From aa9fb5c891d10e84d85ad11b5e5699e7cf6b5e30 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Wed, 26 Mar 2025 20:28:06 -0400 Subject: [PATCH 01/75] Bumps version to 4.7.2 Bumps SQLCipher versions to 4.7.2. Note that 4.7.1 was intentionally omitted due that version number being required to publish a patch to Maven for Community Edition SQLCipher for Android --- CHANGELOG.md | 7 ++++++- SQLCipher.podspec.json | 4 ++-- src/sqlcipher.c | 2 +- test/sqlcipher-pragmas.test | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a126716f3..3873f5d8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ Notable changes to this project are documented in this file. ## [unreleased] - (? 2025 - [unreleased changes]) +## [4.7.2] - (? 2025 - [4.7.2 changes]) +- 4.7.1 intentionally omitted + ## [4.7.0] - (March 2025 - [4.7.0 changes]) - Updates baseline to upstream SQLite 3.49.1, including complete upstream SQLite refactoring of build system to use autosetup - Significantly refactors and optimizes library initialization and cleanup @@ -280,7 +283,9 @@ __BREAKING CHANGE__: `SELECT` statements (now also including schema independent - Change KDF iteration length from 4,000 to 64,000 [unreleased]: https://github.com/sqlcipher/sqlcipher/tree/prerelease -[unreleased changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.7.0...prerelease +[unreleased changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.7.2...prerelease +[4.7.2]: https://github.com/sqlcipher/sqlcipher/tree/v4.7.2 +[4.7.2 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.7.0...v4.7.2 [4.7.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.7.0 [4.7.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.6.1...v4.7.0 [4.6.1]: https://github.com/sqlcipher/sqlcipher/tree/v4.6.1 diff --git a/SQLCipher.podspec.json b/SQLCipher.podspec.json index 62cf97436..88127df25 100644 --- a/SQLCipher.podspec.json +++ b/SQLCipher.podspec.json @@ -18,10 +18,10 @@ "requires_arc": false, "source": { "git": "https://github.com/sqlcipher/sqlcipher.git", - "tag": "v4.7.0" + "tag": "v4.7.2" }, "summary": "Full Database Encryption for SQLite.", - "version": "4.7.0", + "version": "4.7.2", "subspecs": [ { "compiler_flags": [ diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 4024ef6ed..787ade223 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -99,7 +99,7 @@ void sqlite3pager_reset(Pager *pPager); #define CIPHER_STR(s) #s #ifndef CIPHER_VERSION_NUMBER -#define CIPHER_VERSION_NUMBER 4.7.0 +#define CIPHER_VERSION_NUMBER 4.7.2 #endif #ifndef CIPHER_VERSION_BUILD diff --git a/test/sqlcipher-pragmas.test b/test/sqlcipher-pragmas.test index e17921724..11d797e5e 100644 --- a/test/sqlcipher-pragmas.test +++ b/test/sqlcipher-pragmas.test @@ -46,7 +46,7 @@ do_test verify-pragma-cipher-version { execsql { PRAGMA cipher_version; } -} {{4.7.0 community}} +} {{4.7.2 community}} db close file delete -force test.db From 6577e4a4083d525047fc42387ec30607e3566747 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 4 Apr 2025 09:40:15 -0400 Subject: [PATCH 02/75] Bumps version to 4.8.0 --- src/sqlcipher.c | 2 +- test/sqlcipher-pragmas.test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 787ade223..79d546337 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -99,7 +99,7 @@ void sqlite3pager_reset(Pager *pPager); #define CIPHER_STR(s) #s #ifndef CIPHER_VERSION_NUMBER -#define CIPHER_VERSION_NUMBER 4.7.2 +#define CIPHER_VERSION_NUMBER 4.8.0 #endif #ifndef CIPHER_VERSION_BUILD diff --git a/test/sqlcipher-pragmas.test b/test/sqlcipher-pragmas.test index 11d797e5e..6f1f8bbca 100644 --- a/test/sqlcipher-pragmas.test +++ b/test/sqlcipher-pragmas.test @@ -46,7 +46,7 @@ do_test verify-pragma-cipher-version { execsql { PRAGMA cipher_version; } -} {{4.7.2 community}} +} {{4.8.0 community}} db close file delete -force test.db From 756aa78bcda63567441c29420bf7910af6875521 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 4 Apr 2025 10:32:56 -0400 Subject: [PATCH 03/75] Fixes regression in cipher_migrate when migrating a current-version database In previous versions of SQLCipher calling `PRAGMA cipher_migrate` on a database that does not require migration would report success and leave the database in a useable state. A regression exists in 4.7.0 where the migration would report an error and leave the database unusable. This commit restores the former behavior and adds a test for that. --- CHANGELOG.md | 4 ++-- src/sqlcipher.c | 2 +- test/sqlcipher-compatibility.test | 20 +++++++++++++++++--- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3873f5d8b..ef123e06b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,8 @@ Notable changes to this project are documented in this file. ## [unreleased] - (? 2025 - [unreleased changes]) -## [4.7.2] - (? 2025 - [4.7.2 changes]) -- 4.7.1 intentionally omitted +## [4.8.0] - (? 2025 - [4.8.0 changes]) +- Fixes regression in `PRAGMA cipher_migrate` where an error would be thrown when migrating a current-version database ## [4.7.0] - (March 2025 - [4.7.0 changes]) - Updates baseline to upstream SQLite 3.49.1, including complete upstream SQLite refactoring of build system to use autosetup diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 79d546337..ca717a63e 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -2217,7 +2217,6 @@ static int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: reset pager"); handle_error: -cleanup: rc_cleanup = sqlite3_exec(db, "DETACH DATABASE migrate;", NULL, NULL, NULL); if(rc_cleanup != SQLITE_OK) { sqlcipher_log(SQLCIPHER_LOG_WARN, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: DETACH DATABASE migrate failed: %d", rc_cleanup); @@ -2254,6 +2253,7 @@ static int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { ctx->error = rc; /* set flag for deferred error */ } +cleanup: if(pass) sqlcipher_free(pass, pass_sz); if(keyspec) sqlcipher_free(keyspec, keyspec_sz); if(attach_command) sqlcipher_free(attach_command, sqlite3Strlen30(attach_command)); diff --git a/test/sqlcipher-compatibility.test b/test/sqlcipher-compatibility.test index cf8f2b6fc..69630fbf7 100644 --- a/test/sqlcipher-compatibility.test +++ b/test/sqlcipher-compatibility.test @@ -1165,9 +1165,8 @@ do_test migrate-failure { db close file delete -force test.db -# leave database is a readable state after a -# a failed migration -do_test migrate-failure-readable { +# if a migration failes the database should be in a permanent error state +do_test migrate-failure-not-readable { file copy -force $sampleDir/sqlcipher-3.0-testkey.db test.db sqlite_orig db test.db @@ -1192,6 +1191,21 @@ do_test migrate-failure-readable { db close file delete -force test.db +# if cipher_migrate is called on a current-version databse +# is should do nothing and just report OK +do_test migrate-current-format-noop { + file copy -force $sampleDir/sqlcipher-4.0-testkey.db test.db + sqlite_orig db test.db + + execsql { + PRAGMA key = 'testkey'; + PRAGMA cipher_migrate; + SELECT count(*) FROM sqlite_schema; + } +} {ok 0 1} +db close +file delete -force test.db + do_test key-database-by-name { sqlite_orig db test.db execsql { From 91815badc946d3a85112bd1523d5552ec5c6b232 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Tue, 8 Apr 2025 14:30:57 -0400 Subject: [PATCH 04/75] Adds selective locking for shared cache connections Fixes an issue where multiple concurrent connection operations (e.g. open/close) executed in separate threads with shared cache enabled could result in a codec context being freed prior to use. Critical sections of the library now have additional locking on a shared cache mutex when shared cache is in use. Note that despite these improvements use of shared cache is strongly discouraged. --- CHANGELOG.md | 1 + src/pager.c | 12 ++--- src/sqlcipher.c | 120 ++++++++++++++++++++++++++++++++++++++++-------- src/sqlcipher.h | 3 +- 4 files changed, 110 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef123e06b..6c7d2bff3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ Notable changes to this project are documented in this file. ## [4.8.0] - (? 2025 - [4.8.0 changes]) - Fixes regression in `PRAGMA cipher_migrate` where an error would be thrown when migrating a current-version database +- Adds selective locking in critical sections of the library for shared cache connections (Note: use of shared cache is still strongly discouraged) ## [4.7.0] - (March 2025 - [4.7.0 changes]) - Updates baseline to upstream SQLite 3.49.1, including complete upstream SQLite refactoring of build system to use autosetup diff --git a/src/pager.c b/src/pager.c index 3d255fb12..13f242938 100644 --- a/src/pager.c +++ b/src/pager.c @@ -7224,7 +7224,10 @@ const char *sqlite3PagerJournalname(Pager *pPager){ /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC /* -** Set or retrieve the codec for this pager +** Set (or overwrite) the codec for this pager. If there is +** already a codec context (pCodec) attached, it WILL NOT be freed. +** The caller is responsible for freeing the old codec context prior +** setting a new one. */ void sqlcipherPagerSetCodec( Pager *pPager, @@ -7233,11 +7236,7 @@ void sqlcipherPagerSetCodec( void (*xCodecFree)(void*), void *pCodec ){ - if( pPager->xCodecFree ){ - pPager->xCodecFree(pPager->pCodec); - }else{ - pager_reset(pPager); - } + pager_reset(pPager); pPager->xCodec = pPager->memDb ? 0 : xCodec; pPager->xCodecSizeChng = xCodecSizeChng; pPager->xCodecFree = xCodecFree; @@ -7245,6 +7244,7 @@ void sqlcipherPagerSetCodec( setGetterMethod(pPager); pagerReportSize(pPager); } +/* Retrieve the codec for this pager */ void *sqlcipherPagerGetCodec(Pager *pPager){ return pPager->pCodec; } diff --git a/src/sqlcipher.c b/src/sqlcipher.c index ca717a63e..98297b8fa 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -455,7 +455,7 @@ int sqlcipher_extra_init(const char* arg) { for(i = 0; i < SQLCIPHER_MUTEX_COUNT; i++) { if(sqlcipher_static_mutex[i] == NULL) { if((sqlcipher_static_mutex[i] = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST)) == NULL) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_MEMORY, "%s: failed to allocate static mutex %i", __func__, i); + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_MEMORY, "%s: failed to allocate static mutex %d", __func__, i); rc = SQLITE_NOMEM; goto error; } @@ -3212,20 +3212,30 @@ static void* sqlite3Codec(void *iCtx, void *data, Pgno pgno, int mode) { int offset = 0, rc = 0; unsigned char *pData = (unsigned char *) data; int cctx = CIPHER_READ_CTX; + void *out = NULL; + sqlite3_mutex *mutex = ctx->pBt->sharable ? sqlcipher_mutex(SQLCIPHER_MUTEX_SHAREDCACHE) : NULL; + + /* in shared cache mode, this needs to be mutexed to prevent a separate database handle from + * nuking the context on the shared Btree */ + if(mutex) { + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entering mutex %p", __func__, mutex); + sqlite3_mutex_enter(mutex); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entered mutex %p", __func__, mutex); + } sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlite3Codec: pgno=%d, mode=%d, ctx->page_sz=%d", pgno, mode, ctx->page_sz); if(ctx->error != SQLITE_OK) { sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: identified deferred error condition: %d", __func__, rc); sqlcipher_codec_ctx_set_error(ctx, ctx->error); - return NULL; + goto cleanup; } /* call to derive keys if not present yet */ if((rc = sqlcipher_codec_key_derive(ctx)) != SQLITE_OK) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlite3Codec: error occurred during key derivation: %d", rc); - sqlcipher_codec_ctx_set_error(ctx, rc); - return NULL; + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlite3Codec: error occurred during key derivation: %d", rc); + sqlcipher_codec_ctx_set_error(ctx, rc); + goto cleanup; } /* if the plaintext_header_size is negative that means an invalid size was set via @@ -3234,7 +3244,7 @@ static void* sqlite3Codec(void *iCtx, void *data, Pgno pgno, int mode) { if(ctx->plaintext_header_sz < 0) { sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlite3Codec: error invalid ctx->plaintext_header_sz: %d", ctx->plaintext_header_sz); sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); - return NULL; + goto cleanup; } if(pgno == 1) /* adjust starting pointers in data page for header offset on first page*/ @@ -3270,7 +3280,8 @@ static void* sqlite3Codec(void *iCtx, void *data, Pgno pgno, int mode) { SQLCIPHER_FLAG_SET(ctx->flags, CIPHER_FLAG_KEY_USED); } memcpy(pData, ctx->buffer, ctx->page_sz); /* copy buffer data back to pData and return */ - return pData; + out = pData; + goto cleanup; break; case CODEC_WRITE_OP: /* encrypt database page, operate on write context and fall through to case 7, so the write context is used*/ @@ -3283,7 +3294,7 @@ static void* sqlite3Codec(void *iCtx, void *data, Pgno pgno, int mode) { if((rc = sqlcipher_codec_ctx_get_kdf_salt(ctx, &kdf_salt)) != SQLITE_OK) { sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlite3Codec: error retrieving salt: %d", rc); sqlcipher_codec_ctx_set_error(ctx, rc); - return NULL; + goto cleanup; } memcpy(ctx->buffer, ctx->plaintext_header_sz ? pData : kdf_salt, offset); } @@ -3300,24 +3311,53 @@ static void* sqlite3Codec(void *iCtx, void *data, Pgno pgno, int mode) { sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlite3Codec: error encrypting page %d data: %d", pgno, rc); sqlcipher_memset((unsigned char*)ctx->buffer+offset, 0, ctx->page_sz-offset); sqlcipher_codec_ctx_set_error(ctx, rc); - return NULL; + goto cleanup; } SQLCIPHER_FLAG_SET(ctx->flags, CIPHER_FLAG_KEY_USED); - return ctx->buffer; /* return persistent buffer data, pData remains intact */ + out = ctx->buffer; /* return persistent buffer data, pData remains intact */ + goto cleanup; break; default: sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlite3Codec: error unsupported codec mode %d", mode); sqlcipher_codec_ctx_set_error(ctx, SQLITE_ERROR); /* unsupported mode, set error */ - return pData; + out = pData; + goto cleanup; break; } + +cleanup: + if(mutex) { + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: leaving mutex %p", __func__, mutex); + sqlite3_mutex_leave(mutex); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: left mutex %p", __func__, mutex); + } + return out; } +/* This callback will be invoked when a database connection is closed. It is basically a light wrapper + * ariund sqlciher_codec_ctx_free that locks the shared cache mutex if necessary */ static void sqlite3FreeCodecArg(void *pCodecArg) { codec_ctx *ctx = (codec_ctx *) pCodecArg; + sqlite3_mutex *mutex = ctx->pBt->sharable ? sqlcipher_mutex(SQLCIPHER_MUTEX_SHAREDCACHE) : NULL; + if(pCodecArg == NULL) return; + + /* in shared cache mode, this needs to be mutexed to prevent a codec context from being deallocated when + * it is in use by the codec due to cross-database handle access to the shared Btree */ + if(mutex) { + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entering mutex %p", __func__, mutex); + sqlite3_mutex_enter(mutex); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entered mutex %p", __func__, mutex); + } + sqlcipher_codec_ctx_free(&ctx); /* wipe and free allocated memory for the context */ + + if(mutex) { + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: leaving mutex %p", __func__, mutex); + sqlite3_mutex_leave(mutex); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: left mutex %p", __func__, mutex); + } } int sqlcipherCodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) { @@ -3326,6 +3366,7 @@ int sqlcipherCodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) { codec_ctx *ctx = NULL; Pager *pPager = NULL; int rc = SQLITE_OK; + sqlite3_mutex *extra_mutex = NULL; sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: db=%p, nDb=%d", __func__, db, nDb); @@ -3345,24 +3386,58 @@ int sqlcipherCodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) { return SQLITE_MISUSE; } + /* After this point, early returns for API misuse are complete, lock on a mutex and ensure it is cleaned + * up later. If shared cache is enabled then enter a specially defined "global" recursive mutex specifically + * for isolating shared cache connections, otherwise use the built-in databse mutex */ + extra_mutex = pDb->pBt->sharable ? sqlcipher_mutex(SQLCIPHER_MUTEX_SHAREDCACHE) : NULL; + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entering database mutex %p", __func__, db->mutex); sqlite3_mutex_enter(db->mutex); sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entered database mutex %p", __func__, db->mutex); - pPager = pDb->pBt->pBt->pPager; - ctx = (codec_ctx*) sqlcipherPagerGetCodec(pDb->pBt->pBt->pPager); + if(extra_mutex) { + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entering mutex %p", __func__, extra_mutex); + sqlite3_mutex_enter(extra_mutex); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entered mutex %p", __func__, extra_mutex); + } - if(ctx != NULL && SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_KEY_USED)) { - /* there is already a codec attached to this database, so we should not proceed */ - sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_CORE, "%s: disregarding attempt to set key on an previously keyed database connection handle", __func__); - goto cleanup; + pPager = sqlite3BtreePager(pDb->pBt); + ctx = (codec_ctx*) sqlcipherPagerGetCodec(pPager); + + if(ctx != NULL) { + /* There is already a codec attached to this database */ + if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_KEY_USED)) { + /* The key was derived and used successfully, so return early */ + sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_CORE, "%s: disregarding attempt to set key on an previously keyed database connection handle", __func__); + goto cleanup; +#ifndef SQLITE_DEBUG + } else if (pDb->pBt->sharable) { + /* This Btree is participating in shared cache. It would be usafe to reset and reattach a new codec, so return early. + * + * When compiled with SQLITE_DEBUG, all database connections have shared cached enabled. This behavior of disallowing reset + * of the codec on a shared cache connection will break several tests that depend on the the ability to reset the codec, + * like migration tests, repeat-keying tests, etc. Asa result we will disable shared cache handling when compiled with + * SQLIE_DEBUG enabled.*/ + sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_CORE, "%s: disregarding attempt to set key on an shared cache handle", __func__); + goto cleanup; +#endif + } else { + /* To preseve legacy functionality where an incorrect key could be replaced by a correct key without closing the database, + * if the key has not been used, and shared cache is not enabled, reset the codec on this pager entirely. + * This will call sqlcipher_codec_ctx_free directly instead of through sqlite3FreeCodecArg because this function already + * holds the shared cache mutex if it is necessary, and that avoids requiring a more expensive recursive mutex */ + sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_CORE, "%s: resetting existing codec on pager", __func__); + sqlcipher_codec_ctx_free(&ctx); + sqlcipherPagerSetCodec(pPager, NULL, NULL, NULL, NULL); + ctx = NULL; + } } /* check if the sqlite3_file is open, and if not force handle to NULL */ if((fd = sqlite3PagerFile(pPager))->pMethods == 0) fd = NULL; /* point the internal codec argument against the contet to be prepared */ - rc = sqlcipher_codec_ctx_init(&ctx, pDb, pDb->pBt->pBt->pPager, zKey, nKey); + rc = sqlcipher_codec_ctx_init(&ctx, pDb, pPager, zKey, nKey); if(rc != SQLITE_OK) { /* initialization failed, do not attach potentially corrupted context */ @@ -3375,7 +3450,7 @@ int sqlcipherCodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) { } sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: calling sqlcipherPagerSetCodec()", __func__); - sqlcipherPagerSetCodec(sqlite3BtreePager(pDb->pBt), sqlite3Codec, NULL, sqlite3FreeCodecArg, (void *) ctx); + sqlcipherPagerSetCodec(pPager, sqlite3Codec, NULL, sqlite3FreeCodecArg, (void *) ctx); codec_set_btree_to_codec_pagesize(db, pDb, ctx); @@ -3394,6 +3469,13 @@ int sqlcipherCodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) { } cleanup: + + if(extra_mutex) { + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: leaving mutex %p", __func__, extra_mutex); + sqlite3_mutex_leave(extra_mutex); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: left mutex %p", __func__, extra_mutex); + } + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: leaving database mutex %p", __func__, db->mutex); sqlite3_mutex_leave(db->mutex); sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: left database mutex %p", __func__, db->mutex); diff --git a/src/sqlcipher.h b/src/sqlcipher.h index a8a802b4d..40416cb53 100644 --- a/src/sqlcipher.h +++ b/src/sqlcipher.h @@ -114,7 +114,8 @@ sqlcipher_provider* sqlcipher_get_provider(void); #define SQLCIPHER_MUTEX_RESERVED2 4 #define SQLCIPHER_MUTEX_RESERVED3 5 #define SQLCIPHER_MUTEX_MEM 6 -#define SQLCIPHER_MUTEX_COUNT 7 +#define SQLCIPHER_MUTEX_SHAREDCACHE 7 +#define SQLCIPHER_MUTEX_COUNT 8 sqlite3_mutex* sqlcipher_mutex(int); From 542cf9ff057ad28dd4681c6221f0edce2aa30eda Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Thu, 10 Apr 2025 19:54:32 -0400 Subject: [PATCH 05/75] Corrects log statement --- src/sqlcipher.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 98297b8fa..4c9e306c9 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -3457,7 +3457,7 @@ int sqlcipherCodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) { /* force secure delete. This has the benefit of wiping internal data when deleted and also ensures that all pages are written to disk (i.e. not skipped by sqlite3PagerDontWrite optimizations) */ - sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: calling sqlite3BtreeSecureDelete(), __func__"); + sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: calling sqlite3BtreeSecureDelete()", __func__); sqlite3BtreeSecureDelete(pDb->pBt, 1); /* if fd is null, then this is an in-memory database and From e8e2427f6b81e8399b3878088d5fa1801285df96 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Thu, 10 Apr 2025 20:08:51 -0400 Subject: [PATCH 06/75] Standardizes usage of sqlite3BtreePager --- src/sqlcipher.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 4c9e306c9..9476ff366 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -1416,13 +1416,13 @@ static int sqlcipher_codec_ctx_set_kdf_algorithm(codec_ctx *ctx, int algorithm) static void sqlcipher_codec_ctx_set_error(codec_ctx *ctx, int error) { sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_set_error %d", error); - sqlite3pager_error(ctx->pBt->pBt->pPager, error); + sqlite3pager_error(sqlite3BtreePager(ctx->pBt), error); ctx->pBt->pBt->db->errCode = error; ctx->error = error; } static int sqlcipher_codec_ctx_init_kdf_salt(codec_ctx *ctx) { - sqlite3_file *fd = sqlite3PagerFile(ctx->pBt->pBt->pPager); + sqlite3_file *fd = sqlite3PagerFile(sqlite3BtreePager(ctx->pBt)); if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_HAS_KDF_SALT)) { return SQLITE_OK; /* don't reload salt when not needed */ @@ -1971,7 +1971,7 @@ static int sqlcipher_codec_ctx_integrity_check(codec_ctx *ctx, Parse *pParse, ch int rc = 0; char *result; unsigned char *hmac_out = NULL; - sqlite3_file *fd = sqlite3PagerFile(ctx->pBt->pBt->pPager); + sqlite3_file *fd = sqlite3PagerFile(sqlite3BtreePager(ctx->pBt)); i64 file_sz; Vdbe *v = sqlite3GetVdbe(pParse); @@ -2005,7 +2005,7 @@ static int sqlcipher_codec_ctx_integrity_check(codec_ctx *ctx, Parse *pParse, ch int read_sz = ctx->page_sz; /* skip integrity check on PAGER_SJ_PGNO since it will have no valid content */ - if(sqlite3pager_is_sj_pgno(ctx->pBt->pBt->pPager, page)) continue; + if(sqlite3pager_is_sj_pgno(sqlite3BtreePager(ctx->pBt), page)) continue; if(page==1) { int page1_offset = ctx->plaintext_header_sz ? ctx->plaintext_header_sz : FILE_HEADER_SZ; @@ -2170,8 +2170,8 @@ static int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { SQLCIPHER_FLAG_UNSET(ctx->flags, CIPHER_FLAG_KEY_USED); sqlcipherCodecAttach(db, 0, keyspec, keyspec_sz); - srcfile = sqlite3PagerFile(pSrc->pBt->pPager); - destfile = sqlite3PagerFile(pDest->pBt->pPager); + srcfile = sqlite3PagerFile(sqlite3BtreePager(pSrc)); + destfile = sqlite3PagerFile(sqlite3BtreePager(pDest)); sqlite3OsClose(srcfile); sqlite3OsClose(destfile); @@ -2213,7 +2213,7 @@ static int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { goto handle_error; } - sqlite3pager_reset(pDest->pBt->pPager); + sqlite3pager_reset(sqlite3BtreePager(pDest)); sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: reset pager"); handle_error: @@ -2249,7 +2249,7 @@ static int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { if(rc != SQLITE_OK) { sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlcipher_codec_ctx_migrate: an error occurred attempting to migrate the database - last error %d", rc); - sqlite3pager_reset(ctx->pBt->pBt->pPager); + sqlite3pager_reset(sqlite3BtreePager(ctx->pBt)); ctx->error = rc; /* set flag for deferred error */ } @@ -2551,7 +2551,7 @@ static int codec_set_pass_key(sqlite3* db, int nDb, const void *zKey, int nKey, struct Db *pDb = &db->aDb[nDb]; sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "codec_set_pass_key: db=%p nDb=%d for_ctx=%d", db, nDb, for_ctx); if(pDb->pBt) { - codec_ctx *ctx = (codec_ctx*) sqlcipherPagerGetCodec(pDb->pBt->pBt->pPager); + codec_ctx *ctx = (codec_ctx*) sqlcipherPagerGetCodec(sqlite3BtreePager(pDb->pBt)); if(ctx) { return sqlcipher_codec_ctx_set_pass(ctx, zKey, nKey, for_ctx); @@ -2570,7 +2570,7 @@ int sqlcipher_codec_pragma(sqlite3* db, int iDb, Parse *pParse, const char *zLef int rc; if(pDb->pBt) { - ctx = (codec_ctx*) sqlcipherPagerGetCodec(pDb->pBt->pBt->pPager); + ctx = (codec_ctx*) sqlcipherPagerGetCodec(sqlite3BtreePager(pDb->pBt)); } if(sqlite3_stricmp(zLeft, "key") !=0 && sqlite3_stricmp(zLeft, "rekey") != 0) { @@ -3544,9 +3544,9 @@ int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey) { int rc, page_count; Pgno pgno; PgHdr *page; - Pager *pPager = pDb->pBt->pBt->pPager; + Pager *pPager = sqlite3BtreePager(pDb->pBt); - ctx = (codec_ctx*) sqlcipherPagerGetCodec(pDb->pBt->pBt->pPager); + ctx = (codec_ctx*) sqlcipherPagerGetCodec(pPager); if(ctx == NULL) { /* there was no codec attached to this database, so this should do nothing! */ @@ -3617,7 +3617,7 @@ void sqlcipherCodecGetKey(sqlite3* db, int nDb, void **zKey, int *nKey) { struct Db *pDb = &db->aDb[nDb]; sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlcipherCodecGetKey:db=%p, nDb=%d", db, nDb); if( pDb->pBt ) { - codec_ctx *ctx = (codec_ctx*) sqlcipherPagerGetCodec(pDb->pBt->pBt->pPager); + codec_ctx *ctx = (codec_ctx*) sqlcipherPagerGetCodec(sqlite3BtreePager(pDb->pBt)); if(ctx) { /* if the key has not been derived yet, or the key is stored (vi PRAGMA cipher_store_pass) From 69a48ce5b03b0f76cd87bb9d1345a11d7269293f Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 11 Apr 2025 13:15:46 -0400 Subject: [PATCH 07/75] Standardizes on 48K initial private heap allocation --- src/sqlcipher.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 9476ff366..7b7054601 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -288,17 +288,15 @@ static u8* sqlcipher_shield_mask = NULL; /* Establish the default size of the private heap. This can be overriden * at compile time by setting -DSQLCIPHER_PRIVATE_HEAP_SIZE_DEFAULT=X */ #ifndef SQLCIPHER_PRIVATE_HEAP_SIZE_DEFAULT -#ifdef __ANDROID__ -/* On android, the maximim amount of memory that can be memlocked in 64k. - * The default heap size is chosen as 48K, which is either 4 (with 4k page size) - * or 1 (with 16k age size) less than the max. We choose to allocate slightly - * less than the max just in case the app has locked some other page(s) */ +/* On android, the maximim amount of memory that can be memlocked in 64k. This also + * seems to be a popular ulimit on linux distributions, containsers, etc. Therefore + * the default heap size is chosen as 48K, which is either 4 (with 4k page size) + * or 1 (with 16k page size) page less than the max. We choose to allocate slightly + * less than the max just in case the app has locked some other page(s). This + * initial allocation should be enough to support at least 10 concurrent + * sqlcipher-enabled database connections at the same time without requiring any + * overflow allocations */ #define SQLCIPHER_PRIVATE_HEAP_SIZE_DEFAULT 49152 -#else -/* On non-android platforms we'll attempt to allocate and lock a larger private heap - * of around 128k instead. */ -#define SQLCIPHER_PRIVATE_HEAP_SIZE_DEFAULT 131072 -#endif #endif /* if default allocation fails, we'll reduce the size by this amount * and try again. This is also the minimium of the private heap. The minimum From 3eeee394f2cd2e4e2e4cf24548a3369d31738725 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 11 Apr 2025 13:22:46 -0400 Subject: [PATCH 08/75] Removes invasive changes to working set size on windows Commit 61bf74fdf66433a73b86abb65b0c460ee9c35cfe introduced a change to update the minimum working set size on windows to increase lockable pages using VirtualLock. The forced increase in minimum set size should no longer be required with the new private heap, which allocates and locks memory up front to prevent fragmentation. --- src/sqlcipher.c | 53 ++----------------------------------------------- 1 file changed, 2 insertions(+), 51 deletions(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 7b7054601..9e3a4fef2 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -851,58 +851,9 @@ static sqlite3_mem_methods sqlcipher_mem_methods = { void sqlcipher_init_memmethods() { if(sqlcipher_mem_initialized) return; if(sqlite3_config(SQLITE_CONFIG_GETMALLOC, &default_mem_methods) != SQLITE_OK || - sqlite3_config(SQLITE_CONFIG_MALLOC, &sqlcipher_mem_methods) != SQLITE_OK) { - sqlcipher_mem_security_on = sqlcipher_mem_executed = sqlcipher_mem_initialized = 0; + sqlite3_config(SQLITE_CONFIG_MALLOC, &sqlcipher_mem_methods) != SQLITE_OK) { + sqlcipher_mem_security_on = sqlcipher_mem_executed = sqlcipher_mem_initialized = 0; } else { -#if !defined(OMIT_MEMLOCK) && defined(_WIN32) && !(defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_PHONE_APP || WINAPI_FAMILY == WINAPI_FAMILY_PC_APP)) - HANDLE process = NULL; - SYSTEM_INFO info; - SIZE_T dflt_min_size, dflt_max_size, min_size, max_size; - DWORD pid = GetCurrentProcessId(); - - if(pid == 0) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_MEMORY, "sqlcipher_init_memmethods: error calling GetCurrentProcessId: %d", GetLastError()); - goto cleanup; - } - - if((process = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_SET_QUOTA, FALSE, pid)) == NULL) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_MEMORY, "sqlcipher_init_memmethods: error calling OpenProcess for pid=%d: %d", pid, GetLastError()); - goto cleanup; - } - - /* lookup native memory page size for caclulating default working set sizes */ - GetNativeSystemInfo(&info); - - /* https://learn.microsoft.com/en-us/windows/win32/api/memoryapi/nf-memoryapi-setprocessworkingsetsize#parameters - Default Min Set Size is 50 pages. - Default Max Set Size is 345 pages */ - dflt_min_size = info.dwPageSize * 50; - dflt_max_size = info.dwPageSize * 345; - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MEMORY, "sqlcipher_init_memmethods: calculated dflt_min_size=%u dflt_max_size=%u for memory page size %d", dflt_min_size, dflt_max_size, info.dwPageSize); - - /* retrieve current min and max set sizes for comparison */ - if(!GetProcessWorkingSetSize(process, &min_size, &max_size)) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_MEMORY, "sqlcipher_init_memmethods: error calling GetProcessWorkingSetSize %d", GetLastError()); - goto cleanup; - } - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MEMORY, "sqlcipher_init_memmethods: GetProcessWorkingSetSize returned min=%u max=%u", min_size, max_size); - - if(min_size == dflt_min_size && max_size == dflt_max_size) { - /* application has not set any special non-default working set sizes. Caclulate the new min working set size to be - 5 times default to allow greater number of pages to be VirtualLocked, max size will be left unchanged */ - min_size *= 5; - if(!SetProcessWorkingSetSize(process, min_size, max_size)) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_MEMORY, "sqlcipher_init_memmethods: error calling SetProcessWorkingSetSize with min=%u max=%u: %d", min_size, max_size, GetLastError()); - goto cleanup; - } - sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_MEMORY, "sqlcipher_init_memmethods: called SetProcessWorkingSetSize for min=%u max=%u", min_size, max_size); - } else { - sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_MEMORY, "sqlcipher_init_memmethods: application has custom working set sizes min=%u max=%u - skipped alteration of working set sizes", min_size, max_size); - } - -cleanup: - if (process) CloseHandle(process); -#endif sqlcipher_mem_initialized = 1; } } From 3bceefa2746a551d548d66d2c655dbcbdedb1454 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 11 Apr 2025 13:31:45 -0400 Subject: [PATCH 09/75] Removes spurious newlines on log statements --- src/sqlcipher.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 9e3a4fef2..175fe1927 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -368,12 +368,12 @@ sqlite3_mutex* sqlcipher_mutex(int mutex) { } static void sqlcipher_atexit(void) { - sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: calling sqlcipher_extra_shutdown()\n", __func__); + sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: calling sqlcipher_extra_shutdown()", __func__); sqlcipher_extra_shutdown(); } static void sqlcipher_fini(void) { - sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: calling sqlcipher_extra_shutdown()\n", __func__); + sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: calling sqlcipher_extra_shutdown()", __func__); sqlcipher_extra_shutdown(); } @@ -382,7 +382,7 @@ static void sqlcipher_fini(void) { BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) { switch (fdwReason) { case DLL_PROCESS_DETACH: - sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: calling sqlcipher_extra_shutdown()\n", __func__); + sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: calling sqlcipher_extra_shutdown()", __func__); sqlcipher_extra_shutdown(); break; default: @@ -630,7 +630,7 @@ void sqlcipher_extra_shutdown(void) { if(used > 0) { /* don't free the heap so that sqlite treats this as unfreed memory */ sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_MEMORY, - "%s: SQLCipher private heap unfreed memory %u bytes in %d allocations\n", __func__, used, i); + "%s: SQLCipher private heap unfreed memory %u bytes in %d allocations", __func__, used, i); } else { sqlcipher_internal_free(private_heap, private_heap_sz); private_heap = NULL; @@ -3210,7 +3210,7 @@ static void* sqlite3Codec(void *iCtx, void *data, Pgno pgno, int mode) { #ifdef SQLCIPHER_TEST if((cipher_test_flags & TEST_FAIL_DECRYPT) > 0 && sqlcipher_get_test_fail()) { rc = SQLITE_ERROR; - sqlcipher_log(SQLCIPHER_LOG_WARN, SQLCIPHER_LOG_CORE, "sqlite3Codec: simulating decryption failure for pgno=%d, mode=%d, ctx->page_sz=%d\n", pgno, mode, ctx->page_sz); + sqlcipher_log(SQLCIPHER_LOG_WARN, SQLCIPHER_LOG_CORE, "sqlite3Codec: simulating decryption failure for pgno=%d, mode=%d, ctx->page_sz=%d", pgno, mode, ctx->page_sz); } #endif if(rc != SQLITE_OK) { @@ -3251,7 +3251,7 @@ static void* sqlite3Codec(void *iCtx, void *data, Pgno pgno, int mode) { #ifdef SQLCIPHER_TEST if((cipher_test_flags & TEST_FAIL_ENCRYPT) > 0 && sqlcipher_get_test_fail()) { rc = SQLITE_ERROR; - sqlcipher_log(SQLCIPHER_LOG_WARN, SQLCIPHER_LOG_CORE, "sqlite3Codec: simulating encryption failure for pgno=%d, mode=%d, ctx->page_sz=%d\n", pgno, mode, ctx->page_sz); + sqlcipher_log(SQLCIPHER_LOG_WARN, SQLCIPHER_LOG_CORE, "sqlite3Codec: simulating encryption failure for pgno=%d, mode=%d, ctx->page_sz=%d", pgno, mode, ctx->page_sz); } #endif if(rc != SQLITE_OK) { From e1af1318d8e9732c89df1a040f1eaae31f134868 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 11 Apr 2025 13:43:32 -0400 Subject: [PATCH 10/75] Tracks basic private heap usage metrics Adds simple tracking of some stats related to the private heap: 1. current allocated 2. most allocated at one time (high water mark) 3. total allocated on private heap over time 4. total mumber of allocations on private heap over time 5. total allocated by overflow mechanism 6. total number of overflow allocations Basic stats are emitted at INFO log level in the MEMORY source when SQLCipher is shut down. --- src/sqlcipher.c | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 175fe1927..c3abb4a50 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -305,6 +305,12 @@ static u8* sqlcipher_shield_mask = NULL; static volatile size_t private_heap_sz = SQLCIPHER_PRIVATE_HEAP_SIZE_DEFAULT; static u8* private_heap = NULL; +static volatile size_t private_heap_used = 0; /* bytes currently used on private heap */ +static volatile size_t private_heap_hwm = 0; /* larged number of bytes used on the private heap at one time */ +static volatile size_t private_heap_alloc = 0; /* total bytes allocated on private heap over time */ +static volatile u32 private_heap_allocs = 0; /* total number of allocations on private heap over time */ +static volatile size_t private_heap_overflow = 0; /* total bytes overflowing private heap over time */ +static volatile u32 private_heap_overflows = 0; /* number of overlow allocations over time */ /* to prevent excessive fragmentation blocks will * only be split if there are at least this many @@ -639,6 +645,10 @@ void sqlcipher_extra_shutdown(void) { sqlcipher_internal_free(private_heap, private_heap_sz); private_heap = NULL; #endif + sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_MEMORY, + "%s: SQLCipher private heap stats: size=%u, hwm=%u, alloc=%u, allocs=%u, overflow=%u, overflows=%u", __func__, + private_heap_sz, private_heap_hwm, private_heap_alloc, private_heap_allocs, private_heap_overflow, private_heap_overflows + ); } /* free all of sqlcipher's static mutexes */ @@ -675,7 +685,7 @@ void* sqlcipher_memset(void *v, unsigned char value, sqlite_uint64 len) { if (v == NULL) return v; - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MEMORY, "sqlcipher_memset: setting %p[0-%llu]=%d)", a, len, value); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MEMORY, "sqlcipher_memset: setting %p[0-%u]=%d)", a, len, value); for(i = 0; i < len; i++) { a[i] = value; } @@ -936,10 +946,19 @@ void *sqlcipher_malloc(sqlite3_uint64 size) { /* If we were unable to locate a free block large enough to service the request, the fallback behavior will simply attempt to allocate additional memory using malloc. */ if(alloc == NULL) { + private_heap_overflow += size; + private_heap_overflows++; alloc = sqlcipher_internal_malloc(size); - sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_MEMORY, "%s: unable to allocate %llu bytes on private heap, allocated %p using sqlcipher_internal_malloc fallback", __func__, size, alloc); + sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_MEMORY, "%s: unable to allocate %u bytes on private heap, allocated %p using sqlcipher_internal_malloc fallback", __func__, size, alloc); } else { - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MEMORY, "%s allocated %llu bytes on private heap at %p", __func__, size, alloc); + private_heap_used += size; + if(private_heap_used > private_heap_hwm) { + /* if the current bytes allocated on the private heap are greater than the high water mark, set the HWM to the new amount */ + private_heap_hwm = private_heap_used; + } + private_heap_alloc += size; + private_heap_allocs++; + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MEMORY, "%s allocated %u bytes on private heap at %p", __func__, size, alloc); } return alloc; @@ -948,6 +967,7 @@ void *sqlcipher_malloc(sqlite3_uint64 size) { void sqlcipher_free(void *mem, sqlite3_uint64 sz) { private_block *block = NULL, *prev = NULL; void *alloc = NULL; + u32 block_size = 0; block = (private_block *) private_heap; sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "%s: entering SQLCIPHER_MUTEX_MEM", __func__); @@ -962,6 +982,7 @@ void sqlcipher_free(void *mem, sqlite3_uint64 sz) { on to the next block */ if(mem == alloc) { block->is_used = 0; + block_size = block->size; /* retain the acual size of the block in use for stats adjustment */ xoshiro_randomness(alloc, block->size); /* check whether the previous block is free, if so merge*/ @@ -995,10 +1016,11 @@ void sqlcipher_free(void *mem, sqlite3_uint64 sz) { then it was allocated by the fallback mechanism and should be deallocated with free() */ if(!block) { - sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_MEMORY, "%s: unable to find %p with %llu bytes on private heap, calling sqlcipher_internal_free fallback", __func__, mem, sz); + sqlcipher_log(SQLCIPHER_LOG_INFO, SQLCIPHER_LOG_MEMORY, "%s: unable to find %p with %u bytes on private heap, calling sqlcipher_internal_free fallback", __func__, mem, sz); sqlcipher_internal_free(mem, sz); } else { - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MEMORY, "%s freed %llu bytes on private heap at %p", __func__, sz, mem); + private_heap_used -= block_size; + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MEMORY, "%s freed %u bytes (%u total) on private heap at %p", __func__, sz, block_size, mem); } } From 2a073dd49cda9d5c34b0905c863a3d7deb63875a Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Thu, 17 Apr 2025 16:50:04 -0400 Subject: [PATCH 11/75] Update tags in podspec for 4.8.0 --- SQLCipher.podspec.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SQLCipher.podspec.json b/SQLCipher.podspec.json index 88127df25..4f36ed180 100644 --- a/SQLCipher.podspec.json +++ b/SQLCipher.podspec.json @@ -18,10 +18,10 @@ "requires_arc": false, "source": { "git": "https://github.com/sqlcipher/sqlcipher.git", - "tag": "v4.7.2" + "tag": "v4.8.0" }, "summary": "Full Database Encryption for SQLite.", - "version": "4.7.2", + "version": "4.8.0", "subspecs": [ { "compiler_flags": [ From ab47245fc9b7ee078b679ff4a7e18dc23c42e636 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Thu, 17 Apr 2025 16:50:18 -0400 Subject: [PATCH 12/75] Updates CHANGELOG.md for 4.8.0 release --- CHANGELOG.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c7d2bff3..b6cea5032 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,9 +3,12 @@ Notable changes to this project are documented in this file. ## [unreleased] - (? 2025 - [unreleased changes]) -## [4.8.0] - (? 2025 - [4.8.0 changes]) +## [4.8.0] - (April 2025 - [4.8.0 changes]) - Fixes regression in `PRAGMA cipher_migrate` where an error would be thrown when migrating a current-version database - Adds selective locking in critical sections of the library for shared cache connections (Note: use of shared cache is still strongly discouraged) +- Standardizes initial private heap size to 48KB to ensure mlock under constrained limits +- Removes changes to windows working set sizes +- Improvements to logging of memory stats and other cleanup ## [4.7.0] - (March 2025 - [4.7.0 changes]) - Updates baseline to upstream SQLite 3.49.1, including complete upstream SQLite refactoring of build system to use autosetup @@ -284,9 +287,9 @@ __BREAKING CHANGE__: `SELECT` statements (now also including schema independent - Change KDF iteration length from 4,000 to 64,000 [unreleased]: https://github.com/sqlcipher/sqlcipher/tree/prerelease -[unreleased changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.7.2...prerelease -[4.7.2]: https://github.com/sqlcipher/sqlcipher/tree/v4.7.2 -[4.7.2 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.7.0...v4.7.2 +[unreleased changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.8.0...prerelease +[4.8.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.8.0 +[4.8.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.7.0...v4.8.0 [4.7.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.7.0 [4.7.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.6.1...v4.7.0 [4.6.1]: https://github.com/sqlcipher/sqlcipher/tree/v4.6.1 From f25f26d4c03ba61cf2c75cb9aa9d57a532ec1f05 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Wed, 7 May 2025 11:42:55 -0400 Subject: [PATCH 13/75] Bumps version to 4.9.0 --- CHANGELOG.md | 4 ++++ SQLCipher.podspec.json | 4 ++-- src/sqlcipher.c | 2 +- test/sqlcipher-pragmas.test | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6cea5032..5956fe7e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ Notable changes to this project are documented in this file. ## [unreleased] - (? 2025 - [unreleased changes]) +## [4.9.0] - (? 2025 - [4.9.0 changes]) + ## [4.8.0] - (April 2025 - [4.8.0 changes]) - Fixes regression in `PRAGMA cipher_migrate` where an error would be thrown when migrating a current-version database - Adds selective locking in critical sections of the library for shared cache connections (Note: use of shared cache is still strongly discouraged) @@ -288,6 +290,8 @@ __BREAKING CHANGE__: `SELECT` statements (now also including schema independent [unreleased]: https://github.com/sqlcipher/sqlcipher/tree/prerelease [unreleased changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.8.0...prerelease +[4.9.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.9.0 +[4.9.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.8.0...v4.9.0 [4.8.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.8.0 [4.8.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.7.0...v4.8.0 [4.7.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.7.0 diff --git a/SQLCipher.podspec.json b/SQLCipher.podspec.json index 4f36ed180..05c9daf8c 100644 --- a/SQLCipher.podspec.json +++ b/SQLCipher.podspec.json @@ -18,10 +18,10 @@ "requires_arc": false, "source": { "git": "https://github.com/sqlcipher/sqlcipher.git", - "tag": "v4.8.0" + "tag": "v4.9.0" }, "summary": "Full Database Encryption for SQLite.", - "version": "4.8.0", + "version": "4.9.0", "subspecs": [ { "compiler_flags": [ diff --git a/src/sqlcipher.c b/src/sqlcipher.c index c3abb4a50..2a51f01dc 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -99,7 +99,7 @@ void sqlite3pager_reset(Pager *pPager); #define CIPHER_STR(s) #s #ifndef CIPHER_VERSION_NUMBER -#define CIPHER_VERSION_NUMBER 4.8.0 +#define CIPHER_VERSION_NUMBER 4.9.0 #endif #ifndef CIPHER_VERSION_BUILD diff --git a/test/sqlcipher-pragmas.test b/test/sqlcipher-pragmas.test index 6f1f8bbca..2b611001a 100644 --- a/test/sqlcipher-pragmas.test +++ b/test/sqlcipher-pragmas.test @@ -46,7 +46,7 @@ do_test verify-pragma-cipher-version { execsql { PRAGMA cipher_version; } -} {{4.8.0 community}} +} {{4.9.0 community}} db close file delete -force test.db From ca310fa24c456bd41d7809dec49fb3e5c23305e8 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Wed, 7 May 2025 11:50:19 -0400 Subject: [PATCH 14/75] Snapshot of upstream SQLite 3.49.2 --- Makefile.in | 16 +- VERSION | 2 +- auto.def | 96 +++---- autoconf/Makefile.in | 109 ++++---- autoconf/auto.def | 27 +- autosetup/autosetup | 12 +- autosetup/autosetup-find-tclsh | 2 +- autosetup/jimsh0.c | 20 +- autosetup/proj.tcl | 77 ++++-- autosetup/sqlite-config.tcl | 487 +++++++++++++++++++++++++++------ ext/misc/series.c | 71 +++-- main.mk | 159 ++++++----- manifest | 62 ++--- manifest.uuid | 2 +- src/build.c | 1 + src/expr.c | 8 +- src/pragma.h | 88 +++--- src/select.c | 8 +- src/sqliteInt.h | 1 + src/vdbe.c | 1 + src/where.c | 2 +- test/bloom1.test | 41 +++ test/rowvalue.test | 23 ++ test/shell2.test | 12 +- test/tabfunc01.test | 87 +++++- tool/mkpragmatab.tcl | 7 +- tool/mksqlite3h.tcl | 8 +- 27 files changed, 996 insertions(+), 433 deletions(-) diff --git a/Makefile.in b/Makefile.in index 1ff791c8a..60cc228e9 100644 --- a/Makefile.in +++ b/Makefile.in @@ -79,7 +79,7 @@ libdir = @libdir@ INSTALL = @BIN_INSTALL@ AR = @AR@ -AR.flags = cr # TODO? Add a configure test to determine this? +AR.flags = cr CC = @CC@ B.cc = @CC_FOR_BUILD@ @BUILD_CFLAGS@ T.cc = $(CC) @@ -122,13 +122,21 @@ LDFLAGS.rt = @LDFLAGS_RT@ CFLAGS.icu = @CFLAGS_ICU@ LDFLAGS.libsqlite3.soname = @LDFLAGS_LIBSQLITE3_SONAME@ # soname: see https://sqlite.org/src/forumpost/5a3b44f510df8ded -LDFLAGS.libsqlite3.os-specific = @LDFLAGS_MAC_CVERSION@ @LDFLAGS_OUT_IMPLIB@ +LDFLAGS.libsqlite3.os-specific = \ + @LDFLAGS_MAC_CVERSION@ @LDFLAGS_MAC_INSTALL_NAME@ @LDFLAGS_OUT_IMPLIB@ # os-specific: see # - https://sqlite.org/forum/forumpost/9dfd5b8fd525a5d7 # - https://sqlite.org/forum/forumpost/0c7fc097b2 -ENABLE_SHARED = @ENABLE_SHARED@ -ENABLE_STATIC = @ENABLE_STATIC@ +# - https://sqlite.org/forum/forumpost/5651662b8875ec0a + +libsqlite3.DLL.basename = @SQLITE_DLL_BASENAME@ +# DLL.basename: see https://sqlite.org/forum/forumpost/828fdfe904 +libsqlite3.out.implib = @SQLITE_OUT_IMPLIB@ +# libsqlite3.out.implib => the output filename part of LDFLAGS_OUT_IMPLIB. +ENABLE_LIB_SHARED = @ENABLE_LIB_SHARED@ +ENABLE_LIB_STATIC = @ENABLE_LIB_STATIC@ HAVE_WASI_SDK = @HAVE_WASI_SDK@ +libsqlite3.DLL.install-rules = @SQLITE_DLL_INSTALL_RULES@ T.cc.sqlite = $(T.cc) @TARGET_DEBUG@ diff --git a/VERSION b/VERSION index 04da28a74..46325cfb2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.49.1 +3.49.2 diff --git a/auto.def b/auto.def index 9df87f579..8ed599637 100644 --- a/auto.def +++ b/auto.def @@ -4,70 +4,44 @@ # This is the main autosetup-compatible configure script for the # SQLite project. # -# This script should be kept compatible with JimTCL, a copy of which -# is included in this source tree as ./autosetup/jimsh0.c. The number -# of incompatibilities between canonical TCL and JimTCL is very low -# and alternative formulations of incompatible constructs have, so -# far, been easy to find. +# This script and all of its dependencies must be kept compatible with +# JimTCL, a copy of which is included in this source tree as +# ./autosetup/jimsh0.c. The number of incompatibilities between +# canonical TCL and JimTCL is very low and alternative formulations of +# incompatible constructs have, so far, been easy to find. # # JimTCL: https://jim.tcl.tk # use sqlite-config -sqlite-config-bootstrap canonical -sqlite-setup-default-cflags -proj-if-opt-truthy dev { - # --enable-dev needs to come early so that the downstream tests - # which check for the following flags use their updated state. - proj-opt-set all 1 - proj-opt-set debug 1 - proj-opt-set amalgamation 0 - define CFLAGS [get-env CFLAGS {-O0 -g}] - # -------------^^^^^^^ intentionally using [get-env] instead of - # [proj-get-env] here because [sqlite-setup-default-cflags] uses - # [proj-get-env] and we want this to supercede that. +sqlite-configure canonical { + proj-if-opt-truthy dev { + # --enable-dev needs to come early so that the downstream tests + # which check for the following flags use their updated state. + proj-opt-set all 1 + proj-opt-set debug 1 + proj-opt-set amalgamation 0 + define CFLAGS [get-env CFLAGS {-O0 -g}] + # -------------^^^^^^^ intentionally using [get-env] instead of + # [proj-get-env] here because [sqlite-setup-default-cflags] uses + # [proj-get-env] and we want this to supercede that. + } + sqlite-check-common-bins ;# must come before [sqlite-handle-wasi-sdk] + sqlite-handle-wasi-sdk ;# must run relatively early, as it changes the environment + sqlite-check-common-system-deps + + proj-define-for-opt amalgamation USE_AMALGAMATION "Use amalgamation for builds?" + + proj-define-for-opt gcov USE_GCOV "Use gcov?" + + proj-define-for-opt test-status TSTRNNR_OPTS \ + "test-runner flags:" {--status} {} + + proj-define-for-opt linemacros AMALGAMATION_LINE_MACROS \ + "Use #line macros in the amalgamation:" + + define LINK_TOOLS_DYNAMICALLY [proj-opt-was-provided dynlink-tools] + + sqlite-handle-tcl + sqlite-handle-emsdk } - -sqlite-check-common-bins ;# must come before [sqlite-handle-wasi-sdk] -sqlite-handle-wasi-sdk ;# must run relatively early, as it changes the environment -sqlite-check-common-system-deps - -# -# Enable large file support (if special flags are necessary) -# -define HAVE_LFS 0 -if {[opt-bool largefile]} { - cc-check-lfs -} - -proj-define-for-opt shared ENABLE_SHARED "Build shared library?" - -if {![proj-define-for-opt static ENABLE_STATIC \ - "Build static library?"]} { - proj-warn "Static lib build may be implicitly re-activated by other components, e.g. some test apps." -} - -proj-define-for-opt amalgamation USE_AMALGAMATION "Use amalgamation for builds?" - -proj-define-for-opt gcov USE_GCOV "Use gcov?" - -proj-define-for-opt test-status TSTRNNR_OPTS \ - "test-runner flags:" {--status} {} - -proj-define-for-opt linemacros AMALGAMATION_LINE_MACROS \ - "Use #line macros in the amalgamation:" - -define LINK_TOOLS_DYNAMICALLY [proj-opt-was-provided dynlink-tools] - -proj-check-rpath -sqlite-handle-soname -sqlite-handle-debug -sqlite-handle-tcl -sqlite-handle-threadsafe -sqlite-handle-tempstore -sqlite-handle-line-editing -sqlite-handle-load-extension -sqlite-handle-math -sqlite-handle-icu -sqlite-handle-emsdk -sqlite-config-finalize diff --git a/autoconf/Makefile.in b/autoconf/Makefile.in index d494e9d1e..2f6908240 100644 --- a/autoconf/Makefile.in +++ b/autoconf/Makefile.in @@ -127,10 +127,8 @@ OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@ LDFLAGS.libsqlite3.soname = @LDFLAGS_LIBSQLITE3_SONAME@ # soname: see https://sqlite.org/src/forumpost/5a3b44f510df8ded -LDFLAGS.libsqlite3.os-specific = @LDFLAGS_MAC_CVERSION@ @LDFLAGS_OUT_IMPLIB@ -# os-specific: see -# - https://sqlite.org/forum/forumpost/9dfd5b8fd525a5d7 -# - https://sqlite.org/forum/forumpost/0c7fc097b2 +LDFLAGS.libsqlite3.os-specific = \ + @LDFLAGS_MAC_CVERSION@ @LDFLAGS_MAC_INSTALL_NAME@ @LDFLAGS_OUT_IMPLIB@ LDFLAGS.libsqlite3 = \ $(LDFLAGS.rpath) $(LDFLAGS.pthread) \ @@ -143,57 +141,76 @@ sqlite3.o: $(TOP)/sqlite3.h $(TOP)/sqlite3.c $(CC) -c $(TOP)/sqlite3.c -o $@ $(CFLAGS) $(CFLAGS.libsqlite3) libsqlite3.LIB = libsqlite3$(T.lib) -libsqlite3.SO = libsqlite3$(T.dll) +libsqlite3.DLL.basename = @SQLITE_DLL_BASENAME@ +libsqlite3.out.implib = @SQLITE_OUT_IMPLIB@ +libsqlite3.DLL = $(libsqlite3.DLL.basename)$(T.dll) +libsqlite3.DLL.install-rules = @SQLITE_DLL_INSTALL_RULES@ -$(libsqlite3.SO): sqlite3.o +$(libsqlite3.DLL): sqlite3.o $(CC) -o $@ sqlite3.o $(LDFLAGS.shlib) \ $(LDFLAGS) $(LDFLAGS.libsqlite3) \ $(LDFLAGS.libsqlite3.os-specific) $(LDFLAGS.libsqlite3.soname) -all: $(libsqlite3.SO) +$(libsqlite3.DLL)-1: $(libsqlite3.DLL) +$(libsqlite3.DLL)-0: +all: $(libsqlite3.DLL)-$(ENABLE_LIB_SHARED) $(libsqlite3.LIB): sqlite3.o $(AR) $(AR.flags) $@ sqlite3.o -all: $(libsqlite3.LIB) +$(libsqlite3.LIB)-1: $(libsqlite3.LIB) +$(libsqlite3.LIB)-0: +all: $(libsqlite3.LIB)-$(ENABLE_LIB_STATIC) -install-so-1: $(install-dir.lib) $(libsqlite3.SO) - $(INSTALL) $(libsqlite3.SO) "$(install-dir.lib)" - @if [ -f $(libsqlite3.SO).a ]; then \ - $(INSTALL) $(libsqlite3.SO).a "$(install-dir.lib)"; \ +# +# Maintenance reminder: the install-dll-... rules must be kept in sync +# with the main copies rom /main.mk. +# +install-dll-out-implib: $(install-dir.lib) $(libsqlite3.DLL) + if [ x != "x$(libsqlite3.out.implib)" ] && [ -f "$(libsqlite3.out.implib)" ]; then \ + $(INSTALL) $(libsqlite3.out.implib) "$(install-dir.lib)"; \ fi - @echo "Setting up $(libsqlite3.SO) version symlinks..."; \ - if [ x.dll = x$(T.dll) ]; then \ - echo "No library symlinks needed on this platform"; \ - elif [ x.dylib = x$(T.dll) ]; then \ - cd "$(install-dir.lib)" || exit $$?; \ - rm -f libsqlite3.0$(T.dll) libsqlite3.$(PACKAGE_VERSION)$(T.dll) || exit $$?; \ - dllname=libsqlite3.$(PACKAGE_VERSION)$(T.dll); \ - mv $(libsqlite3.SO) $$dllname || exit $$?; \ - ln -s $$dllname $(libsqlite3.SO) || exit $$?; \ - ln -s $$dllname libsqlite3.0$(T.dll) || exit $$?; \ - ls -la $$dllname $(libsqlite3.SO) libsqlite3.0$(T.dll); \ - else \ - cd "$(install-dir.lib)" || exit $$?; \ - rm -f $(libsqlite3.SO).0 $(libsqlite3.SO).$(PACKAGE_VERSION) || exit $$?; \ - mv $(libsqlite3.SO) $(libsqlite3.SO).$(PACKAGE_VERSION) || exit $$?; \ - ln -s $(libsqlite3.SO).$(PACKAGE_VERSION) $(libsqlite3.SO) || exit $$?; \ - ln -s $(libsqlite3.SO).$(PACKAGE_VERSION) $(libsqlite3.SO).0 || exit $$?; \ - ls -la $(libsqlite3.SO) $(libsqlite3.SO).[a03]*; \ - if [ -e $(libsqlite3.SO).0.8.6 ]; then \ - echo "ACHTUNG: legacy libtool-compatible install found. Re-linking it..."; \ - rm -f libsqlite3.la $(libsqlite3.SO).0.8.6 || exit $$?; \ - ln -s $(libsqlite3.SO).$(PACKAGE_VERSION) $(libsqlite3.SO).0.8.6 || exit $$?; \ - ls -la $(libsqlite3.SO).0.8.6; \ - elif [ x1 = "x$(INSTALL_SO_086_LINK)" ]; then \ - echo "ACHTUNG: installing legacy libtool-style links because INSTALL_SO_086_LINK=1"; \ - rm -f libsqlite3.la $(libsqlite3.SO).0.8.6 || exit $$?; \ - ln -s $(libsqlite3.SO).$(PACKAGE_VERSION) $(libsqlite3.SO).0.8.6 || exit $$?; \ - ls -la $(libsqlite3.SO).0.8.6; \ - fi; \ + +install-dll-unix-generic: install-dll-out-implib + $(INSTALL) $(libsqlite3.DLL) "$(install-dir.lib)" + @echo "Setting up $(libsqlite3.DLL) version symlinks..."; \ + cd "$(install-dir.lib)" || exit $$?; \ + rm -f $(libsqlite3.DLL).0 $(libsqlite3.DLL).$(PACKAGE_VERSION) || exit $$?; \ + mv $(libsqlite3.DLL) $(libsqlite3.DLL).$(PACKAGE_VERSION) || exit $$?; \ + ln -s $(libsqlite3.DLL).$(PACKAGE_VERSION) $(libsqlite3.DLL) || exit $$?; \ + ln -s $(libsqlite3.DLL).$(PACKAGE_VERSION) $(libsqlite3.DLL).0 || exit $$?; \ + ls -la $(libsqlite3.DLL) $(libsqlite3.DLL).[a03]*; \ + if [ -e $(libsqlite3.DLL).0.8.6 ]; then \ + echo "ACHTUNG: legacy libtool-compatible install found. Re-linking it..."; \ + rm -f libsqlite3.la $(libsqlite3.DLL).0.8.6 || exit $$?; \ + ln -s $(libsqlite3.DLL).$(PACKAGE_VERSION) $(libsqlite3.DLL).0.8.6 || exit $$?; \ + ls -la $(libsqlite3.DLL).0.8.6; \ + elif [ x1 = "x$(INSTALL_SO_086_LINK)" ]; then \ + echo "ACHTUNG: installing legacy libtool-style links because INSTALL_SO_086_LINK=1"; \ + rm -f libsqlite3.la $(libsqlite3.DLL).0.8.6 || exit $$?; \ + ln -s $(libsqlite3.DLL).$(PACKAGE_VERSION) $(libsqlite3.DLL).0.8.6 || exit $$?; \ + ls -la $(libsqlite3.DLL).0.8.6; \ fi -install-so-0 install-so-: -install-so: install-so-$(ENABLE_LIB_SHARED) -install: install-so +install-dll-msys: install-dll-out-implib $(install-dir.bin) + $(INSTALL) $(libsqlite3.DLL) "$(install-dir.bin)" +# ----------------------------------------------^^^ yes, bin +install-dll-mingw: install-dll-msys +install-dll-cygwin: install-dll-msys + +install-dll-darwin: $(install-dir.lib) $(libsqlite3.DLL) + $(INSTALL) $(libsqlite3.DLL) "$(install-dir.lib)" + @echo "Setting up $(libsqlite3.DLL) version symlinks..."; \ + cd "$(install-dir.lib)" || exit $$?; \ + rm -f libsqlite3.0$(T.dll) libsqlite3.$(PACKAGE_VERSION)$(T.dll) || exit $$?; \ + dllname=libsqlite3.$(PACKAGE_VERSION)$(T.dll); \ + mv $(libsqlite3.DLL) $$dllname || exit $$?; \ + ln -s $$dllname $(libsqlite3.DLL) || exit $$?; \ + ln -s $$dllname libsqlite3.0$(T.dll) || exit $$?; \ + ls -la $$dllname $(libsqlite3.DLL) libsqlite3.0$(T.dll) + +install-dll-1: install-dll-$(libsqlite3.DLL.install-rules) +install-dll-0 install-dll-: +install-dll: install-dll-$(ENABLE_LIB_SHARED) +install: install-dll install-lib-1: $(install-dir.lib) $(libsqlite3.LIB) $(INSTALL.noexec) $(libsqlite3.LIB) "$(install-dir.lib)" @@ -209,7 +226,7 @@ ENABLE_STATIC_SHELL = @ENABLE_STATIC_SHELL@ sqlite3-shell-link-flags.1 = $(TOP)/sqlite3.c $(LDFLAGS.libsqlite3) sqlite3-shell-link-flags.0 = -L. -lsqlite3 $(LDFLAGS.zlib) sqlite3-shell-deps.1 = $(TOP)/sqlite3.c -sqlite3-shell-deps.0 = $(libsqlite3.SO) +sqlite3-shell-deps.0 = $(libsqlite3.DLL) sqlite3$(T.exe): $(TOP)/shell.c $(sqlite3-shell-deps.$(ENABLE_STATIC_SHELL)) $(CC) -o $@ \ $(TOP)/shell.c $(sqlite3-shell-link-flags.$(ENABLE_STATIC_SHELL)) \ @@ -237,7 +254,7 @@ install: install-man1 clean: rm -f *.o sqlite3$(T.exe) - rm -f $(libsqlite3.LIB) $(libsqlite3.SO) $(libsqlite3.SO).a + rm -f $(libsqlite3.LIB) $(libsqlite3.DLL) libsqlite3$(T.dll).a distclean: clean rm -f jimsh0$(T.exe) config.* sqlite3.pc sqlite_cfg.h Makefile diff --git a/autoconf/auto.def b/autoconf/auto.def index 099b52aff..3ba900d95 100644 --- a/autoconf/auto.def +++ b/autoconf/auto.def @@ -3,27 +3,8 @@ # # This is the main autosetup-compatible configure script for the # "autoconf" bundle of the SQLite project. -# -# This script must be kept compatible with JimTCL, a copy of which is -# included in this source tree as ./autosetup/jimsh0.c. -# use sqlite-config -sqlite-config-bootstrap autoconf -sqlite-check-common-bins -sqlite-check-common-system-deps -proj-check-rpath -sqlite-handle-soname -sqlite-setup-default-cflags -sqlite-handle-debug -sqlite-handle-threadsafe -sqlite-handle-tempstore -sqlite-handle-line-editing -sqlite-handle-load-extension -sqlite-handle-math -sqlite-handle-icu - -define ENABLE_STATIC_SHELL [opt-bool static-shell] -define ENABLE_LIB_SHARED [opt-bool shared] -define ENABLE_LIB_STATIC [opt-bool static] - -sqlite-config-finalize +sqlite-configure autoconf { + sqlite-check-common-bins + sqlite-check-common-system-deps +} diff --git a/autosetup/autosetup b/autosetup/autosetup index 1479fca40..239987554 100755 --- a/autosetup/autosetup +++ b/autosetup/autosetup @@ -1634,8 +1634,8 @@ proc automf_command_reference {} { if {[regexp {^#.*@section (.*)$} $line -> section]} { lappend doc($current) [list section $section] - continue - } + continue + } # Find embedded module names if {[regexp {^#.*@module ([^ ]*)} $line -> modulename]} { @@ -1651,7 +1651,7 @@ proc automf_command_reference {} { if {$cmd eq "synopsis:"} { set current $modulename lappend doc($current) [list section "Module: $modulename"] - } else { + } else { lappend doc($current) [list subsection $cmd] } @@ -2088,8 +2088,12 @@ if {$autosetup(iswin)} { proc split-path {} { split [getenv PATH .] : } + # Check for an executable file proc file-isexec {exec} { - file executable $exec + if {[file executable $exec] && [file isfile $exec]} { + return 1 + } + return 0 } } diff --git a/autosetup/autosetup-find-tclsh b/autosetup/autosetup-find-tclsh index 2b2006241..9f6d6e940 100755 --- a/autosetup/autosetup-find-tclsh +++ b/autosetup/autosetup-find-tclsh @@ -9,7 +9,7 @@ for tclsh in ./jimsh0 $autosetup_tclsh jimsh tclsh tclsh8.5 tclsh8.6 tclsh8.7; d done echo 1>&2 "No installed jimsh or tclsh, building local bootstrap jimsh0" for cc in ${CC_FOR_BUILD:-cc} gcc; do - { $cc -o jimsh0 "$d/jimsh0.c"; } >/dev/null 2>&1 || continue + { $cc -o jimsh0 "$d/jimsh0.c"; } 2>/dev/null >/dev/null || continue ./jimsh0 "$d/${1-autosetup-test-tclsh}" && exit 0 done echo 1>&2 "No working C compiler found. Tried ${CC_FOR_BUILD:-cc} and gcc." diff --git a/autosetup/jimsh0.c b/autosetup/jimsh0.c index 0526b9a44..1a6453d0c 100644 --- a/autosetup/jimsh0.c +++ b/autosetup/jimsh0.c @@ -1252,6 +1252,14 @@ int Jim_OpenForRead(const char *filename); #endif +# ifndef MAXPATHLEN +# ifdef PATH_MAX +# define MAXPATHLEN PATH_MAX +# else +# define MAXPATHLEN JIM_PATH_LEN +# endif +# endif + int Jim_FileStoreStatData(Jim_Interp *interp, Jim_Obj *varName, const jim_stat_t *sb); @@ -2088,10 +2096,6 @@ enum wbuftype { #define UNIX_SOCKETS 0 #endif -#ifndef MAXPATHLEN -#define MAXPATHLEN JIM_PATH_LEN -#endif - @@ -4173,14 +4177,6 @@ int Jim_regexpInit(Jim_Interp *interp) #define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) #endif -# ifndef MAXPATHLEN -# ifdef PATH_MAX -# define MAXPATHLEN PATH_MAX -# else -# define MAXPATHLEN JIM_PATH_LEN -# endif -# endif - #if defined(__MINGW32__) || defined(__MSYS__) || defined(_MSC_VER) #define ISWINDOWS 1 diff --git a/autosetup/proj.tcl b/autosetup/proj.tcl index 6a1960f60..4be268da6 100644 --- a/autosetup/proj.tcl +++ b/autosetup/proj.tcl @@ -353,7 +353,6 @@ proc proj-opt-was-provided {key} { # # Returns $val. proc proj-opt-set {flag {val 1}} { - global autosetup if {$flag ni $::autosetup(options)} { # We have to add this to autosetup(options) or else future calls # to [opt-bool $flag] will fail validation of $flag. @@ -363,6 +362,15 @@ proc proj-opt-set {flag {val 1}} { return $val } +######################################################################## +# @proj-opt-exists flag +# +# Returns 1 if the given flag has been defined as a legal configure +# option, else returns 0. +proc proj-opt-exists {flag} { + expr {$flag in $::autosetup(options)}; +} + ######################################################################## # @proj-val-truthy val # @@ -907,6 +915,35 @@ proc proj-check-emsdk {} { return $rc } +######################################################################## +# @proj-cc-check-Wl-flag ?flag ?args?? +# +# Checks whether the given linker flag (and optional arguments) can be +# passed from the compiler to the linker using one of these formats: +# +# - -Wl,flag[,arg1[,...argN]] +# - -Wl,flag -Wl,arg1 ...-Wl,argN +# +# If so, that flag string is returned, else an empty string is +# returned. +proc proj-cc-check-Wl-flag {args} { + cc-with {-link 1} { + # Try -Wl,flag,...args + set fli "-Wl" + foreach f $args { append fli ",$f" } + if {[cc-check-flags $fli]} { + return $fli + } + # Try -Wl,flag -Wl,arg1 ...-Wl,argN + set fli "" + foreach f $args { append fli "-Wl,$f " } + if {[cc-check-flags $fli]} { + return [string trim $fli] + } + return "" + } +} + ######################################################################## # @proj-check-rpath # @@ -918,12 +955,7 @@ proc proj-check-emsdk {} { # --exec-prefix=... or --libdir=... are explicitly passed to # configure then [get-define libdir] is used (noting that it derives # from exec-prefix by default). -# -# Achtung: we have seen platforms which report that a given option -# checked here will work but then fails at build-time, and the current -# order of checks reflects that. proc proj-check-rpath {} { - set rc 1 if {[proj-opt-was-provided libdir] || [proj-opt-was-provided exec-prefix]} { set lp "[get-define libdir]" @@ -934,21 +966,18 @@ proc proj-check-rpath {} { # CFLAGS or LIBS or whatever it is that cc-check-flags updates) then # downstream tests may fail because the resulting rpath gets # implicitly injected into them. - cc-with {} { + cc-with {-link 1} { if {[cc-check-flags "-rpath $lp"]} { define LDFLAGS_RPATH "-rpath $lp" - } elseif {[cc-check-flags "-Wl,-rpath,$lp"]} { - define LDFLAGS_RPATH "-Wl,-rpath,$lp" - } elseif {[cc-check-flags "-Wl,-rpath -Wl,$lp"]} { - define LDFLAGS_RPATH "-Wl,-rpath -Wl,$lp" - } elseif {[cc-check-flags -Wl,-R$lp]} { - define LDFLAGS_RPATH "-Wl,-R$lp" } else { - define LDFLAGS_RPATH "" - set rc 0 + set wl [proj-cc-check-Wl-flag -rpath $lp] + if {"" eq $wl} { + set wl [proj-cc-check-Wl-flag -R$lp] + } + define LDFLAGS_RPATH $wl } } - return $rc + expr {"" ne [get-define LDFLAGS_RPATH]} } ######################################################################## @@ -965,7 +994,7 @@ proc proj-check-rpath {} { # potentially avoid some end-user confusion by using their own lib's # name here (which shows up in the "checking..." output). proc proj-check-soname {{libname "libfoo.so.0"}} { - cc-with {} { + cc-with {-link 1} { if {[cc-check-flags "-Wl,-soname,${libname}"]} { define LDFLAGS_SONAME_PREFIX "-Wl,-soname," return 1 @@ -1138,7 +1167,7 @@ proc proj-xfer-options-aliases {mapping} { ######################################################################## # Arguable/debatable... # -# When _not_ cross-compiling and CC_FOR_BUILD is _not_ explcitely +# When _not_ cross-compiling and CC_FOR_BUILD is _not_ explicitly # specified, force CC_FOR_BUILD to be the same as CC, so that: # # ./configure CC=clang @@ -1146,7 +1175,7 @@ proc proj-xfer-options-aliases {mapping} { # will use CC_FOR_BUILD=clang, instead of cc, for building in-tree # tools. This is based off of an email discussion and is thought to # be likely to cause less confusion than seeing 'cc' invocations -# will when the user passes CC=clang. +# when when the user passes CC=clang. # # Sidebar: if we do this before the cc package is installed, it gets # reverted by that package. Ergo, the cc package init will tell the @@ -1191,11 +1220,11 @@ proc proj-which-linenoise {dotH} { # # In that make invocation, $(libdir) would, at make-time, normally be # hard-coded to /foo/lib, rather than /blah/lib. That happens because -# the autosetup exports conventional $prefix-based values for the -# numerous autoconfig-compatible XYZdir vars at configure-time. What -# we would normally want, however, is that --libdir derives from the -# make-time $(prefix). The distinction between configure-time and -# make-time is the significant factor there. +# autosetup exports conventional $prefix-based values for the numerous +# autoconfig-compatible XYZdir vars at configure-time. What we would +# normally want, however, is that --libdir derives from the make-time +# $(prefix). The distinction between configure-time and make-time is +# the significant factor there. # # This function attempts to reconcile those vars in such a way that # they will derive, at make-time, from $(prefix) in a conventional diff --git a/autosetup/sqlite-config.tcl b/autosetup/sqlite-config.tcl index 7d9a9ea84..c09b8583f 100644 --- a/autosetup/sqlite-config.tcl +++ b/autosetup/sqlite-config.tcl @@ -12,7 +12,19 @@ if {[string first " " $autosetup(builddir)] != -1} { may not contain space characters" } -use cc cc-db cc-shared cc-lib pkg-config proj +use proj +# We want this version info to be emitted up front, but we have to +# 'use system' for --prefix=... to work. Ergo, this bit is up here +# instead of in [sqlite-configure]. +define PACKAGE_VERSION [proj-file-content -trim $::autosetup(srcdir)/VERSION] +if {"--help" ni $::argv} { + msg-result "Configuring SQLite version [get-define PACKAGE_VERSION]" +} +use system ; # Will output "Host System" and "Build System" lines +if {"--help" ni $::argv} { + msg-result "Source dir = $::autosetup(srcdir)" + msg-result "Build dir = $::autosetup(builddir)" +} # # Object for communicating config-time state across various @@ -49,11 +61,17 @@ set sqliteConfig(is-cross-compiling) [proj-is-cross-compiling] ######################################################################## # Processes all configure --flags for this build $buildMode must be # either "canonical" or "autoconf", and others may be added in the -# future. -proc sqlite-config-bootstrap {buildMode} { - if {$buildMode ni {canonical autoconf}} { - user-error "Invalid build mode: $buildMode. Expecting one of: canonical, autoconf" +# future. After bootstrapping, $configScript is eval'd in the caller's +# scope, then post-configuration finalization is run. $configScript is +# intended to hold configure code which is specific to the given +# $buildMode, with the caveat that _some_ build-specific code is +# encapsulated in the configuration finalization step. +proc sqlite-configure {buildMode configScript} { + set allBuildModes {canonical autoconf} + if {$buildMode ni $allBuildModes} { + user-error "Invalid build mode: $buildMode. Expecting one of: $allBuildModes" } + set ::sqliteConfig(build-mode) $buildMode ######################################################################## # A gentle introduction to flags handling in autosetup # @@ -132,7 +150,7 @@ proc sqlite-config-bootstrap {buildMode} { build-modes { {*} { shared=1 => {Disable build of shared libary} - static=1 => {Disable build of static library (mostly)} + static=1 => {Disable build of static library} } {canonical} { amalgamation=1 => {Disable the amalgamation and instead build all files separately} @@ -144,7 +162,9 @@ proc sqlite-config-bootstrap {buildMode} { {*} { threadsafe=1 => {Disable mutexing} with-tempstore:=no => {Use an in-RAM database for temporary tables: never,no,yes,always} - largefile=1 => {Disable large file support} + largefile=1 + => {This legacy flag has no effect on the library but may influence + the contents of the generated sqlite_cfg.h} # ^^^ It's not clear that this actually does anything, as # HAVE_LFS is not checked anywhere in the .c/.h/.in files. load-extension=1 => {Disable loading of external extensions} @@ -229,11 +249,14 @@ proc sqlite-config-bootstrap {buildMode} { # Options for exotic/alternative build modes alternative-builds { {canonical} { + # Potential TODO: add --with-wasi-sdk support to the autoconf + # build with-wasi-sdk:=/opt/wasi-sdk => {Top-most dir of the wasi-sdk for a WASI build} + with-emsdk:=auto => {Top-most dir of the Emscripten SDK installation. - Default = EMSDK env var.} + Needed only by ext/wasm build. Default=EMSDK env var.} } } @@ -241,9 +264,12 @@ proc sqlite-config-bootstrap {buildMode} { packaging { {autoconf} { # --disable-static-shell: https://sqlite.org/forum/forumpost/cc219ee704 - static-shell=1 => {Link the sqlite3 shell app against the DLL instead of embedding sqlite3.c} + static-shell=1 + => {Link the sqlite3 shell app against the DLL instead of embedding sqlite3.c} } {*} { + # A potential TODO without a current use case: + #rpath=1 => {Disable use of the rpath linker flag} # soname: https://sqlite.org/src/forumpost/5a3b44f510df8ded soname:=legacy => {SONAME for libsqlite3.so. "none", or not using this flag, sets no @@ -252,10 +278,21 @@ proc sqlite-config-bootstrap {buildMode} { it to that literal value. Any other value is assumed to be a suffix which gets applied to "libsqlite3.so.", e.g. --soname=9.10 equates to "libsqlite3.so.9.10".} + # dll-basename: https://sqlite.org/forum/forumpost/828fdfe904 + dll-basename:=auto + => {Specifies the base name of the resulting DLL file. + If not provided, libsqlite3 is usually assumed but on some platforms + a platform-dependent default is used. On some platforms this flag + gets automatically enabled if it is not provided. Use "default" to + explicitly disable platform-dependent activation on such systems.} # out-implib: https://sqlite.org/forum/forumpost/0c7fc097b2 - out-implib=0 + out-implib:=auto => {Enable use of --out-implib linker flag to generate an - "import library" for the DLL} + "import library" for the DLL. The output's base name name is + specified by the value, with "auto" meaning to figure out a + name automatically. On some platforms this flag gets + automatically enabled if it is not provided. Use "none" to + explicitly disable this feature on such platforms.} } } @@ -263,8 +300,9 @@ proc sqlite-config-bootstrap {buildMode} { developer { {*} { # Note that using the --debug/--enable-debug flag here - # requires patching autosetup/autosetup to rename the --debug - # to --autosetup-debug. + # requires patching autosetup/autosetup to rename its builtin + # --debug to --autosetup-debug. See details in + # autosetup/README.md#patching. with-debug=0 debug=0 => {Enable debug build flags. This option will impact performance by @@ -275,14 +313,21 @@ proc sqlite-config-bootstrap {buildMode} { => {Enable the SQLITE_ENABLE_STMT_SCANSTATUS feature flag} } {canonical} { - dev => {Enable dev-mode build: automatically enables certain other flags} - test-status => {Enable status of tests} - gcov=0 => {Enable coverage testing using gcov} - linemacros => {Enable #line macros in the amalgamation} - dynlink-tools => {Dynamically link libsqlite3 to certain tools which normally statically embed it} + dev + => {Enable dev-mode build: automatically enables certain other flags} + test-status + => {Enable status of tests} + gcov=0 + => {Enable coverage testing using gcov} + linemacros + => {Enable #line macros in the amalgamation} + dynlink-tools + => {Dynamically link libsqlite3 to certain tools which normally statically embed it} } {*} { - dump-defines=0 => {Dump autosetup defines to $::sqliteConfig(dump-defines-txt) (for build debugging)} + dump-defines=0 + => {Dump autosetup defines to $::sqliteConfig(dump-defines-txt) + (for build debugging)} } } }; # $allOpts @@ -300,14 +345,81 @@ proc sqlite-config-bootstrap {buildMode} { } } #lappend opts "soname:=duplicateEntry => {x}"; #just testing - if {[catch {options $opts}]} { + if {[catch {options $opts} msg xopts]} { # Workaround for # where [options] behaves oddly on _some_ TCL builds when it's # called from deeper than the global scope. - return -code break + dict incr xopts -level + return {*}$xopts $msg + } + # The following uplevel is largely cosmetic, the intent being to put + # the most-frequently-useful info at the top of the ./configure + # output, but also avoiding outputing it if --help is used. + uplevel 1 { + use cc cc-db cc-shared cc-lib pkg-config } sqlite-post-options-init -}; # sqlite-config-bootstrap + uplevel 1 $configScript + sqlite-configure-finalize +}; # sqlite-configure + +######################################################################## +# Performs late-stage config steps common to both the canonical and +# autoconf bundle builds. +proc sqlite-configure-finalize {} { + set buildMode $::sqliteConfig(build-mode) + set isCanonical [expr {$buildMode eq "canonical"}] + set isAutoconf [expr {$buildMode eq "autoconf"}] + + define HAVE_LFS 0 + if {[opt-bool largefile]} { + # + # Insofar as we can determine HAVE_LFS has no effect on the + # library. Perhaps it did back in the early 2000's. The + # --enable/disable-largefile flag is retained because it's + # harmless, but it doesn't do anything useful. It does have + # visible side-effects, though: the generated sqlite_cfg.h may (or + # may not) define HAVE_LFS. + # + cc-check-lfs + } + + if {$isCanonical} { + if {![opt-bool static]} { + proj-indented-notice { + NOTICE: static lib build may be implicitly re-activated by + other components, e.g. some test apps. + } + } + } else { + proj-assert { $isAutoconf } "Invalid build mode" + proj-define-for-opt static-shell ENABLE_STATIC_SHELL \ + "Link library statically into the CLI shell?" + if {![opt-bool shared] && ![opt-bool static-shell]} { + proj-opt-set shared 1 + proj-indented-notice { + NOTICE: ignoring --disable-shared because --disable-static-shell + was specified. + } + } + } + proj-define-for-opt shared ENABLE_LIB_SHARED "Build shared library?" + proj-define-for-opt static ENABLE_LIB_STATIC "Build static library?" + + sqlite-handle-debug + sqlite-handle-rpath + sqlite-handle-soname + sqlite-handle-threadsafe + sqlite-handle-tempstore + sqlite-handle-line-editing + sqlite-handle-load-extension + sqlite-handle-math + sqlite-handle-icu + sqlite-handle-env-quirks + sqlite-process-dot-in-files + sqlite-post-config-validation + sqlite-dump-defines +}; # sqlite-configure-finalize ######################################################################## # Runs some common initialization which must happen immediately after @@ -316,6 +428,10 @@ proc sqlite-config-bootstrap {buildMode} { # top-level build and the "autoconf" build, but it's not intended to # be a catch-all dumping ground for such. proc sqlite-post-options-init {} { + define PACKAGE_NAME "sqlite" + define PACKAGE_URL {https://sqlite.org} + define PACKAGE_BUGREPORT [get-define PACKAGE_URL]/forum + define PACKAGE_STRING "[get-define PACKAGE_NAME] [get-define PACKAGE_VERSION]" # # Carry values from hidden --flag aliases over to their canonical # flag forms. This list must include only options which are common @@ -336,23 +452,7 @@ proc sqlite-post-options-init {} { define SQLITE_OS_WIN 0 } set ::sqliteConfig(msg-debug-enabled) [proj-val-truthy [get-env msg-debug 0]] - sqlite-setup-package-info -} - -######################################################################## -# Called by [sqlite-post-options-init] to set up PACKAGE_NAME and -# related defines. -proc sqlite-setup-package-info {} { - set srcdir $::autosetup(srcdir) - set PACKAGE_VERSION [proj-file-content -trim $srcdir/VERSION] - define PACKAGE_NAME "sqlite" - define PACKAGE_URL {https://sqlite.org} - define PACKAGE_VERSION $PACKAGE_VERSION - define PACKAGE_STRING "[get-define PACKAGE_NAME] $PACKAGE_VERSION" - define PACKAGE_BUGREPORT [get-define PACKAGE_URL]/forum - msg-result "Source dir = $srcdir" - msg-result "Build dir = $::autosetup(builddir)" - msg-result "Configuring SQLite version $PACKAGE_VERSION" + sqlite-setup-default-cflags } ######################################################################## @@ -551,15 +651,41 @@ proc sqlite-setup-default-cflags {} { # Handle various SQLITE_ENABLE_... feature flags. proc sqlite-handle-common-feature-flags {} { msg-result "Feature flags..." - foreach {boolFlag featureFlag ifSetEvalThis} { + if {"tcl-extension" eq $::sqliteConfig(build-mode)} { + set allFlagEnables {fts3 fts4 fts5 rtree geopoly} + } else { + set allFlagEnables {fts4 fts5 rtree rtree geopoly session} + } + if {![opt-bool all]} { + # Special handling for --disable-all + foreach flag $allFlagEnables { + if {![proj-opt-was-provided $flag]} { + proj-opt-set $flag 0 + } + } + } + foreach {boolFlag featureFlag ifSetEvalThis} [proj-strip-hash-comments { all {} { - # The 'all' option must be first in this list. - proj-opt-set fts4 - proj-opt-set fts5 - proj-opt-set geopoly - proj-opt-set rtree - proj-opt-set session + # The 'all' option must be first in this list. This impl makes + # an effort to only apply flags which the user did not already + # apply, so that combinations like (--all --disable-geopoly) + # will indeed disable geopoly. There are corner cases where + # flags which depend on each other will behave in non-intuitive + # ways: + # + # --all --disable-rtree + # + # Will NOT disable geopoly, though geopoly depends on rtree. + # The --geopoly flag, though, will automatically re-enable + # --rtree, so --disable-rtree won't actually disable anything in + # that case. + foreach k $allFlagEnables { + if {![proj-opt-was-provided $k]} { + proj-opt-set $k 1 + } + } } + fts3 -DSQLITE_ENABLE_FTS3 {sqlite-affirm-have-math fts3} fts4 -DSQLITE_ENABLE_FTS4 {sqlite-affirm-have-math fts4} fts5 -DSQLITE_ENABLE_FTS5 {sqlite-affirm-have-math fts5} geopoly -DSQLITE_ENABLE_GEOPOLY {proj-opt-set rtree} @@ -576,7 +702,7 @@ proc sqlite-handle-common-feature-flags {} { } } scanstatus -DSQLITE_ENABLE_STMT_SCANSTATUS {} - } { + }] { if {$boolFlag ni $::autosetup(options)} { # Skip flags which are in the canonical build but not # the autoconf bundle. @@ -798,6 +924,35 @@ proc sqlite-handle-emsdk {} { } } +######################################################################## +# Internal helper for [sqlite-check-line-editing]. Returns a list of +# potential locations under which readline.h might be found. +proc sqlite-get-readline-dir-list {} { + # Historical note: the dirs list, except for the inclusion of + # $prefix and some platform-specific dirs, originates from the + # legacy configure script + set dirs [list [get-define prefix]] + switch -glob -- [get-define host] { + *-linux-android { + # Possibly termux + lappend dirs /data/data/com.termux/files/usr + } + *-mingw32 { + lappend dirs /mingw32 /mingw + } + *-mingw64 { + lappend dirs /mingw64 /mingw + } + } + lappend dirs /usr /usr/local /usr/local/readline /usr/contrib + set rv {} + foreach d $dirs { + if {[file isdir $d]} {lappend rv $d} + } + #msg-debug "sqlite-get-readline-dir-list dirs=$rv" + return $rv +} + ######################################################################## # sqlite-check-line-editing jumps through proverbial hoops to try to # find a working line-editing library, setting: @@ -934,7 +1089,7 @@ proc sqlite-check-line-editing {} { # ^^^ this check is derived from the legacy configure script. proj-warn "Skipping check for readline.h because we're cross-compiling." } else { - set dirs "[get-define prefix] /usr /usr/local /usr/local/readline /usr/contrib /mingw" + set dirs [sqlite-get-readline-dir-list] set subdirs "include/$editLibName" if {"editline" eq $editLibName} { lappend subdirs include/readline @@ -942,16 +1097,14 @@ proc sqlite-check-line-editing {} { # and uses libreadline's header. } lappend subdirs include - # ^^^ The dirs and subdirs lists are, except for the inclusion - # of $prefix and editline, from the legacy configure script set rlInc [proj-search-for-header-dir readline.h \ -dirs $dirs -subdirs $subdirs] if {"" ne $rlInc} { if {[string match */readline $rlInc]} { - set rlInc [file dirname $rlInc]; # shell #include's + set rlInc [file dirname $rlInc]; # CLI shell: #include } elseif {[string match */editline $rlInc]} { set editLibDef HAVE_EDITLINE - set rlInc [file dirname $rlInc]; # shell #include's + set rlInc [file dirname $rlInc]; # CLI shell: #include } set rlInc "-I${rlInc}" } @@ -970,7 +1123,7 @@ proc sqlite-check-line-editing {} { set rlLib "" if {"" ne $rlInc} { set rlLib [opt-val with-readline-ldflags] - if {"" eq $rlLib || "auto" eq $rlLib} { + if {$rlLib eq "auto" || $rlLib eq ""} { set rlLib "" set libTerm "" if {[proj-check-function-in-lib tgetent "$editLibName ncurses curses termcap"]} { @@ -995,7 +1148,7 @@ proc sqlite-check-line-editing {} { # linking to the GPL'd libreadline. Presumably that distinction is # significant for those using --editline. proj-indented-notice { - NOTE: the local libedit but uses so we + NOTE: the local libedit uses so we will compile with -DHAVE_READLINE=1 but will link with libedit. } @@ -1222,11 +1375,11 @@ proc sqlite-handle-math {} { # libtool applied only on Mac platforms. # # Based on https://sqlite.org/forum/forumpost/9dfd5b8fd525a5d7. -proc sqlite-check-mac-cversion {} { +proc sqlite-handle-mac-cversion {} { define LDFLAGS_MAC_CVERSION "" set rc 0 if {[proj-looks-like-mac]} { - cc-with {} { + cc-with {-link 1} { # These version numbers are historical libtool-defined values, not # library-defined ones if {[cc-check-flags "-Wl,-current_version,9.6.0"] @@ -1244,27 +1397,100 @@ proc sqlite-check-mac-cversion {} { } ######################################################################## -# Define LDFLAGS_OUT_IMPLIB to either an empty string or to a +# If this is a Mac platform, check for support for +# -Wl,-install_name,... and, if it's set, define +# LDFLAGS_MAC_INSTALL_NAME to a variant of that string which is +# intended to expand at make-time, else set LDFLAGS_MAC_INSTALL_NAME +# to an empty string. +# +# https://sqlite.org/forum/forumpost/5651662b8875ec0a +proc sqlite-handle-mac-install-name {} { + define LDFLAGS_MAC_INSTALL_NAME ""; # {-Wl,-install_name,"$(install-dir.lib)/$(libsqlite3.DLL)"} + set rc 0 + if {[proj-looks-like-mac]} { + cc-with {-link 1} { + if {[cc-check-flags "-Wl,-install_name,/usr/local/lib/libsqlite3.dylib"]} { + define LDFLAGS_MAC_INSTALL_NAME {-Wl,-install_name,"$(install-dir.lib)/$(libsqlite3.DLL)"} + set rc 1 + } + } + } + return $rc +} + +######################################################################## +# Handles the --dll-basename configure flag. [define]'s +# SQLITE_DLL_BASENAME to the DLL's preferred base name (minus +# extension). If --dll-basename is not provided then this is always +# "libsqlite3", otherwise it may use a different value based on the +# value of [get-define host]. +proc sqlite-handle-dll-basename {} { + if {[proj-opt-was-provided dll-basename]} { + set dn [join [opt-val dll-basename] ""] + if {$dn in {none default}} { set dn libsqlite3 } + } else { + set dn libsqlite3 + } + if {$dn in {auto ""}} { + switch -glob -- [get-define host] { + *-*-cygwin { set dn cygsqlite3-0 } + *-*-ming* { set dn libsqlite3-0 } + *-*-msys { set dn msys-sqlite3-0 } + default { set dn libsqlite3 } + } + } + define SQLITE_DLL_BASENAME $dn +} + +######################################################################## +# [define]s LDFLAGS_OUT_IMPLIB to either an empty string or to a # -Wl,... flag for the platform-specific --out-implib flag, which is # used for building an "import library .dll.a" file on some platforms -# (e.g. mingw). Returns 1 if supported, else 0. +# (e.g. msys2, mingw). Returns 1 if supported, else 0. +# +# The name of the import library is [define]d in SQLITE_OUT_IMPLIB. # # If the configure flag --out-implib is not used then this is a no-op. -# The feature is specifically opt-in because on some platforms the -# feature test will pass but using that flag will fail at link-time -# (e.g. OpenBSD). +# If that flag is used but the capability is not available, a fatal +# error is triggered. +# +# This feature is specifically opt-in because it's supported on far +# more platforms than actually need it and enabling it causes creation +# of libsqlite3.so.a files which are unnecessary in most environments. # # Added in response to: https://sqlite.org/forum/forumpost/0c7fc097b2 -proc sqlite-check-out-implib {} { +# +# Platform notes: +# +# - cygwin sqlite packages historically install no .dll.a file. +# +# - msys2 and mingw sqlite packages historically install +# /usr/lib/libsqlite3.dll.a despite the DLL being in +# /usr/bin/msys-sqlite3-0.dll. +proc sqlite-handle-out-implib {} { define LDFLAGS_OUT_IMPLIB "" + define SQLITE_OUT_IMPLIB "" set rc 0 if {[proj-opt-was-provided out-implib]} { - cc-with {} { - set dll "libsqlite3[get-define TARGET_DLLEXT]" - set flags "-Wl,--out-implib,${dll}.a" - if {[cc-check-flags $flags]} { - define LDFLAGS_OUT_IMPLIB $flags - set rc 1 + set olBaseName [join [opt-val out-implib] ""] + if {$olBaseName in {auto ""}} { + set olBaseName "libsqlite3" ;# [get-define SQLITE_DLL_BASENAME] + # Based on discussions with mingw/msys users, the import lib + # should always be called libsqlite3.dll.a even on platforms + # which rename libsqlite3.dll to something else. + } + if {$olBaseName ne "none"} { + cc-with {-link 1} { + set dll "${olBaseName}[get-define TARGET_DLLEXT]" + set flags [proj-cc-check-Wl-flag --out-implib ${dll}.a] + if {"" ne $flags} { + define LDFLAGS_OUT_IMPLIB $flags + define SQLITE_OUT_IMPLIB ${dll}.a + set rc 1 + } + } + if {!$rc} { + user-error "--out-implib is not supported on this platform" } } } @@ -1272,14 +1498,95 @@ proc sqlite-check-out-implib {} { } ######################################################################## -# Performs late-stage config steps common to both the canonical and -# autoconf bundle builds. -proc sqlite-config-finalize {} { - sqlite-check-mac-cversion - sqlite-check-out-implib - sqlite-process-dot-in-files - sqlite-post-config-validation - sqlite-dump-defines +# If the given platform identifier (defaulting to [get-define host]) +# appears to be one of the Unix-on-Windows environments, returns a +# brief symbolic name for that environment, else returns an empty +# string. +# +# It does not distinguish between msys and msys2, returning msys for +# both. The build does not, as of this writing, specifically support +# msys v1. +proc sqlite-env-is-unix-on-windows {{envTuple ""}} { + if {"" eq $envTuple} { + set envTuple [get-define host] + } + set name "" + switch -glob -- $envTuple { + *-*-cygwin { set name cygwin } + *-*-ming* { set name mingw } + *-*-msys { set name msys } + } + return $name; +} + +######################################################################## +# Performs various tweaks to the build which are only relevant on +# certain platforms, e.g. Mac and "Unix on Windows" platforms (msys2, +# cygwin, ...). +# +# 1) DLL installation: +# +# [define]s SQLITE_DLL_INSTALL_RULES to a symbolic name suffix for a +# set of "make install" rules to use for installation of the DLL +# deliverable. The makefile is tasked with with providing rules named +# install-dll-NAME which runs the installation for that set, as well +# as providing a rule named install-dll which resolves to +# install-dll-NAME (perhaps indirectly, depending on whether the DLL +# is (de)activated). +# +# The default value is "unix-generic". +# +# 2) --out-implib: +# +# On platforms where an "import library" is conventionally used but +# --out-implib was not explicitly used, automatically add that flag. +# This conventionally applies to the "Unix on Windows" environments +# like msys and cygwin. +# +# 3) --dll-basename: +# +# On the same platforms addressed by --out-implib, if --dll-basename +# is not specified, --dll-basename=auto is implied. +proc sqlite-handle-env-quirks {} { + set instName unix-generic; # name of installation rules set + set autoDll 0; # true if --out-implib/--dll-basename should be implied + set host [get-define host] + switch -glob -- $host { + *apple* - + *darwin* { set instName darwin } + default { + set x [sqlite-env-is-unix-on-windows $host] + if {"" ne $x} { + set instName $x + set autoDll 1 + } + } + } + define SQLITE_DLL_INSTALL_RULES $instName + if {$autoDll} { + if {![proj-opt-was-provided out-implib]} { + # Imply --out-implib=auto + proj-indented-notice [subst -nocommands -nobackslashes { + NOTICE: auto-enabling --out-implib for environment [$host]. + Use --out-implib=none to disable this special case + or --out-implib=auto to squelch this notice. + }] + proj-opt-set out-implib auto + } + if {![proj-opt-was-provided dll-basename]} { + # Imply --dll-basename=auto + proj-indented-notice [subst -nocommands -nobackslashes { + NOTICE: auto-enabling --dll-basename for environment [$host]. + Use --dll-basename=default to disable this special case + or --dll-basename=auto to squelch this notice. + }] + proj-opt-set dll-basename auto + } + } + sqlite-handle-dll-basename + sqlite-handle-out-implib + sqlite-handle-mac-cversion + sqlite-handle-mac-install-name } ######################################################################## @@ -1533,9 +1840,9 @@ proc sqlite-check-tcl {} { proj-fatal "No tclConfig.sh found under ${with_tcl}" } } else { - # If we have not yet found a tclConfig.sh file, look in - # $libdir which is set automatically by autosetup or by the - # --prefix command-line option. See + # If we have not yet found a tclConfig.sh file, look in $libdir + # which is set automatically by autosetup or via the --prefix + # command-line option. See # https://sqlite.org/forum/forumpost/e04e693439a22457 set libdir [get-define libdir] if {[file readable "${libdir}/tclConfig.sh"]} { @@ -1721,7 +2028,7 @@ proc sqlite-determine-codegen-tcl {} { } define BTCLSH "\$(TCLSH_CMD)" } - }; # CC swap-out + }; # /define-push $flagsToRestore return $cgtcl }; # sqlite-determine-codegen-tcl @@ -1732,6 +2039,26 @@ proc sqlite-handle-tcl {} { msg-result "TCL for code generation: [sqlite-determine-codegen-tcl]" } +######################################################################## +# Handle the --enable/disable-rpath flag. +proc sqlite-handle-rpath {} { + proj-check-rpath + # autosetup/cc-shared.tcl sets the rpath flag definition in + # [get-define SH_LINKRPATH], but it does so on a per-platform basis + # rather than as a compiler check. Though we should do a proper + # compiler check (as proj-check-rpath does), we may want to consider + # adopting its approach of clearing the rpath flags for environments + # for which sqlite-env-is-unix-on-windows returns a non-empty + # string. + +# if {[proj-opt-truthy rpath]} { +# proj-check-rpath +# } else { +# msg-result "Disabling use of rpath." +# define LDFLAGS_RPATH "" +# } +} + ######################################################################## # If the --dump-defines configure flag is provided then emit a list of # all [define] values to config.defines.txt, else do nothing. diff --git a/ext/misc/series.c b/ext/misc/series.c index aff979692..06f1fd281 100644 --- a/ext/misc/series.c +++ b/ext/misc/series.c @@ -60,8 +60,7 @@ ** step HIDDEN ** ); ** -** The virtual table also has a rowid, logically equivalent to n+1 where -** "n" is the ascending integer in the aforesaid production definition. +** The virtual table also has a rowid which is an alias for the value. ** ** Function arguments in queries against this virtual table are translated ** into equality constraints against successive hidden columns. In other @@ -116,6 +115,7 @@ SQLITE_EXTENSION_INIT1 #include #include #include +#include #ifndef SQLITE_OMIT_VIRTUALTABLE /* @@ -276,6 +276,7 @@ static int seriesConnect( int rc; /* Column numbers */ +#define SERIES_COLUMN_ROWID (-1) #define SERIES_COLUMN_VALUE 0 #define SERIES_COLUMN_START 1 #define SERIES_COLUMN_STOP 2 @@ -363,13 +364,11 @@ static int seriesColumn( #endif /* -** Return the rowid for the current row, logically equivalent to n+1 where -** "n" is the ascending integer in the aforesaid production definition. +** The rowid is the same as the value. */ static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ series_cursor *pCur = (series_cursor*)cur; - sqlite3_uint64 n = pCur->ss.uSeqIndexNow; - *pRowid = (sqlite3_int64)((nss.iValueNow; return SQLITE_OK; } @@ -482,25 +481,52 @@ static int seriesFilter( ** constraints on the "value" column. */ if( idxNum & 0x0080 ){ - iMin = iMax = sqlite3_value_int64(argv[i++]); + if( sqlite3_value_numeric_type(argv[i])==SQLITE_FLOAT ){ + double r = sqlite3_value_double(argv[i++]); + if( r==ceil(r) ){ + iMin = iMax = (sqlite3_int64)r; + }else{ + returnNoRows = 1; + } + }else{ + iMin = iMax = sqlite3_value_int64(argv[i++]); + } }else{ if( idxNum & 0x0300 ){ - iMin = sqlite3_value_int64(argv[i++]); - if( idxNum & 0x0200 ){ - if( iMin==LARGEST_INT64 ){ - returnNoRows = 1; + if( sqlite3_value_numeric_type(argv[i])==SQLITE_FLOAT ){ + double r = sqlite3_value_double(argv[i++]); + if( idxNum & 0x0200 && r==ceil(r) ){ + iMin = (sqlite3_int64)ceil(r+1.0); }else{ - iMin++; + iMin = (sqlite3_int64)ceil(r); + } + }else{ + iMin = sqlite3_value_int64(argv[i++]); + if( idxNum & 0x0200 ){ + if( iMin==LARGEST_INT64 ){ + returnNoRows = 1; + }else{ + iMin++; + } } } } if( idxNum & 0x3000 ){ - iMax = sqlite3_value_int64(argv[i++]); - if( idxNum & 0x2000 ){ - if( iMax==SMALLEST_INT64 ){ - returnNoRows = 1; + if( sqlite3_value_numeric_type(argv[i])==SQLITE_FLOAT ){ + double r = sqlite3_value_double(argv[i++]); + if( (idxNum & 0x2000)!=0 && r==floor(r) ){ + iMax = (sqlite3_int64)(r-1.0); }else{ - iMax--; + iMax = (sqlite3_int64)floor(r); + } + }else{ + iMax = sqlite3_value_int64(argv[i++]); + if( idxNum & 0x2000 ){ + if( iMax==SMALLEST_INT64 ){ + returnNoRows = 1; + }else{ + iMax--; + } } } } @@ -519,8 +545,7 @@ static int seriesFilter( pCur->ss.iBase += ((d+szStep-1)/szStep)*szStep; } if( pCur->ss.iTerm>iMax ){ - sqlite3_uint64 d = pCur->ss.iTerm - iMax; - pCur->ss.iTerm -= ((d+szStep-1)/szStep)*szStep; + pCur->ss.iTerm = iMax; } }else{ sqlite3_int64 szStep = -pCur->ss.iStep; @@ -530,8 +555,7 @@ static int seriesFilter( pCur->ss.iBase -= ((d+szStep-1)/szStep)*szStep; } if( pCur->ss.iTermss.iTerm; - pCur->ss.iTerm += ((d+szStep-1)/szStep)*szStep; + pCur->ss.iTerm = iMin; } } } @@ -659,7 +683,10 @@ static int seriesBestIndex( continue; } if( pConstraint->iColumniColumn==SERIES_COLUMN_VALUE && pConstraint->usable ){ + if( (pConstraint->iColumn==SERIES_COLUMN_VALUE || + pConstraint->iColumn==SERIES_COLUMN_ROWID) + && pConstraint->usable + ){ switch( op ){ case SQLITE_INDEX_CONSTRAINT_EQ: case SQLITE_INDEX_CONSTRAINT_IS: { diff --git a/main.mk b/main.mk index 808f92546..b40b9f968 100644 --- a/main.mk +++ b/main.mk @@ -189,19 +189,19 @@ CFLAGS.readline ?= -I$(prefix)/include # INSTALL ?= install # -# $(ENABLE_SHARED) = +# $(ENABLE_LIB_SHARED) = # -# 1 if libsqlite3.$(T.dll) should be built. +# 1 if libsqlite3$(T.dll) should be built. # -ENABLE_SHARED ?= 1 +ENABLE_LIB_SHARED ?= 1 # -# $(ENABLE_STATIC) = +# $(ENABLE_LIB_STATIC) = # -# 1 if libsqlite3.$(T.lib) should be built. Some components, +# 1 if libsqlite3$(T.lib) should be built. Some components, # e.g. libtclsqlite3 and some test apps, implicitly require the static # library and will ignore this preference. # -ENABLE_STATIC ?= 1 +ENABLE_LIB_STATIC ?= 1 # # $(USE_AMALGAMATION) # @@ -220,7 +220,7 @@ USE_AMALGAMATION ?= 1 # may require that the user specifically prepend "." to their # $LD_LIBRARY_PATH so that the dynamic linker does not pick up a # libsqlite3.so from outside the source tree. Alternately, symlinking -# the in-build-tree $(libsqlite3.SO) to some dir in the system's +# the in-build-tree $(libsqlite3.DLL) to some dir in the system's # library path will work for giving the apps access to the in-tree # DLL. # @@ -1052,8 +1052,35 @@ T.link.tcl = $(T.tcl.env.source); $(T.link) cp fts5.c fts5.h tsrc touch .target_source +# +# libsqlite3.DLL.basename = the base name of the resulting DLL. This +# is typically libsqlite3 but varies wildly on Unix-like Windows +# environments (msys, cygwin, and friends). Conversely, the base name +# of the static library ($(libsqlite3.LIB)) is constant on all tested +# platforms. +# +libsqlite3.DLL.basename ?= libsqlite3 +# +# libsqlite3.DLL => the DLL library +# +libsqlite3.DLL = $(libsqlite3.DLL.basename)$(T.dll) +# +# libsqlite3.out.implib => "import library" file generated by the +# --out-implib linker flag. Not commonly used on Unix systems but is +# on the Windows-side Unix-esque environments and typically as a value +# of "libsqlite3.dll.a". It is expected to match the filename, if any, +# provided by the -Wl,--out-implib,FILENAME flag. +# +libsqlite3.out.implib ?= +# +# libsqlite3.LIB => the static library +# libsqlite3.LIB = libsqlite3$(T.lib) -libsqlite3.SO = libsqlite3$(T.dll) + +# +# libsqlite3.DLL.install-rules => the suffix of the symoblic name of +# the makefile rules for installing the DLL. +libsqlite3.DLL.install-rules ?= unix-generic # Rules to build the LEMON compiler generator # @@ -1357,9 +1384,9 @@ tclsqlite-shell.o: $(T.tcl.env.sh) $(TOP)/src/tclsqlite.c $(DEPS_OBJ_COMMON) tclsqlite-stubs.o: $(T.tcl.env.sh) $(TOP)/src/tclsqlite.c $(DEPS_OBJ_COMMON) $(T.compile.tcl) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c $$TCL_INCLUDE_SPEC -tclsqlite3$(T.exe): $(T.tcl.env.sh) tclsqlite-shell.o $(libsqlite3.SO) +tclsqlite3$(T.exe): $(T.tcl.env.sh) tclsqlite-shell.o $(libsqlite3.DLL) $(T.link.tcl) -o $@ tclsqlite-shell.o \ - $(libsqlite3.SO) $$TCL_INCLUDE_SPEC $$TCL_LIB_SPEC \ + $(libsqlite3.DLL) $$TCL_INCLUDE_SPEC $$TCL_LIB_SPEC \ $(LDFLAGS.libsqlite3) tclsqlite3$(T.exe)-1: tclsqlite3$(T.exe) tclsqlite3$(T.exe)-0 tclsqlite3$(T.exe)-: @@ -1406,32 +1433,33 @@ $(libsqlite3.LIB): $(LIBOBJ) $(AR) $(AR.flags) $@ $(LIBOBJ) $(libsqlite3.LIB)-1: $(libsqlite3.LIB) $(libsqlite3.LIB)-0 $(libsqlite3.LIB)-: -lib: $(libsqlite3.LIB)-$(ENABLE_STATIC) +lib: $(libsqlite3.LIB)-$(ENABLE_LIB_STATIC) all: lib # # Dynamic libsqlite3 # -$(libsqlite3.SO): $(LIBOBJ) +$(libsqlite3.DLL): $(LIBOBJ) $(T.link.shared) -o $@ $(LIBOBJ) $(LDFLAGS.libsqlite3) \ $(LDFLAGS.libsqlite3.os-specific) $(LDFLAGS.libsqlite3.soname) -$(libsqlite3.SO)-1: $(libsqlite3.SO) -$(libsqlite3.SO)-0 $(libsqlite3.SO)-: -so: $(libsqlite3.SO)-$(ENABLE_SHARED) +$(libsqlite3.DLL)-1: $(libsqlite3.DLL) +$(libsqlite3.DLL)-0 $(libsqlite3.DLL)-: +so: $(libsqlite3.DLL)-$(ENABLE_LIB_SHARED) all: so # -# On most Unix-like platforms, install the $(libsqlite3.SO) as -# $(libsqlite3.SO).$(PACKAGE_VERSION) and create symlinks which point +# DLL installation... +# +# On most Unix-like platforms, install the $(libsqlite3.DLL) as +# $(libsqlite3.DLL).$(PACKAGE_VERSION) and create symlinks which point # to it: # # - libsqlite3.so.$(PACKAGE_VERSION) # - libsqlite3.so.0 =symlink-> libsqlite3.so.$(PACKAGE_VERSION) (see below) # - libsqlite3.so =symlink-> libsqlite3.so.3 # -# The symlinks are not added on platforms where $(T.dll) is ".dll", -# and different transformations take place on platforms where $(T.dll) -# is ".dylib". +# Different rules apply for platforms where $(T.dll)==.dylib and for +# the "Unix on Windows" environments. # # The link named libsqlite3.so.0 is provided in an attempt to reduce # downstream disruption when performing upgrades from pre-3.48 to a @@ -1453,7 +1481,7 @@ all: so # 1) If libsqlite3.so.0.8.6 is found in the target installation # directory then it is re-linked to point to the newer-style # names. We cannot retain both the old and new installation because -# they both share the high-level name $(libsqlite3.SO). The +# they both share the high-level name $(libsqlite3.DLL). The # down-side of this is that it may upset packaging tools when we # replace libsqlite3.so (from a legacy package) with a new symlink. # @@ -1467,45 +1495,54 @@ all: so # In either case, libsqlite3.la, if found, is deleted because it would # contain stale state, refering to non-libtool-generated libraries. # -install-so-1: $(install-dir.lib) $(libsqlite3.SO) - $(INSTALL) $(libsqlite3.SO) "$(install-dir.lib)" - @if [ -f $(libsqlite3.SO).a ]; then \ - $(INSTALL) $(libsqlite3.SO).a "$(install-dir.lib)"; \ + +install-dll-out-implib: $(install-dir.lib) $(libsqlite3.DLL) + if [ x != "x$(libsqlite3.out.implib)" ] && [ -f "$(libsqlite3.out.implib)" ]; then \ + $(INSTALL) $(libsqlite3.out.implib) "$(install-dir.lib)"; \ fi - @echo "Setting up $(libsqlite3.SO) version symlinks..."; \ - if [ x.dll = x$(T.dll) ]; then \ - echo "No library symlinks needed on this platform"; \ - elif [ x.dylib = x$(T.dll) ]; then \ - cd "$(install-dir.lib)" || exit $$?; \ - rm -f libsqlite3.0$(T.dll) libsqlite3.$(PACKAGE_VERSION)$(T.dll) || exit $$?; \ - dllname=libsqlite3.$(PACKAGE_VERSION)$(T.dll); \ - mv $(libsqlite3.SO) $$dllname || exit $$?; \ - ln -s $$dllname $(libsqlite3.SO) || exit $$?; \ - ln -s $$dllname libsqlite3.0$(T.dll) || exit $$?; \ - ls -la $$dllname $(libsqlite3.SO) libsqlite3.0$(T.dll); \ - else \ - cd "$(install-dir.lib)" || exit $$?; \ - rm -f $(libsqlite3.SO).0 $(libsqlite3.SO).$(PACKAGE_VERSION) || exit $$?; \ - mv $(libsqlite3.SO) $(libsqlite3.SO).$(PACKAGE_VERSION) || exit $$?; \ - ln -s $(libsqlite3.SO).$(PACKAGE_VERSION) $(libsqlite3.SO) || exit $$?; \ - ln -s $(libsqlite3.SO).$(PACKAGE_VERSION) $(libsqlite3.SO).0 || exit $$?; \ - ls -la $(libsqlite3.SO) $(libsqlite3.SO).[a03]*; \ - if [ -e $(libsqlite3.SO).0.8.6 ]; then \ - echo "ACHTUNG: legacy libtool-compatible install found. Re-linking it..."; \ - rm -f libsqlite3.la $(libsqlite3.SO).0.8.6 || exit $$?; \ - ln -s $(libsqlite3.SO).$(PACKAGE_VERSION) $(libsqlite3.SO).0.8.6 || exit $$?; \ - ls -la $(libsqlite3.SO).0.8.6; \ - elif [ x1 = "x$(INSTALL_SO_086_LINK)" ]; then \ - echo "ACHTUNG: installing legacy libtool-style links because INSTALL_SO_086_LINK=1"; \ - rm -f libsqlite3.la $(libsqlite3.SO).0.8.6 || exit $$?; \ - ln -s $(libsqlite3.SO).$(PACKAGE_VERSION) $(libsqlite3.SO).0.8.6 || exit $$?; \ - ls -la $(libsqlite3.SO).0.8.6; \ - fi; \ + +install-dll-unix-generic: install-dll-out-implib + $(INSTALL) $(libsqlite3.DLL) "$(install-dir.lib)" + @echo "Setting up $(libsqlite3.DLL) version symlinks..."; \ + cd "$(install-dir.lib)" || exit $$?; \ + rm -f $(libsqlite3.DLL).0 $(libsqlite3.DLL).$(PACKAGE_VERSION) || exit $$?; \ + mv $(libsqlite3.DLL) $(libsqlite3.DLL).$(PACKAGE_VERSION) || exit $$?; \ + ln -s $(libsqlite3.DLL).$(PACKAGE_VERSION) $(libsqlite3.DLL) || exit $$?; \ + ln -s $(libsqlite3.DLL).$(PACKAGE_VERSION) $(libsqlite3.DLL).0 || exit $$?; \ + ls -la $(libsqlite3.DLL) $(libsqlite3.DLL).[a03]*; \ + if [ -e $(libsqlite3.DLL).0.8.6 ]; then \ + echo "ACHTUNG: legacy libtool-compatible install found. Re-linking it..."; \ + rm -f libsqlite3.la $(libsqlite3.DLL).0.8.6 || exit $$?; \ + ln -s $(libsqlite3.DLL).$(PACKAGE_VERSION) $(libsqlite3.DLL).0.8.6 || exit $$?; \ + ls -la $(libsqlite3.DLL).0.8.6; \ + elif [ x1 = "x$(INSTALL_SO_086_LINK)" ]; then \ + echo "ACHTUNG: installing legacy libtool-style links because INSTALL_SO_086_LINK=1"; \ + rm -f libsqlite3.la $(libsqlite3.DLL).0.8.6 || exit $$?; \ + ln -s $(libsqlite3.DLL).$(PACKAGE_VERSION) $(libsqlite3.DLL).0.8.6 || exit $$?; \ + ls -la $(libsqlite3.DLL).0.8.6; \ fi -install-so-0 install-so-: -install-so: install-so-$(ENABLE_SHARED) -install: install-so +install-dll-msys: install-dll-out-implib $(install-dir.bin) + $(INSTALL) $(libsqlite3.DLL) "$(install-dir.bin)" +# ----------------------------------------------^^^ yes, bin +install-dll-mingw: install-dll-msys +install-dll-cygwin: install-dll-msys + +install-dll-darwin: $(install-dir.lib) $(libsqlite3.DLL) + $(INSTALL) $(libsqlite3.DLL) "$(install-dir.lib)" + @echo "Setting up $(libsqlite3.DLL) version symlinks..."; \ + cd "$(install-dir.lib)" || exit $$?; \ + rm -f libsqlite3.0$(T.dll) libsqlite3.$(PACKAGE_VERSION)$(T.dll) || exit $$?; \ + dllname=libsqlite3.$(PACKAGE_VERSION)$(T.dll); \ + mv $(libsqlite3.DLL) $$dllname || exit $$?; \ + ln -s $$dllname $(libsqlite3.DLL) || exit $$?; \ + ln -s $$dllname libsqlite3.0$(T.dll) || exit $$?; \ + ls -la $$dllname $(libsqlite3.DLL) libsqlite3.0$(T.dll) + +install-dll-1: install-dll-$(libsqlite3.DLL.install-rules) +install-dll-0 install-dll-: +install-dll: install-dll-$(ENABLE_LIB_SHARED) +install: install-dll # # Install $(libsqlite3.LIB) @@ -1513,7 +1550,7 @@ install: install-so install-lib-1: $(install-dir.lib) $(libsqlite3.LIB) $(INSTALL.noexec) $(libsqlite3.LIB) "$(install-dir.lib)" install-lib-0 install-lib-: -install-lib: install-lib-$(ENABLE_STATIC) +install-lib: install-lib-$(ENABLE_LIB_STATIC) install: install-lib # @@ -1798,7 +1835,7 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl \ # sqlite3_analyzer.flags.1 = -L. -lsqlite3 sqlite3_analyzer.flags.0 = $(LDFLAGS.libsqlite3) -sqlite3_analyzer.deps.1 = $(libsqlite3.SO) +sqlite3_analyzer.deps.1 = $(libsqlite3.DLL) sqlite3_analyzer.deps.0 = sqlite3_analyzer$(T.exe): $(T.tcl.env.sh) sqlite3_analyzer.c \ $(sqlite3_analyzer.deps.$(LINK_TOOLS_DYNAMICALLY)) @@ -2045,7 +2082,7 @@ install: install-shell-$(HAVE_WASI_SDK) # sqldiff.0.deps = $(TOP)/tool/sqldiff.c $(TOP)/ext/misc/sqlite3_stdio.h sqlite3.o sqlite3.h sqldiff.0.rules = $(T.link) -o $@ $(TOP)/tool/sqldiff.c sqlite3.o $(LDFLAGS.libsqlite3) -sqldiff.1.deps = $(TOP)/tool/sqldiff.c $(TOP)/ext/misc/sqlite3_stdio.h $(libsqlite3.SO) +sqldiff.1.deps = $(TOP)/tool/sqldiff.c $(TOP)/ext/misc/sqlite3_stdio.h $(libsqlite3.DLL) sqldiff.1.rules = $(T.link) -o $@ $(TOP)/tool/sqldiff.c -L. -lsqlite3 $(LDFLAGS.configure) sqldiff$(T.exe): $(sqldiff.$(LINK_TOOLS_DYNAMICALLY).deps) $(sqldiff.$(LINK_TOOLS_DYNAMICALLY).rules) @@ -2338,7 +2375,7 @@ tidy: tidy-. rm -f lemon$(B.exe) sqlite*.tar.gz rm -f mkkeywordhash$(B.exe) mksourceid$(B.exe) rm -f parse.* fts5parse.* - rm -f $(libsqlite3.SO) $(libsqlite3.LIB) $(libtclsqlite3.SO) $(libsqlite3.SO).a + rm -f $(libsqlite3.DLL) $(libsqlite3.LIB) $(libtclsqlite3.SO) libsqlite3$(T.dll).a rm -f tclsqlite3$(T.exe) $(TESTPROGS) rm -f LogEst$(T.exe) fts3view$(T.exe) rollback-test$(T.exe) showdb$(T.exe) rm -f showjournal$(T.exe) showstat4$(T.exe) showwal$(T.exe) speedtest1$(T.exe) diff --git a/manifest b/manifest index b7b274ec8..0af6475e3 100644 --- a/manifest +++ b/manifest @@ -1,26 +1,26 @@ -C Version\s3.49.1 -D 2025-02-18T13:38:58.435 +C Version\s3.49.2 +D 2025-05-07T10:39:52.886 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d -F Makefile.in 80be2e281d4647ac15a5bac15d5d20fc76d1cfb3f3f6dc01d1a26e3346a5042a +F Makefile.in d588d01af5f4f1e37468cd197b5614a64fabbab7be17b75728b75388f7e34f5e F Makefile.linux-generic bd3e3cacd369821a6241d4ea1967395c962dfe3057e38cb0a435cee0e8b789d0 F Makefile.msc a9b95ae9807e17f9b0734ebe97d68032141c3f95286bb64593cb73b206f043cf F README.md c3c0f19532ce28f6297a71870f3c7b424729f0e6d9ab889616d3587dd2332159 -F VERSION 652f6a5ba4cffa08baba6313486466378ac404381a2a173fa138879ced5619bc +F VERSION 5008c952d977ab40afea04fd86fa8ed8720ce13a0ea878bf948ab1b26c610d98 F art/icon-243x273.gif 9750b734f82fdb3dc43127753d5e6fbf3b62c9f4e136c2fbf573b2f57ea87af5 F art/icon-80x90.gif 65509ce3e5f86a9cd64fe7fca2d23954199f31fe44c1e09e208c80fb83d87031 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F art/sqlite370.svg 40b7e2fe8aac3add5d56dd86ab8d427a4eca5bcb3fe4f8946cb3794e1821d531 -F auto.def eddf6aef976e2c1a56c0accc3244945e0b22ec6799074c40be160e5a9a5662b0 +F auto.def a8c935b5c3c0b27c6a8b1b788bb47b06cc0ca3e9e92dc1b87e4b02659ba95ff6 F autoconf/Makefile.fallback 22fe523eb36dfce31e0f6349f782eb084e86a5620b2b0b4f84a2d6133f53f5ac -F autoconf/Makefile.in 844bc155e1c02075821530b2b4d996ab3d162e647352bb6e2895f3d0e64ad988 +F autoconf/Makefile.in ec1157e153ecad7c40bb8149250f10ef153dfa4514b93219e1342fe7caa6a533 F autoconf/Makefile.msc 0a071367537dc395285a5d624ac4f99f3a387b27cc5e89752423c0499e15aec4 F autoconf/README.first f1d3876e9a7852c22f275a6f06814e64934cecbc0b5b9617d64849094c1fd136 F autoconf/README.txt 7f01dc3915e2d68f329011073662369e62a0938a2c69398807823c57591cb288 -F autoconf/auto.def 3a318c4898024b35ed61a4876a42e3dcc313f93bd8486874d1ad498b88643d1a +F autoconf/auto.def 8d81c1d728d8462a9b6c1ca0714013bbb097aee0ae5e79309d7939cead98e295 F autoconf/tea/Makefile.in ba0556fee8da09c066bad85a4457904e46ee2c2eabaa309c0e83a78f2f151a8e F autoconf/tea/README.txt 61e62e519579e4a112791354d6d440f8b51ea6db3b0bab58d59f29df42d2dfe3 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43 @@ -38,19 +38,19 @@ F autoconf/tea/win/targets.vc 96a25a1fa6e9e9cfb348fd3760a5395b4ce8acafc8ed10f041 F autosetup/LICENSE 41a26aebdd2cd185d1e2b210f71b7ce234496979f6b35aef2cbf6b80cbed4ce4 F autosetup/README.autosetup a78ff8c4a3d2636a4268736672a74bf14a82f42687fcf0631a70c516075c031e F autosetup/README.md b306314e8a87ccf873cb5b2a360c4a27bbf841df5b76f3acbd65322cff165476 -F autosetup/autosetup df8b53928b1fe3c67db5bc77c8e1eb8160c1b6a26c370e9a06c68748f803b7e4 x +F autosetup/autosetup 74a9782b68d07934510190fbd03fc6ad92e63f0ea3b5cbffa5f0bd271ad60f01 x F autosetup/autosetup-config.guess dfa101c5e8220e864d5e9c72a85e87110df60260d36cb951ad0a85d6d9eaa463 x F autosetup/autosetup-config.sub a38fb074d0dece01cf919e9fb534a26011608aa8fa606490864295328526cd73 x -F autosetup/autosetup-find-tclsh 25905f6c302959db80c2951aa267b4411c5645b598ce761cfc24a166141e2c4c x +F autosetup/autosetup-find-tclsh b08f883f5753cfff1ecb8581f98b314e190b7e3f3059798e274ae5f5aad571af x F autosetup/autosetup-test-tclsh 749d20defee533a3842139df47d700fc7a334a5da7bdbd444ae5331744b06c5f F autosetup/cc-db.tcl 6e0ed90146197a5a05b245e649975c07c548e30926b218ca3e1d4dc034b10a7b F autosetup/cc-lib.tcl 493c5935b5dd3bf9bd4eca89b07c8b1b1a9356d61783035144e21795facf7360 F autosetup/cc-shared.tcl 4f024e94a47f427ba61de1739f6381ef0080210f9fae89112d5c1de1e5460d78 F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e45f -F autosetup/jimsh0.c 6573f6bc6ff204de0139692648d7037ca0b6c067bac83a7b4e087f20a86866a4 +F autosetup/jimsh0.c a57c16e65dcffc9c76e496757cb3f7fb47e01ecbd1631a0a5e01751fc856f049 F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba -F autosetup/proj.tcl 42119a2e899e319b92f3159952b7ef7219c82cb45eeb636aaf8d909def8150f8 -F autosetup/sqlite-config.tcl e1d65cc632e1de94ff39618ac82b649f2062f674ca36dae78ab3573cab096761 +F autosetup/proj.tcl 187d82550cfa55df00e285542e88278c51876d7813d63eaffb2fc5af40566d9f +F autosetup/sqlite-config.tcl 4b0205282099a1016bc9565b7b893eb9d4f739bd76acfcba74ad157d8e13acd7 F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9 F configure 9a00b21dfd13757bbfb8d89b30660a89ec1f8f3a79402b8f9f9b6fc475c3303a x F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad @@ -436,7 +436,7 @@ F ext/misc/regexp.c 388e7f237307c7dfbfb8dde44e097946f6c437801d63f0d7ad63f3320d4e F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 -F ext/misc/series.c cbdda2e2eb8159a1331974d246984c6e2693c6ea93930e6165046c8dbb8db0e9 +F ext/misc/series.c 2ef2f7452d63a8c59295d8503456576882e4de444c06f17f94c1a6c1ca9be6bb F ext/misc/sha1.c cb5002148c2661b5946f34561701e9105e9d339b713ec8ac057fd888b196dcb9 F ext/misc/shathree.c f3a778f27bf3e71b666a77f28e463a3b931c4dbe4219447e61bb678b4bc121c3 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 @@ -702,7 +702,7 @@ F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65a F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 F ext/wasm/wasmfs.make 68999f5bd8c489239592d59a420f8c627c99169bbd6fa16a404751f757b9f702 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk 44392e87994fd3595f64c8973784112b5ae570d9474deaab166218c600953f09 +F main.mk ae183c31bef504b89d1bbcc5b59a07e5eedfcc55bb543b4aa254808f4dd85149 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -722,7 +722,7 @@ F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 F src/btree.c 63ca6b647342e8cef643863cd0962a542f133e1069460725ba4461dcda92b03c F src/btree.h 18e5e7b2124c23426a283523e5f31a4bff029131b795bb82391f9d2f3136fc50 F src/btreeInt.h 98aadb6dcb77b012cab2574d6a728fad56b337fc946839b9898c4b4c969e30b6 -F src/build.c 357f98cdd9fe93f86e93ad3324fd224492480e48506c0c7db8ae3a94f3b5c5ee +F src/build.c 86a7efd263eb7d6a4dc24dda0981c55202b22f84765023d3ef4878d228bee5c6 F src/callback.c acae8c8dddda41ee85cfdf19b926eefe830f371069f8aadca3aa39adf5b1c859 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c d35723024b963edce9c0fad5b3303e8bb9266083784844baed10a6dedfe26f3b @@ -730,7 +730,7 @@ F src/date.c 842c08ac143a56a627b05ac51d68624f2b7b03e3b4cba596205e735eed64ee57 F src/dbpage.c 2e677acb658a29965e55398bbc61161cb7819da538057c8032adac7ab8e4a8c0 F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c F src/delete.c 03a77ba20e54f0f42ebd8eddf15411ed6bdb06a2c472ac4b6b336521bf7cea42 -F src/expr.c 8705be31ee713aaa43c97d91399db09f16ee41b88250406eb99de6b47f550a98 +F src/expr.c 3fc2f37dbc172293239600cc6f041830e7e2bc3ddaaebfbb2a89520317b4a09f F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f F src/func.c 1ab83fd94f97af9797bdf1027169e3e19482fce06b090c3acceb4bf92ae452cd @@ -774,18 +774,18 @@ F src/pcache.c 588cc3c5ccaaadde689ed35ce5c5c891a1f7b1f4d1f56f6cf0143b74d8ee6484 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 49516ad7718a3626f28f710fa7448ef1fce3c07fd169acbb4817341950264319 F src/pragma.c ce1182217aa540e034c6da2f17515e3706bf52c837e8222361be9ccd7a9d495a -F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 +F src/pragma.h 5edad5943dba8ec69727e719fb841b6b505204b9cdbfc50142806f79458fcce6 F src/prepare.c 1832be043fce7d489959aae6f994c452d023914714c4d5457beaed51c0f3d126 F src/printf.c 96f7f8baeedc7639da94e4e7a4a2c200e2537c4eec9e5e1c2ffc821f40eb3105 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c c8a5372b97b2a2e972a280676f06ddb5b74e885d3b1f5ce383f839907b57ef68 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c 83e88fbb36f89b6703b348777491619554f0fd6f917c9fdf51e4c2e9cda6c04e +F src/select.c c94f7fe33a3f481cc68472ba845301e343e0412c216fee06f58ad64e7e88bf45 F src/shell.c.in b377a59822f207106424f08aead37e78b609222e98f86f04cc8a03563ccf3237 F src/sqlite.h.in d2902f13ace94d3d3609646bd6d12a2d7a4f6cbdf6a5a4097580ac305f54c3f0 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 9d7052c71f46ca3a1a4880ba4a62590da334651139a38327d488894d9c883f3b +F src/sqliteInt.h aeb5ed59db92bfa2bf987366769a48681735c9da58e88e76f1f47d1fc26a2e46 F src/sqliteLimit.h 1bbdbf72bd0411d003267ffebc59a262f061df5653027a75627d03f48ca30523 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -849,7 +849,7 @@ F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1 F src/utf.c 8b29d9a5956569ea2700f869669b8ef67a9662ee5e724ff77ab3c387e27094ba F src/util.c e5f6a5eeaa26b69054a43bbd0048cfe3d2851f6961052b35aed8f695df922850 F src/vacuum.c b763b6457bd058d2072ef9364832351fd8d11e8abf70cbb349657360f7d55c40 -F src/vdbe.c b428a751953c0c2ff85e3e152ec16e29d488895cd541c8c20876ff9f3bf6978a +F src/vdbe.c 3c9e4dacc5db859cfe61ed364023f0889ca082061cbb064f44e772d274006ec2 F src/vdbe.h 3d26d5c7660c5c7bd33ffb0d8784615072d8b23c81f8110870efe2631136bc89 F src/vdbeInt.h 895b1ab7536f018d3d70d690f6c0adbd1062b6dddce1c2cad912927856d4033c F src/vdbeapi.c 82fe278a7c71b653235c6f9fb5de0b5de589908dfcb011ba2a782e8becf06f86 @@ -864,7 +864,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 4e6181d8780ab0af2e1388d0754cbe6f2f04593d2b1ab6c41699a89942fd8997 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 -F src/where.c 09dc313e7223ca1217c39c7026b00f16ff449a8323511a762fcba7863a00f4cd +F src/where.c 8bf66eb9911356da811fd5bcbbd42dbc53c9381e04eb7ed32959264e24cf8082 F src/whereInt.h d20cddddb1d61b18d5cb1fcfa9b77fbeebbc4afe44d996e603452a23b3009ee1 F src/wherecode.c 0c3d3199a2b769a5e2bb70feb5003dc85b3d86842ecaf903a47f2b4205ca5dab F src/whereexpr.c 2415c8eee5ff89a8b709d7d83d71c1ff986cd720d0520057e1d8a5371339012a @@ -972,7 +972,7 @@ F test/bind2.test 918bc35135f4141809ead7585909cde57d44db90a7a62aef540127148f91aa F test/bindxfer.test efecd12c580c14df5f4ad3b3e83c667744a4f7e0 F test/bitvec.test 75894a880520164d73b1305c1c3f96882615e142 F test/blob.test e7ac6c7d3a985cc4678c64f325292529a69ae252 -F test/bloom1.test cf613a27054bbaf61c5bfc440a5cfd3ff76798d0695f3fc5e5d1bbc819b8dab1 +F test/bloom1.test 04f3a17df8912bfdc292c41b59d79f93893fe69799f3089a64451f9112f9658f F test/boundary1.tcl 6421b2d920d8b09539503a8673339d32f7609eb1 F test/boundary1.test 66d7f4706ccdb42d58eafdb081de07b0eb42d77b F test/boundary2.tcl e34ef4e930cf1083150d4d2c603e146bd3b76bcb @@ -1572,7 +1572,7 @@ F test/round1.test 29c3c9039936ed024d672f003c4d35ee11c14c0acb75c5f7d6188ff16190c F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test d27191b5ce794c05bf61081e8b2c546a1844c1641321dcaf7fb785234256cc8e -F test/rowvalue.test baf4fa3ec1a8c1c920c3faa5fd25959cb454bbd99ac8960397c34549d9fc4abe +F test/rowvalue.test 0b023643162dfe3036780d78eb8daa52b144b6c2dee1216967cc72eafe19aa3d F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b F test/rowvalue3.test 1347e25ca11c547c5a6ff0cc5626f95aa9740e9275bfaec096029f57cb2130ce F test/rowvalue4.test bac9326d1e886656650f67c0ec484eb5f452244a8209c6af508e9a862ace08ed @@ -1638,7 +1638,7 @@ F test/sharedB.test 1a84863d7a2204e0d42f2e1606577c5e92e4473fa37ea0f5bdf829e4bf8e F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 F test/shell1.test 5d84e415adf7cc4edd5913c4f23c761104ff135b9c190fcf7b430a4cbca6cb65 -F test/shell2.test 01a01f76ed98088ce598794fbf5b359e148271541a8ddbf79d21cc353cc67a24 +F test/shell2.test ac102ebc0a9ec166257600c4ee8bdefec242163afced295f10b004f4af3fc9dd F test/shell3.test db1953a8e59d08e9240b7cc5948878e184f7eb2623591587f8fd1f1a5bd536d8 F test/shell4.test 522fdc628c55eff697b061504fb0a9e4e6dfc5d9087a633ab0f3dd11bcc4f807 F test/shell5.test 0e5f8ce08206b9998a778cfe1989e20e47839153c05af2da29198150172e22fc @@ -1715,7 +1715,7 @@ F test/sync.test 89539f4973c010eda5638407e71ca7fddbcd8e0594f4c9980229f804d433309 F test/sync2.test 8f9f7d4f6d5be8ca8941a8dadcc4299e558cb6a1ff653a9469146c7a76ef2039 F test/syscall.test a067468b43b8cb2305e9f9fe414e5f40c875bb5d2cba5f00b8154396e95fcf37 F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 -F test/tabfunc01.test 7be82bd50c7ede7f01b2dd17cd1b84f352c516078222d0b067d858f081e3f9a7 +F test/tabfunc01.test e85679a3800aa632dee787966b8482fce0bd47629dad82f102fd52f319d2c281 F test/table.test 7862a00b58b5541511a26757ea9c5c7c3f8298766e98aa099deec703d9c0a8e0 F test/tableapi.test ecbcc29c4ab62c1912c3717c48ea5c5e59f7d64e4a91034e6148bd2b82f177f4 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 @@ -2152,13 +2152,13 @@ F tool/mkmsvcmin.tcl d76c45efda1cce2d4005bcea7b8a22bb752e3256009f331120fb4fecb14 F tool/mkopcodec.tcl 33d20791e191df43209b77d37f0ff0904620b28465cca6990cf8d60da61a07ef F tool/mkopcodeh.tcl 2b4e6967a670ef21bf53a164964c35c6163277d002a4c6f56fa231d68c88d023 F tool/mkopts.tcl 680f785fdb09729fd9ac50632413da4eadbdf9071535e3f26d03795828ab07fa -F tool/mkpragmatab.tcl 32e359ccb21011958a821955254bd7a5fa7915d01a8c16fed91ffc8b40cb4adf +F tool/mkpragmatab.tcl f9339d207de40cb57e8dc7a53817dac8e63a4d4f38a7b815e62cd67f68be70d8 F tool/mkshellc.tcl 9ce74de0fa904a2c56a96f8d8b5261246bacb0eaa8d7e184f9e18ff94145ebbc F tool/mksourceid.c 36aa8020014aed0836fd13c51d6dc9219b0df1761d6b5f58ff5b616211b079b9 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 351c55256213154cabb051a3c870ef9f4487de905015141ae50dc7578a901b84 F tool/mksqlite3c.tcl ba13086555b3cb835eba5e47a9250300ab85304d23fd1081abd3f29d8ab71a2b -F tool/mksqlite3h.tcl b05b85c32295bad3fe64807729693d1f19faed3c464c5faac6c53bb6b972ac2f +F tool/mksqlite3h.tcl 989948c6a26e188e673d7c2f2f093ea3acd816ad6ac65bab596280075c8f3a45 F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b F tool/mksrczip.tcl 81efd9974dbb36005383f2cd655520057a2ae5aa85ac2441a80c7c28f803ac52 F tool/mktoolzip.tcl 34b4e92be544f820e2cc26f143f7d5aec511e826ec394cc82969a5dcf7c7a27c @@ -2209,10 +2209,10 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 57eceb361f0290b1326acf791565eb33267a10a5201aee34bdd2d1c93d647ff8 -R 75ce877abe8a04df5ba4cb86e46a277d +P 9d1f01aac909a0689ab881e269c3f1e4b583b0b135689a39fd2822de7a059e5f +R 95cfc65226e423aa18b9e0671f876db9 T +sym-release * -T +sym-version-3.49.1 * +T +sym-version-3.49.2 * U drh -Z ca30243dceef882d9114cf681397efb8 +Z 26aa19dada18db86c2026c3614aef5cb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f6dad3ea6..bbf46906f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -873d4e274b4988d260ba8354a9718324a1c26187a4ab4c1cc0227c03d0f10e70 +17144570b0d96ae63cd6f3edca39e27ebd74925252bbaf6723bcb2f6b4861fb1 diff --git a/src/build.c b/src/build.c index cc29610e0..4e5c7117d 100644 --- a/src/build.c +++ b/src/build.c @@ -4191,6 +4191,7 @@ void sqlite3CreateIndex( assert( j<=0x7fff ); if( j<0 ){ j = pTab->iPKey; + pIndex->bIdxRowid = 1; }else{ if( pTab->aCol[j].notNull==0 ){ pIndex->uniqNotNull = 0; diff --git a/src/expr.c b/src/expr.c index 8f898a1e3..3d0e6db97 100644 --- a/src/expr.c +++ b/src/expr.c @@ -5936,11 +5936,11 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ assert( TK_ISNULL==OP_IsNull ); testcase( op==TK_ISNULL ); assert( TK_NOTNULL==OP_NotNull ); testcase( op==TK_NOTNULL ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - sqlite3VdbeTypeofColumn(v, r1); + assert( regFree1==0 || regFree1==r1 ); + if( regFree1 ) sqlite3VdbeTypeofColumn(v, r1); sqlite3VdbeAddOp2(v, op, r1, dest); VdbeCoverageIf(v, op==TK_ISNULL); VdbeCoverageIf(v, op==TK_NOTNULL); - testcase( regFree1==0 ); break; } case TK_BETWEEN: { @@ -6111,11 +6111,11 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ case TK_ISNULL: case TK_NOTNULL: { r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - sqlite3VdbeTypeofColumn(v, r1); + assert( regFree1==0 || regFree1==r1 ); + if( regFree1 ) sqlite3VdbeTypeofColumn(v, r1); sqlite3VdbeAddOp2(v, op, r1, dest); testcase( op==TK_ISNULL ); VdbeCoverageIf(v, op==TK_ISNULL); testcase( op==TK_NOTNULL ); VdbeCoverageIf(v, op==TK_NOTNULL); - testcase( regFree1==0 ); break; } case TK_BETWEEN: { diff --git a/src/pragma.h b/src/pragma.h index 7270db1db..56a469b9f 100644 --- a/src/pragma.h +++ b/src/pragma.h @@ -83,48 +83,48 @@ static const char *const pragCName[] = { /* 13 */ "pk", /* 14 */ "hidden", /* table_info reuses 8 */ - /* 15 */ "schema", /* Used by: table_list */ - /* 16 */ "name", + /* 15 */ "name", /* Used by: function_list */ + /* 16 */ "builtin", /* 17 */ "type", - /* 18 */ "ncol", - /* 19 */ "wr", - /* 20 */ "strict", - /* 21 */ "seqno", /* Used by: index_xinfo */ - /* 22 */ "cid", - /* 23 */ "name", - /* 24 */ "desc", - /* 25 */ "coll", - /* 26 */ "key", - /* 27 */ "name", /* Used by: function_list */ - /* 28 */ "builtin", - /* 29 */ "type", - /* 30 */ "enc", - /* 31 */ "narg", - /* 32 */ "flags", - /* 33 */ "tbl", /* Used by: stats */ - /* 34 */ "idx", - /* 35 */ "wdth", - /* 36 */ "hght", - /* 37 */ "flgs", - /* 38 */ "seq", /* Used by: index_list */ - /* 39 */ "name", - /* 40 */ "unique", - /* 41 */ "origin", - /* 42 */ "partial", + /* 18 */ "enc", + /* 19 */ "narg", + /* 20 */ "flags", + /* 21 */ "schema", /* Used by: table_list */ + /* 22 */ "name", + /* 23 */ "type", + /* 24 */ "ncol", + /* 25 */ "wr", + /* 26 */ "strict", + /* 27 */ "seqno", /* Used by: index_xinfo */ + /* 28 */ "cid", + /* 29 */ "name", + /* 30 */ "desc", + /* 31 */ "coll", + /* 32 */ "key", + /* 33 */ "seq", /* Used by: index_list */ + /* 34 */ "name", + /* 35 */ "unique", + /* 36 */ "origin", + /* 37 */ "partial", + /* 38 */ "tbl", /* Used by: stats */ + /* 39 */ "idx", + /* 40 */ "wdth", + /* 41 */ "hght", + /* 42 */ "flgs", /* 43 */ "table", /* Used by: foreign_key_check */ /* 44 */ "rowid", /* 45 */ "parent", /* 46 */ "fkid", - /* index_info reuses 21 */ - /* 47 */ "seq", /* Used by: database_list */ - /* 48 */ "name", - /* 49 */ "file", - /* 50 */ "busy", /* Used by: wal_checkpoint */ - /* 51 */ "log", - /* 52 */ "checkpointed", - /* collation_list reuses 38 */ + /* 47 */ "busy", /* Used by: wal_checkpoint */ + /* 48 */ "log", + /* 49 */ "checkpointed", + /* 50 */ "seq", /* Used by: database_list */ + /* 51 */ "name", + /* 52 */ "file", + /* index_info reuses 27 */ /* 53 */ "database", /* Used by: lock_status */ /* 54 */ "status", + /* collation_list reuses 33 */ /* 55 */ "cache_size", /* Used by: default_cache_size */ /* module_list pragma_list reuses 9 */ /* 56 */ "timeout", /* Used by: busy_timeout */ @@ -217,7 +217,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "collation_list", /* ePragTyp: */ PragTyp_COLLATION_LIST, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 38, 2, + /* ColNames: */ 33, 2, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_COMPILEOPTION_DIAGS) @@ -252,7 +252,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "database_list", /* ePragTyp: */ PragTyp_DATABASE_LIST, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 47, 3, + /* ColNames: */ 50, 3, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) @@ -332,7 +332,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "function_list", /* ePragTyp: */ PragTyp_FUNCTION_LIST, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 27, 6, + /* ColNames: */ 15, 6, /* iArg: */ 0 }, #endif #endif @@ -361,17 +361,17 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "index_info", /* ePragTyp: */ PragTyp_INDEX_INFO, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 21, 3, + /* ColNames: */ 27, 3, /* iArg: */ 0 }, {/* zName: */ "index_list", /* ePragTyp: */ PragTyp_INDEX_LIST, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 38, 5, + /* ColNames: */ 33, 5, /* iArg: */ 0 }, {/* zName: */ "index_xinfo", /* ePragTyp: */ PragTyp_INDEX_INFO, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 21, 6, + /* ColNames: */ 27, 6, /* iArg: */ 1 }, #endif #if !defined(SQLITE_OMIT_INTEGRITY_CHECK) @@ -550,7 +550,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "stats", /* ePragTyp: */ PragTyp_STATS, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, - /* ColNames: */ 33, 5, + /* ColNames: */ 38, 5, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) @@ -569,7 +569,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "table_list", /* ePragTyp: */ PragTyp_TABLE_LIST, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1, - /* ColNames: */ 15, 6, + /* ColNames: */ 21, 6, /* iArg: */ 0 }, {/* zName: */ "table_xinfo", /* ePragTyp: */ PragTyp_TABLE_INFO, @@ -646,7 +646,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "wal_checkpoint", /* ePragTyp: */ PragTyp_WAL_CHECKPOINT, /* ePragFlg: */ PragFlg_NeedSchema, - /* ColNames: */ 50, 3, + /* ColNames: */ 47, 3, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) diff --git a/src/select.c b/src/select.c index cf25c8e67..2f11297bf 100644 --- a/src/select.c +++ b/src/select.c @@ -3224,6 +3224,7 @@ static int multiSelect( multi_select_end: pDest->iSdst = dest.iSdst; pDest->nSdst = dest.nSdst; + pDest->iSDParm2 = dest.iSDParm2; if( pDelete ){ sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pDelete); } @@ -7178,6 +7179,7 @@ static void agginfoFree(sqlite3 *db, void *pArg){ ** * There is no WHERE or GROUP BY or HAVING clauses on the subqueries ** * The outer query is a simple count(*) with no WHERE clause or other ** extraneous syntax. +** * None of the subqueries are DISTINCT (forumpost/a860f5fb2e 2025-03-10) ** ** Return TRUE if the optimization is undertaken. */ @@ -7210,7 +7212,11 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ if( pSub->op!=TK_ALL && pSub->pPrior ) return 0; /* Must be UNION ALL */ if( pSub->pWhere ) return 0; /* No WHERE clause */ if( pSub->pLimit ) return 0; /* No LIMIT clause */ - if( pSub->selFlags & SF_Aggregate ) return 0; /* Not an aggregate */ + if( pSub->selFlags & (SF_Aggregate|SF_Distinct) ){ + testcase( pSub->selFlags & SF_Aggregate ); + testcase( pSub->selFlags & SF_Distinct ); + return 0; /* Not an aggregate nor DISTINCT */ + } assert( pSub->pHaving==0 ); /* Due to the previous */ pSub = pSub->pPrior; /* Repeat over compound */ }while( pSub ); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f47009c15..fd4ed5298 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2757,6 +2757,7 @@ struct Index { unsigned bLowQual:1; /* sqlite_stat1 says this is a low-quality index */ unsigned bNoQuery:1; /* Do not use this index to optimize queries */ unsigned bAscKeyBug:1; /* True if the bba7b69f9849b5bf bug applies */ + unsigned bIdxRowid:1; /* One or more of the index keys is the ROWID */ unsigned bHasVCol:1; /* Index references one or more VIRTUAL columns */ unsigned bHasExpr:1; /* Index contains an expression, either a literal ** expression, or a reference to a VIRTUAL column */ diff --git a/src/vdbe.c b/src/vdbe.c index d41ac8d51..3ebe7c049 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3705,6 +3705,7 @@ case OP_MakeRecord: { zHdr += sqlite3PutVarint(zHdr, serial_type); if( pRec->n ){ assert( pRec->z!=0 ); + assert( pRec->z!=(const char*)sqlite3CtypeMap ); memcpy(zPayload, pRec->z, pRec->n); zPayload += pRec->n; } diff --git a/src/where.c b/src/where.c index 5cb52b8ad..ff269750f 100644 --- a/src/where.c +++ b/src/where.c @@ -3469,7 +3469,7 @@ static int whereLoopAddBtreeIndex( if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->u.btree.nEqnColumn && (pNew->u.btree.nEqnKeyCol || - pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY) + (pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY && !pProbe->bIdxRowid)) ){ if( pNew->u.btree.nEq>3 ){ sqlite3ProgressCheck(pParse); diff --git a/test/bloom1.test b/test/bloom1.test index 151f364ae..f8efcc184 100644 --- a/test/bloom1.test +++ b/test/bloom1.test @@ -183,6 +183,47 @@ do_execsql_test 4.3 { do_execsql_test 4.4 { SELECT * FROM t0 LEFT JOIN t1 LEFT JOIN t2 ON (b NOTNULL)==(c IN ()) WHERE c; } {xyz {} 7.0} + +reset_db +do_execsql_test 5.0 { + CREATE TABLE t1 (c1); + INSERT INTO t1 VALUES (101); + CREATE TABLE t2 ( x ); + INSERT INTO t2 VALUES(404); +} + +do_execsql_test 5.1 { + SELECT 'val' in ( + select 'val' from ( select 'valueB' from t1 order by 1 ) + union all + select 'val' + ); +} {1} + +do_execsql_test 5.2 { + select * from t2 + where 'val' in ( + select 'val' from ( select 'valueB' from t1 order by 1 ) + union all + select 'val' + ); +} {404} + +do_execsql_test 5.3 { + SELECT subq_1.c_0 as c_0 + FROM ( SELECT 0 as c_0) as subq_1 + WHERE (subq_1.c_0) IN ( + SELECT subq_2.c_0 as c_0 + FROM ( + SELECT 0 as c_0 + FROM t1 as ref_1 + WHERE (ref_1.c1) = (2) + ORDER BY c_0 desc + ) as subq_2 + UNION ALL + SELECT 0 as c_0 + ); +} {0} finish_test diff --git a/test/rowvalue.test b/test/rowvalue.test index 59b44d938..007e27224 100644 --- a/test/rowvalue.test +++ b/test/rowvalue.test @@ -782,4 +782,27 @@ do_execsql_test 33.3 { SELECT * FROM t1 WHERE (a,b) BETWEEN (2,99) AND (4,0); } {3 100} +# 2025-04-15 https://sqlite.org/forum/forumpost/b9647a113b465950 +# Incorrect result when the schema includes a table with a UNIQUE +# constraint and one of the columns in the UNIQUE constraint is the +# INTEGER PRIMARY KEY, and the columns that UNIQUE constraint are +# used in a rowvalue-IN operator constraint. +# +reset_db +do_execsql_test 34.1 { + CREATE TABLE items ( + Id INTEGER /* rowid alias */, + Item INTEGER /* any type */, + Test TEXT /* TEXT or BLOB */, + Filler, /* any type */ + PRIMARY KEY(Id), + UNIQUE(Item, Id) + ); + INSERT INTO items (Id, Item) + VALUES (1, 2), (2, 2), (3, 3), (4, 5); + UPDATE items SET test='ok' + WHERE (Id, Item) IN (SELECT Id, Item FROM items); + SELECT Id, Item, test FROM items ORDER BY id; +} {1 2 ok 2 2 ok 3 3 ok 4 5 ok} + finish_test diff --git a/test/shell2.test b/test/shell2.test index ee5ae4bdd..3f9fec9ef 100644 --- a/test/shell2.test +++ b/test/shell2.test @@ -224,24 +224,24 @@ do_test shell2-1.4.10 { set res [catchcmd :memory: [string trim { SELECT * FROM generate_series(9223372036854775807,9223372036854775807,1); SELECT * FROM generate_series(9223372036854775807,9223372036854775807,-1); - SELECT avg(rowid),min(value),max(value) FROM generate_series( + SELECT avg(value),min(value),max(value) FROM generate_series( -9223372036854775808,9223372036854775807,1085102592571150095); SELECT * FROM generate_series(-9223372036854775808,9223372036854775807, 9223372036854775807); - SELECT value,rowid FROM generate_series(-4611686018427387904, + SELECT value FROM generate_series(-4611686018427387904, 4611686018427387904, 4611686018427387904) ORDER BY value DESC; SELECT * FROM generate_series(0,-2,-1); SELECT * FROM generate_series(0,-2); SELECT * FROM generate_series(0,2) LIMIT 3;}]] } {0 {9223372036854775807 9223372036854775807 -9.5|-9223372036854775808|9223372036854775807 +-0.5|-9223372036854775808|9223372036854775807 -9223372036854775808 -1 9223372036854775806 -4611686018427387904|3 -0|2 --4611686018427387904|1 +4611686018427387904 +0 +-4611686018427387904 0 -1 -2 diff --git a/test/tabfunc01.test b/test/tabfunc01.test index b6797171e..5938ec6cb 100644 --- a/test/tabfunc01.test +++ b/test/tabfunc01.test @@ -61,10 +61,10 @@ do_execsql_test tabfunc01-1.8 { } {30 25 20 15 10 5 0} do_execsql_test tabfunc01-1.9 { SELECT rowid, * FROM generate_series(0,32,5) ORDER BY value DESC; -} {7 30 6 25 5 20 4 15 3 10 2 5 1 0} +} {30 30 25 25 20 20 15 15 10 10 5 5 0 0} do_execsql_test tabfunc01-1.10 { SELECT rowid, * FROM generate_series(0,32,5) ORDER BY +value DESC; -} {7 30 6 25 5 20 4 15 3 10 2 5 1 0} +} {30 30 25 25 20 20 15 15 10 10 5 5 0 0} do_execsql_test tabfunc01-1.20 { CREATE VIEW v1(a,b) AS VALUES(1,2),(3,4); @@ -181,6 +181,76 @@ do_execsql_test tabfunc01-4.4 { SELECT * FROM (generate_series(1,5,2)) AS x LIMIT 10; } {1 3 5} +# 2025-03-13 forum post bf2dc8e909983511 +# +do_execsql_test tabfunc01-5.1 { + SELECT value + FROM generate_series(60,73,6) + WHERE value=66; +} 66 +do_execsql_test tabfunc01-5.2 { + SELECT value + FROM generate_series(73,60,-6) + WHERE value=67; +} 67 + +# 2025-03-22 forum post 0d5d63257e3ff4f6 +# +do_execsql_test tabfunc01-6.1 { + SELECT value FROM generate_series(1,10) WHERE value<5.5; +} {1 2 3 4 5} +do_execsql_test tabfunc01-6.2 { + SELECT value FROM generate_series(1,10) WHERE value<5.0; +} {1 2 3 4} +do_execsql_test tabfunc01-6.3 { + SELECT value FROM generate_series(1,10) WHERE value<=5.5; +} {1 2 3 4 5} +do_execsql_test tabfunc01-6.4 { + SELECT value FROM generate_series(1,10) WHERE value<=5.0; +} {1 2 3 4 5} +do_execsql_test tabfunc01-6.5 { + SELECT value FROM generate_series(1,10) WHERE value>5.5; +} {6 7 8 9 10} +do_execsql_test tabfunc01-6.6 { + SELECT value FROM generate_series(1,10) WHERE value>5.0; +} {6 7 8 9 10} +do_execsql_test tabfunc01-6.7 { + SELECT value FROM generate_series(1,10) WHERE value>=5.5; +} {6 7 8 9 10} +do_execsql_test tabfunc01-6.8 { + SELECT value FROM generate_series(1,10) WHERE value>=5.0; +} {5 6 7 8 9 10} +do_execsql_test tabfunc01-6.9 { + SELECT value FROM generate_series(10,1,-1) WHERE value<5.5; +} {5 4 3 2 1} +do_execsql_test tabfunc01-6.10 { + SELECT value FROM generate_series(10,1,-1) WHERE value<5.0; +} {4 3 2 1} +do_execsql_test tabfunc01-6.11 { + SELECT value FROM generate_series(10,1,-1) WHERE value<=5.5; +} {5 4 3 2 1} +do_execsql_test tabfunc01-6.12 { + SELECT value FROM generate_series(10,1,-1) WHERE value<=5.0; +} {5 4 3 2 1} +do_execsql_test tabfunc01-6.13 { + SELECT value FROM generate_series(10,1,-1) WHERE value>5.5; +} {10 9 8 7 6} +do_execsql_test tabfunc01-6.14 { + SELECT value FROM generate_series(10,1,-1) WHERE value>5.0; +} {10 9 8 7 6} +do_execsql_test tabfunc01-6.15 { + SELECT value FROM generate_series(10,1,-1) WHERE value>=5.5; +} {10 9 8 7 6} +do_execsql_test tabfunc01-6.16 { + SELECT value FROM generate_series(10,1,-1) WHERE value>=5.0; +} {10 9 8 7 6 5} +do_execsql_test tabfunc01-6.17 { + SELECT value FROM generate_series(1,10) WHERE value==5.5; +} {} +do_execsql_test tabfunc01-6.18 { + SELECT value FROM generate_series(1,10) WHERE value==5.0; +} {5} + # The next series of tests is verifying that virtual table are able # to optimize the IN operator, even on terms that are not marked "omit". # When the generate_series virtual table is compiled for the testfixture, @@ -370,7 +440,18 @@ do_execsql_test 1100 { where (ref_3.value) in (select 1); } {1} - +# 2025-03-18 /forumpost/1e17219c88 +# The generate_series() table-valued function is modified so that its +# rowid is always its value. That way it can be used on the RHS of a +# RIGHT JOIN. +# +do_execsql_test 1200 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(value INT); + INSERT INTO t1 VALUES (1),(2),(3); + SELECT t1.value, t2.value + FROM t1 RIGHT JOIN generate_series(1,3,1) AS t2 USING(value); +} {1 1 2 2 3 3} # Free up memory allocations intarray_addr diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index 2b5b6fdcd..8a0b5c6fa 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -528,10 +528,13 @@ foreach f [lsort [array names allflags]] { set fv [expr {$fv*2}] } -# Sort the column lists so that longer column lists occur first +# Sort the column lists so that longer column lists occur first. +# In the event of a tie, sort column lists lexicographically. # proc colscmp {a b} { - return [expr {[llength $b] - [llength $a]}] + set rc [expr {[llength $b] - [llength $a]}] + if {$rc} {return $rc} + return [string compare $a $b] } set cols_list [lsort -command colscmp $cols_list] diff --git a/tool/mksqlite3h.tcl b/tool/mksqlite3h.tcl index b1d5ecdcd..6bbfa8c8b 100644 --- a/tool/mksqlite3h.tcl +++ b/tool/mksqlite3h.tcl @@ -82,7 +82,13 @@ set nVersion [eval format "%d%03d%03d" [split $zVersion .]] set PWD [pwd] cd $TOP set tmpfile $PWD/tmp-[clock millisec]-[expr {int(rand()*100000000000)}].txt -exec $PWD/mksourceid manifest > $tmpfile +set mksourceid $PWD/mksourceid +if {![file exists $mksourceid] && [file exists ${mksourceid}.exe]} { + # Workaround for Windows-based Unix-like environments + # https://sqlite.org/forum/forumpost/41ba710dd9943453 + set mksourceid ${mksourceid}.exe +} +exec $mksourceid manifest > $tmpfile set fd [open $tmpfile rb] set zSourceId [string trim [read $fd]] close $fd From c8ce6f052d38af2da96d6cc2ec9cd96aa4d59f3f Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 9 May 2025 12:45:40 -0400 Subject: [PATCH 15/75] Removes use of static mutex in sqlcipher_extra_shutdown() The sqlcipher_extra_shutdown() function is called by sqlite3_shutdown() to cleanup internal SQLCipher resources. However, many applications do not call sqlite3_shutdown(). In order to ensure that SQLCipher resources are cleaned up even if sqlite3_shutdown() is omitted, sqlcipher_extra_shutdown() is also called from atexit and from a library finalizer (fini/DllMain). Previously, sqlcipher_extra_shutdown() internally locked a global static mutex prior to clean up. However, this introduced an edge case where, if the library was compiled with SQLITE_OMIT_AUTOINIT, and sqlite_shutdown() was called explicitly, a subsequent call to sqlcipher_extra_shutdown() from atexit or the finalizer could reallocate a new mutex that would never be freed. This change removes the use of the mutex from sqlcipher_extra_shutdown() entirely. The SQLite documentation makes it clear that sqlite3_shutdown() is NOT threadsafe (https://www.sqlite.org/c3ref/initialize.html) so an application must already guarantee that is called from a single thread. Other invocations of sqlcipher_extra_shutdown will also be called in a single-threaded context. As a result sqlcipher_extra_shutdown() should not need to make use of a mutext internally, and its removal solves the previous edge case problem. --- src/sqlcipher.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 2a51f01dc..bd346e5b3 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -589,10 +589,6 @@ int sqlcipher_extra_init(const char* arg) { void sqlcipher_extra_shutdown(void) { int i = 0; sqlcipher_provider *provider = NULL; - sqlite3_mutex *mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER); - if(mutex) { - sqlite3_mutex_enter(mutex); - } /* if sqlcipher hasn't been initialized or the shutdown already completed exit early */ if(!sqlcipher_init || sqlcipher_shutdown) { @@ -663,9 +659,6 @@ void sqlcipher_extra_shutdown(void) { sqlcipher_init = 0; sqlcipher_init_error = SQLITE_ERROR; sqlcipher_shutdown = 1; - if(mutex) { - sqlite3_mutex_leave(mutex); - } } static void sqlcipher_shield(unsigned char *in, int sz) { From c7e811b399379c948b423872ad7ba91d2ce38434 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 9 May 2025 13:16:51 -0400 Subject: [PATCH 16/75] Updates CHANGELOG for 4.9.0 changes --- CHANGELOG.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5956fe7e2..357d62bd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,10 @@ Notable changes to this project are documented in this file. ## [unreleased] - (? 2025 - [unreleased changes]) -## [4.9.0] - (? 2025 - [4.9.0 changes]) - +## [4.9.0] - (May 2025 - [4.9.0 changes]) +- Updates baseline to upstream SQLite 3.46.2 +- Removes use of static mutex in `sqlcipher_extra_shutdown()` + ## [4.8.0] - (April 2025 - [4.8.0 changes]) - Fixes regression in `PRAGMA cipher_migrate` where an error would be thrown when migrating a current-version database - Adds selective locking in critical sections of the library for shared cache connections (Note: use of shared cache is still strongly discouraged) From b9acbb312c9731c7f72d3390087ecf435673ad51 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Tue, 20 May 2025 10:24:18 -0400 Subject: [PATCH 17/75] Bumps version to 4.10.0 --- CHANGELOG.md | 6 +++--- SQLCipher.podspec.json | 4 ++-- src/sqlcipher.c | 2 +- test/sqlcipher-pragmas.test | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 357d62bd5..637e814bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # SQLCipher Change Log Notable changes to this project are documented in this file. -## [unreleased] - (? 2025 - [unreleased changes]) +## [4.10.0] - (? 2025 - [4.10.0 changes]) ## [4.9.0] - (May 2025 - [4.9.0 changes]) - Updates baseline to upstream SQLite 3.46.2 @@ -290,8 +290,8 @@ __BREAKING CHANGE__: `SELECT` statements (now also including schema independent ### Security - Change KDF iteration length from 4,000 to 64,000 -[unreleased]: https://github.com/sqlcipher/sqlcipher/tree/prerelease -[unreleased changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.8.0...prerelease +[4.10.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.10.0 +[4.10.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.9.0...v4.10.0 [4.9.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.9.0 [4.9.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.8.0...v4.9.0 [4.8.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.8.0 diff --git a/SQLCipher.podspec.json b/SQLCipher.podspec.json index 05c9daf8c..343ca965b 100644 --- a/SQLCipher.podspec.json +++ b/SQLCipher.podspec.json @@ -18,10 +18,10 @@ "requires_arc": false, "source": { "git": "https://github.com/sqlcipher/sqlcipher.git", - "tag": "v4.9.0" + "tag": "v4.10.0" }, "summary": "Full Database Encryption for SQLite.", - "version": "4.9.0", + "version": "4.10.0", "subspecs": [ { "compiler_flags": [ diff --git a/src/sqlcipher.c b/src/sqlcipher.c index bd346e5b3..7931a583b 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -99,7 +99,7 @@ void sqlite3pager_reset(Pager *pPager); #define CIPHER_STR(s) #s #ifndef CIPHER_VERSION_NUMBER -#define CIPHER_VERSION_NUMBER 4.9.0 +#define CIPHER_VERSION_NUMBER 4.10.0 #endif #ifndef CIPHER_VERSION_BUILD diff --git a/test/sqlcipher-pragmas.test b/test/sqlcipher-pragmas.test index 2b611001a..5a0a00006 100644 --- a/test/sqlcipher-pragmas.test +++ b/test/sqlcipher-pragmas.test @@ -46,7 +46,7 @@ do_test verify-pragma-cipher-version { execsql { PRAGMA cipher_version; } -} {{4.9.0 community}} +} {{4.10.0 community}} db close file delete -force test.db From 6ce29efc9df8c3246d7a2a23112830c433bdcd13 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Tue, 20 May 2025 10:24:38 -0400 Subject: [PATCH 18/75] Fixes upstream SQLite version in CHANGELOG.md for 4.9.0 release --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 637e814bb..0f1c10436 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Notable changes to this project are documented in this file. ## [4.10.0] - (? 2025 - [4.10.0 changes]) ## [4.9.0] - (May 2025 - [4.9.0 changes]) -- Updates baseline to upstream SQLite 3.46.2 +- Updates baseline to upstream SQLite 3.49.2 - Removes use of static mutex in `sqlcipher_extra_shutdown()` ## [4.8.0] - (April 2025 - [4.8.0 changes]) From 29f05d055d3e2c95879e42f4c49dac5be12896be Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Wed, 28 May 2025 13:33:56 -0400 Subject: [PATCH 19/75] Allows compile time override of default log level via SQLCIPHER_LOG_LEVEL_DEFAULT macro --- src/sqlcipher.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 7931a583b..6eaa5afcb 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -276,6 +276,10 @@ static volatile sqlite3_mem_methods default_mem_methods; static sqlcipher_provider *default_provider = NULL; static sqlite3_mutex* sqlcipher_static_mutex[SQLCIPHER_MUTEX_COUNT]; + +#ifndef SQLCIPHER_LOG_LEVEL_DEFAULT +#define SQLCIPHER_LOG_LEVEL_DEFAULT SQLCIPHER_LOG_WARN +#endif static FILE* sqlcipher_log_file = NULL; static volatile int sqlcipher_log_device = 0; static volatile unsigned int sqlcipher_log_level = SQLCIPHER_LOG_NONE; @@ -440,7 +444,7 @@ int sqlcipher_extra_init(const char* arg) { /* set log level if it is different than the uninitalized default value of NONE */ if(sqlcipher_log_level == SQLCIPHER_LOG_NONE) { - sqlcipher_log_level = SQLCIPHER_LOG_WARN; + sqlcipher_log_level = SQLCIPHER_LOG_LEVEL_DEFAULT; } /* set the default file or device if neither is already set */ From 32ac05d2174d4af7e2ec3ca9169e55a94771ef5e Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Tue, 3 Jun 2025 12:15:16 -0400 Subject: [PATCH 20/75] Fixes detection of CommonCrypto version on macOS --- src/crypto_cc.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/crypto_cc.c b/src/crypto_cc.c index 49be9503a..57465b728 100644 --- a/src/crypto_cc.c +++ b/src/crypto_cc.c @@ -53,16 +53,14 @@ static const char* sqlcipher_cc_get_provider_name(void *ctx) { static const char* sqlcipher_cc_get_provider_version(void *ctx) { #if TARGET_OS_MAC - CFTypeRef version; CFBundleRef bundle = CFBundleGetBundleWithIdentifier(CFSTR("com.apple.security")); - if(bundle == NULL) { - return "unknown"; + CFTypeRef bundle_ver; + if(bundle && (bundle_ver = CFBundleGetValueForInfoDictionaryKey(bundle, kCFBundleVersionKey))) { + const char *ver = CFStringGetCStringPtr(bundle_ver, kCFStringEncodingUTF8); + if(ver) return ver; } - version = CFBundleGetValueForInfoDictionaryKey(bundle, CFSTR("CFBundleShortVersionString")); - return CFStringGetCStringPtr(version, kCFStringEncodingUTF8); -#else - return "unknown"; #endif + return "unknown"; } static int sqlcipher_cc_hmac( From f64f007562418654cd72c14e95218681604e21c9 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Tue, 3 Jun 2025 12:37:19 -0400 Subject: [PATCH 21/75] Further improves CommonCrypto version detection for iOS --- src/crypto_cc.c | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/crypto_cc.c b/src/crypto_cc.c index 57465b728..8c1963a2a 100644 --- a/src/crypto_cc.c +++ b/src/crypto_cc.c @@ -53,13 +53,29 @@ static const char* sqlcipher_cc_get_provider_name(void *ctx) { static const char* sqlcipher_cc_get_provider_version(void *ctx) { #if TARGET_OS_MAC + /* macOS uses com.apple.security with a lowercase s */ CFBundleRef bundle = CFBundleGetBundleWithIdentifier(CFSTR("com.apple.security")); CFTypeRef bundle_ver; - if(bundle && (bundle_ver = CFBundleGetValueForInfoDictionaryKey(bundle, kCFBundleVersionKey))) { - const char *ver = CFStringGetCStringPtr(bundle_ver, kCFStringEncodingUTF8); - if(ver) return ver; + const char *ver; + + /* if the bundle wasn't identified, try secrurity with a capial S (works for iOS) */ + if(!bundle) { + bundle = CFBundleGetBundleWithIdentifier(CFSTR("com.apple.Security")); + } + + /* If the bundle was resolved, retrieve the bundle version key then attempt to convert it to a C string. + * Note that it is possible for CFStringGetCString to return NULL (this is warned against extensively in the + * header), so only return a value if the conversion was successful. */ + if( + bundle + && (bundle_ver = CFBundleGetValueForInfoDictionaryKey(bundle, kCFBundleVersionKey)) + && (ver = CFStringGetCStringPtr(bundle_ver, kCFStringEncodingUTF8)) + ) { + return ver; } + #endif + /* unable to detect the CoreCrypto version, return a fixed string "unknown" */ return "unknown"; } From fb7d92f34bcca8d972b8a083deef0b6a26c0811f Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 20 Jun 2025 12:58:41 -0400 Subject: [PATCH 22/75] Snapshot of upstream SQLite 3.50.1 --- Makefile.in | 27 +- Makefile.msc | 44 +- README.md | 26 +- VERSION | 2 +- auto.def | 21 +- autoconf/Makefile.in | 41 +- autoconf/Makefile.msc | 14 +- autoconf/README.txt | 4 +- autoconf/auto.def | 17 +- autoconf/tea/Makefile.in | 994 ++-- autoconf/tea/README.txt | 72 +- autoconf/tea/_teaish.tester.tcl.in | 49 + autoconf/tea/aclocal.m4 | 9 - autoconf/tea/auto.def | 8 + autoconf/tea/configure | 7 + autoconf/tea/configure.ac.in | 227 - autoconf/tea/doc/sqlite3.n | 4 +- autoconf/tea/pkgIndex.tcl.in | 40 +- autoconf/tea/tclconfig/install-sh | 541 --- autoconf/tea/tclconfig/tcl.m4 | 4119 ----------------- autoconf/tea/teaish.tcl | 565 +++ autoconf/tea/teaish.test.tcl | 14 + autoconf/tea/win/makefile.vc | 61 - autoconf/tea/win/nmakehlp.c | 815 ---- autoconf/tea/win/rules-ext.vc | 123 - autoconf/tea/win/rules.vc | 1913 -------- autoconf/tea/win/targets.vc | 98 - autosetup/README.md | 161 +- {tool => autosetup}/find_tclconfig.tcl | 0 autosetup/jimsh0.c | 37 +- autosetup/proj.tcl | 1272 ++++- autosetup/sqlite-config.tcl | 661 +-- autosetup/teaish/README.txt | 4 + autosetup/teaish/core.tcl | 2539 ++++++++++ autosetup/teaish/feature.tcl | 214 + autosetup/teaish/tester.tcl | 228 + contrib/sqlitecon.tcl | 2 +- doc/jsonb.md | 2 +- doc/lemon.html | 4 +- doc/pager-invariants.txt | 2 +- doc/tcl-extension-testing.md | 158 +- doc/testrunner.md | 88 +- doc/vfs-shm.txt | 4 +- doc/wal-lock.md | 2 +- ext/README.md | 2 +- ext/expert/sqlite3expert.c | 2 +- ext/expert/test_expert.c | 2 +- ext/fts3/README.syntax | 10 +- ext/fts3/fts3.c | 16 +- ext/fts3/fts3Int.h | 30 +- ext/fts3/fts3_expr.c | 56 +- ext/fts3/fts3_hash.c | 2 +- ext/fts3/fts3_porter.c | 2 +- ext/fts3/fts3_snippet.c | 34 +- ext/fts3/fts3_test.c | 6 +- ext/fts3/fts3_tokenize_vtab.c | 2 +- ext/fts3/fts3_write.c | 4 +- ext/fts3/unicode/mkunicode.tcl | 2 +- ext/fts5/fts5Int.h | 18 +- ext/fts5/fts5_aux.c | 2 +- ext/fts5/fts5_buffer.c | 2 +- ext/fts5/fts5_expr.c | 57 +- ext/fts5/fts5_hash.c | 4 +- ext/fts5/fts5_index.c | 86 +- ext/fts5/fts5_main.c | 10 +- ext/fts5/fts5_test_mi.c | 34 +- ext/fts5/fts5_unicode2.c | 1 - ext/fts5/fts5_vocab.c | 6 +- ext/fts5/test/fts5matchinfo.test | 24 + ext/fts5/test/fts5simple3.test | 2 +- ext/jni/GNUmakefile | 6 +- ext/jni/README.md | 2 +- ext/jni/src/c/sqlite3-jni.c | 16 +- ext/jni/src/c/sqlite3-jni.h | 2 +- .../org/sqlite/jni/annotation/NotNull.java | 4 +- ext/jni/src/org/sqlite/jni/capi/CApi.java | 39 +- .../org/sqlite/jni/capi/CallbackProxy.java | 6 +- .../org/sqlite/jni/capi/OutputPointer.java | 42 +- .../sqlite/jni/capi/PrepareMultiCallback.java | 4 +- .../src/org/sqlite/jni/capi/SQLTester.java | 55 +- .../sqlite/jni/capi/TableColumnMetadata.java | 10 +- ext/jni/src/org/sqlite/jni/capi/Tester1.java | 21 +- .../org/sqlite/jni/capi/XDestroyCallback.java | 2 +- .../org/sqlite/jni/fts5/Fts5ExtensionApi.java | 3 +- .../src/org/sqlite/jni/fts5/TesterFts5.java | 95 +- .../jni/fts5/fts5_extension_function.java | 6 +- .../org/sqlite/jni/test-script-interpreter.md | 6 +- .../sqlite/jni/wrapper1/ScalarFunction.java | 4 - .../org/sqlite/jni/wrapper1/SqlFunction.java | 24 +- .../src/org/sqlite/jni/wrapper1/Sqlite.java | 47 +- .../src/org/sqlite/jni/wrapper1/Tester2.java | 17 +- ext/lsm1/lsmInt.h | 2 +- ext/lsm1/lsm_ckpt.c | 2 +- ext/lsm1/lsm_file.c | 4 +- ext/lsm1/lsm_log.c | 2 +- ext/lsm1/lsm_sorted.c | 6 +- ext/misc/README.md | 12 +- ext/misc/amatch.c | 4 +- ext/misc/btreeinfo.c | 2 +- ext/misc/closure.c | 2 +- ext/misc/completion.c | 7 +- ext/misc/csv.c | 2 +- ext/misc/decimal.c | 7 +- ext/misc/fileio.c | 57 +- ext/misc/fossildelta.c | 2 +- ext/misc/fuzzer.c | 6 +- ext/misc/ieee754.c | 2 +- ext/misc/memstat.c | 10 +- ext/misc/normalize.c | 2 +- ext/misc/percentile.c | 2 +- ext/misc/series.c | 4 +- ext/misc/shathree.c | 2 +- ext/misc/spellfix.c | 2 +- ext/misc/totype.c | 2 +- ext/misc/uint.c | 2 +- ext/misc/vfsstat.c | 4 +- ext/misc/vfstrace.c | 4 +- ext/misc/vtablog.c | 4 +- ext/misc/vtshim.c | 2 +- ext/rbu/rbuvacuum2.test | 2 +- ext/rbu/sqlite3rbu.c | 2 +- ext/rbu/sqlite3rbu.h | 6 +- ext/rbu/test_rbu.c | 4 +- ext/recover/dbdata.c | 3 + ext/recover/recoverslowidx.test | 2 + ext/recover/sqlite3recover.c | 63 +- ext/recover/test_recover.c | 2 +- ext/repair/checkindex.c | 2 +- ext/rtree/README | 2 +- ext/rtree/geopoly.c | 4 +- ext/rtree/rtree.c | 25 +- ext/rtree/rtreedoc2.test | 2 +- ext/session/changeset.c | 77 +- ext/session/session1.test | 118 +- ext/session/sessionD.test | 79 +- ext/session/sessionchange.test | 16 + ext/session/sessionconflict.test | 2 +- ext/session/sessioninvert.test | 2 + ext/session/sqlite3session.c | 70 +- ext/session/sqlite3session.h | 27 +- ext/session/test_session.c | 44 +- ext/wasm/GNUmakefile | 38 +- ext/wasm/api/post-js-header.js | 10 +- ext/wasm/api/sqlite3-api-glue.c-pp.js | 2 +- ext/wasm/api/sqlite3-api-prologue.js | 26 +- ext/wasm/api/sqlite3-api-worker1.c-pp.js | 25 +- ext/wasm/api/sqlite3-opfs-async-proxy.js | 4 +- ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js | 231 +- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 2 +- ext/wasm/api/sqlite3-wasm.c | 14 +- ext/wasm/api/sqlite3-worker1-promiser.c-pp.js | 4 +- ext/wasm/c-pp.c | 4 +- ext/wasm/common/whwasmutil.js | 6 +- ext/wasm/demo-jsstorage.js | 2 +- ext/wasm/demo-worker1-promiser.c-pp.js | 5 +- ext/wasm/demo-worker1.js | 5 +- ext/wasm/fiddle.make | 2 +- ext/wasm/fiddle/fiddle.js | 2 +- ext/wasm/fiddle/index.html | 2 +- ext/wasm/index.html | 4 + ext/wasm/jaccwabyt/jaccwabyt.js | 2 +- ext/wasm/jaccwabyt/jaccwabyt.md | 4 +- ext/wasm/mkwasmbuilds.c | 72 +- ext/wasm/tester1.c-pp.js | 21 +- ext/wasm/tests/opfs/sahpool/digest-worker.js | 94 + ext/wasm/tests/opfs/sahpool/digest.html | 151 + ext/wasm/tests/opfs/sahpool/index.html | 31 + .../tests/opfs/sahpool/sahpool-pausing.js | 183 + ext/wasm/tests/opfs/sahpool/sahpool-worker.js | 104 + main.mk | 210 +- manifest | 866 ++-- manifest.uuid | 2 +- src/alter.c | 57 +- src/analyze.c | 17 +- src/attach.c | 9 +- src/bitvec.c | 8 +- src/btmutex.c | 2 +- src/btree.c | 85 +- src/btreeInt.h | 6 + src/build.c | 155 +- src/ctime.c | 796 ---- src/date.c | 2 +- src/dbpage.c | 4 +- src/expr.c | 45 +- src/func.c | 140 +- src/hash.c | 46 +- src/hash.h | 1 + src/insert.c | 40 +- src/json.c | 400 +- src/loadext.c | 4 +- src/main.c | 97 +- src/memdb.c | 4 +- src/os_unix.c | 33 +- src/os_win.c | 1471 ++++-- src/os_win.h | 2 + src/pager.c | 48 +- src/parse.y | 33 +- src/pcache1.c | 20 +- src/pragma.c | 7 +- src/pragma.h | 660 --- src/printf.c | 145 +- src/resolve.c | 110 +- src/select.c | 91 +- src/shell.c.in | 690 ++- src/sqlite.h.in | 113 +- src/sqlite3.rc | 2 +- src/sqlite3ext.h | 4 + src/sqliteInt.h | 142 +- src/sqliteLimit.h | 12 +- src/status.c | 7 +- src/tclsqlite.c | 32 +- src/test1.c | 290 +- src/test2.c | 70 +- src/test3.c | 69 +- src/test4.c | 80 +- src/test5.c | 6 +- src/test6.c | 10 +- src/test9.c | 6 +- src/test_backup.c | 2 +- src/test_blob.c | 2 +- src/test_config.c | 13 + src/test_delete.c | 4 +- src/test_fs.c | 13 +- src/test_hexio.c | 10 +- src/test_init.c | 2 +- src/test_malloc.c | 2 +- src/test_multiplex.c | 4 +- src/test_mutex.c | 4 +- src/test_osinst.c | 27 +- src/test_quota.c | 58 +- src/test_sqllog.c | 2 +- src/test_superlock.c | 2 +- src/test_syscall.c | 4 +- src/test_thread.c | 10 +- src/test_vfs.c | 18 +- src/test_windirent.c | 93 +- src/test_windirent.h | 7 +- src/tokenize.c | 6 +- src/trigger.c | 24 +- src/update.c | 44 +- src/utf.c | 31 +- src/util.c | 8 +- src/vacuum.c | 4 +- src/vdbe.c | 50 +- src/vdbe.h | 4 +- src/vdbeInt.h | 32 +- src/vdbeapi.c | 15 +- src/vdbeaux.c | 48 +- src/vdbeblob.c | 17 +- src/vdbemem.c | 8 +- src/vdbesort.c | 19 +- src/vtab.c | 5 +- src/wal.c | 45 +- src/where.c | 35 +- src/whereInt.h | 7 +- src/wherecode.c | 35 +- src/window.c | 2 +- test/affinity2.test | 2 +- test/affinity3.test | 8 +- test/all.test | 2 +- test/alter3.test | 2 +- test/alter4.test | 4 +- test/altercol.test | 2 +- test/altertab2.test | 2 +- test/analyze3.test | 2 +- test/analyzeC.test | 2 +- test/analyzer1.test | 2 +- test/attach.test | 2 +- test/autoinc.test | 2 +- test/autoindex5.test | 2 +- test/avtrans.test | 2 +- test/backup2.test | 2 +- test/bc_common.tcl | 2 +- test/bloom1.test | 14 +- test/btree01.test | 2 +- test/cast.test | 8 +- test/check.test | 32 + test/chunksize.test | 2 +- test/collate1.test | 2 +- test/collateB.test | 2 +- test/colmeta.test | 2 +- test/colname.test | 2 +- test/conflict.test | 2 +- test/crash8.test | 2 +- test/date.test | 2 +- test/date4.test | 8 +- test/dbfuzz.c | 2 +- test/delete4.test | 2 +- test/enc2.test | 2 +- test/expr.test | 2 +- test/extension01.test | 4 +- test/external_reader.test | 2 +- test/filectrl.test | 14 +- test/fkey6.test | 35 + test/fts3cov.test | 4 +- test/fts3expr2.test | 2 +- test/fts3join.test | 5 +- test/fts4langid.test | 2 +- test/func.test | 2 +- test/func3.test | 2 +- test/func9.test | 12 +- test/fuzz.test | 2 +- test/fuzzcheck.c | 98 +- test/fuzzdata8.db | Bin 4248576 -> 4249600 bytes test/gencol1.test | 6 +- test/in.test | 6 +- test/in4.test | 2 +- test/index.test | 4 +- test/index6.test | 8 +- test/indexedby.test | 2 +- test/indexexpr1.test | 10 +- test/insert.test | 6 +- test/instr.test | 2 +- test/intpkey.test | 4 +- test/io.test | 4 +- test/ioerr.test | 4 +- test/join.test | 59 +- test/join5.test | 4 +- test/joinH.test | 70 + test/journal1.test | 2 +- test/journal3.test | 2 +- test/json/json-speed-check.sh | 2 +- test/json101.test | 8 +- test/json103.test | 2 +- test/like.test | 2 +- test/like3.test | 14 +- test/limit2.test | 8 +- test/loadext.test | 8 +- test/lock.test | 2 +- test/lock_common.tcl | 1 + test/malloc.test | 2 +- test/minmax.test | 2 +- test/misc7.test | 6 +- test/misuse.test | 6 +- test/mmap2.test | 4 +- test/nockpt.test | 6 +- test/orderby1.test | 2 +- test/orderby3.test | 2 +- test/orderby4.test | 2 +- test/oserror.test | 5 +- test/pager1.test | 12 +- test/pager4.test | 2 +- test/pagerfault.test | 2 +- test/permutations.test | 2 +- test/pragma.test | 2 +- test/quota.test | 6 +- test/readonly.test | 2 +- test/recover.test | 3 + test/rowvalue.test | 8 +- test/rowvalue3.test | 2 +- test/rowvalue7.test | 2 +- test/savepoint7.test | 2 +- test/select3.test | 2 +- test/select4.test | 4 +- test/select6.test | 2 +- test/shared.test | 6 +- test/shared3.test | 2 +- test/shared6.test | 2 +- test/shell1.test | 33 +- test/shell3.test | 2 +- test/shell4.test | 2 +- test/shell5.test | 4 +- test/shell8.test | 2 +- test/shellA.test | 257 + test/shmlock.test | 2 +- test/skipscan6.test | 4 +- test/snapshot3.test | 3 + test/speedtest.tcl | 29 +- test/speedtest1.c | 645 ++- test/subquery.test | 38 + test/subquery2.test | 2 +- test/symlink.test | 2 +- test/symlink2.test | 2 +- test/sync.test | 2 +- test/sync2.test | 2 +- test/tabfunc01.test | 2 +- test/table.test | 6 +- test/tableapi.test | 2 +- test/tester.tcl | 24 +- test/testrunner.tcl | 168 +- test/testrunner_data.tcl | 40 +- test/tkt3457.test | 4 +- test/trigger1.test | 2 +- test/triggerG.test | 2 +- test/unixexcl.test | 2 +- test/update.test | 8 +- test/upsert1.test | 2 +- test/uri.test | 4 +- test/vacuum-into.test | 4 +- test/vt100-a.sql | 19 + test/vtabH.test | 4 +- test/wal2.test | 6 +- test/wal6.test | 4 +- test/wal64k.test | 4 +- test/walblock.test | 2 +- test/walcrash4.test | 2 +- test/walmode.test | 2 +- test/walnoshm.test | 1 + test/walro.test | 2 +- test/walsetlk.test | 159 +- test/walsetlk2.test | 267 ++ test/walsetlk3.test | 135 + test/walsetlk_recover.test | 104 + test/walsetlk_snapshot.test | 109 + test/where.test | 8 +- test/where2.test | 2 +- test/whereG.test | 4 +- test/whereL.test | 40 + test/win32heap.test | 2 +- test/win32lock.test | 2 +- test/win32longpath.test | 9 +- test/win32nolock.test | 2 +- test/window1.test | 6 +- test/with1.test | 2 +- test/without_rowid1.test | 35 +- test/writecrash.test | 2 +- test/zipfile.test | 8 +- tool/build-all-msvc.bat | 2 +- tool/buildtclext.tcl | 15 +- tool/genfkey.README | 6 +- tool/kvtest-speed.sh | 35 - tool/lemon.c | 6 +- tool/lempar.c | 2 +- tool/mkautoconfamal.sh | 21 +- tool/mkccode.tcl | 13 +- tool/mkctimec.tcl | 13 +- tool/mkpragmatab.tcl | 8 +- tool/mkspeedsql.tcl | 237 - tool/mksqlite3c.tcl | 2 +- tool/mksqlite3internalh.tcl | 2 +- tool/run-speed-test.sh | 90 - tool/showdb.c | 7 +- tool/soak1.tcl | 2 +- tool/speed-check.sh | 219 - tool/speedtest.tcl | 275 -- tool/speedtest16.c | 171 - tool/speedtest2.tcl | 207 - tool/speedtest8.c | 260 -- tool/speedtest8inst1.c | 218 - tool/split-sqlite3c.tcl | 2 +- tool/sqldiff.c | 2 +- tool/sqlite3_analyzer.c.in | 3 +- tool/sqlite3_rsync.c | 813 +++- tool/src-verify.c | 2 +- tool/srcck1.c | 2 +- tool/srctree-check.tcl | 31 +- tool/stripccomments.c | 2 +- tool/tclConfigShToAutoDef.sh | 29 - tool/warnings.sh | 18 +- 449 files changed, 15404 insertions(+), 15896 deletions(-) create mode 100644 autoconf/tea/_teaish.tester.tcl.in delete mode 100644 autoconf/tea/aclocal.m4 create mode 100644 autoconf/tea/auto.def create mode 100755 autoconf/tea/configure delete mode 100644 autoconf/tea/configure.ac.in delete mode 100644 autoconf/tea/tclconfig/install-sh delete mode 100644 autoconf/tea/tclconfig/tcl.m4 create mode 100644 autoconf/tea/teaish.tcl create mode 100644 autoconf/tea/teaish.test.tcl delete mode 100644 autoconf/tea/win/makefile.vc delete mode 100644 autoconf/tea/win/nmakehlp.c delete mode 100644 autoconf/tea/win/rules-ext.vc delete mode 100644 autoconf/tea/win/rules.vc delete mode 100644 autoconf/tea/win/targets.vc rename {tool => autosetup}/find_tclconfig.tcl (100%) create mode 100644 autosetup/teaish/README.txt create mode 100644 autosetup/teaish/core.tcl create mode 100644 autosetup/teaish/feature.tcl create mode 100644 autosetup/teaish/tester.tcl mode change 100644 => 100755 ext/session/session1.test mode change 100644 => 100755 ext/session/sessionconflict.test mode change 100644 => 100755 ext/session/sessioninvert.test create mode 100644 ext/wasm/tests/opfs/sahpool/digest-worker.js create mode 100644 ext/wasm/tests/opfs/sahpool/digest.html create mode 100644 ext/wasm/tests/opfs/sahpool/index.html create mode 100644 ext/wasm/tests/opfs/sahpool/sahpool-pausing.js create mode 100644 ext/wasm/tests/opfs/sahpool/sahpool-worker.js delete mode 100644 src/ctime.c delete mode 100644 src/pragma.h create mode 100644 test/shellA.test create mode 100644 test/vt100-a.sql create mode 100644 test/walsetlk2.test create mode 100644 test/walsetlk3.test create mode 100644 test/walsetlk_recover.test create mode 100644 test/walsetlk_snapshot.test delete mode 100644 tool/kvtest-speed.sh delete mode 100644 tool/mkspeedsql.tcl delete mode 100644 tool/run-speed-test.sh delete mode 100644 tool/speed-check.sh delete mode 100644 tool/speedtest.tcl delete mode 100644 tool/speedtest16.c delete mode 100644 tool/speedtest2.tcl delete mode 100644 tool/speedtest8.c delete mode 100644 tool/speedtest8inst1.c delete mode 100755 tool/tclConfigShToAutoDef.sh diff --git a/Makefile.in b/Makefile.in index 60cc228e9..57728b049 100644 --- a/Makefile.in +++ b/Makefile.in @@ -138,16 +138,13 @@ ENABLE_LIB_STATIC = @ENABLE_LIB_STATIC@ HAVE_WASI_SDK = @HAVE_WASI_SDK@ libsqlite3.DLL.install-rules = @SQLITE_DLL_INSTALL_RULES@ -T.cc.sqlite = $(T.cc) @TARGET_DEBUG@ +# -fsanitize flags for the fuzzcheck-asap app +CFLAGS.fuzzcheck-asan.fsanitize = @CFLAGS_ASAN_FSANITIZE@ # -# Define -D_HAVE_SQLITE_CONFIG_H so that the code knows it -# can include the generated sqlite_cfg.h. +# Intended to either be empty or be set to -g -DSQLITE_DEBUG=1. # -# main.mk will fill out T.cc.sqlite with additional flags common to -# all builds. -# -T.cc.sqlite += -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite +T.cc.TARGET_DEBUG = @TARGET_DEBUG@ # # $(JIMSH) and $(CFLAGS.jimsh) are documented in main.mk. $(JIMSH) @@ -246,8 +243,8 @@ TSTRNNR_OPTS = @TSTRNNR_OPTS@ CFLAGS.gcov1 = -DSQLITE_COVERAGE_TEST=1 -fprofile-arcs -ftest-coverage LDFLAGS.gcov1 = -lgcov USE_GCOV = @USE_GCOV@ -T.compile.extras = $(CFLAGS.gcov$(USE_GCOV)) -T.link.extras = $(LDFLAGS.gcov$(USE_GCOV)) +T.compile.gcov = $(CFLAGS.gcov$(USE_GCOV)) +T.link.gcov = $(LDFLAGS.gcov$(USE_GCOV)) # # Vars with the AS_ prefix are specifically related to AutoSetup. @@ -264,6 +261,9 @@ AS_AUTORECONFIG = @SQLITE_AUTORECONFIG@ USE_AMALGAMATION ?= @USE_AMALGAMATION@ LINK_TOOLS_DYNAMICALLY ?= @LINK_TOOLS_DYNAMICALLY@ AMALGAMATION_GEN_FLAGS ?= --linemacros=@AMALGAMATION_LINE_MACROS@ +EXTRA_SRC ?= @AMALGAMATION_EXTRA_SRC@ +STATIC_TCLSQLITE3 = @STATIC_TCLSQLITE3@ +STATIC_CLI_SHELL = @STATIC_CLI_SHELL@ # # CFLAGS for sqlite3$(T.exe) @@ -325,13 +325,7 @@ misspell: ./custom.rws has_tclsh84 # perform cleanup known to be relevant to (only) the autosetup-driven # build. # -#clean-autosetup: -# -if [ -f ext/wasm/GNUmakefile ]; then \ -# gmake --no-print-directory --ignore-errors -C ext/wasm clean; \ -# fi >/dev/null 2>&1; true -#clean: clean-autosetup - -distclean-autosetup: clean +distclean-autosetup: rm -f sqlite_cfg.h config.log config.status config.defines.* Makefile sqlite3.pc rm -f $(TOP)/tool/emcc.sh rm -f libsqlite3*$(T.dll) @@ -345,5 +339,4 @@ distclean: distclean-autosetup version-info$(T.exe): $(TOP)/tool/version-info.c Makefile sqlite3.h $(T.link) $(ST_OPT) -o $@ $(TOP)/tool/version-info.c -IS_CROSS_COMPILING = @IS_CROSS_COMPILING@ include $(TOP)/main.mk diff --git a/Makefile.msc b/Makefile.msc index c1a8f88b6..6aef67155 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -294,6 +294,12 @@ SESSION = 0 RBU = 0 !ENDIF +# Set this to non-0 to enable support for blocking locks. +# +!IFNDEF SETLK_TIMEOUT +SETLK_TIMEOUT = 0 +!ENDIF + # Set the source code file to be used by executables and libraries when # they need the amalgamation. # @@ -450,6 +456,10 @@ EXT_FEATURE_FLAGS = !ENDIF !ENDIF +!IF $(SETLK_TIMEOUT)!=0 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SETLK_TIMEOUT +!ENDIF + ############################################################################### ############################### END OF OPTIONS ################################ ############################################################################### @@ -874,7 +884,7 @@ RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 !ENDIF !IF $(DEBUG)>2 -TCC = $(TCC) -DSQLITE_DEBUG=1 +TCC = $(TCC) -DSQLITE_DEBUG=1 -DSQLITE_USE_W32_FOR_CONSOLE_IO RCC = $(RCC) -DSQLITE_DEBUG=1 !IF $(DYNAMIC_SHELL)==0 TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE @@ -1396,7 +1406,7 @@ SRC00 = \ $(TOP)\src\build.c \ $(TOP)\src\callback.c \ $(TOP)\src\complete.c \ - $(TOP)\src\ctime.c \ + ctime.c \ $(TOP)\src\date.c \ $(TOP)\src\dbpage.c \ $(TOP)\src\dbstat.c \ @@ -1495,7 +1505,7 @@ SRC04 = \ SRC05 = \ $(TOP)\src\pager.h \ $(TOP)\src\pcache.h \ - $(TOP)\src\pragma.h \ + pragma.h \ $(TOP)\src\sqlite.h.in \ $(TOP)\src\sqlite3ext.h \ $(TOP)\src\sqliteInt.h \ @@ -1698,7 +1708,7 @@ HDR = \ $(TOP)\src\pager.h \ $(TOP)\src\pcache.h \ parse.h \ - $(TOP)\src\pragma.h \ + pragma.h \ $(SQLITE3H) \ sqlite3ext.h \ $(TOP)\src\sqliteInt.h \ @@ -1962,10 +1972,6 @@ fuzzcheck.exe: $(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H) fuzzcheck-asan.exe: $(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) /fsanitize=address $(FUZZCHECK_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) -run-fuzzcheck: fuzzcheck.exe fuzzcheck-asan.exe - fuzzcheck --spinner $(FUZZDB) - fuzzcheck-asan --spinner $(FUZZDB) - ossshell.exe: $(OSSSHELL_SRC) $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(FUZZCHECK_OPTS) $(OSSSHELL_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) @@ -2099,8 +2105,11 @@ callback.lo: $(TOP)\src\callback.c $(HDR) complete.lo: $(TOP)\src\complete.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\complete.c -ctime.lo: $(TOP)\src\ctime.c $(HDR) - $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\ctime.c +ctime.c: $(TOP)\tool\mkctimec.tcl $(JIM_TCLSH) + $(JIM_TCLSH) $(TOP)\tool\mkctimec.tcl + +ctime.lo: ctime.c $(HDR) + $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c ctime.c date.lo: $(TOP)\src\date.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\date.c @@ -2333,6 +2342,9 @@ parse.c: $(TOP)\src\parse.y lemon.exe copy /B parse.y +,, .\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(EXT_FEATURE_FLAGS) $(OPTS) -S parse.y +pragma.h: $(TOP)\tool\mkpragmatab.tcl $(JIM_TCLSH) + $(JIM_TCLSH) $(TOP)\tool\mkpragmatab.tcl + $(SQLITE3H): $(TOP)\src\sqlite.h.in $(TOP)\manifest mksourceid.exe $(TOP)\VERSION $(JIM_TCLSH) $(JIM_TCLSH) $(TOP)\tool\mksqlite3h.tcl "$(TOP:\=/)" -o $(SQLITE3H) $(MKSQLITE3H_ARGS) @@ -2577,6 +2589,11 @@ testfixture.exe: $(TESTFIXTURE_SRC) $(TESTFIXTURE_DEP) $(SQLITE3H) $(LIBRESOBJS) $(TESTFIXTURE_SRC) \ /link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS) +# A small helper for manually running individual tests +tf.bat: testfixture.exe Makefile.msc + echo @set PATH=$(LIBTCLPATH);%PATH% > $@ + echo .\testfixture.exe %* >> $@ + extensiontest: testfixture.exe testloadext.dll @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS) @@ -2812,7 +2829,7 @@ moreclean: clean clean: del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL del /Q *.bsc *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL - del /Q sqlite3.def tclsqlite3.def 2>NUL + del /Q sqlite3.def tclsqlite3.def ctime.c pragma.h 2>NUL del /Q $(SQLITE3EXE) $(SQLITE3DLL) Replace.exe 2>NUL # <> del /Q $(SQLITE3TCLDLL) pkgIndex.tcl 2>NUL @@ -2828,7 +2845,9 @@ clean: del /Q lsm.dll lsmtest.exe 2>NUL del /Q atrc.exe changesetfuzz.exe dbtotxt.exe index_usage.exe 2>NUL del /Q testloadext.dll 2>NUL - del /Q testfixture.exe test.db 2>NUL + del /Q testfixture.exe test.db tf.bat 2>NUL + del /Q /S testdir 2>/NUL + -rmdir /Q /S testdir 2>NUL del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe dbdump.exe 2>NUL del /Q changeset.exe 2>NUL del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL @@ -2846,4 +2865,5 @@ clean: del /Q fts5.* fts5parse.* 2>NUL del /Q lsm.h lsm1.c 2>NUL del /q src-verify.exe 2>NUL + del /q jimsh.exe jimsh0.exe 2>NUL # <> diff --git a/README.md b/README.md index 689b52dab..0be1fb9f4 100644 --- a/README.md +++ b/README.md @@ -57,18 +57,18 @@ If you do not want to use Fossil, you can download tarballs or ZIP archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows: * Latest trunk check-in as - [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz), - [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip), or - [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar). + [Tarball](https://sqlite.org/src/tarball/sqlite.tar.gz), + [ZIP-archive](https://sqlite.org/src/zip/sqlite.zip), or + [SQLite-archive](https://sqlite.org/src/sqlar/sqlite.sqlar). * Latest release as - [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release), - [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip?r=release), or - [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar?r=release). + [Tarball](https://sqlite.org/src/tarball/sqlite.tar.gz?r=release), + [ZIP-archive](https://sqlite.org/src/zip/sqlite.zip?r=release), or + [SQLite-archive](https://sqlite.org/src/sqlar/sqlite.sqlar?r=release). * For other check-ins, substitute an appropriate branch name or tag or hash prefix in place of "release" in the URLs of the previous - bullet. Or browse the [timeline](https://www.sqlite.org/src/timeline) + bullet. Or browse the [timeline](https://sqlite.org/src/timeline) to locate the check-in desired, click on its information page link, then click on the "Tarball" or "ZIP Archive" links on the information page. @@ -134,7 +134,7 @@ later. The "tclextension-install" target and the test targets that follow all require TCL development libraries too. ("apt install tcl-dev"). It is helpful, but is not required, to install the SQLite TCL extension (the "tclextension-install" target) prior to running tests. The "releasetest" -target has additional requiremenst, such as "valgrind". +target has additional requirements, such as "valgrind". On "make" command-lines, one can add "OPTIONS=..." to specify additional compile-time options over and above those set by ./configure. For example, @@ -308,14 +308,14 @@ individual source file exceeds 32K lines in length. ## How It All Fits Together SQLite is modular in design. -See the [architectural description](https://www.sqlite.org/arch.html) +See the [architectural description](https://sqlite.org/arch.html) for details. Other documents that are useful in helping to understand how SQLite works include the -[file format](https://www.sqlite.org/fileformat2.html) description, -the [virtual machine](https://www.sqlite.org/opcode.html) that runs +[file format](https://sqlite.org/fileformat2.html) description, +the [virtual machine](https://sqlite.org/opcode.html) that runs prepared statements, the description of -[how transactions work](https://www.sqlite.org/atomiccommit.html), and -the [overview of the query planner](https://www.sqlite.org/optoverview.html). +[how transactions work](https://sqlite.org/atomiccommit.html), and +the [overview of the query planner](https://sqlite.org/optoverview.html). Decades of effort have gone into optimizing SQLite, both for small size and high performance. And optimizations tend to result in diff --git a/VERSION b/VERSION index 46325cfb2..b05723fc6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.49.2 +3.50.1 diff --git a/auto.def b/auto.def index 8ed599637..43c597551 100644 --- a/auto.def +++ b/auto.def @@ -12,7 +12,6 @@ # # JimTCL: https://jim.tcl.tk # - use sqlite-config sqlite-configure canonical { proj-if-opt-truthy dev { @@ -25,7 +24,9 @@ sqlite-configure canonical { # -------------^^^^^^^ intentionally using [get-env] instead of # [proj-get-env] here because [sqlite-setup-default-cflags] uses # [proj-get-env] and we want this to supercede that. + sqlite-munge-cflags; # straighten out -DSQLITE_ENABLE/OMIT flags } + sqlite-handle-debug ;# must come after --dev flag check sqlite-check-common-bins ;# must come before [sqlite-handle-wasi-sdk] sqlite-handle-wasi-sdk ;# must run relatively early, as it changes the environment sqlite-check-common-system-deps @@ -40,8 +41,24 @@ sqlite-configure canonical { proj-define-for-opt linemacros AMALGAMATION_LINE_MACROS \ "Use #line macros in the amalgamation:" + define AMALGAMATION_EXTRA_SRC \ + [join [opt-val amalgamation-extra-src ""] " "] + define LINK_TOOLS_DYNAMICALLY [proj-opt-was-provided dynlink-tools] + if {[set fsan [join [opt-val asan-fsanitize] ","]] in {auto ""}} { + set fsan address,bounds-strict + } + define CFLAGS_ASAN_FSANITIZE [proj-check-fsanitize [split $fsan ", "]] + sqlite-handle-tcl sqlite-handle-emsdk -} + + proj-if-opt-truthy static-shells { + proj-opt-set static-tclsqlite3 1 + proj-opt-set static-cli-shell 1 + } + proj-define-for-opt static-tclsqlite3 STATIC_TCLSQLITE3 "Statically link tclsqlite3?" + proj-define-for-opt static-cli-shell STATIC_CLI_SHELL "Statically link CLI shell?" + +}; # sqlite-configure diff --git a/autoconf/Makefile.in b/autoconf/Makefile.in index 2f6908240..a77386fae 100644 --- a/autoconf/Makefile.in +++ b/autoconf/Makefile.in @@ -50,6 +50,7 @@ CC = @CC@ ENABLE_LIB_SHARED = @ENABLE_LIB_SHARED@ ENABLE_LIB_STATIC = @ENABLE_LIB_STATIC@ +HAVE_WASI_SDK = @HAVE_WASI_SDK@ CFLAGS = @CFLAGS@ @CPPFLAGS@ # @@ -71,11 +72,8 @@ LDFLAGS.rt = @LDFLAGS_RT@ LDFLAGS.icu = @LDFLAGS_ICU@ CFLAGS.icu = @CFLAGS_ICU@ -# When cross-compiling, we need to avoid the -s flag because it only -# works on the build host's platform. -INSTALL.strip.1 = $(INSTALL) -INSTALL.strip.0 = $(INSTALL) -s -INSTALL.strip = $(INSTALL.strip.@IS_CROSS_COMPILING@) +# INSTALL reminder: we specifically do not strip binaries, +# as discussed in https://sqlite.org/forum/forumpost/9a67df63eda9925c. INSTALL.noexec = $(INSTALL) -m 0644 install-dir.bin = $(DESTDIR)$(bindir) @@ -193,6 +191,8 @@ install-dll-unix-generic: install-dll-out-implib install-dll-msys: install-dll-out-implib $(install-dir.bin) $(INSTALL) $(libsqlite3.DLL) "$(install-dir.bin)" # ----------------------------------------------^^^ yes, bin +# Each of {msys,mingw,cygwin} uses a different name for the DLL, but +# that is already accounted for via $(libsqlite3.DLL). install-dll-mingw: install-dll-msys install-dll-cygwin: install-dll-msys @@ -218,27 +218,43 @@ install-lib-0 install-lib-: install-lib: install-lib-$(ENABLE_LIB_STATIC) install: install-lib - +# # Flags to link the shell app either directly against sqlite3.c # (ENABLE_STATIC_SHELL==1) or libsqlite3.so (ENABLE_STATIC_SHELL==0). # ENABLE_STATIC_SHELL = @ENABLE_STATIC_SHELL@ sqlite3-shell-link-flags.1 = $(TOP)/sqlite3.c $(LDFLAGS.libsqlite3) -sqlite3-shell-link-flags.0 = -L. -lsqlite3 $(LDFLAGS.zlib) +sqlite3-shell-link-flags.0 = -L. -lsqlite3 $(LDFLAGS.zlib) $(LDFLAGS.math) sqlite3-shell-deps.1 = $(TOP)/sqlite3.c sqlite3-shell-deps.0 = $(libsqlite3.DLL) +# +# STATIC_CLI_SHELL = 1 to statically link sqlite3$(T.exe), else +# 0. Requires static versions of all requisite libraries. Primarily +# intended for use with static-friendly environments like Alpine +# Linux. +# +STATIC_CLI_SHELL = @STATIC_CLI_SHELL@ +# +# sqlite3-shell-static.flags.N = N is $(STATIC_CLI_SHELL) +# +sqlite3-shell-static.flags.1 = -static +sqlite3-shell-static.flags.0 = sqlite3$(T.exe): $(TOP)/shell.c $(sqlite3-shell-deps.$(ENABLE_STATIC_SHELL)) $(CC) -o $@ \ $(TOP)/shell.c $(sqlite3-shell-link-flags.$(ENABLE_STATIC_SHELL)) \ + $(sqlite3-shell-static.flags.$(STATIC_CLI_SHELL)) \ -I. $(OPT_FEATURE_FLAGS) $(SHELL_OPT) \ $(CFLAGS) $(CFLAGS.readline) $(CFLAGS.icu) \ $(LDFLAGS) $(LDFLAGS.readline) -all: sqlite3$(T.exe) +sqlite3$(T.exe)-1: +sqlite3$(T.exe)-0: sqlite3$(T.exe) +all: sqlite3$(T.exe)-$(HAVE_WASI_SDK) -install-shell: sqlite3$(T.exe) $(install-dir.bin) - $(INSTALL.strip) sqlite3$(T.exe) "$(install-dir.bin)" -install: install-shell +install-shell-0: sqlite3$(T.exe) $(install-dir.bin) + $(INSTALL) sqlite3$(T.exe) "$(install-dir.bin)" +install-shell-1: +install: install-shell-$(HAVE_WASI_SDK) install-headers: $(TOP)/sqlite3.h $(install-dir.include) $(INSTALL.noexec) $(TOP)/sqlite3.h $(TOP)/sqlite3ext.h "$(install-dir.include)" @@ -267,15 +283,16 @@ DIST_FILES := \ sqlite3.rc sqlite3rc.h Replace.cs \ sqlite3.pc.in sqlite3.1 +# # Maintenance note: dist_name must be sqlite-$(PACKAGE_VERSION) so # that tool/mkautoconfamal.sh knows how to find it. +# dist_name = sqlite-$(PACKAGE_VERSION) dist_tarball = $(dist_name).tar.gz dist: rm -fr $(dist_name) mkdir -p $(dist_name) cp -rp $(DIST_FILES) $(dist_name)/. - rm -f $(dist_name)/tea/configure.ac.in tar czf $(dist_tarball) $(dist_name) rm -fr $(dist_name) ls -l $(dist_tarball) diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index 47e0a83af..dfa2dcfd5 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -247,6 +247,12 @@ SESSION = 0 RBU = 0 !ENDIF +# Set this to non-0 to enable support for blocking locks. +# +!IFNDEF SETLK_TIMEOUT +SETLK_TIMEOUT = 0 +!ENDIF + # Set the source code file to be used by executables and libraries when # they need the amalgamation. # @@ -372,6 +378,10 @@ EXT_FEATURE_FLAGS = !ENDIF !ENDIF +!IF $(SETLK_TIMEOUT)!=0 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_SETLK_TIMEOUT +!ENDIF + ############################################################################### ############################### END OF OPTIONS ################################ ############################################################################### @@ -714,7 +724,7 @@ RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 !ENDIF !IF $(DEBUG)>2 -TCC = $(TCC) -DSQLITE_DEBUG=1 +TCC = $(TCC) -DSQLITE_DEBUG=1 -DSQLITE_USE_W32_FOR_CONSOLE_IO RCC = $(RCC) -DSQLITE_DEBUG=1 !IF $(DYNAMIC_SHELL)==0 TCC = $(TCC) -DSQLITE_ENABLE_WHERETRACE -DSQLITE_ENABLE_SELECTTRACE @@ -1088,5 +1098,5 @@ $(LIBRESOBJS): $(TOP)\sqlite3.rc rcver.vc $(SQLITE3H) clean: del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL del /Q *.bsc *.cod *.da *.bb *.bbg *.vc gmon.out 2>NUL - del /Q sqlite3.def tclsqlite3.def 2>NUL + del /Q sqlite3.def tclsqlite3.def ctime.c pragma.h 2>NUL del /Q $(SQLITE3EXE) $(SQLITE3DLL) Replace.exe 2>NUL diff --git a/autoconf/README.txt b/autoconf/README.txt index 646c0a121..ca0ed20fd 100644 --- a/autoconf/README.txt +++ b/autoconf/README.txt @@ -27,7 +27,7 @@ the embedded copy of JimTCL). REASONS TO USE THE CANONICAL BUILD SYSTEM RATHER THAN THIS PACKAGE ================================================================== - * the cononical build system allows you to run tests to verify that + * the canonical build system allows you to run tests to verify that the build worked * the canonical build system supports more compile-time options * the canonical build system works for any arbitrary check-in to @@ -90,7 +90,7 @@ Other preprocessor defines Additionally, preprocessor defines may be specified by using the OPTS macro on the NMAKE command line. However, not all possible preprocessor defines may be specified in this manner as some require the amalgamation to be built -with them enabled (see http://www.sqlite.org/compile.html). For example, the +with them enabled (see http://sqlite.org/compile.html). For example, the following will work: "OPTS=-DSQLITE_ENABLE_STAT4=1 -DSQLITE_OMIT_JSON=1" diff --git a/autoconf/auto.def b/autoconf/auto.def index 3ba900d95..c61d81e50 100644 --- a/autoconf/auto.def +++ b/autoconf/auto.def @@ -5,6 +5,21 @@ # "autoconf" bundle of the SQLite project. use sqlite-config sqlite-configure autoconf { - sqlite-check-common-bins + sqlite-handle-debug + sqlite-check-common-bins ;# must come before [sqlite-handle-wasi-sdk] + sqlite-handle-wasi-sdk ;# must run relatively early, as it changes the environment sqlite-check-common-system-deps + proj-define-for-opt static-shell ENABLE_STATIC_SHELL \ + "Link library statically into the CLI shell?" + proj-define-for-opt static-cli-shell STATIC_CLI_SHELL "Statically link CLI shell?" + if {![opt-bool static-shell] && [opt-bool static-cli-shell]} { + proj-fatal "--disable-static-shell and --static-cli-shell are mutualy exclusive" + } + if {![opt-bool shared] && ![opt-bool static-shell]} { + proj-opt-set shared 1 + proj-indented-notice { + NOTICE: ignoring --disable-shared because --disable-static-shell + was specified. + } + } } diff --git a/autoconf/tea/Makefile.in b/autoconf/tea/Makefile.in index cc98ab182..5b2ad4c69 100644 --- a/autoconf/tea/Makefile.in +++ b/autoconf/tea/Makefile.in @@ -1,463 +1,535 @@ -# Makefile.in -- -# -# This file is a Makefile for Sample TEA Extension. If it has the name -# "Makefile.in" then it is a template for a Makefile; to generate the -# actual Makefile, run "./configure", which is a configuration script -# generated by the "autoconf" program (constructs like "@foo@" will get -# replaced in the actual Makefile. -# -# Copyright (c) 1999 Scriptics Corporation. -# Copyright (c) 2002-2005 ActiveState Corporation. -# -# See the file "license.terms" for information on usage and redistribution -# of this file, and for a DISCLAIMER OF ALL WARRANTIES. - -#======================================================================== -# Add additional lines to handle any additional AC_SUBST cases that -# have been added in a customized configure script. -#======================================================================== - -#SAMPLE_NEW_VAR = @SAMPLE_NEW_VAR@ - -#======================================================================== -# Nothing of the variables below this line should need to be changed. -# Please check the TARGETS section below to make sure the make targets -# are correct. -#======================================================================== - -#======================================================================== -# The names of the source files is defined in the configure script. -# The object files are used for linking into the final library. -# This will be used when a dist target is added to the Makefile. -# It is not important to specify the directory, as long as it is the -# $(srcdir) or in the generic, win or unix subdirectory. -#======================================================================== - -PKG_SOURCES = @PKG_SOURCES@ -PKG_OBJECTS = @PKG_OBJECTS@ - -PKG_STUB_SOURCES = @PKG_STUB_SOURCES@ -PKG_STUB_OBJECTS = @PKG_STUB_OBJECTS@ - -#======================================================================== -# PKG_TCL_SOURCES identifies Tcl runtime files that are associated with -# this package that need to be installed, if any. -#======================================================================== - -PKG_TCL_SOURCES = @PKG_TCL_SOURCES@ - -#======================================================================== -# This is a list of public header files to be installed, if any. -#======================================================================== - -PKG_HEADERS = @PKG_HEADERS@ - -#======================================================================== -# "PKG_LIB_FILE" refers to the library (dynamic or static as per -# configuration options) composed of the named objects. -#======================================================================== - -PKG_LIB_FILE = @PKG_LIB_FILE@ -PKG_LIB_FILE8 = @PKG_LIB_FILE8@ -PKG_LIB_FILE9 = @PKG_LIB_FILE9@ -PKG_STUB_LIB_FILE = @PKG_STUB_LIB_FILE@ - -lib_BINARIES = $(PKG_LIB_FILE) -BINARIES = $(lib_BINARIES) - -SHELL = @SHELL@ - -srcdir = @srcdir@ -prefix = @prefix@ -exec_prefix = @exec_prefix@ - -bindir = @bindir@ -libdir = @libdir@ -includedir = @includedir@ -datarootdir = @datarootdir@ -runstatedir = @runstatedir@ -datadir = @datadir@ -mandir = @mandir@ - -DESTDIR = - -PKG_DIR = $(PACKAGE_NAME)$(PACKAGE_VERSION) -pkgdatadir = $(datadir)/$(PKG_DIR) -pkglibdir = $(libdir)/$(PKG_DIR) -pkgincludedir = $(includedir)/$(PKG_DIR) - -top_builddir = @abs_top_builddir@ - -INSTALL_OPTIONS = -INSTALL = @INSTALL@ $(INSTALL_OPTIONS) -INSTALL_DATA_DIR = @INSTALL_DATA_DIR@ -INSTALL_DATA = @INSTALL_DATA@ -INSTALL_PROGRAM = @INSTALL_PROGRAM@ -INSTALL_SCRIPT = @INSTALL_SCRIPT@ -INSTALL_LIBRARY = @INSTALL_LIBRARY@ - -PACKAGE_NAME = @PACKAGE_NAME@ -PACKAGE_VERSION = @PACKAGE_VERSION@ -CC = @CC@ -CFLAGS_DEFAULT = @CFLAGS_DEFAULT@ -CFLAGS_WARNING = @CFLAGS_WARNING@ -EXEEXT = @EXEEXT@ -LDFLAGS_DEFAULT = @LDFLAGS_DEFAULT@ -MAKE_LIB = @MAKE_LIB@ -MAKE_STUB_LIB = @MAKE_STUB_LIB@ -OBJEXT = @OBJEXT@ -RANLIB = @RANLIB@ -RANLIB_STUB = @RANLIB_STUB@ -SHLIB_CFLAGS = @SHLIB_CFLAGS@ -SHLIB_LD = @SHLIB_LD@ -SHLIB_LD_LIBS = @SHLIB_LD_LIBS@ -STLIB_LD = @STLIB_LD@ -#TCL_DEFS = @TCL_DEFS@ -TCL_BIN_DIR = @TCL_BIN_DIR@ -TCL_SRC_DIR = @TCL_SRC_DIR@ -#TK_BIN_DIR = @TK_BIN_DIR@ -#TK_SRC_DIR = @TK_SRC_DIR@ - -# Not used, but retained for reference of what libs Tcl required -#TCL_LIBS = @TCL_LIBS@ - -#======================================================================== -# TCLLIBPATH seeds the auto_path in Tcl's init.tcl so we can test our -# package without installing. The other environment variables allow us -# to test against an uninstalled Tcl. Add special env vars that you -# require for testing here (like TCLX_LIBRARY). -#======================================================================== - -EXTRA_PATH = $(top_builddir):$(TCL_BIN_DIR) -#EXTRA_PATH = $(top_builddir):$(TCL_BIN_DIR):$(TK_BIN_DIR) -TCLLIBPATH = $(top_builddir) -TCLSH_ENV = TCL_LIBRARY=`@CYGPATH@ $(TCL_SRC_DIR)/library` -PKG_ENV = @LD_LIBRARY_PATH_VAR@="$(EXTRA_PATH):$(@LD_LIBRARY_PATH_VAR@)" \ - PATH="$(EXTRA_PATH):$(PATH)" \ - TCLLIBPATH="$(TCLLIBPATH)" - -TCLSH_PROG = @TCLSH_PROG@ -TCLSH = $(TCLSH_ENV) $(PKG_ENV) $(TCLSH_PROG) - -#WISH_ENV = TK_LIBRARY=`@CYGPATH@ $(TK_SRC_DIR)/library` -#WISH_PROG = @WISH_PROG@ -#WISH = $(TCLSH_ENV) $(WISH_ENV) $(PKG_ENV) $(WISH_PROG) - -SHARED_BUILD = @SHARED_BUILD@ - -INCLUDES = @PKG_INCLUDES@ @TCL_INCLUDES@ -I. -I$(srcdir)/.. -#INCLUDES = @PKG_INCLUDES@ @TCL_INCLUDES@ @TK_INCLUDES@ @TK_XINCLUDES@ - -PKG_CFLAGS = @PKG_CFLAGS@ - -# TCL_DEFS is not strictly need here, but if you remove it, then you -# must make sure that configure.ac checks for the necessary components -# that your library may use. TCL_DEFS can actually be a problem if -# you do not compile with a similar machine setup as the Tcl core was -# compiled with. -#DEFS = $(TCL_DEFS) @DEFS@ $(PKG_CFLAGS) -DEFS = @DEFS@ $(PKG_CFLAGS) - -# Move pkgIndex.tcl to 'BINARIES' var if it is generated in the Makefile -CONFIG_CLEAN_FILES = Makefile pkgIndex.tcl -CLEANFILES = @CLEANFILES@ - -CPPFLAGS = @CPPFLAGS@ -LIBS = @PKG_LIBS@ @LIBS@ -AR = @AR@ -CFLAGS = @CFLAGS@ -LDFLAGS = @LDFLAGS@ -LDFLAGS_DEFAULT = @LDFLAGS_DEFAULT@ -COMPILE = $(CC) $(DEFS) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) \ - $(CFLAGS_DEFAULT) $(CFLAGS_WARNING) $(SHLIB_CFLAGS) $(CFLAGS) - -GDB = gdb -VALGRIND = valgrind -VALGRINDARGS = --tool=memcheck --num-callers=8 --leak-resolution=high \ - --leak-check=yes --show-reachable=yes -v - -.SUFFIXES: .c .$(OBJEXT) - -#======================================================================== -# Start of user-definable TARGETS section -#======================================================================== - -#======================================================================== -# TEA TARGETS. Please note that the "libraries:" target refers to platform -# independent files, and the "binaries:" target includes executable programs and -# platform-dependent libraries. Modify these targets so that they install -# the various pieces of your package. The make and install rules -# for the BINARIES that you specified above have already been done. -#======================================================================== - -all: binaries libraries doc - -#======================================================================== -# The binaries target builds executable programs, Windows .dll's, unix -# shared/static libraries, and any other platform-dependent files. -# The list of targets to build for "binaries:" is specified at the top -# of the Makefile, in the "BINARIES" variable. -#======================================================================== - -binaries: $(BINARIES) - -libraries: - -#======================================================================== -# Your doc target should differentiate from doc builds (by the developer) -# and doc installs (see install-doc), which just install the docs on the -# end user machine when building from source. -#======================================================================== - -doc: - @echo "If you have documentation to create, place the commands to" - @echo "build the docs in the 'doc:' target. For example:" - @echo " xml2nroff sample.xml > sample.n" - @echo " xml2html sample.xml > sample.html" - -install: all install-binaries install-libraries install-doc - -install-binaries: binaries install-lib-binaries install-bin-binaries - -#======================================================================== -# This rule installs platform-independent files, such as header files. -# The list=...; for p in $$list handles the empty list case x-platform. -#======================================================================== - -install-libraries: libraries - @$(INSTALL_DATA_DIR) "$(DESTDIR)$(includedir)" - @echo "Installing header files in $(DESTDIR)$(includedir)" - @list='$(PKG_HEADERS)'; for i in $$list; do \ - echo "Installing $(srcdir)/$$i" ; \ - $(INSTALL_DATA) $(srcdir)/$$i "$(DESTDIR)$(includedir)" ; \ - done; - -#======================================================================== -# Install documentation. Unix manpages should go in the $(mandir) -# directory. -#======================================================================== - -install-doc: doc - @$(INSTALL_DATA_DIR) "$(DESTDIR)$(mandir)/mann" - @echo "Installing documentation in $(DESTDIR)$(mandir)" - @list='$(srcdir)/doc/*.n'; for i in $$list; do \ - echo "Installing $$i"; \ - $(INSTALL_DATA) $$i "$(DESTDIR)$(mandir)/mann" ; \ - done - -test: binaries libraries - @echo "SQLite TEA distribution does not include tests" - -shell: binaries libraries - @$(TCLSH) $(SCRIPT) - -gdb: - $(TCLSH_ENV) $(PKG_ENV) $(GDB) $(TCLSH_PROG) $(SCRIPT) - -gdb-test: binaries libraries - $(TCLSH_ENV) $(PKG_ENV) $(GDB) \ - --args $(TCLSH_PROG) `@CYGPATH@ $(srcdir)/tests/all.tcl` \ - $(TESTFLAGS) -singleproc 1 \ - -load "package ifneeded $(PACKAGE_NAME) $(PACKAGE_VERSION) \ - [list load `@CYGPATH@ $(PKG_LIB_FILE)` [string totitle $(PACKAGE_NAME)]]" - -valgrind: binaries libraries - $(TCLSH_ENV) $(PKG_ENV) $(VALGRIND) $(VALGRINDARGS) $(TCLSH_PROG) \ - `@CYGPATH@ $(srcdir)/tests/all.tcl` $(TESTFLAGS) - -valgrindshell: binaries libraries - $(TCLSH_ENV) $(PKG_ENV) $(VALGRIND) $(VALGRINDARGS) $(TCLSH_PROG) $(SCRIPT) - -depend: - -#======================================================================== -# $(PKG_LIB_FILE) should be listed as part of the BINARIES variable -# mentioned above. That will ensure that this target is built when you -# run "make binaries". -# -# The $(PKG_OBJECTS) objects are created and linked into the final -# library. In most cases these object files will correspond to the -# source files above. -#======================================================================== - -$(PKG_LIB_FILE): $(PKG_OBJECTS) - -rm -f $(PKG_LIB_FILE) - ${MAKE_LIB} - $(RANLIB) $(PKG_LIB_FILE) - -$(PKG_STUB_LIB_FILE): $(PKG_STUB_OBJECTS) - -rm -f $(PKG_STUB_LIB_FILE) - ${MAKE_STUB_LIB} - $(RANLIB_STUB) $(PKG_STUB_LIB_FILE) - -#======================================================================== -# We need to enumerate the list of .c to .o lines here. -# -# In the following lines, $(srcdir) refers to the toplevel directory -# containing your extension. If your sources are in a subdirectory, -# you will have to modify the paths to reflect this: -# -# sample.$(OBJEXT): $(srcdir)/generic/sample.c -# $(COMPILE) -c `@CYGPATH@ $(srcdir)/generic/sample.c` -o $@ -# -# Setting the VPATH variable to a list of paths will cause the makefile -# to look into these paths when resolving .c to .obj dependencies. -# As necessary, add $(srcdir):$(srcdir)/compat:.... -#======================================================================== - -VPATH = $(srcdir):$(srcdir)/generic:$(srcdir)/unix:$(srcdir)/win:$(srcdir)/macosx - -.c.@OBJEXT@: - $(COMPILE) -c `@CYGPATH@ $<` -o $@ - - -#======================================================================== -# Distribution creation -# You may need to tweak this target to make it work correctly. -#======================================================================== - -#COMPRESS = tar cvf $(PKG_DIR).tar $(PKG_DIR); compress $(PKG_DIR).tar -COMPRESS = tar zcvf $(PKG_DIR).tar.gz $(PKG_DIR) -DIST_ROOT = /tmp/dist -DIST_DIR = $(DIST_ROOT)/$(PKG_DIR) - -DIST_INSTALL_DATA = CPPROG='cp -p' $(INSTALL) -m 644 -DIST_INSTALL_SCRIPT = CPPROG='cp -p' $(INSTALL) -m 755 - -dist-clean: - rm -rf $(DIST_DIR) $(DIST_ROOT)/$(PKG_DIR).tar.* - -dist: dist-clean $(srcdir)/manifest.uuid - $(INSTALL_DATA_DIR) $(DIST_DIR) - - # TEA files - $(DIST_INSTALL_DATA) $(srcdir)/Makefile.in \ - $(srcdir)/aclocal.m4 $(srcdir)/configure.ac \ - $(DIST_DIR)/ - $(DIST_INSTALL_SCRIPT) $(srcdir)/configure $(DIST_DIR)/ - - $(INSTALL_DATA_DIR) $(DIST_DIR)/tclconfig - $(DIST_INSTALL_DATA) $(srcdir)/tclconfig/README.txt \ - $(srcdir)/manifest.uuid \ - $(srcdir)/tclconfig/tcl.m4 $(srcdir)/tclconfig/install-sh \ - $(DIST_DIR)/tclconfig/ - - # Extension files - $(DIST_INSTALL_DATA) \ - $(srcdir)/ChangeLog \ - $(srcdir)/README.sha \ - $(srcdir)/license.terms \ - $(srcdir)/README \ - $(srcdir)/pkgIndex.tcl.in \ - $(DIST_DIR)/ - - list='demos doc generic library macosx tests unix win'; \ - for p in $$list; do \ - if test -d $(srcdir)/$$p ; then \ - $(INSTALL_DATA_DIR) $(DIST_DIR)/$$p; \ - $(DIST_INSTALL_DATA) $(srcdir)/$$p/* $(DIST_DIR)/$$p/; \ - fi; \ - done - - (cd $(DIST_ROOT); $(COMPRESS);) - -#======================================================================== -# End of user-definable section -#======================================================================== - -#======================================================================== -# Don't modify the file to clean here. Instead, set the "CLEANFILES" -# variable in configure.ac -#======================================================================== - -clean: - -test -z "$(BINARIES)" || rm -f $(BINARIES) - -rm -f *.$(OBJEXT) core *.core - -test -z "$(CLEANFILES)" || rm -f $(CLEANFILES) - -distclean: clean - -rm -f *.tab.c - -rm -f $(CONFIG_CLEAN_FILES) - -rm -f config.cache config.log config.status - -#======================================================================== -# Install binary object libraries. On Windows this includes both .dll and -# .lib files. Because the .lib files are not explicitly listed anywhere, -# we need to deduce their existence from the .dll file of the same name. -# Library files go into the lib directory. -# In addition, this will generate the pkgIndex.tcl -# file in the install location (assuming it can find a usable tclsh shell) -# -# You should not have to modify this target. -#======================================================================== - -install-lib-binaries: binaries - @$(INSTALL_DATA_DIR) "$(DESTDIR)$(pkglibdir)" - @list='$(lib_BINARIES)'; for p in $$list; do \ - if test -f $$p; then \ - echo " $(INSTALL_LIBRARY) $$p $(DESTDIR)$(pkglibdir)/$$p"; \ - $(INSTALL_LIBRARY) $$p "$(DESTDIR)$(pkglibdir)/$$p"; \ - ext=`echo $$p|sed -e "s/.*\.//"`; \ - if test "x$$ext" = "xdll"; then \ - lib=`basename $$p|sed -e 's/.[^.]*$$//'`.lib; \ - if test -f $$lib; then \ - echo " $(INSTALL_DATA) $$lib $(DESTDIR)$(pkglibdir)/$$lib"; \ - $(INSTALL_DATA) $$lib "$(DESTDIR)$(pkglibdir)/$$lib"; \ - fi; \ - fi; \ - fi; \ - done - @list='$(PKG_TCL_SOURCES)'; for p in $$list; do \ - if test -f $(srcdir)/$$p; then \ - destp=`basename $$p`; \ - echo " Install $$destp $(DESTDIR)$(pkglibdir)/$$destp"; \ - $(INSTALL_DATA) $(srcdir)/$$p "$(DESTDIR)$(pkglibdir)/$$destp"; \ - fi; \ - done - @if test "x$(SHARED_BUILD)" = "x1"; then \ - echo " Install pkgIndex.tcl $(DESTDIR)$(pkglibdir)"; \ - $(INSTALL_DATA) pkgIndex.tcl "$(DESTDIR)$(pkglibdir)"; \ +all: +# +# Unless this file is named Makefile.in, you are probably looking +# at an automatically generated/filtered copy and should probably not +# edit it. +# +# This makefile is part of the teaish framework, a tool for building +# Tcl extensions, conceptually related to TEA/tclconfig but using the +# Autosetup configuration system instead of the GNU Autotools. +# +# A copy of this makefile gets processed for each extension separately +# and populated with info about how to build, test, and install the +# extension. +# +# Maintenance reminder: this file needs to stay portable with POSIX +# Make, not just GNU Make. Yes, that's unfortunate because it makes +# some things impossible (like skipping over swathes of rules when +# 'make distclean' is invoked). +# + +CC = @CC@ +INSTALL = @BIN_INSTALL@ +INSTALL.noexec = $(INSTALL) -m 0644 + +# +# Var name prefixes: +# +# teaish. => teaish core +# tx. => teaish extension +# +# Vars with a "tx." or "teaish." prefix are all "public" for purposes +# of the extension makefile, but the extension must not any "teaish." +# vars and must only modify "tx." vars where that allowance is +# specifically noted. +# +# Vars with a "teaish__" prefix are "private" and must not be used by +# the extension makefile. They may change semantics or be removed in +# any given teaish build. +# +tx.name = @TEAISH_NAME@ +tx.version = @TEAISH_VERSION@ +tx.name.pkg = @TEAISH_PKGNAME@ +tx.libdir = @TEAISH_LIBDIR_NAME@ +tx.loadPrefix = @TEAISH_LOAD_PREFIX@ +tx.tcl = @TEAISH_TCL@ +tx.makefile = @TEAISH_MAKEFILE@ +tx.makefile.in = @TEAISH_MAKEFILE_IN@ +tx.dll8.basename = @TEAISH_DLL8_BASENAME@ +tx.dll9.basename = @TEAISH_DLL9_BASENAME@ +tx.dll8 = @TEAISH_DLL8@ +tx.dll9 = @TEAISH_DLL9@ +tx.dll = $(tx.dll$(TCL_MAJOR_VERSION)) +tx.dir = @TEAISH_EXT_DIR@ +@if TEAISH_TM_TCL +# Input filename for tcl::tm-style module +tx.tm = @TEAISH_TM_TCL@ +# Target filename for tcl::tm-style installation +tx.tm.tgt = $(tx.name.pkg)-$(tx.version).tm +@endif + +@if TEAISH_DIST_NAME +tx.name.dist = @TEAISH_DIST_NAME@ +@else +tx.name.dist = $(teaish.name) +@endif + +teaish.dir = @abs_top_srcdir@ +#teaish.dir.autosetup = @TEAISH_AUTOSETUP_DIR@ +teaish.makefile = Makefile +teaish.makefile.in = $(teaish.dir)/Makefile.in +teaish__auto.def = $(teaish.dir)/auto.def + +# +# Autotools-conventional vars. We don't actually use these in this +# makefile but some may be referenced by vars imported via +# tclConfig.sh. They are part of the public API and may be reliably +# depended on from teaish.make.in. +# +bindir = @bindir@ +datadir = @datadir@ +exec_prefix = @exec_prefix@ +includedir = @includedir@ +infodir = @infodir@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +prefix = @prefix@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sysconfdir = @sysconfdir@ + + +# +# Vars derived (mostly) from tclConfig.sh. These may be reliably +# used from the extension makefile. +# +TCLSH = @TCLSH_CMD@ +TCL_CONFIG_SH = @TCL_CONFIG_SH@ +TCL_EXEC_PREFIX = @TCL_EXEC_PREFIX@ +TCL_INCLUDE_SPEC = @TCL_INCLUDE_SPEC@ +TCL_LIBS = @TCL_LIBS@ +TCL_LIB_SPEC = @TCL_LIB_SPEC@ +TCL_MAJOR_VERSION = @TCL_MAJOR_VERSION@ +TCL_MINOR_VERSION = @TCL_MINOR_VERSION@ +TCL_PATCH_LEVEL = @TCL_PATCH_LEVEL@ +TCL_PREFIX = @TCL_PREFIX@ +TCL_SHLIB_SUFFIX = @TCL_SHLIB_SUFFIX@ +TCL_STUB_LIB_SPEC = @TCL_STUB_LIB_SPEC@ +TCL_VERSION = @TCL_VERSION@ +TCLLIBDIR = @TCLLIBDIR@ + +# +# CFLAGS.configure = CFLAGS as known at configure-time. +# +# This ordering is deliberate: flags populated via tcl's +# [teaish-cflags-add] should preceed CFLAGS and CPPFLAGS (which +# typically come from the ./configure command-line invocation). +# +CFLAGS.configure = @SH_CFLAGS@ @TEAISH_CFLAGS@ @CFLAGS@ @CPPFLAGS@ $(TCL_INCLUDE_SPEC) +#CFLAGS.configure += -DUSE_TCL_STUBS=1 + +# +# LDFLAGS.configure = LDFLAGS as known at configure-time. +# +# This ordering is deliberate: flags populated via tcl's +# [teaish-ldflags-add] should precede LDFLAGS (which typically +# comes from the ./configure command-line invocation). +# +LDFLAGS.configure = @TEAISH_LDFLAGS@ @LDFLAGS@ + +# +# Linker flags for linkhing a shared library. +# +LDFLAGS.shlib = @SH_LDFLAGS@ + +# +# The following tx.XYZ vars may be populated/modified by teaish.tcl +# and/or teaish.make. +# + +# +# tx.src is the list of source or object files to include in the +# (single) compiler/linker invocation. This will initially contain any +# sources passed to [teaish-src-add], but may also be appended to by +# teaish.make. +# +tx.src =@TEAISH_EXT_SRC@ + +# +# tx.CFLAGS is typically set by teaish.make, whereas TEAISH_CFLAGS +# gets set up via the configure script. +# +tx.CFLAGS = + +# +# tx.LDFLAGS is typically set by teaish.make, whereas TEAISH_LDFLAGS +# gets set up via the configure script. +# +tx.LDFLAGS = + +# +# The list of 'dist' files may be appended to from teaish.make.in. +# It can also be set up from teaish.tcl using [teaish-dist-add] +# and/or [teaish-src-add -dist ...]. +# +tx.dist.files = @TEAISH_DIST_FILES@ + +# List of deps which may trigger an auto-reconfigure. +# +teaish__autogen.deps = \ + $(tx.makefile.in) $(teaish.makefile.in) \ + $(tx.tcl) \ + @TEAISH_PKGINDEX_TCL_IN@ @TEAISH_TM_TCL_IN@ \ + @AUTODEPS@ + +@if TEAISH_MAKEFILE_IN +$(tx.makefile): $(tx.makefile.in) +@endif + +teaish.autoreconfig = \ + @TEAISH_AUTORECONFIG@ + +# +# Problem: when more than one target can invoke TEAISH_AUTORECONFIG, +# we can get parallel reconfigures running. Thus, targets which +# may require reconfigure should depend on... +# +config.log: $(teaish__autogen.deps) + $(teaish.autoreconfig) +# ^^^ We would love to skip this when running [dist]clean, but there's +# no POSIX Make-portable way to do that. GNU Make can. +.PHONY: reconfigure +reconfigure: + $(teaish.autoreconfig) + +$(teaish.makefile): $(teaish__auto.def) $(teaish.makefile.in) \ + @AUTODEPS@ + +@if TEAISH_TESTER_TCL_IN +@TEAISH_TESTER_TCL_IN@: +@TEAISH_TESTER_TCL@: @TEAISH_TESTER_TCL_IN@ +config.log: @TEAISH_TESTER_TCL@ +@endif + +# +# CC variant for compiling Tcl-using sources. +# +CC.tcl = \ + $(CC) -o $@ $(CFLAGS.configure) $(CFLAGS) $(tx.CFLAGS) + +# +# CC variant for linking $(tx.src) into an extension DLL. Note that +# $(tx.src) must come before $(LDFLAGS...) for linking to third-party +# libs to work. +# +CC.dll = \ + $(CC.tcl) $(tx.src) $(LDFLAGS.shlib) \ + $(LDFLAGS.configure) $(LDFLAGS) $(tx.LDFLAGS) $(TCL_STUB_LIB_SPEC) + +@if TEAISH_ENABLE_DLL +# +# The rest of this makefile exists solely to support this brief +# target: the extension shared lib. +# +$(tx.dll): $(tx.src) config.log + @if [ "x" = "x$(tx.src)" ]; then \ + echo "Makefile var tx.src (source/object files) is empty" 1>&2; \ + exit 1; \ fi + $(CC.dll) + +all: $(tx.dll) +@endif # TEAISH_ENABLE_DLL + +tclsh: $(teaish.makefile) config.log + @{ echo "#!/bin/sh"; echo 'exec $(TCLSH) "$$@"'; } > $@ + @chmod +x $@ + @echo "Created $@" + +# +# Run the generated test script. +# +.PHONY: test-pre test-prepre test-core test test-post test-extension +test-extension: # this name is reserved for use by teaish.make[.in] +@if TEAISH_ENABLE_DLL +test-prepre: $(tx.dll) +@endif +@if TEAISH_TESTER_TCL +test-core.args = @TEAISH_TESTER_TCL@ +@if TEAISH_ENABLE_DLL +test-core.args += '$(tx.dll)' '$(tx.loadPrefix)' +@else +test-core.args += '' '' +@endif +test-core.args += @TEAISH_TESTUTIL_TCL@ +test-core: test-pre + $(TCLSH) $(test-core.args) +test-prepre: @TEAISH_TESTER_TCL@ +@else # !TEAISH_TESTER_TCL +test-prepre: +@endif # TEAISH_TESTER_TCL +test-pre: test-prepre +test-core: test-pre +test-post: test-core +test: test-post -#======================================================================== -# Install binary executables (e.g. .exe files and dependent .dll files) -# This is for files that must go in the bin directory (located next to -# wish and tclsh), like dependent .dll files on Windows. -# -# You should not have to modify this target, except to define bin_BINARIES -# above if necessary. -#======================================================================== - -install-bin-binaries: binaries - @$(INSTALL_DATA_DIR) "$(DESTDIR)$(bindir)" - @list='$(bin_BINARIES)'; for p in $$list; do \ - if test -f $$p; then \ - echo " $(INSTALL_PROGRAM) $$p $(DESTDIR)$(bindir)/$$p"; \ - $(INSTALL_PROGRAM) $$p "$(DESTDIR)$(bindir)/$$p"; \ - fi; \ - done - -.SUFFIXES: .c .$(OBJEXT) - -Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status - cd $(top_builddir) \ - && CONFIG_FILES=$@ CONFIG_HEADERS= $(SHELL) ./config.status - -uninstall-binaries: - list='$(lib_BINARIES)'; for p in $$list; do \ - rm -f "$(DESTDIR)$(pkglibdir)/$$p"; \ - done - list='$(PKG_TCL_SOURCES)'; for p in $$list; do \ - p=`basename $$p`; \ - rm -f "$(DESTDIR)$(pkglibdir)/$$p"; \ - done - list='$(bin_BINARIES)'; for p in $$list; do \ - rm -f "$(DESTDIR)$(bindir)/$$p"; \ - done - -.PHONY: all binaries clean depend distclean doc install libraries test -.PHONY: gdb gdb-test valgrind valgrindshell - -# Tell versions [3.59,3.63) of GNU make to not export all variables. -# Otherwise a system limit (for SysV at least) may be exceeded. -.NOEXPORT: +# +# Cleanup rules... +# +#.PHONY: clean-pre clean-core clean-post clean-extension +# +clean-pre: +clean-core: clean-pre + rm -f $(tx.dll8) $(tx.dll9) tclsh +clean-post: clean-core +clean: clean-post + +.PHONY: distclean-pre distclean-core distclean-post clean-extension +distclean-pre: clean +distclean-core: distclean-pre + rm -f Makefile + rm -f config.log config.defines.txt +@if TEAISH_MAKEFILE_IN +@if TEAISH_MAKEFILE + rm -f @TEAISH_MAKEFILE@ +@endif +@endif +@if TEAISH_TESTER_TCL_IN + rm -f @TEAISH_TESTER_TCL@ +@endif +@if TEAISH_PKGINDEX_TCL_IN + rm -f @TEAISH_PKGINDEX_TCL@ +@endif +@if TEAISH_PKGINIT_TCL_IN + rm -f @TEAISH_PKGINIT_TCL@ +@endif +@if TEAISH_TEST_TCL_IN + rm -f @TEAISH_TEST_TCL@ +@endif +distclean-post: distclean-core +distclean: distclean-post +# +# The (dist)clean-extension targets are reserved for use by +# client-side teaish.make. +# +# Client code which wants to clean up extra stuff should do so by +# adding their cleanup target (e.g. clean-extension) as a dependency +# to the 'clean' target, like so: +# +# clean: distclean-extension +# distclean: distclean-extension +# +distclean-extension: +clean-extension: + +# +# Installation rules... +# +@if TEAISH_ENABLE_INSTALL +.PHONY: install-pre install-core install-post install-test install-prepre install-extension +install-extension: # this name is reserved for use by teaish.make + +@if TEAISH_ENABLE_DLL +install-prepre: $(tx.dll) +@else +install-prepre: +@endif + +@if TEAISH_TM_TCL +install-core.tmdir = $(DESTDIR)@TEAISH_TCL_TM_DIR@ +@endif + +install-pre: install-prepre +install-core: install-pre + @if [ ! -d "$(DESTDIR)$(TCLLIBDIR)" ]; then \ + set -x; $(INSTALL) -d "$(DESTDIR)$(TCLLIBDIR)"; \ + fi +# ^^^^ on some platforms, install -d fails if the target already exists. +@if TEAISH_ENABLE_DLL + $(INSTALL) $(tx.dll) "$(DESTDIR)$(TCLLIBDIR)" +@endif +@if TEAISH_PKGINDEX_TCL + $(INSTALL.noexec) "@TEAISH_PKGINDEX_TCL@" "$(DESTDIR)$(TCLLIBDIR)" +@endif +@if TEAISH_PKGINIT_TCL + $(INSTALL.noexec) "@TEAISH_PKGINIT_TCL@" "$(DESTDIR)$(TCLLIBDIR)" +@endif +@if TEAISH_TM_TCL + @if [ ! -d "$(install-core.tmdir)" ]; then \ + set -x; $(INSTALL) -d "$(install-core.tmdir)"; \ + fi + $(INSTALL.noexec) "@TEAISH_TM_TCL@" "$(install-core.tmdir)/$(tx.tm.tgt)" +@endif +install-test: install-core + @echo "Post-install test of [package require $(tx.name.pkg) $(tx.version)]..."; \ + if echo \ + 'set c 0; ' \ + '@TEAISH_POSTINST_PREREQUIRE@' \ + 'if {[catch {package require $(tx.name.pkg) $(tx.version)}]} {incr c};' \ + 'exit $$c' \ + | $(TCLSH) ; then \ + echo "passed"; \ + else \ + echo "FAILED"; \ + exit 1; \ + fi +install-post: install-test +install: install-post + +# +# Uninstall rules... +# +.PHONY: uninstall uninstall-pre uninstall-core uninstall-post uninstall-extension +uninstall-extension: # this name is reserved for use by teaish.make +uninstall-pre: +uninstall-core: uninstall-pre +@if TEAISH_ENABLE_DLL + rm -fr "$(DESTDIR)$(TCLLIBDIR)" +@endif +@if TEAISH_TM_TCL + rm -f "$(DESTDIR)$(install-core.tmdir)/$(tx.tm.tgt)" +@endif + +uninstall-post: uninstall-core + @echo "Uninstalled Tcl extension $(tx.name) $(tx.version)" +uninstall: uninstall-post +@endif # TEAISH_ENABLE_INSTALL + +@if TEAISH_MAKEFILE_IN +Makefile: $(tx.makefile.in) +config.log: $(teaish.makefile.in) +@endif + +# +# Package archive generation ("dist") rules... +# +@if TEAISH_ENABLE_DIST +@if BIN_TAR +@if BIN_ZIP + +# When installing teaish as part of "make dist", we need to run +# configure with similar flags to what we last configured with but we +# must not pass on any extension-specific flags, as those won't be +# recognized when running in --teaish-install mode, causing +# the sub-configure to fail. +dist.flags = --with-tclsh=$(TCLSH) +dist.reconfig = $(teaish.dir)/configure $(dist.flags) + +# Temp dir for dist.zip. Must be different than dist.tgz or else +# parallel builds may hose the dist. +teaish__dist.tmp.zip = teaish__dist_zip +# +# Make a distribution zip file... +# +dist.basename = $(tx.name.dist)-$(tx.version) +dist.zip = $(dist.basename).zip +.PHONY: dist.zip dist.zip-core dist.zip-post +#dist.zip-pre: +# We apparently can't add a pre-hook here, else "make dist" rebuilds +# the archive each time it's run. +$(dist.zip): $(tx.dist.files) + @rm -fr $(teaish__dist.tmp.zip) + @mkdir -p $(teaish__dist.tmp.zip)/$(dist.basename) + @tar cf $(teaish__dist.tmp.zip)/tmp.tar $(tx.dist.files) + @tar xf $(teaish__dist.tmp.zip)/tmp.tar -C $(teaish__dist.tmp.zip)/$(dist.basename) +@if TEAISH_DIST_FULL + @$(dist.reconfig) \ + --teaish-install=$(teaish__dist.tmp.zip)/$(dist.basename) \ + --t-e-d=$(teaish__dist.tmp.zip)/$(dist.basename) >/dev/null +@endif + @rm -f $(dist.basename)/tmp.tar $(dist.zip) + @cd $(teaish__dist.tmp.zip) && zip -q -r ../$(dist.zip) $(dist.basename) + @rm -fr $(teaish__dist.tmp.zip) + @ls -la $(dist.zip) +dist.zip-core: $(dist.zip) +dist.zip-post: dist.zip-core +dist.zip: dist.zip-post +dist: dist.zip +undist-zip: + rm -f $(dist.zip) +undist: undist-zip +@endif #BIN_ZIP + +# +# Make a distribution tarball... +# +teaish__dist.tmp.tgz = teaish__dist_tgz +dist.tgz = $(dist.basename).tar.gz +.PHONY: dist.tgz dist.tgz-core dist.tgz-post +# dist.tgz-pre: +# see notes in dist.zip +$(dist.tgz): $(tx.dist.files) + @rm -fr $(teaish__dist.tmp.tgz) + @mkdir -p $(teaish__dist.tmp.tgz)/$(dist.basename) + @tar cf $(teaish__dist.tmp.tgz)/tmp.tar $(tx.dist.files) + @tar xf $(teaish__dist.tmp.tgz)/tmp.tar -C $(teaish__dist.tmp.tgz)/$(dist.basename) +@if TEAISH_DIST_FULL + @rm -f $(teaish__dist.tmp.tgz)/$(dist.basename)/pkgIndex.tcl.in; # kludge + @$(dist.reconfig) \ + --teaish-install=$(teaish__dist.tmp.tgz)/$(dist.basename) \ + --t-e-d=$(teaish__dist.tmp.zip)/$(dist.basename) >/dev/null +@endif + @rm -f $(dist.basename)/tmp.tar $(dist.tgz) + @cd $(teaish__dist.tmp.tgz) && tar czf ../$(dist.tgz) $(dist.basename) + @rm -fr $(teaish__dist.tmp.tgz) + @ls -la $(dist.tgz) +dist.tgz-core: $(dist.tgz) +dist.tgz-post: dist.tgz-core +dist.tgz: dist.tgz-post +dist: dist.tgz +undist-tgz: + rm -f $(dist.tgz) +undist: undist-tgz +@else #!BIN_TAR +dist: + @echo "The dist rules require tar, which configure did not find." 1>&2; exit 1 +@endif #BIN_TAR +@else #!TEAISH_ENABLE_DIST +undist: +dist: +@if TEAISH_OUT_OF_EXT_TREE + @echo "'dist' can only be used from an extension's home dir" 1>&2; \ + echo "In this case: @TEAISH_EXT_DIR@" 1>&2; exit 1 +@endif +@endif #TEAISH_ENABLE_DIST + +Makefile: @TEAISH_TCL@ + +@if TEAISH_MAKEFILE_CODE +# +# TEAISH_MAKEFILE_CODE may contain literal makefile code, which +# gets pasted verbatim here. Either [define TEAISH_MAKEFILE_CODE +# ...] or use [teaish-make-add] to incrementally build up this +# content. +# +# +@TEAISH_MAKEFILE_CODE@ +# +@endif + +@if TEAISH_MAKEFILE +# +# TEAISH_MAKEFILE[_IN] defines any extension-specific state this file +# needs. +# +# It must set the following vars if they're not already accounted for +# via teaish.tcl. +# +# - tx.src = list of the extension's source files, being sure to +# prefix each with $(tx.dir) (if it's in the same dir as the +# extension) so that out-of-tree builds can find them. Optionally, +# [define] TEAISH_EXT_SRC or pass them to [teaish-src-add]. +# +# It may optionally set the following vars: +# +# - tx.CFLAGS = CFLAGS/CPPFLAGS. Optionally, [define] TEAISH_CFLAGS +# or pass them to [teaish-cflags-add]. +# +# - tx.LDFLAGS = LDFLAGS. Optionally, [define] TEAISH_LDFLAGS or +# pass them to [teaish-ldflags-add]. +# +# It may optionally hook into various targets as documented in +# /doc/extensions.md in the canonical teaish source tree. +# +# Interestingly, we don't have to pre-filter teaish.makefile.in - we +# can just @include it here. That skips its teaish-specific validation +# though. Hmm. +# +# +Makefile: @TEAISH_MAKEFILE@ +@include @TEAISH_MAKEFILE@ +# +@endif diff --git a/autoconf/tea/README.txt b/autoconf/tea/README.txt index b50d4f29a..fb7cb1924 100644 --- a/autoconf/tea/README.txt +++ b/autoconf/tea/README.txt @@ -1,9 +1,30 @@ -This is the SQLite extension for Tcl using the Tcl Extension -Architecture (TEA). +This is the SQLite extension for Tcl using something akin to +the Tcl Extension Architecture (TEA). To build it: ------------------------ A BETTER WAY --------------------------- + ./configure ...flags... -A better way to build the TCL extension for SQLite is to use the +e.g.: + + ./configure --with-tcl=/path/to/tcl/install/root + +or: + + ./configure --with-tclsh=/path/to/tcl/install/root + +Run ./configure --help for the full list of flags. + +The configuration process will fail if tclConfig.sh cannot be found. + +The makefile will only honor CFLAGS and CPPFLAGS passed to the +configure script, not those directly passed to the makefile. + +Then: + + make test install + +----------------------- THE PREFERRED WAY --------------------------- + +The preferred way to build the TCL extension for SQLite is to use the canonical source code tarball. For Unix: ./configure --with-tclsh=$(TCLSH) @@ -20,24 +41,32 @@ step-by-step instructions at the links below for more information: https://sqlite.org/src/doc/trunk/doc/compile-for-unix.md https://sqlite.org/src/doc/trunk/doc/compile-for-windows.md +And info about the extension's Tcl interface can be found at: + + https://sqlite.org/tclsqlite.html + The whole point of the amalgamation-autoconf tarball (in which this -README.txt file is embedded) is to provide a means of compiling -SQLite that does not require first installing TCL and/or "tclsh". -The canonical Makefile in the SQLite source tree provides more -capabilities (such as the the ability to run test cases to ensure -that the build worked) and is better maintained. The only -downside of the canonical Makfile is that it requires a TCL -installation. But if you are wanting to build the TCL extension for -SQLite, then presumably you already have a TCL installation. So why -not just use the more-capable and better-maintained canoncal Makefile? - -This TEA builder is derived from code found at +README.txt file is embedded) is to provide a means of compiling SQLite +that does not require first installing TCL and/or "tclsh". The +canonical Makefile in the SQLite source tree provides more +capabilities (such as the the ability to run test cases to ensure that +the build worked) and is better maintained. The only downside of the +canonical Makefile is that it requires a TCL installation. But if you +are wanting to build the TCL extension for SQLite, then presumably you +already have a TCL installation. So why not just use the more-capable +and better-maintained canoncal Makefile? + +As of version 3.50.0, this build process uses "teaish": + + https://fossil.wanderinghorse.net/r/teaish + +which is conceptually derived from the pre-3.50 toolchain, TEA: http://core.tcl-lang.org/tclconfig http://core.tcl-lang.org/sampleextension -The SQLite developers do not understand how it works. It seems to -work for us. It might also work for you. But we cannot promise that. +It to works for us. It might also work for you. But we cannot +promise that. If you want to use this TEA builder and it works for you, that's fine. But if you have trouble, the first thing you should do is go back @@ -49,13 +78,11 @@ to using the canonical Makefile in the SQLite source tree. UNIX BUILD ========== -Building under most UNIX systems is easy, just run the configure script -and then run make. For more information about the build process, see -the tcl/unix/README file in the Tcl src dist. The following minimal -example will install the extension in the /opt/tcl directory. +Building under most UNIX systems is easy, just run the configure +script and then run make. For example: $ cd sqlite-*-tea - $ ./configure --prefix=/opt/tcl + $ ./configure --with-tcl=/path/to/tcl/install/root $ make $ make install @@ -68,7 +95,6 @@ generating native Windows binaries. Using the Msys + Mingw build tools means that you can use the same configure script as per the Unix build to create a Makefile. See the tcl/win/README file for the URL of the Msys + Mingw download. - If you have VC++ then you may wish to use the files in the win subdirectory and build the extension using just VC++. These files have been designed to be as generic as possible but will require some diff --git a/autoconf/tea/_teaish.tester.tcl.in b/autoconf/tea/_teaish.tester.tcl.in new file mode 100644 index 000000000..59d11f0a8 --- /dev/null +++ b/autoconf/tea/_teaish.tester.tcl.in @@ -0,0 +1,49 @@ +# -*- tcl -*- +# +# Unless this file is named _teaish.tester.tcl.in, you are probably +# looking at an automatically generated/filtered copy and should +# probably not edit it. +# +# This is the wrapper script invoked by teaish's "make test" recipe. +# It gets passed 3 args: +# +# $1 = the DLL name, or "" if the extension has no DLL +# +# $2 = the "load prefix" for Tcl's [load] or empty if $1 is empty +# +# $3 = the /path/to/teaish/tester.tcl (test utility code) +# +@if TEAISH_VSATISFIES_CODE +@TEAISH_VSATISFIES_CODE@ +@endif +if {[llength [lindex $::argv 0]] > 0} { + load [file normalize [lindex $::argv 0]] [lindex $::argv 1]; + # ----^^^^^^^ needed on Haiku when argv 0 is just a filename, else + # load cannot find the file. +} +source -encoding utf-8 [lindex $::argv 2]; # teaish/tester.tcl +@if TEAISH_PKGINIT_TCL +apply {{file} { + set dir [file dirname $::argv0] + source -encoding utf-8 $file +}} [join {@TEAISH_PKGINIT_TCL@}] +@endif +@if TEAISH_TM_TCL +apply {{file} { + set dir [file dirname $::argv0] + source -encoding utf-8 $file +}} [join {@TEAISH_TM_TCL@}] +@endif +@if TEAISH_TEST_TCL +apply {{file} { + # Populate state for [tester.tcl::teaish-build-flag*] + array set ::teaish__BuildFlags @TEAISH__DEFINES_MAP@ + set dir [file normalize [file dirname $file]] + #test-fail "Just testing" + source -encoding utf-8 $file +}} [join {@TEAISH_TEST_TCL@}] +@else # TEAISH_TEST_TCL +# No $TEAISH_TEST_TCL provided, so here's a default test which simply +# loads the extension. +puts {Extension @TEAISH_NAME@ @TEAISH_VERSION@ successfully loaded from @TEAISH_TESTER_TCL@} +@endif diff --git a/autoconf/tea/aclocal.m4 b/autoconf/tea/aclocal.m4 deleted file mode 100644 index 0b057391d..000000000 --- a/autoconf/tea/aclocal.m4 +++ /dev/null @@ -1,9 +0,0 @@ -# -# Include the TEA standard macro set -# - -builtin(include,tclconfig/tcl.m4) - -# -# Add here whatever m4 macros you want to define for your package -# diff --git a/autoconf/tea/auto.def b/autoconf/tea/auto.def new file mode 100644 index 000000000..7170b3d1f --- /dev/null +++ b/autoconf/tea/auto.def @@ -0,0 +1,8 @@ +#/do/not/tclsh +# ^^^ help out editors which guess this file's content type. +# +# Main configure script entry point for the TEA(ish) framework. All +# extension-specific customization goes in teaish.tcl.in or +# teaish.tcl. +use teaish/core +teaish-configure-core diff --git a/autoconf/tea/configure b/autoconf/tea/configure new file mode 100755 index 000000000..47378126f --- /dev/null +++ b/autoconf/tea/configure @@ -0,0 +1,7 @@ +#!/bin/sh +dir0="`dirname "$0"`" +dirA="$dir0/../autosetup" +# This is the case ^^^^^^^^^^^^ in the SQLite "autoconf" bundle. +WRAPPER="$0"; export WRAPPER; exec "`"$dirA/autosetup-find-tclsh"`" \ + "$dirA/autosetup" --teaish-extension-dir="$dir0" \ + "$@" diff --git a/autoconf/tea/configure.ac.in b/autoconf/tea/configure.ac.in deleted file mode 100644 index a13a1e761..000000000 --- a/autoconf/tea/configure.ac.in +++ /dev/null @@ -1,227 +0,0 @@ -#!/bin/bash -norc -dnl This file is an input file used by the GNU "autoconf" program to -dnl generate the file "configure", which is run during Tcl installation -dnl to configure the system for the local environment. - -#----------------------------------------------------------------------- -# Sample configure.ac for Tcl Extensions. The only places you should -# need to modify this file are marked by the string __CHANGE__ -#----------------------------------------------------------------------- - -#----------------------------------------------------------------------- -# __CHANGE__ -# Set your package name and version numbers here. -# -# This initializes the environment with PACKAGE_NAME and PACKAGE_VERSION -# set as provided. These will also be added as -D defs in your Makefile -# so you can encode the package version directly into the source files. -# This will also define a special symbol for Windows (BUILD_ -# so that we create the export library with the dll. -#----------------------------------------------------------------------- - -AC_INIT([sqlite],[@VERSION@]) - -#-------------------------------------------------------------------- -# Call TEA_INIT as the first TEA_ macro to set up initial vars. -# This will define a ${TEA_PLATFORM} variable == "unix" or "windows" -# as well as PKG_LIB_FILE and PKG_STUB_LIB_FILE. -#-------------------------------------------------------------------- - -TEA_INIT() - -AC_CONFIG_AUX_DIR(tclconfig) - -#-------------------------------------------------------------------- -# Load the tclConfig.sh file -#-------------------------------------------------------------------- - -TEA_PATH_TCLCONFIG -TEA_LOAD_TCLCONFIG - -#-------------------------------------------------------------------- -# Load the tkConfig.sh file if necessary (Tk extension) -#-------------------------------------------------------------------- - -#TEA_PATH_TKCONFIG -#TEA_LOAD_TKCONFIG - -#----------------------------------------------------------------------- -# Handle the --prefix=... option by defaulting to what Tcl gave. -# Must be called after TEA_LOAD_TCLCONFIG and before TEA_SETUP_COMPILER. -#----------------------------------------------------------------------- - -TEA_PREFIX - -#----------------------------------------------------------------------- -# Standard compiler checks. -# This sets up CC by using the CC env var, or looks for gcc otherwise. -# This also calls AC_PROG_CC and a few others to create the basic setup -# necessary to compile executables. -#----------------------------------------------------------------------- - -TEA_SETUP_COMPILER - -#----------------------------------------------------------------------- -# __CHANGE__ -# Specify the C source files to compile in TEA_ADD_SOURCES, -# public headers that need to be installed in TEA_ADD_HEADERS, -# stub library C source files to compile in TEA_ADD_STUB_SOURCES, -# and runtime Tcl library files in TEA_ADD_TCL_SOURCES. -# This defines PKG(_STUB)_SOURCES, PKG(_STUB)_OBJECTS, PKG_HEADERS -# and PKG_TCL_SOURCES. -#----------------------------------------------------------------------- - -TEA_ADD_SOURCES([tclsqlite3.c]) -TEA_ADD_HEADERS([]) -TEA_ADD_INCLUDES([]) -TEA_ADD_LIBS([]) -TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS3=1]) -TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS4=1]) -TEA_ADD_CFLAGS([-DSQLITE_ENABLE_FTS5=1]) -TEA_ADD_CFLAGS([-DSQLITE_3_SUFFIX_ONLY=1]) -TEA_ADD_CFLAGS([-DSQLITE_ENABLE_RTREE=1]) -TEA_ADD_CFLAGS([-DSQLITE_ENABLE_GEOPOLY=1]) -TEA_ADD_CFLAGS([-DSQLITE_ENABLE_MATH_FUNCTIONS=1]) -TEA_ADD_CFLAGS([-DSQLITE_ENABLE_DESERIALIZE=1]) -TEA_ADD_CFLAGS([-DSQLITE_ENABLE_DBPAGE_VTAB=1]) -TEA_ADD_CFLAGS([-DSQLITE_ENABLE_BYTECODE_VTAB=1]) -TEA_ADD_CFLAGS([-DSQLITE_ENABLE_DBSTAT_VTAB=1]) -TEA_ADD_STUB_SOURCES([]) -TEA_ADD_TCL_SOURCES([]) - -#-------------------------------------------------------------------- -# The --with-system-sqlite causes the TCL bindings to SQLite to use -# the system shared library for SQLite rather than statically linking -# against its own private copy. This is dangerous and leads to -# undersirable dependences and is not recommended. -# Patchs from rmax. -#-------------------------------------------------------------------- -AC_ARG_WITH([system-sqlite], - [AS_HELP_STRING([--with-system-sqlite], - [use a system-supplied libsqlite3 instead of the bundled one])], - [], [with_system_sqlite=no]) -if test x$with_system_sqlite != xno; then - AC_CHECK_HEADER([sqlite3.h], - [AC_CHECK_LIB([sqlite3],[sqlite3_initialize], - [AC_DEFINE(USE_SYSTEM_SQLITE) - LIBS="$LIBS -lsqlite3"])]) -fi - -#-------------------------------------------------------------------- -# __CHANGE__ -# -# You can add more files to clean if your extension creates any extra -# files by extending CLEANFILES. -# Add pkgIndex.tcl if it is generated in the Makefile instead of ./configure -# and change Makefile.in to move it from CONFIG_CLEAN_FILES to BINARIES var. -# -# A few miscellaneous platform-specific items: -# TEA_ADD_* any platform specific compiler/build info here. -#-------------------------------------------------------------------- - -#CLEANFILES="$CLEANFILES pkgIndex.tcl" -if test "${TEA_PLATFORM}" = "windows" ; then - # Ensure no empty if clauses - : - #TEA_ADD_SOURCES([win/winFile.c]) - #TEA_ADD_INCLUDES([-I\"$(${CYGPATH} ${srcdir}/win)\"]) -else - # Ensure no empty else clauses - : - #TEA_ADD_SOURCES([unix/unixFile.c]) - #TEA_ADD_LIBS([-lsuperfly]) -fi - -#-------------------------------------------------------------------- -# __CHANGE__ -# Choose which headers you need. Extension authors should try very -# hard to only rely on the Tcl public header files. Internal headers -# contain private data structures and are subject to change without -# notice. -# This MUST be called after TEA_LOAD_TCLCONFIG / TEA_LOAD_TKCONFIG -#-------------------------------------------------------------------- - -TEA_PUBLIC_TCL_HEADERS -#TEA_PRIVATE_TCL_HEADERS - -#TEA_PUBLIC_TK_HEADERS -#TEA_PRIVATE_TK_HEADERS -#TEA_PATH_X - -#-------------------------------------------------------------------- -# Check whether --enable-threads or --disable-threads was given. -# This auto-enables if Tcl was compiled threaded. -#-------------------------------------------------------------------- - -TEA_ENABLE_THREADS -if test "${TCL_THREADS}" = "1" ; then - AC_DEFINE(SQLITE_THREADSAFE, 1, [Trigger sqlite threadsafe build]) - # Not automatically added by Tcl because its assumed Tcl links to them, - # but it may not if it isn't really a threaded build. - TEA_ADD_LIBS([$THREADS_LIBS]) -else - AC_DEFINE(SQLITE_THREADSAFE, 0, [Trigger sqlite non-threadsafe build]) -fi - -#-------------------------------------------------------------------- -# The statement below defines a collection of symbols related to -# building as a shared library instead of a static library. -#-------------------------------------------------------------------- - -TEA_ENABLE_SHARED - -#-------------------------------------------------------------------- -# This macro figures out what flags to use with the compiler/linker -# when building shared/static debug/optimized objects. This information -# can be taken from the tclConfig.sh file, but this figures it all out. -#-------------------------------------------------------------------- - -TEA_CONFIG_CFLAGS - -#-------------------------------------------------------------------- -# Set the default compiler switches based on the --enable-symbols option. -#-------------------------------------------------------------------- - -TEA_ENABLE_SYMBOLS - -#-------------------------------------------------------------------- -# This macro generates a line to use when building a library. It -# depends on values set by the TEA_ENABLE_SHARED, TEA_ENABLE_SYMBOLS, -# and TEA_LOAD_TCLCONFIG macros above. -#-------------------------------------------------------------------- - -TEA_MAKE_LIB - -#-------------------------------------------------------------------- -# Determine the name of the tclsh and/or wish executables in the -# Tcl and Tk build directories or the location they were installed -# into. These paths are used to support running test cases only, -# the Makefile should not be making use of these paths to generate -# a pkgIndex.tcl file or anything else at extension build time. -#-------------------------------------------------------------------- - -TEA_PROG_TCLSH -#TEA_PROG_WISH - -#-------------------------------------------------------------------- -# Setup a *Config.sh.in configuration file. -#-------------------------------------------------------------------- - -#TEA_EXPORT_CONFIG([sample]) -#AC_SUBST(SAMPLE_VAR) - -#-------------------------------------------------------------------- -# Specify files to substitute AC variables in. You may alternatively -# have a special pkgIndex.tcl.in or other files which require -# substituting the AC variables in. Include these here. -#-------------------------------------------------------------------- - -AC_CONFIG_FILES([Makefile pkgIndex.tcl]) -#AC_CONFIG_FILES([sampleConfig.sh]) - -#-------------------------------------------------------------------- -# Finally, substitute all of the various values into the files -# specified with AC_CONFIG_FILES. -#-------------------------------------------------------------------- - -AC_OUTPUT diff --git a/autoconf/tea/doc/sqlite3.n b/autoconf/tea/doc/sqlite3.n index 13913e558..351404634 100644 --- a/autoconf/tea/doc/sqlite3.n +++ b/autoconf/tea/doc/sqlite3.n @@ -11,5 +11,5 @@ SQLite3 is a self-contains, zero-configuration, transactional SQL database engine. This extension provides an easy to use interface for accessing SQLite database files from Tcl. .PP -For full documentation see \fIhttp://www.sqlite.org/\fR and -in particular \fIhttp://www.sqlite.org/tclsqlite.html\fR. +For full documentation see \fIhttps://sqlite.org/\fR and +in particular \fIhttps://sqlite.org/tclsqlite.html\fR. diff --git a/autoconf/tea/pkgIndex.tcl.in b/autoconf/tea/pkgIndex.tcl.in index 666812dee..c93fcc685 100644 --- a/autoconf/tea/pkgIndex.tcl.in +++ b/autoconf/tea/pkgIndex.tcl.in @@ -1,10 +1,40 @@ # -*- tcl -*- -# Tcl package index file, version 1.1 +# Tcl package index file # +# Unless this file is named pkgIndex.tcl.in, you are probably looking +# at an automatically generated/filtered copy and should probably not +# edit it. +# +# Adapted from https://core.tcl-lang.org/tcltls +@if TEAISH_VSATISFIES_CODE +@TEAISH_VSATISFIES_CODE@ +@endif if {[package vsatisfies [package provide Tcl] 9.0-]} { - package ifneeded sqlite3 @PACKAGE_VERSION@ \ - [list load [file join $dir @PKG_LIB_FILE9@] Sqlite3] + package ifneeded {@TEAISH_PKGNAME@} {@TEAISH_VERSION@} [list apply {{dir} { +@if TEAISH_ENABLE_DLL + load [file join $dir {@TEAISH_DLL9@}] @TEAISH_LOAD_PREFIX@ +@endif +@if TEAISH_PKGINIT_TCL_TAIL + set initScript [file join $dir {@TEAISH_PKGINIT_TCL_TAIL@}] + if {[file exists $initScript]} { + source -encoding utf-8 $initScript + } +@endif + }} $dir] } else { - package ifneeded sqlite3 @PACKAGE_VERSION@ \ - [list load [file join $dir @PKG_LIB_FILE8@] Sqlite3] + package ifneeded {@TEAISH_PKGNAME@} {@TEAISH_VERSION@} [list apply {{dir} { +@if TEAISH_ENABLE_DLL + if {[string tolower [file extension {@TEAISH_DLL8@}]] in [list .dll .dylib .so]} { + load [file join $dir {@TEAISH_DLL8@}] @TEAISH_LOAD_PREFIX@ + } else { + load {} @TEAISH_LOAD_PREFIX@ + } +@endif +@if TEAISH_PKGINIT_TCL_TAIL + set initScript [file join $dir {@TEAISH_PKGINIT_TCL_TAIL@}] + if {[file exists $initScript]} { + source -encoding utf-8 $initScript + } +@endif + }} $dir] } diff --git a/autoconf/tea/tclconfig/install-sh b/autoconf/tea/tclconfig/install-sh deleted file mode 100644 index ec298b537..000000000 --- a/autoconf/tea/tclconfig/install-sh +++ /dev/null @@ -1,541 +0,0 @@ -#!/bin/sh -# install - install a program, script, or datafile - -scriptversion=2020-11-14.01; # UTC - -# This originates from X11R5 (mit/util/scripts/install.sh), which was -# later released in X11R6 (xc/config/util/install.sh) with the -# following copyright and license. -# -# Copyright (C) 1994 X Consortium -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN -# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC- -# TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# -# Except as contained in this notice, the name of the X Consortium shall not -# be used in advertising or otherwise to promote the sale, use or other deal- -# ings in this Software without prior written authorization from the X Consor- -# tium. -# -# -# FSF changes to this file are in the public domain. -# -# Calling this script install-sh is preferred over install.sh, to prevent -# 'make' implicit rules from creating a file called install from it -# when there is no Makefile. -# -# This script is compatible with the BSD install script, but was written -# from scratch. - -tab=' ' -nl=' -' -IFS=" $tab$nl" - -# Set DOITPROG to "echo" to test this script. - -doit=${DOITPROG-} -doit_exec=${doit:-exec} - -# Put in absolute file names if you don't have them in your path; -# or use environment vars. - -chgrpprog=${CHGRPPROG-chgrp} -chmodprog=${CHMODPROG-chmod} -chownprog=${CHOWNPROG-chown} -cmpprog=${CMPPROG-cmp} -cpprog=${CPPROG-cp} -mkdirprog=${MKDIRPROG-mkdir} -mvprog=${MVPROG-mv} -rmprog=${RMPROG-rm} -stripprog=${STRIPPROG-strip} - -posix_mkdir= - -# Desired mode of installed file. -mode=0755 - -# Create dirs (including intermediate dirs) using mode 755. -# This is like GNU 'install' as of coreutils 8.32 (2020). -mkdir_umask=22 - -backupsuffix= -chgrpcmd= -chmodcmd=$chmodprog -chowncmd= -mvcmd=$mvprog -rmcmd="$rmprog -f" -stripcmd= - -src= -dst= -dir_arg= -dst_arg= - -copy_on_change=false -is_target_a_directory=possibly - -usage="\ -Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE - or: $0 [OPTION]... SRCFILES... DIRECTORY - or: $0 [OPTION]... -t DIRECTORY SRCFILES... - or: $0 [OPTION]... -d DIRECTORIES... - -In the 1st form, copy SRCFILE to DSTFILE. -In the 2nd and 3rd, copy all SRCFILES to DIRECTORY. -In the 4th, create DIRECTORIES. - -Options: - --help display this help and exit. - --version display version info and exit. - - -c (ignored) - -C install only if different (preserve data modification time) - -d create directories instead of installing files. - -g GROUP $chgrpprog installed files to GROUP. - -m MODE $chmodprog installed files to MODE. - -o USER $chownprog installed files to USER. - -p pass -p to $cpprog. - -s $stripprog installed files. - -S SUFFIX attempt to back up existing files, with suffix SUFFIX. - -t DIRECTORY install into DIRECTORY. - -T report an error if DSTFILE is a directory. - -Environment variables override the default commands: - CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG - RMPROG STRIPPROG - -By default, rm is invoked with -f; when overridden with RMPROG, -it's up to you to specify -f if you want it. - -If -S is not specified, no backups are attempted. - -Email bug reports to bug-automake@gnu.org. -Automake home page: https://www.gnu.org/software/automake/ -" - -while test $# -ne 0; do - case $1 in - -c) ;; - - -C) copy_on_change=true;; - - -d) dir_arg=true;; - - -g) chgrpcmd="$chgrpprog $2" - shift;; - - --help) echo "$usage"; exit $?;; - - -m) mode=$2 - case $mode in - *' '* | *"$tab"* | *"$nl"* | *'*'* | *'?'* | *'['*) - echo "$0: invalid mode: $mode" >&2 - exit 1;; - esac - shift;; - - -o) chowncmd="$chownprog $2" - shift;; - - -p) cpprog="$cpprog -p";; - - -s) stripcmd=$stripprog;; - - -S) backupsuffix="$2" - shift;; - - -t) - is_target_a_directory=always - dst_arg=$2 - # Protect names problematic for 'test' and other utilities. - case $dst_arg in - -* | [=\(\)!]) dst_arg=./$dst_arg;; - esac - shift;; - - -T) is_target_a_directory=never;; - - --version) echo "$0 $scriptversion"; exit $?;; - - --) shift - break;; - - -*) echo "$0: invalid option: $1" >&2 - exit 1;; - - *) break;; - esac - shift -done - -# We allow the use of options -d and -T together, by making -d -# take the precedence; this is for compatibility with GNU install. - -if test -n "$dir_arg"; then - if test -n "$dst_arg"; then - echo "$0: target directory not allowed when installing a directory." >&2 - exit 1 - fi -fi - -if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then - # When -d is used, all remaining arguments are directories to create. - # When -t is used, the destination is already specified. - # Otherwise, the last argument is the destination. Remove it from $@. - for arg - do - if test -n "$dst_arg"; then - # $@ is not empty: it contains at least $arg. - set fnord "$@" "$dst_arg" - shift # fnord - fi - shift # arg - dst_arg=$arg - # Protect names problematic for 'test' and other utilities. - case $dst_arg in - -* | [=\(\)!]) dst_arg=./$dst_arg;; - esac - done -fi - -if test $# -eq 0; then - if test -z "$dir_arg"; then - echo "$0: no input file specified." >&2 - exit 1 - fi - # It's OK to call 'install-sh -d' without argument. - # This can happen when creating conditional directories. - exit 0 -fi - -if test -z "$dir_arg"; then - if test $# -gt 1 || test "$is_target_a_directory" = always; then - if test ! -d "$dst_arg"; then - echo "$0: $dst_arg: Is not a directory." >&2 - exit 1 - fi - fi -fi - -if test -z "$dir_arg"; then - do_exit='(exit $ret); exit $ret' - trap "ret=129; $do_exit" 1 - trap "ret=130; $do_exit" 2 - trap "ret=141; $do_exit" 13 - trap "ret=143; $do_exit" 15 - - # Set umask so as not to create temps with too-generous modes. - # However, 'strip' requires both read and write access to temps. - case $mode in - # Optimize common cases. - *644) cp_umask=133;; - *755) cp_umask=22;; - - *[0-7]) - if test -z "$stripcmd"; then - u_plus_rw= - else - u_plus_rw='% 200' - fi - cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;; - *) - if test -z "$stripcmd"; then - u_plus_rw= - else - u_plus_rw=,u+rw - fi - cp_umask=$mode$u_plus_rw;; - esac -fi - -for src -do - # Protect names problematic for 'test' and other utilities. - case $src in - -* | [=\(\)!]) src=./$src;; - esac - - if test -n "$dir_arg"; then - dst=$src - dstdir=$dst - test -d "$dstdir" - dstdir_status=$? - # Don't chown directories that already exist. - if test $dstdir_status = 0; then - chowncmd="" - fi - else - - # Waiting for this to be detected by the "$cpprog $src $dsttmp" command - # might cause directories to be created, which would be especially bad - # if $src (and thus $dsttmp) contains '*'. - if test ! -f "$src" && test ! -d "$src"; then - echo "$0: $src does not exist." >&2 - exit 1 - fi - - if test -z "$dst_arg"; then - echo "$0: no destination specified." >&2 - exit 1 - fi - dst=$dst_arg - - # If destination is a directory, append the input filename. - if test -d "$dst"; then - if test "$is_target_a_directory" = never; then - echo "$0: $dst_arg: Is a directory" >&2 - exit 1 - fi - dstdir=$dst - dstbase=`basename "$src"` - case $dst in - */) dst=$dst$dstbase;; - *) dst=$dst/$dstbase;; - esac - dstdir_status=0 - else - dstdir=`dirname "$dst"` - test -d "$dstdir" - dstdir_status=$? - fi - fi - - case $dstdir in - */) dstdirslash=$dstdir;; - *) dstdirslash=$dstdir/;; - esac - - obsolete_mkdir_used=false - - if test $dstdir_status != 0; then - case $posix_mkdir in - '') - # With -d, create the new directory with the user-specified mode. - # Otherwise, rely on $mkdir_umask. - if test -n "$dir_arg"; then - mkdir_mode=-m$mode - else - mkdir_mode= - fi - - posix_mkdir=false - # The $RANDOM variable is not portable (e.g., dash). Use it - # here however when possible just to lower collision chance. - tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$ - - trap ' - ret=$? - rmdir "$tmpdir/a/b" "$tmpdir/a" "$tmpdir" 2>/dev/null - exit $ret - ' 0 - - # Because "mkdir -p" follows existing symlinks and we likely work - # directly in world-writeable /tmp, make sure that the '$tmpdir' - # directory is successfully created first before we actually test - # 'mkdir -p'. - if (umask $mkdir_umask && - $mkdirprog $mkdir_mode "$tmpdir" && - exec $mkdirprog $mkdir_mode -p -- "$tmpdir/a/b") >/dev/null 2>&1 - then - if test -z "$dir_arg" || { - # Check for POSIX incompatibilities with -m. - # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or - # other-writable bit of parent directory when it shouldn't. - # FreeBSD 6.1 mkdir -m -p sets mode of existing directory. - test_tmpdir="$tmpdir/a" - ls_ld_tmpdir=`ls -ld "$test_tmpdir"` - case $ls_ld_tmpdir in - d????-?r-*) different_mode=700;; - d????-?--*) different_mode=755;; - *) false;; - esac && - $mkdirprog -m$different_mode -p -- "$test_tmpdir" && { - ls_ld_tmpdir_1=`ls -ld "$test_tmpdir"` - test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1" - } - } - then posix_mkdir=: - fi - rmdir "$tmpdir/a/b" "$tmpdir/a" "$tmpdir" - else - # Remove any dirs left behind by ancient mkdir implementations. - rmdir ./$mkdir_mode ./-p ./-- "$tmpdir" 2>/dev/null - fi - trap '' 0;; - esac - - if - $posix_mkdir && ( - umask $mkdir_umask && - $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir" - ) - then : - else - - # mkdir does not conform to POSIX, - # or it failed possibly due to a race condition. Create the - # directory the slow way, step by step, checking for races as we go. - - case $dstdir in - /*) prefix='/';; - [-=\(\)!]*) prefix='./';; - *) prefix='';; - esac - - oIFS=$IFS - IFS=/ - set -f - set fnord $dstdir - shift - set +f - IFS=$oIFS - - prefixes= - - for d - do - test X"$d" = X && continue - - prefix=$prefix$d - if test -d "$prefix"; then - prefixes= - else - if $posix_mkdir; then - (umask $mkdir_umask && - $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break - # Don't fail if two instances are running concurrently. - test -d "$prefix" || exit 1 - else - case $prefix in - *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;; - *) qprefix=$prefix;; - esac - prefixes="$prefixes '$qprefix'" - fi - fi - prefix=$prefix/ - done - - if test -n "$prefixes"; then - # Don't fail if two instances are running concurrently. - (umask $mkdir_umask && - eval "\$doit_exec \$mkdirprog $prefixes") || - test -d "$dstdir" || exit 1 - obsolete_mkdir_used=true - fi - fi - fi - - if test -n "$dir_arg"; then - { test -z "$chowncmd" || $doit $chowncmd "$dst"; } && - { test -z "$chgrpcmd" || $doit $chgrpcmd "$dst"; } && - { test "$obsolete_mkdir_used$chowncmd$chgrpcmd" = false || - test -z "$chmodcmd" || $doit $chmodcmd $mode "$dst"; } || exit 1 - else - - # Make a couple of temp file names in the proper directory. - dsttmp=${dstdirslash}_inst.$$_ - rmtmp=${dstdirslash}_rm.$$_ - - # Trap to clean up those temp files at exit. - trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0 - - # Copy the file name to the temp name. - (umask $cp_umask && - { test -z "$stripcmd" || { - # Create $dsttmp read-write so that cp doesn't create it read-only, - # which would cause strip to fail. - if test -z "$doit"; then - : >"$dsttmp" # No need to fork-exec 'touch'. - else - $doit touch "$dsttmp" - fi - } - } && - $doit_exec $cpprog "$src" "$dsttmp") && - - # and set any options; do chmod last to preserve setuid bits. - # - # If any of these fail, we abort the whole thing. If we want to - # ignore errors from any of these, just make sure not to ignore - # errors from the above "$doit $cpprog $src $dsttmp" command. - # - { test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } && - { test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } && - { test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } && - { test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } && - - # If -C, don't bother to copy if it wouldn't change the file. - if $copy_on_change && - old=`LC_ALL=C ls -dlL "$dst" 2>/dev/null` && - new=`LC_ALL=C ls -dlL "$dsttmp" 2>/dev/null` && - set -f && - set X $old && old=:$2:$4:$5:$6 && - set X $new && new=:$2:$4:$5:$6 && - set +f && - test "$old" = "$new" && - $cmpprog "$dst" "$dsttmp" >/dev/null 2>&1 - then - rm -f "$dsttmp" - else - # If $backupsuffix is set, and the file being installed - # already exists, attempt a backup. Don't worry if it fails, - # e.g., if mv doesn't support -f. - if test -n "$backupsuffix" && test -f "$dst"; then - $doit $mvcmd -f "$dst" "$dst$backupsuffix" 2>/dev/null - fi - - # Rename the file to the real destination. - $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null || - - # The rename failed, perhaps because mv can't rename something else - # to itself, or perhaps because mv is so ancient that it does not - # support -f. - { - # Now remove or move aside any old file at destination location. - # We try this two ways since rm can't unlink itself on some - # systems and the destination file might be busy for other - # reasons. In this case, the final cleanup might fail but the new - # file should still install successfully. - { - test ! -f "$dst" || - $doit $rmcmd "$dst" 2>/dev/null || - { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null && - { $doit $rmcmd "$rmtmp" 2>/dev/null; :; } - } || - { echo "$0: cannot unlink or rename $dst" >&2 - (exit 1); exit 1 - } - } && - - # Now rename the file to the real destination. - $doit $mvcmd "$dsttmp" "$dst" - } - fi || exit 1 - - trap '' 0 - fi -done - -# Local variables: -# eval: (add-hook 'before-save-hook 'time-stamp) -# time-stamp-start: "scriptversion=" -# time-stamp-format: "%:y-%02m-%02d.%02H" -# time-stamp-time-zone: "UTC0" -# time-stamp-end: "; # UTC" -# End: diff --git a/autoconf/tea/tclconfig/tcl.m4 b/autoconf/tea/tclconfig/tcl.m4 deleted file mode 100644 index 237d50a7b..000000000 --- a/autoconf/tea/tclconfig/tcl.m4 +++ /dev/null @@ -1,4119 +0,0 @@ -# tcl.m4 -- -# -# This file provides a set of autoconf macros to help TEA-enable -# a Tcl extension. -# -# Copyright (c) 1999-2000 Ajuba Solutions. -# Copyright (c) 2002-2005 ActiveState Corporation. -# -# See the file "license.terms" for information on usage and redistribution -# of this file, and for a DISCLAIMER OF ALL WARRANTIES. - -AC_PREREQ([2.69]) - -# Possible values for key variables defined: -# -# TEA_WINDOWINGSYSTEM - win32 aqua x11 (mirrors 'tk windowingsystem') -# TEA_PLATFORM - windows unix -# TEA_TK_EXTENSION - True if this is a Tk extension -# - -#------------------------------------------------------------------------ -# TEA_PATH_TCLCONFIG -- -# -# Locate the tclConfig.sh file and perform a sanity check on -# the Tcl compile flags -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --with-tcl=... -# -# Defines the following vars: -# TCL_BIN_DIR Full path to the directory containing -# the tclConfig.sh file -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_PATH_TCLCONFIG], [ - dnl TEA specific: Make sure we are initialized - AC_REQUIRE([TEA_INIT]) - # - # Ok, lets find the tcl configuration - # First, look for one uninstalled. - # the alternative search directory is invoked by --with-tcl - # - - if test x"${no_tcl}" = x ; then - # we reset no_tcl in case something fails here - no_tcl=true - AC_ARG_WITH(tcl, - AS_HELP_STRING([--with-tcl], - [directory containing tcl configuration (tclConfig.sh)]), - [with_tclconfig="${withval}"]) - AC_ARG_WITH(tcl8, - AS_HELP_STRING([--with-tcl8], - [Compile for Tcl8 in Tcl9 environment]), - [with_tcl8="${withval}"]) - AC_MSG_CHECKING([for Tcl configuration]) - AC_CACHE_VAL(ac_cv_c_tclconfig,[ - - # First check to see if --with-tcl was specified. - if test x"${with_tclconfig}" != x ; then - case "${with_tclconfig}" in - */tclConfig.sh ) - if test -f "${with_tclconfig}"; then - AC_MSG_WARN([--with-tcl argument should refer to directory containing tclConfig.sh, not to tclConfig.sh itself]) - with_tclconfig="`echo "${with_tclconfig}" | sed 's!/tclConfig\.sh$!!'`" - fi ;; - esac - if test -f "${with_tclconfig}/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd "${with_tclconfig}"; pwd)`" - else - AC_MSG_ERROR([${with_tclconfig} directory doesn't contain tclConfig.sh]) - fi - fi - - # then check for a private Tcl installation - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in \ - ../tcl \ - `ls -dr ../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - ../../tcl \ - `ls -dr ../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../tcl[[8-9]].[[0-9]]* 2>/dev/null` \ - ../../../tcl \ - `ls -dr ../../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../../../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../../tcl[[8-9]].[[0-9]]* 2>/dev/null` ; do - if test "${TEA_PLATFORM}" = "windows" \ - -a -f "$i/win/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i/win; pwd)`" - break - fi - if test -f "$i/unix/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i/unix; pwd)`" - break - fi - done - fi - - # on Darwin, check in Framework installation locations - if test "`uname -s`" = "Darwin" -a x"${ac_cv_c_tclconfig}" = x ; then - for i in `ls -d ~/Library/Frameworks 2>/dev/null` \ - `ls -d /Library/Frameworks 2>/dev/null` \ - `ls -d /Network/Library/Frameworks 2>/dev/null` \ - `ls -d /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/Library/Frameworks/Tcl.framework 2>/dev/null` \ - `ls -d /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/Network/Library/Frameworks/Tcl.framework 2>/dev/null` \ - `ls -d /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/System/Library/Frameworks/Tcl.framework 2>/dev/null` \ - ; do - if test -f "$i/Tcl.framework/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i/Tcl.framework; pwd)`" - break - fi - done - fi - - # TEA specific: on Windows, check in common installation locations - if test "${TEA_PLATFORM}" = "windows" \ - -a x"${ac_cv_c_tclconfig}" = x ; then - for i in `ls -d C:/Tcl/lib 2>/dev/null` \ - `ls -d C:/Progra~1/Tcl/lib 2>/dev/null` \ - ; do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i; pwd)`" - break - fi - done - fi - - # check in a few common install locations - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in `ls -d ${libdir} 2>/dev/null` \ - `ls -d ${exec_prefix}/lib 2>/dev/null` \ - `ls -d ${prefix}/lib 2>/dev/null` \ - `ls -d /usr/local/lib 2>/dev/null` \ - `ls -d /usr/contrib/lib 2>/dev/null` \ - `ls -d /usr/pkg/lib 2>/dev/null` \ - `ls -d /usr/lib 2>/dev/null` \ - `ls -d /usr/lib64 2>/dev/null` \ - `ls -d /usr/lib/tcl9.0 2>/dev/null` \ - `ls -d /usr/lib/tcl8.7 2>/dev/null` \ - `ls -d /usr/lib/tcl8.6 2>/dev/null` \ - `ls -d /usr/lib/tcl8.5 2>/dev/null` \ - `ls -d /usr/local/lib/tcl9.0 2>/dev/null` \ - `ls -d /usr/local/lib/tcl8.7 2>/dev/null` \ - `ls -d /usr/local/lib/tcl8.6 2>/dev/null` \ - `ls -d /usr/local/lib/tcl8.5 2>/dev/null` \ - `ls -d /usr/local/lib/tcl/tcl9.0 2>/dev/null` \ - `ls -d /usr/local/lib/tcl/tcl8.7 2>/dev/null` \ - `ls -d /usr/local/lib/tcl/tcl8.6 2>/dev/null` \ - `ls -d /usr/local/lib/tcl/tcl8.5 2>/dev/null` \ - ; do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i; pwd)`" - break - fi - done - fi - - # check in a few other private locations - if test x"${ac_cv_c_tclconfig}" = x ; then - for i in \ - ${srcdir}/../tcl \ - `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]]* 2>/dev/null` ; do - if test "${TEA_PLATFORM}" = "windows" \ - -a -f "$i/win/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i/win; pwd)`" - break - fi - if test -f "$i/unix/tclConfig.sh" ; then - ac_cv_c_tclconfig="`(cd $i/unix; pwd)`" - break - fi - done - fi - ]) - - if test x"${ac_cv_c_tclconfig}" = x ; then - TCL_BIN_DIR="# no Tcl configs found" - AC_MSG_ERROR([Can't find Tcl configuration definitions. Use --with-tcl to specify a directory containing tclConfig.sh]) - else - no_tcl= - TCL_BIN_DIR="${ac_cv_c_tclconfig}" - AC_MSG_RESULT([found ${TCL_BIN_DIR}/tclConfig.sh]) - fi - fi -]) - -#------------------------------------------------------------------------ -# TEA_PATH_TKCONFIG -- -# -# Locate the tkConfig.sh file -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --with-tk=... -# -# Defines the following vars: -# TK_BIN_DIR Full path to the directory containing -# the tkConfig.sh file -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_PATH_TKCONFIG], [ - # - # Ok, lets find the tk configuration - # First, look for one uninstalled. - # the alternative search directory is invoked by --with-tk - # - - if test x"${no_tk}" = x ; then - # we reset no_tk in case something fails here - no_tk=true - AC_ARG_WITH(tk, - AS_HELP_STRING([--with-tk], - [directory containing tk configuration (tkConfig.sh)]), - [with_tkconfig="${withval}"]) - AC_MSG_CHECKING([for Tk configuration]) - AC_CACHE_VAL(ac_cv_c_tkconfig,[ - - # First check to see if --with-tkconfig was specified. - if test x"${with_tkconfig}" != x ; then - case "${with_tkconfig}" in - */tkConfig.sh ) - if test -f "${with_tkconfig}"; then - AC_MSG_WARN([--with-tk argument should refer to directory containing tkConfig.sh, not to tkConfig.sh itself]) - with_tkconfig="`echo "${with_tkconfig}" | sed 's!/tkConfig\.sh$!!'`" - fi ;; - esac - if test -f "${with_tkconfig}/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd "${with_tkconfig}"; pwd)`" - else - AC_MSG_ERROR([${with_tkconfig} directory doesn't contain tkConfig.sh]) - fi - fi - - # then check for a private Tk library - if test x"${ac_cv_c_tkconfig}" = x ; then - for i in \ - ../tk \ - `ls -dr ../tk[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../tk[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../tk[[8-9]].[[0-9]]* 2>/dev/null` \ - ../../tk \ - `ls -dr ../../tk[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../../tk[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../tk[[8-9]].[[0-9]]* 2>/dev/null` \ - ../../../tk \ - `ls -dr ../../../tk[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ../../../tk[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../../tk[[8-9]].[[0-9]]* 2>/dev/null` ; do - if test "${TEA_PLATFORM}" = "windows" \ - -a -f "$i/win/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i/win; pwd)`" - break - fi - if test -f "$i/unix/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i/unix; pwd)`" - break - fi - done - fi - - # on Darwin, check in Framework installation locations - if test "`uname -s`" = "Darwin" -a x"${ac_cv_c_tkconfig}" = x ; then - for i in `ls -d ~/Library/Frameworks 2>/dev/null` \ - `ls -d /Library/Frameworks 2>/dev/null` \ - `ls -d /Network/Library/Frameworks 2>/dev/null` \ - ; do - if test -f "$i/Tk.framework/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i/Tk.framework; pwd)`" - break - fi - done - fi - - # check in a few common install locations - if test x"${ac_cv_c_tkconfig}" = x ; then - for i in `ls -d ${libdir} 2>/dev/null` \ - `ls -d ${exec_prefix}/lib 2>/dev/null` \ - `ls -d ${prefix}/lib 2>/dev/null` \ - `ls -d /usr/local/lib 2>/dev/null` \ - `ls -d /usr/contrib/lib 2>/dev/null` \ - `ls -d /usr/pkg/lib 2>/dev/null` \ - `ls -d /usr/lib/tk9.0 2>/dev/null` \ - `ls -d /usr/lib/tk8.7 2>/dev/null` \ - `ls -d /usr/lib/tk8.6 2>/dev/null` \ - `ls -d /usr/lib/tk8.5 2>/dev/null` \ - `ls -d /usr/lib 2>/dev/null` \ - `ls -d /usr/lib64 2>/dev/null` \ - `ls -d /usr/local/lib/tk9.0 2>/dev/null` \ - `ls -d /usr/local/lib/tk8.7 2>/dev/null` \ - `ls -d /usr/local/lib/tk8.6 2>/dev/null` \ - `ls -d /usr/local/lib/tk8.5 2>/dev/null` \ - `ls -d /usr/local/lib/tcl/tk9.0 2>/dev/null` \ - `ls -d /usr/local/lib/tcl/tk8.7 2>/dev/null` \ - `ls -d /usr/local/lib/tcl/tk8.6 2>/dev/null` \ - `ls -d /usr/local/lib/tcl/tk8.5 2>/dev/null` \ - ; do - if test -f "$i/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i; pwd)`" - break - fi - done - fi - - # TEA specific: on Windows, check in common installation locations - if test "${TEA_PLATFORM}" = "windows" \ - -a x"${ac_cv_c_tkconfig}" = x ; then - for i in `ls -d C:/Tcl/lib 2>/dev/null` \ - `ls -d C:/Progra~1/Tcl/lib 2>/dev/null` \ - ; do - if test -f "$i/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i; pwd)`" - break - fi - done - fi - - # check in a few other private locations - if test x"${ac_cv_c_tkconfig}" = x ; then - for i in \ - ${srcdir}/../tk \ - `ls -dr ${srcdir}/../tk[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \ - `ls -dr ${srcdir}/../tk[[8-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ${srcdir}/../tk[[8-9]].[[0-9]]* 2>/dev/null` ; do - if test "${TEA_PLATFORM}" = "windows" \ - -a -f "$i/win/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i/win; pwd)`" - break - fi - if test -f "$i/unix/tkConfig.sh" ; then - ac_cv_c_tkconfig="`(cd $i/unix; pwd)`" - break - fi - done - fi - ]) - - if test x"${ac_cv_c_tkconfig}" = x ; then - TK_BIN_DIR="# no Tk configs found" - AC_MSG_ERROR([Can't find Tk configuration definitions. Use --with-tk to specify a directory containing tkConfig.sh]) - else - no_tk= - TK_BIN_DIR="${ac_cv_c_tkconfig}" - AC_MSG_RESULT([found ${TK_BIN_DIR}/tkConfig.sh]) - fi - fi -]) - -#------------------------------------------------------------------------ -# TEA_LOAD_TCLCONFIG -- -# -# Load the tclConfig.sh file -# -# Arguments: -# -# Requires the following vars to be set: -# TCL_BIN_DIR -# -# Results: -# -# Substitutes the following vars: -# TCL_BIN_DIR -# TCL_SRC_DIR -# TCL_LIB_FILE -# TCL_ZIP_FILE -# TCL_ZIPFS_SUPPORT -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_LOAD_TCLCONFIG], [ - AC_MSG_CHECKING([for existence of ${TCL_BIN_DIR}/tclConfig.sh]) - - if test -f "${TCL_BIN_DIR}/tclConfig.sh" ; then - AC_MSG_RESULT([loading]) - . "${TCL_BIN_DIR}/tclConfig.sh" - else - AC_MSG_RESULT([could not find ${TCL_BIN_DIR}/tclConfig.sh]) - fi - - # If the TCL_BIN_DIR is the build directory (not the install directory), - # then set the common variable name to the value of the build variables. - # For example, the variable TCL_LIB_SPEC will be set to the value - # of TCL_BUILD_LIB_SPEC. An extension should make use of TCL_LIB_SPEC - # instead of TCL_BUILD_LIB_SPEC since it will work with both an - # installed and uninstalled version of Tcl. - if test -f "${TCL_BIN_DIR}/Makefile" ; then - TCL_LIB_SPEC="${TCL_BUILD_LIB_SPEC}" - TCL_STUB_LIB_SPEC="${TCL_BUILD_STUB_LIB_SPEC}" - TCL_STUB_LIB_PATH="${TCL_BUILD_STUB_LIB_PATH}" - elif test "`uname -s`" = "Darwin"; then - # If Tcl was built as a framework, attempt to use the libraries - # from the framework at the given location so that linking works - # against Tcl.framework installed in an arbitrary location. - case ${TCL_DEFS} in - *TCL_FRAMEWORK*) - if test -f "${TCL_BIN_DIR}/${TCL_LIB_FILE}"; then - for i in "`cd "${TCL_BIN_DIR}"; pwd`" \ - "`cd "${TCL_BIN_DIR}"/../..; pwd`"; do - if test "`basename "$i"`" = "${TCL_LIB_FILE}.framework"; then - TCL_LIB_SPEC="-F`dirname "$i" | sed -e 's/ /\\\\ /g'` -framework ${TCL_LIB_FILE}" - break - fi - done - fi - if test -f "${TCL_BIN_DIR}/${TCL_STUB_LIB_FILE}"; then - TCL_STUB_LIB_SPEC="-L`echo "${TCL_BIN_DIR}" | sed -e 's/ /\\\\ /g'` ${TCL_STUB_LIB_FLAG}" - TCL_STUB_LIB_PATH="${TCL_BIN_DIR}/${TCL_STUB_LIB_FILE}" - fi - ;; - esac - fi - - AC_SUBST(TCL_VERSION) - AC_SUBST(TCL_PATCH_LEVEL) - AC_SUBST(TCL_BIN_DIR) - AC_SUBST(TCL_SRC_DIR) - - AC_SUBST(TCL_LIB_FILE) - AC_SUBST(TCL_LIB_FLAG) - AC_SUBST(TCL_LIB_SPEC) - - AC_SUBST(TCL_STUB_LIB_FILE) - AC_SUBST(TCL_STUB_LIB_FLAG) - AC_SUBST(TCL_STUB_LIB_SPEC) - - AC_MSG_CHECKING([platform]) - hold_cc=$CC; CC="$TCL_CC" - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[ - #ifdef _WIN32 - #error win32 - #endif - ]])],[ - # first test we've already retrieved platform (cross-compile), fallback to unix otherwise: - TEA_PLATFORM="${TEA_PLATFORM-unix}" - CYGPATH=echo - ],[ - TEA_PLATFORM="windows" - AC_CHECK_PROG(CYGPATH, cygpath, cygpath -m, echo) - ]) - CC=$hold_cc - AC_MSG_RESULT($TEA_PLATFORM) - - # The BUILD_$pkg is to define the correct extern storage class - # handling when making this package - AC_DEFINE_UNQUOTED(BUILD_${PACKAGE_NAME}, [], - [Building extension source?]) - # Do this here as we have fully defined TEA_PLATFORM now - if test "${TEA_PLATFORM}" = "windows" ; then - EXEEXT=".exe" - CLEANFILES="$CLEANFILES *.lib *.dll *.pdb *.exp" - fi - - # TEA specific: - AC_SUBST(CLEANFILES) - AC_SUBST(TCL_LIBS) - AC_SUBST(TCL_DEFS) - AC_SUBST(TCL_EXTRA_CFLAGS) - AC_SUBST(TCL_LD_FLAGS) - AC_SUBST(TCL_SHLIB_LD_LIBS) -]) - -#------------------------------------------------------------------------ -# TEA_LOAD_TKCONFIG -- -# -# Load the tkConfig.sh file -# -# Arguments: -# -# Requires the following vars to be set: -# TK_BIN_DIR -# -# Results: -# -# Sets the following vars that should be in tkConfig.sh: -# TK_BIN_DIR -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_LOAD_TKCONFIG], [ - AC_MSG_CHECKING([for existence of ${TK_BIN_DIR}/tkConfig.sh]) - - if test -f "${TK_BIN_DIR}/tkConfig.sh" ; then - AC_MSG_RESULT([loading]) - . "${TK_BIN_DIR}/tkConfig.sh" - else - AC_MSG_RESULT([could not find ${TK_BIN_DIR}/tkConfig.sh]) - fi - - # If the TK_BIN_DIR is the build directory (not the install directory), - # then set the common variable name to the value of the build variables. - # For example, the variable TK_LIB_SPEC will be set to the value - # of TK_BUILD_LIB_SPEC. An extension should make use of TK_LIB_SPEC - # instead of TK_BUILD_LIB_SPEC since it will work with both an - # installed and uninstalled version of Tcl. - if test -f "${TK_BIN_DIR}/Makefile" ; then - TK_LIB_SPEC="${TK_BUILD_LIB_SPEC}" - TK_STUB_LIB_SPEC="${TK_BUILD_STUB_LIB_SPEC}" - TK_STUB_LIB_PATH="${TK_BUILD_STUB_LIB_PATH}" - elif test "`uname -s`" = "Darwin"; then - # If Tk was built as a framework, attempt to use the libraries - # from the framework at the given location so that linking works - # against Tk.framework installed in an arbitrary location. - case ${TK_DEFS} in - *TK_FRAMEWORK*) - if test -f "${TK_BIN_DIR}/${TK_LIB_FILE}"; then - for i in "`cd "${TK_BIN_DIR}"; pwd`" \ - "`cd "${TK_BIN_DIR}"/../..; pwd`"; do - if test "`basename "$i"`" = "${TK_LIB_FILE}.framework"; then - TK_LIB_SPEC="-F`dirname "$i" | sed -e 's/ /\\\\ /g'` -framework ${TK_LIB_FILE}" - break - fi - done - fi - if test -f "${TK_BIN_DIR}/${TK_STUB_LIB_FILE}"; then - TK_STUB_LIB_SPEC="-L` echo "${TK_BIN_DIR}" | sed -e 's/ /\\\\ /g'` ${TK_STUB_LIB_FLAG}" - TK_STUB_LIB_PATH="${TK_BIN_DIR}/${TK_STUB_LIB_FILE}" - fi - ;; - esac - fi - - # TEA specific: Ensure windowingsystem is defined - if test "${TEA_PLATFORM}" = "unix" ; then - case ${TK_DEFS} in - *MAC_OSX_TK*) - AC_DEFINE(MAC_OSX_TK, 1, [Are we building against Mac OS X TkAqua?]) - TEA_WINDOWINGSYSTEM="aqua" - ;; - *) - TEA_WINDOWINGSYSTEM="x11" - ;; - esac - elif test "${TEA_PLATFORM}" = "windows" ; then - TEA_WINDOWINGSYSTEM="win32" - fi - - AC_SUBST(TK_VERSION) - AC_SUBST(TK_BIN_DIR) - AC_SUBST(TK_SRC_DIR) - - AC_SUBST(TK_LIB_FILE) - AC_SUBST(TK_LIB_FLAG) - AC_SUBST(TK_LIB_SPEC) - - AC_SUBST(TK_STUB_LIB_FILE) - AC_SUBST(TK_STUB_LIB_FLAG) - AC_SUBST(TK_STUB_LIB_SPEC) - - # TEA specific: - AC_SUBST(TK_LIBS) - AC_SUBST(TK_XINCLUDES) -]) - -#------------------------------------------------------------------------ -# TEA_PROG_TCLSH -# Determine the fully qualified path name of the tclsh executable -# in the Tcl build directory or the tclsh installed in a bin -# directory. This macro will correctly determine the name -# of the tclsh executable even if tclsh has not yet been -# built in the build directory. The tclsh found is always -# associated with a tclConfig.sh file. This tclsh should be used -# only for running extension test cases. It should never be -# or generation of files (like pkgIndex.tcl) at build time. -# -# Arguments: -# none -# -# Results: -# Substitutes the following vars: -# TCLSH_PROG -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_PROG_TCLSH], [ - AC_MSG_CHECKING([for tclsh]) - if test -f "${TCL_BIN_DIR}/Makefile" ; then - # tclConfig.sh is in Tcl build directory - if test "${TEA_PLATFORM}" = "windows"; then - if test -f "${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}${EXEEXT}" ; then - TCLSH_PROG="${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}${EXEEXT}" - elif test -f "${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}s${EXEEXT}" ; then - TCLSH_PROG="${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}s${EXEEXT}" - elif test -f "${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}t${EXEEXT}" ; then - TCLSH_PROG="${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}t${EXEEXT}" - elif test -f "${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}st${EXEEXT}" ; then - TCLSH_PROG="${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}st${EXEEXT}" - fi - else - TCLSH_PROG="${TCL_BIN_DIR}/tclsh" - fi - else - # tclConfig.sh is in install location - if test "${TEA_PLATFORM}" = "windows"; then - TCLSH_PROG="tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}${EXEEXT}" - else - TCLSH_PROG="tclsh${TCL_MAJOR_VERSION}.${TCL_MINOR_VERSION}" - fi - list="`ls -d ${TCL_BIN_DIR}/../bin 2>/dev/null` \ - `ls -d ${TCL_BIN_DIR}/.. 2>/dev/null` \ - `ls -d ${TCL_PREFIX}/bin 2>/dev/null`" - for i in $list ; do - if test -f "$i/${TCLSH_PROG}" ; then - REAL_TCL_BIN_DIR="`cd "$i"; pwd`/" - break - fi - done - TCLSH_PROG="${REAL_TCL_BIN_DIR}${TCLSH_PROG}" - fi - AC_MSG_RESULT([${TCLSH_PROG}]) - AC_SUBST(TCLSH_PROG) -]) - -#------------------------------------------------------------------------ -# TEA_PROG_WISH -# Determine the fully qualified path name of the wish executable -# in the Tk build directory or the wish installed in a bin -# directory. This macro will correctly determine the name -# of the wish executable even if wish has not yet been -# built in the build directory. The wish found is always -# associated with a tkConfig.sh file. This wish should be used -# only for running extension test cases. It should never be -# or generation of files (like pkgIndex.tcl) at build time. -# -# Arguments: -# none -# -# Results: -# Substitutes the following vars: -# WISH_PROG -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_PROG_WISH], [ - AC_MSG_CHECKING([for wish]) - if test -f "${TK_BIN_DIR}/Makefile" ; then - # tkConfig.sh is in Tk build directory - if test "${TEA_PLATFORM}" = "windows"; then - if test -f "${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}${EXEEXT}" ; then - WISH_PROG="${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}${EXEEXT}" - elif test -f "${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}s${EXEEXT}" ; then - WISH_PROG="${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}$s{EXEEXT}" - elif test -f "${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}t${EXEEXT}" ; then - WISH_PROG="${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}t${EXEEXT}" - elif test -f "${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}st${EXEEXT}" ; then - WISH_PROG="${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}st${EXEEXT}" - fi - else - WISH_PROG="${TK_BIN_DIR}/wish" - fi - else - # tkConfig.sh is in install location - if test "${TEA_PLATFORM}" = "windows"; then - WISH_PROG="wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}${EXEEXT}" - else - WISH_PROG="wish${TK_MAJOR_VERSION}.${TK_MINOR_VERSION}" - fi - list="`ls -d ${TK_BIN_DIR}/../bin 2>/dev/null` \ - `ls -d ${TK_BIN_DIR}/.. 2>/dev/null` \ - `ls -d ${TK_PREFIX}/bin 2>/dev/null`" - for i in $list ; do - if test -f "$i/${WISH_PROG}" ; then - REAL_TK_BIN_DIR="`cd "$i"; pwd`/" - break - fi - done - WISH_PROG="${REAL_TK_BIN_DIR}${WISH_PROG}" - fi - AC_MSG_RESULT([${WISH_PROG}]) - AC_SUBST(WISH_PROG) -]) - -#------------------------------------------------------------------------ -# TEA_ENABLE_SHARED -- -# -# Allows the building of shared libraries -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --enable-shared=yes|no -# --enable-stubs=yes|no -# -# Defines the following vars: -# STATIC_BUILD Used for building import/export libraries -# on Windows. -# -# Sets the following vars: -# SHARED_BUILD Value of 1 or 0 -# STUBS_BUILD Value if 1 or 0 -# USE_TCL_STUBS Value true: if SHARED_BUILD or --enable-stubs -# USE_TCLOO_STUBS Value true: if SHARED_BUILD or --enable-stubs -# USE_TK_STUBS Value true: if SHARED_BUILD or --enable-stubs -# AND TEA_WINDOWING_SYSTEM != "" -#------------------------------------------------------------------------ -AC_DEFUN([TEA_ENABLE_SHARED], [ - AC_MSG_CHECKING([how to build libraries]) - AC_ARG_ENABLE(shared, - AS_HELP_STRING([--enable-shared], - [build and link with shared libraries (default: on)]), - [shared_ok=$enableval], [shared_ok=yes]) - - if test "${enable_shared+set}" = set; then - enableval="$enable_shared" - shared_ok=$enableval - else - shared_ok=yes - fi - - AC_ARG_ENABLE(stubs, - AS_HELP_STRING([--enable-stubs], - [build and link with stub libraries. Always true for shared builds (default: on)]), - [stubs_ok=$enableval], [stubs_ok=yes]) - - if test "${enable_stubs+set}" = set; then - enableval="$enable_stubs" - stubs_ok=$enableval - else - stubs_ok=yes - fi - - # Stubs are always enabled for shared builds - if test "$shared_ok" = "yes" ; then - AC_MSG_RESULT([shared]) - SHARED_BUILD=1 - STUBS_BUILD=1 - else - AC_MSG_RESULT([static]) - SHARED_BUILD=0 - AC_DEFINE(STATIC_BUILD, 1, [This a static build]) - if test "$stubs_ok" = "yes" ; then - STUBS_BUILD=1 - else - STUBS_BUILD=0 - fi - fi - if test "${STUBS_BUILD}" = "1" ; then - AC_DEFINE(USE_TCL_STUBS, 1, [Use Tcl stubs]) - AC_DEFINE(USE_TCLOO_STUBS, 1, [Use TclOO stubs]) - if test "${TEA_WINDOWINGSYSTEM}" != ""; then - AC_DEFINE(USE_TK_STUBS, 1, [Use Tk stubs]) - fi - fi - - AC_SUBST(SHARED_BUILD) - AC_SUBST(STUBS_BUILD) -]) - -#------------------------------------------------------------------------ -# TEA_ENABLE_THREADS -- -# -# Specify if thread support should be enabled. If "yes" is specified -# as an arg (optional), threads are enabled by default, "no" means -# threads are disabled. "yes" is the default. -# -# TCL_THREADS is checked so that if you are compiling an extension -# against a threaded core, your extension must be compiled threaded -# as well. -# -# Note that it is legal to have a thread enabled extension run in a -# threaded or non-threaded Tcl core, but a non-threaded extension may -# only run in a non-threaded Tcl core. -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --enable-threads -# -# Sets the following vars: -# THREADS_LIBS Thread library(s) -# -# Defines the following vars: -# TCL_THREADS -# _REENTRANT -# _THREAD_SAFE -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_ENABLE_THREADS], [ - AC_ARG_ENABLE(threads, - AS_HELP_STRING([--enable-threads], - [build with threads (default: on)]), - [tcl_ok=$enableval], [tcl_ok=yes]) - - if test "${enable_threads+set}" = set; then - enableval="$enable_threads" - tcl_ok=$enableval - else - tcl_ok=yes - fi - - if test "$tcl_ok" = "yes" -o "${TCL_THREADS}" = 1; then - TCL_THREADS=1 - - if test "${TEA_PLATFORM}" != "windows" ; then - # We are always OK on Windows, so check what this platform wants: - - # USE_THREAD_ALLOC tells us to try the special thread-based - # allocator that significantly reduces lock contention - AC_DEFINE(USE_THREAD_ALLOC, 1, - [Do we want to use the threaded memory allocator?]) - AC_DEFINE(_REENTRANT, 1, [Do we want the reentrant OS API?]) - if test "`uname -s`" = "SunOS" ; then - AC_DEFINE(_POSIX_PTHREAD_SEMANTICS, 1, - [Do we really want to follow the standard? Yes we do!]) - fi - AC_DEFINE(_THREAD_SAFE, 1, [Do we want the thread-safe OS API?]) - AC_CHECK_LIB(pthread,pthread_mutex_init,tcl_ok=yes,tcl_ok=no) - if test "$tcl_ok" = "no"; then - # Check a little harder for __pthread_mutex_init in the same - # library, as some systems hide it there until pthread.h is - # defined. We could alternatively do an AC_TRY_COMPILE with - # pthread.h, but that will work with libpthread really doesn't - # exist, like AIX 4.2. [Bug: 4359] - AC_CHECK_LIB(pthread, __pthread_mutex_init, - tcl_ok=yes, tcl_ok=no) - fi - - if test "$tcl_ok" = "yes"; then - # The space is needed - THREADS_LIBS=" -lpthread" - else - AC_CHECK_LIB(pthreads, pthread_mutex_init, - tcl_ok=yes, tcl_ok=no) - if test "$tcl_ok" = "yes"; then - # The space is needed - THREADS_LIBS=" -lpthreads" - else - AC_CHECK_LIB(c, pthread_mutex_init, - tcl_ok=yes, tcl_ok=no) - if test "$tcl_ok" = "no"; then - AC_CHECK_LIB(c_r, pthread_mutex_init, - tcl_ok=yes, tcl_ok=no) - if test "$tcl_ok" = "yes"; then - # The space is needed - THREADS_LIBS=" -pthread" - else - TCL_THREADS=0 - AC_MSG_WARN([Do not know how to find pthread lib on your system - thread support disabled]) - fi - fi - fi - fi - fi - else - TCL_THREADS=0 - fi - # Do checking message here to not mess up interleaved configure output - AC_MSG_CHECKING([for building with threads]) - if test "${TCL_THREADS}" = 1; then - AC_DEFINE(TCL_THREADS, 1, [Are we building with threads enabled?]) - AC_MSG_RESULT([yes (default)]) - else - AC_MSG_RESULT([no]) - fi - # TCL_THREADS sanity checking. See if our request for building with - # threads is the same as the way Tcl was built. If not, warn the user. - case ${TCL_DEFS} in - *THREADS=1*) - if test "${TCL_THREADS}" = "0"; then - AC_MSG_WARN([ - Building ${PACKAGE_NAME} without threads enabled, but building against Tcl - that IS thread-enabled. It is recommended to use --enable-threads.]) - fi - ;; - esac - AC_SUBST(TCL_THREADS) -]) - -#------------------------------------------------------------------------ -# TEA_ENABLE_SYMBOLS -- -# -# Specify if debugging symbols should be used. -# Memory (TCL_MEM_DEBUG) debugging can also be enabled. -# -# Arguments: -# none -# -# TEA varies from core Tcl in that C|LDFLAGS_DEFAULT receives -# the value of C|LDFLAGS_OPTIMIZE|DEBUG already substituted. -# Requires the following vars to be set in the Makefile: -# CFLAGS_DEFAULT -# LDFLAGS_DEFAULT -# -# Results: -# -# Adds the following arguments to configure: -# --enable-symbols -# -# Defines the following vars: -# CFLAGS_DEFAULT Sets to $(CFLAGS_DEBUG) if true -# Sets to "$(CFLAGS_OPTIMIZE) -DNDEBUG" if false -# LDFLAGS_DEFAULT Sets to $(LDFLAGS_DEBUG) if true -# Sets to $(LDFLAGS_OPTIMIZE) if false -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_ENABLE_SYMBOLS], [ - dnl TEA specific: Make sure we are initialized - AC_REQUIRE([TEA_CONFIG_CFLAGS]) - AC_MSG_CHECKING([for build with symbols]) - AC_ARG_ENABLE(symbols, - AS_HELP_STRING([--enable-symbols], - [build with debugging symbols (default: off)]), - [tcl_ok=$enableval], [tcl_ok=no]) - if test "$tcl_ok" = "no"; then - CFLAGS_DEFAULT="${CFLAGS_OPTIMIZE} -DNDEBUG" - LDFLAGS_DEFAULT="${LDFLAGS_OPTIMIZE}" - AC_MSG_RESULT([no]) - AC_DEFINE(TCL_CFG_OPTIMIZED, 1, [Is this an optimized build?]) - else - CFLAGS_DEFAULT="${CFLAGS_DEBUG}" - LDFLAGS_DEFAULT="${LDFLAGS_DEBUG}" - if test "$tcl_ok" = "yes"; then - AC_MSG_RESULT([yes (standard debugging)]) - fi - fi - AC_SUBST(CFLAGS_DEFAULT) - AC_SUBST(LDFLAGS_DEFAULT) - - if test "$tcl_ok" = "mem" -o "$tcl_ok" = "all"; then - AC_DEFINE(TCL_MEM_DEBUG, 1, [Is memory debugging enabled?]) - fi - - if test "$tcl_ok" != "yes" -a "$tcl_ok" != "no"; then - if test "$tcl_ok" = "all"; then - AC_MSG_RESULT([enabled symbols mem debugging]) - else - AC_MSG_RESULT([enabled $tcl_ok debugging]) - fi - fi -]) - -#------------------------------------------------------------------------ -# TEA_ENABLE_LANGINFO -- -# -# Allows use of modern nl_langinfo check for better l10n. -# This is only relevant for Unix. -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --enable-langinfo=yes|no (default is yes) -# -# Defines the following vars: -# HAVE_LANGINFO Triggers use of nl_langinfo if defined. -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_ENABLE_LANGINFO], [ - AC_ARG_ENABLE(langinfo, - AS_HELP_STRING([--enable-langinfo], - [use nl_langinfo if possible to determine encoding at startup, otherwise use old heuristic (default: on)]), - [langinfo_ok=$enableval], [langinfo_ok=yes]) - - HAVE_LANGINFO=0 - if test "$langinfo_ok" = "yes"; then - AC_CHECK_HEADER(langinfo.h,[langinfo_ok=yes],[langinfo_ok=no]) - fi - AC_MSG_CHECKING([whether to use nl_langinfo]) - if test "$langinfo_ok" = "yes"; then - AC_CACHE_VAL(tcl_cv_langinfo_h, [ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[nl_langinfo(CODESET);]])], - [tcl_cv_langinfo_h=yes],[tcl_cv_langinfo_h=no])]) - AC_MSG_RESULT([$tcl_cv_langinfo_h]) - if test $tcl_cv_langinfo_h = yes; then - AC_DEFINE(HAVE_LANGINFO, 1, [Do we have nl_langinfo()?]) - fi - else - AC_MSG_RESULT([$langinfo_ok]) - fi -]) - -#-------------------------------------------------------------------- -# TEA_CONFIG_SYSTEM -# -# Determine what the system is (some things cannot be easily checked -# on a feature-driven basis, alas). This can usually be done via the -# "uname" command. -# -# Arguments: -# none -# -# Results: -# Defines the following var: -# -# system - System/platform/version identification code. -# -#-------------------------------------------------------------------- - -AC_DEFUN([TEA_CONFIG_SYSTEM], [ - AC_CACHE_CHECK([system version], tcl_cv_sys_version, [ - # TEA specific: - if test "${TEA_PLATFORM}" = "windows" ; then - tcl_cv_sys_version=windows - else - tcl_cv_sys_version=`uname -s`-`uname -r` - if test "$?" -ne 0 ; then - AC_MSG_WARN([can't find uname command]) - tcl_cv_sys_version=unknown - else - if test "`uname -s`" = "AIX" ; then - tcl_cv_sys_version=AIX-`uname -v`.`uname -r` - fi - if test "`uname -s`" = "NetBSD" -a -f /etc/debian_version ; then - tcl_cv_sys_version=NetBSD-Debian - fi - fi - fi - ]) - system=$tcl_cv_sys_version -]) - -#-------------------------------------------------------------------- -# TEA_CONFIG_CFLAGS -# -# Try to determine the proper flags to pass to the compiler -# for building shared libraries and other such nonsense. -# -# Arguments: -# none -# -# Results: -# -# Defines and substitutes the following vars: -# -# DL_OBJS, DL_LIBS - removed for TEA, only needed by core. -# LDFLAGS - Flags to pass to the compiler when linking object -# files into an executable application binary such -# as tclsh. -# LD_SEARCH_FLAGS-Flags to pass to ld, such as "-R /usr/local/tcl/lib", -# that tell the run-time dynamic linker where to look -# for shared libraries such as libtcl.so. Depends on -# the variable LIB_RUNTIME_DIR in the Makefile. Could -# be the same as CC_SEARCH_FLAGS if ${CC} is used to link. -# CC_SEARCH_FLAGS-Flags to pass to ${CC}, such as "-Wl,-rpath,/usr/local/tcl/lib", -# that tell the run-time dynamic linker where to look -# for shared libraries such as libtcl.so. Depends on -# the variable LIB_RUNTIME_DIR in the Makefile. -# SHLIB_CFLAGS - Flags to pass to cc when compiling the components -# of a shared library (may request position-independent -# code, among other things). -# SHLIB_LD - Base command to use for combining object files -# into a shared library. -# SHLIB_LD_LIBS - Dependent libraries for the linker to scan when -# creating shared libraries. This symbol typically -# goes at the end of the "ld" commands that build -# shared libraries. The value of the symbol defaults to -# "${LIBS}" if all of the dependent libraries should -# be specified when creating a shared library. If -# dependent libraries should not be specified (as on -# SunOS 4.x, where they cause the link to fail, or in -# general if Tcl and Tk aren't themselves shared -# libraries), then this symbol has an empty string -# as its value. -# SHLIB_SUFFIX - Suffix to use for the names of dynamically loadable -# extensions. An empty string means we don't know how -# to use shared libraries on this platform. -# LIB_SUFFIX - Specifies everything that comes after the "libfoo" -# in a static or shared library name, using the $PACKAGE_VERSION variable -# to put the version in the right place. This is used -# by platforms that need non-standard library names. -# Examples: ${PACKAGE_VERSION}.so.1.1 on NetBSD, since it needs -# to have a version after the .so, and ${PACKAGE_VERSION}.a -# on AIX, since a shared library needs to have -# a .a extension whereas shared objects for loadable -# extensions have a .so extension. Defaults to -# ${PACKAGE_VERSION}${SHLIB_SUFFIX}. -# CFLAGS_DEBUG - -# Flags used when running the compiler in debug mode -# CFLAGS_OPTIMIZE - -# Flags used when running the compiler in optimize mode -# CFLAGS - Additional CFLAGS added as necessary (usually 64-bit) -#-------------------------------------------------------------------- - -AC_DEFUN([TEA_CONFIG_CFLAGS], [ - dnl TEA specific: Make sure we are initialized - AC_REQUIRE([TEA_INIT]) - - # Step 0.a: Enable 64 bit support? - - AC_MSG_CHECKING([if 64bit support is requested]) - AC_ARG_ENABLE(64bit, - AS_HELP_STRING([--enable-64bit], - [enable 64bit support (default: off)]), - [do64bit=$enableval], [do64bit=no]) - AC_MSG_RESULT([$do64bit]) - - # Step 0.b: Enable Solaris 64 bit VIS support? - - AC_MSG_CHECKING([if 64bit Sparc VIS support is requested]) - AC_ARG_ENABLE(64bit-vis, - AS_HELP_STRING([--enable-64bit-vis], - [enable 64bit Sparc VIS support (default: off)]), - [do64bitVIS=$enableval], [do64bitVIS=no]) - AC_MSG_RESULT([$do64bitVIS]) - # Force 64bit on with VIS - AS_IF([test "$do64bitVIS" = "yes"], [do64bit=yes]) - - # Step 0.c: Check if visibility support is available. Do this here so - # that platform specific alternatives can be used below if this fails. - - AC_CACHE_CHECK([if compiler supports visibility "hidden"], - tcl_cv_cc_visibility_hidden, [ - hold_cflags=$CFLAGS; CFLAGS="$CFLAGS -Werror" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[ - extern __attribute__((__visibility__("hidden"))) void f(void); - void f(void) {}]], [[f();]])],[tcl_cv_cc_visibility_hidden=yes], - [tcl_cv_cc_visibility_hidden=no]) - CFLAGS=$hold_cflags]) - AS_IF([test $tcl_cv_cc_visibility_hidden = yes], [ - AC_DEFINE(MODULE_SCOPE, - [extern __attribute__((__visibility__("hidden")))], - [Compiler support for module scope symbols]) - AC_DEFINE(HAVE_HIDDEN, [1], [Compiler support for module scope symbols]) - ]) - - # Step 0.d: Disable -rpath support? - - AC_MSG_CHECKING([if rpath support is requested]) - AC_ARG_ENABLE(rpath, - AS_HELP_STRING([--disable-rpath], - [disable rpath support (default: on)]), - [doRpath=$enableval], [doRpath=yes]) - AC_MSG_RESULT([$doRpath]) - - # Set the variable "system" to hold the name and version number - # for the system. - - TEA_CONFIG_SYSTEM - - # Require ranlib early so we can override it in special cases below. - - AC_REQUIRE([AC_PROG_RANLIB]) - - # Set configuration options based on system name and version. - # This is similar to Tcl's unix/tcl.m4 except that we've added a - # "windows" case and removed some core-only vars. - - do64bit_ok=no - # default to '{$LIBS}' and set to "" on per-platform necessary basis - SHLIB_LD_LIBS='${LIBS}' - # When ld needs options to work in 64-bit mode, put them in - # LDFLAGS_ARCH so they eventually end up in LDFLAGS even if [load] - # is disabled by the user. [Bug 1016796] - LDFLAGS_ARCH="" - UNSHARED_LIB_SUFFIX="" - # TEA specific: use PACKAGE_VERSION instead of VERSION - TCL_TRIM_DOTS='`echo ${PACKAGE_VERSION} | tr -d .`' - ECHO_VERSION='`echo ${PACKAGE_VERSION}`' - TCL_LIB_VERSIONS_OK=ok - CFLAGS_DEBUG=-g - AS_IF([test "$GCC" = yes], [ - CFLAGS_OPTIMIZE=-O2 - CFLAGS_WARNING="-Wall" - ], [ - CFLAGS_OPTIMIZE=-O - CFLAGS_WARNING="" - ]) - AC_CHECK_TOOL(AR, ar) - STLIB_LD='${AR} cr' - LD_LIBRARY_PATH_VAR="LD_LIBRARY_PATH" - AS_IF([test "x$SHLIB_VERSION" = x],[SHLIB_VERSION=""],[SHLIB_VERSION=".$SHLIB_VERSION"]) - case $system in - # TEA specific: - windows) - MACHINE="X86" - if test "$do64bit" != "no" ; then - case "$do64bit" in - amd64|x64|yes) - MACHINE="AMD64" ; # default to AMD64 64-bit build - ;; - arm64|aarch64) - MACHINE="ARM64" - ;; - ia64) - MACHINE="IA64" - ;; - esac - fi - - if test "$GCC" != "yes" ; then - if test "${SHARED_BUILD}" = "0" ; then - runtime=-MT - else - runtime=-MD - fi - case "x`echo \${VisualStudioVersion}`" in - x1[[4-9]]*) - lflags="${lflags} -nodefaultlib:libucrt.lib" - TEA_ADD_LIBS([ucrt.lib]) - ;; - *) - ;; - esac - - if test "$do64bit" != "no" ; then - CC="cl.exe" - RC="rc.exe" - lflags="${lflags} -nologo -MACHINE:${MACHINE} " - LINKBIN="link.exe" - CFLAGS_DEBUG="-nologo -Zi -Od -W3 ${runtime}d" - CFLAGS_OPTIMIZE="-nologo -O2 -W2 ${runtime}" - # Avoid 'unresolved external symbol __security_cookie' - # errors, c.f. http://support.microsoft.com/?id=894573 - TEA_ADD_LIBS([bufferoverflowU.lib]) - else - RC="rc" - lflags="${lflags} -nologo" - LINKBIN="link" - CFLAGS_DEBUG="-nologo -Z7 -Od -W3 -WX ${runtime}d" - CFLAGS_OPTIMIZE="-nologo -O2 -W2 ${runtime}" - fi - fi - - if test "$GCC" = "yes"; then - # mingw gcc mode - AC_CHECK_TOOL(RC, windres) - CFLAGS_DEBUG="-g" - CFLAGS_OPTIMIZE="-O2 -fomit-frame-pointer" - SHLIB_LD='${CC} -shared' - UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.a' - LDFLAGS_CONSOLE="-wl,--subsystem,console ${lflags}" - LDFLAGS_WINDOW="-wl,--subsystem,windows ${lflags}" - - AC_CACHE_CHECK(for cross-compile version of gcc, - ac_cv_cross, - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #ifdef _WIN32 - #error cross-compiler - #endif - ]], [[]])], - [ac_cv_cross=yes], - [ac_cv_cross=no]) - ) - if test "$ac_cv_cross" = "yes"; then - case "$do64bit" in - amd64|x64|yes) - CC="x86_64-w64-mingw32-${CC}" - LD="x86_64-w64-mingw32-ld" - AR="x86_64-w64-mingw32-ar" - RANLIB="x86_64-w64-mingw32-ranlib" - RC="x86_64-w64-mingw32-windres" - ;; - arm64|aarch64) - CC="aarch64-w64-mingw32-clang" - LD="aarch64-w64-mingw32-ld" - AR="aarch64-w64-mingw32-ar" - RANLIB="aarch64-w64-mingw32-ranlib" - RC="aarch64-w64-mingw32-windres" - ;; - *) - CC="i686-w64-mingw32-${CC}" - LD="i686-w64-mingw32-ld" - AR="i686-w64-mingw32-ar" - RANLIB="i686-w64-mingw32-ranlib" - RC="i686-w64-mingw32-windres" - ;; - esac - fi - - else - SHLIB_LD="${LINKBIN} -dll ${lflags}" - # link -lib only works when -lib is the first arg - STLIB_LD="${LINKBIN} -lib ${lflags}" - UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.lib' - PATHTYPE=-w - # For information on what debugtype is most useful, see: - # http://msdn.microsoft.com/library/en-us/dnvc60/html/gendepdebug.asp - # and also - # http://msdn2.microsoft.com/en-us/library/y0zzbyt4%28VS.80%29.aspx - # This essentially turns it all on. - LDFLAGS_DEBUG="-debug -debugtype:cv" - LDFLAGS_OPTIMIZE="-release" - LDFLAGS_CONSOLE="-link -subsystem:console ${lflags}" - LDFLAGS_WINDOW="-link -subsystem:windows ${lflags}" - fi - - SHLIB_SUFFIX=".dll" - SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.dll' - - TCL_LIB_VERSIONS_OK=nodots - ;; - AIX-*) - AS_IF([test "$GCC" != "yes"], [ - # AIX requires the _r compiler when gcc isn't being used - case "${CC}" in - *_r|*_r\ *) - # ok ... - ;; - *) - # Make sure only first arg gets _r - CC=`echo "$CC" | sed -e 's/^\([[^ ]]*\)/\1_r/'` - ;; - esac - AC_MSG_RESULT([Using $CC for compiling with threads]) - ]) - LIBS="$LIBS -lc" - SHLIB_CFLAGS="" - SHLIB_SUFFIX=".so" - - LD_LIBRARY_PATH_VAR="LIBPATH" - - # Check to enable 64-bit flags for compiler/linker - AS_IF([test "$do64bit" = yes], [ - AS_IF([test "$GCC" = yes], [ - AC_MSG_WARN([64bit mode not supported with GCC on $system]) - ], [ - do64bit_ok=yes - CFLAGS="$CFLAGS -q64" - LDFLAGS_ARCH="-q64" - RANLIB="${RANLIB} -X64" - AR="${AR} -X64" - SHLIB_LD_FLAGS="-b64" - ]) - ]) - - AS_IF([test "`uname -m`" = ia64], [ - # AIX-5 uses ELF style dynamic libraries on IA-64, but not PPC - SHLIB_LD="/usr/ccs/bin/ld -G -z text" - AS_IF([test "$GCC" = yes], [ - CC_SEARCH_FLAGS='"-Wl,-R,${LIB_RUNTIME_DIR}"' - ], [ - CC_SEARCH_FLAGS='"-R${LIB_RUNTIME_DIR}"' - ]) - LD_SEARCH_FLAGS='-R "${LIB_RUNTIME_DIR}"' - ], [ - AS_IF([test "$GCC" = yes], [ - SHLIB_LD='${CC} -shared -Wl,-bexpall' - ], [ - SHLIB_LD="/bin/ld -bhalt:4 -bM:SRE -bexpall -H512 -T512 -bnoentry" - LDFLAGS="$LDFLAGS -brtl" - ]) - SHLIB_LD="${SHLIB_LD} ${SHLIB_LD_FLAGS}" - CC_SEARCH_FLAGS='"-L${LIB_RUNTIME_DIR}"' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - ]) - ;; - BeOS*) - SHLIB_CFLAGS="-fPIC" - SHLIB_LD='${CC} -nostart' - SHLIB_SUFFIX=".so" - - #----------------------------------------------------------- - # Check for inet_ntoa in -lbind, for BeOS (which also needs - # -lsocket, even if the network functions are in -lnet which - # is always linked to, for compatibility. - #----------------------------------------------------------- - AC_CHECK_LIB(bind, inet_ntoa, [LIBS="$LIBS -lbind -lsocket"]) - ;; - BSD/OS-2.1*|BSD/OS-3*) - SHLIB_CFLAGS="" - SHLIB_LD="shlicc -r" - SHLIB_SUFFIX=".so" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - BSD/OS-4.*) - SHLIB_CFLAGS="-export-dynamic -fPIC" - SHLIB_LD='${CC} -shared' - SHLIB_SUFFIX=".so" - LDFLAGS="$LDFLAGS -export-dynamic" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - CYGWIN_*) - SHLIB_CFLAGS="" - SHLIB_LD='${CC} -shared' - SHLIB_SUFFIX=".dll" - SHLIB_LD_LIBS="${SHLIB_LD_LIBS} -Wl,--out-implib,\$[@].a" - EXEEXT=".exe" - do64bit_ok=yes - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - dgux*) - SHLIB_CFLAGS="-K PIC" - SHLIB_LD='${CC} -G' - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - Haiku*) - LDFLAGS="$LDFLAGS -Wl,--export-dynamic" - SHLIB_CFLAGS="-fPIC" - SHLIB_SUFFIX=".so" - SHLIB_LD='${CC} ${CFLAGS} ${LDFLAGS} -shared' - AC_CHECK_LIB(network, inet_ntoa, [LIBS="$LIBS -lnetwork"]) - ;; - HP-UX-*.11.*) - # Use updated header definitions where possible - AC_DEFINE(_XOPEN_SOURCE_EXTENDED, 1, [Do we want to use the XOPEN network library?]) - # TEA specific: Needed by Tcl, but not most extensions - #AC_DEFINE(_XOPEN_SOURCE, 1, [Do we want to use the XOPEN network library?]) - #LIBS="$LIBS -lxnet" # Use the XOPEN network library - - AS_IF([test "`uname -m`" = ia64], [ - SHLIB_SUFFIX=".so" - ], [ - SHLIB_SUFFIX=".sl" - ]) - AC_CHECK_LIB(dld, shl_load, tcl_ok=yes, tcl_ok=no) - AS_IF([test "$tcl_ok" = yes], [ - SHLIB_CFLAGS="+z" - SHLIB_LD="ld -b" - LDFLAGS="$LDFLAGS -Wl,-E" - CC_SEARCH_FLAGS='"-Wl,+s,+b,${LIB_RUNTIME_DIR}:."' - LD_SEARCH_FLAGS='+s +b "${LIB_RUNTIME_DIR}:."' - LD_LIBRARY_PATH_VAR="SHLIB_PATH" - ]) - AS_IF([test "$GCC" = yes], [ - SHLIB_LD='${CC} -shared' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - ], [ - CFLAGS="$CFLAGS -z" - ]) - - # Check to enable 64-bit flags for compiler/linker - AS_IF([test "$do64bit" = "yes"], [ - AS_IF([test "$GCC" = yes], [ - case `${CC} -dumpmachine` in - hppa64*) - # 64-bit gcc in use. Fix flags for GNU ld. - do64bit_ok=yes - SHLIB_LD='${CC} -shared' - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"']) - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - ;; - *) - AC_MSG_WARN([64bit mode not supported with GCC on $system]) - ;; - esac - ], [ - do64bit_ok=yes - CFLAGS="$CFLAGS +DD64" - LDFLAGS_ARCH="+DD64" - ]) - ]) ;; - HP-UX-*.08.*|HP-UX-*.09.*|HP-UX-*.10.*) - SHLIB_SUFFIX=".sl" - AC_CHECK_LIB(dld, shl_load, tcl_ok=yes, tcl_ok=no) - AS_IF([test "$tcl_ok" = yes], [ - SHLIB_CFLAGS="+z" - SHLIB_LD="ld -b" - SHLIB_LD_LIBS="" - LDFLAGS="$LDFLAGS -Wl,-E" - CC_SEARCH_FLAGS='"-Wl,+s,+b,${LIB_RUNTIME_DIR}:."' - LD_SEARCH_FLAGS='+s +b "${LIB_RUNTIME_DIR}:."' - LD_LIBRARY_PATH_VAR="SHLIB_PATH" - ]) ;; - IRIX-5.*) - SHLIB_CFLAGS="" - SHLIB_LD="ld -shared -rdata_shared" - SHLIB_SUFFIX=".so" - AC_LIBOBJ(mkstemp) - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"' - LD_SEARCH_FLAGS='-rpath "${LIB_RUNTIME_DIR}"']) - ;; - IRIX-6.*) - SHLIB_CFLAGS="" - SHLIB_LD="ld -n32 -shared -rdata_shared" - SHLIB_SUFFIX=".so" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"' - LD_SEARCH_FLAGS='-rpath "${LIB_RUNTIME_DIR}"']) - AS_IF([test "$GCC" = yes], [ - CFLAGS="$CFLAGS -mabi=n32" - LDFLAGS="$LDFLAGS -mabi=n32" - ], [ - case $system in - IRIX-6.3) - # Use to build 6.2 compatible binaries on 6.3. - CFLAGS="$CFLAGS -n32 -D_OLD_TERMIOS" - ;; - *) - CFLAGS="$CFLAGS -n32" - ;; - esac - LDFLAGS="$LDFLAGS -n32" - ]) - ;; - IRIX64-6.*) - SHLIB_CFLAGS="" - SHLIB_LD="ld -n32 -shared -rdata_shared" - SHLIB_SUFFIX=".so" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"' - LD_SEARCH_FLAGS='-rpath "${LIB_RUNTIME_DIR}"']) - - # Check to enable 64-bit flags for compiler/linker - - AS_IF([test "$do64bit" = yes], [ - AS_IF([test "$GCC" = yes], [ - AC_MSG_WARN([64bit mode not supported by gcc]) - ], [ - do64bit_ok=yes - SHLIB_LD="ld -64 -shared -rdata_shared" - CFLAGS="$CFLAGS -64" - LDFLAGS_ARCH="-64" - ]) - ]) - ;; - Linux*|GNU*|NetBSD-Debian|DragonFly-*|FreeBSD-*) - SHLIB_CFLAGS="-fPIC" - SHLIB_SUFFIX=".so" - - # TEA specific: - CFLAGS_OPTIMIZE="-O2 -fomit-frame-pointer" - - # TEA specific: use LDFLAGS_DEFAULT instead of LDFLAGS - SHLIB_LD='${CC} ${CFLAGS} ${LDFLAGS_DEFAULT} -shared' - LDFLAGS="$LDFLAGS -Wl,--export-dynamic" - - case $system in - DragonFly-*|FreeBSD-*) - AS_IF([test "${TCL_THREADS}" = "1"], [ - # The -pthread needs to go in the LDFLAGS, not LIBS - LIBS=`echo $LIBS | sed s/-pthread//` - CFLAGS="$CFLAGS $PTHREAD_CFLAGS" - LDFLAGS="$LDFLAGS $PTHREAD_LIBS"]) - ;; - esac - - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"']) - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - AS_IF([test "`uname -m`" = "alpha"], [CFLAGS="$CFLAGS -mieee"]) - AS_IF([test $do64bit = yes], [ - AC_CACHE_CHECK([if compiler accepts -m64 flag], tcl_cv_cc_m64, [ - hold_cflags=$CFLAGS - CFLAGS="$CFLAGS -m64" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])], - [tcl_cv_cc_m64=yes],[tcl_cv_cc_m64=no]) - CFLAGS=$hold_cflags]) - AS_IF([test $tcl_cv_cc_m64 = yes], [ - CFLAGS="$CFLAGS -m64" - do64bit_ok=yes - ]) - ]) - - # The combo of gcc + glibc has a bug related to inlining of - # functions like strtod(). The -fno-builtin flag should address - # this problem but it does not work. The -fno-inline flag is kind - # of overkill but it works. Disable inlining only when one of the - # files in compat/*.c is being linked in. - - AS_IF([test x"${USE_COMPAT}" != x],[CFLAGS="$CFLAGS -fno-inline"]) - ;; - Lynx*) - SHLIB_CFLAGS="-fPIC" - SHLIB_SUFFIX=".so" - CFLAGS_OPTIMIZE=-02 - SHLIB_LD='${CC} -shared' - LD_FLAGS="-Wl,--export-dynamic" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"' - LD_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"']) - ;; - OpenBSD-*) - arch=`arch -s` - case "$arch" in - alpha|sparc64) - SHLIB_CFLAGS="-fPIC" - ;; - *) - SHLIB_CFLAGS="-fpic" - ;; - esac - SHLIB_LD='${CC} ${SHLIB_CFLAGS} -shared' - SHLIB_SUFFIX=".so" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"']) - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.so${SHLIB_VERSION}' - LDFLAGS="$LDFLAGS -Wl,-export-dynamic" - CFLAGS_OPTIMIZE="-O2" - # On OpenBSD: Compile with -pthread - # Don't link with -lpthread - LIBS=`echo $LIBS | sed s/-lpthread//` - CFLAGS="$CFLAGS -pthread" - # OpenBSD doesn't do version numbers with dots. - UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.a' - TCL_LIB_VERSIONS_OK=nodots - ;; - NetBSD-*) - # NetBSD has ELF and can use 'cc -shared' to build shared libs - SHLIB_CFLAGS="-fPIC" - SHLIB_LD='${CC} ${SHLIB_CFLAGS} -shared' - SHLIB_SUFFIX=".so" - LDFLAGS="$LDFLAGS -export-dynamic" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"']) - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - # The -pthread needs to go in the CFLAGS, not LIBS - LIBS=`echo $LIBS | sed s/-pthread//` - CFLAGS="$CFLAGS -pthread" - LDFLAGS="$LDFLAGS -pthread" - ;; - Darwin-*) - CFLAGS_OPTIMIZE="-Os" - SHLIB_CFLAGS="-fno-common" - # To avoid discrepancies between what headers configure sees during - # preprocessing tests and compiling tests, move any -isysroot and - # -mmacosx-version-min flags from CFLAGS to CPPFLAGS: - CPPFLAGS="${CPPFLAGS} `echo " ${CFLAGS}" | \ - awk 'BEGIN {FS=" +-";ORS=" "}; {for (i=2;i<=NF;i++) \ - if ([$]i~/^(isysroot|mmacosx-version-min)/) print "-"[$]i}'`" - CFLAGS="`echo " ${CFLAGS}" | \ - awk 'BEGIN {FS=" +-";ORS=" "}; {for (i=2;i<=NF;i++) \ - if (!([$]i~/^(isysroot|mmacosx-version-min)/)) print "-"[$]i}'`" - AS_IF([test $do64bit = yes], [ - case `arch` in - ppc) - AC_CACHE_CHECK([if compiler accepts -arch ppc64 flag], - tcl_cv_cc_arch_ppc64, [ - hold_cflags=$CFLAGS - CFLAGS="$CFLAGS -arch ppc64 -mpowerpc64 -mcpu=G5" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])], - [tcl_cv_cc_arch_ppc64=yes],[tcl_cv_cc_arch_ppc64=no]) - CFLAGS=$hold_cflags]) - AS_IF([test $tcl_cv_cc_arch_ppc64 = yes], [ - CFLAGS="$CFLAGS -arch ppc64 -mpowerpc64 -mcpu=G5" - do64bit_ok=yes - ]);; - i386) - AC_CACHE_CHECK([if compiler accepts -arch x86_64 flag], - tcl_cv_cc_arch_x86_64, [ - hold_cflags=$CFLAGS - CFLAGS="$CFLAGS -arch x86_64" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[]])], - [tcl_cv_cc_arch_x86_64=yes],[tcl_cv_cc_arch_x86_64=no]) - CFLAGS=$hold_cflags]) - AS_IF([test $tcl_cv_cc_arch_x86_64 = yes], [ - CFLAGS="$CFLAGS -arch x86_64" - do64bit_ok=yes - ]);; - *) - AC_MSG_WARN([Don't know how enable 64-bit on architecture `arch`]);; - esac - ], [ - # Check for combined 32-bit and 64-bit fat build - AS_IF([echo "$CFLAGS " |grep -E -q -- '-arch (ppc64|x86_64) ' \ - && echo "$CFLAGS " |grep -E -q -- '-arch (ppc|i386) '], [ - fat_32_64=yes]) - ]) - # TEA specific: use LDFLAGS_DEFAULT instead of LDFLAGS - SHLIB_LD='${CC} -dynamiclib ${CFLAGS} ${LDFLAGS_DEFAULT}' - AC_CACHE_CHECK([if ld accepts -single_module flag], tcl_cv_ld_single_module, [ - hold_ldflags=$LDFLAGS - LDFLAGS="$LDFLAGS -dynamiclib -Wl,-single_module" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[int i;]])], - [tcl_cv_ld_single_module=yes],[tcl_cv_ld_single_module=no]) - LDFLAGS=$hold_ldflags]) - AS_IF([test $tcl_cv_ld_single_module = yes], [ - SHLIB_LD="${SHLIB_LD} -Wl,-single_module" - ]) - # TEA specific: link shlib with current and compatibility version flags - vers=`echo ${PACKAGE_VERSION} | sed -e 's/^\([[0-9]]\{1,5\}\)\(\(\.[[0-9]]\{1,3\}\)\{0,2\}\).*$/\1\2/p' -e d` - SHLIB_LD="${SHLIB_LD} -current_version ${vers:-0} -compatibility_version ${vers:-0}" - SHLIB_SUFFIX=".dylib" - LDFLAGS="$LDFLAGS -headerpad_max_install_names" - AC_CACHE_CHECK([if ld accepts -search_paths_first flag], - tcl_cv_ld_search_paths_first, [ - hold_ldflags=$LDFLAGS - LDFLAGS="$LDFLAGS -Wl,-search_paths_first" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[int i;]])], - [tcl_cv_ld_search_paths_first=yes],[tcl_cv_ld_search_paths_first=no]) - LDFLAGS=$hold_ldflags]) - AS_IF([test $tcl_cv_ld_search_paths_first = yes], [ - LDFLAGS="$LDFLAGS -Wl,-search_paths_first" - ]) - AS_IF([test "$tcl_cv_cc_visibility_hidden" != yes], [ - AC_DEFINE(MODULE_SCOPE, [__private_extern__], - [Compiler support for module scope symbols]) - tcl_cv_cc_visibility_hidden=yes - ]) - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - LD_LIBRARY_PATH_VAR="DYLD_LIBRARY_PATH" - # TEA specific: for combined 32 & 64 bit fat builds of Tk - # extensions, verify that 64-bit build is possible. - AS_IF([test "$fat_32_64" = yes && test -n "${TK_BIN_DIR}"], [ - AS_IF([test "${TEA_WINDOWINGSYSTEM}" = x11], [ - AC_CACHE_CHECK([for 64-bit X11], tcl_cv_lib_x11_64, [ - for v in CFLAGS CPPFLAGS LDFLAGS; do - eval 'hold_'$v'="$'$v'";'$v'="`echo "$'$v' "|sed -e "s/-arch ppc / /g" -e "s/-arch i386 / /g"`"' - done - CPPFLAGS="$CPPFLAGS -I/usr/X11R6/include" - LDFLAGS="$LDFLAGS -L/usr/X11R6/lib -lX11" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]], [[XrmInitialize();]])], - [tcl_cv_lib_x11_64=yes],[tcl_cv_lib_x11_64=no]) - for v in CFLAGS CPPFLAGS LDFLAGS; do - eval $v'="$hold_'$v'"' - done]) - ]) - AS_IF([test "${TEA_WINDOWINGSYSTEM}" = aqua], [ - AC_CACHE_CHECK([for 64-bit Tk], tcl_cv_lib_tk_64, [ - for v in CFLAGS CPPFLAGS LDFLAGS; do - eval 'hold_'$v'="$'$v'";'$v'="`echo "$'$v' "|sed -e "s/-arch ppc / /g" -e "s/-arch i386 / /g"`"' - done - CPPFLAGS="$CPPFLAGS -DUSE_TCL_STUBS=1 -DUSE_TK_STUBS=1 ${TCL_INCLUDES} ${TK_INCLUDES}" - LDFLAGS="$LDFLAGS ${TCL_STUB_LIB_SPEC} ${TK_STUB_LIB_SPEC}" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]], [[Tk_InitStubs(NULL, "", 0);]])], - [tcl_cv_lib_tk_64=yes],[tcl_cv_lib_tk_64=no]) - for v in CFLAGS CPPFLAGS LDFLAGS; do - eval $v'="$hold_'$v'"' - done]) - ]) - # remove 64-bit arch flags from CFLAGS et al. if configuration - # does not support 64-bit. - AS_IF([test "$tcl_cv_lib_tk_64" = no -o "$tcl_cv_lib_x11_64" = no], [ - AC_MSG_NOTICE([Removing 64-bit architectures from compiler & linker flags]) - for v in CFLAGS CPPFLAGS LDFLAGS; do - eval $v'="`echo "$'$v' "|sed -e "s/-arch ppc64 / /g" -e "s/-arch x86_64 / /g"`"' - done]) - ]) - ;; - OS/390-*) - CFLAGS_OPTIMIZE="" # Optimizer is buggy - AC_DEFINE(_OE_SOCKETS, 1, # needed in sys/socket.h - [Should OS/390 do the right thing with sockets?]) - ;; - OSF1-V*) - # Digital OSF/1 - SHLIB_CFLAGS="" - AS_IF([test "$SHARED_BUILD" = 1], [ - SHLIB_LD='ld -shared -expect_unresolved "*"' - ], [ - SHLIB_LD='ld -non_shared -expect_unresolved "*"' - ]) - SHLIB_SUFFIX=".so" - AS_IF([test $doRpath = yes], [ - CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"' - LD_SEARCH_FLAGS='-rpath ${LIB_RUNTIME_DIR}']) - AS_IF([test "$GCC" = yes], [CFLAGS="$CFLAGS -mieee"], [ - CFLAGS="$CFLAGS -DHAVE_TZSET -std1 -ieee"]) - # see pthread_intro(3) for pthread support on osf1, k.furukawa - CFLAGS="$CFLAGS -DHAVE_PTHREAD_ATTR_SETSTACKSIZE" - CFLAGS="$CFLAGS -DTCL_THREAD_STACK_MIN=PTHREAD_STACK_MIN*64" - LIBS=`echo $LIBS | sed s/-lpthreads//` - AS_IF([test "$GCC" = yes], [ - LIBS="$LIBS -lpthread -lmach -lexc" - ], [ - CFLAGS="$CFLAGS -pthread" - LDFLAGS="$LDFLAGS -pthread" - ]) - ;; - QNX-6*) - # QNX RTP - # This may work for all QNX, but it was only reported for v6. - SHLIB_CFLAGS="-fPIC" - SHLIB_LD="ld -Bshareable -x" - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - SCO_SV-3.2*) - AS_IF([test "$GCC" = yes], [ - SHLIB_CFLAGS="-fPIC -melf" - LDFLAGS="$LDFLAGS -melf -Wl,-Bexport" - ], [ - SHLIB_CFLAGS="-Kpic -belf" - LDFLAGS="$LDFLAGS -belf -Wl,-Bexport" - ]) - SHLIB_LD="ld -G" - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - SunOS-5.[[0-6]]) - # Careful to not let 5.10+ fall into this case - - # Note: If _REENTRANT isn't defined, then Solaris - # won't define thread-safe library routines. - - AC_DEFINE(_REENTRANT, 1, [Do we want the reentrant OS API?]) - AC_DEFINE(_POSIX_PTHREAD_SEMANTICS, 1, - [Do we really want to follow the standard? Yes we do!]) - - SHLIB_CFLAGS="-KPIC" - SHLIB_SUFFIX=".so" - AS_IF([test "$GCC" = yes], [ - SHLIB_LD='${CC} -shared' - CC_SEARCH_FLAGS='"-Wl,-R,${LIB_RUNTIME_DIR}"' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - ], [ - SHLIB_LD="/usr/ccs/bin/ld -G -z text" - CC_SEARCH_FLAGS='-R "${LIB_RUNTIME_DIR}"' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - ]) - ;; - SunOS-5*) - # Note: If _REENTRANT isn't defined, then Solaris - # won't define thread-safe library routines. - - AC_DEFINE(_REENTRANT, 1, [Do we want the reentrant OS API?]) - AC_DEFINE(_POSIX_PTHREAD_SEMANTICS, 1, - [Do we really want to follow the standard? Yes we do!]) - - SHLIB_CFLAGS="-KPIC" - - # Check to enable 64-bit flags for compiler/linker - AS_IF([test "$do64bit" = yes], [ - arch=`isainfo` - AS_IF([test "$arch" = "sparcv9 sparc"], [ - AS_IF([test "$GCC" = yes], [ - AS_IF([test "`${CC} -dumpversion | awk -F. '{print [$]1}'`" -lt 3], [ - AC_MSG_WARN([64bit mode not supported with GCC < 3.2 on $system]) - ], [ - do64bit_ok=yes - CFLAGS="$CFLAGS -m64 -mcpu=v9" - LDFLAGS="$LDFLAGS -m64 -mcpu=v9" - SHLIB_CFLAGS="-fPIC" - ]) - ], [ - do64bit_ok=yes - AS_IF([test "$do64bitVIS" = yes], [ - CFLAGS="$CFLAGS -xarch=v9a" - LDFLAGS_ARCH="-xarch=v9a" - ], [ - CFLAGS="$CFLAGS -xarch=v9" - LDFLAGS_ARCH="-xarch=v9" - ]) - # Solaris 64 uses this as well - #LD_LIBRARY_PATH_VAR="LD_LIBRARY_PATH_64" - ]) - ], [AS_IF([test "$arch" = "amd64 i386"], [ - AS_IF([test "$GCC" = yes], [ - case $system in - SunOS-5.1[[1-9]]*|SunOS-5.[[2-9]][[0-9]]*) - do64bit_ok=yes - CFLAGS="$CFLAGS -m64" - LDFLAGS="$LDFLAGS -m64";; - *) - AC_MSG_WARN([64bit mode not supported with GCC on $system]);; - esac - ], [ - do64bit_ok=yes - case $system in - SunOS-5.1[[1-9]]*|SunOS-5.[[2-9]][[0-9]]*) - CFLAGS="$CFLAGS -m64" - LDFLAGS="$LDFLAGS -m64";; - *) - CFLAGS="$CFLAGS -xarch=amd64" - LDFLAGS="$LDFLAGS -xarch=amd64";; - esac - ]) - ], [AC_MSG_WARN([64bit mode not supported for $arch])])]) - ]) - - SHLIB_SUFFIX=".so" - AS_IF([test "$GCC" = yes], [ - SHLIB_LD='${CC} -shared' - CC_SEARCH_FLAGS='"-Wl,-R,${LIB_RUNTIME_DIR}"' - LD_SEARCH_FLAGS=${CC_SEARCH_FLAGS} - AS_IF([test "$do64bit_ok" = yes], [ - AS_IF([test "$arch" = "sparcv9 sparc"], [ - # We need to specify -static-libgcc or we need to - # add the path to the sparv9 libgcc. - # JH: static-libgcc is necessary for core Tcl, but may - # not be necessary for extensions. - SHLIB_LD="$SHLIB_LD -m64 -mcpu=v9 -static-libgcc" - # for finding sparcv9 libgcc, get the regular libgcc - # path, remove so name and append 'sparcv9' - #v9gcclibdir="`gcc -print-file-name=libgcc_s.so` | ..." - #CC_SEARCH_FLAGS="${CC_SEARCH_FLAGS},-R,$v9gcclibdir" - ], [AS_IF([test "$arch" = "amd64 i386"], [ - # JH: static-libgcc is necessary for core Tcl, but may - # not be necessary for extensions. - SHLIB_LD="$SHLIB_LD -m64 -static-libgcc" - ])]) - ]) - ], [ - case $system in - SunOS-5.[[1-9]][[0-9]]*) - # TEA specific: use LDFLAGS_DEFAULT instead of LDFLAGS - SHLIB_LD='${CC} -G -z text ${LDFLAGS_DEFAULT}';; - *) - SHLIB_LD='/usr/ccs/bin/ld -G -z text';; - esac - CC_SEARCH_FLAGS='"-Wl,-R,${LIB_RUNTIME_DIR}"' - LD_SEARCH_FLAGS='-R "${LIB_RUNTIME_DIR}"' - ]) - ;; - UNIX_SV* | UnixWare-5*) - SHLIB_CFLAGS="-KPIC" - SHLIB_LD='${CC} -G' - SHLIB_LD_LIBS="" - SHLIB_SUFFIX=".so" - # Some UNIX_SV* systems (unixware 1.1.2 for example) have linkers - # that don't grok the -Bexport option. Test that it does. - AC_CACHE_CHECK([for ld accepts -Bexport flag], tcl_cv_ld_Bexport, [ - hold_ldflags=$LDFLAGS - LDFLAGS="$LDFLAGS -Wl,-Bexport" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[int i;]])], - [tcl_cv_ld_Bexport=yes],[tcl_cv_ld_Bexport=no]) - LDFLAGS=$hold_ldflags]) - AS_IF([test $tcl_cv_ld_Bexport = yes], [ - LDFLAGS="$LDFLAGS -Wl,-Bexport" - ]) - CC_SEARCH_FLAGS="" - LD_SEARCH_FLAGS="" - ;; - esac - - AS_IF([test "$do64bit" = yes -a "$do64bit_ok" = no], [ - AC_MSG_WARN([64bit support being disabled -- don't know magic for this platform]) - ]) - -dnl # Add any CPPFLAGS set in the environment to our CFLAGS, but delay doing so -dnl # until the end of configure, as configure's compile and link tests use -dnl # both CPPFLAGS and CFLAGS (unlike our compile and link) but configure's -dnl # preprocessing tests use only CPPFLAGS. - AC_CONFIG_COMMANDS_PRE([CFLAGS="${CFLAGS} ${CPPFLAGS}"; CPPFLAGS=""]) - - # Add in the arch flags late to ensure it wasn't removed. - # Not necessary in TEA, but this is aligned with core - LDFLAGS="$LDFLAGS $LDFLAGS_ARCH" - - # If we're running gcc, then change the C flags for compiling shared - # libraries to the right flags for gcc, instead of those for the - # standard manufacturer compiler. - - AS_IF([test "$GCC" = yes], [ - case $system in - AIX-*) ;; - BSD/OS*) ;; - CYGWIN_*|MINGW32_*|MINGW64_*|MSYS_*) ;; - IRIX*) ;; - NetBSD-*|DragonFly-*|FreeBSD-*|OpenBSD-*) ;; - Darwin-*) ;; - SCO_SV-3.2*) ;; - windows) ;; - *) SHLIB_CFLAGS="-fPIC" ;; - esac]) - - AS_IF([test "$tcl_cv_cc_visibility_hidden" != yes], [ - AC_DEFINE(MODULE_SCOPE, [extern], - [No Compiler support for module scope symbols]) - ]) - - AS_IF([test "$SHARED_LIB_SUFFIX" = ""], [ - # TEA specific: use PACKAGE_VERSION instead of VERSION - SHARED_LIB_SUFFIX='${PACKAGE_VERSION}${SHLIB_SUFFIX}']) - AS_IF([test "$UNSHARED_LIB_SUFFIX" = ""], [ - # TEA specific: use PACKAGE_VERSION instead of VERSION - UNSHARED_LIB_SUFFIX='${PACKAGE_VERSION}.a']) - - if test "${GCC}" = "yes" -a ${SHLIB_SUFFIX} = ".dll"; then - AC_CACHE_CHECK(for SEH support in compiler, - tcl_cv_seh, - AC_RUN_IFELSE([AC_LANG_SOURCE([[ -#define WIN32_LEAN_AND_MEAN -#include -#undef WIN32_LEAN_AND_MEAN - - int main(int argc, char** argv) { - int a, b = 0; - __try { - a = 666 / b; - } - __except (EXCEPTION_EXECUTE_HANDLER) { - return 0; - } - return 1; - } - ]])], - [tcl_cv_seh=yes], - [tcl_cv_seh=no], - [tcl_cv_seh=no]) - ) - if test "$tcl_cv_seh" = "no" ; then - AC_DEFINE(HAVE_NO_SEH, 1, - [Defined when mingw does not support SEH]) - fi - - # - # Check to see if the excpt.h include file provided contains the - # definition for EXCEPTION_DISPOSITION; if not, which is the case - # with Cygwin's version as of 2002-04-10, define it to be int, - # sufficient for getting the current code to work. - # - AC_CACHE_CHECK(for EXCEPTION_DISPOSITION support in include files, - tcl_cv_eh_disposition, - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ -# define WIN32_LEAN_AND_MEAN -# include -# undef WIN32_LEAN_AND_MEAN - ]], [[ - EXCEPTION_DISPOSITION x; - ]])], - [tcl_cv_eh_disposition=yes], - [tcl_cv_eh_disposition=no]) - ) - if test "$tcl_cv_eh_disposition" = "no" ; then - AC_DEFINE(EXCEPTION_DISPOSITION, int, - [Defined when cygwin/mingw does not support EXCEPTION DISPOSITION]) - fi - - # Check to see if winnt.h defines CHAR, SHORT, and LONG - # even if VOID has already been #defined. The win32api - # used by mingw and cygwin is known to do this. - - AC_CACHE_CHECK(for winnt.h that ignores VOID define, - tcl_cv_winnt_ignore_void, - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ -#define VOID void -#define WIN32_LEAN_AND_MEAN -#include -#undef WIN32_LEAN_AND_MEAN - ]], [[ - CHAR c; - SHORT s; - LONG l; - ]])], - [tcl_cv_winnt_ignore_void=yes], - [tcl_cv_winnt_ignore_void=no]) - ) - if test "$tcl_cv_winnt_ignore_void" = "yes" ; then - AC_DEFINE(HAVE_WINNT_IGNORE_VOID, 1, - [Defined when cygwin/mingw ignores VOID define in winnt.h]) - fi - fi - - # See if the compiler supports casting to a union type. - # This is used to stop gcc from printing a compiler - # warning when initializing a union member. - - AC_CACHE_CHECK(for cast to union support, - tcl_cv_cast_to_union, - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[ - union foo { int i; double d; }; - union foo f = (union foo) (int) 0; - ]])], - [tcl_cv_cast_to_union=yes], - [tcl_cv_cast_to_union=no]) - ) - if test "$tcl_cv_cast_to_union" = "yes"; then - AC_DEFINE(HAVE_CAST_TO_UNION, 1, - [Defined when compiler supports casting to union type.]) - fi - - AC_CHECK_HEADER(stdbool.h, [AC_DEFINE(HAVE_STDBOOL_H, 1, [Do we have ?])],) - - AC_SUBST(CFLAGS_DEBUG) - AC_SUBST(CFLAGS_OPTIMIZE) - AC_SUBST(CFLAGS_WARNING) - AC_SUBST(LDFLAGS_DEBUG) - AC_SUBST(LDFLAGS_OPTIMIZE) - - AC_SUBST(STLIB_LD) - AC_SUBST(SHLIB_LD) - - AC_SUBST(SHLIB_LD_LIBS) - AC_SUBST(SHLIB_CFLAGS) - - AC_SUBST(LD_LIBRARY_PATH_VAR) - - # These must be called after we do the basic CFLAGS checks and - # verify any possible 64-bit or similar switches are necessary - TEA_TCL_EARLY_FLAGS - TEA_TCL_64BIT_FLAGS -]) - -#-------------------------------------------------------------------- -# TEA_SERIAL_PORT -# -# Determine which interface to use to talk to the serial port. -# Note that #include lines must begin in leftmost column for -# some compilers to recognize them as preprocessor directives, -# and some build environments have stdin not pointing at a -# pseudo-terminal (usually /dev/null instead.) -# -# Arguments: -# none -# -# Results: -# -# Defines only one of the following vars: -# HAVE_SYS_MODEM_H -# USE_TERMIOS -# USE_TERMIO -# USE_SGTTY -#-------------------------------------------------------------------- - -AC_DEFUN([TEA_SERIAL_PORT], [ - AC_CHECK_HEADERS(sys/modem.h) - AC_CACHE_CHECK([termios vs. termio vs. sgtty], tcl_cv_api_serial, [ - AC_RUN_IFELSE([AC_LANG_SOURCE([[ -#include - -int main() { - struct termios t; - if (tcgetattr(0, &t) == 0) { - cfsetospeed(&t, 0); - t.c_cflag |= PARENB | PARODD | CSIZE | CSTOPB; - return 0; - } - return 1; -}]])],[tcl_cv_api_serial=termios],[tcl_cv_api_serial=no],[tcl_cv_api_serial=no]) - if test $tcl_cv_api_serial = no ; then - AC_RUN_IFELSE([AC_LANG_SOURCE([[ -#include - -int main() { - struct termio t; - if (ioctl(0, TCGETA, &t) == 0) { - t.c_cflag |= CBAUD | PARENB | PARODD | CSIZE | CSTOPB; - return 0; - } - return 1; -}]])],[tcl_cv_api_serial=termio],[tcl_cv_api_serial=no],[tcl_cv_api_serial=no]) - fi - if test $tcl_cv_api_serial = no ; then - AC_RUN_IFELSE([AC_LANG_SOURCE([[ -#include - -int main() { - struct sgttyb t; - if (ioctl(0, TIOCGETP, &t) == 0) { - t.sg_ospeed = 0; - t.sg_flags |= ODDP | EVENP | RAW; - return 0; - } - return 1; -}]])],[tcl_cv_api_serial=sgtty],[tcl_cv_api_serial=no],[tcl_cv_api_serial=no]) - fi - if test $tcl_cv_api_serial = no ; then - AC_RUN_IFELSE([AC_LANG_SOURCE([[ -#include -#include - -int main() { - struct termios t; - if (tcgetattr(0, &t) == 0 - || errno == ENOTTY || errno == ENXIO || errno == EINVAL) { - cfsetospeed(&t, 0); - t.c_cflag |= PARENB | PARODD | CSIZE | CSTOPB; - return 0; - } - return 1; -}]])],[tcl_cv_api_serial=termios],[tcl_cv_api_serial=no],[tcl_cv_api_serial=no]) - fi - if test $tcl_cv_api_serial = no; then - AC_RUN_IFELSE([AC_LANG_SOURCE([[ -#include -#include - -int main() { - struct termio t; - if (ioctl(0, TCGETA, &t) == 0 - || errno == ENOTTY || errno == ENXIO || errno == EINVAL) { - t.c_cflag |= CBAUD | PARENB | PARODD | CSIZE | CSTOPB; - return 0; - } - return 1; - }]])],[tcl_cv_api_serial=termio],[tcl_cv_api_serial=no],[tcl_cv_api_serial=no]) - fi - if test $tcl_cv_api_serial = no; then - AC_RUN_IFELSE([AC_LANG_SOURCE([[ -#include -#include - -int main() { - struct sgttyb t; - if (ioctl(0, TIOCGETP, &t) == 0 - || errno == ENOTTY || errno == ENXIO || errno == EINVAL) { - t.sg_ospeed = 0; - t.sg_flags |= ODDP | EVENP | RAW; - return 0; - } - return 1; -}]])],[tcl_cv_api_serial=sgtty],[tcl_cv_api_serial=none],[tcl_cv_api_serial=none]) - fi]) - case $tcl_cv_api_serial in - termios) AC_DEFINE(USE_TERMIOS, 1, [Use the termios API for serial lines]);; - termio) AC_DEFINE(USE_TERMIO, 1, [Use the termio API for serial lines]);; - sgtty) AC_DEFINE(USE_SGTTY, 1, [Use the sgtty API for serial lines]);; - esac -]) - -#-------------------------------------------------------------------- -# TEA_PATH_X -# -# Locate the X11 header files and the X11 library archive. Try -# the ac_path_x macro first, but if it doesn't find the X stuff -# (e.g. because there's no xmkmf program) then check through -# a list of possible directories. Under some conditions the -# autoconf macro will return an include directory that contains -# no include files, so double-check its result just to be safe. -# -# This should be called after TEA_CONFIG_CFLAGS as setting the -# LIBS line can confuse some configure macro magic. -# -# Arguments: -# none -# -# Results: -# -# Sets the following vars: -# XINCLUDES -# XLIBSW -# PKG_LIBS (appends to) -#-------------------------------------------------------------------- - -AC_DEFUN([TEA_PATH_X], [ - if test "${TEA_WINDOWINGSYSTEM}" = "x11" ; then - TEA_PATH_UNIX_X - fi -]) - -AC_DEFUN([TEA_PATH_UNIX_X], [ - AC_PATH_X - not_really_there="" - if test "$no_x" = ""; then - if test "$x_includes" = ""; then - AC_PREPROC_IFELSE([AC_LANG_SOURCE([[#include ]])],[],[not_really_there="yes"]) - else - if test ! -r $x_includes/X11/Xlib.h; then - not_really_there="yes" - fi - fi - fi - if test "$no_x" = "yes" -o "$not_really_there" = "yes"; then - AC_MSG_CHECKING([for X11 header files]) - found_xincludes="no" - AC_PREPROC_IFELSE([AC_LANG_SOURCE([[#include ]])],[found_xincludes="yes"],[found_xincludes="no"]) - if test "$found_xincludes" = "no"; then - dirs="/usr/unsupported/include /usr/local/include /usr/X386/include /usr/X11R6/include /usr/X11R5/include /usr/include/X11R5 /usr/include/X11R4 /usr/openwin/include /usr/X11/include /usr/sww/include" - for i in $dirs ; do - if test -r $i/X11/Xlib.h; then - AC_MSG_RESULT([$i]) - XINCLUDES=" -I$i" - found_xincludes="yes" - break - fi - done - fi - else - if test "$x_includes" != ""; then - XINCLUDES="-I$x_includes" - found_xincludes="yes" - fi - fi - if test "$found_xincludes" = "no"; then - AC_MSG_RESULT([couldn't find any!]) - fi - - if test "$no_x" = yes; then - AC_MSG_CHECKING([for X11 libraries]) - XLIBSW=nope - dirs="/usr/unsupported/lib /usr/local/lib /usr/X386/lib /usr/X11R6/lib /usr/X11R5/lib /usr/lib/X11R5 /usr/lib/X11R4 /usr/openwin/lib /usr/X11/lib /usr/sww/X11/lib" - for i in $dirs ; do - if test -r $i/libX11.a -o -r $i/libX11.so -o -r $i/libX11.sl -o -r $i/libX11.dylib; then - AC_MSG_RESULT([$i]) - XLIBSW="-L$i -lX11" - x_libraries="$i" - break - fi - done - else - if test "$x_libraries" = ""; then - XLIBSW=-lX11 - else - XLIBSW="-L$x_libraries -lX11" - fi - fi - if test "$XLIBSW" = nope ; then - AC_CHECK_LIB(Xwindow, XCreateWindow, XLIBSW=-lXwindow) - fi - if test "$XLIBSW" = nope ; then - AC_MSG_RESULT([could not find any! Using -lX11.]) - XLIBSW=-lX11 - fi - # TEA specific: - if test x"${XLIBSW}" != x ; then - PKG_LIBS="${PKG_LIBS} ${XLIBSW}" - fi -]) - -#-------------------------------------------------------------------- -# TEA_BLOCKING_STYLE -# -# The statements below check for systems where POSIX-style -# non-blocking I/O (O_NONBLOCK) doesn't work or is unimplemented. -# On these systems (mostly older ones), use the old BSD-style -# FIONBIO approach instead. -# -# Arguments: -# none -# -# Results: -# -# Defines some of the following vars: -# HAVE_SYS_IOCTL_H -# HAVE_SYS_FILIO_H -# USE_FIONBIO -# O_NONBLOCK -#-------------------------------------------------------------------- - -AC_DEFUN([TEA_BLOCKING_STYLE], [ - AC_CHECK_HEADERS(sys/ioctl.h) - AC_CHECK_HEADERS(sys/filio.h) - TEA_CONFIG_SYSTEM - AC_MSG_CHECKING([FIONBIO vs. O_NONBLOCK for nonblocking I/O]) - case $system in - OSF*) - AC_DEFINE(USE_FIONBIO, 1, [Should we use FIONBIO?]) - AC_MSG_RESULT([FIONBIO]) - ;; - *) - AC_MSG_RESULT([O_NONBLOCK]) - ;; - esac -]) - -#-------------------------------------------------------------------- -# TEA_TIME_HANDLER -# -# Checks how the system deals with time.h, what time structures -# are used on the system, and what fields the structures have. -# -# Arguments: -# none -# -# Results: -# -# Defines some of the following vars: -# USE_DELTA_FOR_TZ -# HAVE_TM_GMTOFF -# HAVE_TM_TZADJ -# HAVE_TIMEZONE_VAR -# -#-------------------------------------------------------------------- - -AC_DEFUN([TEA_TIME_HANDLER], [ - AC_CHECK_HEADERS(sys/time.h) - AC_HEADER_TIME - AC_STRUCT_TIMEZONE - - AC_CHECK_FUNCS(gmtime_r localtime_r mktime) - - AC_CACHE_CHECK([tm_tzadj in struct tm], tcl_cv_member_tm_tzadj, [ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[struct tm tm; (void)tm.tm_tzadj;]])], - [tcl_cv_member_tm_tzadj=yes], - [tcl_cv_member_tm_tzadj=no])]) - if test $tcl_cv_member_tm_tzadj = yes ; then - AC_DEFINE(HAVE_TM_TZADJ, 1, [Should we use the tm_tzadj field of struct tm?]) - fi - - AC_CACHE_CHECK([tm_gmtoff in struct tm], tcl_cv_member_tm_gmtoff, [ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[struct tm tm; (void)tm.tm_gmtoff;]])], - [tcl_cv_member_tm_gmtoff=yes], - [tcl_cv_member_tm_gmtoff=no])]) - if test $tcl_cv_member_tm_gmtoff = yes ; then - AC_DEFINE(HAVE_TM_GMTOFF, 1, [Should we use the tm_gmtoff field of struct tm?]) - fi - - # - # Its important to include time.h in this check, as some systems - # (like convex) have timezone functions, etc. - # - AC_CACHE_CHECK([long timezone variable], tcl_cv_timezone_long, [ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include -#include ]], - [[extern long timezone; - timezone += 1; - exit (0);]])], - [tcl_cv_timezone_long=yes], [tcl_cv_timezone_long=no])]) - if test $tcl_cv_timezone_long = yes ; then - AC_DEFINE(HAVE_TIMEZONE_VAR, 1, [Should we use the global timezone variable?]) - else - # - # On some systems (eg IRIX 6.2), timezone is a time_t and not a long. - # - AC_CACHE_CHECK([time_t timezone variable], tcl_cv_timezone_time, [ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include -#include ]], - [[extern time_t timezone; - timezone += 1; - exit (0);]])], - [tcl_cv_timezone_time=yes], [tcl_cv_timezone_time=no])]) - if test $tcl_cv_timezone_time = yes ; then - AC_DEFINE(HAVE_TIMEZONE_VAR, 1, [Should we use the global timezone variable?]) - fi - fi -]) - -#-------------------------------------------------------------------- -# TEA_BUGGY_STRTOD -# -# Under Solaris 2.4, strtod returns the wrong value for the -# terminating character under some conditions. Check for this -# and if the problem exists use a substitute procedure -# "fixstrtod" (provided by Tcl) that corrects the error. -# Also, on Compaq's Tru64 Unix 5.0, -# strtod(" ") returns 0.0 instead of a failure to convert. -# -# Arguments: -# none -# -# Results: -# -# Might defines some of the following vars: -# strtod (=fixstrtod) -#-------------------------------------------------------------------- - -AC_DEFUN([TEA_BUGGY_STRTOD], [ - AC_CHECK_FUNC(strtod, tcl_strtod=1, tcl_strtod=0) - if test "$tcl_strtod" = 1; then - AC_CACHE_CHECK([for Solaris2.4/Tru64 strtod bugs], tcl_cv_strtod_buggy,[ - AC_RUN_IFELSE([AC_LANG_SOURCE([[ - #include - extern double strtod(); - int main() { - char *infString="Inf", *nanString="NaN", *spaceString=" "; - char *term; - double value; - value = strtod(infString, &term); - if ((term != infString) && (term[-1] == 0)) { - exit(1); - } - value = strtod(nanString, &term); - if ((term != nanString) && (term[-1] == 0)) { - exit(1); - } - value = strtod(spaceString, &term); - if (term == (spaceString+1)) { - exit(1); - } - exit(0); - }]])], [tcl_cv_strtod_buggy=ok], [tcl_cv_strtod_buggy=buggy], - [tcl_cv_strtod_buggy=buggy])]) - if test "$tcl_cv_strtod_buggy" = buggy; then - AC_LIBOBJ([fixstrtod]) - USE_COMPAT=1 - AC_DEFINE(strtod, fixstrtod, [Do we want to use the strtod() in compat?]) - fi - fi -]) - -#-------------------------------------------------------------------- -# TEA_TCL_LINK_LIBS -# -# Search for the libraries needed to link the Tcl shell. -# Things like the math library (-lm), socket stuff (-lsocket vs. -# -lnsl), zlib (-lz) and libtommath (-ltommath) are dealt with here. -# -# Arguments: -# None. -# -# Results: -# -# Might append to the following vars: -# LIBS -# MATH_LIBS -# -# Might define the following vars: -# HAVE_NET_ERRNO_H -# -#-------------------------------------------------------------------- - -AC_DEFUN([TEA_TCL_LINK_LIBS], [ - #-------------------------------------------------------------------- - # On a few very rare systems, all of the libm.a stuff is - # already in libc.a. Set compiler flags accordingly. - #-------------------------------------------------------------------- - - AC_CHECK_FUNC(sin, MATH_LIBS="", MATH_LIBS="-lm") - - #-------------------------------------------------------------------- - # Interactive UNIX requires -linet instead of -lsocket, plus it - # needs net/errno.h to define the socket-related error codes. - #-------------------------------------------------------------------- - - AC_CHECK_LIB(inet, main, [LIBS="$LIBS -linet"]) - AC_CHECK_HEADER(net/errno.h, [ - AC_DEFINE(HAVE_NET_ERRNO_H, 1, [Do we have ?])]) - - #-------------------------------------------------------------------- - # Check for the existence of the -lsocket and -lnsl libraries. - # The order here is important, so that they end up in the right - # order in the command line generated by make. Here are some - # special considerations: - # 1. Use "connect" and "accept" to check for -lsocket, and - # "gethostbyname" to check for -lnsl. - # 2. Use each function name only once: can't redo a check because - # autoconf caches the results of the last check and won't redo it. - # 3. Use -lnsl and -lsocket only if they supply procedures that - # aren't already present in the normal libraries. This is because - # IRIX 5.2 has libraries, but they aren't needed and they're - # bogus: they goof up name resolution if used. - # 4. On some SVR4 systems, can't use -lsocket without -lnsl too. - # To get around this problem, check for both libraries together - # if -lsocket doesn't work by itself. - #-------------------------------------------------------------------- - - tcl_checkBoth=0 - AC_CHECK_FUNC(connect, tcl_checkSocket=0, tcl_checkSocket=1) - if test "$tcl_checkSocket" = 1; then - AC_CHECK_FUNC(setsockopt, , [AC_CHECK_LIB(socket, setsockopt, - LIBS="$LIBS -lsocket", tcl_checkBoth=1)]) - fi - if test "$tcl_checkBoth" = 1; then - tk_oldLibs=$LIBS - LIBS="$LIBS -lsocket -lnsl" - AC_CHECK_FUNC(accept, tcl_checkNsl=0, [LIBS=$tk_oldLibs]) - fi - AC_CHECK_FUNC(gethostbyname, , [AC_CHECK_LIB(nsl, gethostbyname, - [LIBS="$LIBS -lnsl"])]) - AC_CHECK_FUNC(mp_log_u32, , [AC_CHECK_LIB(tommath, mp_log_u32, - [LIBS="$LIBS -ltommath"])]) - AC_CHECK_FUNC(deflateSetHeader, , [AC_CHECK_LIB(z, deflateSetHeader, - [LIBS="$LIBS -lz"])]) -]) - -#-------------------------------------------------------------------- -# TEA_TCL_EARLY_FLAGS -# -# Check for what flags are needed to be passed so the correct OS -# features are available. -# -# Arguments: -# None -# -# Results: -# -# Might define the following vars: -# _ISOC99_SOURCE -# _FILE_OFFSET_BITS -# -#-------------------------------------------------------------------- - -AC_DEFUN([TEA_TCL_EARLY_FLAG],[ - AC_CACHE_VAL([tcl_cv_flag_]translit($1,[A-Z],[a-z]), - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$2]], [[$3]])], - [tcl_cv_flag_]translit($1,[A-Z],[a-z])=no,[AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[[#define ]$1[ ]m4_default([$4],[1])[ -]$2]], [[$3]])], - [tcl_cv_flag_]translit($1,[A-Z],[a-z])=yes, - [tcl_cv_flag_]translit($1,[A-Z],[a-z])=no)])) - if test ["x${tcl_cv_flag_]translit($1,[A-Z],[a-z])[}" = "xyes"] ; then - AC_DEFINE($1, m4_default([$4],[1]), [Add the ]$1[ flag when building]) - tcl_flags="$tcl_flags $1" - fi -]) - -AC_DEFUN([TEA_TCL_EARLY_FLAGS],[ - AC_MSG_CHECKING([for required early compiler flags]) - tcl_flags="" - TEA_TCL_EARLY_FLAG(_ISOC99_SOURCE,[#include ], - [char *p = (char *)strtoll; char *q = (char *)strtoull;]) - if test "${TCL_MAJOR_VERSION}" -ne 8 ; then - TEA_TCL_EARLY_FLAG(_FILE_OFFSET_BITS,[#include ], - [switch (0) { case 0: case (sizeof(off_t)==sizeof(long long)): ; }],64) - fi - if test "x${tcl_flags}" = "x" ; then - AC_MSG_RESULT([none]) - else - AC_MSG_RESULT([${tcl_flags}]) - fi -]) - -#-------------------------------------------------------------------- -# TEA_TCL_64BIT_FLAGS -# -# Check for what is defined in the way of 64-bit features. -# -# Arguments: -# None -# -# Results: -# -# Might define the following vars: -# TCL_WIDE_INT_IS_LONG -# TCL_WIDE_INT_TYPE -# HAVE_STRUCT_DIRENT64, HAVE_DIR64 -# HAVE_STRUCT_STAT64 -# HAVE_TYPE_OFF64_T -# _TIME_BITS -# -#-------------------------------------------------------------------- - -AC_DEFUN([TEA_TCL_64BIT_FLAGS], [ - AC_MSG_CHECKING([for 64-bit integer type]) - AC_CACHE_VAL(tcl_cv_type_64bit,[ - tcl_cv_type_64bit=none - # See if the compiler knows natively about __int64 - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[__int64 value = (__int64) 0;]])], - [tcl_type_64bit=__int64],[tcl_type_64bit="long long"]) - # See if we could use long anyway Note that we substitute in the - # type that is our current guess for a 64-bit type inside this check - # program, so it should be modified only carefully... - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[switch (0) { - case 1: case (sizeof(${tcl_type_64bit})==sizeof(long)): ; - }]])],[tcl_cv_type_64bit=${tcl_type_64bit}],[])]) - if test "${tcl_cv_type_64bit}" = none ; then - AC_DEFINE(TCL_WIDE_INT_IS_LONG, 1, [Do 'long' and 'long long' have the same size (64-bit)?]) - AC_MSG_RESULT([yes]) - elif test "${tcl_cv_type_64bit}" = "__int64" \ - -a "${TEA_PLATFORM}" = "windows" ; then - # TEA specific: We actually want to use the default tcl.h checks in - # this case to handle both TCL_WIDE_INT_TYPE and TCL_LL_MODIFIER* - AC_MSG_RESULT([using Tcl header defaults]) - else - AC_DEFINE_UNQUOTED(TCL_WIDE_INT_TYPE,${tcl_cv_type_64bit}, - [What type should be used to define wide integers?]) - AC_MSG_RESULT([${tcl_cv_type_64bit}]) - - # Now check for auxiliary declarations - if test "${TCL_MAJOR_VERSION}" -ne 8 ; then - AC_CACHE_CHECK([for 64-bit time_t], tcl_cv_time_t_64,[ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], - [[switch (0) {case 0: case (sizeof(time_t)==sizeof(long long)): ;}]])], - [tcl_cv_time_t_64=yes],[tcl_cv_time_t_64=no])]) - if test "x${tcl_cv_time_t_64}" = "xno" ; then - # Note that _TIME_BITS=64 requires _FILE_OFFSET_BITS=64 - # which SC_TCL_EARLY_FLAGS has defined if necessary. - AC_CACHE_CHECK([if _TIME_BITS=64 enables 64-bit time_t], tcl_cv__time_bits,[ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#define _TIME_BITS 64 -#include ]], - [[switch (0) {case 0: case (sizeof(time_t)==sizeof(long long)): ;}]])], - [tcl_cv__time_bits=yes],[tcl_cv__time_bits=no])]) - if test "x${tcl_cv__time_bits}" = "xyes" ; then - AC_DEFINE(_TIME_BITS, 64, [_TIME_BITS=64 enables 64-bit time_t.]) - fi - fi - fi - - AC_CACHE_CHECK([for struct dirent64], tcl_cv_struct_dirent64,[ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include -#include ]], [[struct dirent64 p;]])], - [tcl_cv_struct_dirent64=yes],[tcl_cv_struct_dirent64=no])]) - if test "x${tcl_cv_struct_dirent64}" = "xyes" ; then - AC_DEFINE(HAVE_STRUCT_DIRENT64, 1, [Is 'struct dirent64' in ?]) - fi - - AC_CACHE_CHECK([for DIR64], tcl_cv_DIR64,[ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include -#include ]], [[struct dirent64 *p; DIR64 d = opendir64("."); - p = readdir64(d); rewinddir64(d); closedir64(d);]])], - [tcl_cv_DIR64=yes], [tcl_cv_DIR64=no])]) - if test "x${tcl_cv_DIR64}" = "xyes" ; then - AC_DEFINE(HAVE_DIR64, 1, [Is 'DIR64' in ?]) - fi - - AC_CACHE_CHECK([for struct stat64], tcl_cv_struct_stat64,[ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[struct stat64 p; -]])], - [tcl_cv_struct_stat64=yes], [tcl_cv_struct_stat64=no])]) - if test "x${tcl_cv_struct_stat64}" = "xyes" ; then - AC_DEFINE(HAVE_STRUCT_STAT64, 1, [Is 'struct stat64' in ?]) - fi - - AC_CHECK_FUNCS(open64 lseek64) - AC_MSG_CHECKING([for off64_t]) - AC_CACHE_VAL(tcl_cv_type_off64_t,[ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[off64_t offset; -]])], - [tcl_cv_type_off64_t=yes], [tcl_cv_type_off64_t=no])]) - dnl Define HAVE_TYPE_OFF64_T only when the off64_t type and the - dnl functions lseek64 and open64 are defined. - if test "x${tcl_cv_type_off64_t}" = "xyes" && \ - test "x${ac_cv_func_lseek64}" = "xyes" && \ - test "x${ac_cv_func_open64}" = "xyes" ; then - AC_DEFINE(HAVE_TYPE_OFF64_T, 1, [Is off64_t in ?]) - AC_MSG_RESULT([yes]) - else - AC_MSG_RESULT([no]) - fi - fi -]) - -## -## Here ends the standard Tcl configuration bits and starts the -## TEA specific functions -## - -#------------------------------------------------------------------------ -# TEA_INIT -- -# -# Init various Tcl Extension Architecture (TEA) variables. -# This should be the first called TEA_* macro. -# -# Arguments: -# none -# -# Results: -# -# Defines and substs the following vars: -# CYGPATH -# EXEEXT -# Defines only: -# TEA_VERSION -# TEA_INITED -# TEA_PLATFORM (windows or unix) -# -# "cygpath" is used on windows to generate native path names for include -# files. These variables should only be used with the compiler and linker -# since they generate native path names. -# -# EXEEXT -# Select the executable extension based on the host type. This -# is a lightweight replacement for AC_EXEEXT that doesn't require -# a compiler. -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_INIT], [ - TEA_VERSION="3.13" - - AC_MSG_CHECKING([TEA configuration]) - if test x"${PACKAGE_NAME}" = x ; then - AC_MSG_ERROR([ -The PACKAGE_NAME variable must be defined by your TEA configure.ac]) - fi - AC_MSG_RESULT([ok (TEA ${TEA_VERSION})]) - - # If the user did not set CFLAGS, set it now to keep macros - # like AC_PROG_CC and AC_TRY_COMPILE from adding "-g -O2". - if test "${CFLAGS+set}" != "set" ; then - CFLAGS="" - fi - - case "`uname -s`" in - *win32*|*WIN32*|*MINGW32_*|*MINGW64_*|*MSYS_*) - AC_CHECK_PROG(CYGPATH, cygpath, cygpath -m, echo) - EXEEXT=".exe" - TEA_PLATFORM="windows" - ;; - *CYGWIN_*) - EXEEXT=".exe" - # CYGPATH and TEA_PLATFORM are determined later in LOAD_TCLCONFIG - ;; - *) - CYGPATH=echo - # Maybe we are cross-compiling.... - case ${host_alias} in - *mingw32*) - EXEEXT=".exe" - TEA_PLATFORM="windows" - ;; - *) - EXEEXT="" - TEA_PLATFORM="unix" - ;; - esac - ;; - esac - - # Check if exec_prefix is set. If not use fall back to prefix. - # Note when adjusted, so that TEA_PREFIX can correct for this. - # This is needed for recursive configures, since autoconf propagates - # $prefix, but not $exec_prefix (doh!). - if test x$exec_prefix = xNONE ; then - exec_prefix_default=yes - exec_prefix=$prefix - fi - - AC_MSG_NOTICE([configuring ${PACKAGE_NAME} ${PACKAGE_VERSION}]) - - AC_SUBST(EXEEXT) - AC_SUBST(CYGPATH) - - # This package name must be replaced statically for AC_SUBST to work - AC_SUBST(PKG_LIB_FILE) - AC_SUBST(PKG_LIB_FILE8) - AC_SUBST(PKG_LIB_FILE9) - - # We AC_SUBST these here to ensure they are subst'ed, - # in case the user doesn't call TEA_ADD_... - AC_SUBST(PKG_STUB_SOURCES) - AC_SUBST(PKG_STUB_OBJECTS) - AC_SUBST(PKG_TCL_SOURCES) - AC_SUBST(PKG_HEADERS) - AC_SUBST(PKG_INCLUDES) - AC_SUBST(PKG_LIBS) - AC_SUBST(PKG_CFLAGS) - - # Configure the installer. - TEA_INSTALLER -]) - -#------------------------------------------------------------------------ -# TEA_ADD_SOURCES -- -# -# Specify one or more source files. Users should check for -# the right platform before adding to their list. -# It is not important to specify the directory, as long as it is -# in the generic, win or unix subdirectory of $(srcdir). -# -# Arguments: -# one or more file names -# -# Results: -# -# Defines and substs the following vars: -# PKG_SOURCES -# PKG_OBJECTS -#------------------------------------------------------------------------ -AC_DEFUN([TEA_ADD_SOURCES], [ - vars="$@" - for i in $vars; do - case $i in - [\$]*) - # allow $-var names - PKG_SOURCES="$PKG_SOURCES $i" - PKG_OBJECTS="$PKG_OBJECTS $i" - ;; - *) - # check for existence - allows for generic/win/unix VPATH - # To add more dirs here (like 'src'), you have to update VPATH - # in Makefile.in as well - if test ! -f "${srcdir}/$i" -a ! -f "${srcdir}/generic/$i" \ - -a ! -f "${srcdir}/win/$i" -a ! -f "${srcdir}/unix/$i" \ - -a ! -f "${srcdir}/macosx/$i" \ - ; then - AC_MSG_ERROR([could not find source file '$i']) - fi - PKG_SOURCES="$PKG_SOURCES $i" - # this assumes it is in a VPATH dir - i=`basename $i` - # handle user calling this before or after TEA_SETUP_COMPILER - if test x"${OBJEXT}" != x ; then - j="`echo $i | sed -e 's/\.[[^.]]*$//'`.${OBJEXT}" - else - j="`echo $i | sed -e 's/\.[[^.]]*$//'`.\${OBJEXT}" - fi - PKG_OBJECTS="$PKG_OBJECTS $j" - ;; - esac - done - AC_SUBST(PKG_SOURCES) - AC_SUBST(PKG_OBJECTS) -]) - -#------------------------------------------------------------------------ -# TEA_ADD_STUB_SOURCES -- -# -# Specify one or more source files. Users should check for -# the right platform before adding to their list. -# It is not important to specify the directory, as long as it is -# in the generic, win or unix subdirectory of $(srcdir). -# -# Arguments: -# one or more file names -# -# Results: -# -# Defines and substs the following vars: -# PKG_STUB_SOURCES -# PKG_STUB_OBJECTS -#------------------------------------------------------------------------ -AC_DEFUN([TEA_ADD_STUB_SOURCES], [ - vars="$@" - for i in $vars; do - # check for existence - allows for generic/win/unix VPATH - if test ! -f "${srcdir}/$i" -a ! -f "${srcdir}/generic/$i" \ - -a ! -f "${srcdir}/win/$i" -a ! -f "${srcdir}/unix/$i" \ - -a ! -f "${srcdir}/macosx/$i" \ - ; then - AC_MSG_ERROR([could not find stub source file '$i']) - fi - PKG_STUB_SOURCES="$PKG_STUB_SOURCES $i" - # this assumes it is in a VPATH dir - i=`basename $i` - # handle user calling this before or after TEA_SETUP_COMPILER - if test x"${OBJEXT}" != x ; then - j="`echo $i | sed -e 's/\.[[^.]]*$//'`.${OBJEXT}" - else - j="`echo $i | sed -e 's/\.[[^.]]*$//'`.\${OBJEXT}" - fi - PKG_STUB_OBJECTS="$PKG_STUB_OBJECTS $j" - done - AC_SUBST(PKG_STUB_SOURCES) - AC_SUBST(PKG_STUB_OBJECTS) -]) - -#------------------------------------------------------------------------ -# TEA_ADD_TCL_SOURCES -- -# -# Specify one or more Tcl source files. These should be platform -# independent runtime files. -# -# Arguments: -# one or more file names -# -# Results: -# -# Defines and substs the following vars: -# PKG_TCL_SOURCES -#------------------------------------------------------------------------ -AC_DEFUN([TEA_ADD_TCL_SOURCES], [ - vars="$@" - for i in $vars; do - # check for existence, be strict because it is installed - if test ! -f "${srcdir}/$i" ; then - AC_MSG_ERROR([could not find tcl source file '${srcdir}/$i']) - fi - PKG_TCL_SOURCES="$PKG_TCL_SOURCES $i" - done - AC_SUBST(PKG_TCL_SOURCES) -]) - -#------------------------------------------------------------------------ -# TEA_ADD_HEADERS -- -# -# Specify one or more source headers. Users should check for -# the right platform before adding to their list. -# -# Arguments: -# one or more file names -# -# Results: -# -# Defines and substs the following vars: -# PKG_HEADERS -#------------------------------------------------------------------------ -AC_DEFUN([TEA_ADD_HEADERS], [ - vars="$@" - for i in $vars; do - # check for existence, be strict because it is installed - if test ! -f "${srcdir}/$i" ; then - AC_MSG_ERROR([could not find header file '${srcdir}/$i']) - fi - PKG_HEADERS="$PKG_HEADERS $i" - done - AC_SUBST(PKG_HEADERS) -]) - -#------------------------------------------------------------------------ -# TEA_ADD_INCLUDES -- -# -# Specify one or more include dirs. Users should check for -# the right platform before adding to their list. -# -# Arguments: -# one or more file names -# -# Results: -# -# Defines and substs the following vars: -# PKG_INCLUDES -#------------------------------------------------------------------------ -AC_DEFUN([TEA_ADD_INCLUDES], [ - vars="$@" - for i in $vars; do - PKG_INCLUDES="$PKG_INCLUDES $i" - done - AC_SUBST(PKG_INCLUDES) -]) - -#------------------------------------------------------------------------ -# TEA_ADD_LIBS -- -# -# Specify one or more libraries. Users should check for -# the right platform before adding to their list. For Windows, -# libraries provided in "foo.lib" format will be converted to -# "-lfoo" when using GCC (mingw). -# -# Arguments: -# one or more file names -# -# Results: -# -# Defines and substs the following vars: -# PKG_LIBS -#------------------------------------------------------------------------ -AC_DEFUN([TEA_ADD_LIBS], [ - vars="$@" - for i in $vars; do - if test "${TEA_PLATFORM}" = "windows" -a "$GCC" = "yes" ; then - # Convert foo.lib to -lfoo for GCC. No-op if not *.lib - i=`echo "$i" | sed -e 's/^\([[^-]].*\)\.[[lL]][[iI]][[bB]][$]/-l\1/'` - fi - PKG_LIBS="$PKG_LIBS $i" - done - AC_SUBST(PKG_LIBS) -]) - -#------------------------------------------------------------------------ -# TEA_ADD_CFLAGS -- -# -# Specify one or more CFLAGS. Users should check for -# the right platform before adding to their list. -# -# Arguments: -# one or more file names -# -# Results: -# -# Defines and substs the following vars: -# PKG_CFLAGS -#------------------------------------------------------------------------ -AC_DEFUN([TEA_ADD_CFLAGS], [ - PKG_CFLAGS="$PKG_CFLAGS $@" - AC_SUBST(PKG_CFLAGS) -]) - -#------------------------------------------------------------------------ -# TEA_ADD_CLEANFILES -- -# -# Specify one or more CLEANFILES. -# -# Arguments: -# one or more file names to clean target -# -# Results: -# -# Appends to CLEANFILES, already defined for subst in LOAD_TCLCONFIG -#------------------------------------------------------------------------ -AC_DEFUN([TEA_ADD_CLEANFILES], [ - CLEANFILES="$CLEANFILES $@" -]) - -#------------------------------------------------------------------------ -# TEA_PREFIX -- -# -# Handle the --prefix=... option by defaulting to what Tcl gave -# -# Arguments: -# none -# -# Results: -# -# If --prefix or --exec-prefix was not specified, $prefix and -# $exec_prefix will be set to the values given to Tcl when it was -# configured. -#------------------------------------------------------------------------ -AC_DEFUN([TEA_PREFIX], [ - if test "${prefix}" = "NONE"; then - prefix_default=yes - if test x"${TCL_PREFIX}" != x; then - AC_MSG_NOTICE([--prefix defaulting to TCL_PREFIX ${TCL_PREFIX}]) - prefix=${TCL_PREFIX} - else - AC_MSG_NOTICE([--prefix defaulting to /usr/local]) - prefix=/usr/local - fi - fi - if test "${exec_prefix}" = "NONE" -a x"${prefix_default}" = x"yes" \ - -o x"${exec_prefix_default}" = x"yes" ; then - if test x"${TCL_EXEC_PREFIX}" != x; then - AC_MSG_NOTICE([--exec-prefix defaulting to TCL_EXEC_PREFIX ${TCL_EXEC_PREFIX}]) - exec_prefix=${TCL_EXEC_PREFIX} - else - AC_MSG_NOTICE([--exec-prefix defaulting to ${prefix}]) - exec_prefix=$prefix - fi - fi -]) - -#------------------------------------------------------------------------ -# TEA_SETUP_COMPILER_CC -- -# -# Do compiler checks the way we want. This is just a replacement -# for AC_PROG_CC in TEA configure.ac files to make them cleaner. -# -# Arguments: -# none -# -# Results: -# -# Sets up CC var and other standard bits we need to make executables. -#------------------------------------------------------------------------ -AC_DEFUN([TEA_SETUP_COMPILER_CC], [ - # Don't put any macros that use the compiler (e.g. AC_TRY_COMPILE) - # in this macro, they need to go into TEA_SETUP_COMPILER instead. - - AC_PROG_CC - AC_PROG_CPP - - #-------------------------------------------------------------------- - # Checks to see if the make program sets the $MAKE variable. - #-------------------------------------------------------------------- - - AC_PROG_MAKE_SET - - #-------------------------------------------------------------------- - # Find ranlib - #-------------------------------------------------------------------- - - AC_CHECK_TOOL(RANLIB, ranlib) - - #-------------------------------------------------------------------- - # Determines the correct binary file extension (.o, .obj, .exe etc.) - #-------------------------------------------------------------------- - - AC_OBJEXT - AC_EXEEXT -]) - -#------------------------------------------------------------------------ -# TEA_SETUP_COMPILER -- -# -# Do compiler checks that use the compiler. This must go after -# TEA_SETUP_COMPILER_CC, which does the actual compiler check. -# -# Arguments: -# none -# -# Results: -# -# Sets up CC var and other standard bits we need to make executables. -#------------------------------------------------------------------------ -AC_DEFUN([TEA_SETUP_COMPILER], [ - # Any macros that use the compiler (e.g. AC_TRY_COMPILE) have to go here. - AC_REQUIRE([TEA_SETUP_COMPILER_CC]) - - #------------------------------------------------------------------------ - # If we're using GCC, see if the compiler understands -pipe. If so, use it. - # It makes compiling go faster. (This is only a performance feature.) - #------------------------------------------------------------------------ - - if test -z "$no_pipe" -a -n "$GCC"; then - AC_CACHE_CHECK([if the compiler understands -pipe], - tcl_cv_cc_pipe, [ - hold_cflags=$CFLAGS; CFLAGS="$CFLAGS -pipe" - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[]])],[tcl_cv_cc_pipe=yes],[tcl_cv_cc_pipe=no]) - CFLAGS=$hold_cflags]) - if test $tcl_cv_cc_pipe = yes; then - CFLAGS="$CFLAGS -pipe" - fi - fi - - if test "${TCL_MAJOR_VERSION}" -lt 9 -a "${TCL_MINOR_VERSION}" -lt 7; then - AC_DEFINE(Tcl_Size, int, [Is 'Tcl_Size' in ?]) - fi - - #-------------------------------------------------------------------- - # Common compiler flag setup - #-------------------------------------------------------------------- - - AC_C_BIGENDIAN(,,,[#]) -]) - -#------------------------------------------------------------------------ -# TEA_MAKE_LIB -- -# -# Generate a line that can be used to build a shared/unshared library -# in a platform independent manner. -# -# Arguments: -# none -# -# Requires: -# -# Results: -# -# Defines the following vars: -# CFLAGS - Done late here to note disturb other AC macros -# MAKE_LIB - Command to execute to build the Tcl library; -# differs depending on whether or not Tcl is being -# compiled as a shared library. -# MAKE_SHARED_LIB Makefile rule for building a shared library -# MAKE_STATIC_LIB Makefile rule for building a static library -# MAKE_STUB_LIB Makefile rule for building a stub library -# VC_MANIFEST_EMBED_DLL Makefile rule for embedded VC manifest in DLL -# VC_MANIFEST_EMBED_EXE Makefile rule for embedded VC manifest in EXE -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_MAKE_LIB], [ - if test "${TEA_PLATFORM}" = "windows" -a "$GCC" != "yes"; then - MAKE_STATIC_LIB="\${STLIB_LD} -out:\[$]@ \$(PKG_OBJECTS)" - MAKE_SHARED_LIB="\${SHLIB_LD} \${LDFLAGS} \${LDFLAGS_DEFAULT} -out:\[$]@ \$(PKG_OBJECTS) \${SHLIB_LD_LIBS}" - AC_EGREP_CPP([manifest needed], [ -#if defined(_MSC_VER) && _MSC_VER >= 1400 -print("manifest needed") -#endif - ], [ - # Could do a CHECK_PROG for mt, but should always be with MSVC8+ - VC_MANIFEST_EMBED_DLL="if test -f \[$]@.manifest ; then mt.exe -nologo -manifest \[$]@.manifest -outputresource:\[$]@\;2 ; fi" - VC_MANIFEST_EMBED_EXE="if test -f \[$]@.manifest ; then mt.exe -nologo -manifest \[$]@.manifest -outputresource:\[$]@\;1 ; fi" - MAKE_SHARED_LIB="${MAKE_SHARED_LIB} ; ${VC_MANIFEST_EMBED_DLL}" - TEA_ADD_CLEANFILES([*.manifest]) - ]) - MAKE_STUB_LIB="\${STLIB_LD} -nodefaultlib -out:\[$]@ \$(PKG_STUB_OBJECTS)" - else - MAKE_STATIC_LIB="\${STLIB_LD} \[$]@ \$(PKG_OBJECTS)" - MAKE_SHARED_LIB="\${SHLIB_LD} \${LDFLAGS} \${LDFLAGS_DEFAULT} -o \[$]@ \$(PKG_OBJECTS) \${SHLIB_LD_LIBS}" - MAKE_STUB_LIB="\${STLIB_LD} \[$]@ \$(PKG_STUB_OBJECTS)" - fi - - if test "${SHARED_BUILD}" = "1" ; then - MAKE_LIB="${MAKE_SHARED_LIB} " - else - MAKE_LIB="${MAKE_STATIC_LIB} " - fi - - #-------------------------------------------------------------------- - # Shared libraries and static libraries have different names. - # Use the double eval to make sure any variables in the suffix is - # substituted. (@@@ Might not be necessary anymore) - #-------------------------------------------------------------------- - - PACKAGE_LIB_PREFIX8="${PACKAGE_LIB_PREFIX}" - PACKAGE_LIB_PREFIX9="${PACKAGE_LIB_PREFIX}tcl9" - if test "${TCL_MAJOR_VERSION}" -gt 8 -a x"${with_tcl8}" = x; then - PACKAGE_LIB_PREFIX="${PACKAGE_LIB_PREFIX9}" - else - PACKAGE_LIB_PREFIX="${PACKAGE_LIB_PREFIX8}" - AC_DEFINE(TCL_MAJOR_VERSION, 8, [Compile for Tcl8?]) - fi - if test "${TEA_PLATFORM}" = "windows" ; then - if test "${SHARED_BUILD}" = "1" ; then - # We force the unresolved linking of symbols that are really in - # the private libraries of Tcl and Tk. - if test x"${TK_BIN_DIR}" != x ; then - SHLIB_LD_LIBS="${SHLIB_LD_LIBS} \"`${CYGPATH} ${TK_BIN_DIR}/${TK_STUB_LIB_FILE}`\"" - fi - SHLIB_LD_LIBS="${SHLIB_LD_LIBS} \"`${CYGPATH} ${TCL_BIN_DIR}/${TCL_STUB_LIB_FILE}`\"" - if test "$GCC" = "yes"; then - SHLIB_LD_LIBS="${SHLIB_LD_LIBS} -static-libgcc" - fi - eval eval "PKG_LIB_FILE8=${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}${SHARED_LIB_SUFFIX}" - eval eval "PKG_LIB_FILE9=${PACKAGE_LIB_PREFIX9}${PACKAGE_NAME}${SHARED_LIB_SUFFIX}" - eval eval "PKG_LIB_FILE=${PACKAGE_LIB_PREFIX}${PACKAGE_NAME}${SHARED_LIB_SUFFIX}" - else - if test "$GCC" = "yes"; then - PACKAGE_LIB_PREFIX=lib${PACKAGE_LIB_PREFIX} - fi - eval eval "PKG_LIB_FILE8=${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}" - eval eval "PKG_LIB_FILE9=${PACKAGE_LIB_PREFIX9}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}" - eval eval "PKG_LIB_FILE=${PACKAGE_LIB_PREFIX}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}" - fi - # Some packages build their own stubs libraries - if test "${TCL_MAJOR_VERSION}" -gt 8 -a x"${with_tcl8}" = x; then - eval eval "PKG_STUB_LIB_FILE=${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}stub.a" - else - eval eval "PKG_STUB_LIB_FILE=${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}stub${UNSHARED_LIB_SUFFIX}" - fi - if test "$GCC" = "yes"; then - PKG_STUB_LIB_FILE=lib${PKG_STUB_LIB_FILE} - fi - # These aren't needed on Windows (either MSVC or gcc) - RANLIB=: - RANLIB_STUB=: - else - RANLIB_STUB="${RANLIB}" - if test "${SHARED_BUILD}" = "1" ; then - SHLIB_LD_LIBS="${SHLIB_LD_LIBS} ${TCL_STUB_LIB_SPEC}" - if test x"${TK_BIN_DIR}" != x ; then - SHLIB_LD_LIBS="${SHLIB_LD_LIBS} ${TK_STUB_LIB_SPEC}" - fi - eval eval "PKG_LIB_FILE8=lib${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}${SHARED_LIB_SUFFIX}" - eval eval "PKG_LIB_FILE9=lib${PACKAGE_LIB_PREFIX9}${PACKAGE_NAME}${SHARED_LIB_SUFFIX}" - eval eval "PKG_LIB_FILE=lib${PACKAGE_LIB_PREFIX}${PACKAGE_NAME}${SHARED_LIB_SUFFIX}" - RANLIB=: - else - eval eval "PKG_LIB_FILE8=lib${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}" - eval eval "PKG_LIB_FILE9=lib${PACKAGE_LIB_PREFIX9}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}" - eval eval "PKG_LIB_FILE=lib${PACKAGE_LIB_PREFIX}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}" - fi - # Some packages build their own stubs libraries - if test "${TCL_MAJOR_VERSION}" -gt 8 -a x"${with_tcl8}" = x; then - eval eval "PKG_STUB_LIB_FILE=lib${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}stub.a" - else - eval eval "PKG_STUB_LIB_FILE=lib${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}stub${UNSHARED_LIB_SUFFIX}" - fi - fi - - # These are escaped so that only CFLAGS is picked up at configure time. - # The other values will be substituted at make time. - CFLAGS="${CFLAGS} \${CFLAGS_DEFAULT} \${CFLAGS_WARNING}" - if test "${SHARED_BUILD}" = "1" ; then - CFLAGS="${CFLAGS} \${SHLIB_CFLAGS}" - fi - - AC_SUBST(MAKE_LIB) - AC_SUBST(MAKE_SHARED_LIB) - AC_SUBST(MAKE_STATIC_LIB) - AC_SUBST(MAKE_STUB_LIB) - # Substitute STUB_LIB_FILE in case package creates a stub library too. - AC_SUBST(PKG_STUB_LIB_FILE) - AC_SUBST(RANLIB_STUB) - AC_SUBST(VC_MANIFEST_EMBED_DLL) - AC_SUBST(VC_MANIFEST_EMBED_EXE) -]) - -#------------------------------------------------------------------------ -# TEA_LIB_SPEC -- -# -# Compute the name of an existing object library located in libdir -# from the given base name and produce the appropriate linker flags. -# -# Arguments: -# basename The base name of the library without version -# numbers, extensions, or "lib" prefixes. -# extra_dir Extra directory in which to search for the -# library. This location is used first, then -# $prefix/$exec-prefix, then some defaults. -# -# Requires: -# TEA_INIT and TEA_PREFIX must be called first. -# -# Results: -# -# Defines the following vars: -# ${basename}_LIB_NAME The computed library name. -# ${basename}_LIB_SPEC The computed linker flags. -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_LIB_SPEC], [ - AC_MSG_CHECKING([for $1 library]) - - # Look in exec-prefix for the library (defined by TEA_PREFIX). - - tea_lib_name_dir="${exec_prefix}/lib" - - # Or in a user-specified location. - - if test x"$2" != x ; then - tea_extra_lib_dir=$2 - else - tea_extra_lib_dir=NONE - fi - - for i in \ - `ls -dr ${tea_extra_lib_dir}/$1[[0-9]]*.lib 2>/dev/null ` \ - `ls -dr ${tea_extra_lib_dir}/lib$1[[0-9]]* 2>/dev/null ` \ - `ls -dr ${tea_lib_name_dir}/$1[[0-9]]*.lib 2>/dev/null ` \ - `ls -dr ${tea_lib_name_dir}/lib$1[[0-9]]* 2>/dev/null ` \ - `ls -dr /usr/lib/$1[[0-9]]*.lib 2>/dev/null ` \ - `ls -dr /usr/lib/lib$1[[0-9]]* 2>/dev/null ` \ - `ls -dr /usr/lib64/$1[[0-9]]*.lib 2>/dev/null ` \ - `ls -dr /usr/lib64/lib$1[[0-9]]* 2>/dev/null ` \ - `ls -dr /usr/local/lib/$1[[0-9]]*.lib 2>/dev/null ` \ - `ls -dr /usr/local/lib/lib$1[[0-9]]* 2>/dev/null ` ; do - if test -f "$i" ; then - tea_lib_name_dir=`dirname $i` - $1_LIB_NAME=`basename $i` - $1_LIB_PATH_NAME=$i - break - fi - done - - if test "${TEA_PLATFORM}" = "windows"; then - $1_LIB_SPEC=\"`${CYGPATH} ${$1_LIB_PATH_NAME} 2>/dev/null`\" - else - # Strip off the leading "lib" and trailing ".a" or ".so" - - tea_lib_name_lib=`echo ${$1_LIB_NAME}|sed -e 's/^lib//' -e 's/\.[[^.]]*$//' -e 's/\.so.*//'` - $1_LIB_SPEC="-L${tea_lib_name_dir} -l${tea_lib_name_lib}" - fi - - if test "x${$1_LIB_NAME}" = x ; then - AC_MSG_ERROR([not found]) - else - AC_MSG_RESULT([${$1_LIB_SPEC}]) - fi -]) - -#------------------------------------------------------------------------ -# TEA_PRIVATE_TCL_HEADERS -- -# -# Locate the private Tcl include files -# -# Arguments: -# -# Requires: -# TCL_SRC_DIR Assumes that TEA_LOAD_TCLCONFIG has -# already been called. -# -# Results: -# -# Substitutes the following vars: -# TCL_TOP_DIR_NATIVE -# TCL_INCLUDES -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_PRIVATE_TCL_HEADERS], [ - # Allow for --with-tclinclude to take effect and define ${ac_cv_c_tclh} - AC_REQUIRE([TEA_PUBLIC_TCL_HEADERS]) - AC_MSG_CHECKING([for Tcl private include files]) - - TCL_SRC_DIR_NATIVE=`${CYGPATH} ${TCL_SRC_DIR}` - TCL_TOP_DIR_NATIVE=\"${TCL_SRC_DIR_NATIVE}\" - - # Check to see if tclPort.h isn't already with the public headers - # Don't look for tclInt.h because that resides with tcl.h in the core - # sources, but the Port headers are in a different directory - if test "${TEA_PLATFORM}" = "windows" -a \ - -f "${ac_cv_c_tclh}/tclWinPort.h"; then - result="private headers found with public headers" - elif test "${TEA_PLATFORM}" = "unix" -a \ - -f "${ac_cv_c_tclh}/tclUnixPort.h"; then - result="private headers found with public headers" - else - TCL_GENERIC_DIR_NATIVE=\"${TCL_SRC_DIR_NATIVE}/generic\" - if test "${TEA_PLATFORM}" = "windows"; then - TCL_PLATFORM_DIR_NATIVE=\"${TCL_SRC_DIR_NATIVE}/win\" - else - TCL_PLATFORM_DIR_NATIVE=\"${TCL_SRC_DIR_NATIVE}/unix\" - fi - # Overwrite the previous TCL_INCLUDES as this should capture both - # public and private headers in the same set. - # We want to ensure these are substituted so as not to require - # any *_NATIVE vars be defined in the Makefile - TCL_INCLUDES="-I${TCL_GENERIC_DIR_NATIVE} -I${TCL_PLATFORM_DIR_NATIVE}" - if test "`uname -s`" = "Darwin"; then - # If Tcl was built as a framework, attempt to use - # the framework's Headers and PrivateHeaders directories - case ${TCL_DEFS} in - *TCL_FRAMEWORK*) - if test -d "${TCL_BIN_DIR}/Headers" -a \ - -d "${TCL_BIN_DIR}/PrivateHeaders"; then - TCL_INCLUDES="-I\"${TCL_BIN_DIR}/Headers\" -I\"${TCL_BIN_DIR}/PrivateHeaders\" ${TCL_INCLUDES}" - else - TCL_INCLUDES="${TCL_INCLUDES} ${TCL_INCLUDE_SPEC} `echo "${TCL_INCLUDE_SPEC}" | sed -e 's/Headers/PrivateHeaders/'`" - fi - ;; - esac - result="Using ${TCL_INCLUDES}" - else - if test ! -f "${TCL_SRC_DIR}/generic/tclInt.h" ; then - AC_MSG_ERROR([Cannot find private header tclInt.h in ${TCL_SRC_DIR}]) - fi - result="Using srcdir found in tclConfig.sh: ${TCL_SRC_DIR}" - fi - fi - - AC_SUBST(TCL_TOP_DIR_NATIVE) - - AC_SUBST(TCL_INCLUDES) - AC_MSG_RESULT([${result}]) -]) - -#------------------------------------------------------------------------ -# TEA_PUBLIC_TCL_HEADERS -- -# -# Locate the installed public Tcl header files -# -# Arguments: -# None. -# -# Requires: -# CYGPATH must be set -# -# Results: -# -# Adds a --with-tclinclude switch to configure. -# Result is cached. -# -# Substitutes the following vars: -# TCL_INCLUDES -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_PUBLIC_TCL_HEADERS], [ - AC_MSG_CHECKING([for Tcl public headers]) - - AC_ARG_WITH(tclinclude, [ --with-tclinclude directory containing the public Tcl header files], with_tclinclude=${withval}) - - AC_CACHE_VAL(ac_cv_c_tclh, [ - # Use the value from --with-tclinclude, if it was given - - if test x"${with_tclinclude}" != x ; then - if test -f "${with_tclinclude}/tcl.h" ; then - ac_cv_c_tclh=${with_tclinclude} - else - AC_MSG_ERROR([${with_tclinclude} directory does not contain tcl.h]) - fi - else - list="" - if test "`uname -s`" = "Darwin"; then - # If Tcl was built as a framework, attempt to use - # the framework's Headers directory - case ${TCL_DEFS} in - *TCL_FRAMEWORK*) - list="`ls -d ${TCL_BIN_DIR}/Headers 2>/dev/null`" - ;; - esac - fi - - # Look in the source dir only if Tcl is not installed, - # and in that situation, look there before installed locations. - if test -f "${TCL_BIN_DIR}/Makefile" ; then - list="$list `ls -d ${TCL_SRC_DIR}/generic 2>/dev/null`" - fi - - # Check order: pkg --prefix location, Tcl's --prefix location, - # relative to directory of tclConfig.sh. - - eval "temp_includedir=${includedir}" - list="$list \ - `ls -d ${temp_includedir} 2>/dev/null` \ - `ls -d ${TCL_PREFIX}/include 2>/dev/null` \ - `ls -d ${TCL_BIN_DIR}/../include 2>/dev/null`" - if test "${TEA_PLATFORM}" != "windows" -o "$GCC" = "yes"; then - list="$list /usr/local/include /usr/include" - if test x"${TCL_INCLUDE_SPEC}" != x ; then - d=`echo "${TCL_INCLUDE_SPEC}" | sed -e 's/^-I//'` - list="$list `ls -d ${d} 2>/dev/null`" - fi - fi - for i in $list ; do - if test -f "$i/tcl.h" ; then - ac_cv_c_tclh=$i - break - fi - done - fi - ]) - - # Print a message based on how we determined the include path - - if test x"${ac_cv_c_tclh}" = x ; then - AC_MSG_ERROR([tcl.h not found. Please specify its location with --with-tclinclude]) - else - AC_MSG_RESULT([${ac_cv_c_tclh}]) - fi - - # Convert to a native path and substitute into the output files. - - INCLUDE_DIR_NATIVE=`${CYGPATH} ${ac_cv_c_tclh}` - - TCL_INCLUDES=-I\"${INCLUDE_DIR_NATIVE}\" - - AC_SUBST(TCL_INCLUDES) -]) - -#------------------------------------------------------------------------ -# TEA_PRIVATE_TK_HEADERS -- -# -# Locate the private Tk include files -# -# Arguments: -# -# Requires: -# TK_SRC_DIR Assumes that TEA_LOAD_TKCONFIG has -# already been called. -# -# Results: -# -# Substitutes the following vars: -# TK_INCLUDES -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_PRIVATE_TK_HEADERS], [ - # Allow for --with-tkinclude to take effect and define ${ac_cv_c_tkh} - AC_REQUIRE([TEA_PUBLIC_TK_HEADERS]) - AC_MSG_CHECKING([for Tk private include files]) - - TK_SRC_DIR_NATIVE=`${CYGPATH} ${TK_SRC_DIR}` - TK_TOP_DIR_NATIVE=\"${TK_SRC_DIR_NATIVE}\" - - # Check to see if tkPort.h isn't already with the public headers - # Don't look for tkInt.h because that resides with tk.h in the core - # sources, but the Port headers are in a different directory - if test "${TEA_PLATFORM}" = "windows" -a \ - -f "${ac_cv_c_tkh}/tkWinPort.h"; then - result="private headers found with public headers" - elif test "${TEA_PLATFORM}" = "unix" -a \ - -f "${ac_cv_c_tkh}/tkUnixPort.h"; then - result="private headers found with public headers" - else - TK_GENERIC_DIR_NATIVE=\"${TK_SRC_DIR_NATIVE}/generic\" - TK_XLIB_DIR_NATIVE=\"${TK_SRC_DIR_NATIVE}/xlib\" - if test "${TEA_PLATFORM}" = "windows"; then - TK_PLATFORM_DIR_NATIVE=\"${TK_SRC_DIR_NATIVE}/win\" - else - TK_PLATFORM_DIR_NATIVE=\"${TK_SRC_DIR_NATIVE}/unix\" - fi - # Overwrite the previous TK_INCLUDES as this should capture both - # public and private headers in the same set. - # We want to ensure these are substituted so as not to require - # any *_NATIVE vars be defined in the Makefile - TK_INCLUDES="-I${TK_GENERIC_DIR_NATIVE} -I${TK_PLATFORM_DIR_NATIVE}" - # Detect and add ttk subdir - if test -d "${TK_SRC_DIR}/generic/ttk"; then - TK_INCLUDES="${TK_INCLUDES} -I\"${TK_SRC_DIR_NATIVE}/generic/ttk\"" - fi - if test "${TEA_WINDOWINGSYSTEM}" != "x11"; then - TK_INCLUDES="${TK_INCLUDES} -I\"${TK_XLIB_DIR_NATIVE}\"" - fi - if test "${TEA_WINDOWINGSYSTEM}" = "aqua"; then - TK_INCLUDES="${TK_INCLUDES} -I\"${TK_SRC_DIR_NATIVE}/macosx\"" - fi - if test "`uname -s`" = "Darwin"; then - # If Tk was built as a framework, attempt to use - # the framework's Headers and PrivateHeaders directories - case ${TK_DEFS} in - *TK_FRAMEWORK*) - if test -d "${TK_BIN_DIR}/Headers" -a \ - -d "${TK_BIN_DIR}/PrivateHeaders"; then - TK_INCLUDES="-I\"${TK_BIN_DIR}/Headers\" -I\"${TK_BIN_DIR}/PrivateHeaders\" ${TK_INCLUDES}" - else - TK_INCLUDES="${TK_INCLUDES} ${TK_INCLUDE_SPEC} `echo "${TK_INCLUDE_SPEC}" | sed -e 's/Headers/PrivateHeaders/'`" - fi - ;; - esac - result="Using ${TK_INCLUDES}" - else - if test ! -f "${TK_SRC_DIR}/generic/tkInt.h" ; then - AC_MSG_ERROR([Cannot find private header tkInt.h in ${TK_SRC_DIR}]) - fi - result="Using srcdir found in tkConfig.sh: ${TK_SRC_DIR}" - fi - fi - - AC_SUBST(TK_TOP_DIR_NATIVE) - AC_SUBST(TK_XLIB_DIR_NATIVE) - - AC_SUBST(TK_INCLUDES) - AC_MSG_RESULT([${result}]) -]) - -#------------------------------------------------------------------------ -# TEA_PUBLIC_TK_HEADERS -- -# -# Locate the installed public Tk header files -# -# Arguments: -# None. -# -# Requires: -# CYGPATH must be set -# -# Results: -# -# Adds a --with-tkinclude switch to configure. -# Result is cached. -# -# Substitutes the following vars: -# TK_INCLUDES -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_PUBLIC_TK_HEADERS], [ - AC_MSG_CHECKING([for Tk public headers]) - - AC_ARG_WITH(tkinclude, [ --with-tkinclude directory containing the public Tk header files], with_tkinclude=${withval}) - - AC_CACHE_VAL(ac_cv_c_tkh, [ - # Use the value from --with-tkinclude, if it was given - - if test x"${with_tkinclude}" != x ; then - if test -f "${with_tkinclude}/tk.h" ; then - ac_cv_c_tkh=${with_tkinclude} - else - AC_MSG_ERROR([${with_tkinclude} directory does not contain tk.h]) - fi - else - list="" - if test "`uname -s`" = "Darwin"; then - # If Tk was built as a framework, attempt to use - # the framework's Headers directory. - case ${TK_DEFS} in - *TK_FRAMEWORK*) - list="`ls -d ${TK_BIN_DIR}/Headers 2>/dev/null`" - ;; - esac - fi - - # Look in the source dir only if Tk is not installed, - # and in that situation, look there before installed locations. - if test -f "${TK_BIN_DIR}/Makefile" ; then - list="$list `ls -d ${TK_SRC_DIR}/generic 2>/dev/null`" - fi - - # Check order: pkg --prefix location, Tk's --prefix location, - # relative to directory of tkConfig.sh, Tcl's --prefix location, - # relative to directory of tclConfig.sh. - - eval "temp_includedir=${includedir}" - list="$list \ - `ls -d ${temp_includedir} 2>/dev/null` \ - `ls -d ${TK_PREFIX}/include 2>/dev/null` \ - `ls -d ${TK_BIN_DIR}/../include 2>/dev/null` \ - `ls -d ${TCL_PREFIX}/include 2>/dev/null` \ - `ls -d ${TCL_BIN_DIR}/../include 2>/dev/null`" - if test "${TEA_PLATFORM}" != "windows" -o "$GCC" = "yes"; then - list="$list /usr/local/include /usr/include" - if test x"${TK_INCLUDE_SPEC}" != x ; then - d=`echo "${TK_INCLUDE_SPEC}" | sed -e 's/^-I//'` - list="$list `ls -d ${d} 2>/dev/null`" - fi - fi - for i in $list ; do - if test -f "$i/tk.h" ; then - ac_cv_c_tkh=$i - break - fi - done - fi - ]) - - # Print a message based on how we determined the include path - - if test x"${ac_cv_c_tkh}" = x ; then - AC_MSG_ERROR([tk.h not found. Please specify its location with --with-tkinclude]) - else - AC_MSG_RESULT([${ac_cv_c_tkh}]) - fi - - # Convert to a native path and substitute into the output files. - - INCLUDE_DIR_NATIVE=`${CYGPATH} ${ac_cv_c_tkh}` - - TK_INCLUDES=-I\"${INCLUDE_DIR_NATIVE}\" - - AC_SUBST(TK_INCLUDES) - - if test "${TEA_WINDOWINGSYSTEM}" != "x11"; then - # On Windows and Aqua, we need the X compat headers - AC_MSG_CHECKING([for X11 header files]) - if test ! -r "${INCLUDE_DIR_NATIVE}/X11/Xlib.h"; then - INCLUDE_DIR_NATIVE="`${CYGPATH} ${TK_SRC_DIR}/xlib`" - TK_XINCLUDES=-I\"${INCLUDE_DIR_NATIVE}\" - AC_SUBST(TK_XINCLUDES) - fi - AC_MSG_RESULT([${INCLUDE_DIR_NATIVE}]) - fi -]) - -#------------------------------------------------------------------------ -# TEA_PATH_CONFIG -- -# -# Locate the ${1}Config.sh file and perform a sanity check on -# the ${1} compile flags. These are used by packages like -# [incr Tk] that load *Config.sh files from more than Tcl and Tk. -# -# Arguments: -# none -# -# Results: -# -# Adds the following arguments to configure: -# --with-$1=... -# -# Defines the following vars: -# $1_BIN_DIR Full path to the directory containing -# the $1Config.sh file -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_PATH_CONFIG], [ - # - # Ok, lets find the $1 configuration - # First, look for one uninstalled. - # the alternative search directory is invoked by --with-$1 - # - - if test x"${no_$1}" = x ; then - # we reset no_$1 in case something fails here - no_$1=true - AC_ARG_WITH($1, [ --with-$1 directory containing $1 configuration ($1Config.sh)], with_$1config=${withval}) - AC_MSG_CHECKING([for $1 configuration]) - AC_CACHE_VAL(ac_cv_c_$1config,[ - - # First check to see if --with-$1 was specified. - if test x"${with_$1config}" != x ; then - case ${with_$1config} in - */$1Config.sh ) - if test -f ${with_$1config}; then - AC_MSG_WARN([--with-$1 argument should refer to directory containing $1Config.sh, not to $1Config.sh itself]) - with_$1config=`echo ${with_$1config} | sed 's!/$1Config\.sh$!!'` - fi;; - esac - if test -f "${with_$1config}/$1Config.sh" ; then - ac_cv_c_$1config=`(cd ${with_$1config}; pwd)` - else - AC_MSG_ERROR([${with_$1config} directory doesn't contain $1Config.sh]) - fi - fi - - # then check for a private $1 installation - if test x"${ac_cv_c_$1config}" = x ; then - for i in \ - ../$1 \ - `ls -dr ../$1*[[0-9]].[[0-9]]*.[[0-9]]* 2>/dev/null` \ - `ls -dr ../$1*[[0-9]].[[0-9]][[0-9]] 2>/dev/null` \ - `ls -dr ../$1*[[0-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../$1*[[0-9]].[[0-9]]* 2>/dev/null` \ - ../../$1 \ - `ls -dr ../../$1*[[0-9]].[[0-9]]*.[[0-9]]* 2>/dev/null` \ - `ls -dr ../../$1*[[0-9]].[[0-9]][[0-9]] 2>/dev/null` \ - `ls -dr ../../$1*[[0-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../$1*[[0-9]].[[0-9]]* 2>/dev/null` \ - ../../../$1 \ - `ls -dr ../../../$1*[[0-9]].[[0-9]]*.[[0-9]]* 2>/dev/null` \ - `ls -dr ../../../$1*[[0-9]].[[0-9]][[0-9]] 2>/dev/null` \ - `ls -dr ../../../$1*[[0-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ../../../$1*[[0-9]].[[0-9]]* 2>/dev/null` \ - ${srcdir}/../$1 \ - `ls -dr ${srcdir}/../$1*[[0-9]].[[0-9]]*.[[0-9]]* 2>/dev/null` \ - `ls -dr ${srcdir}/../$1*[[0-9]].[[0-9]][[0-9]] 2>/dev/null` \ - `ls -dr ${srcdir}/../$1*[[0-9]].[[0-9]] 2>/dev/null` \ - `ls -dr ${srcdir}/../$1*[[0-9]].[[0-9]]* 2>/dev/null` \ - ; do - if test -f "$i/$1Config.sh" ; then - ac_cv_c_$1config=`(cd $i; pwd)` - break - fi - if test -f "$i/unix/$1Config.sh" ; then - ac_cv_c_$1config=`(cd $i/unix; pwd)` - break - fi - done - fi - - # check in a few common install locations - if test x"${ac_cv_c_$1config}" = x ; then - for i in `ls -d ${libdir} 2>/dev/null` \ - `ls -d ${exec_prefix}/lib 2>/dev/null` \ - `ls -d ${prefix}/lib 2>/dev/null` \ - `ls -d /usr/local/lib 2>/dev/null` \ - `ls -d /usr/contrib/lib 2>/dev/null` \ - `ls -d /usr/pkg/lib 2>/dev/null` \ - `ls -d /usr/lib 2>/dev/null` \ - `ls -d /usr/lib64 2>/dev/null` \ - ; do - if test -f "$i/$1Config.sh" ; then - ac_cv_c_$1config=`(cd $i; pwd)` - break - fi - done - fi - ]) - - if test x"${ac_cv_c_$1config}" = x ; then - $1_BIN_DIR="# no $1 configs found" - AC_MSG_WARN([Cannot find $1 configuration definitions]) - exit 0 - else - no_$1= - $1_BIN_DIR=${ac_cv_c_$1config} - AC_MSG_RESULT([found $$1_BIN_DIR/$1Config.sh]) - fi - fi -]) - -#------------------------------------------------------------------------ -# TEA_LOAD_CONFIG -- -# -# Load the $1Config.sh file -# -# Arguments: -# -# Requires the following vars to be set: -# $1_BIN_DIR -# -# Results: -# -# Substitutes the following vars: -# $1_SRC_DIR -# $1_LIB_FILE -# $1_LIB_SPEC -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_LOAD_CONFIG], [ - AC_MSG_CHECKING([for existence of ${$1_BIN_DIR}/$1Config.sh]) - - if test -f "${$1_BIN_DIR}/$1Config.sh" ; then - AC_MSG_RESULT([loading]) - . "${$1_BIN_DIR}/$1Config.sh" - else - AC_MSG_RESULT([file not found]) - fi - - # - # If the $1_BIN_DIR is the build directory (not the install directory), - # then set the common variable name to the value of the build variables. - # For example, the variable $1_LIB_SPEC will be set to the value - # of $1_BUILD_LIB_SPEC. An extension should make use of $1_LIB_SPEC - # instead of $1_BUILD_LIB_SPEC since it will work with both an - # installed and uninstalled version of Tcl. - # - - if test -f "${$1_BIN_DIR}/Makefile" ; then - AC_MSG_WARN([Found Makefile - using build library specs for $1]) - $1_LIB_SPEC=${$1_BUILD_LIB_SPEC} - $1_STUB_LIB_SPEC=${$1_BUILD_STUB_LIB_SPEC} - $1_STUB_LIB_PATH=${$1_BUILD_STUB_LIB_PATH} - $1_INCLUDE_SPEC=${$1_BUILD_INCLUDE_SPEC} - $1_LIBRARY_PATH=${$1_LIBRARY_PATH} - fi - - AC_SUBST($1_VERSION) - AC_SUBST($1_BIN_DIR) - AC_SUBST($1_SRC_DIR) - - AC_SUBST($1_LIB_FILE) - AC_SUBST($1_LIB_SPEC) - - AC_SUBST($1_STUB_LIB_FILE) - AC_SUBST($1_STUB_LIB_SPEC) - AC_SUBST($1_STUB_LIB_PATH) - - # Allow the caller to prevent this auto-check by specifying any 2nd arg - AS_IF([test "x$2" = x], [ - # Check both upper and lower-case variants - # If a dev wanted non-stubs libs, this function could take an option - # to not use _STUB in the paths below - AS_IF([test "x${$1_STUB_LIB_SPEC}" = x], - [TEA_LOAD_CONFIG_LIB(translit($1,[a-z],[A-Z])_STUB)], - [TEA_LOAD_CONFIG_LIB($1_STUB)]) - ]) -]) - -#------------------------------------------------------------------------ -# TEA_LOAD_CONFIG_LIB -- -# -# Helper function to load correct library from another extension's -# ${PACKAGE}Config.sh. -# -# Results: -# Adds to LIBS the appropriate extension library -#------------------------------------------------------------------------ -AC_DEFUN([TEA_LOAD_CONFIG_LIB], [ - AC_MSG_CHECKING([For $1 library for LIBS]) - # This simplifies the use of stub libraries by automatically adding - # the stub lib to your path. Normally this would add to SHLIB_LD_LIBS, - # but this is called before CONFIG_CFLAGS. More importantly, this adds - # to PKG_LIBS, which becomes LIBS, and that is only used by SHLIB_LD. - if test "x${$1_LIB_SPEC}" != "x" ; then - if test "${TEA_PLATFORM}" = "windows" -a "$GCC" != "yes" ; then - TEA_ADD_LIBS([\"`${CYGPATH} ${$1_LIB_PATH}`\"]) - AC_MSG_RESULT([using $1_LIB_PATH ${$1_LIB_PATH}]) - else - TEA_ADD_LIBS([${$1_LIB_SPEC}]) - AC_MSG_RESULT([using $1_LIB_SPEC ${$1_LIB_SPEC}]) - fi - else - AC_MSG_RESULT([file not found]) - fi -]) - -#------------------------------------------------------------------------ -# TEA_EXPORT_CONFIG -- -# -# Define the data to insert into the ${PACKAGE}Config.sh file -# -# Arguments: -# -# Requires the following vars to be set: -# $1 -# -# Results: -# Substitutes the following vars: -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_EXPORT_CONFIG], [ - #-------------------------------------------------------------------- - # These are for $1Config.sh - #-------------------------------------------------------------------- - - # pkglibdir must be a fully qualified path and (not ${exec_prefix}/lib) - eval pkglibdir="[$]{libdir}/$1${PACKAGE_VERSION}" - if test "${TCL_LIB_VERSIONS_OK}" = "ok"; then - eval $1_LIB_FLAG="-l$1${PACKAGE_VERSION}" - eval $1_STUB_LIB_FLAG="-l$1stub${PACKAGE_VERSION}" - else - eval $1_LIB_FLAG="-l$1`echo ${PACKAGE_VERSION} | tr -d .`" - eval $1_STUB_LIB_FLAG="-l$1stub`echo ${PACKAGE_VERSION} | tr -d .`" - fi - if test "${TCL_MAJOR_VERSION}" -gt 8 -a x"${with_tcl8}" = x; then - eval $1_STUB_LIB_FLAG="-l$1stub" - fi - - $1_BUILD_LIB_SPEC="-L`$CYGPATH $(pwd)` ${$1_LIB_FLAG}" - $1_LIB_SPEC="-L`$CYGPATH ${pkglibdir}` ${$1_LIB_FLAG}" - $1_BUILD_STUB_LIB_SPEC="-L`$CYGPATH $(pwd)` [$]{$1_STUB_LIB_FLAG}" - $1_STUB_LIB_SPEC="-L`$CYGPATH ${pkglibdir}` [$]{$1_STUB_LIB_FLAG}" - $1_BUILD_STUB_LIB_PATH="`$CYGPATH $(pwd)`/[$]{PKG_STUB_LIB_FILE}" - $1_STUB_LIB_PATH="`$CYGPATH ${pkglibdir}`/[$]{PKG_STUB_LIB_FILE}" - - AC_SUBST($1_BUILD_LIB_SPEC) - AC_SUBST($1_LIB_SPEC) - AC_SUBST($1_BUILD_STUB_LIB_SPEC) - AC_SUBST($1_STUB_LIB_SPEC) - AC_SUBST($1_BUILD_STUB_LIB_PATH) - AC_SUBST($1_STUB_LIB_PATH) - - AC_SUBST(MAJOR_VERSION) - AC_SUBST(MINOR_VERSION) - AC_SUBST(PATCHLEVEL) -]) - - -#------------------------------------------------------------------------ -# TEA_INSTALLER -- -# -# Configure the installer. -# -# Arguments: -# none -# -# Results: -# Substitutes the following vars: -# INSTALL -# INSTALL_DATA_DIR -# INSTALL_DATA -# INSTALL_PROGRAM -# INSTALL_SCRIPT -# INSTALL_LIBRARY -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_INSTALLER], [ - INSTALL='$(SHELL) $(srcdir)/tclconfig/install-sh -c' - INSTALL_DATA_DIR='${INSTALL} -d -m 755' - INSTALL_DATA='${INSTALL} -m 644' - INSTALL_PROGRAM='${INSTALL} -m 755' - INSTALL_SCRIPT='${INSTALL} -m 755' - - TEA_CONFIG_SYSTEM - case $system in - HP-UX-*) INSTALL_LIBRARY='${INSTALL} -m 755' ;; - *) INSTALL_LIBRARY='${INSTALL} -m 644' ;; - esac - - AC_SUBST(INSTALL) - AC_SUBST(INSTALL_DATA_DIR) - AC_SUBST(INSTALL_DATA) - AC_SUBST(INSTALL_PROGRAM) - AC_SUBST(INSTALL_SCRIPT) - AC_SUBST(INSTALL_LIBRARY) -]) - -### -# Tip 430 - ZipFS Modifications -### -#------------------------------------------------------------------------ -# TEA_ZIPFS_SUPPORT -# Locate a zip encoder installed on the system path, or none. -# -# Arguments: -# none -# -# Results: -# Substitutes the following vars: -# MACHER_PROG -# ZIP_PROG -# ZIP_PROG_OPTIONS -# ZIP_PROG_VFSSEARCH -# ZIP_INSTALL_OBJS -#------------------------------------------------------------------------ - -AC_DEFUN([TEA_ZIPFS_SUPPORT], [ - MACHER_PROG="" - ZIP_PROG="" - ZIP_PROG_OPTIONS="" - ZIP_PROG_VFSSEARCH="" - ZIP_INSTALL_OBJS="" - - AC_MSG_CHECKING([for macher]) - AC_CACHE_VAL(ac_cv_path_macher, [ - search_path=`echo ${PATH} | sed -e 's/:/ /g'` - for dir in $search_path ; do - for j in `ls -r $dir/macher 2> /dev/null` \ - `ls -r $dir/macher 2> /dev/null` ; do - if test x"$ac_cv_path_macher" = x ; then - if test -f "$j" ; then - ac_cv_path_macher=$j - break - fi - fi - done - done - ]) - if test -f "$ac_cv_path_macher" ; then - MACHER_PROG="$ac_cv_path_macher" - AC_MSG_RESULT([$MACHER_PROG]) - AC_MSG_RESULT([Found macher in environment]) - fi - AC_MSG_CHECKING([for zip]) - AC_CACHE_VAL(ac_cv_path_zip, [ - search_path=`echo ${PATH} | sed -e 's/:/ /g'` - for dir in $search_path ; do - for j in `ls -r $dir/zip 2> /dev/null` \ - `ls -r $dir/zip 2> /dev/null` ; do - if test x"$ac_cv_path_zip" = x ; then - if test -f "$j" ; then - ac_cv_path_zip=$j - break - fi - fi - done - done - ]) - if test -f "$ac_cv_path_zip" ; then - ZIP_PROG="$ac_cv_path_zip" - AC_MSG_RESULT([$ZIP_PROG]) - ZIP_PROG_OPTIONS="-rq" - ZIP_PROG_VFSSEARCH="*" - AC_MSG_RESULT([Found INFO Zip in environment]) - # Use standard arguments for zip - else - # It is not an error if an installed version of Zip can't be located. - # We can use the locally distributed minizip instead - ZIP_PROG="./minizip${EXEEXT_FOR_BUILD}" - ZIP_PROG_OPTIONS="-o -r" - ZIP_PROG_VFSSEARCH="*" - ZIP_INSTALL_OBJS="minizip${EXEEXT_FOR_BUILD}" - AC_MSG_RESULT([No zip found on PATH. Building minizip]) - fi - AC_SUBST(MACHER_PROG) - AC_SUBST(ZIP_PROG) - AC_SUBST(ZIP_PROG_OPTIONS) - AC_SUBST(ZIP_PROG_VFSSEARCH) - AC_SUBST(ZIP_INSTALL_OBJS) -]) - -# Local Variables: -# mode: autoconf -# End: diff --git a/autoconf/tea/teaish.tcl b/autoconf/tea/teaish.tcl new file mode 100644 index 000000000..9333495aa --- /dev/null +++ b/autoconf/tea/teaish.tcl @@ -0,0 +1,565 @@ +# Teaish configure script for the SQLite Tcl extension + +# +# State for disparate config-time pieces. +# +array set sqlite__Config [proj-strip-hash-comments { + # + # The list of feature --flags which the --all flag implies. This + # requires special handling in a few places. + # + all-flag-enables {fts3 fts4 fts5 rtree geopoly} + + # >0 if building in the canonical tree. -1=undetermined + is-canonical -1 +}] + +# +# Set up the package info for teaish... +# +apply {{dir} { + # Figure out the version number... + set version "" + if {[file exists $dir/../VERSION]} { + # The canonical SQLite TEA(ish) build + set version [proj-file-content -trim $dir/../VERSION] + set ::sqlite__Config(is-canonical) 1 + set distname sqlite-tcl + } elseif {[file exists $dir/generic/tclsqlite3.c]} { + # The copy from the teaish tree, used as a dev/test bed before + # updating SQLite's tree. + set ::sqlite__Config(is-canonical) 0 + set fd [open $dir/generic/tclsqlite3.c rb] + while {[gets $fd line] >=0} { + if {[regexp {^#define[ ]+SQLITE_VERSION[ ]+"(3.+)"} \ + $line - version]} { + set distname sqlite-teaish + break + } + } + close $fd + } + + if {"" eq $version} { + proj-fatal "Cannot determine the SQLite version number" + } + + proj-assert {$::sqlite__Config(is-canonical) > -1} + proj-assert {[string match 3.*.* $version]} \ + "Unexpected SQLite version: $version" + + set pragmas {} + if {$::sqlite__Config(is-canonical)} { + # Disable "make dist" in the canonical tree. That tree is + # generated from several pieces and creating/testing working + # "dist" rules for that sub-build currently feels unnecessary. The + # copy in the teaish tree, though, should be able to "make dist". + lappend pragmas no-dist + } else { + lappend pragmas full-dist + } + + teaish-pkginfo-set -vars { + -name sqlite + -name.pkg sqlite3 + -version $version + -name.dist $distname + -vsatisfies 8.6- + -libDir sqlite$version + -pragmas $pragmas + } +}} [teaish-get -dir] + +# +# Must return either an empty string or a list in the form accepted by +# autosetup's [options] function. +# +proc teaish-options {} { + # These flags and defaults mostly derive from the historical TEA + # build. Some, like ICU, are taken from the canonical SQLite tree. + return [subst -nocommands -nobackslashes { + with-system-sqlite=0 + => {Use the system-level SQLite instead of the copy in this tree. + Also requires use of --override-sqlite-version so that the build + knows what version number to associate with the system-level SQLite.} + override-sqlite-version:VERSION + => {For use with --with-system-sqlite to set the version number.} + threadsafe=1 => {Disable mutexing} + with-tempstore:=no => {Use an in-RAM database for temporary tables: never,no,yes,always} + load-extension=0 => {Enable loading of external extensions} + math=1 => {Disable math functions} + json=1 => {Disable JSON functions} + fts3 => {Enable the FTS3 extension} + fts4 => {Enable the FTS4 extension} + fts5 => {Enable the FTS5 extension} + update-limit => {Enable the UPDATE/DELETE LIMIT clause} + geopoly => {Enable the GEOPOLY extension} + rtree => {Enable the RTREE extension} + session => {Enable the SESSION extension} + all=1 => {Disable $::sqlite__Config(all-flag-enables)} + with-icu-ldflags:LDFLAGS + => {Enable SQLITE_ENABLE_ICU and add the given linker flags for the + ICU libraries. e.g. on Ubuntu systems, try '-licui18n -licuuc -licudata'.} + with-icu-cflags:CFLAGS + => {Apply extra CFLAGS/CPPFLAGS necessary for building with ICU. + e.g. -I/usr/local/include} + with-icu-config:=auto + => {Enable SQLITE_ENABLE_ICU. Value must be one of: auto, pkg-config, + /path/to/icu-config} + icu-collations=0 + => {Enable SQLITE_ENABLE_ICU_COLLATIONS. Requires --with-icu-ldflags=... + or --with-icu-config} + }] +} + +# +# Gets called by tea-configure-core. Must perform any configuration +# work needed for this extension. +# +proc teaish-configure {} { + use teaish/feature + + teaish-src-add -dist -dir generic/tclsqlite3.c + + if {[proj-opt-was-provided override-sqlite-version]} { + teaish-pkginfo-set -version [opt-val override-sqlite-version] + proj-warn "overriding sqlite version number:" [teaish-pkginfo-get -version] + } elseif {[proj-opt-was-provided with-system-sqlite] + && [opt-val with-system-sqlite] ne "0"} { + proj-fatal "when using --with-system-sqlite also use" \ + "--override-sqlite-version to specify a library version number." + } + + define CFLAGS [proj-get-env CFLAGS {-O2}] + sqlite-munge-cflags + + # + # Add feature flags from legacy configure.ac which are not covered by + # --flags. + # + sqlite-add-feature-flag { + -DSQLITE_3_SUFFIX_ONLY=1 + -DSQLITE_ENABLE_DESERIALIZE=1 + -DSQLITE_ENABLE_DBPAGE_VTAB=1 + -DSQLITE_ENABLE_BYTECODE_VTAB=1 + -DSQLITE_ENABLE_DBSTAT_VTAB=1 + } + + if {[opt-bool with-system-sqlite]} { + msg-result "Using system-level sqlite3." + teaish-cflags-add -DUSE_SYSTEM_SQLITE + teaish-ldflags-add -lsqlite3 + } elseif {$::sqlite__Config(is-canonical)} { + teaish-cflags-add -I[teaish-get -dir]/.. + } + + teaish-check-librt + teaish-check-libz + sqlite-handle-threadsafe + sqlite-handle-tempstore + sqlite-handle-load-extension + sqlite-handle-math + sqlite-handle-icu + + sqlite-handle-common-feature-flags; # must be late in the process +}; # teaish-configure + +define OPT_FEATURE_FLAGS {} ; # -DSQLITE_OMIT/ENABLE flags. +# +# Adds $args, if not empty, to OPT_FEATURE_FLAGS. This is intended only for holding +# -DSQLITE_ENABLE/OMIT/... flags, but that is not enforced here. +proc sqlite-add-feature-flag {args} { + if {"" ne $args} { + define-append OPT_FEATURE_FLAGS {*}$args + } +} + +# +# Check for log(3) in libm and die with an error if it is not +# found. $featureName should be the feature name which requires that +# function (it's used only in error messages). defines LDFLAGS_MATH to +# the required linker flags (which may be empty even if the math APIs +# are found, depending on the OS). +proc sqlite-affirm-have-math {featureName} { + if {"" eq [get-define LDFLAGS_MATH ""]} { + if {![msg-quiet proj-check-function-in-lib log m]} { + user-error "Missing math APIs for $featureName" + } + set lfl [get-define lib_log ""] + undefine lib_log + if {"" ne $lfl} { + user-notice "Forcing requirement of $lfl for $featureName" + } + define LDFLAGS_MATH $lfl + teaish-ldflags-prepend $lfl + } +} + +# +# Handle various SQLITE_ENABLE/OMIT_... feature flags. +proc sqlite-handle-common-feature-flags {} { + msg-result "Feature flags..." + if {![opt-bool all]} { + # Special handling for --disable-all + foreach flag $::sqlite__Config(all-flag-enables) { + if {![proj-opt-was-provided $flag]} { + proj-opt-set $flag 0 + } + } + } + foreach {boolFlag featureFlag ifSetEvalThis} [proj-strip-hash-comments { + all {} { + # The 'all' option must be first in this list. This impl makes + # an effort to only apply flags which the user did not already + # apply, so that combinations like (--all --disable-geopoly) + # will indeed disable geopoly. There are corner cases where + # flags which depend on each other will behave in non-intuitive + # ways: + # + # --all --disable-rtree + # + # Will NOT disable geopoly, though geopoly depends on rtree. + # The --geopoly flag, though, will automatically re-enable + # --rtree, so --disable-rtree won't actually disable anything in + # that case. + foreach k $::sqlite__Config(all-flag-enables) { + if {![proj-opt-was-provided $k]} { + proj-opt-set $k 1 + } + } + } + fts3 -DSQLITE_ENABLE_FTS3 {sqlite-affirm-have-math fts3} + fts4 -DSQLITE_ENABLE_FTS4 {sqlite-affirm-have-math fts4} + fts5 -DSQLITE_ENABLE_FTS5 {sqlite-affirm-have-math fts5} + geopoly -DSQLITE_ENABLE_GEOPOLY {proj-opt-set rtree} + rtree -DSQLITE_ENABLE_RTREE {} + session {-DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK} {} + update-limit -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT {} + scanstatus -DSQLITE_ENABLE_STMT_SCANSTATUS {} + }] { + if {$boolFlag ni $::autosetup(options)} { + # Skip flags which are in the canonical build but not + # the autoconf bundle. + continue + } + proj-if-opt-truthy $boolFlag { + sqlite-add-feature-flag $featureFlag + if {0 != [eval $ifSetEvalThis] && "all" ne $boolFlag} { + msg-result " + $boolFlag" + } + } { + if {"all" ne $boolFlag} { + msg-result " - $boolFlag" + } + } + } + # + # Invert the above loop's logic for some SQLITE_OMIT_... cases. If + # config option $boolFlag is false, [sqlite-add-feature-flag + # $featureFlag], where $featureFlag is intended to be + # -DSQLITE_OMIT_... + foreach {boolFlag featureFlag} { + json -DSQLITE_OMIT_JSON + } { + if {[proj-opt-truthy $boolFlag]} { + msg-result " + $boolFlag" + } else { + sqlite-add-feature-flag $featureFlag + msg-result " - $boolFlag" + } + } + + ## + # Remove duplicates from the final feature flag sets and show them + # to the user. + set oFF [get-define OPT_FEATURE_FLAGS] + if {"" ne $oFF} { + define OPT_FEATURE_FLAGS [lsort -unique $oFF] + msg-result "Library feature flags: [get-define OPT_FEATURE_FLAGS]" + } + if {[lsearch [get-define TARGET_DEBUG ""] -DSQLITE_DEBUG=1] > -1} { + msg-result "Note: this is a debug build, so performance will suffer." + } + teaish-cflags-add -define OPT_FEATURE_FLAGS +}; # sqlite-handle-common-feature-flags + +# +# If --enable-threadsafe is set, this adds -DSQLITE_THREADSAFE=1 to +# OPT_FEATURE_FLAGS and sets LDFLAGS_PTHREAD to the linker flags +# needed for linking pthread (possibly an empty string). If +# --enable-threadsafe is not set, adds -DSQLITE_THREADSAFE=0 to +# OPT_FEATURE_FLAGS and sets LDFLAGS_PTHREAD to an empty string. +# +# It prepends the flags to the global LDFLAGS. +proc sqlite-handle-threadsafe {} { + msg-checking "Support threadsafe operation? " + define LDFLAGS_PTHREAD "" + set enable 0 + if {[proj-opt-was-provided threadsafe]} { + proj-if-opt-truthy threadsafe { + if {[proj-check-function-in-lib pthread_create pthread] + && [proj-check-function-in-lib pthread_mutexattr_init pthread]} { + incr enable + set ldf [get-define lib_pthread_create] + define LDFLAGS_PTHREAD $ldf + teaish-ldflags-prepend $ldf + undefine lib_pthread_create + undefine lib_pthread_mutexattr_init + } else { + user-error "Missing required pthread libraries. Use --disable-threadsafe to disable this check." + } + # Recall that LDFLAGS_PTHREAD might be empty even if pthreads if + # found because it's in -lc on some platforms. + } { + msg-result "Disabled using --disable-threadsafe" + } + } else { + # + # If user does not specify --[disable-]threadsafe then select a + # default based on whether it looks like Tcl has threading + # support. + # + catch { + scan [exec echo {puts [tcl::pkgconfig get threaded]} | [get-define TCLSH_CMD]] \ + %d enable + } + if {$enable} { + set flagName "--threadsafe" + set lblAbled "enabled" + msg-result yes + } else { + set flagName "--disable-threadsafe" + set lblAbled "disabled" + msg-result no + } + msg-result "Defaulting to ${flagName} because Tcl has threading ${lblAbled}." + # ^^^ We (probably) don't need to link against -lpthread in the + # is-enabled case. We might in the case of static linking. Unsure. + } + sqlite-add-feature-flag -DSQLITE_THREADSAFE=${enable} + return $enable +} + +# +# Handles the --enable-load-extension flag. Returns 1 if the support +# is enabled, else 0. If support for that feature is not found, a +# fatal error is triggered if --enable-load-extension is explicitly +# provided, else a loud warning is instead emitted. If +# --disable-load-extension is used, no check is performed. +# +# Makes the following environment changes: +# +# - defines LDFLAGS_DLOPEN to any linker flags needed for this +# feature. It may legally be empty on some systems where dlopen() +# is in libc. +# +# - If the feature is not available, adds +# -DSQLITE_OMIT_LOAD_EXTENSION=1 to the feature flags list. +proc sqlite-handle-load-extension {} { + define LDFLAGS_DLOPEN "" + set found 0 + proj-if-opt-truthy load-extension { + set found [proj-check-function-in-lib dlopen dl] + if {$found} { + set ldf [get-define lib_dlopen] + define LDFLAGS_DLOPEN $ldf + teaish-ldflags-prepend $ldf + undefine lib_dlopen + } else { + if {[proj-opt-was-provided load-extension]} { + # Explicit --enable-load-extension: fail if not found + proj-indented-notice -error { + --enable-load-extension was provided but dlopen() + not found. Use --disable-load-extension to bypass this + check. + } + } else { + # It was implicitly enabled: warn if not found + proj-indented-notice { + WARNING: dlopen() not found, so loadable module support will + be disabled. Use --disable-load-extension to bypass this + check. + } + } + } + } + if {$found} { + msg-result "Loadable extension support enabled." + } else { + msg-result "Disabling loadable extension support. Use --enable-load-extension to enable them." + sqlite-add-feature-flag -DSQLITE_OMIT_LOAD_EXTENSION=1 + } + return $found +} + +# +# ICU - International Components for Unicode +# +# Handles these flags: +# +# --with-icu-ldflags=LDFLAGS +# --with-icu-cflags=CFLAGS +# --with-icu-config[=auto | pkg-config | /path/to/icu-config] +# --enable-icu-collations +# +# --with-icu-config values: +# +# - auto: use the first one of (pkg-config, icu-config) found on the +# system. +# - pkg-config: use only pkg-config to determine flags +# - /path/to/icu-config: use that to determine flags +# +# If --with-icu-config is used as neither pkg-config nor icu-config +# are found, fail fatally. +# +# If both --with-icu-ldflags and --with-icu-config are provided, they +# are cumulative. If neither are provided, icu-collations is not +# honored and a warning is emitted if it is provided. +# +# Design note: though we could automatically enable ICU if the +# icu-config binary or (pkg-config icu-io) are found, we specifically +# do not. ICU is always an opt-in feature. +proc sqlite-handle-icu {} { + define LDFLAGS_LIBICU [join [opt-val with-icu-ldflags ""]] + define CFLAGS_LIBICU [join [opt-val with-icu-cflags ""]] + if {[proj-opt-was-provided with-icu-config]} { + msg-result "Checking for ICU support..." + set icuConfigBin [opt-val with-icu-config] + set tryIcuConfigBin 1; # set to 0 if we end up using pkg-config + if {$icuConfigBin in {auto pkg-config}} { + uplevel 3 { use pkg-config } + if {[pkg-config-init 0] && [pkg-config icu-io]} { + # Maintenance reminder: historical docs say to use both of + # (icu-io, icu-uc). icu-uc lacks a required lib and icu-io has + # all of them on tested OSes. + set tryIcuConfigBin 0 + define LDFLAGS_LIBICU [get-define PKG_ICU_IO_LDFLAGS] + define-append LDFLAGS_LIBICU [get-define PKG_ICU_IO_LIBS] + define CFLAGS_LIBICU [get-define PKG_ICU_IO_CFLAGS] + } elseif {"pkg-config" eq $icuConfigBin} { + proj-fatal "pkg-config cannot find package icu-io" + } else { + proj-assert {"auto" eq $icuConfigBin} + } + } + if {$tryIcuConfigBin} { + if {"auto" eq $icuConfigBin} { + set icuConfigBin [proj-first-bin-of \ + /usr/local/bin/icu-config \ + /usr/bin/icu-config] + if {"" eq $icuConfigBin} { + proj-indented-notice -error { + --with-icu-config=auto cannot find (pkg-config icu-io) or icu-config binary. + On Ubuntu-like systems try: + --with-icu-ldflags='-licui18n -licuuc -licudata' + } + } + } + if {[file-isexec $icuConfigBin]} { + set x [exec $icuConfigBin --ldflags] + if {"" eq $x} { + proj-indented-notice -error \ + [subst { + $icuConfigBin --ldflags returned no data. + On Ubuntu-like systems try: + --with-icu-ldflags='-licui18n -licuuc -licudata' + }] + } + define-append LDFLAGS_LIBICU $x + set x [exec $icuConfigBin --cppflags] + define-append CFLAGS_LIBICU $x + } else { + proj-fatal "--with-icu-config=$icuConfigBin does not refer to an executable" + } + } + } + set ldflags [define LDFLAGS_LIBICU [string trim [get-define LDFLAGS_LIBICU]]] + set cflags [define CFLAGS_LIBICU [string trim [get-define CFLAGS_LIBICU]]] + if {"" ne $ldflags} { + sqlite-add-feature-flag -DSQLITE_ENABLE_ICU + msg-result "Enabling ICU support with flags: $ldflags $cflags" + if {[opt-bool icu-collations]} { + msg-result "Enabling ICU collations." + sqlite-add-feature-flag -DSQLITE_ENABLE_ICU_COLLATIONS + } + teaish-ldflags-prepend $ldflags + teaish-cflags-add $cflags + } elseif {[opt-bool icu-collations]} { + proj-warn "ignoring --enable-icu-collations because neither --with-icu-ldflags nor --with-icu-config provided any linker flags" + } else { + msg-result "ICU support is disabled." + } +}; # sqlite-handle-icu + + +# +# Handles the --with-tempstore flag. +# +# The test fixture likes to set SQLITE_TEMP_STORE on its own, so do +# not set that feature flag unless it was explicitly provided to the +# configure script. +proc sqlite-handle-tempstore {} { + if {[proj-opt-was-provided with-tempstore]} { + set ts [opt-val with-tempstore no] + set tsn 1 + msg-checking "Use an in-RAM database for temporary tables? " + switch -exact -- $ts { + never { set tsn 0 } + no { set tsn 1 } + yes { set tsn 2 } + always { set tsn 3 } + default { + user-error "Invalid --with-tempstore value '$ts'. Use one of: never, no, yes, always" + } + } + msg-result $ts + sqlite-add-feature-flag -DSQLITE_TEMP_STORE=$tsn + } +} + +# +# Handles the --enable-math flag. +proc sqlite-handle-math {} { + proj-if-opt-truthy math { + if {![proj-check-function-in-lib ceil m]} { + user-error "Cannot find libm functions. Use --disable-math to bypass this." + } + set lfl [get-define lib_ceil] + undefine lib_ceil + define LDFLAGS_MATH $lfl + teaish-ldflags-prepend $lfl + sqlite-add-feature-flag -DSQLITE_ENABLE_MATH_FUNCTIONS + msg-result "Enabling math SQL functions" + } { + define LDFLAGS_MATH "" + msg-result "Disabling math SQL functions" + } +} + +# +# Move -DSQLITE_OMIT... and -DSQLITE_ENABLE... flags from CFLAGS and +# CPPFLAGS to OPT_FEATURE_FLAGS and remove them from BUILD_CFLAGS. +proc sqlite-munge-cflags {} { + # Move CFLAGS and CPPFLAGS entries matching -DSQLITE_OMIT* and + # -DSQLITE_ENABLE* to OPT_FEATURE_FLAGS. This behavior is derived + # from the pre-3.48 build. + # + # If any configure flags for features are in conflict with + # CFLAGS/CPPFLAGS-specified feature flags, all bets are off. There + # are no guarantees about which one will take precedence. + foreach flagDef {CFLAGS CPPFLAGS} { + set tmp "" + foreach cf [get-define $flagDef ""] { + switch -glob -- $cf { + -DSQLITE_OMIT* - + -DSQLITE_ENABLE* { + sqlite-add-feature-flag $cf + } + default { + lappend tmp $cf + } + } + } + define $flagDef $tmp + } +} diff --git a/autoconf/tea/teaish.test.tcl b/autoconf/tea/teaish.test.tcl new file mode 100644 index 000000000..b63c9426e --- /dev/null +++ b/autoconf/tea/teaish.test.tcl @@ -0,0 +1,14 @@ +test-expect 1.0-open { + sqlite3 db :memory: +} {} + +test-assert 1.1-version-3.x { + [string match 3.* [db eval {select sqlite_version()}]] +} + +test-expect 1.2-select { + db eval {select 'hi, world',1,2,3} +} {{hi, world} 1 2 3} + + +test-expect 99.0-db-close {db close} {} diff --git a/autoconf/tea/win/makefile.vc b/autoconf/tea/win/makefile.vc deleted file mode 100644 index bb32f1a75..000000000 --- a/autoconf/tea/win/makefile.vc +++ /dev/null @@ -1,61 +0,0 @@ -#------------------------------------------------------------- -*- makefile -*- -# -# Sample makefile for building Tcl extensions. -# -# Basic build, test and install -# nmake /s /nologo /f makefile.vc INSTALLDIR=c:\path\to\tcl -# nmake /s /nologo /f makefile.vc INSTALLDIR=c:\path\to\tcl test -# nmake /s /nologo /f makefile.vc INSTALLDIR=c:\path\to\tcl install -# -# For other build options (debug, static etc.) -# See TIP 477 (https://core.tcl.tk/tips/doc/trunk/tip/477.md) for -# detailed documentation. -# -# See the file "license.terms" for information on usage and redistribution -# of this file, and for a DISCLAIMER OF ALL WARRANTIES. -# -#------------------------------------------------------------------------------ - -# PROJECT is sqlite, not sqlite3 to match TEA AC_INIT definition. -# This makes the generated DLL name also consistent between the two -# except for the "t" suffix which is the convention for nmake builds. -PROJECT = sqlite -PRJ_PACKAGE_TCLNAME = sqlite3 - -!include "rules-ext.vc" - -PRJ_OBJS = $(TMP_DIR)\tclsqlite3.obj - -# Preprocessor macros specific to sqlite3. -PRJ_DEFINES = -I"$(ROOT)\.." -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE \ - -DSQLITE_ENABLE_DBPAGE_VTAB=1 -DSQLITE_ENABLE_DBSTAT_VTAB=1 \ - -DSQLITE_ENABLE_FTS3_PARENTHESIS=1 -DSQLITE_ENABLE_FTS4=1 \ - -DSQLITE_ENABLE_FTS5=1 -DSQLITE_ENABLE_COLUMN_METADATA=1 \ - -DSQLITE_ENABLE_JSON1=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS=1 \ - -DSQLITE_3_SUFFIX_ONLY=1 -DSQLITE_ENABLE_RTREE=1 \ - -DSQLITE_UNTESTABLE=1 -DSQLITE_OMIT_LOOKASIDE=1 \ - -DSQLITE_SECURE_DELETE=1 -DSQLITE_SOUNDEX=1 -DSQLITE_ENABLE_GEOPOLY=1 \ - -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1 \ - -DSQLITE_ENABLE_MATH_FUNCTIONS=1 -DDSQLITE_USE_ALLOCA=1 \ - -DSQLITE_ENABLE_STAT4=1 -DSQLITE_OMIT_DEPRECATED=1 \ - -DSQLITE_WIN32_GETVERSIONEX=0 -DSQLITE_WIN32_NO_ANSI=1 -PRJ_DEFINES = $(PRJ_DEFINES) -I$(TMP_DIR) - -# Standard targets to build, install, test etc. -!include "$(_RULESDIR)\targets.vc" - -# The built-in pkgindex does no suffice for our extension as -# the PROJECT name (sqlite) is not same as init function name (Sqlite3) -pkgindex: - @echo if {[package vsatisfies [package provide Tcl] 9.0-]} { > $(OUT_DIR)\pkgIndex.tcl - @echo package ifneeded $(PRJ_PACKAGE_TCLNAME) $(DOTVERSION) \ - [list load [file join $$dir $(PRJLIBNAME9)] [string totitle $(PRJ_PACKAGE_TCLNAME)]] >> $(OUT_DIR)\pkgIndex.tcl - @echo } else { >> $(OUT_DIR)\pkgIndex.tcl - @echo package ifneeded $(PRJ_PACKAGE_TCLNAME) $(DOTVERSION) \ - [list load [file join $$dir $(PRJLIBNAME8)] [string totitle $(PRJ_PACKAGE_TCLNAME)]] >> $(OUT_DIR)\pkgIndex.tcl - @echo } >> $(OUT_DIR)\pkgIndex.tcl - -# Install the manpage though on Windows, doubt it does much good -install: default-install-docs-n - -# Explicit dependency rules diff --git a/autoconf/tea/win/nmakehlp.c b/autoconf/tea/win/nmakehlp.c deleted file mode 100644 index 2dc33cc65..000000000 --- a/autoconf/tea/win/nmakehlp.c +++ /dev/null @@ -1,815 +0,0 @@ -/* - * ---------------------------------------------------------------------------- - * nmakehlp.c -- - * - * This is used to fix limitations within nmake and the environment. - * - * Copyright (c) 2002 by David Gravereaux. - * Copyright (c) 2006 by Pat Thoyts - * - * See the file "license.terms" for information on usage and redistribution of - * this file, and for a DISCLAIMER OF ALL WARRANTIES. - * ---------------------------------------------------------------------------- - */ - -#define _CRT_SECURE_NO_DEPRECATE -#include -#ifdef _MSC_VER -#pragma comment (lib, "user32.lib") -#pragma comment (lib, "kernel32.lib") -#endif -#include -#include - -/* - * This library is required for x64 builds with _some_ versions of MSVC - */ -#if defined(_M_IA64) || defined(_M_AMD64) -#if _MSC_VER >= 1400 && _MSC_VER < 1500 -#pragma comment(lib, "bufferoverflowU") -#endif -#endif - -/* ISO hack for dumb VC++ */ -#ifdef _MSC_VER -#define snprintf _snprintf -#endif - - -/* protos */ - -static int CheckForCompilerFeature(const char *option); -static int CheckForLinkerFeature(char **options, int count); -static int IsIn(const char *string, const char *substring); -static int SubstituteFile(const char *substs, const char *filename); -static int QualifyPath(const char *path); -static int LocateDependency(const char *keyfile); -static const char *GetVersionFromFile(const char *filename, const char *match, int numdots); -static DWORD WINAPI ReadFromPipe(LPVOID args); - -/* globals */ - -#define CHUNK 25 -#define STATICBUFFERSIZE 1000 -typedef struct { - HANDLE pipe; - char buffer[STATICBUFFERSIZE]; -} pipeinfo; - -pipeinfo Out = {INVALID_HANDLE_VALUE, ""}; -pipeinfo Err = {INVALID_HANDLE_VALUE, ""}; - -/* - * exitcodes: 0 == no, 1 == yes, 2 == error - */ - -int -main( - int argc, - char *argv[]) -{ - char msg[300]; - DWORD dwWritten; - int chars; - const char *s; - - /* - * Make sure children (cl.exe and link.exe) are kept quiet. - */ - - SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX); - - /* - * Make sure the compiler and linker aren't effected by the outside world. - */ - - SetEnvironmentVariable("CL", ""); - SetEnvironmentVariable("LINK", ""); - - if (argc > 1 && *argv[1] == '-') { - switch (*(argv[1]+1)) { - case 'c': - if (argc != 3) { - chars = snprintf(msg, sizeof(msg) - 1, - "usage: %s -c \n" - "Tests for whether cl.exe supports an option\n" - "exitcodes: 0 == no, 1 == yes, 2 == error\n", argv[0]); - WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars, - &dwWritten, NULL); - return 2; - } - return CheckForCompilerFeature(argv[2]); - case 'l': - if (argc < 3) { - chars = snprintf(msg, sizeof(msg) - 1, - "usage: %s -l ? ...?\n" - "Tests for whether link.exe supports an option\n" - "exitcodes: 0 == no, 1 == yes, 2 == error\n", argv[0]); - WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars, - &dwWritten, NULL); - return 2; - } - return CheckForLinkerFeature(&argv[2], argc-2); - case 'f': - if (argc == 2) { - chars = snprintf(msg, sizeof(msg) - 1, - "usage: %s -f \n" - "Find a substring within another\n" - "exitcodes: 0 == no, 1 == yes, 2 == error\n", argv[0]); - WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars, - &dwWritten, NULL); - return 2; - } else if (argc == 3) { - /* - * If the string is blank, there is no match. - */ - - return 0; - } else { - return IsIn(argv[2], argv[3]); - } - case 's': - if (argc == 2) { - chars = snprintf(msg, sizeof(msg) - 1, - "usage: %s -s \n" - "Perform a set of string map type substutitions on a file\n" - "exitcodes: 0\n", - argv[0]); - WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars, - &dwWritten, NULL); - return 2; - } - return SubstituteFile(argv[2], argv[3]); - case 'V': - if (argc != 4) { - chars = snprintf(msg, sizeof(msg) - 1, - "usage: %s -V filename matchstring\n" - "Extract a version from a file:\n" - "eg: pkgIndex.tcl \"package ifneeded http\"", - argv[0]); - WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars, - &dwWritten, NULL); - return 0; - } - s = GetVersionFromFile(argv[2], argv[3], *(argv[1]+2) - '0'); - if (s && *s) { - printf("%s\n", s); - return 0; - } else - return 1; /* Version not found. Return non-0 exit code */ - - case 'Q': - if (argc != 3) { - chars = snprintf(msg, sizeof(msg) - 1, - "usage: %s -Q path\n" - "Emit the fully qualified path\n" - "exitcodes: 0 == no, 1 == yes, 2 == error\n", argv[0]); - WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars, - &dwWritten, NULL); - return 2; - } - return QualifyPath(argv[2]); - - case 'L': - if (argc != 3) { - chars = snprintf(msg, sizeof(msg) - 1, - "usage: %s -L keypath\n" - "Emit the fully qualified path of directory containing keypath\n" - "exitcodes: 0 == success, 1 == not found, 2 == error\n", argv[0]); - WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars, - &dwWritten, NULL); - return 2; - } - return LocateDependency(argv[2]); - } - } - chars = snprintf(msg, sizeof(msg) - 1, - "usage: %s -c|-f|-l|-Q|-s|-V ...\n" - "This is a little helper app to equalize shell differences between WinNT and\n" - "Win9x and get nmake.exe to accomplish its job.\n", - argv[0]); - WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, chars, &dwWritten, NULL); - return 2; -} - -static int -CheckForCompilerFeature( - const char *option) -{ - STARTUPINFO si; - PROCESS_INFORMATION pi; - SECURITY_ATTRIBUTES sa; - DWORD threadID; - char msg[300]; - BOOL ok; - HANDLE hProcess, h, pipeThreads[2]; - char cmdline[100]; - - hProcess = GetCurrentProcess(); - - ZeroMemory(&pi, sizeof(PROCESS_INFORMATION)); - ZeroMemory(&si, sizeof(STARTUPINFO)); - si.cb = sizeof(STARTUPINFO); - si.dwFlags = STARTF_USESTDHANDLES; - si.hStdInput = INVALID_HANDLE_VALUE; - - ZeroMemory(&sa, sizeof(SECURITY_ATTRIBUTES)); - sa.nLength = sizeof(SECURITY_ATTRIBUTES); - sa.lpSecurityDescriptor = NULL; - sa.bInheritHandle = FALSE; - - /* - * Create a non-inheritible pipe. - */ - - CreatePipe(&Out.pipe, &h, &sa, 0); - - /* - * Dupe the write side, make it inheritible, and close the original. - */ - - DuplicateHandle(hProcess, h, hProcess, &si.hStdOutput, 0, TRUE, - DUPLICATE_SAME_ACCESS | DUPLICATE_CLOSE_SOURCE); - - /* - * Same as above, but for the error side. - */ - - CreatePipe(&Err.pipe, &h, &sa, 0); - DuplicateHandle(hProcess, h, hProcess, &si.hStdError, 0, TRUE, - DUPLICATE_SAME_ACCESS | DUPLICATE_CLOSE_SOURCE); - - /* - * Base command line. - */ - - lstrcpy(cmdline, "cl.exe -nologo -c -TC -Zs -X -Fp.\\_junk.pch "); - - /* - * Append our option for testing - */ - - lstrcat(cmdline, option); - - /* - * Filename to compile, which exists, but is nothing and empty. - */ - - lstrcat(cmdline, " .\\nul"); - - ok = CreateProcess( - NULL, /* Module name. */ - cmdline, /* Command line. */ - NULL, /* Process handle not inheritable. */ - NULL, /* Thread handle not inheritable. */ - TRUE, /* yes, inherit handles. */ - DETACHED_PROCESS, /* No console for you. */ - NULL, /* Use parent's environment block. */ - NULL, /* Use parent's starting directory. */ - &si, /* Pointer to STARTUPINFO structure. */ - &pi); /* Pointer to PROCESS_INFORMATION structure. */ - - if (!ok) { - DWORD err = GetLastError(); - int chars = snprintf(msg, sizeof(msg) - 1, - "Tried to launch: \"%s\", but got error [%u]: ", cmdline, err); - - FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM|FORMAT_MESSAGE_IGNORE_INSERTS| - FORMAT_MESSAGE_MAX_WIDTH_MASK, 0L, err, 0, (LPSTR)&msg[chars], - (300-chars), 0); - WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, lstrlen(msg), &err,NULL); - return 2; - } - - /* - * Close our references to the write handles that have now been inherited. - */ - - CloseHandle(si.hStdOutput); - CloseHandle(si.hStdError); - - WaitForInputIdle(pi.hProcess, 5000); - CloseHandle(pi.hThread); - - /* - * Start the pipe reader threads. - */ - - pipeThreads[0] = CreateThread(NULL, 0, ReadFromPipe, &Out, 0, &threadID); - pipeThreads[1] = CreateThread(NULL, 0, ReadFromPipe, &Err, 0, &threadID); - - /* - * Block waiting for the process to end. - */ - - WaitForSingleObject(pi.hProcess, INFINITE); - CloseHandle(pi.hProcess); - - /* - * Wait for our pipe to get done reading, should it be a little slow. - */ - - WaitForMultipleObjects(2, pipeThreads, TRUE, 500); - CloseHandle(pipeThreads[0]); - CloseHandle(pipeThreads[1]); - - /* - * Look for the commandline warning code in both streams. - * - in MSVC 6 & 7 we get D4002, in MSVC 8 we get D9002. - */ - - return !(strstr(Out.buffer, "D4002") != NULL - || strstr(Err.buffer, "D4002") != NULL - || strstr(Out.buffer, "D9002") != NULL - || strstr(Err.buffer, "D9002") != NULL - || strstr(Out.buffer, "D2021") != NULL - || strstr(Err.buffer, "D2021") != NULL); -} - -static int -CheckForLinkerFeature( - char **options, - int count) -{ - STARTUPINFO si; - PROCESS_INFORMATION pi; - SECURITY_ATTRIBUTES sa; - DWORD threadID; - char msg[300]; - BOOL ok; - HANDLE hProcess, h, pipeThreads[2]; - int i; - char cmdline[255]; - - hProcess = GetCurrentProcess(); - - ZeroMemory(&pi, sizeof(PROCESS_INFORMATION)); - ZeroMemory(&si, sizeof(STARTUPINFO)); - si.cb = sizeof(STARTUPINFO); - si.dwFlags = STARTF_USESTDHANDLES; - si.hStdInput = INVALID_HANDLE_VALUE; - - ZeroMemory(&sa, sizeof(SECURITY_ATTRIBUTES)); - sa.nLength = sizeof(SECURITY_ATTRIBUTES); - sa.lpSecurityDescriptor = NULL; - sa.bInheritHandle = TRUE; - - /* - * Create a non-inheritible pipe. - */ - - CreatePipe(&Out.pipe, &h, &sa, 0); - - /* - * Dupe the write side, make it inheritible, and close the original. - */ - - DuplicateHandle(hProcess, h, hProcess, &si.hStdOutput, 0, TRUE, - DUPLICATE_SAME_ACCESS | DUPLICATE_CLOSE_SOURCE); - - /* - * Same as above, but for the error side. - */ - - CreatePipe(&Err.pipe, &h, &sa, 0); - DuplicateHandle(hProcess, h, hProcess, &si.hStdError, 0, TRUE, - DUPLICATE_SAME_ACCESS | DUPLICATE_CLOSE_SOURCE); - - /* - * Base command line. - */ - - lstrcpy(cmdline, "link.exe -nologo "); - - /* - * Append our option for testing. - */ - - for (i = 0; i < count; i++) { - lstrcat(cmdline, " \""); - lstrcat(cmdline, options[i]); - lstrcat(cmdline, "\""); - } - - ok = CreateProcess( - NULL, /* Module name. */ - cmdline, /* Command line. */ - NULL, /* Process handle not inheritable. */ - NULL, /* Thread handle not inheritable. */ - TRUE, /* yes, inherit handles. */ - DETACHED_PROCESS, /* No console for you. */ - NULL, /* Use parent's environment block. */ - NULL, /* Use parent's starting directory. */ - &si, /* Pointer to STARTUPINFO structure. */ - &pi); /* Pointer to PROCESS_INFORMATION structure. */ - - if (!ok) { - DWORD err = GetLastError(); - int chars = snprintf(msg, sizeof(msg) - 1, - "Tried to launch: \"%s\", but got error [%u]: ", cmdline, err); - - FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM|FORMAT_MESSAGE_IGNORE_INSERTS| - FORMAT_MESSAGE_MAX_WIDTH_MASK, 0L, err, 0, (LPSTR)&msg[chars], - (300-chars), 0); - WriteFile(GetStdHandle(STD_ERROR_HANDLE), msg, lstrlen(msg), &err,NULL); - return 2; - } - - /* - * Close our references to the write handles that have now been inherited. - */ - - CloseHandle(si.hStdOutput); - CloseHandle(si.hStdError); - - WaitForInputIdle(pi.hProcess, 5000); - CloseHandle(pi.hThread); - - /* - * Start the pipe reader threads. - */ - - pipeThreads[0] = CreateThread(NULL, 0, ReadFromPipe, &Out, 0, &threadID); - pipeThreads[1] = CreateThread(NULL, 0, ReadFromPipe, &Err, 0, &threadID); - - /* - * Block waiting for the process to end. - */ - - WaitForSingleObject(pi.hProcess, INFINITE); - CloseHandle(pi.hProcess); - - /* - * Wait for our pipe to get done reading, should it be a little slow. - */ - - WaitForMultipleObjects(2, pipeThreads, TRUE, 500); - CloseHandle(pipeThreads[0]); - CloseHandle(pipeThreads[1]); - - /* - * Look for the commandline warning code in the stderr stream. - */ - - return !(strstr(Out.buffer, "LNK1117") != NULL || - strstr(Err.buffer, "LNK1117") != NULL || - strstr(Out.buffer, "LNK4044") != NULL || - strstr(Err.buffer, "LNK4044") != NULL || - strstr(Out.buffer, "LNK4224") != NULL || - strstr(Err.buffer, "LNK4224") != NULL); -} - -static DWORD WINAPI -ReadFromPipe( - LPVOID args) -{ - pipeinfo *pi = (pipeinfo *) args; - char *lastBuf = pi->buffer; - DWORD dwRead; - BOOL ok; - - again: - if (lastBuf - pi->buffer + CHUNK > STATICBUFFERSIZE) { - CloseHandle(pi->pipe); - return (DWORD)-1; - } - ok = ReadFile(pi->pipe, lastBuf, CHUNK, &dwRead, 0L); - if (!ok || dwRead == 0) { - CloseHandle(pi->pipe); - return 0; - } - lastBuf += dwRead; - goto again; - - return 0; /* makes the compiler happy */ -} - -static int -IsIn( - const char *string, - const char *substring) -{ - return (strstr(string, substring) != NULL); -} - -/* - * GetVersionFromFile -- - * Looks for a match string in a file and then returns the version - * following the match where a version is anything acceptable to - * package provide or package ifneeded. - */ - -static const char * -GetVersionFromFile( - const char *filename, - const char *match, - int numdots) -{ - static char szBuffer[100]; - char *szResult = NULL; - FILE *fp = fopen(filename, "rt"); - - if (fp != NULL) { - /* - * Read data until we see our match string. - */ - - while (fgets(szBuffer, sizeof(szBuffer), fp) != NULL) { - LPSTR p, q; - - p = strstr(szBuffer, match); - if (p != NULL) { - /* - * Skip to first digit after the match. - */ - - p += strlen(match); - while (*p && !isdigit((unsigned char)*p)) { - ++p; - } - - /* - * Find ending whitespace. - */ - - q = p; - while (*q && (strchr("0123456789.ab", *q)) && (((!strchr(".ab", *q) - && !strchr("ab", q[-1])) || --numdots))) { - ++q; - } - - *q = 0; - szResult = p; - break; - } - } - fclose(fp); - } - return szResult; -} - -/* - * List helpers for the SubstituteFile function - */ - -typedef struct list_item_t { - struct list_item_t *nextPtr; - char * key; - char * value; -} list_item_t; - -/* insert a list item into the list (list may be null) */ -static list_item_t * -list_insert(list_item_t **listPtrPtr, const char *key, const char *value) -{ - list_item_t *itemPtr = (list_item_t *)malloc(sizeof(list_item_t)); - if (itemPtr) { - itemPtr->key = strdup(key); - itemPtr->value = strdup(value); - itemPtr->nextPtr = NULL; - - while(*listPtrPtr) { - listPtrPtr = &(*listPtrPtr)->nextPtr; - } - *listPtrPtr = itemPtr; - } - return itemPtr; -} - -static void -list_free(list_item_t **listPtrPtr) -{ - list_item_t *tmpPtr, *listPtr = *listPtrPtr; - while (listPtr) { - tmpPtr = listPtr; - listPtr = listPtr->nextPtr; - free(tmpPtr->key); - free(tmpPtr->value); - free(tmpPtr); - } -} - -/* - * SubstituteFile -- - * As windows doesn't provide anything useful like sed and it's unreliable - * to use the tclsh you are building against (consider x-platform builds - - * eg compiling AMD64 target from IX86) we provide a simple substitution - * option here to handle autoconf style substitutions. - * The substitution file is whitespace and line delimited. The file should - * consist of lines matching the regular expression: - * \s*\S+\s+\S*$ - * - * Usage is something like: - * nmakehlp -S << $** > $@ - * @PACKAGE_NAME@ $(PACKAGE_NAME) - * @PACKAGE_VERSION@ $(PACKAGE_VERSION) - * << - */ - -static int -SubstituteFile( - const char *substitutions, - const char *filename) -{ - static char szBuffer[1024], szCopy[1024]; - list_item_t *substPtr = NULL; - FILE *fp, *sp; - - fp = fopen(filename, "rt"); - if (fp != NULL) { - - /* - * Build a list of substutitions from the first filename - */ - - sp = fopen(substitutions, "rt"); - if (sp != NULL) { - while (fgets(szBuffer, sizeof(szBuffer), sp) != NULL) { - unsigned char *ks, *ke, *vs, *ve; - ks = (unsigned char*)szBuffer; - while (ks && *ks && isspace(*ks)) ++ks; - ke = ks; - while (ke && *ke && !isspace(*ke)) ++ke; - vs = ke; - while (vs && *vs && isspace(*vs)) ++vs; - ve = vs; - while (ve && *ve && !(*ve == '\r' || *ve == '\n')) ++ve; - *ke = 0, *ve = 0; - list_insert(&substPtr, (char*)ks, (char*)vs); - } - fclose(sp); - } - - /* debug: dump the list */ -#ifndef NDEBUG - { - int n = 0; - list_item_t *p = NULL; - for (p = substPtr; p != NULL; p = p->nextPtr, ++n) { - fprintf(stderr, "% 3d '%s' => '%s'\n", n, p->key, p->value); - } - } -#endif - - /* - * Run the substitutions over each line of the input - */ - - while (fgets(szBuffer, sizeof(szBuffer), fp) != NULL) { - list_item_t *p = NULL; - for (p = substPtr; p != NULL; p = p->nextPtr) { - char *m = strstr(szBuffer, p->key); - if (m) { - char *cp, *op, *sp; - cp = szCopy; - op = szBuffer; - while (op != m) *cp++ = *op++; - sp = p->value; - while (sp && *sp) *cp++ = *sp++; - op += strlen(p->key); - while (*op) *cp++ = *op++; - *cp = 0; - memcpy(szBuffer, szCopy, sizeof(szCopy)); - } - } - printf("%s", szBuffer); - } - - list_free(&substPtr); - } - fclose(fp); - return 0; -} - -BOOL FileExists(LPCTSTR szPath) -{ -#ifndef INVALID_FILE_ATTRIBUTES - #define INVALID_FILE_ATTRIBUTES ((DWORD)-1) -#endif - DWORD pathAttr = GetFileAttributes(szPath); - return (pathAttr != INVALID_FILE_ATTRIBUTES && - !(pathAttr & FILE_ATTRIBUTE_DIRECTORY)); -} - - -/* - * QualifyPath -- - * - * This composes the current working directory with a provided path - * and returns the fully qualified and normalized path. - * Mostly needed to setup paths for testing. - */ - -static int -QualifyPath( - const char *szPath) -{ - char szCwd[MAX_PATH + 1]; - - GetFullPathName(szPath, sizeof(szCwd)-1, szCwd, NULL); - printf("%s\n", szCwd); - return 0; -} - -/* - * Implements LocateDependency for a single directory. See that command - * for an explanation. - * Returns 0 if found after printing the directory. - * Returns 1 if not found but no errors. - * Returns 2 on any kind of error - * Basically, these are used as exit codes for the process. - */ -static int LocateDependencyHelper(const char *dir, const char *keypath) -{ - HANDLE hSearch; - char path[MAX_PATH+1]; - size_t dirlen; - int keylen, ret; - WIN32_FIND_DATA finfo; - - if (dir == NULL || keypath == NULL) - return 2; /* Have no real error reporting mechanism into nmake */ - dirlen = strlen(dir); - if ((dirlen + 3) > sizeof(path)) - return 2; - strncpy(path, dir, dirlen); - strncpy(path+dirlen, "\\*", 3); /* Including terminating \0 */ - keylen = strlen(keypath); - -#if 0 /* This function is not available in Visual C++ 6 */ - /* - * Use numerics 0 -> FindExInfoStandard, - * 1 -> FindExSearchLimitToDirectories, - * as these are not defined in Visual C++ 6 - */ - hSearch = FindFirstFileEx(path, 0, &finfo, 1, NULL, 0); -#else - hSearch = FindFirstFile(path, &finfo); -#endif - if (hSearch == INVALID_HANDLE_VALUE) - return 1; /* Not found */ - - /* Loop through all subdirs checking if the keypath is under there */ - ret = 1; /* Assume not found */ - do { - int sublen; - /* - * We need to check it is a directory despite the - * FindExSearchLimitToDirectories in the above call. See SDK docs - */ - if ((finfo.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0) - continue; - sublen = strlen(finfo.cFileName); - if ((dirlen+1+sublen+1+keylen+1) > sizeof(path)) - continue; /* Path does not fit, assume not matched */ - strncpy(path+dirlen+1, finfo.cFileName, sublen); - path[dirlen+1+sublen] = '\\'; - strncpy(path+dirlen+1+sublen+1, keypath, keylen+1); - if (FileExists(path)) { - /* Found a match, print to stdout */ - path[dirlen+1+sublen] = '\0'; - QualifyPath(path); - ret = 0; - break; - } - } while (FindNextFile(hSearch, &finfo)); - FindClose(hSearch); - return ret; -} - -/* - * LocateDependency -- - * - * Locates a dependency for a package. - * keypath - a relative path within the package directory - * that is used to confirm it is the correct directory. - * The search path for the package directory is currently only - * the parent and grandparent of the current working directory. - * If found, the command prints - * name_DIRPATH= - * and returns 0. If not found, does not print anything and returns 1. - */ -static int LocateDependency(const char *keypath) -{ - size_t i; - int ret; - static const char *paths[] = {"..", "..\\..", "..\\..\\.."}; - - for (i = 0; i < (sizeof(paths)/sizeof(paths[0])); ++i) { - ret = LocateDependencyHelper(paths[i], keypath); - if (ret == 0) - return ret; - } - return ret; -} - - -/* - * Local variables: - * mode: c - * c-basic-offset: 4 - * fill-column: 78 - * indent-tabs-mode: t - * tab-width: 8 - * End: - */ diff --git a/autoconf/tea/win/rules-ext.vc b/autoconf/tea/win/rules-ext.vc deleted file mode 100644 index 479720a4b..000000000 --- a/autoconf/tea/win/rules-ext.vc +++ /dev/null @@ -1,123 +0,0 @@ -# This file should only be included in makefiles for Tcl extensions, -# NOT in the makefile for Tcl itself. - -!ifndef _RULES_EXT_VC - -# We need to run from the directory the parent makefile is located in. -# nmake does not tell us what makefile was used to invoke it so parent -# makefile has to set the MAKEFILEVC macro or we just make a guess and -# warn if we think that is not the case. -!if "$(MAKEFILEVC)" == "" - -!if exist("$(PROJECT).vc") -MAKEFILEVC = $(PROJECT).vc -!elseif exist("makefile.vc") -MAKEFILEVC = makefile.vc -!endif -!endif # "$(MAKEFILEVC)" == "" - -!if !exist("$(MAKEFILEVC)") -MSG = ^ -You must run nmake from the directory containing the project makefile.^ -If you are doing that and getting this message, set the MAKEFILEVC^ -macro to the name of the project makefile. -!message WARNING: $(MSG) -!endif - -!if "$(PROJECT)" == "tcl" -!error The rules-ext.vc file is not intended for Tcl itself. -!endif - -# We extract version numbers using the nmakehlp program. For now use -# the local copy of nmakehlp. Once we locate Tcl, we will use that -# one if it is newer. -!if "$(MACHINE)" == "IX86" || "$(MACHINE)" == "$(NATIVE_ARCH)" -!if [$(CC) -nologo -DNDEBUG "nmakehlp.c" -link -subsystem:console > nul] -!endif -!else -!if [copy x86_64-w64-mingw32-nmakehlp.exe nmakehlp.exe >NUL] -!endif -!endif - -# First locate the Tcl directory that we are working with. -!if "$(TCLDIR)" != "" - -_RULESDIR = $(TCLDIR:/=\) - -!else - -# If an installation path is specified, that is also the Tcl directory. -# Also Tk never builds against an installed Tcl, it needs Tcl sources -!if defined(INSTALLDIR) && "$(PROJECT)" != "tk" -_RULESDIR=$(INSTALLDIR:/=\) -!else -# Locate Tcl sources -!if [echo _RULESDIR = \> nmakehlp.out] \ - || [nmakehlp -L generic\tcl.h >> nmakehlp.out] -_RULESDIR = ..\..\tcl -!else -!include nmakehlp.out -!endif - -!endif # defined(INSTALLDIR).... - -!endif # ifndef TCLDIR - -# Now look for the targets.vc file under the Tcl root. Note we check this -# file and not rules.vc because the latter also exists on older systems. -!if exist("$(_RULESDIR)\lib\nmake\targets.vc") # Building against installed Tcl -_RULESDIR = $(_RULESDIR)\lib\nmake -!elseif exist("$(_RULESDIR)\win\targets.vc") # Building against Tcl sources -_RULESDIR = $(_RULESDIR)\win -!else -# If we have not located Tcl's targets file, most likely we are compiling -# against an older version of Tcl and so must use our own support files. -_RULESDIR = . -!endif - -!if "$(_RULESDIR)" != "." -# Potentially using Tcl's support files. If this extension has its own -# nmake support files, need to compare the versions and pick newer. - -!if exist("rules.vc") # The extension has its own copy - -!if [echo TCL_RULES_MAJOR = \> versions.vc] \ - && [nmakehlp -V "$(_RULESDIR)\rules.vc" RULES_VERSION_MAJOR >> versions.vc] -!endif -!if [echo TCL_RULES_MINOR = \>> versions.vc] \ - && [nmakehlp -V "$(_RULESDIR)\rules.vc" RULES_VERSION_MINOR >> versions.vc] -!endif - -!if [echo OUR_RULES_MAJOR = \>> versions.vc] \ - && [nmakehlp -V "rules.vc" RULES_VERSION_MAJOR >> versions.vc] -!endif -!if [echo OUR_RULES_MINOR = \>> versions.vc] \ - && [nmakehlp -V "rules.vc" RULES_VERSION_MINOR >> versions.vc] -!endif -!include versions.vc -# We have a newer version of the support files, use them -!if ($(TCL_RULES_MAJOR) != $(OUR_RULES_MAJOR)) || ($(TCL_RULES_MINOR) < $(OUR_RULES_MINOR)) -_RULESDIR = . -!endif - -!endif # if exist("rules.vc") - -!endif # if $(_RULESDIR) != "." - -# Let rules.vc know what copy of nmakehlp.c to use. -NMAKEHLPC = $(_RULESDIR)\nmakehlp.c - -# Get rid of our internal defines before calling rules.vc -!undef TCL_RULES_MAJOR -!undef TCL_RULES_MINOR -!undef OUR_RULES_MAJOR -!undef OUR_RULES_MINOR - -!if exist("$(_RULESDIR)\rules.vc") -!message *** Using $(_RULESDIR)\rules.vc -!include "$(_RULESDIR)\rules.vc" -!else -!error *** Could not locate rules.vc in $(_RULESDIR) -!endif - -!endif # _RULES_EXT_VC \ No newline at end of file diff --git a/autoconf/tea/win/rules.vc b/autoconf/tea/win/rules.vc deleted file mode 100644 index 8ecce0e10..000000000 --- a/autoconf/tea/win/rules.vc +++ /dev/null @@ -1,1913 +0,0 @@ -#------------------------------------------------------------- -*- makefile -*- -# rules.vc -- -# -# Part of the nmake based build system for Tcl and its extensions. -# This file does all the hard work in terms of parsing build options, -# compiler switches, defining common targets and macros. The Tcl makefile -# directly includes this. Extensions include it via "rules-ext.vc". -# -# See TIP 477 (https://core.tcl-lang.org/tips/doc/main/tip/477.md) for -# detailed documentation. -# -# See the file "license.terms" for information on usage and redistribution -# of this file, and for a DISCLAIMER OF ALL WARRANTIES. -# -# Copyright (c) 2001-2003 David Gravereaux. -# Copyright (c) 2003-2008 Patrick Thoyts -# Copyright (c) 2017 Ashok P. Nadkarni -#------------------------------------------------------------------------------ - -!ifndef _RULES_VC -_RULES_VC = 1 - -# The following macros define the version of the rules.vc nmake build system -# For modifications that are not backward-compatible, you *must* change -# the major version. -RULES_VERSION_MAJOR = 1 -RULES_VERSION_MINOR = 12 - -# The PROJECT macro must be defined by parent makefile. -!if "$(PROJECT)" == "" -!error *** Error: Macro PROJECT not defined! Please define it before including rules.vc -!endif - -!if "$(PRJ_PACKAGE_TCLNAME)" == "" -PRJ_PACKAGE_TCLNAME = $(PROJECT) -!endif - -# Also special case Tcl and Tk to save some typing later -DOING_TCL = 0 -DOING_TK = 0 -!if "$(PROJECT)" == "tcl" -DOING_TCL = 1 -!elseif "$(PROJECT)" == "tk" -DOING_TK = 1 -!endif - -!ifndef NEED_TK -# Backwards compatibility -!ifdef PROJECT_REQUIRES_TK -NEED_TK = $(PROJECT_REQUIRES_TK) -!else -NEED_TK = 0 -!endif -!endif - -!ifndef NEED_TCL_SOURCE -NEED_TCL_SOURCE = 0 -!endif - -!ifdef NEED_TK_SOURCE -!if $(NEED_TK_SOURCE) -NEED_TK = 1 -!endif -!else -NEED_TK_SOURCE = 0 -!endif - -################################################################ -# Nmake is a pretty weak environment in syntax and capabilities -# so this file is necessarily verbose. It's broken down into -# the following parts. -# -# 0. Sanity check that compiler environment is set up and initialize -# any built-in settings from the parent makefile -# 1. First define the external tools used for compiling, copying etc. -# as this is independent of everything else. -# 2. Figure out our build structure in terms of the directory, whether -# we are building Tcl or an extension, etc. -# 3. Determine the compiler and linker versions -# 4. Build the nmakehlp helper application -# 5. Determine the supported compiler options and features -# 6. Extract Tcl, Tk, and possibly extensions, version numbers from the -# headers -# 7. Parse the OPTS macro value for user-specified build configuration -# 8. Parse the STATS macro value for statistics instrumentation -# 9. Parse the CHECKS macro for additional compilation checks -# 10. Based on this selected configuration, construct the output -# directory and file paths -# 11. Construct the paths where the package is to be installed -# 12. Set up the actual options passed to compiler and linker based -# on the information gathered above. -# 13. Define some standard build targets and implicit rules. These may -# be optionally disabled by the parent makefile. -# 14. (For extensions only.) Compare the configuration of the target -# Tcl and the extensions and warn against discrepancies. -# -# One final note about the macro names used. They are as they are -# for historical reasons. We would like legacy extensions to -# continue to work with this make include file so be wary of -# changing them for consistency or clarity. - -# 0. Sanity check compiler environment - -# Check to see we are configured to build with MSVC (MSDEVDIR, MSVCDIR or -# VCINSTALLDIR) or with the MS Platform SDK (MSSDK or WindowsSDKDir) - -!if !defined(MSDEVDIR) && !defined(MSVCDIR) && !defined(VCINSTALLDIR) && !defined(MSSDK) && !defined(WINDOWSSDKDIR) -MSG = ^ -Visual C++ compiler environment not initialized. -!error $(MSG) -!endif - -# We need to run from the directory the parent makefile is located in. -# nmake does not tell us what makefile was used to invoke it so parent -# makefile has to set the MAKEFILEVC macro or we just make a guess and -# warn if we think that is not the case. -!if "$(MAKEFILEVC)" == "" - -!if exist("$(PROJECT).vc") -MAKEFILEVC = $(PROJECT).vc -!elseif exist("makefile.vc") -MAKEFILEVC = makefile.vc -!endif -!endif # "$(MAKEFILEVC)" == "" - -!if !exist("$(MAKEFILEVC)") -MSG = ^ -You must run nmake from the directory containing the project makefile.^ -If you are doing that and getting this message, set the MAKEFILEVC^ -macro to the name of the project makefile. -!message WARNING: $(MSG) -!endif - - -################################################################ -# 1. Define external programs being used - -#---------------------------------------------------------- -# Set the proper copy method to avoid overwrite questions -# to the user when copying files and selecting the right -# "delete all" method. -#---------------------------------------------------------- - -RMDIR = rmdir /S /Q -CPY = xcopy /i /y >NUL -CPYDIR = xcopy /e /i /y >NUL -COPY = copy /y >NUL -MKDIR = mkdir - -###################################################################### -# 2. Figure out our build environment in terms of what we're building. -# -# (a) Tcl itself -# (b) Tk -# (c) a Tcl extension using libraries/includes from an *installed* Tcl -# (d) a Tcl extension using libraries/includes from Tcl source directory -# -# This last is needed because some extensions still need -# some Tcl interfaces that are not publicly exposed. -# -# The fragment will set the following macros: -# ROOT - root of this module sources -# COMPATDIR - source directory that holds compatibility sources -# DOCDIR - source directory containing documentation files -# GENERICDIR - platform-independent source directory -# WIN_DIR - Windows-specific source directory -# TESTDIR - directory containing test files -# TOOLSDIR - directory containing build tools -# _TCLDIR - root of the Tcl installation OR the Tcl sources. Not set -# when building Tcl itself. -# _INSTALLDIR - native form of the installation path. For Tcl -# this will be the root of the Tcl installation. For extensions -# this will be the lib directory under the root. -# TCLINSTALL - set to 1 if _TCLDIR refers to -# headers and libraries from an installed Tcl, and 0 if built against -# Tcl sources. Not set when building Tcl itself. Yes, not very well -# named. -# _TCL_H - native path to the tcl.h file -# -# If Tk is involved, also sets the following -# _TKDIR - native form Tk installation OR Tk source. Not set if building -# Tk itself. -# TKINSTALL - set 1 if _TKDIR refers to installed Tk and 0 if Tk sources -# _TK_H - native path to the tk.h file - -# Root directory for sources and assumed subdirectories -ROOT = $(MAKEDIR)\.. -# The following paths CANNOT have spaces in them as they appear on the -# left side of implicit rules. -!ifndef COMPATDIR -COMPATDIR = $(ROOT)\compat -!endif -!ifndef DOCDIR -DOCDIR = $(ROOT)\doc -!endif -!ifndef GENERICDIR -GENERICDIR = $(ROOT)\generic -!endif -!ifndef TOOLSDIR -TOOLSDIR = $(ROOT)\tools -!endif -!ifndef TESTDIR -TESTDIR = $(ROOT)\tests -!endif -!ifndef LIBDIR -!if exist("$(ROOT)\library") -LIBDIR = $(ROOT)\library -!else -LIBDIR = $(ROOT)\lib -!endif -!endif -!ifndef DEMODIR -!if exist("$(LIBDIR)\demos") -DEMODIR = $(LIBDIR)\demos -!else -DEMODIR = $(ROOT)\demos -!endif -!endif # ifndef DEMODIR -# Do NOT use WINDIR because it is Windows internal environment -# variable to point to c:\windows! -WIN_DIR = $(ROOT)\win - -!ifndef RCDIR -!if exist("$(WIN_DIR)\rc") -RCDIR = $(WIN_DIR)\rc -!else -RCDIR = $(WIN_DIR) -!endif -!endif -RCDIR = $(RCDIR:/=\) - -# The target directory where the built packages and binaries will be installed. -# INSTALLDIR is the (optional) path specified by the user. -# _INSTALLDIR is INSTALLDIR using the backslash separator syntax -!ifdef INSTALLDIR -### Fix the path separators. -_INSTALLDIR = $(INSTALLDIR:/=\) -!else -### Assume the normal default. -_INSTALLDIR = $(HOMEDRIVE)\Tcl -!endif - -!if $(DOING_TCL) - -# BEGIN Case 2(a) - Building Tcl itself - -# Only need to define _TCL_H -_TCL_H = ..\generic\tcl.h - -# END Case 2(a) - Building Tcl itself - -!elseif $(DOING_TK) - -# BEGIN Case 2(b) - Building Tk - -TCLINSTALL = 0 # Tk always builds against Tcl source, not an installed Tcl -!if "$(TCLDIR)" == "" -!if [echo TCLDIR = \> nmakehlp.out] \ - || [nmakehlp -L generic\tcl.h >> nmakehlp.out] -!error *** Could not locate Tcl source directory. -!endif -!include nmakehlp.out -!endif # TCLDIR == "" - -_TCLDIR = $(TCLDIR:/=\) -_TCL_H = $(_TCLDIR)\generic\tcl.h -!if !exist("$(_TCL_H)") -!error Could not locate tcl.h. Please set the TCLDIR macro to point to the Tcl *source* directory. -!endif - -_TK_H = ..\generic\tk.h - -# END Case 2(b) - Building Tk - -!else - -# BEGIN Case 2(c) or (d) - Building an extension other than Tk - -# If command line has specified Tcl location through TCLDIR, use it -# else default to the INSTALLDIR setting -!if "$(TCLDIR)" != "" - -_TCLDIR = $(TCLDIR:/=\) -!if exist("$(_TCLDIR)\include\tcl.h") # Case 2(c) with TCLDIR defined -TCLINSTALL = 1 -_TCL_H = $(_TCLDIR)\include\tcl.h -!elseif exist("$(_TCLDIR)\generic\tcl.h") # Case 2(d) with TCLDIR defined -TCLINSTALL = 0 -_TCL_H = $(_TCLDIR)\generic\tcl.h -!endif - -!else # # Case 2(c) for extensions with TCLDIR undefined - -# Need to locate Tcl depending on whether it needs Tcl source or not. -# If we don't, check the INSTALLDIR for an installed Tcl first - -!if exist("$(_INSTALLDIR)\include\tcl.h") && !$(NEED_TCL_SOURCE) - -TCLINSTALL = 1 -TCLDIR = $(_INSTALLDIR)\.. -# NOTE: we will be resetting _INSTALLDIR to _INSTALLDIR/lib for extensions -# later so the \.. accounts for the /lib -_TCLDIR = $(_INSTALLDIR)\.. -_TCL_H = $(_TCLDIR)\include\tcl.h - -!else # exist(...) && !$(NEED_TCL_SOURCE) - -!if [echo _TCLDIR = \> nmakehlp.out] \ - || [nmakehlp -L generic\tcl.h >> nmakehlp.out] -!error *** Could not locate Tcl source directory. -!endif -!include nmakehlp.out -TCLINSTALL = 0 -TCLDIR = $(_TCLDIR) -_TCL_H = $(_TCLDIR)\generic\tcl.h - -!endif # exist(...) && !$(NEED_TCL_SOURCE) - -!endif # TCLDIR - -!ifndef _TCL_H -MSG =^ -Failed to find tcl.h. The TCLDIR macro is set incorrectly or is not set and default path does not contain tcl.h. -!error $(MSG) -!endif - -# Now do the same to locate Tk headers and libs if project requires Tk -!if $(NEED_TK) - -!if "$(TKDIR)" != "" - -_TKDIR = $(TKDIR:/=\) -!if exist("$(_TKDIR)\include\tk.h") -TKINSTALL = 1 -_TK_H = $(_TKDIR)\include\tk.h -!elseif exist("$(_TKDIR)\generic\tk.h") -TKINSTALL = 0 -_TK_H = $(_TKDIR)\generic\tk.h -!endif - -!else # TKDIR not defined - -# Need to locate Tcl depending on whether it needs Tcl source or not. -# If we don't, check the INSTALLDIR for an installed Tcl first - -!if exist("$(_INSTALLDIR)\include\tk.h") && !$(NEED_TK_SOURCE) - -TKINSTALL = 1 -# NOTE: we will be resetting _INSTALLDIR to _INSTALLDIR/lib for extensions -# later so the \.. accounts for the /lib -_TKDIR = $(_INSTALLDIR)\.. -_TK_H = $(_TKDIR)\include\tk.h -TKDIR = $(_TKDIR) - -!else # exist("$(_INSTALLDIR)\include\tk.h") && !$(NEED_TK_SOURCE) - -!if [echo _TKDIR = \> nmakehlp.out] \ - || [nmakehlp -L generic\tk.h >> nmakehlp.out] -!error *** Could not locate Tk source directory. -!endif -!include nmakehlp.out -TKINSTALL = 0 -TKDIR = $(_TKDIR) -_TK_H = $(_TKDIR)\generic\tk.h - -!endif # exist("$(_INSTALLDIR)\include\tk.h") && !$(NEED_TK_SOURCE) - -!endif # TKDIR - -!ifndef _TK_H -MSG =^ -Failed to find tk.h. The TKDIR macro is set incorrectly or is not set and default path does not contain tk.h. -!error $(MSG) -!endif - -!endif # NEED_TK - -!if $(NEED_TCL_SOURCE) && $(TCLINSTALL) -MSG = ^ -*** Warning: This extension requires the source distribution of Tcl.^ -*** Please set the TCLDIR macro to point to the Tcl sources. -!error $(MSG) -!endif - -!if $(NEED_TK_SOURCE) -!if $(TKINSTALL) -MSG = ^ -*** Warning: This extension requires the source distribution of Tk.^ -*** Please set the TKDIR macro to point to the Tk sources. -!error $(MSG) -!endif -!endif - - -# If INSTALLDIR set to Tcl installation root dir then reset to the -# lib dir for installing extensions -!if exist("$(_INSTALLDIR)\include\tcl.h") -_INSTALLDIR=$(_INSTALLDIR)\lib -!endif - -# END Case 2(c) or (d) - Building an extension -!endif # if $(DOING_TCL) - -################################################################ -# 3. Determine compiler version and architecture -# In this section, we figure out the compiler version and the -# architecture for which we are building. This sets the -# following macros: -# VCVERSION - the internal compiler version as 1200, 1400, 1910 etc. -# This is also printed by the compiler in dotted form 19.10 etc. -# VCVER - the "marketing version", for example Visual C++ 6 for internal -# compiler version 1200. This is kept only for legacy reasons as it -# does not make sense for recent Microsoft compilers. Only used for -# output directory names. -# ARCH - set to IX86, ARM64 or AMD64 depending on 32- or 64-bit target -# NATIVE_ARCH - set to IX86, ARM64 or AMD64 for the host machine -# MACHINE - same as $(ARCH) - legacy -# _VC_MANIFEST_EMBED_{DLL,EXE} - commands for embedding a manifest if needed - -cc32 = $(CC) # built-in default. -link32 = link -lib32 = lib -rc32 = $(RC) # built-in default. - -#---------------------------------------------------------------- -# Figure out the compiler architecture and version by writing -# the C macros to a file, preprocessing them with the C -# preprocessor and reading back the created file - -_HASH=^# -_VC_MANIFEST_EMBED_EXE= -_VC_MANIFEST_EMBED_DLL= -VCVER=0 -!if ![echo VCVERSION=_MSC_VER > vercl.x] \ - && ![echo $(_HASH)if defined(_M_IX86) >> vercl.x] \ - && ![echo ARCH=IX86 >> vercl.x] \ - && ![echo $(_HASH)elif defined(_M_AMD64) >> vercl.x] \ - && ![echo ARCH=AMD64 >> vercl.x] \ - && ![echo $(_HASH)elif defined(_M_ARM64) >> vercl.x] \ - && ![echo ARCH=ARM64 >> vercl.x] \ - && ![echo $(_HASH)endif >> vercl.x] \ - && ![$(cc32) -nologo -TC -P vercl.x 2>NUL] -!include vercl.i -!if $(VCVERSION) < 1900 -!if ![echo VCVER= ^\> vercl.vc] \ - && ![set /a $(VCVERSION) / 100 - 6 >> vercl.vc] -!include vercl.vc -!endif -!else -# The simple calculation above does not apply to new Visual Studio releases -# Keep the compiler version in its native form. -VCVER = $(VCVERSION) -!endif -!endif - -!if ![del 2>NUL /q/f vercl.x vercl.i vercl.vc] -!endif - -#---------------------------------------------------------------- -# The MACHINE macro is used by legacy makefiles so set it as well -!ifdef MACHINE -!if "$(MACHINE)" == "x86" -!undef MACHINE -MACHINE = IX86 -!elseif "$(MACHINE)" == "arm64" -!undef MACHINE -MACHINE = ARM64 -!elseif "$(MACHINE)" == "x64" -!undef MACHINE -MACHINE = AMD64 -!endif -!if "$(MACHINE)" != "$(ARCH)" -!error Specified MACHINE macro $(MACHINE) does not match detected target architecture $(ARCH). -!endif -!else -MACHINE=$(ARCH) -!endif - -#--------------------------------------------------------------- -# The PLATFORM_IDENTIFY macro matches the values returned by -# the Tcl platform::identify command -!if "$(MACHINE)" == "AMD64" -PLATFORM_IDENTIFY = win32-x86_64 -!elseif "$(MACHINE)" == "ARM64" -PLATFORM_IDENTIFY = win32-arm -!else -PLATFORM_IDENTIFY = win32-ix86 -!endif - -# The MULTIPLATFORM macro controls whether binary extensions are installed -# in platform-specific directories. Intended to be set/used by extensions. -!ifndef MULTIPLATFORM_INSTALL -MULTIPLATFORM_INSTALL = 0 -!endif - -#------------------------------------------------------------ -# Figure out the *host* architecture by reading the registry - -!if ![reg query HKLM\Hardware\Description\System\CentralProcessor\0 /v Identifier | findstr /i x86] -NATIVE_ARCH=IX86 -!elseif ![reg query HKLM\Hardware\Description\System\CentralProcessor\0 /v Identifier | findstr /i ARM | findstr /i 64-bit] -NATIVE_ARCH=ARM64 -!else -NATIVE_ARCH=AMD64 -!endif - -# Since MSVC8 we must deal with manifest resources. -!if $(VCVERSION) >= 1400 -_VC_MANIFEST_EMBED_EXE=if exist $@.manifest mt -nologo -manifest $@.manifest -outputresource:$@;1 -_VC_MANIFEST_EMBED_DLL=if exist $@.manifest mt -nologo -manifest $@.manifest -outputresource:$@;2 -!endif - -################################################################ -# 4. Build the nmakehlp program -# This is a helper app we need to overcome nmake's limiting -# environment. We will call out to it to get various bits of -# information about supported compiler options etc. -# -# Tcl itself will always use the nmakehlp.c program which is -# in its own source. It will be kept updated there. -# -# Extensions built against an installed Tcl will use the installed -# copy of Tcl's nmakehlp.c if there is one and their own version -# otherwise. In the latter case, they would also be using their own -# rules.vc. Note that older versions of Tcl do not install nmakehlp.c -# or rules.vc. -# -# Extensions built against Tcl sources will use the one from the Tcl source. -# -# When building an extension using a sufficiently new version of Tcl, -# rules-ext.vc will define NMAKEHLPC appropriately to point to the -# copy of nmakehlp.c to be used. - -!ifndef NMAKEHLPC -# Default to the one in the current directory (the extension's own nmakehlp.c) -NMAKEHLPC = nmakehlp.c - -!if !$(DOING_TCL) -!if $(TCLINSTALL) -!if exist("$(_TCLDIR)\lib\nmake\nmakehlp.c") -NMAKEHLPC = $(_TCLDIR)\lib\nmake\nmakehlp.c -!endif -!else # !$(TCLINSTALL) -!if exist("$(_TCLDIR)\win\nmakehlp.c") -NMAKEHLPC = $(_TCLDIR)\win\nmakehlp.c -!endif -!endif # $(TCLINSTALL) -!endif # !$(DOING_TCL) - -!endif # NMAKEHLPC - -# We always build nmakehlp even if it exists since we do not know -# what source it was built from. -!if "$(MACHINE)" == "IX86" || "$(MACHINE)" == "$(NATIVE_ARCH)" -!if [$(cc32) -nologo "$(NMAKEHLPC)" -link -subsystem:console > nul] -!endif -!else -!if [copy $(NMAKEHLPC:nmakehlp.c=x86_64-w64-mingw32-nmakehlp.exe) nmakehlp.exe >NUL] -!endif -!endif - -################################################################ -# 5. Test for compiler features -# Visual C++ compiler options have changed over the years. Check -# which options are supported by the compiler in use. -# -# The following macros are set: -# OPTIMIZATIONS - the compiler flags to be used for optimized builds -# DEBUGFLAGS - the compiler flags to be used for debug builds -# LINKERFLAGS - Flags passed to the linker -# -# Note that these are the compiler settings *available*, not those -# that will be *used*. The latter depends on the OPTS macro settings -# which we have not yet parsed. -# -# Also note that some of the flags in OPTIMIZATIONS are not really -# related to optimization. They are placed there only for legacy reasons -# as some extensions expect them to be included in that macro. - -# -Op improves float consistency. Note only needed for older compilers -# Newer compilers do not need or support this option. -!if [nmakehlp -c -Op] -FPOPTS = -Op -!endif - -# Strict floating point semantics - present in newer compilers in lieu of -Op -!if [nmakehlp -c -fp:strict] -FPOPTS = $(FPOPTS) -fp:strict -!endif - -!if "$(MACHINE)" == "IX86" -### test for pentium errata -!if [nmakehlp -c -QI0f] -!message *** Compiler has 'Pentium 0x0f fix' -FPOPTS = $(FPOPTS) -QI0f -!else -!message *** Compiler does not have 'Pentium 0x0f fix' -!endif -!endif - -### test for optimizations -# /O2 optimization includes /Og /Oi /Ot /Oy /Ob2 /Gs /GF /Gy as per -# documentation. Note we do NOT want /Gs as that inserts a _chkstk -# stack probe at *every* function entry, not just those with more than -# a page of stack allocation resulting in a performance hit. However, -# /O2 documentation is misleading as its stack probes are simply the -# default page size locals allocation probes and not what is implied -# by an explicit /Gs option. - -OPTIMIZATIONS = $(FPOPTS) - -!if [nmakehlp -c -O2] -OPTIMIZING = 1 -OPTIMIZATIONS = $(OPTIMIZATIONS) -O2 -!else -# Legacy, really. All modern compilers support this -!message *** Compiler does not have 'Optimizations' -OPTIMIZING = 0 -!endif - -# Checks for buffer overflows in local arrays -!if [nmakehlp -c -GS] -OPTIMIZATIONS = $(OPTIMIZATIONS) -GS -!endif - -# Link time optimization. Note that this option (potentially) makes -# generated libraries only usable by the specific VC++ version that -# created it. Requires /LTCG linker option -!if [nmakehlp -c -GL] -OPTIMIZATIONS = $(OPTIMIZATIONS) -GL -CC_GL_OPT_ENABLED = 1 -!else -# In newer compilers -GL and -YX are incompatible. -!if [nmakehlp -c -YX] -OPTIMIZATIONS = $(OPTIMIZATIONS) -YX -!endif -!endif # [nmakehlp -c -GL] - -DEBUGFLAGS = $(FPOPTS) - -# Run time error checks. Not available or valid in a release, non-debug build -# RTC is for modern compilers, -GZ is legacy -!if [nmakehlp -c -RTC1] -DEBUGFLAGS = $(DEBUGFLAGS) -RTC1 -!elseif [nmakehlp -c -GZ] -DEBUGFLAGS = $(DEBUGFLAGS) -GZ -!endif - -#---------------------------------------------------------------- -# Linker flags - -# LINKER_TESTFLAGS are for internal use when we call nmakehlp to test -# if the linker supports a specific option. Without these flags link will -# return "LNK1561: entry point must be defined" error compiling from VS-IDE: -# They are not passed through to the actual application / extension -# link rules. -!ifndef LINKER_TESTFLAGS -LINKER_TESTFLAGS = /DLL /NOENTRY /OUT:nmakehlp.out -!endif - -LINKERFLAGS = - -# If compiler has enabled link time optimization, linker must too with -ltcg -!ifdef CC_GL_OPT_ENABLED -!if [nmakehlp -l -ltcg $(LINKER_TESTFLAGS)] -LINKERFLAGS = $(LINKERFLAGS) -ltcg -!endif -!endif - - -################################################################ -# 6. Extract various version numbers from headers -# For Tcl and Tk, version numbers are extracted from tcl.h and tk.h -# respectively. For extensions, versions are extracted from the -# configure.in or configure.ac from the TEA configuration if it -# exists, and unset otherwise. -# Sets the following macros: -# TCL_MAJOR_VERSION -# TCL_MINOR_VERSION -# TCL_RELEASE_SERIAL -# TCL_PATCH_LEVEL -# TCL_PATCH_LETTER -# TCL_VERSION -# TK_MAJOR_VERSION -# TK_MINOR_VERSION -# TK_RELEASE_SERIAL -# TK_PATCH_LEVEL -# TK_PATCH_LETTER -# TK_VERSION -# DOTVERSION - set as (for example) 2.5 -# VERSION - set as (for example 25) -#-------------------------------------------------------------- - -!if [echo REM = This file is generated from rules.vc > versions.vc] -!endif -!if [echo TCL_MAJOR_VERSION = \>> versions.vc] \ - && [nmakehlp -V "$(_TCL_H)" "define TCL_MAJOR_VERSION" >> versions.vc] -!endif -!if [echo TCL_MINOR_VERSION = \>> versions.vc] \ - && [nmakehlp -V "$(_TCL_H)" TCL_MINOR_VERSION >> versions.vc] -!endif -!if [echo TCL_RELEASE_SERIAL = \>> versions.vc] \ - && [nmakehlp -V "$(_TCL_H)" TCL_RELEASE_SERIAL >> versions.vc] -!endif -!if [echo TCL_PATCH_LEVEL = \>> versions.vc] \ - && [nmakehlp -V "$(_TCL_H)" TCL_PATCH_LEVEL >> versions.vc] -!endif - -!if defined(_TK_H) -!if [echo TK_MAJOR_VERSION = \>> versions.vc] \ - && [nmakehlp -V $(_TK_H) "define TK_MAJOR_VERSION" >> versions.vc] -!endif -!if [echo TK_MINOR_VERSION = \>> versions.vc] \ - && [nmakehlp -V $(_TK_H) TK_MINOR_VERSION >> versions.vc] -!endif -!if [echo TK_RELEASE_SERIAL = \>> versions.vc] \ - && [nmakehlp -V "$(_TK_H)" TK_RELEASE_SERIAL >> versions.vc] -!endif -!if [echo TK_PATCH_LEVEL = \>> versions.vc] \ - && [nmakehlp -V $(_TK_H) TK_PATCH_LEVEL >> versions.vc] -!endif -!endif # _TK_H - -!include versions.vc - -TCL_VERSION = $(TCL_MAJOR_VERSION)$(TCL_MINOR_VERSION) -TCL_DOTVERSION = $(TCL_MAJOR_VERSION).$(TCL_MINOR_VERSION) -!if [nmakehlp -f $(TCL_PATCH_LEVEL) "a"] -TCL_PATCH_LETTER = a -!elseif [nmakehlp -f $(TCL_PATCH_LEVEL) "b"] -TCL_PATCH_LETTER = b -!else -TCL_PATCH_LETTER = . -!endif - -!if defined(_TK_H) - -TK_VERSION = $(TK_MAJOR_VERSION)$(TK_MINOR_VERSION) -TK_DOTVERSION = $(TK_MAJOR_VERSION).$(TK_MINOR_VERSION) -!if [nmakehlp -f $(TK_PATCH_LEVEL) "a"] -TK_PATCH_LETTER = a -!elseif [nmakehlp -f $(TK_PATCH_LEVEL) "b"] -TK_PATCH_LETTER = b -!else -TK_PATCH_LETTER = . -!endif - -!endif - -# Set DOTVERSION and VERSION -!if $(DOING_TCL) - -DOTVERSION = $(TCL_MAJOR_VERSION).$(TCL_MINOR_VERSION) -VERSION = $(TCL_VERSION) - -!elseif $(DOING_TK) - -DOTVERSION = $(TK_DOTVERSION) -VERSION = $(TK_VERSION) - -!else # Doing a non-Tk extension - -# If parent makefile has not defined DOTVERSION, try to get it from TEA -# first from a configure.in file, and then from configure.ac -!ifndef DOTVERSION -!if [echo DOTVERSION = \> versions.vc] \ - || [nmakehlp -V $(ROOT)\configure.in ^[$(PROJECT)^] >> versions.vc] -!if [echo DOTVERSION = \> versions.vc] \ - || [nmakehlp -V $(ROOT)\configure.ac ^[$(PROJECT)^] >> versions.vc] -!error *** Could not figure out extension version. Please define DOTVERSION in parent makefile before including rules.vc. -!endif -!endif -!include versions.vc -!endif # DOTVERSION -VERSION = $(DOTVERSION:.=) - -!endif # $(DOING_TCL) ... etc. - -# Windows RC files have 3 version components. Ensure this irrespective -# of how many components the package has specified. Basically, ensure -# minimum 4 components by appending 4 0's and then pick out the first 4. -# Also take care of the fact that DOTVERSION may have "a" or "b" instead -# of "." separating the version components. -DOTSEPARATED=$(DOTVERSION:a=.) -DOTSEPARATED=$(DOTSEPARATED:b=.) -!if [echo RCCOMMAVERSION = \> versions.vc] \ - || [for /f "tokens=1,2,3,4,5* delims=." %a in ("$(DOTSEPARATED).0.0.0.0") do echo %a,%b,%c,%d >> versions.vc] -!error *** Could not generate RCCOMMAVERSION *** -!endif -!include versions.vc - -######################################################################## -# 7. Parse the OPTS macro to work out the requested build configuration. -# Based on this, we will construct the actual switches to be passed to the -# compiler and linker using the macros defined in the previous section. -# The following macros are defined by this section based on OPTS -# STATIC_BUILD - 0 -> Tcl is to be built as a shared library -# 1 -> build as a static library and shell -# TCL_THREADS - legacy but always 1 on Windows since winsock requires it. -# DEBUG - 1 -> debug build, 0 -> release builds -# SYMBOLS - 1 -> generate PDB's, 0 -> no PDB's -# PROFILE - 1 -> generate profiling info, 0 -> no profiling -# PGO - 1 -> profile based optimization, 0 -> no -# MSVCRT - 1 -> link to dynamic C runtime even when building static Tcl build -# 0 -> link to static C runtime for static Tcl build. -# Does not impact shared Tcl builds (STATIC_BUILD == 0) -# Default: 1 for Tcl 8.7 and up, 0 otherwise. -# TCL_USE_STATIC_PACKAGES - 1 -> statically link the registry and dde extensions -# in the Tcl and Wish shell. 0 -> keep them as shared libraries. Does -# not impact shared Tcl builds. Implied by STATIC_BUILD since Tcl 8.7. -# USE_THREAD_ALLOC - 1 -> Use a shared global free pool for allocation. -# 0 -> Use the non-thread allocator. -# UNCHECKED - 1 -> when doing a debug build with symbols, use the release -# C runtime, 0 -> use the debug C runtime. -# USE_STUBS - 1 -> compile to use stubs interfaces, 0 -> direct linking -# CONFIG_CHECK - 1 -> check current build configuration against Tcl -# configuration (ignored for Tcl itself) -# _USE_64BIT_TIME_T - forces a build using 64-bit time_t for 32-bit build -# (CRT library should support this, not needed for Tcl 9.x) -# Further, LINKERFLAGS are modified based on above. - -# Default values for all the above -STATIC_BUILD = 0 -TCL_THREADS = 1 -DEBUG = 0 -SYMBOLS = 0 -PROFILE = 0 -PGO = 0 -MSVCRT = 1 -TCL_USE_STATIC_PACKAGES = 0 -USE_THREAD_ALLOC = 1 -UNCHECKED = 0 -CONFIG_CHECK = 1 -!if $(DOING_TCL) -USE_STUBS = 0 -!else -USE_STUBS = 1 -!endif - -# If OPTS is not empty AND does not contain "none" which turns off all OPTS -# set the above macros based on OPTS content -!if "$(OPTS)" != "" && ![nmakehlp -f "$(OPTS)" "none"] - -# OPTS are specified, parse them - -!if [nmakehlp -f $(OPTS) "static"] -!message *** Doing static -STATIC_BUILD = 1 -!endif - -!if [nmakehlp -f $(OPTS) "nostubs"] -!message *** Not using stubs -USE_STUBS = 0 -!endif - -!if [nmakehlp -f $(OPTS) "nomsvcrt"] -!message *** Doing nomsvcrt -MSVCRT = 0 -!else -!if [nmakehlp -f $(OPTS) "msvcrt"] -!message *** Doing msvcrt -!else -!if $(TCL_MAJOR_VERSION) == 8 && $(TCL_MINOR_VERSION) < 7 && $(STATIC_BUILD) -MSVCRT = 0 -!endif -!endif -!endif # [nmakehlp -f $(OPTS) "nomsvcrt"] - -!if [nmakehlp -f $(OPTS) "staticpkg"] && $(STATIC_BUILD) -!message *** Doing staticpkg -TCL_USE_STATIC_PACKAGES = 1 -!endif - -!if [nmakehlp -f $(OPTS) "nothreads"] -!message *** Compile explicitly for non-threaded tcl -TCL_THREADS = 0 -USE_THREAD_ALLOC= 0 -!endif - -!if [nmakehlp -f $(OPTS) "tcl8"] -!message *** Build for Tcl8 -TCL_BUILD_FOR = 8 -!endif - -!if $(TCL_MAJOR_VERSION) == 8 -!if [nmakehlp -f $(OPTS) "time64bit"] -!message *** Force 64-bit time_t -_USE_64BIT_TIME_T = 1 -!endif -!endif - -# Yes, it's weird that the "symbols" option controls DEBUG and -# the "pdbs" option controls SYMBOLS. That's historical. -!if [nmakehlp -f $(OPTS) "symbols"] -!message *** Doing symbols -DEBUG = 1 -!else -DEBUG = 0 -!endif - -!if [nmakehlp -f $(OPTS) "pdbs"] -!message *** Doing pdbs -SYMBOLS = 1 -!else -SYMBOLS = 0 -!endif - -!if [nmakehlp -f $(OPTS) "profile"] -!message *** Doing profile -PROFILE = 1 -!else -PROFILE = 0 -!endif - -!if [nmakehlp -f $(OPTS) "pgi"] -!message *** Doing profile guided optimization instrumentation -PGO = 1 -!elseif [nmakehlp -f $(OPTS) "pgo"] -!message *** Doing profile guided optimization -PGO = 2 -!else -PGO = 0 -!endif - -!if [nmakehlp -f $(OPTS) "loimpact"] -!message *** Warning: ignoring option "loimpact" - deprecated on modern Windows. -!endif - -# TBD - should get rid of this option -!if [nmakehlp -f $(OPTS) "thrdalloc"] -!message *** Doing thrdalloc -USE_THREAD_ALLOC = 1 -!endif - -!if [nmakehlp -f $(OPTS) "tclalloc"] -USE_THREAD_ALLOC = 0 -!endif - -!if [nmakehlp -f $(OPTS) "unchecked"] -!message *** Doing unchecked -UNCHECKED = 1 -!else -UNCHECKED = 0 -!endif - -!if [nmakehlp -f $(OPTS) "noconfigcheck"] -CONFIG_CHECK = 1 -!else -CONFIG_CHECK = 0 -!endif - -!endif # "$(OPTS)" != "" && ... parsing of OPTS - -# Set linker flags based on above - -!if $(PGO) > 1 -!if [nmakehlp -l -ltcg:pgoptimize $(LINKER_TESTFLAGS)] -LINKERFLAGS = $(LINKERFLAGS:-ltcg=) -ltcg:pgoptimize -!else -MSG=^ -This compiler does not support profile guided optimization. -!error $(MSG) -!endif -!elseif $(PGO) > 0 -!if [nmakehlp -l -ltcg:pginstrument $(LINKER_TESTFLAGS)] -LINKERFLAGS = $(LINKERFLAGS:-ltcg=) -ltcg:pginstrument -!else -MSG=^ -This compiler does not support profile guided optimization. -!error $(MSG) -!endif -!endif - -################################################################ -# 8. Parse the STATS macro to configure code instrumentation -# The following macros are set by this section: -# TCL_MEM_DEBUG - 1 -> enables memory allocation instrumentation -# 0 -> disables -# TCL_COMPILE_DEBUG - 1 -> enables byte compiler logging -# 0 -> disables - -# Default both are off -TCL_MEM_DEBUG = 0 -TCL_COMPILE_DEBUG = 0 - -!if "$(STATS)" != "" && ![nmakehlp -f "$(STATS)" "none"] - -!if [nmakehlp -f $(STATS) "memdbg"] -!message *** Doing memdbg -TCL_MEM_DEBUG = 1 -!else -TCL_MEM_DEBUG = 0 -!endif - -!if [nmakehlp -f $(STATS) "compdbg"] -!message *** Doing compdbg -TCL_COMPILE_DEBUG = 1 -!else -TCL_COMPILE_DEBUG = 0 -!endif - -!endif - -#################################################################### -# 9. Parse the CHECKS macro to configure additional compiler checks -# The following macros are set by this section: -# WARNINGS - compiler switches that control the warnings level -# TCL_NO_DEPRECATED - 1 -> disable support for deprecated functions -# 0 -> enable deprecated functions - -# Defaults - Permit deprecated functions and warning level 3 -TCL_NO_DEPRECATED = 0 -WARNINGS = -W3 - -!if "$(CHECKS)" != "" && ![nmakehlp -f "$(CHECKS)" "none"] - -!if [nmakehlp -f $(CHECKS) "nodep"] -!message *** Doing nodep check -TCL_NO_DEPRECATED = 1 -!endif - -!if [nmakehlp -f $(CHECKS) "fullwarn"] -!message *** Doing full warnings check -WARNINGS = -W4 -!if [nmakehlp -l -warn:3 $(LINKER_TESTFLAGS)] -LINKERFLAGS = $(LINKERFLAGS) -warn:3 -!endif -!endif - -!if [nmakehlp -f $(CHECKS) "64bit"] && [nmakehlp -c -Wp64] -!message *** Doing 64bit portability warnings -WARNINGS = $(WARNINGS) -Wp64 -!endif - -!endif - - -################################################################ -# 10. Construct output directory and file paths -# Figure-out how to name our intermediate and output directories. -# In order to avoid inadvertent mixing of object files built using -# different compilers, build configurations etc., -# -# Naming convention (suffixes): -# t = full thread support. (Not used for Tcl >= 8.7) -# s = static library (as opposed to an import library) -# g = linked to the debug enabled C run-time. -# x = special static build when it links to the dynamic C run-time. -# -# The following macros are set in this section: -# SUFX - the suffix to use for binaries based on above naming convention -# BUILDDIRTOP - the toplevel default output directory -# is of the form {Release,Debug}[_AMD64][_COMPILERVERSION] -# TMP_DIR - directory where object files are created -# OUT_DIR - directory where output executables are created -# Both TMP_DIR and OUT_DIR are defaulted only if not defined by the -# parent makefile (or command line). The default values are -# based on BUILDDIRTOP. -# STUBPREFIX - name of the stubs library for this project -# PRJIMPLIB - output path of the generated project import library -# PRJLIBNAME - name of generated project library -# PRJLIB - output path of generated project library -# PRJSTUBLIBNAME - name of the generated project stubs library -# PRJSTUBLIB - output path of the generated project stubs library -# RESFILE - output resource file (only if not static build) - -SUFX = tsgx - -!if $(DEBUG) -BUILDDIRTOP = Debug -!else -BUILDDIRTOP = Release -!endif - -!if "$(MACHINE)" != "IX86" -BUILDDIRTOP =$(BUILDDIRTOP)_$(MACHINE) -!endif -!if $(VCVER) > 6 -BUILDDIRTOP =$(BUILDDIRTOP)_VC$(VCVER) -!endif - -!if !$(DEBUG) || $(TCL_VERSION) > 86 || $(DEBUG) && $(UNCHECKED) -SUFX = $(SUFX:g=) -!endif - -TMP_DIRFULL = .\$(BUILDDIRTOP)\$(PROJECT)_ThreadedDynamicStaticX - -!if !$(STATIC_BUILD) -TMP_DIRFULL = $(TMP_DIRFULL:Static=) -SUFX = $(SUFX:s=) -EXT = dll -TMP_DIRFULL = $(TMP_DIRFULL:X=) -SUFX = $(SUFX:x=) -!else -TMP_DIRFULL = $(TMP_DIRFULL:Dynamic=) -EXT = lib -!if $(MSVCRT) && $(TCL_VERSION) > 86 || !$(MSVCRT) && $(TCL_VERSION) < 87 -TMP_DIRFULL = $(TMP_DIRFULL:X=) -SUFX = $(SUFX:x=) -!endif -!endif - -!if !$(TCL_THREADS) || $(TCL_VERSION) > 86 -TMP_DIRFULL = $(TMP_DIRFULL:Threaded=) -SUFX = $(SUFX:t=) -!endif - -!ifndef TMP_DIR -TMP_DIR = $(TMP_DIRFULL) -!ifndef OUT_DIR -OUT_DIR = .\$(BUILDDIRTOP) -!endif -!else -!ifndef OUT_DIR -OUT_DIR = $(TMP_DIR) -!endif -!endif - -# Relative paths -> absolute -!if [echo OUT_DIR = \> nmakehlp.out] \ - || [nmakehlp -Q "$(OUT_DIR)" >> nmakehlp.out] -!error *** Could not fully qualify path OUT_DIR=$(OUT_DIR) -!endif -!if [echo TMP_DIR = \>> nmakehlp.out] \ - || [nmakehlp -Q "$(TMP_DIR)" >> nmakehlp.out] -!error *** Could not fully qualify path TMP_DIR=$(TMP_DIR) -!endif -!include nmakehlp.out - -# The name of the stubs library for the project being built -STUBPREFIX = $(PROJECT)stub - -# -# Set up paths to various Tcl executables and libraries needed by extensions -# - -# TIP 430. Unused for 8.6 but no harm defining it to allow a common rules.vc -TCL_ZIP_FILE = libtcl$(TCL_MAJOR_VERSION).$(TCL_MINOR_VERSION)$(TCL_PATCH_LETTER)$(TCL_RELEASE_SERIAL).zip -TK_ZIP_FILE = libtk$(TK_MAJOR_VERSION).$(TK_MINOR_VERSION)$(TK_PATCH_LETTER)$(TK_RELEASE_SERIAL).zip - -!if $(DOING_TCL) -TCLSHNAME = $(PROJECT)sh$(VERSION)$(SUFX).exe -TCLSH = $(OUT_DIR)\$(TCLSHNAME) -TCLIMPLIB = $(OUT_DIR)\$(PROJECT)$(VERSION)$(SUFX).lib -TCLLIBNAME = $(PROJECT)$(VERSION)$(SUFX).$(EXT) -TCLLIB = $(OUT_DIR)\$(TCLLIBNAME) -TCLSCRIPTZIP = $(OUT_DIR)\$(TCL_ZIP_FILE) - -!if $(TCL_MAJOR_VERSION) == 8 -TCLSTUBLIBNAME = $(STUBPREFIX)$(VERSION).lib -!else -TCLSTUBLIBNAME = $(STUBPREFIX).lib -!endif -TCLSTUBLIB = $(OUT_DIR)\$(TCLSTUBLIBNAME) -TCL_INCLUDES = -I"$(WIN_DIR)" -I"$(GENERICDIR)" - -!else # !$(DOING_TCL) - -!if $(TCLINSTALL) # Building against an installed Tcl - -# When building extensions, we need to locate tclsh. Depending on version -# of Tcl we are building against, this may or may not have a "t" suffix. -# Try various possibilities in turn. -TCLSH = $(_TCLDIR)\bin\tclsh$(TCL_VERSION)$(SUFX:t=).exe -!if !exist("$(TCLSH)") -TCLSH = $(_TCLDIR)\bin\tclsh$(TCL_VERSION)t$(SUFX:t=).exe -!endif - -!if $(TCL_MAJOR_VERSION) == 8 -TCLSTUBLIB = $(_TCLDIR)\lib\tclstub$(TCL_VERSION).lib -!else -TCLSTUBLIB = $(_TCLDIR)\lib\tclstub.lib -!endif -TCLIMPLIB = $(_TCLDIR)\lib\tcl$(TCL_VERSION)$(SUFX:t=).lib -# When building extensions, may be linking against Tcl that does not add -# "t" suffix (e.g. 8.5 or 8.7). If lib not found check for that possibility. -!if !exist("$(TCLIMPLIB)") -TCLIMPLIB = $(_TCLDIR)\lib\tcl$(TCL_VERSION)t$(SUFX:t=).lib -!endif -TCL_LIBRARY = $(_TCLDIR)\lib -TCLREGLIB = $(_TCLDIR)\lib\tclreg13$(SUFX:t=).lib -TCLDDELIB = $(_TCLDIR)\lib\tcldde14$(SUFX:t=).lib -TCLSCRIPTZIP = $(_TCLDIR)\lib\$(TCL_ZIP_FILE) -TCLTOOLSDIR = \must\have\tcl\sources\to\build\this\target -TCL_INCLUDES = -I"$(_TCLDIR)\include" - -!else # Building against Tcl sources - -TCLSH = $(_TCLDIR)\win\$(BUILDDIRTOP)\tclsh$(TCL_VERSION)$(SUFX:t=).exe -!if !exist($(TCLSH)) -TCLSH = $(_TCLDIR)\win\$(BUILDDIRTOP)\tclsh$(TCL_VERSION)t$(SUFX:t=).exe -!endif -!if $(TCL_MAJOR_VERSION) == 8 -TCLSTUBLIB = $(_TCLDIR)\win\$(BUILDDIRTOP)\tclstub$(TCL_VERSION).lib -!else -TCLSTUBLIB = $(_TCLDIR)\win\$(BUILDDIRTOP)\tclstub.lib -!endif -TCLIMPLIB = $(_TCLDIR)\win\$(BUILDDIRTOP)\tcl$(TCL_VERSION)$(SUFX:t=).lib -# When building extensions, may be linking against Tcl that does not add -# "t" suffix (e.g. 8.5 or 8.7). If lib not found check for that possibility. -!if !exist("$(TCLIMPLIB)") -TCLIMPLIB = $(_TCLDIR)\win\$(BUILDDIRTOP)\tcl$(TCL_VERSION)t$(SUFX:t=).lib -!endif -TCL_LIBRARY = $(_TCLDIR)\library -TCLREGLIB = $(_TCLDIR)\win\$(BUILDDIRTOP)\tclreg13$(SUFX:t=).lib -TCLDDELIB = $(_TCLDIR)\win\$(BUILDDIRTOP)\tcldde14$(SUFX:t=).lib -TCLSCRIPTZIP = $(_TCLDIR)\win\$(BUILDDIRTOP)\$(TCL_ZIP_FILE) -TCLTOOLSDIR = $(_TCLDIR)\tools -TCL_INCLUDES = -I"$(_TCLDIR)\generic" -I"$(_TCLDIR)\win" - -!endif # TCLINSTALL - -!if !$(STATIC_BUILD) && "$(TCL_BUILD_FOR)" == "8" -tcllibs = "$(TCLSTUBLIB)" -!else -tcllibs = "$(TCLSTUBLIB)" "$(TCLIMPLIB)" -!endif - -!endif # $(DOING_TCL) - -# We need a tclsh that will run on the host machine as part of the build. -# IX86 runs on all architectures. -!ifndef TCLSH_NATIVE -!if "$(MACHINE)" == "IX86" || "$(MACHINE)" == "$(NATIVE_ARCH)" -TCLSH_NATIVE = $(TCLSH) -!else -!error You must explicitly set TCLSH_NATIVE for cross-compilation -!endif -!endif - -# Do the same for Tk and Tk extensions that require the Tk libraries -!if $(DOING_TK) || $(NEED_TK) -WISHNAMEPREFIX = wish -WISHNAME = $(WISHNAMEPREFIX)$(TK_VERSION)$(SUFX).exe -TKLIBNAME8 = tk$(TK_VERSION)$(SUFX).$(EXT) -TKLIBNAME9 = tcl9tk$(TK_VERSION)$(SUFX).$(EXT) -!if $(TCL_MAJOR_VERSION) == 8 || "$(TCL_BUILD_FOR)" == "8" -TKLIBNAME = tk$(TK_VERSION)$(SUFX).$(EXT) -TKIMPLIBNAME = tk$(TK_VERSION)$(SUFX).lib -!else -TKLIBNAME = tcl9tk$(TK_VERSION)$(SUFX).$(EXT) -TKIMPLIBNAME = tcl9tk$(TK_VERSION)$(SUFX).lib -!endif -!if $(TK_MAJOR_VERSION) == 8 -TKSTUBLIBNAME = tkstub$(TK_VERSION).lib -!else -TKSTUBLIBNAME = tkstub.lib -!endif - -!if $(DOING_TK) -WISH = $(OUT_DIR)\$(WISHNAME) -TKSTUBLIB = $(OUT_DIR)\$(TKSTUBLIBNAME) -TKIMPLIB = $(OUT_DIR)\$(TKIMPLIBNAME) -TKLIB = $(OUT_DIR)\$(TKLIBNAME) -TK_INCLUDES = -I"$(WIN_DIR)" -I"$(GENERICDIR)" -TKSCRIPTZIP = $(OUT_DIR)\$(TK_ZIP_FILE) - -!else # effectively NEED_TK - -!if $(TKINSTALL) # Building against installed Tk -WISH = $(_TKDIR)\bin\$(WISHNAME) -TKSTUBLIB = $(_TKDIR)\lib\$(TKSTUBLIBNAME) -TKIMPLIB = $(_TKDIR)\lib\$(TKIMPLIBNAME) -# When building extensions, may be linking against Tk that does not add -# "t" suffix (e.g. 8.5 or 8.7). If lib not found check for that possibility. -!if !exist("$(TKIMPLIB)") -TKIMPLIBNAME = tk$(TK_VERSION)$(SUFX:t=).lib -TKIMPLIB = $(_TKDIR)\lib\$(TKIMPLIBNAME) -!endif -TK_INCLUDES = -I"$(_TKDIR)\include" -TKSCRIPTZIP = $(_TKDIR)\lib\$(TK_ZIP_FILE) - -!else # Building against Tk sources - -WISH = $(_TKDIR)\win\$(BUILDDIRTOP)\$(WISHNAME) -TKSTUBLIB = $(_TKDIR)\win\$(BUILDDIRTOP)\$(TKSTUBLIBNAME) -TKIMPLIB = $(_TKDIR)\win\$(BUILDDIRTOP)\$(TKIMPLIBNAME) -# When building extensions, may be linking against Tk that does not add -# "t" suffix (e.g. 8.5 or 8.7). If lib not found check for that possibility. -!if !exist("$(TKIMPLIB)") -TKIMPLIBNAME = tk$(TK_VERSION)$(SUFX:t=).lib -TKIMPLIB = $(_TKDIR)\win\$(BUILDDIRTOP)\$(TKIMPLIBNAME) -!endif -TK_INCLUDES = -I"$(_TKDIR)\generic" -I"$(_TKDIR)\win" -I"$(_TKDIR)\xlib" -TKSCRIPTZIP = $(_TKDIR)\win\$(BUILDDIRTOP)\$(TK_ZIP_FILE) - -!endif # TKINSTALL - -tklibs = "$(TKSTUBLIB)" "$(TKIMPLIB)" - -!endif # $(DOING_TK) -!endif # $(DOING_TK) || $(NEED_TK) - -# Various output paths -PRJIMPLIB = $(OUT_DIR)\$(PROJECT)$(VERSION)$(SUFX).lib -PRJLIBNAME8 = $(PROJECT)$(VERSION)$(SUFX).$(EXT) -# Even when building against Tcl 8, PRJLIBNAME9 must not have "t" -PRJLIBNAME9 = tcl9$(PROJECT)$(VERSION)$(SUFX:t=).$(EXT) -!if $(TCL_MAJOR_VERSION) == 8 || "$(TCL_BUILD_FOR)" == "8" -PRJLIBNAME = $(PRJLIBNAME8) -!else -PRJLIBNAME = $(PRJLIBNAME9) -!endif -PRJLIB = $(OUT_DIR)\$(PRJLIBNAME) - -!if $(TCL_MAJOR_VERSION) == 8 -PRJSTUBLIBNAME = $(STUBPREFIX)$(VERSION).lib -!else -PRJSTUBLIBNAME = $(STUBPREFIX).lib -!endif -PRJSTUBLIB = $(OUT_DIR)\$(PRJSTUBLIBNAME) - -# If extension parent makefile has not defined a resource definition file, -# we will generate one from standard template. -!if !$(DOING_TCL) && !$(DOING_TK) && !$(STATIC_BUILD) -!ifdef RCFILE -RESFILE = $(TMP_DIR)\$(RCFILE:.rc=.res) -!else -RESFILE = $(TMP_DIR)\$(PROJECT).res -!endif -!endif - -################################################################### -# 11. Construct the paths for the installation directories -# The following macros get defined in this section: -# LIB_INSTALL_DIR - where libraries should be installed -# BIN_INSTALL_DIR - where the executables should be installed -# DOC_INSTALL_DIR - where documentation should be installed -# SCRIPT_INSTALL_DIR - where scripts should be installed -# INCLUDE_INSTALL_DIR - where C include files should be installed -# DEMO_INSTALL_DIR - where demos should be installed -# PRJ_INSTALL_DIR - where package will be installed (not set for Tcl and Tk) - -!if $(DOING_TCL) || $(DOING_TK) -LIB_INSTALL_DIR = $(_INSTALLDIR)\lib -BIN_INSTALL_DIR = $(_INSTALLDIR)\bin -DOC_INSTALL_DIR = $(_INSTALLDIR)\doc -!if $(DOING_TCL) -SCRIPT_INSTALL_DIR = $(_INSTALLDIR)\lib\$(PROJECT)$(TCL_MAJOR_VERSION).$(TCL_MINOR_VERSION) -MODULE_INSTALL_DIR = $(_INSTALLDIR)\lib\tcl$(TCL_MAJOR_VERSION) -!else # DOING_TK -SCRIPT_INSTALL_DIR = $(_INSTALLDIR)\lib\$(PROJECT)$(TK_MAJOR_VERSION).$(TK_MINOR_VERSION) -!endif -DEMO_INSTALL_DIR = $(SCRIPT_INSTALL_DIR)\demos -INCLUDE_INSTALL_DIR = $(_INSTALLDIR)\include - -!else # extension other than Tk - -PRJ_INSTALL_DIR = $(_INSTALLDIR)\$(PROJECT)$(DOTVERSION) -!if $(MULTIPLATFORM_INSTALL) -LIB_INSTALL_DIR = $(PRJ_INSTALL_DIR)\$(PLATFORM_IDENTIFY) -BIN_INSTALL_DIR = $(PRJ_INSTALL_DIR)\$(PLATFORM_IDENTIFY) -!else -LIB_INSTALL_DIR = $(PRJ_INSTALL_DIR) -BIN_INSTALL_DIR = $(PRJ_INSTALL_DIR) -!endif -DOC_INSTALL_DIR = $(PRJ_INSTALL_DIR) -SCRIPT_INSTALL_DIR = $(PRJ_INSTALL_DIR) -DEMO_INSTALL_DIR = $(PRJ_INSTALL_DIR)\demos -INCLUDE_INSTALL_DIR = $(_INSTALLDIR)\..\include - -!endif - -################################################################### -# 12. Set up actual options to be passed to the compiler and linker -# Now we have all the information we need, set up the actual flags and -# options that we will pass to the compiler and linker. The main -# makefile should use these in combination with whatever other flags -# and switches are specific to it. -# The following macros are defined, names are for historical compatibility: -# OPTDEFINES - /Dxxx C macro flags based on user-specified OPTS -# COMPILERFLAGS - /Dxxx C macro flags independent of any configuration options -# crt - Compiler switch that selects the appropriate C runtime -# cdebug - Compiler switches related to debug AND optimizations -# cwarn - Compiler switches that set warning levels -# cflags - complete compiler switches (subsumes cdebug and cwarn) -# ldebug - Linker switches controlling debug information and optimization -# lflags - complete linker switches (subsumes ldebug) except subsystem type -# dlllflags - complete linker switches to build DLLs (subsumes lflags) -# conlflags - complete linker switches for console program (subsumes lflags) -# guilflags - complete linker switches for GUI program (subsumes lflags) -# baselibs - minimum Windows libraries required. Parent makefile can -# define PRJ_LIBS before including rules.rc if additional libs are needed - -OPTDEFINES = /DSTDC_HEADERS /DUSE_NMAKE=1 -!if $(VCVERSION) > 1600 -OPTDEFINES = $(OPTDEFINES) /DHAVE_STDINT_H=1 -!else -OPTDEFINES = $(OPTDEFINES) /DMP_NO_STDINT=1 -!endif -!if $(VCVERSION) >= 1800 -OPTDEFINES = $(OPTDEFINES) /DHAVE_INTTYPES_H=1 /DHAVE_STDBOOL_H=1 -!endif - -!if $(TCL_MEM_DEBUG) -OPTDEFINES = $(OPTDEFINES) /DTCL_MEM_DEBUG -!endif -!if $(TCL_COMPILE_DEBUG) -OPTDEFINES = $(OPTDEFINES) /DTCL_COMPILE_DEBUG /DTCL_COMPILE_STATS -!endif -!if $(TCL_THREADS) && $(TCL_VERSION) < 87 -OPTDEFINES = $(OPTDEFINES) /DTCL_THREADS=1 -!if $(USE_THREAD_ALLOC) && $(TCL_VERSION) < 87 -OPTDEFINES = $(OPTDEFINES) /DUSE_THREAD_ALLOC=1 -!endif -!endif -!if $(STATIC_BUILD) -OPTDEFINES = $(OPTDEFINES) /DSTATIC_BUILD -!elseif $(TCL_VERSION) > 86 -OPTDEFINES = $(OPTDEFINES) /DTCL_WITH_EXTERNAL_TOMMATH -!if "$(MACHINE)" == "AMD64" || "$(MACHINE)" == "ARM64" -OPTDEFINES = $(OPTDEFINES) /DMP_64BIT -!endif -!endif -!if $(TCL_NO_DEPRECATED) -OPTDEFINES = $(OPTDEFINES) /DTCL_NO_DEPRECATED -!endif - -!if $(USE_STUBS) -# Note we do not define USE_TCL_STUBS even when building tk since some -# test targets in tk do not use stubs -!if !$(DOING_TCL) -USE_STUBS_DEFS = /DUSE_TCL_STUBS /DUSE_TCLOO_STUBS -!if $(NEED_TK) -USE_STUBS_DEFS = $(USE_STUBS_DEFS) /DUSE_TK_STUBS -!endif -!endif -!endif # USE_STUBS - -!if !$(DEBUG) -OPTDEFINES = $(OPTDEFINES) /DNDEBUG -!if $(OPTIMIZING) -OPTDEFINES = $(OPTDEFINES) /DTCL_CFG_OPTIMIZED -!endif -!endif -!if $(PROFILE) -OPTDEFINES = $(OPTDEFINES) /DTCL_CFG_PROFILED -!endif -!if "$(MACHINE)" == "AMD64" || "$(MACHINE)" == "ARM64" -OPTDEFINES = $(OPTDEFINES) /DTCL_CFG_DO64BIT -!endif -!if $(VCVERSION) < 1300 -OPTDEFINES = $(OPTDEFINES) /DNO_STRTOI64=1 -!endif - -!if $(TCL_MAJOR_VERSION) == 8 -!if "$(_USE_64BIT_TIME_T)" == "1" -OPTDEFINES = $(OPTDEFINES) /D_USE_64BIT_TIME_T=1 -!endif -!endif -!if "$(TCL_BUILD_FOR)" == "8" -OPTDEFINES = $(OPTDEFINES) /DTCL_MAJOR_VERSION=8 -!endif - -# Like the TEA system only set this non empty for non-Tk extensions -# Note: some extensions use PACKAGE_NAME and others use PACKAGE_TCLNAME -# so we pass both -!if !$(DOING_TCL) && !$(DOING_TK) -PKGNAMEFLAGS = /DPACKAGE_NAME="\"$(PRJ_PACKAGE_TCLNAME)\"" \ - /DPACKAGE_TCLNAME="\"$(PRJ_PACKAGE_TCLNAME)\"" \ - /DPACKAGE_VERSION="\"$(DOTVERSION)\"" \ - /DMODULE_SCOPE=extern -!endif - -# crt picks the C run time based on selected OPTS -!if $(MSVCRT) -!if $(DEBUG) && !$(UNCHECKED) -crt = -MDd -!else -crt = -MD -!endif -!else -!if $(DEBUG) && !$(UNCHECKED) -crt = -MTd -!else -crt = -MT -!endif -!endif - -# cdebug includes compiler options for debugging as well as optimization. -!if $(DEBUG) - -# In debugging mode, optimizations need to be disabled -cdebug = -Zi -Od $(DEBUGFLAGS) - -!else - -cdebug = $(OPTIMIZATIONS) -!if $(SYMBOLS) -cdebug = $(cdebug) -Zi -!endif - -!endif # $(DEBUG) - -# cwarn includes default warning levels, also C4090 (buggy) and C4146 is useless. -cwarn = $(WARNINGS) -wd4090 -wd4146 - -!if "$(MACHINE)" == "AMD64" || "$(MACHINE)" == "ARM64" -# Disable pointer<->int warnings related to cast between different sizes -# There are a gadzillion of these due to use of ClientData and -# clutter up compiler -# output increasing chance of a real warning getting lost. So disable them. -# Eventually some day, Tcl will be 64-bit clean. -cwarn = $(cwarn) -wd4311 -wd4312 -!endif - -### Common compiler options that are architecture specific -!if "$(MACHINE)" == "ARM" -carch = /D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE -!else -carch = -!endif - -# cpuid is only available on intel machines -!if "$(MACHINE)" == "IX86" || "$(MACHINE)" == "AMD64" -carch = $(carch) /DHAVE_CPUID=1 -!endif - -!if $(DEBUG) -# Turn warnings into errors -cwarn = $(cwarn) -WX -!endif - -INCLUDES = $(TCL_INCLUDES) $(TK_INCLUDES) $(PRJ_INCLUDES) -!if !$(DOING_TCL) && !$(DOING_TK) -INCLUDES = $(INCLUDES) -I"$(GENERICDIR)" -I"$(WIN_DIR)" -I"$(COMPATDIR)" -!endif - -# These flags are defined roughly in the order of the pre-reform -# rules.vc/makefile.vc to help visually compare that the pre- and -# post-reform build logs - -# cflags contains generic flags used for building practically all object files -cflags = -nologo -c $(COMPILERFLAGS) $(carch) $(cwarn) -Fp$(TMP_DIR)^\ $(cdebug) - -!if $(TCL_MAJOR_VERSION) == 8 && $(TCL_MINOR_VERSION) < 7 -cflags = $(cflags) -DTcl_Size=int -!endif - -# appcflags contains $(cflags) and flags for building the application -# object files (e.g. tclsh, or wish) pkgcflags contains $(cflags) plus -# flags used for building shared object files The two differ in the -# BUILD_$(PROJECT) macro which should be defined only for the shared -# library *implementation* and not for its caller interface - -appcflags_nostubs = $(cflags) $(crt) $(INCLUDES) $(TCL_DEFINES) $(PRJ_DEFINES) $(OPTDEFINES) -appcflags = $(appcflags_nostubs) $(USE_STUBS_DEFS) -pkgcflags = $(appcflags) $(PKGNAMEFLAGS) /DBUILD_$(PROJECT) -pkgcflags_nostubs = $(appcflags_nostubs) $(PKGNAMEFLAGS) /DBUILD_$(PROJECT) - -# stubscflags contains $(cflags) plus flags used for building a stubs -# library for the package. Note: /DSTATIC_BUILD is defined in -# $(OPTDEFINES) only if the OPTS configuration indicates a static -# library. However the stubs library is ALWAYS static hence included -# here irrespective of the OPTS setting. -# -# TBD - tclvfs has a comment that stubs libs should not be compiled with -GL -# without stating why. Tcl itself compiled stubs libs with this flag. -# so we do not remove it from cflags. -GL may prevent extensions -# compiled with one VC version to fail to link against stubs library -# compiled with another VC version. Check for this and fix accordingly. -stubscflags = $(cflags) $(PKGNAMEFLAGS) $(PRJ_DEFINES) $(OPTDEFINES) /Zl /GL- /DSTATIC_BUILD $(INCLUDES) $(USE_STUBS_DEFS) - -# Link flags - -!if $(DEBUG) -ldebug = -debug -debugtype:cv -!else -ldebug = -release -opt:ref -opt:icf,3 -!if $(SYMBOLS) -ldebug = $(ldebug) -debug -debugtype:cv -!endif -!endif - -# Note: Profiling is currently only possible with the Visual Studio Enterprise -!if $(PROFILE) -ldebug= $(ldebug) -profile -!endif - -### Declarations common to all linker versions -lflags = -nologo -machine:$(MACHINE) $(LINKERFLAGS) $(ldebug) - -!if $(MSVCRT) && !($(DEBUG) && !$(UNCHECKED)) && $(VCVERSION) >= 1900 -lflags = $(lflags) -nodefaultlib:libucrt.lib -!endif - -dlllflags = $(lflags) -dll -conlflags = $(lflags) -subsystem:console -guilflags = $(lflags) -subsystem:windows - -# Libraries that are required for every image. -# Extensions should define any additional libraries with $(PRJ_LIBS) -winlibs = kernel32.lib advapi32.lib - -!if $(NEED_TK) -winlibs = $(winlibs) gdi32.lib user32.lib uxtheme.lib -!endif - -# Avoid 'unresolved external symbol __security_cookie' errors. -# c.f. http://support.microsoft.com/?id=894573 -!if "$(MACHINE)" == "AMD64" -!if $(VCVERSION) > 1399 && $(VCVERSION) < 1500 -winlibs = $(winlibs) bufferoverflowU.lib -!endif -!endif - -baselibs = $(winlibs) $(PRJ_LIBS) - -!if $(MSVCRT) && !($(DEBUG) && !$(UNCHECKED)) && $(VCVERSION) >= 1900 -baselibs = $(baselibs) ucrt.lib -!endif - -################################################################ -# 13. Define standard commands, common make targets and implicit rules - -CCPKGCMD = $(cc32) $(pkgcflags) -Fo$(TMP_DIR)^\ -CCAPPCMD = $(cc32) $(appcflags) -Fo$(TMP_DIR)^\ -CCSTUBSCMD = $(cc32) $(stubscflags) -Fo$(TMP_DIR)^\ - -LIBCMD = $(lib32) -nologo $(LINKERFLAGS) -out:$@ -DLLCMD = $(link32) $(dlllflags) -out:$@ $(baselibs) $(tcllibs) $(tklibs) - -CONEXECMD = $(link32) $(conlflags) -out:$@ $(baselibs) $(tcllibs) $(tklibs) -GUIEXECMD = $(link32) $(guilflags) -out:$@ $(baselibs) $(tcllibs) $(tklibs) -RESCMD = $(rc32) -fo $@ -r -i "$(GENERICDIR)" -i "$(TMP_DIR)" \ - $(TCL_INCLUDES) /DSTATIC_BUILD=$(STATIC_BUILD) \ - /DDEBUG=$(DEBUG) -d UNCHECKED=$(UNCHECKED) \ - /DCOMMAVERSION=$(RCCOMMAVERSION) \ - /DDOTVERSION=\"$(DOTVERSION)\" \ - /DVERSION=\"$(VERSION)\" \ - /DSUFX=\"$(SUFX)\" \ - /DPROJECT=\"$(PROJECT)\" \ - /DPRJLIBNAME=\"$(PRJLIBNAME)\" - -!ifndef DEFAULT_BUILD_TARGET -DEFAULT_BUILD_TARGET = $(PROJECT) -!endif - -default-target: $(DEFAULT_BUILD_TARGET) - -!if $(MULTIPLATFORM_INSTALL) -default-pkgindex: - @echo if {[package vsatisfies [package provide Tcl] 9.0-]} { > $(OUT_DIR)\pkgIndex.tcl - @echo package ifneeded $(PRJ_PACKAGE_TCLNAME) $(DOTVERSION) \ - [list load [file join $$dir $(PLATFORM_IDENTIFY) $(PRJLIBNAME9)]] >> $(OUT_DIR)\pkgIndex.tcl - @echo } else { >> $(OUT_DIR)\pkgIndex.tcl - @echo package ifneeded $(PRJ_PACKAGE_TCLNAME) $(DOTVERSION) \ - [list load [file join $$dir $(PLATFORM_IDENTIFY) $(PRJLIBNAME8)]] >> $(OUT_DIR)\pkgIndex.tcl - @echo } >> $(OUT_DIR)\pkgIndex.tcl -!else -default-pkgindex: - @echo if {[package vsatisfies [package provide Tcl] 9.0-]} { > $(OUT_DIR)\pkgIndex.tcl - @echo package ifneeded $(PRJ_PACKAGE_TCLNAME) $(DOTVERSION) \ - [list load [file join $$dir $(PRJLIBNAME9)]] >> $(OUT_DIR)\pkgIndex.tcl - @echo } else { >> $(OUT_DIR)\pkgIndex.tcl - @echo package ifneeded $(PRJ_PACKAGE_TCLNAME) $(DOTVERSION) \ - [list load [file join $$dir $(PRJLIBNAME8)]] >> $(OUT_DIR)\pkgIndex.tcl - @echo } >> $(OUT_DIR)\pkgIndex.tcl -!endif - -default-pkgindex-tea: - @if exist $(ROOT)\pkgIndex.tcl.in nmakehlp -s << $(ROOT)\pkgIndex.tcl.in > $(OUT_DIR)\pkgIndex.tcl -@PACKAGE_VERSION@ $(DOTVERSION) -@PACKAGE_NAME@ $(PRJ_PACKAGE_TCLNAME) -@PACKAGE_TCLNAME@ $(PRJ_PACKAGE_TCLNAME) -@PKG_LIB_FILE@ $(PRJLIBNAME) -@PKG_LIB_FILE8@ $(PRJLIBNAME8) -@PKG_LIB_FILE9@ $(PRJLIBNAME9) -<< - -default-install: default-install-binaries default-install-libraries -!if $(SYMBOLS) -default-install: default-install-pdbs -!endif - -# Again to deal with historical brokenness, there is some confusion -# in terminlogy. For extensions, the "install-binaries" was used to -# locate target directory for *binary shared libraries* and thus -# the appropriate macro is LIB_INSTALL_DIR since BIN_INSTALL_DIR is -# for executables (exes). On the other hand the "install-libraries" -# target is for *scripts* and should have been called "install-scripts". -default-install-binaries: $(PRJLIB) - @echo Installing binaries to '$(LIB_INSTALL_DIR)' - @if not exist "$(LIB_INSTALL_DIR)" mkdir "$(LIB_INSTALL_DIR)" - @$(CPY) $(PRJLIB) "$(LIB_INSTALL_DIR)" >NUL - -# Alias for default-install-scripts -default-install-libraries: default-install-scripts - -default-install-scripts: $(OUT_DIR)\pkgIndex.tcl - @echo Installing libraries to '$(SCRIPT_INSTALL_DIR)' - @if exist $(LIBDIR) $(CPY) $(LIBDIR)\*.tcl "$(SCRIPT_INSTALL_DIR)" - @echo Installing package index in '$(SCRIPT_INSTALL_DIR)' - @$(CPY) $(OUT_DIR)\pkgIndex.tcl $(SCRIPT_INSTALL_DIR) - -default-install-stubs: - @echo Installing stubs library to '$(SCRIPT_INSTALL_DIR)' - @if not exist "$(SCRIPT_INSTALL_DIR)" mkdir "$(SCRIPT_INSTALL_DIR)" - @$(CPY) $(PRJSTUBLIB) "$(SCRIPT_INSTALL_DIR)" >NUL - -default-install-pdbs: - @echo Installing PDBs to '$(LIB_INSTALL_DIR)' - @if not exist "$(LIB_INSTALL_DIR)" mkdir "$(LIB_INSTALL_DIR)" - @$(CPY) "$(OUT_DIR)\*.pdb" "$(LIB_INSTALL_DIR)\" - -# "emacs font-lock highlighting fix - -default-install-docs-html: - @echo Installing documentation files to '$(DOC_INSTALL_DIR)' - @if not exist "$(DOC_INSTALL_DIR)" mkdir "$(DOC_INSTALL_DIR)" - @if exist $(DOCDIR) for %f in ("$(DOCDIR)\*.html" "$(DOCDIR)\*.css" "$(DOCDIR)\*.png") do @$(COPY) %f "$(DOC_INSTALL_DIR)" - -default-install-docs-n: - @echo Installing documentation files to '$(DOC_INSTALL_DIR)' - @if not exist "$(DOC_INSTALL_DIR)" mkdir "$(DOC_INSTALL_DIR)" - @if exist $(DOCDIR) for %f in ("$(DOCDIR)\*.n") do @$(COPY) %f "$(DOC_INSTALL_DIR)" - -default-install-demos: - @echo Installing demos to '$(DEMO_INSTALL_DIR)' - @if not exist "$(DEMO_INSTALL_DIR)" mkdir "$(DEMO_INSTALL_DIR)" - @if exist $(DEMODIR) $(CPYDIR) "$(DEMODIR)" "$(DEMO_INSTALL_DIR)" - -default-clean: - @echo Cleaning $(TMP_DIR)\* ... - @if exist $(TMP_DIR)\nul $(RMDIR) $(TMP_DIR) - @echo Cleaning $(WIN_DIR)\nmakehlp.obj, nmakehlp.exe ... - @if exist $(WIN_DIR)\nmakehlp.obj del $(WIN_DIR)\nmakehlp.obj - @if exist $(WIN_DIR)\nmakehlp.exe del $(WIN_DIR)\nmakehlp.exe - @if exist $(WIN_DIR)\nmakehlp.out del $(WIN_DIR)\nmakehlp.out - @echo Cleaning $(WIN_DIR)\nmhlp-out.txt ... - @if exist $(WIN_DIR)\nmhlp-out.txt del $(WIN_DIR)\nmhlp-out.txt - @echo Cleaning $(WIN_DIR)\_junk.pch ... - @if exist $(WIN_DIR)\_junk.pch del $(WIN_DIR)\_junk.pch - @echo Cleaning $(WIN_DIR)\vercl.x, vercl.i ... - @if exist $(WIN_DIR)\vercl.x del $(WIN_DIR)\vercl.x - @if exist $(WIN_DIR)\vercl.i del $(WIN_DIR)\vercl.i - @echo Cleaning $(WIN_DIR)\versions.vc, version.vc ... - @if exist $(WIN_DIR)\versions.vc del $(WIN_DIR)\versions.vc - @if exist $(WIN_DIR)\version.vc del $(WIN_DIR)\version.vc - -default-hose: default-clean - @echo Hosing $(OUT_DIR)\* ... - @if exist $(OUT_DIR)\nul $(RMDIR) $(OUT_DIR) - -# Only for backward compatibility -default-distclean: default-hose - -default-setup: - @if not exist $(OUT_DIR)\nul mkdir $(OUT_DIR) - @if not exist $(TMP_DIR)\nul mkdir $(TMP_DIR) - -!if "$(TESTPAT)" != "" -TESTFLAGS = $(TESTFLAGS) -file $(TESTPAT) -!endif - -default-test: default-setup $(PROJECT) - @set TCLLIBPATH=$(OUT_DIR:\=/) - @if exist $(LIBDIR) for %f in ("$(LIBDIR)\*.tcl") do @$(COPY) %f "$(OUT_DIR)" - cd "$(TESTDIR)" && $(DEBUGGER) $(TCLSH) all.tcl $(TESTFLAGS) - -default-shell: default-setup $(PROJECT) - @set TCLLIBPATH=$(OUT_DIR:\=/) - @if exist $(LIBDIR) for %f in ("$(LIBDIR)\*.tcl") do @$(COPY) %f "$(OUT_DIR)" - $(DEBUGGER) $(TCLSH) - -# Generation of Windows version resource -!ifdef RCFILE - -# Note: don't use $** in below rule because there may be other dependencies -# and only the "main" rc must be passed to the resource compiler -$(TMP_DIR)\$(PROJECT).res: $(RCDIR)\$(PROJECT).rc - $(RESCMD) $(RCDIR)\$(PROJECT).rc - -!else - -# If parent makefile has not defined a resource definition file, -# we will generate one from standard template. -$(TMP_DIR)\$(PROJECT).res: $(TMP_DIR)\$(PROJECT).rc - -$(TMP_DIR)\$(PROJECT).rc: - @$(COPY) << $(TMP_DIR)\$(PROJECT).rc -#include - -VS_VERSION_INFO VERSIONINFO - FILEVERSION COMMAVERSION - PRODUCTVERSION COMMAVERSION - FILEFLAGSMASK 0x3fL -#ifdef DEBUG - FILEFLAGS VS_FF_DEBUG -#else - FILEFLAGS 0x0L -#endif - FILEOS VOS_NT_WINDOWS32 - FILETYPE VFT_DLL - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "040904b0" - BEGIN - VALUE "FileDescription", "Tcl extension " PROJECT - VALUE "OriginalFilename", PRJLIBNAME - VALUE "FileVersion", DOTVERSION - VALUE "ProductName", "Package " PROJECT " for Tcl" - VALUE "ProductVersion", DOTVERSION - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x409, 1200 - END -END - -<< - -!endif # ifdef RCFILE - -!ifndef DISABLE_IMPLICIT_RULES -DISABLE_IMPLICIT_RULES = 0 -!endif - -!if !$(DISABLE_IMPLICIT_RULES) -# Implicit rule definitions - only for building library objects. For stubs and -# main application, the makefile should define explicit rules. - -{$(ROOT)}.c{$(TMP_DIR)}.obj:: - $(CCPKGCMD) @<< -$< -<< - -{$(WIN_DIR)}.c{$(TMP_DIR)}.obj:: - $(CCPKGCMD) @<< -$< -<< - -{$(GENERICDIR)}.c{$(TMP_DIR)}.obj:: - $(CCPKGCMD) @<< -$< -<< - -{$(COMPATDIR)}.c{$(TMP_DIR)}.obj:: - $(CCPKGCMD) @<< -$< -<< - -{$(RCDIR)}.rc{$(TMP_DIR)}.res: - $(RESCMD) $< - -{$(WIN_DIR)}.rc{$(TMP_DIR)}.res: - $(RESCMD) $< - -{$(TMP_DIR)}.rc{$(TMP_DIR)}.res: - $(RESCMD) $< - -.SUFFIXES: -.SUFFIXES:.c .rc - -!endif - -################################################################ -# 14. Sanity check selected options against Tcl build options -# When building an extension, certain configuration options should -# match the ones used when Tcl was built. Here we check and -# warn on a mismatch. -!if !$(DOING_TCL) - -!if $(TCLINSTALL) # Building against an installed Tcl -!if exist("$(_TCLDIR)\lib\nmake\tcl.nmake") -TCLNMAKECONFIG = "$(_TCLDIR)\lib\nmake\tcl.nmake" -!endif -!else # !$(TCLINSTALL) - building against Tcl source -!if exist("$(_TCLDIR)\win\$(BUILDDIRTOP)\tcl.nmake") -TCLNMAKECONFIG = "$(_TCLDIR)\win\$(BUILDDIRTOP)\tcl.nmake" -!endif -!endif # TCLINSTALL - -!if $(CONFIG_CHECK) -!ifdef TCLNMAKECONFIG -!include $(TCLNMAKECONFIG) - -!if defined(CORE_MACHINE) && "$(CORE_MACHINE)" != "$(MACHINE)" -!error ERROR: Build target ($(MACHINE)) does not match the Tcl library architecture ($(CORE_MACHINE)). -!endif -!if $(TCL_VERSION) < 87 && defined(CORE_USE_THREAD_ALLOC) && $(CORE_USE_THREAD_ALLOC) != $(USE_THREAD_ALLOC) -!message WARNING: Value of USE_THREAD_ALLOC ($(USE_THREAD_ALLOC)) does not match its Tcl core value ($(CORE_USE_THREAD_ALLOC)). -!endif -!if defined(CORE_DEBUG) && $(CORE_DEBUG) != $(DEBUG) -!message WARNING: Value of DEBUG ($(DEBUG)) does not match its Tcl library configuration ($(DEBUG)). -!endif -!endif - -!endif # TCLNMAKECONFIG - -!endif # !$(DOING_TCL) - - -#---------------------------------------------------------- -# Display stats being used. -#---------------------------------------------------------- - -!if !$(DOING_TCL) -!message *** Building against Tcl at '$(_TCLDIR)' -!endif -!if !$(DOING_TK) && $(NEED_TK) -!message *** Building against Tk at '$(_TKDIR)' -!endif -!message *** Intermediate directory will be '$(TMP_DIR)' -!message *** Output directory will be '$(OUT_DIR)' -!message *** Installation, if selected, will be in '$(_INSTALLDIR)' -!message *** Suffix for binaries will be '$(SUFX)' -!message *** Compiler version $(VCVER). Target $(MACHINE), host $(NATIVE_ARCH). - -!endif # ifdef _RULES_VC diff --git a/autoconf/tea/win/targets.vc b/autoconf/tea/win/targets.vc deleted file mode 100644 index 49ed7d4a4..000000000 --- a/autoconf/tea/win/targets.vc +++ /dev/null @@ -1,98 +0,0 @@ -#------------------------------------------------------------- -*- makefile -*- -# targets.vc -- -# -# Part of the nmake based build system for Tcl and its extensions. -# This file defines some standard targets for the convenience of extensions -# and can be optionally included by the extension makefile. -# See TIP 477 (https://core.tcl-lang.org/tips/doc/main/tip/477.md) for docs. - -$(PROJECT): setup pkgindex $(PRJLIB) - -!ifdef PRJ_STUBOBJS -$(PROJECT): $(PRJSTUBLIB) -$(PRJSTUBLIB): $(PRJ_STUBOBJS) - $(LIBCMD) $** - -$(PRJ_STUBOBJS): - $(CCSTUBSCMD) %s -!endif # PRJ_STUBOBJS - -!ifdef PRJ_MANIFEST -$(PROJECT): $(PRJLIB).manifest -$(PRJLIB).manifest: $(PRJ_MANIFEST) - @nmakehlp -s << $** >$@ -@MACHINE@ $(MACHINE:IX86=X86) -<< -!endif - -!if "$(PROJECT)" != "tcl" && "$(PROJECT)" != "tk" -$(PRJLIB): $(PRJ_OBJS) $(RESFILE) -!if $(STATIC_BUILD) - $(LIBCMD) $** -!else - $(DLLCMD) $** - $(_VC_MANIFEST_EMBED_DLL) -!endif - -@del $*.exp -!endif - -!if "$(PRJ_HEADERS)" != "" && "$(PRJ_OBJS)" != "" -$(PRJ_OBJS): $(PRJ_HEADERS) -!endif - -# If parent makefile has defined stub objects, add their installation -# to the default install -!if "$(PRJ_STUBOBJS)" != "" -default-install: default-install-stubs -!endif - -# Unlike the other default targets, these cannot be in rules.vc because -# the executed command depends on existence of macro PRJ_HEADERS_PUBLIC -# that the parent makefile will not define until after including rules-ext.vc -!if "$(PRJ_HEADERS_PUBLIC)" != "" -default-install: default-install-headers -default-install-headers: - @echo Installing headers to '$(INCLUDE_INSTALL_DIR)' - @for %f in ($(PRJ_HEADERS_PUBLIC)) do @$(COPY) %f "$(INCLUDE_INSTALL_DIR)" -!endif - -!if "$(DISABLE_STANDARD_TARGETS)" == "" -DISABLE_STANDARD_TARGETS = 0 -!endif - -!if "$(DISABLE_TARGET_setup)" == "" -DISABLE_TARGET_setup = 0 -!endif -!if "$(DISABLE_TARGET_install)" == "" -DISABLE_TARGET_install = 0 -!endif -!if "$(DISABLE_TARGET_clean)" == "" -DISABLE_TARGET_clean = 0 -!endif -!if "$(DISABLE_TARGET_test)" == "" -DISABLE_TARGET_test = 0 -!endif -!if "$(DISABLE_TARGET_shell)" == "" -DISABLE_TARGET_shell = 0 -!endif - -!if !$(DISABLE_STANDARD_TARGETS) -!if !$(DISABLE_TARGET_setup) -setup: default-setup -!endif -!if !$(DISABLE_TARGET_install) -install: default-install -!endif -!if !$(DISABLE_TARGET_clean) -clean: default-clean -realclean: hose -hose: default-hose -distclean: realclean default-distclean -!endif -!if !$(DISABLE_TARGET_test) -test: default-test -!endif -!if !$(DISABLE_TARGET_shell) -shell: default-shell -!endif -!endif # DISABLE_STANDARD_TARGETS diff --git a/autosetup/README.md b/autosetup/README.md index 2d6cf723c..3301f5739 100644 --- a/autosetup/README.md +++ b/autosetup/README.md @@ -14,7 +14,9 @@ build infrastructure. It is not an [Autosetup][] reference. - Symbolic Names of Feature Flags - Do Not Update Global Shared State - [Updating Autosetup](#updating) - - [Patching Autosetup for Project-local changes](#patching) + - ***[Patching Autosetup for Project-local changes](#patching)*** +- [Branch-specific Customization](#branch-customization) + ------------------------------------------------------------------------ @@ -42,12 +44,13 @@ following files: (e.g. Fossil), and personal projects of SQLite's developers. It is essentially an amalgamation of a decade's worth of autosetup-related utility code. -- [auto.def][]: the primary driver for the `./configure` process. - When we talk about "the configure script," we're referring to - this file. - [sqlite-config.tcl][]: utility code which is too project-specific for `proj.tcl`. We split this out of `auto.def` so that it can be used by both `auto.def` and... +- [auto.def][]: the primary driver for the `./configure` process. + When we talk about "the configure script," we're technically + referring to this file, though it actually contains very little + of the TCL code. - [autoconf/auto.def][]: the main driver script for the "autoconf" bundle's configure script. It is essentially a slightly trimmed-down version of the main `auto.def` file. The `autoconf` dir was ported @@ -61,11 +64,11 @@ Autosetup API Tips This section briefly covers only APIs which are frequently useful in day-to-day maintenance and might not be immediately recognized as such -obvious from a casual perusal of the relevant TCL files. The complete -docs of those with `proj-` prefix can be found in [proj.tcl][] and -those with an `sqlite-` prefix are in [sqlite-config.tcl][]. The -others are scattered around [the TCL files in -./autosetup](/dir/autosetup). +from a casual perusal of the relevant TCL files. The complete docs of +those with `proj-` prefix can be found in [proj.tcl][] and those with +an `sqlite-` prefix are in [sqlite-config.tcl][]. The others are part +of Autosetup's core packages and are scattered around [the TCL files +in ./autosetup](/dir/autosetup). In (mostly) alphabetical order: @@ -83,19 +86,14 @@ In (mostly) alphabetical order: Works like `get-env` but will, if that function finds no match, look for a file named `./.env-$VAR` and, if found, return its trimmed contents. This can be used, e.g., to set a developer's - local preferences for the default `CFLAGS`. - -- **`define-for-opt flag defineName ?checkingMsg? ?yesVal=1? ?noVal=0?`**\ - `[define $defineName]` to either `$yesVal` or `$noVal`, depending on - whether `--$flag` is truthy or not. `$checkingMsg` is a - human-readable description of the check being made, e.g. "enable foo?" - If no `checkingMsg` is provided, the operation is silent.\ - Potential TODO: change the final two args to `-yes` and `-no` - flags. They're rarely needed, though: search [auto.def][] for - `TSTRNNR_OPTS` for an example of where they are used. + local preferences for the default `CFLAGS`.\ + Tip: adding `-O0` to `.env-CFLAGS` reduces rebuild times + considerably at the cost of performance in `make devtest` and the + like. - **`proj-fatal msg`**\ - Emits `$msg` to stderr and exits with non-zero. + Emits `$msg` to stderr and exits with non-zero. Its differences from + autosetup's `user-error` are purely cosmetic. - **`proj-if-opt-truthy flag thenScript ?elseScript?`**\ Evals `thenScript` if the given `--flag` is truthy, else it @@ -117,7 +115,10 @@ In (mostly) alphabetical order: else 0. This distinction can be used to determine, e.g., whether `--with-readline` was provided or whether we're searching for readline by default. In the former case, failure to find it should - be treated as fatal, where in the latter case it's not. + be treated as fatal, where in the latter case it's not.\ + Unlike most functions which deal with `--flags`, this one does not + validate that `$FLAG` is a registered flag so will not fail fatally + if `$FLAG` is not registered as an Autosetup option. - **`proj-val-truthy value`**\ Returns 1 if `$value` is "truthy," See `proj-opt-truthy` for the definition @@ -139,6 +140,15 @@ In (mostly) alphabetical order: The shell-specific counterpart of `sqlite-add-feature-flag` which only adds the given flag(s) to the CLI-shell-specific CFLAGS. +- **`sqlite-configure BUILD-NAME {script}`**\ + This is where all configure `--flags` are defined for all known + build modes ("canonical" or "autoconf"). After processing all flags, + this function runs `$script`, which contains the build-mode-specific + configuration bits, and then runs any finalization bits which are + common to all build modes. The `auto.def` files are intended to contain + exactly two commands: + `use sqlite-config; sqlite-configure BUILD-NAME {script}` + - **`user-notice msg`**\ Queues `$msg` to be sent to stderr, but does not emit it until either `show-notices` is called or the next time autosetup would @@ -152,13 +162,18 @@ In (mostly) alphabetical order: Ensuring TCL Compatibility ======================================================================== -It is important that any TCL files used by the configure process -remain compatible with both [JimTCL][] and the canonical TCL. Though -JimTCL has outstanding compatibility with canonical TCL, it does have -a few corners with incompatibilities, e.g. regular expressions. If a -script runs in JimTCL without using any JimTCL-specific features, then -it's a certainty that it will run in canonical TCL as well. The -opposite, however, is not _always_ the case. +One of the significant benefits of using Autosetup is that (A) this +project uses many TCL scripts in the build process and (B) Autosetup +comes with a TCL interpreter named [JimTCL][]. + +It is important that any TCL files used by the configure process and +makefiles remain compatible with both [JimTCL][] and the canonical +TCL. Though JimTCL has outstanding compatibility with canonical TCL, +it does have a few corners with incompatibilities, e.g. regular +expressions. If a script runs in JimTCL without using any +JimTCL-specific features, then it's a certainty that it will run in +canonical TCL as well. The opposite, however, is not _always_ the +case. When [`./configure`](/file/configure) is run, it goes through a bootstrapping process to find a suitable TCL with which to run the @@ -187,14 +202,17 @@ compatibility across TCL implementations: before looking for a system-level `tclsh`. Be aware, though, that `make distclean` will remove that file. -**Note that `jimsh0` is distinctly different from the `jimsh`** which -gets built for code-generation purposes. The latter requires +**Note that `./jimsh0` is distinctly different from the `./jimsh`** +which gets built for code-generation purposes. The latter requires non-default build flags to enable features which are platform-dependent, most notably to make its `[file normalize]` work. This means, for example, that the configure script and its utility APIs must not use `[file normalize]`, but autosetup provides a TCL-only implementation of `[file-normalize]` (note the dash) for -portable use in the configure script. +portable use in the configure script. Contrariwise, code-generation +scripts invoked via `make` may use `[file normalize]`, as they'll use +`./jimsh` or `tclsh` instead of `./jimsh0`. + Known TCL Incompatibilities ------------------------------------------------------------------------ @@ -221,6 +239,7 @@ A summary of known incompatibilities in JimTCL - `regsub` does not support the `\y` flag. A workaround is demonstrated in [](/info/c2e5dd791cce3ec4). + Design Conventions ======================================================================== @@ -247,8 +266,9 @@ dots are not permitted. The `X.y` convention is used in the makefiles primarily because the person who did the initial port finds that considerably easier on the eyes and fingers. In practice, the `X_Y` form of such exports is used -exactly once in [Makefile.in][], where it's translated into into `X.y` -form for consumption by [Makefile.in][] and [main.mk][]. For example: +exactly once in [Makefile.in][], where it's translated from `@X_Y@` +into into `X.y` form for consumption by [Makefile.in][] and +[main.mk][]. For example: > ``` @@ -265,9 +285,9 @@ of taste, for which there is no accounting.) Do Not Update Global Shared State ------------------------------------------------------------------------ -In both the legacy Autotools-driven build and in common Autosetup -usage, feature tests performed by the configure script may amend -global flags such as `LIBS`, `LDFLAGS`, and `CFLAGS`[^as-cflags]. That's +In both the legacy Autotools-driven build and common Autosetup usage, +feature tests performed by the configure script may amend global flags +such as `LIBS`, `LDFLAGS`, and `CFLAGS`[^as-cflags]. That's appropriate for a makefile which builds a single deliverable, but less so for makefiles which produce multiple deliverables. Drawbacks of that approach include: @@ -275,8 +295,8 @@ that approach include: - It's unlikely that every single deliverable will require the same core set of those flags. - It can be difficult to determine the origin of any given change to - that global state because those changes are hidden behind voodoo performed - outside the immediate visibility of the configure script's + that global state because those changes are hidden behind voodoo + performed outside the immediate visibility of the configure script's maintainer. - It can force the maintainers of the configure script to place tests in a specific order so that the resulting flags get applied at @@ -357,17 +377,70 @@ Patching Autosetup for Project-local Changes Autosetup reserves the flag name **`--debug`** for its own purposes, and its own special handling of `--enable-...` flags makes `--debug` -an alias for `--enable-debug`. As we have a long history of using -`--enable-debug` for this project's own purposes, we patch autosetup -to use the name `--autosetup-debug` in place of `--debug`. That -requires (as of this writing) four small edits in -[](/file/autosetup/autosetup), as demonstrated in [check-in -3296c8d3](/info/3296c8d3). +an alias for `--enable-debug`. As this project has a long history of +using `--enable-debug`, we patch autosetup to use the name +`--autosetup-debug` in place of `--debug`. That requires (as of this +writing) four small edits in [](/file/autosetup/autosetup), as +demonstrated in [check-in 3296c8d3](/info/3296c8d3). If autosetup is upgraded and this patch is _not_ applied the invoking `./configure` will fail loudly because of the declaration of the `debug` flag in `auto.def` - duplicated flags are not permitted. + +Branch-specific Customization +======================================================================== + +Certain vendor-specific branches require slight configure script +customization. Rather than editing `sqlite-config.tcl` for this, +which frequently leads to merge conflicts, the following approach +is recommended: + +In the vendor-specific branch, create a file named +`autosetup/sqlite-custom.tcl`. + +That file should contain the following content... + +If flag customization is required, add: + +> +``` +proc sqlite-custom-flags {} { + # If any existing --flags require different default values + # then call: + options-defaults { + flag-name new-default-value + ... + } + # ^^^ That will replace the default value but will not update + # the --help text, which may lead to some confusion: + # https://github.com/msteveb/autosetup/issues/77 + + return { + {*} { + new-flag-name => {Help text} + ... + } + }; #see below +} +``` + +That function must return either an empty string or a list in the form +used internally by `sqlite-config.tcl:sqlite-configure`. + +Next, define: + +> +``` +proc sqlite-custom-handle-flags {} { + ... do any custom flag handling here ... +} +``` + +That function, if defined, will be called relatively late in the +configure process, before any filtered files are generated but after +all other significant processing. + [Autosetup]: https://msteveb.github.io/autosetup/ [auto.def]: /file/auto.def diff --git a/tool/find_tclconfig.tcl b/autosetup/find_tclconfig.tcl similarity index 100% rename from tool/find_tclconfig.tcl rename to autosetup/find_tclconfig.tcl diff --git a/autosetup/jimsh0.c b/autosetup/jimsh0.c index 1a6453d0c..b035524c9 100644 --- a/autosetup/jimsh0.c +++ b/autosetup/jimsh0.c @@ -9132,7 +9132,7 @@ int Jim_StringEqObj(Jim_Obj *aObjPtr, Jim_Obj *bObjPtr) const char *sA = Jim_GetString(aObjPtr, &Alen); const char *sB = Jim_GetString(bObjPtr, &Blen); - return Alen == Blen && memcmp(sA, sB, Alen) == 0; + return Alen == Blen && *sA == *sB && memcmp(sA, sB, Alen) == 0; } } @@ -10242,7 +10242,7 @@ static int JimCommandsHT_KeyCompare(void *privdata, const void *key1, const void int len1, len2; const char *str1 = Jim_GetStringNoQualifier((Jim_Obj *)key1, &len1); const char *str2 = Jim_GetStringNoQualifier((Jim_Obj *)key2, &len2); - return len1 == len2 && memcmp(str1, str2, len1) == 0; + return len1 == len2 && *str1 == *str2 && memcmp(str1, str2, len1) == 0; } static void JimCommandsHT_ValDestructor(void *interp, void *val) @@ -13864,6 +13864,13 @@ static int JimExprOpNumUnary(Jim_Interp *interp, struct JimExprNode *node) case JIM_EXPROP_NOT: wC = !bA; break; + case JIM_EXPROP_UNARYPLUS: + case JIM_EXPROP_UNARYMINUS: + rc = JIM_ERR; + Jim_SetResultFormatted(interp, + "can't use non-numeric string as operand of \"%s\"", + node->type == JIM_EXPROP_UNARYPLUS ? "+" : "-"); + break; default: abort(); } @@ -19868,16 +19875,22 @@ static int JimCatchTryHelper(Jim_Interp *interp, int istry, int argc, Jim_Obj *c } else if (errorCodeObj) { int len = Jim_ListLength(interp, argv[idx + 1]); - int i; - ret = JIM_OK; + if (len > Jim_ListLength(interp, errorCodeObj)) { - for (i = 0; i < len; i++) { - Jim_Obj *matchObj = Jim_ListGetIndex(interp, argv[idx + 1], i); - Jim_Obj *objPtr = Jim_ListGetIndex(interp, errorCodeObj, i); - if (Jim_StringCompareObj(interp, matchObj, objPtr, 0) != 0) { - ret = -1; - break; + ret = -1; + } + else { + int i; + ret = JIM_OK; + + for (i = 0; i < len; i++) { + Jim_Obj *matchObj = Jim_ListGetIndex(interp, argv[idx + 1], i); + Jim_Obj *objPtr = Jim_ListGetIndex(interp, errorCodeObj, i); + if (Jim_StringCompareObj(interp, matchObj, objPtr, 0) != 0) { + ret = -1; + break; + } } } } @@ -20253,7 +20266,7 @@ static int Jim_DictCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *arg } case OPT_SET: - return Jim_SetDictKeysVector(interp, argv[2], argv + 3, argc - 4, argv[argc - 1], JIM_ERRMSG); + return Jim_SetDictKeysVector(interp, argv[2], argv + 3, argc - 4, argv[argc - 1], JIM_ERRMSG | JIM_UNSHARED); case OPT_EXISTS:{ int rc = Jim_DictKeysVector(interp, argv[2], argv + 3, argc - 3, &objPtr, JIM_NONE); @@ -20265,7 +20278,7 @@ static int Jim_DictCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *arg } case OPT_UNSET: - if (Jim_SetDictKeysVector(interp, argv[2], argv + 3, argc - 3, NULL, JIM_NONE) != JIM_OK) { + if (Jim_SetDictKeysVector(interp, argv[2], argv + 3, argc - 3, NULL, JIM_UNSHARED) != JIM_OK) { return JIM_ERR; } return JIM_OK; diff --git a/autosetup/proj.tcl b/autosetup/proj.tcl index 4be268da6..133556706 100644 --- a/autosetup/proj.tcl +++ b/autosetup/proj.tcl @@ -8,26 +8,35 @@ # * May you find forgiveness for yourself and forgive others. # * May you share freely, never taking more than you give. # -######################################################################## + +# +# ----- @module proj.tcl ----- +# @section Project-agnostic Helper APIs +# + +# # Routines for Steve Bennett's autosetup which are common to trees # managed in and around the umbrella of the SQLite project. # # The intent is that these routines be relatively generic, independent # of a given project. # +# For practical purposes, the copy of this file hosted in the SQLite +# project is the "canonical" one: +# +# https://sqlite.org/src/file/autosetup/proj.tcl +# # This file was initially derived from one used in the libfossil # project, authored by the same person who ported it here, and this is # noted here only as an indication that there are no licensing issues # despite this code having a handful of near-twins running around a # handful of third-party source trees. # -######################################################################## -# # Design notes: # -# - Symbols with a suffix of _ are intended for internal use within +# - Symbols with _ separators are intended for internal use within # this file, and are not part of the API which auto.def files should -# rely on. +# rely on. Symbols with - separators are public APIs. # # - By and large, autosetup prefers to update global state with the # results of feature checks, e.g. whether the compiler supports flag @@ -45,72 +54,124 @@ # test, downstream tests may not like the $prefix/lib path added by # the rpath test. To avoid such problems, we avoid (intentionally) # updating global state via feature tests. -######################################################################## - -# ----- @module proj.tcl ----- -# @section Project Helper APIs +# -######################################################################## -# $proj_ is an internal-use-only array for storing whatever generic +# +# $proj__Config is an internal-use-only array for storing whatever generic # internal stuff we need stored. -array set proj_ {} -set proj_(isatty) [isatty? stdout] +# +array set ::proj__Config { + self-tests 1 +} -######################################################################## + +# +# List of dot-in files to filter in the final stages of +# configuration. Some configuration steps may append to this. Each +# one in this list which exists will trigger the generation of a +# file with that same name, minus the ".in", in the build directory +# (which differ from the source dir in out-of-tree builds). +# +# See: proj-dot-ins-append and proj-dot-ins-process +# +set ::proj__Config(dot-in-files) [list] +set ::proj__Config(isatty) [isatty? stdout] + +# # @proj-warn msg # -# Emits a warning message to stderr. -proc proj-warn {msg} { +# Emits a warning message to stderr. All args are appended with a +# space between each. +# +proc proj-warn {args} { show-notices - puts stderr "WARNING: $msg" + puts stderr [join [list "WARNING: \[[proj-scope 1]\]: " {*}$args] " "] } -######################################################################## -# @proj-error msg -# -# Emits an error message to stderr and exits with non-0. -proc proj-fatal {msg} { + + +# Internal impl of [proj-fatal] and [proj-error]. It must be called +# using tailcall. +proc proj__faterr {failMode argv} { show-notices - puts stderr "ERROR: $msg" - exit 1 + set lvl 1 + while {"-up" eq [lindex $argv 0]} { + set argv [lassign $argv -] + incr lvl + } + if {$failMode} { + puts stderr [join [list "FATAL: \[[proj-scope $lvl]]: " {*}$argv]] + exit 1 + } else { + error [join [list "\[[proj-scope $lvl]]:" {*}$argv]] + } } -######################################################################## -# @proj-assert script -# -# Kind of like a C assert: if uplevel (eval) of [expr {$script}] is -# false, a fatal error is triggered. The error message, by default, -# includes the body of the failed assertion, but if $descr is set then -# that is used instead. -proc proj-assert {script {descr ""}} { - if {1 == [get-env proj-assert 0]} { + +# +# @proj-fatal ?-up...? msg... +# +# Emits an error message to stderr and exits with non-0. All args are +# appended with a space between each. +# +# The calling scope's name is used in the error message. To instead +# use the name of a call higher up in the stack, use -up once for each +# additional level. +# +proc proj-fatal {args} { + tailcall proj__faterr 1 $args +} + +# +# @proj-error ?-up...? msg... +# +# Works like proj-fatal but uses [error] intead of [exit]. +# +proc proj-error {args} { + tailcall proj__faterr 0 $args +} + +set ::proj__Config(verbose-assert) [get-env proj-assert-verbose 0] +# +# @proj-assert script ?message? +# +# Kind of like a C assert: if uplevel of [list expr $script] is false, +# a fatal error is triggered. The error message, by default, includes +# the body of the failed assertion, but if $msg is set then that is +# used instead. +# +proc proj-assert {script {msg ""}} { + if {1 eq $::proj__Config(verbose-assert)} { msg-result [proj-bold "asserting: $script"] } - set x "expr \{ $script \}" - if {![uplevel 1 $x]} { - if {"" eq $descr} { - set descr $script + if {![uplevel 1 [list expr $script]]} { + if {"" eq $msg} { + set msg $script } - proj-fatal "Assertion failed: $descr" + proj-fatal "Assertion failed in \[[proj-scope 1]\]: $msg" } } -######################################################################## +# # @proj-bold str # # If this function believes that the current console might support # ANSI escape sequences then this returns $str wrapped in a sequence # to bold that text, else it returns $str as-is. -proc proj-bold {str} { - if {$::autosetup(iswin) || !$::proj_(isatty)} { - return $str +# +proc proj-bold {args} { + if {$::autosetup(iswin) || !$::proj__Config(isatty)} { + return [join $args] } - return "\033\[1m${str}\033\[0m" + return "\033\[1m${args}\033\[0m" } -######################################################################## +# # @proj-indented-notice ?-error? ?-notice? msg # # Takes a multi-line message and emits it with consistent indentation. +# It does not perform any line-wrapping of its own. Which output +# routine it uses depends on its flags, defaulting to msg-result. +# For -error and -notice it uses user-notice. # # If the -notice flag it used then it emits using [user-notice], which # means its rendering will (A) go to stderr and (B) be delayed until @@ -121,9 +182,10 @@ proc proj-bold {str} { # # If neither -notice nor -error are used, the message will be sent to # stdout without delay. +# proc proj-indented-notice {args} { set fErr "" - set outFunc "puts" + set outFunc "msg-result" while {[llength $args] > 1} { switch -exact -- [lindex $args 0] { -error { @@ -154,39 +216,24 @@ proc proj-indented-notice {args} { } } -######################################################################## +# # @proj-is-cross-compiling # # Returns 1 if cross-compiling, else 0. +# proc proj-is-cross-compiling {} { - return [expr {[get-define host] ne [get-define build]}] -} - -######################################################################## -# proj-lshift_ shifts $count elements from the list named $listVar -# and returns them as a new list. On empty input, returns "". -# -# Modified slightly from: https://wiki.tcl-lang.org/page/lshift -proc proj-lshift_ {listVar {count 1}} { - upvar 1 $listVar l - if {![info exists l]} { - # make the error message show the real variable name - error "can't read \"$listVar\": no such variable" - } - if {![llength $l]} { - # error Empty - return "" - } - set r [lrange $l 0 [incr count -1]] - set l [lreplace $l [set l 0] $count] - return $r + expr {[get-define host] ne [get-define build]} } -######################################################################## +# +# @proj-strip-hash-comments value +# # Expects to receive string input, which it splits on newlines, strips -# out any lines which begin with an number of whitespace followed by a -# '#', and returns a value containing the [append]ed results of each -# remaining line with a \n between each. +# out any lines which begin with any number of whitespace followed by +# a '#', and returns a value containing the [append]ed results of each +# remaining line with a \n between each. It does not strip out +# comments which appear after the first non-whitespace character. +# proc proj-strip-hash-comments {val} { set x {} foreach line [split $val \n] { @@ -197,28 +244,57 @@ proc proj-strip-hash-comments {val} { return $x } -######################################################################## +# +# @proj-cflags-without-werror +# +# Fetches [define $var], strips out any -Werror entries, and returns +# the new value. This is intended for temporarily stripping -Werror +# from CFLAGS or CPPFLAGS within the scope of a [define-push] block. +# +proc proj-cflags-without-werror {{var CFLAGS}} { + set rv {} + foreach f [get-define $var ""] { + switch -exact -- $f { + -Werror {} + default { lappend rv $f } + } + } + join $rv " " +} + +# # @proj-check-function-in-lib # -# A proxy for cc-check-function-in-lib which does not make any global -# changes to the LIBS define. Returns the result of -# cc-check-function-in-lib (i.e. true or false). The resulting linker -# flags are stored in ${lib_${function}}. +# A proxy for cc-check-function-in-lib with the following differences: +# +# - Does not make any global changes to the LIBS define. +# +# - Strips out the -Werror flag from CFLAGS before running the test, +# as these feature tests will often fail if -Werror is used. +# +# Returns the result of cc-check-function-in-lib (i.e. true or false). +# The resulting linker flags are stored in the [define] named +# lib_${function}. +# proc proj-check-function-in-lib {function libs {otherlibs {}}} { set found 0 - define-push {LIBS} { + define-push {LIBS CFLAGS} { + #puts "CFLAGS before=[get-define CFLAGS]" + define CFLAGS [proj-cflags-without-werror] + #puts "CFLAGS after =[get-define CFLAGS]" set found [cc-check-function-in-lib $function $libs $otherlibs] } return $found } -######################################################################## +# # @proj-search-for-header-dir ?-dirs LIST? ?-subdirs LIST? header # # Searches for $header in a combination of dirs and subdirs, specified # by the -dirs {LIST} and -subdirs {LIST} flags (each of which have # sane defaults). Returns either the first matching dir or an empty # string. The return value does not contain the filename part. +# proc proj-search-for-header-dir {header args} { set subdirs {include} set dirs {/usr /usr/local /mingw} @@ -231,7 +307,7 @@ proc proj-search-for-header-dir {header args} { -dirs { set args [lassign $args - dirs] } -subdirs { set args [lassign $args - subdirs] } default { - proj-fatal "Unhandled argument: $args" + proj-error "Unhandled argument: $args" } } } @@ -245,7 +321,7 @@ proc proj-search-for-header-dir {header args} { return "" } -######################################################################## +# # @proj-find-executable-path ?-v? binaryName # # Works similarly to autosetup's [find-executable-path $binName] but: @@ -253,6 +329,7 @@ proc proj-search-for-header-dir {header args} { # - If the first arg is -v, it's verbose about searching, else it's quiet. # # Returns the full path to the result or an empty string. +# proc proj-find-executable-path {args} { set binName $args set verbose 0 @@ -272,7 +349,7 @@ proc proj-find-executable-path {args} { return $check } -######################################################################## +# # @proj-bin-define binName ?defName? # # Uses [proj-find-executable-path $binName] to (verbosely) search for @@ -282,6 +359,7 @@ proc proj-find-executable-path {args} { # The define'd name is: If $defName is not empty, it is used as-is. If # $defName is empty then "BIN_X" is used, where X is the upper-case # form of $binName with any '-' characters replaced with '_'. +# proc proj-bin-define {binName {defName {}}} { set check [proj-find-executable-path -v $binName] if {"" eq $defName} { @@ -291,7 +369,7 @@ proc proj-bin-define {binName {defName {}}} { return $check } -######################################################################## +# # @proj-first-bin-of bin... # # Looks for the first binary found of the names passed to this @@ -302,11 +380,13 @@ proc proj-bin-define {binName {defName {}}} { # any define'd name that function stores for the result (because the # caller has no sensible way of knowing which result it was unless # they pass only a single argument). +# proc proj-first-bin-of {args} { set rc "" foreach b $args { set u [string toupper $b] - # Note that cc-path-progs defines $u to false if it finds no match. + # Note that cc-path-progs defines $u to "false" if it finds no + # match. if {[cc-path-progs $b]} { set rc [get-define $u] } @@ -316,7 +396,7 @@ proc proj-first-bin-of {args} { return $rc } -######################################################################## +# # @proj-opt-was-provided key # # Returns 1 if the user specifically provided the given configure flag @@ -334,6 +414,9 @@ proc proj-first-bin-of {args} { # to the default value of baz. If the user does not explicitly pass in # --foo-bar (with or without a value) then this returns 0. # +# Calling [proj-opt-set] is, for purposes of the above, equivalent to +# explicitly passing in the flag. +# # Note: unlike most functions which deal with configure --flags, this # one does not validate that $key refers to a pre-defined flag. i.e. # it accepts arbitrary keys, even those not defined via an [options] @@ -341,17 +424,19 @@ proc proj-first-bin-of {args} { # that new options set via that function will cause this function to # return true. (That's an unintended and unavoidable side-effect, not # specifically a feature which should be made use of.) +# proc proj-opt-was-provided {key} { dict exists $::autosetup(optset) $key } -######################################################################## +# # @proj-opt-set flag ?val? # # Force-set autosetup option $flag to $val. The value can be fetched # later with [opt-val], [opt-bool], and friends. # # Returns $val. +# proc proj-opt-set {flag {val 1}} { if {$flag ni $::autosetup(options)} { # We have to add this to autosetup(options) or else future calls @@ -362,29 +447,32 @@ proc proj-opt-set {flag {val 1}} { return $val } -######################################################################## +# # @proj-opt-exists flag # # Returns 1 if the given flag has been defined as a legal configure # option, else returns 0. +# proc proj-opt-exists {flag} { expr {$flag in $::autosetup(options)}; } -######################################################################## +# # @proj-val-truthy val # # Returns 1 if $val appears to be a truthy value, else returns # 0. Truthy values are any of {1 on true yes enabled} +# proc proj-val-truthy {val} { expr {$val in {1 on true yes enabled}} } -######################################################################## +# # @proj-opt-truthy flag # # Returns 1 if [opt-val $flag] appears to be a truthy value or # [opt-bool $flag] is true. See proj-val-truthy. +# proc proj-opt-truthy {flag} { if {[proj-val-truthy [opt-val $flag]]} { return 1 } set rc 0 @@ -395,10 +483,11 @@ proc proj-opt-truthy {flag} { return $rc } -######################################################################## +# # @proj-if-opt-truthy boolFlag thenScript ?elseScript? # # If [proj-opt-truthy $flag] is true, eval $then, else eval $else. +# proc proj-if-opt-truthy {boolFlag thenScript {elseScript {}}} { if {[proj-opt-truthy $boolFlag]} { uplevel 1 $thenScript @@ -407,13 +496,14 @@ proc proj-if-opt-truthy {boolFlag thenScript {elseScript {}}} { } } -######################################################################## +# # @proj-define-for-opt flag def ?msg? ?iftrue? ?iffalse? # # If [proj-opt-truthy $flag] then [define $def $iftrue] else [define # $def $iffalse]. If $msg is not empty, output [msg-checking $msg] and # a [msg-results ...] which corresponds to the result. Returns 1 if # the opt-truthy check passes, else 0. +# proc proj-define-for-opt {flag def {msg ""} {iftrue 1} {iffalse 0}} { if {"" ne $msg} { msg-checking "$msg " @@ -436,45 +526,41 @@ proc proj-define-for-opt {flag def {msg ""} {iftrue 1} {iffalse 0}} { return $rc } -######################################################################## +# # @proj-opt-define-bool ?-v? optName defName ?descr? # # Checks [proj-opt-truthy $optName] and calls [define $defName X] -# where X is 0 for false and 1 for true. descr is an optional +# where X is 0 for false and 1 for true. $descr is an optional # [msg-checking] argument which defaults to $defName. Returns X. # # If args[0] is -v then the boolean semantics are inverted: if # the option is set, it gets define'd to 0, else 1. Returns the # define'd value. +# proc proj-opt-define-bool {args} { set invert 0 if {[lindex $args 0] eq "-v"} { - set invert 1 - set args [lrange $args 1 end] + incr invert + lassign $args - optName defName descr + } else { + lassign $args optName defName descr } - set optName [proj-lshift_ args] - set defName [proj-lshift_ args] - set descr [proj-lshift_ args] if {"" eq $descr} { set descr $defName } + #puts "optName=$optName defName=$defName descr=$descr" set rc 0 - msg-checking "$descr ... " - if {[proj-opt-truthy $optName]} { - if {0 eq $invert} { - set rc 1 - } else { - set rc 0 - } - } elseif {0 ne $invert} { - set rc 1 + msg-checking "[join $descr] ... " + set rc [proj-opt-truthy $optName] + if {$invert} { + set rc [expr {!$rc}] } msg-result $rc define $defName $rc return $rc } -######################################################################## +# # @proj-check-module-loader # # Check for module-loading APIs (libdl/libltdl)... @@ -494,6 +580,7 @@ proc proj-opt-define-bool {args} { # # Note that if it finds LIBLTDL it does not look for LIBDL, so will # report only that is has LIBLTDL. +# proc proj-check-module-loader {} { msg-checking "Looking for module-loader APIs... " if {99 ne [get-define LDFLAGS_MODULE_LOADER 99]} { @@ -539,25 +626,27 @@ proc proj-check-module-loader {} { return $rc } -######################################################################## +# # @proj-no-check-module-loader # # Sets all flags which would be set by proj-check-module-loader to # empty/falsy values, as if those checks had failed to find a module # loader. Intended to be called in place of that function when # a module loader is explicitly not desired. +# proc proj-no-check-module-loader {} { define HAVE_LIBDL 0 define HAVE_LIBLTDL 0 define LDFLAGS_MODULE_LOADER "" } -######################################################################## -# @proj-file-conent ?-trim? filename +# +# @proj-file-content ?-trim? filename # # Opens the given file, reads all of its content, and returns it. If # the first arg is -trim, the contents of the file named by the second # argument are trimmed before returning them. +# proc proj-file-content {args} { set trim 0 set fname $args @@ -565,20 +654,21 @@ proc proj-file-content {args} { set trim 1 lassign $args - fname } - set fp [open $fname r] + set fp [open $fname rb] set rc [read $fp] close $fp if {$trim} { return [string trim $rc] } return $rc } -######################################################################## +# # @proj-file-conent filename # # Returns the contents of the given file as an array of lines, with # the EOL stripped from each input line. +# proc proj-file-content-list {fname} { - set fp [open $fname r] + set fp [open $fname rb] set rc {} while { [gets $fp line] >= 0 } { lappend rc $line @@ -587,22 +677,52 @@ proc proj-file-content-list {fname} { return $rc } -######################################################################## +# +# @proj-file-write ?-ro? fname content +# +# Works like autosetup's [writefile] but explicitly uses binary mode +# to avoid EOL translation on Windows. If $fname already exists, it is +# overwritten, even if it's flagged as read-only. +# +proc proj-file-write {args} { + if {"-ro" eq [lindex $args 0]} { + lassign $args ro fname content + } else { + set ro "" + lassign $args fname content + } + file delete -force -- $fname; # in case it's read-only + set f [open $fname wb] + puts -nonewline $f $content + close $f + if {"" ne $ro} { + catch { + exec chmod -w $fname + #file attributes -w $fname; #jimtcl has no 'attributes' + } + } +} + +# # @proj-check-compile-commands ?configFlag? # # Checks the compiler for compile_commands.json support. If passed an # argument it is assumed to be the name of an autosetup boolean config # which controls whether to run/skip this check. # -# Returns 1 if supported, else 0. Defines MAKE_COMPILATION_DB to "yes" -# if supported, "no" if not. +# Returns 1 if supported, else 0, and defines HAVE_COMPILE_COMMANDS to +# that value. Defines MAKE_COMPILATION_DB to "yes" if supported, "no" +# if not. The use of MAKE_COMPILATION_DB is deprecated/discouraged: +# HAVE_COMPILE_COMMANDS is preferred. +# +# ACHTUNG: this test has a long history of false positive results +# because of compilers reacting differently to the -MJ flag. # -# This test has a long history of false positive results because of -# compilers reacting differently to the -MJ flag. proc proj-check-compile-commands {{configFlag {}}} { msg-checking "compile_commands.json support... " if {"" ne $configFlag && ![proj-opt-truthy $configFlag]} { msg-result "explicitly disabled" + define HAVE_COMPILE_COMMANDS 0 define MAKE_COMPILATION_DB no return 0 } else { @@ -611,31 +731,41 @@ proc proj-check-compile-commands {{configFlag {}}} { # Martin G.'s older systems. drh also reports a false # positive on an unspecified older Mac system. msg-result "compiler supports compile_commands.json" - define MAKE_COMPILATION_DB yes + define MAKE_COMPILATION_DB yes; # deprecated + define HAVE_COMPILE_COMMANDS 1 return 1 } else { msg-result "compiler does not support compile_commands.json" define MAKE_COMPILATION_DB no + define HAVE_COMPILE_COMMANDS 0 return 0 } } } -######################################################################## +# # @proj-touch filename # # Runs the 'touch' external command on one or more files, ignoring any # errors. +# proc proj-touch {filename} { catch { exec touch {*}$filename } } -######################################################################## -# @proj-make-from-dot-in ?-touch? filename... # -# Uses [make-template] to create makefile(-like) file(s) $filename -# from $filename.in but explicitly makes the output read-only, to -# avoid inadvertent editing (who, me?). +# @proj-make-from-dot-in ?-touch? infile ?outfile? +# +# Uses [make-template] to create makefile(-like) file(s) $outfile from +# $infile but explicitly makes the output read-only, to avoid +# inadvertent editing (who, me?). +# +# If $outfile is empty then: +# +# - If $infile is a 2-element list, it is assumed to be an in/out pair, +# and $outfile is set from the 2nd entry in that list. Else... +# +# - $outfile is set to $infile stripped of its extension. # # If the first argument is -touch then the generated file is touched # to update its timestamp. This can be used as a workaround for @@ -644,25 +774,45 @@ proc proj-touch {filename} { # please the build process. # # Failures when running chmod or touch are silently ignored. +# proc proj-make-from-dot-in {args} { - set filename $args + set fIn "" + set fOut "" set touch 0 if {[lindex $args 0] eq "-touch"} { set touch 1 - set filename [lrange $args 1 end] - } - foreach f $filename { - set f [string trim $f] - catch { exec chmod u+w $f } - make-template $f.in $f - if {$touch} { - proj-touch $f + lassign $args - fIn fOut + } else { + lassign $args fIn fOut + } + if {"" eq $fOut} { + if {[llength $fIn]>1} { + lassign $fIn fIn fOut + } else { + set fOut [file rootname $fIn] } - catch { exec chmod -w $f } + } + #puts "filenames=$filename" + if {[file exists $fOut]} { + catch { exec chmod u+w $fOut } + } + #puts "making template: $fIn ==> $fOut" + #define-push {top_srcdir} { + #puts "--- $fIn $fOut top_srcdir=[get-define top_srcdir]" + make-template $fIn $fOut + #puts "--- $fIn $fOut top_srcdir=[get-define top_srcdir]" + # make-template modifies top_srcdir + #} + if {$touch} { + proj-touch $fOut + } + catch { + exec chmod -w $fOut + #file attributes -w $f; #jimtcl has no 'attributes' } } -######################################################################## +# # @proj-check-profile-flag ?flagname? # # Checks for the boolean configure option named by $flagname. If set, @@ -675,6 +825,7 @@ proc proj-make-from-dot-in {args} { # order to avoid potential problems with escaping, space-containing # tokens, and interfering with autosetup's use of these vars, this # routine does not directly modify CFLAGS or LDFLAGS. +# proc proj-check-profile-flag {{flagname profile}} { #puts "flagname=$flagname ?[proj-opt-truthy $flagname]?" if {[proj-opt-truthy $flagname]} { @@ -694,7 +845,7 @@ proc proj-check-profile-flag {{flagname profile}} { return 0 } -######################################################################## +# # @proj-looks-like-windows ?key? # # Returns 1 if this appears to be a Windows environment (MinGw, @@ -705,6 +856,7 @@ proc proj-check-profile-flag {{flagname profile}} { # machine, i.e. the local host). If $key == "build" then some # additional checks may be performed which are not applicable when # $key == "host". +# proc proj-looks-like-windows {{key host}} { global autosetup switch -glob -- [get-define $key] { @@ -716,20 +868,21 @@ proc proj-looks-like-windows {{key host}} { # These apply only to the local OS, not a cross-compilation target, # as the above check potentially can. if {$::autosetup(iswin)} { return 1 } - if {[find-an-executable cygpath] ne "" || $::tcl_platform(os)=="Windows NT"} { + if {[find-an-executable cygpath] ne "" || $::tcl_platform(os) eq "Windows NT"} { return 1 } } return 0 } -######################################################################## +# # @proj-looks-like-mac ?key? # # Looks at either the 'host' (==compilation target platform) or # 'build' (==the being-built-on platform) define value and returns if # if that value seems to indicate that it represents a Mac platform, # else returns 0. +# proc proj-looks-like-mac {{key host}} { switch -glob -- [get-define $key] { *apple* { @@ -741,7 +894,7 @@ proc proj-looks-like-mac {{key host}} { } } -######################################################################## +# # @proj-exe-extension # # Checks autosetup's "host" and "build" defines to see if the build @@ -749,6 +902,7 @@ proc proj-looks-like-mac {{key host}} { # build environment is then BUILD_EXEEXT is [define]'d to ".exe", else # "". If the target, a.k.a. "host", is then TARGET_EXEEXT is # [define]'d to ".exe", else "". +# proc proj-exe-extension {} { set rH "" set rB "" @@ -762,7 +916,7 @@ proc proj-exe-extension {} { define TARGET_EXEEXT $rH } -######################################################################## +# # @proj-dll-extension # # Works like proj-exe-extension except that it defines BUILD_DLLEXT @@ -770,6 +924,7 @@ proc proj-exe-extension {} { # # Trivia: for .dylib files, the linker needs the -dynamiclib flag # instead of -shared. +# proc proj-dll-extension {} { set inner {{key} { switch -glob -- [get-define $key] { @@ -788,12 +943,13 @@ proc proj-dll-extension {} { define TARGET_DLLEXT [apply $inner host] } -######################################################################## +# # @proj-lib-extension # # Static-library counterpart of proj-dll-extension. Defines # BUILD_LIBEXT and TARGET_LIBEXT to the conventional static library # extension for the being-built-on resp. the target platform. +# proc proj-lib-extension {} { set inner {{key} { switch -glob -- [get-define $key] { @@ -811,23 +967,25 @@ proc proj-lib-extension {} { define TARGET_LIBEXT [apply $inner host] } -######################################################################## +# # @proj-file-extensions # # Calls all of the proj-*-extension functions. +# proc proj-file-extensions {} { proj-exe-extension proj-dll-extension proj-lib-extension } -######################################################################## +# # @proj-affirm-files-exist ?-v? filename... # # Expects a list of file names. If any one of them does not exist in # the filesystem, it fails fatally with an informative message. # Returns the last file name it checks. If the first argument is -v # then it emits msg-checking/msg-result messages for each file. +# proc proj-affirm-files-exist {args} { set rc "" set verbose 0 @@ -846,7 +1004,7 @@ proc proj-affirm-files-exist {args} { return rc } -######################################################################## +# # @proj-check-emsdk # # Emscripten is used for doing in-tree builds of web-based WASM stuff, @@ -878,6 +1036,7 @@ proc proj-affirm-files-exist {args} { # but BIN_EMCC is then emcc was not found in the EMSDK_HOME, in which # case we have to rely on the fact that sourcing $EMSDK_ENV_SH from a # shell will add emcc to the $PATH. +# proc proj-check-emsdk {} { set emsdkHome [opt-val with-emsdk] define EMSDK_HOME "" @@ -915,7 +1074,7 @@ proc proj-check-emsdk {} { return $rc } -######################################################################## +# # @proj-cc-check-Wl-flag ?flag ?args?? # # Checks whether the given linker flag (and optional arguments) can be @@ -926,6 +1085,7 @@ proc proj-check-emsdk {} { # # If so, that flag string is returned, else an empty string is # returned. +# proc proj-cc-check-Wl-flag {args} { cc-with {-link 1} { # Try -Wl,flag,...args @@ -944,7 +1104,7 @@ proc proj-cc-check-Wl-flag {args} { } } -######################################################################## +# # @proj-check-rpath # # Tries various approaches to handling the -rpath link-time @@ -955,6 +1115,7 @@ proc proj-cc-check-Wl-flag {args} { # --exec-prefix=... or --libdir=... are explicitly passed to # configure then [get-define libdir] is used (noting that it derives # from exec-prefix by default). +# proc proj-check-rpath {} { if {[proj-opt-was-provided libdir] || [proj-opt-was-provided exec-prefix]} { @@ -980,7 +1141,7 @@ proc proj-check-rpath {} { expr {"" ne [get-define LDFLAGS_RPATH]} } -######################################################################## +# # @proj-check-soname ?libname? # # Checks whether CC supports the -Wl,soname,lib... flag. If so, it @@ -993,6 +1154,7 @@ proc proj-check-rpath {} { # LDFLAGS_SONAME_PREFIX. It is provided so that clients may # potentially avoid some end-user confusion by using their own lib's # name here (which shows up in the "checking..." output). +# proc proj-check-soname {{libname "libfoo.so.0"}} { cc-with {-link 1} { if {[cc-check-flags "-Wl,-soname,${libname}"]} { @@ -1005,12 +1167,47 @@ proc proj-check-soname {{libname "libfoo.so.0"}} { } } -######################################################################## +# +# @proj-check-fsanitize ?list-of-opts? +# +# Checks whether CC supports -fsanitize=X, where X is each entry of +# the given list of flags. If any of those flags are supported, it +# returns the string "-fsanitize=X..." where X... is a comma-separated +# list of all flags from the original set which are supported. If none +# of the given options are supported then it returns an empty string. +# +# Example: +# +# set f [proj-check-fsanitize {address bounds-check just-testing}] +# +# Will, on many systems, resolve to "-fsanitize=address,bounds-check", +# but may also resolve to "-fsanitize=address". +# +proc proj-check-fsanitize {{opts {address bounds-strict}}} { + set sup {} + foreach opt $opts { + # -nooutput is used because -fsanitize=hwaddress will otherwise + # pass this test on x86_64, but then warn at build time that + # "hwaddress is not supported for this target". + cc-with {-nooutput 1} { + if {[cc-check-flags "-fsanitize=$opt"]} { + lappend sup $opt + } + } + } + if {[llength $sup] > 0} { + return "-fsanitize=[join $sup ,]" + } + return "" +} + +# # Internal helper for proj-dump-defs-json. Expects to be passed a # [define] name and the variadic $args which are passed to # proj-dump-defs-json. If it finds a pattern match for the given # $name in the various $args, it returns the type flag for that $name, # e.g. "-str" or "-bare", else returns an empty string. +# proc proj-defs-type_ {name spec} { foreach {type patterns} $spec { foreach pattern $patterns { @@ -1022,28 +1219,30 @@ proc proj-defs-type_ {name spec} { return "" } -######################################################################## +# # Internal helper for proj-defs-format_: returns a JSON-ish quoted # form of the given string-type values. It only performs the most # basic of escaping. The input must not contain any control # characters. +# proc proj-quote-str_ {value} { return \"[string map [list \\ \\\\ \" \\\"] $value]\" } -######################################################################## +# # An internal impl detail of proj-dump-defs-json. Requires a data # type specifier, as used by make-config-header, and a value. Returns -# the formatted value or the value $::proj_(defs-skip) if the caller +# the formatted value or the value $::proj__Config(defs-skip) if the caller # should skip emitting that value. -set proj_(defs-skip) "-proj-defs-format_ sentinel" +# +set ::proj__Config(defs-skip) "-proj-defs-format_ sentinel" proc proj-defs-format_ {type value} { switch -exact -- $type { -bare { # Just output the value unchanged } -none { - set value $::proj_(defs-skip) + set value $::proj__Config(defs-skip) } -str { set value [proj-quote-str_ $value] @@ -1058,14 +1257,14 @@ proc proj-defs-format_ {type value} { set ar {} foreach v $value { set v [proj-defs-format_ -auto $v] - if {$::proj_(defs-skip) ne $v} { + if {$::proj__Config(defs-skip) ne $v} { lappend ar $v } } set value "\[ [join $ar {, }] \]" } "" { - set value $::proj_(defs-skip) + set value $::proj__Config(defs-skip) } default { proj-fatal "Unknown type in proj-dump-defs-json: $type" @@ -1074,7 +1273,9 @@ proc proj-defs-format_ {type value} { return $value } -######################################################################## +# +# @proj-dump-defs-json outfile ...flags +# # This function works almost identically to autosetup's # make-config-header but emits its output in JSON form. It is not a # fully-functional JSON emitter, and will emit broken JSON for @@ -1104,6 +1305,7 @@ proc proj-defs-format_ {type value} { # Neither is especially satisfactory (and the second is useless), and # handling of such values is subject to change if any such values ever # _really_ need to be processed by our source trees. +# proc proj-dump-defs-json {file args} { file mkdir [file dirname $file] set lines {} @@ -1111,7 +1313,7 @@ proc proj-dump-defs-json {file args} { foreach n [lsort [dict keys [all-defines]]] { set type [proj-defs-type_ $n $args] set value [proj-defs-format_ $type [get-define $n]] - if {$::proj_(defs-skip) ne $value} { + if {$::proj__Config(defs-skip) ne $value} { lappend lines "\"$n\": ${value}" } } @@ -1122,7 +1324,7 @@ proc proj-dump-defs-json {file args} { } } -######################################################################## +# # @proj-xfer-option-aliases map # # Expects a list of pairs of configure flags which have been @@ -1152,6 +1354,12 @@ proc proj-dump-defs-json {file args} { # over any values from hidden aliases into their canonical names, such # that [opt-value canonical] will return X if --alias=X is passed to # configure. +# +# That said: autosetup's [opt-str] does support alias forms, but it +# requires that the caller know all possible aliases. It's simpler, in +# terms of options handling, if there's only a single canonical name +# which each down-stream call of [opt-...] has to know. +# proc proj-xfer-options-aliases {mapping} { foreach {hidden - canonical} [proj-strip-hash-comments $mapping] { if {[proj-opt-was-provided $hidden]} { @@ -1164,7 +1372,7 @@ proc proj-xfer-options-aliases {mapping} { } } -######################################################################## +# # Arguable/debatable... # # When _not_ cross-compiling and CC_FOR_BUILD is _not_ explicitly @@ -1180,6 +1388,7 @@ proc proj-xfer-options-aliases {mapping} { # Sidebar: if we do this before the cc package is installed, it gets # reverted by that package. Ergo, the cc package init will tell the # user "Build C compiler...cc" shortly before we tell them otherwise. +# proc proj-redefine-cc-for-build {} { if {![proj-is-cross-compiling] && [get-define CC] ne [get-define CC_FOR_BUILD] @@ -1189,13 +1398,14 @@ proc proj-redefine-cc-for-build {} { } } -######################################################################## +# # @proj-which-linenoise headerFile # # Attempts to determine whether the given linenoise header file is of # the "antirez" or "msteveb" flavor. It returns 2 for msteveb, else 1 # (it does not validate that the header otherwise contains the # linenoise API). +# proc proj-which-linenoise {dotH} { set srcHeader [proj-file-content $dotH] if {[string match *userdata* $srcHeader]} { @@ -1205,7 +1415,7 @@ proc proj-which-linenoise {dotH} { } } -######################################################################## +# # @proj-remap-autoconf-dir-vars # # "Re-map" the autoconf-conventional --XYZdir flags into something @@ -1231,17 +1441,18 @@ proc proj-which-linenoise {dotH} { # manner unless they are explicitly overridden at configure-time, in # which case those overrides takes precedence. # -# Each --XYZdir flag which is explicitly passed to configure is -# exported as-is, as are those which default to some top-level system -# directory, e.g. /etc or /var. All which derive from either $prefix -# or $exec_prefix are exported in the form of a Makefile var -# reference, e.g. libdir=${exec_prefix}/lib. Ergo, if +# Each autoconf-relvant --XYZ flag which is explicitly passed to +# configure is exported as-is, as are those which default to some +# top-level system directory, e.g. /etc or /var. All which derive +# from either $prefix or $exec_prefix are exported in the form of a +# Makefile var reference, e.g. libdir=${exec_prefix}/lib. Ergo, if # --exec-prefix=FOO is passed to configure, libdir will still derive, # at make-time, from whatever exec_prefix is passed to make, and will # use FOO if exec_prefix is not overridden at make-time. Without this # post-processing, libdir would be cemented in as FOO/lib at # configure-time, so could be tedious to override properly via a make # invocation. +# proc proj-remap-autoconf-dir-vars {} { set prefix [get-define prefix] set exec_prefix [get-define exec_prefix $prefix] @@ -1270,18 +1481,19 @@ proc proj-remap-autoconf-dir-vars {} { } # Maintenance reminder: the [join] call is to avoid {braces} # around the output when someone passes in, - # e.g. --libdir=\${prefix}/foo/bar. The Debian package build + # e.g. --libdir=\${prefix}/foo/bar. Debian's SQLite package build # script does that. } } -######################################################################## +# # @proj-env-file flag ?default? # # If a file named .env-$flag exists, this function returns a # trimmed copy of its contents, else it returns $dflt. The intended # usage is that things like developer-specific CFLAGS preferences can # be stored in .env-CFLAGS. +# proc proj-env-file {flag {dflt ""}} { set fn ".env-${flag}" if {[file readable $fn]} { @@ -1290,7 +1502,7 @@ proc proj-env-file {flag {dflt ""}} { return $dflt } -######################################################################## +# # @proj-get-env var ?default? # # Extracts the value of "environment" variable $var from the first of @@ -1301,6 +1513,724 @@ proc proj-env-file {flag {dflt ""}} { # - A file named .env-$var (see [proj-env-file]) # # If none of those are set, $dflt is returned. +# proc proj-get-env {var {dflt ""}} { - return [get-env $var [proj-env-file $var $dflt]] + get-env $var [proj-env-file $var $dflt] +} + +# +# @proj-scope ?lvl? +# +# Returns the name of the _calling_ proc from ($lvl + 1) levels up the +# call stack (where the caller's level will be 1 up from _this_ +# call). If $lvl would resolve to global scope "global scope" is +# returned and if it would be negative then a string indicating such +# is returned (as opposed to throwing an error). +# +proc proj-scope {{lvl 0}} { + #uplevel [expr {$lvl + 1}] {lindex [info level 0] 0} + set ilvl [info level] + set offset [expr {$ilvl - $lvl - 1}] + if { $offset < 0} { + return "invalid scope ($offset)" + } elseif { $offset == 0} { + return "global scope" + } else { + return [lindex [info level $offset] 0] + } +} + +# +# Deprecated name of [proj-scope]. +# +proc proj-current-scope {{lvl 0}} { + puts stderr \ + "Deprecated proj-current-scope called from [proj-scope 1]. Use proj-scope instead." + proj-scope [incr lvl] +} + +# +# Converts parts of tclConfig.sh to autosetup [define]s. +# +# Expects to be passed the name of a value tclConfig.sh or an empty +# string. It converts certain parts of that file's contents to +# [define]s (see the code for the whole list). If $tclConfigSh is an +# empty string then it [define]s the various vars as empty strings. +# +proc proj-tclConfig-sh-to-autosetup {tclConfigSh} { + set shBody {} + set tclVars { + TCL_INCLUDE_SPEC + TCL_LIBS + TCL_LIB_SPEC + TCL_STUB_LIB_SPEC + TCL_EXEC_PREFIX + TCL_PREFIX + TCL_VERSION + TCL_MAJOR_VERSION + TCL_MINOR_VERSION + TCL_PACKAGE_PATH + TCL_PATCH_LEVEL + TCL_SHLIB_SUFFIX + } + # Build a small shell script which proxies the $tclVars from + # $tclConfigSh into autosetup code... + lappend shBody "if test x = \"x${tclConfigSh}\"; then" + foreach v $tclVars { + lappend shBody "$v= ;" + } + lappend shBody "else . \"${tclConfigSh}\"; fi" + foreach v $tclVars { + lappend shBody "echo define $v {\$$v} ;" + } + lappend shBody "exit" + set shBody [join $shBody "\n"] + #puts "shBody=$shBody\n"; exit + eval [exec echo $shBody | sh] +} + +# +# @proj-tweak-default-env-dirs +# +# This function is not useful before [use system] is called to set up +# --prefix and friends. It should be called as soon after [use system] +# as feasible. +# +# For certain target environments, if --prefix is _not_ passed in by +# the user, set the prefix to an environment-specific default. For +# such environments its does [define prefix ...] and [proj-opt-set +# prefix ...], but it does not process vars derived from the prefix, +# e.g. exec-prefix. To do so it is generally necessary to also call +# proj-remap-autoconf-dir-vars late in the config process (immediately +# before ".in" files are filtered). +# +# Similar modifications may be made for --mandir. +# +# Returns 1 if it modifies the environment, else 0. +# +proc proj-tweak-default-env-dirs {} { + set rc 0 + switch -glob -- [get-define host] { + *-haiku { + if {![proj-opt-was-provided prefix]} { + set hdir /boot/home/config/non-packaged + proj-opt-set prefix $hdir + define prefix $hdir + incr rc + } + if {![proj-opt-was-provided mandir]} { + set hdir /boot/system/documentation/man + proj-opt-set mandir $hdir + define mandir $hdir + incr rc + } + } + } + return $rc +} + +# +# @proj-dot-ins-append file ?fileOut ?postProcessScript?? +# +# Queues up an autosetup [make-template]-style file to be processed +# at a later time using [proj-dot-ins-process]. +# +# $file is the input file. If $fileOut is empty then this function +# derives $fileOut from $file, stripping both its directory and +# extension parts. i.e. it defaults to writing the output to the +# current directory (typically $::autosetup(builddir)). +# +# If $postProcessScript is not empty then, during +# [proj-dot-ins-process], it will be eval'd immediately after +# processing the file. In the context of that script, the vars +# $dotInsIn and $dotInsOut will be set to the input and output file +# names. This can be used, for example, to make the output file +# executable or perform validation on its contents. +# +# See [proj-dot-ins-process], [proj-dot-ins-list] +# +proc proj-dot-ins-append {fileIn args} { + set srcdir $::autosetup(srcdir) + switch -exact -- [llength $args] { + 0 { + lappend fileIn [file rootname [file tail $fileIn]] "" + } + 1 { + lappend fileIn [join $args] "" + } + 2 { + lappend fileIn {*}$args + } + default { + proj-fatal "Too many arguments: $fileIn $args" + } + } + #puts "******* [proj-scope]: adding $fileIn" + lappend ::proj__Config(dot-in-files) $fileIn +} + +# +# @proj-dot-ins-list +# +# Returns the current list of [proj-dot-ins-append]'d files, noting +# that each entry is a 3-element list of (inputFileName, +# outputFileName, postProcessScript). +# +proc proj-dot-ins-list {} { + return $::proj__Config(dot-in-files) +} + +# +# @proj-dot-ins-process ?-touch? ?-validate? ?-clear? +# +# Each file which has previously been passed to [proj-dot-ins-append] +# is processed, with its passing its in-file out-file names to +# [proj-make-from-dot-in]. +# +# The intent is that a project accumulate any number of files to +# filter and delay their actual filtering until the last stage of the +# configure script, calling this function at that time. +# +# Optional flags: +# +# -touch: gets passed on to [proj-make-from-dot-in] +# +# -validate: after processing each file, before running the file's +# associated script, if any, it runs the file through +# proj-validate-no-unresolved-ats, erroring out if that does. +# +# -clear: after processing, empty the dot-ins list. This effectively +# makes proj-dot-ins-append available for re-use. +# +proc proj-dot-ins-process {args} { + proj-parse-simple-flags args flags { + -touch "" {return "-touch"} + -clear 0 {expr 1} + -validate 0 {expr 1} + } + if {[llength $args] > 0} { + error "Invalid argument to [proj-scope]: $args" + } + foreach f $::proj__Config(dot-in-files) { + proj-assert {3==[llength $f]} \ + "Expecting proj-dot-ins-list to be stored in 3-entry lists" + lassign $f fIn fOut fScript + #puts "DOING $fIn ==> $fOut" + proj-make-from-dot-in {*}$flags(-touch) $fIn $fOut + if {$flags(-validate)} { + proj-validate-no-unresolved-ats $fOut + } + if {"" ne $fScript} { + uplevel 1 [join [list set dotInsIn $fIn \; \ + set dotInsOut $fOut \; \ + eval \{${fScript}\} \; \ + unset dotInsIn dotInsOut]] + } + } + if {$flags(-clear)} { + set ::proj__Config(dot-in-files) [list] + } +} + +# +# @proj-validate-no-unresolved-ats filenames... +# +# For each filename given to it, it validates that the file has no +# unresolved @VAR@ references. If it finds any, it produces an error +# with location information. +# +# Exception: if a filename matches the pattern {*[Mm]ake*} AND a given +# line begins with a # (not including leading whitespace) then that +# line is ignored for purposes of this validation. The intent is that +# @VAR@ inside of makefile comments should not (necessarily) cause +# validation to fail, as it's sometimes convenient to comment out +# sections during development of a configure script and its +# corresponding makefile(s). +# +proc proj-validate-no-unresolved-ats {args} { + foreach f $args { + set lnno 1 + set isMake [string match {*[Mm]ake*} $f] + foreach line [proj-file-content-list $f] { + if {!$isMake || ![string match "#*" [string trimleft $line]]} { + if {[regexp {(@[A-Za-z0-9_]+@)} $line match]} { + error "Unresolved reference to $match at line $lnno of $f" + } + } + incr lnno + } + } +} + +# +# @proj-first-file-found tgtVar fileList +# +# Searches $fileList for an existing file. If one is found, its name +# is assigned to tgtVar and 1 is returned, else tgtVar is set to "" +# and 0 is returned. +# +proc proj-first-file-found {tgtVar fileList} { + upvar $tgtVar tgt + foreach f $fileList { + if {[file exists $f]} { + set tgt $f + return 1 + } + } + set tgt "" + return 0 +} + +# +# Defines $defName to contain makefile recipe commands for re-running +# the configure script with its current set of $::argv flags. This +# can be used to automatically reconfigure. +# +proc proj-setup-autoreconfig {defName} { + define $defName \ + [join [list \ + cd \"$::autosetup(builddir)\" \ + && [get-define AUTOREMAKE "error - missing @AUTOREMAKE@"]]] +} + +# +# @prop-append-to defineName args... +# +# A proxy for Autosetup's [define-append]. Appends all non-empty $args +# to [define-append $defineName]. +# +proc proj-define-append {defineName args} { + foreach a $args { + if {"" ne $a} { + define-append $defineName {*}$a + } + } +} + +# +# @prod-define-amend ?-p|-prepend? ?-d|-define? defineName args... +# +# A proxy for Autosetup's [define-append]. +# +# Appends all non-empty $args to the define named by $defineName. If +# one of (-p | -prepend) are used it instead prepends them, in their +# given order, to $defineName. +# +# If -define is used then each argument is assumed to be a [define]'d +# flag and [get-define X ""] is used to fetch it. +# +# Re. linker flags: typically, -lXYZ flags need to be in "reverse" +# order, with each -lY resolving symbols for -lX's to its left. This +# order is largely historical, and not relevant on all environments, +# but it is technically correct and still relevant on some +# environments. +# +# See: proj-append-to +# +proc proj-define-amend {args} { + set defName "" + set prepend 0 + set isdefs 0 + set xargs [list] + foreach arg $args { + switch -exact -- $arg { + "" {} + -p - -prepend { incr prepend } + -d - -define { incr isdefs } + default { + if {"" eq $defName} { + set defName $arg + } else { + lappend xargs $arg + } + } + } + } + if {"" eq $defName} { + proj-error "Missing defineName argument in call from [proj-scope 1]" + } + if {$isdefs} { + set args $xargs + set xargs [list] + foreach arg $args { + lappend xargs [get-define $arg ""] + } + set args $xargs + } +# puts "**** args=$args" +# puts "**** xargs=$xargs" + + set args $xargs + if {$prepend} { + lappend args {*}[get-define $defName ""] + define $defName [join $args]; # join to eliminate {} entries + } else { + proj-define-append $defName {*}$args + } +} + +# +# @proj-define-to-cflag ?-list? ?-quote? ?-zero-undef? defineName... +# +# Treat each argument as the name of a [define] and renders it like a +# CFLAGS value in one of the following forms: +# +# -D$name +# -D$name=integer (strict integer matches only) +# '-D$name=value' (without -quote) +# '-D$name="value"' (with -quote) +# +# It treats integers as numbers and everything else as a quoted +# string, noting that it does not handle strings which themselves +# contain quotes. +# +# The -zero-undef flag causes no -D to be emitted for integer values +# of 0. +# +# By default it returns the result as string of all -D... flags, +# but if passed the -list flag it will return a list of the +# individual CFLAGS. +# +proc proj-define-to-cflag {args} { + set rv {} + proj-parse-simple-flags args flags { + -list 0 {expr 1} + -quote 0 {expr 1} + -zero-undef 0 {expr 1} + } + foreach d $args { + set v [get-define $d ""] + set li {} + if {"" eq $d} { + set v "-D${d}" + } elseif {[string is integer -strict $v]} { + if {!$flags(-zero-undef) || $v ne "0"} { + set v "-D${d}=$v" + } + } elseif {$flags(-quote)} { + set v "'-D${d}=\"$v\"'" + } else { + set v "'-D${d}=$v'" + } + lappend rv $v + } + expr {$flags(-list) ? $rv : [join $rv]} +} + + +if {0} { + # Turns out that autosetup's [options-add] essentially does exactly + # this... + + # A list of lists of Autosetup [options]-format --flags definitions. + # Append to this using [proj-options-add] and use + # [proj-options-combine] to merge them into a single list for passing + # to [options]. + # + set ::proj__Config(extra-options) {} + + # @proj-options-add list + # + # Adds a list of options to the pending --flag processing. It must be + # in the format used by Autosetup's [options] function. + # + # This will have no useful effect if called from after [options] + # is called. + # + # Use [proj-options-combine] to get a combined list of all added + # options. + # + # PS: when writing this i wasn't aware of autosetup's [options-add], + # works quite similarly. Only the timing is different. + proc proj-options-add {list} { + lappend ::proj__Config(extra-options) $list + } + + # @proj-options-combine list1 ?...listN? + # + # Expects each argument to be a list of options compatible with + # autosetup's [options] function. This function concatenates the + # contents of each list into a new top-level list, stripping the outer + # list part of each argument, and returning that list + # + # If passed no arguments, it uses the list generated by calls to + # [proj-options-add]. + proc proj-options-combine {args} { + set rv [list] + if {0 == [llength $args]} { + set args $::proj__Config(extra-options) + } + foreach e $args { + lappend rv {*}$e + } + return $rv + } +}; # proj-options-* + +# Internal cache for use via proj-cache-*. +array set proj__Cache {} + +# +# @proj-cache-key arg {addLevel 0} +# +# Helper to generate cache keys for [proj-cache-*]. +# +# $addLevel should almost always be 0. +# +# Returns a cache key for the given argument: +# +# integer: relative call stack levels to get the scope name of for +# use as a key. [proj-scope [expr {1 + $arg + addLevel}]] is +# then used to generate the key. i.e. the default of 0 uses the +# calling scope's name as the key. +# +# Anything else: returned as-is +# +proc proj-cache-key {arg {addLevel 0}} { + if {[string is integer -strict $arg]} { + return [proj-scope [expr {$arg + $addLevel + 1}]] + } + return $arg +} + +# +# @proj-cache-set ?-key KEY? ?-level 0? value +# +# Sets a feature-check cache entry with the given key. +# +# See proj-cache-key for -key's and -level's semantics, noting that +# this function adds one to -level for purposes of that call. +proc proj-cache-set {args} { + proj-parse-simple-flags args flags { + -key => 0 + -level => 0 + } + lassign $args val + set key [proj-cache-key $flags(-key) [expr {1 + $flags(-level)}]] + #puts "** fcheck set $key = $val" + set ::proj__Cache($key) $val +} + +# +# @proj-cache-remove ?key? ?addLevel? +# +# Removes an entry from the proj-cache. +proc proj-cache-remove {{key 0} {addLevel 0}} { + set key [proj-cache-key $key [expr {1 + $addLevel}]] + set rv "" + if {[info exists ::proj__Cache($key)]} { + set rv $::proj__Cache($key) + unset ::proj__Cache($key) + } + return $rv; +} + +# +# @proj-cache-check ?-key KEY? ?-level LEVEL? tgtVarName +# +# Checks for a feature-check cache entry with the given key. +# +# If the feature-check cache has a matching entry then this function +# assigns its value to tgtVar and returns 1, else it assigns tgtVar to +# "" and returns 0. +# +# See proj-cache-key for $key's and $addLevel's semantics, noting that +# this function adds one to $addLevel for purposes of that call. +proc proj-cache-check {args} { + proj-parse-simple-flags args flags { + -key => 0 + -level => 0 + } + lassign $args tgtVar + upvar $tgtVar tgt + set rc 0 + set key [proj-cache-key $flags(-key) [expr {1 + $flags(-level)}]] + #puts "** fcheck get key=$key" + if {[info exists ::proj__Cache($key)]} { + set tgt $::proj__Cache($key) + incr rc + } else { + set tgt "" + } + return $rc +} + +# +# @proj-coalesce ...args +# +# Returns the first argument which is not empty (eq ""), or an empty +# string on no match. +proc proj-coalesce {args} { + foreach arg $args { + if {"" ne $arg} { + return $arg + } + } + return "" +} + +# +# @proj-parse-simple-flags ... +# +# A helper to parse flags from proc argument lists. +# +# Expects a list of arguments to parse, an array name to store any +# -flag values to, and a prototype object which declares the flags. +# +# The prototype must be a list in one of the following forms: +# +# -flag defaultValue {script} +# +# -flag => defaultValue +# -----^--^ (with spaces there!) +# +# Repeated for each flag. +# +# The first form represents a basic flag with no associated +# following argument. The second form extracts its value +# from the following argument in $argvName. +# +# The first argument to this function is the name of a var holding the +# args to parse. It will be overwritten, possibly with a smaller list. +# +# The second argument the name of an array variable to create in the +# caller's scope. (Pneumonic: => points to the next argument.) +# +# For the first form of flag, $script is run in the caller's scope if +# $argv contains -flag, and the result of that script is the new value +# for $tgtArrayName(-flag). This function intercepts [return $val] +# from $script. Any empty script will result in the flag having "" +# assigned to it. +# +# The args list is only inspected until the first argument which is +# not described by $prototype. i.e. the first "non-flag" (not counting +# values consumed for flags defined like --flag=>default). +# +# If a "--" flag is encountered, no more arguments are inspected as +# flags. If "--" is the first non-flag argument, the "--" flag is +# removed from the results but all remaining arguments are passed +# through. If "--" appears after the first non-flag, it is retained. +# +# This function assumes that each flag is unique, and using a flag +# more than once behaves in a last-one-wins fashion. +# +# Any argvName entries not described in $prototype are not treated as +# flags. +# +# Returns the number of flags it processed in $argvName. +# +# Example: +# +# set args [list -foo -bar {blah} 8 9 10 -theEnd] +# proj-parse-simple-flags args flags { +# -foo 0 {expr 1} +# -bar => 0 +# -no-baz 2 {return 0} +# } +# +# After that $flags would contain {-foo 1 -bar {blah} -no-baz 2} +# and $args would be {8 9 10 -theEnd}. +# +# Potential TODOs: consider using lappend instead of set so that any +# given flag can be used more than once. Or add a syntax to indicate +# that multiples are allowed. Also consider searching the whole +# argv list, rather than stopping at the first non-flag +# +proc proj-parse-simple-flags {argvName tgtArrayName prototype} { + upvar $argvName argv + upvar $tgtArrayName tgt + array set dflt {} + array set scripts {} + array set consuming {} + set n [llength $prototype] + # Figure out what our flags are... + for {set i 0} {$i < $n} {incr i} { + set k [lindex $prototype $i] + #puts "**** #$i of $n k=$k" + proj-assert {[string match -* $k]} \ + "Invalid flag value: $k" + set v "" + set s "" + switch -exact -- [lindex $prototype [expr {$i + 1}]] { + => { + incr i 2 + if {$i >= $n} { + proj-error "Missing argument for $k => flag" + } + set consuming($k) 1 + set v [lindex $prototype $i] + } + default { + set v [lindex $prototype [incr i]] + set s [lindex $prototype [incr i]] + set scripts($k) $s + } + } + #puts "**** #$i of $n k=$k v=$v s=$s" + set dflt($k) $v + } + # Now look for those flags in the source list + array set tgt [array get dflt] + unset dflt + set rc 0 + set rv {} + set skipMode 0 + set n [llength $argv] + for {set i 0} {$i < $n} {incr i} { + set arg [lindex $argv $i] + if {$skipMode} { + lappend rv $arg + } elseif {"--" eq $arg} { + incr skipMode + } elseif {[info exists tgt($arg)]} { + if {[info exists consuming($arg)]} { + if {$i + 1 >= $n} { + proj-assert 0 {Cannot happen - bounds already checked} + } + set tgt($arg) [lindex $argv [incr i]] + } elseif {"" eq $scripts($arg)} { + set tgt($arg) "" + } else { + #puts "**** running scripts($arg) $scripts($arg)" + set code [catch {uplevel 1 $scripts($arg)} xrc xopt] + #puts "**** tgt($arg)=$scripts($arg) code=$code rc=$rc" + if {$code in {0 2}} { + set tgt($arg) $xrc + } else { + return {*}$xopt $xrc + } + } + incr rc + } else { + incr skipMode + lappend rv $arg + } + } + set argv $rv + return $rc +} + +if {$::proj__Config(self-tests)} { + apply {{} { + #proj-warn "Test code for proj-cache" + proj-assert {![proj-cache-check -key here check]} + proj-assert {"here" eq [proj-cache-key here]} + proj-assert {"" eq $check} + proj-cache-set -key here thevalue + proj-assert {[proj-cache-check -key here check]} + proj-assert {"thevalue" eq $check} + + proj-assert {![proj-cache-check check]} + #puts "*** key = ([proj-cache-key 0])" + proj-assert {"" eq $check} + proj-cache-set abc + proj-assert {[proj-cache-check check]} + proj-assert {"abc" eq $check} + + #parray ::proj__Cache; + proj-assert {"" ne [proj-cache-remove]} + proj-assert {![proj-cache-check check]} + proj-assert {"" eq [proj-cache-remove]} + proj-assert {"" eq $check} + }} } diff --git a/autosetup/sqlite-config.tcl b/autosetup/sqlite-config.tcl index c09b8583f..85fe41438 100644 --- a/autosetup/sqlite-config.tcl +++ b/autosetup/sqlite-config.tcl @@ -1,6 +1,6 @@ # This file holds functions for autosetup which are specific to the # sqlite build tree. They are in this file, instead of auto.def, so -# that they can be reused in the TEA sub-tree. This file requires +# that they can be reused in the autoconf sub-tree. This file requires # functions from proj.tcl. if {[string first " " $autosetup(srcdir)] != -1} { @@ -11,31 +11,43 @@ if {[string first " " $autosetup(builddir)] != -1} { user-error "The pathname of the build directory\ may not contain space characters" } - +#parray ::autosetup; exit 0 use proj -# We want this version info to be emitted up front, but we have to -# 'use system' for --prefix=... to work. Ergo, this bit is up here -# instead of in [sqlite-configure]. +# +# We want the package version info to be emitted early on, but doing +# so requires a bit of juggling. We have to [use system] for +# --prefix=... to work and to emit the Host/Build system info, but we +# don't want those to interfere with --help output. define PACKAGE_VERSION [proj-file-content -trim $::autosetup(srcdir)/VERSION] if {"--help" ni $::argv} { msg-result "Configuring SQLite version [get-define PACKAGE_VERSION]" } use system ; # Will output "Host System" and "Build System" lines if {"--help" ni $::argv} { + proj-tweak-default-env-dirs msg-result "Source dir = $::autosetup(srcdir)" msg-result "Build dir = $::autosetup(builddir)" + use cc cc-db cc-shared cc-lib pkg-config } # -# Object for communicating config-time state across various +# Object for communicating certain config-time state across various # auto.def-related pieces. -# -array set sqliteConfig [proj-strip-hash-comments { +array set sqliteConfig [subst [proj-strip-hash-comments { + # + # Gets set by [sqlite-configure] (the main configure script driver). + build-mode unknown # # Gets set to 1 when using jimsh for code generation. May affect # later decisions. use-jim-for-codegen 0 # + # Set to 1 when cross-compiling This value may be changed by certain + # build options, so it's important that config code which checks for + # cross-compilation uses this var instead of + # [proj-is-cross-compiling]. + is-cross-compiling [proj-is-cross-compiling] + # # Pass msg-debug=1 to configure to enable obnoxiously loud output # from [msg-debug]. msg-debug-enabled 0 @@ -48,29 +60,52 @@ array set sqliteConfig [proj-strip-hash-comments { # (dump-defines-txt) but also a JSON file named after this option's # value. dump-defines-json "" -}] -# -# Set to 1 when cross-compiling This value may be changed by certain -# build options, so it's important that config code which checks for -# cross-compilation uses this var instead of -# [proj-is-cross-compiling]. -# -set sqliteConfig(is-cross-compiling) [proj-is-cross-compiling] + # + # The list of feature --flags which the --all flag implies. This + # requires special handling in a few places. + # + all-flag-enables {fts4 fts5 rtree geopoly session} + + # + # Default value for the --all flag. Can hypothetically be modified + # by non-canonical builds. + # + all-flag-default 0 +}]] ######################################################################## -# Processes all configure --flags for this build $buildMode must be -# either "canonical" or "autoconf", and others may be added in the +# Processes all configure --flags for this build, run build-specific +# config checks, then finalize the configure process. $buildMode must +# be one of (canonical, autoconf), and others may be added in the # future. After bootstrapping, $configScript is eval'd in the caller's # scope, then post-configuration finalization is run. $configScript is # intended to hold configure code which is specific to the given # $buildMode, with the caveat that _some_ build-specific code is # encapsulated in the configuration finalization step. +# +# The intent is that all (or almost all) build-mode-specific +# configuration goes inside the $configScript argument to this +# function, and that an auto.def file contains only two commands: +# +# use sqlite-config +# sqlite-configure BUILD_NAME { build-specific configure script } +# +# There are snippets of build-mode-specific decision-making in +# [sqlite-configure-finalize] proc sqlite-configure {buildMode configScript} { + proj-assert {$::sqliteConfig(build-mode) eq "unknown"} \ + "sqlite-configure must not be called more than once" set allBuildModes {canonical autoconf} if {$buildMode ni $allBuildModes} { user-error "Invalid build mode: $buildMode. Expecting one of: $allBuildModes" } + if {$::sqliteConfig(all-flag-default)} { + set allFlagHelp "Disable these extensions: $::sqliteConfig(all-flag-enables)" + } else { + set allFlagHelp "Enable these extensions: $::sqliteConfig(all-flag-enables)" + } + set ::sqliteConfig(build-mode) $buildMode ######################################################################## # A gentle introduction to flags handling in autosetup @@ -87,6 +122,7 @@ proc sqlite-configure {buildMode configScript} { # boolopt => "a boolean option which defaults to disabled" # boolopt2=1 => "a boolean option which defaults to enabled" # stringopt: => "an option which takes an argument, e.g. --stringopt=value" + # stringopt:DESCR => As for stringopt: with a description for the value # stringopt2:=value => "an option where the argument is optional and defaults to 'value'" # optalias booltopt3 => "a boolean with a hidden alias. --optalias is not shown in --help" # @@ -128,7 +164,7 @@ proc sqlite-configure {buildMode configScript} { ######################################################################## set allFlags { # Structure: a list of M {Z} pairs, where M is a descriptive - # option group name and Z is a list of X Y pairs. X is a list of + # option group name and Z is a list of X Y pairs. X is a list of # $buildMode name(s) to which the Y flags apply, or {*} to apply # to all builds. Y is a {block} in the form expected by # autosetup's [options] command. Each block which is applicable @@ -148,8 +184,8 @@ proc sqlite-configure {buildMode configScript} { # Options for how to build the library build-modes { - {*} { - shared=1 => {Disable build of shared libary} + {canonical autoconf} { + shared=1 => {Disable build of shared library} static=1 => {Disable build of static library} } {canonical} { @@ -162,12 +198,9 @@ proc sqlite-configure {buildMode configScript} { {*} { threadsafe=1 => {Disable mutexing} with-tempstore:=no => {Use an in-RAM database for temporary tables: never,no,yes,always} - largefile=1 - => {This legacy flag has no effect on the library but may influence - the contents of the generated sqlite_cfg.h} - # ^^^ It's not clear that this actually does anything, as - # HAVE_LFS is not checked anywhere in the .c/.h/.in files. load-extension=1 => {Disable loading of external extensions} + # ^^^ one of the downstream custom builds overrides the load-extension default to 0, which + # confuses the --help text generator. https://github.com/msteveb/autosetup/issues/77 math=1 => {Disable math functions} json=1 => {Disable JSON functions} memsys5 => {Enable MEMSYS5} @@ -179,12 +212,22 @@ proc sqlite-configure {buildMode configScript} { geopoly => {Enable the GEOPOLY extension} rtree => {Enable the RTREE extension} session => {Enable the SESSION extension} - all => {Enable FTS4, FTS5, Geopoly, RTree, Sessions} + all=$::sqliteConfig(all-flag-default) => {$allFlagHelp} + largefile=1 + => {This legacy flag has no effect on the library but may influence + the generated sqlite_cfg.h by adding #define HAVE_LFS} } } # Options for TCL support tcl { + {canonical} { + tcl=1 + => {Disable components which require TCL, including all tests. + This tree requires TCL for code generation but can use the in-tree + copy of autosetup/jimsh0.c for that. The SQLite TCL extension and the + test code require a canonical tclsh.} + } {canonical} { with-tcl:DIR => {Directory containing tclConfig.sh or a directory one level up from @@ -196,17 +239,20 @@ proc sqlite-configure {buildMode configScript} { tclConfig.sh and (B) all TCL-based code generation. Warning: if its containing dir has multiple tclsh versions, it may select the wrong tclConfig.sh!} - tcl=1 - => {Disable components which require TCL, including all tests. - This tree requires TCL for code generation but can use the in-tree - copy of autosetup/jimsh0.c for that. The SQLite TCL extension and the - test code require a canonical tclsh.} + } + {canonical} { + static-tclsqlite3=0 + => {Statically-link tclsqlite3. This only works if TCL support is + enabled and all requisite libraries are available in + static form. Note that glibc is unable to fully statically + link certain libraries required by tclsqlite3, so this won't + work on most Linux environments.} } } # Options for line-editing modes for the CLI shell line-editing { - {*} { + {canonical autoconf} { readline=1 => {Disable readline support} # --with-readline-lib is a backwards-compatible alias for @@ -248,15 +294,33 @@ proc sqlite-configure {buildMode configScript} { # Options for exotic/alternative build modes alternative-builds { - {canonical} { - # Potential TODO: add --with-wasi-sdk support to the autoconf - # build + {canonical autoconf} { with-wasi-sdk:=/opt/wasi-sdk => {Top-most dir of the wasi-sdk for a WASI build} + } + + {*} { + # Note that --static-cli-shell has a completely different + # meaning from --static-shell in the autoconf build! + # --[disable-]static-shell is a legacy flag which we can't + # remove without breaking downstream builds. + static-cli-shell=0 + => {Statically-link the sqlite3 CLI shell. + This only works if the requisite libraries are all available in + static form.} + } + + {canonical} { + static-shells=0 + => {Shorthand for --static-cli-shell --static-tclsqlite3} with-emsdk:=auto => {Top-most dir of the Emscripten SDK installation. - Needed only by ext/wasm build. Default=EMSDK env var.} + Needed only by ext/wasm. Default=EMSDK env var.} + + amalgamation-extra-src:FILES + => {Space-separated list of soure files to append as-is to the resulting + sqlite3.c amalgamation file. May be provided multiple times.} } } @@ -264,10 +328,12 @@ proc sqlite-configure {buildMode configScript} { packaging { {autoconf} { # --disable-static-shell: https://sqlite.org/forum/forumpost/cc219ee704 + # Note that this has a different meaning from --static-cli-shell in the + # canonical build! static-shell=1 => {Link the sqlite3 shell app against the DLL instead of embedding sqlite3.c} } - {*} { + {canonical autoconf} { # A potential TODO without a current use case: #rpath=1 => {Disable use of the rpath linker flag} # soname: https://sqlite.org/src/forumpost/5a3b44f510df8ded @@ -281,16 +347,16 @@ proc sqlite-configure {buildMode configScript} { # dll-basename: https://sqlite.org/forum/forumpost/828fdfe904 dll-basename:=auto => {Specifies the base name of the resulting DLL file. - If not provided, libsqlite3 is usually assumed but on some platforms + If not provided, "libsqlite3" is usually assumed but on some platforms a platform-dependent default is used. On some platforms this flag gets automatically enabled if it is not provided. Use "default" to explicitly disable platform-dependent activation on such systems.} # out-implib: https://sqlite.org/forum/forumpost/0c7fc097b2 out-implib:=auto => {Enable use of --out-implib linker flag to generate an - "import library" for the DLL. The output's base name name is - specified by the value, with "auto" meaning to figure out a - name automatically. On some platforms this flag gets + "import library" for the DLL. The output's base name is + specified by this flag's value, with "auto" meaning to figure + out a name automatically. On some platforms this flag gets automatically enabled if it is not provided. Use "none" to explicitly disable this feature on such platforms.} } @@ -323,6 +389,10 @@ proc sqlite-configure {buildMode configScript} { => {Enable #line macros in the amalgamation} dynlink-tools => {Dynamically link libsqlite3 to certain tools which normally statically embed it} + asan-fsanitize:=auto + => {Comma- or space-separated list of -fsanitize flags for use with the + fuzzcheck-asan tool. Only those which the compiler claims to support + will actually be used. May be provided multiple times.} } {*} { dump-defines=0 @@ -330,47 +400,80 @@ proc sqlite-configure {buildMode configScript} { (for build debugging)} } } - }; # $allOpts + }; # $allFlags + + set allFlags [proj-strip-hash-comments $allFlags] + # ^^^ lappend of [sqlite-custom-flags] introduces weirdness if + # we delay [proj-strip-hash-comments] until after that. + + + ######################################################################## + # sqlite-custom.tcl is intended only for vendor-branch-specific + # customization. See autosetup/README.md#branch-customization for + # details. + if {[file exists $::autosetup(libdir)/sqlite-custom.tcl]} { + uplevel 1 {source $::autosetup(libdir)/sqlite-custom.tcl} + } + + if {[llength [info proc sqlite-custom-flags]] > 0} { + # sqlite-custom-flags is assumed to be imported via + # autosetup/sqlite-custom.tcl. + set scf [sqlite-custom-flags] + if {"" ne $scf} { + lappend allFlags sqlite-custom-flags $scf + } + } - # Filter allOpts to create the set of [options] legal for this build - set opts {} - foreach {group XY} [subst -nobackslashes -nocommands \ - [proj-strip-hash-comments $allFlags]] { + # Filter allFlags to create the set of [options] legal for this build + foreach {group XY} [subst -nobackslashes -nocommands $allFlags] { foreach {X Y} $XY { if { $buildMode in $X || "*" in $X } { - foreach y $Y { - lappend opts $y - } + options-add $Y } } } #lappend opts "soname:=duplicateEntry => {x}"; #just testing - if {[catch {options $opts} msg xopts]} { + if {[catch {options {}} msg xopts]} { # Workaround for # where [options] behaves oddly on _some_ TCL builds when it's # called from deeper than the global scope. dict incr xopts -level return {*}$xopts $msg } - # The following uplevel is largely cosmetic, the intent being to put - # the most-frequently-useful info at the top of the ./configure - # output, but also avoiding outputing it if --help is used. - uplevel 1 { - use cc cc-db cc-shared cc-lib pkg-config - } - sqlite-post-options-init + sqlite-configure-phase1 $buildMode uplevel 1 $configScript sqlite-configure-finalize }; # sqlite-configure ######################################################################## -# Performs late-stage config steps common to both the canonical and -# autoconf bundle builds. -proc sqlite-configure-finalize {} { - set buildMode $::sqliteConfig(build-mode) - set isCanonical [expr {$buildMode eq "canonical"}] - set isAutoconf [expr {$buildMode eq "autoconf"}] - +# Runs "phase 1" of the configure process: after initial --flags +# handling but before the build-specific parts are run. $buildMode +# must be the mode which was passed to [sqlite-configure]. +proc sqlite-configure-phase1 {buildMode} { + define PACKAGE_NAME sqlite + define PACKAGE_URL {https://sqlite.org} + define PACKAGE_BUGREPORT [get-define PACKAGE_URL]/forum + define PACKAGE_STRING "[get-define PACKAGE_NAME] [get-define PACKAGE_VERSION]" + proj-xfer-options-aliases { + # Carry values from hidden --flag aliases over to their canonical + # flag forms. This list must include only options which are common + # to all build modes supported by [sqlite-configure]. + with-readline-inc => with-readline-cflags + with-readline-lib => with-readline-ldflags + with-debug => debug + } + set ::sqliteConfig(msg-debug-enabled) [proj-val-truthy [get-env msg-debug 0]] + proc-debug "msg-debug is enabled" + proj-setup-autoreconfig SQLITE_AUTORECONFIG + proj-file-extensions + if {".exe" eq [get-define TARGET_EXEEXT]} { + define SQLITE_OS_UNIX 0 + define SQLITE_OS_WIN 1 + } else { + define SQLITE_OS_UNIX 1 + define SQLITE_OS_WIN 0 + } + sqlite-setup-default-cflags define HAVE_LFS 0 if {[opt-bool largefile]} { # @@ -380,79 +483,46 @@ proc sqlite-configure-finalize {} { # harmless, but it doesn't do anything useful. It does have # visible side-effects, though: the generated sqlite_cfg.h may (or # may not) define HAVE_LFS. - # cc-check-lfs } - - if {$isCanonical} { - if {![opt-bool static]} { - proj-indented-notice { - NOTICE: static lib build may be implicitly re-activated by - other components, e.g. some test apps. - } - } - } else { - proj-assert { $isAutoconf } "Invalid build mode" - proj-define-for-opt static-shell ENABLE_STATIC_SHELL \ - "Link library statically into the CLI shell?" - if {![opt-bool shared] && ![opt-bool static-shell]} { - proj-opt-set shared 1 - proj-indented-notice { - NOTICE: ignoring --disable-shared because --disable-static-shell - was specified. - } - } + set srcdir $::autosetup(srcdir) + proj-dot-ins-append $srcdir/Makefile.in + if {[file exists $srcdir/sqlite3.pc.in]} { + proj-dot-ins-append $srcdir/sqlite3.pc.in } - proj-define-for-opt shared ENABLE_LIB_SHARED "Build shared library?" - proj-define-for-opt static ENABLE_LIB_STATIC "Build static library?" +}; # sqlite-configure-phase1 - sqlite-handle-debug +######################################################################## +# Performs late-stage config steps common to all supported +# $::sqliteConfig(build-mode) values. +proc sqlite-configure-finalize {} { sqlite-handle-rpath sqlite-handle-soname sqlite-handle-threadsafe sqlite-handle-tempstore - sqlite-handle-line-editing sqlite-handle-load-extension sqlite-handle-math sqlite-handle-icu - sqlite-handle-env-quirks - sqlite-process-dot-in-files - sqlite-post-config-validation - sqlite-dump-defines -}; # sqlite-configure-finalize - -######################################################################## -# Runs some common initialization which must happen immediately after -# autosetup's [options] function is called. This is also a convenient -# place to put some generic pieces common to both the canonical -# top-level build and the "autoconf" build, but it's not intended to -# be a catch-all dumping ground for such. -proc sqlite-post-options-init {} { - define PACKAGE_NAME "sqlite" - define PACKAGE_URL {https://sqlite.org} - define PACKAGE_BUGREPORT [get-define PACKAGE_URL]/forum - define PACKAGE_STRING "[get-define PACKAGE_NAME] [get-define PACKAGE_VERSION]" - # - # Carry values from hidden --flag aliases over to their canonical - # flag forms. This list must include only options which are common - # to both the top-level auto.def and autoconf/auto.def. - # - proj-xfer-options-aliases { - with-readline-inc => with-readline-cflags - with-readline-lib => with-readline-ldflags - with-debug => debug + if {[proj-opt-exists readline]} { + sqlite-handle-line-editing } - sqlite-autoreconfig - proj-file-extensions - if {".exe" eq [get-define TARGET_EXEEXT]} { - define SQLITE_OS_UNIX 0 - define SQLITE_OS_WIN 1 - } else { - define SQLITE_OS_UNIX 1 - define SQLITE_OS_WIN 0 + if {[proj-opt-exists shared]} { + proj-define-for-opt shared ENABLE_LIB_SHARED "Build shared library?" } - set ::sqliteConfig(msg-debug-enabled) [proj-val-truthy [get-env msg-debug 0]] - sqlite-setup-default-cflags + if {[proj-opt-exists static]} { + if {![proj-define-for-opt static ENABLE_LIB_STATIC "Build static library?"]} { + # This notice really only applies to the canonical build... + proj-indented-notice { + NOTICE: static lib build may be implicitly re-activated by + other components, e.g. some test apps. + } + } + } + sqlite-handle-env-quirks + sqlite-handle-common-feature-flags + sqlite-finalize-feature-flags + sqlite-process-dot-in-files; # do not [define] anything after this + sqlite-dump-defines } ######################################################################## @@ -463,28 +533,12 @@ proc msg-debug {msg} { puts stderr [proj-bold "** DEBUG: $msg"] } } - ######################################################################## -# Sets up the SQLITE_AUTORECONFIG define. -proc sqlite-autoreconfig {} { - # - # SQLITE_AUTORECONFIG contains make target rules for re-running the - # configure script with the same arguments it was initially invoked - # with. This can be used to automatically reconfigure - # - set squote {{arg} { - # Wrap $arg in single-quotes if it looks like it might need that - # to avoid mis-handling as a shell argument. We assume that $arg - # will never contain any single-quote characters. - if {[string match {*[ &;$*"]*} $arg]} { return '$arg' } - return $arg - }} - define-append SQLITE_AUTORECONFIG cd [apply $squote $::autosetup(builddir)] \ - && [apply $squote $::autosetup(srcdir)/configure] - #{*}$::autosetup(argv) breaks with --flag='val with spaces', so... - foreach arg $::autosetup(argv) { - define-append SQLITE_AUTORECONFIG [apply $squote $arg] - } +# A [msg-debug] proxy which prepends the name of the current proc to +# the debug message. It is not legal to call this from the global +# scope. +proc proc-debug {msg} { + msg-debug "\[[proj-scope 1]\]: $msg" } define OPT_FEATURE_FLAGS {} ; # -DSQLITE_OMIT/ENABLE flags. @@ -493,7 +547,8 @@ define OPT_SHELL {} ; # Feature-related CFLAGS for the sqlite3 CLI app # Adds $args, if not empty, to OPT_FEATURE_FLAGS. If the first arg is # -shell then it strips that arg and passes the remaining args the # sqlite-add-shell-opt in addition to adding them to -# OPT_FEATURE_FLAGS. +# OPT_FEATURE_FLAGS. This is intended only for holding +# -DSQLITE_ENABLE/OMIT/... flags, but that is not enforced here. proc sqlite-add-feature-flag {args} { set shell "" if {"-shell" eq [lindex $args 0]} { @@ -506,6 +561,8 @@ proc sqlite-add-feature-flag {args} { define-append OPT_FEATURE_FLAGS {*}$args } } + +######################################################################## # Appends $args, if not empty, to OPT_SHELL. proc sqlite-add-shell-opt {args} { if {"" ne $args} { @@ -524,8 +581,12 @@ proc sqlite-affirm-have-math {featureName} { if {![msg-quiet proj-check-function-in-lib log m]} { user-error "Missing math APIs for $featureName" } - define LDFLAGS_MATH [get-define lib_log ""] + set lfl [get-define lib_log ""] undefine lib_log + if {"" ne $lfl} { + user-notice "Forcing requirement of $lfl for $featureName" + } + define LDFLAGS_MATH $lfl } } @@ -543,29 +604,36 @@ proc sqlite-check-common-bins {} { ######################################################################## # Run checks for system-level includes and libs which are common to # both the canonical build and the "autoconf" bundle. +# +# For the canonical build this must come after +# [sqlite-handle-wasi-sdk], as that function may change the +# environment in ways which affect this. proc sqlite-check-common-system-deps {} { - # # Check for needed/wanted data types cc-with {-includes stdint.h} \ {cc-check-types int8_t int16_t int32_t int64_t intptr_t \ uint8_t uint16_t uint32_t uint64_t uintptr_t} - # # Check for needed/wanted functions cc-check-functions gmtime_r isnan localtime_r localtime_s \ - malloc_usable_size strchrnul usleep utime pread pread64 pwrite pwrite64 - - set ldrt "" - # Collapse funcs from librt into LDFLAGS_RT. - # Some systems (ex: SunOS) require -lrt in order to use nanosleep - foreach func {fdatasync nanosleep} { - if {[proj-check-function-in-lib $func rt]} { - lappend ldrt [get-define lib_${func}] + strchrnul usleep utime pread pread64 pwrite pwrite64 + + apply {{} { + set ldrt "" + # Collapse funcs from librt into LDFLAGS_RT. + # Some systems (ex: SunOS) require -lrt in order to use nanosleep + foreach func {fdatasync nanosleep} { + if {[proj-check-function-in-lib $func rt]} { + set ldrt [get-define lib_${func} ""] + undefine lib_${func} + if {"" ne $ldrt} { + break + } + } } - } - define LDFLAGS_RT [join [lsort -unique $ldrt] ""] + define LDFLAGS_RT $ldrt + }} - # # Check for needed/wanted headers cc-check-includes \ sys/types.h sys/stat.h dlfcn.h unistd.h \ @@ -584,27 +652,11 @@ proc sqlite-check-common-system-deps {} { } } -proc sqlite-setup-default-cflags {} { - ######################################################################## - # We differentiate between two C compilers: the one used for binaries - # which are to run on the build system (in autosetup it's called - # CC_FOR_BUILD and in Makefile.in it's $(B.cc)) and the one used for - # compiling binaries for the target system (CC a.k.a. $(T.cc)). - # Normally they're the same, but they will differ when - # cross-compiling. - # - # When cross-compiling we default to not using the -g flag, based on a - # /chat discussion prompted by - # https://sqlite.org/forum/forumpost/9a67df63eda9925c - set defaultCFlags {-O2} - if {!$::sqliteConfig(is-cross-compiling)} { - lappend defaultCFlags -g - } - define CFLAGS [proj-get-env CFLAGS $defaultCFlags] - # BUILD_CFLAGS is the CFLAGS for CC_FOR_BUILD. - define BUILD_CFLAGS [proj-get-env BUILD_CFLAGS {-g}] - - # Copy all CFLAGS and CPPFLAGS entries matching -DSQLITE_OMIT* and +######################################################################## +# Move -DSQLITE_OMIT... and -DSQLITE_ENABLE... flags from CFLAGS and +# CPPFLAGS to OPT_FEATURE_FLAGS and remove them from BUILD_CFLAGS. +proc sqlite-munge-cflags {} { + # Move CFLAGS and CPPFLAGS entries matching -DSQLITE_OMIT* and # -DSQLITE_ENABLE* to OPT_FEATURE_FLAGS. This behavior is derived # from the legacy build and was missing the 3.48.0 release (the # initial Autosetup port). @@ -647,18 +699,37 @@ proc sqlite-setup-default-cflags {} { define BUILD_CFLAGS $tmp } +######################################################################### +# Set up the default CFLAGS and BUILD_CFLAGS values. +proc sqlite-setup-default-cflags {} { + ######################################################################## + # We differentiate between two C compilers: the one used for binaries + # which are to run on the build system (in autosetup it's called + # CC_FOR_BUILD and in Makefile.in it's $(B.cc)) and the one used for + # compiling binaries for the target system (CC a.k.a. $(T.cc)). + # Normally they're the same, but they will differ when + # cross-compiling. + # + # When cross-compiling we default to not using the -g flag, based on a + # /chat discussion prompted by + # https://sqlite.org/forum/forumpost/9a67df63eda9925c + set defaultCFlags {-O2} + if {!$::sqliteConfig(is-cross-compiling)} { + lappend defaultCFlags -g + } + define CFLAGS [proj-get-env CFLAGS $defaultCFlags] + # BUILD_CFLAGS is the CFLAGS for CC_FOR_BUILD. + define BUILD_CFLAGS [proj-get-env BUILD_CFLAGS {-g}] + sqlite-munge-cflags +} + ######################################################################## -# Handle various SQLITE_ENABLE_... feature flags. +# Handle various SQLITE_ENABLE/OMIT_... feature flags. proc sqlite-handle-common-feature-flags {} { msg-result "Feature flags..." - if {"tcl-extension" eq $::sqliteConfig(build-mode)} { - set allFlagEnables {fts3 fts4 fts5 rtree geopoly} - } else { - set allFlagEnables {fts4 fts5 rtree rtree geopoly session} - } if {![opt-bool all]} { # Special handling for --disable-all - foreach flag $allFlagEnables { + foreach flag $::sqliteConfig(all-flag-enables) { if {![proj-opt-was-provided $flag]} { proj-opt-set $flag 0 } @@ -679,7 +750,7 @@ proc sqlite-handle-common-feature-flags {} { # The --geopoly flag, though, will automatically re-enable # --rtree, so --disable-rtree won't actually disable anything in # that case. - foreach k $allFlagEnables { + foreach k $::sqliteConfig(all-flag-enables) { if {![proj-opt-was-provided $k]} { proj-opt-set $k 1 } @@ -734,7 +805,6 @@ proc sqlite-handle-common-feature-flags {} { msg-result " - $boolFlag" } } - } ######################################################################### @@ -751,17 +821,24 @@ proc sqlite-finalize-feature-flags {} { define OPT_SHELL [lsort -unique $oFF] msg-result "Shell options: [get-define OPT_SHELL]" } + if {"" ne [set extraSrc [get-define AMALGAMATION_EXTRA_SRC ""]]} { + proj-assert {"canonical" eq $::sqliteConfig(build-mode)} + msg-result "Appending source files to amalgamation: $extraSrc" + } + if {[lsearch [get-define TARGET_DEBUG ""] -DSQLITE_DEBUG=1] > -1} { + msg-result "Note: this is a debug build, so performance will suffer." + } } ######################################################################## -# Checks for the --debug flag, defining SQLITE_DEBUG to 1 if it is -# true. TARGET_DEBUG gets defined either way, with content depending -# on whether --debug is true or false. +# Checks for the --debug flag and [define]s TARGET_DEBUG based on +# that. TARGET_DEBUG is unused in the autoconf build but that is +# arguably a bug. proc sqlite-handle-debug {} { msg-checking "SQLITE_DEBUG build? " proj-if-opt-truthy debug { - define SQLITE_DEBUG 1 - define TARGET_DEBUG {-g -DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0 -Wall} + define TARGET_DEBUG {-g -DSQLITE_DEBUG=1 -O0 -Wall} + sqlite-add-feature-flag -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE proj-opt-set memsys5 msg-result yes } { @@ -796,7 +873,7 @@ proc sqlite-handle-soname {} { } } } - msg-debug "soname=$soname" + proc-debug "soname=$soname" if {[proj-check-soname $soname]} { define LDFLAGS_LIBSQLITE3_SONAME [get-define LDFLAGS_SONAME_PREFIX]$soname msg-result "Setting SONAME using: [get-define LDFLAGS_LIBSQLITE3_SONAME]" @@ -916,8 +993,10 @@ proc sqlite-handle-emsdk {} { # Maybe there's a copy in the path? proj-bin-define wasm-opt BIN_WASM_OPT } - proj-make-from-dot-in $emccSh $extWasmConfig - catch {exec chmod u+x $emccSh} + proj-dot-ins-append $emccSh.in $emccSh { + catch {exec chmod u+x $dotInsOut} + } + proj-dot-ins-append $extWasmConfig.in $extWasmConfig } else { define EMCC_WRAPPER "" file delete -force -- $emccSh $extWasmConfig @@ -927,6 +1006,11 @@ proc sqlite-handle-emsdk {} { ######################################################################## # Internal helper for [sqlite-check-line-editing]. Returns a list of # potential locations under which readline.h might be found. +# +# On some environments this function may perform extra work to help +# sqlite-check-line-editing figure out how to find libreadline and +# friends. It will communicate those results via means other than the +# result value, e.g. by modifying configure --flags. proc sqlite-get-readline-dir-list {} { # Historical note: the dirs list, except for the inclusion of # $prefix and some platform-specific dirs, originates from the @@ -943,13 +1027,24 @@ proc sqlite-get-readline-dir-list {} { *-mingw64 { lappend dirs /mingw64 /mingw } + *-haiku { + lappend dirs /boot/system/develop/headers + if {[opt-val with-readline-ldflags] in {auto ""}} { + # If the user did not supply their own --with-readline-ldflags + # value, hijack that flag to inject options which are known to + # work on a default Haiku installation. + if {"" ne [glob -nocomplain /boot/system/lib/libreadline*]} { + proj-opt-set with-readline-ldflags {-L/boot/system/lib -lreadline} + } + } + } } lappend dirs /usr /usr/local /usr/local/readline /usr/contrib set rv {} foreach d $dirs { if {[file isdir $d]} {lappend rv $d} } - #msg-debug "sqlite-get-readline-dir-list dirs=$rv" + #proc-debug "dirs=$rv" return $rv } @@ -1090,7 +1185,9 @@ proc sqlite-check-line-editing {} { proj-warn "Skipping check for readline.h because we're cross-compiling." } else { set dirs [sqlite-get-readline-dir-list] - set subdirs "include/$editLibName" + set subdirs [list \ + include/$editLibName \ + readline] if {"editline" eq $editLibName} { lappend subdirs include/readline # ^^^ editline, on some systems, does not have its own header, @@ -1098,7 +1195,8 @@ proc sqlite-check-line-editing {} { } lappend subdirs include set rlInc [proj-search-for-header-dir readline.h \ - -dirs $dirs -subdirs $subdirs] + -dirs $dirs -subdirs $subdirs] + #proc-debug "rlInc=$rlInc" if {"" ne $rlInc} { if {[string match */readline $rlInc]} { set rlInc [file dirname $rlInc]; # CLI shell: #include @@ -1123,7 +1221,8 @@ proc sqlite-check-line-editing {} { set rlLib "" if {"" ne $rlInc} { set rlLib [opt-val with-readline-ldflags] - if {$rlLib eq "auto" || $rlLib eq ""} { + #proc-debug "rlLib=$rlLib" + if {$rlLib in {auto ""}} { set rlLib "" set libTerm "" if {[proj-check-function-in-lib tgetent "$editLibName ncurses curses termcap"]} { @@ -1191,9 +1290,10 @@ proc sqlite-check-line-editing {} { }; # sqlite-check-line-editing ######################################################################## -# Runs sqlite-check-line-editing and adds a message around it In the +# Runs sqlite-check-line-editing and adds a message around it. In the # canonical build this must not be called before -# sqlite-determine-codegen-tcl. +# sqlite-determine-codegen-tcl for reasons now lost to history (and +# might not still be applicable). proc sqlite-handle-line-editing {} { msg-result "Line-editing support for the sqlite3 shell: [sqlite-check-line-editing]" } @@ -1233,7 +1333,7 @@ proc sqlite-handle-icu {} { msg-result "Checking for ICU support..." set icuConfigBin [opt-val with-icu-config] set tryIcuConfigBin 1; # set to 0 if we end up using pkg-config - if {"auto" eq $icuConfigBin || "pkg-config" eq $icuConfigBin} { + if {$icuConfigBin in {auto pkg-config}} { if {[pkg-config-init 0] && [pkg-config icu-io]} { # Maintenance reminder: historical docs say to use both of # (icu-io, icu-uc). icu-uc lacks a required lib and icu-io has @@ -1287,7 +1387,9 @@ proc sqlite-handle-icu {} { if {[opt-bool icu-collations]} { msg-result "Enabling ICU collations." sqlite-add-feature-flag -shell -DSQLITE_ENABLE_ICU_COLLATIONS - # Recall that shell.c builds with sqlite3.c + # Recall that shell.c builds with sqlite3.c except in the case + # of --disable-static-shell, a combination we do not + # specifically attempt to account for. } } elseif {[opt-bool icu-collations]} { proj-warn "ignoring --enable-icu-collations because neither --with-icu-ldflags nor --with-icu-config provided any linker flags" @@ -1301,7 +1403,7 @@ proc sqlite-handle-icu {} { # Handles the --enable-load-extension flag. Returns 1 if the support # is enabled, else 0. If support for that feature is not found, a # fatal error is triggered if --enable-load-extension is explicitly -# provided, else a loud warning is instead emited. If +# provided, else a loud warning is instead emitted. If # --disable-load-extension is used, no check is performed. # # Makes the following environment changes: @@ -1341,8 +1443,8 @@ proc sqlite-handle-load-extension {} { if {$found} { msg-result "Loadable extension support enabled." } else { - msg-result "Disabling loadable extension support. Use --enable-load-extensions to enable them." - sqlite-add-feature-flag {-DSQLITE_OMIT_LOAD_EXTENSION=1} + msg-result "Disabling loadable extension support. Use --enable-load-extension to enable them." + sqlite-add-feature-flag -DSQLITE_OMIT_LOAD_EXTENSION=1 } return $found } @@ -1356,8 +1458,8 @@ proc sqlite-handle-math {} { } define LDFLAGS_MATH [get-define lib_ceil] undefine lib_ceil - sqlite-add-feature-flag {-DSQLITE_ENABLE_MATH_FUNCTIONS} - msg-result "Enabling math SQL functions [get-define LDFLAGS_MATH]" + sqlite-add-feature-flag -DSQLITE_ENABLE_MATH_FUNCTIONS + msg-result "Enabling math SQL functions" } { define LDFLAGS_MATH "" msg-result "Disabling math SQL functions" @@ -1398,7 +1500,7 @@ proc sqlite-handle-mac-cversion {} { ######################################################################## # If this is a Mac platform, check for support for -# -Wl,-install_name,... and, if it's set, define +# -Wl,-install_name,... and, if it's available, define # LDFLAGS_MAC_INSTALL_NAME to a variant of that string which is # intended to expand at make-time, else set LDFLAGS_MAC_INSTALL_NAME # to an empty string. @@ -1421,7 +1523,8 @@ proc sqlite-handle-mac-install-name {} { ######################################################################## # Handles the --dll-basename configure flag. [define]'s # SQLITE_DLL_BASENAME to the DLL's preferred base name (minus -# extension). If --dll-basename is not provided then this is always +# extension). If --dll-basename is not provided (or programmatically +# set - see [sqlite-handle-env-quirks]) then this is always # "libsqlite3", otherwise it may use a different value based on the # value of [get-define host]. proc sqlite-handle-dll-basename {} { @@ -1446,13 +1549,15 @@ proc sqlite-handle-dll-basename {} { # [define]s LDFLAGS_OUT_IMPLIB to either an empty string or to a # -Wl,... flag for the platform-specific --out-implib flag, which is # used for building an "import library .dll.a" file on some platforms -# (e.g. msys2, mingw). Returns 1 if supported, else 0. +# (e.g. msys2, mingw). SQLITE_OUT_IMPLIB is defined to the name of the +# import lib or an empty string. Returns 1 if supported, else 0. # # The name of the import library is [define]d in SQLITE_OUT_IMPLIB. # -# If the configure flag --out-implib is not used then this is a no-op. -# If that flag is used but the capability is not available, a fatal -# error is triggered. +# If the configure flag --out-implib is not used (or programmatically +# set) then this simply sets the above-listed defines to empty strings +# (but see [sqlite-handle-env-quirks]). If that flag is used but the +# capability is not available, a fatal error is triggered. # # This feature is specifically opt-in because it's supported on far # more platforms than actually need it and enabling it causes creation @@ -1466,7 +1571,7 @@ proc sqlite-handle-dll-basename {} { # # - msys2 and mingw sqlite packages historically install # /usr/lib/libsqlite3.dll.a despite the DLL being in -# /usr/bin/msys-sqlite3-0.dll. +# /usr/bin. proc sqlite-handle-out-implib {} { define LDFLAGS_OUT_IMPLIB "" define SQLITE_OUT_IMPLIB "" @@ -1505,7 +1610,8 @@ proc sqlite-handle-out-implib {} { # # It does not distinguish between msys and msys2, returning msys for # both. The build does not, as of this writing, specifically support -# msys v1. +# msys v1. Similarly, this function returns "mingw" for both "mingw32" +# and "mingw64". proc sqlite-env-is-unix-on-windows {{envTuple ""}} { if {"" eq $envTuple} { set envTuple [get-define host] @@ -1516,7 +1622,7 @@ proc sqlite-env-is-unix-on-windows {{envTuple ""}} { *-*-ming* { set name mingw } *-*-msys { set name msys } } - return $name; + return $name } ######################################################################## @@ -1528,7 +1634,7 @@ proc sqlite-env-is-unix-on-windows {{envTuple ""}} { # # [define]s SQLITE_DLL_INSTALL_RULES to a symbolic name suffix for a # set of "make install" rules to use for installation of the DLL -# deliverable. The makefile is tasked with with providing rules named +# deliverable. The makefile is tasked with providing rules named # install-dll-NAME which runs the installation for that set, as well # as providing a rule named install-dll which resolves to # install-dll-NAME (perhaps indirectly, depending on whether the DLL @@ -1540,13 +1646,13 @@ proc sqlite-env-is-unix-on-windows {{envTuple ""}} { # # On platforms where an "import library" is conventionally used but # --out-implib was not explicitly used, automatically add that flag. -# This conventionally applies to the "Unix on Windows" environments -# like msys and cygwin. +# This conventionally applies only to the "Unix on Windows" +# environments like msys and cygwin. # # 3) --dll-basename: # # On the same platforms addressed by --out-implib, if --dll-basename -# is not specified, --dll-basename=auto is implied. +# is not explicitly specified, --dll-basename=auto is implied. proc sqlite-handle-env-quirks {} { set instName unix-generic; # name of installation rules set set autoDll 0; # true if --out-implib/--dll-basename should be implied @@ -1587,24 +1693,17 @@ proc sqlite-handle-env-quirks {} { sqlite-handle-out-implib sqlite-handle-mac-cversion sqlite-handle-mac-install-name + if {[llength [info proc sqlite-custom-handle-flags]] > 0} { + # sqlite-custom-handle-flags is assumed to be imported via a + # client-specific import: autosetup/sqlite-custom.tcl. + sqlite-custom-handle-flags + } } ######################################################################## # Perform some late-stage work and generate the configure-process # output file(s). proc sqlite-process-dot-in-files {} { - ######################################################################## - # When cross-compiling, we have to avoid using the -s flag to - # /usr/bin/install: - # https://sqlite.org/forum/forumpost/9a67df63eda9925c - define IS_CROSS_COMPILING $::sqliteConfig(is-cross-compiling) - - # Finish up handling of the various feature flags here because it's - # convenient for both the canonical build and autoconf bundles that - # it be done here. - sqlite-handle-common-feature-flags - sqlite-finalize-feature-flags - ######################################################################## # "Re-export" the autoconf-conventional --XYZdir flags into something # which is more easily overridable from a make invocation. See the docs @@ -1616,7 +1715,7 @@ proc sqlite-process-dot-in-files {} { # (e.g. [proj-check-rpath]) may do so before we "mangle" them here. proj-remap-autoconf-dir-vars - proj-make-from-dot-in -touch Makefile sqlite3.pc + proj-dot-ins-process -validate make-config-header sqlite_cfg.h \ -bare {SIZEOF_* HAVE_DECL_*} \ -none {HAVE_CFLAG_* LDFLAGS_* SH_* SQLITE_AUTORECONFIG @@ -1626,32 +1725,6 @@ proc sqlite-process-dot-in-files {} { proj-touch sqlite_cfg.h ; # help avoid frequent unnecessary @SQLITE_AUTORECONFIG@ } -######################################################################## -# Perform some high-level validation on the generated files... -# -# 1) Ensure that no unresolved @VAR@ placeholders are in files which -# use those. -# -# 2) TBD -proc sqlite-post-config-validation {} { - # Check #1: ensure that files which get filtered for @VAR@ do not - # contain any unresolved @VAR@ refs. That may indicate an - # unexported/unused var or a typo. - set srcdir $::autosetup(srcdir) - foreach f [list Makefile sqlite3.pc \ - $srcdir/tool/emcc.sh \ - $srcdir/ext/wasm/config.make] { - if {![file exists $f]} continue - set lnno 1 - foreach line [proj-file-content-list $f] { - if {[regexp {(@[A-Za-z0-9_]+@)} $line match]} { - error "Unresolved reference to $match at line $lnno of $f" - } - incr lnno - } - } -} - ######################################################################## # Handle --with-wasi-sdk[=DIR] # @@ -1685,7 +1758,8 @@ proc sqlite-handle-wasi-sdk {} { tcl threadsafe } { - if {[opt-bool $opt]} { + if {[proj-opt-exists $opt] && [opt-bool $opt]} { + # -^^^^ not all builds define all of these flags msg-result " --disable-$opt" proj-opt-set $opt 0 } @@ -1704,7 +1778,7 @@ proc sqlite-handle-wasi-sdk {} { proj-opt-set $opt "" } } - # Remember that we now have a discrepancy beteween + # Remember that we now have a discrepancy between # $::sqliteConfig(is-cross-compiling) and [proj-is-cross-compiling]. set ::sqliteConfig(is-cross-compiling) 1 @@ -1762,12 +1836,10 @@ proc sqlite-check-tcl {} { define TCLLIBDIR "" ; # Installation dir for TCL extension lib define TCL_CONFIG_SH ""; # full path to tclConfig.sh - # Clear out all vars which would be set by tclConfigToAutoDef.sh, so - # that the late-config validation of @VARS@ works even if - # --disable-tcl is used. - foreach k {TCL_INCLUDE_SPEC TCL_LIB_SPEC TCL_STUB_LIB_SPEC TCL_EXEC_PREFIX TCL_VERSION} { - define $k "" - } + # Clear out all vars which would harvest from tclConfig.sh so that + # the late-config validation of @VARS@ works even if --disable-tcl + # is used. + proj-tclConfig-sh-to-autosetup "" file delete -force ".tclenv.sh"; # ensure no stale state from previous configures. if {![opt-bool tcl]} { @@ -1788,14 +1860,14 @@ proc sqlite-check-tcl {} { if {"prefix" eq $with_tcl} { set with_tcl [get-define prefix] } - msg-debug "sqlite-check-tcl: use_tcl ${use_tcl}" - msg-debug "sqlite-check-tcl: with_tclsh=${with_tclsh}" - msg-debug "sqlite-check-tcl: with_tcl=$with_tcl" + proc-debug "use_tcl ${use_tcl}" + proc-debug "with_tclsh=${with_tclsh}" + proc-debug "with_tcl=$with_tcl" if {"" eq $with_tclsh && "" eq $with_tcl} { # If neither --with-tclsh nor --with-tcl are provided, try to find # a workable tclsh. - set with_tclsh [proj-first-bin-of tclsh9.0 tclsh8.6 tclsh] - msg-debug "sqlite-check-tcl: with_tclsh=${with_tclsh}" + set with_tclsh [proj-first-bin-of tclsh9.1 tclsh9.0 tclsh8.6 tclsh] + proc-debug "with_tclsh=${with_tclsh}" } set doConfigLookup 1 ; # set to 0 to test the tclConfig.sh-not-found cases @@ -1809,7 +1881,7 @@ proc sqlite-check-tcl {} { #msg-result "Using tclsh: $with_tclsh" } if {$doConfigLookup && - [catch {exec $with_tclsh $srcdir/tool/find_tclconfig.tcl} result] == 0} { + [catch {exec $with_tclsh $::autosetup(libdir)/find_tclconfig.tcl} result] == 0} { set with_tcl $result } if {"" ne $with_tcl && [file isdir $with_tcl]} { @@ -1820,7 +1892,7 @@ proc sqlite-check-tcl {} { } } set cfg "" - set tclSubdirs {tcl9.0 tcl8.6 lib} + set tclSubdirs {tcl9.1 tcl9.0 tcl8.6 lib} while {$use_tcl} { if {"" ne $with_tcl} { # Ensure that we can find tclConfig.sh under ${with_tcl}/... @@ -1866,23 +1938,25 @@ proc sqlite-check-tcl {} { # Export a subset of tclConfig.sh to the current TCL-space. If $cfg # is an empty string, this emits empty-string entries for the # various options we're interested in. - eval [exec sh "$srcdir/tool/tclConfigShToAutoDef.sh" "$cfg"] - # ---------^^ a Windows/msys workaround, without which it cannot - # exec a .sh file: https://sqlite.org/forum/forumpost/befb352a42a7cd6d + proj-tclConfig-sh-to-autosetup $cfg if {"" eq $with_tclsh && $cfg ne ""} { # We have tclConfig.sh but no tclsh. Attempt to locate a tclsh # based on info from tclConfig.sh. - proj-assert {"" ne [get-define TCL_EXEC_PREFIX]} - set with_tclsh [get-define TCL_EXEC_PREFIX]/bin/tclsh[get-define TCL_VERSION] - if {![file-isexec $with_tclsh]} { - set with_tclsh2 [get-define TCL_EXEC_PREFIX]/bin/tclsh - if {![file-isexec $with_tclsh2]} { - proj-warn "Cannot find a usable tclsh (tried: $with_tclsh $with_tclsh2)" - } else { - set with_tclsh $with_tclsh2 + set tclExecPrefix [get-define TCL_EXEC_PREFIX] + proj-assert {"" ne $tclExecPrefix} + set tryThese [list \ + $tclExecPrefix/bin/tclsh[get-define TCL_VERSION] \ + $tclExecPrefix/bin/tclsh ] + foreach trySh $tryThese { + if {[file-isexec $trySh]} { + set with_tclsh $trySh + break } } + if {![file-isexec $with_tclsh]} { + proj-warn "Cannot find a usable tclsh (tried: $tryThese) + } } define TCLSH_CMD $with_tclsh if {$use_tcl} { @@ -2033,10 +2107,13 @@ proc sqlite-determine-codegen-tcl {} { }; # sqlite-determine-codegen-tcl ######################################################################## -# Runs sqlite-check-tcl and sqlite-determine-codegen-tcl. +# Runs sqlite-check-tcl and, if this is the canonical build, +# sqlite-determine-codegen-tcl. proc sqlite-handle-tcl {} { sqlite-check-tcl - msg-result "TCL for code generation: [sqlite-determine-codegen-tcl]" + if {"canonical" eq $::sqliteConfig(build-mode)} { + msg-result "TCL for code generation: [sqlite-determine-codegen-tcl]" + } } ######################################################################## diff --git a/autosetup/teaish/README.txt b/autosetup/teaish/README.txt new file mode 100644 index 000000000..e11519b04 --- /dev/null +++ b/autosetup/teaish/README.txt @@ -0,0 +1,4 @@ +The *.tcl files in this directory are part of the SQLite's "autoconf" +bundle which are specific to the TEA(-ish) build. During the tarball +generation process, they are copied into /autoconf/autosetup/teaish +(which itself is created as part of that process). diff --git a/autosetup/teaish/core.tcl b/autosetup/teaish/core.tcl new file mode 100644 index 000000000..09017029d --- /dev/null +++ b/autosetup/teaish/core.tcl @@ -0,0 +1,2539 @@ +######################################################################## +# 2025 April 5 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# * May you do good and not evil. +# * May you find forgiveness for yourself and forgive others. +# * May you share freely, never taking more than you give. +# +######################################################################## +# ----- @module teaish.tcl ----- +# @section TEA-ish ((TCL Extension Architecture)-ish) +# +# Functions in this file with a prefix of teaish__ are +# private/internal APIs. Those with a prefix of teaish- are +# public APIs. +# +# Teaish has a hard dependency on proj.tcl, and any public API members +# of that module are considered legal for use by teaish extensions. +# +# Project home page: https://fossil.wanderinghorse.net/r/teaish + +use proj + +# +# API-internal settings and shared state. +array set teaish__Config [proj-strip-hash-comments { + # + # Teaish's version number, not to be confused with + # teaish__PkgInfo(-version). + # + version 0.1-beta + + # set to 1 to enable some internal debugging output + debug-enabled 0 + + # + # 0 = don't yet have extension's pkgindex + # 0x01 = found TEAISH_EXT_DIR/pkgIndex.tcl.in + # 0x02 = found srcdir/pkgIndex.tcl.in + # 0x10 = found TEAISH_EXT_DIR/pkgIndex.tcl (static file) + # 0x20 = static-pkgIndex.tcl pragma: behave as if 0x10 + # 0x100 = disabled by -tm.tcl.in + # 0x200 = disabled by -tm.tcl + # + # Reminder: it's significant that the bottom 4 bits be + # cases where teaish manages ./pkgIndex.tcl. + # + pkgindex-policy 0 + + # + # The pkginit counterpart of pkgindex-policy: + # + # 0 = no pkginit + # 0x01 = found default X.in: generate X from X.in + # 0x10 = found static pkginit file X + # 0x02 = user-provided X.in generates ./X. + # 0x20 = user-provided static pkginit file X + # + # The 0x0f bits indicate that teaish is responsible for cleaning up + # the (generated) pkginit file. + # + pkginit-policy 0 + # + # 0 = no tm.tcl + # 0x01 = tm.tcl.in + # 0x10 = static tm.tcl + tm-policy 0 + + # + # If 1+ then teaish__verbose will emit messages. + # + verbose 0 + + # + # Mapping of pkginfo -flags to their TEAISH_xxx define (if any). + # This must not be modified after initialization. + # + pkginfo-f2d { + -name TEAISH_NAME + -name.dist TEAISH_DIST_NAME + -name.pkg TEAISH_PKGNAME + -version TEAISH_VERSION + -libDir TEAISH_LIBDIR_NAME + -loadPrefix TEAISH_LOAD_PREFIX + -vsatisfies TEAISH_VSATISFIES + -pkgInit.tcl TEAISH_PKGINIT_TCL + -pkgInit.tcl.in TEAISH_PKGINIT_TCL_IN + -url TEAISH_URL + -tm.tcl TEAISH_TM_TCL + -tm.tcl.in TEAISH_TM_TCL_IN + -options {} + -pragmas {} + } + + # + # Queues for use with teaish-checks-queue and teaish-checks-run. + # + queued-checks-pre {} + queued-checks-post {} + + # Whether or not "make dist" parts are enabled. They get enabled + # when building from an extension's dir, disabled when building + # elsewhere. + dist-enabled 1 + # Whether or not "make install" parts are enabled. By default + # they are, but we have a single use case where they're + # both unnecessary and unhelpful, so... + install-enabled 1 + + # By default we enable compilation of a native extension but if the + # extension has no native code or the user wants to take that over + # via teaish.make.in or provide a script-only extension, we will + # elide the default compilation rules if this is 0. + dll-enabled 1 + + # Files to include in the "make dist" bundle. + dist-files {} + + # List of source files for the extension. + extension-src {} + + # Path to the teaish.tcl file. + teaish.tcl {} + + # Dir where teaish.tcl is found. + extension-dir {} + + # Whether the generates TEASH_VSATISFIES_CODE should error out on a + # satisfies error. If 0, it uses return instead of error. + vsatisfies-error 1 + + # Whether or not to allow a "full dist" - a "make dist" build which + # includes both the extension and teaish. By default this is only on + # if the extension dir is teaish's dir. + dist-full-enabled 0 +}] +set teaish__Config(core-dir) $::autosetup(libdir)/teaish + +# +# Array of info managed by teaish-pkginfo-get and friends. Has the +# same set of keys as $teaish__Config(pkginfo-f2d). +# +array set teaish__PkgInfo {} + +# +# Runs {*}$args if $lvl is <= the current verbosity level, else it has +# no side effects. +# +proc teaish__verbose {lvl args} { + if {$lvl <= $::teaish__Config(verbose)} { + {*}$args + } +} + +# +# @teaish-argv-has flags... +# +# Returns true if any arg in $::argv matches any of the given globs, +# else returns false. +# +proc teaish-argv-has {args} { + foreach glob $args { + foreach arg $::argv { + if {[string match $glob $arg]} { + return 1 + } + } + } + return 0 +} + +if {[teaish-argv-has --teaish-verbose --t-v]} { + # Check this early so that we can use verbose-only messages in the + # pre-options-parsing steps. + set ::teaish__Config(verbose) 1 + #teaish__verbose 1 msg-result "--teaish-verbose activated" +} + +msg-quiet use system ; # Outputs "Host System" and "Build System" lines +if {"--help" ni $::argv} { + teaish__verbose 1 msg-result "TEA(ish) Version = $::teaish__Config(version)" + teaish__verbose 1 msg-result "Source dir = $::autosetup(srcdir)" + teaish__verbose 1 msg-result "Build dir = $::autosetup(builddir)" +} + +# +# @teaish-configure-core +# +# Main entry point for the TEA-ish configure process. auto.def's primary +# (ideally only) job should be to call this. +# +proc teaish-configure-core {} { + proj-tweak-default-env-dirs + + set ::teaish__Config(install-mode) [teaish-argv-has --teaish-install*] + set ::teaish__Config(create-ext-mode) \ + [teaish-argv-has --teaish-create-extension=* --t-c-e=*] + set gotExt 0; # True if an extension config is found + if {!$::teaish__Config(create-ext-mode) + && !$::teaish__Config(install-mode)} { + # Don't look for an extension if we're in --t-c-e or --t-i mode + set gotExt [teaish__find_extension] + } + + # + # Set up the core --flags. This needs to come before teaish.tcl is + # sourced so that that file can use teaish-pkginfo-set to append + # options. + # + options-add [proj-strip-hash-comments { + with-tcl:DIR + => {Directory containing tclConfig.sh or a directory one level up from + that, from which we can derive a directory containing tclConfig.sh. + Defaults to the $TCL_HOME environment variable.} + + with-tclsh:PATH + => {Full pathname of tclsh to use. It is used for trying to find + tclConfig.sh. Warning: if its containing dir has multiple tclsh + versions, it may select the wrong tclConfig.sh! + Defaults to the $TCLSH environment variable.} + + # TEA has --with-tclinclude but it appears to only be useful for + # building an extension against an uninstalled copy of TCL's own + # source tree. The policy here is that either we get that info + # from tclConfig.sh or we give up. + # + # with-tclinclude:DIR + # => {Specify the directory which contains the tcl.h. This should not + # normally be required, as that information comes from tclConfig.sh.} + + # We _generally_ want to reduce the possibility of flag collisions with + # extensions, and thus use a teaish-... prefix on most flags. However, + # --teaish-extension-dir is frequently needed, so... + # + # As of this spontaneous moment, we'll settle on using --t-A-X to + # abbreviate --teaish-A...-X... flags when doing so is + # unambiguous... + ted: t-e-d: + teaish-extension-dir:DIR + => {Looks for an extension in the given directory instead of the current + dir.} + + t-c-e: + teaish-create-extension:TARGET_DIRECTORY + => {Writes stub files for creating an extension. Will refuse to overwrite + existing files without --teaish-force.} + + t-f + teaish-force + => {Has a context-dependent meaning (autosetup defines --force for its + own use).} + + t-d-d + teaish-dump-defines + => {Dump all configure-defined vars to config.defines.txt} + + t-v:=0 + teaish-verbose:=0 + => {Enable more (often extraneous) messages from the teaish core.} + + t-d + teaish-debug=0 => {Enable teaish-specific debug output} + + t-i + teaish-install:=auto + => {Installs a copy of teaish, including autosetup, to the target dir. + When used with --teaish-create-extension=DIR, a value of "auto" + (no no value) will inherit that directory.} + + #TODO: --teaish-install-extension:=dir as short for + # --t-c-e=dir --t-i + + t-e-p: + teaish-extension-pkginfo:pkginfo + => {For use with --teaish-create-extension. If used, it must be a + list of arguments for use with teaish-pkginfo-set, e.g. + --teaish-extension-pkginfo="-name Foo -version 2.3"} + + t-v-c + teaish-vsatisfies-check=1 + => {Disable the configure-time "vsatisfies" check on the target tclsh.} + + }]; # main options. + + if {$gotExt} { + # We found an extension. Source it... + set ttcl $::teaish__Config(teaish.tcl) + proj-assert {"" ne [teaish-pkginfo-get -name]} + proj-assert {[file exists $ttcl]} \ + "Expecting to have found teaish.(tcl|config) by now" + if {[string match *.tcl $ttcl]} { + uplevel 1 {source $::teaish__Config(teaish.tcl)} + } else { + teaish-pkginfo-set {*}[proj-file-content -trim $ttcl] + } + unset ttcl + # Set up some default values if the extension did not set them. + # This must happen _after_ it's sourced but before + # teaish-configure is called. + array set f2d $::teaish__Config(pkginfo-f2d) + foreach {pflag key type val} { + - TEAISH_CFLAGS -v "" + - TEAISH_LDFLAGS -v "" + - TEAISH_MAKEFILE -v "" + - TEAISH_MAKEFILE_CODE -v "" + - TEAISH_MAKEFILE_IN -v "" + - TEAISH_PKGINDEX_TCL -v "" + - TEAISH_PKGINDEX_TCL_IN -v "" + - TEAISH_PKGINIT_TCL -v "" + - TEAISH_PKGINIT_TCL_IN -v "" + - TEAISH_PKGINIT_TCL_TAIL -v "" + - TEAISH_TEST_TCL -v "" + - TEAISH_TEST_TCL_IN -v "" + + -version - -v 0.0.0 + -name.pkg - -e {set ::teaish__PkgInfo(-name)} + -name.dist - -e {set ::teaish__PkgInfo(-name)} + -libDir - -e { + join [list \ + $::teaish__PkgInfo(-name.pkg) \ + $::teaish__PkgInfo(-version)] "" + } + -loadPrefix - -e { + string totitle $::teaish__PkgInfo(-name.pkg) + } + -vsatisfies - -v {{Tcl 8.5-}} + -pkgInit.tcl - -v "" + -pkgInit.tcl.in - -v "" + -url - -v "" + -tm.tcl - -v "" + -tm.tcl.in - -v "" + } { + set isPIFlag [expr {"-" ne $pflag}] + if {$isPIFlag} { + if {[info exists ::teaish__PkgInfo($pflag)]} { + # Was already set - skip it. + continue; + } + proj-assert {{-} eq $key} + set key $f2d($pflag) + } + proj-assert {"" ne $key} + set got [get-define $key ""] + if {"" ne $got} { + # Was already set - skip it. + continue + } + switch -exact -- $type { + -v {} + -e { set val [eval $val] } + default { proj-error "Invalid type flag: $type" } + } + #puts "***** defining default $pflag $key {$val} isPIFlag=$isPIFlag got=$got" + define $key $val + if {$isPIFlag} { + set ::teaish__PkgInfo($pflag) $val + } + } + unset isPIFlag pflag key type val + array unset f2d + }; # sourcing extension's teaish.tcl + + if {[llength [info proc teaish-options]] > 0} { + # Add options defined by teaish-options, which is assumed to be + # imported via [teaish-get -teaish-tcl]. + set o [teaish-options] + if {"" ne $o} { + options-add $o + } + } + #set opts [proj-options-combine] + #lappend opts teaish-debug => {x}; #testing dupe entry handling + if {[catch {options {}} msg xopts]} { + # Workaround for + # where [options] behaves oddly on _some_ TCL builds when it's + # called from deeper than the global scope. + dict incr xopts -level + return {*}$xopts $msg + } + + proj-xfer-options-aliases { + t-c-e => teaish-create-extension + t-d => teaish-debug + t-d-d => teaish-dump-defines + ted => teaish-extension-dir + t-e-d => teaish-extension-dir + t-e-p => teaish-extension-pkginfo + t-f => teaish-force + t-i => teaish-install + t-v => teaish-verbose + t-v-c => teaish-vsatisfies-check + } + + scan [opt-val teaish-verbose 0] %d ::teaish__Config(verbose) + set ::teaish__Config(debug-enabled) [opt-bool teaish-debug] + + set exitEarly 0 + if {[proj-opt-was-provided teaish-create-extension]} { + teaish__create_extension [opt-val teaish-create-extension] + incr exitEarly + } + if {$::teaish__Config(install-mode)} { + teaish__install + incr exitEarly + } + + if {$exitEarly} { + file delete -force config.log + return + } + proj-assert {1==$gotExt} "Else we cannot have gotten this far" + + teaish__configure_phase1 +} + + +# +# Internal config-time debugging output routine. It is not legal to +# call this from the global scope. +# +proc teaish-debug {msg} { + if {$::teaish__Config(debug-enabled)} { + puts stderr [proj-bold "** DEBUG: \[[proj-scope 1]\]: $msg"] + } +} + +# +# Runs "phase 1" of the configuration, immediately after processing +# --flags. This is what will import the client-defined teaish.tcl. +# +proc teaish__configure_phase1 {} { + msg-result \ + [join [list "Configuring build of Tcl extension" \ + [proj-bold [teaish-pkginfo-get -name] \ + [teaish-pkginfo-get -version]] "..."]] + + uplevel 1 { + use cc cc-db cc-shared cc-lib; # pkg-config + } + teaish__check_tcl + apply {{} { + # + # If --prefix or --exec-prefix are _not_ provided, use their + # TCL_... counterpart from tclConfig.sh. Caveat: by the time we can + # reach this point, autosetup's system.tcl will have already done + # some non-trivial amount of work with these to create various + # derived values from them, so we temporarily end up with a mishmash + # of autotools-compatibility var values. That will be straightened + # out in the final stage of the configure script via + # [proj-remap-autoconf-dir-vars]. + # + foreach {flag uflag tclVar} { + prefix prefix TCL_PREFIX + exec-prefix exec_prefix TCL_EXEC_PREFIX + } { + if {![proj-opt-was-provided $flag]} { + if {"exec-prefix" eq $flag} { + # If --exec-prefix was not used, ensure that --exec-prefix + # derives from the --prefix we may have just redefined. + set v {${prefix}} + } else { + set v [get-define $tclVar "???"] + teaish__verbose 1 msg-result "Using \$$tclVar for --$flag=$v" + } + proj-assert {"???" ne $v} "Expecting teaish__check_tcl to have defined $tclVar" + #puts "*** $flag $uflag $tclVar = $v" + proj-opt-set $flag $v + define $uflag $v + + # ^^^ As of here, all autotools-compatibility vars which derive + # from --$flag, e.g. --libdir, still derive from the default + # --$flag value which was active when system.tcl was + # included. So long as those flags are not explicitly passed to + # the configure script, those will be straightened out via + # [proj-remap-autoconf-dir-vars]. + } + } + }}; # --[exec-]prefix defaults + teaish__check_common_bins + # + # Set up library file names + # + proj-file-extensions + teaish__define_pkginfo_derived * + + teaish-checks-run -pre + if {[llength [info proc teaish-configure]] > 0} { + # teaish-configure is assumed to be imported via + # teaish.tcl + teaish-configure + } + teaish-checks-run -post + + apply {{} { + # Set up "vsatisfies" code for pkgIndex.tcl.in, + # _teaish.tester.tcl.in, and for a configure-time check. We would + # like to put this before [teaish-checks-run -pre] but it's + # marginally conceivable that a client may need to dynamically + # calculate the vsatisfies and set it via [teaish-configure]. + set vs [get-define TEAISH_VSATISFIES ""] + if {"" eq $vs} return + set code {} + set n 0 + # Treat $vs as a list-of-lists {{Tcl 8.5-} {Foo 1.0- -3.0} ...} + # and generate Tcl which will run package vsatisfies tests with + # that info. + foreach pv $vs { + set n [llength $pv] + if {$n < 2} { + proj-error "-vsatisfies: {$pv} appears malformed. Whole list is: $vs" + } + set pkg [lindex $pv 0] + set vcheck {} + for {set i 1} {$i < $n} {incr i} { + lappend vcheck [lindex $pv $i] + } + if {[opt-bool teaish-vsatisfies-check]} { + set tclsh [get-define TCLSH_CMD] + set vsat "package vsatisfies \[ package provide $pkg \] $vcheck" + set vputs "puts \[ $vsat \]" + #puts "*** vputs = $vputs" + scan [exec echo $vputs | $tclsh] %d vvcheck + if {![info exists vvcheck] || 0 == $vvcheck} { + proj-fatal -up $tclsh "check failed:" $vsat + } + } + if {$::teaish__Config(vsatisfies-error)} { + set vunsat \ + [list error [list Package \ + $::teaish__PkgInfo(-name) $::teaish__PkgInfo(-version) \ + requires $pv]] + } else { + set vunsat return + } + lappend code \ + [string trim [subst -nocommands \ + {if { ![package vsatisfies [package provide $pkg] $vcheck] } {\n $vunsat\n}}]] + }; # foreach pv + define TEAISH_VSATISFIES_CODE [join $code "\n"] + }}; # vsatisfies + + if {[proj-looks-like-windows] || [proj-looks-like-mac]} { + # Without this, linking of an extension will not work on Cygwin or + # Msys2. + msg-result "Using USE_TCL_STUBS for this environment" + teaish-cflags-add -DUSE_TCL_STUBS=1 + } + + #define AS_LIBDIR $::autosetup(libdir) + define TEAISH_TESTUTIL_TCL $::teaish__Config(core-dir)/tester.tcl + + apply {{} { + # + # Ensure we have a pkgIndex.tcl and don't have a stale generated one + # when rebuilding for different --with-tcl=... values. + # + if {!$::teaish__Config(pkgindex-policy)} { + proj-error "Cannot determine which pkgIndex.tcl to use" + } + if {0x300 & $::teaish__Config(pkgindex-policy)} { + teaish__verbose 1 msg-result "pkgIndex disabled by -tm.tcl(.in)" + } else { + set tpi [proj-coalesce \ + [get-define TEAISH_PKGINDEX_TCL_IN] \ + [get-define TEAISH_PKGINDEX_TCL]] + proj-assert {$tpi ne ""} \ + "TEAISH_PKGINDEX_TCL should have been set up by now" + teaish__verbose 1 msg-result "Using pkgIndex from $tpi" + if {0x0f & $::teaish__Config(pkgindex-policy)} { + # Don't leave stale pkgIndex.tcl laying around yet don't delete + # or overwrite a user-managed static pkgIndex.tcl. + file delete -force -- [get-define TEAISH_PKGINDEX_TCL] + proj-dot-ins-append [get-define TEAISH_PKGINDEX_TCL_IN] + } else { + teaish-dist-add [file tail $tpi] + } + } + }}; # $::teaish__Config(pkgindex-policy) + + # + # Ensure we clean up TEAISH_PKGINIT_TCL if needed and @-process + # TEAISH_PKGINIT_TCL_IN if needed. + # + if {0x0f & $::teaish__Config(pkginit-policy)} { + file delete -force -- [get-define TEAISH_PKGINIT_TCL] + proj-dot-ins-append [get-define TEAISH_PKGINIT_TCL_IN] + } + if {0x0f & $::teaish__Config(tm-policy)} { + file delete -force -- [get-define TEAISH_TM_TCL] + proj-dot-ins-append [get-define TEAISH_TM_TCL_IN] + } + + apply {{} { + # Queue up any remaining dot-in files + set dotIns [list] + foreach d { + TEAISH_TESTER_TCL_IN + TEAISH_TEST_TCL_IN + TEAISH_MAKEFILE_IN + } { + lappend dotIns [get-define $d ""] + } + lappend dotIns $::autosetup(srcdir)/Makefile.in; # must be after TEAISH_MAKEFILE_IN + foreach f $dotIns { + if {"" ne $f} { + proj-dot-ins-append $f + } + } + }} + + define TEAISH_DIST_FULL \ + [expr { + $::teaish__Config(dist-enabled) + && $::teaish__Config(dist-full-enabled) + }] + + define TEAISH_AUTOSETUP_DIR $::teaish__Config(core-dir) + define TEAISH_ENABLE_DIST $::teaish__Config(dist-enabled) + define TEAISH_ENABLE_INSTALL $::teaish__Config(install-enabled) + define TEAISH_ENABLE_DLL $::teaish__Config(dll-enabled) + define TEAISH_TCL $::teaish__Config(teaish.tcl) + + define TEAISH_DIST_FILES [join $::teaish__Config(dist-files)] + define TEAISH_EXT_DIR [join $::teaish__Config(extension-dir)] + define TEAISH_EXT_SRC [join $::teaish__Config(extension-src)] + proj-setup-autoreconfig TEAISH_AUTORECONFIG + foreach f { + TEAISH_CFLAGS + TEAISH_LDFLAGS + } { + # Ensure that any of these lists are flattened + define $f [join [get-define $f]] + } + proj-remap-autoconf-dir-vars + set tdefs [teaish__defines_to_list] + define TEAISH__DEFINES_MAP $tdefs; # injected into _teaish.tester.tcl + + # + # NO [define]s after this point! + # + proj-dot-ins-process -validate + proj-if-opt-truthy teaish-dump-defines { + proj-file-write config.defines.txt $tdefs + } + +}; # teaish__configure_phase1 + +# +# Run checks for required binaries. +# +proc teaish__check_common_bins {} { + if {"" eq [proj-bin-define install]} { + proj-warn "Cannot find install binary, so 'make install' will not work." + define BIN_INSTALL false + } + if {"" eq [proj-bin-define zip]} { + proj-warn "Cannot find zip, so 'make dist.zip' will not work." + } + if {"" eq [proj-bin-define tar]} { + proj-warn "Cannot find tar, so 'make dist.tgz' will not work." + } +} + +# +# TCL... +# +# teaish__check_tcl performs most of the --with-tcl and --with-tclsh +# handling. Some related bits and pieces are performed before and +# after that function is called. +# +# Important [define]'d vars: +# +# - TCLSH_CMD is the path to the canonical tclsh or "". +# +# - TCL_CONFIG_SH is the path to tclConfig.sh or "". +# +# - TCLLIBDIR is the dir to which the extension library gets +# - installed. +# +proc teaish__check_tcl {} { + define TCLSH_CMD false ; # Significant is that it exits with non-0 + define TCLLIBDIR "" ; # Installation dir for TCL extension lib + define TCL_CONFIG_SH ""; # full path to tclConfig.sh + + # Clear out all vars which would harvest from tclConfig.sh so that + # the late-config validation of @VARS@ works even if --disable-tcl + # is used. + proj-tclConfig-sh-to-autosetup "" + + # TODO: better document the steps this is taking. + set srcdir $::autosetup(srcdir) + msg-result "Checking for a suitable tcl... " + set use_tcl 1 + set withSh [opt-val with-tclsh [proj-get-env TCLSH]] + set tclHome [opt-val with-tcl [proj-get-env TCL_HOME]] + if {[string match */lib $tclHome]} { + # TEA compatibility kludge: its --with-tcl wants the lib + # dir containing tclConfig.sh. + #proj-warn "Replacing --with-tcl=$tclHome for TEA compatibility" + regsub {/lib^} $tclHome "" tclHome + msg-result "NOTE: stripped /lib suffix from --with-tcl=$tclHome (a TEA-ism)" + } + if {0} { + # This misinteracts with the $TCL_PREFIX default: it will use the + # autosetup-defined --prefix default + if {"prefix" eq $tclHome} { + set tclHome [get-define prefix] + } + } + teaish-debug "use_tcl ${use_tcl}" + teaish-debug "withSh=${withSh}" + teaish-debug "tclHome=$tclHome" + if {"" eq $withSh && "" eq $tclHome} { + # If neither --with-tclsh nor --with-tcl are provided, try to find + # a workable tclsh. + set withSh [proj-first-bin-of tclsh9.1 tclsh9.0 tclsh8.6 tclsh] + teaish-debug "withSh=${withSh}" + } + + set doConfigLookup 1 ; # set to 0 to test the tclConfig.sh-not-found cases + if {"" ne $withSh} { + # --with-tclsh was provided or found above. Validate it and use it + # to trump any value passed via --with-tcl=DIR. + if {![file-isexec $withSh]} { + proj-error "TCL shell $withSh is not executable" + } else { + define TCLSH_CMD $withSh + #msg-result "Using tclsh: $withSh" + } + if {$doConfigLookup && + [catch {exec $withSh $::autosetup(libdir)/find_tclconfig.tcl} result] == 0} { + set tclHome $result + } + if {"" ne $tclHome && [file isdirectory $tclHome]} { + teaish__verbose 1 msg-result "$withSh recommends the tclConfig.sh from $tclHome" + } else { + proj-warn "$withSh is unable to recommend a tclConfig.sh" + set use_tcl 0 + } + } + set cfg "" + set tclSubdirs {tcl9.1 tcl9.0 tcl8.6 tcl8.5 lib} + while {$use_tcl} { + if {"" ne $tclHome} { + # Ensure that we can find tclConfig.sh under ${tclHome}/... + if {$doConfigLookup} { + if {[file readable "${tclHome}/tclConfig.sh"]} { + set cfg "${tclHome}/tclConfig.sh" + } else { + foreach i $tclSubdirs { + if {[file readable "${tclHome}/$i/tclConfig.sh"]} { + set cfg "${tclHome}/$i/tclConfig.sh" + break + } + } + } + } + if {"" eq $cfg} { + proj-error "No tclConfig.sh found under ${tclHome}" + } + } else { + # If we have not yet found a tclConfig.sh file, look in $libdir + # which is set automatically by autosetup or via the --prefix + # command-line option. See + # https://sqlite.org/forum/forumpost/e04e693439a22457 + set libdir [get-define libdir] + if {[file readable "${libdir}/tclConfig.sh"]} { + set cfg "${libdir}/tclConfig.sh" + } else { + foreach i $tclSubdirs { + if {[file readable "${libdir}/$i/tclConfig.sh"]} { + set cfg "${libdir}/$i/tclConfig.sh" + break + } + } + } + if {![file readable $cfg]} { + break + } + } + teaish__verbose 1 msg-result "Using tclConfig.sh = $cfg" + break + }; # while {$use_tcl} + define TCL_CONFIG_SH $cfg + # Export a subset of tclConfig.sh to the current TCL-space. If $cfg + # is an empty string, this emits empty-string entries for the + # various options we're interested in. + proj-tclConfig-sh-to-autosetup $cfg + + if {"" eq $withSh && $cfg ne ""} { + # We have tclConfig.sh but no tclsh. Attempt to locate a tclsh + # based on info from tclConfig.sh. + set tclExecPrefix [get-define TCL_EXEC_PREFIX] + proj-assert {"" ne $tclExecPrefix} + set tryThese [list \ + $tclExecPrefix/bin/tclsh[get-define TCL_VERSION] \ + $tclExecPrefix/bin/tclsh ] + foreach trySh $tryThese { + if {[file-isexec $trySh]} { + set withSh $trySh + break + } + } + if {![file-isexec $withSh]} { + proj-warn "Cannot find a usable tclsh (tried: $tryThese)" + } + } + define TCLSH_CMD $withSh + if {$use_tcl} { + # Set up the TCLLIBDIR + set tcllibdir [get-env TCLLIBDIR ""] + set extDirName [teaish-pkginfo-get -libDir] + if {"" eq $tcllibdir} { + # Attempt to extract TCLLIBDIR from TCL's $auto_path + if {"" ne $withSh && + [catch {exec echo "puts stdout \$auto_path" | "$withSh"} result] == 0} { + foreach i $result { + if {![string match //zip* $i] && [file isdirectory $i]} { + # isdirectory actually passes on //zipfs:/..., but those are + # useless for our purposes + set tcllibdir $i/$extDirName + break + } + } + } else { + proj-error "Cannot determine TCLLIBDIR." + } + } + define TCLLIBDIR $tcllibdir + }; # find TCLLIBDIR + + set gotSh [file-isexec $withSh] + set tmdir ""; # first tcl::tm::list entry + if {$gotSh} { + catch { + set tmli [exec echo {puts [tcl::tm::list]} | $withSh] + # Reminder: this list contains many names of dirs which do not + # exist but are legitimate. If we rely only on an is-dir check, + # we can end up not finding any of the many candidates. + set firstDir "" + foreach d $tmli { + if {"" eq $firstDir && ![string match //*:* $d]} { + # First non-VFS entry, e.g. not //zipfs: + set firstDir $d + } + if {[file isdirectory $d]} { + set tmdir $d + break + } + } + if {"" eq $tmdir} { + set tmdir $firstDir + } + }; # find tcl::tm path + } + define TEAISH_TCL_TM_DIR $tmdir + + # Finally, let's wrap up... + if {$gotSh} { + teaish__verbose 1 msg-result "Using tclsh = $withSh" + if {$cfg ne ""} { + define HAVE_TCL 1 + } else { + proj-warn "Found tclsh but no tclConfig.sh." + } + if {"" eq $tmdir} { + proj-warn "Did not find tcl::tm directory." + } + } + show-notices + # If TCL is not found: if it was explicitly requested then fail + # fatally, else just emit a warning. If we can find the APIs needed + # to generate a working JimTCL then that will suffice for build-time + # TCL purposes (see: proc sqlite-determine-codegen-tcl). + if {!$gotSh} { + proj-error "Did not find tclsh" + } elseif {"" eq $cfg} { + proj-indented-notice -error { + Cannot find a usable tclConfig.sh file. Use --with-tcl=DIR to + specify a directory near which tclConfig.sh can be found, or + --with-tclsh=/path/to/tclsh to allow the tclsh binary to locate + its tclConfig.sh, with the caveat that a symlink to tclsh, or + wrapper script around it, e.g. ~/bin/tclsh -> + $HOME/tcl/9.0/bin/tclsh9.1, may not work because tclsh emits + different library paths for the former than the latter. + } + } + msg-result "Using Tcl [get-define TCL_VERSION] from [get-define TCL_PREFIX]." + teaish__tcl_platform_quirks +}; # teaish__check_tcl + +# +# Perform last-minute platform-specific tweaks to account for quirks. +# +proc teaish__tcl_platform_quirks {} { + define TEAISH_POSTINST_PREREQUIRE "" + switch -glob -- [get-define host] { + *-haiku { + # Haiku's default TCLLIBDIR is "all wrong": it points to a + # read-only virtual filesystem mount-point. We bend it back to + # fit under $TCL_PACKAGE_PATH here. + foreach {k d} { + vj TCL_MAJOR_VERSION + vn TCL_MINOR_VERSION + pp TCL_PACKAGE_PATH + ld TCLLIBDIR + } { + set $k [get-define $d] + } + if {[string match /packages/* $ld]} { + set old $ld + set tail [file tail $ld] + if {8 == $vj} { + set ld "${pp}/tcl${vj}.${vn}/${tail}" + } else { + proj-assert {9 == $vj} + set ld "${pp}/${tail}" + } + define TCLLIBDIR $ld + # [load foo.so], without a directory part, does not work via + # automated tests on Haiku (but works when run + # manually). Similarly, the post-install [package require ...] + # test fails, presumably for a similar reason. We work around + # the former in _teaish.tester.tcl.in. We work around the + # latter by amending the post-install check's ::auto_path (in + # Makefile.in). This code MUST NOT contain any single-quotes. + define TEAISH_POSTINST_PREREQUIRE \ + [join [list set ::auto_path \ + \[ linsert \$::auto_path 0 $ld \] \; \ + ]] + proj-indented-notice [subst -nocommands -nobackslashes { + Haiku users take note: patching target installation dir to match + Tcl's home because Haiku's is not writable. + + Original : $old + Substitute: $ld + }] + } + } + } +}; # teaish__tcl_platform_quirks + +# +# Searches $::argv and/or the build dir and/or the source dir for +# teaish.tcl and friends. Fails if it cannot find teaish.tcl or if +# there are other irreconcilable problems. If it returns 0 then it did +# not find an extension but the --help flag was seen, in which case +# that's not an error. +# +# This does not _load_ the extension, it primarily locates the files +# which make up an extension and fills out no small amount of teaish +# state related to that. +# +proc teaish__find_extension {} { + proj-assert {!$::teaish__Config(install-mode)} + teaish__verbose 1 msg-result "Looking for teaish extension..." + + # Helper for the foreach loop below. + set checkTeaishTcl {{mustHave fid dir} { + set f [file join $dir $fid] + if {[file readable $f]} { + file-normalize $f + } elseif {$mustHave} { + proj-error "Missing required $dir/$fid" + } + }} + + # + # We have to handle some flags manually because the extension must + # be loaded before [options] is run (so that the extension can + # inject its own options). + # + set dirBld $::autosetup(builddir); # dir we're configuring under + set dirSrc $::autosetup(srcdir); # where teaish's configure script lives + set extT ""; # teaish.tcl + set largv {}; # rewritten $::argv + set gotHelpArg 0; # got the --help + foreach arg $::argv { + #puts "*** arg=$arg" + switch -glob -- $arg { + --ted=* - + --t-e-d=* - + --teaish-extension-dir=* { + # Ensure that $extD refers to a directory and contains a + # teaish.tcl. + regexp -- {--[^=]+=(.+)} $arg - extD + set extD [file-normalize $extD] + if {![file isdirectory $extD]} { + proj-error "--teaish-extension-dir value is not a directory: $extD" + } + set extT [apply $checkTeaishTcl 0 teaish.config $extD] + if {"" eq $extT} { + set extT [apply $checkTeaishTcl 1 teaish.tcl $extD] + } + set ::teaish__Config(extension-dir) $extD + } + --help { + incr gotHelpArg + lappend largv $arg + } + default { + lappend largv $arg + } + } + } + set ::argv $largv + + set dirExt $::teaish__Config(extension-dir); # dir with the extension + # + # teaish.tcl is a TCL script which implements various + # interfaces described by this framework. + # + # We use the first one we find in the builddir or srcdir. + # + if {"" eq $extT} { + set flist [list] + proj-assert {$dirExt eq ""} + lappend flist $dirBld/teaish.tcl $dirBld/teaish.config $dirSrc/teaish.tcl + if {![proj-first-file-found extT $flist]} { + if {$gotHelpArg} { + # Tell teaish-configure-core that the lack of extension is not + # an error when --help or --teaish-install is used. + return 0; + } + proj-indented-notice -error " +Did not find any of: $flist + +If you are attempting an out-of-tree build, use + --teaish-extension-dir=/path/to/extension" + } + } + if {![file readable $extT]} { + proj-error "extension tcl file is not readable: $extT" + } + set ::teaish__Config(teaish.tcl) $extT + set dirExt [file dirname $extT] + + set ::teaish__Config(extension-dir) $dirExt + set ::teaish__Config(blddir-is-extdir) [expr {$dirBld eq $dirExt}] + set ::teaish__Config(dist-enabled) $::teaish__Config(blddir-is-extdir); # may change later + set ::teaish__Config(dist-full-enabled) \ + [expr {[file-normalize $::autosetup(srcdir)] + eq [file-normalize $::teaish__Config(extension-dir)]}] + + set addDist {{file} { + teaish-dist-add [file tail $file] + }} + apply $addDist $extT + + teaish__verbose 1 msg-result "Extension dir = [teaish-get -dir]" + teaish__verbose 1 msg-result "Extension config = $extT" + + teaish-pkginfo-set -name [file tail [file dirname $extT]] + + # + # teaish.make[.in] provides some of the info for the main makefile, + # like which source(s) to build and their build flags. + # + # We use the first one of teaish.make.in or teaish.make we find in + # $dirExt. + # + if {[proj-first-file-found extM \ + [list \ + $dirExt/teaish.make.in \ + $dirExt/teaish.make \ + ]]} { + if {[string match *.in $extM]} { + define TEAISH_MAKEFILE_IN $extM + define TEAISH_MAKEFILE [file rootname [file tail $extM]] + } else { + define TEAISH_MAKEFILE_IN "" + define TEAISH_MAKEFILE $extM + } + apply $addDist $extM + teaish__verbose 1 msg-result "Extension makefile = $extM" + } else { + define TEAISH_MAKEFILE_IN "" + define TEAISH_MAKEFILE "" + } + + # Look for teaish.pkginit.tcl[.in] + set piPolicy 0 + if {[proj-first-file-found extI \ + [list \ + $dirExt/teaish.pkginit.tcl.in \ + $dirExt/teaish.pkginit.tcl \ + ]]} { + if {[string match *.in $extI]} { + # Generate teaish.pkginit.tcl from $extI. + define TEAISH_PKGINIT_TCL_IN $extI + define TEAISH_PKGINIT_TCL [file rootname [file tail $extI]] + set piPolicy 0x01 + } else { + # Assume static $extI. + define TEAISH_PKGINIT_TCL_IN "" + define TEAISH_PKGINIT_TCL $extI + set piPolicy 0x10 + } + apply $addDist $extI + teaish__verbose 1 msg-result "Extension post-load init = $extI" + define TEAISH_PKGINIT_TCL_TAIL \ + [file tail [get-define TEAISH_PKGINIT_TCL]]; # for use in pkgIndex.tcl.in + } + set ::teaish__Config(pkginit-policy) $piPolicy + + # Look for pkgIndex.tcl[.in]... + set piPolicy 0 + if {[proj-first-file-found extPI $dirExt/pkgIndex.tcl.in]} { + # Generate ./pkgIndex.tcl from $extPI. + define TEAISH_PKGINDEX_TCL_IN $extPI + define TEAISH_PKGINDEX_TCL [file rootname [file tail $extPI]] + apply $addDist $extPI + set piPolicy 0x01 + } elseif {$dirExt ne $dirSrc + && [proj-first-file-found extPI $dirSrc/pkgIndex.tcl.in]} { + # Generate ./pkgIndex.tcl from $extPI. + define TEAISH_PKGINDEX_TCL_IN $extPI + define TEAISH_PKGINDEX_TCL [file rootname [file tail $extPI]] + set piPolicy 0x02 + } elseif {[proj-first-file-found extPI $dirExt/pkgIndex.tcl]} { + # Assume $extPI's a static file and use it. + define TEAISH_PKGINDEX_TCL_IN "" + define TEAISH_PKGINDEX_TCL $extPI + apply $addDist $extPI + set piPolicy 0x10 + } + # Reminder: we have to delay removal of stale TEAISH_PKGINDEX_TCL + # and the proj-dot-ins-append of TEAISH_PKGINDEX_TCL_IN until much + # later in the process. + set ::teaish__Config(pkgindex-policy) $piPolicy + + # Look for teaish.test.tcl[.in] + proj-assert {"" ne $dirExt} + set flist [list $dirExt/teaish.test.tcl.in $dirExt/teaish.test.tcl] + if {[proj-first-file-found ttt $flist]} { + if {[string match *.in $ttt]} { + # Generate teaish.test.tcl from $ttt + set xt [file rootname [file tail $ttt]] + file delete -force -- $xt; # ensure no stale copy is used + define TEAISH_TEST_TCL $xt + define TEAISH_TEST_TCL_IN $ttt + } else { + define TEAISH_TEST_TCL $ttt + define TEAISH_TEST_TCL_IN "" + } + apply $addDist $ttt + } else { + define TEAISH_TEST_TCL "" + define TEAISH_TEST_TCL_IN "" + } + + # Look for _teaish.tester.tcl[.in] + set flist [list $dirExt/_teaish.tester.tcl.in $dirSrc/_teaish.tester.tcl.in] + if {[proj-first-file-found ttt $flist]} { + # Generate teaish.test.tcl from $ttt + set xt [file rootname [file tail $ttt]] + file delete -force -- $xt; # ensure no stale copy is used + define TEAISH_TESTER_TCL $xt + define TEAISH_TESTER_TCL_IN $ttt + if {[lindex $flist 0] eq $ttt} { + apply $addDist $ttt + } + unset ttt xt + } else { + if {[file exists [set ttt [file join $dirSrc _teaish.tester.tcl.in]]]} { + set xt [file rootname [file tail $ttt]] + define TEAISH_TESTER_TCL $xt + define TEAISH_TESTER_TCL_IN $ttt + } else { + define TEAISH_TESTER_TCL "" + define TEAISH_TESTER_TCL_IN "" + } + } + unset flist + + # TEAISH_OUT_OF_EXT_TREE = 1 if we're building from a dir other + # than the extension's home dir. + define TEAISH_OUT_OF_EXT_TREE \ + [expr {[file-normalize $::autosetup(builddir)] ne \ + [file-normalize $::teaish__Config(extension-dir)]}] + return 1 +}; # teaish__find_extension + +# +# @teaish-cflags-add ?-p|prepend? ?-define? cflags... +# +# Equivalent to [proj-define-amend TEAISH_CFLAGS {*}$args]. +# +proc teaish-cflags-add {args} { + proj-define-amend TEAISH_CFLAGS {*}$args +} + +# +# @teaish-define-to-cflag ?flags? defineName...|{defineName...} +# +# Uses [proj-define-to-cflag] to expand a list of [define] keys, each +# one a separate argument, to CFLAGS-style -D... form then appends +# that to the current TEAISH_CFLAGS. +# +# It accepts these flags from proj-define-to-cflag: -quote, +# -zero-undef. It does _not_ support its -list flag. +# +# It accepts its non-flag argument(s) in 2 forms: (1) each arg is a +# single [define] key or (2) its one arg is a list of such keys. +# +# TODO: document teaish's well-defined (as it were) defines for this +# purpose. At a bare minimum: +# +# - TEAISH_NAME +# - TEAISH_PKGNAME +# - TEAISH_VERSION +# - TEAISH_LIBDIR_NAME +# - TEAISH_LOAD_PREFIX +# - TEAISH_URL +# +proc teaish-define-to-cflag {args} { + set flags {} + while {[string match -* [lindex $args 0]]} { + set arg [lindex $args 0] + switch -exact -- $arg { + -quote - + -zero-undef { + lappend flags $arg + set args [lassign $args -] + } + default break + } + } + if {1 == [llength $args]} { + set args [list {*}[lindex $args 0]] + } + #puts "***** flags=$flags args=$args" + teaish-cflags-add [proj-define-to-cflag {*}$flags {*}$args] +} + +# +# @teaish-cflags-for-tea ?...CFLAGS? +# +# Adds several -DPACKAGE_... CFLAGS using the extension's metadata, +# all as quoted strings. Those symbolic names are commonly used in +# TEA-based builds, and this function is intended to simplify porting +# of such builds. The -D... flags added are: +# +# -DPACKAGE_VERSION=... +# -DPACKAGE_NAME=... +# -DPACKAGE_URL=... +# -DPACKAGE_STRING=... +# +# Any arguments are passed-on as-is to teaish-cflags-add. +# +proc teaish-cflags-for-tea {args} { + set name $::teaish__PkgInfo(-name) + set version $::teaish__PkgInfo(-version) + set pstr [join [list $name $version]] + teaish-cflags-add \ + {*}$args \ + '-DPACKAGE_VERSION="$version"' \ + '-DPACKAGE_NAME="$name"' \ + '-DPACKAGE_STRING="$pstr"' \ + '-DPACKAGE_URL="[teaish-get -url]"' +} + +# +# @teaish-ldflags-add ?-p|-prepend? ?-define? ldflags... +# +# Equivalent to [proj-define-amend TEAISH_LDFLAGS {*}$args]. +# +# Typically, -lXYZ flags need to be in "reverse" order, with each -lY +# resolving symbols for -lX's to its left. This order is largely +# historical, and not relevant on all environments, but it is +# technically correct and still relevant on some environments. +# +# See: teaish-ldflags-prepend +# +proc teaish-ldflags-add {args} { + proj-define-amend TEAISH_LDFLAGS {*}$args +} + +# +# @teaish-ldflags-prepend args... +# +# Functionally equivalent to [teaish-ldflags-add -p {*}$args] +# +proc teaish-ldflags-prepend {args} { + teaish-ldflags-add -p {*}$args +} + +# +# @teaish-src-add ?-dist? ?-dir? src-files... +# +# Appends all non-empty $args to the project's list of C/C++ source or +# (in some cases) object files. +# +# If passed -dist then it also passes each filename, as-is, to +# [teaish-dist-add]. +# +# If passed -dir then each src-file has [teaish-get -dir] prepended to +# it before they're added to the list. As often as not, that will be +# the desired behavior so that out-of-tree builds can find the +# sources, but there are cases where it's not desired (e.g. when using +# a source file from outside of the extension's dir, or when adding +# object files (which are typically in the build tree)). +# +proc teaish-src-add {args} { + set i 0 + proj-parse-simple-flags args flags { + -dist 0 {expr 1} + -dir 0 {expr 1} + } + if {$flags(-dist)} { + teaish-dist-add {*}$args + } + if {$flags(-dir)} { + set xargs {} + foreach arg $args { + if {"" ne $arg} { + lappend xargs [file join $::teaish__Config(extension-dir) $arg] + } + } + set args $xargs + } + lappend ::teaish__Config(extension-src) {*}$args +} + +# +# @teaish-dist-add files-or-dirs... +# +# Adds the given files to the list of files to include with the "make +# dist" rules. +# +# This is a no-op when the current build is not in the extension's +# directory, as dist support is disabled in out-of-tree builds. +# +# It is not legal to call this until [teaish-get -dir] has been +# reliably set (via teaish__find_extension). +# +proc teaish-dist-add {args} { + if {$::teaish__Config(blddir-is-extdir)} { + # ^^^ reminder: we ignore $::teaish__Config(dist-enabled) here + # because the client might want to implement their own dist + # rules. + #proj-warn "**** args=$args" + lappend ::teaish__Config(dist-files) {*}$args + } +} + +# teaish-install-add files... +# Equivalent to [proj-define-apend TEAISH_INSTALL_FILES ...]. +#proc teaish-install-add {args} { +# proj-define-amend TEAISH_INSTALL_FILES {*}$args +#} + +# +# @teash-make-add args... +# +# Appends makefile code to the TEAISH_MAKEFILE_CODE define. Each +# arg may be any of: +# +# -tab: emit a literal tab +# -nl: emit a literal newline +# -nltab: short for -nl -tab +# -bnl: emit a backslash-escaped end-of-line +# -bnltab: short for -eol -tab +# +# Anything else is appended verbatim. This function adds no additional +# spacing between each argument nor between subsequent invocations. +# Generally speaking, a series of calls to this function need to +# be sure to end the series with a newline. +proc teaish-make-add {args} { + set out [get-define TEAISH_MAKEFILE_CODE ""] + foreach a $args { + switch -exact -- $a { + -bnl { set a " \\\n" } + -bnltab { set a " \\\n\t" } + -tab { set a "\t" } + -nl { set a "\n" } + -nltab { set a "\n\t" } + } + append out $a + } + define TEAISH_MAKEFILE_CODE $out +} + +# Internal helper to generate a clean/distclean rule name +proc teaish__cleanup_rule {{tgt clean}} { + set x [incr ::teaish__Config(teaish__cleanup_rule-counter-${tgt})] + return ${tgt}-_${x}_ +} + +# @teaish-make-obj objfile srcfile ?...args? +# +# Uses teaish-make-add to inject makefile rules for $objfile from +# $srcfile, which is assumed to be C code which uses libtcl. Unless +# -recipe is used (see below) it invokes the compiler using the +# makefile-defined $(CC.tcl) which, in the default Makefile.in +# template, includes any flags needed for building against the +# configured Tcl. +# +# This always terminates the resulting code with a newline. +# +# Any arguments after the 2nd may be flags described below or, if no +# -recipe is provided, flags for the compiler call. +# +# -recipe {...} +# Uses the trimmed value of {...} as the recipe, prefixing it with +# a single hard-tab character. +# +# -deps {...} +# List of extra files to list as dependencies of $o. Good luck +# escaping non-trivial cases properly. +# +# -clean +# Generate cleanup rules as well. +proc teaish-make-obj {o src args} { + set consume 0 + set clean 0 + set flag "" + array set flags {} + set xargs {} + foreach arg $args { + if {$consume} { + set consume 0 + set flags($flag) $arg + continue + } + switch -exact -- $arg { + -clean {incr clean} + -recipe - + -deps { + set flag $arg + incr consume + } + default { + lappend xargs $arg + } + } + } + teaish-make-add \ + "# [proj-scope 1] -> [proj-scope] $o $src" -nl \ + "$o: $src $::teaish__Config(teaish.tcl)" + if {[info exists flags(-deps)]} { + teaish-make-add " " [join $flags(-deps)] + } + teaish-make-add -nltab + if {[info exists flags(-recipe)]} { + teaish-make-add [string trim $flags(-recipe)] -nl + } else { + teaish-make-add [join [list \$(CC.tcl) -c $src {*}$xargs]] -nl + } + if {$clean} { + set rule [teaish__cleanup_rule] + teaish-make-add \ + "clean: $rule\n$rule:\n\trm -f \"$o\"\n" + } +} + +# +# @teaish-make-clean ?-r? ?-dist? ...files|{...files} +# +# Adds makefile rules for cleaning up the given files via the "make +# clean" or (if -dist is used) "make distclean" makefile rules. The -r +# flag uses "rm -fr" instead of "rm -f", so be careful with that. +# +# The file names are taken literally as arguments to "rm", so they may +# be shell wildcards to be resolved at cleanup-time. To clean up whole +# directories, pass the -r flag. Each name gets quoted in +# double-quotes, so spaces in names should not be a problem (but +# double-quotes in names will be). +# +proc teaish-make-clean {args} { + if {1 == [llength $args]} { + set args [list {*}[lindex $args 0]] + } + + set tgt clean + set rmflags "-f" + proj-parse-simple-flags args flags { + -dist 0 { + set tgt distclean + } + -r 0 { + set rmflags "-fr" + } + } + set rule [teaish__cleanup_rule $tgt] + teaish-make-add "# [proj-scope 1] -> [proj-scope]: [join $args]\n" + teaish-make-add "${rule}:\n\trm ${rmflags}" + foreach a $args { + teaish-make-add " \"$a\"" + } + teaish-make-add "\n${tgt}: ${rule}\n" +} + +# +# @teaish-make-config-header filename +# +# Invokes autosetup's [make-config-header] and passes it $filename and +# a relatively generic list of options for controlling which defined +# symbols get exported. Clients which need more control over the +# exports can copy/paste/customize this. +# +# The exported file is then passed to [proj-touch] because, in +# practice, that's sometimes necessary to avoid build dependency +# issues. +# +proc teaish-make-config-header {filename} { + make-config-header $filename \ + -none {HAVE_CFLAG_* LDFLAGS_* SH_* TEAISH__* TEAISH_*_CODE} \ + -auto {SIZEOF_* HAVE_* TEAISH_* TCL_*} \ + -none * + proj-touch $filename; # help avoid frequent unnecessary auto-reconfig +} + +# +# @teaish-feature-cache-set $key value +# +# Sets a feature-check cache entry with the given key. +# See proj-cache-set for the key's semantics. $key should +# normally be 0. +# +proc teaish-feature-cache-set {key val} { + proj-cache-set -key $key -level 1 $val +} + +# +# @teaish-feature-cache-check key tgtVarName +# +# Checks for a feature-check cache entry with the given key. +# See proj-cache-set for the key's semantics. +# +# $key should also almost always be 0 but, due to a tclsh +# incompatibility in 1 OS, it cannot have a default value unless it's +# the second argument (but it should be the first one). +# +# If the feature-check cache has a matching entry then this function +# assigns its value to tgtVar and returns 1, else it assigns tgtVar to +# "" and returns 0. +# +# See proj-cache-check for $key's semantics. +# +proc teaish-feature-cache-check {key tgtVar} { + upvar $tgtVar tgt + proj-cache-check -key $key -level 1 tgt +} + +# +# @teaish-check-cached@ ?flags? msg script... +# +# A proxy for feature-test impls which handles caching of a feature +# flag check on per-function basis, using the calling scope's name as +# the cache key. +# +# It emits [msg-checking $msg]. If $msg is empty then it defaults to +# the name of the caller's scope. The -nomsg flag suppresses the +# message for non-cache-hit checks. At the end, it will [msg-result +# "ok"] [msg-result "no"] unless -nostatus is used, in which case the +# caller is responsible for emitting at least a newline when it's +# done. The -msg-0 and -msg-1 flags can be used to change the ok/no +# text. +# +# This function checks for a cache hit before running $script and +# caching the result. If no hit is found then $script is run in the +# calling scope and its result value is stored in the cache. This +# routine will intercept a 'return' from $script. +# +# $script may be a command and its arguments, as opposed to a single +# script block. +# +# Flags: +# +# -nostatus = do not emit "ok" or "no" at the end. This presumes +# that either $script will emit at least one newline before +# returning or the caller will account for it. Because of how this +# function is typically used, -nostatus is not honored when the +# response includes a cached result. +# +# -quiet = disable output from Autosetup's msg-checking and +# msg-result for the duration of the $script check. Note that when +# -quiet is in effect, Autosetup's user-notice can be used to queue +# up output to appear after the check is done. Also note that +# -quiet has no effect on _this_ function, only the $script part. +# +# -nomsg = do not emit $msg for initial check. Like -nostatus, this +# flag is not honored when the response includes a cached result +# because it would otherwise produce no output (which is confusing +# in this context). This is useful when a check runs several other +# verbose checks and they emit all the necessary info. +# +# -msg-0 and -msg-1 MSG = strings to show when the check has failed +# resp. passed. Defaults are "no" and "ok". The 0 and 1 refer to the +# result value from teaish-feature-cache-check. +# +# -key cachekey = set the cache context key. Only needs to be +# explicit when using this function multiple times from a single +# scope. See proj-cache-check and friends for details on the key +# name. Its default is the name of the scope which calls this +# function. +# +proc teaish-check-cached {args} { + proj-parse-simple-flags args flags { + -nostatus 0 {expr 1} + -quiet 0 {expr 1} + -key => 1 + -nomsg 0 {expr 1} + -msg-0 => no + -msg-1 => ok + } + set args [lassign $args msg] + set script [join $args] + if {"" eq $msg} { + set msg [proj-scope 1] + } + if {[teaish-feature-cache-check $flags(-key) check]} { + #if {0 == $flags(-nomsg)} { + msg-checking "${msg} ... (cached) " + #} + #if {!$flags(-nostatus)} { + msg-result $flags(-msg-[expr {0 != ${check}}]) + #} + return $check + } else { + if {0 == $flags(-nomsg)} { + msg-checking "${msg} ... " + } + if {$flags(-quiet)} { + incr ::autosetup(msg-quiet) + } + set code [catch {uplevel 1 $script} rc xopt] + if {$flags(-quiet)} { + incr ::autosetup(msg-quiet) -1 + } + #puts "***** cached-check got code=$code rc=$rc" + if {$code in {0 2}} { + teaish-feature-cache-set 1 $rc + if {!$flags(-nostatus)} { + msg-result $flags(-msg-[expr {0 != ${rc}}]) + } else { + #show-notices; # causes a phantom newline because we're in a + #msg-checking scope, so... + if {[info exists ::autosetup(notices)]} { + show-notices + } + } + } else { + #puts "**** code=$code rc=$rc xopt=$xopt" + teaish-feature-cache-set 1 0 + } + #puts "**** code=$code rc=$rc" + return {*}$xopt $rc + } +} + +# +# Internal helper for teaish__defs_format_: returns a JSON-ish quoted +# form of the given string-type values. +# +# If $asList is true then the return value is in {$value} form. If +# $asList is false it only performs the most basic of escaping and +# the input must not contain any control characters. +# +proc teaish__quote_str {asList value} { + if {$asList} { + return "{${value}}" + } + return \"[string map [list \\ \\\\ \" \\\"] $value]\" +} + +# +# Internal helper for teaish__defines_to_list. Expects to be passed +# a name and the variadic $args which are passed to +# teaish__defines_to_list.. If it finds a pattern match for the +# given $name in the various $args, it returns the type flag for that +# $name, e.g. "-str" or "-bare", else returns an empty string. +# +proc teaish__defs_type {name spec} { + foreach {type patterns} $spec { + foreach pattern $patterns { + if {[string match $pattern $name]} { + return $type + } + } + } + return "" +} + +# +# An internal impl detail. Requires a data type specifier, as used by +# Autosetup's [make-config-header], and a value. Returns the formatted +# value or the value $::teaish__Config(defs-skip) if the caller should +# skip emitting that value. +# +# In addition to -str, -auto, etc., as defined by make-config-header, +# it supports: +# +# -list {...} will cause non-integer values to be quoted in {...} +# instead of quotes. +# +# -autolist {...} works like -auto {...} except that it falls back to +# -list {...} type instead of -str {...} style for non-integers. +# +# -jsarray {...} emits the output in something which, for +# conservative inputs, will be a valid JSON array. It can only +# handle relatively simple values with no control characters in +# them. +# +set teaish__Config(defs-skip) "-teaish__defs_format sentinel" +proc teaish__defs_format {type value} { + switch -exact -- $type { + -bare { + # Just output the value unchanged + } + -none { + set value $::teaish__Config(defs-skip) + } + -str { + set value [teaish__quote_str 0 $value] + } + -auto { + # Automatically determine the type + if {![string is integer -strict $value]} { + set value [teaish__quote_str 0 $value] + } + } + -autolist { + if {![string is integer -strict $value]} { + set value [teaish__quote_str 1 $value] + } + } + -list { + set value [teaish__quote_str 1 $value] + } + -jsarray { + set ar {} + foreach v $value { + if {![string is integer -strict $v]} { + set v [teaish__quote_str 0 $v] + } + if {$::teaish__Config(defs-skip) ne $v} { + lappend ar $v + } + } + set value [concat \[ [join $ar {, }] \]] + } + "" { + # (Much later:) Why do we do this? + set value $::teaish__Config(defs-skip) + } + default { + proj-error \ + "Unknown [proj-scope] -type ($type) called from" \ + [proj-scope 1] + } + } + return $value +} + +# +# Returns Tcl code in the form of code which evaluates to a list of +# configure-time DEFINEs in the form {key val key2 val...}. It may +# misbehave for values which are not numeric or simple strings. Some +# defines are specifically filtered out of the result, either because +# their irrelevant to teaish or because they may be arbitrarily large +# (e.g. makefile content). +# +# The $args are explained in the docs for internal-use-only +# [teaish__defs_format]. The default mode is -autolist. +# +proc teaish__defines_to_list {args} { + set lines {} + lappend lines "\{" + set skipper $::teaish__Config(defs-skip) + set args [list \ + -none { + TEAISH__* + TEAISH_*_CODE + AM_* AS_* + } \ + {*}$args \ + -autolist *] + foreach d [lsort [dict keys [all-defines]]] { + set type [teaish__defs_type $d $args] + set value [teaish__defs_format $type [get-define $d]] + if {$skipper ne $value} { + lappend lines "$d $value" + } + } + lappend lines "\}" + tailcall join $lines "\n" +} + +# +# teaish__pragma ...flags +# +# Offers a way to tweak how teaish's core behaves in some cases, in +# particular those which require changing how the core looks for an +# extension and its files. +# +# Accepts the following flags. Those marked with [L] are safe to use +# during initial loading of tclish.tcl (recall that most teaish APIs +# cannot be used until [teaish-configure] is called). +# +# static-pkgIndex.tcl [L]: Tells teaish that ./pkgIndex.tcl is not +# a generated file, so it will not try to overwrite or delete +# it. Errors out if it does not find pkgIndex.tcl in the +# extension's dir. +# +# no-dist [L]: tells teaish to elide the 'make dist' recipe +# from the generated Makefile. +# +# no-dll [L]: tells teaish to elide the DLL-building recipe +# from the generated Makefile. +# +# no-vsatisfies-error [L]: tells teaish that failure to match the +# -vsatisfies value should simply "return" instead of "error". +# +# no-tester [L]: disables automatic generation of teaish.test.tcl +# even if a copy of _teaish.tester.tcl.in is found. +# +# no-full-dist [L]: changes the "make dist" rules to never include +# a copy of teaish itself. By default it will include itself only +# if the extension lives in the same directory as teaish. +# +# full-dist [L]: changes the "make dist" rules to always include +# a copy of teaish itself. +# +# Emits a warning message for unknown arguments. +# +proc teaish__pragma {args} { + foreach arg $args { + switch -exact -- $arg { + + static-pkgIndex.tcl { + if {$::teaish__Config(tm-policy)} { + proj-fatal -up "Cannot use pragma $arg together with -tm.tcl or -tm.tcl.in." + } + set tpi [file join $::teaish__Config(extension-dir) pkgIndex.tcl] + if {[file exists $tpi]} { + define TEAISH_PKGINDEX_TCL_IN "" + define TEAISH_PKGINDEX_TCL $tpi + set ::teaish__Config(pkgindex-policy) 0x20 + } else { + proj-error "pragma $arg: found no package-local pkgIndex.tcl\[.in]" + } + } + + no-dist { + set ::teaish__Config(dist-enabled) 0 + } + + no-install { + set ::teaish__Config(install-enabled) 0 + } + + full-dist { + set ::teaish__Config(dist-full-enabled) 1 + } + + no-full-dist { + set ::teaish__Config(dist-full-enabled) 0 + } + + no-dll { + set ::teaish__Config(dll-enabled) 0 + } + + no-vsatisfies-error { + set ::teaish__Config(vsatisfies-error) 0 + } + + no-tester { + define TEAISH_TESTER_TCL_IN "" + define TEAISH_TESTER_TCL "" + } + + default { + proj-error "Unknown flag: $arg" + } + } + } +} + +# +# @teaish-pkginfo-set ...flags +# +# The way to set up the initial package state. Used like: +# +# teaish-pkginfo-set -name foo -version 0.1.2 +# +# Or: +# +# teaish-pkginfo-set ?-vars|-subst? {-name foo -version 0.1.2} +# +# The latter may be easier to write for a multi-line invocation. +# +# For the second call form, passing the -vars flag tells it to perform +# a [subst] of (only) variables in the {...} part from the calling +# scope. The -subst flag will cause it to [subst] the {...} with +# command substitution as well (but no backslash substitution). When +# using -subst for string concatenation, e.g. with -libDir +# foo[get-version-number], be sure to wrap the value in braces: +# -libDir {foo[get-version-number]}. +# +# Each pkginfo flag corresponds to one piece of extension package +# info. Teaish provides usable default values for all of these flags, +# but at least the -name and -version should be set by clients. +# e.g. the default -name is the directory name the extension lives in, +# which may change (e.g. when building it from a "make dist" bundle). +# +# The flags: +# +# -name theName: The extension's name. It defaults to the name of the +# directory containing the extension. (In TEA this would be the +# PACKAGE_NAME, not to be confused with...) +# +# -name.pkg pkg-provide-name: The extension's name for purposes of +# Tcl_PkgProvide(), [package require], and friends. It defaults to +# the `-name`, and is normally the same, but some projects (like +# SQLite) have a different name here than they do in their +# historical TEA PACKAGE_NAME. +# +# -version version: The extension's package version. Defaults to +# 0.0.0. +# +# -libDir dirName: The base name of the directory into which this +# extension should be installed. It defaults to a concatenation of +# `-name.pkg` and `-version`. +# +# -loadPrefix prefix: For use as the second argument passed to +# Tcl's `load` command in the package-loading process. It defaults +# to title-cased `-name.pkg` because Tcl's `load` plugin system +# expects it in that form. +# +# -options {...}: If provided, it must be a list compatible with +# Autosetup's `options-add` function. These can also be set up via +# `teaish-options`. +# +# -vsatisfies {{...} ...}: Expects a list-of-lists of conditions +# for Tcl's `package vsatisfies` command: each list entry is a +# sub-list of `{PkgName Condition...}`. Teaish inserts those +# checks via its default pkgIndex.tcl.in and _teaish.tester.tcl.in +# templates to verify that the system's package dependencies meet +# these requirements. The default value is `{{Tcl 8.5-}}` (recall +# that it's a list-of-lists), as 8.5 is the minimum Tcl version +# teaish will run on, but some extensions may require newer +# versions or dependencies on other packages. As a special case, +# if `-vsatisfies` is given a single token, e.g. `8.6-`, then it +# is transformed into `{Tcl $thatToken}`, i.e. it checks the Tcl +# version which the package is being run with. If given multiple +# lists, each `package provides` check is run in the given +# order. Failure to meet a `vsatisfies` condition triggers an +# error. +# +# -url {...}: an optional URL for the extension. +# +# -pragmas {...} A list of infrequently-needed lower-level +# directives which can influence teaish, including: +# +# static-pkgIndex.tcl: tells teaish that the client manages their +# own pkgIndex.tcl, so that teaish won't try to overwrite it +# using a template. +# +# no-dist: tells teaish to elide the "make dist" recipe from the +# makefile so that the client can implement it. +# +# no-dll: tells teaish to elide the makefile rules which build +# the DLL, as well as any templated test script and pkgIndex.tcl +# references to them. The intent here is to (A) support +# client-defined build rules for the DLL and (B) eventually +# support script-only extensions. +# +# Unsupported flags or pragmas will trigger an error. +# +# Potential pothole: setting certain state, e.g. -version, after the +# initial call requires recalculating of some [define]s. Any such +# changes should be made as early as possible in teaish-configure so +# that any later use of those [define]s gets recorded properly (not +# with the old value). This is particularly relevant when it is not +# possible to determine the -version or -name until teaish-configure +# has been called, and it's updated dynamically from +# teaish-configure. Notably: +# +# - If -version or -name are updated, -libDir will almost certainly +# need to be explicitly set along with them. +# +# - If -name is updated, -loadPrefix probably needs to be as well. +# +proc teaish-pkginfo-set {args} { + set doVars 0 + set doCommands 0 + set xargs $args + set recalc {} + foreach arg $args { + switch -exact -- $arg { + -vars { + incr doVars + set xargs [lassign $xargs -] + } + -subst { + incr doVars + incr doCommands + set xargs [lassign $xargs -] + } + default { + break + } + } + } + set args $xargs + unset xargs + if {1 == [llength $args] && [llength [lindex $args 0]] > 1} { + # Transform a single {...} arg into the canonical call form + set a [list {*}[lindex $args 0]] + if {$doVars || $doCommands} { + set sflags -nobackslashes + if {!$doCommands} { + lappend sflags -nocommands + } + set a [uplevel 1 [list subst {*}$sflags $a]] + } + set args $a + } + set sentinel "" + set flagDefs [list] + foreach {f d} $::teaish__Config(pkginfo-f2d) { + lappend flagDefs $f => $sentinel + } + proj-parse-simple-flags args flags $flagDefs + if {[llength $args]} { + proj-error -up "Too many (or unknown) arguments to [proj-scope]: $args" + } + foreach {f d} $::teaish__Config(pkginfo-f2d) { + if {$sentinel eq [set v $flags($f)]} continue + switch -exact -- $f { + + -options { + proj-assert {"" eq $d} + options-add $v + } + + -pragmas { + teaish__pragma {*}$v + } + + -vsatisfies { + if {1 == [llength $v] && 1 == [llength [lindex $v 0]]} { + # Transform X to {Tcl $X} + set v [list [join [list Tcl $v]]] + } + define $d $v + } + + -pkgInit.tcl - + -pkgInit.tcl.in { + if {0x22 & $::teaish__Config(pkginit-policy)} { + proj-fatal "Cannot use -pkgInit.tcl(.in) more than once." + } + set x [file join $::teaish__Config(extension-dir) $v] + set tTail [file tail $v] + if {"-pkgInit.tcl.in" eq $f} { + # Generate pkginit file X from X.in + set pI 0x02 + set tIn $x + set tOut [file rootname $tTail] + set other -pkgInit.tcl + } else { + # Static pkginit file X + set pI 0x20 + set tIn "" + set tOut $x + set other -pkgInit.tcl.in + } + set ::teaish__Config(pkginit-policy) $pI + set ::teaish__PkgInfo($other) {} + define TEAISH_PKGINIT_TCL_IN $tIn + define TEAISH_PKGINIT_TCL $tOut + define TEAISH_PKGINIT_TCL_TAIL $tTail + teaish-dist-add $v + set v $x + } + + -tm.tcl - + -tm.tcl.in { + if {0x30 & $::teaish__Config(pkgindex-policy)} { + proj-fatal "Cannot use $f together with a pkgIndex.tcl." + } elseif {$::teaish__Config(tm-policy)} { + proj-fatal "Cannot use -tm.tcl(.in) more than once." + } + set x [file join $::teaish__Config(extension-dir) $v] + if {"-tm.tcl.in" eq $f} { + # Generate tm file X from X.in + set pT 0x02 + set pI 0x100 + set tIn $x + set tOut [file rootname [file tail $v]] + set other -tm.tcl + } else { + # Static tm file X + set pT 0x20 + set pI 0x200 + set tIn "" + set tOut $x + set other -tm.tcl.in + } + set ::teaish__Config(pkgindex-policy) $pI + set ::teaish__Config(tm-policy) $pT + set ::teaish__PkgInfo($other) {} + define TEAISH_TM_TCL_IN $tIn + define TEAISH_TM_TCL $tOut + define TEAISH_PKGINDEX_TCL "" + define TEAISH_PKGINDEX_TCL_IN "" + define TEAISH_PKGINDEX_TCL_TAIL "" + teaish-dist-add $v + teaish__pragma no-dll + set v $x + } + + default { + proj-assert {"" ne $d} + define $d $v + } + } + set ::teaish__PkgInfo($f) $v + if {$f in {-name -version -libDir -loadPrefix}} { + lappend recalc $f + } + } + if {"" ne $recalc} { + teaish__define_pkginfo_derived $recalc + } +} + +# +# @teaish-pkginfo-get ?arg? +# +# If passed no arguments, it returns the extension config info in the +# same form accepted by teaish-pkginfo-set. +# +# If passed one -flagname arg then it returns the value of that config +# option. +# +# Else it treats arg as the name of caller-scoped variable to +# which this function assigns an array containing the configuration +# state of this extension, in the same structure accepted by +# teaish-pkginfo-set. In this case it returns an empty string. +# +proc teaish-pkginfo-get {args} { + set cases {} + set argc [llength $args] + set rv {} + switch -exact $argc { + 0 { + # Return a list of (-flag value) pairs + lappend cases default {{ + if {[info exists ::teaish__PkgInfo($flag)]} { + lappend rv $flag $::teaish__PkgInfo($flag) + } else { + lappend rv $flag [get-define $defName] + } + }} + } + + 1 { + set arg $args + if {[string match -* $arg]} { + # Return the corresponding -flag's value + lappend cases $arg {{ + if {[info exists ::teaish__PkgInfo($flag)]} { + return $::teaish__PkgInfo($flag) + } else { + return [get-define $defName] + } + }} + } else { + # Populate target with an array of (-flag value). + upvar $arg tgt + array set tgt {} + lappend cases default {{ + if {[info exists ::teaish__PkgInfo($flag)]} { + set tgt($flag) $::teaish__PkgInfo($flag) + } else { + set tgt($flag) [get-define $defName] + } + }} + } + } + + default { + proj-error "invalid arg count from [proj-scope 1]" + } + } + + foreach {flag defName} $::teaish__Config(pkginfo-f2d) { + switch -exact -- $flag [join $cases] + } + if {0 == $argc} { return $rv } +} + +# (Re)set some defines based on pkginfo state. $flags is the list of +# pkginfo -flags which triggered this, or "*" for the initial call. +proc teaish__define_pkginfo_derived {flags} { + set all [expr {{*} in $flags}] + if {$all || "-version" in $flags || "-name" in $flags} { + set name $::teaish__PkgInfo(-name) ; # _not_ -name.pkg + if {[info exists ::teaish__PkgInfo(-version)]} { + set pkgver $::teaish__PkgInfo(-version) + set libname "lib" + if {[string match *-cygwin [get-define host]]} { + set libname cyg + } + define TEAISH_DLL8_BASENAME $libname$name$pkgver + define TEAISH_DLL9_BASENAME ${libname}tcl9$name$pkgver + set ext [get-define TARGET_DLLEXT] + define TEAISH_DLL8 [get-define TEAISH_DLL8_BASENAME]$ext + define TEAISH_DLL9 [get-define TEAISH_DLL9_BASENAME]$ext + } + } + if {$all || "-libDir" in $flags} { + if {[info exists ::teaish__PkgInfo(-libDir)]} { + define TCLLIBDIR \ + [file dirname [get-define TCLLIBDIR]]/$::teaish__PkgInfo(-libDir) + } + } +} + +# +# @teaish-checks-queue -pre|-post args... +# +# Queues one or more arbitrary "feature test" functions to be run when +# teaish-checks-run is called. $flag must be one of -pre or -post to +# specify whether the tests should be run before or after +# teaish-configure is run. Each additional arg is the name of a +# feature-test proc. +# +proc teaish-checks-queue {flag args} { + if {$flag ni {-pre -post}} { + proj-error "illegal flag: $flag" + } + lappend ::teaish__Config(queued-checks${flag}) {*}$args +} + +# +# @teaish-checks-run -pre|-post +# +# Runs all feature checks queued using teaish-checks-queue +# then cleares the queue. +# +proc teaish-checks-run {flag} { + if {$flag ni {-pre -post}} { + proj-error "illegal flag: $flag" + } + #puts "*** running $flag: $::teaish__Config(queued-checks${flag})" + set foo 0 + foreach f $::teaish__Config(queued-checks${flag}) { + if {![teaish-feature-cache-check $f foo]} { + set v [$f] + teaish-feature-cache-set $f $v + } + } + set ::teaish__Config(queued-checks${flag}) {} +} + +# +# A general-purpose getter for various teaish state. Requires one +# flag, which determines its result value. Flags marked with [L] below +# are safe for using at load-time, before teaish-pkginfo-set is called +# +# -dir [L]: returns the extension's directory, which may differ from +# the teaish core dir or the build dir. +# +# -teaish-home [L]: the "home" dir of teaish itself, which may +# differ from the extension dir or build dir. +# +# -build-dir [L]: the build directory (typically the current working +# -dir). +# +# Any of the teaish-pkginfo-get/get flags: returns the same as +# teaish-pkginfo-get. Not safe for use until teaish-pkginfo-set has +# been called. +# +# Triggers an error if passed an unknown flag. +# +proc teaish-get {flag} { + #-teaish.tcl {return $::teaish__Config(teaish.tcl)} + switch -exact -- $flag { + -dir { + return $::teaish__Config(extension-dir) + } + -teaish-home { + return $::autosetup(srcdir) + } + -build-dir { + return $::autosetup(builddir) + } + default { + if {[info exists ::teaish__PkgInfo($flag)]} { + return $::teaish__PkgInfo($flag) + } + } + } + proj-error "Unhandled flag: $flag" +} + +# +# Handles --teaish-create-extension=TARGET-DIR +# +proc teaish__create_extension {dir} { + set force [opt-bool teaish-force] + if {"" eq $dir} { + proj-error "--teaish-create-extension=X requires a directory name." + } + file mkdir $dir/generic + set cwd [pwd] + #set dir [file-normalize [file join $cwd $dir]] + teaish__verbose 1 msg-result "Created dir $dir" + cd $dir + if {!$force} { + # Ensure that we don't blindly overwrite anything + foreach f { + generic/teaish.c + teaish.tcl + teaish.make.in + teaish.test.tcl + } { + if {[file exists $f]} { + error "Cowardly refusing to overwrite $dir/$f. Use --teaish-force to overwrite." + } + } + } + + set name [file tail $dir] + set pkgName $name + set version 0.0.1 + set loadPrefix [string totitle $pkgName] + set content {teaish-pkginfo-set } + #puts "0 content=$content" + if {[opt-str teaish-extension-pkginfo epi]} { + set epi [string trim $epi] + if {[string match "*\n*" $epi]} { + set epi "{$epi}" + } elseif {![string match "{*}" $epi]} { + append content "\{" $epi "\}" + } else { + append content $epi + } + #puts "2 content=$content\nepi=$epi" + } else { + append content [subst -nocommands -nobackslashes {{ + -name ${name} + -name.pkg ${pkgName} + -name.dist ${pkgName} + -version ${version} + -loadPrefix $loadPrefix + -libDir ${name}${version} + -vsatisfies {{Tcl 8.5-}} + -url {} + -options {} + -pragmas {full-dist} + }}] + #puts "3 content=$content" + } + #puts "1 content=$content" + append content "\n" { +#proc teaish-options {} { + # Return a list and/or use \[options-add\] to add new + # configure flags. This is called before teaish's + # bootstrapping is finished, so only teaish-* + # APIs which are explicitly noted as being safe + # early on may be used here. Any autosetup-related + # APIs may be used here. + # + # Return an empty string if there are no options to + # add or if they are added using \[options-add\]. + # + # If there are no options to add, this proc need + # not be defined. +#} + +# Called by teaish once bootstrapping is complete. +# This function is responsible for the client-specific +# parts of the configuration process. +proc teaish-configure {} { + teaish-src-add -dir -dist generic/teaish.c + teaish-define-to-cflag -quote TEAISH_PKGNAME TEAISH_VERSION + + # TODO: your code goes here.. +} +}; # $content + proj-file-write teaish.tcl $content + teaish__verbose 1 msg-result "Created teaish.tcl" + + set content "# Teaish test script. +# When this tcl script is invoked via 'make test' it will have loaded +# the package, run any teaish.pkginit.tcl code, and loaded +# autosetup/teaish/tester.tcl. +" + proj-file-write teaish.test.tcl $content + teaish__verbose 1 msg-result "Created teaish.test.tcl" + + set content [subst -nocommands -nobackslashes { +#include +static int +${loadPrefix}_Cmd(ClientData cdata, Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]){ + Tcl_SetObjResult(interp, Tcl_NewStringObj("this is the ${name} extension", -1)); + return TCL_OK; +} + +extern int DLLEXPORT ${loadPrefix}_Init(Tcl_Interp *interp){ + if (Tcl_InitStubs(interp, TCL_VERSION, 0) == NULL) { + return TCL_ERROR; + } + if (Tcl_PkgProvide(interp, TEAISH_PKGNAME, TEAISH_VERSION) == TCL_ERROR) { + return TCL_ERROR; + } + Tcl_CreateObjCommand(interp, TEAISH_PKGNAME, ${loadPrefix}_Cmd, NULL, NULL); + return TCL_OK; +} +}] + proj-file-write generic/teaish.c $content + teaish__verbose 1 msg-result "Created generic/teaish.c" + + set content "# teaish makefile for the ${name} extension +# tx.src = \$(tx.dir)/generic/teaish.c +# tx.LDFLAGS = +# tx.CFLAGS = +" + proj-file-write teaish.make.in $content + teaish__verbose 1 msg-result "Created teaish.make.in" + + msg-result "Created new extension \[$dir\]." + + cd $cwd + set ::teaish__Config(install-ext-dir) $dir +} + +# +# Internal helper for teaish__install +# +proc teaish__install_file {f destDir force} { + set dest $destDir/[file tail $f] + if {[file isdirectory $f]} { + file mkdir $dest + } elseif {!$force && [file exists $dest]} { + array set st1 [file stat $f] + array set st2 [file stat $dest] + if {($st1(mtime) == $st2(mtime)) + && ($st1(size) == $st2(size))} { + if {[file tail $f] in { + pkgIndex.tcl.in + _teaish.tester.tcl.in + }} { + # Assume they're the same. In the scope of the "make dist" + # rules, this happens legitimately when an extension with a + # copy of teaish installed in the same dir assumes that the + # pkgIndex.tcl.in and _teaish.tester.tcl.in belong to the + # extension, whereas teaish believes they belong to teaish. + # So we end up with dupes of those. + return + } + } + proj-error -up "Cowardly refusing to overwrite \[$dest\]." \ + "Use --teaish-force to enable overwriting." + } else { + # file copy -force $f $destDir; # loses +x bit + # + # JimTcl doesn't have [file attribute], so we can't use that here + # (in the context of an autosetup configure script). + exec cp -p $f $dest + } +} + +# +# Installs a copy of teaish, with autosetup, to $dDest, which defaults +# to the --teaish-install=X or --teash-create-extension=X dir. Won't +# overwrite files unless --teaish-force is used. +# +proc teaish__install {{dDest ""}} { + if {$dDest in {auto ""}} { + set dDest [opt-val teaish-install] + if {$dDest in {auto ""}} { + if {[info exists ::teaish__Config(install-ext-dir)]} { + set dDest $::teaish__Config(install-ext-dir) + } + } + } + set force [opt-bool teaish-force] + if {$dDest in {auto ""}} { + proj-error "Cannot determine installation directory." + } elseif {!$force && [file exists $dDest/auto.def]} { + proj-error \ + "Target dir looks like it already contains teaish and/or autosetup: $dDest" \ + "\nUse --teaish-force to overwrite it." + } + + set dSrc $::autosetup(srcdir) + set dAS $::autosetup(libdir) + set dAST $::teaish__Config(core-dir) + set dASTF $dAST/feature + teaish__verbose 1 msg-result "Installing teaish to \[$dDest\]..." + if {$::teaish__Config(verbose)>1} { + msg-result "dSrc = $dSrc" + msg-result "dAS = $dAS" + msg-result "dAST = $dAST" + msg-result "dASTF = $dASTF" + msg-result "dDest = $dDest" + } + + # Dest subdirs... + set ddAS $dDest/autosetup + set ddAST $ddAS/teaish + set ddASTF $ddAST/feature + foreach {srcDir destDir} [list \ + $dAS $ddAS \ + $dAST $ddAST \ + $dASTF $ddASTF \ + ] { + teaish__verbose 1 msg-result "Copying files to $destDir..." + file mkdir $destDir + foreach f [glob -directory $srcDir *] { + if {[string match {*~} $f] || [string match "#*#" [file tail $f]]} { + # Editor-generated backups and emacs lock files + continue + } + teaish__verbose 2 msg-result "\t$f" + teaish__install_file $f $destDir $force + } + } + teaish__verbose 1 msg-result "Copying files to $dDest..." + foreach f { + auto.def configure Makefile.in pkgIndex.tcl.in + _teaish.tester.tcl.in + } { + teaish__verbose 2 msg-result "\t$f" + teaish__install_file $dSrc/$f $dDest $force + } + set ::teaish__Config(install-self-dir) $dDest + msg-result "Teaish $::teaish__Config(version) installed in \[$dDest\]." +} diff --git a/autosetup/teaish/feature.tcl b/autosetup/teaish/feature.tcl new file mode 100644 index 000000000..6c927d1a7 --- /dev/null +++ b/autosetup/teaish/feature.tcl @@ -0,0 +1,214 @@ +######################################################################## +# 2025 April 7 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# * May you do good and not evil. +# * May you find forgiveness for yourself and forgive others. +# * May you share freely, never taking more than you give. +# +######################################################################## +# ----- @module feature-tests.tcl ----- +# @section TEA-ish collection of feature tests. +# +# Functions in this file with a prefix of teaish__ are +# private/internal APIs. Those with a prefix of teaish- are +# public APIs. + + +# @teaish-check-libz +# +# Checks for zlib.h and the function deflate in libz. If found, +# prepends -lz to the extension's ldflags and returns 1, else returns +# 0. It also defines LDFLAGS_LIBZ to the libs flag. +# +proc teaish-check-libz {} { + teaish-check-cached "Checking for libz" { + set rc 0 + if {[msg-quiet cc-check-includes zlib.h] && [msg-quiet proj-check-function-in-lib deflate z]} { + teaish-ldflags-prepend [define LDFLAGS_LIBZ [get-define lib_deflate]] + undefine lib_deflate + incr rc + } + expr $rc + } +} + +# @teaish-check-librt ?funclist? +# +# Checks whether -lrt is needed for any of the given functions. If +# so, appends -lrt via [teaish-ldflags-prepend] and returns 1, else +# returns 0. It also defines LDFLAGS_LIBRT to the libs flag or an +# empty string. +# +# Some systems (ex: SunOS) require -lrt in order to use nanosleep. +# +proc teaish-check-librt {{funclist {fdatasync nanosleep}}} { + teaish-check-cached -nostatus "Checking whether ($funclist) need librt" { + define LDFLAGS_LIBRT "" + foreach func $funclist { + if {[msg-quiet proj-check-function-in-lib $func rt]} { + set ldrt [get-define lib_${func}] + undefine lib_${func} + if {"" ne $ldrt} { + teaish-ldflags-prepend -r [define LDFLAGS_LIBRT $ldrt] + msg-result $ldrt + return 1 + } else { + msg-result "no lib needed" + return 1 + } + } + } + msg-result "not found" + return 0 + } +} + +# @teaish-check-stdint +# +# A thin proxy for [cc-with] which checks for and the +# various fixed-size int types it declares. It defines HAVE_STDINT_T +# to 0 or 1 and (if it's 1) defines HAVE_XYZ_T for each XYZ int type +# to 0 or 1, depending on whether its available. +proc teaish-check-stdint {} { + teaish-check-cached "Checking for stdint.h" { + msg-quiet cc-with {-includes stdint.h} \ + {cc-check-types int8_t int16_t int32_t int64_t intptr_t \ + uint8_t uint16_t uint32_t uint64_t uintptr_t} + } +} + +# @teaish-is-mingw +# +# Returns 1 if building for mingw, else 0. +proc teaish-is-mingw {} { + return [expr { + [string match *mingw* [get-define host]] && + ![file exists /dev/null] + }] +} + +# @teaish-check-libdl +# +# Checks for whether dlopen() can be found and whether it requires +# -ldl for linking. If found, returns 1, defines LDFLAGS_DLOPEN to the +# linker flags (if any), and passes those flags to +# teaish-ldflags-prepend. It unconditionally defines HAVE_DLOPEN to 0 +# or 1 (the its return result value). +proc teaish-check-dlopen {} { + teaish-check-cached -nostatus "Checking for dlopen()" { + set rc 0 + set lfl "" + if {[cc-with {-includes dlfcn.h} { + cctest -link 1 -declare "extern char* dlerror(void);" -code "dlerror();"}]} { + msg-result "-ldl not needed" + incr rc + } elseif {[cc-check-includes dlfcn.h]} { + incr rc + if {[cc-check-function-in-lib dlopen dl]} { + set lfl [get-define lib_dlopen] + undefine lib_dlopen + msg-result " dlopen() needs $lfl" + } else { + msg-result " - dlopen() not found in libdl. Assuming dlopen() is built-in." + } + } else { + msg-result "not found" + } + teaish-ldflags-prepend [define LDFLAGS_DLOPEN $lfl] + define HAVE_DLOPEN $rc + } +} + +# +# @teaish-check-libmath +# +# Handles the --enable-math flag. Returns 1 if found, else 0. +# If found, it prepends -lm (if needed) to the linker flags. +proc teaish-check-libmath {} { + teaish-check-cached "Checking for libc math library" { + set lfl "" + set rc 0 + if {[msg-quiet proj-check-function-in-lib ceil m]} { + incr rc + set lfl [get-define lib_ceil] + undefine lib_ceil + teaish-ldflags-prepend $lfl + msg-checking "$lfl " + } + define LDFLAGS_LIBMATH $lfl + expr $rc + } +} + +# @teaish-import-features ?-flags? feature-names... +# +# For each $name in feature-names... it invokes: +# +# use teaish/feature/$name +# +# to load TEAISH_AUTOSETUP_DIR/feature/$name.tcl +# +# By default, if a proc named teaish-check-${name}-options is defined +# after sourcing a file, it is called and its result is passed to +# proj-append-options. This can be suppressed with the -no-options +# flag. +# +# Flags: +# +# -no-options: disables the automatic running of +# teaish-check-NAME-options, +# +# -run: if the function teaish-check-NAME exists after importing +# then it is called. This flag must not be used when calling this +# function from teaish-options. This trumps both -pre and -post. +# +# -pre: if the function teaish-check-NAME exists after importing +# then it is passed to [teaish-checks-queue -pre]. +# +# -post: works like -pre but instead uses[teaish-checks-queue -post]. +proc teaish-import-features {args} { + set pk "" + set doOpt 1 + proj-parse-simple-flags args flags { + -no-options 0 {set doOpt 0} + -run 0 {expr 1} + -pre 0 {set pk -pre} + -post 0 {set pk -post} + } + # + # TODO: never import the same module more than once. The "use" + # command is smart enough to not do that but we would need to + # remember whether or not any teaish-check-${arg}* procs have been + # called before, and skip them. + # + if {$flags(-run) && "" ne $pk} { + proj-error "Cannot use both -run and $pk" \ + " (called from [proj-scope 1])" + } + + foreach arg $args { + uplevel "use teaish/feature/$arg" + if {$doOpt} { + set n "teaish-check-${arg}-options" + if {[llength [info proc $n]] > 0} { + if {"" ne [set x [$n]]} { + options-add $x + } + } + } + if {$flags(-run)} { + set n "teaish-check-${arg}" + if {[llength [info proc $n]] > 0} { + uplevel 1 $n + } + } elseif {"" ne $pk} { + set n "teaish-check-${arg}" + if {[llength [info proc $n]] > 0} { + teaish-checks-queue {*}$pk $n + } + } + } +} diff --git a/autosetup/teaish/tester.tcl b/autosetup/teaish/tester.tcl new file mode 100644 index 000000000..d8b5f7a0e --- /dev/null +++ b/autosetup/teaish/tester.tcl @@ -0,0 +1,228 @@ +######################################################################## +# 2025 April 5 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# * May you do good and not evil. +# * May you find forgiveness for yourself and forgive others. +# * May you share freely, never taking more than you give. +# +######################################################################## +# +# Helper routines for running tests on teaish extensions +# +######################################################################## +# ----- @module teaish/tester.tcl ----- +# +# @section TEA-ish Testing APIs. +# +# Though these are part of the autosup dir hierarchy, they are not +# intended to be run from autosetup code. Rather, they're for use +# with/via teaish.tester.tcl and target canonical Tcl only, not JimTcl +# (which the autosetup pieces do target). + +# +# @test-current-scope ?lvl? +# +# Returns the name of the _calling_ proc from ($lvl + 1) levels up the +# call stack (where the caller's level will be 1 up from _this_ +# call). If $lvl would resolve to global scope "global scope" is +# returned and if it would be negative then a string indicating such +# is returned (as opposed to throwing an error). +# +proc test-current-scope {{lvl 0}} { + #uplevel [expr {$lvl + 1}] {lindex [info level 0] 0} + set ilvl [info level] + set offset [expr {$ilvl - $lvl - 1}] + if { $offset < 0} { + return "invalid scope ($offset)" + } elseif { $offset == 0} { + return "global scope" + } else { + return [lindex [info level $offset] 0] + } +} + +# @test-msg +# +# Emits all arugments to stdout. +# +proc test-msg {args} { + puts "$args" +} + +# @test-warn +# +# Emits all arugments to stderr. +# +proc test-warn {args} { + puts stderr "WARNING: $args" +} + +# +# @test-error msg +# +# Triggers a test-failed error with a string describing the calling +# scope and the provided message. +# +proc test-fail {args} { + #puts stderr "ERROR: \[[test-current-scope 1]]: $msg" + #exit 1 + error "FAIL: \[[test-current-scope 1]]: $args" +} + +array set ::test__Counters {} +array set ::test__Config { + verbose-assert 0 verbose-affirm 0 +} + +# Internal impl for affirm and assert. +# +# $args = ?-v? script {msg-on-fail ""} +proc test__affert {failMode args} { + if {$failMode} { + set what assert + } else { + set what affirm + } + set verbose $::test__Config(verbose-$what) + if {"-v" eq [lindex $args 0]} { + lassign $args - script msg + if {1 == [llength $args]} { + # If -v is the only arg, toggle default verbose mode + set ::test__Config(verbose-$what) [expr {!$::test__Config(verbose-$what)}] + return + } + incr verbose + } else { + lassign $args script msg + } + incr ::test__Counters($what) + if {![uplevel 1 [concat expr [list $script]]]} { + if {"" eq $msg} { + set msg $script + } + set txt [join [list $what # $::test__Counters($what) "failed:" $msg]] + if {$failMode} { + puts stderr $txt + exit 1 + } else { + error $txt + } + } elseif {$verbose} { + puts stderr [join [list $what # $::test__Counters($what) "passed:" $script]] + } +} + +# +# @affirm ?-v? script ?msg? +# +# Works like a conventional assert method does, but reports failures +# using [error] instead of [exit]. If -v is used, it reports passing +# assertions to stderr. $script is evaluated in the caller's scope as +# an argument to [expr]. +# +proc affirm {args} { + tailcall test__affert 0 {*}$args +} + +# +# @assert ?-v? script ?msg? +# +# Works like [affirm] but exits on error. +# +proc assert {args} { + tailcall test__affert 1 {*}$args +} + +# +# @test-assert testId script ?msg? +# +# Works like [assert] but emits $testId to stdout first. +# +proc test-assert {testId script {msg ""}} { + puts "test $testId" + tailcall test__affert 1 $script $msg +} + +# +# @test-expect testId script result +# +# Runs $script in the calling scope and compares its result to +# $result, minus any leading or trailing whitespace. If they differ, +# it triggers an [assert]. +# +proc test-expect {testId script result} { + puts "test $testId" + set x [string trim [uplevel 1 $script]] + set result [string trim $result] + tailcall test__affert 0 [list $x eq $result] \ + "\nEXPECTED: <<$result>>\nGOT: <<$x>>" +} + +# +# @test-catch cmd ?...args? +# +# Runs [cmd ...args], repressing any exception except to possibly log +# the failure. Returns 1 if it caught anything, 0 if it didn't. +# +proc test-catch {cmd args} { + if {[catch { + $cmd {*}$args + } rc xopts]} { + puts "[test-current-scope] ignoring failure of: $cmd [lindex $args 0]: $rc" + return 1 + } + return 0 +} + +if {![array exists ::teaish__BuildFlags]} { + array set ::teaish__BuildFlags {} +} + +# +# @teaish-build-flag3 flag tgtVar ?dflt? +# +# If the current build has the configure-time flag named $flag set +# then tgtVar is assigned its value and 1 is returned, else tgtVal is +# assigned $dflt and 0 is returned. +# +# Caveat #1: only valid when called in the context of teaish's default +# "make test" recipe, e.g. from teaish.test.tcl. It is not valid from +# a teaish.tcl configure script because (A) the state it relies on +# doesn't fully exist at that point and (B) that level of the API has +# more direct access to the build state. This function requires that +# an external script have populated its internal state, which is +# normally handled via teaish.tester.tcl.in. +# +# Caveat #2: defines in the style of HAVE_FEATURENAME with a value of +# 0 are, by long-standing configure script conventions, treated as +# _undefined_ here. +# +proc teaish-build-flag3 {flag tgtVar {dflt ""}} { + upvar $tgtVar tgt + if {[info exists ::teaish__BuildFlags($flag)]} { + set tgt $::teaish__BuildFlags($flag) + return 1; + } elseif {0==[array size ::teaish__BuildFlags]} { + test-warn \ + "\[[test-current-scope]] was called from " \ + "[test-current-scope 1] without the build flags imported." + } + set tgt $dflt + return 0 +} + +# +# @teaish-build-flag flag ?dflt? +# +# Convenience form of teaish-build-flag3 which returns the +# configure-time-defined value of $flag or "" if it's not defined (or +# if it's an empty string). +# +proc teaish-build-flag {flag {dflt ""}} { + set tgt "" + teaish-build-flag3 $flag tgt $dflt + return $tgt +} diff --git a/contrib/sqlitecon.tcl b/contrib/sqlitecon.tcl index b5dbcafc2..78463a1ff 100644 --- a/contrib/sqlitecon.tcl +++ b/contrib/sqlitecon.tcl @@ -567,7 +567,7 @@ proc sqlitecon::Cut w { } } -# Do a paste opeation. +# Do a paste operation. # proc sqlitecon::Paste w { if {[sqlitecon::canCut $w]==1} { diff --git a/doc/jsonb.md b/doc/jsonb.md index 5beed1631..ce36f3ead 100644 --- a/doc/jsonb.md +++ b/doc/jsonb.md @@ -281,7 +281,7 @@ happen if and when they are actually needed. A valid JSONB BLOB consists of a single JSON element. The element must exactly fill the BLOB. This one element is often a JSON object or array and those usually contain additional elements as its payload, but the -element can be a primite value such a string, number, boolean, or null. +element can be a primitive value such a string, number, boolean, or null. When the built-in JSON functions are attempting to determine if a BLOB argument is a JSONB or just a random BLOB, they look at the header of diff --git a/doc/lemon.html b/doc/lemon.html index 4147d9b31..965f305c0 100644 --- a/doc/lemon.html +++ b/doc/lemon.html @@ -846,7 +846,7 @@

4.4.7 The %fallback directive

would have generated a syntax error.

The %fallback directive was added to support robust parsing of SQL -syntax in SQLite. +syntax in SQLite. The SQL language contains a large assortment of keywords, each of which appears as a different token to the language parser. SQL contains so many keywords that it can be difficult for programmers to keep up with @@ -881,7 +881,7 @@

4.4.8 The %if directive and its friends

Grammar text in between "%ifdef MACRO" and the next nested "%endif" is ignored unless the "-DMACRO" command-line option is used. Grammar text -betwen "%ifndef MACRO" and the next nested "%endif" is +between "%ifndef MACRO" and the next nested "%endif" is included except when the "-DMACRO" command-line option is used.

The text in between "%if CONDITIONAL" and its diff --git a/doc/pager-invariants.txt b/doc/pager-invariants.txt index 44444dad5..0fea0a698 100644 --- a/doc/pager-invariants.txt +++ b/doc/pager-invariants.txt @@ -45,7 +45,7 @@ *** Definition: Two databases (or the same database at two points it time) are said to be "logically equivalent" if they give the same answer to all queries. Note in particular the content of freelist leaf - pages can be changed arbitarily without effecting the logical equivalence + pages can be changed arbitrarily without effecting the logical equivalence of the database. (7) At any time, if any subset, including the empty set and the total set, diff --git a/doc/tcl-extension-testing.md b/doc/tcl-extension-testing.md index df5f6537b..eb2a8c3a3 100644 --- a/doc/tcl-extension-testing.md +++ b/doc/tcl-extension-testing.md @@ -35,84 +35,69 @@ perspective on how to compile SQLite on unix-like systems. ### 2.1 Setup

    -
  1. - [Fossil](https://fossil-scm.org/) installed. +
  2. + [Fossil][] installed.
  3. Check out source code and set environment variables:
    1. **TCLSOURCE** → - The top-level directory of a Fossil check-out of the TCL source tree. + The top-level directory of a [Fossil][] check-out of the + [TCL source tree][tcl-fossil].
    2. **SQLITESOURCE** → A Fossil check-out of the SQLite source tree. -
    3. **TCLBUILD** → +
    4. **TCLHOME** → A directory that does not exist at the start of the test and which will be deleted at the end of the test, and that will contain the test builds of the TCL libraries and the SQLite TCL Extensions. + It is the top-most installation directory, i.e. the one provided + to Tcl's `./configure --prefix=/path/to/tcl`. +
    5. **TCLVERSION** → + The `X.Y`-form version of Tcl being used: 8.6, 9.0, 9.1...
-### 2.2 Testing TCL 8.6 on unix +### 2.2 Testing TCL 8.x and 9.x on unix + +From a checked-out copy of [the core Tcl tree][tcl-fossil]
    -
  1. `mkdir -p $TCLBUILD/tcl86` -
  2. `cd $TCLSOURCE/unix` -
  3. `fossil up core-8-6-16`
    - ↑ Or some other version of Tcl8.6. +
  4. `TCLVERSION=8.6`
    + ↑ A version of your choice. This process has been tested with + values of 8.6, 9.0, and 9.1 (as of 2025-04-16). The out-of-life + version 8.5 fails some of `make devtest` for undetermined reasons. +
  5. `TCLHOME=$HOME/tcl/$TCLVERSION` +
  6. `TCLSOURCE=/path/to/tcl/checkout` +
  7. `SQLITESOURCE=/path/to/sqlite/checkout` +
  8. `rm -fr $TCLHOME`
    + ↑ Ensure that no stale Tcl installation is laying around. +
  9. `cd $TCLSOURCE` +
  10. `fossil up core-8-6-branch`
    + ↑ The branch corresponding to `$TCLVERSION`, e.g. + `core-9-0-branch` or `trunk`.
  11. `fossil clean -x` -
  12. `./configure --prefix=$TCLBUILD/tcl86 --disable-shared`
    - ↑ The --disable-shared is to avoid the need to set LD_LIBRARY_PATH +
  13. `cd unix` +
  14. `./configure --prefix=$TCLHOME --disable-shared`
    + ↑ The `--disable-shared` is to avoid the need to set `LD_LIBRARY_PATH` when using this Tcl build.
  15. `make install`
  16. `cd $SQLITESOURCE`
  17. `fossil clean -x` -
  18. `./configure --with-tclsh=$TCLBUILD/tcl86/bin/tclsh8.6 --all` +
  19. `./configure --with-tcl=$TCLHOME --all`
  20. `make tclextension-install`
    - ↑ Verify extension installed at $TCLBUILD/tcl86/lib/tcl8.6/sqlite3.* + ↑ Verify extension installed at + `$TCLHOME/lib/tcl${TCLVERSION}/sqlite`.
  21. `make tclextension-list`
    ↑ Verify TCL extension correctly installed.
  22. `make tclextension-verify`
    ↑ Verify that the correct version is installed. -
  23. `$TCLBUILD/tcl86/bin/tclsh8.6 test/testrunner.tcl release --explain`
    +
  24. `$TCLHOME/bin/tclsh[89].[0-9] test/testrunner.tcl release --explain`
    ↑ Verify thousands of lines of output with no errors. Or consider running "devtest" without --explain instead of "release".
-### 2.3 Testing TCL 9.0 on unix - -
    -
  1. `mkdir -p $TCLBUILD/tcl90` -
  2. `fossil up core-9-0-0`
    - ↑ Or some other version of Tcl9 -
  3. `fossil clean -x` -
  4. `./configure --prefix=$TCLBUILD/tcl90 --disable-shared`
    - ↑ The --disable-shared is to avoid the need to set LD_LIBRARY_PATH - when using this Tcl build. -
  5. `make install` -
  6. `cp -r ../library $TCLBUILD/tcl90/lib/tcl9.0`
    - ↑ The Tcl library is not installed by "make install" for Tcl9.0 unless - you also include the --disable-zipfs to ./configure. But if you do that - then the generated tclsh9.0 is no longer stand-alone. On the other hand, - if you don't install the Tcl library, other programs like testfixture - won't be able to find the Tcl library and hence won't work. This - extra installation step resolves the dilemma. - This step is not required when building Tcl8.6, which lacks support for - zipfs and hence always installs its Tcl library. -
  7. `cd $SQLITESOURCE` -
  8. `fossil clean -x` -
  9. `./configure --with-tclsh=$TCLBUILD/tcl90/bin/tclsh9.0 --all` -
  10. `make tclextension-install`
    - ↑ Verify extension installed at $TCLBUILD/tcl90/lib/sqlite3.* -
  11. `make tclextension-list`
    - ↑ Verify TCL extension correctly installed. -
  12. `make tclextension-verify` -
  13. `$TCLBUILD/tcl90/bin/tclsh9.0 test/testrunner.tcl release --explain`
    - ↑ Verify thousands of lines of output with no errors. Or - consider running "devtest" without --explain instead of "release". -
- -### 2.4 Cleanup +### 2.3 Cleanup
    -
  1. `rm -rf $TCLBUILD` +
  2. `rm -rf $TCLHOME`
@@ -123,9 +108,11 @@ perspective on how to compile SQLite on Windows. ### 3.1 Setup for Windows +(These docs are not as up-to-date as the Unix docs, above.) +
    -
  1. - [Fossil](https://fossil-scm.org/) installed. +
  2. + [Fossil][] installed.
  3. Unix-like command-line tools installed. Example: [unxutils](https://unxutils.sourceforge.net/) @@ -206,3 +193,72 @@ perspective on how to compile SQLite on Windows.
    1. `rm -rf %TCLBUILD%`
    + +## 4.0 Testing the TEA(ish) Build (unix only) + +This part requires following the setup instructions for Unix systems, +at the top of this document. + +The former TEA, now TEA(ish), build of this extension uses the same +code as the builds described above but is provided in a form more +convenient for downstream Tcl users. + +It lives in `autoconf/tea` and, as part of the `autoconf` bundle, +_cannot be tested directly from the canonical tree_. Instead it has to +be packaged. + +### 4.1 Teaish Setup + +Follow the same Tcl- and environment-related related setup described +in the first section of this document, up to and including the +installation of Tcl (unless, of course, it was already installed using +those same instructions). + +### 4.2 Teaish Testing + +
      +
    1. `cd $SQLITESOURCE` +
    2. Run either `make snapshot-tarball` or `make amalgamation-tarball` + ↑ + Those steps will leave behind a temp dir called `mkpkg_tmp_dir`, + under which the extension is most readily reached. It can optionally + be extracted from the generated tarball, but that tarball was + generated from this dir, and reusing this dir is a time saver + during development. +
    3. `cd mkpkg_tmp/tea` +
    4. `./configure --with-tcl=$TCLHOME` +
    5. `make test install`
      + ↑ Should run to completion without any errors. +
    6. `make uninstall`
      + ↑ Will uninstall the extension. This _can_ be run + in the same invocation as the `install` target, but only + if the `-j#` make flag is _not_ used. If it is, the + install/uninstall steps will race and make a mess of things. + Parallel builds do not help in this build, anyway, as there's + only a single C file to compile. +
    + +When actively developing and testing the teaish build, which requires +going through the tarball generation, there's a caveat about the +`mkpkg_tmp_dir` dir: it will be deleted every time a tarball is +built, the shell console which is parked in that +directory for testing needs to add `cd $PWD &&` to the start of the +build commands, like: + +> +``` +[user@host:.../mkpkg_tmp_dir/tea]$ \ + cd $PWD && ./configure CFLAGS=-O0 --with-tcl=$TCLHOME \ + && make test install uninstall +``` + +### 4.3 Teaish Cleanup + + +
      +
    1. `rm -rf $TCLHOME` +
    2. `cd $SQLITESOURCE; rm -fr mkpkg_tmp_dir; fossil clean -x` +
    + +[Fossil]: https://fossil-scm.org/home +[tcl-fossil]: https://core.tcl-lang.org/tcl diff --git a/doc/testrunner.md b/doc/testrunner.md index d0248573e..d1696e9d1 100644 --- a/doc/testrunner.md +++ b/doc/testrunner.md @@ -15,6 +15,7 @@
  4. 3.1. Commands to Run SQLite Tests
  5. 3.2. Running ZipVFS Tests
  6. 3.3. Investigating Source Code Test Failures +
  7. 3.4. External Fuzzcheck Databases
  8. 4. Extra testrunner.tcl Options
  9. 5. Controlling CPU Core Utilization @@ -23,47 +24,62 @@ # 1. Overview -testrunner.tcl is a Tcl script used to run multiple SQLite tests using -multiple jobs. It supports the following types of tests: +The testrunner.tcl program is a Tcl script used to run multiple SQLite +tests in parallel, thus reducing testing time on multi-core machines. +It supports the following types of tests: * Tcl test scripts. + * Fuzzcheck tests, including using an external fuzzcheck database. + * Tests run with `make` commands. Examples: - - `make mdevtest` + - `make devtest` - `make releasetest` - `make sdevtest` - `make testrunner` -testrunner.tcl pipes the output of all tests and builds run into log file -**testrunner.log**, created in the current working directory. Search this -file to find details of errors. Suggested search commands: +The testrunner.tcl program stores output of all tests and builds run in +log file **testrunner.log**, created in the current working directory. +Search this file to find details of errors. Suggested search commands: * `grep "^!" testrunner.log` * `grep failed testrunner.log` -testrunner.tcl also populates SQLite database **testrunner.db**. This database -contains details of all tests run, running and to be run. A useful query -might be: +The testrunner.tcl program also populates SQLite database **testrunner.db**. +This database contains details of all tests run, running and to be run. +A useful query might be: ``` SELECT * FROM script WHERE state='failed' ``` +You can get a summary of errors in a prior run by invoking commands like +these: + +``` + tclsh $(TESTDIR)/testrunner.tcl errors + tclsh $(TESTDIR)/testrunner.tcl errors -v +``` + Running the command: ``` - ./testfixture $(TESTDIR)/testrunner.tcl status + tclsh $(TESTDIR)/testrunner.tcl status ``` in the directory containing the testrunner.db database runs various queries to produce a succinct report on the state of a running testrunner.tcl script. -Running: +A good way to keep and eye on test progress is to run either of the two +following commands: ``` - watch ./testfixture $(TESTDIR)/testrunner.tcl status + watch tclsh $(TESTDIR)/testrunner.tcl status + tclsh $(TESTDIR)/testrunner.tcl status -d 2 ``` -in another terminal is a good way to keep an eye on a long running test. +Both of the commands above accomplish about the same thing, but the second +one has the advantage of not requiring "watch" to be installed on your +system. Sometimes testrunner.tcl uses the `testfixture` binary that it is run with to run tests (see "Binary Tests" below). Sometimes it builds testfixture and @@ -239,9 +255,7 @@ of the specific tests run. As with source code tests, one or more patterns may be appended to any of the above commands (mdevtest, sdevtest or release). -In that case only Tcl tests (no fuzz or other tests) that match the specified -pattern are run. For example, to run the just the Tcl rtree tests in all -builds and configurations supported by "release": +Pattern matching is used for both Tcl tests and fuzz tests. ``` tclsh $TESTDIR/testrunner.tcl release rtree% @@ -292,6 +306,34 @@ target to build. This may be used either to run a `make` command test directly, or else to build a testfixture (or testfixture.exe) binary with which to run a Tcl test script, as described above. + +## 3.4 External Fuzzcheck Databases + +Testrunner.tcl will also run fuzzcheck against an external (out of tree) +database, for example fuzzcheck databases generated by dbsqlfuzz. To do +this, simply add the "`--fuzzdb` *FILENAME*" command-line option or set +the FUZZDB environment variable to the name of the external +database. For large external databases, testrunner.tcl will automatically use +the "`--slice`" command-line option of fuzzcheck to divide the work up into +multiple jobs, to increase parallelism. + +Thus, for example, to run a full releasetest including an external +dbsqlfuzz database, run a command like one of these: + +``` + tclsh test/testrunner.tcl releasetest --fuzzdb ../fuzz/20250415.db + FUZZDB=../fuzz/20250415.db make releasetest + nmake /f Makefile.msc FUZZDB=../fuzz/20250415.db releasetest +``` + +The patternlist option to testrunner.tcl will match against fuzzcheck +databases. So if you want to run *only* tests involving the external +database, you can use a command something like this: + +``` + tclsh test/testrunner.tcl releasetest 20250415 --fuzzdb ../fuzz/20250415.db +``` + # 4. Extra testrunner.tcl Options @@ -315,16 +357,22 @@ would normally execute into the testrunner.log file. Example: tclsh $TESTDIR/testrunner.tcl --dryrun mdevtest" ``` -The **--explain** option is similar to --dryrun in that it prevents testrunner.tcl -from building any binaries or running any tests. The difference is that --explain -prints on standard output a human-readable summary of all the builds and tests that -would have been run. +The **--explain** option is similar to --dryrun in that it prevents +testrunner.tcl from building any binaries or running any tests. The +difference is that --explain prints on standard output a human-readable +summary of all the builds and tests that would have been run. ``` # Show what builds and tests would have been run tclsh $TESTDIR/testrunner.tcl --explain mdevtest ``` +The **--status** option uses VT100 escape sequences to display the test +status full-screen. This is similar to running +"`watch test/testrunner status`" in a separate window, just more convenient. +Unfortunately, this option does not work correctly on Windows, due to the +sketchy implementation of VT100 escapes on the Windows console. + # 5. Controlling CPU Core Utilization diff --git a/doc/vfs-shm.txt b/doc/vfs-shm.txt index c1f125a12..a483e9b15 100644 --- a/doc/vfs-shm.txt +++ b/doc/vfs-shm.txt @@ -1,6 +1,6 @@ The 5 states of an historical rollback lock as implemented by the xLock, xUnlock, and xCheckReservedLock methods of the sqlite3_io_methods -objec are: +object are: UNLOCKED SHARED @@ -58,7 +58,7 @@ The meanings of the various wal-index locking states is as follows: A particular lock manager implementation may coalesce one or more of the wal-index locking states, though with a reduction in concurrency. -For example, an implemention might implement only exclusive locking, +For example, an implementation might implement only exclusive locking, in which case all states would be equivalent to CHECKPOINT, meaning that only one reader or one writer or one checkpointer could be active at a time. Or, an implementation might combine READ and READ_FULL into diff --git a/doc/wal-lock.md b/doc/wal-lock.md index d74bb88b6..8df7cc836 100644 --- a/doc/wal-lock.md +++ b/doc/wal-lock.md @@ -12,7 +12,7 @@ facilitates transfer of OS priority between processes when a high priority process is blocked by a lower priority one. Only read/write clients use blocking locks. Clients that have read-only access -to the \*-shm file nevery use blocking locks. +to the \*-shm file never use blocking locks. Threads or processes that access a single database at a time never deadlock as a result of blocking database locks. But it is of course possible for threads diff --git a/ext/README.md b/ext/README.md index 933a33d05..78312819a 100644 --- a/ext/README.md +++ b/ext/README.md @@ -1,6 +1,6 @@ ## Loadable Extensions -Various [loadable extensions](https://www.sqlite.org/loadext.html) for +Various [loadable extensions](https://sqlite.org/loadext.html) for SQLite are found in subfolders. Most subfolders are dedicated to a single loadable extension (for diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c index 93693cfae..ddb36714f 100644 --- a/ext/expert/sqlite3expert.c +++ b/ext/expert/sqlite3expert.c @@ -2052,7 +2052,7 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){ sqlite3_set_authorizer(pNew->dbv, idxAuthCallback, (void*)pNew); } - /* If an error has occurred, free the new object and reutrn NULL. Otherwise, + /* If an error has occurred, free the new object and return NULL. Otherwise, ** return the new sqlite3expert handle. */ if( rc!=SQLITE_OK ){ sqlite3_expert_destroy(pNew); diff --git a/ext/expert/test_expert.c b/ext/expert/test_expert.c index cae5d0f25..4383d7c7b 100644 --- a/ext/expert/test_expert.c +++ b/ext/expert/test_expert.c @@ -28,7 +28,7 @@ static int dbHandleFromObj(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){ Tcl_CmdInfo info; if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){ - Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), 0); + Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), NULL); return TCL_ERROR; } diff --git a/ext/fts3/README.syntax b/ext/fts3/README.syntax index 01bc80c5f..d32ae384c 100644 --- a/ext/fts3/README.syntax +++ b/ext/fts3/README.syntax @@ -62,20 +62,20 @@ matches rows that contain both the "engineering" and "consultancy" tokens in the same column with not more than 10 other words between them. It does not matter which of the two terms occurs first in the document, only that - they be seperated by only 10 tokens or less. The user may also specify + they be separated by only 10 tokens or less. The user may also specify a different required proximity by adding "/N" immediately after the NEAR operator, where N is an integer. For example: MATCH 'engineering NEAR/5 consultancy' - searches for a row containing an instance of each specified token seperated + searches for a row containing an instance of each specified token separated by not more than 5 other tokens. More than one NEAR operator can be used in as sequence. For example this query: MATCH 'reliable NEAR/2 engineering NEAR/5 consultancy' searches for a row that contains an instance of the token "reliable" - seperated by not more than two tokens from an instance of "engineering", + separated by not more than two tokens from an instance of "engineering", which is in turn separated by not more than 5 other tokens from an instance of the term "consultancy". Phrases enclosed in quotes may also be used as arguments to the NEAR operator. @@ -146,7 +146,7 @@ MATCH '(hello world) OR (simple example)' matches documents that contain both "hello" and "world", and documents - that contain both "simple" and "example". It is not possible to forumlate + that contain both "simple" and "example". It is not possible to formulate such a query using the standard syntax. 2) Instead of separating tokens and phrases by whitespace, an AND operator @@ -174,7 +174,7 @@ 4) Unlike in the standard syntax, where the OR operator has a higher precedence than the implicit AND operator, when using the enhanced - syntax implicit and explict AND operators have a higher precedence + syntax implicit and explicit AND operators have a higher precedence than OR operators. Using the enhanced syntax, the following two queries are equivalent: diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 2b2c3b8d2..f178abafe 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -87,7 +87,7 @@ ** Here, array { X } means zero or more occurrences of X, adjacent in ** memory. A "position" is an index of a token in the token stream ** generated by the tokenizer. Note that POS_END and POS_COLUMN occur -** in the same logical place as the position element, and act as sentinals +** in the same logical place as the position element, and act as sentinels ** ending a position list array. POS_END is 0. POS_COLUMN is 1. ** The positions numbers are not stored literally but rather as two more ** than the difference from the prior position, or the just the position plus @@ -295,12 +295,6 @@ # define SQLITE_CORE 1 #endif -#include -#include -#include -#include -#include -#include #include "fts3.h" #ifndef SQLITE_CORE @@ -2639,7 +2633,7 @@ static int fts3DoclistOrMerge( ** sizes of the two inputs, plus enough space for exactly one of the input ** docids to grow. ** - ** A symetric argument may be made if the doclists are in descending + ** A symmetric argument may be made if the doclists are in descending ** order. */ aOut = sqlite3_malloc64((i64)n1+n2+FTS3_VARINT_MAX-1+FTS3_BUFFER_PADDING); @@ -4438,7 +4432,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ nDistance = iPrev - nMaxUndeferred; } - aOut = (char *)sqlite3Fts3MallocZero(nPoslist+FTS3_BUFFER_PADDING); + aOut = (char *)sqlite3Fts3MallocZero(((i64)nPoslist)+FTS3_BUFFER_PADDING); if( !aOut ){ sqlite3_free(aPoslist); return SQLITE_NOMEM; @@ -4737,7 +4731,7 @@ static int incrPhraseTokenNext( ** ** * does not contain any deferred tokens. ** -** Advance it to the next matching documnent in the database and populate +** Advance it to the next matching document in the database and populate ** the Fts3Doclist.pList and nList fields. ** ** If there is no "next" entry and no error occurs, then *pbEof is set to @@ -5744,7 +5738,7 @@ static int fts3EvalNext(Fts3Cursor *pCsr){ } /* -** Restart interation for expression pExpr so that the next call to +** Restart iteration for expression pExpr so that the next call to ** fts3EvalNext() visits the first row. Do not allow incremental ** loading or merging of phrase doclists for this iteration. ** diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 77e6737af..d438549de 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -14,6 +14,13 @@ #ifndef _FTSINT_H #define _FTSINT_H +#include +#include +#include +#include +#include +#include + #if !defined(NDEBUG) && !defined(SQLITE_DEBUG) # define NDEBUG 1 #endif @@ -202,6 +209,19 @@ typedef sqlite3_int64 i64; /* 8-byte signed integer */ #define deliberate_fall_through +/* +** Macros needed to provide flexible arrays in a portable way +*/ +#ifndef offsetof +# define offsetof(STRUCTURE,FIELD) ((size_t)((char*)&((STRUCTURE*)0)->FIELD)) +#endif +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# define FLEXARRAY +#else +# define FLEXARRAY 1 +#endif + + #endif /* SQLITE_AMALGAMATION */ #ifdef SQLITE_DEBUG @@ -306,7 +326,7 @@ struct Fts3Table { #endif #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) - /* True to disable the incremental doclist optimization. This is controled + /* True to disable the incremental doclist optimization. This is controlled ** by special insert command 'test-no-incr-doclist'. */ int bNoIncrDoclist; @@ -358,7 +378,7 @@ struct Fts3Cursor { /* ** The Fts3Cursor.eSearch member is always set to one of the following. -** Actualy, Fts3Cursor.eSearch can be greater than or equal to +** Actually, Fts3Cursor.eSearch can be greater than or equal to ** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index ** of the column to be searched. For example, in ** @@ -431,9 +451,13 @@ struct Fts3Phrase { */ int nToken; /* Number of tokens in the phrase */ int iColumn; /* Index of column this phrase must match */ - Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ + Fts3PhraseToken aToken[FLEXARRAY]; /* One for each token in the phrase */ }; +/* Size (in bytes) of an Fts3Phrase object large enough to hold N tokens */ +#define SZ_FTS3PHRASE(N) \ + (offsetof(Fts3Phrase,aToken)+(N)*sizeof(Fts3PhraseToken)) + /* ** A tree of these objects forms the RHS of a MATCH operator. ** diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 9e201b168..681d4e862 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -161,6 +161,23 @@ int sqlite3Fts3OpenTokenizer( */ static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *); +/* +** Search buffer z[], size n, for a '"' character. Or, if enable_parenthesis +** is defined, search for '(' and ')' as well. Return the index of the first +** such character in the buffer. If there is no such character, return -1. +*/ +static int findBarredChar(const char *z, int n){ + int ii; + for(ii=0; iiiLangid, z, i, &pCursor); + *pnConsumed = n; + rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, n, &pCursor); if( rc==SQLITE_OK ){ const char *zToken; int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; @@ -202,7 +212,18 @@ static int getNextToken( rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); if( rc==SQLITE_OK ){ - nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; + /* Check that this tokenization did not gobble up any " characters. Or, + ** if enable_parenthesis is true, that it did not gobble up any + ** open or close parenthesis characters either. If it did, call + ** getNextToken() again, but pass only that part of the input buffer + ** up to the first such character. */ + int iBarred = findBarredChar(z, iEnd); + if( iBarred>=0 ){ + pModule->xClose(pCursor); + return getNextToken(pParse, iCol, z, iBarred, ppExpr, pnConsumed); + } + + nByte = sizeof(Fts3Expr) + SZ_FTS3PHRASE(1) + nToken; pRet = (Fts3Expr *)sqlite3Fts3MallocZero(nByte); if( !pRet ){ rc = SQLITE_NOMEM; @@ -212,7 +233,7 @@ static int getNextToken( pRet->pPhrase->nToken = 1; pRet->pPhrase->iColumn = iCol; pRet->pPhrase->aToken[0].n = nToken; - pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1]; + pRet->pPhrase->aToken[0].z = (char*)&pRet->pPhrase->aToken[1]; memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken); if( iEnd=0 ){ + *pnConsumed = iBarred; + } rc = SQLITE_OK; } @@ -283,9 +308,9 @@ static int getNextString( Fts3Expr *p = 0; sqlite3_tokenizer_cursor *pCursor = 0; char *zTemp = 0; - int nTemp = 0; + i64 nTemp = 0; - const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase); + const int nSpace = sizeof(Fts3Expr) + SZ_FTS3PHRASE(1); int nToken = 0; /* The final Fts3Expr data structure, including the Fts3Phrase, @@ -657,7 +682,7 @@ static int fts3ExprParse( /* The isRequirePhrase variable is set to true if a phrase or ** an expression contained in parenthesis is required. If a - ** binary operator (AND, OR, NOT or NEAR) is encounted when + ** binary operator (AND, OR, NOT or NEAR) is encountered when ** isRequirePhrase is set, this is a syntax error. */ if( !isPhrase && isRequirePhrase ){ @@ -1239,7 +1264,6 @@ static void fts3ExprTestCommon( } if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){ - sqlite3Fts3ExprFree(pExpr); sqlite3_result_error(context, "Error parsing expression", -1); }else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){ sqlite3_result_error_nomem(context); diff --git a/ext/fts3/fts3_hash.c b/ext/fts3/fts3_hash.c index 63e55b3dc..1918be4cb 100644 --- a/ext/fts3/fts3_hash.c +++ b/ext/fts3/fts3_hash.c @@ -187,7 +187,7 @@ static void fts3HashInsertElement( } -/* Resize the hash table so that it cantains "new_size" buckets. +/* Resize the hash table so that it contains "new_size" buckets. ** "new_size" must be a power of 2. The hash table might fail ** to resize if sqliteMalloc() fails. ** diff --git a/ext/fts3/fts3_porter.c b/ext/fts3/fts3_porter.c index fbe791302..35e81b74a 100644 --- a/ext/fts3/fts3_porter.c +++ b/ext/fts3/fts3_porter.c @@ -256,7 +256,7 @@ static int star_oh(const char *z){ /* ** If the word ends with zFrom and xCond() is true for the stem -** of the word that preceeds the zFrom ending, then change the +** of the word that precedes the zFrom ending, then change the ** ending to zTo. ** ** The input word *pz and zFrom are both in reverse order. zTo diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 8a6ab8ea6..9c7f0ade9 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -108,9 +108,13 @@ struct MatchinfoBuffer { int nElem; int bGlobal; /* Set if global data is loaded */ char *zMatchinfo; - u32 aMatchinfo[1]; + u32 aMI[FLEXARRAY]; }; +/* Size (in bytes) of a MatchinfoBuffer sufficient for N elements */ +#define SZ_MATCHINFOBUFFER(N) \ + (offsetof(MatchinfoBuffer,aMI)+(((N)+1)/2)*sizeof(u64)) + /* ** The snippet() and offsets() functions both return text values. An instance @@ -135,13 +139,13 @@ struct StrBuffer { static MatchinfoBuffer *fts3MIBufferNew(size_t nElem, const char *zMatchinfo){ MatchinfoBuffer *pRet; sqlite3_int64 nByte = sizeof(u32) * (2*(sqlite3_int64)nElem + 1) - + sizeof(MatchinfoBuffer); + + SZ_MATCHINFOBUFFER(1); sqlite3_int64 nStr = strlen(zMatchinfo); pRet = sqlite3Fts3MallocZero(nByte + nStr+1); if( pRet ){ - pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet; - pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + pRet->aMI[0] = (u8*)(&pRet->aMI[1]) - (u8*)pRet; + pRet->aMI[1+nElem] = pRet->aMI[0] + sizeof(u32)*((int)nElem+1); pRet->nElem = (int)nElem; pRet->zMatchinfo = ((char*)pRet) + nByte; @@ -155,10 +159,10 @@ static MatchinfoBuffer *fts3MIBufferNew(size_t nElem, const char *zMatchinfo){ static void fts3MIBufferFree(void *p){ MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]); - assert( (u32*)p==&pBuf->aMatchinfo[1] - || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2] + assert( (u32*)p==&pBuf->aMI[1] + || (u32*)p==&pBuf->aMI[pBuf->nElem+2] ); - if( (u32*)p==&pBuf->aMatchinfo[1] ){ + if( (u32*)p==&pBuf->aMI[1] ){ pBuf->aRef[1] = 0; }else{ pBuf->aRef[2] = 0; @@ -175,18 +179,18 @@ static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ if( p->aRef[1]==0 ){ p->aRef[1] = 1; - aOut = &p->aMatchinfo[1]; + aOut = &p->aMI[1]; xRet = fts3MIBufferFree; } else if( p->aRef[2]==0 ){ p->aRef[2] = 1; - aOut = &p->aMatchinfo[p->nElem+2]; + aOut = &p->aMI[p->nElem+2]; xRet = fts3MIBufferFree; }else{ aOut = (u32*)sqlite3_malloc64(p->nElem * sizeof(u32)); if( aOut ){ xRet = sqlite3_free; - if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32)); + if( p->bGlobal ) memcpy(aOut, &p->aMI[1], p->nElem*sizeof(u32)); } } @@ -196,7 +200,7 @@ static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){ p->bGlobal = 1; - memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32)); + memcpy(&p->aMI[2+p->nElem], &p->aMI[1], p->nElem*sizeof(u32)); } /* @@ -611,7 +615,7 @@ static int fts3StringAppend( } /* If there is insufficient space allocated at StrBuffer.z, use realloc() - ** to grow the buffer until so that it is big enough to accomadate the + ** to grow the buffer until so that it is big enough to accommodate the ** appended data. */ if( pStr->n+nAppend+1>=pStr->nAlloc ){ @@ -1023,16 +1027,16 @@ static size_t fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ break; case FTS3_MATCHINFO_LHITS: - nVal = pInfo->nCol * pInfo->nPhrase; + nVal = (size_t)pInfo->nCol * pInfo->nPhrase; break; case FTS3_MATCHINFO_LHITS_BM: - nVal = pInfo->nPhrase * ((pInfo->nCol + 31) / 32); + nVal = (size_t)pInfo->nPhrase * ((pInfo->nCol + 31) / 32); break; default: assert( cArg==FTS3_MATCHINFO_HITS ); - nVal = pInfo->nCol * pInfo->nPhrase * 3; + nVal = (size_t)pInfo->nCol * pInfo->nPhrase * 3; break; } diff --git a/ext/fts3/fts3_test.c b/ext/fts3/fts3_test.c index 3c42a7bf0..70bccf0c5 100644 --- a/ext/fts3/fts3_test.c +++ b/ext/fts3/fts3_test.c @@ -219,7 +219,7 @@ static int SQLITE_TCLAPI fts3_near_match_cmd( rc = Tcl_ListObjGetElements(interp, pPhrase, &nToken, &apToken); if( rc!=TCL_OK ) goto near_match_out; if( nToken>NM_MAX_TOKEN ){ - Tcl_AppendResult(interp, "Too many tokens in phrase", 0); + Tcl_AppendResult(interp, "Too many tokens in phrase", NULL); rc = TCL_ERROR; goto near_match_out; } @@ -547,7 +547,7 @@ static int SQLITE_TCLAPI fts3_test_varint_cmd( if( w!=w2 || nByte!=nByte2 ){ char *zErr = sqlite3_mprintf("error testing %lld", w); Tcl_ResetResult(interp); - Tcl_AppendResult(interp, zErr, 0); + Tcl_AppendResult(interp, zErr, NULL); return TCL_ERROR; } @@ -557,7 +557,7 @@ static int SQLITE_TCLAPI fts3_test_varint_cmd( if( (int)w!=i || nByte!=nByte2 ){ char *zErr = sqlite3_mprintf("error testing %lld (32-bit)", w); Tcl_ResetResult(interp); - Tcl_AppendResult(interp, zErr, 0); + Tcl_AppendResult(interp, zErr, NULL); return TCL_ERROR; } } diff --git a/ext/fts3/fts3_tokenize_vtab.c b/ext/fts3/fts3_tokenize_vtab.c index 7e8d09bd4..b9d83982c 100644 --- a/ext/fts3/fts3_tokenize_vtab.c +++ b/ext/fts3/fts3_tokenize_vtab.c @@ -346,7 +346,7 @@ static int fts3tokFilterMethod( fts3tokResetCursor(pCsr); if( idxNum==1 ){ const char *zByte = (const char *)sqlite3_value_text(apVal[0]); - int nByte = sqlite3_value_bytes(apVal[0]); + sqlite3_int64 nByte = sqlite3_value_bytes(apVal[0]); pCsr->zInput = sqlite3_malloc64(nByte+1); if( pCsr->zInput==0 ){ rc = SQLITE_NOMEM; diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 5a449dec1..55a3f0a82 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -3956,7 +3956,7 @@ static int fts3IncrmergePush( ** ** It is assumed that the buffer associated with pNode is already large ** enough to accommodate the new entry. The buffer associated with pPrev -** is extended by this function if requrired. +** is extended by this function if required. ** ** If an error (i.e. OOM condition) occurs, an SQLite error code is ** returned. Otherwise, SQLITE_OK. @@ -5619,7 +5619,7 @@ int sqlite3Fts3DeferToken( /* ** SQLite value pRowid contains the rowid of a row that may or may not be ** present in the FTS3 table. If it is, delete it and adjust the contents -** of subsiduary data structures accordingly. +** of subsidiary data structures accordingly. */ static int fts3DeleteByRowid( Fts3Table *p, diff --git a/ext/fts3/unicode/mkunicode.tcl b/ext/fts3/unicode/mkunicode.tcl index 1306629da..3bf866ef7 100644 --- a/ext/fts3/unicode/mkunicode.tcl +++ b/ext/fts3/unicode/mkunicode.tcl @@ -893,7 +893,7 @@ proc print_test_main {} { puts "\}" } -# Proces the command line arguments. Exit early if they are not to +# Process the command line arguments. Exit early if they are not to # our liking. # proc usage {} { diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 832f9ad47..7ad1cc16b 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -20,6 +20,7 @@ SQLITE_EXTENSION_INIT1 #include #include +#include #ifndef SQLITE_AMALGAMATION @@ -75,6 +76,18 @@ typedef sqlite3_uint64 u64; # define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&7)==0) #endif +/* +** Macros needed to provide flexible arrays in a portable way +*/ +#ifndef offsetof +# define offsetof(STRUCTURE,FIELD) ((size_t)((char*)&((STRUCTURE*)0)->FIELD)) +#endif +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# define FLEXARRAY +#else +# define FLEXARRAY 1 +#endif + #endif /* Truncate very long tokens to this many bytes. Hard limit is @@ -147,10 +160,11 @@ typedef struct Fts5Colset Fts5Colset; */ struct Fts5Colset { int nCol; - int aiCol[1]; + int aiCol[FLEXARRAY]; }; - +/* Size (int bytes) of a complete Fts5Colset object with N columns. */ +#define SZ_FTS5COLSET(N) (sizeof(i64)*((N+2)/2)) /************************************************************************** ** Interface to code in fts5_config.c. fts5_config.c contains contains code diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index ad578156d..95b33ea31 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -667,7 +667,7 @@ static int fts5Bm25GetData( ** under consideration. ** ** The problem with this is that if (N < 2*nHit), the IDF is - ** negative. Which is undesirable. So the mimimum allowable IDF is + ** negative. Which is undesirable. So the minimum allowable IDF is ** (1e-6) - roughly the same as a term that appears in just over ** half of set of 5,000,000 documents. */ double idf = log( (nRow - nHit + 0.5) / (nHit + 0.5) ); diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c index 891ef0203..afcd83b6b 100644 --- a/ext/fts5/fts5_buffer.c +++ b/ext/fts5/fts5_buffer.c @@ -308,7 +308,7 @@ char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){ ** * The 52 upper and lower case ASCII characters, and ** * The 10 integer ASCII characters. ** * The underscore character "_" (0x5F). -** * The unicode "subsitute" character (0x1A). +** * The unicode "substitute" character (0x1A). */ int sqlite3Fts5IsBareword(char t){ u8 aBareword[128] = { diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 877c3f79c..0a9b08ed1 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -86,9 +86,13 @@ struct Fts5ExprNode { /* Child nodes. For a NOT node, this array always contains 2 entries. For ** AND or OR nodes, it contains 2 or more entries. */ int nChild; /* Number of child nodes */ - Fts5ExprNode *apChild[1]; /* Array of child nodes */ + Fts5ExprNode *apChild[FLEXARRAY]; /* Array of child nodes */ }; +/* Size (in bytes) of an Fts5ExprNode object that holds up to N children */ +#define SZ_FTS5EXPRNODE(N) \ + (offsetof(Fts5ExprNode,apChild) + (N)*sizeof(Fts5ExprNode*)) + #define Fts5NodeIsString(p) ((p)->eType==FTS5_TERM || (p)->eType==FTS5_STRING) /* @@ -119,9 +123,13 @@ struct Fts5ExprPhrase { Fts5ExprNode *pNode; /* FTS5_STRING node this phrase is part of */ Fts5Buffer poslist; /* Current position list */ int nTerm; /* Number of entries in aTerm[] */ - Fts5ExprTerm aTerm[1]; /* Terms that make up this phrase */ + Fts5ExprTerm aTerm[FLEXARRAY]; /* Terms that make up this phrase */ }; +/* Size (in bytes) of an Fts5ExprPhrase object that holds up to N terms */ +#define SZ_FTS5EXPRPHRASE(N) \ + (offsetof(Fts5ExprPhrase,aTerm) + (N)*sizeof(Fts5ExprTerm)) + /* ** One or more phrases that must appear within a certain token distance of ** each other within each matching document. @@ -130,9 +138,12 @@ struct Fts5ExprNearset { int nNear; /* NEAR parameter */ Fts5Colset *pColset; /* Columns to search (NULL -> all columns) */ int nPhrase; /* Number of entries in aPhrase[] array */ - Fts5ExprPhrase *apPhrase[1]; /* Array of phrase pointers */ + Fts5ExprPhrase *apPhrase[FLEXARRAY]; /* Array of phrase pointers */ }; +/* Size (in bytes) of an Fts5ExprNearset object covering up to N phrases */ +#define SZ_FTS5EXPRNEARSET(N) \ + (offsetof(Fts5ExprNearset,apPhrase)+(N)*sizeof(Fts5ExprPhrase*)) /* ** Parse context. @@ -292,7 +303,7 @@ int sqlite3Fts5ExprNew( /* If the LHS of the MATCH expression was a user column, apply the ** implicit column-filter. */ if( sParse.rc==SQLITE_OK && iColnCol ){ - int n = sizeof(Fts5Colset); + int n = SZ_FTS5COLSET(1); Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n); if( pColset ){ pColset->nCol = 1; @@ -1650,7 +1661,7 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset( if( pParse->rc==SQLITE_OK ){ if( pNear==0 ){ sqlite3_int64 nByte; - nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*); + nByte = SZ_FTS5EXPRNEARSET(SZALLOC+1); pRet = sqlite3_malloc64(nByte); if( pRet==0 ){ pParse->rc = SQLITE_NOMEM; @@ -1661,7 +1672,7 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset( int nNew = pNear->nPhrase + SZALLOC; sqlite3_int64 nByte; - nByte = sizeof(Fts5ExprNearset) + nNew * sizeof(Fts5ExprPhrase*); + nByte = SZ_FTS5EXPRNEARSET(nNew+1); pRet = (Fts5ExprNearset*)sqlite3_realloc64(pNear, nByte); if( pRet==0 ){ pParse->rc = SQLITE_NOMEM; @@ -1752,12 +1763,12 @@ static int fts5ParseTokenize( int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0); pNew = (Fts5ExprPhrase*)sqlite3_realloc64(pPhrase, - sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew + SZ_FTS5EXPRPHRASE(nNew+1) ); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ - if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); + if( pPhrase==0 ) memset(pNew, 0, SZ_FTS5EXPRPHRASE(1)); pCtx->pPhrase = pPhrase = pNew; pNew->nTerm = nNew - SZALLOC; } @@ -1865,7 +1876,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm( if( sCtx.pPhrase==0 ){ /* This happens when parsing a token or quoted phrase that contains ** no token characters at all. (e.g ... MATCH '""'). */ - sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, sizeof(Fts5ExprPhrase)); + sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, SZ_FTS5EXPRPHRASE(1)); }else if( sCtx.pPhrase->nTerm ){ sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix; } @@ -1900,19 +1911,18 @@ int sqlite3Fts5ExprClonePhrase( sizeof(Fts5ExprPhrase*)); } if( rc==SQLITE_OK ){ - pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, - sizeof(Fts5ExprNode)); + pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, SZ_FTS5EXPRNODE(1)); } if( rc==SQLITE_OK ){ - pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, - sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)); + pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, + SZ_FTS5EXPRNEARSET(2)); } if( rc==SQLITE_OK && ALWAYS(pOrig!=0) ){ Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset; if( pColsetOrig ){ sqlite3_int64 nByte; Fts5Colset *pColset; - nByte = sizeof(Fts5Colset) + (pColsetOrig->nCol-1) * sizeof(int); + nByte = SZ_FTS5COLSET(pColsetOrig->nCol); pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&rc, nByte); if( pColset ){ memcpy(pColset, pColsetOrig, (size_t)nByte); @@ -1940,7 +1950,7 @@ int sqlite3Fts5ExprClonePhrase( }else{ /* This happens when parsing a token or quoted phrase that contains ** no token characters at all. (e.g ... MATCH '""'). */ - sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase)); + sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, SZ_FTS5EXPRPHRASE(1)); } } @@ -2005,7 +2015,8 @@ void sqlite3Fts5ParseSetDistance( ); return; } - nNear = nNear * 10 + (p->p[i] - '0'); + if( nNear<214748363 ) nNear = nNear * 10 + (p->p[i] - '0'); + /* ^^^^^^^^^^^^^^^--- Prevent integer overflow */ } }else{ nNear = FTS5_DEFAULT_NEARDIST; @@ -2034,7 +2045,7 @@ static Fts5Colset *fts5ParseColset( assert( pParse->rc==SQLITE_OK ); assert( iCol>=0 && iColpConfig->nCol ); - pNew = sqlite3_realloc64(p, sizeof(Fts5Colset) + sizeof(int)*nCol); + pNew = sqlite3_realloc64(p, SZ_FTS5COLSET(nCol+1)); if( pNew==0 ){ pParse->rc = SQLITE_NOMEM; }else{ @@ -2069,7 +2080,7 @@ Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse *pParse, Fts5Colset *p){ int nCol = pParse->pConfig->nCol; pRet = (Fts5Colset*)sqlite3Fts5MallocZero(&pParse->rc, - sizeof(Fts5Colset) + sizeof(int)*nCol + SZ_FTS5COLSET(nCol+1) ); if( pRet ){ int i; @@ -2130,7 +2141,7 @@ Fts5Colset *sqlite3Fts5ParseColset( static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){ Fts5Colset *pRet; if( pOrig ){ - sqlite3_int64 nByte = sizeof(Fts5Colset) + (pOrig->nCol-1) * sizeof(int); + sqlite3_int64 nByte = SZ_FTS5COLSET(pOrig->nCol); pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte); if( pRet ){ memcpy(pRet, pOrig, (size_t)nByte); @@ -2298,7 +2309,7 @@ static Fts5ExprNode *fts5ParsePhraseToAnd( assert( pNear->nPhrase==1 ); assert( pParse->bPhraseToAnd ); - nByte = sizeof(Fts5ExprNode) + nTerm*sizeof(Fts5ExprNode*); + nByte = SZ_FTS5EXPRNODE(nTerm+1); pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte); if( pRet ){ pRet->eType = FTS5_AND; @@ -2308,7 +2319,7 @@ static Fts5ExprNode *fts5ParsePhraseToAnd( pParse->nPhrase--; for(ii=0; iirc, sizeof(Fts5ExprPhrase) + &pParse->rc, SZ_FTS5EXPRPHRASE(1) ); if( pPhrase ){ if( parseGrowPhraseArray(pParse) ){ @@ -2377,7 +2388,7 @@ Fts5ExprNode *sqlite3Fts5ParseNode( if( pRight->eType==eType ) nChild += pRight->nChild-1; } - nByte = sizeof(Fts5ExprNode) + sizeof(Fts5ExprNode*)*(nChild-1); + nByte = SZ_FTS5EXPRNODE(nChild); pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte); if( pRet ){ @@ -3252,7 +3263,7 @@ int sqlite3Fts5ExprInstToken( } /* -** Clear the token mappings for all Fts5IndexIter objects mannaged by +** Clear the token mappings for all Fts5IndexIter objects managed by ** the expression passed as the only argument. */ void sqlite3Fts5ExprClearTokens(Fts5Expr *pExpr){ diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c index 5e0959aa8..a33dec9a9 100644 --- a/ext/fts5/fts5_hash.c +++ b/ext/fts5/fts5_hash.c @@ -20,7 +20,7 @@ typedef struct Fts5HashEntry Fts5HashEntry; /* ** This file contains the implementation of an in-memory hash table used -** to accumuluate "term -> doclist" content before it is flused to a level-0 +** to accumulate "term -> doclist" content before it is flushed to a level-0 ** segment. */ @@ -77,7 +77,7 @@ struct Fts5HashEntry { }; /* -** Eqivalent to: +** Equivalent to: ** ** char *fts5EntryKey(Fts5HashEntry *pEntry){ return zKey; } */ diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index f56aa82e8..63840de1f 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -422,9 +422,13 @@ struct Fts5Structure { u64 nOriginCntr; /* Origin value for next top-level segment */ int nSegment; /* Total segments in this structure */ int nLevel; /* Number of levels in this index */ - Fts5StructureLevel aLevel[1]; /* Array of nLevel level objects */ + Fts5StructureLevel aLevel[FLEXARRAY]; /* Array of nLevel level objects */ }; +/* Size (in bytes) of an Fts5Structure object holding up to N levels */ +#define SZ_FTS5STRUCTURE(N) \ + (offsetof(Fts5Structure,aLevel) + (N)*sizeof(Fts5StructureLevel)) + /* ** An object of type Fts5SegWriter is used to write to segments. */ @@ -554,11 +558,15 @@ struct Fts5SegIter { ** Array of tombstone pages. Reference counted. */ struct Fts5TombstoneArray { - int nRef; /* Number of pointers to this object */ + int nRef; /* Number of pointers to this object */ int nTombstone; - Fts5Data *apTombstone[1]; /* Array of tombstone pages */ + Fts5Data *apTombstone[FLEXARRAY]; /* Array of tombstone pages */ }; +/* Size (in bytes) of an Fts5TombstoneArray holding up to N tombstones */ +#define SZ_FTS5TOMBSTONEARRAY(N) \ + (offsetof(Fts5TombstoneArray,apTombstone)+(N)*sizeof(Fts5Data*)) + /* ** Argument is a pointer to an Fts5Data structure that contains a ** leaf page. @@ -627,9 +635,12 @@ struct Fts5Iter { i64 iSwitchRowid; /* Firstest rowid of other than aFirst[1] */ Fts5CResult *aFirst; /* Current merge state (see above) */ - Fts5SegIter aSeg[1]; /* Array of segment iterators */ + Fts5SegIter aSeg[FLEXARRAY]; /* Array of segment iterators */ }; +/* Size (in bytes) of an Fts5Iter object holding up to N segment iterators */ +#define SZ_FTS5ITER(N) (offsetof(Fts5Iter,aSeg)+(N)*sizeof(Fts5SegIter)) + /* ** An instance of the following type is used to iterate through the contents ** of a doclist-index record. @@ -656,9 +667,13 @@ struct Fts5DlidxLvl { struct Fts5DlidxIter { int nLvl; int iSegid; - Fts5DlidxLvl aLvl[1]; + Fts5DlidxLvl aLvl[FLEXARRAY]; }; +/* Size (in bytes) of an Fts5DlidxIter object with up to N levels */ +#define SZ_FTS5DLIDXITER(N) \ + (offsetof(Fts5DlidxIter,aLvl)+(N)*sizeof(Fts5DlidxLvl)) + static void fts5PutU16(u8 *aOut, u16 iVal){ aOut[0] = (iVal>>8); aOut[1] = (iVal&0xFF); @@ -1026,7 +1041,7 @@ int sqlite3Fts5StructureTest(Fts5Index *p, void *pStruct){ static void fts5StructureMakeWritable(int *pRc, Fts5Structure **pp){ Fts5Structure *p = *pp; if( *pRc==SQLITE_OK && p->nRef>1 ){ - i64 nByte = sizeof(Fts5Structure)+(p->nLevel-1)*sizeof(Fts5StructureLevel); + i64 nByte = SZ_FTS5STRUCTURE(p->nLevel); Fts5Structure *pNew; pNew = (Fts5Structure*)sqlite3Fts5MallocZero(pRc, nByte); if( pNew ){ @@ -1100,10 +1115,7 @@ static int fts5StructureDecode( ){ return FTS5_CORRUPT; } - nByte = ( - sizeof(Fts5Structure) + /* Main structure */ - sizeof(Fts5StructureLevel) * (nLevel-1) /* aLevel[] array */ - ); + nByte = SZ_FTS5STRUCTURE(nLevel); pRet = (Fts5Structure*)sqlite3Fts5MallocZero(&rc, nByte); if( pRet ){ @@ -1183,10 +1195,7 @@ static void fts5StructureAddLevel(int *pRc, Fts5Structure **ppStruct){ if( *pRc==SQLITE_OK ){ Fts5Structure *pStruct = *ppStruct; int nLevel = pStruct->nLevel; - sqlite3_int64 nByte = ( - sizeof(Fts5Structure) + /* Main structure */ - sizeof(Fts5StructureLevel) * (nLevel+1) /* aLevel[] array */ - ); + sqlite3_int64 nByte = SZ_FTS5STRUCTURE(nLevel+2); pStruct = sqlite3_realloc64(pStruct, nByte); if( pStruct ){ @@ -1725,7 +1734,7 @@ static Fts5DlidxIter *fts5DlidxIterInit( int bDone = 0; for(i=0; p->rc==SQLITE_OK && bDone==0; i++){ - sqlite3_int64 nByte = sizeof(Fts5DlidxIter) + i * sizeof(Fts5DlidxLvl); + sqlite3_int64 nByte = SZ_FTS5DLIDXITER(i+1); Fts5DlidxIter *pNew; pNew = (Fts5DlidxIter*)sqlite3_realloc64(pIter, nByte); @@ -1943,7 +1952,7 @@ static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){ static void fts5SegIterAllocTombstone(Fts5Index *p, Fts5SegIter *pIter){ const int nTomb = pIter->pSeg->nPgTombstone; if( nTomb>0 ){ - int nByte = nTomb * sizeof(Fts5Data*) + sizeof(Fts5TombstoneArray); + int nByte = SZ_FTS5TOMBSTONEARRAY(nTomb+1); Fts5TombstoneArray *pNew; pNew = (Fts5TombstoneArray*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pNew ){ @@ -3404,8 +3413,7 @@ static Fts5Iter *fts5MultiIterAlloc( for(nSlot=2; nSlotaSeg[] */ + SZ_FTS5ITER(nSlot) + /* pNew + pNew->aSeg[] */ sizeof(Fts5CResult) * nSlot /* pNew->aFirst[] */ ); if( pNew ){ @@ -5206,7 +5214,7 @@ static void fts5DoSecureDelete( int iDelKeyOff = 0; /* Offset of deleted key, if any */ nIdx = nPg-iPgIdx; - aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16); + aIdx = sqlite3Fts5MallocZero(&p->rc, ((i64)nIdx)+16); if( p->rc ) return; memcpy(aIdx, &aPg[iPgIdx], nIdx); @@ -5771,7 +5779,7 @@ static Fts5Structure *fts5IndexOptimizeStruct( Fts5Structure *pStruct ){ Fts5Structure *pNew = 0; - sqlite3_int64 nByte = sizeof(Fts5Structure); + sqlite3_int64 nByte = SZ_FTS5STRUCTURE(1); int nSeg = pStruct->nSegment; int i; @@ -5800,7 +5808,8 @@ static Fts5Structure *fts5IndexOptimizeStruct( assert( pStruct->aLevel[i].nMerge<=nThis ); } - nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel); + nByte += (((i64)pStruct->nLevel)+1) * sizeof(Fts5StructureLevel); + assert( nByte==SZ_FTS5STRUCTURE(pStruct->nLevel+2) ); pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pNew ){ @@ -6377,9 +6386,13 @@ struct Fts5TokenDataIter { int nIterAlloc; Fts5PoslistReader *aPoslistReader; int *aPoslistToIter; - Fts5Iter *apIter[1]; + Fts5Iter *apIter[FLEXARRAY]; }; +/* Size in bytes of an Fts5TokenDataIter object holding up to N iterators */ +#define SZ_FTS5TOKENDATAITER(N) \ + (offsetof(Fts5TokenDataIter,apIter) + (N)*sizeof(Fts5Iter)) + /* ** The two input arrays - a1[] and a2[] - are in sorted order. This function ** merges the two arrays together and writes the result to output array @@ -6451,7 +6464,7 @@ static void fts5TokendataIterAppendMap( /* ** Sort the contents of the pT->aMap[] array. ** -** The sorting algorithm requries a malloc(). If this fails, an error code +** The sorting algorithm requires a malloc(). If this fails, an error code ** is left in Fts5Index.rc before returning. */ static void fts5TokendataIterSortMap(Fts5Index *p, Fts5TokenDataIter *pT){ @@ -6642,7 +6655,7 @@ static void fts5SetupPrefixIter( && p->pConfig->bPrefixInsttoken ){ s.pTokendata = &s2; - s2.pT = (Fts5TokenDataIter*)fts5IdxMalloc(p, sizeof(*s2.pT)); + s2.pT = (Fts5TokenDataIter*)fts5IdxMalloc(p, SZ_FTS5TOKENDATAITER(1)); } if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ @@ -6688,7 +6701,8 @@ static void fts5SetupPrefixIter( } } - pData = fts5IdxMalloc(p, sizeof(*pData)+s.doclist.n+FTS5_DATA_ZERO_PADDING); + pData = fts5IdxMalloc(p, sizeof(*pData) + + ((i64)s.doclist.n)+FTS5_DATA_ZERO_PADDING); assert( pData!=0 || p->rc!=SQLITE_OK ); if( pData ){ pData->p = (u8*)&pData[1]; @@ -6769,15 +6783,17 @@ int sqlite3Fts5IndexRollback(Fts5Index *p){ ** and the initial version of the "averages" record (a zero-byte blob). */ int sqlite3Fts5IndexReinit(Fts5Index *p){ - Fts5Structure s; + Fts5Structure *pTmp; + u8 tmpSpace[SZ_FTS5STRUCTURE(1)]; fts5StructureInvalidate(p); fts5IndexDiscardData(p); - memset(&s, 0, sizeof(Fts5Structure)); + pTmp = (Fts5Structure*)tmpSpace; + memset(pTmp, 0, SZ_FTS5STRUCTURE(1)); if( p->pConfig->bContentlessDelete ){ - s.nOriginCntr = 1; + pTmp->nOriginCntr = 1; } fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); - fts5StructureWrite(p, &s); + fts5StructureWrite(p, pTmp); return fts5IndexReturn(p); } @@ -6985,7 +7001,7 @@ static Fts5TokenDataIter *fts5AppendTokendataIter( if( p->rc==SQLITE_OK ){ if( pIn==0 || pIn->nIter==pIn->nIterAlloc ){ int nAlloc = pIn ? pIn->nIterAlloc*2 : 16; - int nByte = nAlloc * sizeof(Fts5Iter*) + sizeof(Fts5TokenDataIter); + int nByte = SZ_FTS5TOKENDATAITER(nAlloc+1); Fts5TokenDataIter *pNew = (Fts5TokenDataIter*)sqlite3_realloc(pIn, nByte); if( pNew==0 ){ @@ -7501,7 +7517,8 @@ static int fts5SetupPrefixIterTokendata( fts5BufferGrow(&p->rc, &token, nToken+1); assert( token.p!=0 || p->rc!=SQLITE_OK ); - ctx.pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(*ctx.pT)); + ctx.pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, + SZ_FTS5TOKENDATAITER(1)); if( p->rc==SQLITE_OK ){ @@ -7632,7 +7649,8 @@ int sqlite3Fts5IndexIterWriteTokendata( if( pIter->nSeg>0 ){ /* This is a prefix term iterator. */ if( pT==0 ){ - pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(*pT)); + pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, + SZ_FTS5TOKENDATAITER(1)); pIter->pTokenDataIter = pT; } if( pT ){ @@ -8666,7 +8684,7 @@ static void fts5DecodeRowid( #if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ - int iSegid, iHeight, iPgno, bDlidx, bTomb; /* Rowid compenents */ + int iSegid, iHeight, iPgno, bDlidx, bTomb; /* Rowid components */ fts5DecodeRowid(iKey, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno); if( iSegid==0 ){ @@ -8912,7 +8930,7 @@ static void fts5DecodeFunction( ** buffer overreads even if the record is corrupt. */ n = sqlite3_value_bytes(apVal[1]); aBlob = sqlite3_value_blob(apVal[1]); - nSpace = n + FTS5_DATA_ZERO_PADDING; + nSpace = ((i64)n) + FTS5_DATA_ZERO_PADDING; a = (u8*)sqlite3Fts5MallocZero(&rc, nSpace); if( a==0 ) goto decode_out; if( n>0 ) memcpy(a, aBlob, n); diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 9f504bb3a..e888abf21 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -170,9 +170,11 @@ struct Fts5Sorter { i64 iRowid; /* Current rowid */ const u8 *aPoslist; /* Position lists for current row */ int nIdx; /* Number of entries in aIdx[] */ - int aIdx[1]; /* Offsets into aPoslist for current row */ + int aIdx[FLEXARRAY]; /* Offsets into aPoslist for current row */ }; +/* Size (int bytes) of an Fts5Sorter object with N indexes */ +#define SZ_FTS5SORTER(N) (offsetof(Fts5Sorter,nIdx)+((N+2)/2)*sizeof(i64)) /* ** Virtual-table cursor object. @@ -1050,7 +1052,7 @@ static int fts5CursorFirstSorted( const char *zRankArgs = pCsr->zRankArgs; nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); - nByte = sizeof(Fts5Sorter) + sizeof(int) * (nPhrase-1); + nByte = SZ_FTS5SORTER(nPhrase); pSorter = (Fts5Sorter*)sqlite3_malloc64(nByte); if( pSorter==0 ) return SQLITE_NOMEM; memset(pSorter, 0, (size_t)nByte); @@ -3801,8 +3803,8 @@ static int fts5Init(sqlite3 *db){ ** its entry point to enable the matchinfo() demo. */ #ifdef SQLITE_FTS5_ENABLE_TEST_MI if( rc==SQLITE_OK ){ - extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3*); - rc = sqlite3Fts5TestRegisterMatchinfo(db); + extern int sqlite3Fts5TestRegisterMatchinfoAPI(fts5_api*); + rc = sqlite3Fts5TestRegisterMatchinfoAPI(&pGlobal->api); } #endif diff --git a/ext/fts5/fts5_test_mi.c b/ext/fts5/fts5_test_mi.c index 6f2d6e7ea..e8648a4d2 100644 --- a/ext/fts5/fts5_test_mi.c +++ b/ext/fts5/fts5_test_mi.c @@ -14,7 +14,7 @@ ** versions of FTS5. It contains the implementation of an FTS5 auxiliary ** function very similar to the FTS4 function matchinfo(): ** -** https://www.sqlite.org/fts3.html#matchinfo +** https://sqlite.org/fts3.html#matchinfo ** ** Known differences are that: ** @@ -393,17 +393,13 @@ static void fts5MatchinfoFunc( } } -int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){ - int rc; /* Return code */ - fts5_api *pApi; /* FTS5 API functions */ - - /* Extract the FTS5 API pointer from the database handle. The - ** fts5_api_from_db() function above is copied verbatim from the - ** FTS5 documentation. Refer there for details. */ - rc = fts5_api_from_db(db, &pApi); - if( rc!=SQLITE_OK ) return rc; +/* +** Register "matchinfo" with global API object pApi. +*/ +int sqlite3Fts5TestRegisterMatchinfoAPI(fts5_api *pApi){ + int rc; - /* If fts5_api_from_db() returns NULL, then either FTS5 is not registered + /* If fts5_api_from_db() returned NULL, then either FTS5 is not registered ** with this database handle, or an error (OOM perhaps?) has occurred. ** ** Also check that the fts5_api object is version 2 or newer. @@ -418,4 +414,20 @@ int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){ return rc; } +/* +** Register "matchinfo" with database handle db. +*/ +int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db){ + int rc; /* Return code */ + fts5_api *pApi; /* FTS5 API functions */ + + /* Extract the FTS5 API pointer from the database handle. The + ** fts5_api_from_db() function above is copied verbatim from the + ** FTS5 documentation. Refer there for details. */ + rc = fts5_api_from_db(db, &pApi); + if( rc!=SQLITE_OK ) return rc; + + return sqlite3Fts5TestRegisterMatchinfoAPI(pApi); +} + #endif /* SQLITE_ENABLE_FTS5 */ diff --git a/ext/fts5/fts5_unicode2.c b/ext/fts5/fts5_unicode2.c index cc164a456..2133d5d5b 100644 --- a/ext/fts5/fts5_unicode2.c +++ b/ext/fts5/fts5_unicode2.c @@ -778,4 +778,3 @@ void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){ } aAscii[0] = 0; /* 0x00 is never a token character */ } - diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c index fb280567f..b157ab0d9 100644 --- a/ext/fts5/fts5_vocab.c +++ b/ext/fts5/fts5_vocab.c @@ -193,12 +193,12 @@ static int fts5VocabInitVtab( *pzErr = sqlite3_mprintf("wrong number of vtable arguments"); rc = SQLITE_ERROR; }else{ - int nByte; /* Bytes of space to allocate */ + i64 nByte; /* Bytes of space to allocate */ const char *zDb = bDb ? argv[3] : argv[1]; const char *zTab = bDb ? argv[4] : argv[3]; const char *zType = bDb ? argv[5] : argv[4]; - int nDb = (int)strlen(zDb)+1; - int nTab = (int)strlen(zTab)+1; + i64 nDb = strlen(zDb)+1; + i64 nTab = strlen(zTab)+1; int eType = 0; rc = fts5VocabTableType(zType, pzErr, &eType); diff --git a/ext/fts5/test/fts5matchinfo.test b/ext/fts5/test/fts5matchinfo.test index 93361a5fe..a3bce869f 100644 --- a/ext/fts5/test/fts5matchinfo.test +++ b/ext/fts5/test/fts5matchinfo.test @@ -520,4 +520,28 @@ do_execsql_test 15.3 { {columnsize {1 1} columntext {c d} columntotalsize {2 2} poslist {} tokenize {c d} rowcount 2} } +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 16.0 { + CREATE TABLE t1(x); + BEGIN EXCLUSIVE; +} + +do_test 16.1 { + set rc [catch { + sqlite3 db2 test.db + db2 eval {SELECT * FROM t1} + } errmsg] + lappend rc $errmsg +} {1 {database is locked}} + +do_execsql_test 16.2 { + ROLLBACK; +} + +do_test 16.3 { + catchsql { SELECT * FROM t1 } db2 +} {0 {}} + finish_test diff --git a/ext/fts5/test/fts5simple3.test b/ext/fts5/test/fts5simple3.test index 680448081..bc3ebfc7c 100644 --- a/ext/fts5/test/fts5simple3.test +++ b/ext/fts5/test/fts5simple3.test @@ -81,7 +81,7 @@ do_execsql_test 3.0 { } #------------------------------------------------------------------------- -# Test that a crash occuring when the second or subsequent tokens in a +# Test that a crash occurring when the second or subsequent tokens in a # phrase matched zero rows has been fixed. # do_execsql_test 4.0 { diff --git a/ext/jni/GNUmakefile b/ext/jni/GNUmakefile index 26a38dad9..f2fc7fb7b 100644 --- a/ext/jni/GNUmakefile +++ b/ext/jni/GNUmakefile @@ -79,7 +79,7 @@ $(bin.version-info): $(dir.tool)/version-info.c $(sqlite3.h) $(dir.top)/Makefile $(MAKE) -C $(dir.top) version-info # Be explicit about which Java files to compile so that we can work on -# in-progress files without requiring them to be in a compilable statae. +# in-progress files without requiring them to be in a compilable state. JAVA_FILES.main := $(patsubst %,$(dir.src.jni)/annotation/%,\ Experimental.java \ NotNull.java \ @@ -429,7 +429,7 @@ $(doc.index): $(JAVA_FILES.main) $(MAKEFILE) .FORCE: doc doc: $(doc.index) javadoc: $(doc.index) -# Force rebild of docs +# Force rebuild of docs redoc: @rm -f $(doc.index) @$(MAKE) doc @@ -451,7 +451,7 @@ distclean: clean -rm -fr $(dir.bld.c) $(dir.doc) ######################################################################## -# disttribution bundle rules... +# distribution bundle rules... ifeq (,$(filter snapshot,$(MAKECMDGOALS))) dist-name-prefix := sqlite-jni diff --git a/ext/jni/README.md b/ext/jni/README.md index fc7b5f761..5ad79fce9 100644 --- a/ext/jni/README.md +++ b/ext/jni/README.md @@ -169,7 +169,7 @@ deliberately return an error code, instead of segfaulting, when passed a `null`. Client-defined callbacks _must never throw exceptions_ unless _very -explitly documented_ as being throw-safe. Exceptions are generally +explicitly documented_ as being throw-safe. Exceptions are generally reserved for higher-level bindings which are constructed to specifically deal with them and ensure that they do not leak C-level resources. In some cases, callback handlers are permitted to throw, in diff --git a/ext/jni/src/c/sqlite3-jni.c b/ext/jni/src/c/sqlite3-jni.c index 5deff19ef..d6723453d 100644 --- a/ext/jni/src/c/sqlite3-jni.c +++ b/ext/jni/src/c/sqlite3-jni.c @@ -133,7 +133,7 @@ ** Which sqlite3.c we're using needs to be configurable to enable ** building against a custom copy, e.g. the SEE variant. We have to ** include sqlite3.c, as opposed to sqlite3.h, in order to get access -** to some interal details like SQLITE_MAX_... and friends. This +** to some internal details like SQLITE_MAX_... and friends. This ** increases the rebuild time considerably but we need this in order ** to access some internal functionality and keep the to-Java-exported ** values of SQLITE_MAX_... and SQLITE_LIMIT_... in sync with the C @@ -948,7 +948,7 @@ static JNIEnv * s3jni_env(void){ } /* -** Fetches the S3JniGlobal.envCache row for the given env, allocing a +** Fetches the S3JniGlobal.envCache row for the given env, allocating a ** row if needed. When a row is allocated, its state is initialized ** insofar as possible. Calls (*env)->FatalError() if allocation of an ** entry fails. That's hypothetically possible but "shouldn't happen." @@ -1603,7 +1603,7 @@ static void * NativePointerHolder__get(JNIEnv * env, jobject jNph, ** 2023-11-09: testing has not revealed any measurable performance ** difference between the approach of passing type T to C compared to ** passing pointer-to-T to C, and adding support for the latter -** everywhere requires sigificantly more code. As of this writing, the +** everywhere requires significantly more code. As of this writing, the ** older/simpler approach is being applied except for (A) where the ** newer approach has already been applied and (B) hot-spot APIs where ** a difference of microseconds (i.e. below our testing measurement @@ -3290,7 +3290,7 @@ static void s3jni_rollback_hook_impl(void *pP){ static jobject s3jni_commit_rollback_hook(int isCommit, JNIEnv * const env, jlong jpDb, jobject jHook){ S3JniDb * ps; - jobject pOld = 0; /* previous hoook */ + jobject pOld = 0; /* previous hook */ S3JniHook * pHook; /* ps->hooks.commit|rollback */ S3JniDb_mutex_enter; @@ -5512,7 +5512,7 @@ static inline jobject new_java_fts5_api(JNIEnv * const env, fts5_api *sv){ ** Returns a per-JNIEnv global ref to the Fts5ExtensionApi singleton ** instance, or NULL on OOM. */ -static jobject s3jni_getFts5ExensionApi(JNIEnv * const env){ +static jobject s3jni_getFts5ExtensionApi(JNIEnv * const env){ if( !SJG.fts5.jExt ){ S3JniGlobal_mutex_enter; if( !SJG.fts5.jExt ){ @@ -5578,7 +5578,7 @@ JniDeclFtsApi(jobject,getInstanceForDb)(JniArgsEnvClass,jobject jDb){ JniDeclFtsXA(jobject,getInstance)(JniArgsEnvClass){ - return s3jni_getFts5ExensionApi(env); + return s3jni_getFts5ExtensionApi(env); } JniDeclFtsXA(jint,xColumnCount)(JniArgsEnvObj,jobject jCtx){ @@ -5641,7 +5641,7 @@ static void s3jni_fts5_extension_function(Fts5ExtensionApi const *pApi, S3JniDeclLocal_env; assert(pAux); - jFXA = s3jni_getFts5ExensionApi(env); + jFXA = s3jni_getFts5ExtensionApi(env); if( !jFXA ) goto error_oom; jpFts = new_java_Fts5Context(env, pFts); if( !jpFts ) goto error_oom; @@ -5696,7 +5696,7 @@ JniDeclFtsApi(jint,xCreateFunction)(JniArgsEnvObj, jstring jName, typedef struct S3JniFts5AuxData S3JniFts5AuxData; /* -** TODO: this middle-man struct is no longer necessary. Conider +** TODO: this middle-man struct is no longer necessary. Consider ** removing it and passing around jObj itself instead. */ struct S3JniFts5AuxData { diff --git a/ext/jni/src/c/sqlite3-jni.h b/ext/jni/src/c/sqlite3-jni.h index 082a20212..6f93bf8ab 100644 --- a/ext/jni/src/c/sqlite3-jni.h +++ b/ext/jni/src/c/sqlite3-jni.h @@ -1342,7 +1342,7 @@ JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1data_1count * Method: sqlite3_db_config * Signature: (Lorg/sqlite/jni/capi/sqlite3;IILorg/sqlite/jni/capi/OutputPointer/Int32;)I */ -JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1db_1config__Lorg_sqlite_jni_capi_sqlite3_2IILorg_sqlite_jni_capi_OutputPointer_Int32_2 +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1db_1config__Lorg_sqlite_jni_capi_sqlite3_2IILorg_sqlite_jni_capi_OutputPointer_00024Int32_2 (JNIEnv *, jclass, jobject, jint, jint, jobject); /* diff --git a/ext/jni/src/org/sqlite/jni/annotation/NotNull.java b/ext/jni/src/org/sqlite/jni/annotation/NotNull.java index 0c31782f2..287308244 100644 --- a/ext/jni/src/org/sqlite/jni/annotation/NotNull.java +++ b/ext/jni/src/org/sqlite/jni/annotation/NotNull.java @@ -31,11 +31,11 @@ never pass a null value to the callback for that parameter.

    Passing a null, for this annotation's definition of null, for - any parameter marked with this annoation specifically invokes + any parameter marked with this annotation specifically invokes undefined behavior (see below).

    Passing 0 (i.e. C NULL) or a negative value for any long-type - parameter marked with this annoation specifically invokes undefined + parameter marked with this annotation specifically invokes undefined behavior (see below). Such values are treated as C pointers in the JNI layer.

    diff --git a/ext/jni/src/org/sqlite/jni/capi/CApi.java b/ext/jni/src/org/sqlite/jni/capi/CApi.java index b5d08306e..731fb0ac3 100644 --- a/ext/jni/src/org/sqlite/jni/capi/CApi.java +++ b/ext/jni/src/org/sqlite/jni/capi/CApi.java @@ -12,14 +12,9 @@ ** This file declares the main JNI bindings for the sqlite3 C API. */ package org.sqlite.jni.capi; +import java.util.Arrays; import java.nio.charset.StandardCharsets; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; -import java.lang.annotation.Documented; -import java.lang.annotation.ElementType; import org.sqlite.jni.annotation.*; -import java.util.Arrays; /** This class contains the entire C-style sqlite3 JNI API binding, @@ -169,7 +164,7 @@ argument is a boolean instead of an int and (B) the returned

    Like the C API, it returns 0 if allocation fails or if initialize is false and no prior aggregate context was allocated for cx. If initialize is true then it returns 0 only on - allocation error. In all casses, 0 is considered the sentinel + allocation error. In all cases, 0 is considered the sentinel "not a key" value. */ public static native long sqlite3_aggregate_context(sqlite3_context cx, boolean initialize); @@ -199,16 +194,16 @@ public static int sqlite3_backup_finish(@NotNull sqlite3_backup b){ } private static native sqlite3_backup sqlite3_backup_init( - @NotNull long ptrToDbDest, @NotNull String destTableName, - @NotNull long ptrToDbSrc, @NotNull String srcTableName + @NotNull long ptrToDbDest, @NotNull String destSchemaName, + @NotNull long ptrToDbSrc, @NotNull String srcSchemaName ); public static sqlite3_backup sqlite3_backup_init( - @NotNull sqlite3 dbDest, @NotNull String destTableName, - @NotNull sqlite3 dbSrc, @NotNull String srcTableName + @NotNull sqlite3 dbDest, @NotNull String destSchemaName, + @NotNull sqlite3 dbSrc, @NotNull String srcSchemaName ){ - return sqlite3_backup_init( dbDest.getNativePointer(), destTableName, - dbSrc.getNativePointer(), srcTableName ); + return sqlite3_backup_init( dbDest.getNativePointer(), destSchemaName, + dbSrc.getNativePointer(), srcSchemaName ); } private static native int sqlite3_backup_pagecount(@NotNull long ptrToBackup); @@ -264,7 +259,7 @@ public static int sqlite3_bind_blob( } /** - Convenience overload which is equivalant to passing its arguments + Convenience overload which is equivalent to passing its arguments to sqlite3_bind_nio_buffer() with the values 0 and -1 for the final two arguments. */ @@ -312,7 +307,7 @@ private static native int sqlite3_bind_java_object( The byte range of the buffer may be restricted by providing a start index and a number of bytes. beginPos may not be negative. - Negative howMany is interpretated as the remainder of the buffer + Negative howMany is interpreted as the remainder of the buffer past the given start position, up to the buffer's limit() (as opposed its capacity()). @@ -563,7 +558,7 @@ public static sqlite3_blob sqlite3_blob_open( sqlite3_blob_open(db.getNativePointer(), dbName, tableName, columnName, iRow, flags, out); return out.take(); - }; + } private static native int sqlite3_blob_read( @NotNull long ptrToBlob, @NotNull byte[] target, int srcOffset @@ -1076,7 +1071,7 @@ private static native int sqlite3_config__SQLLOG( /**

    Works like in the C API with the exception that it only supports - the following subset of configution flags: + the following subset of configuration flags:

    SQLITE_CONFIG_SINGLETHREAD SQLITE_CONFIG_MULTITHREAD @@ -1298,7 +1293,7 @@ public static sqlite3 sqlite3_open(@Nullable String filename){ final OutputPointer.sqlite3 out = new OutputPointer.sqlite3(); sqlite3_open(filename, out); return out.take(); - }; + } public static native int sqlite3_open_v2( @Nullable String filename, @NotNull OutputPointer.sqlite3 ppDb, @@ -1314,7 +1309,7 @@ public static sqlite3 sqlite3_open_v2(@Nullable String filename, int flags, final OutputPointer.sqlite3 out = new OutputPointer.sqlite3(); sqlite3_open_v2(filename, out, flags, zVfs); return out.take(); - }; + } /** The sqlite3_prepare() family of functions require slightly @@ -1410,7 +1405,7 @@ private static native int sqlite3_prepare_v2( /** Works like the canonical sqlite3_prepare_v2() but its "tail" - output paramter is returned as the index offset into the given + output parameter is returned as the index offset into the given byte array at which SQL parsing stopped. */ public static int sqlite3_prepare_v2( @@ -1462,7 +1457,7 @@ private static native int sqlite3_prepare_v3( /** Works like the canonical sqlite3_prepare_v2() but its "tail" - output paramter is returned as the index offset into the given + output parameter is returned as the index offset into the given byte array at which SQL parsing stopped. */ public static int sqlite3_prepare_v3( @@ -1542,7 +1537,7 @@ public static int sqlite3_prepare_multi( int rc = 0; final OutputPointer.sqlite3_stmt outStmt = new OutputPointer.sqlite3_stmt(); while( 0==rc && pos0 ){ sqlChunk = Arrays.copyOfRange(sqlChunk, pos, sqlChunk.length); diff --git a/ext/jni/src/org/sqlite/jni/capi/CallbackProxy.java b/ext/jni/src/org/sqlite/jni/capi/CallbackProxy.java index 7df748e8d..04000a3f3 100644 --- a/ext/jni/src/org/sqlite/jni/capi/CallbackProxy.java +++ b/ext/jni/src/org/sqlite/jni/capi/CallbackProxy.java @@ -13,12 +13,12 @@ */ package org.sqlite.jni.capi; /** - This marker interface exists soley for use as a documentation and + This marker interface exists solely for use as a documentation and class-grouping tool. It should be applied to interfaces or classes which have a call() method implementing some specific callback interface on behalf of the C library. -

    Unless very explicitely documented otherwise, callbacks must +

    Unless very explicitly documented otherwise, callbacks must never throw. Any which do throw but should not might trigger debug output regarding the error, but the exception will not be propagated. For callback interfaces which support returning error @@ -34,7 +34,7 @@ callback interface on behalf of the C library. proxying for, minus the {@code sqlite3_} prefix, plus a {@code Callback} suffix. e.g. {@code sqlite3_busy_handler()}'s callback is named {@code BusyHandlerCallback}. Exceptions are made where that - would potentially be ambiguous, e.g. {@link ConfigSqllogCallback} + would potentially be ambiguous, e.g. {@link ConfigSqlLogCallback} instead of {@code ConfigCallback} because the {@code sqlite3_config()} interface may need to support more callback types in the future. diff --git a/ext/jni/src/org/sqlite/jni/capi/OutputPointer.java b/ext/jni/src/org/sqlite/jni/capi/OutputPointer.java index 7bf7529da..f50d0c57c 100644 --- a/ext/jni/src/org/sqlite/jni/capi/OutputPointer.java +++ b/ext/jni/src/org/sqlite/jni/capi/OutputPointer.java @@ -20,7 +20,7 @@ from the native JNI code is unduly quirky due to a lack of autoboxing at that level. -

    The usage is similar for all of thes types: +

    The usage is similar for all of these types:

    {@code
        OutputPointer.sqlite3 out = new OutputPointer.sqlite3();
    @@ -55,9 +55,9 @@ public static final class sqlite3 {
         /** Sets the current value to null. */
         public void clear(){value = null;}
         /** Returns the current value. */
    -    public final org.sqlite.jni.capi.sqlite3 get(){return value;}
    +    public org.sqlite.jni.capi.sqlite3 get(){return value;}
         /** Equivalent to calling get() then clear(). */
    -    public final org.sqlite.jni.capi.sqlite3 take(){
    +    public org.sqlite.jni.capi.sqlite3 take(){
           final org.sqlite.jni.capi.sqlite3 v = value;
           value = null;
           return v;
    @@ -76,9 +76,9 @@ public static final class sqlite3_blob {
         /** Sets the current value to null. */
         public void clear(){value = null;}
         /** Returns the current value. */
    -    public final org.sqlite.jni.capi.sqlite3_blob get(){return value;}
    +    public org.sqlite.jni.capi.sqlite3_blob get(){return value;}
         /** Equivalent to calling get() then clear(). */
    -    public final org.sqlite.jni.capi.sqlite3_blob take(){
    +    public org.sqlite.jni.capi.sqlite3_blob take(){
           final org.sqlite.jni.capi.sqlite3_blob v = value;
           value = null;
           return v;
    @@ -98,9 +98,9 @@ public static final class sqlite3_stmt {
         /** Sets the current value to null. */
         public void clear(){value = null;}
         /** Returns the current value. */
    -    public final org.sqlite.jni.capi.sqlite3_stmt get(){return value;}
    +    public org.sqlite.jni.capi.sqlite3_stmt get(){return value;}
         /** Equivalent to calling get() then clear(). */
    -    public final org.sqlite.jni.capi.sqlite3_stmt take(){
    +    public org.sqlite.jni.capi.sqlite3_stmt take(){
           final org.sqlite.jni.capi.sqlite3_stmt v = value;
           value = null;
           return v;
    @@ -120,9 +120,9 @@ public static final class sqlite3_value {
         /** Sets the current value to null. */
         public void clear(){value = null;}
         /** Returns the current value. */
    -    public final org.sqlite.jni.capi.sqlite3_value get(){return value;}
    +    public org.sqlite.jni.capi.sqlite3_value get(){return value;}
         /** Equivalent to calling get() then clear(). */
    -    public final org.sqlite.jni.capi.sqlite3_value take(){
    +    public org.sqlite.jni.capi.sqlite3_value take(){
           final org.sqlite.jni.capi.sqlite3_value v = value;
           value = null;
           return v;
    @@ -144,9 +144,9 @@ public static final class Bool {
         /** Initializes with the value v. */
         public Bool(boolean v){value = v;}
         /** Returns the current value. */
    -    public final boolean get(){return value;}
    +    public boolean get(){return value;}
         /** Sets the current value to v. */
    -    public final void set(boolean v){value = v;}
    +    public void set(boolean v){value = v;}
       }
     
       /**
    @@ -164,9 +164,9 @@ public static final class Int32 {
         /** Initializes with the value v. */
         public Int32(int v){value = v;}
         /** Returns the current value. */
    -    public final int get(){return value;}
    +    public int get(){return value;}
         /** Sets the current value to v. */
    -    public final void set(int v){value = v;}
    +    public void set(int v){value = v;}
       }
     
       /**
    @@ -184,9 +184,9 @@ public static final class Int64 {
         /** Initializes with the value v. */
         public Int64(long v){value = v;}
         /** Returns the current value. */
    -    public final long get(){return value;}
    +    public long get(){return value;}
         /** Sets the current value. */
    -    public final void set(long v){value = v;}
    +    public void set(long v){value = v;}
       }
     
       /**
    @@ -204,9 +204,9 @@ public static final class String {
         /** Initializes with the value v. */
         public String(java.lang.String v){value = v;}
         /** Returns the current value. */
    -    public final java.lang.String get(){return value;}
    +    public java.lang.String get(){return value;}
         /** Sets the current value. */
    -    public final void set(java.lang.String v){value = v;}
    +    public void set(java.lang.String v){value = v;}
       }
     
       /**
    @@ -224,9 +224,9 @@ public static final class ByteArray {
         /** Initializes with the value v. */
         public ByteArray(byte[] v){value = v;}
         /** Returns the current value. */
    -    public final byte[] get(){return value;}
    +    public byte[] get(){return value;}
         /** Sets the current value. */
    -    public final void set(byte[] v){value = v;}
    +    public void set(byte[] v){value = v;}
       }
     
       /**
    @@ -246,8 +246,8 @@ public static final class ByteBuffer {
         /** Initializes with the value v. */
         public ByteBuffer(java.nio.ByteBuffer v){value = v;}
         /** Returns the current value. */
    -    public final java.nio.ByteBuffer get(){return value;}
    +    public java.nio.ByteBuffer get(){return value;}
         /** Sets the current value. */
    -    public final void set(java.nio.ByteBuffer v){value = v;}
    +    public void set(java.nio.ByteBuffer v){value = v;}
       }
     }
    diff --git a/ext/jni/src/org/sqlite/jni/capi/PrepareMultiCallback.java b/ext/jni/src/org/sqlite/jni/capi/PrepareMultiCallback.java
    index 9f6dd478c..35bb069c4 100644
    --- a/ext/jni/src/org/sqlite/jni/capi/PrepareMultiCallback.java
    +++ b/ext/jni/src/org/sqlite/jni/capi/PrepareMultiCallback.java
    @@ -20,7 +20,7 @@ public interface PrepareMultiCallback extends CallbackProxy {
     
       /**
          Gets passed a sqlite3_stmt which it may handle in arbitrary ways,
    -     transfering ownership of it to this function.
    +     transferring ownership of it to this function.
     
          sqlite3_prepare_multi() will _not_ finalize st - it is up
          to the call() implementation how st is handled.
    @@ -65,7 +65,7 @@ Calls the call() method of the proxied callback and either returns its
          A PrepareMultiCallback impl which steps entirely through a result set,
          ignoring all non-error results.
       */
    -  public static final class StepAll implements PrepareMultiCallback {
    +  final class StepAll implements PrepareMultiCallback {
         public StepAll(){}
         /**
            Calls sqlite3_step() on st until it returns something other than
    diff --git a/ext/jni/src/org/sqlite/jni/capi/SQLTester.java b/ext/jni/src/org/sqlite/jni/capi/SQLTester.java
    index c68785e2c..bc2e75f8b 100644
    --- a/ext/jni/src/org/sqlite/jni/capi/SQLTester.java
    +++ b/ext/jni/src/org/sqlite/jni/capi/SQLTester.java
    @@ -13,7 +13,6 @@
     ** SQLTester framework.
     */
     package org.sqlite.jni.capi;
    -import java.util.List;
     import java.util.ArrayList;
     import java.util.Arrays;
     import java.nio.charset.StandardCharsets;
    @@ -31,7 +30,7 @@ enum ResultBufferMode {
       ESCAPED,
       //! Append output as-is
       ASIS
    -};
    +}
     
     /**
        Modes to specify how to emit multi-row output from
    @@ -42,7 +41,7 @@ enum ResultRowMode {
       ONELINE,
       //! Add a newline between each result row.
       NEWLINE
    -};
    +}
     
     /**
        Base exception type for test-related failures.
    @@ -264,7 +263,7 @@ public void runTests() throws Exception {
             threw = true;
             outln("🔥EXCEPTION: ",e.getClass().getSimpleName(),": ",e.getMessage());
             ++nAbortedScript;
    -        if( keepGoing ) outln("Continuing anyway becaure of the keep-going option.");
    +        if( keepGoing ) outln("Continuing anyway because of the keep-going option.");
             else if( e.isFatal() ) throw e;
           }finally{
             final long timeEnd = System.currentTimeMillis();
    @@ -278,7 +277,7 @@ public void runTests() throws Exception {
       }
     
       private StringBuilder clearBuffer(StringBuilder b){
    -    b.setLength(0);;
    +    b.setLength(0);
         return b;
       }
     
    @@ -334,7 +333,7 @@ SQLTester affirmDbId(int n) throws IndexOutOfBoundsException {
         return this;
       }
     
    -  sqlite3 setCurrentDb(int n) throws Exception{
    +  sqlite3 setCurrentDb(int n){
         affirmDbId(n);
         iCurrentDb = n;
         return this.aDb[n];
    @@ -342,7 +341,7 @@ sqlite3 setCurrentDb(int n) throws Exception{
     
       sqlite3 getCurrentDb(){ return aDb[iCurrentDb]; }
     
    -  sqlite3 getDbById(int id) throws Exception{
    +  sqlite3 getDbById(int id){
         return affirmDbId(id).aDb[id];
       }
     
    @@ -363,7 +362,7 @@ void closeAllDbs(){
         }
       }
     
    -  sqlite3 openDb(String name, boolean createIfNeeded) throws DbException {
    +  sqlite3 openDb(String name, boolean createIfNeeded) {
         closeDb();
         int flags = SQLITE_OPEN_READWRITE;
         if( createIfNeeded ) flags |= SQLITE_OPEN_CREATE;
    @@ -755,7 +754,7 @@ public abstract void process(
          fall in the inclusive range (min,max) then this function throws. Use
          a max value of -1 to mean unlimited.
       */
    -  protected final void argcCheck(TestScript ts, String[] argv, int min, int max) throws Exception{
    +  protected final void argcCheck(TestScript ts, String[] argv, int min, int max){
         int argc = argv.length-1;
         if(argc=0 && argc>max)){
           if( min==max ){
    @@ -771,16 +770,16 @@ protected final void argcCheck(TestScript ts, String[] argv, int min, int max) t
       /**
          Equivalent to argcCheck(argv,argc,argc).
       */
    -  protected final void argcCheck(TestScript ts, String[] argv, int argc) throws Exception{
    +  protected final void argcCheck(TestScript ts, String[] argv, int argc){
         argcCheck(ts, argv, argc, argc);
       }
     }
     
     //! --close command
     class CloseDbCommand extends Command {
    -  public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{
    +  public void process(SQLTester t, TestScript ts, String[] argv){
         argcCheck(ts,argv,0,1);
    -    Integer id;
    +    int id;
         if(argv.length>1){
           String arg = argv[1];
           if("all".equals(arg)){
    @@ -801,7 +800,7 @@ public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{
     class ColumnNamesCommand extends Command {
       public void process(
         SQLTester st, TestScript ts, String[] argv
    -  ) throws Exception{
    +  ){
         argcCheck(ts,argv,1);
         st.outputColumnNames( Integer.parseInt(argv[1])!=0 );
       }
    @@ -809,7 +808,7 @@ public void process(
     
     //! --db command
     class DbCommand extends Command {
    -  public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{
    +  public void process(SQLTester t, TestScript ts, String[] argv){
         argcCheck(ts,argv,1);
         t.setCurrentDb( Integer.parseInt(argv[1]) );
       }
    @@ -821,7 +820,7 @@ class GlobCommand extends Command {
       public GlobCommand(){}
       protected GlobCommand(boolean negate){ this.negate = negate; }
     
    -  public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{
    +  public void process(SQLTester t, TestScript ts, String[] argv){
         argcCheck(ts,argv,1,-1);
         t.incrementTestCounter();
         final String sql = t.takeInputBuffer();
    @@ -851,7 +850,7 @@ class JsonBlockCommand extends TableResultCommand {
     //! --new command
     class NewDbCommand extends OpenDbCommand {
       public NewDbCommand(){ super(true); }
    -  public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{
    +  public void process(SQLTester t, TestScript ts, String[] argv){
         if(argv.length>1){
           Util.unlink(argv[1]);
         }
    @@ -867,7 +866,7 @@ public NoopCommand(boolean verbose){
         this.verbose = verbose;
       }
       public NoopCommand(){}
    -  public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{
    +  public void process(SQLTester t, TestScript ts, String[] argv){
         if( this.verbose ){
           t.outln("Skipping unhandled command: "+argv[0]);
         }
    @@ -885,7 +884,7 @@ public NotGlobCommand(){
     class NullCommand extends Command {
       public void process(
         SQLTester st, TestScript ts, String[] argv
    -  ) throws Exception{
    +  ){
         argcCheck(ts,argv,1);
         st.setNullValue( argv[1] );
       }
    @@ -896,7 +895,7 @@ class OpenDbCommand extends Command {
       private boolean createIfNeeded = false;
       public OpenDbCommand(){}
       protected OpenDbCommand(boolean c){createIfNeeded = c;}
    -  public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{
    +  public void process(SQLTester t, TestScript ts, String[] argv){
         argcCheck(ts,argv,1);
         t.openDb(argv[1], createIfNeeded);
       }
    @@ -906,7 +905,7 @@ public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{
     class PrintCommand extends Command {
       public void process(
         SQLTester st, TestScript ts, String[] argv
    -  ) throws Exception{
    +  ){
         st.out(ts.getOutputPrefix(),": ");
         if( 1==argv.length ){
           st.out( st.getInputText() );
    @@ -921,7 +920,7 @@ class ResultCommand extends Command {
       private final ResultBufferMode bufferMode;
       protected ResultCommand(ResultBufferMode bm){ bufferMode = bm; }
       public ResultCommand(){ this(ResultBufferMode.ESCAPED); }
    -  public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{
    +  public void process(SQLTester t, TestScript ts, String[] argv){
         argcCheck(ts,argv,0,-1);
         t.incrementTestCounter();
         final String sql = t.takeInputBuffer();
    @@ -939,7 +938,7 @@ public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{
     
     //! --run command
     class RunCommand extends Command {
    -  public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{
    +  public void process(SQLTester t, TestScript ts, String[] argv){
         argcCheck(ts,argv,0,1);
         final sqlite3 db = (1==argv.length)
           ? t.getCurrentDb() : t.getDbById( Integer.parseInt(argv[1]) );
    @@ -959,7 +958,7 @@ class TableResultCommand extends Command {
       private final boolean jsonMode;
       protected TableResultCommand(boolean jsonMode){ this.jsonMode = jsonMode; }
       public TableResultCommand(){ this(false); }
    -  public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{
    +  public void process(SQLTester t, TestScript ts, String[] argv){
         argcCheck(ts,argv,0);
         t.incrementTestCounter();
         String body = ts.fetchCommandBody(t);
    @@ -1002,7 +1001,7 @@ public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{
     
     //! --testcase command
     class TestCaseCommand extends Command {
    -  public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{
    +  public void process(SQLTester t, TestScript ts, String[] argv){
         argcCheck(ts,argv,1);
         ts.setTestCaseName(argv[1]);
         t.clearResultBuffer();
    @@ -1012,7 +1011,7 @@ public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{
     
     //! --verbosity command
     class VerbosityCommand extends Command {
    -  public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{
    +  public void process(SQLTester t, TestScript ts, String[] argv){
         argcCheck(ts,argv,1);
         ts.setVerbosity( Integer.parseInt(argv[1]) );
       }
    @@ -1020,7 +1019,7 @@ public void process(SQLTester t, TestScript ts, String[] argv) throws Exception{
     
     class CommandDispatcher {
     
    -  private static java.util.Map commandMap =
    +  private static final java.util.Map commandMap =
         new java.util.HashMap<>();
     
       /**
    @@ -1244,7 +1243,7 @@ String getLine(){
         }
         cur.pos = i;
         final String rv = cur.sb.toString();
    -    if( i==cur.src.length && 0==rv.length() ){
    +    if( i==cur.src.length && rv.isEmpty() ){
           return null /* EOF */;
         }
         return rv;
    @@ -1364,7 +1363,7 @@ private void checkForDirective(
         if( m.find() ){
           throw new IncompatibleDirective(this, m.group(1)+": "+m.group(3));
         }
    -    if( line.indexOf("\n|")>=0 ){
    +    if( line.contains("\n|") ){
           throw new IncompatibleDirective(this, "newline-pipe combination.");
         }
         return;
    diff --git a/ext/jni/src/org/sqlite/jni/capi/TableColumnMetadata.java b/ext/jni/src/org/sqlite/jni/capi/TableColumnMetadata.java
    index d8b6226ac..54808cd1c 100644
    --- a/ext/jni/src/org/sqlite/jni/capi/TableColumnMetadata.java
    +++ b/ext/jni/src/org/sqlite/jni/capi/TableColumnMetadata.java
    @@ -18,11 +18,11 @@ A wrapper object for use with sqlite3_table_column_metadata().
        They are populated only via that interface.
     */
     public final class TableColumnMetadata {
    -  OutputPointer.Bool pNotNull = new OutputPointer.Bool();
    -  OutputPointer.Bool pPrimaryKey = new OutputPointer.Bool();
    -  OutputPointer.Bool pAutoinc = new OutputPointer.Bool();
    -  OutputPointer.String pzCollSeq = new OutputPointer.String();
    -  OutputPointer.String pzDataType = new OutputPointer.String();
    +  final OutputPointer.Bool pNotNull = new OutputPointer.Bool();
    +  final OutputPointer.Bool pPrimaryKey = new OutputPointer.Bool();
    +  final OutputPointer.Bool pAutoinc = new OutputPointer.Bool();
    +  final OutputPointer.String pzCollSeq = new OutputPointer.String();
    +  final OutputPointer.String pzDataType = new OutputPointer.String();
     
       public TableColumnMetadata(){
       }
    diff --git a/ext/jni/src/org/sqlite/jni/capi/Tester1.java b/ext/jni/src/org/sqlite/jni/capi/Tester1.java
    index 05b1cfeae..a9b766e9f 100644
    --- a/ext/jni/src/org/sqlite/jni/capi/Tester1.java
    +++ b/ext/jni/src/org/sqlite/jni/capi/Tester1.java
    @@ -19,7 +19,6 @@
     import java.util.List;
     import java.util.concurrent.ExecutorService;
     import java.util.concurrent.Executors;
    -import java.util.concurrent.Future;
     
     /**
        An annotation for Tester1 tests which we do not want to run in
    @@ -62,13 +61,13 @@ public class Tester1 implements Runnable {
       //! List of test*() methods to run.
       private static List testMethods = null;
       //! List of exceptions collected by run()
    -  private static List listErrors = new ArrayList<>();
    +  private static final List listErrors = new ArrayList<>();
       private static final class Metrics {
         //! Number of times createNewDb() (or equivalent) is invoked.
         volatile int dbOpen = 0;
       }
     
    -  private Integer tId;
    +  private final Integer tId;
     
       Tester1(Integer id){
         tId = id;
    @@ -78,7 +77,7 @@ private static final class Metrics {
     
       public static synchronized void outln(){
         if( !quietMode ){
    -      System.out.println("");
    +      System.out.println();
         }
       }
     
    @@ -523,7 +522,7 @@ private void testBindFetchText(){
         }
         sqlite3_finalize(stmt);
         affirm(3 == n);
    -    affirm("w😃rldhell🤩!🤩".equals(sbuf.toString()));
    +    affirm("w😃rldhell🤩!🤩".contentEquals(sbuf));
     
         try( sqlite3_stmt stmt2 = prepare(db, "SELECT ?, ?") ){
           rc = sqlite3_bind_text(stmt2, 1, "");
    @@ -668,7 +667,7 @@ private void testCollation(){
         execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')");
         final ValueHolder xDestroyCalled = new ValueHolder<>(0);
         final CollationCallback myCollation = new CollationCallback() {
    -        private String myState =
    +        private final String myState =
               "this is local state. There is much like it, but this is mine.";
             @Override
             // Reverse-sorts its inputs...
    @@ -847,7 +846,7 @@ private void testUdfThrows(){
         SQLFunction funcAgg = new AggregateFunction(){
             @Override public void xStep(sqlite3_context cx, sqlite3_value[] args){
               /** Throwing from here should emit loud noise on stdout or stderr
    -              but the exception is supressed because we have no way to inform
    +              but the exception is suppressed because we have no way to inform
                   sqlite about it from these callbacks. */
               //throw new RuntimeException("Throwing from an xStep");
             }
    @@ -1850,7 +1849,7 @@ private void testPrepareMulti(){
           "; insert into t(a) values(1),(2),(3);",
           "select a from t;"
         };
    -    final List liStmt = new ArrayList();
    +    final List liStmt = new ArrayList<>();
         final PrepareMultiCallback proxy = new PrepareMultiCallback.StepAll();
         final ValueHolder toss = new ValueHolder<>(null);
         PrepareMultiCallback m = new PrepareMultiCallback() {
    @@ -1984,9 +1983,9 @@ to emit the list run by each thread (which may differ from the initial
          -v: emit some developer-mode info at the end.
       */
       public static void main(String[] args) throws Exception {
    -    Integer nThread = 1;
    +    int nThread = 1;
         boolean doSomethingForDev = false;
    -    Integer nRepeat = 1;
    +    int nRepeat = 1;
         boolean forceFail = false;
         boolean sqlLog = false;
         boolean configLog = false;
    @@ -2048,7 +2047,7 @@ public static void main(String[] args) throws Exception {
           final ConfigLogCallback log = new ConfigLogCallback() {
               @Override public void call(int code, String msg){
                 outln("ConfigLogCallback: ",ResultCode.getEntryForInt(code),": ", msg);
    -          };
    +          }
             };
           int rc = sqlite3_config( log );
           affirm( 0==rc );
    diff --git a/ext/jni/src/org/sqlite/jni/capi/XDestroyCallback.java b/ext/jni/src/org/sqlite/jni/capi/XDestroyCallback.java
    index 372e4ec8d..ce6c6a6ab 100644
    --- a/ext/jni/src/org/sqlite/jni/capi/XDestroyCallback.java
    +++ b/ext/jni/src/org/sqlite/jni/capi/XDestroyCallback.java
    @@ -33,5 +33,5 @@ multiple arities or names using sqlite3_create_collation().  If
          each individual reference, leading to memory corruption or a
          crash via duplicate free().
       */
    -  public void xDestroy();
    +  void xDestroy();
     }
    diff --git a/ext/jni/src/org/sqlite/jni/fts5/Fts5ExtensionApi.java b/ext/jni/src/org/sqlite/jni/fts5/Fts5ExtensionApi.java
    index 594f3eaad..f409f4961 100644
    --- a/ext/jni/src/org/sqlite/jni/fts5/Fts5ExtensionApi.java
    +++ b/ext/jni/src/org/sqlite/jni/fts5/Fts5ExtensionApi.java
    @@ -12,7 +12,6 @@
     ** This file is part of the JNI bindings for the sqlite3 C API.
     */
     package org.sqlite.jni.fts5;
    -import java.nio.charset.StandardCharsets;
     import org.sqlite.jni.capi.*;
     import org.sqlite.jni.annotation.*;
     
    @@ -24,7 +23,7 @@ private Fts5ExtensionApi(){}
       private final int iVersion = 2;
     
       /* Callback type for used by xQueryPhrase(). */
    -  public static interface XQueryPhraseCallback {
    +  public interface XQueryPhraseCallback {
         int call(Fts5ExtensionApi fapi, Fts5Context cx);
       }
     
    diff --git a/ext/jni/src/org/sqlite/jni/fts5/TesterFts5.java b/ext/jni/src/org/sqlite/jni/fts5/TesterFts5.java
    index 095e649ca..4d97ced47 100644
    --- a/ext/jni/src/org/sqlite/jni/fts5/TesterFts5.java
    +++ b/ext/jni/src/org/sqlite/jni/fts5/TesterFts5.java
    @@ -12,12 +12,11 @@
     ** This file contains a set of tests for the sqlite3 JNI bindings.
     */
     package org.sqlite.jni.fts5;
    +import java.util.*;
     import static org.sqlite.jni.capi.CApi.*;
     import static org.sqlite.jni.capi.Tester1.*;
     import org.sqlite.jni.capi.*;
    -import org.sqlite.jni.fts5.*;
    -
    -import java.util.*;
    +import java.nio.charset.StandardCharsets;
     
     public class TesterFts5 {
     
    @@ -88,7 +87,7 @@ public void xDestroy(){
       ** thrown.
       */
       private static String[] sqlite3_exec(sqlite3 db, String sql) {
    -    List aOut = new ArrayList();
    +    List aOut = new ArrayList<>();
     
         /* Iterate through the list of SQL statements. For each, step through
         ** it and add any results to the aOut[] array.  */
    @@ -137,14 +136,14 @@ private static void do_execsql_test(sqlite3 db, String sql){
       **     fts5_columncount()
       */
       private static void create_test_functions(sqlite3 db){
    -    /* 
    +    /*
         ** A user-defined-function fts5_rowid() that uses xRowid()
         */
         fts5_extension_function fts5_rowid = new fts5_extension_function(){
           @Override public void call(
    -          Fts5ExtensionApi ext, 
    +          Fts5ExtensionApi ext,
               Fts5Context fCx,
    -          sqlite3_context pCx, 
    +          sqlite3_context pCx,
               sqlite3_value argv[]
           ){
             long rowid = ext.xRowid(fCx);
    @@ -153,14 +152,14 @@ private static void create_test_functions(sqlite3 db){
           public void xDestroy(){ }
         };
     
    -    /* 
    -    ** fts5_columncount() - xColumnCount() 
    +    /*
    +    ** fts5_columncount() - xColumnCount()
         */
         fts5_extension_function fts5_columncount = new fts5_extension_function(){
           @Override public void call(
    -          Fts5ExtensionApi ext, 
    +          Fts5ExtensionApi ext,
               Fts5Context fCx,
    -          sqlite3_context pCx, 
    +          sqlite3_context pCx,
               sqlite3_value argv[]
           ){
             int nCol = ext.xColumnCount(fCx);
    @@ -169,14 +168,14 @@ public void xDestroy(){ }
           public void xDestroy(){ }
         };
     
    -    /* 
    -    ** fts5_columnsize() - xColumnSize() 
    +    /*
    +    ** fts5_columnsize() - xColumnSize()
         */
         fts5_extension_function fts5_columnsize = new fts5_extension_function(){
           @Override public void call(
    -          Fts5ExtensionApi ext, 
    +          Fts5ExtensionApi ext,
               Fts5Context fCx,
    -          sqlite3_context pCx, 
    +          sqlite3_context pCx,
               sqlite3_value argv[]
           ){
             if( argv.length!=1 ){
    @@ -194,14 +193,14 @@ public void xDestroy(){ }
           public void xDestroy(){ }
         };
     
    -    /* 
    -    ** fts5_columntext() - xColumnText() 
    +    /*
    +    ** fts5_columntext() - xColumnText()
         */
         fts5_extension_function fts5_columntext = new fts5_extension_function(){
           @Override public void call(
    -          Fts5ExtensionApi ext, 
    +          Fts5ExtensionApi ext,
               Fts5Context fCx,
    -          sqlite3_context pCx, 
    +          sqlite3_context pCx,
               sqlite3_value argv[]
           ){
             if( argv.length!=1 ){
    @@ -219,14 +218,14 @@ public void xDestroy(){ }
           public void xDestroy(){ }
         };
     
    -    /* 
    -    ** fts5_columntotalsize() - xColumnTotalSize() 
    +    /*
    +    ** fts5_columntotalsize() - xColumnTotalSize()
         */
         fts5_extension_function fts5_columntsize = new fts5_extension_function(){
           @Override public void call(
    -          Fts5ExtensionApi ext, 
    +          Fts5ExtensionApi ext,
               Fts5Context fCx,
    -          sqlite3_context pCx, 
    +          sqlite3_context pCx,
               sqlite3_value argv[]
           ){
             if( argv.length!=1 ){
    @@ -251,9 +250,9 @@ public void xDestroy(){ }
         */
         class fts5_aux implements fts5_extension_function {
           @Override public void call(
    -          Fts5ExtensionApi ext, 
    +          Fts5ExtensionApi ext,
               Fts5Context fCx,
    -          sqlite3_context pCx, 
    +          sqlite3_context pCx,
               sqlite3_value argv[]
           ){
             if( argv.length>1 ){
    @@ -268,13 +267,13 @@ class fts5_aux implements fts5_extension_function {
     
             if( argv.length==1 ){
               String val = sqlite3_value_text16(argv[0]);
    -          if( !val.equals("") ){
    +          if( !val.isEmpty() ){
                 ext.xSetAuxdata(fCx, val);
               }
             }
           }
           public void xDestroy(){ }
    -    };
    +    }
     
         /*
         ** fts5_inst();
    @@ -286,9 +285,9 @@ public void xDestroy(){ }
         */
         fts5_extension_function fts5_inst = new fts5_extension_function(){
           @Override public void call(
    -          Fts5ExtensionApi ext, 
    +          Fts5ExtensionApi ext,
               Fts5Context fCx,
    -          sqlite3_context pCx, 
    +          sqlite3_context pCx,
               sqlite3_value argv[]
           ){
             if( argv.length!=0 ){
    @@ -299,7 +298,7 @@ public void xDestroy(){ }
             OutputPointer.Int32 piPhrase = new OutputPointer.Int32();
             OutputPointer.Int32 piCol = new OutputPointer.Int32();
             OutputPointer.Int32 piOff = new OutputPointer.Int32();
    -        String ret = new String();
    +        String ret = "";
     
             int rc = ext.xInstCount(fCx, pnInst);
             int nInst = pnInst.get();
    @@ -327,9 +326,9 @@ public void xDestroy(){ }
         */
         fts5_extension_function fts5_pinst = new fts5_extension_function(){
           @Override public void call(
    -          Fts5ExtensionApi ext, 
    +          Fts5ExtensionApi ext,
               Fts5Context fCx,
    -          sqlite3_context pCx, 
    +          sqlite3_context pCx,
               sqlite3_value argv[]
           ){
             if( argv.length!=0 ){
    @@ -338,7 +337,7 @@ public void xDestroy(){ }
     
             OutputPointer.Int32 piCol = new OutputPointer.Int32();
             OutputPointer.Int32 piOff = new OutputPointer.Int32();
    -        String ret = new String();
    +        String ret = "";
             int rc = SQLITE_OK;
     
             int nPhrase = ext.xPhraseCount(fCx);
    @@ -350,7 +349,7 @@ public void xDestroy(){ }
                   rc==SQLITE_OK && piCol.get()>=0;
                   ext.xPhraseNext(fCx, pIter, piCol, piOff)
               ){
    -            if( !ret.equals("") ) ret += " ";
    +            if( !ret.isEmpty() ) ret += " ";
                 ret += "{"+ii+" "+piCol.get()+" "+piOff.get()+"}";
               }
             }
    @@ -374,9 +373,9 @@ public void xDestroy(){ }
         */
         fts5_extension_function fts5_pcolinst = new fts5_extension_function(){
           @Override public void call(
    -          Fts5ExtensionApi ext, 
    +          Fts5ExtensionApi ext,
               Fts5Context fCx,
    -          sqlite3_context pCx, 
    +          sqlite3_context pCx,
               sqlite3_value argv[]
           ){
             if( argv.length!=0 ){
    @@ -384,7 +383,7 @@ public void xDestroy(){ }
             }
     
             OutputPointer.Int32 piCol = new OutputPointer.Int32();
    -        String ret = new String();
    +        String ret = "";
             int rc = SQLITE_OK;
     
             int nPhrase = ext.xPhraseCount(fCx);
    @@ -396,7 +395,7 @@ public void xDestroy(){ }
                   rc==SQLITE_OK && piCol.get()>=0;
                   ext.xPhraseNextColumn(fCx, pIter, piCol)
               ){
    -            if( !ret.equals("") ) ret += " ";
    +            if( !ret.isEmpty() ) ret += " ";
                 ret += "{"+ii+" "+piCol.get()+"}";
               }
             }
    @@ -415,9 +414,9 @@ public void xDestroy(){ }
         */
         fts5_extension_function fts5_rowcount = new fts5_extension_function(){
           @Override public void call(
    -          Fts5ExtensionApi ext, 
    +          Fts5ExtensionApi ext,
               Fts5Context fCx,
    -          sqlite3_context pCx, 
    +          sqlite3_context pCx,
               sqlite3_value argv[]
           ){
             if( argv.length!=0 ){
    @@ -440,9 +439,9 @@ public void xDestroy(){ }
         */
         fts5_extension_function fts5_phrasesize = new fts5_extension_function(){
           @Override public void call(
    -          Fts5ExtensionApi ext, 
    +          Fts5ExtensionApi ext,
               Fts5Context fCx,
    -          sqlite3_context pCx, 
    +          sqlite3_context pCx,
               sqlite3_value argv[]
           ){
             if( argv.length!=1 ){
    @@ -464,9 +463,9 @@ public void xDestroy(){ }
         */
         fts5_extension_function fts5_phrasehits = new fts5_extension_function(){
           @Override public void call(
    -          Fts5ExtensionApi ext, 
    +          Fts5ExtensionApi ext,
               Fts5Context fCx,
    -          sqlite3_context pCx, 
    +          sqlite3_context pCx,
               sqlite3_value argv[]
           ){
             if( argv.length!=1 ){
    @@ -503,9 +502,9 @@ public void xDestroy(){ }
         */
         fts5_extension_function fts5_tokenize = new fts5_extension_function(){
           @Override public void call(
    -          Fts5ExtensionApi ext, 
    +          Fts5ExtensionApi ext,
               Fts5Context fCx,
    -          sqlite3_context pCx, 
    +          sqlite3_context pCx,
               sqlite3_value argv[]
           ){
             if( argv.length!=1 ){
    @@ -515,7 +514,7 @@ public void xDestroy(){ }
             int rc = SQLITE_OK;
     
             class MyCallback implements XTokenizeCallback {
    -          private List myList = new ArrayList();
    +          private List myList = new ArrayList<>();
     
               public String getval() {
                 return String.join("+", myList);
    @@ -524,7 +523,7 @@ public String getval() {
               @Override
               public int call(int tFlags, byte[] txt, int iStart, int iEnd){
                 try {
    -              String str = new String(txt, "UTF-8");
    +              String str = new String(txt, StandardCharsets.UTF_8);
                   myList.add(str);
                 } catch (Exception e) {
                 }
    diff --git a/ext/jni/src/org/sqlite/jni/fts5/fts5_extension_function.java b/ext/jni/src/org/sqlite/jni/fts5/fts5_extension_function.java
    index 5e47633ba..6e98f64ff 100644
    --- a/ext/jni/src/org/sqlite/jni/fts5/fts5_extension_function.java
    +++ b/ext/jni/src/org/sqlite/jni/fts5/fts5_extension_function.java
    @@ -39,7 +39,11 @@ void call(Fts5ExtensionApi ext, Fts5Context fCx,
       */
       void xDestroy();
     
    -  public static abstract class Abstract implements fts5_extension_function {
    +  /**
    +     A base implementation of fts5_extension_function() which has a
    +     no-op xDestroy() method.
    +  */
    +  abstract class Abstract implements fts5_extension_function {
         @Override public abstract void call(Fts5ExtensionApi ext, Fts5Context fCx,
                                             sqlite3_context pCx, sqlite3_value argv[]);
         @Override public void xDestroy(){}
    diff --git a/ext/jni/src/org/sqlite/jni/test-script-interpreter.md b/ext/jni/src/org/sqlite/jni/test-script-interpreter.md
    index 939f77e1b..cc7b7e7f9 100644
    --- a/ext/jni/src/org/sqlite/jni/test-script-interpreter.md
    +++ b/ext/jni/src/org/sqlite/jni/test-script-interpreter.md
    @@ -4,7 +4,7 @@
     
     The purpose of the Test Script Interpreter is to read and interpret
     script files that contain SQL commands and desired results.  The
    -interpreter will check results and report any discrepencies found.
    +interpreter will check results and report any discrepancies found.
     
     The test script files are ASCII text files.  The filename always ends with
     ".test".  Each script is evaluated independently; context does not carry
    @@ -101,7 +101,7 @@ interpreter is in a "verbose" mode, the interpreter might choose to emit
     an informational message along the lines of "test script NAME abandoned
     due to unsupported command: --whatever".
     
    -The initial implemention will only recognize a few commands.  Other
    +The initial implementation will only recognize a few commands.  Other
     commands may be added later.  The following is the initial set of
     commands:
     
    @@ -160,7 +160,7 @@ the result buffer.  This distinction does not matter for the --result
     command itself, but it is important for related commands like --glob
     and --notglob.  Sometimes test cases will contains a bunch of SQL
     followed by multiple --glob and/or --notglob statements.  All of the
    -globs should be evaluated agains the result buffer, but the SQL should
    +globs should be evaluated against the result buffer, but the SQL should
     only be run once.  This is accomplished by resetting the input buffer
     but not the result buffer.
     
    diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/ScalarFunction.java b/ext/jni/src/org/sqlite/jni/wrapper1/ScalarFunction.java
    index 067a6983e..c616ae739 100644
    --- a/ext/jni/src/org/sqlite/jni/wrapper1/ScalarFunction.java
    +++ b/ext/jni/src/org/sqlite/jni/wrapper1/ScalarFunction.java
    @@ -12,10 +12,6 @@
     ** This file is part of the wrapper1 interface for sqlite3.
     */
     package org.sqlite.jni.wrapper1;
    -import org.sqlite.jni.capi.CApi;
    -import org.sqlite.jni.annotation.*;
    -import org.sqlite.jni.capi.sqlite3_context;
    -import org.sqlite.jni.capi.sqlite3_value;
     
     /**
        The SqlFunction type for scalar SQL functions.
    diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/SqlFunction.java b/ext/jni/src/org/sqlite/jni/wrapper1/SqlFunction.java
    index dcfc2ebeb..bb0fd0ccd 100644
    --- a/ext/jni/src/org/sqlite/jni/wrapper1/SqlFunction.java
    +++ b/ext/jni/src/org/sqlite/jni/wrapper1/SqlFunction.java
    @@ -22,13 +22,13 @@
     */
     public interface SqlFunction  {
     
    -  public static final int DETERMINISTIC = CApi.SQLITE_DETERMINISTIC;
    -  public static final int INNOCUOUS = CApi.SQLITE_INNOCUOUS;
    -  public static final int DIRECTONLY = CApi.SQLITE_DIRECTONLY;
    -  public static final int SUBTYPE = CApi.SQLITE_SUBTYPE;
    -  public static final int RESULT_SUBTYPE = CApi.SQLITE_RESULT_SUBTYPE;
    -  public static final int UTF8 = CApi.SQLITE_UTF8;
    -  public static final int UTF16 = CApi.SQLITE_UTF16;
    +  int DETERMINISTIC = CApi.SQLITE_DETERMINISTIC;
    +  int INNOCUOUS = CApi.SQLITE_INNOCUOUS;
    +  int DIRECTONLY = CApi.SQLITE_DIRECTONLY;
    +  int SUBTYPE = CApi.SQLITE_SUBTYPE;
    +  int RESULT_SUBTYPE = CApi.SQLITE_RESULT_SUBTYPE;
    +  int UTF8 = CApi.SQLITE_UTF8;
    +  int UTF16 = CApi.SQLITE_UTF16;
     
       /**
          The Arguments type is an abstraction on top of the lower-level
    @@ -36,7 +36,7 @@ public interface SqlFunction  {
          of the lower-level interface, insofar as possible without "leaking"
          those types into this API.
       */
    -  public final static class Arguments implements Iterable{
    +  final class Arguments implements Iterable{
         private final sqlite3_context cx;
         private final sqlite3_value args[];
         public final int length;
    @@ -144,7 +144,7 @@ Analog to sqlite3_set_auxdata() but throws if argNdx is out of
            range.
         */
         public void setAuxData(int argNdx, Object o){
    -      /* From the API docs: https://www.sqlite.org/c3ref/get_auxdata.html
    +      /* From the API docs: https://sqlite.org/c3ref/get_auxdata.html
     
              The value of the N parameter to these interfaces should be
              non-negative. Future enhancements may make use of negative N
    @@ -207,7 +207,7 @@ public java.util.Iterator iterator(){
          Internal-use adapter for wrapping this package's ScalarFunction
          for use with the org.sqlite.jni.capi.ScalarFunction interface.
       */
    -  static final class ScalarAdapter extends org.sqlite.jni.capi.ScalarFunction {
    +  final class ScalarAdapter extends org.sqlite.jni.capi.ScalarFunction {
         private final ScalarFunction impl;
         ScalarAdapter(ScalarFunction impl){
           this.impl = impl;
    @@ -234,8 +234,8 @@ public void xDestroy(){
          Internal-use adapter for wrapping this package's AggregateFunction
          for use with the org.sqlite.jni.capi.AggregateFunction interface.
       */
    -  static /*cannot be final without duplicating the whole body in WindowAdapter*/
       class AggregateAdapter extends org.sqlite.jni.capi.AggregateFunction {
    +  /*cannot be final without duplicating the whole body in WindowAdapter*/
         private final AggregateFunction impl;
         AggregateAdapter(AggregateFunction impl){
           this.impl = impl;
    @@ -277,7 +277,7 @@ public void xDestroy(){
          Internal-use adapter for wrapping this package's WindowFunction
          for use with the org.sqlite.jni.capi.WindowFunction interface.
       */
    -  static final class WindowAdapter extends AggregateAdapter {
    +  final class WindowAdapter extends AggregateAdapter {
         private final WindowFunction impl;
         WindowAdapter(WindowFunction impl){
           super(impl);
    diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java b/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java
    index de131e854..d259e0ce6 100644
    --- a/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java
    +++ b/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java
    @@ -19,7 +19,6 @@
     import org.sqlite.jni.capi.sqlite3_backup;
     import org.sqlite.jni.capi.sqlite3_blob;
     import org.sqlite.jni.capi.OutputPointer;
    -import java.nio.ByteBuffer;
     
     /**
        This class represents a database connection, analog to the C-side
    @@ -388,10 +387,10 @@ public static boolean compileOptionUsed(String optName){
         return CApi.sqlite3_compileoption_used(optName);
       }
     
    -  private static boolean hasNormalizeSql =
    +  private static final boolean hasNormalizeSql =
         compileOptionUsed("ENABLE_NORMALIZE");
     
    -  private static boolean hasSqlLog =
    +  private static final boolean hasSqlLog =
         compileOptionUsed("ENABLE_SQLLOG");
     
       /**
    @@ -450,7 +449,7 @@ public static final class Status {
         long current;
         /** The peak value for the requested status() or libStatus() metric. */
         long peak;
    -  };
    +  }
     
       /**
          As per sqlite3_status64(), but returns its current and high-water
    @@ -661,7 +660,7 @@ public void prepareMulti(String sql, PrepareMulti visitor){
       }
     
       /**
    -     Equivallent to prepareMulti(X,prepFlags,visitor), where X is
    +     Equivalent to prepareMulti(X,prepFlags,visitor), where X is
          sql.getBytes(StandardCharsets.UTF_8).
       */
       public void prepareMulti(String sql, int prepFlags, PrepareMulti visitor){
    @@ -696,7 +695,7 @@ public void prepareMulti(byte sqlUtf8[], int prepFlags, PrepareMulti visitor){
         final org.sqlite.jni.capi.OutputPointer.Int32 oTail =
           new org.sqlite.jni.capi.OutputPointer.Int32();
         while( pos < sqlChunk.length ){
    -      sqlite3_stmt stmt = null;
    +      sqlite3_stmt stmt;
           if( pos>0 ){
             sqlChunk = java.util.Arrays.copyOfRange(sqlChunk, pos, sqlChunk.length);
           }
    @@ -988,15 +987,15 @@ public void trace(int traceMask, TraceCallback callback){
               }
             };
         checkRc( CApi.sqlite3_trace_v2(thisDb(), traceMask, tc) );
    -  };
    +  }
     
       /**
          Corresponds to the sqlite3_stmt class. Use Sqlite.prepare() to
          create new instances.
       */
       public static final class Stmt implements AutoCloseable {
    -    private Sqlite _db = null;
    -    private sqlite3_stmt stmt = null;
    +    private Sqlite _db;
    +    private sqlite3_stmt stmt;
     
         /** Only called by the prepare() factory functions. */
         Stmt(Sqlite db, sqlite3_stmt stmt){
    @@ -1379,9 +1378,9 @@ public static void clearAutoExtensions(){
          Sqlite.initBackup() to create new instances.
       */
       public static final class Backup implements AutoCloseable {
    -    private sqlite3_backup b = null;
    -    private Sqlite dbTo = null;
    -    private Sqlite dbFrom = null;
    +    private sqlite3_backup b;
    +    private Sqlite dbTo;
    +    private Sqlite dbFrom;
     
         Backup(Sqlite dbDest, String schemaDest,Sqlite dbSrc, String schemaSrc){
           this.dbTo = dbDest;
    @@ -1491,7 +1490,7 @@ must compare its two arguments using memcmp(3) semantics.
            Warning: the SQLite core has no mechanism for reporting errors
            from custom collations and its workflow does not accommodate
            propagation of exceptions from callbacks. Any exceptions thrown
    -       from collations will be silently supressed and sorting results
    +       from collations will be silently suppressed and sorting results
            will be unpredictable.
         */
         int call(byte[] lhs, byte[] rhs);
    @@ -1506,7 +1505,7 @@ Analog to sqlite3_create_collation().
       */
       public void createCollation(String name, int encoding, Collation c){
         thisDb();
    -    if( null==name || 0==name.length()){
    +    if( null==name || name.isEmpty()){
           throw new IllegalArgumentException("Collation name may not be null or empty.");
         }
         if( null==c ){
    @@ -1599,11 +1598,12 @@ Analog to sqlite3_busy_handler(). If b is null then any
       public void setBusyHandler( BusyHandler b ){
         org.sqlite.jni.capi.BusyHandlerCallback bhc = null;
         if( null!=b ){
    -      bhc = new org.sqlite.jni.capi.BusyHandlerCallback(){
    +      /*bhc = new org.sqlite.jni.capi.BusyHandlerCallback(){
               @Override public int call(int n){
                 return b.call(n);
               }
    -        };
    +        };*/
    +      bhc = b::call;
         }
         checkRc( CApi.sqlite3_busy_handler(thisDb(), bhc) );
       }
    @@ -1781,9 +1781,10 @@ Note that this API, in contrast to setUpdateHook(),
       public void setProgressHandler( int n, ProgressHandler p ){
         org.sqlite.jni.capi.ProgressHandlerCallback phc = null;
         if( null!=p ){
    -      phc = new org.sqlite.jni.capi.ProgressHandlerCallback(){
    +      /*phc = new org.sqlite.jni.capi.ProgressHandlerCallback(){
               @Override public int call(){ return p.call(); }
    -        };
    +          };*/
    +      phc = p::call;
         }
         CApi.sqlite3_progress_handler( thisDb(), n, phc );
       }
    @@ -1808,11 +1809,12 @@ Analog to sqlite3_set_authorizer(), this sets the current
       public void setAuthorizer( Authorizer a ) {
         org.sqlite.jni.capi.AuthorizerCallback ac = null;
         if( null!=a ){
    -      ac = new org.sqlite.jni.capi.AuthorizerCallback(){
    +      /*ac = new org.sqlite.jni.capi.AuthorizerCallback(){
               @Override public int call(int opId, String s1, String s2, String s3, String s4){
                 return a.call(opId, s1, s2, s3, s4);
               }
    -        };
    +          };*/
    +      ac = a::call;
         }
         checkRc( CApi.sqlite3_set_authorizer( thisDb(), ac ) );
       }
    @@ -1932,11 +1934,12 @@ public static void libConfigLog(ConfigLog log){
         final org.sqlite.jni.capi.ConfigLogCallback l =
           null==log
           ? null
    -      : new org.sqlite.jni.capi.ConfigLogCallback() {
    +      /*: new org.sqlite.jni.capi.ConfigLogCallback() {
               @Override public void call(int errCode, String msg){
                 log.call(errCode, msg);
               }
    -        };
    +          };*/
    +      : log::call;
           checkRcStatic(CApi.sqlite3_config(l));
       }
     
    diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/Tester2.java b/ext/jni/src/org/sqlite/jni/wrapper1/Tester2.java
    index 5ac41323c..528e1f61c 100644
    --- a/ext/jni/src/org/sqlite/jni/wrapper1/Tester2.java
    +++ b/ext/jni/src/org/sqlite/jni/wrapper1/Tester2.java
    @@ -13,7 +13,6 @@
     */
     package org.sqlite.jni.wrapper1;
     import java.nio.charset.StandardCharsets;
    -import java.util.Arrays;
     import java.util.ArrayList;
     import java.util.List;
     import java.util.concurrent.ExecutorService;
    @@ -53,14 +52,14 @@ public class Tester2 implements Runnable {
       //! List of test*() methods to run.
       private static List testMethods = null;
       //! List of exceptions collected by run()
    -  private static List listErrors = new ArrayList<>();
    +  private static final List listErrors = new ArrayList<>();
       private static final class Metrics {
         //! Number of times createNewDb() (or equivalent) is invoked.
         volatile int dbOpen = 0;
       }
     
       //! Instance ID.
    -  private Integer tId;
    +  private final Integer tId;
     
       Tester2(Integer id){
         tId = id;
    @@ -70,7 +69,7 @@ private static final class Metrics {
     
       public static synchronized void outln(){
         if( !quietMode ){
    -      System.out.println("");
    +      System.out.println();
         }
       }
     
    @@ -547,7 +546,7 @@ private synchronized void testAutoExtension(){
           err = e;
         }
         affirm( err!=null );
    -    affirm( err.getMessage().indexOf(toss.value)>=0 );
    +    affirm( err.getMessage().contains(toss.value) );
         toss.value = null;
     
         val.value = 0;
    @@ -616,7 +615,7 @@ private void testCollation(){
         final Sqlite db = openDb();
         execSql(db, "CREATE TABLE t(a); INSERT INTO t(a) VALUES('a'),('b'),('c')");
         final Sqlite.Collation myCollation = new Sqlite.Collation() {
    -        private String myState =
    +        private final String myState =
               "this is local state. There is much like it, but this is mine.";
             @Override
             // Reverse-sorts its inputs...
    @@ -1038,9 +1037,9 @@ to emit the list run by each thread (which may differ from the initial
          -v: emit some developer-mode info at the end.
       */
       public static void main(String[] args) throws Exception {
    -    Integer nThread = 1;
    +    int nThread = 1;
    +    int nRepeat = 1;
         boolean doSomethingForDev = false;
    -    Integer nRepeat = 1;
         boolean forceFail = false;
         boolean sqlLog = false;
         boolean configLog = false;
    @@ -1097,7 +1096,7 @@ public static void main(String[] args) throws Exception {
           Sqlite.libConfigLog( new Sqlite.ConfigLog() {
               @Override public void call(int code, String msg){
                 outln("ConfigLog: ",Sqlite.errstr(code),": ", msg);
    -          };
    +          }
             }
           );
         }
    diff --git a/ext/lsm1/lsmInt.h b/ext/lsm1/lsmInt.h
    index 5060366d0..4e3c5e59c 100644
    --- a/ext/lsm1/lsmInt.h
    +++ b/ext/lsm1/lsmInt.h
    @@ -570,7 +570,7 @@ struct FreelistEntry {
     /*
     ** A snapshot of a database. A snapshot contains all the information required
     ** to read or write a database file on disk. See the description of struct
    -** Database below for futher details.
    +** Database below for further details.
     */
     struct Snapshot {
       Database *pDatabase;            /* Database this snapshot belongs to */
    diff --git a/ext/lsm1/lsm_ckpt.c b/ext/lsm1/lsm_ckpt.c
    index 1c4f788ad..dbfa1a61f 100644
    --- a/ext/lsm1/lsm_ckpt.c
    +++ b/ext/lsm1/lsm_ckpt.c
    @@ -169,7 +169,7 @@
     ** The argument to this macro must be of type u32. On a little-endian
     ** architecture, it returns the u32 value that results from interpreting
     ** the 4 bytes as a big-endian value. On a big-endian architecture, it
    -** returns the value that would be produced by intepreting the 4 bytes
    +** returns the value that would be produced by interpreting the 4 bytes
     ** of the input value as a little-endian integer.
     */
     #define BYTESWAP32(x) ( \
    diff --git a/ext/lsm1/lsm_file.c b/ext/lsm1/lsm_file.c
    index fd78835bb..9f4144618 100644
    --- a/ext/lsm1/lsm_file.c
    +++ b/ext/lsm1/lsm_file.c
    @@ -793,7 +793,7 @@ void lsmFsClose(FileSystem *pFS){
     **
     ** This function returns a pointer to an object that can be linked into
     ** the list described above. The returned object now 'owns' the database
    -** file descriptr, so that when the FileSystem object is destroyed, it
    +** file descriptor, so that when the FileSystem object is destroyed, it
     ** will not be closed. 
     **
     ** This function may be called at most once in the life-time of a 
    @@ -2293,7 +2293,7 @@ int lsmFsMetaPageGet(
             );
           }
     #ifndef NDEBUG
    -      /* pPg->aData causes an uninitialized access via a downstreadm write().
    +      /* pPg->aData causes an uninitialized access via a downstream write().
              After discussion on this list, this memory should not, for performance
              reasons, be memset. However, tracking down "real" misuse is more
              difficult with this "false" positive, so it is set when NDEBUG.
    diff --git a/ext/lsm1/lsm_log.c b/ext/lsm1/lsm_log.c
    index a66e40bcc..3dcef42f7 100644
    --- a/ext/lsm1/lsm_log.c
    +++ b/ext/lsm1/lsm_log.c
    @@ -758,7 +758,7 @@ void lsmLogTell(
     }
     
     /*
    -** Seek (rewind) back to the log file offset stored by an ealier call to
    +** Seek (rewind) back to the log file offset stored by an earlier call to
     ** lsmLogTell() in *pMark.
     */
     void lsmLogSeek(
    diff --git a/ext/lsm1/lsm_sorted.c b/ext/lsm1/lsm_sorted.c
    index 6e5243e85..a72c8cafb 100644
    --- a/ext/lsm1/lsm_sorted.c
    +++ b/ext/lsm1/lsm_sorted.c
    @@ -2652,7 +2652,7 @@ int lsmSortedLoadFreelist(
       void **ppVal,                   /* OUT: Blob containing LSM free-list */
       int *pnVal                      /* OUT: Size of *ppVal blob in bytes */
     ){
    -  MultiCursor *pCsr;              /* Cursor used to retreive free-list */
    +  MultiCursor *pCsr;              /* Cursor used to retrieve free-list */
       int rc = LSM_OK;                /* Return Code */
     
       assert( pDb->pWorker );
    @@ -2764,7 +2764,7 @@ static int mcursorLocationOk(MultiCursor *pCsr, int bDeleteOk){
       }
     
       /* If the cursor points to a system key (free-list entry), and the
    -  ** CURSOR_IGNORE_SYSTEM flag is set, skip thie entry.  */
    +  ** CURSOR_IGNORE_SYSTEM flag is set, skip this entry.  */
       if( (pCsr->flags & CURSOR_IGNORE_SYSTEM) && rtTopic(eType)!=0 ){
         return 0;
       }
    @@ -3586,7 +3586,7 @@ static int mergeWorkerBtreeWrite(
       Hierarchy *p = &pMW->hier;
       lsm_db *pDb = pMW->pDb;         /* Database handle */
       int rc = LSM_OK;                /* Return Code */
    -  int iLevel;                     /* Level of b-tree hierachy to write to */
    +  int iLevel;                     /* Level of b-tree hierarchy to write to */
       int nData;                      /* Size of aData[] in bytes */
       u8 *aData;                      /* Page data for level iLevel */
       int iOff;                       /* Offset on b-tree page to write record to */
    diff --git a/ext/misc/README.md b/ext/misc/README.md
    index 69cb23025..cfc9e867c 100644
    --- a/ext/misc/README.md
    +++ b/ext/misc/README.md
    @@ -1,7 +1,7 @@
     ## Miscellaneous Extensions
     
     This folder contains a collection of smaller loadable extensions.
    -See  for instructions on how
    +See  for instructions on how
     to compile and use loadable extensions.
     Each extension in this folder is implemented in a single file of C code.
     
    @@ -10,9 +10,9 @@ header comments for details about each extension.  Additional notes are
     as follows:
     
       *  **carray.c** —  This module implements the
    -     [carray](https://www.sqlite.org/carray.html) table-valued function.
    +     [carray](https://sqlite.org/carray.html) table-valued function.
          It is a good example of how to go about implementing a custom
    -     [table-valued function](https://www.sqlite.org/vtab.html#tabfunc2).
    +     [table-valued function](https://sqlite.org/vtab.html#tabfunc2).
     
       *  **csv.c** —  A [virtual table](https://sqlite.org/vtab.html)
          for reading 
    @@ -21,7 +21,7 @@ as follows:
       *  **dbdump.c** —  This is not actually a loadable extension, but
          rather a library that implements an approximate equivalent to the
          ".dump" command of the
    -     [command-line shell](https://www.sqlite.org/cli.html).
    +     [command-line shell](https://sqlite.org/cli.html).
     
       *  **json1.c** —  Various SQL functions and table-valued functions
          for processing JSON.  This extension is already built into the
    @@ -29,7 +29,7 @@ as follows:
           for additional information.
     
       *  **memvfs.c** —  This file implements a custom
    -     [VFS](https://www.sqlite.org/vfs.html) that stores an entire database
    +     [VFS](https://sqlite.org/vfs.html) that stores an entire database
          file in a single block of RAM.  It serves as a good example of how
          to implement a simple custom VFS.
     
    @@ -38,7 +38,7 @@ as follows:
          new custom SQL functions for SQLite.
     
       *  **series.c** —  This is an implementation of the
    -     "generate_series" [virtual table](https://www.sqlite.org/vtab.html).
    +     "generate_series" [virtual table](https://sqlite.org/vtab.html).
          It can make a good template for new custom virtual table implementations.
     
       *  **shathree.c** —  An implementation of the sha3() and
    diff --git a/ext/misc/amatch.c b/ext/misc/amatch.c
    index dd9bee53d..b3fcbac50 100644
    --- a/ext/misc/amatch.c
    +++ b/ext/misc/amatch.c
    @@ -482,9 +482,9 @@ struct amatch_rule {
       amatch_rule *pNext;      /* Next rule in order of increasing rCost */
       char *zFrom;             /* Transform from (a string from user input) */
       amatch_cost rCost;       /* Cost of this transformation */
    -  amatch_langid iLang;     /* The langauge to which this rule belongs */
    +  amatch_langid iLang;     /* The language to which this rule belongs */
       amatch_len nFrom, nTo;   /* Length of the zFrom and zTo strings */
    -  char zTo[4];             /* Tranform to V.W value (extra space appended) */
    +  char zTo[4];             /* Transform to V.W value (extra space appended) */
     };
     
     /* 
    diff --git a/ext/misc/btreeinfo.c b/ext/misc/btreeinfo.c
    index 02f8c0319..9c726f5f1 100644
    --- a/ext/misc/btreeinfo.c
    +++ b/ext/misc/btreeinfo.c
    @@ -49,7 +49,7 @@
     ** USAGE EXAMPLES:
     **
     ** Show the table btrees in a schema order with the tables with the most
    -** rows occuring first:
    +** rows occurring first:
     **
     **      SELECT name, nEntry
     **        FROM sqlite_btreeinfo
    diff --git a/ext/misc/closure.c b/ext/misc/closure.c
    index 267ae1c42..14caf271f 100644
    --- a/ext/misc/closure.c
    +++ b/ext/misc/closure.c
    @@ -137,7 +137,7 @@
     **       AND closure.idname='groupId'
     **       AND closure.parentname='parentId';
     **
    -** See the documentation at http://www.sqlite.org/loadext.html for information
    +** See the documentation at http://sqlite.org/loadext.html for information
     ** on how to compile and use loadable extensions such as this one.
     */
     #include "sqlite3ext.h"
    diff --git a/ext/misc/completion.c b/ext/misc/completion.c
    index 54abc0ae1..0a6db1a22 100644
    --- a/ext/misc/completion.c
    +++ b/ext/misc/completion.c
    @@ -41,6 +41,11 @@ SQLITE_EXTENSION_INIT1
     
     #ifndef SQLITE_OMIT_VIRTUALTABLE
     
    +#ifndef IsAlnum
    +#define IsAlnum(X)  isalnum((unsigned char)X)
    +#endif
    +
    +
     /* completion_vtab is a subclass of sqlite3_vtab which will
     ** serve as the underlying representation of a completion virtual table
     */
    @@ -377,7 +382,7 @@ static int completionFilter(
       }
       if( pCur->zLine!=0 && pCur->zPrefix==0 ){
         int i = pCur->nLine;
    -    while( i>0 && (isalnum(pCur->zLine[i-1]) || pCur->zLine[i-1]=='_') ){
    +    while( i>0 && (IsAlnum(pCur->zLine[i-1]) || pCur->zLine[i-1]=='_') ){
           i--;
         }
         pCur->nPrefix = pCur->nLine - i;
    diff --git a/ext/misc/csv.c b/ext/misc/csv.c
    index b38500f4b..8331265aa 100644
    --- a/ext/misc/csv.c
    +++ b/ext/misc/csv.c
    @@ -315,7 +315,7 @@ typedef struct CsvTable {
     } CsvTable;
     
     /* Allowed values for tstFlags */
    -#define CSVTEST_FIDX  0x0001      /* Pretend that constrained searchs cost less*/
    +#define CSVTEST_FIDX  0x0001      /* Pretend that constrained search cost less*/
     
     /* A cursor for the CSV virtual table */
     typedef struct CsvCursor {
    diff --git a/ext/misc/decimal.c b/ext/misc/decimal.c
    index 9365ae68b..60488a001 100644
    --- a/ext/misc/decimal.c
    +++ b/ext/misc/decimal.c
    @@ -27,6 +27,9 @@ SQLITE_EXTENSION_INIT1
     # define UNUSED_PARAMETER(X)  (void)(X)
     #endif
     
    +#ifndef IsSpace
    +#define IsSpace(X)  isspace((unsigned char)X)
    +#endif
     
     /* A decimal object */
     typedef struct Decimal Decimal;
    @@ -76,7 +79,7 @@ static Decimal *decimalNewFromText(const char *zIn, int n){
       p->nFrac = 0;
       p->a = sqlite3_malloc64( n+1 );
       if( p->a==0 ) goto new_from_text_failed;
    -  for(i=0; isspace(zIn[i]); i++){}
    +  for(i=0; IsSpace(zIn[i]); i++){}
       if( zIn[i]=='-' ){
         p->sign = 1;
         i++;
    @@ -731,7 +734,7 @@ static void decimalSubFunc(
       decimal_free(pB);
     }
     
    -/* Aggregate funcion:   decimal_sum(X)
    +/* Aggregate function:   decimal_sum(X)
     **
     ** Works like sum() except that it uses decimal arithmetic for unlimited
     ** precision.
    diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c
    index 80ac3b3e7..96a7f82bd 100644
    --- a/ext/misc/fileio.c
    +++ b/ext/misc/fileio.c
    @@ -98,14 +98,9 @@ SQLITE_EXTENSION_INIT1
     #  include 
     #  include "test_windirent.h"
     #  define dirent DIRENT
    -#  ifndef chmod
    -#    define chmod _chmod
    -#  endif
    -#  ifndef stat
    -#    define stat _stat
    -#  endif
    -#  define mkdir(path,mode) _mkdir(path)
    -#  define lstat(path,buf) stat(path,buf)
    +#  define stat _stat
    +#  define chmod(path,mode) fileio_chmod(path,mode)
    +#  define mkdir(path,mode) fileio_mkdir(path)
     #endif
     #include 
     #include 
    @@ -130,6 +125,40 @@ SQLITE_EXTENSION_INIT1
     #define FSDIR_COLUMN_PATH     4     /* Path to top of search */
     #define FSDIR_COLUMN_DIR      5     /* Path is relative to this directory */
     
    +/*
    +** UTF8 chmod() function for Windows
    +*/
    +#if defined(_WIN32) || defined(WIN32)
    +static int fileio_chmod(const char *zPath, int pmode){
    +  sqlite3_int64 sz = strlen(zPath);
    +  wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(b1[0]) );
    +  int rc;
    +  if( b1==0 ) return -1;
    +  sz = MultiByteToWideChar(CP_UTF8, 0, zPath, sz, b1, sz);
    +  b1[sz] = 0;
    +  rc = _wchmod(b1, pmode);
    +  sqlite3_free(b1);
    +  return rc;
    +}
    +#endif
    +
    +/*
    +** UTF8 mkdir() function for Windows
    +*/
    +#if defined(_WIN32) || defined(WIN32)
    +static int fileio_mkdir(const char *zPath){
    +  sqlite3_int64 sz = strlen(zPath);
    +  wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(b1[0]) );
    +  int rc;
    +  if( b1==0 ) return -1;
    +  sz = MultiByteToWideChar(CP_UTF8, 0, zPath, sz, b1, sz);
    +  b1[sz] = 0;
    +  rc = _wmkdir(b1);
    +  sqlite3_free(b1);
    +  return rc;
    +}
    +#endif
    +
     
     /*
     ** Set the result stored by context ctx to a blob containing the 
    @@ -291,7 +320,13 @@ static int fileStat(
       struct stat *pStatBuf
     ){
     #if defined(_WIN32)
    -  int rc = stat(zPath, pStatBuf);
    +  sqlite3_int64 sz = strlen(zPath);
    +  wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(b1[0]) );
    +  int rc;
    +  if( b1==0 ) return 1;
    +  sz = MultiByteToWideChar(CP_UTF8, 0, zPath, sz, b1, sz);
    +  b1[sz] = 0;
    +  rc = _wstat(b1, pStatBuf);
       if( rc==0 ) statTimesToUtc(zPath, pStatBuf);
       return rc;
     #else
    @@ -309,9 +344,7 @@ static int fileLinkStat(
       struct stat *pStatBuf
     ){
     #if defined(_WIN32)
    -  int rc = lstat(zPath, pStatBuf);
    -  if( rc==0 ) statTimesToUtc(zPath, pStatBuf);
    -  return rc;
    +  return fileStat(zPath, pStatBuf);
     #else
       return lstat(zPath, pStatBuf);
     #endif
    diff --git a/ext/misc/fossildelta.c b/ext/misc/fossildelta.c
    index e638737d2..9f81270d7 100644
    --- a/ext/misc/fossildelta.c
    +++ b/ext/misc/fossildelta.c
    @@ -22,7 +22,7 @@
     ** The delta format is the Fossil delta format, described in a comment
     ** on the delete_create() function implementation below, and also at
     **
    -**    https://www.fossil-scm.org/fossil/doc/trunk/www/delta_format.wiki
    +**    https://fossil-scm.org/fossil/doc/trunk/www/delta_format.wiki
     **
     ** This delta format is used by the RBU extension, which is the main
     ** reason that these routines are included in the extension library.
    diff --git a/ext/misc/fuzzer.c b/ext/misc/fuzzer.c
    index 92b7c0dae..e16d005d9 100644
    --- a/ext/misc/fuzzer.c
    +++ b/ext/misc/fuzzer.c
    @@ -68,7 +68,7 @@
     **       AND distance<200;
     **
     ** This first query outputs the string "abcdefg" and all strings that
    -** can be derived from that string by appling the specified transformations.
    +** can be derived from that string by applying the specified transformations.
     ** The strings are output together with their total transformation cost
     ** (called "distance") and appear in order of increasing cost.  No string
     ** is output more than once.  If there are multiple ways to transform the
    @@ -97,7 +97,7 @@
     **    LIMIT 20
     **
     ** The query above gives the 20 closest words to the $word being tested.
    -** (Note that for good performance, the vocubulary.w column should be
    +** (Note that for good performance, the vocabulary.w column should be
     ** indexed.)
     **
     ** A similar query can be used to find all words in the dictionary that
    @@ -207,7 +207,7 @@ struct fuzzer_rule {
     ** Every stem is added to a hash table as it is output.  Generation of
     ** duplicate stems is suppressed.
     **
    -** Active stems (those that might generate new outputs) are kepts on a linked
    +** Active stems (those that might generate new outputs) are kept on a linked
     ** list sorted by increasing cost.  The cost is the sum of rBaseCost and
     ** pRule->rCost.
     */
    diff --git a/ext/misc/ieee754.c b/ext/misc/ieee754.c
    index 99489fe9c..5ddb4a2fe 100644
    --- a/ext/misc/ieee754.c
    +++ b/ext/misc/ieee754.c
    @@ -79,7 +79,7 @@
     **    WITH c(name,bin) AS (VALUES
     **       ('minimum positive value',        x'0000000000000001'),
     **       ('maximum subnormal value',       x'000fffffffffffff'),
    -**       ('mininum positive nornal value', x'0010000000000000'),
    +**       ('minimum positive normal value', x'0010000000000000'),
     **       ('maximum value',                 x'7fefffffffffffff'))
     **    SELECT c.name, decimal_mul(ieee754_mantissa(c.bin),pow2.v)
     **      FROM pow2, c WHERE pow2.x=ieee754_exponent(c.bin);
    diff --git a/ext/misc/memstat.c b/ext/misc/memstat.c
    index c56af9f29..8e69b4695 100644
    --- a/ext/misc/memstat.c
    +++ b/ext/misc/memstat.c
    @@ -401,8 +401,14 @@ static sqlite3_module memstatModule = {
     
     #endif /* SQLITE_OMIT_VIRTUALTABLE */
     
    -int sqlite3MemstatVtabInit(sqlite3 *db){
    +int sqlite3MemstatVtabInit(
    +  sqlite3 *db,
    +  char **NotUsed1,
    +  const sqlite3_api_routines *NotUsed2
    +){
       int rc = SQLITE_OK;
    +  (void)NotUsed1;
    +  (void)NotUsed2;
     #ifndef SQLITE_OMIT_VIRTUALTABLE
       rc = sqlite3_create_module(db, "sqlite_memstat", &memstatModule, 0);
     #endif
    @@ -421,7 +427,7 @@ int sqlite3_memstat_init(
       int rc = SQLITE_OK;
       SQLITE_EXTENSION_INIT2(pApi);
     #ifndef SQLITE_OMIT_VIRTUALTABLE
    -  rc = sqlite3MemstatVtabInit(db);
    +  rc = sqlite3MemstatVtabInit(db, 0, 0);
     #endif
       return rc;
     }
    diff --git a/ext/misc/normalize.c b/ext/misc/normalize.c
    index 08d7733b9..800e12911 100644
    --- a/ext/misc/normalize.c
    +++ b/ext/misc/normalize.c
    @@ -286,7 +286,7 @@ static const unsigned char sqlite3CtypeMap[256] = {
     #define TK_VARIABLE TK_LITERAL
     #define TK_BLOB     TK_LITERAL
     
    -/* Disable nuisence warnings about case fall-through */
    +/* Disable nuisance warnings about case fall-through */
     #if !defined(deliberate_fall_through) && defined(__GCC__) && __GCC__>=7
     # define deliberate_fall_through __attribute__((fallthrough));
     #else
    diff --git a/ext/misc/percentile.c b/ext/misc/percentile.c
    index 06865185d..98e45cc3a 100644
    --- a/ext/misc/percentile.c
    +++ b/ext/misc/percentile.c
    @@ -205,7 +205,7 @@ static int percentBinarySearch(Percentile *p, double y, int bExact){
     /*
     ** Generate an error for a percentile function.
     **
    -** The error format string must have exactly one occurrance of "%%s()"
    +** The error format string must have exactly one occurrence of "%%s()"
     ** (with two '%' characters).  That substring will be replaced by the name
     ** of the function.
     */
    diff --git a/ext/misc/series.c b/ext/misc/series.c
    index 06f1fd281..22e0f7edb 100644
    --- a/ext/misc/series.c
    +++ b/ext/misc/series.c
    @@ -139,7 +139,7 @@ static sqlite3_int64 genSeqMember(
         smBase += (mxI64 - mxI64/2) * smStep;
       }
       /* Under UBSAN (or on 1's complement machines), must do this last term
    -   * in steps to avoid the dreaded (and harmless) signed multiply overlow. */
    +   * in steps to avoid the dreaded (and harmless) signed multiply overflow. */
       if( ix>=2 ){
         sqlite3_int64 ix2 = (sqlite3_int64)ix/2;
         smBase += ix2*smStep;
    @@ -580,7 +580,7 @@ static int seriesFilter(
       for(i=0; i=zEnd && nDigits>0 && eValid && nonNum==0;
     }
     
    diff --git a/ext/misc/uint.c b/ext/misc/uint.c
    index 286314fef..a527b2f07 100644
    --- a/ext/misc/uint.c
    +++ b/ext/misc/uint.c
    @@ -16,7 +16,7 @@
     ** of digits compare in numeric order.
     **
     **     *   Leading zeros are handled properly, in the sense that
    -**         they do not mess of the maginitude comparison of embedded
    +**         they do not mess of the magnitude comparison of embedded
     **         strings of digits.  "x00123y" is equal to "x123y".
     **
     **     *   Only unsigned integers are recognized.  Plus and minus
    diff --git a/ext/misc/vfsstat.c b/ext/misc/vfsstat.c
    index ba22115ab..504c0b31d 100644
    --- a/ext/misc/vfsstat.c
    +++ b/ext/misc/vfsstat.c
    @@ -22,9 +22,9 @@ SQLITE_EXTENSION_INIT1
     ** most VFS calls to be recorded.
     **
     ** To use this module, first compile it as a loadable extension.  See
    -** https://www.sqlite.org/loadext.html#build for compilations instructions.
    +** https://sqlite.org/loadext.html#build for compilations instructions.
     **
    -** After compliing, load this extension, then open database connections to be
    +** After compiling, load this extension, then open database connections to be
     ** measured.  Query usages status using the vfsstat virtual table:
     **
     **         SELECT * FROM vfsstat;
    diff --git a/ext/misc/vfstrace.c b/ext/misc/vfstrace.c
    index c274558d1..9d75a8b64 100644
    --- a/ext/misc/vfstrace.c
    +++ b/ext/misc/vfstrace.c
    @@ -120,7 +120,7 @@
     **
     ** Individual APIs can be enabled or disabled by name, with or without
     ** the initial "x" character.  For example, to set up for tracing lock
    -** primatives only:
    +** primitives only:
     **
     **    PRAGMA vfstrace('-all, +Lock,Unlock,ShmLock');
     **
    @@ -581,7 +581,7 @@ static int vfstraceFileControl(sqlite3_file *pFile, int op, void *pArg){
           const char *const* a = (const char*const*)pArg;
           if( a[1] && strcmp(a[1],"vfstrace")==0 && a[2] ){
             const u8 *zArg = (const u8*)a[2];
    -        if( zArg[0]>='0' && zArg[0]<=9 ){
    +        if( zArg[0]>='0' && zArg[0]<='9' ){
               pInfo->mTrace = (sqlite3_uint64)strtoll(a[2], 0, 0);
             }else{
               static const struct {
    diff --git a/ext/misc/vtablog.c b/ext/misc/vtablog.c
    index 2cc29c285..e8f084e1b 100644
    --- a/ext/misc/vtablog.c
    +++ b/ext/misc/vtablog.c
    @@ -240,7 +240,7 @@ static int vtablogConnect(
     
     
     /*
    -** This method is the destructor for vtablog_cursor objects.
    +** This method is the destructor for vtablog_vtab objects.
     */
     static int vtablogDisconnect(sqlite3_vtab *pVtab){
       vtablog_vtab *pTab = (vtablog_vtab*)pVtab;
    @@ -252,7 +252,7 @@ static int vtablogDisconnect(sqlite3_vtab *pVtab){
     }
     
     /*
    -** This method is the destructor for vtablog_cursor objects.
    +** This method is (also) the destructor for vtablog_vtab objects.
     */
     static int vtablogDestroy(sqlite3_vtab *pVtab){
       vtablog_vtab *pTab = (vtablog_vtab*)pVtab;
    diff --git a/ext/misc/vtshim.c b/ext/misc/vtshim.c
    index 0709a26a7..3f7945724 100644
    --- a/ext/misc/vtshim.c
    +++ b/ext/misc/vtshim.c
    @@ -425,7 +425,7 @@ static int vtshimRollbackTo(sqlite3_vtab *pBase, int n){
       return rc;
     }
     
    -/* The destructor function for a disposible module */
    +/* The destructor function for a disposable module */
     static void vtshimAuxDestructor(void *pXAux){
       vtshim_aux *pAux = (vtshim_aux*)pXAux;
       assert( pAux->pAllVtab==0 );
    diff --git a/ext/rbu/rbuvacuum2.test b/ext/rbu/rbuvacuum2.test
    index 34ec26188..e338fedda 100644
    --- a/ext/rbu/rbuvacuum2.test
    +++ b/ext/rbu/rbuvacuum2.test
    @@ -171,7 +171,7 @@ foreach {ttt state} {
     #   * Set the state db permissions to the same as those on the db file.
     #
     db close
    -if {$::tcl_platform(platform)=="unix"} {
    +if {$::tcl_platform(platform) eq "unix"} {
       forcedelete test.db
     
       sqlite3 db test.db
    diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c
    index 36688925d..4509986ee 100644
    --- a/ext/rbu/sqlite3rbu.c
    +++ b/ext/rbu/sqlite3rbu.c
    @@ -4822,7 +4822,7 @@ static int rbuVfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
     
       /* If this is an RBU vacuum operation and this is the target database,
       ** pretend that it has at least one page. Otherwise, SQLite will not
    -  ** check for the existance of a *-wal file. rbuVfsRead() contains 
    +  ** check for the existence of a *-wal file. rbuVfsRead() contains 
       ** similar logic.  */
       if( rc==SQLITE_OK && *pSize==0 
        && p->pRbu && rbuIsVacuum(p->pRbu) 
    diff --git a/ext/rbu/sqlite3rbu.h b/ext/rbu/sqlite3rbu.h
    index d156b3178..1534877a9 100644
    --- a/ext/rbu/sqlite3rbu.h
    +++ b/ext/rbu/sqlite3rbu.h
    @@ -49,7 +49,7 @@
     **
     ** "RBU" stands for "Resumable Bulk Update". As in a large database update
     ** transmitted via a wireless network to a mobile device. A transaction
    -** applied using this extension is hence refered to as an "RBU update".
    +** applied using this extension is hence referred to as an "RBU update".
     **
     **
     ** LIMITATIONS
    @@ -346,7 +346,7 @@ SQLITE_API sqlite3rbu *sqlite3rbu_open(
     ** the next call to sqlite3rbu_vacuum() opens a handle that starts a 
     ** new RBU vacuum operation.
     **
    -** As with sqlite3rbu_open(), Zipvfs users should rever to the comment
    +** As with sqlite3rbu_open(), Zipvfs users should refer to the comment
     ** describing the sqlite3rbu_create_vfs() API function below for 
     ** a description of the complications associated with using RBU with 
     ** zipvfs databases.
    @@ -442,7 +442,7 @@ SQLITE_API int sqlite3rbu_savestate(sqlite3rbu *pRbu);
     **
     ** If the RBU update has been completely applied, mark the RBU database
     ** as fully applied. Otherwise, assuming no error has occurred, save the
    -** current state of the RBU update appliation to the RBU database.
    +** current state of the RBU update application to the RBU database.
     **
     ** If an error has already occurred as part of an sqlite3rbu_step()
     ** or sqlite3rbu_open() call, or if one occurs within this function, an
    diff --git a/ext/rbu/test_rbu.c b/ext/rbu/test_rbu.c
    index 969d6208d..6b933cdb2 100644
    --- a/ext/rbu/test_rbu.c
    +++ b/ext/rbu/test_rbu.c
    @@ -139,7 +139,7 @@ static int SQLITE_TCLAPI test_sqlite3rbu_cmd(
           }else{
             Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
             if( zErrmsg ){
    -          Tcl_AppendResult(interp, " - ", zErrmsg, 0);
    +          Tcl_AppendResult(interp, " - ", zErrmsg, NULL);
               sqlite3_free(zErrmsg);
             }
             ret = TCL_ERROR;
    @@ -399,7 +399,7 @@ static int SQLITE_TCLAPI test_sqlite3rbu_internal_test(
     
       db = sqlite3rbu_db(0, 0);
       if( db!=0 ){
    -    Tcl_AppendResult(interp, "sqlite3rbu_db(0, 0)!=0", 0);
    +    Tcl_AppendResult(interp, "sqlite3rbu_db(0, 0)!=0", NULL);
         return TCL_ERROR;
       }
     
    diff --git a/ext/recover/dbdata.c b/ext/recover/dbdata.c
    index 001840b17..225f1a9b3 100644
    --- a/ext/recover/dbdata.c
    +++ b/ext/recover/dbdata.c
    @@ -1008,6 +1008,9 @@ static int sqlite3DbdataRegister(sqlite3 *db){
       return rc;
     }
     
    +#ifdef _WIN32
    +__declspec(dllexport)
    +#endif
     int sqlite3_dbdata_init(
       sqlite3 *db, 
       char **pzErrMsg, 
    diff --git a/ext/recover/recoverslowidx.test b/ext/recover/recoverslowidx.test
    index 269105113..36842b8a6 100644
    --- a/ext/recover/recoverslowidx.test
    +++ b/ext/recover/recoverslowidx.test
    @@ -39,6 +39,7 @@ do_test 1.2 {
     } [list {*}{
       {BEGIN}
       {PRAGMA writable_schema = on}
    +  {PRAGMA foreign_keys = off}
       {PRAGMA encoding = 'UTF-8'}
       {PRAGMA page_size = '1024'}
       {PRAGMA auto_vacuum = '0'}
    @@ -67,6 +68,7 @@ do_test 1.4 {
     } [list {*}{
       {BEGIN}
       {PRAGMA writable_schema = on}
    +  {PRAGMA foreign_keys = off}
       {PRAGMA encoding = 'UTF-8'}
       {PRAGMA page_size = '1024'}
       {PRAGMA auto_vacuum = '0'}
    diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c
    index 58d726f59..d50aa25ea 100644
    --- a/ext/recover/sqlite3recover.c
    +++ b/ext/recover/sqlite3recover.c
    @@ -33,6 +33,16 @@ typedef unsigned int u32;
     typedef unsigned char u8;
     typedef sqlite3_int64 i64;
     
    +/*
    +** Work around C99 "flex-array" syntax for pre-C99 compilers, so as
    +** to avoid complaints from -fsanitize=strict-bounds.
    +*/
    +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)
    +# define FLEXARRAY
    +#else
    +# define FLEXARRAY 1
    +#endif
    +
     typedef struct RecoverTable RecoverTable;
     typedef struct RecoverColumn RecoverColumn;
     
    @@ -140,9 +150,12 @@ struct RecoverColumn {
     typedef struct RecoverBitmap RecoverBitmap;
     struct RecoverBitmap {
       i64 nPg;                        /* Size of bitmap */
    -  u32 aElem[1];                   /* Array of 32-bit bitmasks */
    +  u32 aElem[FLEXARRAY];           /* Array of 32-bit bitmasks */
     };
     
    +/* Size in bytes of a RecoverBitmap object sufficient to cover 32 pages */
    +#define SZ_RECOVERBITMAP_32  (16)
    +
     /*
     ** State variables (part of the sqlite3_recover structure) used while
     ** recovering data for tables identified in the recovered schema (state
    @@ -382,7 +395,7 @@ static int recoverError(
     */
     static RecoverBitmap *recoverBitmapAlloc(sqlite3_recover *p, i64 nPg){
       int nElem = (nPg+1+31) / 32;
    -  int nByte = sizeof(RecoverBitmap) + nElem*sizeof(u32);
    +  int nByte = SZ_RECOVERBITMAP_32 + nElem*sizeof(u32);
       RecoverBitmap *pRet = (RecoverBitmap*)recoverMalloc(p, nByte);
     
       if( pRet ){
    @@ -2574,37 +2587,53 @@ static void recoverUninstallWrapper(sqlite3_recover *p){
     static void recoverStep(sqlite3_recover *p){
       assert( p && p->errCode==SQLITE_OK );
       switch( p->eState ){
    -    case RECOVER_STATE_INIT:
    +    case RECOVER_STATE_INIT: {
    +      int bUseWrapper = 1;
           /* This is the very first call to sqlite3_recover_step() on this object.
           */
           recoverSqlCallback(p, "BEGIN");
           recoverSqlCallback(p, "PRAGMA writable_schema = on");
    +      recoverSqlCallback(p, "PRAGMA foreign_keys = off");
     
           recoverEnterMutex();
    -      recoverInstallWrapper(p);
     
           /* Open the output database. And register required virtual tables and 
           ** user functions with the new handle. */
           recoverOpenOutput(p);
     
    -      /* Open transactions on both the input and output databases. */
    -      sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_RESET_CACHE, 0);
    -      recoverExec(p, p->dbIn, "PRAGMA writable_schema = on");
    -      recoverExec(p, p->dbIn, "BEGIN");
    -      if( p->errCode==SQLITE_OK ) p->bCloseTransaction = 1;
    -      recoverExec(p, p->dbIn, "SELECT 1 FROM sqlite_schema");
    -      recoverTransferSettings(p);
    -      recoverOpenRecovery(p);
    -      recoverCacheSchema(p);
    -
    -      recoverUninstallWrapper(p);
    -      recoverLeaveMutex();
    +      /* Attempt to open a transaction and read page 1 of the input database.
    +      ** Two attempts may be made - one with a wrapper installed to ensure
    +      ** that the database header is sane, and then if that attempt returns
    +      ** SQLITE_NOTADB, then again with no wrapper. The second attempt is
    +      ** required for encrypted databases.  */
    +      if( p->errCode==SQLITE_OK ){
    +        do{
    +          p->errCode = SQLITE_OK;
    +          if( bUseWrapper ) recoverInstallWrapper(p);
    +
    +          /* Open a transaction on the input database. */
    +          sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_RESET_CACHE, 0);
    +          recoverExec(p, p->dbIn, "PRAGMA writable_schema = on");
    +          recoverExec(p, p->dbIn, "BEGIN");
    +          if( p->errCode==SQLITE_OK ) p->bCloseTransaction = 1;
    +          recoverExec(p, p->dbIn, "SELECT 1 FROM sqlite_schema");
    +          recoverTransferSettings(p);
    +          recoverOpenRecovery(p);
    +          recoverCacheSchema(p);
    +
    +          if( bUseWrapper ) recoverUninstallWrapper(p);
    +        }while( p->errCode==SQLITE_NOTADB 
    +             && (bUseWrapper--) 
    +             && SQLITE_OK==sqlite3_exec(p->dbIn, "ROLLBACK", 0, 0, 0)
    +        );
    +      }
     
    +      recoverLeaveMutex();
           recoverExec(p, p->dbOut, "BEGIN");
    -
           recoverWriteSchema1(p);
           p->eState = RECOVER_STATE_WRITING;
           break;
    +    }
           
         case RECOVER_STATE_WRITING: {
           if( p->w1.pTbls==0 ){
    diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c
    index c1c0d8838..f8ab41dc2 100644
    --- a/ext/recover/test_recover.c
    +++ b/ext/recover/test_recover.c
    @@ -58,7 +58,7 @@ static int xSqlCallback(void *pSqlArg, const char *zSql){
     static int getDbPointer(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){
       Tcl_CmdInfo info;
       if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){
    -    Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), 0);
    +    Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), NULL);
         return TCL_ERROR;
       }
       *pDb = *(sqlite3 **)info.objClientData;
    diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c
    index 5f6e646e4..ed30357e5 100644
    --- a/ext/repair/checkindex.c
    +++ b/ext/repair/checkindex.c
    @@ -110,7 +110,7 @@ static int cidxConnect(
           " current_key TEXT,"       /* SQLite quote() text of key values */
           " index_name HIDDEN,"      /* IN: name of the index being scanned */
           " after_key HIDDEN,"       /* IN: Start scanning after this key */
    -      " scanner_sql HIDDEN"      /* debuggingn info: SQL used for scanner */
    +      " scanner_sql HIDDEN"      /* debugging info: SQL used for scanner */
           ")"
       );
       pRet = cidxMalloc(&rc, sizeof(CidxTable));
    diff --git a/ext/rtree/README b/ext/rtree/README
    index 3736f45c5..051fd1339 100644
    --- a/ext/rtree/README
    +++ b/ext/rtree/README
    @@ -24,7 +24,7 @@ and query r-tree structures using ordinary SQL statements.
         3 and 11. Unlike regular SQLite tables, r-tree tables are strongly 
         typed. 
     
    -    The leftmost column is always the pimary key and contains 64-bit 
    +    The leftmost column is always the primary key and contains 64-bit 
         integer values. Each subsequent column contains a 32-bit real
         value. For each pair of real values, the first (leftmost) must be 
         less than or equal to the second. R-tree tables may be 
    diff --git a/ext/rtree/geopoly.c b/ext/rtree/geopoly.c
    index 3e9c2a271..0ae42e7b7 100644
    --- a/ext/rtree/geopoly.c
    +++ b/ext/rtree/geopoly.c
    @@ -771,7 +771,7 @@ static void geopolyBBoxFinal(
     ** Determine if point (x0,y0) is beneath line segment (x1,y1)->(x2,y2).
     ** Returns:
     **
    -**    +2  x0,y0 is on the line segement
    +**    +2  x0,y0 is on the line segment
     **
     **    +1  x0,y0 is beneath line segment
     **
    @@ -877,7 +877,7 @@ static void geopolyWithinFunc(
       sqlite3_free(p2);
     }
     
    -/* Objects used by the overlap algorihm. */
    +/* Objects used by the overlap algorithm. */
     typedef struct GeoEvent GeoEvent;
     typedef struct GeoSegment GeoSegment;
     typedef struct GeoOverlap GeoOverlap;
    diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c
    index 8ed8978bd..d8567b65a 100644
    --- a/ext/rtree/rtree.c
    +++ b/ext/rtree/rtree.c
    @@ -64,6 +64,8 @@
     #endif
     int sqlite3GetToken(const unsigned char*,int*); /* In the SQLite core */
     
    +#include 
    +
     /*
     ** If building separately, we will need some setup that is normally
     ** found in sqliteInt.h
    @@ -94,6 +96,14 @@ typedef unsigned int u32;
     # define ALWAYS(X)      (X)
     # define NEVER(X)       (X)
     #endif
    +#ifndef offsetof
    +#define offsetof(STRUCTURE,FIELD) ((size_t)((char*)&((STRUCTURE*)0)->FIELD))
    +#endif
    +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)
    +# define FLEXARRAY
    +#else
    +# define FLEXARRAY 1
    +#endif
     #endif /* !defined(SQLITE_AMALGAMATION) */
     
     /* Macro to check for 4-byte alignment.  Only used inside of assert() */
    @@ -414,9 +424,13 @@ struct RtreeMatchArg {
       RtreeGeomCallback cb;       /* Info about the callback functions */
       int nParam;                 /* Number of parameters to the SQL function */
       sqlite3_value **apSqlParam; /* Original SQL parameter values */
    -  RtreeDValue aParam[1];      /* Values for parameters to the SQL function */
    +  RtreeDValue aParam[FLEXARRAY]; /* Values for parameters to the SQL function */
     };
     
    +/* Size of an RtreeMatchArg object with N parameters */
    +#define SZ_RTREEMATCHARG(N)  \
    +        (offsetof(RtreeMatchArg,aParam)+(N)*sizeof(RtreeDValue))
    +
     #ifndef MAX
     # define MAX(x,y) ((x) < (y) ? (y) : (x))
     #endif
    @@ -2105,7 +2119,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
     }
     
     /*
    -** Return the N-dimensional volumn of the cell stored in *p.
    +** Return the N-dimensional volume of the cell stored in *p.
     */
     static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){
       RtreeDValue area = (RtreeDValue)1;
    @@ -2836,7 +2850,7 @@ static int deleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell, int iHeight){
     
       return rc;
     }
    -	
    +
     /*
     ** Insert cell pCell into node pNode. Node pNode is the head of a 
     ** subtree iHeight high (leaf nodes have iHeight==0).
    @@ -3871,7 +3885,7 @@ static sqlite3_stmt *rtreeCheckPrepare(
     /*
     ** The second and subsequent arguments to this function are a printf()
     ** style format string and arguments. This function formats the string and
    -** appends it to the report being accumuated in pCheck.
    +** appends it to the report being accumulated in pCheck.
     */
     static void rtreeCheckAppendMsg(RtreeCheck *pCheck, const char *zFmt, ...){
       va_list ap;
    @@ -4369,8 +4383,7 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){
       sqlite3_int64 nBlob;
       int memErr = 0;
     
    -  nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(RtreeDValue)
    -           + nArg*sizeof(sqlite3_value*);
    +  nBlob = SZ_RTREEMATCHARG(nArg) + nArg*sizeof(sqlite3_value*);
       pBlob = (RtreeMatchArg *)sqlite3_malloc64(nBlob);
       if( !pBlob ){
         sqlite3_result_error_nomem(ctx);
    diff --git a/ext/rtree/rtreedoc2.test b/ext/rtree/rtreedoc2.test
    index ca0c6b31b..6032f32fe 100644
    --- a/ext/rtree/rtreedoc2.test
    +++ b/ext/rtree/rtreedoc2.test
    @@ -116,7 +116,7 @@ set testprefix rtreedoc2-2
     # 1-dimensional R*Tree, 4 for a 2-dimensional R*Tree, 6 for a
     # 3-dimensional R*Tree, and so forth.
     #
    -# The second argument refered to above is the length of the list passed
    +# The second argument referred to above is the length of the list passed
     # as the 3rd parameter to the Tcl script.
     #
     do_execsql_test 1.0 {
    diff --git a/ext/session/changeset.c b/ext/session/changeset.c
    index 9cf62949a..fe5c4dc5e 100644
    --- a/ext/session/changeset.c
    +++ b/ext/session/changeset.c
    @@ -28,11 +28,17 @@ static void usage(const char *argv0){
       fprintf(stderr, "Usage: %s FILENAME COMMAND ...\n", argv0);
       fprintf(stderr,
         "COMMANDs:\n"
    -    "   apply DB           Apply the changeset to database file DB\n"
    -    "   concat FILE2 OUT   Concatenate FILENAME and FILE2 into OUT\n"
    -    "   dump               Show the complete content of the changeset\n"
    -    "   invert OUT         Write an inverted changeset into file OUT\n"
    -    "   sql                Give a pseudo-SQL rendering of the changeset\n"
    +    "  apply DB [OPTIONS]   Apply the changeset to database file DB. OPTIONS:\n"
    +    "                          -n|--dryrun     Test run. Don't apply changes\n"
    +    "                          --enablefk      Enable FOREIGN KEY support\n"
    +    "                          --nosavepoint   \\\n"
    +    "                          --invert         \\___  Flags passed into\n"
    +    "                          --ignorenoop     /     changeset_apply_v2()\n"
    +    "                          --fknoaction    /\n"
    +    "  concat FILE2 OUT     Concatenate FILENAME and FILE2 into OUT\n"
    +    "  dump                 Show the complete content of the changeset\n"
    +    "  invert OUT           Write an inverted changeset into file OUT\n"
    +    "  sql                  Give a pseudo-SQL rendering of the changeset\n"
       );
       exit(1);
     }
    @@ -161,7 +167,7 @@ static int conflictCallback(
         case SQLITE_DELETE:     zOp = "DELETE from";   break;
       }
       printf("%s conflict on %s table %s with primary key", zType, zOp, zTab);
    -  for(i=0; inCol==0 ){
         u8 *abPK;
         assert( pTab->azCol==0 || pTab->abPK==0 );
    +    sqlite3_free(pTab->azCol);
    +    pTab->abPK = 0;
         rc = sessionTableInfo(pSession, db, zDb, 
             pTab->zName, &pTab->nCol, &pTab->nTotalCol, 0, &pTab->azCol, 
             &pTab->azDflt, &pTab->aiIdx, &abPK,
    @@ -2216,7 +2218,9 @@ int sqlite3session_diff(
         SessionTable *pTo;            /* Table zTbl */
     
         /* Locate and if necessary initialize the target table object */
    +    pSession->bAutoAttach++;
         rc = sessionFindTable(pSession, zTbl, &pTo);
    +    pSession->bAutoAttach--;
         if( pTo==0 ) goto diff_out;
         if( sessionInitTable(pSession, pTo, pSession->db, pSession->zDb) ){
           rc = pSession->rc;
    @@ -2227,17 +2231,43 @@ int sqlite3session_diff(
         if( rc==SQLITE_OK ){
           int bHasPk = 0;
           int bMismatch = 0;
    -      int nCol;                   /* Columns in zFrom.zTbl */
    +      int nCol = 0;               /* Columns in zFrom.zTbl */
           int bRowid = 0;
    -      u8 *abPK;
    +      u8 *abPK = 0;
           const char **azCol = 0;
    -      rc = sessionTableInfo(0, db, zFrom, zTbl, 
    -          &nCol, 0, 0, &azCol, 0, 0, &abPK, 
    -          pSession->bImplicitPK ? &bRowid : 0
    -      );
    +      char *zDbExists = 0;
    +
    +      /* Check that database zFrom is attached.  */
    +      zDbExists = sqlite3_mprintf("SELECT * FROM %Q.sqlite_schema", zFrom);
    +      if( zDbExists==0 ){
    +        rc = SQLITE_NOMEM;
    +      }else{
    +        sqlite3_stmt *pDbExists = 0;
    +        rc = sqlite3_prepare_v2(db, zDbExists, -1, &pDbExists, 0);
    +        if( rc==SQLITE_ERROR ){
    +          rc = SQLITE_OK;
    +          nCol = -1;
    +        }
    +        sqlite3_finalize(pDbExists);
    +        sqlite3_free(zDbExists);
    +      }
    +
    +      if( rc==SQLITE_OK && nCol==0 ){
    +        rc = sessionTableInfo(0, db, zFrom, zTbl, 
    +            &nCol, 0, 0, &azCol, 0, 0, &abPK, 
    +            pSession->bImplicitPK ? &bRowid : 0
    +        );
    +      }
           if( rc==SQLITE_OK ){
             if( pTo->nCol!=nCol ){
    -          bMismatch = 1;
    +          if( nCol<=0 ){
    +            rc = SQLITE_SCHEMA;
    +            if( pzErrMsg ){
    +              *pzErrMsg = sqlite3_mprintf("no such table: %s.%s", zFrom, zTbl);
    +            }
    +          }else{
    +            bMismatch = 1;
    +          }
             }else{
               int i;
               for(i=0; idb;     /* Source database handle */
       SessionTable *pTab;             /* Used to iterate through attached tables */
    -  SessionBuffer buf = {0,0,0};    /* Buffer in which to accumlate changeset */
    +  SessionBuffer buf = {0,0,0};    /* Buffer in which to accumulate changeset */
       int rc;                         /* Return code */
     
       assert( xOutput==0 || (pnChangeset==0 && ppChangeset==0) );
    @@ -3366,14 +3396,15 @@ int sqlite3changeset_start_v2_strm(
     ** object and the buffer is full, discard some data to free up space.
     */
     static void sessionDiscardData(SessionInput *pIn){
    -  if( pIn->xInput && pIn->iNext>=sessions_strm_chunk_size ){
    -    int nMove = pIn->buf.nBuf - pIn->iNext;
    +  if( pIn->xInput && pIn->iCurrent>=sessions_strm_chunk_size ){
    +    int nMove = pIn->buf.nBuf - pIn->iCurrent;
         assert( nMove>=0 );
         if( nMove>0 ){
    -      memmove(pIn->buf.aBuf, &pIn->buf.aBuf[pIn->iNext], nMove);
    +      memmove(pIn->buf.aBuf, &pIn->buf.aBuf[pIn->iCurrent], nMove);
         }
    -    pIn->buf.nBuf -= pIn->iNext;
    -    pIn->iNext = 0;
    +    pIn->buf.nBuf -= pIn->iCurrent;
    +    pIn->iNext -= pIn->iCurrent;
    +    pIn->iCurrent = 0;
         pIn->nData = pIn->buf.nBuf;
       }
     }
    @@ -3727,8 +3758,8 @@ static int sessionChangesetNextOne(
       p->rc = sessionInputBuffer(&p->in, 2);
       if( p->rc!=SQLITE_OK ) return p->rc;
     
    -  sessionDiscardData(&p->in);
       p->in.iCurrent = p->in.iNext;
    +  sessionDiscardData(&p->in);
     
       /* If the iterator is already at the end of the changeset, return DONE. */
       if( p->in.iNext>=p->in.nData ){
    @@ -6087,14 +6118,19 @@ int sqlite3changegroup_add_change(
       sqlite3_changegroup *pGrp,
       sqlite3_changeset_iter *pIter
     ){
    +  int rc = SQLITE_OK;
    +
       if( pIter->in.iCurrent==pIter->in.iNext 
        || pIter->rc!=SQLITE_OK 
        || pIter->bInvert
       ){
         /* Iterator does not point to any valid entry or is an INVERT iterator. */
    -    return SQLITE_ERROR;
    +    rc = SQLITE_ERROR;
    +  }else{
    +    pIter->in.bNoDiscard = 1;
    +    rc = sessionOneChangeToHash(pGrp, pIter, 0);
       }
    -  return sessionOneChangeToHash(pGrp, pIter, 0);
    +  return rc;
     }
     
     /*
    diff --git a/ext/session/sqlite3session.h b/ext/session/sqlite3session.h
    index f950d419b..4dff5ce87 100644
    --- a/ext/session/sqlite3session.h
    +++ b/ext/session/sqlite3session.h
    @@ -358,9 +358,10 @@ void sqlite3session_table_filter(
     ** is inserted while a session object is enabled, then later deleted while 
     ** the same session object is disabled, no INSERT record will appear in the
     ** changeset, even though the delete took place while the session was disabled.
    -** Or, if one field of a row is updated while a session is disabled, and 
    -** another field of the same row is updated while the session is enabled, the
    -** resulting changeset will contain an UPDATE change that updates both fields.
    +** Or, if one field of a row is updated while a session is enabled, and 
    +** then another field of the same row is updated while the session is disabled,
    +** the resulting changeset will contain an UPDATE change that updates both
    +** fields.
     */
     int sqlite3session_changeset(
       sqlite3_session *pSession,      /* Session object */
    @@ -432,8 +433,9 @@ sqlite3_int64 sqlite3session_changeset_size(sqlite3_session *pSession);
     ** database zFrom the contents of the two compatible tables would be 
     ** identical.
     **
    -** It an error if database zFrom does not exist or does not contain the
    -** required compatible table.
    +** Unless the call to this function is a no-op as described above, it is an
    +** error if database zFrom does not exist or does not contain the required 
    +** compatible table.
     **
     ** If the operation is successful, SQLITE_OK is returned. Otherwise, an SQLite
     ** error code. In this case, if argument pzErrMsg is not NULL, *pzErrMsg
    @@ -568,7 +570,7 @@ int sqlite3changeset_start_v2(
     ** The following flags may passed via the 4th parameter to
     ** [sqlite3changeset_start_v2] and [sqlite3changeset_start_v2_strm]:
     **
    -** 
    SQLITE_CHANGESETAPPLY_INVERT
    +**
    SQLITE_CHANGESETSTART_INVERT
    ** Invert the changeset while iterating through it. This is equivalent to ** inverting a changeset using sqlite3changeset_invert() before applying it. ** It is an error to specify this flag with a patchset. @@ -883,19 +885,6 @@ int sqlite3changeset_concat( void **ppOut /* OUT: Buffer containing output changeset */ ); - -/* -** CAPI3REF: Upgrade the Schema of a Changeset/Patchset -*/ -int sqlite3changeset_upgrade( - sqlite3 *db, - const char *zDb, - int nIn, const void *pIn, /* Input changeset */ - int *pnOut, void **ppOut /* OUT: Inverse of input */ -); - - - /* ** CAPI3REF: Changegroup Handle ** diff --git a/ext/session/test_session.c b/ext/session/test_session.c index 41d6aaa10..f28604abc 100644 --- a/ext/session/test_session.c +++ b/ext/session/test_session.c @@ -34,7 +34,7 @@ struct TestStreamInput { static int dbHandleFromObj(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){ Tcl_CmdInfo info; if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){ - Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), 0); + Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), NULL); return TCL_ERROR; } @@ -130,7 +130,7 @@ static int SQLITE_TCLAPI test_sql_exec_changeset( rc = sql_exec_changeset(db, zSql, &nChangeset, &pChangeset); if( rc!=SQLITE_OK ){ Tcl_ResetResult(interp); - Tcl_AppendResult(interp, "error in sql_exec_changeset()", 0); + Tcl_AppendResult(interp, "error in sql_exec_changeset()", NULL); return TCL_ERROR; } @@ -166,7 +166,7 @@ static int test_session_error(Tcl_Interp *interp, int rc, char *zErr){ extern const char *sqlite3ErrName(int); Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); if( zErr ){ - Tcl_AppendResult(interp, " - ", zErr, 0); + Tcl_AppendResult(interp, " - ", zErr, NULL); sqlite3_free(zErr); } return TCL_ERROR; @@ -435,7 +435,7 @@ static int SQLITE_TCLAPI test_sqlite3session( } if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(objv[2]), &info) ){ - Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(objv[2]), 0); + Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(objv[2]), NULL); return TCL_ERROR; } db = *(sqlite3 **)info.objClientData; @@ -832,7 +832,7 @@ static int SQLITE_TCLAPI testSqlite3changesetApply( return TCL_ERROR; } if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &info) ){ - Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(objv[1]), 0); + Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(objv[1]), NULL); return TCL_ERROR; } db = *(sqlite3 **)info.objClientData; @@ -924,7 +924,7 @@ static int SQLITE_TCLAPI test_sqlite3changeset_apply_replace_all( return TCL_ERROR; } if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &info) ){ - Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(objv[2]), 0); + Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(objv[2]), NULL); return TCL_ERROR; } db = *(sqlite3 **)info.objClientData; @@ -1459,6 +1459,9 @@ struct TestChangegroup { typedef struct TestChangeIter TestChangeIter; struct TestChangeIter { sqlite3_changeset_iter *pIter; + + /* If this iter uses streaming. */ + TestStreamInput in; }; @@ -1551,7 +1554,7 @@ static int SQLITE_TCLAPI test_changegroup_cmd( TestChangeIter *pIter = 0; const char *zIter = Tcl_GetString(objv[2]); if( 0==Tcl_GetCommandInfo(interp, zIter, &cmdInfo) ){ - Tcl_AppendResult(interp, "no such iter: ", Tcl_GetString(objv[2]), 0); + Tcl_AppendResult(interp, "no such iter: ", Tcl_GetString(objv[2]), NULL); return TCL_ERROR; } @@ -1681,6 +1684,7 @@ static int SQLITE_TCLAPI test_sqlite3changeset_start( sqlite3_changeset_iter *pIter = 0; int flags = 0; int rc = SQLITE_OK; + int nAlloc = 0; /* Bytes of space to allocate */ static int iCmd = 1; char zCmd[64]; @@ -1696,18 +1700,36 @@ static int SQLITE_TCLAPI test_sqlite3changeset_start( return TCL_ERROR; } - flags = isInvert ? SQLITE_CHANGESETSTART_INVERT : 0; pChangeset = (void *)Tcl_GetByteArrayFromObj(objv[objc-1], &nChangeset); - rc = sqlite3changeset_start_v2(&pIter, (int)nChangeset, pChangeset, flags); + flags = isInvert ? SQLITE_CHANGESETSTART_INVERT : 0; + + nAlloc = sizeof(TestChangeIter); + if( test_tcl_integer(interp, SESSION_STREAM_TCL_VAR) ){ + nAlloc += nChangeset; + } + pNew = (TestChangeIter*)ckalloc(nAlloc); + memset(pNew, 0, nAlloc); + if( test_tcl_integer(interp, SESSION_STREAM_TCL_VAR) ){ + pNew->in.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR); + pNew->in.nData = nChangeset; + pNew->in.aData = (unsigned char*)&pNew[1]; + memcpy(pNew->in.aData, pChangeset, nChangeset); + } + + if( pNew->in.nStream ){ + void *pCtx = (void*)&pNew->in; + rc = sqlite3changeset_start_v2_strm(&pIter, testStreamInput, pCtx, flags); + }else{ + rc = sqlite3changeset_start_v2(&pIter, (int)nChangeset, pChangeset, flags); + } if( rc!=SQLITE_OK ){ char *zErr = sqlite3_mprintf( "error in sqlite3changeset_start_v2() - %d", rc ); Tcl_AppendResult(interp, zErr, (char*)0); + ckfree(pNew); return TCL_ERROR; } - - pNew = (TestChangeIter*)ckalloc(sizeof(TestChangeIter)); pNew->pIter = pIter; sprintf(zCmd, "csiter%d", iCmd++); diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 21f6de606..6470fd630 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -103,7 +103,7 @@ else ifeq (,$(filter $(OPTIMIZED_TARGETS),$(MAKECMDGOALS))) $(info ==============================================================) $(info == Development build. Make one of (dist, snapshot) for a) - $(info == smaller release build.) + $(info == smaller and faster release build.) $(info ==============================================================) endif endif @@ -136,8 +136,9 @@ JS_BUILD_NAMES := sqlite3 sqlite3-wasmfs # target of browsers. # # - node = for use by node.js for node.js, as opposed to by node.js on -# behalf o browser-side code (use bundler-friendly for that). Note +# behalf of browser-side code (use bundler-friendly for that). Note # that persistent storage (OPFS) is not available in these builds. +# These builds are UNTESTED and UNSUPPORTED! # JS_BUILD_MODES := vanilla esm bunder-friendly node @@ -326,7 +327,7 @@ endif # If the canonical build process finds the file # sqlite3_wasm_extra_init.c in the main wasm build directory, it # arranges to include that file in the build of sqlite3.wasm and -# defines SQLITE_EXTRA_INIT=sqlite3_wasm_extra_init. +# defines SQLITE_EXTRA_INIT_MUTEXED=sqlite3_wasm_extra_init. # # sqlite3_wasm_extra_init() must be a function with this signature: # @@ -358,7 +359,7 @@ endif # Slight caveat: this uses the version info from the in-tree # sqlite3.c/h, which may diff from a user-provided $(sqlite3.c). The # end result is that the generated JS files may have static version -# info from $(bin.version-info) which differ from their runtime-emited +# info from $(bin.version-info) which differ from their runtime-emitted # version info (e.g. from sqlite3_libversion()). bin.version-info := $(dir.top)/version-info .NOTPARALLEL: $(bin.version-info) @@ -409,7 +410,7 @@ DISTCLEAN_FILES += $(bin.stripccomments) # -D... flags which should be included in all invocations should be # appended to $(SQLITE.CALL.C-PP.FILTER.global). bin.c-pp := ./c-pp -$(bin.c-pp): c-pp.c $(sqlite3.c) $(MAKEFILE) +$(bin.c-pp): c-pp.c $(sqlite3.c) # $(MAKEFILE) $(CC) -O0 -o $@ c-pp.c $(sqlite3.c) '-DCMPP_DEFAULT_DELIM="//#"' -I$(dir.top) \ -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_UTF16 \ -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_OMIT_WAL -DSQLITE_THREADSAFE=0 \ @@ -512,7 +513,7 @@ sqlite3-api-build-version.js := $(dir.tmp)/sqlite3-api-build-version.js # sqlite3-api.jses = the list of JS files which make up # $(sqlite3-api.js.in), in the order they need to be assembled. sqlite3-api.jses := $(sqlite3-license-version.js) -# sqlite3-api-prologue.js: initial boostrapping bits: +# sqlite3-api-prologue.js: initial bootstrapping bits: sqlite3-api.jses += $(dir.api)/sqlite3-api-prologue.js # whwhasm.js and jaccwabyt.js: Low-level utils, mostly replacing # Emscripten glue: @@ -596,6 +597,9 @@ emcc.flags += -v # -v is _very_ loud but also informative about what it's doing endif + +# wasmMemory ==> required by our code for use with -sIMPORTED_MEMORY +# Emscripten 4.0.7 (2025-04-15) stops exporting HEAP* by default ######################################################################## # emcc flags for .c/.o. emcc.cflags := @@ -614,11 +618,16 @@ emcc.jsflags += -sDYNAMIC_EXECUTION=0 emcc.jsflags += -sNO_POLYFILL emcc.jsflags += -sEXPORTED_FUNCTIONS=@$(EXPORTED_FUNCTIONS.api) emcc.exportedRuntimeMethods := \ - -sEXPORTED_RUNTIME_METHODS=wasmMemory - # wasmMemory ==> required by our code for use with -sIMPORTED_MEMORY + -sEXPORTED_RUNTIME_METHODS=wasmMemory,HEAP8,HEAPU8,HEAP16,HEAPU16,HEAP32,HEAPU32,HEAP64,HEAPU64 emcc.jsflags += $(emcc.exportedRuntimeMethods) emcc.jsflags += -sUSE_CLOSURE_COMPILER=0 emcc.jsflags += -sIMPORTED_MEMORY +ifeq (,$(filter -O0,$(emcc_opt))) +emcc.assert ?= 0 +else +emcc.assert ?= 2 +endif +emcc.jsflags += -sASSERTIONS=$(emcc.assert) emcc.jsflags += -sSTRICT_JS=0 # STRICT_JS disabled due to: # https://github.com/emscripten-core/emscripten/issues/18610 @@ -885,12 +894,13 @@ EXPORTED_FUNCTIONS.fiddle := $(dir.tmp)/EXPORTED_FUNCTIONS.fiddle ######################################################################## ######################################################################## # We have to ensure that we do not build $(sqlite3*.*js) in parallel -# because they all result in the creation of $(sqlite3.wasm). We have -# no way to build just a .[m]js file without also building the .wasm -# file because the generated .[m]js file has to include info about the -# imports needed by the wasm file, so they have to be built +# for any builds which result in the creation of $(sqlite3.wasm). We +# have no way to build just a .[m]js file without also building the +# .wasm file because the generated .[m]js file has to include info +# about the imports needed by the wasm file, so they have to be built # together. i.e. we're building $(sqlite3.wasm) multiple times, but -# that's unavoidable (and harmless, just a waste of build time). +# that's unavoidable (and harmless, but is a significant waste of +# build time). $(sqlite3.wasm): $(sqlite3.js) $(sqlite3.mjs): $(sqlite3.js) $(sqlite3-bundler-friendly.mjs): $(sqlite3.mjs) @@ -901,7 +911,7 @@ $(sqlite3-node.mjs): $(sqlite3.mjs) # We need separate copies of certain supplementary JS files for the # bundler-friendly build. Concretely, any supplemental JS files which # themselves use importScripts() or Workers or URL() constructors -# which refer to other in-tree (m)JS files quire a bundler-friendly +# which refer to other in-tree (m)JS files require a bundler-friendly # copy. sqlite3-worker1.js.in := $(dir.api)/sqlite3-worker1.c-pp.js sqlite3-worker1-promiser.js.in := $(dir.api)/sqlite3-worker1-promiser.c-pp.js diff --git a/ext/wasm/api/post-js-header.js b/ext/wasm/api/post-js-header.js index a543c14f3..77e3cd227 100644 --- a/ext/wasm/api/post-js-header.js +++ b/ext/wasm/api/post-js-header.js @@ -8,16 +8,16 @@ point the sqlite3 JS API bits will get set up. */ Module.runSQLite3PostLoadInit = function(EmscriptenModule/*the Emscripten-style module object*/){ - /** ^^^ As don't use Module.postRun, as that runs a different time + /** ^^^ Don't use Module.postRun, as that runs a different time depending on whether this file is built with emcc 3.1.x or 4.0.x. This function name is intentionally obnoxiously verbose to ensure that we don't collide with current and future Emscripten symbol names. */ 'use strict'; - //console.warn("This is the start of the Module.postRun handler."); + //console.warn("This is the start of Module.runSQLite3PostLoadInit()"); /* This function will contain at least the following: - - post-js-header.js (this file) + - post-js-header.js => this file - sqlite3-api-prologue.js => Bootstrapping bits to attach the rest to - common/whwasmutil.js => Replacements for much of Emscripten's glue - jaccwabyt/jaccwabyt.js => Jaccwabyt (C/JS struct binding) @@ -26,8 +26,8 @@ Module.runSQLite3PostLoadInit = function(EmscriptenModule/*the Emscripten-style - sqlite3-api-worker1.js => Worker-based API - sqlite3-vfs-helper.c-pp.js => Utilities for VFS impls - sqlite3-vtab-helper.c-pp.js => Utilities for virtual table impls - - sqlite3-vfs-opfs.c-pp.js => OPFS VFS + - sqlite3-vfs-opfs.c-pp.js => OPFS VFS - sqlite3-vfs-opfs-sahpool.c-pp.js => OPFS SAHPool VFS - sqlite3-api-cleanup.js => final API cleanup - - post-js-footer.js => closes this postRun() function + - post-js-footer.js => closes this function */ diff --git a/ext/wasm/api/sqlite3-api-glue.c-pp.js b/ext/wasm/api/sqlite3-api-glue.c-pp.js index a40b83282..a38b9cb5e 100644 --- a/ext/wasm/api/sqlite3-api-glue.c-pp.js +++ b/ext/wasm/api/sqlite3-api-glue.c-pp.js @@ -1835,7 +1835,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ if(!(tgt instanceof StructBinder.StructType)){ toss("Usage error: target object is-not-a StructType."); }else if(!(func instanceof Function) && !wasm.isPtr(func)){ - toss("Usage errror: expecting a Function or WASM pointer to one."); + toss("Usage error: expecting a Function or WASM pointer to one."); } if(1===arguments.length){ return (n,f)=>callee(tgt, n, f, applyArgcCheck); diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 277efa14a..7e128a3fa 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -12,12 +12,12 @@ This file is intended to be combined at build-time with other related code, most notably a header and footer which wraps this - whole file into an Emscripten Module.postRun()-style handler. The - sqlite3 JS API has no hard requirements on Emscripten and does not - expose any Emscripten APIs to clients. It is structured such that - its build can be tweaked to include it in arbitrary WASM - environments which can supply the necessary underlying features - (e.g. a POSIX file I/O layer). + whole file into a single callback which can be run after Emscripten + loads the corresponding WASM module. The sqlite3 JS API has no hard + requirements on Emscripten and does not expose any Emscripten APIs + to clients. It is structured such that its build can be tweaked to + include it in arbitrary WASM environments which can supply the + necessary underlying features (e.g. a POSIX file I/O layer). Main project home page: https://sqlite.org @@ -124,7 +124,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( apiConfig = (globalThis.sqlite3ApiConfig || sqlite3ApiBootstrap.defaultConfig) ){ - if(sqlite3ApiBootstrap.sqlite3){ /* already initalized */ + if(sqlite3ApiBootstrap.sqlite3){ /* already initialized */ (sqlite3ApiBootstrap.sqlite3.config || console).warn( "sqlite3ApiBootstrap() called multiple times.", "Config and external initializers are ignored on calls after the first." @@ -195,7 +195,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( names with prefixes 'sqlite3_' and 'SQLITE_' behave, insofar as possible, identically to the C-native counterparts, as documented at: - https://www.sqlite.org/c3ref/intro.html + https://sqlite.org/c3ref/intro.html A very few exceptions require an additional level of proxy function or may otherwise require special attention in the WASM @@ -663,7 +663,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( free those pointers (using `wasm.uninstallFunction()` or equivalent). - C reference: https://www.sqlite.org/c3ref/create_function.html + C reference: https://sqlite.org/c3ref/create_function.html Maintenance reminder: the ability to add new WASM-accessible functions to the runtime requires that the @@ -869,7 +869,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( Emscripten -sWASM_BIGINT flag, else false. When enabled, certain 64-bit sqlite3 APIs are enabled which are not otherwise enabled due to JS/WASM int64 - impedence mismatches. + impedance mismatches. */ bigIntEnabled: !!config.bigIntEnabled, /** @@ -880,7 +880,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( /** When Emscripten compiles with `-sIMPORTED_MEMORY`, it - initalizes the heap and imports it into wasm, as opposed to + initializes the heap and imports it into wasm, as opposed to the other way around. In this case, the memory is not available via this.exports.memory. */ @@ -1453,7 +1453,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( creates (or overwrites) the given file using those APIs. This is primarily intended for use in Emscripten-based builds where the POSIX APIs are transparently proxied by an in-memory virtual filesystem. - It may behave diffrently in other environments. + It may behave differently in other environments. The first argument must be either a JS string or WASM C-string holding the filename. Note that this routine does _not_ create @@ -1555,7 +1555,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( - "memdb": results are undefined. - "kvvfs": will fail with an I/O error due to strict internal - requirments of that VFS's xTruncate(). + requirements of that VFS's xTruncate(). - "unix" and related: will use the WASM build's equivalent of the POSIX I/O APIs. This will work so long as neither a specific diff --git a/ext/wasm/api/sqlite3-api-worker1.c-pp.js b/ext/wasm/api/sqlite3-api-worker1.c-pp.js index 991862545..5e088f438 100644 --- a/ext/wasm/api/sqlite3-api-worker1.c-pp.js +++ b/ext/wasm/api/sqlite3-api-worker1.c-pp.js @@ -279,11 +279,11 @@ The arguments are in the same form accepted by oo1.DB.exec(), with the exceptions noted below. - If the `countChanges` arguments property (added in version 3.43) is - truthy then the `result` property contained by the returned object - will have a `changeCount` property which holds the number of changes - made by the provided SQL. Because the SQL may contain an arbitrary - number of statements, the `changeCount` is calculated by calling + If `args.countChanges` (added in version 3.43) is truthy then the + `result` property contained by the returned object will have a + `changeCount` property which holds the number of changes made by the + provided SQL. Because the SQL may contain an arbitrary number of + statements, the `changeCount` is calculated by calling `sqlite3_total_changes()` before and after the SQL is evaluated. If the value of `countChanges` is 64 then the `changeCount` property will be returned as a 64-bit integer in the form of a BigInt (noting @@ -292,6 +292,15 @@ calling `sqlite3_total_changes64()` before and after the SQL is evaluated. + If the `args.lastInsertRowId` (added in version 3.50.0) is truthy + then the `result` property contained by the returned object will + have a `lastInsertRowId` will hold a BigInt-type value corresponding + to the result of sqlite3_last_insert_rowid(). This value is only + fetched once, after the SQL is run, regardless of how many + statements the SQL contains. This API has no idea whether the SQL + contains any INSERTs, so it is up to the client to apply/rely on + this property only when it makes sense to do so. + A function-type args.callback property cannot cross the window/Worker boundary, so is not useful here. If args.callback is a string then it is assumed to be a @@ -542,6 +551,12 @@ sqlite3.initWorker1API = function(){ if(undefined !== changeCount){ rc.changeCount = db.changes(true,64===rc.countChanges) - changeCount; } + const lastInsertRowId = !!rc.lastInsertRowId + ? sqlite3.capi.sqlite3_last_insert_rowid(db) + : undefined; + if( undefined!==lastInsertRowId ){ + rc.lastInsertRowId = lastInsertRowId; + } if(rc.callback instanceof Function){ rc.callback = theCallback; /* Post a sentinel message to tell the client that the end diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index 0028c1025..e10d0dd50 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -61,7 +61,7 @@ const installAsyncProxy = function(){ } /** - Will hold state copied to this object from the syncronous side of + Will hold state copied to this object from the synchronous side of this API. */ const state = Object.create(null); @@ -722,7 +722,7 @@ const installAsyncProxy = function(){ https://github.com/tomayac/sqlite-wasm/issues/12 - is reporting that this occassionally, under high loads, + is reporting that this occasionally, under high loads, returns 'ok', which leads to the whichOp being 0 (which isn't a valid operation ID and leads to an exception, along with a corresponding ugly console log diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js index 6551b5c89..73426f203 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js @@ -79,6 +79,48 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ capi.SQLITE_OPEN_MAIN_JOURNAL | capi.SQLITE_OPEN_SUPER_JOURNAL | capi.SQLITE_OPEN_WAL; + const FLAG_COMPUTE_DIGEST_V2 = capi.SQLITE_OPEN_MEMORY + /* Part of the fix for + https://github.com/sqlite/sqlite-wasm/issues/97 + + Summary: prior to version 3.50.0 computeDigest() always computes + a value of [0,0] due to overflows, so it does not do anything + useful. Fixing it invalidates old persistent files, so we + instead only fix it for files created or updated since the bug + was discovered and fixed. + + This flag determines whether we use the broken legacy + computeDigest() or the v2 variant. We only use this flag for + newly-created/overwritten files. Pre-existing files have the + broken digest stored in them so need to continue to use that. + + What this means, in terms of db file compatibility between + versions: + + - DBs created with versions older than this fix (<3.50.0) + can be read by post-fix versions. Such DBs which are written + to in-place (not replaced) by newer versions can still be read + by older versions, as the affected digest is only modified + when the SAH slot is assigned to a given filename. + + - DBs created with post-fix versions will, when read by a pre-fix + version, be seen as having a "bad digest" and will be + unceremoniously replaced by that pre-fix version. When swapping + back to a post-fix version, that version will see that the file + entry is missing the FLAG_COMPUTE_DIGEST_V2 bit so will treat it + as a legacy file. + + This flag is stored in the same memory as the various + SQLITE_OPEN_... flags and we must be careful here to not use a + flag bit which is otherwise relevant for the VFS. + SQLITE_OPEN_MEMORY is handled by sqlite3_open_v2() and friends, + not the VFS, so we'll repurpose that one. If we take a + currently-unused bit and it ends up, at some later point, being + used, we would have to invalidate existing VFS files in order to + move to another bit. Similarly, if the SQLITE_OPEN_MEMORY bit + were ever reassigned (which it won't be!), we'd invalidate all + VFS-side files. + */; /** Subdirectory of the VFS's space where "opaque" (randomly-named) files are stored. Changing this effectively invalidates the data @@ -329,6 +371,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){ const pool = getPoolForVfs(pVfs); try{ + flags &= ~FLAG_COMPUTE_DIGEST_V2; pool.log(`xOpen ${wasm.cstrToJs(zName)} ${flags}`); // First try to open a path that already exists in the file system. const path = (zName && wasm.peek8(zName)) @@ -501,22 +544,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ currently-opened client-specified filenames. */ getFileNames(){ const rc = []; - const iter = this.#mapFilenameToSAH.keys(); - for(const n of iter) rc.push(n); + for(const n of this.#mapFilenameToSAH.keys()) rc.push(n); return rc; } -// #createFileObject(sah,clientName,opaqueName){ -// const f = Object.assign(Object.create(null),{ -// clientName, opaqueName -// }); -// this.#mapSAHToMeta.set(sah, f); -// return f; -// } -// #unmapFileObject(sah){ -// this.#mapSAHToMeta.delete(sah); -// } - /** Adds n files to the pool's capacity. This change is persistent across settings. Returns a Promise which resolves @@ -557,8 +588,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } /** - Releases all currently-opened SAHs. The only legal - operation after this is acquireAccessHandles(). + Releases all currently-opened SAHs. The only legal operation + after this is acquireAccessHandles() or (if this is called from + pauseVfs()) either of isPaused() or unpauseVfs(). */ releaseAccessHandles(){ for(const ah of this.#mapSAHToName.keys()) ah.close(); @@ -568,17 +600,21 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } /** - Opens all files under this.vfsDir/this.#dhOpaque and acquires - a SAH for each. returns a Promise which resolves to no value - but completes once all SAHs are acquired. If acquiring an SAH - throws, SAHPool.$error will contain the corresponding - exception. + Opens all files under this.vfsDir/this.#dhOpaque and acquires a + SAH for each. Returns a Promise which resolves to no value but + completes once all SAHs are acquired. If acquiring an SAH + throws, this.$error will contain the corresponding Error + object. + + If it throws, it releases any SAHs which it may have + acquired before the exception was thrown, leaving the VFS in a + well-defined but unusable state. If clearFiles is true, the client-stored state of each file is cleared when its handle is acquired, including its name, flags, and any data stored after the metadata block. */ - async acquireAccessHandles(clearFiles){ + async acquireAccessHandles(clearFiles=false){ const files = []; for await (const [name,h] of this.#dhOpaque){ if('file'===h.kind){ @@ -631,7 +667,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const fileDigest = new Uint32Array(HEADER_DIGEST_SIZE / 4); sah.read(fileDigest, {at: HEADER_OFFSET_DIGEST}); - const compDigest = this.computeDigest(this.#apBody); + const compDigest = this.computeDigest(this.#apBody, flags); + //warn("getAssociatedPath() flags",'0x'+flags.toString(16), "compDigest", compDigest); if(fileDigest.every((v,i) => v===compDigest[i])){ // Valid digest const pathBytes = this.#apBody.findIndex((v)=>0===v); @@ -640,6 +677,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ // leaving stale db data laying around. sah.truncate(HEADER_OFFSET_DATA); } + //warn("getAssociatedPath() flags",'0x'+flags.toString(16), "compDigest", compDigest,"pathBytes",pathBytes); return pathBytes ? textDecoder.decode(this.#apBody.subarray(0,pathBytes)) : ''; @@ -662,10 +700,17 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ if(HEADER_MAX_PATH_SIZE <= enc.written + 1/*NUL byte*/){ toss("Path too long:",path); } + if(path && flags){ + /* When creating or re-writing files, update their digest, if + needed, to v2. We continue to use v1 for the (!path) case + (empty files) because there's little reason not to use a + digest of 0 for empty entries. */ + flags |= FLAG_COMPUTE_DIGEST_V2; + } this.#apBody.fill(0, enc.written, HEADER_MAX_PATH_SIZE); this.#dvBody.setUint32(HEADER_OFFSET_FLAGS, flags); - - const digest = this.computeDigest(this.#apBody); + const digest = this.computeDigest(this.#apBody, flags); + //console.warn("setAssociatedPath(",path,") digest",digest); sah.write(this.#apBody, {at: 0}); sah.write(digest, {at: HEADER_OFFSET_DIGEST}); sah.flush(); @@ -686,15 +731,22 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ metadata for each file as a validation check. Changing this algorithm invalidates all existing databases for this VFS, so don't do that. + + See the docs for FLAG_COMPUTE_DIGEST_V2 for more details. */ - computeDigest(byteArray){ - let h1 = 0xdeadbeef; - let h2 = 0x41c6ce57; - for(const v of byteArray){ - h1 = 31 * h1 + (v * 307); - h2 = 31 * h2 + (v * 307); + computeDigest(byteArray, fileFlags){ + if( fileFlags & FLAG_COMPUTE_DIGEST_V2 ){ + let h1 = 0xdeadbeef; + let h2 = 0x41c6ce57; + for(const v of byteArray){ + h1 = Math.imul(h1 ^ v, 2654435761); + h2 = Math.imul(h2 ^ v, 104729); + } + return new Uint32Array([h1>>>0, h2>>>0]); + }else{ + /* this is what the buggy legacy computation worked out to */ + return new Uint32Array([0,0]); } - return new Uint32Array([h1>>>0, h2>>>0]); } /** @@ -832,12 +884,18 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Removes this object's sqlite3_vfs registration and shuts down this object, releasing all handles, mappings, and whatnot, including deleting its data directory. There is currently no - way to "revive" the object and reaquire its resources. + way to "revive" the object and reaquire its + resources. Similarly, there is no recovery strategy if removal + of any given SAH fails, so such errors are ignored by this + function. This function is intended primarily for testing. Resolves to true if it did its job, false if the VFS has already been shut down. + + @see pauseVfs() + @see unpauseVfs() */ async removeVfs(){ if(!this.#cVfs.pointer || !this.#dhOpaque) return false; @@ -853,13 +911,77 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ); this.#dhVfsRoot = this.#dhVfsParent = undefined; }catch(e){ - sqlite3.config.error(this.vfsName,"removeVfs() failed:",e); + sqlite3.config.error(this.vfsName,"removeVfs() failed with no recovery strategy:",e); /*otherwise ignored - there is no recovery strategy*/ } return true; } + /** + "Pauses" this VFS by unregistering it from SQLite and + relinquishing all open SAHs, leaving the associated files + intact. If this object is already paused, this is a + no-op. Returns this object. + + This function throws if SQLite has any opened file handles + hosted by this VFS, as the alternative would be to invoke + Undefined Behavior by closing file handles out from under the + library. Similarly, automatically closing any database handles + opened by this VFS would invoke Undefined Behavior in + downstream code which is holding those pointers. + + If this function throws due to open file handles then it has + no side effects. If the OPFS API throws while closing handles + then the VFS is left in an undefined state. + + @see isPaused() + @see unpauseVfs() + */ + pauseVfs(){ + if(this.#mapS3FileToOFile_.size>0){ + sqlite3.SQLite3Error.toss( + capi.SQLITE_MISUSE, "Cannot pause VFS", + this.vfsName,"because it has opened files." + ); + } + if(this.#mapSAHToName.size>0){ + capi.sqlite3_vfs_unregister(this.vfsName); + this.releaseAccessHandles(); + } + return this; + } + + /** + Returns true if this pool is currently paused else false. + + @see pauseVfs() + @see unpauseVfs() + */ + isPaused(){ + return 0===this.#mapSAHToName.size; + } + + /** + "Unpauses" this VFS, reacquiring all SAH's and (if successful) + re-registering it with SQLite. This is a no-op if the VFS is + not currently paused. + + The returned Promise resolves to this object. See + acquireAccessHandles() for how it behaves if it throws due to + SAH acquisition failure. + + @see isPaused() + @see pauseVfs() + */ + async unpauseVfs(){ + if(0===this.#mapSAHToName.size){ + return this.acquireAccessHandles(false). + then(()=>capi.sqlite3_vfs_register(this.#cVfs, 0),this); + } + return this; + } + //! Documented elsewhere in this file. exportFile(name){ const sah = this.#mapFilenameToSAH.get(name) || toss("File not found:",name); @@ -984,6 +1106,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ async removeVfs(){ return this.#p.removeVfs() } + pauseVfs(){ this.#p.pauseVfs(); return this; } + async unpauseVfs(){ return this.#p.unpauseVfs().then(()=>this); } + isPaused(){ return this.#p.isPaused() } + }/* class OpfsSAHPoolUtil */; /** @@ -1038,7 +1164,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - `clearOnInit`: (default=false) if truthy, contents and filename mapping are removed from each SAH it is acquired during - initalization of the VFS, leaving the VFS's storage in a pristine + initialization of the VFS, leaving the VFS's storage in a pristine state. Use this only for databases which need not survive a page reload. @@ -1165,7 +1291,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ VFS-hosted database file. The result of the resolved Promise when called this way is the size of the resulting database. - On succes this routine rewrites the database header bytes in the + On success this routine rewrites the database header bytes in the output file (not the input array) to force disabling of WAL mode. On a write error, the handle is removed from the pool and made @@ -1217,6 +1343,41 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Clears all client-defined state of all SAHs and makes all of them available for re-use by the pool. Results are undefined if any such handles are currently in use, e.g. by an sqlite3 db. + + APIs specific to the "pause" capability (added in version 3.49): + + Summary: "pausing" the VFS disassociates it from SQLite and + relinquishes its SAHs so that they may be opened by another + instance of this VFS (running in a separate tab/page or Worker). + "Unpausing" it takes back control, if able. + + - pauseVfs() + + "Pauses" this VFS by unregistering it from SQLite and + relinquishing all open SAHs, leaving the associated files intact. + This enables pages/tabs to coordinate semi-concurrent usage of + this VFS. If this object is already paused, this is a + no-op. Returns this object. Throws if SQLite has any opened file + handles hosted by this VFS. If this function throws due to open + file handles then it has no side effects. If the OPFS API throws + while closing handles then the VFS is left in an undefined state. + + - isPaused() + + Returns true if this VFS is paused, else false. + + - [async] unpauseVfs() + + Restores the VFS to an active state after having called + pauseVfs() on it. This is a no-op if the VFS is not paused. The + returned Promise resolves to this object on success. A rejected + Promise means there was a problem reacquiring the SAH handles + (possibly because they're in use by another instance or have + since been removed). Generically speaking, there is no recovery + strategy for that type of error, but if the problem is simply + that the OPFS files are locked, then a later attempt to unpause + it, made after the concurrent instance releases the SAHs, may + recover from the situation. */ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ options = Object.assign(Object.create(null), optionDefaults, (options||{})); diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 27f1bfae7..9cacae788 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -459,7 +459,7 @@ const installOpfsVfs = function callee(options){ Runs the given operation (by name) in the async worker counterpart, waits for its response, and returns the result which the async worker writes to SAB[state.opIds.rc]. The - 2nd and subsequent arguments must be the aruguments for the + 2nd and subsequent arguments must be the arguments for the async op. */ const opRun = (op,...args)=>{ diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 461afe066..1850d313c 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -128,7 +128,9 @@ #endif #ifdef SQLITE_WASM_EXTRA_INIT -# define SQLITE_EXTRA_INIT sqlite3_wasm_extra_init +/* SQLITE_EXTRA_INIT vs SQLITE_EXTRA_INIT_MUTEXED: +** see https://sqlite.org/forum/forumpost/14183b98fc0b1dea */ +# define SQLITE_EXTRA_INIT_MUTEXED sqlite3_wasm_extra_init #endif /* @@ -284,7 +286,7 @@ SQLITE_WASM_EXPORT void * sqlite3__wasm_stack_alloc(int n){ /* ** State for the "pseudo-stack" allocator implemented in ** sqlite3__wasm_pstack_xyz(). In order to avoid colliding with -** Emscripten-controled stack space, it carves out a bit of stack +** Emscripten-controlled stack space, it carves out a bit of stack ** memory to use for that purpose. This memory ends up in the ** WASM-managed memory, such that routines which manipulate the wasm ** heap can also be used to manipulate this memory. @@ -311,7 +313,7 @@ SQLITE_WASM_EXPORT void * sqlite3__wasm_pstack_ptr(void){ return PStack.pPos; } /* -** Sets the pstack position poitner to p. Results are undefined if the +** Sets the pstack position pointer to p. Results are undefined if the ** given value did not come from sqlite3__wasm_pstack_ptr(). */ SQLITE_WASM_EXPORT void sqlite3__wasm_pstack_restore(unsigned char * p){ @@ -980,7 +982,7 @@ const char * sqlite3__wasm_enum_json(void){ #undef _DefGroup /* - ** Emit an array of "StructBinder" struct descripions, which look + ** Emit an array of "StructBinder" struct descriptions, which look ** like: ** ** { @@ -1155,7 +1157,7 @@ const char * sqlite3__wasm_enum_json(void){ { /* Validate that the above struct sizeof()s match ** expectations. We could improve upon this by ** checking the offsetof() for each member. */ - const sqlite3_index_info siiCheck; + const sqlite3_index_info siiCheck = {0}; #define IndexSzCheck(T,M) \ (sizeof(T) == sizeof(*siiCheck.M)) if(!IndexSzCheck(sqlite3_index_constraint,aConstraint) @@ -1417,7 +1419,7 @@ int sqlite3__wasm_db_serialize( sqlite3 *pDb, const char *zSchema, ** NULL), or nData is negative, SQLITE_MISUSE are returned. ** ** On success, it creates a new file with the given name, populated -** with the fist nData bytes of pData. If pData is NULL, the file is +** with the first nData bytes of pData. If pData is NULL, the file is ** created and/or truncated to nData bytes. ** ** Whether or not directory components of zFilename are created diff --git a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js index 55e497ead..c043fd148 100644 --- a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js +++ b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js @@ -335,8 +335,8 @@ sqlite3Worker1Promiser.v2 = function(config){ /** When built as a module, we export sqlite3Worker1Promiser.v2() instead of sqlite3Worker1Promise() because (A) its interface is more - conventional for ESM usage and (B) the ESM option export option for - this API did not exist until v2 was created, so there's no backwards + conventional for ESM usage and (B) the ESM export option for this + API did not exist until v2 was created, so there's no backwards incompatibility. */ export default sqlite3Worker1Promiser.v2; diff --git a/ext/wasm/c-pp.c b/ext/wasm/c-pp.c index c67881dd3..318325e93 100644 --- a/ext/wasm/c-pp.c +++ b/ext/wasm/c-pp.c @@ -36,7 +36,7 @@ ** - `#pragma` is in place for adding "meta-commands", but it does not ** yet have any concrete list of documented commands. ** -* - `#stderr` outputs its file name, line number, and the remaininder +* - `#stderr` outputs its file name, line number, and the remainder ** of that line to stderr. ** ** - `#//` acts as a single-line comment, noting that there must be as @@ -1215,7 +1215,7 @@ static void cmpp_kwd_if(CmppKeyword const * pKw, CmppTokenizer *t){ CmppLevel_push(t); break; default: - cmpp_kwd__misuse(pKw, t, "Unpexected keyword token type"); + cmpp_kwd__misuse(pKw, t, "Unexpected keyword token type"); break; } buul = db_define_has((char const *)t->args.argv[1]); diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js index 509d33b37..b4d8f691b 100644 --- a/ext/wasm/common/whwasmutil.js +++ b/ext/wasm/common/whwasmutil.js @@ -798,7 +798,7 @@ globalThis.WhWasmUtilInstaller = function(target){ */ target.peek8 = (...ptr)=>target.peek( (1===ptr.length ? ptr[0] : ptr), 'i8' ); /** - Convience form of poke() intended for setting individual bytes. + Convenience form of poke() intended for setting individual bytes. Its difference from poke() is that it always writes to the i8-sized heap view. */ @@ -1769,7 +1769,7 @@ globalThis.WhWasmUtilInstaller = function(target){ /** If true, the constructor emits a warning. The intent is that this be set to true after bootstrapping of the higher-level client library is complete, to warn downstream clients that - they shouldn't be relying on this implemenation detail which + they shouldn't be relying on this implementation detail which does not have a stable interface. */ xArg.FuncPtrAdapter.warnOnUse = false; @@ -1949,7 +1949,7 @@ globalThis.WhWasmUtilInstaller = function(target){ - `json:dealloc` (results): works exactly like `string:dealloc` but returns the same thing as the `json` adapter. Note the - warning in `string:dealloc` regarding maching allocators and + warning in `string:dealloc` regarding matching allocators and deallocators. The type names for results and arguments are validated when diff --git a/ext/wasm/demo-jsstorage.js b/ext/wasm/demo-jsstorage.js index cf820e403..587aa9cc5 100644 --- a/ext/wasm/demo-jsstorage.js +++ b/ext/wasm/demo-jsstorage.js @@ -103,7 +103,7 @@ if(0===db.selectValue('select count(*) from sqlite_master')){ log("DB is empty. Use the init button to populate it."); }else{ - log("DB contains data from a previous session. Use the Clear Ctorage button to delete it."); + log("DB contains data from a previous session. Use the Clear Storage button to delete it."); btnSelect.click(); } }; diff --git a/ext/wasm/demo-worker1-promiser.c-pp.js b/ext/wasm/demo-worker1-promiser.c-pp.js index f6fc9568a..0b8557b82 100644 --- a/ext/wasm/demo-worker1-promiser.c-pp.js +++ b/ext/wasm/demo-worker1-promiser.c-pp.js @@ -115,6 +115,7 @@ delete globalThis.sqlite3Worker1Promiser; "insert into t(a,b) values(1,2),(3,4),(5,6)" ].join(';'), resultRows: [], columnNames: [], + lastInsertRowId: true, countChanges: sqConfig.bigIntEnabled ? 64 : true }, function(ev){ ev = ev.result; @@ -122,7 +123,9 @@ delete globalThis.sqlite3Worker1Promiser; .assert(0===ev.columnNames.length) .assert(sqConfig.bigIntEnabled ? (3n===ev.changeCount) - : (3===ev.changeCount)); + : (3===ev.changeCount)) + .assert('bigint'===typeof ev.lastInsertRowId) + .assert(ev.lastInsertRowId>=3); }); await wtest('exec',{ diff --git a/ext/wasm/demo-worker1.js b/ext/wasm/demo-worker1.js index 60f5e8dec..1a05cc7ac 100644 --- a/ext/wasm/demo-worker1.js +++ b/ext/wasm/demo-worker1.js @@ -156,11 +156,14 @@ sql: ["create table t(a,b);", "insert into t(a,b) values(1,2),(3,4),(5,6)" ], + lastInsertRowId: true, resultRows: [], columnNames: [] }, function(ev){ ev = ev.result; T.assert(0===ev.resultRows.length) - .assert(0===ev.columnNames.length); + .assert(0===ev.columnNames.length) + .assert('bigint'===typeof ev.lastInsertRowId) + .assert(ev.lastInsertRowId>=3); }); runOneTest('exec',{ sql: 'select a a, b b from t order by a', diff --git a/ext/wasm/fiddle.make b/ext/wasm/fiddle.make index 2a43959e2..8110384a6 100644 --- a/ext/wasm/fiddle.make +++ b/ext/wasm/fiddle.make @@ -91,7 +91,7 @@ all: fiddle fiddle_remote ?= ifeq (,$(fiddle_remote)) ifneq (,$(wildcard /home/stephan)) - fiddle_remote = wh:www/wh/sqlite3/. + fiddle_remote = wh:www/wasm-testing/fiddle/. else ifneq (,$(wildcard /home/drh)) #fiddle_remote = if appropriate, add that user@host:/path here endif diff --git a/ext/wasm/fiddle/fiddle.js b/ext/wasm/fiddle/fiddle.js index d28589835..f0a89f25d 100644 --- a/ext/wasm/fiddle/fiddle.js +++ b/ext/wasm/fiddle/fiddle.js @@ -112,7 +112,7 @@ /** A proxy for localStorage or sessionStorage or a - page-instance-local proxy, if neither one is availble. + page-instance-local proxy, if neither one is available. Which exact storage implementation is uses is unspecified, and apps must not rely on it. diff --git a/ext/wasm/fiddle/index.html b/ext/wasm/fiddle/index.html index f77974931..ca6788ef0 100644 --- a/ext/wasm/fiddle/index.html +++ b/ext/wasm/fiddle/index.html @@ -5,7 +5,7 @@ SQLite3 Fiddle - diff --git a/ext/wasm/index.html b/ext/wasm/index.html index a3d41f1a9..fb8a58560 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -119,6 +119,10 @@
  10. OPFS concurrency tests using multiple workers.
  11. +
  12. OPFS SAHPool cooperative semi-concurrency + demonstrates usage of the OPFS SAHPool VFS's "pause" feature to coordinate + access to a database. +
  13. WASMFS-specific tests which require that diff --git a/ext/wasm/jaccwabyt/jaccwabyt.js b/ext/wasm/jaccwabyt/jaccwabyt.js index 1846441e5..8144e8d62 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.js +++ b/ext/wasm/jaccwabyt/jaccwabyt.js @@ -561,7 +561,7 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ const dbg = ctor.prototype.debugFlags.__flags; /* TODO?: set prototype of descr to an object which can set/fetch - its prefered representation, e.g. conversion to string or mapped + its preferred representation, e.g. conversion to string or mapped function. Advantage: we can avoid doing that via if/else if/else in the get/set methods. */ diff --git a/ext/wasm/jaccwabyt/jaccwabyt.md b/ext/wasm/jaccwabyt/jaccwabyt.md index 431741edc..17bba78cc 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.md +++ b/ext/wasm/jaccwabyt/jaccwabyt.md @@ -176,7 +176,7 @@ essentially boils down to: 3. [Feed (2) to the function generated by (1)](#step-3) to create JS constuctor functions for each struct. This is done at runtime, as opposed to during a build-process step, and can be set up in such a - way that it does not require any maintenace after its initial + way that it does not require any maintenance after its initial setup. 4. [Create and use instances of those structs](#step-4). @@ -950,7 +950,7 @@ const char * wasm__ctype_json(void){ assert(LEVEL<5); memset(pos, '}', LEVEL); pos+=LEVEL; lenCheck //////////////////////////////////////////////////////////////////// - // Macros for emiting StructBinders... + // Macros for emitting StructBinders... #define StructBinder__(TYPE) \ n = 0; \ outf("%s{", (structCount++ ? ", " : "")); \ diff --git a/ext/wasm/mkwasmbuilds.c b/ext/wasm/mkwasmbuilds.c index 91c03b6d4..8aa29c0fe 100644 --- a/ext/wasm/mkwasmbuilds.c +++ b/ext/wasm/mkwasmbuilds.c @@ -45,6 +45,7 @@ ** "sqlite3-wasmfs" build, only "esm" (ES6 Module) is legal. */ #define JS_BUILD_MODES vanilla esm bundler-friendly node +/* Separator to help eyeballs find the different sections */ static const char * zBanner = "\n########################################################################\n"; @@ -135,6 +136,28 @@ static void mk_prologue(void){ } } +/* +** Flags for use with the 3rd argument to mk_pre_post() and +** mk_lib_mode(). +** +** Maintenance reminder: do not combine flags within this enum, +** e.g. LIBMODE_BUNDLER_FRIENDLY=0x02|LIBMODE_ESM, as that will lead +** to breakage in some of the flag checks. +*/ +enum LibModeFlags { + /* Indicates an ESM module build. */ + LIBMODE_ESM = 0x01, + /* Indicates a "bundler-friendly" build mode. */ + LIBMODE_BUNDLER_FRIENDLY = 0x02, + /* Indicates to _not_ add this build to the 'all' target. */ + LIBMODE_DONT_ADD_TO_ALL = 0x04, + /* Indicates a node.js-for-node.js build (untested and + ** unsupported). */ + LIBMODE_NODEJS = 0x08, + /* Indicates a wasmfs build (untested and unsupported). */ + LIBMODE_WASMFS = 0x10 +}; + /* ** Emits makefile code for setting up values for the --pre-js=FILE, ** --post-js=FILE, and --extern-post-js=FILE emcc flags, as well as @@ -142,6 +165,7 @@ static void mk_prologue(void){ */ static void mk_pre_post(const char *zName /* build name */, const char *zMode /* build mode */, + int flags /* LIBMODE_... mask */, const char *zCmppD /* optional -D flags for c-pp for the ** --pre/--post-js files. */){ pf("%s# Begin --pre/--post flags for %s-%s\n", zBanner, zNM); @@ -166,9 +190,10 @@ static void mk_pre_post(const char *zName /* build name */, pf("\tcp $(pre-js.js.%s-%s.intermediary) $@\n", zNM); /* Amend $(pre-js.js.zName-zMode) for all targets except the plain - ** "sqlite3" build... */ + ** "sqlite3" and the "sqlite3-wasmfs" builds... */ if( 0!=strcmp("sqlite3-wasmfs", zName) && 0!=strcmp("sqlite3", zName) ){ +#error "This part ^^^ is needs adapting for use with the LIBMODE_... flags" pf("\t@echo 'Module[xNameOfInstantiateWasm].uri = " "\"%s.wasm\";' >> $@\n", zName); } @@ -184,7 +209,7 @@ static void mk_pre_post(const char *zName /* build name */, pf("$(eval $(call SQLITE.CALL.C-PP.FILTER,$(extern-post-js.js.in),$(extern-post-js.js.%s-%s)," "$(c-pp.D.%s-%s)))\n", zNM, zNM); - /* Combine flags for use with emcc... */ + /* Combined flags for use with emcc... */ pf("pre-post-common.flags.%s-%s := " "$(pre-post-common.flags) " "--post-js=$(post-js.js.%s-%s) " @@ -209,7 +234,7 @@ static void mk_pre_post(const char *zName /* build name */, static void mk_fiddle(){ int i = 0; - mk_pre_post("fiddle-module","vanilla", 0); + mk_pre_post("fiddle-module","vanilla", 0, 0); for( ; i < 2; ++i ){ const char *zTail = i ? ".debug" : ""; const char *zDir = i ? "$(dir.fiddle-debug)" : "$(dir.fiddle)"; @@ -257,7 +282,7 @@ static void mk_fiddle(){ */ static void mk_lib_mode(const char *zName /* build name */, const char *zMode /* build mode */, - int bIsEsm /* true only for ESM build */, + int flags /* LIBMODE_... mask */, const char *zApiJsOut /* name of generated sqlite3-api.js/.mjs */, const char *zJsOut /* name of generated sqlite3.js/.mjs */, const char *zCmppD /* extra -D flags for c-pp */, @@ -276,7 +301,7 @@ static void mk_lib_mode(const char *zName /* build name */, pf("%s# Begin build [%s-%s]\n", zBanner, zNM); pf("# zApiJsOut=%s\n# zJsOut=%s\n# zCmppD=%s\n", zApiJsOut, zJsOut, zCmppD); pf("$(info Setting up build [%s-%s]: %s)\n", zNM, zJsOut); - mk_pre_post(zNM, zCmppD); + mk_pre_post(zNM, flags, zCmppD); pf("\nemcc.flags.%s.%s ?=\n", zNM); if( zEmcc[0] ){ pf("emcc.flags.%s.%s += %s\n", zNM, zEmcc); @@ -303,13 +328,13 @@ static void mk_lib_mode(const char *zName /* build name */, pf("\t\t$(cflags.common) $(SQLITE_OPT) \\\n" "\t\t$(cflags.%s) $(cflags.%s.%s) \\\n" "\t\t$(cflags.wasm_extra_init) $(sqlite3-wasm.cfiles)\n", zName, zNM); - if( bIsEsm ){ + if( (LIBMODE_ESM & flags) || (LIBMODE_NODEJS & flags) ){ /* TODO? Replace this $(call) with the corresponding makefile ** code. OTOH, we also use this $(call) in the speedtest1-wasmfs ** build, which is not part of the rules emitted by this ** program. */ pf("\t@$(call SQLITE.CALL.xJS.ESM-EXPORT-DEFAULT,1,%d)\n", - 0==strcmp("sqlite3-wasmfs", zName) ? 1 : 0); + (LIBMODE_WASMFS & flags) ? 1 : 0); } pf("\t@chmod -x %s; \\\n" "\t\t$(maybe-wasm-strip) %s;\n", @@ -331,16 +356,16 @@ static void mk_lib_mode(const char *zName /* build name */, ** resulting .wasm file is identical for all builds for which zEmcc ** is empty. */ - if( 0==strcmp("bundler-friendly", zMode) - || 0==strcmp("node", zMode) ){ + if( (LIBMODE_BUNDLER_FRIENDLY & flags) + || (LIBMODE_NODEJS & flags) ){ pf("\t@echo 'Patching $@ for %s.wasm...'; \\\n", zName); pf("\t\trm -f %s; \\\n", zWasmOut); pf("\t\tsed -i -e 's/%s-%s.wasm/%s.wasm/g' $@ || exit;\n", /* ^^^^^^ reminder: Mac/BSD sed has no -i flag */ zNM, zName); pf("\t@ls -la $@\n"); - if( 0==strcmp("bundler-friendly", zMode) ){ - /* Avoid a 3rd occurance of the bug fixed by 65798c09a00662a3, + if( LIBMODE_BUNDLER_FRIENDLY & flags ){ + /* Avoid a 3rd occurrence of the bug fixed by 65798c09a00662a3, ** which was (in two cases) caused by makefile refactoring and ** not recognized until after a release was made with the broken ** sqlite3-bundler-friendly.mjs: */ @@ -352,9 +377,7 @@ static void mk_lib_mode(const char *zName /* build name */, }else{ pf("\t@ls -la %s $@\n", zWasmOut); } - if( 0!=strcmp("sqlite3-wasmfs", zName) ){ - /* The sqlite3-wasmfs build is optional and needs to be invoked - ** conditionally using info we don't have here. */ + if( 0==(LIBMODE_DONT_ADD_TO_ALL & flags) ){ pf("all: %s\n", zJsOut); } pf("# End build [%s-%s]%s", zNM, zBanner); @@ -366,22 +389,29 @@ int main(void){ mk_prologue(); mk_lib_mode("sqlite3", "vanilla", 0, "$(sqlite3-api.js)", "$(sqlite3.js)", 0, 0); - mk_lib_mode("sqlite3", "esm", 1, + mk_lib_mode("sqlite3", "esm", LIBMODE_ESM, "$(sqlite3-api.mjs)", "$(sqlite3.mjs)", "-Dtarget=es6-module", 0); - mk_lib_mode("sqlite3", "bundler-friendly", 1, - "$(sqlite3-api-bundler-friendly.mjs)", "$(sqlite3-bundler-friendly.mjs)", + mk_lib_mode("sqlite3", "bundler-friendly", + LIBMODE_BUNDLER_FRIENDLY | LIBMODE_ESM, + "$(sqlite3-api-bundler-friendly.mjs)", + "$(sqlite3-bundler-friendly.mjs)", "$(c-pp.D.sqlite3-esm) -Dtarget=es6-bundler-friendly", 0); - mk_lib_mode("sqlite3" , "node", 1, + mk_lib_mode("sqlite3" , "node", + LIBMODE_NODEJS | LIBMODE_DONT_ADD_TO_ALL, "$(sqlite3-api-node.mjs)", "$(sqlite3-node.mjs)", "$(c-pp.D.sqlite3-bundler-friendly) -Dtarget=node", 0); - mk_lib_mode("sqlite3-wasmfs", "esm" ,1, + mk_lib_mode("sqlite3-wasmfs", "esm" , + LIBMODE_WASMFS | LIBMODE_ESM | LIBMODE_DONT_ADD_TO_ALL, + /* The sqlite3-wasmfs build is optional and needs to be invoked + ** conditionally using info we don't have here. */ "$(sqlite3-api-wasmfs.mjs)", "$(sqlite3-wasmfs.mjs)", "$(c-pp.D.sqlite3-bundler-friendly) -Dwasmfs", "-sEXPORT_ES6 -sUSE_ES6_IMPORT_META"); mk_fiddle(); - mk_pre_post("speedtest1","vanilla", 0); - mk_pre_post("speedtest1-wasmfs","esm", "$(c-pp.D.sqlite3-bundler-friendly) -Dwasmfs"); + mk_pre_post("speedtest1","vanilla", 0, 0); + mk_pre_post("speedtest1-wasmfs","esm", 0, + "$(c-pp.D.sqlite3-bundler-friendly) -Dwasmfs"); return rc; } diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 28d61de07..d30e59e38 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -3189,8 +3189,25 @@ globalThis.sqlite3InitModule = sqlite3InitModule; db.close(); T.assert(1 === u1.getFileCount()); db = new u2.OpfsSAHPoolDb(dbName); - T.assert(1 === u1.getFileCount()); + T.assert(1 === u1.getFileCount()) + .mustThrowMatching( + ()=>u1.pauseVfs(), + (err)=>{ + return capi.SQLITE_MISUSE===err.resultCode + && /^SQLITE_MISUSE: Cannot pause VFS /.test(err.message); + }, + "Cannot pause VFS with opened db." + ); db.close(); + T.assert( u2===u2.pauseVfs() ) + .assert( u2.isPaused() ) + .assert( 0===capi.sqlite3_vfs_find(u2.vfsName) ) + .mustThrowMatching(()=>new u2.OpfsSAHPoolDb(dbName), + /.+no such vfs: .+/, + "VFS is not available") + .assert( u2===await u2.unpauseVfs() ) + .assert( u2===await u1.unpauseVfs(), "unpause is a no-op if the VFS is not paused" ) + .assert( 0!==capi.sqlite3_vfs_find(u2.vfsName) ); const fileNames = u1.getFileNames(); T.assert(1 === fileNames.length) .assert(dbName === fileNames[0]) @@ -3489,7 +3506,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; }); db.exec([ "create table t(a);", - "insert into t(a) values(1),(2),(3);", + "insert into t(a) values(1),(2),(1);", "select auxtest(1,a), auxtest(1,a) from t order by a" ]); }finally{ diff --git a/ext/wasm/tests/opfs/sahpool/digest-worker.js b/ext/wasm/tests/opfs/sahpool/digest-worker.js new file mode 100644 index 000000000..28b3c1673 --- /dev/null +++ b/ext/wasm/tests/opfs/sahpool/digest-worker.js @@ -0,0 +1,94 @@ +/* + 2025-01-31 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file is part of testing the OPFS SAHPool VFS's computeDigest() + fix. See ./digest.html for the details. +*/ +const clog = console.log.bind(console); +const wPost = (type,...args)=>postMessage({type, payload:args}); +const log = (...args)=>{ + clog("Worker:",...args); + wPost('log',...args); +} + +const hasOpfs = ()=>{ + return globalThis.FileSystemHandle + && globalThis.FileSystemDirectoryHandle + && globalThis.FileSystemFileHandle + && globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle + && navigator?.storage?.getDirectory; +}; +if( !hasOpfs() ){ + wPost('error',"OPFS not detected"); + throw new Error("OPFS not detected"); +} + +clog("Importing sqlite3..."); +const searchParams = new URL(self.location.href).searchParams; +importScripts(searchParams.get('sqlite3.dir') + '/sqlite3.js'); + +const runTests = function(sqlite3, poolUtil){ + const fname = '/my.db'; + let db = new poolUtil.OpfsSAHPoolDb(fname); + let n = (new Date()).valueOf(); + try { + db.exec([ + "create table if not exists t(a);" + ]); + db.exec({ + sql: "insert into t(a) values(?)", + bind: n++ + }); + log(fname,"record count: ",db.selectValue("select count(*) from t")); + }finally{ + db.close(); + } + + db = new poolUtil.OpfsSAHPoolDb(fname); + try { + db.exec({ + sql: "insert into t(a) values(?)", + bind: n++ + }); + log(fname,"record count: ",db.selectValue("select count(*) from t")); + }finally{ + db.close(); + } + + const fname2 = '/my2.db'; + db = new poolUtil.OpfsSAHPoolDb(fname2); + try { + db.exec([ + "create table if not exists t(a);" + ]); + db.exec({ + sql: "insert into t(a) values(?)", + bind: n++ + }); + log(fname2,"record count: ",db.selectValue("select count(*) from t")); + }finally{ + db.close(); + } +}; + +globalThis.sqlite3InitModule().then(async function(sqlite3){ + log("sqlite3 version:",sqlite3.version); + const sahPoolConfig = { + name: 'opfs-sahpool-digest', + clearOnInit: false, + initialCapacity: 6 + }; + return sqlite3.installOpfsSAHPoolVfs(sahPoolConfig).then(poolUtil=>{ + log('vfs acquired'); + runTests(sqlite3, poolUtil); + }); +}); diff --git a/ext/wasm/tests/opfs/sahpool/digest.html b/ext/wasm/tests/opfs/sahpool/digest.html new file mode 100644 index 000000000..daa1f7728 --- /dev/null +++ b/ext/wasm/tests/opfs/sahpool/digest.html @@ -0,0 +1,151 @@ + + + + + + + + + sqlite3 tester: OpfsSAHPool Digest + + +

    + +

    + This is a test app for the digest calculation of the OPFS + SAHPool VFS. It requires running it with a new database created using + v3.49.0 or older, then running it again with a newer version, then + again with 3.49.0 or older. +

    +
    + + +
    +
    + + + diff --git a/ext/wasm/tests/opfs/sahpool/index.html b/ext/wasm/tests/opfs/sahpool/index.html new file mode 100644 index 000000000..f3d07f456 --- /dev/null +++ b/ext/wasm/tests/opfs/sahpool/index.html @@ -0,0 +1,31 @@ + + + + + + + + + sqlite3 tester: OpfsSAHPool Pausing + + +

    + +

    + This page provides a very basic demonstration of + "pausing" and "unpausing" the OPFS SAHPool VFS such that + multiple pages or workers can use it by coordinating which + handler may have it open at any given time. +

    +
    + + +
    +
    + + + + diff --git a/ext/wasm/tests/opfs/sahpool/sahpool-pausing.js b/ext/wasm/tests/opfs/sahpool/sahpool-pausing.js new file mode 100644 index 000000000..1aa98d3cb --- /dev/null +++ b/ext/wasm/tests/opfs/sahpool/sahpool-pausing.js @@ -0,0 +1,183 @@ +/* + 2025-01-31 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + These tests are specific to the opfs-sahpool VFS and are limited to + demonstrating its pause/unpause capabilities. + + Most of this file is infrastructure for displaying results to the + user. Search for runTests() to find where the work actually starts. +*/ +'use strict'; +(function(){ + let logClass; + + const mapToString = (v)=>{ + switch(typeof v){ + case 'number': case 'string': case 'boolean': + case 'undefined': case 'bigint': + return ''+v; + default: break; + } + if(null===v) return 'null'; + if(v instanceof Error){ + v = { + message: v.message, + stack: v.stack, + errorClass: v.name + }; + } + return JSON.stringify(v,undefined,2); + }; + const normalizeArgs = (args)=>args.map(mapToString); + const logTarget = document.querySelector('#test-output'); + logClass = function(cssClass,...args){ + const ln = document.createElement('div'); + if(cssClass){ + for(const c of (Array.isArray(cssClass) ? cssClass : [cssClass])){ + ln.classList.add(c); + } + } + ln.append(document.createTextNode(normalizeArgs(args).join(' '))); + logTarget.append(ln); + }; + const cbReverse = document.querySelector('#cb-log-reverse'); + //cbReverse.setAttribute('checked','checked'); + const cbReverseKey = 'tester1:cb-log-reverse'; + const cbReverseIt = ()=>{ + logTarget.classList[cbReverse.checked ? 'add' : 'remove']('reverse'); + //localStorage.setItem(cbReverseKey, cbReverse.checked ? 1 : 0); + }; + cbReverse.addEventListener('change', cbReverseIt, true); + /*if(localStorage.getItem(cbReverseKey)){ + cbReverse.checked = !!(+localStorage.getItem(cbReverseKey)); + }*/ + cbReverseIt(); + + const log = (...args)=>{ + //console.log(...args); + logClass('',...args); + } + const warn = (...args)=>{ + console.warn(...args); + logClass('warning',...args); + } + const error = (...args)=>{ + console.error(...args); + logClass('error',...args); + }; + + const toss = (...args)=>{ + error(...args); + throw new Error(args.join(' ')); + }; + + const endOfWork = (passed=true)=>{ + const eH = document.querySelector('#color-target'); + const eT = document.querySelector('title'); + if(passed){ + log("End of work chain. If you made it this far, you win."); + eH.innerText = 'PASS: '+eH.innerText; + eH.classList.add('tests-pass'); + eT.innerText = 'PASS: '+eT.innerText; + }else{ + eH.innerText = 'FAIL: '+eH.innerText; + eH.classList.add('tests-fail'); + eT.innerText = 'FAIL: '+eT.innerText; + } + }; + + const nextHandlerQueue = []; + + const nextHandler = function(workerId,...msg){ + log(workerId,...msg); + (nextHandlerQueue.shift())(); + }; + + const postThen = function(W, msgType, callback){ + nextHandlerQueue.push(callback); + W.postMessage({type:msgType}); + }; + + /** + Run a series of operations on an sahpool db spanning two workers. + This would arguably be more legible with Promises, but creating a + Promise-based communication channel for this purpose is left as + an exercise for the reader. An example of such a proxy can be + found in the SQLite source tree: + + https://sqlite.org/src/file/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js + */ + const runPyramidOfDoom = function(W1, W2){ + postThen(W1, 'vfs-acquire', function(){ + postThen(W1, 'db-init', function(){ + postThen(W1, 'db-query', function(){ + postThen(W1, 'vfs-pause', function(){ + postThen(W2, 'vfs-acquire', function(){ + postThen(W2, 'db-query', function(){ + postThen(W2, 'vfs-remove', endOfWork); + }); + }); + }); + }); + }); + }); + }; + + const runTests = function(){ + log("Running opfs-sahpool pausing tests..."); + const wjs = 'sahpool-worker.js?sqlite3.dir=../../../jswasm'; + const W1 = new Worker(wjs+'&workerId=w1'), + W2 = new Worker(wjs+'&workerId=w2'); + W1.workerId = 'w1'; + W2.workerId = 'w2'; + let initCount = 0; + const onmessage = function({data}){ + //log("onmessage:",data); + switch(data.type){ + case 'vfs-acquired': + nextHandler(data.workerId, "VFS acquired"); + break; + case 'vfs-paused': + nextHandler(data.workerId, "VFS paused"); + break; + case 'vfs-unpaused': + nextHandler(data.workerId, 'VFS unpaused'); + break; + case 'vfs-removed': + nextHandler(data.workerId, 'VFS removed'); + break; + case 'db-inited': + nextHandler(data.workerId, 'db initialized'); + break; + case 'query-result': + nextHandler(data.workerId, 'query result', data.payload); + break; + case 'log': + log(data.workerId, ':', ...data.payload); + break; + case 'error': + error(data.workerId, ':', ...data.payload); + endOfWork(false); + break; + case 'initialized': + log(data.workerId, ': Worker initialized',...data.payload); + if( 2===++initCount ){ + runPyramidOfDoom(W1, W2); + } + break; + } + }; + W1.onmessage = W2.onmessage = onmessage; + }; + + runTests(); +})(); diff --git a/ext/wasm/tests/opfs/sahpool/sahpool-worker.js b/ext/wasm/tests/opfs/sahpool/sahpool-worker.js new file mode 100644 index 000000000..592f15955 --- /dev/null +++ b/ext/wasm/tests/opfs/sahpool/sahpool-worker.js @@ -0,0 +1,104 @@ +/* + 2025-01-31 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file is part of sahpool-pausing.js's demonstration of the + pause/unpause feature of the opfs-sahpool VFS. +*/ +const searchParams = new URL(self.location.href).searchParams; +const workerId = searchParams.get('workerId'); +const wPost = (type,...args)=>postMessage({type, workerId, payload:args}); +const log = (...args)=>wPost('log',...args); +let capi, wasm, S, poolUtil; + +const sahPoolConfig = { + name: 'opfs-sahpool-pausable', + clearOnInit: false, + initialCapacity: 3 +}; + +importScripts(searchParams.get('sqlite3.dir') + '/sqlite3.js'); + +const sqlExec = function(sql){ + const db = new poolUtil.OpfsSAHPoolDb('/my.db'); + try{ + return db.exec(sql); + }finally{ + db.close(); + } +}; + +const clog = console.log.bind(console); +globalThis.onmessage = function({data}){ + clog(workerId+": onmessage:",data); + switch(data.type){ + case 'vfs-acquire': + if( poolUtil ){ + poolUtil.unpauseVfs().then(()=>wPost('vfs-unpaused')); + }else{ + S.installOpfsSAHPoolVfs(sahPoolConfig).then(pu=>{ + poolUtil = pu; + wPost('vfs-acquired'); + }); + } + break; + case 'db-init': + try{ + sqlExec([ + "DROP TABLE IF EXISTS mytable;", + "CREATE TABLE mytable(a);", + "INSERT INTO mytable(a) VALUES(11),(22),(33)" + ]); + wPost('db-inited'); + }catch(e){ + wPost('error',e.message); + } + break; + case 'db-query': { + const rc = sqlExec({ + sql: 'select * from mytable order by a', + rowMode: 'array', + returnValue: 'resultRows' + }); + wPost('query-result',rc); + break; + } + case 'vfs-remove': + poolUtil.removeVfs().then(()=>wPost('vfs-removed')); + break; + case 'vfs-pause': + poolUtil.pauseVfs(); + wPost('vfs-paused'); + break; + } +}; + +const hasOpfs = ()=>{ + return globalThis.FileSystemHandle + && globalThis.FileSystemDirectoryHandle + && globalThis.FileSystemFileHandle + && globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle + && navigator?.storage?.getDirectory; +}; +if( !hasOpfs() ){ + wPost('error',"OPFS not detected"); +}else{ + globalThis.sqlite3InitModule().then(async function(sqlite3){ + S = sqlite3; + capi = S.capi; + wasm = S.wasm; + log("sqlite3 version:",capi.sqlite3_libversion(), + capi.sqlite3_sourceid()); + //return sqlite3.installOpfsSAHPoolVfs(sahPoolConfig).then(pu=>poolUtil=pu); + }).then(()=>{ + wPost('initialized'); + }); +} diff --git a/main.mk b/main.mk index b40b9f968..fe874b1f3 100644 --- a/main.mk +++ b/main.mk @@ -22,10 +22,10 @@ all: # # $(TOP) = # -# The toplevel directory of the source tree. For canonical builds +# The top-level directory of the source tree. For canonical builds # this is the directory that contains this "Makefile.in" and the -# "configure.in" script. For out-of-tree builds, this will differ -# from $(PWD). +# "auto.def" script. For out-of-tree builds, this will differ from +# $(PWD). # TOP ?= $(PWD) # @@ -88,6 +88,10 @@ T.exe ?= $(B.exe) T.dll ?= $(B.dll) T.lib ?= $(B.lib) # +# HAVE_TCL = 1 to enable full tcl support, else 0. +# +HAVE_TCL ?= 0 +# # $(TCLSH_CMD) = # # The canonical tclsh. @@ -115,8 +119,8 @@ JIMSH ?= ./jimsh$(T.exe) # # The TCL interpreter for in-tree code generation. May be either the # in-tree JimTCL ($(JIMSH)) or the canonical TCL ($(TCLSH_CMD). If -# it's JimTCL, it must be compiled with -DHAVE_REALPATH or -# -DHAVE__FULLPATH. +# it's JimTCL, it must be compiled with -DHAVE_REALPATH (Unix) or +# -DHAVE__FULLPATH (Windows). # B.tclsh ?= $(JIMSH) @@ -187,6 +191,9 @@ CFLAGS.readline ?= -I$(prefix)/include # during installation, which may break the build of targets which are # built after others are installed. # +# Maintenance reminder: we specifically do not strip binaries, as +# discussed in https://sqlite.org/forum/forumpost/9a67df63eda9925c. +# INSTALL ?= install # # $(ENABLE_LIB_SHARED) = @@ -207,7 +214,7 @@ ENABLE_LIB_STATIC ?= 1 # # 1 if the amalgamation (sqlite3.c/h) should be built/used, otherwise # the library is built from all of its original source files. -# Certaint tools, like sqlite3$(T.exe), require the amalgamation and +# Certain tools, like sqlite3$(T.exe), require the amalgamation and # will ignore this preference. # USE_AMALGAMATION ?= 1 @@ -231,6 +238,13 @@ LINK_TOOLS_DYNAMICALLY ?= 0 # Optional flags for the amalgamation generator. # AMALGAMATION_GEN_FLAGS ?= --linemacros=0 +# +# EXTRA_SRC = list of C files to append as-is to the generated +# amalgamation. It should arguably be called AMALGAMATION_EXTRA_SRC +# but this older name is already in use by clients. +# +EXTRA_SRC ?= + # # $(OPT_FEATURE_FLAGS) = # @@ -250,8 +264,9 @@ AMALGAMATION_GEN_FLAGS ?= --linemacros=0 # # $(OPTS)=... is another way of influencing C compilation. It is # distinctly separate from $(OPTIONS) and $(OPT_FEATURE_FLAGS) but, -# like those, $(OPTS) applies to all invocations of $(T.cc). The -# configure process does not set either of $(OPTIONS) or $(OPTS). +# like those, $(OPTS) applies to all invocations of $(T.cc) (and some +# invocations of $(B.cc). The configure process does not set either of +# $(OPTIONS) or $(OPTS). # OPT_FEATURE_FLAGS ?= # @@ -353,12 +368,36 @@ T.cc += $(OPTS) INSTALL.noexec = $(INSTALL) -m 0644 # ^^^ do not use GNU-specific flags to $(INSTALL), e.g. --mode=... +# +# T.compile.gcov = gcov-specific compilation flags for the target +# platform. +# +T.compile.gcov ?= +# +# T.link.gcov = gcov-specific link flags for the target platform. +# +T.link.gcov ?= + # # $(T.compile) = generic target platform compiler invocation, -# differing only from $(T.cc) in that it appends $(T.compile.extras), -# which are primarily intended for use with gcov-related flags. +# differing only from $(T.cc) in that it appends $(T.compile.gcov), +# which is intended for use with gcov-related flags. # -T.compile = $(T.cc) $(T.compile.extras) +T.compile = $(T.cc) $(T.compile.gcov) + +# +# Optionally set by the configure script to include -DSQLITE_DEBUG=1 +# and other debug-related flags. +# +T.cc.TARGET_DEBUG ?= + +# +# Extra CFLAGS for both the core sqlite3 components and extensions. +# +# Define -D_HAVE_SQLITE_CONFIG_H so that the code knows it +# can include the generated sqlite_cfg.h. +# +T.cc.sqlite.extras = -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite $(T.cc.TARGET_DEBUG) # # $(T.cc.sqlite) is $(T.cc) plus any flags which are desired for the @@ -366,7 +405,7 @@ T.compile = $(T.cc) $(T.compile.extras) # will normally get initially populated with flags by the # configure-generated makefile. # -T.cc.sqlite ?= $(T.cc) +T.cc.sqlite ?= $(T.compile) $(T.cc.sqlite.extras) # # $(CFLAGS.intree_includes) = -I... flags relevant specifically to @@ -382,16 +421,16 @@ T.cc.sqlite += $(CFLAGS.intree_includes) # # $(T.cc.extension) = compiler invocation for loadable extensions. # -T.cc.extension = $(T.compile) -I. -I$(TOP)/src -DSQLITE_CORE +T.cc.extension = $(T.compile) -I. -I$(TOP)/src $(T.cc.sqlite.extras) -DSQLITE_CORE # # $(T.link) = compiler invocation for when the target will be an # executable. # -# $(T.link.extras) = optional config-specific flags for $(T.link), -# primarily intended for use with gcov-related flags. +# $(T.link.gcov) = optional config-specific flags for $(T.link), +# intended for use with gcov-related flags. # -T.link = $(T.cc.sqlite) $(T.link.extras) +T.link = $(T.cc.sqlite) $(T.link.gcov) # # $(T.link.shared) = $(T.link) invocation specifically for shared libraries # @@ -448,8 +487,9 @@ $(install-dir.all): # to an empty string. # # 2) Ensure that it is built with -DJIM_COMPAT (which may be -# hard-coded into jimsh0.c). Without this, the [expr] command -# accepts only a single argument. +# hard-coded into jimsh0.c). Without this, the [expr] command accepts +# only a single argument. (That said: the real fix for that is to +# update any scripts which still pass multiple arguments to [expr].) # $(JIMSH): $(TOP)/autosetup/jimsh0.c $(B.cc) -o $@ $(CFLAGS.jimsh) $(TOP)/autosetup/jimsh0.c @@ -548,7 +588,7 @@ SRC = \ $(TOP)/src/build.c \ $(TOP)/src/callback.c \ $(TOP)/src/complete.c \ - $(TOP)/src/ctime.c \ + ctime.c \ $(TOP)/src/date.c \ $(TOP)/src/dbpage.c \ $(TOP)/src/dbstat.c \ @@ -596,7 +636,7 @@ SRC = \ $(TOP)/src/pcache.h \ $(TOP)/src/pcache1.c \ $(TOP)/src/pragma.c \ - $(TOP)/src/pragma.h \ + pragma.h \ $(TOP)/src/prepare.c \ $(TOP)/src/printf.c \ $(TOP)/src/random.c \ @@ -792,7 +832,7 @@ TESTSRC2 = \ $(TOP)/src/bitvec.c \ $(TOP)/src/btree.c \ $(TOP)/src/build.c \ - $(TOP)/src/ctime.c \ + ctime.c \ $(TOP)/src/date.c \ $(TOP)/src/dbpage.c \ $(TOP)/src/dbstat.c \ @@ -857,7 +897,7 @@ HDR = \ $(TOP)/src/pager.h \ $(TOP)/src/pcache.h \ parse.h \ - $(TOP)/src/pragma.h \ + pragma.h \ sqlite3.h \ $(TOP)/src/sqlite3ext.h \ $(TOP)/src/sqliteInt.h \ @@ -956,6 +996,7 @@ FUZZCHECK_OPT += \ -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \ -DSQLITE_ENABLE_STAT4 \ -DSQLITE_ENABLE_STMT_SCANSTATUS \ + -DSQLITE_JSON_MAX_DEPTH=500 \ -DSQLITE_MAX_MEMORY=50000000 \ -DSQLITE_MAX_MMAP_SIZE=0 \ -DSQLITE_OMIT_LOAD_EXTENSION \ @@ -1099,7 +1140,7 @@ sqlite3.h: $(MAKE_SANITY_CHECK) $(TOP)/src/sqlite.h.in \ $(B.tclsh) $(TOP)/tool/mksqlite3h.tcl $(TOP) -o sqlite3.h sqlite3.c: .target_source sqlite3.h $(TOP)/tool/mksqlite3c.tcl src-verify$(B.exe) \ - $(B.tclsh) + $(B.tclsh) $(EXTRA_SRC) $(B.tclsh) $(TOP)/tool/mksqlite3c.tcl $(AMALGAMATION_GEN_FLAGS) $(EXTRA_SRC) cp tsrc/sqlite3ext.h . cp $(TOP)/ext/session/sqlite3session.h . @@ -1164,8 +1205,11 @@ callback.o: $(TOP)/src/callback.c $(DEPS_OBJ_COMMON) complete.o: $(TOP)/src/complete.c $(DEPS_OBJ_COMMON) $(T.cc.sqlite) -c $(TOP)/src/complete.c -ctime.o: $(TOP)/src/ctime.c $(DEPS_OBJ_COMMON) - $(T.cc.sqlite) -c $(TOP)/src/ctime.c +ctime.c: $(TOP)/tool/mkctimec.tcl $(B.tclsh) + $(B.tclsh) $(TOP)/tool/mkctimec.tcl + +ctime.o: ctime.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) -c ctime.c date.o: $(TOP)/src/date.c $(DEPS_OBJ_COMMON) $(T.cc.sqlite) -c $(TOP)/src/date.c @@ -1384,12 +1428,26 @@ tclsqlite-shell.o: $(T.tcl.env.sh) $(TOP)/src/tclsqlite.c $(DEPS_OBJ_COMMON) tclsqlite-stubs.o: $(T.tcl.env.sh) $(TOP)/src/tclsqlite.c $(DEPS_OBJ_COMMON) $(T.compile.tcl) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c $$TCL_INCLUDE_SPEC -tclsqlite3$(T.exe): $(T.tcl.env.sh) tclsqlite-shell.o $(libsqlite3.DLL) +# +# STATIC_TCLSQLITE3 = 1 to statically link tclsqlite3, else +# 0. Requires static versions of all requisite libraries. Primarily +# intended for use with static-friendly environments like Alpine +# Linux. It won't work on glibc-based systems. +# +STATIC_TCLSQLITE3 ?= 0 +# +# tclsqlite3.(deps|flags).N = N is $(STATIC_TCLSQLITE3) +# +tclsqlite3.deps.1 = sqlite3.o +tclsqlite3.flags.1 = -static $(tclsqlite3.deps.1) +tclsqlite3.deps.0 = $(libsqlite3.DLL) +tclsqlite3.flags.0 = $(tclsqlite3.deps.0) +tclsqlite3$(T.exe): $(T.tcl.env.sh) tclsqlite-shell.o $(tclsqlite3.deps.$(STATIC_TCLSQLITE3)) $(T.link.tcl) -o $@ tclsqlite-shell.o \ - $(libsqlite3.DLL) $$TCL_INCLUDE_SPEC $$TCL_LIB_SPEC \ + $(tclsqlite3.flags.$(STATIC_TCLSQLITE3)) $$TCL_INCLUDE_SPEC $$TCL_LIB_SPEC \ $(LDFLAGS.libsqlite3) tclsqlite3$(T.exe)-1: tclsqlite3$(T.exe) -tclsqlite3$(T.exe)-0 tclsqlite3$(T.exe)-: +tclsqlite3$(T.exe)-0: tcl: tclsqlite3$(T.exe)-$(HAVE_TCL) # Rules to build opcodes.c and opcodes.h @@ -1409,6 +1467,9 @@ parse.c: $(TOP)/src/parse.y lemon$(B.exe) cp $(TOP)/src/parse.y . ./lemon$(B.exe) $(OPT_FEATURE_FLAGS) $(OPTS) -S parse.y +pragma.h: $(TOP)/tool/mkpragmatab.tcl $(B.tclsh) + $(B.tclsh) $(TOP)/tool/mkpragmatab.tcl + sqlite3rc.h: $(TOP)/src/sqlite3.rc $(TOP)/VERSION $(B.tclsh) echo '#ifndef SQLITE_RESOURCE_VERSION' >$@ echo -n '#define SQLITE_RESOURCE_VERSION ' >>$@ @@ -1464,7 +1525,7 @@ all: so # The link named libsqlite3.so.0 is provided in an attempt to reduce # downstream disruption when performing upgrades from pre-3.48 to a # version 3.48 or higher. That name is considered a legacy remnant -# and will eventually be removed from this installation process. +# and may eventually be removed from this installation process. # # Historically libtool installed the library like so: # @@ -1493,7 +1554,7 @@ all: so # still expect to see the legacy file names. # # In either case, libsqlite3.la, if found, is deleted because it would -# contain stale state, refering to non-libtool-generated libraries. +# contain stale state, referring to non-libtool-generated libraries. # install-dll-out-implib: $(install-dir.lib) $(libsqlite3.DLL) @@ -1525,6 +1586,8 @@ install-dll-unix-generic: install-dll-out-implib install-dll-msys: install-dll-out-implib $(install-dir.bin) $(INSTALL) $(libsqlite3.DLL) "$(install-dir.bin)" # ----------------------------------------------^^^ yes, bin +# Each of {msys,mingw,cygwin} uses a different name for the DLL, but +# that is already accounted for via $(libsqlite3.DLL). install-dll-mingw: install-dll-msys install-dll-cygwin: install-dll-msys @@ -1568,8 +1631,8 @@ pkgIndex.tcl: pkgIndex.tcl-1: pkgIndex.tcl pkgIndex.tcl-0 pkgIndex.tcl-: tcl: pkgIndex.tcl-$(HAVE_TCL) -libtclsqlite3.SO = libtclsqlite3$(T.dll) -$(libtclsqlite3.SO): $(T.tcl.env.sh) tclsqlite.o $(LIBOBJ) +libtclsqlite3.DLL = libtclsqlite3$(T.dll) +$(libtclsqlite3.DLL): $(T.tcl.env.sh) tclsqlite.o $(LIBOBJ) $(T.tcl.env.source); \ $(T.link.shared) -o $@ tclsqlite.o \ $$TCL_INCLUDE_SPEC $$TCL_STUB_LIB_SPEC $(LDFLAGS.libsqlite3) \ @@ -1577,19 +1640,19 @@ $(libtclsqlite3.SO): $(T.tcl.env.sh) tclsqlite.o $(LIBOBJ) # ^^^ that rpath bit is defined as TCL_LD_SEARCH_FLAGS in # tclConfig.sh, but it's defined in such a way as to be useless for a # _static_ makefile. -$(libtclsqlite3.SO)-1: $(libtclsqlite3.SO) -$(libtclsqlite3.SO)-0 $(libtclsqlite3.SO)-: -libtcl: $(libtclsqlite3.SO)-$(HAVE_TCL) +$(libtclsqlite3.DLL)-1: $(libtclsqlite3.DLL) +$(libtclsqlite3.DLL)-0 $(libtclsqlite3.DLL)-: +libtcl: $(libtclsqlite3.DLL)-$(HAVE_TCL) tcl: libtcl all: tcl -install-tcl-1: $(libtclsqlite3.SO) pkgIndex.tcl +install-tcl-1: $(libtclsqlite3.DLL) pkgIndex.tcl $(T.tcl.env.source); \ $(INSTALL) -d "$(DESTDIR)$$TCLLIBDIR"; \ - $(INSTALL) $(libtclsqlite3.SO) "$(DESTDIR)$$TCLLIBDIR"; \ + $(INSTALL) $(libtclsqlite3.DLL) "$(DESTDIR)$$TCLLIBDIR"; \ $(INSTALL.noexec) pkgIndex.tcl "$(DESTDIR)$$TCLLIBDIR" install-tcl-0 install-tcl-: - @echo "TCL support disabled, so not installing $(libtclsqlite3.SO)" + @echo "TCL support disabled, so not installing $(libtclsqlite3.DLL)" install-tcl: install-tcl-$(HAVE_TCL) install: install-tcl @@ -1772,6 +1835,12 @@ mdevtest: srctree-check has_tclsh85 sdevtest: has_tclsh85 $(TCLSH_CMD) $(TOP)/test/testrunner.tcl sdevtest $(TSTRNNR_OPTS) +# Like releasetest, except it omits srctree-check and verify-source so +# that it can be used on a modified source tree. +# +xdevtest: has_tclsh85 + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl release $(TSTRNNR_OPTS) + # # Validate that various generated files in the source tree # are up-to-date. @@ -1983,7 +2052,7 @@ checksymbols: sqlite3.o nm -g --defined-only sqlite3.o | egrep -v $(VALIDIDS); test $$? -ne 0 echo '0 errors out of 1 tests' -# Build the amalgamation-autoconf package. The amalamgation-tarball target builds +# Build the amalgamation-autoconf package. The amalgamation-tarball target builds # a tarball named for the version number. Ex: sqlite-autoconf-3110000.tar.gz. # The snapshot-tarball target builds a tarball named by the SHA3 hash # @@ -1998,7 +2067,7 @@ snapshot-tarball: sqlite3.c sqlite3rc.h sqlite-src.zip: $(TOP)/tool/mksrczip.tcl $(TCLSH_CMD) $(TOP)/tool/mksrczip.tcl -# Build a ZIP archive of the amaglamation +# Build a ZIP archive of the amalgamation # sqlite-amalgamation.zip: $(TOP)/tool/mkamalzip.tcl sqlite3.c sqlite3.h shell.c sqlite3ext.h $(TCLSH_CMD) $(TOP)/tool/mkamalzip.tcl @@ -2042,6 +2111,18 @@ threadtest5: sqlite3.c $(TOP)/test/threadtest5.c $(T.link) $(TOP)/test/threadtest5.c sqlite3.c -o $@ $(LDFLAGS.libsqlite3) xbin: threadtest5 +# +# STATIC_CLI_SHELL = 1 to statically link sqlite3$(T.exe), else +# 0. Requires static versions of all requisite libraries. Primarily +# intended for use with static-friendly environments like Alpine +# Linux. +# +STATIC_CLI_SHELL ?= 0 +# +# sqlite3-shell-static.flags.N = N is $(STATIC_CLI_SHELL) +# +sqlite3-shell-static.flags.1 = -static +sqlite3-shell-static.flags.0 = # # When building sqlite3$(T.exe) we specifically embed a copy of # sqlite3.c, and not link to libsqlite3.so or libsqlite3.a, because @@ -2054,6 +2135,7 @@ xbin: threadtest5 sqlite3$(T.exe): shell.c sqlite3.c $(T.link) -o $@ \ shell.c sqlite3.c \ + $(sqlite3-shell-static.flags.$(STATIC_CLI_SHELL)) \ $(CFLAGS.readline) $(SHELL_OPT) $(CFLAGS.icu) \ $(LDFLAGS.libsqlite3) $(LDFLAGS.readline) # @@ -2154,8 +2236,11 @@ fuzzcheck$(T.exe): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) fuzzy: fuzzcheck$(T.exe) xbin: fuzzcheck$(T.exe) +# -fsanitize=... flags for fuzzcheck-asan. +CFLAGS.fuzzcheck-asan.fsanitize ?= -fsanitize=address + fuzzcheck-asan$(T.exe): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) - $(T.link) -o $@ -fsanitize=address $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) \ + $(T.link) -o $@ $(CFLAGS.fuzzcheck-asan.fsanitize) $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) \ sqlite3.c $(LDFLAGS.libsqlite3) fuzzy: fuzzcheck-asan$(T.exe) xbin: fuzzcheck-asan$(T.exe) @@ -2166,20 +2251,6 @@ fuzzcheck-ubsan$(T.exe): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) fuzzy: fuzzcheck-ubsan$(T.exe) xbin: fuzzcheck-ubsan$(T.exe) -# Usage: FUZZDB=filename make run-fuzzcheck -# -# Where filename is a fuzzcheck database, this target builds and runs -# fuzzcheck, fuzzcheck-asan, and fuzzcheck-ubsan on that database. -# -# FUZZDB can be a glob pattern of two or more databases. Example: -# -# FUZZDB=test/fuzzdata*.db make run-fuzzcheck -# -run-fuzzcheck: fuzzcheck$(T.exe) fuzzcheck-asan$(T.exe) fuzzcheck-ubsan$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi - ./fuzzcheck$(T.exe) --spinner $(FUZZDB) - ./fuzzcheck-asan$(T.exe) --spinner $(FUZZDB) - ./fuzzcheck-ubsan$(T.exe) --spinner $(FUZZDB) ossshell$(T.exe): $(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h $(T.link) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \ @@ -2327,6 +2398,9 @@ stmt.o: $(TOP)/ext/misc/stmt.c $(DEPS_EXT_COMMON) # # Windows section # +# 2025-03-03: sqlite3.def and sqlite3.dll might no longer be relevant +# in this particular build, but that's difficult to verify. +# dll: sqlite3.dll sqlite3.def: $(LIBOBJ) echo 'EXPORTS' >sqlite3.def @@ -2339,6 +2413,7 @@ sqlite3.dll: $(LIBOBJ) sqlite3.def # # Emit a list of commonly-used targets +# help: @echo; echo "Frequently-used high-level make targets:"; echo; \ echo " - all [default] = builds most components"; \ @@ -2361,21 +2436,22 @@ help: echo +# # Remove build products sufficient so that subsequent makes will recompile # everything from scratch. Do not remove: # # * test results and test logs # * output from ./configure # -tidy-.: -tidy: tidy-. +# +tidy: rm -f *.o *.c *.da *.bb *.bbg gmon.* *.rws sqlite3$(T.exe) rm -f fts5.h keywordhash.h opcodes.h sqlite3.h sqlite3ext.h sqlite3session.h rm -rf .libs .deps tsrc .target_source rm -f lemon$(B.exe) sqlite*.tar.gz rm -f mkkeywordhash$(B.exe) mksourceid$(B.exe) rm -f parse.* fts5parse.* - rm -f $(libsqlite3.DLL) $(libsqlite3.LIB) $(libtclsqlite3.SO) libsqlite3$(T.dll).a + rm -f $(libsqlite3.DLL) $(libsqlite3.LIB) $(libtclsqlite3.DLL) libsqlite3$(T.dll).a rm -f tclsqlite3$(T.exe) $(TESTPROGS) rm -f LogEst$(T.exe) fts3view$(T.exe) rollback-test$(T.exe) showdb$(T.exe) rm -f showjournal$(T.exe) showstat4$(T.exe) showwal$(T.exe) speedtest1$(T.exe) @@ -2395,20 +2471,28 @@ tidy: tidy-. rm -f src-verify$(B.exe) rm -f tclsqlite3.c has_tclsh* $(T.tcl.env.sh) rm -f sqlite3rc.h sqlite3.def + rm -f ctime.c pragma.h # # Removes build products and test logs. Retains ./configure outputs. # -clean-.: -clean: clean-. tidy +clean: tidy rm -rf omittest* testrunner* testdir* -# Clean up everything. No exceptions. -distclean-.: -distclean: distclean-. clean +# +# Clean up everything. No exceptions. From an out-of-tree build which +# starts in an empty directory, this should result in an empty +# directory (assuming the user does not create new files in this +# directory). +# +# The main distclean rules are in Makefile.in. +# +distclean: clean +# # Show important variable settings. +# show-variables: @echo "CC = $(CC)" @echo "B.cc = $(B.cc)" diff --git a/manifest b/manifest index 0af6475e3..77f51e9f0 100644 --- a/manifest +++ b/manifest @@ -1,43 +1,40 @@ -C Version\s3.49.2 -D 2025-05-07T10:39:52.886 +C Version\s3.50.1 +D 2025-06-06T14:52:32.856 +F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d -F Makefile.in d588d01af5f4f1e37468cd197b5614a64fabbab7be17b75728b75388f7e34f5e +F Makefile.in c3e414df4dc8dfb12f1f6baf129fcb6d18cd0ebd3c9109370fb3fceeeef9a37a F Makefile.linux-generic bd3e3cacd369821a6241d4ea1967395c962dfe3057e38cb0a435cee0e8b789d0 -F Makefile.msc a9b95ae9807e17f9b0734ebe97d68032141c3f95286bb64593cb73b206f043cf -F README.md c3c0f19532ce28f6297a71870f3c7b424729f0e6d9ab889616d3587dd2332159 -F VERSION 5008c952d977ab40afea04fd86fa8ed8720ce13a0ea878bf948ab1b26c610d98 +F Makefile.msc 0206f28988bb6634c7e8aff05bf6cfa65d6dfe1d2b6bd95160dd99290a83dfc7 +F README.md e28077cfbef795e99c9c75ed95aa7257a1166709b562076441a8506ac421b7c1 +F VERSION 1283c71d85c60203d57aafc1457c71668ba6090ceaa709a16b90279f54146579 F art/icon-243x273.gif 9750b734f82fdb3dc43127753d5e6fbf3b62c9f4e136c2fbf573b2f57ea87af5 F art/icon-80x90.gif 65509ce3e5f86a9cd64fe7fca2d23954199f31fe44c1e09e208c80fb83d87031 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F art/sqlite370.svg 40b7e2fe8aac3add5d56dd86ab8d427a4eca5bcb3fe4f8946cb3794e1821d531 -F auto.def a8c935b5c3c0b27c6a8b1b788bb47b06cc0ca3e9e92dc1b87e4b02659ba95ff6 +F auto.def 82c32443a91c1062f7a48beec37dbb2d8d03447b1286bce8df5ebb6d8d353f8a F autoconf/Makefile.fallback 22fe523eb36dfce31e0f6349f782eb084e86a5620b2b0b4f84a2d6133f53f5ac -F autoconf/Makefile.in ec1157e153ecad7c40bb8149250f10ef153dfa4514b93219e1342fe7caa6a533 -F autoconf/Makefile.msc 0a071367537dc395285a5d624ac4f99f3a387b27cc5e89752423c0499e15aec4 +F autoconf/Makefile.in d0926d2309e563b5ebdfd711e5c218533acab79829a05eef4e97c3a92e17bf42 +F autoconf/Makefile.msc f15ad424ca2820df8e39d9157965710af0a64d87773706706a12ea4f96e3a0d8 F autoconf/README.first f1d3876e9a7852c22f275a6f06814e64934cecbc0b5b9617d64849094c1fd136 -F autoconf/README.txt 7f01dc3915e2d68f329011073662369e62a0938a2c69398807823c57591cb288 -F autoconf/auto.def 8d81c1d728d8462a9b6c1ca0714013bbb097aee0ae5e79309d7939cead98e295 -F autoconf/tea/Makefile.in ba0556fee8da09c066bad85a4457904e46ee2c2eabaa309c0e83a78f2f151a8e -F autoconf/tea/README.txt 61e62e519579e4a112791354d6d440f8b51ea6db3b0bab58d59f29df42d2dfe3 -F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43 -F autoconf/tea/configure.ac.in da18360dfdeac7414fa8deb549f3d65aeca0ae1150ff1a8b902019b39ce019a4 -F autoconf/tea/doc/sqlite3.n e1fe45d4f5286ee3d0ccc877aca2a0def488e9bb +F autoconf/README.txt b749816b8452b3af994dc6d607394bef3df1736d7e09359f1087de8439a52807 +F autoconf/auto.def 3d994f3a9cc9b712dbce92a5708570ddcf3b988141b6eb738f2ed16127a9f0ac +F autoconf/tea/Makefile.in 14c6a79ce87e10d8a35398f2d0e04e1d83a88eb52ee16ebf0eeaccf005ff84b3 +F autoconf/tea/README.txt 6301adfea2e75eabcc550a1ed4812faaa4024c9584ac89a7f018e39f9bc9defb +F autoconf/tea/_teaish.tester.tcl.in ed5445512e91c12afbbb99771efb68a23be4a046d52d61213fb5b6f010118129 +F autoconf/tea/auto.def ce95b9450e2fa4ba5dc857e208fe10f4e6f2d737796ac3278aee6079db417529 +F autoconf/tea/configure d0b12b984edca6030d1976375b80157ac78b5b90a5b4f0dcee39357f63f4a80b x +F autoconf/tea/doc/sqlite3.n 9a97f4f717ceab73004ea412af7960625c1cb24b5c25e4ae4c8b5d8fa4300f4e F autoconf/tea/license.terms 13bd403c9610fd2b76ece0ab50c4c5eda933d523 -F autoconf/tea/pkgIndex.tcl.in 55aec3c6d7e9a1de9b8d2fdc9c27fd055da3ac3a51b572195e2ae7300bcfd3a2 -F autoconf/tea/tclconfig/install-sh 2182b3705d92e25753411e2c28cf788c69e35a48fbb8aa332e342dfc6b95b80d -F autoconf/tea/tclconfig/tcl.m4 284faa1d9cf66c1efb42817beb5c8a63626fb35bf903993d4f11fde75677cc1a -F autoconf/tea/win/makefile.vc 55721106928894cb818164a8ce054da11d948948f5a92a54d262dd0a6a891d4d -F autoconf/tea/win/nmakehlp.c b01f822eabbe1ed2b64e70882d97d48402b42d2689a1ea00342d1a1a7eaa19cb -F autoconf/tea/win/rules-ext.vc fd5740d97aac8c41c97eaa0fbcc0c15a41b6f7075d5f9f593e147d7a284a247a -F autoconf/tea/win/rules.vc 94a18c3e453535459b4a643983acca52fb8756e79055bd2ad4b0999d66484f4c -F autoconf/tea/win/targets.vc 96a25a1fa6e9e9cfb348fd3760a5395b4ce8acafc8ed10f0412937ec200d5dbd +F autoconf/tea/pkgIndex.tcl.in e07da6b94561f4aa382bab65b1ccceb04701b97bf59d007c1d1f20a222b22d07 +F autoconf/tea/teaish.tcl 81571a9f9ae5c70735595b05586cb2de9d2aea7e32aad10417c4982f2e2f01c8 +F autoconf/tea/teaish.test.tcl cfe94e1fb79dd078f650295be59843d470125e0cc3a17a1414c1fb8d77f4aea6 F autosetup/LICENSE 41a26aebdd2cd185d1e2b210f71b7ce234496979f6b35aef2cbf6b80cbed4ce4 F autosetup/README.autosetup a78ff8c4a3d2636a4268736672a74bf14a82f42687fcf0631a70c516075c031e -F autosetup/README.md b306314e8a87ccf873cb5b2a360c4a27bbf841df5b76f3acbd65322cff165476 +F autosetup/README.md f324bb9f9bf1cc787122034df53fbfdfed28ee2657e6652b763d992ab0d04829 F autosetup/autosetup 74a9782b68d07934510190fbd03fc6ad92e63f0ea3b5cbffa5f0bd271ad60f01 x F autosetup/autosetup-config.guess dfa101c5e8220e864d5e9c72a85e87110df60260d36cb951ad0a85d6d9eaa463 x F autosetup/autosetup-config.sub a38fb074d0dece01cf919e9fb534a26011608aa8fa606490864295328526cd73 x @@ -47,81 +44,86 @@ F autosetup/cc-db.tcl 6e0ed90146197a5a05b245e649975c07c548e30926b218ca3e1d4dc034 F autosetup/cc-lib.tcl 493c5935b5dd3bf9bd4eca89b07c8b1b1a9356d61783035144e21795facf7360 F autosetup/cc-shared.tcl 4f024e94a47f427ba61de1739f6381ef0080210f9fae89112d5c1de1e5460d78 F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e45f -F autosetup/jimsh0.c a57c16e65dcffc9c76e496757cb3f7fb47e01ecbd1631a0a5e01751fc856f049 +F autosetup/find_tclconfig.tcl e64886ffe3b982d4df42cd28ed91fe0b5940c2c5785e126c1821baf61bc86a7e +F autosetup/jimsh0.c 563b966c137a4ce3c9333e5196723b7ac0919140a9d7989eb440463cd855c367 F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba -F autosetup/proj.tcl 187d82550cfa55df00e285542e88278c51876d7813d63eaffb2fc5af40566d9f -F autosetup/sqlite-config.tcl 4b0205282099a1016bc9565b7b893eb9d4f739bd76acfcba74ad157d8e13acd7 +F autosetup/proj.tcl c4a77735b57f3c016a185bff048212a197b77723f9bea6cfafe396e4b542c666 +F autosetup/sqlite-config.tcl ccda82e43e377b832aae72a1678b1dc17dcaff36ed0ebbd8f0cfc88612ae8de3 F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9 +F autosetup/teaish/README.txt b40071e6f8506500a2f7f71d5fc69e0bf87b9d7678dd9da1e5b4d0acbf40b1ca +F autosetup/teaish/core.tcl 42b4f76ebcacafded7c13a225ce55454bd1fcf8a11912729128e1076b12f6433 +F autosetup/teaish/feature.tcl 18194fb79a24d30e5bbdeab40999616f39278b53a27525349ded033af2fd73be +F autosetup/teaish/tester.tcl 091745984473faea6985254b9986c6dfd0cce06f68bc515ba4afc1e6b3742fa8 F configure 9a00b21dfd13757bbfb8d89b30660a89ec1f8f3a79402b8f9f9b6fc475c3303a x -F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad +F contrib/sqlitecon.tcl eb4c6578e08dd353263958da0dc620f8400b869a50d06e271ab0be85a51a08d3 F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd F doc/compile-for-unix.md c9dce1ddd4bf0d25efccc5c63eb047e78c01ce06a6ff29c73e0a8af4a0f4adbc F doc/compile-for-windows.md 5141661e783c9ca9e3fd30e813345898712f5c311d71316f183db87038fa28a6 F doc/json-enhancements.md e356fc834781f1f1aa22ee300027a270b2c960122468499bf347bb123ce1ea4f -F doc/jsonb.md 5fab4b8613aa9153fbeb6259297bd4697988af8b3d23900deba588fa7841456b -F doc/lemon.html 8b266ff711d2ec7f867c3dca37634963f48a630329908cc282beebfa8c708706 -F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710 -F doc/tcl-extension-testing.md 864875c3b672db79e7d42348dd726f9a4fbd852b1d8e5efcf09fe3d1ff6bf2a2 -F doc/testrunner.md 15583cf8c7d8a1c3378fd5d4319ca769a14c4d950a5df9b015d01d5be290dc69 +F doc/jsonb.md ede3238186e3a90bb79d20b2a6a06d0f9429a38e069e9da0efbad0f2ed48b924 +F doc/lemon.html 89ea833a6f71773ab1a9063fbb7fb9b32147bc0b1057b53ecab94a3b30c0aef5 +F doc/pager-invariants.txt 83aa3a4724b2d7970cc3f3461f0295c46d4fc19a835a5781cbb35cb52feb0577 +F doc/tcl-extension-testing.md b88861804fc1eaf83249f8e206334189b61e150c360e1b80d0dcf91af82354f5 +F doc/testrunner.md 5ee928637e03f136a25fef852c5ed975932e31927bd9b05a574424ae18c31019 F doc/trusted-schema.md 33625008620e879c7bcfbbfa079587612c434fa094d338b08242288d358c3e8a F doc/vdbesort-memory.md 4da2639c14cd24a31e0af694b1a8dd37eaf277aff3867e9a8cc14046bc49df56 -F doc/vfs-shm.txt e101f27ea02a8387ce46a05be2b1a902a021d37a -F doc/wal-lock.md 781726aaba20bafeceb7ba9f91d5c98c6731691b30c954e37cf0b49a053d461d -F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd +F doc/vfs-shm.txt 1a55f3f0e7b6745931b117ba5c9df3640d7a0536f532ef0052563100f4416f86 +F doc/wal-lock.md 7db0cd61e2000b545b78ce89b0c2a9a8dd8d64c097839258ac10d7c5c4156ec1 +F ext/README.md 6eb1ac267d917767952ed0ef63f55de003b6a5da433ce1fa389e1a9532e73132 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3 F ext/expert/expert.c d548d603a4cc9e61f446cc179c120c6713511c413f82a4a32b1e1e69d3f086a4 F ext/expert/expert1.test 1d2da6606623b57bb47064e02140823ce1daecd4cacbf402c73ad3473d7f000c -F ext/expert/sqlite3expert.c 494a6b7d4e0ead6dec6a50109dd78fcc054bb1a3fcc29c6f25e06a3685ed557e +F ext/expert/sqlite3expert.c cf4b1e5584862f486a4c6014ddb081831f1c512065dcf35644638d57179979d6 F ext/expert/sqlite3expert.h ca81efc2679a92373a13a3e76a6138d0310e32be53d6c3bfaedabd158ea8969b -F ext/expert/test_expert.c b767b2039a0df707eb3147e86bcf68b252d8455d9a41774b1a836cd052ceca70 +F ext/expert/test_expert.c c395134bd6d4efa594a7d26578a1cb624c4027b79b4b5fcd44736c5ef1f5f725 F ext/fts3/README.content b9078d0843a094d86af0d48dffbff13c906702b4c3558012e67b9c7cc3bf59ee -F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a +F ext/fts3/README.syntax b72477722e9b4fe43f8403227d790a1c94221bfad15c27863a4b36d1052e892b F ext/fts3/README.tokenizers b92bdeb8b46503f0dd301d364efc5ef59ef9fa8e2758b8e742f39fa93a2e422d F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c b840ee915a6fb36571e3fe3c096e8a481a4a9cd8a35199a1b976b132b9f84ad3 +F ext/fts3/fts3.c 4f02858ab845a97bedf06e6cc1fba49a81fe5e00a26df68d0ad0f00a5814fa70 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 2fe7c76dfd7d46dff964d17d3f4c53bca2116cf5d6252552ebbc22e38afdf4e0 +F ext/fts3/fts3Int.h 53f00a0d037c81174af1089ad7776563dcbccb5ed4697405256d3d4260d9fccc F ext/fts3/fts3_aux.c 7eab82a9cf0830f6551ba3abfdbe73ed39e322a4d3940ee82fbf723674ecd9f3 -F ext/fts3/fts3_expr.c 365849a2a1185e19028a9db2d9f1ea63efe909a3a6aca7ec86fc26a13a60bd58 -F ext/fts3/fts3_hash.c 8b6e31bfb0844c27dc6092c2620bdb1fca17ed613072db057d96952c6bdb48b7 +F ext/fts3/fts3_expr.c 5c13796638d8192c388777166075cdc8bc4b6712024cd5b72c31acdbefce5984 +F ext/fts3/fts3_hash.c d9dba473741445789330c7513d4f65737c92df23c3212784312931641814672a F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c 305ce7fb6036484085b5556a9c8e62acdc7763f0f4cdf5fd538212a9f3720116 -F ext/fts3/fts3_porter.c e19807ce0ae31c1c6e9898e89ecc93183d7ec224ea101af039722a4f49e5f2b8 -F ext/fts3/fts3_snippet.c 7a3d5e2cefbb1cb51fb9c65458670cc269647ede18e1ffd57b513f9b4ec10c3e +F ext/fts3/fts3_porter.c 024417020c57dd1ab39816f5fe6cf45222a857b78a1f6412f040ada1ceabd4ff +F ext/fts3/fts3_snippet.c 627d564878f82479ee3e040c89fb15aebf691d81e11c8e451241033c02b9810d F ext/fts3/fts3_term.c 6a96027ad364001432545fe43322b6af04ed28bb5619ec51af1f59d0710d6d69 -F ext/fts3/fts3_test.c 7a9cb3d61774134211bf4bfdf1adcb581a1a0377f2d050a121ae7ab44baef0e3 -F ext/fts3/fts3_tokenize_vtab.c 7fd9ef364f257b97218b9c331f2378e307375c592f70fd541f714e747d944962 +F ext/fts3/fts3_test.c cc329471e573f95a6ea9fbca87e89dcfa1d355591c80172ffcd759ac521d25d8 +F ext/fts3/fts3_tokenize_vtab.c 66eba6c2baa04b2b15e80d68341b8fd0b4d3831f6b2edb33916a2906ff2d4389 F ext/fts3/fts3_tokenizer.c defede96b5dd5d658edfae77355b9c31ea65236eedc7bbe1adbc50d645cca5bc F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c c1de4ae28356ad98ccb8b2e3388a7fdcce7607b5523738c9afb6275dab765154 F ext/fts3/fts3_unicode.c de426ff05c1c2e7bce161cf6b706638419c3a1d9c2667de9cb9dc0458c18e226 F ext/fts3/fts3_unicode2.c 416eb7e1e81142703520d284b768ca2751d40e31fa912cae24ba74860532bf0f -F ext/fts3/fts3_write.c 81cd8f7e8003e427a1801e04842776b731af26dd93af206e4e66ea5ae319cad1 +F ext/fts3/fts3_write.c 4e967e31f34447b5e5a52583250cd22660cd5774c3ba3054d2eb6725f562b45f F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73 F ext/fts3/tool/fts3view.c 413c346399159df81f86c4928b7c4a455caab73bfbc8cd68f950f632e5751674 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 -F ext/fts3/unicode/mkunicode.tcl 63db9624ccf70d4887836c320eda93ab552f21008f3be7ede551eac3ead62baa +F ext/fts3/unicode/mkunicode.tcl cbf5f7b5c8ce8014bad731f246f2e520eece908465de4778c951ca17003381f1 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl 009cf59c77afa86d137b0cca3e3b1a5efbe2264faa2df233f9a7aa8563926d15 F ext/fts5/fts5.h ff5d3cc88b29e41612bfb29eb723e29e38973de62ca75ba3e8f94ccb67f5b5f2 -F ext/fts5/fts5Int.h 6abff7dd770dc5969c994c281e6e77fc277ce414d56cc4a62c145cc7036b0b67 -F ext/fts5/fts5_aux.c 65a0468dd177d6093aa9ae1622e6d86b0136b8d267c62c0ad6493ad1e9a3d759 -F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 +F ext/fts5/fts5Int.h 4c7380ce83e8f6b5b3216ebe2b33093a20fa72e832a88d14023f827b7d8b9933 +F ext/fts5/fts5_aux.c da4a7a9a11ec15c6df0699d908915a209bcde48f0b04101461316b59f71abffb +F ext/fts5/fts5_buffer.c f1e6d0324d7c55329d340673befc26681a372a4d36086caa8d1ec7d7c53066c7 F ext/fts5/fts5_config.c e7d8dd062b44a66cd77e5a0f74f23a2354cd1f3f8575afb967b2773c3384f7f8 -F ext/fts5/fts5_expr.c 69b8d976058512c07dfe86e229521b7a871768157bd1607cedf1a5038dfd72c9 -F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c f1eec0931548b529ddd7ebd274eaef37de7461fe2b0ebdc9818f37324bdf9494 -F ext/fts5/fts5_main.c 9a1daef7247f9b8a50b4159323e340efa6b0e4bea4fcd83580480f94d4f2c888 +F ext/fts5/fts5_expr.c be9e5f7f11d87e7bd3680832c93c13050fe351994b5052b0215c2ef40312c23a +F ext/fts5/fts5_hash.c a6266cedd801ab7964fa9e74ebcdda6d30ec6a96107fa24148ec6b7b5b80f6e0 +F ext/fts5/fts5_index.c d171f2a507abccb3d524bf461b01f0d3971a9bf221be622ac7c671a991cb62ee +F ext/fts5/fts5_main.c 57933c18efe1058d8871199875c7a59744dabc3904f3aefbf9ff4a4e11fc79e2 F ext/fts5/fts5_storage.c 1ad05dab4830a4e2eaf2900bb143477f93bc17437093582f36f4b818809e88d8 F ext/fts5/fts5_tcl.c 7fb5a3d3404099075aaa2457307cb459bbc257c0de3dbd52b1e80a5b503e0329 -F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee +F ext/fts5/fts5_test_mi.c 4308d5658cb1f5eee5998dcbaac7d5bdf7a2ef43c8192ca6e0c843f856ccee26 F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b F ext/fts5/fts5_tokenize.c 49aea8cc400a690a6c4f83c4cedc67f4f8830c6789c4ee343404f62bcaebca7b -F ext/fts5/fts5_unicode2.c 6f9b0fb79a8facaed76628ffd4eb9c16d7f2b84b52872784f617cf3422a9b043 +F ext/fts5/fts5_unicode2.c 536a6dae41d16edadd6a6b58c56e2ebbb133f0dfe757562a2edbcdc9b8362e50 F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d09885d1dadf80 -F ext/fts5/fts5_vocab.c e4830b00809e5da53bc10f93adc59e321407b0f801c7f4167c0e47f5552267e0 +F ext/fts5/fts5_vocab.c ff0441c4ea165081e8152dec6d29056faa0cdc281a9f218a00e3d7aacc1958bc F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl c5aa7cf7148b6dcffb5b61520ae18212baf169936af734ab265143f59db328fe @@ -203,7 +205,7 @@ F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400 F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c F ext/fts5/test/fts5locale.test 83ba7ee12628b540d3098f39c39c1de0c0440eddff8f7512c8c698d0c4a3ae3c -F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 +F ext/fts5/test/fts5matchinfo.test bc9e74157773db7f00aec1e85587f1145956ebdf1672c136f0f04323b2752aa0 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 F ext/fts5/test/fts5misc.test f4dee7da898d605a6488c5b7afaace3158ed6bb9addff78faa1b37b402b77fb9 @@ -243,7 +245,7 @@ F ext/fts5/test/fts5secure8.test 808ade9d172ed07b24b85c57dd53b6d2b1aba018b4e634d F ext/fts5/test/fts5securefault.test c34a28c7cd2f31a8b8907563889e1329a97da975c08df2d951422bcef8e2ebc5 F ext/fts5/test/fts5simple.test 302cdb4f8a3350b091f4f1bccd82d05610428657f6f9e81c17703ba48267ec40 F ext/fts5/test/fts5simple2.test d10d963a357b8ec77b99032e4c816459b4dbdb1f6eee25eada7ef3ed245cb2dc -F ext/fts5/test/fts5simple3.test 146ec3dc8f5763d6212641c9f0a2f1cba41679353d2add7b963beceb115dc7f4 +F ext/fts5/test/fts5simple3.test 4e03b82e669dc07bf9c9a04c306fa493764bc49c93e539896d87d88bd374fece F ext/fts5/test/fts5synonym.test becc8cea6cfc958a50b30c572c68cbfdf7455971d0fe988202ce67638d2c6cf6 F ext/fts5/test/fts5synonym2.test 58f357b997cf2fedeeb9d0de4db9f880fa96fa2fe27a743bfe7d7b96895bdd87 F ext/fts5/test/fts5tok1.test 1f7817499f5971450d8c4a652114b3d833393c8134e32422d0af27884ffe9cef @@ -284,13 +286,13 @@ F ext/intck/intckfault.test cff3f75dff74abb3edfcb13f6aa53f6436746ab64b09fe5e2028 F ext/intck/sqlite3intck.c 0d10df36e2b7b438aa80ecd3f5e584d41b747586b038258fe6b407f66b81e7c5 F ext/intck/sqlite3intck.h 2b40c38e7063ab822c974c0bd4aed97dabb579ccfe2e180a4639bb3bbef0f1c9 F ext/intck/test_intck.c 4f9eaadaedccb9df1d26ba41116a0a8e5b0c5556dc3098c8ff68633adcccdea8 -F ext/jni/GNUmakefile 59eb05f2a363bdfac8d15d66bed624bfe1ff289229184f3861b95f98a19cf4b2 -F ext/jni/README.md d899789a9082a07b99bf30b1bbb6204ae57c060efcaa634536fa669323918f42 +F ext/jni/GNUmakefile 8a94e3a1953b88cf117fb2a5380480feada8b4f5316f02572cab425030a720b4 +F ext/jni/README.md e3fbd47c774683539b7fdc95a667eb9cd6e64d8510f3ee327e7fa0c61c8aa787 F ext/jni/jar-dist.make 030aaa4ae71dd86e4ec5e7c1e6cd86f9dfa47c4592c070d2e35157e42498e1fa -F ext/jni/src/c/sqlite3-jni.c c1292e690a20c7787a63e8d8ac6e2dfed49c97282ed056a7cfda5da461f0b7d8 -F ext/jni/src/c/sqlite3-jni.h 913ab8e8fee432ae40f0e387c8231118d17053714703f5ded18202912a8a3fbf +F ext/jni/src/c/sqlite3-jni.c 74c35473c1fb1756ee911468d5027e4f989eea6764dff429b1214f76d78c431d +F ext/jni/src/c/sqlite3-jni.h cc5fc5cefe2d63f461a4bad90735b34be0d85e5fe6778ae7ba992fae4d223cdf F ext/jni/src/org/sqlite/jni/annotation/Experimental.java 8603498634e41d0f7c70f661f64e05df64376562ea8f126829fd1e0cdd47e82b -F ext/jni/src/org/sqlite/jni/annotation/NotNull.java 38e7e58a69b26dc100e458b31dfa3b2a7d67bc36d051325526ef1987d5bc8a24 +F ext/jni/src/org/sqlite/jni/annotation/NotNull.java be6cc3e8e114485822331630097cc0f816377e8503af2fc02f9305ff2b353917 F ext/jni/src/org/sqlite/jni/annotation/Nullable.java 56e3dee1f3f703a545dfdeddc1c3d64d1581172b1ad01ffcae95c18547fafd90 F ext/jni/src/org/sqlite/jni/annotation/package-info.java 977b374aed9d5853cbf3438ba3b0940abfa2ea4574f702a2448ee143b98ac3ca F ext/jni/src/org/sqlite/jni/capi/AbstractCollationCallback.java 1afa90d3f236f79cc7fcd2497e111992644f7596fbc8e8bcf7f1908ae00acd6c @@ -298,30 +300,30 @@ F ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java 0b72cdff61533b564d65b63 F ext/jni/src/org/sqlite/jni/capi/AuthorizerCallback.java c045a5b47e02bb5f1af91973814a905f12048c428a3504fbc5266d1c1be3de5a F ext/jni/src/org/sqlite/jni/capi/AutoExtensionCallback.java 74cc4998a73d6563542ecb90804a3c4f4e828cb4bd69e61226d1a51f4646e759 F ext/jni/src/org/sqlite/jni/capi/BusyHandlerCallback.java 7b8e19810c42b0ad21a04b5d8c804b32ee5905d137148703f16a75b612c380ca -F ext/jni/src/org/sqlite/jni/capi/CApi.java 27bbd944ea8c147afd25b93f17dc397f3627611ebe2878944a32ffeffc98e99e -F ext/jni/src/org/sqlite/jni/capi/CallbackProxy.java 57e2d275dcebe690b1fc1f3d34eb96879b2d7039bce30b563aee547bf45d8a8b +F ext/jni/src/org/sqlite/jni/capi/CApi.java 09f3a8cf500ecb6fc9df37044bc157d27c17d34db0eec7d9fb7116edeead5a36 +F ext/jni/src/org/sqlite/jni/capi/CallbackProxy.java 1b3baf5b772f266e8baf8f35f0ddc5bd87fc8c4927ec69115c46fd6fba6701c3 F ext/jni/src/org/sqlite/jni/capi/CollationCallback.java e29bcfc540fdd343e2f5cca4d27235113f2886acb13380686756d5cabdfd065a F ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java 5bfa226a8e7a92e804fd52d6e42b4c7b875fa7a94f8e2c330af8cc244a8920ab F ext/jni/src/org/sqlite/jni/capi/CommitHookCallback.java 482f53dfec9e3ac2a9070d3fceebd56250932aaaf7c4f5bc8de29fc011416e0c F ext/jni/src/org/sqlite/jni/capi/ConfigLogCallback.java b995ca412f59b631803b93aa5b3684fce62e335d1e123207084c054abfd488d4 F ext/jni/src/org/sqlite/jni/capi/ConfigSqlLogCallback.java e5723900b6458bc6288f52187090a78ebe0a20f403ac7c887ec9061dfe51aba7 F ext/jni/src/org/sqlite/jni/capi/NativePointerHolder.java b7036dcb1ef1b39f1f36ac605dde0ff1a24a9a01ade6aa1a605039443e089a61 -F ext/jni/src/org/sqlite/jni/capi/OutputPointer.java 246b0e66c4603f41c567105a21189d138aaf8c58203ecd4928802333da553e7c -F ext/jni/src/org/sqlite/jni/capi/PrepareMultiCallback.java 97352091abd7556167f4799076396279a51749fdae2b72a6ba61cd39b3df0359 +F ext/jni/src/org/sqlite/jni/capi/OutputPointer.java 418a82e705ec80b0dabffacfe11b9fab3cb5f2215dcafcfec083eebf5bce9d20 +F ext/jni/src/org/sqlite/jni/capi/PrepareMultiCallback.java 621cd9d887956fb2c99c9be59ee831c00a0b538dbae7313c525cd2936b5d5647 F ext/jni/src/org/sqlite/jni/capi/PreupdateHookCallback.java efcf57545c5e282d1dd332fa63329b3b218d98f356ef107a9dbe3979be82213a F ext/jni/src/org/sqlite/jni/capi/ProgressHandlerCallback.java 01bc0c238eed2d5f93c73522cb7849a445cc9098c2ed1e78248fa20ed1cfde5b F ext/jni/src/org/sqlite/jni/capi/ResultCode.java 8141171f1bcf9f46eef303b9d3c5dc2537a25ad1628f3638398d8a60cacefa7f F ext/jni/src/org/sqlite/jni/capi/RollbackHookCallback.java e172210a2080e851ebb694c70e9f0bf89284237795e38710a7f5f1b61e3f6787 F ext/jni/src/org/sqlite/jni/capi/SQLFunction.java 0d1e9afc9ff8a2adb94a155b72385155fa3b8011a5cca0bb3c28468c7131c1a5 -F ext/jni/src/org/sqlite/jni/capi/SQLTester.java 0b25cde8c5fa77f3e7ad92368acf195c5c64fb1c5273b8ee71b2d7ab812aab34 +F ext/jni/src/org/sqlite/jni/capi/SQLTester.java 3c0babc067d8560627a9ed1b07979f9d4393464e2282c2fca4832052e982c7bc F ext/jni/src/org/sqlite/jni/capi/ScalarFunction.java 93b9700fca4c68075ccab12fe0fbbc76c91cafc9f368e835b9bd7cd7732c8615 -F ext/jni/src/org/sqlite/jni/capi/TableColumnMetadata.java addf120e0e76e5be1ff2260daa7ce305ff9b5fafd64153a7a28e9d8f000a815f -F ext/jni/src/org/sqlite/jni/capi/Tester1.java e5fa17301b7266c1cbe4bcce67788e08e45871c7c72c153d515abb37e501de0a +F ext/jni/src/org/sqlite/jni/capi/TableColumnMetadata.java 9133bb7685901d2edf07801191284975e33b5583ce09dce1c05202ff91e7bb99 +F ext/jni/src/org/sqlite/jni/capi/Tester1.java 9f4f0041e30712b92a86ddb7e1faf956a0c89a7fb0d5daf88cbae9ec263d8453 F ext/jni/src/org/sqlite/jni/capi/TraceV2Callback.java 0a25e117a0daae3394a77f24713e36d7b44c67d6e6d30e9e1d56a63442eef723 F ext/jni/src/org/sqlite/jni/capi/UpdateHookCallback.java c8bdf7848e6599115d601bcc9427ff902cb33129b9be32870ac6808e04b6ae56 F ext/jni/src/org/sqlite/jni/capi/ValueHolder.java 2ce069f3e007fdbbe1f4e507a5a407fc9679da31a0aa40985e6317ed4d5ec7b5 F ext/jni/src/org/sqlite/jni/capi/WindowFunction.java caf4396f91b2567904cf94bc538a069fd62260d975bd037d15a02a890ed1ef9e -F ext/jni/src/org/sqlite/jni/capi/XDestroyCallback.java f3abb8dd7381f53ebba909437090caf68200f06717b8a7d6aa96fa3e8133117d +F ext/jni/src/org/sqlite/jni/capi/XDestroyCallback.java 9c8cc33995a3df385feaf476a8306d29dbab611ed4f55da736597357bde68620 F ext/jni/src/org/sqlite/jni/capi/package-info.java 08ff986a65d2be9162442c82d28a65ce431d826f188520717c2ecb1484d0a50e F ext/jni/src/org/sqlite/jni/capi/sqlite3.java c6a5c555d163d76663534f2b2cce7cab15325b9852d0f58c6688a85e73ae52f0 F ext/jni/src/org/sqlite/jni/capi/sqlite3_backup.java 6742b431cd4d77e8000c1f92ec66265a58414c86bf3b0b5fbcb1164e08477227 @@ -331,21 +333,21 @@ F ext/jni/src/org/sqlite/jni/capi/sqlite3_stmt.java 293b5fa7d5b5724c87de544654ac F ext/jni/src/org/sqlite/jni/capi/sqlite3_value.java e1d62a257c13504b46d39d5c21c49cf157ad73fda00cc5f34c931aa008c37049 F ext/jni/src/org/sqlite/jni/fts5/Fts5.java e94681023785f1eff5399f0ddc82f46b035977d350f14838db659236ebdf6b41 F ext/jni/src/org/sqlite/jni/fts5/Fts5Context.java 338637e6e5a2cc385d962b220f3c1f475cc371d12ae43d18ef27327b6e6225f7 -F ext/jni/src/org/sqlite/jni/fts5/Fts5ExtensionApi.java 7da0fbb5728f7c056a43e6407f13dd0c7c9c445221267786a109b987f5fc8a9d +F ext/jni/src/org/sqlite/jni/fts5/Fts5ExtensionApi.java 2de029b3a12b16f779f4df6381c5f0cd358dd82bdaf99ec837504a1110b829f3 F ext/jni/src/org/sqlite/jni/fts5/Fts5PhraseIter.java 28045042d593a1f1b9b80d54ec77cbf1d8a1bc95e442eceefa9a3a6f56600b0e F ext/jni/src/org/sqlite/jni/fts5/Fts5Tokenizer.java 3c8f677ffb85b8782f865d6fcbc16200b3375d0e3c29ed541a494fde3011bf49 -F ext/jni/src/org/sqlite/jni/fts5/TesterFts5.java 51e16bf9050af7cb246d17d6a19c001cfc916bf20f425c96625aaccaf74688e8 +F ext/jni/src/org/sqlite/jni/fts5/TesterFts5.java 793a5f6f2f581034dc3b503d5023b05e7e38558a80542148b82527dc2a7bf209 F ext/jni/src/org/sqlite/jni/fts5/XTokenizeCallback.java 1efd1220ea328a32f2d2a1b16c735864159e929480f71daad4de9d5944839167 F ext/jni/src/org/sqlite/jni/fts5/fts5_api.java a8e88c3783d21cec51b0748568a96653fead88f8f4953376178d9c7385b197ea -F ext/jni/src/org/sqlite/jni/fts5/fts5_extension_function.java 9e2b954d210d572552b28aca523b272fae14bd41e318921b22f65b728d5bf978 +F ext/jni/src/org/sqlite/jni/fts5/fts5_extension_function.java 1e721873c62df2eb79f45bbf55b8662625365885b02d1d51915f773de16a90e3 F ext/jni/src/org/sqlite/jni/fts5/fts5_tokenizer.java 92bdaa3893bd684533004d64ade23d329843f809cd0d0f4f1a2856da6e6b4d90 -F ext/jni/src/org/sqlite/jni/test-script-interpreter.md 9bf7e9cab1183287b048bb77baee4b266f0c15baf1b624feec12fbf00cfa7e94 +F ext/jni/src/org/sqlite/jni/test-script-interpreter.md d7987b432870d23f7c72a7804d099fe5ccfb945f519ac90a33e189297fbbfa1c F ext/jni/src/org/sqlite/jni/wrapper1/AggregateFunction.java d5c108b02afd3c63c9e5e53f71f85273c1bfdc461ae526e0a0bb2b25e4df6483 -F ext/jni/src/org/sqlite/jni/wrapper1/ScalarFunction.java 43c43adfb7866098aadaaca1620028a6ec82d5193149970019b1cce9eb59fb03 -F ext/jni/src/org/sqlite/jni/wrapper1/SqlFunction.java 27b141f5914c7cb0e40e90a301d5e05b77f3bd42236834a68031b7086381fafd -F ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java ada39f18e4e3e9d4868dadbc3f7bfe1c6c7fde74fb1fb2954c3f0f70120b805c +F ext/jni/src/org/sqlite/jni/wrapper1/ScalarFunction.java 326ffba29aab836a6ea189703c3d7fb573305fd93da2d14b0f9e9dcf314c8290 +F ext/jni/src/org/sqlite/jni/wrapper1/SqlFunction.java e920f7a031e04975579240d4a07ac5e4a9d0f8de31b0aa7a4be753c98ae596c9 +F ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java c82bc00c1988f86246a89f721d3c41f0d952f33f934aa6677ec87f7ca42519a0 F ext/jni/src/org/sqlite/jni/wrapper1/SqliteException.java 982538ddb4c0719ef87dfa664cd137b09890b546029a7477810bd64d4c47ee35 -F ext/jni/src/org/sqlite/jni/wrapper1/Tester2.java ce45f2ec85facbb73690096547ed166e7be82299e3d92eaa206f82b60a6ec969 +F ext/jni/src/org/sqlite/jni/wrapper1/Tester2.java 08f92d52be2cec28a7b4555479cc54b7ebeeb94985256144eeb727154ec3f85b F ext/jni/src/org/sqlite/jni/wrapper1/ValueHolder.java a84e90c43724a69c2ecebd601bc8e5139f869b7d08cb705c77ef757dacdd0593 F ext/jni/src/org/sqlite/jni/wrapper1/WindowFunction.java c7d1452f9ff26175b3c19bbf273116cc2846610af68e01756d755f037fe7319f F ext/jni/src/tests/000-000-sanity.test c3427a0e0ac84d7cbe4c95fdc1cd4b61f9ddcf43443408f3000139478c4dc745 @@ -378,15 +380,15 @@ F ext/lsm1/lsm-test/lsmtest_tdb4.c cbe230727b9413d244062943371af1421ace472ccb023 F ext/lsm1/lsm-test/lsmtest_util.c 241622db5a332a09c8e6e7606b617d288a37b557f7d3bce0bb97809f67cc2806 F ext/lsm1/lsm-test/lsmtest_win32.c 0e0a224674c4d3170631c41b026b56c7e1672b151f5261e1b4cc19068641da2d F ext/lsm1/lsm.h 0f6f64ff071471cb87bf98beb8386566f30ea001 -F ext/lsm1/lsmInt.h 3bcc280347196e4ed14925b64a07685415238bf41317db0598c8d3f6aaceb9c1 -F ext/lsm1/lsm_ckpt.c ad9a8028d401be9e76f20c4d86d49f33f4fc27785577b452ca955094314a72b4 -F ext/lsm1/lsm_file.c 5486f4a63b19e4d7d972ee2482f29ebdf06c29544f31845f713cccb5199f9ad1 -F ext/lsm1/lsm_log.c a8bf334532109bba05b09a504ee45fc393828b0d034ca61ab45e3940709d9a7c +F ext/lsm1/lsmInt.h 8e4ead919951d700c2d065326b22c8bf055b7fec6e75514c6151b5bb5d9d3030 +F ext/lsm1/lsm_ckpt.c 702a08e9fd92695976d759b259e2258330da99b3f8ac845f145e418cb70902ee +F ext/lsm1/lsm_file.c 4bbc4cb1a558089d884e1e5a17b021d9056ae62add32dd6906d070954c7fe954 +F ext/lsm1/lsm_log.c 9450d193db7a50c96805f10f393ac8b08b2009b6330b7df7ae1e4b442ed219a7 F ext/lsm1/lsm_main.c 87770a9c7e73859fce7620cb79623776ba4b30369086229ad82c3e6eeaf45457 F ext/lsm1/lsm_mem.c 4c51ea9fa285ee6e35301b33491642d071740a0a F ext/lsm1/lsm_mutex.c 378edf0a2b142b4f7640ee982df06d50b98788ea F ext/lsm1/lsm_shared.c c67282a4f2c91e2a3362bdd40a81f9041cd587973ffc4bca8b8fbdab5470dee1 -F ext/lsm1/lsm_sorted.c bc276055afc21e7f23538d39d7cf2722379b56c79778ab7232f710e3374d501c +F ext/lsm1/lsm_sorted.c 1bae85469458793adf753f7c5e695d8d15e33f28bc0ca07ebc9f1b58e853c56c F ext/lsm1/lsm_str.c 65e361b488c87b10bf3e5c0070b14ffc602cf84f094880bece77bbf6678bca82 F ext/lsm1/lsm_tree.c 682679d7ef2b8b6f2fe77aeb532c8d29695bca671c220b0abac77069de5fb9fb F ext/lsm1/lsm_unix.c 11e0a5c19d754a4e1d93dfad06de8cc201f10f886b8e61a4c599ed34e334fc24 @@ -396,39 +398,39 @@ F ext/lsm1/lsm_win32.c 0a4acbd7e8d136dd3a5753f0a9e7a9802263a9d96cef3278cf120bcaa F ext/lsm1/test/lsm1_common.tcl 5ed4bab07c93be2e4f300ebe46007ecf4b3e20bc5fbe1dedaf04a8774a6d8d82 F ext/lsm1/test/lsm1_simple.test a04d08e8661ae6fc53786c67f0bd102c6692f003e859dde03ed9ac3f12e066e5 F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f6078e07335398b0 -F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f238c240 -F ext/misc/amatch.c 5001711cbecdd57b288cb613386789f3034e5beb58fbe0c79f2b3d643ffd4e03 +F ext/misc/README.md af13c3bf4405709eb1e2e1e3a39c3be6a15c3189ab8a0642bb107c6eb223b298 +F ext/misc/amatch.c 2db45b1499b275d8340af6337a13d6216e4ceb2ddb41f4042b9801be7b5e593d F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 F ext/misc/base64.c 73c31eb325c71bae2e27276565e3f674fc095d8b0d7a651becb3b241a4d2fa57 F ext/misc/base85.c a70c885c5c9350261ea6e7b166038eab21a09cf4fceae856ce41fae9c2213b60 F ext/misc/basexx.c 89ad6b76558efbceb627afd5e2ef1d84b2e96d9aaf9b7ecb20e3d00b51be6fcf F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a -F ext/misc/btreeinfo.c cb952620eedf5c0b7625b678f0f08e54d2ec0011d4e50efda5ebdc97f3df7d04 +F ext/misc/btreeinfo.c 8f5e6da2c82ec2f06ee0216e922370a436dafdbb06ffa7a552203515ff9e7ddf F ext/misc/carray.c 34fac63770971611c5285de0a9f0ac67d504eaf66be891f637add9290f1c76a5 F ext/misc/carray.h 503209952ccf2431c7fd899ebb92bf46bf7635b38aace42ec8aa1b8d7b6e98a5 F ext/misc/cksumvfs.c 3a7931dd30667be6348af919f3f9e6188dfd7646b42af8e399a499b327f5bd63 -F ext/misc/closure.c 87e0967772e0087e709887ce7ca9cf13aa32d2096e33b5d3382c8b8d477c6cb1 -F ext/misc/completion.c cb978c88d5577821323617a8ea775ce1b920e02dcdb593858f02044a4d008eea +F ext/misc/closure.c 5559daf1daf742228431db929d1aa86dd535a4224cc634a81d2fd0d1e6ad7839 +F ext/misc/completion.c c3c8b3cc1293c34f04f8746a3adfbfedb43f00d113f8c984a1ed09433317e507 F ext/misc/compress.c 2c79a74330e0e0ba6cb3f7397f8ba5af12d46377ef5d3ee075e12dd8a6ed57f0 -F ext/misc/csv.c 575c2c05fba0a451586a4d42c2c81e711780c41e797126f198d8d9e0a308dcdb +F ext/misc/csv.c 7cae8c2666a058a58fb8994ed2457339a06c97d31c251d9a8445cdd966629890 F ext/misc/dbdump.c b8592f6f2da292c62991a13864a60d6c573c47a9cc58362131b9e6a64f823e01 -F ext/misc/decimal.c 172cf81a8634e6a0f0bedaf71a8372fee63348cf5a3c4e1b78bb233c35889fdc +F ext/misc/decimal.c 228d47e9ef4de60daf5851da19e3ac9ac1eda9e94432816914469501db6a1129 F ext/misc/eval.c 04bc9aada78c888394204b4ed996ab834b99726fb59603b0ee3ed6e049755dc1 F ext/misc/explain.c 606100185fb90d6a1eade1ed0414d53503c86820d8956a06e3b0a56291894f2b -F ext/misc/fileio.c 07cf3109ec6452789e3a989a010234e2a17b599ce82ea29212c948572456abac -F ext/misc/fossildelta.c 8c026e086e406e2b69947f1856fa3b848fff5379962276430d10085b8756b05a -F ext/misc/fuzzer.c 8b28acf1a7e95d50e332bdd47e792ff27054ad99d3f9bc2e91273814d4b31a5a -F ext/misc/ieee754.c 62a90978204d2c956d5036eb89e548e736ca5fac0e965912867ddd7bb833256d -F ext/misc/memstat.c 5b284b78be431c1f5fa154b18eade2407e42c65ed32ec9e9fbf195d114778d7d +F ext/misc/fileio.c 34993b810514c58ff99d7b4254d4a388d844a4774ea77bec68a1dafe8de5ce41 +F ext/misc/fossildelta.c 0aeb099e9627eea693cf21ae47826ecd1e0319b93143bed23090838b2ef0c162 +F ext/misc/fuzzer.c 6b231352815304ba60d8e9ec2ee73d4918e74d9b76bda8940ba2b64e8777515e +F ext/misc/ieee754.c c9dd9d77c8e8e18e0a5706f8ffcccf4ccb6562073709f7453d4d73f5122f4362 +F ext/misc/memstat.c 43705d795090efb78c85c736b89251e743c291e23daaa8382fe7a0df2c6a283d F ext/misc/memtrace.c 7c0d115d2ef716ad0ba632c91e05bd119cb16c1aedf3bec9f06196ead2d5537b F ext/misc/memvfs.c 7dffa8cc89c7f2d73da4bd4ccea1bcbd2bd283e3bb4cea398df7c372a197291b F ext/misc/mmapwarm.c a81af4aaec00f24f308e2f4c19bf1d88f3ac3ce848c36daa7a4cd38145c4080d F ext/misc/nextchar.c 7877914c2a80c2f181dd04c3dbef550dfb54c93495dc03da2403b5dd58f34edd F ext/misc/noop.c f1a21cc9b7a4e667e5c8458d80ba680b8bd4315a003f256006046879f679c5a0 -F ext/misc/normalize.c bd84355c118e297522aba74de34a4fd286fc775524e0499b14473918d09ea61f +F ext/misc/normalize.c 4782be3b74b9bd9f67281036ff1f41e5edcad20ad486171a2d671c4bb2586011 F ext/misc/pcachetrace.c f4227ce03fb16aa8d6f321b72dd051097419d7a028a9853af048bee7645cb405 -F ext/misc/percentile.c 82531c62cd015b9cdca95ad6bb10c3a907ceb570d21ebd4fb7d634c809cfb089 +F ext/misc/percentile.c 72e05a21db20a2fa85264b99515941f00ae698824c9db82d7edfbb16cea8ec80 F ext/misc/prefixes.c 82645f79229877afab08c8b08ca1e7fa31921280906b90a61c294e4f540cd2a6 F ext/misc/qpvtab.c fc189e127f68f791af90a487f4460ec91539a716daf45a0c357e963fd47cc06c F ext/misc/randomjson.c ef835fc64289e76ac4873b85fe12f9463a036168d7683cf2b773e36e6262c4ed @@ -436,27 +438,27 @@ F ext/misc/regexp.c 388e7f237307c7dfbfb8dde44e097946f6c437801d63f0d7ad63f3320d4e F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 -F ext/misc/series.c 2ef2f7452d63a8c59295d8503456576882e4de444c06f17f94c1a6c1ca9be6bb +F ext/misc/series.c e212edb2aa00cc778bf29a6d51c51ebb187fae36267f281b484410a3df065dde F ext/misc/sha1.c cb5002148c2661b5946f34561701e9105e9d339b713ec8ac057fd888b196dcb9 -F ext/misc/shathree.c f3a778f27bf3e71b666a77f28e463a3b931c4dbe4219447e61bb678b4bc121c3 +F ext/misc/shathree.c fd22d70620f86a0467acfdd3acd8435d5cb54eb1e2d9ff36ae44e389826993df F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 -F ext/misc/spellfix.c bcc42ef3fd29429bc01a83e751332b8d4690e65d45008449bdffe7656371487f +F ext/misc/spellfix.c 693c8fd3293087fa821322967a97e59dfa24051e5d2ca7fa85790a4034db6fa4 F ext/misc/sqlar.c a6175790482328171da47095f87608b48a476d4fac78d8a9ff18b03a2454f634 F ext/misc/sqlite3_stdio.c 0fe5a45bd332b30aef2b68c64edbe69e31e9c42365b0fa79ce95a034bca6fbb0 F ext/misc/sqlite3_stdio.h f05eaf5e0258f0573910324a789a9586fc360a57678c57a6d63cfaa2245b6176 F ext/misc/stmt.c b090086cd6bd6281c21271d38d576eeffe662f0e6b67536352ce32bbaa438321 F ext/misc/stmtrand.c 59cffa5d8e158943ff1ce078956d8e208e8c04e67307e8f249dece2436dcb7fc F ext/misc/templatevtab.c 10f15b165b95423ddef593bc5dcb915ec4eb5e0f1066d585e5435a368b8bc22b -F ext/misc/totype.c 75ed9827d19cc3b434fc2aeb60725d4d46e1534373615612a4d1cfdcc3d60922 -F ext/misc/uint.c 053fed3bce2e89583afcd4bf804d75d659879bbcedac74d0fa9ed548839a030b +F ext/misc/totype.c ba11aac3c0b52c685bd25aa4e0f80c41c624fb1cc5ab763250e09ddc762bc3a8 +F ext/misc/uint.c 327afc166058acf566f33a15bf47c869d2d3564612644d9ff81a23efc8b36039 F ext/misc/unionvtab.c 716d385256d5fb4beea31b0efede640807e423e85c9784d21d22f0cce010a785 F ext/misc/urifuncs.c f71360d14fa9e7626b563f1f781c6148109462741c5235ac63ae0f8917b9c751 F ext/misc/uuid.c 5bb2264c1b64d163efa46509544fd7500cb8769cb7c16dd52052da8d961505cf F ext/misc/vfslog.c 3932ab932eeb2601dbc4447cb14d445aaa9fbe43b863ef5f014401c3420afd20 -F ext/misc/vfsstat.c a85df08654743922a19410d7b1e3111de41bb7cd07d20dd16eda4e2b808d269d -F ext/misc/vfstrace.c a73386403c350b210dc788a2d23a0f5cc89c49b176109a66af11b5078c116331 -F ext/misc/vtablog.c 1100250ce8782db37c833e3a9a5c9a3ecf1af5e15b8325572b82e6e0a138ffb5 -F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd +F ext/misc/vfsstat.c 0b23c0a69a2b63dc0ef0af44f9c1fc977300c480a1f7a9814500369d8211f56e +F ext/misc/vfstrace.c 0e4b8b17ac0675ea90f6d168d8214687e06ca3efbc0060aad4814994d82b41fb +F ext/misc/vtablog.c a197addbbd1e267a5476274b74953e1b6f050e28516f0a5fe7d6382753165ee6 +F ext/misc/vtshim.c e5bce24ab8c532f4fdc600148718fe1802cb6ed57417f1c1032d8961f72b0e8f F ext/misc/wholenumber.c 0fa0c082676b7868bf2fa918e911133f2b349bcdceabd1198bba5f65b4fc0668 F ext/misc/zipfile.c b62147ac4985eaac4e368d529b1f4f43ad6bc9ac13d6805d907fff3afdac64d3 F ext/misc/zorder.c b0ff58fa643afa1d846786d51ea8d5c4b6b35aa0254ab5a82617db92f3adda64 @@ -501,13 +503,13 @@ F ext/rbu/rbusave.test 588b618dad9d65c4b13d03a79931de82213503fedc26bdf5789c996ec F ext/rbu/rbusplit.test a6dedd23cf37bcf2e8646d9d7139846e96d60d92f9bc6d6ba6ca8c24c0bd1f72 F ext/rbu/rbutemplimit.test 4980df2d4b74f4dd982add8f78809106154ef5a3c4bdce747422ab0b0481e029 F ext/rbu/rbuvacuum.test 542561741ff2b262e3694bc6012b44694ee62c545845319a06f323783b15311e -F ext/rbu/rbuvacuum2.test ae097d04feb041446a74fac94b24bffeb3fdd60e32b848c5611e507ab702b81b +F ext/rbu/rbuvacuum2.test 1a9bd41f127be2826de2a65204df9118525a8af8d16e61e6bc63ba3ac0010a23 F ext/rbu/rbuvacuum3.test 3ce42695fdf21aaa3499e857d7d4253bc499ad759bcd6c9362042c13cd37d8de F ext/rbu/rbuvacuum4.test ffccd22f67e2d0b380d2889685742159dfe0d19a3880ca3d2d1d69eefaebb205 -F ext/rbu/sqlite3rbu.c c07817e89477b8fc286ab6ed87da5bc82fc3490bbbe9e9b22eb2d900e81ee5dc -F ext/rbu/sqlite3rbu.h 9d923eb135c5d04aa6afd7c39ca47b0d1d0707c100e02f19fdde6a494e414304 -F ext/rbu/test_rbu.c b9727c3394307d058e806c1da0f8bb7b24daf3c6bb94cb10cca88ea4d5c806c0 -F ext/recover/dbdata.c 5295f4f922b60d7035b6b9fd5846b13071b9d97ed7fad8496837bb7640d24771 +F ext/rbu/sqlite3rbu.c c208f72f20784bf2f39244b6cdf8019724a706e1246be289e7621c42aad54695 +F ext/rbu/sqlite3rbu.h e3a5bf21e09ca93ce4e8740e00d6a853e90a697968ec0ea98f40826938bdb68e +F ext/rbu/test_rbu.c 8b6e64e486c28c41ef29f6f4ea6be7b3091958987812784904f5e903f6b56418 +F ext/recover/dbdata.c 10d3c56968a9af6853722a47280805ad1564714d79ea45ac6f7da14bb57fd137 F ext/recover/recover1.test e16d78e94183562abff569967b18b7c77451d7044365516cd0fe14713a284851 F ext/recover/recover_common.tcl a61306c1eb45c0c3fc45652c35b2d4ec19729e340bdf65a272ce4c229cefd85a F ext/recover/recoverbuild.test c74170e0f7b02456af41838afeb5353fdb985a48cc2331d661bbabbca7c6b8e3 @@ -521,23 +523,23 @@ F ext/recover/recoverfault2.test 730e7371bcda769554d15460cb23126abba1be8eca9539c F ext/recover/recoverold.test 68db3d6f85dd2b98e785b6c4da4f5eea4bbe52ccf6674d9a94c7506dc92596aa F ext/recover/recoverpgsz.test 88766fcb810e52ee05335c456d4e5fb06d02b73d3ccb48c52bf293434305e2b1 F ext/recover/recoverrowid.test f948bf4024a5f41b0e21b8af80c60564c5b5d78c05a8d64fc00787715ff9f45f -F ext/recover/recoverslowidx.test 5205a9742dd9490ee99950dabb622307355ef1662dea6a3a21030057bfd81411 +F ext/recover/recoverslowidx.test c90d59c46bb8924a973ac6fbc38f3163cee38cc240256addcab1cf1a322c37dc F ext/recover/recoversql.test e66d01f95302a223bcd3fd42b5ee58dc2b53d70afa90b0d00e41e4b8eab20486 -F ext/recover/sqlite3recover.c 788438a6735108d14ca82cf39c59abf8cde2ee384b962fb93e975eb24f2732fe +F ext/recover/sqlite3recover.c 56c216332ea91233d6d820d429f3384adbec9ecedda67aa98186b691d427cc57 F ext/recover/sqlite3recover.h 011c799f02deb70ab685916f6f538e6bb32c4e0025e79bfd0e24ff9c74820959 -F ext/recover/test_recover.c 072260d7452a3b81aba995b2b3269e7ec2aa7f06725544ba4c25b1b0a1dbc61a +F ext/recover/test_recover.c 3d0fb1df7823f5bc22a0b93955034d16a2dfa2eb1e443e9a0123a77f120599a3 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 -F ext/repair/checkindex.c af5c66463f51462d8a6f796b2c44ef8cfa1116bbdc35a15da07c67a705388bfd +F ext/repair/checkindex.c 7639b4f8928f82c10b950169e60cc45a7f6798df0b299771d17bef025736f657 F ext/repair/sqlite3_checker.c.in 445118c5f7fea958b36fba1b2c464283e60ed4842039ddee3265f1698115ebf7 F ext/repair/sqlite3_checker.tcl a9a2caa9660567257c177a91124d8c0dccdfa341e25c51e6da7f1fd9e601eafa F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc78249442da72ff3f8297398a69 F ext/repair/test/checkindex01.test b530f141413b587c9eb78ff734de6bb79bc3515c335096108c12c01bddbadcec F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c -F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/geopoly.c 0dd4775e896cee6067979d67aff7c998e75c2c9d9cd8d62a1a790c09cde7adca -F ext/rtree/rtree.c 4c58755830a0902435322bf61b2994ae02951a039daefb31cff9457d3e2aa201 +F ext/rtree/README 734aa36238bcd2dee91db5dba107d5fcbdb02396612811377a8ad50f1272b1c1 +F ext/rtree/geopoly.c f0573d5109fdc658a180db0db6eec86ab2a1cf5ce58ec66cbf3356167ea757eb +F ext/rtree/rtree.c 811edc5c2f3e13dcee825a8ec6f2ebe29b34bdb0186184d75d461621173638fa F ext/rtree/rtree.h 4a690463901cb5e6127cf05eb8e642f127012fd5003830dbc974eca5802d9412 F ext/rtree/rtree1.test e0608db762b2aadca0ecb6f97396cf66244490adc3ba88f2a292b27be3e1da3e F ext/rtree/rtree2.test 9d9deddbb16fd0c30c36e6b4fdc3ee3132d765567f0f9432ee71e1303d32603d @@ -564,7 +566,7 @@ F ext/rtree/rtreecheck.test 84eedb43b25b3edf591125266d0bb1cebdfcdcc9c4a56b27d85b F ext/rtree/rtreecirc.test aec664eb21ae943aeb344191407afff5d392d3ae9d12b9a112ced0d9c5de298e F ext/rtree/rtreeconnect.test 225ad3fcb483d36cbee423a25052a6bbae762c9576ae9268332360c68c170d3d F ext/rtree/rtreedoc.test d633982d61542f3bc0a0a2df0382a02cc699ac56cbda01130cde6da44a228490 -F ext/rtree/rtreedoc2.test 194ebb7d561452dcdc10bf03f44e30c082c2f0c14efeb07f5e02c7daf8284d93 +F ext/rtree/rtreedoc2.test 0102b4e60f1baf0039bef7c1c4b39f1d3d77a68a5741513a0c190db28b884835 F ext/rtree/rtreedoc3.test 555a878c4d79c4e37fa439a1c3b02ee65d3ebaf75d9e8d96a9c55d66db3efbf8 F ext/rtree/rtreefuzz001.test 44f680a23dbe00d1061dbde381d711119099846d166580c4381e402b9d62cb74 F ext/rtree/sqlite3rtree.h 03c8db3261e435fbddcfc961471795cbf12b24e03001d0015b2636b0f3881373 @@ -573,10 +575,10 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/util/randomshape.tcl 54ee03d0d4a1c621806f7f44d5b78d2db8fac26e0e8687c36c4bd0203b27dbff F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F ext/rtree/visual01.txt e9c2564083bcd30ec51b07f881bffbf0e12b50a3f6fced0c222c5c1d2f94ac66 -F ext/session/changeset.c 7a1e6a14c7e92d36ca177e92e88b5281acd709f3b726298dc34ec0fb58869cb5 +F ext/session/changeset.c 06b585d977391d498746f002b2d5f9315d0d37888ce9551bd0cb30bfe9a4cf47 F ext/session/changesetfuzz.c 227076ab0ae4447d742c01ee88a564da6478bbf26b65108bf8fac9cd8b0b24aa F ext/session/changesetfuzz1.test 15b629004e58d5ffcc852e6842a603775bb64b1ce51254831f3d12b113b616cd -F ext/session/session1.test cc7e58976c2cc6263fb7ef0c5125a98eafc2f213c75929f986768d2dbc224725 +F ext/session/session1.test 5d2502922d38a1579076863827342379a1609ca6bae78c40691a2be1ed1be2aa x F ext/session/session2.test ee83bb973b9ce17ccce4db931cdcdae65eb40bbb22089b2fe6aa4f6be3b9303f F ext/session/session3.test 2cc1629cfb880243aec1a7251145e07b78411d851b39b2aa1390704550db8e6a F ext/session/session4.test 823f6f018fcbb8dacf61e2960f8b3b848d492b094f8b495eae1d9407d9ab7219 @@ -587,7 +589,7 @@ F ext/session/session9.test 4e3aff62d6b4294498ddbe309076de06f4fddffad4fe5f5a6c03 F ext/session/sessionA.test 1feeab0b8e03527f08f2f1defb442da25480138f F ext/session/sessionB.test c4fb7f8a688787111606e123a555f18ee04f65bb9f2a4bb2aa71d55ce4e6d02c F ext/session/sessionC.test f8a5508bc059ae646e5ec9bdbca66ad24bc92fe99fda5790ac57e1f59fce2fdf -F ext/session/sessionD.test f5c6a762d00bc6ca9d561695c322ba8ecca2bed370486707ef37cf565d2f6c73 +F ext/session/sessionD.test 470ff917dc849e2eb78142ade63aaabd729d773833cff0ff01bca0eda68a21ce F ext/session/sessionE.test b2010949c9d7415306f64e3c2072ddabc4b8250c98478d3c0c4d064bce83111d F ext/session/sessionF.test d37ed800881e742c208df443537bf29aa49fd56eac520d0f0c6df3e6320f3401 F ext/session/sessionG.test 3efe388282d641b65485b5462e67851002cd91a282dc95b685d085eb8efdad0a @@ -599,13 +601,13 @@ F ext/session/sessionalter.test e852acb3d2357aac7d0b920a2109da758c4331bfdf85b41d F ext/session/sessionat.test 00c8badb35e43a2f12a716d2734a44d614ff62361979b6b85419035bc04b45ee F ext/session/sessionbig.test 47c381e7acfabeef17d98519a3080d69151723354d220afa2053852182ca7adf F ext/session/sessionblob.test 87faf667870b72f08e91969abd9f52a383ab7b514506ee194d64a39d8faff00a -F ext/session/sessionchange.test 77c4702050f24270b58070e2cf01c95c3d232a3ef164b70f31974b386ce69903 -F ext/session/sessionconflict.test 8b8cbd98548e2e636ddc17d0986276f60e833fb865617dd4f88ea5bbe3a16b96 +F ext/session/sessionchange.test 6618cb1c1338a4b6df173b6ac42d09623fb71269962abf23ebb7617fe9f45a50 +F ext/session/sessionconflict.test 19e4a53795c4c930bfec49e809311e09b2a9e202d9446e56d7a8b139046a0c07 x F ext/session/sessiondiff.test e89f7aedcdd89e5ebac3a455224eb553a171e9586fc3e1e6a7b3388d2648ba8d F ext/session/sessionfault.test c2b43d01213b389a3f518e90775fca2120812ba51e50444c4066962263e45c11 F ext/session/sessionfault2.test b0d6a7c1d7398a7e800d84657404909c7d385965ea8576dc79ed344c46fbf41c F ext/session/sessionfault3.test ce0b5d182133935c224d72507dbf1c5be1a1febf7e85d0b0fbd6d2f724b32b96 -F ext/session/sessioninvert.test 04075517a9497a80d39c495ba6b44f3982c7371129b89e2c52219819bc105a25 +F ext/session/sessioninvert.test 9018f6a7387ac745084b6374c5e1aa14d648b372e6e1181cfab3df632b662d26 x F ext/session/sessionmem.test f2a735db84a3e9e19f571033b725b0b2daf847f3f28b1da55a0c1a4e74f1de09 F ext/session/sessionnoact.test 4c7ae5c7d351cb5323bca62b6b095592ad24bd90a6713c178b62ab0063d23e19 F ext/session/sessionnoop.test a9366a36a95ef85f8a3687856ebef46983df399541174cb1ede2ee53b8011bc7 @@ -615,11 +617,11 @@ F ext/session/sessionrowid.test 85187c2f1b38861a5844868126f69f9ec62223a03449a98a F ext/session/sessionsize.test 8fcf4685993c3dbaa46a24183940ab9f5aa9ed0d23e5fb63bfffbdb56134b795 F ext/session/sessionstat1.test 5e718d5888c0c49bbb33a7a4f816366db85f59f6a4f97544a806421b85dc2dec F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009ecef8511f4cf3fc -F ext/session/sqlite3session.c 52a680dbb03c4734748b215d95987fb4d95ab23baaf053a01ac2626610963b58 -F ext/session/sqlite3session.h 683ccbf16e2c2521661fc4c1cf918ce57002039efbcabcd8097fa4bca569104b -F ext/session/test_session.c 12e0a2c15fd60f92da4bb29c697c9177ff0c0dbcdc5129a54c47e999f147937a +F ext/session/sqlite3session.c 6b0877fe1ab832aa4b85eaca72606dfd1630a1363a1be7af10ee1042a5ec719e +F ext/session/sqlite3session.h 9bb1a6687b467764b35178dc29bbd2c57ab8cd3acdc8a62f088c34ad17e4fe2b +F ext/session/test_session.c 2ddff73ea368d827028c32851b291416e1008845832feb27b751d15e57e13cc3 F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 2b19c10968295f50ebd1c97c33bfb9038bda8e42fe3be9fb0eed8ae634d45742 +F ext/wasm/GNUmakefile d6b869cf3d3eaaec8cf56adf925910c3e443f455562bcb4a4cd2d1e43c933d8d F ext/wasm/README-dist.txt f01081a850ce38a56706af6b481e3a7878e24e42b314cfcd4b129f0f8427066a F ext/wasm/README.md b89605f65661cf35bf034ff6d43e448cc169b8017fc105d498e33b81218b482c F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff @@ -635,52 +637,52 @@ F ext/wasm/api/README.md c64ec8e84449c069e0217706d9d7d31b3bd53627228b2ba0c3cddbd F ext/wasm/api/extern-post-js.c-pp.js 3fcd904f1204685dea84e5ae90d8b7e65a1dcebab1e838386d8328b74cce46c9 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js 365405929f41ca0e6d389ed8a8da3f3c93e11d3ef43a90ae151e37fa9f75bf41 -F ext/wasm/api/post-js-header.js 54b2b4294501b3866245cc94315a16f5424c0e87729d0fb610fba151593c6d26 +F ext/wasm/api/post-js-header.js 53740d824e5d9027eb1e6fd59e216abbd2136740ce260ea5f0699ff2acb0a701 F ext/wasm/api/pre-js.c-pp.js a614a2c82b12c4d96d8e3ba77330329efc53c4d56a8a7e60ade900f341866cfb F ext/wasm/api/sqlite3-api-cleanup.js 3ac1786e461ada63033143be8c3b00b26b939540661f3e839515bb92f2e35359 -F ext/wasm/api/sqlite3-api-glue.c-pp.js 5c0209e6a28164b4c2c1a34b0bb4aee3b7b1a264988d7e71fac08b8ede5b7ae3 +F ext/wasm/api/sqlite3-api-glue.c-pp.js bd8ae059bb3ea489666b2f167025833e55dc5941be99c129a56fc25e541d37ce F ext/wasm/api/sqlite3-api-oo1.c-pp.js f3a8e2004c6625d17946c11f2fb32008be78bc5207bf746fc77d59848813225f -F ext/wasm/api/sqlite3-api-prologue.js 5ff913355b3144f1c9719d0406667fa6e13eb813c71ed7ce29440e2e65363e82 -F ext/wasm/api/sqlite3-api-worker1.c-pp.js 5cc22a3c0d52828cb32aad8691488719f47d27567e63e8bc8b832d74371c352d +F ext/wasm/api/sqlite3-api-prologue.js 8708570165f5b4bce9a78ccd91bc9ddf8735970ac1c4d659e36c9a7d9a644bb4 +F ext/wasm/api/sqlite3-api-worker1.c-pp.js f646a65257973b8c4481f8a6a216370b85644f23e64b126e7ae113570587c0ab F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 3774befd97cd1a5e2895c8225a894aad946848c6d9b4028acc988b5d123475af +F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js bb5e96cd0fd6e1e54538256433f1c60a4e3095063c4d1a79a8a022fc59be9571 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 9b86ca2d8276cf919fbc9ba2a10e9786033b64f92c2db844d951804dee6c4b4e +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 0f68a64e508598910e7c01214ae27d603dfc8baec6a184506fafac603a901931 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 4ab0704ee198de7d1059eccedc7703c931510b588d10af0ee36ea5b3ebbac284 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js e809739d71e8b35dfe1b55d24d91f02d04239e6aef7ca1ea92a15a29e704f616 -F ext/wasm/api/sqlite3-wasm.c 6f9d8529072d072359cd22dc5dfb0572c524684686569cfbd0f9640d7619fc10 -F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 46f303ba8ddd1b2f0a391798837beddfa72e8c897038c8047eda49ce7d5ed46b +F ext/wasm/api/sqlite3-wasm.c 7ea3d4a286a2241f6fcc65c9ff10fc04ee5590f80f40763a57001dd5e93aa4c4 +F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc65debfe43b81fc39fb25c40ad0cc1946bd82580fbf644351107b544d6177ee F ext/wasm/api/sqlite3-worker1.c-pp.js 5e8706c2c4af2a57fbcdc02f4e7ef79869971bc21bb8ede777687786ce1c92d5 F ext/wasm/batch-runner-sahpool.html e9a38fdeb36a13eac7b50241dfe7ae066fe3f51f5c0b0151e7baee5fce0d07a7 F ext/wasm/batch-runner-sahpool.js 54a3ac228e6c4703fe72fb65c897e19156263a51fe9b7e21d2834a45e876aabd F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 F ext/wasm/batch-runner.js 05ec254f5dbfe605146d9640b3db17d6ef8c3fbef6aa8396051ca72bb5884e3f -F ext/wasm/c-pp.c 6d131069644964223305582a80973477fa8b06b57306781690d7874ebd3a4f84 +F ext/wasm/c-pp.c cca55c5b55ebd8d29916adbedb0e40baa12caa9a2e8429f812683c308f9b0e9a F ext/wasm/common/SqliteTestUtil.js 7adaeffef757d8708418dc9190f72df22367b531831775804b31598b44f6aa51 F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15 F ext/wasm/common/testing.css e97549bab24126c24e0daabfe2de9bb478fb0a69fdb2ddd0a73a992c091aad6f -F ext/wasm/common/whwasmutil.js c2e459286c1ada789cda6b17761bb1eea6034be572468eed78c049354f1051ba +F ext/wasm/common/whwasmutil.js 5e7fe4a2547831e003356c201ca28d9a0d6a19ab1fe2c8f63f82464fecd2cef7 F ext/wasm/config.make.in 4bc43443f768a61efd43cf995a5e618f58ac9afc0936706014193537d82c41cb F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 F ext/wasm/demo-123.js c7b3cca50c55841c381a9ca4f9396e5bbdc6114273d0b10a43e378e32e7be5bf F ext/wasm/demo-jsstorage.html 409c4be4af5f207fb2877160724b91b33ea36a3cd8c204e8da1acb828ffe588e -F ext/wasm/demo-jsstorage.js 44e3ae7ec2483b6c511384c3c290beb6f305c721186bcf5398ca4e00004a06b8 +F ext/wasm/demo-jsstorage.js 42131ddfa18e817d0e39ac63745e9ea31553980a5ebd2222e04d4fac60c19837 F ext/wasm/demo-worker1-promiser.c-pp.html 635cf90685805e21772a5f7a35d1ace80f98a9ef7c42ff04d7a125ddca7e5db8 -F ext/wasm/demo-worker1-promiser.c-pp.js fcc628cb42fcfaf07d250477801de1e6deb1e319d003976612a0db8d76b9fccc +F ext/wasm/demo-worker1-promiser.c-pp.js af168699d3cab1c27ad2364ebe06cd49db300bdbf404e23b00d5742ed52816ba F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d -F ext/wasm/demo-worker1.js 836bece8615b17b1b572584f7b15912236a5947fe8c68b98d2737d7e287447ef +F ext/wasm/demo-worker1.js 08720227e98fa5b44761cf6e219269cee3e9dd0421d8d91459535da776950314 F ext/wasm/dist.make 92ef4ffe33022a50f92d602acabad10bd8dd91759f3eb7df27fc6d7d37072b96 F ext/wasm/example_extra_init.c 2347cd69d19d839ef4e5e77b7855103a7fe3ef2af86f2e8c95839afd8b05862f -F ext/wasm/fiddle.make d4969f0322a582c57a22ce3541f10a5b09a609d14eab32891f613f43b3c14d8b +F ext/wasm/fiddle.make c6d7a3d6cc03bb5f21acb295c1233820d0dbf5c6a89b28dc2e093edcc001c45a F ext/wasm/fiddle/fiddle-worker.js 850e66fce39b89d59e161d1abac43a181a4caa89ddeea162765d660277cd84ce -F ext/wasm/fiddle/fiddle.js b444a5646a9aac9f3fc06c53d78af5e1912eb235d69a8e6010723e4eb0e9d4a1 -F ext/wasm/fiddle/index.html c79b1741cbeba78f88af0a84cf5ec7de87a909a6a8d10a369b1f4824c66c2088 +F ext/wasm/fiddle/fiddle.js 2a2f27b4be2674f501fff61c4a09e44dcf2295731a26b5c28e439f3a573bd269 +F ext/wasm/fiddle/index.html 7fcfb221165183bef0e05d5af9ceb79b527e799b1708ab05de0ec0eaebd5b7bf F ext/wasm/index-dist.html 56132399702b15d70c474c3f1952541e25cb0922942868f70daf188f024b3730 -F ext/wasm/index.html e4bbffdb3d40eff12b3f9c7abedef91787e2935620b7f8d40f2c774b80ad8fa9 -F ext/wasm/jaccwabyt/jaccwabyt.js 1264710db3cfbcb6887d95665b7aeba60c1126eaef789ca4cf1a4a17d5bc7f54 -F ext/wasm/jaccwabyt/jaccwabyt.md 59a20df389abcc3606eb4eaea7fb7ba14504beb3e345dbea9b99a0618ba3bec8 -F ext/wasm/mkwasmbuilds.c baf6636e139e2c1e3b56e8dc26073ec80f6d14ae1876b023985315f43ccf312b +F ext/wasm/index.html bcaa00eca521b372a6a62c7e7b17a870b0fcdf3e418a5921df1fd61e5344080d +F ext/wasm/jaccwabyt/jaccwabyt.js 6e4f26d0edb5c2e7d381b7eff1924832a040a12274afab2d1e1789027e9f6c5c +F ext/wasm/jaccwabyt/jaccwabyt.md 1128e3563e7eff90b5a373395251fc76cb32386fad1fea6075b0f34a8f1b9bdf +F ext/wasm/mkwasmbuilds.c 5b096a3c9fdf6e67eb20329dc685f995e820248a67fa970633c2c8f49bf472ad F ext/wasm/module-symbols.html dc476b403369b26a1a23773e13b80f41b9a49f0825e81435fe3600a7cfbbe337 F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96 F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 @@ -696,13 +698,18 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js 05a0143c44a4114aad0ed40ce73c528febc3e0d6b69f48a51c895d7030015b74 +F ext/wasm/tester1.c-pp.js 419717b16e12703487a7ccf3ea4e63d693bdfbf7657e55a7e6c559bbccf027d3 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 +F ext/wasm/tests/opfs/sahpool/digest-worker.js b0ab6218588f1f0a6d15a363b493ceaf29bfb87804d9e0165915a9996377cf79 +F ext/wasm/tests/opfs/sahpool/digest.html 206d08a34dc8bd570b2581d3d9ab3ecad3201b516a598dd096dcf3cf8cd81df8 +F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca01385e2732294b53f4c842328 +F ext/wasm/tests/opfs/sahpool/sahpool-pausing.js f264925cfc82155de38cecb3d204c36e0f6991460fff0cb7c15079454679a4e2 +F ext/wasm/tests/opfs/sahpool/sahpool-worker.js bd25a43fc2ab2d1bafd8f2854ad3943ef673f7c3be03e95ecf1612ff6e8e2a61 F ext/wasm/wasmfs.make 68999f5bd8c489239592d59a420f8c627c99169bbd6fa16a404751f757b9f702 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk ae183c31bef504b89d1bbcc5b59a07e5eedfcc55bb543b4aa254808f4dd85149 +F main.mk 34290a772ec671de1fa5defd4fa4074aad24b1ea7eaabebba071e30564c6498c F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -712,45 +719,44 @@ F mptest/multiwrite01.test dab5c5f8f9534971efce679152c5146da265222d F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 acdff36db796e2d00225b911d3047d580cd136547298435426ce9d40347973cc F sqlite3.pc.in 0977c03a4da7c4204bd60e784a0efb8d51a190448aba78a4e973fe7192bdaf03 -F src/alter.c aa93e37e4a36a0525bbb2a2aeda20d2018f0aa995542c7dc658e031375e3f532 -F src/analyze.c 9a8b67239d899ac12289db5db3f5bfe7f7a0ad1277f80f87ead1d048085876eb -F src/attach.c 3a5cb9ee4aad6c5b22268287340a4f2f7b07959b7a522201be30fee23cd802e9 +F src/alter.c fc7bbbeb9e89c7124bf5772ce474b333b7bdc18d6e080763211a40fde69fb1da +F src/analyze.c 03bcfc083fc0cccaa9ded93604e1d4244ea245c17285d463ef6a60425fcb247d +F src/attach.c 9af61b63b10ee702b1594ecd24fb8cea0839cfdb6addee52fba26fa879f5db9d F src/auth.c 54ab9c6c5803b47c0d45b76ce27eff22a03b4b1f767c5945a3a4eb13aa4c78dc F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 -F src/bitvec.c 9eac5f42c11914d5ef00a75605bb205e934f435c579687f985f1f8b0995c8645 -F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522 -F src/btree.c 63ca6b647342e8cef643863cd0962a542f133e1069460725ba4461dcda92b03c +F src/bitvec.c 782cc29b42b47e7ec6348eb0aaf9ffe60063f498387e7249f458d445af4b53e9 +F src/btmutex.c 30dada73a819a1ef5b7583786370dce1842e12e1ad941e4d05ac29695528daea +F src/btree.c da98489a981c347cc3a3982ea2810bbb583511a73cc34762547f30dbb4cda7f0 F src/btree.h 18e5e7b2124c23426a283523e5f31a4bff029131b795bb82391f9d2f3136fc50 -F src/btreeInt.h 98aadb6dcb77b012cab2574d6a728fad56b337fc946839b9898c4b4c969e30b6 -F src/build.c 86a7efd263eb7d6a4dc24dda0981c55202b22f84765023d3ef4878d228bee5c6 +F src/btreeInt.h 9c0f9ea5c9b5f4dcaea18111d43efe95f2ac276cd86d770dce10fd99ccc93886 +F src/build.c 67c1db4c5e89a8519fe9b6dafc287f6bc3627696b5b8536dc5e06db570d8c05f F src/callback.c acae8c8dddda41ee85cfdf19b926eefe830f371069f8aadca3aa39adf5b1c859 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e -F src/ctime.c d35723024b963edce9c0fad5b3303e8bb9266083784844baed10a6dedfe26f3b -F src/date.c 842c08ac143a56a627b05ac51d68624f2b7b03e3b4cba596205e735eed64ee57 -F src/dbpage.c 2e677acb658a29965e55398bbc61161cb7819da538057c8032adac7ab8e4a8c0 +F src/date.c 9db4d604e699a73e10b8e85a44db074a1f04c0591a77e2abfd77703f50dce1e9 +F src/dbpage.c fcb1aafe00872a8aff9a7aa0ef7ff1b01e5817ec7bbd521f8f3e1e674ac8d609 F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c F src/delete.c 03a77ba20e54f0f42ebd8eddf15411ed6bdb06a2c472ac4b6b336521bf7cea42 -F src/expr.c 3fc2f37dbc172293239600cc6f041830e7e2bc3ddaaebfbb2a89520317b4a09f +F src/expr.c f16fa5cbd849991462edf1d31bb7def5b970bb9611afcb4ea21c77e88e52a220 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f -F src/func.c 1ab83fd94f97af9797bdf1027169e3e19482fce06b090c3acceb4bf92ae452cd +F src/func.c 7686ea382b20e8bfe2ab9de76150c99ee7b6e83523561f3c7787e0f68cb435c2 F src/global.c a19e4b1ca1335f560e9560e590fc13081e21f670643367f99cb9e8f9dc7d615b -F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220 -F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 +F src/hash.c 73934a7f7ab1cb110614a9388cb516893b0cf5b7b69e4fd1a0780ac4ce166be7 +F src/hash.h 46b92795a95bfefb210f52f0c316e9d7cdbcdd7e7fcfb0d8be796d3a5767cddf F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c db8bfff30fd7f71812651df3ddf5d1624b9e19104b31e349cd9055bbc9d622c4 -F src/json.c 68a98c020c22127f2d65f08855f7fc7460ff352a6ce0b543d8931dde83319c22 +F src/insert.c d05934dfab2c5c0c480fc6fd2038f11215661de08ea6ff38d2563216bd555c1b +F src/json.c cb87977b1ee25ee7d27505d65a9261b687395bf895342c8ba566b7c01aee2047 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa -F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 -F src/main.c 54e7f07a41e3424f5c2f39db9e96d5c33bbc1555362478d90700a39b694994e6 +F src/loadext.c d7edd8e671237539d795d30daaf888908a2c82e99bade4c78f3be021e8b7d655 +F src/main.c 07f78d917ffcdf327982840cfd8e855fd000527a2ea5ace372ce4febcbd0bf97 F src/malloc.c 410e570b30c26cc36e3372577df50f7a96ee3eed5b2b161c6b6b48773c650c5e F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2 F src/mem2.c c8bfc9446fd0798bddd495eb5d9dbafa7d4b7287d8c22d50a83ac9daa26d8a75 F src/mem3.c 30301196cace2a085cbedee1326a49f4b26deff0af68774ca82c1f7c06fda4f6 F src/mem5.c b7da5c10a726aacacc9ad7cdcb0667deec643e117591cc69cf9b4b9e7f3e96ff -F src/memdb.c 16679def118b5fd75292a253166d3feba3ec9c6189205bf209643ecdb2174ecc +F src/memdb.c a3feb427cdd4036ea2db0ba56d152f14c8212ca760ccb05fb7aa49ff6b897df3 F src/memjournal.c c283c6c95d940eb9dc70f1863eef3ee40382dbd35e5a1108026e7817c206e8a0 F src/msvc.h 80b35f95d93bf996ccb3e498535255f2ef1118c78764719a7cd15ab4106ccac9 F src/mutex.c 06bcd9c3dbf2d9b21fcd182606c00fafb9bfe0287983c8e17acd13d2c81a2fa9 @@ -764,127 +770,126 @@ F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06 F src/os_kv.c 4d39e1f1c180b11162c6dc4aa8ad34053873a639bac6baae23272fc03349986a F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107 -F src/os_unix.c 4c73f89479d90412cb736a180e9ef89ac1495a158753a7f5de1260c197bc8e1f -F src/os_win.c 49c7725b500f5867e8360e75eeb30f9d70b62fa1f05c8a101da627210578df32 -F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 3a1c4e7f69af482e33c8cba8a75afe0dda0ea6391240adac22b040ce1bdeef44 +F src/os_unix.c 04e054ab86d86a7be99ebe5265922687791a40df5afc781d059beb47f4a40acd +F src/os_win.c b8d3cfdf2f40e2f9715b7d8df64f3c0c7ee18743a2dd0c4fc70c1d57fa1aadc7 +F src/os_win.h 4c247cdb6d407c75186c94a1e84d5a22cbae4adcec93fcae8d2bc1f956fd1f19 +F src/pager.c 23c0f17deb892da6b32fef1f465507df7ab5cd01d774288cb43695658a649259 F src/pager.h 6137149346e6c8a3ddc1eeb40aee46381e9bc8b0fcc6dda8a1efde993c2275b8 -F src/parse.y 5dce477d23c6cd41da97ff9bc4ef93fba0e0a0aaa72a15ddb8a3f71618d76cac +F src/parse.y e426d7323311554c75b0aebc426d0fe3c88d9777ffefed236f343ad9e661dc4c F src/pcache.c 588cc3c5ccaaadde689ed35ce5c5c891a1f7b1f4d1f56f6cf0143b74d8ee6484 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 -F src/pcache1.c 49516ad7718a3626f28f710fa7448ef1fce3c07fd169acbb4817341950264319 -F src/pragma.c ce1182217aa540e034c6da2f17515e3706bf52c837e8222361be9ccd7a9d495a -F src/pragma.h 5edad5943dba8ec69727e719fb841b6b505204b9cdbfc50142806f79458fcce6 +F src/pcache1.c 131ca0daf4e66b4608d2945ae76d6ed90de3f60539afbd5ef9ec65667a5f2fcd +F src/pragma.c 30b535d0a66348df844ee36f890617b4cf45e9a22dcbc47ec3ca92909c50aaf1 F src/prepare.c 1832be043fce7d489959aae6f994c452d023914714c4d5457beaed51c0f3d126 -F src/printf.c 96f7f8baeedc7639da94e4e7a4a2c200e2537c4eec9e5e1c2ffc821f40eb3105 +F src/printf.c 3b91c334f528359145f4dde0dedd945bbb21044d0825ea064934d7222d61662c F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c -F src/resolve.c c8a5372b97b2a2e972a280676f06ddb5b74e885d3b1f5ce383f839907b57ef68 +F src/resolve.c d40fe18d7c2fd0339f5846ffcf7d6809866e380acdf14c76fb2af87e9fe13f64 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c c94f7fe33a3f481cc68472ba845301e343e0412c216fee06f58ad64e7e88bf45 -F src/shell.c.in b377a59822f207106424f08aead37e78b609222e98f86f04cc8a03563ccf3237 -F src/sqlite.h.in d2902f13ace94d3d3609646bd6d12a2d7a4f6cbdf6a5a4097580ac305f54c3f0 -F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 -F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h aeb5ed59db92bfa2bf987366769a48681735c9da58e88e76f1f47d1fc26a2e46 -F src/sqliteLimit.h 1bbdbf72bd0411d003267ffebc59a262f061df5653027a75627d03f48ca30523 -F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b +F src/select.c 7a21df5db6bb1a4c1bb6d9fb76c8e2485a22ff8306519ad69d8ddf0d5fa10903 +F src/shell.c.in ba53a52dafb167ac6320703da741386c34fbcabe8c078a188bb9f89808e3ef8f +F src/sqlite.h.in 22882ddd3a70751aa8864c81993ee4562ed54c2c508b6270f75e223ffee38e1b +F src/sqlite3.rc 015537e6ac1eec6c7050e17b616c2ffe6f70fca241835a84a4f0d5937383c479 +F src/sqlite3ext.h 0bfd049bb2088cc44c2ad54f2079d1c6e43091a4e1ce8868779b75f6c1484f1e +F src/sqliteInt.h c7af428bc3087678f0e6ac7685b9968c7259b428de706ab8c212e486c4ead4d9 +F src/sqliteLimit.h 6d817c28a8f19af95e6f4921933b7fbbca48a962bce0eb0ec81e8bb3ef38e68b +F src/status.c 0e72e4f6be6ccfde2488eb63210297e75f569f3ce9920f6c3d77590ec6ce5ffd F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 -F src/tclsqlite.c 5c1e367e26711044730c93d4b81312170918a8d1fe811f45be740ab48f7de8c1 +F src/tclsqlite.c d0e63ffe7944dd223bf62066d9f982cbee1978811c7fbfd889f4ba9c5baed3d1 F src/tclsqlite.h 65e2c761446e1c9fa0342b7d2612a703483643c8b6a316d12a65b745a4727395 -F src/test1.c 9d2da51b4c33633e7370e4068af6d16d2c52b22a5810ec012ac32e77f8397b64 -F src/test2.c 7ebc518e6735939d8979273a6f7b1d9b5702babf059f6ad62499f7f60a9eb9a3 -F src/test3.c e7573aa0f78ee4e070a4bc8c3493941c1aa64d5c66d4825c74c0f055451f432b -F src/test4.c 13e57ae7ec7a959ee180970aef09deed141252fe9bb07c61054f0dfa4f1dfd5d -F src/test5.c bb87279ed12e199486894e6c83e58dc8cd1de9524ace171d59219d3ab696a0c1 -F src/test6.c 763b92489f11f4a77b773f0d3b8369ab0edd5292ac794043062c337019f12d8a +F src/test1.c 9b54135e5f1352f06b1d23d7c183f124c1f33de6ea8997cd801f0f215c43591d +F src/test2.c 62f0830958f9075692c29c6de51b495ae8969e1bef85f239ffcd9ba5fb44a5ff +F src/test3.c 432646f581d8af1bb495e58fc98234380250954f5d5535e507fc785eccc3987a +F src/test4.c 0ac87fc13cdb334ab3a71823f99b6c32a6bebe5d603cd6a71d84c823d43a25a0 +F src/test5.c 38fa635a70a94f2aa8b47ecbab15d821386205d27ad4159c3551ab3ba45efa11 +F src/test6.c 9722054d37257459f1b8988e59e7db1dd630bfb291f16b2759764e778a9d1899 F src/test8.c 206d8f3cc73950d252906656e2646b5de0d580b07187b635fcb3edd8c2c5fbc0 -F src/test9.c 7a708ad27f8fda79113e5e15de66632710958c401e64c2f22bc04e2f5a7a1b62 +F src/test9.c df9ddc7db6ef1b8cf745866ee229090779728bcbe660c7f297d3127ab21d92af F src/test_autoext.c 14d4bbd3d0bd1eec0f6d16b29e28cf1e2d0b020d454835f0721a5f68121ac10f -F src/test_backup.c bd901e3c116c7f3b3bbbd4aae4ce87d99b400c9cbb0a9e7b4610af451d9719a7 +F src/test_backup.c a2bfd90d2ff2511b8635507bdb30fa9b605ade19c16b533066cae3077f5bdb72 F src/test_bestindex.c 3401bee51665cbf7f9ed2552b5795452a8b86365e4c9ece745b54155a55670c6 -F src/test_blob.c bcdf6a6c22d0bcc13c41479d63692ef413add2a4d30e1e26b9f74ab85b9fb4d5 +F src/test_blob.c 77b994e17f2c87055f44fd96c9a206c5a7155bae2cda2769af60c2f3582f962c F src/test_btree.c 28283787d32b8fa953eb77412ad0de2c9895260e4e5bd5a94b3c7411664f90d5 -F src/test_config.c bff5e1625c007f14a9ea4d346b6a741149b5e1f885c1c7ae69bb28a8ddade151 -F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f +F src/test_config.c 7f412406592794636d6226268e26d413850a9f799bc5f3c01afc2820b165fca8 +F src/test_delete.c d0e8f6dc55cfc98a7c27c057fb88d512260564bf0b611482656c68b8f7f401ed F src/test_demovfs.c 3efa2adf4f21e10d95521721687d5ca047aea91fa62dd8cc22ac9e5a9c942383 F src/test_devsym.c 649434ed34d0b03fbd5a6b42df80f0f9a7e53f94dd1710aad5dd8831e91c4e86 -F src/test_fs.c c411c40baba679536fc34e2679349f59d8225570aed3488b5b3ef1908525a3d5 +F src/test_fs.c 6711fd4c6c05914b613cfc99918a24978452f999ce03fc8f89c9794c03b20a5b F src/test_func.c 858d4dddb7acf88222ebcba7cffb585f6dde83e4a15b838c0d05ccdf8d5219b9 -F src/test_hexio.c 7449504e4bde876ba91b202617a9228c7c8c2e7bd8b957302f3803ac0e9e353c -F src/test_init.c 17313332d58e90defc527129d5eda4a08bd6b6e8de7207a231523c8d98fb445e +F src/test_hexio.c a90baa0a8ab5e7cfe2216a61c9a31cfd1f8378353a3d23e25fa94c09aa755bb0 +F src/test_init.c 1649e02448f536e53172f6b1ff873254fe9a0c6c8a4502a2d25c0cc7b11945ea F src/test_intarray.c 3fcf8ca7bb5c8776ea83f6aa9b66f8df0d1f37a99207b0097c8486f9c15cedbf F src/test_intarray.h 6c3534641108cd1bea517a8e117dcba237081310a29a4c35bd2190caa8972293 F src/test_journal.c a0b9709b2f12b1ec819eea8a1176f283bca6d688a6d4a502bd6fd79786f4e287 F src/test_loadext.c 337056bae59f80b9eb00ba82088b39d0f4fe6dfd -F src/test_malloc.c 4954125ee89aa51d9f641d5cb272cc93ca4cb03dcc7c9c941d70210354c69567 +F src/test_malloc.c f1901a7a32fadff1978d6aca8e3bd2c6cc1e41b790ff19d369cc0367116a1828 F src/test_md5.c 811a45330c9391933360f998156a8907ee29909c828ab83ac05d329942cbea8f -F src/test_multiplex.c b99d7f43ec859e6b93a40aaa5455420b3ad133053cce3db739498d29ea30735f +F src/test_multiplex.c 82f0aa8eee629b6949782cfab8782ed35a9b56dc80d12877af52147f304d22b8 F src/test_multiplex.h f0ff5b6f4462bfd46dac165d6375b9530d08089b7bcbe75e88e0926110db5363 -F src/test_mutex.c f10fcbc2086b19c7b0ddf2752caf2095e42be74d8d7f6093619445b43b1f777b +F src/test_mutex.c dacae6790956c0d4e705aaed2090227792e291b0496cccd688e9994c1e21f740 F src/test_onefile.c f31e52e891c5fef6709b9fcef54ce660648a34172423a9cbdf4cbce3ba0049f4 -F src/test_osinst.c 7aa3feaa3a1da1b5f75bde2ce958dbfe14ec484f065bb2b5b9727d8851fa089b +F src/test_osinst.c 269039d9c0820a02ee928014c30860d57ee757ecda54df42e463d0ca1377b835 F src/test_pcache.c 496da3f7e2ca66aefbc36bbf22138b1eff43ba0dff175c228b760fa020a37bd0 -F src/test_quota.c 07369655d24c3f3fbdbd8fd8f42e856a054a7497846ca1c83ed4be68152a251f +F src/test_quota.c 180e87437250bed7e17e4e61c106730939e39fec9be73d28961f27f579a92078 F src/test_quota.h 2a8ad1952d1d2ca9af0ce0465e56e6c023b5e15d F src/test_rtree.c d844d746a3cc027247318b970025a927f14772339c991f40e7911583ea5ed0d9 F src/test_schema.c b06d3ddc3edc173c143878f3edb869dd200d57d918ae2f38820534f9a5e3d7d9 -F src/test_sqllog.c 540feaea7280cd5f926168aee9deb1065ae136d0bbbe7361e2ef3541783e187a -F src/test_superlock.c 18355ca274746aa6909e3744163e5deb1196a85d5bc64b9cd377273cef626da7 -F src/test_syscall.c 9ad7ab39910c16d29411678d91b0d27a7a996a718df5ee93dcd635e846d0275c +F src/test_sqllog.c 5abf04865758c0a3915b4ec2b2ee5ab75f74c00e2f05bf503b9083e0ab6829d7 +F src/test_superlock.c 3387fc794a68d8c6b6ed059aabacbfe870dc502c5cf65562f36aac78b4a4d629 +F src/test_syscall.c c5bf039261973135068aa68f4d185a6147333dcf266977989f8245b3a1968f1b F src/test_tclsh.c c01706ac60bd3176754d3ccd37da74c6ad97c2e14489f8ed71b497c1c0ac0dd4 F src/test_tclvar.c ae873248a0188459b1c16ca7cc431265dacce524399e8b46725c2b3b7e048424 -F src/test_thread.c d7a8bcea7445f37cc2a1f7f81dd6059634f45e0c61bfe80182b02872fb0328bb +F src/test_thread.c 3edb4a5b5aeb1a6e9a275dccc848ac95acab7f496b3e9230f6d2d04953a2b862 F src/test_vdbecov.c 5c426d9cd2b351f5f9ceb30cabf8c64a63bfcad644c507e0bd9ce2f6ae1a3bf3 -F src/test_vfs.c f298475e468c7e14945b20af885917181090c265aa3c4ade897849c9fbd396f2 -F src/test_windirent.c a895e2c068a06644eef91a7f0a32182445a893b9a0f33d0cdb4283dca2486ac1 -F src/test_windirent.h da2e5b73c32d09905fbdd00f27cd802212a32a58ead882736fe4f5eb775ebc50 +F src/test_vfs.c b4135c1308516adf0dfd494e6d6c33114e03732be899eace0502919b674586b5 +F src/test_windirent.c cbee2b9a118b56b5a26b99c895adcbc861587bc66d24b88d1ad6e4c1d09dad7b +F src/test_windirent.h f8245d8002aa0d4322192d35b0f8bbfc757479e90d60fd0beb386d3913f72cdd F src/test_window.c 6d80e11fba89a1796525e6f0048ff0c7789aa2c6b0b11c80827dc1437bd8ea72 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c 375a772e2342274f4bf73605a70633237da09deed00a9bf4c4816a56777ea7c9 +F src/tokenize.c 3e37ac2b6cbb9b0abe33827b0153c27595269afd7152b48019808974481aca2c F src/treeview.c d85ce76e6d1498d781957c07cb234da6d77ce0ed2d196480d516f54dabc62279 -F src/trigger.c 247e2d712d5edc6021d52a169f6ac9a9c10d7144bc4ac7ea06c1ed2aa414659f -F src/update.c 0e01aa6a3edf9ec112b33eb714b9016a81241497b1fb7c3e74332f4f71756508 +F src/trigger.c 3ffb8ed6b64dbcc0ccae6e82435d01be3bf547e13b814e2d46f7df9bef84748e +F src/update.c 3e5e7ff66fa19ebe4d1b113d480639a24cc1175adbefabbd1a948a07f28e37cf F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1 -F src/utf.c 8b29d9a5956569ea2700f869669b8ef67a9662ee5e724ff77ab3c387e27094ba -F src/util.c e5f6a5eeaa26b69054a43bbd0048cfe3d2851f6961052b35aed8f695df922850 -F src/vacuum.c b763b6457bd058d2072ef9364832351fd8d11e8abf70cbb349657360f7d55c40 -F src/vdbe.c 3c9e4dacc5db859cfe61ed364023f0889ca082061cbb064f44e772d274006ec2 -F src/vdbe.h 3d26d5c7660c5c7bd33ffb0d8784615072d8b23c81f8110870efe2631136bc89 -F src/vdbeInt.h 895b1ab7536f018d3d70d690f6c0adbd1062b6dddce1c2cad912927856d4033c -F src/vdbeapi.c 82fe278a7c71b653235c6f9fb5de0b5de589908dfcb011ba2a782e8becf06f86 -F src/vdbeaux.c 885e16100597507fbbe09d82cbb963bff3fd8a9c1e358dc4f463fc95feb18e8b -F src/vdbeblob.c 255be187436da38b01f276c02e6a08103489bbe2a7c6c21537b7aecbe0e1f797 -F src/vdbemem.c 977438546df236c6a3e7d8b4fe86c0643c13b89b00235db1f11c3a91a4796d30 -F src/vdbesort.c d0a3c7056c081703c8b6d91ad60f17da5e062a5c64bf568ed0fa1b5f4cae311f +F src/utf.c 7267c3fb9e2467020507601af3354c2446c61f444387e094c779dccd5ca62165 +F src/util.c 36fb1150062957280777655976f3f9a75db236cb8207a0770ceae8d5ec17fcd3 +F src/vacuum.c d580ceb395c1ae3d59da41cbfea60683ff7dd2b94ddf4d0f5657620159e2eeb7 +F src/vdbe.c 0feab5781141acca67bd5de84172fff902304274ec5cfe58609f005b8d160050 +F src/vdbe.h 31eddcffc1d14c76c2a20fe4e137e1ee43d44f370896fae14a067052801a3625 +F src/vdbeInt.h 5446f60e89b2aa7cdf3ab0ec4e7b01b8732cd9d52d9092a0b8b1bf700768f784 +F src/vdbeapi.c 28fab30ed0acc981aecfdcaab0a421503609078e29850eb28494816682baf0a7 +F src/vdbeaux.c 948c379976885a073b54cc7d8ffda087dc1a1095d1f5bb8df218796f8c933ac3 +F src/vdbeblob.c b1b4032cac46b41e44b957c4d00aee9851f862dfd85ecb68116ba49884b03dfd +F src/vdbemem.c e67d9c6484d868c879d20c70d00bf4a9058082f1d4058607ca15d50eb3aebc21 +F src/vdbesort.c 706acdc581944cf6381f75c0ccf40f2debf71cdd51c5056592f3b74a1a0c3624 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c fc46b9cbd759dc013f0b3724549cc0d71379183c667df3a5988f7e2f1bd485f3 -F src/vtab.c bd4ab699ac4d1ee6da7339d3fbbb5edf23d9737c1fd322ccd75984329d070472 +F src/vtab.c 828221bdbeaaa6d62126ee6d07fd4ec0d09dcaea846f87ad01944d8b7e548859 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 4e6181d8780ab0af2e1388d0754cbe6f2f04593d2b1ab6c41699a89942fd8997 +F src/wal.c b0f848cfba8dd057f77073493cdd542f9125b4cf87941f53e9d0db21604155c8 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 -F src/where.c 8bf66eb9911356da811fd5bcbbd42dbc53c9381e04eb7ed32959264e24cf8082 -F src/whereInt.h d20cddddb1d61b18d5cb1fcfa9b77fbeebbc4afe44d996e603452a23b3009ee1 -F src/wherecode.c 0c3d3199a2b769a5e2bb70feb5003dc85b3d86842ecaf903a47f2b4205ca5dab +F src/where.c 45a3b496248a0b36d91ce34da3278d54f8fa20e9d3fbd36d45a42051d1118137 +F src/whereInt.h ecdbfb5551cf394f04ec7f0bc7ad963146d80eee3071405ac29aa84950128b8e +F src/wherecode.c 65670d1ef85ef54a4db3826d63be8b646c9ac280962166b645950901ed1bda29 F src/whereexpr.c 2415c8eee5ff89a8b709d7d83d71c1ff986cd720d0520057e1d8a5371339012a -F src/window.c 2bf01f9941a64fbcead61a0e3cb5db3fca5094b30d2ff0d23274c2a81d2e2385 +F src/window.c d01227141f622f24fbe36ca105fbe6ef023f9fd98f1ccd65da95f88886565db5 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 -F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 -F test/affinity3.test f094773025eddf31135c7ad4cde722b7696f8eb07b97511f98585addf2a510a9 +F test/affinity2.test 4d7a34d328e58ca2a2d78fd76c27614a41ca7ddf4312ded9c68c04f430b3b47d +F test/affinity3.test 9b7d1133e11d5edd7805573c4ab6f3ba73b0b74a1f280d5b130d4bf3506a93ff F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggfault.test 777f269d0da5b0c2524c7ff6d99ae9a93db4f1b1839a914dd2a12e3035c29829 F test/aggnested.test 610b0ce2c3e8f3daee25f9752800ee8d785db10da4aa1fbeea0ea1aabaf1d704 F test/aggorderby.test cc3abf5de64d46ff66395ca8c2346b66c2576d5aedb7bffc5b0742508856e3bf F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 -F test/all.test 2ecb8bbd52416642e41c9081182a8df05d42c75637afd4488aace78cc4b69e13 +F test/all.test cf929f721e20960ca9db89471fa44f9176322ba8f25e97193f91881c223643b3 F test/alter.test 3c00eff1e2036b9f93e9cd0f3d3e63750ac87ecb5bc71b9d7bd07cbf2ac4c494 F test/alter2.test 7e3d26ab409df52df887b366a63902c3429b935c41cb962fd58ffc25784f2f19 -F test/alter3.test ffc4ab29ce78a3517a66afd69b2730667e3471622509c283b2bd4c46f680fba3 -F test/alter4.test 716caa071dd8a3c6d57225778d15d3c3cbf5e34b2e84ae44199aeb2bbf50a707 +F test/alter3.test dcdd5f850f30656a45a0f05e41abfb52b74bbf6ccba165d0f7adf6b0116e4fd6 +F test/alter4.test 37cafe164067a6590a0ee4cec780bddbbaa33dc50b11542dcfbe0e65626494fd F test/alterauth.test 63442ba61ceb0c1eeb63aac1f4f5cebfa509d352276059d27106ae256bafc959 F test/alterauth2.test 48967abae0494d9a300d1c92473d99fcb66edfcc23579c89322f033f49410adc -F test/altercol.test 29fed774747777fbbaacdd865b4413ed2d0844a4c824f8af531b5c7d4a832087 +F test/altercol.test b43fb5725332f4cf8cff0280605202c1672e808281accea60a066d2ccc5129e5 F test/altercorrupt.test 2e1d705342cf9d7de884518ddbb053fd52d7e60d2b8869b7b63b2fda68435c12 F test/alterdropcol.test a653a3945f964d26845ec0cd0a8e74189f46de3119a984c5bc45457da392612e F test/alterdropcol2.test 527fce683b200d620f560f666c44ae33e22728e990a10a48a543280dfd4b4d41 @@ -895,29 +900,29 @@ F test/altermalloc2.test 17fb3724c4b004c469c27dc4ef181608aa644555fbd3f3236767584 F test/altermalloc3.test 8040e486368403f2fdd6fc3998258b499bd4cc2f3ddbb5f8f874cd436f076e81 F test/alterqf.test 8ec03d776de9c391daa0078ea8f838903bdcfb11dfae4ba3576b48436834ccba F test/altertab.test 8a2712f9076da5012a002d0b5cc0a421398a5bf61c25bab41b77c427586a7a27 -F test/altertab2.test fff90e3f01e8eb0e09282f538b8ec7cfeb035dbedbe570fe1983440f4613ad0e +F test/altertab2.test 4bad0fa9b1ad6e62d07bc2ddb0807fb98ba80ee06d6593db2e514ec1821cae3a F test/altertab3.test b331ae34e69594e19605e3297805202d6156fcc8f75379dfd972a2e51cae8721 F test/altertrig.test aacc980b657354fe2d3d4d3a004f07d04ccc1a93e5ef82d68a79088c274ddc6b F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 2fb21d7d64748636384e6cb8998dbf83968caf644c07fcb4f76c18f2e7ede94b -F test/analyze3.test 03f4b3d794760cf15da2d85a52df9bae300e51c8fefe9c36cfae1f86dc10d23f +F test/analyze3.test c5156cef33f04b90a6b9e9d5d0bbc273a0fb44147d4508407bf1080811e2c6c8 F test/analyze4.test 68bd069f3ac7ac1e652ddd9f04f57d5606ddb4208450f5297005db7aa0dd707d F test/analyze5.test fa5131952303ac4146aba101b116b9c8cb89e2637531c334a6df7f7d19dddc0d F test/analyze6.test 028f5bdfc9e5b5294768fa9a7185b8cd1d019aa7aab5b2f8ee42d7271d9a3b28 F test/analyze7.test 079d17c495e396bdbd6cc6a083112788a6fbfb3b95c42e760e4270a53c9ead8f F test/analyze8.test 29ef237d8a59b39cc31c3310134fefe96a690b195e3deed5ecb652839089f15c F test/analyze9.test 30e1cb99336045a384a11d97900720184333c88174b3b89bc07444ea39e7df19 -F test/analyzeC.test 1111830ad355d29a294a5dda654dd5f6a8622c6a223a4f7b7b3d091df7a7a42b +F test/analyzeC.test ca20c103809836d15ab7bacb78d3dd9842c7d8743f492e5e5d058b2ffe4cd849 F test/analyzeD.test 485f621cfd2ef0a8f8ac79672586651bfa495bd899db50461bb4b558400ab3c1 F test/analyzeE.test d2ec7921c162cdc33ac8e7eb01f9ebf78100610af7c94c8552bbf551de1fb397 F test/analyzeF.test 40b5cc3ad7b10e81020d7ca86f1417647ecfae7477cfd88acc5aa7ae1068f949 F test/analyzeG.test 623be33038c49648872746c8dd8b23b5792c08fef173c55e82f1b12fca259852 -F test/analyzer1.test 459fa02c445ddbf0101a3bad47b34290a35f2e49 +F test/analyzer1.test b6a624ec0af92eec209e1328465b66937c8fdf2fb442a3fa45321ddb3700f4aa F test/atof1.test bd21c4a0e718ab1470de07a2a79f2544d7903be34feebcc80de04beee4807b00 F test/atomic.test 065a453dde33c77ff586d91ccaa6ed419829d492dbb1a5694b8a09f3f9d7d061 F test/atomic2.test b6863b4aa552543874f80b42fb3063f1c8c2e3d8e56b6562f00a3cc347b5c1da F test/atrc.c c388fac43dbba05c804432a7135ae688b32e8f25818e9994ffba4b64cf60c27c -F test/attach.test 54f8e49e88d0de48f6428267a678465863d2b8f72320612f35bd5c02e240bc2f +F test/attach.test 12b2a9872c1ce20edf40289f00d82d13faae59ced522d98496ab06ad5c5e0a1c F test/attach2.test 6d1e3a457ce260d6fc8e5945c07fba6c76dc2aa90e1c701f067b50ee88f7315a F test/attach3.test c59d92791070c59272e00183b7353eeb94915976 F test/attach4.test 00e754484859998d124d144de6d114d920f2ed6ca2f961e6a7f4183c714f885e @@ -926,20 +931,20 @@ F test/auth.test 5b8558a40571ebc55c1581cb7cec3b2348a699542a0a51b83ef21c6a953d95e F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1 F test/auth3.test 76d20a7fa136d63bcfcf8bcb65c0b1455ed71078d81f22bcd0550d3eb18594ab F test/autoanalyze1.test b9cc3f32a990fa56669b668d237c6d53e983554ae80c0604992e18869a0b2dec -F test/autoinc.test 997d6f185f138229dc4251583a1d04816423dddc2fc034871a01aeb1d728cb39 +F test/autoinc.test 9df9930966dbe92c55ef37a4d89112cfd537be0d0596d397177c12db9e581be0 F test/autoindex1.test 65931519206bbec71948b11e125af0656435a0937973fe5fed70d776a712911f F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df F test/autoindex3.test ca502c8050166ac6107a7b4fe4e951f4d3270a23a958af02b14f1b962b83c4b6 F test/autoindex4.test 3c2105e9172920e26f950ba3c5823e4972190e022c1e6f260ba476b0af24c593 -F test/autoindex5.test 2ee94f033b87ca0160e08d81034c507aff8e230df2627f0304fa309b2fee19a3 +F test/autoindex5.test 3fb938cbf4e7f3896563ce04e2a24b0bc653fc6245b4bf3268cd7b20f441d87f F test/autovacuum.test 00671369bbf96c6a49989a9425f5b78b94075d6a4b031e5e00000c2c32f365df F test/autovacuum2.test 76f7eb4fe6a6bf6d33a196a7141dba98886d2fb53a268d7feca285d5da4759d7 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avfs.test 76f59743dc1f5fa533840d1818b420fe1ee45e21c0fd6bbac7942ba677903128 -F test/avtrans.test b7dc25459ecbd86c6fa9c606ee3068f59d81e225118617dcf2bbb6ded2ade89e +F test/avtrans.test 7a6eae44763293024b137b53ff824d8500d754dbae060a8d940afbacfc1d4a15 F test/backcompat.test f2431465ed668f09fc3f6998e56e893a1506ccea6e8b6f409f085f759f431b48 F test/backup.test 3b08fd4af69f0fa786931103a31f4542b184aba16e239e5f22b18c3c2476697f -F test/backup2.test 8facb54df1388419d34b362ab1f7e233310ff3a3af64e8ad5ec47ba3c2bbe5cf +F test/backup2.test b153553ee5667b0748b43346b0725fbf80ce1f5544613bf087d669778b60ec56 F test/backup4.test 8f6fd48e0dfde77b9a3bb26dc471ede3e101df32 F test/backup5.test ee5da6d7fe5082f5b9b0bbfa31d016f52412a2e4 F test/backup_ioerr.test 4c3c7147cee85b024ecf6e150e090c32fdbb5135 @@ -947,7 +952,7 @@ F test/backup_malloc.test 0c9abdf74c51e7bedb66d504cd684f28d4bd4027 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/badutf2.test f310fd3b24a491b6b77bccdf14923b85d6ebcce751068c180d93a6b8ff854399 F test/basexx1.test d8a50f0744b93dca656625597bcd3499ff4b9a4ea2a82432b119b7d46e3e0c08 -F test/bc_common.tcl b5e42d80305be95697e6370e015af571e5333a1c +F test/bc_common.tcl c70b896d1d4ce72f769d2c7c1fc15b2cb07559eb2093f2736c8ca51664b29ff5 F test/bestindex1.test 856a453dff8c68b4568601eed5a8b5e20b4763af9229f3947c215729ed878db0 F test/bestindex2.test 394ff8fbf34703391247116d6a44e1c50ee7282236ee77909044573cefc37bc0 F test/bestindex3.test 34bea272b0e0f835651b16a3931dbe7ac927039be6b2e1cb617bbe1d584b492b @@ -972,7 +977,7 @@ F test/bind2.test 918bc35135f4141809ead7585909cde57d44db90a7a62aef540127148f91aa F test/bindxfer.test efecd12c580c14df5f4ad3b3e83c667744a4f7e0 F test/bitvec.test 75894a880520164d73b1305c1c3f96882615e142 F test/blob.test e7ac6c7d3a985cc4678c64f325292529a69ae252 -F test/bloom1.test 04f3a17df8912bfdc292c41b59d79f93893fe69799f3089a64451f9112f9658f +F test/bloom1.test 3b6277a647ac503b5d5df331037b0c01c40e88cc9537b94eaf2d8aa334ed4c8f F test/boundary1.tcl 6421b2d920d8b09539503a8673339d32f7609eb1 F test/boundary1.test 66d7f4706ccdb42d58eafdb081de07b0eb42d77b F test/boundary2.tcl e34ef4e930cf1083150d4d2c603e146bd3b76bcb @@ -981,7 +986,7 @@ F test/boundary3.tcl 23361e108a125dca9c4080c2feb884fe54d69243 F test/boundary3.test 56ef82096b4329aca2be74fa1e2b0f762ea0eb45 F test/boundary4.tcl 0bb4b1a94f4fc5ae59b79b9a2b7a140c405e2983 F test/boundary4.test 89e02fa66397b8a325d5eb102b5806f961f8ec4b -F test/btree01.test fef17d9e999ac4f04095948e3438fbe674f4e07bb2c63bb1cad41d87baee077f +F test/btree01.test 7207a813400798c7403dbfdd0b1eb04e204804257de384dd6f221f3b189cb7fc F test/btree02.test 7555a5440453d900410160a52554fe6478af4faf53098f7235f1f443d5a1d6cc F test/btreefault.test a82a23b0578bc587afbf9a622c8f54a54f63762f062ba8a35613cfee38ab42f9 F test/busy.test caff7164c16ce06a53af51f9e4c2753d4cc64250e00790a5e48b9c4f4be37597 @@ -996,18 +1001,18 @@ F test/capi3c.test 31d3a6778f2d06f2d9222bd7660c41a516d1518a059b069e96ebbeadb5a49 F test/capi3d.test 8b778794af891b0dca3d900bd345fbc8ebd2aa2aae425a9dccdd10d5233dfbde F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/carray01.test 23ed7074307c4a829ba5ff2970993a9d87db7c5cdbbe1a2cbef672d0df6d6e31 -F test/cast.test 6d095303492432a973e6dfc0071cb94cac2969ffbe2e6a68432be0c7b3b0a2d3 +F test/cast.test a2a3b32df86e3c0601ffa2e9f028a18796305d251801efea807092dbf374a040 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/changes.test 4377d202a487f66fc2822c1bf57c46798c8b2caf7446f4f701723b1dbb6b86f6 F test/changes2.test 07949edcc732af28cb54276bfb7d99723bccc1e905a423648bf57ac5cb0dc792 -F test/check.test 56e4ed457e9f8683b9fc56f5b964f461f6e8a8dd5a13f3d495408215d66419ed +F test/check.test 3a7972ccbaad80d496833da8714d69d9d5d4ce9e7211af1cd2a06ae488a7de12 F test/checkfault.test da6cb3d50247169efcb20bdf57863a3ccfa1d27d9e55cd324f0680096970f014 -F test/chunksize.test 427d87791743486cbf0c3b8c625002f3255cb3a89c6eba655a98923b1387b760 +F test/chunksize.test faea11c5d6df9d392252a8dd879e1b1d68c9d3e8b7909cbed8bcec3b60c706f1 F test/cksumvfs.test 6f05dc95847c06a3dc10eee6b5ab1351d78314a52d0db15717c9388f4cb96646 F test/close.test eccbad8ecd611d974cbf47278c3d4e5874faf02d811338d5d348af42d56d647c F test/closure01.test 9905883f1b171a4638f98fc764879f154e214a306d3d8daf412a15e7f3a9b1e0 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 -F test/collate1.test 71a6f27fdc93a92f14d8ab80c05e1937656a5a03197e1a10157314554d630ce8 +F test/collate1.test 0890fa372753b59eba53832d37328af815f6b8e4b16761823180eeb62c8e8f64 F test/collate2.test 471c6f74573382b89b0f8b88a05256faa52f7964f9e4799e76708a3b1ece6ba4 F test/collate3.test 89defc49983ddfbf0a0555aca8c0521a676f56a5 F test/collate4.test c953715fb498b87163e3e73dd94356bff1f317bd @@ -1017,11 +1022,11 @@ F test/collate7.test 8ec29d98f3ee4ccebce6e16ce3863fb6b8c7b868 F test/collate8.test cd9b3d3f999b8520ffaa7cc1647061fc5bab1334 F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 -F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151ecac0b95 -F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 -F test/colname.test 387e880eeac0889900f7b3e9703c375515f5d323f71fd4f2bb5428a4ac8e2023 +F test/collateB.test 9c840e21f9aead6fc533cea310f0bd202d5c11511088811b7e93ae7b47fccb24 +F test/colmeta.test 248a644cec4c7c371cf1e107fd8fdba708dc290866c572672b6260e3466cce79 +F test/colname.test 0e32125de701c6bd86247194c6f5639b740b4f71a0d88ee1e67ff3bda9ae99ca F test/columncount.test 6fe99c2f35738b0129357a1cf3fa483f76140f4cd8a89014c88c33c876d2638f -F test/conflict.test b705cddf025a675d3c13d62fa78ab1e2696fb8e07a3d7cccce1596ff8b301492 +F test/conflict.test 3307ffdf988e04b01c4e942d8aa369a977f085bf629f43a627c9a77f39d65926 F test/conflict2.test 5557909ce683b1073982f5d1b61dfb1d41e369533bfdaf003180c5bc87282dd1 F test/conflict3.test 81865d9599609aca394fb3b9cd5f561d4729ea5b176bece3644f6ecb540f88ac F test/contrib01.test 2a1cbc0f2f48955d7d073f725765da6fbceda6b4 @@ -1059,7 +1064,7 @@ F test/crash4.test fe2821baf37168dc59dd733dcf7dba2a401487bc F test/crash5.test 4aa55e7ac3c4bc511873e457aa65d2827d52da9b51e061511899dadcfe22b1e8 F test/crash6.test 4c56f1e40d0291e1110790a99807aa875b1647ba F test/crash7.test 1a194c4900a255258cf94b7fcbfd29536db572df -F test/crash8.test 64366e459c28dd62edfb7ad87253a409c7533b92d16fcc479a6a8131bdcc3100 +F test/crash8.test 3a0c39c079b441a9b1da22e3edb58817f9dd330c4c3a7f9dd1f9f7c8368ea352 F test/crashM.test d95f59046fa749b0d0822edf18a717788c8f318d F test/crashtest1.c 09c1c7d728ccf4feb9e481671e29dda5669bbcc2 F test/createtab.test 85cdfdae5c3de331cd888d6c66e1aba575b47c2e3c3cc4a1d6f54140699f5165 @@ -1069,13 +1074,13 @@ F test/ctime.test 340f362f41f92972bbd71f44e10569a5cc694062b692231bd08aa6fe6c1c47 F test/cursorhint.test 05cf0febe5c5f8a31f199401fd1c9322249e753950d55f26f9d5aca61408a270 F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee9ae9c42f F test/dataversion1.test 6e5e86ac681f0782e766ebcb56c019ae001522d114e0e111e5ebf68ccf2a7bb8 -F test/date.test 8911c3d9fb0e496e92e0259697f431a00707222d2b3438ce1105d1790a3c0d51 +F test/date.test 180301372ed42520ff2d1c9bfa1d2aa726a0530d32452bd3f88f0a1c40f3c21e F test/date2.test 7e12ec14aaf4d5e6294b4ba140445b0eca06ea50062a9c3a69c4ee13d0b6f8b1 F test/date3.test a1b77abf05c6772fe5ca2337cac1398892f2a41e62bce7e6be0f4a08a0e64ae5 -F test/date4.test 75dc8401e8c0639a228cd26a6eaa4ff5ea8ccda912b9853d1c9462c476670e17 +F test/date4.test b5ad22baf7394e008ac59383840159daedd45be31dcf74a3b2450ec0e28955ce F test/date5.test 14ba189bc4d03efc371dd5302e035764f6633355a3e13acb4a45e7b33530231e F test/dbdata.test 042f49acff3438f940eeba5868d3af080ae64ddf26ae78f80c92bec3ca7d8603 -F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e +F test/dbfuzz.c fc566102f72c8af84ae8077b4faf7f056c571e6fa7a32e98b66e42b7505f47b6 F test/dbfuzz001.test 6c9a4622029d69dc38926f115864b055cb2f39badd25ec22cbfb130c8ba8e9c3 F test/dbfuzz2-seed1.db e6225c6f3d7b63f9c5b6867146a5f329d997ab105bee64644dc2b3a2f2aebaee F test/dbfuzz2.c 4b3c12de4d98b1b2d908ab03d217d4619e47c8b23d5e67f8a6f2b1bdee7cae23 @@ -1088,7 +1093,7 @@ F test/default.test c7124864cded213a3f118bc7e2e26f34b7c36dfa26cf6945cc8b7f5db119 F test/delete.test 2686e1c98d552ef37d79ad55b17b93fe96fad9737786917ce3839767f734c48f F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa F test/delete3.test 555e84a00a99230b7d049d477a324a631126a6ab -F test/delete4.test 51fafebe9503a40796d1aae1565c60524cada720e50eecac01b7fd0419d9ea0b +F test/delete4.test 2bcf66fe9978206e8fad1bacb69c00fe05dd1a3f2036c421420f6360f7d18b53 F test/delete_db.test 096d828493c7907f9ea11a7098ea6a0f73edba89406487d5d6cc2228dc4ab8b0 F test/descidx1.test edc8adee58d491b06c7157c50364eaf1c3605c9c19f8093cb1ea2b6184f3ac13 F test/descidx2.test a0ba347037ff3b811f4c6ceca5fd0f9d5d72e74e59f2d9de346a9d2f6ad78298 @@ -1124,7 +1129,7 @@ F test/e_walckpt.test 28c371a6bb5e5fe7f31679c1df1763a19d19e8a0 F test/e_walhook.test 01b494287ba9e60b70f6ebf3c6c62e0ffe01788e344a4846b08e5de0b344cb66 F test/emptytable.test a38110becbdfa6325cd65cb588dca658cd885f62 F test/enc.test b5503a87b31cea8a5084c6e447383f9ca08933bd2f29d97b6b6201081b2343eb -F test/enc2.test 848bf05f15b011719f478dddb7b5e9aea35e39e457493cba4c4eef75d849a5ec +F test/enc2.test 872afe58db772e7dfa1ad8e0759f8cc820e9efc8172d460fae83023101c2e435 F test/enc3.test 55ef64416d72975c66167310a51dc9fc544ba3ae4858b8d5ab22f4cb6500b087 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 F test/eqp.test 82f221e8cd588434d7f3bba9a0f4c78cbe7a541615a41632e12f50608bfb4a99 @@ -1136,15 +1141,15 @@ F test/exclusive.test 7ff63be7503990921838d5c9f77f6e33e68e48ed1a9d48cd28745bf650 F test/exclusive2.test cd70b1d9c6fffd336f9795b711dcc5d9ceba133ad3f7001da3fda63615bdc91e F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac -F test/expr.test 5c06696478212e5a04e04b043f993373f6f8e5ce5a80f5548a84703b123b6caa +F test/expr.test 4ada8eb822c45ef27a36851a258004d43c1e95e7c82585a1217e732084e4482c F test/expr2.test c27327ae9c017a7ff6280123f67aff496f912da74d78c888926d68b46ec75fd8 F test/exprfault.test da33606d799718e2f8e34efd0e5858884a1ad87f608774c552a7f5517cc27181 F test/exprfault2.test c49e84273898969af5dbc4fe6a3f4335f14639799f343590336c9ddf84425965 -F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9 -F test/external_reader.test c7d34694f1b25c32d866f56ac80c1e29edddc42b4ef90cad589263ffac2cde0c +F test/extension01.test 5de412c66276105901c370770175003381fdcb0c4da7054fa43cf4a31e0bfa3a +F test/external_reader.test 6fdec43eeca23eb32faad1e95a4d1abc402bc8b3db70df12d6fc08a637f4a2b5 F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79 F test/fallocate.test 37a62e396a68eeede8f8d2ecf23573a80faceb630788d314d0a073d862616717 -F test/filectrl.test 6e871c2d35dead1d9a88e176e8d2ca094fec6bb3 +F test/filectrl.test 4b720117388cf6766d0b798e2dddd785953f8f371633b0c0084d2f34cf72336a F test/filefmt.test f393e80c4b8d493b7a7f8f3809a8425bbf4292af1f5140f01cb1427798a2bbd4 F test/filter1.test 590f8ba9a0cd0823b80d89ac75c5ce72276189cef9225d2436adaf1ee87f3727 F test/filter2.tcl 44e525497ce07382915f01bd29ffd0fa49dab3adb87253b5e5103ba8f93393e8 @@ -1155,7 +1160,7 @@ F test/fkey2.test 1063d65e5923c054cfb8f0555a92a3ae0fa8c067275a33ee1715bd856cdb30 F test/fkey3.test 76d475c80b84ee7a5d062e56ccb6ea68882e2b49 F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d F test/fkey5.test 6727452e163a427147e84e739da18713da553d79f9783559b04fdcd36d5c7421 -F test/fkey6.test ebd11efb00b9c70b57f4c6b6184445145c96e320329bd90a175036570c5b25ca +F test/fkey6.test 668a7299e75899b0a3342c36df655be57f76a05aca3544bda939a6e676e2f000 F test/fkey7.test 64fb28da03da5dfe3cdef5967aa7e832c2507bf7fb8f0780cacbca1f2338d031 F test/fkey8.test 51deda7f1a1448bca95875e4a6e1a3a75b4bd7215e924e845bd60de60e4d84bf F test/fkey_malloc.test 594a7ea1fbab553c036c70813cd8bd9407d63749 @@ -1197,7 +1202,7 @@ F test/fts3corrupt4.test c7f414fe29b97a478d15c90382c4ae077a2bbd2283bf8c63bf66dad F test/fts3corrupt5.test 0549f85ec4bd22e992f645f13c59b99d652f2f5e643dac75568bfd23a6db7ed5 F test/fts3corrupt6.test f417c910254f32c0bc9ead7affa991a1d5aec35b3b32a183ffb05eea78289525 F test/fts3corrupt7.test 1da31776e24bb91d3c028e663456b61280b121a74496ccf2fef3fe33790ad2b0 -F test/fts3cov.test 7eacdbefd756cfa4dc2241974e3db2834e9b372ca215880e00032222f32194cf +F test/fts3cov.test 1e5ecea0e4c1394cea97adcfb9fd3d2d5998fd563dacf465f413e6c7fa5cffb3 F test/fts3d.test 2bd8c97bcb9975f2334147173b4872505b6a41359a4f9068960a36afe07a679f F test/fts3defer.test f4c20e4c7153d20a98ee49ee5f3faef624fefc9a067f8d8d629db380c4d9f1de F test/fts3defer2.test 3bbe54a7fca7d548bb7ac4f59447ee591070bfbe0c9f3e279defa0b898e9afbb @@ -1206,7 +1211,7 @@ F test/fts3drop.test 1b906e293d6773812587b3dc458cb9e8f3f0c297 F test/fts3dropmod.test 7de242ea1c8a713a8b143ea54468f4b1c4953fa068349e23ac178e2c90c59889 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test ebae205a7a89446c32583bcd492dcb817b9f6b31819bb4dde2583bb99c77e526 -F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a +F test/fts3expr2.test ef381978b9bdcfc6d4b47c86270ba356a75dbff164ce52ff5d18a5924a788614 F test/fts3expr3.test c4d4a7d6327418428c96e0a3a1137c251b8dfbf8 F test/fts3expr4.test 6c7675bbdbffe6ffc95e9e861500b8ac3f739c4d004ffda812f138eeb1b45529 F test/fts3expr5.test a5b9a053becbdb8e973fbf4d6d3abaabeb42d511d1848bd57931f3e0a1cf983e @@ -1217,7 +1222,7 @@ F test/fts3fault3.test ccdd2292dd2d4e21e30fc5f4c8e064f79e516087eec5ff57ab6bc4f6a F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 F test/fts3fuzz001.test c78afcd8ad712ea0b8d2ed50851a8aab3bc9dc52c64a536291e07112f519357c F test/fts3integrity.test 0c6fe7353d7b24d78862f4272ee9df4da2f32b3ff30fa3396945cda8119580a8 -F test/fts3join.test 1a4d786539b2b79a41c28ef2ac22cacd92a8ee830249b68a7dee4a020848e3bb +F test/fts3join.test de31d304ba479043a7d33d2f201c514b3e1da809da6797d7a58704d00e8da2e6 F test/fts3malloc.test b0e4c133b8d61d4f6d112d8110f8320e9e453ef6 F test/fts3matchinfo.test aa66cc50615578b30f6df9984819ae5b702511cf8a94251ec7c594096a703a4a F test/fts3matchinfo2.test 00144e841704b8debfcdf6097969cd9f2a1cf759e2203cda42583648f2e6bf58 @@ -1244,7 +1249,7 @@ F test/fts4growth.test 289833c34ad45a5e6e6133b53b6a71647231fb89d36ddcb8d9c87211b F test/fts4growth2.test 13ad4e76451af6e6906c95cdc725d01b00044269 F test/fts4incr.test 4e353a0bd886ea984e56fce9e77724fc923b8d0d F test/fts4intck1.test 54e7f28e34b72fb0c614d414bb1f568154d463c5a00b20944e893df858372ed4 -F test/fts4langid.test 4be912f42454998e239a2e877600263e0394afbaba03e06cedcc5a08693a345a +F test/fts4langid.test 1cc6fe045f094f1695d0b20e6b817a2ce22ec470cb7c6577470cd71ed0256e90 F test/fts4lastrowid.test 185835895948d5325c7710649824042373b2203149abe8024a9319d25234dfd7 F test/fts4merge.test 57d093660a5093ae6e9fbd2d17592a88b45bbd66db2703c4b640b28828dbe38b F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 @@ -1261,23 +1266,23 @@ F test/fts4umlaut.test fcaca4471de7e78c9d1f7e8976e3e8704d7d8ad979d57a739d00f3f75 F test/fts4unicode.test 82a9c16b68ba2f358a856226bb2ee02f81583797bc4744061c54401bf1a0f4c9 F test/fts4upfrom.test f25835162c989dffd5e2ef91ec24c4848cc9973093e2d492d1c7b32afac1b49d F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d -F test/func.test 15f686741608294340bbea9f35f751074b4cf7df3797724dda40a9f4905ddbe1 +F test/func.test b6b6942b29e1b00120cb4ed00ba8ed8ef561e7ba6ae1c41cc005143c7e6f5bf6 F test/func2.test 69f6ae3751b4ec765bdc3b803c0a255aa0f693f28f44805bef03e6b4a3fd242f -F test/func3.test 600a632c305a88f3946d38f9a51efe145c989b2e13bd2b2a488db47fe76bab6a +F test/func3.test 6b22dfada95b778d01dba75aa00fa8e6a1fb90d30b859241638e7540edcfb3ca F test/func4.test a02e695f62beb31cb092dccf6873ff97543407fff97a5f3ec4da70b5b337bc84 F test/func5.test 863e6d1bd0013d09c17236f8a13ea34008dd857d87d85a13a673960e4c25d82a F test/func6.test 3bc89ec0f2605736d3a118f43d25ef58115a7db4dba8ae939a363917d815c0bb F test/func7.test 7e009275f52c52954c8c028fdb62f8bc16cc47276fcc8753c1d2b22c6e074598 F test/func8.test c4e2ecacf9f16e47a245e7a25fbabcc7e78f9c7c41a80f158527cdfdc6dd299d -F test/func9.test b32d313f679aa9698d52f39519d301c3941823cb72b4e23406c210eadd82c824 +F test/func9.test 62750dbbbcc9a2d241918b5f999f59e2126084d5f81904f9e1d8ee466666a19d F test/fuzz-oss1.test 514dcabb24687818ea949fa6760229eaacad74ca70157743ef36d35bbe01ffb0 -F test/fuzz.test 4608c1310cff4c3014a84bcced6278139743e080046e5f6784b0de7b069371d8 +F test/fuzz.test 819ea7e483bcee91209aacbe6f9eaf3287baa1841479ee5f639f57c5e7c42b86 F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1 F test/fuzz3.test 70ba57260364b83e964707b9d4b5625284239768ab907dd387c740c0370ce313 F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c 6fc952750a69168dd5fea38b9d35cb38475bfda15c8acfd156ac09cd03ddbd3e +F test/fuzzcheck.c 19f8af47a5c4ee2c3943fdee270f1f14e3d83fe968a9737a7557fb4e3c06efc1 F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -1285,13 +1290,13 @@ F test/fuzzdata4.db b502c7d5498261715812dd8b3c2005bad08b3a26e6489414bd13926cd3e4 F test/fuzzdata5.db e35f64af17ec48926481cfaf3b3855e436bd40d1cfe2d59a9474cb4b748a52a5 F test/fuzzdata6.db b8725a5f5cf7a3b7241a9038e57ca7e7cc8c3f4d86b44bd770617bda245ab2b0 F test/fuzzdata7.db 0166b56fd7a6b9636a1d60ef0a060f86ddaecf99400a666bb6e5bbd7199ad1f2 -F test/fuzzdata8.db c6f9cb7d2b808fb10894afe53ef00f51e73e43baa7aabdba7e9af4713fc5b186 +F test/fuzzdata8.db 8f34ae00d8d5d4747dd80983cf46161065e4f78324dcff3c893506ff8db3a4a6 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test f64c4aef4c9e9edf1d6dc0d3f1e65dcc81e67c996403c88d14f09b74807a42bc F test/fuzzinvariants.c 057e910241d85aa4aaf75cef1a7adc45c632b173288d07d9dbbef4e6bda83d5a F test/gcfault.test 4ea410ac161e685f17b19e1f606f58514a2850e806c65b846d05f60d436c5b0d -F test/gencol1.test e169bdfa11c7ed5e9f322a98a7db3afe9e66235750b68c923efee8e1876b46ec +F test/gencol1.test ceb3163b59cb77f4ad57ae4f01a143ce36b06fdd6a8dab1149235db89979ffd8 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 F test/having.test a89236dd8d55aa50c4805f82ac9daf64d477a44d712d8209c118978d0ca21ec9 F test/hexlit.test 4a6a5f46e3c65c4bf1fa06f5dd5a9507a5627751 @@ -1301,10 +1306,10 @@ F test/hook2.test b9ff3b8c6519fb67f33192f1afe86e7782ee4ac8 F test/icu.test 8da7d52cd9722c82f33b0466ed915460cb03c23a38f18a9a2d3ff97da9a4a8c0 F test/ieee754.test b0945d12be7d255f3dfa18e2511b17ca37e0edd2b803231c52d05b86c04ab26e F test/imposter1.test c3f1db2d3db2c24611a6596a3fc0ffc14f1466c8 -F test/in.test d1cad4ededd425568b2e39fb0c31fa9a3772311dd595801ff13ba3912b69bba6 +F test/in.test edf979bff3244b9e47849e2b43886631354c8213791f42da92216f08012141af F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 -F test/in4.test bb767ec1cfd1730256f0a83219f0acda36bc251b63f8b8bb7d8c7cff17875a4f +F test/in4.test e7b1456d423453884aeb9d68fafe9c8e7be7abddc8f028c04205e67820f10772 F test/in5.test 4fd79c70dfa0681313e8cdca07f5ff0400bdc0e20f808a5c59eaef1e4b48082a F test/in6.test f5f40d6816a8bb7c784424b58a10ac38efb76ab29127a2c17399e0cbeeda0e4b F test/in7.test d9efdee00b074a60c6343993b2eda78bc369ab080dad864513c73f8aca89d566 @@ -1319,48 +1324,48 @@ F test/incrvacuum.test 3fa6145f5e71f603554fd7b8ec3da4290b1341029682313285cb5f9e1 F test/incrvacuum2.test 7d26cfda66c7e55898d196de54ac4ec7d86a4e3d F test/incrvacuum3.test 0bf0ffe7f2cbc87ba1d471e4bbadabbf10dacf8d4ee26b3a072708d575d637a9 F test/incrvacuum_ioerr.test 6ae2f783424e47a0033304808fe27789cf93e635 -F test/index.test d866054c88b394fd42cbf2825628f127ca24dfac525fa019069a936674d92cbe +F test/index.test 9a060cbb12b70dd0e6bca0404106242703c0827ba58712f46b0d5a3b3968b341 F test/index2.test f835d5e13ca163bd78c4459ca15fd2e4ed487407 F test/index3.test 51685f39345462b84fcf77eb8537af847fdf438cc96b05c45d6aaca4e473ade0 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 F test/index5.test 8621491915800ec274609e42e02a97d67e9b13e7 -F test/index6.test b376a648e85aa71c50074382784e6cb0c126ec46e43d1ad15af9a4d234c52e65 +F test/index6.test 656502465b30c3f8de4eb956928aaded02cf36128e8fc8be2b95390fb23e74f5 F test/index7.test b238344318e0b4e42126717f6554f0e7dfd0b39cecad4b736039b43e1e3b6eb3 F test/index8.test caa097735c91dbc23d8a402f5e63a2a03c83840ba3928733ed7f9a03f8a912a3 F test/index9.test 2ac891806a4136ef3e91280477e23114e67575207dc331e6797fa0ed9379f997 F test/indexA.test 11d84f6995e6e5b9d8315953fb1b6d29772ee7c7803ee9112715e7e4dd3e4974 -F test/indexedby.test f21eca4f7a6ffe14c8500a7ad6cd53166666c99e5ccd311842a28bc94a195fe0 -F test/indexexpr1.test 928671af9d7374bb56ed4dcfbc157f4eeddb1e86ab5615ceb3ac97a713c2dd8f +F test/indexedby.test 444fb04ce0b21a3daf79f84e6735b49e5a5b3396623b37df5431eb09c8b8f557 +F test/indexexpr1.test d32dba192b8a566a81bb437ee8b6219931458e010c2d94610da775fa9d318f17 F test/indexexpr2.test 1c382e81ef996d8ae8b834a74f2a9013dddf59214c32201d7c8a656d739f999a F test/indexexpr3.test 47b91bc7999805c9a34d356f672259bc49295ecc797448511cae554a309b47cd F test/indexfault.test 98d78a8ff1f5335628b62f886a1cb7c7dac1ef6d48fa39c51ec871c87dce9811 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 -F test/insert.test 4e3f0de67aac3c5be1f4aaedbcea11638f1b5cdc9a3115be14d19aa9db7623c6 +F test/insert.test 97cfb30b83ca1622b9422a1e4c4831b4cb767cf5d654660945036d1e72067e70 F test/insert2.test 4d14b8f1b810a41995f6286b64a6943215d52208 F test/insert3.test 1b7db95a03ad9c5013fdf7d6722b6cd66ee55e30 F test/insert4.test 2bf81535a990c969665d66db51fcf76c23499b39893b5109f413d1de4ad34cd3 F test/insert5.test 394f96728d1258f406fe5f5aeb0aaf29487c39a6 F test/insertfault.test ac63d14ea3b49c573673a572f4014b9117383a03e497c58f308b5c776e4a7f74 -F test/instr.test 107df2b9b74a4b59315916b575590a08f2a714de0754abe541f10a0971d0a2a4 +F test/instr.test 67ba309e9697c24a304e98a7c8f372456177dd4e32237d2a305e1e05f7bb79c2 F test/instrfault.test 95e28efade652e6d51ae11b377088fe523a581a07ec428009e152a4dd0e0f44c F test/intarray.test bb976b0b3df0ebb6a2eddfb61768280440e672beba5460ed49679ea984ccf440 F test/interrupt.test ac1ef50ec9ab8e4f0e17c47629f82539d4b22558904e321ed5abea2e6187da7a F test/interrupt2.test e4408ca770a6feafbadb0801e54a0dcd1a8d108d -F test/intpkey.test aee694afed1a65c86c4e69ad030224b3fc268113d00685234d40079fca16bad3 +F test/intpkey.test 7d54711acf553cdd641a40e9c6cfc2bf1a76070074940c1b126442517054320f F test/intreal.test 68829a8bb073ee1610ca3f8f9e0f99b0371fb36e0fa64862dd5ced4ef03c2343 -F test/io.test f138f3fe696d1ed8c51dfea5b325910d319a1b29e1d25ea57231a02092f02cca -F test/ioerr.test c94eef1cd8bfc36f9aa493e41e151e9160281ac8e2d960cc9dcdcc8e6aa99ab3 +F test/io.test d267fdc8915444a45e19841489033ebe70bb69f6db605b00df70be16b2a80f59 +F test/ioerr.test 78552a95d53b9674d85f63bdf3d76a8df70b4d5dba5a6a1b8a1d60f166cd2c6b F test/ioerr2.test 2593563599e2cc6b6b4fcf5878b177bdd5d8df26 F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 5984da7bf74b6540aa356f2ab0c6ae68a6d12039a3d798a9ac6a100abc17d520 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/istrue.test e7f285bb70282625c258e866ce6337d4c762922f5a300e1b50f958aef6e7d9c9 -F test/join.test f7abfef3faeaf2800308872e33a57e5b6e4a2b44fb8c6b90c6068412e71a6cf4 +F test/join.test 255c1f42b7fe027b518cadb2bf40f41a793a95e7f8db2bceb54faaf59ff19c6c F test/join2.test f59d63264fb24784ae9c3bc9d867eb569cd6d442da5660f8852effe5c1938c27 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 -F test/join5.test 380d12a9350f99f0cc681a4f1fea999886f18b3fe0d71a9b3065bcaead1e007f +F test/join5.test 76f99f69a410241de73fa5cd3f4a6ef469e64bb501a48de2437881792111c589 F test/join6.test f809c025fa253f9e150c0e9afd4cef8813257bceeb6f46e04041228c9403cc2c F test/join7.test 2268dcbb54b724391dda3748ea95c60d960607ffeed67885675998e7117697f6 F test/join8.test d384d63985e3991c404afccadaf3efd1cdf9cd72680167f80e3cb80b95c18c68 @@ -1371,21 +1376,21 @@ F test/joinC.test 1f1a602c2127f55f136e2cbd3bf2d26546614bf8cffe5902ec1ac9c07f87f2 F test/joinD.test 2ce62e7353a0702ca5e70008faf319c1d4686aa19fba34275c6d1da0e960be28 F test/joinE.test d5d182f3812771e2c0d97c9dcf5dbe4c41c8e21c82560e59358731c4a3981d6b F test/joinF.test 53dd66158806823ea680dd7543b5406af151b5aafa5cd06a7f3231cd94938127 -F test/joinH.test 55f69e64da74d4eca2235237f3acb657aef181e22e45daa228e35bba865e0255 -F test/journal1.test c7b768041b7f494471531e17abc2f4f5ebf9e5096984f43ed17c4eb80ba34497 +F test/joinH.test fd76024ff104baec16417db5cafc0894ad4e0863e70803e63c1bba0322706339 +F test/journal1.test bc61a4228db11bffca118bd358ba4b868524bf080f3532749de6c539656e20fa F test/journal2.test 9dac6b4ba0ca79c3b21446bbae993a462c2397c4 -F test/journal3.test 7c3cf23ffc77db06601c1fcfc9743de8441cb77db9d1aa931863d94f5ffa140e +F test/journal3.test e5aeff93a7776cf644dbc48dec277655cff80a1cd24689036abc87869b120ea6 F test/jrnlmode.test 9b5bc01dac22223cb60ec2d5f97acf568d73820794386de5634dcadbea9e1946 F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/json/README.md de59d5ba0bd2796d797115688630a6405bbf43a2891bad445ac6b9f38b83f236 F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd28656fb261bddc8a3f F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 -F test/json/json-speed-check.sh 912ee03e700a65c827ee0c7b4752c21ec5ef69cf7679d2f482ca817042bead52 x +F test/json/json-speed-check.sh 7d5898808ce7542762318306ae6075a30f5e7ee115c4a409f487e123afe91d88 x F test/json/jsonb-q1.txt 1e180fe6491efab307e318b22879e3a736ac9a96539bbde7911a13ee5b33abc7 -F test/json101.test 30db5b055b103ccabc53a29cfe6cda3345d07e171aeb25403dafa04f19e98b19 +F test/json101.test 8237a484c256965eab1678fd950a32ac56325bb7d0dadbd095a46b0ddd95d62b F test/json102.test 9b2e5ada10845ff84853b3feaae2ce51ce7145ae458f74c6a6cecc6ef6ee3ae1 -F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe +F test/json103.test 355746a6b66aa438f214b4fae454b13068fad2444b5f693e0d538ad1c059b264 F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 F test/json105.test 043838b56e68f3252a0dcf5be1689016f6f3f05056f8dcfcdc9d074f4d932988 F test/json106.test 4aed3afd16549045d198a8d9cea00deea96e1f2ecf55864dce96cac558b8abef @@ -1399,28 +1404,28 @@ F test/kvtest.c 6e0228409ea7ca0497dad503fbd109badb5e59545d131014b6aaac68b56f484a F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/lemon-test01.y 70110eff607ab137ccc851edb2bc7e14a6d4f246b5d2d25f82a60b69d87a9ff2 -F test/like.test b3ea2ba3558199aa8f25a42ddeb54772e234fab50868c9f066047acdbda8fc58 +F test/like.test 0036f8fe548fceabd1496361bfa262f35cf5ed17c794bd15e22e9f3de12e0eb0 F test/like2.test d3be15fefee3e02fc88942a9b98f26c5339bbdef7783c90023c092c4955fe3d3 -F test/like3.test b21284df226d6028feeb4dcc56ad9d32673d82c14a63f15f25471292c36491e7 +F test/like3.test 3c9be7a0122908d8ead6aa25e47c2b9787eea88e8062926078ea7b3e95c71d99 F test/limit.test 350f5d03c29e7dff9a2cde016f84f8d368d40bcd02fa2b2a52fa10c4bf3cbfaf -F test/limit2.test 9409b033284642a859fafc95f29a5a6a557bd57c1f0d7c3f554bd64ed69df77e +F test/limit2.test 621188fc3e5c3b8d2ef9827e05fa8313792ae563579073136efd25cb65325f1b F test/literal.test a65dca9fef86e51b8e45544268e37abbd4bb94ba35fd65f6fdcab2f288cd8f79 F test/literal2.tcl 1499037beaf661aeecdbe48801220a181d805372a64c6128d5f26bb6a4a8f0ce F test/literal2.test b149e16b5fc9ee6249069a8858ed41052f222014fe0ba7ad43c2fb989c2dada2 -F test/loadext.test faa4f6eed07a5aac35d57fdd7bc07f8fc82464cfd327567c10cf0ba3c86cde04 +F test/loadext.test 878db71cf74b48250dbe9033bbfe6088ff869db3353ffd4febc68c0cd459959e F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7 -F test/lock.test 05f346b65040b9a27c032c984e1e509dfef1661135b4f26a3ab6d21358277803 +F test/lock.test 675b4367ec58b21009e46982d071c8259255e69072296b7756ea75fac5425d1a F test/lock2.test 5242d8ac4e2d59c403aebff606af449b455aceff F test/lock3.test f271375930711ae044080f4fe6d6eda930870d00 F test/lock4.test 27143363eda1622f03c133efc8db808fc331afd973486cb571ea71cd717d37b8 F test/lock5.test 583cae05992af0f66607286917f7d5f8aed3b6053c52df5994efb98f2a8fdbaf F test/lock6.test ad5b387a3a8096afd3c68a55b9535056431b0cf5 F test/lock7.test 49f1eaff1cdc491cc5dee3669f3c671d9f172431 -F test/lock_common.tcl 2f3f7f2e9637f93ccf609df48ef5b27a50278b6b1cd752b445d52262e5841413 +F test/lock_common.tcl f0a1f7b8f3fbb8629dc6231613a02841736f86ef72151429d5ffc12c7f613fb3 F test/lookaside.test 5a828e7256f1ee4da8e1bdaa03373a3ccdb0f1ff98dfa82e9b76cb41a45b1083 F test/main.test e8752d76233b1c8906cd2c98ad920dba868bd63c87d51d8a2ea5e9cba55dd496 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 -F test/malloc.test 18dd1c4188c81ca79cf123527c71b19ee0c31feb9947fdffb0dc6ceb1436816a +F test/malloc.test f7781c8151179fe4b7f743044a737ac2dfd87bf9cc18dd01398caf28d34e09c5 F test/malloc3.test 6e88bae6312854a4adb4ecc2a6a5ea8c59b4db778b724ba718e1c43fc8c3c136 F test/malloc4.test 957337613002b7058a85116493a262f679f3a261 F test/malloc5.test 2e4ad7684a13389a44a840499cd47173a8d05f22f082d7d083eece433a7a64eb @@ -1455,7 +1460,7 @@ F test/memleak.test c7478f1195d64887dd1c677edc39fa03b5bf29024e6dcc5b5cc554d7ed00 F test/memsubsys1.test 86b8158752af9188ed5b32a30674a1ef71183e6bc4e6808e815cd658ca9058a6 F test/memsubsys2.test 774b93cb09ca50d1b759bb7c645baa2a9ce172edc3a3da67d5150a26a9fc2a08 F test/merge1.test 7dd9dc6838bcd0623a069485fe3a8dd498a051c16e1877cf84f506c0d6a29b43 -F test/minmax.test fe638b55d77d2375531a8f549b338eafcd9adfbd2f72df37ed77d9b26ca0a71a +F test/minmax.test 885d89737b955905b52bd5b203f27b9b56d1e399b8155c2a95ab0f3264e0f582 F test/minmax2.test cf9311babb6f0518d04e42fd6a42c619531c4309a9dd790a2c4e9b3bc595e0de F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354 F test/minmax4.test 272ca395257f05937dc96441c9dde4bc9fbf116a8d4fa02baeb0d13d50e36c87 @@ -1465,12 +1470,12 @@ F test/misc3.test 651b88bca19b8ff6a7b6af73dae00c3fd5b3ea5bee0c0d1d91abd4c4b47487 F test/misc4.test 10cd6addb2fa9093df4751a1b92b50440175dd5468a6ec84d0386e78f087db0e F test/misc5.test 02fcaf4d42405be02ec975e946270a50b0282dac98c78303ade0d1392839d2b8 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 -F test/misc7.test d912f3d45c2989191b797504a220ca225d6be80b21acad22ba0d35f4a9ee4579 +F test/misc7.test d595599972ec0b436985f0f02f243b68500ffc977b9b3194ec66c0866cfddcab F test/misc8.test 08d2380bc435486b12161521f225043ac2be26f02471c2c1ea4cac0b1548edbd -F test/misuse.test 9e7f78402005e833af71dcab32d048003869eca5abcaccc985d4f8dc1d86bcc7 +F test/misuse.test 46d42ffdf375833ea5828796e56f84660344f7548659b493059f152f00e66840 F test/mjournal.test 28a08d5cb5fb5b5702a46e19176e45e964e0800d1f894677169e79f34030e152 F test/mmap1.test 18de3fd7b70a777af6004ca2feecfcdd3d0be17fa04058e808baf530c94b1a1d -F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 +F test/mmap2.test dba452dc7db91e9df10f70bdd73dc4190c7b8ee7b5133b4684f04277ada0b9ac F test/mmap3.test b3c297e78e6a8520aafcc1a8f140535594c9086e F test/mmap4.test 2e2b4e32555b58da15176e6fe750f17c9dcf7f93 F test/mmapcorrupt.test 470fb44fe92e99c1d23701d156f8c17865f5b027063c9119dcfdb842791f4465 @@ -1483,7 +1488,7 @@ F test/multiplex4.test e8ae4c4bd70606a5727743241f13b5701990abe4 F test/mutex1.test 42cb5e244c3a77bb0ef2b967e06fa5e7ba7d32d90a9b20bed98f6f5ede185a25 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test 73ea63ab43668313e2f8cc9ef9e9a966672c7934f3ce76926fbe991235d07d91 -F test/nockpt.test 8c43b25af63b0bd620cf1b003529e37b6f1dc53bd22690e96a1bd73f78dde53a +F test/nockpt.test 3db354270fc63b6871eebd40285d4c55324fb27be629c958adbff6d7fcaa8e14 F test/nolock.test f196cf8b8fbea4e2ca345140a2b3f3b0da45c76e F test/normalize.test f23b6c5926c59548635fcf39678ac613e726121e073dd902a3062fbb83903b72 F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf @@ -1501,10 +1506,10 @@ F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 F test/optfuzz-db01.c 9f2fa80b8f84ebbf1f2e8b13421a4e0477fe300f6686fbd76cac1d2db66e0fdc F test/optfuzz-db01.txt 21f6bdeadc701cf11528276e2a55c70bfcb846ba42df327f979bd9e7b6ce7041 F test/optfuzz.c 690430a0bf0ad047d5a168bf52b05b2ee97aedaad8c14337e9eb5050faa64994 -F test/orderby1.test 7d0e4ee692a3e808c1026b3c483594ad1e468b68b50dcefa0d678a8c05274ceb +F test/orderby1.test 64d6aac62e68c289d378a18cea9b9941d812b6a26f141a18c9fbde79bd7f28db F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 -F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 -F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 +F test/orderby3.test 64465d29ab6dabb38731ff24ead2aa0bb262bb06a7b6aa17c6f874a5f24fee94 +F test/orderby4.test daff719d0eead9ae92863a0b5a2e367bfdc7efcc9dfe16d83d667f84df597eff F test/orderby5.test bd7d9e3380e87e5dcf6ea817ebaab6d15da213c7804b38767e1b3e695e85650b F test/orderby6.test 8b38138ab0972588240b3fca0985d2e400432859 F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da @@ -1512,15 +1517,15 @@ F test/orderby8.test 23ef1a5d72bd3adcc2f65561c654295d1b8047bd F test/orderby9.test 87fb9548debcc2cd141c5299002dd94672fa76a3 F test/orderbyA.test df608e59efc2ef50c1eddf1a773b272de3252e9401bfec86d04b52fd973866d5 F test/orderbyB.test 32576c7b138105bc72f7fbf33bd320ca3a7d303641fc939e0e56af6cba884b3d -F test/oserror.test 1fc9746b83d778e70d115049747ba19c7fba154afce7cc165b09feb6ca6abbc5 +F test/oserror.test ee3fad06ec8671c4d047c2c92a567fc2e0e8161caaec7edd6d48325c5ac97f30 F test/ossfuzz.c 9636dad2092a05a32110df0ca06713038dd0c43dd89a77dabe4b8b0d71096715 F test/ossshell.c f125c5bd16e537a2549aa579b328dd1c59905e7ab1338dfc210e755bb7b69f17 F test/ovfl.test 199c482696defceacee8c8e0e0ef36da62726b2f -F test/pager1.test ffd885cdc98b986c9f746496508c0c4810ed0eaade3575ddf53c222e85880552 +F test/pager1.test b083c2d5d89df8e979658d9320bfc0b9d50b4ef8ae1d9e115a692ff0b9768393 F test/pager2.test c0ede15952b607f9a38f653acdfa73c19e657958e9104aab1a71950ea7b71831 F test/pager3.test 4e9a83d6ca0838d7c602c9eb93d1357562d9059c1e02ffb138a8271020838370 -F test/pager4.test a122e9e6925d5b23b31e3dfef8c6a44bbf19590e -F test/pagerfault.test 63c5da625562c66345ab4528790327ca63db2f6f9cbae2aba8cb7c51de3d1628 +F test/pager4.test b995066c699472614eb5949db5a2e2c51fd463863518afe68675d7fac09216bd +F test/pagerfault.test 43692e660fe480812dc5d44171fdcb8da1a65a644428def1ee9de79edace4028 F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8 F test/pageropt.test 84e4cc5cbca285357f7906e99b21be4f2bf5abc0 @@ -1530,9 +1535,9 @@ F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/pendingrace.test e99efc5ab3584da3dfc8cd6a0ec4e5a42214820574f5ea24ee93f1d84655f463 F test/percentile.test 52ba89d6ee6b65f770972b67dace358bab7cdbd532803d3db157845268e789cd -F test/permutations.test 37650c5286f7d6f322af95cad876b69c6c2c79c28dc649f09de07d3312b1213c +F test/permutations.test 5260363b43b3fad4becb49b58d0b53e6024e1c18dc169b154d0b98db5630bbf2 F test/pg_common.tcl 3b27542224db1e713ae387459b5d117c836a5f6e328846922993b6d2b7640d9f -F test/pragma.test 11cb9310c42f921918f7f563e3c0b6e70f9f9c3a6a1cf12af8fccb6c574f3882 +F test/pragma.test 7d07b7bb76e273215d6a20c4f83c3062cc28976c737ccb70a686025801e86c8f F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f F test/pragma3.test 92a46bbea12322dd94a404f49edcfbfc913a2c98115f0d030a7459bb4712ef31 F test/pragma4.test 396ef9bff1fb966d41721545ad4b12bfc26aae315f5fe51d9b917828d49e6f8e @@ -1549,15 +1554,15 @@ F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 F test/quickcheck.test a4b7e878cd97e46108291c409b0bf8214f29e18fddd68a42bc5c1375ad1fb80a F test/quota-glob.test 32901e9eed6705d68ca3faee2a06b73b57cb3c26 -F test/quota.test bfb269ce81ea52f593f9648316cd5013d766dd2a +F test/quota.test 2747bfdf50d01155c06feb391286bb585f66977feaf5e26e5a37bc00ff7dee7c F test/quota2.test 7dc12e08b11cbc4c16c9ba2aa2e040ea8d8ab4b8 F test/quote.test 7b01b2a261bc26d9821aea9f4941ce1e08191d62fc55ba8862440fb3a59197a4 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 -F test/readonly.test 69a7ccec846cad2e000b3539d56360d02f327061dc5e41f7f9a3e01f19719952 -F test/recover.test a163a156ea9f2beea63fa83c4dcd8dea6e57b8a569fc647155e3d2754eaac1b5 +F test/readonly.test 0d307c335b3421898cfe64a783a376138aa003849b6bff61ee2d21e805bc0051 +F test/recover.test c76d05f33f0271fba0f0752170e03b0ab5952dc61dcea7ab3ba40df03c4c42de F test/regexp1.test 8f2a8bc1569666e29a4cee6c1a666cd224eb6d50e2470d1dc1df995170f3e0f1 F test/regexp2.test 55ed41da802b0e284ac7e2fe944be3948f93ff25abbca0361a609acfed1368b5 F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2118d @@ -1572,13 +1577,13 @@ F test/round1.test 29c3c9039936ed024d672f003c4d35ee11c14c0acb75c5f7d6188ff16190c F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test d27191b5ce794c05bf61081e8b2c546a1844c1641321dcaf7fb785234256cc8e -F test/rowvalue.test 0b023643162dfe3036780d78eb8daa52b144b6c2dee1216967cc72eafe19aa3d +F test/rowvalue.test 9c873b2f6e7ce72b24ef133f93515c07a6a7dac4846a344ebc2af7b8bfdf5147 F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b -F test/rowvalue3.test 1347e25ca11c547c5a6ff0cc5626f95aa9740e9275bfaec096029f57cb2130ce +F test/rowvalue3.test 103e9a224ca0548dd0d67e439f39c5dd16de4200221a333927372408c025324c F test/rowvalue4.test bac9326d1e886656650f67c0ec484eb5f452244a8209c6af508e9a862ace08ed F test/rowvalue5.test 00740304ea6a53a8704640c7405690f0045d5d2a6b4b04dde7bccc14c3068ea7 F test/rowvalue6.test d19b54feb604d5601f8614b15e214e0774c01087 -F test/rowvalue7.test c1cbdbf407029db01f87764097c6ac02a1c5a37efd2776eff32a9cdfdf6f2dba +F test/rowvalue7.test 06ec0aca725bf683313d03793aa2943bc7f45a901848c7056a9665b769c8fc38 F test/rowvalue8.test 5900eddad9e2c3c2e26f1a95f74aafc1232ee5e0 F test/rowvalue9.test 7499a8fd7ca3a3f0e19d94e135355439aa2b596f86b775ca8de79672da2ca378 F test/rowvalueA.test be8d6ad8b476eb24c151bb20bfd487e0d50c5e99618b7b0e656035069d2fc2cf @@ -1591,7 +1596,7 @@ F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test 48a645a7bb3a59a6fcf06a7364cfe5b655c336760de39068f7c241b0fc80d963 -F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa +F test/savepoint7.test 24c69af86d750c80d51cf6500fde9270717f2b6e5658f055b5e75af75d5af179 F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 F test/scanstatus.test b249328caf4d317e71058006872b8012598a5fa045b30bf24a81eeff650ab49e F test/scanstatus2.test d85d17f2b0b4c013dde95232f7beab749f11f0ef847f5ecffb9486d2f5ecf9f9 @@ -1607,10 +1612,10 @@ F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5 F test/seekscan1.test 31af16e3bb3203d153aea320939c5da97ec44705c2710d153c06a01397d45b09 F test/select1.test 692e84cfa29c405854c69e8a4027183d64c22952866a123fabbce741a379e889 F test/select2.test 352480e0e9c66eda9c3044e412abdf5be0215b56 -F test/select3.test 180223af31e1ca5537dd395ef9708ae18e651a233777fd366fd0d75469fc19c6 -F test/select4.test f0684d3da3bccacbe2a1ebadf6fb49d9df6f53acb4c6ebc228a88d0d6054cc7b +F test/select3.test 152ce3978c8600c70413e921e39e4c5aeb8ff1e96dd722442a9981e1b8afc702 +F test/select4.test 21941409ac6b65bfca83de020afb5d976d802d3d8ad216dc774a3fbbf55fe277 F test/select5.test 8afc5e5dcdebc2be54472e73ebd9cd1adef1225fd15d37a1c62f969159f390ae -F test/select6.test d455cc36cb5658ba7002ccbf23d3d392801403e64ac6516190266a8ce167ad39 +F test/select6.test da91e61d26b8dea4b61e4a862088dd6ab19998f7be22a16a5b0cfe806e597639 F test/select7.test b825420da8a0b5722fdb77f3369f6396a3d198c46e8787eb26ff9425d4ac9d27 F test/select8.test 8c8f5ae43894c891efc5755ed905467d1d67ad5d F test/select9.test f7586b207ce2304ab80dc93d3146469a28fd4403621dd3a82d06644563d3c812 @@ -1625,11 +1630,11 @@ F test/selectH.test 0b54599f1917d99568c9b929df22ec6261ed7b6d2f02a46b5945ef81b787 F test/session.test 78fa2365e93d3663a6e933f86e7afc395adf18be F test/sessionfuzz-data1.db 1f8d5def831f19b1c74571037f0d53a588ea49a6c4ca2a028fc0c27ef896dbcb F test/sessionfuzz.c f693b8827034a3bed7616d89c65fb4fe8b7ff3c0f000c6ea6beda69b7f1aced3 -F test/shared.test f022874d9d299fe913529dc10f52ad5a386e4e7ff709270b9b1111b3a0f3420a +F test/shared.test 50bd8091735b272732125928c363476a17b5fb264835de7d19e90c72055c888b F test/shared2.test 03eb4a8d372e290107d34b6ce1809919a698e879 -F test/shared3.test f8cd07c1a2b7cdb315c01671a0b2f8e3830b11ef31da6baa9a9cd8da88965403 +F test/shared3.test cb92d083003ddf0f313166e494ec2fcafa55fdebf648628923ded3169dba8850 F test/shared4.test c75f476804e76e26bf6fa0e7b421fb0ca7d07558 -F test/shared6.test 866bb4982c45ce216c61ded5e8fde4e7e2f3ffa9 +F test/shared6.test 104e1e25b4c4f47aaccca7dba75b3d87bb505b46b009af03ae49bf55b7c4976c F test/shared7.test a81e99f83e6c51b02ac99c96fb3a2a7b5978c956 F test/shared8.test 933ed7d71f598bb6c7a8c192a3cd30f2562fdccf514df383798599c34ffa672f F test/shared9.test 600a257fe9d8b0272746b230e761aa1bd8802ca4cf3ba5b2136b9204f3d51efa @@ -1637,16 +1642,17 @@ F test/sharedA.test 64bdd21216dda2c6a3bd3475348ccdc108160f34682c97f2f51c19fc0e21 F test/sharedB.test 1a84863d7a2204e0d42f2e1606577c5e92e4473fa37ea0f5bdf829e4bf8ee707 F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 -F test/shell1.test 5d84e415adf7cc4edd5913c4f23c761104ff135b9c190fcf7b430a4cbca6cb65 +F test/shell1.test d41257103cf762e1d43f1d07286ac65ed32c5430a19851029bfe06671f5e19fe F test/shell2.test ac102ebc0a9ec166257600c4ee8bdefec242163afced295f10b004f4af3fc9dd -F test/shell3.test db1953a8e59d08e9240b7cc5948878e184f7eb2623591587f8fd1f1a5bd536d8 -F test/shell4.test 522fdc628c55eff697b061504fb0a9e4e6dfc5d9087a633ab0f3dd11bcc4f807 -F test/shell5.test 0e5f8ce08206b9998a778cfe1989e20e47839153c05af2da29198150172e22fc +F test/shell3.test 603b448e917537cf77be0f265c05c6f63bc677c63a533c8e96aae923b56f4a0e +F test/shell4.test ad7eee983b5e7f1dd92d8c87bc0f39474086bc32c980c00f3934c54aabc636a2 +F test/shell5.test d17e7927ab8b7f720efbdd9b5d05fceb6c3c56c25917901b315400214bf24ef4 F test/shell6.test e3b883b61d4916b6906678a35f9d19054861123ad91b856461e0a456273bdbb8 F test/shell7.test 43fd8e511c533bab5232e95c7b4be93b243451709e89582600d4b6e67693d5c3 -F test/shell8.test aea51ecbcd4494c746b096aeff51d841d04d5f0dc4b62eb42427f16109b87acd +F test/shell8.test 641cf21a99c59404c24e3062923734951c4099a6b6b6520de00cf7a1249ee871 F test/shell9.test 8742a5b390cdcef6369f5aa223e415aa4255a4129ef249b177887dc635a87209 -F test/shmlock.test 3dbf017d34ab0c60abe6a44e447d3552154bd0c87b41eaf5ceacd408dd13fda5 +F test/shellA.test 4ecff8b7b2c0122ba8174abfbcc4b0f59e44d80f2a911068f8cd4cfc6661032d +F test/shmlock.test 9f1f729a7fe2c46c88b156af819ac9b72c0714ac6f7246638a73c5752b5fd13c F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 2668e607dcdfa19c52828c09b69685b38da793856582ae31debf79d90c7bbbdc @@ -1655,10 +1661,10 @@ F test/skipscan1.test 9cbbb6575517b15292bd87ee85b853bbd3cd4b4735d69b0f083020cec1 F test/skipscan2.test b032ed3e0ba5caa4df6c43ef22c31566aac67783bc031869155989a7ccdb5bd5 F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 0672103fd2c8f96bd114133f356192b35ece45c794fe3677e1d9e5e3104a608e -F test/skipscan6.test bddbb35dd335e2d21b7791a61e3b2e1f3255dc307ce80aa6fe19cc298e6feb13 +F test/skipscan6.test e2b256cf5d538a605beb97dc97ca5e2836dfc24c5e1d9b7a09e13c069a3b8b49 F test/snapshot.test a504f2e7009f512ef66c719f0ea1c55a556bdaf1e1312c80a04d46fc1a3e9632 F test/snapshot2.test 8d6ff5dd9cc503f6e12d408a30409c3f9c653507b24408d9cd7195931c89bc54 -F test/snapshot3.test 41350216abc6c7da37113ad462259c070786e5ad70bdc8709daaed148b1b3a2c +F test/snapshot3.test 2e0328ba019aa981848e10aded4d7dcd6094ec1f9c6290a34ab18415be0c44eb F test/snapshot4.test d4e9347ef2fcabc491fc893506c7bbaf334da3be111d6eb4f3a97cc623b78322 F test/snapshot_fault.test 129234ceb9b26a0e1000e8563a16e790f5c1412354e70749cbd78c3d5d07d60a F test/snapshot_up.test 77dc7853bfb2b4fa249f76e1714cfa1e596826165d9ef22c06ac3a0b7b778d9a @@ -1680,8 +1686,8 @@ F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 377a0c48e5a92e0b11c1c5ebb1bc9d83a7312c922bc0cb05970ef5d6a96d1f0c F test/speedtest.md ee958457ae1b729d9715ae33c0320600000bf1d9ddea1a88dcf79f56729d6fad -F test/speedtest.tcl 8a9362c1e429318e741b91d26888e7edcc326f98c3aea505ffd618cc5b9e7f0a x -F test/speedtest1.c 204acd8af326bbca2c28f68166635d4574381f4cabbac1bc243663f5dcc5051d +F test/speedtest.tcl 405411356fbc54af15987b7ffeec330a49138f71584220fb8fe1948b2f7ac907 x +F test/speedtest1.c 64b8804b053a796eab22f8b23fb181000f05d7b3e2aa44f022117ea543bc5a2a F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 @@ -1699,8 +1705,8 @@ F test/stmtvtab1.test 6873dfb24f8e79cbb5b799b95c2e4349060eb7a3b811982749a84b3594 F test/strict1.test 4d2b492152b984fd7e8196d23eb88e2ccb0ef9e46ca2f96c2ce7147ceef9d168 F test/strict2.test b22c7a98b5000aef937f1990776497f0e979b1a23bc4f63e2d53b00e59b20070 F test/subjournal.test 8d4e2572c0ee9a15549f0d8e40863161295107e52f07a3e8012a2e1fdd093c49 -F test/subquery.test 903abf41049f8404256f7be24b3151328304a5b25162e17ab0079460237382fc -F test/subquery2.test 90cf944b9de8204569cf656028391e4af1ccc8c0cc02d4ef38ee3be8de1ffb12 +F test/subquery.test 23087f9b1c15ab9cc5231d04946bdebc51db527c95eb9d7434a2222127e17a84 +F test/subquery2.test 5f06ec2dbce42a3f595ab1b73b146592f9ce001cd4ff023d887d643d3560c281 F test/subselect.test 0966aa8e720224dbd6a5e769a3ec2a723e332303 F test/substr.test a673e3763e247e9b5e497a6cacbaf3da2bd8ec8921c0677145c109f2e633f36b F test/subtype1.test 96fd2a59bfc845c955b5f339d23b37ef4d50de5f8a04acd1450a68605fa2e3e7 @@ -1709,15 +1715,15 @@ F test/swarmvtab.test 250231404fcac88f61a6c147bb0e3a118ed879278cd3ccb0ae2d3a729e F test/swarmvtab2.test c948cb2fdfc5b01d85e8f6d6504854202dc1a0782ab2a0ed61538f27cbd0aa5c F test/swarmvtab3.test 41a3ab47cb7a834d4e5336425103b617410a67bb95d335ef536f887587ece073 F test/swarmvtabfault.test 8a67a9f27c61073a47990829e92bc0c64420a807cb642b15a25f6c788210ed95 -F test/symlink.test 4368af0e213dd6e726a6240a16f2bb96a5a58f83f2d5d60652f27547b28cbf06 -F test/symlink2.test bf932ff7fe95c9dbb39d2a990df9098b0ea943233c97e40098e0a8d6b559a96f -F test/sync.test 89539f4973c010eda5638407e71ca7fddbcd8e0594f4c9980229f804d4333092 -F test/sync2.test 8f9f7d4f6d5be8ca8941a8dadcc4299e558cb6a1ff653a9469146c7a76ef2039 +F test/symlink.test 60e16915cd0ee068244563f354ae012149cf7541e922025e31ac613e3fa3e389 +F test/symlink2.test 3cf7f09dcf3cf74541f5fdb1ede3c731e4e35d2018d85efc61e32ac114435ce3 +F test/sync.test a619e407ede58a7b6e3e44375328628559fc9695a9c24c47cb5690a866b0031b +F test/sync2.test 06152269ed73128782c450c355988fe8dd794d305833af75e1a5e79edd4dae47 F test/syscall.test a067468b43b8cb2305e9f9fe414e5f40c875bb5d2cba5f00b8154396e95fcf37 F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 -F test/tabfunc01.test e85679a3800aa632dee787966b8482fce0bd47629dad82f102fd52f319d2c281 -F test/table.test 7862a00b58b5541511a26757ea9c5c7c3f8298766e98aa099deec703d9c0a8e0 -F test/tableapi.test ecbcc29c4ab62c1912c3717c48ea5c5e59f7d64e4a91034e6148bd2b82f177f4 +F test/tabfunc01.test 8a484fe8b19fc24844f72ca1ceb7c9ae8c9a6bca000a5c6ccab5d89f5cfbea4a +F test/table.test e87294bf1c80bfd7792142b84ab32ea5beb4f3f71e535d7fb263a6b2068377bf +F test/tableapi.test e37c33e6be2276e3a96bb54b00eea7f321277115d10e5b30fdb52a112b432750 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test ad0bbd92edabe64cc91d990a0748142fe5ab962d74ac71fa3bfa94d50d2f4c87 F test/tempdb.test 4cdaa23ddd8acb4d79cbb1b68ccdfd09b0537aaba909ca69a876157c2a2cbd08 @@ -1727,9 +1733,9 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d1631311a16 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl 7b44f1a9b9a2de8112695b908afc21dd9a68cd2d44e84b73f1b27b53492c0d59 -F test/testrunner.tcl 90ed8b6c2b26dc1f6af08aeb04670a5df86172f3d9828d8af000f972afa50061 x -F test/testrunner_data.tcl 63ff9eba1d11a3b0a6fc8446d5fa32da21aabda55b994e8fcbd4a8ce81f48378 +F test/tester.tcl 463ae33b8bf75ac77451df19bd65e7c415c2e9891227c7c9e657d0a2d8e1074a +F test/testrunner.tcl 614c4a28f7f730acd7bec53e17d76602fb480e0d538b6ec548169e03a093f92d x +F test/testrunner_data.tcl 02dd645b647d907c959fbf232b7ff7d869c2ae430d5117443fc1e16a0d32243a F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1855,7 +1861,7 @@ F test/tkt3357.test 77c37c6482b526fe89941ce951c22d011f5922ed F test/tkt3419.test 1bbf36d7ea03b638c15804251287c2391f5c1f6b F test/tkt3424.test 61f831bd2b071bd128fa5d00fbda57e656ca5812 F test/tkt3442.test c9d95b4c8f4f35a51b523f35d2afd0ce124937812af296545ad551ff763504fd -F test/tkt3457.test 5b9cc2b6cbbf896e9b973db83f6520f43f326f4d08604372a7b0379625e28412 +F test/tkt3457.test adf048188761581124a2f0f91f9d23a5b76fb425270ecbfd73c9c7949fa10786 F test/tkt3461.test 228ea328a5a21e8663f80ee3d212a6ad92549a19 F test/tkt3493.test 1686cbde85f8721fc1bdc0ee72f2ef2f63139218 F test/tkt3508.test d75704db9501625c7f7deec119fcaf1696aefb7d @@ -1898,7 +1904,7 @@ F test/trans.test 45f6f9ab6f66a7b5744f1caac06b558f95da62501916906cf55586a896f9f4 F test/trans2.test 62bd045bfc7a1c14c5ba83ba64d21ade31583f76 F test/trans3.test 91a100e5412b488e22a655fe423a14c26403ab94 F test/transitive1.test f8ee983600b33d167da1885657f064aec404e1c0d0bc8765fdf163f4c749237a -F test/trigger1.test 2834f8830a1ae338d95c2e3ea0c2a7bc4cda126cdeb715004cf0fd071892e44f +F test/trigger1.test 141bd2bbfa82fa91aed7391f50517373c9823ff8f55af35e56313dbc75112ca1 F test/trigger2.test 30fcb3a6aa6782020d47968735ee6086ed795f73a7affa9406c8d5a36e7b5265 F test/trigger3.test aa640bb2bbb03edd5ff69c055117ea088f121945 F test/trigger4.test 74700b76ebf3947b2f7a92405141eb2cf2a5d359 @@ -1913,7 +1919,7 @@ F test/triggerC.test 29f5a28d0fe39e6e2c01f6e1f53f08c0955170ae10a63ad023e33cb0a16 F test/triggerD.test 8e7f3921a92a5797d472732108109e44575fa650 F test/triggerE.test 612969cb57a4ef792059ad6d01af0117e1ae862c283753ffcc9a6428642b22ee F test/triggerF.test 5d76f0a8c428ff87a4d5ed52da06f6096a2c787a1e21b846111dfac4123de3ad -F test/triggerG.test 2b816093c91ba73c733cfa8aedcc210ad819d72a98b1da30768a3c56505233e9 +F test/triggerG.test b4e3fbccde6cf8995177cd6cad880256c8c00e407e07d8c67149f46106292a2c F test/triggerupfrom.test d1f9e56090408115c522bee626cc33a2f3370f627a5e341d832589d72e3aa271 F test/trustschema1.test d2996bb284859c99956ac706160eab9f086919da738d19bfef3ac431cce8fd47 F test/tt3_checkpoint.c ac7ca661d739280c89d9c253897df64a59a49369bd1247207ac0f655b622579d @@ -1933,9 +1939,9 @@ F test/unionvtab.test e1704ab1b4c1bb3ffc9da4681f8e85a0b909fd80b937984fc94b27415a F test/unionvtabfault.test e8759f3d14fb938ce9657e2342db34aeac0fb9bc1692b0d1ebb0069630151d06 F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 3674e9f2a3f1fbbfd4772ac74b7a97090d0f77d2 -F test/unixexcl.test d936ba2b06794018e136418addd59a2354eeae97 +F test/unixexcl.test d2366ef2d3d95249314307861d748924d9ab4f24305541159a08be61ccd4a9ee F test/unordered.test 0edaf3411d300693bca595897c5201421c6c5ec787990a1dfe2f7f60ae93f1e2 -F test/update.test 85d3f46d0863033370bd881b1097f5694369a8730e53c5f85d96f32b7b310b47 +F test/update.test 258dcf26d401177d3cb7fdf0beab14d671dbe72e8ff6e0435fd85a08fcd57bd9 F test/update2.test 67455bc61fcbcf96923c45b3bc4f87bc72be7d67575ad35f134906148c7b06d3 F test/upfrom1.tcl 8859d9d437f03b44174c4524a7a734a391fd4526fcff65be08285dafc9dc9041 F test/upfrom1.test 8cb06689e99cd707d884faa16da0e8eb26ff658bb01c47ddf72fadade666e6e1 @@ -1943,16 +1949,16 @@ F test/upfrom2.test 66f3ebf721b3cebd922faee5c386bf244f816d416b57c000753ff51af623 F test/upfrom3.test 6130f24ebf97f5ea865e5d2a14a2d543fe5428a62e87cc60f62d875e45c1f5f0 F test/upfrom4.test 78f742a6577c91a7a55c64edb8811004e7c6aa99b8d57b2320f70a918c357807 F test/upfromfault.test 3a10075a0043f0c4fad6614b2c371f88a8ba5a4acab68b907438413865d6a8d6 -F test/upsert1.test beba4316fbd4b7b9d76784313f6129a548cfe7abea04d46db33e2efce1ab0ac2 +F test/upsert1.test 77e3cbabd6b5c773056aca39bda7a690901f1dbd08b0a26ee3b5aec9e9e26198 F test/upsert2.test 720e94d09f7362a282bc69b3c6b83d51daeaaf0440eb4920a08b86518b8c7496 F test/upsert3.test 88d7d590a1948a9cb6eac1b54b0642f67a9f35a1fc0f19b200e97d5d39e3179c F test/upsert4.test 25d2a1da92f149331ae0c51ca6e3eee78189577585eab92de149900d62994fa5 F test/upsert5.test 9953b180d02d1369cdbb6c73c900834e5fef8cb78e98e07511c8762ec21cc176 F test/upsertfault.test f21ca47740841fdb4d61acfa7b17646d773e67724fe8c185b71c018db8a94b35 -F test/uri.test c1abaaaa28e9422d61e5f3f9cbc8ef993ec49fe802f581520731708561d49384 +F test/uri.test 1250724af9beeed2d6c3716f5b990c483200c54f408d3c0ec9543a3c7961f8fc F test/uri2.test 9d3ba7a53ee167572d53a298ee4a5d38ec4a8fb7 F test/utf16align.test 9fde0bb5d3a821594aa68c6829ab9c5453a084384137ebb9f6153e2d678039da -F test/vacuum-into.test 77845cee98770c416dae9b0da6bb3229753861f2da65c11b4f9715d081712d8a +F test/vacuum-into.test 5a489714feecfdabfc7b293be4111564a173dee92c0d6818dd0207f3ade65783 F test/vacuum.test ce91c39f7f91a4273bf620efad21086b5aa6ef1d F test/vacuum2.test 9fd45ce6ce29f5614c249e03938d3567c06a9e772d4f155949f8eafe2d8af520 F test/vacuum3.test d9d9a04ee58c485b94694fd4f68cffaba49c32234fdefe1ac1a622c5e17d4ce3 @@ -1968,6 +1974,7 @@ F test/view.test 3c23d7a068e9e4a0c4e6907498042772adea725f0630c3d9638ffd4e5a08b92 F test/view2.test db32c8138b5b556f610b35dfddd38c5a58a292f07fda5281eedb0851b2672679 F test/view3.test ad8a8290ee2b55ff6ce66c9ef1ce3f1e47926273a3814e1c425293e128a95456 F test/vt02.c 5b44ac67b1a283fedecf2d6e2ceda61e7a157f01d44dcb4490dcb1e87d057060 +F test/vt100-a.sql 631eeab18c5adb531bab79aecf64eee3934b42c75a309ee395c814717a6a7651 F test/vtab1.test 09a72330d0f31eda2ffaa828b06a6b917fb86250ee72de0301570af725774c07 F test/vtab2.test 14d4ab26cee13ba6cf5c5601b158e4f57552d3b055cdd9406cf7f711e9c84082 F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e @@ -1983,7 +1990,7 @@ F test/vtabC.test 4528f459a13136f982e75614d120aef165f17292 F test/vtabD.test 05b3f1d77117271671089e48719524b676842e96 F test/vtabE.test 2a143fe75a11275781d1fd1988d86b66a3f69cb98f4add62e3da8fd0f637b45f F test/vtabF.test 1918844c7c902f6a16c8dacf1ec8f84886d6e78b -F test/vtabH.test 8e338acba32207085b6fe9cb2a58f7b408e74c8e1a2964cbdaca903ac82213cc +F test/vtabH.test a9417d92111629e2c33e0f3b06cc84bcd611753d0fc017b3ce6ae96572bc2f2f F test/vtabI.test 751b07636700dbdea328e4265b6077ccd6811a3f F test/vtabJ.test a6aef49d558af90fae10565b29501f82a95781cb4f797f2d13e2d19f9b6bc77b F test/vtabK.test 13293177528fada1235c0112db0d187d754af1355c5a39371abd365104e3afbf @@ -1995,44 +2002,48 @@ F test/vtabdistinct.test 7688f0889358f849fd60bbfde1ded38b014b18066076d4bfbb75395 F test/vtabdrop.test 65d4cf6722972e5499bdaf0c0d70ee3b8133944a4e4bc31862563f32a7edca12 F test/vtabrhs1.test 9b5ecbc74a689500c33a4b2b36761f9bcc22fcc4e3f9d21066ee0c9c74cf5f6c F test/wal.test 519c550255c78f55959e9159b93ebbfad2b4e9f36f5b76284da41f572f9d27da -F test/wal2.test e89ca97593b5e92849039f6b68ce1719a853ef20fa22c669ec1ac452fbc31cab +F test/wal2.test f058016abe4627d2664db4b4b87990298d925e66a4c5a2c8e674a0ff6f4c841d F test/wal3.test 5de023bb862fd1eb9d2ad26fa8d9c43abb5370582e5b08b2ae0d6f93661bc310 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 -F test/wal6.test b602704e4b066199bc89d91ca9000f335dcf4572 -F test/wal64k.test 2a525c0f45d709bae3765c71045ccec5df7d100ccbd3a7860fdba46c9addb965 +F test/wal6.test 6a773eff47b989c5142d17f2a7778c02d8260149a648d44ef8345aa080e428e3 +F test/wal64k.test bb8c52f0140aae1de877ffed86e2a97d903f98cf9ac263f185d51c58cde92327 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test d9df3fba4caad5854ed69ed673c68482514203c8 F test/wal9.test 378e76a9ad09cd9bee06c172ad3547b0129a6750 F test/wal_common.tcl 204d1721ac13c5e0c7fae6380315b5ab7f4e8423f580d826c5e9df1995cb018d F test/walbak.test 018d4e5a3d45c6298d11b99f09a8ef6876527946 F test/walbig.test f437473a16cfb314867c6b5d1dbcd519e73e3434 -F test/walblock.test be48f3a75eff0b4456209f26b3ce186c2015497d +F test/walblock.test 6bb472e82730e7e4e81395e907a01d8cfc2bd9e1f01f8a9184ca572e2955a4bf F test/walcksum.test ba02b4fe6d22cb42e57a323003cbae62f77a740983e1355b2b520e019ae261c7 F test/walcrash.test 21038858cc552077b0522f50b0fa87e38139306a F test/walcrash2.test a0edab4e5390f03b99a790de89aad15d6ec70b36 F test/walcrash3.test e426aa58122d20f2b9fbe9a507f9eb8cab85b8af -F test/walcrash4.test e7b6e7639a950a0cca8e210e248c8dad4d63bf20 +F test/walcrash4.test 93d8825e9d0b1b183e3a73ee67e5c8fc9d86cdaf1f3a03bcc960c3aeede28001 F test/walfault.test 09b8ad7e52d2f54bce50e31aa7ea51412bb9f70ac13c74e669ddcd8b48b0d98d F test/walfault2.test e039ac66c78d5561683cacde04097213cdad3b58e2b3f3fe1112862217bfd915 F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483 -F test/walmode.test cd6e7cff618eaaa5910ce57c3657aa50110397f86213886a2400afb9bfec7b7b -F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496 +F test/walmode.test 2a5530972948948a17211e070263fcf25ef1ca4e06d742a32d81a470b91441dc +F test/walnoshm.test 844b3eb7d8e8ee76c834ef723babec57b0be51fa52ef7e321c289ed0fe3cddc2 F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8d78a1ec91299163451417b451a2bac3481f8eb9f455b1ca507a6625c927ca6e F test/walprotocol.test 1b3f922125e341703f6e946d77fdc564d38fb3e07a9385cfdc6c99cac1ecf878 F test/walprotocol2.test 7d3b6b4bf0b12f8007121b1e6ef714bc99101fb3b48e46371df1db868eebc131 -F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20 +F test/walro.test 78a84bc0fdae1385c06b017215c426b6845734d6a5a3ac75c918dd9b801b1b9d F test/walro2.test 33955a6fd874dd9724005e17f77fef89d334b3171454a1256fe4941a96766cdc F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68 F test/walseh1.test bae700eb99519b6d5cd3f893c04759accc5a59c391d4189fe4dd6995a533442b -F test/walsetlk.test 34c901443b31ab720afc463f5b236c86ca5c4134402573dce91aa0761de8db5a +F test/walsetlk.test 9079cd8ef82570b8cf0067f31e049a72bec353fb2d5f0cc88f1736dc42ba9704 +F test/walsetlk2.test 9097083633cdf55bf1098b694fb8651d0356d38fef28b869481d18029d7ceaf4 +F test/walsetlk3.test 1b82bd92dea7e58f498b4399b0b3d26773dd8ac5c74205ce4a23c207cb8e85fe +F test/walsetlk_recover.test adccbffc59e365063a4efd2da6b661ae2fcf15d775b6719fe46acd87face08ff +F test/walsetlk_snapshot.test 86d5588380f9927d8fcbbd75133b0a34fddf959378d6823c6f164a390123f70a F test/walshared.test 42e3808582504878af237ea02c42ca793e8a0efaa19df7df26ac573370dbc7a3 F test/walslow.test 0c51843836c9dcf40a5ac05aa781bfb977b396ee2c872d92bd48b79d5dd9aa23 F test/walthread.test 14b20fcfa6ae152f5d8e12f5dc8a8a724b7ef189f5d8ef1e2ceab79f2af51747 F test/walvfs.test e1a6ad0f3c78e98b55c3d5f0889cf366cc0d0a1cb2bccb44ac9ec67384adc4a1 -F test/where.test 59abb854eee24f166b5f7ba9d17eb250abc59ce0a66c48912ffb10763648196d -F test/where2.test 03c21a11e7b90e2845fc3c8b4002fc44cc2797fa74c86ee47d70bd7ea4f29ed6 +F test/where.test 5087c72d26fd075a1644c8512be9fe18de9bf2d2b0754f7fd9b74a1c6540c4fc +F test/where2.test d7d546b8919ed223fb58a6590de296bcc2d5996478412f00bf865030e4f89b3c F test/where3.test 4ccb156ae33de86414a52775a6f590a9d60ba2cbc7a93a24fa331b7bcf5b6030 F test/where4.test 4a371bfcc607f41d233701bdec33ac2972908ba8 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 @@ -2046,12 +2057,12 @@ F test/whereC.test cae295158703cb3fc23bf1a108a9ab730efff0f6 F test/whereD.test c1c335e914e28b122e000e9310f02d2be83e1c9dbca2e29f46bd732703944d1b F test/whereE.test 7a727b5d5b6bc8fa4cef5206e90cc0363e55ca7f0566f6fbad0206e43170f59e F test/whereF.test 926b65519608e3f2aa28720822b9154fb5c7b13519dd78194f434a511ab3dac5 -F test/whereG.test 649d5ad02a87a76ec2ac8de9441e2c83a4dd0f29e459a31215c0533788c6bf07 +F test/whereG.test 875d020ac0a47828b31e36c54f1bf0cf81c9ea43b257bc21286eca1fe9a4880b F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/whereI.test c4bb7e2ca56d49bd8ab5c7bd085b8b83e353922b46904d68aefb3c7468643581 F test/whereJ.test fc05e374cc9f2dc204148d6c06822c380ad388895fe97a6d335b94a26a08aecf F test/whereK.test 0270ab7f04ba5436fb9156d31d642a1c82727f4c4bfe5ba90d435c78cf44684a -F test/whereL.test 438a397fa883b77bb6361c08a8befa41b52e9cfbe15a2a43715d122f8cfa8649 +F test/whereL.test cb115604cc9bd61acbc99a1f1df0eb1ea7a7875a77fef25ba9282f01d10283e1 F test/whereM.test 0dbc9998783458ddcf3cc078ca7c2951d8b2677d472ecf0028f449ed327c0250 F test/whereN.test 63a3584b71acfb6963416de82f26c6b1644abc5ca6080c76546b9246734c8803 F test/wherefault.test 6cf2a9c5712952d463d3f45ebee7f6caf400984df51a195d884cfb7eb0e837a7 @@ -2060,11 +2071,11 @@ F test/wherelimit.test afb46397c6d7e964e6e294ba3569864a0c570fe3807afc634236c2b75 F test/wherelimit2.test b9e4bfe7b4d7c2f85f99cf2bd2c51369378d04b1f3d1b60557423752003bfd90 F test/wherelimit3.test 22d73e046870cf8bbe15573eda6b432b07ebe64a88711f9f849c6b3667c1fae6 F test/widetab1.test c296a98e123762de79917350e45fa33fdf88577a2571eb3a64c8bf7e44ef74d1 -F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2aeee74 -F test/win32lock.test e0924eb8daac02bf80e9da88930747bd44dd9b230b7759fed927b1655b467c9c -F test/win32longpath.test 304006024ca47104bf5a7415ef31ca83ecfc29351af202baf8588b880cffc116 -F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc -F test/window1.test 79dc3b9a2226f622d7e104a1fc750d1c4c3c08d6147b59085bdbe05352947ffa +F test/win32heap.test 1ec2ce646aee491ec23bfcdfd005b33c79f13bf91467966f374a76ffe7c7e85f +F test/win32lock.test e56d7a9b6cf9d5f3867c2dd19ff36c5326881e4038c6867610ecb3a9868ea4eb +F test/win32longpath.test 0f9837039b306735c13521c5f25b6ed42937b600dace58e28a3d2f8baf429b6a +F test/win32nolock.test 95854dc0206b8a95e4aee15a76acc082767b38f079b2e24676aed6cbb0f32798 +F test/window1.test 2a6970e27fd1084c6ded139b0fcd451e116c63f5df3868fdf10abe037d29bd4d F test/window2.tcl 492c125fa550cda1dd3555768a2303b3effbeceee215293adf8871efc25f1476 F test/window2.test e466a88bd626d66edc3d352d7d7e1d5531e0079b549ba44efb029d1fbff9fd3c F test/window3.tcl acea6e86a4324a210fd608d06741010ca83ded9fde438341cb978c49928faf03 @@ -2087,14 +2098,14 @@ F test/windowerr.tcl f5acd6fbc210d7b5546c0e879d157888455cd4a17a1d3f28f07c1c8a387 F test/windowerr.test a8b752402109c15aa1c5efe1b93ccb0ce1ef84fa964ae1cd6684dd0b3cc1819b F test/windowfault.test 15094c1529424e62f798bc679e3fe9dfab6e8ba2f7dfe8c923b6248c31660a7c F test/windowpushd.test c420e2265f0e09a0e798d0513a660d71b51602088d81b3dbd038918ee1339dcc -F test/with1.test b93833890e5d2a368e78747f124503a0159aa029b98e9ed4795ebf630b2efd3d +F test/with1.test 1ee171d7c306ab8b0771f3511d870f56c735607729836585bbceb1fc2f47e0b1 F test/with2.test 181674a6cc86a601ca2ac052741cdfad5b529e07e870435d2f6cdb92d589ff17 F test/with3.test e30369ea27aa27eb1bda4c5e510c8a9f782c8afd2ab99d1a02b8a7f25a5d3e65 F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205 F test/with5.test 6248213c41fab36290b5b73aa3f937309dfba337004d9d8434c3fabc8c7d4be8 F test/with6.test 281e4861b5e517f6c3c2f08517a520c1e2ee7c11966545d3901f258a4fe8ef76 F test/withM.test 693b61765f2b387b5e3e24a4536e2e82de15ff64 -F test/without_rowid1.test a5210b8770dc4736bca4e74bc96588f43025ad03ad6a80f885afd36d9890e217 +F test/without_rowid1.test f6e75e32821eb423ac3812434d12bdd8098f17e3b2206da61575e1db77f82428 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 F test/without_rowid3.test 39ab0dd773eaa62e59b17093f875327630f54c4145458f6d2b053d68d4b2f67b F test/without_rowid4.test 4e08bcbaee0399f35d58b5581881e7a6243d458a @@ -2102,19 +2113,19 @@ F test/without_rowid5.test f14298eb5ac8013894b75141c3f4f5f325a6ad0eded55516eef72 F test/without_rowid6.test efbd7add62c59bf5ca97bf8da674e734e6a70ef979234e816166824b4d258f68 F test/without_rowid7.test d7c59a93d726b55812d620f8f284e01904a5b85f9ee9eea8f2f68571a5e8c40e F test/wordcount.c d721a4b6fae93e6e33449700bce1686bc23257c27425bc3ef1599dc912adec66 -F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc +F test/writecrash.test 13520af28f376bfc8c0bcd130efc1fff20bb165198e8b94cf153f1f754154bb9 F test/zeroblob.test 7b74cefc7b281dfa2b07cd237987fbe94b4a2037a7771e9e83f2d5f608b1d99e F test/zeroblobfault.test 861d8191a0d944dfebb3cb4d2c5b4e46a5a119eaec5a63dd996c2389f8063441 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc -F test/zipfile.test a36327c5697a03150a313ba06ab45842facef8b0c21be19d73a3a4fee58bc54c +F test/zipfile.test c8a7736312c8eb32ee2808121ca22d885a8724d8df4a23d312ddefddadc7f322 F test/zipfile2.test 6df5f5ef9d247756f7200066f43e7f3f52cffff47f0c02cbefe4ce9c3284cb10 F test/zipfilefault.test 44d4d7a7f7cca7521d569d7f71026b241d65a6b1757aa409c1a168827edbbc2c F tool/GetFile.cs 47852aa0d806fe47ed1ac5138bdce7f000fe87aaa7f28107d0cb1e26682aeb44 F tool/GetTclKit.bat d84033c6a93dfe735d247f48ba00292a1cc284dcf69963e5e672444e04534bbf F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 -F tool/build-all-msvc.bat c817b716e0edeecaf265a6775b63e5f45c34a6544f1d4114a222701ed5ac79ab x +F tool/build-all-msvc.bat 1960a7a3e5d8176c4329e31476f6e3dfa9543675355fa9020a569f4452628458 x F tool/build-shell.sh 369c4b171cc877ad974fef691e4da782b4c1e99fe8f4361316c735f64d49280f -F tool/buildtclext.tcl 20726b6b73c7911baa8519a9467b4062104339a5ce57947819884525c56d79e3 +F tool/buildtclext.tcl 85d1bcd5410d0a4c739e2d013106f03674d082cc4d573bcc57a68546b6db67cd F tool/cg_anno.tcl c1f875f5a4c9caca3d59937b16aff716f8b1883935f1b4c9ae23124705bc8099 x F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 F tool/cktclsh.sh 6075eef9c6b9ba4b38fef2ca2a66d25f2311bd3c610498d18a9b01f861629cca @@ -2128,38 +2139,35 @@ F tool/enlargedb.c 3e8b2612b985cfa7e3e8800031ee191b43ae80de96abb5abbd5eada62651e F tool/extract-sqlite3h.tcl 069ceab0cee26cba99952bfa08c0b23e35941c837acabe143f0c355d96c9e2eb x F tool/extract.c 054069d81b095fbdc189a6f5d4466e40380505e2 F tool/fast_vacuum.c c129ae2924a48310c7b766810391da9e8fda532b9f6bd3f9a9e3a799a1b42af9 -F tool/find_tclconfig.tcl e64886ffe3b982d4df42cd28ed91fe0b5940c2c5785e126c1821baf61bc86a7e F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439 F tool/fuzzershell.c 41480c8a1e4749351f381431ecfdfceba645396c5d836f8d26b51a33c4a21b33 -F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 +F tool/genfkey.README e550911fa984c8255ebed2ef97824125d83806eb5232582700de949edf836eb1 F tool/genfkey.test b6afd7b825d797a1e1274f519ab5695373552ecad5cd373530c63533638a5a4f F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/index_usage.c f62a0c701b2c7ff2f3e21d206f093c123f222dbf07136a10ffd1ca15a5c706c5 -F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f -F tool/lemon.c 2418ee31f65764d150f7dd87ef00b4408f1b01a55db0b30bed673a3e336ae718 -F tool/lempar.c e6b649778e5c027c8365ff01d7ef39297cd7285fa1f881cce31792689541e79f +F tool/lemon.c 00535f27e61ae8200ba8402c1753483de39eef2f59b60da8b7951c458a32e017 +F tool/lempar.c bdffd3b233a4e4e78056c9c01fadd2bb3fe902435abde3bce3d769fdf0d5cca2 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c c34e5944318415de513d29a6098df247a9618c96d83c38d4abd88641fe46e669 F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439 F tool/merge-test.tcl de76b62f2de2a92d4c1ca4f976bce0aea6899e0229e250479b229b2a1914b176 F tool/mkamalzip.tcl 8aa5ebe7973c8b8774062d34e15fea9815c4cc2ceea3a9b184695f005910876a -F tool/mkautoconfamal.sh c5e65fa1c922f2e3b3e4f6cd0331ec7d84bdef085f32cb1c46673cdf95ec8090 -F tool/mkccode.tcl 210159febe0ef0ecbc53c79833500663ceaba0115b2b374405818dc835b5f84b x -F tool/mkctimec.tcl ef6a67ec82e5b6fc19152a4c79f237227b18bf67ff16d155bac7adb94355d9cf x +F tool/mkautoconfamal.sh 564378ae48cc8f4c8c68ef6d00228a0b2a70e2e3e4c67f26be1dd05d9730fefd +F tool/mkccode.tcl c42a8f8cf78f92e83795d5447460dbce7aaf78a3bbf9082f1507dc71a3665f3c x +F tool/mkctimec.tcl 11c9eda4a8d18c74b79280b30506d832849fd1855e6d9e95e1fd506f1d211c37 x F tool/mkkeywordhash.c 6b0be901c47f9ad42215fc995eb2f4384ac49213b1fba395102ec3e999acf559 F tool/mkmsvcmin.tcl d76c45efda1cce2d4005bcea7b8a22bb752e3256009f331120fb4fecb14ebb7a F tool/mkopcodec.tcl 33d20791e191df43209b77d37f0ff0904620b28465cca6990cf8d60da61a07ef F tool/mkopcodeh.tcl 2b4e6967a670ef21bf53a164964c35c6163277d002a4c6f56fa231d68c88d023 F tool/mkopts.tcl 680f785fdb09729fd9ac50632413da4eadbdf9071535e3f26d03795828ab07fa -F tool/mkpragmatab.tcl f9339d207de40cb57e8dc7a53817dac8e63a4d4f38a7b815e62cd67f68be70d8 +F tool/mkpragmatab.tcl 3801ce32f8c55fe63a3b279f231fb26c2c1a2ea9a09d2dd599239d87a609acec F tool/mkshellc.tcl 9ce74de0fa904a2c56a96f8d8b5261246bacb0eaa8d7e184f9e18ff94145ebbc F tool/mksourceid.c 36aa8020014aed0836fd13c51d6dc9219b0df1761d6b5f58ff5b616211b079b9 -F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 351c55256213154cabb051a3c870ef9f4487de905015141ae50dc7578a901b84 -F tool/mksqlite3c.tcl ba13086555b3cb835eba5e47a9250300ab85304d23fd1081abd3f29d8ab71a2b +F tool/mksqlite3c.tcl f11b63445c4840509248bd4aa151a81aea25d5415fef71943c8d436eba4f3b3c F tool/mksqlite3h.tcl 989948c6a26e188e673d7c2f2f093ea3acd816ad6ac65bab596280075c8f3a45 -F tool/mksqlite3internalh.tcl eb994013e833359137eb53a55acdad0b5ae1049b +F tool/mksqlite3internalh.tcl 46ef6ed6ccd3c36e23051109dd25085d8edef3887635cea25afa81c4adf4d4db F tool/mksrczip.tcl 81efd9974dbb36005383f2cd655520057a2ae5aa85ac2441a80c7c28f803ac52 F tool/mktoolzip.tcl 34b4e92be544f820e2cc26f143f7d5aec511e826ec394cc82969a5dcf7c7a27c F tool/mkvsix.tcl 67b40996a50f985a573278eea32fc5a5eb6110bdf14d33f1d8086e48c69e540a @@ -2171,48 +2179,40 @@ F tool/pagesig.c f98909b4168d9cac11a2de7f031adea0e2f3131faa7515a72807c03ec58eafe F tool/replace.tcl 511c61acfe563dfb58675efb4628bb158a13d48ff8322123ac447e9d25a82d9a F tool/restore_jrnl.tcl 1079ecba47cc82fa82115b81c1f68097ab1f956f357ee8da5fc4b2589af6bd98 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 -F tool/run-speed-test.sh f95d19fd669b68c4c38b6b475242841d47c66076 -F tool/showdb.c 81b04bfaa9a63665f75945947323aa68b820570aa156b1574f440fc8276092c6 +F tool/showdb.c 3956d71e5193162609a60e8c9edfcf09274c00cfea2b1d221261427adb2b5cca F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818 F tool/showlocks.c 9cc5e66d4ebbf2d194f39db2527ece92077e86ae627ddd233ee48e16e8142564 F tool/showshm.c a0ab6ec32dd1f11218ca2a4018f8fb875b59414801ab8ceed8b2e69b7b45a809 F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c2a1 F tool/showwal.c 11eca547980a066b081f512636151233350ac679f29ecf4ebfce7f4530230b3d -F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe +F tool/soak1.tcl a3892082ed1079671565c044e93b55c3c7f38829aedf53cc597c65d23ffdaddf F tool/spaceanal.tcl 1f83962090a6b60e1d7bf92495d643e622bef9fe82ea3f2d22350dcbce9a12d0 -F tool/speed-check.sh e566ab3934d7d78631743a984ad3f67c331c911bb18ff5d0a6c616a2afee7f91 -F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 -F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e -F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff -F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 -F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x -F tool/split-sqlite3c.tcl 07e18a1d8cc3f6b3a4a1f3528e63c9b29a5c8a7bca0b8d394b231da464ce1247 -F tool/sqldiff.c 2a0987d183027c795ced13d6749061c1d2f38e24eddb428f56fa64c3a8f51e4b -F tool/sqlite3_analyzer.c.in fc7735c499d226a49d843d8209b2543e4e5229eeb71a674c331323a2217b65b4 -F tool/sqlite3_rsync.c 9a1cca2ab1271c59b37a6493c15dc1bcd0ab9149197a9125926bc08dd26b83fb +F tool/split-sqlite3c.tcl 4969fd642dad0ea483e4e104163021d92baf98f6a8eac981fe48525f9b873430 +F tool/sqldiff.c 134be7866be19f8beb32043d5aea5657f01aaeae2df8d33d758ff722c78666b9 +F tool/sqlite3_analyzer.c.in 14f02cb5ec3c264cd6107d1f1dad77092b1cf440fc196c30b69ae87b56a1a43b +F tool/sqlite3_rsync.c 4e152221a51ed2974f08100d86500c2cda549472e48e09699a8011bfb956d00c F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 -F tool/src-verify.c d00f93263aa2fa6ba0cba0106d95458e6effb94fdb5fc634f56834f90c05bbb4 -F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f -F tool/srctree-check.tcl 9dfdaf37653df0b58a309c405cc115026468eee4c1ffad5f2af7428c531e554d +F tool/src-verify.c 6c655d9a8d6b30f3648fc78a79bf3838ed68f8543869d380c43ea9f17b3b8501 +F tool/srcck1.c 559e703c6cca1d70398bdba1d7f91036c1a71adf718a1aaa6401a562ccaed154 +F tool/srctree-check.tcl fa4d82dd3e8a38d5cbce7d6ade8abef2f42b9eca0394484d521dc8d086739460 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 -F tool/stripccomments.c dfe9cc03cf87728ac9836be30763f8aa52b82caca0780b3d3f3572e4643b01d3 +F tool/stripccomments.c 68d2aa8cb504439f541ce66b8f128067612bdd16f5fb7bfe540f3fcb67c9c197 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d F tool/symbols.sh 1612bd947750e21e7b47befad5f6b3825b06cce0705441f903bf35ced65ae9b9 -F tool/tclConfigShToAutoDef.sh 44ec55046d86a3febb2cb3e099399b41794e80e9cd138eee7b9b016f819e882b x F tool/tclConfigShToMake.sh 7c065d81c2d178e15e45a77372c6e5a38b5a1b08755301cd6f20a3a862db7312 x F tool/varint.c 5d94cb5003db9dbbcbcc5df08d66f16071aee003 F tool/vdbe-compress.tcl fa2f37ab39b2a0087fafb6a7f3ce19503e25e624ffa8ed9951717ab72920c088 F tool/vdbe_profile.tcl 3ac5a4a9449f4baf77059358ea050db3e34395ccf59c5464d29b91746d5b961e F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 -F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 +F tool/warnings.sh 1ad0169b022b280bcaaf94a7fa231591be96b514230ab5c98fbf15cd7df842dd F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9d1f01aac909a0689ab881e269c3f1e4b583b0b135689a39fd2822de7a059e5f -R 95cfc65226e423aa18b9e0671f876db9 +P cbab5d86517f0c57c6025aaddbb9408e29bccdc8b158d2b8d40bd2f3b333ef69 +R 2e4328669cf34776de6f2dce452a9a18 T +sym-release * -T +sym-version-3.49.2 * +T +sym-version-3.50.1 * U drh -Z 26aa19dada18db86c2026c3614aef5cb +Z 9176f9c7403d4ffbbc5c61ed5ec557f1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bbf46906f..7df133b85 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -17144570b0d96ae63cd6f3edca39e27ebd74925252bbaf6723bcb2f6b4861fb1 +b77dc5e0f596d2140d9ac682b2893ff65d3a4140aa86067a3efebe29dc914c95 diff --git a/src/alter.c b/src/alter.c index ff2075758..a7255e75e 100644 --- a/src/alter.c +++ b/src/alter.c @@ -531,13 +531,13 @@ void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ assert( pNew->nCol>0 ); nAlloc = (((pNew->nCol-1)/8)*8)+8; assert( nAlloc>=pNew->nCol && nAlloc%8==0 && nAlloc-pNew->nCol<8 ); - pNew->aCol = (Column*)sqlite3DbMallocZero(db, sizeof(Column)*nAlloc); + pNew->aCol = (Column*)sqlite3DbMallocZero(db, sizeof(Column)*(u32)nAlloc); pNew->zName = sqlite3MPrintf(db, "sqlite_altertab_%s", pTab->zName); if( !pNew->aCol || !pNew->zName ){ assert( db->mallocFailed ); goto exit_begin_add_column; } - memcpy(pNew->aCol, pTab->aCol, sizeof(Column)*pNew->nCol); + memcpy(pNew->aCol, pTab->aCol, sizeof(Column)*(size_t)pNew->nCol); for(i=0; inCol; i++){ Column *pCol = &pNew->aCol[i]; pCol->zCnName = sqlite3DbStrDup(db, pCol->zCnName); @@ -632,10 +632,8 @@ void sqlite3AlterRenameColumn( ** altered. Set iCol to be the index of the column being renamed */ zOld = sqlite3NameFromToken(db, pOld); if( !zOld ) goto exit_rename_column; - for(iCol=0; iColnCol; iCol++){ - if( 0==sqlite3StrICmp(pTab->aCol[iCol].zCnName, zOld) ) break; - } - if( iCol==pTab->nCol ){ + iCol = sqlite3ColumnIndex(pTab, zOld); + if( iCol<0 ){ sqlite3ErrorMsg(pParse, "no such column: \"%T\"", pOld); goto exit_rename_column; } @@ -1138,6 +1136,7 @@ static int renameParseSql( int bTemp /* True if SQL is from temp schema */ ){ int rc; + u64 flags; sqlite3ParseObjectInit(p, db); if( zSql==0 ){ @@ -1146,11 +1145,21 @@ static int renameParseSql( if( sqlite3StrNICmp(zSql,"CREATE ",7)!=0 ){ return SQLITE_CORRUPT_BKPT; } - db->init.iDb = bTemp ? 1 : sqlite3FindDbName(db, zDb); + if( bTemp ){ + db->init.iDb = 1; + }else{ + int iDb = sqlite3FindDbName(db, zDb); + assert( iDb>=0 && iDb<=0xff ); + db->init.iDb = (u8)iDb; + } p->eParseMode = PARSE_MODE_RENAME; p->db = db; p->nQueryLoop = 1; + flags = db->flags; + testcase( (db->flags & SQLITE_Comments)==0 && strstr(zSql," /* ")!=0 ); + db->flags |= SQLITE_Comments; rc = sqlite3RunParser(p, zSql); + db->flags = flags; if( db->mallocFailed ) rc = SQLITE_NOMEM; if( rc==SQLITE_OK && NEVER(p->pNewTable==0 && p->pNewIndex==0 && p->pNewTrigger==0) @@ -1213,10 +1222,11 @@ static int renameEditSql( nQuot = sqlite3Strlen30(zQuot)-1; } - assert( nQuot>=nNew ); - zOut = sqlite3DbMallocZero(db, nSql + pRename->nList*nQuot + 1); + assert( nQuot>=nNew && nSql>=0 && nNew>=0 ); + zOut = sqlite3DbMallocZero(db, (u64)nSql + pRename->nList*(u64)nQuot + 1); }else{ - zOut = (char*)sqlite3DbMallocZero(db, (nSql*2+1) * 3); + assert( nSql>0 ); + zOut = (char*)sqlite3DbMallocZero(db, (2*(u64)nSql + 1) * 3); if( zOut ){ zBuf1 = &zOut[nSql*2+1]; zBuf2 = &zOut[nSql*4+2]; @@ -1228,16 +1238,17 @@ static int renameEditSql( ** with the new column name, or with single-quoted versions of themselves. ** All that remains is to construct and return the edited SQL string. */ if( zOut ){ - int nOut = nSql; - memcpy(zOut, zSql, nSql); + i64 nOut = nSql; + assert( nSql>0 ); + memcpy(zOut, zSql, (size_t)nSql); while( pRename->pList ){ int iOff; /* Offset of token to replace in zOut */ - u32 nReplace; + i64 nReplace; const char *zReplace; RenameToken *pBest = renameColumnTokenNext(pRename); if( zNew ){ - if( bQuote==0 && sqlite3IsIdChar(*pBest->t.z) ){ + if( bQuote==0 && sqlite3IsIdChar(*(u8*)pBest->t.z) ){ nReplace = nNew; zReplace = zNew; }else{ @@ -1255,14 +1266,15 @@ static int renameEditSql( memcpy(zBuf1, pBest->t.z, pBest->t.n); zBuf1[pBest->t.n] = 0; sqlite3Dequote(zBuf1); - sqlite3_snprintf(nSql*2, zBuf2, "%Q%s", zBuf1, + assert( nSql < 0x15555554 /* otherwise malloc would have failed */ ); + sqlite3_snprintf((int)(nSql*2), zBuf2, "%Q%s", zBuf1, pBest->t.z[pBest->t.n]=='\'' ? " " : "" ); zReplace = zBuf2; nReplace = sqlite3Strlen30(zReplace); } - iOff = pBest->t.z - zSql; + iOff = (int)(pBest->t.z - zSql); if( pBest->t.n!=nReplace ){ memmove(&zOut[iOff + nReplace], &zOut[iOff + pBest->t.n], nOut - (iOff + pBest->t.n) @@ -1288,11 +1300,12 @@ static int renameEditSql( ** Set all pEList->a[].fg.eEName fields in the expression-list to val. */ static void renameSetENames(ExprList *pEList, int val){ + assert( val==ENAME_NAME || val==ENAME_TAB || val==ENAME_SPAN ); if( pEList ){ int i; for(i=0; inExpr; i++){ assert( val==ENAME_NAME || pEList->a[i].fg.eEName==ENAME_NAME ); - pEList->a[i].fg.eEName = val; + pEList->a[i].fg.eEName = val&0x3; } } } @@ -1549,7 +1562,7 @@ static void renameColumnFunc( if( sParse.pNewTable ){ if( IsView(sParse.pNewTable) ){ Select *pSelect = sParse.pNewTable->u.view.pSelect; - pSelect->selFlags &= ~SF_View; + pSelect->selFlags &= ~(u32)SF_View; sParse.rc = SQLITE_OK; sqlite3SelectPrep(&sParse, pSelect, 0); rc = (db->mallocFailed ? SQLITE_NOMEM : sParse.rc); @@ -1767,7 +1780,7 @@ static void renameTableFunc( sNC.pParse = &sParse; assert( pSelect->selFlags & SF_View ); - pSelect->selFlags &= ~SF_View; + pSelect->selFlags &= ~(u32)SF_View; sqlite3SelectPrep(&sParse, pTab->u.view.pSelect, &sNC); if( sParse.nErr ){ rc = sParse.rc; @@ -1940,7 +1953,7 @@ static void renameQuotefixFunc( if( sParse.pNewTable ){ if( IsView(sParse.pNewTable) ){ Select *pSelect = sParse.pNewTable->u.view.pSelect; - pSelect->selFlags &= ~SF_View; + pSelect->selFlags &= ~(u32)SF_View; sParse.rc = SQLITE_OK; sqlite3SelectPrep(&sParse, pSelect, 0); rc = (db->mallocFailed ? SQLITE_NOMEM : sParse.rc); @@ -2039,10 +2052,10 @@ static void renameTableTest( if( zDb && zInput ){ int rc; Parse sParse; - int flags = db->flags; + u64 flags = db->flags; if( bNoDQS ) db->flags &= ~(SQLITE_DqsDML|SQLITE_DqsDDL); rc = renameParseSql(&sParse, zDb, db, zInput, bTemp); - db->flags |= (flags & (SQLITE_DqsDML|SQLITE_DqsDDL)); + db->flags = flags; if( rc==SQLITE_OK ){ if( isLegacy==0 && sParse.pNewTable && IsView(sParse.pNewTable) ){ NameContext sNC; diff --git a/src/analyze.c b/src/analyze.c index 9213c202b..2721f2523 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -215,7 +215,8 @@ static void openStatTable( sqlite3NestedParse(pParse, "CREATE TABLE %Q.%s(%s)", pDb->zDbSName, zTab, aTable[i].zCols ); - aRoot[i] = (u32)pParse->regRoot; + assert( pParse->isCreate || pParse->nErr ); + aRoot[i] = (u32)pParse->u1.cr.regRoot; aCreateTbl[i] = OPFLAG_P2ISREG; } }else{ @@ -406,7 +407,7 @@ static void statInit( int nCol; /* Number of columns in index being sampled */ int nKeyCol; /* Number of key columns */ int nColUp; /* nCol rounded up for alignment */ - int n; /* Bytes of space to allocate */ + i64 n; /* Bytes of space to allocate */ sqlite3 *db = sqlite3_context_db_handle(context); /* Database connection */ #ifdef SQLITE_ENABLE_STAT4 /* Maximum number of samples. 0 if STAT4 data is not collected */ @@ -442,7 +443,7 @@ static void statInit( p->db = db; p->nEst = sqlite3_value_int64(argv[2]); p->nRow = 0; - p->nLimit = sqlite3_value_int64(argv[3]); + p->nLimit = sqlite3_value_int(argv[3]); p->nCol = nCol; p->nKeyCol = nKeyCol; p->nSkipAhead = 0; @@ -1575,16 +1576,6 @@ static void decodeIntArray( while( z[0]!=0 && z[0]!=' ' ) z++; while( z[0]==' ' ) z++; } - - /* Set the bLowQual flag if the peak number of rows obtained - ** from a full equality match is so large that a full table scan - ** seems likely to be faster than using the index. - */ - if( aLog[0] > 66 /* Index has more than 100 rows */ - && aLog[0] <= aLog[nOut-1] /* And only a single value seen */ - ){ - pIndex->bLowQual = 1; - } } } diff --git a/src/attach.c b/src/attach.c index 399a6cb53..085e1b0ec 100644 --- a/src/attach.c +++ b/src/attach.c @@ -156,7 +156,7 @@ static void attachFunc( if( aNew==0 ) return; memcpy(aNew, db->aDb, sizeof(db->aDb[0])*2); }else{ - aNew = sqlite3DbRealloc(db, db->aDb, sizeof(db->aDb[0])*(db->nDb+1) ); + aNew = sqlite3DbRealloc(db, db->aDb, sizeof(db->aDb[0])*(1+(i64)db->nDb)); if( aNew==0 ) return; } db->aDb = aNew; @@ -227,6 +227,13 @@ static void attachFunc( sqlite3BtreeEnterAll(db); db->init.iDb = 0; db->mDbFlags &= ~(DBFLAG_SchemaKnownOk); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( db->setlkFlags & SQLITE_SETLK_BLOCK_ON_CONNECT ){ + int val = 1; + sqlite3_file *fd = sqlite3PagerFile(sqlite3BtreePager(pNew->pBt)); + sqlite3OsFileControlHint(fd, SQLITE_FCNTL_BLOCK_ON_CONNECT, &val); + } +#endif if( !REOPEN_AS_MEMDB(db) ){ rc = sqlite3Init(db, &zErrDyn); } diff --git a/src/bitvec.c b/src/bitvec.c index 13f87d567..30c4dc7b8 100644 --- a/src/bitvec.c +++ b/src/bitvec.c @@ -67,7 +67,7 @@ ** no fewer collisions than the no-op *1. */ #define BITVEC_HASH(X) (((X)*1)%BITVEC_NINT) -#define BITVEC_NPTR (BITVEC_USIZE/sizeof(Bitvec *)) +#define BITVEC_NPTR ((u32)(BITVEC_USIZE/sizeof(Bitvec *))) /* @@ -250,7 +250,7 @@ void sqlite3BitvecClear(Bitvec *p, u32 i, void *pBuf){ } } if( p->iSize<=BITVEC_NBIT ){ - p->u.aBitmap[i/BITVEC_SZELEM] &= ~(1 << (i&(BITVEC_SZELEM-1))); + p->u.aBitmap[i/BITVEC_SZELEM] &= ~(BITVEC_TELEM)(1<<(i&(BITVEC_SZELEM-1))); }else{ unsigned int j; u32 *aiValues = pBuf; @@ -301,7 +301,7 @@ u32 sqlite3BitvecSize(Bitvec *p){ ** individual bits within V. */ #define SETBIT(V,I) V[I>>3] |= (1<<(I&7)) -#define CLEARBIT(V,I) V[I>>3] &= ~(1<<(I&7)) +#define CLEARBIT(V,I) V[I>>3] &= ~(BITVEC_TELEM)(1<<(I&7)) #define TESTBIT(V,I) (V[I>>3]&(1<<(I&7)))!=0 /* @@ -344,7 +344,7 @@ int sqlite3BitvecBuiltinTest(int sz, int *aOp){ /* Allocate the Bitvec to be tested and a linear array of ** bits to act as the reference */ pBitvec = sqlite3BitvecCreate( sz ); - pV = sqlite3MallocZero( (sz+7)/8 + 1 ); + pV = sqlite3MallocZero( (7+(i64)sz)/8 + 1 ); pTmpSpace = sqlite3_malloc64(BITVEC_SZ); if( pBitvec==0 || pV==0 || pTmpSpace==0 ) goto bitvec_end; diff --git a/src/btmutex.c b/src/btmutex.c index 232831e03..620047c15 100644 --- a/src/btmutex.c +++ b/src/btmutex.c @@ -185,7 +185,7 @@ int sqlite3BtreeHoldsMutex(Btree *p){ */ static void SQLITE_NOINLINE btreeEnterAll(sqlite3 *db){ int i; - int skipOk = 1; + u8 skipOk = 1; Btree *p; assert( sqlite3_mutex_held(db->mutex) ); for(i=0; inDb; i++){ diff --git a/src/btree.c b/src/btree.c index 49eb1d803..25a9b1b4a 100644 --- a/src/btree.c +++ b/src/btree.c @@ -729,7 +729,7 @@ static int saveCursorKey(BtCursor *pCur){ ** below. */ void *pKey; pCur->nKey = sqlite3BtreePayloadSize(pCur); - pKey = sqlite3Malloc( pCur->nKey + 9 + 8 ); + pKey = sqlite3Malloc( ((i64)pCur->nKey) + 9 + 8 ); if( pKey ){ rc = sqlite3BtreePayload(pCur, 0, (int)pCur->nKey, pKey); if( rc==SQLITE_OK ){ @@ -1019,7 +1019,7 @@ void sqlite3BtreeCursorHint(BtCursor *pCur, int eHintType, ...){ */ void sqlite3BtreeCursorHintFlags(BtCursor *pCur, unsigned x){ assert( x==BTREE_SEEK_EQ || x==BTREE_BULKLOAD || x==0 ); - pCur->hints = x; + pCur->hints = (u8)x; } @@ -1213,14 +1213,15 @@ static SQLITE_NOINLINE void btreeParseCellAdjustSizeForOverflow( static int btreePayloadToLocal(MemPage *pPage, i64 nPayload){ int maxLocal; /* Maximum amount of payload held locally */ maxLocal = pPage->maxLocal; + assert( nPayload>=0 ); if( nPayload<=maxLocal ){ - return nPayload; + return (int)nPayload; }else{ int minLocal; /* Minimum amount of payload held locally */ int surplus; /* Overflow payload available for local storage */ minLocal = pPage->minLocal; - surplus = minLocal + (nPayload - minLocal)%(pPage->pBt->usableSize-4); - return ( surplus <= maxLocal ) ? surplus : minLocal; + surplus = (int)(minLocal +(nPayload - minLocal)%(pPage->pBt->usableSize-4)); + return (surplus <= maxLocal) ? surplus : minLocal; } } @@ -1330,11 +1331,13 @@ static void btreeParseCellPtr( pInfo->pPayload = pIter; testcase( nPayload==pPage->maxLocal ); testcase( nPayload==(u32)pPage->maxLocal+1 ); + assert( nPayload>=0 ); + assert( pPage->maxLocal <= BT_MAX_LOCAL ); if( nPayload<=pPage->maxLocal ){ /* This is the (easy) common case where the entire payload fits ** on the local page. No overflow is required. */ - pInfo->nSize = nPayload + (u16)(pIter - pCell); + pInfo->nSize = (u16)nPayload + (u16)(pIter - pCell); if( pInfo->nSize<4 ) pInfo->nSize = 4; pInfo->nLocal = (u16)nPayload; }else{ @@ -1367,11 +1370,13 @@ static void btreeParseCellPtrIndex( pInfo->pPayload = pIter; testcase( nPayload==pPage->maxLocal ); testcase( nPayload==(u32)pPage->maxLocal+1 ); + assert( nPayload>=0 ); + assert( pPage->maxLocal <= BT_MAX_LOCAL ); if( nPayload<=pPage->maxLocal ){ /* This is the (easy) common case where the entire payload fits ** on the local page. No overflow is required. */ - pInfo->nSize = nPayload + (u16)(pIter - pCell); + pInfo->nSize = (u16)nPayload + (u16)(pIter - pCell); if( pInfo->nSize<4 ) pInfo->nSize = 4; pInfo->nLocal = (u16)nPayload; }else{ @@ -1910,14 +1915,14 @@ static SQLITE_INLINE int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ ** at the end of the page. So do additional corruption checks inside this ** routine and return SQLITE_CORRUPT if any problems are found. */ -static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ - u16 iPtr; /* Address of ptr to next freeblock */ - u16 iFreeBlk; /* Address of the next freeblock */ +static int freeSpace(MemPage *pPage, int iStart, int iSize){ + int iPtr; /* Address of ptr to next freeblock */ + int iFreeBlk; /* Address of the next freeblock */ u8 hdr; /* Page header size. 0 or 100 */ - u8 nFrag = 0; /* Reduction in fragmentation */ - u16 iOrigSize = iSize; /* Original value of iSize */ - u16 x; /* Offset to cell content area */ - u32 iEnd = iStart + iSize; /* First byte past the iStart buffer */ + int nFrag = 0; /* Reduction in fragmentation */ + int iOrigSize = iSize; /* Original value of iSize */ + int x; /* Offset to cell content area */ + int iEnd = iStart + iSize; /* First byte past the iStart buffer */ unsigned char *data = pPage->aData; /* Page content */ u8 *pTmp; /* Temporary ptr into data[] */ @@ -1944,7 +1949,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ } iPtr = iFreeBlk; } - if( iFreeBlk>pPage->pBt->usableSize-4 ){ /* TH3: corrupt081.100 */ + if( iFreeBlk>(int)pPage->pBt->usableSize-4 ){ /* TH3: corrupt081.100 */ return SQLITE_CORRUPT_PAGE(pPage); } assert( iFreeBlk>iPtr || iFreeBlk==0 || CORRUPT_DB ); @@ -1959,7 +1964,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ nFrag = iFreeBlk - iEnd; if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_PAGE(pPage); iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]); - if( iEnd > pPage->pBt->usableSize ){ + if( iEnd > (int)pPage->pBt->usableSize ){ return SQLITE_CORRUPT_PAGE(pPage); } iSize = iEnd - iStart; @@ -1980,7 +1985,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ } } if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_PAGE(pPage); - data[hdr+7] -= nFrag; + data[hdr+7] -= (u8)nFrag; } pTmp = &data[hdr+5]; x = get2byte(pTmp); @@ -2001,7 +2006,8 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ /* Insert the new freeblock into the freelist */ put2byte(&data[iPtr], iStart); put2byte(&data[iStart], iFreeBlk); - put2byte(&data[iStart+2], iSize); + assert( iSize>=0 && iSize<=0xffff ); + put2byte(&data[iStart+2], (u16)iSize); } pPage->nFree += iOrigSize; return SQLITE_OK; @@ -2227,7 +2233,7 @@ static int btreeInitPage(MemPage *pPage){ assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); pPage->maskPage = (u16)(pBt->pageSize - 1); pPage->nOverflow = 0; - pPage->cellOffset = pPage->hdrOffset + 8 + pPage->childPtrSize; + pPage->cellOffset = (u16)(pPage->hdrOffset + 8 + pPage->childPtrSize); pPage->aCellIdx = data + pPage->childPtrSize + 8; pPage->aDataEnd = pPage->aData + pBt->pageSize; pPage->aDataOfst = pPage->aData + pPage->childPtrSize; @@ -2261,8 +2267,8 @@ static int btreeInitPage(MemPage *pPage){ static void zeroPage(MemPage *pPage, int flags){ unsigned char *data = pPage->aData; BtShared *pBt = pPage->pBt; - u8 hdr = pPage->hdrOffset; - u16 first; + int hdr = pPage->hdrOffset; + int first; assert( sqlite3PagerPagenumber(pPage->pDbPage)==pPage->pgno || CORRUPT_DB ); assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); @@ -2279,7 +2285,7 @@ static void zeroPage(MemPage *pPage, int flags){ put2byte(&data[hdr+5], pBt->usableSize); pPage->nFree = (u16)(pBt->usableSize - first); decodeFlags(pPage, flags); - pPage->cellOffset = first; + pPage->cellOffset = (u16)first; pPage->aDataEnd = &data[pBt->pageSize]; pPage->aCellIdx = &data[first]; pPage->aDataOfst = &data[pPage->childPtrSize]; @@ -3065,7 +3071,7 @@ int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, int iFix){ BtShared *pBt = p->pBt; assert( nReserve>=0 && nReserve<=255 ); sqlite3BtreeEnter(p); - pBt->nReserveWanted = nReserve; + pBt->nReserveWanted = (u8)nReserve; x = pBt->pageSize - pBt->usableSize; if( nReservebtsFlags & BTS_PAGESIZE_FIXED ){ @@ -3171,7 +3177,7 @@ int sqlite3BtreeSecureDelete(Btree *p, int newFlag){ assert( BTS_FAST_SECURE==(BTS_OVERWRITE|BTS_SECURE_DELETE) ); if( newFlag>=0 ){ p->pBt->btsFlags &= ~BTS_FAST_SECURE; - p->pBt->btsFlags |= BTS_SECURE_DELETE*newFlag; + p->pBt->btsFlags |= (u16)(BTS_SECURE_DELETE*newFlag); } b = (p->pBt->btsFlags & BTS_FAST_SECURE)/BTS_SECURE_DELETE; sqlite3BtreeLeave(p); @@ -3691,6 +3697,13 @@ static SQLITE_NOINLINE int btreeBeginTrans( (void)sqlite3PagerWalWriteLock(pPager, 0); unlockBtreeIfUnused(pBt); } +#if defined(SQLITE_ENABLE_SETLK_TIMEOUT) + if( rc==SQLITE_BUSY_TIMEOUT ){ + /* If a blocking lock timed out, break out of the loop here so that + ** the busy-handler is not invoked. */ + break; + } +#endif }while( (rc&0xFF)==SQLITE_BUSY && pBt->inTransaction==TRANS_NONE && btreeInvokeBusyHandler(pBt) ); sqlite3PagerWalDb(pPager, 0); @@ -6100,7 +6113,7 @@ int sqlite3BtreeIndexMoveto( rc = SQLITE_CORRUPT_PAGE(pPage); goto moveto_index_finish; } - pCellKey = sqlite3Malloc( nCell+nOverrun ); + pCellKey = sqlite3Malloc( (u64)nCell+(u64)nOverrun ); if( pCellKey==0 ){ rc = SQLITE_NOMEM_BKPT; goto moveto_index_finish; @@ -7619,7 +7632,8 @@ static int rebuildPage( } /* The pPg->nFree field is now set incorrectly. The caller will fix it. */ - pPg->nCell = nCell; + assert( nCell < 10922 ); + pPg->nCell = (u16)nCell; pPg->nOverflow = 0; put2byte(&aData[hdr+1], 0); @@ -7866,9 +7880,13 @@ static int editPage( if( pageInsertArray( pPg, pBegin, &pData, pCellptr, iNew+nCell, nNew-nCell, pCArray - ) ) goto editpage_fail; + ) + ){ + goto editpage_fail; + } - pPg->nCell = nNew; + assert( nNew < 10922 ); + pPg->nCell = (u16)nNew; pPg->nOverflow = 0; put2byte(&aData[hdr+3], pPg->nCell); @@ -8177,7 +8195,7 @@ static int balance_nonroot( int pageFlags; /* Value of pPage->aData[0] */ int iSpace1 = 0; /* First unused byte of aSpace1[] */ int iOvflSpace = 0; /* First unused byte of aOvflSpace[] */ - int szScratch; /* Size of scratch memory requested */ + u64 szScratch; /* Size of scratch memory requested */ MemPage *apOld[NB]; /* pPage and up to two siblings */ MemPage *apNew[NB+2]; /* pPage and up to NB siblings after balancing */ u8 *pRight; /* Location in parent of right-sibling pointer */ @@ -9462,7 +9480,7 @@ int sqlite3BtreeInsert( if( pCur->info.nKey==pX->nKey ){ BtreePayload x2; x2.pData = pX->pKey; - x2.nData = pX->nKey; + x2.nData = (int)pX->nKey; assert( pX->nKey<=0x7fffffff ); x2.nZero = 0; return btreeOverwriteCell(pCur, &x2); } @@ -9643,7 +9661,7 @@ int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ getCellInfo(pSrc); if( pSrc->info.nPayload<0x80 ){ - *(aOut++) = pSrc->info.nPayload; + *(aOut++) = (u8)pSrc->info.nPayload; }else{ aOut += sqlite3PutVarint(aOut, pSrc->info.nPayload); } @@ -9656,7 +9674,7 @@ int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ nRem = pSrc->info.nPayload; if( nIn==nRem && nInpPage->maxLocal ){ memcpy(aOut, aIn, nIn); - pBt->nPreformatSize = nIn + (aOut - pBt->pTmpSpace); + pBt->nPreformatSize = nIn + (int)(aOut - pBt->pTmpSpace); return SQLITE_OK; }else{ int rc = SQLITE_OK; @@ -9668,7 +9686,7 @@ int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ u32 nOut; /* Size of output buffer aOut[] */ nOut = btreePayloadToLocal(pDest->pPage, pSrc->info.nPayload); - pBt->nPreformatSize = nOut + (aOut - pBt->pTmpSpace); + pBt->nPreformatSize = (int)nOut + (int)(aOut - pBt->pTmpSpace); if( nOutinfo.nPayload ){ pPgnoOut = &aOut[nOut]; pBt->nPreformatSize += 4; @@ -11289,6 +11307,7 @@ int sqlite3BtreeIsInBackup(Btree *p){ */ void *sqlite3BtreeSchema(Btree *p, int nBytes, void(*xFree)(void *)){ BtShared *pBt = p->pBt; + assert( nBytes==0 || nBytes==sizeof(Schema) ); sqlite3BtreeEnter(p); if( !pBt->pSchema && nBytes ){ pBt->pSchema = sqlite3DbMallocZero(0, nBytes); diff --git a/src/btreeInt.h b/src/btreeInt.h index 121329725..17e3a1add 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -496,6 +496,12 @@ struct CellInfo { */ #define BTCURSOR_MAX_DEPTH 20 +/* +** Maximum amount of storage local to a database page, regardless of +** page size. +*/ +#define BT_MAX_LOCAL 65501 /* 65536 - 35 */ + /* ** A cursor is a pointer to a particular entry within a particular ** b-tree within a database file. diff --git a/src/build.c b/src/build.c index 4e5c7117d..5bd3aac3c 100644 --- a/src/build.c +++ b/src/build.c @@ -68,6 +68,7 @@ static SQLITE_NOINLINE void lockTable( } } + assert( pToplevel->nTableLock < 0x7fff0000 ); nBytes = sizeof(TableLock) * (pToplevel->nTableLock+1); pToplevel->aTableLock = sqlite3DbReallocOrFree(pToplevel->db, pToplevel->aTableLock, nBytes); @@ -168,10 +169,12 @@ void sqlite3FinishCoding(Parse *pParse){ || sqlite3VdbeAssertMayAbort(v, pParse->mayAbort)); if( v ){ if( pParse->bReturning ){ - Returning *pReturning = pParse->u1.pReturning; + Returning *pReturning; int addrRewind; int reg; + assert( !pParse->isCreate ); + pReturning = pParse->u1.d.pReturning; if( pReturning->nRetCol ){ sqlite3VdbeAddOp0(v, OP_FkCheck); addrRewind = @@ -247,7 +250,9 @@ void sqlite3FinishCoding(Parse *pParse){ } if( pParse->bReturning ){ - Returning *pRet = pParse->u1.pReturning; + Returning *pRet; + assert( !pParse->isCreate ); + pRet = pParse->u1.d.pReturning; if( pRet->nRetCol ){ sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRet->iRetCur, pRet->nRetCol); } @@ -1062,10 +1067,16 @@ Index *sqlite3PrimaryKeyIndex(Table *pTab){ ** find the (first) offset of that column in index pIdx. Or return -1 ** if column iCol is not used in index pIdx. */ -i16 sqlite3TableColumnToIndex(Index *pIdx, i16 iCol){ +int sqlite3TableColumnToIndex(Index *pIdx, int iCol){ int i; + i16 iCol16; + assert( iCol>=(-1) && iCol<=SQLITE_MAX_COLUMN ); + assert( pIdx->nColumn<=SQLITE_MAX_COLUMN+1 ); + iCol16 = iCol; for(i=0; inColumn; i++){ - if( iCol==pIdx->aiColumn[i] ) return i; + if( iCol16==pIdx->aiColumn[i] ){ + return i; + } } return -1; } @@ -1319,8 +1330,9 @@ void sqlite3StartTable( /* If the file format and encoding in the database have not been set, ** set them now. */ - reg1 = pParse->regRowid = ++pParse->nMem; - reg2 = pParse->regRoot = ++pParse->nMem; + assert( pParse->isCreate ); + reg1 = pParse->u1.cr.regRowid = ++pParse->nMem; + reg2 = pParse->u1.cr.regRoot = ++pParse->nMem; reg3 = ++pParse->nMem; sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, reg3, BTREE_FILE_FORMAT); sqlite3VdbeUsesBtree(v, iDb); @@ -1335,8 +1347,8 @@ void sqlite3StartTable( ** The record created does not contain anything yet. It will be replaced ** by the real entry in code generated at sqlite3EndTable(). ** - ** The rowid for the new entry is left in register pParse->regRowid. - ** The root page number of the new table is left in reg pParse->regRoot. + ** The rowid for the new entry is left in register pParse->u1.cr.regRowid. + ** The root page of the new table is left in reg pParse->u1.cr.regRoot. ** The rowid and root page number values are needed by the code that ** sqlite3EndTable will generate. */ @@ -1347,7 +1359,7 @@ void sqlite3StartTable( #endif { assert( !pParse->bReturning ); - pParse->u1.addrCrTab = + pParse->u1.cr.addrCrTab = sqlite3VdbeAddOp3(v, OP_CreateBtree, iDb, reg2, BTREE_INTKEY); } sqlite3OpenSchemaTable(pParse, iDb); @@ -1425,7 +1437,8 @@ void sqlite3AddReturning(Parse *pParse, ExprList *pList){ sqlite3ExprListDelete(db, pList); return; } - pParse->u1.pReturning = pRet; + assert( !pParse->isCreate ); + pParse->u1.d.pReturning = pRet; pRet->pParse = pParse; pRet->pReturnEL = pList; sqlite3ParserAddCleanup(pParse, sqlite3DeleteReturning, pRet); @@ -1467,7 +1480,6 @@ void sqlite3AddColumn(Parse *pParse, Token sName, Token sType){ char *zType; Column *pCol; sqlite3 *db = pParse->db; - u8 hName; Column *aNew; u8 eType = COLTYPE_CUSTOM; u8 szEst = 1; @@ -1521,13 +1533,10 @@ void sqlite3AddColumn(Parse *pParse, Token sName, Token sType){ memcpy(z, sName.z, sName.n); z[sName.n] = 0; sqlite3Dequote(z); - hName = sqlite3StrIHash(z); - for(i=0; inCol; i++){ - if( p->aCol[i].hName==hName && sqlite3StrICmp(z, p->aCol[i].zCnName)==0 ){ - sqlite3ErrorMsg(pParse, "duplicate column name: %s", z); - sqlite3DbFree(db, z); - return; - } + if( p->nCol && sqlite3ColumnIndex(p, z)>=0 ){ + sqlite3ErrorMsg(pParse, "duplicate column name: %s", z); + sqlite3DbFree(db, z); + return; } aNew = sqlite3DbRealloc(db,p->aCol,((i64)p->nCol+1)*sizeof(p->aCol[0])); if( aNew==0 ){ @@ -1538,7 +1547,7 @@ void sqlite3AddColumn(Parse *pParse, Token sName, Token sType){ pCol = &p->aCol[p->nCol]; memset(pCol, 0, sizeof(p->aCol[0])); pCol->zCnName = z; - pCol->hName = hName; + pCol->hName = sqlite3StrIHash(z); sqlite3ColumnPropertiesFromName(p, pCol); if( sType.n==0 ){ @@ -1562,9 +1571,14 @@ void sqlite3AddColumn(Parse *pParse, Token sName, Token sType){ pCol->affinity = sqlite3AffinityType(zType, pCol); pCol->colFlags |= COLFLAG_HASTYPE; } + if( p->nCol<=0xff ){ + u8 h = pCol->hName % sizeof(p->aHx); + p->aHx[h] = p->nCol; + } p->nCol++; p->nNVCol++; - pParse->constraintName.n = 0; + assert( pParse->isCreate ); + pParse->u1.cr.constraintName.n = 0; } /* @@ -1828,15 +1842,11 @@ void sqlite3AddPrimaryKey( assert( pCExpr!=0 ); sqlite3StringToId(pCExpr); if( pCExpr->op==TK_ID ){ - const char *zCName; assert( !ExprHasProperty(pCExpr, EP_IntValue) ); - zCName = pCExpr->u.zToken; - for(iCol=0; iColnCol; iCol++){ - if( sqlite3StrICmp(zCName, pTab->aCol[iCol].zCnName)==0 ){ - pCol = &pTab->aCol[iCol]; - makeColumnPartOfPrimaryKey(pParse, pCol); - break; - } + iCol = sqlite3ColumnIndex(pTab, pCExpr->u.zToken); + if( iCol>=0 ){ + pCol = &pTab->aCol[iCol]; + makeColumnPartOfPrimaryKey(pParse, pCol); } } } @@ -1888,8 +1898,10 @@ void sqlite3AddCheckConstraint( && !sqlite3BtreeIsReadonly(db->aDb[db->init.iDb].pBt) ){ pTab->pCheck = sqlite3ExprListAppend(pParse, pTab->pCheck, pCheckExpr); - if( pParse->constraintName.n ){ - sqlite3ExprListSetName(pParse, pTab->pCheck, &pParse->constraintName, 1); + assert( pParse->isCreate ); + if( pParse->u1.cr.constraintName.n ){ + sqlite3ExprListSetName(pParse, pTab->pCheck, + &pParse->u1.cr.constraintName, 1); }else{ Token t; for(zStart++; sqlite3Isspace(zStart[0]); zStart++){} @@ -2084,7 +2096,8 @@ static void identPut(char *z, int *pIdx, char *zSignedIdent){ ** from sqliteMalloc() and must be freed by the calling function. */ static char *createTableStmt(sqlite3 *db, Table *p){ - int i, k, n; + int i, k, len; + i64 n; char *zStmt; char *zSep, *zSep2, *zEnd; Column *pCol; @@ -2108,8 +2121,9 @@ static char *createTableStmt(sqlite3 *db, Table *p){ sqlite3OomFault(db); return 0; } - sqlite3_snprintf(n, zStmt, "CREATE TABLE "); - k = sqlite3Strlen30(zStmt); + assert( n>14 && n<=0x7fffffff ); + memcpy(zStmt, "CREATE TABLE ", 13); + k = 13; identPut(zStmt, &k, p->zName); zStmt[k++] = '('; for(pCol=p->aCol, i=0; inCol; i++, pCol++){ @@ -2121,13 +2135,15 @@ static char *createTableStmt(sqlite3 *db, Table *p){ /* SQLITE_AFF_REAL */ " REAL", /* SQLITE_AFF_FLEXNUM */ " NUM", }; - int len; const char *zType; - sqlite3_snprintf(n-k, &zStmt[k], zSep); - k += sqlite3Strlen30(&zStmt[k]); + len = sqlite3Strlen30(zSep); + assert( k+lenzCnName); + assert( kaffinity-SQLITE_AFF_BLOB >= 0 ); assert( pCol->affinity-SQLITE_AFF_BLOB < ArraySize(azType) ); testcase( pCol->affinity==SQLITE_AFF_BLOB ); @@ -2142,11 +2158,14 @@ static char *createTableStmt(sqlite3 *db, Table *p){ assert( pCol->affinity==SQLITE_AFF_BLOB || pCol->affinity==SQLITE_AFF_FLEXNUM || pCol->affinity==sqlite3AffinityType(zType, 0) ); + assert( k+lennColumn>=N ) return SQLITE_OK; + db = pParse->db; + assert( N>0 ); + assert( N <= SQLITE_MAX_COLUMN*2 /* tag-20250221-1 */ ); + testcase( N==2*pParse->db->aLimit[SQLITE_LIMIT_COLUMN] ); assert( pIdx->isResized==0 ); - nByte = (sizeof(char*) + sizeof(LogEst) + sizeof(i16) + 1)*N; + nByte = (sizeof(char*) + sizeof(LogEst) + sizeof(i16) + 1)*(u64)N; zExtra = sqlite3DbMallocZero(db, nByte); if( zExtra==0 ) return SQLITE_NOMEM_BKPT; memcpy(zExtra, pIdx->azColl, sizeof(char*)*pIdx->nColumn); @@ -2173,7 +2197,7 @@ static int resizeIndexObject(sqlite3 *db, Index *pIdx, int N){ zExtra += sizeof(i16)*N; memcpy(zExtra, pIdx->aSortOrder, pIdx->nColumn); pIdx->aSortOrder = (u8*)zExtra; - pIdx->nColumn = N; + pIdx->nColumn = (u16)N; /* See tag-20250221-1 above for proof of safety */ pIdx->isResized = 1; return SQLITE_OK; } @@ -2339,9 +2363,9 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ ** into BTREE_BLOBKEY. */ assert( !pParse->bReturning ); - if( pParse->u1.addrCrTab ){ + if( pParse->u1.cr.addrCrTab ){ assert( v ); - sqlite3VdbeChangeP3(v, pParse->u1.addrCrTab, BTREE_BLOBKEY); + sqlite3VdbeChangeP3(v, pParse->u1.cr.addrCrTab, BTREE_BLOBKEY); } /* Locate the PRIMARY KEY index. Or, if this table was originally @@ -2427,14 +2451,14 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ pIdx->nColumn = pIdx->nKeyCol; continue; } - if( resizeIndexObject(db, pIdx, pIdx->nKeyCol+n) ) return; + if( resizeIndexObject(pParse, pIdx, pIdx->nKeyCol+n) ) return; for(i=0, j=pIdx->nKeyCol; inKeyCol, pPk, i) ){ testcase( hasColumn(pIdx->aiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ); pIdx->aiColumn[j] = pPk->aiColumn[i]; pIdx->azColl[j] = pPk->azColl[i]; if( pPk->aSortOrder[i] ){ - /* See ticket https://www.sqlite.org/src/info/bba7b69f9849b5bf */ + /* See ticket https://sqlite.org/src/info/bba7b69f9849b5bf */ pIdx->bAscKeyBug = 1; } j++; @@ -2451,7 +2475,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ if( !hasColumn(pPk->aiColumn, nPk, i) && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ) nExtra++; } - if( resizeIndexObject(db, pPk, nPk+nExtra) ) return; + if( resizeIndexObject(pParse, pPk, nPk+nExtra) ) return; for(i=0, j=nPk; inCol; i++){ if( !hasColumn(pPk->aiColumn, j, i) && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 @@ -2781,7 +2805,7 @@ void sqlite3EndTable( /* If this is a CREATE TABLE xx AS SELECT ..., execute the SELECT ** statement to populate the new table. The root-page number for the - ** new table is in register pParse->regRoot. + ** new table is in register pParse->u1.cr.regRoot. ** ** Once the SELECT has been coded by sqlite3Select(), it is in a ** suitable state to query for the column names and types to be used @@ -2812,7 +2836,8 @@ void sqlite3EndTable( regRec = ++pParse->nMem; regRowid = ++pParse->nMem; sqlite3MayAbort(pParse); - sqlite3VdbeAddOp3(v, OP_OpenWrite, iCsr, pParse->regRoot, iDb); + assert( pParse->isCreate ); + sqlite3VdbeAddOp3(v, OP_OpenWrite, iCsr, pParse->u1.cr.regRoot, iDb); sqlite3VdbeChangeP5(v, OPFLAG_P2ISREG); addrTop = sqlite3VdbeCurrentAddr(v) + 1; sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); @@ -2857,6 +2882,7 @@ void sqlite3EndTable( ** schema table. We just need to update that slot with all ** the information we've collected. */ + assert( pParse->isCreate ); sqlite3NestedParse(pParse, "UPDATE %Q." LEGACY_SCHEMA_TABLE " SET type='%s', name=%Q, tbl_name=%Q, rootpage=#%d, sql=%Q" @@ -2865,9 +2891,9 @@ void sqlite3EndTable( zType, p->zName, p->zName, - pParse->regRoot, + pParse->u1.cr.regRoot, zStmt, - pParse->regRowid + pParse->u1.cr.regRowid ); sqlite3DbFree(db, zStmt); sqlite3ChangeCookie(pParse, iDb); @@ -3607,7 +3633,7 @@ void sqlite3CreateForeignKey( }else{ nCol = pFromCol->nExpr; } - nByte = sizeof(*pFKey) + (nCol-1)*sizeof(pFKey->aCol[0]) + pTo->n + 1; + nByte = SZ_FKEY(nCol) + pTo->n + 1; if( pToCol ){ for(i=0; inExpr; i++){ nByte += sqlite3Strlen30(pToCol->a[i].zEName) + 1; @@ -3809,7 +3835,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ ** not work for UNIQUE constraint indexes on WITHOUT ROWID tables ** with DESC primary keys, since those indexes have there keys in ** a different order from the main table. - ** See ticket: https://www.sqlite.org/src/info/bba7b69f9849b5bf + ** See ticket: https://sqlite.org/src/info/bba7b69f9849b5bf */ sqlite3VdbeAddOp1(v, OP_SeekEnd, iIdx); } @@ -3833,13 +3859,14 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ */ Index *sqlite3AllocateIndexObject( sqlite3 *db, /* Database connection */ - i16 nCol, /* Total number of columns in the index */ + int nCol, /* Total number of columns in the index */ int nExtra, /* Number of bytes of extra space to alloc */ char **ppExtra /* Pointer to the "extra" space */ ){ Index *p; /* Allocated index object */ - int nByte; /* Bytes of space for Index object + arrays */ + i64 nByte; /* Bytes of space for Index object + arrays */ + assert( nCol <= 2*db->aLimit[SQLITE_LIMIT_COLUMN] ); nByte = ROUND8(sizeof(Index)) + /* Index structure */ ROUND8(sizeof(char*)*nCol) + /* Index.azColl */ ROUND8(sizeof(LogEst)*(nCol+1) + /* Index.aiRowLogEst */ @@ -3852,8 +3879,9 @@ Index *sqlite3AllocateIndexObject( p->aiRowLogEst = (LogEst*)pExtra; pExtra += sizeof(LogEst)*(nCol+1); p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; p->aSortOrder = (u8*)pExtra; - p->nColumn = nCol; - p->nKeyCol = nCol - 1; + assert( nCol>0 ); + p->nColumn = (u16)nCol; + p->nKeyCol = (u16)(nCol - 1); *ppExtra = ((char*)p) + nByte; } return p; @@ -4665,12 +4693,11 @@ IdList *sqlite3IdListAppend(Parse *pParse, IdList *pList, Token *pToken){ sqlite3 *db = pParse->db; int i; if( pList==0 ){ - pList = sqlite3DbMallocZero(db, sizeof(IdList) ); + pList = sqlite3DbMallocZero(db, SZ_IDLIST(1)); if( pList==0 ) return 0; }else{ IdList *pNew; - pNew = sqlite3DbRealloc(db, pList, - sizeof(IdList) + pList->nId*sizeof(pList->a)); + pNew = sqlite3DbRealloc(db, pList, SZ_IDLIST(pList->nId+1)); if( pNew==0 ){ sqlite3IdListDelete(db, pList); return 0; @@ -4769,8 +4796,7 @@ SrcList *sqlite3SrcListEnlarge( return 0; } if( nAlloc>SQLITE_MAX_SRCLIST ) nAlloc = SQLITE_MAX_SRCLIST; - pNew = sqlite3DbRealloc(db, pSrc, - sizeof(*pSrc) + (nAlloc-1)*sizeof(pSrc->a[0]) ); + pNew = sqlite3DbRealloc(db, pSrc, SZ_SRCLIST(nAlloc)); if( pNew==0 ){ assert( db->mallocFailed ); return 0; @@ -4845,7 +4871,7 @@ SrcList *sqlite3SrcListAppend( assert( pParse->db!=0 ); db = pParse->db; if( pList==0 ){ - pList = sqlite3DbMallocRawNN(pParse->db, sizeof(SrcList) ); + pList = sqlite3DbMallocRawNN(pParse->db, SZ_SRCLIST(1)); if( pList==0 ) return 0; pList->nAlloc = 1; pList->nSrc = 1; @@ -5731,10 +5757,9 @@ With *sqlite3WithAdd( } if( pWith ){ - sqlite3_int64 nByte = sizeof(*pWith) + (sizeof(pWith->a[1]) * pWith->nCte); - pNew = sqlite3DbRealloc(db, pWith, nByte); + pNew = sqlite3DbRealloc(db, pWith, SZ_WITH(pWith->nCte+1)); }else{ - pNew = sqlite3DbMallocZero(db, sizeof(*pWith)); + pNew = sqlite3DbMallocZero(db, SZ_WITH(1)); } assert( (pNew!=0 && zName!=0) || db->mallocFailed ); diff --git a/src/ctime.c b/src/ctime.c deleted file mode 100644 index 9f358bd27..000000000 --- a/src/ctime.c +++ /dev/null @@ -1,796 +0,0 @@ -/* DO NOT EDIT! -** This file is automatically generated by the script in the canonical -** SQLite source tree at tool/mkctimec.tcl. -** -** To modify this header, edit any of the various lists in that script -** which specify categories of generated conditionals in this file. -*/ - -/* -** 2010 February 23 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file implements routines used to report what compile-time options -** SQLite was built with. -*/ -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS /* IMP: R-16824-07538 */ - -/* -** Include the configuration header output by 'configure' if we're using the -** autoconf-based build -*/ -#if defined(_HAVE_SQLITE_CONFIG_H) && !defined(SQLITECONFIG_H) -#include "sqlite_cfg.h" -#define SQLITECONFIG_H 1 -#endif - -/* These macros are provided to "stringify" the value of the define -** for those options in which the value is meaningful. */ -#define CTIMEOPT_VAL_(opt) #opt -#define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt) - -/* Like CTIMEOPT_VAL, but especially for SQLITE_DEFAULT_LOOKASIDE. This -** option requires a separate macro because legal values contain a single -** comma. e.g. (-DSQLITE_DEFAULT_LOOKASIDE="100,100") */ -#define CTIMEOPT_VAL2_(opt1,opt2) #opt1 "," #opt2 -#define CTIMEOPT_VAL2(opt) CTIMEOPT_VAL2_(opt) -#include "sqliteInt.h" - -/* -** An array of names of all compile-time options. This array should -** be sorted A-Z. -** -** This array looks large, but in a typical installation actually uses -** only a handful of compile-time options, so most times this array is usually -** rather short and uses little memory space. -*/ -static const char * const sqlite3azCompileOpt[] = { - -#ifdef SQLITE_32BIT_ROWID - "32BIT_ROWID", -#endif -#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC - "4_BYTE_ALIGNED_MALLOC", -#endif -#ifdef SQLITE_ALLOW_COVERING_INDEX_SCAN -# if SQLITE_ALLOW_COVERING_INDEX_SCAN != 1 - "ALLOW_COVERING_INDEX_SCAN=" CTIMEOPT_VAL(SQLITE_ALLOW_COVERING_INDEX_SCAN), -# endif -#endif -#ifdef SQLITE_ALLOW_ROWID_IN_VIEW - "ALLOW_ROWID_IN_VIEW", -#endif -#ifdef SQLITE_ALLOW_URI_AUTHORITY - "ALLOW_URI_AUTHORITY", -#endif -#ifdef SQLITE_ATOMIC_INTRINSICS - "ATOMIC_INTRINSICS=" CTIMEOPT_VAL(SQLITE_ATOMIC_INTRINSICS), -#endif -#ifdef SQLITE_BITMASK_TYPE - "BITMASK_TYPE=" CTIMEOPT_VAL(SQLITE_BITMASK_TYPE), -#endif -#ifdef SQLITE_BUG_COMPATIBLE_20160819 - "BUG_COMPATIBLE_20160819", -#endif -#ifdef SQLITE_CASE_SENSITIVE_LIKE - "CASE_SENSITIVE_LIKE", -#endif -#ifdef SQLITE_CHECK_PAGES - "CHECK_PAGES", -#endif -#if defined(__clang__) && defined(__clang_major__) - "COMPILER=clang-" CTIMEOPT_VAL(__clang_major__) "." - CTIMEOPT_VAL(__clang_minor__) "." - CTIMEOPT_VAL(__clang_patchlevel__), -#elif defined(_MSC_VER) - "COMPILER=msvc-" CTIMEOPT_VAL(_MSC_VER), -#elif defined(__GNUC__) && defined(__VERSION__) - "COMPILER=gcc-" __VERSION__, -#endif -#ifdef SQLITE_COVERAGE_TEST - "COVERAGE_TEST", -#endif -#ifdef SQLITE_DEBUG - "DEBUG", -#endif -#ifdef SQLITE_DEFAULT_AUTOMATIC_INDEX - "DEFAULT_AUTOMATIC_INDEX", -#endif -#ifdef SQLITE_DEFAULT_AUTOVACUUM - "DEFAULT_AUTOVACUUM", -#endif -#ifdef SQLITE_DEFAULT_CACHE_SIZE - "DEFAULT_CACHE_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_CACHE_SIZE), -#endif -#ifdef SQLITE_DEFAULT_CKPTFULLFSYNC - "DEFAULT_CKPTFULLFSYNC", -#endif -#ifdef SQLITE_DEFAULT_FILE_FORMAT - "DEFAULT_FILE_FORMAT=" CTIMEOPT_VAL(SQLITE_DEFAULT_FILE_FORMAT), -#endif -#ifdef SQLITE_DEFAULT_FILE_PERMISSIONS - "DEFAULT_FILE_PERMISSIONS=" CTIMEOPT_VAL(SQLITE_DEFAULT_FILE_PERMISSIONS), -#endif -#ifdef SQLITE_DEFAULT_FOREIGN_KEYS - "DEFAULT_FOREIGN_KEYS", -#endif -#ifdef SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT - "DEFAULT_JOURNAL_SIZE_LIMIT=" CTIMEOPT_VAL(SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT), -#endif -#ifdef SQLITE_DEFAULT_LOCKING_MODE - "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), -#endif -#ifdef SQLITE_DEFAULT_LOOKASIDE - "DEFAULT_LOOKASIDE=" CTIMEOPT_VAL2(SQLITE_DEFAULT_LOOKASIDE), -#endif -#ifdef SQLITE_DEFAULT_MEMSTATUS -# if SQLITE_DEFAULT_MEMSTATUS != 1 - "DEFAULT_MEMSTATUS=" CTIMEOPT_VAL(SQLITE_DEFAULT_MEMSTATUS), -# endif -#endif -#ifdef SQLITE_DEFAULT_MMAP_SIZE - "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE), -#endif -#ifdef SQLITE_DEFAULT_PAGE_SIZE - "DEFAULT_PAGE_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_PAGE_SIZE), -#endif -#ifdef SQLITE_DEFAULT_PCACHE_INITSZ - "DEFAULT_PCACHE_INITSZ=" CTIMEOPT_VAL(SQLITE_DEFAULT_PCACHE_INITSZ), -#endif -#ifdef SQLITE_DEFAULT_PROXYDIR_PERMISSIONS - "DEFAULT_PROXYDIR_PERMISSIONS=" CTIMEOPT_VAL(SQLITE_DEFAULT_PROXYDIR_PERMISSIONS), -#endif -#ifdef SQLITE_DEFAULT_RECURSIVE_TRIGGERS - "DEFAULT_RECURSIVE_TRIGGERS", -#endif -#ifdef SQLITE_DEFAULT_ROWEST - "DEFAULT_ROWEST=" CTIMEOPT_VAL(SQLITE_DEFAULT_ROWEST), -#endif -#ifdef SQLITE_DEFAULT_SECTOR_SIZE - "DEFAULT_SECTOR_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_SECTOR_SIZE), -#endif -#ifdef SQLITE_DEFAULT_SYNCHRONOUS - "DEFAULT_SYNCHRONOUS=" CTIMEOPT_VAL(SQLITE_DEFAULT_SYNCHRONOUS), -#endif -#ifdef SQLITE_DEFAULT_WAL_AUTOCHECKPOINT - "DEFAULT_WAL_AUTOCHECKPOINT=" CTIMEOPT_VAL(SQLITE_DEFAULT_WAL_AUTOCHECKPOINT), -#endif -#ifdef SQLITE_DEFAULT_WAL_SYNCHRONOUS - "DEFAULT_WAL_SYNCHRONOUS=" CTIMEOPT_VAL(SQLITE_DEFAULT_WAL_SYNCHRONOUS), -#endif -#ifdef SQLITE_DEFAULT_WORKER_THREADS - "DEFAULT_WORKER_THREADS=" CTIMEOPT_VAL(SQLITE_DEFAULT_WORKER_THREADS), -#endif -#ifdef SQLITE_DIRECT_OVERFLOW_READ - "DIRECT_OVERFLOW_READ", -#endif -#ifdef SQLITE_DISABLE_DIRSYNC - "DISABLE_DIRSYNC", -#endif -#ifdef SQLITE_DISABLE_FTS3_UNICODE - "DISABLE_FTS3_UNICODE", -#endif -#ifdef SQLITE_DISABLE_FTS4_DEFERRED - "DISABLE_FTS4_DEFERRED", -#endif -#ifdef SQLITE_DISABLE_INTRINSIC - "DISABLE_INTRINSIC", -#endif -#ifdef SQLITE_DISABLE_LFS - "DISABLE_LFS", -#endif -#ifdef SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS - "DISABLE_PAGECACHE_OVERFLOW_STATS", -#endif -#ifdef SQLITE_DISABLE_SKIPAHEAD_DISTINCT - "DISABLE_SKIPAHEAD_DISTINCT", -#endif -#ifdef SQLITE_DQS - "DQS=" CTIMEOPT_VAL(SQLITE_DQS), -#endif -#ifdef SQLITE_ENABLE_8_3_NAMES - "ENABLE_8_3_NAMES=" CTIMEOPT_VAL(SQLITE_ENABLE_8_3_NAMES), -#endif -#ifdef SQLITE_ENABLE_API_ARMOR - "ENABLE_API_ARMOR", -#endif -#ifdef SQLITE_ENABLE_ATOMIC_WRITE - "ENABLE_ATOMIC_WRITE", -#endif -#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE - "ENABLE_BATCH_ATOMIC_WRITE", -#endif -#ifdef SQLITE_ENABLE_BYTECODE_VTAB - "ENABLE_BYTECODE_VTAB", -#endif -#ifdef SQLITE_ENABLE_CEROD - "ENABLE_CEROD=" CTIMEOPT_VAL(SQLITE_ENABLE_CEROD), -#endif -#ifdef SQLITE_ENABLE_COLUMN_METADATA - "ENABLE_COLUMN_METADATA", -#endif -#ifdef SQLITE_ENABLE_COLUMN_USED_MASK - "ENABLE_COLUMN_USED_MASK", -#endif -#ifdef SQLITE_ENABLE_COSTMULT - "ENABLE_COSTMULT", -#endif -#ifdef SQLITE_ENABLE_CURSOR_HINTS - "ENABLE_CURSOR_HINTS", -#endif -#ifdef SQLITE_ENABLE_DBPAGE_VTAB - "ENABLE_DBPAGE_VTAB", -#endif -#ifdef SQLITE_ENABLE_DBSTAT_VTAB - "ENABLE_DBSTAT_VTAB", -#endif -#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT - "ENABLE_EXPENSIVE_ASSERT", -#endif -#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS - "ENABLE_EXPLAIN_COMMENTS", -#endif -#ifdef SQLITE_ENABLE_FTS3 - "ENABLE_FTS3", -#endif -#ifdef SQLITE_ENABLE_FTS3_PARENTHESIS - "ENABLE_FTS3_PARENTHESIS", -#endif -#ifdef SQLITE_ENABLE_FTS3_TOKENIZER - "ENABLE_FTS3_TOKENIZER", -#endif -#ifdef SQLITE_ENABLE_FTS4 - "ENABLE_FTS4", -#endif -#ifdef SQLITE_ENABLE_FTS5 - "ENABLE_FTS5", -#endif -#ifdef SQLITE_ENABLE_GEOPOLY - "ENABLE_GEOPOLY", -#endif -#ifdef SQLITE_ENABLE_HIDDEN_COLUMNS - "ENABLE_HIDDEN_COLUMNS", -#endif -#ifdef SQLITE_ENABLE_ICU - "ENABLE_ICU", -#endif -#ifdef SQLITE_ENABLE_IOTRACE - "ENABLE_IOTRACE", -#endif -#ifdef SQLITE_ENABLE_LOAD_EXTENSION - "ENABLE_LOAD_EXTENSION", -#endif -#ifdef SQLITE_ENABLE_LOCKING_STYLE - "ENABLE_LOCKING_STYLE=" CTIMEOPT_VAL(SQLITE_ENABLE_LOCKING_STYLE), -#endif -#ifdef SQLITE_ENABLE_MATH_FUNCTIONS - "ENABLE_MATH_FUNCTIONS", -#endif -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - "ENABLE_MEMORY_MANAGEMENT", -#endif -#ifdef SQLITE_ENABLE_MEMSYS3 - "ENABLE_MEMSYS3", -#endif -#ifdef SQLITE_ENABLE_MEMSYS5 - "ENABLE_MEMSYS5", -#endif -#ifdef SQLITE_ENABLE_MULTIPLEX - "ENABLE_MULTIPLEX", -#endif -#ifdef SQLITE_ENABLE_NORMALIZE - "ENABLE_NORMALIZE", -#endif -#ifdef SQLITE_ENABLE_NULL_TRIM - "ENABLE_NULL_TRIM", -#endif -#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC - "ENABLE_OFFSET_SQL_FUNC", -#endif -#ifdef SQLITE_ENABLE_ORDERED_SET_AGGREGATES - "ENABLE_ORDERED_SET_AGGREGATES", -#endif -#ifdef SQLITE_ENABLE_OVERSIZE_CELL_CHECK - "ENABLE_OVERSIZE_CELL_CHECK", -#endif -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK - "ENABLE_PREUPDATE_HOOK", -#endif -#ifdef SQLITE_ENABLE_QPSG - "ENABLE_QPSG", -#endif -#ifdef SQLITE_ENABLE_RBU - "ENABLE_RBU", -#endif -#ifdef SQLITE_ENABLE_RTREE - "ENABLE_RTREE", -#endif -#ifdef SQLITE_ENABLE_SESSION - "ENABLE_SESSION", -#endif -#ifdef SQLITE_ENABLE_SNAPSHOT - "ENABLE_SNAPSHOT", -#endif -#ifdef SQLITE_ENABLE_SORTER_REFERENCES - "ENABLE_SORTER_REFERENCES", -#endif -#ifdef SQLITE_ENABLE_SQLLOG - "ENABLE_SQLLOG", -#endif -#ifdef SQLITE_ENABLE_STAT4 - "ENABLE_STAT4", -#endif -#ifdef SQLITE_ENABLE_STMTVTAB - "ENABLE_STMTVTAB", -#endif -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - "ENABLE_STMT_SCANSTATUS", -#endif -#ifdef SQLITE_ENABLE_TREETRACE - "ENABLE_TREETRACE", -#endif -#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION - "ENABLE_UNKNOWN_SQL_FUNCTION", -#endif -#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY - "ENABLE_UNLOCK_NOTIFY", -#endif -#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT - "ENABLE_UPDATE_DELETE_LIMIT", -#endif -#ifdef SQLITE_ENABLE_URI_00_ERROR - "ENABLE_URI_00_ERROR", -#endif -#ifdef SQLITE_ENABLE_VFSTRACE - "ENABLE_VFSTRACE", -#endif -#ifdef SQLITE_ENABLE_WHERETRACE - "ENABLE_WHERETRACE", -#endif -#ifdef SQLITE_ENABLE_ZIPVFS - "ENABLE_ZIPVFS", -#endif -#ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS - "EXPLAIN_ESTIMATED_ROWS", -#endif -#ifdef SQLITE_EXTRA_AUTOEXT - "EXTRA_AUTOEXT=" CTIMEOPT_VAL(SQLITE_EXTRA_AUTOEXT), -#endif -#ifdef SQLITE_EXTRA_IFNULLROW - "EXTRA_IFNULLROW", -#endif -#ifdef SQLITE_EXTRA_INIT - "EXTRA_INIT=" CTIMEOPT_VAL(SQLITE_EXTRA_INIT), -#endif -#ifdef SQLITE_EXTRA_SHUTDOWN - "EXTRA_SHUTDOWN=" CTIMEOPT_VAL(SQLITE_EXTRA_SHUTDOWN), -#endif -#ifdef SQLITE_FTS3_MAX_EXPR_DEPTH - "FTS3_MAX_EXPR_DEPTH=" CTIMEOPT_VAL(SQLITE_FTS3_MAX_EXPR_DEPTH), -#endif -#ifdef SQLITE_FTS5_ENABLE_TEST_MI - "FTS5_ENABLE_TEST_MI", -#endif -#ifdef SQLITE_FTS5_NO_WITHOUT_ROWID - "FTS5_NO_WITHOUT_ROWID", -#endif -#if HAVE_ISNAN || SQLITE_HAVE_ISNAN - "HAVE_ISNAN", -#endif -#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX -# if SQLITE_HOMEGROWN_RECURSIVE_MUTEX != 1 - "HOMEGROWN_RECURSIVE_MUTEX=" CTIMEOPT_VAL(SQLITE_HOMEGROWN_RECURSIVE_MUTEX), -# endif -#endif -#ifdef SQLITE_IGNORE_AFP_LOCK_ERRORS - "IGNORE_AFP_LOCK_ERRORS", -#endif -#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS - "IGNORE_FLOCK_LOCK_ERRORS", -#endif -#ifdef SQLITE_INLINE_MEMCPY - "INLINE_MEMCPY", -#endif -#ifdef SQLITE_INT64_TYPE - "INT64_TYPE", -#endif -#ifdef SQLITE_INTEGRITY_CHECK_ERROR_MAX - "INTEGRITY_CHECK_ERROR_MAX=" CTIMEOPT_VAL(SQLITE_INTEGRITY_CHECK_ERROR_MAX), -#endif -#ifdef SQLITE_LEGACY_JSON_VALID - "LEGACY_JSON_VALID", -#endif -#ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS - "LIKE_DOESNT_MATCH_BLOBS", -#endif -#ifdef SQLITE_LOCK_TRACE - "LOCK_TRACE", -#endif -#ifdef SQLITE_LOG_CACHE_SPILL - "LOG_CACHE_SPILL", -#endif -#ifdef SQLITE_MALLOC_SOFT_LIMIT - "MALLOC_SOFT_LIMIT=" CTIMEOPT_VAL(SQLITE_MALLOC_SOFT_LIMIT), -#endif -#ifdef SQLITE_MAX_ATTACHED - "MAX_ATTACHED=" CTIMEOPT_VAL(SQLITE_MAX_ATTACHED), -#endif -#ifdef SQLITE_MAX_COLUMN - "MAX_COLUMN=" CTIMEOPT_VAL(SQLITE_MAX_COLUMN), -#endif -#ifdef SQLITE_MAX_COMPOUND_SELECT - "MAX_COMPOUND_SELECT=" CTIMEOPT_VAL(SQLITE_MAX_COMPOUND_SELECT), -#endif -#ifdef SQLITE_MAX_DEFAULT_PAGE_SIZE - "MAX_DEFAULT_PAGE_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_DEFAULT_PAGE_SIZE), -#endif -#ifdef SQLITE_MAX_EXPR_DEPTH - "MAX_EXPR_DEPTH=" CTIMEOPT_VAL(SQLITE_MAX_EXPR_DEPTH), -#endif -#ifdef SQLITE_MAX_FUNCTION_ARG - "MAX_FUNCTION_ARG=" CTIMEOPT_VAL(SQLITE_MAX_FUNCTION_ARG), -#endif -#ifdef SQLITE_MAX_LENGTH - "MAX_LENGTH=" CTIMEOPT_VAL(SQLITE_MAX_LENGTH), -#endif -#ifdef SQLITE_MAX_LIKE_PATTERN_LENGTH - "MAX_LIKE_PATTERN_LENGTH=" CTIMEOPT_VAL(SQLITE_MAX_LIKE_PATTERN_LENGTH), -#endif -#ifdef SQLITE_MAX_MEMORY - "MAX_MEMORY=" CTIMEOPT_VAL(SQLITE_MAX_MEMORY), -#endif -#ifdef SQLITE_MAX_MMAP_SIZE - "MAX_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_MMAP_SIZE), -#endif -#ifdef SQLITE_MAX_MMAP_SIZE_ - "MAX_MMAP_SIZE_=" CTIMEOPT_VAL(SQLITE_MAX_MMAP_SIZE_), -#endif -#ifdef SQLITE_MAX_PAGE_COUNT - "MAX_PAGE_COUNT=" CTIMEOPT_VAL(SQLITE_MAX_PAGE_COUNT), -#endif -#ifdef SQLITE_MAX_PAGE_SIZE - "MAX_PAGE_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_PAGE_SIZE), -#endif -#ifdef SQLITE_MAX_SCHEMA_RETRY - "MAX_SCHEMA_RETRY=" CTIMEOPT_VAL(SQLITE_MAX_SCHEMA_RETRY), -#endif -#ifdef SQLITE_MAX_SQL_LENGTH - "MAX_SQL_LENGTH=" CTIMEOPT_VAL(SQLITE_MAX_SQL_LENGTH), -#endif -#ifdef SQLITE_MAX_TRIGGER_DEPTH - "MAX_TRIGGER_DEPTH=" CTIMEOPT_VAL(SQLITE_MAX_TRIGGER_DEPTH), -#endif -#ifdef SQLITE_MAX_VARIABLE_NUMBER - "MAX_VARIABLE_NUMBER=" CTIMEOPT_VAL(SQLITE_MAX_VARIABLE_NUMBER), -#endif -#ifdef SQLITE_MAX_VDBE_OP - "MAX_VDBE_OP=" CTIMEOPT_VAL(SQLITE_MAX_VDBE_OP), -#endif -#ifdef SQLITE_MAX_WORKER_THREADS - "MAX_WORKER_THREADS=" CTIMEOPT_VAL(SQLITE_MAX_WORKER_THREADS), -#endif -#ifdef SQLITE_MEMDEBUG - "MEMDEBUG", -#endif -#ifdef SQLITE_MIXED_ENDIAN_64BIT_FLOAT - "MIXED_ENDIAN_64BIT_FLOAT", -#endif -#ifdef SQLITE_MMAP_READWRITE - "MMAP_READWRITE", -#endif -#ifdef SQLITE_MUTEX_NOOP - "MUTEX_NOOP", -#endif -#ifdef SQLITE_MUTEX_OMIT - "MUTEX_OMIT", -#endif -#ifdef SQLITE_MUTEX_PTHREADS - "MUTEX_PTHREADS", -#endif -#ifdef SQLITE_MUTEX_W32 - "MUTEX_W32", -#endif -#ifdef SQLITE_NEED_ERR_NAME - "NEED_ERR_NAME", -#endif -#ifdef SQLITE_NO_SYNC - "NO_SYNC", -#endif -#ifdef SQLITE_OMIT_ALTERTABLE - "OMIT_ALTERTABLE", -#endif -#ifdef SQLITE_OMIT_ANALYZE - "OMIT_ANALYZE", -#endif -#ifdef SQLITE_OMIT_ATTACH - "OMIT_ATTACH", -#endif -#ifdef SQLITE_OMIT_AUTHORIZATION - "OMIT_AUTHORIZATION", -#endif -#ifdef SQLITE_OMIT_AUTOINCREMENT - "OMIT_AUTOINCREMENT", -#endif -#ifdef SQLITE_OMIT_AUTOINIT - "OMIT_AUTOINIT", -#endif -#ifdef SQLITE_OMIT_AUTOMATIC_INDEX - "OMIT_AUTOMATIC_INDEX", -#endif -#ifdef SQLITE_OMIT_AUTORESET - "OMIT_AUTORESET", -#endif -#ifdef SQLITE_OMIT_AUTOVACUUM - "OMIT_AUTOVACUUM", -#endif -#ifdef SQLITE_OMIT_BETWEEN_OPTIMIZATION - "OMIT_BETWEEN_OPTIMIZATION", -#endif -#ifdef SQLITE_OMIT_BLOB_LITERAL - "OMIT_BLOB_LITERAL", -#endif -#ifdef SQLITE_OMIT_CAST - "OMIT_CAST", -#endif -#ifdef SQLITE_OMIT_CHECK - "OMIT_CHECK", -#endif -#ifdef SQLITE_OMIT_COMPLETE - "OMIT_COMPLETE", -#endif -#ifdef SQLITE_OMIT_COMPOUND_SELECT - "OMIT_COMPOUND_SELECT", -#endif -#ifdef SQLITE_OMIT_CONFLICT_CLAUSE - "OMIT_CONFLICT_CLAUSE", -#endif -#ifdef SQLITE_OMIT_CTE - "OMIT_CTE", -#endif -#if defined(SQLITE_OMIT_DATETIME_FUNCS) || defined(SQLITE_OMIT_FLOATING_POINT) - "OMIT_DATETIME_FUNCS", -#endif -#ifdef SQLITE_OMIT_DECLTYPE - "OMIT_DECLTYPE", -#endif -#ifdef SQLITE_OMIT_DEPRECATED - "OMIT_DEPRECATED", -#endif -#ifdef SQLITE_OMIT_DESERIALIZE - "OMIT_DESERIALIZE", -#endif -#ifdef SQLITE_OMIT_DISKIO - "OMIT_DISKIO", -#endif -#ifdef SQLITE_OMIT_EXPLAIN - "OMIT_EXPLAIN", -#endif -#ifdef SQLITE_OMIT_FLAG_PRAGMAS - "OMIT_FLAG_PRAGMAS", -#endif -#ifdef SQLITE_OMIT_FLOATING_POINT - "OMIT_FLOATING_POINT", -#endif -#ifdef SQLITE_OMIT_FOREIGN_KEY - "OMIT_FOREIGN_KEY", -#endif -#ifdef SQLITE_OMIT_GET_TABLE - "OMIT_GET_TABLE", -#endif -#ifdef SQLITE_OMIT_HEX_INTEGER - "OMIT_HEX_INTEGER", -#endif -#ifdef SQLITE_OMIT_INCRBLOB - "OMIT_INCRBLOB", -#endif -#ifdef SQLITE_OMIT_INTEGRITY_CHECK - "OMIT_INTEGRITY_CHECK", -#endif -#ifdef SQLITE_OMIT_INTROSPECTION_PRAGMAS - "OMIT_INTROSPECTION_PRAGMAS", -#endif -#ifdef SQLITE_OMIT_JSON - "OMIT_JSON", -#endif -#ifdef SQLITE_OMIT_LIKE_OPTIMIZATION - "OMIT_LIKE_OPTIMIZATION", -#endif -#ifdef SQLITE_OMIT_LOAD_EXTENSION - "OMIT_LOAD_EXTENSION", -#endif -#ifdef SQLITE_OMIT_LOCALTIME - "OMIT_LOCALTIME", -#endif -#ifdef SQLITE_OMIT_LOOKASIDE - "OMIT_LOOKASIDE", -#endif -#ifdef SQLITE_OMIT_MEMORYDB - "OMIT_MEMORYDB", -#endif -#ifdef SQLITE_OMIT_OR_OPTIMIZATION - "OMIT_OR_OPTIMIZATION", -#endif -#ifdef SQLITE_OMIT_PAGER_PRAGMAS - "OMIT_PAGER_PRAGMAS", -#endif -#ifdef SQLITE_OMIT_PARSER_TRACE - "OMIT_PARSER_TRACE", -#endif -#ifdef SQLITE_OMIT_POPEN - "OMIT_POPEN", -#endif -#ifdef SQLITE_OMIT_PRAGMA - "OMIT_PRAGMA", -#endif -#ifdef SQLITE_OMIT_PROGRESS_CALLBACK - "OMIT_PROGRESS_CALLBACK", -#endif -#ifdef SQLITE_OMIT_QUICKBALANCE - "OMIT_QUICKBALANCE", -#endif -#ifdef SQLITE_OMIT_REINDEX - "OMIT_REINDEX", -#endif -#ifdef SQLITE_OMIT_SCHEMA_PRAGMAS - "OMIT_SCHEMA_PRAGMAS", -#endif -#ifdef SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS - "OMIT_SCHEMA_VERSION_PRAGMAS", -#endif -#ifdef SQLITE_OMIT_SEH - "OMIT_SEH", -#endif -#ifdef SQLITE_OMIT_SHARED_CACHE - "OMIT_SHARED_CACHE", -#endif -#ifdef SQLITE_OMIT_SHUTDOWN_DIRECTORIES - "OMIT_SHUTDOWN_DIRECTORIES", -#endif -#ifdef SQLITE_OMIT_SUBQUERY - "OMIT_SUBQUERY", -#endif -#ifdef SQLITE_OMIT_TCL_VARIABLE - "OMIT_TCL_VARIABLE", -#endif -#ifdef SQLITE_OMIT_TEMPDB - "OMIT_TEMPDB", -#endif -#ifdef SQLITE_OMIT_TEST_CONTROL - "OMIT_TEST_CONTROL", -#endif -#ifdef SQLITE_OMIT_TRACE -# if SQLITE_OMIT_TRACE != 1 - "OMIT_TRACE=" CTIMEOPT_VAL(SQLITE_OMIT_TRACE), -# endif -#endif -#ifdef SQLITE_OMIT_TRIGGER - "OMIT_TRIGGER", -#endif -#ifdef SQLITE_OMIT_TRUNCATE_OPTIMIZATION - "OMIT_TRUNCATE_OPTIMIZATION", -#endif -#ifdef SQLITE_OMIT_UTF16 - "OMIT_UTF16", -#endif -#ifdef SQLITE_OMIT_VACUUM - "OMIT_VACUUM", -#endif -#ifdef SQLITE_OMIT_VIEW - "OMIT_VIEW", -#endif -#ifdef SQLITE_OMIT_VIRTUALTABLE - "OMIT_VIRTUALTABLE", -#endif -#ifdef SQLITE_OMIT_WAL - "OMIT_WAL", -#endif -#ifdef SQLITE_OMIT_WSD - "OMIT_WSD", -#endif -#ifdef SQLITE_OMIT_XFER_OPT - "OMIT_XFER_OPT", -#endif -#ifdef SQLITE_PERFORMANCE_TRACE - "PERFORMANCE_TRACE", -#endif -#ifdef SQLITE_POWERSAFE_OVERWRITE -# if SQLITE_POWERSAFE_OVERWRITE != 1 - "POWERSAFE_OVERWRITE=" CTIMEOPT_VAL(SQLITE_POWERSAFE_OVERWRITE), -# endif -#endif -#ifdef SQLITE_PREFER_PROXY_LOCKING - "PREFER_PROXY_LOCKING", -#endif -#ifdef SQLITE_PROXY_DEBUG - "PROXY_DEBUG", -#endif -#ifdef SQLITE_REVERSE_UNORDERED_SELECTS - "REVERSE_UNORDERED_SELECTS", -#endif -#ifdef SQLITE_RTREE_INT_ONLY - "RTREE_INT_ONLY", -#endif -#ifdef SQLITE_SECURE_DELETE - "SECURE_DELETE", -#endif -#ifdef SQLITE_SMALL_STACK - "SMALL_STACK", -#endif -#ifdef SQLITE_SORTER_PMASZ - "SORTER_PMASZ=" CTIMEOPT_VAL(SQLITE_SORTER_PMASZ), -#endif -#ifdef SQLITE_SOUNDEX - "SOUNDEX", -#endif -#ifdef SQLITE_STAT4_SAMPLES - "STAT4_SAMPLES=" CTIMEOPT_VAL(SQLITE_STAT4_SAMPLES), -#endif -#ifdef SQLITE_STMTJRNL_SPILL - "STMTJRNL_SPILL=" CTIMEOPT_VAL(SQLITE_STMTJRNL_SPILL), -#endif -#ifdef SQLITE_SUBSTR_COMPATIBILITY - "SUBSTR_COMPATIBILITY", -#endif -#if (!defined(SQLITE_WIN32_MALLOC) \ - && !defined(SQLITE_ZERO_MALLOC) \ - && !defined(SQLITE_MEMDEBUG) \ - ) || defined(SQLITE_SYSTEM_MALLOC) - "SYSTEM_MALLOC", -#endif -#ifdef SQLITE_TCL - "TCL", -#endif -#ifdef SQLITE_TEMP_STORE - "TEMP_STORE=" CTIMEOPT_VAL(SQLITE_TEMP_STORE), -#endif -#ifdef SQLITE_TEST - "TEST", -#endif -#if defined(SQLITE_THREADSAFE) - "THREADSAFE=" CTIMEOPT_VAL(SQLITE_THREADSAFE), -#elif defined(THREADSAFE) - "THREADSAFE=" CTIMEOPT_VAL(THREADSAFE), -#else - "THREADSAFE=1", -#endif -#ifdef SQLITE_UNLINK_AFTER_CLOSE - "UNLINK_AFTER_CLOSE", -#endif -#ifdef SQLITE_UNTESTABLE - "UNTESTABLE", -#endif -#ifdef SQLITE_USE_ALLOCA - "USE_ALLOCA", -#endif -#ifdef SQLITE_USE_FCNTL_TRACE - "USE_FCNTL_TRACE", -#endif -#ifdef SQLITE_USE_URI - "USE_URI", -#endif -#ifdef SQLITE_VDBE_COVERAGE - "VDBE_COVERAGE", -#endif -#ifdef SQLITE_WIN32_MALLOC - "WIN32_MALLOC", -#endif -#ifdef SQLITE_ZERO_MALLOC - "ZERO_MALLOC", -#endif - -} ; - -const char **sqlite3CompileOptions(int *pnOpt){ - *pnOpt = sizeof(sqlite3azCompileOpt) / sizeof(sqlite3azCompileOpt[0]); - return (const char**)sqlite3azCompileOpt; -} - -#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ diff --git a/src/date.c b/src/date.c index de2736637..1b4f10fb4 100644 --- a/src/date.c +++ b/src/date.c @@ -1357,7 +1357,7 @@ static int daysAfterMonday(DateTime *pDate){ ** In other words, return the day of the week according ** to this code: ** -** 0=Sunday, 1=Monday, 2=Tues, ..., 6=Saturday +** 0=Sunday, 1=Monday, 2=Tuesday, ..., 6=Saturday */ static int daysAfterSunday(DateTime *pDate){ assert( pDate->validJD ); diff --git a/src/dbpage.c b/src/dbpage.c index eb5ab33fe..f9fdcc5a3 100644 --- a/src/dbpage.c +++ b/src/dbpage.c @@ -395,8 +395,8 @@ static int dbpageUpdate( /* "INSERT INTO dbpage($PGNO,NULL)" causes page number $PGNO and ** all subsequent pages to be deleted. */ pTab->iDbTrunc = iDb; - pgno--; - pTab->pgnoTrunc = pgno; + pTab->pgnoTrunc = pgno-1; + pgno = 1; }else{ zErr = "bad page value"; goto update_fail; diff --git a/src/expr.c b/src/expr.c index 3d0e6db97..606a4cd7e 100644 --- a/src/expr.c +++ b/src/expr.c @@ -73,7 +73,9 @@ char sqlite3ExprAffinity(const Expr *pExpr){ pExpr->pLeft->x.pSelect->pEList->a[pExpr->iColumn].pExpr ); } - if( op==TK_VECTOR ){ + if( op==TK_VECTOR + || (op==TK_FUNCTION && pExpr->affExpr==SQLITE_AFF_DEFER) + ){ assert( ExprUseXList(pExpr) ); return sqlite3ExprAffinity(pExpr->x.pList->a[0].pExpr); } @@ -266,7 +268,9 @@ CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ p = p->pLeft; continue; } - if( op==TK_VECTOR ){ + if( op==TK_VECTOR + || (op==TK_FUNCTION && p->affExpr==SQLITE_AFF_DEFER) + ){ assert( ExprUseXList(p) ); p = p->x.pList->a[0].pExpr; continue; @@ -1738,7 +1742,7 @@ static Expr *exprDup( With *sqlite3WithDup(sqlite3 *db, With *p){ With *pRet = 0; if( p ){ - sqlite3_int64 nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1); + sqlite3_int64 nByte = SZ_WITH(p->nCte); pRet = sqlite3DbMallocZero(db, nByte); if( pRet ){ int i; @@ -1849,7 +1853,6 @@ ExprList *sqlite3ExprListDup(sqlite3 *db, const ExprList *p, int flags){ } pItem->zEName = sqlite3DbStrDup(db, pOldItem->zEName); pItem->fg = pOldItem->fg; - pItem->fg.done = 0; pItem->u = pOldItem->u; } return pNew; @@ -1866,11 +1869,9 @@ ExprList *sqlite3ExprListDup(sqlite3 *db, const ExprList *p, int flags){ SrcList *sqlite3SrcListDup(sqlite3 *db, const SrcList *p, int flags){ SrcList *pNew; int i; - int nByte; assert( db!=0 ); if( p==0 ) return 0; - nByte = sizeof(*p) + (p->nSrc>0 ? sizeof(p->a[0]) * (p->nSrc-1) : 0); - pNew = sqlite3DbMallocRawNN(db, nByte ); + pNew = sqlite3DbMallocRawNN(db, SZ_SRCLIST(p->nSrc) ); if( pNew==0 ) return 0; pNew->nSrc = pNew->nAlloc = p->nSrc; for(i=0; inSrc; i++){ @@ -1932,7 +1933,7 @@ IdList *sqlite3IdListDup(sqlite3 *db, const IdList *p){ int i; assert( db!=0 ); if( p==0 ) return 0; - pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew)+(p->nId-1)*sizeof(p->a[0]) ); + pNew = sqlite3DbMallocRawNN(db, SZ_IDLIST(p->nId)); if( pNew==0 ) return 0; pNew->nId = p->nId; for(i=0; inId; i++){ @@ -1964,7 +1965,7 @@ Select *sqlite3SelectDup(sqlite3 *db, const Select *pDup, int flags){ pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); pNew->iLimit = 0; pNew->iOffset = 0; - pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; + pNew->selFlags = p->selFlags & ~(u32)SF_UsesEphemeral; pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = p->nSelectRow; @@ -2016,7 +2017,7 @@ SQLITE_NOINLINE ExprList *sqlite3ExprListAppendNew( struct ExprList_item *pItem; ExprList *pList; - pList = sqlite3DbMallocRawNN(db, sizeof(ExprList)+sizeof(pList->a[0])*4 ); + pList = sqlite3DbMallocRawNN(db, SZ_EXPRLIST(4)); if( pList==0 ){ sqlite3ExprDelete(db, pExpr); return 0; @@ -2036,8 +2037,7 @@ SQLITE_NOINLINE ExprList *sqlite3ExprListAppendGrow( struct ExprList_item *pItem; ExprList *pNew; pList->nAlloc *= 2; - pNew = sqlite3DbRealloc(db, pList, - sizeof(*pList)+(pList->nAlloc-1)*sizeof(pList->a[0])); + pNew = sqlite3DbRealloc(db, pList, SZ_EXPRLIST(pList->nAlloc)); if( pNew==0 ){ sqlite3ExprListDelete(db, pList); sqlite3ExprDelete(db, pExpr); @@ -2966,13 +2966,7 @@ const char *sqlite3RowidAlias(Table *pTab){ int ii; assert( VisibleRowid(pTab) ); for(ii=0; iinCol; iCol++){ - if( sqlite3_stricmp(azOpt[ii], pTab->aCol[iCol].zCnName)==0 ) break; - } - if( iCol==pTab->nCol ){ - return azOpt[ii]; - } + if( sqlite3ColumnIndex(pTab, azOpt[ii])<0 ) return azOpt[ii]; } return 0; } @@ -3376,7 +3370,7 @@ static char *exprINAffinity(Parse *pParse, const Expr *pExpr){ char *zRet; assert( pExpr->op==TK_IN ); - zRet = sqlite3DbMallocRaw(pParse->db, nVal+1); + zRet = sqlite3DbMallocRaw(pParse->db, 1+(i64)nVal); if( zRet ){ int i; for(i=0; idb, pCopy); sqlite3DbFree(pParse->db, dest.zAffSdst); if( addrBloom ){ + /* Remember that location of the Bloom filter in the P3 operand + ** of the OP_Once that began this subroutine. tag-202407032019 */ sqlite3VdbeGetOp(v, addrOnce)->p3 = dest.iSDParm2; if( dest.iSDParm2==0 ){ - sqlite3VdbeChangeToNoop(v, addrBloom); - }else{ - sqlite3VdbeGetOp(v, addrOnce)->p3 = dest.iSDParm2; + /* If the Bloom filter won't actually be used, keep it small */ + sqlite3VdbeGetOp(v, addrBloom)->p1 = 10; } } if( rc ){ @@ -4087,7 +4082,7 @@ static void sqlite3ExprCodeIN( if( ExprHasProperty(pExpr, EP_Subrtn) ){ const VdbeOp *pOp = sqlite3VdbeGetOp(v, pExpr->y.sub.iAddr); assert( pOp->opcode==OP_Once || pParse->nErr ); - if( pOp->opcode==OP_Once && pOp->p3>0 ){ + if( pOp->opcode==OP_Once && pOp->p3>0 ){ /* tag-202407032019 */ assert( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ); sqlite3VdbeAddOp4Int(v, OP_Filter, pOp->p3, destIfFalse, rLhs, nVector); VdbeCoverage(v); @@ -4679,7 +4674,7 @@ static SQLITE_NOINLINE int sqlite3IndexedExprLookup( /* -** Expresion pExpr is guaranteed to be a TK_COLUMN or equivalent. This +** Expression pExpr is guaranteed to be a TK_COLUMN or equivalent. This ** function checks the Parse.pIdxPartExpr list to see if this column ** can be replaced with a constant value. If so, it generates code to ** put the constant value in a register (ideally, but not necessarily, diff --git a/src/func.c b/src/func.c index 5e5b715f2..9e2839336 100644 --- a/src/func.c +++ b/src/func.c @@ -356,11 +356,6 @@ static void substrFunc( i64 p1, p2; assert( argc==3 || argc==2 ); - if( sqlite3_value_type(argv[1])==SQLITE_NULL - || (argc==3 && sqlite3_value_type(argv[2])==SQLITE_NULL) - ){ - return; - } p0type = sqlite3_value_type(argv[0]); p1 = sqlite3_value_int64(argv[1]); if( p0type==SQLITE_BLOB ){ @@ -378,19 +373,23 @@ static void substrFunc( } } } -#ifdef SQLITE_SUBSTR_COMPATIBILITY - /* If SUBSTR_COMPATIBILITY is defined then substr(X,0,N) work the same as - ** as substr(X,1,N) - it returns the first N characters of X. This - ** is essentially a back-out of the bug-fix in check-in [5fc125d362df4b8] - ** from 2009-02-02 for compatibility of applications that exploited the - ** old buggy behavior. */ - if( p1==0 ) p1 = 1; /* */ -#endif if( argc==3 ){ p2 = sqlite3_value_int64(argv[2]); + if( p2==0 && sqlite3_value_type(argv[2])==SQLITE_NULL ) return; }else{ p2 = sqlite3_context_db_handle(context)->aLimit[SQLITE_LIMIT_LENGTH]; } + if( p1==0 ){ +#ifdef SQLITE_SUBSTR_COMPATIBILITY + /* If SUBSTR_COMPATIBILITY is defined then substr(X,0,N) work the same as + ** as substr(X,1,N) - it returns the first N characters of X. This + ** is essentially a back-out of the bug-fix in check-in [5fc125d362df4b8] + ** from 2009-02-02 for compatibility of applications that exploited the + ** old buggy behavior. */ + p1 = 1; /* */ +#endif + if( sqlite3_value_type(argv[1])==SQLITE_NULL ) return; + } if( p1<0 ){ p1 += len; if( p1<0 ){ @@ -1091,7 +1090,7 @@ static const char hexdigits[] = { ** Append to pStr text that is the SQL literal representation of the ** value contained in pValue. */ -void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue){ +void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue, int bEscape){ /* As currently implemented, the string must be initially empty. ** we might relax this requirement in the future, but that will ** require enhancements to the implementation. */ @@ -1139,7 +1138,7 @@ void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue){ } case SQLITE_TEXT: { const unsigned char *zArg = sqlite3_value_text(pValue); - sqlite3_str_appendf(pStr, "%Q", zArg); + sqlite3_str_appendf(pStr, bEscape ? "%#Q" : "%Q", zArg); break; } default: { @@ -1150,6 +1149,105 @@ void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue){ } } +/* +** Return true if z[] begins with N hexadecimal digits, and write +** a decoding of those digits into *pVal. Or return false if any +** one of the first N characters in z[] is not a hexadecimal digit. +*/ +static int isNHex(const char *z, int N, u32 *pVal){ + int i; + int v = 0; + for(i=0; i0 ){ + memmove(&zOut[j], &zIn[i], n); + j += n; + i += n; + } + if( zIn[i+1]=='\\' ){ + i += 2; + zOut[j++] = '\\'; + }else if( sqlite3Isxdigit(zIn[i+1]) ){ + if( !isNHex(&zIn[i+1], 4, &v) ) goto unistr_error; + i += 5; + j += sqlite3AppendOneUtf8Character(&zOut[j], v); + }else if( zIn[i+1]=='+' ){ + if( !isNHex(&zIn[i+2], 6, &v) ) goto unistr_error; + i += 8; + j += sqlite3AppendOneUtf8Character(&zOut[j], v); + }else if( zIn[i+1]=='u' ){ + if( !isNHex(&zIn[i+2], 4, &v) ) goto unistr_error; + i += 6; + j += sqlite3AppendOneUtf8Character(&zOut[j], v); + }else if( zIn[i+1]=='U' ){ + if( !isNHex(&zIn[i+2], 8, &v) ) goto unistr_error; + i += 10; + j += sqlite3AppendOneUtf8Character(&zOut[j], v); + }else{ + goto unistr_error; + } + } + zOut[j] = 0; + sqlite3_result_text64(context, zOut, j, sqlite3_free, SQLITE_UTF8); + return; + +unistr_error: + sqlite3_free(zOut); + sqlite3_result_error(context, "invalid Unicode escape", -1); + return; +} + + /* ** Implementation of the QUOTE() function. ** @@ -1159,6 +1257,10 @@ void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue){ ** as needed. BLOBs are encoded as hexadecimal literals. Strings with ** embedded NUL characters cannot be represented as string literals in SQL ** and hence the returned string literal is truncated prior to the first NUL. +** +** If sqlite3_user_data() is non-zero, then the UNISTR_QUOTE() function is +** implemented instead. The difference is that UNISTR_QUOTE() uses the +** UNISTR() function to escape control characters. */ static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ sqlite3_str str; @@ -1166,7 +1268,7 @@ static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ assert( argc==1 ); UNUSED_PARAMETER(argc); sqlite3StrAccumInit(&str, db, 0, 0, db->aLimit[SQLITE_LIMIT_LENGTH]); - sqlite3QuoteValue(&str,argv[0]); + sqlite3QuoteValue(&str,argv[0],SQLITE_PTR_TO_INT(sqlite3_user_data(context))); sqlite3_result_text(context, sqlite3StrAccumFinish(&str), str.nChar, SQLITE_DYNAMIC); if( str.accError!=SQLITE_OK ){ @@ -1421,7 +1523,7 @@ static void replaceFunc( assert( zRep==sqlite3_value_text(argv[2]) ); nOut = nStr + 1; assert( nOuthtsize = new_size = sqlite3MallocSize(new_ht)/sizeof(struct _ht); memset(new_ht, 0, new_size*sizeof(struct _ht)); for(elem=pH->first, pH->first=0; elem; elem = next_elem){ - unsigned int h = strHash(elem->pKey) % new_size; next_elem = elem->next; - insertElement(pH, &new_ht[h], elem); + insertElement(pH, &new_ht[elem->h % new_size], elem); } return 1; } @@ -152,23 +158,22 @@ static HashElem *findElementWithHash( HashElem *elem; /* Used to loop thru the element list */ unsigned int count; /* Number of elements left to test */ unsigned int h; /* The computed hash */ - static HashElem nullElement = { 0, 0, 0, 0 }; + static HashElem nullElement = { 0, 0, 0, 0, 0 }; + h = strHash(pKey); if( pH->ht ){ /*OPTIMIZATION-IF-TRUE*/ struct _ht *pEntry; - h = strHash(pKey) % pH->htsize; - pEntry = &pH->ht[h]; + pEntry = &pH->ht[h % pH->htsize]; elem = pEntry->chain; count = pEntry->count; }else{ - h = 0; elem = pH->first; count = pH->count; } if( pHash ) *pHash = h; while( count ){ assert( elem!=0 ); - if( sqlite3StrICmp(elem->pKey,pKey)==0 ){ + if( h==elem->h && sqlite3StrICmp(elem->pKey,pKey)==0 ){ return elem; } elem = elem->next; @@ -180,10 +185,9 @@ static HashElem *findElementWithHash( /* Remove a single entry from the hash table given a pointer to that ** element and a hash on the element's key. */ -static void removeElementGivenHash( +static void removeElement( Hash *pH, /* The pH containing "elem" */ - HashElem* elem, /* The element to be removed from the pH */ - unsigned int h /* Hash value for the element */ + HashElem *elem /* The element to be removed from the pH */ ){ struct _ht *pEntry; if( elem->prev ){ @@ -195,7 +199,7 @@ static void removeElementGivenHash( elem->next->prev = elem->prev; } if( pH->ht ){ - pEntry = &pH->ht[h]; + pEntry = &pH->ht[elem->h % pH->htsize]; if( pEntry->chain==elem ){ pEntry->chain = elem->next; } @@ -246,7 +250,7 @@ void *sqlite3HashInsert(Hash *pH, const char *pKey, void *data){ if( elem->data ){ void *old_data = elem->data; if( data==0 ){ - removeElementGivenHash(pH,elem,h); + removeElement(pH,elem); }else{ elem->data = data; elem->pKey = pKey; @@ -257,14 +261,12 @@ void *sqlite3HashInsert(Hash *pH, const char *pKey, void *data){ new_elem = (HashElem*)sqlite3Malloc( sizeof(HashElem) ); if( new_elem==0 ) return data; new_elem->pKey = pKey; + new_elem->h = h; new_elem->data = data; pH->count++; - if( pH->count>=10 && pH->count > 2*pH->htsize ){ - if( rehash(pH, pH->count*2) ){ - assert( pH->htsize>0 ); - h = strHash(pKey) % pH->htsize; - } + if( pH->count>=5 && pH->count > 2*pH->htsize ){ + rehash(pH, pH->count*3); } - insertElement(pH, pH->ht ? &pH->ht[h] : 0, new_elem); + insertElement(pH, pH->ht ? &pH->ht[new_elem->h % pH->htsize] : 0, new_elem); return 0; } diff --git a/src/hash.h b/src/hash.h index 3f491e45c..cff65d6e5 100644 --- a/src/hash.h +++ b/src/hash.h @@ -60,6 +60,7 @@ struct HashElem { HashElem *next, *prev; /* Next and previous elements in the table */ void *data; /* Data associated with this element */ const char *pKey; /* Key associated with this element */ + unsigned int h; /* hash for pKey */ }; /* diff --git a/src/insert.c b/src/insert.c index 83baeece6..fdd9c8da2 100644 --- a/src/insert.c +++ b/src/insert.c @@ -693,7 +693,7 @@ Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow){ f = (f & pLeft->selFlags); } pSelect = sqlite3SelectNew(pParse, pRow, 0, 0, 0, 0, 0, f, 0); - pLeft->selFlags &= ~SF_MultiValue; + pLeft->selFlags &= ~(u32)SF_MultiValue; if( pSelect ){ pSelect->op = TK_ALL; pSelect->pPrior = pLeft; @@ -1075,28 +1075,22 @@ void sqlite3Insert( aTabColMap = sqlite3DbMallocZero(db, pTab->nCol*sizeof(int)); if( aTabColMap==0 ) goto insert_cleanup; for(i=0; inId; i++){ - const char *zCName = pColumn->a[i].zName; - u8 hName = sqlite3StrIHash(zCName); - for(j=0; jnCol; j++){ - if( pTab->aCol[j].hName!=hName ) continue; - if( sqlite3StrICmp(zCName, pTab->aCol[j].zCnName)==0 ){ - if( aTabColMap[j]==0 ) aTabColMap[j] = i+1; - if( i!=j ) bIdListInOrder = 0; - if( j==pTab->iPKey ){ - ipkColumn = i; assert( !withoutRowid ); - } + j = sqlite3ColumnIndex(pTab, pColumn->a[i].zName); + if( j>=0 ){ + if( aTabColMap[j]==0 ) aTabColMap[j] = i+1; + if( i!=j ) bIdListInOrder = 0; + if( j==pTab->iPKey ){ + ipkColumn = i; assert( !withoutRowid ); + } #ifndef SQLITE_OMIT_GENERATED_COLUMNS - if( pTab->aCol[j].colFlags & (COLFLAG_STORED|COLFLAG_VIRTUAL) ){ - sqlite3ErrorMsg(pParse, - "cannot INSERT into generated column \"%s\"", - pTab->aCol[j].zCnName); - goto insert_cleanup; - } -#endif - break; + if( pTab->aCol[j].colFlags & (COLFLAG_STORED|COLFLAG_VIRTUAL) ){ + sqlite3ErrorMsg(pParse, + "cannot INSERT into generated column \"%s\"", + pTab->aCol[j].zCnName); + goto insert_cleanup; } - } - if( j>=pTab->nCol ){ +#endif + }else{ if( sqlite3IsRowid(pColumn->a[i].zName) && !withoutRowid ){ ipkColumn = i; bIdListInOrder = 0; @@ -1394,7 +1388,7 @@ void sqlite3Insert( continue; }else if( pColumn==0 ){ /* Hidden columns that are not explicitly named in the INSERT - ** get there default value */ + ** get their default value */ sqlite3ExprCodeFactorable(pParse, sqlite3ColumnExpr(pTab, &pTab->aCol[i]), iRegStore); @@ -2119,7 +2113,7 @@ void sqlite3GenerateConstraintChecks( ** could happen in any order, but they are grouped up front for ** convenience. ** - ** 2018-08-14: Ticket https://www.sqlite.org/src/info/908f001483982c43 + ** 2018-08-14: Ticket https://sqlite.org/src/info/908f001483982c43 ** The order of constraints used to have OE_Update as (2) and OE_Abort ** and so forth as (1). But apparently PostgreSQL checks the OE_Update ** constraint before any others, so it had to be moved. diff --git a/src/json.c b/src/json.c index a0a075e66..3078be34b 100644 --- a/src/json.c +++ b/src/json.c @@ -23,8 +23,8 @@ ** Beginning with version 3.45.0 (circa 2024-01-01), these routines also ** accept BLOB values that have JSON encoded using a binary representation ** called "JSONB". The name JSONB comes from PostgreSQL, however the on-disk -** format SQLite JSONB is completely different and incompatible with -** PostgreSQL JSONB. +** format for SQLite-JSONB is completely different and incompatible with +** PostgreSQL-JSONB. ** ** Decoding and interpreting JSONB is still O(N) where N is the size of ** the input, the same as text JSON. However, the constant of proportionality @@ -81,7 +81,7 @@ ** ** The payload size need not be expressed in its minimal form. For example, ** if the payload size is 10, the size can be expressed in any of 5 different -** ways: (1) (X>>4)==10, (2) (X>>4)==12 following by on 0x0a byte, +** ways: (1) (X>>4)==10, (2) (X>>4)==12 following by one 0x0a byte, ** (3) (X>>4)==13 followed by 0x00 and 0x0a, (4) (X>>4)==14 followed by ** 0x00 0x00 0x00 0x0a, or (5) (X>>4)==15 followed by 7 bytes of 0x00 and ** a single byte of 0x0a. The shorter forms are preferred, of course, but @@ -91,7 +91,7 @@ ** the size when it becomes known, resulting in a non-minimal encoding. ** ** The value (X>>4)==15 is not actually used in the current implementation -** (as SQLite is currently unable handle BLOBs larger than about 2GB) +** (as SQLite is currently unable to handle BLOBs larger than about 2GB) ** but is included in the design to allow for future enhancements. ** ** The payload follows the header. NULL, TRUE, and FALSE have no payload and @@ -151,23 +151,47 @@ static const char * const jsonbType[] = { ** increase for the text-JSON parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). */ static const char jsonIsSpace[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +#ifdef SQLITE_ASCII +/*0 1 2 3 4 5 6 7 8 9 a b c d e f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, /* 0 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1 */ + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 6 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7 */ + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 8 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 9 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* a */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* b */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* c */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* d */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* e */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* f */ +#endif +#ifdef SQLITE_EBCDIC +/*0 1 2 3 4 5 6 7 8 9 a b c d e f */ + 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, /* 0 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1 */ + 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3 */ + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 6 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7 */ + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 8 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 9 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* a */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* b */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* c */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* d */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* e */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* f */ +#endif + }; #define jsonIsspace(x) (jsonIsSpace[(unsigned char)x]) @@ -175,7 +199,13 @@ static const char jsonIsSpace[] = { ** The set of all space characters recognized by jsonIsspace(). ** Useful as the second argument to strspn(). */ +#ifdef SQLITE_ASCII static const char jsonSpaces[] = "\011\012\015\040"; +#endif +#ifdef SQLITE_EBCDIC +static const char jsonSpaces[] = "\005\045\015\100"; +#endif + /* ** Characters that are special to JSON. Control characters, @@ -184,23 +214,46 @@ static const char jsonSpaces[] = "\011\012\015\040"; ** it in the set of special characters. */ static const char jsonIsOk[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +#ifdef SQLITE_ASCII +/*0 1 2 3 4 5 6 7 8 9 a b c d e f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1 */ + 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, /* 2 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 3 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, /* 5 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 7 */ + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 8 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 9 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* a */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* b */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* c */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* d */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* e */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 /* f */ +#endif +#ifdef SQLITE_EBCDIC +/*0 1 2 3 4 5 6 7 8 9 a b c d e f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1 */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2 */ + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, /* 3 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 5 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, /* 7 */ + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 8 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 9 */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* a */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* b */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* c */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* d */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* e */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 /* f */ +#endif }; /* Objects */ @@ -345,7 +398,7 @@ struct JsonParse { ** Forward references **************************************************************************/ static void jsonReturnStringAsBlob(JsonString*); -static int jsonFuncArgMightBeBinary(sqlite3_value *pJson); +static int jsonArgIsJsonb(sqlite3_value *pJson, JsonParse *p); static u32 jsonTranslateBlobToText(const JsonParse*,u32,JsonString*); static void jsonReturnParse(sqlite3_context*,JsonParse*); static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); @@ -419,7 +472,7 @@ static int jsonCacheInsert( ** most-recently used entry if it isn't so already. ** ** The JsonParse object returned still belongs to the Cache and might -** be deleted at any moment. If the caller whants the JsonParse to +** be deleted at any moment. If the caller wants the JsonParse to ** linger, it needs to increment the nPJRef reference counter. */ static JsonParse *jsonCacheSearch( @@ -763,11 +816,9 @@ static void jsonAppendSqlValue( break; } default: { - if( jsonFuncArgMightBeBinary(pValue) ){ - JsonParse px; - memset(&px, 0, sizeof(px)); - px.aBlob = (u8*)sqlite3_value_blob(pValue); - px.nBlob = sqlite3_value_bytes(pValue); + JsonParse px; + memset(&px, 0, sizeof(px)); + if( jsonArgIsJsonb(pValue, &px) ){ jsonTranslateBlobToText(&px, 0, p); }else if( p->eErr==0 ){ sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); @@ -1086,7 +1137,7 @@ static void jsonWrongNumArgs( */ static int jsonBlobExpand(JsonParse *pParse, u32 N){ u8 *aNew; - u32 t; + u64 t; assert( N>pParse->nBlobAlloc ); if( pParse->nBlobAlloc==0 ){ t = 100; @@ -1096,8 +1147,9 @@ static int jsonBlobExpand(JsonParse *pParse, u32 N){ if( tdb, pParse->aBlob, t); if( aNew==0 ){ pParse->oom = 1; return 1; } + assert( t<0x7fffffff ); pParse->aBlob = aNew; - pParse->nBlobAlloc = t; + pParse->nBlobAlloc = (u32)t; return 0; } @@ -1164,7 +1216,7 @@ static SQLITE_NOINLINE void jsonBlobExpandAndAppendNode( } -/* Append an node type byte together with the payload size and +/* Append a node type byte together with the payload size and ** possibly also the payload. ** ** If aPayload is not NULL, then it is a pointer to the payload which @@ -1233,8 +1285,10 @@ static int jsonBlobChangePayloadSize( nExtra = 1; }else if( szType==13 ){ nExtra = 2; - }else{ + }else if( szType==14 ){ nExtra = 4; + }else{ + nExtra = 8; } if( szPayload<=11 ){ nNeeded = 0; @@ -1704,7 +1758,12 @@ static int jsonTranslateTextToBlob(JsonParse *pParse, u32 i){ || c=='n' || c=='r' || c=='t' || (c=='u' && jsonIs4Hex(&z[j+1])) ){ if( opcode==JSONB_TEXT ) opcode = JSONB_TEXTJ; - }else if( c=='\'' || c=='0' || c=='v' || c=='\n' + }else if( c=='\'' || c=='v' || c=='\n' +#ifdef SQLITE_BUG_COMPATIBLE_20250510 + || (c=='0') /* Legacy bug compatible */ +#else + || (c=='0' && !sqlite3Isdigit(z[j+1])) /* Correct implementation */ +#endif || (0xe2==(u8)c && 0x80==(u8)z[j+1] && (0xa8==(u8)z[j+2] || 0xa9==(u8)z[j+2])) || (c=='x' && jsonIs2Hex(&z[j+1])) ){ @@ -2054,10 +2113,7 @@ static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){ u8 x; u32 sz; u32 n; - if( NEVER(i>pParse->nBlob) ){ - *pSz = 0; - return 0; - } + assert( i<=pParse->nBlob ); x = pParse->aBlob[i]>>4; if( x<=11 ){ sz = x; @@ -2094,15 +2150,15 @@ static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){ *pSz = 0; return 0; } - sz = (pParse->aBlob[i+5]<<24) + (pParse->aBlob[i+6]<<16) + + sz = ((u32)pParse->aBlob[i+5]<<24) + (pParse->aBlob[i+6]<<16) + (pParse->aBlob[i+7]<<8) + pParse->aBlob[i+8]; n = 9; } if( (i64)i+sz+n > pParse->nBlob && (i64)i+sz+n > pParse->nBlob-pParse->delta ){ - sz = 0; - n = 0; + *pSz = 0; + return 0; } *pSz = sz; return n; @@ -2199,9 +2255,12 @@ static u32 jsonTranslateBlobToText( } case JSONB_TEXT: case JSONB_TEXTJ: { - jsonAppendChar(pOut, '"'); - jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n], sz); - jsonAppendChar(pOut, '"'); + if( pOut->nUsed+sz+2<=pOut->nAlloc || jsonStringGrow(pOut, sz+2)==0 ){ + pOut->zBuf[pOut->nUsed] = '"'; + memcpy(pOut->zBuf+pOut->nUsed+1,(const char*)&pParse->aBlob[i+n],sz); + pOut->zBuf[pOut->nUsed+sz+1] = '"'; + pOut->nUsed += sz+2; + } break; } case JSONB_TEXT5: { @@ -2440,33 +2499,6 @@ static u32 jsonTranslateBlobToPrettyText( return i; } - -/* Return true if the input pJson -** -** For performance reasons, this routine does not do a detailed check of the -** input BLOB to ensure that it is well-formed. Hence, false positives are -** possible. False negatives should never occur, however. -*/ -static int jsonFuncArgMightBeBinary(sqlite3_value *pJson){ - u32 sz, n; - const u8 *aBlob; - int nBlob; - JsonParse s; - if( sqlite3_value_type(pJson)!=SQLITE_BLOB ) return 0; - aBlob = sqlite3_value_blob(pJson); - nBlob = sqlite3_value_bytes(pJson); - if( nBlob<1 ) return 0; - if( NEVER(aBlob==0) || (aBlob[0] & 0x0f)>JSONB_OBJECT ) return 0; - memset(&s, 0, sizeof(s)); - s.aBlob = (u8*)aBlob; - s.nBlob = nBlob; - n = jsonbPayloadSize(&s, 0, &sz); - if( n==0 ) return 0; - if( sz+n!=(u32)nBlob ) return 0; - if( (aBlob[0] & 0x0f)<=JSONB_FALSE && sz>0 ) return 0; - return sz+n==(u32)nBlob; -} - /* ** Given that a JSONB_ARRAY object starts at offset i, return ** the number of entries in that array. @@ -2499,6 +2531,82 @@ static void jsonAfterEditSizeAdjust(JsonParse *pParse, u32 iRoot){ pParse->delta += jsonBlobChangePayloadSize(pParse, iRoot, sz); } +/* +** If the JSONB at aIns[0..nIns-1] can be expanded (by denormalizing the +** size field) by d bytes, then write the expansion into aOut[] and +** return true. In this way, an overwrite happens without changing the +** size of the JSONB, which reduces memcpy() operations and also make it +** faster and easier to update the B-Tree entry that contains the JSONB +** in the database. +** +** If the expansion of aIns[] by d bytes cannot be (easily) accomplished +** then return false. +** +** The d parameter is guaranteed to be between 1 and 8. +** +** This routine is an optimization. A correct answer is obtained if it +** always leaves the output unchanged and returns false. +*/ +static int jsonBlobOverwrite( + u8 *aOut, /* Overwrite here */ + const u8 *aIns, /* New content */ + u32 nIns, /* Bytes of new content */ + u32 d /* Need to expand new content by this much */ +){ + u32 szPayload; /* Bytes of payload */ + u32 i; /* New header size, after expansion & a loop counter */ + u8 szHdr; /* Size of header before expansion */ + + /* Lookup table for finding the upper 4 bits of the first byte of the + ** expanded aIns[], based on the size of the expanded aIns[] header: + ** + ** 2 3 4 5 6 7 8 9 */ + static const u8 aType[] = { 0xc0, 0xd0, 0, 0xe0, 0, 0, 0, 0xf0 }; + + if( (aIns[0]&0x0f)<=2 ) return 0; /* Cannot enlarge NULL, true, false */ + switch( aIns[0]>>4 ){ + default: { /* aIns[] header size 1 */ + if( ((1<=2 && i<=9 && aType[i-2]!=0 ); + aOut[0] = (aIns[0] & 0x0f) | aType[i-2]; + memcpy(&aOut[i], &aIns[szHdr], nIns-szHdr); + szPayload = nIns - szHdr; + while( 1/*edit-by-break*/ ){ + i--; + aOut[i] = szPayload & 0xff; + if( i==1 ) break; + szPayload >>= 8; + } + assert( (szPayload>>8)==0 ); + return 1; +} + /* ** Modify the JSONB blob at pParse->aBlob by removing nDel bytes of ** content beginning at iDel, and replacing them with nIns bytes of @@ -2520,6 +2628,11 @@ static void jsonBlobEdit( u32 nIns /* Bytes of content to insert */ ){ i64 d = (i64)nIns - (i64)nDel; + if( d<0 && d>=(-8) && aIns!=0 + && jsonBlobOverwrite(&pParse->aBlob[iDel], aIns, nIns, (int)-d) + ){ + return; + } if( d!=0 ){ if( pParse->nBlob + d > pParse->nBlobAlloc ){ jsonBlobExpand(pParse, pParse->nBlob+d); @@ -2531,7 +2644,9 @@ static void jsonBlobEdit( pParse->nBlob += d; pParse->delta += d; } - if( nIns && aIns ) memcpy(&pParse->aBlob[iDel], aIns, nIns); + if( nIns && aIns ){ + memcpy(&pParse->aBlob[iDel], aIns, nIns); + } } /* @@ -2616,7 +2731,21 @@ static u32 jsonUnescapeOneChar(const char *z, u32 n, u32 *piOut){ case 'r': { *piOut = '\r'; return 2; } case 't': { *piOut = '\t'; return 2; } case 'v': { *piOut = '\v'; return 2; } - case '0': { *piOut = 0; return 2; } + case '0': { + /* JSON5 requires that the \0 escape not be followed by a digit. + ** But SQLite did not enforce this restriction in versions 3.42.0 + ** through 3.49.2. That was a bug. But some applications might have + ** come to depend on that bug. Use the SQLITE_BUG_COMPATIBLE_20250510 + ** option to restore the old buggy behavior. */ +#ifdef SQLITE_BUG_COMPATIBLE_20250510 + /* Legacy bug-compatible behavior */ + *piOut = 0; +#else + /* Correct behavior */ + *piOut = (n>2 && sqlite3Isdigit(z[2])) ? JSON_INVALID_CHAR : 0; +#endif + return 2; + } case '\'': case '"': case '/': @@ -3116,7 +3245,7 @@ static void jsonReturnFromBlob( char *zOut; u32 nOut = sz; z = (const char*)&pParse->aBlob[i+n]; - zOut = sqlite3DbMallocRaw(db, nOut+1); + zOut = sqlite3DbMallocRaw(db, ((u64)nOut)+1); if( zOut==0 ) goto returnfromblob_oom; for(iIn=iOut=0; iInaBlob = (u8*)sqlite3_value_blob(pArg); - pParse->nBlob = sqlite3_value_bytes(pArg); - }else{ + if( !jsonArgIsJsonb(pArg, pParse) ){ sqlite3_result_error(ctx, "JSON cannot hold BLOB values", -1); return 1; } @@ -3294,7 +3420,7 @@ static char *jsonBadPathError( } /* argv[0] is a BLOB that seems likely to be a JSONB. Subsequent -** arguments come in parse where each pair contains a JSON path and +** arguments come in pairs where each pair contains a JSON path and ** content to insert or set at that patch. Do the updates ** and return the result. ** @@ -3365,27 +3491,46 @@ static void jsonInsertIntoBlob( /* ** If pArg is a blob that seems like a JSONB blob, then initialize ** p to point to that JSONB and return TRUE. If pArg does not seem like -** a JSONB blob, then return FALSE; +** a JSONB blob, then return FALSE. +** +** For small BLOBs (having no more than 7 bytes of payload) a full +** validity check is done. So for small BLOBs this routine only returns +** true if the value is guaranteed to be a valid JSONB. For larger BLOBs +** (8 byte or more of payload) only the size of the outermost element is +** checked to verify that the BLOB is superficially valid JSONB. +** +** A full JSONB validation is done on smaller BLOBs because those BLOBs might +** also be text JSON that has been incorrectly cast into a BLOB. +** (See tag-20240123-a and https://sqlite.org/forum/forumpost/012136abd5) +** If the BLOB is 9 bytes are larger, then it is not possible for the +** superficial size check done here to pass if the input is really text +** JSON so we do not need to look deeper in that case. ** -** This routine is only called if it is already known that pArg is a -** blob. The only open question is whether or not the blob appears -** to be a JSONB blob. +** Why we only need to do full JSONB validation for smaller BLOBs: +** +** The first byte of valid JSON text must be one of: '{', '[', '"', ' ', '\n', +** '\r', '\t', '-', or a digit '0' through '9'. Of these, only a subset +** can also be the first byte of JSONB: '{', '[', and digits '3' +** through '9'. In every one of those cases, the payload size is 7 bytes +** or less. So if we do full JSONB validation for every BLOB where the +** payload is less than 7 bytes, we will never get a false positive for +** JSONB on an input that is really text JSON. */ static int jsonArgIsJsonb(sqlite3_value *pArg, JsonParse *p){ u32 n, sz = 0; + u8 c; + if( sqlite3_value_type(pArg)!=SQLITE_BLOB ) return 0; p->aBlob = (u8*)sqlite3_value_blob(pArg); p->nBlob = (u32)sqlite3_value_bytes(pArg); - if( p->nBlob==0 ){ - p->aBlob = 0; - return 0; - } - if( NEVER(p->aBlob==0) ){ - return 0; - } - if( (p->aBlob[0] & 0x0f)<=JSONB_OBJECT + if( p->nBlob>0 + && ALWAYS(p->aBlob!=0) + && ((c = p->aBlob[0]) & 0x0f)<=JSONB_OBJECT && (n = jsonbPayloadSize(p, 0, &sz))>0 && sz+n==p->nBlob - && ((p->aBlob[0] & 0x0f)>JSONB_FALSE || sz==0) + && ((c & 0x0f)>JSONB_FALSE || sz==0) + && (sz>7 + || (c!=0x7b && c!=0x5b && !sqlite3Isdigit(c)) + || jsonbValidityCheck(p, 0, p->nBlob, 1)==0) ){ return 1; } @@ -3463,7 +3608,7 @@ static JsonParse *jsonParseFuncArg( ** JSON functions were suppose to work. From the beginning, blob was ** reserved for expansion and a blob value should have raised an error. ** But it did not, due to a bug. And many applications came to depend - ** upon this buggy behavior, espeically when using the CLI and reading + ** upon this buggy behavior, especially when using the CLI and reading ** JSON text using readfile(), which returns a blob. For this reason ** we will continue to support the bug moving forward. ** See for example https://sqlite.org/forum/forumpost/012136abd5292b8d @@ -4478,21 +4623,17 @@ static void jsonValidFunc( return; } case SQLITE_BLOB: { - if( jsonFuncArgMightBeBinary(argv[0]) ){ + JsonParse py; + memset(&py, 0, sizeof(py)); + if( jsonArgIsJsonb(argv[0], &py) ){ if( flags & 0x04 ){ /* Superficial checking only - accomplished by the - ** jsonFuncArgMightBeBinary() call above. */ + ** jsonArgIsJsonb() call above. */ res = 1; }else if( flags & 0x08 ){ /* Strict checking. Check by translating BLOB->TEXT->BLOB. If ** no errors occur, call that a "strict check". */ - JsonParse px; - u32 iErr; - memset(&px, 0, sizeof(px)); - px.aBlob = (u8*)sqlite3_value_blob(argv[0]); - px.nBlob = sqlite3_value_bytes(argv[0]); - iErr = jsonbValidityCheck(&px, 0, px.nBlob, 1); - res = iErr==0; + res = 0==jsonbValidityCheck(&py, 0, py.nBlob, 1); } break; } @@ -4550,9 +4691,7 @@ static void jsonErrorFunc( UNUSED_PARAMETER(argc); memset(&s, 0, sizeof(s)); s.db = sqlite3_context_db_handle(ctx); - if( jsonFuncArgMightBeBinary(argv[0]) ){ - s.aBlob = (u8*)sqlite3_value_blob(argv[0]); - s.nBlob = sqlite3_value_bytes(argv[0]); + if( jsonArgIsJsonb(argv[0], &s) ){ iErrPos = (i64)jsonbValidityCheck(&s, 0, s.nBlob, 1); }else{ s.zJson = (char*)sqlite3_value_text(argv[0]); @@ -4713,18 +4852,20 @@ static void jsonObjectStep( UNUSED_PARAMETER(argc); pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); if( pStr ){ + z = (const char*)sqlite3_value_text(argv[0]); + n = sqlite3Strlen30(z); if( pStr->zBuf==0 ){ jsonStringInit(pStr, ctx); jsonAppendChar(pStr, '{'); - }else if( pStr->nUsed>1 ){ + }else if( pStr->nUsed>1 && z!=0 ){ jsonAppendChar(pStr, ','); } pStr->pCtx = ctx; - z = (const char*)sqlite3_value_text(argv[0]); - n = sqlite3Strlen30(z); - jsonAppendString(pStr, z, n); - jsonAppendChar(pStr, ':'); - jsonAppendSqlValue(pStr, argv[1]); + if( z!=0 ){ + jsonAppendString(pStr, z, n); + jsonAppendChar(pStr, ':'); + jsonAppendSqlValue(pStr, argv[1]); + } } } static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ @@ -5237,9 +5378,8 @@ static int jsonEachFilter( memset(&p->sParse, 0, sizeof(p->sParse)); p->sParse.nJPRef = 1; p->sParse.db = p->db; - if( jsonFuncArgMightBeBinary(argv[0]) ){ - p->sParse.nBlob = sqlite3_value_bytes(argv[0]); - p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]); + if( jsonArgIsJsonb(argv[0], &p->sParse) ){ + /* We have JSONB */ }else{ p->sParse.zJson = (char*)sqlite3_value_text(argv[0]); p->sParse.nJson = sqlite3_value_bytes(argv[0]); diff --git a/src/loadext.c b/src/loadext.c index 7e0ae2543..40d4f3128 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -517,7 +517,9 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_stmt_explain, /* Version 3.44.0 and later */ sqlite3_get_clientdata, - sqlite3_set_clientdata + sqlite3_set_clientdata, + /* Version 3.50.0 and later */ + sqlite3_setlk_timeout }; /* True if x is the directory separator character diff --git a/src/main.c b/src/main.c index 3f8790d41..9b67de815 100644 --- a/src/main.c +++ b/src/main.c @@ -303,6 +303,14 @@ int sqlite3_initialize(void){ if( rc==SQLITE_OK ){ sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage); +#ifdef SQLITE_EXTRA_INIT_MUTEXED + { + int SQLITE_EXTRA_INIT_MUTEXED(const char*); + rc = SQLITE_EXTRA_INIT_MUTEXED(0); + } +#endif + } + if( rc==SQLITE_OK ){ sqlite3MemoryBarrier(); sqlite3GlobalConfig.isInit = 1; #ifdef SQLITE_EXTRA_INIT @@ -759,17 +767,22 @@ int sqlite3_config(int op, ...){ ** If lookaside is already active, return SQLITE_BUSY. ** ** The sz parameter is the number of bytes in each lookaside slot. -** The cnt parameter is the number of slots. If pStart is NULL the -** space for the lookaside memory is obtained from sqlite3_malloc(). -** If pStart is not NULL then it is sz*cnt bytes of memory to use for -** the lookaside memory. +** The cnt parameter is the number of slots. If pBuf is NULL the +** space for the lookaside memory is obtained from sqlite3_malloc() +** or similar. If pBuf is not NULL then it is sz*cnt bytes of memory +** to use for the lookaside memory. */ -static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ +static int setupLookaside( + sqlite3 *db, /* Database connection being configured */ + void *pBuf, /* Memory to use for lookaside. May be NULL */ + int sz, /* Desired size of each lookaside memory slot */ + int cnt /* Number of slots to allocate */ +){ #ifndef SQLITE_OMIT_LOOKASIDE - void *pStart; - sqlite3_int64 szAlloc; - int nBig; /* Number of full-size slots */ - int nSm; /* Number smaller LOOKASIDE_SMALL-byte slots */ + void *pStart; /* Start of the lookaside buffer */ + sqlite3_int64 szAlloc; /* Total space set aside for lookaside memory */ + int nBig; /* Number of full-size slots */ + int nSm; /* Number smaller LOOKASIDE_SMALL-byte slots */ if( sqlite3LookasideUsed(db,0)>0 ){ return SQLITE_BUSY; @@ -782,19 +795,22 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ sqlite3_free(db->lookaside.pStart); } /* The size of a lookaside slot after ROUNDDOWN8 needs to be larger - ** than a pointer to be useful. + ** than a pointer and small enough to fit in a u16. */ - sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ + sz = ROUNDDOWN8(sz); if( sz<=(int)sizeof(LookasideSlot*) ) sz = 0; if( sz>65528 ) sz = 65528; - if( cnt<0 ) cnt = 0; + /* Count must be at least 1 to be useful, but not so large as to use + ** more than 0x7fff0000 total bytes for lookaside. */ + if( cnt<1 ) cnt = 0; + if( sz>0 && cnt>(0x7fff0000/sz) ) cnt = 0x7fff0000/sz; szAlloc = (i64)sz*(i64)cnt; - if( sz==0 || cnt==0 ){ + if( szAlloc==0 ){ sz = 0; pStart = 0; }else if( pBuf==0 ){ sqlite3BeginBenignMalloc(); - pStart = sqlite3Malloc( szAlloc ); /* IMP: R-61949-35727 */ + pStart = sqlite3Malloc( szAlloc ); sqlite3EndBenignMalloc(); if( pStart ) szAlloc = sqlite3MallocSize(pStart); }else{ @@ -1771,6 +1787,9 @@ int sqlite3_busy_handler( db->busyHandler.pBusyArg = pArg; db->busyHandler.nBusy = 0; db->busyTimeout = 0; +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + db->setlkTimeout = 0; +#endif sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } @@ -1820,12 +1839,47 @@ int sqlite3_busy_timeout(sqlite3 *db, int ms){ sqlite3_busy_handler(db, (int(*)(void*,int))sqliteDefaultBusyCallback, (void*)db); db->busyTimeout = ms; +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + db->setlkTimeout = ms; +#endif }else{ sqlite3_busy_handler(db, 0, 0); } return SQLITE_OK; } +/* +** Set the setlk timeout value. +*/ +int sqlite3_setlk_timeout(sqlite3 *db, int ms, int flags){ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + int iDb; + int bBOC = ((flags & SQLITE_SETLK_BLOCK_ON_CONNECT) ? 1 : 0); +#endif +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + if( ms<-1 ) return SQLITE_RANGE; +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + db->setlkTimeout = ms; + db->setlkFlags = flags; + sqlite3BtreeEnterAll(db); + for(iDb=0; iDbnDb; iDb++){ + Btree *pBt = db->aDb[iDb].pBt; + if( pBt ){ + sqlite3_file *fd = sqlite3PagerFile(sqlite3BtreePager(pBt)); + sqlite3OsFileControlHint(fd, SQLITE_FCNTL_BLOCK_ON_CONNECT, (void*)&bBOC); + } + } + sqlite3BtreeLeaveAll(db); +#endif +#if !defined(SQLITE_ENABLE_API_ARMOR) && !defined(SQLITE_ENABLE_SETLK_TIMEOUT) + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(flags); +#endif + return SQLITE_OK; +} + /* ** Cause any pending operation to stop at its earliest opportunity. */ @@ -3791,7 +3845,7 @@ int sqlite3_set_clientdata( return SQLITE_OK; }else{ size_t n = strlen(zName); - p = sqlite3_malloc64( sizeof(DbClientData)+n+1 ); + p = sqlite3_malloc64( SZ_DBCLIENTDATA(n+1) ); if( p==0 ){ if( xDestructor ) xDestructor(pData); sqlite3_mutex_leave(db->mutex); @@ -3945,13 +3999,10 @@ int sqlite3_table_column_metadata( if( zColumnName==0 ){ /* Query for existence of table only */ }else{ - for(iCol=0; iColnCol; iCol++){ + iCol = sqlite3ColumnIndex(pTab, zColumnName); + if( iCol>=0 ){ pCol = &pTab->aCol[iCol]; - if( 0==sqlite3StrICmp(pCol->zCnName, zColumnName) ){ - break; - } - } - if( iCol==pTab->nCol ){ + }else{ if( HasRowid(pTab) && sqlite3IsRowid(zColumnName) ){ iCol = pTab->iPKey; pCol = iCol>=0 ? &pTab->aCol[iCol] : 0; @@ -4160,8 +4211,8 @@ int sqlite3_test_control(int op, ...){ /* sqlite3_test_control(SQLITE_TESTCTRL_FK_NO_ACTION, sqlite3 *db, int b); ** ** If b is true, then activate the SQLITE_FkNoAction setting. If b is - ** false then clearn that setting. If the SQLITE_FkNoAction setting is - ** abled, all foreign key ON DELETE and ON UPDATE actions behave as if + ** false then clear that setting. If the SQLITE_FkNoAction setting is + ** enabled, all foreign key ON DELETE and ON UPDATE actions behave as if ** they were NO ACTION, regardless of how they are defined. ** ** NB: One must usually run "PRAGMA writable_schema=RESET" after diff --git a/src/memdb.c b/src/memdb.c index d83a51d54..61a378bb3 100644 --- a/src/memdb.c +++ b/src/memdb.c @@ -567,13 +567,13 @@ static int memdbOpen( } if( p==0 ){ MemStore **apNew; - p = sqlite3Malloc( sizeof(*p) + szName + 3 ); + p = sqlite3Malloc( sizeof(*p) + (i64)szName + 3 ); if( p==0 ){ sqlite3_mutex_leave(pVfsMutex); return SQLITE_NOMEM; } apNew = sqlite3Realloc(memdb_g.apMemStore, - sizeof(apNew[0])*(memdb_g.nMemStore+1) ); + sizeof(apNew[0])*(1+(i64)memdb_g.nMemStore) ); if( apNew==0 ){ sqlite3_free(p); sqlite3_mutex_leave(pVfsMutex); diff --git a/src/os_unix.c b/src/os_unix.c index c897895d7..784bc3517 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -284,6 +284,7 @@ struct unixFile { #endif #ifdef SQLITE_ENABLE_SETLK_TIMEOUT unsigned iBusyTimeout; /* Wait this many millisec on locks */ + int bBlockOnConnect; /* True to block for SHARED locks */ #endif #if OS_VXWORKS struct vxworksFileId *pId; /* Unique file ID */ @@ -1677,6 +1678,13 @@ static int unixFileLock(unixFile *pFile, struct flock *pLock){ rc = 0; } }else{ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( pFile->bBlockOnConnect && pLock->l_type==F_RDLCK + && pLock->l_start==SHARED_FIRST && pLock->l_len==SHARED_SIZE + ){ + rc = osFcntl(pFile->h, F_SETLKW, pLock); + }else +#endif rc = osSetPosixAdvisoryLock(pFile->h, pLock, pFile); } return rc; @@ -4038,8 +4046,9 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ #ifdef SQLITE_ENABLE_SETLK_TIMEOUT case SQLITE_FCNTL_LOCK_TIMEOUT: { int iOld = pFile->iBusyTimeout; + int iNew = *(int*)pArg; #if SQLITE_ENABLE_SETLK_TIMEOUT==1 - pFile->iBusyTimeout = *(int*)pArg; + pFile->iBusyTimeout = iNew<0 ? 0x7FFFFFFF : (unsigned)iNew; #elif SQLITE_ENABLE_SETLK_TIMEOUT==2 pFile->iBusyTimeout = !!(*(int*)pArg); #else @@ -4048,7 +4057,12 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ *(int*)pArg = iOld; return SQLITE_OK; } -#endif + case SQLITE_FCNTL_BLOCK_ON_CONNECT: { + int iNew = *(int*)pArg; + pFile->bBlockOnConnect = iNew; + return SQLITE_OK; + } +#endif /* SQLITE_ENABLE_SETLK_TIMEOUT */ #if SQLITE_MAX_MMAP_SIZE>0 case SQLITE_FCNTL_MMAP_SIZE: { i64 newLimit = *(i64*)pArg; @@ -5021,21 +5035,20 @@ static int unixShmLock( /* Check that, if this to be a blocking lock, no locks that occur later ** in the following list than the lock being obtained are already held: ** - ** 1. Checkpointer lock (ofst==1). - ** 2. Write lock (ofst==0). - ** 3. Read locks (ofst>=3 && ofst=3 && ofstexclMask|p->sharedMask); assert( (flags & SQLITE_SHM_UNLOCK) || pDbFd->iBusyTimeout==0 || ( - (ofst!=2) /* not RECOVER */ + (ofst!=2 || lockMask==0) && (ofst!=1 || lockMask==0 || lockMask==2) && (ofst!=0 || lockMask<3) && (ofst<3 || lockMask<(1<iBusyTimeout +#else +# define winFileBusyTimeout(pDbFd) 0 +#endif + /* ** The winVfsAppData structure is used for the pAppData member for all of the ** Win32 VFS variants. @@ -607,7 +617,7 @@ static struct win_syscall { { "FileTimeToLocalFileTime", (SYSCALL)0, 0 }, #endif -#define osFileTimeToLocalFileTime ((BOOL(WINAPI*)(CONST FILETIME*, \ +#define osFileTimeToLocalFileTime ((BOOL(WINAPI*)(const FILETIME*, \ LPFILETIME))aSyscall[11].pCurrent) #if SQLITE_OS_WINCE @@ -616,7 +626,7 @@ static struct win_syscall { { "FileTimeToSystemTime", (SYSCALL)0, 0 }, #endif -#define osFileTimeToSystemTime ((BOOL(WINAPI*)(CONST FILETIME*, \ +#define osFileTimeToSystemTime ((BOOL(WINAPI*)(const FILETIME*, \ LPSYSTEMTIME))aSyscall[12].pCurrent) { "FlushFileBuffers", (SYSCALL)FlushFileBuffers, 0 }, @@ -722,6 +732,12 @@ static struct win_syscall { #define osGetFullPathNameW ((DWORD(WINAPI*)(LPCWSTR,DWORD,LPWSTR, \ LPWSTR*))aSyscall[25].pCurrent) +/* +** For GetLastError(), MSDN says: +** +** Minimum supported client: Windows XP [desktop apps | UWP apps] +** Minimum supported server: Windows Server 2003 [desktop apps | UWP apps] +*/ { "GetLastError", (SYSCALL)GetLastError, 0 }, #define osGetLastError ((DWORD(WINAPI*)(VOID))aSyscall[26].pCurrent) @@ -890,7 +906,7 @@ static struct win_syscall { { "LockFile", (SYSCALL)0, 0 }, #endif -#ifndef osLockFile +#if !defined(osLockFile) && defined(SQLITE_WIN32_HAS_ANSI) #define osLockFile ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ DWORD))aSyscall[47].pCurrent) #endif @@ -954,7 +970,7 @@ static struct win_syscall { { "SystemTimeToFileTime", (SYSCALL)SystemTimeToFileTime, 0 }, -#define osSystemTimeToFileTime ((BOOL(WINAPI*)(CONST SYSTEMTIME*, \ +#define osSystemTimeToFileTime ((BOOL(WINAPI*)(const SYSTEMTIME*, \ LPFILETIME))aSyscall[56].pCurrent) #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT @@ -963,7 +979,7 @@ static struct win_syscall { { "UnlockFile", (SYSCALL)0, 0 }, #endif -#ifndef osUnlockFile +#if !defined(osUnlockFile) && defined(SQLITE_WIN32_HAS_ANSI) #define osUnlockFile ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ DWORD))aSyscall[57].pCurrent) #endif @@ -1004,11 +1020,13 @@ static struct win_syscall { #define osCreateEventExW ((HANDLE(WINAPI*)(LPSECURITY_ATTRIBUTES,LPCWSTR, \ DWORD,DWORD))aSyscall[62].pCurrent) -#if !SQLITE_OS_WINRT +/* +** For WaitForSingleObject(), MSDN says: +** +** Minimum supported client: Windows XP [desktop apps | UWP apps] +** Minimum supported server: Windows Server 2003 [desktop apps | UWP apps] +*/ { "WaitForSingleObject", (SYSCALL)WaitForSingleObject, 0 }, -#else - { "WaitForSingleObject", (SYSCALL)0, 0 }, -#endif #define osWaitForSingleObject ((DWORD(WINAPI*)(HANDLE, \ DWORD))aSyscall[63].pCurrent) @@ -1155,6 +1173,97 @@ static struct win_syscall { #define osFlushViewOfFile \ ((BOOL(WINAPI*)(LPCVOID,SIZE_T))aSyscall[79].pCurrent) +/* +** If SQLITE_ENABLE_SETLK_TIMEOUT is defined, we require CreateEvent() +** to implement blocking locks with timeouts. MSDN says: +** +** Minimum supported client: Windows XP [desktop apps | UWP apps] +** Minimum supported server: Windows Server 2003 [desktop apps | UWP apps] +*/ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + { "CreateEvent", (SYSCALL)CreateEvent, 0 }, +#else + { "CreateEvent", (SYSCALL)0, 0 }, +#endif + +#define osCreateEvent ( \ + (HANDLE(WINAPI*) (LPSECURITY_ATTRIBUTES,BOOL,BOOL,LPCSTR)) \ + aSyscall[80].pCurrent \ +) + +/* +** If SQLITE_ENABLE_SETLK_TIMEOUT is defined, we require CancelIo() +** for the case where a timeout expires and a lock request must be +** cancelled. +** +** Minimum supported client: Windows XP [desktop apps | UWP apps] +** Minimum supported server: Windows Server 2003 [desktop apps | UWP apps] +*/ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + { "CancelIo", (SYSCALL)CancelIo, 0 }, +#else + { "CancelIo", (SYSCALL)0, 0 }, +#endif + +#define osCancelIo ((BOOL(WINAPI*)(HANDLE))aSyscall[81].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) && defined(_WIN32) + { "GetModuleHandleW", (SYSCALL)GetModuleHandleW, 0 }, +#else + { "GetModuleHandleW", (SYSCALL)0, 0 }, +#endif + +#define osGetModuleHandleW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[82].pCurrent) + +#ifndef _WIN32 + { "getenv", (SYSCALL)getenv, 0 }, +#else + { "getenv", (SYSCALL)0, 0 }, +#endif + +#define osGetenv ((const char *(*)(const char *))aSyscall[83].pCurrent) + +#ifndef _WIN32 + { "getcwd", (SYSCALL)getcwd, 0 }, +#else + { "getcwd", (SYSCALL)0, 0 }, +#endif + +#define osGetcwd ((char*(*)(char*,size_t))aSyscall[84].pCurrent) + +#ifndef _WIN32 + { "readlink", (SYSCALL)readlink, 0 }, +#else + { "readlink", (SYSCALL)0, 0 }, +#endif + +#define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[85].pCurrent) + +#ifndef _WIN32 + { "lstat", (SYSCALL)lstat, 0 }, +#else + { "lstat", (SYSCALL)0, 0 }, +#endif + +#define osLstat ((int(*)(const char*,struct stat*))aSyscall[86].pCurrent) + +#ifndef _WIN32 + { "__errno", (SYSCALL)__errno, 0 }, +#else + { "__errno", (SYSCALL)0, 0 }, +#endif + +#define osErrno (*((int*(*)(void))aSyscall[87].pCurrent)()) + +#ifndef _WIN32 + { "cygwin_conv_path", (SYSCALL)cygwin_conv_path, 0 }, +#else + { "cygwin_conv_path", (SYSCALL)0, 0 }, +#endif + +#define osCygwin_conv_path ((size_t(*)(unsigned int, \ + const void *, void *, size_t))aSyscall[88].pCurrent) + }; /* End of the overrideable system calls */ /* @@ -1328,6 +1437,7 @@ int sqlite3_win32_reset_heap(){ } #endif /* SQLITE_WIN32_MALLOC */ +#ifdef _WIN32 /* ** This function outputs the specified (ANSI) string to the Win32 debugger ** (if available). @@ -1370,6 +1480,7 @@ void sqlite3_win32_write_debug(const char *zBuf, int nBuf){ } #endif } +#endif /* _WIN32 */ /* ** The following routine suspends the current thread for at least ms @@ -1453,7 +1564,9 @@ int sqlite3_win32_is_nt(void){ } return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; #elif SQLITE_TEST - return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; + return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2 + || osInterlockedCompareExchange(&sqlite3_os_type, 0, 0)==0 + ; #else /* ** NOTE: All sub-platforms where the GetVersionEx[AW] functions are @@ -1668,6 +1781,7 @@ void sqlite3MemSetDefault(void){ } #endif /* SQLITE_WIN32_MALLOC */ +#ifdef _WIN32 /* ** Convert a UTF-8 string to Microsoft Unicode. ** @@ -1693,6 +1807,7 @@ static LPWSTR winUtf8ToUnicode(const char *zText){ } return zWideText; } +#endif /* _WIN32 */ /* ** Convert a Microsoft Unicode string to UTF-8. @@ -1727,28 +1842,29 @@ static char *winUnicodeToUtf8(LPCWSTR zWideText){ ** Space to hold the returned string is obtained from sqlite3_malloc(). */ static LPWSTR winMbcsToUnicode(const char *zText, int useAnsi){ - int nByte; + int nWideChar; LPWSTR zMbcsText; int codepage = useAnsi ? CP_ACP : CP_OEMCP; - nByte = osMultiByteToWideChar(codepage, 0, zText, -1, NULL, - 0)*sizeof(WCHAR); - if( nByte==0 ){ + nWideChar = osMultiByteToWideChar(codepage, 0, zText, -1, NULL, + 0); + if( nWideChar==0 ){ return 0; } - zMbcsText = sqlite3MallocZero( nByte*sizeof(WCHAR) ); + zMbcsText = sqlite3MallocZero( nWideChar*sizeof(WCHAR) ); if( zMbcsText==0 ){ return 0; } - nByte = osMultiByteToWideChar(codepage, 0, zText, -1, zMbcsText, - nByte); - if( nByte==0 ){ + nWideChar = osMultiByteToWideChar(codepage, 0, zText, -1, zMbcsText, + nWideChar); + if( nWideChar==0 ){ sqlite3_free(zMbcsText); zMbcsText = 0; } return zMbcsText; } +#ifdef _WIN32 /* ** Convert a Microsoft Unicode string to a multi-byte character string, ** using the ANSI or OEM code page. @@ -1776,6 +1892,7 @@ static char *winUnicodeToMbcs(LPCWSTR zWideText, int useAnsi){ } return zText; } +#endif /* _WIN32 */ /* ** Convert a multi-byte character string to UTF-8. @@ -1795,6 +1912,7 @@ static char *winMbcsToUtf8(const char *zText, int useAnsi){ return zTextUtf8; } +#ifdef _WIN32 /* ** Convert a UTF-8 string to a multi-byte character string. ** @@ -1844,6 +1962,7 @@ char *sqlite3_win32_unicode_to_utf8(LPCWSTR zWideText){ #endif return winUnicodeToUtf8(zWideText); } +#endif /* _WIN32 */ /* ** This is a public wrapper for the winMbcsToUtf8() function. @@ -1861,6 +1980,7 @@ char *sqlite3_win32_mbcs_to_utf8(const char *zText){ return winMbcsToUtf8(zText, osAreFileApisANSI()); } +#ifdef _WIN32 /* ** This is a public wrapper for the winMbcsToUtf8() function. */ @@ -1985,6 +2105,7 @@ int sqlite3_win32_set_directory( ){ return sqlite3_win32_set_directory16(type, zValue); } +#endif /* _WIN32 */ /* ** The return value of winGetLastErrorMsg @@ -2533,13 +2654,98 @@ static BOOL winLockFile( ovlp.Offset = offsetLow; ovlp.OffsetHigh = offsetHigh; return osLockFileEx(*phFile, flags, 0, numBytesLow, numBytesHigh, &ovlp); +#ifdef SQLITE_WIN32_HAS_ANSI }else{ return osLockFile(*phFile, offsetLow, offsetHigh, numBytesLow, numBytesHigh); +#endif } #endif } +/* +** Lock a region of nByte bytes starting at offset offset of file hFile. +** Take an EXCLUSIVE lock if parameter bExclusive is true, or a SHARED lock +** otherwise. If nMs is greater than zero and the lock cannot be obtained +** immediately, block for that many ms before giving up. +** +** This function returns SQLITE_OK if the lock is obtained successfully. If +** some other process holds the lock, SQLITE_BUSY is returned if nMs==0, or +** SQLITE_BUSY_TIMEOUT otherwise. Or, if an error occurs, SQLITE_IOERR. +*/ +static int winHandleLockTimeout( + HANDLE hFile, + DWORD offset, + DWORD nByte, + int bExcl, + DWORD nMs +){ + DWORD flags = LOCKFILE_FAIL_IMMEDIATELY | (bExcl?LOCKFILE_EXCLUSIVE_LOCK:0); + int rc = SQLITE_OK; + BOOL ret; + + if( !osIsNT() ){ + ret = winLockFile(&hFile, flags, offset, 0, nByte, 0); + }else{ + OVERLAPPED ovlp; + memset(&ovlp, 0, sizeof(OVERLAPPED)); + ovlp.Offset = offset; + +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( nMs!=0 ){ + flags &= ~LOCKFILE_FAIL_IMMEDIATELY; + } + ovlp.hEvent = osCreateEvent(NULL, TRUE, FALSE, NULL); + if( ovlp.hEvent==NULL ){ + return SQLITE_IOERR_LOCK; + } +#endif + + ret = osLockFileEx(hFile, flags, 0, nByte, 0, &ovlp); + +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + /* If SQLITE_ENABLE_SETLK_TIMEOUT is defined, then the file-handle was + ** opened with FILE_FLAG_OVERHEAD specified. In this case, the call to + ** LockFileEx() may fail because the request is still pending. This can + ** happen even if LOCKFILE_FAIL_IMMEDIATELY was specified. + ** + ** If nMs is 0, then LOCKFILE_FAIL_IMMEDIATELY was set in the flags + ** passed to LockFileEx(). In this case, if the operation is pending, + ** block indefinitely until it is finished. + ** + ** Otherwise, wait for up to nMs ms for the operation to finish. nMs + ** may be set to INFINITE. + */ + if( !ret && GetLastError()==ERROR_IO_PENDING ){ + DWORD nDelay = (nMs==0 ? INFINITE : nMs); + DWORD res = osWaitForSingleObject(ovlp.hEvent, nDelay); + if( res==WAIT_OBJECT_0 ){ + ret = TRUE; + }else if( res==WAIT_TIMEOUT ){ +#if SQLITE_ENABLE_SETLK_TIMEOUT==1 + rc = SQLITE_BUSY_TIMEOUT; +#else + rc = SQLITE_BUSY; +#endif + }else{ + /* Some other error has occurred */ + rc = SQLITE_IOERR_LOCK; + } + + /* If it is still pending, cancel the LockFileEx() call. */ + osCancelIo(hFile); + } + + osCloseHandle(ovlp.hEvent); +#endif + } + + if( rc==SQLITE_OK && !ret ){ + rc = SQLITE_BUSY; + } + return rc; +} + /* ** Unlock a file region. */ @@ -2564,13 +2770,23 @@ static BOOL winUnlockFile( ovlp.Offset = offsetLow; ovlp.OffsetHigh = offsetHigh; return osUnlockFileEx(*phFile, 0, numBytesLow, numBytesHigh, &ovlp); +#ifdef SQLITE_WIN32_HAS_ANSI }else{ return osUnlockFile(*phFile, offsetLow, offsetHigh, numBytesLow, numBytesHigh); +#endif } #endif } +/* +** Remove an nByte lock starting at offset iOff from HANDLE h. +*/ +static int winHandleUnlock(HANDLE h, int iOff, int nByte){ + BOOL ret = winUnlockFile(&h, iOff, 0, nByte, 0); + return (ret ? SQLITE_OK : SQLITE_IOERR_UNLOCK); +} + /***************************************************************************** ** The next group of routines implement the I/O methods specified ** by the sqlite3_io_methods object. @@ -2584,66 +2800,70 @@ static BOOL winUnlockFile( #endif /* -** Move the current position of the file handle passed as the first -** argument to offset iOffset within the file. If successful, return 0. -** Otherwise, set pFile->lastErrno and return non-zero. +** Seek the file handle h to offset nByte of the file. +** +** If successful, return SQLITE_OK. Or, if an error occurs, return an SQLite +** error code. */ -static int winSeekFile(winFile *pFile, sqlite3_int64 iOffset){ +static int winHandleSeek(HANDLE h, sqlite3_int64 iOffset){ + int rc = SQLITE_OK; /* Return value */ + #if !SQLITE_OS_WINRT LONG upperBits; /* Most sig. 32 bits of new offset */ LONG lowerBits; /* Least sig. 32 bits of new offset */ DWORD dwRet; /* Value returned by SetFilePointer() */ - DWORD lastErrno; /* Value returned by GetLastError() */ - - OSTRACE(("SEEK file=%p, offset=%lld\n", pFile->h, iOffset)); upperBits = (LONG)((iOffset>>32) & 0x7fffffff); lowerBits = (LONG)(iOffset & 0xffffffff); + dwRet = osSetFilePointer(h, lowerBits, &upperBits, FILE_BEGIN); + /* API oddity: If successful, SetFilePointer() returns a dword ** containing the lower 32-bits of the new file-offset. Or, if it fails, ** it returns INVALID_SET_FILE_POINTER. However according to MSDN, ** INVALID_SET_FILE_POINTER may also be a valid new offset. So to determine ** whether an error has actually occurred, it is also necessary to call - ** GetLastError(). - */ - dwRet = osSetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); - - if( (dwRet==INVALID_SET_FILE_POINTER - && ((lastErrno = osGetLastError())!=NO_ERROR)) ){ - pFile->lastErrno = lastErrno; - winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, - "winSeekFile", pFile->zPath); - OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); - return 1; + ** GetLastError(). */ + if( dwRet==INVALID_SET_FILE_POINTER ){ + DWORD lastErrno = osGetLastError(); + if( lastErrno!=NO_ERROR ){ + rc = SQLITE_IOERR_SEEK; + } } - - OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); - return 0; #else - /* - ** Same as above, except that this implementation works for WinRT. - */ - + /* This implementation works for WinRT. */ LARGE_INTEGER x; /* The new offset */ BOOL bRet; /* Value returned by SetFilePointerEx() */ x.QuadPart = iOffset; - bRet = osSetFilePointerEx(pFile->h, x, 0, FILE_BEGIN); + bRet = osSetFilePointerEx(h, x, 0, FILE_BEGIN); if(!bRet){ - pFile->lastErrno = osGetLastError(); - winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, - "winSeekFile", pFile->zPath); - OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); - return 1; + rc = SQLITE_IOERR_SEEK; } - - OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); - return 0; #endif + + OSTRACE(("SEEK file=%p, offset=%lld rc=%s\n", h, iOffset, sqlite3ErrName(rc))); + return rc; +} + +/* +** Move the current position of the file handle passed as the first +** argument to offset iOffset within the file. If successful, return 0. +** Otherwise, set pFile->lastErrno and return non-zero. +*/ +static int winSeekFile(winFile *pFile, sqlite3_int64 iOffset){ + int rc; + + rc = winHandleSeek(pFile->h, iOffset); + if( rc!=SQLITE_OK ){ + pFile->lastErrno = osGetLastError(); + winLogError(rc, pFile->lastErrno, "winSeekFile", pFile->zPath); + } + return rc; } + #if SQLITE_MAX_MMAP_SIZE>0 /* Forward references to VFS helper methods used for memory mapped files */ static int winMapfile(winFile*, sqlite3_int64); @@ -2903,6 +3123,60 @@ static int winWrite( return SQLITE_OK; } +/* +** Truncate the file opened by handle h to nByte bytes in size. +*/ +static int winHandleTruncate(HANDLE h, sqlite3_int64 nByte){ + int rc = SQLITE_OK; /* Return code */ + rc = winHandleSeek(h, nByte); + if( rc==SQLITE_OK ){ + if( 0==osSetEndOfFile(h) ){ + rc = SQLITE_IOERR_TRUNCATE; + } + } + return rc; +} + +/* +** Determine the size in bytes of the file opened by the handle passed as +** the first argument. +*/ +static int winHandleSize(HANDLE h, sqlite3_int64 *pnByte){ + int rc = SQLITE_OK; + +#if SQLITE_OS_WINRT + FILE_STANDARD_INFO info; + BOOL b; + b = osGetFileInformationByHandleEx(h, FileStandardInfo, &info, sizeof(info)); + if( b ){ + *pnByte = info.EndOfFile.QuadPart; + }else{ + rc = SQLITE_IOERR_FSTAT; + } +#else + DWORD upperBits = 0; + DWORD lowerBits = 0; + + assert( pnByte ); + lowerBits = osGetFileSize(h, &upperBits); + *pnByte = (((sqlite3_int64)upperBits)<<32) + lowerBits; + if( lowerBits==INVALID_FILE_SIZE && osGetLastError()!=NO_ERROR ){ + rc = SQLITE_IOERR_FSTAT; + } +#endif + + return rc; +} + +/* +** Close the handle passed as the only argument. +*/ +static void winHandleClose(HANDLE h){ + if( h!=INVALID_HANDLE_VALUE ){ + osCloseHandle(h); + } +} + /* ** Truncate an open file to a specified size */ @@ -3158,8 +3432,9 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ ** Different API routines are called depending on whether or not this ** is Win9x or WinNT. */ -static int winGetReadLock(winFile *pFile){ +static int winGetReadLock(winFile *pFile, int bBlock){ int res; + DWORD mask = ~(bBlock ? LOCKFILE_FAIL_IMMEDIATELY : 0); OSTRACE(("READ-LOCK file=%p, lock=%d\n", pFile->h, pFile->locktype)); if( osIsNT() ){ #if SQLITE_OS_WINCE @@ -3169,7 +3444,7 @@ static int winGetReadLock(winFile *pFile){ */ res = winceLockFile(&pFile->h, SHARED_FIRST, 0, 1, 0); #else - res = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS, SHARED_FIRST, 0, + res = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS&mask, SHARED_FIRST, 0, SHARED_SIZE, 0); #endif } @@ -3178,7 +3453,7 @@ static int winGetReadLock(winFile *pFile){ int lk; sqlite3_randomness(sizeof(lk), &lk); pFile->sharedLockByte = (short)((lk & 0x7fffffff)%(SHARED_SIZE - 1)); - res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, + res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS&mask, SHARED_FIRST+pFile->sharedLockByte, 0, 1, 0); } #endif @@ -3273,46 +3548,62 @@ static int winLock(sqlite3_file *id, int locktype){ assert( locktype!=PENDING_LOCK ); assert( locktype!=RESERVED_LOCK || pFile->locktype==SHARED_LOCK ); - /* Lock the PENDING_LOCK byte if we need to acquire a PENDING lock or + /* Lock the PENDING_LOCK byte if we need to acquire an EXCLUSIVE lock or ** a SHARED lock. If we are acquiring a SHARED lock, the acquisition of ** the PENDING_LOCK byte is temporary. */ newLocktype = pFile->locktype; - if( pFile->locktype==NO_LOCK - || (locktype==EXCLUSIVE_LOCK && pFile->locktype<=RESERVED_LOCK) + if( locktype==SHARED_LOCK + || (locktype==EXCLUSIVE_LOCK && pFile->locktype==RESERVED_LOCK) ){ int cnt = 3; - while( cnt-->0 && (res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, - PENDING_BYTE, 0, 1, 0))==0 ){ + + /* Flags for the LockFileEx() call. This should be an exclusive lock if + ** this call is to obtain EXCLUSIVE, or a shared lock if this call is to + ** obtain SHARED. */ + int flags = LOCKFILE_FAIL_IMMEDIATELY; + if( locktype==EXCLUSIVE_LOCK ){ + flags |= LOCKFILE_EXCLUSIVE_LOCK; + } + while( cnt>0 ){ /* Try 3 times to get the pending lock. This is needed to work ** around problems caused by indexing and/or anti-virus software on ** Windows systems. + ** ** If you are using this code as a model for alternative VFSes, do not - ** copy this retry logic. It is a hack intended for Windows only. - */ + ** copy this retry logic. It is a hack intended for Windows only. */ + res = winLockFile(&pFile->h, flags, PENDING_BYTE, 0, 1, 0); + if( res ) break; + lastErrno = osGetLastError(); - OSTRACE(("LOCK-PENDING-FAIL file=%p, count=%d, result=%d\n", - pFile->h, cnt, res)); + OSTRACE(("LOCK-PENDING-FAIL file=%p, count=%d, result=%d\n", + pFile->h, cnt, res + )); + if( lastErrno==ERROR_INVALID_HANDLE ){ pFile->lastErrno = lastErrno; rc = SQLITE_IOERR_LOCK; - OSTRACE(("LOCK-FAIL file=%p, count=%d, rc=%s\n", - pFile->h, cnt, sqlite3ErrName(rc))); + OSTRACE(("LOCK-FAIL file=%p, count=%d, rc=%s\n", + pFile->h, cnt, sqlite3ErrName(rc) + )); return rc; } - if( cnt ) sqlite3_win32_sleep(1); + + cnt--; + if( cnt>0 ) sqlite3_win32_sleep(1); } gotPendingLock = res; - if( !res ){ - lastErrno = osGetLastError(); - } } /* Acquire a shared lock */ if( locktype==SHARED_LOCK && res ){ assert( pFile->locktype==NO_LOCK ); - res = winGetReadLock(pFile); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + res = winGetReadLock(pFile, pFile->bBlockOnConnect); +#else + res = winGetReadLock(pFile, 0); +#endif if( res ){ newLocktype = SHARED_LOCK; }else{ @@ -3350,7 +3641,7 @@ static int winLock(sqlite3_file *id, int locktype){ newLocktype = EXCLUSIVE_LOCK; }else{ lastErrno = osGetLastError(); - winGetReadLock(pFile); + winGetReadLock(pFile, 0); } } @@ -3430,7 +3721,7 @@ static int winUnlock(sqlite3_file *id, int locktype){ type = pFile->locktype; if( type>=EXCLUSIVE_LOCK ){ winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); - if( locktype==SHARED_LOCK && !winGetReadLock(pFile) ){ + if( locktype==SHARED_LOCK && !winGetReadLock(pFile, 0) ){ /* This should never happen. We should always be able to ** reacquire the read lock */ rc = winLogError(SQLITE_IOERR_UNLOCK, osGetLastError(), @@ -3640,6 +3931,28 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ return rc; } #endif + +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + case SQLITE_FCNTL_LOCK_TIMEOUT: { + int iOld = pFile->iBusyTimeout; + int iNew = *(int*)pArg; +#if SQLITE_ENABLE_SETLK_TIMEOUT==1 + pFile->iBusyTimeout = (iNew < 0) ? INFINITE : (DWORD)iNew; +#elif SQLITE_ENABLE_SETLK_TIMEOUT==2 + pFile->iBusyTimeout = (DWORD)(!!iNew); +#else +# error "SQLITE_ENABLE_SETLK_TIMEOUT must be set to 1 or 2" +#endif + *(int*)pArg = iOld; + return SQLITE_OK; + } + case SQLITE_FCNTL_BLOCK_ON_CONNECT: { + int iNew = *(int*)pArg; + pFile->bBlockOnConnect = iNew; + return SQLITE_OK; + } +#endif /* SQLITE_ENABLE_SETLK_TIMEOUT */ + } OSTRACE(("FCNTL file=%p, rc=SQLITE_NOTFOUND\n", pFile->h)); return SQLITE_NOTFOUND; @@ -3720,23 +4033,27 @@ static int winShmMutexHeld(void) { ** ** The following fields are read-only after the object is created: ** -** fid ** zFilename ** ** Either winShmNode.mutex must be held or winShmNode.nRef==0 and ** winShmMutexHeld() is true when reading or writing any other field ** in this structure. ** +** File-handle hSharedShm is used to (a) take the DMS lock, (b) truncate +** the *-shm file if the DMS-locking protocol demands it, and (c) map +** regions of the *-shm file into memory using MapViewOfFile() or +** similar. Other locks are taken by individual clients using the +** winShm.hShm handles. */ struct winShmNode { sqlite3_mutex *mutex; /* Mutex to access this object */ char *zFilename; /* Name of the file */ - winFile hFile; /* File handle from winOpen */ + HANDLE hSharedShm; /* File handle open on zFilename */ + int isUnlocked; /* DMS lock has not yet been obtained */ + int isReadonly; /* True if read-only */ int szRegion; /* Size of shared-memory regions */ int nRegion; /* Size of array apRegion */ - u8 isReadonly; /* True if read-only */ - u8 isUnlocked; /* True if no DMS lock held */ struct ShmRegion { HANDLE hMap; /* File handle from CreateFileMapping */ @@ -3745,7 +4062,6 @@ struct winShmNode { DWORD lastErrno; /* The Windows errno from the last I/O error */ int nRef; /* Number of winShm objects pointing to this */ - winShm *pFirst; /* All winShm objects pointing to this */ winShmNode *pNext; /* Next in list of all winShmNode objects */ #if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) u8 nextShmId; /* Next available winShm.id value */ @@ -3761,23 +4077,15 @@ static winShmNode *winShmNodeList = 0; /* ** Structure used internally by this VFS to record the state of an -** open shared memory connection. -** -** The following fields are initialized when this object is created and -** are read-only thereafter: -** -** winShm.pShmNode -** winShm.id -** -** All other fields are read/write. The winShm.pShmNode->mutex must be held -** while accessing any read/write fields. +** open shared memory connection. There is one such structure for each +** winFile open on a wal mode database. */ struct winShm { winShmNode *pShmNode; /* The underlying winShmNode object */ - winShm *pNext; /* Next winShm with the same winShmNode */ - u8 hasMutex; /* True if holding the winShmNode mutex */ u16 sharedMask; /* Mask of shared locks held */ u16 exclMask; /* Mask of exclusive locks held */ + HANDLE hShm; /* File-handle on *-shm file. For locking. */ + int bReadonly; /* True if hShm is opened read-only */ #if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) u8 id; /* Id of this connection with its winShmNode */ #endif @@ -3789,50 +4097,6 @@ struct winShm { #define WIN_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ #define WIN_SHM_DMS (WIN_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ -/* -** Apply advisory locks for all n bytes beginning at ofst. -*/ -#define WINSHM_UNLCK 1 -#define WINSHM_RDLCK 2 -#define WINSHM_WRLCK 3 -static int winShmSystemLock( - winShmNode *pFile, /* Apply locks to this open shared-memory segment */ - int lockType, /* WINSHM_UNLCK, WINSHM_RDLCK, or WINSHM_WRLCK */ - int ofst, /* Offset to first byte to be locked/unlocked */ - int nByte /* Number of bytes to lock or unlock */ -){ - int rc = 0; /* Result code form Lock/UnlockFileEx() */ - - /* Access to the winShmNode object is serialized by the caller */ - assert( pFile->nRef==0 || sqlite3_mutex_held(pFile->mutex) ); - - OSTRACE(("SHM-LOCK file=%p, lock=%d, offset=%d, size=%d\n", - pFile->hFile.h, lockType, ofst, nByte)); - - /* Release/Acquire the system-level lock */ - if( lockType==WINSHM_UNLCK ){ - rc = winUnlockFile(&pFile->hFile.h, ofst, 0, nByte, 0); - }else{ - /* Initialize the locking parameters */ - DWORD dwFlags = LOCKFILE_FAIL_IMMEDIATELY; - if( lockType == WINSHM_WRLCK ) dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; - rc = winLockFile(&pFile->hFile.h, dwFlags, ofst, 0, nByte, 0); - } - - if( rc!= 0 ){ - rc = SQLITE_OK; - }else{ - pFile->lastErrno = osGetLastError(); - rc = SQLITE_BUSY; - } - - OSTRACE(("SHM-LOCK file=%p, func=%s, errno=%lu, rc=%s\n", - pFile->hFile.h, (lockType == WINSHM_UNLCK) ? "winUnlockFile" : - "winLockFile", pFile->lastErrno, sqlite3ErrName(rc))); - - return rc; -} - /* Forward references to VFS methods */ static int winOpen(sqlite3_vfs*,const char*,sqlite3_file*,int,int*); static int winDelete(sqlite3_vfs *,const char*,int); @@ -3864,11 +4128,7 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); UNUSED_VARIABLE_VALUE(bRc); } - if( p->hFile.h!=NULL && p->hFile.h!=INVALID_HANDLE_VALUE ){ - SimulateIOErrorBenign(1); - winClose((sqlite3_file *)&p->hFile); - SimulateIOErrorBenign(0); - } + winHandleClose(p->hSharedShm); if( deleteFlag ){ SimulateIOErrorBenign(1); sqlite3BeginBenignMalloc(); @@ -3886,42 +4146,239 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ } /* -** The DMS lock has not yet been taken on shm file pShmNode. Attempt to -** take it now. Return SQLITE_OK if successful, or an SQLite error -** code otherwise. -** -** If the DMS cannot be locked because this is a readonly_shm=1 -** connection and no other process already holds a lock, return -** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1. +** The DMS lock has not yet been taken on the shm file associated with +** pShmNode. Take the lock. Truncate the *-shm file if required. +** Return SQLITE_OK if successful, or an SQLite error code otherwise. */ -static int winLockSharedMemory(winShmNode *pShmNode){ - int rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1); +static int winLockSharedMemory(winShmNode *pShmNode, DWORD nMs){ + HANDLE h = pShmNode->hSharedShm; + int rc = SQLITE_OK; + assert( sqlite3_mutex_held(pShmNode->mutex) ); + rc = winHandleLockTimeout(h, WIN_SHM_DMS, 1, 1, 0); if( rc==SQLITE_OK ){ + /* We have an EXCLUSIVE lock on the DMS byte. This means that this + ** is the first process to open the file. Truncate it to zero bytes + ** in this case. */ if( pShmNode->isReadonly ){ - pShmNode->isUnlocked = 1; - winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); - return SQLITE_READONLY_CANTINIT; - }else if( winTruncate((sqlite3_file*)&pShmNode->hFile, 0) ){ - winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); - return winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), - "winLockSharedMemory", pShmNode->zFilename); + rc = SQLITE_READONLY_CANTINIT; + }else{ + rc = winHandleTruncate(h, 0); } + + /* Release the EXCLUSIVE lock acquired above. */ + winUnlockFile(&h, WIN_SHM_DMS, 0, 1, 0); + }else if( (rc & 0xFF)==SQLITE_BUSY ){ + rc = SQLITE_OK; } if( rc==SQLITE_OK ){ - winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + /* Take a SHARED lock on the DMS byte. */ + rc = winHandleLockTimeout(h, WIN_SHM_DMS, 1, 0, nMs); + if( rc==SQLITE_OK ){ + pShmNode->isUnlocked = 0; + } } - return winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); + return rc; } + /* -** Open the shared-memory area associated with database file pDbFd. +** Convert a UTF-8 filename into whatever form the underlying +** operating system wants filenames in. Space to hold the result +** is obtained from malloc and must be freed by the calling +** function +** +** On Cygwin, 3 possible input forms are accepted: +** - If the filename starts with ":/" or ":\", +** it is converted to UTF-16 as-is. +** - If the filename contains '/', it is assumed to be a +** Cygwin absolute path, it is converted to a win32 +** absolute path in UTF-16. +** - Otherwise it must be a filename only, the win32 filename +** is returned in UTF-16. +** Note: If the function cygwin_conv_path() fails, only +** UTF-8 -> UTF-16 conversion will be done. This can only +** happen when the file path >32k, in which case winUtf8ToUnicode() +** will fail too. +*/ +static void *winConvertFromUtf8Filename(const char *zFilename){ + void *zConverted = 0; + if( osIsNT() ){ +#ifdef __CYGWIN__ + int nChar; + LPWSTR zWideFilename; + + if( osCygwin_conv_path && !(winIsDriveLetterAndColon(zFilename) + && winIsDirSep(zFilename[2])) ){ + i64 nByte; + int convertflag = CCP_POSIX_TO_WIN_W; + if( !strchr(zFilename, '/') ) convertflag |= CCP_RELATIVE; + nByte = (i64)osCygwin_conv_path(convertflag, + zFilename, 0, 0); + if( nByte>0 ){ + zConverted = sqlite3MallocZero(12+(u64)nByte); + if ( zConverted==0 ){ + return zConverted; + } + zWideFilename = zConverted; + /* Filenames should be prefixed, except when converted + * full path already starts with "\\?\". */ + if( osCygwin_conv_path(convertflag, zFilename, + zWideFilename+4, nByte)==0 ){ + if( (convertflag&CCP_RELATIVE) ){ + memmove(zWideFilename, zWideFilename+4, nByte); + }else if( memcmp(zWideFilename+4, L"\\\\", 4) ){ + memcpy(zWideFilename, L"\\\\?\\", 8); + }else if( zWideFilename[6]!='?' ){ + memmove(zWideFilename+6, zWideFilename+4, nByte); + memcpy(zWideFilename, L"\\\\?\\UNC", 14); + }else{ + memmove(zWideFilename, zWideFilename+4, nByte); + } + return zConverted; + } + sqlite3_free(zConverted); + } + } + nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0); + if( nChar==0 ){ + return 0; + } + zWideFilename = sqlite3MallocZero( nChar*sizeof(WCHAR)+12 ); + if( zWideFilename==0 ){ + return 0; + } + nChar = osMultiByteToWideChar(CP_UTF8, 0, zFilename, -1, + zWideFilename, nChar); + if( nChar==0 ){ + sqlite3_free(zWideFilename); + zWideFilename = 0; + }else if( nChar>MAX_PATH + && winIsDriveLetterAndColon(zFilename) + && winIsDirSep(zFilename[2]) ){ + memmove(zWideFilename+4, zWideFilename, nChar*sizeof(WCHAR)); + zWideFilename[2] = '\\'; + memcpy(zWideFilename, L"\\\\?\\", 8); + }else if( nChar>MAX_PATH + && winIsDirSep(zFilename[0]) && winIsDirSep(zFilename[1]) + && zFilename[2] != '?' ){ + memmove(zWideFilename+6, zWideFilename, nChar*sizeof(WCHAR)); + memcpy(zWideFilename, L"\\\\?\\UNC", 14); + } + zConverted = zWideFilename; +#else + zConverted = winUtf8ToUnicode(zFilename); +#endif /* __CYGWIN__ */ + } +#if defined(SQLITE_WIN32_HAS_ANSI) && defined(_WIN32) + else{ + zConverted = winUtf8ToMbcs(zFilename, osAreFileApisANSI()); + } +#endif + /* caller will handle out of memory */ + return zConverted; +} + +/* +** This function is used to open a handle on a *-shm file. ** -** When opening a new shared-memory file, if no other instances of that -** file are currently open, in this process or in other processes, then -** the file must be truncated to zero length or have its header cleared. +** If SQLITE_ENABLE_SETLK_TIMEOUT is defined at build time, then the file +** is opened with FILE_FLAG_OVERLAPPED specified. If not, it is not. +*/ +static int winHandleOpen( + const char *zUtf8, /* File to open */ + int *pbReadonly, /* IN/OUT: True for readonly handle */ + HANDLE *ph /* OUT: New HANDLE for file */ +){ + int rc = SQLITE_OK; + void *zConverted = 0; + int bReadonly = *pbReadonly; + HANDLE h = INVALID_HANDLE_VALUE; + +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + const DWORD flag_overlapped = FILE_FLAG_OVERLAPPED; +#else + const DWORD flag_overlapped = 0; +#endif + + /* Convert the filename to the system encoding. */ + zConverted = winConvertFromUtf8Filename(zUtf8); + if( zConverted==0 ){ + OSTRACE(("OPEN name=%s, rc=SQLITE_IOERR_NOMEM", zUtf8)); + rc = SQLITE_IOERR_NOMEM_BKPT; + goto winopenfile_out; + } + + /* Ensure the file we are trying to open is not actually a directory. */ + if( winIsDir(zConverted) ){ + OSTRACE(("OPEN name=%s, rc=SQLITE_CANTOPEN_ISDIR", zUtf8)); + rc = SQLITE_CANTOPEN_ISDIR; + goto winopenfile_out; + } + + /* TODO: platforms. + ** TODO: retry-on-ioerr. + */ + if( osIsNT() ){ +#if SQLITE_OS_WINRT + CREATEFILE2_EXTENDED_PARAMETERS extendedParameters; + memset(&extendedParameters, 0, sizeof(extendedParameters)); + extendedParameters.dwSize = sizeof(extendedParameters); + extendedParameters.dwFileAttributes = FILE_ATTRIBUTE_NORMAL; + extendedParameters.dwFileFlags = flag_overlapped; + extendedParameters.dwSecurityQosFlags = SECURITY_ANONYMOUS; + h = osCreateFile2((LPCWSTR)zConverted, + (GENERIC_READ | (bReadonly ? 0 : GENERIC_WRITE)),/* dwDesiredAccess */ + FILE_SHARE_READ | FILE_SHARE_WRITE, /* dwShareMode */ + OPEN_ALWAYS, /* dwCreationDisposition */ + &extendedParameters + ); +#else + h = osCreateFileW((LPCWSTR)zConverted, /* lpFileName */ + (GENERIC_READ | (bReadonly ? 0 : GENERIC_WRITE)), /* dwDesiredAccess */ + FILE_SHARE_READ | FILE_SHARE_WRITE, /* dwShareMode */ + NULL, /* lpSecurityAttributes */ + OPEN_ALWAYS, /* dwCreationDisposition */ + FILE_ATTRIBUTE_NORMAL|flag_overlapped, + NULL + ); +#endif + }else{ + /* Due to pre-processor directives earlier in this file, + ** SQLITE_WIN32_HAS_ANSI is always defined if osIsNT() is false. */ +#ifdef SQLITE_WIN32_HAS_ANSI + h = osCreateFileA((LPCSTR)zConverted, + (GENERIC_READ | (bReadonly ? 0 : GENERIC_WRITE)), /* dwDesiredAccess */ + FILE_SHARE_READ | FILE_SHARE_WRITE, /* dwShareMode */ + NULL, /* lpSecurityAttributes */ + OPEN_ALWAYS, /* dwCreationDisposition */ + FILE_ATTRIBUTE_NORMAL|flag_overlapped, + NULL + ); +#endif + } + + if( h==INVALID_HANDLE_VALUE ){ + if( bReadonly==0 ){ + bReadonly = 1; + rc = winHandleOpen(zUtf8, &bReadonly, &h); + }else{ + rc = SQLITE_CANTOPEN_BKPT; + } + } + + winopenfile_out: + sqlite3_free(zConverted); + *pbReadonly = bReadonly; + *ph = h; + return rc; +} + + +/* +** Open the shared-memory area associated with database file pDbFd. */ static int winOpenSharedMemory(winFile *pDbFd){ struct winShm *p; /* The connection to be opened */ @@ -3933,98 +4390,83 @@ static int winOpenSharedMemory(winFile *pDbFd){ assert( pDbFd->pShm==0 ); /* Not previously opened */ /* Allocate space for the new sqlite3_shm object. Also speculatively - ** allocate space for a new winShmNode and filename. - */ + ** allocate space for a new winShmNode and filename. */ p = sqlite3MallocZero( sizeof(*p) ); if( p==0 ) return SQLITE_IOERR_NOMEM_BKPT; nName = sqlite3Strlen30(pDbFd->zPath); - pNew = sqlite3MallocZero( sizeof(*pShmNode) + nName + 17 ); + pNew = sqlite3MallocZero( sizeof(*pShmNode) + (i64)nName + 17 ); if( pNew==0 ){ sqlite3_free(p); return SQLITE_IOERR_NOMEM_BKPT; } pNew->zFilename = (char*)&pNew[1]; + pNew->hSharedShm = INVALID_HANDLE_VALUE; + pNew->isUnlocked = 1; sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); sqlite3FileSuffix3(pDbFd->zPath, pNew->zFilename); + /* Open a file-handle on the *-shm file for this connection. This file-handle + ** is only used for locking. The mapping of the *-shm file is created using + ** the shared file handle in winShmNode.hSharedShm. */ + p->bReadonly = sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0); + rc = winHandleOpen(pNew->zFilename, &p->bReadonly, &p->hShm); + /* Look to see if there is an existing winShmNode that can be used. - ** If no matching winShmNode currently exists, create a new one. - */ + ** If no matching winShmNode currently exists, then create a new one. */ winShmEnterMutex(); for(pShmNode = winShmNodeList; pShmNode; pShmNode=pShmNode->pNext){ /* TBD need to come up with better match here. Perhaps - ** use FILE_ID_BOTH_DIR_INFO Structure. - */ + ** use FILE_ID_BOTH_DIR_INFO Structure. */ if( sqlite3StrICmp(pShmNode->zFilename, pNew->zFilename)==0 ) break; } - if( pShmNode ){ - sqlite3_free(pNew); - }else{ - int inFlags = SQLITE_OPEN_WAL; - int outFlags = 0; - + if( pShmNode==0 ){ pShmNode = pNew; - pNew = 0; - ((winFile*)(&pShmNode->hFile))->h = INVALID_HANDLE_VALUE; - pShmNode->pNext = winShmNodeList; - winShmNodeList = pShmNode; + /* Allocate a mutex for this winShmNode object, if one is required. */ if( sqlite3GlobalConfig.bCoreMutex ){ pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutex==0 ){ - rc = SQLITE_IOERR_NOMEM_BKPT; - goto shm_open_err; - } + if( pShmNode->mutex==0 ) rc = SQLITE_IOERR_NOMEM_BKPT; } - if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ - inFlags |= SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; - }else{ - inFlags |= SQLITE_OPEN_READONLY; - } - rc = winOpen(pDbFd->pVfs, pShmNode->zFilename, - (sqlite3_file*)&pShmNode->hFile, - inFlags, &outFlags); - if( rc!=SQLITE_OK ){ - rc = winLogError(rc, osGetLastError(), "winOpenShm", - pShmNode->zFilename); - goto shm_open_err; + /* Open a file-handle to use for mappings, and for the DMS lock. */ + if( rc==SQLITE_OK ){ + HANDLE h = INVALID_HANDLE_VALUE; + pShmNode->isReadonly = p->bReadonly; + rc = winHandleOpen(pNew->zFilename, &pShmNode->isReadonly, &h); + pShmNode->hSharedShm = h; } - if( outFlags==SQLITE_OPEN_READONLY ) pShmNode->isReadonly = 1; - rc = winLockSharedMemory(pShmNode); - if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err; + /* If successful, link the new winShmNode into the global list. If an + ** error occurred, free the object. */ + if( rc==SQLITE_OK ){ + pShmNode->pNext = winShmNodeList; + winShmNodeList = pShmNode; + pNew = 0; + }else{ + sqlite3_mutex_free(pShmNode->mutex); + if( pShmNode->hSharedShm!=INVALID_HANDLE_VALUE ){ + osCloseHandle(pShmNode->hSharedShm); + } + } } - /* Make the new connection a child of the winShmNode */ - p->pShmNode = pShmNode; + /* If no error has occurred, link the winShm object to the winShmNode and + ** the winShm to pDbFd. */ + if( rc==SQLITE_OK ){ + p->pShmNode = pShmNode; + pShmNode->nRef++; #if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) - p->id = pShmNode->nextShmId++; + p->id = pShmNode->nextShmId++; #endif - pShmNode->nRef++; - pDbFd->pShm = p; - winShmLeaveMutex(); - - /* The reference count on pShmNode has already been incremented under - ** the cover of the winShmEnterMutex() mutex and the pointer from the - ** new (struct winShm) object to the pShmNode has been set. All that is - ** left to do is to link the new object into the linked list starting - ** at pShmNode->pFirst. This must be done while holding the pShmNode->mutex - ** mutex. - */ - sqlite3_mutex_enter(pShmNode->mutex); - p->pNext = pShmNode->pFirst; - pShmNode->pFirst = p; - sqlite3_mutex_leave(pShmNode->mutex); - return rc; + pDbFd->pShm = p; + }else if( p ){ + winHandleClose(p->hShm); + sqlite3_free(p); + } - /* Jump here on any error */ -shm_open_err: - winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); - winShmPurge(pDbFd->pVfs, 0); /* This call frees pShmNode if required */ - sqlite3_free(p); - sqlite3_free(pNew); + assert( rc!=SQLITE_OK || pShmNode->isUnlocked==0 || pShmNode->nRegion==0 ); winShmLeaveMutex(); + sqlite3_free(pNew); return rc; } @@ -4039,27 +4481,19 @@ static int winShmUnmap( winFile *pDbFd; /* Database holding shared-memory */ winShm *p; /* The connection to be closed */ winShmNode *pShmNode; /* The underlying shared-memory file */ - winShm **pp; /* For looping over sibling connections */ pDbFd = (winFile*)fd; p = pDbFd->pShm; if( p==0 ) return SQLITE_OK; - pShmNode = p->pShmNode; - - /* Remove connection p from the set of connections associated - ** with pShmNode */ - sqlite3_mutex_enter(pShmNode->mutex); - for(pp=&pShmNode->pFirst; (*pp)!=p; pp = &(*pp)->pNext){} - *pp = p->pNext; + if( p->hShm!=INVALID_HANDLE_VALUE ){ + osCloseHandle(p->hShm); + } - /* Free the connection p */ - sqlite3_free(p); - pDbFd->pShm = 0; - sqlite3_mutex_leave(pShmNode->mutex); + pShmNode = p->pShmNode; + winShmEnterMutex(); /* If pShmNode->nRef has reached 0, then close the underlying - ** shared-memory file, too */ - winShmEnterMutex(); + ** shared-memory file, too. */ assert( pShmNode->nRef>0 ); pShmNode->nRef--; if( pShmNode->nRef==0 ){ @@ -4067,6 +4501,9 @@ static int winShmUnmap( } winShmLeaveMutex(); + /* Free the connection p */ + sqlite3_free(p); + pDbFd->pShm = 0; return SQLITE_OK; } @@ -4081,10 +4518,9 @@ static int winShmLock( ){ winFile *pDbFd = (winFile*)fd; /* Connection holding shared memory */ winShm *p = pDbFd->pShm; /* The shared memory being locked */ - winShm *pX; /* For looping over all siblings */ winShmNode *pShmNode; int rc = SQLITE_OK; /* Result code */ - u16 mask; /* Mask of locks to take or release */ + u16 mask = (u16)((1U<<(ofst+n)) - (1U<pShmNode; @@ -4098,85 +4534,81 @@ static int winShmLock( || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); - mask = (u16)((1U<<(ofst+n)) - (1U<1 || mask==(1<mutex); - if( flags & SQLITE_SHM_UNLOCK ){ - u16 allMask = 0; /* Mask of locks held by siblings */ - - /* See if any siblings hold this same lock */ - for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ - if( pX==p ) continue; - assert( (pX->exclMask & (p->exclMask|p->sharedMask))==0 ); - allMask |= pX->sharedMask; - } + /* Check that, if this to be a blocking lock, no locks that occur later + ** in the following list than the lock being obtained are already held: + ** + ** 1. Recovery lock (ofst==2). + ** 2. Checkpointer lock (ofst==1). + ** 3. Write lock (ofst==0). + ** 4. Read locks (ofst>=3 && ofstexclMask|p->sharedMask); + assert( (flags & SQLITE_SHM_UNLOCK) || pDbFd->iBusyTimeout==0 || ( + (ofst!=2 || lockMask==0) + && (ofst!=1 || lockMask==0 || lockMask==2) + && (ofst!=0 || lockMask<3) + && (ofst<3 || lockMask<(1<exclMask & mask) + ); + if( ((flags & SQLITE_SHM_UNLOCK) && ((p->exclMask|p->sharedMask) & mask)) + || (flags==(SQLITE_SHM_SHARED|SQLITE_SHM_LOCK) && 0==(p->sharedMask & mask)) + || (flags==(SQLITE_SHM_EXCLUSIVE|SQLITE_SHM_LOCK)) + ){ - /* Undo the local locks */ - if( rc==SQLITE_OK ){ - p->exclMask &= ~mask; - p->sharedMask &= ~mask; - } - }else if( flags & SQLITE_SHM_SHARED ){ - u16 allShared = 0; /* Union of locks held by connections other than "p" */ + if( flags & SQLITE_SHM_UNLOCK ){ + /* Case (a) - unlock. */ - /* Find out which shared locks are already held by sibling connections. - ** If any sibling already holds an exclusive lock, go ahead and return - ** SQLITE_BUSY. - */ - for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ - if( (pX->exclMask & mask)!=0 ){ - rc = SQLITE_BUSY; - break; - } - allShared |= pX->sharedMask; - } + assert( (p->exclMask & p->sharedMask)==0 ); + assert( !(flags & SQLITE_SHM_EXCLUSIVE) || (p->exclMask & mask)==mask ); + assert( !(flags & SQLITE_SHM_SHARED) || (p->sharedMask & mask)==mask ); - /* Get shared locks at the system level, if necessary */ - if( rc==SQLITE_OK ){ - if( (allShared & mask)==0 ){ - rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, ofst+WIN_SHM_BASE, n); - }else{ - rc = SQLITE_OK; - } - } + rc = winHandleUnlock(p->hShm, ofst+WIN_SHM_BASE, n); - /* Get the local shared locks */ - if( rc==SQLITE_OK ){ - p->sharedMask |= mask; - } - }else{ - /* Make sure no sibling connections hold locks that will block this - ** lock. If any do, return SQLITE_BUSY right away. - */ - for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ - if( (pX->exclMask & mask)!=0 || (pX->sharedMask & mask)!=0 ){ - rc = SQLITE_BUSY; - break; + /* If successful, also clear the bits in sharedMask/exclMask */ + if( rc==SQLITE_OK ){ + p->exclMask = (p->exclMask & ~mask); + p->sharedMask = (p->sharedMask & ~mask); } - } - - /* Get the exclusive locks at the system level. Then if successful - ** also mark the local connection as being locked. - */ - if( rc==SQLITE_OK ){ - rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, ofst+WIN_SHM_BASE, n); + }else{ + int bExcl = ((flags & SQLITE_SHM_EXCLUSIVE) ? 1 : 0); + DWORD nMs = winFileBusyTimeout(pDbFd); + rc = winHandleLockTimeout(p->hShm, ofst+WIN_SHM_BASE, n, bExcl, nMs); if( rc==SQLITE_OK ){ - assert( (p->sharedMask & mask)==0 ); - p->exclMask |= mask; + if( bExcl ){ + p->exclMask = (p->exclMask | mask); + }else{ + p->sharedMask = (p->sharedMask | mask); + } } } } - sqlite3_mutex_leave(pShmNode->mutex); - OSTRACE(("SHM-LOCK pid=%lu, id=%d, sharedMask=%03x, exclMask=%03x, rc=%s\n", - osGetCurrentProcessId(), p->id, p->sharedMask, p->exclMask, - sqlite3ErrName(rc))); + + OSTRACE(( + "SHM-LOCK(%d,%d,%d) pid=%lu, id=%d, sharedMask=%03x, exclMask=%03x," + " rc=%s\n", + ofst, n, flags, + osGetCurrentProcessId(), p->id, p->sharedMask, p->exclMask, + sqlite3ErrName(rc)) + ); return rc; } @@ -4238,13 +4670,15 @@ static int winShmMap( sqlite3_mutex_enter(pShmNode->mutex); if( pShmNode->isUnlocked ){ - rc = winLockSharedMemory(pShmNode); + /* Take the DMS lock. */ + assert( pShmNode->nRegion==0 ); + rc = winLockSharedMemory(pShmNode, winFileBusyTimeout(pDbFd)); if( rc!=SQLITE_OK ) goto shmpage_out; - pShmNode->isUnlocked = 0; } - assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); + assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); if( pShmNode->nRegion<=iRegion ){ + HANDLE hShared = pShmNode->hSharedShm; struct ShmRegion *apNew; /* New aRegion[] array */ int nByte = (iRegion+1)*szRegion; /* Minimum required file size */ sqlite3_int64 sz; /* Current size of wal-index file */ @@ -4255,10 +4689,9 @@ static int winShmMap( ** Check to see if it has been allocated (i.e. if the wal-index file is ** large enough to contain the requested region). */ - rc = winFileSize((sqlite3_file *)&pShmNode->hFile, &sz); + rc = winHandleSize(hShared, &sz); if( rc!=SQLITE_OK ){ - rc = winLogError(SQLITE_IOERR_SHMSIZE, osGetLastError(), - "winShmMap1", pDbFd->zPath); + rc = winLogError(rc, osGetLastError(), "winShmMap1", pDbFd->zPath); goto shmpage_out; } @@ -4267,19 +4700,17 @@ static int winShmMap( ** zero, exit early. *pp will be set to NULL and SQLITE_OK returned. ** ** Alternatively, if isWrite is non-zero, use ftruncate() to allocate - ** the requested memory region. - */ + ** the requested memory region. */ if( !isWrite ) goto shmpage_out; - rc = winTruncate((sqlite3_file *)&pShmNode->hFile, nByte); + rc = winHandleTruncate(hShared, nByte); if( rc!=SQLITE_OK ){ - rc = winLogError(SQLITE_IOERR_SHMSIZE, osGetLastError(), - "winShmMap2", pDbFd->zPath); + rc = winLogError(rc, osGetLastError(), "winShmMap2", pDbFd->zPath); goto shmpage_out; } } /* Map the requested memory region into this processes address space. */ - apNew = (struct ShmRegion *)sqlite3_realloc64( + apNew = (struct ShmRegion*)sqlite3_realloc64( pShmNode->aRegion, (iRegion+1)*sizeof(apNew[0]) ); if( !apNew ){ @@ -4298,18 +4729,13 @@ static int winShmMap( void *pMap = 0; /* Mapped memory region */ #if SQLITE_OS_WINRT - hMap = osCreateFileMappingFromApp(pShmNode->hFile.h, - NULL, protect, nByte, NULL - ); + hMap = osCreateFileMappingFromApp(hShared, NULL, protect, nByte, NULL); #elif defined(SQLITE_WIN32_HAS_WIDE) - hMap = osCreateFileMappingW(pShmNode->hFile.h, - NULL, protect, 0, nByte, NULL - ); + hMap = osCreateFileMappingW(hShared, NULL, protect, 0, nByte, NULL); #elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA - hMap = osCreateFileMappingA(pShmNode->hFile.h, - NULL, protect, 0, nByte, NULL - ); + hMap = osCreateFileMappingA(hShared, NULL, protect, 0, nByte, NULL); #endif + OSTRACE(("SHM-MAP-CREATE pid=%lu, region=%d, size=%d, rc=%s\n", osGetCurrentProcessId(), pShmNode->nRegion, nByte, hMap ? "ok" : "failed")); @@ -4352,7 +4778,9 @@ static int winShmMap( }else{ *pp = 0; } - if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY; + if( pShmNode->isReadonly && rc==SQLITE_OK ){ + rc = SQLITE_READONLY; + } sqlite3_mutex_leave(pShmNode->mutex); return rc; } @@ -4672,47 +5100,6 @@ static winVfsAppData winNolockAppData = { ** sqlite3_vfs object. */ -#if defined(__CYGWIN__) -/* -** Convert a filename from whatever the underlying operating system -** supports for filenames into UTF-8. Space to hold the result is -** obtained from malloc and must be freed by the calling function. -*/ -static char *winConvertToUtf8Filename(const void *zFilename){ - char *zConverted = 0; - if( osIsNT() ){ - zConverted = winUnicodeToUtf8(zFilename); - } -#ifdef SQLITE_WIN32_HAS_ANSI - else{ - zConverted = winMbcsToUtf8(zFilename, osAreFileApisANSI()); - } -#endif - /* caller will handle out of memory */ - return zConverted; -} -#endif - -/* -** Convert a UTF-8 filename into whatever form the underlying -** operating system wants filenames in. Space to hold the result -** is obtained from malloc and must be freed by the calling -** function. -*/ -static void *winConvertFromUtf8Filename(const char *zFilename){ - void *zConverted = 0; - if( osIsNT() ){ - zConverted = winUtf8ToUnicode(zFilename); - } -#ifdef SQLITE_WIN32_HAS_ANSI - else{ - zConverted = winUtf8ToMbcs(zFilename, osAreFileApisANSI()); - } -#endif - /* caller will handle out of memory */ - return zConverted; -} - /* ** This function returns non-zero if the specified UTF-8 string buffer ** ends with a directory separator character or one was successfully @@ -4725,7 +5112,14 @@ static int winMakeEndInDirSep(int nBuf, char *zBuf){ if( winIsDirSep(zBuf[nLen-1]) ){ return 1; }else if( nLen+1mxPathname; nBuf = nMax + 2; + nMax = pVfs->mxPathname; + nBuf = 2 + (i64)nMax; zBuf = sqlite3MallocZero( nBuf ); if( !zBuf ){ OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); @@ -4802,7 +5197,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ } #if defined(__CYGWIN__) - else{ + else if( osGetenv!=NULL ){ static const char *azDirs[] = { 0, /* getenv("SQLITE_TMPDIR") */ 0, /* getenv("TMPDIR") */ @@ -4818,11 +5213,11 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ unsigned int i; const char *zDir = 0; - if( !azDirs[0] ) azDirs[0] = getenv("SQLITE_TMPDIR"); - if( !azDirs[1] ) azDirs[1] = getenv("TMPDIR"); - if( !azDirs[2] ) azDirs[2] = getenv("TMP"); - if( !azDirs[3] ) azDirs[3] = getenv("TEMP"); - if( !azDirs[4] ) azDirs[4] = getenv("USERPROFILE"); + if( !azDirs[0] ) azDirs[0] = osGetenv("SQLITE_TMPDIR"); + if( !azDirs[1] ) azDirs[1] = osGetenv("TMPDIR"); + if( !azDirs[2] ) azDirs[2] = osGetenv("TMP"); + if( !azDirs[3] ) azDirs[3] = osGetenv("TEMP"); + if( !azDirs[4] ) azDirs[4] = osGetenv("USERPROFILE"); for(i=0; inOut ){ + /* SQLite assumes that xFullPathname() nul-terminates the output buffer + ** even if it returns an error. */ + zOut[iOff] = '\0'; + return SQLITE_CANTOPEN_BKPT; + } + sqlite3_snprintf(nOut-iOff, &zOut[iOff], "%s", zPath); + return SQLITE_OK; +} +#endif /* __CYGWIN__ */ /* ** Turn a relative pathname into a full pathname. Write the full @@ -5605,8 +6049,8 @@ static int winFullPathnameNoMutex( int nFull, /* Size of output buffer in bytes */ char *zFull /* Output buffer */ ){ -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) - DWORD nByte; +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT + int nByte; void *zConverted; char *zOut; #endif @@ -5619,64 +6063,82 @@ static int winFullPathnameNoMutex( zRelative++; } -#if defined(__CYGWIN__) SimulateIOError( return SQLITE_ERROR ); - UNUSED_PARAMETER(nFull); - assert( nFull>=pVfs->mxPathname ); - if ( sqlite3_data_directory && !winIsVerbatimPathname(zRelative) ){ - /* - ** NOTE: We are dealing with a relative path name and the data - ** directory has been set. Therefore, use it as the basis - ** for converting the relative path name to an absolute - ** one by prepending the data directory and a slash. - */ - char *zOut = sqlite3MallocZero( pVfs->mxPathname+1 ); - if( !zOut ){ - return SQLITE_IOERR_NOMEM_BKPT; - } - if( cygwin_conv_path( - (osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A) | - CCP_RELATIVE, zRelative, zOut, pVfs->mxPathname+1)<0 ){ - sqlite3_free(zOut); - return winLogError(SQLITE_CANTOPEN_CONVPATH, (DWORD)errno, - "winFullPathname1", zRelative); - }else{ - char *zUtf8 = winConvertToUtf8Filename(zOut); - if( !zUtf8 ){ - sqlite3_free(zOut); - return SQLITE_IOERR_NOMEM_BKPT; - } - sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s%c%s", - sqlite3_data_directory, winGetDirSep(), zUtf8); - sqlite3_free(zUtf8); - sqlite3_free(zOut); - } - }else{ - char *zOut = sqlite3MallocZero( pVfs->mxPathname+1 ); - if( !zOut ){ - return SQLITE_IOERR_NOMEM_BKPT; - } - if( cygwin_conv_path( - (osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A), - zRelative, zOut, pVfs->mxPathname+1)<0 ){ - sqlite3_free(zOut); - return winLogError(SQLITE_CANTOPEN_CONVPATH, (DWORD)errno, - "winFullPathname2", zRelative); - }else{ - char *zUtf8 = winConvertToUtf8Filename(zOut); - if( !zUtf8 ){ - sqlite3_free(zOut); - return SQLITE_IOERR_NOMEM_BKPT; - } - sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zUtf8); - sqlite3_free(zUtf8); - sqlite3_free(zOut); + +#ifdef __CYGWIN__ + if( osGetcwd ){ + zFull[nFull-1] = '\0'; + if( !winIsDriveLetterAndColon(zRelative) || !winIsDirSep(zRelative[2]) ){ + int rc = SQLITE_OK; + int nLink = 1; /* Number of symbolic links followed so far */ + const char *zIn = zRelative; /* Input path for each iteration of loop */ + char *zDel = 0; + struct stat buf; + + UNUSED_PARAMETER(pVfs); + + do { + /* Call lstat() on path zIn. Set bLink to true if the path is a symbolic + ** link, or false otherwise. */ + int bLink = 0; + if( osLstat && osReadlink ) { + if( osLstat(zIn, &buf)!=0 ){ + int myErrno = osErrno; + if( myErrno!=ENOENT ){ + rc = winLogError(SQLITE_CANTOPEN_BKPT, (DWORD)myErrno, "lstat", zIn); + } + }else{ + bLink = ((buf.st_mode & 0170000) == 0120000); + } + + if( bLink ){ + if( zDel==0 ){ + zDel = sqlite3MallocZero(nFull); + if( zDel==0 ) rc = SQLITE_NOMEM; + }else if( ++nLink>SQLITE_MAX_SYMLINKS ){ + rc = SQLITE_CANTOPEN_BKPT; + } + + if( rc==SQLITE_OK ){ + nByte = osReadlink(zIn, zDel, nFull-1); + if( nByte ==(DWORD)-1 ){ + rc = winLogError(SQLITE_CANTOPEN_BKPT, (DWORD)osErrno, "readlink", zIn); + }else{ + if( zDel[0]!='/' ){ + int n; + for(n = sqlite3Strlen30(zIn); n>0 && zIn[n-1]!='/'; n--); + if( nByte+n+1>nFull ){ + rc = SQLITE_CANTOPEN_BKPT; + }else{ + memmove(&zDel[n], zDel, nByte+1); + memcpy(zDel, zIn, n); + nByte += n; + } + } + zDel[nByte] = '\0'; + } + } + + zIn = zDel; + } + } + + assert( rc!=SQLITE_OK || zIn!=zFull || zIn[0]=='/' ); + if( rc==SQLITE_OK && zIn!=zFull ){ + rc = mkFullPathname(zIn, zFull, nFull); + } + if( bLink==0 ) break; + zIn = zFull; + }while( rc==SQLITE_OK ); + + sqlite3_free(zDel); + winSimplifyName(zFull); + return rc; } } - return SQLITE_OK; -#endif +#endif /* __CYGWIN__ */ -#if (SQLITE_OS_WINCE || SQLITE_OS_WINRT) && !defined(__CYGWIN__) +#if (SQLITE_OS_WINCE || SQLITE_OS_WINRT) && defined(_WIN32) SimulateIOError( return SQLITE_ERROR ); /* WinCE has no concept of a relative pathname, or so I am told. */ /* WinRT has no way to convert a relative path to an absolute one. */ @@ -5695,7 +6157,8 @@ static int winFullPathnameNoMutex( return SQLITE_OK; #endif -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if defined(_WIN32) /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this ** function failing. This function could fail if, for example, the @@ -5713,6 +6176,7 @@ static int winFullPathnameNoMutex( sqlite3_data_directory, winGetDirSep(), zRelative); return SQLITE_OK; } +#endif zConverted = winConvertFromUtf8Filename(zRelative); if( zConverted==0 ){ return SQLITE_IOERR_NOMEM_BKPT; @@ -5751,13 +6215,12 @@ static int winFullPathnameNoMutex( return winLogError(SQLITE_CANTOPEN_FULLPATH, osGetLastError(), "winFullPathname3", zRelative); } - nByte += 3; - zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) ); + zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) + 3*sizeof(zTemp[0]) ); if( zTemp==0 ){ sqlite3_free(zConverted); return SQLITE_IOERR_NOMEM_BKPT; } - nByte = osGetFullPathNameA((char*)zConverted, nByte, zTemp, 0); + nByte = osGetFullPathNameA((char*)zConverted, nByte+3, zTemp, 0); if( nByte==0 ){ sqlite3_free(zConverted); sqlite3_free(zTemp); @@ -5770,7 +6233,26 @@ static int winFullPathnameNoMutex( } #endif if( zOut ){ +#ifdef __CYGWIN__ + if( memcmp(zOut, "\\\\?\\", 4) ){ + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zOut); + }else if( memcmp(zOut+4, "UNC\\", 4) ){ + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zOut+4); + }else{ + char *p = zOut+6; + *p = '\\'; + if( osGetcwd ){ + /* On Cygwin, UNC paths use forward slashes */ + while( *p ){ + if( *p=='\\' ) *p = '/'; + ++p; + } + } + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zOut+6); + } +#else sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zOut); +#endif /* __CYGWIN__ */ sqlite3_free(zOut); return SQLITE_OK; }else{ @@ -5800,25 +6282,8 @@ static int winFullPathname( */ static void *winDlOpen(sqlite3_vfs *pVfs, const char *zFilename){ HANDLE h; -#if defined(__CYGWIN__) - int nFull = pVfs->mxPathname+1; - char *zFull = sqlite3MallocZero( nFull ); - void *zConverted = 0; - if( zFull==0 ){ - OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); - return 0; - } - if( winFullPathname(pVfs, zFilename, nFull, zFull)!=SQLITE_OK ){ - sqlite3_free(zFull); - OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); - return 0; - } - zConverted = winConvertFromUtf8Filename(zFull); - sqlite3_free(zFull); -#else void *zConverted = winConvertFromUtf8Filename(zFilename); UNUSED_PARAMETER(pVfs); -#endif if( zConverted==0 ){ OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); return 0; @@ -6167,7 +6632,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==80 ); + assert( ArraySize(aSyscall)==89 ); /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); diff --git a/src/os_win.h b/src/os_win.h index 27714ed07..a0845f003 100644 --- a/src/os_win.h +++ b/src/os_win.h @@ -22,6 +22,8 @@ #ifdef __CYGWIN__ # include +# include /* amalgamator: dontcache */ +# include /* amalgamator: dontcache */ # include /* amalgamator: dontcache */ #endif diff --git a/src/pager.c b/src/pager.c index ecec892b4..1850ba37b 100644 --- a/src/pager.c +++ b/src/pager.c @@ -700,6 +700,9 @@ struct Pager { Wal *pWal; /* Write-ahead log used by "journal_mode=wal" */ char *zWal; /* File name for write-ahead log */ #endif +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + sqlite3 *dbWal; +#endif }; /* @@ -1291,7 +1294,7 @@ static void checkPage(PgHdr *pPg){ ** If an error occurs while reading from the journal file, an SQLite ** error code is returned. */ -static int readSuperJournal(sqlite3_file *pJrnl, char *zSuper, u32 nSuper){ +static int readSuperJournal(sqlite3_file *pJrnl, char *zSuper, u64 nSuper){ int rc; /* Return code */ u32 len; /* Length in bytes of super-journal name */ i64 szJ; /* Total size in bytes of journal file pJrnl */ @@ -1846,6 +1849,15 @@ static void pager_unlock(Pager *pPager){ if( pagerUseWal(pPager) ){ assert( !isOpen(pPager->jfd) ); + if( pPager->eState==PAGER_ERROR ){ + /* If an IO error occurs in wal.c while attempting to wrap the wal file, + ** then the Wal object may be holding a write-lock but no read-lock. + ** This call ensures that the write-lock is dropped as well. We cannot + ** have sqlite3WalEndReadTransaction() drop the write-lock, as it once + ** did, because this would break "BEGIN EXCLUSIVE" handling for + ** SQLITE_ENABLE_SETLK_TIMEOUT builds. */ + sqlite3WalEndWriteTransaction(pPager->pWal); + } sqlite3WalEndReadTransaction(pPager->pWal); pPager->eState = PAGER_OPEN; }else if( !pPager->exclusiveMode ){ @@ -2527,12 +2539,12 @@ static int pager_delsuper(Pager *pPager, const char *zSuper){ char *zJournal; /* Pointer to one journal within MJ file */ char *zSuperPtr; /* Space to hold super-journal filename */ char *zFree = 0; /* Free this buffer */ - int nSuperPtr; /* Amount of space allocated to zSuperPtr[] */ + i64 nSuperPtr; /* Amount of space allocated to zSuperPtr[] */ /* Allocate space for both the pJournal and pSuper file descriptors. ** If successful, open the super-journal file for reading. */ - pSuper = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile * 2); + pSuper = (sqlite3_file *)sqlite3MallocZero(2 * (i64)pVfs->szOsFile); if( !pSuper ){ rc = SQLITE_NOMEM_BKPT; pJournal = 0; @@ -2550,11 +2562,14 @@ static int pager_delsuper(Pager *pPager, const char *zSuper){ */ rc = sqlite3OsFileSize(pSuper, &nSuperJournal); if( rc!=SQLITE_OK ) goto delsuper_out; - nSuperPtr = pVfs->mxPathname+1; + nSuperPtr = 1 + (i64)pVfs->mxPathname; + assert( nSuperJournal>=0 && nSuperPtr>0 ); zFree = sqlite3Malloc(4 + nSuperJournal + nSuperPtr + 2); if( !zFree ){ rc = SQLITE_NOMEM_BKPT; goto delsuper_out; + }else{ + assert( nSuperJournal<=0x7fffffff ); } zFree[0] = zFree[1] = zFree[2] = zFree[3] = 0; zSuperJournal = &zFree[4]; @@ -2815,7 +2830,7 @@ static int pager_playback(Pager *pPager, int isHot){ ** for pageSize. */ zSuper = pPager->pTmpSpace; - rc = readSuperJournal(pPager->jfd, zSuper, pPager->pVfs->mxPathname+1); + rc = readSuperJournal(pPager->jfd, zSuper, 1+(i64)pPager->pVfs->mxPathname); if( rc==SQLITE_OK && zSuper[0] ){ rc = sqlite3OsAccess(pVfs, zSuper, SQLITE_ACCESS_EXISTS, &res); } @@ -2954,7 +2969,7 @@ static int pager_playback(Pager *pPager, int isHot){ ** which case it requires 4 0x00 bytes in memory immediately before ** the filename. */ zSuper = &pPager->pTmpSpace[4]; - rc = readSuperJournal(pPager->jfd, zSuper, pPager->pVfs->mxPathname+1); + rc = readSuperJournal(pPager->jfd, zSuper, 1+(i64)pPager->pVfs->mxPathname); testcase( rc!=SQLITE_OK ); } if( rc==SQLITE_OK @@ -4724,6 +4739,7 @@ int sqlite3PagerOpen( u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ const char *zUri = 0; /* URI args to copy */ int nUriByte = 1; /* Number of bytes of URI args at *zUri */ + /* Figure out how much space is required for each journal file-handle ** (there are two of them, the main journal and the sub-journal). */ @@ -4750,8 +4766,8 @@ int sqlite3PagerOpen( */ if( zFilename && zFilename[0] ){ const char *z; - nPathname = pVfs->mxPathname+1; - zPathname = sqlite3DbMallocRaw(0, nPathname*2); + nPathname = pVfs->mxPathname + 1; + zPathname = sqlite3DbMallocRaw(0, 2*(i64)nPathname); if( zPathname==0 ){ return SQLITE_NOMEM_BKPT; } @@ -4838,14 +4854,14 @@ int sqlite3PagerOpen( ROUND8(sizeof(*pPager)) + /* Pager structure */ ROUND8(pcacheSize) + /* PCache object */ ROUND8(pVfs->szOsFile) + /* The main db file */ - journalFileSize * 2 + /* The two journal files */ + (u64)journalFileSize * 2 + /* The two journal files */ SQLITE_PTRSIZE + /* Space to hold a pointer */ 4 + /* Database prefix */ - nPathname + 1 + /* database filename */ - nUriByte + /* query parameters */ - nPathname + 8 + 1 + /* Journal filename */ + (u64)nPathname + 1 + /* database filename */ + (u64)nUriByte + /* query parameters */ + (u64)nPathname + 8 + 1 + /* Journal filename */ #ifndef SQLITE_OMIT_WAL - nPathname + 4 + 1 + /* WAL filename */ + (u64)nPathname + 4 + 1 + /* WAL filename */ #endif 3 /* Terminator */ ); @@ -7568,6 +7584,11 @@ static int pagerOpenWal(Pager *pPager){ pPager->fd, pPager->zWal, pPager->exclusiveMode, pPager->journalSizeLimit, &pPager->pWal ); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( rc==SQLITE_OK ){ + sqlite3WalDb(pPager->pWal, pPager->dbWal); + } +#endif } pagerFixMaplimit(pPager); @@ -7687,6 +7708,7 @@ int sqlite3PagerWalWriteLock(Pager *pPager, int bLock){ ** blocking locks are required. */ void sqlite3PagerWalDb(Pager *pPager, sqlite3 *db){ + pPager->dbWal = db; if( pagerUseWal(pPager) ){ sqlite3WalDb(pPager->pWal, db); } diff --git a/src/parse.y b/src/parse.y index e9e2c62e6..a5691cad4 100644 --- a/src/parse.y +++ b/src/parse.y @@ -62,6 +62,11 @@ %include { #include "sqliteInt.h" +/* +** Verify that the pParse->isCreate field is set +*/ +#define ASSERT_IS_CREATE assert(pParse->isCreate) + /* ** Disable all error recovery processing in the parser push-down ** automaton. @@ -125,6 +130,10 @@ static void parserSyntaxError(Parse *pParse, Token *p){ static void disableLookaside(Parse *pParse){ sqlite3 *db = pParse->db; pParse->disableLookaside++; +#ifdef SQLITE_DEBUG + pParse->isCreate = 1; +#endif + memset(&pParse->u1.cr, 0, sizeof(pParse->u1.cr)); DisableLookaside; } @@ -197,7 +206,9 @@ cmd ::= create_table create_table_args. create_table ::= createkw temp(T) TABLE ifnotexists(E) nm(Y) dbnm(Z). { sqlite3StartTable(pParse,&Y,&Z,T,0,0,E); } -createkw(A) ::= CREATE(A). {disableLookaside(pParse);} +createkw(A) ::= CREATE(A). { + disableLookaside(pParse); +} %type ifnotexists {int} ifnotexists(A) ::= . {A = 0;} @@ -312,7 +323,7 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,A,Y);} // %token_class id ID|INDEXED. -// And "ids" is an identifer-or-string. +// And "ids" is an identifier-or-string. // %token_class ids ID|STRING. @@ -373,7 +384,7 @@ scantok(A) ::= . { // carglist ::= carglist ccons. carglist ::= . -ccons ::= CONSTRAINT nm(X). {pParse->constraintName = X;} +ccons ::= CONSTRAINT nm(X). {ASSERT_IS_CREATE; pParse->u1.cr.constraintName = X;} ccons ::= DEFAULT scantok(A) term(X). {sqlite3AddDefaultValue(pParse,X,A.z,&A.z[A.n]);} ccons ::= DEFAULT LP(A) expr(X) RP(Z). @@ -448,9 +459,9 @@ conslist_opt(A) ::= . {A.n = 0; A.z = 0;} conslist_opt(A) ::= COMMA(A) conslist. conslist ::= conslist tconscomma tcons. conslist ::= tcons. -tconscomma ::= COMMA. {pParse->constraintName.n = 0;} +tconscomma ::= COMMA. {ASSERT_IS_CREATE; pParse->u1.cr.constraintName.n = 0;} tconscomma ::= . -tcons ::= CONSTRAINT nm(X). {pParse->constraintName = X;} +tcons ::= CONSTRAINT nm(X). {ASSERT_IS_CREATE; pParse->u1.cr.constraintName = X;} tcons ::= PRIMARY KEY LP sortlist(X) autoinc(I) RP onconf(R). {sqlite3AddPrimaryKey(pParse,X,R,I,0);} tcons ::= UNIQUE LP sortlist(X) RP onconf(R). @@ -606,8 +617,8 @@ selectnowith(A) ::= selectnowith(A) multiselect_op(Y) oneselect(Z). { if( pRhs ){ pRhs->op = (u8)Y; pRhs->pPrior = pLhs; - if( ALWAYS(pLhs) ) pLhs->selFlags &= ~SF_MultiValue; - pRhs->selFlags &= ~SF_MultiValue; + if( ALWAYS(pLhs) ) pLhs->selFlags &= ~(u32)SF_MultiValue; + pRhs->selFlags &= ~(u32)SF_MultiValue; if( Y!=TK_ALL ) pParse->hasCompound = 1; }else{ sqlite3SelectDelete(pParse->db, pLhs); @@ -827,7 +838,7 @@ joinop(X) ::= JOIN_KW(A) nm(B) JOIN. joinop(X) ::= JOIN_KW(A) nm(B) nm(C) JOIN. {X = sqlite3JoinType(pParse,&A,&B,&C);/*X-overwrites-A*/} -// There is a parsing abiguity in an upsert statement that uses a +// There is a parsing ambiguity in an upsert statement that uses a // SELECT on the RHS of a the INSERT: // // INSERT INTO tab SELECT * FROM aaa JOIN bbb ON CONFLICT ... @@ -1208,7 +1219,7 @@ expr(A) ::= idj(X) LP STAR RP. { ** The purpose of this function is to generate an Expr node from the first syntax ** into a TK_FUNCTION node that looks like it came from the second syntax. ** - ** Only functions that have the SQLITE_SELFORDER1 perperty are allowed to do this + ** Only functions that have the SQLITE_SELFORDER1 property are allowed to do this ** transformation. Because DISTINCT is not allowed in the ordered-set aggregate ** syntax, an error is raised if DISTINCT is used. */ @@ -1659,6 +1670,10 @@ trigger_decl(A) ::= temp(T) TRIGGER ifnotexists(NOERR) nm(B) dbnm(Z) ON fullname(E) foreach_clause when_clause(G). { sqlite3BeginTrigger(pParse, &B, &Z, C, D.a, D.b, E, G, T, NOERR); A = (Z.n==0?B:Z); /*A-overwrites-T*/ +#ifdef SQLITE_DEBUG + assert( pParse->isCreate ); /* Set by createkw reduce action */ + pParse->isCreate = 0; /* But, should not be set for CREATE TRIGGER */ +#endif } %type trigger_time {int} diff --git a/src/pcache1.c b/src/pcache1.c index a0a8c7e28..39607328f 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -232,10 +232,6 @@ static SQLITE_WSD struct PCacheGlobal { sqlite3_mutex *mutex; /* Mutex for accessing the following: */ PgFreeslot *pFree; /* Free page blocks */ int nFreeSlot; /* Number of unused pcache slots */ - /* The following value requires a mutex to change. We skip the mutex on - ** reading because (1) most platforms read a 32-bit integer atomically and - ** (2) even if an incorrect value is read, no great harm is done since this - ** is really just an optimization. */ int bUnderPressure; /* True if low on PAGECACHE memory */ } pcache1_g; @@ -283,7 +279,7 @@ void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ pcache1.nReserve = n>90 ? 10 : (n/10 + 1); pcache1.pStart = pBuf; pcache1.pFree = 0; - pcache1.bUnderPressure = 0; + AtomicStore(&pcache1.bUnderPressure,0); while( n-- ){ p = (PgFreeslot*)pBuf; p->pNext = pcache1.pFree; @@ -351,7 +347,7 @@ static void *pcache1Alloc(int nByte){ if( p ){ pcache1.pFree = pcache1.pFree->pNext; pcache1.nFreeSlot--; - pcache1.bUnderPressure = pcache1.nFreeSlot=0 ); sqlite3StatusHighwater(SQLITE_STATUS_PAGECACHE_SIZE, nByte); sqlite3StatusUp(SQLITE_STATUS_PAGECACHE_USED, 1); @@ -390,7 +386,7 @@ static void pcache1Free(void *p){ pSlot->pNext = pcache1.pFree; pcache1.pFree = pSlot; pcache1.nFreeSlot++; - pcache1.bUnderPressure = pcache1.nFreeSlotszPage+pCache->szExtra)<=pcache1.szSlot ){ - return pcache1.bUnderPressure; + return AtomicLoad(&pcache1.bUnderPressure); }else{ return sqlite3HeapNearlyFull(); } @@ -538,12 +534,12 @@ static int pcache1UnderMemoryPressure(PCache1 *pCache){ */ static void pcache1ResizeHash(PCache1 *p){ PgHdr1 **apNew; - unsigned int nNew; - unsigned int i; + u64 nNew; + u32 i; assert( sqlite3_mutex_held(p->pGroup->mutex) ); - nNew = p->nHash*2; + nNew = 2*(u64)p->nHash; if( nNew<256 ){ nNew = 256; } @@ -766,7 +762,7 @@ static void pcache1Destroy(sqlite3_pcache *p); static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ PCache1 *pCache; /* The newly created page cache */ PGroup *pGroup; /* The group the new page cache will belong to */ - int sz; /* Bytes of memory required to allocate the new cache */ + i64 sz; /* Bytes of memory required to allocate the new cache */ assert( (szPage & (szPage-1))==0 && szPage>=512 && szPage<=65536 ); assert( szExtra < 300 ); diff --git a/src/pragma.c b/src/pragma.c index 291ccf7af..2b4d465e7 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -36,7 +36,7 @@ ** the following macro or to the actual analysis_limit if it is non-zero, ** in order to prevent PRAGMA optimize from running for too long. ** -** The value of 2000 is chosen emperically so that the worst-case run-time +** The value of 2000 is chosen empirically so that the worst-case run-time ** for PRAGMA optimize does not exceed 100 milliseconds against a variety ** of test databases on a RaspberryPI-4 compiled using -Os and without ** -DSQLITE_DEBUG. Of course, your mileage may vary. For the purpose of @@ -1153,7 +1153,10 @@ void sqlite3Pragma( } }else{ db->flags &= ~mask; - if( mask==SQLITE_DeferFKs ) db->nDeferredImmCons = 0; + if( mask==SQLITE_DeferFKs ){ + db->nDeferredImmCons = 0; + db->nDeferredCons = 0; + } if( (mask & SQLITE_WriteSchema)!=0 && sqlite3_stricmp(zRight, "reset")==0 ){ diff --git a/src/pragma.h b/src/pragma.h deleted file mode 100644 index 56a469b9f..000000000 --- a/src/pragma.h +++ /dev/null @@ -1,660 +0,0 @@ -/* DO NOT EDIT! -** This file is automatically generated by the script at -** ../tool/mkpragmatab.tcl. To update the set of pragmas, edit -** that script and rerun it. -*/ - -/* The various pragma types */ -#define PragTyp_ACTIVATE_EXTENSIONS 0 -#define PragTyp_ANALYSIS_LIMIT 1 -#define PragTyp_HEADER_VALUE 2 -#define PragTyp_AUTO_VACUUM 3 -#define PragTyp_FLAG 4 -#define PragTyp_BUSY_TIMEOUT 5 -#define PragTyp_CACHE_SIZE 6 -#define PragTyp_CACHE_SPILL 7 -#define PragTyp_CASE_SENSITIVE_LIKE 8 -#define PragTyp_COLLATION_LIST 9 -#define PragTyp_COMPILE_OPTIONS 10 -#define PragTyp_DATA_STORE_DIRECTORY 11 -#define PragTyp_DATABASE_LIST 12 -#define PragTyp_DEFAULT_CACHE_SIZE 13 -#define PragTyp_ENCODING 14 -#define PragTyp_FOREIGN_KEY_CHECK 15 -#define PragTyp_FOREIGN_KEY_LIST 16 -#define PragTyp_FUNCTION_LIST 17 -#define PragTyp_HARD_HEAP_LIMIT 18 -#define PragTyp_INCREMENTAL_VACUUM 19 -#define PragTyp_INDEX_INFO 20 -#define PragTyp_INDEX_LIST 21 -#define PragTyp_INTEGRITY_CHECK 22 -#define PragTyp_JOURNAL_MODE 23 -#define PragTyp_JOURNAL_SIZE_LIMIT 24 -#define PragTyp_LOCK_PROXY_FILE 25 -#define PragTyp_LOCKING_MODE 26 -#define PragTyp_PAGE_COUNT 27 -#define PragTyp_MMAP_SIZE 28 -#define PragTyp_MODULE_LIST 29 -#define PragTyp_OPTIMIZE 30 -#define PragTyp_PAGE_SIZE 31 -#define PragTyp_PRAGMA_LIST 32 -#define PragTyp_SECURE_DELETE 33 -#define PragTyp_SHRINK_MEMORY 34 -#define PragTyp_SOFT_HEAP_LIMIT 35 -#define PragTyp_SYNCHRONOUS 36 -#define PragTyp_TABLE_INFO 37 -#define PragTyp_TABLE_LIST 38 -#define PragTyp_TEMP_STORE 39 -#define PragTyp_TEMP_STORE_DIRECTORY 40 -#define PragTyp_THREADS 41 -#define PragTyp_WAL_AUTOCHECKPOINT 42 -#define PragTyp_WAL_CHECKPOINT 43 -#define PragTyp_LOCK_STATUS 44 -#define PragTyp_STATS 45 - -/* Property flags associated with various pragma. */ -#define PragFlg_NeedSchema 0x01 /* Force schema load before running */ -#define PragFlg_NoColumns 0x02 /* OP_ResultRow called with zero columns */ -#define PragFlg_NoColumns1 0x04 /* zero columns if RHS argument is present */ -#define PragFlg_ReadOnly 0x08 /* Read-only HEADER_VALUE */ -#define PragFlg_Result0 0x10 /* Acts as query when no argument */ -#define PragFlg_Result1 0x20 /* Acts as query when has one argument */ -#define PragFlg_SchemaOpt 0x40 /* Schema restricts name search if present */ -#define PragFlg_SchemaReq 0x80 /* Schema required - "main" is default */ - -/* Names of columns for pragmas that return multi-column result -** or that return single-column results where the name of the -** result column is different from the name of the pragma -*/ -static const char *const pragCName[] = { - /* 0 */ "id", /* Used by: foreign_key_list */ - /* 1 */ "seq", - /* 2 */ "table", - /* 3 */ "from", - /* 4 */ "to", - /* 5 */ "on_update", - /* 6 */ "on_delete", - /* 7 */ "match", - /* 8 */ "cid", /* Used by: table_xinfo */ - /* 9 */ "name", - /* 10 */ "type", - /* 11 */ "notnull", - /* 12 */ "dflt_value", - /* 13 */ "pk", - /* 14 */ "hidden", - /* table_info reuses 8 */ - /* 15 */ "name", /* Used by: function_list */ - /* 16 */ "builtin", - /* 17 */ "type", - /* 18 */ "enc", - /* 19 */ "narg", - /* 20 */ "flags", - /* 21 */ "schema", /* Used by: table_list */ - /* 22 */ "name", - /* 23 */ "type", - /* 24 */ "ncol", - /* 25 */ "wr", - /* 26 */ "strict", - /* 27 */ "seqno", /* Used by: index_xinfo */ - /* 28 */ "cid", - /* 29 */ "name", - /* 30 */ "desc", - /* 31 */ "coll", - /* 32 */ "key", - /* 33 */ "seq", /* Used by: index_list */ - /* 34 */ "name", - /* 35 */ "unique", - /* 36 */ "origin", - /* 37 */ "partial", - /* 38 */ "tbl", /* Used by: stats */ - /* 39 */ "idx", - /* 40 */ "wdth", - /* 41 */ "hght", - /* 42 */ "flgs", - /* 43 */ "table", /* Used by: foreign_key_check */ - /* 44 */ "rowid", - /* 45 */ "parent", - /* 46 */ "fkid", - /* 47 */ "busy", /* Used by: wal_checkpoint */ - /* 48 */ "log", - /* 49 */ "checkpointed", - /* 50 */ "seq", /* Used by: database_list */ - /* 51 */ "name", - /* 52 */ "file", - /* index_info reuses 27 */ - /* 53 */ "database", /* Used by: lock_status */ - /* 54 */ "status", - /* collation_list reuses 33 */ - /* 55 */ "cache_size", /* Used by: default_cache_size */ - /* module_list pragma_list reuses 9 */ - /* 56 */ "timeout", /* Used by: busy_timeout */ -}; - -/* Definitions of all built-in pragmas */ -typedef struct PragmaName { - const char *const zName; /* Name of pragma */ - u8 ePragTyp; /* PragTyp_XXX value */ - u8 mPragFlg; /* Zero or more PragFlg_XXX values */ - u8 iPragCName; /* Start of column names in pragCName[] */ - u8 nPragCName; /* Num of col names. 0 means use pragma name */ - u64 iArg; /* Extra argument */ -} PragmaName; -static const PragmaName aPragmaName[] = { -#if defined(SQLITE_ENABLE_CEROD) - {/* zName: */ "activate_extensions", - /* ePragTyp: */ PragTyp_ACTIVATE_EXTENSIONS, - /* ePragFlg: */ 0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif - {/* zName: */ "analysis_limit", - /* ePragTyp: */ PragTyp_ANALYSIS_LIMIT, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) - {/* zName: */ "application_id", - /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlg: */ PragFlg_NoColumns1|PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ BTREE_APPLICATION_ID }, -#endif -#if !defined(SQLITE_OMIT_AUTOVACUUM) - {/* zName: */ "auto_vacuum", - /* ePragTyp: */ PragTyp_AUTO_VACUUM, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) -#if !defined(SQLITE_OMIT_AUTOMATIC_INDEX) - {/* zName: */ "automatic_index", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_AutoIndex }, -#endif -#endif - {/* zName: */ "busy_timeout", - /* ePragTyp: */ PragTyp_BUSY_TIMEOUT, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 56, 1, - /* iArg: */ 0 }, -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - {/* zName: */ "cache_size", - /* ePragTyp: */ PragTyp_CACHE_SIZE, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "cache_spill", - /* ePragTyp: */ PragTyp_CACHE_SPILL, - /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA) - {/* zName: */ "case_sensitive_like", - /* ePragTyp: */ PragTyp_CASE_SENSITIVE_LIKE, - /* ePragFlg: */ PragFlg_NoColumns, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif - {/* zName: */ "cell_size_check", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_CellSizeCk }, -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "checkpoint_fullfsync", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_CkptFullFSync }, -#endif -#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) - {/* zName: */ "collation_list", - /* ePragTyp: */ PragTyp_COLLATION_LIST, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 33, 2, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_COMPILEOPTION_DIAGS) - {/* zName: */ "compile_options", - /* ePragTyp: */ PragTyp_COMPILE_OPTIONS, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "count_changes", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_CountRows }, -#endif -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_OS_WIN - {/* zName: */ "data_store_directory", - /* ePragTyp: */ PragTyp_DATA_STORE_DIRECTORY, - /* ePragFlg: */ PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) - {/* zName: */ "data_version", - /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlg: */ PragFlg_ReadOnly|PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ BTREE_DATA_VERSION }, -#endif -#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) - {/* zName: */ "database_list", - /* ePragTyp: */ PragTyp_DATABASE_LIST, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 50, 3, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) - {/* zName: */ "default_cache_size", - /* ePragTyp: */ PragTyp_DEFAULT_CACHE_SIZE, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, - /* ColNames: */ 55, 1, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) -#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - {/* zName: */ "defer_foreign_keys", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_DeferFKs }, -#endif -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "empty_result_callbacks", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_NullCallback }, -#endif -#if !defined(SQLITE_OMIT_UTF16) - {/* zName: */ "encoding", - /* ePragTyp: */ PragTyp_ENCODING, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - {/* zName: */ "foreign_key_check", - /* ePragTyp: */ PragTyp_FOREIGN_KEY_CHECK, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 43, 4, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FOREIGN_KEY) - {/* zName: */ "foreign_key_list", - /* ePragTyp: */ PragTyp_FOREIGN_KEY_LIST, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 0, 8, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) -#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - {/* zName: */ "foreign_keys", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_ForeignKeys }, -#endif -#endif -#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) - {/* zName: */ "freelist_count", - /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlg: */ PragFlg_ReadOnly|PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ BTREE_FREE_PAGE_COUNT }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "full_column_names", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_FullColNames }, - {/* zName: */ "fullfsync", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_FullFSync }, -#endif -#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) -#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) - {/* zName: */ "function_list", - /* ePragTyp: */ PragTyp_FUNCTION_LIST, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 15, 6, - /* iArg: */ 0 }, -#endif -#endif - {/* zName: */ "hard_heap_limit", - /* ePragTyp: */ PragTyp_HARD_HEAP_LIMIT, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) -#if !defined(SQLITE_OMIT_CHECK) - {/* zName: */ "ignore_check_constraints", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_IgnoreChecks }, -#endif -#endif -#if !defined(SQLITE_OMIT_AUTOVACUUM) - {/* zName: */ "incremental_vacuum", - /* ePragTyp: */ PragTyp_INCREMENTAL_VACUUM, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_NoColumns, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) - {/* zName: */ "index_info", - /* ePragTyp: */ PragTyp_INDEX_INFO, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 27, 3, - /* iArg: */ 0 }, - {/* zName: */ "index_list", - /* ePragTyp: */ PragTyp_INDEX_LIST, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 33, 5, - /* iArg: */ 0 }, - {/* zName: */ "index_xinfo", - /* ePragTyp: */ PragTyp_INDEX_INFO, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 27, 6, - /* iArg: */ 1 }, -#endif -#if !defined(SQLITE_OMIT_INTEGRITY_CHECK) - {/* zName: */ "integrity_check", - /* ePragTyp: */ PragTyp_INTEGRITY_CHECK, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - {/* zName: */ "journal_mode", - /* ePragTyp: */ PragTyp_JOURNAL_MODE, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, - {/* zName: */ "journal_size_limit", - /* ePragTyp: */ PragTyp_JOURNAL_SIZE_LIMIT, - /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "legacy_alter_table", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_LegacyAlter }, -#endif -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_ENABLE_LOCKING_STYLE - {/* zName: */ "lock_proxy_file", - /* ePragTyp: */ PragTyp_LOCK_PROXY_FILE, - /* ePragFlg: */ PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) - {/* zName: */ "lock_status", - /* ePragTyp: */ PragTyp_LOCK_STATUS, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 53, 2, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - {/* zName: */ "locking_mode", - /* ePragTyp: */ PragTyp_LOCKING_MODE, - /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, - {/* zName: */ "max_page_count", - /* ePragTyp: */ PragTyp_PAGE_COUNT, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, - {/* zName: */ "mmap_size", - /* ePragTyp: */ PragTyp_MMAP_SIZE, - /* ePragFlg: */ 0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) -#if !defined(SQLITE_OMIT_VIRTUALTABLE) -#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) - {/* zName: */ "module_list", - /* ePragTyp: */ PragTyp_MODULE_LIST, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 9, 1, - /* iArg: */ 0 }, -#endif -#endif -#endif - {/* zName: */ "optimize", - /* ePragTyp: */ PragTyp_OPTIMIZE, - /* ePragFlg: */ PragFlg_Result1|PragFlg_NeedSchema, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - {/* zName: */ "page_count", - /* ePragTyp: */ PragTyp_PAGE_COUNT, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, - {/* zName: */ "page_size", - /* ePragTyp: */ PragTyp_PAGE_SIZE, - /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) -#if defined(SQLITE_DEBUG) - {/* zName: */ "parser_trace", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_ParserTrace }, -#endif -#endif -#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) - {/* zName: */ "pragma_list", - /* ePragTyp: */ PragTyp_PRAGMA_LIST, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 9, 1, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "query_only", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_QueryOnly }, -#endif -#if !defined(SQLITE_OMIT_INTEGRITY_CHECK) - {/* zName: */ "quick_check", - /* ePragTyp: */ PragTyp_INTEGRITY_CHECK, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "read_uncommitted", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_ReadUncommit }, - {/* zName: */ "recursive_triggers", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_RecTriggers }, - {/* zName: */ "reverse_unordered_selects", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_ReverseOrder }, -#endif -#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) - {/* zName: */ "schema_version", - /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlg: */ PragFlg_NoColumns1|PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ BTREE_SCHEMA_VERSION }, -#endif -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - {/* zName: */ "secure_delete", - /* ePragTyp: */ PragTyp_SECURE_DELETE, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "short_column_names", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_ShortColNames }, -#endif - {/* zName: */ "shrink_memory", - /* ePragTyp: */ PragTyp_SHRINK_MEMORY, - /* ePragFlg: */ PragFlg_NoColumns, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, - {/* zName: */ "soft_heap_limit", - /* ePragTyp: */ PragTyp_SOFT_HEAP_LIMIT, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) -#if defined(SQLITE_DEBUG) - {/* zName: */ "sql_trace", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_SqlTrace }, -#endif -#endif -#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && defined(SQLITE_DEBUG) - {/* zName: */ "stats", - /* ePragTyp: */ PragTyp_STATS, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, - /* ColNames: */ 38, 5, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - {/* zName: */ "synchronous", - /* ePragTyp: */ PragTyp_SYNCHRONOUS, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) - {/* zName: */ "table_info", - /* ePragTyp: */ PragTyp_TABLE_INFO, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 8, 6, - /* iArg: */ 0 }, - {/* zName: */ "table_list", - /* ePragTyp: */ PragTyp_TABLE_LIST, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1, - /* ColNames: */ 21, 6, - /* iArg: */ 0 }, - {/* zName: */ "table_xinfo", - /* ePragTyp: */ PragTyp_TABLE_INFO, - /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 8, 7, - /* iArg: */ 1 }, -#endif -#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - {/* zName: */ "temp_store", - /* ePragTyp: */ PragTyp_TEMP_STORE, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, - {/* zName: */ "temp_store_directory", - /* ePragTyp: */ PragTyp_TEMP_STORE_DIRECTORY, - /* ePragFlg: */ PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif - {/* zName: */ "threads", - /* ePragTyp: */ PragTyp_THREADS, - /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "trusted_schema", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_TrustedSchema }, -#endif -#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) - {/* zName: */ "user_version", - /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlg: */ PragFlg_NoColumns1|PragFlg_Result0, - /* ColNames: */ 0, 0, - /* iArg: */ BTREE_USER_VERSION }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) -#if defined(SQLITE_DEBUG) - {/* zName: */ "vdbe_addoptrace", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_VdbeAddopTrace }, - {/* zName: */ "vdbe_debug", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_SqlTrace|SQLITE_VdbeListing|SQLITE_VdbeTrace }, - {/* zName: */ "vdbe_eqp", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_VdbeEQP }, - {/* zName: */ "vdbe_listing", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_VdbeListing }, - {/* zName: */ "vdbe_trace", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_VdbeTrace }, -#endif -#endif -#if !defined(SQLITE_OMIT_WAL) - {/* zName: */ "wal_autocheckpoint", - /* ePragTyp: */ PragTyp_WAL_AUTOCHECKPOINT, - /* ePragFlg: */ 0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, - {/* zName: */ "wal_checkpoint", - /* ePragTyp: */ PragTyp_WAL_CHECKPOINT, - /* ePragFlg: */ PragFlg_NeedSchema, - /* ColNames: */ 47, 3, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "writable_schema", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, - /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_WriteSchema|SQLITE_NoSchemaError }, -#endif -}; -/* Number of pragmas: 68 on by default, 78 total. */ diff --git a/src/printf.c b/src/printf.c index 71363f91b..8cb5a43c5 100644 --- a/src/printf.c +++ b/src/printf.c @@ -25,17 +25,17 @@ #define etPERCENT 7 /* Percent symbol. %% */ #define etCHARX 8 /* Characters. %c */ /* The rest are extensions, not normally found in printf() */ -#define etSQLESCAPE 9 /* Strings with '\'' doubled. %q */ -#define etSQLESCAPE2 10 /* Strings with '\'' doubled and enclosed in '', - NULL pointers replaced by SQL NULL. %Q */ -#define etTOKEN 11 /* a pointer to a Token structure */ -#define etSRCITEM 12 /* a pointer to a SrcItem */ -#define etPOINTER 13 /* The %p conversion */ -#define etSQLESCAPE3 14 /* %w -> Strings with '\"' doubled */ -#define etORDINAL 15 /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ -#define etDECIMAL 16 /* %d or %u, but not %x, %o */ +#define etESCAPE_q 9 /* Strings with '\'' doubled. %q */ +#define etESCAPE_Q 10 /* Strings with '\'' doubled and enclosed in '', + NULL pointers replaced by SQL NULL. %Q */ +#define etTOKEN 11 /* a pointer to a Token structure */ +#define etSRCITEM 12 /* a pointer to a SrcItem */ +#define etPOINTER 13 /* The %p conversion */ +#define etESCAPE_w 14 /* %w -> Strings with '\"' doubled */ +#define etORDINAL 15 /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ +#define etDECIMAL 16 /* %d or %u, but not %x, %o */ -#define etINVALID 17 /* Any unrecognized conversion type */ +#define etINVALID 17 /* Any unrecognized conversion type */ /* @@ -74,9 +74,9 @@ static const et_info fmtinfo[] = { { 's', 0, 4, etSTRING, 0, 0 }, { 'g', 0, 1, etGENERIC, 30, 0 }, { 'z', 0, 4, etDYNSTRING, 0, 0 }, - { 'q', 0, 4, etSQLESCAPE, 0, 0 }, - { 'Q', 0, 4, etSQLESCAPE2, 0, 0 }, - { 'w', 0, 4, etSQLESCAPE3, 0, 0 }, + { 'q', 0, 4, etESCAPE_q, 0, 0 }, + { 'Q', 0, 4, etESCAPE_Q, 0, 0 }, + { 'w', 0, 4, etESCAPE_w, 0, 0 }, { 'c', 0, 0, etCHARX, 0, 0 }, { 'o', 8, 0, etRADIX, 0, 2 }, { 'u', 10, 0, etDECIMAL, 0, 0 }, @@ -673,25 +673,7 @@ void sqlite3_str_vappendf( } }else{ unsigned int ch = va_arg(ap,unsigned int); - if( ch<0x00080 ){ - buf[0] = ch & 0xff; - length = 1; - }else if( ch<0x00800 ){ - buf[0] = 0xc0 + (u8)((ch>>6)&0x1f); - buf[1] = 0x80 + (u8)(ch & 0x3f); - length = 2; - }else if( ch<0x10000 ){ - buf[0] = 0xe0 + (u8)((ch>>12)&0x0f); - buf[1] = 0x80 + (u8)((ch>>6) & 0x3f); - buf[2] = 0x80 + (u8)(ch & 0x3f); - length = 3; - }else{ - buf[0] = 0xf0 + (u8)((ch>>18) & 0x07); - buf[1] = 0x80 + (u8)((ch>>12) & 0x3f); - buf[2] = 0x80 + (u8)((ch>>6) & 0x3f); - buf[3] = 0x80 + (u8)(ch & 0x3f); - length = 4; - } + length = sqlite3AppendOneUtf8Character(buf, ch); } if( precision>1 ){ i64 nPrior = 1; @@ -771,22 +753,31 @@ void sqlite3_str_vappendf( while( ii>=0 ) if( (bufpt[ii--] & 0xc0)==0x80 ) width++; } break; - case etSQLESCAPE: /* %q: Escape ' characters */ - case etSQLESCAPE2: /* %Q: Escape ' and enclose in '...' */ - case etSQLESCAPE3: { /* %w: Escape " characters */ + case etESCAPE_q: /* %q: Escape ' characters */ + case etESCAPE_Q: /* %Q: Escape ' and enclose in '...' */ + case etESCAPE_w: { /* %w: Escape " characters */ i64 i, j, k, n; - int needQuote, isnull; + int needQuote = 0; char ch; - char q = ((xtype==etSQLESCAPE3)?'"':'\''); /* Quote character */ char *escarg; + char q; if( bArgList ){ escarg = getTextArg(pArgList); }else{ escarg = va_arg(ap,char*); } - isnull = escarg==0; - if( isnull ) escarg = (xtype==etSQLESCAPE2 ? "NULL" : "(NULL)"); + if( escarg==0 ){ + escarg = (xtype==etESCAPE_Q ? "NULL" : "(NULL)"); + }else if( xtype==etESCAPE_Q ){ + needQuote = 1; + } + if( xtype==etESCAPE_w ){ + q = '"'; + flag_alternateform = 0; + }else{ + q = '\''; + } /* For %q, %Q, and %w, the precision is the number of bytes (or ** characters if the ! flags is present) to use from the input. ** Because of the extra quoting characters inserted, the number @@ -799,7 +790,30 @@ void sqlite3_str_vappendf( while( (escarg[i+1]&0xc0)==0x80 ){ i++; } } } - needQuote = !isnull && xtype==etSQLESCAPE2; + if( flag_alternateform ){ + /* For %#q, do unistr()-style backslash escapes for + ** all control characters, and for backslash itself. + ** For %#Q, do the same but only if there is at least + ** one control character. */ + u32 nBack = 0; + u32 nCtrl = 0; + for(k=0; ketBUFSIZE ){ bufpt = zExtra = printfTempBuf(pAccum, n); @@ -808,13 +822,41 @@ void sqlite3_str_vappendf( bufpt = buf; } j = 0; - if( needQuote ) bufpt[j++] = q; + if( needQuote ){ + if( needQuote==2 ){ + memcpy(&bufpt[j], "unistr('", 8); + j += 8; + }else{ + bufpt[j++] = '\''; + } + } k = i; - for(i=0; i=0x10 ? '1' : '0'; + bufpt[j++] = "0123456789abcdef"[ch&0xf]; + } + } + }else{ + for(i=0; imxAlloc>0 && !isMalloced(p) ); - zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); + zText = sqlite3DbMallocRaw(p->db, 1+(u64)p->nChar ); if( zText ){ memcpy(zText, p->zText, p->nChar+1); p->printfFlags |= SQLITE_PRINTF_MALLOCED; @@ -1302,6 +1344,15 @@ char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ return zBuf; } +/* Maximum size of an sqlite3_log() message. */ +#if defined(SQLITE_MAX_LOG_MESSAGE) + /* Leave the definition as supplied */ +#elif SQLITE_PRINT_BUF_SIZE*10>10000 +# define SQLITE_MAX_LOG_MESSAGE 10000 +#else +# define SQLITE_MAX_LOG_MESSAGE (SQLITE_PRINT_BUF_SIZE*10) +#endif + /* ** This is the routine that actually formats the sqlite3_log() message. ** We house it in a separate routine from sqlite3_log() to avoid using @@ -1318,7 +1369,7 @@ char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ */ static void renderLogMsg(int iErrCode, const char *zFormat, va_list ap){ StrAccum acc; /* String accumulator */ - char zMsg[SQLITE_PRINT_BUF_SIZE*3]; /* Complete log message */ + char zMsg[SQLITE_MAX_LOG_MESSAGE]; /* Complete log message */ sqlite3StrAccumInit(&acc, 0, zMsg, sizeof(zMsg), 0); sqlite3_str_vappendf(&acc, zFormat, ap); diff --git a/src/resolve.c b/src/resolve.c index d6a5144af..3961a2009 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -294,7 +294,6 @@ static int lookupName( Schema *pSchema = 0; /* Schema of the expression */ int eNewExprOp = TK_COLUMN; /* New value for pExpr->op on success */ Table *pTab = 0; /* Table holding the row */ - Column *pCol; /* A column of pTab */ ExprList *pFJMatch = 0; /* Matches for FULL JOIN .. USING */ const char *zCol = pRight->u.zToken; @@ -345,7 +344,6 @@ static int lookupName( if( pSrcList ){ for(i=0, pItem=pSrcList->a; inSrc; i++, pItem++){ - u8 hCol; pTab = pItem->pSTab; assert( pTab!=0 && pTab->zName!=0 ); assert( pTab->nCol>0 || pParse->nErr ); @@ -433,43 +431,38 @@ static int lookupName( sqlite3RenameTokenRemap(pParse, 0, (void*)&pExpr->y.pTab); } } - hCol = sqlite3StrIHash(zCol); - for(j=0, pCol=pTab->aCol; jnCol; j++, pCol++){ - if( pCol->hName==hCol - && sqlite3StrICmp(pCol->zCnName, zCol)==0 - ){ - if( cnt>0 ){ - if( pItem->fg.isUsing==0 - || sqlite3IdListIndex(pItem->u3.pUsing, zCol)<0 - ){ - /* Two or more tables have the same column name which is - ** not joined by USING. This is an error. Signal as much - ** by clearing pFJMatch and letting cnt go above 1. */ - sqlite3ExprListDelete(db, pFJMatch); - pFJMatch = 0; - }else - if( (pItem->fg.jointype & JT_RIGHT)==0 ){ - /* An INNER or LEFT JOIN. Use the left-most table */ - continue; - }else - if( (pItem->fg.jointype & JT_LEFT)==0 ){ - /* A RIGHT JOIN. Use the right-most table */ - cnt = 0; - sqlite3ExprListDelete(db, pFJMatch); - pFJMatch = 0; - }else{ - /* For a FULL JOIN, we must construct a coalesce() func */ - extendFJMatch(pParse, &pFJMatch, pMatch, pExpr->iColumn); - } - } - cnt++; - pMatch = pItem; - /* Substitute the rowid (column -1) for the INTEGER PRIMARY KEY */ - pExpr->iColumn = j==pTab->iPKey ? -1 : (i16)j; - if( pItem->fg.isNestedFrom ){ - sqlite3SrcItemColumnUsed(pItem, j); + j = sqlite3ColumnIndex(pTab, zCol); + if( j>=0 ){ + if( cnt>0 ){ + if( pItem->fg.isUsing==0 + || sqlite3IdListIndex(pItem->u3.pUsing, zCol)<0 + ){ + /* Two or more tables have the same column name which is + ** not joined by USING. This is an error. Signal as much + ** by clearing pFJMatch and letting cnt go above 1. */ + sqlite3ExprListDelete(db, pFJMatch); + pFJMatch = 0; + }else + if( (pItem->fg.jointype & JT_RIGHT)==0 ){ + /* An INNER or LEFT JOIN. Use the left-most table */ + continue; + }else + if( (pItem->fg.jointype & JT_LEFT)==0 ){ + /* A RIGHT JOIN. Use the right-most table */ + cnt = 0; + sqlite3ExprListDelete(db, pFJMatch); + pFJMatch = 0; + }else{ + /* For a FULL JOIN, we must construct a coalesce() func */ + extendFJMatch(pParse, &pFJMatch, pMatch, pExpr->iColumn); } - break; + } + cnt++; + pMatch = pItem; + /* Substitute the rowid (column -1) for the INTEGER PRIMARY KEY */ + pExpr->iColumn = j==pTab->iPKey ? -1 : (i16)j; + if( pItem->fg.isNestedFrom ){ + sqlite3SrcItemColumnUsed(pItem, j); } } if( 0==cnt && VisibleRowid(pTab) ){ @@ -559,23 +552,18 @@ static int lookupName( if( pTab ){ int iCol; - u8 hCol = sqlite3StrIHash(zCol); pSchema = pTab->pSchema; cntTab++; - for(iCol=0, pCol=pTab->aCol; iColnCol; iCol++, pCol++){ - if( pCol->hName==hCol - && sqlite3StrICmp(pCol->zCnName, zCol)==0 - ){ - if( iCol==pTab->iPKey ){ - iCol = -1; - } - break; + iCol = sqlite3ColumnIndex(pTab, zCol); + if( iCol>=0 ){ + if( pTab->iPKey==iCol ) iCol = -1; + }else{ + if( sqlite3IsRowid(zCol) && VisibleRowid(pTab) ){ + iCol = -1; + }else{ + iCol = pTab->nCol; } } - if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) && VisibleRowid(pTab) ){ - /* IMP: R-51414-32910 */ - iCol = -1; - } if( iColnCol ){ cnt++; pMatch = 0; @@ -1214,13 +1202,12 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ ** sqlite_version() that might change over time cannot be used ** in an index or generated column. Curiously, they can be used ** in a CHECK constraint. SQLServer, MySQL, and PostgreSQL all - ** all this. */ + ** allow this. */ sqlite3ResolveNotValid(pParse, pNC, "non-deterministic functions", NC_IdxExpr|NC_PartIdx|NC_GenCol, 0, pExpr); }else{ assert( (NC_SelfRef & 0xff)==NC_SelfRef ); /* Must fit in 8 bits */ pExpr->op2 = pNC->ncFlags & NC_SelfRef; - if( pNC->ncFlags & NC_FromDDL ) ExprSetProperty(pExpr, EP_FromDDL); } if( (pDef->funcFlags & SQLITE_FUNC_INTERNAL)!=0 && pParse->nested==0 @@ -1236,6 +1223,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ if( (pDef->funcFlags & (SQLITE_FUNC_DIRECT|SQLITE_FUNC_UNSAFE))!=0 && !IN_RENAME_OBJECT ){ + if( pNC->ncFlags & NC_FromDDL ) ExprSetProperty(pExpr, EP_FromDDL); sqlite3ExprFunctionUsable(pParse, pExpr, pDef); } } @@ -2289,20 +2277,22 @@ int sqlite3ResolveSelfReference( Expr *pExpr, /* Expression to resolve. May be NULL. */ ExprList *pList /* Expression list to resolve. May be NULL. */ ){ - SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ + SrcList *pSrc; /* Fake SrcList for pParse->pNewTable */ NameContext sNC; /* Name context for pParse->pNewTable */ int rc; + u8 srcSpace[SZ_SRCLIST_1]; /* Memory space for the fake SrcList */ assert( type==0 || pTab!=0 ); assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr || type==NC_GenCol || pTab==0 ); memset(&sNC, 0, sizeof(sNC)); - memset(&sSrc, 0, sizeof(sSrc)); + pSrc = (SrcList*)srcSpace; + memset(pSrc, 0, SZ_SRCLIST_1); if( pTab ){ - sSrc.nSrc = 1; - sSrc.a[0].zName = pTab->zName; - sSrc.a[0].pSTab = pTab; - sSrc.a[0].iCursor = -1; + pSrc->nSrc = 1; + pSrc->a[0].zName = pTab->zName; + pSrc->a[0].pSTab = pTab; + pSrc->a[0].iCursor = -1; if( pTab->pSchema!=pParse->db->aDb[1].pSchema ){ /* Cause EP_FromDDL to be set on TK_FUNCTION nodes of non-TEMP ** schema elements */ @@ -2310,7 +2300,7 @@ int sqlite3ResolveSelfReference( } } sNC.pParse = pParse; - sNC.pSrcList = &sSrc; + sNC.pSrcList = pSrc; sNC.ncFlags = type | NC_IsDDL; if( (rc = sqlite3ResolveExprNames(&sNC, pExpr))!=SQLITE_OK ) return rc; if( pList ) rc = sqlite3ResolveExprListNames(&sNC, pList); diff --git a/src/select.c b/src/select.c index 2f11297bf..8e4c939cd 100644 --- a/src/select.c +++ b/src/select.c @@ -154,7 +154,7 @@ Select *sqlite3SelectNew( pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = 0; - if( pSrc==0 ) pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*pSrc)); + if( pSrc==0 ) pSrc = sqlite3DbMallocZero(pParse->db, SZ_SRCLIST_1); pNew->pSrc = pSrc; pNew->pWhere = pWhere; pNew->pGroupBy = pGroupBy; @@ -319,10 +319,33 @@ int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *pC){ */ int sqlite3ColumnIndex(Table *pTab, const char *zCol){ int i; - u8 h = sqlite3StrIHash(zCol); - Column *pCol; - for(pCol=pTab->aCol, i=0; inCol; pCol++, i++){ - if( pCol->hName==h && sqlite3StrICmp(pCol->zCnName, zCol)==0 ) return i; + u8 h; + const Column *aCol; + int nCol; + + h = sqlite3StrIHash(zCol); + aCol = pTab->aCol; + nCol = pTab->nCol; + + /* See if the aHx gives us a lucky match */ + i = pTab->aHx[h % sizeof(pTab->aHx)]; + assert( i=nCol ) break; } return -1; } @@ -573,7 +596,7 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ } pE1 = sqlite3CreateColumnExpr(db, pSrc, iLeft, iLeftCol); sqlite3SrcItemColumnUsed(&pSrc->a[iLeft], iLeftCol); - if( (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ + if( (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 && pParse->nErr==0 ){ /* This branch runs if the query contains one or more RIGHT or FULL ** JOINs. If only a single table on the left side of this join ** contains the zName column, then this branch is a no-op. @@ -589,6 +612,8 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ */ ExprList *pFuncArgs = 0; /* Arguments to the coalesce() */ static const Token tkCoalesce = { "coalesce", 8 }; + assert( pE1!=0 ); + ExprSetProperty(pE1, EP_CanBeNull); while( tableAndColumnIndex(pSrc, iLeft+1, i, zName, &iLeft, &iLeftCol, pRight->fg.isSynthUsing)!=0 ){ if( pSrc->a[iLeft].fg.isUsing==0 @@ -605,7 +630,13 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ if( pFuncArgs ){ pFuncArgs = sqlite3ExprListAppend(pParse, pFuncArgs, pE1); pE1 = sqlite3ExprFunction(pParse, pFuncArgs, &tkCoalesce, 0); + if( pE1 ){ + pE1->affExpr = SQLITE_AFF_DEFER; + } } + }else if( (pSrc->a[i+1].fg.jointype & JT_LEFT)!=0 && pParse->nErr==0 ){ + assert( pE1!=0 ); + ExprSetProperty(pE1, EP_CanBeNull); } pE2 = sqlite3CreateColumnExpr(db, pSrc, i+1, iRightCol); sqlite3SrcItemColumnUsed(pRight, iRightCol); @@ -1514,8 +1545,8 @@ static void selectInnerLoop( ** X extra columns. */ KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ - int nExtra = (N+X)*(sizeof(CollSeq*)+1) - sizeof(CollSeq*); - KeyInfo *p = sqlite3DbMallocRawNN(db, sizeof(KeyInfo) + nExtra); + int nExtra = (N+X)*(sizeof(CollSeq*)+1); + KeyInfo *p = sqlite3DbMallocRawNN(db, SZ_KEYINFO(0) + nExtra); if( p ){ p->aSortFlags = (u8*)&p->aColl[N+X]; p->nKeyField = (u16)N; @@ -1523,7 +1554,7 @@ KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ p->enc = ENC(db); p->db = db; p->nRef = 1; - memset(&p[1], 0, nExtra); + memset(p->aColl, 0, nExtra); }else{ return (KeyInfo*)sqlite3OomFault(db); } @@ -4213,9 +4244,9 @@ static int compoundHasDifferentAffinities(Select *p){ ** from 2015-02-09.) ** ** (3) If the subquery is the right operand of a LEFT JOIN then -** (3a) the subquery may not be a join and -** (3b) the FROM clause of the subquery may not contain a virtual -** table and +** (3a) the subquery may not be a join +** (**) Was (3b): "the FROM clause of the subquery may not contain +** a virtual table" ** (**) Was: "The outer query may not have a GROUP BY." This case ** is now managed correctly ** (3d) the outer query may not be DISTINCT. @@ -4431,7 +4462,7 @@ static int flattenSubquery( */ if( (pSubitem->fg.jointype & (JT_OUTER|JT_LTORJ))!=0 ){ if( pSubSrc->nSrc>1 /* (3a) */ - || IsVirtual(pSubSrc->a[0].pSTab) /* (3b) */ + /**** || IsVirtual(pSubSrc->a[0].pSTab) (3b)-omitted */ || (p->selFlags & SF_Distinct)!=0 /* (3d) */ || (pSubitem->fg.jointype & JT_RIGHT)!=0 /* (26) */ ){ @@ -4835,7 +4866,8 @@ static void constInsert( return; /* Already present. Return without doing anything. */ } } - if( sqlite3ExprAffinity(pColumn)==SQLITE_AFF_BLOB ){ + assert( SQLITE_AFF_NONEbHasAffBlob = 1; } @@ -4910,7 +4942,8 @@ static int propagateConstantExprRewriteOne( if( pColumn==pExpr ) continue; if( pColumn->iTable!=pExpr->iTable ) continue; if( pColumn->iColumn!=pExpr->iColumn ) continue; - if( bIgnoreAffBlob && sqlite3ExprAffinity(pColumn)==SQLITE_AFF_BLOB ){ + assert( SQLITE_AFF_NONEpWinDefn = 0; #endif - p->selFlags &= ~SF_Compound; + p->selFlags &= ~(u32)SF_Compound; assert( (p->selFlags & SF_Converted)==0 ); p->selFlags |= SF_Converted; assert( pNew->pPrior!=0 ); @@ -6040,7 +6073,7 @@ static int selectExpander(Walker *pWalker, Select *p){ pEList = p->pEList; if( pParse->pWith && (p->selFlags & SF_View) ){ if( p->pWith==0 ){ - p->pWith = (With*)sqlite3DbMallocZero(db, sizeof(With)); + p->pWith = (With*)sqlite3DbMallocZero(db, SZ_WITH(1) ); if( p->pWith==0 ){ return WRC_Abort; } @@ -7228,14 +7261,14 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ pExpr = 0; pSub = sqlite3SubqueryDetach(db, pFrom); sqlite3SrcListDelete(db, p->pSrc); - p->pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*p->pSrc)); + p->pSrc = sqlite3DbMallocZero(pParse->db, SZ_SRCLIST_1); while( pSub ){ Expr *pTerm; pPrior = pSub->pPrior; pSub->pPrior = 0; pSub->pNext = 0; pSub->selFlags |= SF_Aggregate; - pSub->selFlags &= ~SF_Compound; + pSub->selFlags &= ~(u32)SF_Compound; pSub->nSelectRow = 0; sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, pSub->pEList); pTerm = pPrior ? sqlite3ExprDup(db, pCount, 0) : pCount; @@ -7250,7 +7283,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ pSub = pPrior; } p->pEList->a[0].pExpr = pExpr; - p->selFlags &= ~SF_Aggregate; + p->selFlags &= ~(u32)SF_Aggregate; #if TREETRACE_ENABLED if( sqlite3TreeTrace & 0x200 ){ @@ -7457,7 +7490,7 @@ int sqlite3Select( testcase( pParse->earlyCleanup ); p->pOrderBy = 0; } - p->selFlags &= ~SF_Distinct; + p->selFlags &= ~(u32)SF_Distinct; p->selFlags |= SF_NoopOrderBy; } sqlite3SelectPrep(pParse, p, 0); @@ -7496,7 +7529,7 @@ int sqlite3Select( ** and leaving this flag set can cause errors if a compound sub-query ** in p->pSrc is flattened into this query and this function called ** again as part of compound SELECT processing. */ - p->selFlags &= ~SF_UFSrcCheck; + p->selFlags &= ~(u32)SF_UFSrcCheck; } if( pDest->eDest==SRT_Output ){ @@ -7985,7 +8018,7 @@ int sqlite3Select( && p->pWin==0 #endif ){ - p->selFlags &= ~SF_Distinct; + p->selFlags &= ~(u32)SF_Distinct; pGroupBy = p->pGroupBy = sqlite3ExprListDup(db, pEList, 0); if( pGroupBy ){ for(i=0; inExpr; i++){ @@ -8094,6 +8127,12 @@ int sqlite3Select( if( pWInfo==0 ) goto select_end; if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){ p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo); + if( pDest->eDest<=SRT_DistQueue && pDest->eDest>=SRT_DistFifo ){ + /* TUNING: For a UNION CTE, because UNION is implies DISTINCT, + ** reduce the estimated output row count by 8 (LogEst 30). + ** Search for tag-20250414a to see other cases */ + p->nSelectRow -= 30; + } } if( sDistinct.isTnct && sqlite3WhereIsDistinct(pWInfo) ){ sDistinct.eTnctType = sqlite3WhereIsDistinct(pWInfo); @@ -8467,6 +8506,10 @@ int sqlite3Select( if( iOrderByCol ){ Expr *pX = p->pEList->a[iOrderByCol-1].pExpr; Expr *pBase = sqlite3ExprSkipCollateAndLikely(pX); + while( ALWAYS(pBase!=0) && pBase->op==TK_IF_NULL_ROW ){ + pX = pBase->pLeft; + pBase = sqlite3ExprSkipCollateAndLikely(pX); + } if( ALWAYS(pBase!=0) && pBase->op!=TK_AGG_COLUMN && pBase->op!=TK_REGISTER diff --git a/src/shell.c.in b/src/shell.c.in index fcc9316b0..8660bd78a 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -218,6 +218,8 @@ typedef unsigned char u8; #define IsSpace(X) isspace((unsigned char)X) #define IsDigit(X) isdigit((unsigned char)X) #define ToLower(X) (char)tolower((unsigned char)X) +#define IsAlnum(X) isalnum((unsigned char)X) +#define IsAlpha(X) isalpha((unsigned char)X) #if defined(_WIN32) || defined(WIN32) #if SQLITE_OS_WINRT @@ -470,7 +472,7 @@ static char *Argv0; ** Prompt strings. Initialized in main. Settable with ** .prompt main continue */ -#define PROMPT_LEN_MAX 20 +#define PROMPT_LEN_MAX 128 /* First line prompt. default: "sqlite> " */ static char mainPrompt[PROMPT_LEN_MAX]; /* Continuation prompt. default: " ...> " */ @@ -782,6 +784,23 @@ int cli_wcswidth(const char *z){ } #endif +/* +** Check to see if z[] is a valid VT100 escape. If it is, then +** return the number of bytes in the escape sequence. Return 0 if +** z[] is not a VT100 escape. +** +** This routine assumes that z[0] is \033 (ESC). +*/ +static int isVt100(const unsigned char *z){ + int i; + if( z[1]!='[' ) return 0; + i = 2; + while( z[i]>=0x30 && z[i]<=0x3f ){ i++; } + while( z[i]>=0x20 && z[i]<=0x2f ){ i++; } + if( z[i]<0x40 || z[i]>0x7e ) return 0; + return i+1; +} + /* ** Output string zUtf to stdout as w characters. If w is negative, ** then right-justify the text. W is the width in UTF-8 characters, not @@ -797,6 +816,7 @@ static void utf8_width_print(FILE *out, int w, const char *zUtf){ unsigned char c; int i = 0; int n = 0; + int k; int aw = w<0 ? -w : w; if( zUtf==0 ) zUtf = ""; while( (c = a[i])!=0 ){ @@ -809,6 +829,8 @@ static void utf8_width_print(FILE *out, int w, const char *zUtf){ } i += len; n += x; + }else if( c==0x1b && (k = isVt100(&a[i]))>0 ){ + i += k; }else if( n>=aw ){ break; }else{ @@ -1127,9 +1149,9 @@ static void appendText(ShellText *p, const char *zAppend, char quote){ static char quoteChar(const char *zName){ int i; if( zName==0 ) return '"'; - if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"'; + if( !IsAlpha(zName[0]) && zName[0]!='_' ) return '"'; for(i=0; zName[i]; i++){ - if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"'; + if( !IsAlnum(zName[i]) && zName[i]!='_' ) return '"'; } return sqlite3_keyword_check(zName, i) ? '"' : 0; } @@ -1221,30 +1243,6 @@ static void shellDtostr( sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT); } - -/* -** SQL function: shell_module_schema(X) -** -** Return a fake schema for the table-valued function or eponymous virtual -** table X. -*/ -static void shellModuleSchema( - sqlite3_context *pCtx, - int nVal, - sqlite3_value **apVal -){ - const char *zName; - char *zFake; - UNUSED_PARAMETER(nVal); - zName = (const char*)sqlite3_value_text(apVal[0]); - zFake = zName? shellFakeSchema(sqlite3_context_db_handle(pCtx), 0, zName) : 0; - if( zFake ){ - sqlite3_result_text(pCtx, sqlite3_mprintf("/* %s */", zFake), - -1, sqlite3_free); - free(zFake); - } -} - /* ** SQL function: shell_add_schema(S,X) ** @@ -1446,6 +1444,7 @@ struct ShellState { u8 bSafeModePersist; /* The long-term value of bSafeMode */ u8 eRestoreState; /* See comments above doAutoDetectRestore() */ u8 crlfMode; /* Do NL-to-CRLF translations when enabled (maybe) */ + u8 eEscMode; /* Escape mode for text output */ ColModeOpts cmOpts; /* Option values affecting columnar mode output */ unsigned statsOn; /* True to display memory stats before each finalize */ unsigned mEqpLines; /* Mask of vertical lines in the EQP output graph */ @@ -1546,6 +1545,15 @@ static ShellState shellState; ** top-level SQL statement */ #define SHELL_PROGRESS_ONCE 0x04 /* Cancel the --limit after firing once */ +/* Allowed values for ShellState.eEscMode. The default value should +** be 0, so to change the default, reorder the names. +*/ +#define SHELL_ESC_ASCII 0 /* Substitute ^Y for X where Y=X+0x40 */ +#define SHELL_ESC_SYMBOL 1 /* Substitute U+2400 graphics */ +#define SHELL_ESC_OFF 2 /* Send characters verbatim */ + +static const char *shell_EscModeNames[] = { "ascii", "symbol", "off" }; + /* ** These are the allowed shellFlgs values */ @@ -1883,59 +1891,75 @@ static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){ } /* -** Find a string that is not found anywhere in z[]. Return a pointer -** to that string. +** Output the given string as a quoted string using SQL quoting conventions: ** -** Try to use zA and zB first. If both of those are already found in z[] -** then make up some string and store it in the buffer zBuf. -*/ -static const char *unused_string( - const char *z, /* Result must not appear anywhere in z */ - const char *zA, const char *zB, /* Try these first */ - char *zBuf /* Space to store a generated string */ -){ - unsigned i = 0; - if( strstr(z, zA)==0 ) return zA; - if( strstr(z, zB)==0 ) return zB; - do{ - sqlite3_snprintf(20,zBuf,"(%s%u)", zA, i++); - }while( strstr(z,zBuf)!=0 ); - return zBuf; -} - -/* -** Output the given string as a quoted string using SQL quoting conventions. +** (1) Single quotes (') within the string are doubled +** (2) The whle string is enclosed in '...' +** (3) Control characters other than \n, \t, and \r\n are escaped +** using \u00XX notation and if such substitutions occur, +** the whole string is enclosed in unistr('...') instead of '...'. +** +** Step (3) is omitted if the control-character escape mode is OFF. ** -** See also: output_quoted_escaped_string() +** See also: output_quoted_escaped_string() which does the same except +** that it does not make exceptions for \n, \t, and \r\n in step (3). */ -static void output_quoted_string(ShellState *p, const char *z){ +static void output_quoted_string(ShellState *p, const char *zInX){ int i; - char c; + int needUnistr = 0; + int needDblQuote = 0; + const unsigned char *z = (const unsigned char*)zInX; + unsigned char c; FILE *out = p->out; sqlite3_fsetmode(out, _O_BINARY); if( z==0 ) return; - for(i=0; (c = z[i])!=0 && c!='\''; i++){} - if( c==0 ){ + for(i=0; (c = z[i])!=0; i++){ + if( c=='\'' ){ needDblQuote = 1; } + if( c>0x1f ) continue; + if( c=='\t' || c=='\n' ) continue; + if( c=='\r' && z[i+1]=='\n' ) continue; + needUnistr = 1; + break; + } + if( (needDblQuote==0 && needUnistr==0) + || (needDblQuote==0 && p->eEscMode==SHELL_ESC_OFF) + ){ sqlite3_fprintf(out, "'%s'",z); + }else if( p->eEscMode==SHELL_ESC_OFF ){ + char *zEncoded = sqlite3_mprintf("%Q", z); + sqlite3_fputs(zEncoded, out); + sqlite3_free(zEncoded); }else{ - sqlite3_fputs("'", out); + if( needUnistr ){ + sqlite3_fputs("unistr('", out); + }else{ + sqlite3_fputs("'", out); + } while( *z ){ - for(i=0; (c = z[i])!=0 && c!='\''; i++){} - if( c=='\'' ) i++; + for(i=0; (c = z[i])!=0; i++){ + if( c=='\'' ) break; + if( c>0x1f ) continue; + if( c=='\t' || c=='\n' ) continue; + if( c=='\r' && z[i+1]=='\n' ) continue; + break; + } if( i ){ sqlite3_fprintf(out, "%.*s", i, z); z += i; } + if( c==0 ) break; if( c=='\'' ){ - sqlite3_fputs("'", out); - continue; - } - if( c==0 ){ - break; + sqlite3_fputs("''", out); + }else{ + sqlite3_fprintf(out, "\\u%04x", c); } z++; } - sqlite3_fputs("'", out); + if( needUnistr ){ + sqlite3_fputs("')", out); + }else{ + sqlite3_fputs("'", out); + } } setCrlfMode(p); } @@ -1950,61 +1974,15 @@ static void output_quoted_string(ShellState *p, const char *z){ ** escape mechanism. */ static void output_quoted_escaped_string(ShellState *p, const char *z){ - int i; - char c; - FILE *out = p->out; - sqlite3_fsetmode(out, _O_BINARY); - for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){} - if( c==0 ){ - sqlite3_fprintf(out, "'%s'",z); + char *zEscaped; + sqlite3_fsetmode(p->out, _O_BINARY); + if( p->eEscMode==SHELL_ESC_OFF ){ + zEscaped = sqlite3_mprintf("%Q", z); }else{ - const char *zNL = 0; - const char *zCR = 0; - int nNL = 0; - int nCR = 0; - char zBuf1[20], zBuf2[20]; - for(i=0; z[i]; i++){ - if( z[i]=='\n' ) nNL++; - if( z[i]=='\r' ) nCR++; - } - if( nNL ){ - sqlite3_fputs("replace(", out); - zNL = unused_string(z, "\\n", "\\012", zBuf1); - } - if( nCR ){ - sqlite3_fputs("replace(", out); - zCR = unused_string(z, "\\r", "\\015", zBuf2); - } - sqlite3_fputs("'", out); - while( *z ){ - for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){} - if( c=='\'' ) i++; - if( i ){ - sqlite3_fprintf(out, "%.*s", i, z); - z += i; - } - if( c=='\'' ){ - sqlite3_fputs("'", out); - continue; - } - if( c==0 ){ - break; - } - z++; - if( c=='\n' ){ - sqlite3_fputs(zNL, out); - continue; - } - sqlite3_fputs(zCR, out); - } - sqlite3_fputs("'", out); - if( nCR ){ - sqlite3_fprintf(out, ",'%s',char(13))", zCR); - } - if( nNL ){ - sqlite3_fprintf(out, ",'%s',char(10))", zNL); - } + zEscaped = sqlite3_mprintf("%#Q", z); } + sqlite3_fputs(zEscaped, p->out); + sqlite3_free(zEscaped); setCrlfMode(p); } @@ -2154,6 +2132,93 @@ static void output_json_string(FILE *out, const char *z, i64 n){ sqlite3_fputs(zq, out); } +/* +** Escape the input string if it is needed and in accordance with +** eEscMode. +** +** Escaping is needed if the string contains any control characters +** other than \t, \n, and \r\n +** +** If no escaping is needed (the common case) then set *ppFree to NULL +** and return the original string. If escapingn is needed, write the +** escaped string into memory obtained from sqlite3_malloc64() or the +** equivalent, and return the new string and set *ppFree to the new string +** as well. +** +** The caller is responsible for freeing *ppFree if it is non-NULL in order +** to reclaim memory. +*/ +static const char *escapeOutput( + ShellState *p, + const char *zInX, + char **ppFree +){ + i64 i, j; + i64 nCtrl = 0; + unsigned char *zIn; + unsigned char c; + unsigned char *zOut; + + + /* No escaping if disabled */ + if( p->eEscMode==SHELL_ESC_OFF ){ + *ppFree = 0; + return zInX; + } + + /* Count the number of control characters in the string. */ + zIn = (unsigned char*)zInX; + for(i=0; (c = zIn[i])!=0; i++){ + if( c<=0x1f + && c!='\t' + && c!='\n' + && (c!='\r' || zIn[i+1]!='\n') + ){ + nCtrl++; + } + } + if( nCtrl==0 ){ + *ppFree = 0; + return zInX; + } + if( p->eEscMode==SHELL_ESC_SYMBOL ) nCtrl *= 2; + zOut = sqlite3_malloc64( i + nCtrl + 1 ); + shell_check_oom(zOut); + for(i=j=0; (c = zIn[i])!=0; i++){ + if( c>0x1f + || c=='\t' + || c=='\n' + || (c=='\r' && zIn[i+1]=='\n') + ){ + continue; + } + if( i>0 ){ + memcpy(&zOut[j], zIn, i); + j += i; + } + zIn += i+1; + i = -1; + switch( p->eEscMode ){ + case SHELL_ESC_SYMBOL: + zOut[j++] = 0xe2; + zOut[j++] = 0x90; + zOut[j++] = 0x80+c; + break; + case SHELL_ESC_ASCII: + zOut[j++] = '^'; + zOut[j++] = 0x40+c; + break; + } + } + if( i>0 ){ + memcpy(&zOut[j], zIn, i); + j += i; + } + zOut[j] = 0; + *ppFree = (char*)zOut; + return (char*)zOut; +} + /* ** Output the given string with characters that are special to ** HTML escaped. @@ -2597,8 +2662,12 @@ static int shell_callback( } if( p->cnt++>0 ) sqlite3_fputs(p->rowSeparator, p->out); for(i=0; inullValue, &pFree); sqlite3_fprintf(p->out, "%*s = %s%s", w, azCol[i], - azArg[i] ? azArg[i] : p->nullValue, p->rowSeparator); + pDisplay, p->rowSeparator); + if( pFree ) sqlite3_free(pFree); } break; } @@ -2668,6 +2737,8 @@ static int shell_callback( char cEnd = 0; char c; int nLine = 0; + int isIndex; + int isWhere = 0; assert( nArg==1 ); if( azArg[0]==0 ) break; if( sqlite3_strlike("CREATE VIEW%", azArg[0], 0)==0 @@ -2676,6 +2747,8 @@ static int shell_callback( sqlite3_fprintf(p->out, "%s;\n", azArg[0]); break; } + isIndex = sqlite3_strlike("CREATE INDEX%", azArg[0], 0)==0 + || sqlite3_strlike("CREATE UNIQUE INDEX%", azArg[0], 0)==0; z = sqlite3_mprintf("%s", azArg[0]); shell_check_oom(z); j = 0; @@ -2705,14 +2778,26 @@ static int shell_callback( nParen++; }else if( c==')' ){ nParen--; - if( nLine>0 && nParen==0 && j>0 ){ + if( nLine>0 && nParen==0 && j>0 && !isWhere ){ printSchemaLineN(p->out, z, j, "\n"); j = 0; } + }else if( (c=='w' || c=='W') + && nParen==0 && isIndex + && sqlite3_strnicmp("WHERE",&z[i],5)==0 + && !IsAlnum(z[i+5]) && z[i+5]!='_' ){ + isWhere = 1; + }else if( isWhere && (c=='A' || c=='a') + && nParen==0 + && sqlite3_strnicmp("AND",&z[i],3)==0 + && !IsAlnum(z[i+3]) && z[i+3]!='_' ){ + printSchemaLineN(p->out, z, j, "\n "); + j = 0; } z[j++] = c; if( nParen==1 && cEnd==0 && (c=='(' || c=='\n' || (c==',' && !wsToEol(z+i+1))) + && !isWhere ){ if( c=='\n' ) j--; printSchemaLineN(p->out, z, j, "\n "); @@ -2730,15 +2815,23 @@ static int shell_callback( case MODE_List: { if( p->cnt++==0 && p->showHeader ){ for(i=0; iout, "%s%s", azCol[i], + char *z = azCol[i]; + char *pFree; + const char *zOut = escapeOutput(p, z, &pFree); + sqlite3_fprintf(p->out, "%s%s", zOut, i==nArg-1 ? p->rowSeparator : p->colSeparator); + if( pFree ) sqlite3_free(pFree); } } if( azArg==0 ) break; for(i=0; inullValue; - sqlite3_fputs(z, p->out); + zOut = escapeOutput(p, z, &pFree); + sqlite3_fputs(zOut, p->out); + if( pFree ) sqlite3_free(pFree); sqlite3_fputs((icolSeparator : p->rowSeparator, p->out); } break; @@ -3857,6 +3950,7 @@ static void print_box_row_separator( ** the last line, write a NULL into *pzTail. (*pzTail is not allocated.) */ static char *translateForDisplayAndDup( + ShellState *p, /* To access current settings */ const unsigned char *z, /* Input text to be transformed */ const unsigned char **pzTail, /* OUT: Tail of the input for next line */ int mxWidth, /* Max width. 0 means no limit */ @@ -3891,6 +3985,7 @@ static char *translateForDisplayAndDup( j++; continue; } + if( c==0 || c=='\n' || (c=='\r' && z[i+1]=='\n') ) break; if( c=='\t' ){ do{ n++; @@ -3899,16 +3994,23 @@ static char *translateForDisplayAndDup( i++; continue; } - break; + if( c==0x1b && p->eEscMode==SHELL_ESC_OFF && (k = isVt100(&z[i]))>0 ){ + i += k; + j += k; + }else{ + n++; + j += 3; + i++; + } } if( n>=mxWidth && bWordWrap ){ /* Perhaps try to back up to a better place to break the line */ for(k=i; k>i/2; k--){ - if( isspace(z[k-1]) ) break; + if( IsSpace(z[k-1]) ) break; } if( k<=i/2 ){ for(k=i; k>i/2; k--){ - if( isalnum(z[k-1])!=isalnum(z[k]) && (z[k]&0xc0)!=0x80 ) break; + if( IsAlnum(z[k-1])!=IsAlnum(z[k]) && (z[k]&0xc0)!=0x80 ) break; } } if( k<=i/2 ){ @@ -3946,6 +4048,7 @@ static char *translateForDisplayAndDup( zOut[j++] = z[i++]; continue; } + if( c==0 ) break; if( z[i]=='\t' ){ do{ n++; @@ -3954,12 +4057,44 @@ static char *translateForDisplayAndDup( i++; continue; } - break; + switch( p->eEscMode ){ + case SHELL_ESC_SYMBOL: + zOut[j++] = 0xe2; + zOut[j++] = 0x90; + zOut[j++] = 0x80 + c; + break; + case SHELL_ESC_ASCII: + zOut[j++] = '^'; + zOut[j++] = 0x40 + c; + break; + case SHELL_ESC_OFF: { + int nn; + if( c==0x1b && (nn = isVt100(&z[i]))>0 ){ + memcpy(&zOut[j], &z[i], nn); + j += nn; + i += nn - 1; + }else{ + zOut[j++] = c; + } + break; + } + } + i++; } zOut[j] = 0; return (char*)zOut; } +/* Return true if the text string z[] contains characters that need +** unistr() escaping. +*/ +static int needUnistr(const unsigned char *z){ + unsigned char c; + if( z==0 ) return 0; + while( (c = *z)>0x1f || c=='\t' || c=='\n' || (c=='\r' && z[1]=='\n') ){ z++; } + return c!=0; +} + /* Extract the value of the i-th current column for pStmt as an SQL literal ** value. Memory is obtained from sqlite3_malloc64() and must be freed by ** the caller. @@ -3974,7 +4109,8 @@ static char *quoted_column(sqlite3_stmt *pStmt, int i){ return sqlite3_mprintf("%s",sqlite3_column_text(pStmt,i)); } case SQLITE_TEXT: { - return sqlite3_mprintf("%Q",sqlite3_column_text(pStmt,i)); + const unsigned char *zText = sqlite3_column_text(pStmt,i); + return sqlite3_mprintf(needUnistr(zText)?"%#Q":"%Q",zText); } case SQLITE_BLOB: { int j; @@ -4066,7 +4202,7 @@ static void exec_prepared_stmt_columnar( if( wx<0 ) wx = -wx; uz = (const unsigned char*)sqlite3_column_name(pStmt,i); if( uz==0 ) uz = (u8*)""; - azData[i] = translateForDisplayAndDup(uz, &zNotUsed, wx, bw); + azData[i] = translateForDisplayAndDup(p, uz, &zNotUsed, wx, bw); } do{ int useNextLine = bNextLine; @@ -4090,6 +4226,7 @@ static void exec_prepared_stmt_columnar( uz = azNextLine[i]; if( uz==0 ) uz = (u8*)zEmpty; }else if( p->cmOpts.bQuote ){ + assert( azQuoted!=0 ); sqlite3_free(azQuoted[i]); azQuoted[i] = quoted_column(pStmt,i); uz = (const unsigned char*)azQuoted[i]; @@ -4098,7 +4235,7 @@ static void exec_prepared_stmt_columnar( if( uz==0 ) uz = (u8*)zShowNull; } azData[nRow*nColumn + i] - = translateForDisplayAndDup(uz, &azNextLine[i], wx, bw); + = translateForDisplayAndDup(p, uz, &azNextLine[i], wx, bw); if( azNextLine[i] ){ bNextLine = 1; abRowDiv[nRow-1] = 0; @@ -5021,7 +5158,7 @@ static const char *(azHelp[]) = { #else ".log on|off Turn logging on or off.", #endif - ".mode MODE ?OPTIONS? Set output mode", + ".mode ?MODE? ?OPTIONS? Set output mode", " MODE is one of:", " ascii Columns/rows delimited by 0x1F and 0x1E", " box Tables using unicode box-drawing characters", @@ -5039,6 +5176,7 @@ static const char *(azHelp[]) = { " tabs Tab-separated values", " tcl TCL list elements", " OPTIONS: (for columnar modes or insert mode):", + " --escape T ctrl-char escape; T is one of: symbol, ascii, off", " --wrap N Wrap output lines to no longer than N characters", " --wordwrap B Wrap or not at word boundaries per B (on/off)", " --ww Shorthand for \"--wordwrap 1\"", @@ -5076,6 +5214,7 @@ static const char *(azHelp[]) = { #ifndef SQLITE_SHELL_FIDDLE ".output ?FILE? Send output to FILE or stdout if FILE is omitted", " If FILE begins with '|' then open it as a pipe.", + " If FILE is 'off' then output is disabled.", " Options:", " --bom Prefix output with a UTF8 byte-order mark", " -e Send output to the system text editor", @@ -5201,103 +5340,104 @@ static const char *(azHelp[]) = { }; /* -** Output help text. +** Output help text for commands that match zPattern. +** +** * If zPattern is NULL, then show all documented commands, but +** only give a one-line summary of each. ** -** zPattern describes the set of commands for which help text is provided. -** If zPattern is NULL, then show all commands, but only give a one-line -** description of each. +** * If zPattern is "-a" or "-all" or "--all" then show all help text +** for all commands except undocumented commands. ** -** Return the number of matches. +** * If zPattern is "0" then show all help for undocumented commands. +** Undocumented commands begin with "," instead of "." in the azHelp[] +** array. +** +** * If zPattern is a prefix for one or more documented commands, then +** show help for those commands. If only a single command matches the +** prefix, show the full text of the help. If multiple commands match, +** Only show just the first line of each. +** +** * Otherwise, show the complete text of any documented command for which +** zPattern is a LIKE match for any text within that command help +** text. +** +** Return the number commands that match zPattern. */ static int showHelp(FILE *out, const char *zPattern){ int i = 0; int j = 0; int n = 0; char *zPat; - if( zPattern==0 - || zPattern[0]=='0' - || cli_strcmp(zPattern,"-a")==0 - || cli_strcmp(zPattern,"-all")==0 - || cli_strcmp(zPattern,"--all")==0 + if( zPattern==0 ){ + /* Show just the first line for all help topics */ + zPattern = "[a-z]"; + }else if( cli_strcmp(zPattern,"-a")==0 + || cli_strcmp(zPattern,"-all")==0 + || cli_strcmp(zPattern,"--all")==0 ){ - enum HelpWanted { HW_NoCull = 0, HW_SummaryOnly = 1, HW_Undoc = 2 }; - enum HelpHave { HH_Undoc = 2, HH_Summary = 1, HH_More = 0 }; - /* Show all or most commands - ** *zPattern==0 => summary of documented commands only - ** *zPattern=='0' => whole help for undocumented commands - ** Otherwise => whole help for documented commands - */ - enum HelpWanted hw = HW_SummaryOnly; - enum HelpHave hh = HH_More; - if( zPattern!=0 ){ - hw = (*zPattern=='0')? HW_NoCull|HW_Undoc : HW_NoCull; - } + /* Show everything except undocumented commands */ + zPattern = "."; + }else if( cli_strcmp(zPattern,"0")==0 ){ + /* Show complete help text of undocumented commands */ + int show = 0; for(i=0; ipLog; + UNUSED_PARAMETER(nVal); + zName = (const char*)sqlite3_value_text(apVal[0]); + + /* Temporarily disable the ".log" when calling shellFakeSchema() because + ** shellFakeSchema() might generate failures for some ephemeral virtual + ** tables due to missing arguments. Example: fts4aux. + ** https://sqlite.org/forum/forumpost/42fe6520b803be51 */ + p->pLog = 0; + zFake = zName? shellFakeSchema(sqlite3_context_db_handle(pCtx), 0, zName) : 0; + p->pLog = pSavedLog; + + if( zFake ){ + sqlite3_result_text(pCtx, sqlite3_mprintf("/* %s */", zFake), + -1, sqlite3_free); + free(zFake); + } +} + /* Flags for open_db(). ** ** The default behavior of open_db() is to exit(1) if the database fails to @@ -5690,7 +5863,7 @@ static void open_db(ShellState *p, int openFlags){ shellDtostr, 0, 0); sqlite3_create_function(p->db, "shell_add_schema", 3, SQLITE_UTF8, 0, shellAddSchemaName, 0, 0); - sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, 0, + sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, p, shellModuleSchema, 0, 0); sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p, shellPutsFunc, 0, 0); @@ -5814,7 +5987,7 @@ static void linenoise_completion( #endif if( nLine>(i64)sizeof(zBuf)-30 ) return; if( zLine[0]=='.' || zLine[0]=='#') return; - for(i=nLine-1; i>=0 && (isalnum(zLine[i]) || zLine[i]=='_'); i--){} + for(i=nLine-1; i>=0 && (IsAlnum(zLine[i]) || zLine[i]=='_'); i--){} if( i==nLine-1 ) return; iStart = i+1; memcpy(zBuf, zLine, iStart); @@ -6693,7 +6866,7 @@ static int shell_dbtotxt_command(ShellState *p, int nArg, char **azArg){ for(j=0; j<16 && aLine[j]==0; j++){} if( j==16 ) continue; if( !seenPageLabel ){ - sqlite3_fprintf(p->out, "| page %lld offset %lld\n", pgno, pgno*pgSz); + sqlite3_fprintf(p->out, "| page %lld offset %lld\n",pgno,(pgno-1)*pgSz); seenPageLabel = 1; } sqlite3_fprintf(p->out, "| %5d:", i); @@ -8054,6 +8227,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ sqlite3_recover_config(p, SQLITE_RECOVER_ROWIDS, (void*)&bRowids); sqlite3_recover_config(p, SQLITE_RECOVER_FREELIST_CORRUPT,(void*)&bFreelist); + sqlite3_fprintf(pState->out, ".dbconfig defensive off\n"); sqlite3_recover_run(p); if( sqlite3_recover_errcode(p)!=SQLITE_OK ){ const char *zErr = sqlite3_recover_errmsg(p); @@ -9779,24 +9953,52 @@ static int do_meta_command(char *zLine, ShellState *p){ const char *zMode = 0; const char *zTabname = 0; int i, n2; + int chng = 0; /* 0x01: change to cmopts. 0x02: Any other change */ ColModeOpts cmOpts = ColModeOpts_default; for(i=1; ieEscMode = k; + chng |= 2; + break; + } + } + if( k>=ArraySize(shell_EscModeNames) ){ + sqlite3_fprintf(stderr, "unknown control character escape mode \"%s\"" + " - choices:", zEsc); + for(k=0; kmode==MODE_Column || (p->mode>=MODE_Markdown && p->mode<=MODE_Box) ){ sqlite3_fprintf(p->out, - "current output mode: %s --wrap %d --wordwrap %s --%squote\n", + "current output mode: %s --wrap %d --wordwrap %s " + "--%squote --escape %s\n", modeDescr[p->mode], p->cmOpts.iWrap, p->cmOpts.bWordWrap ? "on" : "off", - p->cmOpts.bQuote ? "" : "no"); + p->cmOpts.bQuote ? "" : "no", + shell_EscModeNames[p->eEscMode] + ); }else{ sqlite3_fprintf(p->out, - "current output mode: %s\n", modeDescr[p->mode]); + "current output mode: %s --escape %s\n", + modeDescr[p->mode], + shell_EscModeNames[p->eEscMode] + ); } + } + if( zMode==0 ){ zMode = modeDescr[p->mode]; + if( (chng&1)==0 ) cmOpts = p->cmOpts; } n2 = strlen30(zMode); if( cli_strncmp(zMode,"lines",n2)==0 ){ @@ -9866,6 +10078,11 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( cli_strncmp(zMode,"insert",n2)==0 ){ p->mode = MODE_Insert; set_table_name(p, zTabname ? zTabname : "table"); + if( p->eEscMode==SHELL_ESC_OFF ){ + ShellSetFlag(p, SHFLG_Newlines); + }else{ + ShellClearFlag(p, SHFLG_Newlines); + } }else if( cli_strncmp(zMode,"quote",n2)==0 ){ p->mode = MODE_Quote; sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); @@ -10027,9 +10244,9 @@ static int do_meta_command(char *zLine, ShellState *p){ ){ char *zFile = 0; int i; - int eMode = 0; - int bOnce = 0; /* 0: .output, 1: .once, 2: .excel/.www */ - int bPlain = 0; /* --plain option */ + int eMode = 0; /* 0: .outout/.once, 'x'=.excel, 'w'=.www */ + int bOnce = 0; /* 0: .output, 1: .once, 2: .excel/.www */ + int bPlain = 0; /* --plain option */ static const char *zBomUtf8 = "\357\273\277"; const char *zBom = 0; @@ -10058,14 +10275,22 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( c=='o' && cli_strcmp(z,"-w")==0 ){ eMode = 'w'; /* Web browser */ }else{ - sqlite3_fprintf(p->out, + sqlite3_fprintf(p->out, "ERROR: unknown option: \"%s\". Usage:\n", azArg[i]); showHelp(p->out, azArg[0]); rc = 1; goto meta_command_exit; } }else if( zFile==0 && eMode==0 ){ - zFile = sqlite3_mprintf("%s", z); + if( cli_strcmp(z, "off")==0 ){ +#ifdef _WIN32 + zFile = sqlite3_mprintf("nul"); +#else + zFile = sqlite3_mprintf("/dev/null"); +#endif + }else{ + zFile = sqlite3_mprintf("%s", z); + } if( zFile && zFile[0]=='|' ){ while( i+10 && '\r'==zBegin[nZ-1]){ @@ -12586,6 +12811,7 @@ static const char zOptions[] = " -deserialize open the database using sqlite3_deserialize()\n" #endif " -echo print inputs before execution\n" + " -escape T ctrl-char escape; T is one of: symbol, ascii, off\n" " -init FILENAME read/process named file\n" " -[no]header turn headers on or off\n" #if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) @@ -12633,7 +12859,7 @@ static const char zOptions[] = #endif ; static void usage(int showDetail){ - sqlite3_fprintf(stderr,"Usage: %s [OPTIONS] [FILENAME [SQL]]\n" + sqlite3_fprintf(stderr,"Usage: %s [OPTIONS] [FILENAME [SQL...]]\n" "FILENAME is the name of an SQLite database. A new database is created\n" "if the file does not previously exist. Defaults to :memory:.\n", Argv0); if( showDetail ){ @@ -13019,6 +13245,9 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ ShellSetFlag(&data,SHFLG_TestingMode); }else if( cli_strcmp(z,"-safe")==0 ){ /* no-op - catch this on the second pass */ + }else if( cli_strcmp(z,"-escape")==0 && i+1=ArraySize(shell_EscModeNames) ){ + sqlite3_fprintf(stderr, "unknown control character escape mode \"%s\"" + " - choices:", zEsc); + for(k=0; kFossil configuration management +** Fossil configuration management ** system. ^The SQLITE_SOURCE_ID macro evaluates to ** a string which identifies a particular check-in of SQLite ** within its configuration management system. ^The SQLITE_SOURCE_ID @@ -1163,6 +1163,12 @@ struct sqlite3_io_methods { ** the value that M is to be set to. Before returning, the 32-bit signed ** integer is overwritten with the previous value of M. ** +**
  14. [[SQLITE_FCNTL_BLOCK_ON_CONNECT]] +** The [SQLITE_FCNTL_BLOCK_ON_CONNECT] opcode is used to configure the +** VFS to block when taking a SHARED lock to connect to a wal mode database. +** This is used to implement the functionality associated with +** SQLITE_SETLK_BLOCK_ON_CONNECT. +** **
  15. [[SQLITE_FCNTL_DATA_VERSION]] ** The [SQLITE_FCNTL_DATA_VERSION] opcode is used to detect changes to ** a database file. The argument is a pointer to a 32-bit unsigned integer. @@ -1259,6 +1265,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_CKSM_FILE 41 #define SQLITE_FCNTL_RESET_CACHE 42 #define SQLITE_FCNTL_NULL_IO 43 +#define SQLITE_FCNTL_BLOCK_ON_CONNECT 44 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE @@ -1989,13 +1996,16 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_CONFIG_LOOKASIDE]]
    SQLITE_CONFIG_LOOKASIDE
    **
    ^(The SQLITE_CONFIG_LOOKASIDE option takes two arguments that determine -** the default size of lookaside memory on each [database connection]. +** the default size of [lookaside memory] on each [database connection]. ** The first argument is the -** size of each lookaside buffer slot and the second is the number of -** slots allocated to each database connection.)^ ^(SQLITE_CONFIG_LOOKASIDE -** sets the default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE] -** option to [sqlite3_db_config()] can be used to change the lookaside -** configuration on individual connections.)^
    +** size of each lookaside buffer slot ("sz") and the second is the number of +** slots allocated to each database connection ("cnt").)^ +** ^(SQLITE_CONFIG_LOOKASIDE sets the default lookaside size. +** The [SQLITE_DBCONFIG_LOOKASIDE] option to [sqlite3_db_config()] can +** be used to change the lookaside configuration on individual connections.)^ +** The [-DSQLITE_DEFAULT_LOOKASIDE] option can be used to change the +** default lookaside configuration at compile-time. +** ** ** [[SQLITE_CONFIG_PCACHE2]]
    SQLITE_CONFIG_PCACHE2
    **
    ^(The SQLITE_CONFIG_PCACHE2 option takes a single argument which is @@ -2232,31 +2242,50 @@ struct sqlite3_mem_methods { ** [[SQLITE_DBCONFIG_LOOKASIDE]] **
    SQLITE_DBCONFIG_LOOKASIDE
    **
    The SQLITE_DBCONFIG_LOOKASIDE option is used to adjust the -** configuration of the lookaside memory allocator within a database +** configuration of the [lookaside memory allocator] within a database ** connection. ** The arguments to the SQLITE_DBCONFIG_LOOKASIDE option are not ** in the [DBCONFIG arguments|usual format]. ** The SQLITE_DBCONFIG_LOOKASIDE option takes three arguments, not two, ** so that a call to [sqlite3_db_config()] that uses SQLITE_DBCONFIG_LOOKASIDE ** should have a total of five parameters. -** ^The first argument (the third parameter to [sqlite3_db_config()] is a +**
      +**
    1. The first argument ("buf") is a ** pointer to a memory buffer to use for lookaside memory. -** ^The first argument after the SQLITE_DBCONFIG_LOOKASIDE verb -** may be NULL in which case SQLite will allocate the -** lookaside buffer itself using [sqlite3_malloc()]. ^The second argument is the -** size of each lookaside buffer slot. ^The third argument is the number of -** slots. The size of the buffer in the first argument must be greater than -** or equal to the product of the second and third arguments. The buffer -** must be aligned to an 8-byte boundary. ^If the second argument to -** SQLITE_DBCONFIG_LOOKASIDE is not a multiple of 8, it is internally -** rounded down to the next smaller multiple of 8. ^(The lookaside memory +** The first argument may be NULL in which case SQLite will allocate the +** lookaside buffer itself using [sqlite3_malloc()]. +**

    2. The second argument ("sz") is the +** size of each lookaside buffer slot. Lookaside is disabled if "sz" +** is less than 8. The "sz" argument should be a multiple of 8 less than +** 65536. If "sz" does not meet this constraint, it is reduced in size until +** it does. +**

    3. The third argument ("cnt") is the number of slots. Lookaside is disabled +** if "cnt"is less than 1. The "cnt" value will be reduced, if necessary, so +** that the product of "sz" and "cnt" does not exceed 2,147,418,112. The "cnt" +** parameter is usually chosen so that the product of "sz" and "cnt" is less +** than 1,000,000. +**

    +**

    If the "buf" argument is not NULL, then it must +** point to a memory buffer with a size that is greater than +** or equal to the product of "sz" and "cnt". +** The buffer must be aligned to an 8-byte boundary. +** The lookaside memory ** configuration for a database connection can only be changed when that ** connection is not currently using lookaside memory, or in other words -** when the "current value" returned by -** [sqlite3_db_status](D,[SQLITE_DBSTATUS_LOOKASIDE_USED],...) is zero. +** when the value returned by [SQLITE_DBSTATUS_LOOKASIDE_USED] is zero. ** Any attempt to change the lookaside memory configuration when lookaside ** memory is in use leaves the configuration unchanged and returns -** [SQLITE_BUSY].)^

    +** [SQLITE_BUSY]. +** If the "buf" argument is NULL and an attempt +** to allocate memory based on "sz" and "cnt" fails, then +** lookaside is silently disabled. +**

    +** The [SQLITE_CONFIG_LOOKASIDE] configuration option can be used to set the +** default lookaside configuration at initialization. The +** [-DSQLITE_DEFAULT_LOOKASIDE] option can be used to set the default lookaside +** configuration at compile-time. Typical values for lookaside are 1200 for +** "sz" and 40 to 100 for "cnt". +** ** ** [[SQLITE_DBCONFIG_ENABLE_FKEY]] **

    SQLITE_DBCONFIG_ENABLE_FKEY
    @@ -2993,6 +3022,44 @@ int sqlite3_busy_handler(sqlite3*,int(*)(void*,int),void*); */ int sqlite3_busy_timeout(sqlite3*, int ms); +/* +** CAPI3REF: Set the Setlk Timeout +** METHOD: sqlite3 +** +** This routine is only useful in SQLITE_ENABLE_SETLK_TIMEOUT builds. If +** the VFS supports blocking locks, it sets the timeout in ms used by +** eligible locks taken on wal mode databases by the specified database +** handle. In non-SQLITE_ENABLE_SETLK_TIMEOUT builds, or if the VFS does +** not support blocking locks, this function is a no-op. +** +** Passing 0 to this function disables blocking locks altogether. Passing +** -1 to this function requests that the VFS blocks for a long time - +** indefinitely if possible. The results of passing any other negative value +** are undefined. +** +** Internally, each SQLite database handle store two timeout values - the +** busy-timeout (used for rollback mode databases, or if the VFS does not +** support blocking locks) and the setlk-timeout (used for blocking locks +** on wal-mode databases). The sqlite3_busy_timeout() method sets both +** values, this function sets only the setlk-timeout value. Therefore, +** to configure separate busy-timeout and setlk-timeout values for a single +** database handle, call sqlite3_busy_timeout() followed by this function. +** +** Whenever the number of connections to a wal mode database falls from +** 1 to 0, the last connection takes an exclusive lock on the database, +** then checkpoints and deletes the wal file. While it is doing this, any +** new connection that tries to read from the database fails with an +** SQLITE_BUSY error. Or, if the SQLITE_SETLK_BLOCK_ON_CONNECT flag is +** passed to this API, the new connection blocks until the exclusive lock +** has been released. +*/ +int sqlite3_setlk_timeout(sqlite3*, int ms, int flags); + +/* +** CAPI3REF: Flags for sqlite3_setlk_timeout() +*/ +#define SQLITE_SETLK_BLOCK_ON_CONNECT 0x01 + /* ** CAPI3REF: Convenience Routines For Running Queries ** METHOD: sqlite3 @@ -5108,7 +5175,7 @@ const void *sqlite3_column_decltype16(sqlite3_stmt*,int); ** other than [SQLITE_ROW] before any subsequent invocation of ** sqlite3_step(). Failure to reset the prepared statement using ** [sqlite3_reset()] would result in an [SQLITE_MISUSE] return from -** sqlite3_step(). But after [version 3.6.23.1] ([dateof:3.6.23.1], +** sqlite3_step(). But after [version 3.6.23.1] ([dateof:3.6.23.1]), ** sqlite3_step() began ** calling [sqlite3_reset()] automatically in this circumstance rather ** than returning [SQLITE_MISUSE]. This is not considered a compatibility @@ -7004,6 +7071,8 @@ int sqlite3_autovacuum_pages( ** ** ^The second argument is a pointer to the function to invoke when a ** row is updated, inserted or deleted in a rowid table. +** ^The update hook is disabled by invoking sqlite3_update_hook() +** with a NULL pointer as the second parameter. ** ^The first argument to the callback is a copy of the third argument ** to sqlite3_update_hook(). ** ^The second callback argument is one of [SQLITE_INSERT], [SQLITE_DELETE], diff --git a/src/sqlite3.rc b/src/sqlite3.rc index 5a856490d..aad468d34 100644 --- a/src/sqlite3.rc +++ b/src/sqlite3.rc @@ -70,7 +70,7 @@ BEGIN VALUE "FileDescription", "SQLite is a software library that implements a self-contained, serverless, zero-configuration, transactional SQL database engine." VALUE "FileVersion", SQLITE_VERSION VALUE "InternalName", "sqlite3" - VALUE "LegalCopyright", "http://www.sqlite.org/copyright.html" + VALUE "LegalCopyright", "http://sqlite.org/copyright.html" VALUE "ProductName", "SQLite" VALUE "ProductVersion", SQLITE_VERSION VALUE "SourceId", SQLITE_SOURCE_ID diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index ae0949baf..cf775dfbd 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -366,6 +366,8 @@ struct sqlite3_api_routines { /* Version 3.44.0 and later */ void *(*get_clientdata)(sqlite3*,const char*); int (*set_clientdata)(sqlite3*, const char*, void*, void(*)(void*)); + /* Version 3.50.0 and later */ + int (*setlk_timeout)(sqlite3*,int,int); }; /* @@ -699,6 +701,8 @@ typedef int (*sqlite3_loadext_entry)( /* Version 3.44.0 and later */ #define sqlite3_get_clientdata sqlite3_api->get_clientdata #define sqlite3_set_clientdata sqlite3_api->set_clientdata +/* Version 3.50.0 and later */ +#define sqlite3_setlk_timeout sqlite3_api->setlk_timeout #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) diff --git a/src/sqliteInt.h b/src/sqliteInt.h index fd4ed5298..41adccf69 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -765,7 +765,17 @@ ** ourselves. */ #ifndef offsetof -#define offsetof(STRUCTURE,FIELD) ((int)((char*)&((STRUCTURE*)0)->FIELD)) +#define offsetof(STRUCTURE,FIELD) ((size_t)((char*)&((STRUCTURE*)0)->FIELD)) +#endif + +/* +** Work around C99 "flex-array" syntax for pre-C99 compilers, so as +** to avoid complaints from -fsanitize=strict-bounds. +*/ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# define FLEXARRAY +#else +# define FLEXARRAY 1 #endif /* @@ -843,6 +853,11 @@ typedef INT16_TYPE i16; /* 2-byte signed integer */ typedef UINT8_TYPE u8; /* 1-byte unsigned integer */ typedef INT8_TYPE i8; /* 1-byte signed integer */ +/* A bitfield type for use inside of structures. Always follow with :N where +** N is the number of bits. +*/ +typedef unsigned bft; /* Bit Field Type */ + /* ** SQLITE_MAX_U32 is a u64 constant that is the maximum u64 value ** that can be stored in a u32 without loss of data. The value @@ -1011,6 +1026,14 @@ typedef INT16_TYPE LogEst; #define LARGEST_UINT64 (0xffffffff|(((u64)0xffffffff)<<32)) #define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) +/* +** Macro SMXV(n) return the maximum value that can be held in variable n, +** assuming n is a signed integer type. UMXV(n) is similar for unsigned +** integer types. +*/ +#define SMXV(n) ((((i64)1)<<(sizeof(n)-1))-1) +#define UMXV(n) ((((i64)1)<<(sizeof(n)))-1) + /* ** Round up a number to the next larger multiple of 8. This is used ** to force 8-byte alignment on 64-bit architectures. @@ -1748,6 +1771,10 @@ struct sqlite3 { Savepoint *pSavepoint; /* List of active savepoints */ int nAnalysisLimit; /* Number of index rows to ANALYZE */ int busyTimeout; /* Busy handler timeout, in msec */ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + int setlkTimeout; /* Blocking lock timeout, in msec. -1 -> inf. */ + int setlkFlags; /* Flags passed to setlk_timeout() */ +#endif int nSavepoint; /* Number of non-transaction savepoints */ int nStatement; /* Number of nested statement-transactions */ i64 nDeferredCons; /* Net deferred constraints this transaction. */ @@ -2302,6 +2329,7 @@ struct CollSeq { #define SQLITE_AFF_INTEGER 0x44 /* 'D' */ #define SQLITE_AFF_REAL 0x45 /* 'E' */ #define SQLITE_AFF_FLEXNUM 0x46 /* 'F' */ +#define SQLITE_AFF_DEFER 0x58 /* 'X' - defer computation until later */ #define sqlite3IsNumericAffinity(X) ((X)>=SQLITE_AFF_NUMERIC) @@ -2426,6 +2454,7 @@ struct Table { } u; Trigger *pTrigger; /* List of triggers on this object */ Schema *pSchema; /* Schema that contains this table */ + u8 aHx[16]; /* Column aHt[K%sizeof(aHt)] might have hash K */ }; /* @@ -2559,9 +2588,13 @@ struct FKey { struct sColMap { /* Mapping of columns in pFrom to columns in zTo */ int iFrom; /* Index of column in pFrom */ char *zCol; /* Name of column in zTo. If NULL use PRIMARY KEY */ - } aCol[1]; /* One entry for each of nCol columns */ + } aCol[FLEXARRAY]; /* One entry for each of nCol columns */ }; +/* The size (in bytes) of an FKey object holding N columns. The answer +** does NOT include space to hold the zTo name. */ +#define SZ_FKEY(N) (offsetof(FKey,aCol)+(N)*sizeof(struct sColMap)) + /* ** SQLite supports many different ways to resolve a constraint ** error. ROLLBACK processing means that a constraint violation @@ -2623,9 +2656,12 @@ struct KeyInfo { u16 nAllField; /* Total columns, including key plus others */ sqlite3 *db; /* The database connection */ u8 *aSortFlags; /* Sort order for each column. */ - CollSeq *aColl[1]; /* Collating sequence for each term of the key */ + CollSeq *aColl[FLEXARRAY]; /* Collating sequence for each term of the key */ }; +/* The size (in bytes) of a KeyInfo object with up to N fields */ +#define SZ_KEYINFO(N) (offsetof(KeyInfo,aColl) + (N)*sizeof(CollSeq*)) + /* ** Allowed bit values for entries in the KeyInfo.aSortFlags[] array. */ @@ -2745,7 +2781,7 @@ struct Index { Pgno tnum; /* DB Page containing root of this index */ LogEst szIdxRow; /* Estimated average row size in bytes */ u16 nKeyCol; /* Number of columns forming the key */ - u16 nColumn; /* Number of columns stored in the index */ + u16 nColumn; /* Nr columns in btree. Can be 2*Table.nCol */ u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ unsigned idxType:2; /* 0:Normal 1:UNIQUE, 2:PRIMARY KEY, 3:IPK */ unsigned bUnordered:1; /* Use this index for == or IN queries only */ @@ -2754,7 +2790,6 @@ struct Index { unsigned isCovering:1; /* True if this is a covering index */ unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ unsigned hasStat1:1; /* aiRowLogEst values come from sqlite_stat1 */ - unsigned bLowQual:1; /* sqlite_stat1 says this is a low-quality index */ unsigned bNoQuery:1; /* Do not use this index to optimize queries */ unsigned bAscKeyBug:1; /* True if the bba7b69f9849b5bf bug applies */ unsigned bIdxRowid:1; /* One or more of the index keys is the ROWID */ @@ -3084,10 +3119,10 @@ struct Expr { /* Macros can be used to test, set, or clear bits in the ** Expr.flags field. */ -#define ExprHasProperty(E,P) (((E)->flags&(P))!=0) -#define ExprHasAllProperty(E,P) (((E)->flags&(P))==(P)) -#define ExprSetProperty(E,P) (E)->flags|=(P) -#define ExprClearProperty(E,P) (E)->flags&=~(P) +#define ExprHasProperty(E,P) (((E)->flags&(u32)(P))!=0) +#define ExprHasAllProperty(E,P) (((E)->flags&(u32)(P))==(u32)(P)) +#define ExprSetProperty(E,P) (E)->flags|=(u32)(P) +#define ExprClearProperty(E,P) (E)->flags&=~(u32)(P) #define ExprAlwaysTrue(E) (((E)->flags&(EP_OuterON|EP_IsTrue))==EP_IsTrue) #define ExprAlwaysFalse(E) (((E)->flags&(EP_OuterON|EP_IsFalse))==EP_IsFalse) #define ExprIsFullSize(E) (((E)->flags&(EP_Reduced|EP_TokenOnly))==0) @@ -3199,9 +3234,14 @@ struct ExprList { int iConstExprReg; /* Register in which Expr value is cached. Used only ** by Parse.pConstExpr */ } u; - } a[1]; /* One slot for each expression in the list */ + } a[FLEXARRAY]; /* One slot for each expression in the list */ }; +/* The size (in bytes) of an ExprList object that is big enough to hold +** as many as N expressions. */ +#define SZ_EXPRLIST(N) \ + (offsetof(ExprList,a) + (N)*sizeof(struct ExprList_item)) + /* ** Allowed values for Expr.a.eEName */ @@ -3229,9 +3269,12 @@ struct IdList { int nId; /* Number of identifiers on the list */ struct IdList_item { char *zName; /* Name of the identifier */ - } a[1]; + } a[FLEXARRAY]; }; +/* The size (in bytes) of an IdList object that can hold up to N IDs. */ +#define SZ_IDLIST(N) (offsetof(IdList,a)+(N)*sizeof(struct IdList_item)) + /* ** Allowed values for IdList.eType, which determines which value of the a.u4 ** is valid. @@ -3351,11 +3394,19 @@ struct OnOrUsing { ** */ struct SrcList { - int nSrc; /* Number of tables or subqueries in the FROM clause */ - u32 nAlloc; /* Number of entries allocated in a[] below */ - SrcItem a[1]; /* One entry for each identifier on the list */ + int nSrc; /* Number of tables or subqueries in the FROM clause */ + u32 nAlloc; /* Number of entries allocated in a[] below */ + SrcItem a[FLEXARRAY]; /* One entry for each identifier on the list */ }; +/* Size (in bytes) of a SrcList object that can hold as many as N +** SrcItem objects. */ +#define SZ_SRCLIST(N) (offsetof(SrcList,a)+(N)*sizeof(SrcItem)) + +/* Size (in bytes( of a SrcList object that holds 1 SrcItem. This is a +** special case of SZ_SRCITEM(1) that comes up often. */ +#define SZ_SRCLIST_1 (offsetof(SrcList,a)+sizeof(SrcItem)) + /* ** Permitted values of the SrcList.a.jointype field */ @@ -3824,25 +3875,32 @@ struct Parse { char *zErrMsg; /* An error message */ Vdbe *pVdbe; /* An engine for executing database bytecode */ int rc; /* Return code from execution */ - u8 colNamesSet; /* TRUE after OP_ColumnName has been issued to pVdbe */ - u8 checkSchema; /* Causes schema cookie check after an error */ + LogEst nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ u8 nested; /* Number of nested calls to the parser/code generator */ u8 nTempReg; /* Number of temporary registers in aTempReg[] */ u8 isMultiWrite; /* True if statement may modify/insert multiple rows */ u8 mayAbort; /* True if statement may throw an ABORT exception */ u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ - u8 okConstFactor; /* OK to factor out constants */ u8 disableLookaside; /* Number of times lookaside has been disabled */ u8 prepFlags; /* SQLITE_PREPARE_* flags */ u8 withinRJSubrtn; /* Nesting level for RIGHT JOIN body subroutines */ - u8 bHasWith; /* True if statement contains WITH */ u8 mSubrtnSig; /* mini Bloom filter on available SubrtnSig.selId */ + u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ + u8 bReturning; /* Coding a RETURNING trigger */ + u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ + u8 disableTriggers; /* True to disable triggers */ #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) u8 earlyCleanup; /* OOM inside sqlite3ParserAddCleanup() */ #endif #ifdef SQLITE_DEBUG u8 ifNotExists; /* Might be true if IF NOT EXISTS. Assert()s only */ + u8 isCreate; /* CREATE TABLE, INDEX, or VIEW (but not TRIGGER) + ** and ALTER TABLE ADD COLUMN. */ #endif + bft colNamesSet :1; /* TRUE after OP_ColumnName has been issued to pVdbe */ + bft bHasWith :1; /* True if statement contains WITH */ + bft okConstFactor :1; /* OK to factor out constants */ + bft checkSchema :1; /* Causes schema cookie check after an error */ int nRangeReg; /* Size of the temporary register block */ int iRangeReg; /* First register in temporary register block */ int nErr; /* Number of errors seen */ @@ -3857,12 +3915,9 @@ struct Parse { ExprList *pConstExpr;/* Constant expressions */ IndexedExpr *pIdxEpr;/* List of expressions used by active indexes */ IndexedExpr *pIdxPartExpr; /* Exprs constrained by index WHERE clauses */ - Token constraintName;/* Name of the constraint currently being parsed */ yDbMask writeMask; /* Start a write transaction on these databases */ yDbMask cookieMask; /* Bitmask of schema verified databases */ - int regRowid; /* Register holding rowid of CREATE TABLE entry */ - int regRoot; /* Register holding root page number for new objects */ - int nMaxArg; /* Max args passed to user function by sub-program */ + int nMaxArg; /* Max args to xUpdate and xFilter vtab methods */ int nSelect; /* Number of SELECT stmts. Counter for Select.selId */ #ifndef SQLITE_OMIT_PROGRESS_CALLBACK u32 nProgressSteps; /* xProgress steps taken during sqlite3_prepare() */ @@ -3876,17 +3931,6 @@ struct Parse { Table *pTriggerTab; /* Table triggers are being coded for */ TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ ParseCleanup *pCleanup; /* List of cleanup operations to run after parse */ - union { - int addrCrTab; /* Address of OP_CreateBtree on CREATE TABLE */ - Returning *pReturning; /* The RETURNING clause */ - } u1; - u32 oldmask; /* Mask of old.* columns referenced */ - u32 newmask; /* Mask of new.* columns referenced */ - LogEst nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ - u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ - u8 bReturning; /* Coding a RETURNING trigger */ - u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ - u8 disableTriggers; /* True to disable triggers */ /************************************************************************** ** Fields above must be initialized to zero. The fields that follow, @@ -3898,6 +3942,19 @@ struct Parse { int aTempReg[8]; /* Holding area for temporary registers */ Parse *pOuterParse; /* Outer Parse object when nested */ Token sNameToken; /* Token with unqualified schema object name */ + u32 oldmask; /* Mask of old.* columns referenced */ + u32 newmask; /* Mask of new.* columns referenced */ + union { + struct { /* These fields available when isCreate is true */ + int addrCrTab; /* Address of OP_CreateBtree on CREATE TABLE */ + int regRowid; /* Register holding rowid of CREATE TABLE entry */ + int regRoot; /* Register holding root page for new objects */ + Token constraintName; /* Name of the constraint currently being parsed */ + } cr; + struct { /* These fields available to all other statements */ + Returning *pReturning; /* The RETURNING clause */ + } d; + } u1; /************************************************************************ ** Above is constant between recursions. Below is reset before and after @@ -4413,9 +4470,13 @@ struct With { int nCte; /* Number of CTEs in the WITH clause */ int bView; /* Belongs to the outermost Select of a view */ With *pOuter; /* Containing WITH clause, or NULL */ - Cte a[1]; /* For each CTE in the WITH clause.... */ + Cte a[FLEXARRAY]; /* For each CTE in the WITH clause.... */ }; +/* The size (in bytes) of a With object that can hold as many +** as N different CTEs. */ +#define SZ_WITH(N) (offsetof(With,a) + (N)*sizeof(Cte)) + /* ** The Cte object is not guaranteed to persist for the entire duration ** of code generation. (The query flattener or other parser tree @@ -4444,9 +4505,13 @@ struct DbClientData { DbClientData *pNext; /* Next in a linked list */ void *pData; /* The data */ void (*xDestructor)(void*); /* Destructor. Might be NULL */ - char zName[1]; /* Name of this client data. MUST BE LAST */ + char zName[FLEXARRAY]; /* Name of this client data. MUST BE LAST */ }; +/* The size (in bytes) of a DbClientData object that can has a name +** that is N bytes long, including the zero-terminator. */ +#define SZ_DBCLIENTDATA(N) (offsetof(DbClientData,zName)+(N)) + #ifdef SQLITE_DEBUG /* ** An instance of the TreeView object is used for printing the content of @@ -4889,7 +4954,7 @@ void sqlite3SubqueryColumnTypes(Parse*,Table*,Select*,char); Table *sqlite3ResultSetOfSelect(Parse*,Select*,char); void sqlite3OpenSchemaTable(Parse *, int); Index *sqlite3PrimaryKeyIndex(Table*); -i16 sqlite3TableColumnToIndex(Index*, i16); +int sqlite3TableColumnToIndex(Index*, int); #ifdef SQLITE_OMIT_GENERATED_COLUMNS # define sqlite3TableColumnToStorage(T,X) (X) /* No-op pass-through */ # define sqlite3StorageColumnToTable(T,X) (X) /* No-op pass-through */ @@ -4987,7 +5052,7 @@ void sqlite3SrcListAssignCursors(Parse*, SrcList*); void sqlite3IdListDelete(sqlite3*, IdList*); void sqlite3ClearOnOrUsing(sqlite3*, OnOrUsing*); void sqlite3SrcListDelete(sqlite3*, SrcList*); -Index *sqlite3AllocateIndexObject(sqlite3*,i16,int,char**); +Index *sqlite3AllocateIndexObject(sqlite3*,int,int,char**); void sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*, Expr*, int, int, u8); void sqlite3DropIndex(Parse*, SrcList*, int); @@ -5123,7 +5188,8 @@ Select *sqlite3SelectDup(sqlite3*,const Select*,int); FuncDef *sqlite3FunctionSearch(int,const char*); void sqlite3InsertBuiltinFuncs(FuncDef*,int); FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8); -void sqlite3QuoteValue(StrAccum*,sqlite3_value*); +void sqlite3QuoteValue(StrAccum*,sqlite3_value*,int); +int sqlite3AppendOneUtf8Character(char*, u32); void sqlite3RegisterBuiltinFunctions(void); void sqlite3RegisterDateTimeFunctions(void); void sqlite3RegisterJsonFunctions(void); diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index 620b0f8e5..ec774889b 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -36,14 +36,22 @@ ** * Terms in the GROUP BY or ORDER BY clauses of a SELECT statement. ** * Terms in the VALUES clause of an INSERT statement ** -** The hard upper limit here is 32676. Most database people will +** The hard upper limit here is 32767. Most database people will ** tell you that in a well-normalized database, you usually should ** not have more than a dozen or so columns in any table. And if ** that is the case, there is no point in having more than a few ** dozen values in any of the other situations described above. +** +** An index can only have SQLITE_MAX_COLUMN columns from the user +** point of view, but the underlying b-tree that implements the index +** might have up to twice as many columns in a WITHOUT ROWID table, +** since must also store the primary key at the end. Hence the +** column count for Index is u16 instead of i16. */ -#ifndef SQLITE_MAX_COLUMN +#if !defined(SQLITE_MAX_COLUMN) # define SQLITE_MAX_COLUMN 2000 +#elif SQLITE_MAX_COLUMN>32767 +# error SQLITE_MAX_COLUMN may not exceed 32767 #endif /* diff --git a/src/status.c b/src/status.c index a462c9429..b0a47c7f8 100644 --- a/src/status.c +++ b/src/status.c @@ -192,8 +192,9 @@ int sqlite3LookasideUsed(sqlite3 *db, int *pHighwater){ nInit += countLookasideSlots(db->lookaside.pSmallInit); nFree += countLookasideSlots(db->lookaside.pSmallFree); #endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ - if( pHighwater ) *pHighwater = db->lookaside.nSlot - nInit; - return db->lookaside.nSlot - (nInit+nFree); + assert( db->lookaside.nSlot >= nInit+nFree ); + if( pHighwater ) *pHighwater = (int)(db->lookaside.nSlot - nInit); + return (int)(db->lookaside.nSlot - (nInit+nFree)); } /* @@ -246,7 +247,7 @@ int sqlite3_db_status( assert( (op-SQLITE_DBSTATUS_LOOKASIDE_HIT)>=0 ); assert( (op-SQLITE_DBSTATUS_LOOKASIDE_HIT)<3 ); *pCurrent = 0; - *pHighwater = db->lookaside.anStat[op - SQLITE_DBSTATUS_LOOKASIDE_HIT]; + *pHighwater = (int)db->lookaside.anStat[op-SQLITE_DBSTATUS_LOOKASIDE_HIT]; if( resetFlag ){ db->lookaside.anStat[op - SQLITE_DBSTATUS_LOOKASIDE_HIT] = 0; } diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 824e8c4d3..8c40b8692 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -76,7 +76,9 @@ # define SQLITE_PTRSIZE 8 # endif # endif /* SQLITE_PTRSIZE */ -# if defined(HAVE_STDINT_H) +# if defined(HAVE_STDINT_H) || (defined(__STDC_VERSION__) && \ + (__STDC_VERSION__ >= 199901L)) +# include typedef uintptr_t uptr; # elif SQLITE_PTRSIZE==4 typedef unsigned int uptr; @@ -1232,6 +1234,7 @@ static int auth_callback( } #endif /* SQLITE_OMIT_AUTHORIZATION */ +#if 0 /* ** This routine reads a line of text from FILE in, stores ** the text in memory obtained from malloc() and returns a pointer @@ -1276,6 +1279,7 @@ static char *local_getline(char *zPrompt, FILE *in){ zLine = realloc( zLine, n+1 ); return zLine; } +#endif /* @@ -2057,7 +2061,7 @@ static int SQLITE_TCLAPI DbObjCmd( ** (4) Name of the database (ex: "main", "temp") ** (5) Name of trigger that is doing the access ** - ** The callback should return on of the following strings: SQLITE_OK, + ** The callback should return one of the following strings: SQLITE_OK, ** SQLITE_IGNORE, or SQLITE_DENY. Any other return value is an error. ** ** If this method is invoked with no arguments, the current authorization @@ -2520,9 +2524,10 @@ static int SQLITE_TCLAPI DbObjCmd( char *zLine; /* A single line of input from the file */ char **azCol; /* zLine[] broken up into columns */ const char *zCommit; /* How to commit changes */ - FILE *in; /* The input file */ + Tcl_Channel in; /* The input file */ int lineno = 0; /* Line number of input file */ char zLineNum[80]; /* Line number print buffer */ + Tcl_Obj *str; Tcl_Obj *pResult; /* interp result */ const char *zSep; @@ -2601,23 +2606,27 @@ static int SQLITE_TCLAPI DbObjCmd( sqlite3_finalize(pStmt); return TCL_ERROR; } - in = fopen(zFile, "rb"); + in = Tcl_OpenFileChannel(interp, zFile, "rb", 0666); if( in==0 ){ - Tcl_AppendResult(interp, "Error: cannot open file: ", zFile, (char*)0); sqlite3_finalize(pStmt); return TCL_ERROR; } + Tcl_SetChannelOption(NULL, in, "-translation", "auto"); azCol = malloc( sizeof(azCol[0])*(nCol+1) ); if( azCol==0 ) { Tcl_AppendResult(interp, "Error: can't malloc()", (char*)0); - fclose(in); + Tcl_Close(interp, in); return TCL_ERROR; } + str = Tcl_NewObj(); + Tcl_IncrRefCount(str); (void)sqlite3_exec(pDb->db, "BEGIN", 0, 0, 0); zCommit = "COMMIT"; - while( (zLine = local_getline(0, in))!=0 ){ + while( Tcl_GetsObj(in, str)>=0 ) { char *z; + Tcl_Size byteLen; lineno++; + zLine = (char *)Tcl_GetByteArrayFromObj(str, &byteLen); azCol[0] = zLine; for(i=0, z=zLine; *z; z++){ if( *z==zSep[0] && strncmp(z, zSep, nSep)==0 ){ @@ -2655,15 +2664,16 @@ static int SQLITE_TCLAPI DbObjCmd( } sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); - free(zLine); + Tcl_SetObjLength(str, 0); if( rc!=SQLITE_OK ){ Tcl_AppendResult(interp,"Error: ", sqlite3_errmsg(pDb->db), (char*)0); zCommit = "ROLLBACK"; break; } } + Tcl_DecrRefCount(str); free(azCol); - fclose(in); + Tcl_Close(interp, in); sqlite3_finalize(pStmt); (void)sqlite3_exec(pDb->db, zCommit, 0, 0, 0); @@ -4034,8 +4044,8 @@ EXTERN int sqlite_Init(Tcl_Interp *interp){ return Sqlite3_Init(interp);} #if defined(TCLSH) /* This is the main routine for an ordinary TCL shell. If there are -** are arguments, run the first argument as a script. Otherwise, -** read TCL commands from standard input +** arguments, run the first argument as a script. Otherwise, read TCL +** commands from standard input */ static const char *tclsh_main_loop(void){ static const char zMainloop[] = diff --git a/src/test1.c b/src/test1.c index c204335dc..bb2f2d3b9 100644 --- a/src/test1.c +++ b/src/test1.c @@ -102,7 +102,7 @@ static int SQLITE_TCLAPI get_sqlite_pointer( } p = (struct SqliteDb*)cmdInfo.objClientData; sqlite3_snprintf(sizeof(zBuf), zBuf, "%p", p->db); - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); return TCL_OK; } @@ -153,7 +153,7 @@ int sqlite3TestErrCode(Tcl_Interp *interp, sqlite3 *db, int rc){ "error code %s (%d) does not match sqlite3_errcode %s (%d)", t1ErrorName(rc), rc, t1ErrorName(r2), r2); Tcl_ResetResult(interp); - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); return 1; } return 0; @@ -504,7 +504,7 @@ static int SQLITE_TCLAPI test_mprintf_z( for(i=2; isizeof(zStr) ) n = sizeof(zStr); sqlite3_snprintf(sizeof(zStr), zStr, "abcdefghijklmnopqrstuvwxyz"); sqlite3_snprintf(n, zStr, zFormat, a1); - Tcl_AppendResult(interp, zStr, 0); + Tcl_AppendResult(interp, zStr, NULL); return TCL_OK; } @@ -639,12 +639,12 @@ static int SQLITE_TCLAPI test_last_rowid( char zBuf[30]; if( argc!=2 ){ - Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " DB\"", 0); + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " DB\"", NULL); return TCL_ERROR; } if( getDbPointer(interp, argv[1], &db) ) return TCL_ERROR; sqlite3_snprintf(sizeof(zBuf), zBuf, "%lld", sqlite3_last_insert_rowid(db)); - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); return SQLITE_OK; } @@ -1385,7 +1385,7 @@ static int SQLITE_TCLAPI sqlite3_mprintf_int( if( Tcl_GetInt(interp, argv[i], &a[i-2]) ) return TCL_ERROR; } z = sqlite3_mprintf(argv[1], a[0], a[1], a[2]); - Tcl_AppendResult(interp, z, 0); + Tcl_AppendResult(interp, z, NULL); sqlite3_free(z); return TCL_OK; } @@ -1411,12 +1411,12 @@ static int SQLITE_TCLAPI sqlite3_mprintf_int64( } for(i=2; i<5; i++){ if( sqlite3Atoi64(argv[i], &a[i-2], sqlite3Strlen30(argv[i]), SQLITE_UTF8) ){ - Tcl_AppendResult(interp, "argument is not a valid 64-bit integer", 0); + Tcl_AppendResult(interp, "argument is not a valid 64-bit integer", NULL); return TCL_ERROR; } } z = sqlite3_mprintf(argv[1], a[0], a[1], a[2]); - Tcl_AppendResult(interp, z, 0); + Tcl_AppendResult(interp, z, NULL); sqlite3_free(z); return TCL_OK; } @@ -1449,7 +1449,7 @@ static int SQLITE_TCLAPI sqlite3_mprintf_long( a[i-2] &= (((u64)1)<<(sizeof(int)*8))-1; } z = sqlite3_mprintf(argv[1], a[0], a[1], a[2]); - Tcl_AppendResult(interp, z, 0); + Tcl_AppendResult(interp, z, NULL); sqlite3_free(z); return TCL_OK; } @@ -1476,7 +1476,7 @@ static int SQLITE_TCLAPI sqlite3_mprintf_str( if( Tcl_GetInt(interp, argv[i], &a[i-2]) ) return TCL_ERROR; } z = sqlite3_mprintf(argv[1], a[0], a[1], argc>4 ? argv[4] : NULL); - Tcl_AppendResult(interp, z, 0); + Tcl_AppendResult(interp, z, NULL); sqlite3_free(z); return TCL_OK; } @@ -1502,7 +1502,7 @@ static int SQLITE_TCLAPI sqlite3_snprintf_str( } if( Tcl_GetInt(interp, argv[1], &n) ) return TCL_ERROR; if( n<0 ){ - Tcl_AppendResult(interp, "N must be non-negative", 0); + Tcl_AppendResult(interp, "N must be non-negative", NULL); return TCL_ERROR; } for(i=3; i<5; i++){ @@ -1510,7 +1510,7 @@ static int SQLITE_TCLAPI sqlite3_snprintf_str( } z = sqlite3_malloc( n+1 ); sqlite3_snprintf(n, z, argv[2], a[0], a[1], argc>4 ? argv[5] : NULL); - Tcl_AppendResult(interp, z, 0); + Tcl_AppendResult(interp, z, NULL); sqlite3_free(z); return TCL_OK; } @@ -1539,7 +1539,7 @@ static int SQLITE_TCLAPI sqlite3_mprintf_double( } if( Tcl_GetDouble(interp, argv[4], &r) ) return TCL_ERROR; z = sqlite3_mprintf(argv[1], a[0], a[1], r); - Tcl_AppendResult(interp, z, 0); + Tcl_AppendResult(interp, z, NULL); sqlite3_free(z); return TCL_OK; } @@ -1569,7 +1569,7 @@ static int SQLITE_TCLAPI sqlite3_mprintf_scaled( if( Tcl_GetDouble(interp, argv[i], &r[i-2]) ) return TCL_ERROR; } z = sqlite3_mprintf(argv[1], r[0]*r[1]); - Tcl_AppendResult(interp, z, 0); + Tcl_AppendResult(interp, z, NULL); sqlite3_free(z); return TCL_OK; } @@ -1594,7 +1594,7 @@ static int SQLITE_TCLAPI sqlite3_mprintf_stronly( return TCL_ERROR; } z = sqlite3_mprintf(argv[1], argv[2]); - Tcl_AppendResult(interp, z, 0); + Tcl_AppendResult(interp, z, NULL); sqlite3_free(z); return TCL_OK; } @@ -1621,14 +1621,14 @@ static int SQLITE_TCLAPI sqlite3_mprintf_hexdouble( return TCL_ERROR; } if( sscanf(argv[2], "%08x%08x", &x2, &x1)!=2 ){ - Tcl_AppendResult(interp, "2nd argument should be 16-characters of hex", 0); + Tcl_AppendResult(interp, "2nd argument should be 16-characters of hex", NULL); return TCL_ERROR; } d = x2; d = (d<<32) + x1; memcpy(&r, &d, sizeof(r)); z = sqlite3_mprintf(argv[1], r); - Tcl_AppendResult(interp, z, 0); + Tcl_AppendResult(interp, z, NULL); sqlite3_free(z); return TCL_OK; } @@ -1746,7 +1746,7 @@ static int SQLITE_TCLAPI test_table_column_metadata( &zDatatype, &zCollseq, ¬null, &primarykey, &autoincrement); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3_errmsg(db), 0); + Tcl_AppendResult(interp, sqlite3_errmsg(db), NULL); return TCL_ERROR; } @@ -2023,7 +2023,7 @@ static int SQLITE_TCLAPI test_create_function_v2( ); if( rc!=SQLITE_OK ){ Tcl_ResetResult(interp); - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } return TCL_OK; @@ -2212,7 +2212,7 @@ static int SQLITE_TCLAPI test_register_func( rc = sqlite3_create_function(db, argv[2], -1, SQLITE_UTF8, 0, testFunc, 0, 0); if( rc!=0 ){ - Tcl_AppendResult(interp, sqlite3ErrStr(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrStr(rc), NULL); return TCL_ERROR; } if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR; @@ -2773,7 +2773,7 @@ static int SQLITE_TCLAPI test_snapshot_open_blob( zName = Tcl_GetString(objv[2]); pBlob = Tcl_GetByteArrayFromObj(objv[3], &nBlob); if( nBlob!=sizeof(sqlite3_snapshot) ){ - Tcl_AppendResult(interp, "bad SNAPSHOT", 0); + Tcl_AppendResult(interp, "bad SNAPSHOT", NULL); return TCL_ERROR; } rc = sqlite3_snapshot_open(db, zName, (sqlite3_snapshot*)pBlob); @@ -2810,7 +2810,7 @@ static int SQLITE_TCLAPI test_snapshot_cmp_blob( p2 = Tcl_GetByteArrayFromObj(objv[2], &n2); if( n1!=sizeof(sqlite3_snapshot) || n1!=n2 ){ - Tcl_AppendResult(interp, "bad SNAPSHOT", 0); + Tcl_AppendResult(interp, "bad SNAPSHOT", NULL); return TCL_ERROR; } @@ -2867,7 +2867,7 @@ static int SQLITE_TCLAPI test_atomic_batch_write( rc = sqlite3_open(zFile, &db); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3_errmsg(db), 0); + Tcl_AppendResult(interp, sqlite3_errmsg(db), NULL); sqlite3_close(db); return TCL_ERROR; } @@ -2909,7 +2909,7 @@ static int SQLITE_TCLAPI test_next_stmt( pStmt = sqlite3_next_stmt(db, pStmt); if( pStmt ){ if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ) return TCL_ERROR; - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); } return TCL_OK; } @@ -3211,7 +3211,7 @@ static int SQLITE_TCLAPI test_bind( if( rc ){ char zBuf[50]; sqlite3_snprintf(sizeof(zBuf), zBuf, "(%d) ", rc); - Tcl_AppendResult(interp, zBuf, sqlite3ErrStr(rc), 0); + Tcl_AppendResult(interp, zBuf, sqlite3ErrStr(rc), NULL); return TCL_ERROR; } return TCL_OK; @@ -3343,14 +3343,14 @@ static int SQLITE_TCLAPI test_collate( if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR; if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } return TCL_OK; bad_args: Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " ", 0); + Tcl_GetStringFromObj(objv[0], 0), " ", NULL); return TCL_ERROR; } @@ -3629,7 +3629,7 @@ static int SQLITE_TCLAPI test_function( return TCL_OK; bad_args: Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " ", 0); + Tcl_GetStringFromObj(objv[0], 0), " ", NULL); #endif /* SQLITE_OMIT_UTF16 */ return TCL_ERROR; } @@ -3750,7 +3750,7 @@ static int SQLITE_TCLAPI test_bind_zeroblob64( rc = sqlite3_bind_zeroblob64(pStmt, idx, n); if( sqlite3TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR; if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } @@ -3777,7 +3777,7 @@ static int SQLITE_TCLAPI test_bind_int( if( objc!=4 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " STMT N VALUE", 0); + Tcl_GetStringFromObj(objv[0], 0), " STMT N VALUE", NULL); return TCL_ERROR; } @@ -3954,7 +3954,7 @@ static int SQLITE_TCLAPI test_bind_int64( if( objc!=4 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " STMT N VALUE", 0); + Tcl_GetStringFromObj(objv[0], 0), " STMT N VALUE", NULL); return TCL_ERROR; } @@ -4010,7 +4010,7 @@ static int SQLITE_TCLAPI test_bind_double( if( objc!=4 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " STMT N VALUE", 0); + Tcl_GetStringFromObj(objv[0], 0), " STMT N VALUE", NULL); return TCL_ERROR; } @@ -4067,7 +4067,7 @@ static int SQLITE_TCLAPI test_bind_null( if( objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " STMT N", 0); + Tcl_GetStringFromObj(objv[0], 0), " STMT N", NULL); return TCL_ERROR; } @@ -4107,7 +4107,7 @@ static int SQLITE_TCLAPI test_bind_text( if( objc!=5 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " STMT N VALUE BYTES", 0); + Tcl_GetStringFromObj(objv[0], 0), " STMT N VALUE BYTES", NULL); return TCL_ERROR; } @@ -4167,7 +4167,7 @@ static int SQLITE_TCLAPI test_bind_text16( if( objc!=5 && objc!=6){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " STMT N VALUE BYTES", 0); + Tcl_GetStringFromObj(objv[0], 0), " STMT N VALUE BYTES", NULL); return TCL_ERROR; } @@ -4189,7 +4189,7 @@ static int SQLITE_TCLAPI test_bind_text16( free(toFree); if( sqlite3TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR; if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } @@ -4220,7 +4220,7 @@ static int SQLITE_TCLAPI test_bind_blob( if( objc!=5 && objc!=6 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " STMT N DATA BYTES", 0); + Tcl_GetStringFromObj(objv[0], 0), " STMT N DATA BYTES", NULL); return TCL_ERROR; } @@ -4718,12 +4718,12 @@ static int SQLITE_TCLAPI test_ex_errcode( if( objc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " DB", 0); + Tcl_GetString(objv[0]), " DB", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; rc = sqlite3_extended_errcode(db); - Tcl_AppendResult(interp, (char *)t1ErrorName(rc), 0); + Tcl_AppendResult(interp, (char *)t1ErrorName(rc), NULL); return TCL_OK; } @@ -4745,12 +4745,12 @@ static int SQLITE_TCLAPI test_errcode( if( objc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " DB", 0); + Tcl_GetString(objv[0]), " DB", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; rc = sqlite3_errcode(db); - Tcl_AppendResult(interp, (char *)t1ErrorName(rc), 0); + Tcl_AppendResult(interp, (char *)t1ErrorName(rc), NULL); return TCL_OK; } @@ -4771,7 +4771,7 @@ static int SQLITE_TCLAPI test_errmsg( if( objc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " DB", 0); + Tcl_GetString(objv[0]), " DB", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; @@ -4799,7 +4799,7 @@ static int SQLITE_TCLAPI test_error_offset( if( objc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " DB", 0); + Tcl_GetString(objv[0]), " DB", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; @@ -4831,7 +4831,7 @@ static int SQLITE_TCLAPI test_errmsg16( if( objc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " DB", 0); + Tcl_GetString(objv[0]), " DB", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; @@ -4870,7 +4870,7 @@ static int SQLITE_TCLAPI test_prepare( if( objc!=5 && objc!=4 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " DB sql bytes ?tailvar?", 0); + Tcl_GetString(objv[0]), " DB sql bytes ?tailvar?", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; @@ -4892,13 +4892,13 @@ static int SQLITE_TCLAPI test_prepare( if( rc!=SQLITE_OK ){ assert( pStmt==0 ); sqlite3_snprintf(sizeof(zBuf), zBuf, "(%d) ", rc); - Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), 0); + Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), NULL); return TCL_ERROR; } if( pStmt ){ if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ) return TCL_ERROR; - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); } return TCL_OK; } @@ -4929,7 +4929,7 @@ static int SQLITE_TCLAPI test_prepare_v2( if( objc!=5 && objc!=4 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " DB sql bytes tailvar", 0); + Tcl_GetString(objv[0]), " DB sql bytes tailvar", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; @@ -4966,13 +4966,13 @@ static int SQLITE_TCLAPI test_prepare_v2( if( rc!=SQLITE_OK ){ assert( pStmt==0 ); sqlite3_snprintf(sizeof(zBuf), zBuf, "(%d) ", rc); - Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), 0); + Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), NULL); return TCL_ERROR; } if( pStmt ){ if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ) return TCL_ERROR; - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); } return TCL_OK; } @@ -5003,7 +5003,7 @@ static int SQLITE_TCLAPI test_prepare_v3( if( objc!=6 && objc!=5 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " DB sql bytes flags tailvar", 0); + Tcl_GetString(objv[0]), " DB sql bytes flags tailvar", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; @@ -5039,13 +5039,13 @@ static int SQLITE_TCLAPI test_prepare_v3( if( rc!=SQLITE_OK ){ assert( pStmt==0 ); sqlite3_snprintf(sizeof(zBuf), zBuf, "(%d) ", rc); - Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), 0); + Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), NULL); return TCL_ERROR; } if( pStmt ){ if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ) return TCL_ERROR; - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); } return TCL_OK; } @@ -5070,7 +5070,7 @@ static int SQLITE_TCLAPI test_prepare_tkt3134( if( objc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " DB sql bytes tailvar", 0); + Tcl_GetString(objv[0]), " DB sql bytes tailvar", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; @@ -5080,13 +5080,13 @@ static int SQLITE_TCLAPI test_prepare_tkt3134( if( rc!=SQLITE_OK ){ assert( pStmt==0 ); sqlite3_snprintf(sizeof(zBuf), zBuf, "(%d) ", rc); - Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), 0); + Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), NULL); return TCL_ERROR; } if( pStmt ){ if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ) return TCL_ERROR; - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); } return TCL_OK; } @@ -5118,7 +5118,7 @@ static int SQLITE_TCLAPI test_prepare16( if( objc!=5 && objc!=4 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " DB sql bytes ?tailvar?", 0); + Tcl_GetString(objv[0]), " DB sql bytes ?tailvar?", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; @@ -5146,7 +5146,7 @@ static int SQLITE_TCLAPI test_prepare16( if( pStmt ){ if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ) return TCL_ERROR; } - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); #endif /* SQLITE_OMIT_UTF16 */ return TCL_OK; } @@ -5178,7 +5178,7 @@ static int SQLITE_TCLAPI test_prepare16_v2( if( objc!=5 && objc!=4 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " DB sql bytes ?tailvar?", 0); + Tcl_GetString(objv[0]), " DB sql bytes ?tailvar?", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; @@ -5206,7 +5206,7 @@ static int SQLITE_TCLAPI test_prepare16_v2( if( pStmt ){ if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ) return TCL_ERROR; } - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); #endif /* SQLITE_OMIT_UTF16 */ return TCL_OK; } @@ -5226,7 +5226,7 @@ static int SQLITE_TCLAPI test_open( if( objc!=3 && objc!=2 && objc!=1 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " filename options-list", 0); + Tcl_GetString(objv[0]), " filename options-list", NULL); return TCL_ERROR; } @@ -5234,7 +5234,7 @@ static int SQLITE_TCLAPI test_open( sqlite3_open(zFilename, &db); if( sqlite3TestMakePointerStr(interp, zBuf, db) ) return TCL_ERROR; - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); return TCL_OK; } @@ -5305,7 +5305,7 @@ static int SQLITE_TCLAPI test_open_v2( rc = sqlite3_open_v2(zFilename, &db, flags, zVfs); if( sqlite3TestMakePointerStr(interp, zBuf, db) ) return TCL_ERROR; - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); return TCL_OK; } @@ -5325,7 +5325,7 @@ static int SQLITE_TCLAPI test_open16( if( objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " filename options-list", 0); + Tcl_GetString(objv[0]), " filename options-list", NULL); return TCL_ERROR; } @@ -5333,7 +5333,7 @@ static int SQLITE_TCLAPI test_open16( sqlite3_open16(zFilename, &db); if( sqlite3TestMakePointerStr(interp, zBuf, db) ) return TCL_ERROR; - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); #endif /* SQLITE_OMIT_UTF16 */ return TCL_OK; } @@ -5409,7 +5409,7 @@ static int SQLITE_TCLAPI test_step( if( objc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " STMT", 0); + Tcl_GetString(objv[0]), " STMT", NULL); return TCL_ERROR; } @@ -5493,7 +5493,7 @@ static int SQLITE_TCLAPI test_column_count( if( objc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " STMT column", 0); + Tcl_GetString(objv[0]), " STMT column", NULL); return TCL_ERROR; } @@ -5520,7 +5520,7 @@ static int SQLITE_TCLAPI test_column_type( if( objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " STMT column", 0); + Tcl_GetString(objv[0]), " STMT column", NULL); return TCL_ERROR; } @@ -5569,7 +5569,7 @@ static int SQLITE_TCLAPI test_column_int64( if( objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " STMT column", 0); + Tcl_GetString(objv[0]), " STMT column", NULL); return TCL_ERROR; } @@ -5598,7 +5598,7 @@ static int SQLITE_TCLAPI test_column_blob( if( objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " STMT column", 0); + Tcl_GetString(objv[0]), " STMT column", NULL); return TCL_ERROR; } @@ -5628,7 +5628,7 @@ static int SQLITE_TCLAPI test_column_double( if( objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " STMT column", 0); + Tcl_GetString(objv[0]), " STMT column", NULL); return TCL_ERROR; } @@ -5655,7 +5655,7 @@ static int SQLITE_TCLAPI test_data_count( if( objc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " STMT column", 0); + Tcl_GetString(objv[0]), " STMT column", NULL); return TCL_ERROR; } @@ -5688,7 +5688,7 @@ static int SQLITE_TCLAPI test_stmt_utf8( xFuncU = (const unsigned char*(*)(sqlite3_stmt*,int))xFunc; if( objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " STMT column", 0); + Tcl_GetString(objv[0]), " STMT column", NULL); return TCL_ERROR; } @@ -5746,7 +5746,7 @@ static int SQLITE_TCLAPI test_stmt_utf16( xFunc = (const void *(*)(sqlite3_stmt*, int))clientData; if( objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " STMT column", 0); + Tcl_GetString(objv[0]), " STMT column", NULL); return TCL_ERROR; } @@ -5787,7 +5787,7 @@ static int SQLITE_TCLAPI test_stmt_int( xFunc = (int (*)(sqlite3_stmt*, int))clientData; if( objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetString(objv[0]), " STMT column", 0); + Tcl_GetString(objv[0]), " STMT column", NULL); return TCL_ERROR; } @@ -5811,7 +5811,7 @@ static int SQLITE_TCLAPI test_interrupt( ){ sqlite3 *db; if( argc!=2 ){ - Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " DB", 0); + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " DB", NULL); return TCL_ERROR; } if( getDbPointer(interp, argv[1], &db) ) return TCL_ERROR; @@ -5833,7 +5833,7 @@ static int SQLITE_TCLAPI test_is_interrupted( sqlite3 *db; int rc; if( argc!=2 ){ - Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " DB", 0); + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " DB", NULL); return TCL_ERROR; } if( getDbPointer(interp, argv[1], &db) ) return TCL_ERROR; @@ -5915,7 +5915,7 @@ static int SQLITE_TCLAPI get_autocommit( } if( getDbPointer(interp, argv[1], &db) ) return TCL_ERROR; sqlite3_snprintf(sizeof(zBuf), zBuf, "%d", sqlite3_get_autocommit(db)); - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); return TCL_OK; } @@ -5936,13 +5936,49 @@ static int SQLITE_TCLAPI test_busy_timeout( sqlite3 *db; if( argc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " DB", 0); + " DB", NULL); return TCL_ERROR; } if( getDbPointer(interp, argv[1], &db) ) return TCL_ERROR; if( Tcl_GetInt(interp, argv[2], &ms) ) return TCL_ERROR; rc = sqlite3_busy_timeout(db, ms); - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); + return TCL_OK; +} + +/* +** Usage: sqlite3_setlk_timeout ?-blockonconnect? DB MS +** +** Set the setlk timeout. +*/ +static int SQLITE_TCLAPI test_setlk_timeout( + void * clientData, + Tcl_Interp *interp, + int argc, + char **argv +){ + int rc, ms; + sqlite3 *db; + int bBlockOnConnect = 0; + + if( argc==4 ){ + const char *zArg = argv[1]; + const size_t nArg = strlen(zArg); + if( nArg>=2 && nArg<=15 && memcmp(zArg, "-blockonconnect", nArg)==0 ){ + bBlockOnConnect = 1; + } + } + if( argc!=(3+bBlockOnConnect) ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " ?-blockonconnect? DB MS", NULL); + return TCL_ERROR; + } + if( getDbPointer(interp, argv[argc-2], &db) ) return TCL_ERROR; + if( Tcl_GetInt(interp, argv[argc-1], &ms) ) return TCL_ERROR; + rc = sqlite3_setlk_timeout( + db, ms, (bBlockOnConnect ? SQLITE_SETLK_BLOCK_ON_CONNECT : 0) + ); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_OK; } @@ -6354,7 +6390,7 @@ static int SQLITE_TCLAPI test_pager_refcounts( if( objc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " DB", 0); + Tcl_GetStringFromObj(objv[0], 0), " DB", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; @@ -6596,7 +6632,7 @@ static int SQLITE_TCLAPI file_control_test( if( objc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " DB", 0); + Tcl_GetStringFromObj(objv[0], 0), " DB", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; @@ -6631,7 +6667,7 @@ static int SQLITE_TCLAPI file_control_lasterrno_test( if( objc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " DB", 0); + Tcl_GetStringFromObj(objv[0], 0), " DB", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ @@ -6644,7 +6680,7 @@ static int SQLITE_TCLAPI file_control_lasterrno_test( } if( iArg!=0 ) { Tcl_AppendResult(interp, "Unexpected non-zero errno: ", - Tcl_GetStringFromObj(Tcl_NewIntObj(iArg), 0), " ", 0); + Tcl_GetStringFromObj(Tcl_NewIntObj(iArg), 0), " ", NULL); return TCL_ERROR; } return TCL_OK; @@ -6780,7 +6816,7 @@ static int SQLITE_TCLAPI file_control_lockproxy_test( if( objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " DB PWD", 0); + Tcl_GetStringFromObj(objv[0], 0), " DB PWD", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ @@ -6816,7 +6852,7 @@ static int SQLITE_TCLAPI file_control_lockproxy_test( rc = sqlite3_file_control(db, NULL, SQLITE_GET_LOCKPROXYFILE, &testPath); if( strncmp(proxyPath,testPath,11) ){ Tcl_AppendResult(interp, "Lock proxy file did not match the " - "previously assigned value", 0); + "previously assigned value", NULL); return TCL_ERROR; } if( rc ){ @@ -6853,7 +6889,7 @@ static int SQLITE_TCLAPI file_control_win32_av_retry( if( objc!=4 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " DB NRETRY DELAY", 0); + Tcl_GetStringFromObj(objv[0], 0), " DB NRETRY DELAY", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ @@ -6886,7 +6922,7 @@ static int file_control_win32_get_handle( if( objc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " DB", 0); + Tcl_GetStringFromObj(objv[0], 0), " DB", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ @@ -6918,7 +6954,7 @@ static int SQLITE_TCLAPI file_control_win32_set_handle( if( objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " DB HANDLE", 0); + Tcl_GetStringFromObj(objv[0], 0), " DB HANDLE", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ @@ -6954,7 +6990,7 @@ static int SQLITE_TCLAPI file_control_persist_wal( if( objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " DB FLAG", 0); + Tcl_GetStringFromObj(objv[0], 0), " DB FLAG", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ @@ -6986,7 +7022,7 @@ static int SQLITE_TCLAPI file_control_powersafe_overwrite( if( objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " DB FLAG", 0); + Tcl_GetStringFromObj(objv[0], 0), " DB FLAG", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ @@ -7017,7 +7053,7 @@ static int SQLITE_TCLAPI file_control_vfsname( if( objc!=2 && objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " DB ?AUXDB?", 0); + Tcl_GetStringFromObj(objv[0], 0), " DB ?AUXDB?", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ @@ -7079,7 +7115,7 @@ static int SQLITE_TCLAPI file_control_tempfilename( if( objc!=2 && objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " DB ?AUXDB?", 0); + Tcl_GetStringFromObj(objv[0], 0), " DB ?AUXDB?", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ @@ -7112,7 +7148,7 @@ static int SQLITE_TCLAPI file_control_external_reader( if( objc!=2 && objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " DB ?AUXDB?", 0); + Tcl_GetStringFromObj(objv[0], 0), " DB ?AUXDB?", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ @@ -7196,7 +7232,7 @@ static int SQLITE_TCLAPI test_limit( if( objc!=4 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " DB ID VALUE", 0); + Tcl_GetStringFromObj(objv[0], 0), " DB ID VALUE", NULL); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; @@ -7507,7 +7543,7 @@ static int SQLITE_TCLAPI test_wal_checkpoint_v2( if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ const char *zErrCode = sqlite3ErrName(rc); Tcl_ResetResult(interp); - Tcl_AppendResult(interp, zErrCode, " - ", (char *)sqlite3_errmsg(db), 0); + Tcl_AppendResult(interp, zErrCode, " - ", (char *)sqlite3_errmsg(db), NULL); return TCL_ERROR; } @@ -7939,7 +7975,7 @@ static int SQLITE_TCLAPI test_getrusage( */ struct win32FileLocker { char *evName; /* Name of event to signal thread startup */ - HANDLE h; /* Handle of the file to be locked */ + sqlite3_file *pFd; /* Handle of the file to be locked */ int delay1; /* Delay before locking */ int delay2; /* Delay before unlocking */ int ok; /* Finished ok */ @@ -7948,13 +7984,15 @@ struct win32FileLocker { #endif -#if SQLITE_OS_WIN +#ifdef _WIN32 #include /* ** The background thread that does file locking. */ static void SQLITE_CDECL win32_file_locker(void *pAppData){ struct win32FileLocker *p = (struct win32FileLocker*)pAppData; + sqlite3_file *pFd = p->pFd; + HANDLE h = INVALID_HANDLE_VALUE; if( p->evName ){ HANDLE ev = OpenEvent(EVENT_MODIFY_STATE, FALSE, p->evName); if ( ev ){ @@ -7963,21 +8001,23 @@ static void SQLITE_CDECL win32_file_locker(void *pAppData){ } } if( p->delay1 ) Sleep(p->delay1); - if( LockFile(p->h, 0, 0, 100000000, 0) ){ + pFd->pMethods->xFileControl(pFd, SQLITE_FCNTL_WIN32_GET_HANDLE, (void*)&h); + if( LockFile(h, 0, 0, 100000000, 0) ){ Sleep(p->delay2); - UnlockFile(p->h, 0, 0, 100000000, 0); + UnlockFile(h, 0, 0, 100000000, 0); p->ok = 1; }else{ p->err = 1; } - CloseHandle(p->h); - p->h = 0; + pFd->pMethods->xClose(pFd); + sqlite3_free(pFd); + p->pFd = 0; p->delay1 = 0; p->delay2 = 0; } #endif -#if SQLITE_OS_WIN +#ifdef _WIN32 /* ** lock_win32_file FILENAME DELAY1 DELAY2 ** @@ -7991,37 +8031,56 @@ static int SQLITE_TCLAPI win32_file_lock( Tcl_Obj *CONST objv[] ){ static struct win32FileLocker x = { "win32_file_lock", 0, 0, 0, 0, 0 }; - const char *zFilename; + const char *zFilename = 0; + Tcl_Size nFilename = 0; + char *zTerm = 0; char zBuf[200]; int retry = 0; HANDLE ev; DWORD wResult; + sqlite3_vfs *pVfs = 0; + int flags = SQLITE_OPEN_MAIN_DB | SQLITE_OPEN_READWRITE; + int rc = SQLITE_OK; if( objc!=4 && objc!=1 ){ Tcl_WrongNumArgs(interp, 1, objv, "FILENAME DELAY1 DELAY2"); return TCL_ERROR; } if( objc==1 ){ + HANDLE h = INVALID_HANDLE_VALUE; + if( x.pFd ){ + x.pFd->pMethods->xFileControl( + x.pFd, SQLITE_FCNTL_WIN32_GET_HANDLE, (void*)&h + ); + } sqlite3_snprintf(sizeof(zBuf), zBuf, "%d %d %d %d %d", - x.ok, x.err, x.delay1, x.delay2, x.h); + x.ok, x.err, x.delay1, x.delay2, h); Tcl_AppendResult(interp, zBuf, (char*)0); return TCL_OK; } - while( x.h && retry<30 ){ + while( x.pFd && retry<30 ){ retry++; Sleep(100); } - if( x.h ){ + if( x.pFd ){ Tcl_AppendResult(interp, "busy", (char*)0); return TCL_ERROR; } if( Tcl_GetIntFromObj(interp, objv[2], &x.delay1) ) return TCL_ERROR; if( Tcl_GetIntFromObj(interp, objv[3], &x.delay2) ) return TCL_ERROR; - zFilename = Tcl_GetString(objv[1]); - x.h = CreateFile(zFilename, GENERIC_READ|GENERIC_WRITE, - FILE_SHARE_READ|FILE_SHARE_WRITE, 0, OPEN_ALWAYS, - FILE_ATTRIBUTE_NORMAL, 0); - if( !x.h ){ + pVfs = sqlite3_vfs_find(0); + x.pFd = (sqlite3_file*)sqlite3_malloc(pVfs->szOsFile); + + /* xOpen() must be passed a dual-nul-terminated string preceded in memory + ** by 4 0x00 bytes. */ + zFilename = Tcl_GetStringFromObj(objv[1], &nFilename); + zTerm = (char*)sqlite3_malloc(nFilename+6); + memset(zTerm, 0, nFilename+6); + memcpy(&zTerm[4], zFilename, nFilename); + rc = pVfs->xOpen(pVfs, &zTerm[4], x.pFd, flags, &flags); + sqlite3_free(zTerm); + + if( rc!=SQLITE_OK ){ Tcl_AppendResult(interp, "cannot open file: ", zFilename, (char*)0); return TCL_ERROR; } @@ -8767,7 +8826,7 @@ static int SQLITE_TCLAPI guess_number_of_cores( Tcl_Obj *CONST objv[] ){ unsigned int nCore = 1; -#if SQLITE_OS_WIN +#ifdef _WIN32 SYSTEM_INFO sysinfo; GetSystemInfo(&sysinfo); nCore = (unsigned int)sysinfo.dwNumberOfProcessors; @@ -8848,6 +8907,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite_delete_collation", (Tcl_CmdProc*)delete_collation }, { "sqlite3_get_autocommit", (Tcl_CmdProc*)get_autocommit }, { "sqlite3_busy_timeout", (Tcl_CmdProc*)test_busy_timeout }, + { "sqlite3_setlk_timeout", (Tcl_CmdProc*)test_setlk_timeout }, { "printf", (Tcl_CmdProc*)test_printf }, { "sqlite3IoTrace", (Tcl_CmdProc*)test_io_trace }, { "clang_sanitize_address", (Tcl_CmdProc*)clang_sanitize_address }, @@ -8946,7 +9006,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "database_never_corrupt", database_never_corrupt, 0}, { "database_may_be_corrupt", database_may_be_corrupt, 0}, { "optimization_control", optimization_control,0}, -#if SQLITE_OS_WIN +#ifdef _WIN32 { "lock_win32_file", win32_file_lock, 0 }, #endif { "tcl_objproc", runAsObjProc, 0 }, diff --git a/src/test2.c b/src/test2.c index a9549aa7f..899728ead 100644 --- a/src/test2.c +++ b/src/test2.c @@ -51,7 +51,7 @@ static int SQLITE_TCLAPI pager_open( char zBuf[100]; if( argc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " FILENAME N-PAGE\"", 0); + " FILENAME N-PAGE\"", NULL); return TCL_ERROR; } if( Tcl_GetInt(interp, argv[2], &nPage) ) return TCL_ERROR; @@ -59,14 +59,14 @@ static int SQLITE_TCLAPI pager_open( SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MAIN_DB, pager_test_reiniter); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } sqlite3PagerSetCachesize(pPager, nPage); pageSize = test_pagesize; sqlite3PagerSetPagesize(pPager, &pageSize, -1); sqlite3_snprintf(sizeof(zBuf),zBuf,"%p",pPager); - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); return TCL_OK; } @@ -85,13 +85,13 @@ static int SQLITE_TCLAPI pager_close( int rc; if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID\"", 0); + " ID\"", NULL); return TCL_ERROR; } pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerClose(pPager, 0); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } return TCL_OK; @@ -112,13 +112,13 @@ static int SQLITE_TCLAPI pager_rollback( int rc; if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID\"", 0); + " ID\"", NULL); return TCL_ERROR; } pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerRollback(pPager); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } return TCL_OK; @@ -139,18 +139,18 @@ static int SQLITE_TCLAPI pager_commit( int rc; if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID\"", 0); + " ID\"", NULL); return TCL_ERROR; } pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerCommitPhaseOne(pPager, 0, 0); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } rc = sqlite3PagerCommitPhaseTwo(pPager); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } return TCL_OK; @@ -171,13 +171,13 @@ static int SQLITE_TCLAPI pager_stmt_begin( int rc; if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID\"", 0); + " ID\"", NULL); return TCL_ERROR; } pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerOpenSavepoint(pPager, 1); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } return TCL_OK; @@ -198,14 +198,14 @@ static int SQLITE_TCLAPI pager_stmt_rollback( int rc; if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID\"", 0); + " ID\"", NULL); return TCL_ERROR; } pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, 0); sqlite3PagerSavepoint(pPager, SAVEPOINT_RELEASE, 0); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } return TCL_OK; @@ -226,13 +226,13 @@ static int SQLITE_TCLAPI pager_stmt_commit( int rc; if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID\"", 0); + " ID\"", NULL); return TCL_ERROR; } pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_RELEASE, 0); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } return TCL_OK; @@ -253,7 +253,7 @@ static int SQLITE_TCLAPI pager_stats( int i, *a; if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID\"", 0); + " ID\"", NULL); return TCL_ERROR; } pPager = sqlite3TestTextToPtr(argv[1]); @@ -287,13 +287,13 @@ static int SQLITE_TCLAPI pager_pagecount( int nPage; if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID\"", 0); + " ID\"", NULL); return TCL_ERROR; } pPager = sqlite3TestTextToPtr(argv[1]); sqlite3PagerPagecount(pPager, &nPage); sqlite3_snprintf(sizeof(zBuf), zBuf, "%d", nPage); - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); return TCL_OK; } @@ -315,7 +315,7 @@ static int SQLITE_TCLAPI page_get( int rc; if( argc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID PGNO\"", 0); + " ID PGNO\"", NULL); return TCL_ERROR; } pPager = sqlite3TestTextToPtr(argv[1]); @@ -325,11 +325,11 @@ static int SQLITE_TCLAPI page_get( rc = sqlite3PagerGet(pPager, pgno, &pPage, 0); } if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf),zBuf,"%p",pPage); - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); return TCL_OK; } @@ -351,7 +351,7 @@ static int SQLITE_TCLAPI page_lookup( int pgno; if( argc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID PGNO\"", 0); + " ID PGNO\"", NULL); return TCL_ERROR; } pPager = sqlite3TestTextToPtr(argv[1]); @@ -359,7 +359,7 @@ static int SQLITE_TCLAPI page_lookup( pPage = sqlite3PagerLookup(pPager, pgno); if( pPage ){ sqlite3_snprintf(sizeof(zBuf),zBuf,"%p",pPage); - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); } return TCL_OK; } @@ -377,7 +377,7 @@ static int SQLITE_TCLAPI pager_truncate( int pgno; if( argc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID PGNO\"", 0); + " ID PGNO\"", NULL); return TCL_ERROR; } pPager = sqlite3TestTextToPtr(argv[1]); @@ -401,7 +401,7 @@ static int SQLITE_TCLAPI page_unref( DbPage *pPage; if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " PAGE\"", 0); + " PAGE\"", NULL); return TCL_ERROR; } pPage = (DbPage *)sqlite3TestTextToPtr(argv[1]); @@ -424,12 +424,12 @@ static int SQLITE_TCLAPI page_read( DbPage *pPage; if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " PAGE\"", 0); + " PAGE\"", NULL); return TCL_ERROR; } pPage = sqlite3TestTextToPtr(argv[1]); memcpy(zBuf, sqlite3PagerGetData(pPage), sizeof(zBuf)); - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); return TCL_OK; } @@ -448,12 +448,12 @@ static int SQLITE_TCLAPI page_number( DbPage *pPage; if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " PAGE\"", 0); + " PAGE\"", NULL); return TCL_ERROR; } pPage = (DbPage *)sqlite3TestTextToPtr(argv[1]); sqlite3_snprintf(sizeof(zBuf), zBuf, "%d", sqlite3PagerPagenumber(pPage)); - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); return TCL_OK; } @@ -473,13 +473,13 @@ static int SQLITE_TCLAPI page_write( int rc; if( argc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " PAGE DATA\"", 0); + " PAGE DATA\"", NULL); return TCL_ERROR; } pPage = (DbPage *)sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerWrite(pPage); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } pData = sqlite3PagerGetData(pPage); @@ -513,7 +513,7 @@ static int SQLITE_TCLAPI fake_big_file( int nFile; if( argc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " N-MEGABYTES FILE\"", 0); + " N-MEGABYTES FILE\"", NULL); return TCL_ERROR; } if( Tcl_GetInt(interp, argv[1], &n) ) return TCL_ERROR; @@ -536,7 +536,7 @@ static int SQLITE_TCLAPI fake_big_file( (SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_DB), 0 ); if( rc ){ - Tcl_AppendResult(interp, "open failed: ", sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, "open failed: ", sqlite3ErrName(rc), NULL); sqlite3_free(zFile); return TCL_ERROR; } @@ -546,7 +546,7 @@ static int SQLITE_TCLAPI fake_big_file( sqlite3OsCloseFree(fd); sqlite3_free(zFile); if( rc ){ - Tcl_AppendResult(interp, "write failed: ", sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, "write failed: ", sqlite3ErrName(rc), NULL); return TCL_ERROR; } return TCL_OK; diff --git a/src/test3.c b/src/test3.c index f1b2b0168..8fbb96a80 100644 --- a/src/test3.c +++ b/src/test3.c @@ -46,9 +46,10 @@ static int SQLITE_TCLAPI btree_open( char *zFilename; if( argc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " FILENAME NCACHE FLAGS\"", 0); + " FILENAME NCACHE FLAGS\"", NULL); return TCL_ERROR; } + if( Tcl_GetInt(interp, argv[2], &nCache) ) return TCL_ERROR; nRefSqlite3++; if( nRefSqlite3==1 ){ @@ -65,12 +66,12 @@ static int SQLITE_TCLAPI btree_open( SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MAIN_DB); sqlite3_free(zFilename); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } sqlite3BtreeSetCacheSize(pBt, nCache); sqlite3_snprintf(sizeof(zBuf), zBuf,"%p", pBt); - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); return TCL_OK; } @@ -89,13 +90,13 @@ static int SQLITE_TCLAPI btree_close( int rc; if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID\"", 0); + " ID\"", NULL); return TCL_ERROR; } pBt = sqlite3TestTextToPtr(argv[1]); rc = sqlite3BtreeClose(pBt); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } nRefSqlite3--; @@ -124,7 +125,7 @@ static int SQLITE_TCLAPI btree_begin_transaction( int rc; if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID\"", 0); + " ID\"", NULL); return TCL_ERROR; } pBt = sqlite3TestTextToPtr(argv[1]); @@ -132,7 +133,7 @@ static int SQLITE_TCLAPI btree_begin_transaction( rc = sqlite3BtreeBeginTrans(pBt, 1, 0); sqlite3BtreeLeave(pBt); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } return TCL_OK; @@ -155,7 +156,7 @@ static int SQLITE_TCLAPI btree_pager_stats( if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID\"", 0); + " ID\"", NULL); return TCL_ERROR; } pBt = sqlite3TestTextToPtr(argv[1]); @@ -208,7 +209,7 @@ static int SQLITE_TCLAPI btree_cursor( if( argc!=4 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID TABLENUM WRITEABLE\"", 0); + " ID TABLENUM WRITEABLE\"", NULL); return TCL_ERROR; } pBt = sqlite3TestTextToPtr(argv[1]); @@ -229,11 +230,11 @@ static int SQLITE_TCLAPI btree_cursor( sqlite3_mutex_leave(pBt->db->mutex); if( rc ){ ckfree((char *)pCur); - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf), zBuf,"%p", pCur); - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); return SQLITE_OK; } @@ -253,7 +254,7 @@ static int SQLITE_TCLAPI btree_close_cursor( if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID\"", 0); + " ID\"", NULL); return TCL_ERROR; } pCur = sqlite3TestTextToPtr(argv[1]); @@ -271,7 +272,7 @@ static int SQLITE_TCLAPI btree_close_cursor( #endif ckfree((char *)pCur); if( rc ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } return SQLITE_OK; @@ -297,7 +298,7 @@ static int SQLITE_TCLAPI btree_next( if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID\"", 0); + " ID\"", NULL); return TCL_ERROR; } pCur = sqlite3TestTextToPtr(argv[1]); @@ -309,11 +310,11 @@ static int SQLITE_TCLAPI btree_next( } sqlite3BtreeLeave(pCur->pBtree); if( rc ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf),zBuf,"%d",res); - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); return SQLITE_OK; } @@ -336,7 +337,7 @@ static int SQLITE_TCLAPI btree_first( if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID\"", 0); + " ID\"", NULL); return TCL_ERROR; } pCur = sqlite3TestTextToPtr(argv[1]); @@ -344,11 +345,11 @@ static int SQLITE_TCLAPI btree_first( rc = sqlite3BtreeFirst(pCur, &res); sqlite3BtreeLeave(pCur->pBtree); if( rc ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf),zBuf,"%d",res); - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); return SQLITE_OK; } @@ -370,7 +371,7 @@ static int SQLITE_TCLAPI btree_eof( if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID\"", 0); + " ID\"", NULL); return TCL_ERROR; } pCur = sqlite3TestTextToPtr(argv[1]); @@ -378,7 +379,7 @@ static int SQLITE_TCLAPI btree_eof( rc = sqlite3BtreeEof(pCur); sqlite3BtreeLeave(pCur->pBtree); sqlite3_snprintf(sizeof(zBuf),zBuf, "%d", rc); - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); return SQLITE_OK; } @@ -399,7 +400,7 @@ static int SQLITE_TCLAPI btree_payload_size( if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID\"", 0); + " ID\"", NULL); return TCL_ERROR; } pCur = sqlite3TestTextToPtr(argv[1]); @@ -407,7 +408,7 @@ static int SQLITE_TCLAPI btree_payload_size( n = sqlite3BtreePayloadSize(pCur); sqlite3BtreeLeave(pCur->pBtree); sqlite3_snprintf(sizeof(zBuf),zBuf, "%u", n); - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); return SQLITE_OK; } @@ -437,7 +438,7 @@ static int SQLITE_TCLAPI btree_varint_test( unsigned char zBuf[100]; if( argc!=5 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " START MULTIPLIER COUNT INCREMENT\"", 0); + " START MULTIPLIER COUNT INCREMENT\"", NULL); return TCL_ERROR; } if( Tcl_GetInt(interp, argv[1], (int*)&start) ) return TCL_ERROR; @@ -452,20 +453,20 @@ static int SQLITE_TCLAPI btree_varint_test( if( n1>9 || n1<1 ){ sqlite3_snprintf(sizeof(zErr), zErr, "putVarint returned %d - should be between 1 and 9", n1); - Tcl_AppendResult(interp, zErr, 0); + Tcl_AppendResult(interp, zErr, NULL); return TCL_ERROR; } n2 = getVarint(zBuf, &out); if( n1!=n2 ){ sqlite3_snprintf(sizeof(zErr), zErr, "putVarint returned %d and getVarint returned %d", n1, n2); - Tcl_AppendResult(interp, zErr, 0); + Tcl_AppendResult(interp, zErr, NULL); return TCL_ERROR; } if( in!=out ){ sqlite3_snprintf(sizeof(zErr), zErr, "Wrote 0x%016llx and got back 0x%016llx", in, out); - Tcl_AppendResult(interp, zErr, 0); + Tcl_AppendResult(interp, zErr, NULL); return TCL_ERROR; } if( (in & 0xffffffff)==in ){ @@ -476,14 +477,14 @@ static int SQLITE_TCLAPI btree_varint_test( sqlite3_snprintf(sizeof(zErr), zErr, "putVarint returned %d and GetVarint32 returned %d", n1, n2); - Tcl_AppendResult(interp, zErr, 0); + Tcl_AppendResult(interp, zErr, NULL); return TCL_ERROR; } if( in!=out ){ sqlite3_snprintf(sizeof(zErr), zErr, "Wrote 0x%016llx and got back 0x%016llx from GetVarint32", in, out); - Tcl_AppendResult(interp, zErr, 0); + Tcl_AppendResult(interp, zErr, NULL); return TCL_ERROR; } } @@ -523,12 +524,12 @@ static int SQLITE_TCLAPI btree_from_db( if( argc!=2 && argc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " DB-HANDLE ?N?\"", 0); + " DB-HANDLE ?N?\"", NULL); return TCL_ERROR; } if( 1!=Tcl_GetCommandInfo(interp, argv[1], &info) ){ - Tcl_AppendResult(interp, "No such db-handle: \"", argv[1], "\"", 0); + Tcl_AppendResult(interp, "No such db-handle: \"", argv[1], "\"", NULL); return TCL_ERROR; } if( argc==3 ){ @@ -561,7 +562,7 @@ static int SQLITE_TCLAPI btree_ismemdb( if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID\"", 0); + " ID\"", NULL); return TCL_ERROR; } pBt = sqlite3TestTextToPtr(argv[1]); @@ -591,7 +592,7 @@ static int SQLITE_TCLAPI btree_set_cache_size( if( argc!=3 ){ Tcl_AppendResult( - interp, "wrong # args: should be \"", argv[0], " BT NCACHE\"", 0); + interp, "wrong # args: should be \"", argv[0], " BT NCACHE\"", NULL); return TCL_ERROR; } pBt = sqlite3TestTextToPtr(argv[1]); @@ -646,7 +647,7 @@ static int SQLITE_TCLAPI btree_insert( Tcl_ResetResult(interp); if( rc ){ - Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), NULL); return TCL_ERROR; } return TCL_OK; diff --git a/src/test4.c b/src/test4.c index 8a68f7d3e..07236b3e5 100644 --- a/src/test4.c +++ b/src/test4.c @@ -119,7 +119,7 @@ static void *test_thread_main(void *pArg){ */ static int parse_thread_id(Tcl_Interp *interp, const char *zArg){ if( zArg==0 || zArg[0]==0 || zArg[1]!=0 || !isupper((unsigned char)zArg[0]) ){ - Tcl_AppendResult(interp, "thread ID must be an upper case letter", 0); + Tcl_AppendResult(interp, "thread ID must be an upper case letter", NULL); return -1; } return zArg[0] - 'A'; @@ -143,13 +143,13 @@ static int SQLITE_TCLAPI tcl_thread_create( if( argc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID FILENAME", 0); + " ID FILENAME", NULL); return TCL_ERROR; } i = parse_thread_id(interp, argv[1]); if( i<0 ) return TCL_ERROR; if( threadset[i].busy ){ - Tcl_AppendResult(interp, "thread ", argv[1], " is already running", 0); + Tcl_AppendResult(interp, "thread ", argv[1], " is already running", NULL); return TCL_ERROR; } threadset[i].busy = 1; @@ -159,7 +159,7 @@ static int SQLITE_TCLAPI tcl_thread_create( threadset[i].completed = 0; rc = pthread_create(&x, 0, test_thread_main, &threadset[i]); if( rc ){ - Tcl_AppendResult(interp, "failed to create the thread", 0); + Tcl_AppendResult(interp, "failed to create the thread", NULL); sqlite3_free(threadset[i].zFilename); threadset[i].busy = 0; return TCL_ERROR; @@ -192,13 +192,13 @@ static int SQLITE_TCLAPI tcl_thread_wait( if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID", 0); + " ID", NULL); return TCL_ERROR; } i = parse_thread_id(interp, argv[1]); if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ - Tcl_AppendResult(interp, "no such thread", 0); + Tcl_AppendResult(interp, "no such thread", NULL); return TCL_ERROR; } test_thread_wait(&threadset[i]); @@ -236,7 +236,7 @@ static int SQLITE_TCLAPI tcl_thread_halt( if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID", 0); + " ID", NULL); return TCL_ERROR; } if( argv[1][0]=='*' && argv[1][1]==0 ){ @@ -247,7 +247,7 @@ static int SQLITE_TCLAPI tcl_thread_halt( i = parse_thread_id(interp, argv[1]); if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ - Tcl_AppendResult(interp, "no such thread", 0); + Tcl_AppendResult(interp, "no such thread", NULL); return TCL_ERROR; } test_stop_thread(&threadset[i]); @@ -272,18 +272,18 @@ static int SQLITE_TCLAPI tcl_thread_argc( if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID", 0); + " ID", NULL); return TCL_ERROR; } i = parse_thread_id(interp, argv[1]); if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ - Tcl_AppendResult(interp, "no such thread", 0); + Tcl_AppendResult(interp, "no such thread", NULL); return TCL_ERROR; } test_thread_wait(&threadset[i]); sqlite3_snprintf(sizeof(zBuf), zBuf, "%d", threadset[i].argc); - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); return TCL_OK; } @@ -304,22 +304,22 @@ static int SQLITE_TCLAPI tcl_thread_argv( if( argc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID N", 0); + " ID N", NULL); return TCL_ERROR; } i = parse_thread_id(interp, argv[1]); if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ - Tcl_AppendResult(interp, "no such thread", 0); + Tcl_AppendResult(interp, "no such thread", NULL); return TCL_ERROR; } if( Tcl_GetInt(interp, argv[2], &n) ) return TCL_ERROR; test_thread_wait(&threadset[i]); if( n<0 || n>=threadset[i].argc ){ - Tcl_AppendResult(interp, "column number out of range", 0); + Tcl_AppendResult(interp, "column number out of range", NULL); return TCL_ERROR; } - Tcl_AppendResult(interp, threadset[i].argv[n], 0); + Tcl_AppendResult(interp, threadset[i].argv[n], NULL); return TCL_OK; } @@ -340,22 +340,22 @@ static int SQLITE_TCLAPI tcl_thread_colname( if( argc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID N", 0); + " ID N", NULL); return TCL_ERROR; } i = parse_thread_id(interp, argv[1]); if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ - Tcl_AppendResult(interp, "no such thread", 0); + Tcl_AppendResult(interp, "no such thread", NULL); return TCL_ERROR; } if( Tcl_GetInt(interp, argv[2], &n) ) return TCL_ERROR; test_thread_wait(&threadset[i]); if( n<0 || n>=threadset[i].argc ){ - Tcl_AppendResult(interp, "column number out of range", 0); + Tcl_AppendResult(interp, "column number out of range", NULL); return TCL_ERROR; } - Tcl_AppendResult(interp, threadset[i].colv[n], 0); + Tcl_AppendResult(interp, threadset[i].colv[n], NULL); return TCL_OK; } @@ -376,18 +376,18 @@ static int SQLITE_TCLAPI tcl_thread_result( if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID", 0); + " ID", NULL); return TCL_ERROR; } i = parse_thread_id(interp, argv[1]); if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ - Tcl_AppendResult(interp, "no such thread", 0); + Tcl_AppendResult(interp, "no such thread", NULL); return TCL_ERROR; } test_thread_wait(&threadset[i]); zName = sqlite3ErrName(threadset[i].rc); - Tcl_AppendResult(interp, zName, 0); + Tcl_AppendResult(interp, zName, NULL); return TCL_OK; } @@ -407,17 +407,17 @@ static int SQLITE_TCLAPI tcl_thread_error( if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID", 0); + " ID", NULL); return TCL_ERROR; } i = parse_thread_id(interp, argv[1]); if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ - Tcl_AppendResult(interp, "no such thread", 0); + Tcl_AppendResult(interp, "no such thread", NULL); return TCL_ERROR; } test_thread_wait(&threadset[i]); - Tcl_AppendResult(interp, threadset[i].zErr, 0); + Tcl_AppendResult(interp, threadset[i].zErr, NULL); return TCL_OK; } @@ -451,13 +451,13 @@ static int SQLITE_TCLAPI tcl_thread_compile( int i; if( argc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID SQL", 0); + " ID SQL", NULL); return TCL_ERROR; } i = parse_thread_id(interp, argv[1]); if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ - Tcl_AppendResult(interp, "no such thread", 0); + Tcl_AppendResult(interp, "no such thread", NULL); return TCL_ERROR; } test_thread_wait(&threadset[i]); @@ -505,13 +505,13 @@ static int SQLITE_TCLAPI tcl_thread_step( int i; if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " IDL", 0); + " IDL", NULL); return TCL_ERROR; } i = parse_thread_id(interp, argv[1]); if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ - Tcl_AppendResult(interp, "no such thread", 0); + Tcl_AppendResult(interp, "no such thread", NULL); return TCL_ERROR; } test_thread_wait(&threadset[i]); @@ -548,13 +548,13 @@ static int SQLITE_TCLAPI tcl_thread_finalize( int i; if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " IDL", 0); + " IDL", NULL); return TCL_ERROR; } i = parse_thread_id(interp, argv[1]); if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ - Tcl_AppendResult(interp, "no such thread", 0); + Tcl_AppendResult(interp, "no such thread", NULL); return TCL_ERROR; } test_thread_wait(&threadset[i]); @@ -581,20 +581,20 @@ static int SQLITE_TCLAPI tcl_thread_swap( sqlite3 *temp; if( argc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID1 ID2", 0); + " ID1 ID2", NULL); return TCL_ERROR; } i = parse_thread_id(interp, argv[1]); if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ - Tcl_AppendResult(interp, "no such thread", 0); + Tcl_AppendResult(interp, "no such thread", NULL); return TCL_ERROR; } test_thread_wait(&threadset[i]); j = parse_thread_id(interp, argv[2]); if( j<0 ) return TCL_ERROR; if( !threadset[j].busy ){ - Tcl_AppendResult(interp, "no such thread", 0); + Tcl_AppendResult(interp, "no such thread", NULL); return TCL_ERROR; } test_thread_wait(&threadset[j]); @@ -622,13 +622,13 @@ static int SQLITE_TCLAPI tcl_thread_db_get( extern int sqlite3TestMakePointerStr(Tcl_Interp*, char*, void*); if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID", 0); + " ID", NULL); return TCL_ERROR; } i = parse_thread_id(interp, argv[1]); if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ - Tcl_AppendResult(interp, "no such thread", 0); + Tcl_AppendResult(interp, "no such thread", NULL); return TCL_ERROR; } test_thread_wait(&threadset[i]); @@ -653,13 +653,13 @@ static int SQLITE_TCLAPI tcl_thread_db_put( extern void *sqlite3TestTextToPtr(const char *); if( argc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID DB", 0); + " ID DB", NULL); return TCL_ERROR; } i = parse_thread_id(interp, argv[1]); if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ - Tcl_AppendResult(interp, "no such thread", 0); + Tcl_AppendResult(interp, "no such thread", NULL); return TCL_ERROR; } test_thread_wait(&threadset[i]); @@ -685,13 +685,13 @@ static int SQLITE_TCLAPI tcl_thread_stmt_get( extern int sqlite3TestMakePointerStr(Tcl_Interp*, char*, void*); if( argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], - " ID", 0); + " ID", NULL); return TCL_ERROR; } i = parse_thread_id(interp, argv[1]); if( i<0 ) return TCL_ERROR; if( !threadset[i].busy ){ - Tcl_AppendResult(interp, "no such thread", 0); + Tcl_AppendResult(interp, "no such thread", NULL); return TCL_ERROR; } test_thread_wait(&threadset[i]); diff --git a/src/test5.c b/src/test5.c index 334b5d07f..06d2de911 100644 --- a/src/test5.c +++ b/src/test5.c @@ -67,7 +67,7 @@ static int SQLITE_TCLAPI test_value_overhead( if( objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " ", 0); + Tcl_GetStringFromObj(objv[0], 0), " ", NULL); return TCL_ERROR; } @@ -106,7 +106,7 @@ static u8 name_to_enc(Tcl_Interp *interp, Tcl_Obj *pObj){ } } if( !pEnc->enc ){ - Tcl_AppendResult(interp, "No such encoding: ", z, 0); + Tcl_AppendResult(interp, "No such encoding: ", z, NULL); } if( pEnc->enc==SQLITE_UTF16 ){ return SQLITE_UTF16NATIVE; @@ -135,7 +135,7 @@ static int SQLITE_TCLAPI test_translate( if( objc!=4 && objc!=5 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", Tcl_GetStringFromObj(objv[0], 0), - " ", 0 + " ", NULL ); return TCL_ERROR; } diff --git a/src/test6.c b/src/test6.c index 76db640c4..aee7bf12a 100644 --- a/src/test6.c +++ b/src/test6.c @@ -755,12 +755,12 @@ static int processDevSymArgs( ){ Tcl_AppendResult(interp, "Bad option: \"", zOpt, - "\" - must be \"-characteristics\" or \"-sectorsize\"", 0 + "\" - must be \"-characteristics\" or \"-sectorsize\"", NULL ); return TCL_ERROR; } if( i==objc-1 ){ - Tcl_AppendResult(interp, "Option requires an argument: \"", zOpt, "\"",0); + Tcl_AppendResult(interp, "Option requires an argument: \"", zOpt, "\"", NULL); return TCL_ERROR; } @@ -934,7 +934,7 @@ static int SQLITE_TCLAPI crashParamsObjCmd( zCrashFile = Tcl_GetStringFromObj(objv[objc-1], &nCrashFile); if( nCrashFile>=sizeof(g.zCrashFile) ){ - Tcl_AppendResult(interp, "Filename is too long: \"", zCrashFile, "\"", 0); + Tcl_AppendResult(interp, "Filename is too long: \"", zCrashFile, "\"", NULL); goto error; } if( Tcl_GetIntFromObj(interp, objv[objc-2], &iDelay) ){ @@ -1044,7 +1044,7 @@ static int SQLITE_TCLAPI jtObjCmd( if( objc==3 ){ if( strcmp(zParent, "-default") ){ Tcl_AppendResult(interp, - "bad option \"", zParent, "\": must be -default", 0 + "bad option \"", zParent, "\": must be -default", NULL ); return TCL_ERROR; } @@ -1055,7 +1055,7 @@ static int SQLITE_TCLAPI jtObjCmd( zParent = 0; } if( jt_register(zParent, objc==3) ){ - Tcl_AppendResult(interp, "Error in jt_register", 0); + Tcl_AppendResult(interp, "Error in jt_register", NULL); return TCL_ERROR; } diff --git a/src/test9.c b/src/test9.c index b5362adb7..62b16795e 100644 --- a/src/test9.c +++ b/src/test9.c @@ -56,7 +56,7 @@ static int SQLITE_TCLAPI c_collation_test( error_out: Tcl_ResetResult(interp); - Tcl_AppendResult(interp, "Error testing function: ", zErrFunction, 0); + Tcl_AppendResult(interp, "Error testing function: ", zErrFunction, NULL); return TCL_ERROR; } @@ -96,7 +96,7 @@ static int SQLITE_TCLAPI c_realloc_test( error_out: Tcl_ResetResult(interp); - Tcl_AppendResult(interp, "Error testing function: ", zErrFunction, 0); + Tcl_AppendResult(interp, "Error testing function: ", zErrFunction, NULL); return TCL_ERROR; } @@ -174,7 +174,7 @@ static int SQLITE_TCLAPI c_misuse_test( error_out: Tcl_ResetResult(interp); - Tcl_AppendResult(interp, "Error testing function: ", zErrFunction, 0); + Tcl_AppendResult(interp, "Error testing function: ", zErrFunction, NULL); return TCL_ERROR; } diff --git a/src/test_backup.c b/src/test_backup.c index 8051888ee..ae2348ebc 100644 --- a/src/test_backup.c +++ b/src/test_backup.c @@ -135,7 +135,7 @@ static int SQLITE_TCLAPI backupTestInit( pBackup = sqlite3_backup_init(pDestDb, zDestName, pSrcDb, zSrcName); if( !pBackup ){ - Tcl_AppendResult(interp, "sqlite3_backup_init() failed", 0); + Tcl_AppendResult(interp, "sqlite3_backup_init() failed", NULL); return TCL_ERROR; } diff --git a/src/test_blob.c b/src/test_blob.c index bddad240c..ae5a73417 100644 --- a/src/test_blob.c +++ b/src/test_blob.c @@ -237,7 +237,7 @@ static int SQLITE_TCLAPI test_blob_read( if( nByte>0 ){ zBuf = (unsigned char *)Tcl_AttemptAlloc(nByte); if( zBuf==0 ){ - Tcl_AppendResult(interp, "out of memory in " __FILE__, 0); + Tcl_AppendResult(interp, "out of memory in " __FILE__, NULL); return TCL_ERROR; } } diff --git a/src/test_config.c b/src/test_config.c index c8ce2ab88..bdfb31e7d 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -88,6 +88,12 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "win32malloc", "0", TCL_GLOBAL_ONLY); #endif +#if defined(SQLITE_OS_WINRT) && SQLITE_OS_WINRT + Tcl_SetVar2(interp, "sqlite_options", "winrt", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "winrt", "0", TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_DEBUG Tcl_SetVar2(interp, "sqlite_options", "debug", "1", TCL_GLOBAL_ONLY); #else @@ -781,6 +787,13 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); Tcl_SetVar2(interp, "sqlite_options", "windowfunc", "1", TCL_GLOBAL_ONLY); #endif +#if !defined(SQLITE_ENABLE_SETLK_TIMEOUT) + Tcl_SetVar2(interp, "sqlite_options", "setlk_timeout", "0", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "setlk_timeout", + STRINGVALUE(SQLITE_ENABLE_SETLK_TIMEOUT), TCL_GLOBAL_ONLY); +#endif + #define LINKVAR(x) { \ static const int cv_ ## x = SQLITE_ ## x; \ Tcl_LinkVar(interp, "SQLITE_" #x, (char *)&(cv_ ## x), \ diff --git a/src/test_delete.c b/src/test_delete.c index 68fdbc6a7..3f34a72c4 100644 --- a/src/test_delete.c +++ b/src/test_delete.c @@ -63,7 +63,7 @@ static int sqlite3DeleteUnlinkIfExists( int *pbExists ){ int rc = SQLITE_ERROR; -#if SQLITE_OS_WIN +#ifdef _WIN32 if( pVfs ){ if( pbExists ) *pbExists = 1; rc = pVfs->xDelete(pVfs, zFile, 0); @@ -115,7 +115,7 @@ SQLITE_API int sqlite3_delete_database( { "%s-wal%03d", SQLITE_MULTIPLEX_WAL_8_3_OFFSET, 1 }, }; -#ifdef SQLITE_OS_WIN +#ifdef _WIN32 sqlite3_vfs *pVfs = sqlite3_vfs_find("win32"); #else sqlite3_vfs *pVfs = 0; diff --git a/src/test_fs.c b/src/test_fs.c index d821a83b9..1c47bbaac 100644 --- a/src/test_fs.c +++ b/src/test_fs.c @@ -69,18 +69,15 @@ #include #include -#if SQLITE_OS_UNIX || defined(__MINGW_H) +#if !defined(_WIN32) || defined(__MSVCRT__) # include # include # ifndef DIRENT # define DIRENT dirent # endif -#endif -#if SQLITE_OS_WIN +#else # include -# if !defined(__MINGW_H) -# include "test_windirent.h" -# endif +# include "test_windirent.h" # ifndef S_ISREG # define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG) # endif @@ -485,7 +482,7 @@ static int fstreeFilter( int nDir; char aWild[2] = { '\0', '\0' }; -#if SQLITE_OS_WIN +#ifdef _WIN32 const char *zDrive = windirent_getenv("fstreeDrive"); if( zDrive==0 ){ zDrive = windirent_getenv("SystemDrive"); @@ -538,7 +535,7 @@ static int fstreeFilter( sqlite3_bind_text(pCsr->pStmt, 2, zRoot, nRoot, SQLITE_TRANSIENT); sqlite3_bind_text(pCsr->pStmt, 3, zPrefix, nPrefix, SQLITE_TRANSIENT); -#if SQLITE_OS_WIN +#ifdef _WIN32 sqlite3_free(zPrefix); sqlite3_free(zRoot); #endif diff --git a/src/test_hexio.c b/src/test_hexio.c index 1a21e89aa..048ab1324 100644 --- a/src/test_hexio.c +++ b/src/test_hexio.c @@ -122,7 +122,7 @@ static int SQLITE_TCLAPI hexio_read( in = fopen(zFile, "r"); } if( in==0 ){ - Tcl_AppendResult(interp, "cannot open input file ", zFile, 0); + Tcl_AppendResult(interp, "cannot open input file ", zFile, NULL); return TCL_ERROR; } fseek(in, offset, SEEK_SET); @@ -132,7 +132,7 @@ static int SQLITE_TCLAPI hexio_read( got = 0; } sqlite3TestBinToHex(zBuf, got); - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); sqlite3_free(zBuf); return TCL_OK; } @@ -175,7 +175,7 @@ static int SQLITE_TCLAPI hexio_write( out = fopen(zFile, "r+"); } if( out==0 ){ - Tcl_AppendResult(interp, "cannot open output file ", zFile, 0); + Tcl_AppendResult(interp, "cannot open output file ", zFile, NULL); return TCL_ERROR; } fseek(out, offset, SEEK_SET); @@ -324,12 +324,12 @@ static int SQLITE_TCLAPI utf8_to_utf8( z[n] = 0; nOut = sqlite3Utf8To8(z); sqlite3TestBinToHex(z,nOut); - Tcl_AppendResult(interp, (char*)z, 0); + Tcl_AppendResult(interp, (char*)z, NULL); sqlite3_free(z); return TCL_OK; #else Tcl_AppendResult(interp, - "[utf8_to_utf8] unavailable - SQLITE_DEBUG not defined", 0 + "[utf8_to_utf8] unavailable - SQLITE_DEBUG not defined", NULL ); return TCL_ERROR; #endif diff --git a/src/test_init.c b/src/test_init.c index f7b85875b..0c6ac8eb5 100644 --- a/src/test_init.c +++ b/src/test_init.c @@ -201,7 +201,7 @@ static int SQLITE_TCLAPI init_wrapper_install( }else if( strcmp(z, "pcache")==0 ){ wrapped.pcache_fail = 1; }else{ - Tcl_AppendResult(interp, "Unknown argument: \"", z, "\""); + Tcl_AppendResult(interp, "Unknown argument: \"", z, "\"", NULL); return TCL_ERROR; } } diff --git a/src/test_malloc.c b/src/test_malloc.c index 8d6c4fa50..1c19d896f 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -646,7 +646,7 @@ static int SQLITE_TCLAPI test_memdebug_fail( } if( zErr ){ - Tcl_AppendResult(interp, zErr, zOption, 0); + Tcl_AppendResult(interp, zErr, zOption, NULL); return TCL_ERROR; } } diff --git a/src/test_multiplex.c b/src/test_multiplex.c index e5b43f4cc..82551200f 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -1315,8 +1315,8 @@ static int SQLITE_TCLAPI test_multiplex_control( } if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &cmdInfo) ){ - Tcl_AppendResult(interp, "expected database handle, got \"", 0); - Tcl_AppendResult(interp, Tcl_GetString(objv[1]), "\"", 0); + Tcl_AppendResult(interp, "expected database handle, got \"", NULL); + Tcl_AppendResult(interp, Tcl_GetString(objv[1]), "\"", NULL); return TCL_ERROR; }else{ db = *(sqlite3 **)cmdInfo.objClientData; diff --git a/src/test_mutex.c b/src/test_mutex.c index e60a06df3..de064de4c 100644 --- a/src/test_mutex.c +++ b/src/test_mutex.c @@ -225,8 +225,8 @@ static int SQLITE_TCLAPI test_install_mutex_counters( assert(isInstall==0 || isInstall==1); assert(g.isInstalled==0 || g.isInstalled==1); if( isInstall==g.isInstalled ){ - Tcl_AppendResult(interp, "mutex counters are ", 0); - Tcl_AppendResult(interp, isInstall?"already installed":"not installed", 0); + Tcl_AppendResult(interp, "mutex counters are ", NULL); + Tcl_AppendResult(interp, isInstall?"already installed":"not installed", NULL); return TCL_ERROR; } diff --git a/src/test_osinst.c b/src/test_osinst.c index 2d03d2bbc..e776d89e5 100644 --- a/src/test_osinst.c +++ b/src/test_osinst.c @@ -71,9 +71,8 @@ #include "sqlite3.h" -#include "os_setup.h" -#if SQLITE_OS_WIN -# include "os_win.h" +#ifdef _WIN32 +#include #endif #include @@ -219,14 +218,7 @@ static sqlite3_io_methods vfslog_io_methods = { vfslogShmUnmap /* xShmUnmap */ }; -#if SQLITE_OS_UNIX && !defined(NO_GETTOD) -#include -static sqlite3_uint64 vfslog_time(){ - struct timeval sTime; - gettimeofday(&sTime, 0); - return sTime.tv_usec + (sqlite3_uint64)sTime.tv_sec * 1000000; -} -#elif SQLITE_OS_WIN +#ifdef _WIN32 #include static sqlite3_uint64 vfslog_time(){ FILETIME ft; @@ -241,6 +233,13 @@ static sqlite3_uint64 vfslog_time(){ /* ft is 100-nanosecond intervals, we want microseconds */ return u64time /(sqlite3_uint64)10; } +#elif !defined(NO_GETTOD) +#include +static sqlite3_uint64 vfslog_time(){ + struct timeval sTime; + gettimeofday(&sTime, 0); + return sTime.tv_usec + (sqlite3_uint64)sTime.tv_sec * 1000000; +} #else static sqlite3_uint64 vfslog_time(){ return 0; @@ -1146,7 +1145,7 @@ static int SQLITE_TCLAPI test_vfslog( zMsg = Tcl_GetString(objv[3]); rc = sqlite3_vfslog_annotate(zVfs, zMsg); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, "failed", 0); + Tcl_AppendResult(interp, "failed", (char*)0); return TCL_ERROR; } break; @@ -1160,7 +1159,7 @@ static int SQLITE_TCLAPI test_vfslog( zVfs = Tcl_GetString(objv[2]); rc = sqlite3_vfslog_finalize(zVfs); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, "failed", 0); + Tcl_AppendResult(interp, "failed", (char*)0); return TCL_ERROR; } break; @@ -1180,7 +1179,7 @@ static int SQLITE_TCLAPI test_vfslog( if( *zParent=='\0' ) zParent = 0; rc = sqlite3_vfslog_new(zVfs, zParent, zLog); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, "failed", 0); + Tcl_AppendResult(interp, "failed", (char*)0); return TCL_ERROR; } break; diff --git a/src/test_quota.c b/src/test_quota.c index 1bfc5ce11..d2f9cddd1 100644 --- a/src/test_quota.c +++ b/src/test_quota.c @@ -44,14 +44,12 @@ #define sqlite3_mutex_notheld(X) ((void)(X),1) #endif /* SQLITE_THREADSAFE==0 */ -#include "os_setup.h" -#if SQLITE_OS_UNIX -# include -#endif -#if SQLITE_OS_WIN -# include "os_win.h" +#ifdef _WIN32 +# include # include +#else +# include #endif @@ -130,7 +128,7 @@ struct quota_FILE { FILE *f; /* Open stdio file pointer */ sqlite3_int64 iOfst; /* Current offset into the file */ quotaFile *pFile; /* The file record in the quota system */ -#if SQLITE_OS_WIN +#ifdef _WIN32 char *zMbcsName; /* Full MBCS pathname of the file */ #endif }; @@ -375,7 +373,7 @@ static quotaFile *quotaFindFile( ** used to store the returned pointer when done. */ static char *quota_utf8_to_mbcs(const char *zUtf8){ -#if SQLITE_OS_WIN +#ifdef _WIN32 size_t n; /* Bytes in zUtf8 */ int nWide; /* number of UTF-16 characters */ int nMbcs; /* Bytes of MBCS */ @@ -389,7 +387,11 @@ static char *quota_utf8_to_mbcs(const char *zUtf8){ zTmpWide = (LPWSTR)sqlite3_malloc( (nWide+1)*sizeof(zTmpWide[0]) ); if( zTmpWide==0 ) return 0; MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, zTmpWide, nWide); +#ifdef SQLITE_OS_WINRT + codepage = CP_ACP; +#else codepage = AreFileApisANSI() ? CP_ACP : CP_OEMCP; +#endif nMbcs = WideCharToMultiByte(codepage, 0, zTmpWide, nWide, 0, 0, 0, 0); zMbcs = nMbcs ? (char*)sqlite3_malloc( nMbcs+1 ) : 0; if( zMbcs ){ @@ -406,7 +408,7 @@ static char *quota_utf8_to_mbcs(const char *zUtf8){ ** Deallocate any memory allocated by quota_utf8_to_mbcs(). */ static void quota_mbcs_free(char *zOld){ -#if SQLITE_OS_WIN +#ifdef _WIN32 sqlite3_free(zOld); #else /* No-op on unix */ @@ -966,7 +968,7 @@ quota_FILE *sqlite3_quota_fopen(const char *zFilename, const char *zMode){ } quotaLeave(); sqlite3_free(zFull); -#if SQLITE_OS_WIN +#ifdef _WIN32 p->zMbcsName = zFullTranslated; #endif return p; @@ -1069,7 +1071,7 @@ int sqlite3_quota_fclose(quota_FILE *p){ } quotaLeave(); } -#if SQLITE_OS_WIN +#ifdef _WIN32 quota_mbcs_free(p->zMbcsName); #endif sqlite3_free(p); @@ -1083,11 +1085,10 @@ int sqlite3_quota_fflush(quota_FILE *p, int doFsync){ int rc; rc = fflush(p->f); if( rc==0 && doFsync ){ -#if SQLITE_OS_UNIX - rc = fsync(fileno(p->f)); -#endif -#if SQLITE_OS_WIN +#ifdef _WIN32 rc = _commit(_fileno(p->f)); +#else + rc = fsync(fileno(p->f)); #endif } return rc!=0; @@ -1139,17 +1140,16 @@ int sqlite3_quota_ftruncate(quota_FILE *p, sqlite3_int64 szNew){ pGroup->iSize += szNew - pFile->iSize; quotaLeave(); } -#if SQLITE_OS_UNIX - rc = ftruncate(fileno(p->f), szNew); -#endif -#if SQLITE_OS_WIN -# if defined(__MINGW32__) && defined(SQLITE_TEST) +#ifdef _WIN32 +# if defined(__MSVCRT__) && defined(SQLITE_TEST) /* _chsize_s() is missing from MingW (as of 2012-11-06). Use ** _chsize() as a work-around for testing purposes. */ rc = _chsize(_fileno(p->f), (long)szNew); # else rc = _chsize_s(_fileno(p->f), szNew); # endif +#else + rc = ftruncate(fileno(p->f), szNew); #endif if( pFile && rc==0 ){ quotaGroup *pGroup = pFile->pGroup; @@ -1168,13 +1168,12 @@ int sqlite3_quota_ftruncate(quota_FILE *p, sqlite3_int64 szNew){ */ int sqlite3_quota_file_mtime(quota_FILE *p, time_t *pTime){ int rc; -#if SQLITE_OS_UNIX - struct stat buf; - rc = fstat(fileno(p->f), &buf); -#endif -#if SQLITE_OS_WIN +#ifdef _WIN32 struct _stati64 buf; rc = _stati64(p->zMbcsName, &buf); +#else + struct stat buf; + rc = fstat(fileno(p->f), &buf); #endif if( rc==0 ) *pTime = buf.st_mtime; return rc; @@ -1186,13 +1185,12 @@ int sqlite3_quota_file_mtime(quota_FILE *p, time_t *pTime){ */ sqlite3_int64 sqlite3_quota_file_truesize(quota_FILE *p){ int rc; -#if SQLITE_OS_UNIX - struct stat buf; - rc = fstat(fileno(p->f), &buf); -#endif -#if SQLITE_OS_WIN +#ifdef _WIN32 struct _stati64 buf; rc = _stati64(p->zMbcsName, &buf); +#else + struct stat buf; + rc = fstat(fileno(p->f), &buf); #endif return rc==0 ? buf.st_size : -1; } diff --git a/src/test_sqllog.c b/src/test_sqllog.c index 9ae0e5068..5abf59a8b 100644 --- a/src/test_sqllog.c +++ b/src/test_sqllog.c @@ -84,7 +84,7 @@ #include #include static int getProcessId(void){ -#if SQLITE_OS_WIN +#ifdef _WIN32 return (int)_getpid(); #else return (int)getpid(); diff --git a/src/test_superlock.c b/src/test_superlock.c index 7f3bf163a..82997927c 100644 --- a/src/test_superlock.c +++ b/src/test_superlock.c @@ -338,7 +338,7 @@ static int SQLITE_TCLAPI superlock_cmd( if( rc!=SQLITE_OK ){ extern const char *sqlite3ErrStr(int); Tcl_ResetResult(interp); - Tcl_AppendResult(interp, sqlite3ErrStr(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrStr(rc), NULL); return TCL_ERROR; } diff --git a/src/test_syscall.c b/src/test_syscall.c index af2ae1001..35c303f8e 100644 --- a/src/test_syscall.c +++ b/src/test_syscall.c @@ -686,7 +686,7 @@ static int SQLITE_TCLAPI test_syscall_pagesize( } }else{ if( pgsz<512 || (pgsz & (pgsz-1)) ){ - Tcl_AppendResult(interp, "pgsz out of range", 0); + Tcl_AppendResult(interp, "pgsz out of range", NULL); return TCL_ERROR; } gSyscall.orig_getpagesize = pVfs->xGetSystemCall(pVfs, "getpagesize"); @@ -729,7 +729,7 @@ static int SQLITE_TCLAPI test_syscall( return TCL_ERROR; } if( pVfs->iVersion<3 || pVfs->xSetSystemCall==0 ){ - Tcl_AppendResult(interp, "VFS does not support xSetSystemCall", 0); + Tcl_AppendResult(interp, "VFS does not support xSetSystemCall", NULL); rc = TCL_ERROR; }else{ rc = Tcl_GetIndexFromObjStruct(interp, diff --git a/src/test_thread.c b/src/test_thread.c index 7c06d110a..98ef2c246 100644 --- a/src/test_thread.c +++ b/src/test_thread.c @@ -201,7 +201,7 @@ static int SQLITE_TCLAPI sqlthread_spawn( rc = Tcl_CreateThread(&x, tclScriptThread, (void *)pNew, nStack, flags); if( rc!=TCL_OK ){ - Tcl_AppendResult(interp, "Error in Tcl_CreateThread()", 0); + Tcl_AppendResult(interp, "Error in Tcl_CreateThread()", NULL); ckfree((char *)pNew); return TCL_ERROR; } @@ -235,7 +235,7 @@ static int SQLITE_TCLAPI sqlthread_parent( UNUSED_PARAMETER(objc); if( p==0 ){ - Tcl_AppendResult(interp, "no parent thread", 0); + Tcl_AppendResult(interp, "no parent thread", NULL); return TCL_ERROR; } @@ -287,7 +287,7 @@ static int SQLITE_TCLAPI sqlthread_open( sqlite3_busy_handler(db, xBusy, 0); if( sqlite3TestMakePointerStr(interp, zBuf, db) ) return TCL_ERROR; - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); return TCL_OK; } @@ -614,13 +614,13 @@ static int SQLITE_TCLAPI blocking_prepare_v2_proc( if( rc!=SQLITE_OK ){ assert( pStmt==0 ); sqlite3_snprintf(sizeof(zBuf), zBuf, "%s ", (char *)sqlite3ErrName(rc)); - Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), 0); + Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), NULL); return TCL_ERROR; } if( pStmt ){ if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ) return TCL_ERROR; - Tcl_AppendResult(interp, zBuf, 0); + Tcl_AppendResult(interp, zBuf, NULL); } return TCL_OK; } diff --git a/src/test_vfs.c b/src/test_vfs.c index 9f84b4f80..0d90a53a5 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -130,8 +130,9 @@ struct Testvfs { #define TESTVFS_LOCK_MASK 0x00040000 #define TESTVFS_CKLOCK_MASK 0x00080000 #define TESTVFS_FCNTL_MASK 0x00100000 +#define TESTVFS_SLEEP_MASK 0x00200000 -#define TESTVFS_ALL_MASK 0x001FFFFF +#define TESTVFS_ALL_MASK 0x003FFFFF #define TESTVFS_MAX_PAGES 1024 @@ -813,6 +814,10 @@ static int tvfsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ ** actually slept. */ static int tvfsSleep(sqlite3_vfs *pVfs, int nMicro){ + Testvfs *p = (Testvfs *)pVfs->pAppData; + if( p->pScript && (p->mask&TESTVFS_SLEEP_MASK) ){ + tvfsExecTcl(p, "xSleep", Tcl_NewIntObj(nMicro), 0, 0, 0); + } return sqlite3OsSleep(PARENTVFS(pVfs), nMicro); } @@ -1133,7 +1138,7 @@ static int SQLITE_TCLAPI testvfs_obj_cmd( ); if( rc!=SQLITE_OK ){ Tcl_AppendResult(interp, "failed to get full path: ", - Tcl_GetString(objv[2]), 0); + Tcl_GetString(objv[2]), NULL); ckfree(zName); return TCL_ERROR; } @@ -1142,7 +1147,7 @@ static int SQLITE_TCLAPI testvfs_obj_cmd( } ckfree(zName); if( !pBuffer ){ - Tcl_AppendResult(interp, "no such file: ", Tcl_GetString(objv[2]), 0); + Tcl_AppendResult(interp, "no such file: ", Tcl_GetString(objv[2]), NULL); return TCL_ERROR; } if( objc==4 ){ @@ -1197,6 +1202,7 @@ static int SQLITE_TCLAPI testvfs_obj_cmd( { "xLock", TESTVFS_LOCK_MASK }, { "xCheckReservedLock", TESTVFS_CKLOCK_MASK }, { "xFileControl", TESTVFS_FCNTL_MASK }, + { "xSleep", TESTVFS_SLEEP_MASK }, }; Tcl_Obj **apElem = 0; Tcl_Size nElem = 0; @@ -1219,7 +1225,7 @@ static int SQLITE_TCLAPI testvfs_obj_cmd( } } if( iMethod==ArraySize(vfsmethod) ){ - Tcl_AppendResult(interp, "unknown method: ", zElem, 0); + Tcl_AppendResult(interp, "unknown method: ", zElem, NULL); return TCL_ERROR; } } @@ -1347,7 +1353,7 @@ static int SQLITE_TCLAPI testvfs_obj_cmd( return TCL_ERROR; } if( aFlag[idx].iValue<0 && nFlags>1 ){ - Tcl_AppendResult(interp, "bad flags: ", Tcl_GetString(objv[2]), 0); + Tcl_AppendResult(interp, "bad flags: ", Tcl_GetString(objv[2]), NULL); return TCL_ERROR; } iNew |= aFlag[idx].iValue; @@ -1667,7 +1673,7 @@ static int SQLITE_TCLAPI test_vfs_set_readmark( return TCL_ERROR; } if( pShm==0 ){ - Tcl_AppendResult(interp, "*-shm is not yet mapped", 0); + Tcl_AppendResult(interp, "*-shm is not yet mapped", NULL); return TCL_ERROR; } aShm = (u32*)pShm; diff --git a/src/test_windirent.c b/src/test_windirent.c index 62165c4be..de4192d7c 100644 --- a/src/test_windirent.c +++ b/src/test_windirent.c @@ -48,11 +48,13 @@ const char *windirent_getenv( ** Implementation of the POSIX opendir() function using the MSVCRT. */ LPDIR opendir( - const char *dirname + const char *dirname /* Directory name, UTF8 encoding */ ){ - struct _finddata_t data; + struct _wfinddata_t data; LPDIR dirp = (LPDIR)sqlite3_malloc(sizeof(DIR)); SIZE_T namesize = sizeof(data.name) / sizeof(data.name[0]); + wchar_t *b1; + sqlite3_int64 sz; if( dirp==NULL ) return NULL; memset(dirp, 0, sizeof(DIR)); @@ -62,9 +64,25 @@ LPDIR opendir( dirname = windirent_getenv("SystemDrive"); } - memset(&data, 0, sizeof(struct _finddata_t)); - _snprintf(data.name, namesize, "%s\\*", dirname); - dirp->d_handle = _findfirst(data.name, &data); + memset(&data, 0, sizeof(data)); + sz = strlen(dirname); + b1 = sqlite3_malloc64( (sz+3)*sizeof(b1[0]) ); + if( b1==0 ){ + closedir(dirp); + return NULL; + } + sz = MultiByteToWideChar(CP_UTF8, 0, dirname, sz, b1, sz); + b1[sz++] = '\\'; + b1[sz++] = '*'; + b1[sz] = 0; + if( sz+1>(sqlite3_int64)namesize ){ + closedir(dirp); + sqlite3_free(b1); + return NULL; + } + memcpy(data.name, b1, (sz+1)*sizeof(b1[0])); + sqlite3_free(b1); + dirp->d_handle = _wfindfirst(data.name, &data); if( dirp->d_handle==BAD_INTPTR_T ){ closedir(dirp); @@ -75,8 +93,8 @@ LPDIR opendir( if( is_filtered(data) ){ next: - memset(&data, 0, sizeof(struct _finddata_t)); - if( _findnext(dirp->d_handle, &data)==-1 ){ + memset(&data, 0, sizeof(data)); + if( _wfindnext(dirp->d_handle, &data)==-1 ){ closedir(dirp); return NULL; } @@ -86,9 +104,8 @@ LPDIR opendir( } dirp->d_first.d_attributes = data.attrib; - strncpy(dirp->d_first.d_name, data.name, NAME_MAX); - dirp->d_first.d_name[NAME_MAX] = '\0'; - + WideCharToMultiByte(CP_UTF8, 0, data.name, -1, + dirp->d_first.d_name, DIRENT_NAME_MAX, 0, 0); return dirp; } @@ -98,7 +115,7 @@ LPDIR opendir( LPDIRENT readdir( LPDIR dirp ){ - struct _finddata_t data; + struct _wfinddata_t data; if( dirp==NULL ) return NULL; @@ -111,65 +128,19 @@ LPDIRENT readdir( next: - memset(&data, 0, sizeof(struct _finddata_t)); - if( _findnext(dirp->d_handle, &data)==-1 ) return NULL; + memset(&data, 0, sizeof(data)); + if( _wfindnext(dirp->d_handle, &data)==-1 ) return NULL; /* TODO: Remove this block to allow hidden and/or system files. */ if( is_filtered(data) ) goto next; dirp->d_next.d_ino++; dirp->d_next.d_attributes = data.attrib; - strncpy(dirp->d_next.d_name, data.name, NAME_MAX); - dirp->d_next.d_name[NAME_MAX] = '\0'; - + WideCharToMultiByte(CP_UTF8, 0, data.name, -1, + dirp->d_next.d_name, DIRENT_NAME_MAX, 0, 0); return &dirp->d_next; } -/* -** Implementation of the POSIX readdir_r() function using the MSVCRT. -*/ -INT readdir_r( - LPDIR dirp, - LPDIRENT entry, - LPDIRENT *result -){ - struct _finddata_t data; - - if( dirp==NULL ) return EBADF; - - if( dirp->d_first.d_ino==0 ){ - dirp->d_first.d_ino++; - dirp->d_next.d_ino++; - - entry->d_ino = dirp->d_first.d_ino; - entry->d_attributes = dirp->d_first.d_attributes; - strncpy(entry->d_name, dirp->d_first.d_name, NAME_MAX); - entry->d_name[NAME_MAX] = '\0'; - - *result = entry; - return 0; - } - -next: - - memset(&data, 0, sizeof(struct _finddata_t)); - if( _findnext(dirp->d_handle, &data)==-1 ){ - *result = NULL; - return ENOENT; - } - - /* TODO: Remove this block to allow hidden and/or system files. */ - if( is_filtered(data) ) goto next; - - entry->d_ino = (ino_t)-1; /* not available */ - entry->d_attributes = data.attrib; - strncpy(entry->d_name, data.name, NAME_MAX); - entry->d_name[NAME_MAX] = '\0'; - - *result = entry; - return 0; -} - /* ** Implementation of the POSIX closedir() function using the MSVCRT. */ diff --git a/src/test_windirent.h b/src/test_windirent.h index 28ce66778..527dfaa8f 100644 --- a/src/test_windirent.h +++ b/src/test_windirent.h @@ -88,6 +88,7 @@ # else # define NAME_MAX (260) # endif +# define DIRENT_NAME_MAX (NAME_MAX) #endif /* @@ -131,8 +132,7 @@ struct DIR { /* ** Provide a macro, for use by the implementation, to determine if a ** particular directory entry should be skipped over when searching for -** the next directory entry that should be returned by the readdir() or -** readdir_r() functions. +** the next directory entry that should be returned by the readdir(). */ #ifndef is_filtered @@ -148,12 +148,11 @@ extern const char *windirent_getenv(const char *name); /* ** Finally, we can provide the function prototypes for the opendir(), -** readdir(), readdir_r(), and closedir() POSIX functions. +** readdir(), and closedir() POSIX functions. */ extern LPDIR opendir(const char *dirname); extern LPDIRENT readdir(LPDIR dirp); -extern INT readdir_r(LPDIR dirp, LPDIRENT entry, LPDIRENT *result); extern INT closedir(LPDIR dirp); #endif /* defined(WIN32) && defined(_MSC_VER) */ diff --git a/src/tokenize.c b/src/tokenize.c index fe300ca52..e4d9f5371 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -692,7 +692,11 @@ int sqlite3RunParser(Parse *pParse, const char *zSql){ assert( n==6 ); tokenType = analyzeFilterKeyword((const u8*)&zSql[6], lastTokenParsed); #endif /* SQLITE_OMIT_WINDOWFUNC */ - }else if( tokenType==TK_COMMENT && (db->flags & SQLITE_Comments)!=0 ){ + }else if( tokenType==TK_COMMENT + && (db->init.busy || (db->flags & SQLITE_Comments)!=0) + ){ + /* Ignore SQL comments if either (1) we are reparsing the schema or + ** (2) SQLITE_DBCONFIG_ENABLE_COMMENTS is turned on (the default). */ zSql += n; continue; }else if( tokenType!=TK_QNUMBER ){ diff --git a/src/trigger.c b/src/trigger.c index e306a2e66..779da5e5f 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -70,7 +70,8 @@ Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){ assert( pParse->db->pVtabCtx==0 ); #endif assert( pParse->bReturning ); - assert( &(pParse->u1.pReturning->retTrig) == pTrig ); + assert( !pParse->isCreate ); + assert( &(pParse->u1.d.pReturning->retTrig) == pTrig ); pTrig->table = pTab->zName; pTrig->pTabSchema = pTab->pSchema; pTrig->pNext = pList; @@ -1038,7 +1039,8 @@ static void codeReturningTrigger( ExprList *pNew; Returning *pReturning; Select sSelect; - SrcList sFrom; + SrcList *pFrom; + u8 fromSpace[SZ_SRCLIST_1]; assert( v!=0 ); if( !pParse->bReturning ){ @@ -1047,19 +1049,21 @@ static void codeReturningTrigger( return; } assert( db->pParse==pParse ); - pReturning = pParse->u1.pReturning; + assert( !pParse->isCreate ); + pReturning = pParse->u1.d.pReturning; if( pTrigger != &(pReturning->retTrig) ){ /* This RETURNING trigger is for a different statement */ return; } memset(&sSelect, 0, sizeof(sSelect)); - memset(&sFrom, 0, sizeof(sFrom)); + pFrom = (SrcList*)fromSpace; + memset(pFrom, 0, SZ_SRCLIST_1); sSelect.pEList = sqlite3ExprListDup(db, pReturning->pReturnEL, 0); - sSelect.pSrc = &sFrom; - sFrom.nSrc = 1; - sFrom.a[0].pSTab = pTab; - sFrom.a[0].zName = pTab->zName; /* tag-20240424-1 */ - sFrom.a[0].iCursor = -1; + sSelect.pSrc = pFrom; + pFrom->nSrc = 1; + pFrom->a[0].pSTab = pTab; + pFrom->a[0].zName = pTab->zName; /* tag-20240424-1 */ + pFrom->a[0].iCursor = -1; sqlite3SelectPrep(pParse, &sSelect, 0); if( pParse->nErr==0 ){ assert( db->mallocFailed==0 ); @@ -1277,6 +1281,8 @@ static TriggerPrg *codeRowTrigger( sSubParse.eTriggerOp = pTrigger->op; sSubParse.nQueryLoop = pParse->nQueryLoop; sSubParse.prepFlags = pParse->prepFlags; + sSubParse.oldmask = 0; + sSubParse.newmask = 0; v = sqlite3GetVdbe(&sSubParse); if( v ){ diff --git a/src/update.c b/src/update.c index a8e7f7780..979afea1f 100644 --- a/src/update.c +++ b/src/update.c @@ -465,38 +465,32 @@ void sqlite3Update( */ chngRowid = chngPk = 0; for(i=0; inExpr; i++){ - u8 hCol = sqlite3StrIHash(pChanges->a[i].zEName); /* If this is an UPDATE with a FROM clause, do not resolve expressions ** here. The call to sqlite3Select() below will do that. */ if( nChangeFrom==0 && sqlite3ResolveExprNames(&sNC, pChanges->a[i].pExpr) ){ goto update_cleanup; } - for(j=0; jnCol; j++){ - if( pTab->aCol[j].hName==hCol - && sqlite3StrICmp(pTab->aCol[j].zCnName, pChanges->a[i].zEName)==0 - ){ - if( j==pTab->iPKey ){ - chngRowid = 1; - pRowidExpr = pChanges->a[i].pExpr; - iRowidExpr = i; - }else if( pPk && (pTab->aCol[j].colFlags & COLFLAG_PRIMKEY)!=0 ){ - chngPk = 1; - } + j = sqlite3ColumnIndex(pTab, pChanges->a[i].zEName); + if( j>=0 ){ + if( j==pTab->iPKey ){ + chngRowid = 1; + pRowidExpr = pChanges->a[i].pExpr; + iRowidExpr = i; + }else if( pPk && (pTab->aCol[j].colFlags & COLFLAG_PRIMKEY)!=0 ){ + chngPk = 1; + } #ifndef SQLITE_OMIT_GENERATED_COLUMNS - else if( pTab->aCol[j].colFlags & COLFLAG_GENERATED ){ - testcase( pTab->aCol[j].colFlags & COLFLAG_VIRTUAL ); - testcase( pTab->aCol[j].colFlags & COLFLAG_STORED ); - sqlite3ErrorMsg(pParse, - "cannot UPDATE generated column \"%s\"", - pTab->aCol[j].zCnName); - goto update_cleanup; - } -#endif - aXRef[j] = i; - break; + else if( pTab->aCol[j].colFlags & COLFLAG_GENERATED ){ + testcase( pTab->aCol[j].colFlags & COLFLAG_VIRTUAL ); + testcase( pTab->aCol[j].colFlags & COLFLAG_STORED ); + sqlite3ErrorMsg(pParse, + "cannot UPDATE generated column \"%s\"", + pTab->aCol[j].zCnName); + goto update_cleanup; } - } - if( j>=pTab->nCol ){ +#endif + aXRef[j] = i; + }else{ if( pPk==0 && sqlite3IsRowid(pChanges->a[i].zEName) ){ j = -1; chngRowid = 1; diff --git a/src/utf.c b/src/utf.c index c934bb234..2efcd6791 100644 --- a/src/utf.c +++ b/src/utf.c @@ -105,6 +105,35 @@ static const unsigned char sqlite3Utf8Trans1[] = { } \ } +/* +** Write a single UTF8 character whose value is v into the +** buffer starting at zOut. zOut must be sized to hold at +** least four bytes. Return the number of bytes needed +** to encode the new character. +*/ +int sqlite3AppendOneUtf8Character(char *zOut, u32 v){ + if( v<0x00080 ){ + zOut[0] = (u8)(v & 0xff); + return 1; + } + if( v<0x00800 ){ + zOut[0] = 0xc0 + (u8)((v>>6) & 0x1f); + zOut[1] = 0x80 + (u8)(v & 0x3f); + return 2; + } + if( v<0x10000 ){ + zOut[0] = 0xe0 + (u8)((v>>12) & 0x0f); + zOut[1] = 0x80 + (u8)((v>>6) & 0x3f); + zOut[2] = 0x80 + (u8)(v & 0x3f); + return 3; + } + zOut[0] = 0xf0 + (u8)((v>>18) & 0x07); + zOut[1] = 0x80 + (u8)((v>>12) & 0x3f); + zOut[2] = 0x80 + (u8)((v>>6) & 0x3f); + zOut[3] = 0x80 + (u8)(v & 0x3f); + return 4; +} + /* ** Translate a single UTF-8 character. Return the unicode value. ** @@ -526,7 +555,7 @@ int sqlite3Utf16ByteLen(const void *zIn, int nByte, int nChar){ int n = 0; if( SQLITE_UTF16NATIVE==SQLITE_UTF16LE ) z++; - while( n=0xd8 && c<0xdc && z<=zEnd && z[0]>=0xdc && z[0]<0xe0 ) z += 2; diff --git a/src/util.c b/src/util.c index ecce460e0..8e4fd516e 100644 --- a/src/util.c +++ b/src/util.c @@ -1130,7 +1130,11 @@ void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRound){ } p->z = &p->zBuf[i+1]; assert( i+p->n < sizeof(p->zBuf) ); - while( ALWAYS(p->n>0) && p->z[p->n-1]=='0' ){ p->n--; } + assert( p->n>0 ); + while( p->z[p->n-1]=='0' ){ + p->n--; + assert( p->n>0 ); + } } /* @@ -1635,7 +1639,7 @@ int sqlite3MulInt64(i64 *pA, i64 iB){ } /* -** Compute the absolute value of a 32-bit signed integer, of possible. Or +** Compute the absolute value of a 32-bit signed integer, if possible. Or ** if the integer has a value of -2147483648, return +2147483647 */ int sqlite3AbsInt32(int x){ diff --git a/src/vacuum.c b/src/vacuum.c index e203f68c6..96d77e5bc 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -116,7 +116,7 @@ void sqlite3Vacuum(Parse *pParse, Token *pNm, Expr *pInto){ #else /* When SQLITE_BUG_COMPATIBLE_20160819 is defined, unrecognized arguments ** to VACUUM are silently ignored. This is a back-out of a bug fix that - ** occurred on 2016-08-19 (https://www.sqlite.org/src/info/083f9e6270). + ** occurred on 2016-08-19 (https://sqlite.org/src/info/083f9e6270). ** The buggy behavior is required for binary compatibility with some ** legacy applications. */ iDb = sqlite3FindDb(pParse->db, pNm); @@ -195,7 +195,7 @@ SQLITE_NOINLINE int sqlite3RunVacuum( saved_nChange = db->nChange; saved_nTotalChange = db->nTotalChange; saved_mTrace = db->mTrace; - db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks; + db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_Comments; db->mDbFlags |= DBFLAG_PreferBuiltin | DBFLAG_Vacuum; db->flags &= ~(u64)(SQLITE_ForeignKeys | SQLITE_ReverseOrder | SQLITE_Defensive | SQLITE_CountRows); diff --git a/src/vdbe.c b/src/vdbe.c index 3ebe7c049..29b6f9a65 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -276,11 +276,11 @@ static VdbeCursor *allocateCursor( */ Mem *pMem = iCur>0 ? &p->aMem[p->nMem-iCur] : p->aMem; - int nByte; + i64 nByte; VdbeCursor *pCx = 0; - nByte = - ROUND8P(sizeof(VdbeCursor)) + 2*sizeof(u32)*nField + - (eCurType==CURTYPE_BTREE?sqlite3BtreeCursorSize():0); + nByte = SZ_VDBECURSOR(nField); + assert( ROUND8(nByte)==nByte ); + if( eCurType==CURTYPE_BTREE ) nByte += sqlite3BtreeCursorSize(); assert( iCur>=0 && iCurnCursor ); if( p->apCsr[iCur] ){ /*OPTIMIZATION-IF-FALSE*/ @@ -304,7 +304,7 @@ static VdbeCursor *allocateCursor( pMem->szMalloc = 0; return 0; } - pMem->szMalloc = nByte; + pMem->szMalloc = (int)nByte; } p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->zMalloc; @@ -313,8 +313,8 @@ static VdbeCursor *allocateCursor( pCx->nField = nField; pCx->aOffset = &pCx->aType[nField]; if( eCurType==CURTYPE_BTREE ){ - pCx->uc.pCursor = (BtCursor*) - &pMem->z[ROUND8P(sizeof(VdbeCursor))+2*sizeof(u32)*nField]; + assert( ROUND8(SZ_VDBECURSOR(nField))==SZ_VDBECURSOR(nField) ); + pCx->uc.pCursor = (BtCursor*)&pMem->z[SZ_VDBECURSOR(nField)]; sqlite3BtreeCursorZero(pCx->uc.pCursor); } return pCx; @@ -1318,7 +1318,7 @@ case OP_Halt: { sqlite3VdbeError(p, "%s", pOp->p4.z); } pcx = (int)(pOp - aOp); - sqlite3_log(pOp->p1, "abort at %d in [%s]: %s", pcx, p->zSql, p->zErrMsg); + sqlite3_log(pOp->p1, "abort at %d: %s; [%s]", pcx, p->zErrMsg, p->zSql); } rc = sqlite3VdbeHalt(p); assert( rc==SQLITE_BUSY || rc==SQLITE_OK || rc==SQLITE_ERROR ); @@ -2644,7 +2644,7 @@ case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ break; } -/* Opcode: Once P1 P2 * * * +/* Opcode: Once P1 P2 P3 * * ** ** Fall through to the next instruction the first time this opcode is ** encountered on each invocation of the byte-code program. Jump to P2 @@ -2660,6 +2660,12 @@ case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ ** whether or not the jump should be taken. The bitmask is necessary ** because the self-altering code trick does not work for recursive ** triggers. +** +** The P3 operand is not used directly by this opcode. However P3 is +** used by the code generator as follows: If this opcode is the start +** of a subroutine and that subroutine uses a Bloom filter, then P3 will +** be the register that holds that Bloom filter. See tag-202407032019 +** in the source code for implementation details. */ case OP_Once: { /* jump */ u32 iAddr; /* Address of this instruction */ @@ -6057,7 +6063,7 @@ case OP_RowData: { /* The OP_RowData opcodes always follow OP_NotExists or ** OP_SeekRowid or OP_Rewind/Op_Next with no intervening instructions ** that might invalidate the cursor. - ** If this where not the case, on of the following assert()s + ** If this were not the case, one of the following assert()s ** would fail. Should this ever change (because of changes in the code ** generator) then the fix would be to insert a call to ** sqlite3VdbeCursorMoveto(). @@ -7326,7 +7332,7 @@ case OP_RowSetTest: { /* jump, in1, in3 */ */ case OP_Program: { /* jump0 */ int nMem; /* Number of memory registers for sub-program */ - int nByte; /* Bytes of runtime space required for sub-program */ + i64 nByte; /* Bytes of runtime space required for sub-program */ Mem *pRt; /* Register to allocate runtime space */ Mem *pMem; /* Used to iterate through memory cells */ Mem *pEnd; /* Last memory cell in new array */ @@ -7377,7 +7383,7 @@ case OP_Program: { /* jump0 */ nByte = ROUND8(sizeof(VdbeFrame)) + nMem * sizeof(Mem) + pProgram->nCsr * sizeof(VdbeCursor*) - + (pProgram->nOp + 7)/8; + + (7 + (i64)pProgram->nOp)/8; pFrame = sqlite3DbMallocZero(db, nByte); if( !pFrame ){ goto no_mem; @@ -7385,7 +7391,7 @@ case OP_Program: { /* jump0 */ sqlite3VdbeMemRelease(pRt); pRt->flags = MEM_Blob|MEM_Dyn; pRt->z = (char*)pFrame; - pRt->n = nByte; + pRt->n = (int)nByte; pRt->xDel = sqlite3VdbeFrameMemDel; pFrame->v = p; @@ -7484,12 +7490,14 @@ case OP_Param: { /* out2 */ ** statement counter is incremented (immediate foreign key constraints). */ case OP_FkCounter: { - if( db->flags & SQLITE_DeferFKs ){ - db->nDeferredImmCons += pOp->p2; - }else if( pOp->p1 ){ + if( pOp->p1 ){ db->nDeferredCons += pOp->p2; }else{ - p->nFkConstraint += pOp->p2; + if( db->flags & SQLITE_DeferFKs ){ + db->nDeferredImmCons += pOp->p2; + }else{ + p->nFkConstraint += pOp->p2; + } } break; } @@ -7704,7 +7712,7 @@ case OP_AggStep: { ** ** Note: We could avoid this by using a regular memory cell from aMem[] for ** the accumulator, instead of allocating one here. */ - nAlloc = ROUND8P( sizeof(pCtx[0]) + (n-1)*sizeof(sqlite3_value*) ); + nAlloc = ROUND8P( SZ_CONTEXT(n) ); pCtx = sqlite3DbMallocRawNN(db, nAlloc + sizeof(Mem)); if( pCtx==0 ) goto no_mem; pCtx->pOut = (Mem*)((u8*)pCtx + nAlloc); @@ -8364,6 +8372,7 @@ case OP_VFilter: { /* jump, ncycle */ /* Invoke the xFilter method */ apArg = p->apArg; + assert( nArg<=p->napArg ); for(i = 0; ivtabOnConflict; apArg = p->apArg; pX = &aMem[pOp->p3]; + assert( nArg<=p->napArg ); for(i=0; irc = rc; sqlite3SystemError(db, rc); testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(rc, "statement aborts at %d: [%s] %s", - (int)(pOp - aOp), p->zSql, p->zErrMsg); + sqlite3_log(rc, "statement aborts at %d: %s; [%s]", + (int)(pOp - aOp), p->zErrMsg, p->zSql); if( p->eVdbeState==VDBE_RUN_STATE ) sqlite3VdbeHalt(p); if( rc==SQLITE_IOERR_NOMEM ) sqlite3OomFault(db); if( rc==SQLITE_CORRUPT && db->autoCommit==0 ){ diff --git a/src/vdbe.h b/src/vdbe.h index 476f1b4ea..dc98e270e 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -319,8 +319,8 @@ int sqlite3NotPureFunc(sqlite3_context*); int sqlite3VdbeBytecodeVtabInit(sqlite3*); #endif -/* Use SQLITE_ENABLE_COMMENTS to enable generation of extra comments on -** each VDBE opcode. +/* Use SQLITE_ENABLE_EXPLAIN_COMMENTS to enable generation of extra +** comments on each VDBE opcode. ** ** Use the SQLITE_ENABLE_MODULE_COMMENTS macro to see some extra no-op ** comments in VDBE programs that show key decision points in the code diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 24cf1ac56..13262cd4e 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -133,12 +133,19 @@ struct VdbeCursor { #endif VdbeTxtBlbCache *pCache; /* Cache of large TEXT or BLOB values */ - /* 2*nField extra array elements allocated for aType[], beyond the one - ** static element declared in the structure. nField total array slots for - ** aType[] and nField+1 array slots for aOffset[] */ - u32 aType[1]; /* Type values record decode. MUST BE LAST */ + /* Space is allocated for aType to hold at least 2*nField+1 entries: + ** nField slots for aType[] and nField+1 array slots for aOffset[] */ + u32 aType[FLEXARRAY]; /* Type values record decode. MUST BE LAST */ }; +/* +** The size (in bytes) of a VdbeCursor object that has an nField value of N +** or less. The value of SZ_VDBECURSOR(n) is guaranteed to be a multiple +** of 8. +*/ +#define SZ_VDBECURSOR(N) \ + (ROUND8(offsetof(VdbeCursor,aType)) + ((N)+1)*sizeof(u64)) + /* Return true if P is a null-only cursor */ #define IsNullCursor(P) \ @@ -395,13 +402,16 @@ struct sqlite3_context { u8 enc; /* Encoding to use for results */ u8 skipFlag; /* Skip accumulator loading if true */ u16 argc; /* Number of arguments */ - sqlite3_value *argv[1]; /* Argument set */ + sqlite3_value *argv[FLEXARRAY]; /* Argument set */ }; -/* A bitfield type for use inside of structures. Always follow with :N where -** N is the number of bits. +/* +** The size (in bytes) of an sqlite3_context object that holds N +** argv[] arguments. */ -typedef unsigned bft; /* Bit Field Type */ +#define SZ_CONTEXT(N) \ + (offsetof(sqlite3_context,argv)+(N)*sizeof(sqlite3_value*)) + /* The ScanStatus object holds a single value for the ** sqlite3_stmt_scanstatus() interface. @@ -462,7 +472,7 @@ struct Vdbe { i64 nStmtDefCons; /* Number of def. constraints when stmt started */ i64 nStmtDefImmCons; /* Number of def. imm constraints when stmt started */ Mem *aMem; /* The memory locations */ - Mem **apArg; /* Arguments to currently executing user function */ + Mem **apArg; /* Arguments xUpdate and xFilter vtab methods */ VdbeCursor **apCsr; /* One element of this array for each open cursor */ Mem *aVar; /* Values for the OP_Variable opcode. */ @@ -482,6 +492,7 @@ struct Vdbe { #ifdef SQLITE_DEBUG int rcApp; /* errcode set by sqlite3_result_error_code() */ u32 nWrite; /* Number of write operations that have occurred */ + int napArg; /* Size of the apArg[] array */ #endif u16 nResColumn; /* Number of columns in one row of the result set */ u16 nResAlloc; /* Column slots allocated to aColName[] */ @@ -534,7 +545,7 @@ struct PreUpdate { VdbeCursor *pCsr; /* Cursor to read old values from */ int op; /* One of SQLITE_INSERT, UPDATE, DELETE */ u8 *aRecord; /* old.* database record */ - KeyInfo keyinfo; + KeyInfo *pKeyinfo; /* Key information */ UnpackedRecord *pUnpacked; /* Unpacked version of aRecord[] */ UnpackedRecord *pNewUnpacked; /* Unpacked version of new.* record */ int iNewReg; /* Register for new.* values */ @@ -546,6 +557,7 @@ struct PreUpdate { Table *pTab; /* Schema object being updated */ Index *pPk; /* PK index if pTab is WITHOUT ROWID */ sqlite3_value **apDflt; /* Array of default values, if required */ + u8 keyinfoSpace[SZ_KEYINFO(0)]; /* Space to hold pKeyinfo[0] content */ }; /* diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 0dc09d501..ed9549462 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1808,7 +1808,7 @@ int sqlite3_bind_text64( assert( xDel!=SQLITE_DYNAMIC ); if( enc!=SQLITE_UTF8 ){ if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; - nData &= ~(u16)1; + nData &= ~(u64)1; } return bindText(pStmt, i, zData, nData, xDel, enc); } @@ -2216,7 +2216,7 @@ int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ if( !aRec ) goto preupdate_old_out; rc = sqlite3BtreePayload(p->pCsr->uc.pCursor, 0, nRec, aRec); if( rc==SQLITE_OK ){ - p->pUnpacked = vdbeUnpackRecord(&p->keyinfo, nRec, aRec); + p->pUnpacked = vdbeUnpackRecord(p->pKeyinfo, nRec, aRec); if( !p->pUnpacked ) rc = SQLITE_NOMEM; } if( rc!=SQLITE_OK ){ @@ -2233,7 +2233,9 @@ int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ Column *pCol = &p->pTab->aCol[iIdx]; if( pCol->iDflt>0 ){ if( p->apDflt==0 ){ - int nByte = sizeof(sqlite3_value*)*p->pTab->nCol; + int nByte; + assert( sizeof(sqlite3_value*)*UMXV(p->pTab->nCol) < 0x7fffffff ); + nByte = sizeof(sqlite3_value*)*p->pTab->nCol; p->apDflt = (sqlite3_value**)sqlite3DbMallocZero(db, nByte); if( p->apDflt==0 ) goto preupdate_old_out; } @@ -2279,7 +2281,7 @@ int sqlite3_preupdate_count(sqlite3 *db){ #else p = db->pPreUpdate; #endif - return (p ? p->keyinfo.nKeyField : 0); + return (p ? p->pKeyinfo->nKeyField : 0); } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ @@ -2362,7 +2364,7 @@ int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ Mem *pData = &p->v->aMem[p->iNewReg]; rc = ExpandBlob(pData); if( rc!=SQLITE_OK ) goto preupdate_new_out; - pUnpack = vdbeUnpackRecord(&p->keyinfo, pData->n, pData->z); + pUnpack = vdbeUnpackRecord(p->pKeyinfo, pData->n, pData->z); if( !pUnpack ){ rc = SQLITE_NOMEM; goto preupdate_new_out; @@ -2383,7 +2385,8 @@ int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ */ assert( p->op==SQLITE_UPDATE ); if( !p->aNew ){ - p->aNew = (Mem *)sqlite3DbMallocZero(db, sizeof(Mem) * p->pCsr->nField); + assert( sizeof(Mem)*UMXV(p->pCsr->nField) < 0x7fffffff ); + p->aNew = (Mem *)sqlite3DbMallocZero(db, sizeof(Mem)*p->pCsr->nField); if( !p->aNew ){ rc = SQLITE_NOMEM; goto preupdate_new_out; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 81dca10f0..a6798e62d 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -445,12 +445,10 @@ int sqlite3VdbeAddFunctionCall( int eCallCtx /* Calling context */ ){ Vdbe *v = pParse->pVdbe; - int nByte; int addr; sqlite3_context *pCtx; assert( v ); - nByte = sizeof(*pCtx) + (nArg-1)*sizeof(sqlite3_value*); - pCtx = sqlite3DbMallocRawNN(pParse->db, nByte); + pCtx = sqlite3DbMallocRawNN(pParse->db, SZ_CONTEXT(nArg)); if( pCtx==0 ){ assert( pParse->db->mallocFailed ); freeEphemeralFunction(pParse->db, (FuncDef*)pFunc); @@ -726,7 +724,7 @@ static Op *opIterNext(VdbeOpIter *p){ } if( pRet->p4type==P4_SUBPROGRAM ){ - int nByte = (p->nSub+1)*sizeof(SubProgram*); + i64 nByte = (1+(u64)p->nSub)*sizeof(SubProgram*); int j; for(j=0; jnSub; j++){ if( p->apSub[j]==pRet->p4.pProgram ) break; @@ -856,8 +854,8 @@ void sqlite3VdbeAssertAbortable(Vdbe *p){ ** (1) For each jump instruction with a negative P2 value (a label) ** resolve the P2 value to an actual address. ** -** (2) Compute the maximum number of arguments used by any SQL function -** and store that value in *pMaxFuncArgs. +** (2) Compute the maximum number of arguments used by the xUpdate/xFilter +** methods of any virtual table and store that value in *pMaxVtabArgs. ** ** (3) Update the Vdbe.readOnly and Vdbe.bIsReader flags to accurately ** indicate what the prepared statement actually does. @@ -870,8 +868,8 @@ void sqlite3VdbeAssertAbortable(Vdbe *p){ ** script numbers the opcodes correctly. Changes to this routine must be ** coordinated with changes to mkopcodeh.tcl. */ -static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ - int nMaxArgs = *pMaxFuncArgs; +static void resolveP2Values(Vdbe *p, int *pMaxVtabArgs){ + int nMaxVtabArgs = *pMaxVtabArgs; Op *pOp; Parse *pParse = p->pParse; int *aLabel = pParse->aLabel; @@ -916,15 +914,19 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ } #ifndef SQLITE_OMIT_VIRTUALTABLE case OP_VUpdate: { - if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2; + if( pOp->p2>nMaxVtabArgs ) nMaxVtabArgs = pOp->p2; break; } case OP_VFilter: { int n; + /* The instruction immediately prior to VFilter will be an + ** OP_Integer that sets the "argc" value for the VFilter. See + ** the code where OP_VFilter is generated at tag-20250207a. */ assert( (pOp - p->aOp) >= 3 ); assert( pOp[-1].opcode==OP_Integer ); + assert( pOp[-1].p2==pOp->p3+1 ); n = pOp[-1].p1; - if( n>nMaxArgs ) nMaxArgs = n; + if( n>nMaxVtabArgs ) nMaxVtabArgs = n; /* Fall through into the default case */ /* no break */ deliberate_fall_through } @@ -965,7 +967,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ pParse->aLabel = 0; } pParse->nLabel = 0; - *pMaxFuncArgs = nMaxArgs; + *pMaxVtabArgs = nMaxVtabArgs; assert( p->bIsReader!=0 || DbMaskAllZero(p->btreeMask) ); } @@ -1194,7 +1196,7 @@ void sqlite3VdbeScanStatus( const char *zName /* Name of table or index being scanned */ ){ if( IS_STMT_SCANSTATUS(p->db) ){ - sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus); + i64 nByte = (1+(i64)p->nScan) * sizeof(ScanStatus); ScanStatus *aNew; aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); if( aNew ){ @@ -1304,6 +1306,9 @@ void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){ */ void sqlite3VdbeTypeofColumn(Vdbe *p, int iDest){ VdbeOp *pOp = sqlite3VdbeGetLastOp(p); +#ifdef SQLITE_DEBUG + while( pOp->opcode==OP_ReleaseReg ) pOp--; +#endif if( pOp->p3==iDest && pOp->opcode==OP_Column ){ pOp->p5 |= OPFLAG_TYPEOFARG; } @@ -2643,7 +2648,7 @@ void sqlite3VdbeMakeReady( int nVar; /* Number of parameters */ int nMem; /* Number of VM memory registers */ int nCursor; /* Number of cursors required */ - int nArg; /* Number of arguments in subprograms */ + int nArg; /* Max number args to xFilter or xUpdate */ int n; /* Loop counter */ struct ReusableSpace x; /* Reusable bulk memory */ @@ -2715,6 +2720,9 @@ void sqlite3VdbeMakeReady( p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*)); } } +#ifdef SQLITE_DEBUG + p->napArg = nArg; +#endif if( db->mallocFailed ){ p->nVar = 0; @@ -4212,6 +4220,7 @@ UnpackedRecord *sqlite3VdbeAllocUnpackedRecord( ){ UnpackedRecord *p; /* Unpacked record to return */ int nByte; /* Number of bytes required for *p */ + assert( sizeof(UnpackedRecord) + sizeof(Mem)*65536 < 0x7fffffff ); nByte = ROUND8P(sizeof(UnpackedRecord)) + sizeof(Mem)*(pKeyInfo->nKeyField+1); p = (UnpackedRecord *)sqlite3DbMallocRaw(pKeyInfo->db, nByte); if( !p ) return 0; @@ -5518,10 +5527,11 @@ void sqlite3VdbePreUpdateHook( preupdate.pCsr = pCsr; preupdate.op = op; preupdate.iNewReg = iReg; - preupdate.keyinfo.db = db; - preupdate.keyinfo.enc = ENC(db); - preupdate.keyinfo.nKeyField = pTab->nCol; - preupdate.keyinfo.aSortFlags = (u8*)&fakeSortOrder; + preupdate.pKeyinfo = (KeyInfo*)&preupdate.keyinfoSpace; + preupdate.pKeyinfo->db = db; + preupdate.pKeyinfo->enc = ENC(db); + preupdate.pKeyinfo->nKeyField = pTab->nCol; + preupdate.pKeyinfo->aSortFlags = (u8*)&fakeSortOrder; preupdate.iKey1 = iKey1; preupdate.iKey2 = iKey2; preupdate.pTab = pTab; @@ -5531,8 +5541,8 @@ void sqlite3VdbePreUpdateHook( db->xPreUpdateCallback(db->pPreUpdateArg, db, op, zDb, zTbl, iKey1, iKey2); db->pPreUpdate = 0; sqlite3DbFree(db, preupdate.aRecord); - vdbeFreeUnpacked(db, preupdate.keyinfo.nKeyField+1, preupdate.pUnpacked); - vdbeFreeUnpacked(db, preupdate.keyinfo.nKeyField+1, preupdate.pNewUnpacked); + vdbeFreeUnpacked(db, preupdate.pKeyinfo->nKeyField+1,preupdate.pUnpacked); + vdbeFreeUnpacked(db, preupdate.pKeyinfo->nKeyField+1,preupdate.pNewUnpacked); sqlite3VdbeMemRelease(&preupdate.oldipk); if( preupdate.aNew ){ int i; diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 6cb36da37..42edcf7de 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -133,6 +133,7 @@ int sqlite3_blob_open( char *zErr = 0; Table *pTab; Incrblob *pBlob = 0; + int iDb; Parse sParse; #ifdef SQLITE_ENABLE_API_ARMOR @@ -178,7 +179,10 @@ int sqlite3_blob_open( sqlite3ErrorMsg(&sParse, "cannot open view: %s", zTable); } #endif - if( !pTab ){ + if( pTab==0 + || ((iDb = sqlite3SchemaToIndex(db, pTab->pSchema))==1 && + sqlite3OpenTempDatabase(&sParse)) + ){ if( sParse.zErrMsg ){ sqlite3DbFree(db, zErr); zErr = sParse.zErrMsg; @@ -189,15 +193,11 @@ int sqlite3_blob_open( goto blob_open_out; } pBlob->pTab = pTab; - pBlob->zDb = db->aDb[sqlite3SchemaToIndex(db, pTab->pSchema)].zDbSName; + pBlob->zDb = db->aDb[iDb].zDbSName; /* Now search pTab for the exact column. */ - for(iCol=0; iColnCol; iCol++) { - if( sqlite3StrICmp(pTab->aCol[iCol].zCnName, zColumn)==0 ){ - break; - } - } - if( iCol==pTab->nCol ){ + iCol = sqlite3ColumnIndex(pTab, zColumn); + if( iCol<0 ){ sqlite3DbFree(db, zErr); zErr = sqlite3MPrintf(db, "no such column: \"%s\"", zColumn); rc = SQLITE_ERROR; @@ -277,7 +277,6 @@ int sqlite3_blob_open( {OP_Halt, 0, 0, 0}, /* 5 */ }; Vdbe *v = (Vdbe *)pBlob->pStmt; - int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); VdbeOp *aOp; sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, wrFlag, diff --git a/src/vdbemem.c b/src/vdbemem.c index 38ba5abe8..6db9e4b1a 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -141,7 +141,7 @@ static void vdbeMemRenderNum(int sz, char *zBuf, Mem *p){ ** corresponding string value, then it is important that the string be ** derived from the numeric value, not the other way around, to ensure ** that the index and table are consistent. See ticket -** https://www.sqlite.org/src/info/343634942dd54ab (2018-01-31) for +** https://sqlite.org/src/info/343634942dd54ab (2018-01-31) for ** an example. ** ** This routine looks at pMem to verify that if it has both a numeric @@ -327,7 +327,7 @@ void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ return; } if( pMem->enc!=SQLITE_UTF8 ) return; - if( NEVER(pMem->z==0) ) return; + assert( pMem->z!=0 ); if( pMem->flags & MEM_Dyn ){ if( pMem->xDel==sqlite3_free && sqlite3_msize(pMem->z) >= (u64)(pMem->n+1) @@ -1440,7 +1440,7 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ if( pRec==0 ){ Index *pIdx = p->pIdx; /* Index being probed */ - int nByte; /* Bytes of space to allocate */ + i64 nByte; /* Bytes of space to allocate */ int i; /* Counter variable */ int nCol = pIdx->nColumn; /* Number of index columns including rowid */ @@ -1506,7 +1506,7 @@ static int valueFromFunction( ){ sqlite3_context ctx; /* Context object for function invocation */ sqlite3_value **apVal = 0; /* Function arguments */ - int nVal = 0; /* Size of apVal[] array */ + int nVal = 0; /* Number of function arguments */ FuncDef *pFunc = 0; /* Function definition */ sqlite3_value *pVal = 0; /* New value */ int rc = SQLITE_OK; /* Return code */ diff --git a/src/vdbesort.c b/src/vdbesort.c index 239c0a0f3..6b1b4cff5 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -332,9 +332,12 @@ struct VdbeSorter { u8 iPrev; /* Previous thread used to flush PMA */ u8 nTask; /* Size of aTask[] array */ u8 typeMask; - SortSubtask aTask[1]; /* One or more subtasks */ + SortSubtask aTask[FLEXARRAY]; /* One or more subtasks */ }; +/* Size (in bytes) of a VdbeSorter object that works with N or fewer subtasks */ +#define SZ_VDBESORTER(N) (offsetof(VdbeSorter,aTask)+(N)*sizeof(SortSubtask)) + #define SORTER_TYPE_INTEGER 0x01 #define SORTER_TYPE_TEXT 0x02 @@ -936,7 +939,7 @@ int sqlite3VdbeSorterInit( VdbeSorter *pSorter; /* The new sorter */ KeyInfo *pKeyInfo; /* Copy of pCsr->pKeyInfo with db==0 */ int szKeyInfo; /* Size of pCsr->pKeyInfo in bytes */ - int sz; /* Size of pSorter in bytes */ + i64 sz; /* Size of pSorter in bytes */ int rc = SQLITE_OK; #if SQLITE_MAX_WORKER_THREADS==0 # define nWorker 0 @@ -964,8 +967,10 @@ int sqlite3VdbeSorterInit( assert( pCsr->pKeyInfo ); assert( !pCsr->isEphemeral ); assert( pCsr->eCurType==CURTYPE_SORTER ); - szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nKeyField-1)*sizeof(CollSeq*); - sz = sizeof(VdbeSorter) + nWorker * sizeof(SortSubtask); + assert( sizeof(KeyInfo) + UMXV(pCsr->pKeyInfo->nKeyField)*sizeof(CollSeq*) + < 0x7fffffff ); + szKeyInfo = SZ_KEYINFO(pCsr->pKeyInfo->nKeyField); + sz = SZ_VDBESORTER(nWorker+1); pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo); pCsr->uc.pSorter = pSorter; @@ -1177,7 +1182,7 @@ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ */ static MergeEngine *vdbeMergeEngineNew(int nReader){ int N = 2; /* Smallest power of two >= nReader */ - int nByte; /* Total bytes of space to allocate */ + i64 nByte; /* Total bytes of space to allocate */ MergeEngine *pNew; /* Pointer to allocated object to return */ assert( nReader<=SORTER_MAX_MERGE_COUNT ); @@ -1429,6 +1434,10 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ p->u.pNext = 0; for(i=0; aSlot[i]; i++){ p = vdbeSorterMerge(pTask, p, aSlot[i]); + /* ,--Each aSlot[] holds twice as much as the previous. So we cannot use + ** | up all 64 aSlots[] with only a 64-bit address space. + ** v */ + assert( iregRowid holds the rowid of an + ** The VM register number pParse->u1.cr.regRowid holds the rowid of an ** entry in the sqlite_schema table that was created for this vtab ** by sqlite3StartTable(). */ iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( pParse->isCreate ); sqlite3NestedParse(pParse, "UPDATE %Q." LEGACY_SCHEMA_TABLE " " "SET type='table', name=%Q, tbl_name=%Q, rootpage=0, sql=%Q " @@ -492,7 +493,7 @@ void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ pTab->zName, pTab->zName, zStmt, - pParse->regRowid + pParse->u1.cr.regRowid ); v = sqlite3GetVdbe(pParse); sqlite3ChangeCookie(pParse, iDb); diff --git a/src/wal.c b/src/wal.c index 42ce3cb97..1fd5b201c 100644 --- a/src/wal.c +++ b/src/wal.c @@ -502,6 +502,11 @@ struct WalCkptInfo { /* ** An open write-ahead log file is represented by an instance of the ** following object. +** +** writeLock: +** This is usually set to 1 whenever the WRITER lock is held. However, +** if it is set to 2, then the WRITER lock is held but must be released +** by walHandleException() if a SEH exception is thrown. */ struct Wal { sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */ @@ -592,9 +597,13 @@ struct WalIterator { u32 *aPgno; /* Array of page numbers. */ int nEntry; /* Nr. of entries in aPgno[] and aIndex[] */ int iZero; /* Frame number associated with aPgno[0] */ - } aSegment[1]; /* One for every 32KB page in the wal-index */ + } aSegment[FLEXARRAY]; /* One for every 32KB page in the wal-index */ }; +/* Size (in bytes) of a WalIterator object suitable for N or fewer segments */ +#define SZ_WALITERATOR(N) \ + (offsetof(WalIterator,aSegment)*(N)*sizeof(struct WalSegment)) + /* ** Define the parameters of the hash tables in the wal-index file. There ** is a hash-table following every HASHTABLE_NPAGE page numbers in the @@ -753,7 +762,7 @@ static SQLITE_NOINLINE int walIndexPageRealloc( /* Enlarge the pWal->apWiData[] array if required */ if( pWal->nWiData<=iPage ){ - sqlite3_int64 nByte = sizeof(u32*)*(iPage+1); + sqlite3_int64 nByte = sizeof(u32*)*(1+(i64)iPage); volatile u32 **apNew; apNew = (volatile u32 **)sqlite3Realloc((void *)pWal->apWiData, nByte); if( !apNew ){ @@ -862,10 +871,8 @@ static void walChecksumBytes( s1 = s2 = 0; } - assert( nByte>=8 ); - assert( (nByte&0x00000007)==0 ); - assert( nByte<=65536 ); - assert( nByte%4==0 ); + /* nByte is a multiple of 8 between 8 and 65536 */ + assert( nByte>=8 && (nByte&7)==0 && nByte<=65536 ); if( !nativeCksum ){ do { @@ -1955,8 +1962,7 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ /* Allocate space for the WalIterator object. */ nSegment = walFramePage(iLast) + 1; - nByte = sizeof(WalIterator) - + (nSegment-1)*sizeof(struct WalSegment) + nByte = SZ_WALITERATOR(nSegment) + iLast*sizeof(ht_slot); p = (WalIterator *)sqlite3_malloc64(nByte + sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) @@ -2027,7 +2033,7 @@ static int walEnableBlockingMs(Wal *pWal, int nMs){ static int walEnableBlocking(Wal *pWal){ int res = 0; if( pWal->db ){ - int tmout = pWal->db->busyTimeout; + int tmout = pWal->db->setlkTimeout; if( tmout ){ res = walEnableBlockingMs(pWal, tmout); } @@ -2413,7 +2419,9 @@ static int walHandleException(Wal *pWal){ static const int S = 1; static const int E = (1<lockMask & ~( + u32 mUnlock; + if( pWal->writeLock==2 ) pWal->writeLock = 0; + mUnlock = pWal->lockMask & ~( (pWal->readLock<0 ? 0 : (S << WAL_READ_LOCK(pWal->readLock))) | (pWal->writeLock ? (E << WAL_WRITE_LOCK) : 0) | (pWal->ckptLock ? (E << WAL_CKPT_LOCK) : 0) @@ -2685,7 +2693,12 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ if( bWriteLock || SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) ){ - pWal->writeLock = 1; + /* If the write-lock was just obtained, set writeLock to 2 instead of + ** the usual 1. This causes walIndexPage() to behave as if the + ** write-lock were held (so that it allocates new pages as required), + ** and walHandleException() to unlock the write-lock if a SEH exception + ** is thrown. */ + if( !bWriteLock ) pWal->writeLock = 2; if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ badHdr = walIndexTryHdr(pWal, pChanged); if( badHdr ){ @@ -3049,7 +3062,6 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){ rc = walIndexReadHdr(pWal, pChanged); } #ifdef SQLITE_ENABLE_SETLK_TIMEOUT - walDisableBlocking(pWal); if( rc==SQLITE_BUSY_TIMEOUT ){ rc = SQLITE_BUSY; *pCnt |= WAL_RETRY_BLOCKED_MASK; @@ -3064,6 +3076,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){ ** WAL_RETRY this routine will be called again and will probably be ** right on the second iteration. */ + (void)walEnableBlocking(pWal); if( pWal->apWiData[0]==0 ){ /* This branch is taken when the xShmMap() method returns SQLITE_BUSY. ** We assume this is a transient condition, so return WAL_RETRY. The @@ -3080,6 +3093,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){ rc = SQLITE_BUSY_RECOVERY; } } + walDisableBlocking(pWal); if( rc!=SQLITE_OK ){ return rc; } @@ -3470,8 +3484,11 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ ** read-lock. */ void sqlite3WalEndReadTransaction(Wal *pWal){ - sqlite3WalEndWriteTransaction(pWal); +#ifndef SQLITE_ENABLE_SETLK_TIMEOUT + assert( pWal->writeLock==0 || pWal->readLock<0 ); +#endif if( pWal->readLock>=0 ){ + sqlite3WalEndWriteTransaction(pWal); walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); pWal->readLock = -1; } @@ -3664,7 +3681,7 @@ int sqlite3WalBeginWriteTransaction(Wal *pWal){ ** read-transaction was even opened, making this call a no-op. ** Return early. */ if( pWal->writeLock ){ - assert( !memcmp(&pWal->hdr,(void *)walIndexHdr(pWal),sizeof(WalIndexHdr)) ); + assert( !memcmp(&pWal->hdr,(void*)pWal->apWiData[0],sizeof(WalIndexHdr)) ); return SQLITE_OK; } #endif diff --git a/src/where.c b/src/where.c index ff269750f..9561a75ea 100644 --- a/src/where.c +++ b/src/where.c @@ -35,11 +35,16 @@ struct HiddenIndexInfo { int eDistinct; /* Value to return from sqlite3_vtab_distinct() */ u32 mIn; /* Mask of terms that are IN (...) */ u32 mHandleIn; /* Terms that vtab will handle as IN (...) */ - sqlite3_value *aRhs[1]; /* RHS values for constraints. MUST BE LAST - ** because extra space is allocated to hold up - ** to nTerm such values */ + sqlite3_value *aRhs[FLEXARRAY]; /* RHS values for constraints. MUST BE LAST + ** Extra space is allocated to hold up + ** to nTerm such values */ }; +/* Size (in bytes) of a HiddenIndeInfo object sufficient to hold as +** many as N constraints */ +#define SZ_HIDDENINDEXINFO(N) \ + (offsetof(HiddenIndexInfo,aRhs) + (N)*sizeof(sqlite3_value*)) + /* Forward declaration of methods */ static int whereLoopResize(sqlite3*, WhereLoop*, int); @@ -1104,6 +1109,8 @@ static SQLITE_NOINLINE void constructAutomaticIndex( } /* Construct the Index object to describe this index */ + assert( nKeyCol <= pTable->nCol + MAX(0, pTable->nCol - BMS + 1) ); + /* ^-- This guarantees that the number of index columns will fit in the u16 */ pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+HasRowid(pTable), 0, &zNotUsed); if( pIdx==0 ) goto end_auto_index_create; @@ -1515,8 +1522,8 @@ static sqlite3_index_info *allocateIndexInfo( */ pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo) + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm - + sizeof(*pIdxOrderBy)*nOrderBy + sizeof(*pHidden) - + sizeof(sqlite3_value*)*nTerm ); + + sizeof(*pIdxOrderBy)*nOrderBy + + SZ_HIDDENINDEXINFO(nTerm) ); if( pIdxInfo==0 ){ sqlite3ErrorMsg(pParse, "out of memory"); return 0; @@ -3152,11 +3159,8 @@ static int whereLoopAddBtreeIndex( assert( pNew->u.btree.nBtm==0 ); opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS; } - if( pProbe->bUnordered || pProbe->bLowQual ){ - if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); - if( pProbe->bLowQual && pSrc->fg.isIndexedBy==0 ){ - opMask &= ~(WO_EQ|WO_IN|WO_IS); - } + if( pProbe->bUnordered ){ + opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); } assert( pNew->u.btree.nEqnColumn ); @@ -3598,6 +3602,7 @@ static int whereUsablePartialIndex( if( (!ExprHasProperty(pExpr, EP_OuterON) || pExpr->w.iJoin==iTab) && ((jointype & JT_OUTER)==0 || ExprHasProperty(pExpr, EP_OuterON)) && sqlite3ExprImpliesExpr(pParse, pExpr, pWhere, iTab) + && !sqlite3ExprImpliesExpr(pParse, pExpr, pWhere, -1) && (pTerm->wtFlags & TERM_VNULL)==0 ){ return 1; @@ -4093,7 +4098,7 @@ static int whereLoopAddBtree( && (HasRowid(pTab) || pWInfo->pSelect!=0 || sqlite3FaultSim(700)) ){ WHERETRACE(0x200, - ("-> %s a covering index according to bitmasks\n", + ("-> %s is a covering index according to bitmasks\n", pProbe->zName, m==0 ? "is" : "is not")); pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; } @@ -6710,10 +6715,7 @@ WhereInfo *sqlite3WhereBegin( ** field (type Bitmask) it must be aligned on an 8-byte boundary on ** some architectures. Hence the ROUND8() below. */ - nByteWInfo = ROUND8P(sizeof(WhereInfo)); - if( nTabList>1 ){ - nByteWInfo = ROUND8P(nByteWInfo + (nTabList-1)*sizeof(WhereLevel)); - } + nByteWInfo = SZ_WHEREINFO(nTabList); pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop)); if( db->mallocFailed ){ sqlite3DbFree(db, pWInfo); @@ -6930,7 +6932,8 @@ WhereInfo *sqlite3WhereBegin( } /* TUNING: Assume that a DISTINCT clause on a subquery reduces - ** the output size by a factor of 8 (LogEst -30). + ** the output size by a factor of 8 (LogEst -30). Search for + ** tag-20250414a to see other cases. */ if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 ){ WHERETRACE(0x0080,("nRowOut reduced from %d to %d due to DISTINCT\n", diff --git a/src/whereInt.h b/src/whereInt.h index 8ba8a7072..40a720ab9 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -501,9 +501,14 @@ struct WhereInfo { Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ WhereClause sWC; /* Decomposition of the WHERE clause */ WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ - WhereLevel a[1]; /* Information about each nest loop in WHERE */ + WhereLevel a[FLEXARRAY]; /* Information about each nest loop in WHERE */ }; +/* +** The size (in bytes) of a WhereInfo object that holds N WhereLevels. +*/ +#define SZ_WHEREINFO(N) ROUND8(offsetof(WhereInfo,a)+(N)*sizeof(WhereLevel)) + /* ** Private interfaces - callable only by other where.c routines. ** diff --git a/src/wherecode.c b/src/wherecode.c index 045653aac..8e3e56cb1 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -542,7 +542,7 @@ static void adjustOrderByCol(ExprList *pOrderBy, ExprList *pEList){ /* ** pX is an expression of the form: (vector) IN (SELECT ...) ** In other words, it is a vector IN operator with a SELECT clause on the -** LHS. But not all terms in the vector are indexable and the terms might +** RHS. But not all terms in the vector are indexable and the terms might ** not be in the correct order for indexing. ** ** This routine makes a copy of the input pX expression and then adjusts @@ -600,7 +600,7 @@ static Expr *removeUnindexableInClauseTerms( iField = pLoop->aLTerm[i]->u.x.iField - 1; if( pOrigRhs->a[iField].pExpr==0 ) continue; /* Duplicate PK column */ pRhs = sqlite3ExprListAppend(pParse, pRhs, pOrigRhs->a[iField].pExpr); - pOrigRhs->a[iField].pExpr = 0; + pOrigRhs->a[iField].pExpr = 0; if( pRhs ) pRhs->a[pRhs->nExpr-1].u.x.iOrderByCol = iField+1; if( pOrigLhs ){ assert( pOrigLhs->a[iField].pExpr!=0 ); @@ -1608,6 +1608,9 @@ Bitmask sqlite3WhereCodeOneLoopStart( } sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg); sqlite3VdbeAddOp2(v, OP_Integer, nConstraint, iReg+1); + /* The instruction immediately prior to OP_VFilter must be an OP_Integer + ** that sets the "argc" value for xVFilter. This is necessary for + ** resolveP2() to work correctly. See tag-20250207a. */ sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, pLoop->u.vtab.idxStr, pLoop->u.vtab.needFree ? P4_DYNAMIC : P4_STATIC); @@ -2198,12 +2201,13 @@ Bitmask sqlite3WhereCodeOneLoopStart( if( pLevel->iLeftJoin==0 ){ /* If a partial index is driving the loop, try to eliminate WHERE clause ** terms from the query that must be true due to the WHERE clause of - ** the partial index. + ** the partial index. This optimization does not work on an outer join, + ** as shown by: ** - ** 2019-11-02 ticket 623eff57e76d45f6: This optimization does not work - ** for a LEFT JOIN. + ** 2019-11-02 ticket 623eff57e76d45f6 (LEFT JOIN) + ** 2025-05-29 forum post 7dee41d32506c4ae (RIGHT JOIN) */ - if( pIdx->pPartIdxWhere ){ + if( pIdx->pPartIdxWhere && pLevel->pRJ==0 ){ whereApplyPartialIndexConstraints(pIdx->pPartIdxWhere, iCur, pWC); } }else{ @@ -2310,8 +2314,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( int nNotReady; /* The number of notReady tables */ SrcItem *origSrc; /* Original list of tables */ nNotReady = pWInfo->nLevel - iLevel - 1; - pOrTab = sqlite3DbMallocRawNN(db, - sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); + pOrTab = sqlite3DbMallocRawNN(db, SZ_SRCLIST(nNotReady+1)); if( pOrTab==0 ) return notReady; pOrTab->nAlloc = (u8)(nNotReady + 1); pOrTab->nSrc = pOrTab->nAlloc; @@ -2362,7 +2365,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( ** ** This optimization also only applies if the (x1 OR x2 OR ...) term ** is not contained in the ON clause of a LEFT JOIN. - ** See ticket http://www.sqlite.org/src/info/f2369304e4 + ** See ticket http://sqlite.org/src/info/f2369304e4 ** ** 2022-02-04: Do not push down slices of a row-value comparison. ** In other words, "w" or "y" may not be a slice of a vector. Otherwise, @@ -2854,7 +2857,8 @@ SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( WhereInfo *pSubWInfo; WhereLoop *pLoop = pLevel->pWLoop; SrcItem *pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; - SrcList sFrom; + SrcList *pFrom; + u8 fromSpace[SZ_SRCLIST_1]; Bitmask mAll = 0; int k; @@ -2898,13 +2902,14 @@ SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( sqlite3ExprDup(pParse->db, pTerm->pExpr, 0)); } } - sFrom.nSrc = 1; - sFrom.nAlloc = 1; - memcpy(&sFrom.a[0], pTabItem, sizeof(SrcItem)); - sFrom.a[0].fg.jointype = 0; + pFrom = (SrcList*)fromSpace; + pFrom->nSrc = 1; + pFrom->nAlloc = 1; + memcpy(&pFrom->a[0], pTabItem, sizeof(SrcItem)); + pFrom->a[0].fg.jointype = 0; assert( pParse->withinRJSubrtn < 100 ); pParse->withinRJSubrtn++; - pSubWInfo = sqlite3WhereBegin(pParse, &sFrom, pSubWhere, 0, 0, 0, + pSubWInfo = sqlite3WhereBegin(pParse, pFrom, pSubWhere, 0, 0, 0, WHERE_RIGHT_JOIN, 0); if( pSubWInfo ){ int iCur = pLevel->iTabCur; diff --git a/src/window.c b/src/window.c index 8373e3641..948b0728a 100644 --- a/src/window.c +++ b/src/window.c @@ -995,7 +995,7 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){ p->pWhere = 0; p->pGroupBy = 0; p->pHaving = 0; - p->selFlags &= ~SF_Aggregate; + p->selFlags &= ~(u32)SF_Aggregate; p->selFlags |= SF_WinRewrite; /* Create the ORDER BY clause for the sub-select. This is the concatenation diff --git a/test/affinity2.test b/test/affinity2.test index 6ad257ac3..59f9dada5 100644 --- a/test/affinity2.test +++ b/test/affinity2.test @@ -116,7 +116,7 @@ do_execsql_test 507 { SELECT * FROM t0 WHERE +-+'ce' >= t0.c0; } {-1 {}} -# 2019-08-30 ticket https://www.sqlite.org/src/info/40812aea1fde9594 +# 2019-08-30 ticket https://sqlite.org/src/info/40812aea1fde9594 # # Due to some differences in floating point computations, these tests do not # work under valgrind. diff --git a/test/affinity3.test b/test/affinity3.test index 3695ea847..be415de46 100644 --- a/test/affinity3.test +++ b/test/affinity3.test @@ -11,14 +11,14 @@ # # Test cases for bugs: # -# https://www.sqlite.org/src/info/91e2e8ba6ff2e2 -# https://www.sqlite.org/src/info/7ffd1ca1d2ad4ecf +# https://sqlite.org/src/info/91e2e8ba6ff2e2 +# https://sqlite.org/src/info/7ffd1ca1d2ad4ecf # set testdir [file dirname $argv0] source $testdir/tester.tcl -# Ticket https://www.sqlite.org/src/info/91e2e8ba6ff2e2 (2011-09-19) +# Ticket https://sqlite.org/src/info/91e2e8ba6ff2e2 (2011-09-19) # Automatic index causes undesired type conversions # do_execsql_test affinity3-100 { @@ -87,7 +87,7 @@ do_execsql_test affinity3-142 { SELECT id, (apr / 100), typeof(apr) apr_type FROM v2rjrj; } {1 0.12 real 2 0.1201 real} -# Ticket https://www.sqlite.org/src/info/7ffd1ca1d2ad4ecf (2017-01-16) +# Ticket https://sqlite.org/src/info/7ffd1ca1d2ad4ecf (2017-01-16) # Incorrect affinity when using automatic indexes # do_execsql_test affinity3-200 { diff --git a/test/all.test b/test/all.test index 46e8115fb..22d7b8dae 100644 --- a/test/all.test +++ b/test/all.test @@ -42,7 +42,7 @@ run_test_suite pcache100 run_test_suite prepare run_test_suite mmap -if {$::tcl_platform(platform)=="unix"} { +if {$::tcl_platform(platform) eq "unix"} { ifcapable !default_autovacuum { run_test_suite autovacuum_crash } diff --git a/test/alter3.test b/test/alter3.test index c6f26b0c5..464e20aee 100644 --- a/test/alter3.test +++ b/test/alter3.test @@ -101,7 +101,7 @@ do_test alter3-1.7 { } {{CREATE TABLE t3(a, b, c VARCHAR(10, 20), UNIQUE(a, b))}} do_test alter3-1.99 { catchsql { - # May not exist if foriegn-keys are omitted at compile time. + # May not exist if foreign-keys are omitted at compile time. DROP TABLE t2; } execsql { diff --git a/test/alter4.test b/test/alter4.test index c63ba6b07..7e2d7e66a 100644 --- a/test/alter4.test +++ b/test/alter4.test @@ -110,7 +110,7 @@ do_test alter4-1.7 { } {{CREATE TABLE t3(a, b, c VARCHAR(10, 20), UNIQUE(a, b))}} do_test alter4-1.99 { catchsql { - # May not exist if foriegn-keys are omitted at compile time. + # May not exist if foreign-keys are omitted at compile time. DROP TABLE t2; } execsql { @@ -379,7 +379,7 @@ do_execsql_test alter4-9.3 { # Confirm that doing an ALTER TABLE on a legacy format database # does not corrupt DESC indexes. # -# Ticket https://www.sqlite.org/src/tktview/f68bf68513a1c +# Ticket https://sqlite.org/src/tktview/f68bf68513a1c # do_test alter4-10.1 { db close diff --git a/test/altercol.test b/test/altercol.test index f44aa2e06..5f7de57a4 100644 --- a/test/altercol.test +++ b/test/altercol.test @@ -796,7 +796,7 @@ do_execsql_test 19.1 { {CREATE VIEW v2(e) AS SELECT coalesce(t2.c,t1.f) FROM t1, t2 WHERE t1.b=t2.d} } -# 2019-01-08: https://www.sqlite.org/src/tktview/bc8d94f0fbd633fd9a051e3 +# 2019-01-08: https://sqlite.org/src/tktview/bc8d94f0fbd633fd9a051e3 # # ALTER TABLE RENAME COLUMN does not work for tables that have redundant # UNIQUE constraints. diff --git a/test/altertab2.test b/test/altertab2.test index 576dc4967..56e42f1a6 100644 --- a/test/altertab2.test +++ b/test/altertab2.test @@ -351,7 +351,7 @@ do_execsql_test 8.5 { SELECT sql FROM sqlite_master WHERE name = 'v4' } {{CREATE VIEW v4 AS SELECT * FROM t4 WHERE (c=1 AND 0) OR b=2}} -# 2019-06-10 https://www.sqlite.org/src/info/533010b8cacebe82 +# 2019-06-10 https://sqlite.org/src/info/533010b8cacebe82 reset_db do_catchsql_test 8.6 { CREATE TABLE t0(c0); diff --git a/test/analyze3.test b/test/analyze3.test index c5d7a7cb1..033dfa5df 100644 --- a/test/analyze3.test +++ b/test/analyze3.test @@ -48,7 +48,7 @@ if {[permutation]=="prepare"} { # query plan when there is no way in which replanning the # query may produce a superior outcome. # -# analyze3-4.*: Test that SQL or authorization callback errors occuring +# analyze3-4.*: Test that SQL or authorization callback errors occurring # within sqlite3Reprepare() are handled correctly. # # analyze3-5.*: Check that the query plans of applicable statements are diff --git a/test/analyzeC.test b/test/analyzeC.test index 2f43d57a1..f5bcaeb78 100644 --- a/test/analyzeC.test +++ b/test/analyzeC.test @@ -133,7 +133,7 @@ do_execsql_test 4.3 { } {/.*INDEX t1ca.*/} # 2019-08-15. -# Ticket https://www.sqlite.org/src/tktview/e4598ecbdd18bd82945f602901 +# Ticket https://sqlite.org/src/tktview/e4598ecbdd18bd82945f602901 # The sz=N parameter in the sqlite_stat1 table needs to have a value of # 2 or more to avoid a division by zero in the query planner. # diff --git a/test/analyzer1.test b/test/analyzer1.test index 51b5f8b6a..1f0b0f637 100644 --- a/test/analyzer1.test +++ b/test/analyzer1.test @@ -19,7 +19,7 @@ ifcapable !vtab { return } -if {$tcl_platform(platform)=="windows"} { +if {$tcl_platform(platform) eq "windows"} { set PROG "sqlite3_analyzer.exe" } else { set PROG "./sqlite3_analyzer" diff --git a/test/attach.test b/test/attach.test index 557201d65..3445c43fa 100644 --- a/test/attach.test +++ b/test/attach.test @@ -759,7 +759,7 @@ do_test attach-6.1 { ATTACH DATABASE 'no-such-file' AS nosuch; } } {0 {}} -if {$tcl_platform(platform)=="unix"} { +if {$tcl_platform(platform) eq "unix"} { do_test attach-6.2 { sqlite3 dbx cannot-read dbx eval {CREATE TABLE t1(a,b,c)} diff --git a/test/autoinc.test b/test/autoinc.test index 2c7ee2a7e..9f869f35e 100644 --- a/test/autoinc.test +++ b/test/autoinc.test @@ -669,7 +669,7 @@ ifcapable trigger { } {1 124 2 10123} } -# 2016-10-03 ticket https://www.sqlite.org/src/tktview/7b3328086a5c1 +# 2016-10-03 ticket https://sqlite.org/src/tktview/7b3328086a5c1 # Make sure autoincrement plays nicely with the xfer optimization # do_execsql_test autoinc-10.1 { diff --git a/test/autoindex5.test b/test/autoindex5.test index aa8dec27d..adfc3e5f7 100644 --- a/test/autoindex5.test +++ b/test/autoindex5.test @@ -142,7 +142,7 @@ do_catchsql_test 2.2 { ); } {0 9} -# Ticket https://www.sqlite.org/src/info/787fa716be3a7f65 +# Ticket https://sqlite.org/src/info/787fa716be3a7f65 # Segfault due to multiple uses of the same subquery where the # subquery is implemented via coroutine. # diff --git a/test/avtrans.test b/test/avtrans.test index 6fc4a3e39..b483c71a4 100644 --- a/test/avtrans.test +++ b/test/avtrans.test @@ -903,7 +903,7 @@ for {set i 2} {$i<=$limit} {incr i} { INSERT INTO t3 SELECT randstr(10,400) FROM t3 WHERE random()%10==0; } } {} - if {$tcl_platform(platform)=="unix"} { + if {$tcl_platform(platform) eq "unix"} { do_test avtrans-9.$i.4-$cnt { expr {$sqlite_sync_count>0} } 1 diff --git a/test/backup2.test b/test/backup2.test index 1822e2dbf..095ecc752 100644 --- a/test/backup2.test +++ b/test/backup2.test @@ -141,7 +141,7 @@ do_test backup2-9 { # Try to restore from an unreadable file. # -if {$tcl_platform(platform)=="windows"} { +if {$::tcl_platform(os) eq "Windows NT"} { set msg {cannot open source database: unable to open database file} } elseif {[string match *BSD $tcl_platform(os)]} { set msg {} diff --git a/test/bc_common.tcl b/test/bc_common.tcl index c47f99681..953ce6262 100644 --- a/test/bc_common.tcl +++ b/test/bc_common.tcl @@ -9,7 +9,7 @@ proc bc_find_binaries {zCaption} { set binaries [list] set self [info nameofexec] set pattern "$self?*" - if {$::tcl_platform(platform)=="windows"} { + if {$::tcl_platform(platform) eq "windows"} { set pattern [string map {\.exe {}} $pattern] } foreach file [glob -nocomplain $pattern] { diff --git a/test/bloom1.test b/test/bloom1.test index f8efcc184..09553c3b9 100644 --- a/test/bloom1.test +++ b/test/bloom1.test @@ -224,6 +224,18 @@ do_execsql_test 5.3 { SELECT 0 as c_0 ); } {0} - + +# 2025-04-30 https://sqlite.org/forum/forumpost/792a09cb3df9e69f +# A continuation of the above. +# +do_execsql_test 6.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a); + SELECT 111 IN ( + SELECT 222 FROM (SELECT 333 ORDER BY 1) + UNION ALL + SELECT 444 FROM (SELECT 555 FROM t1 ORDER BY 1) + ); +} 0 finish_test diff --git a/test/btree01.test b/test/btree01.test index 6e4717ae6..b1909a3ad 100644 --- a/test/btree01.test +++ b/test/btree01.test @@ -17,7 +17,7 @@ source $testdir/tester.tcl set testprefix btree01 # The refactoring on the b-tree balance() routine in check-in -# http://www.sqlite.org/src/info/face33bea1ba3a (2014-10-27) +# http://sqlite.org/src/info/face33bea1ba3a (2014-10-27) # caused the integrity_check on the following SQL to fail. # do_execsql_test btree01-1.1 { diff --git a/test/cast.test b/test/cast.test index 1c9071d77..c1b564328 100644 --- a/test/cast.test +++ b/test/cast.test @@ -385,7 +385,7 @@ do_execsql_test cast-6.1 { } {9000000000000000001 9000000000000000001 9000000000000000001 9000000000000000001} # 2019-06-07 -# https://www.sqlite.org/src/info/4c2d7639f076aa7c +# https://sqlite.org/src/info/4c2d7639f076aa7c do_execsql_test cast-7.1 { SELECT CAST('-' AS NUMERIC); } {0} @@ -400,7 +400,7 @@ do_execsql_test cast-7.4 { } {0} # 2019-06-07 -# https://www.sqlite.org/src/info/e8bedb2a184001bb +# https://sqlite.org/src/info/e8bedb2a184001bb do_execsql_test cast-7.10 { SELECT '' - 2851427734582196970; } {-2851427734582196970} @@ -412,7 +412,7 @@ do_execsql_test cast-7.12 { } {-1} # 2019-06-10 -# https://www.sqlite.org/src/info/dd6bffbfb6e61db9 +# https://sqlite.org/src/info/dd6bffbfb6e61db9 # # EVIDENCE-OF: R-55084-10555 Casting a TEXT or BLOB value into NUMERIC # yields either an INTEGER or a REAL result. @@ -441,7 +441,7 @@ do_execsql_test cast-7.33 { } 0 # 2019-06-12 -# https://www.sqlite.org/src/info/674385aeba91c774 +# https://sqlite.org/src/info/674385aeba91c774 # do_execsql_test cast-7.40 { SELECT CAST('-0.0' AS numeric); diff --git a/test/check.test b/test/check.test index 10d1cf4be..c3beb2f5d 100644 --- a/test/check.test +++ b/test/check.test @@ -612,4 +612,36 @@ do_catchsql_test 12.81 { INSERT INTO t1(a) VALUES(456); } {1 {CHECK constraint failed: a NOT BETWEEN +a AND 999999}} +#------------------------------------------------------------------------- + +reset_db + +do_execsql_test 13.1.0 { + CREATE TABLE Table0 (Col0 , CHECK(Table0.Col0 NOT NULL ) ) ; + REPLACE INTO Table0 VALUES (hex(randomblob(100000))); +} +integrity_check 13.1.1 +do_execsql_test 13.1.2 { + UPDATE OR REPLACE Table0 SET Col0 = Table0.Col0 ; +} +integrity_check 13.1.3 +do_execsql_test 13.1.4 { + SELECT length(col0) FROM table0; +} {200000} + +do_execsql_test 13.2.0 { + CREATE TABLE t2 (x , CHECK((NOT (x ISNULL) ))); + REPLACE INTO t2 VALUES (hex(randomblob(100000))); +} +do_execsql_test 13.2.1 { + SELECT length(x) FROM t2 +} {200000} +do_execsql_test 13.2.2 { + UPDATE OR REPLACE t2 SET x = x; +} +do_execsql_test 13.2.3 { + SELECT length(x) FROM t2 +} {200000} +integrity_check 13.2.4 + finish_test diff --git a/test/chunksize.test b/test/chunksize.test index 47d118d84..be3795fc5 100644 --- a/test/chunksize.test +++ b/test/chunksize.test @@ -14,7 +14,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix chunksize -if {$::tcl_platform(platform)!="unix"} { +if {$::tcl_platform(platform) ne "unix"} { finish_test return } diff --git a/test/collate1.test b/test/collate1.test index b65b85047..da662bee6 100644 --- a/test/collate1.test +++ b/test/collate1.test @@ -23,7 +23,7 @@ set testprefix collate1 # collate1-1.* - Single-field ORDER BY with an explicit COLLATE clause. # collate1-2.* - Multi-field ORDER BY with an explicit COLLATE clause. # collate1-3.* - ORDER BY using a default collation type. Also that an -# explict collate type overrides a default collate type. +# explicit collate type overrides a default collate type. # collate1-4.* - ORDER BY using a data type. # diff --git a/test/collateB.test b/test/collateB.test index 3c8d50d8e..678eb5af0 100644 --- a/test/collateB.test +++ b/test/collateB.test @@ -62,7 +62,7 @@ do_execsql_test collateB-1.17 { } {1 11 1 11 |} #------------------------------------------------------------------------- -# Test an assert() failure that was occuring if an index were created +# Test an assert() failure that was occurring if an index were created # on a column explicitly declared "COLLATE binary". reset_db do_execsql_test 2.1 { diff --git a/test/colmeta.test b/test/colmeta.test index 21b0e0f38..43d2c83cb 100644 --- a/test/colmeta.test +++ b/test/colmeta.test @@ -97,7 +97,7 @@ foreach {tn params results} $tests { } # Calling sqlite3_table_column_metadata with a NULL column name merely -# checks for the existance of the table. +# checks for the existence of the table. # do_test colmeta-300 { catch {sqlite3_table_column_metadata $::DB main xyzzy} res diff --git a/test/colname.test b/test/colname.test index 24fb51acf..0907df641 100644 --- a/test/colname.test +++ b/test/colname.test @@ -378,7 +378,7 @@ do_test colname-9.210 { execsql2 {SELECT t1.a, v3.a AS n FROM t1 JOIN v3} } {a 1 n 3} -# 2017-12-23: Ticket https://www.sqlite.org/src/info/3b4450072511e621 +# 2017-12-23: Ticket https://sqlite.org/src/info/3b4450072511e621 # Inconsistent column names in CREATE TABLE AS # # Verify that the names of columns in the created table of a CREATE TABLE AS diff --git a/test/conflict.test b/test/conflict.test index ba3ad9b0d..54c01d36d 100644 --- a/test/conflict.test +++ b/test/conflict.test @@ -824,7 +824,7 @@ do_test conflict-13.2 { } {1 3} -# Ticket https://www.sqlite.org/src/tktview/e6f1f2e34dceeb1ed61531c7e9 +# Ticket https://sqlite.org/src/tktview/e6f1f2e34dceeb1ed61531c7e9 # Verify that it is not possible to sneak a NULL value into a NOT NULL # column using REPLACE. # diff --git a/test/crash8.test b/test/crash8.test index c07829979..b2b01183f 100644 --- a/test/crash8.test +++ b/test/crash8.test @@ -356,7 +356,7 @@ ifcapable pragma { # is not created on F2FS file-systems that support atomic # write. So do not run these tests in that case either. # -if {$::tcl_platform(platform)=="unix" && [atomic_batch_write test.db]==0 } { +if {$::tcl_platform(os) ne "Windows NT" && [atomic_batch_write test.db]==0 } { for {set i 1} {$i < 10} {incr i} { catch { db close } forcedelete test.db test.db-journal diff --git a/test/date.test b/test/date.test index 2042880a9..8badccf3f 100644 --- a/test/date.test +++ b/test/date.test @@ -584,7 +584,7 @@ datetest 16.30 {datetime(5373484,'-14712 years')} {-4713-12-31 12:00:00} datetest 16.31 {datetime(5373484,'-14713 years')} NULL # 2017-03-02: Wrong 'start of day' computation. -# https://www.sqlite.org/src/info/6097cb92745327a1 +# https://sqlite.org/src/info/6097cb92745327a1 # datetest 17.1 {datetime(2457754, 'start of day')} {2016-12-31 00:00:00} datetest 17.2 {datetime(2457828)} {2017-03-15 12:00:00} diff --git a/test/date4.test b/test/date4.test index 56a9090b1..4e936b71c 100644 --- a/test/date4.test +++ b/test/date4.test @@ -24,7 +24,13 @@ ifcapable {!datetime} { } if {$tcl_platform(os)=="Linux"} { - set FMT {%d,%e,%F,%H,%k,%I,%l,%j,%m,%M,%u,%w,%W,%Y,%%,%P,%p,%U,%V,%G,%g} + if {"" eq [strftime {%P} 1]} { + # This is probably musl libc, which does not support + # %k, %l, %P + set FMT {%d,%e,%F,%H,%I,%j,%m,%M,%u,%w,%W,%Y,%%,%p,%U,%V,%G,%g} + } else { + set FMT {%d,%e,%F,%H,%k,%I,%l,%j,%m,%M,%u,%w,%W,%Y,%%,%P,%p,%U,%V,%G,%g} + } } else { set FMT {%d,%e,%F,%H,%I,%j,%p,%R,%u,%w,%W,%%} } diff --git a/test/dbfuzz.c b/test/dbfuzz.c index ca1c6ea21..1587bacbb 100644 --- a/test/dbfuzz.c +++ b/test/dbfuzz.c @@ -368,7 +368,7 @@ static int inmemDelete( return SQLITE_IOERR_DELETE; } -/* Check for the existance of a file +/* Check for the existence of a file */ static int inmemAccess( sqlite3_vfs *pVfs, diff --git a/test/delete4.test b/test/delete4.test index 8d6a1b8c7..34151446b 100644 --- a/test/delete4.test +++ b/test/delete4.test @@ -170,7 +170,7 @@ do_execsql_test 5.0 { } {} # 2016-05-02 -# Ticket https://www.sqlite.org/src/tktview/dc6ebeda93960877 +# Ticket https://sqlite.org/src/tktview/dc6ebeda93960877 # A subquery in the WHERE clause of a one-pass DELETE can cause an # incorrect answer. # diff --git a/test/enc2.test b/test/enc2.test index 81e7bfd68..f7446a40e 100644 --- a/test/enc2.test +++ b/test/enc2.test @@ -63,7 +63,7 @@ set dbcontents { INSERT INTO t1 VALUES('one', 'I', 1); } # This proc tests that we can open and manipulate the test.db -# database, and that it is possible to retreive values in +# database, and that it is possible to retrieve values in # various text encodings. # proc run_test_script {t enc} { diff --git a/test/expr.test b/test/expr.test index 55b1dc950..71518df69 100644 --- a/test/expr.test +++ b/test/expr.test @@ -1002,7 +1002,7 @@ do_execsql_test expr-13.9 { SELECT '' <= ""; } {1} -# 2018-02-26. Ticket https://www.sqlite.org/src/tktview/36fae083b450e3af85 +# 2018-02-26. Ticket https://sqlite.org/src/tktview/36fae083b450e3af85 # do_execsql_test expr-14.1 { DROP TABLE IF EXISTS t1; diff --git a/test/extension01.test b/test/extension01.test index 97b772680..ba8a44edb 100644 --- a/test/extension01.test +++ b/test/extension01.test @@ -60,7 +60,7 @@ do_test 1.5 { } {0} do_test 1.6 { - if {$::tcl_platform(platform)=="unix"} { + if {$::tcl_platform(os) ne "Windows NT"} { file attributes ./file2.txt -permissions r--r--r-- } else { file attributes ./file2.txt -readonly 1 @@ -70,7 +70,7 @@ do_test 1.6 { } } {nil} do_test 1.7 { - if {$::tcl_platform(platform)=="unix"} { + if {$::tcl_platform(os) ne "Windows NT"} { file attributes ./file2.txt -permissions rw-r--r-- } else { file attributes ./file2.txt -readonly 0 diff --git a/test/external_reader.test b/test/external_reader.test index 5d293981c..d56aa4e26 100644 --- a/test/external_reader.test +++ b/test/external_reader.test @@ -19,7 +19,7 @@ ifcapable !wal { finish_test return } -if {$::tcl_platform(platform)!="unix"} { +if {$::tcl_platform(os) eq "Windows NT"} { finish_test return } diff --git a/test/filectrl.test b/test/filectrl.test index 460b71e25..9b1a1c758 100644 --- a/test/filectrl.test +++ b/test/filectrl.test @@ -36,16 +36,18 @@ do_test filectrl-1.5 { sqlite3 db test_control_lockproxy.db file_control_lockproxy_test db [get_pwd] } {} -do_test filectrl-1.6 { - sqlite3 db test.db - set fn [file_control_tempfilename db] - set fn -} {/etilqs_/} +ifcapable !winrt { + do_test filectrl-1.6 { + sqlite3 db test.db + set fn [file_control_tempfilename db] + set fn + } {/etilqs_/} +} db close forcedelete .test_control_lockproxy.db-conch test.proxy forcedelete test.db test2.db -if {$tcl_platform(platform)=="windows"} { +if {$tcl_platform(platform) eq "windows"} { do_test filectrl-2.1 { sqlite3 db test2.db set size [file size test2.db] diff --git a/test/fkey6.test b/test/fkey6.test index 72de926b5..415daeaf3 100644 --- a/test/fkey6.test +++ b/test/fkey6.test @@ -267,5 +267,40 @@ do_execsql_test 5.1 { COMMIT; } +#------------------------------------------------------------------------- +# +reset_db + +ifcapable fts5 { +if {[permutation]!="inmemory_journal"} { + do_execsql_test 6.1 { + PRAGMA auto_vacuum = 0; + PRAGMA writable_schema = 1; + INSERT INTO sqlite_schema + VALUES('table', 't1', 't1', 2, 'CREATE TABLE t1(x INTEGER PRIMARY KEY)'); + } + db close + sqlite3 db test.db + do_execsql_test 6.1 { + PRAGMA foreign_keys = 1; + PRAGMA writable_schema = 1; + } + do_execsql_test 6.2 { + CREATE TABLE t2( + y INTEGER PRIMARY KEY, + z INTEGER REFERENCES t1(x) DEFERRABLE INITIALLY DEFERRED + ); + } + do_execsql_test 6.3 { + BEGIN; + INSERT INTO t2 VALUES(1,0),(2,1); + CREATE VIRTUAL TABLE t3 USING fts5(a, b, content='', tokendata=1); + INSERT INTO t3 VALUES(3,3); + PRAGMA defer_foreign_keys=ON; + DELETE FROM t2; + COMMIT; + } +} +} finish_test diff --git a/test/fts3cov.test b/test/fts3cov.test index 5d8383657..d01791bbe 100644 --- a/test/fts3cov.test +++ b/test/fts3cov.test @@ -25,7 +25,7 @@ set testprefix fts3cov # When it first needs to read a block from the %_segments table, the FTS3 # module compiles an SQL statement for that purpose. The statement is # stored and reused each subsequent time a block is read. This test case -# tests the effects of an OOM error occuring while compiling the statement. +# tests the effects of an OOM error occurring while compiling the statement. # # Similarly, when FTS3 first needs to scan through a set of segment leaves # to find a set of documents that matches a term, it allocates a string @@ -277,7 +277,7 @@ do_test fts3cov-7.2 { # pending-terms table must be flushed each time a document with a docid # less than or equal to the previous docid is modified. # -# This test checks the effects of an OOM error occuring when the +# This test checks the effects of an OOM error occurring when the # pending-terms table is flushed for this reason as part of a DELETE # statement. # diff --git a/test/fts3expr2.test b/test/fts3expr2.test index c3d161730..6fb133f17 100644 --- a/test/fts3expr2.test +++ b/test/fts3expr2.test @@ -46,7 +46,7 @@ ifcapable !fts3 { # * Whether or not superflous parenthesis are included. i.e. if # "a OR b AND (c OR d)" or "a OR (b AND (c OR d))" is generated. # -# * Whether or not explict AND operators are used. i.e. if +# * Whether or not explicit AND operators are used. i.e. if # "a OR b AND c" or "a OR b c" is generated. # diff --git a/test/fts3join.test b/test/fts3join.test index cbd08b63f..9171c817b 100644 --- a/test/fts3join.test +++ b/test/fts3join.test @@ -97,11 +97,8 @@ do_eqp_test 4.2 { WHERE t4.y = ?; } { QUERY PLAN - |--MATERIALIZE rr - | `--SCAN ft4 VIRTUAL TABLE INDEX 3: |--SCAN t4 - |--BLOOM FILTER ON rr (docid=?) - `--SEARCH rr USING AUTOMATIC COVERING INDEX (docid=?) LEFT-JOIN + `--SCAN ft4 VIRTUAL TABLE INDEX 3: LEFT-JOIN } finish_test diff --git a/test/fts4langid.test b/test/fts4langid.test index 7be594bd5..84424ff3c 100644 --- a/test/fts4langid.test +++ b/test/fts4langid.test @@ -435,7 +435,7 @@ do_test 5.3.2 { for {set i 0} {$i < 20} {incr i} { execsql { INSERT INTO t6(content, lid) VALUES( - 'I (row '||$i||') belong to langauge N!', $lid + 'I (row '||$i||') belong to language N!', $lid ); } } diff --git a/test/func.test b/test/func.test index 85c9ada7e..4e5f617e7 100644 --- a/test/func.test +++ b/test/func.test @@ -1241,7 +1241,7 @@ do_test func-24.12 { WHEN 'program' THEN null ELSE t1 END) FROM tbl1 } } {,is,free,software} -# Tests to verify ticket http://www.sqlite.org/src/tktview/55746f9e65f8587c0 +# Tests to verify ticket http://sqlite.org/src/tktview/55746f9e65f8587c0 do_test func-24.13 { execsql { SELECT typeof(group_concat(x)) FROM (SELECT '' AS x); diff --git a/test/func3.test b/test/func3.test index 0221a0dfd..518bd51c7 100644 --- a/test/func3.test +++ b/test/func3.test @@ -154,7 +154,7 @@ do_test func3-5.39 { } [db eval {EXPLAIN SELECT min(1.0+'2.0',4*11)}] # Unlikely() does not preserve the affinity of X. -# ticket https://www.sqlite.org/src/tktview/0c620df60b +# ticket https://sqlite.org/src/tktview/0c620df60b # do_execsql_test func3-5.40 { SELECT likely(CAST(1 AS INT))=='1'; diff --git a/test/func9.test b/test/func9.test index 6cf9fc31e..42138ab2e 100644 --- a/test/func9.test +++ b/test/func9.test @@ -9,7 +9,7 @@ # #************************************************************************* # -# Test cases for SQL newer functions +# Test cases for some newer SQL functions # set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -36,5 +36,15 @@ do_catchsql_test func9-160 { SELECT concat_ws(','); } {1 {wrong number of arguments to function concat_ws()}} +# https://sqlite.org/forum/forumpost/4c344ca61f (2025-03-02) +do_execsql_test func9-200 { + SELECT unistr('G\u00e4ste'); +} {Gäste} +do_execsql_test func9-210 { + SELECT unistr_quote(unistr('G\u00e4ste')); +} {'Gäste'} +do_execsql_test func9-220 { + SELECT format('%#Q',unistr('G\u00e4ste')); +} {'Gäste'} finish_test diff --git a/test/fuzz.test b/test/fuzz.test index 83dc79bc7..7002054be 100644 --- a/test/fuzz.test +++ b/test/fuzz.test @@ -130,7 +130,7 @@ do_test fuzz-1.12.1 { # The following query was crashing. The later subquery (in the FROM) # clause was flattened into the parent, but the code was not repairng # the "b" reference in the other sub-query. When the query was executed, - # that "b" refered to a non-existant vdbe table-cursor. + # that "b" referred to a non-existant vdbe table-cursor. # execsql { SELECT 1 IN ( SELECT b UNION SELECT 1 ) FROM (SELECT b FROM abc); diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index 84e3f3289..09898d7b3 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -88,6 +88,11 @@ #include "sqlite3recover.h" #define ISSPACE(X) isspace((unsigned char)(X)) #define ISDIGIT(X) isdigit((unsigned char)(X)) +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# define FLEXARRAY +#else +# define FLEXARRAY 1 +#endif #ifdef __unix__ @@ -129,9 +134,12 @@ struct Blob { int id; /* Id of this Blob */ int seq; /* Sequence number */ int sz; /* Size of this Blob in bytes */ - unsigned char a[1]; /* Blob content. Extra space allocated as needed. */ + unsigned char a[FLEXARRAY]; /* Blob content. Allocated as needed. */ }; +/* Size in bytes of a Blob object sufficient to store N byte of content */ +#define SZ_BLOB(N) (offsetof(Blob,a) + (((N)+7)&~7)) + /* ** Maximum number of files in the in-memory virtual filesystem. */ @@ -383,6 +391,21 @@ static void renderDbSqlForCLI( if( nSql>0 && zSql[nSql-1]!='\n' ) fprintf(out, "\n"); } +/* +** Find the tail (the last component) of a pathname. +*/ +static const char *pathTail(const char *zPath){ + const char *zTail = zPath; + while( zPath[0] ){ + if( zPath[0]=='/' && zPath[1]!=0 ) zTail = &zPath[1]; +#ifndef __unix__ + if( zPath[0]=='\\' && zPath[1]!=0 ) zTail = &zPath[1]; +#endif + zPath++; + } + return zTail; +} + /* ** Read the complete content of a file into memory. Add a 0x00 terminator ** and return a pointer to the result. @@ -512,13 +535,15 @@ static void blobListLoadFromDb( int *pN, /* OUT: Write number of blobs loaded here */ Blob **ppList /* OUT: Write the head of the blob list here */ ){ - Blob head; + Blob *head; Blob *p; sqlite3_stmt *pStmt; int n = 0; int rc; char *z2; + unsigned char tmp[SZ_BLOB(8)]; + head = (Blob*)tmp; if( firstId>0 ){ z2 = sqlite3_mprintf("%s WHERE rowid BETWEEN %d AND %d", zSql, firstId, lastId); @@ -528,11 +553,11 @@ static void blobListLoadFromDb( rc = sqlite3_prepare_v2(db, z2, -1, &pStmt, 0); sqlite3_free(z2); if( rc ) fatalError("%s", sqlite3_errmsg(db)); - head.pNext = 0; - p = &head; + head->pNext = 0; + p = head; while( SQLITE_ROW==sqlite3_step(pStmt) ){ int sz = sqlite3_column_bytes(pStmt, 1); - Blob *pNew = safe_realloc(0, sizeof(*pNew)+sz ); + Blob *pNew = safe_realloc(0, SZ_BLOB(sz+1)); pNew->id = sqlite3_column_int(pStmt, 0); pNew->sz = sz; pNew->seq = n++; @@ -544,7 +569,7 @@ static void blobListLoadFromDb( } sqlite3_finalize(pStmt); *pN = n; - *ppList = head.pNext; + *ppList = head->pNext; } /* @@ -1594,7 +1619,7 @@ static int inmemDelete( return SQLITE_IOERR_DELETE; } -/* Check for the existance of a file +/* Check for the existence of a file */ static int inmemAccess( sqlite3_vfs *pVfs, @@ -1837,6 +1862,7 @@ static void showHelp(void){ "Read databases and SQL scripts from SOURCE-DB and execute each script against\n" "each database, checking for crashes and memory leaks.\n" "Options:\n" +" --brief Output only a summary of results at the end\n" " --cell-size-check Set the PRAGMA cell_size_check=ON\n" " --dbid M..N Use only the databases where dbid between M and N\n" " \"M..\" for M and afterwards. Just \"M\" for M only\n" @@ -1863,6 +1889,7 @@ static void showHelp(void){ " --result-trace Show the results of each SQL command\n" " --script Output CLI script instead of running tests\n" " --skip N Skip the first N test cases\n" +" --slice M N Run only the M-th out of each group of N tests\n" " --spinner Use a spinner to show progress\n" " --sqlid M..N Use only SQL where sqlid between M..N\n" " \"M..\" for M and afterwards. Just \"M\" for M only\n" @@ -1877,6 +1904,7 @@ static void showHelp(void){ int main(int argc, char **argv){ sqlite3_int64 iBegin; /* Start time of this program */ int quietFlag = 0; /* True if --quiet or -q */ + int briefFlag = 0; /* Output summary report at the end */ int verboseFlag = 0; /* True if --verbose or -v */ char *zInsSql = 0; /* SQL statement for --load-db or --load-sql */ int iFirstInsArg = 0; /* First argv[] for --load-db or --load-sql */ @@ -1924,6 +1952,8 @@ int main(int argc, char **argv){ int nV; /* How much to increase verbosity with -vvvv */ sqlite3_int64 tmStart; /* Start of each test */ int iEstTime = 0; /* LPF for the time-to-go */ + int iSliceSz = 0; /* Divide the test space into this many pieces */ + int iSliceIdx = 0; /* Only run the piece with this index */ sqlite3_config(SQLITE_CONFIG_URI,1); registerOomSimulator(); @@ -1944,6 +1974,12 @@ int main(int argc, char **argv){ if( z[0]=='-' ){ z++; if( z[0]=='-' ) z++; + if( strcmp(z,"brief")==0 ){ + briefFlag = 1; + quietFlag = 1; + verboseFlag = 1; + eVerbosity = 0; + }else if( strcmp(z,"cell-size-check")==0 ){ cellSzCkFlag = 1; }else @@ -2034,6 +2070,7 @@ int main(int argc, char **argv){ g.uRandom = atoi(argv[++i]); }else if( strcmp(z,"quiet")==0 || strcmp(z,"q")==0 ){ + briefFlag = 0; quietFlag = 1; verboseFlag = 0; eVerbosity = 0; @@ -2052,12 +2089,19 @@ int main(int argc, char **argv){ if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); nSkip = atoi(argv[++i]); }else + if( strcmp(z,"slice")==0 ){ + if( i>=argc-2 ) fatalError("missing arguments on %s", argv[i]); + iSliceIdx = integerValue(argv[++i]); + iSliceSz = integerValue(argv[++i]); + /* --slice implices --brief */ + briefFlag = 1; + quietFlag = 1; + verboseFlag = 1; + eVerbosity = 0; + }else if( strcmp(z,"spinner")==0 ){ bSpinner = 1; }else - if( strcmp(z,"timer")==0 ){ - bTimer = 1; - }else if( strcmp(z,"sqlid")==0 ){ const char *zDotDot; if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); @@ -2083,10 +2127,14 @@ int main(int argc, char **argv){ fatalError("timeout is not available on non-unix systems"); #endif }else + if( strcmp(z,"timer")==0 ){ + bTimer = 1; + }else if( strcmp(z,"vdbe-debug")==0 ){ bVdbeDebug = 1; }else if( strcmp(z,"verbose")==0 ){ + briefFlag = 0; quietFlag = 0; verboseFlag++; eVerbosity++; @@ -2153,6 +2201,12 @@ int main(int argc, char **argv){ fatalError("cannot import into more than one database"); } } + if( iSliceSz<=iSliceIdx + || iSliceSz<=0 + || iSliceIdx<0 + ){ + iSliceSz = iSliceIdx = 0; + } /* Process each source database separately */ for(iSrcDb=0; iSrcDbpNext){ tmStart = timeOfDay(); if( isDbSql(pSql->a, pSql->sz) ){ + if( iSliceSz>0 && (nTest%iSliceSz)!=iSliceIdx ){ + nTest++; + continue; + } sqlite3_snprintf(sizeof(g.zTestName), g.zTestName, "sqlid=%d",pSql->id); if( bScript ){ /* No progress output */ @@ -2500,6 +2558,10 @@ int main(int argc, char **argv){ for(pDb=g.pFirstDb; pDb; pDb=pDb->pNext){ int openFlags; const char *zVfs = "inmem"; + if( iSliceSz>0 && (nTest%iSliceSz)!=iSliceIdx ){ + nTest++; + continue; + } sqlite3_snprintf(sizeof(g.zTestName), g.zTestName, "sqlid=%d,dbid=%d", pSql->id, pDb->id); if( bScript ){ @@ -2606,7 +2668,20 @@ int main(int argc, char **argv){ } } } - if( bScript ){ + if( briefFlag ){ + sqlite3_int64 iElapse = timeOfDay() - iBegin; + if( iSliceSz>0 ){ + printf("%s %s: slice %d/%d of %d tests, %d.%03d seconds\n", + pathTail(argv[0]), pathTail(g.zDbFile), + iSliceIdx, iSliceSz, nTest, + (int)(iElapse/1000), (int)(iElapse%1000)); + }else{ + printf("%s %s: 0 errors, %d tests, %d.%03d seconds\n", + pathTail(argv[0]), pathTail(g.zDbFile), nTest, + (int)(iElapse/1000), (int)(iElapse%1000)); + } + iBegin = timeOfDay(); + }else if( bScript ){ /* No progress output */ }else if( bSpinner ){ int nTotal = g.nDb*g.nSql; @@ -2624,6 +2699,7 @@ int main(int argc, char **argv){ } /* End loop over all source databases */ + if( !quietFlag && !bScript ){ sqlite3_int64 iElapse = timeOfDay() - iBegin; if( g.nInvariant ){ diff --git a/test/fuzzdata8.db b/test/fuzzdata8.db index 469df2c681a13064074d0a6e2fac0cf5939dbbcc..bfa3e3ecd09c7ef66a63cc958568e341dbdbd4a0 100644 GIT binary patch delta 3168 zcmZvedsGzH9mn_1?6SMC%4}Fc5q1_>z(qioHxXtAZCWX)a3pP_pbK0d5j>G58hg^y zK=rnbNqhw5XCf-7HR2AlewUMm906sfi*cRioA30i#=B&iVfSxcAQ7 z&;8Et_nWEMx|pkISj?>$X-?2+gk;T=%Wq|Lf-XT5yHu#e1S5huf_Z|q1nUU)B3Mtb zfnX!S-URy)Y$A9F!9xl5C0HQXOz=|#`w?s*co@O{1P2g2oZt}z2NFDz;2?s73APd} z5^N(lgy2wu!w9w$98Pcq!I1=yYQm1%+lE(=iD3!JaA?i&L{RQH_Huj<{ppTlG>4)f z;+|tvfe=l~*pg6E%HnV`m&Kvv1s1hrjz;7A6PeB8zsM{WFA>b*8B)UHF;dLpVN%56 zKH_3=2bsZQEh%JiEt$sRJAc`B&%qSMG@2=zDTXPQX$(^wlanc)DS;`GDTyhWDTOJO zDUIoArm;-vblbg*f*iv+4fjXn)|{*ATzE|XCol56R;G_M(Y8s?=Hxh{N5w}k zii~uO^&UTY;>4`{vEJ03C5isc>E@%H;q7QLRjb)tP}MlgM%iW-)a;UywYq6q%?2+v zID+P5X*;ibmz{ncWUdOAA!eqd^*#Oti8rn0!O-@LKhL-Rk$i6$=1VJmp1L>qP zsfkA$z#T66K|`#so?7QiiUAeU8H=RboT0AHObWD`55c@H!vep1ReDQT!#3CmD@*OA z&^6f}L~XhDKO=}*CJ|V1G2KMBR7+D4)ZQ>?wQS`ksGM(qj=I)J>)GG}lR$^QEA=37 z-Rhg)40X%xGoWLvJrb7HNEaw3(gqIwjk+f}LPz`Cu{T;0x{1!Ouk)l0WH-Fxg9?}NK$r~_I#`6XJrLn`6XF32m^PockU zkaqB>f!19~8;$6gCh07P8lm!-Uk=UPEA7#vJs|wj?^iUtRocs=4`BW)d;pjnmOp^_ zw*QxOL7TLULrqY$epmn<*)Cn>xn>(6F(aUAy49b?pO89vADWVwmYA4S66cJA%wK4u zpuUzbtuD6KP`Oh&%Avhfdsdof^m1u6cU?(f?o`G>S4BWJ?A+*ozxuNDP`8^M;Q%da zNgaZy*)8>O=pd9AK7EbexFT72)B>5;q{C40mUR})|6WR^Mc+yrIdll>z77h6uHe)h zb-0jz@}0C(?_H$TT$-EokX=yzjnPSW{UGJ?y8Ud6BT&Qz$I!U@*29Q)-X%X@fzay;v4zHO_` zT1@8{6QJgx@(y(wT~_f%SzAlkK64nWOTTM<&t>4NfV_Ny_tg`@WM+RezhrdZ zg7ZcB_?`+2dBk09wXIgM{|(NE*3)pfNt~^|_K$l_z6M9GWMo4{q+9{Y7Wp*G#^TpEb%(4s}D`SK>mb9U+$i%h`Nbep<9pWf;ra z=DjknMLmON#;pOF@k9Rv{;|LR{~6r>+n)a4dF;P|{`#!Ek4JZ(_}|p7r1op0={9(ksx__jHAo+vlF0BzFxhEicW_(p7e(M8H-e93 zB`t&l>dDAdt2vaTvzID02t|NMW}5+&n_{r~%u!G|MvR2qI60rbwn|yTBer&je2HFcRYYE05k>PF1AMSEi|b`ImOW!lYE9(y^T`m6KSkV z-j3+5HYJ}!>|elb351{7S`g(vRkV7qdCcdvc3>Z(bu@09asq*^(dbpxV5wA}5%WJ% C35gQ` delta 3074 zcmZvedsI}%9mjX>-DP)S(YvyOBJ5pYSzHiY5D=rxy_lRf79()voVEs(g;A-*Q>l;k zw6?;Dlbkg1N#f5aD)>kuih>BEMtr0u(P|V;Ont1Cnny_bdM!>k^iPRh_deFm)MDlmt@}LWzZvP)Y_+GLRC95-TNxC<&v)M#*4G!YPTMWC$fgDT$r(F$&3=qzp{=x?|UrRCg;bf!)>@GhOf?LX;sZoi;q+@7JO+#aP~ zZV%8BZuih>+-{>&x!pjExLxyV!#y`wB3BaE2(DzV6t0n690B9HnOvi} zvbe@@J;ODYYaDC1mt8c`v_&WU1^IMaYECZz)5;O_*LvONqMGIz4tVrDmLa4Jf6sgq zupHh-aHifcMX%cs#D|Bn!g9P#G`!8xY9Z&%cmp7U0GR1aaQK2ENE(v zkAmF0Lo5)rJU)_5nuA+Kv>tr%I1HLn1{#@t9#&1Lh)r9FZwscndMkZVuloS3>#}X| zyG3}3VJ+XF9#&L1E1)ya8O0p=&Q}m5uEZ3UUmRy)@2tU75Y*i;>Ggc(MyQ_S{1q#y z#q0UY`4)){SdY6Al-%ka-vsr`oYSCVtTO?Y*5M0`yoNUjXfyLoa?1#$ho%F}w*#otTNV0T+oN9g%qB-FUl*cCz{_W0DXXxfg#XpeCq37B-RP@5e1hvIG= z)B;%3h3at)6a&|<`RC_o}>%gbQkhan3 zW75+z$~>ta$azkm2wUsKidCidTBe-EM+C%}{v4id4w|Od-7U!g>sfU?bXG;=LgU8p z`>QVFM}}s8gni80o)wCi^(yWb(ECtX{LDA(#x-mc(SFFefe%8}68j98a}#GV@3(lP zfZCz{a#SRAMrTdbUKg{EyKtj1xJ0kJG%Mo~f1vWJIhF0agY!khUcSX4@Cwn%%yZv< z5V5WYI7~o?!FydSXTKe;tPqW*nPd2@55NnN(NNN$>|`B!Wt52iR&zQp4pP$OIQD`^ zX%UT?X_P2)Ms%?%5;ONed)u`SS(Ft~Vm#iSB z8kBZH%h^wAa^hT1aN0z45^ipl{{;L*&qM7Y^=(!Xs%#R_DSxJpK1`?mnN+3Q6jW-^ zRj(e$4p%B;1vBA09pm~8nq4N(Q>TUYFr4j!`8jaR^XyWg;u6t60uj&kK`hn~&-)`@ z@JA$?+b;SemMgbRsEqBeRGt+KpK`JalC7gI#)T~Qe~r$!B^QGZ)x2BhSW#)Q*f|Vo~gzCqaH8+2M%A!&V{NZ zr3%yr<&>6>r9C6xfK;P)L2{bx)N=A#&&&TpOo$Q#i3ios!IQ24)Vbv8>~^DK5zx0_ z&9#k$j)!6}%rUcYqr`_MfIq$8`6#qDN2YLR<4xp1i_O;YNoT ze&AQSp+6`o$REn4@~SrmKuIP)sP7bB_2&Meg8iXF0--GaP+CQ?_e0fZg&>LNpfs>w zHmMJrHIQvkAX`}fY&L(k!B5#(UcTnBaDTWFD)-m`PBru=UE8bjt{vH@YfHoYyLP$C zyLME+U0aIo?b>rx-nH%iJn~bX-T*rMaTcmwrl46`NvpjLGK7py=X683;#hN6^oKoH ziX=e3OH9zb=>*iBoud_VtiWb2RcjH72Vasbueu;lg0I@uF0sR){a^%$CJ z)XAbN)eQy5O|iX~4Rlx?CQxt6(b}R3P(4ykfc$i&kS$uRE*8;nC}>hHv5QAkS=45@ zS;1~)HewaW)MZ*%kP>^3A$@8{V7Z&s@n%%PmNu%Df+3cVCbQ=IcKjpzp&iN&?VA+H z85m-F+(kQE)i?o-WX*b|t%r7k_WyeGwV3DtLZ|j=D%3x~^MJ#(vy8*)Ndcv?6iGQF zYVJ+cW{hGfMA?Se&SPq!fYQP0vqi#>c=0x80} { if {$i<=0x9F || $tcl_version>=9.0} continue - if {$tcl_platform(platform)=="windows"} continue + if {$tcl_platform(platform) eq "windows"} continue } - if {$i>=0xE0 && $tcl_platform(os)=="OpenBSD"} continue - if {$i>=0xE0 && $i<=0xEF && $tcl_platform(os)=="Linux"} continue + if {$i>=0xE0 && $tcl_platform(os) eq "OpenBSD"} continue + if {$i>=0xE0 && $i<=0xEF && $tcl_platform(os) eq "Linux"} continue set hex [format %02X $i] set char [subst \\x$hex]; set oldChar $char set escapes [list] - if {$tcl_platform(platform)=="windows"} { + if {$tcl_platform(platform) eq "windows"} { # # NOTE: On Windows, we need to escape all the whitespace characters, # the alarm (\a) character, and those with special meaning to @@ -1223,7 +1230,7 @@ do_test shell1-8.1 { FROM pow2, c WHERE pow2.x=ieee754_exponent(c.n); } } {0 47.49000000000000198951966012828052043914794921875} -do_test shell1-8.2 { +do_test_with_ansi_output shell1-8.2 { catchcmd :memory: { .mode box SELECT ieee754(47.49) AS x; @@ -1233,7 +1240,7 @@ SELECT ieee754(47.49) AS x; ├───────────────────────────────┤ │ ieee754(6683623321994527,-47) │ └───────────────────────────────┘}} -do_test shell1-8.3 { +do_test_with_ansi_output shell1-8.3 { catchcmd ":memory: --box" { select ieee754(6683623321994527,-47) as x; } @@ -1249,7 +1256,7 @@ do_test shell1-8.4 { +------------------+-----+ | 6683623321994527 | -47 | +------------------+-----+}} -do_test shell1-8.5 { +do_test_with_ansi_output shell1-8.5 { catchcmd ":memory: --box" { create table t(a text, b int); insert into t values ('too long for one line', 1), ('shorter', NULL); diff --git a/test/shell3.test b/test/shell3.test index ef3ea784f..6bb49f5c3 100644 --- a/test/shell3.test +++ b/test/shell3.test @@ -36,7 +36,7 @@ sqlite3 db test.db # different. This causes problems for the tests below. To avoid # issues, these tests are disabled for windows. # -if {$::tcl_platform(platform)=="windows"} { +if {$::tcl_platform(platform) eq "windows"} { finish_test return } diff --git a/test/shell4.test b/test/shell4.test index 4b7e9b8ee..4275911ef 100644 --- a/test/shell4.test +++ b/test/shell4.test @@ -140,7 +140,7 @@ do_test shell4-3.1 { close $fd exec $::CLI_ONLY :memory: --interactive ".read t1.txt" } {squirrel} -do_test shell4-3.2 { +do_test_with_ansi_output shell4-3.2 { set fd [open t1.txt wb] puts $fd "SELECT 'pound: \302\243';" close $fd diff --git a/test/shell5.test b/test/shell5.test index 8eb905974..70a2298bc 100644 --- a/test/shell5.test +++ b/test/shell5.test @@ -553,7 +553,7 @@ Columns renamed during .import shell5.csv due to duplicates: # Tests for preserving utf-8 that is not also ASCII. # -do_test shell5-6.1 { +do_test_with_ansi_output shell5-6.1 { set out [open shell5.csv w] fconfigure $out -translation lf puts $out {あい,うえお} @@ -566,7 +566,7 @@ SELECT * FROM t1;} } {0 { あい = 1 うえお = 2}} -do_test shell5-6.2 { +do_test_with_ansi_output shell5-6.2 { set out [open shell5.csv w] fconfigure $out -translation lf puts $out {1,2} diff --git a/test/shell8.test b/test/shell8.test index ca37598e9..e55539636 100644 --- a/test/shell8.test +++ b/test/shell8.test @@ -55,7 +55,7 @@ proc dir_to_list {dirname {n -1}} { set res [list] foreach f [glob -nocomplain $dirname/*] { set mtime [file mtime $f] - if {$::tcl_platform(platform)!="windows"} { + if {$::tcl_platform(platform) ne "windows"} { set perm [file attributes $f -perm] } else { set perm 0 diff --git a/test/shellA.test b/test/shellA.test new file mode 100644 index 000000000..f3959d428 --- /dev/null +++ b/test/shellA.test @@ -0,0 +1,257 @@ +# 2025-02-24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# TESTRUNNER: shell +# +# Test cases for the command-line shell - focusing on .mode and +# especially control-character escaping and the --escape option. +# +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set CLI [test_cli_invocation] + +do_execsql_test shellA-1.0 { + CREATE TABLE t1(a INT, x TEXT); + INSERT INTO t1 VALUES + (1, 'line with '' single quote'), + (2, concat(char(0x1b),'[31mVT-100 codes',char(0x1b),'[0m')), + (3, NULL), + (4, 1234), + (5, 568.25), + (6, unistr('new\u000aline')), + (7, unistr('carriage\u000dreturn')), + (8, 'last line'); +} {} + +# Initial verification that the database created correctly +# and that our calls to the CLI are working. +# +do_test_with_ansi_output shellA-1.2 { + exec {*}$CLI test.db {.mode box --escape symbol} {SELECT * FROM t1;} +} { +┌───┬──────────────────────────┐ +│ a │ x │ +├───┼──────────────────────────┤ +│ 1 │ line with ' single quote │ +├───┼──────────────────────────┤ +│ 2 │ ␛[31mVT-100 codes␛[0m │ +├───┼──────────────────────────┤ +│ 3 │ │ +├───┼──────────────────────────┤ +│ 4 │ 1234 │ +├───┼──────────────────────────┤ +│ 5 │ 568.25 │ +├───┼──────────────────────────┤ +│ 6 │ new │ +│ │ line │ +├───┼──────────────────────────┤ +│ 7 │ carriage␍return │ +├───┼──────────────────────────┤ +│ 8 │ last line │ +└───┴──────────────────────────┘ +} + +# ".mode list" +# +do_test shellA-1.3 { + exec {*}$CLI test.db {SELECT x FROM t1 WHERE a=2;} +} { +^[[31mVT-100 codes^[[0m +} +do_test_with_ansi_output shellA-1.4 { + exec {*}$CLI test.db --escape symbol {SELECT x FROM t1 WHERE a=2;} +} { +␛[31mVT-100 codes␛[0m +} +do_test shellA-1.5 { + exec {*}$CLI test.db --escape ascii {SELECT x FROM t1 WHERE a=2;} +} { +^[[31mVT-100 codes^[[0m +} +do_test_with_ansi_output shellA-1.6 { + exec {*}$CLI test.db {.mode list --escape symbol} {SELECT x FROM t1 WHERE a=2;} +} { +␛[31mVT-100 codes␛[0m +} +do_test shellA-1.7 { + exec {*}$CLI test.db {.mode list --escape ascii} {SELECT x FROM t1 WHERE a=2;} +} { +^[[31mVT-100 codes^[[0m +} +do_test shellA-1.8 { + file delete -force out.txt + exec {*}$CLI test.db {.mode list --escape off} {SELECT x FROM t1 WHERE a=7;} \ + >out.txt + set fd [open out.txt rb] + set res [read $fd] + close $fd + string trim $res +} "carriage\rreturn" +do_test shellA-1.9 { + set rc [catch { + exec {*}$CLI test.db {.mode test --escape xyz} + } msg] + lappend rc $msg +} {1 {unknown control character escape mode "xyz" - choices: ascii symbol off}} +do_test shellA-1.10 { + set rc [catch { + exec {*}$CLI --escape abc test.db .q + } msg] + lappend rc $msg +} {1 {unknown control character escape mode "abc" - choices: ascii symbol off}} + +# ".mode quote" +# +do_test shellA-2.1 { + exec {*}$CLI test.db --quote {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} +} { +1,'line with '' single quote' +2,unistr('\u001b[31mVT-100 codes\u001b[0m') +6,'new +line' +7,unistr('carriage\u000dreturn') +8,'last line' +} +do_test shellA-2.2 { + exec {*}$CLI test.db --quote {.mode} +} {current output mode: quote --escape ascii} +do_test shellA-2.3 { + exec {*}$CLI test.db --quote --escape SYMBOL {.mode} +} {current output mode: quote --escape symbol} +do_test shellA-2.4 { + exec {*}$CLI test.db --quote --escape OFF {.mode} +} {current output mode: quote --escape off} + + +# ".mode line" +# +do_test_with_ansi_output shellA-3.1 { + exec {*}$CLI test.db --line --escape symbol \ + {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} +} { + a = 1 + x = line with ' single quote + + a = 2 + x = ␛[31mVT-100 codes␛[0m + + a = 6 + x = new +line + + a = 7 + x = carriage␍return + + a = 8 + x = last line +} +do_test shellA-3.2 { + exec {*}$CLI test.db --line --escape ascii \ + {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} +} { + a = 1 + x = line with ' single quote + + a = 2 + x = ^[[31mVT-100 codes^[[0m + + a = 6 + x = new +line + + a = 7 + x = carriage^Mreturn + + a = 8 + x = last line +} + +# ".mode box" +# +do_test_with_ansi_output shellA-4.1 { + exec {*}$CLI test.db --box --escape ascii \ + {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} +} { +┌───┬──────────────────────────┐ +│ a │ x │ +├───┼──────────────────────────┤ +│ 1 │ line with ' single quote │ +├───┼──────────────────────────┤ +│ 2 │ ^[[31mVT-100 codes^[[0m │ +├───┼──────────────────────────┤ +│ 6 │ new │ +│ │ line │ +├───┼──────────────────────────┤ +│ 7 │ carriage^Mreturn │ +├───┼──────────────────────────┤ +│ 8 │ last line │ +└───┴──────────────────────────┘ +} +do_test_with_ansi_output shellA-4.2 { + exec {*}$CLI test.db {.mode qbox} {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} +} { +┌───┬───────────────────────────────────────────┐ +│ a │ x │ +├───┼───────────────────────────────────────────┤ +│ 1 │ 'line with '' single quote' │ +├───┼───────────────────────────────────────────┤ +│ 2 │ unistr('\u001b[31mVT-100 codes\u001b[0m') │ +├───┼───────────────────────────────────────────┤ +│ 6 │ 'new │ +│ │ line' │ +├───┼───────────────────────────────────────────┤ +│ 7 │ unistr('carriage\u000dreturn') │ +├───┼───────────────────────────────────────────┤ +│ 8 │ 'last line' │ +└───┴───────────────────────────────────────────┘ +} + +# ".mode insert" +# +do_test shellA-5.1 { + exec {*}$CLI test.db {.mode insert t1 --escape ascii} \ + {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} +} { +INSERT INTO t1 VALUES(1,'line with '' single quote'); +INSERT INTO t1 VALUES(2,unistr('\u001b[31mVT-100 codes\u001b[0m')); +INSERT INTO t1 VALUES(6,unistr('new\u000aline')); +INSERT INTO t1 VALUES(7,unistr('carriage\u000dreturn')); +INSERT INTO t1 VALUES(8,'last line'); +} +do_test shellA-5.2 { + exec {*}$CLI test.db {.mode insert t1 --escape symbol} \ + {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} +} { +INSERT INTO t1 VALUES(1,'line with '' single quote'); +INSERT INTO t1 VALUES(2,unistr('\u001b[31mVT-100 codes\u001b[0m')); +INSERT INTO t1 VALUES(6,unistr('new\u000aline')); +INSERT INTO t1 VALUES(7,unistr('carriage\u000dreturn')); +INSERT INTO t1 VALUES(8,'last line'); +} +do_test shellA-5.3 { + file delete -force out.txt + exec {*}$CLI test.db {.mode insert t1 --escape off} \ + {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} >out.txt + set fd [open out.txt rb] + set res [read $fd] + close $fd + string trim [string map [list \r\n \n] $res] +} " +INSERT INTO t1 VALUES(1,'line with '' single quote'); +INSERT INTO t1 VALUES(2,'\033\13331mVT-100 codes\033\1330m'); +INSERT INTO t1 VALUES(6,'new +line'); +INSERT INTO t1 VALUES(7,'carriage\rreturn'); +INSERT INTO t1 VALUES(8,'last line'); +" + +finish_test diff --git a/test/shmlock.test b/test/shmlock.test index 89b29fd7a..fce0cf8f5 100644 --- a/test/shmlock.test +++ b/test/shmlock.test @@ -114,7 +114,7 @@ sqlite3 db0 test.db sqlite3 db1 test.db do_test 3.1 { execsql { SELECT * FROM t1 } db0 } {1 2} do_test 3.2 { execsql { SELECT * FROM t1 } db1 } {1 2} -if {$tcl_platform(platform)=="windows"} { +if {$tcl_platform(os) eq "Windows NT"} { set isWindows 1 } else { set isWindows 0 diff --git a/test/skipscan6.test b/test/skipscan6.test index 4f592bc0e..a97f440ee 100644 --- a/test/skipscan6.test +++ b/test/skipscan6.test @@ -167,10 +167,10 @@ do_execsql_test 3.0 { INSERT INTO t2 SELECT * FROM t3; ANALYZE; - SELECT * FROM sqlite_stat1; + SELECT * FROM sqlite_stat1 ORDER BY +idx; } { - t2 t2_ba {100 20 1 1} t2 t2_a {100 1} + t2 t2_ba {100 20 1 1} t3 t3_a {100 1} t3 t3_ba {100 20 1 1} } diff --git a/test/snapshot3.test b/test/snapshot3.test index 470d463a6..6d57b1d0c 100644 --- a/test/snapshot3.test +++ b/test/snapshot3.test @@ -96,6 +96,9 @@ do_test 1.8 { list [catch { sqlite3_snapshot_open_blob db3 main $snap } msg] $msg } {1 SQLITE_ERROR_SNAPSHOT} +db3 close +db2 close + #------------------------------------------------------------------------- reset_db do_execsql_test 2.0 { diff --git a/test/speedtest.tcl b/test/speedtest.tcl index 93b407c94..7cd3b5fa1 100755 --- a/test/speedtest.tcl +++ b/test/speedtest.tcl @@ -25,8 +25,10 @@ Other options include: --help Show this help screen. --lean "Lean" mode. --lookaside N SZ Lookahead uses N slots of SZ bytes each. + --osmalloc Use the OS native malloc() instead of MEMSYS5 --pagesize N Use N as the page size. --quiet | -q "Quite". Put results in file but don't pop up editor + --size N Change the test size. 100 means 100%. Default: 5. --testset TEST Specify the specific testset to use. The default is "mix1". Other options include: "main", "json", "cte", "orm", "fp", "rtree". @@ -39,7 +41,8 @@ set cc gcc set testset mix1 set dryrun 0 set quiet 0 -set speedtestflags {--shrink-memory --reprepare --stats --heap 40000000 64} +set osmalloc 0 +set speedtestflags {--shrink-memory --reprepare --stats} lappend speedtestflags --journal wal --size 5 for {set i 0} {$i<[llength $argv]} {incr i} { @@ -78,11 +81,23 @@ for {set i 0} {$i<[llength $argv]} {incr i} { incr i set testset [lindex $argv $i] } + -size - + --size { + incr i + set newsize [lindex $argv $i] + if {$newsize<1} {set newsize 1} + set speedtestflags \ + [regsub {.-size \d+} $speedtestflags "-size $newsize"] + } -n - -dryrun - --dryrun { set dryrun 1 } + -osmalloc - + --osmalloc { + set osmalloc 1 + } -? - -help - --help { @@ -130,10 +145,13 @@ for {set i 0} {$i<[llength $argv]} {incr i} { if {[lsearch -glob $cflags -O*]<0} { lappend cflags -Os } -if {[lsearch -glob $cflags -DSQLITE_ENABLE_MEMSYS*]<0} { +if {!$osmalloc} { + append speedtestflags { --heap 40000000 64} +} +if {!$osmalloc && [lsearch -glob $cflags {-DSQLITE_ENABLE_MEMSYS*}]<0} { lappend cflags -DSQLITE_ENABLE_MEMSYS5 } -if {[lsearch -glob $cflags -DSQLITE_ENABLE_RTREE*]<0} { +if {[lsearch -glob $cflags {-DSQLITE_ENABLE_RTREE*}]<0} { lappend cflags -DSQLITE_ENABLE_RTREE } if {$srcfile==""} { @@ -174,9 +192,14 @@ if {!$dryrun} { } lappend speedtestflags --testset $testset set stcmd [list valgrind --tool=cachegrind ./speedtest1 {*}$speedtestflags] +lappend stcmd speedtest1.db lappend stcmd >valgrind-out.txt 2>valgrind-err.txt puts $stcmd if {!$dryrun} { + foreach file {speedtest1.db speedtest1.db-journal speedtest1.db-wal + speedtest1.db-shm} { + if {[file exists $file]} {file delete $file} + } exec {*}$stcmd } diff --git a/test/speedtest1.c b/test/speedtest1.c index 9d8ddc454..10cd30f1c 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -35,6 +35,7 @@ static const char zHelp[] = " --exclusive Enable locking_mode=EXCLUSIVE\n" " --explain Like --sqlonly but with added EXPLAIN keywords\n" " --fullfsync Enable fullfsync=TRUE\n" + " --hard-heap-limit N The hard limit on the maximum heap size\n" " --heap SZ MIN Memory allocator uses SZ bytes & min allocation MIN\n" " --incrvacuum Enable incremenatal vacuum mode\n" " --journal M Set the journal_mode to M\n" @@ -60,14 +61,14 @@ static const char zHelp[] = " --sqlonly No-op. Only show the SQL that would have been run.\n" " --shrink-memory Invoke sqlite3_db_release_memory() frequently.\n" " --size N Relative test size. Default=100\n" + " --soft-heap-limit N The soft limit on the maximum heap size\n" " --strict Use STRICT table where appropriate\n" " --stats Show statistics at the end\n" " --stmtscanstatus Activate SQLITE_DBCONFIG_STMT_SCANSTATUS\n" " --temp N N from 0 to 9. 0: no temp table. 9: all temp tables\n" - " --testset T Run test-set T (main, cte, rtree, orm, fp, json," - " debug)\n" - " Can be a comma-separated list of values, with /SCALE\n" - " suffixes or macro \"mix1\"\n" + " --testset T Run test-set T (main, cte, rtree, orm, fp, json,\n" + " star, app, debug). Can be a comma-separated list\n" + " of values, with /SCALE suffixes or macro \"mix1\"\n" " --trace Turn on SQL tracing\n" " --threads N Use up to N threads for sorting\n" " --utf16be Set text encoding to UTF-16BE\n" @@ -113,6 +114,8 @@ struct HashContext { /* All global state is held in this structure */ static struct Global { sqlite3 *db; /* The open database connection */ + const char *zDbName; /* Name of the database file */ + const char *zVfs; /* --vfs NAME */ sqlite3_stmt *pStmt; /* Current SQL statement */ sqlite3_int64 iStart; /* Start-time for the current test */ sqlite3_int64 iTotal; /* Total time */ @@ -541,6 +544,7 @@ char *speedtest1_once(const char *zFormat, ...){ char *zSql; sqlite3_stmt *pStmt; char *zResult = 0; + int rc; va_start(ap, zFormat); zSql = sqlite3_vmprintf(zFormat, ap); va_end(ap); @@ -560,6 +564,12 @@ char *speedtest1_once(const char *zFormat, ...){ const char *z = (const char*)sqlite3_column_text(pStmt, 0); if( z ) zResult = sqlite3_mprintf("%s", z); } + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "%s\nError code %d: %s\n", + sqlite3_sql(pStmt), rc, sqlite3_errmsg(g.db)); + exit(1); + } sqlite3_finalize(pStmt); } sqlite3_free(zSql); @@ -589,7 +599,7 @@ void speedtest1_prepare(const char *zFormat, ...){ /* Run an SQL statement previously prepared */ void speedtest1_run(void){ - int i, n, len; + int i, n, len, rc; if( g.bSqlOnly ) return; assert( g.pStmt ); g.nResult = 0; @@ -648,12 +658,22 @@ void speedtest1_run(void){ if( g.bReprepare ){ sqlite3_stmt *pNew; sqlite3_prepare_v2(g.db, sqlite3_sql(g.pStmt), -1, &pNew, 0); - sqlite3_finalize(g.pStmt); + rc = sqlite3_finalize(g.pStmt); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "%s\nError code %d: %s\n", + sqlite3_sql(pNew), rc, sqlite3_errmsg(g.db)); + exit(1); + } g.pStmt = pNew; }else #endif { - sqlite3_reset(g.pStmt); + rc = sqlite3_reset(g.pStmt); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "%s\nError code %d: %s\n", + sqlite3_sql(g.pStmt), rc, sqlite3_errmsg(g.db)); + exit(1); + } } speedtest1_shrink_memory(); } @@ -1458,6 +1478,561 @@ void testset_fp(void){ speedtest1_end_test(); } +/* +** A testset for star-schema queries. +*/ +void testset_star(void){ + int n; + int i; + n = g.szTest*50; + speedtest1_begin_test(100, "Create a fact table with %d entries", n); + speedtest1_exec( + "CREATE TABLE facttab(" + " attr01 INT," + " attr02 INT," + " attr03 INT," + " data01 TEXT," + " attr04 INT," + " attr05 INT," + " attr06 INT," + " attr07 INT," + " attr08 INT," + " factid INTEGER PRIMARY KEY," + " data02 TEXT" + ");" + ); + speedtest1_exec( + "WITH RECURSIVE counter(nnn) AS" + "(VALUES(1) UNION ALL SELECT nnn+1 FROM counter WHERE nnn<%d)" + "INSERT INTO facttab(attr01,attr02,attr03,attr04,attr05," + "attr06,attr07,attr08,data01,data02)" + "SELECT random()%%12, random()%%13, random()%%14, random()%%15," + "random()%%16, random()%%17, random()%%18, random()%%19," + "concat('data-',nnn), format('%%x',random()) FROM counter;", + n + ); + speedtest1_end_test(); + + speedtest1_begin_test(110, "Create indexes on all attributes columns"); + for(i=1; i<=8; i++){ + speedtest1_exec( + "CREATE INDEX fact_attr%02d ON facttab(attr%02d)", i, i + ); + } + speedtest1_end_test(); + + speedtest1_begin_test(120, "Create dimension tables"); + for(i=1; i<=8; i++){ + speedtest1_exec( + "CREATE TABLE dimension%02d(" + "beta%02d INT, " + "content%02d TEXT, " + "rate%02d REAL)", + i, i, i, i + ); + speedtest1_exec( + "WITH RECURSIVE ctr(nn) AS" + " (VALUES(1) UNION ALL SELECT nn+1 FROM ctr WHERE nn<%d)" + " INSERT INTO dimension%02d" + " SELECT nn%%(%d), concat('content-%02d-',nn)," + " (random()%%10000)*0.125 FROM ctr;", + 4*(i+1), i, 2*(i+1), i + ); + if( i&2 ){ + speedtest1_exec( + "CREATE INDEX dim%02d ON dimension%02d(beta%02d);", + i, i, i + ); + }else{ + speedtest1_exec( + "CREATE INDEX dim%02d ON dimension%02d(beta%02d,content%02d);", + i, i, i, i + ); + } + } + speedtest1_end_test(); + + speedtest1_begin_test(130, "Star query over the entire fact table"); + speedtest1_exec( + "SELECT count(*), max(content04), min(content03), sum(rate04), avg(rate05)" + " FROM facttab, dimension01, dimension02, dimension03, dimension04," + " dimension05, dimension06, dimension07, dimension08" + " WHERE attr01=beta01" + " AND attr02=beta02" + " AND attr03=beta03" + " AND attr04=beta04" + " AND attr05=beta05" + " AND attr06=beta06" + " AND attr07=beta07" + " AND attr08=beta08" + ";" + ); + speedtest1_end_test(); + + speedtest1_begin_test(130, "Star query with LEFT JOINs"); + speedtest1_exec( + "SELECT count(*), max(content04), min(content03), sum(rate04), avg(rate05)" + " FROM facttab LEFT JOIN dimension01 ON attr01=beta01" + " LEFT JOIN dimension02 ON attr02=beta02" + " JOIN dimension03 ON attr03=beta03" + " JOIN dimension04 ON attr04=beta04" + " JOIN dimension05 ON attr05=beta05" + " LEFT JOIN dimension06 ON attr06=beta06" + " JOIN dimension07 ON attr07=beta07" + " JOIN dimension08 ON attr08=beta08" + " WHERE facttab.data01 LIKE 'data-9%%'" + ";" + ); + speedtest1_end_test(); +} + +/* +** Tests that simulate an application opening and closing an SQLite database +** frequently. Fossil is used as the model. The focus here is on rapidly +** parsing the database schema and rapidly generating prepared statements, +** in other words, rapid start-up of Fossil-like applications. +** +** The same database has no data, so the performance of sqlite3_step() is +** not significant to this testset. +*/ +static void testset_app(void){ + int i, n; + speedtest1_begin_test(100, "Generate a Fossil-like database schema"); + speedtest1_exec( + "BEGIN;" + "CREATE TABLE blob(\n" + " rid INTEGER PRIMARY KEY,\n" + " rcvid INTEGER,\n" + " size INTEGER,\n" + " uuid TEXT UNIQUE NOT NULL,\n" + " content BLOB,\n" + " CHECK( length(uuid)>=40 AND rid>0 )\n" + ");\n" + "CREATE TABLE delta(\n" + " rid INTEGER PRIMARY KEY,\n" + " srcid INTEGER NOT NULL REFERENCES blob\n" + ");\n" + "CREATE TABLE rcvfrom(\n" + " rcvid INTEGER PRIMARY KEY,\n" + " uid INTEGER REFERENCES user,\n" + " mtime DATETIME,\n" + " nonce TEXT UNIQUE,\n" + " ipaddr TEXT\n" + ");\n" + "CREATE TABLE private(rid INTEGER PRIMARY KEY);\n" + "CREATE TABLE accesslog(\n" + " uname TEXT,\n" + " ipaddr TEXT,\n" + " success BOOLEAN,\n" + " mtime TIMESTAMP\n" + ");\n" + "CREATE TABLE user(\n" + " uid INTEGER PRIMARY KEY,\n" + " login TEXT UNIQUE,\n" + " pw TEXT,\n" + " cap TEXT,\n" + " cookie TEXT,\n" + " ipaddr TEXT,\n" + " cexpire DATETIME,\n" + " info TEXT,\n" + " mtime DATE,\n" + " photo BLOB\n" + ", jx TEXT DEFAULT '{}');\n" + "CREATE TABLE reportfmt(\n" + " rn INTEGER PRIMARY KEY,\n" + " owner TEXT,\n" + " title TEXT UNIQUE,\n" + " mtime INTEGER,\n" + " cols TEXT,\n" + " sqlcode TEXT\n" + ", jx TEXT DEFAULT '{}');\n" + "CREATE TABLE config(\n" + " name TEXT PRIMARY KEY NOT NULL,\n" + " value CLOB, mtime INTEGER,\n" + " CHECK( typeof(name)='text' AND length(name)>=1 )\n" + ") WITHOUT ROWID;\n" + "CREATE TABLE shun(uuid PRIMARY KEY, mtime INTEGER, scom TEXT)\n" + " WITHOUT ROWID;\n" + "CREATE TABLE concealed(\n" + " hash TEXT PRIMARY KEY,\n" + " content TEXT\n" + ", mtime INTEGER) WITHOUT ROWID;\n" + "CREATE TABLE admin_log(\n" + " id INTEGER PRIMARY KEY,\n" + " time INTEGER, -- Seconds since 1970\n" + " page TEXT, -- path of page\n" + " who TEXT, -- User who made the change\n" + " what TEXT -- What changed\n" + ");\n" + "CREATE TABLE unversioned(\n" + " name TEXT PRIMARY KEY,\n" + " rcvid INTEGER,\n" + " mtime DATETIME,\n" + " hash TEXT,\n" + " sz INTEGER,\n" + " encoding INT,\n" + " content BLOB\n" + ") WITHOUT ROWID;\n" + "CREATE TABLE subscriber(\n" + " subscriberId INTEGER PRIMARY KEY,\n" + " subscriberCode BLOB DEFAULT (randomblob(32)) UNIQUE,\n" + " semail TEXT UNIQUE COLLATE nocase,\n" + " suname TEXT,\n" + " sverified BOOLEAN DEFAULT true,\n" + " sdonotcall BOOLEAN,\n" + " sdigest BOOLEAN,\n" + " ssub TEXT,\n" + " sctime INTDATE,\n" + " mtime INTDATE,\n" + " smip TEXT\n" + ", lastContact INT);\n" + "CREATE TABLE pending_alert(\n" + " eventid TEXT PRIMARY KEY,\n" + " sentSep BOOLEAN DEFAULT false,\n" + " sentDigest BOOLEAN DEFAULT false\n" + ", sentMod BOOLEAN DEFAULT false) WITHOUT ROWID;\n" + "CREATE TABLE filename(\n" + " fnid INTEGER PRIMARY KEY,\n" + " name TEXT UNIQUE\n" + ") STRICT;\n" + "CREATE TABLE mlink(\n" + " mid INTEGER,\n" + " fid INTEGER,\n" + " pmid INTEGER,\n" + " pid INTEGER,\n" + " fnid INTEGER REFERENCES filename,\n" + " pfnid INTEGER,\n" + " mperm INTEGER,\n" + " isaux INT DEFAULT 0\n" + ") STRICT;\n" + "CREATE TABLE plink(\n" + " pid INTEGER REFERENCES blob,\n" + " cid INTEGER REFERENCES blob,\n" + " isprim INT,\n" + " mtime REAL,\n" + " baseid INTEGER REFERENCES blob,\n" + " UNIQUE(pid, cid)\n" + ") STRICT;\n" + "CREATE TABLE leaf(rid INTEGER PRIMARY KEY);\n" + "CREATE TABLE event(\n" + " type TEXT,\n" + " mtime REAL,\n" + " objid INTEGER PRIMARY KEY,\n" + " tagid INTEGER,\n" + " uid INTEGER REFERENCES user,\n" + " bgcolor TEXT,\n" + " euser TEXT,\n" + " user TEXT,\n" + " ecomment TEXT,\n" + " comment TEXT,\n" + " brief TEXT,\n" + " omtime REAL\n" + ") STRICT;\n" + "CREATE TABLE phantom(\n" + " rid INTEGER PRIMARY KEY\n" + ");\n" + "CREATE TABLE orphan(\n" + " rid INTEGER PRIMARY KEY,\n" + " baseline INTEGER\n" + ") STRICT;\n" + "CREATE TABLE unclustered(\n" + " rid INTEGER PRIMARY KEY\n" + ");\n" + "CREATE TABLE unsent(\n" + " rid INTEGER PRIMARY KEY\n" + ");\n" + "CREATE TABLE tag(\n" + " tagid INTEGER PRIMARY KEY,\n" + " tagname TEXT UNIQUE\n" + ") STRICT;\n" + "CREATE TABLE tagxref(\n" + " tagid INTEGER REFERENCES tag,\n" + " tagtype INTEGER,\n" + " srcid INTEGER REFERENCES blob,\n" + " origid INTEGER REFERENCES blob,\n" + " value TEXT,\n" + " mtime REAL,\n" + " rid INTEGER REFERENCES blob,\n" + " UNIQUE(rid, tagid)\n" + ") STRICT;\n" + "CREATE TABLE backlink(\n" + " target TEXT,\n" + " srctype INT,\n" + " srcid INT,\n" + " mtime REAL,\n" + " UNIQUE(target, srctype, srcid)\n" + ") STRICT;\n" + "CREATE TABLE attachment(\n" + " attachid INTEGER PRIMARY KEY,\n" + " isLatest INT DEFAULT 0,\n" + " mtime REAL,\n" + " src TEXT,\n" + " target TEXT,\n" + " filename TEXT,\n" + " comment TEXT,\n" + " user TEXT\n" + ") STRICT;\n" + "CREATE TABLE cherrypick(\n" + " parentid INT,\n" + " childid INT,\n" + " isExclude INT DEFAULT false,\n" + " PRIMARY KEY(parentid, childid)\n" + ") WITHOUT ROWID, STRICT;\n" + "CREATE TABLE vcache(\n" + " vid INTEGER, -- check-in ID\n" + " fname TEXT, -- filename\n" + " rid INTEGER, -- artifact ID\n" + " PRIMARY KEY(vid,fname)\n" + ") WITHOUT ROWID;\n" + "CREATE TABLE synclog(\n" + " sfrom TEXT,\n" + " sto TEXT,\n" + " stime INT NOT NULL,\n" + " stype TEXT,\n" + " PRIMARY KEY(sfrom,sto)\n" + ") WITHOUT ROWID;\n" + "CREATE TABLE chat(\n" + " msgid INTEGER PRIMARY KEY AUTOINCREMENT,\n" + " mtime JULIANDAY,\n" + " lmtime TEXT,\n" + " xfrom TEXT,\n" + " xmsg TEXT,\n" + " fname TEXT,\n" + " fmime TEXT,\n" + " mdel INT,\n" + " file BLOB\n" + ");\n" + "CREATE TABLE ftsdocs(\n" + " rowid INTEGER PRIMARY KEY,\n" + " type CHAR(1),\n" + " rid INTEGER,\n" + " name TEXT,\n" + " idxed BOOLEAN,\n" + " label TEXT,\n" + " url TEXT,\n" + " mtime DATE,\n" + " bx TEXT,\n" + " UNIQUE(type,rid)\n" + ");\n" + "CREATE TABLE ticket(\n" + " -- Do not change any column that begins with tkt_\n" + " tkt_id INTEGER PRIMARY KEY,\n" + " tkt_uuid TEXT UNIQUE,\n" + " tkt_mtime DATE,\n" + " tkt_ctime DATE,\n" + " -- Add as many fields as required below this line\n" + " type TEXT,\n" + " status TEXT,\n" + " subsystem TEXT,\n" + " priority TEXT,\n" + " severity TEXT,\n" + " foundin TEXT,\n" + " private_contact TEXT,\n" + " resolution TEXT,\n" + " title TEXT,\n" + " comment TEXT\n" + ");\n" + "CREATE TABLE ticketchng(\n" + " -- Do not change any column that begins with tkt_\n" + " tkt_id INTEGER REFERENCES ticket,\n" + " tkt_rid INTEGER REFERENCES blob,\n" + " tkt_mtime DATE,\n" + " tkt_user TEXT,\n" + " -- Add as many fields as required below this line\n" + " login TEXT,\n" + " username TEXT,\n" + " mimetype TEXT,\n" + " icomment TEXT\n" + ");\n" + "CREATE TABLE forumpost(\n" + " fpid INTEGER PRIMARY KEY,\n" + " froot INT,\n" + " fprev INT,\n" + " firt INT,\n" + " fmtime REAL\n" + ");\n" + "CREATE INDEX delta_i1 ON delta(srcid);\n" + "CREATE INDEX blob_rcvid ON blob(rcvid);\n" + "CREATE INDEX subscriberUname\n" + " ON subscriber(suname) WHERE suname IS NOT NULL;\n" + "CREATE INDEX mlink_i1 ON mlink(mid);\n" + "CREATE INDEX mlink_i2 ON mlink(fnid);\n" + "CREATE INDEX mlink_i3 ON mlink(fid);\n" + "CREATE INDEX mlink_i4 ON mlink(pid);\n" + "CREATE INDEX plink_i2 ON plink(cid,pid);\n" + "CREATE INDEX event_i1 ON event(mtime);\n" + "CREATE INDEX orphan_baseline ON orphan(baseline);\n" + "CREATE INDEX tagxref_i1 ON tagxref(tagid, mtime);\n" + "CREATE INDEX backlink_src ON backlink(srcid, srctype);\n" + "CREATE INDEX attachment_idx1 ON attachment(target, filename, mtime);\n" + "CREATE INDEX attachment_idx2 ON attachment(src);\n" + "CREATE INDEX cherrypick_cid ON cherrypick(childid);\n" + "CREATE INDEX ftsdocIdxed ON ftsdocs(type,rid,name) WHERE idxed==0;\n" + "CREATE INDEX ftsdocName ON ftsdocs(name) WHERE type='w';\n" + "CREATE INDEX ticketchng_idx1 ON ticketchng(tkt_id, tkt_mtime);\n" + "CREATE INDEX forumthread ON forumpost(froot,fmtime);\n" + "CREATE VIEW artifact(rid,rcvid,size,atype,srcid,hash,content) AS\n" + " SELECT blob.rid,rcvid,size,1,srcid,uuid,content\n" + " FROM blob LEFT JOIN delta ON (blob.rid=delta.rid);\n" + "CREATE VIEW ftscontent AS\n" + " SELECT rowid, type, rid, name, idxed, label, url, mtime,\n" + " title(type,rid,name) AS 'title', body(type,rid,name) AS 'body'\n" + " FROM ftsdocs;\n" + ); + if( sqlite3_compileoption_used("ENABLE_FTS5") ){ + speedtest1_exec( + "CREATE VIRTUAL TABLE ftsidx\n" + " USING fts5(content=\"ftscontent\", title, body);\n" + "CREATE VIRTUAL TABLE chatfts1 USING fts5(\n" + " xmsg, content=chat, content_rowid=msgid,tokenize=porter);\n" + ); + }else{ + speedtest1_exec( + "CREATE TABLE ftsidx_data(id INTEGER PRIMARY KEY, block BLOB);\n" + "CREATE TABLE ftsidx_idx(segid, term, pgno, PRIMARY KEY(segid, term))\n" + " WITHOUT ROWID;\n" + "CREATE TABLE ftsidx_docsize(id INTEGER PRIMARY KEY, sz BLOB);\n" + "CREATE TABLE ftsidx_config(k PRIMARY KEY, v) WITHOUT ROWID;\n" + "CREATE TABLE chatfts1_data(id INTEGER PRIMARY KEY, block BLOB);\n" + "CREATE TABLE chatfts1_idx(segid, term, pgno, PRIMARY KEY(segid, term))\n" + " WITHOUT ROWID;\n" + "CREATE TABLE chatfts1_docsize(id INTEGER PRIMARY KEY, sz BLOB);\n" + "CREATE TABLE chatfts1_config(k PRIMARY KEY, v) WITHOUT ROWID;\n" + ); + } + speedtest1_exec( + "ANALYZE sqlite_schema;\n" + "INSERT INTO sqlite_stat1(tbl,idx,stat) VALUES\n" + " ('ftsidx_config','ftsidx_config','1 1'),\n" + " ('ftsidx_idx','ftsidx_idx','4215 401 1'),\n" + " ('user','sqlite_autoindex_user_1','25 1'),\n" + " ('phantom',NULL,'26'),\n" + " ('reportfmt','sqlite_autoindex_reportfmt_1','9 1'),\n" + " ('rcvfrom','sqlite_autoindex_rcvfrom_1','18445 401'),\n" + " ('private',NULL,'99'),\n" + " ('mlink','mlink_i4','116678 401'),\n" + " ('mlink','mlink_i3','121212 2'),\n" + " ('mlink','mlink_i2','106372 401'),\n" + " ('mlink','mlink_i1','99298 5'),\n" + " ('ftsidx_data',NULL,'3795'),\n" + " ('leaf',NULL,'1559'),\n" + " ('delta','delta_i1','66340 1'),\n" + " ('unversioned','unversioned','3 1'),\n" + " ('pending_alert','pending_alert','3 1'),\n" + " ('cherrypick','cherrypick_cid','680 2'),\n" + " ('cherrypick','cherrypick','628 1 1'),\n" + " ('config','config','128 1'),\n" + " ('ftsidx_docsize',NULL,'33848'),\n" + " ('event','event_i1','36096 1'),\n" + " ('plink','plink_i2','38236 1 1'),\n" + " ('plink','sqlite_autoindex_plink_1','38357 1 1'),\n" + " ('shun','shun','10 1'),\n" + " ('concealed','concealed','110 1'),\n" + " ('vcache','vcache','1888 401 1'),\n" + " ('ftsdocs','ftsdocName','19 1'),\n" + " ('ftsdocs','ftsdocIdxed','168 84 1 1'),\n" + " ('ftsdocs','sqlite_autoindex_ftsdocs_1','37312 401 1'),\n" + " ('subscriber','subscriberUname','5 1'),\n" + " ('subscriber','sqlite_autoindex_subscriber_2','37 1'),\n" + " ('subscriber','sqlite_autoindex_subscriber_1','37 1'),\n" + " ('tag','sqlite_autoindex_tag_1','2990 1'),\n" + " ('filename','sqlite_autoindex_filename_1','3168 1'),\n" + " ('chat',NULL,'56124'),\n" + " ('tagxref','tagxref_i1','40992 401 2'),\n" + " ('tagxref','sqlite_autoindex_tagxref_1','79233 3 1'),\n" + " ('attachment','attachment_idx2','11 1'),\n" + " ('attachment','attachment_idx1','11 2 2 1'),\n" + " ('blob','blob_rcvid','128240 201'),\n" + " ('blob','sqlite_autoindex_blob_1','126480 1'),\n" + " ('synclog','synclog','12 3 1'),\n" + " ('backlink','backlink_src','2160 2 2'),\n" + " ('backlink','sqlite_autoindex_backlink_1','2340 2 2 1'),\n" + " ('accesslog',NULL,'38'),\n" + " ('chatfts1_config','chatfts1_config','1 1'),\n" + " ('chatfts1_idx','chatfts1_idx','688 230 1'),\n" + " ('ticket','sqlite_autoindex_ticket_1','794 1'),\n" + " ('ticketchng','ticketchng_idx1','2089 3 1'),\n" + " ('forumpost','forumthread','4 4 1'),\n" + " ('unclustered',NULL,'12');\n" + "COMMIT;" + ); + speedtest1_end_test(); + + n = g.szTest*3; + speedtest1_begin_test(110, "Open and use the database %d times", n); + for(i=0; i=$date OR parent.pid=$pid)\n" + " ORDER BY mtime DESC LIMIT 10\n" + " )\n" + " INSERT OR IGNORE INTO ok SELECT rid FROM ancestor;" + ); + sqlite3_close(dbAux); + g.db = dbMain; + } + speedtest1_end_test(); +} + #ifdef SQLITE_ENABLE_RTREE /* Generate two numbers between 1 and mx. The first number is less than ** the second. Usually the numbers are near each other but can sometimes @@ -2393,6 +2968,8 @@ int main(int argc, char **argv){ int doIncrvac = 0; /* True for --incrvacuum */ const char *zJMode = 0; /* Journal mode */ const char *zKey = 0; /* Encryption key */ + int nHardHeapLmt = 0; /* The hard heap limit */ + int nSoftHeapLmt = 0; /* The soft heap limit */ int nLook = -1, szLook = 0; /* --lookaside configuration */ int noSync = 0; /* True for --nosync */ int pageSize = 0; /* Desired page size. 0 means default */ @@ -2404,11 +2981,9 @@ int main(int argc, char **argv){ int memDb = 0; /* --memdb. Use an in-memory database */ int openFlags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE ; /* SQLITE_OPEN_xxx flags. */ - char *zTSet = "main"; /* Which --testset torun */ - const char * zVfs = 0; /* --vfs NAME */ + char *zTSet = "mix1"; /* Which --testset torun */ int doTrace = 0; /* True for --trace */ const char *zEncoding = 0; /* --utf16be or --utf16le */ - const char *zDbName = 0; /* Name of the test database */ void *pHeap = 0; /* Allocated heap space */ void *pLook = 0; /* Allocated lookaside space */ @@ -2417,6 +2992,10 @@ int main(int argc, char **argv){ int i; /* Loop counter */ int rc; /* API return code */ + /* "mix1" is a macro testset: */ + static char zMix1Tests[] = + "main,orm/25,cte/20,json,fp/3,parsenumber/25,rtree/10,star,app"; + #ifdef SQLITE_SPEEDTEST1_WASM /* Resetting all state is important for the WASM build, which may ** call main() multiple times. */ @@ -2435,6 +3014,8 @@ int main(int argc, char **argv){ sqlite3_libversion(), sqlite3_sourceid()); /* Process command-line arguments */ + g.zDbName = 0; + g.zVfs = 0; g.zWR = ""; g.zNN = ""; g.zPK = "UNIQUE"; @@ -2461,6 +3042,10 @@ int main(int argc, char **argv){ }else if( strcmp(z,"explain")==0 ){ g.bSqlOnly = 1; g.bExplain = 1; + }else if( strcmp(z,"hard-heap-limit")==0 ){ + ARGC_VALUE_CHECK(1); + nHardHeapLmt = integerValue(argv[i+1]); + i += 1; }else if( strcmp(z,"heap")==0 ){ ARGC_VALUE_CHECK(2); nHeap = integerValue(argv[i+1]); @@ -2550,6 +3135,10 @@ int main(int argc, char **argv){ }else if( strcmp(z,"size")==0 ){ ARGC_VALUE_CHECK(1); g.szTest = g.szBase = integerValue(argv[++i]); + }else if( strcmp(z,"soft-heap-limit")==0 ){ + ARGC_VALUE_CHECK(1); + nSoftHeapLmt = integerValue(argv[i+1]); + i += 1; }else if( strcmp(z,"stats")==0 ){ showStats = 1; }else if( strcmp(z,"temp")==0 ){ @@ -2560,10 +3149,8 @@ int main(int argc, char **argv){ } g.eTemp = argv[i][0] - '0'; }else if( strcmp(z,"testset")==0 ){ - static char zMix1Tests[] = "main,orm/25,cte/20,json,fp/3,parsenumber/25,rtree/10"; ARGC_VALUE_CHECK(1); zTSet = argv[++i]; - if( strcmp(zTSet,"mix1")==0 ) zTSet = zMix1Tests; }else if( strcmp(z,"trace")==0 ){ doTrace = 1; }else if( strcmp(z,"threads")==0 ){ @@ -2580,7 +3167,7 @@ int main(int argc, char **argv){ #endif }else if( strcmp(z,"vfs")==0 ){ ARGC_VALUE_CHECK(1); - zVfs = argv[++i]; + g.zVfs = argv[++i]; }else if( strcmp(z,"reserve")==0 ){ ARGC_VALUE_CHECK(1); g.nReserve = atoi(argv[++i]); @@ -2610,8 +3197,8 @@ int main(int argc, char **argv){ fatal_error("unknown option: %s\nUse \"%s -?\" for help\n", argv[i], argv[0]); } - }else if( zDbName==0 ){ - zDbName = argv[i]; + }else if( g.zDbName==0 ){ + g.zDbName = argv[i]; }else{ fatal_error("surplus argument: %s\nUse \"%s -?\" for help\n", argv[i], argv[0]); @@ -2640,8 +3227,8 @@ int main(int argc, char **argv){ #endif sqlite3_initialize(); - if( zDbName!=0 ){ - sqlite3_vfs *pVfs = sqlite3_vfs_find(zVfs); + if( g.zDbName!=0 ){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(g.zVfs); /* For some VFSes, e.g. opfs, unlink() is not sufficient. Use the ** selected (or default) VFS's xDelete method to delete the ** database. This is specifically important for the "opfs" VFS @@ -2649,15 +3236,15 @@ int main(int argc, char **argv){ ** can be cleaned up properly. For historical compatibility, we'll ** also simply unlink(). */ if( pVfs!=0 ){ - pVfs->xDelete(pVfs, zDbName, 1); + pVfs->xDelete(pVfs, g.zDbName, 1); } - unlink(zDbName); + unlink(g.zDbName); } /* Open the database and the input file */ - if( sqlite3_open_v2(memDb ? ":memory:" : zDbName, &g.db, - openFlags, zVfs) ){ - fatal_error("Cannot open database file: %s\n", zDbName); + if( sqlite3_open_v2(memDb ? ":memory:" : g.zDbName, &g.db, + openFlags, g.zVfs) ){ + fatal_error("Cannot open database file: %s\n", g.zDbName); } #if SQLITE_VERSION_NUMBER>=3006001 if( nLook>0 && szLook>0 ){ @@ -2713,8 +3300,18 @@ int main(int argc, char **argv){ if( zJMode ){ speedtest1_exec("PRAGMA journal_mode=%s", zJMode); } + if( nHardHeapLmt>0 ){ + speedtest1_exec("PRAGMA hard_heap_limit=%d", nHardHeapLmt); + } + if( nSoftHeapLmt>0 ){ + speedtest1_exec("PRAGMA soft_heap_limit=%d", nSoftHeapLmt); + } + if( zJMode ){ + speedtest1_exec("PRAGMA journal_mode=%s", zJMode); + } if( g.bExplain ) printf(".explain\n.echo on\n"); + if( strcmp(zTSet,"mix1")==0 ) zTSet = zMix1Tests; do{ char *zThisTest = zTSet; char *zSep; @@ -2749,6 +3346,10 @@ int main(int argc, char **argv){ testset_orm(); }else if( strcmp(zThisTest,"cte")==0 ){ testset_cte(); + }else if( strcmp(zThisTest,"star")==0 ){ + testset_star(); + }else if( strcmp(zThisTest,"app")==0 ){ + testset_app(); }else if( strcmp(zThisTest,"fp")==0 ){ testset_fp(); }else if( strcmp(zThisTest,"json")==0 ){ diff --git a/test/subquery.test b/test/subquery.test index 17061d4b6..898c8f756 100644 --- a/test/subquery.test +++ b/test/subquery.test @@ -650,6 +650,44 @@ do_eqp_test subquery-10.2 { # |--SCAN v2 # `--SEARCH t1 USING INDEX x12 (aa=?) # +#--------------------------------------------------------------------------- +# Follow-up on 2025-04-14. Performance issue found while working +# on Fossil (Fossil check-in 2025-04-13T19:54). +# +do_execsql_test 10.3 { + CREATE TABLE blob( + rid INTEGER PRIMARY KEY, + size INT, + uuid TEXT + ); + CREATE TABLE delta( + rid INTEGER PRIMARY KEY, + srcid INT + ); + CREATE INDEX delta_i1 ON delta(srcid); +} +do_eqp_test subquery-10.4 { + WITH RECURSIVE deltasof(rid) AS ( + SELECT rid FROM delta WHERE srcid=125020 + UNION + SELECT delta.rid FROM deltasof, delta + WHERE delta.srcid=deltasof.rid) + SELECT deltasof.rid, blob.uuid FROM deltasof, blob + WHERE blob.rid=deltasof.rid; +} { + QUERY PLAN + |--CO-ROUTINE deltasof + | |--SETUP + | | `--SEARCH delta USING COVERING INDEX delta_i1 (srcid=?) + | `--RECURSIVE STEP + | |--SCAN deltasof + | `--SEARCH delta USING COVERING INDEX delta_i1 (srcid=?) + |--SCAN deltasof + `--SEARCH blob USING INTEGER PRIMARY KEY (rowid=?) +} +# ^^^^^^^^^^^^^^^^^^ +# deltasof should be the outer loop and blob the inner loop +# Prior to the fix, SQLite was doing it the other way around. #----------------------------------------------------------------------------- # 2024-04-25 Column affinities for columns of compound subqueries diff --git a/test/subquery2.test b/test/subquery2.test index 0c1bdc669..8513dc75c 100644 --- a/test/subquery2.test +++ b/test/subquery2.test @@ -104,7 +104,7 @@ do_execsql_test 2.2 { } {2 3 3 6 4 10} ############################################################################ -# Ticket http://www.sqlite.org/src/info/d11a6e908f (2014-09-20) +# Ticket http://sqlite.org/src/info/d11a6e908f (2014-09-20) # Query planner fault on three-way nested join with compound inner SELECT # do_execsql_test 3.0 { diff --git a/test/symlink.test b/test/symlink.test index 685cae5a4..fc78a0472 100644 --- a/test/symlink.test +++ b/test/symlink.test @@ -17,7 +17,7 @@ source $testdir/tester.tcl set testprefix symlink # This only runs on unix. -if {$::tcl_platform(platform)!="unix"} { +if {$::tcl_platform(os) eq "Windows NT"} { finish_test return } diff --git a/test/symlink2.test b/test/symlink2.test index 9a2237e4c..7305f6dd3 100644 --- a/test/symlink2.test +++ b/test/symlink2.test @@ -17,7 +17,7 @@ source $testdir/tester.tcl set testprefix symlink2 # This only runs on Windows. -if {$::tcl_platform(platform)!="windows"} { +if {$::tcl_platform(platform) ne "windows"} { finish_test return } diff --git a/test/sync.test b/test/sync.test index 023425e6b..b24800d10 100644 --- a/test/sync.test +++ b/test/sync.test @@ -34,7 +34,7 @@ if {[atomic_batch_write test.db]} { set sqlite_sync_count 0 proc cond_incr_sync_count {adj} { global sqlite_sync_count - if {$::tcl_platform(platform) == "windows"} { + if {$::tcl_platform(os) eq "Windows NT"} { incr sqlite_sync_count $adj } else { ifcapable !dirsync { diff --git a/test/sync2.test b/test/sync2.test index 89e66c845..ce8132c71 100644 --- a/test/sync2.test +++ b/test/sync2.test @@ -26,7 +26,7 @@ ifcapable !pager_pragmas||!attach||!dirsync { finish_test return } -if {$::tcl_platform(platform)!="unix" +if {$::tcl_platform(os) eq "Windows NT" || [permutation] == "journaltest" || [permutation] == "inmemory_journal" || [atomic_batch_write test.db] diff --git a/test/tabfunc01.test b/test/tabfunc01.test index 5938ec6cb..0e29c3568 100644 --- a/test/tabfunc01.test +++ b/test/tabfunc01.test @@ -339,7 +339,7 @@ do_test tabfunc01-750 { } } {5.0 x5 | 7.0 x7 | 13.0 x13 | 17.0 x17 | 23.0 x23 |} -# ticket https://www.sqlite.org/src/info/2ae0c599b735d59e +# ticket https://sqlite.org/src/info/2ae0c599b735d59e # Verification of testtag-20230227a do_test tabfunc01-751 { db eval { diff --git a/test/table.test b/test/table.test index b961207f8..46ecd7d23 100644 --- a/test/table.test +++ b/test/table.test @@ -792,7 +792,7 @@ do_catchsql_test table-16.7 { INSERT INTO t16 DEFAULT VALUES; } {1 {unknown function: string_agg()}} -# Ticket [https://www.sqlite.org/src/info/094d39a4c95ee4abbc417f04214617675ba15c63] +# Ticket [https://sqlite.org/src/info/094d39a4c95ee4abbc417f04214617675ba15c63] # describes a assertion fault that occurs on a CREATE TABLE .. AS SELECT statement. # the following test verifies that the problem has been fixed. # @@ -810,7 +810,7 @@ do_execsql_test table-17.1 { } {1 1 | 2 2 |} # 2015-06-16 -# Ticket [https://www.sqlite.org/src/tktview/873cae2b6e25b1991ce5e9b782f9cd0409b96063] +# Ticket [https://sqlite.org/src/tktview/873cae2b6e25b1991ce5e9b782f9cd0409b96063] # Make sure a CREATE TABLE AS statement correctly rolls back partial changes to the # sqlite_master table when the SELECT on the right-hand side aborts. # @@ -825,7 +825,7 @@ do_execsql_test table-18.2 { } {ok} # 2015-09-09 -# Ticket [https://www.sqlite.org/src/info/acd12990885d9276] +# Ticket [https://sqlite.org/src/info/acd12990885d9276] # "CREATE TABLE ... AS SELECT ... FROM sqlite_master" fails because the row # in the sqlite_master table for the next table is initially populated # with a NULL instead of a record created by OP_Record. diff --git a/test/tableapi.test b/test/tableapi.test index 02633cdca..772073747 100644 --- a/test/tableapi.test +++ b/test/tableapi.test @@ -230,7 +230,7 @@ ifcapable schema_pragmas { } # do_malloc_test closes and deletes the usual db connections and files on -# each iteration. $::dbx is a seperate connection, and on Windows, will +# each iteration. $::dbx is a separate connection, and on Windows, will # cause the file deletion of test.db to fail, so we move the close of $::dbx # up to here before the do_malloc_test. do_test tableapi-99.0 { diff --git a/test/tester.tcl b/test/tester.tcl index b5f49ebde..3fad39668 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -809,6 +809,15 @@ proc do_test {name cmd expected} { flush stdout } +# Like do_test except the test is not run in a slave interpreter +# on Windows because of issues with ANSI and UTF8 I/O on Win11. +# +proc do_test_with_ansi_output {name cmd expected} { + if {![info exists ::SLAVE] || $::tcl_platform(platform) ne "windows"} { + uplevel 1 [list do_test $name $cmd $expected] + } +} + proc dumpbytes {s} { set r "" for {set i 0} {$i < [string length $s]} {incr i} { @@ -863,7 +872,7 @@ proc catchcmdex {db {cmd ""}} { proc filepath_normalize {p} { # test cases should be written to assume "unix"-like file paths - if {$::tcl_platform(platform)!="unix"} { + if {$::tcl_platform(platform) ne "unix"} { string map [list \\ / \{/ / .db\} .db] \ [regsub -nocase -all {[a-z]:[/\\]+} $p {/}] } { @@ -1724,7 +1733,7 @@ proc ifcapable {expr code {else ""} {elsecode ""}} { return -code $c $r } -# This proc execs a seperate process that crashes midway through executing +# This proc execs a separate process that crashes midway through executing # the SQL script $sql on database test.db. # # The crash occurs during a sync() of file $crashfile. When the crash @@ -1776,6 +1785,11 @@ proc crashsql {args} { # cfSync(), which can be different then what TCL uses by # default, so here we force it to the "nativename" format. set cfile [string map {\\ \\\\} [file nativename [file join [get_pwd] $crashfile]]] + ifcapable winrt { + # Except on winrt. Winrt has no way to transform a relative path into + # an absolute one, so it just uses the relative paths. + set cfile $crashfile + } set f [open crash.tcl w] puts $f "sqlite3_initialize ; sqlite3_shutdown" @@ -1818,7 +1832,7 @@ proc crashsql {args} { # error message. We map that to the expected message # so that we don't have to change all of the test # cases. - if {$::tcl_platform(platform)=="windows"} { + if {$::tcl_platform(platform) eq "windows"} { if {$msg=="child killed: unknown signal"} { set msg "child process exited abnormally" } @@ -1869,7 +1883,7 @@ proc crash_on_write {args} { # error message. We map that to the expected message # so that we don't have to change all of the test # cases. - if {$::tcl_platform(platform)=="windows"} { + if {$::tcl_platform(platform) eq "windows"} { if {$msg=="child killed: unknown signal"} { set msg "child process exited abnormally" } @@ -2518,7 +2532,7 @@ proc test_restore_config_pagecache {} { } proc test_binary_name {nm} { - if {$::tcl_platform(platform)=="windows"} { + if {$::tcl_platform(platform) eq "windows"} { set ret "$nm.exe" } else { set ret $nm diff --git a/test/testrunner.tcl b/test/testrunner.tcl index d365092e0..0c6982f42 100755 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -30,15 +30,15 @@ proc find_interpreter {} { && [file executable ./testfixture] } { puts "Failed to find tcl package sqlite3. Restarting with ./testfixture.." - set status [catch { - exec ./testfixture [info script] {*}$::argv >@ stdout + set status [catch { + exec [trd_get_bin_name testfixture] [info script] {*}$::argv >@ stdout } msg] exit $status } } if {$rc} { puts "Cannot find tcl package sqlite3: Trying to build it now..." - if {$::tcl_platform(platform)=="windows"} { + if {$::tcl_platform(platform) eq "windows"} { set bat [open make-tcl-extension.bat w] puts $bat "nmake /f Makefile.msc tclextension" close $bat @@ -98,6 +98,7 @@ Usage: --config CONFIGS Only use configs on comma-separate list CONFIGS --dryrun Write what would have happened to testrunner.log --explain Write summary to stdout + --fuzzdb FILENAME Additional external fuzzcheck database --jobs NUM Run tests using NUM separate processes --omit CONFIGS Omit configs on comma-separated list CONFIGS --status Show the full "status" report while running @@ -169,8 +170,7 @@ Full documentation here: https://sqlite.org/src/doc/trunk/doc/testrunner.md proc guess_number_of_cores {} { if {[catch {number_of_cores} ret]} { set ret 4 - - if {$::tcl_platform(platform)=="windows"} { + if {$::tcl_platform(platform) eq "windows"} { catch { set ret $::env(NUMBER_OF_PROCESSORS) } } else { if {$::tcl_platform(os)=="Darwin"} { @@ -237,7 +237,7 @@ switch -nocase -glob -- $tcl_platform(os) { set TRG(run) run.sh set TRG(runcmd) "bash run.sh" } - *linux* { + *linux* - MSYS_NT* - MINGW64_NT* - MINGW32_NT* { set TRG(platform) linux set TRG(make) make.sh set TRG(makecmd) "bash make.sh" @@ -263,11 +263,25 @@ switch -nocase -glob -- $tcl_platform(os) { set TRG(shell) sqlite3.exe set TRG(run) run.bat set TRG(runcmd) "run.bat" + if {"unix" eq $tcl_platform(platform)} { + # Presumably cygwin. This block gets testrunner.tcl started on + # Cygwin but then downstream tests all fail, at least in part + # because of the discrepancies in build target names which need + # .exe on cygwin but not on other Unix-like platforms. + set TRG(platform) cygwin + set TRG(make) make.sh + set TRG(makecmd) "bash make.sh" + set TRG(testfixture) testfixture + set TRG(shell) sqlite3 + set TRG(run) run.sh + set TRG(runcmd) "bash run.sh" + } } default { + puts "tcl_platform(os)=$::tcl_platform(os)" error "cannot determine platform!" } -} +} #------------------------------------------------------------------------- #------------------------------------------------------------------------- @@ -518,7 +532,7 @@ proc show_status {db cls} { (SELECT value FROM config WHERE name='start') }] - set total 0 + set totalw 0 foreach s {"" ready running done failed omit} { set S($s) 0; set W($s) 0; } set workpending 0 $db eval { @@ -543,7 +557,7 @@ proc show_status {db cls} { flush stdout } puts [format %-79.79s "Command: \[testrunner.tcl$cmdline\]"] - puts [format %-79.79s "Summary: [elapsetime $tm], $fin/$total jobs,\ + puts [format %-79.79s "Summary: [elapsetime $tm], $fin/$totalw jobs,\ $ne errors, $nt tests"] set srcdir [file dirname [file dirname $TRG(info_script)]] @@ -798,12 +812,15 @@ for {set ii 0} {$ii < [llength $argv]} {incr ii} { } elseif {($n>2 && [string match "$a*" --omit]) || $a=="-c"} { incr ii set TRG(omitconfig) [lindex $argv $ii] + } elseif {($n>2 && [string match "$a*" --fuzzdb])} { + incr ii + set env(FUZZDB) [lindex $argv $ii] } elseif {[string match "$a*" --stop-on-error]} { set TRG(stopOnError) 1 } elseif {[string match "$a*" --stop-on-coredump]} { set TRG(stopOnCore) 1 } elseif {[string match "$a*" --status]} { - if {$tcl_platform(platform)=="windows"} { + if {$tcl_platform(platform) eq "windows"} { puts stdout \ "The --status option is not available on Windows. A suggested work-around" puts stdout \ @@ -989,6 +1006,35 @@ proc add_job {args} { trdb last_insert_rowid } + +# Look to see if $jobcmd matches any of the glob patterns given in +# $patternlist. Return true if there is a match. Return false +# if no match is seen. +# +# An empty patternlist matches everything +# +proc job_matches_any_pattern {patternlist jobcmd} { + set bMatch 0 + if {[llength $patternlist]==0} {return 1} + foreach p $patternlist { + set p [string trim $p *] + if {[string index $p 0]=="^"} { + set p [string range $p 1 end] + } else { + set p "*$p" + } + if {[string index $p end]=="\$"} { + set p [string range $p 0 end-1] + } else { + set p "$p*" + } + if {[string match $p $jobcmd]} { + set bMatch 1 + break + } + } + return $bMatch +} # Argument $build is either an empty string, or else a list of length 3 @@ -1002,6 +1048,7 @@ proc add_job {args} { # proc add_tcl_jobs {build config patternlist {shelldepid ""}} { global TRG + set ntcljob 0 set topdir [file dirname $::testdir] set testrunner_tcl [file normalize [info script]] @@ -1019,26 +1066,8 @@ proc add_tcl_jobs {build config patternlist {shelldepid ""}} { # The ::testspec array is populated by permutations.test foreach f [dict get $::testspec($config) -files] { - if {[llength $patternlist]>0} { - set bMatch 0 - foreach p $patternlist { - set p [string trim $p *] - if {[string index $p 0]=="^"} { - set p [string range $p 1 end] - } else { - set p "*$p" - } - if {[string index $p end]=="\$"} { - set p [string range $p 0 end-1] - } else { - set p "$p*" - } - if {[string match $p "$config [file tail $f]"]} { - set bMatch 1 - break - } - } - if {$bMatch==0} continue + if {![job_matches_any_pattern $patternlist "$config [file tail $f]"]} { + continue } if {[file pathtype $f]!="absolute"} { set f [file join $::testdir $f] } @@ -1063,6 +1092,7 @@ proc add_tcl_jobs {build config patternlist {shelldepid ""}} { set depid [lindex $build 0] if {$shelldepid!="" && [lsearch $lProp shell]>=0} { set depid $shelldepid } + incr ntcljob add_job \ -displaytype tcl \ -displayname $displayname \ @@ -1070,6 +1100,10 @@ proc add_tcl_jobs {build config patternlist {shelldepid ""}} { -depid $depid \ -priority $priority } + if {$ntcljob==0 && [llength $build]>0} { + set bldid [lindex $build 0] + trdb eval {DELETE FROM jobs WHERE rowid=$bldid} + } } proc add_build_job {buildname target {postcmd ""} {depid ""}} { @@ -1132,26 +1166,57 @@ proc add_make_job {bld target} { -priority 1 } -proc add_fuzztest_jobs {buildname} { +proc add_fuzztest_jobs {buildname patternlist} { + global env TRG + # puts buildname=$buildname - foreach {interpreter scripts} [trd_fuzztest_data] { + foreach {interpreter scripts} [trd_fuzztest_data $buildname] { + set bldDone 0 set subcmd [lrange $interpreter 1 end] set interpreter [lindex $interpreter 0] - set bld [add_build_job $buildname $interpreter] - foreach {depid dirname displayname} $bld {} + if {[string match fuzzcheck* $interpreter] + && [info exists env(FUZZDB)] + && [file readable $env(FUZZDB)] + && $buildname ne "Windows-Win32Heap" + && $buildname ne "Windows-Memdebug" + } { + set TRG(FUZZDB) $env(FUZZDB) + set fname [file normalize $env(FUZZDB)] + set N [expr {([file size $fname]+4999999)/5000000}] + for {set i 0} {$i<$N} {incr i} { + lappend scripts [list --slice $i $N $fname] + } + } foreach s $scripts { # Fuzz data files fuzzdata1.db and fuzzdata2.db are larger than # the others. So ensure that these are run as a higher priority. - set tail [file tail $s] - if {$tail=="fuzzdata1.db" || $tail=="fuzzdata2.db"} { + if {[llength $s]==1} { + set tail [file tail $s] + } else { + set fname [lindex $s end] + set tail [lrange $s 0 end-1] + lappend tail [file tail $fname] + } + if {![job_matches_any_pattern $patternlist "$interpreter $tail"]} { + continue + } + if {!$bldDone} { + set bld [add_build_job $buildname $interpreter] + foreach {depid dirname displayname} $bld {} + set bldDone 1 + } + if {[string match ?-slice* $tail]} { + set priority 15 + } elseif {$tail=="fuzzdata1.db" + || $tail=="fuzzdata2.db" + || $tail=="fuzzdata8.db"} { set priority 5 } else { set priority 1 } - add_job \ -displaytype fuzz \ -displayname "$buildname $interpreter $tail" \ @@ -1187,9 +1252,7 @@ proc add_devtest_jobs {lBld patternlist} { foreach b $lBld { set bld [add_build_job $b $TRG(testfixture)] add_tcl_jobs $bld veryquick $patternlist SHELL - if {$patternlist==""} { - add_fuzztest_jobs $b - } + add_fuzztest_jobs $b $patternlist if {[trdb one "SELECT EXISTS (SELECT 1 FROM jobs WHERE depid='SHELL')"]} { set sbld [add_shell_build_job $b [lindex $bld 1] [lindex $bld 0]] @@ -1263,13 +1326,11 @@ proc add_jobs_from_cmdline {patternlist} { add_tcl_jobs $bld $c $patternlist SHELL } - if {$patternlist==""} { - foreach e [trd_extras $TRG(platform) $b] { - if {$e=="fuzztest"} { - add_fuzztest_jobs $b - } else { - add_make_job $bld $e - } + foreach e [trd_extras $TRG(platform) $b] { + if {$e=="fuzztest"} { + add_fuzztest_jobs $b $patternlist + } elseif {[job_matches_any_pattern $patternlist $e]} { + add_make_job $bld $e } } @@ -1486,7 +1547,7 @@ proc progress_report {} { global TRG if {$TRG(fullstatus)} { - if {$::tcl_platform(platform)=="windows"} { + if {$::tcl_platform(platform) eq "windows"} { exec [info nameofexe] $::argv0 status --cls } else { show_status trdb 1 @@ -1589,6 +1650,9 @@ proc run_testset {} { puts "\nTest database is $TRG(dbname)" puts "Test log is $TRG(logname)" + if {[info exists TRG(FUZZDB)]} { + puts "Extra fuzzcheck data taken from $TRG(FUZZDB)" + } trdb eval { SELECT sum(ntest) AS totaltest, sum(nerr) AS totalerr @@ -1641,7 +1705,13 @@ proc explain_layer {indent depid} { puts "${indent}$displayname in $dirname" explain_layer "${indent} " $jobid } elseif {$showtests} { - set tail [lindex $displayname end] + if {[lindex $displayname end-3] eq "--slice"} { + set M [lindex $displayname end-2] + set N [lindex $displayname end-1] + set tail "[lindex $displayname end] (slice $M/$N)" + } else { + set tail [lindex $displayname end] + } set e1 [lindex $displayname 1] if {[string match config=* $e1]} { set cfg [string range $e1 7 end] diff --git a/test/testrunner_data.tcl b/test/testrunner_data.tcl index 2cfa7f3b3..3998bd9cc 100644 --- a/test/testrunner_data.tcl +++ b/test/testrunner_data.tcl @@ -37,6 +37,7 @@ namespace eval trd { set tcltest(win.Windows-Memdebug) veryquick set tcltest(win.Windows-Win32Heap) veryquick set tcltest(win.Windows-Sanitize) veryquick + set tcltest(win.Windows-WinRT) veryquick set tcltest(win.Default) full # Extra [make xyz] tests that should be run for various builds. @@ -358,11 +359,17 @@ namespace eval trd { set build(Windows-Win32Heap) { WIN32HEAP=1 DEBUG=4 + ENABLE_SETLK=1 } set build(Windows-Sanitize) { ASAN=1 } + set build(Windows-WinRT) { + FOR_WINRT=1 + ENABLE_SETLK=1 + -DSQLITE_TEMP_STORE=3 + } } @@ -423,7 +430,7 @@ proc trd_extras {platform bld} { # Usage: # -# trd_fuzztest_data +# trd_fuzztest_data $buildname # # This returns data used by testrunner.tcl to run commands equivalent # to [make fuzztest]. The returned value is a list, which should be @@ -443,16 +450,25 @@ proc trd_extras {platform bld} { # directory containing this file). "fuzzcheck" and "sessionfuzz" have .exe # extensions on windows. # -proc trd_fuzztest_data {} { +proc trd_fuzztest_data {buildname} { set EXE "" set lFuzzDb [glob [file join $::testdir fuzzdata*.db]] set lSessionDb [glob [file join $::testdir sessionfuzz-data*.db]] + set sanBuilds {All-Debug Apple Have-Not Update-Delete-Limit} - if {$::tcl_platform(platform)=="windows"} { + if {$::tcl_platform(platform) eq "windows"} { return [list fuzzcheck.exe $lFuzzDb] + } else { + set lRet [list [trd_get_bin_name fuzzcheck] $lFuzzDb] + if {[lsearch $sanBuilds $buildname]>=0} { + lappend lRet [trd_get_bin_name fuzzcheck-asan] $lFuzzDb + if {$::tcl_platform(os) ne "OpenBSD"} { + lappend lRet [trd_get_bin_name fuzzcheck-ubsan] $lFuzzDb + } + } + lappend lRet {sessionfuzz run} $lSessionDb + return $lRet } - - return [list fuzzcheck $lFuzzDb {sessionfuzz run} $lSessionDb] } @@ -520,7 +536,7 @@ proc make_script {cfg srcdir bMsvc} { set configOpts [list] ;# Extra args for [configure] # Define either SQLITE_OS_WIN or SQLITE_OS_UNIX, as appropriate. - if {$::tcl_platform(platform)=="windows"} { + if {$::tcl_platform(os) eq "Windows NT"} { lappend opts -DSQLITE_OS_WIN=1 } else { lappend opts -DSQLITE_OS_UNIX=1 @@ -681,3 +697,15 @@ proc trd_test_script_properties {path} { set trd_test_script_properties_cache($path) } + +# Usage: +# +# trd_get_bin_name executable-file-name +# +# If the tcl platform is "unix", return $bin, else return +# ${bin}.exe. +proc trd_get_bin_name {bin} { + global tcl_platform + if {"unix" eq $tcl_platform(platform)} {return $bin} + return $bin.exe +} diff --git a/test/tkt3457.test b/test/tkt3457.test index 027349463..17b6c72cd 100644 --- a/test/tkt3457.test +++ b/test/tkt3457.test @@ -15,10 +15,10 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -if {$tcl_platform(platform) != "unix"} { +if {[llength [info commands test_syscall]]==0} { finish_test return -} +} if {[atomic_batch_write test.db]} { finish_test return diff --git a/test/trigger1.test b/test/trigger1.test index afeb7ddcc..67943677f 100644 --- a/test/trigger1.test +++ b/test/trigger1.test @@ -751,7 +751,7 @@ do_execsql_test trigger1-18.1 { SELECT * FROM t18; } {1 3 2} ;# Not: 1 1001 1000 -# 2018-04-26 ticket [https://www.sqlite.org/src/tktview/d85fffd6ffe856092e] +# 2018-04-26 ticket [https://sqlite.org/src/tktview/d85fffd6ffe856092e] # VDBE Program uses an expired value. # do_execsql_test trigger1-19.0 { diff --git a/test/triggerG.test b/test/triggerG.test index 8cf20b5ec..b078fffb2 100644 --- a/test/triggerG.test +++ b/test/triggerG.test @@ -19,7 +19,7 @@ ifcapable {!trigger} { } # Test cases for ticket -# https://www.sqlite.org/src/tktview/06796225f59c057cd120f +# https://sqlite.org/src/tktview/06796225f59c057cd120f # # The OP_Once opcode was not working correctly for recursive triggers. # diff --git a/test/unixexcl.test b/test/unixexcl.test index 8e9c4644d..c24945e5e 100644 --- a/test/unixexcl.test +++ b/test/unixexcl.test @@ -18,7 +18,7 @@ source $testdir/tester.tcl source $testdir/lock_common.tcl source $testdir/malloc_common.tcl -if {$::tcl_platform(platform)!="unix" || [info commands test_syscall]==""} { +if {[llength [info commands test_syscall]]==0} { finish_test return } diff --git a/test/update.test b/test/update.test index bf7666662..0a380fa03 100644 --- a/test/update.test +++ b/test/update.test @@ -616,7 +616,7 @@ do_test update-14.4 { } ;# ifcapable {trigger} -# Ticket [https://www.sqlite.org/src/tktview/43107840f1c02] on 2014-10-29 +# Ticket [https://sqlite.org/src/tktview/43107840f1c02] on 2014-10-29 # An assertion fault on UPDATE # ifcapable altertable { @@ -703,7 +703,7 @@ do_execsql_test update-19.10 { SELECT * FROM t1; } {2 2} -# 2019-12-29 ticket https://www.sqlite.org/src/info/314cc133e5ada126 +# 2019-12-29 ticket https://sqlite.org/src/info/314cc133e5ada126 # REPLACE conflict resolution during an UPDATE causes a DELETE trigger # to fire. If that DELETE trigger subsequently modifies the row # being updated, bad things can happen. Prevent this by prohibiting @@ -711,8 +711,8 @@ do_execsql_test update-19.10 { # REPLACE conflict resolution on the UPDATE. # # See also tickets: -# https://www.sqlite.org/src/info/c1e19e12046d23fe 2019-10-25 -# https://www.sqlite.org/src/info/a8a4847a2d96f5de 2019-10-16 +# https://sqlite.org/src/info/c1e19e12046d23fe 2019-10-25 +# https://sqlite.org/src/info/a8a4847a2d96f5de 2019-10-16 # reset_db do_execsql_test update-20.10 { diff --git a/test/upsert1.test b/test/upsert1.test index 8af273a89..49168f840 100644 --- a/test/upsert1.test +++ b/test/upsert1.test @@ -129,7 +129,7 @@ do_execsql_test upsert1-610 { } {ok} # 2018-08-14 -# Ticket https://www.sqlite.org/src/info/908f001483982c43 +# Ticket https://sqlite.org/src/info/908f001483982c43 # If there are multiple uniqueness contraints, the UPSERT should fire # if the one constraint it targets fails, regardless of whether or not # the other constraints pass or fail. In other words, the UPSERT constraint diff --git a/test/uri.test b/test/uri.test index 2b388c400..74da225ac 100644 --- a/test/uri.test +++ b/test/uri.test @@ -59,7 +59,7 @@ foreach {tn uri file} { if {[string first %00 $uri]>=0} continue } - if {$tcl_platform(platform)=="windows"} { + if {$tcl_platform(platform) eq "windows"} { # # NOTE: Due to limits on legal characters for file names imposed by # Windows, we must skip the final two tests here (i.e. the @@ -306,7 +306,7 @@ foreach {tn uri res} { 6 "file://x/PWD/test.db" {invalid uri authority: x} } { - if {$tcl_platform(platform)=="windows"} { + if {$tcl_platform(platform) eq "windows"} { set uri [string map [list PWD [string range [get_pwd] 3 end]] $uri] } else { set uri [string map [list PWD [string range [get_pwd] 1 end]] $uri] diff --git a/test/vacuum-into.test b/test/vacuum-into.test index d559b7fb3..c041ebe7a 100644 --- a/test/vacuum-into.test +++ b/test/vacuum-into.test @@ -111,7 +111,7 @@ do_catchsql_test vacuum-into-420 { # The ability to VACUUM INTO a read-only database db close -if {$tcl_platform(platform)=="windows"} { +if {$tcl_platform(platform) eq "windows"} { file attributes test.db -readonly 1 } else { file attributes test.db -permissions 292 ;# 292 == 0444 @@ -121,7 +121,7 @@ forcedelete test.db2 do_execsql_test vacuum-into-500 { VACUUM INTO 'test.db2'; } -if {$tcl_platform(platform)=="windows"} { +if {$tcl_platform(platform) eq "windows"} { file attributes test.db -readonly 0 } else { file attributes test.db -permissions 420 ;# 420 = 0644 diff --git a/test/vt100-a.sql b/test/vt100-a.sql new file mode 100644 index 000000000..a0d3f46be --- /dev/null +++ b/test/vt100-a.sql @@ -0,0 +1,19 @@ +/* +** Run this script using the "sqlite3" command-line shell +** test test formatting of output text that contains +** vt100 escape sequences. +*/ +.mode box -escape off +CREATE TEMP TABLE t1(a,b,c); +INSERT INTO t1 VALUES + ('one','twotwotwo','thirty-three'), + (unistr('\u001b[91mRED\u001b[0m'),'fourfour','fifty-five'), + ('six','seven','eighty-eight'); +.print With -escape off +SELECT * FROM t1; +.mode box -escape ascii +.print With -escape ascii +SELECT * FROM t1; +.mode box -escape symbol +.print With -escape symbol +SELECT * FROM t1; diff --git a/test/vtabH.test b/test/vtabH.test index cf3dcafd9..07704cefb 100644 --- a/test/vtabH.test +++ b/test/vtabH.test @@ -124,13 +124,13 @@ foreach ::tclvar_set_omit {0 1} { #------------------------------------------------------------------------- # -if {$tcl_platform(platform)=="windows"} { +if {$tcl_platform(platform) eq "windows"} { set drive [string range [pwd] 0 1] set ::env(fstreeDrive) $drive } reset_db register_fs_module db -if {$tcl_platform(platform)!="windows" || \ +if {$tcl_platform(platform) ne "windows" || \ [regexp -nocase -- {^[A-Z]:} $drive]} { do_execsql_test 3.0 { SELECT name FROM fsdir WHERE dir = '.' AND name = 'test.db'; diff --git a/test/wal2.test b/test/wal2.test index 5ef303edc..064bed0b2 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -26,7 +26,7 @@ ifcapable !wal {finish_test ; return } set sqlite_sync_count 0 proc cond_incr_sync_count {adj} { global sqlite_sync_count - if {$::tcl_platform(platform) == "windows"} { + if {$::tcl_platform(os) eq "Windows NT"} { incr sqlite_sync_count $adj } { ifcapable !dirsync { @@ -1038,7 +1038,7 @@ tvfs delete # the new files with the same file-system permissions as the database # file itself. Test this. # -if {$::tcl_platform(platform) == "unix"} { +if {$::tcl_platform(os) ne "Windows NT"} { faultsim_delete_and_reopen # Changed on 2012-02-13: umask is deliberately ignored for -wal files. #set umask [exec /bin/sh -c umask] @@ -1094,7 +1094,7 @@ if {$::tcl_platform(platform) == "unix"} { # database, wal or shm files cannot be opened, or can only be opened # read-only. # -if {$::tcl_platform(platform) == "unix"} { +if {$::tcl_platform(os) ne "Windows NT"} { proc perm {} { set L [list] foreach f {test.db test.db-wal test.db-shm} { diff --git a/test/wal6.test b/test/wal6.test index 9bbc58409..081608cd3 100644 --- a/test/wal6.test +++ b/test/wal6.test @@ -46,7 +46,7 @@ foreach jmode $all_journal_modes { # Under Windows, you'll get an error trying to delete # a file this is already opened. Close the first connection # so the other tests work. -if {$tcl_platform(platform)=="windows"} { +if {$::tcl_platform(os) eq "Windows NT"} { if {$jmode=="persist" || $jmode=="truncate"} { db close } @@ -61,7 +61,7 @@ if {$tcl_platform(platform)=="windows"} { } db2 } {wal 1 2 3 4} -if {$tcl_platform(platform)=="windows"} { +if {$::tcl_platform(os) eq "Windows NT"} { if {$jmode=="persist" || $jmode=="truncate"} { sqlite3 db test.db } diff --git a/test/wal64k.test b/test/wal64k.test index 8ff8e4b77..bacb14328 100644 --- a/test/wal64k.test +++ b/test/wal64k.test @@ -19,10 +19,10 @@ set testprefix wal64k ifcapable !wal {finish_test ; return } -if {$tcl_platform(platform) != "unix"} { +if {[llength [info commands test_syscall]]==0} { finish_test return -} +} db close test_syscall pagesize 65536 diff --git a/test/walblock.test b/test/walblock.test index 23167a883..86a52b3f9 100644 --- a/test/walblock.test +++ b/test/walblock.test @@ -17,7 +17,7 @@ source $testdir/wal_common.tcl finish_test; return; # Feature currently not implemented. ifcapable !wal {finish_test ; return } -if {$::tcl_platform(platform)!="unix"} { finish_test ; return } +if {$::tcl_platform(platform) ne "unix"} { finish_test ; return } set testprefix walblock catch { db close } diff --git a/test/walcrash4.test b/test/walcrash4.test index 80839b39e..43292def4 100644 --- a/test/walcrash4.test +++ b/test/walcrash4.test @@ -38,7 +38,7 @@ faultsim_save_and_close # The error message is different on unix and windows # -if {$::tcl_platform(platform)=="windows"} { +if {$::tcl_platform(platform) eq "windows"} { set msg "child killed: unknown signal" } else { set msg "child process exited abnormally" diff --git a/test/walmode.test b/test/walmode.test index f760823c8..1c3325acf 100644 --- a/test/walmode.test +++ b/test/walmode.test @@ -47,7 +47,7 @@ do_test walmode-1.2 { if {[atomic_batch_write test.db]==0} { set expected_sync_count 3 - if {$::tcl_platform(platform)!="windows"} { + if {$::tcl_platform(os) ne "Windows NT"} { ifcapable dirsync { incr expected_sync_count } diff --git a/test/walnoshm.test b/test/walnoshm.test index d4082178d..f546c29b1 100644 --- a/test/walnoshm.test +++ b/test/walnoshm.test @@ -104,6 +104,7 @@ do_test 2.1.5 { } {exclusive delete a b c d e f g h} do_test 2.2.1 { + db2 close forcecopy test.db test2.db forcecopy test.db-wal test2.db-wal sqlite3 db3 test2.db -vfs tvfsshm diff --git a/test/walro.test b/test/walro.test index cae52db6d..a39b844d9 100644 --- a/test/walro.test +++ b/test/walro.test @@ -19,7 +19,7 @@ set ::testprefix walro # These tests are only going to work on unix. # -if {$::tcl_platform(platform) != "unix"} { +if {$::tcl_platform(os) eq "Windows NT"} { finish_test return } diff --git a/test/walsetlk.test b/test/walsetlk.test index 1e0923822..969bcaf46 100644 --- a/test/walsetlk.test +++ b/test/walsetlk.test @@ -80,6 +80,19 @@ db2 close #------------------------------------------------------------------------- do_multiclient_test tn { + + testvfs tvfs -fullshm 1 + db close + sqlite3 db test.db -vfs tvfs + tvfs script xSleep_callback + tvfs filter xSleep + + set ::sleep_count 0 + proc xSleep_callback {xSleep nMs} { + after [expr $nMs / 1000] + incr ::sleep_count + } + do_test 2.$tn.1 { sql1 { PRAGMA journal_mode = wal; @@ -132,7 +145,6 @@ do_multiclient_test tn { set us [lindex [time { catch {db eval "PRAGMA wal_checkpoint=RESTART"} }] 0] expr $us>1000000 && $us<4000000 } {1} - do_test 2.$tn.9 { sql3 { INSERT INTO t1 VALUES(11, 12); @@ -178,13 +190,50 @@ do_multiclient_test tn { set us [lindex [time { catch {db eval "PRAGMA wal_checkpoint=RESTART"} }] 0] expr $us>1000000 && $us<4000000 } {1} - + + db close + tvfs delete + + # Set bSleep to true if it is expected that the above used xSleep() to + # wait for locks. bSleep is true unless SQLITE_ENABLE_SETLK_TIMEOUT is + # set to 1 and either: + # + # * the OS is windows, or + # * the OS is unix and the tests were run with each connection + # in a separate process. + # + set bSleep 1 + if {$::sqlite_options(setlk_timeout)==1} { + if {$::tcl_platform(platform) eq "windows"} { + set bSleep 0 + } + if {$::tcl_platform(platform) eq "unix"} { + set bSleep [expr $tn==2] + } + } + + do_test 2.$tn.15.$bSleep { + expr $::sleep_count > 0 + } $bSleep } #------------------------------------------------------------------------- reset_db -sqlite3 db2 test.db + +testvfs tvfs -fullshm 1 +tvfs script xSleep_callback +tvfs filter xSleep + +set ::sleep_count 0 +proc xSleep_callback {xSleep nMs} { + after [expr $nMs / 1000] + incr ::sleep_count + breakpoint +} + +sqlite3 db2 test.db -vfs tvfs db2 timeout 1000 + do_execsql_test 3.0 { PRAGMA journal_mode = wal; CREATE TABLE x1(x, y); @@ -192,8 +241,110 @@ do_execsql_test 3.0 { INSERT INTO x1 VALUES(1, 2); } {wal} -do_test 3.1 { +do_execsql_test -db db2 3.1a { + SELECT * FROM x1 +} {} + +do_test 3.1b { list [catch { db2 eval {BEGIN EXCLUSIVE} } msg] $msg } {1 {database is locked}} +# Set bExpect to true if calls to xSleep() are expected. Such calls are +# expected unless this is an SQLITE_ENABLE_SETLK_TIMEOUT=1 build. +set bExpect 1 +if {$tcl_platform(platform) eq "windows" && $::sqlite_options(setlk_timeout)==1} { + set bExpect 0 +} +do_test 3.2 { + expr {$::sleep_count > 0} +} $bExpect +set ::sleep_count 0 + +do_execsql_test 3.3 { + COMMIT; +} + +# Launch a non-blocking testfixture process to write-lock the +# database for 2000 ms. +testfixture_nb done { + sqlite3 db test.db + db eval { + BEGIN EXCLUSIVE; + INSERT INTO x1 VALUES(3, 4); + } + after 2000 + db eval { + COMMIT + } +} + +after 500 {set ok 1} +vwait ok + +db2 timeout 5000 +do_test 3.4 { + set t [lindex [time { db2 eval { BEGIN EXCLUSIVE } }] 0] + expr ($t>1000000) +} {1} + +# Set bExpect to true if calls to xSleep() are expected. Such calls are +# expected unless this is an SQLITE_ENABLE_SETLK_TIMEOUT=1 build. +set bExpect 1 +if {$::sqlite_options(setlk_timeout)==1} { + set bExpect 0 +} +do_test 3.5 { + expr {$::sleep_count > 0} +} $bExpect + +do_execsql_test -db db2 3.6 { + INSERT INTO x1 VALUES(5, 6); + COMMIT; + SELECT * FROM x1; +} {1 2 3 4 5 6} + +# Launch a non-blocking testfixture process to write-lock the +# database for 2000 ms. +testfixture_nb done { + sqlite3 db test.db + db eval { + BEGIN EXCLUSIVE; + INSERT INTO x1 VALUES(7, 8); + } + after 2000 + db eval { + COMMIT + } + db close +} + +after 500 {set ok 1} +vwait ok + +db2 timeout 0x7FFFFFFF +do_test 3.7 { + set t [lindex [time { db2 eval { BEGIN EXCLUSIVE } }] 0] + expr ($t>1000000) +} {1} + +# Set bExpect to true if calls to xSleep() are expected. Such calls are +# expected unless this is an SQLITE_ENABLE_SETLK_TIMEOUT=1 build. +set bExpect 1 +if {$::sqlite_options(setlk_timeout)==1} { + set bExpect 0 +} +do_test 3.8 { + expr {$::sleep_count > 0} +} $bExpect + +do_execsql_test -db db2 3.9 { + INSERT INTO x1 VALUES(9, 10); + COMMIT; + SELECT * FROM x1; +} {1 2 3 4 5 6 7 8 9 10} + +db2 close +tvfs delete + finish_test + diff --git a/test/walsetlk2.test b/test/walsetlk2.test new file mode 100644 index 000000000..92630b3fd --- /dev/null +++ b/test/walsetlk2.test @@ -0,0 +1,267 @@ +# 2025 Jan 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# TESTRUNNER: slow +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set testprefix walsetlk2 + +ifcapable !wal {finish_test ; return } +ifcapable !setlk_timeout {finish_test ; return } + +#------------------------------------------------------------------------- +# Check that xShmLock calls are as expected for write transactions in +# setlk mode. +# +reset_db + +do_execsql_test 1.0 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 2, 3); +} {wal} +db close + +testvfs tvfs +tvfs script xShmLock_callback +tvfs filter xShmLock + +set ::xshmlock [list] +proc xShmLock_callback {method path name detail} { + lappend ::xshmlock $detail +} + +sqlite3 db test.db -vfs tvfs +db timeout 1000 + +do_execsql_test 1.1 { + SELECT * FROM t1 +} {1 2 3} + +do_execsql_test 1.2 { + INSERT INTO t1 VALUES(4, 5, 6); +} + +set ::xshmlock [list] +do_execsql_test 1.3 { + INSERT INTO t1 VALUES(7, 8, 9); +} + +do_test 1.4 { + set ::xshmlock +} [list \ + {0 1 lock exclusive} \ + {4 1 lock exclusive} {4 1 unlock exclusive} \ + {4 1 lock shared} \ + {0 1 unlock exclusive} \ + {4 1 unlock shared} +] + +do_execsql_test 1.5.1 { SELECT * FROM t1 } {1 2 3 4 5 6 7 8 9} +set ::xshmlock [list] +do_execsql_test 1.5.2 { + INSERT INTO t1 VALUES(10, 11, 12); +} +do_test 1.5.3 { + set ::xshmlock +} [list \ + {0 1 lock exclusive} \ + {4 1 lock shared} \ + {0 1 unlock exclusive} \ + {4 1 unlock shared} +] + +db close +tvfs delete + +#------------------------------------------------------------------------- +# Check that if sqlite3_setlk_timeout() is used, blocking locks timeout +# but other operations do not use the retry mechanism. +# +reset_db + +do_execsql_test 2.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2), (3, 4); +} + +sqlite3_setlk_timeout db 2000 + +# Launch a non-blocking testfixture process to write-lock the +# database for 2000 ms. +testfixture_nb done { + sqlite3 db test.db + db eval { + BEGIN EXCLUSIVE; + INSERT INTO t1 VALUES(5, 6); + } + after 2000 + db eval { + COMMIT + } + db close +} + +after 500 {set ok 1} +vwait ok + +do_catchsql_test 2.1 { + INSERT INTO t1 VALUES(7, 8); +} {1 {database is locked}} + +sqlite3_busy_timeout db 2000 + +do_catchsql_test 2.2 { + INSERT INTO t1 VALUES(7, 8); +} {0 {}} + +do_execsql_test 2.3 { + SELECT * FROM t1 +} {1 2 3 4 5 6 7 8} + +do_execsql_test 2.4 { + PRAGMA journal_mode = wal; +} {wal} + +db close +sqlite3 db test.db + +do_execsql_test 2.5 { + INSERT INTO t1 VALUES(9, 10); +} + +if {$::sqlite_options(setlk_timeout)==1} { + +sqlite3_setlk_timeout db 2000 + +# Launch a non-blocking testfixture process to write-lock the +# database for 2000 ms. +testfixture_nb done { + sqlite3 db test.db + db eval { + BEGIN EXCLUSIVE; + INSERT INTO t1 VALUES(11, 12); + } + after 2000 + db eval { + COMMIT + } + db close +} + +after 500 {set ok 1} +vwait ok + +do_catchsql_test 2.6 { + INSERT INTO t1 VALUES(13, 14); +} {0 {}} + +do_execsql_test 2.7 { + SELECT * FROM t1 +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14} + +} + + +#------------------------------------------------------------------------- +# Check that if sqlite3_setlk_timeout(-1) is called, blocking locks are +# enabled and last for a few seconds at least. Difficult to test that they +# really do block indefinitely. +# +reset_db + +if {$::sqlite_options(setlk_timeout)==1} { +do_execsql_test 3.0 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 'one'), (3, 'three'); +} {wal} + +sqlite3_setlk_timeout db -1 + +# Launch a non-blocking testfixture process to write-lock the +# database for 2000 ms. +testfixture_nb done { + sqlite3 db test.db + db eval { + BEGIN EXCLUSIVE; + INSERT INTO t1 VALUES(5, 'five'); + } + after 2000 + db eval { + COMMIT + } + db close +} + +after 500 {set ok 1} +vwait ok + +breakpoint +do_catchsql_test 3.1 { + INSERT INTO t1 VALUES(7, 'seven'); +} {0 {}} + +# Launch another non-blocking testfixture process to write-lock the +# database for 2000 ms. +testfixture_nb done { + sqlite3 db test.db + db eval { + BEGIN EXCLUSIVE; + INSERT INTO t1 VALUES(9, 'nine'); + } + after 2000 + db eval { + COMMIT + } + db close +} + +after 500 {set ok 1} +vwait ok + +do_catchsql_test 3.2 { + INSERT INTO t1 VALUES(9, 'ten'); +} {1 {UNIQUE constraint failed: t1.a}} + +do_execsql_test 3.3 { + SELECT * FROM t1 +} {1 one 3 three 5 five 7 seven 9 nine} + +db close + +# Launch another non-blocking testfixture process to write-lock the +# database for 2000 ms. +testfixture_nb done { + sqlite3 db test.db + db eval { + BEGIN EXCLUSIVE; + INSERT INTO t1 VALUES(11, 'eleven'); + } + after 2000 + db eval { + COMMIT + } +} + +sqlite3 db test.db +sqlite3_setlk_timeout db -1 +do_catchsql_test 3.4 { + INSERT INTO t1 VALUES(13, 'thirteen'); +} {0 {}} + +} + +finish_test + diff --git a/test/walsetlk3.test b/test/walsetlk3.test new file mode 100644 index 000000000..b091b183d --- /dev/null +++ b/test/walsetlk3.test @@ -0,0 +1,135 @@ +# 2020 May 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# TESTRUNNER: slow +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set testprefix walsetlk3 + +ifcapable !wal {finish_test ; return } +ifcapable !setlk_timeout {finish_test ; return } + +do_execsql_test 1.0 { + CREATE TABLE t1(x, y); + PRAGMA journal_mode = wal; + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); +} {wal} + +db close + +proc sql_block_on_close {sql} { + testfixture_nb done [string map [list %SQL% $sql] { + testvfs tvfs + tvfs script xWrite + tvfs filter xWrite + + set ::delay_done 0 + proc xWrite {method fname args} { + if {[file tail $fname]=="test.db" && $::delay_done==0} { + after 3000 + set ::delay_done 1 + } + return 0 + } + + sqlite3 db test.db -vfs tvfs + db eval {%SQL%} + db close + }] +} + +# Start a second process that writes to the db, then blocks within the +# [db close] holding an EXCLUSIVE on the db in order to checkpoint and +# delete the wal file. Then try to read the db. +# +# Without the SQLITE_SETLK_BLOCK_ON_CONNECT flag, this should fail with +# SQLITE_BUSY. +# +sql_block_on_close { + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t1 VALUES(7, 8); +} +after 500 {set ok 1} +vwait ok +sqlite3 db test.db +sqlite3_setlk_timeout db 2000 +do_catchsql_test 1.1 { + SELECT * FROM t1 +} {1 {database is locked}} + +vwait ::done + +# But with SQLITE_SETLK_BLOCK_ON_CONNECT flag, it should succeed. +# +sql_block_on_close { + INSERT INTO t1 VALUES(9, 10); + INSERT INTO t1 VALUES(11, 12); +} +after 500 {set ok 1} +vwait ok +sqlite3 db test.db +sqlite3_setlk_timeout -block db 2000 +do_catchsql_test 1.2 { + SELECT * FROM t1 +} {0 {1 2 3 4 5 6 7 8 9 10 11 12}} + +vwait ::done + +#------------------------------------------------------------------------- +# Check that the SQLITE_SETLK_BLOCK_ON_CONNECT does not cause connections +# to block when taking a SHARED lock on a rollback mode database. +# +reset_db +do_execsql_test 2.1 { + CREATE TABLE x1(a); + INSERT INTO x1 VALUES(1), (2), (3); +} + +proc sql_block_on_write {sql} { + testfixture_nb done [string map [list %SQL% $sql] { + sqlite3 db test.db + db eval "BEGIN EXCLUSIVE" + db eval {%SQL%} + after 3000 + db eval COMMIT + db close + }] +} + +db close +sql_block_on_write { + INSERT INTO x1 VALUES(4); +} + +after 500 {set ok 1} +vwait ok + +sqlite3 db test.db +sqlite3_setlk_timeout -block db 2000 + +do_catchsql_test 2.2 { + SELECT * FROM x1 +} {1 {database is locked}} + +vwait ::done +after 500 {set ok 1} +vwait ok + +do_catchsql_test 2.3 { + SELECT * FROM x1 +} {0 {1 2 3 4}} + +finish_test + diff --git a/test/walsetlk_recover.test b/test/walsetlk_recover.test new file mode 100644 index 000000000..1daece747 --- /dev/null +++ b/test/walsetlk_recover.test @@ -0,0 +1,104 @@ +# 2025 May 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# TESTRUNNER: slow +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set testprefix walsetlk_recover + +ifcapable !wal {finish_test ; return } +# ifcapable !setlk_timeout {finish_test ; return } + +do_execsql_test 1.0 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); +} {wal} + +db_save_and_close +db_restore + +testfixture_nb myvar { + + testvfs tvfs -fullshm 1 + sqlite3 db test.db -vfs tvfs + tvfs script vfs_callback + tvfs filter xRead + + set ::done 0 + proc vfs_callback {method file args} { + if {$::done==0 && [string match *wal $file]} { + after 4000 + set ::done 1 + } + return "SQLITE_OK" + } + + db eval { + SELECT * FROM t1 + } + + db close +} + +# Give the [testfixture_nb] command time to start +after 1000 {set xyz 1} +vwait xyz + +testvfs tvfs -fullshm 1 +sqlite3 db test.db -vfs tvfs + +tvfs script sleep_callback +tvfs filter xSleep +set ::sleep_count 0 +proc sleep_callback {args} { + incr ::sleep_count +} + +sqlite3 db test.db -vfs tvfs +db timeout 500 +set tm [lindex [time { + catch { + db eval {SELECT * FROM t1} + } msg +}] 0] + +do_test 1.2 { set ::msg } {database is locked} +do_test 1.3.($::tm) { expr $::tm>400000 && $::tm<2000000 } 1 + +vwait myvar + +do_execsql_test 1.4 { + SELECT * FROM t1 +} {1 2 3 4 5 6} + +db close +tvfs delete + +# All SQLite builds should pass the tests above. SQLITE_ENABLE_SETLK_TIMEOUT=1 +# builds do so without calling the VFS xSleep method. +if {$::sqlite_options(setlk_timeout)==1} { + do_test 1.5.1 { + set ::sleep_count + } 0 +} else { + do_test 1.5.2 { + expr $::sleep_count>0 + } 1 +} + +finish_test + diff --git a/test/walsetlk_snapshot.test b/test/walsetlk_snapshot.test new file mode 100644 index 000000000..e05ad69cc --- /dev/null +++ b/test/walsetlk_snapshot.test @@ -0,0 +1,109 @@ +# 2025 May 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# TESTRUNNER: slow +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set testprefix walsetlk_snapshot + +ifcapable !wal {finish_test ; return } +ifcapable !snapshot {finish_test; return} + +db close +testvfs tvfs -fullshm 1 +sqlite3 db test.db -vfs tvfs +tvfs script sleep_callback +tvfs filter xSleep +set ::sleep_count 0 +proc sleep_callback {args} { + incr ::sleep_count +} + +do_execsql_test 1.0 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); +} {wal} + +do_test 1.1 { + db eval BEGIN + set ::snap [sqlite3_snapshot_get db main] + db eval { + INSERT INTO t1 VALUES(7, 8); + COMMIT; + } +} {} + +testfixture_nb myvar { + + testvfs tvfs -fullshm 1 + sqlite3 db test.db -vfs tvfs + tvfs script vfs_callback + tvfs filter {xWrite} + + set ::done 0 + proc vfs_callback {args} { + if {$::done==0} { + after 4000 + set ::done 1 + } + return "SQLITE_OK" + } + + db eval { + PRAGMA wal_checkpoint; + } + + db close +} + +# Give the [testfixture_nb] command time to start +after 1000 {set xyz 1} +vwait xyz + +db timeout 500 +set tm [lindex [time { + catch { + db eval BEGIN + sqlite3_snapshot_open db main $::snap + } msg +}] 0] + +do_test 1.2 { set ::msg } {SQLITE_BUSY} +do_test 1.3.($::tm) { expr $::tm<2000000 } 1 + +do_execsql_test 1.4 { + SELECT * FROM t1 +} {1 2 3 4 5 6 7 8} + +sqlite3_snapshot_free $::snap + +vwait myvar + +# All SQLite builds should pass the tests above. SQLITE_ENABLE_SETLK_TIMEOUT=1 +# builds do so without calling the VFS xSleep method. +if {$::sqlite_options(setlk_timeout)==1} { + do_test 1.5.1 { + set ::sleep_count + } 0 +} else { + do_test 1.5.2 { + expr $::sleep_count>0 + } 1 +} + +finish_test + diff --git a/test/where.test b/test/where.test index 0a8cfd572..c377006fb 100644 --- a/test/where.test +++ b/test/where.test @@ -1388,7 +1388,7 @@ do_execsql_test where-19.0 { SELECT t191.rowid FROM t192, t191 WHERE (a=y OR b=y) AND x=?1; } {/.* sqlite_autoindex_t191_1 .* sqlite_autoindex_t191_2 .*/} -# 2018-04-24 ticket [https://www.sqlite.org/src/info/4ba5abf65c5b0f9a] +# 2018-04-24 ticket [https://sqlite.org/src/info/4ba5abf65c5b0f9a] # Index on expressions leads to an incorrect answer for a LEFT JOIN # do_execsql_test where-20.0 { @@ -1422,7 +1422,7 @@ do_execsql_test where-21.1 { 4 0 1 } -# 2018-11-05: ticket [https://www.sqlite.org/src/tktview/65eb38f6e46de8c75e188a] +# 2018-11-05: ticket [https://sqlite.org/src/tktview/65eb38f6e46de8c75e188a] # Incorrect result in LEFT JOIN when STAT4 is enabled. # sqlite3 db :memory: @@ -1435,7 +1435,7 @@ do_execsql_test where-22.1 { } {5} # 20190-02-22: A bug introduced by checkin -# https://www.sqlite.org/src/info/fa792714ae62fa98. +# https://sqlite.org/src/info/fa792714ae62fa98. # do_execsql_test where-23.0 { DROP TABLE IF EXISTS t1; @@ -1547,7 +1547,7 @@ do_catchsql_test where-25.5 { ON CONFLICT(c) DO UPDATE SET b=NULL } {1 {corrupt database}} -# 2019-08-21 Ticket https://www.sqlite.org/src/info/d9f584e936c7a8d0 +# 2019-08-21 Ticket https://sqlite.org/src/info/d9f584e936c7a8d0 # db close sqlite3 db :memory: diff --git a/test/where2.test b/test/where2.test index 7a7e9b92e..a38f6e54b 100644 --- a/test/where2.test +++ b/test/where2.test @@ -767,7 +767,7 @@ do_execsql_test where2-13.1 { SELECT * FROM t13 WHERE (1=2 AND a=3) OR a=4; } {4 5} -# https://www.sqlite.org/src/info/5e3c886796e5512e (2016-03-09) +# https://sqlite.org/src/info/5e3c886796e5512e (2016-03-09) # Correlated subquery on the RHS of an IN operator # do_execsql_test where2-14.1 { diff --git a/test/whereG.test b/test/whereG.test index c15405823..6ee863481 100644 --- a/test/whereG.test +++ b/test/whereG.test @@ -237,7 +237,7 @@ do_eqp_test 5.3.3 { } {SCAN t1} # 2015-06-18 -# Ticket [https://www.sqlite.org/see/tktview/472f0742a1868fb58862bc588ed70] +# Ticket [https://sqlite.org/see/tktview/472f0742a1868fb58862bc588ed70] # do_execsql_test 6.0 { DROP TABLE IF EXISTS t1; @@ -273,7 +273,7 @@ do_execsql_test 7.3 { } {1 3 1 4 9 3 9 4} # 2019-08-22 -# Ticket https://www.sqlite.org/src/info/7e07a3dbf5a8cd26 +# Ticket https://sqlite.org/src/info/7e07a3dbf5a8cd26 # do_execsql_test 8.1 { DROP TABLE IF EXISTS t0; diff --git a/test/whereL.test b/test/whereL.test index 2e9ae219e..ffbae02b8 100644 --- a/test/whereL.test +++ b/test/whereL.test @@ -254,4 +254,44 @@ do_execsql_test 810 { SELECT * FROM v0 LEFT JOIN t0 ON c31} { append LDFLAGS $OPTS } + if {$tcl_platform(os) eq "Windows NT"} { + set OUT cyg + } else { + set OUT lib + } if {$TCLMAJOR>8} { - set OUT libtcl9sqlite$VERSION.$SUFFIX + set OUT ${OUT}tcl9sqlite$VERSION.$SUFFIX } else { - set OUT libsqlite$VERSION.$SUFFIX + set OUT ${OUT}sqlite$VERSION.$SUFFIX } set @ $OUT; # Workaround for https://sqlite.org/forum/forumpost/0683a49cb02f31a1 # in which Gentoo edits their tclConfig.sh to include an soname @@ -295,7 +300,7 @@ package ifneeded sqlite3 $VERSION \\ # Generate and execute the command with which to do the compilation. # - set cmd "$CMD tclsqlite3.c -o $OUT $LIBS" + set cmd "$CMD -DUSE_TCL_STUBS tclsqlite3.c -o $OUT $LIBS" puts $cmd file delete -force $OUT catch {exec {*}$cmd} errmsg diff --git a/tool/genfkey.README b/tool/genfkey.README index 57cdff87f..560994504 100644 --- a/tool/genfkey.README +++ b/tool/genfkey.README @@ -63,7 +63,7 @@ CAPABILITIES LIMITATIONS - Apart from those limitiations described above: + Apart from those limitations described above: * Implicit mapping to composite primary keys is not supported. If a parent table has a composite primary key, then any child table @@ -108,7 +108,7 @@ USAGE If errors are found and the --ignore-errors option is passed, then no error messages are printed. No "CREATE TRIGGER" statements are generated - for foriegn-key definitions that contained errors, they are silently + for foreign-key definitions that contained errors, they are silently ignored by subsequent processing. All triggers generated by this command have names that match the pattern @@ -129,7 +129,7 @@ USAGE they will be printed to stdout, but this can be configured using other dot-commands (i.e. ".output"). - The simplest way to activate the foriegn key definitions in a database + The simplest way to activate the foreign key definitions in a database is simply to open it using the shell tool and enter the command ".genfkey --exec": diff --git a/tool/kvtest-speed.sh b/tool/kvtest-speed.sh deleted file mode 100644 index 5f2c8345b..000000000 --- a/tool/kvtest-speed.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# -# A script for running speed tests using kvtest. -# -# The test database must be set up first. Recommended -# command-line: -# -# ./kvtest init kvtest.db --count 100K --size 12K --variance 5K - -if test "$1" = "" -then - echo "Usage: $0 OUTPUTFILE [OPTIONS]" - exit -fi -NAME=$1 -shift -OPTS="-DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DIRECT_OVERFLOW_READ -DUSE_PREAD" -KVARGS="--count 100K --stats" -gcc -g -Os -I. $OPTS $* kvtest.c sqlite3.c -o kvtest - -# First run using SQL -rm cachegrind.out.[1-9][0-9]* -valgrind --tool=cachegrind ./kvtest run kvtest.db $KVARGS 2>&1 | tee summary-kvtest-$NAME.txt -mv cachegrind.out.[1-9][0-9]* cachegrind.out.sql-$NAME -cg_anno.tcl cachegrind.out.sql-$NAME >cout-kvtest-sql-$NAME.txt - -# Second run using the sqlite3_blob object -valgrind --tool=cachegrind ./kvtest run kvtest.db $KVARGS --blob-api 2>&1 | tee -a summary-kvtest-$NAME.txt -mv cachegrind.out.[1-9][0-9]* cachegrind.out.$NAME -cg_anno.tcl cachegrind.out.$NAME >cout-kvtest-$NAME.txt - -# Diff the sqlite3_blob API analysis for non-trunk runs. -if test "$NAME" != "trunk"; then - fossil test-diff --tk cout-kvtest-trunk.txt cout-kvtest-$NAME.txt & -fi diff --git a/tool/lemon.c b/tool/lemon.c index 5747520b6..795c3a216 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -3695,7 +3695,7 @@ PRIVATE int compute_action(struct lemon *lemp, struct action *ap) switch( ap->type ){ case SHIFT: act = ap->x.stp->statenum; break; case SHIFTREDUCE: { - /* Since a SHIFT is inherient after a prior REDUCE, convert any + /* Since a SHIFT is inherent after a prior REDUCE, convert any ** SHIFTREDUCE action with a nonterminal on the LHS into a simple ** REDUCE action: */ if( ap->sp->index>=lemp->nterminal @@ -4035,10 +4035,10 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){ } } if( lhsdirect ){ - sprintf(zLhs, "yymsp[%d].minor.yy%d",1-rp->nrhs,rp->lhs->dtnum); + lemon_sprintf(zLhs, "yymsp[%d].minor.yy%d",1-rp->nrhs,rp->lhs->dtnum); }else{ rc = 1; - sprintf(zLhs, "yylhsminor.yy%d",rp->lhs->dtnum); + lemon_sprintf(zLhs, "yylhsminor.yy%d",rp->lhs->dtnum); } append_str(0,0,0,0); diff --git a/tool/lempar.c b/tool/lempar.c index 851a0e2e5..74314efea 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -13,7 +13,7 @@ ** ** The "lemon" program processes an LALR(1) input grammar file, then uses ** this template to construct a parser. The "lemon" program inserts text -** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the +** at each "%%" line. Also, any "P-a-r-s-e" identifier prefix (without the ** interstitial "-" characters) contained in this template is changed into ** the value of the %name directive from the grammar. Otherwise, the content ** of this template is copied straight through into the generate parser diff --git a/tool/mkautoconfamal.sh b/tool/mkautoconfamal.sh index c26ac8c73..b750593c9 100644 --- a/tool/mkautoconfamal.sh +++ b/tool/mkautoconfamal.sh @@ -25,18 +25,6 @@ VERSION=`cat $TOP/VERSION` HASH=`cut -c1-10 $TOP/manifest.uuid` DATETIME=`grep '^D' $TOP/manifest | tr -c -d '[0-9]' | cut -c1-12` -# Inject the current version into the TEA autoconf file. -# -sed -e "s/@VERSION@/$VERSION/" \ - < $TOP/autoconf/tea/configure.ac.in \ - > $TOP/autoconf/tea/configure.ac -# And then verify that that worked... -# -if grep $VERSION $TOP/autoconf/tea/configure.ac > /dev/null -then echo "TEA version number ok" -else echo "TEA version number mismatch. Should be $VERSION"; exit 1 -fi - # If this script is given an argument of --snapshot, then generate a # snapshot tarball named for the current checkout SHA hash, rather than # the version number. @@ -75,7 +63,7 @@ cp $TOP/main.mk $TMPSPACE cd $TMPSPACE # Clean up emacs-generated backup files from the target -rm -f ./autosetup/*~ +rm -f ./autosetup/*~ ./autosetup/teaish/*~ rm -f ./*~ #if true; then @@ -95,12 +83,9 @@ cat < tea/generic/tclsqlite3.c EOF cat $TOP/src/tclsqlite.c >> tea/generic/tclsqlite3.c -cd tea -rm -f configure.ac.in -autoconf -rm -rf autom4te.cache +find . -type f -name '*~' -exec rm -f \{} \; +find . -type f -name '#*#' -exec rm -f \{} \; -cd ../ ./configure && make dist tar xzf sqlite-$VERSION.tar.gz mv sqlite-$VERSION $TARBALLNAME diff --git a/tool/mkccode.tcl b/tool/mkccode.tcl index ecafbdadb..8b4fae82c 100755 --- a/tool/mkccode.tcl +++ b/tool/mkccode.tcl @@ -1,17 +1,16 @@ -#!/usr/bin/tclsh +#!/bin/env tclsh # -# Use this script to build C-language source code for a program that uses -# tclsqlite.c together with custom TCL scripts and/or C extensions for -# either SQLite or TCL. +# This script is used to amalgamate C source code files into a single +# unit. # # Usage example: # # tclsh mkccode.tcl -DENABLE_FEATURE_XYZ demoapp.c.in >demoapp.c # # The demoapp.c.in file contains a mixture of C code, TCL script, and -# processing directives used by mktclsqliteprog.tcl to build the final C-code -# output file. Most lines of demoapp.c.in are copied straight through into -# the output. The following control directives are recognized: +# processing directives used by mkccode.tcl to build the final C-code +# output file. Most lines of demoapp.c.in are copied straight through +# into the output. The following control directives are recognized: # # BEGIN_STRING # diff --git a/tool/mkctimec.tcl b/tool/mkctimec.tcl index 69d25c678..1c59131b0 100755 --- a/tool/mkctimec.tcl +++ b/tool/mkctimec.tcl @@ -4,13 +4,13 @@ # # const char **azCompileOpt[] # -# definition used in src/ctime.c, run this script from -# the checkout root. It generates src/ctime.c . +# definition used in ctime.c, run this script from +# the checkout root. It generates ctime.c . # -# Results are normally written into src/ctime.c. But if an argument is +# Results are normally written into ctime.c. But if an argument is # provided, results are written there instead. Examples: # -# tclsh tool/mkctimec.tcl ;# <-- results to src/ctime.c +# tclsh tool/mkctimec.tcl ;# <-- ctime.c # # tclsh tool/mkctimec.tcl /dev/tty ;# <-- results to the terminal # @@ -108,6 +108,7 @@ set boolean_defnil_options { SQLITE_ALLOW_ROWID_IN_VIEW SQLITE_ALLOW_URI_AUTHORITY SQLITE_BUG_COMPATIBLE_20160819 + SQLITE_BUG_COMPATIBLE_20250510 SQLITE_CASE_SENSITIVE_LIKE SQLITE_CHECK_PAGES SQLITE_COVERAGE_TEST @@ -167,6 +168,7 @@ set boolean_defnil_options { SQLITE_ENABLE_RBU SQLITE_ENABLE_RTREE SQLITE_ENABLE_SESSION + SQLITE_ENABLE_SETLK_TIMEOUT SQLITE_ENABLE_SNAPSHOT SQLITE_ENABLE_SORTER_REFERENCES SQLITE_ENABLE_SQLLOG @@ -319,6 +321,7 @@ set value_options { SQLITE_ENABLE_LOCKING_STYLE SQLITE_EXTRA_AUTOEXT SQLITE_EXTRA_INIT + SQLITE_EXTRA_INIT_MUTEXED SQLITE_EXTRA_SHUTDOWN SQLITE_FTS3_MAX_EXPR_DEPTH SQLITE_INTEGRITY_CHECK_ERROR_MAX @@ -440,7 +443,7 @@ foreach v $value2_options { if {$argc>0} { set destfile [lindex $argv 0] } else { - set destfile "[file dir [file dir [file normal $argv0]]]/src/ctime.c" + set destfile ctime.c puts "Overwriting $destfile..." } diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index 8a0b5c6fa..70988cf82 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -4,16 +4,16 @@ # # To add new pragmas, first add the name and other relevant attributes # of the pragma to the "pragma_def" object below. Then run this script -# to generate the ../src/pragma.h header file that contains macros and +# to generate the pragma.h header file that contains macros and # the lookup table needed for pragma name lookup in the pragma.c module. # Then add the extra "case PragTyp_XXXXX:" and subsequent code for the # new pragma in ../src/pragma.c. # -# The results are normally written into the ../src/pragma.h file. However, +# The results are normally written into the pragma.h file. However, # if an alternative output file name is provided as an argument, then # results are written into the alternative. For example: # -# tclsh tool/mkpragmatab.tcl ;# <--- Results to src/pragma.h +# tclsh tool/mkpragmatab.tcl ;# <--- Results to pragma.h # # tclsh tool/mkpragmatab.tcl /dev/tty ;# <-- results to terminal # @@ -413,7 +413,7 @@ set pragma_def { if {$argc>0} { set destfile [lindex $argv 0] } else { - set destfile "[file dir [file dir [file normal $argv0]]]/src/pragma.h" + set destfile "pragma.h" puts "Overwriting $destfile with new pragma table..." } set fd [open $destfile wb] diff --git a/tool/mkspeedsql.tcl b/tool/mkspeedsql.tcl deleted file mode 100644 index 04bafc04c..000000000 --- a/tool/mkspeedsql.tcl +++ /dev/null @@ -1,237 +0,0 @@ -# 2008 October 9 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#************************************************************************* -# This file generates SQL text used for performance testing. -# -# $Id: mkspeedsql.tcl,v 1.1 2008/10/09 17:57:34 drh Exp $ -# - -# Set a uniform random seed -expr srand(0) - -# The number_name procedure below converts its argment (an integer) -# into a string which is the English-language name for that number. -# -# Example: -# -# puts [number_name 123] -> "one hundred twenty three" -# -set ones {zero one two three four five six seven eight nine - ten eleven twelve thirteen fourteen fifteen sixteen seventeen - eighteen nineteen} -set tens {{} ten twenty thirty forty fifty sixty seventy eighty ninety} -proc number_name {n} { - if {$n>=1000} { - set txt "[number_name [expr {$n/1000}]] thousand" - set n [expr {$n%1000}] - } else { - set txt {} - } - if {$n>=100} { - append txt " [lindex $::ones [expr {$n/100}]] hundred" - set n [expr {$n%100}] - } - if {$n>=20} { - append txt " [lindex $::tens [expr {$n/10}]]" - set n [expr {$n%10}] - } - if {$n>0} { - append txt " [lindex $::ones $n]" - } - set txt [string trim $txt] - if {$txt==""} {set txt zero} - return $txt -} - -# Create a database schema. -# -puts { - PRAGMA page_size=1024; - PRAGMA cache_size=8192; - PRAGMA locking_mode=EXCLUSIVE; - CREATE TABLE t1(a INTEGER, b INTEGER, c TEXT); - CREATE TABLE t2(a INTEGER, b INTEGER, c TEXT); - CREATE INDEX i2a ON t2(a); - CREATE INDEX i2b ON t2(b); - SELECT name FROM sqlite_master ORDER BY 1; -} - - -# 50000 INSERTs on an unindexed table -# -set t1c_list {} -puts {BEGIN;} -for {set i 1} {$i<=50000} {incr i} { - set r [expr {int(rand()*500000)}] - set x [number_name $r] - lappend t1c_list $x - puts "INSERT INTO t1 VALUES($i,$r,'$x');" -} -puts {COMMIT;} - -# 50000 INSERTs on an indexed table -# -puts {BEGIN;} -for {set i 1} {$i<=50000} {incr i} { - set r [expr {int(rand()*500000)}] - puts "INSERT INTO t2 VALUES($i,$r,'[number_name $r]');" -} -puts {COMMIT;} - - -# 50 SELECTs on an integer comparison. There is no index so -# a full table scan is required. -# -for {set i 0} {$i<50} {incr i} { - set lwr [expr {$i*100}] - set upr [expr {($i+10)*100}] - puts "SELECT count(*), avg(b) FROM t1 WHERE b>=$lwr AND b<$upr;" -} - -# 50 SELECTs on an LIKE comparison. There is no index so a full -# table scan is required. -# -for {set i 0} {$i<50} {incr i} { - puts "SELECT count(*), avg(b) FROM t1 WHERE c LIKE '%[number_name $i]%';" -} - -# Create indices -# -puts {BEGIN;} -puts { - CREATE INDEX i1a ON t1(a); - CREATE INDEX i1b ON t1(b); - CREATE INDEX i1c ON t1(c); -} -puts {COMMIT;} - -# 5000 SELECTs on an integer comparison where the integer is -# indexed. -# -set sql {} -for {set i 0} {$i<5000} {incr i} { - set lwr [expr {$i*100}] - set upr [expr {($i+10)*100}] - puts "SELECT count(*), avg(b) FROM t1 WHERE b>=$lwr AND b<$upr;" -} - -# 100000 random SELECTs against rowid. -# -for {set i 1} {$i<=100000} {incr i} { - set id [expr {int(rand()*50000)+1}] - puts "SELECT c FROM t1 WHERE rowid=$id;" -} - -# 100000 random SELECTs against a unique indexed column. -# -for {set i 1} {$i<=100000} {incr i} { - set id [expr {int(rand()*50000)+1}] - puts "SELECT c FROM t1 WHERE a=$id;" -} - -# 50000 random SELECTs against an indexed column text column -# -set nt1c [llength $t1c_list] -for {set i 0} {$i<50000} {incr i} { - set r [expr {int(rand()*$nt1c)}] - set c [lindex $t1c_list $i] - puts "SELECT c FROM t1 WHERE c='$c';" -} - - -# Vacuum -puts {VACUUM;} - -# 5000 updates of ranges where the field being compared is indexed. -# -puts {BEGIN;} -for {set i 0} {$i<5000} {incr i} { - set lwr [expr {$i*2}] - set upr [expr {($i+1)*2}] - puts "UPDATE t1 SET b=b*2 WHERE a>=$lwr AND a<$upr;" -} -puts {COMMIT;} - -# 50000 single-row updates. An index is used to find the row quickly. -# -puts {BEGIN;} -for {set i 0} {$i<50000} {incr i} { - set r [expr {int(rand()*500000)}] - puts "UPDATE t1 SET b=$r WHERE a=$i;" -} -puts {COMMIT;} - -# 1 big text update that touches every row in the table. -# -puts { - UPDATE t1 SET c=a; -} - -# Many individual text updates. Each row in the table is -# touched through an index. -# -puts {BEGIN;} -for {set i 1} {$i<=50000} {incr i} { - set r [expr {int(rand()*500000)}] - puts "UPDATE t1 SET c='[number_name $r]' WHERE a=$i;" -} -puts {COMMIT;} - -# Delete all content in a table. -# -puts {DELETE FROM t1;} - -# Copy one table into another -# -puts {INSERT INTO t1 SELECT * FROM t2;} - -# Delete all content in a table, one row at a time. -# -puts {DELETE FROM t1 WHERE 1;} - -# Refill the table yet again -# -puts {INSERT INTO t1 SELECT * FROM t2;} - -# Drop the table and recreate it without its indices. -# -puts {BEGIN;} -puts { - DROP TABLE t1; - CREATE TABLE t1(a INTEGER, b INTEGER, c TEXT); -} -puts {COMMIT;} - -# Refill the table yet again. This copy should be faster because -# there are no indices to deal with. -# -puts {INSERT INTO t1 SELECT * FROM t2;} - -# Select 20000 rows from the table at random. -# -puts { - SELECT rowid FROM t1 ORDER BY random() LIMIT 20000; -} - -# Delete 20000 random rows from the table. -# -puts { - DELETE FROM t1 WHERE rowid IN - (SELECT rowid FROM t1 ORDER BY random() LIMIT 20000); -} -puts {SELECT count(*) FROM t1;} - -# Delete 20000 more rows at random from the table. -# -puts { - DELETE FROM t1 WHERE rowid IN - (SELECT rowid FROM t1 ORDER BY random() LIMIT 20000); -} -puts {SELECT count(*) FROM t1;} diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 1d0f89236..7b6f57e42 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -111,7 +111,7 @@ puts $out [subst \ ** separate file. This file contains only code for the core SQLite library. **}] set srcroot [file dirname [file dirname [info script]]] -if {$tcl_platform(platform)=="windows"} { +if {$tcl_platform(platform) eq "windows"} { set vsrcprog src-verify.exe } else { set vsrcprog ./src-verify diff --git a/tool/mksqlite3internalh.tcl b/tool/mksqlite3internalh.tcl index 8db593fe7..e1a42ee77 100644 --- a/tool/mksqlite3internalh.tcl +++ b/tool/mksqlite3internalh.tcl @@ -92,7 +92,7 @@ proc section_comment {text} { # Read the source file named $filename and write it into the # sqlite3.c output file. If any #include statements are seen, -# process them approprately. +# process them appropriately. # proc copy_file {filename} { global seen_hdr available_hdr out diff --git a/tool/run-speed-test.sh b/tool/run-speed-test.sh deleted file mode 100644 index 0e970ea0f..000000000 --- a/tool/run-speed-test.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash -# -# This is a template for a script used for day-to-day size and -# performance monitoring of SQLite. Typical usage: -# -# sh run-speed-test.sh trunk # Baseline measurement of trunk -# sh run-speed-test.sh x1 # Measure some experimental change -# fossil test-diff --tk cout-trunk.txt cout-x1.txt # View chanages -# -# There are multiple output files, all with a base name given by -# the first argument: -# -# summary-$BASE.txt # Copy of standard output -# cout-$BASE.txt # cachegrind output -# explain-$BASE.txt # EXPLAIN listings (only with --explain) -# -if test "$1" = "" -then - echo "Usage: $0 OUTPUTFILE [OPTIONS]" - exit -fi -NAME=$1 -shift -CC_OPTS="-DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_MEMSYS5" -SPEEDTEST_OPTS="--shrink-memory --reprepare --heap 10000000 64" -SIZE=5 -doExplain=0 -while test "$1" != ""; do - case $1 in - --reprepare) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --autovacuum) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --utf16be) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --stats) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --without-rowid) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --nomemstat) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --wal) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal" - ;; - --size) - shift; SIZE=$1 - ;; - --explain) - doExplain=1 - ;; - --heap) - CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_MEMSYS5" - shift; - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --heap $1 64" - ;; - *) - CC_OPTS="$CC_OPTS $1" - ;; - esac - shift -done -SPEEDTEST_OPTS="$SPEEDTEST_OPTS --size $SIZE" -echo "NAME = $NAME" | tee summary-$NAME.txt -echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt -echo "CC_OPTS = $CC_OPTS" | tee -a summary-$NAME.txt -rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o -gcc -g -Os -Wall -I. $CC_OPTS -c sqlite3.c -size sqlite3.o | tee -a summary-$NAME.txt -if test $doExplain -eq 1; then - gcc -g -Os -Wall -I. $CC_OPTS \ - -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ - ./shell.c ./sqlite3.c -o sqlite3 -ldl -lpthread -fi -SRC=./speedtest1.c -gcc -g -Os -Wall -I. $CC_OPTS $SRC ./sqlite3.o -o speedtest1 -ldl -lpthread -ls -l speedtest1 | tee -a summary-$NAME.txt -valgrind --tool=cachegrind ./speedtest1 speedtest1.db \ - $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt -size sqlite3.o | tee -a summary-$NAME.txt -wc sqlite3.c -cg_anno.tcl cachegrind.out.* >cout-$NAME.txt -if test $doExplain -eq 1; then - ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt -fi diff --git a/tool/showdb.c b/tool/showdb.c index 12c2e271b..f0bd9737c 100644 --- a/tool/showdb.c +++ b/tool/showdb.c @@ -27,7 +27,7 @@ typedef sqlite3_uint64 u64; /* unsigned 64-bit */ static struct GlobalData { - u32 pagesize; /* Size of a database page */ + i64 pagesize; /* Size of a database page */ int dbfd; /* File descriptor for reading the DB */ u32 mxPage; /* Last page number */ int perLine; /* HEX elements to print per line */ @@ -1178,7 +1178,7 @@ int main(int argc, char **argv){ if( g.pagesize==0 ) g.pagesize = 1024; sqlite3_free(zPgSz); - printf("Pagesize: %d\n", g.pagesize); + printf("Pagesize: %d\n", (int)g.pagesize); g.mxPage = (u32)((szFile+g.pagesize-1)/g.pagesize); printf("Available pages: 1..%u\n", g.mxPage); @@ -1218,7 +1218,8 @@ int main(int argc, char **argv){ iEnd = strtol(&zLeft[2], 0, 0); checkPageValidity(iEnd); }else if( zLeft && zLeft[0]=='b' ){ - int ofst, nByte, hdrSize; + i64 ofst; + int nByte, hdrSize; unsigned char *a; if( iStart==1 ){ ofst = hdrSize = 100; diff --git a/tool/soak1.tcl b/tool/soak1.tcl index 846f90593..e09c566b8 100644 --- a/tool/soak1.tcl +++ b/tool/soak1.tcl @@ -4,7 +4,7 @@ # # tclsh soak1.tcl local-makefile.mk ?target? ?scenario? # -# This generates many variations on local-makefile.mk (by modifing +# This generates many variations on local-makefile.mk (by modifying # the OPT = lines) and runs them will fulltest, one by one. The # constructed makefiles are named "soak1.mk". # diff --git a/tool/speed-check.sh b/tool/speed-check.sh deleted file mode 100644 index 8a9e67a38..000000000 --- a/tool/speed-check.sh +++ /dev/null @@ -1,219 +0,0 @@ -#!/bin/bash -# -# This is a template for a script used for day-to-day size and -# performance monitoring of SQLite. Typical usage: -# -# sh speed-check.sh trunk # Baseline measurement of trunk -# sh speed-check.sh x1 # Measure some experimental change -# fossil xdiff --tk cout-trunk.txt cout-x1.txt # View chanages -# -# There are multiple output files, all with a base name given by -# the first argument: -# -# summary-$BASE.txt # Copy of standard output -# cout-$BASE.txt # cachegrind output -# explain-$BASE.txt # EXPLAIN listings (only with --explain) -# -if test "$1" = "" -then - echo "Usage: $0 OUTPUTFILE [OPTIONS]" - exit -fi -NAME=$1 -shift -#CC_OPTS="-DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_MEMSYS5" -CC_OPTS="-DSQLITE_ENABLE_MEMSYS5" -CC=gcc -SPEEDTEST_OPTS="--shrink-memory --reprepare --stats --heap 10000000 64" -SIZE=5 -LEAN_OPTS="-DSQLITE_THREADSAFE=0" -LEAN_OPTS="$LEAN_OPTS -DSQLITE_DEFAULT_MEMSTATUS=0" -LEAN_OPTS="$LEAN_OPTS -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1" -LEAN_OPTS="$LEAN_OPTS -DSQLITE_LIKE_DOESNT_MATCH_BLOBS" -LEAN_OPTS="$LEAN_OPTS -DSQLITE_MAX_EXPR_DEPTH=0" -LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_DECLTYPE" -LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_DEPRECATED" -LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_PROGRESS_CALLBACK" -LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_SHARED_CACHE" -LEAN_OPTS="$LEAN_OPTS -DSQLITE_USE_ALLOCA" -BASELINE="trunk" -doExplain=0 -doCachegrind=1 -doVdbeProfile=0 -doWal=1 -doDiff=1 -while test "$1" != ""; do - case $1 in - --nodiff) - doDiff=0 - ;; - --reprepare) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --autovacuum) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --utf16be) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --stats) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --without-rowid) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --strict) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --nomemstat) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --multithread) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --singlethread) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --serialized) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1" - ;; - --temp) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --temp 6" - ;; - --legacy) - doWal=0 - CC_OPTS="$CC_OPTS -DSPEEDTEST_OMIT_HASH" - ;; - --verify) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --verify" - ;; - --wal) - doWal=1 - ;; - --size) - shift; SIZE=$1 - ;; - --cachesize) - shift; SPEEDTEST_OPTS="$SPEEDTEST_OPTS --cachesize $1" - ;; - --stmtcache) - shift; SPEEDTEST_OPTS="$SPEEDTEST_OPTS --stmtcache $1" - ;; - --checkpoint) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --checkpoint" - ;; - --explain) - doExplain=1 - ;; - --vdbeprofile) - rm -f vdbe_profile.out - CC_OPTS="$CC_OPTS -DVDBE_PROFILE" - doCachegrind=0 - doVdbeProfile=1 - ;; - --lean) - CC_OPTS="$CC_OPTS $LEAN_OPTS" - ;; - --clang) - CC=clang - ;; - --icc) - CC=/home/drh/intel/bin/icc - ;; - --gcc7) - CC=gcc-7 - ;; - --heap) - CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_MEMSYS5" - shift; - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --heap $1 64" - ;; - --lookaside) - shift; - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --lookaside $1 $2" - shift; - ;; - --repeat) - CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_RCACHE" - shift; - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --repeat $1" - ;; - --mmap) - shift; - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --mmap $1" - ;; - --rtree) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset rtree" - CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_RTREE" - ;; - --persist) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --persist" - ;; - --orm) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset orm" - ;; - --cte) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset cte" - ;; - --fp) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset fp" - ;; - --parsenumber) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset parsenumber" - ;; - --stmtscanstatus) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --stmtscanstatus" - ;; - -*) - CC_OPTS="$CC_OPTS $1" - ;; - *) - BASELINE=$1 - ;; - esac - shift -done -if test $doWal -eq 1; then - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal" -fi -SPEEDTEST_OPTS="$SPEEDTEST_OPTS --size $SIZE" -echo "NAME = $NAME" | tee summary-$NAME.txt -echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt -echo "CC_OPTS = $CC_OPTS" | tee -a summary-$NAME.txt -rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o -if test $doVdbeProfile -eq 1; then - rm -f vdbe_profile.out -fi -$CC -g -Os -Wall -I. $CC_OPTS -c sqlite3.c -size sqlite3.o | tee -a summary-$NAME.txt -if test $doExplain -eq 1; then - $CC -g -Os -Wall -I. $CC_OPTS \ - -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ - ./shell.c ./sqlite3.c -o sqlite3 -ldl -lpthread -fi -SRC=./speedtest1.c -$CC -g -Os -Wall -I. $CC_OPTS $SRC ./sqlite3.o -o speedtest1 -ldl -lpthread -ls -l speedtest1 | tee -a summary-$NAME.txt -if test $doCachegrind -eq 1; then - valgrind --tool=cachegrind ./speedtest1 speedtest1.db \ - $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt -else - ./speedtest1 speedtest1.db $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt -fi -size sqlite3.o | tee -a summary-$NAME.txt -wc sqlite3.c -if test $doCachegrind -eq 1; then - cg_anno.tcl cachegrind.out.* >cout-$NAME.txt - echo '*****************************************************' >>cout-$NAME.txt - sed 's/^[0-9=-]\{9\}/==00000==/' summary-$NAME.txt >>cout-$NAME.txt -fi -if test $doExplain -eq 1; then - ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt -fi -if test $doVdbeProfile -eq 1; then - tclsh ../sqlite/tool/vdbe_profile.tcl >vdbeprofile-$NAME.txt - open vdbeprofile-$NAME.txt -fi -if test "$NAME" != "$BASELINE" -a $doVdbeProfile -ne 1 -a $doDiff -ne 0; then - fossil test-diff --tk -c 20 cout-$BASELINE.txt cout-$NAME.txt -fi diff --git a/tool/speedtest.tcl b/tool/speedtest.tcl deleted file mode 100644 index ef39dc546..000000000 --- a/tool/speedtest.tcl +++ /dev/null @@ -1,275 +0,0 @@ -#!/usr/bin/tclsh -# -# Run this script using TCLSH to do a speed comparison between -# various versions of SQLite and PostgreSQL and MySQL -# - -# Run a test -# -set cnt 1 -proc runtest {title} { - global cnt - set sqlfile test$cnt.sql - puts "

    Test $cnt: $title

    " - incr cnt - set fd [open $sqlfile r] - set sql [string trim [read $fd [file size $sqlfile]]] - close $fd - set sx [split $sql \n] - set n [llength $sx] - if {$n>8} { - set sql {} - for {set i 0} {$i<3} {incr i} {append sql [lindex $sx $i]
    \n} - append sql "... [expr {$n-6}] lines omitted
    \n" - for {set i [expr {$n-3}]} {$i<$n} {incr i} { - append sql [lindex $sx $i]
    \n - } - } else { - regsub -all \n [string trim $sql]
    sql - } - puts "
    " - puts "$sql" - puts "
    " - set format {} - set delay 1000 -# exec sync; after $delay; -# set t [time "exec psql drh <$sqlfile" 1] -# set t [expr {[lindex $t 0]/1000000.0}] -# puts [format $format PostgreSQL: $t] - exec sync; after $delay; - set t [time "exec mysql -f drh <$sqlfile" 1] - set t [expr {[lindex $t 0]/1000000.0}] - puts [format $format MySQL: $t] -# set t [time "exec ./sqlite232 s232.db <$sqlfile" 1] -# set t [expr {[lindex $t 0]/1000000.0}] -# puts [format $format {SQLite 2.3.2:} $t] -# set t [time "exec ./sqlite-100 s100.db <$sqlfile" 1] -# set t [expr {[lindex $t 0]/1000000.0}] -# puts [format $format {SQLite 2.4 (cache=100):} $t] - exec sync; after $delay; - set t [time "exec ./sqlite248 s2k.db <$sqlfile" 1] - set t [expr {[lindex $t 0]/1000000.0}] - puts [format $format {SQLite 2.4.8:} $t] - exec sync; after $delay; - set t [time "exec ./sqlite248 sns.db <$sqlfile" 1] - set t [expr {[lindex $t 0]/1000000.0}] - puts [format $format {SQLite 2.4.8 (nosync):} $t] - exec sync; after $delay; - set t [time "exec ./sqlite2412 s2kb.db <$sqlfile" 1] - set t [expr {[lindex $t 0]/1000000.0}] - puts [format $format {SQLite 2.4.12:} $t] - exec sync; after $delay; - set t [time "exec ./sqlite2412 snsb.db <$sqlfile" 1] - set t [expr {[lindex $t 0]/1000000.0}] - puts [format $format {SQLite 2.4.12 (nosync):} $t] -# set t [time "exec ./sqlite-t1 st1.db <$sqlfile" 1] -# set t [expr {[lindex $t 0]/1000000.0}] -# puts [format $format {SQLite 2.4 (test):} $t] - puts "
    %s   %.3f
    " -} - -# Initialize the environment -# -expr srand(1) -catch {exec /bin/sh -c {rm -f s*.db}} -set fd [open clear.sql w] -puts $fd { - drop table t1; - drop table t2; -} -close $fd -catch {exec psql drh =1000} { - set txt "[number_name [expr {$n/1000}]] thousand" - set n [expr {$n%1000}] - } else { - set txt {} - } - if {$n>=100} { - append txt " [lindex $::ones [expr {$n/100}]] hundred" - set n [expr {$n%100}] - } - if {$n>=20} { - append txt " [lindex $::tens [expr {$n/10}]]" - set n [expr {$n%10}] - } - if {$n>0} { - append txt " [lindex $::ones $n]" - } - set txt [string trim $txt] - if {$txt==""} {set txt zero} - return $txt -} - - - -set fd [open test$cnt.sql w] -puts $fd "CREATE TABLE t1(a INTEGER, b INTEGER, c VARCHAR(100));" -for {set i 1} {$i<=1000} {incr i} { - set r [expr {int(rand()*100000)}] - puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');" -} -close $fd -runtest {1000 INSERTs} - - - -set fd [open test$cnt.sql w] -puts $fd "BEGIN;" -puts $fd "CREATE TABLE t2(a INTEGER, b INTEGER, c VARCHAR(100));" -for {set i 1} {$i<=25000} {incr i} { - set r [expr {int(rand()*500000)}] - puts $fd "INSERT INTO t2 VALUES($i,$r,'[number_name $r]');" -} -puts $fd "COMMIT;" -close $fd -runtest {25000 INSERTs in a transaction} - - - -set fd [open test$cnt.sql w] -for {set i 0} {$i<100} {incr i} { - set lwr [expr {$i*100}] - set upr [expr {($i+10)*100}] - puts $fd "SELECT count(*), avg(b) FROM t2 WHERE b>=$lwr AND b<$upr;" -} -close $fd -runtest {100 SELECTs without an index} - - - -set fd [open test$cnt.sql w] -for {set i 1} {$i<=100} {incr i} { - puts $fd "SELECT count(*), avg(b) FROM t2 WHERE c LIKE '%[number_name $i]%';" -} -close $fd -runtest {100 SELECTs on a string comparison} - - - -set fd [open test$cnt.sql w] -puts $fd {CREATE INDEX i2a ON t2(a);} -puts $fd {CREATE INDEX i2b ON t2(b);} -close $fd -runtest {Creating an index} - - - -set fd [open test$cnt.sql w] -for {set i 0} {$i<5000} {incr i} { - set lwr [expr {$i*100}] - set upr [expr {($i+1)*100}] - puts $fd "SELECT count(*), avg(b) FROM t2 WHERE b>=$lwr AND b<$upr;" -} -close $fd -runtest {5000 SELECTs with an index} - - - -set fd [open test$cnt.sql w] -puts $fd "BEGIN;" -for {set i 0} {$i<1000} {incr i} { - set lwr [expr {$i*10}] - set upr [expr {($i+1)*10}] - puts $fd "UPDATE t1 SET b=b*2 WHERE a>=$lwr AND a<$upr;" -} -puts $fd "COMMIT;" -close $fd -runtest {1000 UPDATEs without an index} - - - -set fd [open test$cnt.sql w] -puts $fd "BEGIN;" -for {set i 1} {$i<=25000} {incr i} { - set r [expr {int(rand()*500000)}] - puts $fd "UPDATE t2 SET b=$r WHERE a=$i;" -} -puts $fd "COMMIT;" -close $fd -runtest {25000 UPDATEs with an index} - - -set fd [open test$cnt.sql w] -puts $fd "BEGIN;" -for {set i 1} {$i<=25000} {incr i} { - set r [expr {int(rand()*500000)}] - puts $fd "UPDATE t2 SET c='[number_name $r]' WHERE a=$i;" -} -puts $fd "COMMIT;" -close $fd -runtest {25000 text UPDATEs with an index} - - - -set fd [open test$cnt.sql w] -puts $fd "BEGIN;" -puts $fd "INSERT INTO t1 SELECT * FROM t2;" -puts $fd "INSERT INTO t2 SELECT * FROM t1;" -puts $fd "COMMIT;" -close $fd -runtest {INSERTs from a SELECT} - - - -set fd [open test$cnt.sql w] -puts $fd {DELETE FROM t2 WHERE c LIKE '%fifty%';} -close $fd -runtest {DELETE without an index} - - - -set fd [open test$cnt.sql w] -puts $fd {DELETE FROM t2 WHERE a>10 AND a<20000;} -close $fd -runtest {DELETE with an index} - - - -set fd [open test$cnt.sql w] -puts $fd {INSERT INTO t2 SELECT * FROM t1;} -close $fd -runtest {A big INSERT after a big DELETE} - - - -set fd [open test$cnt.sql w] -puts $fd {BEGIN;} -puts $fd {DELETE FROM t1;} -for {set i 1} {$i<=3000} {incr i} { - set r [expr {int(rand()*100000)}] - puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');" -} -puts $fd {COMMIT;} -close $fd -runtest {A big DELETE followed by many small INSERTs} - - - -set fd [open test$cnt.sql w] -puts $fd {DROP TABLE t1;} -puts $fd {DROP TABLE t2;} -close $fd -runtest {DROP TABLE} diff --git a/tool/speedtest16.c b/tool/speedtest16.c deleted file mode 100644 index 993cc1926..000000000 --- a/tool/speedtest16.c +++ /dev/null @@ -1,171 +0,0 @@ -/* -** Performance test for SQLite. -** -** This program reads ASCII text from a file named on the command-line. -** It converts each SQL statement into UTF16 and submits it to SQLite -** for evaluation. A new UTF16 database is created at the beginning of -** the program. All statements are timed using the high-resolution timer -** built into Intel-class processors. -** -** To compile this program, first compile the SQLite library separately -** will full optimizations. For example: -** -** gcc -c -O6 -DSQLITE_THREADSAFE=0 sqlite3.c -** -** Then link against this program. But to do optimize this program -** because that defeats the hi-res timer. -** -** gcc speedtest16.c sqlite3.o -ldl -I../src -** -** Then run this program with a single argument which is the name of -** a file containing SQL script that you want to test: -** -** ./a.out database.db test.sql -*/ -#include -#include -#include -#include -#include -#include "sqlite3.h" - -#define ISSPACE(X) isspace((unsigned char)(X)) - -/* -** hwtime.h contains inline assembler code for implementing -** high-performance timing routines. -*/ -#include "hwtime.h" - -/* -** Convert a zero-terminated ASCII string into a zero-terminated -** UTF-16le string. Memory to hold the returned string comes -** from malloc() and should be freed by the caller. -*/ -static void *asciiToUtf16le(const char *z){ - int n = strlen(z); - char *z16; - int i, j; - - z16 = malloc( n*2 + 2 ); - for(i=j=0; i<=n; i++){ - z16[j++] = z[i]; - z16[j++] = 0; - } - return (void*)z16; -} - -/* -** Timers -*/ -static sqlite_uint64 prepTime = 0; -static sqlite_uint64 runTime = 0; -static sqlite_uint64 finalizeTime = 0; - -/* -** Prepare and run a single statement of SQL. -*/ -static void prepareAndRun(sqlite3 *db, const char *zSql){ - void *utf16; - sqlite3_stmt *pStmt; - const void *stmtTail; - sqlite_uint64 iStart, iElapse; - int rc; - - printf("****************************************************************\n"); - printf("SQL statement: [%s]\n", zSql); - utf16 = asciiToUtf16le(zSql); - iStart = sqlite3Hwtime(); - rc = sqlite3_prepare16_v2(db, utf16, -1, &pStmt, &stmtTail); - iElapse = sqlite3Hwtime() - iStart; - prepTime += iElapse; - printf("sqlite3_prepare16_v2() returns %d in %llu cycles\n", rc, iElapse); - if( rc==SQLITE_OK ){ - int nRow = 0; - iStart = sqlite3Hwtime(); - while( (rc=sqlite3_step(pStmt))==SQLITE_ROW ){ nRow++; } - iElapse = sqlite3Hwtime() - iStart; - runTime += iElapse; - printf("sqlite3_step() returns %d after %d rows in %llu cycles\n", - rc, nRow, iElapse); - iStart = sqlite3Hwtime(); - rc = sqlite3_finalize(pStmt); - iElapse = sqlite3Hwtime() - iStart; - finalizeTime += iElapse; - printf("sqlite3_finalize() returns %d in %llu cycles\n", rc, iElapse); - } - free(utf16); -} - -int main(int argc, char **argv){ - void *utf16; - sqlite3 *db; - int rc; - int nSql; - char *zSql; - int i, j; - FILE *in; - sqlite_uint64 iStart, iElapse; - sqlite_uint64 iSetup = 0; - int nStmt = 0; - int nByte = 0; - - if( argc!=3 ){ - fprintf(stderr, "Usage: %s FILENAME SQL-SCRIPT\n" - "Runs SQL-SCRIPT as UTF16 against a UTF16 database\n", - argv[0]); - exit(1); - } - in = fopen(argv[2], "r"); - fseek(in, 0L, SEEK_END); - nSql = ftell(in); - zSql = malloc( nSql+1 ); - fseek(in, 0L, SEEK_SET); - nSql = fread(zSql, 1, nSql, in); - zSql[nSql] = 0; - - printf("SQLite version: %d\n", sqlite3_libversion_number()); - unlink(argv[1]); - utf16 = asciiToUtf16le(argv[1]); - iStart = sqlite3Hwtime(); - rc = sqlite3_open16(utf16, &db); - iElapse = sqlite3Hwtime() - iStart; - iSetup = iElapse; - printf("sqlite3_open16() returns %d in %llu cycles\n", rc, iElapse); - free(utf16); - for(i=j=0; jTest $cnt: $title" - incr cnt - set fd [open $sqlfile r] - set sql [string trim [read $fd [file size $sqlfile]]] - close $fd - set sx [split $sql \n] - set n [llength $sx] - if {$n>8} { - set sql {} - for {set i 0} {$i<3} {incr i} {append sql [lindex $sx $i]
    \n} - append sql "... [expr {$n-6}] lines omitted
    \n" - for {set i [expr {$n-3}]} {$i<$n} {incr i} { - append sql [lindex $sx $i]
    \n - } - } else { - regsub -all \n [string trim $sql]
    sql - } - puts "
    " - puts "$sql" - puts "
    " - set format {} - set delay 1000 - exec sync; after $delay; - set t [time "exec psql drh <$sqlfile" 1] - set t [expr {[lindex $t 0]/1000000.0}] - puts [format $format PostgreSQL: $t] - exec sync; after $delay; - set t [time "exec mysql -f drh <$sqlfile" 1] - set t [expr {[lindex $t 0]/1000000.0}] - puts [format $format MySQL: $t] -# set t [time "exec ./sqlite232 s232.db <$sqlfile" 1] -# set t [expr {[lindex $t 0]/1000000.0}] -# puts [format $format {SQLite 2.3.2:} $t] -# set t [time "exec ./sqlite-100 s100.db <$sqlfile" 1] -# set t [expr {[lindex $t 0]/1000000.0}] -# puts [format $format {SQLite 2.4 (cache=100):} $t] - exec sync; after $delay; - set t [time "exec ./sqlite240 s2k.db <$sqlfile" 1] - set t [expr {[lindex $t 0]/1000000.0}] - puts [format $format {SQLite 2.4:} $t] - exec sync; after $delay; - set t [time "exec ./sqlite240 sns.db <$sqlfile" 1] - set t [expr {[lindex $t 0]/1000000.0}] - puts [format $format {SQLite 2.4 (nosync):} $t] -# set t [time "exec ./sqlite-t1 st1.db <$sqlfile" 1] -# set t [expr {[lindex $t 0]/1000000.0}] -# puts [format $format {SQLite 2.4 (test):} $t] - puts "
    %s   %.3f
    " -} - -# Initialize the environment -# -expr srand(1) -catch {exec /bin/sh -c {rm -f s*.db}} -set fd [open clear.sql w] -puts $fd { - drop table t1; - drop table t2; -} -close $fd -catch {exec psql drh =1000} { - set txt "[number_name [expr {$n/1000}]] thousand" - set n [expr {$n%1000}] - } else { - set txt {} - } - if {$n>=100} { - append txt " [lindex $::ones [expr {$n/100}]] hundred" - set n [expr {$n%100}] - } - if {$n>=20} { - append txt " [lindex $::tens [expr {$n/10}]]" - set n [expr {$n%10}] - } - if {$n>0} { - append txt " [lindex $::ones $n]" - } - set txt [string trim $txt] - if {$txt==""} {set txt zero} - return $txt -} - - -set fd [open test$cnt.sql w] -puts $fd "BEGIN;" -puts $fd "CREATE TABLE t1(a INTEGER, b INTEGER, c VARCHAR(100));" -for {set i 1} {$i<=25000} {incr i} { - set r [expr {int(rand()*500000)}] - puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');" -} -puts $fd "COMMIT;" -close $fd -runtest {25000 INSERTs in a transaction} - - -set fd [open test$cnt.sql w] -puts $fd "DELETE FROM t1;" -close $fd -runtest {DELETE everything} - - -set fd [open test$cnt.sql w] -puts $fd "BEGIN;" -for {set i 1} {$i<=25000} {incr i} { - set r [expr {int(rand()*500000)}] - puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');" -} -puts $fd "COMMIT;" -close $fd -runtest {25000 INSERTs in a transaction} - - -set fd [open test$cnt.sql w] -puts $fd "DELETE FROM t1;" -close $fd -runtest {DELETE everything} - - -set fd [open test$cnt.sql w] -puts $fd "BEGIN;" -for {set i 1} {$i<=25000} {incr i} { - set r [expr {int(rand()*500000)}] - puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');" -} -puts $fd "COMMIT;" -close $fd -runtest {25000 INSERTs in a transaction} - - -set fd [open test$cnt.sql w] -puts $fd "DELETE FROM t1;" -close $fd -runtest {DELETE everything} - - -set fd [open test$cnt.sql w] -puts $fd "BEGIN;" -for {set i 1} {$i<=25000} {incr i} { - set r [expr {int(rand()*500000)}] - puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');" -} -puts $fd "COMMIT;" -close $fd -runtest {25000 INSERTs in a transaction} - - -set fd [open test$cnt.sql w] -puts $fd "DELETE FROM t1;" -close $fd -runtest {DELETE everything} - - -set fd [open test$cnt.sql w] -puts $fd "BEGIN;" -for {set i 1} {$i<=25000} {incr i} { - set r [expr {int(rand()*500000)}] - puts $fd "INSERT INTO t1 VALUES($i,$r,'[number_name $r]');" -} -puts $fd "COMMIT;" -close $fd -runtest {25000 INSERTs in a transaction} - - -set fd [open test$cnt.sql w] -puts $fd "DELETE FROM t1;" -close $fd -runtest {DELETE everything} - - -set fd [open test$cnt.sql w] -puts $fd {DROP TABLE t1;} -close $fd -runtest {DROP TABLE} diff --git a/tool/speedtest8.c b/tool/speedtest8.c deleted file mode 100644 index 051fc8981..000000000 --- a/tool/speedtest8.c +++ /dev/null @@ -1,260 +0,0 @@ -/* -** Performance test for SQLite. -** -** This program reads ASCII text from a file named on the command-line -** and submits that text to SQLite for evaluation. A new database -** is created at the beginning of the program. All statements are -** timed using the high-resolution timer built into Intel-class processors. -** -** To compile this program, first compile the SQLite library separately -** will full optimizations. For example: -** -** gcc -c -O6 -DSQLITE_THREADSAFE=0 sqlite3.c -** -** Then link against this program. But to do optimize this program -** because that defeats the hi-res timer. -** -** gcc speedtest8.c sqlite3.o -ldl -I../src -** -** Then run this program with a single argument which is the name of -** a file containing SQL script that you want to test: -** -** ./a.out test.db test.sql -*/ -#include -#include -#include -#include -#include - -#if defined(_MSC_VER) -#include -#else -#include -#include -#include -#endif - -#include "sqlite3.h" - -/* -** hwtime.h contains inline assembler code for implementing -** high-performance timing routines. -*/ -#include "hwtime.h" - -/* -** Timers -*/ -static sqlite_uint64 prepTime = 0; -static sqlite_uint64 runTime = 0; -static sqlite_uint64 finalizeTime = 0; - -/* -** Prepare and run a single statement of SQL. -*/ -static void prepareAndRun(sqlite3 *db, const char *zSql, int bQuiet){ - sqlite3_stmt *pStmt; - const char *stmtTail; - sqlite_uint64 iStart, iElapse; - int rc; - - if (!bQuiet){ - printf("***************************************************************\n"); - } - if (!bQuiet) printf("SQL statement: [%s]\n", zSql); - iStart = sqlite3Hwtime(); - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &stmtTail); - iElapse = sqlite3Hwtime() - iStart; - prepTime += iElapse; - if (!bQuiet){ - printf("sqlite3_prepare_v2() returns %d in %llu cycles\n", rc, iElapse); - } - if( rc==SQLITE_OK ){ - int nRow = 0; - iStart = sqlite3Hwtime(); - while( (rc=sqlite3_step(pStmt))==SQLITE_ROW ){ nRow++; } - iElapse = sqlite3Hwtime() - iStart; - runTime += iElapse; - if (!bQuiet){ - printf("sqlite3_step() returns %d after %d rows in %llu cycles\n", - rc, nRow, iElapse); - } - iStart = sqlite3Hwtime(); - rc = sqlite3_finalize(pStmt); - iElapse = sqlite3Hwtime() - iStart; - finalizeTime += iElapse; - if (!bQuiet){ - printf("sqlite3_finalize() returns %d in %llu cycles\n", rc, iElapse); - } - } -} - -int main(int argc, char **argv){ - sqlite3 *db; - int rc; - int nSql; - char *zSql; - int i, j; - FILE *in; - sqlite_uint64 iStart, iElapse; - sqlite_uint64 iSetup = 0; - int nStmt = 0; - int nByte = 0; - const char *zArgv0 = argv[0]; - int bQuiet = 0; -#if !defined(_MSC_VER) - struct tms tmsStart, tmsEnd; - clock_t clkStart, clkEnd; -#endif - -#ifdef HAVE_OSINST - extern sqlite3_vfs *sqlite3_instvfs_binarylog(char *, char *, char *); - extern void sqlite3_instvfs_destroy(sqlite3_vfs *); - sqlite3_vfs *pVfs = 0; -#endif - - while (argc>3) - { -#ifdef HAVE_OSINST - if( argc>4 && (strcmp(argv[1], "-log")==0) ){ - pVfs = sqlite3_instvfs_binarylog("oslog", 0, argv[2]); - sqlite3_vfs_register(pVfs, 1); - argv += 2; - argc -= 2; - continue; - } -#endif - - /* - ** Increasing the priority slightly above normal can help with - ** repeatability of testing. Note that with Cygwin, -5 equates - ** to "High", +5 equates to "Low", and anything in between - ** equates to "Normal". - */ - if( argc>4 && (strcmp(argv[1], "-priority")==0) ){ -#if defined(_MSC_VER) - int new_priority = atoi(argv[2]); - if(!SetPriorityClass(GetCurrentProcess(), - (new_priority<=-5) ? HIGH_PRIORITY_CLASS : - (new_priority<=0) ? ABOVE_NORMAL_PRIORITY_CLASS : - (new_priority==0) ? NORMAL_PRIORITY_CLASS : - (new_priority<5) ? BELOW_NORMAL_PRIORITY_CLASS : - IDLE_PRIORITY_CLASS)){ - printf ("error setting priority\n"); - exit(2); - } -#else - struct sched_param myParam; - sched_getparam(0, &myParam); - printf ("Current process priority is %d.\n", (int)myParam.sched_priority); - myParam.sched_priority = atoi(argv[2]); - printf ("Setting process priority to %d.\n", (int)myParam.sched_priority); - if (sched_setparam (0, &myParam) != 0){ - printf ("error setting priority\n"); - exit(2); - } -#endif - argv += 2; - argc -= 2; - continue; - } - - if( argc>3 && strcmp(argv[1], "-quiet")==0 ){ - bQuiet = -1; - argv++; - argc--; - continue; - } - - break; - } - - if( argc!=3 ){ - fprintf(stderr, "Usage: %s [options] FILENAME SQL-SCRIPT\n" - "Runs SQL-SCRIPT against a UTF8 database\n" - "\toptions:\n" -#ifdef HAVE_OSINST - "\t-log \n" -#endif - "\t-priority : set priority of task\n" - "\t-quiet : only display summary results\n", - zArgv0); - exit(1); - } - - in = fopen(argv[2], "r"); - fseek(in, 0L, SEEK_END); - nSql = ftell(in); - zSql = malloc( nSql+1 ); - fseek(in, 0L, SEEK_SET); - nSql = fread(zSql, 1, nSql, in); - zSql[nSql] = 0; - - printf("SQLite version: %d\n", sqlite3_libversion_number()); - unlink(argv[1]); -#if !defined(_MSC_VER) - clkStart = times(&tmsStart); -#endif - iStart = sqlite3Hwtime(); - rc = sqlite3_open(argv[1], &db); - iElapse = sqlite3Hwtime() - iStart; - iSetup = iElapse; - if (!bQuiet) printf("sqlite3_open() returns %d in %llu cycles\n", rc, iElapse); - for(i=j=0; j=6 && memcmp(&zSql[i], ".crash",6)==0 ) exit(1); - nStmt++; - nByte += n; - prepareAndRun(db, &zSql[i], bQuiet); - } - zSql[j] = ';'; - i = j+1; - } - } - } - iStart = sqlite3Hwtime(); - sqlite3_close(db); - iElapse = sqlite3Hwtime() - iStart; -#if !defined(_MSC_VER) - clkEnd = times(&tmsEnd); -#endif - iSetup += iElapse; - if (!bQuiet) printf("sqlite3_close() returns in %llu cycles\n", iElapse); - - printf("\n"); - printf("Statements run: %15d stmts\n", nStmt); - printf("Bytes of SQL text: %15d bytes\n", nByte); - printf("Total prepare time: %15llu cycles\n", prepTime); - printf("Total run time: %15llu cycles\n", runTime); - printf("Total finalize time: %15llu cycles\n", finalizeTime); - printf("Open/Close time: %15llu cycles\n", iSetup); - printf("Total time: %15llu cycles\n", - prepTime + runTime + finalizeTime + iSetup); - -#if !defined(_MSC_VER) - printf("\n"); - printf("Total user CPU time: %15.3g secs\n", (tmsEnd.tms_utime - tmsStart.tms_utime)/(double)CLOCKS_PER_SEC ); - printf("Total system CPU time: %15.3g secs\n", (tmsEnd.tms_stime - tmsStart.tms_stime)/(double)CLOCKS_PER_SEC ); - printf("Total real time: %15.3g secs\n", (clkEnd -clkStart)/(double)CLOCKS_PER_SEC ); -#endif - -#ifdef HAVE_OSINST - if( pVfs ){ - sqlite3_instvfs_destroy(pVfs); - printf("vfs log written to %s\n", argv[0]); - } -#endif - - return 0; -} diff --git a/tool/speedtest8inst1.c b/tool/speedtest8inst1.c deleted file mode 100644 index ceaeca0f1..000000000 --- a/tool/speedtest8inst1.c +++ /dev/null @@ -1,218 +0,0 @@ -/* -** Performance test for SQLite. -** -** This program reads ASCII text from a file named on the command-line -** and submits that text to SQLite for evaluation. A new database -** is created at the beginning of the program. All statements are -** timed using the high-resolution timer built into Intel-class processors. -** -** To compile this program, first compile the SQLite library separately -** will full optimizations. For example: -** -** gcc -c -O6 -DSQLITE_THREADSAFE=0 sqlite3.c -** -** Then link against this program. But to do optimize this program -** because that defeats the hi-res timer. -** -** gcc speedtest8.c sqlite3.o -ldl -I../src -** -** Then run this program with a single argument which is the name of -** a file containing SQL script that you want to test: -** -** ./a.out test.db test.sql -*/ -#include -#include -#include -#include -#include -#include -#include "sqlite3.h" - -#define ISSPACE(X) isspace((unsigned char)(X)) - -#include "test_osinst.c" - -/* -** Prepare and run a single statement of SQL. -*/ -static void prepareAndRun(sqlite3_vfs *pInstVfs, sqlite3 *db, const char *zSql){ - sqlite3_stmt *pStmt; - const char *stmtTail; - int rc; - char zMessage[1024]; - zMessage[1023] = '\0'; - - sqlite3_uint64 iTime; - - sqlite3_snprintf(1023, zMessage, "sqlite3_prepare_v2: %s", zSql); - sqlite3_instvfs_binarylog_marker(pInstVfs, zMessage); - - iTime = sqlite3Hwtime(); - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &stmtTail); - iTime = sqlite3Hwtime() - iTime; - sqlite3_instvfs_binarylog_call(pInstVfs,BINARYLOG_PREPARE_V2,iTime,rc,zSql); - - if( rc==SQLITE_OK ){ - int nRow = 0; - - sqlite3_snprintf(1023, zMessage, "sqlite3_step loop: %s", zSql); - sqlite3_instvfs_binarylog_marker(pInstVfs, zMessage); - iTime = sqlite3Hwtime(); - while( (rc=sqlite3_step(pStmt))==SQLITE_ROW ){ nRow++; } - iTime = sqlite3Hwtime() - iTime; - sqlite3_instvfs_binarylog_call(pInstVfs, BINARYLOG_STEP, iTime, rc, zSql); - - sqlite3_snprintf(1023, zMessage, "sqlite3_finalize: %s", zSql); - sqlite3_instvfs_binarylog_marker(pInstVfs, zMessage); - iTime = sqlite3Hwtime(); - rc = sqlite3_finalize(pStmt); - iTime = sqlite3Hwtime() - iTime; - sqlite3_instvfs_binarylog_call(pInstVfs, BINARYLOG_FINALIZE, iTime, rc, zSql); - } -} - -static int stringcompare(const char *zLeft, const char *zRight){ - int ii; - for(ii=0; zLeft[ii] && zRight[ii]; ii++){ - if( zLeft[ii]!=zRight[ii] ) return 0; - } - return( zLeft[ii]==zRight[ii] ); -} - -static char *readScriptFile(const char *zFile, int *pnScript){ - sqlite3_vfs *pVfs = sqlite3_vfs_find(0); - sqlite3_file *p; - int rc; - sqlite3_int64 nByte; - char *zData = 0; - int flags = SQLITE_OPEN_READONLY|SQLITE_OPEN_MAIN_DB; - - p = (sqlite3_file *)malloc(pVfs->szOsFile); - rc = pVfs->xOpen(pVfs, zFile, p, flags, &flags); - if( rc!=SQLITE_OK ){ - goto error_out; - } - - rc = p->pMethods->xFileSize(p, &nByte); - if( rc!=SQLITE_OK ){ - goto close_out; - } - - zData = (char *)malloc(nByte+1); - rc = p->pMethods->xRead(p, zData, nByte, 0); - if( rc!=SQLITE_OK ){ - goto close_out; - } - zData[nByte] = '\0'; - - p->pMethods->xClose(p); - free(p); - *pnScript = nByte; - return zData; - -close_out: - p->pMethods->xClose(p); - -error_out: - free(p); - free(zData); - return 0; -} - -int main(int argc, char **argv){ - - const char zUsageMsg[] = - "Usage: %s options...\n" - " where available options are:\n" - "\n" - " -db DATABASE-FILE (database file to operate on)\n" - " -script SCRIPT-FILE (script file to read sql from)\n" - " -log LOG-FILE (log file to create)\n" - " -logdata (log all data to log file)\n" - "\n" - " Options -db, -script and -log are compulsory\n" - "\n" - ; - - const char *zDb = 0; - const char *zScript = 0; - const char *zLog = 0; - int logdata = 0; - - int ii; - int i, j; - int rc; - - sqlite3_vfs *pInstVfs; /* Instrumentation VFS */ - - char *zSql = 0; - int nSql; - - sqlite3 *db; - - for(ii=1; iiiSize==0 ) HashInit(pCx, 160); + if( eType==SQLITE_BLOB ){ + HashUpdate(pCx, sqlite3_value_blob(argv[0]), nByte); + }else{ + HashUpdate(pCx, sqlite3_value_text(argv[0]), nByte); + } +} +static void agghashFinal(sqlite3_context *context){ + HashContext *pCx = (HashContext*)sqlite3_aggregate_context(context, 0); + if( pCx ){ + sqlite3_result_blob(context, HashFinal(pCx), 160/8, SQLITE_TRANSIENT); + } +} + /* Register the hash function */ static int hashRegister(sqlite3 *db){ - return sqlite3_create_function(db, "hash", 1, + int rc; + rc = sqlite3_create_function(db, "hash", 1, SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, 0, hashFunc, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "agghash", 1, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, 0, agghashStep, agghashFinal); + } + return rc; } /* End of the hashing logic @@ -836,6 +932,25 @@ static void logError(SQLiteRsync *p, const char *zFormat, ...){ p->nErr++; } +/* +** Append text to the debugging mesage file, if an that file is +** specified. +*/ +static void debugMessage(SQLiteRsync *p, const char *zFormat, ...){ + if( p->zDebugFile ){ + if( p->pDebug==0 ){ + p->pDebug = fopen(p->zDebugFile, "wb"); + } + if( p->pDebug ){ + va_list ap; + va_start(ap, zFormat); + vfprintf(p->pDebug, zFormat, ap); + va_end(ap); + fflush(p->pDebug); + } + } +} + /* Read a single big-endian 32-bit unsigned integer from the input ** stream. Return 0 on success and 1 if there are any errors. @@ -1190,6 +1305,13 @@ static void closeDb(SQLiteRsync *p){ ** nPage, and szPage. Then enter a loop responding to message from ** the replica: ** +** REPLICA_BEGIN iProtocol +** +** An optional message sent by the replica in response to the +** prior ORIGIN_BEGIN with a counter-proposal for the protocol +** level. If seen, try to reduce the protocol level to what is +** requested and send a new ORGIN_BEGIN. +** ** REPLICA_ERROR size text ** ** Report an error from the replica and quit @@ -1200,30 +1322,42 @@ static void closeDb(SQLiteRsync *p){ ** ** REPLICA_HASH hash ** -** The argument is the 20-byte SHA1 hash for the next page -** page hashes appear in sequential order with no gaps. +** The argument is the 20-byte SHA1 hash for the next page or +** block of pages. Hashes appear in sequential order with no gaps, +** unless there is an intervening REPLICA_CONFIG message. +** +** REPLICA_CONFIG pgno cnt +** +** Set counters used by REPLICA_HASH. The next hash will start +** on page pgno and all subsequent hashes will cover cnt pages +** each. Note that for a multi-page hash, the hash value is +** actually a hash of the individual page hashes. ** ** REPLICA_READY ** ** The replica has sent all the hashes that it intends to send. ** This side (the origin) can now start responding with page -** content for pages that do not have a matching hash. +** content for pages that do not have a matching hash or with +** ORIGIN_DETAIL messages with requests for more detail. */ static void originSide(SQLiteRsync *p){ int rc = 0; int c = 0; unsigned int nPage = 0; - unsigned int iPage = 0; + unsigned int iHash = 1; /* Pgno for next hash to receive */ + unsigned int nHash = 1; /* Number of pages per hash received */ + unsigned int mxHash = 0; /* Maximum hash value received */ unsigned int lockBytePage = 0; unsigned int szPg = 0; - sqlite3_stmt *pCkHash = 0; - sqlite3_stmt *pInsHash = 0; + sqlite3_stmt *pCkHash = 0; /* Verify hash on a single page */ + sqlite3_stmt *pCkHashN = 0; /* Verify a multi-page hash */ + sqlite3_stmt *pInsHash = 0; /* Record a bad hash */ char buf[200]; p->isReplica = 0; if( p->bCommCheck ){ infoMsg(p, "origin zOrigin=%Q zReplica=%Q isRemote=%d protocol=%d", - p->zOrigin, p->zReplica, p->isRemote, PROTOCOL_VERSION); + p->zOrigin, p->zReplica, p->isRemote, p->iProtocol); writeByte(p, ORIGIN_END); fflush(p->pOut); }else{ @@ -1237,9 +1371,11 @@ static void originSide(SQLiteRsync *p){ } hashRegister(p->db); runSql(p, "BEGIN"); - runSqlReturnText(p, buf, "PRAGMA journal_mode"); - if( sqlite3_stricmp(buf,"wal")!=0 ){ - reportError(p, "Origin database is not in WAL mode"); + if( p->bWalOnly ){ + runSqlReturnText(p, buf, "PRAGMA journal_mode"); + if( sqlite3_stricmp(buf,"wal")!=0 ){ + reportError(p, "Origin database is not in WAL mode"); + } } runSqlReturnUInt(p, &nPage, "PRAGMA page_count"); runSqlReturnUInt(p, &szPg, "PRAGMA page_size"); @@ -1247,13 +1383,15 @@ static void originSide(SQLiteRsync *p){ if( p->nErr==0 ){ /* Send the ORIGIN_BEGIN message */ writeByte(p, ORIGIN_BEGIN); - writeByte(p, PROTOCOL_VERSION); + writeByte(p, p->iProtocol); writePow2(p, szPg); writeUint32(p, nPage); fflush(p->pOut); + if( p->zDebugFile ){ + debugMessage(p, "-> ORIGIN_BEGIN %u %u %u\n", p->iProtocol,szPg,nPage); + } p->nPage = nPage; p->szPage = szPg; - p->iProtocol = PROTOCOL_VERSION; lockBytePage = (1<<30)/szPg + 1; } } @@ -1266,11 +1404,24 @@ static void originSide(SQLiteRsync *p){ ** that is larger than what it knows about. The replica sends back ** a counter-proposal of an earlier protocol which the origin can ** accept by resending a new ORIGIN_BEGIN. */ - p->iProtocol = readByte(p); - writeByte(p, ORIGIN_BEGIN); - writeByte(p, p->iProtocol); - writePow2(p, p->szPage); - writeUint32(p, p->nPage); + u8 newProtocol = readByte(p); + if( p->zDebugFile ){ + debugMessage(p, "<- REPLICA_BEGIN %d\n", (int)newProtocol); + } + if( newProtocol < p->iProtocol ){ + p->iProtocol = newProtocol; + writeByte(p, ORIGIN_BEGIN); + writeByte(p, p->iProtocol); + writePow2(p, p->szPage); + writeUint32(p, p->nPage); + fflush(p->pOut); + if( p->zDebugFile ){ + debugMessage(p, "-> ORIGIN_BEGIN %d %d %u\n", p->iProtocol, + p->szPage, p->nPage); + } + }else{ + reportError(p, "Invalid REPLICA_BEGIN reply"); + } break; } case REPLICA_MSG: @@ -1278,25 +1429,73 @@ static void originSide(SQLiteRsync *p){ readAndDisplayMessage(p, c); break; } + case REPLICA_CONFIG: { + readUint32(p, &iHash); + readUint32(p, &nHash); + if( p->zDebugFile ){ + debugMessage(p, "<- REPLICA_CONFIG %u %u\n", iHash, nHash); + } + break; + } case REPLICA_HASH: { + int bMatch = 0; if( pCkHash==0 ){ - runSql(p, "CREATE TEMP TABLE badHash(pgno INTEGER PRIMARY KEY)"); + runSql(p, "CREATE TEMP TABLE badHash(" + " pgno INTEGER PRIMARY KEY," + " sz INT)"); pCkHash = prepareStmt(p, - "SELECT pgno FROM sqlite_dbpage('main')" - " WHERE pgno=?1 AND hash(data)!=?2" + "SELECT hash(data)==?3 FROM sqlite_dbpage('main')" + " WHERE pgno=?1" ); if( pCkHash==0 ) break; - pInsHash = prepareStmt(p, "INSERT INTO badHash VALUES(?)"); + pInsHash = prepareStmt(p, "INSERT INTO badHash VALUES(?1,?2)"); if( pInsHash==0 ) break; } p->nHashSent++; - iPage++; - sqlite3_bind_int64(pCkHash, 1, iPage); readBytes(p, 20, buf); - sqlite3_bind_blob(pCkHash, 2, buf, 20, SQLITE_STATIC); - rc = sqlite3_step(pCkHash); - if( rc==SQLITE_ROW ){ - sqlite3_bind_int64(pInsHash, 1, sqlite3_column_int64(pCkHash, 0)); + if( nHash>1 ){ + if( pCkHashN==0 ){ + pCkHashN = prepareStmt(p, + "WITH c(n) AS " + " (VALUES(?1) UNION ALL SELECT n+1 FROM c WHERE ndb)); + } + sqlite3_reset(pCkHashN); + }else{ + sqlite3_bind_int64(pCkHash, 1, iHash); + sqlite3_bind_blob(pCkHash, 3, buf, 20, SQLITE_STATIC); + rc = sqlite3_step(pCkHash); + if( rc==SQLITE_ERROR ){ + reportError(p, "SQL statement [%s] failed: %s", + sqlite3_sql(pCkHash), sqlite3_errmsg(p->db)); + }else if( rc==SQLITE_ROW && sqlite3_column_int(pCkHash,0) ){ + bMatch = 1; + } + sqlite3_reset(pCkHash); + } + if( p->zDebugFile ){ + debugMessage(p, "<- REPLICA_HASH %u %u %s %08x...\n", + iHash, nHash, + bMatch ? "match" : "fail", + *(unsigned int*)buf + ); + } + if( !bMatch ){ + sqlite3_bind_int64(pInsHash, 1, iHash); + sqlite3_bind_int64(pInsHash, 2, nHash); rc = sqlite3_step(pInsHash); if( rc!=SQLITE_DONE ){ reportError(p, "SQL statement [%s] failed: %s", @@ -1304,42 +1503,74 @@ static void originSide(SQLiteRsync *p){ } sqlite3_reset(pInsHash); } - else if( rc!=SQLITE_DONE ){ - reportError(p, "SQL statement [%s] failed: %s", - sqlite3_sql(pCkHash), sqlite3_errmsg(p->db)); - } - sqlite3_reset(pCkHash); + if( iHash+nHash>mxHash ) mxHash = iHash+nHash; + iHash += nHash; break; } case REPLICA_READY: { + int nMulti = 0; sqlite3_stmt *pStmt; - sqlite3_finalize(pCkHash); - sqlite3_finalize(pInsHash); - pCkHash = 0; - pInsHash = 0; - if( iPage+1nPage ){ - runSql(p, "WITH RECURSIVE c(n) AS" - " (VALUES(%d) UNION ALL SELECT n+1 FROM c WHERE n<%d)" - " INSERT INTO badHash SELECT n FROM c", - iPage+1, p->nPage); + if( p->zDebugFile ){ + debugMessage(p, "<- REPLICA_READY\n"); } - runSql(p, "DELETE FROM badHash WHERE pgno=%d", lockBytePage); - pStmt = prepareStmt(p, - "SELECT pgno, data" - " FROM badHash JOIN sqlite_dbpage('main') USING(pgno)"); + p->nRound++; + pStmt = prepareStmt(p,"SELECT pgno, sz FROM badHash WHERE sz>1"); if( pStmt==0 ) break; - while( sqlite3_step(pStmt)==SQLITE_ROW && p->nErr==0 && p->nWrErr==0 ){ + while( sqlite3_step(pStmt)==SQLITE_ROW ){ unsigned int pgno = (unsigned int)sqlite3_column_int64(pStmt,0); - const void *pContent = sqlite3_column_blob(pStmt, 1); - writeByte(p, ORIGIN_PAGE); + unsigned int cnt = (unsigned int)sqlite3_column_int64(pStmt,1); + writeByte(p, ORIGIN_DETAIL); writeUint32(p, pgno); - writeBytes(p, szPg, pContent); - p->nPageSent++; + writeUint32(p, cnt); + nMulti++; + if( p->zDebugFile ){ + debugMessage(p, "-> ORIGIN_DETAIL %u %u\n", pgno, cnt); + } } sqlite3_finalize(pStmt); - writeByte(p, ORIGIN_TXN); - writeUint32(p, nPage); - writeByte(p, ORIGIN_END); + if( nMulti ){ + runSql(p, "DELETE FROM badHash WHERE sz>1"); + writeByte(p, ORIGIN_READY); + if( p->zDebugFile ) debugMessage(p, "-> ORIGIN_READY\n"); + }else{ + sqlite3_finalize(pCkHash); + sqlite3_finalize(pCkHashN); + sqlite3_finalize(pInsHash); + pCkHash = 0; + pInsHash = 0; + if( mxHash<=p->nPage ){ + runSql(p, "WITH RECURSIVE c(n) AS" + " (VALUES(%d) UNION ALL SELECT n+1 FROM c WHERE n<%d)" + " INSERT INTO badHash SELECT n, 1 FROM c", + mxHash, p->nPage); + } + runSql(p, "DELETE FROM badHash WHERE pgno=%d", lockBytePage); + pStmt = prepareStmt(p, + "SELECT pgno, data" + " FROM badHash JOIN sqlite_dbpage('main') USING(pgno)"); + if( pStmt==0 ) break; + while( sqlite3_step(pStmt)==SQLITE_ROW + && p->nErr==0 + && p->nWrErr==0 + ){ + unsigned int pgno = (unsigned int)sqlite3_column_int64(pStmt,0); + const void *pContent = sqlite3_column_blob(pStmt, 1); + writeByte(p, ORIGIN_PAGE); + writeUint32(p, pgno); + writeBytes(p, szPg, pContent); + p->nPageSent++; + if( p->zDebugFile ){ + debugMessage(p, "-> ORIGIN_PAGE %u\n", pgno); + } + } + sqlite3_finalize(pStmt); + writeByte(p, ORIGIN_TXN); + writeUint32(p, nPage); + if( p->zDebugFile ){ + debugMessage(p, "-> ORIGIN_TXN %u\n", nPage); + } + writeByte(p, ORIGIN_END); + } fflush(p->pOut); break; } @@ -1356,6 +1587,103 @@ static void originSide(SQLiteRsync *p){ closeDb(p); } +/* +** Send a REPLICA_HASH message for each entry in the sendHash table. +** The sendHash table looks like this: +** +** CREATE TABLE sendHash( +** fpg INTEGER PRIMARY KEY, -- Page number of the hash +** npg INT -- Number of pages in this hash +** ); +** +** If iHash is page number for the next page that the origin will +** be expecting, and nHash is the number of pages that the origin will +** be expecting in the hash that follows. Send a REPLICA_CONFIG message +** if either of these values if not correct. +*/ +static void sendHashMessages( + SQLiteRsync *p, /* The replica-side of the sync */ + unsigned int iHash, /* Next page expected by origin */ + unsigned int nHash /* Next number of pages expected by origin */ +){ + sqlite3_stmt *pStmt; + pStmt = prepareStmt(p, + "SELECT if(npg==1," + " (SELECT hash(data) FROM sqlite_dbpage('replica') WHERE pgno=fpg)," + " (WITH RECURSIVE c(n) AS" + " (SELECT fpg UNION ALL SELECT n+1 FROM c WHERE nnErr==0 && p->nWrErr==0 ){ + const unsigned char *a = sqlite3_column_blob(pStmt, 0); + unsigned int pgno = (unsigned int)sqlite3_column_int64(pStmt, 1); + unsigned int npg = (unsigned int)sqlite3_column_int64(pStmt, 2); + if( pgno!=iHash || npg!=nHash ){ + writeByte(p, REPLICA_CONFIG); + writeUint32(p, pgno); + writeUint32(p, npg); + if( p->zDebugFile ){ + debugMessage(p, "-> REPLICA_CONFIG %u %u\n", pgno, npg); + } + } + if( a==0 ){ + if( p->zDebugFile ){ + debugMessage(p, "# Oops: No hash for %u %u\n", pgno, npg); + } + }else{ + writeByte(p, REPLICA_HASH); + writeBytes(p, 20, a); + if( p->zDebugFile ){ + debugMessage(p, "-> REPLICA_HASH %u %u (%08x...)\n", + pgno, npg, *(unsigned int*)a); + } + } + p->nHashSent++; + iHash = pgno + npg; + nHash = npg; + } + sqlite3_finalize(pStmt); + runSql(p, "DELETE FROM sendHash"); + writeByte(p, REPLICA_READY); + fflush(p->pOut); + p->nRound++; + if( p->zDebugFile ) debugMessage(p, "-> REPLICA_READY\n", iHash); +} + +/* +** Make entries in the sendHash table to send hashes for +** npg (mnemonic: Number of PaGes) pages starting with fpg +** (mnemonic: First PaGe). +*/ +static void subdivideHashRange( + SQLiteRsync *p, /* The replica-side of the sync */ + unsigned int fpg, /* First page of the range */ + unsigned int npg /* Number of pages */ +){ + unsigned int nChunk; /* How many pages to request per hash */ + sqlite3_uint64 iEnd; /* One more than the last page */ + if( npg<=30 ){ + nChunk = 1; + }else if( npg<=1000 ){ + nChunk = 30; + }else{ + nChunk = 1000; + } + iEnd = fpg; + iEnd += npg; + runSql(p, + "WITH RECURSIVE c(n) AS" + " (VALUES(%u) UNION ALL SELECT n+%u FROM c WHERE n<%llu)" + "REPLACE INTO sendHash(fpg,npg)" + " SELECT n, min(%llu-n,%u) FROM c", + fpg, nChunk, iEnd-nChunk, iEnd, nChunk + ); +} + /* ** Run the replica-side protocol. The protocol is passive in the sense ** that it only response to message from the origin side. @@ -1366,15 +1694,35 @@ static void originSide(SQLiteRsync *p){ ** each page in the origin database (sent as a single-byte power-of-2), ** and the number of pages in the origin database. ** This procedure checks compatibility, and if everything is ok, -** it starts sending hashes of pages already present back to the origin. +** it starts sending hashes back to the origin using REPLICA_HASH +** and/or REPLICA_CONFIG message, followed by a single REPLICA_READY. +** REPLICA_CONFIG is only sent if the protocol is 2 or greater. +** +** ORIGIN_ERROR size text +** +** Report an error and quit. +** +** ORIGIN_DETAIL pgno cnt ** -** ORIGIN_ERROR size text +** The origin reports that a multi-page hash starting at pgno and +** spanning cnt pages failed to match. The origin is requesting +** details (more REPLICA_HASH message with a smaller cnt). The +** replica must wait on ORIGIN_READY before sending its reply. ** -** Report the received error and quit. +** ORIGIN_READY ** -** ORIGIN_PAGE pgno content +** After sending one or more ORIGIN_DETAIL messages, the ORIGIN_READY +** is sent by the origin to indicate that it has finished sending +** requests for detail and is ready for the replicate to reply +** with a new round of REPLICA_CONFIG and REPLICA_HASH messages. ** -** Update the content of the given page. +** ORIGIN_PAGE pgno content +** +** Once the origin believes it knows exactly which pages need to be +** updated in the replica, it starts sending those pages using these +** messages. These messages will only appear immediately after +** REPLICA_READY. The origin never mixes ORIGIN_DETAIL and +** ORIGIN_PAGE messages in the same batch. ** ** ORIGIN_TXN pgno ** @@ -1389,15 +1737,17 @@ static void replicaSide(SQLiteRsync *p){ int c; sqlite3_stmt *pIns = 0; unsigned int szOPage = 0; + char eJMode = 0; /* Journal mode prior to sync */ char buf[65536]; p->isReplica = 1; if( p->bCommCheck ){ infoMsg(p, "replica zOrigin=%Q zReplica=%Q isRemote=%d protocol=%d", - p->zOrigin, p->zReplica, p->isRemote, PROTOCOL_VERSION); + p->zOrigin, p->zReplica, p->isRemote, p->iProtocol); writeByte(p, REPLICA_END); fflush(p->pOut); } + if( p->iProtocol<=0 ) p->iProtocol = PROTOCOL_VERSION; /* Respond to message from the origin. The origin will initiate the ** the conversation with an ORIGIN_BEGIN message. @@ -1413,22 +1763,31 @@ static void replicaSide(SQLiteRsync *p){ unsigned int nOPage = 0; unsigned int nRPage = 0, szRPage = 0; int rc = 0; - sqlite3_stmt *pStmt = 0; + u8 iProtocol; closeDb(p); - p->iProtocol = readByte(p); + iProtocol = readByte(p); szOPage = readPow2(p); readUint32(p, &nOPage); + if( p->zDebugFile ){ + debugMessage(p, "<- ORIGIN_BEGIN %d %d %u\n", iProtocol, szOPage, + nOPage); + } if( p->nErr ) break; - if( p->iProtocol>PROTOCOL_VERSION ){ + if( iProtocol>p->iProtocol ){ /* If the protocol version on the origin side is larger, send back ** a REPLICA_BEGIN message with the protocol version number of the ** replica side. This gives the origin an opportunity to resend ** a new ORIGIN_BEGIN with a reduced protocol version. */ writeByte(p, REPLICA_BEGIN); - writeByte(p, PROTOCOL_VERSION); + writeByte(p, p->iProtocol); + fflush(p->pOut); + if( p->zDebugFile ){ + debugMessage(p, "-> REPLICA_BEGIN %u\n", p->iProtocol); + } break; } + p->iProtocol = iProtocol; p->nPage = nOPage; p->szPage = szOPage; rc = sqlite3_open(":memory:", &p->db); @@ -1453,20 +1812,30 @@ static void replicaSide(SQLiteRsync *p){ closeDb(p); break; } + runSql(p, + "CREATE TABLE sendHash(" + " fpg INTEGER PRIMARY KEY," /* The page number of hash to send */ + " npg INT" /* Number of pages in this hash */ + ")" + ); hashRegister(p->db); if( runSqlReturnUInt(p, &nRPage, "PRAGMA replica.page_count") ){ break; } if( nRPage==0 ){ runSql(p, "PRAGMA replica.page_size=%u", szOPage); - runSql(p, "PRAGMA replica.journal_mode=WAL"); runSql(p, "SELECT * FROM replica.sqlite_schema"); } runSql(p, "BEGIN IMMEDIATE"); runSqlReturnText(p, buf, "PRAGMA replica.journal_mode"); if( strcmp(buf, "wal")!=0 ){ - reportError(p, "replica is not in WAL mode"); - break; + if( p->bWalOnly && nRPage>0 ){ + reportError(p, "replica is not in WAL mode"); + break; + } + eJMode = 1; /* Non-WAL mode prior to sync */ + }else{ + eJMode = 2; /* WAL-mode prior to sync */ } runSqlReturnUInt(p, &nRPage, "PRAGMA replica.page_count"); runSqlReturnUInt(p, &szRPage, "PRAGMA replica.page_size"); @@ -1475,26 +1844,43 @@ static void replicaSide(SQLiteRsync *p){ "replica is %d bytes", szOPage, szRPage); break; } - - pStmt = prepareStmt(p, - "SELECT hash(data) FROM sqlite_dbpage('replica')" - " WHERE pgno<=min(%d,%d)" - " ORDER BY pgno", nRPage, nOPage); - while( sqlite3_step(pStmt)==SQLITE_ROW && p->nErr==0 && p->nWrErr==0 ){ - const unsigned char *a = sqlite3_column_blob(pStmt, 0); - writeByte(p, REPLICA_HASH); - writeBytes(p, 20, a); - p->nHashSent++; + if( p->iProtocol<2 || nRPage<=100 ){ + runSql(p, + "WITH RECURSIVE c(n) AS" + "(VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<%d)" + "INSERT INTO sendHash(fpg, npg) SELECT n, 1 FROM c", + nRPage); + }else{ + runSql(p,"INSERT INTO sendHash VALUES(1,1)"); + subdivideHashRange(p, 2, nRPage); } - sqlite3_finalize(pStmt); - writeByte(p, REPLICA_READY); - fflush(p->pOut); + sendHashMessages(p, 1, 1); runSql(p, "PRAGMA writable_schema=ON"); break; } + case ORIGIN_DETAIL: { + unsigned int fpg, npg; + readUint32(p, &fpg); + readUint32(p, &npg); + if( p->zDebugFile ){ + debugMessage(p, "<- ORIGIN_DETAIL %u %u\n", fpg, npg); + } + subdivideHashRange(p, fpg, npg); + break; + } + case ORIGIN_READY: { + if( p->zDebugFile ){ + debugMessage(p, "<- ORIGIN_READY\n"); + } + sendHashMessages(p, 0, 0); + break; + } case ORIGIN_TXN: { unsigned int nOPage = 0; readUint32(p, &nOPage); + if( p->zDebugFile ){ + debugMessage(p, "<- ORIGIN_TXN %u\n", nOPage); + } if( pIns==0 ){ /* Nothing has changed */ runSql(p, "COMMIT"); @@ -1522,6 +1908,9 @@ static void replicaSide(SQLiteRsync *p){ unsigned int pgno = 0; int rc; readUint32(p, &pgno); + if( p->zDebugFile ){ + debugMessage(p, "<- ORIGIN_PAGE %u\n", pgno); + } if( p->nErr ) break; if( pIns==0 ){ pIns = prepareStmt(p, @@ -1531,6 +1920,11 @@ static void replicaSide(SQLiteRsync *p){ } readBytes(p, szOPage, buf); if( p->nErr ) break; + if( pgno==1 && eJMode==2 && buf[18]==1 ){ + /* Do not switch the replica out of WAL mode if it started in + ** WAL mode */ + buf[18] = buf[19] = 2; + } p->nPageSent++; sqlite3_bind_int64(pIns, 1, pgno); sqlite3_bind_blob(pIns, 2, buf, szOPage, SQLITE_STATIC); @@ -1619,9 +2013,9 @@ static char *hostSeparator(const char *zIn){ zIn++; } return zPath; - } + /* ** Parse command-line arguments. Dispatch subroutines to do the ** requested work. @@ -1664,16 +2058,19 @@ int main(int argc, char const * const *argv){ sqlite3_int64 tmEnd; sqlite3_int64 tmElapse; const char *zRemoteErrFile = 0; + const char *zRemoteDebugFile = 0; #define cli_opt_val cmdline_option_value(argc, argv, ++i) memset(&ctx, 0, sizeof(ctx)); + ctx.iProtocol = PROTOCOL_VERSION; for(i=1; iPROTOCOL_VERSION ){ + ctx.iProtocol = PROTOCOL_VERSION; + } + continue; + } + if( strcmp(z, "-ssh")==0 ){ zSsh = cli_opt_val; continue; } - if( strcmp(z, "--exe")==0 ){ + if( strcmp(z, "-exe")==0 ){ zExe = cli_opt_val; continue; } - if( strcmp(z, "--logfile")==0 ){ + if( strcmp(z, "-wal-only")==0 ){ + ctx.bWalOnly = 1; + continue; + } + if( strcmp(z, "-version")==0 ){ + printf("%s\n", sqlite3_sourceid()); + return 0; + } + if( strcmp(z, "-help")==0 || strcmp(z, "--help")==0 + || strcmp(z, "-?")==0 + ){ + printf("%s", zUsage); + return 0; + } + if( strcmp(z, "-logfile")==0 ){ /* DEBUG OPTION: --logfile FILENAME ** Cause all local output traffic to be duplicated in FILENAME */ const char *zLog = cli_opt_val; @@ -1701,48 +2121,51 @@ int main(int argc, char const * const *argv){ } continue; } - if( strcmp(z, "--errorfile")==0 ){ + if( strcmp(z, "-errorfile")==0 ){ /* DEBUG OPTION: --errorfile FILENAME ** Error messages on the local side are written into FILENAME */ ctx.zErrFile = cli_opt_val; continue; } - if( strcmp(z, "--remote-errorfile")==0 ){ + if( strcmp(z, "-remote-errorfile")==0 ){ /* DEBUG OPTION: --remote-errorfile FILENAME ** Error messages on the remote side are written into FILENAME on ** the remote side. */ zRemoteErrFile = cli_opt_val; continue; } - if( strcmp(z, "-help")==0 || strcmp(z, "--help")==0 - || strcmp(z, "-?")==0 - ){ - printf("%s", zUsage); - return 0; + if( strcmp(z, "-debugfile")==0 ){ + /* DEBUG OPTION: --debugfile FILENAME + ** Debugging messages on the local side are written into FILENAME */ + ctx.zDebugFile = cli_opt_val; + continue; } - if( strcmp(z, "--version")==0 ){ - printf("%s\n", sqlite3_sourceid()); - return 0; + if( strcmp(z, "-remote-debugfile")==0 ){ + /* DEBUG OPTION: --remote-debugfile FILENAME + ** Error messages on the remote side are written into FILENAME on + ** the remote side. */ + zRemoteDebugFile = cli_opt_val; + continue; } - if( z[0]=='-' ){ - if( strcmp(z,"--commcheck")==0 ){ /* DEBUG ONLY */ - /* Run a communication check with the remote side. Do not attempt - ** to exchange any database connection */ - ctx.bCommCheck = 1; - continue; - } - if( strcmp(z,"--arg-escape-check")==0 ){ /* DEBUG ONLY */ - /* Test the append_escaped_arg() routine by using it to render a - ** copy of the input command-line, assuming all arguments except - ** this one are filenames. */ - sqlite3_str *pStr = sqlite3_str_new(0); - int k; - for(k=0; k=2 ) printf("%s\n", zCmd); - if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){ - fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd); - return 1; + /* Remote ORIGIN and local REPLICA */ + for(iRetry=0; 1 /*exit-via-break*/; iRetry++){ + sqlite3_str *pStr = sqlite3_str_new(0); + append_escaped_arg(pStr, zSsh, 1); + sqlite3_str_appendf(pStr, " -e none"); + append_escaped_arg(pStr, ctx.zOrigin, 0); + if( iRetry ) add_path_argument(pStr); + append_escaped_arg(pStr, zExe, 1); + append_escaped_arg(pStr, "--origin", 0); + if( ctx.bCommCheck ){ + append_escaped_arg(pStr, "--commcheck", 0); + if( ctx.eVerbose==0 ) ctx.eVerbose = 1; + } + if( zRemoteErrFile ){ + append_escaped_arg(pStr, "--errorfile", 0); + append_escaped_arg(pStr, zRemoteErrFile, 1); + } + if( zRemoteDebugFile ){ + append_escaped_arg(pStr, "--debugfile", 0); + append_escaped_arg(pStr, zRemoteDebugFile, 1); + } + if( ctx.bWalOnly ){ + append_escaped_arg(pStr, "--wal-only", 0); + } + append_escaped_arg(pStr, zDiv, 1); + append_escaped_arg(pStr, file_tail(ctx.zReplica), 1); + if( ctx.eVerbose<2 && iRetry==0 ){ + append_escaped_arg(pStr, "2>/dev/null", 0); + } + zCmd = sqlite3_str_finish(pStr); + if( ctx.eVerbose>=2 ) printf("%s\n", zCmd); + if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){ + if( iRetry>=1 ){ + fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd); + return 1; + } + if( ctx.eVerbose>=2 ){ + printf("ssh FAILED. Retry with a PATH= argument...\n"); + } + continue; + } + replicaSide(&ctx); + if( ctx.nHashSent==0 && iRetry==0 ) continue; + break; } - replicaSide(&ctx); }else if( (zDiv = hostSeparator(ctx.zReplica))!=0 ){ /* Local ORIGIN and remote REPLICA */ - sqlite3_str *pStr = sqlite3_str_new(0); - append_escaped_arg(pStr, zSsh, 1); - sqlite3_str_appendf(pStr, " -e none"); + int iRetry; *(zDiv++) = 0; - append_escaped_arg(pStr, ctx.zReplica, 0); - append_escaped_arg(pStr, zExe, 1); - append_escaped_arg(pStr, "--replica", 0); - if( ctx.bCommCheck ){ - append_escaped_arg(pStr, "--commcheck", 0); - if( ctx.eVerbose==0 ) ctx.eVerbose = 1; - } - if( zRemoteErrFile ){ - append_escaped_arg(pStr, "--errorfile", 0); - append_escaped_arg(pStr, zRemoteErrFile, 1); - } - append_escaped_arg(pStr, file_tail(ctx.zOrigin), 1); - append_escaped_arg(pStr, zDiv, 1); - zCmd = sqlite3_str_finish(pStr); - if( ctx.eVerbose>=2 ) printf("%s\n", zCmd); - if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){ - fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd); - return 1; + for(iRetry=0; 1 /*exit-by-break*/; iRetry++){ + sqlite3_str *pStr = sqlite3_str_new(0); + append_escaped_arg(pStr, zSsh, 1); + sqlite3_str_appendf(pStr, " -e none"); + append_escaped_arg(pStr, ctx.zReplica, 0); + if( iRetry==1 ) add_path_argument(pStr); + append_escaped_arg(pStr, zExe, 1); + append_escaped_arg(pStr, "--replica", 0); + if( ctx.bCommCheck ){ + append_escaped_arg(pStr, "--commcheck", 0); + if( ctx.eVerbose==0 ) ctx.eVerbose = 1; + } + if( zRemoteErrFile ){ + append_escaped_arg(pStr, "--errorfile", 0); + append_escaped_arg(pStr, zRemoteErrFile, 1); + } + if( zRemoteDebugFile ){ + append_escaped_arg(pStr, "--debugfile", 0); + append_escaped_arg(pStr, zRemoteDebugFile, 1); + } + if( ctx.bWalOnly ){ + append_escaped_arg(pStr, "--wal-only", 0); + } + append_escaped_arg(pStr, file_tail(ctx.zOrigin), 1); + append_escaped_arg(pStr, zDiv, 1); + if( ctx.eVerbose<2 && iRetry==0 ){ + append_escaped_arg(pStr, "2>/dev/null", 0); + } + zCmd = sqlite3_str_finish(pStr); + if( ctx.eVerbose>=2 ) printf("%s\n", zCmd); + if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){ + if( iRetry>=1 ){ + fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd); + return 1; + }else if( ctx.eVerbose>=2 ){ + printf("ssh FAILED. Retry with a PATH= argument...\n"); + } + continue; + } + originSide(&ctx); + if( ctx.nHashSent==0 && iRetry==0 ) continue; + break; } - originSide(&ctx); }else{ /* Local ORIGIN and REPLICA */ sqlite3_str *pStr = sqlite3_str_new(0); @@ -1866,6 +2332,10 @@ int main(int argc, char const * const *argv){ append_escaped_arg(pStr, "--errorfile", 0); append_escaped_arg(pStr, zRemoteErrFile, 1); } + if( zRemoteDebugFile ){ + append_escaped_arg(pStr, "--debugfile", 0); + append_escaped_arg(pStr, zRemoteDebugFile, 1); + } append_escaped_arg(pStr, ctx.zOrigin, 1); append_escaped_arg(pStr, ctx.zReplica, 1); zCmd = sqlite3_str_finish(pStr); @@ -1909,6 +2379,11 @@ int main(int argc, char const * const *argv){ printf("%s\n", zMsg); sqlite3_free(zMsg); } + if( ctx.eVerbose>=3 ){ + printf("hashes: %lld hash-rounds: %d" + " page updates: %d protocol: %d\n", + ctx.nHashSent, ctx.nRound, ctx.nPageSent, ctx.iProtocol); + } } sqlite3_free(zCmd); if( pIn!=0 && pOut!=0 ){ diff --git a/tool/src-verify.c b/tool/src-verify.c index 0c7ed6f4c..6dc9f7259 100644 --- a/tool/src-verify.c +++ b/tool/src-verify.c @@ -752,7 +752,7 @@ void sha1sum_file(const char *zFilename, char *zCksum){ SHA1Final(zResult, &ctx); DigestToBase16(zResult, zCksum, 20); } - + /* ** Decode a fossilized string in-place. */ diff --git a/tool/srcck1.c b/tool/srcck1.c index 20084ac47..5a1158beb 100644 --- a/tool/srcck1.c +++ b/tool/srcck1.c @@ -13,7 +13,7 @@ ** The aim of this utility is to prevent recurrences of errors such ** as the one fixed at: ** -** https://www.sqlite.org/src/info/a2952231ac7abe16 +** https://sqlite.org/src/info/a2952231ac7abe16 ** ** Note that another similar error was found by this utility when it was ** first written. That other error was fixed by the same check-in that diff --git a/tool/srctree-check.tcl b/tool/srctree-check.tcl index 027b19370..921bb0976 100644 --- a/tool/srctree-check.tcl +++ b/tool/srctree-check.tcl @@ -4,8 +4,7 @@ # various aspects of the source tree are up-to-date. Items checked include: # # * Makefile.msc and autoconf/Makefile.msc agree -# * src/ctime.tcl is consistent with tool/mkctimec.tcl -# * src/pragma.h agrees with tool/mkpragmatab.tcl +# * VERSION agrees with autoconf/tea/configure.ac # # Other tests might be added later. # @@ -46,31 +45,3 @@ if {$f1 != $f2} { puts "...... Fix: tclsh tool/mkmsvcmin.tcl" incr NERR } - -######################### src/pragma.h ######################################## - -set f1 [readfile $ROOT/src/pragma.h] -exec $TCLSH $ROOT/tool/mkpragmatab.tcl tmp2.txt -set f2 [readfile tmp2.txt] -file delete tmp2.txt -if {$f1 != $f2} { - puts "ERROR: ./src/pragma.h does not agree with ./tool/mkpragmatab.tcl" - puts "...... Fix: tclsh tool/mkpragmatab.tcl" - incr NERR -} - -######################### src/ctime.c ######################################## - -set f1 [readfile $ROOT/src/ctime.c] -exec $TCLSH $ROOT/tool/mkctimec.tcl tmp3.txt -set f2 [readfile tmp3.txt] -file delete tmp3.txt -if {$f1 != $f2} { - puts "ERROR: ./src/ctime.c does not agree with ./tool/mkctimec.tcl" - puts "..... Fix: tclsh tool/mkctimec.tcl" - incr NERR -} - -# If any errors are seen, exit 1 so that the build will fail. -# -if {$NERR>0} {exit 1} diff --git a/tool/stripccomments.c b/tool/stripccomments.c index 1bdb5c6b8..1d8525278 100644 --- a/tool/stripccomments.c +++ b/tool/stripccomments.c @@ -117,7 +117,7 @@ void do_it_all(void){ https://github.com/emscripten-core/emscripten/issues/23412 - Such regexes will always necessarily be preceeded by a + Such regexes will always necessarily be preceded by a backslash, though. It is hypothetically possible for a legitimate comment diff --git a/tool/tclConfigShToAutoDef.sh b/tool/tclConfigShToAutoDef.sh deleted file mode 100755 index d12657063..000000000 --- a/tool/tclConfigShToAutoDef.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -# -# A level of indirection for use soley by the configure script -# (auto.def). -# -# Expects to be passed a full path to a tclConfig.sh. It sources it -# and emits TCL code which sets some vars which are exported by -# tclConfig.sh. -# -# This script expects that the caller has already validated that the -# file exists, is not a directory, and is readable. -# -# If passed no filename, or an empty one, then it emits config code -# suitable for the "config not found" case. -if test x = "x$1"; then - TCL_INCLUDE_SPEC= - TCL_LIB_SPEC= - TCL_STUB_LIB_SPEC= - TCL_EXEC_PREFIX= - TCL_VERSION= -else - . "$1" -fi - -echo "define TCL_INCLUDE_SPEC {$TCL_INCLUDE_SPEC} ;" -echo "define TCL_LIB_SPEC {$TCL_LIB_SPEC} ;" -echo "define TCL_STUB_LIB_SPEC {$TCL_STUB_LIB_SPEC} ;" -echo "define TCL_EXEC_PREFIX {$TCL_EXEC_PREFIX} ;" -echo "define TCL_VERSION {$TCL_VERSION} ;" diff --git a/tool/warnings.sh b/tool/warnings.sh index 2b962d15e..b589780ea 100644 --- a/tool/warnings.sh +++ b/tool/warnings.sh @@ -10,7 +10,7 @@ if uname | grep -i openbsd ; then WARNING_ANDROID_OPTS=-Wall else # Use these for testing on Linux and Mac OSX: - WARNING_OPTS="-Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long" + WARNING_OPTS="-Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -Wno-array-bounds" gccvers=`gcc -v 2>&1 | grep '^gcc version'` if test "$gccvers" '<' 'gcc version 6' then @@ -22,15 +22,15 @@ fi rm -f sqlite3.c make sqlite3.c -echo '********** No optimizations. Includes FTS4/5, GEOPOLY, JSON1 ***' -echo '********** ' Options: $WARNING_OPTS +echo '**** No optimizations. Includes FTS4/5, GEOPOLY, JSON1 ***' +echo '****' $WARNING_OPTS gcc -c $WARNING_OPTS -std=c89 \ -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_GEOPOLY \ -DSQLITE_ENABLE_FTS5 \ sqlite3.c if test x`uname` = 'xLinux'; then -echo '********** Android configuration ******************************' -echo '********** ' Options: $WARNING_ANDROID_OPTS +echo '**** Android configuration ******************************' +echo '****' $WARNING_ANDROID_OPTS gcc -c \ -DSQLITE_HAVE_ISNAN \ -DSQLITE_DEFAULT_JOURNAL_SIZE_LIMIT=1048576 \ @@ -52,13 +52,13 @@ gcc -c \ $WARNING_ANDROID_OPTS \ -Os sqlite3.c shell.c fi -echo '********** No optimizations. ENABLE_STAT4. THREADSAFE=0 *******' -echo '********** ' Options: $WARNING_OPTS +echo '**** No optimizations. ENABLE_STAT4. THREADSAFE=0 *******' +echo '****' $WARNING_OPTS gcc -c $WARNING_OPTS -std=c89 \ -ansi -DSQLITE_ENABLE_STAT4 -DSQLITE_THREADSAFE=0 \ sqlite3.c -echo '********** Optimized -O3. Includes FTS4/5, GEOPOLY, JSON1 ******' -echo '********** ' Options: $WARNING_OPTS +echo '**** Optimized -O3. Includes FTS4/5, GEOPOLY, JSON1 ******' +echo '****' $WARNING_OPTS gcc -O3 -c $WARNING_OPTS -std=c89 \ -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_GEOPOLY \ -DSQLITE_ENABLE_FTS5 \ From 1eef2b438442c46cf40d690530d7b19c9c40ceed Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 20 Jun 2025 14:23:50 -0400 Subject: [PATCH 23/75] Fixes issue building with -fsantize=address on macOS --- src/sqlcipher.c | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 6eaa5afcb..884f7bb4f 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -388,21 +388,26 @@ static void sqlcipher_fini(void) { } #if defined(_WIN32) -#ifndef SQLCIPHER_OMIT_DLLMAIN -BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) { - switch (fdwReason) { - case DLL_PROCESS_DETACH: - sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: calling sqlcipher_extra_shutdown()", __func__); - sqlcipher_extra_shutdown(); - break; - default: - break; + #ifndef SQLCIPHER_OMIT_DLLMAIN + BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) { + switch (fdwReason) { + case DLL_PROCESS_DETACH: + sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "%s: calling sqlcipher_extra_shutdown()", __func__); + sqlcipher_extra_shutdown(); + break; + default: + break; + } + return TRUE; } - return TRUE; -} -#endif + #endif #elif defined(__APPLE__) -static void (*const sqlcipher_fini_func)(void) __attribute__((used, section("__DATA,__mod_term_func"))) = sqlcipher_fini; + #if !defined(__has_feature) || !__has_feature(address_sanitizer) + static void (*const sqlcipher_fini_func)(void) __attribute__((used, section("__DATA,__mod_term_func"))) = sqlcipher_fini; + #else + static void sqlcipher_cleanup_destructor(void) __attribute__((destructor)); + static void sqlcipher_cleanup_destructor(void) { sqlcipher_fini(); } + #endif #else static void (*const sqlcipher_fini_func)(void) __attribute__((used, section(".fini_array"))) = sqlcipher_fini; #endif From d83817ecdd0c25c824f446574488895d8ebb3443 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Mon, 30 Jun 2025 11:51:48 -0400 Subject: [PATCH 24/75] Snapshot of upstream SQLite 3.50.2 --- VERSION | 2 +- ext/fts5/fts5_storage.c | 23 +++-- ext/fts5/test/fts5faultI.test | 20 +++++ ext/fts5/test/fts5unicode4.test | 30 +++++++ ext/misc/fileio.c | 15 ++-- ext/wasm/GNUmakefile | 8 +- manifest | 48 +++++------ manifest.uuid | 2 +- src/bitvec.c | 4 +- src/expr.c | 16 +++- src/func.c | 6 +- src/main.c | 2 + src/sqlite.h.in | 90 +++++++++---------- src/sqliteInt.h | 10 +-- src/wal.c | 4 + src/whereexpr.c | 36 +++++--- test/func9.test | 3 + test/join.test | 27 ++++++ test/walcksum.test | 148 ++++++++++++++++++++++++++++++++ test/walsetlk2.test | 2 + 20 files changed, 389 insertions(+), 107 deletions(-) diff --git a/VERSION b/VERSION index b05723fc6..18d0478a0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.50.1 +3.50.2 diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 2b43016be..76820e85b 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -539,6 +539,7 @@ static int fts5StorageDeleteFromIndex( for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){ if( pConfig->abUnindexed[iCol-1]==0 ){ sqlite3_value *pVal = 0; + sqlite3_value *pFree = 0; const char *pText = 0; int nText = 0; const char *pLoc = 0; @@ -555,11 +556,22 @@ static int fts5StorageDeleteFromIndex( if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); }else{ - pText = (const char*)sqlite3_value_text(pVal); - nText = sqlite3_value_bytes(pVal); - if( pConfig->bLocale && pSeek ){ - pLoc = (const char*)sqlite3_column_text(pSeek, iCol + pConfig->nCol); - nLoc = sqlite3_column_bytes(pSeek, iCol + pConfig->nCol); + if( sqlite3_value_type(pVal)!=SQLITE_TEXT ){ + /* Make a copy of the value to work with. This is because the call + ** to sqlite3_value_text() below forces the type of the value to + ** SQLITE_TEXT, and we may need to use it again later. */ + pFree = pVal = sqlite3_value_dup(pVal); + if( pVal==0 ){ + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + if( pConfig->bLocale && pSeek ){ + pLoc = (const char*)sqlite3_column_text(pSeek, iCol+pConfig->nCol); + nLoc = sqlite3_column_bytes(pSeek, iCol + pConfig->nCol); + } } } @@ -575,6 +587,7 @@ static int fts5StorageDeleteFromIndex( } sqlite3Fts5ClearLocale(pConfig); } + sqlite3_value_free(pFree); } } if( rc==SQLITE_OK && p->nTotalRow<1 ){ diff --git a/ext/fts5/test/fts5faultI.test b/ext/fts5/test/fts5faultI.test index ab84d37de..a2b04af8f 100644 --- a/ext/fts5/test/fts5faultI.test +++ b/ext/fts5/test/fts5faultI.test @@ -325,5 +325,25 @@ ifcapable foreignkey { } } +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 13.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b); + INSERT INTO t1 VALUES('abc def', X'123456'); +} +faultsim_save_and_close + + +do_faultsim_test 13 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + UPDATE t1 SET a='def abc' + } +} -test { + faultsim_test_result {0 {}} +} + finish_test diff --git a/ext/fts5/test/fts5unicode4.test b/ext/fts5/test/fts5unicode4.test index dc225cb5e..f006d6c0a 100644 --- a/ext/fts5/test/fts5unicode4.test +++ b/ext/fts5/test/fts5unicode4.test @@ -28,4 +28,34 @@ do_execsql_test 1.1 { INSERT INTO sss VALUES('まりや'); } +foreach {tn enc tok} { + 1 utf-8 ascii + 2 utf-16 ascii + 3 utf-8 unicode61 + 4 utf-16 unicode61 +} { + reset_db + + do_execsql_test 1.$tn.0 " + PRAGMA encoding = '$enc'; + CREATE VIRTUAL TABLE vt2 USING fts5(c0, c1, tokenize=$tok); + " + + do_execsql_test 1.$tn.1 { + INSERT INTO vt2(c0, c1) VALUES ('bhal', x'17db'); + } + + do_execsql_test 1.$tn.2 { + UPDATE vt2 SET c0='bhal'; + } + + do_execsql_test 1.$tn.3 { + INSERT INTO vt2(vt2) VALUES('integrity-check') + } + + do_execsql_test 1.$tn.4 { + SELECT quote(c1) FROM vt2 + } {X'17DB'} +} + finish_test diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c index 96a7f82bd..03c911712 100644 --- a/ext/misc/fileio.c +++ b/ext/misc/fileio.c @@ -92,13 +92,14 @@ SQLITE_EXTENSION_INIT1 # include # include # include +# define STRUCT_STAT struct stat #else # include "windows.h" # include # include # include "test_windirent.h" # define dirent DIRENT -# define stat _stat +# define STRUCT_STAT struct _stat # define chmod(path,mode) fileio_chmod(path,mode) # define mkdir(path,mode) fileio_mkdir(path) #endif @@ -289,7 +290,7 @@ LPWSTR utf8_to_utf16(const char *z){ */ static void statTimesToUtc( const char *zPath, - struct stat *pStatBuf + STRUCT_STAT *pStatBuf ){ HANDLE hFindFile; WIN32_FIND_DATAW fd; @@ -317,7 +318,7 @@ static void statTimesToUtc( */ static int fileStat( const char *zPath, - struct stat *pStatBuf + STRUCT_STAT *pStatBuf ){ #if defined(_WIN32) sqlite3_int64 sz = strlen(zPath); @@ -341,7 +342,7 @@ static int fileStat( */ static int fileLinkStat( const char *zPath, - struct stat *pStatBuf + STRUCT_STAT *pStatBuf ){ #if defined(_WIN32) return fileStat(zPath, pStatBuf); @@ -374,7 +375,7 @@ static int makeDirectory( int i = 1; while( rc==SQLITE_OK ){ - struct stat sStat; + STRUCT_STAT sStat; int rc2; for(; zCopy[i]!='/' && iu.aHash, sizeof(p->u.aHash)); memset(p->u.apSub, 0, sizeof(p->u.apSub)); - p->iDivisor = (p->iSize + BITVEC_NPTR - 1)/BITVEC_NPTR; + p->iDivisor = p->iSize/BITVEC_NPTR; + if( (p->iSize%BITVEC_NPTR)!=0 ) p->iDivisor++; + if( p->iDivisoriDivisor = BITVEC_NBIT; rc = sqlite3BitvecSet(p, i); for(j=0; jdb->aLimit[SQLITE_LIMIT_COLUMN]; + assert( mxTerm <= SMXV(i16) ); assert( pAggInfo->iFirstReg==0 ); pCol = pAggInfo->aCol; for(k=0; knColumn; k++, pCol++){ @@ -7028,6 +7030,10 @@ static void findOrCreateAggInfoColumn( assert( pParse->db->mallocFailed ); return; } + if( k>mxTerm ){ + sqlite3ErrorMsg(pParse, "more than %d aggregate terms", mxTerm); + k = mxTerm; + } pCol = &pAggInfo->aCol[k]; assert( ExprUseYTab(pExpr) ); pCol->pTab = pExpr->y.pTab; @@ -7061,6 +7067,7 @@ static void findOrCreateAggInfoColumn( if( pExpr->op==TK_COLUMN ){ pExpr->op = TK_AGG_COLUMN; } + assert( k <= SMXV(pExpr->iAgg) ); pExpr->iAgg = (i16)k; } @@ -7145,13 +7152,19 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ ** function that is already in the pAggInfo structure */ struct AggInfo_func *pItem = pAggInfo->aFunc; + int mxTerm = pParse->db->aLimit[SQLITE_LIMIT_COLUMN]; + assert( mxTerm <= SMXV(i16) ); for(i=0; inFunc; i++, pItem++){ if( NEVER(pItem->pFExpr==pExpr) ) break; if( sqlite3ExprCompare(0, pItem->pFExpr, pExpr, -1)==0 ){ break; } } - if( i>=pAggInfo->nFunc ){ + if( i>mxTerm ){ + sqlite3ErrorMsg(pParse, "more than %d aggregate terms", mxTerm); + i = mxTerm; + assert( inFunc ); + }else if( i>=pAggInfo->nFunc ){ /* pExpr is original. Make a new entry in pAggInfo->aFunc[] */ u8 enc = ENC(pParse->db); @@ -7205,6 +7218,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ */ assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); ExprSetVVAProperty(pExpr, EP_NoReduce); + assert( i <= SMXV(pExpr->iAgg) ); pExpr->iAgg = (i16)i; pExpr->pAggInfo = pAggInfo; return WRC_Prune; diff --git a/src/func.c b/src/func.c index 9e2839336..b0a1359b7 100644 --- a/src/func.c +++ b/src/func.c @@ -1667,7 +1667,7 @@ static void concatFuncCore( int nSep, const char *zSep ){ - i64 j, k, n = 0; + i64 j, n = 0; int i; char *z; for(i=0; i0 ){ + if( sqlite3_value_type(argv[i])!=SQLITE_NULL ){ + int k = sqlite3_value_bytes(argv[i]); const char *v = (const char*)sqlite3_value_text(argv[i]); if( v!=0 ){ if( j>0 && nSep>0 ){ diff --git a/src/main.c b/src/main.c index 9b67de815..0dd9926fe 100644 --- a/src/main.c +++ b/src/main.c @@ -1861,6 +1861,7 @@ int sqlite3_setlk_timeout(sqlite3 *db, int ms, int flags){ #endif if( ms<-1 ) return SQLITE_RANGE; #ifdef SQLITE_ENABLE_SETLK_TIMEOUT + sqlite3_mutex_enter(db->mutex); db->setlkTimeout = ms; db->setlkFlags = flags; sqlite3BtreeEnterAll(db); @@ -1872,6 +1873,7 @@ int sqlite3_setlk_timeout(sqlite3 *db, int ms, int flags){ } } sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); #endif #if !defined(SQLITE_ENABLE_API_ARMOR) && !defined(SQLITE_ENABLE_SETLK_TIMEOUT) UNUSED_PARAMETER(db); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index d3164d906..4a79945f3 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -4079,7 +4079,7 @@ sqlite3_file *sqlite3_database_file_object(const char*); ** ** The sqlite3_create_filename(D,J,W,N,P) allocates memory to hold a version of ** database filename D with corresponding journal file J and WAL file W and -** with N URI parameters key/values pairs in the array P. The result from +** an array P of N URI Key/Value pairs. The result from ** sqlite3_create_filename(D,J,W,N,P) is a pointer to a database filename that ** is safe to pass to routines like: **
      @@ -4760,7 +4760,7 @@ typedef struct sqlite3_context sqlite3_context; ** METHOD: sqlite3_stmt ** ** ^(In the SQL statement text input to [sqlite3_prepare_v2()] and its variants, -** literals may be replaced by a [parameter] that matches one of following +** literals may be replaced by a [parameter] that matches one of the following ** templates: ** **
        @@ -4805,7 +4805,7 @@ typedef struct sqlite3_context sqlite3_context; ** ** [[byte-order determination rules]] ^The byte-order of ** UTF16 input text is determined by the byte-order mark (BOM, U+FEFF) -** found in first character, which is removed, or in the absence of a BOM +** found in the first character, which is removed, or in the absence of a BOM ** the byte order is the native byte order of the host ** machine for sqlite3_bind_text16() or the byte order specified in ** the 6th parameter for sqlite3_bind_text64().)^ @@ -4825,7 +4825,7 @@ typedef struct sqlite3_context sqlite3_context; ** or sqlite3_bind_text16() or sqlite3_bind_text64() then ** that parameter must be the byte offset ** where the NUL terminator would occur assuming the string were NUL -** terminated. If any NUL characters occurs at byte offsets less than +** terminated. If any NUL characters occur at byte offsets less than ** the value of the fourth parameter then the resulting string value will ** contain embedded NULs. The result of expressions involving strings ** with embedded NULs is undefined. @@ -5037,7 +5037,7 @@ const void *sqlite3_column_name16(sqlite3_stmt*, int N); ** METHOD: sqlite3_stmt ** ** ^These routines provide a means to determine the database, table, and -** table column that is the origin of a particular result column in +** table column that is the origin of a particular result column in a ** [SELECT] statement. ** ^The name of the database or table or column can be returned as ** either a UTF-8 or UTF-16 string. ^The _database_ routines return @@ -5606,8 +5606,8 @@ int sqlite3_reset(sqlite3_stmt *pStmt); ** ** For best security, the [SQLITE_DIRECTONLY] flag is recommended for ** all application-defined SQL functions that do not need to be -** used inside of triggers, view, CHECK constraints, or other elements of -** the database schema. This flags is especially recommended for SQL +** used inside of triggers, views, CHECK constraints, or other elements of +** the database schema. This flag is especially recommended for SQL ** functions that have side effects or reveal internal application state. ** Without this flag, an attacker might be able to modify the schema of ** a database file to include invocations of the function with parameters @@ -5638,7 +5638,7 @@ int sqlite3_reset(sqlite3_stmt *pStmt); ** [user-defined window functions|available here]. ** ** ^(If the final parameter to sqlite3_create_function_v2() or -** sqlite3_create_window_function() is not NULL, then it is destructor for +** sqlite3_create_window_function() is not NULL, then it is the destructor for ** the application data pointer. The destructor is invoked when the function ** is deleted, either by being overloaded or when the database connection ** closes.)^ ^The destructor is also invoked if the call to @@ -6038,7 +6038,7 @@ unsigned int sqlite3_value_subtype(sqlite3_value*); ** METHOD: sqlite3_value ** ** ^The sqlite3_value_dup(V) interface makes a copy of the [sqlite3_value] -** object D and returns a pointer to that copy. ^The [sqlite3_value] returned +** object V and returns a pointer to that copy. ^The [sqlite3_value] returned ** is a [protected sqlite3_value] object even if the input is not. ** ^The sqlite3_value_dup(V) interface returns NULL if V is NULL or if a ** memory allocation fails. ^If V is a [pointer value], then the result @@ -6076,7 +6076,7 @@ void sqlite3_value_free(sqlite3_value*); ** allocation error occurs. ** ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is -** determined by the N parameter on first successful call. Changing the +** determined by the N parameter on the first successful call. Changing the ** value of N in any subsequent call to sqlite3_aggregate_context() within ** the same aggregate function instance will not resize the memory ** allocation.)^ Within the xFinal callback, it is customary to set @@ -6238,7 +6238,7 @@ void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); ** ** Security Warning: These interfaces should not be exposed in scripting ** languages or in other circumstances where it might be possible for an -** an attacker to invoke them. Any agent that can invoke these interfaces +** attacker to invoke them. Any agent that can invoke these interfaces ** can probably also take control of the process. ** ** Database connection client data is only available for SQLite @@ -6352,7 +6352,7 @@ typedef void (*sqlite3_destructor_type)(void*); ** pointed to by the 2nd parameter are taken as the application-defined ** function result. If the 3rd parameter is non-negative, then it ** must be the byte offset into the string where the NUL terminator would -** appear if the string where NUL terminated. If any NUL characters occur +** appear if the string were NUL terminated. If any NUL characters occur ** in the string at a byte offset that is less than the value of the 3rd ** parameter, then the resulting string will contain embedded NULs and the ** result of expressions operating on strings with embedded NULs is undefined. @@ -6410,7 +6410,7 @@ typedef void (*sqlite3_destructor_type)(void*); ** string and preferably a string literal. The sqlite3_result_pointer() ** routine is part of the [pointer passing interface] added for SQLite 3.20.0. ** -** If these routines are called from within the different thread +** If these routines are called from within a different thread ** than the one containing the application-defined function that received ** the [sqlite3_context] pointer, the results are undefined. */ @@ -6816,7 +6816,7 @@ sqlite3 *sqlite3_db_handle(sqlite3_stmt*); ** METHOD: sqlite3 ** ** ^The sqlite3_db_name(D,N) interface returns a pointer to the schema name -** for the N-th database on database connection D, or a NULL pointer of N is +** for the N-th database on database connection D, or a NULL pointer if N is ** out of range. An N value of 0 means the main database file. An N of 1 is ** the "temp" schema. Larger values of N correspond to various ATTACH-ed ** databases. @@ -6911,7 +6911,7 @@ int sqlite3_txn_state(sqlite3*,const char *zSchema); **
        The SQLITE_TXN_READ state means that the database is currently ** in a read transaction. Content has been read from the database file ** but nothing in the database file has changed. The transaction state -** will advanced to SQLITE_TXN_WRITE if any changes occur and there are +** will be advanced to SQLITE_TXN_WRITE if any changes occur and there are ** no other conflicting concurrent write transactions. The transaction ** state will revert to SQLITE_TXN_NONE following a [ROLLBACK] or ** [COMMIT].
        @@ -6920,7 +6920,7 @@ int sqlite3_txn_state(sqlite3*,const char *zSchema); **
        The SQLITE_TXN_WRITE state means that the database is currently ** in a write transaction. Content has been written to the database file ** but has not yet committed. The transaction state will change to -** to SQLITE_TXN_NONE at the next [ROLLBACK] or [COMMIT].
        +** SQLITE_TXN_NONE at the next [ROLLBACK] or [COMMIT]. */ #define SQLITE_TXN_NONE 0 #define SQLITE_TXN_READ 1 @@ -7201,7 +7201,7 @@ int sqlite3_db_release_memory(sqlite3*); ** CAPI3REF: Impose A Limit On Heap Size ** ** These interfaces impose limits on the amount of heap memory that will be -** by all database connections within a single process. +** used by all database connections within a single process. ** ** ^The sqlite3_soft_heap_limit64() interface sets and/or queries the ** soft limit on the amount of heap memory that may be allocated by SQLite. @@ -7259,7 +7259,7 @@ int sqlite3_db_release_memory(sqlite3*); **
      )^ ** ** The circumstances under which SQLite will enforce the heap limits may -** changes in future releases of SQLite. +** change in future releases of SQLite. */ sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); sqlite3_int64 sqlite3_hard_heap_limit64(sqlite3_int64 N); @@ -7374,8 +7374,8 @@ int sqlite3_table_column_metadata( ** ^The entry point is zProc. ** ^(zProc may be 0, in which case SQLite will try to come up with an ** entry point name on its own. It first tries "sqlite3_extension_init". -** If that does not work, it constructs a name "sqlite3_X_init" where the -** X is consists of the lower-case equivalent of all ASCII alphabetic +** If that does not work, it constructs a name "sqlite3_X_init" where +** X consists of the lower-case equivalent of all ASCII alphabetic ** characters in the filename from the last "/" to the first following ** "." and omitting any initial "lib".)^ ** ^The sqlite3_load_extension() interface returns @@ -7446,7 +7446,7 @@ int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** ^(Even though the function prototype shows that xEntryPoint() takes ** no arguments and returns void, SQLite invokes xEntryPoint() with three ** arguments and expects an integer result as if the signature of the -** entry point where as follows: +** entry point were as follows: ** **
       **    int xEntryPoint(
      @@ -7610,7 +7610,7 @@ struct sqlite3_module {
       ** virtual table and might not be checked again by the byte code.)^ ^(The
       ** aConstraintUsage[].omit flag is an optimization hint. When the omit flag
       ** is left in its default setting of false, the constraint will always be
      -** checked separately in byte code.  If the omit flag is change to true, then
      +** checked separately in byte code.  If the omit flag is changed to true, then
       ** the constraint may or may not be checked in byte code.  In other words,
       ** when the omit flag is true there is no guarantee that the constraint will
       ** not be checked again using byte code.)^
      @@ -7636,7 +7636,7 @@ struct sqlite3_module {
       ** The xBestIndex method may optionally populate the idxFlags field with a
       ** mask of SQLITE_INDEX_SCAN_* flags. One such flag is
       ** [SQLITE_INDEX_SCAN_HEX], which if set causes the [EXPLAIN QUERY PLAN]
      -** output to show the idxNum has hex instead of as decimal.  Another flag is
      +** output to show the idxNum as hex instead of as decimal.  Another flag is
       ** SQLITE_INDEX_SCAN_UNIQUE, which if set indicates that the query plan will
       ** return at most one row.
       **
      @@ -7777,7 +7777,7 @@ struct sqlite3_index_info {
       ** the implementation of the [virtual table module].   ^The fourth
       ** parameter is an arbitrary client data pointer that is passed through
       ** into the [xCreate] and [xConnect] methods of the virtual table module
      -** when a new virtual table is be being created or reinitialized.
      +** when a new virtual table is being created or reinitialized.
       **
       ** ^The sqlite3_create_module_v2() interface has a fifth parameter which
       ** is a pointer to a destructor for the pClientData.  ^SQLite will
      @@ -7942,7 +7942,7 @@ typedef struct sqlite3_blob sqlite3_blob;
       ** in *ppBlob. Otherwise an [error code] is returned and, unless the error
       ** code is SQLITE_MISUSE, *ppBlob is set to NULL.)^ ^This means that, provided
       ** the API is not misused, it is always safe to call [sqlite3_blob_close()]
      -** on *ppBlob after this function it returns.
      +** on *ppBlob after this function returns.
       **
       ** This function fails with SQLITE_ERROR if any of the following are true:
       ** 
        @@ -8062,7 +8062,7 @@ int sqlite3_blob_close(sqlite3_blob *); ** ** ^Returns the size in bytes of the BLOB accessible via the ** successfully opened [BLOB handle] in its only argument. ^The -** incremental blob I/O routines can only read or overwriting existing +** incremental blob I/O routines can only read or overwrite existing ** blob content; they cannot change the size of a blob. ** ** This routine only works on a [BLOB handle] which has been created @@ -8212,7 +8212,7 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); ** ^The sqlite3_mutex_alloc() routine allocates a new ** mutex and returns a pointer to it. ^The sqlite3_mutex_alloc() ** routine returns NULL if it is unable to allocate the requested -** mutex. The argument to sqlite3_mutex_alloc() must one of these +** mutex. The argument to sqlite3_mutex_alloc() must be one of these ** integer constants: ** **
          @@ -8445,7 +8445,7 @@ int sqlite3_mutex_notheld(sqlite3_mutex*); ** CAPI3REF: Retrieve the mutex for a database connection ** METHOD: sqlite3 ** -** ^This interface returns a pointer the [sqlite3_mutex] object that +** ^This interface returns a pointer to the [sqlite3_mutex] object that ** serializes access to the [database connection] given in the argument ** when the [threading mode] is Serialized. ** ^If the [threading mode] is Single-thread or Multi-thread then this @@ -8568,7 +8568,7 @@ int sqlite3_test_control(int op, ...); ** CAPI3REF: SQL Keyword Checking ** ** These routines provide access to the set of SQL language keywords -** recognized by SQLite. Applications can uses these routines to determine +** recognized by SQLite. Applications can use these routines to determine ** whether or not a specific identifier needs to be escaped (for example, ** by enclosing in double-quotes) so as not to confuse the parser. ** @@ -8736,7 +8736,7 @@ void sqlite3_str_reset(sqlite3_str*); ** content of the dynamic string under construction in X. The value ** returned by [sqlite3_str_value(X)] is managed by the sqlite3_str object X ** and might be freed or altered by any subsequent method on the same -** [sqlite3_str] object. Applications must not used the pointer returned +** [sqlite3_str] object. Applications must not use the pointer returned by ** [sqlite3_str_value(X)] after any subsequent method call on the same ** object. ^Applications may change the content of the string returned ** by [sqlite3_str_value(X)] as long as they do not write into any bytes @@ -8822,7 +8822,7 @@ int sqlite3_status64( ** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE] ** buffer and where forced to overflow to [sqlite3_malloc()]. The ** returned value includes allocations that overflowed because they -** where too large (they were larger than the "sz" parameter to +** were too large (they were larger than the "sz" parameter to ** [SQLITE_CONFIG_PAGECACHE]) and allocations that overflowed because ** no space was left in the page cache.)^ ** @@ -8906,28 +8906,29 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** [[SQLITE_DBSTATUS_LOOKASIDE_HIT]] ^(
          SQLITE_DBSTATUS_LOOKASIDE_HIT
          **
          This parameter returns the number of malloc attempts that were ** satisfied using lookaside memory. Only the high-water value is meaningful; -** the current value is always zero.)^ +** the current value is always zero.
          )^ ** ** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE]] ** ^(
          SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE
          -**
          This parameter returns the number malloc attempts that might have +**
          This parameter returns the number of malloc attempts that might have ** been satisfied using lookaside memory but failed due to the amount of ** memory requested being larger than the lookaside slot size. ** Only the high-water value is meaningful; -** the current value is always zero.)^ +** the current value is always zero.
          )^ ** ** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL]] ** ^(
          SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL
          -**
          This parameter returns the number malloc attempts that might have +**
          This parameter returns the number of malloc attempts that might have ** been satisfied using lookaside memory but failed due to all lookaside ** memory already being in use. ** Only the high-water value is meaningful; -** the current value is always zero.)^ +** the current value is always zero.
          )^ ** ** [[SQLITE_DBSTATUS_CACHE_USED]] ^(
          SQLITE_DBSTATUS_CACHE_USED
          **
          This parameter returns the approximate number of bytes of heap ** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. +**
          ** ** [[SQLITE_DBSTATUS_CACHE_USED_SHARED]] ** ^(
          SQLITE_DBSTATUS_CACHE_USED_SHARED
          @@ -8936,10 +8937,10 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** memory used by that pager cache is divided evenly between the attached ** connections.)^ In other words, if none of the pager caches associated ** with the database connection are shared, this request returns the same -** value as DBSTATUS_CACHE_USED. Or, if one or more or the pager caches are +** value as DBSTATUS_CACHE_USED. Or, if one or more of the pager caches are ** shared, the value returned by this call will be smaller than that returned ** by DBSTATUS_CACHE_USED. ^The highwater mark associated with -** SQLITE_DBSTATUS_CACHE_USED_SHARED is always 0. +** SQLITE_DBSTATUS_CACHE_USED_SHARED is always 0. ** ** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(
          SQLITE_DBSTATUS_SCHEMA_USED
          **
          This parameter returns the approximate number of bytes of heap @@ -8949,6 +8950,7 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** schema memory is shared with other database connections due to ** [shared cache mode] being enabled. ** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. +**
          ** ** [[SQLITE_DBSTATUS_STMT_USED]] ^(
          SQLITE_DBSTATUS_STMT_USED
          **
          This parameter returns the approximate number of bytes of heap @@ -8985,7 +8987,7 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** been written to disk in the middle of a transaction due to the page ** cache overflowing. Transactions are more efficient if they are written ** to disk all at once. When pages spill mid-transaction, that introduces -** additional overhead. This parameter can be used help identify +** additional overhead. This parameter can be used to help identify ** inefficiencies that can be resolved by increasing the cache size. **
          ** @@ -9465,7 +9467,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** external process or via a database connection other than the one being ** used by the backup operation, then the backup will be automatically ** restarted by the next call to sqlite3_backup_step(). ^If the source -** database is modified by the using the same database connection as is used +** database is modified by using the same database connection as is used ** by the backup operation, then the backup database is automatically ** updated at the same time. ** @@ -9482,7 +9484,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** and may not be used following a call to sqlite3_backup_finish(). ** ** ^The value returned by sqlite3_backup_finish is [SQLITE_OK] if no -** sqlite3_backup_step() errors occurred, regardless or whether or not +** sqlite3_backup_step() errors occurred, regardless of whether or not ** sqlite3_backup_step() completed. ** ^If an out-of-memory condition or IO error occurred during any prior ** sqlite3_backup_step() call on the same [sqlite3_backup] object, then @@ -10521,14 +10523,14 @@ int sqlite3_stmt_scanstatus( int idx, /* Index of loop to report on */ int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ void *pOut /* Result written here */ -); +); int sqlite3_stmt_scanstatus_v2( sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ int idx, /* Index of loop to report on */ int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ int flags, /* Mask of flags defined below */ void *pOut /* Result written here */ -); +); /* ** CAPI3REF: Prepared Statement Scan Status @@ -10552,7 +10554,7 @@ void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); ** METHOD: sqlite3 ** ** ^If a write-transaction is open on [database connection] D when the -** [sqlite3_db_cacheflush(D)] interface invoked, any dirty +** [sqlite3_db_cacheflush(D)] interface is invoked, any dirty ** pages in the pager-cache that are not currently in use are written out ** to disk. A dirty page may be in use if a database cursor created by an ** active SQL statement is reading from it, or if it is page 1 of a database diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 41adccf69..5cc134a9d 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1031,8 +1031,8 @@ typedef INT16_TYPE LogEst; ** assuming n is a signed integer type. UMXV(n) is similar for unsigned ** integer types. */ -#define SMXV(n) ((((i64)1)<<(sizeof(n)-1))-1) -#define UMXV(n) ((((i64)1)<<(sizeof(n)))-1) +#define SMXV(n) ((((i64)1)<<(sizeof(n)*8-1))-1) +#define UMXV(n) ((((i64)1)<<(sizeof(n)*8))-1) /* ** Round up a number to the next larger multiple of 8. This is used @@ -2880,7 +2880,7 @@ struct AggInfo { ** from source tables rather than from accumulators */ u8 useSortingIdx; /* In direct mode, reference the sorting index rather ** than the source table */ - u16 nSortingColumn; /* Number of columns in the sorting index */ + u32 nSortingColumn; /* Number of columns in the sorting index */ int sortingIdx; /* Cursor number of the sorting index */ int sortingIdxPTab; /* Cursor number of pseudo-table */ int iFirstReg; /* First register in range for aCol[] and aFunc[] */ @@ -2889,8 +2889,8 @@ struct AggInfo { Table *pTab; /* Source table */ Expr *pCExpr; /* The original expression */ int iTable; /* Cursor number of the source table */ - i16 iColumn; /* Column number within the source table */ - i16 iSorterColumn; /* Column number in the sorting index */ + int iColumn; /* Column number within the source table */ + int iSorterColumn; /* Column number in the sorting index */ } *aCol; int nColumn; /* Number of used entries in aCol[] */ int nAccumulator; /* Number of columns that show through to the output. diff --git a/src/wal.c b/src/wal.c index 1fd5b201c..41018b584 100644 --- a/src/wal.c +++ b/src/wal.c @@ -3781,6 +3781,7 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal); } SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) + pWal->iReCksum = 0; } return rc; } @@ -3828,6 +3829,9 @@ int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ walCleanupHash(pWal); } SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) + if( pWal->iReCksum>pWal->hdr.mxFrame ){ + pWal->iReCksum = 0; + } } return rc; diff --git a/src/whereexpr.c b/src/whereexpr.c index 4a24dadd2..26bcae71c 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -931,30 +931,42 @@ static void exprAnalyzeOrTerm( ** 1. The SQLITE_Transitive optimization must be enabled ** 2. Must be either an == or an IS operator ** 3. Not originating in the ON clause of an OUTER JOIN -** 4. The affinities of A and B must be compatible -** 5a. Both operands use the same collating sequence OR -** 5b. The overall collating sequence is BINARY +** 4. The operator is not IS or else the query does not contain RIGHT JOIN +** 5. The affinities of A and B must be compatible +** 6a. Both operands use the same collating sequence OR +** 6b. The overall collating sequence is BINARY ** If this routine returns TRUE, that means that the RHS can be substituted ** for the LHS anyplace else in the WHERE clause where the LHS column occurs. ** This is an optimization. No harm comes from returning 0. But if 1 is ** returned when it should not be, then incorrect answers might result. */ -static int termIsEquivalence(Parse *pParse, Expr *pExpr){ +static int termIsEquivalence(Parse *pParse, Expr *pExpr, SrcList *pSrc){ char aff1, aff2; CollSeq *pColl; - if( !OptimizationEnabled(pParse->db, SQLITE_Transitive) ) return 0; - if( pExpr->op!=TK_EQ && pExpr->op!=TK_IS ) return 0; - if( ExprHasProperty(pExpr, EP_OuterON) ) return 0; + if( !OptimizationEnabled(pParse->db, SQLITE_Transitive) ) return 0; /* (1) */ + if( pExpr->op!=TK_EQ && pExpr->op!=TK_IS ) return 0; /* (2) */ + if( ExprHasProperty(pExpr, EP_OuterON) ) return 0; /* (3) */ + assert( pSrc!=0 ); + if( pExpr->op==TK_IS + && pSrc->nSrc + && (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 + ){ + return 0; /* (4) */ + } aff1 = sqlite3ExprAffinity(pExpr->pLeft); aff2 = sqlite3ExprAffinity(pExpr->pRight); if( aff1!=aff2 && (!sqlite3IsNumericAffinity(aff1) || !sqlite3IsNumericAffinity(aff2)) ){ - return 0; + return 0; /* (5) */ } pColl = sqlite3ExprCompareCollSeq(pParse, pExpr); - if( sqlite3IsBinary(pColl) ) return 1; - return sqlite3ExprCollSeqMatch(pParse, pExpr->pLeft, pExpr->pRight); + if( !sqlite3IsBinary(pColl) + && !sqlite3ExprCollSeqMatch(pParse, pExpr->pLeft, pExpr->pRight) + ){ + return 0; /* (6) */ + } + return 1; } /* @@ -1219,8 +1231,8 @@ static void exprAnalyze( if( op==TK_IS ) pNew->wtFlags |= TERM_IS; pTerm = &pWC->a[idxTerm]; pTerm->wtFlags |= TERM_COPIED; - - if( termIsEquivalence(pParse, pDup) ){ + assert( pWInfo->pTabList!=0 ); + if( termIsEquivalence(pParse, pDup, pWInfo->pTabList) ){ pTerm->eOperator |= WO_EQUIV; eExtraOp = WO_EQUIV; } diff --git a/test/func9.test b/test/func9.test index 42138ab2e..2383b76f6 100644 --- a/test/func9.test +++ b/test/func9.test @@ -26,6 +26,9 @@ do_catchsql_test func9-120 { do_execsql_test func9-130 { SELECT concat_ws(',',1,2,3,4,5,6,7,8,NULL,9,10,11,12); } {1,2,3,4,5,6,7,8,9,10,11,12} +do_execsql_test func9-131 { + SELECT concat_ws(',',1,2,3,4,'',6,7,8,NULL,9,10,11,12); +} {1,2,3,4,,6,7,8,9,10,11,12} do_execsql_test func9-140 { SELECT concat_ws(NULL,1,2,3,4,5,6,7,8,NULL,9,10,11,12); } {{}} diff --git a/test/join.test b/test/join.test index ef2f6335c..b33a7560a 100644 --- a/test/join.test +++ b/test/join.test @@ -1342,4 +1342,31 @@ do_execsql_test join-31.8 { SELECT * FROM t3 LEFT JOIN t2 ON true JOIN t4 ON true NATURAL LEFT JOIN t1; } {3 NULL 4 NULL} +# 2025-06-16 https://sqlite.org/forum/forumpost/68f29a2005 +# +# The transitive-constraint optimization was not working for RIGHT JOIN. +# +reset_db +db null NULL +do_execsql_test join-32.1 { + CREATE TABLE t0(w INT); + CREATE TABLE t1(x INT); + CREATE TABLE t2(y INT UNIQUE); + CREATE VIEW v0(z) AS SELECT CAST(x AS INT) FROM t1 LEFT JOIN t2 ON true; + INSERT INTO t1(x) VALUES(123); + INSERT INTO t2(y) VALUES(NULL); +} +do_execsql_test join-32.2 { + SELECT * + FROM t0 JOIN v0 ON w=z + RIGHT JOIN t1 ON true + INNER JOIN t2 ON y IS z; +} {NULL NULL 123 NULL} +do_execsql_test join-32.3 { + SELECT * + FROM t0 JOIN v0 ON w=z + RIGHT JOIN t1 ON true + INNER JOIN t2 ON +y IS z; +} {NULL NULL 123 NULL} + finish_test diff --git a/test/walcksum.test b/test/walcksum.test index 10329ba6c..0c9a7e55c 100644 --- a/test/walcksum.test +++ b/test/walcksum.test @@ -16,6 +16,7 @@ source $testdir/lock_common.tcl source $testdir/wal_common.tcl ifcapable !wal {finish_test ; return } +set testprefix walcksum # Read and return the contents of file $filename. Treat the content as # binary data. @@ -331,5 +332,152 @@ do_test walcksum-2.1 { catch { db close } catch { db2 close } +#------------------------------------------------------------------------- +# Test cases based on the bug reported at: +# +# +# +reset_db + +do_execsql_test 3.0 { + PRAGMA auto_vacuum = 0; + PRAGMA synchronous = NORMAL; + PRAGMA journal_mode = WAL; + PRAGMA cache_size = 1; + + CREATE TABLE t1 (i INTEGER PRIMARY KEY, b BLOB, t TEXT); + PRAGMA wal_checkpoint; + INSERT INTO t1 VALUES(1, randomblob(2048), 'one'); +} {wal 0 2 2} + +do_execsql_test 3.1 { + BEGIN; + INSERT INTO t1 VALUES(2, randomblob(2048), 'two'); + SAVEPOINT one; + INSERT INTO t1 VALUES(3, randomblob(2048), 'three'); + INSERT INTO t1 VALUES(4, randomblob(2048), 'four'); + INSERT INTO t1 VALUES(5, randomblob(2048), 'five'); + INSERT INTO t1 VALUES(6, randomblob(2048), 'six'); + INSERT INTO t1 VALUES(7, randomblob(2048), 'seven'); + + UPDATE t1 SET b=randomblob(2048) WHERE i=5; + UPDATE t1 SET b=randomblob(2048) WHERE i=6; + UPDATE t1 SET b=randomblob(2048) WHERE i=7; + ROLLBACK TO one; + INSERT INTO t1 VALUES(8, NULL, 'eight'); + COMMIT; +} {} + +do_execsql_test 3.2 { + SELECT i, t FROM t1 +} {1 one 2 two 8 eight} + +forcecopy test.db test2.db +forcecopy test.db-wal test2.db-wal + +sqlite3 db2 test2.db +do_test 1.3 { + execsql { + SELECT i, t FROM t1 + } db2 +} {1 one 2 two 8 eight} + +catch { db2 close } + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 4.0 { + PRAGMA auto_vacuum = 0; + PRAGMA synchronous = NORMAL; + PRAGMA journal_mode = WAL; + PRAGMA cache_size = 1; + + CREATE TABLE t1 (i INTEGER PRIMARY KEY, b BLOB, t TEXT); + PRAGMA wal_checkpoint; + INSERT INTO t1 VALUES(1, randomblob(2048), 'one'); +} {wal 0 2 2} + +do_execsql_test 4.1.1 { + SAVEPOINT one; + INSERT INTO t1 VALUES(2, randomblob(2048), 'two'); + INSERT INTO t1 VALUES(3, randomblob(2048), 'three'); + INSERT INTO t1 VALUES(4, randomblob(2048), 'four'); + INSERT INTO t1 VALUES(5, randomblob(2048), 'five'); + INSERT INTO t1 VALUES(6, randomblob(2048), 'six'); + INSERT INTO t1 VALUES(7, randomblob(2048), 'seven'); + + UPDATE t1 SET b=randomblob(2048) WHERE i=5; + UPDATE t1 SET b=randomblob(2048) WHERE i=6; + UPDATE t1 SET b=randomblob(2048) WHERE i=7; +} + +do_execsql_test 4.1.2 { + ROLLBACK TO one; + INSERT INTO t1 VALUES(8, NULL, 'eight'); + RELEASE one; +} {} + +do_execsql_test 4.2 { + SELECT i, t FROM t1 +} {1 one 8 eight} + +forcecopy test.db test2.db +forcecopy test.db-wal test2.db-wal + +sqlite3 db2 test2.db +do_test 4.3 { + execsql { + SELECT i, t FROM t1 + } db2 +} {1 one 8 eight} + +catch { db2 close } + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 5.0 { + PRAGMA auto_vacuum = 0; + PRAGMA synchronous = NORMAL; + PRAGMA journal_mode = WAL; + PRAGMA cache_size = 1; + + CREATE TABLE t1 (i INTEGER PRIMARY KEY, b BLOB, t TEXT); + INSERT INTO t1 VALUES(1, randomblob(2048), 'one'); + INSERT INTO t1 VALUES(2, randomblob(2048), 'two'); + INSERT INTO t1 VALUES(3, randomblob(2048), 'three'); + PRAGMA wal_checkpoint; +} {wal 0 14 14} + +do_execsql_test 5.1 { + BEGIN; + SELECT count(*) FROM t1; + SAVEPOINT one; + INSERT INTO t1 VALUES(4, randomblob(2048), 'four'); + INSERT INTO t1 VALUES(5, randomblob(2048), 'five'); + INSERT INTO t1 VALUES(6, randomblob(2048), 'six'); + INSERT INTO t1 VALUES(7, randomblob(2048), 'seven'); + ROLLBACK TO one; + INSERT INTO t1 VALUES(8, randomblob(2048), 'eight'); + INSERT INTO t1 VALUES(9, randomblob(2048), 'nine'); + COMMIT; +} {3} + +forcecopy test.db test2.db +forcecopy test.db-wal test2.db-wal + +sqlite3 db2 test2.db +do_test 5.2 { + execsql { + SELECT i, t FROM t1 + } db2 +} {1 one 2 two 3 three 8 eight 9 nine} +db2 close + +do_execsql_test 5.3 { + SELECT i, t FROM t1 +} {1 one 2 two 3 three 8 eight 9 nine} + finish_test diff --git a/test/walsetlk2.test b/test/walsetlk2.test index 92630b3fd..7ffd8f03d 100644 --- a/test/walsetlk2.test +++ b/test/walsetlk2.test @@ -90,6 +90,8 @@ tvfs delete # but other operations do not use the retry mechanism. # reset_db +db close +sqlite3 db test.db -fullmutex 1 do_execsql_test 2.0 { CREATE TABLE t1(a, b); From bfc94fdbdd9e665c00537b8f6884f766f8f3c632 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 18 Jul 2025 15:38:18 -0400 Subject: [PATCH 25/75] Snapshot of upstream SQLite 3.50.3 --- VERSION | 2 +- ext/fts5/fts5_index.c | 4 +- ext/fts5/test/fts5corrupt8.test | 53 +++++++++++++++++++++ ext/wasm/fiddle.make | 3 +- manifest | 46 +++++++++--------- manifest.uuid | 2 +- sqlite3.pc.in | 2 +- src/build.c | 1 - src/expr.c | 2 +- src/parse.y | 19 ++++++-- src/sqlite.h.in | 84 ++++++++++++++++----------------- src/sqliteInt.h | 1 - src/tokenize.c | 2 +- src/where.c | 18 ++++++- src/wherecode.c | 29 +++++------- test/altertab2.test | 2 +- test/altertab3.test | 12 +---- test/parser1.test | 22 +++++++++ test/rowvalue.test | 37 +++++++++++++++ 19 files changed, 229 insertions(+), 112 deletions(-) diff --git a/VERSION b/VERSION index 18d0478a0..6a6d84a54 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.50.2 +3.50.3 diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 63840de1f..13e3740fe 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -1950,9 +1950,9 @@ static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){ ** leave an error in the Fts5Index object. */ static void fts5SegIterAllocTombstone(Fts5Index *p, Fts5SegIter *pIter){ - const int nTomb = pIter->pSeg->nPgTombstone; + const i64 nTomb = (i64)pIter->pSeg->nPgTombstone; if( nTomb>0 ){ - int nByte = SZ_FTS5TOMBSTONEARRAY(nTomb+1); + i64 nByte = SZ_FTS5TOMBSTONEARRAY(nTomb+1); Fts5TombstoneArray *pNew; pNew = (Fts5TombstoneArray*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pNew ){ diff --git a/ext/fts5/test/fts5corrupt8.test b/ext/fts5/test/fts5corrupt8.test index d642920e4..c0019137d 100644 --- a/ext/fts5/test/fts5corrupt8.test +++ b/ext/fts5/test/fts5corrupt8.test @@ -90,5 +90,58 @@ do_execsql_test 3.7 { SELECT * FROM sqlite_schema } +#------------------------------------------------------------------------- +reset_db + +proc hex_to_blob {hex} { + binary encode hex $hex +} +db func hex_to_blob hex_to_blob + +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x1 USING fts5(x, content='', contentless_delete=1); + BEGIN; + INSERT INTO x1(rowid, x) VALUES(1, 'a b c d e f g h'); + INSERT INTO x1(rowid, x) VALUES(2, 'a b c d e f g h'); + COMMIT; + DELETE FROM x1 WHERE rowid=1; +} + +do_execsql_test 4.1 { + SELECT hex(block) FROM x1_data WHERE id=10 +} { + 00000000FF00000101010200010101010101010102 +} + +do_execsql_test 4.2.1 { + UPDATE x1_data SET block= + X'00000000FF00000101010200010101010101819C9B95A8000102' + WHERE id=10; +} + +do_catchsql_test 4.2.2 { + SELECT * FROM x1('c d e'); +} {1 {out of memory}} + +do_execsql_test 4.3.1 { + UPDATE x1_data SET block= + X'00000000FF000001010102000101010101019282AFF9A0000102' + WHERE id=10; +} + +do_catchsql_test 4.3.2 { + SELECT * FROM x1('c d e'); +} {1 {out of memory}} + +do_execsql_test 4.4.1 { + UPDATE x1_data SET block= + X'00000000FF000001010102000101010101018181808080130102' + WHERE id=10; +} + +do_catchsql_test 4.3.2 { + SELECT * FROM x1('c d e'); +} {1 {out of memory}} + finish_test diff --git a/ext/wasm/fiddle.make b/ext/wasm/fiddle.make index 8110384a6..5b1eb5e77 100644 --- a/ext/wasm/fiddle.make +++ b/ext/wasm/fiddle.make @@ -40,9 +40,8 @@ fiddle.emcc-flags = \ -sWASM_BIGINT=$(emcc.WASM_BIGINT) \ -sEXPORT_NAME=$(sqlite3.js.init-func) \ -Wno-limited-postlink-optimizations \ - $(emcc.exportedRuntimeMethods) \ + $(emcc.exportedRuntimeMethods),FS \ -sEXPORTED_FUNCTIONS=@$(abspath $(EXPORTED_FUNCTIONS.fiddle)) \ - -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory \ $(SQLITE_OPT.full-featured) \ $(SQLITE_OPT.common) \ $(SHELL_OPT) \ diff --git a/manifest b/manifest index 00377be59..0ee8c25cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.50.2 -D 2025-06-28T14:00:48.788 +C Version\s3.50.3 +D 2025-07-17T13:25:10.611 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -8,7 +8,7 @@ F Makefile.in c3e414df4dc8dfb12f1f6baf129fcb6d18cd0ebd3c9109370fb3fceeeef9a37a F Makefile.linux-generic bd3e3cacd369821a6241d4ea1967395c962dfe3057e38cb0a435cee0e8b789d0 F Makefile.msc 0206f28988bb6634c7e8aff05bf6cfa65d6dfe1d2b6bd95160dd99290a83dfc7 F README.md e28077cfbef795e99c9c75ed95aa7257a1166709b562076441a8506ac421b7c1 -F VERSION 56fffb7f40e1117f5ab0cfe06e918a82d522c686d1dfff7f570f4ca414eb256a +F VERSION 613f62a5375c819156a56fa263adc08f8ee5998f45a946d3c5d594385b2ef736 F art/icon-243x273.gif 9750b734f82fdb3dc43127753d5e6fbf3b62c9f4e136c2fbf573b2f57ea87af5 F art/icon-80x90.gif 65509ce3e5f86a9cd64fe7fca2d23954199f31fe44c1e09e208c80fb83d87031 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -114,7 +114,7 @@ F ext/fts5/fts5_buffer.c f1e6d0324d7c55329d340673befc26681a372a4d36086caa8d1ec7d F ext/fts5/fts5_config.c e7d8dd062b44a66cd77e5a0f74f23a2354cd1f3f8575afb967b2773c3384f7f8 F ext/fts5/fts5_expr.c be9e5f7f11d87e7bd3680832c93c13050fe351994b5052b0215c2ef40312c23a F ext/fts5/fts5_hash.c a6266cedd801ab7964fa9e74ebcdda6d30ec6a96107fa24148ec6b7b5b80f6e0 -F ext/fts5/fts5_index.c d171f2a507abccb3d524bf461b01f0d3971a9bf221be622ac7c671a991cb62ee +F ext/fts5/fts5_index.c 5896ca188574f728426a8f03752dfbf4fad48d1a4b7450b29d8d3acae1739761 F ext/fts5/fts5_main.c 57933c18efe1058d8871199875c7a59744dabc3904f3aefbf9ff4a4e11fc79e2 F ext/fts5/fts5_storage.c 19bc7c4cbe1e6a2dd9849ef7d84b5ca1fcbf194cefc3e386b901e00e08bf05c2 F ext/fts5/fts5_tcl.c 7fb5a3d3404099075aaa2457307cb459bbc257c0de3dbd52b1e80a5b503e0329 @@ -168,7 +168,7 @@ F ext/fts5/test/fts5corrupt4.test dc08d19f5b8943e95a7778a7d8da592042504faf18dd93 F ext/fts5/test/fts5corrupt5.test bcf0801b0c991eadae3cb8e978e82b4bf01412cb4df41874a90d5aa279c7cc96 F ext/fts5/test/fts5corrupt6.test 2d72db743db7b5d9c9a6d0cfef24d799ed1aa5e8192b66c40e871a37ed9eed06 F ext/fts5/test/fts5corrupt7.test 4e830875c33b9ea3c4cf1ba71e692b63893cbb4faae8c69b1071889dc26e211c -F ext/fts5/test/fts5corrupt8.test b81d802e41631e98100f49a1aadeeffef860e30a62d6ed7d743c2797c477239e +F ext/fts5/test/fts5corrupt8.test ffaaad3524c79621042eaf176d9206f68c5f07611c4d24943e60a3b6ea0026e8 F ext/fts5/test/fts5delete.test 2a5008f8b1174ef41d1974e606928c20e4f9da77d9f8347aed818994d89cced4 F ext/fts5/test/fts5detail.test 54015e9c43ec4ba542cfb93268abdf280e0300f350efd08ee411284b03595cc4 F ext/fts5/test/fts5determin.test 1b77879b2ae818b5b71c859e534ee334dac088b7cf3ff3bf76a2c82b1c788d11 @@ -674,7 +674,7 @@ F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2 F ext/wasm/demo-worker1.js 08720227e98fa5b44761cf6e219269cee3e9dd0421d8d91459535da776950314 F ext/wasm/dist.make 92ef4ffe33022a50f92d602acabad10bd8dd91759f3eb7df27fc6d7d37072b96 F ext/wasm/example_extra_init.c 2347cd69d19d839ef4e5e77b7855103a7fe3ef2af86f2e8c95839afd8b05862f -F ext/wasm/fiddle.make c6d7a3d6cc03bb5f21acb295c1233820d0dbf5c6a89b28dc2e093edcc001c45a +F ext/wasm/fiddle.make 2df87f12bcbae2c966c2cef34ce71bb1584c440c69e14ca6d32f443d8d550dc5 F ext/wasm/fiddle/fiddle-worker.js 850e66fce39b89d59e161d1abac43a181a4caa89ddeea162765d660277cd84ce F ext/wasm/fiddle/fiddle.js 2a2f27b4be2674f501fff61c4a09e44dcf2295731a26b5c28e439f3a573bd269 F ext/wasm/fiddle/index.html 7fcfb221165183bef0e05d5af9ceb79b527e799b1708ab05de0ec0eaebd5b7bf @@ -718,7 +718,7 @@ F mptest/mptest.c aa41ace6dbc5050d76b02548d3521e6bbccae4f0 F mptest/multiwrite01.test dab5c5f8f9534971efce679152c5146da265222d F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 acdff36db796e2d00225b911d3047d580cd136547298435426ce9d40347973cc -F sqlite3.pc.in 0977c03a4da7c4204bd60e784a0efb8d51a190448aba78a4e973fe7192bdaf03 +F sqlite3.pc.in e6dee284fba59ef500092fdc1843df3be8433323a3733c91da96690a50a5b398 F src/alter.c fc7bbbeb9e89c7124bf5772ce474b333b7bdc18d6e080763211a40fde69fb1da F src/analyze.c 03bcfc083fc0cccaa9ded93604e1d4244ea245c17285d463ef6a60425fcb247d F src/attach.c 9af61b63b10ee702b1594ecd24fb8cea0839cfdb6addee52fba26fa879f5db9d @@ -729,14 +729,14 @@ F src/btmutex.c 30dada73a819a1ef5b7583786370dce1842e12e1ad941e4d05ac29695528daea F src/btree.c da98489a981c347cc3a3982ea2810bbb583511a73cc34762547f30dbb4cda7f0 F src/btree.h 18e5e7b2124c23426a283523e5f31a4bff029131b795bb82391f9d2f3136fc50 F src/btreeInt.h 9c0f9ea5c9b5f4dcaea18111d43efe95f2ac276cd86d770dce10fd99ccc93886 -F src/build.c 67c1db4c5e89a8519fe9b6dafc287f6bc3627696b5b8536dc5e06db570d8c05f +F src/build.c 67159d31bfc565e5f23a7be8159325bae599bddd19fc584ac2511b947cf341d3 F src/callback.c acae8c8dddda41ee85cfdf19b926eefe830f371069f8aadca3aa39adf5b1c859 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/date.c 9db4d604e699a73e10b8e85a44db074a1f04c0591a77e2abfd77703f50dce1e9 F src/dbpage.c fcb1aafe00872a8aff9a7aa0ef7ff1b01e5817ec7bbd521f8f3e1e674ac8d609 F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c F src/delete.c 03a77ba20e54f0f42ebd8eddf15411ed6bdb06a2c472ac4b6b336521bf7cea42 -F src/expr.c 41f193b96b3757db1a82e7590823ca833f6215b38d7b17c98d181208cf9cdd6b +F src/expr.c 7e7f57247e864aaa46af2fd6bf3f7433dbefffc470687158340e5d68a15c4469 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f F src/func.c de47a8295503aa130baae5e6d9868ecf4f7c4dbffa65d83ad1f70bdbac0ee2d6 @@ -775,7 +775,7 @@ F src/os_win.c b8d3cfdf2f40e2f9715b7d8df64f3c0c7ee18743a2dd0c4fc70c1d57fa1aadc7 F src/os_win.h 4c247cdb6d407c75186c94a1e84d5a22cbae4adcec93fcae8d2bc1f956fd1f19 F src/pager.c 23c0f17deb892da6b32fef1f465507df7ab5cd01d774288cb43695658a649259 F src/pager.h 6137149346e6c8a3ddc1eeb40aee46381e9bc8b0fcc6dda8a1efde993c2275b8 -F src/parse.y e426d7323311554c75b0aebc426d0fe3c88d9777ffefed236f343ad9e661dc4c +F src/parse.y 619c3e92a54686c5e47923688c4b9bf7ec534a4690db5677acc28b299c403250 F src/pcache.c 588cc3c5ccaaadde689ed35ce5c5c891a1f7b1f4d1f56f6cf0143b74d8ee6484 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 131ca0daf4e66b4608d2945ae76d6ed90de3f60539afbd5ef9ec65667a5f2fcd @@ -787,10 +787,10 @@ F src/resolve.c d40fe18d7c2fd0339f5846ffcf7d6809866e380acdf14c76fb2af87e9fe13f64 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 7a21df5db6bb1a4c1bb6d9fb76c8e2485a22ff8306519ad69d8ddf0d5fa10903 F src/shell.c.in ba53a52dafb167ac6320703da741386c34fbcabe8c078a188bb9f89808e3ef8f -F src/sqlite.h.in 18897d0b0cc064c9d6139cb0f56682d5da9ad0cc098b04f492458cdc0fc30e01 +F src/sqlite.h.in 9ae373d11e1b11ac9c81c508523ae37f1619e739858280078ee9fb4e1e62d3ed F src/sqlite3.rc 015537e6ac1eec6c7050e17b616c2ffe6f70fca241835a84a4f0d5937383c479 F src/sqlite3ext.h 0bfd049bb2088cc44c2ad54f2079d1c6e43091a4e1ce8868779b75f6c1484f1e -F src/sqliteInt.h e29d71b90bde0b2e06c813bd40d2265d1292a2ad7062ef7b2a1b9207dc9ecd98 +F src/sqliteInt.h 08a7dcb8db162875b3dd5229eb0bf75691150ac54fea1ff3670391bd6ef40546 F src/sqliteLimit.h 6d817c28a8f19af95e6f4921933b7fbbca48a962bce0eb0ec81e8bb3ef38e68b F src/status.c 0e72e4f6be6ccfde2488eb63210297e75f569f3ce9920f6c3d77590ec6ce5ffd F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -846,7 +846,7 @@ F src/test_windirent.h f8245d8002aa0d4322192d35b0f8bbfc757479e90d60fd0beb386d391 F src/test_window.c 6d80e11fba89a1796525e6f0048ff0c7789aa2c6b0b11c80827dc1437bd8ea72 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c 3e37ac2b6cbb9b0abe33827b0153c27595269afd7152b48019808974481aca2c +F src/tokenize.c 8400646d2830afc2f2dc465a75e3a92e4bedeea623f19dbd79c0c12d0dd6dda2 F src/treeview.c d85ce76e6d1498d781957c07cb234da6d77ce0ed2d196480d516f54dabc62279 F src/trigger.c 3ffb8ed6b64dbcc0ccae6e82435d01be3bf547e13b814e2d46f7df9bef84748e F src/update.c 3e5e7ff66fa19ebe4d1b113d480639a24cc1175adbefabbd1a948a07f28e37cf @@ -869,9 +869,9 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 20be6f0a25a80b7897cf2a5369bfd37ef198e6f0b6cdef16d83eee856056b159 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 -F src/where.c 45a3b496248a0b36d91ce34da3278d54f8fa20e9d3fbd36d45a42051d1118137 +F src/where.c 4b9f73b1633b4dcddd82abc69aa6384dbef528289e8daa29521c3112b82fd1b9 F src/whereInt.h ecdbfb5551cf394f04ec7f0bc7ad963146d80eee3071405ac29aa84950128b8e -F src/wherecode.c 65670d1ef85ef54a4db3826d63be8b646c9ac280962166b645950901ed1bda29 +F src/wherecode.c 66684a11713667f532a69214ac552acee7e0825e2f1b71abaebe05511d190fb4 F src/whereexpr.c 1c60db88b6e8472f4864b98014d1b2d9d16bdd42d5d181ec515acbcdeddc18db F src/window.c d01227141f622f24fbe36ca105fbe6ef023f9fd98f1ccd65da95f88886565db5 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -900,8 +900,8 @@ F test/altermalloc2.test 17fb3724c4b004c469c27dc4ef181608aa644555fbd3f3236767584 F test/altermalloc3.test 8040e486368403f2fdd6fc3998258b499bd4cc2f3ddbb5f8f874cd436f076e81 F test/alterqf.test 8ec03d776de9c391daa0078ea8f838903bdcfb11dfae4ba3576b48436834ccba F test/altertab.test 8a2712f9076da5012a002d0b5cc0a421398a5bf61c25bab41b77c427586a7a27 -F test/altertab2.test 4bad0fa9b1ad6e62d07bc2ddb0807fb98ba80ee06d6593db2e514ec1821cae3a -F test/altertab3.test b331ae34e69594e19605e3297805202d6156fcc8f75379dfd972a2e51cae8721 +F test/altertab2.test 0889ba0700cc1cdb7bc7d25975aa61fece34f621de963d0886e2395716b38576 +F test/altertab3.test 471b8898d10bbc6488db9c23dc76811f405de6707d2d342b1b8b6fd1f13cd3c8 F test/altertrig.test aacc980b657354fe2d3d4d3a004f07d04ccc1a93e5ef82d68a79088c274ddc6b F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 2fb21d7d64748636384e6cb8998dbf83968caf644c07fcb4f76c18f2e7ede94b @@ -1530,7 +1530,7 @@ F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8 F test/pageropt.test 84e4cc5cbca285357f7906e99b21be4f2bf5abc0 F test/pagesize.test 5769fc62d8c890a83a503f67d47508dfdc543305 -F test/parser1.test 6ccdf5e459a5dc4673d3273dc311a7e9742ca952dd0551a6a6320d27035ce4b3 +F test/parser1.test 131f4733472252d53d8ed681115257866f55740ab697fa05900d766049348f27 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/pendingrace.test e99efc5ab3584da3dfc8cd6a0ec4e5a42214820574f5ea24ee93f1d84655f463 @@ -1577,7 +1577,7 @@ F test/round1.test 29c3c9039936ed024d672f003c4d35ee11c14c0acb75c5f7d6188ff16190c F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test d27191b5ce794c05bf61081e8b2c546a1844c1641321dcaf7fb785234256cc8e -F test/rowvalue.test 9c873b2f6e7ce72b24ef133f93515c07a6a7dac4846a344ebc2af7b8bfdf5147 +F test/rowvalue.test 8a3f0fea3a3cbbfc7cb9885b76185a774cd8d891e0c133e289567c755d39eb9f F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b F test/rowvalue3.test 103e9a224ca0548dd0d67e439f39c5dd16de4200221a333927372408c025324c F test/rowvalue4.test bac9326d1e886656650f67c0ec484eb5f452244a8209c6af508e9a862ace08ed @@ -2209,10 +2209,10 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 1ad0169b022b280bcaaf94a7fa231591be96b514230ab5c98fbf15cd7df842dd F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4ae45291e6aac984af49c6da8e03216cf96b97b2ca7b11d5bcdb90b1f827fdaf -R de044a43142203d291ffaa11945f35d7 +P 63595b74956a9391f03a273204c80ecd0ba946846b7aa0195b9095fe8b6a78e5 +R 987003a9608f543153f11a48f885adf4 T +sym-release * -T +sym-version-3.50.2 * +T +sym-version-3.50.3 * U drh -Z 74a7bf04be5e68b03e7a14d88f83dabb +Z 351123ab09d4bd77e28a506f711aa720 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 18e0261b3..d2cf7ea04 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2af157d77fb1304a74176eaee7fbc7c7e932d946bf25325e9c26c91db19e3079 +3ce993b8657d6d9deda380a93cdd6404a8c8ba1b185b2bc423703e41ae5f2543 diff --git a/sqlite3.pc.in b/sqlite3.pc.in index a9f941b1e..723dd5156 100644 --- a/sqlite3.pc.in +++ b/sqlite3.pc.in @@ -9,5 +9,5 @@ Name: SQLite Description: SQL database engine Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lsqlite3 -Libs.private: @LDFLAGS_MATH@ @LDFLAGS_ZLIB@ @LDFLAGS_ICU@ +Libs.private: @LDFLAGS_MATH@ @LDFLAGS_ZLIB@ @LDFLAGS_DLOPEN@ @LDFLAGS_PTHREAD@ @LDFLAGS_ICU@ Cflags: -I${includedir} diff --git a/src/build.c b/src/build.c index 5bd3aac3c..27d7b499d 100644 --- a/src/build.c +++ b/src/build.c @@ -4219,7 +4219,6 @@ void sqlite3CreateIndex( assert( j<=0x7fff ); if( j<0 ){ j = pTab->iPKey; - pIndex->bIdxRowid = 1; }else{ if( pTab->aCol[j].notNull==0 ){ pIndex->uniqNotNull = 0; diff --git a/src/expr.c b/src/expr.c index 349015f48..7e13e5bc0 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1144,7 +1144,7 @@ Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ return pLeft; }else{ u32 f = pLeft->flags | pRight->flags; - if( (f&(EP_OuterON|EP_InnerON|EP_IsFalse))==EP_IsFalse + if( (f&(EP_OuterON|EP_InnerON|EP_IsFalse|EP_HasFunc))==EP_IsFalse && !IN_RENAME_OBJECT ){ sqlite3ExprDeferredDelete(pParse, pLeft); diff --git a/src/parse.y b/src/parse.y index a5691cad4..617eb7303 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1429,12 +1429,21 @@ expr(A) ::= expr(A) between_op(N) expr(X) AND expr(Y). [BETWEEN] { ** expr1 IN () ** expr1 NOT IN () ** - ** simplify to constants 0 (false) and 1 (true), respectively, - ** regardless of the value of expr1. + ** simplify to constants 0 (false) and 1 (true), respectively. + ** + ** Except, do not apply this optimization if expr1 contains a function + ** because that function might be an aggregate (we don't know yet whether + ** it is or not) and if it is an aggregate, that could change the meaning + ** of the whole query. */ - sqlite3ExprUnmapAndDelete(pParse, A); - A = sqlite3Expr(pParse->db, TK_STRING, N ? "true" : "false"); - if( A ) sqlite3ExprIdToTrueFalse(A); + Expr *pB = sqlite3Expr(pParse->db, TK_STRING, N ? "true" : "false"); + if( pB ) sqlite3ExprIdToTrueFalse(pB); + if( !ExprHasProperty(A, EP_HasFunc) ){ + sqlite3ExprUnmapAndDelete(pParse, A); + A = pB; + }else{ + A = sqlite3PExpr(pParse, N ? TK_OR : TK_AND, pB, A); + } }else{ Expr *pRHS = Y->a[0].pExpr; if( Y->nExpr==1 && sqlite3ExprIsConstant(pParse,pRHS) && A->op!=TK_VECTOR ){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 4a79945f3..809f6462b 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -9058,13 +9058,13 @@ int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** [[SQLITE_STMTSTATUS_SORT]]
          SQLITE_STMTSTATUS_SORT
          **
          ^This is the number of sort operations that have occurred. ** A non-zero value in this counter may indicate an opportunity to -** improvement performance through careful use of indices.
          +** improve performance through careful use of indices. ** ** [[SQLITE_STMTSTATUS_AUTOINDEX]]
          SQLITE_STMTSTATUS_AUTOINDEX
          **
          ^This is the number of rows inserted into transient indices that ** were created automatically in order to help joins run faster. ** A non-zero value in this counter may indicate an opportunity to -** improvement performance by adding permanent indices that do not +** improve performance by adding permanent indices that do not ** need to be reinitialized each time the statement is run.
          ** ** [[SQLITE_STMTSTATUS_VM_STEP]]
          SQLITE_STMTSTATUS_VM_STEP
          @@ -9073,19 +9073,19 @@ int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** to 2147483647. The number of virtual machine operations can be ** used as a proxy for the total work done by the prepared statement. ** If the number of virtual machine operations exceeds 2147483647 -** then the value returned by this statement status code is undefined. +** then the value returned by this statement status code is undefined. ** ** [[SQLITE_STMTSTATUS_REPREPARE]]
          SQLITE_STMTSTATUS_REPREPARE
          **
          ^This is the number of times that the prepare statement has been ** automatically regenerated due to schema changes or changes to -** [bound parameters] that might affect the query plan. +** [bound parameters] that might affect the query plan.
          ** ** [[SQLITE_STMTSTATUS_RUN]]
          SQLITE_STMTSTATUS_RUN
          **
          ^This is the number of times that the prepared statement has ** been run. A single "run" for the purposes of this counter is one ** or more calls to [sqlite3_step()] followed by a call to [sqlite3_reset()]. ** The counter is incremented on the first [sqlite3_step()] call of each -** cycle. +** cycle.
          ** ** [[SQLITE_STMTSTATUS_FILTER_MISS]] ** [[SQLITE_STMTSTATUS_FILTER HIT]] @@ -9095,7 +9095,7 @@ int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** step was bypassed because a Bloom filter returned not-found. The ** corresponding SQLITE_STMTSTATUS_FILTER_MISS value is the number of ** times that the Bloom filter returned a find, and thus the join step -** had to be processed as normal. +** had to be processed as normal. ** ** [[SQLITE_STMTSTATUS_MEMUSED]]
          SQLITE_STMTSTATUS_MEMUSED
          **
          ^This is the approximate number of bytes of heap memory @@ -9200,9 +9200,9 @@ struct sqlite3_pcache_page { ** SQLite will typically create one cache instance for each open database file, ** though this is not guaranteed. ^The ** first parameter, szPage, is the size in bytes of the pages that must -** be allocated by the cache. ^szPage will always a power of two. ^The +** be allocated by the cache. ^szPage will always be a power of two. ^The ** second parameter szExtra is a number of bytes of extra storage -** associated with each page cache entry. ^The szExtra parameter will +** associated with each page cache entry. ^The szExtra parameter will be ** a number less than 250. SQLite will use the ** extra szExtra bytes on each page to store metadata about the underlying ** database page on disk. The value passed into szExtra depends @@ -9210,17 +9210,17 @@ struct sqlite3_pcache_page { ** ^The third argument to xCreate(), bPurgeable, is true if the cache being ** created will be used to cache database pages of a file stored on disk, or ** false if it is used for an in-memory database. The cache implementation -** does not have to do anything special based with the value of bPurgeable; +** does not have to do anything special based upon the value of bPurgeable; ** it is purely advisory. ^On a cache where bPurgeable is false, SQLite will ** never invoke xUnpin() except to deliberately delete a page. ** ^In other words, calls to xUnpin() on a cache with bPurgeable set to ** false will always have the "discard" flag set to true. -** ^Hence, a cache created with bPurgeable false will +** ^Hence, a cache created with bPurgeable set to false will ** never contain any unpinned pages. ** ** [[the xCachesize() page cache method]] ** ^(The xCachesize() method may be called at any time by SQLite to set the -** suggested maximum cache-size (number of pages stored by) the cache +** suggested maximum cache-size (number of pages stored) for the cache ** instance passed as the first argument. This is the value configured using ** the SQLite "[PRAGMA cache_size]" command.)^ As with the bPurgeable ** parameter, the implementation is not required to do anything with this @@ -9247,12 +9247,12 @@ struct sqlite3_pcache_page { ** implementation must return a pointer to the page buffer with its content ** intact. If the requested page is not already in the cache, then the ** cache implementation should use the value of the createFlag -** parameter to help it determined what action to take: +** parameter to help it determine what action to take: ** ** **
          createFlag Behavior when page is not already in cache **
          0 Do not allocate a new page. Return NULL. -**
          1 Allocate a new page if it easy and convenient to do so. +**
          1 Allocate a new page if it is easy and convenient to do so. ** Otherwise return NULL. **
          2 Make every effort to allocate a new page. Only return ** NULL if allocating a new page is effectively impossible. @@ -9269,7 +9269,7 @@ struct sqlite3_pcache_page { ** as its second argument. If the third parameter, discard, is non-zero, ** then the page must be evicted from the cache. ** ^If the discard parameter is -** zero, then the page may be discarded or retained at the discretion of +** zero, then the page may be discarded or retained at the discretion of the ** page cache implementation. ^The page cache implementation ** may choose to evict unpinned pages at any time. ** @@ -9287,7 +9287,7 @@ struct sqlite3_pcache_page { ** When SQLite calls the xTruncate() method, the cache must discard all ** existing cache entries with page numbers (keys) greater than or equal ** to the value of the iLimit parameter passed to xTruncate(). If any -** of these pages are pinned, they are implicitly unpinned, meaning that +** of these pages are pinned, they become implicitly unpinned, meaning that ** they can be safely discarded. ** ** [[the xDestroy() page cache method]] @@ -9586,7 +9586,7 @@ int sqlite3_backup_pagecount(sqlite3_backup *p); ** application receives an SQLITE_LOCKED error, it may call the ** sqlite3_unlock_notify() method with the blocked connection handle as ** the first argument to register for a callback that will be invoked -** when the blocking connections current transaction is concluded. ^The +** when the blocking connection's current transaction is concluded. ^The ** callback is invoked from within the [sqlite3_step] or [sqlite3_close] ** call that concludes the blocking connection's transaction. ** @@ -9606,7 +9606,7 @@ int sqlite3_backup_pagecount(sqlite3_backup *p); ** blocked connection already has a registered unlock-notify callback, ** then the new callback replaces the old.)^ ^If sqlite3_unlock_notify() is ** called with a NULL pointer as its second argument, then any existing -** unlock-notify callback is canceled. ^The blocked connections +** unlock-notify callback is canceled. ^The blocked connection's ** unlock-notify callback may also be canceled by closing the blocked ** connection using [sqlite3_close()]. ** @@ -10004,7 +10004,7 @@ int sqlite3_vtab_config(sqlite3*, int op, ...); ** support constraints. In this configuration (which is the default) if ** a call to the [xUpdate] method returns [SQLITE_CONSTRAINT], then the entire ** statement is rolled back as if [ON CONFLICT | OR ABORT] had been -** specified as part of the users SQL statement, regardless of the actual +** specified as part of the user's SQL statement, regardless of the actual ** ON CONFLICT mode specified. ** ** If X is non-zero, then the virtual table implementation guarantees @@ -10038,7 +10038,7 @@ int sqlite3_vtab_config(sqlite3*, int op, ...); ** [[SQLITE_VTAB_INNOCUOUS]]
          SQLITE_VTAB_INNOCUOUS
          **
          Calls of the form ** [sqlite3_vtab_config](db,SQLITE_VTAB_INNOCUOUS) from within the -** the [xConnect] or [xCreate] methods of a [virtual table] implementation +** [xConnect] or [xCreate] methods of a [virtual table] implementation ** identify that virtual table as being safe to use from within triggers ** and views. Conceptually, the SQLITE_VTAB_INNOCUOUS tag means that the ** virtual table can do no serious harm even if it is controlled by a @@ -10206,7 +10206,7 @@ const char *sqlite3_vtab_collation(sqlite3_index_info*,int); **
          ** ** ^For the purposes of comparing virtual table output values to see if the -** values are same value for sorting purposes, two NULL values are considered +** values are the same value for sorting purposes, two NULL values are considered ** to be the same. In other words, the comparison operator is "IS" ** (or "IS NOT DISTINCT FROM") and not "==". ** @@ -10216,7 +10216,7 @@ const char *sqlite3_vtab_collation(sqlite3_index_info*,int); ** ** ^A virtual table implementation is always free to return rows in any order ** it wants, as long as the "orderByConsumed" flag is not set. ^When the -** the "orderByConsumed" flag is unset, the query planner will add extra +** "orderByConsumed" flag is unset, the query planner will add extra ** [bytecode] to ensure that the final results returned by the SQL query are ** ordered correctly. The use of the "orderByConsumed" flag and the ** sqlite3_vtab_distinct() interface is merely an optimization. ^Careful @@ -10313,7 +10313,7 @@ int sqlite3_vtab_in(sqlite3_index_info*, int iCons, int bHandle); ** sqlite3_vtab_in_next(X,P) should be one of the parameters to the ** xFilter method which invokes these routines, and specifically ** a parameter that was previously selected for all-at-once IN constraint -** processing use the [sqlite3_vtab_in()] interface in the +** processing using the [sqlite3_vtab_in()] interface in the ** [xBestIndex|xBestIndex method]. ^(If the X parameter is not ** an xFilter argument that was selected for all-at-once IN constraint ** processing, then these routines return [SQLITE_ERROR].)^ @@ -10368,7 +10368,7 @@ int sqlite3_vtab_in_next(sqlite3_value *pVal, sqlite3_value **ppOut); ** and only if *V is set to a value. ^The sqlite3_vtab_rhs_value(P,J,V) ** inteface returns SQLITE_NOTFOUND if the right-hand side of the J-th ** constraint is not available. ^The sqlite3_vtab_rhs_value() interface -** can return an result code other than SQLITE_OK or SQLITE_NOTFOUND if +** can return a result code other than SQLITE_OK or SQLITE_NOTFOUND if ** something goes wrong. ** ** The sqlite3_vtab_rhs_value() interface is usually only successful if @@ -10396,8 +10396,8 @@ int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); ** KEYWORDS: {conflict resolution mode} ** ** These constants are returned by [sqlite3_vtab_on_conflict()] to -** inform a [virtual table] implementation what the [ON CONFLICT] mode -** is for the SQL statement being evaluated. +** inform a [virtual table] implementation of the [ON CONFLICT] mode +** for the SQL statement being evaluated. ** ** Note that the [SQLITE_IGNORE] constant is also used as a potential ** return value from the [sqlite3_set_authorizer()] callback and that @@ -10437,39 +10437,39 @@ int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); ** [[SQLITE_SCANSTAT_EST]]
          SQLITE_SCANSTAT_EST
          **
          ^The "double" variable pointed to by the V parameter will be set to the ** query planner's estimate for the average number of rows output from each -** iteration of the X-th loop. If the query planner's estimates was accurate, +** iteration of the X-th loop. If the query planner's estimate was accurate, ** then this value will approximate the quotient NVISIT/NLOOP and the ** product of this value for all prior loops with the same SELECTID will -** be the NLOOP value for the current loop. +** be the NLOOP value for the current loop.
          ** ** [[SQLITE_SCANSTAT_NAME]]
          SQLITE_SCANSTAT_NAME
          **
          ^The "const char *" variable pointed to by the V parameter will be set ** to a zero-terminated UTF-8 string containing the name of the index or table -** used for the X-th loop. +** used for the X-th loop.
          ** ** [[SQLITE_SCANSTAT_EXPLAIN]]
          SQLITE_SCANSTAT_EXPLAIN
          **
          ^The "const char *" variable pointed to by the V parameter will be set ** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] -** description for the X-th loop. +** description for the X-th loop.
          ** ** [[SQLITE_SCANSTAT_SELECTID]]
          SQLITE_SCANSTAT_SELECTID
          **
          ^The "int" variable pointed to by the V parameter will be set to the ** id for the X-th query plan element. The id value is unique within the ** statement. The select-id is the same value as is output in the first -** column of an [EXPLAIN QUERY PLAN] query. +** column of an [EXPLAIN QUERY PLAN] query.
          ** ** [[SQLITE_SCANSTAT_PARENTID]]
          SQLITE_SCANSTAT_PARENTID
          **
          The "int" variable pointed to by the V parameter will be set to the -** the id of the parent of the current query element, if applicable, or +** id of the parent of the current query element, if applicable, or ** to zero if the query element has no parent. This is the same value as -** returned in the second column of an [EXPLAIN QUERY PLAN] query. +** returned in the second column of an [EXPLAIN QUERY PLAN] query.
          ** ** [[SQLITE_SCANSTAT_NCYCLE]]
          SQLITE_SCANSTAT_NCYCLE
          **
          The sqlite3_int64 output value is set to the number of cycles, ** according to the processor time-stamp counter, that elapsed while the ** query element was being processed. This value is not available for ** all query elements - if it is unavailable the output variable is -** set to -1. +** set to -1.
          ** */ #define SQLITE_SCANSTAT_NLOOP 0 @@ -10510,8 +10510,8 @@ int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); ** sqlite3_stmt_scanstatus_v2() with a zeroed flags parameter. ** ** Parameter "idx" identifies the specific query element to retrieve statistics -** for. Query elements are numbered starting from zero. A value of -1 may be -** to query for statistics regarding the entire query. ^If idx is out of range +** for. Query elements are numbered starting from zero. A value of -1 may +** retrieve statistics for the entire query. ^If idx is out of range ** - less than -1 or greater than or equal to the total number of query ** elements used to implement the statement - a non-zero value is returned and ** the variable that pOut points to is unchanged. @@ -10668,8 +10668,8 @@ int sqlite3_db_cacheflush(sqlite3*); ** triggers; and so forth. ** ** When the [sqlite3_blob_write()] API is used to update a blob column, -** the pre-update hook is invoked with SQLITE_DELETE. This is because the -** in this case the new values are not available. In this case, when a +** the pre-update hook is invoked with SQLITE_DELETE, because +** the new values are not yet available. In this case, when a ** callback made with op==SQLITE_DELETE is actually a write using the ** sqlite3_blob_write() API, the [sqlite3_preupdate_blobwrite()] returns ** the index of the column being written. In other cases, where the @@ -10922,7 +10922,7 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); ** For an ordinary on-disk database file, the serialization is just a ** copy of the disk file. For an in-memory database or a "TEMP" database, ** the serialization is the same sequence of bytes which would be written -** to disk if that database where backed up to disk. +** to disk if that database were backed up to disk. ** ** The usual case is that sqlite3_serialize() copies the serialization of ** the database into memory obtained from [sqlite3_malloc64()] and returns @@ -10931,7 +10931,7 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); ** contains the SQLITE_SERIALIZE_NOCOPY bit, then no memory allocations ** are made, and the sqlite3_serialize() function will return a pointer ** to the contiguous memory representation of the database that SQLite -** is currently using for that database, or NULL if the no such contiguous +** is currently using for that database, or NULL if no such contiguous ** memory representation of the database exists. A contiguous memory ** representation of the database will usually only exist if there has ** been a prior call to [sqlite3_deserialize(D,S,...)] with the same @@ -11002,7 +11002,7 @@ unsigned char *sqlite3_serialize( ** database is currently in a read transaction or is involved in a backup ** operation. ** -** It is not possible to deserialized into the TEMP database. If the +** It is not possible to deserialize into the TEMP database. If the ** S argument to sqlite3_deserialize(D,S,P,N,M,F) is "temp" then the ** function returns SQLITE_ERROR. ** @@ -11024,7 +11024,7 @@ int sqlite3_deserialize( sqlite3 *db, /* The database connection */ const char *zSchema, /* Which DB to reopen with the deserialization */ unsigned char *pData, /* The serialized database content */ - sqlite3_int64 szDb, /* Number bytes in the deserialization */ + sqlite3_int64 szDb, /* Number of bytes in the deserialization */ sqlite3_int64 szBuf, /* Total size of buffer pData[] */ unsigned mFlags /* Zero or more SQLITE_DESERIALIZE_* flags */ ); @@ -11032,7 +11032,7 @@ int sqlite3_deserialize( /* ** CAPI3REF: Flags for sqlite3_deserialize() ** -** The following are allowed values for 6th argument (the F argument) to +** The following are allowed values for the 6th argument (the F argument) to ** the [sqlite3_deserialize(D,S,P,N,M,F)] interface. ** ** The SQLITE_DESERIALIZE_FREEONCLOSE means that the database serialization diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5cc134a9d..dee1d07fe 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2792,7 +2792,6 @@ struct Index { unsigned hasStat1:1; /* aiRowLogEst values come from sqlite_stat1 */ unsigned bNoQuery:1; /* Do not use this index to optimize queries */ unsigned bAscKeyBug:1; /* True if the bba7b69f9849b5bf bug applies */ - unsigned bIdxRowid:1; /* One or more of the index keys is the ROWID */ unsigned bHasVCol:1; /* Index references one or more VIRTUAL columns */ unsigned bHasExpr:1; /* Index contains an expression, either a literal ** expression, or a reference to a VIRTUAL column */ diff --git a/src/tokenize.c b/src/tokenize.c index e4d9f5371..6f7bab35b 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -199,7 +199,7 @@ static int getToken(const unsigned char **pz){ int t; /* Token type to return */ do { z += sqlite3GetToken(z, &t); - }while( t==TK_SPACE ); + }while( t==TK_SPACE || t==TK_COMMENT ); if( t==TK_ID || t==TK_STRING || t==TK_JOIN_KW diff --git a/src/where.c b/src/where.c index 9561a75ea..d1312952d 100644 --- a/src/where.c +++ b/src/where.c @@ -3233,6 +3233,7 @@ static int whereLoopAddBtreeIndex( if( ExprUseXSelect(pExpr) ){ /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */ int i; + int bRedundant = 0; nIn = 46; assert( 46==sqlite3LogEst(25) ); /* The expression may actually be of the form (x, y) IN (SELECT...). @@ -3241,7 +3242,20 @@ static int whereLoopAddBtreeIndex( ** for each such term. The following loop checks that pTerm is the ** first such term in use, and sets nIn back to 0 if it is not. */ for(i=0; inLTerm-1; i++){ - if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ) nIn = 0; + if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ){ + nIn = 0; + if( pNew->aLTerm[i]->u.x.iField == pTerm->u.x.iField ){ + /* Detect when two or more columns of an index match the same + ** column of a vector IN operater, and avoid adding the column + ** to the WhereLoop more than once. See tag-20250707-01 + ** in test/rowvalue.test */ + bRedundant = 1; + } + } + } + if( bRedundant ){ + pNew->nLTerm--; + continue; } }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ /* "x IN (value, value, ...)" */ @@ -3473,7 +3487,7 @@ static int whereLoopAddBtreeIndex( if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->u.btree.nEqnColumn && (pNew->u.btree.nEqnKeyCol || - (pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY && !pProbe->bIdxRowid)) + pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY) ){ if( pNew->u.btree.nEq>3 ){ sqlite3ProgressCheck(pParse); diff --git a/src/wherecode.c b/src/wherecode.c index 8e3e56cb1..823b4884e 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -598,7 +598,9 @@ static Expr *removeUnindexableInClauseTerms( int iField; assert( (pLoop->aLTerm[i]->eOperator & (WO_OR|WO_AND))==0 ); iField = pLoop->aLTerm[i]->u.x.iField - 1; - if( pOrigRhs->a[iField].pExpr==0 ) continue; /* Duplicate PK column */ + if( NEVER(pOrigRhs->a[iField].pExpr==0) ){ + continue; /* Duplicate PK column */ + } pRhs = sqlite3ExprListAppend(pParse, pRhs, pOrigRhs->a[iField].pExpr); pOrigRhs->a[iField].pExpr = 0; if( pRhs ) pRhs->a[pRhs->nExpr-1].u.x.iOrderByCol = iField+1; @@ -695,7 +697,7 @@ static SQLITE_NOINLINE void codeINTerm( return; } } - for(i=iEq;inLTerm; i++){ + for(i=iEq; inLTerm; i++){ assert( pLoop->aLTerm[i]!=0 ); if( pLoop->aLTerm[i]->pExpr==pX ) nEq++; } @@ -704,22 +706,13 @@ static SQLITE_NOINLINE void codeINTerm( if( !ExprUseXSelect(pX) || pX->x.pSelect->pEList->nExpr==1 ){ eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, 0, &iTab); }else{ - Expr *pExpr = pTerm->pExpr; - if( pExpr->iTable==0 || !ExprHasProperty(pExpr, EP_Subrtn) ){ - sqlite3 *db = pParse->db; - pX = removeUnindexableInClauseTerms(pParse, iEq, pLoop, pX); - if( !db->mallocFailed ){ - aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*nEq); - eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap,&iTab); - pExpr->iTable = iTab; - } - sqlite3ExprDelete(db, pX); - }else{ - int n = sqlite3ExprVectorSize(pX->pLeft); - aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*MAX(nEq,n)); - eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap, &iTab); + sqlite3 *db = pParse->db; + Expr *pXMod = removeUnindexableInClauseTerms(pParse, iEq, pLoop, pX); + if( !db->mallocFailed ){ + aiMap = (int*)sqlite3DbMallocZero(db, sizeof(int)*nEq); + eType = sqlite3FindInIndex(pParse, pXMod, IN_INDEX_LOOP, 0, aiMap, &iTab); } - pX = pExpr; + sqlite3ExprDelete(db, pXMod); } if( eType==IN_INDEX_INDEX_DESC ){ @@ -749,7 +742,7 @@ static SQLITE_NOINLINE void codeINTerm( if( pIn ){ int iMap = 0; /* Index in aiMap[] */ pIn += i; - for(i=iEq;inLTerm; i++){ + for(i=iEq; inLTerm; i++){ if( pLoop->aLTerm[i]->pExpr==pX ){ int iOut = iTarget + i - iEq; if( eType==IN_INDEX_ROWID ){ diff --git a/test/altertab2.test b/test/altertab2.test index 56e42f1a6..f2a1d74c4 100644 --- a/test/altertab2.test +++ b/test/altertab2.test @@ -358,7 +358,7 @@ do_catchsql_test 8.6 { CREATE INDEX i0 ON t0(likelihood(1,2) AND 0); ALTER TABLE t0 RENAME TO t1; SELECT sql FROM sqlite_master WHERE name='i0'; -} {1 {error in index i0: second argument to likelihood() must be a constant between 0.0 and 1.0}} +} {1 {second argument to likelihood() must be a constant between 0.0 and 1.0}} reset_db diff --git a/test/altertab3.test b/test/altertab3.test index 5f5c11b0b..92060fb41 100644 --- a/test/altertab3.test +++ b/test/altertab3.test @@ -190,14 +190,14 @@ do_execsql_test 8.1 { } do_execsql_test 8.2.1 { CREATE TABLE t2 (c0); - CREATE INDEX i2 ON t2((LIKELIHOOD(c0, 100) IN ())); + CREATE INDEX i2 ON t2((LIKELIHOOD(c0, 1.0) IN ())); ALTER TABLE t2 RENAME COLUMN c0 TO c1; } do_execsql_test 8.2.2 { SELECT sql FROM sqlite_master WHERE tbl_name = 't2'; } { {CREATE TABLE t2 (c1)} - {CREATE INDEX i2 ON t2((LIKELIHOOD(c0, 100) IN ()))} + {CREATE INDEX i2 ON t2((LIKELIHOOD(c1, 1.0) IN ()))} } do_test 8.2.3 { sqlite3 db2 test.db @@ -662,14 +662,6 @@ do_execsql_test 27.2 { {CREATE TABLE t1(a, b AS ((WITH w1 (xyz) AS ( SELECT t1.b FROM t1 ) SELECT 123) IN ()))} } -do_execsql_test 27.3 { - CREATE TABLE t0(c0 , c1 AS (CASE TRUE NOT IN () WHEN NULL THEN CASE + 0xa ISNULL WHEN NOT + 0x9 THEN t0.c1 ELSE CURRENT_TIME LIKE CAST (t0.c1 REGEXP '-([1-9]\d*.\d*|0\.\d*[1-9]\d*)'ESCAPE (c1) COLLATE BINARY BETWEEN c1 AND c1 NOT IN (WITH t4 (c0) AS (WITH t3 (c0) AS NOT MATERIALIZED (WITH RECURSIVE t2 (c0) AS (WITH RECURSIVE t1 AS (VALUES (x'717171ff71717171' ) ) SELECT DISTINCT t0.c0 FROM t0 NOT INDEXED WHERE t0.c0 =t0.c0 GROUP BY 0x9 ) SELECT DISTINCT t0.c0 FROM t0 NOT INDEXED WHERE t0.c0 =t0.c1 ) SELECT DISTINCT t0.c0 FROM t0 NOT INDEXED WHERE t0.c0 =t0.c0 GROUP BY typeof(0x9 ) ) SELECT DISTINCT t0.c0 FROM t0 NOT INDEXED WHERE t0.c0 =t0.c0 GROUP BY typeof(typeof(0x9 ) ) ) IN t0 BETWEEN typeof(typeof(typeof(hex(*) FILTER (WHERE + x'5ccd1e68' ) ) ) ) AND 1 >0xa AS BLOB (+4.4E4 , -0xe ) ) END <> c1 IN () END ) VIRTUAL , c35 PRIMARY KEY , c60 , c64 NUMERIC (-6.8 , -0xE ) ) WITHOUT ROWID ; -} {} - -do_execsql_test 27.4 { - ALTER TABLE t0 DROP COLUMN c60; -} {} - #------------------------------------------------------------------------- reset_db do_execsql_test 28.1 { diff --git a/test/parser1.test b/test/parser1.test index ad95b4909..b8d3d8b42 100644 --- a/test/parser1.test +++ b/test/parser1.test @@ -100,4 +100,26 @@ do_execsql_test parser1-3.1 { PRAGMA foreign_key_list(t301); } {0 0 t300 c2 id RESTRICT CASCADE NONE 1 0 t300 c1 id RESTRICT CASCADE NONE} +# 2025-07-01 https://sqlite.org/forum/forumpost/f4878de3e7dd4764 +# Do not allow parse-time optimizations to omit aggregate functions, +# because doing so can change the meaning of the query. +# +unset -nocomplain zero +set zero [expr {0+0}] +do_execsql_test parser1-4.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(x); + SELECT max(x) AND $zero FROM t1; +} 0 +do_execsql_test parser1-4.2 { + SELECT max(x) AND 0 FROM t1; +} 0 +do_execsql_test parser1-4.3 { + SELECT max(x) IN () FROM t1; +} 0 +do_execsql_test parser1-4.4 { + SELECT max(x) NOT IN () FROM t1; +} 1 + + finish_test diff --git a/test/rowvalue.test b/test/rowvalue.test index e2688e903..387780c45 100644 --- a/test/rowvalue.test +++ b/test/rowvalue.test @@ -788,6 +788,9 @@ do_execsql_test 33.3 { # INTEGER PRIMARY KEY, and the columns that UNIQUE constraint are # used in a rowvalue-IN operator constraint. # +# 2025-07-07 Discovered that the original fix was incomplete and +# new tests added. See tag-20250707-01 in the code. +# reset_db do_execsql_test 34.1 { CREATE TABLE items ( @@ -804,5 +807,39 @@ do_execsql_test 34.1 { WHERE (Id, Item) IN (SELECT Id, Item FROM items); SELECT Id, Item, test FROM items ORDER BY id; } {1 2 ok 2 2 ok 3 3 ok 4 5 ok} +db null NULL +do_execsql_test 34.2 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); + CREATE INDEX idx ON t1(b,a); + INSERT INTO t1(a,b) VALUES (1, 22); + SELECT * FROM t1 INDEXED BY idx WHERE (b,a) IN (SELECT b,a FROM t1); +} {1 22 NULL NULL} +do_execsql_test 34.3 { + DROP TABLE t1; + CREATE TABLE t1(a, b, c, d); + CREATE INDEX idx ON t1(b,a,a); + INSERT INTO t1(a,b) VALUES (1, 22); + SELECT * FROM t1 INDEXED BY idx WHERE (b,a) IN (SELECT b,a FROM t1); +} {1 22 NULL NULL} +do_execsql_test 34.4 { + DROP TABLE t1; + CREATE TABLE t1(id INTEGER PRIMARY KEY, a INT); + CREATE INDEX t1a ON t1(a,id); -- index includes PRIMARY KEY + CREATE TABLE t2(id INTEGER PRIMARY KEY); + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<100) + INSERT INTO t1(id,a) SELECT n, 777 FROM c; + INSERT INTO t2 SELECT id FROM t1; + SELECT * + FROM t1 JOIN t2 USING(id) + WHERE t1.a=777 AND t2.id>999 + ORDER BY t1.id; +} {} +do_execsql_test 34.5 { + EXPLAIN QUERY PLAN + SELECT * + FROM t1 JOIN t2 USING(id) + WHERE t1.a=777 AND t2.id>999 + ORDER BY t1.id; +} {/SEARCH t1 USING COVERING INDEX t1a .a=. AND id>../} finish_test From 4b546c015b93d4a142d5f435791bb0c0399c24be Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Tue, 22 Jul 2025 09:20:25 -0400 Subject: [PATCH 26/75] Updates CHANGELOG.md with relevant changes for 4.10.0 --- CHANGELOG.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f1c10436..bbcb9113c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,14 @@ # SQLCipher Change Log Notable changes to this project are documented in this file. -## [4.10.0] - (? 2025 - [4.10.0 changes]) +## [4.11.0] - (? 2025 - [4.11.0 changes]) + +## [4.10.0] - (July 2025 - [4.10.0 changes]) +- Updates baseline to latest SQLite 3.50.3 +- Allows compile time override of default log level via `SQLCIPHER_LOG_LEVEL_DEFAULT` macro +- Fixes issue building with `-fstanitize=address` on macOS +- Fixes detection of CommonCrypto version on macOS +- Improves CommonCrypto version detection on iOS ## [4.9.0] - (May 2025 - [4.9.0 changes]) - Updates baseline to upstream SQLite 3.49.2 @@ -290,6 +297,8 @@ __BREAKING CHANGE__: `SELECT` statements (now also including schema independent ### Security - Change KDF iteration length from 4,000 to 64,000 +[4.11.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.11.0 +[4.11.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.10.0...v4.11.0 [4.10.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.10.0 [4.10.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.9.0...v4.10.0 [4.9.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.9.0 From da0a22872215399ddb0bc25cdf73b5b089adaaa2 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 1 Aug 2025 11:31:27 -0400 Subject: [PATCH 27/75] Snapshot of upstream SQLite 3.50.4 --- VERSION | 2 +- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/expr.c | 25 +++++++++++++++++++++++++ src/select.c | 1 + src/sqliteInt.h | 2 ++ src/where.c | 3 +-- 7 files changed, 42 insertions(+), 15 deletions(-) diff --git a/VERSION b/VERSION index 6a6d84a54..acecb4fcb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.50.3 +3.50.4 diff --git a/manifest b/manifest index 0ee8c25cf..62c1d4b0c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.50.3 -D 2025-07-17T13:25:10.611 +C Version\s3.50.4 +D 2025-07-30T19:33:53.432 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -8,7 +8,7 @@ F Makefile.in c3e414df4dc8dfb12f1f6baf129fcb6d18cd0ebd3c9109370fb3fceeeef9a37a F Makefile.linux-generic bd3e3cacd369821a6241d4ea1967395c962dfe3057e38cb0a435cee0e8b789d0 F Makefile.msc 0206f28988bb6634c7e8aff05bf6cfa65d6dfe1d2b6bd95160dd99290a83dfc7 F README.md e28077cfbef795e99c9c75ed95aa7257a1166709b562076441a8506ac421b7c1 -F VERSION 613f62a5375c819156a56fa263adc08f8ee5998f45a946d3c5d594385b2ef736 +F VERSION 33191da58a15d51e312248db3124d33e1dbc1797d7f8fe14d9df5ff07430b503 F art/icon-243x273.gif 9750b734f82fdb3dc43127753d5e6fbf3b62c9f4e136c2fbf573b2f57ea87af5 F art/icon-80x90.gif 65509ce3e5f86a9cd64fe7fca2d23954199f31fe44c1e09e208c80fb83d87031 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -736,7 +736,7 @@ F src/date.c 9db4d604e699a73e10b8e85a44db074a1f04c0591a77e2abfd77703f50dce1e9 F src/dbpage.c fcb1aafe00872a8aff9a7aa0ef7ff1b01e5817ec7bbd521f8f3e1e674ac8d609 F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c F src/delete.c 03a77ba20e54f0f42ebd8eddf15411ed6bdb06a2c472ac4b6b336521bf7cea42 -F src/expr.c 7e7f57247e864aaa46af2fd6bf3f7433dbefffc470687158340e5d68a15c4469 +F src/expr.c 72e7cddcb2ea06de48f40ee301b81608d86a17b73781e42901ce585f53074589 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f F src/func.c de47a8295503aa130baae5e6d9868ecf4f7c4dbffa65d83ad1f70bdbac0ee2d6 @@ -785,12 +785,12 @@ F src/printf.c 3b91c334f528359145f4dde0dedd945bbb21044d0825ea064934d7222d61662c F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d40fe18d7c2fd0339f5846ffcf7d6809866e380acdf14c76fb2af87e9fe13f64 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c 7a21df5db6bb1a4c1bb6d9fb76c8e2485a22ff8306519ad69d8ddf0d5fa10903 +F src/select.c 5c129b669317a1d57283055482b9c1e105199a7e47d69526491ca165d3376999 F src/shell.c.in ba53a52dafb167ac6320703da741386c34fbcabe8c078a188bb9f89808e3ef8f F src/sqlite.h.in 9ae373d11e1b11ac9c81c508523ae37f1619e739858280078ee9fb4e1e62d3ed F src/sqlite3.rc 015537e6ac1eec6c7050e17b616c2ffe6f70fca241835a84a4f0d5937383c479 F src/sqlite3ext.h 0bfd049bb2088cc44c2ad54f2079d1c6e43091a4e1ce8868779b75f6c1484f1e -F src/sqliteInt.h 08a7dcb8db162875b3dd5229eb0bf75691150ac54fea1ff3670391bd6ef40546 +F src/sqliteInt.h 0c24996d4c02eca6fd971ecbe5a565aaaf5f024265a53ce62d77ee366fcc7c6a F src/sqliteLimit.h 6d817c28a8f19af95e6f4921933b7fbbca48a962bce0eb0ec81e8bb3ef38e68b F src/status.c 0e72e4f6be6ccfde2488eb63210297e75f569f3ce9920f6c3d77590ec6ce5ffd F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -869,7 +869,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 20be6f0a25a80b7897cf2a5369bfd37ef198e6f0b6cdef16d83eee856056b159 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 -F src/where.c 4b9f73b1633b4dcddd82abc69aa6384dbef528289e8daa29521c3112b82fd1b9 +F src/where.c 09fea7e821556e8f8851b68edc39d14becb6ee64bea88e34b05ceeaadcdfaecd F src/whereInt.h ecdbfb5551cf394f04ec7f0bc7ad963146d80eee3071405ac29aa84950128b8e F src/wherecode.c 66684a11713667f532a69214ac552acee7e0825e2f1b71abaebe05511d190fb4 F src/whereexpr.c 1c60db88b6e8472f4864b98014d1b2d9d16bdd42d5d181ec515acbcdeddc18db @@ -2209,10 +2209,10 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 1ad0169b022b280bcaaf94a7fa231591be96b514230ab5c98fbf15cd7df842dd F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 63595b74956a9391f03a273204c80ecd0ba946846b7aa0195b9095fe8b6a78e5 -R 987003a9608f543153f11a48f885adf4 +P f39a0865d336d4a964627d4819ce7ba8dafd43a0e8e2a4431ea431ef73dd96c3 +R 5c3da0295a5c868b2c70cbb25c7978d3 T +sym-release * -T +sym-version-3.50.3 * +T +sym-version-3.50.4 * U drh -Z 351123ab09d4bd77e28a506f711aa720 +Z e0e73bda1cca0dba2213fe2c15c764d6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d2cf7ea04..1ed3eba93 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3ce993b8657d6d9deda380a93cdd6404a8c8ba1b185b2bc423703e41ae5f2543 +4d8adfb30e03f9cf27f800a2c1ba3c48fb4ca1b08b0f5ed59a4d5ecbf45e20a3 diff --git a/src/expr.c b/src/expr.c index 7e13e5bc0..677e4fe92 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4898,6 +4898,12 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3VdbeLoadString(v, target, pExpr->u.zToken); return target; } + case TK_NULLS: { + /* Set a range of registers to NULL. pExpr->y.nReg registers starting + ** with target */ + sqlite3VdbeAddOp3(v, OP_Null, 0, target, target + pExpr->y.nReg - 1); + return target; + } default: { /* Make NULL the default case so that if a bug causes an illegal ** Expr node to be passed into this function, it will be handled @@ -5582,6 +5588,25 @@ int sqlite3ExprCodeRunJustOnce( return regDest; } +/* +** Make arrangements to invoke OP_Null on a range of registers +** during initialization. +*/ +SQLITE_NOINLINE void sqlite3ExprNullRegisterRange( + Parse *pParse, /* Parsing context */ + int iReg, /* First register to set to NULL */ + int nReg /* Number of sequential registers to NULL out */ +){ + u8 okConstFactor = pParse->okConstFactor; + Expr t; + memset(&t, 0, sizeof(t)); + t.op = TK_NULLS; + t.y.nReg = nReg; + pParse->okConstFactor = 1; + sqlite3ExprCodeRunJustOnce(pParse, &t, iReg); + pParse->okConstFactor = okConstFactor; +} + /* ** Generate code to evaluate an expression and store the results ** into a register. Return the register number where the results diff --git a/src/select.c b/src/select.c index 8e4c939cd..dc4e87393 100644 --- a/src/select.c +++ b/src/select.c @@ -8363,6 +8363,7 @@ int sqlite3Select( sqlite3VdbeAddOp2(v, OP_Integer, 0, iAbortFlag); VdbeComment((v, "clear abort flag")); sqlite3VdbeAddOp3(v, OP_Null, 0, iAMem, iAMem+pGroupBy->nExpr-1); + sqlite3ExprNullRegisterRange(pParse, iAMem, pGroupBy->nExpr); /* Begin a loop that will extract all source rows in GROUP BY order. ** This might involve two separate loops with an OP_Sort in between, or diff --git a/src/sqliteInt.h b/src/sqliteInt.h index dee1d07fe..76aef3d12 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3064,6 +3064,7 @@ struct Expr { Table *pTab; /* TK_COLUMN: Table containing column. Can be NULL ** for a column of an index on an expression */ Window *pWin; /* EP_WinFunc: Window/Filter defn for a function */ + int nReg; /* TK_NULLS: Number of registers to NULL out */ struct { /* TK_IN, TK_SELECT, and TK_EXISTS */ int iAddr; /* Subroutine entry address */ int regReturn; /* Register used to hold return address */ @@ -5098,6 +5099,7 @@ void sqlite3ExprCodeGeneratedColumn(Parse*, Table*, Column*, int); void sqlite3ExprCodeCopy(Parse*, Expr*, int); void sqlite3ExprCodeFactorable(Parse*, Expr*, int); int sqlite3ExprCodeRunJustOnce(Parse*, Expr*, int); +void sqlite3ExprNullRegisterRange(Parse*, int, int); int sqlite3ExprCodeTemp(Parse*, Expr*, int*); int sqlite3ExprCodeTarget(Parse*, Expr*, int); int sqlite3ExprCodeExprList(Parse*, ExprList*, int, int, u8); diff --git a/src/where.c b/src/where.c index d1312952d..b0b4de424 100644 --- a/src/where.c +++ b/src/where.c @@ -4030,6 +4030,7 @@ static int whereLoopAddBtree( pNew->u.btree.nEq = 0; pNew->u.btree.nBtm = 0; pNew->u.btree.nTop = 0; + pNew->u.btree.nDistinctCol = 0; pNew->nSkip = 0; pNew->nLTerm = 0; pNew->iSortIdx = 0; @@ -5098,8 +5099,6 @@ static i8 wherePathSatisfiesOrderBy( obSat = obDone; } break; - }else if( wctrlFlags & WHERE_DISTINCTBY ){ - pLoop->u.btree.nDistinctCol = 0; } iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; From d41a25f448ba08ce24c0a599cf322046bdaa135a Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 1 Aug 2025 11:43:28 -0400 Subject: [PATCH 28/75] Updates CHANGELOG to reference 3.50.4 upstream merge --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbcb9113c..f10939324 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,8 @@ Notable changes to this project are documented in this file. ## [4.11.0] - (? 2025 - [4.11.0 changes]) -## [4.10.0] - (July 2025 - [4.10.0 changes]) -- Updates baseline to latest SQLite 3.50.3 +## [4.10.0] - (August 2025 - [4.10.0 changes]) +- Updates baseline to SQLite 3.50.4 - Allows compile time override of default log level via `SQLCIPHER_LOG_LEVEL_DEFAULT` macro - Fixes issue building with `-fstanitize=address` on macOS - Fixes detection of CommonCrypto version on macOS From 88dd0c3370fd37b2c7900d0a7de93ea9f55a8c11 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 8 Aug 2025 14:13:40 -0400 Subject: [PATCH 29/75] Converts log output to UTF-16 when writing to stdout or stderr on Windows --- src/sqlcipher.c | 46 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 884f7bb4f..1c558a526 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -2266,6 +2266,43 @@ static int sqlcipher_codec_add_random(codec_ctx *ctx, const char *zRight, int ra return SQLITE_ERROR; } +#if defined(_WIN32) +/* On windows convert to utf-16 when writing to stderr or stdout to avoid + * a potential exception when writing mixed context to those streams + * when using the shell. */ +static int sqlcipher_fprintf(FILE* stream, const char* format, ...) { + int sz; + va_list ap; + + if (stream == stderr || stream == stdout) { + char* buffer = NULL; + wchar_t* wbuffer = NULL; + + va_start(ap, format); + buffer = sqlite3_vmprintf(format, ap); + va_end(ap); + sz = (int)strlen(buffer); + + wbuffer = sqlite3_malloc((sz + 1) * sizeof(wchar_t)); + if (wbuffer == NULL) return NULL; + + sz = MultiByteToWideChar(CP_UTF8, 0, buffer, sz, wbuffer, sz); + wbuffer[sz] = NULL; + fputws(wbuffer, stream); + + sqlite3_free(wbuffer); + sqlite3_free(buffer); + } else { + va_start(ap, format); + sz = vfprintf(stream, format, ap); + va_end(ap); + } + return sz; +} +#else +#define sqlcipher_fprintf fprintf +#endif + #if !defined(SQLITE_OMIT_TRACE) #define SQLCIPHER_PROFILE_FMT "Elapsed time:%.3f ms - %s\n" @@ -2283,7 +2320,7 @@ static int sqlcipher_profile_callback(unsigned int trace, void *file, void *stmt #endif #endif } else { - fprintf(f, SQLCIPHER_PROFILE_FMT, elapsed, sqlite3_sql((sqlite3_stmt*)stmt)); + sqlcipher_fprintf(f, SQLCIPHER_PROFILE_FMT, elapsed, sqlite3_sql((sqlite3_stmt*)stmt)); } return SQLITE_OK; } @@ -2387,8 +2424,9 @@ void sqlcipher_log(unsigned int level, unsigned int source, const char *message, #ifdef CODEC_DEBUG #if defined(SQLCIPHER_OMIT_LOG_DEVICE) || (!defined(__ANDROID__) && !defined(__APPLE__)) - vfprintf(stderr, message, params); - fprintf(stderr, "\n"); + sqlite3_vsnprintf(MAX_LOG_LEN, formatted, message, params); + sqlcipher_fprintf(stderr, formatted); + sqlcipher_fprintf(stderr, "\n"); goto end; #else #if defined(__ANDROID__) @@ -2447,7 +2485,7 @@ void sqlcipher_log(unsigned int level, unsigned int source, const char *message, localtime_r(&sec, &tt); #endif if(strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", &tt)) { - fprintf((FILE*)sqlcipher_log_file, "%s.%03d: %s\n", buffer, ms, formatted); + sqlcipher_fprintf((FILE*)sqlcipher_log_file, "%s.%03d: %s\n", buffer, ms, formatted); goto end; } } From aad96d9cb7cf02edd4e0a4ea15e0ef5fb43edd1b Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Tue, 12 Aug 2025 09:53:34 -0400 Subject: [PATCH 30/75] Relocates defines for SQLCIPHER_EN/DECRYPT to sqlcipher.h for non-amalgamation builds --- src/crypto_cc.c | 2 +- src/crypto_libtomcrypt.c | 2 +- src/crypto_nss.c | 2 +- src/sqlcipher.c | 15 ++++++--------- src/sqlcipher.h | 3 +++ 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/crypto_cc.c b/src/crypto_cc.c index 8c1963a2a..37669cb46 100644 --- a/src/crypto_cc.c +++ b/src/crypto_cc.c @@ -139,7 +139,7 @@ static int sqlcipher_cc_cipher( ) { CCCryptorRef cryptor; size_t tmp_csz, csz; - CCOperation op = mode == CIPHER_ENCRYPT ? kCCEncrypt : kCCDecrypt; + CCOperation op = mode == SQLCIPHER_ENCRYPT ? kCCEncrypt : kCCDecrypt; if(CCCryptorCreate(op, kCCAlgorithmAES128, 0, key, kCCKeySizeAES256, iv, &cryptor) != kCCSuccess) return SQLITE_ERROR; if(CCCryptorUpdate(cryptor, in, in_sz, out, in_sz, &tmp_csz) != kCCSuccess) return SQLITE_ERROR; diff --git a/src/crypto_libtomcrypt.c b/src/crypto_libtomcrypt.c index 9a9bb17b7..b648faa7c 100644 --- a/src/crypto_libtomcrypt.c +++ b/src/crypto_libtomcrypt.c @@ -232,7 +232,7 @@ static int sqlcipher_ltc_cipher( if((cipher_idx = find_cipher(LTC_CIPHER)) == -1) return SQLITE_ERROR; if((rc = cbc_start(cipher_idx, iv, key, key_sz, 0, &cbc)) != CRYPT_OK) return SQLITE_ERROR; - rc = mode == 1 ? cbc_encrypt(in, out, in_sz, &cbc) : cbc_decrypt(in, out, in_sz, &cbc); + rc = mode == SQLCIPHER_ENCRYPT ? cbc_encrypt(in, out, in_sz, &cbc) : cbc_decrypt(in, out, in_sz, &cbc); if(rc != CRYPT_OK) return SQLITE_ERROR; cbc_done(&cbc); return SQLITE_OK; diff --git a/src/crypto_nss.c b/src/crypto_nss.c index c0fd9dbdb..e3555bb66 100644 --- a/src/crypto_nss.c +++ b/src/crypto_nss.c @@ -265,7 +265,7 @@ static int sqlcipher_nss_cipher( CKA_ENCRYPT, &keyItem, NULL); if (symKey == NULL) goto error; SECStatus rv; - if (mode == CIPHER_ENCRYPT) { + if (mode == SQLCIPHER_ENCRYPT) { rv = PK11_Encrypt(symKey, CKM_AES_CBC, ¶ms, out, &outLen, in_sz + 16, in, in_sz); } else { diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 1c558a526..3d0acd8c0 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -106,9 +106,6 @@ void sqlite3pager_reset(Pager *pPager); #define CIPHER_VERSION_BUILD community #endif -#define CIPHER_DECRYPT 0 -#define CIPHER_ENCRYPT 1 - #define CIPHER_READ_CTX 0 #define CIPHER_WRITE_CTX 1 #define CIPHER_READWRITE_CTX 2 @@ -1673,14 +1670,14 @@ static int sqlcipher_page_cipher(codec_ctx *ctx, int for_ctx, Pgno pgno, int mod goto error; } - if(mode == CIPHER_ENCRYPT) { + if(mode == SQLCIPHER_ENCRYPT) { /* start at front of the reserve block, write random data to the end */ if(ctx->provider->random(ctx->provider_ctx, iv_out, ctx->reserve_sz) != SQLITE_OK) goto error; - } else { /* CIPHER_DECRYPT */ + } else { /* SQLCIPHER_DECRYPT */ memcpy(iv_out, iv_in, ctx->iv_sz); /* copy the iv from the input to output buffer */ } - if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_HMAC) && (mode == CIPHER_DECRYPT)) { + if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_HMAC) && (mode == SQLCIPHER_DECRYPT)) { if(sqlcipher_page_hmac(ctx, c_ctx, pgno, in, size + ctx->iv_sz, hmac_out) != SQLITE_OK) { sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: hmac operation on decrypt failed for pgno=%d", __func__, pgno); goto error; @@ -1715,7 +1712,7 @@ static int sqlcipher_page_cipher(codec_ctx *ctx, int for_ctx, Pgno pgno, int mod goto error; }; - if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_HMAC) && (mode == CIPHER_ENCRYPT)) { + if(SQLCIPHER_FLAG_GET(ctx->flags, CIPHER_FLAG_HMAC) && (mode == SQLCIPHER_ENCRYPT)) { if(sqlcipher_page_hmac(ctx, c_ctx, pgno, out_start, size + ctx->iv_sz, hmac_out) != SQLITE_OK) { sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: hmac operation on encrypt failed for pgno=%d", __func__, pgno); goto error; @@ -3268,7 +3265,7 @@ static void* sqlite3Codec(void *iCtx, void *data, Pgno pgno, int mode) { if(pgno == 1) /* copy initial part of file header or SQLite magic to buffer */ memcpy(ctx->buffer, ctx->plaintext_header_sz ? pData : (void *) SQLITE_FILE_HEADER, offset); - rc = sqlcipher_page_cipher(ctx, cctx, pgno, CIPHER_DECRYPT, ctx->page_sz - offset, pData + offset, (unsigned char*)ctx->buffer + offset); + rc = sqlcipher_page_cipher(ctx, cctx, pgno, SQLCIPHER_DECRYPT, ctx->page_sz - offset, pData + offset, (unsigned char*)ctx->buffer + offset); #ifdef SQLCIPHER_TEST if((cipher_test_flags & TEST_FAIL_DECRYPT) > 0 && sqlcipher_get_test_fail()) { rc = SQLITE_ERROR; @@ -3309,7 +3306,7 @@ static void* sqlite3Codec(void *iCtx, void *data, Pgno pgno, int mode) { } memcpy(ctx->buffer, ctx->plaintext_header_sz ? pData : kdf_salt, offset); } - rc = sqlcipher_page_cipher(ctx, cctx, pgno, CIPHER_ENCRYPT, ctx->page_sz - offset, pData + offset, (unsigned char*)ctx->buffer + offset); + rc = sqlcipher_page_cipher(ctx, cctx, pgno, SQLCIPHER_ENCRYPT, ctx->page_sz - offset, pData + offset, (unsigned char*)ctx->buffer + offset); #ifdef SQLCIPHER_TEST if((cipher_test_flags & TEST_FAIL_ENCRYPT) > 0 && sqlcipher_get_test_fail()) { rc = SQLITE_ERROR; diff --git a/src/sqlcipher.h b/src/sqlcipher.h index 40416cb53..6fe600f28 100644 --- a/src/sqlcipher.h +++ b/src/sqlcipher.h @@ -37,6 +37,9 @@ #include "sqlite3.h" +#define SQLCIPHER_DECRYPT 0 +#define SQLCIPHER_ENCRYPT 1 + #define SQLCIPHER_HMAC_SHA1 0 #define SQLCIPHER_HMAC_SHA1_LABEL "HMAC_SHA1" #define SQLCIPHER_HMAC_SHA256 1 From c89bab0b416163efd998d7c65846bbe15514499f Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Tue, 12 Aug 2025 10:04:46 -0400 Subject: [PATCH 31/75] Fixes call to provider free_ctx to pass ** --- src/sqlcipher.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 3d0acd8c0..40f755799 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -550,7 +550,7 @@ int sqlcipher_extra_init(const char* arg) { } } - default_provider->ctx_free(provider_ctx); + default_provider->ctx_free(&provider_ctx); sqlcipher_init = 1; sqlcipher_shutdown = 0; From e3ee52a711a0c998056c7e7fdec5296e91e13c91 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Thu, 14 Aug 2025 14:47:30 -0400 Subject: [PATCH 32/75] Relocates includes to support non-amalgamated builds --- src/sqlcipher.c | 8 +++----- src/sqlcipher.h | 1 + 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 40f755799..292ce7907 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -31,11 +31,6 @@ /* BEGIN SQLCIPHER */ #ifdef SQLITE_HAS_CODEC -#include "sqliteInt.h" -#include "btreeInt.h" -#include "pager.h" -#include "vdbeInt.h" - #if !defined(SQLCIPHER_OMIT_LOG_DEVICE) #if defined(__ANDROID__) #include @@ -64,6 +59,9 @@ #include #include "sqlcipher.h" +#include "btreeInt.h" +#include "pager.h" +#include "vdbeInt.h" #if !defined(SQLITE_EXTRA_INIT) || !defined(SQLITE_EXTRA_SHUTDOWN) #error "SQLCipher must be compiled with -DSQLITE_EXTRA_INIT=sqlcipher_extra_init -DSQLITE_EXTRA_SHUTDOWN=sqlcipher_extra_shutdown" diff --git a/src/sqlcipher.h b/src/sqlcipher.h index 6fe600f28..de25d0b1f 100644 --- a/src/sqlcipher.h +++ b/src/sqlcipher.h @@ -36,6 +36,7 @@ #define SQLCIPHER_H #include "sqlite3.h" +#include "sqliteInt.h" #define SQLCIPHER_DECRYPT 0 #define SQLCIPHER_ENCRYPT 1 From cd8909864d9421f3dfb33afebeb873c035112694 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Wed, 20 Aug 2025 12:34:36 -0400 Subject: [PATCH 33/75] Updates version to 4.11.0 --- CHANGELOG.md | 2 ++ src/sqlcipher.c | 2 +- test/sqlcipher-pragmas.test | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f10939324..b33b61903 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ Notable changes to this project are documented in this file. ## [4.11.0] - (? 2025 - [4.11.0 changes]) +- Converts log output to UTF-16 when writing to stdout or stderr on Windows +- Fixes scope issues to allow --disable-amalgamation to work properly ## [4.10.0] - (August 2025 - [4.10.0 changes]) - Updates baseline to SQLite 3.50.4 diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 292ce7907..ec1e39eae 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -97,7 +97,7 @@ void sqlite3pager_reset(Pager *pPager); #define CIPHER_STR(s) #s #ifndef CIPHER_VERSION_NUMBER -#define CIPHER_VERSION_NUMBER 4.10.0 +#define CIPHER_VERSION_NUMBER 4.11.0 #endif #ifndef CIPHER_VERSION_BUILD diff --git a/test/sqlcipher-pragmas.test b/test/sqlcipher-pragmas.test index 5a0a00006..ae03bf676 100644 --- a/test/sqlcipher-pragmas.test +++ b/test/sqlcipher-pragmas.test @@ -46,7 +46,7 @@ do_test verify-pragma-cipher-version { execsql { PRAGMA cipher_version; } -} {{4.10.0 community}} +} {{4.11.0 community}} db close file delete -force test.db From ea37f3d271146eec5a7a2e5439922a6b46244c88 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Wed, 20 Aug 2025 12:35:28 -0400 Subject: [PATCH 34/75] Removes CocoaPod support (SQLCipher.podspec.json) --- CHANGELOG.md | 1 + SQLCipher.podspec.json | 100 ----------------------------------------- 2 files changed, 1 insertion(+), 100 deletions(-) delete mode 100644 SQLCipher.podspec.json diff --git a/CHANGELOG.md b/CHANGELOG.md index b33b61903..e1b86fea0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Notable changes to this project are documented in this file. ## [4.11.0] - (? 2025 - [4.11.0 changes]) - Converts log output to UTF-16 when writing to stdout or stderr on Windows - Fixes scope issues to allow --disable-amalgamation to work properly +- Removes CocoaPods support (SQLCipher.podspec.json) ## [4.10.0] - (August 2025 - [4.10.0 changes]) - Updates baseline to SQLite 3.50.4 diff --git a/SQLCipher.podspec.json b/SQLCipher.podspec.json deleted file mode 100644 index 343ca965b..000000000 --- a/SQLCipher.podspec.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "authors": "Zetetic LLC", - "default_subspecs": "standard", - "description": "SQLCipher is an open source extension to SQLite that provides transparent 256-bit AES encryption of database files.", - "homepage": "https://www.zetetic.net/sqlcipher/", - "license": { - "type": "BSD-3-Clause", - "file": "LICENSE.txt" - }, - "name": "SQLCipher", - "platforms": { - "ios": "12.0", - "osx": "10.13", - "tvos": "12.0", - "watchos": "7.0" - }, - "prepare_command": "./configure && make sqlite3.c", - "requires_arc": false, - "source": { - "git": "https://github.com/sqlcipher/sqlcipher.git", - "tag": "v4.10.0" - }, - "summary": "Full Database Encryption for SQLite.", - "version": "4.10.0", - "subspecs": [ - { - "compiler_flags": [ - "-DNDEBUG", - "-DSQLITE_HAS_CODEC", - "-DSQLITE_TEMP_STORE=2", - "-DSQLITE_SOUNDEX", - "-DSQLITE_THREADSAFE", - "-DSQLITE_ENABLE_RTREE", - "-DSQLITE_ENABLE_STAT3", - "-DSQLITE_ENABLE_STAT4", - "-DSQLITE_ENABLE_COLUMN_METADATA", - "-DSQLITE_ENABLE_MEMORY_MANAGEMENT", - "-DSQLITE_ENABLE_LOAD_EXTENSION", - "-DSQLITE_ENABLE_FTS4", - "-DSQLITE_ENABLE_FTS4_UNICODE61", - "-DSQLITE_ENABLE_FTS3_PARENTHESIS", - "-DSQLITE_ENABLE_UNLOCK_NOTIFY", - "-DSQLITE_ENABLE_JSON1", - "-DSQLITE_ENABLE_FTS5", - "-DSQLCIPHER_CRYPTO_CC", - "-DHAVE_USLEEP=1", - "-DSQLITE_MAX_VARIABLE_NUMBER=99999", - "-DSQLITE_EXTRA_INIT=sqlcipher_extra_init", - "-DSQLITE_EXTRA_SHUTDOWN=sqlcipher_extra_shutdown" - ], - "frameworks": [ - "Foundation", - "Security" - ], - "name": "common", - "source_files": "sqlite3.{h,c}", - "resource_bundles": {"SQLCipher": ["sqlcipher-resources/PrivacyInfo.xcprivacy"]}, - "xcconfig": { - "HEADER_SEARCH_PATHS": "$(PODS_ROOT)/SQLCipher", - "GCC_PREPROCESSOR_DEFINITIONS": "SQLITE_HAS_CODEC=1", - "OTHER_CFLAGS": "$(inherited) -DSQLITE_HAS_CODEC -DSQLITE_TEMP_STORE=2 -DSQLITE_SOUNDEX -DSQLITE_THREADSAFE -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_STAT3 -DSQLITE_ENABLE_STAT4 -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_LOAD_EXTENSION -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS4_UNICODE61 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_UNLOCK_NOTIFY -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5 -DSQLCIPHER_CRYPTO_CC -DHAVE_USLEEP=1 -DSQLITE_MAX_VARIABLE_NUMBER=99999 -DSQLITE_EXTRA_INIT=sqlcipher_extra_init -DSQLITE_EXTRA_SHUTDOWN=sqlcipher_extra_shutdown" - }, - "user_target_xcconfig": { - "GCC_PREPROCESSOR_DEFINITIONS": "_SQLITE3_H_=1 _FTS5_H=1 _SQLITE3RTREE_H_=1" - } - }, - { - "dependencies": { - "SQLCipher/common": [ - - ] - }, - "name": "standard" - }, - { - "compiler_flags": "", - "dependencies": { - "SQLCipher/common": [ - - ] - }, - "name": "fts", - "xcconfig": { - "OTHER_CFLAGS": "$(inherited)" - } - }, - { - "compiler_flags": "", - "dependencies": { - "SQLCipher/common": [ - - ] - }, - "name": "unlock_notify", - "xcconfig": { - "OTHER_CFLAGS": "$(inherited)" - } - } - ] -} From 7dbc295e6852d123b8c36f6cade2fbd5342349ca Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Thu, 21 Aug 2025 21:51:36 -0400 Subject: [PATCH 35/75] Replaces fortuna seeding mechanism for libtomcrypt with rng_get_bytes() --- CHANGELOG.md | 1 + src/crypto_libtomcrypt.c | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1b86fea0..ca9b45a15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Notable changes to this project are documented in this file. ## [4.11.0] - (? 2025 - [4.11.0 changes]) - Converts log output to UTF-16 when writing to stdout or stderr on Windows - Fixes scope issues to allow --disable-amalgamation to work properly +- Replaces fortuna seeding mechanism for libtomcrypt with rng_get_bytes() - Removes CocoaPods support (SQLCipher.podspec.json) ## [4.10.0] - (August 2025 - [4.10.0 changes]) diff --git a/src/crypto_libtomcrypt.c b/src/crypto_libtomcrypt.c index b648faa7c..c8ee378d3 100644 --- a/src/crypto_libtomcrypt.c +++ b/src/crypto_libtomcrypt.c @@ -73,6 +73,7 @@ static int sqlcipher_ltc_add_random(void *ctx, const void *buffer, int length) { static int sqlcipher_ltc_activate(void *ctx) { unsigned char random_buffer[FORTUNA_MAX_SZ]; + int bytes = 0; sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_activate: entering SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); @@ -94,8 +95,9 @@ static int sqlcipher_ltc_activate(void *ctx) { ltc_ref_count++; #ifndef SQLCIPHER_TEST - sqlite3_randomness(FORTUNA_MAX_SZ, random_buffer); + bytes = rng_get_bytes(random_buffer, FORTUNA_MAX_SZ, NULL); #endif + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_PROVIDER, "sqlcipher_ltc_activate: seeded fortuna with %d bytes from rng_get_bytes", bytes); if(sqlcipher_ltc_add_random(ctx, random_buffer, FORTUNA_MAX_SZ) != SQLITE_OK) { return SQLITE_ERROR; From d16327695d8f9795bc0254d30565d53fe29f5687 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Mon, 25 Aug 2025 14:40:17 -0400 Subject: [PATCH 36/75] Fixes several compiler warnings --- src/sqlcipher.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index ec1e39eae..0424828ce 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -475,11 +475,11 @@ int sqlcipher_extra_init(const char* arg) { while(private_heap_sz >= SQLCIPHER_PRIVATE_HEAP_SIZE_STEP) { /* attempt to allocate the private heap. If allocation fails, reduce the size and try again */ if((private_heap = sqlcipher_internal_malloc(private_heap_sz))) { - xoshiro_randomness(private_heap, private_heap_sz); + xoshiro_randomness(private_heap, (int) private_heap_sz); /* initialize the head block of the linked list at the start of the heap */ private_block *head = (private_block *) private_heap; head->is_used = 0; - head->size = private_heap_sz - sizeof(private_block); + head->size = (u32) private_heap_sz - sizeof(private_block); head->next = NULL; break; } @@ -542,7 +542,7 @@ int sqlcipher_extra_init(const char* arg) { sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_MEMORY, "%s: failed to allocate shield mask", __func__); goto error; } - if((rc = default_provider->random(provider_ctx, sqlcipher_shield_mask, sqlcipher_shield_mask_sz)) != SQLITE_OK) { + if((rc = default_provider->random(provider_ctx, sqlcipher_shield_mask, (int) sqlcipher_shield_mask_sz)) != SQLITE_OK) { sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_MEMORY, "%s: failed to generate requisite random mask data %d", __func__, rc); goto error; } @@ -2282,7 +2282,7 @@ static int sqlcipher_fprintf(FILE* stream, const char* format, ...) { if (wbuffer == NULL) return NULL; sz = MultiByteToWideChar(CP_UTF8, 0, buffer, sz, wbuffer, sz); - wbuffer[sz] = NULL; + wbuffer[sz] = (wchar_t) 0; fputws(wbuffer, stream); sqlite3_free(wbuffer); From 04ebb65f91cfc55686b6063415ec1fbc328694e9 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Mon, 25 Aug 2025 14:40:31 -0400 Subject: [PATCH 37/75] Corrects return value from sqlcipher_fprintf --- src/sqlcipher.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 0424828ce..fb70160f7 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -2279,7 +2279,7 @@ static int sqlcipher_fprintf(FILE* stream, const char* format, ...) { sz = (int)strlen(buffer); wbuffer = sqlite3_malloc((sz + 1) * sizeof(wchar_t)); - if (wbuffer == NULL) return NULL; + if (wbuffer == NULL) return -1; sz = MultiByteToWideChar(CP_UTF8, 0, buffer, sz, wbuffer, sz); wbuffer[sz] = (wchar_t) 0; From 1e6efba8b5c0ff42244b1acd6f2b2520a5fca666 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Tue, 23 Sep 2025 10:22:49 -0400 Subject: [PATCH 38/75] Fixes check for __has_feature to resolve GHI #572 --- src/sqlcipher.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index fb70160f7..66e00dce1 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -397,11 +397,11 @@ static void sqlcipher_fini(void) { } #endif #elif defined(__APPLE__) - #if !defined(__has_feature) || !__has_feature(address_sanitizer) - static void (*const sqlcipher_fini_func)(void) __attribute__((used, section("__DATA,__mod_term_func"))) = sqlcipher_fini; - #else + #if defined(__has_feature) && __has_feature(address_sanitizer) static void sqlcipher_cleanup_destructor(void) __attribute__((destructor)); static void sqlcipher_cleanup_destructor(void) { sqlcipher_fini(); } + #else + static void (*const sqlcipher_fini_func)(void) __attribute__((used, section("__DATA,__mod_term_func"))) = sqlcipher_fini; #endif #else static void (*const sqlcipher_fini_func)(void) __attribute__((used, section(".fini_array"))) = sqlcipher_fini; From c6e8461bfa0b7317dfd96278dd950b1eb6b21a1a Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Tue, 30 Sep 2025 16:39:04 -0400 Subject: [PATCH 39/75] Updates CHANGELOG.md with relevant changes for 4.11.0 --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca9b45a15..c067b7b12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,16 @@ # SQLCipher Change Log Notable changes to this project are documented in this file. -## [4.11.0] - (? 2025 - [4.11.0 changes]) +## [4.11.0] - (October 2025 - [4.11.0 changes]) - Converts log output to UTF-16 when writing to stdout or stderr on Windows - Fixes scope issues to allow --disable-amalgamation to work properly - Replaces fortuna seeding mechanism for libtomcrypt with rng_get_bytes() - Removes CocoaPods support (SQLCipher.podspec.json) +- Fixes includes and macros to support non-amalgamated builds +- Fixes check for __has_feature to resolve issue with compilers that don't support it +- Corrects return value from sqlcipher_fprintf +- Fixes use of provider free_ctx +- Fixes some compiler warnings ## [4.10.0] - (August 2025 - [4.10.0 changes]) - Updates baseline to SQLite 3.50.4 From 4f7fda428a2781e7c4c89538e61bb8b03885e8d6 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 10 Oct 2025 13:07:43 -0400 Subject: [PATCH 40/75] Updates version number to 4.12.0 --- CHANGELOG.md | 4 ++++ src/sqlcipher.c | 2 +- test/sqlcipher-pragmas.test | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c067b7b12..8b5ff1965 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # SQLCipher Change Log Notable changes to this project are documented in this file. +## [4.12.0] - (? 2025 - [4.12.0 changes]) + ## [4.11.0] - (October 2025 - [4.11.0 changes]) - Converts log output to UTF-16 when writing to stdout or stderr on Windows - Fixes scope issues to allow --disable-amalgamation to work properly @@ -306,6 +308,8 @@ __BREAKING CHANGE__: `SELECT` statements (now also including schema independent ### Security - Change KDF iteration length from 4,000 to 64,000 +[4.12.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.12.0 +[4.12.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.11.0...v4.12.0 [4.11.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.11.0 [4.11.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.10.0...v4.11.0 [4.10.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.10.0 diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 66e00dce1..e6a94ee51 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -97,7 +97,7 @@ void sqlite3pager_reset(Pager *pPager); #define CIPHER_STR(s) #s #ifndef CIPHER_VERSION_NUMBER -#define CIPHER_VERSION_NUMBER 4.11.0 +#define CIPHER_VERSION_NUMBER 4.12.0 #endif #ifndef CIPHER_VERSION_BUILD diff --git a/test/sqlcipher-pragmas.test b/test/sqlcipher-pragmas.test index ae03bf676..88cf71d5a 100644 --- a/test/sqlcipher-pragmas.test +++ b/test/sqlcipher-pragmas.test @@ -46,7 +46,7 @@ do_test verify-pragma-cipher-version { execsql { PRAGMA cipher_version; } -} {{4.11.0 community}} +} {{4.12.0 community}} db close file delete -force test.db From 3607cf6833338eb08bbeae50132600c9de65a48f Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Mon, 25 Aug 2025 11:34:01 -0400 Subject: [PATCH 41/75] Removes providers for LibTomCrypt and NSS --- Makefile.msc | 2 - main.mk | 8 - src/crypto_libtomcrypt.c | 316 -------------------------------------- src/crypto_nss.c | 323 --------------------------------------- src/sqlcipher.c | 8 - tool/mksqlite3c.tcl | 2 - 6 files changed, 659 deletions(-) delete mode 100644 src/crypto_libtomcrypt.c delete mode 100644 src/crypto_nss.c diff --git a/Makefile.msc b/Makefile.msc index e7412dedc..27a72b2f4 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1397,8 +1397,6 @@ LIBRESOBJS = SRC00 = \ $(TOP)\src\sqlcipher.c \ $(TOP)\src\crypto_cc.c \ - $(TOP)\src\crypto_libtomcrypt.c \ - $(TOP)\src\crypto_nss.c \ $(TOP)\src\crypto_openssl.c \ $(TOP)\src\sqlcipher.h \ $(TOP)\src\alter.c \ diff --git a/main.mk b/main.mk index 450b235b2..e87e1f4ad 100644 --- a/main.mk +++ b/main.mk @@ -533,15 +533,11 @@ clean: clean-sanity-check SQLCIPHER_OBJ = \ sqlcipher.o \ crypto_openssl.o \ - crypto_libtomcrypt.o \ - crypto_nss.o \ crypto_cc.o SQLCIPHER_SRC = \ $(TOP)/src/sqlcipher.h \ $(TOP)/src/sqlcipher.c \ - $(TOP)/src/crypto_libtomcrypt.c \ - $(TOP)/src/crypto_nss.c \ $(TOP)/src/crypto_openssl.c \ $(TOP)/src/crypto_cc.c @@ -549,10 +545,6 @@ sqlcipher.o: $(TOP)/src/sqlcipher.c $(DEPS_OBJ_COMMON) $(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/sqlcipher.c crypto_openssl.o: $(TOP)/src/crypto_openssl.c $(DEPS_OBJ_COMMON) $(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/crypto_openssl.c -crypto_nss.o: $(TOP)/src/crypto_nss.c $(DEPS_OBJ_COMMON) - $(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/crypto_nss.c -crypto_libtomcrypt.o: $(TOP)/src/crypto_libtomcrypt.c $(DEPS_OBJ_COMMON) - $(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/crypto_libtomcrypt.c crypto_cc.o: $(TOP)/src/crypto_cc.c $(DEPS_OBJ_COMMON) $(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/crypto_cc.c diff --git a/src/crypto_libtomcrypt.c b/src/crypto_libtomcrypt.c deleted file mode 100644 index c8ee378d3..000000000 --- a/src/crypto_libtomcrypt.c +++ /dev/null @@ -1,316 +0,0 @@ -/* -** SQLCipher -** http://sqlcipher.net -** -** Copyright (c) 2008 - 2013, ZETETIC LLC -** All rights reserved. -** -** Redistribution and use in source and binary forms, with or without -** modification, are permitted provided that the following conditions are met: -** * Redistributions of source code must retain the above copyright -** notice, this list of conditions and the following disclaimer. -** * Redistributions in binary form must reproduce the above copyright -** notice, this list of conditions and the following disclaimer in the -** documentation and/or other materials provided with the distribution. -** * Neither the name of the ZETETIC LLC nor the -** names of its contributors may be used to endorse or promote products -** derived from this software without specific prior written permission. -** -** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY -** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY -** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -** -*/ -/* BEGIN SQLCIPHER */ -#ifdef SQLITE_HAS_CODEC -#ifdef SQLCIPHER_CRYPTO_LIBTOMCRYPT -#include "sqliteInt.h" -#include "sqlcipher.h" -#include - -#define FORTUNA_MAX_SZ 32 -static prng_state prng; -static volatile unsigned int ltc_init = 0; -static volatile unsigned int ltc_ref_count = 0; - -#define LTC_CIPHER "rijndael" - -static int sqlcipher_ltc_add_random(void *ctx, const void *buffer, int length) { - int rc = 0; - int data_to_read = length; - int block_sz = data_to_read < FORTUNA_MAX_SZ ? data_to_read : FORTUNA_MAX_SZ; - const unsigned char * data = (const unsigned char *)buffer; - - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_add_random: entering SQLCIPHER_MUTEX_PROVIDER_RAND"); - sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_RAND)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_add_random: entered SQLCIPHER_MUTEX_PROVIDER_RAND"); - - while(data_to_read > 0){ - rc = fortuna_add_entropy(data, block_sz, &prng); - rc = rc != CRYPT_OK ? SQLITE_ERROR : SQLITE_OK; - if(rc != SQLITE_OK){ - break; - } - data_to_read -= block_sz; - data += block_sz; - block_sz = data_to_read < FORTUNA_MAX_SZ ? data_to_read : FORTUNA_MAX_SZ; - } - fortuna_ready(&prng); - - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_add_random: leaving SQLCIPHER_MUTEX_PROVIDER_RAND"); - sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_RAND)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_add_random: left SQLCIPHER_MUTEX_PROVIDER_RAND"); - - return rc; -} - -static int sqlcipher_ltc_activate(void *ctx) { - unsigned char random_buffer[FORTUNA_MAX_SZ]; - int bytes = 0; - - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_activate: entering SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_activate: entered SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - - sqlcipher_memset(random_buffer, 0, FORTUNA_MAX_SZ); - if(ltc_init == 0) { - if(register_prng(&fortuna_desc) < 0) return SQLITE_ERROR; - if(register_cipher(&rijndael_desc) < 0) return SQLITE_ERROR; - if(register_hash(&sha512_desc) < 0) return SQLITE_ERROR; - if(register_hash(&sha256_desc) < 0) return SQLITE_ERROR; - if(register_hash(&sha1_desc) < 0) return SQLITE_ERROR; - if(fortuna_start(&prng) != CRYPT_OK) { - return SQLITE_ERROR; - } - - ltc_init = 1; - } - ltc_ref_count++; - -#ifndef SQLCIPHER_TEST - bytes = rng_get_bytes(random_buffer, FORTUNA_MAX_SZ, NULL); -#endif - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_PROVIDER, "sqlcipher_ltc_activate: seeded fortuna with %d bytes from rng_get_bytes", bytes); - - if(sqlcipher_ltc_add_random(ctx, random_buffer, FORTUNA_MAX_SZ) != SQLITE_OK) { - return SQLITE_ERROR; - } - sqlcipher_memset(random_buffer, 0, FORTUNA_MAX_SZ); - - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_activate: leaving SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_activate: left SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - - return SQLITE_OK; -} - -static int sqlcipher_ltc_deactivate(void *ctx) { - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_deactivate: entering SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_deactivate: entered SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - - ltc_ref_count--; - if(ltc_ref_count == 0){ - fortuna_done(&prng); - sqlcipher_memset((void *)&prng, 0, sizeof(prng)); - } - - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_deactivate: leaving SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_deactivate: left SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - - return SQLITE_OK; -} - -static const char* sqlcipher_ltc_get_provider_name(void *ctx) { - return "libtomcrypt"; -} - -static const char* sqlcipher_ltc_get_provider_version(void *ctx) { - return SCRYPT; -} - -static int sqlcipher_ltc_random(void *ctx, void *buffer, int length) { - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_random: entering SQLCIPHER_MUTEX_PROVIDER_RAND"); - sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_RAND)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_random: entered SQLCIPHER_MUTEX_PROVIDER_RAND"); - - fortuna_read(buffer, length, &prng); - - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_random: leaving SQLCIPHER_MUTEX_PROVIDER_RAND"); - sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_RAND)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_random: left SQLCIPHER_MUTEX_PROVIDER_RAND"); - - return SQLITE_OK; -} - -static int sqlcipher_ltc_hmac( - void *ctx, int algorithm, - const unsigned char *hmac_key, int key_sz, - const unsigned char *in, int in_sz, - const unsigned char *in2, int in2_sz, - unsigned char *out -) { - int rc, hash_idx; - hmac_state hmac; - unsigned long outlen; - switch(algorithm) { - case SQLCIPHER_HMAC_SHA1: - hash_idx = find_hash("sha1"); - break; - case SQLCIPHER_HMAC_SHA256: - hash_idx = find_hash("sha256"); - break; - case SQLCIPHER_HMAC_SHA512: - hash_idx = find_hash("sha512"); - break; - default: - return SQLITE_ERROR; - } - - if(hash_idx < 0) return SQLITE_ERROR; - outlen = hash_descriptor[hash_idx].hashsize; - - if(in == NULL) return SQLITE_ERROR; - if((rc = hmac_init(&hmac, hash_idx, hmac_key, key_sz)) != CRYPT_OK) return SQLITE_ERROR; - if((rc = hmac_process(&hmac, in, in_sz)) != CRYPT_OK) return SQLITE_ERROR; - if(in2 != NULL && (rc = hmac_process(&hmac, in2, in2_sz)) != CRYPT_OK) return SQLITE_ERROR; - if((rc = hmac_done(&hmac, out, &outlen)) != CRYPT_OK) return SQLITE_ERROR; - return SQLITE_OK; -} - -static int sqlcipher_ltc_kdf( - void *ctx, int algorithm, - const unsigned char *pass, int pass_sz, - const unsigned char* salt, int salt_sz, - int workfactor, - int key_sz, unsigned char *key -) { - int rc, hash_idx; - unsigned long outlen = key_sz; - - switch(algorithm) { - case SQLCIPHER_HMAC_SHA1: - hash_idx = find_hash("sha1"); - break; - case SQLCIPHER_HMAC_SHA256: - hash_idx = find_hash("sha256"); - break; - case SQLCIPHER_HMAC_SHA512: - hash_idx = find_hash("sha512"); - break; - default: - return SQLITE_ERROR; - } - if(hash_idx < 0) return SQLITE_ERROR; - - if((rc = pkcs_5_alg2(pass, pass_sz, salt, salt_sz, - workfactor, hash_idx, key, &outlen)) != CRYPT_OK) { - return SQLITE_ERROR; - } - return SQLITE_OK; -} - -static const char* sqlcipher_ltc_get_cipher(void *ctx) { - return "aes-256-cbc"; -} - -static int sqlcipher_ltc_cipher( - void *ctx, int mode, - const unsigned char *key, int key_sz, - const unsigned char *iv, - const unsigned char *in, int in_sz, - unsigned char *out -) { - int rc, cipher_idx; - symmetric_CBC cbc; - - if((cipher_idx = find_cipher(LTC_CIPHER)) == -1) return SQLITE_ERROR; - if((rc = cbc_start(cipher_idx, iv, key, key_sz, 0, &cbc)) != CRYPT_OK) return SQLITE_ERROR; - rc = mode == SQLCIPHER_ENCRYPT ? cbc_encrypt(in, out, in_sz, &cbc) : cbc_decrypt(in, out, in_sz, &cbc); - if(rc != CRYPT_OK) return SQLITE_ERROR; - cbc_done(&cbc); - return SQLITE_OK; -} - -static int sqlcipher_ltc_get_key_sz(void *ctx) { - int cipher_idx = find_cipher(LTC_CIPHER); - return cipher_descriptor[cipher_idx].max_key_length; -} - -static int sqlcipher_ltc_get_iv_sz(void *ctx) { - int cipher_idx = find_cipher(LTC_CIPHER); - return cipher_descriptor[cipher_idx].block_length; -} - -static int sqlcipher_ltc_get_block_sz(void *ctx) { - int cipher_idx = find_cipher(LTC_CIPHER); - return cipher_descriptor[cipher_idx].block_length; -} - -static int sqlcipher_ltc_get_hmac_sz(void *ctx, int algorithm) { - int hash_idx; - switch(algorithm) { - case SQLCIPHER_HMAC_SHA1: - hash_idx = find_hash("sha1"); - break; - case SQLCIPHER_HMAC_SHA256: - hash_idx = find_hash("sha256"); - break; - case SQLCIPHER_HMAC_SHA512: - hash_idx = find_hash("sha512"); - break; - default: - return 0; - } - - if(hash_idx < 0) return 0; - - return hash_descriptor[hash_idx].hashsize; -} - -static int sqlcipher_ltc_ctx_init(void **ctx) { - sqlcipher_ltc_activate(NULL); - return SQLITE_OK; -} - -static int sqlcipher_ltc_ctx_free(void **ctx) { - sqlcipher_ltc_deactivate(&ctx); - return SQLITE_OK; -} - -static int sqlcipher_ltc_fips_status(void *ctx) { - return 0; -} - -int sqlcipher_ltc_setup(sqlcipher_provider *p) { - p->init = NULL; - p->shutdown = NULL; - p->get_provider_name = sqlcipher_ltc_get_provider_name; - p->random = sqlcipher_ltc_random; - p->hmac = sqlcipher_ltc_hmac; - p->kdf = sqlcipher_ltc_kdf; - p->cipher = sqlcipher_ltc_cipher; - p->get_cipher = sqlcipher_ltc_get_cipher; - p->get_key_sz = sqlcipher_ltc_get_key_sz; - p->get_iv_sz = sqlcipher_ltc_get_iv_sz; - p->get_block_sz = sqlcipher_ltc_get_block_sz; - p->get_hmac_sz = sqlcipher_ltc_get_hmac_sz; - p->ctx_init = sqlcipher_ltc_ctx_init; - p->ctx_free = sqlcipher_ltc_ctx_free; - p->add_random = sqlcipher_ltc_add_random; - p->fips_status = sqlcipher_ltc_fips_status; - p->get_provider_version = sqlcipher_ltc_get_provider_version; - return SQLITE_OK; -} - -#endif -#endif -/* END SQLCIPHER */ diff --git a/src/crypto_nss.c b/src/crypto_nss.c deleted file mode 100644 index e3555bb66..000000000 --- a/src/crypto_nss.c +++ /dev/null @@ -1,323 +0,0 @@ -/* -** SQLCipher -** http://sqlcipher.net -** -** Copyright (c) 2008 - 2013, ZETETIC LLC -** All rights reserved. -** -** Redistribution and use in source and binary forms, with or without -** modification, are permitted provided that the following conditions are met: -** * Redistributions of source code must retain the above copyright -** notice, this list of conditions and the following disclaimer. -** * Redistributions in binary form must reproduce the above copyright -** notice, this list of conditions and the following disclaimer in the -** documentation and/or other materials provided with the distribution. -** * Neither the name of the ZETETIC LLC nor the -** names of its contributors may be used to endorse or promote products -** derived from this software without specific prior written permission. -** -** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY -** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY -** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -** -*/ -/* BEGIN SQLCIPHER */ -#ifdef SQLITE_HAS_CODEC -#ifdef SQLCIPHER_CRYPTO_NSS -#include "sqlcipher.h" -#include -#include -#include - -static NSSInitContext* nss_init_context = NULL; -static unsigned int nss_init_count = 0; - -int sqlcipher_nss_setup(sqlcipher_provider *p); - -static int sqlcipher_nss_activate(void *ctx) { - - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_nss_activate: entering SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_nss_activate: entered SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - if (nss_init_context == NULL) { - nss_init_context = NSS_InitContext("", "", "", "", NULL, - NSS_INIT_READONLY | NSS_INIT_NOCERTDB | NSS_INIT_NOMODDB | - NSS_INIT_FORCEOPEN | NSS_INIT_OPTIMIZESPACE | NSS_INIT_NOROOTINIT); - } - nss_init_count++; - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_nss_activate: leaving SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_nss_activate: left SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - return SQLITE_OK; -} - -static int sqlcipher_nss_deactivate(void *ctx) { - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_nss_activate: entering SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_nss_activate: entered SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - - nss_init_count--; - if (nss_init_count == 0 && nss_init_context != NULL) { - NSS_ShutdownContext(nss_init_context); - nss_init_context = NULL; - } - - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_nss_activate: leaving SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); - sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_nss_activate: left SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); - return SQLITE_OK; -} - -static int sqlcipher_nss_add_random(void *ctx, const void *buffer, int length) { - return SQLITE_OK; -} - -/* generate a defined number of random bytes */ -static int sqlcipher_nss_random (void *ctx, void *buffer, int length) { - // PK11_GenerateRandom should be thread-safe. - return (PK11_GenerateRandom((unsigned char *)buffer, length) == SECSuccess) ? SQLITE_OK : SQLITE_ERROR; -} - -static const char* sqlcipher_nss_get_provider_name(void *ctx) { - return "nss"; -} - -static const char* sqlcipher_nss_get_provider_version(void *ctx) { - return NSS_GetVersion(); -} - -static const char* sqlcipher_nss_get_cipher(void *ctx) { - return "aes-256-cbc"; -} - -static int sqlcipher_nss_get_key_sz(void *ctx) { - return AES_256_KEY_LENGTH; -} - -static int sqlcipher_nss_get_iv_sz(void *ctx) { - return AES_BLOCK_SIZE; -} - -static int sqlcipher_nss_get_block_sz(void *ctx) { - return AES_BLOCK_SIZE; -} - -static int sqlcipher_nss_get_hmac_sz(void *ctx, int algorithm) { - switch(algorithm) { - case SQLCIPHER_HMAC_SHA1: - return SHA1_LENGTH; - break; - case SQLCIPHER_HMAC_SHA256: - return SHA256_LENGTH; - break; - case SQLCIPHER_HMAC_SHA512: - return SHA512_LENGTH; - break; - default: - return 0; - } -} - -static int sqlcipher_nss_hmac( - void *ctx, int algorithm, - const unsigned char *hmac_key, int key_sz, - const unsigned char *in, int in_sz, - const unsigned char *in2, int in2_sz, - unsigned char *out -) { - int rc = SQLITE_OK; - unsigned int length; - unsigned int outLen; - PK11Context* context = NULL; - PK11SlotInfo * slot = NULL; - PK11SymKey* symKey = NULL; - if(in == NULL) goto error; - CK_MECHANISM_TYPE mech; - switch(algorithm) { - case SQLCIPHER_HMAC_SHA1: - mech = CKM_SHA_1_HMAC; - break; - case SQLCIPHER_HMAC_SHA256: - mech = CKM_SHA256_HMAC; - break; - case SQLCIPHER_HMAC_SHA512: - mech = CKM_SHA512_HMAC; - break; - default: - goto error; - } - length = sqlcipher_nss_get_hmac_sz(ctx, algorithm); - slot = PK11_GetInternalSlot(); - if (slot == NULL) goto error; - SECItem keyItem; - keyItem.data = hmac_key; - keyItem.len = key_sz; - symKey = PK11_ImportSymKey(slot, mech, PK11_OriginUnwrap, - CKA_SIGN, &keyItem, NULL); - if (symKey == NULL) goto error; - SECItem noParams; - noParams.data = 0; - noParams.len = 0; - context = PK11_CreateContextBySymKey(mech, CKA_SIGN, symKey, &noParams); - if (context == NULL) goto error; - if (PK11_DigestBegin(context) != SECSuccess) goto error; - if (PK11_DigestOp(context, in, in_sz) != SECSuccess) goto error; - if (in2 != NULL) { - if (PK11_DigestOp(context, in2, in2_sz) != SECSuccess) goto error; - } - if (PK11_DigestFinal(context, out, &outLen, length) != SECSuccess) goto error; - - goto cleanup; - error: - rc = SQLITE_ERROR; - cleanup: - if (context) PK11_DestroyContext(context, PR_TRUE); - if (symKey) PK11_FreeSymKey(symKey); - if (slot) PK11_FreeSlot(slot); - return rc; -} - -static int sqlcipher_nss_kdf( - void *ctx, int algorithm, - const unsigned char *pass, int pass_sz, - const unsigned char* salt, int salt_sz, - int workfactor, - int key_sz, unsigned char *key -) { - int rc = SQLITE_OK; - PK11SlotInfo * slot = NULL; - SECAlgorithmID * algid = NULL; - PK11SymKey* symKey = NULL; - SECOidTag oidtag; - switch(algorithm) { - case SQLCIPHER_HMAC_SHA1: - oidtag = SEC_OID_HMAC_SHA1; - break; - case SQLCIPHER_HMAC_SHA256: - oidtag = SEC_OID_HMAC_SHA256; - break; - case SQLCIPHER_HMAC_SHA512: - oidtag = SEC_OID_HMAC_SHA512; - break; - default: - goto error; - } - SECItem secSalt; - secSalt.data = salt; - secSalt.len = salt_sz; - // Always pass SEC_OID_HMAC_SHA1 (i.e. PBMAC1) as this parameter - // is unused for key generation. It is currently only used - // for PBKDF2 authentication or key (un)wrapping when specifying an - // encryption algorithm (PBES2). - algid = PK11_CreatePBEV2AlgorithmID(SEC_OID_PKCS5_PBKDF2, SEC_OID_HMAC_SHA1, - oidtag, key_sz, workfactor, &secSalt); - if (algid == NULL) goto error; - slot = PK11_GetInternalSlot(); - if (slot == NULL) goto error; - SECItem pwItem; - pwItem.data = (unsigned char *) pass; // PK11_PBEKeyGen doesn't modify the key. - pwItem.len = pass_sz; - symKey = PK11_PBEKeyGen(slot, algid, &pwItem, PR_FALSE, NULL); - if (symKey == NULL) goto error; - if (PK11_ExtractKeyValue(symKey) != SECSuccess) goto error; - // No need to free keyData as it is a buffer managed by symKey. - SECItem* keyData = PK11_GetKeyData(symKey); - if (keyData == NULL) goto error; - memcpy(key, keyData->data, key_sz); - - goto cleanup; - error: - rc = SQLITE_ERROR; - cleanup: - if (slot) PK11_FreeSlot(slot); - if (algid) SECOID_DestroyAlgorithmID(algid, PR_TRUE); - if (symKey) PK11_FreeSymKey(symKey); - return rc; -} - -static int sqlcipher_nss_cipher( - void *ctx, int mode, - const unsigned char *key, int key_sz, - const unsigned char *iv, - const unsigned char *in, - int in_sz, unsigned char *out -) { - int rc = SQLITE_OK; - PK11SlotInfo * slot = NULL; - PK11SymKey* symKey = NULL; - unsigned int outLen; - SECItem params; - params.data = iv; - params.len = sqlcipher_nss_get_iv_sz(ctx); - slot = PK11_GetInternalSlot(); - if (slot == NULL) goto error; - SECItem keyItem; - keyItem.data = key; - keyItem.len = key_sz; - symKey = PK11_ImportSymKey(slot, CKM_AES_CBC, PK11_OriginUnwrap, - CKA_ENCRYPT, &keyItem, NULL); - if (symKey == NULL) goto error; - SECStatus rv; - if (mode == SQLCIPHER_ENCRYPT) { - rv = PK11_Encrypt(symKey, CKM_AES_CBC, ¶ms, out, &outLen, - in_sz + 16, in, in_sz); - } else { - rv = PK11_Decrypt(symKey, CKM_AES_CBC, ¶ms, out, &outLen, - in_sz + 16, in, in_sz); - } - if (rv != SECSuccess) goto error; - - goto cleanup; - error: - rc = SQLITE_ERROR; - cleanup: - if (slot) PK11_FreeSlot(slot); - if (symKey) PK11_FreeSymKey(symKey); - return rc; -} - -static int sqlcipher_nss_ctx_init(void **ctx) { - sqlcipher_nss_activate(NULL); - return SQLITE_OK; -} - -static int sqlcipher_nss_ctx_free(void **ctx) { - sqlcipher_nss_deactivate(NULL); - return SQLITE_OK; -} - -static int sqlcipher_nss_fips_status(void *ctx) { - return 0; -} - -int sqlcipher_nss_setup(sqlcipher_provider *p) { - p->init = NULL; - p->shutdown = NULL; - p->random = sqlcipher_nss_random; - p->get_provider_name = sqlcipher_nss_get_provider_name; - p->hmac = sqlcipher_nss_hmac; - p->kdf = sqlcipher_nss_kdf; - p->cipher = sqlcipher_nss_cipher; - p->get_cipher = sqlcipher_nss_get_cipher; - p->get_key_sz = sqlcipher_nss_get_key_sz; - p->get_iv_sz = sqlcipher_nss_get_iv_sz; - p->get_block_sz = sqlcipher_nss_get_block_sz; - p->get_hmac_sz = sqlcipher_nss_get_hmac_sz; - p->ctx_init = sqlcipher_nss_ctx_init; - p->ctx_free = sqlcipher_nss_ctx_free; - p->add_random = sqlcipher_nss_add_random; - p->fips_status = sqlcipher_nss_fips_status; - p->get_provider_version = sqlcipher_nss_get_provider_version; - return SQLITE_OK; -} - -#endif -#endif -/* END SQLCIPHER */ diff --git a/src/sqlcipher.c b/src/sqlcipher.c index e6a94ee51..0a617c1cd 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -84,8 +84,6 @@ void sqlite3pager_reset(Pager *pPager); /* end extensions defined in pager.c */ #if !defined (SQLCIPHER_CRYPTO_CC) \ - && !defined (SQLCIPHER_CRYPTO_LIBTOMCRYPT) \ - && !defined (SQLCIPHER_CRYPTO_NSS) \ && !defined (SQLCIPHER_CRYPTO_OPENSSL) \ && !defined (SQLCIPHER_CRYPTO_CUSTOM) #define SQLCIPHER_CRYPTO_OPENSSL @@ -502,12 +500,6 @@ int sqlcipher_extra_init(const char* arg) { #if defined (SQLCIPHER_CRYPTO_CC) extern int sqlcipher_cc_setup(sqlcipher_provider *p); sqlcipher_cc_setup(p); -#elif defined (SQLCIPHER_CRYPTO_LIBTOMCRYPT) - extern int sqlcipher_ltc_setup(sqlcipher_provider *p); - sqlcipher_ltc_setup(p); -#elif defined (SQLCIPHER_CRYPTO_NSS) - extern int sqlcipher_nss_setup(sqlcipher_provider *p); - sqlcipher_nss_setup(p); #elif defined (SQLCIPHER_CRYPTO_OPENSSL) extern int sqlcipher_openssl_setup(sqlcipher_provider *p); sqlcipher_openssl_setup(p); diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index b8aa28796..b8c91209b 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -436,8 +436,6 @@ set flist { memjournal.c sqlcipher.c - crypto_libtomcrypt.c - crypto_nss.c crypto_openssl.c crypto_cc.c From 5b5ce4888e64606e07eec1a76d13ffca044a63b6 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Mon, 13 Oct 2025 14:10:36 -0400 Subject: [PATCH 42/75] Checks __has_feature on separate line prior to use to resolve GHI #572 --- src/sqlcipher.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 0a617c1cd..c1a3c8879 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -395,11 +395,15 @@ static void sqlcipher_fini(void) { } #endif #elif defined(__APPLE__) - #if defined(__has_feature) && __has_feature(address_sanitizer) - static void sqlcipher_cleanup_destructor(void) __attribute__((destructor)); - static void sqlcipher_cleanup_destructor(void) { sqlcipher_fini(); } + #if defined(__has_feature) + #if __has_feature(address_sanitizer) + static void sqlcipher_cleanup_destructor(void) __attribute__((destructor)); + static void sqlcipher_cleanup_destructor(void) { sqlcipher_fini(); } + #else + static void (*const sqlcipher_fini_func)(void) __attribute__((used, section("__DATA,__mod_term_func"))) = sqlcipher_fini; + #endif #else - static void (*const sqlcipher_fini_func)(void) __attribute__((used, section("__DATA,__mod_term_func"))) = sqlcipher_fini; + static void (*const sqlcipher_fini_func)(void) __attribute__((used, section("__DATA,__mod_term_func"))) = sqlcipher_fini; #endif #else static void (*const sqlcipher_fini_func)(void) __attribute__((used, section(".fini_array"))) = sqlcipher_fini; From 7f1d4d5afd497db4544bf74ef02a1e241cdfbbb8 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 17 Oct 2025 13:34:46 -0400 Subject: [PATCH 43/75] Standardizes criteria for cipher_migrate tests --- test/sqlcipher-compatibility.test | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/test/sqlcipher-compatibility.test b/test/sqlcipher-compatibility.test index 69630fbf7..89a0080cc 100644 --- a/test/sqlcipher-compatibility.test +++ b/test/sqlcipher-compatibility.test @@ -1052,36 +1052,44 @@ file delete -force test.db do_test migrate-1.1.8-database-to-current-format { file copy -force $sampleDir/sqlcipher-1.1.8-testkey.db test.db sqlite_orig db test.db - execsql { + set rc {} + + lappend rc [execsql { PRAGMA key = 'testkey'; PRAGMA cipher_migrate; - } + SELECT count(*) FROM sqlite_schema; + }] db close sqlite_orig db test.db - execsql { + lappend rc [execsql { PRAGMA key = 'testkey'; SELECT count(*) FROM sqlite_schema; - } -} {ok 1} + PRAGMA journal_mode; + }] +} {{ok 0 1} {ok 1 delete}} db close file delete -force test.db do_test migrate-2-0-le-database-to-current-format { file copy -force $sampleDir/sqlcipher-2.0-le-testkey.db test.db sqlite_orig db test.db - execsql { + set rc {} + + lappend rc [execsql { PRAGMA key = 'testkey'; PRAGMA cipher_migrate; - } + SELECT count(*) FROM sqlite_schema; + }] db close sqlite_orig db test.db - execsql { + lappend rc [execsql { PRAGMA key = 'testkey'; SELECT count(*) FROM sqlite_schema; - } -} {ok 1} + PRAGMA journal_mode; + }] +} {{ok 0 1} {ok 1 delete}} db close file delete -force test.db From 746c43764206b1b20c7e0434f9650125761b02d7 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Thu, 6 Nov 2025 12:31:24 -0500 Subject: [PATCH 44/75] Snapshot of upstream SQLite 3.51.0 --- LICENSE.md | 26 +- Makefile.in | 27 +- Makefile.msc | 66 +- README.md | 42 +- VERSION | 2 +- auto.def | 5 + autoconf/Makefile.in | 22 +- autoconf/Makefile.msc | 3 + autoconf/tea/Makefile.in | 79 +- autoconf/tea/README.txt | 18 +- autoconf/tea/_teaish.tester.tcl.in | 3 +- autoconf/tea/configure | 17 +- autoconf/tea/doc/sqlite3.n | 15 - autoconf/tea/teaish.tcl | 10 +- autosetup/README.md | 7 +- autosetup/autosetup | 18 +- autosetup/cc-shared.tcl | 10 +- autosetup/jimsh0.c | 37 +- autosetup/proj.tcl | 575 +- autosetup/sqlite-config.tcl | 207 +- autosetup/teaish/core.tcl | 135 +- autosetup/teaish/tester.tcl | 71 +- doc/compile-for-windows.md | 25 +- doc/jsonb.md | 17 +- ext/expert/sqlite3expert.c | 52 +- ext/fts3/fts3Int.h | 22 +- ext/fts3/fts3_snippet.c | 4 - ext/fts3/fts3_write.c | 8 +- ext/fts5/fts5Int.h | 25 +- ext/fts5/fts5_expr.c | 11 +- ext/fts5/fts5_index.c | 190 +- ext/fts5/fts5_main.c | 57 +- ext/fts5/test/fts5aa.test | 2 +- ext/fts5/test/fts5ab.test | 33 + ext/fts5/test/fts5corrupt.test | 7 +- ext/fts5/test/fts5corrupt2.test | 12 +- ext/fts5/test/fts5corrupt3.test | 297 +- ext/fts5/test/fts5corrupt5.test | 8 +- ext/fts5/test/fts5corrupt7.test | 2 +- ext/fts5/test/fts5corrupt8.test | 2 +- ext/fts5/test/fts5corruptbig.test | 53 + ext/fts5/test/fts5integrity.test | 6 + ext/fts5/test/fts5join.test | 69 + ext/fts5/test/fts5leftjoin.test | 49 + ext/fts5/test/fts5misc.test | 12 + ext/fts5/test/fts5onepass.test | 8 +- ext/fts5/test/fts5rebuild.test | 2 +- ext/intck/intck1.test | 2 + ext/intck/intck2.test | 1 + ext/jni/src/c/sqlite3-jni.c | 25 +- ext/jni/src/c/sqlite3-jni.h | 10 +- .../sqlite/jni/capi/AggregateFunction.java | 4 +- ext/jni/src/org/sqlite/jni/capi/CApi.java | 7 +- ext/jni/src/org/sqlite/jni/capi/Tester1.java | 14 + ext/lsm1/Makefile | 56 - ext/lsm1/Makefile.msc | 102 - ext/lsm1/lsm-test/README | 40 - ext/lsm1/lsm-test/lsmtest.h | 303 - ext/lsm1/lsm-test/lsmtest1.c | 656 -- ext/lsm1/lsm-test/lsmtest2.c | 488 -- ext/lsm1/lsm-test/lsmtest3.c | 238 - ext/lsm1/lsm-test/lsmtest4.c | 127 - ext/lsm1/lsm-test/lsmtest5.c | 633 -- ext/lsm1/lsm-test/lsmtest6.c | 661 -- ext/lsm1/lsm-test/lsmtest7.c | 206 - ext/lsm1/lsm-test/lsmtest8.c | 324 - ext/lsm1/lsm-test/lsmtest9.c | 140 - ext/lsm1/lsm-test/lsmtest_bt.c | 71 - ext/lsm1/lsm-test/lsmtest_datasource.c | 96 - ext/lsm1/lsm-test/lsmtest_func.c | 177 - ext/lsm1/lsm-test/lsmtest_io.c | 248 - ext/lsm1/lsm-test/lsmtest_main.c | 1548 ---- ext/lsm1/lsm-test/lsmtest_mem.c | 409 -- ext/lsm1/lsm-test/lsmtest_tdb.c | 846 --- ext/lsm1/lsm-test/lsmtest_tdb.h | 174 - ext/lsm1/lsm-test/lsmtest_tdb2.cc | 369 - ext/lsm1/lsm-test/lsmtest_tdb3.c | 1429 ---- ext/lsm1/lsm-test/lsmtest_tdb4.c | 980 --- ext/lsm1/lsm-test/lsmtest_util.c | 223 - ext/lsm1/lsm-test/lsmtest_win32.c | 30 - ext/lsm1/lsm.h | 684 -- ext/lsm1/lsmInt.h | 997 --- ext/lsm1/lsm_ckpt.c | 1239 ---- ext/lsm1/lsm_file.c | 3311 --------- ext/lsm1/lsm_log.c | 1156 --- ext/lsm1/lsm_main.c | 1008 --- ext/lsm1/lsm_mem.c | 104 - ext/lsm1/lsm_mutex.c | 88 - ext/lsm1/lsm_shared.c | 1994 ------ ext/lsm1/lsm_sorted.c | 6195 ----------------- ext/lsm1/lsm_str.c | 148 - ext/lsm1/lsm_tree.c | 2465 ------- ext/lsm1/lsm_unix.c | 753 -- ext/lsm1/lsm_varint.c | 201 - ext/lsm1/lsm_vtab.c | 1084 --- ext/lsm1/lsm_win32.c | 1063 --- ext/lsm1/test/lsm1_common.tcl | 38 - ext/lsm1/test/lsm1_simple.test | 152 - ext/lsm1/tool/mklsm1c.tcl | 88 - ext/misc/README.md | 10 - ext/misc/amatch.c | 10 +- ext/misc/base64.c | 6 +- ext/misc/base85.c | 2 +- ext/misc/carray.h | 50 - ext/misc/cksumvfs.c | 43 +- ext/misc/completion.c | 3 + ext/misc/dbdump.c | 12 +- ext/misc/decimal.c | 31 +- ext/misc/fileio.c | 134 +- ext/misc/fossildelta.c | 32 +- ext/misc/ieee754.c | 3 + ext/misc/memvfs.c | 575 -- ext/misc/normalize.c | 7 +- ext/misc/regexp.c | 118 +- ext/misc/series.c | 527 +- ext/misc/vtablog.c | 59 +- ext/misc/windirent.h | 163 + ext/misc/zipfile.c | 49 +- ext/misc/zorder.c | 60 +- ext/rtree/rtree.c | 10 +- ext/rtree/rtreeH.test | 19 + ext/session/session9.test | 35 + ext/session/sessionI.test | 88 + ext/session/sessionnoop2.test | 144 + ext/session/sqlite3session.c | 233 +- ext/session/sqlite3session.h | 82 +- ext/session/test_session.c | 225 +- ext/wasm/GNUmakefile | 1568 +++-- ext/wasm/README.md | 30 +- ext/wasm/SQLTester/SQLTester.mjs | 30 +- ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core | 2 + .../api/EXPORTED_FUNCTIONS.sqlite3-extras | 33 +- ext/wasm/api/README.md | 139 +- ext/wasm/api/extern-post-js.c-pp.js | 115 +- ext/wasm/api/post-js-footer.js | 5 +- ext/wasm/api/post-js-header.js | 50 +- ext/wasm/api/pre-js.c-pp.js | 206 +- ext/wasm/api/sqlite3-api-cleanup.js | 97 +- ext/wasm/api/sqlite3-api-glue.c-pp.js | 971 +-- ext/wasm/api/sqlite3-api-oo1.c-pp.js | 464 +- ext/wasm/api/sqlite3-api-prologue.js | 459 +- ext/wasm/api/sqlite3-api-worker1.c-pp.js | 17 +- ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js | 16 +- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 19 +- ext/wasm/api/sqlite3-vtab-helper.c-pp.js | 12 +- ext/wasm/api/sqlite3-wasm.c | 82 +- ext/wasm/api/sqlite3-worker1-promiser.c-pp.js | 33 +- ext/wasm/api/sqlite3-worker1.c-pp.js | 8 +- ext/wasm/batch-runner-sahpool.html | 86 - ext/wasm/batch-runner-sahpool.js | 341 - ext/wasm/batch-runner.html | 90 - ext/wasm/batch-runner.js | 604 -- ext/wasm/c-pp-lite.c | 2767 ++++++++ ext/wasm/c-pp.c | 1531 ---- ext/wasm/common/whwasmutil.js | 1094 +-- ext/wasm/config.make.in | 13 +- ext/wasm/demo-worker1-promiser.c-pp.js | 4 +- ext/wasm/dist.make | 154 - ext/wasm/fiddle.make | 193 - ext/wasm/fiddle/fiddle-worker.js | 183 +- ext/wasm/fiddle/fiddle.js | 322 +- ext/wasm/fiddle/index.html | 189 +- ext/wasm/index-dist.html | 5 +- ext/wasm/index.html | 63 +- ext/wasm/jaccwabyt/jaccwabyt.js | 153 +- ext/wasm/jaccwabyt/jaccwabyt.md | 86 +- ext/wasm/mkdist.sh | 210 + ext/wasm/mkwasmbuilds.c | 1246 +++- ext/wasm/module-symbols.html | 16 +- ext/wasm/speedtest1-wasmfs.mjs | 2 +- ext/wasm/speedtest1-worker.html | 9 +- ext/wasm/speedtest1-worker.js | 5 + ext/wasm/speedtest1.html | 12 +- ...1-worker.html => tester1-worker.c-pp.html} | 27 +- ext/wasm/tester1.c-pp.html | 17 +- ext/wasm/tester1.c-pp.js | 775 ++- ext/wasm/wasmfs.make | 111 - main.mk | 122 +- manifest | 555 +- manifest.uuid | 2 +- mptest/mptest.c | 4 + sqlite3.1 | 28 +- src/bitvec.c | 110 +- src/btree.c | 56 +- src/btree.h | 1 + src/build.c | 38 +- {ext/misc => src}/carray.c | 154 +- src/date.c | 8 +- src/dbpage.c | 30 +- src/expr.c | 368 +- src/func.c | 522 +- src/insert.c | 5 +- src/json.c | 119 +- src/loadext.c | 5 +- src/main.c | 39 +- src/os_kv.c | 22 +- src/os_setup.h | 4 +- src/os_unix.c | 364 +- src/os_win.c | 418 +- src/pager.c | 23 +- src/pragma.c | 28 +- src/prepare.c | 6 +- src/printf.c | 128 +- src/resolve.c | 25 +- src/select.c | 298 +- src/shell.c.in | 762 +- src/sqlite.h.in | 327 +- src/sqlite3ext.h | 7 + src/sqliteInt.h | 61 +- src/sqliteLimit.h | 2 +- src/status.c | 91 +- src/tclsqlite.c | 138 +- src/tclsqlite.h | 2 +- src/test1.c | 187 +- src/test_config.c | 6 + src/test_fs.c | 12 +- src/test_malloc.c | 1 + src/test_windirent.c | 162 - src/test_windirent.h | 158 - src/tokenize.c | 13 +- src/treeview.c | 6 +- src/trigger.c | 9 +- src/vacuum.c | 3 +- src/vdbe.c | 124 +- src/vdbe.h | 7 +- src/vdbeInt.h | 12 +- src/vdbeapi.c | 15 +- src/vdbeaux.c | 144 +- src/vdbeblob.c | 33 +- src/vdbemem.c | 1 + src/vdbesort.c | 36 +- src/vdbetrace.c | 10 +- src/vxworks.h | 4 +- src/wal.c | 61 +- src/where.c | 66 +- src/whereInt.h | 4 + src/wherecode.c | 107 +- src/whereexpr.c | 41 +- src/window.c | 8 +- test/aggorderby.test | 12 + test/basexx1.test | 13 + test/bestindexC.test | 74 + test/bestindexE.test | 130 + test/between.test | 17 + test/bigrow.test | 11 + test/carray01.test | 2 +- test/carray02.test | 162 + test/carrayfault.test | 87 + test/cksumvfs.test | 68 +- test/date.test | 22 +- test/dblwidth-a.sql | 20 + test/dbstatus2.test | 49 + test/decimal.test | 19 + test/e_expr.test | 4 + test/eqp.test | 3 +- test/existsexpr.test | 432 ++ test/existsexpr2.test | 96 + test/existsfault.test | 49 + test/expr.test | 3 + test/fts3atoken2.test | 106 + test/func9.test | 13 + test/fuzzcheck.c | 37 +- test/fuzzinvariants.c | 29 +- test/hook.test | 10 + test/ieee754.test | 12 + test/imposter1.test | 1 + test/incrblob4.test | 98 + test/joinH.test | 63 + test/joinI.test | 125 + test/json101.test | 8 +- test/notnull2.test | 2 +- test/ossfuzz.c | 5 + test/percentile.test | 1 - test/permutations.test | 12 + test/regexp2.test | 15 + test/reservebytes.test | 51 + test/rowvalue.test | 13 + test/shell1.test | 83 + test/shell2.test | 92 + test/shell4.test | 2 +- test/speedtest.tcl | 2 +- test/speedtest1.c | 32 +- test/strict1.test | 97 + test/tabfunc01.test | 219 +- test/tclsqlite.test | 153 +- test/testloadext.c | 98 + test/testrunner.tcl | 178 +- test/testrunner_data.tcl | 2 +- test/testrunner_estwork.tcl | 488 ++ test/vacuum.test | 21 + test/vtabH.test | 4 +- test/walckptnoop.test | 110 + test/walthread.test | 47 +- test/where2.test | 16 + test/window1.test | 30 + test/without_rowid7.test | 63 + test/zipfile.test | 14 + test/zipfile2.test | 59 + tool/buildtclext.tcl | 68 +- tool/lemon.c | 6 +- tool/loadfts.c | 1 + tool/mkautoconfamal.sh | 14 +- tool/mkctimec.tcl | 2 + tool/mkshellc.tcl | 2 +- tool/mksqlite3c.tcl | 1 + tool/mksqlite3h.tcl | 35 +- tool/mktoolzip.tcl | 64 +- tool/mkwinarm64ec.tcl | 45 + tool/showstat4.c | 1 + tool/sqlite3_rsync.c | 10 +- tool/version-info.c | 12 +- tool/warnings.sh | 26 +- 312 files changed, 19651 insertions(+), 43589 deletions(-) delete mode 100644 autoconf/tea/doc/sqlite3.n create mode 100644 ext/fts5/test/fts5corruptbig.test create mode 100644 ext/fts5/test/fts5join.test delete mode 100644 ext/lsm1/Makefile delete mode 100644 ext/lsm1/Makefile.msc delete mode 100644 ext/lsm1/lsm-test/README delete mode 100644 ext/lsm1/lsm-test/lsmtest.h delete mode 100644 ext/lsm1/lsm-test/lsmtest1.c delete mode 100644 ext/lsm1/lsm-test/lsmtest2.c delete mode 100644 ext/lsm1/lsm-test/lsmtest3.c delete mode 100644 ext/lsm1/lsm-test/lsmtest4.c delete mode 100644 ext/lsm1/lsm-test/lsmtest5.c delete mode 100644 ext/lsm1/lsm-test/lsmtest6.c delete mode 100644 ext/lsm1/lsm-test/lsmtest7.c delete mode 100644 ext/lsm1/lsm-test/lsmtest8.c delete mode 100644 ext/lsm1/lsm-test/lsmtest9.c delete mode 100644 ext/lsm1/lsm-test/lsmtest_bt.c delete mode 100644 ext/lsm1/lsm-test/lsmtest_datasource.c delete mode 100644 ext/lsm1/lsm-test/lsmtest_func.c delete mode 100644 ext/lsm1/lsm-test/lsmtest_io.c delete mode 100644 ext/lsm1/lsm-test/lsmtest_main.c delete mode 100644 ext/lsm1/lsm-test/lsmtest_mem.c delete mode 100644 ext/lsm1/lsm-test/lsmtest_tdb.c delete mode 100644 ext/lsm1/lsm-test/lsmtest_tdb.h delete mode 100644 ext/lsm1/lsm-test/lsmtest_tdb2.cc delete mode 100644 ext/lsm1/lsm-test/lsmtest_tdb3.c delete mode 100644 ext/lsm1/lsm-test/lsmtest_tdb4.c delete mode 100644 ext/lsm1/lsm-test/lsmtest_util.c delete mode 100644 ext/lsm1/lsm-test/lsmtest_win32.c delete mode 100644 ext/lsm1/lsm.h delete mode 100644 ext/lsm1/lsmInt.h delete mode 100644 ext/lsm1/lsm_ckpt.c delete mode 100644 ext/lsm1/lsm_file.c delete mode 100644 ext/lsm1/lsm_log.c delete mode 100644 ext/lsm1/lsm_main.c delete mode 100644 ext/lsm1/lsm_mem.c delete mode 100644 ext/lsm1/lsm_mutex.c delete mode 100644 ext/lsm1/lsm_shared.c delete mode 100644 ext/lsm1/lsm_sorted.c delete mode 100644 ext/lsm1/lsm_str.c delete mode 100644 ext/lsm1/lsm_tree.c delete mode 100644 ext/lsm1/lsm_unix.c delete mode 100644 ext/lsm1/lsm_varint.c delete mode 100644 ext/lsm1/lsm_vtab.c delete mode 100644 ext/lsm1/lsm_win32.c delete mode 100644 ext/lsm1/test/lsm1_common.tcl delete mode 100644 ext/lsm1/test/lsm1_simple.test delete mode 100644 ext/lsm1/tool/mklsm1c.tcl delete mode 100644 ext/misc/carray.h delete mode 100644 ext/misc/memvfs.c create mode 100644 ext/misc/windirent.h create mode 100644 ext/session/sessionI.test delete mode 100644 ext/wasm/batch-runner-sahpool.html delete mode 100644 ext/wasm/batch-runner-sahpool.js delete mode 100644 ext/wasm/batch-runner.html delete mode 100644 ext/wasm/batch-runner.js create mode 100644 ext/wasm/c-pp-lite.c delete mode 100644 ext/wasm/c-pp.c delete mode 100644 ext/wasm/dist.make delete mode 100644 ext/wasm/fiddle.make create mode 100755 ext/wasm/mkdist.sh rename ext/wasm/{tester1-worker.html => tester1-worker.c-pp.html} (73%) delete mode 100644 ext/wasm/wasmfs.make rename {ext/misc => src}/carray.c (82%) delete mode 100644 src/test_windirent.c delete mode 100644 src/test_windirent.h create mode 100644 test/bestindexE.test create mode 100644 test/carray02.test create mode 100644 test/carrayfault.test create mode 100644 test/dblwidth-a.sql create mode 100644 test/existsexpr.test create mode 100644 test/existsexpr2.test create mode 100644 test/existsfault.test create mode 100644 test/fts3atoken2.test create mode 100644 test/joinI.test create mode 100644 test/reservebytes.test create mode 100644 test/testloadext.c create mode 100644 test/testrunner_estwork.tcl create mode 100644 test/walckptnoop.test create mode 100644 tool/mkwinarm64ec.tcl diff --git a/LICENSE.md b/LICENSE.md index ebfc07760..5f59bb8fe 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -30,14 +30,12 @@ The public domain source files usually contain a header comment similar to the following to make it clear that the software is public domain. -> ~~~ -The author disclaims copyright to this source code. In place of -a legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. -~~~ +> The author disclaims copyright to this source code. In place of +> a legal notice, here is a blessing: +> +> * May you do good and not evil. +> * May you find forgiveness for yourself and forgive others. +> * May you share freely, never taking more than you give. Almost every file you find in this source repository will be public domain. But there are a small number of exceptions: @@ -49,9 +47,7 @@ This repository contains a (relatively) small amount of non-public-domain code used to help implement the configuration and build logic. In other words, there are some non-public-domain files used to implement: -> ~~~ -./configure && make -~~~ +> ./configure && make In all cases, the non-public-domain files included with this repository have generous BSD-style licenses. So anyone is free to @@ -69,7 +65,7 @@ third-party build scripts in order to compile SQLite. Non-public-domain code included in this respository includes: * The ["autosetup"](http://msteveb.github.io/autosetup/) configuration - system that is contained (mostly) the autosetup/ directory, but also + system that is contained (mostly) in the autosetup/ directory, but also includes the "./configure" script at the top-level of this archive. Autosetup has a separate BSD-style license. See the [autosetup/LICENSE](http://msteveb.github.io/autosetup/license/) @@ -79,13 +75,11 @@ Non-public-domain code included in this respository includes: software found in the legacy autoconf/ directory and its subdirectories. -The following unix shell command is can be run from the top-level +The following unix shell command can be run from the top-level of this source repository in order to remove all non-public-domain code: -> ~~~ -rm -rf configure autosetup autoconf -~~~ +> rm -rf configure autosetup autoconf If you unpack this source repository and then run the command above, what is left will be 100% public domain. diff --git a/Makefile.in b/Makefile.in index 57728b049..a597a86a3 100644 --- a/Makefile.in +++ b/Makefile.in @@ -11,17 +11,6 @@ # all: ######################################################################## -# -# Known TODOs/FIXMEs/TOIMPROVEs for the autosetup port, in no -# particular order... -# -# - TEA pieces. -# -# - Replace the autotools-specific distribution deliverable(s). -# -# - Confirm whether cross-compilation works and patch it -# appropriately. -# # Maintenance reminders: # # - This makefile should remain as POSIX-make-compatible as possible: @@ -213,6 +202,13 @@ TCL_CONFIG_SH = @TCL_CONFIG_SH@ #TCL_STUB_LIB_SPEC = @TCL_STUB_LIB_SPEC@ #TCL_EXEC_PREFIX = @TCL_EXEC_PREFIX@ #TCL_VERSION = @TCL_VERSION@ +TCL_MAJOR_VERSION = @TCL_MAJOR_VERSION@ +# ^^^ main.mk optionally uses this for determining the Tcl extension's +# DLL name. +TCL_EXT_DLL_BASENAME = @TCL_EXT_DLL_BASENAME@ +# ^^^ base name of the Tcl extension DLL. It varies by platform and +# Tcl version. + # # $(TCLLIBDIR) = where to install the tcl plugin. If this is empty, it # is calculated at make-time by the targets which need it but we @@ -257,7 +253,9 @@ AS_AUTO_DEF = $(TOP)/auto.def # invoked with to produce this makefile. # AS_AUTORECONFIG = @SQLITE_AUTORECONFIG@ - +.PHONY: config reconfigure +config reconfigure: + $(AS_AUTORECONFIG) USE_AMALGAMATION ?= @USE_AMALGAMATION@ LINK_TOOLS_DYNAMICALLY ?= @LINK_TOOLS_DYNAMICALLY@ AMALGAMATION_GEN_FLAGS ?= --linemacros=@AMALGAMATION_LINE_MACROS@ @@ -315,9 +313,8 @@ fiddle: sqlite3.c shell.c ./custom.rws: ./tool/custom.txt @echo 'Updating custom dictionary from tool/custom.txt' aspell --lang=en create master ./custom.rws < ./tool/custom.txt -# Note that jimsh does not work here: -# https://github.com/msteveb/jimtcl/issues/319 misspell: ./custom.rws has_tclsh84 +# $(JIMSH) does not work with spellsift.tcl $(TCLSH_CMD) ./tool/spellsift.tcl ./src/*.c ./src/*.h ./src/*.in # @@ -334,7 +331,7 @@ distclean: distclean-autosetup # # tool/version-info: a utility for emitting sqlite3 version info -# in various forms. +# in various forms. It's used by ext/wasm/. # version-info$(T.exe): $(TOP)/tool/version-info.c Makefile sqlite3.h $(T.link) $(ST_OPT) -o $@ $(TOP)/tool/version-info.c diff --git a/Makefile.msc b/Makefile.msc index 6aef67155..3c075fac3 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -405,6 +405,7 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_BYTECODE_VTAB=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_CARRAY=1 !ENDIF OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 !ENDIF @@ -427,6 +428,7 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1 # Always enable math functions on Windows OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_MATH_FUNCTIONS +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PERCENTILE # Should the rbu extension be enabled? If so, add compilation options # to enable it. @@ -1343,7 +1345,7 @@ LTLIBS = $(LTLIBS) $(LIBICU) # LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ backup.lo bitvec.lo btmutex.lo btree.lo build.lo \ - callback.lo complete.lo ctime.lo \ + callback.lo carray.lo complete.lo ctime.lo \ date.lo dbpage.lo dbstat.lo delete.lo \ expr.lo fault.lo fkey.lo \ fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \ @@ -1405,6 +1407,7 @@ SRC00 = \ $(TOP)\src\btree.c \ $(TOP)\src\build.c \ $(TOP)\src\callback.c \ + $(TOP)\src\carray.c \ $(TOP)\src\complete.c \ ctime.c \ $(TOP)\src\date.c \ @@ -1623,7 +1626,6 @@ TESTSRC = \ $(TOP)\src\test_thread.c \ $(TOP)\src\test_vdbecov.c \ $(TOP)\src\test_vfs.c \ - $(TOP)\src\test_windirent.c \ $(TOP)\src\test_window.c \ $(TOP)\src\test_wsd.c \ $(TOP)\ext\fts3\fts3_term.c \ @@ -1639,7 +1641,6 @@ TESTEXT = \ $(TOP)\ext\misc\amatch.c \ $(TOP)\ext\misc\appendvfs.c \ $(TOP)\ext\misc\basexx.c \ - $(TOP)\ext\misc\carray.c \ $(TOP)\ext\misc\cksumvfs.c \ $(TOP)\ext\misc\closure.c \ $(TOP)\ext\misc\csv.c \ @@ -1655,7 +1656,6 @@ TESTEXT = \ $(TOP)\ext\misc\mmapwarm.c \ $(TOP)\ext\misc\nextchar.c \ $(TOP)\ext\misc\normalize.c \ - $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\prefixes.c \ $(TOP)\ext\misc\qpvtab.c \ $(TOP)\ext\misc\randomjson.c \ @@ -1767,6 +1767,7 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_DQS=0 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_PERCENTILE=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_STRICT_SUBTYPE=1 @@ -1781,6 +1782,7 @@ FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -I$(TOP)\test -I$(TOP)\ext\recover FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MEMSYS5 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OSS_FUZZ FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_CARRAY FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBSTAT_VTAB FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB @@ -1794,6 +1796,7 @@ FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MATH_FUNCTIONS FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MEMSYS5 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_NORMALIZE FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_PERCENTILE FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_PREUPDATE_HOOK FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_RTREE FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_SESSION @@ -1824,7 +1827,6 @@ FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\fuzzinvariants.c FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\vt02.c FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\dbdata.c FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\sqlite3recover.c -FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\percentile.c FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\randomjson.c OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c @@ -2102,6 +2104,9 @@ build.lo: $(TOP)\src\build.c $(HDR) callback.lo: $(TOP)\src\callback.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\callback.c +carray.lo: $(TOP)\src\carray.c $(HDR) + $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\carray.c + complete.lo: $(TOP)\src\complete.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\complete.c @@ -2385,7 +2390,6 @@ SHELL_DEP = \ $(TOP)\ext\misc\ieee754.c \ $(TOP)\ext\misc\memtrace.c \ $(TOP)\ext\misc\pcachetrace.c \ - $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\series.c \ $(TOP)\ext\misc\sha1.c \ @@ -2395,12 +2399,11 @@ SHELL_DEP = \ $(TOP)\ext\misc\sqlite3_stdio.h \ $(TOP)\ext\misc\uint.c \ $(TOP)\ext\misc\vfstrace.c \ + $(TOP)\ext\misc\windirent.h \ $(TOP)\ext\misc\zipfile.c \ $(TOP)\ext\recover\dbdata.c \ $(TOP)\ext\recover\sqlite3recover.c \ - $(TOP)\ext\recover\sqlite3recover.h \ - $(TOP)\src\test_windirent.c \ - $(TOP)\src\test_windirent.h + $(TOP)\ext\recover\sqlite3recover.h # If use of zlib is enabled, add the "zipfile.c" source file. # @@ -2487,24 +2490,6 @@ FTS5_SRC = \ $(TOP)\ext\fts5\fts5_varint.c \ $(TOP)\ext\fts5\fts5_vocab.c -LSM1_SRC = \ - $(TOP)\ext\lsm1\lsm.h \ - $(TOP)\ext\lsm1\lsmInt.h \ - $(TOP)\ext\lsm1\lsm_ckpt.c \ - $(TOP)\ext\lsm1\lsm_file.c \ - $(TOP)\ext\lsm1\lsm_log.c \ - $(TOP)\ext\lsm1\lsm_main.c \ - $(TOP)\ext\lsm1\lsm_mem.c \ - $(TOP)\ext\lsm1\lsm_mutex.c \ - $(TOP)\ext\lsm1\lsm_shared.c \ - $(TOP)\ext\lsm1\lsm_sorted.c \ - $(TOP)\ext\lsm1\lsm_str.c \ - $(TOP)\ext\lsm1\lsm_tree.c \ - $(TOP)\ext\lsm1\lsm_unix.c \ - $(TOP)\ext\lsm1\lsm_varint.c \ - $(TOP)\ext\lsm1\lsm_vtab.c \ - $(TOP)\ext\lsm1\lsm_win32.c - fts5parse.c: $(TOP)\ext\fts5\fts5parse.y lemon.exe copy /Y $(TOP)\ext\fts5\fts5parse.y . copy /B fts5parse.y +,, @@ -2518,11 +2503,6 @@ fts5.c: $(FTS5_SRC) $(JIM_TCLSH) copy /Y $(TOP)\ext\fts5\fts5.h . copy /B fts5.h +,, -lsm1.c: $(LSM1_SRC) $(JIM_TCLSH) - $(JIM_TCLSH) $(TOP)\ext\lsm1\tool\mklsm1c.tcl - copy /Y $(TOP)\ext\lsm1\lsm.h . - copy /B lsm.h +,, - fts5.lo: fts5.c $(HDR) $(EXTHDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c fts5.c @@ -2550,6 +2530,8 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_BYTECODE_VTAB=1 +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_CARRAY=1 +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_PERCENTILE=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CKSUMVFS_STATIC=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS) TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_STATIC_RANDOMJSON @@ -2585,7 +2567,7 @@ sqlite_tcl.h: testfixture.exe: $(TESTFIXTURE_SRC) $(TESTFIXTURE_DEP) $(SQLITE3H) $(LIBRESOBJS) $(HDR) $(SQLITE_TCL_DEP) $(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \ - -DBUILD_sqlite -I$(TCLINCDIR) \ + -DBUILD_sqlite -I$(TCLINCDIR) -I$(TOP)\ext\misc \ $(TESTFIXTURE_SRC) \ /link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS) @@ -2598,9 +2580,12 @@ extensiontest: testfixture.exe testloadext.dll @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS) -tool-zip: testfixture.exe sqlite3.exe sqldiff.exe sqlite3_analyzer.exe sqlite3_rsync.exe $(TOP)\tool\mktoolzip.tcl +tool-zip: testfixture.exe sqlite3.exe sqldiff.exe sqlite3_analyzer.exe sqlite3_rsync.exe sqlite3.dll $(TOP)\tool\mktoolzip.tcl .\testfixture.exe $(TOP)\tool\mktoolzip.tcl +snapshot-zip: testfixture.exe sqlite3.exe sqldiff.exe sqlite3_analyzer.exe sqlite3_rsync.exe sqlite3.dll $(TOP)\tool\mktoolzip.tcl + .\testfixture.exe $(TOP)\tool\mktoolzip.tcl --snapshot + coretestprogs: testfixture.exe sqlite3.exe testprogs: $(TESTPROGS) srcck1.exe fuzzcheck.exe sessionfuzz.exe @@ -2674,7 +2659,14 @@ srctree-check: $(TOP)\tool\srctree-check.tcl # Testing for a release # -releasetest: +releasetest: verify-source + $(TCLSH_CMD) $(TOP)\test\testrunner.tcl release + +# xdevtest is like releasetest, except that it skips the +# dependency on verify-source so that xdevtest can be run from +# a modified source tree. +# +xdevtest: $(TCLSH_CMD) $(TOP)\test\testrunner.tcl release @@ -2819,9 +2811,6 @@ tcl-env: @echo JIM_TCLSH = $(JIM_TCLSH) @echo VISUALSTUDIOVERSION = $(VISUALSTUDIOVERSION) -LSMDIR=$(TOP)\ext\lsm1 -!INCLUDE $(LSMDIR)\Makefile.msc - moreclean: clean del /Q $(SQLITE3C) $(SQLITE3H) 2>NUL # <> @@ -2863,7 +2852,6 @@ clean: del /Q kvtest.exe ossshell.exe scrub.exe 2>NUL del /Q showshm.exe sqlite3_checker.* sqlite3_expert.exe 2>NUL del /Q fts5.* fts5parse.* 2>NUL - del /Q lsm.h lsm1.c 2>NUL del /q src-verify.exe 2>NUL del /q jimsh.exe jimsh0.exe 2>NUL # <> diff --git a/README.md b/README.md index 0be1fb9f4..d44bf5cb7 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ This repository contains the complete source code for the [SQLite database engine](https://sqlite.org/), including -many test scripts. However, other test scripts -and most of the documentation are managed separately. +many tests. Additional tests and most documentation +are managed separately. See the [on-line documentation](https://sqlite.org/) for more information about what SQLite is and how it works from a user's perspective. This @@ -53,39 +53,32 @@ then no longer be fully in the public domain. ## Obtaining The SQLite Source Code -If you do not want to use Fossil, you can download tarballs or ZIP -archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows: +Source code tarballs or ZIP archives are available at: - * Latest trunk check-in as - [Tarball](https://sqlite.org/src/tarball/sqlite.tar.gz), - [ZIP-archive](https://sqlite.org/src/zip/sqlite.zip), or - [SQLite-archive](https://sqlite.org/src/sqlar/sqlite.sqlar). + * [Latest trunk check-in](https://sqlite.org/src/rchvdwnld/trunk). - * Latest release as - [Tarball](https://sqlite.org/src/tarball/sqlite.tar.gz?r=release), - [ZIP-archive](https://sqlite.org/src/zip/sqlite.zip?r=release), or - [SQLite-archive](https://sqlite.org/src/sqlar/sqlite.sqlar?r=release). + * [Latest release](https://sqlite.org/src/rchvdwnld/release) - * For other check-ins, substitute an appropriate branch name or - tag or hash prefix in place of "release" in the URLs of the previous - bullet. Or browse the [timeline](https://sqlite.org/src/timeline) - to locate the check-in desired, click on its information page link, - then click on the "Tarball" or "ZIP Archive" links on the information - page. + * For other check-ins, browse the + [project timeline](https://sqlite.org/src/timeline?y=ci) and + click on the check-in hash of the check-in you want to download. + On the resulting "info" page, click one of the options to the + right of the "**Downloads:**" label in the "**Overview**" section + near the top. To access sources directly using [Fossil](https://fossil-scm.org/home), first install Fossil version 2.0 or later. -Source tarballs and precompiled binaries available at +Source tarballs and precompiled binaries for Fossil are available at . Fossil is a stand-alone program. To install, simply download or build the single -executable file and put that file someplace on your $PATH. +executable file and put that file someplace on your $PATH or %PATH%. Then run commands like this: mkdir -p ~/sqlite cd ~/sqlite fossil open https://sqlite.org/src -The "fossil open" command will take two or three minutes. Afterwards, +The initial "fossil open" command will take two or three minutes. Afterwards, you can do fast, bandwidth-efficient updates to the whatever versions of SQLite you like. Some examples: @@ -152,7 +145,8 @@ show what changes are needed. ## Compiling for Windows Using MSVC On Windows, everything can be compiled with MSVC. -You will also need a working installation of TCL. +You will also need a working installation of TCL if you want to run tests. +TCL is not required if you just want to build SQLite itself. See the [compile-for-windows.md](doc/compile-for-windows.md) document for additional information about how to install MSVC and TCL and configure your build environment. @@ -164,8 +158,8 @@ TCL library, using a command like this: SQLite uses "tclsh.exe" as part of the build process, and so that program will need to be somewhere on your %PATH%. SQLite itself -does not contain any TCL code, but it does use TCL to help with the -build process and to run tests. You may need to install TCL development +does not contain any TCL code, but it does use TCL to run tests. +You may need to install TCL development libraries in order to successfully complete some makefile targets. It is helpful, but is not required, to install the SQLite TCL extension (the "tclextension-install" target) prior to running tests. diff --git a/VERSION b/VERSION index acecb4fcb..84ba969c4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.50.4 +3.51.0 diff --git a/auto.def b/auto.def index 43c597551..214ef2230 100644 --- a/auto.def +++ b/auto.def @@ -12,6 +12,11 @@ # # JimTCL: https://jim.tcl.tk # +# Code-diver notes: APIs names starting with "sqlite-" are specific to +# this project and can be found in autosetup/sqlite-config.tcl. Names +# starting with "proj-" are project-agnostic and found in +# autosetup/proj.tcl. +# use sqlite-config sqlite-configure canonical { proj-if-opt-truthy dev { diff --git a/autoconf/Makefile.in b/autoconf/Makefile.in index a77386fae..c938ffe1b 100644 --- a/autoconf/Makefile.in +++ b/autoconf/Makefile.in @@ -117,6 +117,19 @@ sqlite_cfg.h: $(AS_AUTO_DEF) # CFLAGS for sqlite3$(T.exe) # SHELL_OPT ?= @OPT_SHELL@ +SHELL_OPT += -DSQLITE_DQS=0 +SHELL_OPT += -DSQLITE_ENABLE_FTS4 +#SHELL_OPT += -DSQLITE_ENABLE_FTS5 +SHELL_OPT += -DSQLITE_ENABLE_RTREE +SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS +SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION +SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB +SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB +SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB +SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB +SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC +SHELL_OPT += -DSQLITE_ENABLE_PERCENTILE +SHELL_OPT += -DSQLITE_STRICT_SUBTYPE=1 # # Library-level feature flags @@ -222,9 +235,12 @@ install: install-lib # Flags to link the shell app either directly against sqlite3.c # (ENABLE_STATIC_SHELL==1) or libsqlite3.so (ENABLE_STATIC_SHELL==0). # +# Maintenance reminder: placement of $(LDFLAGS) is more relevant for +# some platforms than others: +# https://sqlite.org/forum/forumpost/d80ecdaddd ENABLE_STATIC_SHELL = @ENABLE_STATIC_SHELL@ -sqlite3-shell-link-flags.1 = $(TOP)/sqlite3.c $(LDFLAGS.libsqlite3) -sqlite3-shell-link-flags.0 = -L. -lsqlite3 $(LDFLAGS.zlib) $(LDFLAGS.math) +sqlite3-shell-link-flags.1 = $(TOP)/sqlite3.c $(LDFLAGS) $(LDFLAGS.libsqlite3) +sqlite3-shell-link-flags.0 = $(LDFLAGS) -L. -lsqlite3 $(LDFLAGS.zlib) $(LDFLAGS.math) sqlite3-shell-deps.1 = $(TOP)/sqlite3.c sqlite3-shell-deps.0 = $(libsqlite3.DLL) # @@ -245,7 +261,7 @@ sqlite3$(T.exe): $(TOP)/shell.c $(sqlite3-shell-deps.$(ENABLE_STATIC_SHELL)) $(sqlite3-shell-static.flags.$(STATIC_CLI_SHELL)) \ -I. $(OPT_FEATURE_FLAGS) $(SHELL_OPT) \ $(CFLAGS) $(CFLAGS.readline) $(CFLAGS.icu) \ - $(LDFLAGS) $(LDFLAGS.readline) + $(LDFLAGS.readline) sqlite3$(T.exe)-1: sqlite3$(T.exe)-0: sqlite3$(T.exe) diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index dfa2dcfd5..4f96e3b18 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -327,6 +327,7 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_BYTECODE_VTAB=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_CARRAY=1 !ENDIF OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 !ENDIF @@ -349,6 +350,7 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PREUPDATE_HOOK=1 # Always enable math functions on Windows OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_MATH_FUNCTIONS +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_PERCENTILE # Should the rbu extension be enabled? If so, add compilation options # to enable it. @@ -1015,6 +1017,7 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_DQS=0 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_PERCENTILE=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_STRICT_SUBTYPE=1 diff --git a/autoconf/tea/Makefile.in b/autoconf/tea/Makefile.in index 5b2ad4c69..ccf9a7b94 100644 --- a/autoconf/tea/Makefile.in +++ b/autoconf/tea/Makefile.in @@ -146,7 +146,7 @@ LDFLAGS.shlib = @SH_LDFLAGS@ # sources passed to [teaish-src-add], but may also be appended to by # teaish.make. # -tx.src =@TEAISH_EXT_SRC@ +tx.src = @TEAISH_EXT_SRC@ # # tx.CFLAGS is typically set by teaish.make, whereas TEAISH_CFLAGS @@ -167,6 +167,11 @@ tx.LDFLAGS = # tx.dist.files = @TEAISH_DIST_FILES@ +# +# The base name for a distribution tar/zip file. +# +tx.dist.basename = $(tx.name.dist)-$(tx.version) + # List of deps which may trigger an auto-reconfigure. # teaish__autogen.deps = \ @@ -199,9 +204,14 @@ $(teaish.makefile): $(teaish__auto.def) $(teaish.makefile.in) \ @AUTODEPS@ @if TEAISH_TESTER_TCL_IN -@TEAISH_TESTER_TCL_IN@: -@TEAISH_TESTER_TCL@: @TEAISH_TESTER_TCL_IN@ -config.log: @TEAISH_TESTER_TCL@ +@TEAISH_TESTER_TCL_IN@: $(teaish__autogen.deps) +config.log: @TEAISH_TESTER_TCL_IN@ +@TEAISH_TESTER_TCL@: @TEAISH_TESTER_TCL_IN@ +@endif +@if TEAISH_TEST_TCL_IN +@TEAISH_TEST_TCL_IN@: $(teaish__autogen.deps) +config.log: @TEAISH_TEST_TCL_IN@ +@TEAISH_TEST_TCL@: @TEAISH_TEST_TCL_IN@ @endif # @@ -217,7 +227,7 @@ CC.tcl = \ # CC.dll = \ $(CC.tcl) $(tx.src) $(LDFLAGS.shlib) \ - $(LDFLAGS.configure) $(LDFLAGS) $(tx.LDFLAGS) $(TCL_STUB_LIB_SPEC) + $(tx.LDFLAGS) $(LDFLAGS.configure) $(LDFLAGS) $(TCL_STUB_LIB_SPEC) @if TEAISH_ENABLE_DLL # @@ -248,16 +258,25 @@ test-extension: # this name is reserved for use by teaish.make[.in] test-prepre: $(tx.dll) @endif @if TEAISH_TESTER_TCL -test-core.args = @TEAISH_TESTER_TCL@ +teaish.tester.tcl = @TEAISH_TESTER_TCL@ +test-core.args = $(teaish.tester.tcl) @if TEAISH_ENABLE_DLL test-core.args += '$(tx.dll)' '$(tx.loadPrefix)' @else test-core.args += '' '' @endif test-core.args += @TEAISH_TESTUTIL_TCL@ +# Clients may pass additional args via test.args=... +# and ::argv will be rewritten before the test script loads, to +# remove $(test-core.args) +test.args ?= test-core: test-pre - $(TCLSH) $(test-core.args) -test-prepre: @TEAISH_TESTER_TCL@ + $(TCLSH) $(test-core.args) $(test.args) +test-gdb: $(teaish.tester.tcl) + gdb --args $(TCLSH) $(test-core.args) $(test.args) +test-vg.flags ?= --leak-check=full -v --show-reachable=yes --track-origins=yes +test-vg: $(teaish.tester.tcl) + valgrind $(test-vg.flags) $(TCLSH) $(test-core.args) $(test.args) @else # !TEAISH_TESTER_TCL test-prepre: @endif # TEAISH_TESTER_TCL @@ -288,7 +307,7 @@ distclean-core: distclean-pre @endif @endif @if TEAISH_TESTER_TCL_IN - rm -f @TEAISH_TESTER_TCL@ + rm -f $(teaish.tester.tcl) @endif @if TEAISH_PKGINDEX_TCL_IN rm -f @TEAISH_PKGINDEX_TCL@ @@ -355,10 +374,15 @@ install-core: install-pre @endif install-test: install-core @echo "Post-install test of [package require $(tx.name.pkg) $(tx.version)]..."; \ + set xtra=""; \ + if [ x != "x$(DESTDIR)" ]; then \ + xtra='set ::auto_path [linsert $$::auto_path 0 [file normalize $(DESTDIR)$(TCLLIBDIR)/..]];'; \ + fi; \ if echo \ - 'set c 0; ' \ + 'set c 0; ' $$xtra \ '@TEAISH_POSTINST_PREREQUIRE@' \ - 'if {[catch {package require $(tx.name.pkg) $(tx.version)}]} {incr c};' \ + 'if {[catch {package require $(tx.name.pkg) $(tx.version)} xc]} {incr c};' \ + 'if {$$c && "" ne $$xc} {puts $$xc; puts "auto_path=$$::auto_path"};' \ 'exit $$c' \ | $(TCLSH) ; then \ echo "passed"; \ @@ -406,7 +430,7 @@ config.log: $(teaish.makefile.in) # recognized when running in --teaish-install mode, causing # the sub-configure to fail. dist.flags = --with-tclsh=$(TCLSH) -dist.reconfig = $(teaish.dir)/configure $(dist.flags) +dist.reconfig = $(teaish.dir)/configure $(tx.dist.reconfig-flags) $(dist.flags) # Temp dir for dist.zip. Must be different than dist.tgz or else # parallel builds may hose the dist. @@ -414,24 +438,23 @@ teaish__dist.tmp.zip = teaish__dist_zip # # Make a distribution zip file... # -dist.basename = $(tx.name.dist)-$(tx.version) -dist.zip = $(dist.basename).zip +dist.zip = $(tx.dist.basename).zip .PHONY: dist.zip dist.zip-core dist.zip-post #dist.zip-pre: # We apparently can't add a pre-hook here, else "make dist" rebuilds # the archive each time it's run. $(dist.zip): $(tx.dist.files) @rm -fr $(teaish__dist.tmp.zip) - @mkdir -p $(teaish__dist.tmp.zip)/$(dist.basename) + @mkdir -p $(teaish__dist.tmp.zip)/$(tx.dist.basename) @tar cf $(teaish__dist.tmp.zip)/tmp.tar $(tx.dist.files) - @tar xf $(teaish__dist.tmp.zip)/tmp.tar -C $(teaish__dist.tmp.zip)/$(dist.basename) + @tar xf $(teaish__dist.tmp.zip)/tmp.tar -C $(teaish__dist.tmp.zip)/$(tx.dist.basename) @if TEAISH_DIST_FULL @$(dist.reconfig) \ - --teaish-install=$(teaish__dist.tmp.zip)/$(dist.basename) \ - --t-e-d=$(teaish__dist.tmp.zip)/$(dist.basename) >/dev/null + --teaish-install=$(teaish__dist.tmp.zip)/$(tx.dist.basename) \ + --t-e-d=$(teaish__dist.tmp.zip)/$(tx.dist.basename) >/dev/null @endif - @rm -f $(dist.basename)/tmp.tar $(dist.zip) - @cd $(teaish__dist.tmp.zip) && zip -q -r ../$(dist.zip) $(dist.basename) + @rm -f $(tx.dist.basename)/tmp.tar $(dist.zip) + @cd $(teaish__dist.tmp.zip) && zip -q -r ../$(dist.zip) $(tx.dist.basename) @rm -fr $(teaish__dist.tmp.zip) @ls -la $(dist.zip) dist.zip-core: $(dist.zip) @@ -447,23 +470,23 @@ undist: undist-zip # Make a distribution tarball... # teaish__dist.tmp.tgz = teaish__dist_tgz -dist.tgz = $(dist.basename).tar.gz +dist.tgz = $(tx.dist.basename).tar.gz .PHONY: dist.tgz dist.tgz-core dist.tgz-post # dist.tgz-pre: # see notes in dist.zip $(dist.tgz): $(tx.dist.files) @rm -fr $(teaish__dist.tmp.tgz) - @mkdir -p $(teaish__dist.tmp.tgz)/$(dist.basename) + @mkdir -p $(teaish__dist.tmp.tgz)/$(tx.dist.basename) @tar cf $(teaish__dist.tmp.tgz)/tmp.tar $(tx.dist.files) - @tar xf $(teaish__dist.tmp.tgz)/tmp.tar -C $(teaish__dist.tmp.tgz)/$(dist.basename) + @tar xf $(teaish__dist.tmp.tgz)/tmp.tar -C $(teaish__dist.tmp.tgz)/$(tx.dist.basename) @if TEAISH_DIST_FULL - @rm -f $(teaish__dist.tmp.tgz)/$(dist.basename)/pkgIndex.tcl.in; # kludge + @rm -f $(teaish__dist.tmp.tgz)/$(tx.dist.basename)/pkgIndex.tcl.in; # kludge @$(dist.reconfig) \ - --teaish-install=$(teaish__dist.tmp.tgz)/$(dist.basename) \ - --t-e-d=$(teaish__dist.tmp.zip)/$(dist.basename) >/dev/null + --teaish-install=$(teaish__dist.tmp.tgz)/$(tx.dist.basename) \ + --t-e-d=$(teaish__dist.tmp.zip)/$(tx.dist.basename) >/dev/null @endif - @rm -f $(dist.basename)/tmp.tar $(dist.tgz) - @cd $(teaish__dist.tmp.tgz) && tar czf ../$(dist.tgz) $(dist.basename) + @rm -f $(tx.dist.basename)/tmp.tar $(dist.tgz) + @cd $(teaish__dist.tmp.tgz) && tar czf ../$(dist.tgz) $(tx.dist.basename) @rm -fr $(teaish__dist.tmp.tgz) @ls -la $(dist.tgz) dist.tgz-core: $(dist.tgz) diff --git a/autoconf/tea/README.txt b/autoconf/tea/README.txt index fb7cb1924..122b08d32 100644 --- a/autoconf/tea/README.txt +++ b/autoconf/tea/README.txt @@ -83,22 +83,12 @@ script and then run make. For example: $ cd sqlite-*-tea $ ./configure --with-tcl=/path/to/tcl/install/root - $ make + $ make test $ make install WINDOWS BUILD ============= -The recommended method to build extensions under windows is to use the -Msys + Mingw build process. This provides a Unix-style build while -generating native Windows binaries. Using the Msys + Mingw build tools -means that you can use the same configure script as per the Unix build -to create a Makefile. See the tcl/win/README file for the URL of -the Msys + Mingw download. -If you have VC++ then you may wish to use the files in the win -subdirectory and build the extension using just VC++. These files have -been designed to be as generic as possible but will require some -additional maintenance by the project developer to synchronise with -the TEA configure.in and Makefile.in files. Instructions for using the -VC++ makefile are written in the first part of the Makefile.vc -file. +On Windows this build is known to work on Cygwin and some Msys2 +environments. We do not currently support Microsoft makefiles for +native Windows builds. diff --git a/autoconf/tea/_teaish.tester.tcl.in b/autoconf/tea/_teaish.tester.tcl.in index 59d11f0a8..e04d8e63e 100644 --- a/autoconf/tea/_teaish.tester.tcl.in +++ b/autoconf/tea/_teaish.tester.tcl.in @@ -21,7 +21,8 @@ if {[llength [lindex $::argv 0]] > 0} { # ----^^^^^^^ needed on Haiku when argv 0 is just a filename, else # load cannot find the file. } -source -encoding utf-8 [lindex $::argv 2]; # teaish/tester.tcl +set ::argv [lassign $argv - -] +source -encoding utf-8 [lindex $::argv 0]; # teaish/tester.tcl @if TEAISH_PKGINIT_TCL apply {{file} { set dir [file dirname $::argv0] diff --git a/autoconf/tea/configure b/autoconf/tea/configure index 47378126f..01b3abcc2 100755 --- a/autoconf/tea/configure +++ b/autoconf/tea/configure @@ -1,7 +1,20 @@ #!/bin/sh +# Look for and run autosetup... dir0="`dirname "$0"`" -dirA="$dir0/../autosetup" -# This is the case ^^^^^^^^^^^^ in the SQLite "autoconf" bundle. +dirA="$dir0" +if [ -d $dirA/autosetup ]; then + # A local copy of autosetup + dirA=$dirA/autosetup +elif [ -d $dirA/../autosetup ]; then + # SQLite "autoconf" bundle + dirA=$dirA/../autosetup +elif [ -d $dirA/../../autosetup ]; then + # SQLite canonical source tree + dirA=$dirA/../../autosetup +else + echo "$0: Cannot find autosetup" 1>&2 + exit 1 +fi WRAPPER="$0"; export WRAPPER; exec "`"$dirA/autosetup-find-tclsh"`" \ "$dirA/autosetup" --teaish-extension-dir="$dir0" \ "$@" diff --git a/autoconf/tea/doc/sqlite3.n b/autoconf/tea/doc/sqlite3.n deleted file mode 100644 index 351404634..000000000 --- a/autoconf/tea/doc/sqlite3.n +++ /dev/null @@ -1,15 +0,0 @@ -.TH sqlite3 n 4.1 "Tcl-Extensions" -.HS sqlite3 tcl -.BS -.SH NAME -sqlite3 \- an interface to the SQLite3 database engine -.SH SYNOPSIS -\fBsqlite3\fI command_name ?filename?\fR -.br -.SH DESCRIPTION -SQLite3 is a self-contains, zero-configuration, transactional SQL database -engine. This extension provides an easy to use interface for accessing -SQLite database files from Tcl. -.PP -For full documentation see \fIhttps://sqlite.org/\fR and -in particular \fIhttps://sqlite.org/tclsqlite.html\fR. diff --git a/autoconf/tea/teaish.tcl b/autoconf/tea/teaish.tcl index 9333495aa..47e0ea701 100644 --- a/autoconf/tea/teaish.tcl +++ b/autoconf/tea/teaish.tcl @@ -64,12 +64,18 @@ apply {{dir} { -name.pkg sqlite3 -version $version -name.dist $distname - -vsatisfies 8.6- -libDir sqlite$version -pragmas $pragmas + -src generic/tclsqlite3.c } + # We should also have: + # -vsatisfies 8.6- + # But at least one platform is failing this vsatisfies check + # for no apparent reason: + # https://sqlite.org/forum/forumpost/fde857fb8101a4be }} [teaish-get -dir] + # # Must return either an empty string or a list in the form accepted by # autosetup's [options] function. @@ -119,8 +125,6 @@ proc teaish-options {} { proc teaish-configure {} { use teaish/feature - teaish-src-add -dist -dir generic/tclsqlite3.c - if {[proj-opt-was-provided override-sqlite-version]} { teaish-pkginfo-set -version [opt-val override-sqlite-version] proj-warn "overriding sqlite version number:" [teaish-pkginfo-get -version] diff --git a/autosetup/README.md b/autosetup/README.md index 3301f5739..c8da5c643 100644 --- a/autosetup/README.md +++ b/autosetup/README.md @@ -380,8 +380,9 @@ and its own special handling of `--enable-...` flags makes `--debug` an alias for `--enable-debug`. As this project has a long history of using `--enable-debug`, we patch autosetup to use the name `--autosetup-debug` in place of `--debug`. That requires (as of this -writing) four small edits in [](/file/autosetup/autosetup), as -demonstrated in [check-in 3296c8d3](/info/3296c8d3). +writing) four small edits in +[/autosetup/autosetup](/file/autosetup/autosetup), as demonstrated in +[check-in 3296c8d3](/info/3296c8d3). If autosetup is upgraded and this patch is _not_ applied the invoking `./configure` will fail loudly because of the declaration of the @@ -426,7 +427,7 @@ proc sqlite-custom-flags {} { ``` That function must return either an empty string or a list in the form -used internally by `sqlite-config.tcl:sqlite-configure`. +used internally by [sqlite-config.tcl][]'s `sqlite-configure`. Next, define: diff --git a/autosetup/autosetup b/autosetup/autosetup index 239987554..c3a31bec5 100755 --- a/autosetup/autosetup +++ b/autosetup/autosetup @@ -406,8 +406,8 @@ proc options-add {opts} { # Find the corresponding value in the user options # and set the default if necessary if {[string match "-*" $opt]} { - # This is a documentation-only option, like "-C " - set opthelp $opt + # We no longer support documentation-only options, like "-C " + autosetup-error "Option $opt is not supported" } elseif {$colon eq ""} { # Boolean option lappend autosetup(options) $name @@ -1611,7 +1611,7 @@ proc autosetup_output_block {type lines} { # Generate a command reference from inline documentation proc automf_command_reference {} { lappend files $::autosetup(prog) - lappend files {*}[lsort [glob -nocomplain $::autosetup(libdir)/*.tcl]] + lappend files {*}[lsort [glob -nocomplain $::autosetup(libdir)/{*/*.tcl,*.tcl}]] # We want to process all non-module files before module files # and then modules in alphabetical order. @@ -2124,7 +2124,7 @@ if {$autosetup(istcl)} { set frame [info frame -$i] if {[dict exists $frame file]} { # We don't need proc, so use "" - lappend stacktrace "" [dict get $frame file] [dict get $frame line] + lappend stacktrace "" [dict get $frame file] [dict get $frame line] "" } } return $stacktrace @@ -2181,8 +2181,12 @@ proc error-location {msg} { if {$::autosetup(debug)} { return -code error $msg } - # Search back through the stack trace for the first error in a .def file - foreach {p f l} [stacktrace] { + set vars {p f l cmd} + if {!$::autosetup(istcl) && ![dict exists $::tcl_platform stackFormat]} { + # Older versions of Jim had a 3 element stacktrace + set vars {p f l} + } + foreach $vars [stacktrace] { if {[string match *.def $f]} { return "[relative-path $f]:$l: Error: $msg" } @@ -2534,7 +2538,7 @@ if {[catch {main $argv} msg opts] == 1} { show-notices autosetup-full-error [error-dump $msg $opts $autosetup(debug)] if {!$autosetup(debug)} { - puts stderr "Try: '[file tail $autosetup(exe)] --autosetup-debug' for a full stack trace" + puts stderr "Try: '[file tail $autosetup(exe)] --debug' for a full stack trace" } exit 1 } diff --git a/autosetup/cc-shared.tcl b/autosetup/cc-shared.tcl index cbe568018..1fa200eec 100644 --- a/autosetup/cc-shared.tcl +++ b/autosetup/cc-shared.tcl @@ -89,13 +89,15 @@ switch -glob -- [get-define host] { define SH_SOPREFIX -Wl,-h, } } - *-*-hpux { - # XXX: These haven't been tested - define SHOBJ_CFLAGS "+O3 +z" + *-*-hpux* { + define SHOBJ_CFLAGS +z define SHOBJ_LDFLAGS -b define SH_CFLAGS +z + define SH_LDFLAGS -b define SH_LINKFLAGS -Wl,+s - define LD_LIBRARY_PATH SHLIB_PATH + define SH_LINKRPATH "-Wl,+b -Wl,%s" + define SH_SOPREFIX -Wl,+h, + define STRIPLIBFLAGS -Wl,-s } *-*-haiku { define SHOBJ_CFLAGS "" diff --git a/autosetup/jimsh0.c b/autosetup/jimsh0.c index b035524c9..1a6453d0c 100644 --- a/autosetup/jimsh0.c +++ b/autosetup/jimsh0.c @@ -9132,7 +9132,7 @@ int Jim_StringEqObj(Jim_Obj *aObjPtr, Jim_Obj *bObjPtr) const char *sA = Jim_GetString(aObjPtr, &Alen); const char *sB = Jim_GetString(bObjPtr, &Blen); - return Alen == Blen && *sA == *sB && memcmp(sA, sB, Alen) == 0; + return Alen == Blen && memcmp(sA, sB, Alen) == 0; } } @@ -10242,7 +10242,7 @@ static int JimCommandsHT_KeyCompare(void *privdata, const void *key1, const void int len1, len2; const char *str1 = Jim_GetStringNoQualifier((Jim_Obj *)key1, &len1); const char *str2 = Jim_GetStringNoQualifier((Jim_Obj *)key2, &len2); - return len1 == len2 && *str1 == *str2 && memcmp(str1, str2, len1) == 0; + return len1 == len2 && memcmp(str1, str2, len1) == 0; } static void JimCommandsHT_ValDestructor(void *interp, void *val) @@ -13864,13 +13864,6 @@ static int JimExprOpNumUnary(Jim_Interp *interp, struct JimExprNode *node) case JIM_EXPROP_NOT: wC = !bA; break; - case JIM_EXPROP_UNARYPLUS: - case JIM_EXPROP_UNARYMINUS: - rc = JIM_ERR; - Jim_SetResultFormatted(interp, - "can't use non-numeric string as operand of \"%s\"", - node->type == JIM_EXPROP_UNARYPLUS ? "+" : "-"); - break; default: abort(); } @@ -19875,22 +19868,16 @@ static int JimCatchTryHelper(Jim_Interp *interp, int istry, int argc, Jim_Obj *c } else if (errorCodeObj) { int len = Jim_ListLength(interp, argv[idx + 1]); + int i; - if (len > Jim_ListLength(interp, errorCodeObj)) { + ret = JIM_OK; - ret = -1; - } - else { - int i; - ret = JIM_OK; - - for (i = 0; i < len; i++) { - Jim_Obj *matchObj = Jim_ListGetIndex(interp, argv[idx + 1], i); - Jim_Obj *objPtr = Jim_ListGetIndex(interp, errorCodeObj, i); - if (Jim_StringCompareObj(interp, matchObj, objPtr, 0) != 0) { - ret = -1; - break; - } + for (i = 0; i < len; i++) { + Jim_Obj *matchObj = Jim_ListGetIndex(interp, argv[idx + 1], i); + Jim_Obj *objPtr = Jim_ListGetIndex(interp, errorCodeObj, i); + if (Jim_StringCompareObj(interp, matchObj, objPtr, 0) != 0) { + ret = -1; + break; } } } @@ -20266,7 +20253,7 @@ static int Jim_DictCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *arg } case OPT_SET: - return Jim_SetDictKeysVector(interp, argv[2], argv + 3, argc - 4, argv[argc - 1], JIM_ERRMSG | JIM_UNSHARED); + return Jim_SetDictKeysVector(interp, argv[2], argv + 3, argc - 4, argv[argc - 1], JIM_ERRMSG); case OPT_EXISTS:{ int rc = Jim_DictKeysVector(interp, argv[2], argv + 3, argc - 3, &objPtr, JIM_NONE); @@ -20278,7 +20265,7 @@ static int Jim_DictCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *arg } case OPT_UNSET: - if (Jim_SetDictKeysVector(interp, argv[2], argv + 3, argc - 3, NULL, JIM_UNSHARED) != JIM_OK) { + if (Jim_SetDictKeysVector(interp, argv[2], argv + 3, argc - 3, NULL, JIM_NONE) != JIM_OK) { return JIM_ERR; } return JIM_OK; diff --git a/autosetup/proj.tcl b/autosetup/proj.tcl index 133556706..86f4df44e 100644 --- a/autosetup/proj.tcl +++ b/autosetup/proj.tcl @@ -60,10 +60,11 @@ # $proj__Config is an internal-use-only array for storing whatever generic # internal stuff we need stored. # -array set ::proj__Config { - self-tests 1 -} - +array set ::proj__Config [subst { + self-tests [get-env proj.self-tests 0] + verbose-assert [get-env proj.assert-verbose 0] + isatty [isatty? stdout] +}] # # List of dot-in files to filter in the final stages of @@ -75,7 +76,6 @@ array set ::proj__Config { # See: proj-dot-ins-append and proj-dot-ins-process # set ::proj__Config(dot-in-files) [list] -set ::proj__Config(isatty) [isatty? stdout] # # @proj-warn msg @@ -85,28 +85,29 @@ set ::proj__Config(isatty) [isatty? stdout] # proc proj-warn {args} { show-notices - puts stderr [join [list "WARNING: \[[proj-scope 1]\]: " {*}$args] " "] + puts stderr [join [list "WARNING:" \[ [proj-scope 1] \]: {*}$args] " "] } +# # Internal impl of [proj-fatal] and [proj-error]. It must be called # using tailcall. -proc proj__faterr {failMode argv} { +# +proc proj__faterr {failMode args} { show-notices set lvl 1 - while {"-up" eq [lindex $argv 0]} { - set argv [lassign $argv -] + while {"-up" eq [lindex $args 0]} { + set args [lassign $args -] incr lvl } if {$failMode} { - puts stderr [join [list "FATAL: \[[proj-scope $lvl]]: " {*}$argv]] + puts stderr [join [list "FATAL:" \[ [proj-scope $lvl] \]: {*}$args]] exit 1 } else { - error [join [list "\[[proj-scope $lvl]]:" {*}$argv]] + error [join [list in \[ [proj-scope $lvl] \]: {*}$args]] } } - # # @proj-fatal ?-up...? msg... # @@ -118,7 +119,7 @@ proc proj__faterr {failMode argv} { # additional level. # proc proj-fatal {args} { - tailcall proj__faterr 1 $args + tailcall proj__faterr 1 {*}$args } # @@ -127,10 +128,9 @@ proc proj-fatal {args} { # Works like proj-fatal but uses [error] intead of [exit]. # proc proj-error {args} { - tailcall proj__faterr 0 $args + tailcall proj__faterr 0 {*}$args } -set ::proj__Config(verbose-assert) [get-env proj-assert-verbose 0] # # @proj-assert script ?message? # @@ -147,7 +147,7 @@ proc proj-assert {script {msg ""}} { if {"" eq $msg} { set msg $script } - proj-fatal "Assertion failed in \[[proj-scope 1]\]: $msg" + tailcall proj__faterr 1 "Assertion failed:" $msg } } @@ -378,8 +378,8 @@ proc proj-bin-define {binName {defName {}}} { # # Despite using cc-path-progs to do the search, this function clears # any define'd name that function stores for the result (because the -# caller has no sensible way of knowing which result it was unless -# they pass only a single argument). +# caller has no sensible way of knowing which [define] name it has +# unless they pass only a single argument). # proc proj-first-bin-of {args} { set rc "" @@ -451,7 +451,9 @@ proc proj-opt-set {flag {val 1}} { # @proj-opt-exists flag # # Returns 1 if the given flag has been defined as a legal configure -# option, else returns 0. +# option, else returns 0. Options set via proj-opt-set "exist" for +# this purpose even if they were not defined via autosetup's +# [options] function. # proc proj-opt-exists {flag} { expr {$flag in $::autosetup(options)}; @@ -555,7 +557,7 @@ proc proj-opt-define-bool {args} { if {$invert} { set rc [expr {!$rc}] } - msg-result $rc + msg-result [string map {0 no 1 yes} $rc] define $defName $rc return $rc } @@ -704,11 +706,20 @@ proc proj-file-write {args} { } # -# @proj-check-compile-commands ?configFlag? +# @proj-check-compile-commands ?-assume-for-clang? ?configFlag? +# +# Checks the compiler for compile_commands.json support. If +# $configFlag is not empty then it is assumed to be the name of an +# autosetup boolean config which controls whether to run/skip this +# check. # -# Checks the compiler for compile_commands.json support. If passed an -# argument it is assumed to be the name of an autosetup boolean config -# which controls whether to run/skip this check. +# If -assume-for-clang is provided and $configFlag is not empty and CC +# matches *clang* and no --$configFlag was explicitly provided to the +# configure script then behave as if --$configFlag had been provided. +# To disable that assumption, either don't pass -assume-for-clang or +# pass --$configFlag=0 to the configure script. (The reason for this +# behavior is that clang supports compile-commands but some other +# compilers report false positives with these tests.) # # Returns 1 if supported, else 0, and defines HAVE_COMPILE_COMMANDS to # that value. Defines MAKE_COMPILATION_DB to "yes" if supported, "no" @@ -716,12 +727,38 @@ proc proj-file-write {args} { # HAVE_COMPILE_COMMANDS is preferred. # # ACHTUNG: this test has a long history of false positive results -# because of compilers reacting differently to the -MJ flag. -# -proc proj-check-compile-commands {{configFlag {}}} { +# because of compilers reacting differently to the -MJ flag. Because +# of this, it is recommended that this support be an opt-in feature, +# rather than an on-by-default default one. That is: in the +# configure script define the option as +# {--the-flag-name=0 => {Enable ....}} +# +proc proj-check-compile-commands {args} { + set i 0 + set configFlag {} + set fAssumeForClang 0 + set doAssume 0 msg-checking "compile_commands.json support... " - if {"" ne $configFlag && ![proj-opt-truthy $configFlag]} { - msg-result "explicitly disabled" + if {"-assume-for-clang" eq [lindex $args 0]} { + lassign $args - configFlag + incr fAssumeForClang + } elseif {1 == [llength $args]} { + lassign $args configFlag + } else { + proj-error "Invalid arguments" + } + if {1 == $fAssumeForClang && "" ne $configFlag} { + if {[string match *clang* [get-define CC]] + && ![proj-opt-was-provided $configFlag] + && ![proj-opt-truthy $configFlag]} { + proj-indented-notice [subst -nocommands -nobackslashes { + CC appears to be clang, so assuming that --$configFlag is likely + to work. To disable this assumption use --$configFlag=0.}] + incr doAssume + } + } + if {!$doAssume && "" ne $configFlag && ![proj-opt-truthy $configFlag]} { + msg-result "check disabled. Use --${configFlag} to enable it." define HAVE_COMPILE_COMMANDS 0 define MAKE_COMPILATION_DB no return 0 @@ -730,7 +767,7 @@ proc proj-check-compile-commands {{configFlag {}}} { # This test reportedly incorrectly succeeds on one of # Martin G.'s older systems. drh also reports a false # positive on an unspecified older Mac system. - msg-result "compiler supports compile_commands.json" + msg-result "compiler supports -MJ. Assuming it's useful for compile_commands.json" define MAKE_COMPILATION_DB yes; # deprecated define HAVE_COMPILE_COMMANDS 1 return 1 @@ -885,7 +922,9 @@ proc proj-looks-like-windows {{key host}} { # proc proj-looks-like-mac {{key host}} { switch -glob -- [get-define $key] { - *apple* { + *-*-darwin* { + # https://sqlite.org/forum/forumpost/7b218c3c9f207646 + # There's at least one Linux out there which matches *apple*. return 1 } default { @@ -927,17 +966,13 @@ proc proj-exe-extension {} { # proc proj-dll-extension {} { set inner {{key} { - switch -glob -- [get-define $key] { - *apple* { - return ".dylib" - } - *-*-ming* - *-*-cygwin - *-*-msys { - return ".dll" - } - default { - return ".so" - } + if {[proj-looks-like-mac $key]} { + return ".dylib" } + if {[proj-looks-like-windows $key]} { + return ".dll" + } + return ".so" }} define BUILD_DLLEXT [apply $inner build] define TARGET_DLLEXT [apply $inner host] @@ -1135,6 +1170,10 @@ proc proj-check-rpath {} { if {"" eq $wl} { set wl [proj-cc-check-Wl-flag -R$lp] } + if {"" eq $wl} { + # HP-UX: https://sqlite.org/forum/forumpost/d80ecdaddd + set wl [proj-cc-check-Wl-flag +b $lp] + } define LDFLAGS_RPATH $wl } } @@ -1144,7 +1183,7 @@ proc proj-check-rpath {} { # # @proj-check-soname ?libname? # -# Checks whether CC supports the -Wl,soname,lib... flag. If so, it +# Checks whether CC supports the -Wl,-soname,lib... flag. If so, it # returns 1 and defines LDFLAGS_SONAME_PREFIX to the flag's prefix, to # which the client would need to append "libwhatever.N". If not, it # returns 0 and defines LDFLAGS_SONAME_PREFIX to an empty string. @@ -1160,6 +1199,10 @@ proc proj-check-soname {{libname "libfoo.so.0"}} { if {[cc-check-flags "-Wl,-soname,${libname}"]} { define LDFLAGS_SONAME_PREFIX "-Wl,-soname," return 1 + } elseif {[cc-check-flags "-Wl,+h,${libname}"]} { + # HP-UX: https://sqlite.org/forum/forumpost/d80ecdaddd + define LDFLAGS_SONAME_PREFIX "-Wl,+h," + return 1 } else { define LDFLAGS_SONAME_PREFIX "" return 0 @@ -1606,7 +1649,7 @@ proc proj-tclConfig-sh-to-autosetup {tclConfigSh} { # # Similar modifications may be made for --mandir. # -# Returns 1 if it modifies the environment, else 0. +# Returns >0 if it modifies the environment, else 0. # proc proj-tweak-default-env-dirs {} { set rc 0 @@ -1645,7 +1688,11 @@ proc proj-tweak-default-env-dirs {} { # processing the file. In the context of that script, the vars # $dotInsIn and $dotInsOut will be set to the input and output file # names. This can be used, for example, to make the output file -# executable or perform validation on its contents. +# executable or perform validation on its contents: +# +## proj-dot-ins-append my.sh.in my.sh { +## catch {exec chmod u+x $dotInsOut} +## } # # See [proj-dot-ins-process], [proj-dot-ins-list] # @@ -1665,7 +1712,7 @@ proc proj-dot-ins-append {fileIn args} { proj-fatal "Too many arguments: $fileIn $args" } } - #puts "******* [proj-scope]: adding $fileIn" + #puts "******* [proj-scope]: adding [llength $fileIn]-length item: $fileIn" lappend ::proj__Config(dot-in-files) $fileIn } @@ -1703,17 +1750,18 @@ proc proj-dot-ins-list {} { # makes proj-dot-ins-append available for re-use. # proc proj-dot-ins-process {args} { - proj-parse-simple-flags args flags { + proj-parse-flags args flags { -touch "" {return "-touch"} -clear 0 {expr 1} -validate 0 {expr 1} } + #puts "args=$args"; parray flags if {[llength $args] > 0} { error "Invalid argument to [proj-scope]: $args" } foreach f $::proj__Config(dot-in-files) { proj-assert {3==[llength $f]} \ - "Expecting proj-dot-ins-list to be stored in 3-entry lists" + "Expecting proj-dot-ins-list to be stored in 3-entry lists. Got: $f" lassign $f fIn fOut fScript #puts "DOING $fIn ==> $fOut" proj-make-from-dot-in {*}$flags(-touch) $fIn $fOut @@ -1753,7 +1801,7 @@ proc proj-validate-no-unresolved-ats {args} { set isMake [string match {*[Mm]ake*} $f] foreach line [proj-file-content-list $f] { if {!$isMake || ![string match "#*" [string trimleft $line]]} { - if {[regexp {(@[A-Za-z0-9_]+@)} $line match]} { + if {[regexp {(@[A-Za-z0-9_\.]+@)} $line match]} { error "Unresolved reference to $match at line $lnno of $f" } } @@ -1893,7 +1941,7 @@ proc proj-define-amend {args} { # proc proj-define-to-cflag {args} { set rv {} - proj-parse-simple-flags args flags { + proj-parse-flags args flags { -list 0 {expr 1} -quote 0 {expr 1} -zero-undef 0 {expr 1} @@ -2001,7 +2049,7 @@ proc proj-cache-key {arg {addLevel 0}} { # See proj-cache-key for -key's and -level's semantics, noting that # this function adds one to -level for purposes of that call. proc proj-cache-set {args} { - proj-parse-simple-flags args flags { + proj-parse-flags args flags { -key => 0 -level => 0 } @@ -2037,7 +2085,7 @@ proc proj-cache-remove {{key 0} {addLevel 0}} { # See proj-cache-key for $key's and $addLevel's semantics, noting that # this function adds one to $addLevel for purposes of that call. proc proj-cache-check {args} { - proj-parse-simple-flags args flags { + proj-parse-flags args flags { -key => 0 -level => 0 } @@ -2070,147 +2118,316 @@ proc proj-coalesce {args} { } # -# @proj-parse-simple-flags ... +# @proj-parse-flags argvListName targetArrayName {prototype} # # A helper to parse flags from proc argument lists. # -# Expects a list of arguments to parse, an array name to store any -# -flag values to, and a prototype object which declares the flags. +# The first argument is the name of a var holding the args to +# parse. It will be overwritten, possibly with a smaller list. # -# The prototype must be a list in one of the following forms: +# The second argument is the name of an array variable to create in +# the caller's scope. # -# -flag defaultValue {script} +# The third argument, $prototype, is a description of how to handle +# the flags. Each entry in that list must be in one of the +# following forms: # -# -flag => defaultValue -# -----^--^ (with spaces there!) +# -flag defaultValue ?-literal|-call|-apply? +# script|number|incr|proc-name|{apply $aLambda} # -# Repeated for each flag. +# -flag* ...as above... # -# The first form represents a basic flag with no associated -# following argument. The second form extracts its value -# from the following argument in $argvName. +# -flag => defaultValue ?-call proc-name-and-args|-apply lambdaExpr? # -# The first argument to this function is the name of a var holding the -# args to parse. It will be overwritten, possibly with a smaller list. +# -flag* => ...as above... # -# The second argument the name of an array variable to create in the -# caller's scope. (Pneumonic: => points to the next argument.) +# :PRAGMA # -# For the first form of flag, $script is run in the caller's scope if -# $argv contains -flag, and the result of that script is the new value -# for $tgtArrayName(-flag). This function intercepts [return $val] -# from $script. Any empty script will result in the flag having "" -# assigned to it. +# The first two forms represents a basic flag with no associated +# following argument. The third and fourth forms, called arg-consuming +# flags, extract the value from the following argument in $argvName +# (pneumonic: => points to the next argument.). The :PRAGMA form +# offers a way to configure certain aspects of this call. # -# The args list is only inspected until the first argument which is -# not described by $prototype. i.e. the first "non-flag" (not counting -# values consumed for flags defined like --flag=>default). +# If $argv contains any given flag from $prototype, its default value +# is overridden depending on several factors: # -# If a "--" flag is encountered, no more arguments are inspected as -# flags. If "--" is the first non-flag argument, the "--" flag is -# removed from the results but all remaining arguments are passed -# through. If "--" appears after the first non-flag, it is retained. +# - If the -literal flag is used, or the flag's script is a number, +# value is used verbatim. +# +# - Else if the -call flag is used, the argument must be a proc name +# and any leading arguments, e.g. {apply $myLambda}. The proc is passed +# the (flag, value) as arguments (non-consuming flags will get +# passed the flag's current/starting value and consuming flags will +# get the next argument). Its result becomes the result of the +# flag. +# +# - Else if -apply X is used, it's effectively shorthand for -call +# {apply X}. Its argument may either be a $lambaRef or a {{f v} +# {body}} construct. +# +# - Else if $script is one of the following values, it is treated as +# the result of... +# +# - incr: increments the current value of the flag. +# +# - Else $script is eval'd to get its result value. That result +# becomes the new flag value for $tgtArrayName(-flag). This +# function intercepts [return $val] from eval'ing $script. Any +# empty script will result in the flag having "" assigned to it. +# +# Unless the -flag has a trailing asterisk, e.g. -flag*, this function +# assumes that each flag is unique, and using a flag more than once +# causes an error to be triggered. the -flag* forms works similarly +# except that may appear in $argv any number of times: +# +# - For non-arg-consuming flags, each invocation of -flag causes the +# result of $script to overwrite the previous value. e.g. so +# {-flag* {x} {incr foo}} has a default value of x, but passing in +# -flag twice would change it to the result of incrementing foo +# twice. This form can be used to implement, e.g., increasing +# verbosity levels by passing -verbose multiple times. +# +# - For arg-consuming flags, the given flag starts with value X, but +# if the flag is provided in $argv, the default is cleared, then +# each instance of -flag causes its value to be appended to the +# result, so {-flag* => {a b c}} defaults to {a b c}, but passing +# in -flag y -flag z would change it to {y z}, not {a b c y z}.. +# +# By default, the args list is only inspected until the first argument +# which is not described by $prototype. i.e. the first "non-flag" (not +# counting values consumed for flags defined like -flag => default). +# The :all-flags pragma (see below) can modify this behavior. # -# This function assumes that each flag is unique, and using a flag -# more than once behaves in a last-one-wins fashion. +# If a "--" flag is encountered, no more arguments are inspected as +# flags unless the :all-flags pragma (see below) is in effect. The +# first instance of "--" is removed from the target result list but +# all remaining instances of "--" are are passed through. # -# Any argvName entries not described in $prototype are not treated as -# flags. +# Any argvName entries not described in $prototype are considered to +# be "non-flags" for purposes of this function, even if they +# ostensibly look like flags. # -# Returns the number of flags it processed in $argvName. +# Returns the number of flags it processed in $argvName, not counting +# "--". # # Example: # -# set args [list -foo -bar {blah} 8 9 10 -theEnd] -# proj-parse-simple-flags args flags { -# -foo 0 {expr 1} -# -bar => 0 -# -no-baz 2 {return 0} -# } +## set args [list -foo -bar {blah} -z 8 9 10 -theEnd] +## proj-parse-flags args flags { +## -foo 0 {expr 1} +## -bar => 0 +## -no-baz 1 {return 0} +## -z 0 2 +## } # -# After that $flags would contain {-foo 1 -bar {blah} -no-baz 2} +# After that $flags would contain {-foo 1 -bar {blah} -no-baz 1 -z 2} # and $args would be {8 9 10 -theEnd}. # -# Potential TODOs: consider using lappend instead of set so that any -# given flag can be used more than once. Or add a syntax to indicate -# that multiples are allowed. Also consider searching the whole -# argv list, rather than stopping at the first non-flag +# Pragmas: +# +# Passing :PRAGMAS to this function may modify how it works. The +# following pragmas are supported (note the leading ":"): +# +# :all-flags indicates that the whole input list should be scanned, +# not stopping at the first non-flag or "--". # -proc proj-parse-simple-flags {argvName tgtArrayName prototype} { +proc proj-parse-flags {argvName tgtArrayName prototype} { upvar $argvName argv - upvar $tgtArrayName tgt - array set dflt {} - array set scripts {} - array set consuming {} + upvar $tgtArrayName outFlags + array set flags {}; # staging area + array set blob {}; # holds markers for various per-key state and options + set incrSkip 1; # 1 if we stop at the first non-flag, else 0 + # Parse $prototype for flag definitions... set n [llength $prototype] - # Figure out what our flags are... + set checkProtoFlag { + #puts "**** checkProtoFlag #$i of $n k=$k fv=$fv" + switch -exact -- $fv { + -literal { + proj-assert {![info exists blob(${k}.consumes)]} + set blob(${k}.script) [list expr [lindex $prototype [incr i]]] + } + -apply { + set fv [lindex $prototype [incr i]] + if {2 == [llength $fv]} { + # Treat this as a lambda literal + set fv [list $fv] + } + lappend blob(${k}.call) "apply $fv" + } + -call { + # arg is either a proc name or {apply $aLambda} + set fv [lindex $prototype [incr i]] + lappend blob(${k}.call) $fv + } + default { + proj-assert {![info exists blob(${k}.consumes)]} + set blob(${k}.script) $fv + } + } + if {$i >= $n} { + proj-error -up "[proj-scope]: Missing argument for $k flag" + } + } for {set i 0} {$i < $n} {incr i} { set k [lindex $prototype $i] #puts "**** #$i of $n k=$k" + + # Check for :PRAGMA... + switch -exact -- $k { + :all-flags { + set incrSkip 0 + continue + } + } + proj-assert {[string match -* $k]} \ - "Invalid flag value: $k" - set v "" - set s "" + "Invalid argument: $k" + + if {[string match {*\*} $k]} { + # Re-map -foo* to -foo and flag -foo as a repeatable flag + set k [string map {* ""} $k] + incr blob(${k}.multi) + } + + if {[info exists flags($k)]} { + proj-error -up "[proj-scope]: Duplicated prototype for flag $k" + } + switch -exact -- [lindex $prototype [expr {$i + 1}]] { => { + # -flag => DFLT ?-subflag arg? incr i 2 if {$i >= $n} { - proj-error "Missing argument for $k => flag" + proj-error -up "[proj-scope]: Missing argument for $k => flag" + } + incr blob(${k}.consumes) + set vi [lindex $prototype $i] + if {$vi in {-apply -call}} { + proj-error -up "[proj-scope]: Missing default value for $k flag" + } else { + set fv [lindex $prototype [expr {$i + 1}]] + if {$fv in {-apply -call}} { + incr i + eval $checkProtoFlag + } } - set consuming($k) 1 - set v [lindex $prototype $i] } default { - set v [lindex $prototype [incr i]] - set s [lindex $prototype [incr i]] - set scripts($k) $s + # -flag VALUE ?flag? SCRIPT + set vi [lindex $prototype [incr i]] + set fv [lindex $prototype [incr i]] + eval $checkProtoFlag } } - #puts "**** #$i of $n k=$k v=$v s=$s" - set dflt($k) $v + #puts "**** #$i of $n k=$k vi=$vi" + set flags($k) $vi } - # Now look for those flags in the source list - array set tgt [array get dflt] - unset dflt + #puts "-- flags"; parray flags + #puts "-- blob"; parray blob set rc 0 - set rv {} + set rv {}; # staging area for the target argv value set skipMode 0 set n [llength $argv] + # Now look for those flags in $argv... for {set i 0} {$i < $n} {incr i} { set arg [lindex $argv $i] + #puts "-- [proj-scope] arg=$arg" if {$skipMode} { lappend rv $arg } elseif {"--" eq $arg} { - incr skipMode - } elseif {[info exists tgt($arg)]} { - if {[info exists consuming($arg)]} { - if {$i + 1 >= $n} { - proj-assert 0 {Cannot happen - bounds already checked} + # "--" is the conventional way to end processing of args + if {[incr blob(--)] > 1} { + # Elide only the first one + lappend rv $arg + } + incr skipMode $incrSkip + } elseif {[info exists flags($arg)]} { + # A known flag... + set isMulti [info exists blob(${arg}.multi)] + incr blob(${arg}.seen) + if {1 < $blob(${arg}.seen) && !$isMulti} { + proj-error -up [proj-scope] "$arg flag was used multiple times" + } + set vMode 0; # 0=as-is, 1=eval, 2=call + set isConsuming [info exists blob(${arg}.consumes)] + if {$isConsuming} { + incr i + if {$i >= $n} { + proj-error -up [proj-scope] "is missing argument for $arg flag" + } + set vv [lindex $argv $i] + } elseif {[info exists blob(${arg}.script)]} { + set vMode 1 + set vv $blob(${arg}.script) + } else { + set vv $flags($arg) + } + + if {[info exists blob(${arg}.call)]} { + set vMode 2 + set vv [concat {*}$blob(${arg}.call) $arg $vv] + } elseif {$isConsuming} { + proj-assert {!$vMode} + # fall through + } elseif {"" eq $vv || [string is double -strict $vv]} { + set vMode 0 + } elseif {$vv in {incr}} { + set vMode 0 + switch -exact $vv { + incr { + set xx $flags($k); incr xx; set vv $xx; unset xx + } + default { + proj-error "Unhandled \$vv value $vv" + } } - set tgt($arg) [lindex $argv [incr i]] - } elseif {"" eq $scripts($arg)} { - set tgt($arg) "" } else { - #puts "**** running scripts($arg) $scripts($arg)" - set code [catch {uplevel 1 $scripts($arg)} xrc xopt] - #puts "**** tgt($arg)=$scripts($arg) code=$code rc=$rc" - if {$code in {0 2}} { - set tgt($arg) $xrc + set vv [list eval $vv] + set vMode 1 + } + if {$vMode} { + set code [catch [list uplevel 1 $vv] vv xopt] + if {$code ni {0 2}} { + return {*}$xopt $vv + } + } + if {$isConsuming && $isMulti} { + if {1 == $blob(${arg}.seen)} { + # On the first hit, overwrite the default with a new list. + set flags($arg) [list $vv] } else { - return {*}$xopt $xrc + # On subsequent hits, append to the list. + lappend flags($arg) $vv } + } else { + set flags($arg) $vv } incr rc } else { - incr skipMode + # Non-flag + incr skipMode $incrSkip lappend rv $arg } } set argv $rv + array set outFlags [array get flags] + #puts "-- rv=$rv argv=$argv flags="; parray flags return $rc +}; # proj-parse-flags + +# +# Older (deprecated) name of proj-parse-flags. +# +proc proj-parse-simple-flags {args} { + tailcall proj-parse-flags {*}$args } if {$::proj__Config(self-tests)} { + set __ova $::proj__Config(verbose-assert); + set ::proj__Config(verbose-assert) 1 + puts "Running [info script] self-tests..." + # proj-cache... apply {{} { #proj-warn "Test code for proj-cache" proj-assert {![proj-cache-check -key here check]} @@ -2233,4 +2450,100 @@ if {$::proj__Config(self-tests)} { proj-assert {"" eq [proj-cache-remove]} proj-assert {"" eq $check} }} -} + + # proj-parse-flags ... + apply {{} { + set foo 3 + set argv {-a "hi - world" -b -b -b -- -a {bye bye} -- -d -D c -a "" --} + proj-parse-flags argv flags { + :all-flags + -a* => "gets overwritten" + -b* 7 {incr foo} + -d 1 0 + -D 0 1 + } + + #puts "-- argv = $argv"; parray flags; + proj-assert {"-- c --" eq $argv} + proj-assert {$flags(-a) eq "{hi - world} {bye bye} {}"} + proj-assert {$foo == 6} + proj-assert {$flags(-b) eq $foo} + proj-assert {$flags(-d) == 0} + proj-assert {$flags(-D) == 1} + set foo 0 + foreach x $flags(-a) { + proj-assert {$x in {{hi - world} {bye bye} {}}} + incr foo + } + proj-assert {3 == $foo} + + set argv {-a {hi world} -b -maybe -- -a {bye bye} -- -b c --} + set foo 0 + proj-parse-flags argv flags { + -a => "aaa" + -b 0 {incr foo} + -maybe no -literal yes + } + #parray flags; puts "--- argv = $argv" + proj-assert {"-a {bye bye} -- -b c --" eq $argv} + proj-assert {$flags(-a) eq "hi world"} + proj-assert {1 == $flags(-b)} + proj-assert {"yes" eq $flags(-maybe)} + + set argv {-f -g -a aaa -M -M -M -L -H -A AAA a b c} + set foo 0 + set myLambda {{flag val} { + proj-assert {$flag in {-f -g -M}} + #puts "myLambda flag=$flag val=$val" + incr val + }} + proc myNonLambda {flag val} { + proj-assert {$flag in {-A -a}} + #puts "myNonLambda flag=$flag val=$val" + concat $val $val + } + proj-parse-flags argv flags { + -f 0 -call {apply $myLambda} + -g 2 -apply $myLambda + -h 3 -apply $myLambda + -H 30 33 + -a => aAAAa -apply {{f v} { + set v + }} + -A => AaaaA -call myNonLambda + -B => 17 -call myNonLambda + -M* 0 -apply $myLambda + -L "" -literal $myLambda + } + rename myNonLambda "" + #puts "--- argv = $argv"; parray flags + proj-assert {$flags(-f) == 1} + proj-assert {$flags(-g) == 3} + proj-assert {$flags(-h) == 3} + proj-assert {$flags(-H) == 33} + proj-assert {$flags(-a) == {aaa}} + proj-assert {$flags(-A) eq "AAA AAA"} + proj-assert {$flags(-B) == 17} + proj-assert {$flags(-M) == 3} + proj-assert {$flags(-L) eq $myLambda} + + set argv {-touch -validate} + proj-parse-flags argv flags { + -touch "" {return "-touch"} + -validate 0 1 + } + #puts "----- argv = $argv"; parray flags + proj-assert {$flags(-touch) eq "-touch"} + proj-assert {$flags(-validate) == 1} + proj-assert {$argv eq {}} + + set argv {-i -i -i} + proj-parse-flags argv flags { + -i* 0 incr + } + proj-assert {3 == $flags(-i)} + }} + set ::proj__Config(verbose-assert) $__ova + unset __ova + puts "Done running [info script] self-tests." +}; # proj- API self-tests diff --git a/autosetup/sqlite-config.tcl b/autosetup/sqlite-config.tcl index 85fe41438..7c798b31a 100644 --- a/autosetup/sqlite-config.tcl +++ b/autosetup/sqlite-config.tcl @@ -1,7 +1,7 @@ # This file holds functions for autosetup which are specific to the # sqlite build tree. They are in this file, instead of auto.def, so # that they can be reused in the autoconf sub-tree. This file requires -# functions from proj.tcl. +# functions from the project-agnostic proj.tcl. if {[string first " " $autosetup(srcdir)] != -1} { user-error "The pathname of the source tree\ @@ -11,7 +11,7 @@ if {[string first " " $autosetup(builddir)] != -1} { user-error "The pathname of the build directory\ may not contain space characters" } -#parray ::autosetup; exit 0 + use proj # # We want the package version info to be emitted early on, but doing @@ -65,11 +65,12 @@ array set sqliteConfig [subst [proj-strip-hash-comments { # The list of feature --flags which the --all flag implies. This # requires special handling in a few places. # - all-flag-enables {fts4 fts5 rtree geopoly session} + all-flag-enables {fts4 fts5 rtree geopoly session dbpage dbstat carray} # # Default value for the --all flag. Can hypothetically be modified - # by non-canonical builds. + # by non-canonical builds (it was added for a Tcl extension build + # mode which was eventually removed). # all-flag-default 0 }]] @@ -92,7 +93,7 @@ array set sqliteConfig [subst [proj-strip-hash-comments { # sqlite-configure BUILD_NAME { build-specific configure script } # # There are snippets of build-mode-specific decision-making in -# [sqlite-configure-finalize] +# [sqlite-configure-finalize], which gets run after $configScript. proc sqlite-configure {buildMode configScript} { proj-assert {$::sqliteConfig(build-mode) eq "unknown"} \ "sqlite-configure must not be called more than once" @@ -112,8 +113,10 @@ proc sqlite-configure {buildMode configScript} { # # Reference: https://msteveb.github.io/autosetup/developer/ # - # All configure flags must be described in an 'options' call. The - # general syntax is: + # All configure flags must be described in one or more calls to + # autosetup's [options] and [options-add] functions. The general + # syntax of the single argument to those functions is a list contain + # a mapping of flags to help text: # # FLAG => {Help text} # @@ -164,13 +167,18 @@ proc sqlite-configure {buildMode configScript} { ######################################################################## set allFlags { # Structure: a list of M {Z} pairs, where M is a descriptive - # option group name and Z is a list of X Y pairs. X is a list of + # option group name and Z is a list of X Y pairs. X is a list of # $buildMode name(s) to which the Y flags apply, or {*} to apply # to all builds. Y is a {block} in the form expected by - # autosetup's [options] command. Each block which is applicable - # to $buildMode is appended to a new list before that list is - # passed on to [options]. The order of each Y and sub-Y is - # retained, which is significant for rendering of --help. + # autosetup's [options] and [options-add] command. Each block + # which is applicable to $buildMode is passed on to + # [options-add]. The order of each Y and sub-Y is retained, which + # is significant for rendering of --help. + # + # Maintenance note: [options] does not support comments in + # options, but we filter this object through + # [proj-strip-hash-comments] to remove them before passing them on + # to [options]. # When writing {help text blocks}, be aware that: # @@ -180,7 +188,7 @@ proc sqlite-configure {buildMode configScript} { # pretty-printed. # # B) Vars and commands are NOT expanded, but we use a [subst] call - # below which will replace (only) var refs. + # below which will replace (only) $var refs. # Options for how to build the library build-modes { @@ -212,11 +220,19 @@ proc sqlite-configure {buildMode configScript} { geopoly => {Enable the GEOPOLY extension} rtree => {Enable the RTREE extension} session => {Enable the SESSION extension} + dbpage => {Enable the sqlite3_dbpage extension} + dbstat => {Enable the sqlite3_dbstat extension} + carray=1 => {Disable the CARRAY extension} all=$::sqliteConfig(all-flag-default) => {$allFlagHelp} largefile=1 => {This legacy flag has no effect on the library but may influence the generated sqlite_cfg.h by adding #define HAVE_LFS} } + {canonical} { + column-metadata => {Enable the column metadata APIs} + # ^^^ Affects how sqlite3.c is generated, so is not available in + # the autoconf build. + } } # Options for TCL support @@ -227,8 +243,6 @@ proc sqlite-configure {buildMode configScript} { This tree requires TCL for code generation but can use the in-tree copy of autosetup/jimsh0.c for that. The SQLite TCL extension and the test code require a canonical tclsh.} - } - {canonical} { with-tcl:DIR => {Directory containing tclConfig.sh or a directory one level up from that, from which we can derive a directory containing tclConfig.sh. @@ -236,11 +250,10 @@ proc sqlite-configure {buildMode configScript} { the --prefix flag.} with-tclsh:PATH => {Full pathname of tclsh to use. It is used for (A) trying to find - tclConfig.sh and (B) all TCL-based code generation. Warning: if - its containing dir has multiple tclsh versions, it may select the + tclConfig.sh and (B) all TCL-based code generation. Use --with-tcl + unless you have a specific need for this flag. Warning: if its + containing dir has multiple tclsh versions, it may select the wrong tclConfig.sh!} - } - {canonical} { static-tclsqlite3=0 => {Statically-link tclsqlite3. This only works if TCL support is enabled and all requisite libraries are available in @@ -319,7 +332,7 @@ proc sqlite-configure {buildMode configScript} { Needed only by ext/wasm. Default=EMSDK env var.} amalgamation-extra-src:FILES - => {Space-separated list of soure files to append as-is to the resulting + => {Space-separated list of source files to append as-is to the resulting sqlite3.c amalgamation file. May be provided multiple times.} } } @@ -334,8 +347,7 @@ proc sqlite-configure {buildMode configScript} { => {Link the sqlite3 shell app against the DLL instead of embedding sqlite3.c} } {canonical autoconf} { - # A potential TODO without a current use case: - #rpath=1 => {Disable use of the rpath linker flag} + rpath=1 => {Disable use of the rpath linker flag} # soname: https://sqlite.org/src/forumpost/5a3b44f510df8ded soname:=legacy => {SONAME for libsqlite3.so. "none", or not using this flag, sets no @@ -406,7 +418,6 @@ proc sqlite-configure {buildMode configScript} { # ^^^ lappend of [sqlite-custom-flags] introduces weirdness if # we delay [proj-strip-hash-comments] until after that. - ######################################################################## # sqlite-custom.tcl is intended only for vendor-branch-specific # customization. See autosetup/README.md#branch-customization for @@ -424,6 +435,8 @@ proc sqlite-configure {buildMode configScript} { } } + #lappend allFlags just-testing {{*} {soname:=duplicateEntry => {x}}} + # Filter allFlags to create the set of [options] legal for this build foreach {group XY} [subst -nobackslashes -nocommands $allFlags] { foreach {X Y} $XY { @@ -432,7 +445,7 @@ proc sqlite-configure {buildMode configScript} { } } } - #lappend opts "soname:=duplicateEntry => {x}"; #just testing + if {[catch {options {}} msg xopts]} { # Workaround for # where [options] behaves oddly on _some_ TCL builds when it's @@ -447,8 +460,9 @@ proc sqlite-configure {buildMode configScript} { ######################################################################## # Runs "phase 1" of the configure process: after initial --flags -# handling but before the build-specific parts are run. $buildMode -# must be the mode which was passed to [sqlite-configure]. +# handling but before sqlite-configure's $configScript argument is +# run. $buildMode must be the mode which was passed to +# [sqlite-configure]. proc sqlite-configure-phase1 {buildMode} { define PACKAGE_NAME sqlite define PACKAGE_URL {https://sqlite.org} @@ -490,6 +504,7 @@ proc sqlite-configure-phase1 {buildMode} { if {[file exists $srcdir/sqlite3.pc.in]} { proj-dot-ins-append $srcdir/sqlite3.pc.in } + sqlite-handle-hpux; # must be relatively early so that other config tests can work }; # sqlite-configure-phase1 ######################################################################## @@ -616,7 +631,7 @@ proc sqlite-check-common-system-deps {} { # Check for needed/wanted functions cc-check-functions gmtime_r isnan localtime_r localtime_s \ - strchrnul usleep utime pread pread64 pwrite pwrite64 + usleep utime pread pread64 pwrite pwrite64 apply {{} { set ldrt "" @@ -772,7 +787,11 @@ proc sqlite-handle-common-feature-flags {} { sqlite-add-feature-flag -DSQLITE_ENABLE_MEMSYS3 } } - scanstatus -DSQLITE_ENABLE_STMT_SCANSTATUS {} + scanstatus -DSQLITE_ENABLE_STMT_SCANSTATUS {} + column-metadata -DSQLITE_ENABLE_COLUMN_METADATA {} + dbpage -DSQLITE_ENABLE_DBPAGE_VTAB {} + dbstat -DSQLITE_ENABLE_DBSTAT_VTAB {} + carray -DSQLITE_ENABLE_CARRAY {} }] { if {$boolFlag ni $::autosetup(options)} { # Skip flags which are in the canonical build but not @@ -1014,7 +1033,7 @@ proc sqlite-handle-emsdk {} { proc sqlite-get-readline-dir-list {} { # Historical note: the dirs list, except for the inclusion of # $prefix and some platform-specific dirs, originates from the - # legacy configure script + # legacy configure script. set dirs [list [get-define prefix]] switch -glob -- [get-define host] { *-linux-android { @@ -1032,7 +1051,7 @@ proc sqlite-get-readline-dir-list {} { if {[opt-val with-readline-ldflags] in {auto ""}} { # If the user did not supply their own --with-readline-ldflags # value, hijack that flag to inject options which are known to - # work on a default Haiku installation. + # work on Haiku OS installations. if {"" ne [glob -nocomplain /boot/system/lib/libreadline*]} { proj-opt-set with-readline-ldflags {-L/boot/system/lib -lreadline} } @@ -1082,8 +1101,8 @@ proc sqlite-get-readline-dir-list {} { # 4) Default to automatic search for optional readline # # 5) Try to find readline or editline. If it's not found AND the -# corresponding --FEATURE flag was explicitly given, fail fatally, -# else fail silently. +# corresponding --FEATURE flag was explicitly given then fail +# fatally, else fail non-fatally. proc sqlite-check-line-editing {} { msg-result "Checking for line-editing capability..." define HAVE_READLINE 0 @@ -1096,7 +1115,7 @@ proc sqlite-check-line-editing {} { # if the library is not found. set libsForReadline {readline edit} ; # -l names to check for readline(). # The libedit check changes this. - set editLibName "readline" ; # "readline" or "editline" + set editLibName "readline" ; # "readline" or "editline" set editLibDef "HAVE_READLINE" ; # "HAVE_READLINE" or "HAVE_EDITLINE" set dirLn [opt-val with-linenoise] if {"" ne $dirLn} { @@ -1113,7 +1132,7 @@ proc sqlite-check-line-editing {} { foreach f $lnCOpts { if {[file exists $dirLn/$f]} { set lnC $dirLn/$f - break; + break } } if {"" eq $lnC} { @@ -1223,16 +1242,18 @@ proc sqlite-check-line-editing {} { set rlLib [opt-val with-readline-ldflags] #proc-debug "rlLib=$rlLib" if {$rlLib in {auto ""}} { - set rlLib "" - set libTerm "" - if {[proj-check-function-in-lib tgetent "$editLibName ncurses curses termcap"]} { + set rlLib "" ; # make sure it's not "auto", as we may append to it below + set libTerm ""; # lib with tgetent(3) + if {[proj-check-function-in-lib tgetent [list $editLibName ncurses curses termcap]]} { # ^^^ that libs list comes from the legacy configure script ^^^ set libTerm [get-define lib_tgetent] undefine lib_tgetent } if {$editLibName eq $libTerm} { + # tgetent(3) was found in the editing library set rlLib $libTerm } elseif {[proj-check-function-in-lib readline $libsForReadline $libTerm]} { + # tgetent(3) was found in an external lib set rlLib [get-define lib_readline] lappend rlLib $libTerm undefine lib_readline @@ -1262,8 +1283,8 @@ proc sqlite-check-line-editing {} { msg-result "Using $editLibName flags: $rlInc $rlLib" # Check whether rl_completion_matches() has a signature we can use # and disable that sub-feature if it doesn't. - if {![cctest \ - -cflags "$rlInc -D${editLibDef}" -libs $rlLib -nooutput 1 -source { + if {![cctest -cflags "$rlInc -D${editLibDef}" -libs $rlLib -nooutput 1 \ + -source { #include #ifdef HAVE_EDITLINE #include @@ -1316,7 +1337,7 @@ proc sqlite-handle-line-editing {} { # - pkg-config: use only pkg-config to determine flags # - /path/to/icu-config: use that to determine flags # -# If --with-icu-config is used as neither pkg-config nor icu-config +# If --with-icu-config is used and neither pkg-config nor icu-config # are found, fail fatally. # # If both --with-icu-ldflags and --with-icu-config are provided, they @@ -1409,39 +1430,50 @@ proc sqlite-handle-icu {} { # Makes the following environment changes: # # - defines LDFLAGS_DLOPEN to any linker flags needed for this -# feature. It may legally be empty on some systems where dlopen() -# is in libc. +# feature. It may legally be empty on (A) some systems where +# dlopen() is in libc and (B) certain Unix-esque Windows +# environments which identify as Windows for SQLite's purposes so +# use LoadLibrary(). # # - If the feature is not available, adds # -DSQLITE_OMIT_LOAD_EXTENSION=1 to the feature flags list. proc sqlite-handle-load-extension {} { define LDFLAGS_DLOPEN "" set found 0 + set suffix "" proj-if-opt-truthy load-extension { - set found [proj-check-function-in-lib dlopen dl] - if {$found} { - define LDFLAGS_DLOPEN [get-define lib_dlopen] - undefine lib_dlopen - } else { - if {[proj-opt-was-provided load-extension]} { - # Explicit --enable-load-extension: fail if not found - proj-indented-notice -error { - --enable-load-extension was provided but dlopen() - not found. Use --disable-load-extension to bypass this - check. - } - } else { - # It was implicitly enabled: warn if not found - proj-indented-notice { - WARNING: dlopen() not found, so loadable module support will - be disabled. Use --disable-load-extension to bypass this - check. + switch -glob -- [get-define host] { + *-*-mingw* - *windows* { + incr found + set suffix "Using LoadLibrary()" + } + default { + set found [proj-check-function-in-lib dlopen dl] + if {$found} { + set suffix [define LDFLAGS_DLOPEN [get-define lib_dlopen]] + undefine lib_dlopen + } else { + if {[proj-opt-was-provided load-extension]} { + # Explicit --enable-load-extension: fail if not found + proj-indented-notice -error { + --enable-load-extension was provided but dlopen() + not found. Use --disable-load-extension to bypass this + check. + } + } else { + # It was implicitly enabled: warn if not found + proj-indented-notice { + WARNING: dlopen() not found, so loadable module support will + be disabled. Use --disable-load-extension to bypass this + check. + } + } } } } } if {$found} { - msg-result "Loadable extension support enabled." + msg-result "Loadable extension support enabled. $suffix" } else { msg-result "Disabling loadable extension support. Use --enable-load-extension to enable them." sqlite-add-feature-flag -DSQLITE_OMIT_LOAD_EXTENSION=1 @@ -1458,7 +1490,7 @@ proc sqlite-handle-math {} { } define LDFLAGS_MATH [get-define lib_ceil] undefine lib_ceil - sqlite-add-feature-flag -DSQLITE_ENABLE_MATH_FUNCTIONS + sqlite-add-feature-flag -DSQLITE_ENABLE_MATH_FUNCTIONS -DSQLITE_ENABLE_PERCENTILE msg-result "Enabling math SQL functions" } { define LDFLAGS_MATH "" @@ -1520,6 +1552,19 @@ proc sqlite-handle-mac-install-name {} { return $rc } +# +# Checks specific to HP-UX. +# +proc sqlite-handle-hpux {} { + switch -glob -- [get-define host] { + *hpux* { + if {[cc-check-flags "-Ae"]} { + define-append CFLAGS -Ae + } + } + } +} + ######################################################################## # Handles the --dll-basename configure flag. [define]'s # SQLITE_DLL_BASENAME to the DLL's preferred base name (minus @@ -1969,13 +2014,14 @@ proc sqlite-check-tcl {} { # TCLLIBDIR from here, which will cause the canonical makefile to # use this one rather than to re-calculate it at make-time. set tcllibdir [get-env TCLLIBDIR ""] + set sq3Ver [get-define PACKAGE_VERSION] if {"" eq $tcllibdir} { # Attempt to extract TCLLIBDIR from TCL's $auto_path if {"" ne $with_tclsh && [catch {exec echo "puts stdout \$auto_path" | "$with_tclsh"} result] == 0} { foreach i $result { if {[file isdir $i]} { - set tcllibdir $i/sqlite3 + set tcllibdir $i/sqlite${sq3Ver} break } } @@ -2111,15 +2157,31 @@ proc sqlite-determine-codegen-tcl {} { # sqlite-determine-codegen-tcl. proc sqlite-handle-tcl {} { sqlite-check-tcl - if {"canonical" eq $::sqliteConfig(build-mode)} { - msg-result "TCL for code generation: [sqlite-determine-codegen-tcl]" + if {"canonical" ne $::sqliteConfig(build-mode)} return + msg-result "TCL for code generation: [sqlite-determine-codegen-tcl]" + + # Determine the base name of the Tcl extension's DLL + # + if {[get-define HAVE_TCL]} { + if {[string match *-cygwin [get-define host]]} { + set libname cyg + } else { + set libname lib + } + if {[get-define TCL_MAJOR_VERSION] > 8} { + append libname tcl9 + } + append libname sqlite + } else { + set libname "" } + define TCL_EXT_DLL_BASENAME $libname + # The extension is added in the makefile } ######################################################################## # Handle the --enable/disable-rpath flag. proc sqlite-handle-rpath {} { - proj-check-rpath # autosetup/cc-shared.tcl sets the rpath flag definition in # [get-define SH_LINKRPATH], but it does so on a per-platform basis # rather than as a compiler check. Though we should do a proper @@ -2128,12 +2190,13 @@ proc sqlite-handle-rpath {} { # for which sqlite-env-is-unix-on-windows returns a non-empty # string. -# if {[proj-opt-truthy rpath]} { -# proj-check-rpath -# } else { -# msg-result "Disabling use of rpath." -# define LDFLAGS_RPATH "" -# } + # https://sqlite.org/forum/forumpost/13cac3b56516f849 + if {[proj-opt-truthy rpath]} { + proj-check-rpath + } else { + msg-result "Disabling use of rpath." + define LDFLAGS_RPATH "" + } } ######################################################################## diff --git a/autosetup/teaish/core.tcl b/autosetup/teaish/core.tcl index 09017029d..a4a6b001f 100644 --- a/autosetup/teaish/core.tcl +++ b/autosetup/teaish/core.tcl @@ -92,6 +92,7 @@ array set teaish__Config [proj-strip-hash-comments { -tm.tcl.in TEAISH_TM_TCL_IN -options {} -pragmas {} + -src {} } # @@ -331,29 +332,33 @@ proc teaish-configure-core {} { -url - -v "" -tm.tcl - -v "" -tm.tcl.in - -v "" + -src - -v "" } { + #proj-assert 0 {Just testing} set isPIFlag [expr {"-" ne $pflag}] if {$isPIFlag} { if {[info exists ::teaish__PkgInfo($pflag)]} { # Was already set - skip it. continue; } - proj-assert {{-} eq $key} + proj-assert {{-} eq $key};# "Unexpected pflag=$pflag key=$key type=$type val=$val" set key $f2d($pflag) } - proj-assert {"" ne $key} - set got [get-define $key ""] - if {"" ne $got} { - # Was already set - skip it. - continue + if {"" ne $key} { + if {"" ne [get-define $key ""]} { + # Was already set - skip it. + continue + } } switch -exact -- $type { -v {} -e { set val [eval $val] } default { proj-error "Invalid type flag: $type" } } - #puts "***** defining default $pflag $key {$val} isPIFlag=$isPIFlag got=$got" - define $key $val + #puts "***** defining default $pflag $key {$val} isPIFlag=$isPIFlag" + if {$key ne ""} { + define $key $val + } if {$isPIFlag} { set ::teaish__PkgInfo($pflag) $val } @@ -541,10 +546,10 @@ proc teaish__configure_phase1 {} { define TEAISH_VSATISFIES_CODE [join $code "\n"] }}; # vsatisfies - if {[proj-looks-like-windows] || [proj-looks-like-mac]} { + if {[proj-looks-like-windows]} { # Without this, linking of an extension will not work on Cygwin or # Msys2. - msg-result "Using USE_TCL_STUBS for this environment" + msg-result "Using USE_TCL_STUBS for Unix(ish)-on-Windows environment" teaish-cflags-add -DUSE_TCL_STUBS=1 } @@ -585,7 +590,8 @@ proc teaish__configure_phase1 {} { # if {0x0f & $::teaish__Config(pkginit-policy)} { file delete -force -- [get-define TEAISH_PKGINIT_TCL] - proj-dot-ins-append [get-define TEAISH_PKGINIT_TCL_IN] + proj-dot-ins-append [get-define TEAISH_PKGINIT_TCL_IN] \ + [get-define TEAISH_PKGINIT_TCL] } if {0x0f & $::teaish__Config(tm-policy)} { file delete -force -- [get-define TEAISH_TM_TCL] @@ -595,17 +601,20 @@ proc teaish__configure_phase1 {} { apply {{} { # Queue up any remaining dot-in files set dotIns [list] - foreach d { - TEAISH_TESTER_TCL_IN - TEAISH_TEST_TCL_IN - TEAISH_MAKEFILE_IN + foreach {dIn => dOut} { + TEAISH_TESTER_TCL_IN => TEAISH_TESTER_TCL + TEAISH_TEST_TCL_IN => TEAISH_TEST_TCL + TEAISH_MAKEFILE_IN => TEAISH_MAKEFILE } { - lappend dotIns [get-define $d ""] - } - lappend dotIns $::autosetup(srcdir)/Makefile.in; # must be after TEAISH_MAKEFILE_IN - foreach f $dotIns { - if {"" ne $f} { - proj-dot-ins-append $f + lappend dotIns [get-define $dIn ""] [get-define $dOut ""] + } + lappend dotIns $::autosetup(srcdir)/Makefile.in Makefile; # must be after TEAISH_MAKEFILE_IN. + # Much later: probably because of timestamps for deps purposes :-? + #puts "dotIns=$dotIns" + foreach {i o} $dotIns { + if {"" ne $i && "" ne $o} { + #puts " pre-dot-ins-append: \[$i\] -> \[$o\]" + proj-dot-ins-append $i $o } } }} @@ -640,10 +649,10 @@ proc teaish__configure_phase1 {} { # # NO [define]s after this point! # - proj-dot-ins-process -validate proj-if-opt-truthy teaish-dump-defines { proj-file-write config.defines.txt $tdefs } + proj-dot-ins-process -validate }; # teaish__configure_phase1 @@ -1068,7 +1077,7 @@ If you are attempting an out-of-tree build, use ]]} { if {[string match *.in $extM]} { define TEAISH_MAKEFILE_IN $extM - define TEAISH_MAKEFILE [file rootname [file tail $extM]] + define TEAISH_MAKEFILE _[file rootname [file tail $extM]] } else { define TEAISH_MAKEFILE_IN "" define TEAISH_MAKEFILE $extM @@ -1136,8 +1145,8 @@ If you are attempting an out-of-tree build, use set flist [list $dirExt/teaish.test.tcl.in $dirExt/teaish.test.tcl] if {[proj-first-file-found ttt $flist]} { if {[string match *.in $ttt]} { - # Generate teaish.test.tcl from $ttt - set xt [file rootname [file tail $ttt]] + # Generate _teaish.test.tcl from $ttt + set xt _[file rootname [file tail $ttt]] file delete -force -- $xt; # ensure no stale copy is used define TEAISH_TEST_TCL $xt define TEAISH_TEST_TCL_IN $ttt @@ -1304,7 +1313,6 @@ proc teaish-ldflags-prepend {args} { # object files (which are typically in the build tree)). # proc teaish-src-add {args} { - set i 0 proj-parse-simple-flags args flags { -dist 0 {expr 1} -dir 0 {expr 1} @@ -1389,7 +1397,7 @@ proc teaish__cleanup_rule {{tgt clean}} { return ${tgt}-_${x}_ } -# @teaish-make-obj objfile srcfile ?...args? +# @teaish-make-obj ?flags? ?...args? # # Uses teaish-make-add to inject makefile rules for $objfile from # $srcfile, which is assumed to be C code which uses libtcl. Unless @@ -1403,43 +1411,45 @@ proc teaish__cleanup_rule {{tgt clean}} { # Any arguments after the 2nd may be flags described below or, if no # -recipe is provided, flags for the compiler call. # +# -obj obj-filename.o +# +# -src src-filename.c +# # -recipe {...} # Uses the trimmed value of {...} as the recipe, prefixing it with # a single hard-tab character. # # -deps {...} -# List of extra files to list as dependencies of $o. Good luck -# escaping non-trivial cases properly. +# List of extra files to list as dependencies of $o. # # -clean # Generate cleanup rules as well. -proc teaish-make-obj {o src args} { - set consume 0 - set clean 0 - set flag "" - array set flags {} - set xargs {} - foreach arg $args { - if {$consume} { - set consume 0 - set flags($flag) $arg - continue - } - switch -exact -- $arg { - -clean {incr clean} - -recipe - - -deps { - set flag $arg - incr consume - } - default { - lappend xargs $arg - } +proc teaish-make-obj {args} { + proj-parse-simple-flags args flags { + -clean 0 {expr 1} + -recipe => {} + -deps => {} + -obj => {} + -src => {} + } + #parray flags + if {"" eq $flags(-obj)} { + set args [lassign $args flags(-obj)] + if {"" eq $flags(-obj)} { + proj-error "Missing -obj flag." } } + foreach f {-deps -src} { + set flags($f) [string trim [string map {\n " "} $flags($f)]] + } + foreach f {-deps -src} { + set flags($f) [string trim $flags($f)] + } + #parray flags + #puts "-- args=$args" teaish-make-add \ - "# [proj-scope 1] -> [proj-scope] $o $src" -nl \ - "$o: $src $::teaish__Config(teaish.tcl)" + "# [proj-scope 1] -> [proj-scope] $flags(-obj) $flags(-src)" -nl \ + "$flags(-obj): $flags(-src) $::teaish__Config(teaish.tcl)" if {[info exists flags(-deps)]} { teaish-make-add " " [join $flags(-deps)] } @@ -1447,12 +1457,12 @@ proc teaish-make-obj {o src args} { if {[info exists flags(-recipe)]} { teaish-make-add [string trim $flags(-recipe)] -nl } else { - teaish-make-add [join [list \$(CC.tcl) -c $src {*}$xargs]] -nl + teaish-make-add [join [list \$(CC.tcl) -c $flags(-src) {*}$args]] -nl } - if {$clean} { + if {$flags(-clean)} { set rule [teaish__cleanup_rule] teaish-make-add \ - "clean: $rule\n$rule:\n\trm -f \"$o\"\n" + "clean: $rule\n$rule:\n\trm -f \"$flags(-obj)\"\n" } } @@ -2080,6 +2090,17 @@ proc teaish-pkginfo-set {args} { set v $x } + -src { + set d $::teaish__Config(extension-dir) + foreach f $v { + lappend ::teaish__Config(dist-files) $f + lappend ::teaish__Config(extension-src) $d/$f + lappend ::teaish__PkgInfo(-src) $f + # ^^^ so that default-value initialization in + # teaish-configure-core recognizes that it's been set. + } + } + -tm.tcl - -tm.tcl.in { if {0x30 & $::teaish__Config(pkgindex-policy)} { @@ -2517,7 +2538,7 @@ proc teaish__install {{dDest ""}} { ] { teaish__verbose 1 msg-result "Copying files to $destDir..." file mkdir $destDir - foreach f [glob -directory $srcDir *] { + foreach f [glob -nocomplain -directory $srcDir *] { if {[string match {*~} $f] || [string match "#*#" [file tail $f]]} { # Editor-generated backups and emacs lock files continue diff --git a/autosetup/teaish/tester.tcl b/autosetup/teaish/tester.tcl index d8b5f7a0e..a25b366e8 100644 --- a/autosetup/teaish/tester.tcl +++ b/autosetup/teaish/tester.tcl @@ -99,7 +99,7 @@ proc test__affert {failMode args} { lassign $args script msg } incr ::test__Counters($what) - if {![uplevel 1 [concat expr [list $script]]]} { + if {![uplevel 1 expr [list $script]]} { if {"" eq $msg} { set msg $script } @@ -136,6 +136,40 @@ proc assert {args} { tailcall test__affert 1 {*}$args } +# +# @assert-matches ?-e? pattern ?-e? rhs ?msg? +# +# Equivalent to assert {[string match $pattern $rhs]} except that +# if either of those are prefixed with an -e flag, they are eval'd +# and their results are used. +# +proc assert-matches {args} { + set evalLhs 0 + set evalRhs 0 + if {"-e" eq [lindex $args 0]} { + incr evalLhs + set args [lassign $args -] + } + set args [lassign $args pattern] + if {"-e" eq [lindex $args 0]} { + incr evalRhs + set args [lassign $args -] + } + set args [lassign $args rhs msg] + + if {$evalLhs} { + set pattern [uplevel 1 $pattern] + } + if {$evalRhs} { + set rhs [uplevel 1 $rhs] + } + #puts "***pattern=$pattern\n***rhs=$rhs" + tailcall test__affert 1 \ + [join [list \[ string match [list $pattern] [list $rhs] \]]] $msg + # why does this not work? [list \[ string match [list $pattern] [list $rhs] \]] $msg + # "\[string match [list $pattern] [list $rhs]\]" +} + # # @test-assert testId script ?msg? # @@ -157,7 +191,7 @@ proc test-expect {testId script result} { puts "test $testId" set x [string trim [uplevel 1 $script]] set result [string trim $result] - tailcall test__affert 0 [list $x eq $result] \ + tailcall test__affert 0 [list "{$x}" eq "{$result}"] \ "\nEXPECTED: <<$result>>\nGOT: <<$x>>" } @@ -169,7 +203,7 @@ proc test-expect {testId script result} { # proc test-catch {cmd args} { if {[catch { - $cmd {*}$args + uplevel 1 $cmd {*}$args } rc xopts]} { puts "[test-current-scope] ignoring failure of: $cmd [lindex $args 0]: $rc" return 1 @@ -177,6 +211,37 @@ proc test-catch {cmd args} { return 0 } +# +# @test-catch-matching pattern (script|cmd args...) +# +# Works like test-catch, but it expects its argument(s) to to throw an +# error matching the given string (checked with [string match]). If +# they do not throw, or the error does not match $pattern, this +# function throws, else it returns 1. +# +# If there is no second argument, the $cmd is assumed to be a script, +# and will be eval'd in the caller's scope. +# +# TODO: add -glob and -regex flags to control matching flavor. +# +proc test-catch-matching {pattern cmd args} { + if {[catch { + #puts "**** catch-matching cmd=$cmd args=$args" + if {0 == [llength $args]} { + uplevel 1 $cmd {*}$args + } else { + $cmd {*}$args + } + } rc xopts]} { + if {[string match $pattern $rc]} { + return 1 + } else { + error "[test-current-scope] exception does not match {$pattern}: {$rc}" + } + } + error "[test-current-scope] expecting to see an error matching {$pattern}" +} + if {![array exists ::teaish__BuildFlags]} { array set ::teaish__BuildFlags {} } diff --git a/doc/compile-for-windows.md b/doc/compile-for-windows.md index 2e6228633..0e59c83fe 100644 --- a/doc/compile-for-windows.md +++ b/doc/compile-for-windows.md @@ -1,7 +1,7 @@ # Notes On Compiling SQLite On Windows 11 Below are step-by-step instructions on how to build SQLite from -canonical source on a new Windows 11 PC, as of 2024-10-09. +canonical source on a new Windows 11 PC, as of 2025-10-31. See [](./compile-for-unix.md) for a similar guide for unix-like systems, including MacOS. @@ -24,7 +24,8 @@ systems, including MacOS. application to your task bar, as you will use it a lot. Bring up an instance of this command prompt and do all of the subsequent steps in that "x64 Native Tools" command prompt. (Or use "x86" if you want - a 32-bit build.) The subsequent steps will not work in a vanilla + a 32-bit build. Or use "ARM64" if you want to do a build for Windows + on ARM.) The subsequent steps will not work in a vanilla DOS prompt. Nor will they work in PowerShell. 3. *(Optional):* Install TCL development libraries. @@ -133,9 +134,7 @@ following minor changes: The command the developers use for building the deliverable DLL on the [download page](https://sqlite.org/download.html) is as follows: -> ~~~~ -nmake /f Makefile.msc sqlite3.dll USE_NATIVE_LIBPATHS=1 "OPTS=-DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_FTS5=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_JSON1=1 -DSQLITE_ENABLE_GEOPOLY=1 -DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 -DSQLITE_ENABLE_SERIALIZE=1 -DSQLITE_ENABLE_MATH_FUNCTIONS=1" -~~~~ +> nmake /f Makefile.msc sqlite3.dll USE_NATIVE_LIBPATHS=1 "OPTS=-DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_FTS5=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_JSON1=1 -DSQLITE_ENABLE_GEOPOLY=1 -DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 -DSQLITE_ENABLE_SERIALIZE=1 -DSQLITE_ENABLE_MATH_FUNCTIONS=1" That command generates both the sqlite3.dll and sqlite3.def files. The same command works for both 32-bit and 64-bit builds. @@ -147,9 +146,7 @@ with TCL in order to function. The [sqlite3_analyzer.exe program](https://sqlit is an example. You can build as described above, and then enter: -> ~~~~ -nmake /f Makefile.msc sqlite3_analyzer.exe -~~~~ +> nmake /f Makefile.msc sqlite3_analyzer.exe And you will end up with a working executable. However, that executable will depend on having the "tcl98.dll" library somewhere on your %PATH%. @@ -175,17 +172,11 @@ statically linked so that it does not depend on separate DLL: 5. CD into your SQLite source code directory and build the desired utility program, but add the following extra argument to the nmake command line: -
          -      STATICALLY_LINK_TCL=1
          -      
          +
          STATICALLY_LINK_TCL=1

          So, for example, to build a statically linked version of sqlite3_analyzer.exe, you might type: -

          -      nmake /f Makefile.msc STATICALLY_LINK_TCL=1 sqlite3_analyzer.exe
          -      
          +
          nmake /f Makefile.msc STATICALLY_LINK_TCL=1 sqlite3_analyzer.exe
          6. After your executable is built, you can verify that it does not depend on the TCL DLL by running: -
          -      dumpbin /dependents sqlite3_analyzer.exe
          -      
          +
          dumpbin /dependents sqlite3_analyzer.exe
          diff --git a/doc/jsonb.md b/doc/jsonb.md index ce36f3ead..63ce77b17 100644 --- a/doc/jsonb.md +++ b/doc/jsonb.md @@ -149,11 +149,10 @@ RFC 8259 format, without extensions. The payload is the ASCII text representation of that numeric value.
        • INT5 → -The element is a JSON integer value that is not in the -canonical format. The payload is the ASCII -text representation of that numeric value. Because the payload is in a -non-standard format, it will need to be translated when the JSONB is -converted into RFC 8259 text JSON. +The element is a JSON integer literal in hexadecimal notation. +The payload is the ASCII text representation of the literal. +Because the payload is in a non-standard format, it will need +to be translated when the JSONB is converted into RFC 8259 text JSON.

        • FLOAT → The element is a JSON floating-point value in the canonical @@ -162,10 +161,10 @@ text representation of that numeric value.

        • FLOAT5 → The element is a JSON floating-point value that is not in the -canonical format. The payload is the ASCII -text representation of that numeric value. Because the payload is in a -non-standard format, it will need to be translated when the JSONB is -converted into RFC 8259 text JSON. +canonical JSON format but rather the extended JSON5 format. +The payload is the ASCII text representation of that numeric value. +Because the payload is in a non-standard format, it will need to +be translated when the JSONB is converted into RFC 8259 text JSON.

        • TEXT → The element is a JSON string value that does not contain diff --git a/ext/expert/sqlite3expert.c b/ext/expert/sqlite3expert.c index ddb36714f..c430c3ae9 100644 --- a/ext/expert/sqlite3expert.c +++ b/ext/expert/sqlite3expert.c @@ -172,11 +172,11 @@ struct sqlite3expert { ** Allocate and return nByte bytes of zeroed memory using sqlite3_malloc(). ** If the allocation fails, set *pRc to SQLITE_NOMEM and return NULL. */ -static void *idxMalloc(int *pRc, int nByte){ +static void *idxMalloc(int *pRc, i64 nByte){ void *pRet; assert( *pRc==SQLITE_OK ); assert( nByte>0 ); - pRet = sqlite3_malloc(nByte); + pRet = sqlite3_malloc64(nByte); if( pRet ){ memset(pRet, 0, nByte); }else{ @@ -243,7 +243,7 @@ static int idxHashAdd( return 1; } } - pEntry = idxMalloc(pRc, sizeof(IdxHashEntry) + nKey+1 + nVal+1); + pEntry = idxMalloc(pRc, sizeof(IdxHashEntry) + (i64)nKey+1 + (i64)nVal+1); if( pEntry ){ pEntry->zKey = (char*)&pEntry[1]; memcpy(pEntry->zKey, zKey, nKey); @@ -378,15 +378,15 @@ struct ExpertCsr { }; static char *expertDequote(const char *zIn){ - int n = STRLEN(zIn); - char *zRet = sqlite3_malloc(n); + i64 n = STRLEN(zIn); + char *zRet = sqlite3_malloc64(n); assert( zIn[0]=='\'' ); assert( zIn[n-1]=='\'' ); if( zRet ){ - int iOut = 0; - int iIn = 0; + i64 iOut = 0; + i64 iIn = 0; for(iIn=1; iIn<(n-1); iIn++){ if( zIn[iIn]=='\'' ){ assert( zIn[iIn+1]=='\'' ); @@ -699,7 +699,7 @@ static int idxGetTableInfo( sqlite3_stmt *p1 = 0; int nCol = 0; int nTab; - int nByte; + i64 nByte; IdxTable *pNew = 0; int rc, rc2; char *pCsr = 0; @@ -791,14 +791,14 @@ static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){ va_list ap; char *zAppend = 0; char *zRet = 0; - int nIn = zIn ? STRLEN(zIn) : 0; - int nAppend = 0; + i64 nIn = zIn ? STRLEN(zIn) : 0; + i64 nAppend = 0; va_start(ap, zFmt); if( *pRc==SQLITE_OK ){ zAppend = sqlite3_vmprintf(zFmt, ap); if( zAppend ){ nAppend = STRLEN(zAppend); - zRet = (char*)sqlite3_malloc(nIn + nAppend + 1); + zRet = (char*)sqlite3_malloc64(nIn + nAppend + 1); } if( zAppend && zRet ){ if( nIn ) memcpy(zRet, zIn, nIn); @@ -1562,8 +1562,8 @@ struct IdxRemCtx { int eType; /* SQLITE_NULL, INTEGER, REAL, TEXT, BLOB */ i64 iVal; /* SQLITE_INTEGER value */ double rVal; /* SQLITE_FLOAT value */ - int nByte; /* Bytes of space allocated at z */ - int n; /* Size of buffer z */ + i64 nByte; /* Bytes of space allocated at z */ + i64 n; /* Size of buffer z */ char *z; /* SQLITE_TEXT/BLOB value */ } aSlot[1]; }; @@ -1599,11 +1599,13 @@ static void idxRemFunc( break; case SQLITE_BLOB: - sqlite3_result_blob(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT); + assert( pSlot->n <= 0x7fffffff ); + sqlite3_result_blob(pCtx, pSlot->z, (int)pSlot->n, SQLITE_TRANSIENT); break; case SQLITE_TEXT: - sqlite3_result_text(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT); + assert( pSlot->n <= 0x7fffffff ); + sqlite3_result_text(pCtx, pSlot->z, (int)pSlot->n, SQLITE_TRANSIENT); break; } @@ -1623,10 +1625,10 @@ static void idxRemFunc( case SQLITE_BLOB: case SQLITE_TEXT: { - int nByte = sqlite3_value_bytes(argv[1]); + i64 nByte = sqlite3_value_bytes(argv[1]); const void *pData = 0; if( nByte>pSlot->nByte ){ - char *zNew = (char*)sqlite3_realloc(pSlot->z, nByte*2); + char *zNew = (char*)sqlite3_realloc64(pSlot->z, nByte*2); if( zNew==0 ){ sqlite3_result_error_nomem(pCtx); return; @@ -1681,7 +1683,7 @@ static int idxPopulateOneStat1( int nCol = 0; int i; sqlite3_stmt *pQuery = 0; - int *aStat = 0; + i64 *aStat = 0; int rc = SQLITE_OK; assert( p->iSample>0 ); @@ -1727,7 +1729,7 @@ static int idxPopulateOneStat1( sqlite3_free(zQuery); if( rc==SQLITE_OK ){ - aStat = (int*)idxMalloc(&rc, sizeof(int)*(nCol+1)); + aStat = (i64*)idxMalloc(&rc, sizeof(i64)*(nCol+1)); } if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){ IdxHashEntry *pEntry; @@ -1744,11 +1746,11 @@ static int idxPopulateOneStat1( } if( rc==SQLITE_OK ){ - int s0 = aStat[0]; - zStat = sqlite3_mprintf("%d", s0); + i64 s0 = aStat[0]; + zStat = sqlite3_mprintf("%lld", s0); if( zStat==0 ) rc = SQLITE_NOMEM; for(i=1; rc==SQLITE_OK && i<=nCol; i++){ - zStat = idxAppendText(&rc, zStat, " %d", (s0+aStat[i]/2) / aStat[i]); + zStat = idxAppendText(&rc, zStat, " %lld", (s0+aStat[i]/2) / aStat[i]); } } @@ -1827,7 +1829,7 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){ rc = sqlite3_exec(p->dbm, "ANALYZE; PRAGMA writable_schema=1", 0, 0, 0); if( rc==SQLITE_OK ){ - int nByte = sizeof(struct IdxRemCtx) + (sizeof(struct IdxRemSlot) * nMax); + i64 nByte = sizeof(struct IdxRemCtx) + (sizeof(struct IdxRemSlot) * nMax); pCtx = (struct IdxRemCtx*)idxMalloc(&rc, nByte); } @@ -1844,7 +1846,7 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){ } if( rc==SQLITE_OK ){ - pCtx->nSlot = nMax+1; + pCtx->nSlot = (i64)nMax+1; rc = idxPrepareStmt(p->dbm, &pAllIndex, pzErr, zAllIndex); } if( rc==SQLITE_OK ){ @@ -2111,7 +2113,7 @@ int sqlite3_expert_sql( if( pStmt ){ IdxStatement *pNew; const char *z = sqlite3_sql(pStmt); - int n = STRLEN(z); + i64 n = STRLEN(z); pNew = (IdxStatement*)idxMalloc(&rc, sizeof(IdxStatement) + n+1); if( rc==SQLITE_OK ){ pNew->zSql = (char*)&pNew[1]; diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index d438549de..e98b90a75 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -14,6 +14,13 @@ #ifndef _FTSINT_H #define _FTSINT_H +/* +** Activate assert() only if SQLITE_TEST is enabled. +*/ +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif + #include #include #include @@ -21,10 +28,6 @@ #include #include -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif - /* FTS3/FTS4 require virtual tables */ #ifdef SQLITE_OMIT_VIRTUALTABLE # undef SQLITE_ENABLE_FTS3 @@ -44,7 +47,7 @@ /* If not building as part of the core, include sqlite3ext.h. */ #ifndef SQLITE_CORE -# include "sqlite3ext.h" +# include "sqlite3ext.h" SQLITE_EXTENSION_INIT3 #endif @@ -186,13 +189,6 @@ typedef sqlite3_int64 i64; /* 8-byte signed integer */ */ #define UNUSED_PARAMETER(x) (void)(x) -/* -** Activate assert() only if SQLITE_TEST is enabled. -*/ -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif - /* ** The TESTONLY macro is used to enclose variable declarations or ** other bits of code that are needed to support the arguments @@ -213,7 +209,7 @@ typedef sqlite3_int64 i64; /* 8-byte signed integer */ ** Macros needed to provide flexible arrays in a portable way */ #ifndef offsetof -# define offsetof(STRUCTURE,FIELD) ((size_t)((char*)&((STRUCTURE*)0)->FIELD)) +# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) #endif #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) # define FLEXARRAY diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 9c7f0ade9..62e27d30b 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -17,10 +17,6 @@ #include #include -#ifndef SQLITE_AMALGAMATION -typedef sqlite3_int64 i64; -#endif - /* ** Characters that may appear in the second argument to matchinfo(). */ diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 55a3f0a82..19dff31f0 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -3714,8 +3714,8 @@ struct NodeWriter { ** to an appendable b-tree segment. */ struct IncrmergeWriter { - int nLeafEst; /* Space allocated for leaf blocks */ - int nWork; /* Number of leaf pages flushed */ + i64 nLeafEst; /* Space allocated for leaf blocks */ + i64 nWork; /* Number of leaf pages flushed */ sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ int iIdx; /* Index of *output* segment in iAbsLevel+1 */ sqlite3_int64 iStart; /* Block number of first allocated block */ @@ -4461,7 +4461,7 @@ static int fts3IncrmergeWriter( ){ int rc; /* Return Code */ int i; /* Iterator variable */ - int nLeafEst = 0; /* Blocks allocated for leaf nodes */ + i64 nLeafEst = 0; /* Blocks allocated for leaf nodes */ sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ @@ -4471,7 +4471,7 @@ static int fts3IncrmergeWriter( sqlite3_bind_int64(pLeafEst, 1, iAbsLevel); sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment); if( SQLITE_ROW==sqlite3_step(pLeafEst) ){ - nLeafEst = sqlite3_column_int(pLeafEst, 0); + nLeafEst = sqlite3_column_int64(pLeafEst, 0); } rc = sqlite3_reset(pLeafEst); } diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 7ad1cc16b..a13a65d3c 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -60,27 +60,20 @@ typedef sqlite3_uint64 u64; # define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) -/* The uptr type is an unsigned integer large enough to hold a pointer +/* +** This macro is used in a single assert() within fts5 to check that an +** allocation is aligned to an 8-byte boundary. But it is a complicated +** macro to get right for multiple platforms without generating warnings. +** So instead of reproducing the entire definition from sqliteInt.h, we +** just do without this assert() for the rare non-amalgamation builds. */ -#if defined(HAVE_STDINT_H) - typedef uintptr_t uptr; -#elif SQLITE_PTRSIZE==4 - typedef u32 uptr; -#else - typedef u64 uptr; -#endif - -#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC -# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&3)==0) -#else -# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&7)==0) -#endif +#define EIGHT_BYTE_ALIGNMENT(x) 1 /* ** Macros needed to provide flexible arrays in a portable way */ #ifndef offsetof -# define offsetof(STRUCTURE,FIELD) ((size_t)((char*)&((STRUCTURE*)0)->FIELD)) +# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) #endif #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) # define FLEXARRAY @@ -822,7 +815,7 @@ int sqlite3Fts5ExprPattern( ** i64 iRowid = sqlite3Fts5ExprRowid(pExpr); ** } */ -int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, int bDesc); +int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, i64, int bDesc); int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax); int sqlite3Fts5ExprEof(Fts5Expr*); i64 sqlite3Fts5ExprRowid(Fts5Expr*); diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 0a9b08ed1..352df81f4 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1549,7 +1549,13 @@ static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){ ** Return SQLITE_OK if successful, or an SQLite error code otherwise. It ** is not considered an error if the query does not match any documents. */ -int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){ +int sqlite3Fts5ExprFirst( + Fts5Expr *p, + Fts5Index *pIdx, + i64 iFirst, + i64 iLast, + int bDesc +){ Fts5ExprNode *pRoot = p->pRoot; int rc; /* Return code */ @@ -1571,6 +1577,9 @@ int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){ assert( pRoot->bEof==0 ); rc = fts5ExprNodeNext(p, pRoot, 0, 0); } + if( fts5RowidCmp(p, pRoot->iRowid, iLast)>0 ){ + pRoot->bEof = 1; + } return rc; } diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 13e3740fe..a5a37f758 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -554,6 +554,36 @@ struct Fts5SegIter { u8 bDel; /* True if the delete flag is set */ }; +static int fts5IndexCorruptRowid(Fts5Index *pIdx, i64 iRowid){ + pIdx->rc = FTS5_CORRUPT; + sqlite3Fts5ConfigErrmsg(pIdx->pConfig, + "fts5: corruption found reading blob %lld from table \"%s\"", + iRowid, pIdx->pConfig->zName + ); + return SQLITE_CORRUPT_VTAB; +} +#define FTS5_CORRUPT_ROWID(pIdx, iRowid) fts5IndexCorruptRowid(pIdx, iRowid) + +static int fts5IndexCorruptIter(Fts5Index *pIdx, Fts5SegIter *pIter){ + pIdx->rc = FTS5_CORRUPT; + sqlite3Fts5ConfigErrmsg(pIdx->pConfig, + "fts5: corruption on page %d, segment %d, table \"%s\"", + pIter->iLeafPgno, pIter->pSeg->iSegid, pIdx->pConfig->zName + ); + return SQLITE_CORRUPT_VTAB; +} +#define FTS5_CORRUPT_ITER(pIdx, pIter) fts5IndexCorruptIter(pIdx, pIter) + +static int fts5IndexCorruptIdx(Fts5Index *pIdx){ + pIdx->rc = FTS5_CORRUPT; + sqlite3Fts5ConfigErrmsg(pIdx->pConfig, + "fts5: corruption in table \"%s\"", pIdx->pConfig->zName + ); + return SQLITE_CORRUPT_VTAB; +} +#define FTS5_CORRUPT_IDX(pIdx) fts5IndexCorruptIdx(pIdx) + + /* ** Array of tombstone pages. Reference counted. */ @@ -843,13 +873,13 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ ** All the reasons those functions might return SQLITE_ERROR - missing ** table, missing row, non-blob/text in block column - indicate ** backing store corruption. */ - if( rc==SQLITE_ERROR ) rc = FTS5_CORRUPT; + if( rc==SQLITE_ERROR ) rc = FTS5_CORRUPT_ROWID(p, iRowid); if( rc==SQLITE_OK ){ u8 *aOut = 0; /* Read blob data into this buffer */ - int nByte = sqlite3_blob_bytes(p->pReader); - int szData = (sizeof(Fts5Data) + 7) & ~7; - sqlite3_int64 nAlloc = szData + nByte + FTS5_DATA_PADDING; + i64 nByte = sqlite3_blob_bytes(p->pReader); + i64 szData = (sizeof(Fts5Data) + 7) & ~7; + i64 nAlloc = szData + nByte + FTS5_DATA_PADDING; pRet = (Fts5Data*)sqlite3_malloc64(nAlloc); if( pRet ){ pRet->nn = nByte; @@ -893,7 +923,7 @@ static Fts5Data *fts5LeafRead(Fts5Index *p, i64 iRowid){ Fts5Data *pRet = fts5DataRead(p, iRowid); if( pRet ){ if( pRet->nn<4 || pRet->szLeaf>pRet->nn ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); fts5DataRelease(pRet); pRet = 0; } @@ -1252,8 +1282,14 @@ static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){ /* TODO: Do we need this if the leaf-index is appended? Probably... */ memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING); p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet); - if( p->rc==SQLITE_OK && (pConfig->pgsz==0 || pConfig->iCookie!=iCookie) ){ - p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); + if( p->rc==SQLITE_OK ){ + if( (pConfig->pgsz==0 || pConfig->iCookie!=iCookie) ){ + p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); + } + }else if( p->rc==SQLITE_CORRUPT_VTAB ){ + sqlite3Fts5ConfigErrmsg(p->pConfig, + "fts5: corrupt structure record for table \"%s\"", p->pConfig->zName + ); } fts5DataRelease(pData); if( p->rc!=SQLITE_OK ){ @@ -1876,7 +1912,7 @@ static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ while( iOff>=pIter->pLeaf->szLeaf ){ fts5SegIterNextPage(p, pIter); if( pIter->pLeaf==0 ){ - if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; + if( p->rc==SQLITE_OK ) FTS5_CORRUPT_ITER(p, pIter); return; } iOff = 4; @@ -1908,7 +1944,7 @@ static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){ iOff += fts5GetVarint32(&a[iOff], nNew); if( iOff+nNew>pIter->pLeaf->szLeaf || nKeep>pIter->term.n || nNew==0 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } pIter->term.n = nKeep; @@ -2103,7 +2139,7 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){ iRowidOff = fts5LeafFirstRowidOff(pNew); if( iRowidOff ){ if( iRowidOff>=pNew->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); }else{ pIter->pLeaf = pNew; pIter->iLeafOffset = iRowidOff; @@ -2337,7 +2373,7 @@ static void fts5SegIterNext( } assert_nc( iOffszLeaf ); if( iOff>pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } } @@ -2445,18 +2481,20 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ fts5DataRelease(pIter->pLeaf); pIter->pLeaf = pLast; pIter->iLeafPgno = pgnoLast; - iOff = fts5LeafFirstRowidOff(pLast); - if( iOff>pLast->szLeaf ){ - p->rc = FTS5_CORRUPT; - return; - } - iOff += fts5GetVarint(&pLast->p[iOff], (u64*)&pIter->iRowid); - pIter->iLeafOffset = iOff; + if( p->rc==SQLITE_OK ){ + iOff = fts5LeafFirstRowidOff(pLast); + if( iOff>pLast->szLeaf ){ + FTS5_CORRUPT_ITER(p, pIter); + return; + } + iOff += fts5GetVarint(&pLast->p[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; - if( fts5LeafIsTermless(pLast) ){ - pIter->iEndofDoclist = pLast->nn+1; - }else{ - pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast); + if( fts5LeafIsTermless(pLast) ){ + pIter->iEndofDoclist = pLast->nn+1; + }else{ + pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast); + } } } @@ -2526,7 +2564,7 @@ static void fts5LeafSeek( iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff); iOff = iTermOff; if( iOff>n ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } @@ -2569,7 +2607,7 @@ static void fts5LeafSeek( iOff = iTermOff; if( iOff>=n ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } @@ -2591,7 +2629,7 @@ static void fts5LeafSeek( iPgidx = (u32)pIter->pLeaf->szLeaf; iPgidx += fts5GetVarint32(&pIter->pLeaf->p[iPgidx], iOff); if( iOff<4 || (i64)iOff>=pIter->pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; }else{ nKeep = 0; @@ -2606,7 +2644,7 @@ static void fts5LeafSeek( search_success: if( (i64)iOff+nNew>n || nNew<1 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ITER(p, pIter); return; } pIter->iLeafOffset = iOff + nNew; @@ -3071,7 +3109,7 @@ static void fts5SegIterGotoPage( assert( iLeafPgno>pIter->iLeafPgno ); if( iLeafPgno>pIter->pSeg->pgnoLast ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); }else{ fts5DataRelease(pIter->pNextLeaf); pIter->pNextLeaf = 0; @@ -3086,7 +3124,7 @@ static void fts5SegIterGotoPage( u8 *a = pIter->pLeaf->p; int n = pIter->pLeaf->szLeaf; if( iOff<4 || iOff>=n ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); }else{ iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); pIter->iLeafOffset = iOff; @@ -3565,7 +3603,7 @@ static void fts5ChunkIterate( if( nRem<=0 ){ break; }else if( pSeg->pSeg==0 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); return; }else{ pgno++; @@ -4668,7 +4706,7 @@ static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){ ** a single page has been assigned to more than one segment. In ** this case a prior iteration of this loop may have corrupted the ** segment currently being trimmed. */ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iLeafRowid); }else{ fts5BufferZero(&buf); fts5BufferGrow(&p->rc, &buf, pData->nn); @@ -5135,7 +5173,7 @@ static void fts5SecureDeleteOverflow( }else if( bDetailNone ){ break; }else if( iNext>=pLeaf->szLeaf || pLeaf->nnszLeaf || iNext<4 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); break; }else{ int nShift = iNext - 4; @@ -5155,7 +5193,7 @@ static void fts5SecureDeleteOverflow( i1 += fts5GetVarint32(&aPg[i1], iFirst); if( iFirstrc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); break; } aIdx = sqlite3Fts5MallocZero(&p->rc, (pLeaf->nn-pLeaf->szLeaf)+2); @@ -5378,14 +5416,14 @@ static void fts5DoSecureDelete( nSuffix = (nPrefix2 + nSuffix2) - nPrefix; if( (iKeyOff+nSuffix)>iPgIdx || (iNextOff+nSuffix2)>iPgIdx ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); }else{ if( iKey!=1 ){ iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix); } iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix); if( nPrefix2>pSeg->term.n ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); }else if( nPrefix2>nPrefix ){ memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix); iOff += (nPrefix2-nPrefix); @@ -5809,7 +5847,7 @@ static Fts5Structure *fts5IndexOptimizeStruct( } nByte += (((i64)pStruct->nLevel)+1) * sizeof(Fts5StructureLevel); - assert( nByte==SZ_FTS5STRUCTURE(pStruct->nLevel+2) ); + assert( nByte==(i64)SZ_FTS5STRUCTURE(pStruct->nLevel+2) ); pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pNew ){ @@ -6178,7 +6216,7 @@ static void fts5MergePrefixLists( } if( pHead==0 || pHead->pNext==0 ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_IDX(p); break; } @@ -6215,7 +6253,7 @@ static void fts5MergePrefixLists( assert_nc( tmp.n+nTail<=nTmp ); assert( tmp.n+nTail<=nTmp+nMerge*10 ); if( tmp.n+nTail>nTmp-FTS5_DATA_ZERO_PADDING ){ - if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; + if( p->rc==SQLITE_OK ) FTS5_CORRUPT_IDX(p); break; } fts5BufferSafeAppendVarint(&out, (tmp.n+nTail) * 2); @@ -6784,11 +6822,14 @@ int sqlite3Fts5IndexRollback(Fts5Index *p){ */ int sqlite3Fts5IndexReinit(Fts5Index *p){ Fts5Structure *pTmp; - u8 tmpSpace[SZ_FTS5STRUCTURE(1)]; + union { + Fts5Structure sFts; + u8 tmpSpace[SZ_FTS5STRUCTURE(1)]; + } uFts; fts5StructureInvalidate(p); fts5IndexDiscardData(p); - pTmp = (Fts5Structure*)tmpSpace; - memset(pTmp, 0, SZ_FTS5STRUCTURE(1)); + pTmp = &uFts.sFts; + memset(uFts.tmpSpace, 0, sizeof(uFts.tmpSpace)); if( p->pConfig->bContentlessDelete ){ pTmp->nOriginCntr = 1; } @@ -8248,19 +8289,27 @@ static int fts5TestUtf8(const char *z, int n){ /* ** This function is also purely an internal test. It does not contribute to ** FTS functionality, or even the integrity-check, in any way. +** +** This function sets output variable (*pbFail) to true if the test fails. Or +** leaves it unchanged if the test succeeds. */ static void fts5TestTerm( Fts5Index *p, Fts5Buffer *pPrev, /* Previous term */ const char *z, int n, /* Possibly new term to test */ u64 expected, - u64 *pCksum + u64 *pCksum, + int *pbFail ){ int rc = p->rc; if( pPrev->n==0 ){ fts5BufferSet(&rc, pPrev, n, (const u8*)z); }else - if( rc==SQLITE_OK && (pPrev->n!=n || memcmp(pPrev->p, z, n)) ){ + if( *pbFail==0 + && rc==SQLITE_OK + && (pPrev->n!=n || memcmp(pPrev->p, z, n)) + && (p->pHash==0 || p->pHash->nEntry==0) + ){ u64 cksum3 = *pCksum; const char *zTerm = (const char*)&pPrev->p[1]; /* term sans prefix-byte */ int nTerm = pPrev->n-1; /* Size of zTerm in bytes */ @@ -8310,7 +8359,7 @@ static void fts5TestTerm( fts5BufferSet(&rc, pPrev, n, (const u8*)z); if( rc==SQLITE_OK && cksum3!=expected ){ - rc = FTS5_CORRUPT; + *pbFail = 1; } *pCksum = cksum3; } @@ -8319,7 +8368,7 @@ static void fts5TestTerm( #else # define fts5TestDlidxReverse(x,y,z) -# define fts5TestTerm(u,v,w,x,y,z) +# define fts5TestTerm(t,u,v,w,x,y,z) #endif /* @@ -8344,14 +8393,17 @@ static void fts5IndexIntegrityCheckEmpty( for(i=iFirst; p->rc==SQLITE_OK && i<=iLast; i++){ Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, i)); if( pLeaf ){ - if( !fts5LeafIsTermless(pLeaf) ) p->rc = FTS5_CORRUPT; - if( i>=iNoRowid && 0!=fts5LeafFirstRowidOff(pLeaf) ) p->rc = FTS5_CORRUPT; + if( !fts5LeafIsTermless(pLeaf) + || (i>=iNoRowid && 0!=fts5LeafFirstRowidOff(pLeaf)) + ){ + FTS5_CORRUPT_ROWID(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, i)); + } } fts5DataRelease(pLeaf); } } -static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ +static void fts5IntegrityCheckPgidx(Fts5Index *p, i64 iRowid, Fts5Data *pLeaf){ i64 iTermOff = 0; int ii; @@ -8369,12 +8421,12 @@ static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ iOff = iTermOff; if( iOff>=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); }else if( iTermOff==nIncr ){ int nByte; iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); if( (iOff+nByte)>pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); }else{ fts5BufferSet(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); } @@ -8383,7 +8435,7 @@ static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ iOff += fts5GetVarint32(&pLeaf->p[iOff], nKeep); iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); if( nKeep>buf1.n || (iOff+nByte)>pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRowid); }else{ buf1.n = nKeep; fts5BufferAppendBlob(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); @@ -8391,7 +8443,7 @@ static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ if( p->rc==SQLITE_OK ){ res = fts5BufferCompare(&buf1, &buf2); - if( res<=0 ) p->rc = FTS5_CORRUPT; + if( res<=0 ) FTS5_CORRUPT_ROWID(p, iRowid); } } fts5BufferSet(&p->rc, &buf2, buf1.n, buf1.p); @@ -8452,7 +8504,7 @@ static void fts5IndexIntegrityCheckSegment( ** entry even if all the terms are removed from it by secure-delete ** operations. */ }else{ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRow); } }else{ @@ -8464,15 +8516,15 @@ static void fts5IndexIntegrityCheckSegment( iOff = fts5LeafFirstTermOff(pLeaf); iRowidOff = fts5LeafFirstRowidOff(pLeaf); if( iRowidOff>=iOff || iOff>=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iRow); }else{ iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm); res = fts5Memcmp(&pLeaf->p[iOff], zIdxTerm, MIN(nTerm, nIdxTerm)); if( res==0 ) res = nTerm - nIdxTerm; - if( res<0 ) p->rc = FTS5_CORRUPT; + if( res<0 ) FTS5_CORRUPT_ROWID(p, iRow); } - fts5IntegrityCheckPgidx(p, pLeaf); + fts5IntegrityCheckPgidx(p, iRow, pLeaf); } fts5DataRelease(pLeaf); if( p->rc ) break; @@ -8502,7 +8554,7 @@ static void fts5IndexIntegrityCheckSegment( iKey = FTS5_SEGMENT_ROWID(iSegid, iPg); pLeaf = fts5DataRead(p, iKey); if( pLeaf ){ - if( fts5LeafFirstRowidOff(pLeaf)!=0 ) p->rc = FTS5_CORRUPT; + if( fts5LeafFirstRowidOff(pLeaf)!=0 ) FTS5_CORRUPT_ROWID(p, iKey); fts5DataRelease(pLeaf); } } @@ -8517,12 +8569,12 @@ static void fts5IndexIntegrityCheckSegment( int iRowidOff = fts5LeafFirstRowidOff(pLeaf); ASSERT_SZLEAF_OK(pLeaf); if( iRowidOff>=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iKey); }else if( bSecureDelete==0 || iRowidOff>0 ){ i64 iDlRowid = fts5DlidxIterRowid(pDlidx); fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid); if( iRowidrc = FTS5_CORRUPT; + FTS5_CORRUPT_ROWID(p, iKey); } } fts5DataRelease(pLeaf); @@ -8574,6 +8626,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){ /* Used by extra internal tests only run if NDEBUG is not defined */ u64 cksum3 = 0; /* Checksum based on contents of indexes */ Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */ + int bTestFail = 0; #endif const int flags = FTS5INDEX_QUERY_NOOUTPUT; @@ -8616,7 +8669,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){ char *z = (char*)fts5MultiIterTerm(pIter, &n); /* If this is a new term, query for it. Update cksum3 with the results. */ - fts5TestTerm(p, &term, z, n, cksum2, &cksum3); + fts5TestTerm(p, &term, z, n, cksum2, &cksum3, &bTestFail); if( p->rc ) break; if( eDetail==FTS5_DETAIL_NONE ){ @@ -8634,15 +8687,26 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){ } } } - fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3); + fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3, &bTestFail); fts5MultiIterFree(pIter); - if( p->rc==SQLITE_OK && bUseCksum && cksum!=cksum2 ) p->rc = FTS5_CORRUPT; - - fts5StructureRelease(pStruct); + if( p->rc==SQLITE_OK && bUseCksum && cksum!=cksum2 ){ + p->rc = FTS5_CORRUPT; + sqlite3Fts5ConfigErrmsg(p->pConfig, + "fts5: checksum mismatch for table \"%s\"", p->pConfig->zName + ); + } #ifdef SQLITE_DEBUG + /* In SQLITE_DEBUG builds, expensive extra checks were run as part of + ** the integrity-check above. If no other errors were detected, but one + ** of these tests failed, set the result to SQLITE_CORRUPT_VTAB here. */ + if( p->rc==SQLITE_OK && bTestFail ){ + p->rc = FTS5_CORRUPT; + } fts5BufferFree(&term); #endif + + fts5StructureRelease(pStruct); fts5BufferFree(&poslist); return fts5IndexReturn(p); } diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index e888abf21..f45b9ef90 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -511,6 +511,17 @@ static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){ #endif } +static void fts5SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=3008002 +#ifndef SQLITE_CORE + if( sqlite3_libversion_number()>=3008002 ) +#endif + { + pIdxInfo->estimatedRows = nRow; + } +#endif +} + static int fts5UsePatternMatch( Fts5Config *pConfig, struct sqlite3_index_constraint *p @@ -646,7 +657,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ nSeenMatch++; idxStr[iIdxStr++] = 'M'; sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol); - idxStr += strlen(&idxStr[iIdxStr]); + iIdxStr += (int)strlen(&idxStr[iIdxStr]); assert( idxStr[iIdxStr]=='\0' ); } pInfo->aConstraintUsage[i].argvIndex = ++iCons; @@ -665,6 +676,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ idxStr[iIdxStr++] = '='; bSeenEq = 1; pInfo->aConstraintUsage[i].argvIndex = ++iCons; + pInfo->aConstraintUsage[i].omit = 1; } } } @@ -712,17 +724,21 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ /* Calculate the estimated cost based on the flags set in idxFlags. */ if( bSeenEq ){ - pInfo->estimatedCost = nSeenMatch ? 1000.0 : 10.0; - if( nSeenMatch==0 ) fts5SetUniqueFlag(pInfo); - }else if( bSeenLt && bSeenGt ){ - pInfo->estimatedCost = nSeenMatch ? 5000.0 : 250000.0; - }else if( bSeenLt || bSeenGt ){ - pInfo->estimatedCost = nSeenMatch ? 7500.0 : 750000.0; + pInfo->estimatedCost = nSeenMatch ? 1000.0 : 25.0; + fts5SetUniqueFlag(pInfo); + fts5SetEstimatedRows(pInfo, 1); }else{ - pInfo->estimatedCost = nSeenMatch ? 10000.0 : 1000000.0; - } - for(i=1; iestimatedCost *= 0.4; + if( bSeenLt && bSeenGt ){ + pInfo->estimatedCost = nSeenMatch ? 5000.0 : 750000.0; + }else if( bSeenLt || bSeenGt ){ + pInfo->estimatedCost = nSeenMatch ? 7500.0 : 2250000.0; + }else{ + pInfo->estimatedCost = nSeenMatch ? 10000.0 : 3000000.0; + } + for(i=1; iestimatedCost *= 0.4; + } + fts5SetEstimatedRows(pInfo, (i64)(pInfo->estimatedCost / 4.0)); } pInfo->idxNum = idxFlags; @@ -921,7 +937,9 @@ static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){ int bDesc = pCsr->bDesc; i64 iRowid = sqlite3Fts5ExprRowid(pCsr->pExpr); - rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->p.pIndex, iRowid, bDesc); + rc = sqlite3Fts5ExprFirst( + pCsr->pExpr, pTab->p.pIndex, iRowid, pCsr->iLastRowid, bDesc + ); if( rc==SQLITE_OK && iRowid!=sqlite3Fts5ExprRowid(pCsr->pExpr) ){ *pbSkip = 1; } @@ -1093,7 +1111,9 @@ static int fts5CursorFirstSorted( static int fts5CursorFirst(Fts5FullTable *pTab, Fts5Cursor *pCsr, int bDesc){ int rc; Fts5Expr *pExpr = pCsr->pExpr; - rc = sqlite3Fts5ExprFirst(pExpr, pTab->p.pIndex, pCsr->iFirstRowid, bDesc); + rc = sqlite3Fts5ExprFirst( + pExpr, pTab->p.pIndex, pCsr->iFirstRowid, pCsr->iLastRowid, bDesc + ); if( sqlite3Fts5ExprEof(pExpr) ){ CsrFlagSet(pCsr, FTS5CSR_EOF); } @@ -3601,9 +3621,9 @@ static void fts5LocaleFunc( sqlite3_value **apArg /* Function arguments */ ){ const char *zLocale = 0; - int nLocale = 0; + i64 nLocale = 0; const char *zText = 0; - int nText = 0; + i64 nText = 0; assert( nArg==2 ); UNUSED_PARAM(nArg); @@ -3620,10 +3640,10 @@ static void fts5LocaleFunc( Fts5Global *p = (Fts5Global*)sqlite3_user_data(pCtx); u8 *pBlob = 0; u8 *pCsr = 0; - int nBlob = 0; + i64 nBlob = 0; nBlob = FTS5_LOCALE_HDR_SIZE + nLocale + 1 + nText; - pBlob = (u8*)sqlite3_malloc(nBlob); + pBlob = (u8*)sqlite3_malloc64(nBlob); if( pBlob==0 ){ sqlite3_result_error_nomem(pCtx); return; @@ -3701,8 +3721,9 @@ static int fts5IntegrityMethod( " FTS5 table %s.%s: %s", zSchema, zTabname, sqlite3_errstr(rc)); } + }else if( (rc&0xff)==SQLITE_CORRUPT ){ + rc = SQLITE_OK; } - sqlite3Fts5IndexCloseReader(pTab->p.pIndex); pTab->p.pConfig->pzErrmsg = 0; diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index bcad9e724..184cb77b8 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -428,7 +428,7 @@ do_execsql_test 15.1 { } do_catchsql_test 15.2 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {1 {fts5: checksum mismatch for table "t1"}} #------------------------------------------------------------------------- # diff --git a/ext/fts5/test/fts5ab.test b/ext/fts5/test/fts5ab.test index 7e312286f..a74c0f888 100644 --- a/ext/fts5/test/fts5ab.test +++ b/ext/fts5/test/fts5ab.test @@ -294,6 +294,39 @@ do_execsql_test 7.0 { INSERT INTO x1 VALUES($doc); } +#------------------------------------------------------------------------- +# Forum post: https://sqlite.org/forum/forumpost/ea4d8c9acb +# +reset_db +do_execsql_test 8.0 { + PRAGMA encoding = 'UTF-16le'; + CREATE VIRTUAL TABLE vt0 USING fts5(c0); +} +set v [db one {SELECT x'2a12'}] +do_execsql_test 8.1 { + INSERT INTO vt0 VALUES ($v); +} +do_execsql_test 8.2 { + SELECT quote(c0) FROM vt0 +} {X'2A12'} +do_execsql_test 8.3 { + INSERT INTO vt0(vt0) VALUES('integrity-check'); +} {} +reset_db +do_execsql_test 8.4 { + PRAGMA encoding = 'UTF-16le'; + CREATE VIRTUAL TABLE vt0 USING fts5(c0); +} +do_execsql_test 8.5 { + INSERT INTO vt0 VALUES (x'2a12'); +} +do_execsql_test 8.6 { + SELECT quote(c0) FROM vt0 +} {X'2A12'} +do_execsql_test 8.7 { + INSERT INTO vt0(vt0) VALUES('integrity-check'); +} {} + } ;# foreach_detail_mode... diff --git a/ext/fts5/test/fts5corrupt.test b/ext/fts5/test/fts5corrupt.test index 0abd8b86d..8788bc2ed 100644 --- a/ext/fts5/test/fts5corrupt.test +++ b/ext/fts5/test/fts5corrupt.test @@ -47,11 +47,10 @@ do_test 1.3 { DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', $segid, 4); } catchsql { INSERT INTO t1(t1) VALUES('integrity-check') } -} {1 {database disk image is malformed}} +} {1 {fts5: corruption found reading blob 137438953476 from table "t1"}} do_execsql_test 1.3b { PRAGMA integrity_check(t1); -} {{malformed inverted index for FTS5 table main.t1}} - +} {{fts5: corruption found reading blob 137438953476 from table "t1"}} do_test 1.4 { db_restore_and_reopen @@ -61,7 +60,7 @@ do_test 1.4 { rowid = fts5_rowid('segment', $segid, 4); } catchsql { INSERT INTO t1(t1) VALUES('integrity-check') } -} {1 {database disk image is malformed}} +} {1 {fts5: corruption found reading blob 137438953476 from table "t1"}} db_restore_and_reopen #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r} diff --git a/ext/fts5/test/fts5corrupt2.test b/ext/fts5/test/fts5corrupt2.test index 6b4d6d411..fd2a841c7 100644 --- a/ext/fts5/test/fts5corrupt2.test +++ b/ext/fts5/test/fts5corrupt2.test @@ -109,12 +109,12 @@ for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} { do_catchsql_test 2.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check'); - } {1 {database disk image is malformed}} + } {/1.*fts5: corruption.*/} do_test 2.$i.3 { set res [catchsql {SELECT rowid FROM t1 WHERE t1 MATCH 'x*'}] expr { - $res=="1 {database disk image is malformed}" + [string match {*fts5: corruption*} $res] || $res=="0 {$all}" } } 1 @@ -160,17 +160,17 @@ foreach {tn hdr} { close $fd set res [catchsql {SELECT rowid FROM x3 WHERE x3 MATCH 'x AND a'}] - if {$res == "1 {database disk image is malformed}"} {incr nCorrupt} + if {[string match {*fts5: corruption*} $res]} {incr nCorrupt} set {} 1 } {1} if {($tn2 % 10)==0 && $existing != $hdr} { do_test 3.$tn.$tn2.2 { catchsql { INSERT INTO x3(x3) VALUES('integrity-check') } - } {1 {database disk image is malformed}} + } {/.*fts5: corruption.*/} do_execsql_test 3.$tn.$tn2.3 { PRAGMA integrity_check(x3); - } {{malformed inverted index for FTS5 table main.x3}} + } {/.*fts5: corruption.*/} } execsql ROLLBACK @@ -209,7 +209,7 @@ foreach {tn nCut} { set res [catchsql { SELECT rowid FROM x4 WHERE x4 MATCH 'a' ORDER BY 1 DESC }] - if {$res == "1 {database disk image is malformed}"} {incr nCorrupt} + if {[string match {*fts5: corruption*} $res]} {incr nCorrupt} set {} 1 } {1} diff --git a/ext/fts5/test/fts5corrupt3.test b/ext/fts5/test/fts5corrupt3.test index 437c4842c..eab4c3c91 100644 --- a/ext/fts5/test/fts5corrupt3.test +++ b/ext/fts5/test/fts5corrupt3.test @@ -102,7 +102,7 @@ proc do_3_test {tn} { list [ catch { db eval {SELECT rowid FROM t1 WHERE t1 MATCH 'x*'} } msg ] $msg - } {1 {database disk image is malformed}} + } {/.*fts5: corruption.*/} catch { db eval ROLLBACK } } } @@ -273,7 +273,7 @@ do_execsql_test 6.1.1 { } do_catchsql_test 6.1.2 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corruption.*/} #------- reset_db @@ -289,7 +289,7 @@ do_execsql_test 6.2.1 { } do_catchsql_test 6.2.2 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corruption.*/} #------- reset_db @@ -308,7 +308,7 @@ do_execsql_test 6.3.1 { } do_catchsql_test 6.3.2 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corruption.*/} do_execsql_test 6.3.3 { ROLLBACK; BEGIN; @@ -319,7 +319,7 @@ do_execsql_test 6.3.3 { } do_catchsql_test 6.3.3 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corruption.*/} do_execsql_test 6.3.4 { ROLLBACK; BEGIN; @@ -330,7 +330,7 @@ do_execsql_test 6.3.4 { } do_catchsql_test 6.3.5 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corruption.*/} do_execsql_test 6.3.6 { ROLLBACK; BEGIN; @@ -341,7 +341,7 @@ do_execsql_test 6.3.6 { } do_catchsql_test 6.3.5 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corruption.*/} #------------------------------------------------------------------------ @@ -374,7 +374,7 @@ do_test 7.1 { db eval BEGIN db eval {DELETE FROM t5_data WHERE rowid = $i} set r [catchsql { INSERT INTO t5(t5) VALUES('integrity-check')} ] - if {$r != "1 {database disk image is malformed}"} { error $r } + if {![string match {*fts5: corruption*} $r]} { error $r } db eval ROLLBACK } } {} @@ -399,7 +399,7 @@ do_test 9.1.1 { } {} do_catchsql_test 9.1.2 { SELECT * FROM t1('one AND two'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} do_test 9.2.1 { set blob "12345678" ;# cookie @@ -411,7 +411,7 @@ do_test 9.2.1 { } {} do_catchsql_test 9.2.2 { SELECT * FROM t1('one AND two'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -497,7 +497,7 @@ do_test 10.0 { } {} do_catchsql_test 10.1 { SELECT * FROM t1 WHERE t1 MATCH 'abandon'; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- # @@ -678,13 +678,13 @@ do_test 12.0 { | end c2.db }]} {} -do_catchsql_test 11.1 { +do_catchsql_test 12.1 { SELECT * FROM t1 WHERE t1 MATCH 'abandon'; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} -do_catchsql_test 11.2 { +do_catchsql_test 12.2 { INSERT INTO t1(t1, rank) VALUES('merge', 500); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- # @@ -870,7 +870,7 @@ do_test 14.0 { do_catchsql_test 14.1 { INSERT INTO t1(t1) VALUES('optimize'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #--------------------------------------------------------------------------- # @@ -1040,7 +1040,7 @@ do_test 16.0 { do_catchsql_test 16.1 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #-------------------------------------------------------------------------- reset_db @@ -1126,7 +1126,7 @@ do_test 17.0 { do_catchsql_test 17.1 { SELECT * FROM t1 WHERE t1 MATCH 'abandon'; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #-------------------------------------------------------------------------- reset_db @@ -1435,7 +1435,7 @@ do_test 18.0 { do_catchsql_test 18.1 { INSERT INTO t1(t1) VALUES('optimize'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #-------------------------------------------------------------------------- reset_db @@ -1546,7 +1546,7 @@ do_test 19.0 { do_catchsql_test 19.1 { INSERT INTO t1(t1) VALUES('optimize'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #-------------------------------------------------------------------------- reset_db @@ -1630,7 +1630,7 @@ do_test 20.0 { do_catchsql_test 20.1 { SELECT * FROM t1 WHERE t1 MATCH 'abandon'; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -1764,7 +1764,7 @@ do_test 21.0 { do_catchsql_test 21.1 { DELETE FROM t1 WHERE t1 MATCH 'ab*ndon'; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- # @@ -2100,7 +2100,7 @@ do_test 22.0 { do_catchsql_test 22.1 { INSERT INTO t1(t1) VALUES('optimize'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #-------------------------------------------------------------------------- reset_db @@ -2211,7 +2211,7 @@ do_test 23.0 { do_catchsql_test 23.1 { INSERT INTO t1(t1) VALUES('optimize'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #-------------------------------------------------------------------------- reset_db @@ -2429,7 +2429,7 @@ do_test 24.0 { do_catchsql_test 24.1 { UPDATE t1 SET b=quote(zeroblob(200)) WHERE a MATCH 'thread*'; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} do_catchsql_test 24.2 { INSERT INTO t1(t1) VALUES('integrity-check'); @@ -2518,7 +2518,7 @@ do_test 25.0 { do_catchsql_test 25.1 { INSERT INTO t1(t1) VALUES('rebuild'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} do_execsql_test 25.2 { PRAGMA page_size=512; @@ -3011,7 +3011,7 @@ do_test 27.0 { do_catchsql_test 27.1 { DELETE FROM t1 WHERE a MATCH 'fts*'; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -3700,7 +3700,7 @@ do_catchsql_test 32.1 { highlight(t1, 2, '[', ']') FROM t1('g + h') WHERE rank MATCH 'bm25(1.0, 1.0)' ORDER BY rank; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} do_catchsql_test 32.2 { SELECT * FROM t3; @@ -5351,7 +5351,7 @@ do_execsql_test 41.0 { do_catchsql_test 41.1 { INSERT INTO t1(t1) VALUES('optimize'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} do_catchsql_test 41.2 { INSERT INTO t1(t1) VALUES('integrity-check'); @@ -5573,7 +5573,7 @@ do_test 42.0 { do_catchsql_test 42.1 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {1 {fts5: checksum mismatch for table "t1"}} #------------------------------------------------------------------------- reset_db @@ -5813,7 +5813,7 @@ do_execsql_test 44.1 { do_catchsql_test 44.2 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} do_catchsql_test 44.3 { SELECT snippet(t1, -1, '.', '..', '', 2 ) FROM t1('g h') ORDER BY rank; @@ -6644,7 +6644,7 @@ do_test 48.0 { do_catchsql_test 48.1 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {1 {fts5: corruption on page 1, segment 1, table "t1"}} #-------------------------------------------------------------------------- reset_db @@ -6917,7 +6917,6 @@ REPLACE INTO t1_data VALUES(1,X'2eb1182424'); REPLACE INTO t1_data VALUES(10,X'000000000102080002010101020107'); INSERT INTO t1_data VALUES(137438953473,X'0000032b0230300102060102060102061f0203010203010203010832303136303630390102070102070102070101340102050102050102050101350102040102040102040207303030303030301c023d010204010204010662696e6172790306010202030601020203060102020306010202030601020203060102020306010202030601020203060102020306010202030601020203060102020108636f6d70696c657201020201020201020201066462737461740702030102030102030204656275670402020102020102020107656e61626c6507020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020201020202087874656e73696f6e1f02040102040102040104667473340a02030102030102030401350d020301020301020301036763630102030102030102030206656f706f6c7910020301020301020301056a736f6e3113020301020301020301046c6f61641f020301020301020301036d61781c02020102020102020205656d6f72791c020301020301020304047379733516020301020301020301066e6f6361736502060102020306010202030601020213060102020306010202030601020203060102020306010202030601020203060102020306010202030601020201046f6d69741f0202010202010202010572747265651902030102030102030402696d010601020203060102020306010202030601020203060102020306010202030601020203060102020306010202030601020203060102020306010202010a7468726561647361666522020201020201020201047674616207020401020401020401017801060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102ad060101020106010102010601010201060101020106010101010601010201060101020106010102010601010201060101020106010102010601010201060101020106010102010601010201060101020415130c0c124413110f47130efc0e11100f0e100f440f1040150f'); INSERT INTO t1_data VALUES(274877906945,X'00000e96023030011a042319320d3b123d812b5a31120110446e581b66814a05010a4537814274010e8102815c810f3d0104846d01081581204401103741043c59416b44010a404655265301103f73811a11114213010a821235820f020135030484320201360104816a020162020484550302390301710a04824a020166030483690201670704837d0201690404822602016a0504825c02026b620504817502016f0904810d0303e79c88060482760201700a04826302017204048155020373c2be050481130201770204846202027962050482710202c2ba010482140203e58496070483330204e8b2b879010483710101310110545c0c814b0e3a6501082c815d5b011a2a0e2f0d765c3d686014061d0d0112810733112c2e82141101048313010e5c6f632e813e42010c811882370548010e19158146822f1f01104d364a708146135a010a237b0a55210201610904841703027678090481270201620304810002026374060484660202657a0704827602016601048351090483540301660704814b02016b03025f0304c582caba0204816602016c01025f0302cebc0904843e02016e0804821802016f0a04817503016f070483100201720304822c020484380201740404842e0102460201770104812f0204836c02027a6f040483110202cebc02048267040161020484650205d5bd62cebc0604845b0204f2a580880204842a0206f38184a179670502750204f696a3aa0a04814601013201063330390110812281378114600d010c03716c5e822d010e81226b542a814d010a72740f83000108813a1e0b010c5681046f812c010c07814a777328011664244219531b1a2f811e4a010c4d81557c7f1b0201300704810702013307048230010484050202367807048175020239710804832502016204026b0204814d020363306108048262020265650504817602026667070483150201690704832f0301360a04814d02016c08024702016d0304843e0303cfb2630204814002016e0804837503016203048416030370c2be0304821b02016f0604834b0201700504816b030175070210020273660604841c020676c2bac2b2640604830a02017704027d02017808048141010482700201790504811d0202c2ba0502470206ca8d73ecbab9010483340204f09e9ab504048367010133010c3e04814f82250114812e814b2e0411811305010c811337811e6e010c82085e2b0e5d010c61812054811c01148122451b0781050c813d010c17823762643e011e080c1720814a10364306143b0d33260112810f0c2a810c816b13010a8163810e470201370404811102056176c2aa36050481530202646e0904846202026a730404827402016b0a020c02036f616b0404830a0201750504820d02017605025e0201770a04820702027a73050482460202c2ba0604824a020483330203cba434040483200203cebc790304847e01013401124181442c1d091f81580108601d8336011081320a2b8125820001123b0b81158116811f070110078112817a817308010e6682410d810e2601122d0d6413378147351e01105081021d3525812d01128246510a622204054101105c1b620e81302b05020130020483480104822702013102027e020132030483270201350304844802023770030207020261710604823f0201640802570204830a020265770304831f020168070483210201690204825b02016f020481280704835402057037e18b8d0904810802017304048439030172020481440303c2bd6b0a02630201760202490804815e0201770304816e050483550201780704816902017a070483280207c2b2f093aabc780a0482240301b303024e0301bc0604837a0202cb800204834a0202cebc04048201040484410203d3ad770a04814f0202d5a508048371010135011630817e0f81040d2c041552010c813d3b7e8115010c40692182693a01121f810d810d0a32814701101d1d1f642281742e01068229240110811231810a387f4c01100f50810f8165810d0114811f26443152593c104a010e641e1a3357820e020132030481540201340204815402023778040207020163060481020204815b020164020483010201670a0481540304f0948f870904811a0201690704835502016d0604832203027b01022803017a0a025102016e020484260404816002026f690502680301720104834e02017208024f02021d020475e69c8e0504814c0202767602025f0302de870a04837b020178090483200104835003026a72090484400201790504837204022302017a0104836b0202c2aa0a0481070303bcc2b3090484370203c7866403027501013601087c158303011212814305813e7b0e090118141a1c49713a211e0c74630f010a59826d8113011203328166037781561a01101d7f1d2a1f822533010e820e070f7b40160110811f40292c813226010a2d20824a32010a81418158670201300304840a0201660404817d02016d0804826902016e080626817f0104820b02016f0404825a02017003023b020272750804840c0301760a04844d02017403025b020175080484200202766e060482360303c2ba6c030482220202c2aa0204810e0301b20204835703048421040484240301bc0802410202d2a1010482630204e1b18f3704048354010137010c08816337812f01101382211532424d39010881248123010e7724810267815f011081236029813e273301101b7b29812a5b813b01128150810324814b220b01060c8417010e165b6c81708117010a1782346f6c0201380804816803016b0604840102013908023f0201610204816c0201630302760201640304832604023e0204833902016502048203020266770804821a0201670a04830002026964080484500304f29e9eb70802250201700204811b0201710904832d0201730304826b05048403020174010481000205776a62c2b2050482630201790202260206c2b2eaaeb464050485020301b3080482480303bac7af0a0484550203c695650804822a0202ceb90a0481170202dbae05023f010138011819814a2703390a61090c6912011a21181304812523811b5f164e050114110f35128123423f810c010c817573817c03010e7182590c812b0401142503597e6e0e2f3a3759011252813a811a2b75091a010882162a31010e17450a81048279010858658208020231670a048205020361c2b906023b02016301048236020164050482520201650904833d0201670904811b0201690604825002026a7a0604837c02016e0204832002017002026d0302c789010481020201720504835e0201740604810002017502020a0702630302676306025f0303e4a0a70102640203786a75010484440201790104841402047ac2be72070481340207c2ba3766673576090482790301bd07048142010481600202c7b309027a0401740604823b0202d2bf0304830f0204e989a6300a02600204f4bd91b60702120101390106518369010e19254641823711010c258267288121010e817c810d2b17250110810a578133812f4c0110415681067b288121010e0881208119347101140b8131543c8100343d1101088203813e01100d742e3230820f3802013006048107020134080481440202356501021a02013808048147020162020482230201630304833c030162050483390206656cf093b5bd070484140202677303048502020769e3ad9669c2b90304847402016b0804836402016c0404841f02016d010481250904825202016e06025402016f0704842a0201700a04834a0201720a0483530201750304822e020676f097b18374030482140201780604833d0202c2b9010481550301be070483720204826f0202ca80060481630202d5a80504833f0101610114551047810e130a78660c011a364611206c0b13080705733d5501240f08070c090b0c20813d1471042e4351131e011204412d814f0913104201263036110d060b1f811a301b0f4e1a29092f181c012808071e221a2a81075b320503065a0f140c1a0a26011c07231d0e6f3715063b760c6b091501121111303e3a71566d6d010e0867814d816a0c01181e18240d41724d221b3f384b0201300204830b060483080302c2b30204837f0201310a026e020134050481560301730902690201360406827c0f0201370204825d06020a0201390904815b0304f3bfb2a70a04822c0201610404810e020262660604841a03017608025b02016309088112825f020164090482310201650a0484480302396f01025d02026774010482090302df9b06048321020168070661813303016f03048248020169030483610504814001048401010483460301300704824203016a0204824402016a0504813303021d03017209048412030277380804824502016c0104814c02036ec2ba0804835702016f0204811d030176020238020270360a04840d02017107048201030469ca99690602350201720304812d0104845e02017304022c02017409022b020175050484140302caaf0402410201760404831f020177010484180704845203026f6d04024b02017a030482660202c2aa0204810a0301b30a04817b0302bd76020483780302be6a0302440202c58207025e0202c69901027a0302ad77010483200206c993f099b183070484270203caa1660204841a0204f29788ac0804831d0204f4ba9f950504843c010162011c0e33810216341c2413042130780501184d373e53131f2f052907423e010c830e3781390e011c1320461f81041b811b041e15243d011e241b10816c310b130c3133033b0741011a11816d3139100c13140b395848011c580e411a06304306810a3138330d011a441707092c70140c1643813920010c73653581374f010c826c81210f0402013803024f07048172020161020481650204810c020162030483470301370404813602016307048379020483280201640304815501048176010481060201650104812a0104841f0201660504821f0201670302700201680804846403016b0a04831d0301700904845502016908025c02016a010483560904827602026b6306023402016c0404832602016d040484410204825702016e0504831603027831020482160302c2be0504827d02026f6a05048121020171030232020483220304845402023a0201720204845c0304846e03013607048224020174060484480201780704844303016f0604814f045807070a0707070707080709070709070808090a5c07080708080b07060a06080707070b0a0b0808070b0a0b0a55070b08080a0908080707060709070709070706080c060b07070c0a66070b08080609070607080c0909660b06070707080a0807070b0b0707080a0b07070d0607080c0908630707070b07070a070d060b0707090a07080b080a070809085f0707070c0706080706070809080f06080a5807070607060e070807080907070b070b060c0709090807690808070707070708070608070709070809070a0d0b07070809095b070707070707070c080d07070b06070707070c07080b0808811a0b08060706080a070a07080609070707080808071307070a0708070907060807090b06060707070b0707080708070707080c090a0a81080a0b07070b0f0b0706070707060b07070b07080808110b070707'); -INSERT INTO t1_data VALUES(274877906946,X'00000e880330627a020482240202c2aa0a04833f0301b30704844b0302b9650704824f0301ba0204845f0202c9820a0483640202d194060482300203e19cbd0904844b0203e691b4050483510205e78dadde9b0104821201016301142a6c033b8151085c094601140b813d49313f81110e1c011681163611221527257f5d38011c150f22811a0a3c12350631238117011c3e26420b402c1d81080c40150b2f01181c3143382640273d60132e070118663b1d162a1b0e2e8111393e0110821117310e52811c01141a2f49810181391f2b130112323c0305812a6f2e390201320204842702023334020481340201350202610201360304844603023362040484470203376a360a04826b02013808026203016f0704830d02013902025502016106088170827a0301320a04820403016c0404831502016204048327050484030201630a04814302016401048349020484760302430301640204845d02016504048249020367c2bd07020a0302cebc0902150201680a02500201690204846102016a06024c0301770504842b02016b0104830e0704811803023370040483580301710404845f02016c0504844f0204820d0204837f0302c2aa0104833702016d0104844c02016e0804834f03026c6a07025702046fecbd9a01023a020270330204830a0301740304837c020271350204811e0201720706833b310206736ef09289b70104832b020174070483290301320204827c02017608022802017806025b0302c2b30904835202027978080483040303c9b56f0904846a0202c2aa04048127050482120301b30504813a0301b901024204016a0704840c0202c5820704823c0202c999010482470202cebc0602400203db91670602730202dca7050482760203e1a3950304817e0203e786a702048273010164011a0612105b292b817c1211080d5a01147c1d420b35451c36811a011e0e168117081c0c2e051d474055192d011e02050a1c81180420250f815f300f21011c02316a37143321443a10042d54230112761428810e4750054101101805072b8215294e0116680f0f5381445a3e0b070901224e4a41210c361c281b101c43051325130f01185a1e19108106300f2e3f4538020130060481370202327305022002013405048168020335377a03021802013704027b02013804048260020161030882118101030263650904814d0201620604822502016303048419030135090483240201640602280301380404817e0201650404823c0304f097ac9f010482680201660a0267030566e2b6936f0104821c0201680704813302016b0604832002016c01026a0301610204843202016d0804845e03026d6d03023c020270730a04817f0304c2be797a0804832e020271710504835f02017203021b0201740204825a02017706027202017805048451010683572e03016e0a04814c0201790304811702017a080484450302c2b90204837b0202c2aa0604825b0303b273630104841c0301b9050485040301bc0a027b0303bd37680502670202c98b0204826b0203cfa1740504823a0203d199610202350203e3bf87040483570204f1baaba90504817301016501120b8104392d0d20180f011645213f292e4d0d082f8165011e0b400c07341b2329307f193338173a012055292409050c560a272a0f4403245718011a1c3a183f1c43264c3126060829012081208102043a044d0621650b180e150e011a066c030e513d7d265e1313130c0118171953040457347b114d191901261b1c060c26090d6f0d332a1519096e03101d1d012207342f1f2c7e2517251d0f310d2a17081e02013005020a02013308048247020135030483660201380704841b030132030482180201610102600604825f0304c2bdc2ba02023a02016209021f0201630604813002056663cebc610604841e0201670504816a0104842703037177310604833f03027872080482350201680604825e02016a040483320104840d02016b0304813f02026c6408020602016d070481590104837d03016c08020d02016e060484630301780404815c02016f0104826b0804825e02017008027b0302c2aa0504847c0201710404836d0201730402510302cebc08048338020375c7bf05048344020177010482660304822303026479070481630204786ec2ba0204814f0202796a0a04834f02017a01048407030484660604810603026561020483180204c2b278390504813d0301b304024f04026536010483110302b9330604813e0301be0304840b040484560202c4a702022d0206c6a5f7ada9990402350202c79f090481180202caa60502140204cebcc2bd080483320202db900a0481250205f4b5aa9079040484360204f7b985bd0204835e01016601128101285c096981190e01121f813f0d431a8135530114698102813228492f190a011260161881328101812601188155780d813257050c0b04060114161681340772811b5e25011c4505810e13290b253a0c0c0a1a4b011a3714133e1235812b136b062c0b011a6b591356810c3c240906250b1001148127810d413e0e81090d020231680504822c0302c2bc0204830c0201360104840c0303656a740902110302d1950504824f0201370902130201390a0482530303e0a9ae0904844c0201610804810d0201620a04810c03023039030481330302356902048268020163060483470201650504822d02016706048200030483560302713509025002026a79010484410804825902016c0504822002016e07020d0304843a02016f0204842d0201710304837604048361030482430201720904840f020173020482520804810b02017406068425320201760202420201780804845d02037979650704814802017a080481110301780804812c0202c2bd070484600202cebc0508813082410102770401610202250202ddb40302310205f19e9a937a030482410101670118365558195a0a062d0581260a011881068143330844041f0a1851011a2025141e1081204f550e077521011a193b1f58351912265681220821011812070528472f4e2f407f204a01124d1e1f811b810d7b4d01180f1d3481034a35580a12811f011c2303340d1470150778070c812331011620411939703c032915143f01104281116a3d323c6a0201350604842902013703023f0201390a04816e0201610604826402016202023f0201630a04843302016409048258020165020482620404814702016602061b833f0201670404827d03026369020227020169040483490301670604847602016a0604845d0404840902016c04024601088150822b010482350204830a0301690304844902016d0202710704820002016f03048509020482380304836603026e74010483580201700504817a020171020481080204826a0201720304837b0202410302160302c2b3080484550201730904816e020174040248010484280304834e020275730404836e020176060484720104815c02017709026a020178060275040483790201790504821501026c030170080483770304cfb269710704815102017a0204813303016f090483160202c2aa050481400301ba0804810b0401630804830d0301be0604844d0202c8a30a02110203cba0640204816a0202cebf060482420204e487856e030481080204ec97bd6d080484080205f09eb3a0770502260208f687999931ed878703048424010168011a12460e090c036e151b812e065501161708411982151738471f35011a2d1c0678340c1f04425c21200c010c2a087f255d4a011a0d0b6c33814a212c3a0a401b1e011c501f2381010a0481201c0c6012280118150b5228520e0a036c1c8123011a15810a060408030a81563f381601185b1b06212a143f332a60160e011a221b1e62411d2048090e0b0f5502033072350804826a020131020482530201320304823b0201330104814502033677380204813102026174020483540304dbbf6f620404835d0201620504846a0104831803017103048323020164060483740201660a02410201690102130104821402016a0504823f02016c0404832e02016d09022602026e640a04822702016f070482000301750402670201710304813106020d0202726304048220020173060484530201750504831e020483400302c2bd0704843a020177020483470203786371030483740201790904810002017a0302300202c2b9080483280301bc010481700303bd33720304825e0205c99973c2bd040483160203d5a6330a04842b0204e7b3b3300904813c0205f099a68f72090209010169011c21101d4b2d0e0e066b4253074c140118070a0910447556030833541d01163733816837402b3909122501183c5b1139102e2d430c662334011e27050f21621230323503332b6a0332011e1e07031843202e6e3c2850094d410c01163955220b16812d24521212011681250b0a3505460481176f011a2c09266b162968051c0a1481170116022e1e820c352037263a070201310104825d0602110303696869020484070201320204826e0201610304832203016f040232020162050642843f03048336020484540301370804833c03026c6105027a0203636165090483120201640502770304833e03017107022f0301780702470201650a04811e020167080238020168040481160102230404826b030170040483000201690304836f0302766c0304811402016b0504812e02026c6108027702016d0308827d81530604837302026e790904842602016f06048208020170060481680302320201710204812902017307048255020274320104822a02017506026803016e0a04821303017207025f0302c2b90504834a020177020483130201780604836b0402210301320604847302017a010483130202c2aa0804823c0301b9030482600301ba0104845304016c0504837e0202c3b8080484600204cf9d6379020483660202d3860704812e0203e3a4be0402560203e58784010481210204f09e95ac0102580204f5aea5890a023301016a01123428131a1f6c81445601141e227c1a7b5f1918810301182318812e17455605460d811c011a28820221311a6e12093f050a0c0120082c0b0f1362074457460c3b070d5132011c2143052a20133d160a358117591f01103136813b136e6247011c100e4c28060d16815a320a3e11070124462c03582e262d45110804113326040808070807080809090b81050708060708090607060907070b070e07070807060706070b08070f0807070709080708080c070706060808090c07060708080708080909811307070708060709080707070607070a060b070706070707080a080607060c070707080809070608080908090a812406070707070a0906070b0b0908070b07070b0607070b0608070608090b080a080f080a0608080b070b08070a080b0a810408080708080607090707080807070b070c070a070f070b080607090707080d06070b810b070607070607070b08070707070b14070a0f08070b0d08070e080b060a0a070a0707080707070709080a0a0a0e810f0907070709080a0b0707060a0707060807060a08070b08070907060807090b090a0a81140a09070706100707090a060607060e07070807070d08070a07070806070608070a070708070707080a0808090909'); INSERT INTO t1_data VALUES(274877906947,X'00080e7f073c23110a1a18392f66090524183704276d6703306a320404824e030164080483520305c2bd7ac2bc0604815a0201360704833202016106021f020482400201630304822a0708817e8204030173040483500201640404824803016d0804824a03017709023002016606048367020268680a04815802016902088339811804027f0302656e0704834e0303d5a5370604816702036a3366090484470303c2ba660904826e02016b0904837c02016c07021403026c610604835802016d0204816802016e0104831202016f0104822f020270720602060201710704822202017206048174020273690204824602017409020c020175090482140201760a0482720301660404824403016a090484290201790404845703025d0203c2bc33040484620301bd0304824c020484540202c78607022403019a010482380202ca87070484390202d39d030485050203e184940404831b0203e6a881060483480203e8b18c0a04816d0203ee8d850104814801016b0110467257393c81272c011a053e815d3b190517064524521f011c3823590a8115372004313b1f3216011a5a20780b102d0804426916112c011a182f810781082d12137026161501221a180516811611051c131207811515173501180320112581062e05621c1407011c2d0e0617811522062208065a21520114582841621e6c203f1e2001161647411a272533815b1c2602013009048309020232630104835a0301720104817f0201330604836f0302ddb5080482560202347a07048102020135020483460104827b02043678ca800a04835f0201370404814b0104846002016103048246010482220301700204833f0201620404824d060481150201650304824f02016606088110834c0201670604821d0303c2be66010481790201680404843b030176050482270201690a04830e02016e0904844202016f040481010301630304822f020270640204822f03016e0704845802027177090482710206736ec2b2796a0104832e0306dab1d485377004048304020174050481700201750a0212020378627604048164030173080483190201790704833d0204823a02017a0506820e67030178070484530202c2b2060481500104823f020483030301b3020484310301bc04027e0402caaf0a026a0301be040482590204842e0202540202c79f0804824d0202cfa30804815a0204f29a92970204823301016c01140f63351a0a653b650d22011c09117a3e1538123537046a15043101141310082f49052f772b0c011c11121781583c2a5010133228241301287f3e0a2b1244080503060a100f413b4f0d070e2a01103e4e1f04814e7b1601183d0404052877111f230f811d01123a100f053e5c076910011a031732102381243d1b1727507301180e5d273e810803812e0f192a02013301048271070481330204821d020134080263020135060481280201610104830a0201640604826d020165060483050201670204841c0504841c0304841b0201680a04845602016a0104811c01024f030481080204813102016b0204837008024502016d0404836c04068207780301670704842302016f0404821203016d040484490301720404837e0201700104821d03048407030165050483050201720a04811602017307023502017407020503016f080484240302c2b90504821b020175090484090201770a088119822503026d6905048300020178030482680604812a0201790104830c0204833a0303d9a06806022002017a060482600203c2aa33030481560301b904020a0301bd0504820f0202d0b90904817c0202d3820202200202daa9080482030203de966e02024b0202df9d080484350204f098b0a20604845e01016d01220304456608322258060a031d4c38340f090112310c070e4238626e6601124a318109030513812f0118240d561e533742188113101b01160b24444b224d44814d4806011c05774e483410330d23541b28090401141f29062581131e221b6d011e81053a037a03320b0e4c24360d2310011a0e321d3c141825111d54637a1c0114093d3c2e58571a35293a0201350104840a03017701048330020136060217020138030483370201610a0482650201620504815e0201630704827701048201020164080483690201660804846703016904048113020167070483080201680504837d05022c0302cdb10a04815f0201690104833a0404824302026a360a04823b02016b0a04813502016d0504831a0204833803021a02026e360404825e02016f080484140201720304844b0404816603056ff09d899b0304823f020275390204816e0301780a04824202017604088308812703027902027770050482040201790104827e040482750204812902017a060483030304c2bdc2b30104836f0203c2aa62040484040301b903021c0302bd6b090484300301be0704814d0202c99402025603049a65656b090484020202ca92090482060203d19a730504844a0203d49f690804836e0202dfa8020482710204f09180860704822901016e011a0c0b8104243647521f43231f36011a2e1b33432c3d0b414905054d17011010573a6c0a816c1801160e063582340a5239050b06011a4481063d1b67250f2044200839012044591d1857291214135814101a1b361d011225067e8147111a4a4301166b13362e17195f3812186f01141c465b032b290406373301182a152a2281300f8107054e3f02023274080481770305c2bacf8168020481450201340604832f0201350704842e02013605020e0201380404841d0201610404810d020483750201620304812b020484230301610804834503026c6a0304816d0201630102380305613577337405048359020165040482720201660904826202066736f094b0af0a0482250201680104811f02016b0304847202016c0404822403016f0904822c0302c2b301025002016d0504817b01023f02016e020483090802040303e7bda10804832d02036f6b740404811402017005048419020484220202716506026303026b760904830a020172080482430304706c73620504825f02017308048413020174080481070201760104827f0204836e020477e7b89a0104840e02017a030483700206c2aa35657065050482740301b30804842b04046cc2be78090481040301b903020d0301bc010484260904813f0203c7a5620302330203c99f36050481010301a30704815b0202ca8b090483250202cdbb0604820a0202cebc0102170401380304842b0207eca2a6f29c87950904824001016f01221d17052b58101241060e3a201f1021633a0114816919811c142443100801280426080e2620042a812c531a490e121707131710011273432e493347811a340112195f671f46721c325e0118380c052b812822478107600b0116021c21821b2019263433040126021b05351b2a286b05181f071b5628111a330a012014533e073d0c0e5469141d1e2734050901220318051b44412803632e0642370e0a3a2b020131070481770201320a04812f0202346e04022b020136030483590304f09a81b60404834702016105048210020162030205030167010268020163010481540604820202048300020264310902420201650804834b02016703048247030365c6b602048205030573f098b890030481450201690204832802016a0a04826703016c0104825e02016b0604815e03016c03048334030677c2aa74c2bc09023d02016c0304823903027777060484540303df866c0104815b02016d0204811f0303796f7704020302016e0204814d07024a02016f0902680201700604840a0104831a0204835a020172070484440201730a023703026b660604830e030278790304815b0201750904822402017704025b020178030482350307f4b2a3896a343407026b0201790902720302633409020402017a020484590302dea004025f0202c2b20204816b050481200202de900402160204ee85a5770204822c0101700114143a0d391a60812d4e09011a2f313104201c372c3a3411321b011a268140144226334145050d1c4d01164e081f20671f088107237901186b123c1f6d07261e2b732e210116511116342a3d32376e083001106882257a0a17141101163039192b0c05812d735f3b01262a3e0841030b17181411051e0a18530e272b6d01182f322b260e24581d5381050f02013104048353020132050482370301690204843e02013302020d0201340104841f0201350a048139020137050482770201380204833a02016105025a0504832f0201620304836d020163050484100304832003016c030481290201640304837e020482490304822b010482290201660a04827002046964dc960204833102016c0704814502016d010482000201700104817e02037176760904821f020473eb91a708048152020174090482770201750404831e01063a825d03017a0904826a020276730a0254020177080260030277630104815c020178070481220202020301720804841202017a0204834c0202c2aa040484010301ba080482580202c6a3020481320203cdbf690502790202ce90070483140301bc030481470205d1a371cebc060481590203d2976a0404830c0203dfba6e0604814b01017101163732393b8120422f054b0e010e030b211d815d1c01165641757c080d81311d090e0112816581542d2313054301224e07121706516606080e39102d231c4b39010a2d81402d5e011a132527428114080d6e1111721c011a814a1a341538251023100d1c4c011e22182622623712411e38162a182d3b01142b67611981470f1f1f250201310a04824e0202336207048217020238730204815a0303cf886d06025a02033962620404833803016f0a04814e020161060483140302726d050483450301790904810c02036376690504811c02016403024d020165010483280802550301650904827f0304ebb8b561070482340201670302670301660804810e020168040482340201690704844d0302616404020f02016a030481060301700704827802016b070481240104814b0302c2bd0504816102016c0604837f03017a0404837902036dc2b90804810002016e0904821602016f0304812c03016401024b02017103048233060483600303e5848e04023a020172040481050305f3978aa06c070481151708070b070a0d0707070607080c080909090706080707070707070806070707070a090b070708080909090981140708070708080b0a0b0b070b070907090707070707070807080c0c070609070b0807100706070e08080a81110f06070707070f07120a0c070707070b0707060607080709080b0b080709060708070808080a810f0707060707070b070707070a080b08070e08070b0b08070c080f070a09060807070a080909080a810b080b070706070b0b0708060b07070c07070707070a0a09090b0708070a07070b0a070c070a060b080907080807070d8123070707070a0706060f070707090b07070707070b07080907080a060f070608080706070c060707070c070a810f07070706070707070a070b0713070a070707090a070c070706080a07070807080808070b0909810607080808090707080709060a070a060707070707070b080707090707060b0807'); INSERT INTO t1_data VALUES(274877906948,X'00000e8a0330717304048359030134050481100203756371070482190201760704817b0301770804821d0201770204844d0201780204836c0404826103017504026a0202c2aa020482420301b305022c040267390602570301b9080481540301bc0102290206c99cf6b5aa80080481430202cebc02026e02048120010172011a1c2f15158108048125463f251d010811811539011412423105812181171549011847284a30234e5b33042632120118351e8113817d0f2b220d111901264f104a211004061d0a2a0b35121a0a2118341f011c81160a1b030d2a0610243e445f0c011c6f0c1e3b1768141e322717500b140110537f810169811625011a492847203e210f532c16480627020135020481780302caae0104811a020261330904846c0201620804812d0201630404814a0201650704837503017301048276020168050213020169010484540604842b0302796f0504833302016a0304831b02016b0604701302016d0604815302016e070483630204815202027071020481520201710104835b02037266700704843002027362040481490201740904817d020175020481040304f59e9c9407048218020176060484060204776dc2aa020483070201780504812503016601048159020279790704840502027a730904826c030178030482740202c2aa04023d010483540301bd06026b0203cab877010483290202cdbf060482410202cebc04025c0401690204827f04016c0904840e0202cf880104835d0203dfbe6c01025b0204f0aeb7b2030481680205f1a7b5bb390504826a010173011a22810d12415003071f81181839011a3220221511546d810012052b57011a0c4274300d154e81111f041e10011e293f4213051b2276560817312811170120092136122418370e4e782b3912080f3201262b3b340f222b0c09142a0822116a135c1c130c0114320c4e385a0d0415075f01163543340f06362381133c0c012224180981742048191d110e180e180d310f011a20632450281f043027114b034e0203316a61030482160201360104826e0201380a0484570201610104844d0504814a0201620904820f020164010481630304810403027761040258020165080481550302c9b70204827a0204677367690102660201680104831b020169020483760301720a04814502016a04020c02016b04023405023c030269630504840502016c08048463020170030482670202716d080482000201720304835c03016507024d020174030482520504821503016505048207020175040208030137060481710201760a04833d0201780404832302027a6f080481200202c2ba0204812c0301bd0404821e0205c3a66865730702540202c7890804816f0205cebcc7af730504815c0202d2930202540202db89020481160203e8b8a00304825d0205f0958db331070481620305989b8569780a04813d0204f69299a5020210010174011a7b3829100a4e511f1a281c17140114812626032c372634234c01140a520e815a810815200501123e4f3531042d57615b011a041f3e64070f1f1913274a20770114811d0f5d743e0634161c01162c2782130c1b810520280d01164a513110480b402b810d13011e522d08042c1146137012201e810512011a290903182c05301e5d811944290201390a04836d0301610104836e02016208027803036cc2bc07048261020263640a0481480301730704813602016408022b020165050484310301720702260301780402500203666d73070484470201670104825b02016a040481590304836702016c0304835601025e02016d060484110301340204813702016f04021802017006068336280201710304813507025902017201026508022903016d0604812f0201740304827e03016a010482440201750404834b02017604021a010484150504836a02057773c2b2380602520201780204823e0302cebc080484040303d2956403048171020179020484240204813c0202c2b303048307020484410301be0204816f030484250203c798680104843d03019c030482570204e19ea86a020482350302a0950a0482280204e5a4bc780304810e0101750126090a35030a03220a1731630f31252f0c4b1e31011e39200e3715282a03103b56090f6b1501121d4916246e6d460d6501162609380406361e816d203f011a22166008124f58202e182025150114390f3a25713f0e3f715c011a5a11191123466025710c313312011e3c191326811c1444055f1f5109051201143b106f1181000d068155012043381381020d81080d0603171824260a0201300404844a02013207024203026b390204833803066eeebabb35660604842102013308020302023f02013503048243020436716b66040481440201380a04843d020261690704836002016203048209030484670201630a04841d020264790104822b0201650502340302c2bd060484300201670202620201680704810d020169010484430104843402026b67040481540306eea3ad77c2aa0a04836702016c020258040482270104830a02016d0404824102036e716a0604843e02016f0104832a04020c0204836803036530650a04817402017005025b0301630604843d020171070275020273720404827002017504048133010484120301370102270201760104814f03026203016d0904844802017701048375080481220301660202330301700a04827f020378c2b2030484710202796806023c0301730704813d02017a0404815f0104817202048407010483390202c2b204026802048254040266640204831e0301b3090482230302b963060482010401750a0483290306ba35f2999dac09020f0203ca926e090483350203cdb4780302350202cebc0a020f0204d7a7696d010483100206f097bc996d71040481480101760114185b2258291610821c0e01160272173107154f5b813722011a81020c200e1826250d39811f07011a7911152a2a45131504422c81070120050d3f5b23342e3e4139032a3813042d0116592d1c15630c0c0a814649011c1a362f5c4a35511f0804033e372b01102981262a352e8205010e0b4b6282388106011e26810a2d125f361a12170d1721311e0201300204832b0201320104811b050483790201350204832202016404024c0202657a090483710202666307022703016c07048362030277750604842f0201670304843d0104844c0203686f7a07027202016a0102430204847502046ce0a2b20704810302016d0204827b02026e330804826c02016f0104835502017102027606026c020172060483490202736505048371020174030484130302387602025d02017502048345020376346b0904825c0201770904814b0303c2bd720804812b020178040483600201790804816402017a08026e0202c2b90704811c0301bd0504821e0204cdbcc7a108021a0202cfb8080481490204f09f96a50204842101017701180207232d37812d0c045c4a0a011a06163b3408171c52213a26592201206d08581605811a171e0c0a1347104914011282181324082b73320f01122f6e811d2c3d410a44011e551414206a092f133f333d150a3e0f011e235b170e37060627471b13373b3e27011a0e1e816b270c10102d53381045011a060e1e254d044932651234691e011a158138300a04810c0a8121071802013003027707048214020131080481370201320a04835502013304048271020538ceb369650802310201390a0481440202616c0502570302c7a104026c0201620804811c02016301048168020241020164040481560104820b020165080484460102400202686c060232020269670604827302016a0108810c826e0704824e02026b6c0604816902016c0304831c02016d0304811903016407022b02016e01022e0604845a030237780702040301710804815102016f0104842903017a0704826902017003048445020482080202713002024e0201720804822003016202022302017303048111010482790204812d03067479c39f66700a0210020174020229020175090484430301690604820402017606020202017a01021b0104843a0202c2b90604842b0301bd030484570202c69b0504815d0202c8a30a02240202ceb8090484690301bc0404832f0202df85020481230101780116812b0a16810e4b045a3b2a01205b1305811134092f62072343100f0f05011e5734152612030b4c4134123009361601121781653207780a6a0d01164a25210824138107738139011481341f088158060c8133010e5920193a4c2331011a0510358101231a1b3609702732011a2f07631610033436810256174c011a1342040a58110721378139101602033067750802720201320604834303017109048244020133090481700203366f79010483520201390404810902016301048411050483420201640a048432020265310704832b02026774040481000302d5b2010259020369756c0504832902016c07048365030233700904824102016d010482670404834c0504830c02016e0104831604048120030169090481750303eba6990104835f02016f0604834c030379c2ba0a0481560201700404815e06048256020174040482370201760a04820d0201770604811c02017a0404812e06020b0202c2b9030481660301bd0a04816c0301be030483620206c99b6d7777750304835b0206ceb0646b66610402490202d8bf030482250206e8bfbc626964080482510204f09c9a9e0404830f010179010e2081335661371c01220e4e2718124f0d0649812b0b0a063b040b011402741d1235810805211a011409161d732b8106325f6a01182e330325068107703728302b011e3723081c0d0a3f810c183e061b067f0106834a12011a044030185a1e810704220a0541011245602b0e421441817801144b03811a1a29614e224b02013003048139020132050485050304f09caba6010482230201350104822708048413020138020483430201610402230201620902440108812181070301300304815e020263710404831002016403048226010483660604823c020165010484510201670104816407048418030334c2b20a0481120201690404814c02016a010481500904810f02036b75610304836402016d01027902016f0304817b03056f6373cebc0a0483010201700104817d020171050482680104843b0302383203048128040807090707070b0608060707060c0b810e07080807070707060b080707070b0807090807070a070a07070808070b06090807070708080a0b81230907070b070b0707080907070706090807070807060b07060707070808070a080b0708090b0b09810a0707060908070607060609070b0a070706080a09070707070e0a0708090b0c0b09070a080a811a0706080c09070a07080b0708060806070b080c0e07090e09060706080b060a070b0607090707130b080708070b0908070a0c810d070b0706080707080b080a0a070807090708070707090709070706080709080a81170a0707070a070707070a0b0a07080d080707060a070707070b0707060f0b060707060a08070807080708810d0807070709070b070808070907080f0b070907090b0707070a0807070c0b080c0a810107070a0b07060c07080f070b09070b0906070b070b'); INSERT INTO t1_data VALUES(274877906949,X'00000e5c033079720404826c0404833002021b03026f6b020482100201740904842e010484150303c2b36f0204840e0201760704826b02017708048273020578f48ba5b50a0481400201790904845902017a050483280304c2bac2bd0404841e0203c2aa680904843a0301bc070481100301be0304820304016a050481630202ca8103027b0202cb860604840d0204ceb56e370204832e0302bc740202520202cf8d07027e0205d8ae39c2aa0104813c0204e887b3770404816a0204f1bfb0970204832f01017a0118101e282f07045961813a193e011e69162f0d2b051c060f084460063053011e06810c20330d0733815c220515220c011210290a7e07810c3a18011a1f2f064a19155212472781047c010a123e45825501166c6062182718167131092f0112331b812c0b6e81470b01184f3a230d45261e271c36111701140a8128456b291248391a0201300104840d0204812502013401048318020137070483070201380904823b03016704026402013907023403016e060481380201610204812f0404824504045243020162050481150104824003016a0804827a020163070484590204817903017a02048209020164020484200203653077030482610302737601048424020166070481730201670304841e0504840e03016b0504825302016901048235010484400404841002016a010483680404845f02046bc2b97407023302026c7a0704812d02027161070209030378cebc0a021f020172040483750404815f0201730304813c030131020482040201740704824d0302793901027502017604048423020177040620813103048313020179030484540204833c04022503017a0904826302017a050483530203c2b2660404840f0307ba6272f397bd92010483070301be020482760202c6b9050482410203c9a3650604812c0202cbae020482180202d38c0704812f0204f096adb7070483490204f6a69c8b030484390102c2aa010c815e81147969010c811c827f0417010e81077a4c03815f010a2e8100820c010a810d148140010c0a7481201a13010c0b831525323d010a8206358129010c21637a33812701083c8340390301300504820a0304326939770404841c03016107048334030165030484150302696c0704810103016c0704812a03027178090481620301730504830e0301750504846303017802021c0301790304836803017a0604834e040135030482650302cdbf010483330304ceb23169070484090303e4849a0504817d0201b201088219744601082210832c010629846001121c8137182211816232010a81668122690108817281080110816a433581102f3e010e7481001b3481190106814e1b010e3646823a810e070301310504840b030161010481720301660604822f0301680804821003016a0704844803016b0104845b03016c0504823403026f700802340301710804826004026d3106024a03017509068458070301770a04836c04016b01021f0301790a0482790302c2b20a02270401b3040482130303ceae6e010484330302df9e040482100201b3010881656e750106820b50010e81434a27048153010a812b068122010843810c0401048211010a50817d812e010c811a8163810801061e832c010c811f81418101030163030481210301640504831102048104030482440301680a048202040271300404841303066b6576cebc6a0104843603026f7a08026d0303756a7808023e030176040483610301770704814403027973020484360302c2b906020c0401bc010484490401bd0804835c0302d38c0304846f0304eaae96750a0484020201b9010c3b824018322d010e0468315a5c817901067d583401060b810e010865118169010c0e07826b813a01066f8265010c3b8239633852010e61161e7030821b01068241210301350804843d0301620302480301690402610403e7a1910704812103016d080269030673c2bac2b36c090483070301760504821d02048454030278350702620302c2ba010483320304ca897a750804813b0401b5070482350303cebc6d050483120302d199020483020402ad660604840c0302d2a1010482550302daa30202340201ba01026d01085119826a010c2f5e82008110010882028221010481090108821b812d01068128460208810b8264010c810971812d440301310304820e0301320504825e05048221030333756d010482720301340502700301640604813d03016607020c0301690204823a03016b0104812203016c0704842103016d0304835f03016e0a04813c03016f0304834d030173060482660401710604810f0301740104843c0303d48174040482690201bc010482690108813c83290114090f816045242e148111010a810616822701048456010e2c81167a638115010a3a3b83204001067a8367010c8114127a2265010a824f7f5c230301300604833e0302616f020483560301640104827d090481280301650502150301660804841a03016804048221030169030481120301720104827803017606023d04016e020481700302c2aa0904836c0401ba090484100201bd01088240224f010843813674020a81185068620106822a44010e14154a8101825e010c19814a1b826c010c81221f81651b010a815a4d812c01082d7281160301300804843c030231300304836703013509048353030365c2bc0304814d0302667504027a0301670604831503016c09025d03016d03048178030172020483620301730104816b0402c2b30404823203017401027204016a08048451030175090237040177090481690301770302190301780a04823d03027a350a0481260302c2aa060481620401b3090481090304f098a1a30a04825d0201be010a0d730b816f010c821d81236c2b010c6f8208358119010681236801087d4e831e0108823c8235010667794b010683165e010e05812e3d3c820d010c81148123817403043531c2ba0204824c03083875c2bceba7957a03020803016302020804036dcebc0304814e0301640304843704033379790504823203016b08023603016d070481090304825804013707027a03016f080484530301710a04842203017809025e0304c2aa6135010481460401b90a04836b0401bc0704814f0306c6ab66e3afb9080482660103c39f350304821f0201b0050481110301680602030201b8040481310604810403016c0a024803017108024b0201be06048323030482650301300504812d0304f48990ae060482740103c491680a0483700301760a0481290201a7010484090202b177020482380201b3030482280102c5800602090301360104826a02018202048173040483780301370104814102018b090482730201930504810f0102c680020483790404830c020183020228020185030484400704820b02028d7a0a04821c0201920404835e02019507048437010484070301760204836102019a020214080483100303d795680504820602019e030481050201a30404820a05048436030177070484290201a50704843b0201a8060482690304816f03027735020483630202ab79050484430201b6060481050201b904024003016d070481130201bd090484350201bf06048361030163020481580102c781040484560201830604833a020286650404842a0301750a04821a020189070483370303e0b9b30904836602018c0104816c04020602021d030171050484570201960a04842d03016b0204821903016c04021e03016e0204845002019a0a04844402019d0704835103013103021702039f77750704827e0201a1090483340201a301022b0404831d0304616363750502520302cfb2040483720201a50a0481530202ad760902310201af0104811d0908816281390301640404843c0201b3080483240201bb030483480201bd040484290304840a0203bf646a0404843f03017a040482450102c89d05020f0201a1090483550201a3060483550201ab0904813203017a04024a0201ad030482210201b10204824302025d0303656577030483600202b46e010482240201b6090484410204bce39f9d020484570102c9870604813202018b0804843603017a06027102018d080481130204840c02018f0104836309026202029164020484430201930504816d0201970304843402039c6c73070483700201a0070483470201a203048262030364d7960a0484010201a80304832a060484390202a96a080481170303c69532090481440201ac040214020481180202ae770604832c0201b0090482300201b40804845a0201b5070482110302716f010482160201ba0504837b0201bb02048149030165050483250201bc07088105833002067f824d0302d19f0402330201bd04024d03017a0904814f0102ca800a048321020181040481580201820604821e020383693707048417020287630704840002018a0904834c0104835c02018b06022f02018c0704814702018d0704810a0104831a020490e7b38308048423020191020482510201920304832f030135030483710201950304833207026802019902026b02019b0904832a02019d09024c02019f0802210201a4010484080202a5710a0481680203a777670804831e0301790902100201ae0402700104834e0201af040484650202b673080484190201b705048221010483520201b808048465030167010484400104cb81cab90104834802018602048317020689777568cf920804842602018b0404817b020191050484200202a0720304833e0201a3040625827f0201a4030485080102cdb1070881408205020483700201b30404840e0202b735080482640201b80504815a0201b90a0482370206bb31cebc6c730304842f030133070483010103ce80370304826d0201900202770201ae0504844e0201b1050484080201b20904815f0201b3040483430201b40702600201b50804823b0201b90304846e0302c9a1070483300201ba0404844d0104847e0201bb050482780201bc010882378223010e811105814d8134010a8142823f2d0106811c52010a6e1814816b02088168824b010a438137812d011019060c6b812f811c010a81314e811b03033366660404825503013502048263030238620904820703016303048120040f080b0907070b07070a0907070707080a07070b0a0a81060b0707070606070f0b070b07070908070b070f0b090807080b07070707070c0e0707090d07080908080a0a50070a0707080708070706070707080a094d07070707070707070707080706070707090844070f07080c0708070708070707080a4707060609060c0b07080a07090808080737070b09060706070707070707070707094807080b0607070707060708074107080709070706070707080607060706070808070a460a0d06090709060b060707060a07070c0907060b06060b070a090707080707070b0707070c060b08070b070a09070b07070b08080706070707070807080707090d070707060707070609070a090807070d0707070b09070707070706070a0908070a0807060b0a080707090707090b08090a08070707080707070e07060708070709080b06070b0a0707070a06070606070809060a07080b07070a070c07070808070e070807070c07090607070707060707080b0743090708'); @@ -6976,7 +6975,7 @@ COMMIT; do_catchsql_test 51.1 { SELECT max(rowid)==0 FROM t1('e*'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #-------------------------------------------------------------------------- reset_db @@ -8752,7 +8751,7 @@ do_test 60.0 { do_catchsql_test 60.2 { SELECT (matchinfo(t1,591)) FROM t1 WHERE t1 MATCH 'e*eŸ' -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- do_test 61.0 { @@ -9773,7 +9772,7 @@ do_test 66.0 { do_catchsql_test 66.1 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- # @@ -10107,7 +10106,7 @@ do_test 68.0 { do_catchsql_test 68.1 { PRAGMA reverse_unordered_selects=ON; INSERT INTO t1(t1) SELECT x FROM t2; -} {1 {database disk image is malformed}} +} {1 {fts5: corruption on page 1, segment 1, table "t1"}} #------------------------------------------------------------------------- reset_db @@ -10323,7 +10322,7 @@ do_test 69.0 { do_catchsql_test 69.2 { SELECT * FROM t1 WHERE a MATCH 'fx*' -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -10506,7 +10505,7 @@ do_test 71.0 { do_catchsql_test 71.2 { INSERT INTO t1(t1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -10633,9 +10632,9 @@ do_catchsql_test 72.1 { INSERT INTO ttt(ttt) VALUES('integrity-check'); } {1 {database disk image is malformed}} -do_catchsql_test 72.1 { +do_catchsql_test 72.2 { SELECT 1 FROM ttt('e* NOT ee*'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -10761,7 +10760,7 @@ do_test 73.0 { do_catchsql_test 73.1 { SELECT snippet(ttt,ttt, NOT 54 ), * FROM ttt('e* NOT ee*e* NOT ee* NOT ee*e* NOT e*') ; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -15910,8 +15909,220 @@ do_catchsql_test 82.4 { SAVEPOINT b; } {1 {database disk image is malformed}} +#------------------------------------------------------------------------- +reset_db +do_test 83.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 24576 pagesize 4096 filename crash-c4a4c5492615bd.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 06 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 06 0e 0f 00 0f aa 0f 53 ...............S +| 112: 0e e8 0e 8b 0e 33 0e 0f 00 01 00 00 00 00 00 00 .....3.......... +| 3584: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 ................ +| 3600: 06 06 17 11 11 01 31 74 61 62 6c 65 62 62 62 62 ......1tablebbbb +| 3616: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 62 62 .CREATE TABLE bb +| 3632: 28 61 29 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 (a)V.......table +| 3648: 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 t1_configt1_conf +| 3664: 69 67 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 ig.CREATE TABLE +| 3680: 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 't1_config'(k PR +| 3696: 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 20 57 49 IMARY KEY, v) WI +| 3712: 54 48 4f 55 54 20 52 4f 57 49 44 5b 04 07 17 21 THOUT ROWID[...! +| 3728: 21 01 81 01 74 61 62 6c 65 74 31 5f 64 6f 63 73 !...tablet1_docs +| 3744: 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 izet1_docsize.CR +| 3760: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 EATE TABLE 't1_d +| 3776: 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 54 45 47 ocsize'(id INTEG +| 3792: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3808: 73 7a 20 42 4c 4f 42 29 69 03 07 17 19 19 01 81 sz BLOB)i....... +| 3824: 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 -tablet1_idxt1_i +| 3840: 64 78 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 dx.CREATE TABLE +| 3856: 27 74 31 5f 69 64 78 27 28 73 65 67 69 64 2c 20 't1_idx'(segid, +| 3872: 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d term, pgno, PRIM +| 3888: 41 52 59 20 4b 45 59 28 73 65 67 69 64 2c 20 74 ARY KEY(segid, t +| 3904: 65 72 6d 29 29 20 57 49 54 48 4f 55 54 20 52 4f erm)) WITHOUT RO +| 3920: 57 49 44 55 02 07 17 1b 1b 01 81 01 74 61 62 6c WIDU........tabl +| 3936: 65 74 31 5f 64 61 74 61 74 31 5f 64 61 74 61 02 et1_datat1_data. +| 3952: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 CREATE TABLE 't1 +| 3968: 5f 64 61 74 61 27 28 69 64 20 49 4e 54 45 47 45 _data'(id INTEGE +| 3984: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 R PRIMARY KEY, b +| 4000: 6c 6f 63 6b 20 42 4c 4f 42 29 54 01 07 17 11 11 lock BLOB)T..... +| 4016: 08 81 15 74 61 62 6c 65 74 31 74 31 43 52 45 41 ...tablet1t1CREA +| 4032: 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 4c 45 TE VIRTUAL TABLE +| 4048: 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 28 61 t1 USING fts5(a +| 4064: 2c 62 2c 70 72 65 66 69 78 3d 22 31 2c 32 2c 33 ,b,prefix=.1,2,3 +| 4080: 2c 34 22 2c 20 63 6f 6e 74 65 6e 74 3d 22 22 29 ,4., content=..) +| page 2 offset 4096 +| 0: 0d 0b 6a 00 37 09 4c 02 0f e7 09 4c 0f c6 0f a4 ..j.7.L....L.... +| 16: 0f 88 0f 6d 0f 4b 0f 2c 0f 0e 0e ec 0e cd 0e ad ...m.K.,........ +| 32: 0e 8e 0e 6c 0e 4b 0e 29 0e 08 0d e6 0d c4 0d b5 ...l.K.)........ +| 48: 0d 97 0d 76 0d 54 0d 31 0d 15 0c f3 0c d3 0c b5 ...v.T.1........ +| 64: 0c 95 0c 73 0c 54 0c 32 0c 10 0b ee 0b cc 0b b0 ...s.T.2........ +| 80: 0b 8d 0b 7e 0b 48 0b 2e 0b 00 4a ef fa cc 0a ad ...~.H....J..... +| 96: 0a 8c 0a 6d 0a 4d 0a 2b 0a 0c 09 00 00 00 00 00 ...m.M.+........ +| 2368: 00 00 00 00 00 00 00 00 00 00 00 00 15 0a 03 00 ................ +| 2384: 30 00 00 00 01 01 03 35 00 03 01 01 12 02 01 12 0......5........ +| 2400: 03 01 11 1c 8c 80 80 80 80 10 03 00 3e 00 00 00 ............>... +| 2416: 17 01 05 05 34 74 61 62 6c 03 02 03 01 04 77 68 ....4tabl.....wh +| 2432: 65 72 03 02 06 09 1b 8c 80 80 80 80 0f 03 00 3c er.............< +| 2448: 00 00 00 16 05 34 66 74 73 34 03 02 02 01 04 6e .....4fts4.....n +| 2464: 75 6d 62 03 07 01 04 09 1b 8c 80 80 80 80 0e 03 umb............. +| 2480: 00 3c 00 00 00 16 04 33 74 68 65 03 06 01 01 04 .<.....3the..... +| 2496: 01 03 77 68 65 03 02 04 04 0a 1b 8c 80 80 80 80 ..whe........... +| 2512: 0d 03 00 3c 00 00 00 16 04 33 6e 75 6d 03 06 01 ...<.....3num... +| 2528: 01 05 01 03 74 61 62 03 02 03 04 0a 19 8c 80 80 ....tab......... +| 2544: 80 80 0c 03 00 38 00 00 00 14 03 32 77 68 03 02 .....8.....2wh.. +| 2560: 04 00 04 33 66 74 73 03 02 02 04 07 18 8c 80 80 ...3fts......... +| 2576: 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 03 02 .....6.....2ta.. +| 2592: 03 02 01 68 03 06 01 01 04 04 07 1b 8c 80 80 80 ...h............ +| 2608: 80 0a 03 00 3c 00 00 00 16 03 32 6e 75 03 06 01 ....<.....2nu... +| 2624: 01 05 01 02 6f 66 03 06 01 01 06 04 09 19 8c 80 ....of.......... +| 2640: 80 80 80 09 03 00 38 00 00 00 14 03 32 66 74 03 ......8.....2ft. +| 2656: 02 02 01 02 69 73 02 06 01 01 03 04 07 18 8c 80 ....is.......... +| 2672: 80 80 22 08 03 00 36 00 00 00 13 02 31 74 03 08 ......6.....1t.. +| 2688: 03 01 01 04 01 01 77 03 02 04 04 09 1a 8c 80 80 ......w......... +| 2704: 80 80 07 03 00 3a 00 00 00 15 02 31 6e 03 08 01 .....:.....1n... +| 2720: 01 02 05 01 01 6f 03 06 01 01 06 04 09 18 8c 80 .....o.......... +| 2736: 80 80 80 06 03 00 36 00 00 00 13 04 02 31 66 03 ......6......1f. +| 2752: 02 02 01 01 69 03 06 01 01 03 05 06 1c 8c 80 80 ....i........... +| 2768: 80 80 05 03 00 3e 00 00 00 17 04 30 74 68 65 03 .....>.....0the. +| 2784: 06 01 01 04 01 05 77 68 65 72 65 03 02 04 0a 15 ......where..... +| 2800: 8c 80 80 80 80 04 03 00 30 00 00 00 11 01 01 06 ........0....... +| 2816: 06 30 74 61 62 6c 65 03 01 f3 07 1c 8c 80 80 80 .0table......... +| 2832: 80 03 03 00 3e 00 00 00 17 07 30 6e 75 6d 62 65 ....>.....0numbe +| 2848: 72 03 06 01 01 05 01 02 6f 66 03 06 04 0d 13 8c r.......of...... +| 2864: 80 80 80 80 02 03 00 2c 00 00 00 0f 01 01 03 02 .......,........ +| 2880: 30 6e 03 06 01 01 02 07 1b 8c 80 80 80 80 01 03 0n.............. +| 2896: 00 3c 00 00 00 16 08 30 66 74 73 34 61 75 78 03 .<.....0fts4aux. +| 2912: 02 02 01 02 69 73 03 06 04 0c 00 00 00 14 2a 00 ....is........*. +| 2928: 00 00 01 01 02 24 00 02 01 01 12 02 01 12 08 88 .....$.......... +| 2944: 80 80 80 80 12 03 00 16 00 00 00 05 02 1c 88 80 ................ +| 2960: 80 80 80 11 03 00 3e 00 00 00 17 05 34 72 6f 77 ......>.....4row +| 2976: 73 02 06 01 01 05 01 04 74 68 65 72 02 02 04 0b s.......ther.... +| 2992: 15 88 80 80 80 80 10 03 00 30 00 00 00 11 02 01 .........0...... +| 3008: 01 07 05 34 62 65 74 77 02 02 04 08 1b 88 80 80 ...4betw........ +| 3024: 80 80 0f 03 00 3c 00 00 00 16 04 04 33 72 6f 77 .....<......3row +| 3040: 02 06 01 01 05 01 03 74 68 65 02 08 05 0a 1b 88 .......the...... +| 3056: 80 80 80 80 0e 03 00 3c 00 00 00 16 01 01 02 04 .......<........ +| 3072: 33 61 72 65 02 02 b3 01 03 62 65 74 02 02 07 08 3are.....bet.... +| 3088: 1b 88 80 80 80 80 0d 03 00 3c 00 00 00 16 03 32 .........<.....2 +| 3104: 74 68 02 08 02 01 01 07 00 04 33 61 6e 64 02 06 th........3and.. +| 3120: 04 0a 1b 88 80 80 80 80 0c 03 00 3c 00 00 00 16 ...........<.... +| 3136: 03 32 69 6e 02 06 01 01 06 01 02 72 6f 02 06 01 .2in.......ro... +| 3152: 01 05 04 09 18 88 80 80 80 80 0b 03 00 36 00 0f .............6.. +| 3168: f0 13 02 03 32 61 72 02 02 03 01 02 62 65 02 02 ....2ar.....be.. +| 3184: 03 05 07 1b 88 80 80 80 80 0a 03 00 3c dd 00 00 ............<... +| 3200: 18 c2 31 74 02 08 02 01 01 07 00 03 32 61 6e 02 ..1t........2an. +| 3216: 06 01 01 04 09 19 88 80 80 80 80 09 03 00 38 00 ..............8. +| 3232: 00 00 14 02 31 6e 02 06 01 01 03 01 01 72 02 06 ....1n.......r.. +| 3248: 01 01 05 04 08 17 88 80 80 80 80 08 03 00 34 00 ..............4. +| 3264: 00 00 12 02 31 62 02 02 04 01 01 69 02 06 01 01 ....1b.....i.... +| 3280: 06 04 06 19 88 80 90 80 80 07 03 00 38 00 00 00 ............8... +| 3296: 14 04 02 31 32 02 02 05 01 01 61 02 08 03 01 01 ...12.....a..... +| 3312: 02 05 06 1b 88 80 80 80 80 06 03 00 3c 00 00 00 ............<... +| 3328: 16 06 30 74 68 65 72 65 02 02 02 00 02 31 31 02 ..0there.....11. +| 3344: 06 01 01 04 0a 15 88 80 80 80 80 05 03 00 30 00 ..............0. +| 3360: 00 00 11 01 01 05 04 30 74 68 65 02 06 01 01 07 .......0the..... +| 3376: 07 1c 88 80 80 80 80 04 03 00 3e 00 00 00 17 01 ..........>..... +| 3392: 01 06 02 30 6e 02 06 01 01 03 01 04 72 6f 77 73 ...0n.......rows +| 3408: 02 06 07 08 1b 88 80 80 80 80 03 03 00 3c 00 00 .............<.. +| 3424: 00 16 08 30 62 65 74 77 65 65 6e 02 02 04 01 02 ...0between..... +| 3440: 69 6e 02 06 04 0c 1a 88 80 80 80 80 02 03 00 3a in.............: +| 3456: 00 00 00 15 04 30 61 6e 64 02 06 01 01 02 02 02 .....0and....... +| 3472: 72 65 02 02 03 04 0a 17 88 80 80 80 80 01 03 00 re.............. +| 3488: 34 00 00 0c 52 02 30 31 02 06 01 01 04 01 01 32 4...R.01.......2 +| 3504: 02 02 05 04 08 08 84 80 80 80 80 12 03 00 16 00 ................ +| 3520: 00 00 05 04 1b 84 80 80 80 80 11 03 00 3c 00 00 .............<.. +| 3536: 00 16 05 34 74 61 62 6c 01 06 00 f1 05 02 03 65 ...4tabl.......e +| 3552: 72 6d 01 02 04 0b 1b 84 80 80 80 80 10 03 00 3c rm.............< +| 3568: 00 00 00 16 05 34 65 61 63 68 01 02 03 01 04 70 .....4each.....p +| 3584: 72 65 73 01 02 05 04 09 1a 84 80 80 80 80 0f 03 res............. +| 3600: 00 3a 00 00 00 15 04 33 74 65 72 01 02 04 02 02 .:.....3ter..... +| 3616: 68 65 01 06 01 01 03 04 08 1b 84 80 80 80 80 0e he.............. +| 3632: 03 00 3c 00 00 00 16 04 33 70 72 65 01 02 05 01 ..<.....3pre.... +| 3648: 03 74 61 62 01 06 01 01 05 04 08 1a 84 80 80 80 .tab............ +| 3664: 80 0d 03 00 3a 00 00 00 15 04 33 66 6f 72 01 02 ....:.....3for.. +| 3680: 02 02 02 74 73 01 06 01 01 04 04 08 1b 84 80 80 ...ts........... +| 3696: 80 80 0c 03 00 3c 00 00 00 16 03 32 74 68 01 06 .....<.....2th.. +| 3712: 01 01 03 00 04 33 65 61 63 01 02 03 04 09 18 74 .....3eac......t +| 3728: 80 80 80 80 0b 03 00 36 00 00 00 13 03 32 74 61 .......6.....2ta +| 3744: 01 06 01 01 05 02 01 65 01 02 04 04 09 19 84 80 .......e........ +| 3760: 80 80 80 0a 03 00 38 00 00 00 14 03 32 69 6e 01 ......8.....2in. +| 3776: 06 01 01 02 11 02 70 62 01 02 05 04 09 18 84 80 ......pb........ +| 3792: 80 80 80 09 03 00 36 00 00 00 13 03 32 66 6f 01 ......6.....2fo. +| 3808: 02 02 02 01 74 01 06 01 01 04 04 07 1b 84 80 80 ....t........... +| 3824: 80 80 08 03 00 3c 0d c0 00 16 12 31 74 01 0a 04 .....<.....1t... +| 3840: 01 01 03 04 00 03 32 65 61 01 02 03 04 0a 17 84 ......2ea....... +| 3856: 80 80 80 80 07 03 00 34 00 00 00 12 02 31 69 01 .......4.....1i. +| 3872: 06 01 01 02 01 01 70 01 02 05 04 08 18 84 80 80 ......p......... +| 3888: 80 80 06 03 00 36 00 00 00 13 02 31 65 01 02 03 .....6.....1e... +| 3904: 01 01 66 01 08 02 5b 01 04 04 06 1b 84 80 80 80 ..f...[......... +| 3920: 80 05 03 00 3c 00 00 00 16 05 30 74 65 72 6d 01 ....<.....0term. +| 3936: 02 04 02 02 68 65 01 06 01 01 03 04 09 14 84 80 ....he.......... +| 3952: 80 80 80 04 03 00 2e 00 00 00 10 06 30 74 61 62 ............0tab +| 3968: 6c 65 01 06 01 01 05 04 15 84 80 80 80 80 03 03 le.............. +| 3984: 00 30 00 00 00 11 01 f8 30 70 72 65 73 65 6e 74 .0......0present +| 4000: 01 02 05 05 1b 84 80 80 80 80 02 03 00 3c 00 00 .............<.. +| 4016: 00 16 04 30 66 74 73 01 06 01 01 04 01 02 69 6e ...0fts.......in +| 4032: 01 06 01 01 04 0a 1a 84 80 80 80 80 01 03 00 3a ...............: +| 4048: 00 00 00 15 05 30 65 61 63 68 01 02 03 01 03 66 .....0each.....f +| 4064: 6f 72 01 02 02 04 09 06 01 03 00 12 03 0b 0f 00 or.............. +| 4080: 00 08 8c 80 80 80 80 11 03 00 16 00 00 00 05 04 ................ +| page 3 offset 8192 +| 0: 0a 00 00 00 32 0e 4f 00 0f fa 0f f1 0f e9 0f e1 ....2.O......... +| 16: 0f d8 0f d1 0f c9 0f c1 0f b9 0f b1 0f a9 0f a0 ................ +| 32: 0f 98 0f 90 0f 87 0f 80 0f 78 0f 71 0f 68 0f 5f .........x.q.h._ +| 48: 0f 56 0f 4d 0f 41 0f 38 0f 2f 0f 26 0f 1d 0f 13 .V.M.A.8./.&.... +| 64: 0f 0a 0f 01 0e f7 0e ee 0e e6 0e dd 0e d6 0e cd ................ +| 80: 0e c3 0e ba 0e b0 0e a8 0e 9f 0e 96 0e 00 00 00 ................ +| 3648: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 ................ +| 3664: 04 01 10 01 03 34 74 20 07 04 02 4e 01 03 34 1e .....4t ...N..4. +| 3680: 09 04 01 12 01 03 33 74 68 1c 08 04 01 10 01 03 ......3th....... +| 3696: 33 6e 1a 08 04 01 10 01 03 32 77 18 08 04 01 10 3n.......2w..... +| 3712: 01 03 32 74 16 08 04 01 10 01 03 32 6e 14 07 04 ..2t.......2n... +| 3728: 01 0e 01 03 32 12 08 04 01 10 01 03 31 74 10 08 ....2.......1t.. +| 3744: 04 01 10 01 03 31 6e 0e 07 04 01 0e 01 03 31 0c .....1n.......1. +| 3760: 09 04 01 12 01 03 30 74 68 0a 08 04 01 10 01 03 ......0th....... +| 3776: 30 74 08 19 04 01 12 01 03 30 6e 75 06 08 04 01 0t.......0nu.... +| 3792: 10 01 03 30 6e 04 06 04 01 0c 01 03 02 08 04 01 ...0n........... +| 3808: 10 01 02 34 72 22 07 04 01 0e 01 02 34 20 08 04 ...4r.......4 .. +| 3824: 01 10 01 02 33 72 1e 09 04 01 12 01 02 33 61 72 ....3r.......3ar +| 3840: 1c 08 04 01 10 01 02 32 74 1a 08 04 01 10 01 02 .......2t....... +| 3856: 32 69 18 09 04 01 12 01 02 32 60 82 16 08 04 01 2i.......2`..... +| 3872: 10 01 02 31 74 14 08 04 01 10 01 02 31 6e 12 08 ...1t.......1n.. +| 3888: 04 01 10 01 02 31 62 10 08 04 01 10 01 02 31 32 .....1b.......12 +| 3904: 0e 0b 04 01 16 01 02 30 74 68 65 72 0c 08 04 01 .......0ther.... +| 3920: 10 01 02 30 74 0a 08 04 01 10 01 02 30 6e 08 08 ...0t.......0n.. +| 3936: 04 01 10 01 02 30 62 06 08 04 01 10 01 02 30 61 .....0b.......0a +| 3952: 04 06 04 01 0c 01 02 02 07 04 09 10 01 34 74 22 .............4t. +| 3968: 06 04 09 0e 01 34 20 08 04 09 12 01 33 74 65 1e .....4 .....3te. +| 3984: 07 04 09 10 01 33 70 1c 07 04 09 10 01 33 66 1a .....3p......3f. +| 4000: 08 04 09 12 01 32 74 68 18 07 04 09 10 01 32 74 .....2th......2t +| 4016: 16 07 04 09 10 01 32 69 14 07 04 09 10 01 32 66 ......2i......2f +| 4032: 12 07 04 09 10 01 31 74 10 07 04 09 10 01 31 69 ......1t......1i +| 4048: 0e 06 04 09 0e 01 31 0c 08 04 09 12 01 30 74 65 ......1......0te +| 4064: 0a 07 04 09 10 01 30 74 08 07 04 09 10 01 30 70 ......0t......0p +| 4080: 06 08 04 09 12 01 30 66 74 04 05 04 09 0c 01 02 ......0ft....... +| page 4 offset 12288 +| 0: 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 05 03 03 00 10 ................ +| 4080: 03 05 05 02 03 00 10 04 06 05 01 03 00 10 04 04 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 02 0f eb 00 0f eb 0f f4 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 08 03 15 01 70 ...............p +| 4080: 67 83 7a 18 0b 03 1b 01 76 65 72 73 69 6f 6e 04 g.z.....version. +| page 6 offset 20480 +| 0: 0d 00 00 00 03 0f f2 00 0f fc 0f 00 00 00 00 00 ................ +| 4080: 00 00 03 03 02 01 03 03 02 02 01 02 02 01 02 09 ................ +| end crash-c4a4c5492615bd.db +}]} {} + +do_catchsql_test 83.1 { + SELECT * FROM t1('R*R*R*R*R*R*R*R*') WHERE (a,b)<=(current_date,0 BETWEEN 'a'<>11 AND '') ORDER BY rowid DESC; +} {/.*fts5: corruption found/} sqlite3_fts5_may_be_corrupt 0 finish_test - diff --git a/ext/fts5/test/fts5corrupt5.test b/ext/fts5/test/fts5corrupt5.test index 6a70fc7e4..4b21a9ff7 100644 --- a/ext/fts5/test/fts5corrupt5.test +++ b/ext/fts5/test/fts5corrupt5.test @@ -237,7 +237,7 @@ do_test 1.0 { do_catchsql_test 1.1 { SELECT * FROM t1('R*') WHERE (a,b)<=(current_date,0) ORDER BY rowid DESC; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- # @@ -450,7 +450,7 @@ do_test 2.0 { do_catchsql_test 2.1 { SELECT * FROM t1('R*R*R*R*') WHERE (a,b)<=(current_date,0) ORDER BY rowid DESC; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -569,7 +569,7 @@ do_test 3.0 { do_catchsql_test 3.1 { UPDATE t1 SET b=quote(zeroblob(200)) WHERE a MATCH 'thra*T'; -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db @@ -878,7 +878,7 @@ do_execsql_test 5.1 { } do_catchsql_test 5.4 { UPDATE t1 SET content=randomblob(500); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} #------------------------------------------------------------------------- reset_db diff --git a/ext/fts5/test/fts5corrupt7.test b/ext/fts5/test/fts5corrupt7.test index 41e359f42..23061a1cb 100644 --- a/ext/fts5/test/fts5corrupt7.test +++ b/ext/fts5/test/fts5corrupt7.test @@ -123,6 +123,6 @@ do_execsql_test 2.2 { do_catchsql_test 2.3 { DELETE FROM t1 WHERE rowid = 1 -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} finish_test diff --git a/ext/fts5/test/fts5corrupt8.test b/ext/fts5/test/fts5corrupt8.test index c0019137d..471a1b0e3 100644 --- a/ext/fts5/test/fts5corrupt8.test +++ b/ext/fts5/test/fts5corrupt8.test @@ -32,7 +32,7 @@ sqlite3 db test.db do_catchsql_test 1.2 { SELECT * FROM t1 -} {1 {database disk image is malformed}} +} {1 {fts5: corrupt structure record for table "t1"}} do_catchsql_test 1.3 { DROP TABLE t1 } {0 {}} diff --git a/ext/fts5/test/fts5corruptbig.test b/ext/fts5/test/fts5corruptbig.test new file mode 100644 index 000000000..6019f17ee --- /dev/null +++ b/ext/fts5/test/fts5corruptbig.test @@ -0,0 +1,53 @@ +# 2025 October 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This test is focused on really large position lists. Those that require +# 4 or 5 byte position-list size varints. Because of the amount of memory +# required, these tests only run on 64-bit platforms. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5corruptbig + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +if { $tcl_platform(wordSize)<8 } { + finish_test + return +} + +if { $SQLITE_MAX_LENGTH!=0x7FFFFFFF } { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); +} + +do_execsql_test 1.1 { + UPDATE t1_data SET block = zeroblob(2147483640) WHERE id=10; +} + +do_execsql_test 1.2 { + SELECT id, length(block) FROM t1_data +} {1 0 10 2147483640} + +do_catchsql_test 1.3 { + SELECT * FROM t1('abc') +} {1 {out of memory}} + +finish_test + diff --git a/ext/fts5/test/fts5integrity.test b/ext/fts5/test/fts5integrity.test index 5c4002180..4bf120c44 100644 --- a/ext/fts5/test/fts5integrity.test +++ b/ext/fts5/test/fts5integrity.test @@ -37,6 +37,12 @@ do_execsql_test 2.1 { INSERT INTO yy(yy) VALUES('integrity-check'); } +db close +sqlite3 db test.db +do_execsql_test 2.1 { + INSERT INTO yy(yy) VALUES('integrity-check'); +} + #-------------------------------------------------------------------- # do_execsql_test 3.0 { diff --git a/ext/fts5/test/fts5join.test b/ext/fts5/test/fts5join.test new file mode 100644 index 000000000..e4d3b69b7 --- /dev/null +++ b/ext/fts5/test/fts5join.test @@ -0,0 +1,69 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5join + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE vt USING fts5(x); + INSERT INTO vt VALUES('abc'); + INSERT INTO vt VALUES('xyz'); + + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TIMESTAMP); + INSERT INTO t1 VALUES(1, 1), (2, 2); + CREATE INDEX i1 ON t1(b); +} + +# set sqlite_where_trace [expr 0xFFF] + +do_eqp_test 1.1 { + SELECT * FROM vt, t1 WHERE vt.rowid = t1.rowid ORDER BY t1.rowid; +} { + QUERY PLAN + |--SCAN t1 + `--SCAN vt VIRTUAL TABLE INDEX 0:= +} + +do_eqp_test 1.2 { + SELECT * FROM vt, t1 WHERE vt.rowid = t1.rowid AND b>? ORDER BY b LIMIT 10 +} { + QUERY PLAN + |--SEARCH t1 USING COVERING INDEX i1 (b>?) + `--SCAN vt VIRTUAL TABLE INDEX 0:= +} + +do_eqp_test 1.3 { + SELECT * FROM vt, t1 WHERE vt.rowid = t1.rowid AND b>? +} { + QUERY PLAN + |--SEARCH t1 USING COVERING INDEX i1 (b>?) + `--SCAN vt VIRTUAL TABLE INDEX 0:= +} + +do_eqp_test 1.4 { + SELECT * FROM vt, t1 WHERE vt.rowid = t1.rowid ORDER BY b +} { + QUERY PLAN + |--SCAN t1 USING COVERING INDEX i1 + `--SCAN vt VIRTUAL TABLE INDEX 0:= +} + + +finish_test diff --git a/ext/fts5/test/fts5leftjoin.test b/ext/fts5/test/fts5leftjoin.test index 4ef6a8961..69a172bd4 100644 --- a/ext/fts5/test/fts5leftjoin.test +++ b/ext/fts5/test/fts5leftjoin.test @@ -40,4 +40,53 @@ do_execsql_test 1.2 { SELECT * FROM t1 LEFT JOIN vt ON (vt MATCH 'abc') } {1 abc 2 abc} + +do_execsql_test 1.3 { + DELETE FROM t1; + INSERT INTO t1 VALUES(14); +} + +do_execsql_test 1.4 { + SELECT * FROM vt LEFT JOIN t1 ON vt.rowid = 1; +} { + abc 14 + xyz {} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t0 USING fts5(a,b); + INSERT INTO t0(a,b)VALUES(1,0); + CREATE TABLE t1(x); +} + +do_execsql_test 2.1 { + SELECT * FROM t0 LEFT JOIN t1; +} {1 0 {}} + +breakpoint +do_catchsql_test 2.2 { + SELECT * FROM t0 LEFT JOIN t1 ON t0.b MATCH '1'; +} {1 {no query solution}} + +do_execsql_test 2.3 { + SELECT * FROM t0 LEFT JOIN t1 ON +b MATCH '1'; +} {1 0 {}} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t0 USING fts5(c0, c1); + INSERT INTO t0(c0,c1) VALUES (1,0); +} + +do_catchsql_test 3.1 { + SELECT * FROM t0 + LEFT JOIN ( SELECT 0 AS col_0 ) + ON ((((t0.c1 MATCH '1')AND(CASE WHEN t0.c0 THEN CAST(t0.c1 AS INTEGER) ELSE 1 END)))); +} {1 {no query solution}} + + finish_test diff --git a/ext/fts5/test/fts5misc.test b/ext/fts5/test/fts5misc.test index 2aca1986a..817be9560 100644 --- a/ext/fts5/test/fts5misc.test +++ b/ext/fts5/test/fts5misc.test @@ -685,5 +685,17 @@ do_execsql_test 26.1 { COMMIT; } +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 27.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(a, b); + INSERT INTO ft1(rowid, a, b) VALUES(3, '3', '3'); +} + +do_execsql_test 27.1 { + SELECT * FROM ft1 WHERE rowid=3 AND b MATCH 'hello'; +} + finish_test diff --git a/ext/fts5/test/fts5onepass.test b/ext/fts5/test/fts5onepass.test index 01021ed34..b33409675 100644 --- a/ext/fts5/test/fts5onepass.test +++ b/ext/fts5/test/fts5onepass.test @@ -38,15 +38,15 @@ foreach {tn sql uses} { 1.2 { DELETE FROM ft WHERE rowid=? } 0 1.3 { DELETE FROM ft WHERE rowid=? } 0 1.4 { DELETE FROM ft WHERE ft MATCH '1' } 1 - 1.5 { DELETE FROM ft WHERE ft MATCH '1' AND rowid=? } 1 - 1.6 { DELETE FROM ft WHERE ft MATCH '1' AND rowid=? } 1 + 1.5 { DELETE FROM ft WHERE ft MATCH '1' AND rowid=? } 0 + 1.6 { DELETE FROM ft WHERE ft MATCH '1' AND rowid=? } 0 2.1 { UPDATE ft SET content='a b c' } 1 2.2 { UPDATE ft SET content='a b c' WHERE rowid=? } 0 2.3 { UPDATE ft SET content='a b c' WHERE rowid=? } 0 2.4 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' } 1 - 2.5 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' AND rowid=? } 1 - 2.6 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' AND rowid=? } 1 + 2.5 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' AND rowid=? } 0 + 2.6 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' AND rowid=? } 0 } { do_test 1.$tn { sql_uses_stmt db $sql } $uses } diff --git a/ext/fts5/test/fts5rebuild.test b/ext/fts5/test/fts5rebuild.test index d74b148fb..065d16b91 100644 --- a/ext/fts5/test/fts5rebuild.test +++ b/ext/fts5/test/fts5rebuild.test @@ -46,7 +46,7 @@ do_execsql_test 1.5 { do_catchsql_test 1.6 { INSERT INTO f1(f1) VALUES('integrity-check'); -} {1 {database disk image is malformed}} +} {/.*fts5: corrupt.*/} do_execsql_test 1.7 { INSERT INTO f1(f1) VALUES('rebuild'); diff --git a/ext/intck/intck1.test b/ext/intck/intck1.test index 187132f76..ef29c2c54 100644 --- a/ext/intck/intck1.test +++ b/ext/intck/intck1.test @@ -139,6 +139,7 @@ do_test 2.3 { sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 db eval { + PRAGMA writable_schema=on; DELETE FROM imp1 WHERE rowid=1; DELETE FROM imp2 WHERE rowid=2; } @@ -174,6 +175,7 @@ do_test 3.2 { sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 db eval { + PRAGMA writable_schema=on; DELETE FROM imp1 WHERE a=5; } execsql_pp { diff --git a/ext/intck/intck2.test b/ext/intck/intck2.test index 23b241b5a..a35bbc70c 100644 --- a/ext/intck/intck2.test +++ b/ext/intck/intck2.test @@ -29,6 +29,7 @@ do_execsql_test 1.0 { proc imposter_edit {obj create sql} { sqlite3 xdb test.db set pgno [xdb one {SELECT rootpage FROM sqlite_schema WHERE name=$obj}] + xdb eval {PRAGMA Writable_schema=ON} sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER xdb main 1 $pgno xdb eval $create diff --git a/ext/jni/src/c/sqlite3-jni.c b/ext/jni/src/c/sqlite3-jni.c index d6723453d..f130eff04 100644 --- a/ext/jni/src/c/sqlite3-jni.c +++ b/ext/jni/src/c/sqlite3-jni.c @@ -168,6 +168,10 @@ ** Creates a verbose JNI function name. Suffix must be ** the JNI-mangled form of the function's name, minus the ** prefix seen in this macro. +** +** If you get java.lang.UnsatisfiedLinkError when calling newly-added +** native bindings, be sure that the mangled name is correct. It can +** be found in the generated sqlite3-jni.h. */ #define JniFuncName(Suffix) \ Java_org_sqlite_jni_capi_CApi_sqlite3_ ## Suffix @@ -177,10 +181,10 @@ JNIEXPORT ReturnType JNICALL JniFuncName(Suffix) /* -** S3JniApi's intent is that CFunc be the C API func(s) the -** being-declared JNI function is wrapping, making it easier to find -** that function's JNI-side entry point. The other args are for JniDecl. -** See the many examples in this file. +** S3JniApi's intent is that CFunc be the name(s) of the C API func(s) +** the being-declared JNI function is wrapping, making it easier to +** find those bindings' JNI-side entry points. The other args are for +** JniDecl. See the many examples in this file. */ #define S3JniApi(CFunc,ReturnType,Suffix) JniDecl(ReturnType,Suffix) @@ -3856,6 +3860,19 @@ S3JniApi(sqlite3_errmsg(),jstring,1errmsg)( effect should be identical to using errmsg16(), however. */; } +S3JniApi(sqlite3_set_errmsg(),jint,1set_1errmsg)( + JniArgsEnvClass, jobject jpDb, jint errCode, jstring msg +){ + sqlite3 * const pDb = PtrGet_sqlite3(jpDb); + const char *zUtf8; + jint rc; + if( !pDb ) return SQLITE_MISUSE; + zUtf8 = msg ? s3jni_jstring_to_mutf8(msg) : NULL; + rc = sqlite3_set_errmsg(pDb, (int)errCode, zUtf8); + s3jni_mutf8_release(msg, zUtf8); + return rc; +} + S3JniApi(sqlite3_errstr(),jstring,1errstr)( JniArgsEnvClass, jint rcCode ){ diff --git a/ext/jni/src/c/sqlite3-jni.h b/ext/jni/src/c/sqlite3-jni.h index 6f93bf8ab..81af5cbde 100644 --- a/ext/jni/src/c/sqlite3-jni.h +++ b/ext/jni/src/c/sqlite3-jni.h @@ -1342,7 +1342,7 @@ JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1data_1count * Method: sqlite3_db_config * Signature: (Lorg/sqlite/jni/capi/sqlite3;IILorg/sqlite/jni/capi/OutputPointer/Int32;)I */ -JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1db_1config__Lorg_sqlite_jni_capi_sqlite3_2IILorg_sqlite_jni_capi_OutputPointer_00024Int32_2 +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1db_1config__Lorg_sqlite_jni_capi_sqlite3_2IILorg_sqlite_jni_capi_OutputPointer_Int32_2 (JNIEnv *, jclass, jobject, jint, jint, jobject); /* @@ -1417,6 +1417,14 @@ JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1errcode JNIEXPORT jstring JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1errmsg (JNIEnv *, jclass, jobject); +/* + * Class: org_sqlite_jni_capi_CApi + * Method: sqlite3_set_errmsg + * Signature: (Lorg/sqlite/jni/capi/sqlite3;ILjava/lang/String;)I + */ +JNIEXPORT jint JNICALL Java_org_sqlite_jni_capi_CApi_sqlite3_1set_1errmsg + (JNIEnv *, jclass, jobject, jint, jstring); + /* * Class: org_sqlite_jni_capi_CApi * Method: sqlite3_error_offset diff --git a/ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java b/ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java index 1fa6c6b80..912f6ed5b 100644 --- a/ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java +++ b/ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java @@ -120,7 +120,7 @@ as the xValue() and xInverse() methods of the {@link WindowFunction} argument, the context is set to the given initial value. On all other calls, the 2nd argument is ignored. - @see SQLFunction.PerContextState#getAggregateState + @see AggregateFunction.PerContextState#getAggregateState */ protected final ValueHolder getAggregateState(sqlite3_context cx, T initialValue){ return map.getAggregateState(cx, initialValue); @@ -130,7 +130,7 @@ protected final ValueHolder getAggregateState(sqlite3_context cx, T initialVa To be called from the implementation's xFinal() method to fetch the final state of the UDF and remove its mapping. - see SQLFunction.PerContextState#takeAggregateState + see AggregateFunction.PerContextState#takeAggregateState */ protected final T takeAggregateState(sqlite3_context cx){ return map.takeAggregateState(cx); diff --git a/ext/jni/src/org/sqlite/jni/capi/CApi.java b/ext/jni/src/org/sqlite/jni/capi/CApi.java index 731fb0ac3..0b840c362 100644 --- a/ext/jni/src/org/sqlite/jni/capi/CApi.java +++ b/ext/jni/src/org/sqlite/jni/capi/CApi.java @@ -1197,10 +1197,15 @@ public static native int sqlite3_db_status( public static native String sqlite3_errmsg(@NotNull sqlite3 db); + /** Added in 3.51.0. */ + public static native int sqlite3_set_errmsg(@NotNull sqlite3 db, + int resultCode, + String msg); + private static native int sqlite3_error_offset(@NotNull long ptrToDb); /** - Note that the returned byte offset values assume UTF-8-encoded + Caveat: the returned byte offset values assume UTF-8-encoded inputs, so won't always match character offsets in Java Strings. */ public static int sqlite3_error_offset(@NotNull sqlite3 db){ diff --git a/ext/jni/src/org/sqlite/jni/capi/Tester1.java b/ext/jni/src/org/sqlite/jni/capi/Tester1.java index a9b766e9f..9d14c954b 100644 --- a/ext/jni/src/org/sqlite/jni/capi/Tester1.java +++ b/ext/jni/src/org/sqlite/jni/capi/Tester1.java @@ -1874,6 +1874,20 @@ private void testPrepareMulti(){ sqlite3_close_v2(db); } + private void testSetErrmsg(){ + final sqlite3 db = createNewDb(); + + int rc = sqlite3_set_errmsg(db, SQLITE_RANGE, "nope"); + affirm( 0==rc ); + affirm( SQLITE_MISUSE == sqlite3_set_errmsg(null, 0, null) ); + affirm( "nope".equals(sqlite3_errmsg(db)) ); + affirm( SQLITE_RANGE == sqlite3_errcode(db) ); + rc = sqlite3_set_errmsg(db, 0, null); + affirm( "not an error".equals(sqlite3_errmsg(db)) ); + affirm( 0 == sqlite3_errcode(db) ); + sqlite3_close_v2(db); + } + /* Copy/paste/rename this to add new tests. */ private void _testTemplate(){ final sqlite3 db = createNewDb(); diff --git a/ext/lsm1/Makefile b/ext/lsm1/Makefile deleted file mode 100644 index d497a1d13..000000000 --- a/ext/lsm1/Makefile +++ /dev/null @@ -1,56 +0,0 @@ -# -# This Makefile is designed for use with main.mk in the root directory of -# this project. After including main.mk, the users makefile should contain: -# -# LSMDIR=$(TOP)/ext/lsm1/ -# LSMOPTS=-fPIC -# include $(LSMDIR)/Makefile -# -# The most useful targets are [lsmtest] and [lsm.so]. -# - -LSMOBJ = \ - lsm_ckpt.o \ - lsm_file.o \ - lsm_log.o \ - lsm_main.o \ - lsm_mem.o \ - lsm_mutex.o \ - lsm_shared.o \ - lsm_sorted.o \ - lsm_str.o \ - lsm_tree.o \ - lsm_unix.o \ - lsm_win32.o \ - lsm_varint.o \ - lsm_vtab.o - -LSMHDR = \ - $(LSMDIR)/lsm.h \ - $(LSMDIR)/lsmInt.h - -LSMTESTSRC = $(LSMDIR)/lsm-test/lsmtest1.c $(LSMDIR)/lsm-test/lsmtest2.c \ - $(LSMDIR)/lsm-test/lsmtest3.c $(LSMDIR)/lsm-test/lsmtest4.c \ - $(LSMDIR)/lsm-test/lsmtest5.c $(LSMDIR)/lsm-test/lsmtest6.c \ - $(LSMDIR)/lsm-test/lsmtest7.c $(LSMDIR)/lsm-test/lsmtest8.c \ - $(LSMDIR)/lsm-test/lsmtest9.c \ - $(LSMDIR)/lsm-test/lsmtest_datasource.c \ - $(LSMDIR)/lsm-test/lsmtest_func.c $(LSMDIR)/lsm-test/lsmtest_io.c \ - $(LSMDIR)/lsm-test/lsmtest_main.c $(LSMDIR)/lsm-test/lsmtest_mem.c \ - $(LSMDIR)/lsm-test/lsmtest_tdb.c $(LSMDIR)/lsm-test/lsmtest_tdb3.c \ - $(LSMDIR)/lsm-test/lsmtest_util.c $(LSMDIR)/lsm-test/lsmtest_win32.c - - -# all: lsm.so - -LSMOPTS += -fPIC -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR) -DHAVE_ZLIB - -lsm.so: $(LSMOBJ) - $(T.link) -shared -fPIC -o lsm.so $(LSMOBJ) - -%.o: $(LSMDIR)/%.c $(LSMHDR) sqlite3.h - $(T.link) $(LSMOPTS) -c $< - -lsmtest$(EXE): $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) sqlite3.o - # $(T.link) -c $(TOP)/lsm-test/lsmtest_tdb2.cc - $(T.link) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB) -lz diff --git a/ext/lsm1/Makefile.msc b/ext/lsm1/Makefile.msc deleted file mode 100644 index 3e5a3b331..000000000 --- a/ext/lsm1/Makefile.msc +++ /dev/null @@ -1,102 +0,0 @@ -# -# This Makefile is designed for use with Makefile.msc in the root directory -# of this project. The Makefile.msc should contain: -# -# LSMDIR=$(TOP)\ext\lsm1 -# !INCLUDE $(LSMDIR)\Makefile.msc -# -# The most useful targets are [lsmtest.exe] and [lsm.dll]. -# - -LSMOBJ = \ - lsm_ckpt.lo \ - lsm_file.lo \ - lsm_log.lo \ - lsm_main.lo \ - lsm_mem.lo \ - lsm_mutex.lo \ - lsm_shared.lo \ - lsm_sorted.lo \ - lsm_str.lo \ - lsm_tree.lo \ - lsm_unix.lo \ - lsm_win32.lo \ - lsm_varint.lo \ - lsm_vtab.lo - -LSMHDR = \ - $(LSMDIR)\lsm.h \ - $(LSMDIR)\lsmInt.h - -LSMTESTSRC = $(LSMDIR)\lsm-test\lsmtest1.c $(LSMDIR)\lsm-test\lsmtest2.c \ - $(LSMDIR)\lsm-test\lsmtest3.c $(LSMDIR)\lsm-test\lsmtest4.c \ - $(LSMDIR)\lsm-test\lsmtest5.c $(LSMDIR)\lsm-test\lsmtest6.c \ - $(LSMDIR)\lsm-test\lsmtest7.c $(LSMDIR)\lsm-test\lsmtest8.c \ - $(LSMDIR)\lsm-test\lsmtest9.c \ - $(LSMDIR)\lsm-test\lsmtest_datasource.c \ - $(LSMDIR)\lsm-test\lsmtest_func.c $(LSMDIR)\lsm-test\lsmtest_io.c \ - $(LSMDIR)\lsm-test\lsmtest_main.c $(LSMDIR)\lsm-test\lsmtest_mem.c \ - $(LSMDIR)\lsm-test\lsmtest_tdb.c $(LSMDIR)\lsm-test\lsmtest_tdb3.c \ - $(LSMDIR)\lsm-test\lsmtest_util.c $(LSMDIR)\lsm-test\lsmtest_win32.c - -# all: lsm.dll lsmtest.exe - -LSMOPTS = $(NO_WARN) -DLSM_MUTEX_WIN32=1 -I$(LSMDIR) - -!IF $(DEBUG)>2 -LSMOPTS = $(LSMOPTS) -DLSM_DEBUG=1 -!ENDIF - -!IF $(MEMDEBUG)!=0 -LSMOPTS = $(LSMOPTS) -DLSM_DEBUG_MEM=1 -!ENDIF - -lsm_ckpt.lo: $(LSMDIR)\lsm_ckpt.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_ckpt.c - -lsm_file.lo: $(LSMDIR)\lsm_file.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_file.c - -lsm_log.lo: $(LSMDIR)\lsm_log.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_log.c - -lsm_main.lo: $(LSMDIR)\lsm_main.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_main.c - -lsm_mem.lo: $(LSMDIR)\lsm_mem.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_mem.c - -lsm_mutex.lo: $(LSMDIR)\lsm_mutex.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_mutex.c - -lsm_shared.lo: $(LSMDIR)\lsm_shared.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_shared.c - -lsm_sorted.lo: $(LSMDIR)\lsm_sorted.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_sorted.c - -lsm_str.lo: $(LSMDIR)\lsm_str.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_str.c - -lsm_tree.lo: $(LSMDIR)\lsm_tree.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_tree.c - -lsm_unix.lo: $(LSMDIR)\lsm_unix.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_unix.c - -lsm_win32.lo: $(LSMDIR)\lsm_win32.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_win32.c - -lsm_varint.lo: $(LSMDIR)\lsm_varint.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_varint.c - -lsm_vtab.lo: $(LSMDIR)\lsm_vtab.c $(LSMHDR) $(SQLITE3H) - $(LTCOMPILE) $(LSMOPTS) -c $(LSMDIR)\lsm_vtab.c - -lsm.dll: $(LSMOBJ) - $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ $(LSMOBJ) - copy /Y $@ $(LSMDIR)\$@ - -lsmtest.exe: $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) $(LIBOBJ) - $(LTLINK) $(LSMOPTS) $(LSMTESTSRC) /link $(LSMOBJ) $(LIBOBJ) - copy /Y $@ $(LSMDIR)\$@ diff --git a/ext/lsm1/lsm-test/README b/ext/lsm1/lsm-test/README deleted file mode 100644 index 80654ee97..000000000 --- a/ext/lsm1/lsm-test/README +++ /dev/null @@ -1,40 +0,0 @@ - - -Organization of test case files: - - lsmtest1.c: Data tests. Tests that perform many inserts and deletes on a - database file, then verify that the contents of the database can - be queried. - - lsmtest2.c: Crash tests. Tests that attempt to verify that the database - recovers correctly following an application or system crash. - - lsmtest3.c: Rollback tests. Tests that focus on the explicit rollback of - transactions and sub-transactions. - - lsmtest4.c: Multi-client tests. - - lsmtest5.c: Multi-client tests with a different thread for each client. - - lsmtest6.c: OOM injection tests. - - lsmtest7.c: API tests. - - lsmtest8.c: Writer crash tests. Tests in this file attempt to verify that - the system recovers and other clients proceed unaffected if - a process fails in the middle of a write transaction. - - The difference from lsmtest2.c is that this file tests - live-recovery (recovery from a failure that occurs while other - clients are still running) whereas lsmtest2.c tests recovery - from a system or power failure. - - lsmtest9.c: More data tests. These focus on testing that calling - lsm_work(nMerge=1) to compact the database does not corrupt it. - In other words, that databases containing block-redirects - can be read and written. - - - - - diff --git a/ext/lsm1/lsm-test/lsmtest.h b/ext/lsm1/lsm-test/lsmtest.h deleted file mode 100644 index ca60424ad..000000000 --- a/ext/lsm1/lsm-test/lsmtest.h +++ /dev/null @@ -1,303 +0,0 @@ - -#ifndef __WRAPPER_INT_H_ -#define __WRAPPER_INT_H_ - -#include "lsmtest_tdb.h" -#include "sqlite3.h" -#include "lsm.h" - -#include -#include -#include -#include -#include -#ifndef _WIN32 -# include -#endif -#include -#include -#include -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef _WIN32 -# include "windows.h" -# define gettimeofday win32GetTimeOfDay -# define F_OK (0) -# define sleep(sec) Sleep(1000 * (sec)) -# define usleep(usec) Sleep(((usec) + 999) / 1000) -# ifdef _MSC_VER -# include -# define snprintf _snprintf -# define fsync(fd) FlushFileBuffers((HANDLE)_get_osfhandle((fd))) -# define fdatasync(fd) FlushFileBuffers((HANDLE)_get_osfhandle((fd))) -# define __va_copy(dst,src) ((dst) = (src)) -# define ftruncate(fd,sz) ((_chsize_s((fd), (sz))==0) ? 0 : -1) -# else -# error Unsupported C compiler for Windows. -# endif -int win32GetTimeOfDay(struct timeval *, void *); -#endif - -#ifndef _LSM_INT_H -typedef unsigned int u32; -typedef unsigned char u8; -typedef long long int i64; -typedef unsigned long long int u64; -#endif - - -#define ArraySize(x) ((int)(sizeof(x) / sizeof((x)[0]))) - -#define MIN(x,y) ((x)<(y) ? (x) : (y)) -#define MAX(x,y) ((x)>(y) ? (x) : (y)) - -#define unused_parameter(x) (void)(x) - -#define TESTDB_DEFAULT_PAGE_SIZE 4096 -#define TESTDB_DEFAULT_CACHE_SIZE 2048 - -#ifndef _O_BINARY -# define _O_BINARY (0) -#endif - -/* -** Ideally, these should be in wrapper.c. But they are here instead so that -** they can be used by the C++ database wrappers in wrapper2.cc. -*/ -typedef struct DatabaseMethods DatabaseMethods; -struct TestDb { - DatabaseMethods const *pMethods; /* Database methods */ - const char *zLibrary; /* Library name for tdb_open() */ -}; -struct DatabaseMethods { - int (*xClose)(TestDb *); - int (*xWrite)(TestDb *, void *, int , void *, int); - int (*xDelete)(TestDb *, void *, int); - int (*xDeleteRange)(TestDb *, void *, int, void *, int); - int (*xFetch)(TestDb *, void *, int, void **, int *); - int (*xScan)(TestDb *, void *, int, void *, int, void *, int, - void (*)(void *, void *, int , void *, int) - ); - int (*xBegin)(TestDb *, int); - int (*xCommit)(TestDb *, int); - int (*xRollback)(TestDb *, int); -}; - -/* -** Functions in wrapper2.cc (a C++ source file). wrapper2.cc contains the -** wrapper for Kyoto Cabinet. Kyoto cabinet has a C API, but -** the primary interface is the C++ API. -*/ -int test_kc_open(const char*, const char *zFilename, int bClear, TestDb **ppDb); -int test_kc_close(TestDb *); -int test_kc_write(TestDb *, void *, int , void *, int); -int test_kc_delete(TestDb *, void *, int); -int test_kc_delete_range(TestDb *, void *, int, void *, int); -int test_kc_fetch(TestDb *, void *, int, void **, int *); -int test_kc_scan(TestDb *, void *, int, void *, int, void *, int, - void (*)(void *, void *, int , void *, int) -); - -int test_mdb_open(const char*, const char *zFile, int bClear, TestDb **ppDb); -int test_mdb_close(TestDb *); -int test_mdb_write(TestDb *, void *, int , void *, int); -int test_mdb_delete(TestDb *, void *, int); -int test_mdb_fetch(TestDb *, void *, int, void **, int *); -int test_mdb_scan(TestDb *, void *, int, void *, int, void *, int, - void (*)(void *, void *, int , void *, int) -); - -/* -** Functions in wrapper3.c. This file contains the tdb wrapper for lsm. -** The wrapper for lsm is a bit more involved than the others, as it -** includes code for a couple of different lsm configurations, and for -** various types of fault injection and robustness testing. -*/ -int test_lsm_open(const char*, const char *zFile, int bClear, TestDb **ppDb); -int test_lsm_lomem_open(const char*, const char*, int bClear, TestDb **ppDb); -int test_lsm_lomem2_open(const char*, const char*, int bClear, TestDb **ppDb); -int test_lsm_zip_open(const char*, const char*, int bClear, TestDb **ppDb); -int test_lsm_small_open(const char*, const char*, int bClear, TestDb **ppDb); -int test_lsm_mt2(const char*, const char *zFile, int bClear, TestDb **ppDb); -int test_lsm_mt3(const char*, const char *zFile, int bClear, TestDb **ppDb); - -int tdb_lsm_configure(lsm_db *, const char *); - -/* Functions in lsmtest_tdb4.c */ -int test_bt_open(const char*, const char *zFile, int bClear, TestDb **ppDb); -int test_fbt_open(const char*, const char *zFile, int bClear, TestDb **ppDb); -int test_fbts_open(const char*, const char *zFile, int bClear, TestDb **ppDb); - - -/* Functions in testutil.c. */ -int testPrngInit(void); -u32 testPrngValue(u32 iVal); -void testPrngArray(u32 iVal, u32 *aOut, int nOut); -void testPrngString(u32 iVal, char *aOut, int nOut); - -void testErrorInit(int argc, char **); -void testPrintError(const char *zFormat, ...); -void testPrintUsage(const char *zArgs); -void testPrintFUsage(const char *zFormat, ...); -void testTimeInit(void); -int testTimeGet(void); - -/* Functions in testmem.c. */ -void testMallocInstall(lsm_env *pEnv); -void testMallocUninstall(lsm_env *pEnv); -void testMallocCheck(lsm_env *pEnv, int *, int *, FILE *); -void testMallocOom(lsm_env *pEnv, int, int, void(*)(void*), void *); -void testMallocOomEnable(lsm_env *pEnv, int); - -/* lsmtest.c */ -TestDb *testOpen(const char *zSystem, int, int *pRc); -void testReopen(TestDb **ppDb, int *pRc); -void testClose(TestDb **ppDb); - -void testFetch(TestDb *, void *, int, void *, int, int *); -void testWrite(TestDb *, void *, int, void *, int, int *); -void testDelete(TestDb *, void *, int, int *); -void testDeleteRange(TestDb *, void *, int, void *, int, int *); -void testWriteStr(TestDb *, const char *, const char *zVal, int *pRc); -void testFetchStr(TestDb *, const char *, const char *, int *pRc); - -void testBegin(TestDb *pDb, int iTrans, int *pRc); -void testCommit(TestDb *pDb, int iTrans, int *pRc); - -void test_failed(void); - -char *testMallocPrintf(const char *zFormat, ...); -char *testMallocVPrintf(const char *zFormat, va_list ap); -int testGlobMatch(const char *zPattern, const char *zStr); - -void testScanCompare(TestDb *, TestDb *, int, void *, int, void *, int, int *); -void testFetchCompare(TestDb *, TestDb *, void *, int, int *); - -void *testMalloc(int); -void *testMallocCopy(void *pCopy, int nByte); -void *testRealloc(void *, int); -void testFree(void *); - -/* lsmtest_bt.c */ -int do_bt(int nArg, char **azArg); - -/* testio.c */ -int testVfsConfigureDb(TestDb *pDb); - -/* testfunc.c */ -int do_show(int nArg, char **azArg); -int do_work(int nArg, char **azArg); - -/* testio.c */ -int do_io(int nArg, char **azArg); - -/* lsmtest2.c */ -void do_crash_test(const char *zPattern, int *pRc); -int do_rollback_test(int nArg, char **azArg); - -/* test3.c */ -void test_rollback(const char *zSystem, const char *zPattern, int *pRc); - -/* test4.c */ -void test_mc(const char *zSystem, const char *zPattern, int *pRc); - -/* test5.c */ -void test_mt(const char *zSystem, const char *zPattern, int *pRc); - -/* lsmtest6.c */ -void test_oom(const char *zPattern, int *pRc); -void testDeleteLsmdb(const char *zFile); - -void testSaveDb(const char *zFile, const char *zAuxExt); -void testRestoreDb(const char *zFile, const char *zAuxExt); -void testCopyLsmdb(const char *zFrom, const char *zTo); - -/* lsmtest7.c */ -void test_api(const char *zPattern, int *pRc); - -/* lsmtest8.c */ -void do_writer_crash_test(const char *zPattern, int *pRc); - -/************************************************************************* -** Interface to functionality in test_datasource.c. -*/ -typedef struct Datasource Datasource; -typedef struct DatasourceDefn DatasourceDefn; - -struct DatasourceDefn { - int eType; /* A TEST_DATASOURCE_* value */ - int nMinKey; /* Minimum key size */ - int nMaxKey; /* Maximum key size */ - int nMinVal; /* Minimum value size */ - int nMaxVal; /* Maximum value size */ -}; - -#define TEST_DATASOURCE_RANDOM 1 -#define TEST_DATASOURCE_SEQUENCE 2 - -char *testDatasourceName(const DatasourceDefn *); -Datasource *testDatasourceNew(const DatasourceDefn *); -void testDatasourceFree(Datasource *); -void testDatasourceEntry(Datasource *, int, void **, int *, void **, int *); -/* End of test_datasource.c interface. -*************************************************************************/ -void testDatasourceFetch( - TestDb *pDb, /* Database handle */ - Datasource *pData, - int iKey, - int *pRc /* IN/OUT: Error code */ -); - -void testWriteDatasource(TestDb *, Datasource *, int, int *); -void testWriteDatasourceRange(TestDb *, Datasource *, int, int, int *); -void testDeleteDatasource(TestDb *, Datasource *, int, int *); -void testDeleteDatasourceRange(TestDb *, Datasource *, int, int, int *); - - -/* test1.c */ -void test_data_1(const char *, const char *, int *pRc); -void test_data_2(const char *, const char *, int *pRc); -void test_data_3(const char *, const char *, int *pRc); -void testDbContents(TestDb *, Datasource *, int, int, int, int, int, int *); -void testCaseProgress(int, int, int, int *); -int testCaseNDot(void); - -void testCompareDb(Datasource *, int, int, TestDb *, TestDb *, int *); -int testControlDb(TestDb **ppDb); - -typedef struct CksumDb CksumDb; -CksumDb *testCksumArrayNew(Datasource *, int, int, int); -char *testCksumArrayGet(CksumDb *, int); -void testCksumArrayFree(CksumDb *); -void testCaseStart(int *pRc, char *zFmt, ...); -void testCaseFinish(int rc); -void testCaseSkip(void); -int testCaseBegin(int *, const char *, const char *, ...); - -#define TEST_CKSUM_BYTES 29 -int testCksumDatabase(TestDb *pDb, char *zOut); -int testCountDatabase(TestDb *pDb); -void testCompareInt(int, int, int *); -void testCompareStr(const char *z1, const char *z2, int *pRc); - -/* lsmtest9.c */ -void test_data_4(const char *, const char *, int *pRc); - - -/* -** Similar to the Tcl_GetIndexFromObjStruct() Tcl library function. -*/ -#define testArgSelect(w,x,y,z) testArgSelectX(w,x,sizeof(w[0]),y,z) -int testArgSelectX(void *, const char *, int, const char *, int *); - -#ifdef __cplusplus -} /* End of the 'extern "C"' block */ -#endif - -#endif diff --git a/ext/lsm1/lsm-test/lsmtest1.c b/ext/lsm1/lsm-test/lsmtest1.c deleted file mode 100644 index 1ce2cc058..000000000 --- a/ext/lsm1/lsm-test/lsmtest1.c +++ /dev/null @@ -1,656 +0,0 @@ - -#include "lsmtest.h" - -#define DATA_SEQUENTIAL TEST_DATASOURCE_SEQUENCE -#define DATA_RANDOM TEST_DATASOURCE_RANDOM - -typedef struct Datatest1 Datatest1; -typedef struct Datatest2 Datatest2; - -/* -** An instance of the following structure contains parameters used to -** customize the test function in this file. Test procedure: -** -** 1. Create a data-source based on the "datasource definition" vars. -** -** 2. Insert nRow key value pairs into the database. -** -** 3. Delete all keys from the database. Deletes are done in the same -** order as the inserts. -** -** During steps 2 and 3 above, after each Datatest1.nVerify inserts or -** deletes, the following: -** -** a. Run Datasource.nTest key lookups and check the results are as expected. -** -** b. If Datasource.bTestScan is true, run a handful (8) of range -** queries (scanning forwards and backwards). Check that the results -** are as expected. -** -** c. Close and reopen the database. Then run (a) and (b) again. -*/ -struct Datatest1 { - /* Datasource definition */ - DatasourceDefn defn; - - /* Test procedure parameters */ - int nRow; /* Number of rows to insert then delete */ - int nVerify; /* How often to verify the db contents */ - int nTest; /* Number of keys to test (0==all) */ - int bTestScan; /* True to do scan tests */ -}; - -/* -** An instance of the following data structure is used to describe the -** second type of test case in this file. The chief difference between -** these tests and those described by Datatest1 is that these tests also -** experiment with range-delete operations. Tests proceed as follows: -** -** 1. Open the datasource described by Datatest2.defn. -** -** 2. Open a connection on an empty database. -** -** 3. Do this Datatest2.nIter times: -** -** a) Insert Datatest2.nWrite key-value pairs from the datasource. -** -** b) Select two pseudo-random keys and use them as the start -** and end points of a range-delete operation. -** -** c) Verify that the contents of the database are as expected (see -** below for details). -** -** d) Close and then reopen the database handle. -** -** e) Verify that the contents of the database are still as expected. -** -** The inserts and range deletes are run twice - once on the database being -** tested and once using a control system (sqlite3, kc etc. - something that -** works). In order to verify that the contents of the db being tested are -** correct, the test runs a bunch of scans and lookups on both the test and -** control databases. If the results are the same, the test passes. -*/ -struct Datatest2 { - DatasourceDefn defn; - int nRange; - int nWrite; /* Number of writes per iteration */ - int nIter; /* Total number of iterations to run */ -}; - -/* -** Generate a unique name for the test case pTest with database system -** zSystem. -*/ -static char *getName(const char *zSystem, int bRecover, Datatest1 *pTest){ - char *zRet; - char *zData; - zData = testDatasourceName(&pTest->defn); - zRet = testMallocPrintf("data.%s.%s.rec=%d.%d.%d", - zSystem, zData, bRecover, pTest->nRow, pTest->nVerify - ); - testFree(zData); - return zRet; -} - -int testControlDb(TestDb **ppDb){ -#ifdef HAVE_KYOTOCABINET - return tdb_open("kyotocabinet", "tmp.db", 1, ppDb); -#else - return tdb_open("sqlite3", "", 1, ppDb); -#endif -} - -void testDatasourceFetch( - TestDb *pDb, /* Database handle */ - Datasource *pData, - int iKey, - int *pRc /* IN/OUT: Error code */ -){ - void *pKey; int nKey; /* Database key to query for */ - void *pVal; int nVal; /* Expected result of query */ - - testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal); - testFetch(pDb, pKey, nKey, pVal, nVal, pRc); -} - -/* -** This function is called to test that the contents of database pDb -** are as expected. In this case, expected is defined as containing -** key-value pairs iFirst through iLast, inclusive, from data source -** pData. In other words, a loop like the following could be used to -** construct a database with identical contents from scratch. -** -** for(i=iFirst; i<=iLast; i++){ -** testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal); -** // insert (pKey, nKey) -> (pVal, nVal) into database -** } -** -** The key domain consists of keys 0 to (nRow-1), inclusive, from -** data source pData. For both scan and lookup tests, keys are selected -** pseudo-randomly from within this set. -** -** This function runs nLookupTest lookup tests and nScanTest scan tests. -** -** A lookup test consists of selecting a key from the domain and querying -** pDb for it. The test fails if the presence of the key and, if present, -** the associated value do not match the expectations defined above. -** -** A scan test involves selecting a key from the domain and running -** the following queries: -** -** 1. Scan all keys equal to or greater than the key, in ascending order. -** 2. Scan all keys equal to or smaller than the key, in descending order. -** -** Additionally, if nLookupTest is greater than zero, the following are -** run once: -** -** 1. Scan all keys in the db, in ascending order. -** 2. Scan all keys in the db, in descending order. -** -** As you would assume, the test fails if the returned values do not match -** expectations. -*/ -void testDbContents( - TestDb *pDb, /* Database handle being tested */ - Datasource *pData, /* pDb contains data from here */ - int nRow, /* Size of key domain */ - int iFirst, /* Index of first key from pData in pDb */ - int iLast, /* Index of last key from pData in pDb */ - int nLookupTest, /* Number of lookup tests to run */ - int nScanTest, /* Number of scan tests to run */ - int *pRc /* IN/OUT: Error code */ -){ - int j; - int rc = *pRc; - - if( rc==0 && nScanTest ){ - TestDb *pDb2 = 0; - - /* Open a control db (i.e. one that we assume works) */ - rc = testControlDb(&pDb2); - - for(j=iFirst; rc==0 && j<=iLast; j++){ - void *pKey; int nKey; /* Database key to insert */ - void *pVal; int nVal; /* Database value to insert */ - testDatasourceEntry(pData, j, &pKey, &nKey, &pVal, &nVal); - rc = tdb_write(pDb2, pKey, nKey, pVal, nVal); - } - - if( rc==0 ){ - int iKey1; - int iKey2; - void *pKey1; int nKey1; /* Start key */ - void *pKey2; int nKey2; /* Final key */ - - iKey1 = testPrngValue((iFirst<<8) + (iLast<<16)) % nRow; - iKey2 = testPrngValue((iLast<<8) + (iFirst<<16)) % nRow; - testDatasourceEntry(pData, iKey1, &pKey2, &nKey1, 0, 0); - pKey1 = testMalloc(nKey1+1); - memcpy(pKey1, pKey2, nKey1+1); - testDatasourceEntry(pData, iKey2, &pKey2, &nKey2, 0, 0); - - testScanCompare(pDb2, pDb, 0, 0, 0, 0, 0, &rc); - testScanCompare(pDb2, pDb, 0, 0, 0, pKey2, nKey2, &rc); - testScanCompare(pDb2, pDb, 0, pKey1, nKey1, 0, 0, &rc); - testScanCompare(pDb2, pDb, 0, pKey1, nKey1, pKey2, nKey2, &rc); - testScanCompare(pDb2, pDb, 1, 0, 0, 0, 0, &rc); - testScanCompare(pDb2, pDb, 1, 0, 0, pKey2, nKey2, &rc); - testScanCompare(pDb2, pDb, 1, pKey1, nKey1, 0, 0, &rc); - testScanCompare(pDb2, pDb, 1, pKey1, nKey1, pKey2, nKey2, &rc); - testFree(pKey1); - } - tdb_close(pDb2); - } - - /* Test some lookups. */ - for(j=0; rc==0 && j=nRow ){ - iKey = j; - }else{ - iKey = testPrngValue(j + (iFirst<<8) + (iLast<<16)) % nRow; - } - - testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal); - if( iFirst>iKey || iKey>iLast ){ - pVal = 0; - nVal = -1; - } - - testFetch(pDb, pKey, nKey, pVal, nVal, &rc); - } - - *pRc = rc; -} - -/* -** This function should be called during long running test cases to output -** the progress dots (...) to stdout. -*/ -void testCaseProgress(int i, int n, int nDot, int *piDot){ - int iDot = *piDot; - while( iDot < ( ((nDot*2+1) * i) / (n*2) ) ){ - printf("."); - fflush(stdout); - iDot++; - } - *piDot = iDot; -} - -int testCaseNDot(void){ return 20; } - -#if 0 -static void printScanCb( - void *pCtx, void *pKey, int nKey, void *pVal, int nVal -){ - printf("%s\n", (char *)pKey); - fflush(stdout); -} -#endif - -void testReopenRecover(TestDb **ppDb, int *pRc){ - if( *pRc==0 ){ - const char *zLib = tdb_library_name(*ppDb); - const char *zDflt = tdb_default_db(zLib); - testCopyLsmdb(zDflt, "bak.db"); - testClose(ppDb); - testCopyLsmdb("bak.db", zDflt); - *pRc = tdb_open(zLib, 0, 0, ppDb); - } -} - - -static void doDataTest1( - const char *zSystem, /* Database system to test */ - int bRecover, - Datatest1 *p, /* Structure containing test parameters */ - int *pRc /* OUT: Error code */ -){ - int i; - int iDot; - int rc = LSM_OK; - Datasource *pData; - TestDb *pDb; - int iToggle = 0; - - /* Start the test case, open a database and allocate the datasource. */ - pDb = testOpen(zSystem, 1, &rc); - pData = testDatasourceNew(&p->defn); - - i = 0; - iDot = 0; - while( rc==LSM_OK && inRow ){ - - /* Insert some data */ - testWriteDatasourceRange(pDb, pData, i, p->nVerify, &rc); - i += p->nVerify; - - if( iToggle ) testBegin(pDb, 1, &rc); - /* Check that the db content is correct. */ - testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc); - if( iToggle ) testCommit(pDb, 0, &rc); - iToggle = (iToggle+1)%2; - - if( bRecover ){ - testReopenRecover(&pDb, &rc); - }else{ - testReopen(&pDb, &rc); - } - - /* Check that the db content is still correct. */ - testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc); - - /* Update the progress dots... */ - testCaseProgress(i, p->nRow, testCaseNDot()/2, &iDot); - } - - i = 0; - iDot = 0; - while( rc==LSM_OK && inRow ){ - - /* Delete some entries */ - testDeleteDatasourceRange(pDb, pData, i, p->nVerify, &rc); - i += p->nVerify; - - /* Check that the db content is correct. */ - testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc); - - /* Close and reopen the database. */ - if( bRecover ){ - testReopenRecover(&pDb, &rc); - }else{ - testReopen(&pDb, &rc); - } - - /* Check that the db content is still correct. */ - testDbContents(pDb, pData, p->nRow, i, p->nRow-1,p->nTest,p->bTestScan,&rc); - - /* Update the progress dots... */ - testCaseProgress(i, p->nRow, testCaseNDot()/2, &iDot); - } - - /* Free the datasource, close the database and finish the test case. */ - testDatasourceFree(pData); - tdb_close(pDb); - testCaseFinish(rc); - *pRc = rc; -} - - -void test_data_1( - const char *zSystem, /* Database system name */ - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - Datatest1 aTest[] = { - { {DATA_RANDOM, 500,600, 1000,2000}, 1000, 100, 10, 0}, - { {DATA_RANDOM, 20,25, 100,200}, 1000, 250, 1000, 1}, - { {DATA_RANDOM, 8,10, 100,200}, 1000, 250, 1000, 1}, - { {DATA_RANDOM, 8,10, 10,20}, 1000, 250, 1000, 1}, - { {DATA_RANDOM, 8,10, 1000,2000}, 1000, 250, 1000, 1}, - { {DATA_RANDOM, 8,100, 10000,20000}, 100, 25, 100, 1}, - { {DATA_RANDOM, 80,100, 10,20}, 1000, 250, 1000, 1}, - { {DATA_RANDOM, 5000,6000, 10,20}, 100, 25, 100, 1}, - { {DATA_SEQUENTIAL, 5,10, 10,20}, 1000, 250, 1000, 1}, - { {DATA_SEQUENTIAL, 5,10, 100,200}, 1000, 250, 1000, 1}, - { {DATA_SEQUENTIAL, 5,10, 1000,2000}, 1000, 250, 1000, 1}, - { {DATA_SEQUENTIAL, 5,100, 10000,20000}, 100, 25, 100, 1}, - { {DATA_RANDOM, 10,10, 100,100}, 100000, 1000, 100, 0}, - { {DATA_SEQUENTIAL, 10,10, 100,100}, 100000, 1000, 100, 0}, - }; - - int i; - int bRecover; - - for(bRecover=0; bRecover<2; bRecover++){ - if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break; - for(i=0; *pRc==LSM_OK && idefn); - rc = testControlDb(&pControl); - - if( tdb_lsm(pDb) ){ - int nBuf = 32 * 1024 * 1024; - lsm_config(tdb_lsm(pDb), LSM_CONFIG_AUTOFLUSH, &nBuf); - } - - for(i=0; rc==0 && inIter; i++){ - void *pKey1; int nKey1; - void *pKey2; int nKey2; - int ii; - int nRange = MIN(p->nIter*p->nWrite, p->nRange); - - for(ii=0; rc==0 && iinWrite; ii++){ - int iKey = (i*p->nWrite + ii) % p->nRange; - testWriteDatasource(pControl, pData, iKey, &rc); - testWriteDatasource(pDb, pData, iKey, &rc); - } - - testDatasourceEntry(pData, i+1000000, &pKey1, &nKey1, 0, 0); - pKey1 = testMallocCopy(pKey1, nKey1); - testDatasourceEntry(pData, i+2000000, &pKey2, &nKey2, 0, 0); - - testDeleteRange(pDb, pKey1, nKey1, pKey2, nKey2, &rc); - testDeleteRange(pControl, pKey1, nKey1, pKey2, nKey2, &rc); - testFree(pKey1); - - testCompareDb(pData, nRange, i, pControl, pDb, &rc); - if( bRecover ){ - testReopenRecover(&pDb, &rc); - }else{ - testReopen(&pDb, &rc); - } - testCompareDb(pData, nRange, i, pControl, pDb, &rc); - - /* Update the progress dots... */ - testCaseProgress(i, p->nIter, testCaseNDot(), &iDot); - } - - testClose(&pDb); - testClose(&pControl); - testDatasourceFree(pData); - testCaseFinish(rc); - *pRc = rc; -} - -static char *getName2(const char *zSystem, int bRecover, Datatest2 *pTest){ - char *zRet; - char *zData; - zData = testDatasourceName(&pTest->defn); - zRet = testMallocPrintf("data2.%s.%s.rec=%d.%d.%d.%d", - zSystem, zData, bRecover, pTest->nRange, pTest->nWrite, pTest->nIter - ); - testFree(zData); - return zRet; -} - -void test_data_2( - const char *zSystem, /* Database system name */ - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - Datatest2 aTest[] = { - /* defn, nRange, nWrite, nIter */ - { {DATA_RANDOM, 20,25, 100,200}, 10000, 10, 50 }, - { {DATA_RANDOM, 20,25, 100,200}, 10000, 200, 50 }, - { {DATA_RANDOM, 20,25, 100,200}, 100, 10, 1000 }, - { {DATA_RANDOM, 20,25, 100,200}, 100, 200, 50 }, - }; - - int i; - int bRecover; - - for(bRecover=0; bRecover<2; bRecover++){ - if( bRecover==1 && memcmp(zSystem, "lsm", 3) ) break; - for(i=0; *pRc==LSM_OK && i> 24) & 0xFF; - aBuf[1] = (iVal >> 16) & 0xFF; - aBuf[2] = (iVal >> 8) & 0xFF; - aBuf[3] = (iVal >> 0) & 0xFF; -} - -void dt3PutKey(u8 *aBuf, int iKey){ - assert( iKey<100000 && iKey>=0 ); - sprintf((char *)aBuf, "%.5d", iKey); -} - -static void doDataTest3( - const char *zSystem, /* Database system to test */ - Datatest3 *p, /* Structure containing test parameters */ - int *pRc /* OUT: Error code */ -){ - int iDot = 0; - int rc = *pRc; - TestDb *pDb; - u8 *abPresent; /* Array of boolean */ - char *aVal; /* Buffer to hold values */ - int i; - u32 iSeq = 10; /* prng counter */ - - abPresent = (u8 *)testMalloc(p->nRange+1); - aVal = (char *)testMalloc(p->nValMax+1); - pDb = testOpen(zSystem, 1, &rc); - - for(i=0; inIter && rc==0; i++){ - int ii; - - testCaseProgress(i, p->nIter, testCaseNDot(), &iDot); - - /* Perform nWrite inserts */ - for(ii=0; iinWrite; ii++){ - u8 aKey[6]; - u32 iKey; - int nVal; - - iKey = (testPrngValue(iSeq++) % p->nRange) + 1; - nVal = (testPrngValue(iSeq++) % (p->nValMax - p->nValMin)) + p->nValMin; - testPrngString(testPrngValue(iSeq++), aVal, nVal); - dt3PutKey(aKey, iKey); - - testWrite(pDb, aKey, sizeof(aKey)-1, aVal, nVal, &rc); - abPresent[iKey] = 1; - } - - /* Perform nDelete deletes */ - for(ii=0; iinDelete; ii++){ - u8 aKey1[6]; - u8 aKey2[6]; - u32 iKey; - - iKey = (testPrngValue(iSeq++) % p->nRange) + 1; - dt3PutKey(aKey1, iKey-1); - dt3PutKey(aKey2, iKey+1); - - testDeleteRange(pDb, aKey1, sizeof(aKey1)-1, aKey2, sizeof(aKey2)-1, &rc); - abPresent[iKey] = 0; - } - - testReopen(&pDb, &rc); - - for(ii=1; rc==0 && ii<=p->nRange; ii++){ - int nDbVal; - void *pDbVal; - u8 aKey[6]; - int dbrc; - - dt3PutKey(aKey, ii); - dbrc = tdb_fetch(pDb, aKey, sizeof(aKey)-1, &pDbVal, &nDbVal); - testCompareInt(0, dbrc, &rc); - - if( abPresent[ii] ){ - testCompareInt(1, (nDbVal>0), &rc); - }else{ - testCompareInt(1, (nDbVal<0), &rc); - } - } - } - - testClose(&pDb); - testCaseFinish(rc); - *pRc = rc; -} - -static char *getName3(const char *zSystem, Datatest3 *p){ - return testMallocPrintf("data3.%s.%d.%d.%d.%d.(%d..%d)", - zSystem, p->nRange, p->nIter, p->nWrite, p->nDelete, - p->nValMin, p->nValMax - ); -} - -void test_data_3( - const char *zSystem, /* Database system name */ - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - Datatest3 aTest[] = { - /* nRange, nIter, nWrite, nDelete, nValMin, nValMax */ - { 100, 1000, 5, 5, 50, 100 }, - { 100, 1000, 2, 2, 5, 10 }, - }; - - int i; - - for(i=0; *pRc==LSM_OK && inRow++; - for(i=0; icksum1 += ((u8 *)pKey)[i]; - p->cksum2 += p->cksum1; - } - for(i=0; icksum1 += ((u8 *)pVal)[i]; - p->cksum2 += p->cksum1; - } -} - -/* -** tdb_scan() callback used by testCountDatabase() -*/ -static void scanCountDb( - void *pCtx, - void *pKey, int nKey, - void *pVal, int nVal -){ - Cksum *p = (Cksum *)pCtx; - p->nRow++; - - unused_parameter(pKey); - unused_parameter(nKey); - unused_parameter(pVal); - unused_parameter(nVal); -} - - -/* -** Iterate through the entire contents of database pDb. Write a checksum -** string based on the db contents into buffer zOut before returning. A -** checksum string is at most 29 (TEST_CKSUM_BYTES) bytes in size: -** -** * 32-bit integer (10 bytes) -** * 1 space (1 byte) -** * 32-bit hex (8 bytes) -** * 1 space (1 byte) -** * 32-bit hex (8 bytes) -** * nul-terminator (1 byte) -** -** The number of entries in the database is returned. -*/ -int testCksumDatabase( - TestDb *pDb, /* Database handle */ - char *zOut /* Buffer to write checksum to */ -){ - Cksum cksum; - memset(&cksum, 0, sizeof(Cksum)); - tdb_scan(pDb, (void *)&cksum, 0, 0, 0, 0, 0, scanCksumDb); - sprintf(zOut, "%d %x %x", - cksum.nRow, (u32)cksum.cksum1, (u32)cksum.cksum2 - ); - assert( strlen(zOut)0 ); */ - if( testrc==0 ) testrc = lsm_checkpoint(db, 0); - } - tdb_close(pDb); - - /* Check that the database content is still correct */ - testCompareCksumLsmdb(DBNAME, - bCompress, testCksumArrayGet(pCksumDb, nRow), 0, pRc); - } - - testCksumArrayFree(pCksumDb); - testDatasourceFree(pData); -} - -/* -** This test verifies that if a system crash occurs while committing a -** transaction to the log file, no earlier transactions are lost or damaged. -*/ -static void crash_test2(int bCompress, int *pRc){ - const char *DBNAME = "testdb.lsm"; - const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 12, 16, 1000, 1000}; - - const int nIter = 200; - const int nInsert = 20; - - int i; - int iDot = 0; - Datasource *pData; - CksumDb *pCksumDb; - TestDb *pDb; - - /* Allocate datasource. And calculate the expected checksums. */ - pData = testDatasourceNew(&defn); - pCksumDb = testCksumArrayNew(pData, 100, 100+nInsert, 1); - - /* Setup and save the initial database. */ - testSetupSavedLsmdb("", DBNAME, pData, 100, pRc); - - for(i=0; izTest) ){ - p->x(p->bCompress, pRc); - testCaseFinish(*pRc); - } - } -} diff --git a/ext/lsm1/lsm-test/lsmtest3.c b/ext/lsm1/lsm-test/lsmtest3.c deleted file mode 100644 index 760dec300..000000000 --- a/ext/lsm1/lsm-test/lsmtest3.c +++ /dev/null @@ -1,238 +0,0 @@ - - -/* -** This file contains tests related to the explicit rollback of database -** transactions and sub-transactions. -*/ - - -/* -** Repeat 2000 times (until the db contains 100,000 entries): -** -** 1. Open a transaction and insert 500 rows, opening a nested -** sub-transaction each 100 rows. -** -** 2. Roll back to each sub-transaction savepoint. Check the database -** checksum looks Ok. -** -** 3. Every second iteration, roll back the main transaction. Check the -** db checksum is correct. Every other iteration, commit the main -** transaction (increasing the size of the db by 100 rows). -*/ - - -#include "lsmtest.h" - -struct CksumDb { - int nFirst; - int nLast; - int nStep; - char **azCksum; -}; - -CksumDb *testCksumArrayNew( - Datasource *pData, - int nFirst, - int nLast, - int nStep -){ - TestDb *pDb; - CksumDb *pRet; - int i; - int nEntry; - int rc = 0; - - assert( nLast>=nFirst && ((nLast-nFirst)%nStep)==0 ); - - pRet = malloc(sizeof(CksumDb)); - memset(pRet, 0, sizeof(CksumDb)); - pRet->nFirst = nFirst; - pRet->nLast = nLast; - pRet->nStep = nStep; - nEntry = 1 + ((nLast - nFirst) / nStep); - - /* Allocate space so that azCksum is an array of nEntry pointers to - ** buffers each TEST_CKSUM_BYTES in size. */ - pRet->azCksum = (char **)malloc(nEntry * (sizeof(char *) + TEST_CKSUM_BYTES)); - for(i=0; iazCksum[nEntry]); - pRet->azCksum[i] = &pStart[i * TEST_CKSUM_BYTES]; - } - - tdb_open("lsm", "tempdb.lsm", 1, &pDb); - testWriteDatasourceRange(pDb, pData, 0, nFirst, &rc); - for(i=0; iazCksum[i]); - if( i==nEntry ) break; - testWriteDatasourceRange(pDb, pData, nFirst+i*nStep, nStep, &rc); - } - - tdb_close(pDb); - - return pRet; -} - -char *testCksumArrayGet(CksumDb *p, int nRow){ - int i; - assert( nRow>=p->nFirst ); - assert( nRow<=p->nLast ); - assert( ((nRow-p->nFirst) % p->nStep)==0 ); - - i = (nRow - p->nFirst) / p->nStep; - return p->azCksum[i]; -} - -void testCksumArrayFree(CksumDb *p){ - free(p->azCksum); - memset(p, 0x55, sizeof(*p)); - free(p); -} - -/* End of CksumDb code. -**************************************************************************/ - -/* -** Test utility function. Write key-value pair $i from datasource pData -** into database pDb. -*/ -void testWriteDatasource(TestDb *pDb, Datasource *pData, int i, int *pRc){ - void *pKey; int nKey; - void *pVal; int nVal; - testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal); - testWrite(pDb, pKey, nKey, pVal, nVal, pRc); -} - -/* -** Test utility function. Delete datasource pData key $i from database pDb. -*/ -void testDeleteDatasource(TestDb *pDb, Datasource *pData, int i, int *pRc){ - void *pKey; int nKey; - testDatasourceEntry(pData, i, &pKey, &nKey, 0, 0); - testDelete(pDb, pKey, nKey, pRc); -} - -/* -** This function inserts nWrite key/value pairs into database pDb - the -** nWrite key value pairs starting at iFirst from data source pData. -*/ -void testWriteDatasourceRange( - TestDb *pDb, /* Database to write to */ - Datasource *pData, /* Data source to read values from */ - int iFirst, /* Index of first key/value pair */ - int nWrite, /* Number of key/value pairs to write */ - int *pRc /* IN/OUT: Error code */ -){ - int i; - for(i=0; i2 && rc==0; iTrans--){ - tdb_rollback(pDb, iTrans); - nCurrent -= 100; - testCksumDatabase(pDb, zCksum); - testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc); - } - - if( i%2 ){ - tdb_rollback(pDb, 0); - nCurrent -= 100; - testCksumDatabase(pDb, zCksum); - testCompareStr(zCksum, testCksumArrayGet(pCksum, nCurrent), &rc); - }else{ - tdb_commit(pDb, 0); - } - } - testCaseFinish(rc); - - skip_rollback_test: - tdb_close(pDb); - testCksumArrayFree(pCksum); - return rc; -} - -void test_rollback( - const char *zSystem, - const char *zPattern, - int *pRc -){ - if( *pRc==0 ){ - int bRun = 1; - - if( zPattern ){ - char *zName = getName(zSystem); - bRun = testGlobMatch(zPattern, zName); - testFree(zName); - } - - if( bRun ){ - DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 10, 15, 50, 100 }; - Datasource *pData = testDatasourceNew(&defn); - *pRc = rollback_test_1(zSystem, pData); - testDatasourceFree(pData); - } - } -} diff --git a/ext/lsm1/lsm-test/lsmtest4.c b/ext/lsm1/lsm-test/lsmtest4.c deleted file mode 100644 index a47241db9..000000000 --- a/ext/lsm1/lsm-test/lsmtest4.c +++ /dev/null @@ -1,127 +0,0 @@ - -/* -** This file contains test cases involving multiple database clients. -*/ - -#include "lsmtest.h" - -/* -** The following code implements test cases "mc1.*". -** -** This test case uses one writer and $nReader readers. All connections -** are driven by a single thread. All connections are opened at the start -** of the test and remain open until the test is finished. -** -** The test consists of $nStep steps. Each step the following is performed: -** -** 1. The writer inserts $nWriteStep records into the db. -** -** 2. The writer checks that the contents of the db are as expected. -** -** 3. Each reader that currently has an open read transaction also checks -** that the contents of the db are as expected (according to the snapshot -** the read transaction is reading - see below). -** -** After step 1, reader 1 opens a read transaction. After step 2, reader -** 2 opens a read transaction, and so on. At step ($nReader+1), reader 1 -** closes the current read transaction and opens a new one. And so on. -** The result is that at step N (for N > $nReader), there exists a reader -** with an open read transaction reading the snapshot committed following -** steps (N-$nReader-1) to N. -*/ -typedef struct Mctest Mctest; -struct Mctest { - DatasourceDefn defn; /* Datasource to use */ - int nStep; /* Total number of steps in test */ - int nWriteStep; /* Number of rows to insert each step */ - int nReader; /* Number of read connections */ -}; -static void do_mc_test( - const char *zSystem, /* Database system to test */ - Mctest *pTest, - int *pRc /* IN/OUT: return code */ -){ - const int nDomain = pTest->nStep * pTest->nWriteStep; - Datasource *pData; /* Source of data */ - TestDb *pDb; /* First database connection (writer) */ - int iReader; /* Used to iterate through aReader */ - int iStep; /* Current step in test */ - int iDot = 0; /* Current step in test */ - - /* Array of reader connections */ - struct Reader { - TestDb *pDb; /* Connection handle */ - int iLast; /* Current snapshot contains keys 0..iLast */ - } *aReader; - - /* Create a data source */ - pData = testDatasourceNew(&pTest->defn); - - /* Open the writer connection */ - pDb = testOpen(zSystem, 1, pRc); - - /* Allocate aReader */ - aReader = (struct Reader *)testMalloc(sizeof(aReader[0]) * pTest->nReader); - for(iReader=0; iReadernReader; iReader++){ - aReader[iReader].pDb = testOpen(zSystem, 0, pRc); - } - - for(iStep=0; iStepnStep; iStep++){ - int iLast; - int iBegin; /* Start read trans using aReader[iBegin] */ - - /* Insert nWriteStep more records into the database */ - int iFirst = iStep*pTest->nWriteStep; - testWriteDatasourceRange(pDb, pData, iFirst, pTest->nWriteStep, pRc); - - /* Check that the db is Ok according to the writer */ - iLast = (iStep+1) * pTest->nWriteStep - 1; - testDbContents(pDb, pData, nDomain, 0, iLast, iLast, 1, pRc); - - /* Have reader (iStep % nReader) open a read transaction here. */ - iBegin = (iStep % pTest->nReader); - if( iBeginnReader && aReader[iReader].iLast; iReader++){ - iLast = aReader[iReader].iLast; - testDbContents( - aReader[iReader].pDb, pData, nDomain, 0, iLast, iLast, 1, pRc - ); - } - - /* Report progress */ - testCaseProgress(iStep, pTest->nStep, testCaseNDot(), &iDot); - } - - /* Close all readers */ - for(iReader=0; iReadernReader; iReader++){ - testClose(&aReader[iReader].pDb); - } - testFree(aReader); - - /* Close the writer-connection and free the datasource */ - testClose(&pDb); - testDatasourceFree(pData); -} - - -void test_mc( - const char *zSystem, /* Database system name */ - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - int i; - Mctest aTest[] = { - { { TEST_DATASOURCE_RANDOM, 10,10, 100,100 }, 100, 10, 5 }, - }; - - for(i=0; i "k.0000000045". -** -** As well as the key/value pairs, the database also contains checksum -** entries. The checksums form a hierarchy - for every F key/value -** entries there is one level 1 checksum. And for each F level 1 checksums -** there is one level 2 checksum. And so on. -** -** Checksum keys are encoded as the two byte "c." followed by the -** checksum level, followed by a 10 digit decimal number containing -** the value of the first key that contributes to the checksum value. -** For example, assuming F==10, the level 1 checksum that spans keys -** 10 to 19 is "c.1.0000000010". -** -** Clients may perform one of two operations on the database: a read -** or a write. -** -** READ OPERATIONS: -** -** A read operation scans a range of F key/value pairs. It computes -** the expected checksum and then compares the computed value to the -** actual value stored in the level 1 checksum entry. It then scans -** the group of F level 1 checksums, and compares the computed checksum -** to the associated level 2 checksum value, and so on until the -** highest level checksum value has been verified. -** -** If a checksum ever fails to match the expected value, the test -** has failed. -** -** WRITE OPERATIONS: -** -** A write operation involves writing (possibly clobbering) a single -** key/value pair. The associated level 1 checksum is then recalculated -** updated. Then the level 2 checksum, and so on until the highest -** level checksum has been modified. -** -** All updates occur inside a single transaction. -** -** INTERFACE: -** -** The interface used by test cases to read and write the db consists -** of type DbParameters and the following functions: -** -** dbReadOperation() -** dbWriteOperation() -*/ - -#include "lsmtest.h" - -typedef struct DbParameters DbParameters; -struct DbParameters { - int nFanout; /* Checksum fanout (F) */ - int nKey; /* Size of key space (N) */ -}; - -#define DB_KEY_BYTES (2+5+10+1) - -/* -** Argument aBuf[] must point to a buffer at least DB_KEY_BYTES in size. -** This function populates the buffer with a nul-terminated key string -** corresponding to key iKey. -*/ -static void dbFormatKey( - DbParameters *pParam, - int iLevel, - int iKey, /* Key value */ - char *aBuf /* Write key string here */ -){ - if( iLevel==0 ){ - snprintf(aBuf, DB_KEY_BYTES, "k.%.10d", iKey); - }else{ - int f = 1; - int i; - for(i=0; inFanout; - snprintf(aBuf, DB_KEY_BYTES, "c.%d.%.10d", iLevel, f*(iKey/f)); - } -} - -/* -** Argument aBuf[] must point to a buffer at least DB_KEY_BYTES in size. -** This function populates the buffer with the string representation of -** checksum value iVal. -*/ -static void dbFormatCksumValue(u32 iVal, char *aBuf){ - snprintf(aBuf, DB_KEY_BYTES, "%.10u", iVal); -} - -/* -** Return the highest level of checksum in the database described -** by *pParam. -*/ -static int dbMaxLevel(DbParameters *pParam){ - int iMax; - int n = 1; - for(iMax=0; nnKey; iMax++){ - n = n * pParam->nFanout; - } - return iMax; -} - -static void dbCksum( - void *pCtx, /* IN/OUT: Pointer to u32 containing cksum */ - void *pKey, int nKey, /* Database key. Unused. */ - void *pVal, int nVal /* Database value. Checksum this. */ -){ - u8 *aVal = (u8 *)pVal; - u32 *pCksum = (u32 *)pCtx; - u32 cksum = *pCksum; - int i; - - unused_parameter(pKey); - unused_parameter(nKey); - - for(i=0; inFanout entries at level -** iLevel-1. -*/ -static u32 dbComputeCksum( - DbParameters *pParam, /* Database parameters */ - TestDb *pDb, /* Database connection handle */ - int iLevel, /* Level of checksum to compute */ - int iKey, /* Compute checksum for this key */ - int *pRc /* IN/OUT: Error code */ -){ - u32 cksum = 0; - if( *pRc==0 ){ - int nFirst; - int nLast; - int iFirst = 0; - int iLast = 0; - int i; - int f = 1; - char zFirst[DB_KEY_BYTES]; - char zLast[DB_KEY_BYTES]; - - assert( iLevel>=1 ); - for(i=0; inFanout; - - iFirst = f*(iKey/f); - iLast = iFirst + f - 1; - dbFormatKey(pParam, iLevel-1, iFirst, zFirst); - dbFormatKey(pParam, iLevel-1, iLast, zLast); - nFirst = strlen(zFirst); - nLast = strlen(zLast); - - *pRc = tdb_scan(pDb, (u32*)&cksum, 0, zFirst, nFirst, zLast, nLast,dbCksum); - } - - return cksum; -} - -static void dbReadOperation( - DbParameters *pParam, /* Database parameters */ - TestDb *pDb, /* Database connection handle */ - void (*xDelay)(void *), - void *pDelayCtx, - int iKey, /* Key to read */ - int *pRc /* IN/OUT: Error code */ -){ - const int iMax = dbMaxLevel(pParam); - int i; - - if( tdb_transaction_support(pDb) ) testBegin(pDb, 1, pRc); - for(i=1; *pRc==0 && i<=iMax; i++){ - char zCksum[DB_KEY_BYTES]; - char zKey[DB_KEY_BYTES]; - u32 iCksum = 0; - - iCksum = dbComputeCksum(pParam, pDb, i, iKey, pRc); - if( iCksum ){ - if( xDelay && i==1 ) xDelay(pDelayCtx); - dbFormatCksumValue(iCksum, zCksum); - dbFormatKey(pParam, i, iKey, zKey); - testFetchStr(pDb, zKey, zCksum, pRc); - } - } - if( tdb_transaction_support(pDb) ) testCommit(pDb, 0, pRc); -} - -static int dbWriteOperation( - DbParameters *pParam, /* Database parameters */ - TestDb *pDb, /* Database connection handle */ - int iKey, /* Key to write to */ - const char *zValue, /* Nul-terminated value to write */ - int *pRc /* IN/OUT: Error code */ -){ - const int iMax = dbMaxLevel(pParam); - char zKey[DB_KEY_BYTES]; - int i; - int rc; - - assert( iKey>=0 && iKeynKey ); - dbFormatKey(pParam, 0, iKey, zKey); - - /* Open a write transaction. This may fail - SQLITE4_BUSY */ - if( *pRc==0 && tdb_transaction_support(pDb) ){ - rc = tdb_begin(pDb, 2); - if( rc==5 ) return 0; - *pRc = rc; - } - - testWriteStr(pDb, zKey, zValue, pRc); - for(i=1; i<=iMax; i++){ - char zCksum[DB_KEY_BYTES]; - u32 iCksum = 0; - - iCksum = dbComputeCksum(pParam, pDb, i, iKey, pRc); - dbFormatCksumValue(iCksum, zCksum); - dbFormatKey(pParam, i, iKey, zKey); - testWriteStr(pDb, zKey, zCksum, pRc); - } - if( tdb_transaction_support(pDb) ) testCommit(pDb, 0, pRc); - return 1; -} - -/************************************************************************* -** The following block contains testXXX() functions that implement a -** wrapper around the systems native multi-thread support. There are no -** synchronization primitives - just functions to launch and join -** threads. Wrapper functions are: -** -** testThreadSupport() -** -** testThreadInit() -** testThreadShutdown() -** testThreadLaunch() -** testThreadWait() -** -** testThreadSetHalt() -** testThreadGetHalt() -** testThreadSetResult() -** testThreadGetResult() -** -** testThreadEnterMutex() -** testThreadLeaveMutex() -*/ -typedef struct ThreadSet ThreadSet; -#ifdef LSM_MUTEX_PTHREADS - -#include -#include - -typedef struct Thread Thread; -struct Thread { - int rc; - char *zMsg; - pthread_t id; - void (*xMain)(ThreadSet *, int, void *); - void *pCtx; - ThreadSet *pThreadSet; -}; - -struct ThreadSet { - int bHalt; /* Halt flag */ - int nThread; /* Number of threads */ - Thread *aThread; /* Array of Thread structures */ - pthread_mutex_t mutex; /* Mutex used for cheating */ -}; - -/* -** Return true if this build supports threads, or false otherwise. If -** this function returns false, no other testThreadXXX() functions should -** be called. -*/ -static int testThreadSupport(){ return 1; } - -/* -** Allocate and return a thread-set handle with enough space allocated -** to handle up to nMax threads. Each call to this function should be -** matched by a call to testThreadShutdown() to delete the object. -*/ -static ThreadSet *testThreadInit(int nMax){ - int nByte; /* Total space to allocate */ - ThreadSet *p; /* Return value */ - - nByte = sizeof(ThreadSet) + sizeof(struct Thread) * nMax; - p = (ThreadSet *)testMalloc(nByte); - p->nThread = nMax; - p->aThread = (Thread *)&p[1]; - pthread_mutex_init(&p->mutex, 0); - - return p; -} - -/* -** Delete a thread-set object and release all resources held by it. -*/ -static void testThreadShutdown(ThreadSet *p){ - int i; - for(i=0; inThread; i++){ - testFree(p->aThread[i].zMsg); - } - pthread_mutex_destroy(&p->mutex); - testFree(p); -} - -static void *ttMain(void *pArg){ - Thread *pThread = (Thread *)pArg; - int iThread; - iThread = (pThread - pThread->pThreadSet->aThread); - pThread->xMain(pThread->pThreadSet, iThread, pThread->pCtx); - return 0; -} - -/* -** Launch a new thread. -*/ -static int testThreadLaunch( - ThreadSet *p, - int iThread, - void (*xMain)(ThreadSet *, int, void *), - void *pCtx -){ - int rc; - Thread *pThread; - - assert( iThread>=0 && iThreadnThread ); - - pThread = &p->aThread[iThread]; - assert( pThread->pThreadSet==0 ); - pThread->xMain = xMain; - pThread->pCtx = pCtx; - pThread->pThreadSet = p; - rc = pthread_create(&pThread->id, 0, ttMain, (void *)pThread); - - return rc; -} - -/* -** Set the thread-set "halt" flag. -*/ -static void testThreadSetHalt(ThreadSet *pThreadSet){ - pThreadSet->bHalt = 1; -} - -/* -** Return the current value of the thread-set "halt" flag. -*/ -static int testThreadGetHalt(ThreadSet *pThreadSet){ - return pThreadSet->bHalt; -} - -static void testThreadSleep(ThreadSet *pThreadSet, int nMs){ - int nRem = nMs; - while( nRem>0 && testThreadGetHalt(pThreadSet)==0 ){ - usleep(50000); - nRem -= 50; - } -} - -/* -** Wait for all threads launched to finish before returning. If nMs -** is greater than zero, set the "halt" flag to tell all threads -** to halt after waiting nMs milliseconds. -*/ -static void testThreadWait(ThreadSet *pThreadSet, int nMs){ - int i; - - testThreadSleep(pThreadSet, nMs); - testThreadSetHalt(pThreadSet); - for(i=0; inThread; i++){ - Thread *pThread = &pThreadSet->aThread[i]; - if( pThread->xMain ){ - pthread_join(pThread->id, 0); - } - } -} - -/* -** Set the result for thread iThread. -*/ -static void testThreadSetResult( - ThreadSet *pThreadSet, /* Thread-set handle */ - int iThread, /* Set result for this thread */ - int rc, /* Result error code */ - char *zFmt, /* Result string format */ - ... /* Result string formatting args... */ -){ - va_list ap; - - testFree(pThreadSet->aThread[iThread].zMsg); - pThreadSet->aThread[iThread].rc = rc; - pThreadSet->aThread[iThread].zMsg = 0; - if( zFmt ){ - va_start(ap, zFmt); - pThreadSet->aThread[iThread].zMsg = testMallocVPrintf(zFmt, ap); - va_end(ap); - } -} - -/* -** Retrieve the result for thread iThread. -*/ -static int testThreadGetResult( - ThreadSet *pThreadSet, /* Thread-set handle */ - int iThread, /* Get result for this thread */ - const char **pzRes /* OUT: Pointer to result string */ -){ - if( pzRes ) *pzRes = pThreadSet->aThread[iThread].zMsg; - return pThreadSet->aThread[iThread].rc; -} - -/* -** Enter and leave the test case mutex. -*/ -#if 0 -static void testThreadEnterMutex(ThreadSet *p){ - pthread_mutex_lock(&p->mutex); -} -static void testThreadLeaveMutex(ThreadSet *p){ - pthread_mutex_unlock(&p->mutex); -} -#endif -#endif - -#if !defined(LSM_MUTEX_PTHREADS) -static int testThreadSupport(){ return 0; } - -#define testThreadInit(a) 0 -#define testThreadShutdown(a) -#define testThreadLaunch(a,b,c,d) 0 -#define testThreadWait(a,b) -#define testThreadSetHalt(a) -#define testThreadGetHalt(a) 0 -#define testThreadGetResult(a,b,c) 0 -#define testThreadSleep(a,b) 0 - -static void testThreadSetResult(ThreadSet *a, int b, int c, char *d, ...){ - unused_parameter(a); - unused_parameter(b); - unused_parameter(c); - unused_parameter(d); -} -#endif -/* End of threads wrapper. -*************************************************************************/ - -/************************************************************************* -** Below this point is the third part of this file - the implementation -** of the mt1.* tests. -*/ -typedef struct Mt1Test Mt1Test; -struct Mt1Test { - DbParameters param; /* Description of database to read/write */ - int nReadwrite; /* Number of read/write threads */ - int nFastReader; /* Number of fast reader threads */ - int nSlowReader; /* Number of slow reader threads */ - int nMs; /* How long to run for */ - const char *zSystem; /* Database system to test */ -}; - -typedef struct Mt1DelayCtx Mt1DelayCtx; -struct Mt1DelayCtx { - ThreadSet *pSet; /* Threadset to sleep within */ - int nMs; /* Sleep in ms */ -}; - -static void xMt1Delay(void *pCtx){ - Mt1DelayCtx *p = (Mt1DelayCtx *)pCtx; - testThreadSleep(p->pSet, p->nMs); -} - -#define MT1_THREAD_RDWR 0 -#define MT1_THREAD_SLOW 1 -#define MT1_THREAD_FAST 2 - -static void xMt1Work(lsm_db *pDb, void *pCtx){ -#if 0 - char *z = 0; - lsm_info(pDb, LSM_INFO_DB_STRUCTURE, &z); - printf("%s\n", z); - fflush(stdout); -#endif -} - -/* -** This is the main() proc for all threads in test case "mt1". -*/ -static void mt1Main(ThreadSet *pThreadSet, int iThread, void *pCtx){ - Mt1Test *p = (Mt1Test *)pCtx; /* Test parameters */ - Mt1DelayCtx delay; - int nRead = 0; /* Number of calls to dbReadOperation() */ - int nWrite = 0; /* Number of completed database writes */ - int rc = 0; /* Error code */ - int iPrng; /* Prng argument variable */ - TestDb *pDb; /* Database handle */ - int eType; - - delay.pSet = pThreadSet; - delay.nMs = 0; - if( iThreadnReadwrite ){ - eType = MT1_THREAD_RDWR; - }else if( iThread<(p->nReadwrite+p->nFastReader) ){ - eType = MT1_THREAD_FAST; - }else{ - eType = MT1_THREAD_SLOW; - delay.nMs = (p->nMs / 20); - } - - /* Open a new database connection. Initialize the pseudo-random number - ** argument based on the thread number. */ - iPrng = testPrngValue(iThread); - pDb = testOpen(p->zSystem, 0, &rc); - - if( rc==0 ){ - tdb_lsm_config_work_hook(pDb, xMt1Work, 0); - } - - /* Loop until either an error occurs or some other thread sets the - ** halt flag. */ - while( rc==0 && testThreadGetHalt(pThreadSet)==0 ){ - int iKey; - - /* Perform a read operation on an arbitrarily selected key. */ - iKey = (testPrngValue(iPrng++) % p->param.nKey); - dbReadOperation(&p->param, pDb, xMt1Delay, (void *)&delay, iKey, &rc); - if( rc ) continue; - nRead++; - - /* Attempt to write an arbitrary key value pair (and update the associated - ** checksum entries). dbWriteOperation() returns 1 if the write is - ** successful, or 0 if it failed with an LSM_BUSY error. */ - if( eType==MT1_THREAD_RDWR ){ - char aValue[50]; - char aRnd[25]; - - iKey = (testPrngValue(iPrng++) % p->param.nKey); - testPrngString(iPrng, aRnd, sizeof(aRnd)); - iPrng += sizeof(aRnd); - snprintf(aValue, sizeof(aValue), "%d.%s", iThread, aRnd); - nWrite += dbWriteOperation(&p->param, pDb, iKey, aValue, &rc); - } - } - testClose(&pDb); - - /* If an error has occured, set the thread error code and the threadset - ** halt flag to tell the other test threads to halt. Otherwise, set the - ** thread error code to 0 and post a message with the number of read - ** and write operations completed. */ - if( rc ){ - testThreadSetResult(pThreadSet, iThread, rc, 0); - testThreadSetHalt(pThreadSet); - }else{ - testThreadSetResult(pThreadSet, iThread, 0, "r/w: %d/%d", nRead, nWrite); - } -} - -static void do_test_mt1( - const char *zSystem, /* Database system name */ - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - Mt1Test aTest[] = { - /* param, nReadwrite, nFastReader, nSlowReader, nMs, zSystem */ - { {10, 1000}, 4, 0, 0, 10000, 0 }, - { {10, 1000}, 4, 4, 2, 100000, 0 }, - { {10, 100000}, 4, 0, 0, 10000, 0 }, - { {10, 100000}, 4, 4, 2, 100000, 0 }, - }; - int i; - - for(i=0; *pRc==0 && iparam.nFanout, p->param.nKey, - p->nMs, p->nReadwrite, p->nFastReader, p->nSlowReader - ); - if( bRun ){ - TestDb *pDb; - ThreadSet *pSet; - int iThread; - int nThread; - - p->zSystem = zSystem; - pDb = testOpen(zSystem, 1, pRc); - - nThread = p->nReadwrite + p->nFastReader + p->nSlowReader; - pSet = testThreadInit(nThread); - for(iThread=0; *pRc==0 && iThreadnMs); - for(iThread=0; *pRc==0 && iThreadiNext = 1; - p->bEnable = 1; - p->nFail = 1; - p->pEnv = tdb_lsm_env(); -} - -static void xOomHook(OomTest *p){ - p->nFail++; -} - -static int testOomContinue(OomTest *p){ - if( p->rc!=0 || (p->iNext>1 && p->nFail==0) ){ - return 0; - } - p->nFail = 0; - testMallocOom(p->pEnv, p->iNext, 0, (void (*)(void*))xOomHook, (void *)p); - return 1; -} - -static void testOomEnable(OomTest *p, int bEnable){ - p->bEnable = bEnable; - testMallocOomEnable(p->pEnv, bEnable); -} - -static void testOomNext(OomTest *p){ - p->iNext++; -} - -static int testOomHit(OomTest *p){ - return (p->nFail>0); -} - -static int testOomFinish(OomTest *p){ - return p->rc; -} - -static void testOomAssert(OomTest *p, int bVal){ - if( bVal==0 ){ - test_failed(); - p->rc = 1; - } -} - -/* -** Test that the error code matches the state of the OomTest object passed -** as the first argument. Specifically, check that rc is LSM_NOMEM if an -** OOM error has already been injected, or LSM_OK if not. -*/ -static void testOomAssertRc(OomTest *p, int rc){ - testOomAssert(p, rc==LSM_OK || rc==LSM_NOMEM); - testOomAssert(p, testOomHit(p)==(rc==LSM_NOMEM) || p->bEnable==0 ); -} - -static void testOomOpen( - OomTest *pOom, - const char *zName, - lsm_db **ppDb, - int *pRc -){ - if( *pRc==LSM_OK ){ - int rc; - rc = lsm_new(tdb_lsm_env(), ppDb); - if( rc==LSM_OK ) rc = lsm_open(*ppDb, zName); - testOomAssertRc(pOom, rc); - *pRc = rc; - } -} - -static void testOomFetch( - OomTest *pOom, - lsm_db *pDb, - void *pKey, int nKey, - void *pVal, int nVal, - int *pRc -){ - testOomAssertRc(pOom, *pRc); - if( *pRc==LSM_OK ){ - lsm_cursor *pCsr; - int rc; - - rc = lsm_csr_open(pDb, &pCsr); - if( rc==LSM_OK ) rc = lsm_csr_seek(pCsr, pKey, nKey, 0); - testOomAssertRc(pOom, rc); - - if( rc==LSM_OK ){ - const void *p; int n; - testOomAssert(pOom, lsm_csr_valid(pCsr)); - - rc = lsm_csr_key(pCsr, &p, &n); - testOomAssertRc(pOom, rc); - testOomAssert(pOom, rc!=LSM_OK || (n==nKey && memcmp(pKey, p, nKey)==0) ); - } - - if( rc==LSM_OK ){ - const void *p; int n; - testOomAssert(pOom, lsm_csr_valid(pCsr)); - - rc = lsm_csr_value(pCsr, &p, &n); - testOomAssertRc(pOom, rc); - testOomAssert(pOom, rc!=LSM_OK || (n==nVal && memcmp(pVal, p, nVal)==0) ); - } - - lsm_csr_close(pCsr); - *pRc = rc; - } -} - -static void testOomWrite( - OomTest *pOom, - lsm_db *pDb, - void *pKey, int nKey, - void *pVal, int nVal, - int *pRc -){ - testOomAssertRc(pOom, *pRc); - if( *pRc==LSM_OK ){ - int rc; - - rc = lsm_insert(pDb, pKey, nKey, pVal, nVal); - testOomAssertRc(pOom, rc); - - *pRc = rc; - } -} - - -static void testOomFetchStr( - OomTest *pOom, - lsm_db *pDb, - const char *zKey, - const char *zVal, - int *pRc -){ - int nKey = strlen(zKey); - int nVal = strlen(zVal); - testOomFetch(pOom, pDb, (void *)zKey, nKey, (void *)zVal, nVal, pRc); -} - -static void testOomFetchData( - OomTest *pOom, - lsm_db *pDb, - Datasource *pData, - int iKey, - int *pRc -){ - void *pKey; int nKey; - void *pVal; int nVal; - testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal); - testOomFetch(pOom, pDb, pKey, nKey, pVal, nVal, pRc); -} - -static void testOomWriteStr( - OomTest *pOom, - lsm_db *pDb, - const char *zKey, - const char *zVal, - int *pRc -){ - int nKey = strlen(zKey); - int nVal = strlen(zVal); - testOomWrite(pOom, pDb, (void *)zKey, nKey, (void *)zVal, nVal, pRc); -} - -static void testOomWriteData( - OomTest *pOom, - lsm_db *pDb, - Datasource *pData, - int iKey, - int *pRc -){ - void *pKey; int nKey; - void *pVal; int nVal; - testDatasourceEntry(pData, iKey, &pKey, &nKey, &pVal, &nVal); - testOomWrite(pOom, pDb, pKey, nKey, pVal, nVal, pRc); -} - -static void testOomScan( - OomTest *pOom, - lsm_db *pDb, - int bReverse, - const void *pKey, int nKey, - int nScan, - int *pRc -){ - if( *pRc==0 ){ - int rc; - int iScan = 0; - lsm_cursor *pCsr; - int (*xAdvance)(lsm_cursor *) = 0; - - - rc = lsm_csr_open(pDb, &pCsr); - testOomAssertRc(pOom, rc); - - if( rc==LSM_OK ){ - if( bReverse ){ - rc = lsm_csr_seek(pCsr, pKey, nKey, LSM_SEEK_LE); - xAdvance = lsm_csr_prev; - }else{ - rc = lsm_csr_seek(pCsr, pKey, nKey, LSM_SEEK_GE); - xAdvance = lsm_csr_next; - } - } - testOomAssertRc(pOom, rc); - - while( rc==LSM_OK && lsm_csr_valid(pCsr) && iScan "one" -** "two" -> "four" -** "three" -> "nine" -** "four" -> "sixteen" -** "five" -> "twentyfive" -** "six" -> "thirtysix" -** "seven" -> "fourtynine" -** "eight" -> "sixtyfour" -*/ -static void setup_populate_db(void){ - const char *azStr[] = { - "one", "one", - "two", "four", - "three", "nine", - "four", "sixteen", - "five", "twentyfive", - "six", "thirtysix", - "seven", "fourtynine", - "eight", "sixtyfour", - }; - int rc; - int ii; - lsm_db *pDb; - - testDeleteLsmdb(LSMTEST6_TESTDB); - - rc = lsm_new(tdb_lsm_env(), &pDb); - if( rc==LSM_OK ) rc = lsm_open(pDb, LSMTEST6_TESTDB); - - for(ii=0; rc==LSM_OK && iiiInsStart, pStep->nIns, pRc); - testDeleteDatasourceRange(pDb, pData, pStep->iDelStart, pStep->nDel, pRc); - if( *pRc==0 ){ - int nSave = -1; - int nBuf = 64; - lsm_db *db = tdb_lsm(pDb); - - lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nSave); - lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nBuf); - lsm_begin(db, 1); - lsm_commit(db, 0); - lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nSave); - - *pRc = lsm_work(db, 0, 0, 0); - if( *pRc==0 ){ - *pRc = lsm_checkpoint(db, 0); - } - } -} - -static void doSetupStepArray( - TestDb *pDb, - Datasource *pData, - const SetupStep *aStep, - int nStep -){ - int i; - for(i=0; i -void testReadFile(const char *zFile, int iOff, void *pOut, int nByte, int *pRc){ - if( *pRc==0 ){ - FILE *fd; - fd = fopen(zFile, "rb"); - if( fd==0 ){ - *pRc = 1; - }else{ - if( 0!=fseek(fd, iOff, SEEK_SET) ){ - *pRc = 1; - }else{ - assert( nByte>=0 ); - if( (size_t)nByte!=fread(pOut, 1, nByte, fd) ){ - *pRc = 1; - } - } - fclose(fd); - } - } -} - -void testWriteFile( - const char *zFile, - int iOff, - void *pOut, - int nByte, - int *pRc -){ - if( *pRc==0 ){ - FILE *fd; - fd = fopen(zFile, "r+b"); - if( fd==0 ){ - *pRc = 1; - }else{ - if( 0!=fseek(fd, iOff, SEEK_SET) ){ - *pRc = 1; - }else{ - assert( nByte>=0 ); - if( (size_t)nByte!=fwrite(pOut, 1, nByte, fd) ){ - *pRc = 1; - } - } - fclose(fd); - } - } -} - -static ShmHeader *getShmHeader(const char *zDb){ - int rc = 0; - char *zShm = testMallocPrintf("%s-shm", zDb); - ShmHeader *pHdr; - - pHdr = testMalloc(sizeof(ShmHeader)); - testReadFile(zShm, 0, (void *)pHdr, sizeof(ShmHeader), &rc); - assert( rc==0 ); - - return pHdr; -} - -/* -** This function makes a copy of the three files associated with LSM -** database zDb (i.e. if zDb is "test.db", it makes copies of "test.db", -** "test.db-log" and "test.db-shm"). -** -** It then opens a new database connection to the copy with the xLock() call -** instrumented so that it appears that some other process already connected -** to the db (holding a shared lock on DMS2). This prevents recovery from -** running. Then: -** -** 1) Check that the checksum of the database is zCksum. -** 2) Write a few keys to the database. Then delete the same keys. -** 3) Check that the checksum is zCksum. -** 4) Flush the db to disk and run a checkpoint. -** 5) Check once more that the checksum is still zCksum. -*/ -static void doLiveRecovery(const char *zDb, const char *zCksum, int *pRc){ - if( *pRc==LSM_OK ){ - const DatasourceDefn defn = {TEST_DATASOURCE_RANDOM, 20, 25, 100, 500}; - Datasource *pData; - const char *zCopy = "testcopy.lsm"; - char zCksum2[TEST_CKSUM_BYTES]; - TestDb *pDb = 0; - int rc; - - pData = testDatasourceNew(&defn); - - testCopyLsmdb(zDb, zCopy); - rc = tdb_lsm_open("test_no_recovery=1", zCopy, 0, &pDb); - if( rc==0 ){ - ShmHeader *pHdr; - lsm_db *db; - testCksumDatabase(pDb, zCksum2); - testCompareStr(zCksum, zCksum2, &rc); - - testWriteDatasourceRange(pDb, pData, 1, 10, &rc); - testDeleteDatasourceRange(pDb, pData, 1, 10, &rc); - - /* Test that the two tree-headers are now consistent. */ - pHdr = getShmHeader(zCopy); - if( rc==0 && memcmp(&pHdr->hdr1, &pHdr->hdr2, sizeof(pHdr->hdr1)) ){ - rc = 1; - } - testFree(pHdr); - - if( rc==0 ){ - int nBuf = 64; - db = tdb_lsm(pDb); - lsm_config(db, LSM_CONFIG_AUTOFLUSH, &nBuf); - lsm_begin(db, 1); - lsm_commit(db, 0); - rc = lsm_work(db, 0, 0, 0); - } - - testCksumDatabase(pDb, zCksum2); - testCompareStr(zCksum, zCksum2, &rc); - } - - testDatasourceFree(pData); - testClose(&pDb); - testDeleteLsmdb(zCopy); - *pRc = rc; - } -} - -static void doWriterCrash1(int *pRc){ - const int nWrite = 2000; - const int nStep = 10; - const int iWriteStart = 20000; - int rc = 0; - TestDb *pDb = 0; - Datasource *pData = 0; - - rc = tdb_lsm_open("autowork=0", "testdb.lsm", 1, &pDb); - if( rc==0 ){ - int iDot = 0; - char zCksum[TEST_CKSUM_BYTES]; - int i; - setupDatabase1(pDb, &pData); - testCksumDatabase(pDb, zCksum); - testBegin(pDb, 2, &rc); - for(i=0; rc==0 && ihdr1, &pHdr1->hdr1, sizeof(pHdr1->hdr1)); - pHdr2->bWriter = 1; - testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); - doLiveRecovery("testdb.lsm", zCksum1, &rc); - - /* If both tree-headers are valid, tree-header-1 is used. */ - memcpy(&pHdr2->hdr1, &pHdr2->hdr2, sizeof(pHdr1->hdr1)); - memcpy(&pHdr2->hdr2, &pHdr1->hdr1, sizeof(pHdr1->hdr1)); - pHdr2->bWriter = 1; - testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); - doLiveRecovery("testdb.lsm", zCksum2, &rc); - - /* If tree-header 1 is invalid, tree-header-2 is used */ - memcpy(&pHdr2->hdr2, &pHdr2->hdr1, sizeof(pHdr1->hdr1)); - pHdr2->hdr1.aCksum[0] = 5; - pHdr2->hdr1.aCksum[0] = 6; - pHdr2->bWriter = 1; - testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); - doLiveRecovery("testdb.lsm", zCksum2, &rc); - - /* If tree-header 2 is invalid, tree-header-1 is used */ - memcpy(&pHdr2->hdr1, &pHdr2->hdr2, sizeof(pHdr1->hdr1)); - pHdr2->hdr2.aCksum[0] = 5; - pHdr2->hdr2.aCksum[0] = 6; - pHdr2->bWriter = 1; - testWriteFile("testdb.lsm-shm", 0, (void *)pHdr2, sizeof(ShmHeader), &rc); - doLiveRecovery("testdb.lsm", zCksum2, &rc); - - testFree(pHdr1); - testFree(pHdr2); - testClose(&pDb); - } - - *pRc = rc; -} - -void do_writer_crash_test(const char *zPattern, int *pRc){ - struct Test { - const char *zName; - void (*xFunc)(int *); - } aTest[] = { - { "writercrash1.lsm", doWriterCrash1 }, - { "writercrash2.lsm", doWriterCrash2 }, - }; - int i; - for(i=0; izName) ){ - p->xFunc(pRc); - testCaseFinish(*pRc); - } - } - -} diff --git a/ext/lsm1/lsm-test/lsmtest9.c b/ext/lsm1/lsm-test/lsmtest9.c deleted file mode 100644 index b01de0d4e..000000000 --- a/ext/lsm1/lsm-test/lsmtest9.c +++ /dev/null @@ -1,140 +0,0 @@ - -#include "lsmtest.h" - -#define DATA_SEQUENTIAL TEST_DATASOURCE_SEQUENCE -#define DATA_RANDOM TEST_DATASOURCE_RANDOM - -typedef struct Datatest4 Datatest4; - -/* -** Test overview: -** -** 1. Insert (Datatest4.nRec) records into a database. -** -** 2. Repeat (Datatest4.nRepeat) times: -** -** 2a. Delete 2/3 of the records in the database. -** -** 2b. Run lsm_work(nMerge=1). -** -** 2c. Insert as many records as were deleted in 2a. -** -** 2d. Check database content is as expected. -** -** 2e. If (Datatest4.bReopen) is true, close and reopen the database. -*/ -struct Datatest4 { - /* Datasource definition */ - DatasourceDefn defn; - - int nRec; - int nRepeat; - int bReopen; -}; - -static void doDataTest4( - const char *zSystem, /* Database system to test */ - Datatest4 *p, /* Structure containing test parameters */ - int *pRc /* OUT: Error code */ -){ - lsm_db *db = 0; - TestDb *pDb; - TestDb *pControl; - Datasource *pData; - int i; - int rc = 0; - int iDot = 0; - int bMultiThreaded = 0; /* True for MT LSM database */ - - int nRecOn3 = (p->nRec / 3); - int iData = 0; - - /* Start the test case, open a database and allocate the datasource. */ - rc = testControlDb(&pControl); - pDb = testOpen(zSystem, 1, &rc); - pData = testDatasourceNew(&p->defn); - if( rc==0 ){ - db = tdb_lsm(pDb); - bMultiThreaded = tdb_lsm_multithread(pDb); - } - - testWriteDatasourceRange(pControl, pData, iData, nRecOn3*3, &rc); - testWriteDatasourceRange(pDb, pData, iData, nRecOn3*3, &rc); - - for(i=0; rc==0 && inRepeat; i++){ - - testDeleteDatasourceRange(pControl, pData, iData, nRecOn3*2, &rc); - testDeleteDatasourceRange(pDb, pData, iData, nRecOn3*2, &rc); - - if( db ){ - int nDone; -#if 0 - fprintf(stderr, "lsm_work() start...\n"); fflush(stderr); -#endif - do { - nDone = 0; - rc = lsm_work(db, 1, (1<<30), &nDone); - }while( rc==0 && nDone>0 ); - if( bMultiThreaded && rc==LSM_BUSY ) rc = LSM_OK; -#if 0 - fprintf(stderr, "lsm_work() done...\n"); fflush(stderr); -#endif - } - -if( i+1nRepeat ){ - iData += (nRecOn3*2); - testWriteDatasourceRange(pControl, pData, iData+nRecOn3, nRecOn3*2, &rc); - testWriteDatasourceRange(pDb, pData, iData+nRecOn3, nRecOn3*2, &rc); - - testCompareDb(pData, nRecOn3*3, iData, pControl, pDb, &rc); - - /* If Datatest4.bReopen is true, close and reopen the database */ - if( p->bReopen ){ - testReopen(&pDb, &rc); - if( rc==0 ) db = tdb_lsm(pDb); - } -} - - /* Update the progress dots... */ - testCaseProgress(i, p->nRepeat, testCaseNDot(), &iDot); - } - - testClose(&pDb); - testClose(&pControl); - testDatasourceFree(pData); - testCaseFinish(rc); - *pRc = rc; -} - -static char *getName4(const char *zSystem, Datatest4 *pTest){ - char *zRet; - char *zData; - zData = testDatasourceName(&pTest->defn); - zRet = testMallocPrintf("data4.%s.%s.%d.%d.%d", - zSystem, zData, pTest->nRec, pTest->nRepeat, pTest->bReopen - ); - testFree(zData); - return zRet; -} - -void test_data_4( - const char *zSystem, /* Database system name */ - const char *zPattern, /* Run test cases that match this pattern */ - int *pRc /* IN/OUT: Error code */ -){ - Datatest4 aTest[] = { - /* defn, nRec, nRepeat, bReopen */ - { {DATA_RANDOM, 20,25, 500,600}, 10000, 10, 0 }, - { {DATA_RANDOM, 20,25, 500,600}, 10000, 10, 1 }, - }; - - int i; - - for(i=0; *pRc==LSM_OK && ieType ){ - case TEST_DATASOURCE_RANDOM: { - int nRange = (1 + p->nMaxKey - p->nMinKey); - nKey = (int)( testPrngValue((u32)iData) % nRange ) + p->nMinKey; - testPrngString((u32)iData, p->aKey, nKey); - break; - } - case TEST_DATASOURCE_SEQUENCE: - nKey = sprintf(p->aKey, "%012d", iData); - break; - } - *ppKey = p->aKey; - *pnKey = nKey; - } - if( ppVal ){ - u32 nVal = testPrngValue((u32)iData)%(1+p->nMaxVal-p->nMinVal)+p->nMinVal; - testPrngString((u32)~iData, p->aVal, (int)nVal); - *ppVal = p->aVal; - *pnVal = (int)nVal; - } -} - -void testDatasourceFree(Datasource *p){ - testFree(p); -} - -/* -** Return a pointer to a nul-terminated string that corresponds to the -** contents of the datasource-definition passed as the first argument. -** The caller should eventually free the returned pointer using testFree(). -*/ -char *testDatasourceName(const DatasourceDefn *p){ - char *zRet; - zRet = testMallocPrintf("%s.(%d-%d).(%d-%d)", - (p->eType==TEST_DATASOURCE_SEQUENCE ? "seq" : "rnd"), - p->nMinKey, p->nMaxKey, - p->nMinVal, p->nMaxVal - ); - return zRet; -} - -Datasource *testDatasourceNew(const DatasourceDefn *pDefn){ - Datasource *p; - int nMinKey; - int nMaxKey; - int nMinVal; - int nMaxVal; - - if( pDefn->eType==TEST_DATASOURCE_SEQUENCE ){ - nMinKey = 128; - nMaxKey = 128; - }else{ - nMinKey = MAX(0, pDefn->nMinKey); - nMaxKey = MAX(nMinKey, pDefn->nMaxKey); - } - nMinVal = MAX(0, pDefn->nMinVal); - nMaxVal = MAX(nMinVal, pDefn->nMaxVal); - - p = (Datasource *)testMalloc(sizeof(Datasource) + nMaxKey + nMaxVal + 1); - p->eType = pDefn->eType; - p->nMinKey = nMinKey; - p->nMinVal = nMinVal; - p->nMaxKey = nMaxKey; - p->nMaxVal = nMaxVal; - - p->aKey = (char *)&p[1]; - p->aVal = &p->aKey[nMaxKey]; - return p; -}; diff --git a/ext/lsm1/lsm-test/lsmtest_func.c b/ext/lsm1/lsm-test/lsmtest_func.c deleted file mode 100644 index eb8346aa8..000000000 --- a/ext/lsm1/lsm-test/lsmtest_func.c +++ /dev/null @@ -1,177 +0,0 @@ - -#include "lsmtest.h" - - -int do_work(int nArg, char **azArg){ - struct Option { - const char *zName; - } aOpt [] = { - { "-nmerge" }, - { "-nkb" }, - { 0 } - }; - - lsm_db *pDb; - int rc; - int i; - const char *zDb; - int nMerge = 1; - int nKB = (1<<30); - - if( nArg==0 ) goto usage; - zDb = azArg[nArg-1]; - for(i=0; i<(nArg-1); i++){ - int iSel; - rc = testArgSelect(aOpt, "option", azArg[i], &iSel); - if( rc ) return rc; - switch( iSel ){ - case 0: - i++; - if( i==(nArg-1) ) goto usage; - nMerge = atoi(azArg[i]); - break; - case 1: - i++; - if( i==(nArg-1) ) goto usage; - nKB = atoi(azArg[i]); - break; - } - } - - rc = lsm_new(0, &pDb); - if( rc!=LSM_OK ){ - testPrintError("lsm_open(): rc=%d\n", rc); - }else{ - rc = lsm_open(pDb, zDb); - if( rc!=LSM_OK ){ - testPrintError("lsm_open(): rc=%d\n", rc); - }else{ - int n = -1; - lsm_config(pDb, LSM_CONFIG_BLOCK_SIZE, &n); - n = n*2; - lsm_config(pDb, LSM_CONFIG_AUTOCHECKPOINT, &n); - - rc = lsm_work(pDb, nMerge, nKB, 0); - if( rc!=LSM_OK ){ - testPrintError("lsm_work(): rc=%d\n", rc); - } - } - } - if( rc==LSM_OK ){ - rc = lsm_checkpoint(pDb, 0); - } - - lsm_close(pDb); - return rc; - - usage: - testPrintUsage("?-optimize? ?-n N? DATABASE"); - return -1; -} - - -/* -** lsmtest show ?-config LSM-CONFIG? DATABASE ?COMMAND ?PGNO?? -*/ -int do_show(int nArg, char **azArg){ - lsm_db *pDb; - int rc; - const char *zDb; - - int eOpt = LSM_INFO_DB_STRUCTURE; - unsigned int iPg = 0; - int bConfig = 0; - const char *zConfig = ""; - - struct Option { - const char *zName; - int bConfig; - int eOpt; - } aOpt [] = { - { "array", 0, LSM_INFO_ARRAY_STRUCTURE }, - { "array-pages", 0, LSM_INFO_ARRAY_PAGES }, - { "blocksize", 1, LSM_CONFIG_BLOCK_SIZE }, - { "pagesize", 1, LSM_CONFIG_PAGE_SIZE }, - { "freelist", 0, LSM_INFO_FREELIST }, - { "page-ascii", 0, LSM_INFO_PAGE_ASCII_DUMP }, - { "page-hex", 0, LSM_INFO_PAGE_HEX_DUMP }, - { 0, 0 } - }; - - char *z = 0; - int iDb = 0; /* Index of DATABASE in azArg[] */ - - /* Check if there is a "-config" option: */ - if( nArg>2 && strlen(azArg[0])>1 - && memcmp(azArg[0], "-config", strlen(azArg[0]))==0 - ){ - zConfig = azArg[1]; - iDb = 2; - } - if( nArg<(iDb+1) ) goto usage; - - if( nArg>(iDb+1) ){ - rc = testArgSelect(aOpt, "option", azArg[iDb+1], &eOpt); - if( rc!=0 ) return rc; - bConfig = aOpt[eOpt].bConfig; - eOpt = aOpt[eOpt].eOpt; - if( (bConfig==0 && eOpt==LSM_INFO_FREELIST) - || (bConfig==1 && eOpt==LSM_CONFIG_BLOCK_SIZE) - || (bConfig==1 && eOpt==LSM_CONFIG_PAGE_SIZE) - ){ - if( nArg!=(iDb+2) ) goto usage; - }else{ - if( nArg!=(iDb+3) ) goto usage; - iPg = atoi(azArg[iDb+2]); - } - } - zDb = azArg[iDb]; - - rc = lsm_new(0, &pDb); - tdb_lsm_configure(pDb, zConfig); - if( rc!=LSM_OK ){ - testPrintError("lsm_new(): rc=%d\n", rc); - }else{ - rc = lsm_open(pDb, zDb); - if( rc!=LSM_OK ){ - testPrintError("lsm_open(): rc=%d\n", rc); - } - } - - if( rc==LSM_OK ){ - if( bConfig==0 ){ - switch( eOpt ){ - case LSM_INFO_DB_STRUCTURE: - case LSM_INFO_FREELIST: - rc = lsm_info(pDb, eOpt, &z); - break; - case LSM_INFO_ARRAY_STRUCTURE: - case LSM_INFO_ARRAY_PAGES: - case LSM_INFO_PAGE_ASCII_DUMP: - case LSM_INFO_PAGE_HEX_DUMP: - rc = lsm_info(pDb, eOpt, iPg, &z); - break; - default: - assert( !"no chance" ); - } - - if( rc==LSM_OK ){ - printf("%s\n", z ? z : ""); - fflush(stdout); - } - lsm_free(lsm_get_env(pDb), z); - }else{ - int iRes = -1; - lsm_config(pDb, eOpt, &iRes); - printf("%d\n", iRes); - fflush(stdout); - } - } - - lsm_close(pDb); - return rc; - - usage: - testPrintUsage("DATABASE ?array|page-ascii|page-hex PGNO?"); - return -1; -} diff --git a/ext/lsm1/lsm-test/lsmtest_io.c b/ext/lsm1/lsm-test/lsmtest_io.c deleted file mode 100644 index 7aa5d1094..000000000 --- a/ext/lsm1/lsm-test/lsmtest_io.c +++ /dev/null @@ -1,248 +0,0 @@ - -/* -** SUMMARY -** -** This file implements the 'io' subcommand of the test program. It is used -** for testing the performance of various combinations of write() and fsync() -** system calls. All operations occur on a single file, which may or may not -** exist when a test is started. -** -** A test consists of a series of commands. Each command is either a write -** or an fsync. A write is specified as "@", where -** is the amount of data written, and is the offset of the file -** to write to. An or an is specified as an integer number -** of bytes. Or, if postfixed with a "K", "M" or "G", an integer number of -** KB, MB or GB, respectively. An fsync is simply "S". All commands are -** case-insensitive. -** -** Example test program: -** -** 2M@6M 1492K@4M S 4096@4K S -** -** This program writes 2 MB of data starting at the offset 6MB offset of -** the file, followed by 1492 KB of data written at the 4MB offset of the -** file, followed by a call to fsync(), a write of 4KB of data at byte -** offset 4096, and finally another call to fsync(). -** -** Commands may either be specified on the command line (one command per -** command line argument) or read from stdin. Commands read from stdin -** must be separated by white-space. -** -** COMMAND LINE INVOCATION -** -** The sub-command implemented in this file must be invoked with at least -** two arguments - the path to the file to write to and the page-size to -** use for writing. If there are more than two arguments, then each -** subsequent argument is assumed to be a test command. If there are exactly -** two arguments, the test commands are read from stdin. -** -** A write command does not result in a single call to system call write(). -** Instead, the specified region is written sequentially using one or -** more calls to write(), each of which writes not more than one page of -** data. For example, if the page-size is 4KB, the command "2M@6M" results -** in 512 calls to write(), each of which writes 4KB of data. -** -** EXAMPLES -** -** Two equivalent examples: -** -** $ lsmtest io testfile.db 4KB 2M@6M 1492K@4M S 4096@4K S -** 3544K written in 129 ms -** $ echo "2M@6M 1492K@4M S 4096@4K S" | lsmtest io testfile.db 4096 -** 3544K written in 127 ms -** -*/ - -#include "lsmtest.h" - -typedef struct IoContext IoContext; - -struct IoContext { - int fd; - int nWrite; -}; - -/* -** As isspace(3) -*/ -static int safe_isspace(char c){ - if( c&0x80) return 0; - return isspace(c); -} - -/* -** As isdigit(3) -*/ -static int safe_isdigit(char c){ - if( c&0x80) return 0; - return isdigit(c); -} - -static i64 getNextSize(char *zIn, char **pzOut, int *pRc){ - i64 iRet = 0; - if( *pRc==0 ){ - char *z = zIn; - - if( !safe_isdigit(*z) ){ - *pRc = 1; - return 0; - } - - /* Process digits */ - while( safe_isdigit(*z) ){ - iRet = iRet*10 + (*z - '0'); - z++; - } - - /* Process suffix */ - switch( *z ){ - case 'k': case 'K': - iRet = iRet * 1024; - z++; - break; - - case 'm': case 'M': - iRet = iRet * 1024 * 1024; - z++; - break; - - case 'g': case 'G': - iRet = iRet * 1024 * 1024 * 1024; - z++; - break; - } - - if( pzOut ) *pzOut = z; - } - return iRet; -} - -static int doOneCmd( - IoContext *pCtx, - u8 *aData, - int pgsz, - char *zCmd, - char **pzOut -){ - char c; - char *z = zCmd; - - while( safe_isspace(*z) ) z++; - c = *z; - - if( c==0 ){ - if( pzOut ) *pzOut = z; - return 0; - } - - if( c=='s' || c=='S' ){ - if( pzOut ) *pzOut = &z[1]; - return fdatasync(pCtx->fd); - } - - if( safe_isdigit(c) ){ - i64 iOff = 0; - int nByte = 0; - int rc = 0; - int nPg; - int iPg; - - nByte = (int)getNextSize(z, &z, &rc); - if( rc || *z!='@' ) goto bad_command; - z++; - iOff = getNextSize(z, &z, &rc); - if( rc || (safe_isspace(*z)==0 && *z!='\0') ) goto bad_command; - if( pzOut ) *pzOut = z; - - nPg = (nByte+pgsz-1) / pgsz; - lseek(pCtx->fd, (off_t)iOff, SEEK_SET); - for(iPg=0; iPgfd, aData, pgsz); - } - pCtx->nWrite += nByte/1024; - - return 0; - } - - bad_command: - testPrintError("unrecognized command: %s", zCmd); - return 1; -} - -static int readStdin(char **pzOut){ - int nAlloc = 128; - char *zOut = 0; - int nOut = 0; - - while( !feof(stdin) ){ - int nRead; - - nAlloc = nAlloc*2; - zOut = realloc(zOut, nAlloc); - nRead = fread(&zOut[nOut], 1, nAlloc-nOut-1, stdin); - - if( nRead==0 ) break; - nOut += nRead; - zOut[nOut] = '\0'; - } - - *pzOut = zOut; - return 0; -} - -int do_io(int nArg, char **azArg){ - IoContext ctx; - int pgsz; - char *zFile; - char *zPgsz; - int i; - int rc = 0; - - char *zStdin = 0; - char *z; - - u8 *aData; - - memset(&ctx, 0, sizeof(IoContext)); - if( nArg<2 ){ - testPrintUsage("FILE PGSZ ?CMD-1 ...?"); - return -1; - } - zFile = azArg[0]; - zPgsz = azArg[1]; - - pgsz = (int)getNextSize(zPgsz, 0, &rc); - if( pgsz<=0 ){ - testPrintError("Ridiculous page size: %d", pgsz); - return -1; - } - aData = malloc(pgsz); - memset(aData, 0x77, pgsz); - - ctx.fd = open(zFile, O_RDWR|O_CREAT|_O_BINARY, 0644); - if( ctx.fd<0 ){ - perror("open: "); - return -1; - } - - if( nArg==2 ){ - readStdin(&zStdin); - testTimeInit(); - z = zStdin; - while( *z && rc==0 ){ - rc = doOneCmd(&ctx, aData, pgsz, z, &z); - } - }else{ - testTimeInit(); - for(i=2; i - -void test_failed(){ - assert( 0 ); - return; -} - -#define testSetError(rc) testSetErrorFunc(rc, pRc, __FILE__, __LINE__) -static void testSetErrorFunc(int rc, int *pRc, const char *zFile, int iLine){ - if( rc ){ - *pRc = rc; - fprintf(stderr, "FAILED (%s:%d) rc=%d ", zFile, iLine, rc); - test_failed(); - } -} - -static int lsm_memcmp(u8 *a, u8 *b, int c){ - int i; - for(i=0; i0 && lsm_memcmp(pVal, pDbVal, nVal))) ){ - testSetError(1); - } - } -} - -void testWrite( - TestDb *pDb, /* Database handle */ - void *pKey, int nKey, /* Key to query database for */ - void *pVal, int nVal, /* Value to write */ - int *pRc /* IN/OUT: Error code */ -){ - if( *pRc==0 ){ - int rc; -static int nCall = 0; -nCall++; - rc = tdb_write(pDb, pKey, nKey, pVal, nVal); - testSetError(rc); - } -} -void testDelete( - TestDb *pDb, /* Database handle */ - void *pKey, int nKey, /* Key to query database for */ - int *pRc /* IN/OUT: Error code */ -){ - if( *pRc==0 ){ - int rc; - *pRc = rc = tdb_delete(pDb, pKey, nKey); - testSetError(rc); - } -} -void testDeleteRange( - TestDb *pDb, /* Database handle */ - void *pKey1, int nKey1, - void *pKey2, int nKey2, - int *pRc /* IN/OUT: Error code */ -){ - if( *pRc==0 ){ - int rc; - *pRc = rc = tdb_delete_range(pDb, pKey1, nKey1, pKey2, nKey2); - testSetError(rc); - } -} - -void testBegin(TestDb *pDb, int iTrans, int *pRc){ - if( *pRc==0 ){ - int rc; - rc = tdb_begin(pDb, iTrans); - testSetError(rc); - } -} -void testCommit(TestDb *pDb, int iTrans, int *pRc){ - if( *pRc==0 ){ - int rc; - rc = tdb_commit(pDb, iTrans); - testSetError(rc); - } -} -#if 0 /* unused */ -static void testRollback(TestDb *pDb, int iTrans, int *pRc){ - if( *pRc==0 ){ - int rc; - rc = tdb_rollback(pDb, iTrans); - testSetError(rc); - } -} -#endif - -void testWriteStr( - TestDb *pDb, /* Database handle */ - const char *zKey, /* Key to query database for */ - const char *zVal, /* Value to write */ - int *pRc /* IN/OUT: Error code */ -){ - int nVal = (zVal ? strlen(zVal) : 0); - testWrite(pDb, (void *)zKey, strlen(zKey), (void *)zVal, nVal, pRc); -} - -#if 0 /* unused */ -static void testDeleteStr(TestDb *pDb, const char *zKey, int *pRc){ - testDelete(pDb, (void *)zKey, strlen(zKey), pRc); -} -#endif -void testFetchStr( - TestDb *pDb, /* Database handle */ - const char *zKey, /* Key to query database for */ - const char *zVal, /* Value to write */ - int *pRc /* IN/OUT: Error code */ -){ - int nVal = (zVal ? strlen(zVal) : 0); - testFetch(pDb, (void *)zKey, strlen(zKey), (void *)zVal, nVal, pRc); -} - -void testFetchCompare( - TestDb *pControl, - TestDb *pDb, - void *pKey, int nKey, - int *pRc -){ - int rc; - void *pDbVal1; - void *pDbVal2; - int nDbVal1; - int nDbVal2; - - static int nCall = 0; - nCall++; - - rc = tdb_fetch(pControl, pKey, nKey, &pDbVal1, &nDbVal1); - testSetError(rc); - - rc = tdb_fetch(pDb, pKey, nKey, &pDbVal2, &nDbVal2); - testSetError(rc); - - if( *pRc==0 - && (nDbVal1!=nDbVal2 || (nDbVal1>0 && memcmp(pDbVal1, pDbVal2, nDbVal1))) - ){ - testSetError(1); - } -} - -typedef struct ScanResult ScanResult; -struct ScanResult { - TestDb *pDb; - - int nRow; - u32 cksum1; - u32 cksum2; - void *pKey1; int nKey1; - void *pKey2; int nKey2; - - int bReverse; - int nPrevKey; - u8 aPrevKey[256]; -}; - -static int keyCompare(void *pKey1, int nKey1, void *pKey2, int nKey2){ - int res; - res = memcmp(pKey1, pKey2, MIN(nKey1, nKey2)); - if( res==0 ){ - res = nKey1 - nKey2; - } - return res; -} - -int test_scan_debug = 0; - -static void scanCompareCb( - void *pCtx, - void *pKey, int nKey, - void *pVal, int nVal -){ - ScanResult *p = (ScanResult *)pCtx; - u8 *aKey = (u8 *)pKey; - u8 *aVal = (u8 *)pVal; - int i; - - if( test_scan_debug ){ - printf("%d: %.*s\n", p->nRow, nKey, (char *)pKey); - fflush(stdout); - } -#if 0 - if( test_scan_debug ) printf("%.20s\n", (char *)pVal); -#endif - -#if 0 - /* Check tdb_fetch() matches */ - int rc = 0; - testFetch(p->pDb, pKey, nKey, pVal, nVal, &rc); - assert( rc==0 ); -#endif - - /* Update the checksum data */ - p->nRow++; - for(i=0; icksum1 += ((int)aKey[i] << (i&0x0F)); - p->cksum2 += p->cksum1; - } - for(i=0; icksum1 += ((int)aVal[i] << (i&0x0F)); - p->cksum2 += p->cksum1; - } - - /* Check that the delivered row is not out of order. */ - if( nKey<(int)sizeof(p->aPrevKey) ){ - if( p->nPrevKey ){ - int res = keyCompare(p->aPrevKey, p->nPrevKey, pKey, nKey); - if( (res<0 && p->bReverse) || (res>0 && p->bReverse==0) ){ - testPrintError("Returned key out of order at %s:%d\n", - __FILE__, __LINE__ - ); - } - } - - p->nPrevKey = nKey; - memcpy(p->aPrevKey, pKey, MIN(p->nPrevKey, nKey)); - } - - /* Check that the delivered row is within range. */ - if( p->pKey1 && ( - (memcmp(p->pKey1, pKey, MIN(p->nKey1, nKey))>0) - || (memcmp(p->pKey1, pKey, MIN(p->nKey1, nKey))==0 && p->nKey1>nKey) - )){ - testPrintError("Returned key too small at %s:%d\n", __FILE__, __LINE__); - } - if( p->pKey2 && ( - (memcmp(p->pKey2, pKey, MIN(p->nKey2, nKey))<0) - || (memcmp(p->pKey2, pKey, MIN(p->nKey2, nKey))==0 && p->nKey2=0 ); - zRet = (char *)testMalloc(nByte+1); - vsnprintf(zRet, nByte+1, zFormat, ap); - return zRet; -} - -char *testMallocPrintf(const char *zFormat, ...){ - va_list ap; - char *zRet; - - va_start(ap, zFormat); - zRet = testMallocVPrintf(zFormat, ap); - va_end(ap); - - return zRet; -} - - -/* -** A wrapper around malloc(3). -** -** This function should be used for all allocations made by test procedures. -** It has the following properties: -** -** * Test code may assume that allocations may not fail. -** * Returned memory is always zeroed. -** -** Allocations made using testMalloc() should be freed using testFree(). -*/ -void *testMalloc(int n){ - u8 *p = (u8*)malloc(n + 8); - memset(p, 0, n+8); - *(int*)p = n; - return (void*)&p[8]; -} - -void *testMallocCopy(void *pCopy, int nByte){ - void *pRet = testMalloc(nByte); - memcpy(pRet, pCopy, nByte); - return pRet; -} - -void *testRealloc(void *ptr, int n){ - if( ptr ){ - u8 *p = (u8*)ptr - 8; - int nOrig = *(int*)p; - p = (u8*)realloc(p, n+8); - if( nOrig1 ){ - testPrintError("Usage: test ?PATTERN?\n"); - return 1; - } - if( nArg==1 ){ - zPattern = azArg[0]; - } - - for(j=0; tdb_system_name(j); j++){ - rc = 0; - - test_data_1(tdb_system_name(j), zPattern, &rc); - test_data_2(tdb_system_name(j), zPattern, &rc); - test_data_3(tdb_system_name(j), zPattern, &rc); - test_data_4(tdb_system_name(j), zPattern, &rc); - test_rollback(tdb_system_name(j), zPattern, &rc); - test_mc(tdb_system_name(j), zPattern, &rc); - test_mt(tdb_system_name(j), zPattern, &rc); - - if( rc ) nFail++; - } - - rc = 0; - test_oom(zPattern, &rc); - if( rc ) nFail++; - - rc = 0; - test_api(zPattern, &rc); - if( rc ) nFail++; - - rc = 0; - do_crash_test(zPattern, &rc); - if( rc ) nFail++; - - rc = 0; - do_writer_crash_test(zPattern, &rc); - if( rc ) nFail++; - - return (nFail!=0); -} - -static lsm_db *configure_lsm_db(TestDb *pDb){ - lsm_db *pLsm; - pLsm = tdb_lsm(pDb); - if( pLsm ){ - tdb_lsm_config_str(pDb, "mmap=1 autowork=1 automerge=4 worker_automerge=4"); - } - return pLsm; -} - -typedef struct WriteHookEvent WriteHookEvent; -struct WriteHookEvent { - i64 iOff; - int nData; - int nUs; -}; -WriteHookEvent prev = {0, 0, 0}; - -static void flushPrev(FILE *pOut){ - if( prev.nData ){ - fprintf(pOut, "w %s %lld %d %d\n", "d", prev.iOff, prev.nData, prev.nUs); - prev.nData = 0; - } -} - -#if 0 /* unused */ -static void do_speed_write_hook2( - void *pCtx, - int bLog, - i64 iOff, - int nData, - int nUs -){ - FILE *pOut = (FILE *)pCtx; - if( bLog ) return; - - if( prev.nData && nData && iOff==prev.iOff+prev.nData ){ - prev.nData += nData; - prev.nUs += nUs; - }else{ - flushPrev(pOut); - if( nData==0 ){ - fprintf(pOut, "s %s 0 0 %d\n", (bLog ? "l" : "d"), nUs); - }else{ - prev.iOff = iOff; - prev.nData = nData; - prev.nUs = nUs; - } - } -} -#endif - -#define ST_REPEAT 0 -#define ST_WRITE 1 -#define ST_PAUSE 2 -#define ST_FETCH 3 -#define ST_SCAN 4 -#define ST_NSCAN 5 -#define ST_KEYSIZE 6 -#define ST_VALSIZE 7 -#define ST_TRANS 8 - - -static void print_speed_test_help(){ - printf( -"\n" -"Repeat the following $repeat times:\n" -" 1. Insert $write key-value pairs. One transaction for each write op.\n" -" 2. Pause for $pause ms.\n" -" 3. Perform $fetch queries on the database.\n" -"\n" -" Keys are $keysize bytes in size. Values are $valsize bytes in size\n" -" Both keys and values are pseudo-randomly generated\n" -"\n" -"Options are:\n" -" -repeat $repeat (default value 10)\n" -" -write $write (default value 10000)\n" -" -pause $pause (default value 0)\n" -" -fetch $fetch (default value 0)\n" -" -keysize $keysize (default value 12)\n" -" -valsize $valsize (default value 100)\n" -" -system $system (default value \"lsm\")\n" -" -trans $trans (default value 0)\n" -"\n" -); -} - -int do_speed_test2(int nArg, char **azArg){ - struct Option { - const char *zOpt; - int eVal; - int iDefault; - } aOpt[] = { - { "-repeat", ST_REPEAT, 10}, - { "-write", ST_WRITE, 10000}, - { "-pause", ST_PAUSE, 0}, - { "-fetch", ST_FETCH, 0}, - { "-scan", ST_SCAN, 0}, - { "-nscan", ST_NSCAN, 0}, - { "-keysize", ST_KEYSIZE, 12}, - { "-valsize", ST_VALSIZE, 100}, - { "-trans", ST_TRANS, 0}, - { "-system", -1, 0}, - { "help", -2, 0}, - {0, 0, 0} - }; - int i; - int aParam[9]; - int rc = 0; - int bReadonly = 0; - int nContent = 0; - - TestDb *pDb; - Datasource *pData; - DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 0, 0, 0, 0 }; - char *zSystem = ""; - int bLsm = 1; - FILE *pLog = 0; - -#ifdef NDEBUG - /* If NDEBUG is defined, disable the dynamic memory related checks in - ** lsmtest_mem.c. They slow things down. */ - testMallocUninstall(tdb_lsm_env()); -#endif - - /* Initialize aParam[] with default values. */ - for(i=0; i=0 ){ - aParam[aOpt[iSel].eVal] = atoi(azArg[i+1]); - }else{ - zSystem = azArg[i+1]; - bLsm = 0; -#if 0 - for(j=0; zSystem[j]; j++){ - if( zSystem[j]=='=' ) bLsm = 1; - } -#endif - } - } - - printf("#"); - for(i=0; i=0 ){ - printf(" %s=%d", &aOpt[i].zOpt[1], aParam[aOpt[i].eVal]); - }else if( aOpt[i].eVal==-1 ){ - printf(" %s=\"%s\"", &aOpt[i].zOpt[1], zSystem); - } - } - } - printf("\n"); - - defn.nMinKey = defn.nMaxKey = aParam[ST_KEYSIZE]; - defn.nMinVal = defn.nMaxVal = aParam[ST_VALSIZE]; - pData = testDatasourceNew(&defn); - - if( aParam[ST_WRITE]==0 ){ - bReadonly = 1; - } - - if( bLsm ){ - rc = tdb_lsm_open(zSystem, "testdb.lsm", !bReadonly, &pDb); - }else{ - pDb = testOpen(zSystem, !bReadonly, &rc); - } - if( rc!=0 ) return rc; - if( bReadonly ){ - nContent = testCountDatabase(pDb); - } - -#if 0 - pLog = fopen("/tmp/speed.log", "w"); - tdb_lsm_write_hook(pDb, do_speed_write_hook2, (void *)pLog); -#endif - - for(i=0; i=nArg ){ - testPrintError("option %s requires an argument\n", aOpt[iSel].zOpt); - return 1; - } - if( aOpt[iSel].isSwitch==1 ){ - nRow = atoi(azArg[i]); - } - if( aOpt[iSel].isSwitch==2 ){ - nSleep = atoi(azArg[i]); - } - if( aOpt[iSel].isSwitch==3 ){ - struct Mode { - const char *zMode; - int doReadTest; - int doWriteTest; - } aMode[] = {{"ro", 1, 0} , {"rw", 1, 1}, {"wo", 0, 1}, {0, 0, 0}}; - int iMode; - rc = testArgSelect(aMode, "option", azArg[i], &iMode); - if( rc ) return rc; - doReadTest = aMode[iMode].doReadTest; - doWriteTest = aMode[iMode].doWriteTest; - } - if( aOpt[iSel].isSwitch==4 ){ - /* The "-out FILE" switch. This option is used to specify a file to - ** write the gnuplot script to. */ - zOut = azArg[i]; - } - }else{ - /* A db name */ - rc = testArgSelect(aOpt, "system", azArg[i], &iSel); - if( rc ) return rc; - sys_mask |= (1< 100000) ? 100000 : nSelStep; - - aTime = malloc(sizeof(int) * ArraySize(aSys) * nRow/nStep); - aWrite = malloc(sizeof(int) * nRow/nStep); - aSelTime = malloc(sizeof(int) * ArraySize(aSys) * nRow/nSelStep); - - /* This loop collects the INSERT speed data. */ - if( doWriteTest ){ - printf("Writing output to file \"%s\".\n", zOut); - - for(j=0; aSys[j].zLibrary; j++){ - FILE *pLog = 0; - TestDb *pDb; /* Database being tested */ - lsm_db *pLsm; - int iDot = 0; - - if( ((1<nData ){ - fprintf(pHook->pOut, "write %s %d %d\n", - (pHook->bLog ? "log" : "db"), (int)pHook->iOff, pHook->nData - ); - pHook->nData = 0; - fflush(pHook->pOut); - } -} - -static void do_insert_write_hook( - void *pCtx, - int bLog, - i64 iOff, - int nData, - int nUs -){ - InsertWriteHook *pHook = (InsertWriteHook *)pCtx; - if( bLog ) return; - - if( nData==0 ){ - flushHook(pHook); - fprintf(pHook->pOut, "sync %s\n", (bLog ? "log" : "db")); - }else if( pHook->nData - && bLog==pHook->bLog - && iOff==(pHook->iOff+pHook->nData) - ){ - pHook->nData += nData; - }else{ - flushHook(pHook); - pHook->bLog = bLog; - pHook->iOff = iOff; - pHook->nData = nData; - } -} - -static int do_replay(int nArg, char **azArg){ - char aBuf[4096]; - FILE *pInput; - FILE *pClose = 0; - const char *zDb; - - lsm_env *pEnv; - lsm_file *pOut; - int rc; - - if( nArg!=2 ){ - testPrintError("Usage: replay WRITELOG FILE\n"); - return 1; - } - - if( strcmp(azArg[0], "-")==0 ){ - pInput = stdin; - }else{ - pClose = pInput = fopen(azArg[0], "r"); - } - zDb = azArg[1]; - pEnv = tdb_lsm_env(); - rc = pEnv->xOpen(pEnv, zDb, 0, &pOut); - if( rc!=LSM_OK ) return rc; - - while( feof(pInput)==0 ){ - char zLine[80]; - fgets(zLine, sizeof(zLine)-1, pInput); - zLine[sizeof(zLine)-1] = '\0'; - - if( 0==memcmp("sync db", zLine, 7) ){ - rc = pEnv->xSync(pOut); - if( rc!=0 ) break; - }else{ - int iOff; - int nData; - int nMatch; - nMatch = sscanf(zLine, "write db %d %d", &iOff, &nData); - if( nMatch==2 ){ - int i; - for(i=0; ixWrite(pOut, iOff+i, aBuf, sizeof(aBuf)); - if( rc!=0 ) break; - } - } - } - } - if( pClose ) fclose(pClose); - pEnv->xClose(pOut); - - return rc; -} - -static int do_insert(int nArg, char **azArg){ - const char *zDb = "lsm"; - TestDb *pDb = 0; - int i; - int rc; - const int nRow = 1 * 1000 * 1000; - - DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 8, 15, 80, 150 }; - Datasource *pData = 0; - - if( nArg>1 ){ - testPrintError("Usage: insert ?DATABASE?\n"); - return 1; - } - if( nArg==1 ){ zDb = azArg[0]; } - - testMallocUninstall(tdb_lsm_env()); - for(i=0; zDb[i] && zDb[i]!='='; i++); - if( zDb[i] ){ - rc = tdb_lsm_open(zDb, "testdb.lsm", 1, &pDb); - }else{ - rc = tdb_open(zDb, 0, 1, &pDb); - } - - if( rc!=0 ){ - testPrintError("Error opening db \"%s\": %d\n", zDb, rc); - }else{ - InsertWriteHook hook; - memset(&hook, 0, sizeof(hook)); - hook.pOut = fopen("writelog.txt", "w"); - - pData = testDatasourceNew(&defn); - tdb_lsm_config_work_hook(pDb, do_insert_work_hook, 0); - tdb_lsm_write_hook(pDb, do_insert_write_hook, (void *)&hook); - - if( rc==0 ){ - for(i=0; i -#include - -static void lsmtest_rusage_report(void){ - struct rusage r; - memset(&r, 0, sizeof(r)); - - getrusage(RUSAGE_SELF, &r); - printf("# getrusage: { ru_maxrss %d ru_oublock %d ru_inblock %d }\n", - (int)r.ru_maxrss, (int)r.ru_oublock, (int)r.ru_inblock - ); -} -#else -static void lsmtest_rusage_report(void){ - /* no-op */ -} -#endif - -int main(int argc, char **argv){ - struct TestFunc { - const char *zName; - int bRusageReport; - int (*xFunc)(int, char **); - } aTest[] = { - {"random", 1, do_random_tests}, - {"writespeed", 1, do_writer_test}, - {"io", 1, st_do_io}, - - {"insert", 1, do_insert}, - {"replay", 1, do_replay}, - - {"speed", 1, do_speed_tests}, - {"speed2", 1, do_speed_test2}, - {"show", 0, st_do_show}, - {"work", 1, st_do_work}, - {"test", 1, do_test}, - - {0, 0} - }; - int rc; /* Return Code */ - int iFunc; /* Index into aTest[] */ - - int nLeakAlloc = 0; /* Allocations leaked by lsm */ - int nLeakByte = 0; /* Bytes leaked by lsm */ - -#ifdef LSM_DEBUG_MEM - FILE *pReport = 0; /* lsm malloc() report file */ - const char *zReport = "malloc.txt generated"; -#else - const char *zReport = "malloc.txt NOT generated"; -#endif - - testMallocInstall(tdb_lsm_env()); - - if( argc<2 ){ - testPrintError("Usage: %s sub-command ?args...?\n", argv[0]); - return -1; - } - - /* Initialize error reporting */ - testErrorInit(argc, argv); - - /* Initialize PRNG system */ - testPrngInit(); - - rc = testArgSelect(aTest, "sub-command", argv[1], &iFunc); - if( rc==0 ){ - rc = aTest[iFunc].xFunc(argc-2, &argv[2]); - } - -#ifdef LSM_DEBUG_MEM - pReport = fopen("malloc.txt", "w"); - testMallocCheck(tdb_lsm_env(), &nLeakAlloc, &nLeakByte, pReport); - fclose(pReport); -#else - testMallocCheck(tdb_lsm_env(), &nLeakAlloc, &nLeakByte, 0); -#endif - - if( nLeakAlloc ){ - testPrintError("Leaked %d bytes in %d allocations (%s)\n", - nLeakByte, nLeakAlloc, zReport - ); - if( rc==0 ) rc = -1; - } - testMallocUninstall(tdb_lsm_env()); - - if( aTest[iFunc].bRusageReport ){ - lsmtest_rusage_report(); - } - return rc; -} diff --git a/ext/lsm1/lsm-test/lsmtest_mem.c b/ext/lsm1/lsm-test/lsmtest_mem.c deleted file mode 100644 index 4c35e849f..000000000 --- a/ext/lsm1/lsm-test/lsmtest_mem.c +++ /dev/null @@ -1,409 +0,0 @@ - -#include -#include -#include - -#define ArraySize(x) ((int)(sizeof(x) / sizeof((x)[0]))) - -#define MIN(x,y) ((x)<(y) ? (x) : (y)) - -typedef unsigned int u32; -typedef unsigned char u8; -typedef long long int i64; -typedef unsigned long long int u64; - -#if defined(__GLIBC__) && defined(LSM_DEBUG_MEM) - extern int backtrace(void**,int); - extern void backtrace_symbols_fd(void*const*,int,int); -# define TM_BACKTRACE 12 -#else -# define backtrace(A,B) 1 -# define backtrace_symbols_fd(A,B,C) -#endif - - -typedef struct TmBlockHdr TmBlockHdr; -typedef struct TmAgg TmAgg; -typedef struct TmGlobal TmGlobal; - -struct TmGlobal { - /* Linked list of all currently outstanding allocations. And a table of - ** all allocations, past and present, indexed by backtrace() info. */ - TmBlockHdr *pFirst; -#ifdef TM_BACKTRACE - TmAgg *aHash[10000]; -#endif - - /* Underlying malloc/realloc/free functions */ - void *(*xMalloc)(int); /* underlying malloc(3) function */ - void *(*xRealloc)(void *, int); /* underlying realloc(3) function */ - void (*xFree)(void *); /* underlying free(3) function */ - - /* Mutex to protect pFirst and aHash */ - void (*xEnterMutex)(TmGlobal*); /* Call this to enter the mutex */ - void (*xLeaveMutex)(TmGlobal*); /* Call this to leave mutex */ - void (*xDelMutex)(TmGlobal*); /* Call this to delete mutex */ - void *pMutex; /* Mutex handle */ - - void *(*xSaveMalloc)(void *, size_t); - void *(*xSaveRealloc)(void *, void *, size_t); - void (*xSaveFree)(void *, void *); - - /* OOM injection scheduling. If nCountdown is greater than zero when a - ** malloc attempt is made, it is decremented. If this means nCountdown - ** transitions from 1 to 0, then the allocation fails. If bPersist is true - ** when this happens, nCountdown is then incremented back to 1 (so that the - ** next attempt fails too). - */ - int nCountdown; - int bPersist; - int bEnable; - void (*xHook)(void *); - void *pHookCtx; -}; - -struct TmBlockHdr { - TmBlockHdr *pNext; - TmBlockHdr *pPrev; - int nByte; -#ifdef TM_BACKTRACE - TmAgg *pAgg; -#endif - u32 iForeGuard; -}; - -#ifdef TM_BACKTRACE -struct TmAgg { - int nAlloc; /* Number of allocations at this path */ - int nByte; /* Total number of bytes allocated */ - int nOutAlloc; /* Number of outstanding allocations */ - int nOutByte; /* Number of outstanding bytes */ - void *aFrame[TM_BACKTRACE]; /* backtrace() output */ - TmAgg *pNext; /* Next object in hash-table collision */ -}; -#endif - -#define FOREGUARD 0x80F5E153 -#define REARGUARD 0xE4676B53 -static const u32 rearguard = REARGUARD; - -#define ROUND8(x) (((x)+7)&~7) - -#define BLOCK_HDR_SIZE (ROUND8( sizeof(TmBlockHdr) )) - -static void lsmtest_oom_error(void){ - static int nErr = 0; - nErr++; -} - -static void tmEnterMutex(TmGlobal *pTm){ - pTm->xEnterMutex(pTm); -} -static void tmLeaveMutex(TmGlobal *pTm){ - pTm->xLeaveMutex(pTm); -} - -static void *tmMalloc(TmGlobal *pTm, int nByte){ - TmBlockHdr *pNew; /* New allocation header block */ - u8 *pUser; /* Return value */ - int nReq; /* Total number of bytes requested */ - - assert( sizeof(rearguard)==4 ); - nReq = BLOCK_HDR_SIZE + nByte + 4; - pNew = (TmBlockHdr *)pTm->xMalloc(nReq); - memset(pNew, 0, sizeof(TmBlockHdr)); - - tmEnterMutex(pTm); - assert( pTm->nCountdown>=0 ); - assert( pTm->bPersist==0 || pTm->bPersist==1 ); - - if( pTm->bEnable && pTm->nCountdown==1 ){ - /* Simulate an OOM error. */ - lsmtest_oom_error(); - pTm->xFree(pNew); - pTm->nCountdown = pTm->bPersist; - if( pTm->xHook ) pTm->xHook(pTm->pHookCtx); - pUser = 0; - }else{ - if( pTm->bEnable && pTm->nCountdown ) pTm->nCountdown--; - - pNew->iForeGuard = FOREGUARD; - pNew->nByte = nByte; - pNew->pNext = pTm->pFirst; - - if( pTm->pFirst ){ - pTm->pFirst->pPrev = pNew; - } - pTm->pFirst = pNew; - - pUser = &((u8 *)pNew)[BLOCK_HDR_SIZE]; - memset(pUser, 0x56, nByte); - memcpy(&pUser[nByte], &rearguard, 4); - -#ifdef TM_BACKTRACE - { - TmAgg *pAgg; - int i; - u32 iHash = 0; - void *aFrame[TM_BACKTRACE]; - memset(aFrame, 0, sizeof(aFrame)); - backtrace(aFrame, TM_BACKTRACE); - - for(i=0; iaHash); - - for(pAgg=pTm->aHash[iHash]; pAgg; pAgg=pAgg->pNext){ - if( memcmp(pAgg->aFrame, aFrame, sizeof(aFrame))==0 ) break; - } - if( !pAgg ){ - pAgg = (TmAgg *)pTm->xMalloc(sizeof(TmAgg)); - memset(pAgg, 0, sizeof(TmAgg)); - memcpy(pAgg->aFrame, aFrame, sizeof(aFrame)); - pAgg->pNext = pTm->aHash[iHash]; - pTm->aHash[iHash] = pAgg; - } - pAgg->nAlloc++; - pAgg->nByte += nByte; - pAgg->nOutAlloc++; - pAgg->nOutByte += nByte; - pNew->pAgg = pAgg; - } -#endif - } - - tmLeaveMutex(pTm); - return pUser; -} - -static void tmFree(TmGlobal *pTm, void *p){ - if( p ){ - TmBlockHdr *pHdr; - u8 *pUser = (u8 *)p; - - tmEnterMutex(pTm); - pHdr = (TmBlockHdr *)(pUser - BLOCK_HDR_SIZE); - assert( pHdr->iForeGuard==FOREGUARD ); - assert( 0==memcmp(&pUser[pHdr->nByte], &rearguard, 4) ); - - if( pHdr->pPrev ){ - assert( pHdr->pPrev->pNext==pHdr ); - pHdr->pPrev->pNext = pHdr->pNext; - }else{ - assert( pHdr==pTm->pFirst ); - pTm->pFirst = pHdr->pNext; - } - if( pHdr->pNext ){ - assert( pHdr->pNext->pPrev==pHdr ); - pHdr->pNext->pPrev = pHdr->pPrev; - } - -#ifdef TM_BACKTRACE - pHdr->pAgg->nOutAlloc--; - pHdr->pAgg->nOutByte -= pHdr->nByte; -#endif - - tmLeaveMutex(pTm); - memset(pUser, 0x58, pHdr->nByte); - memset(pHdr, 0x57, sizeof(TmBlockHdr)); - pTm->xFree(pHdr); - } -} - -static void *tmRealloc(TmGlobal *pTm, void *p, int nByte){ - void *pNew; - - pNew = tmMalloc(pTm, nByte); - if( pNew && p ){ - TmBlockHdr *pHdr; - u8 *pUser = (u8 *)p; - pHdr = (TmBlockHdr *)(pUser - BLOCK_HDR_SIZE); - memcpy(pNew, p, MIN(nByte, pHdr->nByte)); - tmFree(pTm, p); - } - return pNew; -} - -static void tmMallocOom( - TmGlobal *pTm, - int nCountdown, - int bPersist, - void (*xHook)(void *), - void *pHookCtx -){ - assert( nCountdown>=0 ); - assert( bPersist==0 || bPersist==1 ); - pTm->nCountdown = nCountdown; - pTm->bPersist = bPersist; - pTm->xHook = xHook; - pTm->pHookCtx = pHookCtx; - pTm->bEnable = 1; -} - -static void tmMallocOomEnable( - TmGlobal *pTm, - int bEnable -){ - pTm->bEnable = bEnable; -} - -static void tmMallocCheck( - TmGlobal *pTm, - int *pnLeakAlloc, - int *pnLeakByte, - FILE *pFile -){ - TmBlockHdr *pHdr; - int nLeak = 0; - int nByte = 0; - - if( pTm==0 ) return; - - for(pHdr=pTm->pFirst; pHdr; pHdr=pHdr->pNext){ - nLeak++; - nByte += pHdr->nByte; - } - if( pnLeakAlloc ) *pnLeakAlloc = nLeak; - if( pnLeakByte ) *pnLeakByte = nByte; - -#ifdef TM_BACKTRACE - if( pFile ){ - int i; - fprintf(pFile, "LEAKS\n"); - for(i=0; iaHash); i++){ - TmAgg *pAgg; - for(pAgg=pTm->aHash[i]; pAgg; pAgg=pAgg->pNext){ - if( pAgg->nOutAlloc ){ - int j; - fprintf(pFile, "%d %d ", pAgg->nOutByte, pAgg->nOutAlloc); - for(j=0; jaFrame[j]); - } - fprintf(pFile, "\n"); - } - } - } - fprintf(pFile, "\nALLOCATIONS\n"); - for(i=0; iaHash); i++){ - TmAgg *pAgg; - for(pAgg=pTm->aHash[i]; pAgg; pAgg=pAgg->pNext){ - int j; - fprintf(pFile, "%d %d ", pAgg->nByte, pAgg->nAlloc); - for(j=0; jaFrame[j]); - fprintf(pFile, "\n"); - } - } - } -#else - (void)pFile; -#endif -} - - -#include "lsm.h" -#include "stdlib.h" - -typedef struct LsmMutex LsmMutex; -struct LsmMutex { - lsm_env *pEnv; - lsm_mutex *pMutex; -}; - -static void tmLsmMutexEnter(TmGlobal *pTm){ - LsmMutex *p = (LsmMutex *)pTm->pMutex; - p->pEnv->xMutexEnter(p->pMutex); -} -static void tmLsmMutexLeave(TmGlobal *pTm){ - LsmMutex *p = (LsmMutex *)(pTm->pMutex); - p->pEnv->xMutexLeave(p->pMutex); -} -static void tmLsmMutexDel(TmGlobal *pTm){ - LsmMutex *p = (LsmMutex *)pTm->pMutex; - pTm->xFree(p); -} -static void *tmLsmMalloc(int n){ return malloc(n); } -static void tmLsmFree(void *ptr){ free(ptr); } -static void *tmLsmRealloc(void *ptr, int n){ return realloc(ptr, n); } - -static void *tmLsmEnvMalloc(lsm_env *p, size_t n){ - return tmMalloc((TmGlobal *)(p->pMemCtx), n); -} -static void tmLsmEnvFree(lsm_env *p, void *ptr){ - tmFree((TmGlobal *)(p->pMemCtx), ptr); -} -static void *tmLsmEnvRealloc(lsm_env *p, void *ptr, size_t n){ - return tmRealloc((TmGlobal *)(p->pMemCtx), ptr, n); -} - -void testMallocInstall(lsm_env *pEnv){ - TmGlobal *pGlobal; - LsmMutex *pMutex; - assert( pEnv->pMemCtx==0 ); - - /* Allocate and populate a TmGlobal structure. */ - pGlobal = (TmGlobal *)tmLsmMalloc(sizeof(TmGlobal)); - memset(pGlobal, 0, sizeof(TmGlobal)); - pGlobal->xMalloc = tmLsmMalloc; - pGlobal->xRealloc = tmLsmRealloc; - pGlobal->xFree = tmLsmFree; - pMutex = (LsmMutex *)pGlobal->xMalloc(sizeof(LsmMutex)); - pMutex->pEnv = pEnv; - pEnv->xMutexStatic(pEnv, LSM_MUTEX_HEAP, &pMutex->pMutex); - pGlobal->xEnterMutex = tmLsmMutexEnter; - pGlobal->xLeaveMutex = tmLsmMutexLeave; - pGlobal->xDelMutex = tmLsmMutexDel; - pGlobal->pMutex = (void *)pMutex; - - pGlobal->xSaveMalloc = pEnv->xMalloc; - pGlobal->xSaveRealloc = pEnv->xRealloc; - pGlobal->xSaveFree = pEnv->xFree; - - /* Set up pEnv to the use the new TmGlobal */ - pEnv->pMemCtx = (void *)pGlobal; - pEnv->xMalloc = tmLsmEnvMalloc; - pEnv->xRealloc = tmLsmEnvRealloc; - pEnv->xFree = tmLsmEnvFree; -} - -void testMallocUninstall(lsm_env *pEnv){ - TmGlobal *p = (TmGlobal *)pEnv->pMemCtx; - pEnv->pMemCtx = 0; - if( p ){ - pEnv->xMalloc = p->xSaveMalloc; - pEnv->xRealloc = p->xSaveRealloc; - pEnv->xFree = p->xSaveFree; - p->xDelMutex(p); - tmLsmFree(p); - } -} - -void testMallocCheck( - lsm_env *pEnv, - int *pnLeakAlloc, - int *pnLeakByte, - FILE *pFile -){ - if( pEnv->pMemCtx==0 ){ - *pnLeakAlloc = 0; - *pnLeakByte = 0; - }else{ - tmMallocCheck((TmGlobal *)(pEnv->pMemCtx), pnLeakAlloc, pnLeakByte, pFile); - } -} - -void testMallocOom( - lsm_env *pEnv, - int nCountdown, - int bPersist, - void (*xHook)(void *), - void *pHookCtx -){ - TmGlobal *pTm = (TmGlobal *)(pEnv->pMemCtx); - tmMallocOom(pTm, nCountdown, bPersist, xHook, pHookCtx); -} - -void testMallocOomEnable(lsm_env *pEnv, int bEnable){ - TmGlobal *pTm = (TmGlobal *)(pEnv->pMemCtx); - tmMallocOomEnable(pTm, bEnable); -} diff --git a/ext/lsm1/lsm-test/lsmtest_tdb.c b/ext/lsm1/lsm-test/lsmtest_tdb.c deleted file mode 100644 index 8f63f64ac..000000000 --- a/ext/lsm1/lsm-test/lsmtest_tdb.c +++ /dev/null @@ -1,846 +0,0 @@ - -/* -** This program attempts to test the correctness of some facets of the -** LSM database library. Specifically, that the contents of the database -** are maintained correctly during a series of inserts and deletes. -*/ - - -#include "lsmtest_tdb.h" -#include "lsm.h" - -#include "lsmtest.h" - -#include -#include -#include -#ifndef _WIN32 -# include -#endif -#include - - -typedef struct SqlDb SqlDb; - -static int error_transaction_function(TestDb *p, int iLevel){ - unused_parameter(p); - unused_parameter(iLevel); - return -1; -} - - -/************************************************************************* -** Begin wrapper for LevelDB. -*/ -#ifdef HAVE_LEVELDB - -#include - -typedef struct LevelDb LevelDb; -struct LevelDb { - TestDb base; - leveldb_t *db; - leveldb_options_t *pOpt; - leveldb_writeoptions_t *pWriteOpt; - leveldb_readoptions_t *pReadOpt; - - char *pVal; -}; - -static int test_leveldb_close(TestDb *pTestDb){ - LevelDb *pDb = (LevelDb *)pTestDb; - - leveldb_close(pDb->db); - leveldb_writeoptions_destroy(pDb->pWriteOpt); - leveldb_readoptions_destroy(pDb->pReadOpt); - leveldb_options_destroy(pDb->pOpt); - free(pDb->pVal); - free(pDb); - - return 0; -} - -static int test_leveldb_write( - TestDb *pTestDb, - void *pKey, - int nKey, - void *pVal, - int nVal -){ - LevelDb *pDb = (LevelDb *)pTestDb; - char *zErr = 0; - leveldb_put(pDb->db, pDb->pWriteOpt, pKey, nKey, pVal, nVal, &zErr); - return (zErr!=0); -} - -static int test_leveldb_delete(TestDb *pTestDb, void *pKey, int nKey){ - LevelDb *pDb = (LevelDb *)pTestDb; - char *zErr = 0; - leveldb_delete(pDb->db, pDb->pWriteOpt, pKey, nKey, &zErr); - return (zErr!=0); -} - -static int test_leveldb_fetch( - TestDb *pTestDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - LevelDb *pDb = (LevelDb *)pTestDb; - char *zErr = 0; - size_t nVal = 0; - - if( pKey==0 ) return 0; - free(pDb->pVal); - pDb->pVal = leveldb_get(pDb->db, pDb->pReadOpt, pKey, nKey, &nVal, &zErr); - *ppVal = (void *)(pDb->pVal); - if( pDb->pVal==0 ){ - *pnVal = -1; - }else{ - *pnVal = (int)nVal; - } - - return (zErr!=0); -} - -static int test_leveldb_scan( - TestDb *pTestDb, - void *pCtx, - int bReverse, - void *pKey1, int nKey1, /* Start of search */ - void *pKey2, int nKey2, /* End of search */ - void (*xCallback)(void *, void *, int , void *, int) -){ - LevelDb *pDb = (LevelDb *)pTestDb; - leveldb_iterator_t *iter; - - iter = leveldb_create_iterator(pDb->db, pDb->pReadOpt); - - if( bReverse==0 ){ - if( pKey1 ){ - leveldb_iter_seek(iter, pKey1, nKey1); - }else{ - leveldb_iter_seek_to_first(iter); - } - }else{ - if( pKey2 ){ - leveldb_iter_seek(iter, pKey2, nKey2); - - if( leveldb_iter_valid(iter)==0 ){ - leveldb_iter_seek_to_last(iter); - }else{ - const char *k; size_t n; - int res; - k = leveldb_iter_key(iter, &n); - res = memcmp(k, pKey2, MIN(n, nKey2)); - if( res==0 ) res = n - nKey2; - assert( res>=0 ); - if( res>0 ){ - leveldb_iter_prev(iter); - } - } - }else{ - leveldb_iter_seek_to_last(iter); - } - } - - - while( leveldb_iter_valid(iter) ){ - const char *k; size_t n; - const char *v; size_t n2; - int res; - - k = leveldb_iter_key(iter, &n); - if( bReverse==0 && pKey2 ){ - res = memcmp(k, pKey2, MIN(n, nKey2)); - if( res==0 ) res = n - nKey2; - if( res>0 ) break; - } - if( bReverse!=0 && pKey1 ){ - res = memcmp(k, pKey1, MIN(n, nKey1)); - if( res==0 ) res = n - nKey1; - if( res<0 ) break; - } - - v = leveldb_iter_value(iter, &n2); - - xCallback(pCtx, (void *)k, n, (void *)v, n2); - - if( bReverse==0 ){ - leveldb_iter_next(iter); - }else{ - leveldb_iter_prev(iter); - } - } - - leveldb_iter_destroy(iter); - return 0; -} - -static int test_leveldb_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - static const DatabaseMethods LeveldbMethods = { - test_leveldb_close, - test_leveldb_write, - test_leveldb_delete, - 0, - test_leveldb_fetch, - test_leveldb_scan, - error_transaction_function, - error_transaction_function, - error_transaction_function - }; - - LevelDb *pLevelDb; - char *zErr = 0; - - if( bClear ){ - char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename); - system(zCmd); - sqlite3_free(zCmd); - } - - pLevelDb = (LevelDb *)malloc(sizeof(LevelDb)); - memset(pLevelDb, 0, sizeof(LevelDb)); - - pLevelDb->pOpt = leveldb_options_create(); - leveldb_options_set_create_if_missing(pLevelDb->pOpt, 1); - pLevelDb->pWriteOpt = leveldb_writeoptions_create(); - pLevelDb->pReadOpt = leveldb_readoptions_create(); - - pLevelDb->db = leveldb_open(pLevelDb->pOpt, zFilename, &zErr); - - if( zErr ){ - test_leveldb_close((TestDb *)pLevelDb); - *ppDb = 0; - return 1; - } - - *ppDb = (TestDb *)pLevelDb; - pLevelDb->base.pMethods = &LeveldbMethods; - return 0; -} -#endif /* HAVE_LEVELDB */ -/* -** End wrapper for LevelDB. -*************************************************************************/ - -#ifdef HAVE_KYOTOCABINET -static int kc_close(TestDb *pTestDb){ - return test_kc_close(pTestDb); -} - -static int kc_write( - TestDb *pTestDb, - void *pKey, - int nKey, - void *pVal, - int nVal -){ - return test_kc_write(pTestDb, pKey, nKey, pVal, nVal); -} - -static int kc_delete(TestDb *pTestDb, void *pKey, int nKey){ - return test_kc_delete(pTestDb, pKey, nKey); -} - -static int kc_delete_range( - TestDb *pTestDb, - void *pKey1, int nKey1, - void *pKey2, int nKey2 -){ - return test_kc_delete_range(pTestDb, pKey1, nKey1, pKey2, nKey2); -} - -static int kc_fetch( - TestDb *pTestDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - if( pKey==0 ) return LSM_OK; - return test_kc_fetch(pTestDb, pKey, nKey, ppVal, pnVal); -} - -static int kc_scan( - TestDb *pTestDb, - void *pCtx, - int bReverse, - void *pFirst, int nFirst, - void *pLast, int nLast, - void (*xCallback)(void *, void *, int , void *, int) -){ - return test_kc_scan( - pTestDb, pCtx, bReverse, pFirst, nFirst, pLast, nLast, xCallback - ); -} - -static int kc_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - static const DatabaseMethods KcdbMethods = { - kc_close, - kc_write, - kc_delete, - kc_delete_range, - kc_fetch, - kc_scan, - error_transaction_function, - error_transaction_function, - error_transaction_function - }; - - int rc; - TestDb *pTestDb = 0; - - rc = test_kc_open(zFilename, bClear, &pTestDb); - if( rc!=0 ){ - *ppDb = 0; - return rc; - } - pTestDb->pMethods = &KcdbMethods; - *ppDb = pTestDb; - return 0; -} -#endif /* HAVE_KYOTOCABINET */ -/* -** End wrapper for Kyoto cabinet. -*************************************************************************/ - -#ifdef HAVE_MDB -static int mdb_close(TestDb *pTestDb){ - return test_mdb_close(pTestDb); -} - -static int mdb_write( - TestDb *pTestDb, - void *pKey, - int nKey, - void *pVal, - int nVal -){ - return test_mdb_write(pTestDb, pKey, nKey, pVal, nVal); -} - -static int mdb_delete(TestDb *pTestDb, void *pKey, int nKey){ - return test_mdb_delete(pTestDb, pKey, nKey); -} - -static int mdb_fetch( - TestDb *pTestDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - if( pKey==0 ) return LSM_OK; - return test_mdb_fetch(pTestDb, pKey, nKey, ppVal, pnVal); -} - -static int mdb_scan( - TestDb *pTestDb, - void *pCtx, - int bReverse, - void *pFirst, int nFirst, - void *pLast, int nLast, - void (*xCallback)(void *, void *, int , void *, int) -){ - return test_mdb_scan( - pTestDb, pCtx, bReverse, pFirst, nFirst, pLast, nLast, xCallback - ); -} - -static int mdb_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - static const DatabaseMethods KcdbMethods = { - mdb_close, - mdb_write, - mdb_delete, - 0, - mdb_fetch, - mdb_scan, - error_transaction_function, - error_transaction_function, - error_transaction_function - }; - - int rc; - TestDb *pTestDb = 0; - - rc = test_mdb_open(zSpec, zFilename, bClear, &pTestDb); - if( rc!=0 ){ - *ppDb = 0; - return rc; - } - pTestDb->pMethods = &KcdbMethods; - *ppDb = pTestDb; - return 0; -} -#endif /* HAVE_MDB */ - -/************************************************************************* -** Begin wrapper for SQLite. -*/ - -/* -** nOpenTrans: -** The number of open nested transactions, in the same sense as used -** by the tdb_begin/commit/rollback and SQLite 4 KV interfaces. If this -** value is 0, there are no transactions open at all. If it is 1, then -** there is a read transaction. If it is 2 or greater, then there are -** (nOpenTrans-1) nested write transactions open. -*/ -struct SqlDb { - TestDb base; - sqlite3 *db; - sqlite3_stmt *pInsert; - sqlite3_stmt *pDelete; - sqlite3_stmt *pDeleteRange; - sqlite3_stmt *pFetch; - sqlite3_stmt *apScan[8]; - - int nOpenTrans; - - /* Used by sql_fetch() to allocate space for results */ - int nAlloc; - u8 *aAlloc; -}; - -static int sql_close(TestDb *pTestDb){ - SqlDb *pDb = (SqlDb *)pTestDb; - sqlite3_finalize(pDb->pInsert); - sqlite3_finalize(pDb->pDelete); - sqlite3_finalize(pDb->pDeleteRange); - sqlite3_finalize(pDb->pFetch); - sqlite3_finalize(pDb->apScan[0]); - sqlite3_finalize(pDb->apScan[1]); - sqlite3_finalize(pDb->apScan[2]); - sqlite3_finalize(pDb->apScan[3]); - sqlite3_finalize(pDb->apScan[4]); - sqlite3_finalize(pDb->apScan[5]); - sqlite3_finalize(pDb->apScan[6]); - sqlite3_finalize(pDb->apScan[7]); - sqlite3_close(pDb->db); - free((char *)pDb->aAlloc); - free((char *)pDb); - return SQLITE_OK; -} - -static int sql_write( - TestDb *pTestDb, - void *pKey, - int nKey, - void *pVal, - int nVal -){ - SqlDb *pDb = (SqlDb *)pTestDb; - sqlite3_bind_blob(pDb->pInsert, 1, pKey, nKey, SQLITE_STATIC); - sqlite3_bind_blob(pDb->pInsert, 2, pVal, nVal, SQLITE_STATIC); - sqlite3_step(pDb->pInsert); - return sqlite3_reset(pDb->pInsert); -} - -static int sql_delete(TestDb *pTestDb, void *pKey, int nKey){ - SqlDb *pDb = (SqlDb *)pTestDb; - sqlite3_bind_blob(pDb->pDelete, 1, pKey, nKey, SQLITE_STATIC); - sqlite3_step(pDb->pDelete); - return sqlite3_reset(pDb->pDelete); -} - -static int sql_delete_range( - TestDb *pTestDb, - void *pKey1, int nKey1, - void *pKey2, int nKey2 -){ - SqlDb *pDb = (SqlDb *)pTestDb; - sqlite3_bind_blob(pDb->pDeleteRange, 1, pKey1, nKey1, SQLITE_STATIC); - sqlite3_bind_blob(pDb->pDeleteRange, 2, pKey2, nKey2, SQLITE_STATIC); - sqlite3_step(pDb->pDeleteRange); - return sqlite3_reset(pDb->pDeleteRange); -} - -static int sql_fetch( - TestDb *pTestDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - SqlDb *pDb = (SqlDb *)pTestDb; - int rc; - - sqlite3_reset(pDb->pFetch); - if( pKey==0 ){ - assert( ppVal==0 ); - assert( pnVal==0 ); - return LSM_OK; - } - - sqlite3_bind_blob(pDb->pFetch, 1, pKey, nKey, SQLITE_STATIC); - rc = sqlite3_step(pDb->pFetch); - if( rc==SQLITE_ROW ){ - int nVal = sqlite3_column_bytes(pDb->pFetch, 0); - u8 *aVal = (void *)sqlite3_column_blob(pDb->pFetch, 0); - - if( nVal>pDb->nAlloc ){ - free(pDb->aAlloc); - pDb->aAlloc = (u8 *)malloc(nVal*2); - pDb->nAlloc = nVal*2; - } - memcpy(pDb->aAlloc, aVal, nVal); - *pnVal = nVal; - *ppVal = (void *)pDb->aAlloc; - }else{ - *pnVal = -1; - *ppVal = 0; - } - - rc = sqlite3_reset(pDb->pFetch); - return rc; -} - -static int sql_scan( - TestDb *pTestDb, - void *pCtx, - int bReverse, - void *pFirst, int nFirst, - void *pLast, int nLast, - void (*xCallback)(void *, void *, int , void *, int) -){ - SqlDb *pDb = (SqlDb *)pTestDb; - sqlite3_stmt *pScan; - - assert( bReverse==1 || bReverse==0 ); - pScan = pDb->apScan[(pFirst==0) + (pLast==0)*2 + bReverse*4]; - - if( pFirst ) sqlite3_bind_blob(pScan, 1, pFirst, nFirst, SQLITE_STATIC); - if( pLast ) sqlite3_bind_blob(pScan, 2, pLast, nLast, SQLITE_STATIC); - - while( SQLITE_ROW==sqlite3_step(pScan) ){ - void *pKey; int nKey; - void *pVal; int nVal; - - nKey = sqlite3_column_bytes(pScan, 0); - pKey = (void *)sqlite3_column_blob(pScan, 0); - nVal = sqlite3_column_bytes(pScan, 1); - pVal = (void *)sqlite3_column_blob(pScan, 1); - - xCallback(pCtx, pKey, nKey, pVal, nVal); - } - return sqlite3_reset(pScan); -} - -static int sql_begin(TestDb *pTestDb, int iLevel){ - int i; - SqlDb *pDb = (SqlDb *)pTestDb; - - /* iLevel==0 is a no-op */ - if( iLevel==0 ) return 0; - - /* If there are no transactions at all open, open a read transaction. */ - if( pDb->nOpenTrans==0 ){ - int rc = sqlite3_exec(pDb->db, - "BEGIN; SELECT * FROM sqlite_schema LIMIT 1;" , 0, 0, 0 - ); - if( rc!=0 ) return rc; - pDb->nOpenTrans = 1; - } - - /* Open any required write transactions */ - for(i=pDb->nOpenTrans; idb, zSql, 0, 0, 0); - sqlite3_free(zSql); - if( rc!=SQLITE_OK ) return rc; - } - - pDb->nOpenTrans = iLevel; - return 0; -} - -static int sql_commit(TestDb *pTestDb, int iLevel){ - SqlDb *pDb = (SqlDb *)pTestDb; - assert( iLevel>=0 ); - - /* Close the read transaction if requested. */ - if( pDb->nOpenTrans>=1 && iLevel==0 ){ - int rc = sqlite3_exec(pDb->db, "COMMIT", 0, 0, 0); - if( rc!=0 ) return rc; - pDb->nOpenTrans = 0; - } - - /* Close write transactions as required */ - if( pDb->nOpenTrans>iLevel ){ - char *zSql = sqlite3_mprintf("RELEASE x%d", iLevel); - int rc = sqlite3_exec(pDb->db, zSql, 0, 0, 0); - sqlite3_free(zSql); - if( rc!=0 ) return rc; - } - - pDb->nOpenTrans = iLevel; - return 0; -} - -static int sql_rollback(TestDb *pTestDb, int iLevel){ - SqlDb *pDb = (SqlDb *)pTestDb; - assert( iLevel>=0 ); - - if( pDb->nOpenTrans>=1 && iLevel==0 ){ - /* Close the read transaction if requested. */ - int rc = sqlite3_exec(pDb->db, "ROLLBACK", 0, 0, 0); - if( rc!=0 ) return rc; - }else if( pDb->nOpenTrans>1 && iLevel==1 ){ - /* Or, rollback and close the top-level write transaction */ - int rc = sqlite3_exec(pDb->db, "ROLLBACK TO x1; RELEASE x1;", 0, 0, 0); - if( rc!=0 ) return rc; - }else{ - /* Or, just roll back some nested transactions */ - char *zSql = sqlite3_mprintf("ROLLBACK TO x%d", iLevel-1); - int rc = sqlite3_exec(pDb->db, zSql, 0, 0, 0); - sqlite3_free(zSql); - if( rc!=0 ) return rc; - } - - pDb->nOpenTrans = iLevel; - return 0; -} - -static int sql_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - static const DatabaseMethods SqlMethods = { - sql_close, - sql_write, - sql_delete, - sql_delete_range, - sql_fetch, - sql_scan, - sql_begin, - sql_commit, - sql_rollback - }; - const char *zCreate = "CREATE TABLE IF NOT EXISTS t1(k PRIMARY KEY, v)"; - const char *zInsert = "REPLACE INTO t1 VALUES(?, ?)"; - const char *zDelete = "DELETE FROM t1 WHERE k = ?"; - const char *zRange = "DELETE FROM t1 WHERE k>? AND k= ?1 ORDER BY k"; - const char *zScan3 = "SELECT * FROM t1 ORDER BY k"; - - const char *zScan4 = - "SELECT * FROM t1 WHERE k BETWEEN ?1 AND ?2 ORDER BY k DESC"; - const char *zScan5 = "SELECT * FROM t1 WHERE k <= ?2 ORDER BY k DESC"; - const char *zScan6 = "SELECT * FROM t1 WHERE k >= ?1 ORDER BY k DESC"; - const char *zScan7 = "SELECT * FROM t1 ORDER BY k DESC"; - - int rc; - SqlDb *pDb; - char *zPragma; - - if( bClear && zFilename && zFilename[0] ){ - unlink(zFilename); - } - - pDb = (SqlDb *)malloc(sizeof(SqlDb)); - memset(pDb, 0, sizeof(SqlDb)); - pDb->base.pMethods = &SqlMethods; - - if( 0!=(rc = sqlite3_open(zFilename, &pDb->db)) - || 0!=(rc = sqlite3_exec(pDb->db, zCreate, 0, 0, 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zInsert, -1, &pDb->pInsert, 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zDelete, -1, &pDb->pDelete, 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zRange, -1, &pDb->pDeleteRange, 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zFetch, -1, &pDb->pFetch, 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan0, -1, &pDb->apScan[0], 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan1, -1, &pDb->apScan[1], 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan2, -1, &pDb->apScan[2], 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan3, -1, &pDb->apScan[3], 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan4, -1, &pDb->apScan[4], 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan5, -1, &pDb->apScan[5], 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan6, -1, &pDb->apScan[6], 0)) - || 0!=(rc = sqlite3_prepare_v2(pDb->db, zScan7, -1, &pDb->apScan[7], 0)) - ){ - *ppDb = 0; - sql_close((TestDb *)pDb); - return rc; - } - - zPragma = sqlite3_mprintf("PRAGMA page_size=%d", TESTDB_DEFAULT_PAGE_SIZE); - sqlite3_exec(pDb->db, zPragma, 0, 0, 0); - sqlite3_free(zPragma); - zPragma = sqlite3_mprintf("PRAGMA cache_size=%d", TESTDB_DEFAULT_CACHE_SIZE); - sqlite3_exec(pDb->db, zPragma, 0, 0, 0); - sqlite3_free(zPragma); - - /* sqlite3_exec(pDb->db, "PRAGMA locking_mode=EXCLUSIVE", 0, 0, 0); */ - sqlite3_exec(pDb->db, "PRAGMA synchronous=OFF", 0, 0, 0); - sqlite3_exec(pDb->db, "PRAGMA journal_mode=WAL", 0, 0, 0); - sqlite3_exec(pDb->db, "PRAGMA wal_autocheckpoint=4096", 0, 0, 0); - if( zSpec ){ - rc = sqlite3_exec(pDb->db, zSpec, 0, 0, 0); - if( rc!=SQLITE_OK ){ - sql_close((TestDb *)pDb); - return rc; - } - } - - *ppDb = (TestDb *)pDb; - return 0; -} -/* -** End wrapper for SQLite. -*************************************************************************/ - -/************************************************************************* -** Begin exported functions. -*/ -static struct Lib { - const char *zName; - const char *zDefaultDb; - int (*xOpen)(const char *, const char *zFilename, int bClear, TestDb **ppDb); -} aLib[] = { - { "sqlite3", "testdb.sqlite", sql_open }, - { "lsm_small", "testdb.lsm_small", test_lsm_small_open }, - { "lsm_lomem", "testdb.lsm_lomem", test_lsm_lomem_open }, - { "lsm_lomem2", "testdb.lsm_lomem2", test_lsm_lomem2_open }, -#ifdef HAVE_ZLIB - { "lsm_zip", "testdb.lsm_zip", test_lsm_zip_open }, -#endif - { "lsm", "testdb.lsm", test_lsm_open }, -#ifdef LSM_MUTEX_PTHREADS - { "lsm_mt2", "testdb.lsm_mt2", test_lsm_mt2 }, - { "lsm_mt3", "testdb.lsm_mt3", test_lsm_mt3 }, -#endif -#ifdef HAVE_LEVELDB - { "leveldb", "testdb.leveldb", test_leveldb_open }, -#endif -#ifdef HAVE_KYOTOCABINET - { "kyotocabinet", "testdb.kc", kc_open }, -#endif -#ifdef HAVE_MDB - { "mdb", "./testdb.mdb", mdb_open } -#endif -}; - -const char *tdb_system_name(int i){ - if( i<0 || i>=ArraySize(aLib) ) return 0; - return aLib[i].zName; -} - -const char *tdb_default_db(const char *zSys){ - int i; - for(i=0; izLibrary = aLib[i].zName; - } - break; - } - } - - if( rc ){ - /* Failed to find the requested database library. Return an error. */ - *ppDb = 0; - } - return rc; -} - -int tdb_close(TestDb *pDb){ - if( pDb ){ - return pDb->pMethods->xClose(pDb); - } - return 0; -} - -int tdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){ - return pDb->pMethods->xWrite(pDb, pKey, nKey, pVal, nVal); -} - -int tdb_delete(TestDb *pDb, void *pKey, int nKey){ - return pDb->pMethods->xDelete(pDb, pKey, nKey); -} - -int tdb_delete_range( - TestDb *pDb, void *pKey1, int nKey1, void *pKey2, int nKey2 -){ - return pDb->pMethods->xDeleteRange(pDb, pKey1, nKey1, pKey2, nKey2); -} - -int tdb_fetch(TestDb *pDb, void *pKey, int nKey, void **ppVal, int *pnVal){ - return pDb->pMethods->xFetch(pDb, pKey, nKey, ppVal, pnVal); -} - -int tdb_scan( - TestDb *pDb, /* Database handle */ - void *pCtx, /* Context pointer to pass to xCallback */ - int bReverse, /* True to scan in reverse order */ - void *pKey1, int nKey1, /* Start of search */ - void *pKey2, int nKey2, /* End of search */ - void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) -){ - return pDb->pMethods->xScan( - pDb, pCtx, bReverse, pKey1, nKey1, pKey2, nKey2, xCallback - ); -} - -int tdb_begin(TestDb *pDb, int iLevel){ - return pDb->pMethods->xBegin(pDb, iLevel); -} -int tdb_commit(TestDb *pDb, int iLevel){ - return pDb->pMethods->xCommit(pDb, iLevel); -} -int tdb_rollback(TestDb *pDb, int iLevel){ - return pDb->pMethods->xRollback(pDb, iLevel); -} - -int tdb_transaction_support(TestDb *pDb){ - return (pDb->pMethods->xBegin != error_transaction_function); -} - -const char *tdb_library_name(TestDb *pDb){ - return pDb->zLibrary; -} - -/* -** End exported functions. -*************************************************************************/ diff --git a/ext/lsm1/lsm-test/lsmtest_tdb.h b/ext/lsm1/lsm-test/lsmtest_tdb.h deleted file mode 100644 index c55b6e2f8..000000000 --- a/ext/lsm1/lsm-test/lsmtest_tdb.h +++ /dev/null @@ -1,174 +0,0 @@ - -/* -** This file is the interface to a very simple database library used for -** testing. The interface is similar to that of the LSM. The main virtue -** of this library is that the same API may be used to access a key-value -** store implemented by LSM, SQLite or another database system. Which -** makes it easy to use for correctness and performance tests. -*/ - -#ifndef __WRAPPER_H_ -#define __WRAPPER_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "lsm.h" - -typedef struct TestDb TestDb; - -/* -** Open a new database connection. The first argument is the name of the -** database library to use. e.g. something like: -** -** "sqlite3" -** "lsm" -** -** See function tdb_system_name() for a list of available database systems. -** -** The second argument is the name of the database to open (e.g. a filename). -** -** If the third parameter is non-zero, then any existing database by the -** name of zDb is removed before opening a new one. If it is zero, then an -** existing database may be opened. -*/ -int tdb_open(const char *zLibrary, const char *zDb, int bClear, TestDb **ppDb); - -/* -** Close a database handle. -*/ -int tdb_close(TestDb *pDb); - -/* -** Write a new key/value into the database. -*/ -int tdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal); - -/* -** Delete a key from the database. -*/ -int tdb_delete(TestDb *pDb, void *pKey, int nKey); - -/* -** Delete a range of keys from the database. -*/ -int tdb_delete_range(TestDb *, void *pKey1, int nKey1, void *pKey2, int nKey2); - -/* -** Query the database for key (pKey/nKey). If no entry is found, set *ppVal -** to 0 and *pnVal to -1 before returning. Otherwise, set *ppVal and *pnVal -** to a pointer to and size of the value associated with (pKey/nKey). -*/ -int tdb_fetch(TestDb *pDb, void *pKey, int nKey, void **ppVal, int *pnVal); - -/* -** Open and close nested transactions. Currently, these functions only -** work for SQLite3 and LSM systems. Use the tdb_transaction_support() -** function to determine if a given TestDb handle supports these methods. -** -** These functions and the iLevel parameter follow the same conventions as -** the SQLite 4 transaction interface. Note that this is slightly different -** from the way LSM does things. As follows: -** -** tdb_begin(): -** A successful call to tdb_begin() with (iLevel>1) guarantees that -** there are at least (iLevel-1) write transactions open. If iLevel==1, -** then it guarantees that at least a read-transaction is open. Calling -** tdb_begin() with iLevel==0 is a no-op. -** -** tdb_commit(): -** A successful call to tdb_commit() with (iLevel>1) guarantees that -** there are at most (iLevel-1) write transactions open. If iLevel==1, -** then it guarantees that there are no write transactions open (although -** a read-transaction may remain open). Calling tdb_commit() with -** iLevel==0 ensures that all transactions, read or write, have been -** closed and committed. -** -** tdb_rollback(): -** This call is similar to tdb_commit(), except that instead of committing -** transactions, it reverts them. For example, calling tdb_rollback() with -** iLevel==2 ensures that there is at most one write transaction open, and -** restores the database to the state that it was in when that transaction -** was opened. -** -** In other words, tdb_commit() just closes transactions - tdb_rollback() -** closes transactions and then restores the database to the state it -** was in before those transactions were even opened. -*/ -int tdb_begin(TestDb *pDb, int iLevel); -int tdb_commit(TestDb *pDb, int iLevel); -int tdb_rollback(TestDb *pDb, int iLevel); - -/* -** Return true if transactions are supported, or false otherwise. -*/ -int tdb_transaction_support(TestDb *pDb); - -/* -** Return the name of the database library (as passed to tdb_open()) used -** by the handled passed as the first argument. -*/ -const char *tdb_library_name(TestDb *pDb); - -/* -** Scan a range of database keys. Invoke the callback function for each -** key visited. -*/ -int tdb_scan( - TestDb *pDb, /* Database handle */ - void *pCtx, /* Context pointer to pass to xCallback */ - int bReverse, /* True to scan in reverse order */ - void *pKey1, int nKey1, /* Start of search */ - void *pKey2, int nKey2, /* End of search */ - void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) -); - -const char *tdb_system_name(int i); -const char *tdb_default_db(const char *zSys); - -int tdb_lsm_open(const char *zCfg, const char *zDb, int bClear, TestDb **ppDb); - -/* -** If the TestDb handle passed as an argument is a wrapper around an LSM -** database, return the LSM handle. Otherwise, if the argument is some other -** database system, return NULL. -*/ -lsm_db *tdb_lsm(TestDb *pDb); - -/* -** Return true if the db passed as an argument is a multi-threaded LSM -** connection. -*/ -int tdb_lsm_multithread(TestDb *pDb); - -/* -** Return a pointer to the lsm_env object used by all lsm database -** connections initialized as a copy of the object returned by -** lsm_default_env(). It may be modified (e.g. to override functions) -** if the caller can guarantee that it is not already in use. -*/ -lsm_env *tdb_lsm_env(void); - -/* -** The following functions only work with LSM database handles. It is -** illegal to call them with any other type of database handle specified -** as an argument. -*/ -void tdb_lsm_enable_log(TestDb *pDb, int bEnable); -void tdb_lsm_application_crash(TestDb *pDb); -void tdb_lsm_prepare_system_crash(TestDb *pDb); -void tdb_lsm_system_crash(TestDb *pDb); -void tdb_lsm_prepare_sync_crash(TestDb *pDb, int iSync); - - -void tdb_lsm_safety(TestDb *pDb, int eMode); -void tdb_lsm_config_work_hook(TestDb *pDb, void (*)(lsm_db *, void *), void *); -void tdb_lsm_write_hook(TestDb *, void(*)(void*,int,lsm_i64,int,int), void*); -int tdb_lsm_config_str(TestDb *pDb, const char *zStr); - -#ifdef __cplusplus -} /* End of the 'extern "C"' block */ -#endif - -#endif diff --git a/ext/lsm1/lsm-test/lsmtest_tdb2.cc b/ext/lsm1/lsm-test/lsmtest_tdb2.cc deleted file mode 100644 index 86ebb4958..000000000 --- a/ext/lsm1/lsm-test/lsmtest_tdb2.cc +++ /dev/null @@ -1,369 +0,0 @@ - - -#include "lsmtest.h" -#include - -#ifdef HAVE_KYOTOCABINET -#include "kcpolydb.h" -extern "C" { - struct KcDb { - TestDb base; - kyotocabinet::TreeDB* db; - char *pVal; - }; -} - -int test_kc_open(const char *zFilename, int bClear, TestDb **ppDb){ - KcDb *pKcDb; - int ok; - int rc = 0; - - if( bClear ){ - char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename); - system(zCmd); - sqlite3_free(zCmd); - } - - pKcDb = (KcDb *)malloc(sizeof(KcDb)); - memset(pKcDb, 0, sizeof(KcDb)); - - - pKcDb->db = new kyotocabinet::TreeDB(); - pKcDb->db->tune_page(TESTDB_DEFAULT_PAGE_SIZE); - pKcDb->db->tune_page_cache( - TESTDB_DEFAULT_PAGE_SIZE * TESTDB_DEFAULT_CACHE_SIZE - ); - ok = pKcDb->db->open(zFilename, - kyotocabinet::PolyDB::OWRITER | kyotocabinet::PolyDB::OCREATE - ); - if( ok==0 ){ - free(pKcDb); - pKcDb = 0; - rc = 1; - } - - *ppDb = (TestDb *)pKcDb; - return rc; -} - -int test_kc_close(TestDb *pDb){ - KcDb *pKcDb = (KcDb *)pDb; - if( pKcDb->pVal ){ - delete [] pKcDb->pVal; - } - pKcDb->db->close(); - delete pKcDb->db; - free(pKcDb); - return 0; -} - -int test_kc_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){ - KcDb *pKcDb = (KcDb *)pDb; - int ok; - - ok = pKcDb->db->set((const char *)pKey, nKey, (const char *)pVal, nVal); - return (ok ? 0 : 1); -} - -int test_kc_delete(TestDb *pDb, void *pKey, int nKey){ - KcDb *pKcDb = (KcDb *)pDb; - int ok; - - ok = pKcDb->db->remove((const char *)pKey, nKey); - return (ok ? 0 : 1); -} - -int test_kc_delete_range( - TestDb *pDb, - void *pKey1, int nKey1, - void *pKey2, int nKey2 -){ - int res; - KcDb *pKcDb = (KcDb *)pDb; - kyotocabinet::DB::Cursor* pCur = pKcDb->db->cursor(); - - if( pKey1 ){ - res = pCur->jump((const char *)pKey1, nKey1); - }else{ - res = pCur->jump(); - } - - while( 1 ){ - const char *pKey; size_t nKey; - const char *pVal; size_t nVal; - - pKey = pCur->get(&nKey, &pVal, &nVal); - if( pKey==0 ) break; - -#ifndef NDEBUG - if( pKey1 ){ - res = memcmp(pKey, pKey1, MIN((size_t)nKey1, nKey)); - assert( res>0 || (res==0 && nKey>nKey1) ); - } -#endif - - if( pKey2 ){ - res = memcmp(pKey, pKey2, MIN((size_t)nKey2, nKey)); - if( res>0 || (res==0 && (size_t)nKey2remove(); - delete [] pKey; - } - - delete pCur; - return 0; -} - -int test_kc_fetch( - TestDb *pDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - KcDb *pKcDb = (KcDb *)pDb; - size_t nVal; - - if( pKcDb->pVal ){ - delete [] pKcDb->pVal; - pKcDb->pVal = 0; - } - - pKcDb->pVal = pKcDb->db->get((const char *)pKey, nKey, &nVal); - if( pKcDb->pVal ){ - *ppVal = pKcDb->pVal; - *pnVal = nVal; - }else{ - *ppVal = 0; - *pnVal = -1; - } - - return 0; -} - -int test_kc_scan( - TestDb *pDb, /* Database handle */ - void *pCtx, /* Context pointer to pass to xCallback */ - int bReverse, /* True for a reverse order scan */ - void *pKey1, int nKey1, /* Start of search */ - void *pKey2, int nKey2, /* End of search */ - void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) -){ - KcDb *pKcDb = (KcDb *)pDb; - kyotocabinet::DB::Cursor* pCur = pKcDb->db->cursor(); - int res; - - if( bReverse==0 ){ - if( pKey1 ){ - res = pCur->jump((const char *)pKey1, nKey1); - }else{ - res = pCur->jump(); - } - }else{ - if( pKey2 ){ - res = pCur->jump_back((const char *)pKey2, nKey2); - }else{ - res = pCur->jump_back(); - } - } - - while( res ){ - const char *pKey; size_t nKey; - const char *pVal; size_t nVal; - pKey = pCur->get(&nKey, &pVal, &nVal); - - if( bReverse==0 && pKey2 ){ - res = memcmp(pKey, pKey2, MIN((size_t)nKey2, nKey)); - if( res>0 || (res==0 && (size_t)nKey2nKey) ){ - delete [] pKey; - break; - } - } - - xCallback(pCtx, (void *)pKey, (int)nKey, (void *)pVal, (int)nVal); - delete [] pKey; - - if( bReverse ){ - res = pCur->step_back(); - }else{ - res = pCur->step(); - } - } - - delete pCur; - return 0; -} -#endif /* HAVE_KYOTOCABINET */ - -#ifdef HAVE_MDB -#include "lmdb.h" - -extern "C" { - struct MdbDb { - TestDb base; - MDB_env *env; - MDB_dbi dbi; - }; -} - -int test_mdb_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - MDB_txn *txn; - MdbDb *pMdb; - int rc; - - if( bClear ){ - char *zCmd = sqlite3_mprintf("rm -rf %s\n", zFilename); - system(zCmd); - sqlite3_free(zCmd); - } - - pMdb = (MdbDb *)malloc(sizeof(MdbDb)); - memset(pMdb, 0, sizeof(MdbDb)); - - rc = mdb_env_create(&pMdb->env); - if( rc==0 ) rc = mdb_env_set_mapsize(pMdb->env, 1*1024*1024*1024); - if( rc==0 ) rc = mdb_env_open(pMdb->env, zFilename, MDB_NOSYNC|MDB_NOSUBDIR, 0600); - if( rc==0 ) rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn); - if( rc==0 ){ - rc = mdb_open(txn, NULL, 0, &pMdb->dbi); - mdb_txn_commit(txn); - } - - *ppDb = (TestDb *)pMdb; - return rc; -} - -int test_mdb_close(TestDb *pDb){ - MdbDb *pMdb = (MdbDb *)pDb; - - mdb_close(pMdb->env, pMdb->dbi); - mdb_env_close(pMdb->env); - free(pMdb); - return 0; -} - -int test_mdb_write(TestDb *pDb, void *pKey, int nKey, void *pVal, int nVal){ - int rc; - MdbDb *pMdb = (MdbDb *)pDb; - MDB_val val; - MDB_val key; - MDB_txn *txn; - - val.mv_size = nVal; - val.mv_data = pVal; - key.mv_size = nKey; - key.mv_data = pKey; - - rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn); - if( rc==0 ){ - rc = mdb_put(txn, pMdb->dbi, &key, &val, 0); - if( rc==0 ){ - rc = mdb_txn_commit(txn); - }else{ - mdb_txn_abort(txn); - } - } - - return rc; -} - -int test_mdb_delete(TestDb *pDb, void *pKey, int nKey){ - int rc; - MdbDb *pMdb = (MdbDb *)pDb; - MDB_val key; - MDB_txn *txn; - - key.mv_size = nKey; - key.mv_data = pKey; - rc = mdb_txn_begin(pMdb->env, NULL, 0, &txn); - if( rc==0 ){ - rc = mdb_del(txn, pMdb->dbi, &key, 0); - if( rc==0 ){ - rc = mdb_txn_commit(txn); - }else{ - mdb_txn_abort(txn); - } - } - - return rc; -} - -int test_mdb_fetch( - TestDb *pDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - int rc; - MdbDb *pMdb = (MdbDb *)pDb; - MDB_val key; - MDB_txn *txn; - - key.mv_size = nKey; - key.mv_data = pKey; - - rc = mdb_txn_begin(pMdb->env, NULL, MDB_RDONLY, &txn); - if( rc==0 ){ - MDB_val val = {0, 0}; - rc = mdb_get(txn, pMdb->dbi, &key, &val); - if( rc==MDB_NOTFOUND ){ - rc = 0; - *ppVal = 0; - *pnVal = -1; - }else{ - *ppVal = val.mv_data; - *pnVal = val.mv_size; - } - mdb_txn_commit(txn); - } - - return rc; -} - -int test_mdb_scan( - TestDb *pDb, /* Database handle */ - void *pCtx, /* Context pointer to pass to xCallback */ - int bReverse, /* True for a reverse order scan */ - void *pKey1, int nKey1, /* Start of search */ - void *pKey2, int nKey2, /* End of search */ - void (*xCallback)(void *pCtx, void *pKey, int nKey, void *pVal, int nVal) -){ - MdbDb *pMdb = (MdbDb *)pDb; - int rc; - MDB_cursor_op op = bReverse ? MDB_PREV : MDB_NEXT; - MDB_txn *txn; - - rc = mdb_txn_begin(pMdb->env, NULL, MDB_RDONLY, &txn); - if( rc==0 ){ - MDB_cursor *csr; - MDB_val key = {0, 0}; - MDB_val val = {0, 0}; - - rc = mdb_cursor_open(txn, pMdb->dbi, &csr); - if( rc==0 ){ - while( mdb_cursor_get(csr, &key, &val, op)==0 ){ - xCallback(pCtx, key.mv_data, key.mv_size, val.mv_data, val.mv_size); - } - mdb_cursor_close(csr); - } - } - - return rc; -} - -#endif /* HAVE_MDB */ diff --git a/ext/lsm1/lsm-test/lsmtest_tdb3.c b/ext/lsm1/lsm-test/lsmtest_tdb3.c deleted file mode 100644 index e29497af2..000000000 --- a/ext/lsm1/lsm-test/lsmtest_tdb3.c +++ /dev/null @@ -1,1429 +0,0 @@ - -#include "lsmtest_tdb.h" -#include "lsm.h" -#include "lsmtest.h" - -#include -#include -#include -#ifndef _WIN32 -# include -#endif -#include - -#ifndef _WIN32 -# include -#endif - -typedef struct LsmDb LsmDb; -typedef struct LsmWorker LsmWorker; -typedef struct LsmFile LsmFile; - -#define LSMTEST_DFLT_MT_MAX_CKPT (8*1024) -#define LSMTEST_DFLT_MT_MIN_CKPT (2*1024) - -#ifdef LSM_MUTEX_PTHREADS -#include - -#define LSMTEST_THREAD_CKPT 1 -#define LSMTEST_THREAD_WORKER 2 -#define LSMTEST_THREAD_WORKER_AC 3 - -/* -** There are several different types of worker threads that run in different -** test configurations, depending on the value of LsmWorker.eType. -** -** 1. Checkpointer. -** 2. Worker with auto-checkpoint. -** 3. Worker without auto-checkpoint. -*/ -struct LsmWorker { - LsmDb *pDb; /* Main database structure */ - lsm_db *pWorker; /* Worker database handle */ - pthread_t worker_thread; /* Worker thread */ - pthread_cond_t worker_cond; /* Condition var the worker waits on */ - pthread_mutex_t worker_mutex; /* Mutex used with worker_cond */ - int bDoWork; /* Set to true by client when there is work */ - int worker_rc; /* Store error code here */ - int eType; /* LSMTEST_THREAD_XXX constant */ - int bBlock; -}; -#else -struct LsmWorker { int worker_rc; int bBlock; }; -#endif - -static void mt_shutdown(LsmDb *); - -lsm_env *tdb_lsm_env(void){ - static int bInit = 0; - static lsm_env env; - if( bInit==0 ){ - memcpy(&env, lsm_default_env(), sizeof(env)); - bInit = 1; - } - return &env; -} - -typedef struct FileSector FileSector; -typedef struct FileData FileData; - -struct FileSector { - u8 *aOld; /* Old data for this sector */ -}; - -struct FileData { - int nSector; /* Allocated size of apSector[] array */ - FileSector *aSector; /* Array of file sectors */ -}; - -/* -** bPrepareCrash: -** If non-zero, the file wrappers maintain enough in-memory data to -** simulate the effect of a power-failure on the file-system (i.e. that -** unsynced sectors may be written, not written, or overwritten with -** arbitrary data when the crash occurs). -** -** bCrashed: -** Set to true after a crash is simulated. Once this variable is true, all -** VFS methods other than xClose() return LSM_IOERR as soon as they are -** called (without affecting the contents of the file-system). -** -** env: -** The environment object used by all lsm_db* handles opened by this -** object (i.e. LsmDb.db plus any worker connections). Variable env.pVfsCtx -** always points to the containing LsmDb structure. -*/ -struct LsmDb { - TestDb base; /* Base class - methods table */ - lsm_env env; /* Environment used by connection db */ - char *zName; /* Database file name */ - lsm_db *db; /* LSM database handle */ - - lsm_cursor *pCsr; /* Cursor held open during read transaction */ - void *pBuf; /* Buffer for tdb_fetch() output */ - int nBuf; /* Allocated (not used) size of pBuf */ - - /* Crash testing related state */ - int bCrashed; /* True once a crash has occurred */ - int nAutoCrash; /* Number of syncs until a crash */ - int bPrepareCrash; /* True to store writes in memory */ - - /* Unsynced data (while crash testing) */ - int szSector; /* Assumed size of disk sectors (512B) */ - FileData aFile[2]; /* Database and log file data */ - - /* Other test instrumentation */ - int bNoRecovery; /* If true, assume DMS2 is locked */ - - /* Work hook redirection */ - void (*xWork)(lsm_db *, void *); - void *pWorkCtx; - - /* IO logging hook */ - void (*xWriteHook)(void *, int, lsm_i64, int, int); - void *pWriteCtx; - - /* Worker threads (for lsm_mt) */ - int nMtMinCkpt; - int nMtMaxCkpt; - int eMode; - int nWorker; - LsmWorker *aWorker; -}; - -#define LSMTEST_MODE_SINGLETHREAD 1 -#define LSMTEST_MODE_BACKGROUND_CKPT 2 -#define LSMTEST_MODE_BACKGROUND_WORK 3 -#define LSMTEST_MODE_BACKGROUND_BOTH 4 - -/************************************************************************* -************************************************************************** -** Begin test VFS code. -*/ - -struct LsmFile { - lsm_file *pReal; /* Real underlying file */ - int bLog; /* True for log file. False for db file */ - LsmDb *pDb; /* Database handle that uses this file */ -}; - -static int testEnvFullpath( - lsm_env *pEnv, /* Environment for current LsmDb */ - const char *zFile, /* Relative path name */ - char *zOut, /* Output buffer */ - int *pnOut /* IN/OUT: Size of output buffer */ -){ - lsm_env *pRealEnv = tdb_lsm_env(); - return pRealEnv->xFullpath(pRealEnv, zFile, zOut, pnOut); -} - -static int testEnvOpen( - lsm_env *pEnv, /* Environment for current LsmDb */ - const char *zFile, /* Name of file to open */ - int flags, - lsm_file **ppFile /* OUT: New file handle object */ -){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmDb *pDb = (LsmDb *)pEnv->pVfsCtx; - int rc; /* Return Code */ - LsmFile *pRet; /* The new file handle */ - int nFile; /* Length of string zFile in bytes */ - - nFile = strlen(zFile); - pRet = (LsmFile *)testMalloc(sizeof(LsmFile)); - pRet->pDb = pDb; - pRet->bLog = (nFile > 4 && 0==memcmp("-log", &zFile[nFile-4], 4)); - - rc = pRealEnv->xOpen(pRealEnv, zFile, flags, &pRet->pReal); - if( rc!=LSM_OK ){ - testFree(pRet); - pRet = 0; - } - - *ppFile = (lsm_file *)pRet; - return rc; -} - -static int testEnvRead(lsm_file *pFile, lsm_i64 iOff, void *pData, int nData){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - if( p->pDb->bCrashed ) return LSM_IOERR; - return pRealEnv->xRead(p->pReal, iOff, pData, nData); -} - -static int testEnvWrite(lsm_file *pFile, lsm_i64 iOff, void *pData, int nData){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - LsmDb *pDb = p->pDb; - - if( pDb->bCrashed ) return LSM_IOERR; - - if( pDb->bPrepareCrash ){ - FileData *pData2 = &pDb->aFile[p->bLog]; - int iFirst; - int iLast; - int iSector; - - iFirst = (int)(iOff / pDb->szSector); - iLast = (int)((iOff + nData - 1) / pDb->szSector); - - if( pData2->nSector<(iLast+1) ){ - int nNew = ( ((iLast + 1) + 63) / 64 ) * 64; - assert( nNew>iLast ); - pData2->aSector = (FileSector *)testRealloc( - pData2->aSector, nNew*sizeof(FileSector) - ); - memset(&pData2->aSector[pData2->nSector], - 0, (nNew - pData2->nSector) * sizeof(FileSector) - ); - pData2->nSector = nNew; - } - - for(iSector=iFirst; iSector<=iLast; iSector++){ - if( pData2->aSector[iSector].aOld==0 ){ - u8 *aOld = (u8 *)testMalloc(pDb->szSector); - pRealEnv->xRead( - p->pReal, (lsm_i64)iSector*pDb->szSector, aOld, pDb->szSector - ); - pData2->aSector[iSector].aOld = aOld; - } - } - } - - if( pDb->xWriteHook ){ - int rc; - int nUs; - struct timeval t1; - struct timeval t2; - - gettimeofday(&t1, 0); - assert( nData>0 ); - rc = pRealEnv->xWrite(p->pReal, iOff, pData, nData); - gettimeofday(&t2, 0); - - nUs = (t2.tv_sec - t1.tv_sec) * 1000000 + (t2.tv_usec - t1.tv_usec); - pDb->xWriteHook(pDb->pWriteCtx, p->bLog, iOff, nData, nUs); - return rc; - } - - return pRealEnv->xWrite(p->pReal, iOff, pData, nData); -} - -static void doSystemCrash(LsmDb *pDb); - -static int testEnvSync(lsm_file *pFile){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - LsmDb *pDb = p->pDb; - FileData *pData = &pDb->aFile[p->bLog]; - int i; - - if( pDb->bCrashed ) return LSM_IOERR; - - if( pDb->nAutoCrash ){ - pDb->nAutoCrash--; - if( pDb->nAutoCrash==0 ){ - doSystemCrash(pDb); - pDb->bCrashed = 1; - return LSM_IOERR; - } - } - - if( pDb->bPrepareCrash ){ - for(i=0; inSector; i++){ - testFree(pData->aSector[i].aOld); - pData->aSector[i].aOld = 0; - } - } - - if( pDb->xWriteHook ){ - int rc; - int nUs; - struct timeval t1; - struct timeval t2; - - gettimeofday(&t1, 0); - rc = pRealEnv->xSync(p->pReal); - gettimeofday(&t2, 0); - - nUs = (t2.tv_sec - t1.tv_sec) * 1000000 + (t2.tv_usec - t1.tv_usec); - pDb->xWriteHook(pDb->pWriteCtx, p->bLog, 0, 0, nUs); - return rc; - } - - return pRealEnv->xSync(p->pReal); -} - -static int testEnvTruncate(lsm_file *pFile, lsm_i64 iOff){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - if( p->pDb->bCrashed ) return LSM_IOERR; - return pRealEnv->xTruncate(p->pReal, iOff); -} - -static int testEnvSectorSize(lsm_file *pFile){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - return pRealEnv->xSectorSize(p->pReal); -} - -static int testEnvRemap( - lsm_file *pFile, - lsm_i64 iMin, - void **ppOut, - lsm_i64 *pnOut -){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - return pRealEnv->xRemap(p->pReal, iMin, ppOut, pnOut); -} - -static int testEnvFileid( - lsm_file *pFile, - void *ppOut, - int *pnOut -){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - return pRealEnv->xFileid(p->pReal, ppOut, pnOut); -} - -static int testEnvClose(lsm_file *pFile){ - lsm_env *pRealEnv = tdb_lsm_env(); - LsmFile *p = (LsmFile *)pFile; - - pRealEnv->xClose(p->pReal); - testFree(p); - return LSM_OK; -} - -static int testEnvUnlink(lsm_env *pEnv, const char *zFile){ - lsm_env *pRealEnv = tdb_lsm_env(); - unused_parameter(pEnv); - return pRealEnv->xUnlink(pRealEnv, zFile); -} - -static int testEnvLock(lsm_file *pFile, int iLock, int eType){ - LsmFile *p = (LsmFile *)pFile; - lsm_env *pRealEnv = tdb_lsm_env(); - - if( iLock==2 && eType==LSM_LOCK_EXCL && p->pDb->bNoRecovery ){ - return LSM_BUSY; - } - return pRealEnv->xLock(p->pReal, iLock, eType); -} - -static int testEnvTestLock(lsm_file *pFile, int iLock, int nLock, int eType){ - LsmFile *p = (LsmFile *)pFile; - lsm_env *pRealEnv = tdb_lsm_env(); - - if( iLock==2 && eType==LSM_LOCK_EXCL && p->pDb->bNoRecovery ){ - return LSM_BUSY; - } - return pRealEnv->xTestLock(p->pReal, iLock, nLock, eType); -} - -static int testEnvShmMap(lsm_file *pFile, int iRegion, int sz, void **pp){ - LsmFile *p = (LsmFile *)pFile; - lsm_env *pRealEnv = tdb_lsm_env(); - return pRealEnv->xShmMap(p->pReal, iRegion, sz, pp); -} - -static void testEnvShmBarrier(void){ -} - -static int testEnvShmUnmap(lsm_file *pFile, int bDel){ - LsmFile *p = (LsmFile *)pFile; - lsm_env *pRealEnv = tdb_lsm_env(); - return pRealEnv->xShmUnmap(p->pReal, bDel); -} - -static int testEnvSleep(lsm_env *pEnv, int us){ - lsm_env *pRealEnv = tdb_lsm_env(); - return pRealEnv->xSleep(pRealEnv, us); -} - -static void doSystemCrash(LsmDb *pDb){ - lsm_env *pEnv = tdb_lsm_env(); - int iFile; - int iSeed = pDb->aFile[0].nSector + pDb->aFile[1].nSector; - - char *zFile = pDb->zName; - char *zFree = 0; - - for(iFile=0; iFile<2; iFile++){ - lsm_file *pFile = 0; - int i; - - pEnv->xOpen(pEnv, zFile, 0, &pFile); - for(i=0; iaFile[iFile].nSector; i++){ - u8 *aOld = pDb->aFile[iFile].aSector[i].aOld; - if( aOld ){ - int iOpt = testPrngValue(iSeed++) % 3; - switch( iOpt ){ - case 0: - break; - - case 1: - testPrngArray(iSeed++, (u32 *)aOld, pDb->szSector/4); - /* Fall-through */ - - case 2: - pEnv->xWrite( - pFile, (lsm_i64)i * pDb->szSector, aOld, pDb->szSector - ); - break; - } - testFree(aOld); - pDb->aFile[iFile].aSector[i].aOld = 0; - } - } - pEnv->xClose(pFile); - zFree = zFile = sqlite3_mprintf("%s-log", pDb->zName); - } - - sqlite3_free(zFree); -} -/* -** End test VFS code. -************************************************************************** -*************************************************************************/ - -/************************************************************************* -************************************************************************** -** Begin test compression hooks. -*/ - -#ifdef HAVE_ZLIB -#include - -static int testZipBound(void *pCtx, int nSrc){ - return compressBound(nSrc); -} - -static int testZipCompress( - void *pCtx, /* Context pointer */ - char *aOut, int *pnOut, /* OUT: Buffer containing compressed data */ - const char *aIn, int nIn /* Buffer containing input data */ -){ - uLongf n = *pnOut; /* In/out buffer size for compress() */ - int rc; /* compress() return code */ - - rc = compress((Bytef*)aOut, &n, (Bytef*)aIn, nIn); - *pnOut = n; - return (rc==Z_OK ? 0 : LSM_ERROR); -} - -static int testZipUncompress( - void *pCtx, /* Context pointer */ - char *aOut, int *pnOut, /* OUT: Buffer containing uncompressed data */ - const char *aIn, int nIn /* Buffer containing input data */ -){ - uLongf n = *pnOut; /* In/out buffer size for uncompress() */ - int rc; /* uncompress() return code */ - - rc = uncompress((Bytef*)aOut, &n, (Bytef*)aIn, nIn); - *pnOut = n; - return (rc==Z_OK ? 0 : LSM_ERROR); -} - -static int testConfigureCompression(lsm_db *pDb){ - static lsm_compress zip = { - 0, /* Context pointer (unused) */ - 1, /* Id value */ - testZipBound, /* xBound method */ - testZipCompress, /* xCompress method */ - testZipUncompress /* xUncompress method */ - }; - return lsm_config(pDb, LSM_CONFIG_SET_COMPRESSION, &zip); -} -#endif /* ifdef HAVE_ZLIB */ - -/* -** End test compression hooks. -************************************************************************** -*************************************************************************/ - -static int test_lsm_close(TestDb *pTestDb){ - int i; - int rc = LSM_OK; - LsmDb *pDb = (LsmDb *)pTestDb; - - lsm_csr_close(pDb->pCsr); - lsm_close(pDb->db); - - /* If this is a multi-threaded database, wait on the worker threads. */ - mt_shutdown(pDb); - for(i=0; inWorker && rc==LSM_OK; i++){ - rc = pDb->aWorker[i].worker_rc; - } - - for(i=0; iaFile[0].nSector; i++){ - testFree(pDb->aFile[0].aSector[i].aOld); - } - testFree(pDb->aFile[0].aSector); - for(i=0; iaFile[1].nSector; i++){ - testFree(pDb->aFile[1].aSector[i].aOld); - } - testFree(pDb->aFile[1].aSector); - - memset(pDb, sizeof(LsmDb), 0x11); - testFree((char *)pDb->pBuf); - testFree((char *)pDb); - return rc; -} - -static void mt_signal_worker(LsmDb*, int); - -static int waitOnCheckpointer(LsmDb *pDb, lsm_db *db){ - int nSleep = 0; - int nKB; - int rc; - - do { - nKB = 0; - rc = lsm_info(db, LSM_INFO_CHECKPOINT_SIZE, &nKB); - if( rc!=LSM_OK || nKBnMtMaxCkpt ) break; -#ifdef LSM_MUTEX_PTHREADS - mt_signal_worker(pDb, - (pDb->eMode==LSMTEST_MODE_BACKGROUND_CKPT ? 0 : 1) - ); -#endif - usleep(5000); - nSleep += 5; - }while( 1 ); - -#if 0 - if( nSleep ) printf("# waitOnCheckpointer(): nSleep=%d\n", nSleep); -#endif - - return rc; -} - -static int waitOnWorker(LsmDb *pDb){ - int rc; - int nLimit = -1; - int nSleep = 0; - - rc = lsm_config(pDb->db, LSM_CONFIG_AUTOFLUSH, &nLimit); - do { - int nOld, nNew, rc2; - rc2 = lsm_info(pDb->db, LSM_INFO_TREE_SIZE, &nOld, &nNew); - if( rc2!=LSM_OK ) return rc2; - if( nOld==0 || nNew<(nLimit/2) ) break; -#ifdef LSM_MUTEX_PTHREADS - mt_signal_worker(pDb, 0); -#endif - usleep(5000); - nSleep += 5; - }while( 1 ); - -#if 0 - if( nSleep ) printf("# waitOnWorker(): nSleep=%d\n", nSleep); -#endif - - return rc; -} - -static int test_lsm_write( - TestDb *pTestDb, - void *pKey, - int nKey, - void *pVal, - int nVal -){ - LsmDb *pDb = (LsmDb *)pTestDb; - int rc = LSM_OK; - - if( pDb->eMode==LSMTEST_MODE_BACKGROUND_CKPT ){ - rc = waitOnCheckpointer(pDb, pDb->db); - }else if( - pDb->eMode==LSMTEST_MODE_BACKGROUND_WORK - || pDb->eMode==LSMTEST_MODE_BACKGROUND_BOTH - ){ - rc = waitOnWorker(pDb); - } - - if( rc==LSM_OK ){ - rc = lsm_insert(pDb->db, pKey, nKey, pVal, nVal); - } - return rc; -} - -static int test_lsm_delete(TestDb *pTestDb, void *pKey, int nKey){ - LsmDb *pDb = (LsmDb *)pTestDb; - return lsm_delete(pDb->db, pKey, nKey); -} - -static int test_lsm_delete_range( - TestDb *pTestDb, - void *pKey1, int nKey1, - void *pKey2, int nKey2 -){ - LsmDb *pDb = (LsmDb *)pTestDb; - return lsm_delete_range(pDb->db, pKey1, nKey1, pKey2, nKey2); -} - -static int test_lsm_fetch( - TestDb *pTestDb, - void *pKey, - int nKey, - void **ppVal, - int *pnVal -){ - int rc; - LsmDb *pDb = (LsmDb *)pTestDb; - lsm_cursor *csr; - - if( pKey==0 ) return LSM_OK; - - if( pDb->pCsr==0 ){ - rc = lsm_csr_open(pDb->db, &csr); - if( rc!=LSM_OK ) return rc; - }else{ - csr = pDb->pCsr; - } - - rc = lsm_csr_seek(csr, pKey, nKey, LSM_SEEK_EQ); - if( rc==LSM_OK ){ - if( lsm_csr_valid(csr) ){ - const void *pVal; int nVal; - rc = lsm_csr_value(csr, &pVal, &nVal); - if( nVal>pDb->nBuf ){ - testFree(pDb->pBuf); - pDb->pBuf = testMalloc(nVal*2); - pDb->nBuf = nVal*2; - } - memcpy(pDb->pBuf, pVal, nVal); - *ppVal = pDb->pBuf; - *pnVal = nVal; - }else{ - *ppVal = 0; - *pnVal = -1; - } - } - if( pDb->pCsr==0 ){ - lsm_csr_close(csr); - } - return rc; -} - -static int test_lsm_scan( - TestDb *pTestDb, - void *pCtx, - int bReverse, - void *pFirst, int nFirst, - void *pLast, int nLast, - void (*xCallback)(void *, void *, int , void *, int) -){ - LsmDb *pDb = (LsmDb *)pTestDb; - lsm_cursor *csr; - lsm_cursor *csr2 = 0; - int rc; - - if( pDb->pCsr==0 ){ - rc = lsm_csr_open(pDb->db, &csr); - if( rc!=LSM_OK ) return rc; - }else{ - rc = LSM_OK; - csr = pDb->pCsr; - } - - /* To enhance testing, if both pLast and pFirst are defined, seek the - ** cursor to the "end" boundary here. Then the next block seeks it to - ** the "start" ready for the scan. The point is to test that cursors - ** can be reused. */ - if( pLast && pFirst ){ - if( bReverse ){ - rc = lsm_csr_seek(csr, pFirst, nFirst, LSM_SEEK_LE); - }else{ - rc = lsm_csr_seek(csr, pLast, nLast, LSM_SEEK_GE); - } - } - - if( bReverse ){ - if( pLast ){ - rc = lsm_csr_seek(csr, pLast, nLast, LSM_SEEK_LE); - }else{ - rc = lsm_csr_last(csr); - } - }else{ - if( pFirst ){ - rc = lsm_csr_seek(csr, pFirst, nFirst, LSM_SEEK_GE); - }else{ - rc = lsm_csr_first(csr); - } - } - - while( rc==LSM_OK && lsm_csr_valid(csr) ){ - const void *pKey; int nKey; - const void *pVal; int nVal; - int cmp; - - lsm_csr_key(csr, &pKey, &nKey); - lsm_csr_value(csr, &pVal, &nVal); - - if( bReverse && pFirst ){ - cmp = memcmp(pFirst, pKey, MIN(nKey, nFirst)); - if( cmp>0 || (cmp==0 && nFirst>nKey) ) break; - }else if( bReverse==0 && pLast ){ - cmp = memcmp(pLast, pKey, MIN(nKey, nLast)); - if( cmp<0 || (cmp==0 && nLastpCsr==0 ){ - lsm_csr_close(csr); - } - return rc; -} - -static int test_lsm_begin(TestDb *pTestDb, int iLevel){ - int rc = LSM_OK; - LsmDb *pDb = (LsmDb *)pTestDb; - - /* iLevel==0 is a no-op. */ - if( iLevel==0 ) return 0; - - if( pDb->pCsr==0 ) rc = lsm_csr_open(pDb->db, &pDb->pCsr); - if( rc==LSM_OK && iLevel>1 ){ - rc = lsm_begin(pDb->db, iLevel-1); - } - - return rc; -} -static int test_lsm_commit(TestDb *pTestDb, int iLevel){ - LsmDb *pDb = (LsmDb *)pTestDb; - - /* If iLevel==0, close any open read transaction */ - if( iLevel==0 && pDb->pCsr ){ - lsm_csr_close(pDb->pCsr); - pDb->pCsr = 0; - } - - /* If iLevel==0, close any open read transaction */ - return lsm_commit(pDb->db, MAX(0, iLevel-1)); -} -static int test_lsm_rollback(TestDb *pTestDb, int iLevel){ - LsmDb *pDb = (LsmDb *)pTestDb; - - /* If iLevel==0, close any open read transaction */ - if( iLevel==0 && pDb->pCsr ){ - lsm_csr_close(pDb->pCsr); - pDb->pCsr = 0; - } - - return lsm_rollback(pDb->db, MAX(0, iLevel-1)); -} - -/* -** A log message callback registered with lsm connections. Prints all -** messages to stderr. -*/ -static void xLog(void *pCtx, int rc, const char *z){ - unused_parameter(rc); - /* fprintf(stderr, "lsm: rc=%d \"%s\"\n", rc, z); */ - if( pCtx ) fprintf(stderr, "%s: ", (char *)pCtx); - fprintf(stderr, "%s\n", z); - fflush(stderr); -} - -static void xWorkHook(lsm_db *db, void *pArg){ - LsmDb *p = (LsmDb *)pArg; - if( p->xWork ) p->xWork(db, p->pWorkCtx); -} - -#define TEST_NO_RECOVERY -1 -#define TEST_COMPRESSION -3 - -#define TEST_MT_MODE -2 -#define TEST_MT_MIN_CKPT -4 -#define TEST_MT_MAX_CKPT -5 - - -int test_lsm_config_str( - LsmDb *pLsm, - lsm_db *db, - int bWorker, - const char *zStr, - int *pnThread -){ - struct CfgParam { - const char *zParam; - int bWorker; - int eParam; - } aParam[] = { - { "autoflush", 0, LSM_CONFIG_AUTOFLUSH }, - { "page_size", 0, LSM_CONFIG_PAGE_SIZE }, - { "block_size", 0, LSM_CONFIG_BLOCK_SIZE }, - { "safety", 0, LSM_CONFIG_SAFETY }, - { "autowork", 0, LSM_CONFIG_AUTOWORK }, - { "autocheckpoint", 0, LSM_CONFIG_AUTOCHECKPOINT }, - { "mmap", 0, LSM_CONFIG_MMAP }, - { "use_log", 0, LSM_CONFIG_USE_LOG }, - { "automerge", 0, LSM_CONFIG_AUTOMERGE }, - { "max_freelist", 0, LSM_CONFIG_MAX_FREELIST }, - { "multi_proc", 0, LSM_CONFIG_MULTIPLE_PROCESSES }, - { "worker_automerge", 1, LSM_CONFIG_AUTOMERGE }, - { "test_no_recovery", 0, TEST_NO_RECOVERY }, - { "bg_min_ckpt", 0, TEST_NO_RECOVERY }, - - { "mt_mode", 0, TEST_MT_MODE }, - { "mt_min_ckpt", 0, TEST_MT_MIN_CKPT }, - { "mt_max_ckpt", 0, TEST_MT_MAX_CKPT }, - -#ifdef HAVE_ZLIB - { "compression", 0, TEST_COMPRESSION }, -#endif - { 0, 0 } - }; - const char *z = zStr; - int nThread = 1; - - if( zStr==0 ) return 0; - - assert( db ); - while( z[0] ){ - const char *zStart; - - /* Skip whitespace */ - while( *z==' ' ) z++; - zStart = z; - - while( *z && *z!='=' ) z++; - if( *z ){ - int eParam; - int i; - int iVal; - int iMul = 1; - int rc; - char zParam[32]; - int nParam = z-zStart; - if( nParam==0 || nParam>sizeof(zParam)-1 ) goto syntax_error; - - memcpy(zParam, zStart, nParam); - zParam[nParam] = '\0'; - rc = testArgSelect(aParam, "param", zParam, &i); - if( rc!=0 ) return rc; - eParam = aParam[i].eParam; - - z++; - zStart = z; - while( *z>='0' && *z<='9' ) z++; - if( *z=='k' || *z=='K' ){ - iMul = 1; - z++; - }else if( *z=='M' || *z=='M' ){ - iMul = 1024; - z++; - } - nParam = z-zStart; - if( nParam==0 || nParam>sizeof(zParam)-1 ) goto syntax_error; - memcpy(zParam, zStart, nParam); - zParam[nParam] = '\0'; - iVal = atoi(zParam) * iMul; - - if( eParam>0 ){ - if( bWorker || aParam[i].bWorker==0 ){ - lsm_config(db, eParam, &iVal); - } - }else{ - switch( eParam ){ - case TEST_NO_RECOVERY: - if( pLsm ) pLsm->bNoRecovery = iVal; - break; - case TEST_MT_MODE: - if( pLsm ) nThread = iVal; - break; - case TEST_MT_MIN_CKPT: - if( pLsm && iVal>0 ) pLsm->nMtMinCkpt = iVal*1024; - break; - case TEST_MT_MAX_CKPT: - if( pLsm && iVal>0 ) pLsm->nMtMaxCkpt = iVal*1024; - break; -#ifdef HAVE_ZLIB - case TEST_COMPRESSION: - testConfigureCompression(db); - break; -#endif - } - } - }else if( z!=zStart ){ - goto syntax_error; - } - } - - if( pnThread ) *pnThread = nThread; - if( pLsm && pLsm->nMtMaxCkpt < pLsm->nMtMinCkpt ){ - pLsm->nMtMinCkpt = pLsm->nMtMaxCkpt; - } - - return 0; - syntax_error: - testPrintError("syntax error at: \"%s\"\n", z); - return 1; -} - -int tdb_lsm_config_str(TestDb *pDb, const char *zStr){ - int rc = 0; - if( tdb_lsm(pDb) ){ -#ifdef LSM_MUTEX_PTHREADS - int i; -#endif - LsmDb *pLsm = (LsmDb *)pDb; - - rc = test_lsm_config_str(pLsm, pLsm->db, 0, zStr, 0); -#ifdef LSM_MUTEX_PTHREADS - for(i=0; rc==0 && inWorker; i++){ - rc = test_lsm_config_str(0, pLsm->aWorker[i].pWorker, 1, zStr, 0); - } -#endif - } - return rc; -} - -int tdb_lsm_configure(lsm_db *db, const char *zConfig){ - return test_lsm_config_str(0, db, 0, zConfig, 0); -} - -static int testLsmStartWorkers(LsmDb *, int, const char *, const char *); - -static int testLsmOpen( - const char *zCfg, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - static const DatabaseMethods LsmMethods = { - test_lsm_close, - test_lsm_write, - test_lsm_delete, - test_lsm_delete_range, - test_lsm_fetch, - test_lsm_scan, - test_lsm_begin, - test_lsm_commit, - test_lsm_rollback - }; - - int rc; - int nFilename; - LsmDb *pDb; - - /* If the bClear flag is set, delete any existing database. */ - assert( zFilename); - if( bClear ) testDeleteLsmdb(zFilename); - nFilename = strlen(zFilename); - - pDb = (LsmDb *)testMalloc(sizeof(LsmDb) + nFilename + 1); - memset(pDb, 0, sizeof(LsmDb)); - pDb->base.pMethods = &LsmMethods; - pDb->zName = (char *)&pDb[1]; - memcpy(pDb->zName, zFilename, nFilename + 1); - - /* Default the sector size used for crash simulation to 512 bytes. - ** Todo: There should be an OS method to obtain this value - just as - ** there is in SQLite. For now, LSM assumes that it is smaller than - ** the page size (default 4KB). - */ - pDb->szSector = 256; - - /* Default values for the mt_min_ckpt and mt_max_ckpt parameters. */ - pDb->nMtMinCkpt = LSMTEST_DFLT_MT_MIN_CKPT; - pDb->nMtMaxCkpt = LSMTEST_DFLT_MT_MAX_CKPT; - - memcpy(&pDb->env, tdb_lsm_env(), sizeof(lsm_env)); - pDb->env.pVfsCtx = (void *)pDb; - pDb->env.xFullpath = testEnvFullpath; - pDb->env.xOpen = testEnvOpen; - pDb->env.xRead = testEnvRead; - pDb->env.xWrite = testEnvWrite; - pDb->env.xTruncate = testEnvTruncate; - pDb->env.xSync = testEnvSync; - pDb->env.xSectorSize = testEnvSectorSize; - pDb->env.xRemap = testEnvRemap; - pDb->env.xFileid = testEnvFileid; - pDb->env.xClose = testEnvClose; - pDb->env.xUnlink = testEnvUnlink; - pDb->env.xLock = testEnvLock; - pDb->env.xTestLock = testEnvTestLock; - pDb->env.xShmBarrier = testEnvShmBarrier; - pDb->env.xShmMap = testEnvShmMap; - pDb->env.xShmUnmap = testEnvShmUnmap; - pDb->env.xSleep = testEnvSleep; - - rc = lsm_new(&pDb->env, &pDb->db); - if( rc==LSM_OK ){ - int nThread = 1; - lsm_config_log(pDb->db, xLog, 0); - lsm_config_work_hook(pDb->db, xWorkHook, (void *)pDb); - - rc = test_lsm_config_str(pDb, pDb->db, 0, zCfg, &nThread); - if( rc==LSM_OK ) rc = lsm_open(pDb->db, zFilename); - - pDb->eMode = nThread; -#ifdef LSM_MUTEX_PTHREADS - if( rc==LSM_OK && nThread>1 ){ - testLsmStartWorkers(pDb, nThread, zFilename, zCfg); - } -#endif - - if( rc!=LSM_OK ){ - test_lsm_close((TestDb *)pDb); - pDb = 0; - } - } - - *ppDb = (TestDb *)pDb; - return rc; -} - -int test_lsm_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - return testLsmOpen(zSpec, zFilename, bClear, ppDb); -} - -int test_lsm_small_open( - const char *zSpec, - const char *zFile, - int bClear, - TestDb **ppDb -){ - const char *zCfg = "page_size=256 block_size=64 mmap=1024"; - return testLsmOpen(zCfg, zFile, bClear, ppDb); -} - -int test_lsm_lomem_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - /* "max_freelist=4 autocheckpoint=32" */ - const char *zCfg = - "page_size=256 block_size=64 autoflush=16 " - "autocheckpoint=32" - "mmap=0 " - ; - return testLsmOpen(zCfg, zFilename, bClear, ppDb); -} - -int test_lsm_lomem2_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - /* "max_freelist=4 autocheckpoint=32" */ - const char *zCfg = - "page_size=512 block_size=64 autoflush=0 mmap=0 " - ; - return testLsmOpen(zCfg, zFilename, bClear, ppDb); -} - -int test_lsm_zip_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - const char *zCfg = - "page_size=256 block_size=64 autoflush=16 " - "autocheckpoint=32 compression=1 mmap=0 " - ; - return testLsmOpen(zCfg, zFilename, bClear, ppDb); -} - -lsm_db *tdb_lsm(TestDb *pDb){ - if( pDb->pMethods->xClose==test_lsm_close ){ - return ((LsmDb *)pDb)->db; - } - return 0; -} - -int tdb_lsm_multithread(TestDb *pDb){ - int ret = 0; - if( tdb_lsm(pDb) ){ - ret = ((LsmDb*)pDb)->eMode!=LSMTEST_MODE_SINGLETHREAD; - } - return ret; -} - -void tdb_lsm_enable_log(TestDb *pDb, int bEnable){ - lsm_db *db = tdb_lsm(pDb); - if( db ){ - lsm_config_log(db, (bEnable ? xLog : 0), (void *)"client"); - } -} - -void tdb_lsm_application_crash(TestDb *pDb){ - if( tdb_lsm(pDb) ){ - LsmDb *p = (LsmDb *)pDb; - p->bCrashed = 1; - } -} - -void tdb_lsm_prepare_system_crash(TestDb *pDb){ - if( tdb_lsm(pDb) ){ - LsmDb *p = (LsmDb *)pDb; - p->bPrepareCrash = 1; - } -} - -void tdb_lsm_system_crash(TestDb *pDb){ - if( tdb_lsm(pDb) ){ - LsmDb *p = (LsmDb *)pDb; - p->bCrashed = 1; - doSystemCrash(p); - } -} - -void tdb_lsm_safety(TestDb *pDb, int eMode){ - assert( eMode==LSM_SAFETY_OFF - || eMode==LSM_SAFETY_NORMAL - || eMode==LSM_SAFETY_FULL - ); - if( tdb_lsm(pDb) ){ - int iParam = eMode; - LsmDb *p = (LsmDb *)pDb; - lsm_config(p->db, LSM_CONFIG_SAFETY, &iParam); - } -} - -void tdb_lsm_prepare_sync_crash(TestDb *pDb, int iSync){ - assert( iSync>0 ); - if( tdb_lsm(pDb) ){ - LsmDb *p = (LsmDb *)pDb; - p->nAutoCrash = iSync; - p->bPrepareCrash = 1; - } -} - -void tdb_lsm_config_work_hook( - TestDb *pDb, - void (*xWork)(lsm_db *, void *), - void *pWorkCtx -){ - if( tdb_lsm(pDb) ){ - LsmDb *p = (LsmDb *)pDb; - p->xWork = xWork; - p->pWorkCtx = pWorkCtx; - } -} - -void tdb_lsm_write_hook( - TestDb *pDb, - void (*xWrite)(void *, int, lsm_i64, int, int), - void *pWriteCtx -){ - if( tdb_lsm(pDb) ){ - LsmDb *p = (LsmDb *)pDb; - p->xWriteHook = xWrite; - p->pWriteCtx = pWriteCtx; - } -} - -int tdb_lsm_open(const char *zCfg, const char *zDb, int bClear, TestDb **ppDb){ - return testLsmOpen(zCfg, zDb, bClear, ppDb); -} - -#ifdef LSM_MUTEX_PTHREADS - -/* -** Signal worker thread iWorker that there may be work to do. -*/ -static void mt_signal_worker(LsmDb *pDb, int iWorker){ - LsmWorker *p = &pDb->aWorker[iWorker]; - pthread_mutex_lock(&p->worker_mutex); - p->bDoWork = 1; - pthread_cond_signal(&p->worker_cond); - pthread_mutex_unlock(&p->worker_mutex); -} - -/* -** This routine is used as the main() for all worker threads. -*/ -static void *worker_main(void *pArg){ - LsmWorker *p = (LsmWorker *)pArg; - lsm_db *pWorker; /* Connection to access db through */ - - pthread_mutex_lock(&p->worker_mutex); - while( (pWorker = p->pWorker) ){ - int rc = LSM_OK; - - /* Do some work. If an error occurs, exit. */ - - pthread_mutex_unlock(&p->worker_mutex); - if( p->eType==LSMTEST_THREAD_CKPT ){ - int nKB = 0; - rc = lsm_info(pWorker, LSM_INFO_CHECKPOINT_SIZE, &nKB); - if( rc==LSM_OK && nKB>=p->pDb->nMtMinCkpt ){ - rc = lsm_checkpoint(pWorker, 0); - } - }else{ - int nWrite; - do { - - if( p->eType==LSMTEST_THREAD_WORKER ){ - waitOnCheckpointer(p->pDb, pWorker); - } - - nWrite = 0; - rc = lsm_work(pWorker, 0, 256, &nWrite); - - if( p->eType==LSMTEST_THREAD_WORKER && nWrite ){ - mt_signal_worker(p->pDb, 1); - } - }while( nWrite && p->pWorker ); - } - pthread_mutex_lock(&p->worker_mutex); - - if( rc!=LSM_OK && rc!=LSM_BUSY ){ - p->worker_rc = rc; - break; - } - - /* The thread will wake up when it is signaled either because another - ** thread has created some work for this one or because the connection - ** is being closed. */ - if( p->pWorker && p->bDoWork==0 ){ - pthread_cond_wait(&p->worker_cond, &p->worker_mutex); - } - p->bDoWork = 0; - } - pthread_mutex_unlock(&p->worker_mutex); - - return 0; -} - - -static void mt_stop_worker(LsmDb *pDb, int iWorker){ - LsmWorker *p = &pDb->aWorker[iWorker]; - if( p->pWorker ){ - void *pDummy; - lsm_db *pWorker; - - /* Signal the worker to stop */ - pthread_mutex_lock(&p->worker_mutex); - pWorker = p->pWorker; - p->pWorker = 0; - pthread_cond_signal(&p->worker_cond); - pthread_mutex_unlock(&p->worker_mutex); - - /* Join the worker thread. */ - pthread_join(p->worker_thread, &pDummy); - - /* Free resources allocated in mt_start_worker() */ - pthread_cond_destroy(&p->worker_cond); - pthread_mutex_destroy(&p->worker_mutex); - lsm_close(pWorker); - } -} - -static void mt_shutdown(LsmDb *pDb){ - int i; - for(i=0; inWorker; i++){ - mt_stop_worker(pDb, i); - } -} - -/* -** This callback is invoked by LSM when the client database writes to -** the database file (i.e. to flush the contents of the in-memory tree). -** This implies there may be work to do on the database, so signal -** the worker threads. -*/ -static void mt_client_work_hook(lsm_db *db, void *pArg){ - LsmDb *pDb = (LsmDb *)pArg; /* LsmDb database handle */ - - /* Invoke the user level work-hook, if any. */ - if( pDb->xWork ) pDb->xWork(db, pDb->pWorkCtx); - - /* Wake up worker thread 0. */ - mt_signal_worker(pDb, 0); -} - -static void mt_worker_work_hook(lsm_db *db, void *pArg){ - LsmDb *pDb = (LsmDb *)pArg; /* LsmDb database handle */ - - /* Invoke the user level work-hook, if any. */ - if( pDb->xWork ) pDb->xWork(db, pDb->pWorkCtx); -} - -/* -** Launch worker thread iWorker for database connection pDb. -*/ -static int mt_start_worker( - LsmDb *pDb, /* Main database structure */ - int iWorker, /* Worker number to start */ - const char *zFilename, /* File name of database to open */ - const char *zCfg, /* Connection configuration string */ - int eType /* Type of worker thread */ -){ - int rc = 0; /* Return code */ - LsmWorker *p; /* Object to initialize */ - - assert( iWorkernWorker ); - assert( eType==LSMTEST_THREAD_CKPT - || eType==LSMTEST_THREAD_WORKER - || eType==LSMTEST_THREAD_WORKER_AC - ); - - p = &pDb->aWorker[iWorker]; - p->eType = eType; - p->pDb = pDb; - - /* Open the worker connection */ - if( rc==0 ) rc = lsm_new(&pDb->env, &p->pWorker); - if( zCfg ){ - test_lsm_config_str(pDb, p->pWorker, 1, zCfg, 0); - } - if( rc==0 ) rc = lsm_open(p->pWorker, zFilename); - lsm_config_log(p->pWorker, xLog, (void *)"worker"); - - /* Configure the work-hook */ - if( rc==0 ){ - lsm_config_work_hook(p->pWorker, mt_worker_work_hook, (void *)pDb); - } - - if( eType==LSMTEST_THREAD_WORKER ){ - test_lsm_config_str(0, p->pWorker, 1, "autocheckpoint=0", 0); - } - - /* Kick off the worker thread. */ - if( rc==0 ) rc = pthread_cond_init(&p->worker_cond, 0); - if( rc==0 ) rc = pthread_mutex_init(&p->worker_mutex, 0); - if( rc==0 ) rc = pthread_create(&p->worker_thread, 0, worker_main, (void *)p); - - return rc; -} - - -static int testLsmStartWorkers( - LsmDb *pDb, int eModel, const char *zFilename, const char *zCfg -){ - int rc; - - if( eModel<1 || eModel>4 ) return 1; - if( eModel==1 ) return 0; - - /* Configure a work-hook for the client connection. Worker 0 is signalled - ** every time the users connection writes to the database. */ - lsm_config_work_hook(pDb->db, mt_client_work_hook, (void *)pDb); - - /* Allocate space for two worker connections. They may not both be - ** used, but both are allocated. */ - pDb->aWorker = (LsmWorker *)testMalloc(sizeof(LsmWorker) * 2); - memset(pDb->aWorker, 0, sizeof(LsmWorker) * 2); - - switch( eModel ){ - case LSMTEST_MODE_BACKGROUND_CKPT: - pDb->nWorker = 1; - test_lsm_config_str(0, pDb->db, 0, "autocheckpoint=0", 0); - rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_CKPT); - break; - - case LSMTEST_MODE_BACKGROUND_WORK: - pDb->nWorker = 1; - test_lsm_config_str(0, pDb->db, 0, "autowork=0", 0); - rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_WORKER_AC); - break; - - case LSMTEST_MODE_BACKGROUND_BOTH: - pDb->nWorker = 2; - test_lsm_config_str(0, pDb->db, 0, "autowork=0", 0); - rc = mt_start_worker(pDb, 0, zFilename, zCfg, LSMTEST_THREAD_WORKER); - if( rc==0 ){ - rc = mt_start_worker(pDb, 1, zFilename, zCfg, LSMTEST_THREAD_CKPT); - } - break; - } - - return rc; -} - - -int test_lsm_mt2( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - const char *zCfg = "mt_mode=2"; - return testLsmOpen(zCfg, zFilename, bClear, ppDb); -} - -int test_lsm_mt3( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - const char *zCfg = "mt_mode=4"; - return testLsmOpen(zCfg, zFilename, bClear, ppDb); -} - -#else -static void mt_shutdown(LsmDb *pDb) { - unused_parameter(pDb); -} -int test_lsm_mt(const char *zFilename, int bClear, TestDb **ppDb){ - unused_parameter(zFilename); - unused_parameter(bClear); - unused_parameter(ppDb); - testPrintError("threads unavailable - recompile with LSM_MUTEX_PTHREADS\n"); - return 1; -} -#endif diff --git a/ext/lsm1/lsm-test/lsmtest_tdb4.c b/ext/lsm1/lsm-test/lsmtest_tdb4.c deleted file mode 100644 index 1f9292852..000000000 --- a/ext/lsm1/lsm-test/lsmtest_tdb4.c +++ /dev/null @@ -1,980 +0,0 @@ - -/* -** This file contains the TestDb bt wrapper. -*/ - -#include "lsmtest_tdb.h" -#include "lsmtest.h" -#include -#include "bt.h" - -#include - -typedef struct BtDb BtDb; -typedef struct BtFile BtFile; - -/* Background checkpointer interface (see implementations below). */ -typedef struct bt_ckpter bt_ckpter; -static int bgc_attach(BtDb *pDb, const char*); -static int bgc_detach(BtDb *pDb); - -/* -** Each database or log file opened by a database handle is wrapped by -** an object of the following type. -*/ -struct BtFile { - BtDb *pBt; /* Database handle that opened this file */ - bt_env *pVfs; /* Underlying VFS */ - bt_file *pFile; /* File handle belonging to underlying VFS */ - int nSectorSize; /* Size of sectors in bytes */ - int nSector; /* Allocated size of nSector array */ - u8 **apSector; /* Original sector data */ -}; - -/* -** nCrashSync: -** If this value is non-zero, then a "crash-test" is running. If -** nCrashSync==1, then the crash is simulated during the very next -** call to the xSync() VFS method (on either the db or log file). -** If nCrashSync==2, the following call to xSync(), and so on. -** -** bCrash: -** After a crash is simulated, this variable is set. Any subsequent -** attempts to write to a file or modify the file system in any way -** fail once this is set. All the caller can do is close the connection. -** -** bFastInsert: -** If this variable is set to true, then a BT_CONTROL_FAST_INSERT_OP -** control is issued before each callto BtReplace() or BtCsrOpen(). -*/ -struct BtDb { - TestDb base; /* Base class */ - bt_db *pBt; /* bt database handle */ - sqlite4_env *pEnv; /* SQLite environment (for malloc/free) */ - bt_env *pVfs; /* Underlying VFS */ - int bFastInsert; /* True to use fast-insert */ - - /* Space for bt_fetch() results */ - u8 *aBuffer; /* Space to store results */ - int nBuffer; /* Allocated size of aBuffer[] in bytes */ - int nRef; - - /* Background checkpointer used by mt connections */ - bt_ckpter *pCkpter; - - /* Stuff used for crash test simulation */ - BtFile *apFile[2]; /* Database and log files used by pBt */ - bt_env env; /* Private VFS for this object */ - int nCrashSync; /* Number of syncs until crash (see above) */ - int bCrash; /* True once a crash has been simulated */ -}; - -static int btVfsFullpath( - sqlite4_env *pEnv, - bt_env *pVfs, - const char *z, - char **pzOut -){ - BtDb *pBt = (BtDb*)pVfs->pVfsCtx; - if( pBt->bCrash ) return SQLITE4_IOERR; - return pBt->pVfs->xFullpath(pEnv, pBt->pVfs, z, pzOut); -} - -static int btVfsOpen( - sqlite4_env *pEnv, - bt_env *pVfs, - const char *zFile, - int flags, bt_file **ppFile -){ - BtFile *p; - BtDb *pBt = (BtDb*)pVfs->pVfsCtx; - int rc; - - if( pBt->bCrash ) return SQLITE4_IOERR; - - p = (BtFile*)testMalloc(sizeof(BtFile)); - if( !p ) return SQLITE4_NOMEM; - if( flags & BT_OPEN_DATABASE ){ - pBt->apFile[0] = p; - }else if( flags & BT_OPEN_LOG ){ - pBt->apFile[1] = p; - } - if( (flags & BT_OPEN_SHARED)==0 ){ - p->pBt = pBt; - } - p->pVfs = pBt->pVfs; - - rc = pBt->pVfs->xOpen(pEnv, pVfs, zFile, flags, &p->pFile); - if( rc!=SQLITE4_OK ){ - testFree(p); - p = 0; - }else{ - pBt->nRef++; - } - - *ppFile = (bt_file*)p; - return rc; -} - -static int btVfsSize(bt_file *pFile, sqlite4_int64 *piRes){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - return p->pVfs->xSize(p->pFile, piRes); -} - -static int btVfsRead(bt_file *pFile, sqlite4_int64 iOff, void *pBuf, int nBuf){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - return p->pVfs->xRead(p->pFile, iOff, pBuf, nBuf); -} - -static int btFlushSectors(BtFile *p, int iFile){ - sqlite4_int64 iSz; - int rc; - int i; - u8 *aTmp = 0; - - rc = p->pBt->pVfs->xSize(p->pFile, &iSz); - for(i=0; rc==SQLITE4_OK && inSector; i++){ - if( p->pBt->bCrash && p->apSector[i] ){ - - /* The system is simulating a crash. There are three choices for - ** this sector: - ** - ** 1) Leave it as it is (simulating a successful write), - ** 2) Restore the original data (simulating a lost write), - ** 3) Populate the disk sector with garbage data. - */ - sqlite4_int64 iSOff = p->nSectorSize*i; - int nWrite = MIN(p->nSectorSize, iSz - iSOff); - - if( nWrite ){ - u8 *aWrite = 0; - int iOpt = (testPrngValue(i) % 3) + 1; - if( iOpt==1 ){ - aWrite = p->apSector[i]; - }else if( iOpt==3 ){ - if( aTmp==0 ) aTmp = testMalloc(p->nSectorSize); - aWrite = aTmp; - testPrngArray(i*13, (u32*)aWrite, nWrite/sizeof(u32)); - } - -#if 0 -fprintf(stderr, "handle sector %d of %s with %s\n", i, - iFile==0 ? "db" : "log", - iOpt==1 ? "rollback" : iOpt==2 ? "write" : "omit" -); -fflush(stderr); -#endif - - if( aWrite ){ - rc = p->pBt->pVfs->xWrite(p->pFile, iSOff, aWrite, nWrite); - } - } - } - testFree(p->apSector[i]); - p->apSector[i] = 0; - } - - testFree(aTmp); - return rc; -} - -static int btSaveSectors(BtFile *p, sqlite4_int64 iOff, int nBuf){ - int rc; - sqlite4_int64 iSz; /* Size of file on disk */ - int iFirst; /* First sector affected */ - int iSector; /* Current sector */ - int iLast; /* Last sector affected */ - - if( p->nSectorSize==0 ){ - p->nSectorSize = p->pBt->pVfs->xSectorSize(p->pFile); - if( p->nSectorSize<512 ) p->nSectorSize = 512; - } - iLast = (iOff+nBuf-1) / p->nSectorSize; - iFirst = iOff / p->nSectorSize; - - rc = p->pBt->pVfs->xSize(p->pFile, &iSz); - for(iSector=iFirst; rc==SQLITE4_OK && iSector<=iLast; iSector++){ - int nRead; - sqlite4_int64 iSOff = iSector * p->nSectorSize; - u8 *aBuf = testMalloc(p->nSectorSize); - nRead = MIN(p->nSectorSize, (iSz - iSOff)); - if( nRead>0 ){ - rc = p->pBt->pVfs->xRead(p->pFile, iSOff, aBuf, nRead); - } - - while( rc==SQLITE4_OK && iSector>=p->nSector ){ - int nNew = p->nSector + 32; - u8 **apNew = (u8**)testMalloc(nNew * sizeof(u8*)); - memcpy(apNew, p->apSector, p->nSector*sizeof(u8*)); - testFree(p->apSector); - p->apSector = apNew; - p->nSector = nNew; - } - - p->apSector[iSector] = aBuf; - } - - return rc; -} - -static int btVfsWrite(bt_file *pFile, sqlite4_int64 iOff, void *pBuf, int nBuf){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - if( p->pBt && p->pBt->nCrashSync ){ - btSaveSectors(p, iOff, nBuf); - } - return p->pVfs->xWrite(p->pFile, iOff, pBuf, nBuf); -} - -static int btVfsTruncate(bt_file *pFile, sqlite4_int64 iOff){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - return p->pVfs->xTruncate(p->pFile, iOff); -} - -static int btVfsSync(bt_file *pFile){ - int rc = SQLITE4_OK; - BtFile *p = (BtFile*)pFile; - BtDb *pBt = p->pBt; - - if( pBt ){ - if( pBt->bCrash ) return SQLITE4_IOERR; - if( pBt->nCrashSync ){ - pBt->nCrashSync--; - pBt->bCrash = (pBt->nCrashSync==0); - if( pBt->bCrash ){ - btFlushSectors(pBt->apFile[0], 0); - btFlushSectors(pBt->apFile[1], 1); - rc = SQLITE4_IOERR; - }else{ - btFlushSectors(p, 0); - } - } - } - - if( rc==SQLITE4_OK ){ - rc = p->pVfs->xSync(p->pFile); - } - return rc; -} - -static int btVfsSectorSize(bt_file *pFile){ - BtFile *p = (BtFile*)pFile; - return p->pVfs->xSectorSize(p->pFile); -} - -static void btDeref(BtDb *p){ - p->nRef--; - assert( p->nRef>=0 ); - if( p->nRef<=0 ) testFree(p); -} - -static int btVfsClose(bt_file *pFile){ - BtFile *p = (BtFile*)pFile; - BtDb *pBt = p->pBt; - int rc; - if( pBt ){ - btFlushSectors(p, 0); - if( p==pBt->apFile[0] ) pBt->apFile[0] = 0; - if( p==pBt->apFile[1] ) pBt->apFile[1] = 0; - } - testFree(p->apSector); - rc = p->pVfs->xClose(p->pFile); -#if 0 - btDeref(p->pBt); -#endif - testFree(p); - return rc; -} - -static int btVfsUnlink(sqlite4_env *pEnv, bt_env *pVfs, const char *zFile){ - BtDb *pBt = (BtDb*)pVfs->pVfsCtx; - if( pBt->bCrash ) return SQLITE4_IOERR; - return pBt->pVfs->xUnlink(pEnv, pBt->pVfs, zFile); -} - -static int btVfsLock(bt_file *pFile, int iLock, int eType){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - return p->pVfs->xLock(p->pFile, iLock, eType); -} - -static int btVfsTestLock(bt_file *pFile, int iLock, int nLock, int eType){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - return p->pVfs->xTestLock(p->pFile, iLock, nLock, eType); -} - -static int btVfsShmMap(bt_file *pFile, int iChunk, int sz, void **ppOut){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - return p->pVfs->xShmMap(p->pFile, iChunk, sz, ppOut); -} - -static void btVfsShmBarrier(bt_file *pFile){ - BtFile *p = (BtFile*)pFile; - return p->pVfs->xShmBarrier(p->pFile); -} - -static int btVfsShmUnmap(bt_file *pFile, int bDelete){ - BtFile *p = (BtFile*)pFile; - if( p->pBt && p->pBt->bCrash ) return SQLITE4_IOERR; - return p->pVfs->xShmUnmap(p->pFile, bDelete); -} - -static int bt_close(TestDb *pTestDb){ - BtDb *p = (BtDb*)pTestDb; - int rc = sqlite4BtClose(p->pBt); - free(p->aBuffer); - if( p->apFile[0] ) p->apFile[0]->pBt = 0; - if( p->apFile[1] ) p->apFile[1]->pBt = 0; - bgc_detach(p); - testFree(p); - return rc; -} - -static int btMinTransaction(BtDb *p, int iMin, int *piLevel){ - int iLevel; - int rc = SQLITE4_OK; - - iLevel = sqlite4BtTransactionLevel(p->pBt); - if( iLevelpBt, iMin); - *piLevel = iLevel; - }else{ - *piLevel = -1; - } - - return rc; -} -static int btRestoreTransaction(BtDb *p, int iLevel, int rcin){ - int rc = rcin; - if( iLevel>=0 ){ - if( rc==SQLITE4_OK ){ - rc = sqlite4BtCommit(p->pBt, iLevel); - }else{ - sqlite4BtRollback(p->pBt, iLevel); - } - assert( iLevel==sqlite4BtTransactionLevel(p->pBt) ); - } - return rc; -} - -static int bt_write(TestDb *pTestDb, void *pK, int nK, void *pV, int nV){ - BtDb *p = (BtDb*)pTestDb; - int iLevel; - int rc; - - rc = btMinTransaction(p, 2, &iLevel); - if( rc==SQLITE4_OK ){ - if( p->bFastInsert ) sqlite4BtControl(p->pBt, BT_CONTROL_FAST_INSERT_OP, 0); - rc = sqlite4BtReplace(p->pBt, pK, nK, pV, nV); - rc = btRestoreTransaction(p, iLevel, rc); - } - return rc; -} - -static int bt_delete(TestDb *pTestDb, void *pK, int nK){ - return bt_write(pTestDb, pK, nK, 0, -1); -} - -static int bt_delete_range( - TestDb *pTestDb, - void *pKey1, int nKey1, - void *pKey2, int nKey2 -){ - BtDb *p = (BtDb*)pTestDb; - bt_cursor *pCsr = 0; - int rc = SQLITE4_OK; - int iLevel; - - rc = btMinTransaction(p, 2, &iLevel); - if( rc==SQLITE4_OK ){ - if( p->bFastInsert ) sqlite4BtControl(p->pBt, BT_CONTROL_FAST_INSERT_OP, 0); - rc = sqlite4BtCsrOpen(p->pBt, 0, &pCsr); - } - while( rc==SQLITE4_OK ){ - const void *pK; - int n; - int nCmp; - int res; - - rc = sqlite4BtCsrSeek(pCsr, pKey1, nKey1, BT_SEEK_GE); - if( rc==SQLITE4_INEXACT ) rc = SQLITE4_OK; - if( rc!=SQLITE4_OK ) break; - - rc = sqlite4BtCsrKey(pCsr, &pK, &n); - if( rc!=SQLITE4_OK ) break; - - nCmp = MIN(n, nKey1); - res = memcmp(pKey1, pK, nCmp); - assert( res<0 || (res==0 && nKey1<=n) ); - if( res==0 && nKey1==n ){ - rc = sqlite4BtCsrNext(pCsr); - if( rc!=SQLITE4_OK ) break; - rc = sqlite4BtCsrKey(pCsr, &pK, &n); - if( rc!=SQLITE4_OK ) break; - } - - nCmp = MIN(n, nKey2); - res = memcmp(pKey2, pK, nCmp); - if( res<0 || (res==0 && nKey2<=n) ) break; - - rc = sqlite4BtDelete(pCsr); - } - if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK; - - sqlite4BtCsrClose(pCsr); - - rc = btRestoreTransaction(p, iLevel, rc); - return rc; -} - -static int bt_fetch( - TestDb *pTestDb, - void *pK, int nK, - void **ppVal, int *pnVal -){ - BtDb *p = (BtDb*)pTestDb; - bt_cursor *pCsr = 0; - int iLevel; - int rc = SQLITE4_OK; - - iLevel = sqlite4BtTransactionLevel(p->pBt); - if( iLevel==0 ){ - rc = sqlite4BtBegin(p->pBt, 1); - if( rc!=SQLITE4_OK ) return rc; - } - - if( p->bFastInsert ) sqlite4BtControl(p->pBt, BT_CONTROL_FAST_INSERT_OP, 0); - rc = sqlite4BtCsrOpen(p->pBt, 0, &pCsr); - if( rc==SQLITE4_OK ){ - rc = sqlite4BtCsrSeek(pCsr, pK, nK, BT_SEEK_EQ); - if( rc==SQLITE4_OK ){ - const void *pV = 0; - int nV = 0; - rc = sqlite4BtCsrData(pCsr, 0, -1, &pV, &nV); - if( rc==SQLITE4_OK ){ - if( nV>p->nBuffer ){ - free(p->aBuffer); - p->aBuffer = (u8*)malloc(nV*2); - p->nBuffer = nV*2; - } - memcpy(p->aBuffer, pV, nV); - *pnVal = nV; - *ppVal = (void*)(p->aBuffer); - } - - }else if( rc==SQLITE4_INEXACT || rc==SQLITE4_NOTFOUND ){ - *ppVal = 0; - *pnVal = -1; - rc = SQLITE4_OK; - } - sqlite4BtCsrClose(pCsr); - } - - if( iLevel==0 ) sqlite4BtCommit(p->pBt, 0); - return rc; -} - -static int bt_scan( - TestDb *pTestDb, - void *pCtx, - int bReverse, - void *pFirst, int nFirst, - void *pLast, int nLast, - void (*xCallback)(void *, void *, int , void *, int) -){ - BtDb *p = (BtDb*)pTestDb; - bt_cursor *pCsr = 0; - int rc; - int iLevel; - - rc = btMinTransaction(p, 1, &iLevel); - - if( rc==SQLITE4_OK ){ - if( p->bFastInsert ) sqlite4BtControl(p->pBt, BT_CONTROL_FAST_INSERT_OP, 0); - rc = sqlite4BtCsrOpen(p->pBt, 0, &pCsr); - } - if( rc==SQLITE4_OK ){ - if( bReverse ){ - if( pLast ){ - rc = sqlite4BtCsrSeek(pCsr, pLast, nLast, BT_SEEK_LE); - }else{ - rc = sqlite4BtCsrLast(pCsr); - } - }else{ - rc = sqlite4BtCsrSeek(pCsr, pFirst, nFirst, BT_SEEK_GE); - } - if( rc==SQLITE4_INEXACT ) rc = SQLITE4_OK; - - while( rc==SQLITE4_OK ){ - const void *pK = 0; int nK = 0; - const void *pV = 0; int nV = 0; - - rc = sqlite4BtCsrKey(pCsr, &pK, &nK); - if( rc==SQLITE4_OK ){ - rc = sqlite4BtCsrData(pCsr, 0, -1, &pV, &nV); - } - - if( rc!=SQLITE4_OK ) break; - if( bReverse ){ - if( pFirst ){ - int res; - int nCmp = MIN(nK, nFirst); - res = memcmp(pFirst, pK, nCmp); - if( res>0 || (res==0 && nKnLast) ) break; - } - } - - xCallback(pCtx, (void*)pK, nK, (void*)pV, nV); - if( bReverse ){ - rc = sqlite4BtCsrPrev(pCsr); - }else{ - rc = sqlite4BtCsrNext(pCsr); - } - } - if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK; - - sqlite4BtCsrClose(pCsr); - } - - rc = btRestoreTransaction(p, iLevel, rc); - return rc; -} - -static int bt_begin(TestDb *pTestDb, int iLvl){ - BtDb *p = (BtDb*)pTestDb; - int rc = sqlite4BtBegin(p->pBt, iLvl); - return rc; -} - -static int bt_commit(TestDb *pTestDb, int iLvl){ - BtDb *p = (BtDb*)pTestDb; - int rc = sqlite4BtCommit(p->pBt, iLvl); - return rc; -} - -static int bt_rollback(TestDb *pTestDb, int iLvl){ - BtDb *p = (BtDb*)pTestDb; - int rc = sqlite4BtRollback(p->pBt, iLvl); - return rc; -} - -static int testParseOption( - const char **pzIn, /* IN/OUT: pointer to next option */ - const char **pzOpt, /* OUT: nul-terminated option name */ - const char **pzArg, /* OUT: nul-terminated option argument */ - char *pSpace /* Temporary space for output params */ -){ - const char *p = *pzIn; - const char *pStart; - int n; - - char *pOut = pSpace; - - while( *p==' ' ) p++; - pStart = p; - while( *p && *p!='=' ) p++; - if( *p==0 ) return 1; - - n = (p - pStart); - memcpy(pOut, pStart, n); - *pzOpt = pOut; - pOut += n; - *pOut++ = '\0'; - - p++; - pStart = p; - while( *p && *p!=' ' ) p++; - n = (p - pStart); - - memcpy(pOut, pStart, n); - *pzArg = pOut; - pOut += n; - *pOut++ = '\0'; - - *pzIn = p; - return 0; -} - -static int testParseInt(const char *z, int *piVal){ - int i = 0; - const char *p = z; - - while( *p>='0' && *p<='9' ){ - i = i*10 + (*p - '0'); - p++; - } - if( *p=='K' || *p=='k' ){ - i = i * 1024; - p++; - }else if( *p=='M' || *p=='m' ){ - i = i * 1024 * 1024; - p++; - } - - if( *p ) return SQLITE4_ERROR; - *piVal = i; - return SQLITE4_OK; -} - -static int testBtConfigure(BtDb *pDb, const char *zCfg, int *pbMt){ - int rc = SQLITE4_OK; - - if( zCfg ){ - struct CfgParam { - const char *zParam; - int eParam; - } aParam[] = { - { "safety", BT_CONTROL_SAFETY }, - { "autockpt", BT_CONTROL_AUTOCKPT }, - { "multiproc", BT_CONTROL_MULTIPROC }, - { "blksz", BT_CONTROL_BLKSZ }, - { "pagesz", BT_CONTROL_PAGESZ }, - { "mt", -1 }, - { "fastinsert", -2 }, - { 0, 0 } - }; - const char *z = zCfg; - int n = strlen(z); - char *aSpace; - const char *zOpt; - const char *zArg; - - aSpace = (char*)testMalloc(n+2); - while( rc==SQLITE4_OK && 0==testParseOption(&z, &zOpt, &zArg, aSpace) ){ - int i; - int iVal; - rc = testArgSelect(aParam, "param", zOpt, &i); - if( rc!=SQLITE4_OK ) break; - - rc = testParseInt(zArg, &iVal); - if( rc!=SQLITE4_OK ) break; - - switch( aParam[i].eParam ){ - case -1: - *pbMt = iVal; - break; - case -2: - pDb->bFastInsert = 1; - break; - default: - rc = sqlite4BtControl(pDb->pBt, aParam[i].eParam, (void*)&iVal); - break; - } - } - testFree(aSpace); - } - - return rc; -} - - -int test_bt_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - - static const DatabaseMethods SqlMethods = { - bt_close, - bt_write, - bt_delete, - bt_delete_range, - bt_fetch, - bt_scan, - bt_begin, - bt_commit, - bt_rollback - }; - BtDb *p = 0; - bt_db *pBt = 0; - int rc; - sqlite4_env *pEnv = sqlite4_env_default(); - - if( bClear && zFilename && zFilename[0] ){ - char *zLog = sqlite3_mprintf("%s-wal", zFilename); - unlink(zFilename); - unlink(zLog); - sqlite3_free(zLog); - } - - rc = sqlite4BtNew(pEnv, 0, &pBt); - if( rc==SQLITE4_OK ){ - int mt = 0; /* True for multi-threaded connection */ - - p = (BtDb*)testMalloc(sizeof(BtDb)); - p->base.pMethods = &SqlMethods; - p->pBt = pBt; - p->pEnv = pEnv; - p->nRef = 1; - - p->env.pVfsCtx = (void*)p; - p->env.xFullpath = btVfsFullpath; - p->env.xOpen = btVfsOpen; - p->env.xSize = btVfsSize; - p->env.xRead = btVfsRead; - p->env.xWrite = btVfsWrite; - p->env.xTruncate = btVfsTruncate; - p->env.xSync = btVfsSync; - p->env.xSectorSize = btVfsSectorSize; - p->env.xClose = btVfsClose; - p->env.xUnlink = btVfsUnlink; - p->env.xLock = btVfsLock; - p->env.xTestLock = btVfsTestLock; - p->env.xShmMap = btVfsShmMap; - p->env.xShmBarrier = btVfsShmBarrier; - p->env.xShmUnmap = btVfsShmUnmap; - - sqlite4BtControl(pBt, BT_CONTROL_GETVFS, (void*)&p->pVfs); - sqlite4BtControl(pBt, BT_CONTROL_SETVFS, (void*)&p->env); - - rc = testBtConfigure(p, zSpec, &mt); - if( rc==SQLITE4_OK ){ - rc = sqlite4BtOpen(pBt, zFilename); - } - - if( rc==SQLITE4_OK && mt ){ - int nAuto = 0; - rc = bgc_attach(p, zSpec); - sqlite4BtControl(pBt, BT_CONTROL_AUTOCKPT, (void*)&nAuto); - } - } - - if( rc!=SQLITE4_OK && p ){ - bt_close(&p->base); - } - - *ppDb = &p->base; - return rc; -} - -int test_fbt_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - return test_bt_open("fast=1", zFilename, bClear, ppDb); -} - -int test_fbts_open( - const char *zSpec, - const char *zFilename, - int bClear, - TestDb **ppDb -){ - return test_bt_open("fast=1 blksz=32K pagesz=512", zFilename, bClear, ppDb); -} - - -void tdb_bt_prepare_sync_crash(TestDb *pTestDb, int iSync){ - BtDb *p = (BtDb*)pTestDb; - assert( pTestDb->pMethods->xClose==bt_close ); - assert( p->bCrash==0 ); - p->nCrashSync = iSync; -} - -bt_db *tdb_bt(TestDb *pDb){ - if( pDb->pMethods->xClose==bt_close ){ - return ((BtDb *)pDb)->pBt; - } - return 0; -} - -/************************************************************************* -** Beginning of code for background checkpointer. -*/ - -struct bt_ckpter { - sqlite4_buffer file; /* File name */ - sqlite4_buffer spec; /* Options */ - int nLogsize; /* Minimum log size to checkpoint */ - int nRef; /* Number of clients */ - - int bDoWork; /* Set by client threads */ - pthread_t ckpter_thread; /* Checkpointer thread */ - pthread_cond_t ckpter_cond; /* Condition var the ckpter waits on */ - pthread_mutex_t ckpter_mutex; /* Mutex used with ckpter_cond */ - - bt_ckpter *pNext; /* Next object in list at gBgc.pCkpter */ -}; - -static struct GlobalBackgroundCheckpointer { - bt_ckpter *pCkpter; /* Linked list of checkpointers */ -} gBgc; - -static void *bgc_main(void *pArg){ - BtDb *pDb = 0; - int rc; - int mt; - bt_ckpter *pCkpter = (bt_ckpter*)pArg; - - rc = test_bt_open("", (char*)pCkpter->file.p, 0, (TestDb**)&pDb); - assert( rc==SQLITE4_OK ); - rc = testBtConfigure(pDb, (char*)pCkpter->spec.p, &mt); - - while( pCkpter->nRef>0 ){ - bt_db *db = pDb->pBt; - int nLog = 0; - - sqlite4BtBegin(db, 1); - sqlite4BtCommit(db, 0); - sqlite4BtControl(db, BT_CONTROL_LOGSIZE, (void*)&nLog); - - if( nLog>=pCkpter->nLogsize ){ - int rc; - bt_checkpoint ckpt; - memset(&ckpt, 0, sizeof(bt_checkpoint)); - ckpt.nFrameBuffer = nLog/2; - rc = sqlite4BtControl(db, BT_CONTROL_CHECKPOINT, (void*)&ckpt); - assert( rc==SQLITE4_OK ); - sqlite4BtControl(db, BT_CONTROL_LOGSIZE, (void*)&nLog); - } - - /* The thread will wake up when it is signaled either because another - ** thread has created some work for this one or because the connection - ** is being closed. */ - pthread_mutex_lock(&pCkpter->ckpter_mutex); - if( pCkpter->bDoWork==0 ){ - pthread_cond_wait(&pCkpter->ckpter_cond, &pCkpter->ckpter_mutex); - } - pCkpter->bDoWork = 0; - pthread_mutex_unlock(&pCkpter->ckpter_mutex); - } - - if( pDb ) bt_close((TestDb*)pDb); - return 0; -} - -static void bgc_logsize_cb(void *pCtx, int nLogsize){ - bt_ckpter *p = (bt_ckpter*)pCtx; - if( nLogsize>=p->nLogsize ){ - pthread_mutex_lock(&p->ckpter_mutex); - p->bDoWork = 1; - pthread_cond_signal(&p->ckpter_cond); - pthread_mutex_unlock(&p->ckpter_mutex); - } -} - -static int bgc_attach(BtDb *pDb, const char *zSpec){ - int rc; - int n; - bt_info info; - bt_ckpter *pCkpter; - - /* Figure out the full path to the database opened by handle pDb. */ - info.eType = BT_INFO_FILENAME; - info.pgno = 0; - sqlite4_buffer_init(&info.output, 0); - rc = sqlite4BtControl(pDb->pBt, BT_CONTROL_INFO, (void*)&info); - if( rc!=SQLITE4_OK ) return rc; - - sqlite4_mutex_enter(sqlite4_mutex_alloc(pDb->pEnv, SQLITE4_MUTEX_STATIC_KV)); - - /* Search for an existing bt_ckpter object. */ - n = info.output.n; - for(pCkpter=gBgc.pCkpter; pCkpter; pCkpter=pCkpter->pNext){ - if( n==pCkpter->file.n && 0==memcmp(info.output.p, pCkpter->file.p, n) ){ - break; - } - } - - /* Failed to find a suitable checkpointer. Create a new one. */ - if( pCkpter==0 ){ - bt_logsizecb cb; - - pCkpter = testMalloc(sizeof(bt_ckpter)); - memcpy(&pCkpter->file, &info.output, sizeof(sqlite4_buffer)); - info.output.p = 0; - pCkpter->pNext = gBgc.pCkpter; - pCkpter->nLogsize = 1000; - gBgc.pCkpter = pCkpter; - pCkpter->nRef = 1; - - sqlite4_buffer_init(&pCkpter->spec, 0); - rc = sqlite4_buffer_set(&pCkpter->spec, zSpec, strlen(zSpec)+1); - assert( rc==SQLITE4_OK ); - - /* Kick off the checkpointer thread. */ - if( rc==0 ) rc = pthread_cond_init(&pCkpter->ckpter_cond, 0); - if( rc==0 ) rc = pthread_mutex_init(&pCkpter->ckpter_mutex, 0); - if( rc==0 ){ - rc = pthread_create(&pCkpter->ckpter_thread, 0, bgc_main, (void*)pCkpter); - } - assert( rc==0 ); /* todo: Fix this */ - - /* Set up the logsize callback for the client thread */ - cb.pCtx = (void*)pCkpter; - cb.xLogsize = bgc_logsize_cb; - sqlite4BtControl(pDb->pBt, BT_CONTROL_LOGSIZECB, (void*)&cb); - }else{ - pCkpter->nRef++; - } - - /* Assuming a checkpointer was encountered or effected, attach the - ** connection to it. */ - if( pCkpter ){ - pDb->pCkpter = pCkpter; - } - - sqlite4_mutex_leave(sqlite4_mutex_alloc(pDb->pEnv, SQLITE4_MUTEX_STATIC_KV)); - sqlite4_buffer_clear(&info.output); - return rc; -} - -static int bgc_detach(BtDb *pDb){ - int rc = SQLITE4_OK; - bt_ckpter *pCkpter = pDb->pCkpter; - if( pCkpter ){ - int bShutdown = 0; /* True if this is the last reference */ - - sqlite4_mutex_enter(sqlite4_mutex_alloc(pDb->pEnv,SQLITE4_MUTEX_STATIC_KV)); - pCkpter->nRef--; - if( pCkpter->nRef==0 ){ - bt_ckpter **pp; - - *pp = pCkpter->pNext; - for(pp=&gBgc.pCkpter; *pp!=pCkpter; pp=&((*pp)->pNext)); - bShutdown = 1; - } - sqlite4_mutex_leave(sqlite4_mutex_alloc(pDb->pEnv,SQLITE4_MUTEX_STATIC_KV)); - - if( bShutdown ){ - void *pDummy; - - /* Signal the checkpointer thread. */ - pthread_mutex_lock(&pCkpter->ckpter_mutex); - pCkpter->bDoWork = 1; - pthread_cond_signal(&pCkpter->ckpter_cond); - pthread_mutex_unlock(&pCkpter->ckpter_mutex); - - /* Join the checkpointer thread. */ - pthread_join(pCkpter->ckpter_thread, &pDummy); - pthread_cond_destroy(&pCkpter->ckpter_cond); - pthread_mutex_destroy(&pCkpter->ckpter_mutex); - - sqlite4_buffer_clear(&pCkpter->file); - sqlite4_buffer_clear(&pCkpter->spec); - testFree(pCkpter); - } - - pDb->pCkpter = 0; - } - return rc; -} - -/* -** End of background checkpointer. -*************************************************************************/ diff --git a/ext/lsm1/lsm-test/lsmtest_util.c b/ext/lsm1/lsm-test/lsmtest_util.c deleted file mode 100644 index adab8a53e..000000000 --- a/ext/lsm1/lsm-test/lsmtest_util.c +++ /dev/null @@ -1,223 +0,0 @@ - -#include "lsmtest.h" -#include -#include -#include -#ifndef _WIN32 -# include -#endif - -/* -** Global variables used within this module. -*/ -static struct TestutilGlobal { - char **argv; - int argc; -} g = {0, 0}; - -static struct TestutilRnd { - unsigned int aRand1[2048]; /* Bits 0..10 */ - unsigned int aRand2[2048]; /* Bits 11..21 */ - unsigned int aRand3[1024]; /* Bits 22..31 */ -} r; - -/************************************************************************* -** The following block is a copy of the implementation of SQLite function -** sqlite3_randomness. This version has two important differences: -** -** 1. It always uses the same seed. So the sequence of random data output -** is the same for every run of the program. -** -** 2. It is not threadsafe. -*/ -static struct sqlite3PrngType { - unsigned char i, j; /* State variables */ - unsigned char s[256]; /* State variables */ -} sqlite3Prng = { - 0xAF, 0x28, - { - 0x71, 0xF5, 0xB4, 0x6E, 0x80, 0xAB, 0x1D, 0xB8, - 0xFB, 0xB7, 0x49, 0xBF, 0xFF, 0x72, 0x2D, 0x14, - 0x79, 0x09, 0xE3, 0x78, 0x76, 0xB0, 0x2C, 0x0A, - 0x8E, 0x23, 0xEE, 0xDF, 0xE0, 0x9A, 0x2F, 0x67, - 0xE1, 0xBE, 0x0E, 0xA7, 0x08, 0x97, 0xEB, 0x77, - 0x78, 0xBA, 0x9D, 0xCA, 0x49, 0x4C, 0x60, 0x9A, - 0xF6, 0xBD, 0xDA, 0x7F, 0xBC, 0x48, 0x58, 0x52, - 0xE5, 0xCD, 0x83, 0x72, 0x23, 0x52, 0xFF, 0x6D, - 0xEF, 0x0F, 0x82, 0x29, 0xA0, 0x83, 0x3F, 0x7D, - 0xA4, 0x88, 0x31, 0xE7, 0x88, 0x92, 0x3B, 0x9B, - 0x3B, 0x2C, 0xC2, 0x4C, 0x71, 0xA2, 0xB0, 0xEA, - 0x36, 0xD0, 0x00, 0xF1, 0xD3, 0x39, 0x17, 0x5D, - 0x2A, 0x7A, 0xE4, 0xAD, 0xE1, 0x64, 0xCE, 0x0F, - 0x9C, 0xD9, 0xF5, 0xED, 0xB0, 0x22, 0x5E, 0x62, - 0x97, 0x02, 0xA3, 0x8C, 0x67, 0x80, 0xFC, 0x88, - 0x14, 0x0B, 0x15, 0x10, 0x0F, 0xC7, 0x40, 0xD4, - 0xF1, 0xF9, 0x0E, 0x1A, 0xCE, 0xB9, 0x1E, 0xA1, - 0x72, 0x8E, 0xD7, 0x78, 0x39, 0xCD, 0xF4, 0x5D, - 0x2A, 0x59, 0x26, 0x34, 0xF2, 0x73, 0x0B, 0xA0, - 0x02, 0x51, 0x2C, 0x03, 0xA3, 0xA7, 0x43, 0x13, - 0xE8, 0x98, 0x2B, 0xD2, 0x53, 0xF8, 0xEE, 0x91, - 0x7D, 0xE7, 0xE3, 0xDA, 0xD5, 0xBB, 0xC0, 0x92, - 0x9D, 0x98, 0x01, 0x2C, 0xF9, 0xB9, 0xA0, 0xEB, - 0xCF, 0x32, 0xFA, 0x01, 0x49, 0xA5, 0x1D, 0x9A, - 0x76, 0x86, 0x3F, 0x40, 0xD4, 0x89, 0x8F, 0x9C, - 0xE2, 0xE3, 0x11, 0x31, 0x37, 0xB2, 0x49, 0x28, - 0x35, 0xC0, 0x99, 0xB6, 0xD0, 0xBC, 0x66, 0x35, - 0xF7, 0x83, 0x5B, 0xD7, 0x37, 0x1A, 0x2B, 0x18, - 0xA6, 0xFF, 0x8D, 0x7C, 0x81, 0xA8, 0xFC, 0x9E, - 0xC4, 0xEC, 0x80, 0xD0, 0x98, 0xA7, 0x76, 0xCC, - 0x9C, 0x2F, 0x7B, 0xFF, 0x8E, 0x0E, 0xBB, 0x90, - 0xAE, 0x13, 0x06, 0xF5, 0x1C, 0x4E, 0x52, 0xF7 - } -}; - -/* Generate and return single random byte */ -static unsigned char randomByte(void){ - unsigned char t; - sqlite3Prng.i++; - t = sqlite3Prng.s[sqlite3Prng.i]; - sqlite3Prng.j += t; - sqlite3Prng.s[sqlite3Prng.i] = sqlite3Prng.s[sqlite3Prng.j]; - sqlite3Prng.s[sqlite3Prng.j] = t; - t += sqlite3Prng.s[sqlite3Prng.i]; - return sqlite3Prng.s[t]; -} - -/* -** Return N random bytes. -*/ -static void randomBlob(int nBuf, unsigned char *zBuf){ - int i; - for(i=0; i>11) & 0x000007FF] ^ - r.aRand3[(iVal>>22) & 0x000003FF] - ; -} - -void testPrngArray(unsigned int iVal, unsigned int *aOut, int nOut){ - int i; - for(i=0; izName; - pEntry=(struct Entry *)&((unsigned char *)pEntry)[sz] - ){ - if( zPrev ){ testPrintError("%s, ", zPrev); } - zPrev = pEntry->zName; - } - testPrintError("or %s\n", zPrev); -} - -int testArgSelectX( - void *aData, - const char *zType, - int sz, - const char *zArg, - int *piOut -){ - struct Entry { const char *zName; }; - struct Entry *pEntry; - int nArg = strlen(zArg); - - int i = 0; - int iOut = -1; - int nOut = 0; - - for(pEntry=(struct Entry *)aData; - pEntry->zName; - pEntry=(struct Entry *)&((unsigned char *)pEntry)[sz] - ){ - int nName = strlen(pEntry->zName); - if( nArg<=nName && memcmp(pEntry->zName, zArg, nArg)==0 ){ - iOut = i; - if( nName==nArg ){ - nOut = 1; - break; - } - nOut++; - } - i++; - } - - if( nOut!=1 ){ - argError(aData, zType, sz, zArg); - }else{ - *piOut = iOut; - } - return (nOut!=1); -} - -struct timeval zero_time; - -void testTimeInit(void){ - gettimeofday(&zero_time, 0); -} - -int testTimeGet(void){ - struct timeval now; - gettimeofday(&now, 0); - return - (((int)now.tv_sec - (int)zero_time.tv_sec)*1000) + - (((int)now.tv_usec - (int)zero_time.tv_usec)/1000); -} diff --git a/ext/lsm1/lsm-test/lsmtest_win32.c b/ext/lsm1/lsm-test/lsmtest_win32.c deleted file mode 100644 index 947272336..000000000 --- a/ext/lsm1/lsm-test/lsmtest_win32.c +++ /dev/null @@ -1,30 +0,0 @@ - -#include "lsmtest.h" - -#ifdef _WIN32 - -#define TICKS_PER_SECOND (10000000) -#define TICKS_PER_MICROSECOND (10) -#define TICKS_UNIX_EPOCH (116444736000000000LL) - -int win32GetTimeOfDay( - struct timeval *tp, - void *tzp -){ - FILETIME fileTime; - ULONGLONG ticks; - ULONGLONG unixTicks; - - unused_parameter(tzp); - memset(&fileTime, 0, sizeof(FILETIME)); - GetSystemTimeAsFileTime(&fileTime); - ticks = (ULONGLONG)fileTime.dwHighDateTime << 32; - ticks |= (ULONGLONG)fileTime.dwLowDateTime; - unixTicks = ticks - TICKS_UNIX_EPOCH; - tp->tv_sec = (long)(unixTicks / TICKS_PER_SECOND); - unixTicks -= ((ULONGLONG)tp->tv_sec * TICKS_PER_SECOND); - tp->tv_usec = (long)(unixTicks / TICKS_PER_MICROSECOND); - - return 0; -} -#endif diff --git a/ext/lsm1/lsm.h b/ext/lsm1/lsm.h deleted file mode 100644 index 48701c4c5..000000000 --- a/ext/lsm1/lsm.h +++ /dev/null @@ -1,684 +0,0 @@ -/* -** 2011-08-10 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file defines the LSM API. -*/ -#ifndef _LSM_H -#define _LSM_H -#include -#ifdef __cplusplus -extern "C" { -#endif - -/* -** Opaque handle types. -*/ -typedef struct lsm_compress lsm_compress; /* Compression library functions */ -typedef struct lsm_compress_factory lsm_compress_factory; -typedef struct lsm_cursor lsm_cursor; /* Database cursor handle */ -typedef struct lsm_db lsm_db; /* Database connection handle */ -typedef struct lsm_env lsm_env; /* Runtime environment */ -typedef struct lsm_file lsm_file; /* OS file handle */ -typedef struct lsm_mutex lsm_mutex; /* Mutex handle */ - -/* 64-bit integer type used for file offsets. */ -typedef long long int lsm_i64; /* 64-bit signed integer type */ - -/* Candidate values for the 3rd argument to lsm_env.xLock() */ -#define LSM_LOCK_UNLOCK 0 -#define LSM_LOCK_SHARED 1 -#define LSM_LOCK_EXCL 2 - -/* Flags for lsm_env.xOpen() */ -#define LSM_OPEN_READONLY 0x0001 - -/* -** CAPI: Database Runtime Environment -** -** Run-time environment used by LSM -*/ -struct lsm_env { - int nByte; /* Size of this structure in bytes */ - int iVersion; /* Version number of this structure (1) */ - /****** file i/o ***********************************************/ - void *pVfsCtx; - int (*xFullpath)(lsm_env*, const char *, char *, int *); - int (*xOpen)(lsm_env*, const char *, int flags, lsm_file **); - int (*xRead)(lsm_file *, lsm_i64, void *, int); - int (*xWrite)(lsm_file *, lsm_i64, void *, int); - int (*xTruncate)(lsm_file *, lsm_i64); - int (*xSync)(lsm_file *); - int (*xSectorSize)(lsm_file *); - int (*xRemap)(lsm_file *, lsm_i64, void **, lsm_i64*); - int (*xFileid)(lsm_file *, void *pBuf, int *pnBuf); - int (*xClose)(lsm_file *); - int (*xUnlink)(lsm_env*, const char *); - int (*xLock)(lsm_file*, int, int); - int (*xTestLock)(lsm_file*, int, int, int); - int (*xShmMap)(lsm_file*, int, int, void **); - void (*xShmBarrier)(void); - int (*xShmUnmap)(lsm_file*, int); - /****** memory allocation ****************************************/ - void *pMemCtx; - void *(*xMalloc)(lsm_env*, size_t); /* malloc(3) function */ - void *(*xRealloc)(lsm_env*, void *, size_t); /* realloc(3) function */ - void (*xFree)(lsm_env*, void *); /* free(3) function */ - size_t (*xSize)(lsm_env*, void *); /* xSize function */ - /****** mutexes ****************************************************/ - void *pMutexCtx; - int (*xMutexStatic)(lsm_env*,int,lsm_mutex**); /* Obtain a static mutex */ - int (*xMutexNew)(lsm_env*, lsm_mutex**); /* Get a new dynamic mutex */ - void (*xMutexDel)(lsm_mutex *); /* Delete an allocated mutex */ - void (*xMutexEnter)(lsm_mutex *); /* Grab a mutex */ - int (*xMutexTry)(lsm_mutex *); /* Attempt to obtain a mutex */ - void (*xMutexLeave)(lsm_mutex *); /* Leave a mutex */ - int (*xMutexHeld)(lsm_mutex *); /* Return true if mutex is held */ - int (*xMutexNotHeld)(lsm_mutex *); /* Return true if mutex not held */ - /****** other ****************************************************/ - int (*xSleep)(lsm_env*, int microseconds); - - /* New fields may be added in future releases, in which case the - ** iVersion value will increase. */ -}; - -/* -** Values that may be passed as the second argument to xMutexStatic. -*/ -#define LSM_MUTEX_GLOBAL 1 -#define LSM_MUTEX_HEAP 2 - -/* -** CAPI: LSM Error Codes -*/ -#define LSM_OK 0 -#define LSM_ERROR 1 -#define LSM_BUSY 5 -#define LSM_NOMEM 7 -#define LSM_READONLY 8 -#define LSM_IOERR 10 -#define LSM_CORRUPT 11 -#define LSM_FULL 13 -#define LSM_CANTOPEN 14 -#define LSM_PROTOCOL 15 -#define LSM_MISUSE 21 - -#define LSM_MISMATCH 50 - - -#define LSM_IOERR_NOENT (LSM_IOERR | (1<<8)) - -/* -** CAPI: Creating and Destroying Database Connection Handles -** -** Open and close a database connection handle. -*/ -int lsm_new(lsm_env*, lsm_db **ppDb); -int lsm_close(lsm_db *pDb); - -/* -** CAPI: Connecting to a Database -*/ -int lsm_open(lsm_db *pDb, const char *zFilename); - -/* -** CAPI: Obtaining pointers to database environments -** -** Return a pointer to the environment used by the database connection -** passed as the first argument. Assuming the argument is valid, this -** function always returns a valid environment pointer - it cannot fail. -*/ -lsm_env *lsm_get_env(lsm_db *pDb); - -/* -** The lsm_default_env() function returns a pointer to the default LSM -** environment for the current platform. -*/ -lsm_env *lsm_default_env(void); - - -/* -** CAPI: Configuring a database connection. -** -** The lsm_config() function is used to configure a database connection. -*/ -int lsm_config(lsm_db *, int, ...); - -/* -** The following values may be passed as the second argument to lsm_config(). -** -** LSM_CONFIG_AUTOFLUSH: -** A read/write integer parameter. -** -** This value determines the amount of data allowed to accumulate in a -** live in-memory tree before it is marked as old. After committing a -** transaction, a connection checks if the size of the live in-memory tree, -** including data structure overhead, is greater than the value of this -** option in KB. If it is, and there is not already an old in-memory tree, -** the live in-memory tree is marked as old. -** -** The maximum allowable value is 1048576 (1GB). There is no minimum -** value. If this parameter is set to zero, then an attempt is made to -** mark the live in-memory tree as old after each transaction is committed. -** -** The default value is 1024 (1MB). -** -** LSM_CONFIG_PAGE_SIZE: -** A read/write integer parameter. This parameter may only be set before -** lsm_open() has been called. -** -** LSM_CONFIG_BLOCK_SIZE: -** A read/write integer parameter. -** -** This parameter may only be set before lsm_open() has been called. It -** must be set to a power of two between 64 and 65536, inclusive (block -** sizes between 64KB and 64MB). -** -** If the connection creates a new database, the block size of the new -** database is set to the value of this option in KB. After lsm_open() -** has been called, querying this parameter returns the actual block -** size of the opened database. -** -** The default value is 1024 (1MB blocks). -** -** LSM_CONFIG_SAFETY: -** A read/write integer parameter. Valid values are 0, 1 (the default) -** and 2. This parameter determines how robust the database is in the -** face of a system crash (e.g. a power failure or operating system -** crash). As follows: -** -** 0 (off): No robustness. A system crash may corrupt the database. -** -** 1 (normal): Some robustness. A system crash may not corrupt the -** database file, but recently committed transactions may -** be lost following recovery. -** -** 2 (full): Full robustness. A system crash may not corrupt the -** database file. Following recovery the database file -** contains all successfully committed transactions. -** -** LSM_CONFIG_AUTOWORK: -** A read/write integer parameter. -** -** LSM_CONFIG_AUTOCHECKPOINT: -** A read/write integer parameter. -** -** If this option is set to non-zero value N, then a checkpoint is -** automatically attempted after each N KB of data have been written to -** the database file. -** -** The amount of uncheckpointed data already written to the database file -** is a global parameter. After performing database work (writing to the -** database file), the process checks if the total amount of uncheckpointed -** data exceeds the value of this paramter. If so, a checkpoint is performed. -** This means that this option may cause the connection to perform a -** checkpoint even if the current connection has itself written very little -** data into the database file. -** -** The default value is 2048 (checkpoint every 2MB). -** -** LSM_CONFIG_MMAP: -** A read/write integer parameter. If this value is set to 0, then the -** database file is accessed using ordinary read/write IO functions. Or, -** if it is set to 1, then the database file is memory mapped and accessed -** that way. If this parameter is set to any value N greater than 1, then -** up to the first N KB of the file are memory mapped, and any remainder -** accessed using read/write IO. -** -** The default value is 1 on 64-bit platforms and 32768 on 32-bit platforms. -** -** -** LSM_CONFIG_USE_LOG: -** A read/write boolean parameter. True (the default) to use the log -** file normally. False otherwise. -** -** LSM_CONFIG_AUTOMERGE: -** A read/write integer parameter. The minimum number of segments to -** merge together at a time. Default value 4. -** -** LSM_CONFIG_MAX_FREELIST: -** A read/write integer parameter. The maximum number of free-list -** entries that are stored in a database checkpoint (the others are -** stored elsewhere in the database). -** -** There is no reason for an application to configure or query this -** parameter. It is only present because configuring a small value -** makes certain parts of the lsm code easier to test. -** -** LSM_CONFIG_MULTIPLE_PROCESSES: -** A read/write boolean parameter. This parameter may only be set before -** lsm_open() has been called. If true, the library uses shared-memory -** and posix advisory locks to co-ordinate access by clients from within -** multiple processes. Otherwise, if false, all database clients must be -** located in the same process. The default value is true. -** -** LSM_CONFIG_SET_COMPRESSION: -** Set the compression methods used to compress and decompress database -** content. The argument to this option should be a pointer to a structure -** of type lsm_compress. The lsm_config() method takes a copy of the -** structures contents. -** -** This option may only be used before lsm_open() is called. Invoking it -** after lsm_open() has been called results in an LSM_MISUSE error. -** -** LSM_CONFIG_GET_COMPRESSION: -** Query the compression methods used to compress and decompress database -** content. -** -** LSM_CONFIG_SET_COMPRESSION_FACTORY: -** Configure a factory method to be invoked in case of an LSM_MISMATCH -** error. -** -** LSM_CONFIG_READONLY: -** A read/write boolean parameter. This parameter may only be set before -** lsm_open() is called. -*/ -#define LSM_CONFIG_AUTOFLUSH 1 -#define LSM_CONFIG_PAGE_SIZE 2 -#define LSM_CONFIG_SAFETY 3 -#define LSM_CONFIG_BLOCK_SIZE 4 -#define LSM_CONFIG_AUTOWORK 5 -#define LSM_CONFIG_MMAP 7 -#define LSM_CONFIG_USE_LOG 8 -#define LSM_CONFIG_AUTOMERGE 9 -#define LSM_CONFIG_MAX_FREELIST 10 -#define LSM_CONFIG_MULTIPLE_PROCESSES 11 -#define LSM_CONFIG_AUTOCHECKPOINT 12 -#define LSM_CONFIG_SET_COMPRESSION 13 -#define LSM_CONFIG_GET_COMPRESSION 14 -#define LSM_CONFIG_SET_COMPRESSION_FACTORY 15 -#define LSM_CONFIG_READONLY 16 - -#define LSM_SAFETY_OFF 0 -#define LSM_SAFETY_NORMAL 1 -#define LSM_SAFETY_FULL 2 - -/* -** CAPI: Compression and/or Encryption Hooks -*/ -struct lsm_compress { - void *pCtx; - unsigned int iId; - int (*xBound)(void *, int nSrc); - int (*xCompress)(void *, char *, int *, const char *, int); - int (*xUncompress)(void *, char *, int *, const char *, int); - void (*xFree)(void *pCtx); -}; - -struct lsm_compress_factory { - void *pCtx; - int (*xFactory)(void *, lsm_db *, unsigned int); - void (*xFree)(void *pCtx); -}; - -#define LSM_COMPRESSION_EMPTY 0 -#define LSM_COMPRESSION_NONE 1 - -/* -** CAPI: Allocating and Freeing Memory -** -** Invoke the memory allocation functions that belong to environment -** pEnv. Or the system defaults if no memory allocation functions have -** been registered. -*/ -void *lsm_malloc(lsm_env*, size_t); -void *lsm_realloc(lsm_env*, void *, size_t); -void lsm_free(lsm_env*, void *); - -/* -** CAPI: Querying a Connection For Operational Data -** -** Query a database connection for operational statistics or data. -*/ -int lsm_info(lsm_db *, int, ...); - -int lsm_get_user_version(lsm_db *, unsigned int *); -int lsm_set_user_version(lsm_db *, unsigned int); - -/* -** The following values may be passed as the second argument to lsm_info(). -** -** LSM_INFO_NWRITE: -** The third parameter should be of type (int *). The location pointed -** to by the third parameter is set to the number of 4KB pages written to -** the database file during the lifetime of this connection. -** -** LSM_INFO_NREAD: -** The third parameter should be of type (int *). The location pointed -** to by the third parameter is set to the number of 4KB pages read from -** the database file during the lifetime of this connection. -** -** LSM_INFO_DB_STRUCTURE: -** The third argument should be of type (char **). The location pointed -** to is populated with a pointer to a nul-terminated string containing -** the string representation of a Tcl data-structure reflecting the -** current structure of the database file. Specifically, the current state -** of the worker snapshot. The returned string should be eventually freed -** by the caller using lsm_free(). -** -** The returned list contains one element for each level in the database, -** in order from most to least recent. Each element contains a -** single element for each segment comprising the corresponding level, -** starting with the lhs segment, then each of the rhs segments (if any) -** in order from most to least recent. -** -** Each segment element is itself a list of 4 integer values, as follows: -** -**

          1. First page of segment -**
          2. Last page of segment -**
          3. Root page of segment (if applicable) -**
          4. Total number of pages in segment -**
          -** -** LSM_INFO_ARRAY_STRUCTURE: -** There should be two arguments passed following this option (i.e. a -** total of four arguments passed to lsm_info()). The first argument -** should be the page number of the first page in a database array -** (perhaps obtained from an earlier INFO_DB_STRUCTURE call). The second -** trailing argument should be of type (char **). The location pointed -** to is populated with a pointer to a nul-terminated string that must -** be eventually freed using lsm_free() by the caller. -** -** The output string contains the text representation of a Tcl list of -** integers. Each pair of integers represent a range of pages used by -** the identified array. For example, if the array occupies database -** pages 993 to 1024, then pages 2048 to 2777, then the returned string -** will be "993 1024 2048 2777". -** -** If the specified integer argument does not correspond to the first -** page of any database array, LSM_ERROR is returned and the output -** pointer is set to a NULL value. -** -** LSM_INFO_LOG_STRUCTURE: -** The third argument should be of type (char **). The location pointed -** to is populated with a pointer to a nul-terminated string containing -** the string representation of a Tcl data-structure. The returned -** string should be eventually freed by the caller using lsm_free(). -** -** The Tcl structure returned is a list of six integers that describe -** the current structure of the log file. -** -** LSM_INFO_ARRAY_PAGES: -** -** LSM_INFO_PAGE_ASCII_DUMP: -** As with LSM_INFO_ARRAY_STRUCTURE, there should be two arguments passed -** with calls that specify this option - an integer page number and a -** (char **) used to return a nul-terminated string that must be later -** freed using lsm_free(). In this case the output string is populated -** with a human-readable description of the page content. -** -** If the page cannot be decoded, it is not an error. In this case the -** human-readable output message will report the systems failure to -** interpret the page data. -** -** LSM_INFO_PAGE_HEX_DUMP: -** This argument is similar to PAGE_ASCII_DUMP, except that keys and -** values are represented using hexadecimal notation instead of ascii. -** -** LSM_INFO_FREELIST: -** The third argument should be of type (char **). The location pointed -** to is populated with a pointer to a nul-terminated string containing -** the string representation of a Tcl data-structure. The returned -** string should be eventually freed by the caller using lsm_free(). -** -** The Tcl structure returned is a list containing one element for each -** free block in the database. The element itself consists of two -** integers - the block number and the id of the snapshot that freed it. -** -** LSM_INFO_CHECKPOINT_SIZE: -** The third argument should be of type (int *). The location pointed to -** by this argument is populated with the number of KB written to the -** database file since the most recent checkpoint. -** -** LSM_INFO_TREE_SIZE: -** If this value is passed as the second argument to an lsm_info() call, it -** should be followed by two arguments of type (int *) (for a total of four -** arguments). -** -** At any time, there are either one or two tree structures held in shared -** memory that new database clients will access (there may also be additional -** tree structures being used by older clients - this API does not provide -** information on them). One tree structure - the current tree - is used to -** accumulate new data written to the database. The other tree structure - -** the old tree - is a read-only tree holding older data and may be flushed -** to disk at any time. -** -** Assuming no error occurs, the location pointed to by the first of the two -** (int *) arguments is set to the size of the old in-memory tree in KB. -** The second is set to the size of the current, or live in-memory tree. -** -** LSM_INFO_COMPRESSION_ID: -** This value should be followed by a single argument of type -** (unsigned int *). If successful, the location pointed to is populated -** with the database compression id before returning. -*/ -#define LSM_INFO_NWRITE 1 -#define LSM_INFO_NREAD 2 -#define LSM_INFO_DB_STRUCTURE 3 -#define LSM_INFO_LOG_STRUCTURE 4 -#define LSM_INFO_ARRAY_STRUCTURE 5 -#define LSM_INFO_PAGE_ASCII_DUMP 6 -#define LSM_INFO_PAGE_HEX_DUMP 7 -#define LSM_INFO_FREELIST 8 -#define LSM_INFO_ARRAY_PAGES 9 -#define LSM_INFO_CHECKPOINT_SIZE 10 -#define LSM_INFO_TREE_SIZE 11 -#define LSM_INFO_FREELIST_SIZE 12 -#define LSM_INFO_COMPRESSION_ID 13 - - -/* -** CAPI: Opening and Closing Write Transactions -** -** These functions are used to open and close transactions and nested -** sub-transactions. -** -** The lsm_begin() function is used to open transactions and sub-transactions. -** A successful call to lsm_begin() ensures that there are at least iLevel -** nested transactions open. To open a top-level transaction, pass iLevel=1. -** To open a sub-transaction within the top-level transaction, iLevel=2. -** Passing iLevel=0 is a no-op. -** -** lsm_commit() is used to commit transactions and sub-transactions. A -** successful call to lsm_commit() ensures that there are at most iLevel -** nested transactions open. To commit a top-level transaction, pass iLevel=0. -** To commit all sub-transactions inside the main transaction, pass iLevel=1. -** -** Function lsm_rollback() is used to roll back transactions and -** sub-transactions. A successful call to lsm_rollback() restores the database -** to the state it was in when the iLevel'th nested sub-transaction (if any) -** was first opened. And then closes transactions to ensure that there are -** at most iLevel nested transactions open. Passing iLevel=0 rolls back and -** closes the top-level transaction. iLevel=1 also rolls back the top-level -** transaction, but leaves it open. iLevel=2 rolls back the sub-transaction -** nested directly inside the top-level transaction (and leaves it open). -*/ -int lsm_begin(lsm_db *pDb, int iLevel); -int lsm_commit(lsm_db *pDb, int iLevel); -int lsm_rollback(lsm_db *pDb, int iLevel); - -/* -** CAPI: Writing to a Database -** -** Write a new value into the database. If a value with a duplicate key -** already exists it is replaced. -*/ -int lsm_insert(lsm_db*, const void *pKey, int nKey, const void *pVal, int nVal); - -/* -** Delete a value from the database. No error is returned if the specified -** key value does not exist in the database. -*/ -int lsm_delete(lsm_db *, const void *pKey, int nKey); - -/* -** Delete all database entries with keys that are greater than (pKey1/nKey1) -** and smaller than (pKey2/nKey2). Note that keys (pKey1/nKey1) and -** (pKey2/nKey2) themselves, if they exist in the database, are not deleted. -** -** Return LSM_OK if successful, or an LSM error code otherwise. -*/ -int lsm_delete_range(lsm_db *, - const void *pKey1, int nKey1, const void *pKey2, int nKey2 -); - -/* -** CAPI: Explicit Database Work and Checkpointing -** -** This function is called by a thread to work on the database structure. -*/ -int lsm_work(lsm_db *pDb, int nMerge, int nKB, int *pnWrite); - -int lsm_flush(lsm_db *pDb); - -/* -** Attempt to checkpoint the current database snapshot. Return an LSM -** error code if an error occurs or LSM_OK otherwise. -** -** If the current snapshot has already been checkpointed, calling this -** function is a no-op. In this case if pnKB is not NULL, *pnKB is -** set to 0. Or, if the current snapshot is successfully checkpointed -** by this function and pbKB is not NULL, *pnKB is set to the number -** of bytes written to the database file since the previous checkpoint -** (the same measure as returned by the LSM_INFO_CHECKPOINT_SIZE query). -*/ -int lsm_checkpoint(lsm_db *pDb, int *pnKB); - -/* -** CAPI: Opening and Closing Database Cursors -** -** Open and close a database cursor. -*/ -int lsm_csr_open(lsm_db *pDb, lsm_cursor **ppCsr); -int lsm_csr_close(lsm_cursor *pCsr); - -/* -** CAPI: Positioning Database Cursors -** -** If the fourth parameter is LSM_SEEK_EQ, LSM_SEEK_GE or LSM_SEEK_LE, -** this function searches the database for an entry with key (pKey/nKey). -** If an error occurs, an LSM error code is returned. Otherwise, LSM_OK. -** -** If no error occurs and the requested key is present in the database, the -** cursor is left pointing to the entry with the specified key. Or, if the -** specified key is not present in the database the state of the cursor -** depends on the value passed as the final parameter, as follows: -** -** LSM_SEEK_EQ: -** The cursor is left at EOF (invalidated). A call to lsm_csr_valid() -** returns non-zero. -** -** LSM_SEEK_LE: -** The cursor is left pointing to the largest key in the database that -** is smaller than (pKey/nKey). If the database contains no keys smaller -** than (pKey/nKey), the cursor is left at EOF. -** -** LSM_SEEK_GE: -** The cursor is left pointing to the smallest key in the database that -** is larger than (pKey/nKey). If the database contains no keys larger -** than (pKey/nKey), the cursor is left at EOF. -** -** If the fourth parameter is LSM_SEEK_LEFAST, this function searches the -** database in a similar manner to LSM_SEEK_LE, with two differences: -** -**
          1. Even if a key can be found (the cursor is not left at EOF), the -** lsm_csr_value() function may not be used (attempts to do so return -** LSM_MISUSE). -** -**
          2. The key that the cursor is left pointing to may be one that has -** been recently deleted from the database. In this case it is -** guaranteed that the returned key is larger than any key currently -** in the database that is less than or equal to (pKey/nKey). -**
          -** -** LSM_SEEK_LEFAST requests are intended to be used to allocate database -** keys. -*/ -int lsm_csr_seek(lsm_cursor *pCsr, const void *pKey, int nKey, int eSeek); - -int lsm_csr_first(lsm_cursor *pCsr); -int lsm_csr_last(lsm_cursor *pCsr); - -/* -** Advance the specified cursor to the next or previous key in the database. -** Return LSM_OK if successful, or an LSM error code otherwise. -** -** Functions lsm_csr_seek(), lsm_csr_first() and lsm_csr_last() are "seek" -** functions. Whether or not lsm_csr_next and lsm_csr_prev may be called -** successfully also depends on the most recent seek function called on -** the cursor. Specifically: -** -**
            -**
          • At least one seek function must have been called on the cursor. -**
          • To call lsm_csr_next(), the most recent call to a seek function must -** have been either lsm_csr_first() or a call to lsm_csr_seek() specifying -** LSM_SEEK_GE. -**
          • To call lsm_csr_prev(), the most recent call to a seek function must -** have been either lsm_csr_last() or a call to lsm_csr_seek() specifying -** LSM_SEEK_LE. -**
          -** -** Otherwise, if the above conditions are not met when lsm_csr_next or -** lsm_csr_prev is called, LSM_MISUSE is returned and the cursor position -** remains unchanged. -*/ -int lsm_csr_next(lsm_cursor *pCsr); -int lsm_csr_prev(lsm_cursor *pCsr); - -/* -** Values that may be passed as the fourth argument to lsm_csr_seek(). -*/ -#define LSM_SEEK_LEFAST -2 -#define LSM_SEEK_LE -1 -#define LSM_SEEK_EQ 0 -#define LSM_SEEK_GE 1 - -/* -** CAPI: Extracting Data From Database Cursors -** -** Retrieve data from a database cursor. -*/ -int lsm_csr_valid(lsm_cursor *pCsr); -int lsm_csr_key(lsm_cursor *pCsr, const void **ppKey, int *pnKey); -int lsm_csr_value(lsm_cursor *pCsr, const void **ppVal, int *pnVal); - -/* -** If no error occurs, this function compares the database key passed via -** the pKey/nKey arguments with the key that the cursor passed as the first -** argument currently points to. If the cursors key is less than, equal to -** or greater than pKey/nKey, *piRes is set to less than, equal to or greater -** than zero before returning. LSM_OK is returned in this case. -** -** Or, if an error occurs, an LSM error code is returned and the final -** value of *piRes is undefined. If the cursor does not point to a valid -** key when this function is called, LSM_MISUSE is returned. -*/ -int lsm_csr_cmp(lsm_cursor *pCsr, const void *pKey, int nKey, int *piRes); - -/* -** CAPI: Change these!! -** -** Configure a callback to which debugging and other messages should -** be directed. Only useful for debugging lsm. -*/ -void lsm_config_log(lsm_db *, void (*)(void *, int, const char *), void *); - -/* -** Configure a callback that is invoked if the database connection ever -** writes to the database file. -*/ -void lsm_config_work_hook(lsm_db *, void (*)(lsm_db *, void *), void *); - -/* ENDOFAPI */ -#ifdef __cplusplus -} /* End of the 'extern "C"' block */ -#endif -#endif /* ifndef _LSM_H */ diff --git a/ext/lsm1/lsmInt.h b/ext/lsm1/lsmInt.h deleted file mode 100644 index 4e3c5e59c..000000000 --- a/ext/lsm1/lsmInt.h +++ /dev/null @@ -1,997 +0,0 @@ -/* -** 2011-08-18 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** Internal structure definitions for the LSM module. -*/ -#ifndef _LSM_INT_H -#define _LSM_INT_H - -#include "lsm.h" -#include -#include - -#include -#include -#include -#include - -#ifdef _WIN32 -# ifdef _MSC_VER -# define snprintf _snprintf -# endif -#else -# include -#endif - -#ifdef NDEBUG -# ifdef LSM_DEBUG_EXPENSIVE -# undef LSM_DEBUG_EXPENSIVE -# endif -# ifdef LSM_DEBUG -# undef LSM_DEBUG -# endif -#else -# ifndef LSM_DEBUG -# define LSM_DEBUG -# endif -#endif - -/* #define LSM_DEBUG_EXPENSIVE 1 */ - -/* -** Default values for various data structure parameters. These may be -** overridden by calls to lsm_config(). -*/ -#define LSM_DFLT_PAGE_SIZE (4 * 1024) -#define LSM_DFLT_BLOCK_SIZE (1 * 1024 * 1024) -#define LSM_DFLT_AUTOFLUSH (1 * 1024 * 1024) -#define LSM_DFLT_AUTOCHECKPOINT (i64)(2 * 1024 * 1024) -#define LSM_DFLT_AUTOWORK 1 -#define LSM_DFLT_LOG_SIZE (128*1024) -#define LSM_DFLT_AUTOMERGE 4 -#define LSM_DFLT_SAFETY LSM_SAFETY_NORMAL -#define LSM_DFLT_MMAP (LSM_IS_64_BIT ? 1 : 32768) -#define LSM_DFLT_MULTIPLE_PROCESSES 1 -#define LSM_DFLT_USE_LOG 1 - -/* Initial values for log file checksums. These are only used if the -** database file does not contain a valid checkpoint. */ -#define LSM_CKSUM0_INIT 42 -#define LSM_CKSUM1_INIT 42 - -/* "mmap" mode is currently only used in environments with 64-bit address -** spaces. The following macro is used to test for this. */ -#define LSM_IS_64_BIT (sizeof(void*)==8) - -#define LSM_AUTOWORK_QUANT 32 - -typedef struct Database Database; -typedef struct DbLog DbLog; -typedef struct FileSystem FileSystem; -typedef struct Freelist Freelist; -typedef struct FreelistEntry FreelistEntry; -typedef struct Level Level; -typedef struct LogMark LogMark; -typedef struct LogRegion LogRegion; -typedef struct LogWriter LogWriter; -typedef struct LsmString LsmString; -typedef struct Mempool Mempool; -typedef struct Merge Merge; -typedef struct MergeInput MergeInput; -typedef struct MetaPage MetaPage; -typedef struct MultiCursor MultiCursor; -typedef struct Page Page; -typedef struct Redirect Redirect; -typedef struct Segment Segment; -typedef struct SegmentMerger SegmentMerger; -typedef struct ShmChunk ShmChunk; -typedef struct ShmHeader ShmHeader; -typedef struct ShmReader ShmReader; -typedef struct Snapshot Snapshot; -typedef struct TransMark TransMark; -typedef struct Tree Tree; -typedef struct TreeCursor TreeCursor; -typedef struct TreeHeader TreeHeader; -typedef struct TreeMark TreeMark; -typedef struct TreeRoot TreeRoot; - -#ifndef _SQLITEINT_H_ -typedef unsigned char u8; -typedef unsigned short int u16; -typedef unsigned int u32; -typedef lsm_i64 i64; -typedef unsigned long long int u64; -#endif - -/* A page number is a 64-bit integer. */ -typedef i64 LsmPgno; - -#ifdef LSM_DEBUG -int lsmErrorBkpt(int); -#else -# define lsmErrorBkpt(x) (x) -#endif - -#define LSM_PROTOCOL_BKPT lsmErrorBkpt(LSM_PROTOCOL) -#define LSM_IOERR_BKPT lsmErrorBkpt(LSM_IOERR) -#define LSM_NOMEM_BKPT lsmErrorBkpt(LSM_NOMEM) -#define LSM_CORRUPT_BKPT lsmErrorBkpt(LSM_CORRUPT) -#define LSM_MISUSE_BKPT lsmErrorBkpt(LSM_MISUSE) - -#define unused_parameter(x) (void)(x) -#define array_size(x) (sizeof(x)/sizeof(x[0])) - - -/* The size of each shared-memory chunk */ -#define LSM_SHM_CHUNK_SIZE (32*1024) - -/* The number of bytes reserved at the start of each shm chunk for MM. */ -#define LSM_SHM_CHUNK_HDR (sizeof(ShmChunk)) - -/* The number of available read locks. */ -#define LSM_LOCK_NREADER 6 - -/* The number of available read-write client locks. */ -#define LSM_LOCK_NRWCLIENT 16 - -/* Lock definitions. -*/ -#define LSM_LOCK_DMS1 1 /* Serialize connect/disconnect ops */ -#define LSM_LOCK_DMS2 2 /* Read-write connections */ -#define LSM_LOCK_DMS3 3 /* Read-only connections */ -#define LSM_LOCK_WRITER 4 -#define LSM_LOCK_WORKER 5 -#define LSM_LOCK_CHECKPOINTER 6 -#define LSM_LOCK_ROTRANS 7 -#define LSM_LOCK_READER(i) ((i) + LSM_LOCK_ROTRANS + 1) -#define LSM_LOCK_RWCLIENT(i) ((i) + LSM_LOCK_READER(LSM_LOCK_NREADER)) - -#define LSM_N_LOCK LSM_LOCK_RWCLIENT(LSM_LOCK_NRWCLIENT) - -/* -** Meta-page size and usable size. -*/ -#define LSM_META_PAGE_SIZE 4096 - -#define LSM_META_RW_PAGE_SIZE (LSM_META_PAGE_SIZE - LSM_N_LOCK) - -/* -** Hard limit on the number of free-list entries that may be stored in -** a checkpoint (the remainder are stored as a system record in the LSM). -** See also LSM_CONFIG_MAX_FREELIST. -*/ -#define LSM_MAX_FREELIST_ENTRIES 24 - -#define LSM_MAX_BLOCK_REDIRECTS 16 - -#define LSM_ATTEMPTS_BEFORE_PROTOCOL 10000 - - -/* -** Each entry stored in the LSM (or in-memory tree structure) has an -** associated mask of the following flags. -*/ -#define LSM_START_DELETE 0x01 /* Start of open-ended delete range */ -#define LSM_END_DELETE 0x02 /* End of open-ended delete range */ -#define LSM_POINT_DELETE 0x04 /* Delete this key */ -#define LSM_INSERT 0x08 /* Insert this key and value */ -#define LSM_SEPARATOR 0x10 /* True if entry is separator key only */ -#define LSM_SYSTEMKEY 0x20 /* True if entry is a system key (FREELIST) */ - -#define LSM_CONTIGUOUS 0x40 /* Used in lsm_tree.c */ - -/* -** A string that can grow by appending. -*/ -struct LsmString { - lsm_env *pEnv; /* Run-time environment */ - int n; /* Size of string. -1 indicates error */ - int nAlloc; /* Space allocated for z[] */ - char *z; /* The string content */ -}; - -typedef struct LsmFile LsmFile; -struct LsmFile { - lsm_file *pFile; - LsmFile *pNext; -}; - -/* -** An instance of the following type is used to store an ordered list of -** u32 values. -** -** Note: This is a place-holder implementation. It should be replaced by -** a version that avoids making a single large allocation when the array -** contains a large number of values. For this reason, the internals of -** this object should only manipulated by the intArrayXXX() functions in -** lsm_tree.c. -*/ -typedef struct IntArray IntArray; -struct IntArray { - int nAlloc; - int nArray; - u32 *aArray; -}; - -struct Redirect { - int n; /* Number of redirects */ - struct RedirectEntry { - int iFrom; - int iTo; - } *a; -}; - -/* -** An instance of this structure represents a point in the history of the -** tree structure to roll back to. Refer to comments in lsm_tree.c for -** details. -*/ -struct TreeMark { - u32 iRoot; /* Offset of root node in shm file */ - u32 nHeight; /* Current height of tree structure */ - u32 iWrite; /* Write offset in shm file */ - u32 nChunk; /* Number of chunks in shared-memory file */ - u32 iFirst; /* First chunk in linked list */ - u32 iNextShmid; /* Next id to allocate */ - int iRollback; /* Index in lsm->rollback to revert to */ -}; - -/* -** An instance of this structure represents a point in the database log. -*/ -struct LogMark { - i64 iOff; /* Offset into log (see lsm_log.c) */ - int nBuf; /* Size of in-memory buffer here */ - u8 aBuf[8]; /* Bytes of content in aBuf[] */ - u32 cksum0; /* Checksum 0 at offset (iOff-nBuf) */ - u32 cksum1; /* Checksum 1 at offset (iOff-nBuf) */ -}; - -struct TransMark { - TreeMark tree; - LogMark log; -}; - -/* -** A structure that defines the start and end offsets of a region in the -** log file. The size of the region in bytes is (iEnd - iStart), so if -** iEnd==iStart the region is zero bytes in size. -*/ -struct LogRegion { - i64 iStart; /* Start of region in log file */ - i64 iEnd; /* End of region in log file */ -}; - -struct DbLog { - u32 cksum0; /* Checksum 0 at offset iOff */ - u32 cksum1; /* Checksum 1 at offset iOff */ - i64 iSnapshotId; /* Log space has been reclaimed to this ss */ - LogRegion aRegion[3]; /* Log file regions (see docs in lsm_log.c) */ -}; - -struct TreeRoot { - u32 iRoot; - u32 nHeight; - u32 nByte; /* Total size of this tree in bytes */ - u32 iTransId; -}; - -/* -** Tree header structure. -*/ -struct TreeHeader { - u32 iUsedShmid; /* Id of first shm chunk used by this tree */ - u32 iNextShmid; /* Shm-id of next chunk allocated */ - u32 iFirst; /* Chunk number of smallest shm-id */ - u32 nChunk; /* Number of chunks in shared-memory file */ - TreeRoot root; /* Root and height of current tree */ - u32 iWrite; /* Write offset in shm file */ - TreeRoot oldroot; /* Root and height of the previous tree */ - u32 iOldShmid; /* Last shm-id used by previous tree */ - u32 iUsrVersion; /* get/set_user_version() value */ - i64 iOldLog; /* Log offset associated with old tree */ - u32 oldcksum0; - u32 oldcksum1; - DbLog log; /* Current layout of log file */ - u32 aCksum[2]; /* Checksums 1 and 2. */ -}; - -/* -** Database handle structure. -** -** mLock: -** A bitmask representing the locks currently held by the connection. -** An LSM database supports N distinct locks, where N is some number less -** than or equal to 32. Locks are numbered starting from 1 (see the -** definitions for LSM_LOCK_WRITER and co.). -** -** The least significant 32-bits in mLock represent EXCLUSIVE locks. The -** most significant are SHARED locks. So, if a connection holds a SHARED -** lock on lock region iLock, then the following is true: -** -** (mLock & ((iLock+32-1) << 1)) -** -** Or for an EXCLUSIVE lock: -** -** (mLock & ((iLock-1) << 1)) -** -** pCsr: -** Points to the head of a linked list that contains all currently open -** cursors. Once this list becomes empty, the user has no outstanding -** cursors and the database handle can be successfully closed. -** -** pCsrCache: -** This list contains cursor objects that have been closed using -** lsm_csr_close(). Each time a cursor is closed, it is shifted from -** the pCsr list to this list. When a new cursor is opened, this list -** is inspected to see if there exists a cursor object that can be -** reused. This is an optimization only. -*/ -struct lsm_db { - - /* Database handle configuration */ - lsm_env *pEnv; /* runtime environment */ - int (*xCmp)(void *, int, void *, int); /* Compare function */ - - /* Values configured by calls to lsm_config */ - int eSafety; /* LSM_SAFETY_OFF, NORMAL or FULL */ - int bAutowork; /* Configured by LSM_CONFIG_AUTOWORK */ - int nTreeLimit; /* Configured by LSM_CONFIG_AUTOFLUSH */ - int nMerge; /* Configured by LSM_CONFIG_AUTOMERGE */ - int bUseLog; /* Configured by LSM_CONFIG_USE_LOG */ - int nDfltPgsz; /* Configured by LSM_CONFIG_PAGE_SIZE */ - int nDfltBlksz; /* Configured by LSM_CONFIG_BLOCK_SIZE */ - int nMaxFreelist; /* Configured by LSM_CONFIG_MAX_FREELIST */ - int iMmap; /* Configured by LSM_CONFIG_MMAP */ - i64 nAutockpt; /* Configured by LSM_CONFIG_AUTOCHECKPOINT */ - int bMultiProc; /* Configured by L_C_MULTIPLE_PROCESSES */ - int bReadonly; /* Configured by LSM_CONFIG_READONLY */ - lsm_compress compress; /* Compression callbacks */ - lsm_compress_factory factory; /* Compression callback factory */ - - /* Sub-system handles */ - FileSystem *pFS; /* On-disk portion of database */ - Database *pDatabase; /* Database shared data */ - - int iRwclient; /* Read-write client lock held (-1 == none) */ - - /* Client transaction context */ - Snapshot *pClient; /* Client snapshot */ - int iReader; /* Read lock held (-1 == unlocked) */ - int bRoTrans; /* True if a read-only db trans is open */ - MultiCursor *pCsr; /* List of all open cursors */ - LogWriter *pLogWriter; /* Context for writing to the log file */ - int nTransOpen; /* Number of opened write transactions */ - int nTransAlloc; /* Allocated size of aTrans[] array */ - TransMark *aTrans; /* Array of marks for transaction rollback */ - IntArray rollback; /* List of tree-nodes to roll back */ - int bDiscardOld; /* True if lsmTreeDiscardOld() was called */ - - MultiCursor *pCsrCache; /* List of all closed cursors */ - - /* Worker context */ - Snapshot *pWorker; /* Worker snapshot (or NULL) */ - Freelist *pFreelist; /* See sortedNewToplevel() */ - int bUseFreelist; /* True to use pFreelist */ - int bIncrMerge; /* True if currently doing a merge */ - - int bInFactory; /* True if within factory.xFactory() */ - - /* Debugging message callback */ - void (*xLog)(void *, int, const char *); - void *pLogCtx; - - /* Work done notification callback */ - void (*xWork)(lsm_db *, void *); - void *pWorkCtx; - - u64 mLock; /* Mask of current locks. See lsmShmLock(). */ - lsm_db *pNext; /* Next connection to same database */ - - int nShm; /* Size of apShm[] array */ - void **apShm; /* Shared memory chunks */ - ShmHeader *pShmhdr; /* Live shared-memory header */ - TreeHeader treehdr; /* Local copy of tree-header */ - u32 aSnapshot[LSM_META_PAGE_SIZE / sizeof(u32)]; -}; - -struct Segment { - LsmPgno iFirst; /* First page of this run */ - LsmPgno iLastPg; /* Last page of this run */ - LsmPgno iRoot; /* Root page number (if any) */ - LsmPgno nSize; /* Size of this run in pages */ - - Redirect *pRedirect; /* Block redirects (or NULL) */ -}; - -/* -** iSplitTopic/pSplitKey/nSplitKey: -** If nRight>0, this buffer contains a copy of the largest key that has -** already been written to the left-hand-side of the level. -*/ -struct Level { - Segment lhs; /* Left-hand (main) segment */ - int nRight; /* Size of apRight[] array */ - Segment *aRhs; /* Old segments being merged into this */ - int iSplitTopic; /* Split key topic (if nRight>0) */ - void *pSplitKey; /* Pointer to split-key (if nRight>0) */ - int nSplitKey; /* Number of bytes in split-key */ - - u16 iAge; /* Number of times data has been written */ - u16 flags; /* Mask of LEVEL_XXX bits */ - Merge *pMerge; /* Merge operation currently underway */ - Level *pNext; /* Next level in tree */ -}; - -/* -** The Level.flags field is set to a combination of the following bits. -** -** LEVEL_FREELIST_ONLY: -** Set if the level consists entirely of free-list entries. -** -** LEVEL_INCOMPLETE: -** This is set while a new toplevel level is being constructed. It is -** never set for any level other than a new toplevel. -*/ -#define LEVEL_FREELIST_ONLY 0x0001 -#define LEVEL_INCOMPLETE 0x0002 - - -/* -** A structure describing an ongoing merge. There is an instance of this -** structure for every Level currently undergoing a merge in the worker -** snapshot. -** -** It is assumed that code that uses an instance of this structure has -** access to the associated Level struct. -** -** iOutputOff: -** The byte offset to write to next within the last page of the -** output segment. -*/ -struct MergeInput { - LsmPgno iPg; /* Page on which next input is stored */ - int iCell; /* Cell containing next input to merge */ -}; -struct Merge { - int nInput; /* Number of input runs being merged */ - MergeInput *aInput; /* Array nInput entries in size */ - MergeInput splitkey; /* Location in file of current splitkey */ - int nSkip; /* Number of separators entries to skip */ - int iOutputOff; /* Write offset on output page */ - LsmPgno iCurrentPtr; /* Current pointer value */ -}; - -/* -** The first argument to this macro is a pointer to a Segment structure. -** Returns true if the structure instance indicates that the separators -** array is valid. -*/ -#define segmentHasSeparators(pSegment) ((pSegment)->sep.iFirst>0) - -/* -** The values that accompany the lock held by a database reader. -*/ -struct ShmReader { - u32 iTreeId; - i64 iLsmId; -}; - -/* -** An instance of this structure is stored in the first shared-memory -** page. The shared-memory header. -** -** bWriter: -** Immediately after opening a write transaction taking the WRITER lock, -** each writer client sets this flag. It is cleared right before the -** WRITER lock is relinquished. If a subsequent writer finds that this -** flag is already set when a write transaction is opened, this indicates -** that a previous writer failed mid-transaction. -** -** iMetaPage: -** If the database file does not contain a valid, synced, checkpoint, this -** value is set to 0. Otherwise, it is set to the meta-page number that -** contains the most recently written checkpoint (either 1 or 2). -** -** hdr1, hdr2: -** The two copies of the in-memory tree header. Two copies are required -** in case a writer fails while updating one of them. -*/ -struct ShmHeader { - u32 aSnap1[LSM_META_PAGE_SIZE / 4]; - u32 aSnap2[LSM_META_PAGE_SIZE / 4]; - u32 bWriter; - u32 iMetaPage; - TreeHeader hdr1; - TreeHeader hdr2; - ShmReader aReader[LSM_LOCK_NREADER]; -}; - -/* -** An instance of this structure is stored at the start of each shared-memory -** chunk except the first (which is the header chunk - see above). -*/ -struct ShmChunk { - u32 iShmid; - u32 iNext; -}; - -/* -** Maximum number of shared-memory chunks allowed in the *-shm file. Since -** each shared-memory chunk is 32KB in size, this is a theoretical limit only. -*/ -#define LSM_MAX_SHMCHUNKS (1<<30) - -/* Return true if shm-sequence "a" is larger than or equal to "b" */ -#define shm_sequence_ge(a, b) (((u32)a-(u32)b) < LSM_MAX_SHMCHUNKS) - -#define LSM_APPLIST_SZ 4 - -/* -** An instance of the following structure stores the in-memory part of -** the current free block list. This structure is to the free block list -** as the in-memory tree is to the users database content. The contents -** of the free block list is found by merging the in-memory components -** with those stored in the LSM, just as the contents of the database is -** found by merging the in-memory tree with the user data entries in the -** LSM. -** -** Each FreelistEntry structure in the array represents either an insert -** or delete operation on the free-list. For deletes, the FreelistEntry.iId -** field is set to -1. For inserts, it is set to zero or greater. -** -** The array of FreelistEntry structures is always sorted in order of -** block number (ascending). -** -** When the in-memory free block list is written into the LSM, each insert -** operation is written separately. The entry key is the bitwise inverse -** of the block number as a 32-bit big-endian integer. This is done so that -** the entries in the LSM are sorted in descending order of block id. -** The associated value is the snapshot id, formated as a varint. -*/ -struct Freelist { - FreelistEntry *aEntry; /* Free list entries */ - int nEntry; /* Number of valid slots in aEntry[] */ - int nAlloc; /* Allocated size of aEntry[] */ -}; -struct FreelistEntry { - u32 iBlk; /* Block number */ - i64 iId; /* Largest snapshot id to use this block */ -}; - -/* -** A snapshot of a database. A snapshot contains all the information required -** to read or write a database file on disk. See the description of struct -** Database below for further details. -*/ -struct Snapshot { - Database *pDatabase; /* Database this snapshot belongs to */ - u32 iCmpId; /* Id of compression scheme */ - Level *pLevel; /* Pointer to level 0 of snapshot (or NULL) */ - i64 iId; /* Snapshot id */ - i64 iLogOff; /* Log file offset */ - Redirect redirect; /* Block redirection array */ - - /* Used by worker snapshots only */ - int nBlock; /* Number of blocks in database file */ - LsmPgno aiAppend[LSM_APPLIST_SZ]; /* Append point list */ - Freelist freelist; /* Free block list */ - u32 nWrite; /* Total number of pages written to disk */ -}; -#define LSM_INITIAL_SNAPSHOT_ID 11 - -/* -** Functions from file "lsm_ckpt.c". -*/ -int lsmCheckpointWrite(lsm_db *, u32 *); -int lsmCheckpointLevels(lsm_db *, int, void **, int *); -int lsmCheckpointLoadLevels(lsm_db *pDb, void *pVal, int nVal); - -int lsmCheckpointRecover(lsm_db *); -int lsmCheckpointDeserialize(lsm_db *, int, u32 *, Snapshot **); - -int lsmCheckpointLoadWorker(lsm_db *pDb); -int lsmCheckpointStore(lsm_db *pDb, int); - -int lsmCheckpointLoad(lsm_db *pDb, int *); -int lsmCheckpointLoadOk(lsm_db *pDb, int); -int lsmCheckpointClientCacheOk(lsm_db *); - -u32 lsmCheckpointNBlock(u32 *); -i64 lsmCheckpointId(u32 *, int); -u32 lsmCheckpointNWrite(u32 *, int); -i64 lsmCheckpointLogOffset(u32 *); -int lsmCheckpointPgsz(u32 *); -int lsmCheckpointBlksz(u32 *); -void lsmCheckpointLogoffset(u32 *aCkpt, DbLog *pLog); -void lsmCheckpointZeroLogoffset(lsm_db *); - -int lsmCheckpointSaveWorker(lsm_db *pDb, int); -int lsmDatabaseFull(lsm_db *pDb); -int lsmCheckpointSynced(lsm_db *pDb, i64 *piId, i64 *piLog, u32 *pnWrite); - -int lsmCheckpointSize(lsm_db *db, int *pnByte); - -int lsmInfoCompressionId(lsm_db *db, u32 *piCmpId); - -/* -** Functions from file "lsm_tree.c". -*/ -int lsmTreeNew(lsm_env *, int (*)(void *, int, void *, int), Tree **ppTree); -void lsmTreeRelease(lsm_env *, Tree *); -int lsmTreeInit(lsm_db *); -int lsmTreeRepair(lsm_db *); - -void lsmTreeMakeOld(lsm_db *pDb); -void lsmTreeDiscardOld(lsm_db *pDb); -int lsmTreeHasOld(lsm_db *pDb); - -int lsmTreeSize(lsm_db *); -int lsmTreeEndTransaction(lsm_db *pDb, int bCommit); -int lsmTreeLoadHeader(lsm_db *pDb, int *); -int lsmTreeLoadHeaderOk(lsm_db *, int); - -int lsmTreeInsert(lsm_db *pDb, void *pKey, int nKey, void *pVal, int nVal); -int lsmTreeDelete(lsm_db *db, void *pKey1, int nKey1, void *pKey2, int nKey2); -void lsmTreeRollback(lsm_db *pDb, TreeMark *pMark); -void lsmTreeMark(lsm_db *pDb, TreeMark *pMark); - -int lsmTreeCursorNew(lsm_db *pDb, int, TreeCursor **); -void lsmTreeCursorDestroy(TreeCursor *); - -int lsmTreeCursorSeek(TreeCursor *pCsr, void *pKey, int nKey, int *pRes); -int lsmTreeCursorNext(TreeCursor *pCsr); -int lsmTreeCursorPrev(TreeCursor *pCsr); -int lsmTreeCursorEnd(TreeCursor *pCsr, int bLast); -void lsmTreeCursorReset(TreeCursor *pCsr); -int lsmTreeCursorKey(TreeCursor *pCsr, int *pFlags, void **ppKey, int *pnKey); -int lsmTreeCursorFlags(TreeCursor *pCsr); -int lsmTreeCursorValue(TreeCursor *pCsr, void **ppVal, int *pnVal); -int lsmTreeCursorValid(TreeCursor *pCsr); -int lsmTreeCursorSave(TreeCursor *pCsr); - -void lsmFlagsToString(int flags, char *zFlags); - -/* -** Functions from file "mem.c". -*/ -void *lsmMalloc(lsm_env*, size_t); -void lsmFree(lsm_env*, void *); -void *lsmRealloc(lsm_env*, void *, size_t); -void *lsmReallocOrFree(lsm_env*, void *, size_t); -void *lsmReallocOrFreeRc(lsm_env *, void *, size_t, int *); - -void *lsmMallocZeroRc(lsm_env*, size_t, int *); -void *lsmMallocRc(lsm_env*, size_t, int *); - -void *lsmMallocZero(lsm_env *pEnv, size_t); -char *lsmMallocStrdup(lsm_env *pEnv, const char *); - -/* -** Functions from file "lsm_mutex.c". -*/ -int lsmMutexStatic(lsm_env*, int, lsm_mutex **); -int lsmMutexNew(lsm_env*, lsm_mutex **); -void lsmMutexDel(lsm_env*, lsm_mutex *); -void lsmMutexEnter(lsm_env*, lsm_mutex *); -int lsmMutexTry(lsm_env*, lsm_mutex *); -void lsmMutexLeave(lsm_env*, lsm_mutex *); - -#ifndef NDEBUG -int lsmMutexHeld(lsm_env *, lsm_mutex *); -int lsmMutexNotHeld(lsm_env *, lsm_mutex *); -#endif - -/************************************************************************** -** Start of functions from "lsm_file.c". -*/ -int lsmFsOpen(lsm_db *, const char *, int); -int lsmFsOpenLog(lsm_db *, int *); -void lsmFsCloseLog(lsm_db *); -void lsmFsClose(FileSystem *); - -int lsmFsUnmap(FileSystem *); - -int lsmFsConfigure(lsm_db *db); - -int lsmFsBlockSize(FileSystem *); -void lsmFsSetBlockSize(FileSystem *, int); -int lsmFsMoveBlock(FileSystem *pFS, Segment *pSeg, int iTo, int iFrom); - -int lsmFsPageSize(FileSystem *); -void lsmFsSetPageSize(FileSystem *, int); - -int lsmFsFileid(lsm_db *pDb, void **ppId, int *pnId); - -/* Creating, populating, gobbling and deleting sorted runs. */ -void lsmFsGobble(lsm_db *, Segment *, LsmPgno *, int); -int lsmFsSortedDelete(FileSystem *, Snapshot *, int, Segment *); -int lsmFsSortedFinish(FileSystem *, Segment *); -int lsmFsSortedAppend(FileSystem *, Snapshot *, Level *, int, Page **); -int lsmFsSortedPadding(FileSystem *, Snapshot *, Segment *); - -/* Functions to retrieve the lsm_env pointer from a FileSystem or Page object */ -lsm_env *lsmFsEnv(FileSystem *); -lsm_env *lsmPageEnv(Page *); -FileSystem *lsmPageFS(Page *); - -int lsmFsSectorSize(FileSystem *); - -void lsmSortedSplitkey(lsm_db *, Level *, int *); - -/* Reading sorted run content. */ -int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg); -int lsmFsDbPageGet(FileSystem *, Segment *, LsmPgno, Page **); -int lsmFsDbPageNext(Segment *, Page *, int eDir, Page **); - -u8 *lsmFsPageData(Page *, int *); -int lsmFsPageRelease(Page *); -int lsmFsPagePersist(Page *); -void lsmFsPageRef(Page *); -LsmPgno lsmFsPageNumber(Page *); - -int lsmFsNRead(FileSystem *); -int lsmFsNWrite(FileSystem *); - -int lsmFsMetaPageGet(FileSystem *, int, int, MetaPage **); -int lsmFsMetaPageRelease(MetaPage *); -u8 *lsmFsMetaPageData(MetaPage *, int *); - -#ifdef LSM_DEBUG -int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg); -int lsmFsIntegrityCheck(lsm_db *); -#endif - -LsmPgno lsmFsRedirectPage(FileSystem *, Redirect *, LsmPgno); - -int lsmFsPageWritable(Page *); - -/* Functions to read, write and sync the log file. */ -int lsmFsWriteLog(FileSystem *pFS, i64 iOff, LsmString *pStr); -int lsmFsSyncLog(FileSystem *pFS); -int lsmFsReadLog(FileSystem *pFS, i64 iOff, int nRead, LsmString *pStr); -int lsmFsTruncateLog(FileSystem *pFS, i64 nByte); -int lsmFsTruncateDb(FileSystem *pFS, i64 nByte); -int lsmFsCloseAndDeleteLog(FileSystem *pFS); - -LsmFile *lsmFsDeferClose(FileSystem *pFS); - -/* And to sync the db file */ -int lsmFsSyncDb(FileSystem *, int); - -void lsmFsFlushWaiting(FileSystem *, int *); - -/* Used by lsm_info(ARRAY_STRUCTURE) and lsm_config(MMAP) */ -int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, LsmPgno iFirst, char **pz); -int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut); -int lsmConfigMmap(lsm_db *pDb, int *piParam); - -int lsmEnvOpen(lsm_env *, const char *, int, lsm_file **); -int lsmEnvClose(lsm_env *pEnv, lsm_file *pFile); -int lsmEnvLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int eLock); -int lsmEnvTestLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int nLock, int); - -int lsmEnvShmMap(lsm_env *, lsm_file *, int, int, void **); -void lsmEnvShmBarrier(lsm_env *); -void lsmEnvShmUnmap(lsm_env *, lsm_file *, int); - -void lsmEnvSleep(lsm_env *, int); - -int lsmFsReadSyncedId(lsm_db *db, int, i64 *piVal); - -int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, LsmPgno, int *); - -void lsmFsPurgeCache(FileSystem *); - -/* -** End of functions from "lsm_file.c". -**************************************************************************/ - -/* -** Functions from file "lsm_sorted.c". -*/ -int lsmInfoPageDump(lsm_db *, LsmPgno, int, char **); -void lsmSortedCleanup(lsm_db *); -int lsmSortedAutoWork(lsm_db *, int nUnit); - -int lsmSortedWalkFreelist(lsm_db *, int, int (*)(void *, int, i64), void *); - -int lsmSaveWorker(lsm_db *, int); - -int lsmFlushTreeToDisk(lsm_db *pDb); - -void lsmSortedRemap(lsm_db *pDb); - -void lsmSortedFreeLevel(lsm_env *pEnv, Level *); - -int lsmSortedAdvanceAll(lsm_db *pDb); - -int lsmSortedLoadMerge(lsm_db *, Level *, u32 *, int *); -int lsmSortedLoadFreelist(lsm_db *pDb, void **, int *); - -void *lsmSortedSplitKey(Level *pLevel, int *pnByte); - -void lsmSortedSaveTreeCursors(lsm_db *); - -int lsmMCursorNew(lsm_db *, MultiCursor **); -void lsmMCursorClose(MultiCursor *, int); -int lsmMCursorSeek(MultiCursor *, int, void *, int , int); -int lsmMCursorFirst(MultiCursor *); -int lsmMCursorPrev(MultiCursor *); -int lsmMCursorLast(MultiCursor *); -int lsmMCursorValid(MultiCursor *); -int lsmMCursorNext(MultiCursor *); -int lsmMCursorKey(MultiCursor *, void **, int *); -int lsmMCursorValue(MultiCursor *, void **, int *); -int lsmMCursorType(MultiCursor *, int *); -lsm_db *lsmMCursorDb(MultiCursor *); -void lsmMCursorFreeCache(lsm_db *); - -int lsmSaveCursors(lsm_db *pDb); -int lsmRestoreCursors(lsm_db *pDb); - -void lsmSortedDumpStructure(lsm_db *pDb, Snapshot *, int, int, const char *); -void lsmFsDumpBlocklists(lsm_db *); - -void lsmSortedExpandBtreePage(Page *pPg, int nOrig); - -void lsmPutU32(u8 *, u32); -u32 lsmGetU32(u8 *); -u64 lsmGetU64(u8 *); - -/* -** Functions from "lsm_varint.c". -*/ -int lsmVarintPut32(u8 *, int); -int lsmVarintGet32(u8 *, int *); -int lsmVarintPut64(u8 *aData, i64 iVal); -int lsmVarintGet64(const u8 *aData, i64 *piVal); - -int lsmVarintLen64(i64); - -int lsmVarintLen32(int); -int lsmVarintSize(u8 c); - -/* -** Functions from file "main.c". -*/ -void lsmLogMessage(lsm_db *, int, const char *, ...); -int lsmInfoFreelist(lsm_db *pDb, char **pzOut); - -/* -** Functions from file "lsm_log.c". -*/ -int lsmLogBegin(lsm_db *pDb); -int lsmLogWrite(lsm_db *, int, void *, int, void *, int); -int lsmLogCommit(lsm_db *); -void lsmLogEnd(lsm_db *pDb, int bCommit); -void lsmLogTell(lsm_db *, LogMark *); -void lsmLogSeek(lsm_db *, LogMark *); -void lsmLogClose(lsm_db *); - -int lsmLogRecover(lsm_db *); -int lsmInfoLogStructure(lsm_db *pDb, char **pzVal); - -/* Valid values for the second argument to lsmLogWrite(). */ -#define LSM_WRITE 0x06 -#define LSM_DELETE 0x08 -#define LSM_DRANGE 0x0A - -/************************************************************************** -** Functions from file "lsm_shared.c". -*/ - -int lsmDbDatabaseConnect(lsm_db*, const char *); -void lsmDbDatabaseRelease(lsm_db *); - -int lsmBeginReadTrans(lsm_db *); -int lsmBeginWriteTrans(lsm_db *); -int lsmBeginFlush(lsm_db *); - -int lsmDetectRoTrans(lsm_db *db, int *); -int lsmBeginRoTrans(lsm_db *db); - -int lsmBeginWork(lsm_db *); -void lsmFinishWork(lsm_db *, int, int *); - -int lsmFinishRecovery(lsm_db *); -void lsmFinishReadTrans(lsm_db *); -int lsmFinishWriteTrans(lsm_db *, int); -int lsmFinishFlush(lsm_db *, int); - -int lsmSnapshotSetFreelist(lsm_db *, int *, int); - -Snapshot *lsmDbSnapshotClient(lsm_db *); -Snapshot *lsmDbSnapshotWorker(lsm_db *); - -void lsmSnapshotSetCkptid(Snapshot *, i64); - -Level *lsmDbSnapshotLevel(Snapshot *); -void lsmDbSnapshotSetLevel(Snapshot *, Level *); - -void lsmDbRecoveryComplete(lsm_db *, int); - -int lsmBlockAllocate(lsm_db *, int, int *); -int lsmBlockFree(lsm_db *, int); -int lsmBlockRefree(lsm_db *, int); - -void lsmFreelistDeltaBegin(lsm_db *); -void lsmFreelistDeltaEnd(lsm_db *); -int lsmFreelistDelta(lsm_db *pDb); - -DbLog *lsmDatabaseLog(lsm_db *pDb); - -#ifdef LSM_DEBUG - int lsmHoldingClientMutex(lsm_db *pDb); - int lsmShmAssertLock(lsm_db *db, int iLock, int eOp); - int lsmShmAssertWorker(lsm_db *db); -#endif - -void lsmFreeSnapshot(lsm_env *, Snapshot *); - - -/* Candidate values for the 3rd argument to lsmShmLock() */ -#define LSM_LOCK_UNLOCK 0 -#define LSM_LOCK_SHARED 1 -#define LSM_LOCK_EXCL 2 - -int lsmShmCacheChunks(lsm_db *db, int nChunk); -int lsmShmLock(lsm_db *db, int iLock, int eOp, int bBlock); -int lsmShmTestLock(lsm_db *db, int iLock, int nLock, int eOp); -void lsmShmBarrier(lsm_db *db); - -#ifdef LSM_DEBUG -void lsmShmHasLock(lsm_db *db, int iLock, int eOp); -#else -# define lsmShmHasLock(x,y,z) -#endif - -int lsmReadlock(lsm_db *, i64 iLsm, u32 iShmMin, u32 iShmMax); - -int lsmLsmInUse(lsm_db *db, i64 iLsmId, int *pbInUse); -int lsmTreeInUse(lsm_db *db, u32 iLsmId, int *pbInUse); -int lsmFreelistAppend(lsm_env *pEnv, Freelist *p, int iBlk, i64 iId); - -int lsmDbMultiProc(lsm_db *); -void lsmDbDeferredClose(lsm_db *, lsm_file *, LsmFile *); -LsmFile *lsmDbRecycleFd(lsm_db *); - -int lsmWalkFreelist(lsm_db *, int, int (*)(void *, int, i64), void *); - -int lsmCheckCompressionId(lsm_db *, u32); - - -/************************************************************************** -** functions in lsm_str.c -*/ -void lsmStringInit(LsmString*, lsm_env *pEnv); -int lsmStringExtend(LsmString*, int); -int lsmStringAppend(LsmString*, const char *, int); -void lsmStringVAppendf(LsmString*, const char *zFormat, va_list, va_list); -void lsmStringAppendf(LsmString*, const char *zFormat, ...); -void lsmStringClear(LsmString*); -char *lsmMallocPrintf(lsm_env*, const char*, ...); -int lsmStringBinAppend(LsmString *pStr, const u8 *a, int n); - -int lsmStrlen(const char *zName); - - - -/* -** Round up a number to the next larger multiple of 8. This is used -** to force 8-byte alignment on 64-bit architectures. -*/ -#define ROUND8(x) (((x)+7)&~7) - -#define LSM_MIN(x,y) ((x)>(y) ? (y) : (x)) -#define LSM_MAX(x,y) ((x)>(y) ? (x) : (y)) - -#endif diff --git a/ext/lsm1/lsm_ckpt.c b/ext/lsm1/lsm_ckpt.c deleted file mode 100644 index dbfa1a61f..000000000 --- a/ext/lsm1/lsm_ckpt.c +++ /dev/null @@ -1,1239 +0,0 @@ -/* -** 2011-09-11 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file contains code to read and write checkpoints. -** -** A checkpoint represents the database layout at a single point in time. -** It includes a log offset. When an existing database is opened, the -** current state is determined by reading the newest checkpoint and updating -** it with all committed transactions from the log that follow the specified -** offset. -*/ -#include "lsmInt.h" - -/* -** CHECKPOINT BLOB FORMAT: -** -** A checkpoint blob is a series of unsigned 32-bit integers stored in -** big-endian byte order. As follows: -** -** Checkpoint header (see the CKPT_HDR_XXX #defines): -** -** 1. The checkpoint id MSW. -** 2. The checkpoint id LSW. -** 3. The number of integer values in the entire checkpoint, including -** the two checksum values. -** 4. The compression scheme id. -** 5. The total number of blocks in the database. -** 6. The block size. -** 7. The number of levels. -** 8. The nominal database page size. -** 9. The number of pages (in total) written to the database file. -** -** Log pointer: -** -** 1. The log offset MSW. -** 2. The log offset LSW. -** 3. Log checksum 0. -** 4. Log checksum 1. -** -** Note that the "log offset" is not the literal byte offset. Instead, -** it is the byte offset multiplied by 2, with least significant bit -** toggled each time the log pointer value is changed. This is to make -** sure that this field changes each time the log pointer is updated, -** even if the log file itself is disabled. See lsmTreeMakeOld(). -** -** See ckptExportLog() and ckptImportLog(). -** -** Append points: -** -** 8 integers (4 * 64-bit page numbers). See ckptExportAppendlist(). -** -** For each level in the database, a level record. Formatted as follows: -** -** 0. Age of the level (least significant 16-bits). And flags mask (most -** significant 16-bits). -** 1. The number of right-hand segments (nRight, possibly 0), -** 2. Segment record for left-hand segment (8 integers defined below), -** 3. Segment record for each right-hand segment (8 integers defined below), -** 4. If nRight>0, The number of segments involved in the merge -** 5. if nRight>0, Current nSkip value (see Merge structure defn.), -** 6. For each segment in the merge: -** 5a. Page number of next cell to read during merge (this field -** is 64-bits - 2 integers) -** 5b. Cell number of next cell to read during merge -** 7. Page containing current split-key (64-bits - 2 integers). -** 8. Cell within page containing current split-key. -** 9. Current pointer value (64-bits - 2 integers). -** -** The block redirect array: -** -** 1. Number of redirections (maximum LSM_MAX_BLOCK_REDIRECTS). -** 2. For each redirection: -** a. "from" block number -** b. "to" block number -** -** The in-memory freelist entries. Each entry is either an insert or a -** delete. The in-memory freelist is to the free-block-list as the -** in-memory tree is to the users database content. -** -** 1. Number of free-list entries stored in checkpoint header. -** 2. Number of free blocks (in total). -** 3. Total number of blocks freed during database lifetime. -** 4. For each entry: -** 2a. Block number of free block. -** 2b. A 64-bit integer (MSW followed by LSW). -1 for a delete entry, -** or the associated checkpoint id for an insert. -** -** The checksum: -** -** 1. Checksum value 1. -** 2. Checksum value 2. -** -** In the above, a segment record consists of the following four 64-bit -** fields (converted to 2 * u32 by storing the MSW followed by LSW): -** -** 1. First page of array, -** 2. Last page of array, -** 3. Root page of array (or 0), -** 4. Size of array in pages. -*/ - -/* -** LARGE NUMBERS OF LEVEL RECORDS: -** -** A limit on the number of rhs segments that may be present in the database -** file. Defining this limit ensures that all level records fit within -** the 4096 byte limit for checkpoint blobs. -** -** The number of right-hand-side segments in a database is counted as -** follows: -** -** * For each level in the database not undergoing a merge, add 1. -** -** * For each level in the database that is undergoing a merge, add -** the number of segments on the rhs of the level. -** -** A level record not undergoing a merge is 10 integers. A level record -** with nRhs rhs segments and (nRhs+1) input segments (i.e. including the -** separators from the next level) is (11*nRhs+20) integers. The maximum -** per right-hand-side level is therefore 21 integers. So the maximum -** size of all level records in a checkpoint is 21*40=820 integers. -** -** TODO: Before pointer values were changed from 32 to 64 bits, the above -** used to come to 420 bytes - leaving significant space for a free-list -** prefix. No more. To fix this, reduce the size of the level records in -** a db snapshot, and improve management of the free-list tail in -** lsm_sorted.c. -*/ -#define LSM_MAX_RHS_SEGMENTS 40 - -/* -** LARGE NUMBERS OF FREELIST ENTRIES: -** -** There is also a limit (LSM_MAX_FREELIST_ENTRIES - defined in lsmInt.h) -** on the number of free-list entries stored in a checkpoint. Since each -** free-list entry consists of 3 integers, the maximum free-list size is -** 3*100=300 integers. Combined with the limit on rhs segments defined -** above, this ensures that a checkpoint always fits within a 4096 byte -** meta page. -** -** If the database contains more than 100 free blocks, the "overflow" flag -** in the checkpoint header is set and the remainder are stored in the -** system FREELIST entry in the LSM (along with user data). The value -** accompanying the FREELIST key in the LSM is, like a checkpoint, an array -** of 32-bit big-endian integers. As follows: -** -** For each entry: -** a. Block number of free block. -** b. MSW of associated checkpoint id. -** c. LSW of associated checkpoint id. -** -** The number of entries is not required - it is implied by the size of the -** value blob containing the integer array. -** -** Note that the limit defined by LSM_MAX_FREELIST_ENTRIES is a hard limit. -** The actual value used may be configured using LSM_CONFIG_MAX_FREELIST. -*/ - -/* -** The argument to this macro must be of type u32. On a little-endian -** architecture, it returns the u32 value that results from interpreting -** the 4 bytes as a big-endian value. On a big-endian architecture, it -** returns the value that would be produced by interpreting the 4 bytes -** of the input value as a little-endian integer. -*/ -#define BYTESWAP32(x) ( \ - (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8) \ - + (((x)&0x00FF0000)>>8) + (((x)&0xFF000000)>>24) \ -) - -static const int one = 1; -#define LSM_LITTLE_ENDIAN (*(u8 *)(&one)) - -/* Sizes, in integers, of various parts of the checkpoint. */ -#define CKPT_HDR_SIZE 9 -#define CKPT_LOGPTR_SIZE 4 -#define CKPT_APPENDLIST_SIZE (LSM_APPLIST_SZ * 2) - -/* A #define to describe each integer in the checkpoint header. */ -#define CKPT_HDR_ID_MSW 0 -#define CKPT_HDR_ID_LSW 1 -#define CKPT_HDR_NCKPT 2 -#define CKPT_HDR_CMPID 3 -#define CKPT_HDR_NBLOCK 4 -#define CKPT_HDR_BLKSZ 5 -#define CKPT_HDR_NLEVEL 6 -#define CKPT_HDR_PGSZ 7 -#define CKPT_HDR_NWRITE 8 - -#define CKPT_HDR_LO_MSW 9 -#define CKPT_HDR_LO_LSW 10 -#define CKPT_HDR_LO_CKSUM1 11 -#define CKPT_HDR_LO_CKSUM2 12 - -typedef struct CkptBuffer CkptBuffer; - -/* -** Dynamic buffer used to accumulate data for a checkpoint. -*/ -struct CkptBuffer { - lsm_env *pEnv; - int nAlloc; - u32 *aCkpt; -}; - -/* -** Calculate the checksum of the checkpoint specified by arguments aCkpt and -** nCkpt. Store the checksum in *piCksum1 and *piCksum2 before returning. -** -** The value of the nCkpt parameter includes the two checksum values at -** the end of the checkpoint. They are not used as inputs to the checksum -** calculation. The checksum is based on the array of (nCkpt-2) integers -** at aCkpt[]. -*/ -static void ckptChecksum(u32 *aCkpt, u32 nCkpt, u32 *piCksum1, u32 *piCksum2){ - u32 i; - u32 cksum1 = 1; - u32 cksum2 = 2; - - if( nCkpt % 2 ){ - cksum1 += aCkpt[nCkpt-3] & 0x0000FFFF; - cksum2 += aCkpt[nCkpt-3] & 0xFFFF0000; - } - - for(i=0; (i+3)=p->nAlloc ){ - int nNew = LSM_MAX(8, iIdx*2); - p->aCkpt = (u32 *)lsmReallocOrFree(p->pEnv, p->aCkpt, nNew*sizeof(u32)); - if( !p->aCkpt ){ - *pRc = LSM_NOMEM_BKPT; - return; - } - p->nAlloc = nNew; - } - p->aCkpt[iIdx] = iVal; -} - -/* -** Argument aInt points to an array nInt elements in size. Switch the -** endian-ness of each element of the array. -*/ -static void ckptChangeEndianness(u32 *aInt, int nInt){ - if( LSM_LITTLE_ENDIAN ){ - int i; - for(i=0; iaCkpt, nCkpt+2, &aCksum[0], &aCksum[1]); - ckptSetValue(p, nCkpt, aCksum[0], pRc); - ckptSetValue(p, nCkpt+1, aCksum[1], pRc); - } -} - -static void ckptAppend64(CkptBuffer *p, int *piOut, i64 iVal, int *pRc){ - int iOut = *piOut; - ckptSetValue(p, iOut++, (iVal >> 32) & 0xFFFFFFFF, pRc); - ckptSetValue(p, iOut++, (iVal & 0xFFFFFFFF), pRc); - *piOut = iOut; -} - -static i64 ckptRead64(u32 *a){ - return (((i64)a[0]) << 32) + (i64)a[1]; -} - -static i64 ckptGobble64(u32 *a, int *piIn){ - int iIn = *piIn; - *piIn += 2; - return ckptRead64(&a[iIn]); -} - - -/* -** Append a 6-value segment record corresponding to pSeg to the checkpoint -** buffer passed as the third argument. -*/ -static void ckptExportSegment( - Segment *pSeg, - CkptBuffer *p, - int *piOut, - int *pRc -){ - ckptAppend64(p, piOut, pSeg->iFirst, pRc); - ckptAppend64(p, piOut, pSeg->iLastPg, pRc); - ckptAppend64(p, piOut, pSeg->iRoot, pRc); - ckptAppend64(p, piOut, pSeg->nSize, pRc); -} - -static void ckptExportLevel( - Level *pLevel, /* Level object to serialize */ - CkptBuffer *p, /* Append new level record to this ckpt */ - int *piOut, /* IN/OUT: Size of checkpoint so far */ - int *pRc /* IN/OUT: Error code */ -){ - int iOut = *piOut; - Merge *pMerge; - - pMerge = pLevel->pMerge; - ckptSetValue(p, iOut++, (u32)pLevel->iAge + (u32)(pLevel->flags<<16), pRc); - ckptSetValue(p, iOut++, pLevel->nRight, pRc); - ckptExportSegment(&pLevel->lhs, p, &iOut, pRc); - - assert( (pLevel->nRight>0)==(pMerge!=0) ); - if( pMerge ){ - int i; - for(i=0; inRight; i++){ - ckptExportSegment(&pLevel->aRhs[i], p, &iOut, pRc); - } - assert( pMerge->nInput==pLevel->nRight - || pMerge->nInput==pLevel->nRight+1 - ); - ckptSetValue(p, iOut++, pMerge->nInput, pRc); - ckptSetValue(p, iOut++, pMerge->nSkip, pRc); - for(i=0; inInput; i++){ - ckptAppend64(p, &iOut, pMerge->aInput[i].iPg, pRc); - ckptSetValue(p, iOut++, pMerge->aInput[i].iCell, pRc); - } - ckptAppend64(p, &iOut, pMerge->splitkey.iPg, pRc); - ckptSetValue(p, iOut++, pMerge->splitkey.iCell, pRc); - ckptAppend64(p, &iOut, pMerge->iCurrentPtr, pRc); - } - - *piOut = iOut; -} - -/* -** Populate the log offset fields of the checkpoint buffer. 4 values. -*/ -static void ckptExportLog( - lsm_db *pDb, - int bFlush, - CkptBuffer *p, - int *piOut, - int *pRc -){ - int iOut = *piOut; - - assert( iOut==CKPT_HDR_LO_MSW ); - - if( bFlush ){ - i64 iOff = pDb->treehdr.iOldLog; - ckptAppend64(p, &iOut, iOff, pRc); - ckptSetValue(p, iOut++, pDb->treehdr.oldcksum0, pRc); - ckptSetValue(p, iOut++, pDb->treehdr.oldcksum1, pRc); - }else{ - for(; iOut<=CKPT_HDR_LO_CKSUM2; iOut++){ - ckptSetValue(p, iOut, pDb->pShmhdr->aSnap2[iOut], pRc); - } - } - - assert( *pRc || iOut==CKPT_HDR_LO_CKSUM2+1 ); - *piOut = iOut; -} - -static void ckptExportAppendlist( - lsm_db *db, /* Database connection */ - CkptBuffer *p, /* Checkpoint buffer to write to */ - int *piOut, /* IN/OUT: Offset within checkpoint buffer */ - int *pRc /* IN/OUT: Error code */ -){ - int i; - LsmPgno *aiAppend = db->pWorker->aiAppend; - - for(i=0; ipFS; /* File system object */ - Snapshot *pSnap = pDb->pWorker; /* Worker snapshot */ - int nLevel = 0; /* Number of levels in checkpoint */ - int iLevel; /* Used to count out nLevel levels */ - int iOut = 0; /* Current offset in aCkpt[] */ - Level *pLevel; /* Level iterator */ - int i; /* Iterator used while serializing freelist */ - CkptBuffer ckpt; - - /* Initialize the output buffer */ - memset(&ckpt, 0, sizeof(CkptBuffer)); - ckpt.pEnv = pDb->pEnv; - iOut = CKPT_HDR_SIZE; - - /* Write the log offset into the checkpoint. */ - ckptExportLog(pDb, bLog, &ckpt, &iOut, &rc); - - /* Write the append-point list */ - ckptExportAppendlist(pDb, &ckpt, &iOut, &rc); - - /* Figure out how many levels will be written to the checkpoint. */ - for(pLevel=lsmDbSnapshotLevel(pSnap); pLevel; pLevel=pLevel->pNext) nLevel++; - - /* Serialize nLevel levels. */ - iLevel = 0; - for(pLevel=lsmDbSnapshotLevel(pSnap); iLevelpNext){ - ckptExportLevel(pLevel, &ckpt, &iOut, &rc); - iLevel++; - } - - /* Write the block-redirect list */ - ckptSetValue(&ckpt, iOut++, pSnap->redirect.n, &rc); - for(i=0; iredirect.n; i++){ - ckptSetValue(&ckpt, iOut++, pSnap->redirect.a[i].iFrom, &rc); - ckptSetValue(&ckpt, iOut++, pSnap->redirect.a[i].iTo, &rc); - } - - /* Write the freelist */ - assert( pSnap->freelist.nEntry<=pDb->nMaxFreelist ); - if( rc==LSM_OK ){ - int nFree = pSnap->freelist.nEntry; - ckptSetValue(&ckpt, iOut++, nFree, &rc); - for(i=0; ifreelist.aEntry[i]; - ckptSetValue(&ckpt, iOut++, p->iBlk, &rc); - ckptSetValue(&ckpt, iOut++, (p->iId >> 32) & 0xFFFFFFFF, &rc); - ckptSetValue(&ckpt, iOut++, p->iId & 0xFFFFFFFF, &rc); - } - } - - /* Write the checkpoint header */ - assert( iId>=0 ); - assert( pSnap->iCmpId==pDb->compress.iId - || pSnap->iCmpId==LSM_COMPRESSION_EMPTY - ); - ckptSetValue(&ckpt, CKPT_HDR_ID_MSW, (u32)(iId>>32), &rc); - ckptSetValue(&ckpt, CKPT_HDR_ID_LSW, (u32)(iId&0xFFFFFFFF), &rc); - ckptSetValue(&ckpt, CKPT_HDR_NCKPT, iOut+2, &rc); - ckptSetValue(&ckpt, CKPT_HDR_CMPID, pDb->compress.iId, &rc); - ckptSetValue(&ckpt, CKPT_HDR_NBLOCK, pSnap->nBlock, &rc); - ckptSetValue(&ckpt, CKPT_HDR_BLKSZ, lsmFsBlockSize(pFS), &rc); - ckptSetValue(&ckpt, CKPT_HDR_NLEVEL, nLevel, &rc); - ckptSetValue(&ckpt, CKPT_HDR_PGSZ, lsmFsPageSize(pFS), &rc); - ckptSetValue(&ckpt, CKPT_HDR_NWRITE, pSnap->nWrite, &rc); - - if( bCksum ){ - ckptAddChecksum(&ckpt, iOut, &rc); - }else{ - ckptSetValue(&ckpt, iOut, 0, &rc); - ckptSetValue(&ckpt, iOut+1, 0, &rc); - } - iOut += 2; - assert( iOut<=1024 ); - -#ifdef LSM_LOG_FREELIST - lsmLogMessage(pDb, rc, - "ckptExportSnapshot(): id=%lld freelist: %d", iId, pSnap->freelist.nEntry - ); - for(i=0; ifreelist.nEntry; i++){ - lsmLogMessage(pDb, rc, - "ckptExportSnapshot(): iBlk=%d id=%lld", - pSnap->freelist.aEntry[i].iBlk, - pSnap->freelist.aEntry[i].iId - ); - } -#endif - - *ppCkpt = (void *)ckpt.aCkpt; - if( pnCkpt ) *pnCkpt = sizeof(u32)*iOut; - return rc; -} - - -/* -** Helper function for ckptImport(). -*/ -static void ckptNewSegment( - u32 *aIn, - int *piIn, - Segment *pSegment /* Populate this structure */ -){ - assert( pSegment->iFirst==0 && pSegment->iLastPg==0 ); - assert( pSegment->nSize==0 && pSegment->iRoot==0 ); - pSegment->iFirst = ckptGobble64(aIn, piIn); - pSegment->iLastPg = ckptGobble64(aIn, piIn); - pSegment->iRoot = ckptGobble64(aIn, piIn); - pSegment->nSize = ckptGobble64(aIn, piIn); - assert( pSegment->iFirst ); -} - -static int ckptSetupMerge(lsm_db *pDb, u32 *aInt, int *piIn, Level *pLevel){ - Merge *pMerge; /* Allocated Merge object */ - int nInput; /* Number of input segments in merge */ - int iIn = *piIn; /* Next value to read from aInt[] */ - int i; /* Iterator variable */ - int nByte; /* Number of bytes to allocate */ - - /* Allocate the Merge object. If malloc() fails, return LSM_NOMEM. */ - nInput = (int)aInt[iIn++]; - nByte = sizeof(Merge) + sizeof(MergeInput) * nInput; - pMerge = (Merge *)lsmMallocZero(pDb->pEnv, nByte); - if( !pMerge ) return LSM_NOMEM_BKPT; - pLevel->pMerge = pMerge; - - /* Populate the Merge object. */ - pMerge->aInput = (MergeInput *)&pMerge[1]; - pMerge->nInput = nInput; - pMerge->iOutputOff = -1; - pMerge->nSkip = (int)aInt[iIn++]; - for(i=0; iaInput[i].iPg = ckptGobble64(aInt, &iIn); - pMerge->aInput[i].iCell = (int)aInt[iIn++]; - } - pMerge->splitkey.iPg = ckptGobble64(aInt, &iIn); - pMerge->splitkey.iCell = (int)aInt[iIn++]; - pMerge->iCurrentPtr = ckptGobble64(aInt, &iIn); - - /* Set *piIn and return LSM_OK. */ - *piIn = iIn; - return LSM_OK; -} - - -static int ckptLoadLevels( - lsm_db *pDb, - u32 *aIn, - int *piIn, - int nLevel, - Level **ppLevel -){ - int i; - int rc = LSM_OK; - Level *pRet = 0; - Level **ppNext; - int iIn = *piIn; - - ppNext = &pRet; - for(i=0; rc==LSM_OK && ipEnv, sizeof(Level), &rc); - if( rc==LSM_OK ){ - pLevel->iAge = (u16)(aIn[iIn] & 0x0000FFFF); - pLevel->flags = (u16)((aIn[iIn]>>16) & 0x0000FFFF); - iIn++; - pLevel->nRight = aIn[iIn++]; - if( pLevel->nRight ){ - int nByte = sizeof(Segment) * pLevel->nRight; - pLevel->aRhs = (Segment *)lsmMallocZeroRc(pDb->pEnv, nByte, &rc); - } - if( rc==LSM_OK ){ - *ppNext = pLevel; - ppNext = &pLevel->pNext; - - /* Allocate the main segment */ - ckptNewSegment(aIn, &iIn, &pLevel->lhs); - - /* Allocate each of the right-hand segments, if any */ - for(iRight=0; iRightnRight; iRight++){ - ckptNewSegment(aIn, &iIn, &pLevel->aRhs[iRight]); - } - - /* Set up the Merge object, if required */ - if( pLevel->nRight>0 ){ - rc = ckptSetupMerge(pDb, aIn, &iIn, pLevel); - } - } - } - } - - if( rc!=LSM_OK ){ - /* An OOM must have occurred. Free any level structures allocated and - ** return the error to the caller. */ - lsmSortedFreeLevel(pDb->pEnv, pRet); - pRet = 0; - } - - *ppLevel = pRet; - *piIn = iIn; - return rc; -} - - -int lsmCheckpointLoadLevels(lsm_db *pDb, void *pVal, int nVal){ - int rc = LSM_OK; - if( nVal>0 ){ - u32 *aIn; - - aIn = lsmMallocRc(pDb->pEnv, nVal, &rc); - if( aIn ){ - Level *pLevel = 0; - Level *pParent; - - int nIn; - int nLevel; - int iIn = 1; - memcpy(aIn, pVal, nVal); - nIn = nVal / sizeof(u32); - - ckptChangeEndianness(aIn, nIn); - nLevel = aIn[0]; - rc = ckptLoadLevels(pDb, aIn, &iIn, nLevel, &pLevel); - lsmFree(pDb->pEnv, aIn); - assert( rc==LSM_OK || pLevel==0 ); - if( rc==LSM_OK ){ - pParent = lsmDbSnapshotLevel(pDb->pWorker); - assert( pParent ); - while( pParent->pNext ) pParent = pParent->pNext; - pParent->pNext = pLevel; - } - } - } - - return rc; -} - -/* -** Return the data for the LEVELS record. -** -** The size of the checkpoint that can be stored in the database header -** must not exceed 1024 32-bit integers. Normally, it does not. However, -** if it does, part of the checkpoint must be stored in the LSM. This -** routine returns that part. -*/ -int lsmCheckpointLevels( - lsm_db *pDb, /* Database handle */ - int nLevel, /* Number of levels to write to blob */ - void **paVal, /* OUT: Pointer to LEVELS blob */ - int *pnVal /* OUT: Size of LEVELS blob in bytes */ -){ - Level *p; /* Used to iterate through levels */ - int nAll= 0; - int rc; - int i; - int iOut; - CkptBuffer ckpt; - assert( nLevel>0 ); - - for(p=lsmDbSnapshotLevel(pDb->pWorker); p; p=p->pNext) nAll++; - - assert( nAll>nLevel ); - nAll -= nLevel; - for(p=lsmDbSnapshotLevel(pDb->pWorker); p && nAll>0; p=p->pNext) nAll--; - - memset(&ckpt, 0, sizeof(CkptBuffer)); - ckpt.pEnv = pDb->pEnv; - - ckptSetValue(&ckpt, 0, nLevel, &rc); - iOut = 1; - for(i=0; rc==LSM_OK && ipNext; - } - assert( rc!=LSM_OK || p==0 ); - - if( rc==LSM_OK ){ - ckptChangeEndianness(ckpt.aCkpt, iOut); - *paVal = (void *)ckpt.aCkpt; - *pnVal = iOut * sizeof(u32); - }else{ - *pnVal = 0; - *paVal = 0; - } - - return rc; -} - -/* -** Read the checkpoint id from meta-page pPg. -*/ -static i64 ckptLoadId(MetaPage *pPg){ - i64 ret = 0; - if( pPg ){ - int nData; - u8 *aData = lsmFsMetaPageData(pPg, &nData); - ret = (((i64)lsmGetU32(&aData[CKPT_HDR_ID_MSW*4])) << 32) + - ((i64)lsmGetU32(&aData[CKPT_HDR_ID_LSW*4])); - } - return ret; -} - -/* -** Return true if the buffer passed as an argument contains a valid -** checkpoint. -*/ -static int ckptChecksumOk(u32 *aCkpt){ - u32 nCkpt = aCkpt[CKPT_HDR_NCKPT]; - u32 cksum1; - u32 cksum2; - - if( nCkpt(LSM_META_RW_PAGE_SIZE)/sizeof(u32) ){ - return 0; - } - ckptChecksum(aCkpt, nCkpt, &cksum1, &cksum2); - return (cksum1==aCkpt[nCkpt-2] && cksum2==aCkpt[nCkpt-1]); -} - -/* -** Attempt to load a checkpoint from meta page iMeta. -** -** This function is a no-op if *pRc is set to any value other than LSM_OK -** when it is called. If an error occurs, *pRc is set to an LSM error code -** before returning. -** -** If no error occurs and the checkpoint is successfully loaded, copy it to -** ShmHeader.aSnap1[] and ShmHeader.aSnap2[], and set ShmHeader.iMetaPage -** to indicate its origin. In this case return 1. Or, if the checkpoint -** cannot be loaded (because the checksum does not compute), return 0. -*/ -static int ckptTryLoad(lsm_db *pDb, MetaPage *pPg, u32 iMeta, int *pRc){ - int bLoaded = 0; /* Return value */ - if( *pRc==LSM_OK ){ - int rc = LSM_OK; /* Error code */ - u32 *aCkpt = 0; /* Pointer to buffer containing checkpoint */ - u32 nCkpt; /* Number of elements in aCkpt[] */ - int nData; /* Bytes of data in aData[] */ - u8 *aData; /* Meta page data */ - - aData = lsmFsMetaPageData(pPg, &nData); - nCkpt = (u32)lsmGetU32(&aData[CKPT_HDR_NCKPT*sizeof(u32)]); - if( nCkpt<=nData/sizeof(u32) && nCkpt>CKPT_HDR_NCKPT ){ - aCkpt = (u32 *)lsmMallocRc(pDb->pEnv, nCkpt*sizeof(u32), &rc); - } - if( aCkpt ){ - memcpy(aCkpt, aData, nCkpt*sizeof(u32)); - ckptChangeEndianness(aCkpt, nCkpt); - if( ckptChecksumOk(aCkpt) ){ - ShmHeader *pShm = pDb->pShmhdr; - memcpy(pShm->aSnap1, aCkpt, nCkpt*sizeof(u32)); - memcpy(pShm->aSnap2, aCkpt, nCkpt*sizeof(u32)); - memcpy(pDb->aSnapshot, aCkpt, nCkpt*sizeof(u32)); - pShm->iMetaPage = iMeta; - bLoaded = 1; - } - } - - lsmFree(pDb->pEnv, aCkpt); - *pRc = rc; - } - return bLoaded; -} - -/* -** Initialize the shared-memory header with an empty snapshot. This function -** is called when no valid snapshot can be found in the database header. -*/ -static void ckptLoadEmpty(lsm_db *pDb){ - u32 aCkpt[] = { - 0, /* CKPT_HDR_ID_MSW */ - 10, /* CKPT_HDR_ID_LSW */ - 0, /* CKPT_HDR_NCKPT */ - LSM_COMPRESSION_EMPTY, /* CKPT_HDR_CMPID */ - 0, /* CKPT_HDR_NBLOCK */ - 0, /* CKPT_HDR_BLKSZ */ - 0, /* CKPT_HDR_NLEVEL */ - 0, /* CKPT_HDR_PGSZ */ - 0, /* CKPT_HDR_NWRITE */ - 0, 0, 1234, 5678, /* The log pointer and initial checksum */ - 0,0,0,0, 0,0,0,0, /* The append list */ - 0, /* The redirected block list */ - 0, /* The free block list */ - 0, 0 /* Space for checksum values */ - }; - u32 nCkpt = array_size(aCkpt); - ShmHeader *pShm = pDb->pShmhdr; - - aCkpt[CKPT_HDR_NCKPT] = nCkpt; - aCkpt[CKPT_HDR_BLKSZ] = pDb->nDfltBlksz; - aCkpt[CKPT_HDR_PGSZ] = pDb->nDfltPgsz; - ckptChecksum(aCkpt, array_size(aCkpt), &aCkpt[nCkpt-2], &aCkpt[nCkpt-1]); - - memcpy(pShm->aSnap1, aCkpt, nCkpt*sizeof(u32)); - memcpy(pShm->aSnap2, aCkpt, nCkpt*sizeof(u32)); - memcpy(pDb->aSnapshot, aCkpt, nCkpt*sizeof(u32)); -} - -/* -** This function is called as part of database recovery to initialize the -** ShmHeader.aSnap1[] and ShmHeader.aSnap2[] snapshots. -*/ -int lsmCheckpointRecover(lsm_db *pDb){ - int rc = LSM_OK; /* Return Code */ - i64 iId1; /* Id of checkpoint on meta-page 1 */ - i64 iId2; /* Id of checkpoint on meta-page 2 */ - int bLoaded = 0; /* True once checkpoint has been loaded */ - int cmp; /* True if (iId2>iId1) */ - MetaPage *apPg[2] = {0, 0}; /* Meta-pages 1 and 2 */ - - rc = lsmFsMetaPageGet(pDb->pFS, 0, 1, &apPg[0]); - if( rc==LSM_OK ) rc = lsmFsMetaPageGet(pDb->pFS, 0, 2, &apPg[1]); - - iId1 = ckptLoadId(apPg[0]); - iId2 = ckptLoadId(apPg[1]); - cmp = (iId2 > iId1); - bLoaded = ckptTryLoad(pDb, apPg[cmp?1:0], (cmp?2:1), &rc); - if( bLoaded==0 ){ - bLoaded = ckptTryLoad(pDb, apPg[cmp?0:1], (cmp?1:2), &rc); - } - - /* The database does not contain a valid checkpoint. Initialize the shared - ** memory header with an empty checkpoint. */ - if( bLoaded==0 ){ - ckptLoadEmpty(pDb); - } - - lsmFsMetaPageRelease(apPg[0]); - lsmFsMetaPageRelease(apPg[1]); - - return rc; -} - -/* -** Store the snapshot in pDb->aSnapshot[] in meta-page iMeta. -*/ -int lsmCheckpointStore(lsm_db *pDb, int iMeta){ - MetaPage *pPg = 0; - int rc; - - assert( iMeta==1 || iMeta==2 ); - rc = lsmFsMetaPageGet(pDb->pFS, 1, iMeta, &pPg); - if( rc==LSM_OK ){ - u8 *aData; - int nData; - int nCkpt; - - nCkpt = (int)pDb->aSnapshot[CKPT_HDR_NCKPT]; - aData = lsmFsMetaPageData(pPg, &nData); - memcpy(aData, pDb->aSnapshot, nCkpt*sizeof(u32)); - ckptChangeEndianness((u32 *)aData, nCkpt); - rc = lsmFsMetaPageRelease(pPg); - } - - return rc; -} - -/* -** Copy the current client snapshot from shared-memory to pDb->aSnapshot[]. -*/ -int lsmCheckpointLoad(lsm_db *pDb, int *piRead){ - int nRem = LSM_ATTEMPTS_BEFORE_PROTOCOL; - ShmHeader *pShm = pDb->pShmhdr; - while( (nRem--)>0 ){ - int nInt; - - nInt = pShm->aSnap1[CKPT_HDR_NCKPT]; - if( nInt<=(LSM_META_RW_PAGE_SIZE / sizeof(u32)) ){ - memcpy(pDb->aSnapshot, pShm->aSnap1, nInt*sizeof(u32)); - if( ckptChecksumOk(pDb->aSnapshot) ){ - if( piRead ) *piRead = 1; - return LSM_OK; - } - } - - nInt = pShm->aSnap2[CKPT_HDR_NCKPT]; - if( nInt<=(LSM_META_RW_PAGE_SIZE / sizeof(u32)) ){ - memcpy(pDb->aSnapshot, pShm->aSnap2, nInt*sizeof(u32)); - if( ckptChecksumOk(pDb->aSnapshot) ){ - if( piRead ) *piRead = 2; - return LSM_OK; - } - } - - lsmShmBarrier(pDb); - } - return LSM_PROTOCOL_BKPT; -} - -int lsmInfoCompressionId(lsm_db *db, u32 *piCmpId){ - int rc; - - assert( db->pClient==0 && db->pWorker==0 ); - rc = lsmCheckpointLoad(db, 0); - if( rc==LSM_OK ){ - *piCmpId = db->aSnapshot[CKPT_HDR_CMPID]; - } - - return rc; -} - -int lsmCheckpointLoadOk(lsm_db *pDb, int iSnap){ - u32 *aShm; - assert( iSnap==1 || iSnap==2 ); - aShm = (iSnap==1) ? pDb->pShmhdr->aSnap1 : pDb->pShmhdr->aSnap2; - return (lsmCheckpointId(pDb->aSnapshot, 0)==lsmCheckpointId(aShm, 0) ); -} - -int lsmCheckpointClientCacheOk(lsm_db *pDb){ - return ( pDb->pClient - && pDb->pClient->iId==lsmCheckpointId(pDb->aSnapshot, 0) - && pDb->pClient->iId==lsmCheckpointId(pDb->pShmhdr->aSnap1, 0) - && pDb->pClient->iId==lsmCheckpointId(pDb->pShmhdr->aSnap2, 0) - ); -} - -int lsmCheckpointLoadWorker(lsm_db *pDb){ - int rc; - ShmHeader *pShm = pDb->pShmhdr; - int nInt1; - int nInt2; - - /* Must be holding the WORKER lock to do this. Or DMS2. */ - assert( - lsmShmAssertLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL) - || lsmShmAssertLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL) - ); - - /* Check that the two snapshots match. If not, repair them. */ - nInt1 = pShm->aSnap1[CKPT_HDR_NCKPT]; - nInt2 = pShm->aSnap2[CKPT_HDR_NCKPT]; - if( nInt1!=nInt2 || memcmp(pShm->aSnap1, pShm->aSnap2, nInt2*sizeof(u32)) ){ - if( ckptChecksumOk(pShm->aSnap1) ){ - memcpy(pShm->aSnap2, pShm->aSnap1, sizeof(u32)*nInt1); - }else if( ckptChecksumOk(pShm->aSnap2) ){ - memcpy(pShm->aSnap1, pShm->aSnap2, sizeof(u32)*nInt2); - }else{ - return LSM_PROTOCOL_BKPT; - } - } - - rc = lsmCheckpointDeserialize(pDb, 1, pShm->aSnap1, &pDb->pWorker); - if( pDb->pWorker ) pDb->pWorker->pDatabase = pDb->pDatabase; - - if( rc==LSM_OK ){ - rc = lsmCheckCompressionId(pDb, pDb->pWorker->iCmpId); - } - -#if 0 - assert( rc!=LSM_OK || lsmFsIntegrityCheck(pDb) ); -#endif - return rc; -} - -int lsmCheckpointDeserialize( - lsm_db *pDb, - int bInclFreelist, /* If true, deserialize free-list */ - u32 *aCkpt, - Snapshot **ppSnap -){ - int rc = LSM_OK; - Snapshot *pNew; - - pNew = (Snapshot *)lsmMallocZeroRc(pDb->pEnv, sizeof(Snapshot), &rc); - if( rc==LSM_OK ){ - Level *pLvl; - int nFree; - int i; - int nLevel = (int)aCkpt[CKPT_HDR_NLEVEL]; - int iIn = CKPT_HDR_SIZE + CKPT_APPENDLIST_SIZE + CKPT_LOGPTR_SIZE; - - pNew->iId = lsmCheckpointId(aCkpt, 0); - pNew->nBlock = aCkpt[CKPT_HDR_NBLOCK]; - pNew->nWrite = aCkpt[CKPT_HDR_NWRITE]; - rc = ckptLoadLevels(pDb, aCkpt, &iIn, nLevel, &pNew->pLevel); - pNew->iLogOff = lsmCheckpointLogOffset(aCkpt); - pNew->iCmpId = aCkpt[CKPT_HDR_CMPID]; - - /* Make a copy of the append-list */ - for(i=0; iaiAppend[i] = ckptRead64(a); - } - - /* Read the block-redirect list */ - pNew->redirect.n = aCkpt[iIn++]; - if( pNew->redirect.n ){ - pNew->redirect.a = lsmMallocZeroRc(pDb->pEnv, - (sizeof(struct RedirectEntry) * LSM_MAX_BLOCK_REDIRECTS), &rc - ); - if( rc==LSM_OK ){ - for(i=0; iredirect.n; i++){ - pNew->redirect.a[i].iFrom = aCkpt[iIn++]; - pNew->redirect.a[i].iTo = aCkpt[iIn++]; - } - } - for(pLvl=pNew->pLevel; pLvl->pNext; pLvl=pLvl->pNext); - if( pLvl->nRight ){ - pLvl->aRhs[pLvl->nRight-1].pRedirect = &pNew->redirect; - }else{ - pLvl->lhs.pRedirect = &pNew->redirect; - } - } - - /* Copy the free-list */ - if( rc==LSM_OK && bInclFreelist ){ - nFree = aCkpt[iIn++]; - if( nFree ){ - pNew->freelist.aEntry = (FreelistEntry *)lsmMallocZeroRc( - pDb->pEnv, sizeof(FreelistEntry)*nFree, &rc - ); - if( rc==LSM_OK ){ - int j; - for(j=0; jfreelist.aEntry[j]; - p->iBlk = aCkpt[iIn++]; - p->iId = ((i64)(aCkpt[iIn])<<32) + aCkpt[iIn+1]; - iIn += 2; - } - pNew->freelist.nEntry = pNew->freelist.nAlloc = nFree; - } - } - } - } - - if( rc!=LSM_OK ){ - lsmFreeSnapshot(pDb->pEnv, pNew); - pNew = 0; - } - - *ppSnap = pNew; - return rc; -} - -/* -** Connection pDb must be the worker connection in order to call this -** function. It returns true if the database already contains the maximum -** number of levels or false otherwise. -** -** This is used when flushing the in-memory tree to disk. If the database -** is already full, then the caller should invoke lsm_work() or similar -** until it is not full before creating a new level by flushing the in-memory -** tree to disk. Limiting the number of levels in the database ensures that -** the records describing them always fit within the checkpoint blob. -*/ -int lsmDatabaseFull(lsm_db *pDb){ - Level *p; - int nRhs = 0; - - assert( lsmShmAssertLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL) ); - assert( pDb->pWorker ); - - for(p=pDb->pWorker->pLevel; p; p=p->pNext){ - nRhs += (p->nRight ? p->nRight : 1); - } - - return (nRhs >= LSM_MAX_RHS_SEGMENTS); -} - -/* -** The connection passed as the only argument is currently the worker -** connection. Some work has been performed on the database by the connection, -** but no new snapshot has been written into shared memory. -** -** This function updates the shared-memory worker and client snapshots with -** the new snapshot produced by the work performed by pDb. -** -** If successful, LSM_OK is returned. Otherwise, if an error occurs, an LSM -** error code is returned. -*/ -int lsmCheckpointSaveWorker(lsm_db *pDb, int bFlush){ - Snapshot *pSnap = pDb->pWorker; - ShmHeader *pShm = pDb->pShmhdr; - void *p = 0; - int n = 0; - int rc; - - pSnap->iId++; - rc = ckptExportSnapshot(pDb, bFlush, pSnap->iId, 1, &p, &n); - if( rc!=LSM_OK ) return rc; - assert( ckptChecksumOk((u32 *)p) ); - - assert( n<=LSM_META_RW_PAGE_SIZE ); - memcpy(pShm->aSnap2, p, n); - lsmShmBarrier(pDb); - memcpy(pShm->aSnap1, p, n); - lsmFree(pDb->pEnv, p); - - /* assert( lsmFsIntegrityCheck(pDb) ); */ - return LSM_OK; -} - -/* -** This function is used to determine the snapshot-id of the most recently -** checkpointed snapshot. Variable ShmHeader.iMetaPage indicates which of -** the two meta-pages said snapshot resides on (if any). -** -** If successful, this function loads the snapshot from the meta-page, -** verifies its checksum and sets *piId to the snapshot-id before returning -** LSM_OK. Or, if the checksum attempt fails, *piId is set to zero and -** LSM_OK returned. If an error occurs, an LSM error code is returned and -** the final value of *piId is undefined. -*/ -int lsmCheckpointSynced(lsm_db *pDb, i64 *piId, i64 *piLog, u32 *pnWrite){ - int rc = LSM_OK; - MetaPage *pPg; - u32 iMeta; - - iMeta = pDb->pShmhdr->iMetaPage; - if( iMeta==1 || iMeta==2 ){ - rc = lsmFsMetaPageGet(pDb->pFS, 0, iMeta, &pPg); - if( rc==LSM_OK ){ - int nCkpt; - int nData; - u8 *aData; - - aData = lsmFsMetaPageData(pPg, &nData); - assert( nData==LSM_META_RW_PAGE_SIZE ); - nCkpt = lsmGetU32(&aData[CKPT_HDR_NCKPT*sizeof(u32)]); - if( nCkpt<(LSM_META_RW_PAGE_SIZE/sizeof(u32)) ){ - u32 *aCopy = lsmMallocRc(pDb->pEnv, sizeof(u32) * nCkpt, &rc); - if( aCopy ){ - memcpy(aCopy, aData, nCkpt*sizeof(u32)); - ckptChangeEndianness(aCopy, nCkpt); - if( ckptChecksumOk(aCopy) ){ - if( piId ) *piId = lsmCheckpointId(aCopy, 0); - if( piLog ) *piLog = (lsmCheckpointLogOffset(aCopy) >> 1); - if( pnWrite ) *pnWrite = aCopy[CKPT_HDR_NWRITE]; - } - lsmFree(pDb->pEnv, aCopy); - } - } - lsmFsMetaPageRelease(pPg); - } - } - - if( (iMeta!=1 && iMeta!=2) || rc!=LSM_OK || pDb->pShmhdr->iMetaPage!=iMeta ){ - if( piId ) *piId = 0; - if( piLog ) *piLog = 0; - if( pnWrite ) *pnWrite = 0; - } - return rc; -} - -/* -** Return the checkpoint-id of the checkpoint array passed as the first -** argument to this function. If the second argument is true, then assume -** that the checkpoint is made up of 32-bit big-endian integers. If it -** is false, assume that the integers are in machine byte order. -*/ -i64 lsmCheckpointId(u32 *aCkpt, int bDisk){ - i64 iId; - if( bDisk ){ - u8 *aData = (u8 *)aCkpt; - iId = (((i64)lsmGetU32(&aData[CKPT_HDR_ID_MSW*4])) << 32); - iId += ((i64)lsmGetU32(&aData[CKPT_HDR_ID_LSW*4])); - }else{ - iId = ((i64)aCkpt[CKPT_HDR_ID_MSW] << 32) + (i64)aCkpt[CKPT_HDR_ID_LSW]; - } - return iId; -} - -u32 lsmCheckpointNBlock(u32 *aCkpt){ - return aCkpt[CKPT_HDR_NBLOCK]; -} - -u32 lsmCheckpointNWrite(u32 *aCkpt, int bDisk){ - if( bDisk ){ - return lsmGetU32((u8 *)&aCkpt[CKPT_HDR_NWRITE]); - }else{ - return aCkpt[CKPT_HDR_NWRITE]; - } -} - -i64 lsmCheckpointLogOffset(u32 *aCkpt){ - return ((i64)aCkpt[CKPT_HDR_LO_MSW] << 32) + (i64)aCkpt[CKPT_HDR_LO_LSW]; -} - -int lsmCheckpointPgsz(u32 *aCkpt){ return (int)aCkpt[CKPT_HDR_PGSZ]; } - -int lsmCheckpointBlksz(u32 *aCkpt){ return (int)aCkpt[CKPT_HDR_BLKSZ]; } - -void lsmCheckpointLogoffset( - u32 *aCkpt, - DbLog *pLog -){ - pLog->aRegion[2].iStart = (lsmCheckpointLogOffset(aCkpt) >> 1); - - pLog->cksum0 = aCkpt[CKPT_HDR_LO_CKSUM1]; - pLog->cksum1 = aCkpt[CKPT_HDR_LO_CKSUM2]; - pLog->iSnapshotId = lsmCheckpointId(aCkpt, 0); -} - -void lsmCheckpointZeroLogoffset(lsm_db *pDb){ - u32 nCkpt; - - nCkpt = pDb->aSnapshot[CKPT_HDR_NCKPT]; - assert( nCkpt>CKPT_HDR_NCKPT ); - assert( nCkpt==pDb->pShmhdr->aSnap1[CKPT_HDR_NCKPT] ); - assert( 0==memcmp(pDb->aSnapshot, pDb->pShmhdr->aSnap1, nCkpt*sizeof(u32)) ); - assert( 0==memcmp(pDb->aSnapshot, pDb->pShmhdr->aSnap2, nCkpt*sizeof(u32)) ); - - pDb->aSnapshot[CKPT_HDR_LO_MSW] = 0; - pDb->aSnapshot[CKPT_HDR_LO_LSW] = 0; - ckptChecksum(pDb->aSnapshot, nCkpt, - &pDb->aSnapshot[nCkpt-2], &pDb->aSnapshot[nCkpt-1] - ); - - memcpy(pDb->pShmhdr->aSnap1, pDb->aSnapshot, nCkpt*sizeof(u32)); - memcpy(pDb->pShmhdr->aSnap2, pDb->aSnapshot, nCkpt*sizeof(u32)); -} - -/* -** Set the output variable to the number of KB of data written into the -** database file since the most recent checkpoint. -*/ -int lsmCheckpointSize(lsm_db *db, int *pnKB){ - int rc = LSM_OK; - u32 nSynced; - - /* Set nSynced to the number of pages that had been written when the - ** database was last checkpointed. */ - rc = lsmCheckpointSynced(db, 0, 0, &nSynced); - - if( rc==LSM_OK ){ - u32 nPgsz = db->pShmhdr->aSnap1[CKPT_HDR_PGSZ]; - u32 nWrite = db->pShmhdr->aSnap1[CKPT_HDR_NWRITE]; - *pnKB = (int)(( ((i64)(nWrite - nSynced) * nPgsz) + 1023) / 1024); - } - - return rc; -} diff --git a/ext/lsm1/lsm_file.c b/ext/lsm1/lsm_file.c deleted file mode 100644 index 9f4144618..000000000 --- a/ext/lsm1/lsm_file.c +++ /dev/null @@ -1,3311 +0,0 @@ -/* -** 2011-08-26 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** NORMAL DATABASE FILE FORMAT -** -** The following database file format concepts are used by the code in -** this file to read and write the database file. -** -** Pages: -** -** A database file is divided into pages. The first 8KB of the file consists -** of two 4KB meta-pages. The meta-page size is not configurable. The -** remainder of the file is made up of database pages. The default database -** page size is 4KB. Database pages are aligned to page-size boundaries, -** so if the database page size is larger than 8KB there is a gap between -** the end of the meta pages and the start of the database pages. -** -** Database pages are numbered based on their position in the file. Page N -** begins at byte offset ((N-1)*pgsz). This means that page 1 does not -** exist - since it would always overlap with the meta pages. If the -** page-size is (say) 512 bytes, then the first usable page in the database -** is page 33. -** -** It is assumed that the first two meta pages and the data that follows -** them are located on different disk sectors. So that if a power failure -** while writing to a meta page there is no risk of damage to the other -** meta page or any other part of the database file. TODO: This may need -** to be revisited. -** -** Blocks: -** -** The database file is also divided into blocks. The default block size is -** 1MB. When writing to the database file, an attempt is made to write data -** in contiguous block-sized chunks. -** -** The first and last page on each block are special in that they are 4 -** bytes smaller than all other pages. This is because the last four bytes -** of space on the first and last pages of each block are reserved for -** pointers to other blocks (i.e. a 32-bit block number). -** -** Runs: -** -** A run is a sequence of pages that the upper layer uses to store a -** sorted array of database keys (and accompanying data - values, FC -** pointers and so on). Given a page within a run, it is possible to -** navigate to the next page in the run as follows: -** -** a) if the current page is not the last in a block, the next page -** in the run is located immediately after the current page, OR -** -** b) if the current page is the last page in a block, the next page -** in the run is the first page on the block identified by the -** block pointer stored in the last 4 bytes of the current block. -** -** It is possible to navigate to the previous page in a similar fashion, -** using the block pointer embedded in the last 4 bytes of the first page -** of each block as required. -** -** The upper layer is responsible for identifying by page number the -** first and last page of any run that it needs to navigate - there are -** no "end-of-run" markers stored or identified by this layer. This is -** necessary as clients reading different database snapshots may access -** different subsets of a run. -** -** THE LOG FILE -** -** This file opens and closes the log file. But it does not contain any -** logic related to the log file format. Instead, it exports the following -** functions that are used by the code in lsm_log.c to read and write the -** log file: -** -** lsmFsOpenLog -** lsmFsWriteLog -** lsmFsSyncLog -** lsmFsReadLog -** lsmFsTruncateLog -** lsmFsCloseAndDeleteLog -** -** COMPRESSED DATABASE FILE FORMAT -** -** The compressed database file format is very similar to the normal format. -** The file still begins with two 4KB meta-pages (which are never compressed). -** It is still divided into blocks. -** -** The first and last four bytes of each block are reserved for 32-bit -** pointer values. Similar to the way four bytes are carved from the end of -** the first and last page of each block in uncompressed databases. From -** the point of view of the upper layer, all pages are the same size - this -** is different from the uncompressed format where the first and last pages -** on each block are 4 bytes smaller than the others. -** -** Pages are stored in variable length compressed form, as follows: -** -** * 3-byte size field containing the size of the compressed page image -** in bytes. The most significant bit of each byte of the size field -** is always set. The remaining 7 bits are used to store a 21-bit -** integer value (in big-endian order - the first byte in the field -** contains the most significant 7 bits). Since the maximum allowed -** size of a compressed page image is (2^17 - 1) bytes, there are -** actually 4 unused bits in the size field. -** -** In other words, if the size of the compressed page image is nSz, -** the header can be serialized as follows: -** -** u8 aHdr[3] -** aHdr[0] = 0x80 | (u8)(nSz >> 14); -** aHdr[1] = 0x80 | (u8)(nSz >> 7); -** aHdr[2] = 0x80 | (u8)(nSz >> 0); -** -** * Compressed page image. -** -** * A second copy of the 3-byte record header. -** -** A page number is a byte offset into the database file. So the smallest -** possible page number is 8192 (immediately after the two meta-pages). -** The first and root page of a segment are identified by a page number -** corresponding to the byte offset of the first byte in the corresponding -** page record. The last page of a segment is identified by the byte offset -** of the last byte in its record. -** -** Unlike uncompressed pages, compressed page records may span blocks. -** -** Sometimes, in order to avoid touching sectors that contain synced data -** when writing, it is necessary to insert unused space between compressed -** page records. This can be done as follows: -** -** * For less than 6 bytes of empty space, the first and last byte -** of the free space contain the total number of free bytes. For -** example: -** -** Block of 4 free bytes: 0x04 0x?? 0x?? 0x04 -** Block of 2 free bytes: 0x02 0x02 -** A single free byte: 0x01 -** -** * For 6 or more bytes of empty space, a record similar to a -** compressed page record is added to the segment. A padding record -** is distinguished from a compressed page record by the most -** significant bit of the second byte of the size field, which is -** cleared instead of set. -*/ -#include "lsmInt.h" - -#include -#include -#include - -/* -** File-system object. Each database connection allocates a single instance -** of the following structure. It is used for all access to the database and -** log files. -** -** The database file may be accessed via two methods - using mmap() or using -** read() and write() calls. In the general case both methods are used - a -** prefix of the file is mapped into memory and the remainder accessed using -** read() and write(). This is helpful when accessing very large files (or -** files that may grow very large during the lifetime of a database -** connection) on systems with 32-bit address spaces. However, it also requires -** that this object manage two distinct types of Page objects simultaneously - -** those that carry pointers to the mapped file and those that carry arrays -** populated by read() calls. -** -** pFree: -** The head of a singly-linked list that containing currently unused Page -** structures suitable for use as mmap-page handles. Connected by the -** Page.pFreeNext pointers. -** -** pMapped: -** The head of a singly-linked list that contains all pages that currently -** carry pointers to the mapped region. This is used if the region is -** every remapped - the pointers carried by existing pages can be adjusted -** to account for the remapping. Connected by the Page.pMappedNext pointers. -** -** pWaiting: -** When the upper layer wishes to append a new b-tree page to a segment, -** it allocates a Page object that carries a malloc'd block of memory - -** regardless of the mmap-related configuration. The page is not assigned -** a page number at first. When the upper layer has finished constructing -** the page contents, it calls lsmFsPagePersist() to assign a page number -** to it. At this point it is likely that N pages have been written to the -** segment, the (N+1)th page is still outstanding and the b-tree page is -** assigned page number (N+2). To avoid writing page (N+2) before page -** (N+1), the recently completed b-tree page is held in the singly linked -** list headed by pWaiting until page (N+1) has been written. -** -** Function lsmFsFlushWaiting() is responsible for eventually writing -** waiting pages to disk. -** -** apHash/nHash: -** Hash table used to store all Page objects that carry malloc'd arrays, -** except those b-tree pages that have not yet been assigned page numbers. -** Once they have been assigned page numbers - they are added to this -** hash table. -** -** Hash table overflow chains are connected using the Page.pHashNext -** pointers. -** -** pLruFirst, pLruLast: -** The first and last entries in a doubly-linked list of pages. This -** list contains all pages with malloc'd data that are present in the -** hash table and have a ref-count of zero. -*/ -struct FileSystem { - lsm_db *pDb; /* Database handle that owns this object */ - lsm_env *pEnv; /* Environment pointer */ - char *zDb; /* Database file name */ - char *zLog; /* Database file name */ - int nMetasize; /* Size of meta pages in bytes */ - int nMetaRwSize; /* Read/written size of meta pages in bytes */ - i64 nPagesize; /* Database page-size in bytes */ - i64 nBlocksize; /* Database block-size in bytes */ - - /* r/w file descriptors for both files. */ - LsmFile *pLsmFile; /* Used after lsm_close() to link into list */ - lsm_file *fdDb; /* Database file */ - lsm_file *fdLog; /* Log file */ - int szSector; /* Database file sector size */ - - /* If this is a compressed database, a pointer to the compression methods. - ** For an uncompressed database, a NULL pointer. */ - lsm_compress *pCompress; - u8 *aIBuffer; /* Buffer to compress to */ - u8 *aOBuffer; /* Buffer to uncompress from */ - int nBuffer; /* Allocated size of above buffers in bytes */ - - /* mmap() page related things */ - i64 nMapLimit; /* Maximum bytes of file to map */ - void *pMap; /* Current mapping of database file */ - i64 nMap; /* Bytes mapped at pMap */ - Page *pFree; /* Unused Page structures */ - Page *pMapped; /* List of Page structs that point to pMap */ - - /* Page cache parameters for non-mmap() pages */ - int nCacheMax; /* Configured cache size (in pages) */ - int nCacheAlloc; /* Current cache size (in pages) */ - Page *pLruFirst; /* Head of the LRU list */ - Page *pLruLast; /* Tail of the LRU list */ - int nHash; /* Number of hash slots in hash table */ - Page **apHash; /* nHash Hash slots */ - Page *pWaiting; /* b-tree pages waiting to be written */ - - /* Statistics */ - int nOut; /* Number of outstanding pages */ - int nWrite; /* Total number of pages written */ - int nRead; /* Total number of pages read */ -}; - -/* -** Database page handle. -** -** pSeg: -** When lsmFsSortedAppend() is called on a compressed database, the new -** page is not assigned a page number or location in the database file -** immediately. Instead, these are assigned by the lsmFsPagePersist() call -** right before it writes the compressed page image to disk. -** -** The lsmFsSortedAppend() function sets the pSeg pointer to point to the -** segment that the new page will be a part of. It is unset by -** lsmFsPagePersist() after the page is written to disk. -*/ -struct Page { - u8 *aData; /* Buffer containing page data */ - int nData; /* Bytes of usable data at aData[] */ - LsmPgno iPg; /* Page number */ - int nRef; /* Number of outstanding references */ - int flags; /* Combination of PAGE_XXX flags */ - Page *pHashNext; /* Next page in hash table slot */ - Page *pLruNext; /* Next page in LRU list */ - Page *pLruPrev; /* Previous page in LRU list */ - FileSystem *pFS; /* File system that owns this page */ - - /* Only used in compressed database mode: */ - int nCompress; /* Compressed size (or 0 for uncomp. db) */ - int nCompressPrev; /* Compressed size of prev page */ - Segment *pSeg; /* Segment this page will be written to */ - - /* Pointers for singly linked lists */ - Page *pWaitingNext; /* Next page in FileSystem.pWaiting list */ - Page *pFreeNext; /* Next page in FileSystem.pFree list */ - Page *pMappedNext; /* Next page in FileSystem.pMapped list */ -}; - -/* -** Meta-data page handle. There are two meta-data pages at the start of -** the database file, each FileSystem.nMetasize bytes in size. -*/ -struct MetaPage { - int iPg; /* Either 1 or 2 */ - int bWrite; /* Write back to db file on release */ - u8 *aData; /* Pointer to buffer */ - FileSystem *pFS; /* FileSystem that owns this page */ -}; - -/* -** Values for LsmPage.flags -*/ -#define PAGE_DIRTY 0x00000001 /* Set if page is dirty */ -#define PAGE_FREE 0x00000002 /* Set if Page.aData requires lsmFree() */ -#define PAGE_HASPREV 0x00000004 /* Set if page is first on uncomp. block */ - -/* -** Number of pgsz byte pages omitted from the start of block 1. The start -** of block 1 contains two 4096 byte meta pages (8192 bytes in total). -*/ -#define BLOCK1_HDR_SIZE(pgsz) LSM_MAX(1, 8192/(pgsz)) - -/* -** If NDEBUG is not defined, set a breakpoint in function lsmIoerrBkpt() -** to catch IO errors (any error returned by a VFS method). -*/ -#ifndef NDEBUG -static void lsmIoerrBkpt(void){ - static int nErr = 0; - nErr++; -} -static int IOERR_WRAPPER(int rc){ - if( rc!=LSM_OK ) lsmIoerrBkpt(); - return rc; -} -#else -# define IOERR_WRAPPER(rc) (rc) -#endif - -#ifdef NDEBUG -# define assert_lists_are_ok(x) -#else -static Page *fsPageFindInHash(FileSystem *pFS, LsmPgno iPg, int *piHash); - -static void assert_lists_are_ok(FileSystem *pFS){ -#if 0 - Page *p; - - assert( pFS->nMapLimit>=0 ); - - /* Check that all pages in the LRU list have nRef==0, pointers to buffers - ** in heap memory, and corresponding entries in the hash table. */ - for(p=pFS->pLruFirst; p; p=p->pLruNext){ - assert( p==pFS->pLruFirst || p->pLruPrev!=0 ); - assert( p==pFS->pLruLast || p->pLruNext!=0 ); - assert( p->pLruPrev==0 || p->pLruPrev->pLruNext==p ); - assert( p->pLruNext==0 || p->pLruNext->pLruPrev==p ); - assert( p->nRef==0 ); - assert( p->flags & PAGE_FREE ); - assert( p==fsPageFindInHash(pFS, p->iPg, 0) ); - } -#endif -} -#endif - -/* -** Wrappers around the VFS methods of the lsm_env object: -** -** lsmEnvOpen() -** lsmEnvRead() -** lsmEnvWrite() -** lsmEnvSync() -** lsmEnvSectorSize() -** lsmEnvClose() -** lsmEnvTruncate() -** lsmEnvUnlink() -** lsmEnvRemap() -*/ -int lsmEnvOpen(lsm_env *pEnv, const char *zFile, int flags, lsm_file **ppNew){ - return pEnv->xOpen(pEnv, zFile, flags, ppNew); -} - -static int lsmEnvRead( - lsm_env *pEnv, - lsm_file *pFile, - lsm_i64 iOff, - void *pRead, - int nRead -){ - return IOERR_WRAPPER( pEnv->xRead(pFile, iOff, pRead, nRead) ); -} - -static int lsmEnvWrite( - lsm_env *pEnv, - lsm_file *pFile, - lsm_i64 iOff, - const void *pWrite, - int nWrite -){ - return IOERR_WRAPPER( pEnv->xWrite(pFile, iOff, (void *)pWrite, nWrite) ); -} - -static int lsmEnvSync(lsm_env *pEnv, lsm_file *pFile){ - return IOERR_WRAPPER( pEnv->xSync(pFile) ); -} - -static int lsmEnvSectorSize(lsm_env *pEnv, lsm_file *pFile){ - return pEnv->xSectorSize(pFile); -} - -int lsmEnvClose(lsm_env *pEnv, lsm_file *pFile){ - return IOERR_WRAPPER( pEnv->xClose(pFile) ); -} - -static int lsmEnvTruncate(lsm_env *pEnv, lsm_file *pFile, lsm_i64 nByte){ - return IOERR_WRAPPER( pEnv->xTruncate(pFile, nByte) ); -} - -static int lsmEnvUnlink(lsm_env *pEnv, const char *zDel){ - return IOERR_WRAPPER( pEnv->xUnlink(pEnv, zDel) ); -} - -static int lsmEnvRemap( - lsm_env *pEnv, - lsm_file *pFile, - i64 szMin, - void **ppMap, - i64 *pszMap -){ - return pEnv->xRemap(pFile, szMin, ppMap, pszMap); -} - -int lsmEnvLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int eLock){ - if( pFile==0 ) return LSM_OK; - return pEnv->xLock(pFile, iLock, eLock); -} - -int lsmEnvTestLock( - lsm_env *pEnv, - lsm_file *pFile, - int iLock, - int nLock, - int eLock -){ - return pEnv->xTestLock(pFile, iLock, nLock, eLock); -} - -int lsmEnvShmMap( - lsm_env *pEnv, - lsm_file *pFile, - int iChunk, - int sz, - void **ppOut -){ - return pEnv->xShmMap(pFile, iChunk, sz, ppOut); -} - -void lsmEnvShmBarrier(lsm_env *pEnv){ - pEnv->xShmBarrier(); -} - -void lsmEnvShmUnmap(lsm_env *pEnv, lsm_file *pFile, int bDel){ - pEnv->xShmUnmap(pFile, bDel); -} - -void lsmEnvSleep(lsm_env *pEnv, int nUs){ - pEnv->xSleep(pEnv, nUs); -} - - -/* -** Write the contents of string buffer pStr into the log file, starting at -** offset iOff. -*/ -int lsmFsWriteLog(FileSystem *pFS, i64 iOff, LsmString *pStr){ - assert( pFS->fdLog ); - return lsmEnvWrite(pFS->pEnv, pFS->fdLog, iOff, pStr->z, pStr->n); -} - -/* -** fsync() the log file. -*/ -int lsmFsSyncLog(FileSystem *pFS){ - assert( pFS->fdLog ); - return lsmEnvSync(pFS->pEnv, pFS->fdLog); -} - -/* -** Read nRead bytes of data starting at offset iOff of the log file. Append -** the results to string buffer pStr. -*/ -int lsmFsReadLog(FileSystem *pFS, i64 iOff, int nRead, LsmString *pStr){ - int rc; /* Return code */ - assert( pFS->fdLog ); - rc = lsmStringExtend(pStr, nRead); - if( rc==LSM_OK ){ - rc = lsmEnvRead(pFS->pEnv, pFS->fdLog, iOff, &pStr->z[pStr->n], nRead); - pStr->n += nRead; - } - return rc; -} - -/* -** Truncate the log file to nByte bytes in size. -*/ -int lsmFsTruncateLog(FileSystem *pFS, i64 nByte){ - if( pFS->fdLog==0 ) return LSM_OK; - return lsmEnvTruncate(pFS->pEnv, pFS->fdLog, nByte); -} - -/* -** Truncate the db file to nByte bytes in size. -*/ -int lsmFsTruncateDb(FileSystem *pFS, i64 nByte){ - if( pFS->fdDb==0 ) return LSM_OK; - return lsmEnvTruncate(pFS->pEnv, pFS->fdDb, nByte); -} - -/* -** Close the log file. Then delete it from the file-system. This function -** is called during database shutdown only. -*/ -int lsmFsCloseAndDeleteLog(FileSystem *pFS){ - char *zDel; - - if( pFS->fdLog ){ - lsmEnvClose(pFS->pEnv, pFS->fdLog ); - pFS->fdLog = 0; - } - - zDel = lsmMallocPrintf(pFS->pEnv, "%s-log", pFS->zDb); - if( zDel ){ - lsmEnvUnlink(pFS->pEnv, zDel); - lsmFree(pFS->pEnv, zDel); - } - return LSM_OK; -} - -/* -** Return true if page iReal of the database should be accessed using mmap. -** False otherwise. -*/ -static int fsMmapPage(FileSystem *pFS, LsmPgno iReal){ - return ((i64)iReal*pFS->nPagesize <= pFS->nMapLimit); -} - -/* -** Given that there are currently nHash slots in the hash table, return -** the hash key for file iFile, page iPg. -*/ -static int fsHashKey(int nHash, LsmPgno iPg){ - return (iPg % nHash); -} - -/* -** This is a helper function for lsmFsOpen(). It opens a single file on -** disk (either the database or log file). -*/ -static lsm_file *fsOpenFile( - FileSystem *pFS, /* File system object */ - int bReadonly, /* True to open this file read-only */ - int bLog, /* True for log, false for db */ - int *pRc /* IN/OUT: Error code */ -){ - lsm_file *pFile = 0; - if( *pRc==LSM_OK ){ - int flags = (bReadonly ? LSM_OPEN_READONLY : 0); - const char *zPath = (bLog ? pFS->zLog : pFS->zDb); - - *pRc = lsmEnvOpen(pFS->pEnv, zPath, flags, &pFile); - } - return pFile; -} - -/* -** If it is not already open, this function opens the log file. It returns -** LSM_OK if successful (or if the log file was already open) or an LSM -** error code otherwise. -** -** The log file must be opened before any of the following may be called: -** -** lsmFsWriteLog -** lsmFsSyncLog -** lsmFsReadLog -*/ -int lsmFsOpenLog(lsm_db *db, int *pbOpen){ - int rc = LSM_OK; - FileSystem *pFS = db->pFS; - - if( 0==pFS->fdLog ){ - pFS->fdLog = fsOpenFile(pFS, db->bReadonly, 1, &rc); - - if( rc==LSM_IOERR_NOENT && db->bReadonly ){ - rc = LSM_OK; - } - } - - if( pbOpen ) *pbOpen = (pFS->fdLog!=0); - return rc; -} - -/* -** Close the log file, if it is open. -*/ -void lsmFsCloseLog(lsm_db *db){ - FileSystem *pFS = db->pFS; - if( pFS->fdLog ){ - lsmEnvClose(pFS->pEnv, pFS->fdLog); - pFS->fdLog = 0; - } -} - -/* -** Open a connection to a database stored within the file-system. -** -** If parameter bReadonly is true, then open a read-only file-descriptor -** on the database file. It is possible that bReadonly will be false even -** if the user requested that pDb be opened read-only. This is because the -** file-descriptor may later on be recycled by a read-write connection. -** If the db file can be opened for read-write access, it always is. Parameter -** bReadonly is only ever true if it has already been determined that the -** db can only be opened for read-only access. -** -** Return LSM_OK if successful or an lsm error code otherwise. -*/ -int lsmFsOpen( - lsm_db *pDb, /* Database connection to open fd for */ - const char *zDb, /* Full path to database file */ - int bReadonly /* True to open db file read-only */ -){ - FileSystem *pFS; - int rc = LSM_OK; - int nDb = strlen(zDb); - int nByte; - - assert( pDb->pFS==0 ); - assert( pDb->pWorker==0 && pDb->pClient==0 ); - - nByte = sizeof(FileSystem) + nDb+1 + nDb+4+1; - pFS = (FileSystem *)lsmMallocZeroRc(pDb->pEnv, nByte, &rc); - if( pFS ){ - LsmFile *pLsmFile; - pFS->zDb = (char *)&pFS[1]; - pFS->zLog = &pFS->zDb[nDb+1]; - pFS->nPagesize = LSM_DFLT_PAGE_SIZE; - pFS->nBlocksize = LSM_DFLT_BLOCK_SIZE; - pFS->nMetasize = LSM_META_PAGE_SIZE; - pFS->nMetaRwSize = LSM_META_RW_PAGE_SIZE; - pFS->pDb = pDb; - pFS->pEnv = pDb->pEnv; - - /* Make a copy of the database and log file names. */ - memcpy(pFS->zDb, zDb, nDb+1); - memcpy(pFS->zLog, zDb, nDb); - memcpy(&pFS->zLog[nDb], "-log", 5); - - /* Allocate the hash-table here. At some point, it should be changed - ** so that it can grow dynamicly. */ - pFS->nCacheMax = 2048*1024 / pFS->nPagesize; - pFS->nHash = 4096; - pFS->apHash = lsmMallocZeroRc(pDb->pEnv, sizeof(Page *) * pFS->nHash, &rc); - - /* Open the database file */ - pLsmFile = lsmDbRecycleFd(pDb); - if( pLsmFile ){ - pFS->pLsmFile = pLsmFile; - pFS->fdDb = pLsmFile->pFile; - memset(pLsmFile, 0, sizeof(LsmFile)); - }else{ - pFS->pLsmFile = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmFile), &rc); - if( rc==LSM_OK ){ - pFS->fdDb = fsOpenFile(pFS, bReadonly, 0, &rc); - } - } - - if( rc!=LSM_OK ){ - lsmFsClose(pFS); - pFS = 0; - }else{ - pFS->szSector = lsmEnvSectorSize(pFS->pEnv, pFS->fdDb); - } - } - - pDb->pFS = pFS; - return rc; -} - -/* -** Configure the file-system object according to the current values of -** the LSM_CONFIG_MMAP and LSM_CONFIG_SET_COMPRESSION options. -*/ -int lsmFsConfigure(lsm_db *db){ - FileSystem *pFS = db->pFS; - if( pFS ){ - lsm_env *pEnv = pFS->pEnv; - Page *pPg; - - assert( pFS->nOut==0 ); - assert( pFS->pWaiting==0 ); - assert( pFS->pMapped==0 ); - - /* Reset any compression/decompression buffers already allocated */ - lsmFree(pEnv, pFS->aIBuffer); - lsmFree(pEnv, pFS->aOBuffer); - pFS->nBuffer = 0; - - /* Unmap the file, if it is currently mapped */ - if( pFS->pMap ){ - lsmEnvRemap(pEnv, pFS->fdDb, -1, &pFS->pMap, &pFS->nMap); - pFS->nMapLimit = 0; - } - - /* Free all allocated page structures */ - pPg = pFS->pLruFirst; - while( pPg ){ - Page *pNext = pPg->pLruNext; - assert( pPg->flags & PAGE_FREE ); - lsmFree(pEnv, pPg->aData); - lsmFree(pEnv, pPg); - pPg = pNext; - } - - pPg = pFS->pFree; - while( pPg ){ - Page *pNext = pPg->pFreeNext; - lsmFree(pEnv, pPg); - pPg = pNext; - } - - /* Zero pointers that point to deleted page objects */ - pFS->nCacheAlloc = 0; - pFS->pLruFirst = 0; - pFS->pLruLast = 0; - pFS->pFree = 0; - if( pFS->apHash ){ - memset(pFS->apHash, 0, pFS->nHash*sizeof(pFS->apHash[0])); - } - - /* Configure the FileSystem object */ - if( db->compress.xCompress ){ - pFS->pCompress = &db->compress; - pFS->nMapLimit = 0; - }else{ - pFS->pCompress = 0; - if( db->iMmap==1 ){ - /* Unlimited */ - pFS->nMapLimit = (i64)1 << 60; - }else{ - /* iMmap is a limit in KB. Set nMapLimit to the same value in bytes. */ - pFS->nMapLimit = (i64)db->iMmap * 1024; - } - } - } - - return LSM_OK; -} - -/* -** Close and destroy a FileSystem object. -*/ -void lsmFsClose(FileSystem *pFS){ - if( pFS ){ - Page *pPg; - lsm_env *pEnv = pFS->pEnv; - - assert( pFS->nOut==0 ); - pPg = pFS->pLruFirst; - while( pPg ){ - Page *pNext = pPg->pLruNext; - if( pPg->flags & PAGE_FREE ) lsmFree(pEnv, pPg->aData); - lsmFree(pEnv, pPg); - pPg = pNext; - } - - pPg = pFS->pFree; - while( pPg ){ - Page *pNext = pPg->pFreeNext; - if( pPg->flags & PAGE_FREE ) lsmFree(pEnv, pPg->aData); - lsmFree(pEnv, pPg); - pPg = pNext; - } - - if( pFS->fdDb ) lsmEnvClose(pFS->pEnv, pFS->fdDb ); - if( pFS->fdLog ) lsmEnvClose(pFS->pEnv, pFS->fdLog ); - lsmFree(pEnv, pFS->pLsmFile); - lsmFree(pEnv, pFS->apHash); - lsmFree(pEnv, pFS->aIBuffer); - lsmFree(pEnv, pFS->aOBuffer); - lsmFree(pEnv, pFS); - } -} - -/* -** This function is called when closing a database handle (i.e. lsm_close()) -** if there exist other connections to the same database within this process. -** In that case the file-descriptor open on the database file is not closed -** when the FileSystem object is destroyed, as this would cause any POSIX -** locks held by the other connections to be silently dropped (see "man close" -** for details). Instead, the file-descriptor is stored in a list by the -** lsm_shared.c module until it is either closed or reused. -** -** This function returns a pointer to an object that can be linked into -** the list described above. The returned object now 'owns' the database -** file descriptor, so that when the FileSystem object is destroyed, it -** will not be closed. -** -** This function may be called at most once in the life-time of a -** FileSystem object. The results of any operations involving the database -** file descriptor are undefined once this function has been called. -** -** None of this is necessary on non-POSIX systems. But we do it anyway in -** the name of using as similar code as possible on all platforms. -*/ -LsmFile *lsmFsDeferClose(FileSystem *pFS){ - LsmFile *p = pFS->pLsmFile; - assert( p->pNext==0 ); - p->pFile = pFS->fdDb; - pFS->fdDb = 0; - pFS->pLsmFile = 0; - return p; -} - -/* -** Allocate a buffer and populate it with the output of the xFileid() -** method of the database file handle. If successful, set *ppId to point -** to the buffer and *pnId to the number of bytes in the buffer and return -** LSM_OK. Otherwise, set *ppId and *pnId to zero and return an LSM -** error code. -*/ -int lsmFsFileid(lsm_db *pDb, void **ppId, int *pnId){ - lsm_env *pEnv = pDb->pEnv; - FileSystem *pFS = pDb->pFS; - int rc; - int nId = 0; - void *pId; - - rc = pEnv->xFileid(pFS->fdDb, 0, &nId); - pId = lsmMallocZeroRc(pEnv, nId, &rc); - if( rc==LSM_OK ) rc = pEnv->xFileid(pFS->fdDb, pId, &nId); - - if( rc!=LSM_OK ){ - lsmFree(pEnv, pId); - pId = 0; - nId = 0; - } - - *ppId = pId; - *pnId = nId; - return rc; -} - -/* -** Return the nominal page-size used by this file-system. Actual pages -** may be smaller or larger than this value. -*/ -int lsmFsPageSize(FileSystem *pFS){ - return pFS->nPagesize; -} - -/* -** Return the block-size used by this file-system. -*/ -int lsmFsBlockSize(FileSystem *pFS){ - return pFS->nBlocksize; -} - -/* -** Configure the nominal page-size used by this file-system. Actual -** pages may be smaller or larger than this value. -*/ -void lsmFsSetPageSize(FileSystem *pFS, int nPgsz){ - pFS->nPagesize = nPgsz; - pFS->nCacheMax = 2048*1024 / pFS->nPagesize; -} - -/* -** Configure the block-size used by this file-system. -*/ -void lsmFsSetBlockSize(FileSystem *pFS, int nBlocksize){ - pFS->nBlocksize = nBlocksize; -} - -/* -** Return the page number of the first page on block iBlock. Blocks are -** numbered starting from 1. -** -** For a compressed database, page numbers are byte offsets. The first -** page on each block is the byte offset immediately following the 4-byte -** "previous block" pointer at the start of each block. -*/ -static LsmPgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){ - LsmPgno iPg; - if( pFS->pCompress ){ - if( iBlock==1 ){ - iPg = pFS->nMetasize * 2 + 4; - }else{ - iPg = pFS->nBlocksize * (LsmPgno)(iBlock-1) + 4; - } - }else{ - const i64 nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); - if( iBlock==1 ){ - iPg = 1 + ((pFS->nMetasize*2 + pFS->nPagesize - 1) / pFS->nPagesize); - }else{ - iPg = 1 + (iBlock-1) * nPagePerBlock; - } - } - return iPg; -} - -/* -** Return the page number of the last page on block iBlock. Blocks are -** numbered starting from 1. -** -** For a compressed database, page numbers are byte offsets. The first -** page on each block is the byte offset of the byte immediately before -** the 4-byte "next block" pointer at the end of each block. -*/ -static LsmPgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){ - if( pFS->pCompress ){ - return pFS->nBlocksize * (LsmPgno)iBlock - 1 - 4; - }else{ - const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); - return iBlock * nPagePerBlock; - } -} - -/* -** Return the block number of the block that page iPg is located on. -** Blocks are numbered starting from 1. -*/ -static int fsPageToBlock(FileSystem *pFS, LsmPgno iPg){ - if( pFS->pCompress ){ - return (int)((iPg / pFS->nBlocksize) + 1); - }else{ - return (int)(1 + ((iPg-1) / (pFS->nBlocksize / pFS->nPagesize))); - } -} - -/* -** Return true if page iPg is the last page on its block. -** -** This function is only called in non-compressed database mode. -*/ -static int fsIsLast(FileSystem *pFS, LsmPgno iPg){ - const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); - assert( !pFS->pCompress ); - return ( iPg && (iPg % nPagePerBlock)==0 ); -} - -/* -** Return true if page iPg is the first page on its block. -** -** This function is only called in non-compressed database mode. -*/ -static int fsIsFirst(FileSystem *pFS, LsmPgno iPg){ - const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); - assert( !pFS->pCompress ); - return ( (iPg % nPagePerBlock)==1 - || (iPgnData; - } - return pPage->aData; -} - -/* -** Return the page number of a page. -*/ -LsmPgno lsmFsPageNumber(Page *pPage){ - /* assert( (pPage->flags & PAGE_DIRTY)==0 ); */ - return pPage ? pPage->iPg : 0; -} - -/* -** Page pPg is currently part of the LRU list belonging to pFS. Remove -** it from the list. pPg->pLruNext and pPg->pLruPrev are cleared by this -** operation. -*/ -static void fsPageRemoveFromLru(FileSystem *pFS, Page *pPg){ - assert( pPg->pLruNext || pPg==pFS->pLruLast ); - assert( pPg->pLruPrev || pPg==pFS->pLruFirst ); - if( pPg->pLruNext ){ - pPg->pLruNext->pLruPrev = pPg->pLruPrev; - }else{ - pFS->pLruLast = pPg->pLruPrev; - } - if( pPg->pLruPrev ){ - pPg->pLruPrev->pLruNext = pPg->pLruNext; - }else{ - pFS->pLruFirst = pPg->pLruNext; - } - pPg->pLruPrev = 0; - pPg->pLruNext = 0; -} - -/* -** Page pPg is not currently part of the LRU list belonging to pFS. Add it. -*/ -static void fsPageAddToLru(FileSystem *pFS, Page *pPg){ - assert( pPg->pLruNext==0 && pPg->pLruPrev==0 ); - pPg->pLruPrev = pFS->pLruLast; - if( pPg->pLruPrev ){ - pPg->pLruPrev->pLruNext = pPg; - }else{ - pFS->pLruFirst = pPg; - } - pFS->pLruLast = pPg; -} - -/* -** Page pPg is currently stored in the apHash/nHash hash table. Remove it. -*/ -static void fsPageRemoveFromHash(FileSystem *pFS, Page *pPg){ - int iHash; - Page **pp; - - iHash = fsHashKey(pFS->nHash, pPg->iPg); - for(pp=&pFS->apHash[iHash]; *pp!=pPg; pp=&(*pp)->pHashNext); - *pp = pPg->pHashNext; - pPg->pHashNext = 0; -} - -/* -** Free a Page object allocated by fsPageBuffer(). -*/ -static void fsPageBufferFree(Page *pPg){ - pPg->pFS->nCacheAlloc--; - lsmFree(pPg->pFS->pEnv, pPg->aData); - lsmFree(pPg->pFS->pEnv, pPg); -} - - -/* -** Purge the cache of all non-mmap pages with nRef==0. -*/ -void lsmFsPurgeCache(FileSystem *pFS){ - Page *pPg; - - pPg = pFS->pLruFirst; - while( pPg ){ - Page *pNext = pPg->pLruNext; - assert( pPg->flags & PAGE_FREE ); - fsPageRemoveFromHash(pFS, pPg); - fsPageBufferFree(pPg); - pPg = pNext; - } - pFS->pLruFirst = 0; - pFS->pLruLast = 0; - - assert( pFS->nCacheAlloc<=pFS->nOut && pFS->nCacheAlloc>=0 ); -} - -/* -** Search the hash-table for page iPg. If an entry is round, return a pointer -** to it. Otherwise, return NULL. -** -** Either way, if argument piHash is not NULL set *piHash to the hash slot -** number that page iPg would be stored in before returning. -*/ -static Page *fsPageFindInHash(FileSystem *pFS, LsmPgno iPg, int *piHash){ - Page *p; /* Return value */ - int iHash = fsHashKey(pFS->nHash, iPg); - - if( piHash ) *piHash = iHash; - for(p=pFS->apHash[iHash]; p; p=p->pHashNext){ - if( p->iPg==iPg) break; - } - return p; -} - -/* -** Allocate and return a non-mmap Page object. If there are already -** nCacheMax such Page objects outstanding, try to recycle an existing -** Page instead. -*/ -static int fsPageBuffer( - FileSystem *pFS, - Page **ppOut -){ - int rc = LSM_OK; - Page *pPage = 0; - if( pFS->pLruFirst==0 || pFS->nCacheAllocnCacheMax ){ - /* Allocate a new Page object */ - pPage = lsmMallocZero(pFS->pEnv, sizeof(Page)); - if( !pPage ){ - rc = LSM_NOMEM_BKPT; - }else{ - pPage->aData = (u8 *)lsmMalloc(pFS->pEnv, pFS->nPagesize); - if( !pPage->aData ){ - lsmFree(pFS->pEnv, pPage); - rc = LSM_NOMEM_BKPT; - pPage = 0; - }else{ - pFS->nCacheAlloc++; - } - } - }else{ - /* Reuse an existing Page object */ - u8 *aData; - pPage = pFS->pLruFirst; - aData = pPage->aData; - fsPageRemoveFromLru(pFS, pPage); - fsPageRemoveFromHash(pFS, pPage); - - memset(pPage, 0, sizeof(Page)); - pPage->aData = aData; - } - - if( pPage ){ - pPage->flags = PAGE_FREE; - } - *ppOut = pPage; - return rc; -} - -/* -** Assuming *pRc is initially LSM_OK, attempt to ensure that the -** memory-mapped region is at least iSz bytes in size. If it is not already, -** iSz bytes in size, extend it and update the pointers associated with any -** outstanding Page objects. -** -** If *pRc is not LSM_OK when this function is called, it is a no-op. -** Otherwise, *pRc is set to an lsm error code if an error occurs, or -** left unmodified otherwise. -** -** This function is never called in compressed database mode. -*/ -static void fsGrowMapping( - FileSystem *pFS, /* File system object */ - i64 iSz, /* Minimum size to extend mapping to */ - int *pRc /* IN/OUT: Error code */ -){ - assert( PAGE_HASPREV==4 ); - - if( *pRc==LSM_OK && iSz>pFS->nMap ){ - int rc; - u8 *aOld = pFS->pMap; - rc = lsmEnvRemap(pFS->pEnv, pFS->fdDb, iSz, &pFS->pMap, &pFS->nMap); - if( rc==LSM_OK && pFS->pMap!=aOld ){ - Page *pFix; - i64 iOff = (u8 *)pFS->pMap - aOld; - for(pFix=pFS->pMapped; pFix; pFix=pFix->pMappedNext){ - pFix->aData += iOff; - } - lsmSortedRemap(pFS->pDb); - } - *pRc = rc; - } -} - -/* -** If it is mapped, unmap the database file. -*/ -int lsmFsUnmap(FileSystem *pFS){ - int rc = LSM_OK; - if( pFS ){ - rc = lsmEnvRemap(pFS->pEnv, pFS->fdDb, -1, &pFS->pMap, &pFS->nMap); - } - return rc; -} - -/* -** fsync() the database file. -*/ -int lsmFsSyncDb(FileSystem *pFS, int nBlock){ - return lsmEnvSync(pFS->pEnv, pFS->fdDb); -} - -/* -** If block iBlk has been redirected according to the redirections in the -** object passed as the first argument, return the destination block to -** which it is redirected. Otherwise, return a copy of iBlk. -*/ -static int fsRedirectBlock(Redirect *p, int iBlk){ - if( p ){ - int i; - for(i=0; in; i++){ - if( iBlk==p->a[i].iFrom ) return p->a[i].iTo; - } - } - assert( iBlk!=0 ); - return iBlk; -} - -/* -** If page iPg has been redirected according to the redirections in the -** object passed as the second argument, return the destination page to -** which it is redirected. Otherwise, return a copy of iPg. -*/ -LsmPgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, LsmPgno iPg){ - LsmPgno iReal = iPg; - - if( pRedir ){ - const int nPagePerBlock = ( - pFS->pCompress ? pFS->nBlocksize : (pFS->nBlocksize / pFS->nPagesize) - ); - int iBlk = fsPageToBlock(pFS, iPg); - int i; - for(i=0; in; i++){ - int iFrom = pRedir->a[i].iFrom; - if( iFrom>iBlk ) break; - if( iFrom==iBlk ){ - int iTo = pRedir->a[i].iTo; - iReal = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock; - if( iTo==1 ){ - iReal += (fsFirstPageOnBlock(pFS, 1)-1); - } - break; - } - } - } - - assert( iReal!=0 ); - return iReal; -} - -/* Required by the circular fsBlockNext<->fsPageGet dependency. */ -static int fsPageGet(FileSystem *, Segment *, LsmPgno, int, Page **, int *); - -/* -** Parameter iBlock is a database file block. This function reads the value -** stored in the blocks "next block" pointer and stores it in *piNext. -** LSM_OK is returned if everything is successful, or an LSM error code -** otherwise. -*/ -static int fsBlockNext( - FileSystem *pFS, /* File-system object handle */ - Segment *pSeg, /* Use this segment for block redirects */ - int iBlock, /* Read field from this block */ - int *piNext /* OUT: Next block in linked list */ -){ - int rc; - int iRead; /* Read block from here */ - - if( pSeg ){ - iRead = fsRedirectBlock(pSeg->pRedirect, iBlock); - }else{ - iRead = iBlock; - } - - assert( pFS->nMapLimit==0 || pFS->pCompress==0 ); - if( pFS->pCompress ){ - i64 iOff; /* File offset to read data from */ - u8 aNext[4]; /* 4-byte pointer read from db file */ - - iOff = (i64)iRead * pFS->nBlocksize - sizeof(aNext); - rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aNext, sizeof(aNext)); - if( rc==LSM_OK ){ - *piNext = (int)lsmGetU32(aNext); - } - }else{ - const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); - Page *pLast; - rc = fsPageGet(pFS, 0, iRead*nPagePerBlock, 0, &pLast, 0); - if( rc==LSM_OK ){ - *piNext = lsmGetU32(&pLast->aData[pFS->nPagesize-4]); - lsmFsPageRelease(pLast); - } - } - - if( pSeg ){ - *piNext = fsRedirectBlock(pSeg->pRedirect, *piNext); - } - return rc; -} - -/* -** Return the page number of the last page on the same block as page iPg. -*/ -LsmPgno fsLastPageOnPagesBlock(FileSystem *pFS, LsmPgno iPg){ - return fsLastPageOnBlock(pFS, fsPageToBlock(pFS, iPg)); -} - -/* -** Read nData bytes of data from offset iOff of the database file into -** buffer aData. If this means reading past the end of a block, follow -** the block pointer to the next block and continue reading. -** -** Offset iOff is an absolute offset - not subject to any block redirection. -** However any block pointer followed is. Use pSeg->pRedirect in this case. -** -** This function is only called in compressed database mode. -*/ -static int fsReadData( - FileSystem *pFS, /* File-system handle */ - Segment *pSeg, /* Block redirection */ - i64 iOff, /* Read data from this offset */ - u8 *aData, /* Buffer to read data into */ - int nData /* Number of bytes to read */ -){ - i64 iEob; /* End of block */ - int nRead; - int rc; - - assert( pFS->pCompress ); - - iEob = fsLastPageOnPagesBlock(pFS, iOff) + 1; - nRead = (int)LSM_MIN(iEob - iOff, nData); - - rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aData, nRead); - if( rc==LSM_OK && nRead!=nData ){ - int iBlk; - - rc = fsBlockNext(pFS, pSeg, fsPageToBlock(pFS, iOff), &iBlk); - if( rc==LSM_OK ){ - i64 iOff2 = fsFirstPageOnBlock(pFS, iBlk); - rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff2, &aData[nRead], nData-nRead); - } - } - - return rc; -} - -/* -** Parameter iBlock is a database file block. This function reads the value -** stored in the blocks "previous block" pointer and stores it in *piPrev. -** LSM_OK is returned if everything is successful, or an LSM error code -** otherwise. -*/ -static int fsBlockPrev( - FileSystem *pFS, /* File-system object handle */ - Segment *pSeg, /* Use this segment for block redirects */ - int iBlock, /* Read field from this block */ - int *piPrev /* OUT: Previous block in linked list */ -){ - int rc = LSM_OK; /* Return code */ - - assert( pFS->nMapLimit==0 || pFS->pCompress==0 ); - assert( iBlock>0 ); - - if( pFS->pCompress ){ - i64 iOff = fsFirstPageOnBlock(pFS, iBlock) - 4; - u8 aPrev[4]; /* 4-byte pointer read from db file */ - rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aPrev, sizeof(aPrev)); - if( rc==LSM_OK ){ - Redirect *pRedir = (pSeg ? pSeg->pRedirect : 0); - *piPrev = fsRedirectBlock(pRedir, (int)lsmGetU32(aPrev)); - } - }else{ - assert( 0 ); - } - return rc; -} - -/* -** Encode and decode routines for record size fields. -*/ -static void putRecordSize(u8 *aBuf, int nByte, int bFree){ - aBuf[0] = (u8)(nByte >> 14) | 0x80; - aBuf[1] = ((u8)(nByte >> 7) & 0x7F) | (bFree ? 0x00 : 0x80); - aBuf[2] = (u8)nByte | 0x80; -} -static int getRecordSize(u8 *aBuf, int *pbFree){ - int nByte; - nByte = (aBuf[0] & 0x7F) << 14; - nByte += (aBuf[1] & 0x7F) << 7; - nByte += (aBuf[2] & 0x7F); - *pbFree = !(aBuf[1] & 0x80); - return nByte; -} - -/* -** Subtract iSub from database file offset iOff and set *piRes to the -** result. If doing so means passing the start of a block, follow the -** block pointer stored in the first 4 bytes of the block. -** -** Offset iOff is an absolute offset - not subject to any block redirection. -** However any block pointer followed is. Use pSeg->pRedirect in this case. -** -** Return LSM_OK if successful or an lsm error code if an error occurs. -*/ -static int fsSubtractOffset( - FileSystem *pFS, - Segment *pSeg, - i64 iOff, - int iSub, - i64 *piRes -){ - i64 iStart; - int iBlk = 0; - int rc; - - assert( pFS->pCompress ); - - iStart = fsFirstPageOnBlock(pFS, fsPageToBlock(pFS, iOff)); - if( (iOff-iSub)>=iStart ){ - *piRes = (iOff-iSub); - return LSM_OK; - } - - rc = fsBlockPrev(pFS, pSeg, fsPageToBlock(pFS, iOff), &iBlk); - *piRes = fsLastPageOnBlock(pFS, iBlk) - iSub + (iOff - iStart + 1); - return rc; -} - -/* -** Add iAdd to database file offset iOff and set *piRes to the -** result. If doing so means passing the end of a block, follow the -** block pointer stored in the last 4 bytes of the block. -** -** Offset iOff is an absolute offset - not subject to any block redirection. -** However any block pointer followed is. Use pSeg->pRedirect in this case. -** -** Return LSM_OK if successful or an lsm error code if an error occurs. -*/ -static int fsAddOffset( - FileSystem *pFS, - Segment *pSeg, - i64 iOff, - int iAdd, - i64 *piRes -){ - i64 iEob; - int iBlk; - int rc; - - assert( pFS->pCompress ); - - iEob = fsLastPageOnPagesBlock(pFS, iOff); - if( (iOff+iAdd)<=iEob ){ - *piRes = (iOff+iAdd); - return LSM_OK; - } - - rc = fsBlockNext(pFS, pSeg, fsPageToBlock(pFS, iOff), &iBlk); - *piRes = fsFirstPageOnBlock(pFS, iBlk) + iAdd - (iEob - iOff + 1); - return rc; -} - -/* -** If it is not already allocated, allocate either the FileSystem.aOBuffer (if -** bWrite is true) or the FileSystem.aIBuffer (if bWrite is false). Return -** LSM_OK if successful if the attempt to allocate memory fails. -*/ -static int fsAllocateBuffer(FileSystem *pFS, int bWrite){ - u8 **pp; /* Pointer to either aIBuffer or aOBuffer */ - - assert( pFS->pCompress ); - - /* If neither buffer has been allocated, figure out how large they - ** should be. Store this value in FileSystem.nBuffer. */ - if( pFS->nBuffer==0 ){ - assert( pFS->aIBuffer==0 && pFS->aOBuffer==0 ); - pFS->nBuffer = pFS->pCompress->xBound(pFS->pCompress->pCtx, pFS->nPagesize); - if( pFS->nBuffer<(pFS->szSector+6) ){ - pFS->nBuffer = pFS->szSector+6; - } - } - - pp = (bWrite ? &pFS->aOBuffer : &pFS->aIBuffer); - if( *pp==0 ){ - *pp = lsmMalloc(pFS->pEnv, LSM_MAX(pFS->nBuffer, pFS->nPagesize)); - if( *pp==0 ) return LSM_NOMEM_BKPT; - } - - return LSM_OK; -} - -/* -** This function is only called in compressed database mode. It reads and -** uncompresses the compressed data for page pPg from the database and -** populates the pPg->aData[] buffer and pPg->nCompress field. -** -** It is possible that instead of a page record, there is free space -** at offset pPg->iPgno. In this case no data is read from the file, but -** output variable *pnSpace is set to the total number of free bytes. -** -** LSM_OK is returned if successful, or an LSM error code otherwise. -*/ -static int fsReadPagedata( - FileSystem *pFS, /* File-system handle */ - Segment *pSeg, /* pPg is part of this segment */ - Page *pPg, /* Page to read and uncompress data for */ - int *pnSpace /* OUT: Total bytes of free space */ -){ - lsm_compress *p = pFS->pCompress; - i64 iOff = pPg->iPg; - u8 aSz[3]; - int rc; - - assert( p && pPg->nCompress==0 ); - - if( fsAllocateBuffer(pFS, 0) ) return LSM_NOMEM; - - rc = fsReadData(pFS, pSeg, iOff, aSz, sizeof(aSz)); - - if( rc==LSM_OK ){ - int bFree; - if( aSz[0] & 0x80 ){ - pPg->nCompress = (int)getRecordSize(aSz, &bFree); - }else{ - pPg->nCompress = (int)aSz[0] - sizeof(aSz)*2; - bFree = 1; - } - if( bFree ){ - if( pnSpace ){ - *pnSpace = pPg->nCompress + sizeof(aSz)*2; - }else{ - rc = LSM_CORRUPT_BKPT; - } - }else{ - rc = fsAddOffset(pFS, pSeg, iOff, 3, &iOff); - if( rc==LSM_OK ){ - if( pPg->nCompress>pFS->nBuffer ){ - rc = LSM_CORRUPT_BKPT; - }else{ - rc = fsReadData(pFS, pSeg, iOff, pFS->aIBuffer, pPg->nCompress); - } - if( rc==LSM_OK ){ - int n = pFS->nPagesize; - rc = p->xUncompress(p->pCtx, - (char *)pPg->aData, &n, - (const char *)pFS->aIBuffer, pPg->nCompress - ); - if( rc==LSM_OK && n!=pPg->pFS->nPagesize ){ - rc = LSM_CORRUPT_BKPT; - } - } - } - } - } - return rc; -} - -/* -** Return a handle for a database page. -** -** If this file-system object is accessing a compressed database it may be -** that there is no page record at database file offset iPg. Instead, there -** may be a free space record. In this case, set *ppPg to NULL and *pnSpace -** to the total number of free bytes before returning. -** -** If no error occurs, LSM_OK is returned. Otherwise, an lsm error code. -*/ -static int fsPageGet( - FileSystem *pFS, /* File-system handle */ - Segment *pSeg, /* Block redirection to use (or NULL) */ - LsmPgno iPg, /* Page id */ - int noContent, /* True to not load content from disk */ - Page **ppPg, /* OUT: New page handle */ - int *pnSpace /* OUT: Bytes of free space */ -){ - Page *p; - int iHash; - int rc = LSM_OK; - - /* In most cases iReal is the same as iPg. Except, if pSeg->pRedirect is - ** not NULL, and the block containing iPg has been redirected, then iReal - ** is the page number after redirection. */ - LsmPgno iReal = lsmFsRedirectPage(pFS, (pSeg ? pSeg->pRedirect : 0), iPg); - - assert_lists_are_ok(pFS); - assert( iPg>=fsFirstPageOnBlock(pFS, 1) ); - assert( iReal>=fsFirstPageOnBlock(pFS, 1) ); - *ppPg = 0; - - /* Search the hash-table for the page */ - p = fsPageFindInHash(pFS, iReal, &iHash); - - if( p ){ - assert( p->flags & PAGE_FREE ); - if( p->nRef==0 ) fsPageRemoveFromLru(pFS, p); - }else{ - - if( fsMmapPage(pFS, iReal) ){ - i64 iEnd = (i64)iReal * pFS->nPagesize; - fsGrowMapping(pFS, iEnd, &rc); - if( rc!=LSM_OK ) return rc; - - if( pFS->pFree ){ - p = pFS->pFree; - pFS->pFree = p->pFreeNext; - assert( p->nRef==0 ); - }else{ - p = lsmMallocZeroRc(pFS->pEnv, sizeof(Page), &rc); - if( rc ) return rc; - p->pFS = pFS; - } - p->aData = &((u8 *)pFS->pMap)[pFS->nPagesize * (iReal-1)]; - p->iPg = iReal; - - /* This page now carries a pointer to the mapping. Link it in to - ** the FileSystem.pMapped list. */ - assert( p->pMappedNext==0 ); - p->pMappedNext = pFS->pMapped; - pFS->pMapped = p; - - assert( pFS->pCompress==0 ); - assert( (p->flags & PAGE_FREE)==0 ); - }else{ - rc = fsPageBuffer(pFS, &p); - if( rc==LSM_OK ){ - int nSpace = 0; - p->iPg = iReal; - p->nRef = 0; - p->pFS = pFS; - assert( p->flags==0 || p->flags==PAGE_FREE ); - -#ifdef LSM_DEBUG - memset(p->aData, 0x56, pFS->nPagesize); -#endif - assert( p->pLruNext==0 && p->pLruPrev==0 ); - if( noContent==0 ){ - if( pFS->pCompress ){ - rc = fsReadPagedata(pFS, pSeg, p, &nSpace); - }else{ - int nByte = pFS->nPagesize; - i64 iOff = (i64)(iReal-1) * pFS->nPagesize; - rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, p->aData, nByte); - } - pFS->nRead++; - } - - /* If the xRead() call was successful (or not attempted), link the - ** page into the page-cache hash-table. Otherwise, if it failed, - ** free the buffer. */ - if( rc==LSM_OK && nSpace==0 ){ - p->pHashNext = pFS->apHash[iHash]; - pFS->apHash[iHash] = p; - }else{ - fsPageBufferFree(p); - p = 0; - if( pnSpace ) *pnSpace = nSpace; - } - } - } - - assert( (rc==LSM_OK && (p || (pnSpace && *pnSpace))) - || (rc!=LSM_OK && p==0) - ); - } - - if( rc==LSM_OK && p ){ - if( pFS->pCompress==0 && (fsIsLast(pFS, iReal) || fsIsFirst(pFS, iReal)) ){ - p->nData = pFS->nPagesize - 4; - if( fsIsFirst(pFS, iReal) && p->nRef==0 ){ - p->aData += 4; - p->flags |= PAGE_HASPREV; - } - }else{ - p->nData = pFS->nPagesize; - } - pFS->nOut += (p->nRef==0); - p->nRef++; - } - *ppPg = p; - return rc; -} - -/* -** Read the 64-bit checkpoint id of the checkpoint currently stored on meta -** page iMeta of the database file. If no error occurs, store the id value -** in *piVal and return LSM_OK. Otherwise, return an LSM error code and leave -** *piVal unmodified. -** -** If a checkpointer connection is currently updating meta-page iMeta, or an -** earlier checkpointer crashed while doing so, the value read into *piVal -** may be garbage. It is the callers responsibility to deal with this. -*/ -int lsmFsReadSyncedId(lsm_db *db, int iMeta, i64 *piVal){ - FileSystem *pFS = db->pFS; - int rc = LSM_OK; - - assert( iMeta==1 || iMeta==2 ); - if( pFS->nMapLimit>0 ){ - fsGrowMapping(pFS, iMeta*LSM_META_PAGE_SIZE, &rc); - if( rc==LSM_OK ){ - *piVal = (i64)lsmGetU64(&((u8 *)pFS->pMap)[(iMeta-1)*LSM_META_PAGE_SIZE]); - } - }else{ - MetaPage *pMeta = 0; - rc = lsmFsMetaPageGet(pFS, 0, iMeta, &pMeta); - if( rc==LSM_OK ){ - *piVal = (i64)lsmGetU64(pMeta->aData); - lsmFsMetaPageRelease(pMeta); - } - } - - return rc; -} - - -/* -** Return true if the first or last page of segment pRun falls between iFirst -** and iLast, inclusive, and pRun is not equal to pIgnore. -*/ -static int fsRunEndsBetween( - Segment *pRun, - Segment *pIgnore, - LsmPgno iFirst, - LsmPgno iLast -){ - return (pRun!=pIgnore && ( - (pRun->iFirst>=iFirst && pRun->iFirst<=iLast) - || (pRun->iLastPg>=iFirst && pRun->iLastPg<=iLast) - )); -} - -/* -** Return true if level pLevel contains a segment other than pIgnore for -** which the first or last page is between iFirst and iLast, inclusive. -*/ -static int fsLevelEndsBetween( - Level *pLevel, - Segment *pIgnore, - LsmPgno iFirst, - LsmPgno iLast -){ - int i; - - if( fsRunEndsBetween(&pLevel->lhs, pIgnore, iFirst, iLast) ){ - return 1; - } - for(i=0; inRight; i++){ - if( fsRunEndsBetween(&pLevel->aRhs[i], pIgnore, iFirst, iLast) ){ - return 1; - } - } - - return 0; -} - -/* -** Block iBlk is no longer in use by segment pIgnore. If it is not in use -** by any other segment, move it to the free block list. -*/ -static int fsFreeBlock( - FileSystem *pFS, /* File system object */ - Snapshot *pSnapshot, /* Worker snapshot */ - Segment *pIgnore, /* Ignore this run when searching */ - int iBlk /* Block number of block to free */ -){ - int rc = LSM_OK; /* Return code */ - LsmPgno iFirst; /* First page on block iBlk */ - LsmPgno iLast; /* Last page on block iBlk */ - Level *pLevel; /* Used to iterate through levels */ - - int iIn; /* Used to iterate through append points */ - int iOut = 0; /* Used to output append points */ - LsmPgno *aApp = pSnapshot->aiAppend; - - iFirst = fsFirstPageOnBlock(pFS, iBlk); - iLast = fsLastPageOnBlock(pFS, iBlk); - - /* Check if any other run in the snapshot has a start or end page - ** within this block. If there is such a run, return early. */ - for(pLevel=lsmDbSnapshotLevel(pSnapshot); pLevel; pLevel=pLevel->pNext){ - if( fsLevelEndsBetween(pLevel, pIgnore, iFirst, iLast) ){ - return LSM_OK; - } - } - - /* Remove any entries that lie on this block from the append-list. */ - for(iIn=0; iIniLast ){ - aApp[iOut++] = aApp[iIn]; - } - } - while( iOutpDb, iBlk); - } - return rc; -} - -/* -** Delete or otherwise recycle the blocks currently occupied by run pDel. -*/ -int lsmFsSortedDelete( - FileSystem *pFS, - Snapshot *pSnapshot, - int bZero, /* True to zero the Segment structure */ - Segment *pDel -){ - if( pDel->iFirst ){ - int rc = LSM_OK; - - int iBlk; - int iLastBlk; - - iBlk = fsPageToBlock(pFS, pDel->iFirst); - iLastBlk = fsPageToBlock(pFS, pDel->iLastPg); - - /* Mark all blocks currently used by this sorted run as free */ - while( iBlk && rc==LSM_OK ){ - int iNext = 0; - if( iBlk!=iLastBlk ){ - rc = fsBlockNext(pFS, pDel, iBlk, &iNext); - }else if( bZero==0 && pDel->iLastPg!=fsLastPageOnBlock(pFS, iLastBlk) ){ - break; - } - rc = fsFreeBlock(pFS, pSnapshot, pDel, iBlk); - iBlk = iNext; - } - - if( pDel->pRedirect ){ - assert( pDel->pRedirect==&pSnapshot->redirect ); - pSnapshot->redirect.n = 0; - } - - if( bZero ) memset(pDel, 0, sizeof(Segment)); - } - return LSM_OK; -} - -/* -** aPgno is an array containing nPgno page numbers. Return the smallest page -** number from the array that falls on block iBlk. Or, if none of the pages -** in aPgno[] fall on block iBlk, return 0. -*/ -static LsmPgno firstOnBlock( - FileSystem *pFS, - int iBlk, - LsmPgno *aPgno, - int nPgno -){ - LsmPgno iRet = 0; - int i; - for(i=0; ipRedirect, iPg)); -} - -/* -** Return true if the second argument is not NULL and any of the first -** last or root pages lie on a redirected block. -*/ -static int fsSegmentRedirects(FileSystem *pFS, Segment *p){ - return (p && ( - fsPageRedirects(pFS, p, p->iFirst) - || fsPageRedirects(pFS, p, p->iRoot) - || fsPageRedirects(pFS, p, p->iLastPg) - )); -} -#endif - -/* -** Argument aPgno is an array of nPgno page numbers. All pages belong to -** the segment pRun. This function gobbles from the start of the run to the -** first page that appears in aPgno[] (i.e. so that the aPgno[] entry is -** the new first page of the run). -*/ -void lsmFsGobble( - lsm_db *pDb, - Segment *pRun, - LsmPgno *aPgno, - int nPgno -){ - int rc = LSM_OK; - FileSystem *pFS = pDb->pFS; - Snapshot *pSnapshot = pDb->pWorker; - int iBlk; - - assert( pRun->nSize>0 ); - assert( 0==fsSegmentRedirects(pFS, pRun) ); - assert( nPgno>0 && 0==fsPageRedirects(pFS, pRun, aPgno[0]) ); - - iBlk = fsPageToBlock(pFS, pRun->iFirst); - pRun->nSize += (pRun->iFirst - fsFirstPageOnBlock(pFS, iBlk)); - - while( rc==LSM_OK ){ - int iNext = 0; - LsmPgno iFirst = firstOnBlock(pFS, iBlk, aPgno, nPgno); - if( iFirst ){ - pRun->iFirst = iFirst; - break; - } - rc = fsBlockNext(pFS, pRun, iBlk, &iNext); - if( rc==LSM_OK ) rc = fsFreeBlock(pFS, pSnapshot, pRun, iBlk); - pRun->nSize -= ( - 1 + fsLastPageOnBlock(pFS, iBlk) - fsFirstPageOnBlock(pFS, iBlk) - ); - iBlk = iNext; - } - - pRun->nSize -= (pRun->iFirst - fsFirstPageOnBlock(pFS, iBlk)); - assert( pRun->nSize>0 ); -} - -/* -** This function is only used in compressed database mode. -** -** Argument iPg is the page number (byte offset) of a page within segment -** pSeg. The page record, including all headers, is nByte bytes in size. -** Before returning, set *piNext to the page number of the next page in -** the segment, or to zero if iPg is the last. -** -** In other words, do: -** -** *piNext = iPg + nByte; -** -** But take block overflow and redirection into account. -*/ -static int fsNextPageOffset( - FileSystem *pFS, /* File system object */ - Segment *pSeg, /* Segment to move within */ - LsmPgno iPg, /* Offset of current page */ - int nByte, /* Size of current page including headers */ - LsmPgno *piNext /* OUT: Offset of next page. Or zero (EOF) */ -){ - LsmPgno iNext; - int rc; - - assert( pFS->pCompress ); - - rc = fsAddOffset(pFS, pSeg, iPg, nByte-1, &iNext); - if( pSeg && iNext==pSeg->iLastPg ){ - iNext = 0; - }else if( rc==LSM_OK ){ - rc = fsAddOffset(pFS, pSeg, iNext, 1, &iNext); - } - - *piNext = iNext; - return rc; -} - -/* -** This function is only used in compressed database mode. -** -** Argument iPg is the page number of a pagethat appears in segment pSeg. -** This function determines the page number of the previous page in the -** same run. *piPrev is set to the previous page number before returning. -** -** LSM_OK is returned if no error occurs. Otherwise, an lsm error code. -** If any value other than LSM_OK is returned, then the final value of -** *piPrev is undefined. -*/ -static int fsGetPageBefore( - FileSystem *pFS, - Segment *pSeg, - LsmPgno iPg, - LsmPgno *piPrev -){ - u8 aSz[3]; - int rc; - i64 iRead; - - assert( pFS->pCompress ); - - rc = fsSubtractOffset(pFS, pSeg, iPg, sizeof(aSz), &iRead); - if( rc==LSM_OK ) rc = fsReadData(pFS, pSeg, iRead, aSz, sizeof(aSz)); - - if( rc==LSM_OK ){ - int bFree; - int nSz; - if( aSz[2] & 0x80 ){ - nSz = getRecordSize(aSz, &bFree) + sizeof(aSz)*2; - }else{ - nSz = (int)(aSz[2] & 0x7F); - bFree = 1; - } - rc = fsSubtractOffset(pFS, pSeg, iPg, nSz, piPrev); - } - - return rc; -} - -/* -** The first argument to this function is a valid reference to a database -** file page that is part of a sorted run. If parameter eDir is -1, this -** function attempts to locate and load the previous page in the same run. -** Or, if eDir is +1, it attempts to find the next page in the same run. -** The results of passing an eDir value other than positive or negative one -** are undefined. -** -** If parameter pRun is not NULL then it must point to the run that page -** pPg belongs to. In this case, if pPg is the first or last page of the -** run, and the request is for the previous or next page, respectively, -** *ppNext is set to NULL before returning LSM_OK. If pRun is NULL, then it -** is assumed that the next or previous page, as requested, exists. -** -** If the previous/next page does exist and is successfully loaded, *ppNext -** is set to point to it and LSM_OK is returned. Otherwise, if an error -** occurs, *ppNext is set to NULL and and lsm error code returned. -** -** Page references returned by this function should be released by the -** caller using lsmFsPageRelease(). -*/ -int lsmFsDbPageNext(Segment *pRun, Page *pPg, int eDir, Page **ppNext){ - int rc = LSM_OK; - FileSystem *pFS = pPg->pFS; - LsmPgno iPg = pPg->iPg; - - assert( 0==fsSegmentRedirects(pFS, pRun) ); - if( pFS->pCompress ){ - int nSpace = pPg->nCompress + 2*3; - - do { - if( eDir>0 ){ - rc = fsNextPageOffset(pFS, pRun, iPg, nSpace, &iPg); - }else{ - if( iPg==pRun->iFirst ){ - iPg = 0; - }else{ - rc = fsGetPageBefore(pFS, pRun, iPg, &iPg); - } - } - - nSpace = 0; - if( iPg!=0 ){ - rc = fsPageGet(pFS, pRun, iPg, 0, ppNext, &nSpace); - assert( (*ppNext==0)==(rc!=LSM_OK || nSpace>0) ); - }else{ - *ppNext = 0; - } - }while( nSpace>0 && rc==LSM_OK ); - - }else{ - Redirect *pRedir = pRun ? pRun->pRedirect : 0; - assert( eDir==1 || eDir==-1 ); - if( eDir<0 ){ - if( pRun && iPg==pRun->iFirst ){ - *ppNext = 0; - return LSM_OK; - }else if( fsIsFirst(pFS, iPg) ){ - assert( pPg->flags & PAGE_HASPREV ); - iPg = fsLastPageOnBlock(pFS, lsmGetU32(&pPg->aData[-4])); - }else{ - iPg--; - } - }else{ - if( pRun ){ - if( iPg==pRun->iLastPg ){ - *ppNext = 0; - return LSM_OK; - } - } - - if( fsIsLast(pFS, iPg) ){ - int iBlk = fsRedirectBlock( - pRedir, lsmGetU32(&pPg->aData[pFS->nPagesize-4]) - ); - iPg = fsFirstPageOnBlock(pFS, iBlk); - }else{ - iPg++; - } - } - rc = fsPageGet(pFS, pRun, iPg, 0, ppNext, 0); - } - - return rc; -} - -/* -** This function is called when creating a new segment to determine if the -** first part of it can be written following an existing segment on an -** already allocated block. If it is possible, the page number of the first -** page to use for the new segment is returned. Otherwise zero. -** -** If argument pLvl is not NULL, then this function will not attempt to -** start the new segment immediately following any segment that is part -** of the right-hand-side of pLvl. -*/ -static LsmPgno findAppendPoint(FileSystem *pFS, Level *pLvl){ - int i; - LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend; - LsmPgno iRet = 0; - - for(i=LSM_APPLIST_SZ-1; iRet==0 && i>=0; i--){ - if( (iRet = aiAppend[i]) ){ - if( pLvl ){ - int iBlk = fsPageToBlock(pFS, iRet); - int j; - for(j=0; iRet && jnRight; j++){ - if( fsPageToBlock(pFS, pLvl->aRhs[j].iLastPg)==iBlk ){ - iRet = 0; - } - } - } - if( iRet ) aiAppend[i] = 0; - } - } - return iRet; -} - -/* -** Append a page to the left-hand-side of pLvl. Set the ref-count to 1 and -** return a pointer to it. The page is writable until either -** lsmFsPagePersist() is called on it or the ref-count drops to zero. -*/ -int lsmFsSortedAppend( - FileSystem *pFS, - Snapshot *pSnapshot, - Level *pLvl, - int bDefer, - Page **ppOut -){ - int rc = LSM_OK; - Page *pPg = 0; - LsmPgno iApp = 0; - LsmPgno iNext = 0; - Segment *p = &pLvl->lhs; - LsmPgno iPrev = p->iLastPg; - - *ppOut = 0; - assert( p->pRedirect==0 ); - - if( pFS->pCompress || bDefer ){ - /* In compressed database mode the page is not assigned a page number - ** or location in the database file at this point. This will be done - ** by the lsmFsPagePersist() call. */ - rc = fsPageBuffer(pFS, &pPg); - if( rc==LSM_OK ){ - pPg->pFS = pFS; - pPg->pSeg = p; - pPg->iPg = 0; - pPg->flags |= PAGE_DIRTY; - pPg->nData = pFS->nPagesize; - assert( pPg->aData ); - if( pFS->pCompress==0 ) pPg->nData -= 4; - - pPg->nRef = 1; - pFS->nOut++; - } - }else{ - if( iPrev==0 ){ - iApp = findAppendPoint(pFS, pLvl); - }else if( fsIsLast(pFS, iPrev) ){ - int iNext2; - rc = fsBlockNext(pFS, 0, fsPageToBlock(pFS, iPrev), &iNext2); - if( rc!=LSM_OK ) return rc; - iApp = fsFirstPageOnBlock(pFS, iNext2); - }else{ - iApp = iPrev + 1; - } - - /* If this is the first page allocated, or if the page allocated is the - ** last in the block, also allocate the next block here. */ - if( iApp==0 || fsIsLast(pFS, iApp) ){ - int iNew; /* New block number */ - - rc = lsmBlockAllocate(pFS->pDb, 0, &iNew); - if( rc!=LSM_OK ) return rc; - if( iApp==0 ){ - iApp = fsFirstPageOnBlock(pFS, iNew); - }else{ - iNext = fsFirstPageOnBlock(pFS, iNew); - } - } - - /* Grab the new page. */ - pPg = 0; - rc = fsPageGet(pFS, 0, iApp, 1, &pPg, 0); - assert( rc==LSM_OK || pPg==0 ); - - /* If this is the first or last page of a block, fill in the pointer - ** value at the end of the new page. */ - if( rc==LSM_OK ){ - p->nSize++; - p->iLastPg = iApp; - if( p->iFirst==0 ) p->iFirst = iApp; - pPg->flags |= PAGE_DIRTY; - - if( fsIsLast(pFS, iApp) ){ - lsmPutU32(&pPg->aData[pFS->nPagesize-4], fsPageToBlock(pFS, iNext)); - }else if( fsIsFirst(pFS, iApp) ){ - lsmPutU32(&pPg->aData[-4], fsPageToBlock(pFS, iPrev)); - } - } - } - - *ppOut = pPg; - return rc; -} - -/* -** Mark the segment passed as the second argument as finished. Once a segment -** is marked as finished it is not possible to append any further pages to -** it. -** -** Return LSM_OK if successful or an lsm error code if an error occurs. -*/ -int lsmFsSortedFinish(FileSystem *pFS, Segment *p){ - int rc = LSM_OK; - if( p && p->iLastPg ){ - assert( p->pRedirect==0 ); - - /* Check if the last page of this run happens to be the last of a block. - ** If it is, then an extra block has already been allocated for this run. - ** Shift this extra block back to the free-block list. - ** - ** Otherwise, add the first free page in the last block used by the run - ** to the lAppend list. - */ - if( fsLastPageOnPagesBlock(pFS, p->iLastPg)!=p->iLastPg ){ - int i; - LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend; - for(i=0; iiLastPg+1; - break; - } - } - }else if( pFS->pCompress==0 ){ - Page *pLast; - rc = fsPageGet(pFS, 0, p->iLastPg, 0, &pLast, 0); - if( rc==LSM_OK ){ - int iBlk = (int)lsmGetU32(&pLast->aData[pFS->nPagesize-4]); - lsmBlockRefree(pFS->pDb, iBlk); - lsmFsPageRelease(pLast); - } - }else{ - int iBlk = 0; - rc = fsBlockNext(pFS, p, fsPageToBlock(pFS, p->iLastPg), &iBlk); - if( rc==LSM_OK ){ - lsmBlockRefree(pFS->pDb, iBlk); - } - } - } - return rc; -} - -/* -** Obtain a reference to page number iPg. -** -** Return LSM_OK if successful, or an lsm error code if an error occurs. -*/ -int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, LsmPgno iPg, Page **ppPg){ - return fsPageGet(pFS, pSeg, iPg, 0, ppPg, 0); -} - -/* -** Obtain a reference to the last page in the segment passed as the -** second argument. -** -** Return LSM_OK if successful, or an lsm error code if an error occurs. -*/ -int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg){ - int rc; - LsmPgno iPg = pSeg->iLastPg; - if( pFS->pCompress ){ - int nSpace; - iPg++; - do { - nSpace = 0; - rc = fsGetPageBefore(pFS, pSeg, iPg, &iPg); - if( rc==LSM_OK ){ - rc = fsPageGet(pFS, pSeg, iPg, 0, ppPg, &nSpace); - } - }while( rc==LSM_OK && nSpace>0 ); - - }else{ - rc = fsPageGet(pFS, pSeg, iPg, 0, ppPg, 0); - } - return rc; -} - -/* -** Return a reference to meta-page iPg. If successful, LSM_OK is returned -** and *ppPg populated with the new page reference. The reference should -** be released by the caller using lsmFsPageRelease(). -** -** Otherwise, if an error occurs, *ppPg is set to NULL and an LSM error -** code is returned. -*/ -int lsmFsMetaPageGet( - FileSystem *pFS, /* File-system connection */ - int bWrite, /* True for write access, false for read */ - int iPg, /* Either 1 or 2 */ - MetaPage **ppPg /* OUT: Pointer to MetaPage object */ -){ - int rc = LSM_OK; - MetaPage *pPg; - assert( iPg==1 || iPg==2 ); - - pPg = lsmMallocZeroRc(pFS->pEnv, sizeof(Page), &rc); - - if( pPg ){ - i64 iOff = (iPg-1) * pFS->nMetasize; - if( pFS->nMapLimit>0 ){ - fsGrowMapping(pFS, 2*pFS->nMetasize, &rc); - pPg->aData = (u8 *)(pFS->pMap) + iOff; - }else{ - pPg->aData = lsmMallocRc(pFS->pEnv, pFS->nMetasize, &rc); - if( rc==LSM_OK && bWrite==0 ){ - rc = lsmEnvRead( - pFS->pEnv, pFS->fdDb, iOff, pPg->aData, pFS->nMetaRwSize - ); - } -#ifndef NDEBUG - /* pPg->aData causes an uninitialized access via a downstream write(). - After discussion on this list, this memory should not, for performance - reasons, be memset. However, tracking down "real" misuse is more - difficult with this "false" positive, so it is set when NDEBUG. - */ - else if( rc==LSM_OK ){ - memset( pPg->aData, 0x77, pFS->nMetasize ); - } -#endif - } - - if( rc!=LSM_OK ){ - if( pFS->nMapLimit==0 ) lsmFree(pFS->pEnv, pPg->aData); - lsmFree(pFS->pEnv, pPg); - pPg = 0; - }else{ - pPg->iPg = iPg; - pPg->bWrite = bWrite; - pPg->pFS = pFS; - } - } - - *ppPg = pPg; - return rc; -} - -/* -** Release a meta-page reference obtained via a call to lsmFsMetaPageGet(). -*/ -int lsmFsMetaPageRelease(MetaPage *pPg){ - int rc = LSM_OK; - if( pPg ){ - FileSystem *pFS = pPg->pFS; - - if( pFS->nMapLimit==0 ){ - if( pPg->bWrite ){ - i64 iOff = (pPg->iPg==2 ? pFS->nMetasize : 0); - int nWrite = pFS->nMetaRwSize; - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iOff, pPg->aData, nWrite); - } - lsmFree(pFS->pEnv, pPg->aData); - } - - lsmFree(pFS->pEnv, pPg); - } - return rc; -} - -/* -** Return a pointer to a buffer containing the data associated with the -** meta-page passed as the first argument. If parameter pnData is not NULL, -** set *pnData to the size of the meta-page in bytes before returning. -*/ -u8 *lsmFsMetaPageData(MetaPage *pPg, int *pnData){ - if( pnData ) *pnData = pPg->pFS->nMetaRwSize; - return pPg->aData; -} - -/* -** Return true if page is currently writable. This is used in assert() -** statements only. -*/ -#ifndef NDEBUG -int lsmFsPageWritable(Page *pPg){ - return (pPg->flags & PAGE_DIRTY) ? 1 : 0; -} -#endif - -/* -** This is called when block iFrom is being redirected to iTo. If page -** number (*piPg) lies on block iFrom, then calculate the equivalent -** page on block iTo and set *piPg to this value before returning. -*/ -static void fsMovePage( - FileSystem *pFS, /* File system object */ - int iTo, /* Destination block */ - int iFrom, /* Source block */ - LsmPgno *piPg /* IN/OUT: Page number */ -){ - LsmPgno iPg = *piPg; - if( iFrom==fsPageToBlock(pFS, iPg) ){ - const int nPagePerBlock = ( - pFS->pCompress ? pFS ->nBlocksize : (pFS->nBlocksize / pFS->nPagesize) - ); - *piPg = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock; - } -} - -/* -** Copy the contents of block iFrom to block iTo. -** -** It is safe to assume that there are no outstanding references to pages -** on block iTo. And that block iFrom is not currently being written. In -** other words, the data can be read and written directly. -*/ -int lsmFsMoveBlock(FileSystem *pFS, Segment *pSeg, int iTo, int iFrom){ - Snapshot *p = pFS->pDb->pWorker; - int rc = LSM_OK; - int i; - i64 nMap; - - i64 iFromOff = (i64)(iFrom-1) * pFS->nBlocksize; - i64 iToOff = (i64)(iTo-1) * pFS->nBlocksize; - - assert( iTo!=1 ); - assert( iFrom>iTo ); - - /* Grow the mapping as required. */ - nMap = LSM_MIN(pFS->nMapLimit, (i64)iFrom * pFS->nBlocksize); - fsGrowMapping(pFS, nMap, &rc); - - if( rc==LSM_OK ){ - const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize); - int nSz = pFS->nPagesize; - u8 *aBuf = 0; - u8 *aData = 0; - - for(i=0; rc==LSM_OK && inMapLimit ){ - u8 *aMap = (u8 *)(pFS->pMap); - aData = &aMap[iOff]; - }else{ - if( aBuf==0 ){ - aBuf = (u8 *)lsmMallocRc(pFS->pEnv, nSz, &rc); - if( aBuf==0 ) break; - } - aData = aBuf; - rc = lsmEnvRead(pFS->pEnv, pFS->fdDb, iOff, aData, nSz); - } - - /* Copy aData to the to page */ - if( rc==LSM_OK ){ - iOff = iToOff + i*nSz; - if( (iOff+nSz)<=pFS->nMapLimit ){ - u8 *aMap = (u8 *)(pFS->pMap); - memcpy(&aMap[iOff], aData, nSz); - }else{ - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iOff, aData, nSz); - } - } - } - lsmFree(pFS->pEnv, aBuf); - lsmFsPurgeCache(pFS); - } - - /* Update append-point list if necessary */ - for(i=0; iaiAppend[i]); - } - - /* Update the Segment structure itself */ - fsMovePage(pFS, iTo, iFrom, &pSeg->iFirst); - fsMovePage(pFS, iTo, iFrom, &pSeg->iLastPg); - fsMovePage(pFS, iTo, iFrom, &pSeg->iRoot); - - return rc; -} - -/* -** Append raw data to a segment. Return the database file offset that the -** data is written to (this may be used as the page number if the data -** being appended is a new page record). -** -** This function is only used in compressed database mode. -*/ -static LsmPgno fsAppendData( - FileSystem *pFS, /* File-system handle */ - Segment *pSeg, /* Segment to append to */ - const u8 *aData, /* Buffer containing data to write */ - int nData, /* Size of buffer aData[] in bytes */ - int *pRc /* IN/OUT: Error code */ -){ - LsmPgno iRet = 0; - int rc = *pRc; - assert( pFS->pCompress ); - if( rc==LSM_OK ){ - int nRem = 0; - int nWrite = 0; - LsmPgno iLastOnBlock; - LsmPgno iApp = pSeg->iLastPg+1; - - /* If this is the first data written into the segment, find an append-point - ** or allocate a new block. */ - if( iApp==1 ){ - pSeg->iFirst = iApp = findAppendPoint(pFS, 0); - if( iApp==0 ){ - int iBlk; - rc = lsmBlockAllocate(pFS->pDb, 0, &iBlk); - pSeg->iFirst = iApp = fsFirstPageOnBlock(pFS, iBlk); - } - } - iRet = iApp; - - /* Write as much data as is possible at iApp (usually all of it). */ - iLastOnBlock = fsLastPageOnPagesBlock(pFS, iApp); - if( rc==LSM_OK ){ - int nSpace = (int)(iLastOnBlock - iApp + 1); - nWrite = LSM_MIN(nData, nSpace); - nRem = nData - nWrite; - assert( nWrite>=0 ); - if( nWrite!=0 ){ - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iApp, aData, nWrite); - } - iApp += nWrite; - } - - /* If required, allocate a new block and write the rest of the data - ** into it. Set the next and previous block pointers to link the new - ** block to the old. */ - assert( nRem<=0 || (iApp-1)==iLastOnBlock ); - if( rc==LSM_OK && (iApp-1)==iLastOnBlock ){ - u8 aPtr[4]; /* Space to serialize a u32 */ - int iBlk; /* New block number */ - - if( nWrite>0 ){ - /* Allocate a new block. */ - rc = lsmBlockAllocate(pFS->pDb, 0, &iBlk); - - /* Set the "next" pointer on the old block */ - if( rc==LSM_OK ){ - assert( iApp==(fsPageToBlock(pFS, iApp)*pFS->nBlocksize)-4 ); - lsmPutU32(aPtr, iBlk); - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iApp, aPtr, sizeof(aPtr)); - } - - /* Set the "prev" pointer on the new block */ - if( rc==LSM_OK ){ - LsmPgno iWrite; - lsmPutU32(aPtr, fsPageToBlock(pFS, iApp)); - iWrite = fsFirstPageOnBlock(pFS, iBlk); - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iWrite-4, aPtr, sizeof(aPtr)); - if( nRem>0 ) iApp = iWrite; - } - }else{ - /* The next block is already allocated. */ - assert( nRem>0 ); - assert( pSeg->pRedirect==0 ); - rc = fsBlockNext(pFS, 0, fsPageToBlock(pFS, iApp), &iBlk); - iRet = iApp = fsFirstPageOnBlock(pFS, iBlk); - } - - /* Write the remaining data into the new block */ - if( rc==LSM_OK && nRem>0 ){ - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iApp, &aData[nWrite], nRem); - iApp += nRem; - } - } - - pSeg->iLastPg = iApp-1; - *pRc = rc; - } - - return iRet; -} - -/* -** This function is only called in compressed database mode. It -** compresses the contents of page pPg and writes the result to the -** buffer at pFS->aOBuffer. The size of the compressed data is stored in -** pPg->nCompress. -** -** If buffer pFS->aOBuffer[] has not been allocated then this function -** allocates it. If this fails, LSM_NOMEM is returned. Otherwise, LSM_OK. -*/ -static int fsCompressIntoBuffer(FileSystem *pFS, Page *pPg){ - lsm_compress *p = pFS->pCompress; - - if( fsAllocateBuffer(pFS, 1) ) return LSM_NOMEM; - assert( pPg->nData==pFS->nPagesize ); - - pPg->nCompress = pFS->nBuffer; - return p->xCompress(p->pCtx, - (char *)pFS->aOBuffer, &pPg->nCompress, - (const char *)pPg->aData, pPg->nData - ); -} - -/* -** Append a new page to segment pSeg. Set output variable *piNew to the -** page number of the new page before returning. -** -** If the new page is the last on its block, then the 'next' block that -** will be used by the segment is allocated here too. In this case output -** variable *piNext is set to the block number of the next block. -** -** If the new page is the first on its block but not the first in the -** entire segment, set output variable *piPrev to the block number of -** the previous block in the segment. -** -** LSM_OK is returned if successful, or an lsm error code otherwise. If -** any value other than LSM_OK is returned, then the final value of all -** output variables is undefined. -*/ -static int fsAppendPage( - FileSystem *pFS, - Segment *pSeg, - LsmPgno *piNew, - int *piPrev, - int *piNext -){ - LsmPgno iPrev = pSeg->iLastPg; - int rc; - assert( iPrev!=0 ); - - *piPrev = 0; - *piNext = 0; - - if( fsIsLast(pFS, iPrev) ){ - /* Grab the first page on the next block (which has already be - ** allocated). In this case set *piPrev to tell the caller to set - ** the "previous block" pointer in the first 4 bytes of the page. - */ - int iNext; - int iBlk = fsPageToBlock(pFS, iPrev); - assert( pSeg->pRedirect==0 ); - rc = fsBlockNext(pFS, 0, iBlk, &iNext); - if( rc!=LSM_OK ) return rc; - *piNew = fsFirstPageOnBlock(pFS, iNext); - *piPrev = iBlk; - }else{ - *piNew = iPrev+1; - if( fsIsLast(pFS, *piNew) ){ - /* Allocate the next block here. */ - int iBlk; - rc = lsmBlockAllocate(pFS->pDb, 0, &iBlk); - if( rc!=LSM_OK ) return rc; - *piNext = iBlk; - } - } - - pSeg->nSize++; - pSeg->iLastPg = *piNew; - return LSM_OK; -} - -/* -** Flush all pages in the FileSystem.pWaiting list to disk. -*/ -void lsmFsFlushWaiting(FileSystem *pFS, int *pRc){ - int rc = *pRc; - Page *pPg; - - pPg = pFS->pWaiting; - pFS->pWaiting = 0; - - while( pPg ){ - Page *pNext = pPg->pWaitingNext; - if( rc==LSM_OK ) rc = lsmFsPagePersist(pPg); - assert( pPg->nRef==1 ); - lsmFsPageRelease(pPg); - pPg = pNext; - } - *pRc = rc; -} - -/* -** If there exists a hash-table entry associated with page iPg, remove it. -*/ -static void fsRemoveHashEntry(FileSystem *pFS, LsmPgno iPg){ - Page *p; - int iHash = fsHashKey(pFS->nHash, iPg); - - for(p=pFS->apHash[iHash]; p && p->iPg!=iPg; p=p->pHashNext); - - if( p ){ - assert( p->nRef==0 || (p->flags & PAGE_FREE)==0 ); - fsPageRemoveFromHash(pFS, p); - p->iPg = 0; - iHash = fsHashKey(pFS->nHash, 0); - p->pHashNext = pFS->apHash[iHash]; - pFS->apHash[iHash] = p; - } -} - -/* -** If the page passed as an argument is dirty, update the database file -** (or mapping of the database file) with its current contents and mark -** the page as clean. -** -** Return LSM_OK if the operation is a success, or an LSM error code -** otherwise. -*/ -int lsmFsPagePersist(Page *pPg){ - int rc = LSM_OK; - if( pPg && (pPg->flags & PAGE_DIRTY) ){ - FileSystem *pFS = pPg->pFS; - - if( pFS->pCompress ){ - int iHash; /* Hash key of assigned page number */ - u8 aSz[3]; /* pPg->nCompress as a 24-bit big-endian */ - assert( pPg->pSeg && pPg->iPg==0 && pPg->nCompress==0 ); - - /* Compress the page image. */ - rc = fsCompressIntoBuffer(pFS, pPg); - - /* Serialize the compressed size into buffer aSz[] */ - putRecordSize(aSz, pPg->nCompress, 0); - - /* Write the serialized page record into the database file. */ - pPg->iPg = fsAppendData(pFS, pPg->pSeg, aSz, sizeof(aSz), &rc); - fsAppendData(pFS, pPg->pSeg, pFS->aOBuffer, pPg->nCompress, &rc); - fsAppendData(pFS, pPg->pSeg, aSz, sizeof(aSz), &rc); - - /* Now that it has a page number, insert the page into the hash table */ - iHash = fsHashKey(pFS->nHash, pPg->iPg); - pPg->pHashNext = pFS->apHash[iHash]; - pFS->apHash[iHash] = pPg; - - pPg->pSeg->nSize += (sizeof(aSz) * 2) + pPg->nCompress; - - pPg->flags &= ~PAGE_DIRTY; - pFS->nWrite++; - }else{ - - if( pPg->iPg==0 ){ - /* No page number has been assigned yet. This occurs with pages used - ** in the b-tree hierarchy. They were not assigned page numbers when - ** they were created as doing so would cause this call to - ** lsmFsPagePersist() to write an out-of-order page. Instead a page - ** number is assigned here so that the page data will be appended - ** to the current segment. - */ - Page **pp; - int iPrev = 0; - int iNext = 0; - int iHash; - - assert( pPg->pSeg->iFirst ); - assert( pPg->flags & PAGE_FREE ); - assert( (pPg->flags & PAGE_HASPREV)==0 ); - assert( pPg->nData==pFS->nPagesize-4 ); - - rc = fsAppendPage(pFS, pPg->pSeg, &pPg->iPg, &iPrev, &iNext); - if( rc!=LSM_OK ) return rc; - - assert( pPg->flags & PAGE_FREE ); - iHash = fsHashKey(pFS->nHash, pPg->iPg); - fsRemoveHashEntry(pFS, pPg->iPg); - pPg->pHashNext = pFS->apHash[iHash]; - pFS->apHash[iHash] = pPg; - assert( pPg->pHashNext==0 || pPg->pHashNext->iPg!=pPg->iPg ); - - if( iPrev ){ - assert( iNext==0 ); - memmove(&pPg->aData[4], pPg->aData, pPg->nData); - lsmPutU32(pPg->aData, iPrev); - pPg->flags |= PAGE_HASPREV; - pPg->aData += 4; - }else if( iNext ){ - assert( iPrev==0 ); - lsmPutU32(&pPg->aData[pPg->nData], iNext); - }else{ - int nData = pPg->nData; - pPg->nData += 4; - lsmSortedExpandBtreePage(pPg, nData); - } - - pPg->nRef++; - for(pp=&pFS->pWaiting; *pp; pp=&(*pp)->pWaitingNext); - *pp = pPg; - assert( pPg->pWaitingNext==0 ); - - }else{ - i64 iOff; /* Offset to write within database file */ - - iOff = (i64)pFS->nPagesize * (i64)(pPg->iPg-1); - if( fsMmapPage(pFS, pPg->iPg)==0 ){ - u8 *aData = pPg->aData - (pPg->flags & PAGE_HASPREV); - rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iOff, aData, pFS->nPagesize); - }else if( pPg->flags & PAGE_FREE ){ - fsGrowMapping(pFS, iOff + pFS->nPagesize, &rc); - if( rc==LSM_OK ){ - u8 *aTo = &((u8 *)(pFS->pMap))[iOff]; - u8 *aFrom = pPg->aData - (pPg->flags & PAGE_HASPREV); - memcpy(aTo, aFrom, pFS->nPagesize); - lsmFree(pFS->pEnv, aFrom); - pFS->nCacheAlloc--; - pPg->aData = aTo + (pPg->flags & PAGE_HASPREV); - pPg->flags &= ~PAGE_FREE; - fsPageRemoveFromHash(pFS, pPg); - pPg->pMappedNext = pFS->pMapped; - pFS->pMapped = pPg; - } - } - - lsmFsFlushWaiting(pFS, &rc); - pPg->flags &= ~PAGE_DIRTY; - pFS->nWrite++; - } - } - } - - return rc; -} - -/* -** For non-compressed databases, this function is a no-op. For compressed -** databases, it adds a padding record to the segment passed as the third -** argument. -** -** The size of the padding records is selected so that the last byte -** written is the last byte of a disk sector. This means that if a -** snapshot is taken and checkpointed, subsequent worker processes will -** not write to any sector that contains checkpointed data. -*/ -int lsmFsSortedPadding( - FileSystem *pFS, - Snapshot *pSnapshot, - Segment *pSeg -){ - int rc = LSM_OK; - if( pFS->pCompress && pSeg->iFirst ){ - LsmPgno iLast2; - LsmPgno iLast = pSeg->iLastPg; /* Current last page of segment */ - int nPad; /* Bytes of padding required */ - u8 aSz[3]; - - iLast2 = (1 + iLast/pFS->szSector) * pFS->szSector - 1; - assert( fsPageToBlock(pFS, iLast)==fsPageToBlock(pFS, iLast2) ); - nPad = (int)(iLast2 - iLast); - - if( iLast2>fsLastPageOnPagesBlock(pFS, iLast) ){ - nPad -= 4; - } - assert( nPad>=0 ); - - if( nPad>=6 ){ - pSeg->nSize += nPad; - nPad -= 6; - putRecordSize(aSz, nPad, 1); - fsAppendData(pFS, pSeg, aSz, sizeof(aSz), &rc); - memset(pFS->aOBuffer, 0, nPad); - fsAppendData(pFS, pSeg, pFS->aOBuffer, nPad, &rc); - fsAppendData(pFS, pSeg, aSz, sizeof(aSz), &rc); - }else if( nPad>0 ){ - u8 aBuf[5] = {0,0,0,0,0}; - aBuf[0] = (u8)nPad; - aBuf[nPad-1] = (u8)nPad; - fsAppendData(pFS, pSeg, aBuf, nPad, &rc); - } - - assert( rc!=LSM_OK - || pSeg->iLastPg==fsLastPageOnPagesBlock(pFS, pSeg->iLastPg) - || ((pSeg->iLastPg + 1) % pFS->szSector)==0 - ); - } - - return rc; -} - - -/* -** Increment the reference count on the page object passed as the first -** argument. -*/ -void lsmFsPageRef(Page *pPg){ - if( pPg ){ - pPg->nRef++; - } -} - -/* -** Release a page-reference obtained using fsPageGet(). -*/ -int lsmFsPageRelease(Page *pPg){ - int rc = LSM_OK; - if( pPg ){ - assert( pPg->nRef>0 ); - pPg->nRef--; - if( pPg->nRef==0 ){ - FileSystem *pFS = pPg->pFS; - rc = lsmFsPagePersist(pPg); - pFS->nOut--; - - assert( pPg->pFS->pCompress - || fsIsFirst(pPg->pFS, pPg->iPg)==0 - || (pPg->flags & PAGE_HASPREV) - ); - pPg->aData -= (pPg->flags & PAGE_HASPREV); - pPg->flags &= ~PAGE_HASPREV; - - if( (pPg->flags & PAGE_FREE)==0 ){ - /* Removed from mapped list */ - Page **pp; - for(pp=&pFS->pMapped; (*pp)!=pPg; pp=&(*pp)->pMappedNext); - *pp = pPg->pMappedNext; - pPg->pMappedNext = 0; - - /* Add to free list */ - pPg->pFreeNext = pFS->pFree; - pFS->pFree = pPg; - }else{ - fsPageAddToLru(pFS, pPg); - } - } - } - - return rc; -} - -/* -** Return the total number of pages read from the database file. -*/ -int lsmFsNRead(FileSystem *pFS){ return pFS->nRead; } - -/* -** Return the total number of pages written to the database file. -*/ -int lsmFsNWrite(FileSystem *pFS){ return pFS->nWrite; } - -/* -** Return a copy of the environment pointer used by the file-system object. -*/ -lsm_env *lsmFsEnv(FileSystem *pFS){ - return pFS->pEnv; -} - -/* -** Return a copy of the environment pointer used by the file-system object -** to which this page belongs. -*/ -lsm_env *lsmPageEnv(Page *pPg) { - return pPg->pFS->pEnv; -} - -/* -** Return a pointer to the file-system object associated with the Page -** passed as the only argument. -*/ -FileSystem *lsmPageFS(Page *pPg){ - return pPg->pFS; -} - -/* -** Return the sector-size as reported by the log file handle. -*/ -int lsmFsSectorSize(FileSystem *pFS){ - return pFS->szSector; -} - -/* -** Helper function for lsmInfoArrayStructure(). -*/ -static Segment *startsWith(Segment *pRun, LsmPgno iFirst){ - return (iFirst==pRun->iFirst) ? pRun : 0; -} - -/* -** Return the segment that starts with page iFirst, if any. If no such segment -** can be found, return NULL. -*/ -static Segment *findSegment(Snapshot *pWorker, LsmPgno iFirst){ - Level *pLvl; /* Used to iterate through db levels */ - Segment *pSeg = 0; /* Pointer to segment to return */ - - for(pLvl=lsmDbSnapshotLevel(pWorker); pLvl && pSeg==0; pLvl=pLvl->pNext){ - if( 0==(pSeg = startsWith(&pLvl->lhs, iFirst)) ){ - int i; - for(i=0; inRight; i++){ - if( (pSeg = startsWith(&pLvl->aRhs[i], iFirst)) ) break; - } - } - } - - return pSeg; -} - -/* -** This function implements the lsm_info(LSM_INFO_ARRAY_STRUCTURE) request. -** If successful, *pzOut is set to point to a nul-terminated string -** containing the array structure and LSM_OK is returned. The caller should -** eventually free the string using lsmFree(). -** -** If an error occurs, *pzOut is set to NULL and an LSM error code returned. -*/ -int lsmInfoArrayStructure( - lsm_db *pDb, - int bBlock, /* True for block numbers only */ - LsmPgno iFirst, - char **pzOut -){ - int rc = LSM_OK; - Snapshot *pWorker; /* Worker snapshot */ - Segment *pArray = 0; /* Array to report on */ - int bUnlock = 0; - - *pzOut = 0; - if( iFirst==0 ) return LSM_ERROR; - - /* Obtain the worker snapshot */ - pWorker = pDb->pWorker; - if( !pWorker ){ - rc = lsmBeginWork(pDb); - if( rc!=LSM_OK ) return rc; - pWorker = pDb->pWorker; - bUnlock = 1; - } - - /* Search for the array that starts on page iFirst */ - pArray = findSegment(pWorker, iFirst); - - if( pArray==0 ){ - /* Could not find the requested array. This is an error. */ - rc = LSM_ERROR; - }else{ - FileSystem *pFS = pDb->pFS; - LsmString str; - int iBlk; - int iLastBlk; - - iBlk = fsPageToBlock(pFS, pArray->iFirst); - iLastBlk = fsPageToBlock(pFS, pArray->iLastPg); - - lsmStringInit(&str, pDb->pEnv); - if( bBlock ){ - lsmStringAppendf(&str, "%d", iBlk); - while( iBlk!=iLastBlk ){ - fsBlockNext(pFS, pArray, iBlk, &iBlk); - lsmStringAppendf(&str, " %d", iBlk); - } - }else{ - lsmStringAppendf(&str, "%d", pArray->iFirst); - while( iBlk!=iLastBlk ){ - lsmStringAppendf(&str, " %d", fsLastPageOnBlock(pFS, iBlk)); - fsBlockNext(pFS, pArray, iBlk, &iBlk); - lsmStringAppendf(&str, " %d", fsFirstPageOnBlock(pFS, iBlk)); - } - lsmStringAppendf(&str, " %d", pArray->iLastPg); - } - - *pzOut = str.z; - } - - if( bUnlock ){ - int rcwork = LSM_BUSY; - lsmFinishWork(pDb, 0, &rcwork); - } - return rc; -} - -int lsmFsSegmentContainsPg( - FileSystem *pFS, - Segment *pSeg, - LsmPgno iPg, - int *pbRes -){ - Redirect *pRedir = pSeg->pRedirect; - int rc = LSM_OK; - int iBlk; - int iLastBlk; - int iPgBlock; /* Block containing page iPg */ - - iPgBlock = fsPageToBlock(pFS, pSeg->iFirst); - iBlk = fsRedirectBlock(pRedir, fsPageToBlock(pFS, pSeg->iFirst)); - iLastBlk = fsRedirectBlock(pRedir, fsPageToBlock(pFS, pSeg->iLastPg)); - - while( iBlk!=iLastBlk && iBlk!=iPgBlock && rc==LSM_OK ){ - rc = fsBlockNext(pFS, pSeg, iBlk, &iBlk); - } - - *pbRes = (iBlk==iPgBlock); - return rc; -} - -/* -** This function implements the lsm_info(LSM_INFO_ARRAY_PAGES) request. -** If successful, *pzOut is set to point to a nul-terminated string -** containing the array structure and LSM_OK is returned. The caller should -** eventually free the string using lsmFree(). -** -** If an error occurs, *pzOut is set to NULL and an LSM error code returned. -*/ -int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut){ - int rc = LSM_OK; - Snapshot *pWorker; /* Worker snapshot */ - Segment *pSeg = 0; /* Array to report on */ - int bUnlock = 0; - - *pzOut = 0; - if( iFirst==0 ) return LSM_ERROR; - - /* Obtain the worker snapshot */ - pWorker = pDb->pWorker; - if( !pWorker ){ - rc = lsmBeginWork(pDb); - if( rc!=LSM_OK ) return rc; - pWorker = pDb->pWorker; - bUnlock = 1; - } - - /* Search for the array that starts on page iFirst */ - pSeg = findSegment(pWorker, iFirst); - - if( pSeg==0 ){ - /* Could not find the requested array. This is an error. */ - rc = LSM_ERROR; - }else{ - Page *pPg = 0; - FileSystem *pFS = pDb->pFS; - LsmString str; - - lsmStringInit(&str, pDb->pEnv); - rc = lsmFsDbPageGet(pFS, pSeg, iFirst, &pPg); - while( rc==LSM_OK && pPg ){ - Page *pNext = 0; - lsmStringAppendf(&str, " %lld", lsmFsPageNumber(pPg)); - rc = lsmFsDbPageNext(pSeg, pPg, 1, &pNext); - lsmFsPageRelease(pPg); - pPg = pNext; - } - - if( rc!=LSM_OK ){ - lsmFree(pDb->pEnv, str.z); - }else{ - *pzOut = str.z; - } - } - - if( bUnlock ){ - int rcwork = LSM_BUSY; - lsmFinishWork(pDb, 0, &rcwork); - } - return rc; -} - -/* -** The following macros are used by the integrity-check code. Associated with -** each block in the database is an 8-bit bit mask (the entry in the aUsed[] -** array). As the integrity-check meanders through the database, it sets the -** following bits to indicate how each block is used. -** -** INTEGRITY_CHECK_FIRST_PG: -** First page of block is in use by sorted run. -** -** INTEGRITY_CHECK_LAST_PG: -** Last page of block is in use by sorted run. -** -** INTEGRITY_CHECK_USED: -** At least one page of the block is in use by a sorted run. -** -** INTEGRITY_CHECK_FREE: -** The free block list contains an entry corresponding to this block. -*/ -#define INTEGRITY_CHECK_FIRST_PG 0x01 -#define INTEGRITY_CHECK_LAST_PG 0x02 -#define INTEGRITY_CHECK_USED 0x04 -#define INTEGRITY_CHECK_FREE 0x08 - -/* -** Helper function for lsmFsIntegrityCheck() -*/ -static void checkBlocks( - FileSystem *pFS, - Segment *pSeg, - int bExtra, /* If true, count the "next" block if any */ - int nUsed, - u8 *aUsed -){ - if( pSeg ){ - if( pSeg && pSeg->nSize>0 ){ - int rc; - int iBlk; /* Current block (during iteration) */ - int iLastBlk; /* Last block of segment */ - int iFirstBlk; /* First block of segment */ - int bLastIsLastOnBlock; /* True iLast is the last on its block */ - - assert( 0==fsSegmentRedirects(pFS, pSeg) ); - iBlk = iFirstBlk = fsPageToBlock(pFS, pSeg->iFirst); - iLastBlk = fsPageToBlock(pFS, pSeg->iLastPg); - - bLastIsLastOnBlock = (fsLastPageOnBlock(pFS, iLastBlk)==pSeg->iLastPg); - assert( iBlk>0 ); - - do { - /* iBlk is a part of this sorted run. */ - aUsed[iBlk-1] |= INTEGRITY_CHECK_USED; - - /* If the first page of this block is also part of the segment, - ** set the flag to indicate that the first page of iBlk is in use. - */ - if( fsFirstPageOnBlock(pFS, iBlk)==pSeg->iFirst || iBlk!=iFirstBlk ){ - assert( (aUsed[iBlk-1] & INTEGRITY_CHECK_FIRST_PG)==0 ); - aUsed[iBlk-1] |= INTEGRITY_CHECK_FIRST_PG; - } - - /* Unless the sorted run finishes before the last page on this block, - ** the last page of this block is also in use. */ - if( iBlk!=iLastBlk || bLastIsLastOnBlock ){ - assert( (aUsed[iBlk-1] & INTEGRITY_CHECK_LAST_PG)==0 ); - aUsed[iBlk-1] |= INTEGRITY_CHECK_LAST_PG; - } - - /* Special case. The sorted run being scanned is the output run of - ** a level currently undergoing an incremental merge. The sorted - ** run ends on the last page of iBlk, but the next block has already - ** been allocated. So mark it as in use as well. */ - if( iBlk==iLastBlk && bLastIsLastOnBlock && bExtra ){ - int iExtra = 0; - rc = fsBlockNext(pFS, pSeg, iBlk, &iExtra); - assert( rc==LSM_OK ); - - assert( aUsed[iExtra-1]==0 ); - aUsed[iExtra-1] |= INTEGRITY_CHECK_USED; - aUsed[iExtra-1] |= INTEGRITY_CHECK_FIRST_PG; - aUsed[iExtra-1] |= INTEGRITY_CHECK_LAST_PG; - } - - /* Move on to the next block in the sorted run. Or set iBlk to zero - ** in order to break out of the loop if this was the last block in - ** the run. */ - if( iBlk==iLastBlk ){ - iBlk = 0; - }else{ - rc = fsBlockNext(pFS, pSeg, iBlk, &iBlk); - assert( rc==LSM_OK ); - } - }while( iBlk ); - } - } -} - -typedef struct CheckFreelistCtx CheckFreelistCtx; -struct CheckFreelistCtx { - u8 *aUsed; - int nBlock; -}; -static int checkFreelistCb(void *pCtx, int iBlk, i64 iSnapshot){ - CheckFreelistCtx *p = (CheckFreelistCtx *)pCtx; - - assert( iBlk>=1 ); - assert( iBlk<=p->nBlock ); - assert( p->aUsed[iBlk-1]==0 ); - p->aUsed[iBlk-1] = INTEGRITY_CHECK_FREE; - return 0; -} - -/* -** This function checks that all blocks in the database file are accounted -** for. For each block, exactly one of the following must be true: -** -** + the block is part of a sorted run, or -** + the block is on the free-block list -** -** This function also checks that there are no references to blocks with -** out-of-range block numbers. -** -** If no errors are found, non-zero is returned. If an error is found, an -** assert() fails. -*/ -int lsmFsIntegrityCheck(lsm_db *pDb){ - CheckFreelistCtx ctx; - FileSystem *pFS = pDb->pFS; - int i; - int rc; - Freelist freelist = {0, 0, 0}; - u8 *aUsed; - Level *pLevel; - Snapshot *pWorker = pDb->pWorker; - int nBlock = pWorker->nBlock; - -#if 0 - static int nCall = 0; - nCall++; - printf("%d calls\n", nCall); -#endif - - aUsed = lsmMallocZero(pDb->pEnv, nBlock); - if( aUsed==0 ){ - /* Malloc has failed. Since this function is only called within debug - ** builds, this probably means the user is running an OOM injection test. - ** Regardless, it will not be possible to run the integrity-check at this - ** time, so assume the database is Ok and return non-zero. */ - return 1; - } - - for(pLevel=pWorker->pLevel; pLevel; pLevel=pLevel->pNext){ - int j; - checkBlocks(pFS, &pLevel->lhs, (pLevel->nRight!=0), nBlock, aUsed); - for(j=0; jnRight; j++){ - checkBlocks(pFS, &pLevel->aRhs[j], 0, nBlock, aUsed); - } - } - - /* Mark all blocks in the free-list as used */ - ctx.aUsed = aUsed; - ctx.nBlock = nBlock; - rc = lsmWalkFreelist(pDb, 0, checkFreelistCb, (void *)&ctx); - - if( rc==LSM_OK ){ - for(i=0; ipEnv, aUsed); - lsmFree(pDb->pEnv, freelist.aEntry); - - return 1; -} - -#ifndef NDEBUG -/* -** Return true if pPg happens to be the last page in segment pSeg. Or false -** otherwise. This function is only invoked as part of assert() conditions. -*/ -int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg){ - if( pPg->pFS->pCompress ){ - LsmPgno iNext = 0; - int rc; - rc = fsNextPageOffset(pPg->pFS, pSeg, pPg->iPg, pPg->nCompress+6, &iNext); - return (rc!=LSM_OK || iNext==0); - } - return (pPg->iPg==pSeg->iLastPg); -} -#endif diff --git a/ext/lsm1/lsm_log.c b/ext/lsm1/lsm_log.c deleted file mode 100644 index 3dcef42f7..000000000 --- a/ext/lsm1/lsm_log.c +++ /dev/null @@ -1,1156 +0,0 @@ -/* -** 2011-08-13 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file contains the implementation of LSM database logging. Logging -** has one purpose in LSM - to make transactions durable. -** -** When data is written to an LSM database, it is initially stored in an -** in-memory tree structure. Since this structure is in volatile memory, -** if a power failure or application crash occurs it may be lost. To -** prevent loss of data in this case, each time a record is written to the -** in-memory tree an equivalent record is appended to the log on disk. -** If a power failure or application crash does occur, data can be recovered -** by reading the log. -** -** A log file consists of the following types of records representing data -** written into the database: -** -** LOG_WRITE: A key-value pair written to the database. -** LOG_DELETE: A delete key issued to the database. -** LOG_COMMIT: A transaction commit. -** -** And the following types of records for ancillary purposes.. -** -** LOG_EOF: A record indicating the end of a log file. -** LOG_PAD1: A single byte padding record. -** LOG_PAD2: An N byte padding record (N>1). -** LOG_JUMP: A pointer to another offset within the log file. -** -** Each transaction written to the log contains one or more LOG_WRITE and/or -** LOG_DELETE records, followed by a LOG_COMMIT record. The LOG_COMMIT record -** contains an 8-byte checksum based on all previous data written to the -** log file. -** -** LOG CHECKSUMS & RECOVERY -** -** Checksums are found in two types of log records: LOG_COMMIT and -** LOG_CKSUM records. In order to recover content from a log, a client -** reads each record from the start of the log, calculating a checksum as -** it does. Each time a LOG_COMMIT or LOG_CKSUM is encountered, the -** recovery process verifies that the checksum stored in the log -** matches the calculated checksum. If it does not, the recovery process -** can stop reading the log. -** -** If a recovery process reads records (other than COMMIT or CKSUM) -** consisting of at least LSM_CKSUM_MAXDATA bytes, then the next record in -** the log must be either a LOG_CKSUM or LOG_COMMIT record. If it is -** not, the recovery process also stops reading the log. -** -** To recover the log file, it must be read twice. The first time to -** determine the location of the last valid commit record. And the second -** time to load data into the in-memory tree. -** -** Todo: Surely there is a better way... -** -** LOG WRAPPING -** -** If the log file were never deleted or wrapped, it would be possible to -** read it from start to end each time is required recovery (i.e each time -** the number of database clients changes from 0 to 1). Effectively reading -** the entire history of the database each time. This would quickly become -** inefficient. Additionally, since the log file would grow without bound, -** it wastes storage space. -** -** Instead, part of each checkpoint written into the database file contains -** a log offset (and other information required to read the log starting at -** at this offset) at which to begin recovery. Offset $O. -** -** Once a checkpoint has been written and synced into the database file, it -** is guaranteed that no recovery process will need to read any data before -** offset $O of the log file. It is therefore safe to begin overwriting -** any data that occurs before offset $O. -** -** This implementation separates the log into three regions mapped into -** the log file - regions 0, 1 and 2. During recovery, regions are read -** in ascending order (i.e. 0, then 1, then 2). Each region is zero or -** more bytes in size. -** -** |---1---|..|--0--|.|--2--|.... -** -** New records are always appended to the end of region 2. -** -** Initially (when it is empty), all three regions are zero bytes in size. -** Each of them are located at the beginning of the file. As records are -** added to the log, region 2 grows, so that the log consists of a zero -** byte region 1, followed by a zero byte region 0, followed by an N byte -** region 2. After one or more checkpoints have been written to disk, -** the start point of region 2 is moved to $O. For example: -** -** A) ||.........|--2--|.... -** -** (both regions 0 and 1 are 0 bytes in size at offset 0). -** -** Eventually, the log wraps around to write new records into the start. -** At this point, region 2 is renamed to region 0. Region 0 is renamed -** to region 2. After appending a few records to the new region 2, the -** log file looks like this: -** -** B) ||--2--|...|--0--|.... -** -** (region 1 is still 0 bytes in size, located at offset 0). -** -** Any checkpoints made at this point may reduce the size of region 0. -** However, if they do not, and region 2 expands so that it is about to -** overwrite the start of region 0, then region 2 is renamed to region 1, -** and a new region 2 created at the end of the file following the existing -** region 0. -** -** C) |---1---|..|--0--|.|-2-| -** -** In this state records are appended to region 2 until checkpoints have -** contracted regions 0 AND 1 UNTil they are both zero bytes in size. They -** are then shifted to the start of the log file, leaving the system in -** the equivalent of state A above. -** -** Alternatively, state B may transition directly to state A if the size -** of region 0 is reduced to zero bytes before region 2 threatens to -** encroach upon it. -** -** LOG_PAD1 & LOG_PAD2 RECORDS -** -** PAD1 and PAD2 records may appear in a log file at any point. They allow -** a process writing the log file align the beginning of transactions with -** the beginning of disk sectors, which increases robustness. -** -** RECORD FORMATS: -** -** LOG_EOF: * A single 0x00 byte. -** -** LOG_PAD1: * A single 0x01 byte. -** -** LOG_PAD2: * A single 0x02 byte, followed by -** * The number of unused bytes (N) as a varint, -** * An N byte block of unused space. -** -** LOG_COMMIT: * A single 0x03 byte. -** * An 8-byte checksum. -** -** LOG_JUMP: * A single 0x04 byte. -** * Absolute file offset to jump to, encoded as a varint. -** -** LOG_WRITE: * A single 0x06 or 0x07 byte, -** * The number of bytes in the key, encoded as a varint, -** * The number of bytes in the value, encoded as a varint, -** * If the first byte was 0x07, an 8 byte checksum. -** * The key data, -** * The value data. -** -** LOG_DELETE: * A single 0x08 or 0x09 byte, -** * The number of bytes in the key, encoded as a varint, -** * If the first byte was 0x09, an 8 byte checksum. -** * The key data. -** -** Varints are as described in lsm_varint.c (SQLite 4 format). -** -** CHECKSUMS: -** -** The checksum is calculated using two 32-bit unsigned integers, s0 and -** s1. The initial value for both is 42. It is updated each time a record -** is written into the log file by treating the encoded (binary) record as -** an array of 32-bit little-endian integers. Then, if x[] is the integer -** array, updating the checksum accumulators as follows: -** -** for i from 0 to n-1 step 2: -** s0 += x[i] + s1; -** s1 += x[i+1] + s0; -** endfor -** -** If the record is not an even multiple of 8-bytes in size it is padded -** with zeroes to make it so before the checksum is updated. -** -** The checksum stored in a COMMIT, WRITE or DELETE is based on all bytes -** up to the start of the 8-byte checksum itself, including the COMMIT, -** WRITE or DELETE fields that appear before the checksum in the record. -** -** VARINT FORMAT -** -** See lsm_varint.c. -*/ - -#ifndef _LSM_INT_H -# include "lsmInt.h" -#endif - -/* Log record types */ -#define LSM_LOG_EOF 0x00 -#define LSM_LOG_PAD1 0x01 -#define LSM_LOG_PAD2 0x02 -#define LSM_LOG_COMMIT 0x03 -#define LSM_LOG_JUMP 0x04 - -#define LSM_LOG_WRITE 0x06 -#define LSM_LOG_WRITE_CKSUM 0x07 - -#define LSM_LOG_DELETE 0x08 -#define LSM_LOG_DELETE_CKSUM 0x09 - -#define LSM_LOG_DRANGE 0x0A -#define LSM_LOG_DRANGE_CKSUM 0x0B - -/* Require a checksum every 32KB. */ -#define LSM_CKSUM_MAXDATA (32*1024) - -/* Do not wrap a log file smaller than this in bytes. */ -#define LSM_MIN_LOGWRAP (128*1024) - -/* -** szSector: -** Commit records must be aligned to end on szSector boundaries. If -** the safety-mode is set to NORMAL or OFF, this value is 1. Otherwise, -** if the safety-mode is set to FULL, it is the size of the file-system -** sectors as reported by lsmFsSectorSize(). -*/ -struct LogWriter { - u32 cksum0; /* Checksum 0 at offset iOff */ - u32 cksum1; /* Checksum 1 at offset iOff */ - int iCksumBuf; /* Bytes of buf that have been checksummed */ - i64 iOff; /* Offset at start of buffer buf */ - int szSector; /* Sector size for this transaction */ - LogRegion jump; /* Avoid writing to this region */ - i64 iRegion1End; /* End of first region written by trans */ - i64 iRegion2Start; /* Start of second regions written by trans */ - LsmString buf; /* Buffer containing data not yet written */ -}; - -/* -** Return the result of interpreting the first 4 bytes in buffer aIn as -** a 32-bit unsigned little-endian integer. -*/ -static u32 getU32le(u8 *aIn){ - return ((u32)aIn[3] << 24) - + ((u32)aIn[2] << 16) - + ((u32)aIn[1] << 8) - + ((u32)aIn[0]); -} - - -/* -** This function is the same as logCksum(), except that pointer "a" need -** not be aligned to an 8-byte boundary or padded with zero bytes. This -** version is slower, but sometimes more convenient to use. -*/ -static void logCksumUnaligned( - char *z, /* Input buffer */ - int n, /* Size of input buffer in bytes */ - u32 *pCksum0, /* IN/OUT: Checksum value 1 */ - u32 *pCksum1 /* IN/OUT: Checksum value 2 */ -){ - u8 *a = (u8 *)z; - u32 cksum0 = *pCksum0; - u32 cksum1 = *pCksum1; - int nIn = (n/8) * 8; - int i; - - assert( n>0 ); - for(i=0; inIn ); - memcpy(aBuf, &a[nIn], n-nIn); - cksum0 += getU32le(aBuf) + cksum1; - cksum1 += getU32le(&aBuf[4]) + cksum0; - } - - *pCksum0 = cksum0; - *pCksum1 = cksum1; -} - -/* -** Update pLog->cksum0 and pLog->cksum1 so that the first nBuf bytes in the -** write buffer (pLog->buf) are included in the checksum. -*/ -static void logUpdateCksum(LogWriter *pLog, int nBuf){ - assert( (pLog->iCksumBuf % 8)==0 ); - assert( pLog->iCksumBuf<=nBuf ); - assert( (nBuf % 8)==0 || nBuf==pLog->buf.n ); - if( nBuf>pLog->iCksumBuf ){ - logCksumUnaligned( - &pLog->buf.z[pLog->iCksumBuf], nBuf-pLog->iCksumBuf, - &pLog->cksum0, &pLog->cksum1 - ); - } - pLog->iCksumBuf = nBuf; -} - -static i64 firstByteOnSector(LogWriter *pLog, i64 iOff){ - return (iOff / pLog->szSector) * pLog->szSector; -} -static i64 lastByteOnSector(LogWriter *pLog, i64 iOff){ - return firstByteOnSector(pLog, iOff) + pLog->szSector - 1; -} - -/* -** If possible, reclaim log file space. Log file space is reclaimed after -** a snapshot that points to the same data in the database file is synced -** into the db header. -*/ -static int logReclaimSpace(lsm_db *pDb){ - int rc; - int iMeta; - int bRotrans; /* True if there exists some ro-trans */ - - /* Test if there exists some other connection with a read-only transaction - ** open. If there does, then log file space may not be reclaimed. */ - rc = lsmDetectRoTrans(pDb, &bRotrans); - if( rc!=LSM_OK || bRotrans ) return rc; - - iMeta = (int)pDb->pShmhdr->iMetaPage; - if( iMeta==1 || iMeta==2 ){ - DbLog *pLog = &pDb->treehdr.log; - i64 iSyncedId; - - /* Read the snapshot-id of the snapshot stored on meta-page iMeta. Note - ** that in theory, the value read is untrustworthy (due to a race - ** condition - see comments above lsmFsReadSyncedId()). So it is only - ** ever used to conclude that no log space can be reclaimed. If it seems - ** to indicate that it may be possible to reclaim log space, a - ** second call to lsmCheckpointSynced() (which does return trustworthy - ** values) is made below to confirm. */ - rc = lsmFsReadSyncedId(pDb, iMeta, &iSyncedId); - - if( rc==LSM_OK && pLog->iSnapshotId!=iSyncedId ){ - i64 iSnapshotId = 0; - i64 iOff = 0; - rc = lsmCheckpointSynced(pDb, &iSnapshotId, &iOff, 0); - if( rc==LSM_OK && pLog->iSnapshotIdaRegion[iRegion]; - if( iOff>=p->iStart && iOff<=p->iEnd ) break; - p->iStart = 0; - p->iEnd = 0; - } - assert( iRegion<3 ); - pLog->aRegion[iRegion].iStart = iOff; - pLog->iSnapshotId = iSnapshotId; - } - } - } - return rc; -} - -/* -** This function is called when a write-transaction is first opened. It -** is assumed that the caller is holding the client-mutex when it is -** called. -** -** Before returning, this function allocates the LogWriter object that -** will be used to write to the log file during the write transaction. -** LSM_OK is returned if no error occurs, otherwise an LSM error code. -*/ -int lsmLogBegin(lsm_db *pDb){ - int rc = LSM_OK; - LogWriter *pNew; - LogRegion *aReg; - - if( pDb->bUseLog==0 ) return LSM_OK; - - /* If the log file has not yet been opened, open it now. Also allocate - ** the LogWriter structure, if it has not already been allocated. */ - rc = lsmFsOpenLog(pDb, 0); - if( pDb->pLogWriter==0 ){ - pNew = lsmMallocZeroRc(pDb->pEnv, sizeof(LogWriter), &rc); - if( pNew ){ - lsmStringInit(&pNew->buf, pDb->pEnv); - rc = lsmStringExtend(&pNew->buf, 2); - } - pDb->pLogWriter = pNew; - }else{ - pNew = pDb->pLogWriter; - assert( (u8 *)(&pNew[1])==(u8 *)(&((&pNew->buf)[1])) ); - memset(pNew, 0, ((u8 *)&pNew->buf) - (u8 *)pNew); - pNew->buf.n = 0; - } - - if( rc==LSM_OK ){ - /* The following call detects whether or not a new snapshot has been - ** synced into the database file. If so, it updates the contents of - ** the pDb->treehdr.log structure to reclaim any space in the log - ** file that is no longer required. - ** - ** TODO: Calling this every transaction is overkill. And since the - ** call has to read and checksum a snapshot from the database file, - ** it is expensive. It would be better to figure out a way so that - ** this is only called occasionally - say for every 32KB written to - ** the log file. - */ - rc = logReclaimSpace(pDb); - } - if( rc!=LSM_OK ){ - lsmLogClose(pDb); - return rc; - } - - /* Set the effective sector-size for this transaction. Sectors are assumed - ** to be one byte in size if the safety-mode is OFF or NORMAL, or as - ** reported by lsmFsSectorSize if it is FULL. */ - if( pDb->eSafety==LSM_SAFETY_FULL ){ - pNew->szSector = lsmFsSectorSize(pDb->pFS); - assert( pNew->szSector>0 ); - }else{ - pNew->szSector = 1; - } - - /* There are now three scenarios: - ** - ** 1) Regions 0 and 1 are both zero bytes in size and region 2 begins - ** at a file offset greater than LSM_MIN_LOGWRAP. In this case, wrap - ** around to the start and write data into the start of the log file. - ** - ** 2) Region 1 is zero bytes in size and region 2 occurs earlier in the - ** file than region 0. In this case, append data to region 2, but - ** remember to jump over region 1 if required. - ** - ** 3) Region 2 is the last in the file. Append to it. - */ - aReg = &pDb->treehdr.log.aRegion[0]; - - assert( aReg[0].iEnd==0 || aReg[0].iEnd>aReg[0].iStart ); - assert( aReg[1].iEnd==0 || aReg[1].iEnd>aReg[1].iStart ); - - pNew->cksum0 = pDb->treehdr.log.cksum0; - pNew->cksum1 = pDb->treehdr.log.cksum1; - - if( aReg[0].iEnd==0 && aReg[1].iEnd==0 && aReg[2].iStart>=LSM_MIN_LOGWRAP ){ - /* Case 1. Wrap around to the start of the file. Write an LSM_LOG_JUMP - ** into the log file in this case. Pad it out to 8 bytes using a PAD2 - ** record so that the checksums can be updated immediately. */ - u8 aJump[] = { - LSM_LOG_PAD2, 0x04, 0x00, 0x00, 0x00, 0x00, LSM_LOG_JUMP, 0x00 - }; - - lsmStringBinAppend(&pNew->buf, aJump, sizeof(aJump)); - logUpdateCksum(pNew, pNew->buf.n); - rc = lsmFsWriteLog(pDb->pFS, aReg[2].iEnd, &pNew->buf); - pNew->iCksumBuf = pNew->buf.n = 0; - - aReg[2].iEnd += 8; - pNew->jump = aReg[0] = aReg[2]; - aReg[2].iStart = aReg[2].iEnd = 0; - }else if( aReg[1].iEnd==0 && aReg[2].iEndiOff = aReg[2].iEnd; - pNew->jump = aReg[0]; - }else{ - /* Case 3. */ - assert( aReg[2].iStart>=aReg[0].iEnd && aReg[2].iStart>=aReg[1].iEnd ); - pNew->iOff = aReg[2].iEnd; - } - - if( pNew->jump.iStart ){ - i64 iRound; - assert( pNew->jump.iStart>pNew->iOff ); - - iRound = firstByteOnSector(pNew, pNew->jump.iStart); - if( iRound>pNew->iOff ) pNew->jump.iStart = iRound; - pNew->jump.iEnd = lastByteOnSector(pNew, pNew->jump.iEnd); - } - - assert( pDb->pLogWriter==pNew ); - return rc; -} - -/* -** This function is called when a write-transaction is being closed. -** Parameter bCommit is true if the transaction is being committed, -** or false otherwise. The caller must hold the client-mutex to call -** this function. -** -** A call to this function deletes the LogWriter object allocated by -** lsmLogBegin(). If the transaction is being committed, the shared state -** in *pLog is updated before returning. -*/ -void lsmLogEnd(lsm_db *pDb, int bCommit){ - DbLog *pLog; - LogWriter *p; - p = pDb->pLogWriter; - - if( p==0 ) return; - pLog = &pDb->treehdr.log; - - if( bCommit ){ - pLog->aRegion[2].iEnd = p->iOff; - pLog->cksum0 = p->cksum0; - pLog->cksum1 = p->cksum1; - if( p->iRegion1End ){ - /* This happens when the transaction had to jump over some other - ** part of the log. */ - assert( pLog->aRegion[1].iEnd==0 ); - assert( pLog->aRegion[2].iStartiRegion1End ); - pLog->aRegion[1].iStart = pLog->aRegion[2].iStart; - pLog->aRegion[1].iEnd = p->iRegion1End; - pLog->aRegion[2].iStart = p->iRegion2Start; - } - } -} - -static int jumpIfRequired( - lsm_db *pDb, - LogWriter *pLog, - int nReq, - int *pbJump -){ - /* Determine if it is necessary to add an LSM_LOG_JUMP to jump over the - ** jump region before writing the LSM_LOG_WRITE or DELETE record. This - ** is necessary if there is insufficient room between the current offset - ** and the jump region to fit the new WRITE/DELETE record and the largest - ** possible JUMP record with up to 7 bytes of padding (a total of 17 - ** bytes). */ - if( (pLog->jump.iStart > (pLog->iOff + pLog->buf.n)) - && (pLog->jump.iStart < (pLog->iOff + pLog->buf.n + (nReq + 17))) - ){ - int rc; /* Return code */ - i64 iJump; /* Offset to jump to */ - u8 aJump[10]; /* Encoded jump record */ - int nJump; /* Valid bytes in aJump[] */ - int nPad; /* Bytes of padding required */ - - /* Serialize the JUMP record */ - iJump = pLog->jump.iEnd+1; - aJump[0] = LSM_LOG_JUMP; - nJump = 1 + lsmVarintPut64(&aJump[1], iJump); - - /* Adding padding to the contents of the buffer so that it will be a - ** multiple of 8 bytes in size after the JUMP record is appended. This - ** is not strictly required, it just makes the keeping the running - ** checksum up to date in this file a little simpler. */ - nPad = (pLog->buf.n + nJump) % 8; - if( nPad ){ - u8 aPad[7] = {0,0,0,0,0,0,0}; - nPad = 8-nPad; - if( nPad==1 ){ - aPad[0] = LSM_LOG_PAD1; - }else{ - aPad[0] = LSM_LOG_PAD2; - aPad[1] = (u8)(nPad-2); - } - rc = lsmStringBinAppend(&pLog->buf, aPad, nPad); - if( rc!=LSM_OK ) return rc; - } - - /* Append the JUMP record to the buffer. Then flush the buffer to disk - ** and update the checksums. The next write to the log file (assuming - ** there is no transaction rollback) will be to offset iJump (just past - ** the jump region). */ - rc = lsmStringBinAppend(&pLog->buf, aJump, nJump); - if( rc!=LSM_OK ) return rc; - assert( (pLog->buf.n % 8)==0 ); - rc = lsmFsWriteLog(pDb->pFS, pLog->iOff, &pLog->buf); - if( rc!=LSM_OK ) return rc; - logUpdateCksum(pLog, pLog->buf.n); - pLog->iRegion1End = (pLog->iOff + pLog->buf.n); - pLog->iRegion2Start = iJump; - pLog->iOff = iJump; - pLog->iCksumBuf = pLog->buf.n = 0; - if( pbJump ) *pbJump = 1; - } - - return LSM_OK; -} - -static int logCksumAndFlush(lsm_db *pDb){ - int rc; /* Return code */ - LogWriter *pLog = pDb->pLogWriter; - - /* Calculate the checksum value. Append it to the buffer. */ - logUpdateCksum(pLog, pLog->buf.n); - lsmPutU32((u8 *)&pLog->buf.z[pLog->buf.n], pLog->cksum0); - pLog->buf.n += 4; - lsmPutU32((u8 *)&pLog->buf.z[pLog->buf.n], pLog->cksum1); - pLog->buf.n += 4; - - /* Write the contents of the buffer to disk. */ - rc = lsmFsWriteLog(pDb->pFS, pLog->iOff, &pLog->buf); - pLog->iOff += pLog->buf.n; - pLog->iCksumBuf = pLog->buf.n = 0; - - return rc; -} - -/* -** Write the contents of the log-buffer to disk. Then write either a CKSUM -** or COMMIT record, depending on the value of parameter eType. -*/ -static int logFlush(lsm_db *pDb, int eType){ - int rc; - int nReq; - LogWriter *pLog = pDb->pLogWriter; - - assert( eType==LSM_LOG_COMMIT ); - assert( pLog ); - - /* Commit record is always 9 bytes in size. */ - nReq = 9; - if( eType==LSM_LOG_COMMIT && pLog->szSector>1 ) nReq += pLog->szSector + 17; - rc = jumpIfRequired(pDb, pLog, nReq, 0); - - /* If this is a COMMIT, add padding to the log so that the COMMIT record - ** is aligned against the end of a disk sector. In other words, add padding - ** so that the first byte following the COMMIT record lies on a different - ** sector. */ - if( eType==LSM_LOG_COMMIT && pLog->szSector>1 ){ - int nPad; /* Bytes of padding to add */ - - /* Determine the value of nPad. */ - nPad = ((pLog->iOff + pLog->buf.n + 9) % pLog->szSector); - if( nPad ) nPad = pLog->szSector - nPad; - rc = lsmStringExtend(&pLog->buf, nPad); - if( rc!=LSM_OK ) return rc; - - while( nPad ){ - if( nPad==1 ){ - pLog->buf.z[pLog->buf.n++] = LSM_LOG_PAD1; - nPad = 0; - }else{ - int n = LSM_MIN(200, nPad-2); - pLog->buf.z[pLog->buf.n++] = LSM_LOG_PAD2; - pLog->buf.z[pLog->buf.n++] = (char)n; - nPad -= 2; - memset(&pLog->buf.z[pLog->buf.n], 0x2B, n); - pLog->buf.n += n; - nPad -= n; - } - } - } - - /* Make sure there is room in the log-buffer to add the CKSUM or COMMIT - ** record. Then add the first byte of it. */ - rc = lsmStringExtend(&pLog->buf, 9); - if( rc!=LSM_OK ) return rc; - pLog->buf.z[pLog->buf.n++] = (char)eType; - memset(&pLog->buf.z[pLog->buf.n], 0, 8); - - rc = logCksumAndFlush(pDb); - - /* If this is a commit and synchronous=full, sync the log to disk. */ - if( rc==LSM_OK && eType==LSM_LOG_COMMIT && pDb->eSafety==LSM_SAFETY_FULL ){ - rc = lsmFsSyncLog(pDb->pFS); - } - return rc; -} - -/* -** Append an LSM_LOG_WRITE (if nVal>=0) or LSM_LOG_DELETE (if nVal<0) -** record to the database log. -*/ -int lsmLogWrite( - lsm_db *pDb, /* Database handle */ - int eType, - void *pKey, int nKey, /* Database key to write to log */ - void *pVal, int nVal /* Database value (or nVal<0) to write */ -){ - int rc = LSM_OK; - LogWriter *pLog; /* Log object to write to */ - int nReq; /* Bytes of space required in log */ - int bCksum = 0; /* True to embed a checksum in this record */ - - assert( eType==LSM_WRITE || eType==LSM_DELETE || eType==LSM_DRANGE ); - assert( LSM_LOG_WRITE==LSM_WRITE ); - assert( LSM_LOG_DELETE==LSM_DELETE ); - assert( LSM_LOG_DRANGE==LSM_DRANGE ); - assert( (eType==LSM_LOG_DELETE)==(nVal<0) ); - - if( pDb->bUseLog==0 ) return LSM_OK; - pLog = pDb->pLogWriter; - - /* Determine how many bytes of space are required, assuming that a checksum - ** will be embedded in this record (even though it may not be). */ - nReq = 1 + lsmVarintLen32(nKey) + 8 + nKey; - if( eType!=LSM_LOG_DELETE ) nReq += lsmVarintLen32(nVal) + nVal; - - /* Jump over the jump region if required. Set bCksum to true to tell the - ** code below to include a checksum in the record if either (a) writing - ** this record would mean that more than LSM_CKSUM_MAXDATA bytes of data - ** have been written to the log since the last checksum, or (b) the jump - ** is taken. */ - rc = jumpIfRequired(pDb, pLog, nReq, &bCksum); - if( (pLog->buf.n+nReq) > LSM_CKSUM_MAXDATA ) bCksum = 1; - - if( rc==LSM_OK ){ - rc = lsmStringExtend(&pLog->buf, nReq); - } - if( rc==LSM_OK ){ - u8 *a = (u8 *)&pLog->buf.z[pLog->buf.n]; - - /* Write the record header - the type byte followed by either 1 (for - ** DELETE) or 2 (for WRITE) varints. */ - assert( LSM_LOG_WRITE_CKSUM == (LSM_LOG_WRITE | 0x0001) ); - assert( LSM_LOG_DELETE_CKSUM == (LSM_LOG_DELETE | 0x0001) ); - assert( LSM_LOG_DRANGE_CKSUM == (LSM_LOG_DRANGE | 0x0001) ); - *(a++) = (u8)eType | (u8)bCksum; - a += lsmVarintPut32(a, nKey); - if( eType!=LSM_LOG_DELETE ) a += lsmVarintPut32(a, nVal); - - if( bCksum ){ - pLog->buf.n = (a - (u8 *)pLog->buf.z); - rc = logCksumAndFlush(pDb); - a = (u8 *)&pLog->buf.z[pLog->buf.n]; - } - - memcpy(a, pKey, nKey); - a += nKey; - if( eType!=LSM_LOG_DELETE ){ - memcpy(a, pVal, nVal); - a += nVal; - } - pLog->buf.n = a - (u8 *)pLog->buf.z; - assert( pLog->buf.n<=pLog->buf.nAlloc ); - } - - return rc; -} - -/* -** Append an LSM_LOG_COMMIT record to the database log. -*/ -int lsmLogCommit(lsm_db *pDb){ - if( pDb->bUseLog==0 ) return LSM_OK; - return logFlush(pDb, LSM_LOG_COMMIT); -} - -/* -** Store the current offset and other checksum related information in the -** structure *pMark. Later, *pMark can be passed to lsmLogSeek() to "rewind" -** the LogWriter object to the current log file offset. This is used when -** rolling back savepoint transactions. -*/ -void lsmLogTell( - lsm_db *pDb, /* Database handle */ - LogMark *pMark /* Populate this object with current offset */ -){ - LogWriter *pLog; - int nCksum; - - if( pDb->bUseLog==0 ) return; - pLog = pDb->pLogWriter; - nCksum = pLog->buf.n & 0xFFFFFFF8; - logUpdateCksum(pLog, nCksum); - assert( pLog->iCksumBuf==nCksum ); - pMark->nBuf = pLog->buf.n - nCksum; - memcpy(pMark->aBuf, &pLog->buf.z[nCksum], pMark->nBuf); - - pMark->iOff = pLog->iOff + pLog->buf.n; - pMark->cksum0 = pLog->cksum0; - pMark->cksum1 = pLog->cksum1; -} - -/* -** Seek (rewind) back to the log file offset stored by an earlier call to -** lsmLogTell() in *pMark. -*/ -void lsmLogSeek( - lsm_db *pDb, /* Database handle */ - LogMark *pMark /* Object containing log offset to seek to */ -){ - LogWriter *pLog; - - if( pDb->bUseLog==0 ) return; - pLog = pDb->pLogWriter; - - assert( pMark->iOff<=pLog->iOff+pLog->buf.n ); - if( (pMark->iOff & 0xFFFFFFF8)>=pLog->iOff ){ - pLog->buf.n = (int)(pMark->iOff - pLog->iOff); - pLog->iCksumBuf = (pLog->buf.n & 0xFFFFFFF8); - }else{ - pLog->buf.n = pMark->nBuf; - memcpy(pLog->buf.z, pMark->aBuf, pMark->nBuf); - pLog->iCksumBuf = 0; - pLog->iOff = pMark->iOff - pMark->nBuf; - } - pLog->cksum0 = pMark->cksum0; - pLog->cksum1 = pMark->cksum1; - - if( pMark->iOff > pLog->iRegion1End ) pLog->iRegion1End = 0; - if( pMark->iOff > pLog->iRegion2Start ) pLog->iRegion2Start = 0; -} - -/* -** This function does the work for an lsm_info(LOG_STRUCTURE) request. -*/ -int lsmInfoLogStructure(lsm_db *pDb, char **pzVal){ - int rc = LSM_OK; - char *zVal = 0; - - /* If there is no read or write transaction open, read the latest - ** tree-header from shared-memory to report on. If necessary, update - ** it based on the contents of the database header. - ** - ** No locks are taken here - these are passive read operations only. - */ - if( pDb->pCsr==0 && pDb->nTransOpen==0 ){ - rc = lsmTreeLoadHeader(pDb, 0); - if( rc==LSM_OK ) rc = logReclaimSpace(pDb); - } - - if( rc==LSM_OK ){ - DbLog *pLog = &pDb->treehdr.log; - zVal = lsmMallocPrintf(pDb->pEnv, - "%d %d %d %d %d %d", - (int)pLog->aRegion[0].iStart, (int)pLog->aRegion[0].iEnd, - (int)pLog->aRegion[1].iStart, (int)pLog->aRegion[1].iEnd, - (int)pLog->aRegion[2].iStart, (int)pLog->aRegion[2].iEnd - ); - if( !zVal ) rc = LSM_NOMEM_BKPT; - } - - *pzVal = zVal; - return rc; -} - -/************************************************************************* -** Begin code for log recovery. -*/ - -typedef struct LogReader LogReader; -struct LogReader { - FileSystem *pFS; /* File system to read from */ - i64 iOff; /* File offset at end of buf content */ - int iBuf; /* Current read offset in buf */ - LsmString buf; /* Buffer containing file content */ - - int iCksumBuf; /* Offset in buf corresponding to cksum[01] */ - u32 cksum0; /* Checksum 0 at offset iCksumBuf */ - u32 cksum1; /* Checksum 1 at offset iCksumBuf */ -}; - -static void logReaderBlob( - LogReader *p, /* Log reader object */ - LsmString *pBuf, /* Dynamic storage, if required */ - int nBlob, /* Number of bytes to read */ - u8 **ppBlob, /* OUT: Pointer to blob read */ - int *pRc /* IN/OUT: Error code */ -){ - static const int LOG_READ_SIZE = 512; - int rc = *pRc; /* Return code */ - int nReq = nBlob; /* Bytes required */ - - while( rc==LSM_OK && nReq>0 ){ - int nAvail; /* Bytes of data available in p->buf */ - if( p->buf.n==p->iBuf ){ - int nCksum; /* Total bytes requiring checksum */ - int nCarry = 0; /* Total bytes requiring checksum */ - - nCksum = p->iBuf - p->iCksumBuf; - if( nCksum>0 ){ - nCarry = nCksum % 8; - nCksum = ((nCksum / 8) * 8); - if( nCksum>0 ){ - logCksumUnaligned( - &p->buf.z[p->iCksumBuf], nCksum, &p->cksum0, &p->cksum1 - ); - } - } - if( nCarry>0 ) memcpy(p->buf.z, &p->buf.z[p->iBuf-nCarry], nCarry); - p->buf.n = nCarry; - p->iBuf = nCarry; - - rc = lsmFsReadLog(p->pFS, p->iOff, LOG_READ_SIZE, &p->buf); - if( rc!=LSM_OK ) break; - p->iCksumBuf = 0; - p->iOff += LOG_READ_SIZE; - } - - nAvail = p->buf.n - p->iBuf; - if( ppBlob && nReq==nBlob && nBlob<=nAvail ){ - *ppBlob = (u8 *)&p->buf.z[p->iBuf]; - p->iBuf += nBlob; - nReq = 0; - }else{ - int nCopy = LSM_MIN(nAvail, nReq); - if( nBlob==nReq ){ - pBuf->n = 0; - } - rc = lsmStringBinAppend(pBuf, (u8 *)&p->buf.z[p->iBuf], nCopy); - nReq -= nCopy; - p->iBuf += nCopy; - if( nReq==0 && ppBlob ){ - *ppBlob = (u8*)pBuf->z; - } - } - } - - *pRc = rc; -} - -static void logReaderVarint( - LogReader *p, - LsmString *pBuf, - int *piVal, /* OUT: Value read from log */ - int *pRc /* IN/OUT: Error code */ -){ - if( *pRc==LSM_OK ){ - u8 *aVarint; - if( p->buf.n==p->iBuf ){ - logReaderBlob(p, 0, 10, &aVarint, pRc); - if( LSM_OK==*pRc ) p->iBuf -= (10 - lsmVarintGet32(aVarint, piVal)); - }else{ - logReaderBlob(p, pBuf, lsmVarintSize(p->buf.z[p->iBuf]), &aVarint, pRc); - if( LSM_OK==*pRc ) lsmVarintGet32(aVarint, piVal); - } - } -} - -static void logReaderByte(LogReader *p, u8 *pByte, int *pRc){ - u8 *pPtr = 0; - logReaderBlob(p, 0, 1, &pPtr, pRc); - if( pPtr ) *pByte = *pPtr; -} - -static void logReaderCksum(LogReader *p, LsmString *pBuf, int *pbEof, int *pRc){ - if( *pRc==LSM_OK ){ - u8 *pPtr = 0; - u32 cksum0, cksum1; - int nCksum = p->iBuf - p->iCksumBuf; - - /* Update in-memory (expected) checksums */ - assert( nCksum>=0 ); - logCksumUnaligned(&p->buf.z[p->iCksumBuf], nCksum, &p->cksum0, &p->cksum1); - p->iCksumBuf = p->iBuf + 8; - logReaderBlob(p, pBuf, 8, &pPtr, pRc); - assert( pPtr || *pRc ); - - /* Read the checksums from the log file. Set *pbEof if they do not match. */ - if( pPtr ){ - cksum0 = lsmGetU32(pPtr); - cksum1 = lsmGetU32(&pPtr[4]); - *pbEof = (cksum0!=p->cksum0 || cksum1!=p->cksum1); - p->iCksumBuf = p->iBuf; - } - } -} - -static void logReaderInit( - lsm_db *pDb, /* Database handle */ - DbLog *pLog, /* Log object associated with pDb */ - int bInitBuf, /* True if p->buf is uninitialized */ - LogReader *p /* Initialize this LogReader object */ -){ - p->pFS = pDb->pFS; - p->iOff = pLog->aRegion[2].iStart; - p->cksum0 = pLog->cksum0; - p->cksum1 = pLog->cksum1; - if( bInitBuf ){ lsmStringInit(&p->buf, pDb->pEnv); } - p->buf.n = 0; - p->iCksumBuf = 0; - p->iBuf = 0; -} - -/* -** This function is called after reading the header of a LOG_DELETE or -** LOG_WRITE record. Parameter nByte is the total size of the key and -** value that follow the header just read. Return true if the size and -** position of the record indicate that it should contain a checksum. -*/ -static int logRequireCksum(LogReader *p, int nByte){ - return ((p->iBuf + nByte - p->iCksumBuf) > LSM_CKSUM_MAXDATA); -} - -/* -** Recover the contents of the log file. -*/ -int lsmLogRecover(lsm_db *pDb){ - LsmString buf1; /* Key buffer */ - LsmString buf2; /* Value buffer */ - LogReader reader; /* Log reader object */ - int rc = LSM_OK; /* Return code */ - int nCommit = 0; /* Number of transactions to recover */ - int iPass; - int nJump = 0; /* Number of LSM_LOG_JUMP records in pass 0 */ - DbLog *pLog; - int bOpen; - - rc = lsmFsOpenLog(pDb, &bOpen); - if( rc!=LSM_OK ) return rc; - - rc = lsmTreeInit(pDb); - if( rc!=LSM_OK ) return rc; - - pLog = &pDb->treehdr.log; - lsmCheckpointLogoffset(pDb->pShmhdr->aSnap2, pLog); - - logReaderInit(pDb, pLog, 1, &reader); - lsmStringInit(&buf1, pDb->pEnv); - lsmStringInit(&buf2, pDb->pEnv); - - /* The outer for() loop runs at most twice. The first iteration is to - ** count the number of committed transactions in the log. The second - ** iterates through those transactions and updates the in-memory tree - ** structure with their contents. */ - if( bOpen ){ - for(iPass=0; iPass<2 && rc==LSM_OK; iPass++){ - int bEof = 0; - - while( rc==LSM_OK && !bEof ){ - u8 eType = 0; - logReaderByte(&reader, &eType, &rc); - - switch( eType ){ - case LSM_LOG_PAD1: - break; - - case LSM_LOG_PAD2: { - int nPad; - logReaderVarint(&reader, &buf1, &nPad, &rc); - logReaderBlob(&reader, &buf1, nPad, 0, &rc); - break; - } - - case LSM_LOG_DRANGE: - case LSM_LOG_DRANGE_CKSUM: - case LSM_LOG_WRITE: - case LSM_LOG_WRITE_CKSUM: { - int nKey; - int nVal; - u8 *aVal; - logReaderVarint(&reader, &buf1, &nKey, &rc); - logReaderVarint(&reader, &buf2, &nVal, &rc); - - if( eType==LSM_LOG_WRITE_CKSUM || eType==LSM_LOG_DRANGE_CKSUM ){ - logReaderCksum(&reader, &buf1, &bEof, &rc); - }else{ - bEof = logRequireCksum(&reader, nKey+nVal); - } - if( bEof ) break; - - logReaderBlob(&reader, &buf1, nKey, 0, &rc); - logReaderBlob(&reader, &buf2, nVal, &aVal, &rc); - if( iPass==1 && rc==LSM_OK ){ - if( eType==LSM_LOG_WRITE || eType==LSM_LOG_WRITE_CKSUM ){ - rc = lsmTreeInsert(pDb, (u8 *)buf1.z, nKey, aVal, nVal); - }else{ - rc = lsmTreeDelete(pDb, (u8 *)buf1.z, nKey, aVal, nVal); - } - } - break; - } - - case LSM_LOG_DELETE: - case LSM_LOG_DELETE_CKSUM: { - int nKey; u8 *aKey; - logReaderVarint(&reader, &buf1, &nKey, &rc); - - if( eType==LSM_LOG_DELETE_CKSUM ){ - logReaderCksum(&reader, &buf1, &bEof, &rc); - }else{ - bEof = logRequireCksum(&reader, nKey); - } - if( bEof ) break; - - logReaderBlob(&reader, &buf1, nKey, &aKey, &rc); - if( iPass==1 && rc==LSM_OK ){ - rc = lsmTreeInsert(pDb, aKey, nKey, NULL, -1); - } - break; - } - - case LSM_LOG_COMMIT: - logReaderCksum(&reader, &buf1, &bEof, &rc); - if( bEof==0 ){ - nCommit++; - assert( nCommit>0 || iPass==1 ); - if( nCommit==0 ) bEof = 1; - } - break; - - case LSM_LOG_JUMP: { - int iOff = 0; - logReaderVarint(&reader, &buf1, &iOff, &rc); - if( rc==LSM_OK ){ - if( iPass==1 ){ - if( pLog->aRegion[2].iStart==0 ){ - assert( pLog->aRegion[1].iStart==0 ); - pLog->aRegion[1].iEnd = reader.iOff; - }else{ - assert( pLog->aRegion[0].iStart==0 ); - pLog->aRegion[0].iStart = pLog->aRegion[2].iStart; - pLog->aRegion[0].iEnd = reader.iOff-reader.buf.n+reader.iBuf; - } - pLog->aRegion[2].iStart = iOff; - }else{ - if( (nJump++)==2 ){ - bEof = 1; - } - } - - reader.iOff = iOff; - reader.buf.n = reader.iBuf; - } - break; - } - - default: - /* Including LSM_LOG_EOF */ - bEof = 1; - break; - } - } - - if( rc==LSM_OK && iPass==0 ){ - if( nCommit==0 ){ - if( pLog->aRegion[2].iStart==0 ){ - iPass = 1; - }else{ - pLog->aRegion[2].iStart = 0; - iPass = -1; - lsmCheckpointZeroLogoffset(pDb); - } - } - logReaderInit(pDb, pLog, 0, &reader); - nCommit = nCommit * -1; - } - } - } - - /* Initialize DbLog object */ - if( rc==LSM_OK ){ - pLog->aRegion[2].iEnd = reader.iOff - reader.buf.n + reader.iBuf; - pLog->cksum0 = reader.cksum0; - pLog->cksum1 = reader.cksum1; - } - - if( rc==LSM_OK ){ - rc = lsmFinishRecovery(pDb); - }else{ - lsmFinishRecovery(pDb); - } - - if( pDb->bRoTrans ){ - lsmFsCloseLog(pDb); - } - - lsmStringClear(&buf1); - lsmStringClear(&buf2); - lsmStringClear(&reader.buf); - return rc; -} - -void lsmLogClose(lsm_db *db){ - if( db->pLogWriter ){ - lsmFree(db->pEnv, db->pLogWriter->buf.z); - lsmFree(db->pEnv, db->pLogWriter); - db->pLogWriter = 0; - } -} diff --git a/ext/lsm1/lsm_main.c b/ext/lsm1/lsm_main.c deleted file mode 100644 index f2b353105..000000000 --- a/ext/lsm1/lsm_main.c +++ /dev/null @@ -1,1008 +0,0 @@ -/* -** 2011-08-18 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** The main interface to the LSM module. -*/ -#include "lsmInt.h" - - -#ifdef LSM_DEBUG -/* -** This function returns a copy of its only argument. -** -** When the library is built with LSM_DEBUG defined, this function is called -** whenever an error code is generated (not propagated - generated). So -** if the library is mysteriously returning (say) LSM_IOERR, a breakpoint -** may be set in this function to determine why. -*/ -int lsmErrorBkpt(int rc){ - /* Set breakpoint here! */ - return rc; -} - -/* -** This function contains various assert() statements that test that the -** lsm_db structure passed as an argument is internally consistent. -*/ -static void assert_db_state(lsm_db *pDb){ - - /* If there is at least one cursor or a write transaction open, the database - ** handle must be holding a pointer to a client snapshot. And the reverse - ** - if there are no open cursors and no write transactions then there must - ** not be a client snapshot. */ - - assert( (pDb->pCsr!=0||pDb->nTransOpen>0)==(pDb->iReader>=0||pDb->bRoTrans) ); - - assert( (pDb->iReader<0 && pDb->bRoTrans==0) || pDb->pClient!=0 ); - - assert( pDb->nTransOpen>=0 ); -} -#else -# define assert_db_state(x) -#endif - -/* -** The default key-compare function. -*/ -static int xCmp(void *p1, int n1, void *p2, int n2){ - int res; - res = memcmp(p1, p2, LSM_MIN(n1, n2)); - if( res==0 ) res = (n1-n2); - return res; -} - -static void xLog(void *pCtx, int rc, const char *z){ - (void)(rc); - (void)(pCtx); - fprintf(stderr, "%s\n", z); - fflush(stderr); -} - -/* -** Allocate a new db handle. -*/ -int lsm_new(lsm_env *pEnv, lsm_db **ppDb){ - lsm_db *pDb; - - /* If the user did not provide an environment, use the default. */ - if( pEnv==0 ) pEnv = lsm_default_env(); - assert( pEnv ); - - /* Allocate the new database handle */ - *ppDb = pDb = (lsm_db *)lsmMallocZero(pEnv, sizeof(lsm_db)); - if( pDb==0 ) return LSM_NOMEM_BKPT; - - /* Initialize the new object */ - pDb->pEnv = pEnv; - pDb->nTreeLimit = LSM_DFLT_AUTOFLUSH; - pDb->nAutockpt = LSM_DFLT_AUTOCHECKPOINT; - pDb->bAutowork = LSM_DFLT_AUTOWORK; - pDb->eSafety = LSM_DFLT_SAFETY; - pDb->xCmp = xCmp; - pDb->nDfltPgsz = LSM_DFLT_PAGE_SIZE; - pDb->nDfltBlksz = LSM_DFLT_BLOCK_SIZE; - pDb->nMerge = LSM_DFLT_AUTOMERGE; - pDb->nMaxFreelist = LSM_MAX_FREELIST_ENTRIES; - pDb->bUseLog = LSM_DFLT_USE_LOG; - pDb->iReader = -1; - pDb->iRwclient = -1; - pDb->bMultiProc = LSM_DFLT_MULTIPLE_PROCESSES; - pDb->iMmap = LSM_DFLT_MMAP; - pDb->xLog = xLog; - pDb->compress.iId = LSM_COMPRESSION_NONE; - return LSM_OK; -} - -lsm_env *lsm_get_env(lsm_db *pDb){ - assert( pDb->pEnv ); - return pDb->pEnv; -} - -/* -** If database handle pDb is currently holding a client snapshot, but does -** not have any open cursors or write transactions, release it. -*/ -static void dbReleaseClientSnapshot(lsm_db *pDb){ - if( pDb->nTransOpen==0 && pDb->pCsr==0 ){ - lsmFinishReadTrans(pDb); - } -} - -static int getFullpathname( - lsm_env *pEnv, - const char *zRel, - char **pzAbs -){ - int nAlloc = 0; - char *zAlloc = 0; - int nReq = 0; - int rc; - - do{ - nAlloc = nReq; - rc = pEnv->xFullpath(pEnv, zRel, zAlloc, &nReq); - if( nReq>nAlloc ){ - zAlloc = lsmReallocOrFreeRc(pEnv, zAlloc, nReq, &rc); - } - }while( nReq>nAlloc && rc==LSM_OK ); - - if( rc!=LSM_OK ){ - lsmFree(pEnv, zAlloc); - zAlloc = 0; - } - *pzAbs = zAlloc; - return rc; -} - -/* -** Check that the bits in the db->mLock mask are consistent with the -** value stored in db->iRwclient. An assert shall fail otherwise. -*/ -static void assertRwclientLockValue(lsm_db *db){ -#ifndef NDEBUG - u64 msk; /* Mask of mLock bits for RWCLIENT locks */ - u64 rwclient = 0; /* Bit corresponding to db->iRwclient */ - - if( db->iRwclient>=0 ){ - rwclient = ((u64)1 << (LSM_LOCK_RWCLIENT(db->iRwclient)-1)); - } - msk = ((u64)1 << (LSM_LOCK_RWCLIENT(LSM_LOCK_NRWCLIENT)-1)) - 1; - msk -= (((u64)1 << (LSM_LOCK_RWCLIENT(0)-1)) - 1); - - assert( (db->mLock & msk)==rwclient ); -#endif -} - -/* -** Open a new connection to database zFilename. -*/ -int lsm_open(lsm_db *pDb, const char *zFilename){ - int rc; - - if( pDb->pDatabase ){ - rc = LSM_MISUSE; - }else{ - char *zFull; - - /* Translate the possibly relative pathname supplied by the user into - ** an absolute pathname. This is required because the supplied path - ** is used (either directly or with "-log" appended to it) for more - ** than one purpose - to open both the database and log files, and - ** perhaps to unlink the log file during disconnection. An absolute - ** path is required to ensure that the correct files are operated - ** on even if the application changes the cwd. */ - rc = getFullpathname(pDb->pEnv, zFilename, &zFull); - assert( rc==LSM_OK || zFull==0 ); - - /* Connect to the database. */ - if( rc==LSM_OK ){ - rc = lsmDbDatabaseConnect(pDb, zFull); - } - - if( pDb->bReadonly==0 ){ - /* Configure the file-system connection with the page-size and block-size - ** of this database. Even if the database file is zero bytes in size - ** on disk, these values have been set in shared-memory by now, and so - ** are guaranteed not to change during the lifetime of this connection. - */ - if( rc==LSM_OK && LSM_OK==(rc = lsmCheckpointLoad(pDb, 0)) ){ - lsmFsSetPageSize(pDb->pFS, lsmCheckpointPgsz(pDb->aSnapshot)); - lsmFsSetBlockSize(pDb->pFS, lsmCheckpointBlksz(pDb->aSnapshot)); - } - } - - lsmFree(pDb->pEnv, zFull); - assertRwclientLockValue(pDb); - } - - assert( pDb->bReadonly==0 || pDb->bReadonly==1 ); - assert( rc!=LSM_OK || (pDb->pShmhdr==0)==(pDb->bReadonly==1) ); - - return rc; -} - -int lsm_close(lsm_db *pDb){ - int rc = LSM_OK; - if( pDb ){ - assert_db_state(pDb); - if( pDb->pCsr || pDb->nTransOpen ){ - rc = LSM_MISUSE_BKPT; - }else{ - lsmMCursorFreeCache(pDb); - lsmFreeSnapshot(pDb->pEnv, pDb->pClient); - pDb->pClient = 0; - - assertRwclientLockValue(pDb); - - lsmDbDatabaseRelease(pDb); - lsmLogClose(pDb); - lsmFsClose(pDb->pFS); - /* assert( pDb->mLock==0 ); */ - - /* Invoke any destructors registered for the compression or - ** compression factory callbacks. */ - if( pDb->factory.xFree ) pDb->factory.xFree(pDb->factory.pCtx); - if( pDb->compress.xFree ) pDb->compress.xFree(pDb->compress.pCtx); - - lsmFree(pDb->pEnv, pDb->rollback.aArray); - lsmFree(pDb->pEnv, pDb->aTrans); - lsmFree(pDb->pEnv, pDb->apShm); - lsmFree(pDb->pEnv, pDb); - } - } - return rc; -} - -int lsm_config(lsm_db *pDb, int eParam, ...){ - int rc = LSM_OK; - va_list ap; - va_start(ap, eParam); - - switch( eParam ){ - case LSM_CONFIG_AUTOFLUSH: { - /* This parameter is read and written in KB. But all internal - ** processing is done in bytes. */ - int *piVal = va_arg(ap, int *); - int iVal = *piVal; - if( iVal>=0 && iVal<=(1024*1024) ){ - pDb->nTreeLimit = iVal*1024; - } - *piVal = (pDb->nTreeLimit / 1024); - break; - } - - case LSM_CONFIG_AUTOWORK: { - int *piVal = va_arg(ap, int *); - if( *piVal>=0 ){ - pDb->bAutowork = *piVal; - } - *piVal = pDb->bAutowork; - break; - } - - case LSM_CONFIG_AUTOCHECKPOINT: { - /* This parameter is read and written in KB. But all internal processing - ** (including the lsm_db.nAutockpt variable) is done in bytes. */ - int *piVal = va_arg(ap, int *); - if( *piVal>=0 ){ - int iVal = *piVal; - pDb->nAutockpt = (i64)iVal * 1024; - } - *piVal = (int)(pDb->nAutockpt / 1024); - break; - } - - case LSM_CONFIG_PAGE_SIZE: { - int *piVal = va_arg(ap, int *); - if( pDb->pDatabase ){ - /* If lsm_open() has been called, this is a read-only parameter. - ** Set the output variable to the page-size according to the - ** FileSystem object. */ - *piVal = lsmFsPageSize(pDb->pFS); - }else{ - if( *piVal>=256 && *piVal<=65536 && ((*piVal-1) & *piVal)==0 ){ - pDb->nDfltPgsz = *piVal; - }else{ - *piVal = pDb->nDfltPgsz; - } - } - break; - } - - case LSM_CONFIG_BLOCK_SIZE: { - /* This parameter is read and written in KB. But all internal - ** processing is done in bytes. */ - int *piVal = va_arg(ap, int *); - if( pDb->pDatabase ){ - /* If lsm_open() has been called, this is a read-only parameter. - ** Set the output variable to the block-size in KB according to the - ** FileSystem object. */ - *piVal = lsmFsBlockSize(pDb->pFS) / 1024; - }else{ - int iVal = *piVal; - if( iVal>=64 && iVal<=65536 && ((iVal-1) & iVal)==0 ){ - pDb->nDfltBlksz = iVal * 1024; - }else{ - *piVal = pDb->nDfltBlksz / 1024; - } - } - break; - } - - case LSM_CONFIG_SAFETY: { - int *piVal = va_arg(ap, int *); - if( *piVal>=0 && *piVal<=2 ){ - pDb->eSafety = *piVal; - } - *piVal = pDb->eSafety; - break; - } - - case LSM_CONFIG_MMAP: { - int *piVal = va_arg(ap, int *); - if( pDb->iReader<0 && *piVal>=0 ){ - pDb->iMmap = *piVal; - rc = lsmFsConfigure(pDb); - } - *piVal = pDb->iMmap; - break; - } - - case LSM_CONFIG_USE_LOG: { - int *piVal = va_arg(ap, int *); - if( pDb->nTransOpen==0 && (*piVal==0 || *piVal==1) ){ - pDb->bUseLog = *piVal; - } - *piVal = pDb->bUseLog; - break; - } - - case LSM_CONFIG_AUTOMERGE: { - int *piVal = va_arg(ap, int *); - if( *piVal>1 ) pDb->nMerge = *piVal; - *piVal = pDb->nMerge; - break; - } - - case LSM_CONFIG_MAX_FREELIST: { - int *piVal = va_arg(ap, int *); - if( *piVal>=2 && *piVal<=LSM_MAX_FREELIST_ENTRIES ){ - pDb->nMaxFreelist = *piVal; - } - *piVal = pDb->nMaxFreelist; - break; - } - - case LSM_CONFIG_MULTIPLE_PROCESSES: { - int *piVal = va_arg(ap, int *); - if( pDb->pDatabase ){ - /* If lsm_open() has been called, this is a read-only parameter. - ** Set the output variable to true if this connection is currently - ** in multi-process mode. */ - *piVal = lsmDbMultiProc(pDb); - }else{ - pDb->bMultiProc = *piVal = (*piVal!=0); - } - break; - } - - case LSM_CONFIG_READONLY: { - int *piVal = va_arg(ap, int *); - /* If lsm_open() has been called, this is a read-only parameter. */ - if( pDb->pDatabase==0 && *piVal>=0 ){ - pDb->bReadonly = *piVal = (*piVal!=0); - } - *piVal = pDb->bReadonly; - break; - } - - case LSM_CONFIG_SET_COMPRESSION: { - lsm_compress *p = va_arg(ap, lsm_compress *); - if( pDb->iReader>=0 && pDb->bInFactory==0 ){ - /* May not change compression schemes with an open transaction */ - rc = LSM_MISUSE_BKPT; - }else{ - if( pDb->compress.xFree ){ - /* Invoke any destructor belonging to the current compression. */ - pDb->compress.xFree(pDb->compress.pCtx); - } - if( p->xBound==0 ){ - memset(&pDb->compress, 0, sizeof(lsm_compress)); - pDb->compress.iId = LSM_COMPRESSION_NONE; - }else{ - memcpy(&pDb->compress, p, sizeof(lsm_compress)); - } - rc = lsmFsConfigure(pDb); - } - break; - } - - case LSM_CONFIG_SET_COMPRESSION_FACTORY: { - lsm_compress_factory *p = va_arg(ap, lsm_compress_factory *); - if( pDb->factory.xFree ){ - /* Invoke any destructor belonging to the current factory. */ - pDb->factory.xFree(pDb->factory.pCtx); - } - memcpy(&pDb->factory, p, sizeof(lsm_compress_factory)); - break; - } - - case LSM_CONFIG_GET_COMPRESSION: { - lsm_compress *p = va_arg(ap, lsm_compress *); - memcpy(p, &pDb->compress, sizeof(lsm_compress)); - break; - } - - default: - rc = LSM_MISUSE; - break; - } - - va_end(ap); - return rc; -} - -void lsmAppendSegmentList(LsmString *pStr, char *zPre, Segment *pSeg){ - lsmStringAppendf(pStr, "%s{%lld %lld %lld %lld}", zPre, - pSeg->iFirst, pSeg->iLastPg, pSeg->iRoot, pSeg->nSize - ); -} - -static int infoGetWorker(lsm_db *pDb, Snapshot **pp, int *pbUnlock){ - int rc = LSM_OK; - - assert( *pbUnlock==0 ); - if( !pDb->pWorker ){ - rc = lsmBeginWork(pDb); - if( rc!=LSM_OK ) return rc; - *pbUnlock = 1; - } - if( pp ) *pp = pDb->pWorker; - return rc; -} - -static void infoFreeWorker(lsm_db *pDb, int bUnlock){ - if( bUnlock ){ - int rcdummy = LSM_BUSY; - lsmFinishWork(pDb, 0, &rcdummy); - } -} - -int lsmStructList( - lsm_db *pDb, /* Database handle */ - char **pzOut /* OUT: Nul-terminated string (tcl list) */ -){ - Level *pTopLevel = 0; /* Top level of snapshot to report on */ - int rc = LSM_OK; - Level *p; - LsmString s; - Snapshot *pWorker; /* Worker snapshot */ - int bUnlock = 0; - - /* Obtain the worker snapshot */ - rc = infoGetWorker(pDb, &pWorker, &bUnlock); - if( rc!=LSM_OK ) return rc; - - /* Format the contents of the snapshot as text */ - pTopLevel = lsmDbSnapshotLevel(pWorker); - lsmStringInit(&s, pDb->pEnv); - for(p=pTopLevel; rc==LSM_OK && p; p=p->pNext){ - int i; - lsmStringAppendf(&s, "%s{%d", (s.n ? " " : ""), (int)p->iAge); - lsmAppendSegmentList(&s, " ", &p->lhs); - for(i=0; rc==LSM_OK && inRight; i++){ - lsmAppendSegmentList(&s, " ", &p->aRhs[i]); - } - lsmStringAppend(&s, "}", 1); - } - rc = s.n>=0 ? LSM_OK : LSM_NOMEM; - - /* Release the snapshot and return */ - infoFreeWorker(pDb, bUnlock); - *pzOut = s.z; - return rc; -} - -static int infoFreelistCb(void *pCtx, int iBlk, i64 iSnapshot){ - LsmString *pStr = (LsmString *)pCtx; - lsmStringAppendf(pStr, "%s{%d %lld}", (pStr->n?" ":""), iBlk, iSnapshot); - return 0; -} - -int lsmInfoFreelist(lsm_db *pDb, char **pzOut){ - Snapshot *pWorker; /* Worker snapshot */ - int bUnlock = 0; - LsmString s; - int rc; - - /* Obtain the worker snapshot */ - rc = infoGetWorker(pDb, &pWorker, &bUnlock); - if( rc!=LSM_OK ) return rc; - - lsmStringInit(&s, pDb->pEnv); - rc = lsmWalkFreelist(pDb, 0, infoFreelistCb, &s); - if( rc!=LSM_OK ){ - lsmFree(pDb->pEnv, s.z); - }else{ - *pzOut = s.z; - } - - /* Release the snapshot and return */ - infoFreeWorker(pDb, bUnlock); - return rc; -} - -static int infoTreeSize(lsm_db *db, int *pnOldKB, int *pnNewKB){ - ShmHeader *pShm = db->pShmhdr; - TreeHeader *p = &pShm->hdr1; - - /* The following code suffers from two race conditions, as it accesses and - ** trusts the contents of shared memory without verifying checksums: - ** - ** * The two values read - TreeHeader.root.nByte and oldroot.nByte - are - ** 32-bit fields. It is assumed that reading from one of these - ** is atomic - that it is not possible to read a partially written - ** garbage value. However the two values may be mutually inconsistent. - ** - ** * TreeHeader.iLogOff is a 64-bit value. And lsmCheckpointLogOffset() - ** reads a 64-bit value from a snapshot stored in shared memory. It - ** is assumed that in each case it is possible to read a partially - ** written garbage value. If this occurs, then the value returned - ** for the size of the "old" tree may reflect the size of an "old" - ** tree that was recently flushed to disk. - ** - ** Given the context in which this function is called (as a result of an - ** lsm_info(LSM_INFO_TREE_SIZE) request), neither of these are considered to - ** be problems. - */ - *pnNewKB = ((int)p->root.nByte + 1023) / 1024; - if( p->iOldShmid ){ - if( p->iOldLog==lsmCheckpointLogOffset(pShm->aSnap1) ){ - *pnOldKB = 0; - }else{ - *pnOldKB = ((int)p->oldroot.nByte + 1023) / 1024; - } - }else{ - *pnOldKB = 0; - } - - return LSM_OK; -} - -int lsm_info(lsm_db *pDb, int eParam, ...){ - int rc = LSM_OK; - va_list ap; - va_start(ap, eParam); - - switch( eParam ){ - case LSM_INFO_NWRITE: { - int *piVal = va_arg(ap, int *); - *piVal = lsmFsNWrite(pDb->pFS); - break; - } - - case LSM_INFO_NREAD: { - int *piVal = va_arg(ap, int *); - *piVal = lsmFsNRead(pDb->pFS); - break; - } - - case LSM_INFO_DB_STRUCTURE: { - char **pzVal = va_arg(ap, char **); - rc = lsmStructList(pDb, pzVal); - break; - } - - case LSM_INFO_ARRAY_STRUCTURE: { - LsmPgno pgno = va_arg(ap, LsmPgno); - char **pzVal = va_arg(ap, char **); - rc = lsmInfoArrayStructure(pDb, 0, pgno, pzVal); - break; - } - - case LSM_INFO_ARRAY_PAGES: { - LsmPgno pgno = va_arg(ap, LsmPgno); - char **pzVal = va_arg(ap, char **); - rc = lsmInfoArrayPages(pDb, pgno, pzVal); - break; - } - - case LSM_INFO_PAGE_HEX_DUMP: - case LSM_INFO_PAGE_ASCII_DUMP: { - LsmPgno pgno = va_arg(ap, LsmPgno); - char **pzVal = va_arg(ap, char **); - int bUnlock = 0; - rc = infoGetWorker(pDb, 0, &bUnlock); - if( rc==LSM_OK ){ - int bHex = (eParam==LSM_INFO_PAGE_HEX_DUMP); - rc = lsmInfoPageDump(pDb, pgno, bHex, pzVal); - } - infoFreeWorker(pDb, bUnlock); - break; - } - - case LSM_INFO_LOG_STRUCTURE: { - char **pzVal = va_arg(ap, char **); - rc = lsmInfoLogStructure(pDb, pzVal); - break; - } - - case LSM_INFO_FREELIST: { - char **pzVal = va_arg(ap, char **); - rc = lsmInfoFreelist(pDb, pzVal); - break; - } - - case LSM_INFO_CHECKPOINT_SIZE: { - int *pnKB = va_arg(ap, int *); - rc = lsmCheckpointSize(pDb, pnKB); - break; - } - - case LSM_INFO_TREE_SIZE: { - int *pnOld = va_arg(ap, int *); - int *pnNew = va_arg(ap, int *); - rc = infoTreeSize(pDb, pnOld, pnNew); - break; - } - - case LSM_INFO_COMPRESSION_ID: { - unsigned int *piOut = va_arg(ap, unsigned int *); - if( pDb->pClient ){ - *piOut = pDb->pClient->iCmpId; - }else{ - rc = lsmInfoCompressionId(pDb, piOut); - } - break; - } - - default: - rc = LSM_MISUSE; - break; - } - - va_end(ap); - return rc; -} - -static int doWriteOp( - lsm_db *pDb, - int bDeleteRange, - const void *pKey, int nKey, /* Key to write or delete */ - const void *pVal, int nVal /* Value to write. Or nVal==-1 for a delete */ -){ - int rc = LSM_OK; /* Return code */ - int bCommit = 0; /* True to commit before returning */ - - if( pDb->nTransOpen==0 ){ - bCommit = 1; - rc = lsm_begin(pDb, 1); - } - - if( rc==LSM_OK ){ - int eType = (bDeleteRange ? LSM_DRANGE : (nVal>=0?LSM_WRITE:LSM_DELETE)); - rc = lsmLogWrite(pDb, eType, (void *)pKey, nKey, (void *)pVal, nVal); - } - - lsmSortedSaveTreeCursors(pDb); - - if( rc==LSM_OK ){ - int pgsz = lsmFsPageSize(pDb->pFS); - int nQuant = LSM_AUTOWORK_QUANT * pgsz; - int nBefore; - int nAfter; - int nDiff; - - if( nQuant>pDb->nTreeLimit ){ - nQuant = LSM_MAX(pDb->nTreeLimit, pgsz); - } - - nBefore = lsmTreeSize(pDb); - if( bDeleteRange ){ - rc = lsmTreeDelete(pDb, (void *)pKey, nKey, (void *)pVal, nVal); - }else{ - rc = lsmTreeInsert(pDb, (void *)pKey, nKey, (void *)pVal, nVal); - } - - nAfter = lsmTreeSize(pDb); - nDiff = (nAfter/nQuant) - (nBefore/nQuant); - if( rc==LSM_OK && pDb->bAutowork && nDiff!=0 ){ - rc = lsmSortedAutoWork(pDb, nDiff * LSM_AUTOWORK_QUANT); - } - } - - /* If a transaction was opened at the start of this function, commit it. - ** Or, if an error has occurred, roll it back. */ - if( bCommit ){ - if( rc==LSM_OK ){ - rc = lsm_commit(pDb, 0); - }else{ - lsm_rollback(pDb, 0); - } - } - - return rc; -} - -/* -** Write a new value into the database. -*/ -int lsm_insert( - lsm_db *db, /* Database connection */ - const void *pKey, int nKey, /* Key to write or delete */ - const void *pVal, int nVal /* Value to write. Or nVal==-1 for a delete */ -){ - return doWriteOp(db, 0, pKey, nKey, pVal, nVal); -} - -/* -** Delete a value from the database. -*/ -int lsm_delete(lsm_db *db, const void *pKey, int nKey){ - return doWriteOp(db, 0, pKey, nKey, 0, -1); -} - -/* -** Delete a range of database keys. -*/ -int lsm_delete_range( - lsm_db *db, /* Database handle */ - const void *pKey1, int nKey1, /* Lower bound of range to delete */ - const void *pKey2, int nKey2 /* Upper bound of range to delete */ -){ - int rc = LSM_OK; - if( db->xCmp((void *)pKey1, nKey1, (void *)pKey2, nKey2)<0 ){ - rc = doWriteOp(db, 1, pKey1, nKey1, pKey2, nKey2); - } - return rc; -} - -/* -** Open a new cursor handle. -** -** If there are currently no other open cursor handles, and no open write -** transaction, open a read transaction here. -*/ -int lsm_csr_open(lsm_db *pDb, lsm_cursor **ppCsr){ - int rc = LSM_OK; /* Return code */ - MultiCursor *pCsr = 0; /* New cursor object */ - - /* Open a read transaction if one is not already open. */ - assert_db_state(pDb); - - if( pDb->pShmhdr==0 ){ - assert( pDb->bReadonly ); - rc = lsmBeginRoTrans(pDb); - }else if( pDb->iReader<0 ){ - rc = lsmBeginReadTrans(pDb); - } - - /* Allocate the multi-cursor. */ - if( rc==LSM_OK ){ - rc = lsmMCursorNew(pDb, &pCsr); - } - - /* If an error has occured, set the output to NULL and delete any partially - ** allocated cursor. If this means there are no open cursors, release the - ** client snapshot. */ - if( rc!=LSM_OK ){ - lsmMCursorClose(pCsr, 0); - dbReleaseClientSnapshot(pDb); - } - - assert_db_state(pDb); - *ppCsr = (lsm_cursor *)pCsr; - return rc; -} - -/* -** Close a cursor opened using lsm_csr_open(). -*/ -int lsm_csr_close(lsm_cursor *p){ - if( p ){ - lsm_db *pDb = lsmMCursorDb((MultiCursor *)p); - assert_db_state(pDb); - lsmMCursorClose((MultiCursor *)p, 1); - dbReleaseClientSnapshot(pDb); - assert_db_state(pDb); - } - return LSM_OK; -} - -/* -** Attempt to seek the cursor to the database entry specified by pKey/nKey. -** If an error occurs (e.g. an OOM or IO error), return an LSM error code. -** Otherwise, return LSM_OK. -*/ -int lsm_csr_seek(lsm_cursor *pCsr, const void *pKey, int nKey, int eSeek){ - return lsmMCursorSeek((MultiCursor *)pCsr, 0, (void *)pKey, nKey, eSeek); -} - -int lsm_csr_next(lsm_cursor *pCsr){ - return lsmMCursorNext((MultiCursor *)pCsr); -} - -int lsm_csr_prev(lsm_cursor *pCsr){ - return lsmMCursorPrev((MultiCursor *)pCsr); -} - -int lsm_csr_first(lsm_cursor *pCsr){ - return lsmMCursorFirst((MultiCursor *)pCsr); -} - -int lsm_csr_last(lsm_cursor *pCsr){ - return lsmMCursorLast((MultiCursor *)pCsr); -} - -int lsm_csr_valid(lsm_cursor *pCsr){ - return lsmMCursorValid((MultiCursor *)pCsr); -} - -int lsm_csr_key(lsm_cursor *pCsr, const void **ppKey, int *pnKey){ - return lsmMCursorKey((MultiCursor *)pCsr, (void **)ppKey, pnKey); -} - -int lsm_csr_value(lsm_cursor *pCsr, const void **ppVal, int *pnVal){ - return lsmMCursorValue((MultiCursor *)pCsr, (void **)ppVal, pnVal); -} - -void lsm_config_log( - lsm_db *pDb, - void (*xLog)(void *, int, const char *), - void *pCtx -){ - pDb->xLog = xLog; - pDb->pLogCtx = pCtx; -} - -void lsm_config_work_hook( - lsm_db *pDb, - void (*xWork)(lsm_db *, void *), - void *pCtx -){ - pDb->xWork = xWork; - pDb->pWorkCtx = pCtx; -} - -void lsmLogMessage(lsm_db *pDb, int rc, const char *zFormat, ...){ - if( pDb->xLog ){ - LsmString s; - va_list ap, ap2; - lsmStringInit(&s, pDb->pEnv); - va_start(ap, zFormat); - va_start(ap2, zFormat); - lsmStringVAppendf(&s, zFormat, ap, ap2); - va_end(ap); - va_end(ap2); - pDb->xLog(pDb->pLogCtx, rc, s.z); - lsmStringClear(&s); - } -} - -int lsm_begin(lsm_db *pDb, int iLevel){ - int rc; - - assert_db_state( pDb ); - rc = (pDb->bReadonly ? LSM_READONLY : LSM_OK); - - /* A value less than zero means open one more transaction. */ - if( iLevel<0 ) iLevel = pDb->nTransOpen + 1; - if( iLevel>pDb->nTransOpen ){ - int i; - - /* Extend the pDb->aTrans[] array if required. */ - if( rc==LSM_OK && pDb->nTransAllocpEnv, pDb->aTrans, nByte); - if( !aNew ){ - rc = LSM_NOMEM; - }else{ - nByte = sizeof(TransMark) * (iLevel+1 - pDb->nTransAlloc); - memset(&aNew[pDb->nTransAlloc], 0, nByte); - pDb->nTransAlloc = iLevel+1; - pDb->aTrans = aNew; - } - } - - if( rc==LSM_OK && pDb->nTransOpen==0 ){ - rc = lsmBeginWriteTrans(pDb); - } - - if( rc==LSM_OK ){ - for(i=pDb->nTransOpen; iaTrans[i].tree); - lsmLogTell(pDb, &pDb->aTrans[i].log); - } - pDb->nTransOpen = iLevel; - } - } - - return rc; -} - -int lsm_commit(lsm_db *pDb, int iLevel){ - int rc = LSM_OK; - - assert_db_state( pDb ); - - /* A value less than zero means close the innermost nested transaction. */ - if( iLevel<0 ) iLevel = LSM_MAX(0, pDb->nTransOpen - 1); - - if( iLevelnTransOpen ){ - if( iLevel==0 ){ - int rc2; - /* Commit the transaction to disk. */ - if( rc==LSM_OK ) rc = lsmLogCommit(pDb); - if( rc==LSM_OK && pDb->eSafety==LSM_SAFETY_FULL ){ - rc = lsmFsSyncLog(pDb->pFS); - } - rc2 = lsmFinishWriteTrans(pDb, (rc==LSM_OK)); - if( rc==LSM_OK ) rc = rc2; - } - pDb->nTransOpen = iLevel; - } - dbReleaseClientSnapshot(pDb); - return rc; -} - -int lsm_rollback(lsm_db *pDb, int iLevel){ - int rc = LSM_OK; - assert_db_state( pDb ); - - if( pDb->nTransOpen ){ - /* A value less than zero means close the innermost nested transaction. */ - if( iLevel<0 ) iLevel = LSM_MAX(0, pDb->nTransOpen - 1); - - if( iLevel<=pDb->nTransOpen ){ - TransMark *pMark = &pDb->aTrans[(iLevel==0 ? 0 : iLevel-1)]; - lsmTreeRollback(pDb, &pMark->tree); - if( iLevel ) lsmLogSeek(pDb, &pMark->log); - pDb->nTransOpen = iLevel; - } - - if( pDb->nTransOpen==0 ){ - lsmFinishWriteTrans(pDb, 0); - } - dbReleaseClientSnapshot(pDb); - } - - return rc; -} - -int lsm_get_user_version(lsm_db *pDb, unsigned int *piUsr){ - int rc = LSM_OK; /* Return code */ - - /* Open a read transaction if one is not already open. */ - assert_db_state(pDb); - if( pDb->pShmhdr==0 ){ - assert( pDb->bReadonly ); - rc = lsmBeginRoTrans(pDb); - }else if( pDb->iReader<0 ){ - rc = lsmBeginReadTrans(pDb); - } - - /* Allocate the multi-cursor. */ - if( rc==LSM_OK ){ - *piUsr = pDb->treehdr.iUsrVersion; - } - - dbReleaseClientSnapshot(pDb); - assert_db_state(pDb); - return rc; -} - -int lsm_set_user_version(lsm_db *pDb, unsigned int iUsr){ - int rc = LSM_OK; /* Return code */ - int bCommit = 0; /* True to commit before returning */ - - if( pDb->nTransOpen==0 ){ - bCommit = 1; - rc = lsm_begin(pDb, 1); - } - - if( rc==LSM_OK ){ - pDb->treehdr.iUsrVersion = iUsr; - } - - /* If a transaction was opened at the start of this function, commit it. - ** Or, if an error has occurred, roll it back. */ - if( bCommit ){ - if( rc==LSM_OK ){ - rc = lsm_commit(pDb, 0); - }else{ - lsm_rollback(pDb, 0); - } - } - - return rc; -} diff --git a/ext/lsm1/lsm_mem.c b/ext/lsm1/lsm_mem.c deleted file mode 100644 index 13dd9fe31..000000000 --- a/ext/lsm1/lsm_mem.c +++ /dev/null @@ -1,104 +0,0 @@ -/* -** 2011-08-18 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Helper routines for memory allocation. -*/ -#include "lsmInt.h" - -/* -** The following routines are called internally by LSM sub-routines. In -** this case a valid environment pointer must be supplied. -*/ -void *lsmMalloc(lsm_env *pEnv, size_t N){ - assert( pEnv ); - return pEnv->xMalloc(pEnv, N); -} -void lsmFree(lsm_env *pEnv, void *p){ - assert( pEnv ); - pEnv->xFree(pEnv, p); -} -void *lsmRealloc(lsm_env *pEnv, void *p, size_t N){ - assert( pEnv ); - return pEnv->xRealloc(pEnv, p, N); -} - -/* -** Core memory allocation routines for LSM. -*/ -void *lsm_malloc(lsm_env *pEnv, size_t N){ - return lsmMalloc(pEnv ? pEnv : lsm_default_env(), N); -} -void lsm_free(lsm_env *pEnv, void *p){ - lsmFree(pEnv ? pEnv : lsm_default_env(), p); -} -void *lsm_realloc(lsm_env *pEnv, void *p, size_t N){ - return lsmRealloc(pEnv ? pEnv : lsm_default_env(), p, N); -} - -void *lsmMallocZero(lsm_env *pEnv, size_t N){ - void *pRet; - assert( pEnv ); - pRet = lsmMalloc(pEnv, N); - if( pRet ) memset(pRet, 0, N); - return pRet; -} - -void *lsmMallocRc(lsm_env *pEnv, size_t N, int *pRc){ - void *pRet = 0; - if( *pRc==LSM_OK ){ - pRet = lsmMalloc(pEnv, N); - if( pRet==0 ){ - *pRc = LSM_NOMEM_BKPT; - } - } - return pRet; -} - -void *lsmMallocZeroRc(lsm_env *pEnv, size_t N, int *pRc){ - void *pRet = 0; - if( *pRc==LSM_OK ){ - pRet = lsmMallocZero(pEnv, N); - if( pRet==0 ){ - *pRc = LSM_NOMEM_BKPT; - } - } - return pRet; -} - -void *lsmReallocOrFree(lsm_env *pEnv, void *p, size_t N){ - void *pNew; - pNew = lsm_realloc(pEnv, p, N); - if( !pNew ) lsm_free(pEnv, p); - return pNew; -} - -void *lsmReallocOrFreeRc(lsm_env *pEnv, void *p, size_t N, int *pRc){ - void *pRet = 0; - if( *pRc ){ - lsmFree(pEnv, p); - }else{ - pRet = lsmReallocOrFree(pEnv, p, N); - if( !pRet ) *pRc = LSM_NOMEM_BKPT; - } - return pRet; -} - -char *lsmMallocStrdup(lsm_env *pEnv, const char *zIn){ - int nByte; - char *zRet; - nByte = strlen(zIn); - zRet = lsmMalloc(pEnv, nByte+1); - if( zRet ){ - memcpy(zRet, zIn, nByte+1); - } - return zRet; -} diff --git a/ext/lsm1/lsm_mutex.c b/ext/lsm1/lsm_mutex.c deleted file mode 100644 index cb99b2a61..000000000 --- a/ext/lsm1/lsm_mutex.c +++ /dev/null @@ -1,88 +0,0 @@ -/* -** 2012-01-30 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Mutex functions for LSM. -*/ -#include "lsmInt.h" - -/* -** Allocate a new mutex. -*/ -int lsmMutexNew(lsm_env *pEnv, lsm_mutex **ppNew){ - return pEnv->xMutexNew(pEnv, ppNew); -} - -/* -** Return a handle for one of the static mutexes. -*/ -int lsmMutexStatic(lsm_env *pEnv, int iMutex, lsm_mutex **ppStatic){ - return pEnv->xMutexStatic(pEnv, iMutex, ppStatic); -} - -/* -** Free a mutex allocated by lsmMutexNew(). -*/ -void lsmMutexDel(lsm_env *pEnv, lsm_mutex *pMutex){ - if( pMutex ) pEnv->xMutexDel(pMutex); -} - -/* -** Enter a mutex. -*/ -void lsmMutexEnter(lsm_env *pEnv, lsm_mutex *pMutex){ - pEnv->xMutexEnter(pMutex); -} - -/* -** Attempt to enter a mutex, but do not block. If successful, return zero. -** Otherwise, if the mutex is already held by some other thread and is not -** entered, return non zero. -** -** Each successful call to this function must be matched by a call to -** lsmMutexLeave(). -*/ -int lsmMutexTry(lsm_env *pEnv, lsm_mutex *pMutex){ - return pEnv->xMutexTry(pMutex); -} - -/* -** Leave a mutex. -*/ -void lsmMutexLeave(lsm_env *pEnv, lsm_mutex *pMutex){ - pEnv->xMutexLeave(pMutex); -} - -#ifndef NDEBUG -/* -** Return non-zero if the mutex passed as the second argument is held -** by the calling thread, or zero otherwise. If the implementation is not -** able to tell if the mutex is held by the caller, it should return -** non-zero. -** -** This function is only used as part of assert() statements. -*/ -int lsmMutexHeld(lsm_env *pEnv, lsm_mutex *pMutex){ - return pEnv->xMutexHeld ? pEnv->xMutexHeld(pMutex) : 1; -} - -/* -** Return non-zero if the mutex passed as the second argument is not -** held by the calling thread, or zero otherwise. If the implementation -** is not able to tell if the mutex is held by the caller, it should -** return non-zero. -** -** This function is only used as part of assert() statements. -*/ -int lsmMutexNotHeld(lsm_env *pEnv, lsm_mutex *pMutex){ - return pEnv->xMutexNotHeld ? pEnv->xMutexNotHeld(pMutex) : 1; -} -#endif diff --git a/ext/lsm1/lsm_shared.c b/ext/lsm1/lsm_shared.c deleted file mode 100644 index 09f933848..000000000 --- a/ext/lsm1/lsm_shared.c +++ /dev/null @@ -1,1994 +0,0 @@ -/* -** 2012-01-23 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Utilities used to help multiple LSM clients to coexist within the -** same process space. -*/ -#include "lsmInt.h" - -/* -** Global data. All global variables used by code in this file are grouped -** into the following structure instance. -** -** pDatabase: -** Linked list of all Database objects allocated within this process. -** This list may not be traversed without holding the global mutex (see -** functions enterGlobalMutex() and leaveGlobalMutex()). -*/ -static struct SharedData { - Database *pDatabase; /* Linked list of all Database objects */ -} gShared; - -/* -** Database structure. There is one such structure for each distinct -** database accessed by this process. They are stored in the singly linked -** list starting at global variable gShared.pDatabase. Database objects are -** reference counted. Once the number of connections to the associated -** database drops to zero, they are removed from the linked list and deleted. -** -** pFile: -** In multi-process mode, this file descriptor is used to obtain locks -** and to access shared-memory. In single process mode, its only job is -** to hold the exclusive lock on the file. -** -*/ -struct Database { - /* Protected by the global mutex (enterGlobalMutex/leaveGlobalMutex): */ - char *zName; /* Canonical path to database file */ - int nName; /* strlen(zName) */ - int nDbRef; /* Number of associated lsm_db handles */ - Database *pDbNext; /* Next Database structure in global list */ - - /* Protected by the local mutex (pClientMutex) */ - int bReadonly; /* True if Database.pFile is read-only */ - int bMultiProc; /* True if running in multi-process mode */ - lsm_file *pFile; /* Used for locks/shm in multi-proc mode */ - LsmFile *pLsmFile; /* List of deferred closes */ - lsm_mutex *pClientMutex; /* Protects the apShmChunk[] and pConn */ - int nShmChunk; /* Number of entries in apShmChunk[] array */ - void **apShmChunk; /* Array of "shared" memory regions */ - lsm_db *pConn; /* List of connections to this db. */ -}; - -/* -** Functions to enter and leave the global mutex. This mutex is used -** to protect the global linked-list headed at gShared.pDatabase. -*/ -static int enterGlobalMutex(lsm_env *pEnv){ - lsm_mutex *p; - int rc = lsmMutexStatic(pEnv, LSM_MUTEX_GLOBAL, &p); - if( rc==LSM_OK ) lsmMutexEnter(pEnv, p); - return rc; -} -static void leaveGlobalMutex(lsm_env *pEnv){ - lsm_mutex *p; - lsmMutexStatic(pEnv, LSM_MUTEX_GLOBAL, &p); - lsmMutexLeave(pEnv, p); -} - -#ifdef LSM_DEBUG -static int holdingGlobalMutex(lsm_env *pEnv){ - lsm_mutex *p; - lsmMutexStatic(pEnv, LSM_MUTEX_GLOBAL, &p); - return lsmMutexHeld(pEnv, p); -} -#endif - -#if 0 -static void assertNotInFreelist(Freelist *p, int iBlk){ - int i; - for(i=0; inEntry; i++){ - assert( p->aEntry[i].iBlk!=iBlk ); - } -} -#else -# define assertNotInFreelist(x,y) -#endif - -/* -** Append an entry to the free-list. If (iId==-1), this is a delete. -*/ -int freelistAppend(lsm_db *db, u32 iBlk, i64 iId){ - lsm_env *pEnv = db->pEnv; - Freelist *p; - int i; - - assert( iId==-1 || iId>=0 ); - p = db->bUseFreelist ? db->pFreelist : &db->pWorker->freelist; - - /* Extend the space allocated for the freelist, if required */ - assert( p->nAlloc>=p->nEntry ); - if( p->nAlloc==p->nEntry ){ - int nNew; - int nByte; - FreelistEntry *aNew; - - nNew = (p->nAlloc==0 ? 4 : p->nAlloc*2); - nByte = sizeof(FreelistEntry) * nNew; - aNew = (FreelistEntry *)lsmRealloc(pEnv, p->aEntry, nByte); - if( !aNew ) return LSM_NOMEM_BKPT; - p->nAlloc = nNew; - p->aEntry = aNew; - } - - for(i=0; inEntry; i++){ - assert( i==0 || p->aEntry[i].iBlk > p->aEntry[i-1].iBlk ); - if( p->aEntry[i].iBlk>=iBlk ) break; - } - - if( inEntry && p->aEntry[i].iBlk==iBlk ){ - /* Clobber an existing entry */ - p->aEntry[i].iId = iId; - }else{ - /* Insert a new entry into the list */ - int nByte = sizeof(FreelistEntry)*(p->nEntry-i); - memmove(&p->aEntry[i+1], &p->aEntry[i], nByte); - p->aEntry[i].iBlk = iBlk; - p->aEntry[i].iId = iId; - p->nEntry++; - } - - return LSM_OK; -} - -/* -** This function frees all resources held by the Database structure passed -** as the only argument. -*/ -static void freeDatabase(lsm_env *pEnv, Database *p){ - assert( holdingGlobalMutex(pEnv) ); - if( p ){ - /* Free the mutexes */ - lsmMutexDel(pEnv, p->pClientMutex); - - if( p->pFile ){ - lsmEnvClose(pEnv, p->pFile); - } - - /* Free the array of shm pointers */ - lsmFree(pEnv, p->apShmChunk); - - /* Free the memory allocated for the Database struct itself */ - lsmFree(pEnv, p); - } -} - -typedef struct DbTruncateCtx DbTruncateCtx; -struct DbTruncateCtx { - int nBlock; - i64 iInUse; -}; - -static int dbTruncateCb(void *pCtx, int iBlk, i64 iSnapshot){ - DbTruncateCtx *p = (DbTruncateCtx *)pCtx; - if( iBlk!=p->nBlock || (p->iInUse>=0 && iSnapshot>=p->iInUse) ) return 1; - p->nBlock--; - return 0; -} - -static int dbTruncate(lsm_db *pDb, i64 iInUse){ - int rc = LSM_OK; -#if 0 - int i; - DbTruncateCtx ctx; - - assert( pDb->pWorker ); - ctx.nBlock = pDb->pWorker->nBlock; - ctx.iInUse = iInUse; - - rc = lsmWalkFreelist(pDb, 1, dbTruncateCb, (void *)&ctx); - for(i=ctx.nBlock+1; rc==LSM_OK && i<=pDb->pWorker->nBlock; i++){ - rc = freelistAppend(pDb, i, -1); - } - - if( rc==LSM_OK ){ -#ifdef LSM_LOG_FREELIST - if( ctx.nBlock!=pDb->pWorker->nBlock ){ - lsmLogMessage(pDb, 0, - "dbTruncate(): truncated db to %d blocks",ctx.nBlock - ); - } -#endif - pDb->pWorker->nBlock = ctx.nBlock; - } -#endif - return rc; -} - - -/* -** This function is called during database shutdown (when the number of -** connections drops from one to zero). It truncates the database file -** to as small a size as possible without truncating away any blocks that -** contain data. -*/ -static int dbTruncateFile(lsm_db *pDb){ - int rc; - - assert( pDb->pWorker==0 ); - assert( lsmShmAssertLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL) ); - rc = lsmCheckpointLoadWorker(pDb); - - if( rc==LSM_OK ){ - DbTruncateCtx ctx; - - /* Walk the database free-block-list in reverse order. Set ctx.nBlock - ** to the block number of the last block in the database that actually - ** contains data. */ - ctx.nBlock = pDb->pWorker->nBlock; - ctx.iInUse = -1; - rc = lsmWalkFreelist(pDb, 1, dbTruncateCb, (void *)&ctx); - - /* If the last block that contains data is not already the last block in - ** the database file, truncate the database file so that it is. */ - if( rc==LSM_OK ){ - rc = lsmFsTruncateDb( - pDb->pFS, (i64)ctx.nBlock*lsmFsBlockSize(pDb->pFS) - ); - } - } - - lsmFreeSnapshot(pDb->pEnv, pDb->pWorker); - pDb->pWorker = 0; - return rc; -} - -static void doDbDisconnect(lsm_db *pDb){ - int rc; - - if( pDb->bReadonly ){ - lsmShmLock(pDb, LSM_LOCK_DMS3, LSM_LOCK_UNLOCK, 0); - }else{ - /* Block for an exclusive lock on DMS1. This lock serializes all calls - ** to doDbConnect() and doDbDisconnect() across all processes. */ - rc = lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL, 1); - if( rc==LSM_OK ){ - - lsmShmLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_UNLOCK, 0); - - /* Try an exclusive lock on DMS2. If successful, this is the last - ** connection to the database. In this case flush the contents of the - ** in-memory tree to disk and write a checkpoint. */ - rc = lsmShmTestLock(pDb, LSM_LOCK_DMS2, 1, LSM_LOCK_EXCL); - if( rc==LSM_OK ){ - rc = lsmShmTestLock(pDb, LSM_LOCK_CHECKPOINTER, 1, LSM_LOCK_EXCL); - } - if( rc==LSM_OK ){ - int bReadonly = 0; /* True if there exist read-only conns. */ - - /* Flush the in-memory tree, if required. If there is data to flush, - ** this will create a new client snapshot in Database.pClient. The - ** checkpoint (serialization) of this snapshot may be written to disk - ** by the following block. - ** - ** There is no need to take a WRITER lock here. That there are no - ** other locks on DMS2 guarantees that there are no other read-write - ** connections at this time (and the lock on DMS1 guarantees that - ** no new ones may appear). - */ - rc = lsmTreeLoadHeader(pDb, 0); - if( rc==LSM_OK && (lsmTreeHasOld(pDb) || lsmTreeSize(pDb)>0) ){ - rc = lsmFlushTreeToDisk(pDb); - } - - /* Now check if there are any read-only connections. If there are, - ** then do not truncate the db file or unlink the shared-memory - ** region. */ - if( rc==LSM_OK ){ - rc = lsmShmTestLock(pDb, LSM_LOCK_DMS3, 1, LSM_LOCK_EXCL); - if( rc==LSM_BUSY ){ - bReadonly = 1; - rc = LSM_OK; - } - } - - /* Write a checkpoint to disk. */ - if( rc==LSM_OK ){ - rc = lsmCheckpointWrite(pDb, 0); - } - - /* If the checkpoint was written successfully, delete the log file - ** and, if possible, truncate the database file. */ - if( rc==LSM_OK ){ - int bRotrans = 0; - Database *p = pDb->pDatabase; - - /* The log file may only be deleted if there are no clients - ** read-only clients running rotrans transactions. */ - rc = lsmDetectRoTrans(pDb, &bRotrans); - if( rc==LSM_OK && bRotrans==0 ){ - lsmFsCloseAndDeleteLog(pDb->pFS); - } - - /* The database may only be truncated if there exist no read-only - ** clients - either connected or running rotrans transactions. */ - if( bReadonly==0 && bRotrans==0 ){ - lsmFsUnmap(pDb->pFS); - dbTruncateFile(pDb); - if( p->pFile && p->bMultiProc ){ - lsmEnvShmUnmap(pDb->pEnv, p->pFile, 1); - } - } - } - } - } - - if( pDb->iRwclient>=0 ){ - lsmShmLock(pDb, LSM_LOCK_RWCLIENT(pDb->iRwclient), LSM_LOCK_UNLOCK, 0); - pDb->iRwclient = -1; - } - - lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); - } - pDb->pShmhdr = 0; -} - -static int doDbConnect(lsm_db *pDb){ - const int nUsMax = 100000; /* Max value for nUs */ - int nUs = 1000; /* us to wait between DMS1 attempts */ - int rc; - - /* Obtain a pointer to the shared-memory header */ - assert( pDb->pShmhdr==0 ); - assert( pDb->bReadonly==0 ); - - /* Block for an exclusive lock on DMS1. This lock serializes all calls - ** to doDbConnect() and doDbDisconnect() across all processes. */ - while( 1 ){ - rc = lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL, 1); - if( rc!=LSM_BUSY ) break; - lsmEnvSleep(pDb->pEnv, nUs); - nUs = nUs * 2; - if( nUs>nUsMax ) nUs = nUsMax; - } - if( rc==LSM_OK ){ - rc = lsmShmCacheChunks(pDb, 1); - } - if( rc!=LSM_OK ) return rc; - pDb->pShmhdr = (ShmHeader *)pDb->apShm[0]; - - /* Try an exclusive lock on DMS2/DMS3. If successful, this is the first - ** and only connection to the database. In this case initialize the - ** shared-memory and run log file recovery. */ - assert( LSM_LOCK_DMS3==1+LSM_LOCK_DMS2 ); - rc = lsmShmTestLock(pDb, LSM_LOCK_DMS2, 2, LSM_LOCK_EXCL); - if( rc==LSM_OK ){ - memset(pDb->pShmhdr, 0, sizeof(ShmHeader)); - rc = lsmCheckpointRecover(pDb); - if( rc==LSM_OK ){ - rc = lsmLogRecover(pDb); - } - if( rc==LSM_OK ){ - ShmHeader *pShm = pDb->pShmhdr; - pShm->aReader[0].iLsmId = lsmCheckpointId(pShm->aSnap1, 0); - pShm->aReader[0].iTreeId = pDb->treehdr.iUsedShmid; - } - }else if( rc==LSM_BUSY ){ - rc = LSM_OK; - } - - /* Take a shared lock on DMS2. In multi-process mode this lock "cannot" - ** fail, as connections may only hold an exclusive lock on DMS2 if they - ** first hold an exclusive lock on DMS1. And this connection is currently - ** holding the exclusive lock on DSM1. - ** - ** However, if some other connection has the database open in single-process - ** mode, this operation will fail. In this case, return the error to the - ** caller - the attempt to connect to the db has failed. - */ - if( rc==LSM_OK ){ - rc = lsmShmLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_SHARED, 0); - } - - /* If anything went wrong, unlock DMS2. Otherwise, try to take an exclusive - ** lock on one of the LSM_LOCK_RWCLIENT() locks. Unlock DMS1 in any case. */ - if( rc!=LSM_OK ){ - pDb->pShmhdr = 0; - }else{ - int i; - for(i=0; iiRwclient = i; - if( rc2!=LSM_BUSY ){ - rc = rc2; - break; - } - } - } - lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); - - return rc; -} - -static int dbOpenSharedFd(lsm_env *pEnv, Database *p, int bRoOk){ - int rc; - - rc = lsmEnvOpen(pEnv, p->zName, 0, &p->pFile); - if( rc==LSM_IOERR && bRoOk ){ - rc = lsmEnvOpen(pEnv, p->zName, LSM_OPEN_READONLY, &p->pFile); - p->bReadonly = 1; - } - - return rc; -} - -/* -** Return a reference to the shared Database handle for the database -** identified by canonical path zName. If this is the first connection to -** the named database, a new Database object is allocated. Otherwise, a -** pointer to an existing object is returned. -** -** If successful, *ppDatabase is set to point to the shared Database -** structure and LSM_OK returned. Otherwise, *ppDatabase is set to NULL -** and and LSM error code returned. -** -** Each successful call to this function should be (eventually) matched -** by a call to lsmDbDatabaseRelease(). -*/ -int lsmDbDatabaseConnect( - lsm_db *pDb, /* Database handle */ - const char *zName /* Full-path to db file */ -){ - lsm_env *pEnv = pDb->pEnv; - int rc; /* Return code */ - Database *p = 0; /* Pointer returned via *ppDatabase */ - int nName = lsmStrlen(zName); - - assert( pDb->pDatabase==0 ); - rc = enterGlobalMutex(pEnv); - if( rc==LSM_OK ){ - - /* Search the global list for an existing object. TODO: Need something - ** better than the memcmp() below to figure out if a given Database - ** object represents the requested file. */ - for(p=gShared.pDatabase; p; p=p->pDbNext){ - if( nName==p->nName && 0==memcmp(zName, p->zName, nName) ) break; - } - - /* If no suitable Database object was found, allocate a new one. */ - if( p==0 ){ - p = (Database *)lsmMallocZeroRc(pEnv, sizeof(Database)+nName+1, &rc); - - /* If the allocation was successful, fill in other fields and - ** allocate the client mutex. */ - if( rc==LSM_OK ){ - p->bMultiProc = pDb->bMultiProc; - p->zName = (char *)&p[1]; - p->nName = nName; - memcpy((void *)p->zName, zName, nName+1); - rc = lsmMutexNew(pEnv, &p->pClientMutex); - } - - /* If nothing has gone wrong so far, open the shared fd. And if that - ** succeeds and this connection requested single-process mode, - ** attempt to take the exclusive lock on DMS2. */ - if( rc==LSM_OK ){ - int bReadonly = (pDb->bReadonly && pDb->bMultiProc); - rc = dbOpenSharedFd(pDb->pEnv, p, bReadonly); - } - - if( rc==LSM_OK && p->bMultiProc==0 ){ - /* Hold an exclusive lock DMS1 while grabbing DMS2. This ensures - ** that any ongoing call to doDbDisconnect() (even one in another - ** process) is finished before proceeding. */ - assert( p->bReadonly==0 ); - rc = lsmEnvLock(pDb->pEnv, p->pFile, LSM_LOCK_DMS1, LSM_LOCK_EXCL); - if( rc==LSM_OK ){ - rc = lsmEnvLock(pDb->pEnv, p->pFile, LSM_LOCK_DMS2, LSM_LOCK_EXCL); - lsmEnvLock(pDb->pEnv, p->pFile, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK); - } - } - - if( rc==LSM_OK ){ - p->pDbNext = gShared.pDatabase; - gShared.pDatabase = p; - }else{ - freeDatabase(pEnv, p); - p = 0; - } - } - - if( p ){ - p->nDbRef++; - } - leaveGlobalMutex(pEnv); - - if( p ){ - lsmMutexEnter(pDb->pEnv, p->pClientMutex); - pDb->pNext = p->pConn; - p->pConn = pDb; - lsmMutexLeave(pDb->pEnv, p->pClientMutex); - } - } - - pDb->pDatabase = p; - if( rc==LSM_OK ){ - assert( p ); - rc = lsmFsOpen(pDb, zName, p->bReadonly); - } - - /* If the db handle is read-write, then connect to the system now. Run - ** recovery as necessary. Or, if this is a read-only database handle, - ** defer attempting to connect to the system until a read-transaction - ** is opened. */ - if( rc==LSM_OK ){ - rc = lsmFsConfigure(pDb); - } - if( rc==LSM_OK && pDb->bReadonly==0 ){ - rc = doDbConnect(pDb); - } - - return rc; -} - -static void dbDeferClose(lsm_db *pDb){ - if( pDb->pFS ){ - LsmFile *pLsmFile; - Database *p = pDb->pDatabase; - pLsmFile = lsmFsDeferClose(pDb->pFS); - pLsmFile->pNext = p->pLsmFile; - p->pLsmFile = pLsmFile; - } -} - -LsmFile *lsmDbRecycleFd(lsm_db *db){ - LsmFile *pRet; - Database *p = db->pDatabase; - lsmMutexEnter(db->pEnv, p->pClientMutex); - if( (pRet = p->pLsmFile)!=0 ){ - p->pLsmFile = pRet->pNext; - } - lsmMutexLeave(db->pEnv, p->pClientMutex); - return pRet; -} - -/* -** Release a reference to a Database object obtained from -** lsmDbDatabaseConnect(). There should be exactly one call to this function -** for each successful call to Find(). -*/ -void lsmDbDatabaseRelease(lsm_db *pDb){ - Database *p = pDb->pDatabase; - if( p ){ - lsm_db **ppDb; - - if( pDb->pShmhdr ){ - doDbDisconnect(pDb); - } - - lsmFsUnmap(pDb->pFS); - lsmMutexEnter(pDb->pEnv, p->pClientMutex); - for(ppDb=&p->pConn; *ppDb!=pDb; ppDb=&((*ppDb)->pNext)); - *ppDb = pDb->pNext; - dbDeferClose(pDb); - lsmMutexLeave(pDb->pEnv, p->pClientMutex); - - enterGlobalMutex(pDb->pEnv); - p->nDbRef--; - if( p->nDbRef==0 ){ - LsmFile *pIter; - LsmFile *pNext; - Database **pp; - - /* Remove the Database structure from the linked list. */ - for(pp=&gShared.pDatabase; *pp!=p; pp=&((*pp)->pDbNext)); - *pp = p->pDbNext; - - /* If they were allocated from the heap, free the shared memory chunks */ - if( p->bMultiProc==0 ){ - int i; - for(i=0; inShmChunk; i++){ - lsmFree(pDb->pEnv, p->apShmChunk[i]); - } - } - - /* Close any outstanding file descriptors */ - for(pIter=p->pLsmFile; pIter; pIter=pNext){ - pNext = pIter->pNext; - lsmEnvClose(pDb->pEnv, pIter->pFile); - lsmFree(pDb->pEnv, pIter); - } - freeDatabase(pDb->pEnv, p); - } - leaveGlobalMutex(pDb->pEnv); - } -} - -Level *lsmDbSnapshotLevel(Snapshot *pSnapshot){ - return pSnapshot->pLevel; -} - -void lsmDbSnapshotSetLevel(Snapshot *pSnap, Level *pLevel){ - pSnap->pLevel = pLevel; -} - -/* TODO: Shuffle things around to get rid of this */ -static int firstSnapshotInUse(lsm_db *, i64 *); - -/* -** Context object used by the lsmWalkFreelist() utility. -*/ -typedef struct WalkFreelistCtx WalkFreelistCtx; -struct WalkFreelistCtx { - lsm_db *pDb; - int bReverse; - Freelist *pFreelist; - int iFree; - int (*xUsr)(void *, int, i64); /* User callback function */ - void *pUsrctx; /* User callback context */ - int bDone; /* Set to true after xUsr() returns true */ -}; - -/* -** Callback used by lsmWalkFreelist(). -*/ -static int walkFreelistCb(void *pCtx, int iBlk, i64 iSnapshot){ - WalkFreelistCtx *p = (WalkFreelistCtx *)pCtx; - const int iDir = (p->bReverse ? -1 : 1); - Freelist *pFree = p->pFreelist; - - assert( p->bDone==0 ); - assert( iBlk>=0 ); - if( pFree ){ - while( (p->iFree < pFree->nEntry) && p->iFree>=0 ){ - FreelistEntry *pEntry = &pFree->aEntry[p->iFree]; - if( (p->bReverse==0 && pEntry->iBlk>(u32)iBlk) - || (p->bReverse!=0 && pEntry->iBlk<(u32)iBlk) - ){ - break; - }else{ - p->iFree += iDir; - if( pEntry->iId>=0 - && p->xUsr(p->pUsrctx, pEntry->iBlk, pEntry->iId) - ){ - p->bDone = 1; - return 1; - } - if( pEntry->iBlk==(u32)iBlk ) return 0; - } - } - } - - if( p->xUsr(p->pUsrctx, iBlk, iSnapshot) ){ - p->bDone = 1; - return 1; - } - return 0; -} - -/* -** The database handle passed as the first argument must be the worker -** connection. This function iterates through the contents of the current -** free block list, invoking the supplied callback once for each list -** element. -** -** The difference between this function and lsmSortedWalkFreelist() is -** that lsmSortedWalkFreelist() only considers those free-list elements -** stored within the LSM. This function also merges in any in-memory -** elements. -*/ -int lsmWalkFreelist( - lsm_db *pDb, /* Database handle (must be worker) */ - int bReverse, /* True to iterate from largest to smallest */ - int (*x)(void *, int, i64), /* Callback function */ - void *pCtx /* First argument to pass to callback */ -){ - const int iDir = (bReverse ? -1 : 1); - int rc; - int iCtx; - - WalkFreelistCtx ctx[2]; - - ctx[0].pDb = pDb; - ctx[0].bReverse = bReverse; - ctx[0].pFreelist = &pDb->pWorker->freelist; - if( ctx[0].pFreelist && bReverse ){ - ctx[0].iFree = ctx[0].pFreelist->nEntry-1; - }else{ - ctx[0].iFree = 0; - } - ctx[0].xUsr = walkFreelistCb; - ctx[0].pUsrctx = (void *)&ctx[1]; - ctx[0].bDone = 0; - - ctx[1].pDb = pDb; - ctx[1].bReverse = bReverse; - ctx[1].pFreelist = pDb->pFreelist; - if( ctx[1].pFreelist && bReverse ){ - ctx[1].iFree = ctx[1].pFreelist->nEntry-1; - }else{ - ctx[1].iFree = 0; - } - ctx[1].xUsr = x; - ctx[1].pUsrctx = pCtx; - ctx[1].bDone = 0; - - rc = lsmSortedWalkFreelist(pDb, bReverse, walkFreelistCb, (void *)&ctx[0]); - - if( ctx[0].bDone==0 ){ - for(iCtx=0; iCtx<2; iCtx++){ - int i; - WalkFreelistCtx *p = &ctx[iCtx]; - for(i=p->iFree; - p->pFreelist && rc==LSM_OK && ipFreelist->nEntry && i>=0; - i += iDir - ){ - FreelistEntry *pEntry = &p->pFreelist->aEntry[i]; - if( pEntry->iId>=0 && p->xUsr(p->pUsrctx, pEntry->iBlk, pEntry->iId) ){ - return LSM_OK; - } - } - } - } - - return rc; -} - - -typedef struct FindFreeblockCtx FindFreeblockCtx; -struct FindFreeblockCtx { - i64 iInUse; - int iRet; - int bNotOne; -}; - -static int findFreeblockCb(void *pCtx, int iBlk, i64 iSnapshot){ - FindFreeblockCtx *p = (FindFreeblockCtx *)pCtx; - if( iSnapshotiInUse && (iBlk!=1 || p->bNotOne==0) ){ - p->iRet = iBlk; - return 1; - } - return 0; -} - -static int findFreeblock(lsm_db *pDb, i64 iInUse, int bNotOne, int *piRet){ - int rc; /* Return code */ - FindFreeblockCtx ctx; /* Context object */ - - ctx.iInUse = iInUse; - ctx.iRet = 0; - ctx.bNotOne = bNotOne; - rc = lsmWalkFreelist(pDb, 0, findFreeblockCb, (void *)&ctx); - *piRet = ctx.iRet; - - return rc; -} - -/* -** Allocate a new database file block to write data to, either by extending -** the database file or by recycling a free-list entry. The worker snapshot -** must be held in order to call this function. -** -** If successful, *piBlk is set to the block number allocated and LSM_OK is -** returned. Otherwise, *piBlk is zeroed and an lsm error code returned. -*/ -int lsmBlockAllocate(lsm_db *pDb, int iBefore, int *piBlk){ - Snapshot *p = pDb->pWorker; - int iRet = 0; /* Block number of allocated block */ - int rc = LSM_OK; - i64 iInUse = 0; /* Snapshot id still in use */ - i64 iSynced = 0; /* Snapshot id synced to disk */ - - assert( p ); - -#ifdef LSM_LOG_FREELIST - { - static int nCall = 0; - char *zFree = 0; - nCall++; - rc = lsmInfoFreelist(pDb, &zFree); - if( rc!=LSM_OK ) return rc; - lsmLogMessage(pDb, 0, "lsmBlockAllocate(): %d freelist: %s", nCall, zFree); - lsmFree(pDb->pEnv, zFree); - } -#endif - - /* Set iInUse to the smallest snapshot id that is either: - ** - ** * Currently in use by a database client, - ** * May be used by a database client in the future, or - ** * Is the most recently checkpointed snapshot (i.e. the one that will - ** be used following recovery if a failure occurs at this point). - */ - rc = lsmCheckpointSynced(pDb, &iSynced, 0, 0); - if( rc==LSM_OK && iSynced==0 ) iSynced = p->iId; - iInUse = iSynced; - if( rc==LSM_OK && pDb->iReader>=0 ){ - assert( pDb->pClient ); - iInUse = LSM_MIN(iInUse, pDb->pClient->iId); - } - if( rc==LSM_OK ) rc = firstSnapshotInUse(pDb, &iInUse); - -#ifdef LSM_LOG_FREELIST - { - lsmLogMessage(pDb, 0, "lsmBlockAllocate(): " - "snapshot-in-use: %lld (iSynced=%lld) (client-id=%lld)", - iInUse, iSynced, (pDb->iReader>=0 ? pDb->pClient->iId : 0) - ); - } -#endif - - - /* Unless there exists a read-only transaction (which prevents us from - ** recycling any blocks regardless, query the free block list for a - ** suitable block to reuse. - ** - ** It might seem more natural to check for a read-only transaction at - ** the start of this function. However, it is better do wait until after - ** the call to lsmCheckpointSynced() to do so. - */ - if( rc==LSM_OK ){ - int bRotrans; - rc = lsmDetectRoTrans(pDb, &bRotrans); - - if( rc==LSM_OK && bRotrans==0 ){ - rc = findFreeblock(pDb, iInUse, (iBefore>0), &iRet); - } - } - - if( iBefore>0 && (iRet<=0 || iRet>=iBefore) ){ - iRet = 0; - - }else if( rc==LSM_OK ){ - /* If a block was found in the free block list, use it and remove it from - ** the list. Otherwise, if no suitable block was found, allocate one from - ** the end of the file. */ - if( iRet>0 ){ -#ifdef LSM_LOG_FREELIST - lsmLogMessage(pDb, 0, - "reusing block %d (snapshot-in-use=%lld)", iRet, iInUse); -#endif - rc = freelistAppend(pDb, iRet, -1); - if( rc==LSM_OK ){ - rc = dbTruncate(pDb, iInUse); - } - }else{ - iRet = ++(p->nBlock); -#ifdef LSM_LOG_FREELIST - lsmLogMessage(pDb, 0, "extending file to %d blocks", iRet); -#endif - } - } - - assert( iBefore>0 || iRet>0 || rc!=LSM_OK ); - *piBlk = iRet; - return rc; -} - -/* -** Free a database block. The worker snapshot must be held in order to call -** this function. -** -** If successful, LSM_OK is returned. Otherwise, an lsm error code (e.g. -** LSM_NOMEM). -*/ -int lsmBlockFree(lsm_db *pDb, int iBlk){ - Snapshot *p = pDb->pWorker; - assert( lsmShmAssertWorker(pDb) ); - -#ifdef LSM_LOG_FREELIST - lsmLogMessage(pDb, LSM_OK, "lsmBlockFree(): Free block %d", iBlk); -#endif - - return freelistAppend(pDb, iBlk, p->iId); -} - -/* -** Refree a database block. The worker snapshot must be held in order to call -** this function. -** -** Refreeing is required when a block is allocated using lsmBlockAllocate() -** but then not used. This function is used to push the block back onto -** the freelist. Refreeing a block is different from freeing is, as a refreed -** block may be reused immediately. Whereas a freed block can not be reused -** until (at least) after the next checkpoint. -*/ -int lsmBlockRefree(lsm_db *pDb, int iBlk){ - int rc = LSM_OK; /* Return code */ - -#ifdef LSM_LOG_FREELIST - lsmLogMessage(pDb, LSM_OK, "lsmBlockRefree(): Refree block %d", iBlk); -#endif - - rc = freelistAppend(pDb, iBlk, 0); - return rc; -} - -/* -** If required, copy a database checkpoint from shared memory into the -** database itself. -** -** The WORKER lock must not be held when this is called. This is because -** this function may indirectly call fsync(). And the WORKER lock should -** not be held that long (in case it is required by a client flushing an -** in-memory tree to disk). -*/ -int lsmCheckpointWrite(lsm_db *pDb, u32 *pnWrite){ - int rc; /* Return Code */ - u32 nWrite = 0; - - assert( pDb->pWorker==0 ); - assert( 1 || pDb->pClient==0 ); - assert( lsmShmAssertLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_UNLOCK) ); - - rc = lsmShmLock(pDb, LSM_LOCK_CHECKPOINTER, LSM_LOCK_EXCL, 0); - if( rc!=LSM_OK ) return rc; - - rc = lsmCheckpointLoad(pDb, 0); - if( rc==LSM_OK ){ - int nBlock = lsmCheckpointNBlock(pDb->aSnapshot); - ShmHeader *pShm = pDb->pShmhdr; - int bDone = 0; /* True if checkpoint is already stored */ - - /* Check if this checkpoint has already been written to the database - ** file. If so, set variable bDone to true. */ - if( pShm->iMetaPage ){ - MetaPage *pPg; /* Meta page */ - u8 *aData; /* Meta-page data buffer */ - int nData; /* Size of aData[] in bytes */ - i64 iCkpt; /* Id of checkpoint just loaded */ - i64 iDisk = 0; /* Id of checkpoint already stored in db */ - iCkpt = lsmCheckpointId(pDb->aSnapshot, 0); - rc = lsmFsMetaPageGet(pDb->pFS, 0, pShm->iMetaPage, &pPg); - if( rc==LSM_OK ){ - aData = lsmFsMetaPageData(pPg, &nData); - iDisk = lsmCheckpointId((u32 *)aData, 1); - nWrite = lsmCheckpointNWrite((u32 *)aData, 1); - lsmFsMetaPageRelease(pPg); - } - bDone = (iDisk>=iCkpt); - } - - if( rc==LSM_OK && bDone==0 ){ - int iMeta = (pShm->iMetaPage % 2) + 1; - if( pDb->eSafety!=LSM_SAFETY_OFF ){ - rc = lsmFsSyncDb(pDb->pFS, nBlock); - } - if( rc==LSM_OK ) rc = lsmCheckpointStore(pDb, iMeta); - if( rc==LSM_OK && pDb->eSafety!=LSM_SAFETY_OFF){ - rc = lsmFsSyncDb(pDb->pFS, 0); - } - if( rc==LSM_OK ){ - pShm->iMetaPage = iMeta; - nWrite = lsmCheckpointNWrite(pDb->aSnapshot, 0) - nWrite; - } -#ifdef LSM_LOG_WORK - lsmLogMessage(pDb, 0, "finish checkpoint %d", - (int)lsmCheckpointId(pDb->aSnapshot, 0) - ); -#endif - } - } - - lsmShmLock(pDb, LSM_LOCK_CHECKPOINTER, LSM_LOCK_UNLOCK, 0); - if( pnWrite && rc==LSM_OK ) *pnWrite = nWrite; - return rc; -} - -int lsmBeginWork(lsm_db *pDb){ - int rc; - - /* Attempt to take the WORKER lock */ - rc = lsmShmLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_EXCL, 0); - - /* Deserialize the current worker snapshot */ - if( rc==LSM_OK ){ - rc = lsmCheckpointLoadWorker(pDb); - } - return rc; -} - -void lsmFreeSnapshot(lsm_env *pEnv, Snapshot *p){ - if( p ){ - lsmSortedFreeLevel(pEnv, p->pLevel); - lsmFree(pEnv, p->freelist.aEntry); - lsmFree(pEnv, p->redirect.a); - lsmFree(pEnv, p); - } -} - -/* -** Attempt to populate one of the read-lock slots to contain lock values -** iLsm/iShm. Or, if such a slot exists already, this function is a no-op. -** -** It is not an error if no slot can be populated because the write-lock -** cannot be obtained. If any other error occurs, return an LSM error code. -** Otherwise, LSM_OK. -** -** This function is called at various points to try to ensure that there -** always exists at least one read-lock slot that can be used by a read-only -** client. And so that, in the usual case, there is an "exact match" available -** whenever a read transaction is opened by any client. At present this -** function is called when: -** -** * A write transaction that called lsmTreeDiscardOld() is committed, and -** * Whenever the working snapshot is updated (i.e. lsmFinishWork()). -*/ -static int dbSetReadLock(lsm_db *db, i64 iLsm, u32 iShm){ - int rc = LSM_OK; - ShmHeader *pShm = db->pShmhdr; - int i; - - /* Check if there is already a slot containing the required values. */ - for(i=0; iaReader[i]; - if( p->iLsmId==iLsm && p->iTreeId==iShm ) return LSM_OK; - } - - /* Iterate through all read-lock slots, attempting to take a write-lock - ** on each of them. If a write-lock succeeds, populate the locked slot - ** with the required values and break out of the loop. */ - for(i=0; rc==LSM_OK && iaReader[i]; - p->iLsmId = iLsm; - p->iTreeId = iShm; - lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_UNLOCK, 0); - break; - } - } - - return rc; -} - -/* -** Release the read-lock currently held by connection db. -*/ -int dbReleaseReadlock(lsm_db *db){ - int rc = LSM_OK; - if( db->iReader>=0 ){ - rc = lsmShmLock(db, LSM_LOCK_READER(db->iReader), LSM_LOCK_UNLOCK, 0); - db->iReader = -1; - } - db->bRoTrans = 0; - return rc; -} - - -/* -** Argument bFlush is true if the contents of the in-memory tree has just -** been flushed to disk. The significance of this is that once the snapshot -** created to hold the updated state of the database is synced to disk, log -** file space can be recycled. -*/ -void lsmFinishWork(lsm_db *pDb, int bFlush, int *pRc){ - int rc = *pRc; - assert( rc!=0 || pDb->pWorker ); - if( pDb->pWorker ){ - /* If no error has occurred, serialize the worker snapshot and write - ** it to shared memory. */ - if( rc==LSM_OK ){ - rc = lsmSaveWorker(pDb, bFlush); - } - - /* Assuming no error has occurred, update a read lock slot with the - ** new snapshot id (see comments above function dbSetReadLock()). */ - if( rc==LSM_OK ){ - if( pDb->iReader<0 ){ - rc = lsmTreeLoadHeader(pDb, 0); - } - if( rc==LSM_OK ){ - rc = dbSetReadLock(pDb, pDb->pWorker->iId, pDb->treehdr.iUsedShmid); - } - } - - /* Free the snapshot object. */ - lsmFreeSnapshot(pDb->pEnv, pDb->pWorker); - pDb->pWorker = 0; - } - - lsmShmLock(pDb, LSM_LOCK_WORKER, LSM_LOCK_UNLOCK, 0); - *pRc = rc; -} - -/* -** Called when recovery is finished. -*/ -int lsmFinishRecovery(lsm_db *pDb){ - lsmTreeEndTransaction(pDb, 1); - return LSM_OK; -} - -/* -** Check if the currently configured compression functions -** (LSM_CONFIG_SET_COMPRESSION) are compatible with a database that has its -** compression id set to iReq. Compression routines are compatible if iReq -** is zero (indicating the database is empty), or if it is equal to the -** compression id of the configured compression routines. -** -** If the check shows that the current compression are incompatible and there -** is a compression factory registered, give it a chance to install new -** compression routines. -** -** If, after any registered factory is invoked, the compression functions -** are still incompatible, return LSM_MISMATCH. Otherwise, LSM_OK. -*/ -int lsmCheckCompressionId(lsm_db *pDb, u32 iReq){ - if( iReq!=LSM_COMPRESSION_EMPTY && pDb->compress.iId!=iReq ){ - if( pDb->factory.xFactory ){ - pDb->bInFactory = 1; - pDb->factory.xFactory(pDb->factory.pCtx, pDb, iReq); - pDb->bInFactory = 0; - } - if( pDb->compress.iId!=iReq ){ - /* Incompatible */ - return LSM_MISMATCH; - } - } - /* Compatible */ - return LSM_OK; -} - -/* -** Begin a read transaction. This function is a no-op if the connection -** passed as the only argument already has an open read transaction. -*/ -int lsmBeginReadTrans(lsm_db *pDb){ - const int MAX_READLOCK_ATTEMPTS = 10; - const int nMaxAttempt = (pDb->bRoTrans ? 1 : MAX_READLOCK_ATTEMPTS); - - int rc = LSM_OK; /* Return code */ - int iAttempt = 0; - - assert( pDb->pWorker==0 ); - - while( rc==LSM_OK && pDb->iReader<0 && (iAttempt++)pCsr==0 && pDb->nTransOpen==0 ); - - /* Load the in-memory tree header. */ - rc = lsmTreeLoadHeader(pDb, &iTreehdr); - - /* Load the database snapshot */ - if( rc==LSM_OK ){ - if( lsmCheckpointClientCacheOk(pDb)==0 ){ - lsmFreeSnapshot(pDb->pEnv, pDb->pClient); - pDb->pClient = 0; - lsmMCursorFreeCache(pDb); - lsmFsPurgeCache(pDb->pFS); - rc = lsmCheckpointLoad(pDb, &iSnap); - }else{ - iSnap = 1; - } - } - - /* Take a read-lock on the tree and snapshot just loaded. Then check - ** that the shared-memory still contains the same values. If so, proceed. - ** Otherwise, relinquish the read-lock and retry the whole procedure - ** (starting with loading the in-memory tree header). */ - if( rc==LSM_OK ){ - u32 iShmMax = pDb->treehdr.iUsedShmid; - u32 iShmMin = pDb->treehdr.iNextShmid+1-LSM_MAX_SHMCHUNKS; - rc = lsmReadlock( - pDb, lsmCheckpointId(pDb->aSnapshot, 0), iShmMin, iShmMax - ); - if( rc==LSM_OK ){ - if( lsmTreeLoadHeaderOk(pDb, iTreehdr) - && lsmCheckpointLoadOk(pDb, iSnap) - ){ - /* Read lock has been successfully obtained. Deserialize the - ** checkpoint just loaded. TODO: This will be removed after - ** lsm_sorted.c is changed to work directly from the serialized - ** version of the snapshot. */ - if( pDb->pClient==0 ){ - rc = lsmCheckpointDeserialize(pDb, 0, pDb->aSnapshot,&pDb->pClient); - } - assert( (rc==LSM_OK)==(pDb->pClient!=0) ); - assert( pDb->iReader>=0 ); - - /* Check that the client has the right compression hooks loaded. - ** If not, set rc to LSM_MISMATCH. */ - if( rc==LSM_OK ){ - rc = lsmCheckCompressionId(pDb, pDb->pClient->iCmpId); - } - }else{ - rc = dbReleaseReadlock(pDb); - } - } - - if( rc==LSM_BUSY ){ - rc = LSM_OK; - } - } -#if 0 -if( rc==LSM_OK && pDb->pClient ){ - fprintf(stderr, - "reading %p: snapshot:%d used-shmid:%d trans-id:%d iOldShmid=%d\n", - (void *)pDb, - (int)pDb->pClient->iId, (int)pDb->treehdr.iUsedShmid, - (int)pDb->treehdr.root.iTransId, - (int)pDb->treehdr.iOldShmid - ); -} -#endif - } - - if( rc==LSM_OK ){ - rc = lsmShmCacheChunks(pDb, pDb->treehdr.nChunk); - } - if( rc!=LSM_OK ){ - dbReleaseReadlock(pDb); - } - if( pDb->pClient==0 && rc==LSM_OK ) rc = LSM_BUSY; - return rc; -} - -/* -** This function is used by a read-write connection to determine if there -** are currently one or more read-only transactions open on the database -** (in this context a read-only transaction is one opened by a read-only -** connection on a non-live database). -** -** If no error occurs, LSM_OK is returned and *pbExists is set to true if -** some other connection has a read-only transaction open, or false -** otherwise. If an error occurs an LSM error code is returned and the final -** value of *pbExist is undefined. -*/ -int lsmDetectRoTrans(lsm_db *db, int *pbExist){ - int rc; - - /* Only a read-write connection may use this function. */ - assert( db->bReadonly==0 ); - - rc = lsmShmTestLock(db, LSM_LOCK_ROTRANS, 1, LSM_LOCK_EXCL); - if( rc==LSM_BUSY ){ - *pbExist = 1; - rc = LSM_OK; - }else{ - *pbExist = 0; - } - - return rc; -} - -/* -** db is a read-only database handle in the disconnected state. This function -** attempts to open a read-transaction on the database. This may involve -** connecting to the database system (opening shared memory etc.). -*/ -int lsmBeginRoTrans(lsm_db *db){ - int rc = LSM_OK; - - assert( db->bReadonly && db->pShmhdr==0 ); - assert( db->iReader<0 ); - - if( db->bRoTrans==0 ){ - - /* Attempt a shared-lock on DMS1. */ - rc = lsmShmLock(db, LSM_LOCK_DMS1, LSM_LOCK_SHARED, 0); - if( rc!=LSM_OK ) return rc; - - rc = lsmShmTestLock( - db, LSM_LOCK_RWCLIENT(0), LSM_LOCK_NREADER, LSM_LOCK_SHARED - ); - if( rc==LSM_OK ){ - /* System is not live. Take a SHARED lock on the ROTRANS byte and - ** release DMS1. Locking ROTRANS tells all read-write clients that they - ** may not recycle any disk space from within the database or log files, - ** as a read-only client may be using it. */ - rc = lsmShmLock(db, LSM_LOCK_ROTRANS, LSM_LOCK_SHARED, 0); - lsmShmLock(db, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); - - if( rc==LSM_OK ){ - db->bRoTrans = 1; - rc = lsmShmCacheChunks(db, 1); - if( rc==LSM_OK ){ - db->pShmhdr = (ShmHeader *)db->apShm[0]; - memset(db->pShmhdr, 0, sizeof(ShmHeader)); - rc = lsmCheckpointRecover(db); - if( rc==LSM_OK ){ - rc = lsmLogRecover(db); - } - } - } - }else if( rc==LSM_BUSY ){ - /* System is live! */ - rc = lsmShmLock(db, LSM_LOCK_DMS3, LSM_LOCK_SHARED, 0); - lsmShmLock(db, LSM_LOCK_DMS1, LSM_LOCK_UNLOCK, 0); - if( rc==LSM_OK ){ - rc = lsmShmCacheChunks(db, 1); - if( rc==LSM_OK ){ - db->pShmhdr = (ShmHeader *)db->apShm[0]; - } - } - } - - /* In 'lsm_open()' we don't update the page and block sizes in the - ** Filesystem for 'readonly' connection. Because member 'db->pShmhdr' is a - ** nullpointer, this prevents loading a checkpoint. Now that the system is - ** live this member should be set. So we can update both values in - ** the Filesystem. - ** - ** Configure the file-system connection with the page-size and block-size - ** of this database. Even if the database file is zero bytes in size - ** on disk, these values have been set in shared-memory by now, and so - ** are guaranteed not to change during the lifetime of this connection. */ - if( LSM_OK==rc - && 0==lsmCheckpointClientCacheOk(db) - && LSM_OK==(rc=lsmCheckpointLoad(db, 0)) - ){ - lsmFsSetPageSize(db->pFS, lsmCheckpointPgsz(db->aSnapshot)); - lsmFsSetBlockSize(db->pFS, lsmCheckpointBlksz(db->aSnapshot)); - } - - if( rc==LSM_OK ){ - rc = lsmBeginReadTrans(db); - } - } - - return rc; -} - -/* -** Close the currently open read transaction. -*/ -void lsmFinishReadTrans(lsm_db *pDb){ - - /* Worker connections should not be closing read transactions. And - ** read transactions should only be closed after all cursors and write - ** transactions have been closed. Finally pClient should be non-NULL - ** only iff pDb->iReader>=0. */ - assert( pDb->pWorker==0 ); - assert( pDb->pCsr==0 && pDb->nTransOpen==0 ); - - if( pDb->bRoTrans ){ - int i; - for(i=0; inShm; i++){ - lsmFree(pDb->pEnv, pDb->apShm[i]); - } - lsmFree(pDb->pEnv, pDb->apShm); - pDb->apShm = 0; - pDb->nShm = 0; - pDb->pShmhdr = 0; - - lsmShmLock(pDb, LSM_LOCK_ROTRANS, LSM_LOCK_UNLOCK, 0); - } - dbReleaseReadlock(pDb); -} - -/* -** Open a write transaction. -*/ -int lsmBeginWriteTrans(lsm_db *pDb){ - int rc = LSM_OK; /* Return code */ - ShmHeader *pShm = pDb->pShmhdr; /* Shared memory header */ - - assert( pDb->nTransOpen==0 ); - assert( pDb->bDiscardOld==0 ); - assert( pDb->bReadonly==0 ); - - /* If there is no read-transaction open, open one now. */ - if( pDb->iReader<0 ){ - rc = lsmBeginReadTrans(pDb); - } - - /* Attempt to take the WRITER lock */ - if( rc==LSM_OK ){ - rc = lsmShmLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_EXCL, 0); - } - - /* If the previous writer failed mid-transaction, run emergency rollback. */ - if( rc==LSM_OK && pShm->bWriter ){ - rc = lsmTreeRepair(pDb); - if( rc==LSM_OK ) pShm->bWriter = 0; - } - - /* Check that this connection is currently reading from the most recent - ** version of the database. If not, return LSM_BUSY. */ - if( rc==LSM_OK && memcmp(&pShm->hdr1, &pDb->treehdr, sizeof(TreeHeader)) ){ - rc = LSM_BUSY; - } - - if( rc==LSM_OK ){ - rc = lsmLogBegin(pDb); - } - - /* If everything was successful, set the "transaction-in-progress" flag - ** and return LSM_OK. Otherwise, if some error occurred, relinquish the - ** WRITER lock and return an error code. */ - if( rc==LSM_OK ){ - TreeHeader *p = &pDb->treehdr; - pShm->bWriter = 1; - p->root.iTransId++; - if( lsmTreeHasOld(pDb) && p->iOldLog==pDb->pClient->iLogOff ){ - lsmTreeDiscardOld(pDb); - pDb->bDiscardOld = 1; - } - }else{ - lsmShmLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_UNLOCK, 0); - if( pDb->pCsr==0 ) lsmFinishReadTrans(pDb); - } - return rc; -} - -/* -** End the current write transaction. The connection is left with an open -** read transaction. It is an error to call this if there is no open write -** transaction. -** -** If the transaction was committed, then a commit record has already been -** written into the log file when this function is called. Or, if the -** transaction was rolled back, both the log file and in-memory tree -** structure have already been restored. In either case, this function -** merely releases locks and other resources held by the write-transaction. -** -** LSM_OK is returned if successful, or an LSM error code otherwise. -*/ -int lsmFinishWriteTrans(lsm_db *pDb, int bCommit){ - int rc = LSM_OK; - int bFlush = 0; - - lsmLogEnd(pDb, bCommit); - if( rc==LSM_OK && bCommit && lsmTreeSize(pDb)>pDb->nTreeLimit ){ - bFlush = 1; - lsmTreeMakeOld(pDb); - } - lsmTreeEndTransaction(pDb, bCommit); - - if( rc==LSM_OK ){ - if( bFlush && pDb->bAutowork ){ - rc = lsmSortedAutoWork(pDb, 1); - }else if( bCommit && pDb->bDiscardOld ){ - rc = dbSetReadLock(pDb, pDb->pClient->iId, pDb->treehdr.iUsedShmid); - } - } - pDb->bDiscardOld = 0; - lsmShmLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_UNLOCK, 0); - - if( bFlush && pDb->bAutowork==0 && pDb->xWork ){ - pDb->xWork(pDb, pDb->pWorkCtx); - } - return rc; -} - - -/* -** Return non-zero if the caller is holding the client mutex. -*/ -#ifdef LSM_DEBUG -int lsmHoldingClientMutex(lsm_db *pDb){ - return lsmMutexHeld(pDb->pEnv, pDb->pDatabase->pClientMutex); -} -#endif - -static int slotIsUsable(ShmReader *p, i64 iLsm, u32 iShmMin, u32 iShmMax){ - return( - p->iLsmId && p->iLsmId<=iLsm - && shm_sequence_ge(iShmMax, p->iTreeId) - && shm_sequence_ge(p->iTreeId, iShmMin) - ); -} - -/* -** Obtain a read-lock on database version identified by the combination -** of snapshot iLsm and tree iTree. Return LSM_OK if successful, or -** an LSM error code otherwise. -*/ -int lsmReadlock(lsm_db *db, i64 iLsm, u32 iShmMin, u32 iShmMax){ - int rc = LSM_OK; - ShmHeader *pShm = db->pShmhdr; - int i; - - assert( db->iReader<0 ); - assert( shm_sequence_ge(iShmMax, iShmMin) ); - - /* This is a no-op if the read-only transaction flag is set. */ - if( db->bRoTrans ){ - db->iReader = 0; - return LSM_OK; - } - - /* Search for an exact match. */ - for(i=0; db->iReader<0 && rc==LSM_OK && iaReader[i]; - if( p->iLsmId==iLsm && p->iTreeId==iShmMax ){ - rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_SHARED, 0); - if( rc==LSM_OK && p->iLsmId==iLsm && p->iTreeId==iShmMax ){ - db->iReader = i; - }else if( rc==LSM_BUSY ){ - rc = LSM_OK; - } - } - } - - /* Try to obtain a write-lock on each slot, in order. If successful, set - ** the slot values to iLsm/iTree. */ - for(i=0; db->iReader<0 && rc==LSM_OK && iaReader[i]; - p->iLsmId = iLsm; - p->iTreeId = iShmMax; - rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_SHARED, 0); - assert( rc!=LSM_BUSY ); - if( rc==LSM_OK ) db->iReader = i; - } - } - - /* Search for any usable slot */ - for(i=0; db->iReader<0 && rc==LSM_OK && iaReader[i]; - if( slotIsUsable(p, iLsm, iShmMin, iShmMax) ){ - rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_SHARED, 0); - if( rc==LSM_OK && slotIsUsable(p, iLsm, iShmMin, iShmMax) ){ - db->iReader = i; - }else if( rc==LSM_BUSY ){ - rc = LSM_OK; - } - } - } - - if( rc==LSM_OK && db->iReader<0 ){ - rc = LSM_BUSY; - } - return rc; -} - -/* -** This is used to check if there exists a read-lock locking a particular -** version of either the in-memory tree or database file. -** -** If iLsmId is non-zero, then it is a snapshot id. If there exists a -** read-lock using this snapshot or newer, set *pbInUse to true. Or, -** if there is no such read-lock, set it to false. -** -** Or, if iLsmId is zero, then iShmid is a shared-memory sequence id. -** Search for a read-lock using this sequence id or newer. etc. -*/ -static int isInUse(lsm_db *db, i64 iLsmId, u32 iShmid, int *pbInUse){ - ShmHeader *pShm = db->pShmhdr; - int i; - int rc = LSM_OK; - - for(i=0; rc==LSM_OK && iaReader[i]; - if( p->iLsmId ){ - if( (iLsmId!=0 && p->iLsmId!=0 && iLsmId>=p->iLsmId) - || (iLsmId==0 && shm_sequence_ge(p->iTreeId, iShmid)) - ){ - rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_EXCL, 0); - if( rc==LSM_OK ){ - p->iLsmId = 0; - lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_UNLOCK, 0); - } - } - } - } - - if( rc==LSM_BUSY ){ - *pbInUse = 1; - return LSM_OK; - } - *pbInUse = 0; - return rc; -} - -/* -** This function is called by worker connections to determine the smallest -** snapshot id that is currently in use by a database client. The worker -** connection uses this result to determine whether or not it is safe to -** recycle a database block. -*/ -static int firstSnapshotInUse( - lsm_db *db, /* Database handle */ - i64 *piInUse /* IN/OUT: Smallest snapshot id in use */ -){ - ShmHeader *pShm = db->pShmhdr; - i64 iInUse = *piInUse; - int i; - - assert( iInUse>0 ); - for(i=0; iaReader[i]; - if( p->iLsmId ){ - i64 iThis = p->iLsmId; - if( iThis!=0 && iInUse>iThis ){ - int rc = lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_EXCL, 0); - if( rc==LSM_OK ){ - p->iLsmId = 0; - lsmShmLock(db, LSM_LOCK_READER(i), LSM_LOCK_UNLOCK, 0); - }else if( rc==LSM_BUSY ){ - iInUse = iThis; - }else{ - /* Some error other than LSM_BUSY. Return the error code to - ** the caller in this case. */ - return rc; - } - } - } - } - - *piInUse = iInUse; - return LSM_OK; -} - -int lsmTreeInUse(lsm_db *db, u32 iShmid, int *pbInUse){ - if( db->treehdr.iUsedShmid==iShmid ){ - *pbInUse = 1; - return LSM_OK; - } - return isInUse(db, 0, iShmid, pbInUse); -} - -int lsmLsmInUse(lsm_db *db, i64 iLsmId, int *pbInUse){ - if( db->pClient && db->pClient->iId<=iLsmId ){ - *pbInUse = 1; - return LSM_OK; - } - return isInUse(db, iLsmId, 0, pbInUse); -} - -/* -** This function may only be called after a successful call to -** lsmDbDatabaseConnect(). It returns true if the connection is in -** multi-process mode, or false otherwise. -*/ -int lsmDbMultiProc(lsm_db *pDb){ - return pDb->pDatabase && pDb->pDatabase->bMultiProc; -} - - -/************************************************************************* -************************************************************************** -************************************************************************** -************************************************************************** -************************************************************************** -*************************************************************************/ - -/* -** Ensure that database connection db has cached pointers to at least the -** first nChunk chunks of shared memory. -*/ -int lsmShmCacheChunks(lsm_db *db, int nChunk){ - int rc = LSM_OK; - if( nChunk>db->nShm ){ - static const int NINCR = 16; - Database *p = db->pDatabase; - lsm_env *pEnv = db->pEnv; - int nAlloc; - int i; - - /* Ensure that the db->apShm[] array is large enough. If an attempt to - ** allocate memory fails, return LSM_NOMEM immediately. The apShm[] array - ** is always extended in multiples of 16 entries - so the actual allocated - ** size can be inferred from nShm. */ - nAlloc = ((db->nShm + NINCR - 1) / NINCR) * NINCR; - while( nChunk>=nAlloc ){ - void **apShm; - nAlloc += NINCR; - apShm = lsmRealloc(pEnv, db->apShm, sizeof(void*)*nAlloc); - if( !apShm ) return LSM_NOMEM_BKPT; - db->apShm = apShm; - } - - if( db->bRoTrans ){ - for(i=db->nShm; rc==LSM_OK && iapShm[i] = lsmMallocZeroRc(pEnv, LSM_SHM_CHUNK_SIZE, &rc); - db->nShm++; - } - - }else{ - - /* Enter the client mutex */ - lsmMutexEnter(pEnv, p->pClientMutex); - - /* Extend the Database objects apShmChunk[] array if necessary. Using the - ** same pattern as for the lsm_db.apShm[] array above. */ - nAlloc = ((p->nShmChunk + NINCR - 1) / NINCR) * NINCR; - while( nChunk>=nAlloc ){ - void **apShm; - nAlloc += NINCR; - apShm = lsmRealloc(pEnv, p->apShmChunk, sizeof(void*)*nAlloc); - if( !apShm ){ - rc = LSM_NOMEM_BKPT; - break; - } - p->apShmChunk = apShm; - } - - for(i=db->nShm; rc==LSM_OK && i=p->nShmChunk ){ - void *pChunk = 0; - if( p->bMultiProc==0 ){ - /* Single process mode */ - pChunk = lsmMallocZeroRc(pEnv, LSM_SHM_CHUNK_SIZE, &rc); - }else{ - /* Multi-process mode */ - rc = lsmEnvShmMap(pEnv, p->pFile, i, LSM_SHM_CHUNK_SIZE, &pChunk); - } - if( rc==LSM_OK ){ - p->apShmChunk[i] = pChunk; - p->nShmChunk++; - } - } - if( rc==LSM_OK ){ - db->apShm[i] = p->apShmChunk[i]; - db->nShm++; - } - } - - /* Release the client mutex */ - lsmMutexLeave(pEnv, p->pClientMutex); - } - } - - return rc; -} - -static int lockSharedFile(lsm_env *pEnv, Database *p, int iLock, int eOp){ - int rc = LSM_OK; - if( p->bMultiProc ){ - rc = lsmEnvLock(pEnv, p->pFile, iLock, eOp); - } - return rc; -} - -/* -** Test if it would be possible for connection db to obtain a lock of type -** eType on the nLock locks starting at iLock. If so, return LSM_OK. If it -** would not be possible to obtain the lock due to a lock held by another -** connection, return LSM_BUSY. If an IO or other error occurs (i.e. in the -** lsm_env.xTestLock function), return some other LSM error code. -** -** Note that this function never actually locks the database - it merely -** queries the system to see if there exists a lock that would prevent -** it from doing so. -*/ -int lsmShmTestLock( - lsm_db *db, - int iLock, - int nLock, - int eOp -){ - int rc = LSM_OK; - lsm_db *pIter; - Database *p = db->pDatabase; - int i; - u64 mask = 0; - - for(i=iLock; i<(iLock+nLock); i++){ - mask |= ((u64)1 << (iLock-1)); - if( eOp==LSM_LOCK_EXCL ) mask |= ((u64)1 << (iLock+32-1)); - } - - lsmMutexEnter(db->pEnv, p->pClientMutex); - for(pIter=p->pConn; pIter; pIter=pIter->pNext){ - if( pIter!=db && (pIter->mLock & mask) ){ - assert( pIter!=db ); - break; - } - } - - if( pIter ){ - rc = LSM_BUSY; - }else if( p->bMultiProc ){ - rc = lsmEnvTestLock(db->pEnv, p->pFile, iLock, nLock, eOp); - } - - lsmMutexLeave(db->pEnv, p->pClientMutex); - return rc; -} - -/* -** Attempt to obtain the lock identified by the iLock and bExcl parameters. -** If successful, return LSM_OK. If the lock cannot be obtained because -** there exists some other conflicting lock, return LSM_BUSY. If some other -** error occurs, return an LSM error code. -** -** Parameter iLock must be one of LSM_LOCK_WRITER, WORKER or CHECKPOINTER, -** or else a value returned by the LSM_LOCK_READER macro. -*/ -int lsmShmLock( - lsm_db *db, - int iLock, - int eOp, /* One of LSM_LOCK_UNLOCK, SHARED or EXCL */ - int bBlock /* True for a blocking lock */ -){ - lsm_db *pIter; - const u64 me = ((u64)1 << (iLock-1)); - const u64 ms = ((u64)1 << (iLock+32-1)); - int rc = LSM_OK; - Database *p = db->pDatabase; - - assert( eOp!=LSM_LOCK_EXCL || p->bReadonly==0 ); - assert( iLock>=1 && iLock<=LSM_LOCK_RWCLIENT(LSM_LOCK_NRWCLIENT-1) ); - assert( LSM_LOCK_RWCLIENT(LSM_LOCK_NRWCLIENT-1)<=32 ); - assert( eOp==LSM_LOCK_UNLOCK || eOp==LSM_LOCK_SHARED || eOp==LSM_LOCK_EXCL ); - - /* Check for a no-op. Proceed only if this is not one of those. */ - if( (eOp==LSM_LOCK_UNLOCK && (db->mLock & (me|ms))!=0) - || (eOp==LSM_LOCK_SHARED && (db->mLock & (me|ms))!=ms) - || (eOp==LSM_LOCK_EXCL && (db->mLock & me)==0) - ){ - int nExcl = 0; /* Number of connections holding EXCLUSIVE */ - int nShared = 0; /* Number of connections holding SHARED */ - lsmMutexEnter(db->pEnv, p->pClientMutex); - - /* Figure out the locks currently held by this process on iLock, not - ** including any held by connection db. */ - for(pIter=p->pConn; pIter; pIter=pIter->pNext){ - assert( (pIter->mLock & me)==0 || (pIter->mLock & ms)!=0 ); - if( pIter!=db ){ - if( pIter->mLock & me ){ - nExcl++; - }else if( pIter->mLock & ms ){ - nShared++; - } - } - } - assert( nExcl==0 || nExcl==1 ); - assert( nExcl==0 || nShared==0 ); - assert( nExcl==0 || (db->mLock & (me|ms))==0 ); - - switch( eOp ){ - case LSM_LOCK_UNLOCK: - if( nShared==0 ){ - lockSharedFile(db->pEnv, p, iLock, LSM_LOCK_UNLOCK); - } - db->mLock &= ~(me|ms); - break; - - case LSM_LOCK_SHARED: - if( nExcl ){ - rc = LSM_BUSY; - }else{ - if( nShared==0 ){ - rc = lockSharedFile(db->pEnv, p, iLock, LSM_LOCK_SHARED); - } - if( rc==LSM_OK ){ - db->mLock |= ms; - db->mLock &= ~me; - } - } - break; - - default: - assert( eOp==LSM_LOCK_EXCL ); - if( nExcl || nShared ){ - rc = LSM_BUSY; - }else{ - rc = lockSharedFile(db->pEnv, p, iLock, LSM_LOCK_EXCL); - if( rc==LSM_OK ){ - db->mLock |= (me|ms); - } - } - break; - } - - lsmMutexLeave(db->pEnv, p->pClientMutex); - } - - return rc; -} - -#ifdef LSM_DEBUG - -int shmLockType(lsm_db *db, int iLock){ - const u64 me = ((u64)1 << (iLock-1)); - const u64 ms = ((u64)1 << (iLock+32-1)); - - if( db->mLock & me ) return LSM_LOCK_EXCL; - if( db->mLock & ms ) return LSM_LOCK_SHARED; - return LSM_LOCK_UNLOCK; -} - -/* -** The arguments passed to this function are similar to those passed to -** the lsmShmLock() function. However, instead of obtaining a new lock -** this function returns true if the specified connection already holds -** (or does not hold) such a lock, depending on the value of eOp. As -** follows: -** -** (eOp==LSM_LOCK_UNLOCK) -> true if db has no lock on iLock -** (eOp==LSM_LOCK_SHARED) -> true if db has at least a SHARED lock on iLock. -** (eOp==LSM_LOCK_EXCL) -> true if db has an EXCLUSIVE lock on iLock. -*/ -int lsmShmAssertLock(lsm_db *db, int iLock, int eOp){ - int ret = 0; - int eHave; - - assert( iLock>=1 && iLock<=LSM_LOCK_READER(LSM_LOCK_NREADER-1) ); - assert( iLock<=16 ); - assert( eOp==LSM_LOCK_UNLOCK || eOp==LSM_LOCK_SHARED || eOp==LSM_LOCK_EXCL ); - - eHave = shmLockType(db, iLock); - - switch( eOp ){ - case LSM_LOCK_UNLOCK: - ret = (eHave==LSM_LOCK_UNLOCK); - break; - case LSM_LOCK_SHARED: - ret = (eHave!=LSM_LOCK_UNLOCK); - break; - case LSM_LOCK_EXCL: - ret = (eHave==LSM_LOCK_EXCL); - break; - default: - assert( !"bad eOp value passed to lsmShmAssertLock()" ); - break; - } - - return ret; -} - -int lsmShmAssertWorker(lsm_db *db){ - return lsmShmAssertLock(db, LSM_LOCK_WORKER, LSM_LOCK_EXCL) && db->pWorker; -} - -/* -** This function does not contribute to library functionality, and is not -** included in release builds. It is intended to be called from within -** an interactive debugger. -** -** When called, this function prints a single line of human readable output -** to stdout describing the locks currently held by the connection. For -** example: -** -** (gdb) call print_db_locks(pDb) -** (shared on dms2) (exclusive on writer) -*/ -void print_db_locks(lsm_db *db){ - int iLock; - for(iLock=0; iLock<16; iLock++){ - int bOne = 0; - const char *azLock[] = {0, "shared", "exclusive"}; - const char *azName[] = { - 0, "dms1", "dms2", "writer", "worker", "checkpointer", - "reader0", "reader1", "reader2", "reader3", "reader4", "reader5" - }; - int eHave = shmLockType(db, iLock); - if( azLock[eHave] ){ - printf("%s(%s on %s)", (bOne?" ":""), azLock[eHave], azName[iLock]); - bOne = 1; - } - } - printf("\n"); -} -void print_all_db_locks(lsm_db *db){ - lsm_db *p; - for(p=db->pDatabase->pConn; p; p=p->pNext){ - printf("%s connection %p ", ((p==db)?"*":""), p); - print_db_locks(p); - } -} -#endif - -void lsmShmBarrier(lsm_db *db){ - lsmEnvShmBarrier(db->pEnv); -} - -int lsm_checkpoint(lsm_db *pDb, int *pnKB){ - int rc; /* Return code */ - u32 nWrite = 0; /* Number of pages checkpointed */ - - /* Attempt the checkpoint. If successful, nWrite is set to the number of - ** pages written between this and the previous checkpoint. */ - rc = lsmCheckpointWrite(pDb, &nWrite); - - /* If required, calculate the output variable (KB of data checkpointed). - ** Set it to zero if an error occured. */ - if( pnKB ){ - int nKB = 0; - if( rc==LSM_OK && nWrite ){ - nKB = (((i64)nWrite * lsmFsPageSize(pDb->pFS)) + 1023) / 1024; - } - *pnKB = nKB; - } - - return rc; -} diff --git a/ext/lsm1/lsm_sorted.c b/ext/lsm1/lsm_sorted.c deleted file mode 100644 index a72c8cafb..000000000 --- a/ext/lsm1/lsm_sorted.c +++ /dev/null @@ -1,6195 +0,0 @@ -/* -** 2011-08-14 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** PAGE FORMAT: -** -** The maximum page size is 65536 bytes. -** -** Since all records are equal to or larger than 2 bytes in size, and -** some space within the page is consumed by the page footer, there must -** be less than 2^15 records on each page. -** -** Each page ends with a footer that describes the pages contents. This -** footer serves as similar purpose to the page header in an SQLite database. -** A footer is used instead of a header because it makes it easier to -** populate a new page based on a sorted list of key/value pairs. -** -** The footer consists of the following values (starting at the end of -** the page and continuing backwards towards the start). All values are -** stored as unsigned big-endian integers. -** -** * Number of records on page (2 bytes). -** * Flags field (2 bytes). -** * Left-hand pointer value (8 bytes). -** * The starting offset of each record (2 bytes per record). -** -** Records may span pages. Unless it happens to be an exact fit, the part -** of the final record that starts on page X that does not fit on page X -** is stored at the start of page (X+1). This means there may be pages where -** (N==0). And on most pages the first record that starts on the page will -** not start at byte offset 0. For example: -** -** aaaaa bbbbb ccc
          cc eeeee fffff g
          gggg.... -** -** RECORD FORMAT: -** -** The first byte of the record is a flags byte. It is a combination -** of the following flags (defined in lsmInt.h): -** -** LSM_START_DELETE -** LSM_END_DELETE -** LSM_POINT_DELETE -** LSM_INSERT -** LSM_SEPARATOR -** LSM_SYSTEMKEY -** -** Immediately following the type byte is a pointer to the smallest key -** in the next file that is larger than the key in the current record. The -** pointer is encoded as a varint. When added to the 32-bit page number -** stored in the footer, it is the page number of the page that contains the -** smallest key in the next sorted file that is larger than this key. -** -** Next is the number of bytes in the key, encoded as a varint. -** -** If the LSM_INSERT flag is set, the number of bytes in the value, as -** a varint, is next. -** -** Finally, the blob of data containing the key, and for LSM_INSERT -** records, the value as well. -*/ - -#ifndef _LSM_INT_H -# include "lsmInt.h" -#endif - -#define LSM_LOG_STRUCTURE 0 -#define LSM_LOG_DATA 0 - -/* -** Macros to help decode record types. -*/ -#define rtTopic(eType) ((eType) & LSM_SYSTEMKEY) -#define rtIsDelete(eType) (((eType) & 0x0F)==LSM_POINT_DELETE) - -#define rtIsSeparator(eType) (((eType) & LSM_SEPARATOR)!=0) -#define rtIsWrite(eType) (((eType) & LSM_INSERT)!=0) -#define rtIsSystem(eType) (((eType) & LSM_SYSTEMKEY)!=0) - -/* -** The following macros are used to access a page footer. -*/ -#define SEGMENT_NRECORD_OFFSET(pgsz) ((pgsz) - 2) -#define SEGMENT_FLAGS_OFFSET(pgsz) ((pgsz) - 2 - 2) -#define SEGMENT_POINTER_OFFSET(pgsz) ((pgsz) - 2 - 2 - 8) -#define SEGMENT_CELLPTR_OFFSET(pgsz, iCell) ((pgsz) - 2 - 2 - 8 - 2 - (iCell)*2) - -#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry-1) - -#define SEGMENT_BTREE_FLAG 0x0001 -#define PGFTR_SKIP_NEXT_FLAG 0x0002 -#define PGFTR_SKIP_THIS_FLAG 0x0004 - - -#ifndef LSM_SEGMENTPTR_FREE_THRESHOLD -# define LSM_SEGMENTPTR_FREE_THRESHOLD 1024 -#endif - -typedef struct SegmentPtr SegmentPtr; -typedef struct LsmBlob LsmBlob; - -struct LsmBlob { - lsm_env *pEnv; - void *pData; - int nData; - int nAlloc; -}; - -/* -** A SegmentPtr object may be used for one of two purposes: -** -** * To iterate and/or seek within a single Segment (the combination of a -** main run and an optional sorted run). -** -** * To iterate through the separators array of a segment. -*/ -struct SegmentPtr { - Level *pLevel; /* Level object segment is part of */ - Segment *pSeg; /* Segment to access */ - - /* Current page. See segmentPtrLoadPage(). */ - Page *pPg; /* Current page */ - u16 flags; /* Copy of page flags field */ - int nCell; /* Number of cells on pPg */ - LsmPgno iPtr; /* Base cascade pointer */ - - /* Current cell. See segmentPtrLoadCell() */ - int iCell; /* Current record within page pPg */ - int eType; /* Type of current record */ - LsmPgno iPgPtr; /* Cascade pointer offset */ - void *pKey; int nKey; /* Key associated with current record */ - void *pVal; int nVal; /* Current record value (eType==WRITE only) */ - - /* Blobs used to allocate buffers for pKey and pVal as required */ - LsmBlob blob1; - LsmBlob blob2; -}; - -/* -** Used to iterate through the keys stored in a b-tree hierarchy from start -** to finish. Only First() and Next() operations are required. -** -** btreeCursorNew() -** btreeCursorFirst() -** btreeCursorNext() -** btreeCursorFree() -** btreeCursorPosition() -** btreeCursorRestore() -*/ -typedef struct BtreePg BtreePg; -typedef struct BtreeCursor BtreeCursor; -struct BtreePg { - Page *pPage; - int iCell; -}; -struct BtreeCursor { - Segment *pSeg; /* Iterate through this segments btree */ - FileSystem *pFS; /* File system to read pages from */ - int nDepth; /* Allocated size of aPg[] */ - int iPg; /* Current entry in aPg[]. -1 -> EOF. */ - BtreePg *aPg; /* Pages from root to current location */ - - /* Cache of current entry. pKey==0 for EOF. */ - void *pKey; - int nKey; - int eType; - LsmPgno iPtr; - - /* Storage for key, if not local */ - LsmBlob blob; -}; - - -/* -** A cursor used for merged searches or iterations through up to one -** Tree structure and any number of sorted files. -** -** lsmMCursorNew() -** lsmMCursorSeek() -** lsmMCursorNext() -** lsmMCursorPrev() -** lsmMCursorFirst() -** lsmMCursorLast() -** lsmMCursorKey() -** lsmMCursorValue() -** lsmMCursorValid() -** -** iFree: -** This variable is only used by cursors providing input data for a -** new top-level segment. Such cursors only ever iterate forwards, not -** backwards. -*/ -struct MultiCursor { - lsm_db *pDb; /* Connection that owns this cursor */ - MultiCursor *pNext; /* Next cursor owned by connection pDb */ - int flags; /* Mask of CURSOR_XXX flags */ - - int eType; /* Cache of current key type */ - LsmBlob key; /* Cache of current key (or NULL) */ - LsmBlob val; /* Cache of current value */ - - /* All the component cursors: */ - TreeCursor *apTreeCsr[2]; /* Up to two tree cursors */ - int iFree; /* Next element of free-list (-ve for eof) */ - SegmentPtr *aPtr; /* Array of segment pointers */ - int nPtr; /* Size of array aPtr[] */ - BtreeCursor *pBtCsr; /* b-tree cursor (db writes only) */ - - /* Comparison results */ - int nTree; /* Size of aTree[] array */ - int *aTree; /* Array of comparison results */ - - /* Used by cursors flushing the in-memory tree only */ - void *pSystemVal; /* Pointer to buffer to free */ - - /* Used by worker cursors only */ - LsmPgno *pPrevMergePtr; -}; - -/* -** The following constants are used to assign integers to each component -** cursor of a multi-cursor. -*/ -#define CURSOR_DATA_TREE0 0 /* Current tree cursor (apTreeCsr[0]) */ -#define CURSOR_DATA_TREE1 1 /* The "old" tree, if any (apTreeCsr[1]) */ -#define CURSOR_DATA_SYSTEM 2 /* Free-list entries (new-toplevel only) */ -#define CURSOR_DATA_SEGMENT 3 /* First segment pointer (aPtr[0]) */ - -/* -** CURSOR_IGNORE_DELETE -** If set, this cursor will not visit SORTED_DELETE keys. -** -** CURSOR_FLUSH_FREELIST -** This cursor is being used to create a new toplevel. It should also -** iterate through the contents of the in-memory free block list. -** -** CURSOR_IGNORE_SYSTEM -** If set, this cursor ignores system keys. -** -** CURSOR_NEXT_OK -** Set if it is Ok to call lsm_csr_next(). -** -** CURSOR_PREV_OK -** Set if it is Ok to call lsm_csr_prev(). -** -** CURSOR_READ_SEPARATORS -** Set if this cursor should visit the separator keys in segment -** aPtr[nPtr-1]. -** -** CURSOR_SEEK_EQ -** Cursor has undergone a successful lsm_csr_seek(LSM_SEEK_EQ) operation. -** The key and value are stored in MultiCursor.key and MultiCursor.val -** respectively. -*/ -#define CURSOR_IGNORE_DELETE 0x00000001 -#define CURSOR_FLUSH_FREELIST 0x00000002 -#define CURSOR_IGNORE_SYSTEM 0x00000010 -#define CURSOR_NEXT_OK 0x00000020 -#define CURSOR_PREV_OK 0x00000040 -#define CURSOR_READ_SEPARATORS 0x00000080 -#define CURSOR_SEEK_EQ 0x00000100 - -typedef struct MergeWorker MergeWorker; -typedef struct Hierarchy Hierarchy; - -struct Hierarchy { - Page **apHier; - int nHier; -}; - -/* -** aSave: -** When mergeWorkerNextPage() is called to advance to the next page in -** the output segment, if the bStore flag for an element of aSave[] is -** true, it is cleared and the corresponding iPgno value is set to the -** page number of the page just completed. -** -** aSave[0] is used to record the pointer value to be pushed into the -** b-tree hierarchy. aSave[1] is used to save the page number of the -** page containing the indirect key most recently written to the b-tree. -** see mergeWorkerPushHierarchy() for details. -*/ -struct MergeWorker { - lsm_db *pDb; /* Database handle */ - Level *pLevel; /* Worker snapshot Level being merged */ - MultiCursor *pCsr; /* Cursor to read new segment contents from */ - int bFlush; /* True if this is an in-memory tree flush */ - Hierarchy hier; /* B-tree hierarchy under construction */ - Page *pPage; /* Current output page */ - int nWork; /* Number of calls to mergeWorkerNextPage() */ - LsmPgno *aGobble; /* Gobble point for each input segment */ - - LsmPgno iIndirect; - struct SavedPgno { - LsmPgno iPgno; - int bStore; - } aSave[2]; -}; - -#ifdef LSM_DEBUG_EXPENSIVE -static int assertPointersOk(lsm_db *, Segment *, Segment *, int); -static int assertBtreeOk(lsm_db *, Segment *); -static void assertRunInOrder(lsm_db *pDb, Segment *pSeg); -#else -#define assertRunInOrder(x,y) -#define assertBtreeOk(x,y) -#endif - - -struct FilePage { u8 *aData; int nData; }; -static u8 *fsPageData(Page *pPg, int *pnData){ - *pnData = ((struct FilePage *)(pPg))->nData; - return ((struct FilePage *)(pPg))->aData; -} -/*UNUSED static u8 *fsPageDataPtr(Page *pPg){ - return ((struct FilePage *)(pPg))->aData; -}*/ - -/* -** Write nVal as a 16-bit unsigned big-endian integer into buffer aOut. -*/ -void lsmPutU16(u8 *aOut, u16 nVal){ - aOut[0] = (u8)((nVal>>8) & 0xFF); - aOut[1] = (u8)(nVal & 0xFF); -} - -void lsmPutU32(u8 *aOut, u32 nVal){ - aOut[0] = (u8)((nVal>>24) & 0xFF); - aOut[1] = (u8)((nVal>>16) & 0xFF); - aOut[2] = (u8)((nVal>> 8) & 0xFF); - aOut[3] = (u8)((nVal ) & 0xFF); -} - -int lsmGetU16(u8 *aOut){ - return (aOut[0] << 8) + aOut[1]; -} - -u32 lsmGetU32(u8 *aOut){ - return ((u32)aOut[0] << 24) - + ((u32)aOut[1] << 16) - + ((u32)aOut[2] << 8) - + ((u32)aOut[3]); -} - -u64 lsmGetU64(u8 *aOut){ - return ((u64)aOut[0] << 56) - + ((u64)aOut[1] << 48) - + ((u64)aOut[2] << 40) - + ((u64)aOut[3] << 32) - + ((u64)aOut[4] << 24) - + ((u32)aOut[5] << 16) - + ((u32)aOut[6] << 8) - + ((u32)aOut[7]); -} - -void lsmPutU64(u8 *aOut, u64 nVal){ - aOut[0] = (u8)((nVal>>56) & 0xFF); - aOut[1] = (u8)((nVal>>48) & 0xFF); - aOut[2] = (u8)((nVal>>40) & 0xFF); - aOut[3] = (u8)((nVal>>32) & 0xFF); - aOut[4] = (u8)((nVal>>24) & 0xFF); - aOut[5] = (u8)((nVal>>16) & 0xFF); - aOut[6] = (u8)((nVal>> 8) & 0xFF); - aOut[7] = (u8)((nVal ) & 0xFF); -} - -static int sortedBlobGrow(lsm_env *pEnv, LsmBlob *pBlob, int nData){ - assert( pBlob->pEnv==pEnv || (pBlob->pEnv==0 && pBlob->pData==0) ); - if( pBlob->nAllocpData = lsmReallocOrFree(pEnv, pBlob->pData, nData); - if( !pBlob->pData ) return LSM_NOMEM_BKPT; - pBlob->nAlloc = nData; - pBlob->pEnv = pEnv; - } - return LSM_OK; -} - -static int sortedBlobSet(lsm_env *pEnv, LsmBlob *pBlob, void *pData, int nData){ - if( sortedBlobGrow(pEnv, pBlob, nData) ) return LSM_NOMEM; - memcpy(pBlob->pData, pData, nData); - pBlob->nData = nData; - return LSM_OK; -} - -#if 0 -static int sortedBlobCopy(LsmBlob *pDest, LsmBlob *pSrc){ - return sortedBlobSet(pDest, pSrc->pData, pSrc->nData); -} -#endif - -static void sortedBlobFree(LsmBlob *pBlob){ - assert( pBlob->pEnv || pBlob->pData==0 ); - if( pBlob->pData ) lsmFree(pBlob->pEnv, pBlob->pData); - memset(pBlob, 0, sizeof(LsmBlob)); -} - -static int sortedReadData( - Segment *pSeg, - Page *pPg, - int iOff, - int nByte, - void **ppData, - LsmBlob *pBlob -){ - int rc = LSM_OK; - int iEnd; - int nData; - int nCell; - u8 *aData; - - aData = fsPageData(pPg, &nData); - nCell = lsmGetU16(&aData[SEGMENT_NRECORD_OFFSET(nData)]); - iEnd = SEGMENT_EOF(nData, nCell); - assert( iEnd>0 && iEndnData = nByte; - aDest = (u8 *)pBlob->pData; - *ppData = pBlob->pData; - - /* Increment the pointer pages ref-count. */ - lsmFsPageRef(pPg); - - while( rc==LSM_OK ){ - Page *pNext; - int flags; - - /* Copy data from pPg into the output buffer. */ - int nCopy = LSM_MIN(nRem, iEnd-i); - if( nCopy>0 ){ - memcpy(&aDest[nByte-nRem], &aData[i], nCopy); - nRem -= nCopy; - i += nCopy; - assert( nRem==0 || i==iEnd ); - } - assert( nRem>=0 ); - if( nRem==0 ) break; - i -= iEnd; - - /* Grab the next page in the segment */ - - do { - rc = lsmFsDbPageNext(pSeg, pPg, 1, &pNext); - if( rc==LSM_OK && pNext==0 ){ - rc = LSM_CORRUPT_BKPT; - } - if( rc ) break; - lsmFsPageRelease(pPg); - pPg = pNext; - aData = fsPageData(pPg, &nData); - flags = lsmGetU16(&aData[SEGMENT_FLAGS_OFFSET(nData)]); - }while( flags&SEGMENT_BTREE_FLAG ); - - iEnd = SEGMENT_EOF(nData, lsmGetU16(&aData[nData-2])); - assert( iEnd>0 && iEnd=0 ); - aCell = pageGetCell(aData, nData, iCell); - lsmVarintGet64(&aCell[1], &iRet); - return iRet; -} - -static u8 *pageGetKey( - Segment *pSeg, /* Segment pPg belongs to */ - Page *pPg, /* Page to read from */ - int iCell, /* Index of cell on page to read */ - int *piTopic, /* OUT: Topic associated with this key */ - int *pnKey, /* OUT: Size of key in bytes */ - LsmBlob *pBlob /* If required, use this for dynamic memory */ -){ - u8 *pKey; - i64 nDummy; - int eType; - u8 *aData; - int nData; - - aData = fsPageData(pPg, &nData); - - assert( !(pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG) ); - assert( iCellpData || nKey==pBlob->nData ); - if( (void *)aKey!=pBlob->pData ){ - rc = sortedBlobSet(pEnv, pBlob, aKey, nKey); - } - - return rc; -} - -static LsmPgno pageGetBtreeRef(Page *pPg, int iKey){ - LsmPgno iRef; - u8 *aData; - int nData; - u8 *aCell; - - aData = fsPageData(pPg, &nData); - aCell = pageGetCell(aData, nData, iKey); - assert( aCell[0]==0 ); - aCell++; - aCell += lsmVarintGet64(aCell, &iRef); - lsmVarintGet64(aCell, &iRef); - assert( iRef>0 ); - return iRef; -} - -#define GETVARINT64(a, i) (((i)=((u8*)(a))[0])<=240?1:lsmVarintGet64((a), &(i))) -#define GETVARINT32(a, i) (((i)=((u8*)(a))[0])<=240?1:lsmVarintGet32((a), &(i))) - -static int pageGetBtreeKey( - Segment *pSeg, /* Segment page pPg belongs to */ - Page *pPg, - int iKey, - LsmPgno *piPtr, - int *piTopic, - void **ppKey, - int *pnKey, - LsmBlob *pBlob -){ - u8 *aData; - int nData; - u8 *aCell; - int eType; - - aData = fsPageData(pPg, &nData); - assert( SEGMENT_BTREE_FLAG & pageGetFlags(aData, nData) ); - assert( iKey>=0 && iKeypData; - *pnKey = pBlob->nData; - }else{ - aCell += GETVARINT32(aCell, *pnKey); - *ppKey = aCell; - } - if( piTopic ) *piTopic = rtTopic(eType); - - return LSM_OK; -} - -static int btreeCursorLoadKey(BtreeCursor *pCsr){ - int rc = LSM_OK; - if( pCsr->iPg<0 ){ - pCsr->pKey = 0; - pCsr->nKey = 0; - pCsr->eType = 0; - }else{ - LsmPgno dummy; - int iPg = pCsr->iPg; - int iCell = pCsr->aPg[iPg].iCell; - while( iCell<0 && (--iPg)>=0 ){ - iCell = pCsr->aPg[iPg].iCell-1; - } - if( iPg<0 || iCell<0 ) return LSM_CORRUPT_BKPT; - - rc = pageGetBtreeKey( - pCsr->pSeg, - pCsr->aPg[iPg].pPage, iCell, - &dummy, &pCsr->eType, &pCsr->pKey, &pCsr->nKey, &pCsr->blob - ); - pCsr->eType |= LSM_SEPARATOR; - } - - return rc; -} - -static LsmPgno btreeCursorPtr(u8 *aData, int nData, int iCell){ - int nCell; - - nCell = pageGetNRec(aData, nData); - if( iCell>=nCell ){ - return pageGetPtr(aData, nData); - } - return pageGetRecordPtr(aData, nData, iCell); -} - -static int btreeCursorNext(BtreeCursor *pCsr){ - int rc = LSM_OK; - - BtreePg *pPg = &pCsr->aPg[pCsr->iPg]; - int nCell; - u8 *aData; - int nData; - - assert( pCsr->iPg>=0 ); - assert( pCsr->iPg==pCsr->nDepth-1 ); - - aData = fsPageData(pPg->pPage, &nData); - nCell = pageGetNRec(aData, nData); - assert( pPg->iCell<=nCell ); - pPg->iCell++; - if( pPg->iCell==nCell ){ - LsmPgno iLoad; - - /* Up to parent. */ - lsmFsPageRelease(pPg->pPage); - pPg->pPage = 0; - pCsr->iPg--; - while( pCsr->iPg>=0 ){ - pPg = &pCsr->aPg[pCsr->iPg]; - aData = fsPageData(pPg->pPage, &nData); - if( pPg->iCellpPage); - pCsr->iPg--; - } - - /* Read the key */ - rc = btreeCursorLoadKey(pCsr); - - /* Unless the cursor is at EOF, descend to cell -1 (yes, negative one) of - ** the left-most most descendent. */ - if( pCsr->iPg>=0 ){ - pCsr->aPg[pCsr->iPg].iCell++; - - iLoad = btreeCursorPtr(aData, nData, pPg->iCell); - do { - Page *pLoad; - pCsr->iPg++; - rc = lsmFsDbPageGet(pCsr->pFS, pCsr->pSeg, iLoad, &pLoad); - pCsr->aPg[pCsr->iPg].pPage = pLoad; - pCsr->aPg[pCsr->iPg].iCell = 0; - if( rc==LSM_OK ){ - if( pCsr->iPg==(pCsr->nDepth-1) ) break; - aData = fsPageData(pLoad, &nData); - iLoad = btreeCursorPtr(aData, nData, 0); - } - }while( rc==LSM_OK && pCsr->iPg<(pCsr->nDepth-1) ); - pCsr->aPg[pCsr->iPg].iCell = -1; - } - - }else{ - rc = btreeCursorLoadKey(pCsr); - } - - if( rc==LSM_OK && pCsr->iPg>=0 ){ - aData = fsPageData(pCsr->aPg[pCsr->iPg].pPage, &nData); - pCsr->iPtr = btreeCursorPtr(aData, nData, pCsr->aPg[pCsr->iPg].iCell+1); - } - - return rc; -} - -static void btreeCursorFree(BtreeCursor *pCsr){ - if( pCsr ){ - int i; - lsm_env *pEnv = lsmFsEnv(pCsr->pFS); - for(i=0; i<=pCsr->iPg; i++){ - lsmFsPageRelease(pCsr->aPg[i].pPage); - } - sortedBlobFree(&pCsr->blob); - lsmFree(pEnv, pCsr->aPg); - lsmFree(pEnv, pCsr); - } -} - -static int btreeCursorFirst(BtreeCursor *pCsr){ - int rc; - - Page *pPg = 0; - FileSystem *pFS = pCsr->pFS; - LsmPgno iPg = pCsr->pSeg->iRoot; - - do { - rc = lsmFsDbPageGet(pFS, pCsr->pSeg, iPg, &pPg); - assert( (rc==LSM_OK)==(pPg!=0) ); - if( rc==LSM_OK ){ - u8 *aData; - int nData; - int flags; - - aData = fsPageData(pPg, &nData); - flags = pageGetFlags(aData, nData); - if( (flags & SEGMENT_BTREE_FLAG)==0 ) break; - - if( (pCsr->nDepth % 8)==0 ){ - int nNew = pCsr->nDepth + 8; - pCsr->aPg = (BtreePg *)lsmReallocOrFreeRc( - lsmFsEnv(pFS), pCsr->aPg, sizeof(BtreePg) * nNew, &rc - ); - if( rc==LSM_OK ){ - memset(&pCsr->aPg[pCsr->nDepth], 0, sizeof(BtreePg) * 8); - } - } - - if( rc==LSM_OK ){ - assert( pCsr->aPg[pCsr->nDepth].iCell==0 ); - pCsr->aPg[pCsr->nDepth].pPage = pPg; - pCsr->nDepth++; - iPg = pageGetRecordPtr(aData, nData, 0); - } - } - }while( rc==LSM_OK ); - lsmFsPageRelease(pPg); - pCsr->iPg = pCsr->nDepth-1; - - if( rc==LSM_OK && pCsr->nDepth ){ - pCsr->aPg[pCsr->iPg].iCell = -1; - rc = btreeCursorNext(pCsr); - } - - return rc; -} - -static void btreeCursorPosition(BtreeCursor *pCsr, MergeInput *p){ - if( pCsr->iPg>=0 ){ - p->iPg = lsmFsPageNumber(pCsr->aPg[pCsr->iPg].pPage); - p->iCell = ((pCsr->aPg[pCsr->iPg].iCell + 1) << 8) + pCsr->nDepth; - }else{ - p->iPg = 0; - p->iCell = 0; - } -} - -static void btreeCursorSplitkey(BtreeCursor *pCsr, MergeInput *p){ - int iCell = pCsr->aPg[pCsr->iPg].iCell; - if( iCell>=0 ){ - p->iCell = iCell; - p->iPg = lsmFsPageNumber(pCsr->aPg[pCsr->iPg].pPage); - }else{ - int i; - for(i=pCsr->iPg-1; i>=0; i--){ - if( pCsr->aPg[i].iCell>0 ) break; - } - assert( i>=0 ); - p->iCell = pCsr->aPg[i].iCell-1; - p->iPg = lsmFsPageNumber(pCsr->aPg[i].pPage); - } -} - -static int sortedKeyCompare( - int (*xCmp)(void *, int, void *, int), - int iLhsTopic, void *pLhsKey, int nLhsKey, - int iRhsTopic, void *pRhsKey, int nRhsKey -){ - int res = iLhsTopic - iRhsTopic; - if( res==0 ){ - res = xCmp(pLhsKey, nLhsKey, pRhsKey, nRhsKey); - } - return res; -} - -static int btreeCursorRestore( - BtreeCursor *pCsr, - int (*xCmp)(void *, int, void *, int), - MergeInput *p -){ - int rc = LSM_OK; - - if( p->iPg ){ - lsm_env *pEnv = lsmFsEnv(pCsr->pFS); - int iCell; /* Current cell number on leaf page */ - LsmPgno iLeaf; /* Page number of current leaf page */ - int nDepth; /* Depth of b-tree structure */ - Segment *pSeg = pCsr->pSeg; - - /* Decode the MergeInput structure */ - iLeaf = p->iPg; - nDepth = (p->iCell & 0x00FF); - iCell = (p->iCell >> 8) - 1; - - /* Allocate the BtreeCursor.aPg[] array */ - assert( pCsr->aPg==0 ); - pCsr->aPg = (BtreePg *)lsmMallocZeroRc(pEnv, sizeof(BtreePg) * nDepth, &rc); - - /* Populate the last entry of the aPg[] array */ - if( rc==LSM_OK ){ - Page **pp = &pCsr->aPg[nDepth-1].pPage; - pCsr->iPg = nDepth-1; - pCsr->nDepth = nDepth; - pCsr->aPg[pCsr->iPg].iCell = iCell; - rc = lsmFsDbPageGet(pCsr->pFS, pSeg, iLeaf, pp); - } - - /* Populate any other aPg[] array entries */ - if( rc==LSM_OK && nDepth>1 ){ - LsmBlob blob = {0,0,0}; - void *pSeek; - int nSeek; - int iTopicSeek; - int iPg = 0; - LsmPgno iLoad = pSeg->iRoot; - Page *pPg = pCsr->aPg[nDepth-1].pPage; - - if( pageObjGetNRec(pPg)==0 ){ - /* This can happen when pPg is the right-most leaf in the b-tree. - ** In this case, set the iTopicSeek/pSeek/nSeek key to a value - ** greater than any real key. */ - assert( iCell==-1 ); - iTopicSeek = 1000; - pSeek = 0; - nSeek = 0; - }else{ - LsmPgno dummy; - rc = pageGetBtreeKey(pSeg, pPg, - 0, &dummy, &iTopicSeek, &pSeek, &nSeek, &pCsr->blob - ); - } - - do { - Page *pPg2; - rc = lsmFsDbPageGet(pCsr->pFS, pSeg, iLoad, &pPg2); - assert( rc==LSM_OK || pPg2==0 ); - if( rc==LSM_OK ){ - u8 *aData; /* Buffer containing page data */ - int nData; /* Size of aData[] in bytes */ - int iMin; - int iMax; - int iCell2; - - aData = fsPageData(pPg2, &nData); - assert( (pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG) ); - - iLoad = pageGetPtr(aData, nData); - iCell2 = pageGetNRec(aData, nData); - iMax = iCell2-1; - iMin = 0; - - while( iMax>=iMin ){ - int iTry = (iMin+iMax)/2; - void *pKey; int nKey; /* Key for cell iTry */ - int iTopic; /* Topic for key pKeyT/nKeyT */ - LsmPgno iPtr; /* Pointer for cell iTry */ - int res; /* (pSeek - pKeyT) */ - - rc = pageGetBtreeKey( - pSeg, pPg2, iTry, &iPtr, &iTopic, &pKey, &nKey, &blob - ); - if( rc!=LSM_OK ) break; - - res = sortedKeyCompare( - xCmp, iTopicSeek, pSeek, nSeek, iTopic, pKey, nKey - ); - assert( res!=0 ); - - if( res<0 ){ - iLoad = iPtr; - iCell2 = iTry; - iMax = iTry-1; - }else{ - iMin = iTry+1; - } - } - - pCsr->aPg[iPg].pPage = pPg2; - pCsr->aPg[iPg].iCell = iCell2; - iPg++; - assert( iPg!=nDepth-1 - || lsmFsRedirectPage(pCsr->pFS, pSeg->pRedirect, iLoad)==iLeaf - ); - } - }while( rc==LSM_OK && iPg<(nDepth-1) ); - sortedBlobFree(&blob); - } - - /* Load the current key and pointer */ - if( rc==LSM_OK ){ - BtreePg *pBtreePg; - u8 *aData; - int nData; - - pBtreePg = &pCsr->aPg[pCsr->iPg]; - aData = fsPageData(pBtreePg->pPage, &nData); - pCsr->iPtr = btreeCursorPtr(aData, nData, pBtreePg->iCell+1); - if( pBtreePg->iCell<0 ){ - LsmPgno dummy; - int i; - for(i=pCsr->iPg-1; i>=0; i--){ - if( pCsr->aPg[i].iCell>0 ) break; - } - assert( i>=0 ); - rc = pageGetBtreeKey(pSeg, - pCsr->aPg[i].pPage, pCsr->aPg[i].iCell-1, - &dummy, &pCsr->eType, &pCsr->pKey, &pCsr->nKey, &pCsr->blob - ); - pCsr->eType |= LSM_SEPARATOR; - - }else{ - rc = btreeCursorLoadKey(pCsr); - } - } - } - return rc; -} - -static int btreeCursorNew( - lsm_db *pDb, - Segment *pSeg, - BtreeCursor **ppCsr -){ - int rc = LSM_OK; - BtreeCursor *pCsr; - - assert( pSeg->iRoot ); - pCsr = lsmMallocZeroRc(pDb->pEnv, sizeof(BtreeCursor), &rc); - if( pCsr ){ - pCsr->pFS = pDb->pFS; - pCsr->pSeg = pSeg; - pCsr->iPg = -1; - } - - *ppCsr = pCsr; - return rc; -} - -static void segmentPtrSetPage(SegmentPtr *pPtr, Page *pNext){ - lsmFsPageRelease(pPtr->pPg); - if( pNext ){ - int nData; - u8 *aData = fsPageData(pNext, &nData); - pPtr->nCell = pageGetNRec(aData, nData); - pPtr->flags = (u16)pageGetFlags(aData, nData); - pPtr->iPtr = pageGetPtr(aData, nData); - } - pPtr->pPg = pNext; -} - -/* -** Load a new page into the SegmentPtr object pPtr. -*/ -static int segmentPtrLoadPage( - FileSystem *pFS, - SegmentPtr *pPtr, /* Load page into this SegmentPtr object */ - LsmPgno iNew /* Page number of new page */ -){ - Page *pPg = 0; /* The new page */ - int rc; /* Return Code */ - - rc = lsmFsDbPageGet(pFS, pPtr->pSeg, iNew, &pPg); - assert( rc==LSM_OK || pPg==0 ); - segmentPtrSetPage(pPtr, pPg); - - return rc; -} - -static int segmentPtrReadData( - SegmentPtr *pPtr, - int iOff, - int nByte, - void **ppData, - LsmBlob *pBlob -){ - return sortedReadData(pPtr->pSeg, pPtr->pPg, iOff, nByte, ppData, pBlob); -} - -static int segmentPtrNextPage( - SegmentPtr *pPtr, /* Load page into this SegmentPtr object */ - int eDir /* +1 for next(), -1 for prev() */ -){ - Page *pNext; /* New page to load */ - int rc; /* Return code */ - - assert( eDir==1 || eDir==-1 ); - assert( pPtr->pPg ); - assert( pPtr->pSeg || eDir>0 ); - - rc = lsmFsDbPageNext(pPtr->pSeg, pPtr->pPg, eDir, &pNext); - assert( rc==LSM_OK || pNext==0 ); - segmentPtrSetPage(pPtr, pNext); - return rc; -} - -static int segmentPtrLoadCell( - SegmentPtr *pPtr, /* Load page into this SegmentPtr object */ - int iNew /* Cell number of new cell */ -){ - int rc = LSM_OK; - if( pPtr->pPg ){ - u8 *aData; /* Pointer to page data buffer */ - int iOff; /* Offset in aData[] to read from */ - int nPgsz; /* Size of page (aData[]) in bytes */ - - assert( iNewnCell ); - pPtr->iCell = iNew; - aData = fsPageData(pPtr->pPg, &nPgsz); - iOff = lsmGetU16(&aData[SEGMENT_CELLPTR_OFFSET(nPgsz, pPtr->iCell)]); - pPtr->eType = aData[iOff]; - iOff++; - iOff += GETVARINT64(&aData[iOff], pPtr->iPgPtr); - iOff += GETVARINT32(&aData[iOff], pPtr->nKey); - if( rtIsWrite(pPtr->eType) ){ - iOff += GETVARINT32(&aData[iOff], pPtr->nVal); - } - assert( pPtr->nKey>=0 ); - - rc = segmentPtrReadData( - pPtr, iOff, pPtr->nKey, &pPtr->pKey, &pPtr->blob1 - ); - if( rc==LSM_OK && rtIsWrite(pPtr->eType) ){ - rc = segmentPtrReadData( - pPtr, iOff+pPtr->nKey, pPtr->nVal, &pPtr->pVal, &pPtr->blob2 - ); - }else{ - pPtr->nVal = 0; - pPtr->pVal = 0; - } - } - - return rc; -} - - -static Segment *sortedSplitkeySegment(Level *pLevel){ - Merge *pMerge = pLevel->pMerge; - MergeInput *p = &pMerge->splitkey; - Segment *pSeg; - int i; - - for(i=0; inInput; i++){ - if( p->iPg==pMerge->aInput[i].iPg ) break; - } - if( pMerge->nInput==(pLevel->nRight+1) && i>=(pMerge->nInput-1) ){ - pSeg = &pLevel->pNext->lhs; - }else{ - pSeg = &pLevel->aRhs[i]; - } - - return pSeg; -} - -static void sortedSplitkey(lsm_db *pDb, Level *pLevel, int *pRc){ - Segment *pSeg; - Page *pPg = 0; - lsm_env *pEnv = pDb->pEnv; /* Environment handle */ - int rc = *pRc; - Merge *pMerge = pLevel->pMerge; - - pSeg = sortedSplitkeySegment(pLevel); - if( rc==LSM_OK ){ - rc = lsmFsDbPageGet(pDb->pFS, pSeg, pMerge->splitkey.iPg, &pPg); - } - if( rc==LSM_OK ){ - int iTopic; - LsmBlob blob = {0, 0, 0, 0}; - u8 *aData; - int nData; - - aData = lsmFsPageData(pPg, &nData); - if( pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG ){ - void *pKey; - int nKey; - LsmPgno dummy; - rc = pageGetBtreeKey(pSeg, - pPg, pMerge->splitkey.iCell, &dummy, &iTopic, &pKey, &nKey, &blob - ); - if( rc==LSM_OK && blob.pData!=pKey ){ - rc = sortedBlobSet(pEnv, &blob, pKey, nKey); - } - }else{ - rc = pageGetKeyCopy( - pEnv, pSeg, pPg, pMerge->splitkey.iCell, &iTopic, &blob - ); - } - - pLevel->iSplitTopic = iTopic; - pLevel->pSplitKey = blob.pData; - pLevel->nSplitKey = blob.nData; - lsmFsPageRelease(pPg); - } - - *pRc = rc; -} - -/* -** Reset a segment cursor. Also free its buffers if they are nThreshold -** bytes or larger in size. -*/ -static void segmentPtrReset(SegmentPtr *pPtr, int nThreshold){ - lsmFsPageRelease(pPtr->pPg); - pPtr->pPg = 0; - pPtr->nCell = 0; - pPtr->pKey = 0; - pPtr->nKey = 0; - pPtr->pVal = 0; - pPtr->nVal = 0; - pPtr->eType = 0; - pPtr->iCell = 0; - if( pPtr->blob1.nAlloc>=nThreshold ) sortedBlobFree(&pPtr->blob1); - if( pPtr->blob2.nAlloc>=nThreshold ) sortedBlobFree(&pPtr->blob2); -} - -static int segmentPtrIgnoreSeparators(MultiCursor *pCsr, SegmentPtr *pPtr){ - return (pCsr->flags & CURSOR_READ_SEPARATORS)==0 - || (pPtr!=&pCsr->aPtr[pCsr->nPtr-1]); -} - -static int segmentPtrAdvance( - MultiCursor *pCsr, - SegmentPtr *pPtr, - int bReverse -){ - int eDir = (bReverse ? -1 : 1); - Level *pLvl = pPtr->pLevel; - do { - int rc; - int iCell; /* Number of new cell in page */ - int svFlags = 0; /* SegmentPtr.eType before advance */ - - iCell = pPtr->iCell + eDir; - assert( pPtr->pPg ); - assert( iCell<=pPtr->nCell && iCell>=-1 ); - - if( bReverse && pPtr->pSeg!=&pPtr->pLevel->lhs ){ - svFlags = pPtr->eType; - assert( svFlags ); - } - - if( iCell>=pPtr->nCell || iCell<0 ){ - do { - rc = segmentPtrNextPage(pPtr, eDir); - }while( rc==LSM_OK - && pPtr->pPg - && (pPtr->nCell==0 || (pPtr->flags & SEGMENT_BTREE_FLAG) ) - ); - if( rc!=LSM_OK ) return rc; - iCell = bReverse ? (pPtr->nCell-1) : 0; - } - rc = segmentPtrLoadCell(pPtr, iCell); - if( rc!=LSM_OK ) return rc; - - if( svFlags && pPtr->pPg ){ - int res = sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, - pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey - ); - if( res<0 ) segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); - } - - if( pPtr->pPg==0 && (svFlags & LSM_END_DELETE) ){ - Segment *pSeg = pPtr->pSeg; - rc = lsmFsDbPageGet(pCsr->pDb->pFS, pSeg, pSeg->iFirst, &pPtr->pPg); - if( rc!=LSM_OK ) return rc; - pPtr->eType = LSM_START_DELETE | LSM_POINT_DELETE; - pPtr->eType |= (pLvl->iSplitTopic ? LSM_SYSTEMKEY : 0); - pPtr->pKey = pLvl->pSplitKey; - pPtr->nKey = pLvl->nSplitKey; - } - - }while( pCsr - && pPtr->pPg - && segmentPtrIgnoreSeparators(pCsr, pPtr) - && rtIsSeparator(pPtr->eType) - ); - - return LSM_OK; -} - -static void segmentPtrEndPage( - FileSystem *pFS, - SegmentPtr *pPtr, - int bLast, - int *pRc -){ - if( *pRc==LSM_OK ){ - Segment *pSeg = pPtr->pSeg; - Page *pNew = 0; - if( bLast ){ - *pRc = lsmFsDbPageLast(pFS, pSeg, &pNew); - }else{ - *pRc = lsmFsDbPageGet(pFS, pSeg, pSeg->iFirst, &pNew); - } - segmentPtrSetPage(pPtr, pNew); - } -} - - -/* -** Try to move the segment pointer passed as the second argument so that it -** points at either the first (bLast==0) or last (bLast==1) cell in the valid -** region of the segment defined by pPtr->iFirst and pPtr->iLast. -** -** Return LSM_OK if successful or an lsm error code if something goes -** wrong (IO error, OOM etc.). -*/ -static int segmentPtrEnd(MultiCursor *pCsr, SegmentPtr *pPtr, int bLast){ - Level *pLvl = pPtr->pLevel; - int rc = LSM_OK; - FileSystem *pFS = pCsr->pDb->pFS; - int bIgnore; - - segmentPtrEndPage(pFS, pPtr, bLast, &rc); - while( rc==LSM_OK && pPtr->pPg - && (pPtr->nCell==0 || (pPtr->flags & SEGMENT_BTREE_FLAG)) - ){ - rc = segmentPtrNextPage(pPtr, (bLast ? -1 : 1)); - } - - if( rc==LSM_OK && pPtr->pPg ){ - rc = segmentPtrLoadCell(pPtr, bLast ? (pPtr->nCell-1) : 0); - if( rc==LSM_OK && bLast && pPtr->pSeg!=&pLvl->lhs ){ - int res = sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, - pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey - ); - if( res<0 ) segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); - } - } - - bIgnore = segmentPtrIgnoreSeparators(pCsr, pPtr); - if( rc==LSM_OK && pPtr->pPg && bIgnore && rtIsSeparator(pPtr->eType) ){ - rc = segmentPtrAdvance(pCsr, pPtr, bLast); - } - -#if 0 - if( bLast && rc==LSM_OK && pPtr->pPg - && pPtr->pSeg==&pLvl->lhs - && pLvl->nRight && (pPtr->eType & LSM_START_DELETE) - ){ - pPtr->iCell++; - pPtr->eType = LSM_END_DELETE | (pLvl->iSplitTopic); - pPtr->pKey = pLvl->pSplitKey; - pPtr->nKey = pLvl->nSplitKey; - pPtr->pVal = 0; - pPtr->nVal = 0; - } -#endif - - return rc; -} - -static void segmentPtrKey(SegmentPtr *pPtr, void **ppKey, int *pnKey){ - assert( pPtr->pPg ); - *ppKey = pPtr->pKey; - *pnKey = pPtr->nKey; -} - -#if 0 /* NOT USED */ -static char *keyToString(lsm_env *pEnv, void *pKey, int nKey){ - int i; - u8 *aKey = (u8 *)pKey; - char *zRet = (char *)lsmMalloc(pEnv, nKey+1); - - for(i=0; ipDb->pFS); - LsmBlob blob = {0, 0, 0}; - int eDir; - int iTopic = 0; /* TODO: Fix me */ - - for(eDir=-1; eDir<=1; eDir+=2){ - Page *pTest = pPtr->pPg; - - lsmFsPageRef(pTest); - while( pTest ){ - Segment *pSeg = pPtr->pSeg; - Page *pNext; - - int rc = lsmFsDbPageNext(pSeg, pTest, eDir, &pNext); - lsmFsPageRelease(pTest); - if( rc ) return 1; - pTest = pNext; - - if( pTest ){ - int nData; - u8 *aData = fsPageData(pTest, &nData); - int nCell = pageGetNRec(aData, nData); - int flags = pageGetFlags(aData, nData); - if( nCell && 0==(flags&SEGMENT_BTREE_FLAG) ){ - int nPgKey; - int iPgTopic; - u8 *pPgKey; - int res; - int iCell; - - iCell = ((eDir < 0) ? (nCell-1) : 0); - pPgKey = pageGetKey(pSeg, pTest, iCell, &iPgTopic, &nPgKey, &blob); - res = iTopic - iPgTopic; - if( res==0 ) res = pCsr->pDb->xCmp(pKey, nKey, pPgKey, nPgKey); - if( (eDir==1 && res>0) || (eDir==-1 && res<0) ){ - /* Taking this branch means something has gone wrong. */ - char *zMsg = lsmMallocPrintf(pEnv, "Key \"%s\" is not on page %d", - keyToString(pEnv, pKey, nKey), lsmFsPageNumber(pPtr->pPg) - ); - fprintf(stderr, "%s\n", zMsg); - assert( !"assertKeyLocation() failed" ); - } - lsmFsPageRelease(pTest); - pTest = 0; - } - } - } - } - - sortedBlobFree(&blob); - return 1; -} -#endif - -#ifndef NDEBUG -static int assertSeekResult( - MultiCursor *pCsr, - SegmentPtr *pPtr, - int iTopic, - void *pKey, - int nKey, - int eSeek -){ - if( pPtr->pPg ){ - int res; - res = sortedKeyCompare(pCsr->pDb->xCmp, iTopic, pKey, nKey, - rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey - ); - - if( eSeek==LSM_SEEK_EQ ) return (res==0); - if( eSeek==LSM_SEEK_LE ) return (res>=0); - if( eSeek==LSM_SEEK_GE ) return (res<=0); - } - - return 1; -} -#endif - -static int segmentPtrSearchOversized( - MultiCursor *pCsr, /* Cursor context */ - SegmentPtr *pPtr, /* Pointer to seek */ - int iTopic, /* Topic of key to search for */ - void *pKey, int nKey /* Key to seek to */ -){ - int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; - int rc = LSM_OK; - - /* If the OVERSIZED flag is set, then there is no pointer in the - ** upper level to the next page in the segment that contains at least - ** one key. So compare the largest key on the current page with the - ** key being sought (pKey/nKey). If (pKey/nKey) is larger, advance - ** to the next page in the segment that contains at least one key. - */ - while( rc==LSM_OK && (pPtr->flags & PGFTR_SKIP_NEXT_FLAG) ){ - u8 *pLastKey; - int nLastKey; - int iLastTopic; - int res; /* Result of comparison */ - Page *pNext; - - /* Load the last key on the current page. */ - pLastKey = pageGetKey(pPtr->pSeg, - pPtr->pPg, pPtr->nCell-1, &iLastTopic, &nLastKey, &pPtr->blob1 - ); - - /* If the loaded key is >= than (pKey/nKey), break out of the loop. - ** If (pKey/nKey) is present in this array, it must be on the current - ** page. */ - res = sortedKeyCompare( - xCmp, iLastTopic, pLastKey, nLastKey, iTopic, pKey, nKey - ); - if( res>=0 ) break; - - /* Advance to the next page that contains at least one key. */ - pNext = pPtr->pPg; - lsmFsPageRef(pNext); - while( 1 ){ - Page *pLoad; - u8 *aData; int nData; - - rc = lsmFsDbPageNext(pPtr->pSeg, pNext, 1, &pLoad); - lsmFsPageRelease(pNext); - pNext = pLoad; - if( pNext==0 ) break; - - assert( rc==LSM_OK ); - aData = lsmFsPageData(pNext, &nData); - if( (pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG)==0 - && pageGetNRec(aData, nData)>0 - ){ - break; - } - } - if( pNext==0 ) break; - segmentPtrSetPage(pPtr, pNext); - - /* This should probably be an LSM_CORRUPT error. */ - assert( rc!=LSM_OK || (pPtr->flags & PGFTR_SKIP_THIS_FLAG) ); - } - - return rc; -} - -static int ptrFwdPointer( - Page *pPage, - int iCell, - Segment *pSeg, - LsmPgno *piPtr, - int *pbFound -){ - Page *pPg = pPage; - int iFirst = iCell; - int rc = LSM_OK; - - do { - Page *pNext = 0; - u8 *aData; - int nData; - - aData = lsmFsPageData(pPg, &nData); - if( (pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG)==0 ){ - int i; - int nCell = pageGetNRec(aData, nData); - for(i=iFirst; ipPg && rc==LSM_OK ){ - int res = sortedKeyCompare(pCsr->pDb->xCmp, - pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey, - rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey - ); - if( res<=0 ) break; - rc = segmentPtrAdvance(pCsr, pPtr, 0); - } - return rc; -} - - -/* -** This function is called as part of a SEEK_GE op on a multi-cursor if the -** FC pointer read from segment *pPtr comes from an entry with the -** LSM_START_DELETE flag set. In this case the pointer value cannot be -** trusted. Instead, the pointer that should be followed is that associated -** with the next entry in *pPtr that does not have LSM_START_DELETE set. -** -** Why the pointers can't be trusted: -** -** -** -** TODO: This is a stop-gap solution: -** -** At the moment, this function is called from within segmentPtrSeek(), -** as part of the initial lsmMCursorSeek() call. However, consider a -** database where the following has occurred: -** -** 1. A range delete removes keys 1..9999 using a range delete. -** 2. Keys 1 through 9999 are reinserted. -** 3. The levels containing the ops in 1. and 2. above are merged. Call -** this level N. Level N contains FC pointers to level N+1. -** -** Then, if the user attempts to query for (key>=2 LIMIT 10), the -** lsmMCursorSeek() call will iterate through 9998 entries searching for a -** pointer down to the level N+1 that is never actually used. It would be -** much better if the multi-cursor could do this lazily - only seek to the -** level (N+1) page after the user has moved the cursor on level N passed -** the big range-delete. -*/ -static int segmentPtrFwdPointer( - MultiCursor *pCsr, /* Multi-cursor pPtr belongs to */ - SegmentPtr *pPtr, /* Segment-pointer to extract FC ptr from */ - LsmPgno *piPtr /* OUT: FC pointer value */ -){ - Level *pLvl = pPtr->pLevel; - Level *pNext = pLvl->pNext; - Page *pPg = pPtr->pPg; - int rc; - int bFound; - LsmPgno iOut = 0; - - if( pPtr->pSeg==&pLvl->lhs || pPtr->pSeg==&pLvl->aRhs[pLvl->nRight-1] ){ - if( pNext==0 - || (pNext->nRight==0 && pNext->lhs.iRoot) - || (pNext->nRight!=0 && pNext->aRhs[0].iRoot) - ){ - /* Do nothing. The pointer will not be used anyway. */ - return LSM_OK; - } - }else{ - if( pPtr[1].pSeg->iRoot ){ - return LSM_OK; - } - } - - /* Search for a pointer within the current segment. */ - lsmFsPageRef(pPg); - rc = ptrFwdPointer(pPg, pPtr->iCell, pPtr->pSeg, &iOut, &bFound); - - if( rc==LSM_OK && bFound==0 ){ - /* This case happens when pPtr points to the left-hand-side of a segment - ** currently undergoing an incremental merge. In this case, jump to the - ** oldest segment in the right-hand-side of the same level and continue - ** searching. But - do not consider any keys smaller than the levels - ** split-key. */ - SegmentPtr ptr; - - if( pPtr->pLevel->nRight==0 || pPtr->pSeg!=&pPtr->pLevel->lhs ){ - return LSM_CORRUPT_BKPT; - } - - memset(&ptr, 0, sizeof(SegmentPtr)); - ptr.pLevel = pPtr->pLevel; - ptr.pSeg = &ptr.pLevel->aRhs[ptr.pLevel->nRight-1]; - rc = sortedRhsFirst(pCsr, ptr.pLevel, &ptr); - if( rc==LSM_OK ){ - rc = ptrFwdPointer(ptr.pPg, ptr.iCell, ptr.pSeg, &iOut, &bFound); - ptr.pPg = 0; - } - segmentPtrReset(&ptr, 0); - } - - *piPtr = iOut; - return rc; -} - -static int segmentPtrSeek( - MultiCursor *pCsr, /* Cursor context */ - SegmentPtr *pPtr, /* Pointer to seek */ - int iTopic, /* Key topic to seek to */ - void *pKey, int nKey, /* Key to seek to */ - int eSeek, /* Search bias - see above */ - LsmPgno *piPtr, /* OUT: FC pointer */ - int *pbStop -){ - int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; - int res = 0; /* Result of comparison operation */ - int rc = LSM_OK; - int iMin; - int iMax; - LsmPgno iPtrOut = 0; - - /* If the current page contains an oversized entry, then there are no - ** pointers to one or more of the subsequent pages in the sorted run. - ** The following call ensures that the segment-ptr points to the correct - ** page in this case. */ - rc = segmentPtrSearchOversized(pCsr, pPtr, iTopic, pKey, nKey); - iPtrOut = pPtr->iPtr; - - /* Assert that this page is the right page of this segment for the key - ** that we are searching for. Do this by loading page (iPg-1) and testing - ** that pKey/nKey is greater than all keys on that page, and then by - ** loading (iPg+1) and testing that pKey/nKey is smaller than all - ** the keys it houses. - ** - ** TODO: With range-deletes in the tree, the test described above may fail. - */ -#if 0 - assert( assertKeyLocation(pCsr, pPtr, pKey, nKey) ); -#endif - - assert( pPtr->nCell>0 - || pPtr->pSeg->nSize==1 - || lsmFsDbPageIsLast(pPtr->pSeg, pPtr->pPg) - ); - if( pPtr->nCell==0 ){ - segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); - }else{ - iMin = 0; - iMax = pPtr->nCell-1; - - while( 1 ){ - int iTry = (iMin+iMax)/2; - void *pKeyT; int nKeyT; /* Key for cell iTry */ - int iTopicT; - - assert( iTryeType); - - res = sortedKeyCompare(xCmp, iTopicT, pKeyT, nKeyT, iTopic, pKey, nKey); - if( res<=0 ){ - iPtrOut = pPtr->iPtr + pPtr->iPgPtr; - } - - if( res==0 || iMin==iMax ){ - break; - }else if( res>0 ){ - iMax = LSM_MAX(iTry-1, iMin); - }else{ - iMin = iTry+1; - } - } - - if( rc==LSM_OK ){ - assert( res==0 || (iMin==iMax && iMin>=0 && iMinnCell) ); - if( res ){ - rc = segmentPtrLoadCell(pPtr, iMin); - } - assert( rc!=LSM_OK || res>0 || iPtrOut==(pPtr->iPtr + pPtr->iPgPtr) ); - - if( rc==LSM_OK ){ - switch( eSeek ){ - case LSM_SEEK_EQ: { - int eType = pPtr->eType; - if( (res<0 && (eType & LSM_START_DELETE)) - || (res>0 && (eType & LSM_END_DELETE)) - || (res==0 && (eType & LSM_POINT_DELETE)) - ){ - *pbStop = 1; - }else if( res==0 && (eType & LSM_INSERT) ){ - lsm_env *pEnv = pCsr->pDb->pEnv; - *pbStop = 1; - pCsr->eType = pPtr->eType; - rc = sortedBlobSet(pEnv, &pCsr->key, pPtr->pKey, pPtr->nKey); - if( rc==LSM_OK ){ - rc = sortedBlobSet(pEnv, &pCsr->val, pPtr->pVal, pPtr->nVal); - } - pCsr->flags |= CURSOR_SEEK_EQ; - } - segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); - break; - } - case LSM_SEEK_LE: - if( res>0 ) rc = segmentPtrAdvance(pCsr, pPtr, 1); - break; - case LSM_SEEK_GE: { - /* Figure out if we need to 'skip' the pointer forward or not */ - if( (res<=0 && (pPtr->eType & LSM_START_DELETE)) - || (res>0 && (pPtr->eType & LSM_END_DELETE)) - ){ - rc = segmentPtrFwdPointer(pCsr, pPtr, &iPtrOut); - } - if( res<0 && rc==LSM_OK ){ - rc = segmentPtrAdvance(pCsr, pPtr, 0); - } - break; - } - } - } - } - - /* If the cursor seek has found a separator key, and this cursor is - ** supposed to ignore separators keys, advance to the next entry. */ - if( rc==LSM_OK && pPtr->pPg - && segmentPtrIgnoreSeparators(pCsr, pPtr) - && rtIsSeparator(pPtr->eType) - ){ - assert( eSeek!=LSM_SEEK_EQ ); - rc = segmentPtrAdvance(pCsr, pPtr, eSeek==LSM_SEEK_LE); - } - } - - assert( rc!=LSM_OK || assertSeekResult(pCsr,pPtr,iTopic,pKey,nKey,eSeek) ); - *piPtr = iPtrOut; - return rc; -} - -static int seekInBtree( - MultiCursor *pCsr, /* Multi-cursor object */ - Segment *pSeg, /* Seek within this segment */ - int iTopic, - void *pKey, int nKey, /* Key to seek to */ - LsmPgno *aPg, /* OUT: Page numbers */ - Page **ppPg /* OUT: Leaf (sorted-run) page reference */ -){ - int i = 0; - int rc; - LsmPgno iPg; - Page *pPg = 0; - LsmBlob blob = {0, 0, 0}; - - iPg = pSeg->iRoot; - do { - LsmPgno *piFirst = 0; - if( aPg ){ - aPg[i++] = iPg; - piFirst = &aPg[i]; - } - - rc = lsmFsDbPageGet(pCsr->pDb->pFS, pSeg, iPg, &pPg); - assert( rc==LSM_OK || pPg==0 ); - if( rc==LSM_OK ){ - u8 *aData; /* Buffer containing page data */ - int nData; /* Size of aData[] in bytes */ - int iMin; - int iMax; - int nRec; - int flags; - - aData = fsPageData(pPg, &nData); - flags = pageGetFlags(aData, nData); - if( (flags & SEGMENT_BTREE_FLAG)==0 ) break; - - iPg = pageGetPtr(aData, nData); - nRec = pageGetNRec(aData, nData); - - iMin = 0; - iMax = nRec-1; - while( iMax>=iMin ){ - int iTry = (iMin+iMax)/2; - void *pKeyT; int nKeyT; /* Key for cell iTry */ - int iTopicT; /* Topic for key pKeyT/nKeyT */ - LsmPgno iPtr; /* Pointer associated with cell iTry */ - int res; /* (pKey - pKeyT) */ - - rc = pageGetBtreeKey( - pSeg, pPg, iTry, &iPtr, &iTopicT, &pKeyT, &nKeyT, &blob - ); - if( rc!=LSM_OK ) break; - if( piFirst && pKeyT==blob.pData ){ - *piFirst = pageGetBtreeRef(pPg, iTry); - piFirst = 0; - i++; - } - - res = sortedKeyCompare( - pCsr->pDb->xCmp, iTopic, pKey, nKey, iTopicT, pKeyT, nKeyT - ); - if( res<0 ){ - iPg = iPtr; - iMax = iTry-1; - }else{ - iMin = iTry+1; - } - } - lsmFsPageRelease(pPg); - pPg = 0; - } - }while( rc==LSM_OK ); - - sortedBlobFree(&blob); - assert( (rc==LSM_OK)==(pPg!=0) ); - if( ppPg ){ - *ppPg = pPg; - }else{ - lsmFsPageRelease(pPg); - } - return rc; -} - -static int seekInSegment( - MultiCursor *pCsr, - SegmentPtr *pPtr, - int iTopic, - void *pKey, int nKey, - LsmPgno iPg, /* Page to search */ - int eSeek, /* Search bias - see above */ - LsmPgno *piPtr, /* OUT: FC pointer */ - int *pbStop /* OUT: Stop search flag */ -){ - LsmPgno iPtr = iPg; - int rc = LSM_OK; - - if( pPtr->pSeg->iRoot ){ - Page *pPg; - assert( pPtr->pSeg->iRoot!=0 ); - rc = seekInBtree(pCsr, pPtr->pSeg, iTopic, pKey, nKey, 0, &pPg); - if( rc==LSM_OK ) segmentPtrSetPage(pPtr, pPg); - }else{ - if( iPtr==0 ){ - iPtr = pPtr->pSeg->iFirst; - } - if( rc==LSM_OK ){ - rc = segmentPtrLoadPage(pCsr->pDb->pFS, pPtr, iPtr); - } - } - - if( rc==LSM_OK ){ - rc = segmentPtrSeek(pCsr, pPtr, iTopic, pKey, nKey, eSeek, piPtr, pbStop); - } - return rc; -} - -/* -** Seek each segment pointer in the array of (pLvl->nRight+1) at aPtr[]. -** -** pbStop: -** This parameter is only significant if parameter eSeek is set to -** LSM_SEEK_EQ. In this case, it is set to true before returning if -** the seek operation is finished. This can happen in two ways: -** -** a) A key matching (pKey/nKey) is found, or -** b) A point-delete or range-delete deleting the key is found. -** -** In case (a), the multi-cursor CURSOR_SEEK_EQ flag is set and the pCsr->key -** and pCsr->val blobs populated before returning. -*/ -static int seekInLevel( - MultiCursor *pCsr, /* Sorted cursor object to seek */ - SegmentPtr *aPtr, /* Pointer to array of (nRhs+1) SPs */ - int eSeek, /* Search bias - see above */ - int iTopic, /* Key topic to search for */ - void *pKey, int nKey, /* Key to search for */ - LsmPgno *piPgno, /* IN/OUT: fraction cascade pointer (or 0) */ - int *pbStop /* OUT: See above */ -){ - Level *pLvl = aPtr[0].pLevel; /* Level to seek within */ - int rc = LSM_OK; /* Return code */ - LsmPgno iOut = 0; /* Pointer to return to caller */ - int res = -1; /* Result of xCmp(pKey, split) */ - int nRhs = pLvl->nRight; /* Number of right-hand-side segments */ - int bStop = 0; - - /* If this is a composite level (one currently undergoing an incremental - ** merge), figure out if the search key is larger or smaller than the - ** levels split-key. */ - if( nRhs ){ - res = sortedKeyCompare(pCsr->pDb->xCmp, iTopic, pKey, nKey, - pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey - ); - } - - /* If (res<0), then key pKey/nKey is smaller than the split-key (or this - ** is not a composite level and there is no split-key). Search the - ** left-hand-side of the level in this case. */ - if( res<0 ){ - int i; - LsmPgno iPtr = 0; - if( nRhs==0 ) iPtr = *piPgno; - - rc = seekInSegment( - pCsr, &aPtr[0], iTopic, pKey, nKey, iPtr, eSeek, &iOut, &bStop - ); - if( rc==LSM_OK && nRhs>0 && eSeek==LSM_SEEK_GE && aPtr[0].pPg==0 ){ - res = 0; - } - for(i=1; i<=nRhs; i++){ - segmentPtrReset(&aPtr[i], LSM_SEGMENTPTR_FREE_THRESHOLD); - } - } - - if( res>=0 ){ - int bHit = 0; /* True if at least one rhs is not EOF */ - LsmPgno iPtr = *piPgno; - int i; - segmentPtrReset(&aPtr[0], LSM_SEGMENTPTR_FREE_THRESHOLD); - for(i=1; rc==LSM_OK && i<=nRhs && bStop==0; i++){ - SegmentPtr *pPtr = &aPtr[i]; - iOut = 0; - rc = seekInSegment( - pCsr, pPtr, iTopic, pKey, nKey, iPtr, eSeek, &iOut, &bStop - ); - iPtr = iOut; - - /* If the segment-pointer has settled on a key that is smaller than - ** the splitkey, invalidate the segment-pointer. */ - if( pPtr->pPg ){ - res = sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, - pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey - ); - if( res<0 ){ - if( pPtr->eType & LSM_START_DELETE ){ - pPtr->eType &= ~LSM_INSERT; - pPtr->pKey = pLvl->pSplitKey; - pPtr->nKey = pLvl->nSplitKey; - pPtr->pVal = 0; - pPtr->nVal = 0; - }else{ - segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); - } - } - } - - if( aPtr[i].pKey ) bHit = 1; - } - - if( rc==LSM_OK && eSeek==LSM_SEEK_LE && bHit==0 ){ - rc = segmentPtrEnd(pCsr, &aPtr[0], 1); - } - } - - assert( eSeek==LSM_SEEK_EQ || bStop==0 ); - *piPgno = iOut; - *pbStop = bStop; - return rc; -} - -static void multiCursorGetKey( - MultiCursor *pCsr, - int iKey, - int *peType, /* OUT: Key type (SORTED_WRITE etc.) */ - void **ppKey, /* OUT: Pointer to buffer containing key */ - int *pnKey /* OUT: Size of *ppKey in bytes */ -){ - int nKey = 0; - void *pKey = 0; - int eType = 0; - - switch( iKey ){ - case CURSOR_DATA_TREE0: - case CURSOR_DATA_TREE1: { - TreeCursor *pTreeCsr = pCsr->apTreeCsr[iKey-CURSOR_DATA_TREE0]; - if( lsmTreeCursorValid(pTreeCsr) ){ - lsmTreeCursorKey(pTreeCsr, &eType, &pKey, &nKey); - } - break; - } - - case CURSOR_DATA_SYSTEM: { - Snapshot *pWorker = pCsr->pDb->pWorker; - if( pWorker && (pCsr->flags & CURSOR_FLUSH_FREELIST) ){ - int nEntry = pWorker->freelist.nEntry; - if( pCsr->iFree < (nEntry*2) ){ - FreelistEntry *aEntry = pWorker->freelist.aEntry; - int i = nEntry - 1 - (pCsr->iFree / 2); - u32 iKey2 = 0; - - if( (pCsr->iFree % 2) ){ - eType = LSM_END_DELETE|LSM_SYSTEMKEY; - iKey2 = aEntry[i].iBlk-1; - }else if( aEntry[i].iId>=0 ){ - eType = LSM_INSERT|LSM_SYSTEMKEY; - iKey2 = aEntry[i].iBlk; - - /* If the in-memory entry immediately before this one was a - ** DELETE, and the block number is one greater than the current - ** block number, mark this entry as an "end-delete-range". */ - if( i<(nEntry-1) && aEntry[i+1].iBlk==iKey2+1 && aEntry[i+1].iId<0 ){ - eType |= LSM_END_DELETE; - } - - }else{ - eType = LSM_START_DELETE|LSM_SYSTEMKEY; - iKey2 = aEntry[i].iBlk + 1; - } - - /* If the in-memory entry immediately after this one is a - ** DELETE, and the block number is one less than the current - ** key, mark this entry as an "start-delete-range". */ - if( i>0 && aEntry[i-1].iBlk==iKey2-1 && aEntry[i-1].iId<0 ){ - eType |= LSM_START_DELETE; - } - - pKey = pCsr->pSystemVal; - nKey = 4; - lsmPutU32(pKey, ~iKey2); - } - } - break; - } - - default: { - int iPtr = iKey - CURSOR_DATA_SEGMENT; - assert( iPtr>=0 ); - if( iPtr==pCsr->nPtr ){ - if( pCsr->pBtCsr ){ - pKey = pCsr->pBtCsr->pKey; - nKey = pCsr->pBtCsr->nKey; - eType = pCsr->pBtCsr->eType; - } - }else if( iPtrnPtr ){ - SegmentPtr *pPtr = &pCsr->aPtr[iPtr]; - if( pPtr->pPg ){ - pKey = pPtr->pKey; - nKey = pPtr->nKey; - eType = pPtr->eType; - } - } - break; - } - } - - if( peType ) *peType = eType; - if( pnKey ) *pnKey = nKey; - if( ppKey ) *ppKey = pKey; -} - -static int sortedDbKeyCompare( - MultiCursor *pCsr, - int iLhsFlags, void *pLhsKey, int nLhsKey, - int iRhsFlags, void *pRhsKey, int nRhsKey -){ - int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; - int res; - - /* Compare the keys, including the system flag. */ - res = sortedKeyCompare(xCmp, - rtTopic(iLhsFlags), pLhsKey, nLhsKey, - rtTopic(iRhsFlags), pRhsKey, nRhsKey - ); - - /* If a key has the LSM_START_DELETE flag set, but not the LSM_INSERT or - ** LSM_POINT_DELETE flags, it is considered a delta larger. This prevents - ** the beginning of an open-ended set from masking a database entry or - ** delete at a lower level. */ - if( res==0 && (pCsr->flags & CURSOR_IGNORE_DELETE) ){ - const int m = LSM_POINT_DELETE|LSM_INSERT|LSM_END_DELETE |LSM_START_DELETE; - int iDel1 = 0; - int iDel2 = 0; - - if( LSM_START_DELETE==(iLhsFlags & m) ) iDel1 = +1; - if( LSM_END_DELETE ==(iLhsFlags & m) ) iDel1 = -1; - if( LSM_START_DELETE==(iRhsFlags & m) ) iDel2 = +1; - if( LSM_END_DELETE ==(iRhsFlags & m) ) iDel2 = -1; - - res = (iDel1 - iDel2); - } - - return res; -} - -static void multiCursorDoCompare(MultiCursor *pCsr, int iOut, int bReverse){ - int i1; - int i2; - int iRes; - void *pKey1; int nKey1; int eType1; - void *pKey2; int nKey2; int eType2; - const int mul = (bReverse ? -1 : 1); - - assert( pCsr->aTree && iOutnTree ); - if( iOut>=(pCsr->nTree/2) ){ - i1 = (iOut - pCsr->nTree/2) * 2; - i2 = i1 + 1; - }else{ - i1 = pCsr->aTree[iOut*2]; - i2 = pCsr->aTree[iOut*2+1]; - } - - multiCursorGetKey(pCsr, i1, &eType1, &pKey1, &nKey1); - multiCursorGetKey(pCsr, i2, &eType2, &pKey2, &nKey2); - - if( pKey1==0 ){ - iRes = i2; - }else if( pKey2==0 ){ - iRes = i1; - }else{ - int res; - - /* Compare the keys */ - res = sortedDbKeyCompare(pCsr, - eType1, pKey1, nKey1, eType2, pKey2, nKey2 - ); - - res = res * mul; - if( res==0 ){ - /* The two keys are identical. Normally, this means that the key from - ** the newer run clobbers the old. However, if the newer key is a - ** separator key, or a range-delete-boundary only, do not allow it - ** to clobber an older entry. */ - int nc1 = (eType1 & (LSM_INSERT|LSM_POINT_DELETE))==0; - int nc2 = (eType2 & (LSM_INSERT|LSM_POINT_DELETE))==0; - iRes = (nc1 > nc2) ? i2 : i1; - }else if( res<0 ){ - iRes = i1; - }else{ - iRes = i2; - } - } - - pCsr->aTree[iOut] = iRes; -} - -/* -** This function advances segment pointer iPtr belonging to multi-cursor -** pCsr forward (bReverse==0) or backward (bReverse!=0). -** -** If the segment pointer points to a segment that is part of a composite -** level, then the following special case is handled. -** -** * If iPtr is the lhs of a composite level, and the cursor is being -** advanced forwards, and segment iPtr is at EOF, move all pointers -** that correspond to rhs segments of the same level to the first -** key in their respective data. -*/ -static int segmentCursorAdvance( - MultiCursor *pCsr, - int iPtr, - int bReverse -){ - int rc; - SegmentPtr *pPtr = &pCsr->aPtr[iPtr]; - Level *pLvl = pPtr->pLevel; - int bComposite; /* True if pPtr is part of composite level */ - - /* Advance the segment-pointer object. */ - rc = segmentPtrAdvance(pCsr, pPtr, bReverse); - if( rc!=LSM_OK ) return rc; - - bComposite = (pLvl->nRight>0 && pCsr->nPtr>pLvl->nRight); - if( bComposite && pPtr->pPg==0 ){ - int bFix = 0; - if( (bReverse==0)==(pPtr->pSeg==&pLvl->lhs) ){ - int i; - if( bReverse ){ - SegmentPtr *pLhs = &pCsr->aPtr[iPtr - 1 - (pPtr->pSeg - pLvl->aRhs)]; - for(i=0; inRight; i++){ - if( pLhs[i+1].pPg ) break; - } - if( i==pLvl->nRight ){ - bFix = 1; - rc = segmentPtrEnd(pCsr, pLhs, 1); - } - }else{ - bFix = 1; - for(i=0; rc==LSM_OK && inRight; i++){ - rc = sortedRhsFirst(pCsr, pLvl, &pCsr->aPtr[iPtr+1+i]); - } - } - } - - if( bFix ){ - int i; - for(i=pCsr->nTree-1; i>0; i--){ - multiCursorDoCompare(pCsr, i, bReverse); - } - } - } - -#if 0 - if( bComposite && pPtr->pSeg==&pLvl->lhs /* lhs of composite level */ - && bReverse==0 /* csr advanced forwards */ - && pPtr->pPg==0 /* segment at EOF */ - ){ - int i; - for(i=0; rc==LSM_OK && inRight; i++){ - rc = sortedRhsFirst(pCsr, pLvl, &pCsr->aPtr[iPtr+1+i]); - } - for(i=pCsr->nTree-1; i>0; i--){ - multiCursorDoCompare(pCsr, i, 0); - } - } -#endif - - return rc; -} - -static void mcursorFreeComponents(MultiCursor *pCsr){ - int i; - lsm_env *pEnv = pCsr->pDb->pEnv; - - /* Close the tree cursor, if any. */ - lsmTreeCursorDestroy(pCsr->apTreeCsr[0]); - lsmTreeCursorDestroy(pCsr->apTreeCsr[1]); - - /* Reset the segment pointers */ - for(i=0; inPtr; i++){ - segmentPtrReset(&pCsr->aPtr[i], 0); - } - - /* And the b-tree cursor, if any */ - btreeCursorFree(pCsr->pBtCsr); - - /* Free allocations */ - lsmFree(pEnv, pCsr->aPtr); - lsmFree(pEnv, pCsr->aTree); - lsmFree(pEnv, pCsr->pSystemVal); - - /* Zero fields */ - pCsr->nPtr = 0; - pCsr->aPtr = 0; - pCsr->nTree = 0; - pCsr->aTree = 0; - pCsr->pSystemVal = 0; - pCsr->apTreeCsr[0] = 0; - pCsr->apTreeCsr[1] = 0; - pCsr->pBtCsr = 0; -} - -void lsmMCursorFreeCache(lsm_db *pDb){ - MultiCursor *p; - MultiCursor *pNext; - for(p=pDb->pCsrCache; p; p=pNext){ - pNext = p->pNext; - lsmMCursorClose(p, 0); - } - pDb->pCsrCache = 0; -} - -/* -** Close the cursor passed as the first argument. -** -** If the bCache parameter is true, then shift the cursor to the pCsrCache -** list for possible reuse instead of actually deleting it. -*/ -void lsmMCursorClose(MultiCursor *pCsr, int bCache){ - if( pCsr ){ - lsm_db *pDb = pCsr->pDb; - MultiCursor **pp; /* Iterator variable */ - - /* The cursor may or may not be currently part of the linked list - ** starting at lsm_db.pCsr. If it is, extract it. */ - for(pp=&pDb->pCsr; *pp; pp=&((*pp)->pNext)){ - if( *pp==pCsr ){ - *pp = pCsr->pNext; - break; - } - } - - if( bCache ){ - int i; /* Used to iterate through segment-pointers */ - - /* Release any page references held by this cursor. */ - assert( !pCsr->pBtCsr ); - for(i=0; inPtr; i++){ - SegmentPtr *pPtr = &pCsr->aPtr[i]; - lsmFsPageRelease(pPtr->pPg); - pPtr->pPg = 0; - } - - /* Reset the tree cursors */ - lsmTreeCursorReset(pCsr->apTreeCsr[0]); - lsmTreeCursorReset(pCsr->apTreeCsr[1]); - - /* Add the cursor to the pCsrCache list */ - pCsr->pNext = pDb->pCsrCache; - pDb->pCsrCache = pCsr; - }else{ - /* Free the allocation used to cache the current key, if any. */ - sortedBlobFree(&pCsr->key); - sortedBlobFree(&pCsr->val); - - /* Free the component cursors */ - mcursorFreeComponents(pCsr); - - /* Free the cursor structure itself */ - lsmFree(pDb->pEnv, pCsr); - } - } -} - -#define TREE_NONE 0 -#define TREE_OLD 1 -#define TREE_BOTH 2 - -/* -** Parameter eTree is one of TREE_OLD or TREE_BOTH. -*/ -static int multiCursorAddTree(MultiCursor *pCsr, Snapshot *pSnap, int eTree){ - int rc = LSM_OK; - lsm_db *db = pCsr->pDb; - - /* Add a tree cursor on the 'old' tree, if it exists. */ - if( eTree!=TREE_NONE - && lsmTreeHasOld(db) - && db->treehdr.iOldLog!=pSnap->iLogOff - ){ - rc = lsmTreeCursorNew(db, 1, &pCsr->apTreeCsr[1]); - } - - /* Add a tree cursor on the 'current' tree, if required. */ - if( rc==LSM_OK && eTree==TREE_BOTH ){ - rc = lsmTreeCursorNew(db, 0, &pCsr->apTreeCsr[0]); - } - - return rc; -} - -static int multiCursorAddRhs(MultiCursor *pCsr, Level *pLvl){ - int i; - int nRhs = pLvl->nRight; - - assert( pLvl->nRight>0 ); - assert( pCsr->aPtr==0 ); - pCsr->aPtr = lsmMallocZero(pCsr->pDb->pEnv, sizeof(SegmentPtr) * nRhs); - if( !pCsr->aPtr ) return LSM_NOMEM_BKPT; - pCsr->nPtr = nRhs; - - for(i=0; iaPtr[i].pSeg = &pLvl->aRhs[i]; - pCsr->aPtr[i].pLevel = pLvl; - } - - return LSM_OK; -} - -static void multiCursorAddOne(MultiCursor *pCsr, Level *pLvl, int *pRc){ - if( *pRc==LSM_OK ){ - int iPtr = pCsr->nPtr; - int i; - pCsr->aPtr[iPtr].pLevel = pLvl; - pCsr->aPtr[iPtr].pSeg = &pLvl->lhs; - iPtr++; - for(i=0; inRight; i++){ - pCsr->aPtr[iPtr].pLevel = pLvl; - pCsr->aPtr[iPtr].pSeg = &pLvl->aRhs[i]; - iPtr++; - } - - if( pLvl->nRight && pLvl->pSplitKey==0 ){ - sortedSplitkey(pCsr->pDb, pLvl, pRc); - } - pCsr->nPtr = iPtr; - } -} - -static int multiCursorAddAll(MultiCursor *pCsr, Snapshot *pSnap){ - Level *pLvl; - int nPtr = 0; - int rc = LSM_OK; - - for(pLvl=pSnap->pLevel; pLvl; pLvl=pLvl->pNext){ - /* If the LEVEL_INCOMPLETE flag is set, then this function is being - ** called (indirectly) from within a sortedNewToplevel() call to - ** construct pLvl. In this case ignore pLvl - this cursor is going to - ** be used to retrieve a freelist entry from the LSM, and the partially - ** complete level may confuse it. */ - if( pLvl->flags & LEVEL_INCOMPLETE ) continue; - nPtr += (1 + pLvl->nRight); - } - - assert( pCsr->aPtr==0 ); - pCsr->aPtr = lsmMallocZeroRc(pCsr->pDb->pEnv, sizeof(SegmentPtr) * nPtr, &rc); - - for(pLvl=pSnap->pLevel; pLvl; pLvl=pLvl->pNext){ - if( (pLvl->flags & LEVEL_INCOMPLETE)==0 ){ - multiCursorAddOne(pCsr, pLvl, &rc); - } - } - - return rc; -} - -static int multiCursorInit(MultiCursor *pCsr, Snapshot *pSnap){ - int rc; - rc = multiCursorAddAll(pCsr, pSnap); - if( rc==LSM_OK ){ - rc = multiCursorAddTree(pCsr, pSnap, TREE_BOTH); - } - pCsr->flags |= (CURSOR_IGNORE_SYSTEM | CURSOR_IGNORE_DELETE); - return rc; -} - -static MultiCursor *multiCursorNew(lsm_db *db, int *pRc){ - MultiCursor *pCsr; - pCsr = (MultiCursor *)lsmMallocZeroRc(db->pEnv, sizeof(MultiCursor), pRc); - if( pCsr ){ - pCsr->pNext = db->pCsr; - db->pCsr = pCsr; - pCsr->pDb = db; - } - return pCsr; -} - - -void lsmSortedRemap(lsm_db *pDb){ - MultiCursor *pCsr; - for(pCsr=pDb->pCsr; pCsr; pCsr=pCsr->pNext){ - int iPtr; - if( pCsr->pBtCsr ){ - btreeCursorLoadKey(pCsr->pBtCsr); - } - for(iPtr=0; iPtrnPtr; iPtr++){ - segmentPtrLoadCell(&pCsr->aPtr[iPtr], pCsr->aPtr[iPtr].iCell); - } - } -} - -static void multiCursorReadSeparators(MultiCursor *pCsr){ - if( pCsr->nPtr>0 ){ - pCsr->flags |= CURSOR_READ_SEPARATORS; - } -} - -/* -** Have this cursor skip over SORTED_DELETE entries. -*/ -static void multiCursorIgnoreDelete(MultiCursor *pCsr){ - if( pCsr ) pCsr->flags |= CURSOR_IGNORE_DELETE; -} - -/* -** If the free-block list is not empty, then have this cursor visit a key -** with (a) the system bit set, and (b) the key "FREELIST" and (c) a value -** blob containing the serialized free-block list. -*/ -static int multiCursorVisitFreelist(MultiCursor *pCsr){ - int rc = LSM_OK; - pCsr->flags |= CURSOR_FLUSH_FREELIST; - pCsr->pSystemVal = lsmMallocRc(pCsr->pDb->pEnv, 4 + 8, &rc); - return rc; -} - -/* -** Allocate and return a new database cursor. -** -** This method should only be called to allocate user cursors. As it may -** recycle a cursor from lsm_db.pCsrCache. -*/ -int lsmMCursorNew( - lsm_db *pDb, /* Database handle */ - MultiCursor **ppCsr /* OUT: Allocated cursor */ -){ - MultiCursor *pCsr = 0; - int rc = LSM_OK; - - if( pDb->pCsrCache ){ - int bOld; /* True if there is an old in-memory tree */ - - /* Remove a cursor from the pCsrCache list and add it to the open list. */ - pCsr = pDb->pCsrCache; - pDb->pCsrCache = pCsr->pNext; - pCsr->pNext = pDb->pCsr; - pDb->pCsr = pCsr; - - /* The cursor can almost be used as is, except that the old in-memory - ** tree cursor may be present and not required, or required and not - ** present. Fix this if required. */ - bOld = (lsmTreeHasOld(pDb) && pDb->treehdr.iOldLog!=pDb->pClient->iLogOff); - if( !bOld && pCsr->apTreeCsr[1] ){ - lsmTreeCursorDestroy(pCsr->apTreeCsr[1]); - pCsr->apTreeCsr[1] = 0; - }else if( bOld && !pCsr->apTreeCsr[1] ){ - rc = lsmTreeCursorNew(pDb, 1, &pCsr->apTreeCsr[1]); - } - - pCsr->flags = (CURSOR_IGNORE_SYSTEM | CURSOR_IGNORE_DELETE); - - }else{ - pCsr = multiCursorNew(pDb, &rc); - if( rc==LSM_OK ) rc = multiCursorInit(pCsr, pDb->pClient); - } - - if( rc!=LSM_OK ){ - lsmMCursorClose(pCsr, 0); - pCsr = 0; - } - assert( (rc==LSM_OK)==(pCsr!=0) ); - *ppCsr = pCsr; - return rc; -} - -static int multiCursorGetVal( - MultiCursor *pCsr, - int iVal, - void **ppVal, - int *pnVal -){ - int rc = LSM_OK; - - *ppVal = 0; - *pnVal = 0; - - switch( iVal ){ - case CURSOR_DATA_TREE0: - case CURSOR_DATA_TREE1: { - TreeCursor *pTreeCsr = pCsr->apTreeCsr[iVal-CURSOR_DATA_TREE0]; - if( lsmTreeCursorValid(pTreeCsr) ){ - lsmTreeCursorValue(pTreeCsr, ppVal, pnVal); - }else{ - *ppVal = 0; - *pnVal = 0; - } - break; - } - - case CURSOR_DATA_SYSTEM: { - Snapshot *pWorker = pCsr->pDb->pWorker; - if( pWorker - && (pCsr->iFree % 2)==0 - && pCsr->iFree < (pWorker->freelist.nEntry*2) - ){ - int iEntry = pWorker->freelist.nEntry - 1 - (pCsr->iFree / 2); - u8 *aVal = &((u8 *)(pCsr->pSystemVal))[4]; - lsmPutU64(aVal, pWorker->freelist.aEntry[iEntry].iId); - *ppVal = aVal; - *pnVal = 8; - } - break; - } - - default: { - int iPtr = iVal-CURSOR_DATA_SEGMENT; - if( iPtrnPtr ){ - SegmentPtr *pPtr = &pCsr->aPtr[iPtr]; - if( pPtr->pPg ){ - *ppVal = pPtr->pVal; - *pnVal = pPtr->nVal; - } - } - } - } - - assert( rc==LSM_OK || (*ppVal==0 && *pnVal==0) ); - return rc; -} - -static int multiCursorAdvance(MultiCursor *pCsr, int bReverse); - -/* -** This function is called by worker connections to walk the part of the -** free-list stored within the LSM data structure. -*/ -int lsmSortedWalkFreelist( - lsm_db *pDb, /* Database handle */ - int bReverse, /* True to iterate from largest to smallest */ - int (*x)(void *, int, i64), /* Callback function */ - void *pCtx /* First argument to pass to callback */ -){ - MultiCursor *pCsr; /* Cursor used to read db */ - int rc = LSM_OK; /* Return Code */ - Snapshot *pSnap = 0; - - assert( pDb->pWorker ); - if( pDb->bIncrMerge ){ - rc = lsmCheckpointDeserialize(pDb, 0, pDb->pShmhdr->aSnap1, &pSnap); - if( rc!=LSM_OK ) return rc; - }else{ - pSnap = pDb->pWorker; - } - - pCsr = multiCursorNew(pDb, &rc); - if( pCsr ){ - rc = multiCursorAddAll(pCsr, pSnap); - pCsr->flags |= CURSOR_IGNORE_DELETE; - } - - if( rc==LSM_OK ){ - if( bReverse==0 ){ - rc = lsmMCursorLast(pCsr); - }else{ - rc = lsmMCursorSeek(pCsr, 1, "", 0, LSM_SEEK_GE); - } - - while( rc==LSM_OK && lsmMCursorValid(pCsr) && rtIsSystem(pCsr->eType) ){ - void *pKey; int nKey; - void *pVal = 0; int nVal = 0; - - rc = lsmMCursorKey(pCsr, &pKey, &nKey); - if( rc==LSM_OK ) rc = lsmMCursorValue(pCsr, &pVal, &nVal); - if( rc==LSM_OK && (nKey!=4 || nVal!=8) ) rc = LSM_CORRUPT_BKPT; - - if( rc==LSM_OK ){ - int iBlk; - i64 iSnap; - iBlk = (int)(~(lsmGetU32((u8 *)pKey))); - iSnap = (i64)lsmGetU64((u8 *)pVal); - if( x(pCtx, iBlk, iSnap) ) break; - rc = multiCursorAdvance(pCsr, !bReverse); - } - } - } - - lsmMCursorClose(pCsr, 0); - if( pSnap!=pDb->pWorker ){ - lsmFreeSnapshot(pDb->pEnv, pSnap); - } - - return rc; -} - -int lsmSortedLoadFreelist( - lsm_db *pDb, /* Database handle (must be worker) */ - void **ppVal, /* OUT: Blob containing LSM free-list */ - int *pnVal /* OUT: Size of *ppVal blob in bytes */ -){ - MultiCursor *pCsr; /* Cursor used to retrieve free-list */ - int rc = LSM_OK; /* Return Code */ - - assert( pDb->pWorker ); - assert( *ppVal==0 && *pnVal==0 ); - - pCsr = multiCursorNew(pDb, &rc); - if( pCsr ){ - rc = multiCursorAddAll(pCsr, pDb->pWorker); - pCsr->flags |= CURSOR_IGNORE_DELETE; - } - - if( rc==LSM_OK ){ - rc = lsmMCursorLast(pCsr); - if( rc==LSM_OK - && rtIsWrite(pCsr->eType) && rtIsSystem(pCsr->eType) - && pCsr->key.nData==8 - && 0==memcmp(pCsr->key.pData, "FREELIST", 8) - ){ - void *pVal; int nVal; /* Value read from database */ - rc = lsmMCursorValue(pCsr, &pVal, &nVal); - if( rc==LSM_OK ){ - *ppVal = lsmMallocRc(pDb->pEnv, nVal, &rc); - if( *ppVal ){ - memcpy(*ppVal, pVal, nVal); - *pnVal = nVal; - } - } - } - - lsmMCursorClose(pCsr, 0); - } - - return rc; -} - -static int multiCursorAllocTree(MultiCursor *pCsr){ - int rc = LSM_OK; - if( pCsr->aTree==0 ){ - int nByte; /* Bytes of space to allocate */ - int nMin; /* Total number of cursors being merged */ - - nMin = CURSOR_DATA_SEGMENT + pCsr->nPtr + (pCsr->pBtCsr!=0); - pCsr->nTree = 2; - while( pCsr->nTreenTree = pCsr->nTree*2; - } - - nByte = sizeof(int)*pCsr->nTree*2; - pCsr->aTree = (int *)lsmMallocZeroRc(pCsr->pDb->pEnv, nByte, &rc); - } - return rc; -} - -static void multiCursorCacheKey(MultiCursor *pCsr, int *pRc){ - if( *pRc==LSM_OK ){ - void *pKey; - int nKey; - multiCursorGetKey(pCsr, pCsr->aTree[1], &pCsr->eType, &pKey, &nKey); - *pRc = sortedBlobSet(pCsr->pDb->pEnv, &pCsr->key, pKey, nKey); - } -} - -#ifdef LSM_DEBUG_EXPENSIVE -static void assertCursorTree(MultiCursor *pCsr){ - int bRev = !!(pCsr->flags & CURSOR_PREV_OK); - int *aSave = pCsr->aTree; - int nSave = pCsr->nTree; - int rc; - - pCsr->aTree = 0; - pCsr->nTree = 0; - rc = multiCursorAllocTree(pCsr); - if( rc==LSM_OK ){ - int i; - for(i=pCsr->nTree-1; i>0; i--){ - multiCursorDoCompare(pCsr, i, bRev); - } - - assert( nSave==pCsr->nTree - && 0==memcmp(aSave, pCsr->aTree, sizeof(int)*nSave) - ); - - lsmFree(pCsr->pDb->pEnv, pCsr->aTree); - } - - pCsr->aTree = aSave; - pCsr->nTree = nSave; -} -#else -# define assertCursorTree(x) -#endif - -static int mcursorLocationOk(MultiCursor *pCsr, int bDeleteOk){ - int eType = pCsr->eType; - int iKey; - int i; - int rdmask; - - assert( pCsr->flags & (CURSOR_NEXT_OK|CURSOR_PREV_OK) ); - assertCursorTree(pCsr); - - rdmask = (pCsr->flags & CURSOR_NEXT_OK) ? LSM_END_DELETE : LSM_START_DELETE; - - /* If the cursor does not currently point to an actual database key (i.e. - ** it points to a delete key, or the start or end of a range-delete), and - ** the CURSOR_IGNORE_DELETE flag is set, skip past this entry. */ - if( (pCsr->flags & CURSOR_IGNORE_DELETE) && bDeleteOk==0 ){ - if( (eType & LSM_INSERT)==0 ) return 0; - } - - /* If the cursor points to a system key (free-list entry), and the - ** CURSOR_IGNORE_SYSTEM flag is set, skip this entry. */ - if( (pCsr->flags & CURSOR_IGNORE_SYSTEM) && rtTopic(eType)!=0 ){ - return 0; - } - -#ifndef NDEBUG - /* This block fires assert() statements to check one of the assumptions - ** in the comment below - that if the lhs sub-cursor of a level undergoing - ** a merge is valid, then all the rhs sub-cursors must be at EOF. - ** - ** Also assert that all rhs sub-cursors are either at EOF or point to - ** a key that is not less than the level split-key. */ - for(i=0; inPtr; i++){ - SegmentPtr *pPtr = &pCsr->aPtr[i]; - Level *pLvl = pPtr->pLevel; - if( pLvl->nRight && pPtr->pPg ){ - if( pPtr->pSeg==&pLvl->lhs ){ - int j; - for(j=0; jnRight; j++) assert( pPtr[j+1].pPg==0 ); - }else{ - int res = sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(pPtr->eType), pPtr->pKey, pPtr->nKey, - pLvl->iSplitTopic, pLvl->pSplitKey, pLvl->nSplitKey - ); - assert( res>=0 ); - } - } - } -#endif - - /* Now check if this key has already been deleted by a range-delete. If - ** so, skip past it. - ** - ** Assume, for the moment, that the tree contains no levels currently - ** undergoing incremental merge, and that this cursor is iterating forwards - ** through the database keys. The cursor currently points to a key in - ** level L. This key has already been deleted if any of the sub-cursors - ** that point to levels newer than L (or to the in-memory tree) point to - ** a key greater than the current key with the LSM_END_DELETE flag set. - ** - ** Or, if the cursor is iterating backwards through data keys, if any - ** such sub-cursor points to a key smaller than the current key with the - ** LSM_START_DELETE flag set. - ** - ** Why it works with levels undergoing a merge too: - ** - ** When a cursor iterates forwards, the sub-cursors for the rhs of a - ** level are only activated once the lhs reaches EOF. So when iterating - ** forwards, the keys visited are the same as if the level was completely - ** merged. - ** - ** If the cursor is iterating backwards, then the lhs sub-cursor is not - ** initialized until the last of the rhs sub-cursors has reached EOF. - ** Additionally, if the START_DELETE flag is set on the last entry (in - ** reverse order - so the entry with the smallest key) of a rhs sub-cursor, - ** then a pseudo-key equal to the levels split-key with the END_DELETE - ** flag set is visited by the sub-cursor. - */ - iKey = pCsr->aTree[1]; - for(i=0; iflags & CURSOR_IGNORE_DELETE)==0 - ){ - void *pKey; int nKey; - multiCursorGetKey(pCsr, i, 0, &pKey, &nKey); - if( 0==sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(eType), pCsr->key.pData, pCsr->key.nData, - rtTopic(csrflags), pKey, nKey - )){ - continue; - } - } - return 0; - } - } - - /* The current cursor position is one this cursor should visit. Return 1. */ - return 1; -} - -static int multiCursorSetupTree(MultiCursor *pCsr, int bRev){ - int rc; - - rc = multiCursorAllocTree(pCsr); - if( rc==LSM_OK ){ - int i; - for(i=pCsr->nTree-1; i>0; i--){ - multiCursorDoCompare(pCsr, i, bRev); - } - } - - assertCursorTree(pCsr); - multiCursorCacheKey(pCsr, &rc); - - if( rc==LSM_OK && mcursorLocationOk(pCsr, 0)==0 ){ - rc = multiCursorAdvance(pCsr, bRev); - } - return rc; -} - - -static int multiCursorEnd(MultiCursor *pCsr, int bLast){ - int rc = LSM_OK; - int i; - - pCsr->flags &= ~(CURSOR_NEXT_OK | CURSOR_PREV_OK | CURSOR_SEEK_EQ); - pCsr->flags |= (bLast ? CURSOR_PREV_OK : CURSOR_NEXT_OK); - pCsr->iFree = 0; - - /* Position the two in-memory tree cursors */ - for(i=0; rc==LSM_OK && i<2; i++){ - if( pCsr->apTreeCsr[i] ){ - rc = lsmTreeCursorEnd(pCsr->apTreeCsr[i], bLast); - } - } - - for(i=0; rc==LSM_OK && inPtr; i++){ - SegmentPtr *pPtr = &pCsr->aPtr[i]; - Level *pLvl = pPtr->pLevel; - int iRhs; - int bHit = 0; - - if( bLast ){ - for(iRhs=0; iRhsnRight && rc==LSM_OK; iRhs++){ - rc = segmentPtrEnd(pCsr, &pPtr[iRhs+1], 1); - if( pPtr[iRhs+1].pPg ) bHit = 1; - } - if( bHit==0 && rc==LSM_OK ){ - rc = segmentPtrEnd(pCsr, pPtr, 1); - }else{ - segmentPtrReset(pPtr, LSM_SEGMENTPTR_FREE_THRESHOLD); - } - }else{ - int bLhs = (pPtr->pSeg==&pLvl->lhs); - assert( pPtr->pSeg==&pLvl->lhs || pPtr->pSeg==&pLvl->aRhs[0] ); - - if( bLhs ){ - rc = segmentPtrEnd(pCsr, pPtr, 0); - if( pPtr->pKey ) bHit = 1; - } - for(iRhs=0; iRhsnRight && rc==LSM_OK; iRhs++){ - if( bHit ){ - segmentPtrReset(&pPtr[iRhs+1], LSM_SEGMENTPTR_FREE_THRESHOLD); - }else{ - rc = sortedRhsFirst(pCsr, pLvl, &pPtr[iRhs+bLhs]); - } - } - } - i += pLvl->nRight; - } - - /* And the b-tree cursor, if applicable */ - if( rc==LSM_OK && pCsr->pBtCsr ){ - assert( bLast==0 ); - rc = btreeCursorFirst(pCsr->pBtCsr); - } - - if( rc==LSM_OK ){ - rc = multiCursorSetupTree(pCsr, bLast); - } - - return rc; -} - - -int mcursorSave(MultiCursor *pCsr){ - int rc = LSM_OK; - if( pCsr->aTree ){ - int iTree = pCsr->aTree[1]; - if( iTree==CURSOR_DATA_TREE0 || iTree==CURSOR_DATA_TREE1 ){ - multiCursorCacheKey(pCsr, &rc); - } - } - mcursorFreeComponents(pCsr); - return rc; -} - -int mcursorRestore(lsm_db *pDb, MultiCursor *pCsr){ - int rc; - rc = multiCursorInit(pCsr, pDb->pClient); - if( rc==LSM_OK && pCsr->key.pData ){ - rc = lsmMCursorSeek(pCsr, - rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, +1 - ); - } - return rc; -} - -int lsmSaveCursors(lsm_db *pDb){ - int rc = LSM_OK; - MultiCursor *pCsr; - - for(pCsr=pDb->pCsr; rc==LSM_OK && pCsr; pCsr=pCsr->pNext){ - rc = mcursorSave(pCsr); - } - return rc; -} - -int lsmRestoreCursors(lsm_db *pDb){ - int rc = LSM_OK; - MultiCursor *pCsr; - - for(pCsr=pDb->pCsr; rc==LSM_OK && pCsr; pCsr=pCsr->pNext){ - rc = mcursorRestore(pDb, pCsr); - } - return rc; -} - -int lsmMCursorFirst(MultiCursor *pCsr){ - return multiCursorEnd(pCsr, 0); -} - -int lsmMCursorLast(MultiCursor *pCsr){ - return multiCursorEnd(pCsr, 1); -} - -lsm_db *lsmMCursorDb(MultiCursor *pCsr){ - return pCsr->pDb; -} - -void lsmMCursorReset(MultiCursor *pCsr){ - int i; - lsmTreeCursorReset(pCsr->apTreeCsr[0]); - lsmTreeCursorReset(pCsr->apTreeCsr[1]); - for(i=0; inPtr; i++){ - segmentPtrReset(&pCsr->aPtr[i], LSM_SEGMENTPTR_FREE_THRESHOLD); - } - pCsr->key.nData = 0; -} - -static int treeCursorSeek( - MultiCursor *pCsr, - TreeCursor *pTreeCsr, - void *pKey, int nKey, - int eSeek, - int *pbStop -){ - int rc = LSM_OK; - if( pTreeCsr ){ - int res = 0; - lsmTreeCursorSeek(pTreeCsr, pKey, nKey, &res); - switch( eSeek ){ - case LSM_SEEK_EQ: { - int eType = lsmTreeCursorFlags(pTreeCsr); - if( (res<0 && (eType & LSM_START_DELETE)) - || (res>0 && (eType & LSM_END_DELETE)) - || (res==0 && (eType & LSM_POINT_DELETE)) - ){ - *pbStop = 1; - }else if( res==0 && (eType & LSM_INSERT) ){ - lsm_env *pEnv = pCsr->pDb->pEnv; - void *p; int n; /* Key/value from tree-cursor */ - *pbStop = 1; - pCsr->flags |= CURSOR_SEEK_EQ; - rc = lsmTreeCursorKey(pTreeCsr, &pCsr->eType, &p, &n); - if( rc==LSM_OK ) rc = sortedBlobSet(pEnv, &pCsr->key, p, n); - if( rc==LSM_OK ) rc = lsmTreeCursorValue(pTreeCsr, &p, &n); - if( rc==LSM_OK ) rc = sortedBlobSet(pEnv, &pCsr->val, p, n); - } - lsmTreeCursorReset(pTreeCsr); - break; - } - case LSM_SEEK_GE: - if( res<0 && lsmTreeCursorValid(pTreeCsr) ){ - lsmTreeCursorNext(pTreeCsr); - } - break; - default: - if( res>0 ){ - assert( lsmTreeCursorValid(pTreeCsr) ); - lsmTreeCursorPrev(pTreeCsr); - } - break; - } - } - return rc; -} - - -/* -** Seek the cursor. -*/ -int lsmMCursorSeek( - MultiCursor *pCsr, - int iTopic, - void *pKey, int nKey, - int eSeek -){ - int eESeek = eSeek; /* Effective eSeek parameter */ - int bStop = 0; /* Set to true to halt search operation */ - int rc = LSM_OK; /* Return code */ - int iPtr = 0; /* Used to iterate through pCsr->aPtr[] */ - LsmPgno iPgno = 0; /* FC pointer value */ - - assert( pCsr->apTreeCsr[0]==0 || iTopic==0 ); - assert( pCsr->apTreeCsr[1]==0 || iTopic==0 ); - - if( eESeek==LSM_SEEK_LEFAST ) eESeek = LSM_SEEK_LE; - - assert( eESeek==LSM_SEEK_EQ || eESeek==LSM_SEEK_LE || eESeek==LSM_SEEK_GE ); - assert( (pCsr->flags & CURSOR_FLUSH_FREELIST)==0 ); - assert( pCsr->nPtr==0 || pCsr->aPtr[0].pLevel ); - - pCsr->flags &= ~(CURSOR_NEXT_OK | CURSOR_PREV_OK | CURSOR_SEEK_EQ); - rc = treeCursorSeek(pCsr, pCsr->apTreeCsr[0], pKey, nKey, eESeek, &bStop); - if( rc==LSM_OK && bStop==0 ){ - rc = treeCursorSeek(pCsr, pCsr->apTreeCsr[1], pKey, nKey, eESeek, &bStop); - } - - /* Seek all segment pointers. */ - for(iPtr=0; iPtrnPtr && rc==LSM_OK && bStop==0; iPtr++){ - SegmentPtr *pPtr = &pCsr->aPtr[iPtr]; - assert( pPtr->pSeg==&pPtr->pLevel->lhs ); - rc = seekInLevel(pCsr, pPtr, eESeek, iTopic, pKey, nKey, &iPgno, &bStop); - iPtr += pPtr->pLevel->nRight; - } - - if( eSeek!=LSM_SEEK_EQ ){ - if( rc==LSM_OK ){ - rc = multiCursorAllocTree(pCsr); - } - if( rc==LSM_OK ){ - int i; - for(i=pCsr->nTree-1; i>0; i--){ - multiCursorDoCompare(pCsr, i, eESeek==LSM_SEEK_LE); - } - if( eSeek==LSM_SEEK_GE ) pCsr->flags |= CURSOR_NEXT_OK; - if( eSeek==LSM_SEEK_LE ) pCsr->flags |= CURSOR_PREV_OK; - } - - multiCursorCacheKey(pCsr, &rc); - if( rc==LSM_OK && eSeek!=LSM_SEEK_LEFAST && 0==mcursorLocationOk(pCsr, 0) ){ - switch( eESeek ){ - case LSM_SEEK_EQ: - lsmMCursorReset(pCsr); - break; - case LSM_SEEK_GE: - rc = lsmMCursorNext(pCsr); - break; - default: - rc = lsmMCursorPrev(pCsr); - break; - } - } - } - - return rc; -} - -int lsmMCursorValid(MultiCursor *pCsr){ - int res = 0; - if( pCsr->flags & CURSOR_SEEK_EQ ){ - res = 1; - }else if( pCsr->aTree ){ - int iKey = pCsr->aTree[1]; - if( iKey==CURSOR_DATA_TREE0 || iKey==CURSOR_DATA_TREE1 ){ - res = lsmTreeCursorValid(pCsr->apTreeCsr[iKey-CURSOR_DATA_TREE0]); - }else{ - void *pKey; - multiCursorGetKey(pCsr, iKey, 0, &pKey, 0); - res = pKey!=0; - } - } - return res; -} - -static int mcursorAdvanceOk( - MultiCursor *pCsr, - int bReverse, - int *pRc -){ - void *pNew; /* Pointer to buffer containing new key */ - int nNew; /* Size of buffer pNew in bytes */ - int eNewType; /* Type of new record */ - - if( *pRc ) return 1; - - /* Check the current key value. If it is not greater than (if bReverse==0) - ** or less than (if bReverse!=0) the key currently cached in pCsr->key, - ** then the cursor has not yet been successfully advanced. - */ - multiCursorGetKey(pCsr, pCsr->aTree[1], &eNewType, &pNew, &nNew); - if( pNew ){ - int typemask = (pCsr->flags & CURSOR_IGNORE_DELETE) ? ~(0) : LSM_SYSTEMKEY; - int res = sortedDbKeyCompare(pCsr, - eNewType & typemask, pNew, nNew, - pCsr->eType & typemask, pCsr->key.pData, pCsr->key.nData - ); - - if( (bReverse==0 && res<=0) || (bReverse!=0 && res>=0) ){ - return 0; - } - - multiCursorCacheKey(pCsr, pRc); - assert( pCsr->eType==eNewType ); - - /* If this cursor is configured to skip deleted keys, and the current - ** cursor points to a SORTED_DELETE entry, then the cursor has not been - ** successfully advanced. - ** - ** Similarly, if the cursor is configured to skip system keys and the - ** current cursor points to a system key, it has not yet been advanced. - */ - if( *pRc==LSM_OK && 0==mcursorLocationOk(pCsr, 0) ) return 0; - } - return 1; -} - -static void flCsrAdvance(MultiCursor *pCsr){ - assert( pCsr->flags & CURSOR_FLUSH_FREELIST ); - if( pCsr->iFree % 2 ){ - pCsr->iFree++; - }else{ - int nEntry = pCsr->pDb->pWorker->freelist.nEntry; - FreelistEntry *aEntry = pCsr->pDb->pWorker->freelist.aEntry; - - int i = nEntry - 1 - (pCsr->iFree / 2); - - /* If the current entry is a delete and the "end-delete" key will not - ** be attached to the next entry, increment iFree by 1 only. */ - if( aEntry[i].iId<0 ){ - while( 1 ){ - if( i==0 || aEntry[i-1].iBlk!=aEntry[i].iBlk-1 ){ - pCsr->iFree--; - break; - } - if( aEntry[i-1].iId>=0 ) break; - pCsr->iFree += 2; - i--; - } - } - pCsr->iFree += 2; - } -} - -static int multiCursorAdvance(MultiCursor *pCsr, int bReverse){ - int rc = LSM_OK; /* Return Code */ - if( lsmMCursorValid(pCsr) ){ - do { - int iKey = pCsr->aTree[1]; - - assertCursorTree(pCsr); - - /* If this multi-cursor is advancing forwards, and the sub-cursor - ** being advanced is the one that separator keys may be being read - ** from, record the current absolute pointer value. */ - if( pCsr->pPrevMergePtr ){ - if( iKey==(CURSOR_DATA_SEGMENT+pCsr->nPtr) ){ - assert( pCsr->pBtCsr ); - *pCsr->pPrevMergePtr = pCsr->pBtCsr->iPtr; - }else if( pCsr->pBtCsr==0 && pCsr->nPtr>0 - && iKey==(CURSOR_DATA_SEGMENT+pCsr->nPtr-1) - ){ - SegmentPtr *pPtr = &pCsr->aPtr[iKey-CURSOR_DATA_SEGMENT]; - *pCsr->pPrevMergePtr = pPtr->iPtr+pPtr->iPgPtr; - } - } - - if( iKey==CURSOR_DATA_TREE0 || iKey==CURSOR_DATA_TREE1 ){ - TreeCursor *pTreeCsr = pCsr->apTreeCsr[iKey-CURSOR_DATA_TREE0]; - if( bReverse ){ - rc = lsmTreeCursorPrev(pTreeCsr); - }else{ - rc = lsmTreeCursorNext(pTreeCsr); - } - }else if( iKey==CURSOR_DATA_SYSTEM ){ - assert( pCsr->flags & CURSOR_FLUSH_FREELIST ); - assert( bReverse==0 ); - flCsrAdvance(pCsr); - }else if( iKey==(CURSOR_DATA_SEGMENT+pCsr->nPtr) ){ - assert( bReverse==0 && pCsr->pBtCsr ); - rc = btreeCursorNext(pCsr->pBtCsr); - }else{ - rc = segmentCursorAdvance(pCsr, iKey-CURSOR_DATA_SEGMENT, bReverse); - } - if( rc==LSM_OK ){ - int i; - for(i=(iKey+pCsr->nTree)/2; i>0; i=i/2){ - multiCursorDoCompare(pCsr, i, bReverse); - } - assertCursorTree(pCsr); - } - }while( mcursorAdvanceOk(pCsr, bReverse, &rc)==0 ); - } - return rc; -} - -int lsmMCursorNext(MultiCursor *pCsr){ - if( (pCsr->flags & CURSOR_NEXT_OK)==0 ) return LSM_MISUSE_BKPT; - return multiCursorAdvance(pCsr, 0); -} - -int lsmMCursorPrev(MultiCursor *pCsr){ - if( (pCsr->flags & CURSOR_PREV_OK)==0 ) return LSM_MISUSE_BKPT; - return multiCursorAdvance(pCsr, 1); -} - -int lsmMCursorKey(MultiCursor *pCsr, void **ppKey, int *pnKey){ - if( (pCsr->flags & CURSOR_SEEK_EQ) || pCsr->aTree==0 ){ - *pnKey = pCsr->key.nData; - *ppKey = pCsr->key.pData; - }else{ - int iKey = pCsr->aTree[1]; - - if( iKey==CURSOR_DATA_TREE0 || iKey==CURSOR_DATA_TREE1 ){ - TreeCursor *pTreeCsr = pCsr->apTreeCsr[iKey-CURSOR_DATA_TREE0]; - lsmTreeCursorKey(pTreeCsr, 0, ppKey, pnKey); - }else{ - int nKey; - -#ifndef NDEBUG - void *pKey; - int eType; - multiCursorGetKey(pCsr, iKey, &eType, &pKey, &nKey); - assert( eType==pCsr->eType ); - assert( nKey==pCsr->key.nData ); - assert( memcmp(pKey, pCsr->key.pData, nKey)==0 ); -#endif - - nKey = pCsr->key.nData; - if( nKey==0 ){ - *ppKey = 0; - }else{ - *ppKey = pCsr->key.pData; - } - *pnKey = nKey; - } - } - return LSM_OK; -} - -/* -** Compare the current key that cursor csr points to with pKey/nKey. Set -** *piRes to the result and return LSM_OK. -*/ -int lsm_csr_cmp(lsm_cursor *csr, const void *pKey, int nKey, int *piRes){ - MultiCursor *pCsr = (MultiCursor *)csr; - void *pCsrkey; int nCsrkey; - int rc; - rc = lsmMCursorKey(pCsr, &pCsrkey, &nCsrkey); - if( rc==LSM_OK ){ - int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; - *piRes = sortedKeyCompare(xCmp, 0, pCsrkey, nCsrkey, 0, (void *)pKey, nKey); - } - return rc; -} - -int lsmMCursorValue(MultiCursor *pCsr, void **ppVal, int *pnVal){ - void *pVal; - int nVal; - int rc; - if( (pCsr->flags & CURSOR_SEEK_EQ) || pCsr->aTree==0 ){ - rc = LSM_OK; - nVal = pCsr->val.nData; - pVal = pCsr->val.pData; - }else{ - - assert( pCsr->aTree ); - assert( mcursorLocationOk(pCsr, (pCsr->flags & CURSOR_IGNORE_DELETE)) ); - - rc = multiCursorGetVal(pCsr, pCsr->aTree[1], &pVal, &nVal); - if( pVal && rc==LSM_OK ){ - rc = sortedBlobSet(pCsr->pDb->pEnv, &pCsr->val, pVal, nVal); - pVal = pCsr->val.pData; - } - - if( rc!=LSM_OK ){ - pVal = 0; - nVal = 0; - } - } - *ppVal = pVal; - *pnVal = nVal; - return rc; -} - -int lsmMCursorType(MultiCursor *pCsr, int *peType){ - assert( pCsr->aTree ); - multiCursorGetKey(pCsr, pCsr->aTree[1], peType, 0, 0); - return LSM_OK; -} - -/* -** Buffer aData[], size nData, is assumed to contain a valid b-tree -** hierarchy page image. Return the offset in aData[] of the next free -** byte in the data area (where a new cell may be written if there is -** space). -*/ -static int mergeWorkerPageOffset(u8 *aData, int nData){ - int nRec; - int iOff; - int nKey; - int eType; - i64 nDummy; - - - nRec = lsmGetU16(&aData[SEGMENT_NRECORD_OFFSET(nData)]); - iOff = lsmGetU16(&aData[SEGMENT_CELLPTR_OFFSET(nData, nRec-1)]); - eType = aData[iOff++]; - assert( eType==0 - || eType==(LSM_SYSTEMKEY|LSM_SEPARATOR) - || eType==(LSM_SEPARATOR) - ); - - iOff += lsmVarintGet64(&aData[iOff], &nDummy); - iOff += lsmVarintGet32(&aData[iOff], &nKey); - - return iOff + (eType ? nKey : 0); -} - -/* -** Following a checkpoint operation, database pages that are part of the -** checkpointed state of the LSM are deemed read-only. This includes the -** right-most page of the b-tree hierarchy of any separators array under -** construction, and all pages between it and the b-tree root, inclusive. -** This is a problem, as when further pages are appended to the separators -** array, entries must be added to the indicated b-tree hierarchy pages. -** -** This function copies all such b-tree pages to new locations, so that -** they can be modified as required. -** -** The complication is that not all database pages are the same size - due -** to the way the file.c module works some (the first and last in each block) -** are 4 bytes smaller than the others. -*/ -static int mergeWorkerMoveHierarchy( - MergeWorker *pMW, /* Merge worker */ - int bSep /* True for separators run */ -){ - lsm_db *pDb = pMW->pDb; /* Database handle */ - int rc = LSM_OK; /* Return code */ - int i; - Page **apHier = pMW->hier.apHier; - int nHier = pMW->hier.nHier; - - for(i=0; rc==LSM_OK && ipFS, pDb->pWorker, pMW->pLevel, 1, &pNew); - assert( rc==LSM_OK ); - - if( rc==LSM_OK ){ - u8 *a1; int n1; - u8 *a2; int n2; - - a1 = fsPageData(pNew, &n1); - a2 = fsPageData(apHier[i], &n2); - - assert( n1==n2 || n1+4==n2 ); - - if( n1==n2 ){ - memcpy(a1, a2, n2); - }else{ - int nEntry = pageGetNRec(a2, n2); - int iEof1 = SEGMENT_EOF(n1, nEntry); - int iEof2 = SEGMENT_EOF(n2, nEntry); - - memcpy(a1, a2, iEof2 - 4); - memcpy(&a1[iEof1], &a2[iEof2], n2 - iEof2); - } - - lsmFsPageRelease(apHier[i]); - apHier[i] = pNew; - -#if 0 - assert( n1==n2 || n1+4==n2 || n2+4==n1 ); - if( n1>=n2 ){ - /* If n1 (size of the new page) is equal to or greater than n2 (the - ** size of the old page), then copy the data into the new page. If - ** n1==n2, this could be done with a single memcpy(). However, - ** since sometimes n1>n2, the page content and footer must be copied - ** separately. */ - int nEntry = pageGetNRec(a2, n2); - int iEof1 = SEGMENT_EOF(n1, nEntry); - int iEof2 = SEGMENT_EOF(n2, nEntry); - memcpy(a1, a2, iEof2); - memcpy(&a1[iEof1], &a2[iEof2], n2 - iEof2); - lsmFsPageRelease(apHier[i]); - apHier[i] = pNew; - }else{ - lsmPutU16(&a1[SEGMENT_FLAGS_OFFSET(n1)], SEGMENT_BTREE_FLAG); - lsmPutU16(&a1[SEGMENT_NRECORD_OFFSET(n1)], 0); - lsmPutU64(&a1[SEGMENT_POINTER_OFFSET(n1)], 0); - i = i - 1; - lsmFsPageRelease(pNew); - } -#endif - } - } - -#ifdef LSM_DEBUG - if( rc==LSM_OK ){ - for(i=0; ipLevel->lhs; - p = &pMW->hier; - - if( p->apHier==0 && pSeg->iRoot!=0 ){ - FileSystem *pFS = pMW->pDb->pFS; - lsm_env *pEnv = pMW->pDb->pEnv; - Page **apHier = 0; - int nHier = 0; - LsmPgno iPg = pSeg->iRoot; - - do { - Page *pPg = 0; - u8 *aData; - int nData; - int flags; - - rc = lsmFsDbPageGet(pFS, pSeg, iPg, &pPg); - if( rc!=LSM_OK ) break; - - aData = fsPageData(pPg, &nData); - flags = pageGetFlags(aData, nData); - if( flags&SEGMENT_BTREE_FLAG ){ - Page **apNew = (Page **)lsmRealloc( - pEnv, apHier, sizeof(Page *)*(nHier+1) - ); - if( apNew==0 ){ - rc = LSM_NOMEM_BKPT; - break; - } - apHier = apNew; - memmove(&apHier[1], &apHier[0], sizeof(Page *) * nHier); - nHier++; - - apHier[0] = pPg; - iPg = pageGetPtr(aData, nData); - }else{ - lsmFsPageRelease(pPg); - break; - } - }while( 1 ); - - if( rc==LSM_OK ){ - u8 *aData; - int nData; - aData = fsPageData(apHier[0], &nData); - pMW->aSave[0].iPgno = pageGetPtr(aData, nData); - p->nHier = nHier; - p->apHier = apHier; - rc = mergeWorkerMoveHierarchy(pMW, 0); - }else{ - int i; - for(i=0; ihier; - lsm_db *pDb = pMW->pDb; /* Database handle */ - int rc = LSM_OK; /* Return Code */ - int iLevel; /* Level of b-tree hierarchy to write to */ - int nData; /* Size of aData[] in bytes */ - u8 *aData; /* Page data for level iLevel */ - int iOff; /* Offset on b-tree page to write record to */ - int nRec; /* Initial number of records on b-tree page */ - - /* iKeyPg should be zero for an ordinary b-tree key, or non-zero for an - ** indirect key. The flags byte for an indirect key is 0x00. */ - assert( (eType==0)==(iKeyPg!=0) ); - - /* The MergeWorker.apHier[] array contains the right-most leaf of the b-tree - ** hierarchy, the root node, and all nodes that lie on the path between. - ** apHier[0] is the right-most leaf and apHier[pMW->nHier-1] is the current - ** root page. - ** - ** This loop searches for a node with enough space to store the key on, - ** starting with the leaf and iterating up towards the root. When the loop - ** exits, the key may be written to apHier[iLevel]. */ - for(iLevel=0; iLevel<=p->nHier; iLevel++){ - int nByte; /* Number of free bytes required */ - - if( iLevel==p->nHier ){ - /* Extend the array and allocate a new root page. */ - Page **aNew; - aNew = (Page **)lsmRealloc( - pMW->pDb->pEnv, p->apHier, sizeof(Page *)*(p->nHier+1) - ); - if( !aNew ){ - return LSM_NOMEM_BKPT; - } - p->apHier = aNew; - }else{ - Page *pOld; - int nFree; - - /* If the key will fit on this page, break out of the loop here. - ** The new entry will be written to page apHier[iLevel]. */ - pOld = p->apHier[iLevel]; - assert( lsmFsPageWritable(pOld) ); - aData = fsPageData(pOld, &nData); - if( eType==0 ){ - nByte = 2 + 1 + lsmVarintLen64(iPtr) + lsmVarintLen64(iKeyPg); - }else{ - nByte = 2 + 1 + lsmVarintLen64(iPtr) + lsmVarintLen32(nKey) + nKey; - } - - nRec = pageGetNRec(aData, nData); - nFree = SEGMENT_EOF(nData, nRec) - mergeWorkerPageOffset(aData, nData); - if( nByte<=nFree ) break; - - /* Otherwise, this page is full. Set the right-hand-child pointer - ** to iPtr and release it. */ - lsmPutU64(&aData[SEGMENT_POINTER_OFFSET(nData)], iPtr); - assert( lsmFsPageNumber(pOld)==0 ); - rc = lsmFsPagePersist(pOld); - if( rc==LSM_OK ){ - iPtr = lsmFsPageNumber(pOld); - lsmFsPageRelease(pOld); - } - } - - /* Allocate a new page for apHier[iLevel]. */ - p->apHier[iLevel] = 0; - if( rc==LSM_OK ){ - rc = lsmFsSortedAppend( - pDb->pFS, pDb->pWorker, pMW->pLevel, 1, &p->apHier[iLevel] - ); - } - if( rc!=LSM_OK ) return rc; - - aData = fsPageData(p->apHier[iLevel], &nData); - memset(aData, 0, nData); - lsmPutU16(&aData[SEGMENT_FLAGS_OFFSET(nData)], SEGMENT_BTREE_FLAG); - lsmPutU16(&aData[SEGMENT_NRECORD_OFFSET(nData)], 0); - - if( iLevel==p->nHier ){ - p->nHier++; - break; - } - } - - /* Write the key into page apHier[iLevel]. */ - aData = fsPageData(p->apHier[iLevel], &nData); - iOff = mergeWorkerPageOffset(aData, nData); - nRec = pageGetNRec(aData, nData); - lsmPutU16(&aData[SEGMENT_CELLPTR_OFFSET(nData, nRec)], (u16)iOff); - lsmPutU16(&aData[SEGMENT_NRECORD_OFFSET(nData)], (u16)(nRec+1)); - if( eType==0 ){ - aData[iOff++] = 0x00; - iOff += lsmVarintPut64(&aData[iOff], iPtr); - iOff += lsmVarintPut64(&aData[iOff], iKeyPg); - }else{ - aData[iOff++] = eType; - iOff += lsmVarintPut64(&aData[iOff], iPtr); - iOff += lsmVarintPut32(&aData[iOff], nKey); - memcpy(&aData[iOff], pKey, nKey); - } - - return rc; -} - -static int mergeWorkerBtreeIndirect(MergeWorker *pMW){ - int rc = LSM_OK; - if( pMW->iIndirect ){ - LsmPgno iKeyPg = pMW->aSave[1].iPgno; - rc = mergeWorkerBtreeWrite(pMW, 0, pMW->iIndirect, iKeyPg, 0, 0); - pMW->iIndirect = 0; - } - return rc; -} - -/* -** Append the database key (iTopic/pKey/nKey) to the b-tree under -** construction. This key has not yet been written to a segment page. -** The pointer that will accompany the new key in the b-tree - that -** points to the completed segment page that contains keys smaller than -** (pKey/nKey) is currently stored in pMW->aSave[0].iPgno. -*/ -static int mergeWorkerPushHierarchy( - MergeWorker *pMW, /* Merge worker object */ - int iTopic, /* Topic value for this key */ - void *pKey, /* Pointer to key buffer */ - int nKey /* Size of pKey buffer in bytes */ -){ - int rc = LSM_OK; /* Return Code */ - LsmPgno iPtr; /* Pointer value to accompany pKey/nKey */ - - assert( pMW->aSave[0].bStore==0 ); - assert( pMW->aSave[1].bStore==0 ); - rc = mergeWorkerBtreeIndirect(pMW); - - /* Obtain the absolute pointer value to store along with the key in the - ** page body. This pointer points to a page that contains keys that are - ** smaller than pKey/nKey. */ - iPtr = pMW->aSave[0].iPgno; - assert( iPtr!=0 ); - - /* Determine if the indirect format should be used. */ - if( (nKey*4 > lsmFsPageSize(pMW->pDb->pFS)) ){ - pMW->iIndirect = iPtr; - pMW->aSave[1].bStore = 1; - }else{ - rc = mergeWorkerBtreeWrite( - pMW, (u8)(iTopic | LSM_SEPARATOR), iPtr, 0, pKey, nKey - ); - } - - /* Ensure that the SortedRun.iRoot field is correct. */ - return rc; -} - -static int mergeWorkerFinishHierarchy( - MergeWorker *pMW /* Merge worker object */ -){ - int i; /* Used to loop through apHier[] */ - int rc = LSM_OK; /* Return code */ - LsmPgno iPtr; /* New right-hand-child pointer value */ - - iPtr = pMW->aSave[0].iPgno; - for(i=0; ihier.nHier && rc==LSM_OK; i++){ - Page *pPg = pMW->hier.apHier[i]; - int nData; /* Size of aData[] in bytes */ - u8 *aData; /* Page data for pPg */ - - aData = fsPageData(pPg, &nData); - lsmPutU64(&aData[SEGMENT_POINTER_OFFSET(nData)], iPtr); - - rc = lsmFsPagePersist(pPg); - iPtr = lsmFsPageNumber(pPg); - lsmFsPageRelease(pPg); - } - - if( pMW->hier.nHier ){ - pMW->pLevel->lhs.iRoot = iPtr; - lsmFree(pMW->pDb->pEnv, pMW->hier.apHier); - pMW->hier.apHier = 0; - pMW->hier.nHier = 0; - } - - return rc; -} - -static int mergeWorkerAddPadding( - MergeWorker *pMW /* Merge worker object */ -){ - FileSystem *pFS = pMW->pDb->pFS; - return lsmFsSortedPadding(pFS, pMW->pDb->pWorker, &pMW->pLevel->lhs); -} - -/* -** Release all page references currently held by the merge-worker passed -** as the only argument. Unless an error has occurred, all pages have -** already been released. -*/ -static void mergeWorkerReleaseAll(MergeWorker *pMW){ - int i; - lsmFsPageRelease(pMW->pPage); - pMW->pPage = 0; - - for(i=0; ihier.nHier; i++){ - lsmFsPageRelease(pMW->hier.apHier[i]); - pMW->hier.apHier[i] = 0; - } - lsmFree(pMW->pDb->pEnv, pMW->hier.apHier); - pMW->hier.apHier = 0; - pMW->hier.nHier = 0; -} - -static int keyszToSkip(FileSystem *pFS, int nKey){ - int nPgsz; /* Nominal database page size */ - nPgsz = lsmFsPageSize(pFS); - return LSM_MIN(((nKey * 4) / nPgsz), 3); -} - -/* -** Release the reference to the current output page of merge-worker *pMW -** (reference pMW->pPage). Set the page number values in aSave[] as -** required (see comments above struct MergeWorker for details). -*/ -static int mergeWorkerPersistAndRelease(MergeWorker *pMW){ - int rc; - int i; - - assert( pMW->pPage || (pMW->aSave[0].bStore==0 && pMW->aSave[1].bStore==0) ); - - /* Persist the page */ - rc = lsmFsPagePersist(pMW->pPage); - - /* If required, save the page number. */ - for(i=0; i<2; i++){ - if( pMW->aSave[i].bStore ){ - pMW->aSave[i].iPgno = lsmFsPageNumber(pMW->pPage); - pMW->aSave[i].bStore = 0; - } - } - - /* Release the completed output page. */ - lsmFsPageRelease(pMW->pPage); - pMW->pPage = 0; - return rc; -} - -/* -** Advance to the next page of an output run being populated by merge-worker -** pMW. The footer of the new page is initialized to indicate that it contains -** zero records. The flags field is cleared. The page footer pointer field -** is set to iFPtr. -** -** If successful, LSM_OK is returned. Otherwise, an error code. -*/ -static int mergeWorkerNextPage( - MergeWorker *pMW, /* Merge worker object to append page to */ - LsmPgno iFPtr /* Pointer value for footer of new page */ -){ - int rc = LSM_OK; /* Return code */ - Page *pNext = 0; /* New page appended to run */ - lsm_db *pDb = pMW->pDb; /* Database handle */ - - rc = lsmFsSortedAppend(pDb->pFS, pDb->pWorker, pMW->pLevel, 0, &pNext); - assert( rc || pMW->pLevel->lhs.iFirst>0 || pMW->pDb->compress.xCompress ); - - if( rc==LSM_OK ){ - u8 *aData; /* Data buffer belonging to page pNext */ - int nData; /* Size of aData[] in bytes */ - - rc = mergeWorkerPersistAndRelease(pMW); - - pMW->pPage = pNext; - pMW->pLevel->pMerge->iOutputOff = 0; - aData = fsPageData(pNext, &nData); - lsmPutU16(&aData[SEGMENT_NRECORD_OFFSET(nData)], 0); - lsmPutU16(&aData[SEGMENT_FLAGS_OFFSET(nData)], 0); - lsmPutU64(&aData[SEGMENT_POINTER_OFFSET(nData)], iFPtr); - pMW->nWork++; - } - - return rc; -} - -/* -** Write a blob of data into an output segment being populated by a -** merge-worker object. If argument bSep is true, write into the separators -** array. Otherwise, the main array. -** -** This function is used to write the blobs of data for keys and values. -*/ -static int mergeWorkerData( - MergeWorker *pMW, /* Merge worker object */ - int bSep, /* True to write to separators run */ - LsmPgno iFPtr, /* Footer ptr for new pages */ - u8 *aWrite, /* Write data from this buffer */ - int nWrite /* Size of aWrite[] in bytes */ -){ - int rc = LSM_OK; /* Return code */ - int nRem = nWrite; /* Number of bytes still to write */ - - while( rc==LSM_OK && nRem>0 ){ - Merge *pMerge = pMW->pLevel->pMerge; - int nCopy; /* Number of bytes to copy */ - u8 *aData; /* Pointer to buffer of current output page */ - int nData; /* Size of aData[] in bytes */ - int nRec; /* Number of records on current output page */ - int iOff; /* Offset in aData[] to write to */ - - assert( lsmFsPageWritable(pMW->pPage) ); - - aData = fsPageData(pMW->pPage, &nData); - nRec = pageGetNRec(aData, nData); - iOff = pMerge->iOutputOff; - nCopy = LSM_MIN(nRem, SEGMENT_EOF(nData, nRec) - iOff); - - memcpy(&aData[iOff], &aWrite[nWrite-nRem], nCopy); - nRem -= nCopy; - - if( nRem>0 ){ - rc = mergeWorkerNextPage(pMW, iFPtr); - }else{ - pMerge->iOutputOff = iOff + nCopy; - } - } - - return rc; -} - - -/* -** The MergeWorker passed as the only argument is working to merge two or -** more existing segments together (not to flush an in-memory tree). It -** has not yet written the first key to the first page of the output. -*/ -static int mergeWorkerFirstPage(MergeWorker *pMW){ - int rc = LSM_OK; /* Return code */ - Page *pPg = 0; /* First page of run pSeg */ - LsmPgno iFPtr = 0; /* Pointer value read from footer of pPg */ - MultiCursor *pCsr = pMW->pCsr; - - assert( pMW->pPage==0 ); - - if( pCsr->pBtCsr ){ - rc = LSM_OK; - iFPtr = pMW->pLevel->pNext->lhs.iFirst; - }else if( pCsr->nPtr>0 ){ - Segment *pSeg; - pSeg = pCsr->aPtr[pCsr->nPtr-1].pSeg; - rc = lsmFsDbPageGet(pMW->pDb->pFS, pSeg, pSeg->iFirst, &pPg); - if( rc==LSM_OK ){ - u8 *aData; /* Buffer for page pPg */ - int nData; /* Size of aData[] in bytes */ - aData = fsPageData(pPg, &nData); - iFPtr = pageGetPtr(aData, nData); - lsmFsPageRelease(pPg); - } - } - - if( rc==LSM_OK ){ - rc = mergeWorkerNextPage(pMW, iFPtr); - if( pCsr->pPrevMergePtr ) *pCsr->pPrevMergePtr = iFPtr; - pMW->aSave[0].bStore = 1; - } - - return rc; -} - -static int mergeWorkerWrite( - MergeWorker *pMW, /* Merge worker object to write into */ - int eType, /* One of SORTED_SEPARATOR, WRITE or DELETE */ - void *pKey, int nKey, /* Key value */ - void *pVal, int nVal, /* Value value */ - LsmPgno iPtr /* Absolute value of page pointer, or 0 */ -){ - int rc = LSM_OK; /* Return code */ - Merge *pMerge; /* Persistent part of level merge state */ - int nHdr; /* Space required for this record header */ - Page *pPg; /* Page to write to */ - u8 *aData; /* Data buffer for page pWriter->pPage */ - int nData = 0; /* Size of buffer aData[] in bytes */ - int nRec = 0; /* Number of records on page pPg */ - LsmPgno iFPtr = 0; /* Value of pointer in footer of pPg */ - LsmPgno iRPtr = 0; /* Value of pointer written into record */ - int iOff = 0; /* Current write offset within page pPg */ - Segment *pSeg; /* Segment being written */ - int flags = 0; /* If != 0, flags value for page footer */ - int bFirst = 0; /* True for first key of output run */ - - pMerge = pMW->pLevel->pMerge; - pSeg = &pMW->pLevel->lhs; - - if( pSeg->iFirst==0 && pMW->pPage==0 ){ - rc = mergeWorkerFirstPage(pMW); - bFirst = 1; - } - pPg = pMW->pPage; - if( pPg ){ - aData = fsPageData(pPg, &nData); - nRec = pageGetNRec(aData, nData); - iFPtr = pageGetPtr(aData, nData); - iRPtr = iPtr ? (iPtr - iFPtr) : 0; - } - - /* Figure out how much space is required by the new record. The space - ** required is divided into two sections: the header and the body. The - ** header consists of the intial varint fields. The body are the blobs - ** of data that correspond to the key and value data. The entire header - ** must be stored on the page. The body may overflow onto the next and - ** subsequent pages. - ** - ** The header space is: - ** - ** 1) record type - 1 byte. - ** 2) Page-pointer-offset - 1 varint - ** 3) Key size - 1 varint - ** 4) Value size - 1 varint (only if LSM_INSERT flag is set) - */ - if( rc==LSM_OK ){ - nHdr = 1 + lsmVarintLen64(iRPtr) + lsmVarintLen32(nKey); - if( rtIsWrite(eType) ) nHdr += lsmVarintLen32(nVal); - - /* If the entire header will not fit on page pPg, or if page pPg is - ** marked read-only, advance to the next page of the output run. */ - iOff = pMerge->iOutputOff; - if( iOff<0 || pPg==0 || iOff+nHdr > SEGMENT_EOF(nData, nRec+1) ){ - if( iOff>=0 && pPg ){ - /* Zero any free space on the page */ - assert( aData ); - memset(&aData[iOff], 0, SEGMENT_EOF(nData, nRec)-iOff); - } - iFPtr = *pMW->pCsr->pPrevMergePtr; - iRPtr = iPtr ? (iPtr - iFPtr) : 0; - iOff = 0; - nRec = 0; - rc = mergeWorkerNextPage(pMW, iFPtr); - pPg = pMW->pPage; - } - } - - /* If this record header will be the first on the page, and the page is - ** not the very first in the entire run, add a copy of the key to the - ** b-tree hierarchy. - */ - if( rc==LSM_OK && nRec==0 && bFirst==0 ){ - assert( pMerge->nSkip>=0 ); - - if( pMerge->nSkip==0 ){ - rc = mergeWorkerPushHierarchy(pMW, rtTopic(eType), pKey, nKey); - assert( pMW->aSave[0].bStore==0 ); - pMW->aSave[0].bStore = 1; - pMerge->nSkip = keyszToSkip(pMW->pDb->pFS, nKey); - }else{ - pMerge->nSkip--; - flags = PGFTR_SKIP_THIS_FLAG; - } - - if( pMerge->nSkip ) flags |= PGFTR_SKIP_NEXT_FLAG; - } - - /* Update the output segment */ - if( rc==LSM_OK ){ - aData = fsPageData(pPg, &nData); - - /* Update the page footer. */ - lsmPutU16(&aData[SEGMENT_NRECORD_OFFSET(nData)], (u16)(nRec+1)); - lsmPutU16(&aData[SEGMENT_CELLPTR_OFFSET(nData, nRec)], (u16)iOff); - if( flags ) lsmPutU16(&aData[SEGMENT_FLAGS_OFFSET(nData)], (u16)flags); - - /* Write the entry header into the current page. */ - aData[iOff++] = (u8)eType; /* 1 */ - iOff += lsmVarintPut64(&aData[iOff], iRPtr); /* 2 */ - iOff += lsmVarintPut32(&aData[iOff], nKey); /* 3 */ - if( rtIsWrite(eType) ) iOff += lsmVarintPut32(&aData[iOff], nVal); /* 4 */ - pMerge->iOutputOff = iOff; - - /* Write the key and data into the segment. */ - assert( iFPtr==pageGetPtr(aData, nData) ); - rc = mergeWorkerData(pMW, 0, iFPtr+iRPtr, pKey, nKey); - if( rc==LSM_OK && rtIsWrite(eType) ){ - if( rc==LSM_OK ){ - rc = mergeWorkerData(pMW, 0, iFPtr+iRPtr, pVal, nVal); - } - } - } - - return rc; -} - - -/* -** Free all resources allocated by mergeWorkerInit(). -*/ -static void mergeWorkerShutdown(MergeWorker *pMW, int *pRc){ - int i; /* Iterator variable */ - int rc = *pRc; - MultiCursor *pCsr = pMW->pCsr; - - /* Unless the merge has finished, save the cursor position in the - ** Merge.aInput[] array. See function mergeWorkerInit() for the - ** code to restore a cursor position based on aInput[]. */ - if( rc==LSM_OK && pCsr ){ - Merge *pMerge = pMW->pLevel->pMerge; - if( lsmMCursorValid(pCsr) ){ - int bBtree = (pCsr->pBtCsr!=0); - int iPtr; - - /* pMerge->nInput==0 indicates that this is a FlushTree() operation. */ - assert( pMerge->nInput==0 || pMW->pLevel->nRight>0 ); - assert( pMerge->nInput==0 || pMerge->nInput==(pCsr->nPtr+bBtree) ); - - for(i=0; i<(pMerge->nInput-bBtree); i++){ - SegmentPtr *pPtr = &pCsr->aPtr[i]; - if( pPtr->pPg ){ - pMerge->aInput[i].iPg = lsmFsPageNumber(pPtr->pPg); - pMerge->aInput[i].iCell = pPtr->iCell; - }else{ - pMerge->aInput[i].iPg = 0; - pMerge->aInput[i].iCell = 0; - } - } - if( bBtree && pMerge->nInput ){ - assert( i==pCsr->nPtr ); - btreeCursorPosition(pCsr->pBtCsr, &pMerge->aInput[i]); - } - - /* Store the location of the split-key */ - iPtr = pCsr->aTree[1] - CURSOR_DATA_SEGMENT; - if( iPtrnPtr ){ - pMerge->splitkey = pMerge->aInput[iPtr]; - }else{ - btreeCursorSplitkey(pCsr->pBtCsr, &pMerge->splitkey); - } - } - - /* Zero any free space left on the final page. This helps with - ** compression if using a compression hook. And prevents valgrind - ** from complaining about uninitialized byte passed to write(). */ - if( pMW->pPage ){ - int nData; - u8 *aData = fsPageData(pMW->pPage, &nData); - int iOff = pMerge->iOutputOff; - int iEof = SEGMENT_EOF(nData, pageGetNRec(aData, nData)); - memset(&aData[iOff], 0, iEof - iOff); - } - - pMerge->iOutputOff = -1; - } - - lsmMCursorClose(pCsr, 0); - - /* Persist and release the output page. */ - if( rc==LSM_OK ) rc = mergeWorkerPersistAndRelease(pMW); - if( rc==LSM_OK ) rc = mergeWorkerBtreeIndirect(pMW); - if( rc==LSM_OK ) rc = mergeWorkerFinishHierarchy(pMW); - if( rc==LSM_OK ) rc = mergeWorkerAddPadding(pMW); - lsmFsFlushWaiting(pMW->pDb->pFS, &rc); - mergeWorkerReleaseAll(pMW); - - lsmFree(pMW->pDb->pEnv, pMW->aGobble); - pMW->aGobble = 0; - pMW->pCsr = 0; - - *pRc = rc; -} - -/* -** The cursor passed as the first argument is being used as the input for -** a merge operation. When this function is called, *piFlags contains the -** database entry flags for the current entry. The entry about to be written -** to the output. -** -** Note that this function only has to work for cursors configured to -** iterate forwards (not backwards). -*/ -static void mergeRangeDeletes(MultiCursor *pCsr, int *piVal, int *piFlags){ - int f = *piFlags; - int iKey = pCsr->aTree[1]; - int i; - - assert( pCsr->flags & CURSOR_NEXT_OK ); - if( pCsr->flags & CURSOR_IGNORE_DELETE ){ - /* The ignore-delete flag is set when the output of the merge will form - ** the oldest level in the database. In this case there is no point in - ** retaining any range-delete flags. */ - assert( (f & LSM_POINT_DELETE)==0 ); - f &= ~(LSM_START_DELETE|LSM_END_DELETE); - }else{ - for(i=0; i<(CURSOR_DATA_SEGMENT + pCsr->nPtr); i++){ - if( i!=iKey ){ - int eType; - void *pKey; - int nKey; - int res; - multiCursorGetKey(pCsr, i, &eType, &pKey, &nKey); - - if( pKey ){ - res = sortedKeyCompare(pCsr->pDb->xCmp, - rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, - rtTopic(eType), pKey, nKey - ); - assert( res<=0 ); - if( res==0 ){ - if( (f & (LSM_INSERT|LSM_POINT_DELETE))==0 ){ - if( eType & LSM_INSERT ){ - f |= LSM_INSERT; - *piVal = i; - } - else if( eType & LSM_POINT_DELETE ){ - f |= LSM_POINT_DELETE; - } - } - f |= (eType & (LSM_END_DELETE|LSM_START_DELETE)); - } - - if( i>iKey && (eType & LSM_END_DELETE) && res<0 ){ - if( f & (LSM_INSERT|LSM_POINT_DELETE) ){ - f |= (LSM_END_DELETE|LSM_START_DELETE); - }else{ - f = 0; - } - break; - } - } - } - } - - assert( (f & LSM_INSERT)==0 || (f & LSM_POINT_DELETE)==0 ); - if( (f & LSM_START_DELETE) - && (f & LSM_END_DELETE) - && (f & LSM_POINT_DELETE ) - ){ - f = 0; - } - } - - *piFlags = f; -} - -static int mergeWorkerStep(MergeWorker *pMW){ - lsm_db *pDb = pMW->pDb; /* Database handle */ - MultiCursor *pCsr; /* Cursor to read input data from */ - int rc = LSM_OK; /* Return code */ - int eType; /* SORTED_SEPARATOR, WRITE or DELETE */ - void *pKey; int nKey; /* Key */ - LsmPgno iPtr; - int iVal; - - pCsr = pMW->pCsr; - - /* Pull the next record out of the source cursor. */ - lsmMCursorKey(pCsr, &pKey, &nKey); - eType = pCsr->eType; - - /* Figure out if the output record may have a different pointer value - ** than the previous. This is the case if the current key is identical to - ** a key that appears in the lowest level run being merged. If so, set - ** iPtr to the absolute pointer value. If not, leave iPtr set to zero, - ** indicating that the output pointer value should be a copy of the pointer - ** value written with the previous key. */ - iPtr = (pCsr->pPrevMergePtr ? *pCsr->pPrevMergePtr : 0); - if( pCsr->pBtCsr ){ - BtreeCursor *pBtCsr = pCsr->pBtCsr; - if( pBtCsr->pKey ){ - int res = rtTopic(pBtCsr->eType) - rtTopic(eType); - if( res==0 ) res = pDb->xCmp(pBtCsr->pKey, pBtCsr->nKey, pKey, nKey); - if( 0==res ) iPtr = pBtCsr->iPtr; - assert( res>=0 ); - } - }else if( pCsr->nPtr ){ - SegmentPtr *pPtr = &pCsr->aPtr[pCsr->nPtr-1]; - if( pPtr->pPg - && 0==pDb->xCmp(pPtr->pKey, pPtr->nKey, pKey, nKey) - ){ - iPtr = pPtr->iPtr+pPtr->iPgPtr; - } - } - - iVal = pCsr->aTree[1]; - mergeRangeDeletes(pCsr, &iVal, &eType); - - if( eType!=0 ){ - if( pMW->aGobble ){ - int iGobble = pCsr->aTree[1] - CURSOR_DATA_SEGMENT; - if( iGobblenPtr && iGobble>=0 ){ - SegmentPtr *pGobble = &pCsr->aPtr[iGobble]; - if( (pGobble->flags & PGFTR_SKIP_THIS_FLAG)==0 ){ - pMW->aGobble[iGobble] = lsmFsPageNumber(pGobble->pPg); - } - } - } - - /* If this is a separator key and we know that the output pointer has not - ** changed, there is no point in writing an output record. Otherwise, - ** proceed. */ - if( rc==LSM_OK && (rtIsSeparator(eType)==0 || iPtr!=0) ){ - /* Write the record into the main run. */ - void *pVal; int nVal; - rc = multiCursorGetVal(pCsr, iVal, &pVal, &nVal); - if( pVal && rc==LSM_OK ){ - assert( nVal>=0 ); - rc = sortedBlobSet(pDb->pEnv, &pCsr->val, pVal, nVal); - pVal = pCsr->val.pData; - } - if( rc==LSM_OK ){ - rc = mergeWorkerWrite(pMW, eType, pKey, nKey, pVal, nVal, iPtr); - } - } - } - - /* Advance the cursor to the next input record (assuming one exists). */ - assert( lsmMCursorValid(pMW->pCsr) ); - if( rc==LSM_OK ) rc = lsmMCursorNext(pMW->pCsr); - - return rc; -} - -static int mergeWorkerDone(MergeWorker *pMW){ - return pMW->pCsr==0 || !lsmMCursorValid(pMW->pCsr); -} - -static void sortedFreeLevel(lsm_env *pEnv, Level *p){ - if( p ){ - lsmFree(pEnv, p->pSplitKey); - lsmFree(pEnv, p->pMerge); - lsmFree(pEnv, p->aRhs); - lsmFree(pEnv, p); - } -} - -static void sortedInvokeWorkHook(lsm_db *pDb){ - if( pDb->xWork ){ - pDb->xWork(pDb, pDb->pWorkCtx); - } -} - -static int sortedNewToplevel( - lsm_db *pDb, /* Connection handle */ - int eTree, /* One of the TREE_XXX constants */ - int *pnWrite /* OUT: Number of database pages written */ -){ - int rc = LSM_OK; /* Return Code */ - MultiCursor *pCsr = 0; - Level *pNext = 0; /* The current top level */ - Level *pNew; /* The new level itself */ - Segment *pLinked = 0; /* Delete separators from this segment */ - Level *pDel = 0; /* Delete this entire level */ - int nWrite = 0; /* Number of database pages written */ - Freelist freelist; - - if( eTree!=TREE_NONE ){ - rc = lsmShmCacheChunks(pDb, pDb->treehdr.nChunk); - } - - assert( pDb->bUseFreelist==0 ); - pDb->pFreelist = &freelist; - pDb->bUseFreelist = 1; - memset(&freelist, 0, sizeof(freelist)); - - /* Allocate the new level structure to write to. */ - pNext = lsmDbSnapshotLevel(pDb->pWorker); - pNew = (Level *)lsmMallocZeroRc(pDb->pEnv, sizeof(Level), &rc); - if( pNew ){ - pNew->pNext = pNext; - lsmDbSnapshotSetLevel(pDb->pWorker, pNew); - } - - /* Create a cursor to gather the data required by the new segment. The new - ** segment contains everything in the tree and pointers to the next segment - ** in the database (if any). */ - pCsr = multiCursorNew(pDb, &rc); - if( pCsr ){ - pCsr->pDb = pDb; - rc = multiCursorVisitFreelist(pCsr); - if( rc==LSM_OK ){ - rc = multiCursorAddTree(pCsr, pDb->pWorker, eTree); - } - if( rc==LSM_OK && pNext && pNext->pMerge==0 ){ - if( (pNext->flags & LEVEL_FREELIST_ONLY) ){ - pDel = pNext; - pCsr->aPtr = lsmMallocZeroRc(pDb->pEnv, sizeof(SegmentPtr), &rc); - multiCursorAddOne(pCsr, pNext, &rc); - }else if( eTree!=TREE_NONE && pNext->lhs.iRoot ){ - pLinked = &pNext->lhs; - rc = btreeCursorNew(pDb, pLinked, &pCsr->pBtCsr); - } - } - - /* If this will be the only segment in the database, discard any delete - ** markers present in the in-memory tree. */ - if( pNext==0 ){ - multiCursorIgnoreDelete(pCsr); - } - } - - if( rc!=LSM_OK ){ - lsmMCursorClose(pCsr, 0); - }else{ - LsmPgno iLeftPtr = 0; - Merge merge; /* Merge object used to create new level */ - MergeWorker mergeworker; /* MergeWorker object for the same purpose */ - - memset(&merge, 0, sizeof(Merge)); - memset(&mergeworker, 0, sizeof(MergeWorker)); - - pNew->pMerge = &merge; - pNew->flags |= LEVEL_INCOMPLETE; - mergeworker.pDb = pDb; - mergeworker.pLevel = pNew; - mergeworker.pCsr = pCsr; - pCsr->pPrevMergePtr = &iLeftPtr; - - /* Mark the separators array for the new level as a "phantom". */ - mergeworker.bFlush = 1; - - /* Do the work to create the new merged segment on disk */ - if( rc==LSM_OK ) rc = lsmMCursorFirst(pCsr); - while( rc==LSM_OK && mergeWorkerDone(&mergeworker)==0 ){ - rc = mergeWorkerStep(&mergeworker); - } - mergeWorkerShutdown(&mergeworker, &rc); - assert( rc!=LSM_OK || mergeworker.nWork==0 || pNew->lhs.iFirst ); - if( rc==LSM_OK && pNew->lhs.iFirst ){ - rc = lsmFsSortedFinish(pDb->pFS, &pNew->lhs); - } - nWrite = mergeworker.nWork; - pNew->flags &= ~LEVEL_INCOMPLETE; - if( eTree==TREE_NONE ){ - pNew->flags |= LEVEL_FREELIST_ONLY; - } - pNew->pMerge = 0; - } - - if( rc!=LSM_OK || pNew->lhs.iFirst==0 ){ - assert( rc!=LSM_OK || pDb->pWorker->freelist.nEntry==0 ); - lsmDbSnapshotSetLevel(pDb->pWorker, pNext); - sortedFreeLevel(pDb->pEnv, pNew); - }else{ - if( pLinked ){ - pLinked->iRoot = 0; - }else if( pDel ){ - assert( pNew->pNext==pDel ); - pNew->pNext = pDel->pNext; - lsmFsSortedDelete(pDb->pFS, pDb->pWorker, 1, &pDel->lhs); - sortedFreeLevel(pDb->pEnv, pDel); - } - -#if LSM_LOG_STRUCTURE - lsmSortedDumpStructure(pDb, pDb->pWorker, LSM_LOG_DATA, 0, "new-toplevel"); -#endif - - if( freelist.nEntry ){ - Freelist *p = &pDb->pWorker->freelist; - lsmFree(pDb->pEnv, p->aEntry); - memcpy(p, &freelist, sizeof(freelist)); - freelist.aEntry = 0; - }else{ - pDb->pWorker->freelist.nEntry = 0; - } - - assertBtreeOk(pDb, &pNew->lhs); - sortedInvokeWorkHook(pDb); - } - - if( pnWrite ) *pnWrite = nWrite; - pDb->pWorker->nWrite += nWrite; - pDb->pFreelist = 0; - pDb->bUseFreelist = 0; - lsmFree(pDb->pEnv, freelist.aEntry); - return rc; -} - -/* -** The nMerge levels in the LSM beginning with pLevel consist of a -** left-hand-side segment only. Replace these levels with a single new -** level consisting of a new empty segment on the left-hand-side and the -** nMerge segments from the replaced levels on the right-hand-side. -** -** Also, allocate and populate a Merge object and set Level.pMerge to -** point to it. -*/ -static int sortedMergeSetup( - lsm_db *pDb, /* Database handle */ - Level *pLevel, /* First level to merge */ - int nMerge, /* Merge this many levels together */ - Level **ppNew /* New, merged, level */ -){ - int rc = LSM_OK; /* Return Code */ - Level *pNew; /* New Level object */ - int bUseNext = 0; /* True to link in next separators */ - Merge *pMerge; /* New Merge object */ - int nByte; /* Bytes of space allocated at pMerge */ - -#ifdef LSM_DEBUG - int iLevel; - Level *pX = pLevel; - for(iLevel=0; iLevelnRight==0 ); - pX = pX->pNext; - } -#endif - - /* Allocate the new Level object */ - pNew = (Level *)lsmMallocZeroRc(pDb->pEnv, sizeof(Level), &rc); - if( pNew ){ - pNew->aRhs = (Segment *)lsmMallocZeroRc(pDb->pEnv, - nMerge * sizeof(Segment), &rc); - } - - /* Populate the new Level object */ - if( rc==LSM_OK ){ - Level *pNext = 0; /* Level following pNew */ - int i; - int bFreeOnly = 1; - Level *pTopLevel; - Level *p = pLevel; - Level **pp; - pNew->nRight = nMerge; - pNew->iAge = pLevel->iAge+1; - for(i=0; inRight==0 ); - pNext = p->pNext; - pNew->aRhs[i] = p->lhs; - if( (p->flags & LEVEL_FREELIST_ONLY)==0 ) bFreeOnly = 0; - sortedFreeLevel(pDb->pEnv, p); - p = pNext; - } - - if( bFreeOnly ) pNew->flags |= LEVEL_FREELIST_ONLY; - - /* Replace the old levels with the new. */ - pTopLevel = lsmDbSnapshotLevel(pDb->pWorker); - pNew->pNext = p; - for(pp=&pTopLevel; *pp!=pLevel; pp=&((*pp)->pNext)); - *pp = pNew; - lsmDbSnapshotSetLevel(pDb->pWorker, pTopLevel); - - /* Determine whether or not the next separators will be linked in */ - if( pNext && pNext->pMerge==0 && pNext->lhs.iRoot && pNext - && (bFreeOnly==0 || (pNext->flags & LEVEL_FREELIST_ONLY)) - ){ - bUseNext = 1; - } - } - - /* Allocate the merge object */ - nByte = sizeof(Merge) + sizeof(MergeInput) * (nMerge + bUseNext); - pMerge = (Merge *)lsmMallocZeroRc(pDb->pEnv, nByte, &rc); - if( pMerge ){ - pMerge->aInput = (MergeInput *)&pMerge[1]; - pMerge->nInput = nMerge + bUseNext; - pNew->pMerge = pMerge; - } - - *ppNew = pNew; - return rc; -} - -static int mergeWorkerInit( - lsm_db *pDb, /* Db connection to do merge work */ - Level *pLevel, /* Level to work on merging */ - MergeWorker *pMW /* Object to initialize */ -){ - int rc = LSM_OK; /* Return code */ - Merge *pMerge = pLevel->pMerge; /* Persistent part of merge state */ - MultiCursor *pCsr = 0; /* Cursor opened for pMW */ - Level *pNext = pLevel->pNext; /* Next level in LSM */ - - assert( pDb->pWorker ); - assert( pLevel->pMerge ); - assert( pLevel->nRight>0 ); - - memset(pMW, 0, sizeof(MergeWorker)); - pMW->pDb = pDb; - pMW->pLevel = pLevel; - pMW->aGobble = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*pLevel->nRight,&rc); - - /* Create a multi-cursor to read the data to write to the new - ** segment. The new segment contains: - ** - ** 1. Records from LHS of each of the nMerge levels being merged. - ** 2. Separators from either the last level being merged, or the - ** separators attached to the LHS of the following level, or neither. - ** - ** If the new level is the lowest (oldest) in the db, discard any - ** delete keys. Key annihilation. - */ - pCsr = multiCursorNew(pDb, &rc); - if( pCsr ){ - pCsr->flags |= CURSOR_NEXT_OK; - rc = multiCursorAddRhs(pCsr, pLevel); - } - if( rc==LSM_OK && pMerge->nInput > pLevel->nRight ){ - rc = btreeCursorNew(pDb, &pNext->lhs, &pCsr->pBtCsr); - }else if( pNext ){ - multiCursorReadSeparators(pCsr); - }else{ - multiCursorIgnoreDelete(pCsr); - } - - assert( rc!=LSM_OK || pMerge->nInput==(pCsr->nPtr+(pCsr->pBtCsr!=0)) ); - pMW->pCsr = pCsr; - - /* Load the b-tree hierarchy into memory. */ - if( rc==LSM_OK ) rc = mergeWorkerLoadHierarchy(pMW); - if( rc==LSM_OK && pMW->hier.nHier==0 ){ - pMW->aSave[0].iPgno = pLevel->lhs.iFirst; - } - - /* Position the cursor. */ - if( rc==LSM_OK ){ - pCsr->pPrevMergePtr = &pMerge->iCurrentPtr; - if( pLevel->lhs.iFirst==0 ){ - /* The output array is still empty. So position the cursor at the very - ** start of the input. */ - rc = multiCursorEnd(pCsr, 0); - }else{ - /* The output array is non-empty. Position the cursor based on the - ** page/cell data saved in the Merge.aInput[] array. */ - int i; - for(i=0; rc==LSM_OK && inPtr; i++){ - MergeInput *pInput = &pMerge->aInput[i]; - if( pInput->iPg ){ - SegmentPtr *pPtr; - assert( pCsr->aPtr[i].pPg==0 ); - pPtr = &pCsr->aPtr[i]; - rc = segmentPtrLoadPage(pDb->pFS, pPtr, pInput->iPg); - if( rc==LSM_OK && pPtr->nCell>0 ){ - rc = segmentPtrLoadCell(pPtr, pInput->iCell); - } - } - } - - if( rc==LSM_OK && pCsr->pBtCsr ){ - int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp; - assert( i==pCsr->nPtr ); - rc = btreeCursorRestore(pCsr->pBtCsr, xCmp, &pMerge->aInput[i]); - } - - if( rc==LSM_OK ){ - rc = multiCursorSetupTree(pCsr, 0); - } - } - pCsr->flags |= CURSOR_NEXT_OK; - } - - return rc; -} - -static int sortedBtreeGobble( - lsm_db *pDb, /* Worker connection */ - MultiCursor *pCsr, /* Multi-cursor being used for a merge */ - int iGobble /* pCsr->aPtr[] entry to operate on */ -){ - int rc = LSM_OK; - if( rtTopic(pCsr->eType)==0 ){ - Segment *pSeg = pCsr->aPtr[iGobble].pSeg; - LsmPgno *aPg; - int nPg; - - /* Seek from the root of the b-tree to the segment leaf that may contain - ** a key equal to the one multi-cursor currently points to. Record the - ** page number of each b-tree page and the leaf. The segment may be - ** gobbled up to (but not including) the first of these page numbers. - */ - assert( pSeg->iRoot>0 ); - aPg = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*32, &rc); - if( rc==LSM_OK ){ - rc = seekInBtree(pCsr, pSeg, - rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, aPg, 0 - ); - } - - if( rc==LSM_OK ){ - for(nPg=0; aPg[nPg]; nPg++); - lsmFsGobble(pDb, pSeg, aPg, nPg); - } - - lsmFree(pDb->pEnv, aPg); - } - return rc; -} - -/* -** Argument p points to a level of age N. Return the number of levels in -** the linked list starting at p that have age=N (always at least 1). -*/ -static int sortedCountLevels(Level *p){ - int iAge = p->iAge; - int nRet = 0; - do { - nRet++; - p = p->pNext; - }while( p && p->iAge==iAge ); - return nRet; -} - -static int sortedSelectLevel(lsm_db *pDb, int nMerge, Level **ppOut){ - Level *pTopLevel = lsmDbSnapshotLevel(pDb->pWorker); - int rc = LSM_OK; - Level *pLevel = 0; /* Output value */ - Level *pBest = 0; /* Best level to work on found so far */ - int nBest; /* Number of segments merged at pBest */ - Level *pThis = 0; /* First in run of levels with age=iAge */ - int nThis = 0; /* Number of levels starting at pThis */ - - assert( nMerge>=1 ); - nBest = LSM_MAX(1, nMerge-1); - - /* Find the longest contiguous run of levels not currently undergoing a - ** merge with the same age in the structure. Or the level being merged - ** with the largest number of right-hand segments. Work on it. */ - for(pLevel=pTopLevel; pLevel; pLevel=pLevel->pNext){ - if( pLevel->nRight==0 && pThis && pLevel->iAge==pThis->iAge ){ - nThis++; - }else{ - if( nThis>nBest ){ - if( (pLevel->iAge!=pThis->iAge+1) - || (pLevel->nRight==0 && sortedCountLevels(pLevel)<=pDb->nMerge) - ){ - pBest = pThis; - nBest = nThis; - } - } - if( pLevel->nRight ){ - if( pLevel->nRight>nBest ){ - nBest = pLevel->nRight; - pBest = pLevel; - } - nThis = 0; - pThis = 0; - }else{ - pThis = pLevel; - nThis = 1; - } - } - } - if( nThis>nBest ){ - assert( pThis ); - pBest = pThis; - nBest = nThis; - } - - if( pBest==0 && nMerge==1 ){ - int nFree = 0; - int nUsr = 0; - for(pLevel=pTopLevel; pLevel; pLevel=pLevel->pNext){ - assert( !pLevel->nRight ); - if( pLevel->flags & LEVEL_FREELIST_ONLY ){ - nFree++; - }else{ - nUsr++; - } - } - if( nUsr>1 ){ - pBest = pTopLevel; - nBest = nFree + nUsr; - } - } - - if( pBest ){ - if( pBest->nRight==0 ){ - rc = sortedMergeSetup(pDb, pBest, nBest, ppOut); - }else{ - *ppOut = pBest; - } - } - - return rc; -} - -static int sortedDbIsFull(lsm_db *pDb){ - Level *pTop = lsmDbSnapshotLevel(pDb->pWorker); - - if( lsmDatabaseFull(pDb) ) return 1; - if( pTop && pTop->iAge==0 - && (pTop->nRight || sortedCountLevels(pTop)>=pDb->nMerge) - ){ - return 1; - } - return 0; -} - -typedef struct MoveBlockCtx MoveBlockCtx; -struct MoveBlockCtx { - int iSeen; /* Previous free block on list */ - int iFrom; /* Total number of blocks in file */ -}; - -static int moveBlockCb(void *pCtx, int iBlk, i64 iSnapshot){ - MoveBlockCtx *p = (MoveBlockCtx *)pCtx; - assert( p->iFrom==0 ); - if( iBlk==(p->iSeen-1) ){ - p->iSeen = iBlk; - return 0; - } - p->iFrom = p->iSeen-1; - return 1; -} - -/* -** This function is called to further compact a database for which all -** of the content has already been merged into a single segment. If -** possible, it moves the contents of a single block from the end of the -** file to a free-block that lies closer to the start of the file (allowing -** the file to be eventually truncated). -*/ -static int sortedMoveBlock(lsm_db *pDb, int *pnWrite){ - Snapshot *p = pDb->pWorker; - Level *pLvl = lsmDbSnapshotLevel(p); - int iFrom; /* Block to move */ - int iTo; /* Destination to move block to */ - int rc; /* Return code */ - - MoveBlockCtx sCtx; - - assert( pLvl->pNext==0 && pLvl->nRight==0 ); - assert( p->redirect.n<=LSM_MAX_BLOCK_REDIRECTS ); - - *pnWrite = 0; - - /* Check that the redirect array is not already full. If it is, return - ** without moving any database content. */ - if( p->redirect.n>=LSM_MAX_BLOCK_REDIRECTS ) return LSM_OK; - - /* Find the last block of content in the database file. Do this by - ** traversing the free-list in reverse (descending block number) order. - ** The first block not on the free list is the one that will be moved. - ** Since the db consists of a single segment, there is no ambiguity as - ** to which segment the block belongs to. */ - sCtx.iSeen = p->nBlock+1; - sCtx.iFrom = 0; - rc = lsmWalkFreelist(pDb, 1, moveBlockCb, &sCtx); - if( rc!=LSM_OK || sCtx.iFrom==0 ) return rc; - iFrom = sCtx.iFrom; - - /* Find the first free block in the database, ignoring block 1. Block - ** 1 is tricky as it is smaller than the other blocks. */ - rc = lsmBlockAllocate(pDb, iFrom, &iTo); - if( rc!=LSM_OK || iTo==0 ) return rc; - assert( iTo!=1 && iTopFS, &pLvl->lhs, iTo, iFrom); - if( rc==LSM_OK ){ - if( p->redirect.a==0 ){ - int nByte = sizeof(struct RedirectEntry) * LSM_MAX_BLOCK_REDIRECTS; - p->redirect.a = lsmMallocZeroRc(pDb->pEnv, nByte, &rc); - } - if( rc==LSM_OK ){ - - /* Check if the block just moved was already redirected. */ - int i; - for(i=0; iredirect.n; i++){ - if( p->redirect.a[i].iTo==iFrom ) break; - } - - if( i==p->redirect.n ){ - /* Block iFrom was not already redirected. Add a new array entry. */ - memmove(&p->redirect.a[1], &p->redirect.a[0], - sizeof(struct RedirectEntry) * p->redirect.n - ); - p->redirect.a[0].iFrom = iFrom; - p->redirect.a[0].iTo = iTo; - p->redirect.n++; - }else{ - /* Block iFrom was already redirected. Overwrite existing entry. */ - p->redirect.a[i].iTo = iTo; - } - - rc = lsmBlockFree(pDb, iFrom); - - *pnWrite = lsmFsBlockSize(pDb->pFS) / lsmFsPageSize(pDb->pFS); - pLvl->lhs.pRedirect = &p->redirect; - } - } - -#if LSM_LOG_STRUCTURE - if( rc==LSM_OK ){ - char aBuf[64]; - sprintf(aBuf, "move-block %d/%d", p->redirect.n-1, LSM_MAX_BLOCK_REDIRECTS); - lsmSortedDumpStructure(pDb, pDb->pWorker, LSM_LOG_DATA, 0, aBuf); - } -#endif - return rc; -} - -/* -*/ -static int mergeInsertFreelistSegments( - lsm_db *pDb, - int nFree, - MergeWorker *pMW -){ - int rc = LSM_OK; - if( nFree>0 ){ - MultiCursor *pCsr = pMW->pCsr; - Level *pLvl = pMW->pLevel; - SegmentPtr *aNew1; - Segment *aNew2; - - Level *pIter; - Level *pNext; - int i = 0; - - aNew1 = (SegmentPtr *)lsmMallocZeroRc( - pDb->pEnv, sizeof(SegmentPtr) * (pCsr->nPtr+nFree), &rc - ); - if( rc ) return rc; - memcpy(&aNew1[nFree], pCsr->aPtr, sizeof(SegmentPtr)*pCsr->nPtr); - pCsr->nPtr += nFree; - lsmFree(pDb->pEnv, pCsr->aTree); - lsmFree(pDb->pEnv, pCsr->aPtr); - pCsr->aTree = 0; - pCsr->aPtr = aNew1; - - aNew2 = (Segment *)lsmMallocZeroRc( - pDb->pEnv, sizeof(Segment) * (pLvl->nRight+nFree), &rc - ); - if( rc ) return rc; - memcpy(&aNew2[nFree], pLvl->aRhs, sizeof(Segment)*pLvl->nRight); - pLvl->nRight += nFree; - lsmFree(pDb->pEnv, pLvl->aRhs); - pLvl->aRhs = aNew2; - - for(pIter=pDb->pWorker->pLevel; rc==LSM_OK && pIter!=pLvl; pIter=pNext){ - Segment *pSeg = &pLvl->aRhs[i]; - memcpy(pSeg, &pIter->lhs, sizeof(Segment)); - - pCsr->aPtr[i].pSeg = pSeg; - pCsr->aPtr[i].pLevel = pLvl; - rc = segmentPtrEnd(pCsr, &pCsr->aPtr[i], 0); - - pDb->pWorker->pLevel = pNext = pIter->pNext; - sortedFreeLevel(pDb->pEnv, pIter); - i++; - } - assert( i==nFree ); - assert( rc!=LSM_OK || pDb->pWorker->pLevel==pLvl ); - - for(i=nFree; inPtr; i++){ - pCsr->aPtr[i].pSeg = &pLvl->aRhs[i]; - } - - lsmFree(pDb->pEnv, pMW->aGobble); - pMW->aGobble = 0; - } - return rc; -} - -static int sortedWork( - lsm_db *pDb, /* Database handle. Must be worker. */ - int nWork, /* Number of pages of work to do */ - int nMerge, /* Try to merge this many levels at once */ - int bFlush, /* Set if call is to make room for a flush */ - int *pnWrite /* OUT: Actual number of pages written */ -){ - int rc = LSM_OK; /* Return Code */ - int nRemaining = nWork; /* Units of work to do before returning */ - Snapshot *pWorker = pDb->pWorker; - - assert( pWorker ); - if( lsmDbSnapshotLevel(pWorker)==0 ) return LSM_OK; - - while( nRemaining>0 ){ - Level *pLevel = 0; - - /* Find a level to work on. */ - rc = sortedSelectLevel(pDb, nMerge, &pLevel); - assert( rc==LSM_OK || pLevel==0 ); - - if( pLevel==0 ){ - int nDone = 0; - Level *pTopLevel = lsmDbSnapshotLevel(pDb->pWorker); - if( bFlush==0 && nMerge==1 && pTopLevel && pTopLevel->pNext==0 ){ - rc = sortedMoveBlock(pDb, &nDone); - } - nRemaining -= nDone; - - /* Could not find any work to do. Finished. */ - if( nDone==0 ) break; - }else{ - int bSave = 0; - Freelist freelist = {0, 0, 0}; - MergeWorker mergeworker; /* State used to work on the level merge */ - - assert( pDb->bIncrMerge==0 ); - assert( pDb->pFreelist==0 && pDb->bUseFreelist==0 ); - - pDb->bIncrMerge = 1; - rc = mergeWorkerInit(pDb, pLevel, &mergeworker); - assert( mergeworker.nWork==0 ); - - while( rc==LSM_OK - && 0==mergeWorkerDone(&mergeworker) - && (mergeworker.nWorkbUseFreelist) - ){ - int eType = rtTopic(mergeworker.pCsr->eType); - rc = mergeWorkerStep(&mergeworker); - - /* If the cursor now points at the first entry past the end of the - ** user data (i.e. either to EOF or to the first free-list entry - ** that will be added to the run), then check if it is possible to - ** merge in any free-list entries that are either in-memory or in - ** free-list-only blocks. */ - if( rc==LSM_OK && nMerge==1 && eType==0 - && (rtTopic(mergeworker.pCsr->eType) || mergeWorkerDone(&mergeworker)) - ){ - int nFree = 0; /* Number of free-list-only levels to merge */ - Level *pLvl; - assert( pDb->pFreelist==0 && pDb->bUseFreelist==0 ); - - /* Now check if all levels containing data newer than this one - ** are single-segment free-list only levels. If so, they will be - ** merged in now. */ - for(pLvl=pDb->pWorker->pLevel; - pLvl!=mergeworker.pLevel && (pLvl->flags & LEVEL_FREELIST_ONLY); - pLvl=pLvl->pNext - ){ - assert( pLvl->nRight==0 ); - nFree++; - } - if( pLvl==mergeworker.pLevel ){ - - rc = mergeInsertFreelistSegments(pDb, nFree, &mergeworker); - if( rc==LSM_OK ){ - rc = multiCursorVisitFreelist(mergeworker.pCsr); - } - if( rc==LSM_OK ){ - rc = multiCursorSetupTree(mergeworker.pCsr, 0); - pDb->pFreelist = &freelist; - pDb->bUseFreelist = 1; - } - } - } - } - nRemaining -= LSM_MAX(mergeworker.nWork, 1); - - if( rc==LSM_OK ){ - /* Check if the merge operation is completely finished. If not, - ** gobble up (declare eligible for recycling) any pages from rhs - ** segments for which the content has been completely merged into - ** the lhs of the level. */ - if( mergeWorkerDone(&mergeworker)==0 ){ - int i; - for(i=0; inRight; i++){ - SegmentPtr *pGobble = &mergeworker.pCsr->aPtr[i]; - if( pGobble->pSeg->iRoot ){ - rc = sortedBtreeGobble(pDb, mergeworker.pCsr, i); - }else if( mergeworker.aGobble[i] ){ - lsmFsGobble(pDb, pGobble->pSeg, &mergeworker.aGobble[i], 1); - } - } - }else{ - int i; - int bEmpty; - mergeWorkerShutdown(&mergeworker, &rc); - bEmpty = (pLevel->lhs.iFirst==0); - - if( bEmpty==0 && rc==LSM_OK ){ - rc = lsmFsSortedFinish(pDb->pFS, &pLevel->lhs); - } - - if( pDb->bUseFreelist ){ - Freelist *p = &pDb->pWorker->freelist; - lsmFree(pDb->pEnv, p->aEntry); - memcpy(p, &freelist, sizeof(freelist)); - pDb->bUseFreelist = 0; - pDb->pFreelist = 0; - bSave = 1; - } - - for(i=0; inRight; i++){ - lsmFsSortedDelete(pDb->pFS, pWorker, 1, &pLevel->aRhs[i]); - } - - if( bEmpty ){ - /* If the new level is completely empty, remove it from the - ** database snapshot. This can only happen if all input keys were - ** annihilated. Since keys are only annihilated if the new level - ** is the last in the linked list (contains the most ancient of - ** database content), this guarantees that pLevel->pNext==0. */ - Level *pTop; /* Top level of worker snapshot */ - Level **pp; /* Read/write iterator for Level.pNext list */ - - assert( pLevel->pNext==0 ); - - /* Remove the level from the worker snapshot. */ - pTop = lsmDbSnapshotLevel(pWorker); - for(pp=&pTop; *pp!=pLevel; pp=&((*pp)->pNext)); - *pp = pLevel->pNext; - lsmDbSnapshotSetLevel(pWorker, pTop); - - /* Free the Level structure. */ - sortedFreeLevel(pDb->pEnv, pLevel); - }else{ - - /* Free the separators of the next level, if required. */ - if( pLevel->pMerge->nInput > pLevel->nRight ){ - assert( pLevel->pNext->lhs.iRoot ); - pLevel->pNext->lhs.iRoot = 0; - } - - /* Zero the right-hand-side of pLevel */ - lsmFree(pDb->pEnv, pLevel->aRhs); - pLevel->nRight = 0; - pLevel->aRhs = 0; - - /* Free the Merge object */ - lsmFree(pDb->pEnv, pLevel->pMerge); - pLevel->pMerge = 0; - } - - if( bSave && rc==LSM_OK ){ - pDb->bIncrMerge = 0; - rc = lsmSaveWorker(pDb, 0); - } - } - } - - /* Clean up the MergeWorker object initialized above. If no error - ** has occurred, invoke the work-hook to inform the application that - ** the database structure has changed. */ - mergeWorkerShutdown(&mergeworker, &rc); - pDb->bIncrMerge = 0; - if( rc==LSM_OK ) sortedInvokeWorkHook(pDb); - -#if LSM_LOG_STRUCTURE - lsmSortedDumpStructure(pDb, pDb->pWorker, LSM_LOG_DATA, 0, "work"); -#endif - assertBtreeOk(pDb, &pLevel->lhs); - assertRunInOrder(pDb, &pLevel->lhs); - - /* If bFlush is true and the database is no longer considered "full", - ** break out of the loop even if nRemaining is still greater than - ** zero. The caller has an in-memory tree to flush to disk. */ - if( bFlush && sortedDbIsFull(pDb)==0 ) break; - } - } - - if( pnWrite ) *pnWrite = (nWork - nRemaining); - pWorker->nWrite += (nWork - nRemaining); - -#ifdef LSM_LOG_WORK - lsmLogMessage(pDb, rc, "sortedWork(): %d pages", (nWork-nRemaining)); -#endif - return rc; -} - -/* -** The database connection passed as the first argument must be a worker -** connection. This function checks if there exists an "old" in-memory tree -** ready to be flushed to disk. If so, true is returned. Otherwise false. -** -** If an error occurs, *pRc is set to an LSM error code before returning. -** It is assumed that *pRc is set to LSM_OK when this function is called. -*/ -static int sortedTreeHasOld(lsm_db *pDb, int *pRc){ - int rc = LSM_OK; - int bRet = 0; - - assert( pDb->pWorker ); - if( *pRc==LSM_OK ){ - if( rc==LSM_OK - && pDb->treehdr.iOldShmid - && pDb->treehdr.iOldLog!=pDb->pWorker->iLogOff - ){ - bRet = 1; - }else{ - bRet = 0; - } - *pRc = rc; - } - assert( *pRc==LSM_OK || bRet==0 ); - return bRet; -} - -/* -** Create a new free-list only top-level segment. Return LSM_OK if successful -** or an LSM error code if some error occurs. -*/ -static int sortedNewFreelistOnly(lsm_db *pDb){ - return sortedNewToplevel(pDb, TREE_NONE, 0); -} - -int lsmSaveWorker(lsm_db *pDb, int bFlush){ - Snapshot *p = pDb->pWorker; - if( p->freelist.nEntry>pDb->nMaxFreelist ){ - int rc = sortedNewFreelistOnly(pDb); - if( rc!=LSM_OK ) return rc; - } - return lsmCheckpointSaveWorker(pDb, bFlush); -} - -static int doLsmSingleWork( - lsm_db *pDb, - int bShutdown, - int nMerge, /* Minimum segments to merge together */ - int nPage, /* Number of pages to write to disk */ - int *pnWrite, /* OUT: Pages actually written to disk */ - int *pbCkpt /* OUT: True if an auto-checkpoint is req. */ -){ - Snapshot *pWorker; /* Worker snapshot */ - int rc = LSM_OK; /* Return code */ - int bDirty = 0; - int nMax = nPage; /* Maximum pages to write to disk */ - int nRem = nPage; - int bCkpt = 0; - - assert( nPage>0 ); - - /* Open the worker 'transaction'. It will be closed before this function - ** returns. */ - assert( pDb->pWorker==0 ); - rc = lsmBeginWork(pDb); - if( rc!=LSM_OK ) return rc; - pWorker = pDb->pWorker; - - /* If this connection is doing auto-checkpoints, set nMax (and nRem) so - ** that this call stops writing when the auto-checkpoint is due. The - ** caller will do the checkpoint, then possibly call this function again. */ - if( bShutdown==0 && pDb->nAutockpt ){ - u32 nSync; - u32 nUnsync; - int nPgsz; - - lsmCheckpointSynced(pDb, 0, 0, &nSync); - nUnsync = lsmCheckpointNWrite(pDb->pShmhdr->aSnap1, 0); - nPgsz = lsmCheckpointPgsz(pDb->pShmhdr->aSnap1); - - nMax = (int)LSM_MIN(nMax, (pDb->nAutockpt/nPgsz) - (int)(nUnsync-nSync)); - if( nMaxnTransOpen==0 ){ - rc = lsmTreeLoadHeader(pDb, 0); - } - if( sortedTreeHasOld(pDb, &rc) ){ - /* sortedDbIsFull() returns non-zero if either (a) there are too many - ** levels in total in the db, or (b) there are too many levels with the - ** the same age in the db. Either way, call sortedWork() to merge - ** existing segments together until this condition is cleared. */ - if( sortedDbIsFull(pDb) ){ - int nPg = 0; - rc = sortedWork(pDb, nRem, nMerge, 1, &nPg); - nRem -= nPg; - assert( rc!=LSM_OK || nRem<=0 || !sortedDbIsFull(pDb) ); - bDirty = 1; - } - - if( rc==LSM_OK && nRem>0 ){ - int nPg = 0; - rc = sortedNewToplevel(pDb, TREE_OLD, &nPg); - nRem -= nPg; - if( rc==LSM_OK ){ - if( pDb->nTransOpen>0 ){ - lsmTreeDiscardOld(pDb); - } - rc = lsmSaveWorker(pDb, 1); - bDirty = 0; - } - } - } - - /* If nPage is still greater than zero, do some merging. */ - if( rc==LSM_OK && nRem>0 && bShutdown==0 ){ - int nPg = 0; - rc = sortedWork(pDb, nRem, nMerge, 0, &nPg); - nRem -= nPg; - if( nPg ) bDirty = 1; - } - - /* If the in-memory part of the free-list is too large, write a new - ** top-level containing just the in-memory free-list entries to disk. */ - if( rc==LSM_OK && pDb->pWorker->freelist.nEntry > pDb->nMaxFreelist ){ - while( rc==LSM_OK && lsmDatabaseFull(pDb) ){ - int nPg = 0; - rc = sortedWork(pDb, 16, nMerge, 1, &nPg); - nRem -= nPg; - } - if( rc==LSM_OK ){ - rc = sortedNewFreelistOnly(pDb); - } - bDirty = 1; - } - - if( rc==LSM_OK ){ - *pnWrite = (nMax - nRem); - *pbCkpt = (bCkpt && nRem<=0); - if( nMerge==1 && pDb->nAutockpt>0 && *pnWrite>0 - && pWorker->pLevel - && pWorker->pLevel->nRight==0 - && pWorker->pLevel->pNext==0 - ){ - *pbCkpt = 1; - } - } - - if( rc==LSM_OK && bDirty ){ - lsmFinishWork(pDb, 0, &rc); - }else{ - int rcdummy = LSM_BUSY; - lsmFinishWork(pDb, 0, &rcdummy); - *pnWrite = 0; - } - assert( pDb->pWorker==0 ); - return rc; -} - -static int doLsmWork(lsm_db *pDb, int nMerge, int nPage, int *pnWrite){ - int rc = LSM_OK; /* Return code */ - int nWrite = 0; /* Number of pages written */ - - assert( nMerge>=1 ); - - if( nPage!=0 ){ - int bCkpt = 0; - do { - int nThis = 0; - int nReq = (nPage>=0) ? (nPage-nWrite) : ((int)0x7FFFFFFF); - - bCkpt = 0; - rc = doLsmSingleWork(pDb, 0, nMerge, nReq, &nThis, &bCkpt); - nWrite += nThis; - if( rc==LSM_OK && bCkpt ){ - rc = lsm_checkpoint(pDb, 0); - } - }while( rc==LSM_OK && bCkpt && (nWritenTransOpen || pDb->pCsr ) return LSM_MISUSE_BKPT; - if( nMerge<=0 ) nMerge = pDb->nMerge; - - lsmFsPurgeCache(pDb->pFS); - - /* Convert from KB to pages */ - nPgsz = lsmFsPageSize(pDb->pFS); - if( nKB>=0 ){ - nPage = ((i64)nKB * 1024 + nPgsz - 1) / nPgsz; - }else{ - nPage = -1; - } - - rc = doLsmWork(pDb, nMerge, nPage, &nWrite); - - if( pnWrite ){ - /* Convert back from pages to KB */ - *pnWrite = (int)(((i64)nWrite * 1024 + nPgsz - 1) / nPgsz); - } - return rc; -} - -int lsm_flush(lsm_db *db){ - int rc; - - if( db->nTransOpen>0 || db->pCsr ){ - rc = LSM_MISUSE_BKPT; - }else{ - rc = lsmBeginWriteTrans(db); - if( rc==LSM_OK ){ - lsmFlushTreeToDisk(db); - lsmTreeDiscardOld(db); - lsmTreeMakeOld(db); - lsmTreeDiscardOld(db); - } - - if( rc==LSM_OK ){ - rc = lsmFinishWriteTrans(db, 1); - }else{ - lsmFinishWriteTrans(db, 0); - } - lsmFinishReadTrans(db); - } - - return rc; -} - -/* -** This function is called in auto-work mode to perform merging work on -** the data structure. It performs enough merging work to prevent the -** height of the tree from growing indefinitely assuming that roughly -** nUnit database pages worth of data have been written to the database -** (i.e. the in-memory tree) since the last call. -*/ -int lsmSortedAutoWork( - lsm_db *pDb, /* Database handle */ - int nUnit /* Pages of data written to in-memory tree */ -){ - int rc = LSM_OK; /* Return code */ - int nDepth = 0; /* Current height of tree (longest path) */ - Level *pLevel; /* Used to iterate through levels */ - int bRestore = 0; - - assert( pDb->pWorker==0 ); - assert( pDb->nTransOpen>0 ); - - /* Determine how many units of work to do before returning. One unit of - ** work is achieved by writing one page (~4KB) of merged data. */ - for(pLevel=lsmDbSnapshotLevel(pDb->pClient); pLevel; pLevel=pLevel->pNext){ - /* nDepth += LSM_MAX(1, pLevel->nRight); */ - nDepth += 1; - } - if( lsmTreeHasOld(pDb) ){ - nDepth += 1; - bRestore = 1; - rc = lsmSaveCursors(pDb); - if( rc!=LSM_OK ) return rc; - } - - if( nDepth>0 ){ - int nRemaining; /* Units of work to do before returning */ - - nRemaining = nUnit * nDepth; -#ifdef LSM_LOG_WORK - lsmLogMessage(pDb, rc, "lsmSortedAutoWork(): %d*%d = %d pages", - nUnit, nDepth, nRemaining); -#endif - assert( nRemaining>=0 ); - rc = doLsmWork(pDb, pDb->nMerge, nRemaining, 0); - if( rc==LSM_BUSY ) rc = LSM_OK; - - if( bRestore && pDb->pCsr ){ - lsmMCursorFreeCache(pDb); - lsmFreeSnapshot(pDb->pEnv, pDb->pClient); - pDb->pClient = 0; - if( rc==LSM_OK ){ - rc = lsmCheckpointLoad(pDb, 0); - } - if( rc==LSM_OK ){ - rc = lsmCheckpointDeserialize(pDb, 0, pDb->aSnapshot, &pDb->pClient); - } - if( rc==LSM_OK ){ - rc = lsmRestoreCursors(pDb); - } - } - } - - return rc; -} - -/* -** This function is only called during system shutdown. The contents of -** any in-memory trees present (old or current) are written out to disk. -*/ -int lsmFlushTreeToDisk(lsm_db *pDb){ - int rc; - - rc = lsmBeginWork(pDb); - while( rc==LSM_OK && sortedDbIsFull(pDb) ){ - rc = sortedWork(pDb, 256, pDb->nMerge, 1, 0); - } - - if( rc==LSM_OK ){ - rc = sortedNewToplevel(pDb, TREE_BOTH, 0); - } - - lsmFinishWork(pDb, 1, &rc); - return rc; -} - -/* -** Return a string representation of the segment passed as the only argument. -** Space for the returned string is allocated using lsmMalloc(), and should -** be freed by the caller using lsmFree(). -*/ -static char *segToString(lsm_env *pEnv, Segment *pSeg, int nMin){ - LsmPgno nSize = pSeg->nSize; - LsmPgno iRoot = pSeg->iRoot; - LsmPgno iFirst = pSeg->iFirst; - LsmPgno iLast = pSeg->iLastPg; - char *z; - - char *z1; - char *z2; - int nPad; - - z1 = lsmMallocPrintf(pEnv, "%d.%d", iFirst, iLast); - if( iRoot ){ - z2 = lsmMallocPrintf(pEnv, "root=%lld", iRoot); - }else{ - z2 = lsmMallocPrintf(pEnv, "size=%lld", nSize); - } - - nPad = nMin - 2 - strlen(z1) - 1 - strlen(z2); - nPad = LSM_MAX(0, nPad); - - if( iRoot ){ - z = lsmMallocPrintf(pEnv, "/%s %*s%s\\", z1, nPad, "", z2); - }else{ - z = lsmMallocPrintf(pEnv, "|%s %*s%s|", z1, nPad, "", z2); - } - lsmFree(pEnv, z1); - lsmFree(pEnv, z2); - - return z; -} - -static int fileToString( - lsm_db *pDb, /* For xMalloc() */ - char *aBuf, - int nBuf, - int nMin, - Segment *pSeg -){ - int i = 0; - if( pSeg ){ - char *zSeg; - - zSeg = segToString(pDb->pEnv, pSeg, nMin); - snprintf(&aBuf[i], nBuf-i, "%s", zSeg); - i += strlen(&aBuf[i]); - lsmFree(pDb->pEnv, zSeg); - -#ifdef LSM_LOG_FREELIST - lsmInfoArrayStructure(pDb, 1, pSeg->iFirst, &zSeg); - snprintf(&aBuf[i], nBuf-1, " (%s)", zSeg); - i += strlen(&aBuf[i]); - lsmFree(pDb->pEnv, zSeg); -#endif - aBuf[nBuf] = 0; - }else{ - aBuf[0] = '\0'; - } - - return i; -} - -void sortedDumpPage(lsm_db *pDb, Segment *pRun, Page *pPg, int bVals){ - LsmBlob blob = {0, 0, 0}; /* LsmBlob used for keys */ - LsmString s; - int i; - - int nRec; - LsmPgno iPtr; - int flags; - u8 *aData; - int nData; - - aData = fsPageData(pPg, &nData); - - nRec = pageGetNRec(aData, nData); - iPtr = pageGetPtr(aData, nData); - flags = pageGetFlags(aData, nData); - - lsmStringInit(&s, pDb->pEnv); - lsmStringAppendf(&s,"nCell=%d iPtr=%lld flags=%d {", nRec, iPtr, flags); - if( flags&SEGMENT_BTREE_FLAG ) iPtr = 0; - - for(i=0; ipFS, pRun, iRef, &pRef); - aKey = pageGetKey(pRun, pRef, 0, &iTopic, &nKey, &blob); - }else{ - aCell += lsmVarintGet32(aCell, &nKey); - if( rtIsWrite(eType) ) aCell += lsmVarintGet32(aCell, &nVal); - sortedReadData(0, pPg, (aCell-aData), nKey+nVal, (void **)&aKey, &blob); - aVal = &aKey[nKey]; - iTopic = eType; - } - - lsmStringAppendf(&s, "%s%2X:", (i==0?"":" "), iTopic); - for(iChar=0; iChar0 && bVals ){ - lsmStringAppendf(&s, "##"); - for(iChar=0; iCharpFS, pSeg, iRef, &pRef); - pageGetKeyCopy(pDb->pEnv, pSeg, pRef, 0, &dummy, pBlob); - aKey = (u8 *)pBlob->pData; - nKey = pBlob->nData; - lsmFsPageRelease(pRef); - }else{ - aKey = (u8 *)""; - nKey = 11; - } - }else{ - aCell += lsmVarintGet32(aCell, &nKey); - if( rtIsWrite(eType) ) aCell += lsmVarintGet32(aCell, &nVal); - sortedReadData(pSeg, pPg, (aCell-aData), nKey+nVal, (void **)&aKey, pBlob); - aVal = &aKey[nKey]; - } - - if( peType ) *peType = eType; - if( piPgPtr ) *piPgPtr = iPgPtr; - if( paKey ) *paKey = aKey; - if( paVal ) *paVal = aVal; - if( pnKey ) *pnKey = nKey; - if( pnVal ) *pnVal = nVal; -} - -static int infoAppendBlob(LsmString *pStr, int bHex, u8 *z, int n){ - int iChar; - for(iChar=0; iCharpClient || pDb->pWorker ); - pSnap = pDb->pClient; - if( pSnap==0 ) pSnap = pDb->pWorker; - if( pSnap->redirect.n>0 ){ - Level *pLvl; - int bUse = 0; - for(pLvl=pSnap->pLevel; pLvl->pNext; pLvl=pLvl->pNext); - pSeg = (pLvl->nRight==0 ? &pLvl->lhs : &pLvl->aRhs[pLvl->nRight-1]); - rc = lsmFsSegmentContainsPg(pDb->pFS, pSeg, iPg, &bUse); - if( bUse==0 ){ - pSeg = 0; - } - } - - /* iPg is a real page number (not subject to redirection). So it is safe - ** to pass a NULL in place of the segment pointer as the second argument - ** to lsmFsDbPageGet() here. */ - if( rc==LSM_OK ){ - rc = lsmFsDbPageGet(pDb->pFS, 0, iPg, &pPg); - } - - if( rc==LSM_OK ){ - LsmBlob blob = {0, 0, 0, 0}; - int nKeyWidth = 0; - LsmString str; - int nRec; - LsmPgno iPtr; - int flags2; - int iCell; - u8 *aData; int nData; /* Page data and size thereof */ - - aData = fsPageData(pPg, &nData); - nRec = pageGetNRec(aData, nData); - iPtr = pageGetPtr(aData, nData); - flags2 = pageGetFlags(aData, nData); - - lsmStringInit(&str, pDb->pEnv); - lsmStringAppendf(&str, "Page : %lld (%d bytes)\n", iPg, nData); - lsmStringAppendf(&str, "nRec : %d\n", nRec); - lsmStringAppendf(&str, "iPtr : %lld\n", iPtr); - lsmStringAppendf(&str, "flags: %04x\n", flags2); - lsmStringAppendf(&str, "\n"); - - for(iCell=0; iCellnKeyWidth ) nKeyWidth = nKey; - } - if( bHex ) nKeyWidth = nKeyWidth * 2; - - for(iCell=0; iCell0 && bValues ){ - lsmStringAppendf(&str, "%*s", nKeyWidth - (nKey*(1+bHex)), ""); - lsmStringAppendf(&str, " "); - infoAppendBlob(&str, bHex, aVal, nVal); - } - if( rtTopic(eType) ){ - int iBlk = (int)~lsmGetU32(aKey); - lsmStringAppendf(&str, " (block=%d", iBlk); - if( nVal>0 ){ - i64 iSnap = lsmGetU64(aVal); - lsmStringAppendf(&str, " snapshot=%lld", iSnap); - } - lsmStringAppendf(&str, ")"); - } - lsmStringAppendf(&str, "\n"); - } - - if( bData ){ - lsmStringAppendf(&str, "\n-------------------" - "-------------------------------------------------------------\n"); - lsmStringAppendf(&str, "Page %d\n", - iPg, (iPg-1)*nData, iPg*nData - 1); - for(i=0; inData ){ - lsmStringAppendf(&str, " "); - }else{ - lsmStringAppendf(&str, "%02x ", aData[i+j]); - } - } - lsmStringAppendf(&str, " "); - for(j=0; jnData ){ - lsmStringAppendf(&str, " "); - }else{ - lsmStringAppendf(&str,"%c", isprint(aData[i+j]) ? aData[i+j] : '.'); - } - } - lsmStringAppendf(&str,"\n"); - } - } - - *pzOut = str.z; - sortedBlobFree(&blob); - lsmFsPageRelease(pPg); - } - - return rc; -} - -int lsmInfoPageDump( - lsm_db *pDb, /* Database handle */ - LsmPgno iPg, /* Page number of page to dump */ - int bHex, /* True to output key/value in hex form */ - char **pzOut /* OUT: lsmMalloc'd string */ -){ - int flags = INFO_PAGE_DUMP_DATA | INFO_PAGE_DUMP_VALUES; - if( bHex ) flags |= INFO_PAGE_DUMP_HEX; - return infoPageDump(pDb, iPg, flags, pzOut); -} - -void sortedDumpSegment(lsm_db *pDb, Segment *pRun, int bVals){ - assert( pDb->xLog ); - if( pRun && pRun->iFirst ){ - int flags = (bVals ? INFO_PAGE_DUMP_VALUES : 0); - char *zSeg; - Page *pPg; - - zSeg = segToString(pDb->pEnv, pRun, 0); - lsmLogMessage(pDb, LSM_OK, "Segment: %s", zSeg); - lsmFree(pDb->pEnv, zSeg); - - lsmFsDbPageGet(pDb->pFS, pRun, pRun->iFirst, &pPg); - while( pPg ){ - Page *pNext; - char *z = 0; - infoPageDump(pDb, lsmFsPageNumber(pPg), flags, &z); - lsmLogMessage(pDb, LSM_OK, "%s", z); - lsmFree(pDb->pEnv, z); -#if 0 - sortedDumpPage(pDb, pRun, pPg, bVals); -#endif - lsmFsDbPageNext(pRun, pPg, 1, &pNext); - lsmFsPageRelease(pPg); - pPg = pNext; - } - } -} - -/* -** Invoke the log callback zero or more times with messages that describe -** the current database structure. -*/ -void lsmSortedDumpStructure( - lsm_db *pDb, /* Database handle (used for xLog callback) */ - Snapshot *pSnap, /* Snapshot to dump */ - int bKeys, /* Output the keys from each segment */ - int bVals, /* Output the values from each segment */ - const char *zWhy /* Caption to print near top of dump */ -){ - Snapshot *pDump = pSnap; - Level *pTopLevel; - char *zFree = 0; - - assert( pSnap ); - pTopLevel = lsmDbSnapshotLevel(pDump); - if( pDb->xLog && pTopLevel ){ - static int nCall = 0; - Level *pLevel; - int iLevel = 0; - - nCall++; - lsmLogMessage(pDb, LSM_OK, "Database structure %d (%s)", nCall, zWhy); - -#if 0 - if( nCall==1031 || nCall==1032 ) bKeys=1; -#endif - - for(pLevel=pTopLevel; pLevel; pLevel=pLevel->pNext){ - char zLeft[1024]; - char zRight[1024]; - int i = 0; - - Segment *aLeft[24]; - Segment *aRight[24]; - - int nLeft = 0; - int nRight = 0; - - Segment *pSeg = &pLevel->lhs; - aLeft[nLeft++] = pSeg; - - for(i=0; inRight; i++){ - aRight[nRight++] = &pLevel->aRhs[i]; - } - -#ifdef LSM_LOG_FREELIST - if( nRight ){ - memmove(&aRight[1], aRight, sizeof(aRight[0])*nRight); - aRight[0] = 0; - nRight++; - } -#endif - - for(i=0; iiAge, (int)pLevel->flags - ); - }else{ - zLevel[0] = '\0'; - } - - if( nRight==0 ){ - iPad = 10; - } - - lsmLogMessage(pDb, LSM_OK, "% 25s % *s% -35s %s", - zLevel, iPad, "", zLeft, zRight - ); - } - - iLevel++; - } - - if( bKeys ){ - for(pLevel=pTopLevel; pLevel; pLevel=pLevel->pNext){ - int i; - sortedDumpSegment(pDb, &pLevel->lhs, bVals); - for(i=0; inRight; i++){ - sortedDumpSegment(pDb, &pLevel->aRhs[i], bVals); - } - } - } - } - - lsmInfoFreelist(pDb, &zFree); - lsmLogMessage(pDb, LSM_OK, "Freelist: %s", zFree); - lsmFree(pDb->pEnv, zFree); - - assert( lsmFsIntegrityCheck(pDb) ); -} - -void lsmSortedFreeLevel(lsm_env *pEnv, Level *pLevel){ - Level *pNext; - Level *p; - - for(p=pLevel; p; p=pNext){ - pNext = p->pNext; - sortedFreeLevel(pEnv, p); - } -} - -void lsmSortedSaveTreeCursors(lsm_db *pDb){ - MultiCursor *pCsr; - for(pCsr=pDb->pCsr; pCsr; pCsr=pCsr->pNext){ - lsmTreeCursorSave(pCsr->apTreeCsr[0]); - lsmTreeCursorSave(pCsr->apTreeCsr[1]); - } -} - -void lsmSortedExpandBtreePage(Page *pPg, int nOrig){ - u8 *aData; - int nData; - int nEntry; - int iHdr; - - aData = lsmFsPageData(pPg, &nData); - nEntry = pageGetNRec(aData, nOrig); - iHdr = SEGMENT_EOF(nOrig, nEntry); - memmove(&aData[iHdr + (nData-nOrig)], &aData[iHdr], nOrig-iHdr); -} - -#ifdef LSM_DEBUG_EXPENSIVE -static void assertRunInOrder(lsm_db *pDb, Segment *pSeg){ - Page *pPg = 0; - LsmBlob blob1 = {0, 0, 0, 0}; - LsmBlob blob2 = {0, 0, 0, 0}; - - lsmFsDbPageGet(pDb->pFS, pSeg, pSeg->iFirst, &pPg); - while( pPg ){ - u8 *aData; int nData; - Page *pNext; - - aData = lsmFsPageData(pPg, &nData); - if( 0==(pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG) ){ - int i; - int nRec = pageGetNRec(aData, nData); - for(i=0; ipEnv, pSeg, pPg, i, &iTopic1, &blob1); - - if( i==0 && blob2.nData ){ - assert( sortedKeyCompare( - pDb->xCmp, iTopic2, blob2.pData, blob2.nData, - iTopic1, blob1.pData, blob1.nData - )<0 ); - } - - if( i<(nRec-1) ){ - pageGetKeyCopy(pDb->pEnv, pSeg, pPg, i+1, &iTopic2, &blob2); - assert( sortedKeyCompare( - pDb->xCmp, iTopic1, blob1.pData, blob1.nData, - iTopic2, blob2.pData, blob2.nData - )<0 ); - } - } - } - - lsmFsDbPageNext(pSeg, pPg, 1, &pNext); - lsmFsPageRelease(pPg); - pPg = pNext; - } - - sortedBlobFree(&blob1); - sortedBlobFree(&blob2); -} -#endif - -#ifdef LSM_DEBUG_EXPENSIVE -/* -** This function is only included in the build if LSM_DEBUG_EXPENSIVE is -** defined. Its only purpose is to evaluate various assert() statements to -** verify that the database is well formed in certain respects. -** -** More specifically, it checks that the array pOne contains the required -** pointers to pTwo. Array pTwo must be a main array. pOne may be either a -** separators array or another main array. If pOne does not contain the -** correct set of pointers, an assert() statement fails. -*/ -static int assertPointersOk( - lsm_db *pDb, /* Database handle */ - Segment *pOne, /* Segment containing pointers */ - Segment *pTwo, /* Segment containing pointer targets */ - int bRhs /* True if pTwo may have been Gobble()d */ -){ - int rc = LSM_OK; /* Error code */ - SegmentPtr ptr1; /* Iterates through pOne */ - SegmentPtr ptr2; /* Iterates through pTwo */ - LsmPgno iPrev; - - assert( pOne && pTwo ); - - memset(&ptr1, 0, sizeof(ptr1)); - memset(&ptr2, 0, sizeof(ptr1)); - ptr1.pSeg = pOne; - ptr2.pSeg = pTwo; - segmentPtrEndPage(pDb->pFS, &ptr1, 0, &rc); - segmentPtrEndPage(pDb->pFS, &ptr2, 0, &rc); - - /* Check that the footer pointer of the first page of pOne points to - ** the first page of pTwo. */ - iPrev = pTwo->iFirst; - if( ptr1.iPtr!=iPrev && !bRhs ){ - assert( 0 ); - } - - if( rc==LSM_OK && ptr1.nCell>0 ){ - rc = segmentPtrLoadCell(&ptr1, 0); - } - - while( rc==LSM_OK && ptr2.pPg ){ - LsmPgno iThis; - - /* Advance to the next page of segment pTwo that contains at least - ** one cell. Break out of the loop if the iterator reaches EOF. */ - do{ - rc = segmentPtrNextPage(&ptr2, 1); - assert( rc==LSM_OK ); - }while( rc==LSM_OK && ptr2.pPg && ptr2.nCell==0 ); - if( rc!=LSM_OK || ptr2.pPg==0 ) break; - iThis = lsmFsPageNumber(ptr2.pPg); - - if( (ptr2.flags & (PGFTR_SKIP_THIS_FLAG|SEGMENT_BTREE_FLAG))==0 ){ - - /* Load the first cell in the array pTwo page. */ - rc = segmentPtrLoadCell(&ptr2, 0); - - /* Iterate forwards through pOne, searching for a key that matches the - ** key ptr2.pKey/nKey. This key should have a pointer to the page that - ** ptr2 currently points to. */ - while( rc==LSM_OK ){ - int res = rtTopic(ptr1.eType) - rtTopic(ptr2.eType); - if( res==0 ){ - res = pDb->xCmp(ptr1.pKey, ptr1.nKey, ptr2.pKey, ptr2.nKey); - } - - if( res<0 ){ - assert( bRhs || ptr1.iPtr+ptr1.iPgPtr==iPrev ); - }else if( res>0 ){ - assert( 0 ); - }else{ - assert( ptr1.iPtr+ptr1.iPgPtr==iThis ); - iPrev = iThis; - break; - } - - rc = segmentPtrAdvance(0, &ptr1, 0); - if( ptr1.pPg==0 ){ - assert( 0 ); - } - } - } - } - - segmentPtrReset(&ptr1, 0); - segmentPtrReset(&ptr2, 0); - return LSM_OK; -} - -/* -** This function is only included in the build if LSM_DEBUG_EXPENSIVE is -** defined. Its only purpose is to evaluate various assert() statements to -** verify that the database is well formed in certain respects. -** -** More specifically, it checks that the b-tree embedded in array pRun -** contains the correct keys. If not, an assert() fails. -*/ -static int assertBtreeOk( - lsm_db *pDb, - Segment *pSeg -){ - int rc = LSM_OK; /* Return code */ - if( pSeg->iRoot ){ - LsmBlob blob = {0, 0, 0}; /* Buffer used to cache overflow keys */ - FileSystem *pFS = pDb->pFS; /* File system to read from */ - Page *pPg = 0; /* Main run page */ - BtreeCursor *pCsr = 0; /* Btree cursor */ - - rc = btreeCursorNew(pDb, pSeg, &pCsr); - if( rc==LSM_OK ){ - rc = btreeCursorFirst(pCsr); - } - if( rc==LSM_OK ){ - rc = lsmFsDbPageGet(pFS, pSeg, pSeg->iFirst, &pPg); - } - - while( rc==LSM_OK ){ - Page *pNext; - u8 *aData; - int nData; - int flags; - - rc = lsmFsDbPageNext(pSeg, pPg, 1, &pNext); - lsmFsPageRelease(pPg); - pPg = pNext; - if( pPg==0 ) break; - aData = fsPageData(pPg, &nData); - flags = pageGetFlags(aData, nData); - if( rc==LSM_OK - && 0==((SEGMENT_BTREE_FLAG|PGFTR_SKIP_THIS_FLAG) & flags) - && 0!=pageGetNRec(aData, nData) - ){ - u8 *pKey; - int nKey; - int iTopic; - pKey = pageGetKey(pSeg, pPg, 0, &iTopic, &nKey, &blob); - assert( nKey==pCsr->nKey && 0==memcmp(pKey, pCsr->pKey, nKey) ); - assert( lsmFsPageNumber(pPg)==pCsr->iPtr ); - rc = btreeCursorNext(pCsr); - } - } - assert( rc!=LSM_OK || pCsr->pKey==0 ); - - if( pPg ) lsmFsPageRelease(pPg); - - btreeCursorFree(pCsr); - sortedBlobFree(&blob); - } - - return rc; -} -#endif /* ifdef LSM_DEBUG_EXPENSIVE */ diff --git a/ext/lsm1/lsm_str.c b/ext/lsm1/lsm_str.c deleted file mode 100644 index 9b1b63cee..000000000 --- a/ext/lsm1/lsm_str.c +++ /dev/null @@ -1,148 +0,0 @@ -/* -** 2012-04-27 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Dynamic string functions. -*/ -#include "lsmInt.h" - -/* -** Turn bulk and uninitialized memory into an LsmString object -*/ -void lsmStringInit(LsmString *pStr, lsm_env *pEnv){ - memset(pStr, 0, sizeof(pStr[0])); - pStr->pEnv = pEnv; -} - -/* -** Increase the memory allocated for holding the string. Realloc as needed. -** -** If a memory allocation error occurs, set pStr->n to -1 and free the existing -** allocation. If a prior memory allocation has occurred, this routine is a -** no-op. -*/ -int lsmStringExtend(LsmString *pStr, int nNew){ - assert( nNew>0 ); - if( pStr->n<0 ) return LSM_NOMEM; - if( pStr->n + nNew >= pStr->nAlloc ){ - int nAlloc = pStr->n + nNew + 100; - char *zNew = lsmRealloc(pStr->pEnv, pStr->z, nAlloc); - if( zNew==0 ){ - lsmFree(pStr->pEnv, pStr->z); - nAlloc = 0; - pStr->n = -1; - } - pStr->nAlloc = nAlloc; - pStr->z = zNew; - } - return (pStr->z ? LSM_OK : LSM_NOMEM_BKPT); -} - -/* -** Clear an LsmString object, releasing any allocated memory that it holds. -** This also clears the error indication (if any). -*/ -void lsmStringClear(LsmString *pStr){ - lsmFree(pStr->pEnv, pStr->z); - lsmStringInit(pStr, pStr->pEnv); -} - -/* -** Append N bytes of text to the end of an LsmString object. If -** N is negative, append the entire string. -** -** If the string is in an error state, this routine is a no-op. -*/ -int lsmStringAppend(LsmString *pStr, const char *z, int N){ - int rc; - if( N<0 ) N = (int)strlen(z); - rc = lsmStringExtend(pStr, N+1); - if( pStr->nAlloc ){ - memcpy(pStr->z+pStr->n, z, N+1); - pStr->n += N; - } - return rc; -} - -int lsmStringBinAppend(LsmString *pStr, const u8 *a, int n){ - int rc; - rc = lsmStringExtend(pStr, n); - if( pStr->nAlloc ){ - memcpy(pStr->z+pStr->n, a, n); - pStr->n += n; - } - return rc; -} - -/* -** Append printf-formatted content to an LsmString. -*/ -void lsmStringVAppendf( - LsmString *pStr, - const char *zFormat, - va_list ap1, - va_list ap2 -){ -#if (!defined(__STDC_VERSION__) || (__STDC_VERSION__<199901L)) && \ - !defined(__APPLE__) - extern int vsnprintf(char *str, size_t size, const char *format, va_list ap) - /* Compatibility crutch for C89 compilation mode. sqlite3_vsnprintf() - does not work identically and causes test failures if used here. - For the time being we are assuming that the target has vsnprintf(), - but that is not guaranteed to be the case for pure C89 platforms. - */; -#endif - int nWrite; - int nAvail; - - nAvail = pStr->nAlloc - pStr->n; - nWrite = vsnprintf(pStr->z + pStr->n, nAvail, zFormat, ap1); - - if( nWrite>=nAvail ){ - lsmStringExtend(pStr, nWrite+1); - if( pStr->nAlloc==0 ) return; - nWrite = vsnprintf(pStr->z + pStr->n, nWrite+1, zFormat, ap2); - } - - pStr->n += nWrite; - pStr->z[pStr->n] = 0; -} - -void lsmStringAppendf(LsmString *pStr, const char *zFormat, ...){ - va_list ap, ap2; - va_start(ap, zFormat); - va_start(ap2, zFormat); - lsmStringVAppendf(pStr, zFormat, ap, ap2); - va_end(ap); - va_end(ap2); -} - -int lsmStrlen(const char *zName){ - int nRet = 0; - while( zName[nRet] ) nRet++; - return nRet; -} - -/* -** Write into memory obtained from lsm_malloc(). -*/ -char *lsmMallocPrintf(lsm_env *pEnv, const char *zFormat, ...){ - LsmString s; - va_list ap, ap2; - lsmStringInit(&s, pEnv); - va_start(ap, zFormat); - va_start(ap2, zFormat); - lsmStringVAppendf(&s, zFormat, ap, ap2); - va_end(ap); - va_end(ap2); - if( s.n<0 ) return 0; - return (char *)lsmReallocOrFree(pEnv, s.z, s.n+1); -} diff --git a/ext/lsm1/lsm_tree.c b/ext/lsm1/lsm_tree.c deleted file mode 100644 index 1a199fc1c..000000000 --- a/ext/lsm1/lsm_tree.c +++ /dev/null @@ -1,2465 +0,0 @@ -/* -** 2011-08-18 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file contains the implementation of an in-memory tree structure. -** -** Technically the tree is a B-tree of order 4 (in the Knuth sense - each -** node may have up to 4 children). Keys are stored within B-tree nodes by -** reference. This may be slightly slower than a conventional red-black -** tree, but it is simpler. It is also an easier structure to modify to -** create a version that supports nested transaction rollback. -** -** This tree does not currently support a delete operation. One is not -** required. When LSM deletes a key from a database, it inserts a DELETE -** marker into the data structure. As a result, although the value associated -** with a key stored in the in-memory tree structure may be modified, no -** keys are ever removed. -*/ - -/* -** MVCC NOTES -** -** The in-memory tree structure supports SQLite-style MVCC. This means -** that while one client is writing to the tree structure, other clients -** may still be querying an older snapshot of the tree. -** -** One way to implement this is to use an append-only b-tree. In this -** case instead of modifying nodes in-place, a copy of the node is made -** and the required modifications made to the copy. The parent of the -** node is then modified (to update the pointer so that it points to -** the new copy), which causes a copy of the parent to be made, and so on. -** This means that each time the tree is written to a new root node is -** created. A snapshot is identified by the root node that it uses. -** -** The problem with the above is that each time the tree is written to, -** a copy of the node structure modified and all of its ancestor nodes -** is made. This may prove excessive with large tree structures. -** -** To reduce this overhead, the data structure used for a tree node is -** designed so that it may be edited in place exactly once without -** affecting existing users. In other words, the node structure is capable -** of storing two separate versions of the node at the same time. -** When a node is to be edited, if the node structure already contains -** two versions, a copy is made as in the append-only approach. Or, if -** it only contains a single version, it is edited in place. -** -** This reduces the overhead so that, roughly, one new node structure -** must be allocated for each write (on top of those allocations that -** would have been required by a non-MVCC tree). Logic: Assume that at -** any time, 50% of nodes in the tree already contain 2 versions. When -** a new entry is written to a node, there is a 50% chance that a copy -** of the node will be required. And a 25% chance that a copy of its -** parent is required. And so on. -** -** ROLLBACK -** -** The in-memory tree also supports transaction and sub-transaction -** rollback. In order to rollback to point in time X, the following is -** necessary: -** -** 1. All memory allocated since X must be freed, and -** 2. All "v2" data adding to nodes that existed at X should be zeroed. -** 3. The root node must be restored to its X value. -** -** The Mempool object used to allocate memory for the tree supports -** operation (1) - see the lsmPoolMark() and lsmPoolRevert() functions. -** -** To support (2), all nodes that have v2 data are part of a singly linked -** list, sorted by the age of the v2 data (nodes that have had data added -** most recently are at the end of the list). So to zero all v2 data added -** since X, the linked list is traversed from the first node added following -** X onwards. -** -*/ - -#ifndef _LSM_INT_H -# include "lsmInt.h" -#endif - -#include - -#define MAX_DEPTH 32 - -typedef struct TreeKey TreeKey; -typedef struct TreeNode TreeNode; -typedef struct TreeLeaf TreeLeaf; -typedef struct NodeVersion NodeVersion; - -struct TreeOld { - u32 iShmid; /* Last shared-memory chunk in use by old */ - u32 iRoot; /* Offset of root node in shm file */ - u32 nHeight; /* Height of tree structure */ -}; - -#if 0 -/* -** assert() that a TreeKey.flags value is sane. Usage: -** -** assert( lsmAssertFlagsOk(pTreeKey->flags) ); -*/ -static int lsmAssertFlagsOk(u8 keyflags){ - /* At least one flag must be set. Otherwise, what is this key doing? */ - assert( keyflags!=0 ); - - /* The POINT_DELETE and INSERT flags cannot both be set. */ - assert( (keyflags & LSM_POINT_DELETE)==0 || (keyflags & LSM_INSERT)==0 ); - - /* If both the START_DELETE and END_DELETE flags are set, then the INSERT - ** flag must also be set. In other words - the three DELETE flags cannot - ** all be set */ - assert( (keyflags & LSM_END_DELETE)==0 - || (keyflags & LSM_START_DELETE)==0 - || (keyflags & LSM_POINT_DELETE)==0 - ); - - return 1; -} -#endif -static int assert_delete_ranges_match(lsm_db *); -static int treeCountEntries(lsm_db *db); - -/* -** Container for a key-value pair. Within the *-shm file, each key/value -** pair is stored in a single allocation (which may not actually be -** contiguous in memory). Layout is the TreeKey structure, followed by -** the nKey bytes of key blob, followed by the nValue bytes of value blob -** (if nValue is non-negative). -*/ -struct TreeKey { - int nKey; /* Size of pKey in bytes */ - int nValue; /* Size of pValue. Or negative. */ - u8 flags; /* Various LSM_XXX flags */ -}; - -#define TKV_KEY(p) ((void *)&(p)[1]) -#define TKV_VAL(p) ((void *)(((u8 *)&(p)[1]) + (p)->nKey)) - -/* -** A single tree node. A node structure may contain up to 3 key/value -** pairs. Internal (non-leaf) nodes have up to 4 children. -** -** TODO: Update the format of this to be more compact. Get it working -** first though... -*/ -struct TreeNode { - u32 aiKeyPtr[3]; /* Array of pointers to TreeKey objects */ - - /* The following fields are present for interior nodes only, not leaves. */ - u32 aiChildPtr[4]; /* Array of pointers to child nodes */ - - /* The extra child pointer slot. */ - u32 iV2; /* Transaction number of v2 */ - u8 iV2Child; /* apChild[] entry replaced by pV2Ptr */ - u32 iV2Ptr; /* Substitute pointer */ -}; - -struct TreeLeaf { - u32 aiKeyPtr[3]; /* Array of pointers to TreeKey objects */ -}; - -typedef struct TreeBlob TreeBlob; -struct TreeBlob { - int n; - u8 *a; -}; - -/* -** Cursor for searching a tree structure. -** -** If a cursor does not point to any element (a.k.a. EOF), then the -** TreeCursor.iNode variable is set to a negative value. Otherwise, the -** cursor currently points to key aiCell[iNode] on node apTreeNode[iNode]. -** -** Entries in the apTreeNode[] and aiCell[] arrays contain the node and -** index of the TreeNode.apChild[] pointer followed to descend to the -** current element. Hence apTreeNode[0] always contains the root node of -** the tree. -*/ -struct TreeCursor { - lsm_db *pDb; /* Database handle for this cursor */ - TreeRoot *pRoot; /* Root node and height of tree to access */ - int iNode; /* Cursor points at apTreeNode[iNode] */ - TreeNode *apTreeNode[MAX_DEPTH];/* Current position in tree */ - u8 aiCell[MAX_DEPTH]; /* Current position in tree */ - TreeKey *pSave; /* Saved key */ - TreeBlob blob; /* Dynamic storage for a key */ -}; - -/* -** A value guaranteed to be larger than the largest possible transaction -** id (TreeHeader.iTransId). -*/ -#define WORKING_VERSION (1<<30) - -static int tblobGrow(lsm_db *pDb, TreeBlob *p, int n, int *pRc){ - if( n>p->n ){ - lsmFree(pDb->pEnv, p->a); - p->a = lsmMallocRc(pDb->pEnv, n, pRc); - p->n = n; - } - return (p->a==0); -} -static void tblobFree(lsm_db *pDb, TreeBlob *p){ - lsmFree(pDb->pEnv, p->a); -} - - -/*********************************************************************** -** Start of IntArray methods. */ -/* -** Append value iVal to the contents of IntArray *p. Return LSM_OK if -** successful, or LSM_NOMEM if an OOM condition is encountered. -*/ -static int intArrayAppend(lsm_env *pEnv, IntArray *p, u32 iVal){ - assert( p->nArray<=p->nAlloc ); - if( p->nArray>=p->nAlloc ){ - u32 *aNew; - int nNew = p->nArray ? p->nArray*2 : 128; - aNew = lsmRealloc(pEnv, p->aArray, nNew*sizeof(u32)); - if( !aNew ) return LSM_NOMEM_BKPT; - p->aArray = aNew; - p->nAlloc = nNew; - } - - p->aArray[p->nArray++] = iVal; - return LSM_OK; -} - -/* -** Zero the IntArray object. -*/ -static void intArrayFree(lsm_env *pEnv, IntArray *p){ - p->nArray = 0; -} - -/* -** Return the number of entries currently in the int-array object. -*/ -static int intArraySize(IntArray *p){ - return p->nArray; -} - -/* -** Return a copy of the iIdx'th entry in the int-array. -*/ -static u32 intArrayEntry(IntArray *p, int iIdx){ - return p->aArray[iIdx]; -} - -/* -** Truncate the int-array so that all but the first nVal values are -** discarded. -*/ -static void intArrayTruncate(IntArray *p, int nVal){ - p->nArray = nVal; -} -/* End of IntArray methods. -***********************************************************************/ - -static int treeKeycmp(void *p1, int n1, void *p2, int n2){ - int res; - res = memcmp(p1, p2, LSM_MIN(n1, n2)); - if( res==0 ) res = (n1-n2); - return res; -} - -/* -** The pointer passed as the first argument points to an interior node, -** not a leaf. This function returns the offset of the iCell'th child -** sub-tree of the node. -*/ -static u32 getChildPtr(TreeNode *p, int iVersion, int iCell){ - assert( iVersion>=0 ); - assert( iCell>=0 && iCell<=array_size(p->aiChildPtr) ); - if( p->iV2 && p->iV2<=(u32)iVersion && iCell==p->iV2Child ) return p->iV2Ptr; - return p->aiChildPtr[iCell]; -} - -/* -** Given an offset within the *-shm file, return the associated chunk number. -*/ -static int treeOffsetToChunk(u32 iOff){ - assert( LSM_SHM_CHUNK_SIZE==(1<<15) ); - return (int)(iOff>>15); -} - -#define treeShmptrUnsafe(pDb, iPtr) \ -(&((u8*)((pDb)->apShm[(iPtr)>>15]))[(iPtr) & (LSM_SHM_CHUNK_SIZE-1)]) - -/* -** Return a pointer to the mapped memory location associated with *-shm -** file offset iPtr. -*/ -static void *treeShmptr(lsm_db *pDb, u32 iPtr){ - - assert( (iPtr>>15)<(u32)pDb->nShm ); - assert( pDb->apShm[iPtr>>15] ); - - return iPtr ? treeShmptrUnsafe(pDb, iPtr) : 0; -} - -static ShmChunk * treeShmChunk(lsm_db *pDb, int iChunk){ - return (ShmChunk *)(pDb->apShm[iChunk]); -} - -static ShmChunk * treeShmChunkRc(lsm_db *pDb, int iChunk, int *pRc){ - assert( *pRc==LSM_OK ); - if( iChunknShm || LSM_OK==(*pRc = lsmShmCacheChunks(pDb, iChunk+1)) ){ - return (ShmChunk *)(pDb->apShm[iChunk]); - } - return 0; -} - - -#ifndef NDEBUG -static void assertIsWorkingChild( - lsm_db *db, - TreeNode *pNode, - TreeNode *pParent, - int iCell -){ - TreeNode *p; - u32 iPtr = getChildPtr(pParent, WORKING_VERSION, iCell); - p = treeShmptr(db, iPtr); - assert( p==pNode ); -} -#else -# define assertIsWorkingChild(w,x,y,z) -#endif - -/* Values for the third argument to treeShmkey(). */ -#define TKV_LOADKEY 1 -#define TKV_LOADVAL 2 - -static TreeKey *treeShmkey( - lsm_db *pDb, /* Database handle */ - u32 iPtr, /* Shmptr to TreeKey struct */ - int eLoad, /* Either zero or a TREEKEY_LOADXXX value */ - TreeBlob *pBlob, /* Used if dynamic memory is required */ - int *pRc /* IN/OUT: Error code */ -){ - TreeKey *pRet; - - assert( eLoad==TKV_LOADKEY || eLoad==TKV_LOADVAL ); - pRet = (TreeKey *)treeShmptr(pDb, iPtr); - if( pRet ){ - int nReq; /* Bytes of space required at pRet */ - int nAvail; /* Bytes of space available at pRet */ - - nReq = sizeof(TreeKey) + pRet->nKey; - if( eLoad==TKV_LOADVAL && pRet->nValue>0 ){ - nReq += pRet->nValue; - } - assert( LSM_SHM_CHUNK_SIZE==(1<<15) ); - nAvail = LSM_SHM_CHUNK_SIZE - (iPtr & (LSM_SHM_CHUNK_SIZE-1)); - - if( nAvaila[nLoad], p, n); - nLoad += n; - if( nLoad==nReq ) break; - - pChunk = treeShmChunk(pDb, treeOffsetToChunk(iPtr)); - assert( pChunk ); - iPtr = (pChunk->iNext * LSM_SHM_CHUNK_SIZE) + LSM_SHM_CHUNK_HDR; - nAvail = LSM_SHM_CHUNK_SIZE - LSM_SHM_CHUNK_HDR; - } - } - pRet = (TreeKey *)(pBlob->a); - } - } - - return pRet; -} - -#if defined(LSM_DEBUG) && defined(LSM_EXPENSIVE_ASSERT) -void assert_leaf_looks_ok(TreeNode *pNode){ - assert( pNode->apKey[1] ); -} - -void assert_node_looks_ok(TreeNode *pNode, int nHeight){ - if( pNode ){ - assert( pNode->apKey[1] ); - if( nHeight>1 ){ - int i; - assert( getChildPtr(pNode, WORKING_VERSION, 1) ); - assert( getChildPtr(pNode, WORKING_VERSION, 2) ); - for(i=0; i<4; i++){ - assert_node_looks_ok(getChildPtr(pNode, WORKING_VERSION, i), nHeight-1); - } - } - } -} - -/* -** Run various assert() statements to check that the working-version of the -** tree is correct in the following respects: -** -** * todo... -*/ -void assert_tree_looks_ok(int rc, Tree *pTree){ -} -#else -# define assert_tree_looks_ok(x,y) -#endif - -void lsmFlagsToString(int flags, char *zFlags){ - - zFlags[0] = (flags & LSM_END_DELETE) ? ']' : '.'; - - /* Only one of LSM_POINT_DELETE, LSM_INSERT and LSM_SEPARATOR should ever - ** be set. If this is not true, write a '?' to the output. */ - switch( flags & (LSM_POINT_DELETE|LSM_INSERT|LSM_SEPARATOR) ){ - case 0: zFlags[1] = '.'; break; - case LSM_POINT_DELETE: zFlags[1] = '-'; break; - case LSM_INSERT: zFlags[1] = '+'; break; - case LSM_SEPARATOR: zFlags[1] = '^'; break; - default: zFlags[1] = '?'; break; - } - - zFlags[2] = (flags & LSM_SYSTEMKEY) ? '*' : '.'; - zFlags[3] = (flags & LSM_START_DELETE) ? '[' : '.'; - zFlags[4] = '\0'; -} - -#ifdef LSM_DEBUG - -/* -** Pointer pBlob points to a buffer containing a blob of binary data -** nBlob bytes long. Append the contents of this blob to *pStr, with -** each octet represented by a 2-digit hexadecimal number. For example, -** if the input blob is three bytes in size and contains {0x01, 0x44, 0xFF}, -** then "0144ff" is appended to *pStr. -*/ -static void lsmAppendStrBlob(LsmString *pStr, void *pBlob, int nBlob){ - int i; - lsmStringExtend(pStr, nBlob*2); - if( pStr->nAlloc==0 ) return; - for(i=0; i='a' && c<='z' ){ - pStr->z[pStr->n++] = c; - }else if( c!=0 || nBlob==1 || i!=(nBlob-1) ){ - pStr->z[pStr->n++] = "0123456789abcdef"[(c>>4)&0xf]; - pStr->z[pStr->n++] = "0123456789abcdef"[c&0xf]; - } - } - pStr->z[pStr->n] = 0; -} - -#if 0 /* NOT USED */ -/* -** Append nIndent space (0x20) characters to string *pStr. -*/ -static void lsmAppendIndent(LsmString *pStr, int nIndent){ - int i; - lsmStringExtend(pStr, nIndent); - for(i=0; ipEnv); - - /* Append each key to string s. */ - for(i=0; i<3; i++){ - u32 iPtr = pNode->aiKeyPtr[i]; - if( iPtr ){ - TreeKey *pKey = treeShmkey(pDb, pNode->aiKeyPtr[i],TKV_LOADKEY, &b,&rc); - strAppendFlags(&s, pKey->flags); - lsmAppendStrBlob(&s, TKV_KEY(pKey), pKey->nKey); - lsmStringAppend(&s, " ", -1); - } - } - - printf("% 6d %.*sleaf%.*s: %s\n", - iNode, nPath, zPath, 20-nPath-4, zSpace, s.z - ); - lsmStringClear(&s); - }else{ - for(i=0; i<4 && nHeight>0; i++){ - u32 iPtr = getChildPtr(pNode, pDb->treehdr.root.iTransId, i); - zPath[nPath] = (char)(i+'0'); - zPath[nPath+1] = '/'; - - if( iPtr ){ - dump_node_contents(pDb, iPtr, zPath, nPath+2, nHeight-1); - } - if( i!=3 && pNode->aiKeyPtr[i] ){ - TreeKey *pKey = treeShmkey(pDb, pNode->aiKeyPtr[i], TKV_LOADKEY,&b,&rc); - lsmStringInit(&s, pDb->pEnv); - strAppendFlags(&s, pKey->flags); - lsmAppendStrBlob(&s, TKV_KEY(pKey), pKey->nKey); - printf("% 6d %.*s%.*s: %s\n", - iNode, nPath+1, zPath, 20-nPath-1, zSpace, s.z); - lsmStringClear(&s); - } - } - } - - tblobFree(pDb, &b); -} - -void dump_tree_contents(lsm_db *pDb, const char *zCaption){ - char zPath[64]; - TreeRoot *p = &pDb->treehdr.root; - printf("\n%s\n", zCaption); - zPath[0] = '/'; - if( p->iRoot ){ - dump_node_contents(pDb, p->iRoot, zPath, 1, p->nHeight-1); - } - fflush(stdout); -} - -#endif - -/* -** Initialize a cursor object, the space for which has already been -** allocated. -*/ -static void treeCursorInit(lsm_db *pDb, int bOld, TreeCursor *pCsr){ - memset(pCsr, 0, sizeof(TreeCursor)); - pCsr->pDb = pDb; - if( bOld ){ - pCsr->pRoot = &pDb->treehdr.oldroot; - }else{ - pCsr->pRoot = &pDb->treehdr.root; - } - pCsr->iNode = -1; -} - -/* -** Return a pointer to the mapping of the TreeKey object that the cursor -** is pointing to. -*/ -static TreeKey *csrGetKey(TreeCursor *pCsr, TreeBlob *pBlob, int *pRc){ - TreeKey *pRet; - lsm_db *pDb = pCsr->pDb; - u32 iPtr = pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[pCsr->aiCell[pCsr->iNode]]; - - assert( iPtr ); - pRet = (TreeKey*)treeShmptrUnsafe(pDb, iPtr); - if( !(pRet->flags & LSM_CONTIGUOUS) ){ - pRet = treeShmkey(pDb, iPtr, TKV_LOADVAL, pBlob, pRc); - } - - return pRet; -} - -/* -** Save the current position of tree cursor pCsr. -*/ -int lsmTreeCursorSave(TreeCursor *pCsr){ - int rc = LSM_OK; - if( pCsr && pCsr->pSave==0 ){ - int iNode = pCsr->iNode; - if( iNode>=0 ){ - pCsr->pSave = csrGetKey(pCsr, &pCsr->blob, &rc); - } - pCsr->iNode = -1; - } - return rc; -} - -/* -** Restore the position of a saved tree cursor. -*/ -static int treeCursorRestore(TreeCursor *pCsr, int *pRes){ - int rc = LSM_OK; - if( pCsr->pSave ){ - TreeKey *pKey = pCsr->pSave; - pCsr->pSave = 0; - if( pRes ){ - rc = lsmTreeCursorSeek(pCsr, TKV_KEY(pKey), pKey->nKey, pRes); - } - } - return rc; -} - -/* -** Allocate nByte bytes of space within the *-shm file. If successful, -** return LSM_OK and set *piPtr to the offset within the file at which -** the allocated space is located. -*/ -static u32 treeShmalloc(lsm_db *pDb, int bAlign, int nByte, int *pRc){ - u32 iRet = 0; - if( *pRc==LSM_OK ){ - const static int CHUNK_SIZE = LSM_SHM_CHUNK_SIZE; - const static int CHUNK_HDR = LSM_SHM_CHUNK_HDR; - u32 iWrite; /* Current write offset */ - u32 iEof; /* End of current chunk */ - int iChunk; /* Current chunk */ - - assert( nByte <= (CHUNK_SIZE-CHUNK_HDR) ); - - /* Check if there is enough space on the current chunk to fit the - ** new allocation. If not, link in a new chunk and put the new - ** allocation at the start of it. */ - iWrite = pDb->treehdr.iWrite; - if( bAlign ){ - iWrite = (iWrite + 3) & ~0x0003; - assert( (iWrite % 4)==0 ); - } - - assert( iWrite ); - iChunk = treeOffsetToChunk(iWrite-1); - iEof = (iChunk+1) * CHUNK_SIZE; - assert( iEof>=iWrite && (iEof-iWrite)<(u32)CHUNK_SIZE ); - if( (iWrite+nByte)>iEof ){ - ShmChunk *pHdr; /* Header of chunk just finished (iChunk) */ - ShmChunk *pFirst; /* Header of chunk treehdr.iFirst */ - ShmChunk *pNext; /* Header of new chunk */ - int iNext = 0; /* Next chunk */ - int rc = LSM_OK; - - pFirst = treeShmChunk(pDb, pDb->treehdr.iFirst); - - assert( shm_sequence_ge(pDb->treehdr.iUsedShmid, pFirst->iShmid) ); - assert( (pDb->treehdr.iNextShmid+1-pDb->treehdr.nChunk)==pFirst->iShmid ); - - /* Check if the chunk at the start of the linked list is still in - ** use. If not, reuse it. If so, allocate a new chunk by appending - ** to the *-shm file. */ - if( pDb->treehdr.iUsedShmid!=pFirst->iShmid ){ - int bInUse; - rc = lsmTreeInUse(pDb, pFirst->iShmid, &bInUse); - if( rc!=LSM_OK ){ - *pRc = rc; - return 0; - } - if( bInUse==0 ){ - iNext = pDb->treehdr.iFirst; - pDb->treehdr.iFirst = pFirst->iNext; - assert( pDb->treehdr.iFirst ); - } - } - if( iNext==0 ) iNext = pDb->treehdr.nChunk++; - - /* Set the header values for the new chunk */ - pNext = treeShmChunkRc(pDb, iNext, &rc); - if( pNext ){ - pNext->iNext = 0; - pNext->iShmid = (pDb->treehdr.iNextShmid++); - }else{ - *pRc = rc; - return 0; - } - - /* Set the header values for the chunk just finished */ - pHdr = (ShmChunk *)treeShmptr(pDb, iChunk*CHUNK_SIZE); - pHdr->iNext = iNext; - - /* Advance to the next chunk */ - iWrite = iNext * CHUNK_SIZE + CHUNK_HDR; - } - - /* Allocate space at iWrite. */ - iRet = iWrite; - pDb->treehdr.iWrite = iWrite + nByte; - pDb->treehdr.root.nByte += nByte; - } - return iRet; -} - -/* -** Allocate and zero nByte bytes of space within the *-shm file. -*/ -static void *treeShmallocZero(lsm_db *pDb, int nByte, u32 *piPtr, int *pRc){ - u32 iPtr; - void *p; - iPtr = treeShmalloc(pDb, 1, nByte, pRc); - p = treeShmptr(pDb, iPtr); - if( p ){ - assert( *pRc==LSM_OK ); - memset(p, 0, nByte); - *piPtr = iPtr; - } - return p; -} - -static TreeNode *newTreeNode(lsm_db *pDb, u32 *piPtr, int *pRc){ - return treeShmallocZero(pDb, sizeof(TreeNode), piPtr, pRc); -} - -static TreeLeaf *newTreeLeaf(lsm_db *pDb, u32 *piPtr, int *pRc){ - return treeShmallocZero(pDb, sizeof(TreeLeaf), piPtr, pRc); -} - -static TreeKey *newTreeKey( - lsm_db *pDb, - u32 *piPtr, - void *pKey, int nKey, /* Key data */ - void *pVal, int nVal, /* Value data (or nVal<0 for delete) */ - int *pRc -){ - TreeKey *p; - u32 iPtr; - u32 iEnd; - int nRem; - u8 *a; - int n; - - /* Allocate space for the TreeKey structure itself */ - *piPtr = iPtr = treeShmalloc(pDb, 1, sizeof(TreeKey), pRc); - p = treeShmptr(pDb, iPtr); - if( *pRc ) return 0; - p->nKey = nKey; - p->nValue = nVal; - - /* Allocate and populate the space required for the key and value. */ - n = nRem = nKey; - a = (u8 *)pKey; - while( a ){ - while( nRem>0 ){ - u8 *aAlloc; - int nAlloc; - u32 iWrite; - - iWrite = (pDb->treehdr.iWrite & (LSM_SHM_CHUNK_SIZE-1)); - iWrite = LSM_MAX(iWrite, LSM_SHM_CHUNK_HDR); - nAlloc = LSM_MIN((LSM_SHM_CHUNK_SIZE-iWrite), (u32)nRem); - - aAlloc = treeShmptr(pDb, treeShmalloc(pDb, 0, nAlloc, pRc)); - if( aAlloc==0 ) break; - memcpy(aAlloc, &a[n-nRem], nAlloc); - nRem -= nAlloc; - } - a = pVal; - n = nRem = nVal; - pVal = 0; - } - - iEnd = iPtr + sizeof(TreeKey) + nKey + LSM_MAX(0, nVal); - if( (iPtr & ~(LSM_SHM_CHUNK_SIZE-1))!=(iEnd & ~(LSM_SHM_CHUNK_SIZE-1)) ){ - p->flags = 0; - }else{ - p->flags = LSM_CONTIGUOUS; - } - - if( *pRc ) return 0; -#if 0 - printf("store: %d %s\n", (int)iPtr, (char *)pKey); -#endif - return p; -} - -static TreeNode *copyTreeNode( - lsm_db *pDb, - TreeNode *pOld, - u32 *piNew, - int *pRc -){ - TreeNode *pNew; - - pNew = newTreeNode(pDb, piNew, pRc); - if( pNew ){ - memcpy(pNew->aiKeyPtr, pOld->aiKeyPtr, sizeof(pNew->aiKeyPtr)); - memcpy(pNew->aiChildPtr, pOld->aiChildPtr, sizeof(pNew->aiChildPtr)); - if( pOld->iV2 ) pNew->aiChildPtr[pOld->iV2Child] = pOld->iV2Ptr; - } - return pNew; -} - -static TreeNode *copyTreeLeaf( - lsm_db *pDb, - TreeLeaf *pOld, - u32 *piNew, - int *pRc -){ - TreeLeaf *pNew; - pNew = newTreeLeaf(pDb, piNew, pRc); - if( pNew ){ - memcpy(pNew, pOld, sizeof(TreeLeaf)); - } - return (TreeNode *)pNew; -} - -/* -** The tree cursor passed as the second argument currently points to an -** internal node (not a leaf). Specifically, to a sub-tree pointer. This -** function replaces the sub-tree that the cursor currently points to -** with sub-tree pNew. -** -** The sub-tree may be replaced either by writing the "v2 data" on the -** internal node, or by allocating a new TreeNode structure and then -** calling this function on the parent of the internal node. -*/ -static int treeUpdatePtr(lsm_db *pDb, TreeCursor *pCsr, u32 iNew){ - int rc = LSM_OK; - if( pCsr->iNode<0 ){ - /* iNew is the new root node */ - pDb->treehdr.root.iRoot = iNew; - }else{ - /* If this node already has version 2 content, allocate a copy and - ** update the copy with the new pointer value. Otherwise, store the - ** new pointer as v2 data within the current node structure. */ - - TreeNode *p; /* The node to be modified */ - int iChildPtr; /* apChild[] entry to modify */ - - p = pCsr->apTreeNode[pCsr->iNode]; - iChildPtr = pCsr->aiCell[pCsr->iNode]; - - if( p->iV2 ){ - /* The "allocate new TreeNode" option */ - u32 iCopy; - TreeNode *pCopy; - pCopy = copyTreeNode(pDb, p, &iCopy, &rc); - if( pCopy ){ - assert( rc==LSM_OK ); - pCopy->aiChildPtr[iChildPtr] = iNew; - pCsr->iNode--; - rc = treeUpdatePtr(pDb, pCsr, iCopy); - } - }else{ - /* The "v2 data" option */ - u32 iPtr; - assert( pDb->treehdr.root.iTransId>0 ); - - if( pCsr->iNode ){ - iPtr = getChildPtr( - pCsr->apTreeNode[pCsr->iNode-1], - pDb->treehdr.root.iTransId, pCsr->aiCell[pCsr->iNode-1] - ); - }else{ - iPtr = pDb->treehdr.root.iRoot; - } - rc = intArrayAppend(pDb->pEnv, &pDb->rollback, iPtr); - - if( rc==LSM_OK ){ - p->iV2 = pDb->treehdr.root.iTransId; - p->iV2Child = (u8)iChildPtr; - p->iV2Ptr = iNew; - } - } - } - - return rc; -} - -/* -** Cursor pCsr points at a node that is part of pTree. This function -** inserts a new key and optionally child node pointer into that node. -** -** The position into which the new key and pointer are inserted is -** determined by the iSlot parameter. The new key will be inserted to -** the left of the key currently stored in apKey[iSlot]. Or, if iSlot is -** greater than the index of the rightmost key in the node. -** -** Pointer pLeftPtr points to a child tree that contains keys that are -** smaller than pTreeKey. -*/ -static int treeInsert( - lsm_db *pDb, /* Database handle */ - TreeCursor *pCsr, /* Cursor indicating path to insert at */ - u32 iLeftPtr, /* Left child pointer */ - u32 iTreeKey, /* Location of key to insert */ - u32 iRightPtr, /* Right child pointer */ - int iSlot /* Position to insert key into */ -){ - int rc = LSM_OK; - TreeNode *pNode = pCsr->apTreeNode[pCsr->iNode]; - - /* Check if the node is currently full. If so, split pNode in two and - ** call this function recursively to add a key to the parent. Otherwise, - ** insert the new key directly into pNode. */ - assert( pNode->aiKeyPtr[1] ); - if( pNode->aiKeyPtr[0] && pNode->aiKeyPtr[2] ){ - u32 iLeft; TreeNode *pLeft; /* New left-hand sibling node */ - u32 iRight; TreeNode *pRight; /* New right-hand sibling node */ - - pLeft = newTreeNode(pDb, &iLeft, &rc); - pRight = newTreeNode(pDb, &iRight, &rc); - if( rc ) return rc; - - pLeft->aiChildPtr[1] = getChildPtr(pNode, WORKING_VERSION, 0); - pLeft->aiKeyPtr[1] = pNode->aiKeyPtr[0]; - pLeft->aiChildPtr[2] = getChildPtr(pNode, WORKING_VERSION, 1); - - pRight->aiChildPtr[1] = getChildPtr(pNode, WORKING_VERSION, 2); - pRight->aiKeyPtr[1] = pNode->aiKeyPtr[2]; - pRight->aiChildPtr[2] = getChildPtr(pNode, WORKING_VERSION, 3); - - if( pCsr->iNode==0 ){ - /* pNode is the root of the tree. Grow the tree by one level. */ - u32 iRoot; TreeNode *pRoot; /* New root node */ - - pRoot = newTreeNode(pDb, &iRoot, &rc); - pRoot->aiKeyPtr[1] = pNode->aiKeyPtr[1]; - pRoot->aiChildPtr[1] = iLeft; - pRoot->aiChildPtr[2] = iRight; - - pDb->treehdr.root.iRoot = iRoot; - pDb->treehdr.root.nHeight++; - }else{ - - pCsr->iNode--; - rc = treeInsert(pDb, pCsr, - iLeft, pNode->aiKeyPtr[1], iRight, pCsr->aiCell[pCsr->iNode] - ); - } - - assert( pLeft->iV2==0 ); - assert( pRight->iV2==0 ); - switch( iSlot ){ - case 0: - pLeft->aiKeyPtr[0] = iTreeKey; - pLeft->aiChildPtr[0] = iLeftPtr; - if( iRightPtr ) pLeft->aiChildPtr[1] = iRightPtr; - break; - case 1: - pLeft->aiChildPtr[3] = (iRightPtr ? iRightPtr : pLeft->aiChildPtr[2]); - pLeft->aiKeyPtr[2] = iTreeKey; - pLeft->aiChildPtr[2] = iLeftPtr; - break; - case 2: - pRight->aiKeyPtr[0] = iTreeKey; - pRight->aiChildPtr[0] = iLeftPtr; - if( iRightPtr ) pRight->aiChildPtr[1] = iRightPtr; - break; - case 3: - pRight->aiChildPtr[3] = (iRightPtr ? iRightPtr : pRight->aiChildPtr[2]); - pRight->aiKeyPtr[2] = iTreeKey; - pRight->aiChildPtr[2] = iLeftPtr; - break; - } - - }else{ - TreeNode *pNew; - u32 *piKey; - u32 *piChild; - u32 iStore = 0; - u32 iNew = 0; - int i; - - /* Allocate a new version of node pNode. */ - pNew = newTreeNode(pDb, &iNew, &rc); - if( rc ) return rc; - - piKey = pNew->aiKeyPtr; - piChild = pNew->aiChildPtr; - - for(i=0; iaiKeyPtr[i] ){ - *(piKey++) = pNode->aiKeyPtr[i]; - *(piChild++) = getChildPtr(pNode, WORKING_VERSION, i); - } - } - - *piKey++ = iTreeKey; - *piChild++ = iLeftPtr; - - iStore = iRightPtr; - for(i=iSlot; i<3; i++){ - if( pNode->aiKeyPtr[i] ){ - *(piKey++) = pNode->aiKeyPtr[i]; - *(piChild++) = iStore ? iStore : getChildPtr(pNode, WORKING_VERSION, i); - iStore = 0; - } - } - - if( iStore ){ - *piChild = iStore; - }else{ - *piChild = getChildPtr(pNode, WORKING_VERSION, - (pNode->aiKeyPtr[2] ? 3 : 2) - ); - } - pCsr->iNode--; - rc = treeUpdatePtr(pDb, pCsr, iNew); - } - - return rc; -} - -static int treeInsertLeaf( - lsm_db *pDb, /* Database handle */ - TreeCursor *pCsr, /* Cursor structure */ - u32 iTreeKey, /* Key pointer to insert */ - int iSlot /* Insert key to the left of this */ -){ - int rc = LSM_OK; /* Return code */ - TreeNode *pLeaf = pCsr->apTreeNode[pCsr->iNode]; - TreeLeaf *pNew; - u32 iNew; - - assert( iSlot>=0 && iSlot<=4 ); - assert( pCsr->iNode>0 ); - assert( pLeaf->aiKeyPtr[1] ); - - pCsr->iNode--; - - pNew = newTreeLeaf(pDb, &iNew, &rc); - if( pNew ){ - if( pLeaf->aiKeyPtr[0] && pLeaf->aiKeyPtr[2] ){ - /* The leaf is full. Split it in two. */ - TreeLeaf *pRight; - u32 iRight; - pRight = newTreeLeaf(pDb, &iRight, &rc); - if( pRight ){ - assert( rc==LSM_OK ); - pNew->aiKeyPtr[1] = pLeaf->aiKeyPtr[0]; - pRight->aiKeyPtr[1] = pLeaf->aiKeyPtr[2]; - switch( iSlot ){ - case 0: pNew->aiKeyPtr[0] = iTreeKey; break; - case 1: pNew->aiKeyPtr[2] = iTreeKey; break; - case 2: pRight->aiKeyPtr[0] = iTreeKey; break; - case 3: pRight->aiKeyPtr[2] = iTreeKey; break; - } - - rc = treeInsert(pDb, pCsr, iNew, pLeaf->aiKeyPtr[1], iRight, - pCsr->aiCell[pCsr->iNode] - ); - } - }else{ - int iOut = 0; - int i; - for(i=0; i<4; i++){ - if( i==iSlot ) pNew->aiKeyPtr[iOut++] = iTreeKey; - if( i<3 && pLeaf->aiKeyPtr[i] ){ - pNew->aiKeyPtr[iOut++] = pLeaf->aiKeyPtr[i]; - } - } - rc = treeUpdatePtr(pDb, pCsr, iNew); - } - } - - return rc; -} - -void lsmTreeMakeOld(lsm_db *pDb){ - - /* A write transaction must be open. Otherwise the code below that - ** assumes (pDb->pClient->iLogOff) is current may malfunction. - ** - ** Update: currently this assert fails due to lsm_flush(), which does - ** not set nTransOpen. - */ - assert( /* pDb->nTransOpen>0 && */ pDb->iReader>=0 ); - - if( pDb->treehdr.iOldShmid==0 ){ - pDb->treehdr.iOldLog = (pDb->treehdr.log.aRegion[2].iEnd << 1); - pDb->treehdr.iOldLog |= (~(pDb->pClient->iLogOff) & (i64)0x0001); - - pDb->treehdr.oldcksum0 = pDb->treehdr.log.cksum0; - pDb->treehdr.oldcksum1 = pDb->treehdr.log.cksum1; - pDb->treehdr.iOldShmid = pDb->treehdr.iNextShmid-1; - memcpy(&pDb->treehdr.oldroot, &pDb->treehdr.root, sizeof(TreeRoot)); - - pDb->treehdr.root.iTransId = 1; - pDb->treehdr.root.iRoot = 0; - pDb->treehdr.root.nHeight = 0; - pDb->treehdr.root.nByte = 0; - } -} - -void lsmTreeDiscardOld(lsm_db *pDb){ - assert( lsmShmAssertLock(pDb, LSM_LOCK_WRITER, LSM_LOCK_EXCL) - || lsmShmAssertLock(pDb, LSM_LOCK_DMS2, LSM_LOCK_EXCL) - ); - pDb->treehdr.iUsedShmid = pDb->treehdr.iOldShmid; - pDb->treehdr.iOldShmid = 0; -} - -int lsmTreeHasOld(lsm_db *pDb){ - return pDb->treehdr.iOldShmid!=0; -} - -/* -** This function is called during recovery to initialize the -** tree header. Only the database connections private copy of the tree-header -** is initialized here - it will be copied into shared memory if log file -** recovery is successful. -*/ -int lsmTreeInit(lsm_db *pDb){ - ShmChunk *pOne; - int rc = LSM_OK; - - memset(&pDb->treehdr, 0, sizeof(TreeHeader)); - pDb->treehdr.root.iTransId = 1; - pDb->treehdr.iFirst = 1; - pDb->treehdr.nChunk = 2; - pDb->treehdr.iWrite = LSM_SHM_CHUNK_SIZE + LSM_SHM_CHUNK_HDR; - pDb->treehdr.iNextShmid = 2; - pDb->treehdr.iUsedShmid = 1; - - pOne = treeShmChunkRc(pDb, 1, &rc); - if( pOne ){ - pOne->iNext = 0; - pOne->iShmid = 1; - } - return rc; -} - -static void treeHeaderChecksum( - TreeHeader *pHdr, - u32 *aCksum -){ - u32 cksum1 = 0x12345678; - u32 cksum2 = 0x9ABCDEF0; - u32 *a = (u32 *)pHdr; - int i; - - assert( (offsetof(TreeHeader, aCksum) + sizeof(u32)*2)==sizeof(TreeHeader) ); - assert( (sizeof(TreeHeader) % (sizeof(u32)*2))==0 ); - - for(i=0; i<(offsetof(TreeHeader, aCksum) / sizeof(u32)); i+=2){ - cksum1 += a[i]; - cksum2 += (cksum1 + a[i+1]); - } - aCksum[0] = cksum1; - aCksum[1] = cksum2; -} - -/* -** Return true if the checksum stored in TreeHeader object *pHdr is -** consistent with the contents of its other fields. -*/ -static int treeHeaderChecksumOk(TreeHeader *pHdr){ - u32 aCksum[2]; - treeHeaderChecksum(pHdr, aCksum); - return (0==memcmp(aCksum, pHdr->aCksum, sizeof(aCksum))); -} - -/* -** This type is used by functions lsmTreeRepair() and treeSortByShmid() to -** make relinking the linked list of shared-memory chunks easier. -*/ -typedef struct ShmChunkLoc ShmChunkLoc; -struct ShmChunkLoc { - ShmChunk *pShm; - u32 iLoc; -}; - -/* -** This function checks that the linked list of shared memory chunks -** that starts at chunk db->treehdr.iFirst: -** -** 1) Includes all chunks in the shared-memory region, and -** 2) Links them together in order of ascending shm-id. -** -** If no error occurs and the conditions above are met, LSM_OK is returned. -** -** If either of the conditions are untrue, LSM_CORRUPT is returned. Or, if -** an error is encountered before the checks are completed, another LSM error -** code (i.e. LSM_IOERR or LSM_NOMEM) may be returned. -*/ -static int treeCheckLinkedList(lsm_db *db){ - int rc = LSM_OK; - int nVisit = 0; - ShmChunk *p; - - p = treeShmChunkRc(db, db->treehdr.iFirst, &rc); - while( rc==LSM_OK && p ){ - if( p->iNext ){ - if( p->iNext>=db->treehdr.nChunk ){ - rc = LSM_CORRUPT_BKPT; - }else{ - ShmChunk *pNext = treeShmChunkRc(db, p->iNext, &rc); - if( rc==LSM_OK ){ - if( pNext->iShmid!=p->iShmid+1 ){ - rc = LSM_CORRUPT_BKPT; - } - p = pNext; - } - } - }else{ - p = 0; - } - nVisit++; - } - - if( rc==LSM_OK && (u32)nVisit!=db->treehdr.nChunk-1 ){ - rc = LSM_CORRUPT_BKPT; - } - return rc; -} - -/* -** Iterate through the current in-memory tree. If there are any v2-pointers -** with transaction ids larger than db->treehdr.iTransId, zero them. -*/ -static int treeRepairPtrs(lsm_db *db){ - int rc = LSM_OK; - - if( db->treehdr.root.nHeight>1 ){ - TreeCursor csr; /* Cursor used to iterate through tree */ - u32 iTransId = db->treehdr.root.iTransId; - - /* Initialize the cursor structure. Also decrement the nHeight variable - ** in the tree-header. This will prevent the cursor from visiting any - ** leaf nodes. */ - db->treehdr.root.nHeight--; - treeCursorInit(db, 0, &csr); - - rc = lsmTreeCursorEnd(&csr, 0); - while( rc==LSM_OK && lsmTreeCursorValid(&csr) ){ - TreeNode *pNode = csr.apTreeNode[csr.iNode]; - if( pNode->iV2>iTransId ){ - pNode->iV2Child = 0; - pNode->iV2Ptr = 0; - pNode->iV2 = 0; - } - rc = lsmTreeCursorNext(&csr); - } - tblobFree(csr.pDb, &csr.blob); - - db->treehdr.root.nHeight++; - } - - return rc; -} - -static int treeRepairList(lsm_db *db){ - int rc = LSM_OK; - int i; - ShmChunk *p; - ShmChunk *pMin = 0; - u32 iMin = 0; - - /* Iterate through all shm chunks. Find the smallest shm-id present in - ** the shared-memory region. */ - for(i=1; rc==LSM_OK && (u32)itreehdr.nChunk; i++){ - p = treeShmChunkRc(db, i, &rc); - if( p && (pMin==0 || shm_sequence_ge(pMin->iShmid, p->iShmid)) ){ - pMin = p; - iMin = i; - } - } - - /* Fix the shm-id values on any chunks with a shm-id greater than or - ** equal to treehdr.iNextShmid. Then do a merge-sort of all chunks to - ** fix the ShmChunk.iNext pointers. - */ - if( rc==LSM_OK ){ - int nSort; - int nByte; - u32 iPrevShmid; - ShmChunkLoc *aSort; - - /* Allocate space for a merge sort. */ - nSort = 1; - while( (u32)nSort < (db->treehdr.nChunk-1) ) nSort = nSort * 2; - nByte = sizeof(ShmChunkLoc) * nSort * 2; - aSort = lsmMallocZeroRc(db->pEnv, nByte, &rc); - iPrevShmid = pMin->iShmid; - - /* Fix all shm-ids, if required. */ - if( rc==LSM_OK ){ - iPrevShmid = pMin->iShmid-1; - for(i=1; (u32)itreehdr.nChunk; i++){ - p = treeShmChunk(db, i); - aSort[i-1].pShm = p; - aSort[i-1].iLoc = i; - if( (u32)i!=db->treehdr.iFirst ){ - if( shm_sequence_ge(p->iShmid, db->treehdr.iNextShmid) ){ - p->iShmid = iPrevShmid--; - } - } - } - if( iMin!=db->treehdr.iFirst ){ - p = treeShmChunk(db, db->treehdr.iFirst); - p->iShmid = iPrevShmid; - } - } - - if( rc==LSM_OK ){ - ShmChunkLoc *aSpace = &aSort[nSort]; - for(i=0; iiShmid, iPrevShmid) ); - assert( aSpace[aSort[i].pShm->iShmid - iPrevShmid].pShm==0 ); - aSpace[aSort[i].pShm->iShmid - iPrevShmid] = aSort[i]; - } - } - - if( aSpace[nSort-1].pShm ) aSpace[nSort-1].pShm->iNext = 0; - for(i=0; iiNext = aSpace[i+1].iLoc; - } - } - - rc = treeCheckLinkedList(db); - lsmFree(db->pEnv, aSort); - } - } - - return rc; -} - -/* -** This function is called as part of opening a write-transaction if the -** writer-flag is already set - indicating that the previous writer -** failed before ending its transaction. -*/ -int lsmTreeRepair(lsm_db *db){ - int rc = LSM_OK; - TreeHeader hdr; - ShmHeader *pHdr = db->pShmhdr; - - /* Ensure that the two tree-headers are consistent. Copy one over the other - ** if necessary. Prefer the data from a tree-header for which the checksum - ** computes. Or, if they both compute, prefer tree-header-1. */ - if( memcmp(&pHdr->hdr1, &pHdr->hdr2, sizeof(TreeHeader)) ){ - if( treeHeaderChecksumOk(&pHdr->hdr1) ){ - memcpy(&pHdr->hdr2, &pHdr->hdr1, sizeof(TreeHeader)); - }else{ - memcpy(&pHdr->hdr1, &pHdr->hdr2, sizeof(TreeHeader)); - } - } - - /* Save the connections current copy of the tree-header. It will be - ** restored before returning. */ - memcpy(&hdr, &db->treehdr, sizeof(TreeHeader)); - - /* Walk the tree. Zero any v2 pointers with a transaction-id greater than - ** the transaction-id currently in the tree-headers. */ - rc = treeRepairPtrs(db); - - /* Repair the linked list of shared-memory chunks. */ - if( rc==LSM_OK ){ - rc = treeRepairList(db); - } - - memcpy(&db->treehdr, &hdr, sizeof(TreeHeader)); - return rc; -} - -static void treeOverwriteKey(lsm_db *db, TreeCursor *pCsr, u32 iKey, int *pRc){ - if( *pRc==LSM_OK ){ - TreeRoot *p = &db->treehdr.root; - TreeNode *pNew; - u32 iNew; - TreeNode *pNode = pCsr->apTreeNode[pCsr->iNode]; - int iCell = pCsr->aiCell[pCsr->iNode]; - - /* Create a copy of this node */ - if( (pCsr->iNode>0 && (u32)pCsr->iNode==(p->nHeight-1)) ){ - pNew = copyTreeLeaf(db, (TreeLeaf *)pNode, &iNew, pRc); - }else{ - pNew = copyTreeNode(db, pNode, &iNew, pRc); - } - - if( pNew ){ - /* Modify the value in the new version */ - pNew->aiKeyPtr[iCell] = iKey; - - /* Change the pointer in the parent (if any) to point at the new - ** TreeNode */ - pCsr->iNode--; - treeUpdatePtr(db, pCsr, iNew); - } - } -} - -static int treeNextIsEndDelete(lsm_db *db, TreeCursor *pCsr){ - int iNode = pCsr->iNode; - int iCell = pCsr->aiCell[iNode]+1; - - /* Cursor currently points to a leaf node. */ - assert( (u32)pCsr->iNode==(db->treehdr.root.nHeight-1) ); - - while( iNode>=0 ){ - TreeNode *pNode = pCsr->apTreeNode[iNode]; - if( iCell<3 && pNode->aiKeyPtr[iCell] ){ - int rc = LSM_OK; - TreeKey *pKey = treeShmptr(db, pNode->aiKeyPtr[iCell]); - assert( rc==LSM_OK ); - return ((pKey->flags & LSM_END_DELETE) ? 1 : 0); - } - iNode--; - iCell = pCsr->aiCell[iNode]; - } - - return 0; -} - -static int treePrevIsStartDelete(lsm_db *db, TreeCursor *pCsr){ - int iNode = pCsr->iNode; - - /* Cursor currently points to a leaf node. */ - assert( (u32)pCsr->iNode==(db->treehdr.root.nHeight-1) ); - - while( iNode>=0 ){ - TreeNode *pNode = pCsr->apTreeNode[iNode]; - int iCell = pCsr->aiCell[iNode]-1; - if( iCell>=0 && pNode->aiKeyPtr[iCell] ){ - int rc = LSM_OK; - TreeKey *pKey = treeShmptr(db, pNode->aiKeyPtr[iCell]); - assert( rc==LSM_OK ); - return ((pKey->flags & LSM_START_DELETE) ? 1 : 0); - } - iNode--; - } - - return 0; -} - - -static int treeInsertEntry( - lsm_db *pDb, /* Database handle */ - int flags, /* Flags associated with entry */ - void *pKey, /* Pointer to key data */ - int nKey, /* Size of key data in bytes */ - void *pVal, /* Pointer to value data (or NULL) */ - int nVal /* Bytes in value data (or -ve for delete) */ -){ - int rc = LSM_OK; /* Return Code */ - TreeKey *pTreeKey; /* New key-value being inserted */ - u32 iTreeKey; - TreeRoot *p = &pDb->treehdr.root; - TreeCursor csr; /* Cursor to seek to pKey/nKey */ - int res = 0; /* Result of seek operation on csr */ - - assert( nVal>=0 || pVal==0 ); - assert_tree_looks_ok(LSM_OK, pTree); - assert( flags==LSM_INSERT || flags==LSM_POINT_DELETE - || flags==LSM_START_DELETE || flags==LSM_END_DELETE - ); - assert( (flags & LSM_CONTIGUOUS)==0 ); -#if 0 - dump_tree_contents(pDb, "before"); -#endif - - if( p->iRoot ){ - TreeKey *pRes; /* Key at end of seek operation */ - treeCursorInit(pDb, 0, &csr); - - /* Seek to the leaf (or internal node) that the new key belongs on */ - rc = lsmTreeCursorSeek(&csr, pKey, nKey, &res); - pRes = csrGetKey(&csr, &csr.blob, &rc); - if( rc!=LSM_OK ) return rc; - assert( pRes ); - - if( flags==LSM_START_DELETE ){ - /* When inserting a start-delete-range entry, if the key that - ** occurs immediately before the new entry is already a START_DELETE, - ** then the new entry is not required. */ - if( (res<=0 && (pRes->flags & LSM_START_DELETE)) - || (res>0 && treePrevIsStartDelete(pDb, &csr)) - ){ - goto insert_entry_out; - } - }else if( flags==LSM_END_DELETE ){ - /* When inserting an start-delete-range entry, if the key that - ** occurs immediately after the new entry is already an END_DELETE, - ** then the new entry is not required. */ - if( (res<0 && treeNextIsEndDelete(pDb, &csr)) - || (res>=0 && (pRes->flags & LSM_END_DELETE)) - ){ - goto insert_entry_out; - } - } - - if( res==0 && (flags & (LSM_END_DELETE|LSM_START_DELETE)) ){ - if( pRes->flags & LSM_INSERT ){ - nVal = pRes->nValue; - pVal = TKV_VAL(pRes); - } - flags = flags | pRes->flags; - } - - if( flags & (LSM_INSERT|LSM_POINT_DELETE) ){ - if( (res<0 && (pRes->flags & LSM_START_DELETE)) - || (res>0 && (pRes->flags & LSM_END_DELETE)) - ){ - flags = flags | (LSM_END_DELETE|LSM_START_DELETE); - }else if( res==0 ){ - flags = flags | (pRes->flags & (LSM_END_DELETE|LSM_START_DELETE)); - } - } - }else{ - memset(&csr, 0, sizeof(TreeCursor)); - } - - /* Allocate and populate a new key-value pair structure */ - pTreeKey = newTreeKey(pDb, &iTreeKey, pKey, nKey, pVal, nVal, &rc); - if( rc!=LSM_OK ) return rc; - assert( pTreeKey->flags==0 || pTreeKey->flags==LSM_CONTIGUOUS ); - pTreeKey->flags |= flags; - - if( p->iRoot==0 ){ - /* The tree is completely empty. Add a new root node and install - ** (pKey/nKey) as the middle entry. Even though it is a leaf at the - ** moment, use newTreeNode() to allocate the node (i.e. allocate enough - ** space for the fields used by interior nodes). This is because the - ** treeInsert() routine may convert this node to an interior node. */ - TreeNode *pRoot = newTreeNode(pDb, &p->iRoot, &rc); - if( rc==LSM_OK ){ - assert( p->nHeight==0 ); - pRoot->aiKeyPtr[1] = iTreeKey; - p->nHeight = 1; - } - }else{ - if( res==0 ){ - /* The search found a match within the tree. */ - treeOverwriteKey(pDb, &csr, iTreeKey, &rc); - }else{ - /* The cursor now points to the leaf node into which the new entry should - ** be inserted. There may or may not be a free slot within the leaf for - ** the new key-value pair. - ** - ** iSlot is set to the index of the key within pLeaf that the new key - ** should be inserted to the left of (or to a value 1 greater than the - ** index of the rightmost key if the new key is larger than all keys - ** currently stored in the node). - */ - int iSlot = csr.aiCell[csr.iNode] + (res<0); - if( csr.iNode==0 ){ - rc = treeInsert(pDb, &csr, 0, iTreeKey, 0, iSlot); - }else{ - rc = treeInsertLeaf(pDb, &csr, iTreeKey, iSlot); - } - } - } - -#if 0 - dump_tree_contents(pDb, "after"); -#endif - insert_entry_out: - tblobFree(pDb, &csr.blob); - assert_tree_looks_ok(rc, pTree); - return rc; -} - -/* -** Insert a new entry into the in-memory tree. -** -** If the value of the 5th parameter, nVal, is negative, then a delete-marker -** is inserted into the tree. In this case the value pointer, pVal, must be -** NULL. -*/ -int lsmTreeInsert( - lsm_db *pDb, /* Database handle */ - void *pKey, /* Pointer to key data */ - int nKey, /* Size of key data in bytes */ - void *pVal, /* Pointer to value data (or NULL) */ - int nVal /* Bytes in value data (or -ve for delete) */ -){ - int flags; - if( nVal<0 ){ - flags = LSM_POINT_DELETE; - }else{ - flags = LSM_INSERT; - } - - return treeInsertEntry(pDb, flags, pKey, nKey, pVal, nVal); -} - -static int treeDeleteEntry(lsm_db *db, TreeCursor *pCsr, u32 iNewptr){ - TreeRoot *p = &db->treehdr.root; - TreeNode *pNode = pCsr->apTreeNode[pCsr->iNode]; - int iSlot = pCsr->aiCell[pCsr->iNode]; - int bLeaf; - int rc = LSM_OK; - - assert( pNode->aiKeyPtr[1] ); - assert( pNode->aiKeyPtr[iSlot] ); - assert( iSlot==0 || iSlot==1 || iSlot==2 ); - assert( ((u32)pCsr->iNode==(db->treehdr.root.nHeight-1))==(iNewptr==0) ); - - bLeaf = ((u32)pCsr->iNode==(p->nHeight-1) && p->nHeight>1); - - if( pNode->aiKeyPtr[0] || pNode->aiKeyPtr[2] ){ - /* There are currently at least 2 keys on this node. So just create - ** a new copy of the node with one of the keys removed. If the node - ** happens to be the root node of the tree, allocate an entire - ** TreeNode structure instead of just a TreeLeaf. */ - TreeNode *pNew; - u32 iNew; - - if( bLeaf ){ - pNew = (TreeNode *)newTreeLeaf(db, &iNew, &rc); - }else{ - pNew = newTreeNode(db, &iNew, &rc); - } - if( pNew ){ - int i; - int iOut = 1; - for(i=0; i<4; i++){ - if( i==iSlot ){ - i++; - if( bLeaf==0 ) pNew->aiChildPtr[iOut] = iNewptr; - if( i<3 ) pNew->aiKeyPtr[iOut] = pNode->aiKeyPtr[i]; - iOut++; - }else if( bLeaf || p->nHeight==1 ){ - if( i<3 && pNode->aiKeyPtr[i] ){ - pNew->aiKeyPtr[iOut++] = pNode->aiKeyPtr[i]; - } - }else{ - if( getChildPtr(pNode, WORKING_VERSION, i) ){ - pNew->aiChildPtr[iOut] = getChildPtr(pNode, WORKING_VERSION, i); - if( i<3 ) pNew->aiKeyPtr[iOut] = pNode->aiKeyPtr[i]; - iOut++; - } - } - } - assert( iOut<=4 ); - assert( bLeaf || pNew->aiChildPtr[0]==0 ); - pCsr->iNode--; - rc = treeUpdatePtr(db, pCsr, iNew); - } - - }else if( pCsr->iNode==0 ){ - /* Removing the only key in the root node. iNewptr is the new root. */ - assert( iSlot==1 ); - db->treehdr.root.iRoot = iNewptr; - db->treehdr.root.nHeight--; - - }else{ - /* There is only one key on this node and the node is not the root - ** node. Find a peer for this node. Then redistribute the contents of - ** the peer and the parent cell between the parent and either one or - ** two new nodes. */ - TreeNode *pParent; /* Parent tree node */ - int iPSlot; - u32 iPeer; /* Pointer to peer leaf node */ - int iDir; - TreeNode *pPeer; /* The peer leaf node */ - TreeNode *pNew1; u32 iNew1; /* First new leaf node */ - - assert( iSlot==1 ); - - pParent = pCsr->apTreeNode[pCsr->iNode-1]; - iPSlot = pCsr->aiCell[pCsr->iNode-1]; - - if( iPSlot>0 && getChildPtr(pParent, WORKING_VERSION, iPSlot-1) ){ - iDir = -1; - }else{ - iDir = +1; - } - iPeer = getChildPtr(pParent, WORKING_VERSION, iPSlot+iDir); - pPeer = (TreeNode *)treeShmptr(db, iPeer); - assertIsWorkingChild(db, pNode, pParent, iPSlot); - - /* Allocate the first new leaf node. This is always required. */ - if( bLeaf ){ - pNew1 = (TreeNode *)newTreeLeaf(db, &iNew1, &rc); - }else{ - pNew1 = (TreeNode *)newTreeNode(db, &iNew1, &rc); - } - - if( pPeer->aiKeyPtr[0] && pPeer->aiKeyPtr[2] ){ - /* Peer node is completely full. This means that two new leaf nodes - ** and a new parent node are required. */ - - TreeNode *pNew2; u32 iNew2; /* Second new leaf node */ - TreeNode *pNewP; u32 iNewP; /* New parent node */ - - if( bLeaf ){ - pNew2 = (TreeNode *)newTreeLeaf(db, &iNew2, &rc); - }else{ - pNew2 = (TreeNode *)newTreeNode(db, &iNew2, &rc); - } - pNewP = copyTreeNode(db, pParent, &iNewP, &rc); - - if( iDir==-1 ){ - pNew1->aiKeyPtr[1] = pPeer->aiKeyPtr[0]; - if( bLeaf==0 ){ - pNew1->aiChildPtr[1] = getChildPtr(pPeer, WORKING_VERSION, 0); - pNew1->aiChildPtr[2] = getChildPtr(pPeer, WORKING_VERSION, 1); - } - - pNewP->aiChildPtr[iPSlot-1] = iNew1; - pNewP->aiKeyPtr[iPSlot-1] = pPeer->aiKeyPtr[1]; - pNewP->aiChildPtr[iPSlot] = iNew2; - - pNew2->aiKeyPtr[0] = pPeer->aiKeyPtr[2]; - pNew2->aiKeyPtr[1] = pParent->aiKeyPtr[iPSlot-1]; - if( bLeaf==0 ){ - pNew2->aiChildPtr[0] = getChildPtr(pPeer, WORKING_VERSION, 2); - pNew2->aiChildPtr[1] = getChildPtr(pPeer, WORKING_VERSION, 3); - pNew2->aiChildPtr[2] = iNewptr; - } - }else{ - pNew1->aiKeyPtr[1] = pParent->aiKeyPtr[iPSlot]; - if( bLeaf==0 ){ - pNew1->aiChildPtr[1] = iNewptr; - pNew1->aiChildPtr[2] = getChildPtr(pPeer, WORKING_VERSION, 0); - } - - pNewP->aiChildPtr[iPSlot] = iNew1; - pNewP->aiKeyPtr[iPSlot] = pPeer->aiKeyPtr[0]; - pNewP->aiChildPtr[iPSlot+1] = iNew2; - - pNew2->aiKeyPtr[0] = pPeer->aiKeyPtr[1]; - pNew2->aiKeyPtr[1] = pPeer->aiKeyPtr[2]; - if( bLeaf==0 ){ - pNew2->aiChildPtr[0] = getChildPtr(pPeer, WORKING_VERSION, 1); - pNew2->aiChildPtr[1] = getChildPtr(pPeer, WORKING_VERSION, 2); - pNew2->aiChildPtr[2] = getChildPtr(pPeer, WORKING_VERSION, 3); - } - } - assert( pCsr->iNode>=1 ); - pCsr->iNode -= 2; - if( rc==LSM_OK ){ - assert( pNew1->aiKeyPtr[1] && pNew2->aiKeyPtr[1] ); - rc = treeUpdatePtr(db, pCsr, iNewP); - } - }else{ - int iKOut = 0; - int iPOut = 0; - int i; - - pCsr->iNode--; - - if( iDir==1 ){ - pNew1->aiKeyPtr[iKOut++] = pParent->aiKeyPtr[iPSlot]; - if( bLeaf==0 ) pNew1->aiChildPtr[iPOut++] = iNewptr; - } - for(i=0; i<3; i++){ - if( pPeer->aiKeyPtr[i] ){ - pNew1->aiKeyPtr[iKOut++] = pPeer->aiKeyPtr[i]; - } - } - if( bLeaf==0 ){ - for(i=0; i<4; i++){ - if( getChildPtr(pPeer, WORKING_VERSION, i) ){ - pNew1->aiChildPtr[iPOut++] = getChildPtr(pPeer, WORKING_VERSION, i); - } - } - } - if( iDir==-1 ){ - iPSlot--; - pNew1->aiKeyPtr[iKOut++] = pParent->aiKeyPtr[iPSlot]; - if( bLeaf==0 ) pNew1->aiChildPtr[iPOut++] = iNewptr; - pCsr->aiCell[pCsr->iNode] = (u8)iPSlot; - } - - rc = treeDeleteEntry(db, pCsr, iNew1); - } - } - - return rc; -} - -/* -** Delete a range of keys from the tree structure (i.e. the lsm_delete_range() -** function, not lsm_delete()). -** -** This is a two step process: -** -** 1) Remove all entries currently stored in the tree that have keys -** that fall into the deleted range. -** -** TODO: There are surely good ways to optimize this step - removing -** a range of keys from a b-tree. But for now, this function removes -** them one at a time using the usual approach. -** -** 2) Unless the largest key smaller than or equal to (pKey1/nKey1) is -** already marked as START_DELETE, insert a START_DELETE key. -** Similarly, unless the smallest key greater than or equal to -** (pKey2/nKey2) is already START_END, insert a START_END key. -*/ -int lsmTreeDelete( - lsm_db *db, - void *pKey1, int nKey1, /* Start of range */ - void *pKey2, int nKey2 /* End of range */ -){ - int rc = LSM_OK; - int bDone = 0; - TreeRoot *p = &db->treehdr.root; - TreeBlob blob = {0, 0}; - - /* The range must be sensible - that (key1 < key2). */ - assert( treeKeycmp(pKey1, nKey1, pKey2, nKey2)<0 ); - assert( assert_delete_ranges_match(db) ); - -#if 0 - static int nCall = 0; - printf("\n"); - nCall++; - printf("%d delete %s .. %s\n", nCall, (char *)pKey1, (char *)pKey2); - dump_tree_contents(db, "before delete"); -#endif - - /* Step 1. This loop runs until the tree contains no keys within the - ** range being deleted. Or until an error occurs. */ - while( bDone==0 && rc==LSM_OK ){ - int res; - TreeCursor csr; /* Cursor to seek to first key in range */ - void *pDel; int nDel; /* Key to (possibly) delete this iteration */ -#ifndef NDEBUG - int nEntry = treeCountEntries(db); -#endif - - /* Seek the cursor to the first entry in the tree greater than pKey1. */ - treeCursorInit(db, 0, &csr); - lsmTreeCursorSeek(&csr, pKey1, nKey1, &res); - if( res<=0 && lsmTreeCursorValid(&csr) ) lsmTreeCursorNext(&csr); - - /* If there is no such entry, or if it is greater than pKey2, then the - ** tree now contains no keys in the range being deleted. In this case - ** break out of the loop. */ - bDone = 1; - if( lsmTreeCursorValid(&csr) ){ - lsmTreeCursorKey(&csr, 0, &pDel, &nDel); - if( treeKeycmp(pDel, nDel, pKey2, nKey2)<0 ) bDone = 0; - } - - if( bDone==0 ){ - if( (u32)csr.iNode==(p->nHeight-1) ){ - /* The element to delete already lies on a leaf node */ - rc = treeDeleteEntry(db, &csr, 0); - }else{ - /* 1. Overwrite the current key with a copy of the next key in the - ** tree (key N). - ** - ** 2. Seek to key N (cursor will stop at the internal node copy of - ** N). Move to the next key (original copy of N). Delete - ** this entry. - */ - u32 iKey; - TreeKey *pKey; - int iNode = csr.iNode; - lsmTreeCursorNext(&csr); - assert( (u32)csr.iNode==(p->nHeight-1) ); - - iKey = csr.apTreeNode[csr.iNode]->aiKeyPtr[csr.aiCell[csr.iNode]]; - lsmTreeCursorPrev(&csr); - - treeOverwriteKey(db, &csr, iKey, &rc); - pKey = treeShmkey(db, iKey, TKV_LOADKEY, &blob, &rc); - if( pKey ){ - rc = lsmTreeCursorSeek(&csr, TKV_KEY(pKey), pKey->nKey, &res); - } - if( rc==LSM_OK ){ - assert( res==0 && csr.iNode==iNode ); - rc = lsmTreeCursorNext(&csr); - if( rc==LSM_OK ){ - rc = treeDeleteEntry(db, &csr, 0); - } - } - } - } - - /* Clean up any memory allocated by the cursor. */ - tblobFree(db, &csr.blob); -#if 0 - dump_tree_contents(db, "ddd delete"); -#endif - assert( bDone || treeCountEntries(db)==(nEntry-1) ); - } - -#if 0 - dump_tree_contents(db, "during delete"); -#endif - - /* Now insert the START_DELETE and END_DELETE keys. */ - if( rc==LSM_OK ){ - rc = treeInsertEntry(db, LSM_START_DELETE, pKey1, nKey1, 0, -1); - } -#if 0 - dump_tree_contents(db, "during delete 2"); -#endif - if( rc==LSM_OK ){ - rc = treeInsertEntry(db, LSM_END_DELETE, pKey2, nKey2, 0, -1); - } - -#if 0 - dump_tree_contents(db, "after delete"); -#endif - - tblobFree(db, &blob); - assert( assert_delete_ranges_match(db) ); - return rc; -} - -/* -** Return, in bytes, the amount of memory currently used by the tree -** structure. -*/ -int lsmTreeSize(lsm_db *pDb){ - return pDb->treehdr.root.nByte; -} - -/* -** Open a cursor on the in-memory tree pTree. -*/ -int lsmTreeCursorNew(lsm_db *pDb, int bOld, TreeCursor **ppCsr){ - TreeCursor *pCsr; - *ppCsr = pCsr = lsmMalloc(pDb->pEnv, sizeof(TreeCursor)); - if( pCsr ){ - treeCursorInit(pDb, bOld, pCsr); - return LSM_OK; - } - return LSM_NOMEM_BKPT; -} - -/* -** Close an in-memory tree cursor. -*/ -void lsmTreeCursorDestroy(TreeCursor *pCsr){ - if( pCsr ){ - tblobFree(pCsr->pDb, &pCsr->blob); - lsmFree(pCsr->pDb->pEnv, pCsr); - } -} - -void lsmTreeCursorReset(TreeCursor *pCsr){ - if( pCsr ){ - pCsr->iNode = -1; - pCsr->pSave = 0; - } -} - -#ifndef NDEBUG -static int treeCsrCompare(TreeCursor *pCsr, void *pKey, int nKey, int *pRc){ - TreeKey *p; - int cmp = 0; - assert( pCsr->iNode>=0 ); - p = csrGetKey(pCsr, &pCsr->blob, pRc); - if( p ){ - cmp = treeKeycmp(TKV_KEY(p), p->nKey, pKey, nKey); - } - return cmp; -} -#endif - - -/* -** Attempt to seek the cursor passed as the first argument to key (pKey/nKey) -** in the tree structure. If an exact match for the key is found, leave the -** cursor pointing to it and set *pRes to zero before returning. If an -** exact match cannot be found, do one of the following: -** -** * Leave the cursor pointing to the smallest element in the tree that -** is larger than the key and set *pRes to +1, or -** -** * Leave the cursor pointing to the largest element in the tree that -** is smaller than the key and set *pRes to -1, or -** -** * If the tree is empty, leave the cursor at EOF and set *pRes to -1. -*/ -int lsmTreeCursorSeek(TreeCursor *pCsr, void *pKey, int nKey, int *pRes){ - int rc = LSM_OK; /* Return code */ - lsm_db *pDb = pCsr->pDb; - TreeRoot *pRoot = pCsr->pRoot; - u32 iNodePtr; /* Location of current node in search */ - - /* Discard any saved position data */ - treeCursorRestore(pCsr, 0); - - iNodePtr = pRoot->iRoot; - if( iNodePtr==0 ){ - /* Either an error occurred or the tree is completely empty. */ - assert( rc!=LSM_OK || pRoot->iRoot==0 ); - *pRes = -1; - pCsr->iNode = -1; - }else{ - TreeBlob b = {0, 0}; - int res = 0; /* Result of comparison function */ - int iNode = -1; - while( iNodePtr ){ - TreeNode *pNode; /* Node at location iNodePtr */ - int iTest; /* Index of second key to test (0 or 2) */ - u32 iTreeKey; - TreeKey *pTreeKey; /* Key to compare against */ - - pNode = (TreeNode *)treeShmptrUnsafe(pDb, iNodePtr); - iNode++; - pCsr->apTreeNode[iNode] = pNode; - - /* Compare (pKey/nKey) with the key in the middle slot of B-tree node - ** pNode. The middle slot is never empty. If the comparison is a match, - ** then the search is finished. Break out of the loop. */ - pTreeKey = (TreeKey*)treeShmptrUnsafe(pDb, pNode->aiKeyPtr[1]); - if( !(pTreeKey->flags & LSM_CONTIGUOUS) ){ - pTreeKey = treeShmkey(pDb, pNode->aiKeyPtr[1], TKV_LOADKEY, &b, &rc); - if( rc!=LSM_OK ) break; - } - res = treeKeycmp((void *)&pTreeKey[1], pTreeKey->nKey, pKey, nKey); - if( res==0 ){ - pCsr->aiCell[iNode] = 1; - break; - } - - /* Based on the results of the previous comparison, compare (pKey/nKey) - ** to either the left or right key of the B-tree node, if such a key - ** exists. */ - iTest = (res>0 ? 0 : 2); - iTreeKey = pNode->aiKeyPtr[iTest]; - if( iTreeKey ){ - pTreeKey = (TreeKey*)treeShmptrUnsafe(pDb, iTreeKey); - if( !(pTreeKey->flags & LSM_CONTIGUOUS) ){ - pTreeKey = treeShmkey(pDb, iTreeKey, TKV_LOADKEY, &b, &rc); - if( rc ) break; - } - res = treeKeycmp((void *)&pTreeKey[1], pTreeKey->nKey, pKey, nKey); - if( res==0 ){ - pCsr->aiCell[iNode] = (u8)iTest; - break; - } - }else{ - iTest = 1; - } - - if( (u32)iNode<(pRoot->nHeight-1) ){ - iNodePtr = getChildPtr(pNode, pRoot->iTransId, iTest + (res<0)); - }else{ - iNodePtr = 0; - } - pCsr->aiCell[iNode] = (u8)(iTest + (iNodePtr && (res<0))); - } - - *pRes = res; - pCsr->iNode = iNode; - tblobFree(pDb, &b); - } - - /* assert() that *pRes has been set properly */ -#ifndef NDEBUG - if( rc==LSM_OK && lsmTreeCursorValid(pCsr) ){ - int cmp = treeCsrCompare(pCsr, pKey, nKey, &rc); - assert( rc!=LSM_OK || *pRes==cmp || (*pRes ^ cmp)>0 ); - } -#endif - - return rc; -} - -int lsmTreeCursorNext(TreeCursor *pCsr){ -#ifndef NDEBUG - TreeKey *pK1; - TreeBlob key1 = {0, 0}; -#endif - lsm_db *pDb = pCsr->pDb; - TreeRoot *pRoot = pCsr->pRoot; - const int iLeaf = pRoot->nHeight-1; - int iCell; - int rc = LSM_OK; - TreeNode *pNode; - - /* Restore the cursor position, if required */ - int iRestore = 0; - treeCursorRestore(pCsr, &iRestore); - if( iRestore>0 ) return LSM_OK; - - /* Save a pointer to the current key. This is used in an assert() at the - ** end of this function - to check that the 'next' key really is larger - ** than the current key. */ -#ifndef NDEBUG - pK1 = csrGetKey(pCsr, &key1, &rc); - if( rc!=LSM_OK ) return rc; -#endif - - assert( lsmTreeCursorValid(pCsr) ); - assert( pCsr->aiCell[pCsr->iNode]<3 ); - - pNode = pCsr->apTreeNode[pCsr->iNode]; - iCell = ++pCsr->aiCell[pCsr->iNode]; - - /* If the current node is not a leaf, and the current cell has sub-tree - ** associated with it, descend to the left-most key on the left-most - ** leaf of the sub-tree. */ - if( pCsr->iNodeiTransId, iCell) ){ - do { - u32 iNodePtr; - pCsr->iNode++; - iNodePtr = getChildPtr(pNode, pRoot->iTransId, iCell); - pNode = (TreeNode *)treeShmptr(pDb, iNodePtr); - pCsr->apTreeNode[pCsr->iNode] = pNode; - iCell = pCsr->aiCell[pCsr->iNode] = (pNode->aiKeyPtr[0]==0); - }while( pCsr->iNode < iLeaf ); - } - - /* Otherwise, the next key is found by following pointer up the tree - ** until there is a key immediately to the right of the pointer followed - ** to reach the sub-tree containing the current key. */ - else if( iCell>=3 || pNode->aiKeyPtr[iCell]==0 ){ - while( (--pCsr->iNode)>=0 ){ - iCell = pCsr->aiCell[pCsr->iNode]; - if( iCell<3 && pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[iCell] ) break; - } - } - -#ifndef NDEBUG - if( pCsr->iNode>=0 ){ - TreeKey *pK2 = csrGetKey(pCsr, &pCsr->blob, &rc); - assert( rc||treeKeycmp(TKV_KEY(pK2),pK2->nKey,TKV_KEY(pK1),pK1->nKey)>=0 ); - } - tblobFree(pDb, &key1); -#endif - - return rc; -} - -int lsmTreeCursorPrev(TreeCursor *pCsr){ -#ifndef NDEBUG - TreeKey *pK1; - TreeBlob key1 = {0, 0}; -#endif - lsm_db *pDb = pCsr->pDb; - TreeRoot *pRoot = pCsr->pRoot; - const int iLeaf = pRoot->nHeight-1; - int iCell; - int rc = LSM_OK; - TreeNode *pNode; - - /* Restore the cursor position, if required */ - int iRestore = 0; - treeCursorRestore(pCsr, &iRestore); - if( iRestore<0 ) return LSM_OK; - - /* Save a pointer to the current key. This is used in an assert() at the - ** end of this function - to check that the 'next' key really is smaller - ** than the current key. */ -#ifndef NDEBUG - pK1 = csrGetKey(pCsr, &key1, &rc); - if( rc!=LSM_OK ) return rc; -#endif - - assert( lsmTreeCursorValid(pCsr) ); - pNode = pCsr->apTreeNode[pCsr->iNode]; - iCell = pCsr->aiCell[pCsr->iNode]; - assert( iCell>=0 && iCell<3 ); - - /* If the current node is not a leaf, and the current cell has sub-tree - ** associated with it, descend to the right-most key on the right-most - ** leaf of the sub-tree. */ - if( pCsr->iNodeiTransId, iCell) ){ - do { - u32 iNodePtr; - pCsr->iNode++; - iNodePtr = getChildPtr(pNode, pRoot->iTransId, iCell); - pNode = (TreeNode *)treeShmptr(pDb, iNodePtr); - if( rc!=LSM_OK ) break; - pCsr->apTreeNode[pCsr->iNode] = pNode; - iCell = 1 + (pNode->aiKeyPtr[2]!=0) + (pCsr->iNode < iLeaf); - pCsr->aiCell[pCsr->iNode] = (u8)iCell; - }while( pCsr->iNode < iLeaf ); - } - - /* Otherwise, the next key is found by following pointer up the tree until - ** there is a key immediately to the left of the pointer followed to reach - ** the sub-tree containing the current key. */ - else{ - do { - iCell = pCsr->aiCell[pCsr->iNode]-1; - if( iCell>=0 && pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[iCell] ) break; - }while( (--pCsr->iNode)>=0 ); - pCsr->aiCell[pCsr->iNode] = (u8)iCell; - } - -#ifndef NDEBUG - if( pCsr->iNode>=0 ){ - TreeKey *pK2 = csrGetKey(pCsr, &pCsr->blob, &rc); - assert( rc || treeKeycmp(TKV_KEY(pK2),pK2->nKey,TKV_KEY(pK1),pK1->nKey)<0 ); - } - tblobFree(pDb, &key1); -#endif - - return rc; -} - -/* -** Move the cursor to the first (bLast==0) or last (bLast!=0) entry in the -** in-memory tree. -*/ -int lsmTreeCursorEnd(TreeCursor *pCsr, int bLast){ - lsm_db *pDb = pCsr->pDb; - TreeRoot *pRoot = pCsr->pRoot; - int rc = LSM_OK; - - u32 iNodePtr; - pCsr->iNode = -1; - - /* Discard any saved position data */ - treeCursorRestore(pCsr, 0); - - iNodePtr = pRoot->iRoot; - while( iNodePtr ){ - int iCell; - TreeNode *pNode; - - pNode = (TreeNode *)treeShmptr(pDb, iNodePtr); - if( rc ) break; - - if( bLast ){ - iCell = ((pNode->aiKeyPtr[2]==0) ? 2 : 3); - }else{ - iCell = ((pNode->aiKeyPtr[0]==0) ? 1 : 0); - } - pCsr->iNode++; - pCsr->apTreeNode[pCsr->iNode] = pNode; - - if( (u32)pCsr->iNodenHeight-1 ){ - iNodePtr = getChildPtr(pNode, pRoot->iTransId, iCell); - }else{ - iNodePtr = 0; - } - pCsr->aiCell[pCsr->iNode] = (u8)(iCell - (iNodePtr==0 && bLast)); - } - - return rc; -} - -int lsmTreeCursorFlags(TreeCursor *pCsr){ - int flags = 0; - if( pCsr && pCsr->iNode>=0 ){ - int rc = LSM_OK; - TreeKey *pKey = (TreeKey *)treeShmptrUnsafe(pCsr->pDb, - pCsr->apTreeNode[pCsr->iNode]->aiKeyPtr[pCsr->aiCell[pCsr->iNode]] - ); - assert( rc==LSM_OK ); - flags = (pKey->flags & ~LSM_CONTIGUOUS); - } - return flags; -} - -int lsmTreeCursorKey(TreeCursor *pCsr, int *pFlags, void **ppKey, int *pnKey){ - TreeKey *pTreeKey; - int rc = LSM_OK; - - assert( lsmTreeCursorValid(pCsr) ); - - pTreeKey = pCsr->pSave; - if( !pTreeKey ){ - pTreeKey = csrGetKey(pCsr, &pCsr->blob, &rc); - } - if( rc==LSM_OK ){ - *pnKey = pTreeKey->nKey; - if( pFlags ) *pFlags = pTreeKey->flags; - *ppKey = (void *)&pTreeKey[1]; - } - - return rc; -} - -int lsmTreeCursorValue(TreeCursor *pCsr, void **ppVal, int *pnVal){ - int res = 0; - int rc; - - rc = treeCursorRestore(pCsr, &res); - if( res==0 ){ - TreeKey *pTreeKey = csrGetKey(pCsr, &pCsr->blob, &rc); - if( rc==LSM_OK ){ - if( pTreeKey->flags & LSM_INSERT ){ - *pnVal = pTreeKey->nValue; - *ppVal = TKV_VAL(pTreeKey); - }else{ - *ppVal = 0; - *pnVal = -1; - } - } - }else{ - *ppVal = 0; - *pnVal = 0; - } - - return rc; -} - -/* -** Return true if the cursor currently points to a valid entry. -*/ -int lsmTreeCursorValid(TreeCursor *pCsr){ - return (pCsr && (pCsr->pSave || pCsr->iNode>=0)); -} - -/* -** Store a mark in *pMark. Later on, a call to lsmTreeRollback() with a -** pointer to the same TreeMark structure may be used to roll the tree -** contents back to their current state. -*/ -void lsmTreeMark(lsm_db *pDb, TreeMark *pMark){ - pMark->iRoot = pDb->treehdr.root.iRoot; - pMark->nHeight = pDb->treehdr.root.nHeight; - pMark->iWrite = pDb->treehdr.iWrite; - pMark->nChunk = pDb->treehdr.nChunk; - pMark->iNextShmid = pDb->treehdr.iNextShmid; - pMark->iRollback = intArraySize(&pDb->rollback); -} - -/* -** Roll back to mark pMark. Structure *pMark should have been previously -** populated by a call to lsmTreeMark(). -*/ -void lsmTreeRollback(lsm_db *pDb, TreeMark *pMark){ - int iIdx; - int nIdx; - u32 iNext; - ShmChunk *pChunk; - u32 iChunk; - u32 iShmid; - - /* Revert all required v2 pointers. */ - nIdx = intArraySize(&pDb->rollback); - for(iIdx = pMark->iRollback; iIdxrollback, iIdx)); - assert( pNode ); - pNode->iV2 = 0; - pNode->iV2Child = 0; - pNode->iV2Ptr = 0; - } - intArrayTruncate(&pDb->rollback, pMark->iRollback); - - /* Restore the free-chunk list. */ - assert( pMark->iWrite!=0 ); - iChunk = treeOffsetToChunk(pMark->iWrite-1); - pChunk = treeShmChunk(pDb, iChunk); - iNext = pChunk->iNext; - pChunk->iNext = 0; - - pChunk = treeShmChunk(pDb, pDb->treehdr.iFirst); - iShmid = pChunk->iShmid-1; - - while( iNext ){ - u32 iFree = iNext; /* Current chunk being rollback-freed */ - ShmChunk *pFree; /* Pointer to chunk iFree */ - - pFree = treeShmChunk(pDb, iFree); - iNext = pFree->iNext; - - if( iFreenChunk ){ - pFree->iNext = pDb->treehdr.iFirst; - pFree->iShmid = iShmid--; - pDb->treehdr.iFirst = iFree; - } - } - - /* Restore the tree-header fields */ - pDb->treehdr.root.iRoot = pMark->iRoot; - pDb->treehdr.root.nHeight = pMark->nHeight; - pDb->treehdr.iWrite = pMark->iWrite; - pDb->treehdr.nChunk = pMark->nChunk; - pDb->treehdr.iNextShmid = pMark->iNextShmid; -} - -/* -** Load the in-memory tree header from shared-memory into pDb->treehdr. -** If the header cannot be loaded, return LSM_PROTOCOL. -** -** If the header is successfully loaded and parameter piRead is not NULL, -** is is set to 1 if the header was loaded from ShmHeader.hdr1, or 2 if -** the header was loaded from ShmHeader.hdr2. -*/ -int lsmTreeLoadHeader(lsm_db *pDb, int *piRead){ - int nRem = LSM_ATTEMPTS_BEFORE_PROTOCOL; - while( (nRem--)>0 ){ - ShmHeader *pShm = pDb->pShmhdr; - - memcpy(&pDb->treehdr, &pShm->hdr1, sizeof(TreeHeader)); - if( treeHeaderChecksumOk(&pDb->treehdr) ){ - if( piRead ) *piRead = 1; - return LSM_OK; - } - memcpy(&pDb->treehdr, &pShm->hdr2, sizeof(TreeHeader)); - if( treeHeaderChecksumOk(&pDb->treehdr) ){ - if( piRead ) *piRead = 2; - return LSM_OK; - } - - lsmShmBarrier(pDb); - } - return LSM_PROTOCOL_BKPT; -} - -int lsmTreeLoadHeaderOk(lsm_db *pDb, int iRead){ - TreeHeader *p = (iRead==1) ? &pDb->pShmhdr->hdr1 : &pDb->pShmhdr->hdr2; - assert( iRead==1 || iRead==2 ); - return (0==memcmp(pDb->treehdr.aCksum, p->aCksum, sizeof(u32)*2)); -} - -/* -** This function is called to conclude a transaction. If argument bCommit -** is true, the transaction is committed. Otherwise it is rolled back. -*/ -int lsmTreeEndTransaction(lsm_db *pDb, int bCommit){ - ShmHeader *pShm = pDb->pShmhdr; - - treeHeaderChecksum(&pDb->treehdr, pDb->treehdr.aCksum); - memcpy(&pShm->hdr2, &pDb->treehdr, sizeof(TreeHeader)); - lsmShmBarrier(pDb); - memcpy(&pShm->hdr1, &pDb->treehdr, sizeof(TreeHeader)); - pShm->bWriter = 0; - intArrayFree(pDb->pEnv, &pDb->rollback); - - return LSM_OK; -} - -#ifndef NDEBUG -static int assert_delete_ranges_match(lsm_db *db){ - int prev = 0; - TreeBlob blob = {0, 0}; - TreeCursor csr; /* Cursor used to iterate through tree */ - int rc; - - treeCursorInit(db, 0, &csr); - for( rc = lsmTreeCursorEnd(&csr, 0); - rc==LSM_OK && lsmTreeCursorValid(&csr); - rc = lsmTreeCursorNext(&csr) - ){ - TreeKey *pKey = csrGetKey(&csr, &blob, &rc); - if( rc!=LSM_OK ) break; - assert( ((prev&LSM_START_DELETE)==0)==((pKey->flags&LSM_END_DELETE)==0) ); - prev = pKey->flags; - } - - tblobFree(csr.pDb, &csr.blob); - tblobFree(csr.pDb, &blob); - - return 1; -} - -static int treeCountEntries(lsm_db *db){ - TreeCursor csr; /* Cursor used to iterate through tree */ - int rc; - int nEntry = 0; - - treeCursorInit(db, 0, &csr); - for( rc = lsmTreeCursorEnd(&csr, 0); - rc==LSM_OK && lsmTreeCursorValid(&csr); - rc = lsmTreeCursorNext(&csr) - ){ - nEntry++; - } - - tblobFree(csr.pDb, &csr.blob); - - return nEntry; -} -#endif diff --git a/ext/lsm1/lsm_unix.c b/ext/lsm1/lsm_unix.c deleted file mode 100644 index 88952d15f..000000000 --- a/ext/lsm1/lsm_unix.c +++ /dev/null @@ -1,753 +0,0 @@ -/* -** 2011-12-03 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Unix-specific run-time environment implementation for LSM. -*/ - -#ifndef _WIN32 - -#if defined(__GNUC__) || defined(__TINYC__) -/* workaround for ftruncate() visibility on gcc. */ -# ifndef _XOPEN_SOURCE -# define _XOPEN_SOURCE 500 -# endif -#endif - -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include - -#include -#include "lsmInt.h" - -/* There is no fdatasync() call on Android */ -#ifdef __ANDROID__ -# define fdatasync(x) fsync(x) -#endif - -/* -** An open file is an instance of the following object -*/ -typedef struct PosixFile PosixFile; -struct PosixFile { - lsm_env *pEnv; /* The run-time environment */ - const char *zName; /* Full path to file */ - int fd; /* The open file descriptor */ - int shmfd; /* Shared memory file-descriptor */ - void *pMap; /* Pointer to mapping of file fd */ - off_t nMap; /* Size of mapping at pMap in bytes */ - int nShm; /* Number of entries in array apShm[] */ - void **apShm; /* Array of 32K shared memory segments */ -}; - -static char *posixShmFile(PosixFile *p){ - char *zShm; - int nName = strlen(p->zName); - zShm = (char *)lsmMalloc(p->pEnv, nName+4+1); - if( zShm ){ - memcpy(zShm, p->zName, nName); - memcpy(&zShm[nName], "-shm", 5); - } - return zShm; -} - -static int lsmPosixOsOpen( - lsm_env *pEnv, - const char *zFile, - int flags, - lsm_file **ppFile -){ - int rc = LSM_OK; - PosixFile *p; - - p = lsm_malloc(pEnv, sizeof(PosixFile)); - if( p==0 ){ - rc = LSM_NOMEM; - }else{ - int bReadonly = (flags & LSM_OPEN_READONLY); - int oflags = (bReadonly ? O_RDONLY : (O_RDWR|O_CREAT)); - memset(p, 0, sizeof(PosixFile)); - p->zName = zFile; - p->pEnv = pEnv; - p->fd = open(zFile, oflags, 0644); - if( p->fd<0 ){ - lsm_free(pEnv, p); - p = 0; - if( errno==ENOENT ){ - rc = lsmErrorBkpt(LSM_IOERR_NOENT); - }else{ - rc = LSM_IOERR_BKPT; - } - } - } - - *ppFile = (lsm_file *)p; - return rc; -} - -static int lsmPosixOsWrite( - lsm_file *pFile, /* File to write to */ - lsm_i64 iOff, /* Offset to write to */ - void *pData, /* Write data from this buffer */ - int nData /* Bytes of data to write */ -){ - int rc = LSM_OK; - PosixFile *p = (PosixFile *)pFile; - off_t offset; - - offset = lseek(p->fd, (off_t)iOff, SEEK_SET); - if( offset!=iOff ){ - rc = LSM_IOERR_BKPT; - }else{ - ssize_t prc = write(p->fd, pData, (size_t)nData); - if( prc<0 ) rc = LSM_IOERR_BKPT; - } - - return rc; -} - -static int lsmPosixOsTruncate( - lsm_file *pFile, /* File to write to */ - lsm_i64 nSize /* Size to truncate file to */ -){ - PosixFile *p = (PosixFile *)pFile; - int rc = LSM_OK; /* Return code */ - int prc; /* Posix Return Code */ - struct stat sStat; /* Result of fstat() invocation */ - - prc = fstat(p->fd, &sStat); - if( prc==0 && sStat.st_size>nSize ){ - prc = ftruncate(p->fd, (off_t)nSize); - } - if( prc<0 ) rc = LSM_IOERR_BKPT; - - return rc; -} - -static int lsmPosixOsRead( - lsm_file *pFile, /* File to read from */ - lsm_i64 iOff, /* Offset to read from */ - void *pData, /* Read data into this buffer */ - int nData /* Bytes of data to read */ -){ - int rc = LSM_OK; - PosixFile *p = (PosixFile *)pFile; - off_t offset; - - offset = lseek(p->fd, (off_t)iOff, SEEK_SET); - if( offset!=iOff ){ - rc = LSM_IOERR_BKPT; - }else{ - ssize_t prc = read(p->fd, pData, (size_t)nData); - if( prc<0 ){ - rc = LSM_IOERR_BKPT; - }else if( prcpMap ){ - prc = msync(p->pMap, p->nMap, MS_SYNC); - } - if( prc==0 ) prc = fdatasync(p->fd); - if( prc<0 ) rc = LSM_IOERR_BKPT; -#else - (void)pFile; -#endif - - return rc; -} - -static int lsmPosixOsSectorSize(lsm_file *pFile){ - return 512; -} - -static int lsmPosixOsRemap( - lsm_file *pFile, - lsm_i64 iMin, - void **ppOut, - lsm_i64 *pnOut -){ - off_t iSz; - int prc; - PosixFile *p = (PosixFile *)pFile; - struct stat buf; - - /* If the file is between 0 and 2MB in size, extend it in chunks of 256K. - ** Thereafter, in chunks of 1MB at a time. */ - const int aIncrSz[] = {256*1024, 1024*1024}; - int nIncrSz = aIncrSz[iMin>(2*1024*1024)]; - - if( p->pMap ){ - munmap(p->pMap, p->nMap); - *ppOut = p->pMap = 0; - *pnOut = p->nMap = 0; - } - - if( iMin>=0 ){ - memset(&buf, 0, sizeof(buf)); - prc = fstat(p->fd, &buf); - if( prc!=0 ) return LSM_IOERR_BKPT; - iSz = buf.st_size; - if( iSzfd, iSz); - if( prc!=0 ) return LSM_IOERR_BKPT; - } - - p->pMap = mmap(0, iSz, PROT_READ|PROT_WRITE, MAP_SHARED, p->fd, 0); - if( p->pMap==MAP_FAILED ){ - p->pMap = 0; - return LSM_IOERR_BKPT; - } - p->nMap = iSz; - } - - *ppOut = p->pMap; - *pnOut = p->nMap; - return LSM_OK; -} - -static int lsmPosixOsFullpath( - lsm_env *pEnv, - const char *zName, - char *zOut, - int *pnOut -){ - int nBuf = *pnOut; - int nReq; - - if( zName[0]!='/' ){ - char *z; - char *zTmp; - int nTmp = 512; - zTmp = lsmMalloc(pEnv, nTmp); - while( zTmp ){ - z = getcwd(zTmp, nTmp); - if( z || errno!=ERANGE ) break; - nTmp = nTmp*2; - zTmp = lsmReallocOrFree(pEnv, zTmp, nTmp); - } - if( zTmp==0 ) return LSM_NOMEM_BKPT; - if( z==0 ) return LSM_IOERR_BKPT; - assert( z==zTmp ); - - nTmp = strlen(zTmp); - nReq = nTmp + 1 + strlen(zName) + 1; - if( nReq<=nBuf ){ - memcpy(zOut, zTmp, nTmp); - zOut[nTmp] = '/'; - memcpy(&zOut[nTmp+1], zName, strlen(zName)+1); - } - lsmFree(pEnv, zTmp); - }else{ - nReq = strlen(zName)+1; - if( nReq<=nBuf ){ - memcpy(zOut, zName, strlen(zName)+1); - } - } - - *pnOut = nReq; - return LSM_OK; -} - -static int lsmPosixOsFileid( - lsm_file *pFile, - void *pBuf, - int *pnBuf -){ - int prc; - int nBuf; - int nReq; - PosixFile *p = (PosixFile *)pFile; - struct stat buf; - - nBuf = *pnBuf; - nReq = (sizeof(buf.st_dev) + sizeof(buf.st_ino)); - *pnBuf = nReq; - if( nReq>nBuf ) return LSM_OK; - - memset(&buf, 0, sizeof(buf)); - prc = fstat(p->fd, &buf); - if( prc!=0 ) return LSM_IOERR_BKPT; - - memcpy(pBuf, &buf.st_dev, sizeof(buf.st_dev)); - memcpy(&(((u8 *)pBuf)[sizeof(buf.st_dev)]), &buf.st_ino, sizeof(buf.st_ino)); - return LSM_OK; -} - -static int lsmPosixOsUnlink(lsm_env *pEnv, const char *zFile){ - int prc = unlink(zFile); - return prc ? LSM_IOERR_BKPT : LSM_OK; -} - -static int lsmPosixOsLock(lsm_file *pFile, int iLock, int eType){ - int rc = LSM_OK; - PosixFile *p = (PosixFile *)pFile; - static const short aType[3] = { F_UNLCK, F_RDLCK, F_WRLCK }; - struct flock lock; - - assert( aType[LSM_LOCK_UNLOCK]==F_UNLCK ); - assert( aType[LSM_LOCK_SHARED]==F_RDLCK ); - assert( aType[LSM_LOCK_EXCL]==F_WRLCK ); - assert( eType>=0 && eType0 && iLock<=32 ); - - memset(&lock, 0, sizeof(lock)); - lock.l_whence = SEEK_SET; - lock.l_len = 1; - lock.l_type = aType[eType]; - lock.l_start = (4096-iLock); - - if( fcntl(p->fd, F_SETLK, &lock) ){ - int e = errno; - if( e==EACCES || e==EAGAIN ){ - rc = LSM_BUSY; - }else{ - rc = LSM_IOERR_BKPT; - } - } - - return rc; -} - -static int lsmPosixOsTestLock(lsm_file *pFile, int iLock, int nLock, int eType){ - int rc = LSM_OK; - PosixFile *p = (PosixFile *)pFile; - static const short aType[3] = { 0, F_RDLCK, F_WRLCK }; - struct flock lock; - - assert( eType==LSM_LOCK_SHARED || eType==LSM_LOCK_EXCL ); - assert( aType[LSM_LOCK_SHARED]==F_RDLCK ); - assert( aType[LSM_LOCK_EXCL]==F_WRLCK ); - assert( eType>=0 && eType0 && iLock<=32 ); - - memset(&lock, 0, sizeof(lock)); - lock.l_whence = SEEK_SET; - lock.l_len = nLock; - lock.l_type = aType[eType]; - lock.l_start = (4096-iLock-nLock+1); - - if( fcntl(p->fd, F_GETLK, &lock) ){ - rc = LSM_IOERR_BKPT; - }else if( lock.l_type!=F_UNLCK ){ - rc = LSM_BUSY; - } - - return rc; -} - -static int lsmPosixOsShmMap(lsm_file *pFile, int iChunk, int sz, void **ppShm){ - PosixFile *p = (PosixFile *)pFile; - - *ppShm = 0; - assert( sz==LSM_SHM_CHUNK_SIZE ); - if( iChunk>=p->nShm ){ - int i; - void **apNew; - int nNew = iChunk+1; - off_t nReq = nNew * LSM_SHM_CHUNK_SIZE; - struct stat sStat; - - /* If the shared-memory file has not been opened, open it now. */ - if( p->shmfd<=0 ){ - char *zShm = posixShmFile(p); - if( !zShm ) return LSM_NOMEM_BKPT; - p->shmfd = open(zShm, O_RDWR|O_CREAT, 0644); - lsmFree(p->pEnv, zShm); - if( p->shmfd<0 ){ - return LSM_IOERR_BKPT; - } - } - - /* If the shared-memory file is not large enough to contain the - ** requested chunk, cause it to grow. */ - if( fstat(p->shmfd, &sStat) ){ - return LSM_IOERR_BKPT; - } - if( sStat.st_sizeshmfd, nReq) ){ - return LSM_IOERR_BKPT; - } - } - - apNew = (void **)lsmRealloc(p->pEnv, p->apShm, sizeof(void *) * nNew); - if( !apNew ) return LSM_NOMEM_BKPT; - for(i=p->nShm; iapShm = apNew; - p->nShm = nNew; - } - - if( p->apShm[iChunk]==0 ){ - p->apShm[iChunk] = mmap(0, LSM_SHM_CHUNK_SIZE, - PROT_READ|PROT_WRITE, MAP_SHARED, p->shmfd, iChunk*LSM_SHM_CHUNK_SIZE - ); - if( p->apShm[iChunk]==MAP_FAILED ){ - p->apShm[iChunk] = 0; - return LSM_IOERR_BKPT; - } - } - - *ppShm = p->apShm[iChunk]; - return LSM_OK; -} - -static void lsmPosixOsShmBarrier(void){ -} - -static int lsmPosixOsShmUnmap(lsm_file *pFile, int bDelete){ - PosixFile *p = (PosixFile *)pFile; - if( p->shmfd>0 ){ - int i; - for(i=0; inShm; i++){ - if( p->apShm[i] ){ - munmap(p->apShm[i], LSM_SHM_CHUNK_SIZE); - p->apShm[i] = 0; - } - } - close(p->shmfd); - p->shmfd = 0; - if( bDelete ){ - char *zShm = posixShmFile(p); - if( zShm ) unlink(zShm); - lsmFree(p->pEnv, zShm); - } - } - return LSM_OK; -} - - -static int lsmPosixOsClose(lsm_file *pFile){ - PosixFile *p = (PosixFile *)pFile; - lsmPosixOsShmUnmap(pFile, 0); - if( p->pMap ) munmap(p->pMap, p->nMap); - close(p->fd); - lsm_free(p->pEnv, p->apShm); - lsm_free(p->pEnv, p); - return LSM_OK; -} - -static int lsmPosixOsSleep(lsm_env *pEnv, int us){ -#if 0 - /* Apparently on Android usleep() returns void */ - if( usleep(us) ) return LSM_IOERR; -#endif - usleep(us); - return LSM_OK; -} - -/**************************************************************************** -** Memory allocation routines. -*/ -#define BLOCK_HDR_SIZE ROUND8( sizeof(size_t) ) - -static void *lsmPosixOsMalloc(lsm_env *pEnv, size_t N){ - unsigned char * m; - N += BLOCK_HDR_SIZE; - m = (unsigned char *)malloc(N); - *((size_t*)m) = N; - return m + BLOCK_HDR_SIZE; -} - -static void lsmPosixOsFree(lsm_env *pEnv, void *p){ - if(p){ - free( ((unsigned char *)p) - BLOCK_HDR_SIZE ); - } -} - -static void *lsmPosixOsRealloc(lsm_env *pEnv, void *p, size_t N){ - unsigned char * m = (unsigned char *)p; - if(1>N){ - lsmPosixOsFree( pEnv, p ); - return NULL; - }else if(NULL==p){ - return lsmPosixOsMalloc(pEnv, N); - }else{ - void * re = NULL; - m -= BLOCK_HDR_SIZE; -#if 0 /* arguable: don't shrink */ - size_t * sz = (size_t*)m; - if(*sz >= (size_t)N){ - return p; - } -#endif - re = realloc( m, N + BLOCK_HDR_SIZE ); - if(re){ - m = (unsigned char *)re; - *((size_t*)m) = N; - return m + BLOCK_HDR_SIZE; - }else{ - return NULL; - } - } -} - -static size_t lsmPosixOsMSize(lsm_env *pEnv, void *p){ - unsigned char * m = (unsigned char *)p; - return *((size_t*)(m-BLOCK_HDR_SIZE)); -} -#undef BLOCK_HDR_SIZE - - -#ifdef LSM_MUTEX_PTHREADS -/************************************************************************* -** Mutex methods for pthreads based systems. If LSM_MUTEX_PTHREADS is -** missing then a no-op implementation of mutexes found in lsm_mutex.c -** will be used instead. -*/ -#include - -typedef struct PthreadMutex PthreadMutex; -struct PthreadMutex { - lsm_env *pEnv; - pthread_mutex_t mutex; -#ifdef LSM_DEBUG - pthread_t owner; -#endif -}; - -#ifdef LSM_DEBUG -# define LSM_PTHREAD_STATIC_MUTEX { 0, PTHREAD_MUTEX_INITIALIZER, 0 } -#else -# define LSM_PTHREAD_STATIC_MUTEX { 0, PTHREAD_MUTEX_INITIALIZER } -#endif - -static int lsmPosixOsMutexStatic( - lsm_env *pEnv, - int iMutex, - lsm_mutex **ppStatic -){ - static PthreadMutex sMutex[2] = { - LSM_PTHREAD_STATIC_MUTEX, - LSM_PTHREAD_STATIC_MUTEX - }; - - assert( iMutex==LSM_MUTEX_GLOBAL || iMutex==LSM_MUTEX_HEAP ); - assert( LSM_MUTEX_GLOBAL==1 && LSM_MUTEX_HEAP==2 ); - - *ppStatic = (lsm_mutex *)&sMutex[iMutex-1]; - return LSM_OK; -} - -static int lsmPosixOsMutexNew(lsm_env *pEnv, lsm_mutex **ppNew){ - PthreadMutex *pMutex; /* Pointer to new mutex */ - pthread_mutexattr_t attr; /* Attributes object */ - - pMutex = (PthreadMutex *)lsmMallocZero(pEnv, sizeof(PthreadMutex)); - if( !pMutex ) return LSM_NOMEM_BKPT; - - pMutex->pEnv = pEnv; - pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); - pthread_mutex_init(&pMutex->mutex, &attr); - pthread_mutexattr_destroy(&attr); - - *ppNew = (lsm_mutex *)pMutex; - return LSM_OK; -} - -static void lsmPosixOsMutexDel(lsm_mutex *p){ - PthreadMutex *pMutex = (PthreadMutex *)p; - pthread_mutex_destroy(&pMutex->mutex); - lsmFree(pMutex->pEnv, pMutex); -} - -static void lsmPosixOsMutexEnter(lsm_mutex *p){ - PthreadMutex *pMutex = (PthreadMutex *)p; - pthread_mutex_lock(&pMutex->mutex); - -#ifdef LSM_DEBUG - assert( !pthread_equal(pMutex->owner, pthread_self()) ); - pMutex->owner = pthread_self(); - assert( pthread_equal(pMutex->owner, pthread_self()) ); -#endif -} - -static int lsmPosixOsMutexTry(lsm_mutex *p){ - int ret; - PthreadMutex *pMutex = (PthreadMutex *)p; - ret = pthread_mutex_trylock(&pMutex->mutex); -#ifdef LSM_DEBUG - if( ret==0 ){ - assert( !pthread_equal(pMutex->owner, pthread_self()) ); - pMutex->owner = pthread_self(); - assert( pthread_equal(pMutex->owner, pthread_self()) ); - } -#endif - return ret; -} - -static void lsmPosixOsMutexLeave(lsm_mutex *p){ - PthreadMutex *pMutex = (PthreadMutex *)p; -#ifdef LSM_DEBUG - assert( pthread_equal(pMutex->owner, pthread_self()) ); - pMutex->owner = 0; - assert( !pthread_equal(pMutex->owner, pthread_self()) ); -#endif - pthread_mutex_unlock(&pMutex->mutex); -} - -#ifdef LSM_DEBUG -static int lsmPosixOsMutexHeld(lsm_mutex *p){ - PthreadMutex *pMutex = (PthreadMutex *)p; - return pMutex ? pthread_equal(pMutex->owner, pthread_self()) : 1; -} -static int lsmPosixOsMutexNotHeld(lsm_mutex *p){ - PthreadMutex *pMutex = (PthreadMutex *)p; - return pMutex ? !pthread_equal(pMutex->owner, pthread_self()) : 1; -} -#endif -/* -** End of pthreads mutex implementation. -*************************************************************************/ -#else -/************************************************************************* -** Noop mutex implementation -*/ -typedef struct NoopMutex NoopMutex; -struct NoopMutex { - lsm_env *pEnv; /* Environment handle (for xFree()) */ - int bHeld; /* True if mutex is held */ - int bStatic; /* True for a static mutex */ -}; -static NoopMutex aStaticNoopMutex[2] = { - {0, 0, 1}, - {0, 0, 1}, -}; - -static int lsmPosixOsMutexStatic( - lsm_env *pEnv, - int iMutex, - lsm_mutex **ppStatic -){ - assert( iMutex>=1 && iMutex<=(int)array_size(aStaticNoopMutex) ); - *ppStatic = (lsm_mutex *)&aStaticNoopMutex[iMutex-1]; - return LSM_OK; -} -static int lsmPosixOsMutexNew(lsm_env *pEnv, lsm_mutex **ppNew){ - NoopMutex *p; - p = (NoopMutex *)lsmMallocZero(pEnv, sizeof(NoopMutex)); - if( p ) p->pEnv = pEnv; - *ppNew = (lsm_mutex *)p; - return (p ? LSM_OK : LSM_NOMEM_BKPT); -} -static void lsmPosixOsMutexDel(lsm_mutex *pMutex) { - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bStatic==0 && p->pEnv ); - lsmFree(p->pEnv, p); -} -static void lsmPosixOsMutexEnter(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bHeld==0 ); - p->bHeld = 1; -} -static int lsmPosixOsMutexTry(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bHeld==0 ); - p->bHeld = 1; - return 0; -} -static void lsmPosixOsMutexLeave(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bHeld==1 ); - p->bHeld = 0; -} -#ifdef LSM_DEBUG -static int lsmPosixOsMutexHeld(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - return p ? p->bHeld : 1; -} -static int lsmPosixOsMutexNotHeld(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - return p ? !p->bHeld : 1; -} -#endif -/***************************************************************************/ -#endif /* else LSM_MUTEX_NONE */ - -/* Without LSM_DEBUG, the MutexHeld tests are never called */ -#ifndef LSM_DEBUG -# define lsmPosixOsMutexHeld 0 -# define lsmPosixOsMutexNotHeld 0 -#endif - -lsm_env *lsm_default_env(void){ - static lsm_env posix_env = { - sizeof(lsm_env), /* nByte */ - 1, /* iVersion */ - /***** file i/o ******************/ - 0, /* pVfsCtx */ - lsmPosixOsFullpath, /* xFullpath */ - lsmPosixOsOpen, /* xOpen */ - lsmPosixOsRead, /* xRead */ - lsmPosixOsWrite, /* xWrite */ - lsmPosixOsTruncate, /* xTruncate */ - lsmPosixOsSync, /* xSync */ - lsmPosixOsSectorSize, /* xSectorSize */ - lsmPosixOsRemap, /* xRemap */ - lsmPosixOsFileid, /* xFileid */ - lsmPosixOsClose, /* xClose */ - lsmPosixOsUnlink, /* xUnlink */ - lsmPosixOsLock, /* xLock */ - lsmPosixOsTestLock, /* xTestLock */ - lsmPosixOsShmMap, /* xShmMap */ - lsmPosixOsShmBarrier, /* xShmBarrier */ - lsmPosixOsShmUnmap, /* xShmUnmap */ - /***** memory allocation *********/ - 0, /* pMemCtx */ - lsmPosixOsMalloc, /* xMalloc */ - lsmPosixOsRealloc, /* xRealloc */ - lsmPosixOsFree, /* xFree */ - lsmPosixOsMSize, /* xSize */ - /***** mutexes *********************/ - 0, /* pMutexCtx */ - lsmPosixOsMutexStatic, /* xMutexStatic */ - lsmPosixOsMutexNew, /* xMutexNew */ - lsmPosixOsMutexDel, /* xMutexDel */ - lsmPosixOsMutexEnter, /* xMutexEnter */ - lsmPosixOsMutexTry, /* xMutexTry */ - lsmPosixOsMutexLeave, /* xMutexLeave */ - lsmPosixOsMutexHeld, /* xMutexHeld */ - lsmPosixOsMutexNotHeld, /* xMutexNotHeld */ - /***** other *********************/ - lsmPosixOsSleep, /* xSleep */ - }; - return &posix_env; -} - -#endif diff --git a/ext/lsm1/lsm_varint.c b/ext/lsm1/lsm_varint.c deleted file mode 100644 index f690e3b06..000000000 --- a/ext/lsm1/lsm_varint.c +++ /dev/null @@ -1,201 +0,0 @@ - -/* -** 2012-02-08 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** SQLite4-compatible varint implementation. -*/ -#include "lsmInt.h" - -/************************************************************************* -** The following is a copy of the varint.c module from SQLite 4. -*/ - -/* -** Decode the varint in z[]. Write the integer value into *pResult and -** return the number of bytes in the varint. -*/ -static int lsmSqlite4GetVarint64(const unsigned char *z, u64 *pResult){ - unsigned int x; - if( z[0]<=240 ){ - *pResult = z[0]; - return 1; - } - if( z[0]<=248 ){ - *pResult = (z[0]-241)*256 + z[1] + 240; - return 2; - } - if( z[0]==249 ){ - *pResult = 2288 + 256*z[1] + z[2]; - return 3; - } - if( z[0]==250 ){ - *pResult = (z[1]<<16) + (z[2]<<8) + z[3]; - return 4; - } - x = (z[1]<<24) + (z[2]<<16) + (z[3]<<8) + z[4]; - if( z[0]==251 ){ - *pResult = x; - return 5; - } - if( z[0]==252 ){ - *pResult = (((u64)x)<<8) + z[5]; - return 6; - } - if( z[0]==253 ){ - *pResult = (((u64)x)<<16) + (z[5]<<8) + z[6]; - return 7; - } - if( z[0]==254 ){ - *pResult = (((u64)x)<<24) + (z[5]<<16) + (z[6]<<8) + z[7]; - return 8; - } - *pResult = (((u64)x)<<32) + - (0xffffffff & ((z[5]<<24) + (z[6]<<16) + (z[7]<<8) + z[8])); - return 9; -} - -/* -** Write a 32-bit unsigned integer as 4 big-endian bytes. -*/ -static void lsmVarintWrite32(unsigned char *z, unsigned int y){ - z[0] = (unsigned char)(y>>24); - z[1] = (unsigned char)(y>>16); - z[2] = (unsigned char)(y>>8); - z[3] = (unsigned char)(y); -} - -/* -** Write a varint into z[]. The buffer z[] must be at least 9 characters -** long to accommodate the largest possible varint. Return the number of -** bytes of z[] used. -*/ -static int lsmSqlite4PutVarint64(unsigned char *z, u64 x){ - unsigned int w, y; - if( x<=240 ){ - z[0] = (unsigned char)x; - return 1; - } - if( x<=2287 ){ - y = (unsigned int)(x - 240); - z[0] = (unsigned char)(y/256 + 241); - z[1] = (unsigned char)(y%256); - return 2; - } - if( x<=67823 ){ - y = (unsigned int)(x - 2288); - z[0] = 249; - z[1] = (unsigned char)(y/256); - z[2] = (unsigned char)(y%256); - return 3; - } - y = (unsigned int)x; - w = (unsigned int)(x>>32); - if( w==0 ){ - if( y<=16777215 ){ - z[0] = 250; - z[1] = (unsigned char)(y>>16); - z[2] = (unsigned char)(y>>8); - z[3] = (unsigned char)(y); - return 4; - } - z[0] = 251; - lsmVarintWrite32(z+1, y); - return 5; - } - if( w<=255 ){ - z[0] = 252; - z[1] = (unsigned char)w; - lsmVarintWrite32(z+2, y); - return 6; - } - if( w<=32767 ){ - z[0] = 253; - z[1] = (unsigned char)(w>>8); - z[2] = (unsigned char)w; - lsmVarintWrite32(z+3, y); - return 7; - } - if( w<=16777215 ){ - z[0] = 254; - z[1] = (unsigned char)(w>>16); - z[2] = (unsigned char)(w>>8); - z[3] = (unsigned char)w; - lsmVarintWrite32(z+4, y); - return 8; - } - z[0] = 255; - lsmVarintWrite32(z+1, w); - lsmVarintWrite32(z+5, y); - return 9; -} - -/* -** End of SQLite 4 code. -*************************************************************************/ - -int lsmVarintPut64(u8 *aData, i64 iVal){ - return lsmSqlite4PutVarint64(aData, (u64)iVal); -} - -int lsmVarintGet64(const u8 *aData, i64 *piVal){ - return lsmSqlite4GetVarint64(aData, (u64 *)piVal); -} - -int lsmVarintPut32(u8 *aData, int iVal){ - return lsmSqlite4PutVarint64(aData, (u64)iVal); -} - -int lsmVarintGet32(u8 *z, int *piVal){ - u64 i; - int ret; - - if( z[0]<=240 ){ - *piVal = z[0]; - return 1; - } - if( z[0]<=248 ){ - *piVal = (z[0]-241)*256 + z[1] + 240; - return 2; - } - if( z[0]==249 ){ - *piVal = 2288 + 256*z[1] + z[2]; - return 3; - } - if( z[0]==250 ){ - *piVal = (z[1]<<16) + (z[2]<<8) + z[3]; - return 4; - } - - ret = lsmSqlite4GetVarint64(z, &i); - *piVal = (int)i; - return ret; -} - -int lsmVarintLen32(int n){ - u8 aData[9]; - return lsmVarintPut32(aData, n); -} - -int lsmVarintLen64(i64 n){ - u8 aData[9]; - return lsmVarintPut64(aData, n); -} - -/* -** The argument is the first byte of a varint. This function returns the -** total number of bytes in the entire varint (including the first byte). -*/ -int lsmVarintSize(u8 c){ - if( c<241 ) return 1; - if( c<249 ) return 2; - return (int)(c - 246); -} diff --git a/ext/lsm1/lsm_vtab.c b/ext/lsm1/lsm_vtab.c deleted file mode 100644 index 8c21923e1..000000000 --- a/ext/lsm1/lsm_vtab.c +++ /dev/null @@ -1,1084 +0,0 @@ -/* -** 2015-11-16 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file implements a virtual table for SQLite3 around the LSM -** storage engine from SQLite4. -** -** USAGE -** -** CREATE VIRTUAL TABLE demo USING lsm1(filename,key,keytype,value1,...); -** -** The filename parameter is the name of the LSM database file, which is -** separate and distinct from the SQLite3 database file. -** -** The keytype must be one of: UINT, TEXT, BLOB. All keys must be of that -** one type. "UINT" means unsigned integer. The values may be of any -** SQLite datatype: BLOB, TEXT, INTEGER, FLOAT, or NULL. -** -** The virtual table contains read-only hidden columns: -** -** lsm1_key A BLOB which is the raw LSM key. If the "keytype" -** is BLOB or TEXT then this column is exactly the -** same as the key. For the UINT keytype, this column -** will be a variable-length integer encoding of the key. -** -** lsm1_value A BLOB which is the raw LSM value. All of the value -** columns are packed into this BLOB using the encoding -** described below. -** -** Attempts to write values into the lsm1_key and lsm1_value columns are -** silently ignored. -** -** EXAMPLE -** -** The virtual table declared this way: -** -** CREATE VIRTUAL TABLE demo2 USING lsm1('x.lsm',id,UINT,a,b,c,d); -** -** Results in a new virtual table named "demo2" that acts as if it has -** the following schema: -** -** CREATE TABLE demo2( -** id UINT PRIMARY KEY ON CONFLICT REPLACE, -** a ANY, -** b ANY, -** c ANY, -** d ANY, -** lsm1_key BLOB HIDDEN, -** lsm1_value BLOB HIDDEN -** ) WITHOUT ROWID; -** -** -** -** INTERNALS -** -** The key encoding for BLOB and TEXT is just a copy of the blob or text. -** UTF-8 is used for text. The key encoding for UINT is the variable-length -** integer format at https://sqlite.org/src4/doc/trunk/www/varint.wiki. -** -** The values are encoded as a single blob (since that is what lsm stores as -** its content). There is a "type integer" followed by "content" for each -** value, alternating back and forth. The content might be empty. -** -** TYPE1 CONTENT1 TYPE2 CONTENT2 TYPE3 CONTENT3 .... -** -** Each "type integer" is encoded as a variable-length integer in the -** format of the link above. Let the type integer be T. The actual -** datatype is an integer 0-5 equal to T%6. Values 1 through 5 correspond -** to SQLITE_INTEGER through SQLITE_NULL. The size of the content in bytes -** is T/6. Type value 0 means that the value is an integer whose actual -** values is T/6 and there is no content. The type-value-0 integer format -** only works for integers in the range of 0 through 40. -** -** There is no content for NULL or type-0 integers. For BLOB and TEXT -** values, the content is the blob data or the UTF-8 text data. For -** non-negative integers X, the content is a variable-length integer X*2. -** For negative integers Y, the content is varaible-length integer (1-Y)*2+1. -** For FLOAT values, the content is the IEEE754 floating point value in -** native byte-order. This means that FLOAT values will be corrupted when -** database file is moved between big-endian and little-endian machines. -*/ -#include "sqlite3ext.h" -SQLITE_EXTENSION_INIT1 -#include "lsm.h" -#include -#include - -/* Forward declaration of subclasses of virtual table objects */ -typedef struct lsm1_vtab lsm1_vtab; -typedef struct lsm1_cursor lsm1_cursor; -typedef struct lsm1_vblob lsm1_vblob; - -/* Primitive types */ -typedef unsigned char u8; -typedef unsigned int u32; -typedef sqlite3_uint64 u64; - -/* An open connection to an LSM table */ -struct lsm1_vtab { - sqlite3_vtab base; /* Base class - must be first */ - lsm_db *pDb; /* Open connection to the LSM table */ - u8 keyType; /* SQLITE_BLOB, _TEXT, or _INTEGER */ - u32 nVal; /* Number of value columns */ -}; - - -/* lsm1_cursor is a subclass of sqlite3_vtab_cursor which will -** serve as the underlying representation of a cursor that scans -** over rows of the result -*/ -struct lsm1_cursor { - sqlite3_vtab_cursor base; /* Base class - must be first */ - lsm_cursor *pLsmCur; /* The LSM cursor */ - u8 isDesc; /* 0: scan forward. 1: scan reverse */ - u8 atEof; /* True if the scan is complete */ - u8 bUnique; /* True if no more than one row of output */ - u8 *zData; /* Content of the current row */ - u32 nData; /* Number of bytes in the current row */ - u8 *aeType; /* Types for all column values */ - u32 *aiOfst; /* Offsets to the various fields */ - u32 *aiLen; /* Length of each field */ - u8 *pKey2; /* Loop termination key, or NULL */ - u32 nKey2; /* Length of the loop termination key */ -}; - -/* An extensible buffer object. -** -** Content can be appended. Space to hold new content is automatically -** allocated. -*/ -struct lsm1_vblob { - u8 *a; /* Space to hold content, from sqlite3_malloc64() */ - u64 n; /* Bytes of space used */ - u64 nAlloc; /* Bytes of space allocated */ - u8 errNoMem; /* True if a memory allocation error has been seen */ -}; - -#if defined(__GNUC__) -# define LSM1_NOINLINE __attribute__((noinline)) -#elif defined(_MSC_VER) && _MSC_VER>=1310 -# define LSM1_NOINLINE __declspec(noinline) -#else -# define LSM1_NOINLINE -#endif - - -/* Increase the available space in the vblob object so that it can hold -** at least N more bytes. Return the number of errors. -*/ -static int lsm1VblobEnlarge(lsm1_vblob *p, u32 N){ - if( p->n+N>p->nAlloc ){ - if( p->errNoMem ) return 1; - p->nAlloc += N + (p->nAlloc ? p->nAlloc : N); - p->a = sqlite3_realloc64(p->a, p->nAlloc); - if( p->a==0 ){ - p->n = 0; - p->nAlloc = 0; - p->errNoMem = 1; - return 1; - } - p->nAlloc = sqlite3_msize(p->a); - } - return 0; -} - -/* Append N bytes to a vblob after first enlarging it */ -static LSM1_NOINLINE void lsm1VblobEnlargeAndAppend( - lsm1_vblob *p, - const u8 *pData, - u32 N -){ - if( p->n+N>p->nAlloc && lsm1VblobEnlarge(p, N) ) return; - memcpy(p->a+p->n, pData, N); - p->n += N; -} - -/* Append N bytes to a vblob */ -static void lsm1VblobAppend(lsm1_vblob *p, const u8 *pData, u32 N){ - sqlite3_int64 n = p->n; - if( n+N>p->nAlloc ){ - lsm1VblobEnlargeAndAppend(p, pData, N); - }else{ - p->n += N; - memcpy(p->a+n, pData, N); - } -} - -/* append text to a vblob */ -static void lsm1VblobAppendText(lsm1_vblob *p, const char *z){ - lsm1VblobAppend(p, (u8*)z, (u32)strlen(z)); -} - -/* Dequote the string */ -static void lsm1Dequote(char *z){ - int j; - char cQuote = z[0]; - size_t i, n; - - if( cQuote!='\'' && cQuote!='"' ) return; - n = strlen(z); - if( n<2 || z[n-1]!=z[0] ) return; - for(i=1, j=0; ikeyType = keyType; - rc = lsm_new(0, &pNew->pDb); - if( rc ){ - *pzErr = sqlite3_mprintf("lsm_new failed with error code %d", rc); - rc = SQLITE_ERROR; - goto connect_failed; - } - zFilename = sqlite3_mprintf("%s", argv[3]); - lsm1Dequote(zFilename); - rc = lsm_open(pNew->pDb, zFilename); - sqlite3_free(zFilename); - if( rc ){ - *pzErr = sqlite3_mprintf("lsm_open failed with %d", rc); - rc = SQLITE_ERROR; - goto connect_failed; - } - - memset(&sql, 0, sizeof(sql)); - lsm1VblobAppendText(&sql, "CREATE TABLE x("); - lsm1VblobAppendText(&sql, argv[4]); - lsm1VblobAppendText(&sql, " "); - lsm1VblobAppendText(&sql, argv[5]); - lsm1VblobAppendText(&sql, " PRIMARY KEY"); - for(i=6; inVal++; - } - lsm1VblobAppendText(&sql, - ", lsm1_command HIDDEN" - ", lsm1_key HIDDEN" - ", lsm1_value HIDDEN) WITHOUT ROWID"); - lsm1VblobAppend(&sql, (u8*)"", 1); - if( sql.errNoMem ){ - rc = SQLITE_NOMEM; - goto connect_failed; - } - rc = sqlite3_declare_vtab(db, (const char*)sql.a); - sqlite3_free(sql.a); - -connect_failed: - if( rc!=SQLITE_OK ){ - if( pNew ){ - if( pNew->pDb ) lsm_close(pNew->pDb); - sqlite3_free(pNew); - } - *ppVtab = 0; - } - return rc; -} - -/* -** This method is the destructor for lsm1_cursor objects. -*/ -static int lsm1Disconnect(sqlite3_vtab *pVtab){ - lsm1_vtab *p = (lsm1_vtab*)pVtab; - lsm_close(p->pDb); - sqlite3_free(p); - return SQLITE_OK; -} - -/* -** Constructor for a new lsm1_cursor object. -*/ -static int lsm1Open(sqlite3_vtab *pVtab, sqlite3_vtab_cursor **ppCursor){ - lsm1_vtab *p = (lsm1_vtab*)pVtab; - lsm1_cursor *pCur; - int rc; - pCur = sqlite3_malloc64( sizeof(*pCur) - + p->nVal*(sizeof(pCur->aiOfst)+sizeof(pCur->aiLen)+1) ); - if( pCur==0 ) return SQLITE_NOMEM; - memset(pCur, 0, sizeof(*pCur)); - pCur->aiOfst = (u32*)&pCur[1]; - pCur->aiLen = &pCur->aiOfst[p->nVal]; - pCur->aeType = (u8*)&pCur->aiLen[p->nVal]; - *ppCursor = &pCur->base; - rc = lsm_csr_open(p->pDb, &pCur->pLsmCur); - if( rc==LSM_OK ){ - rc = SQLITE_OK; - }else{ - sqlite3_free(pCur); - *ppCursor = 0; - rc = SQLITE_ERROR; - } - return rc; -} - -/* -** Destructor for a lsm1_cursor. -*/ -static int lsm1Close(sqlite3_vtab_cursor *cur){ - lsm1_cursor *pCur = (lsm1_cursor*)cur; - sqlite3_free(pCur->pKey2); - lsm_csr_close(pCur->pLsmCur); - sqlite3_free(pCur); - return SQLITE_OK; -} - - -/* -** Advance a lsm1_cursor to its next row of output. -*/ -static int lsm1Next(sqlite3_vtab_cursor *cur){ - lsm1_cursor *pCur = (lsm1_cursor*)cur; - int rc = LSM_OK; - if( pCur->bUnique ){ - pCur->atEof = 1; - }else{ - if( pCur->isDesc ){ - rc = lsm_csr_prev(pCur->pLsmCur); - }else{ - rc = lsm_csr_next(pCur->pLsmCur); - } - if( rc==LSM_OK && lsm_csr_valid(pCur->pLsmCur)==0 ){ - pCur->atEof = 1; - } - if( pCur->pKey2 && pCur->atEof==0 ){ - const u8 *pVal; - u32 nVal; - assert( pCur->isDesc==0 ); - rc = lsm_csr_key(pCur->pLsmCur, (const void**)&pVal, (int*)&nVal); - if( rc==LSM_OK ){ - u32 len = pCur->nKey2; - int c; - if( len>nVal ) len = nVal; - c = memcmp(pVal, pCur->pKey2, len); - if( c==0 ) c = nVal - pCur->nKey2; - if( c>0 ) pCur->atEof = 1; - } - } - pCur->zData = 0; - } - return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; -} - -/* -** Return TRUE if the cursor has been moved off of the last -** row of output. -*/ -static int lsm1Eof(sqlite3_vtab_cursor *cur){ - lsm1_cursor *pCur = (lsm1_cursor*)cur; - return pCur->atEof; -} - -/* -** Rowids are not supported by the underlying virtual table. So always -** return 0 for the rowid. -*/ -static int lsm1Rowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ - *pRowid = 0; - return SQLITE_OK; -} - -/* -** Type prefixes on LSM keys -*/ -#define LSM1_TYPE_NEGATIVE 0 -#define LSM1_TYPE_POSITIVE 1 -#define LSM1_TYPE_TEXT 2 -#define LSM1_TYPE_BLOB 3 - -/* -** Write a 32-bit unsigned integer as 4 big-endian bytes. -*/ -static void varintWrite32(unsigned char *z, unsigned int y){ - z[0] = (unsigned char)(y>>24); - z[1] = (unsigned char)(y>>16); - z[2] = (unsigned char)(y>>8); - z[3] = (unsigned char)(y); -} - -/* -** Write a varint into z[]. The buffer z[] must be at least 9 characters -** long to accommodate the largest possible varint. Return the number of -** bytes of z[] used. -*/ -static int lsm1PutVarint64(unsigned char *z, sqlite3_uint64 x){ - unsigned int w, y; - if( x<=240 ){ - z[0] = (unsigned char)x; - return 1; - } - if( x<=2287 ){ - y = (unsigned int)(x - 240); - z[0] = (unsigned char)(y/256 + 241); - z[1] = (unsigned char)(y%256); - return 2; - } - if( x<=67823 ){ - y = (unsigned int)(x - 2288); - z[0] = 249; - z[1] = (unsigned char)(y/256); - z[2] = (unsigned char)(y%256); - return 3; - } - y = (unsigned int)x; - w = (unsigned int)(x>>32); - if( w==0 ){ - if( y<=16777215 ){ - z[0] = 250; - z[1] = (unsigned char)(y>>16); - z[2] = (unsigned char)(y>>8); - z[3] = (unsigned char)(y); - return 4; - } - z[0] = 251; - varintWrite32(z+1, y); - return 5; - } - if( w<=255 ){ - z[0] = 252; - z[1] = (unsigned char)w; - varintWrite32(z+2, y); - return 6; - } - if( w<=65535 ){ - z[0] = 253; - z[1] = (unsigned char)(w>>8); - z[2] = (unsigned char)w; - varintWrite32(z+3, y); - return 7; - } - if( w<=16777215 ){ - z[0] = 254; - z[1] = (unsigned char)(w>>16); - z[2] = (unsigned char)(w>>8); - z[3] = (unsigned char)w; - varintWrite32(z+4, y); - return 8; - } - z[0] = 255; - varintWrite32(z+1, w); - varintWrite32(z+5, y); - return 9; -} - -/* Append non-negative integer x as a variable-length integer. -*/ -static void lsm1VblobAppendVarint(lsm1_vblob *p, sqlite3_uint64 x){ - sqlite3_int64 n = p->n; - if( n+9>p->nAlloc && lsm1VblobEnlarge(p, 9) ) return; - p->n += lsm1PutVarint64(p->a+p->n, x); -} - -/* -** Decode the varint in the first n bytes z[]. Write the integer value -** into *pResult and return the number of bytes in the varint. -** -** If the decode fails because there are not enough bytes in z[] then -** return 0; -*/ -static int lsm1GetVarint64( - const unsigned char *z, - int n, - sqlite3_uint64 *pResult -){ - unsigned int x; - if( n<1 ) return 0; - if( z[0]<=240 ){ - *pResult = z[0]; - return 1; - } - if( z[0]<=248 ){ - if( n<2 ) return 0; - *pResult = (z[0]-241)*256 + z[1] + 240; - return 2; - } - if( n=0 ){ - u = (sqlite3_uint64)v; - return lsm1PutVarint64(z, u*2); - }else{ - u = (sqlite3_uint64)(-1-v); - return lsm1PutVarint64(z, u*2+1); - } -} - -/* Decoded a signed varint. */ -static int lsm1GetSignedVarint64( - const unsigned char *z, - int n, - sqlite3_int64 *pResult -){ - sqlite3_uint64 u = 0; - n = lsm1GetVarint64(z, n, &u); - if( u&1 ){ - *pResult = -1 - (sqlite3_int64)(u>>1); - }else{ - *pResult = (sqlite3_int64)(u>>1); - } - return n; -} - - -/* -** Read the value part of the key-value pair and decode it into columns. -*/ -static int lsm1DecodeValues(lsm1_cursor *pCur){ - lsm1_vtab *pTab = (lsm1_vtab*)(pCur->base.pVtab); - int i, n; - int rc; - u8 eType; - sqlite3_uint64 v; - - if( pCur->zData ) return 1; - rc = lsm_csr_value(pCur->pLsmCur, (const void**)&pCur->zData, - (int*)&pCur->nData); - if( rc ) return 0; - for(i=n=0; inVal; i++){ - v = 0; - n += lsm1GetVarint64(pCur->zData+n, pCur->nData-n, &v); - pCur->aeType[i] = eType = (u8)(v%6); - if( eType==0 ){ - pCur->aiOfst[i] = (u32)(v/6); - pCur->aiLen[i] = 0; - }else{ - pCur->aiOfst[i] = n; - n += (pCur->aiLen[i] = (u32)(v/6)); - } - if( n>pCur->nData ) break; - } - if( inVal ){ - pCur->zData = 0; - return 0; - } - return 1; -} - -/* -** Return values of columns for the row at which the lsm1_cursor -** is currently pointing. -*/ -static int lsm1Column( - sqlite3_vtab_cursor *cur, /* The cursor */ - sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ - int i /* Which column to return */ -){ - lsm1_cursor *pCur = (lsm1_cursor*)cur; - lsm1_vtab *pTab = (lsm1_vtab*)(cur->pVtab); - if( i==0 ){ - /* The key column */ - const void *pVal; - int nVal; - if( lsm_csr_key(pCur->pLsmCur, &pVal, &nVal)==LSM_OK ){ - if( pTab->keyType==SQLITE_BLOB ){ - sqlite3_result_blob(ctx, pVal, nVal, SQLITE_TRANSIENT); - }else if( pTab->keyType==SQLITE_TEXT ){ - sqlite3_result_text(ctx,(const char*)pVal, nVal, SQLITE_TRANSIENT); - }else{ - const unsigned char *z = (const unsigned char*)pVal; - sqlite3_uint64 v1; - lsm1GetVarint64(z, nVal, &v1); - sqlite3_result_int64(ctx, (sqlite3_int64)v1); - } - } - }else if( i>pTab->nVal ){ - if( i==pTab->nVal+2 ){ /* lsm1_key */ - const void *pVal; - int nVal; - if( lsm_csr_key(pCur->pLsmCur, &pVal, &nVal)==LSM_OK ){ - sqlite3_result_blob(ctx, pVal, nVal, SQLITE_TRANSIENT); - } - }else if( i==pTab->nVal+3 ){ /* lsm1_value */ - const void *pVal; - int nVal; - if( lsm_csr_value(pCur->pLsmCur, &pVal, &nVal)==LSM_OK ){ - sqlite3_result_blob(ctx, pVal, nVal, SQLITE_TRANSIENT); - } - } - }else if( lsm1DecodeValues(pCur) ){ - /* The i-th value column (where leftmost is 1) */ - const u8 *zData; - u32 nData; - i--; - zData = pCur->zData + pCur->aiOfst[i]; - nData = pCur->aiLen[i]; - switch( pCur->aeType[i] ){ - case 0: { /* in-line integer */ - sqlite3_result_int(ctx, pCur->aiOfst[i]); - break; - } - case SQLITE_INTEGER: { - sqlite3_int64 v; - lsm1GetSignedVarint64(zData, nData, &v); - sqlite3_result_int64(ctx, v); - break; - } - case SQLITE_FLOAT: { - double v; - if( nData==sizeof(v) ){ - memcpy(&v, zData, sizeof(v)); - sqlite3_result_double(ctx, v); - } - break; - } - case SQLITE_TEXT: { - sqlite3_result_text(ctx, (const char*)zData, nData, SQLITE_TRANSIENT); - break; - } - case SQLITE_BLOB: { - sqlite3_result_blob(ctx, zData, nData, SQLITE_TRANSIENT); - break; - } - default: { - /* A NULL. Do nothing */ - } - } - } - return SQLITE_OK; -} - -/* Parameter "pValue" contains an SQL value that is to be used as -** a key in an LSM table. The type of the key is determined by -** "keyType". Extract the raw bytes used for the key in LSM1. -*/ -static void lsm1KeyFromValue( - int keyType, /* The key type */ - sqlite3_value *pValue, /* The key value */ - u8 *pBuf, /* Storage space for a generated key */ - const u8 **ppKey, /* OUT: the bytes of the key */ - int *pnKey /* OUT: size of the key */ -){ - if( keyType==SQLITE_BLOB ){ - *ppKey = (const u8*)sqlite3_value_blob(pValue); - *pnKey = sqlite3_value_bytes(pValue); - }else if( keyType==SQLITE_TEXT ){ - *ppKey = (const u8*)sqlite3_value_text(pValue); - *pnKey = sqlite3_value_bytes(pValue); - }else{ - sqlite3_int64 v = sqlite3_value_int64(pValue); - if( v<0 ) v = 0; - *pnKey = lsm1PutVarint64(pBuf, v); - *ppKey = pBuf; - } -} - -/* Move to the first row to return. -*/ -static int lsm1Filter( - sqlite3_vtab_cursor *pVtabCursor, - int idxNum, const char *idxStr, - int argc, sqlite3_value **argv -){ - lsm1_cursor *pCur = (lsm1_cursor *)pVtabCursor; - lsm1_vtab *pTab = (lsm1_vtab*)(pCur->base.pVtab); - int rc = LSM_OK; - int seekType = -1; - const u8 *pVal = 0; - int nVal; - u8 keyType = pTab->keyType; - u8 aKey1[16]; - - pCur->atEof = 1; - sqlite3_free(pCur->pKey2); - pCur->pKey2 = 0; - if( idxNum<99 ){ - lsm1KeyFromValue(keyType, argv[0], aKey1, &pVal, &nVal); - } - switch( idxNum ){ - case 0: { /* key==argv[0] */ - assert( argc==1 ); - seekType = LSM_SEEK_EQ; - pCur->isDesc = 0; - pCur->bUnique = 1; - break; - } - case 1: { /* key>=argv[0] AND key<=argv[1] */ - u8 aKey[12]; - seekType = LSM_SEEK_GE; - pCur->isDesc = 0; - pCur->bUnique = 0; - if( keyType==SQLITE_INTEGER ){ - sqlite3_int64 v = sqlite3_value_int64(argv[1]); - if( v<0 ) v = 0; - pCur->nKey2 = lsm1PutVarint64(aKey, (sqlite3_uint64)v); - pCur->pKey2 = sqlite3_malloc( pCur->nKey2 ); - if( pCur->pKey2==0 ) return SQLITE_NOMEM; - memcpy(pCur->pKey2, aKey, pCur->nKey2); - }else{ - pCur->nKey2 = sqlite3_value_bytes(argv[1]); - pCur->pKey2 = sqlite3_malloc( pCur->nKey2 ); - if( pCur->pKey2==0 ) return SQLITE_NOMEM; - if( keyType==SQLITE_BLOB ){ - memcpy(pCur->pKey2, sqlite3_value_blob(argv[1]), pCur->nKey2); - }else{ - memcpy(pCur->pKey2, sqlite3_value_text(argv[1]), pCur->nKey2); - } - } - break; - } - case 2: { /* key>=argv[0] */ - seekType = LSM_SEEK_GE; - pCur->isDesc = 0; - pCur->bUnique = 0; - break; - } - case 3: { /* key<=argv[0] */ - seekType = LSM_SEEK_LE; - pCur->isDesc = 1; - pCur->bUnique = 0; - break; - } - default: { /* full table scan */ - pCur->isDesc = 0; - pCur->bUnique = 0; - break; - } - } - if( pVal ){ - rc = lsm_csr_seek(pCur->pLsmCur, pVal, nVal, seekType); - }else{ - rc = lsm_csr_first(pCur->pLsmCur); - } - if( rc==LSM_OK && lsm_csr_valid(pCur->pLsmCur)!=0 ){ - pCur->atEof = 0; - } - return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; -} - -/* -** Only comparisons against the key are allowed. The idxNum defines -** which comparisons are available: -** -** 0 key==?1 -** 1 key>=?1 AND key<=?2 -** 2 key>?1 or key>=?1 -** 3 keyaConstraint; - for(i=0; inConstraint; i++, pConstraint++){ - if( pConstraint->usable==0 ) continue; - if( pConstraint->iColumn!=0 ) continue; - switch( pConstraint->op ){ - case SQLITE_INDEX_CONSTRAINT_EQ: { - if( idxNum>0 ){ - argIdx = i; - iIdx2 = -1; - idxNum = 0; - omit1 = 1; - } - break; - } - case SQLITE_INDEX_CONSTRAINT_GE: - case SQLITE_INDEX_CONSTRAINT_GT: { - if( idxNum==99 ){ - argIdx = i; - idxNum = 2; - omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_GE; - }else if( idxNum==3 ){ - iIdx2 = idxNum; - omit2 = omit1; - argIdx = i; - idxNum = 1; - omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_GE; - } - break; - } - case SQLITE_INDEX_CONSTRAINT_LE: - case SQLITE_INDEX_CONSTRAINT_LT: { - if( idxNum==99 ){ - argIdx = i; - idxNum = 3; - omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE; - }else if( idxNum==2 ){ - iIdx2 = i; - idxNum = 1; - omit1 = pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE; - } - break; - } - } - } - if( argIdx>=0 ){ - pIdxInfo->aConstraintUsage[argIdx].argvIndex = ++nArg; - pIdxInfo->aConstraintUsage[argIdx].omit = omit1; - } - if( iIdx2>=0 ){ - pIdxInfo->aConstraintUsage[iIdx2].argvIndex = ++nArg; - pIdxInfo->aConstraintUsage[iIdx2].omit = omit2; - } - if( idxNum==0 ){ - pIdxInfo->estimatedCost = (double)1; - pIdxInfo->estimatedRows = 1; - pIdxInfo->orderByConsumed = 1; - }else if( idxNum==1 ){ - pIdxInfo->estimatedCost = (double)100; - pIdxInfo->estimatedRows = 100; - }else if( idxNum<99 ){ - pIdxInfo->estimatedCost = (double)5000; - pIdxInfo->estimatedRows = 5000; - }else{ - /* Full table scan */ - pIdxInfo->estimatedCost = (double)2147483647; - pIdxInfo->estimatedRows = 2147483647; - } - pIdxInfo->idxNum = idxNum; - return SQLITE_OK; -} - -/* -** The xUpdate method is normally used for INSERT, REPLACE, UPDATE, and -** DELETE. But this virtual table only supports INSERT and REPLACE. -** DELETE is accomplished by inserting a record with a value of NULL. -** UPDATE is achieved by using REPLACE. -*/ -int lsm1Update( - sqlite3_vtab *pVTab, - int argc, - sqlite3_value **argv, - sqlite_int64 *pRowid -){ - lsm1_vtab *p = (lsm1_vtab*)pVTab; - int nKey, nKey2; - int i; - int rc = LSM_OK; - const u8 *pKey, *pKey2; - unsigned char aKey[16]; - unsigned char pSpace[16]; - lsm1_vblob val; - - if( argc==1 ){ - /* DELETE the record whose key is argv[0] */ - lsm1KeyFromValue(p->keyType, argv[0], aKey, &pKey, &nKey); - lsm_delete(p->pDb, pKey, nKey); - return SQLITE_OK; - } - - if( sqlite3_value_type(argv[0])!=SQLITE_NULL ){ - /* An UPDATE */ - lsm1KeyFromValue(p->keyType, argv[0], aKey, &pKey, &nKey); - lsm1KeyFromValue(p->keyType, argv[1], pSpace, &pKey2, &nKey2); - if( nKey!=nKey2 || memcmp(pKey, pKey2, nKey)!=0 ){ - /* The UPDATE changes the PRIMARY KEY value. DELETE the old key */ - lsm_delete(p->pDb, pKey, nKey); - } - /* Fall through into the INSERT case to complete the UPDATE */ - } - - /* "INSERT INTO tab(lsm1_command) VALUES('....')" is used to implement - ** special commands. - */ - if( sqlite3_value_type(argv[3+p->nVal])!=SQLITE_NULL ){ - return SQLITE_OK; - } - lsm1KeyFromValue(p->keyType, argv[2], aKey, &pKey, &nKey); - memset(&val, 0, sizeof(val)); - for(i=0; inVal; i++){ - sqlite3_value *pArg = argv[3+i]; - u8 eType = sqlite3_value_type(pArg); - switch( eType ){ - case SQLITE_NULL: { - lsm1VblobAppendVarint(&val, SQLITE_NULL); - break; - } - case SQLITE_INTEGER: { - sqlite3_int64 v = sqlite3_value_int64(pArg); - if( v>=0 && v<=240/6 ){ - lsm1VblobAppendVarint(&val, v*6); - }else{ - int n = lsm1PutSignedVarint64(pSpace, v); - lsm1VblobAppendVarint(&val, SQLITE_INTEGER + n*6); - lsm1VblobAppend(&val, pSpace, n); - } - break; - } - case SQLITE_FLOAT: { - double r = sqlite3_value_double(pArg); - lsm1VblobAppendVarint(&val, SQLITE_FLOAT + 8*6); - lsm1VblobAppend(&val, (u8*)&r, sizeof(r)); - break; - } - case SQLITE_BLOB: { - int n = sqlite3_value_bytes(pArg); - lsm1VblobAppendVarint(&val, n*6 + SQLITE_BLOB); - lsm1VblobAppend(&val, sqlite3_value_blob(pArg), n); - break; - } - case SQLITE_TEXT: { - int n = sqlite3_value_bytes(pArg); - lsm1VblobAppendVarint(&val, n*6 + SQLITE_TEXT); - lsm1VblobAppend(&val, sqlite3_value_text(pArg), n); - break; - } - } - } - if( val.errNoMem ){ - return SQLITE_NOMEM; - } - rc = lsm_insert(p->pDb, pKey, nKey, val.a, val.n); - sqlite3_free(val.a); - return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; -} - -/* Begin a transaction -*/ -static int lsm1Begin(sqlite3_vtab *pVtab){ - lsm1_vtab *p = (lsm1_vtab*)pVtab; - int rc = lsm_begin(p->pDb, 1); - return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; -} - -/* Phase 1 of a transaction commit. -*/ -static int lsm1Sync(sqlite3_vtab *pVtab){ - return SQLITE_OK; -} - -/* Commit a transaction -*/ -static int lsm1Commit(sqlite3_vtab *pVtab){ - lsm1_vtab *p = (lsm1_vtab*)pVtab; - int rc = lsm_commit(p->pDb, 0); - return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; -} - -/* Rollback a transaction -*/ -static int lsm1Rollback(sqlite3_vtab *pVtab){ - lsm1_vtab *p = (lsm1_vtab*)pVtab; - int rc = lsm_rollback(p->pDb, 0); - return rc==LSM_OK ? SQLITE_OK : SQLITE_ERROR; -} - -/* -** This following structure defines all the methods for the -** generate_lsm1 virtual table. -*/ -static sqlite3_module lsm1Module = { - 0, /* iVersion */ - lsm1Connect, /* xCreate */ - lsm1Connect, /* xConnect */ - lsm1BestIndex, /* xBestIndex */ - lsm1Disconnect, /* xDisconnect */ - lsm1Disconnect, /* xDestroy */ - lsm1Open, /* xOpen - open a cursor */ - lsm1Close, /* xClose - close a cursor */ - lsm1Filter, /* xFilter - configure scan constraints */ - lsm1Next, /* xNext - advance a cursor */ - lsm1Eof, /* xEof - check for end of scan */ - lsm1Column, /* xColumn - read data */ - lsm1Rowid, /* xRowid - read data */ - lsm1Update, /* xUpdate */ - lsm1Begin, /* xBegin */ - lsm1Sync, /* xSync */ - lsm1Commit, /* xCommit */ - lsm1Rollback, /* xRollback */ - 0, /* xFindMethod */ - 0, /* xRename */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0, /* xRollbackTo */ - 0, /* xShadowName */ - 0 /* xIntegrity */ -}; - - -#ifdef _WIN32 -__declspec(dllexport) -#endif -int sqlite3_lsm_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - int rc = SQLITE_OK; - SQLITE_EXTENSION_INIT2(pApi); - rc = sqlite3_create_module(db, "lsm1", &lsm1Module, 0); - return rc; -} diff --git a/ext/lsm1/lsm_win32.c b/ext/lsm1/lsm_win32.c deleted file mode 100644 index 6c5d06b4c..000000000 --- a/ext/lsm1/lsm_win32.c +++ /dev/null @@ -1,1063 +0,0 @@ -/* -** 2011-12-03 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Win32-specific run-time environment implementation for LSM. -*/ - -#ifdef _WIN32 - -#include -#include - -#include -#include -#include -#include - -#include "windows.h" - -#include "lsmInt.h" - -/* -** An open file is an instance of the following object -*/ -typedef struct Win32File Win32File; -struct Win32File { - lsm_env *pEnv; /* The run-time environment */ - const char *zName; /* Full path to file */ - - HANDLE hFile; /* Open file handle */ - HANDLE hShmFile; /* File handle for *-shm file */ - - SYSTEM_INFO sysInfo; /* Operating system information */ - HANDLE hMap; /* File handle for mapping */ - LPVOID pMap; /* Pointer to mapping of file fd */ - size_t nMap; /* Size of mapping at pMap in bytes */ - int nShm; /* Number of entries in ahShm[]/apShm[] */ - LPHANDLE ahShm; /* Array of handles for shared mappings */ - LPVOID *apShm; /* Array of 32K shared memory segments */ -}; - -static char *win32ShmFile(Win32File *pWin32File){ - char *zShm; - int nName = strlen(pWin32File->zName); - zShm = (char *)lsmMallocZero(pWin32File->pEnv, nName+4+1); - if( zShm ){ - memcpy(zShm, pWin32File->zName, nName); - memcpy(&zShm[nName], "-shm", 5); - } - return zShm; -} - -static int win32Sleep(int us){ - Sleep((us + 999) / 1000); - return LSM_OK; -} - -/* -** The number of times that an I/O operation will be retried following a -** locking error - probably caused by antivirus software. Also the initial -** delay before the first retry. The delay increases linearly with each -** retry. -*/ -#ifndef LSM_WIN32_IOERR_RETRY -# define LSM_WIN32_IOERR_RETRY 10 -#endif -#ifndef LSM_WIN32_IOERR_RETRY_DELAY -# define LSM_WIN32_IOERR_RETRY_DELAY 25000 -#endif -static int win32IoerrRetry = LSM_WIN32_IOERR_RETRY; -static int win32IoerrRetryDelay = LSM_WIN32_IOERR_RETRY_DELAY; - -/* -** The "win32IoerrCanRetry1" macro is used to determine if a particular -** I/O error code obtained via GetLastError() is eligible to be retried. -** It must accept the error code DWORD as its only argument and should -** return non-zero if the error code is transient in nature and the -** operation responsible for generating the original error might succeed -** upon being retried. The argument to this macro should be a variable. -** -** Additionally, a macro named "win32IoerrCanRetry2" may be defined. If -** it is defined, it will be consulted only when the macro -** "win32IoerrCanRetry1" returns zero. The "win32IoerrCanRetry2" macro -** is completely optional and may be used to include additional error -** codes in the set that should result in the failing I/O operation being -** retried by the caller. If defined, the "win32IoerrCanRetry2" macro -** must exhibit external semantics identical to those of the -** "win32IoerrCanRetry1" macro. -*/ -#if !defined(win32IoerrCanRetry1) -#define win32IoerrCanRetry1(a) (((a)==ERROR_ACCESS_DENIED) || \ - ((a)==ERROR_SHARING_VIOLATION) || \ - ((a)==ERROR_LOCK_VIOLATION) || \ - ((a)==ERROR_DEV_NOT_EXIST) || \ - ((a)==ERROR_NETNAME_DELETED) || \ - ((a)==ERROR_SEM_TIMEOUT) || \ - ((a)==ERROR_NETWORK_UNREACHABLE)) -#endif - -/* -** If an I/O error occurs, invoke this routine to see if it should be -** retried. Return TRUE to retry. Return FALSE to give up with an -** error. -*/ -static int win32RetryIoerr( - lsm_env *pEnv, - int *pnRetry -){ - DWORD lastErrno; - if( *pnRetry>=win32IoerrRetry ){ - return 0; - } - lastErrno = GetLastError(); - if( win32IoerrCanRetry1(lastErrno) ){ - win32Sleep(win32IoerrRetryDelay*(1+*pnRetry)); - ++*pnRetry; - return 1; - } -#if defined(win32IoerrCanRetry2) - else if( win32IoerrCanRetry2(lastErrno) ){ - win32Sleep(win32IoerrRetryDelay*(1+*pnRetry)); - ++*pnRetry; - return 1; - } -#endif - return 0; -} - -/* -** Convert a UTF-8 string to Microsoft Unicode. -** -** Space to hold the returned string is obtained from lsmMalloc(). -*/ -static LPWSTR win32Utf8ToUnicode(lsm_env *pEnv, const char *zText){ - int nChar; - LPWSTR zWideText; - - nChar = MultiByteToWideChar(CP_UTF8, 0, zText, -1, NULL, 0); - if( nChar==0 ){ - return 0; - } - zWideText = lsmMallocZero(pEnv, nChar * sizeof(WCHAR)); - if( zWideText==0 ){ - return 0; - } - nChar = MultiByteToWideChar(CP_UTF8, 0, zText, -1, zWideText, nChar); - if( nChar==0 ){ - lsmFree(pEnv, zWideText); - zWideText = 0; - } - return zWideText; -} - -/* -** Convert a Microsoft Unicode string to UTF-8. -** -** Space to hold the returned string is obtained from lsmMalloc(). -*/ -static char *win32UnicodeToUtf8(lsm_env *pEnv, LPCWSTR zWideText){ - int nByte; - char *zText; - - nByte = WideCharToMultiByte(CP_UTF8, 0, zWideText, -1, 0, 0, 0, 0); - if( nByte == 0 ){ - return 0; - } - zText = lsmMallocZero(pEnv, nByte); - if( zText==0 ){ - return 0; - } - nByte = WideCharToMultiByte(CP_UTF8, 0, zWideText, -1, zText, nByte, 0, 0); - if( nByte == 0 ){ - lsmFree(pEnv, zText); - zText = 0; - } - return zText; -} - -#if !defined(win32IsNotFound) -#define win32IsNotFound(a) (((a)==ERROR_FILE_NOT_FOUND) || \ - ((a)==ERROR_PATH_NOT_FOUND)) -#endif - -static int win32Open( - lsm_env *pEnv, - const char *zFile, - int flags, - LPHANDLE phFile -){ - int rc; - LPWSTR zConverted; - - zConverted = win32Utf8ToUnicode(pEnv, zFile); - if( zConverted==0 ){ - rc = LSM_NOMEM_BKPT; - }else{ - int bReadonly = (flags & LSM_OPEN_READONLY); - DWORD dwDesiredAccess; - DWORD dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; - DWORD dwCreationDisposition; - DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; - HANDLE hFile; - int nRetry = 0; - if( bReadonly ){ - dwDesiredAccess = GENERIC_READ; - dwCreationDisposition = OPEN_EXISTING; - }else{ - dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; - dwCreationDisposition = OPEN_ALWAYS; - } - while( (hFile = CreateFileW((LPCWSTR)zConverted, - dwDesiredAccess, - dwShareMode, NULL, - dwCreationDisposition, - dwFlagsAndAttributes, - NULL))==INVALID_HANDLE_VALUE && - win32RetryIoerr(pEnv, &nRetry) ){ - /* Noop */ - } - lsmFree(pEnv, zConverted); - if( hFile!=INVALID_HANDLE_VALUE ){ - *phFile = hFile; - rc = LSM_OK; - }else{ - if( win32IsNotFound(GetLastError()) ){ - rc = lsmErrorBkpt(LSM_IOERR_NOENT); - }else{ - rc = LSM_IOERR_BKPT; - } - } - } - return rc; -} - -static int lsmWin32OsOpen( - lsm_env *pEnv, - const char *zFile, - int flags, - lsm_file **ppFile -){ - int rc = LSM_OK; - Win32File *pWin32File; - - pWin32File = lsmMallocZero(pEnv, sizeof(Win32File)); - if( pWin32File==0 ){ - rc = LSM_NOMEM_BKPT; - }else{ - HANDLE hFile = NULL; - - rc = win32Open(pEnv, zFile, flags, &hFile); - if( rc==LSM_OK ){ - memset(&pWin32File->sysInfo, 0, sizeof(SYSTEM_INFO)); - GetSystemInfo(&pWin32File->sysInfo); - pWin32File->pEnv = pEnv; - pWin32File->zName = zFile; - pWin32File->hFile = hFile; - }else{ - lsmFree(pEnv, pWin32File); - pWin32File = 0; - } - } - *ppFile = (lsm_file *)pWin32File; - return rc; -} - -static int lsmWin32OsWrite( - lsm_file *pFile, /* File to write to */ - lsm_i64 iOff, /* Offset to write to */ - void *pData, /* Write data from this buffer */ - int nData /* Bytes of data to write */ -){ - Win32File *pWin32File = (Win32File *)pFile; - OVERLAPPED overlapped; /* The offset for WriteFile. */ - u8 *aRem = (u8 *)pData; /* Data yet to be written */ - int nRem = nData; /* Number of bytes yet to be written */ - int nRetry = 0; /* Number of retrys */ - - memset(&overlapped, 0, sizeof(OVERLAPPED)); - overlapped.Offset = (LONG)(iOff & 0XFFFFFFFF); - overlapped.OffsetHigh = (LONG)((iOff>>32) & 0x7FFFFFFF); - while( nRem>0 ){ - DWORD nWrite = 0; /* Bytes written using WriteFile */ - if( !WriteFile(pWin32File->hFile, aRem, nRem, &nWrite, &overlapped) ){ - if( win32RetryIoerr(pWin32File->pEnv, &nRetry) ) continue; - break; - } - assert( nWrite==0 || nWrite<=(DWORD)nRem ); - if( nWrite==0 || nWrite>(DWORD)nRem ){ - break; - } - iOff += nWrite; - overlapped.Offset = (LONG)(iOff & 0xFFFFFFFF); - overlapped.OffsetHigh = (LONG)((iOff>>32) & 0x7FFFFFFF); - aRem += nWrite; - nRem -= nWrite; - } - if( nRem!=0 ) return LSM_IOERR_BKPT; - return LSM_OK; -} - -static int win32Truncate( - HANDLE hFile, - lsm_i64 nSize -){ - LARGE_INTEGER offset; - offset.QuadPart = nSize; - if( !SetFilePointerEx(hFile, offset, 0, FILE_BEGIN) ){ - return LSM_IOERR_BKPT; - } - if (!SetEndOfFile(hFile) ){ - return LSM_IOERR_BKPT; - } - return LSM_OK; -} - -static int lsmWin32OsTruncate( - lsm_file *pFile, /* File to write to */ - lsm_i64 nSize /* Size to truncate file to */ -){ - Win32File *pWin32File = (Win32File *)pFile; - return win32Truncate(pWin32File->hFile, nSize); -} - -static int lsmWin32OsRead( - lsm_file *pFile, /* File to read from */ - lsm_i64 iOff, /* Offset to read from */ - void *pData, /* Read data into this buffer */ - int nData /* Bytes of data to read */ -){ - Win32File *pWin32File = (Win32File *)pFile; - OVERLAPPED overlapped; /* The offset for ReadFile */ - DWORD nRead = 0; /* Bytes read using ReadFile */ - int nRetry = 0; /* Number of retrys */ - - memset(&overlapped, 0, sizeof(OVERLAPPED)); - overlapped.Offset = (LONG)(iOff & 0XFFFFFFFF); - overlapped.OffsetHigh = (LONG)((iOff>>32) & 0X7FFFFFFF); - while( !ReadFile(pWin32File->hFile, pData, nData, &nRead, &overlapped) && - GetLastError()!=ERROR_HANDLE_EOF ){ - if( win32RetryIoerr(pWin32File->pEnv, &nRetry) ) continue; - return LSM_IOERR_BKPT; - } - if( nRead<(DWORD)nData ){ - /* Unread parts of the buffer must be zero-filled */ - memset(&((char*)pData)[nRead], 0, nData - nRead); - } - return LSM_OK; -} - -static int lsmWin32OsSync(lsm_file *pFile){ - int rc = LSM_OK; - -#ifndef LSM_NO_SYNC - Win32File *pWin32File = (Win32File *)pFile; - - if( pWin32File->pMap!=NULL ){ - if( !FlushViewOfFile(pWin32File->pMap, 0) ){ - rc = LSM_IOERR_BKPT; - } - } - if( rc==LSM_OK && !FlushFileBuffers(pWin32File->hFile) ){ - rc = LSM_IOERR_BKPT; - } -#else - unused_parameter(pFile); -#endif - - return rc; -} - -static int lsmWin32OsSectorSize(lsm_file *pFile){ - return 512; -} - -static void win32Unmap(Win32File *pWin32File){ - if( pWin32File->pMap!=NULL ){ - UnmapViewOfFile(pWin32File->pMap); - pWin32File->pMap = NULL; - pWin32File->nMap = 0; - } - if( pWin32File->hMap!=NULL ){ - CloseHandle(pWin32File->hMap); - pWin32File->hMap = NULL; - } -} - -static int lsmWin32OsRemap( - lsm_file *pFile, - lsm_i64 iMin, - void **ppOut, - lsm_i64 *pnOut -){ - Win32File *pWin32File = (Win32File *)pFile; - - /* If the file is between 0 and 2MB in size, extend it in chunks of 256K. - ** Thereafter, in chunks of 1MB at a time. */ - const int aIncrSz[] = {256*1024, 1024*1024}; - int nIncrSz = aIncrSz[iMin>(2*1024*1024)]; - - *ppOut = NULL; - *pnOut = 0; - - win32Unmap(pWin32File); - if( iMin>=0 ){ - LARGE_INTEGER fileSize; - DWORD dwSizeHigh; - DWORD dwSizeLow; - HANDLE hMap; - LPVOID pMap; - memset(&fileSize, 0, sizeof(LARGE_INTEGER)); - if( !GetFileSizeEx(pWin32File->hFile, &fileSize) ){ - return LSM_IOERR_BKPT; - } - assert( fileSize.QuadPart>=0 ); - if( fileSize.QuadPart> 32); - hMap = CreateFileMappingW(pWin32File->hFile, NULL, PAGE_READWRITE, - dwSizeHigh, dwSizeLow, NULL); - if( hMap==NULL ){ - return LSM_IOERR_BKPT; - } - pWin32File->hMap = hMap; - assert( fileSize.QuadPart<=0xFFFFFFFF ); - pMap = MapViewOfFile(hMap, FILE_MAP_WRITE | FILE_MAP_READ, 0, 0, - (SIZE_T)fileSize.QuadPart); - if( pMap==NULL ){ - return LSM_IOERR_BKPT; - } - pWin32File->pMap = pMap; - pWin32File->nMap = (SIZE_T)fileSize.QuadPart; - } - *ppOut = pWin32File->pMap; - *pnOut = pWin32File->nMap; - return LSM_OK; -} - -static BOOL win32IsDriveLetterAndColon( - const char *zPathname -){ - return ( isalpha(zPathname[0]) && zPathname[1]==':' ); -} - -static int lsmWin32OsFullpath( - lsm_env *pEnv, - const char *zName, - char *zOut, - int *pnOut -){ - DWORD nByte; - void *zConverted; - LPWSTR zTempWide; - char *zTempUtf8; - - if( zName[0]=='/' && win32IsDriveLetterAndColon(zName+1) ){ - zName++; - } - zConverted = win32Utf8ToUnicode(pEnv, zName); - if( zConverted==0 ){ - return LSM_NOMEM_BKPT; - } - nByte = GetFullPathNameW((LPCWSTR)zConverted, 0, 0, 0); - if( nByte==0 ){ - lsmFree(pEnv, zConverted); - return LSM_IOERR_BKPT; - } - nByte += 3; - zTempWide = lsmMallocZero(pEnv, nByte * sizeof(zTempWide[0])); - if( zTempWide==0 ){ - lsmFree(pEnv, zConverted); - return LSM_NOMEM_BKPT; - } - nByte = GetFullPathNameW((LPCWSTR)zConverted, nByte, zTempWide, 0); - if( nByte==0 ){ - lsmFree(pEnv, zConverted); - lsmFree(pEnv, zTempWide); - return LSM_IOERR_BKPT; - } - lsmFree(pEnv, zConverted); - zTempUtf8 = win32UnicodeToUtf8(pEnv, zTempWide); - lsmFree(pEnv, zTempWide); - if( zTempUtf8 ){ - int nOut = *pnOut; - int nLen = strlen(zTempUtf8) + 1; - if( nLen<=nOut ){ - snprintf(zOut, nOut, "%s", zTempUtf8); - } - lsmFree(pEnv, zTempUtf8); - *pnOut = nLen; - return LSM_OK; - }else{ - return LSM_NOMEM_BKPT; - } -} - -static int lsmWin32OsFileid( - lsm_file *pFile, - void *pBuf, - int *pnBuf -){ - int nBuf; - int nReq; - u8 *pBuf2 = (u8 *)pBuf; - Win32File *pWin32File = (Win32File *)pFile; - BY_HANDLE_FILE_INFORMATION fileInfo; - - nBuf = *pnBuf; - nReq = (sizeof(fileInfo.dwVolumeSerialNumber) + - sizeof(fileInfo.nFileIndexHigh) + - sizeof(fileInfo.nFileIndexLow)); - *pnBuf = nReq; - if( nReq>nBuf ) return LSM_OK; - memset(&fileInfo, 0, sizeof(BY_HANDLE_FILE_INFORMATION)); - if( !GetFileInformationByHandle(pWin32File->hFile, &fileInfo) ){ - return LSM_IOERR_BKPT; - } - nReq = sizeof(fileInfo.dwVolumeSerialNumber); - memcpy(pBuf2, &fileInfo.dwVolumeSerialNumber, nReq); - pBuf2 += nReq; - nReq = sizeof(fileInfo.nFileIndexHigh); - memcpy(pBuf, &fileInfo.nFileIndexHigh, nReq); - pBuf2 += nReq; - nReq = sizeof(fileInfo.nFileIndexLow); - memcpy(pBuf2, &fileInfo.nFileIndexLow, nReq); - return LSM_OK; -} - -static int win32Delete( - lsm_env *pEnv, - const char *zFile -){ - int rc; - LPWSTR zConverted; - - zConverted = win32Utf8ToUnicode(pEnv, zFile); - if( zConverted==0 ){ - rc = LSM_NOMEM_BKPT; - }else{ - int nRetry = 0; - DWORD attr; - - do { - attr = GetFileAttributesW(zConverted); - if ( attr==INVALID_FILE_ATTRIBUTES ){ - rc = LSM_IOERR_BKPT; - break; - } - if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ - rc = LSM_IOERR_BKPT; /* Files only. */ - break; - } - if ( DeleteFileW(zConverted) ){ - rc = LSM_OK; /* Deleted OK. */ - break; - } - if ( !win32RetryIoerr(pEnv, &nRetry) ){ - rc = LSM_IOERR_BKPT; /* No more retries. */ - break; - } - }while( 1 ); - } - lsmFree(pEnv, zConverted); - return rc; -} - -static int lsmWin32OsUnlink(lsm_env *pEnv, const char *zFile){ - return win32Delete(pEnv, zFile); -} - -#if !defined(win32IsLockBusy) -#define win32IsLockBusy(a) (((a)==ERROR_LOCK_VIOLATION) || \ - ((a)==ERROR_IO_PENDING)) -#endif - -static int win32LockFile( - Win32File *pWin32File, - int iLock, - int nLock, - int eType -){ - OVERLAPPED ovlp; - - assert( LSM_LOCK_UNLOCK==0 ); - assert( LSM_LOCK_SHARED==1 ); - assert( LSM_LOCK_EXCL==2 ); - assert( eType>=LSM_LOCK_UNLOCK && eType<=LSM_LOCK_EXCL ); - assert( nLock>=0 ); - assert( iLock>0 && iLock<=32 ); - - memset(&ovlp, 0, sizeof(OVERLAPPED)); - ovlp.Offset = (4096-iLock-nLock+1); - if( eType>LSM_LOCK_UNLOCK ){ - DWORD flags = LOCKFILE_FAIL_IMMEDIATELY; - if( eType>=LSM_LOCK_EXCL ) flags |= LOCKFILE_EXCLUSIVE_LOCK; - if( !LockFileEx(pWin32File->hFile, flags, 0, (DWORD)nLock, 0, &ovlp) ){ - if( win32IsLockBusy(GetLastError()) ){ - return LSM_BUSY; - }else{ - return LSM_IOERR_BKPT; - } - } - }else{ - if( !UnlockFileEx(pWin32File->hFile, 0, (DWORD)nLock, 0, &ovlp) ){ - return LSM_IOERR_BKPT; - } - } - return LSM_OK; -} - -static int lsmWin32OsLock(lsm_file *pFile, int iLock, int eType){ - Win32File *pWin32File = (Win32File *)pFile; - return win32LockFile(pWin32File, iLock, 1, eType); -} - -static int lsmWin32OsTestLock(lsm_file *pFile, int iLock, int nLock, int eType){ - int rc; - Win32File *pWin32File = (Win32File *)pFile; - rc = win32LockFile(pWin32File, iLock, nLock, eType); - if( rc!=LSM_OK ) return rc; - win32LockFile(pWin32File, iLock, nLock, LSM_LOCK_UNLOCK); - return LSM_OK; -} - -static int lsmWin32OsShmMap(lsm_file *pFile, int iChunk, int sz, void **ppShm){ - int rc; - Win32File *pWin32File = (Win32File *)pFile; - int iOffset = iChunk * sz; - int iOffsetShift = iOffset % pWin32File->sysInfo.dwAllocationGranularity; - int nNew = iChunk + 1; - lsm_i64 nReq = nNew * sz; - - *ppShm = NULL; - assert( sz>=0 ); - assert( sz==LSM_SHM_CHUNK_SIZE ); - if( iChunk>=pWin32File->nShm ){ - LPHANDLE ahNew; - LPVOID *apNew; - LARGE_INTEGER fileSize; - - /* If the shared-memory file has not been opened, open it now. */ - if( pWin32File->hShmFile==NULL ){ - char *zShm = win32ShmFile(pWin32File); - if( !zShm ) return LSM_NOMEM_BKPT; - rc = win32Open(pWin32File->pEnv, zShm, 0, &pWin32File->hShmFile); - lsmFree(pWin32File->pEnv, zShm); - if( rc!=LSM_OK ){ - return rc; - } - } - - /* If the shared-memory file is not large enough to contain the - ** requested chunk, cause it to grow. */ - memset(&fileSize, 0, sizeof(LARGE_INTEGER)); - if( !GetFileSizeEx(pWin32File->hShmFile, &fileSize) ){ - return LSM_IOERR_BKPT; - } - assert( fileSize.QuadPart>=0 ); - if( fileSize.QuadParthShmFile, nReq); - if( rc!=LSM_OK ){ - return rc; - } - } - - ahNew = (LPHANDLE)lsmMallocZero(pWin32File->pEnv, sizeof(HANDLE) * nNew); - if( !ahNew ) return LSM_NOMEM_BKPT; - apNew = (LPVOID *)lsmMallocZero(pWin32File->pEnv, sizeof(LPVOID) * nNew); - if( !apNew ){ - lsmFree(pWin32File->pEnv, ahNew); - return LSM_NOMEM_BKPT; - } - memcpy(ahNew, pWin32File->ahShm, sizeof(HANDLE) * pWin32File->nShm); - memcpy(apNew, pWin32File->apShm, sizeof(LPVOID) * pWin32File->nShm); - lsmFree(pWin32File->pEnv, pWin32File->ahShm); - pWin32File->ahShm = ahNew; - lsmFree(pWin32File->pEnv, pWin32File->apShm); - pWin32File->apShm = apNew; - pWin32File->nShm = nNew; - } - - if( pWin32File->ahShm[iChunk]==NULL ){ - HANDLE hMap; - assert( nReq<=0xFFFFFFFF ); - hMap = CreateFileMappingW(pWin32File->hShmFile, NULL, PAGE_READWRITE, 0, - (DWORD)nReq, NULL); - if( hMap==NULL ){ - return LSM_IOERR_BKPT; - } - pWin32File->ahShm[iChunk] = hMap; - } - if( pWin32File->apShm[iChunk]==NULL ){ - LPVOID pMap; - pMap = MapViewOfFile(pWin32File->ahShm[iChunk], - FILE_MAP_WRITE | FILE_MAP_READ, 0, - iOffset - iOffsetShift, sz + iOffsetShift); - if( pMap==NULL ){ - return LSM_IOERR_BKPT; - } - pWin32File->apShm[iChunk] = pMap; - } - if( iOffsetShift!=0 ){ - char *p = (char *)pWin32File->apShm[iChunk]; - *ppShm = (void *)&p[iOffsetShift]; - }else{ - *ppShm = pWin32File->apShm[iChunk]; - } - return LSM_OK; -} - -static void lsmWin32OsShmBarrier(void){ - MemoryBarrier(); -} - -static int lsmWin32OsShmUnmap(lsm_file *pFile, int bDelete){ - Win32File *pWin32File = (Win32File *)pFile; - - if( pWin32File->hShmFile!=NULL ){ - int i; - for(i=0; inShm; i++){ - if( pWin32File->apShm[i]!=NULL ){ - UnmapViewOfFile(pWin32File->apShm[i]); - pWin32File->apShm[i] = NULL; - } - if( pWin32File->ahShm[i]!=NULL ){ - CloseHandle(pWin32File->ahShm[i]); - pWin32File->ahShm[i] = NULL; - } - } - CloseHandle(pWin32File->hShmFile); - pWin32File->hShmFile = NULL; - if( bDelete ){ - char *zShm = win32ShmFile(pWin32File); - if( zShm ){ win32Delete(pWin32File->pEnv, zShm); } - lsmFree(pWin32File->pEnv, zShm); - } - } - return LSM_OK; -} - -#define MX_CLOSE_ATTEMPT 3 -static int lsmWin32OsClose(lsm_file *pFile){ - int rc; - int nRetry = 0; - Win32File *pWin32File = (Win32File *)pFile; - lsmWin32OsShmUnmap(pFile, 0); - win32Unmap(pWin32File); - do{ - if( pWin32File->hFile==NULL ){ - rc = LSM_IOERR_BKPT; - break; - } - rc = CloseHandle(pWin32File->hFile); - if( rc ){ - pWin32File->hFile = NULL; - rc = LSM_OK; - break; - } - if( ++nRetry>=MX_CLOSE_ATTEMPT ){ - rc = LSM_IOERR_BKPT; - break; - } - }while( 1 ); - lsmFree(pWin32File->pEnv, pWin32File->ahShm); - lsmFree(pWin32File->pEnv, pWin32File->apShm); - lsmFree(pWin32File->pEnv, pWin32File); - return rc; -} - -static int lsmWin32OsSleep(lsm_env *pEnv, int us){ - unused_parameter(pEnv); - return win32Sleep(us); -} - -/**************************************************************************** -** Memory allocation routines. -*/ - -static void *lsmWin32OsMalloc(lsm_env *pEnv, size_t N){ - assert( HeapValidate(GetProcessHeap(), 0, NULL) ); - return HeapAlloc(GetProcessHeap(), 0, (SIZE_T)N); -} - -static void lsmWin32OsFree(lsm_env *pEnv, void *p){ - assert( HeapValidate(GetProcessHeap(), 0, NULL) ); - if( p ){ - HeapFree(GetProcessHeap(), 0, p); - } -} - -static void *lsmWin32OsRealloc(lsm_env *pEnv, void *p, size_t N){ - unsigned char *m = (unsigned char *)p; - assert( HeapValidate(GetProcessHeap(), 0, NULL) ); - if( 1>N ){ - lsmWin32OsFree(pEnv, p); - return NULL; - }else if( NULL==p ){ - return lsmWin32OsMalloc(pEnv, N); - }else{ -#if 0 /* arguable: don't shrink */ - SIZE_T sz = HeapSize(GetProcessHeap(), 0, m); - if( sz>=(SIZE_T)N ){ - return p; - } -#endif - return HeapReAlloc(GetProcessHeap(), 0, m, N); - } -} - -static size_t lsmWin32OsMSize(lsm_env *pEnv, void *p){ - assert( HeapValidate(GetProcessHeap(), 0, NULL) ); - return (size_t)HeapSize(GetProcessHeap(), 0, p); -} - - -#ifdef LSM_MUTEX_WIN32 -/************************************************************************* -** Mutex methods for Win32 based systems. If LSM_MUTEX_WIN32 is -** missing then a no-op implementation of mutexes found below will be -** used instead. -*/ -#include "windows.h" - -typedef struct Win32Mutex Win32Mutex; -struct Win32Mutex { - lsm_env *pEnv; - CRITICAL_SECTION mutex; -#ifdef LSM_DEBUG - DWORD owner; -#endif -}; - -#ifndef WIN32_MUTEX_INITIALIZER -# define WIN32_MUTEX_INITIALIZER { 0 } -#endif - -#ifdef LSM_DEBUG -# define LSM_WIN32_STATIC_MUTEX { 0, WIN32_MUTEX_INITIALIZER, 0 } -#else -# define LSM_WIN32_STATIC_MUTEX { 0, WIN32_MUTEX_INITIALIZER } -#endif - -static int lsmWin32OsMutexStatic( - lsm_env *pEnv, - int iMutex, - lsm_mutex **ppStatic -){ - static volatile LONG initialized = 0; - static Win32Mutex sMutex[2] = { - LSM_WIN32_STATIC_MUTEX, - LSM_WIN32_STATIC_MUTEX - }; - - assert( iMutex==LSM_MUTEX_GLOBAL || iMutex==LSM_MUTEX_HEAP ); - assert( LSM_MUTEX_GLOBAL==1 && LSM_MUTEX_HEAP==2 ); - - if( InterlockedCompareExchange(&initialized, 1, 0)==0 ){ - int i; - for(i=0; ipEnv = pEnv; - InitializeCriticalSection(&pMutex->mutex); - - *ppNew = (lsm_mutex *)pMutex; - return LSM_OK; -} - -static void lsmWin32OsMutexDel(lsm_mutex *p){ - Win32Mutex *pMutex = (Win32Mutex *)p; - DeleteCriticalSection(&pMutex->mutex); - lsmFree(pMutex->pEnv, pMutex); -} - -static void lsmWin32OsMutexEnter(lsm_mutex *p){ - Win32Mutex *pMutex = (Win32Mutex *)p; - EnterCriticalSection(&pMutex->mutex); - -#ifdef LSM_DEBUG - assert( pMutex->owner!=GetCurrentThreadId() ); - pMutex->owner = GetCurrentThreadId(); - assert( pMutex->owner==GetCurrentThreadId() ); -#endif -} - -static int lsmWin32OsMutexTry(lsm_mutex *p){ - BOOL bRet; - Win32Mutex *pMutex = (Win32Mutex *)p; - bRet = TryEnterCriticalSection(&pMutex->mutex); -#ifdef LSM_DEBUG - if( bRet ){ - assert( pMutex->owner!=GetCurrentThreadId() ); - pMutex->owner = GetCurrentThreadId(); - assert( pMutex->owner==GetCurrentThreadId() ); - } -#endif - return !bRet; -} - -static void lsmWin32OsMutexLeave(lsm_mutex *p){ - Win32Mutex *pMutex = (Win32Mutex *)p; -#ifdef LSM_DEBUG - assert( pMutex->owner==GetCurrentThreadId() ); - pMutex->owner = 0; - assert( pMutex->owner!=GetCurrentThreadId() ); -#endif - LeaveCriticalSection(&pMutex->mutex); -} - -#ifdef LSM_DEBUG -static int lsmWin32OsMutexHeld(lsm_mutex *p){ - Win32Mutex *pMutex = (Win32Mutex *)p; - return pMutex ? pMutex->owner==GetCurrentThreadId() : 1; -} -static int lsmWin32OsMutexNotHeld(lsm_mutex *p){ - Win32Mutex *pMutex = (Win32Mutex *)p; - return pMutex ? pMutex->owner!=GetCurrentThreadId() : 1; -} -#endif -/* -** End of Win32 mutex implementation. -*************************************************************************/ -#else -/************************************************************************* -** Noop mutex implementation -*/ -typedef struct NoopMutex NoopMutex; -struct NoopMutex { - lsm_env *pEnv; /* Environment handle (for xFree()) */ - int bHeld; /* True if mutex is held */ - int bStatic; /* True for a static mutex */ -}; -static NoopMutex aStaticNoopMutex[2] = { - {0, 0, 1}, - {0, 0, 1}, -}; - -static int lsmWin32OsMutexStatic( - lsm_env *pEnv, - int iMutex, - lsm_mutex **ppStatic -){ - assert( iMutex>=1 && iMutex<=(int)array_size(aStaticNoopMutex) ); - *ppStatic = (lsm_mutex *)&aStaticNoopMutex[iMutex-1]; - return LSM_OK; -} -static int lsmWin32OsMutexNew(lsm_env *pEnv, lsm_mutex **ppNew){ - NoopMutex *p; - p = (NoopMutex *)lsmMallocZero(pEnv, sizeof(NoopMutex)); - if( p ) p->pEnv = pEnv; - *ppNew = (lsm_mutex *)p; - return (p ? LSM_OK : LSM_NOMEM_BKPT); -} -static void lsmWin32OsMutexDel(lsm_mutex *pMutex) { - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bStatic==0 && p->pEnv ); - lsmFree(p->pEnv, p); -} -static void lsmWin32OsMutexEnter(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bHeld==0 ); - p->bHeld = 1; -} -static int lsmWin32OsMutexTry(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bHeld==0 ); - p->bHeld = 1; - return 0; -} -static void lsmWin32OsMutexLeave(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - assert( p->bHeld==1 ); - p->bHeld = 0; -} -#ifdef LSM_DEBUG -static int lsmWin32OsMutexHeld(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - return p ? p->bHeld : 1; -} -static int lsmWin32OsMutexNotHeld(lsm_mutex *pMutex){ - NoopMutex *p = (NoopMutex *)pMutex; - return p ? !p->bHeld : 1; -} -#endif -/***************************************************************************/ -#endif /* else LSM_MUTEX_NONE */ - -/* Without LSM_DEBUG, the MutexHeld tests are never called */ -#ifndef LSM_DEBUG -# define lsmWin32OsMutexHeld 0 -# define lsmWin32OsMutexNotHeld 0 -#endif - -lsm_env *lsm_default_env(void){ - static lsm_env win32_env = { - sizeof(lsm_env), /* nByte */ - 1, /* iVersion */ - /***** file i/o ******************/ - 0, /* pVfsCtx */ - lsmWin32OsFullpath, /* xFullpath */ - lsmWin32OsOpen, /* xOpen */ - lsmWin32OsRead, /* xRead */ - lsmWin32OsWrite, /* xWrite */ - lsmWin32OsTruncate, /* xTruncate */ - lsmWin32OsSync, /* xSync */ - lsmWin32OsSectorSize, /* xSectorSize */ - lsmWin32OsRemap, /* xRemap */ - lsmWin32OsFileid, /* xFileid */ - lsmWin32OsClose, /* xClose */ - lsmWin32OsUnlink, /* xUnlink */ - lsmWin32OsLock, /* xLock */ - lsmWin32OsTestLock, /* xTestLock */ - lsmWin32OsShmMap, /* xShmMap */ - lsmWin32OsShmBarrier, /* xShmBarrier */ - lsmWin32OsShmUnmap, /* xShmUnmap */ - /***** memory allocation *********/ - 0, /* pMemCtx */ - lsmWin32OsMalloc, /* xMalloc */ - lsmWin32OsRealloc, /* xRealloc */ - lsmWin32OsFree, /* xFree */ - lsmWin32OsMSize, /* xSize */ - /***** mutexes *********************/ - 0, /* pMutexCtx */ - lsmWin32OsMutexStatic, /* xMutexStatic */ - lsmWin32OsMutexNew, /* xMutexNew */ - lsmWin32OsMutexDel, /* xMutexDel */ - lsmWin32OsMutexEnter, /* xMutexEnter */ - lsmWin32OsMutexTry, /* xMutexTry */ - lsmWin32OsMutexLeave, /* xMutexLeave */ - lsmWin32OsMutexHeld, /* xMutexHeld */ - lsmWin32OsMutexNotHeld, /* xMutexNotHeld */ - /***** other *********************/ - lsmWin32OsSleep, /* xSleep */ - }; - return &win32_env; -} - -#endif diff --git a/ext/lsm1/test/lsm1_common.tcl b/ext/lsm1/test/lsm1_common.tcl deleted file mode 100644 index 0e6cd84e3..000000000 --- a/ext/lsm1/test/lsm1_common.tcl +++ /dev/null @@ -1,38 +0,0 @@ -# 2014 Dec 19 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# - -if {![info exists testdir]} { - set testdir [file join [file dirname [info script]] .. .. .. test] -} -source $testdir/tester.tcl - -# Check if the lsm1 extension has been compiled. -if {$::tcl_platform(platform) == "windows"} { - set lsm1 lsm.dll -} else { - set lsm1 lsm.so -} - -if {[file exists [file join .. $lsm1]]} { - proc return_if_no_lsm1 {} {} -} else { - proc return_if_no_lsm1 {} { - finish_test - return -code return - } - return -} - -proc load_lsm1_vtab {db} { - db enable_load_extension 1 - db eval {SELECT load_extension('../lsm')} -} diff --git a/ext/lsm1/test/lsm1_simple.test b/ext/lsm1/test/lsm1_simple.test deleted file mode 100644 index 2eab50a83..000000000 --- a/ext/lsm1/test/lsm1_simple.test +++ /dev/null @@ -1,152 +0,0 @@ -# 2017 July 14 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#************************************************************************* -# This file implements regression tests for SQLite library. The -# focus of this script is testing the lsm1 virtual table module. -# - -source [file join [file dirname [info script]] lsm1_common.tcl] -set testprefix lsm1_simple -return_if_no_lsm1 -load_lsm1_vtab db - -forcedelete testlsm.db - -do_execsql_test 100 { - CREATE VIRTUAL TABLE x1 USING lsm1(testlsm.db,a,UINT,b,c,d); - PRAGMA table_info(x1); -} { - 0 a UINT 1 {} 1 - 1 b {} 0 {} 0 - 2 c {} 0 {} 0 - 3 d {} 0 {} 0 -} - -do_execsql_test 110 { - INSERT INTO x1(a,b,c,d) VALUES(15, 11, 22, 33),(8,'banjo',x'333231',NULL), - (12,NULL,3.25,-559281390); - SELECT a, quote(b), quote(c), quote(d) FROM x1; -} {8 'banjo' X'333231' NULL 12 NULL 3.25 -559281390 15 11 22 33} -do_execsql_test 111 { - SELECT a, quote(lsm1_key), quote(lsm1_value) FROM x1; -} {8 X'08' X'2162616E6A6F1633323105' 12 X'0C' X'05320000000000000A401FFB42ABE9DB' 15 X'0F' X'4284C6'} - -do_execsql_test 120 { - UPDATE x1 SET d = d+1.0 WHERE a=15; - SELECT a, quote(b), quote(c), quote(d) FROM x1; -} {8 'banjo' X'333231' NULL 12 NULL 3.25 -559281390 15 11 22 34.0} - -do_execsql_test 130 { - UPDATE x1 SET a=123456789 WHERE a=12; - SELECT a, quote(b), quote(c), quote(d) FROM x1; -} {8 'banjo' X'333231' NULL 15 11 22 34.0 123456789 NULL 3.25 -559281390} -do_execsql_test 131 { - SELECT quote(lsm1_key), printf('0x%x',a) FROM x1 WHERE a > 100000000; -} {X'FB075BCD15' 0x75bcd15} - -do_execsql_test 140 { - DELETE FROM x1 WHERE a=15; - SELECT a, quote(b), quote(c), quote(d) FROM x1; -} {8 'banjo' X'333231' NULL 123456789 NULL 3.25 -559281390} - -do_test 150 { - lsort [glob testlsm.db*] -} {testlsm.db testlsm.db-log testlsm.db-shm} - -db close -do_test 160 { - lsort [glob testlsm.db*] -} {testlsm.db} - -forcedelete testlsm.db -forcedelete test.db -sqlite3 db test.db -load_lsm1_vtab db - - -do_execsql_test 200 { - CREATE VIRTUAL TABLE x1 USING lsm1(testlsm.db,a,TEXT,b,c,d); - PRAGMA table_info(x1); -} { - 0 a TEXT 1 {} 1 - 1 b {} 0 {} 0 - 2 c {} 0 {} 0 - 3 d {} 0 {} 0 -} -do_execsql_test 210 { - INSERT INTO x1(a,b,c,d) VALUES(15, 11, 22, 33),(8,'banjo',x'333231',NULL), - (12,NULL,3.25,-559281390); - SELECT quote(a), quote(b), quote(c), quote(d), '|' FROM x1; -} {'12' NULL 3.25 -559281390 | '15' 11 22 33 | '8' 'banjo' X'333231' NULL |} -do_execsql_test 211 { - SELECT quote(a), quote(lsm1_key), quote(lsm1_value), '|' FROM x1; -} {'12' X'3132' X'05320000000000000A401FFB42ABE9DB' | '15' X'3135' X'4284C6' | '8' X'38' X'2162616E6A6F1633323105' |} -do_execsql_test 212 { - SELECT quote(a), quote(lsm1_key), quote(lsm1_value) FROM x1 WHERE a='12'; -} {'12' X'3132' X'05320000000000000A401FFB42ABE9DB'} - -#------------------------------------------------------------------------- -reset_db -forcedelete testlsm.db -load_lsm1_vtab db -do_execsql_test 300 { - CREATE VIRTUAL TABLE x1 USING lsm1(testlsm.db,a,TEXT,b,c,d); -} -do_eqp_test 310 { - SELECT * FROM x1 WHERE a=? -} {SCAN TABLE x1 VIRTUAL TABLE INDEX 0:} - -do_eqp_test 320 { - SELECT * FROM x1 WHERE a>? -} {SCAN TABLE x1 VIRTUAL TABLE INDEX 2:} - -do_eqp_test 330 { - SELECT * FROM x1 WHERE a 'five'; -} {4 1 3 2} -do_execsql_test 421 { - SELECT b FROM x1 WHERE a <= 'three'; -} {3 1 4 5} - -finish_test diff --git a/ext/lsm1/tool/mklsm1c.tcl b/ext/lsm1/tool/mklsm1c.tcl deleted file mode 100644 index d4a317b70..000000000 --- a/ext/lsm1/tool/mklsm1c.tcl +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/sh -# restart with tclsh \ -exec tclsh "$0" "$@" - -set srcdir [file dirname [file dirname [info script]]] -set G(src) [string map [list %dir% $srcdir] { - %dir%/lsm.h - %dir%/lsmInt.h - %dir%/lsm_vtab.c - %dir%/lsm_ckpt.c - %dir%/lsm_file.c - %dir%/lsm_log.c - %dir%/lsm_main.c - %dir%/lsm_mem.c - %dir%/lsm_mutex.c - %dir%/lsm_shared.c - %dir%/lsm_sorted.c - %dir%/lsm_str.c - %dir%/lsm_tree.c - %dir%/lsm_unix.c - %dir%/lsm_varint.c - %dir%/lsm_win32.c -}] - -set G(hdr) { - -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1) - -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif -#if defined(NDEBUG) && defined(SQLITE_DEBUG) -# undef NDEBUG -#endif - -} - -set G(footer) { - -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1) */ -} - -#------------------------------------------------------------------------- -# Read and return the entire contents of text file $zFile from disk. -# -proc readfile {zFile} { - set fd [open $zFile] - set data [read $fd] - close $fd - return $data -} - -proc lsm1c_init {zOut} { - global G - set G(fd) stdout - set G(fd) [open $zOut w] - - puts -nonewline $G(fd) $G(hdr) -} - -proc lsm1c_printfile {zIn} { - global G - set data [readfile $zIn] - set zTail [file tail $zIn] - puts $G(fd) "#line 1 \"$zTail\"" - - foreach line [split $data "\n"] { - if {[regexp {^# *include.*lsm} $line]} { - set line "/* $line */" - } elseif { [regexp {^(const )?[a-zA-Z][a-zA-Z0-9]* [*]?lsm[^_]} $line] } { - set line "static $line" - } - puts $G(fd) $line - } -} - -proc lsm1c_close {} { - global G - puts -nonewline $G(fd) $G(footer) - if {$G(fd)!="stdout"} { - close $G(fd) - } -} - - -lsm1c_init lsm1.c -foreach f $G(src) { lsm1c_printfile $f } -lsm1c_close diff --git a/ext/misc/README.md b/ext/misc/README.md index cfc9e867c..6d34ab9af 100644 --- a/ext/misc/README.md +++ b/ext/misc/README.md @@ -9,11 +9,6 @@ Each source file contains a description in its header comment. See the header comments for details about each extension. Additional notes are as follows: - * **carray.c** — This module implements the - [carray](https://sqlite.org/carray.html) table-valued function. - It is a good example of how to go about implementing a custom - [table-valued function](https://sqlite.org/vtab.html#tabfunc2). - * **csv.c** — A [virtual table](https://sqlite.org/vtab.html) for reading [Comma-Separated-Value (CSV) files](https://en.wikipedia.org/wiki/Comma-separated_values). @@ -28,11 +23,6 @@ as follows: [SQLite amalgamation](https://sqlite.org/amalgamation.html). See for additional information. - * **memvfs.c** — This file implements a custom - [VFS](https://sqlite.org/vfs.html) that stores an entire database - file in a single block of RAM. It serves as a good example of how - to implement a simple custom VFS. - * **rot13.c** — This file implements the very simple rot13() substitution function. This file makes a good template for implementing new custom SQL functions for SQLite. diff --git a/ext/misc/amatch.c b/ext/misc/amatch.c index b3fcbac50..587c610b9 100644 --- a/ext/misc/amatch.c +++ b/ext/misc/amatch.c @@ -514,7 +514,7 @@ struct amatch_cursor { sqlite3_int64 iRowid; /* The rowid of the current word */ amatch_langid iLang; /* Use this language ID */ amatch_cost rLimit; /* Maximum cost of any term */ - int nBuf; /* Space allocated for zBuf */ + sqlite3_int64 nBuf; /* Space allocated for zBuf */ int oomErr; /* True following an OOM error */ int nWord; /* Number of amatch_word objects */ char *zBuf; /* Temp-use buffer space */ @@ -1039,7 +1039,7 @@ static void amatchAddWord( nTail = (int)strlen(zWordTail); if( nBase+nTail+3>pCur->nBuf ){ pCur->nBuf = nBase+nTail+100; - pCur->zBuf = sqlite3_realloc(pCur->zBuf, pCur->nBuf); + pCur->zBuf = sqlite3_realloc64(pCur->zBuf, pCur->nBuf); if( pCur->zBuf==0 ){ pCur->nBuf = 0; return; @@ -1105,13 +1105,13 @@ static int amatchNext(sqlite3_vtab_cursor *cur){ amatch_avl *pNode; int isMatch = 0; amatch_vtab *p = pCur->pVtab; - int nWord; + sqlite3_int64 nWord; int rc; int i; const char *zW; amatch_rule *pRule; char *zBuf = 0; - char nBuf = 0; + sqlite3_int64 nBuf = 0; char zNext[8]; char zNextIn[8]; int nNextIn; @@ -1158,7 +1158,7 @@ static int amatchNext(sqlite3_vtab_cursor *cur){ nWord = (int)strlen(pWord->zWord+2); if( nWord+20>nBuf ){ nBuf = (char)(nWord+100); - zBuf = sqlite3_realloc(zBuf, nBuf); + zBuf = sqlite3_realloc64(zBuf, nBuf); if( zBuf==0 ) return SQLITE_NOMEM; } amatchStrcpy(zBuf, pWord->zWord+2); diff --git a/ext/misc/base64.c b/ext/misc/base64.c index 17b3bbfc7..2da767bb0 100644 --- a/ext/misc/base64.c +++ b/ext/misc/base64.c @@ -207,7 +207,9 @@ static u8* fromBase64( char *pIn, int ncIn, u8 *pOut ){ /* This function does the work for the SQLite base64(x) UDF. */ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ - int nb, nc, nv = sqlite3_value_bytes(av[0]); + sqlite3_int64 nb; + sqlite3_int64 nv = sqlite3_value_bytes(av[0]); + sqlite3_int64 nc; int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), SQLITE_LIMIT_LENGTH, -1); char *cBuf; @@ -216,7 +218,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ switch( sqlite3_value_type(av[0]) ){ case SQLITE_BLOB: nb = nv; - nc = 4*(nv+2/3); /* quads needed */ + nc = 4*((nv+2)/3); /* quads needed */ nc += (nc+(B64_DARK_MAX-1))/B64_DARK_MAX + 1; /* LFs and a 0-terminator */ if( nvMax < nc ){ sqlite3_result_error(context, "blob expanded to base64 too big", -1); diff --git a/ext/misc/base85.c b/ext/misc/base85.c index eaf1732c4..63245e2e4 100644 --- a/ext/misc/base85.c +++ b/ext/misc/base85.c @@ -286,7 +286,7 @@ static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ /* This function does the work for the SQLite base85(x) UDF. */ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ - int nb, nc, nv = sqlite3_value_bytes(av[0]); + sqlite3_int64 nb, nc, nv = sqlite3_value_bytes(av[0]); int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), SQLITE_LIMIT_LENGTH, -1); char *cBuf; diff --git a/ext/misc/carray.h b/ext/misc/carray.h deleted file mode 100644 index ae0a9e8ab..000000000 --- a/ext/misc/carray.h +++ /dev/null @@ -1,50 +0,0 @@ -/* -** 2020-11-17 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** Interface definitions for the CARRAY table-valued function -** extension. -*/ - -#ifndef _CARRAY_H -#define _CARRAY_H - -#include "sqlite3.h" /* Required for error code definitions */ - -#ifdef __cplusplus -extern "C" { -#endif - -/* Use this interface to bind an array to the single-argument version -** of CARRAY(). -*/ -SQLITE_API int sqlite3_carray_bind( - sqlite3_stmt *pStmt, /* Statement to be bound */ - int i, /* Parameter index */ - void *aData, /* Pointer to array data */ - int nData, /* Number of data elements */ - int mFlags, /* CARRAY flags */ - void (*xDel)(void*) /* Destructgor for aData*/ -); - -/* Allowed values for the mFlags parameter to sqlite3_carray_bind(). -*/ -#define CARRAY_INT32 0 /* Data is 32-bit signed integers */ -#define CARRAY_INT64 1 /* Data is 64-bit signed integers */ -#define CARRAY_DOUBLE 2 /* Data is doubles */ -#define CARRAY_TEXT 3 /* Data is char* */ -#define CARRAY_BLOB 4 /* Data is struct iovec */ - -#ifdef __cplusplus -} /* end of the 'extern "C"' block */ -#endif - -#endif /* ifndef _CARRAY_H */ diff --git a/ext/misc/cksumvfs.c b/ext/misc/cksumvfs.c index 2d7f6584e..6491e85cd 100644 --- a/ext/misc/cksumvfs.c +++ b/ext/misc/cksumvfs.c @@ -197,8 +197,6 @@ struct CksmFile { char computeCksm; /* True to compute checksums. ** Always true if reserve size is 8. */ char verifyCksm; /* True to verify checksums */ - char isWal; /* True if processing a WAL file */ - char inCkpt; /* Currently doing a checkpoint */ CksmFile *pPartner; /* Ptr from WAL to main-db, or from main-db to WAL */ }; @@ -444,11 +442,9 @@ static int cksmRead( ** (1) the size indicates that we are dealing with a complete ** database page ** (2) checksum verification is enabled - ** (3) we are not in the middle of checkpoint */ if( iAmt>=512 && (iAmt & (iAmt-1))==0 /* (1) */ && p->verifyCksm /* (2) */ - && !p->inCkpt /* (3) */ ){ u8 cksum[8]; cksmCompute((u8*)zBuf, iAmt-8, cksum); @@ -489,7 +485,6 @@ static int cksmWrite( */ if( iAmt>=512 && p->computeCksm - && !p->inCkpt ){ cksmCompute((u8*)zBuf, iAmt-8, ((u8*)zBuf)+iAmt-8); } @@ -576,21 +571,6 @@ static int cksmFileControl(sqlite3_file *pFile, int op, void *pArg){ /* Do not allow page size changes on a checksum database */ return SQLITE_OK; } - }else if( op==SQLITE_FCNTL_CKPT_START || op==SQLITE_FCNTL_CKPT_DONE ){ - p->inCkpt = op==SQLITE_FCNTL_CKPT_START; - if( p->pPartner ) p->pPartner->inCkpt = p->inCkpt; - }else if( op==SQLITE_FCNTL_CKSM_FILE ){ - /* This VFS needs to obtain a pointer to the corresponding database - ** file handle from within xOpen() calls to open wal files. To do this, - ** it uses the sqlite3_database_file_object() API to obtain a pointer - ** to the file-handle used by SQLite to access the db file. This is - ** fine if cksmvfs happens to be the top-level VFS, but not if there - ** are one or more wrapper VFS. To handle this case, this file-control - ** is used to extract the cksmvfs file-handle from any wrapper file - ** handle. */ - sqlite3_file **ppFile = (sqlite3_file**)pArg; - *ppFile = (sqlite3_file*)p; - return SQLITE_OK; } rc = pFile->pMethods->xFileControl(pFile, op, pArg); if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ @@ -611,8 +591,10 @@ static int cksmSectorSize(sqlite3_file *pFile){ ** Return the device characteristic flags supported by a cksm-file. */ static int cksmDeviceCharacteristics(sqlite3_file *pFile){ + int devchar = 0; pFile = ORIGFILE(pFile); - return pFile->pMethods->xDeviceCharacteristics(pFile); + devchar = pFile->pMethods->xDeviceCharacteristics(pFile); + return (devchar & ~SQLITE_IOCAP_SUBPAGE_READ); } /* Create a shared memory file mapping */ @@ -689,7 +671,7 @@ static int cksmOpen( sqlite3_vfs *pSubVfs; int rc; pSubVfs = ORIGVFS(pVfs); - if( (flags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_WAL))==0 ){ + if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){ return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags); } p = (CksmFile*)pFile; @@ -698,19 +680,6 @@ static int cksmOpen( pFile->pMethods = &cksm_io_methods; rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags); if( rc ) goto cksm_open_done; - if( flags & SQLITE_OPEN_WAL ){ - sqlite3_file *pDb = sqlite3_database_file_object(zName); - rc = pDb->pMethods->xFileControl(pDb, SQLITE_FCNTL_CKSM_FILE, (void*)&pDb); - assert( rc==SQLITE_OK ); - p->pPartner = (CksmFile*)pDb; - assert( p->pPartner->pPartner==0 ); - p->pPartner->pPartner = p; - p->isWal = 1; - p->computeCksm = p->pPartner->computeCksm; - }else{ - p->isWal = 0; - p->computeCksm = 0; - } p->zFName = zName; cksm_open_done: if( rc ) pFile->pMethods = 0; @@ -820,9 +789,7 @@ static int cksmRegisterFunc( */ static int cksmRegisterVfs(void){ int rc = SQLITE_OK; - sqlite3_vfs *pOrig; - if( sqlite3_vfs_find("cksmvfs")!=0 ) return SQLITE_OK; - pOrig = sqlite3_vfs_find(0); + sqlite3_vfs *pOrig = sqlite3_vfs_find(0); if( pOrig==0 ) return SQLITE_ERROR; cksm_vfs.iVersion = pOrig->iVersion; cksm_vfs.pAppData = pOrig; diff --git a/ext/misc/completion.c b/ext/misc/completion.c index 0a6db1a22..67b40d84d 100644 --- a/ext/misc/completion.c +++ b/ext/misc/completion.c @@ -370,6 +370,7 @@ static int completionFilter( if( pCur->nPrefix>0 ){ pCur->zPrefix = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg])); if( pCur->zPrefix==0 ) return SQLITE_NOMEM; + pCur->nPrefix = (int)strlen(pCur->zPrefix); } iArg = 1; } @@ -378,6 +379,7 @@ static int completionFilter( if( pCur->nLine>0 ){ pCur->zLine = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg])); if( pCur->zLine==0 ) return SQLITE_NOMEM; + pCur->nLine = (int)strlen(pCur->zLine); } } if( pCur->zLine!=0 && pCur->zPrefix==0 ){ @@ -389,6 +391,7 @@ static int completionFilter( if( pCur->nPrefix>0 ){ pCur->zPrefix = sqlite3_mprintf("%.*s", pCur->nPrefix, pCur->zLine + i); if( pCur->zPrefix==0 ) return SQLITE_NOMEM; + pCur->nPrefix = (int)strlen(pCur->zPrefix); } } pCur->iRowid = 0; diff --git a/ext/misc/dbdump.c b/ext/misc/dbdump.c index ecf7d810d..0b93d2b9b 100644 --- a/ext/misc/dbdump.c +++ b/ext/misc/dbdump.c @@ -67,9 +67,9 @@ struct DState { */ typedef struct DText DText; struct DText { - char *z; /* The text */ - int n; /* Number of bytes of content in z[] */ - int nAlloc; /* Number of bytes allocated to z[] */ + char *z; /* The text */ + sqlite3_int64 n; /* Number of bytes of content in z[] */ + sqlite3_int64 nAlloc; /* Number of bytes allocated to z[] */ }; /* @@ -107,7 +107,7 @@ static void appendText(DText *p, char const *zAppend, char quote){ if( p->n+len>=p->nAlloc ){ char *zNew; p->nAlloc = p->nAlloc*2 + len + 20; - zNew = sqlite3_realloc(p->z, p->nAlloc); + zNew = sqlite3_realloc64(p->z, p->nAlloc); if( zNew==0 ){ freeText(p); return; @@ -179,8 +179,8 @@ static char **tableColumnList(DState *p, const char *zTab){ char **azCol = 0; sqlite3_stmt *pStmt = 0; char *zSql; - int nCol = 0; - int nAlloc = 0; + sqlite3_int64 nCol = 0; + sqlite3_int64 nAlloc = 0; int nPK = 0; /* Number of PRIMARY KEY columns seen */ int isIPK = 0; /* True if one PRIMARY KEY column of type INTEGER */ int preserveRowid = 1; diff --git a/ext/misc/decimal.c b/ext/misc/decimal.c index 60488a001..f87699f96 100644 --- a/ext/misc/decimal.c +++ b/ext/misc/decimal.c @@ -128,7 +128,8 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ } } if( iExp>0 ){ - p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); + p->a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + + (sqlite3_int64)iExp + 1 ); if( p->a==0 ) goto new_from_text_failed; memset(p->a+p->nDigit, 0, iExp); p->nDigit += iExp; @@ -147,7 +148,8 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ } } if( iExp>0 ){ - p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); + p->a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + + (sqlite3_int64)iExp + 1 ); if( p->a==0 ) goto new_from_text_failed; memmove(p->a+iExp, p->a, p->nDigit); memset(p->a, 0, iExp); @@ -155,6 +157,10 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ p->nFrac += iExp; } } + if( p->sign ){ + for(i=0; inDigit && p->a[i]==0; i++){} + if( i>=p->nDigit ) p->sign = 0; + } return p; new_from_text_failed: @@ -247,7 +253,7 @@ static void decimal_result(sqlite3_context *pCtx, Decimal *p){ sqlite3_result_null(pCtx); return; } - z = sqlite3_malloc( p->nDigit+4 ); + z = sqlite3_malloc64( (sqlite3_int64)p->nDigit+4 ); if( z==0 ){ sqlite3_result_error_nomem(pCtx); return; @@ -312,7 +318,7 @@ static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ for(nZero=0; nZeroa[nZero]==0; nZero++){} nFrac = p->nFrac + (nDigit - p->nDigit); nDigit -= nZero; - z = sqlite3_malloc( nDigit+20 ); + z = sqlite3_malloc64( (sqlite3_int64)nDigit+20 ); if( z==0 ){ sqlite3_result_error_nomem(pCtx); return; @@ -357,13 +363,21 @@ static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ ** pB!=0 ** pB->isNull==0 */ -static int decimal_cmp(const Decimal *pA, const Decimal *pB){ +static int decimal_cmp(Decimal *pA, Decimal *pB){ int nASig, nBSig, rc, n; + while( pA->nFrac>0 && pA->a[pA->nDigit-1]==0 ){ + pA->nDigit--; + pA->nFrac--; + } + while( pB->nFrac>0 && pB->a[pB->nDigit-1]==0 ){ + pB->nDigit--; + pB->nFrac--; + } if( pA->sign!=pB->sign ){ return pA->sign ? -1 : +1; } if( pA->sign ){ - const Decimal *pTemp = pA; + Decimal *pTemp = pA; pA = pB; pB = pTemp; } @@ -525,7 +539,8 @@ static void decimalMul(Decimal *pA, Decimal *pB){ ){ goto mul_end; } - acc = sqlite3_malloc64( pA->nDigit + pB->nDigit + 2 ); + acc = sqlite3_malloc64( (sqlite3_int64)pA->nDigit + + (sqlite3_int64)pB->nDigit + 2 ); if( acc==0 ){ pA->oom = 1; goto mul_end; @@ -612,7 +627,7 @@ static Decimal *decimalFromDouble(double r){ isNeg = 0; } memcpy(&a,&r,sizeof(a)); - if( a==0 ){ + if( a==0 || a==(sqlite3_int64)0x8000000000000000LL){ e = 0; m = 0; }else{ diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c index 03c911712..d78b14877 100644 --- a/ext/misc/fileio.c +++ b/ext/misc/fileio.c @@ -67,6 +67,7 @@ ** data: For a regular file, a blob containing the file data. For a ** symlink, a text value containing the text of the link. For a ** directory, NULL. +** level: Directory hierarchy level. Topmost is 1. ** ** If a non-NULL value is specified for the optional $dir parameter and ** $path is a relative path, then $path is interpreted relative to $dir. @@ -94,11 +95,8 @@ SQLITE_EXTENSION_INIT1 # include # define STRUCT_STAT struct stat #else -# include "windows.h" -# include +# include "windirent.h" # include -# include "test_windirent.h" -# define dirent DIRENT # define STRUCT_STAT struct _stat # define chmod(path,mode) fileio_chmod(path,mode) # define mkdir(path,mode) fileio_mkdir(path) @@ -117,14 +115,16 @@ SQLITE_EXTENSION_INIT1 /* ** Structure of the fsdir() table-valued function */ - /* 0 1 2 3 4 5 */ -#define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)" + /* 0 1 2 3 4 5 6 */ +#define FSDIR_SCHEMA "(name,mode,mtime,data,level,path HIDDEN,dir HIDDEN)" + #define FSDIR_COLUMN_NAME 0 /* Name of the file */ #define FSDIR_COLUMN_MODE 1 /* Access mode */ #define FSDIR_COLUMN_MTIME 2 /* Last modification time */ #define FSDIR_COLUMN_DATA 3 /* File content */ -#define FSDIR_COLUMN_PATH 4 /* Path to top of search */ -#define FSDIR_COLUMN_DIR 5 /* Path is relative to this directory */ +#define FSDIR_COLUMN_LEVEL 4 /* Level. Topmost is 1 */ +#define FSDIR_COLUMN_PATH 5 /* Path to top of search */ +#define FSDIR_COLUMN_DIR 6 /* Path is relative to this directory */ /* ** UTF8 chmod() function for Windows @@ -621,6 +621,7 @@ struct fsdir_cursor { sqlite3_vtab_cursor base; /* Base class - must be first */ int nLvl; /* Number of entries in aLvl[] array */ + int mxLvl; /* Maximum level */ int iLvl; /* Index of current entry */ FsdirLevel *aLvl; /* Hierarchy of directories being traversed */ @@ -739,7 +740,7 @@ static int fsdirNext(sqlite3_vtab_cursor *cur){ mode_t m = pCur->sStat.st_mode; pCur->iRowid++; - if( S_ISDIR(m) ){ + if( S_ISDIR(m) && pCur->iLvl+3mxLvl ){ /* Descend into this directory */ int iNew = pCur->iLvl + 1; FsdirLevel *pLvl; @@ -759,7 +760,7 @@ static int fsdirNext(sqlite3_vtab_cursor *cur){ pCur->zPath = 0; pLvl->pDir = opendir(pLvl->zDir); if( pLvl->pDir==0 ){ - fsdirSetErrmsg(pCur, "cannot read directory: %s", pCur->zPath); + fsdirSetErrmsg(pCur, "cannot read directory: %s", pLvl->zDir); return SQLITE_ERROR; } } @@ -847,7 +848,11 @@ static int fsdirColumn( }else{ readFileContents(ctx, pCur->zPath); } + break; } + case FSDIR_COLUMN_LEVEL: + sqlite3_result_int(ctx, pCur->iLvl+2); + break; case FSDIR_COLUMN_PATH: default: { /* The FSDIR_COLUMN_PATH and FSDIR_COLUMN_DIR are input parameters. @@ -881,8 +886,11 @@ static int fsdirEof(sqlite3_vtab_cursor *cur){ /* ** xFilter callback. ** -** idxNum==1 PATH parameter only -** idxNum==2 Both PATH and DIR supplied +** idxNum bit Meaning +** 0x01 PATH=N +** 0x02 DIR=N +** 0x04 LEVEL0 ); zDir = (const char*)sqlite3_value_text(argv[0]); if( zDir==0 ){ fsdirSetErrmsg(pCur, "table function fsdir requires a non-NULL argument"); return SQLITE_ERROR; } - if( argc==2 ){ - pCur->zBase = (const char*)sqlite3_value_text(argv[1]); + i = 1; + if( (idxNum & 0x02)!=0 ){ + assert( argc>i ); + pCur->zBase = (const char*)sqlite3_value_text(argv[i++]); + } + if( (idxNum & 0x0c)!=0 ){ + assert( argc>i ); + pCur->mxLvl = sqlite3_value_int(argv[i++]); + if( idxNum & 0x08 ) pCur->mxLvl++; + if( pCur->mxLvl<=0 ) pCur->mxLvl = 1000000000; + }else{ + pCur->mxLvl = 1000000000; } if( pCur->zBase ){ pCur->nBase = (int)strlen(pCur->zBase)+1; @@ -935,10 +954,11 @@ static int fsdirFilter( ** In this implementation idxNum is used to represent the ** query plan. idxStr is unused. ** -** The query plan is represented by values of idxNum: +** The query plan is represented by bits in idxNum: ** -** (1) The path value is supplied by argv[0] -** (2) Path is in argv[0] and dir is in argv[1] +** 0x01 The path value is supplied by argv[0] +** 0x02 dir is in argv[1] +** 0x04 maxdepth is in argv[1] or [2] */ static int fsdirBestIndex( sqlite3_vtab *tab, @@ -947,6 +967,9 @@ static int fsdirBestIndex( int i; /* Loop over constraints */ int idxPath = -1; /* Index in pIdxInfo->aConstraint of PATH= */ int idxDir = -1; /* Index in pIdxInfo->aConstraint of DIR= */ + int idxLevel = -1; /* Index in pIdxInfo->aConstraint of LEVEL< or <= */ + int idxLevelEQ = 0; /* 0x08 for LEVEL<= or LEVEL=. 0x04 for LEVEL< */ + int omitLevel = 0; /* omit the LEVEL constraint */ int seenPath = 0; /* True if an unusable PATH= constraint is seen */ int seenDir = 0; /* True if an unusable DIR= constraint is seen */ const struct sqlite3_index_constraint *pConstraint; @@ -954,25 +977,48 @@ static int fsdirBestIndex( (void)tab; pConstraint = pIdxInfo->aConstraint; for(i=0; inConstraint; i++, pConstraint++){ - if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; - switch( pConstraint->iColumn ){ - case FSDIR_COLUMN_PATH: { - if( pConstraint->usable ){ - idxPath = i; - seenPath = 0; - }else if( idxPath<0 ){ - seenPath = 1; + if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + switch( pConstraint->iColumn ){ + case FSDIR_COLUMN_PATH: { + if( pConstraint->usable ){ + idxPath = i; + seenPath = 0; + }else if( idxPath<0 ){ + seenPath = 1; + } + break; } - break; - } - case FSDIR_COLUMN_DIR: { - if( pConstraint->usable ){ - idxDir = i; - seenDir = 0; - }else if( idxDir<0 ){ - seenDir = 1; + case FSDIR_COLUMN_DIR: { + if( pConstraint->usable ){ + idxDir = i; + seenDir = 0; + }else if( idxDir<0 ){ + seenDir = 1; + } + break; + } + case FSDIR_COLUMN_LEVEL: { + if( pConstraint->usable && idxLevel<0 ){ + idxLevel = i; + idxLevelEQ = 0x08; + omitLevel = 0; + } + break; } - break; + } + }else + if( pConstraint->iColumn==FSDIR_COLUMN_LEVEL + && pConstraint->usable + && idxLevel<0 + ){ + if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE ){ + idxLevel = i; + idxLevelEQ = 0x08; + omitLevel = 1; + }else if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT ){ + idxLevel = i; + idxLevelEQ = 0x04; + omitLevel = 1; } } } @@ -989,14 +1035,20 @@ static int fsdirBestIndex( }else{ pIdxInfo->aConstraintUsage[idxPath].omit = 1; pIdxInfo->aConstraintUsage[idxPath].argvIndex = 1; + pIdxInfo->idxNum = 0x01; + pIdxInfo->estimatedCost = 1.0e9; + i = 2; if( idxDir>=0 ){ pIdxInfo->aConstraintUsage[idxDir].omit = 1; - pIdxInfo->aConstraintUsage[idxDir].argvIndex = 2; - pIdxInfo->idxNum = 2; - pIdxInfo->estimatedCost = 10.0; - }else{ - pIdxInfo->idxNum = 1; - pIdxInfo->estimatedCost = 100.0; + pIdxInfo->aConstraintUsage[idxDir].argvIndex = i++; + pIdxInfo->idxNum |= 0x02; + pIdxInfo->estimatedCost /= 1.0e4; + } + if( idxLevel>=0 ){ + pIdxInfo->aConstraintUsage[idxLevel].omit = omitLevel; + pIdxInfo->aConstraintUsage[idxLevel].argvIndex = i++; + pIdxInfo->idxNum |= idxLevelEQ; + pIdxInfo->estimatedCost /= 1.0e4; } } diff --git a/ext/misc/fossildelta.c b/ext/misc/fossildelta.c index 9f81270d7..d24a87700 100644 --- a/ext/misc/fossildelta.c +++ b/ext/misc/fossildelta.c @@ -27,7 +27,7 @@ ** This delta format is used by the RBU extension, which is the main ** reason that these routines are included in the extension library. ** RBU does not use this extension directly. Rather, this extension is -** provided as a convenience to developers who want to analyze RBU files +** provided as a convenience to developers who want to analyze RBU files ** that contain deltas. */ #include @@ -868,11 +868,21 @@ static int deltaparsevtabNext(sqlite3_vtab_cursor *cur){ int i = 0; pCur->iCursor = pCur->iNext; + if( pCur->iCursor >= pCur->nDelta ){ + pCur->eOp = DELTAPARSE_OP_ERROR; + pCur->iNext = pCur->nDelta; + return SQLITE_OK; + } z = pCur->aDelta + pCur->iCursor; pCur->a1 = deltaGetInt(&z, &i); switch( z[0] ){ case '@': { z++; + if( pCur->iNext>=pCur->nDelta ){ + pCur->eOp = DELTAPARSE_OP_ERROR; + pCur->iNext = pCur->nDelta; + break; + } pCur->a2 = deltaGetInt(&z, &i); pCur->eOp = DELTAPARSE_OP_COPY; pCur->iNext = (int)(&z[1] - pCur->aDelta); @@ -926,8 +936,12 @@ static int deltaparsevtabColumn( if( pCur->eOp==DELTAPARSE_OP_COPY ){ sqlite3_result_int(ctx, pCur->a2); }else if( pCur->eOp==DELTAPARSE_OP_INSERT ){ - sqlite3_result_blob(ctx, pCur->aDelta+pCur->a2, pCur->a1, - SQLITE_TRANSIENT); + if( pCur->a2 + pCur->a1 > pCur->nDelta ){ + sqlite3_result_zeroblob(ctx, pCur->a1); + }else{ + sqlite3_result_blob(ctx, pCur->aDelta+pCur->a2, pCur->a1, + SQLITE_TRANSIENT); + } } break; } @@ -955,17 +969,17 @@ static int deltaparsevtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ */ static int deltaparsevtabEof(sqlite3_vtab_cursor *cur){ deltaparsevtab_cursor *pCur = (deltaparsevtab_cursor*)cur; - return pCur->eOp==DELTAPARSE_OP_EOF; + return pCur->eOp==DELTAPARSE_OP_EOF || pCur->iCursor>=pCur->nDelta; } /* ** This method is called to "rewind" the deltaparsevtab_cursor object back ** to the first row of output. This method is always called at least -** once prior to any call to deltaparsevtabColumn() or deltaparsevtabRowid() or +** once prior to any call to deltaparsevtabColumn() or deltaparsevtabRowid() or ** deltaparsevtabEof(). */ static int deltaparsevtabFilter( - sqlite3_vtab_cursor *pVtabCursor, + sqlite3_vtab_cursor *pVtabCursor, int idxNum, const char *idxStr, int argc, sqlite3_value **argv ){ @@ -1031,7 +1045,7 @@ static int deltaparsevtabBestIndex( } /* -** This following structure defines all the methods for the +** This following structure defines all the methods for the ** virtual table. */ static sqlite3_module deltaparsevtabModule = { @@ -1068,8 +1082,8 @@ static sqlite3_module deltaparsevtabModule = { __declspec(dllexport) #endif int sqlite3_fossildelta_init( - sqlite3 *db, - char **pzErrMsg, + sqlite3 *db, + char **pzErrMsg, const sqlite3_api_routines *pApi ){ static const int enc = SQLITE_UTF8|SQLITE_INNOCUOUS; diff --git a/ext/misc/ieee754.c b/ext/misc/ieee754.c index 5ddb4a2fe..7f1c6d9e1 100644 --- a/ext/misc/ieee754.c +++ b/ext/misc/ieee754.c @@ -134,6 +134,9 @@ static void ieee754func( if( a==0 ){ e = 0; m = 0; + }else if( a==(sqlite3_int64)0x8000000000000000LL ){ + e = -1996; + m = -1; }else{ e = a>>52; m = a & ((((sqlite3_int64)1)<<52)-1); diff --git a/ext/misc/memvfs.c b/ext/misc/memvfs.c deleted file mode 100644 index 83fc9468e..000000000 --- a/ext/misc/memvfs.c +++ /dev/null @@ -1,575 +0,0 @@ -/* -** 2016-09-07 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This is an in-memory VFS implementation. The application supplies -** a chunk of memory to hold the database file. -** -** Because there is place to store a rollback or wal journal, the database -** must use one of journal_mode=MEMORY or journal_mode=NONE. -** -** USAGE: -** -** sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336&max=65536", &db, -** SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI, -** "memvfs"); -** -** These are the query parameters: -** -** ptr= The address of the memory buffer that holds the database. -** -** sz= The current size the database file -** -** maxsz= The maximum size of the database. In other words, the -** amount of space allocated for the ptr= buffer. -** -** freeonclose= If true, then sqlite3_free() is called on the ptr= -** value when the connection closes. -** -** The ptr= and sz= query parameters are required. If maxsz= is omitted, -** then it defaults to the sz= value. Parameter values can be in either -** decimal or hexadecimal. The filename in the URI is ignored. -*/ -#include -SQLITE_EXTENSION_INIT1 -#include -#include - - -/* -** Forward declaration of objects used by this utility -*/ -typedef struct sqlite3_vfs MemVfs; -typedef struct MemFile MemFile; - -/* Access to a lower-level VFS that (might) implement dynamic loading, -** access to randomness, etc. -*/ -#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) - -/* An open file */ -struct MemFile { - sqlite3_file base; /* IO methods */ - sqlite3_int64 sz; /* Size of the file */ - sqlite3_int64 szMax; /* Space allocated to aData */ - unsigned char *aData; /* content of the file */ - int bFreeOnClose; /* Invoke sqlite3_free() on aData at close */ -}; - -/* -** Methods for MemFile -*/ -static int memClose(sqlite3_file*); -static int memRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); -static int memWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); -static int memTruncate(sqlite3_file*, sqlite3_int64 size); -static int memSync(sqlite3_file*, int flags); -static int memFileSize(sqlite3_file*, sqlite3_int64 *pSize); -static int memLock(sqlite3_file*, int); -static int memUnlock(sqlite3_file*, int); -static int memCheckReservedLock(sqlite3_file*, int *pResOut); -static int memFileControl(sqlite3_file*, int op, void *pArg); -static int memSectorSize(sqlite3_file*); -static int memDeviceCharacteristics(sqlite3_file*); -static int memShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); -static int memShmLock(sqlite3_file*, int offset, int n, int flags); -static void memShmBarrier(sqlite3_file*); -static int memShmUnmap(sqlite3_file*, int deleteFlag); -static int memFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); -static int memUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); - -/* -** Methods for MemVfs -*/ -static int memOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); -static int memDelete(sqlite3_vfs*, const char *zName, int syncDir); -static int memAccess(sqlite3_vfs*, const char *zName, int flags, int *); -static int memFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); -static void *memDlOpen(sqlite3_vfs*, const char *zFilename); -static void memDlError(sqlite3_vfs*, int nByte, char *zErrMsg); -static void (*memDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); -static void memDlClose(sqlite3_vfs*, void*); -static int memRandomness(sqlite3_vfs*, int nByte, char *zOut); -static int memSleep(sqlite3_vfs*, int microseconds); -static int memCurrentTime(sqlite3_vfs*, double*); -static int memGetLastError(sqlite3_vfs*, int, char *); -static int memCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); - -static sqlite3_vfs mem_vfs = { - 2, /* iVersion */ - 0, /* szOsFile (set when registered) */ - 1024, /* mxPathname */ - 0, /* pNext */ - "memvfs", /* zName */ - 0, /* pAppData (set when registered) */ - memOpen, /* xOpen */ - memDelete, /* xDelete */ - memAccess, /* xAccess */ - memFullPathname, /* xFullPathname */ - memDlOpen, /* xDlOpen */ - memDlError, /* xDlError */ - memDlSym, /* xDlSym */ - memDlClose, /* xDlClose */ - memRandomness, /* xRandomness */ - memSleep, /* xSleep */ - memCurrentTime, /* xCurrentTime */ - memGetLastError, /* xGetLastError */ - memCurrentTimeInt64 /* xCurrentTimeInt64 */ -}; - -static const sqlite3_io_methods mem_io_methods = { - 3, /* iVersion */ - memClose, /* xClose */ - memRead, /* xRead */ - memWrite, /* xWrite */ - memTruncate, /* xTruncate */ - memSync, /* xSync */ - memFileSize, /* xFileSize */ - memLock, /* xLock */ - memUnlock, /* xUnlock */ - memCheckReservedLock, /* xCheckReservedLock */ - memFileControl, /* xFileControl */ - memSectorSize, /* xSectorSize */ - memDeviceCharacteristics, /* xDeviceCharacteristics */ - memShmMap, /* xShmMap */ - memShmLock, /* xShmLock */ - memShmBarrier, /* xShmBarrier */ - memShmUnmap, /* xShmUnmap */ - memFetch, /* xFetch */ - memUnfetch /* xUnfetch */ -}; - - - -/* -** Close an mem-file. -** -** The pData pointer is owned by the application, so there is nothing -** to free. -*/ -static int memClose(sqlite3_file *pFile){ - MemFile *p = (MemFile *)pFile; - if( p->bFreeOnClose ) sqlite3_free(p->aData); - return SQLITE_OK; -} - -/* -** Read data from an mem-file. -*/ -static int memRead( - sqlite3_file *pFile, - void *zBuf, - int iAmt, - sqlite_int64 iOfst -){ - MemFile *p = (MemFile *)pFile; - memcpy(zBuf, p->aData+iOfst, iAmt); - return SQLITE_OK; -} - -/* -** Write data to an mem-file. -*/ -static int memWrite( - sqlite3_file *pFile, - const void *z, - int iAmt, - sqlite_int64 iOfst -){ - MemFile *p = (MemFile *)pFile; - if( iOfst+iAmt>p->sz ){ - if( iOfst+iAmt>p->szMax ) return SQLITE_FULL; - if( iOfst>p->sz ) memset(p->aData+p->sz, 0, iOfst-p->sz); - p->sz = iOfst+iAmt; - } - memcpy(p->aData+iOfst, z, iAmt); - return SQLITE_OK; -} - -/* -** Truncate an mem-file. -*/ -static int memTruncate(sqlite3_file *pFile, sqlite_int64 size){ - MemFile *p = (MemFile *)pFile; - if( size>p->sz ){ - if( size>p->szMax ) return SQLITE_FULL; - memset(p->aData+p->sz, 0, size-p->sz); - } - p->sz = size; - return SQLITE_OK; -} - -/* -** Sync an mem-file. -*/ -static int memSync(sqlite3_file *pFile, int flags){ - return SQLITE_OK; -} - -/* -** Return the current file-size of an mem-file. -*/ -static int memFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ - MemFile *p = (MemFile *)pFile; - *pSize = p->sz; - return SQLITE_OK; -} - -/* -** Lock an mem-file. -*/ -static int memLock(sqlite3_file *pFile, int eLock){ - return SQLITE_OK; -} - -/* -** Unlock an mem-file. -*/ -static int memUnlock(sqlite3_file *pFile, int eLock){ - return SQLITE_OK; -} - -/* -** Check if another file-handle holds a RESERVED lock on an mem-file. -*/ -static int memCheckReservedLock(sqlite3_file *pFile, int *pResOut){ - *pResOut = 0; - return SQLITE_OK; -} - -/* -** File control method. For custom operations on an mem-file. -*/ -static int memFileControl(sqlite3_file *pFile, int op, void *pArg){ - MemFile *p = (MemFile *)pFile; - int rc = SQLITE_NOTFOUND; - if( op==SQLITE_FCNTL_VFSNAME ){ - *(char**)pArg = sqlite3_mprintf("mem(%p,%lld)", p->aData, p->sz); - rc = SQLITE_OK; - } - return rc; -} - -/* -** Return the sector-size in bytes for an mem-file. -*/ -static int memSectorSize(sqlite3_file *pFile){ - return 1024; -} - -/* -** Return the device characteristic flags supported by an mem-file. -*/ -static int memDeviceCharacteristics(sqlite3_file *pFile){ - return SQLITE_IOCAP_ATOMIC | - SQLITE_IOCAP_POWERSAFE_OVERWRITE | - SQLITE_IOCAP_SAFE_APPEND | - SQLITE_IOCAP_SEQUENTIAL; -} - -/* Create a shared memory file mapping */ -static int memShmMap( - sqlite3_file *pFile, - int iPg, - int pgsz, - int bExtend, - void volatile **pp -){ - return SQLITE_IOERR_SHMMAP; -} - -/* Perform locking on a shared-memory segment */ -static int memShmLock(sqlite3_file *pFile, int offset, int n, int flags){ - return SQLITE_IOERR_SHMLOCK; -} - -/* Memory barrier operation on shared memory */ -static void memShmBarrier(sqlite3_file *pFile){ - return; -} - -/* Unmap a shared memory segment */ -static int memShmUnmap(sqlite3_file *pFile, int deleteFlag){ - return SQLITE_OK; -} - -/* Fetch a page of a memory-mapped file */ -static int memFetch( - sqlite3_file *pFile, - sqlite3_int64 iOfst, - int iAmt, - void **pp -){ - MemFile *p = (MemFile *)pFile; - *pp = (void*)(p->aData + iOfst); - return SQLITE_OK; -} - -/* Release a memory-mapped page */ -static int memUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ - return SQLITE_OK; -} - -/* -** Open an mem file handle. -*/ -static int memOpen( - sqlite3_vfs *pVfs, - const char *zName, - sqlite3_file *pFile, - int flags, - int *pOutFlags -){ - MemFile *p = (MemFile*)pFile; - memset(p, 0, sizeof(*p)); - if( (flags & SQLITE_OPEN_MAIN_DB)==0 ) return SQLITE_CANTOPEN; - p->aData = (unsigned char*)sqlite3_uri_int64(zName,"ptr",0); - if( p->aData==0 ) return SQLITE_CANTOPEN; - p->sz = sqlite3_uri_int64(zName,"sz",0); - if( p->sz<0 ) return SQLITE_CANTOPEN; - p->szMax = sqlite3_uri_int64(zName,"max",p->sz); - if( p->szMaxsz ) return SQLITE_CANTOPEN; - p->bFreeOnClose = sqlite3_uri_boolean(zName,"freeonclose",0); - pFile->pMethods = &mem_io_methods; - return SQLITE_OK; -} - -/* -** Delete the file located at zPath. If the dirSync argument is true, -** ensure the file-system modifications are synced to disk before -** returning. -*/ -static int memDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ - return SQLITE_IOERR_DELETE; -} - -/* -** Test for access permissions. Return true if the requested permission -** is available, or false otherwise. -*/ -static int memAccess( - sqlite3_vfs *pVfs, - const char *zPath, - int flags, - int *pResOut -){ - *pResOut = 0; - return SQLITE_OK; -} - -/* -** Populate buffer zOut with the full canonical pathname corresponding -** to the pathname in zPath. zOut is guaranteed to point to a buffer -** of at least (INST_MAX_PATHNAME+1) bytes. -*/ -static int memFullPathname( - sqlite3_vfs *pVfs, - const char *zPath, - int nOut, - char *zOut -){ - sqlite3_snprintf(nOut, zOut, "%s", zPath); - return SQLITE_OK; -} - -/* -** Open the dynamic library located at zPath and return a handle. -*/ -static void *memDlOpen(sqlite3_vfs *pVfs, const char *zPath){ - return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); -} - -/* -** Populate the buffer zErrMsg (size nByte bytes) with a human readable -** utf-8 string describing the most recent error encountered associated -** with dynamic libraries. -*/ -static void memDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ - ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); -} - -/* -** Return a pointer to the symbol zSymbol in the dynamic library pHandle. -*/ -static void (*memDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ - return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); -} - -/* -** Close the dynamic library handle pHandle. -*/ -static void memDlClose(sqlite3_vfs *pVfs, void *pHandle){ - ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); -} - -/* -** Populate the buffer pointed to by zBufOut with nByte bytes of -** random data. -*/ -static int memRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ - return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); -} - -/* -** Sleep for nMicro microseconds. Return the number of microseconds -** actually slept. -*/ -static int memSleep(sqlite3_vfs *pVfs, int nMicro){ - return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); -} - -/* -** Return the current time as a Julian Day number in *pTimeOut. -*/ -static int memCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ - return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); -} - -static int memGetLastError(sqlite3_vfs *pVfs, int a, char *b){ - return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); -} -static int memCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ - return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); -} - -#ifdef MEMVFS_TEST -/* -** memvfs_from_file(FILENAME, MAXSIZE) -** -** This an SQL function used to help in testing the memvfs VFS. The -** function reads the content of a file into memory and then returns -** a URI that can be handed to ATTACH to attach the memory buffer as -** a database. Example: -** -** ATTACH memvfs_from_file('test.db',1048576) AS inmem; -** -** The optional MAXSIZE argument gives the size of the memory allocation -** used to hold the database. If omitted, it defaults to the size of the -** file on disk. -*/ -#include -static void memvfsFromFileFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - unsigned char *p; - sqlite3_int64 sz; - sqlite3_int64 szMax; - FILE *in; - const char *zFilename = (const char*)sqlite3_value_text(argv[0]); - char *zUri; - - if( zFilename==0 ) return; - in = fopen(zFilename, "rb"); - if( in==0 ) return; - fseek(in, 0, SEEK_END); - szMax = sz = ftell(in); - rewind(in); - if( argc>=2 ){ - szMax = sqlite3_value_int64(argv[1]); - if( szMaxzName,"memvfs")!=0 ) return; - rc = sqlite3_file_control(db, zSchema, SQLITE_FCNTL_FILE_POINTER, &p); - if( rc ) return; - fwrite(p->aData, 1, (size_t)p->sz, out); - fclose(out); -} -#endif /* MEMVFS_TEST */ - -#ifdef MEMVFS_TEST -/* Called for each new database connection */ -static int memvfsRegister( - sqlite3 *db, - char **pzErrMsg, - const struct sqlite3_api_routines *pThunk -){ - sqlite3_create_function(db, "memvfs_from_file", 1, SQLITE_UTF8, 0, - memvfsFromFileFunc, 0, 0); - sqlite3_create_function(db, "memvfs_from_file", 2, SQLITE_UTF8, 0, - memvfsFromFileFunc, 0, 0); - sqlite3_create_function(db, "memvfs_to_file", 2, SQLITE_UTF8, 0, - memvfsToFileFunc, 0, 0); - return SQLITE_OK; -} -#endif /* MEMVFS_TEST */ - - -#ifdef _WIN32 -__declspec(dllexport) -#endif -/* -** This routine is called when the extension is loaded. -** Register the new VFS. -*/ -int sqlite3_memvfs_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - int rc = SQLITE_OK; - SQLITE_EXTENSION_INIT2(pApi); - mem_vfs.pAppData = sqlite3_vfs_find(0); - if( mem_vfs.pAppData==0 ) return SQLITE_ERROR; - mem_vfs.szOsFile = sizeof(MemFile); - rc = sqlite3_vfs_register(&mem_vfs, 1); -#ifdef MEMVFS_TEST - if( rc==SQLITE_OK ){ - rc = sqlite3_auto_extension((void(*)(void))memvfsRegister); - } - if( rc==SQLITE_OK ){ - rc = memvfsRegister(db, pzErrMsg, pApi); - } -#endif - if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; - return rc; -} diff --git a/ext/misc/normalize.c b/ext/misc/normalize.c index 800e12911..44ddcd388 100644 --- a/ext/misc/normalize.c +++ b/ext/misc/normalize.c @@ -297,8 +297,9 @@ static const unsigned char sqlite3CtypeMap[256] = { ** Return the length (in bytes) of the token that begins at z[0]. ** Store the token type in *tokenType before returning. */ -static int sqlite3GetToken(const unsigned char *z, int *tokenType){ - int i, c; +static sqlite3_int64 sqlite3GetToken(const unsigned char *z, int *tokenType){ + sqlite3_int64 i; + int c; switch( aiClass[*z] ){ /* Switch on the character-class of the first byte ** of the token. See the comment on the CC_ defines ** above. */ @@ -559,7 +560,7 @@ char *sqlite3_normalize(const char *zSql){ int i; /* Next character to read from zSql[] */ int j; /* Next slot to fill in on z[] */ int tokenType; /* Type of the next token */ - int n; /* Size of the next token */ + sqlite3_int64 n; /* Size of the next token */ int k; /* Loop counter */ nSql = strlen(zSql); diff --git a/ext/misc/regexp.c b/ext/misc/regexp.c index 71c7ea4c2..f1babf4ab 100644 --- a/ext/misc/regexp.c +++ b/ext/misc/regexp.c @@ -55,6 +55,8 @@ ** to p copies of X following by q-p copies of X? and that the size of the ** regular expression in the O(N*M) performance bound is computed after ** this expansion. +** +** To help prevent DoS attacks, the maximum size of the NFA is restricted. */ #include #include @@ -96,32 +98,6 @@ SQLITE_EXTENSION_INIT1 #define RE_OP_BOUNDARY 17 /* Boundary between word and non-word */ #define RE_OP_ATSTART 18 /* Currently at the start of the string */ -#if defined(SQLITE_DEBUG) -/* Opcode names used for symbolic debugging */ -static const char *ReOpName[] = { - "EOF", - "MATCH", - "ANY", - "ANYSTAR", - "FORK", - "GOTO", - "ACCEPT", - "CC_INC", - "CC_EXC", - "CC_VALUE", - "CC_RANGE", - "WORD", - "NOTWORD", - "DIGIT", - "NOTDIGIT", - "SPACE", - "NOTSPACE", - "BOUNDARY", - "ATSTART", -}; -#endif /* SQLITE_DEBUG */ - - /* Each opcode is a "state" in the NFA */ typedef unsigned short ReStateNumber; @@ -158,6 +134,7 @@ struct ReCompiled { int nInit; /* Number of bytes in zInit */ unsigned nState; /* Number of entries in aOp[] and aArg[] */ unsigned nAlloc; /* Slots allocated for aOp[] and aArg[] */ + unsigned mxAlloc; /* Complexity limit */ }; /* Add a state to the given state set if it is not already there */ @@ -372,14 +349,15 @@ static int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){ /* Resize the opcode and argument arrays for an RE under construction. */ -static int re_resize(ReCompiled *p, int N){ +static int re_resize(ReCompiled *p, unsigned int N){ char *aOp; int *aArg; + if( N>p->mxAlloc ){ p->zErr = "REGEXP pattern too big"; return 1; } aOp = sqlite3_realloc64(p->aOp, N*sizeof(p->aOp[0])); - if( aOp==0 ) return 1; + if( aOp==0 ){ p->zErr = "out of memory"; return 1; } p->aOp = aOp; aArg = sqlite3_realloc64(p->aArg, N*sizeof(p->aArg[0])); - if( aArg==0 ) return 1; + if( aArg==0 ){ p->zErr = "out of memory"; return 1; } p->aArg = aArg; p->nAlloc = N; return 0; @@ -410,7 +388,7 @@ static int re_append(ReCompiled *p, int op, int arg){ /* Make a copy of N opcodes starting at iStart onto the end of the RE ** under construction. */ -static void re_copy(ReCompiled *p, int iStart, int N){ +static void re_copy(ReCompiled *p, int iStart, unsigned int N){ if( p->nState+N>=p->nAlloc && re_resize(p, p->nAlloc*2+N) ) return; memcpy(&p->aOp[p->nState], &p->aOp[iStart], N*sizeof(p->aOp[0])); memcpy(&p->aArg[p->nState], &p->aArg[iStart], N*sizeof(p->aArg[0])); @@ -563,18 +541,26 @@ static const char *re_subcompile_string(ReCompiled *p){ break; } case '{': { - int m = 0, n = 0; - int sz, j; + unsigned int m = 0, n = 0; + unsigned int sz, j; if( iPrev<0 ) return "'{m,n}' without operand"; - while( (c=rePeek(p))>='0' && c<='9' ){ m = m*10 + c - '0'; p->sIn.i++; } + while( (c=rePeek(p))>='0' && c<='9' ){ + m = m*10 + c - '0'; + if( m*2>p->mxAlloc ) return "REGEXP pattern too big"; + p->sIn.i++; + } n = m; if( c==',' ){ p->sIn.i++; n = 0; - while( (c=rePeek(p))>='0' && c<='9' ){ n = n*10 + c-'0'; p->sIn.i++; } + while( (c=rePeek(p))>='0' && c<='9' ){ + n = n*10 + c-'0'; + if( n*2>p->mxAlloc ) return "REGEXP pattern too big"; + p->sIn.i++; + } } if( c!='}' ) return "unmatched '{'"; - if( n>0 && nsIn.i++; sz = p->nState - iPrev; if( m==0 ){ @@ -590,7 +576,7 @@ static const char *re_subcompile_string(ReCompiled *p){ re_copy(p, iPrev, sz); } if( n==0 && m>0 ){ - re_append(p, RE_OP_FORK, -sz); + re_append(p, RE_OP_FORK, -(int)sz); } break; } @@ -656,8 +642,7 @@ static const char *re_subcompile_string(ReCompiled *p){ ** regular expression. Applications should invoke this routine once ** for every call to re_compile() to avoid memory leaks. */ -static void re_free(void *p){ - ReCompiled *pRe = (ReCompiled*)p; +static void re_free(ReCompiled *pRe){ if( pRe ){ sqlite3_free(pRe->aOp); sqlite3_free(pRe->aArg); @@ -665,13 +650,27 @@ static void re_free(void *p){ } } +/* +** Version of re_free() that accepts a pointer of type (void*). Required +** to satisfy sanitizers when the re_free() function is called via a +** function pointer. +*/ +static void re_free_voidptr(void *p){ + re_free((ReCompiled*)p); +} + /* ** Compile a textual regular expression in zIn[] into a compiled regular ** expression suitable for us by re_match() and return a pointer to the ** compiled regular expression in *ppRe. Return NULL on success or an ** error message if something goes wrong. */ -static const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){ +static const char *re_compile( + ReCompiled **ppRe, /* OUT: write compiled NFA here */ + const char *zIn, /* Input regular expression */ + int mxRe, /* Complexity limit */ + int noCase /* True for caseless comparisons */ +){ ReCompiled *pRe; const char *zErr; int i, j; @@ -683,9 +682,11 @@ static const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){ } memset(pRe, 0, sizeof(*pRe)); pRe->xNextChar = noCase ? re_next_char_nocase : re_next_char; + pRe->mxAlloc = mxRe; if( re_resize(pRe, 30) ){ + zErr = pRe->zErr; re_free(pRe); - return "out of memory"; + return zErr; } if( zIn[0]=='^' ){ zIn++; @@ -738,6 +739,14 @@ static const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){ return pRe->zErr; } +/* +** Compute a reasonable limit on the length of the REGEXP NFA. +*/ +static int re_maxlen(sqlite3_context *context){ + sqlite3 *db = sqlite3_context_db_handle(context); + return 75 + sqlite3_limit(db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH,-1)/2; +} + /* ** Implementation of the regexp() SQL function. This function implements ** the build-in REGEXP operator. The first argument to the function is the @@ -763,7 +772,8 @@ static void re_sql_func( if( pRe==0 ){ zPattern = (const char*)sqlite3_value_text(argv[0]); if( zPattern==0 ) return; - zErr = re_compile(&pRe, zPattern, sqlite3_user_data(context)!=0); + zErr = re_compile(&pRe, zPattern, re_maxlen(context), + sqlite3_user_data(context)!=0); if( zErr ){ re_free(pRe); sqlite3_result_error(context, zErr, -1); @@ -780,7 +790,7 @@ static void re_sql_func( sqlite3_result_int(context, re_match(pRe, zStr, -1)); } if( setAux ){ - sqlite3_set_auxdata(context, 0, pRe, (void(*)(void*))re_free); + sqlite3_set_auxdata(context, 0, pRe, re_free_voidptr); } } @@ -805,10 +815,32 @@ static void re_bytecode_func( int n; char *z; (void)argc; + static const char *ReOpName[] = { + "EOF", + "MATCH", + "ANY", + "ANYSTAR", + "FORK", + "GOTO", + "ACCEPT", + "CC_INC", + "CC_EXC", + "CC_VALUE", + "CC_RANGE", + "WORD", + "NOTWORD", + "DIGIT", + "NOTDIGIT", + "SPACE", + "NOTSPACE", + "BOUNDARY", + "ATSTART", + }; zPattern = (const char*)sqlite3_value_text(argv[0]); if( zPattern==0 ) return; - zErr = re_compile(&pRe, zPattern, sqlite3_user_data(context)!=0); + zErr = re_compile(&pRe, zPattern, re_maxlen(context), + sqlite3_user_data(context)!=0); if( zErr ){ re_free(pRe); sqlite3_result_error(context, zErr, -1); diff --git a/ext/misc/series.c b/ext/misc/series.c index 22e0f7edb..ffdb23c1a 100644 --- a/ext/misc/series.c +++ b/ext/misc/series.c @@ -30,19 +30,20 @@ ** SELECT * FROM generate_series(0,100,5); ** ** The query above returns integers from 0 through 100 counting by steps -** of 5. +** of 5. In other words, 0, 5, 10, 15, ..., 90, 95, 100. There are a total +** of 21 rows. ** ** SELECT * FROM generate_series(0,100); ** -** Integers from 0 through 100 with a step size of 1. +** Integers from 0 through 100 with a step size of 1. 101 rows. ** ** SELECT * FROM generate_series(20) LIMIT 10; ** -** Integers 20 through 29. +** Integers 20 through 29. 10 rows. ** ** SELECT * FROM generate_series(0,-100,-5); ** -** Integers 0 -5 -10 ... -100. +** Integers 0 -5 -10 ... -100. 21 rows. ** ** SELECT * FROM generate_series(0,-1); ** @@ -118,139 +119,88 @@ SQLITE_EXTENSION_INIT1 #include #ifndef SQLITE_OMIT_VIRTUALTABLE -/* -** Return that member of a generate_series(...) sequence whose 0-based -** index is ix. The 0th member is given by smBase. The sequence members -** progress per ix increment by smStep. -*/ -static sqlite3_int64 genSeqMember( - sqlite3_int64 smBase, - sqlite3_int64 smStep, - sqlite3_uint64 ix -){ - static const sqlite3_uint64 mxI64 = - ((sqlite3_uint64)0x7fffffff)<<32 | 0xffffffff; - if( ix>=mxI64 ){ - /* Get ix into signed i64 range. */ - ix -= mxI64; - /* With 2's complement ALU, this next can be 1 step, but is split into - * 2 for UBSAN's satisfaction (and hypothetical 1's complement ALUs.) */ - smBase += (mxI64/2) * smStep; - smBase += (mxI64 - mxI64/2) * smStep; - } - /* Under UBSAN (or on 1's complement machines), must do this last term - * in steps to avoid the dreaded (and harmless) signed multiply overflow. */ - if( ix>=2 ){ - sqlite3_int64 ix2 = (sqlite3_int64)ix/2; - smBase += ix2*smStep; - ix -= ix2; - } - return smBase + ((sqlite3_int64)ix)*smStep; -} +/* series_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result. +** +** iOBase, iOTerm, and iOStep are the original values of the +** start=, stop=, and step= constraints on the query. These are +** the values reported by the start, stop, and step columns of the +** virtual table. +** +** iBase, iTerm, iStep, and bDescp are the actual values used to generate +** the sequence. These might be different from the iOxxxx values. +** For example in +** +** SELECT value FROM generate_series(1,11,2) +** WHERE value BETWEEN 4 AND 8; +** +** The iOBase is 1, but the iBase is 5. iOTerm is 11 but iTerm is 7. +** Another example: +** +** SELECT value FROM generate_series(1,15,3) ORDER BY value DESC; +** +** The cursor initialization for the above query is: +** +** iOBase = 1 iBase = 13 +** iOTerm = 15 iTerm = 1 +** iOStep = 3 iStep = 3 bDesc = 1 +** +** The actual step size is unsigned so that can have a value of +** +9223372036854775808 which is needed for querys like this: +** +** SELECT value +** FROM generate_series(9223372036854775807, +** -9223372036854775808, +** -9223372036854775808) +** ORDER BY value ASC; +** +** The setup for the previous query will be: +** +** iOBase = 9223372036854775807 iBase = -1 +** iOTerm = -9223372036854775808 iTerm = 9223372036854775807 +** iOStep = -9223372036854775808 iStep = 9223372036854775808 bDesc = 0 +*/ typedef unsigned char u8; - -typedef struct SequenceSpec { - sqlite3_int64 iOBase; /* Original starting value ("start") */ - sqlite3_int64 iOTerm; /* Original terminal value ("stop") */ - sqlite3_int64 iBase; /* Starting value to actually use */ - sqlite3_int64 iTerm; /* Terminal value to actually use */ - sqlite3_int64 iStep; /* Increment ("step") */ - sqlite3_uint64 uSeqIndexMax; /* maximum sequence index (aka "n") */ - sqlite3_uint64 uSeqIndexNow; /* Current index during generation */ - sqlite3_int64 iValueNow; /* Current value during generation */ - u8 isNotEOF; /* Sequence generation not exhausted */ - u8 isReversing; /* Sequence is being reverse generated */ -} SequenceSpec; +typedef struct series_cursor series_cursor; +struct series_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_int64 iOBase; /* Original starting value ("start") */ + sqlite3_int64 iOTerm; /* Original terminal value ("stop") */ + sqlite3_int64 iOStep; /* Original step value */ + sqlite3_int64 iBase; /* Starting value to actually use */ + sqlite3_int64 iTerm; /* Terminal value to actually use */ + sqlite3_uint64 iStep; /* The step size */ + sqlite3_int64 iValue; /* Current value */ + u8 bDesc; /* iStep is really negative */ + u8 bDone; /* True if stepped past last element */ +}; /* -** Prepare a SequenceSpec for use in generating an integer series -** given initialized iBase, iTerm and iStep values. Sequence is -** initialized per given isReversing. Other members are computed. +** Computed the difference between two 64-bit signed integers using a +** convoluted computation designed to work around the silly restriction +** against signed integer overflow in C. */ -static void setupSequence( SequenceSpec *pss ){ - int bSameSigns; - pss->uSeqIndexMax = 0; - pss->isNotEOF = 0; - bSameSigns = (pss->iBase < 0)==(pss->iTerm < 0); - if( pss->iTerm < pss->iBase ){ - sqlite3_uint64 nuspan = 0; - if( bSameSigns ){ - nuspan = (sqlite3_uint64)(pss->iBase - pss->iTerm); - }else{ - /* Under UBSAN (or on 1's complement machines), must do this in steps. - * In this clause, iBase>=0 and iTerm<0 . */ - nuspan = 1; - nuspan += pss->iBase; - nuspan += -(pss->iTerm+1); - } - if( pss->iStep<0 ){ - pss->isNotEOF = 1; - if( nuspan==ULONG_MAX ){ - pss->uSeqIndexMax = ( pss->iStep>LLONG_MIN )? nuspan/-pss->iStep : 1; - }else if( pss->iStep>LLONG_MIN ){ - pss->uSeqIndexMax = nuspan/-pss->iStep; - } - } - }else if( pss->iTerm > pss->iBase ){ - sqlite3_uint64 puspan = 0; - if( bSameSigns ){ - puspan = (sqlite3_uint64)(pss->iTerm - pss->iBase); - }else{ - /* Under UBSAN (or on 1's complement machines), must do this in steps. - * In this clause, iTerm>=0 and iBase<0 . */ - puspan = 1; - puspan += pss->iTerm; - puspan += -(pss->iBase+1); - } - if( pss->iStep>0 ){ - pss->isNotEOF = 1; - pss->uSeqIndexMax = puspan/pss->iStep; - } - }else if( pss->iTerm == pss->iBase ){ - pss->isNotEOF = 1; - pss->uSeqIndexMax = 0; - } - pss->uSeqIndexNow = (pss->isReversing)? pss->uSeqIndexMax : 0; - pss->iValueNow = (pss->isReversing) - ? genSeqMember(pss->iBase, pss->iStep, pss->uSeqIndexMax) - : pss->iBase; -} +static sqlite3_uint64 span64(sqlite3_int64 a, sqlite3_int64 b){ + assert( a>=b ); + return (*(sqlite3_uint64*)&a) - (*(sqlite3_uint64*)&b); +} /* -** Progress sequence generator to yield next value, if any. -** Leave its state to either yield next value or be at EOF. -** Return whether there is a next value, or 0 at EOF. +** Add or substract an unsigned 64-bit integer from a signed 64-bit integer +** and return the new signed 64-bit integer. */ -static int progressSequence( SequenceSpec *pss ){ - if( !pss->isNotEOF ) return 0; - if( pss->isReversing ){ - if( pss->uSeqIndexNow > 0 ){ - pss->uSeqIndexNow--; - pss->iValueNow -= pss->iStep; - }else{ - pss->isNotEOF = 0; - } - }else{ - if( pss->uSeqIndexNow < pss->uSeqIndexMax ){ - pss->uSeqIndexNow++; - pss->iValueNow += pss->iStep; - }else{ - pss->isNotEOF = 0; - } - } - return pss->isNotEOF; +static sqlite3_int64 add64(sqlite3_int64 a, sqlite3_uint64 b){ + sqlite3_uint64 x = *(sqlite3_uint64*)&a; + x += b; + return *(sqlite3_int64*)&x; +} +static sqlite3_int64 sub64(sqlite3_int64 a, sqlite3_uint64 b){ + sqlite3_uint64 x = *(sqlite3_uint64*)&a; + x -= b; + return *(sqlite3_int64*)&x; } - -/* series_cursor is a subclass of sqlite3_vtab_cursor which will -** serve as the underlying representation of a cursor that scans -** over rows of the result -*/ -typedef struct series_cursor series_cursor; -struct series_cursor { - sqlite3_vtab_cursor base; /* Base class - must be first */ - SequenceSpec ss; /* (this) Derived class data */ -}; /* ** The seriesConnect() method is invoked to create a new @@ -332,7 +282,15 @@ static int seriesClose(sqlite3_vtab_cursor *cur){ */ static int seriesNext(sqlite3_vtab_cursor *cur){ series_cursor *pCur = (series_cursor*)cur; - progressSequence( & pCur->ss ); + if( pCur->iValue==pCur->iTerm ){ + pCur->bDone = 1; + }else if( pCur->bDesc ){ + pCur->iValue = sub64(pCur->iValue, pCur->iStep); + assert( pCur->iValue>=pCur->iTerm ); + }else{ + pCur->iValue = add64(pCur->iValue, pCur->iStep); + assert( pCur->iValue<=pCur->iTerm ); + } return SQLITE_OK; } @@ -348,19 +306,19 @@ static int seriesColumn( series_cursor *pCur = (series_cursor*)cur; sqlite3_int64 x = 0; switch( i ){ - case SERIES_COLUMN_START: x = pCur->ss.iOBase; break; - case SERIES_COLUMN_STOP: x = pCur->ss.iOTerm; break; - case SERIES_COLUMN_STEP: x = pCur->ss.iStep; break; - default: x = pCur->ss.iValueNow; break; + case SERIES_COLUMN_START: x = pCur->iOBase; break; + case SERIES_COLUMN_STOP: x = pCur->iOTerm; break; + case SERIES_COLUMN_STEP: x = pCur->iOStep; break; + default: x = pCur->iValue; break; } sqlite3_result_int64(ctx, x); return SQLITE_OK; } #ifndef LARGEST_UINT64 -#define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) -#define LARGEST_UINT64 (0xffffffff|(((sqlite3_uint64)0xffffffff)<<32)) -#define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) +#define LARGEST_INT64 ((sqlite3_int64)0x7fffffffffffffffLL) +#define LARGEST_UINT64 ((sqlite3_uint64)0xffffffffffffffffULL) +#define SMALLEST_INT64 ((sqlite3_int64)0x8000000000000000LL) #endif /* @@ -368,7 +326,7 @@ static int seriesColumn( */ static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ series_cursor *pCur = (series_cursor*)cur; - *pRowid = pCur->ss.iValueNow; + *pRowid = pCur->iValue; return SQLITE_OK; } @@ -378,7 +336,7 @@ static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ */ static int seriesEof(sqlite3_vtab_cursor *cur){ series_cursor *pCur = (series_cursor*)cur; - return !pCur->ss.isNotEOF; + return pCur->bDone; } /* True to cause run-time checking of the start=, stop=, and/or step= @@ -389,6 +347,59 @@ static int seriesEof(sqlite3_vtab_cursor *cur){ # define SQLITE_SERIES_CONSTRAINT_VERIFY 0 #endif +/* +** Return the number of steps between pCur->iBase and pCur->iTerm if +** the step width is pCur->iStep. +*/ +static sqlite3_uint64 seriesSteps(series_cursor *pCur){ + if( pCur->bDesc ){ + assert( pCur->iBase >= pCur->iTerm ); + return span64(pCur->iBase, pCur->iTerm)/pCur->iStep; + }else{ + assert( pCur->iBase <= pCur->iTerm ); + return span64(pCur->iTerm, pCur->iBase)/pCur->iStep; + } +} + +#if defined(SQLITE_ENABLE_MATH_FUNCTIONS) || defined(_WIN32) +/* +** Case 1 (the most common case): +** The standard math library is available so use ceil() and floor() from there. +*/ +static double seriesCeil(double r){ return ceil(r); } +static double seriesFloor(double r){ return floor(r); } +#elif defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) +/* +** Case 2 (2nd most common): Use GCC/Clang builtins +*/ +static double seriesCeil(double r){ return __builtin_ceil(r); } +static double seriesFloor(double r){ return __builtin_floor(r); } +#else +/* +** Case 3 (rarely happens): Use home-grown ceil() and floor() routines. +*/ +static double seriesCeil(double r){ + sqlite3_int64 x; + if( r!=r ) return r; + if( r<=(-4503599627370496.0) ) return r; + if( r>=(+4503599627370496.0) ) return r; + x = (sqlite3_int64)r; + if( r==(double)x ) return r; + if( r>(double)x ) x++; + return (double)x; +} +static double seriesFloor(double r){ + sqlite3_int64 x; + if( r!=r ) return r; + if( r<=(-4503599627370496.0) ) return r; + if( r>=(+4503599627370496.0) ) return r; + x = (sqlite3_int64)r; + if( r==(double)x ) return r; + if( r<(double)x ) x--; + return (double)x; +} +#endif + /* ** This method is called to "rewind" the series_cursor object back ** to the first row of output. This method is always called at least @@ -422,33 +433,42 @@ static int seriesFilter( int argc, sqlite3_value **argv ){ series_cursor *pCur = (series_cursor *)pVtabCursor; - int i = 0; - int returnNoRows = 0; - sqlite3_int64 iMin = SMALLEST_INT64; - sqlite3_int64 iMax = LARGEST_INT64; - sqlite3_int64 iLimit = 0; - sqlite3_int64 iOffset = 0; + int iArg = 0; /* Arguments used so far */ + int i; /* Loop counter */ + sqlite3_int64 iMin = SMALLEST_INT64; /* Smallest allowed output value */ + sqlite3_int64 iMax = LARGEST_INT64; /* Largest allowed output value */ + sqlite3_int64 iLimit = 0; /* if >0, the value of the LIMIT */ + sqlite3_int64 iOffset = 0; /* if >0, the value of the OFFSET */ (void)idxStrUnused; + + /* If any constraints have a NULL value, then return no rows. + ** See ticket https://sqlite.org/src/info/fac496b61722daf2 + */ + for(i=0; i
      )^ ** -** ^The sqlite3_version[] string constant contains the text of [SQLITE_VERSION] -** macro. ^The sqlite3_libversion() function returns a pointer to the -** to the sqlite3_version[] string constant. The sqlite3_libversion() +** ^The sqlite3_version[] string constant contains the text of the +** [SQLITE_VERSION] macro. ^The sqlite3_libversion() function returns a +** pointer to the sqlite3_version[] string constant. The sqlite3_libversion() ** function is provided for use in DLLs since DLL users usually do not have ** direct access to string constants within the DLL. ^The ** sqlite3_libversion_number() function returns an integer equal to @@ -370,7 +373,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** without having to use a lot of C code. ** ** ^The sqlite3_exec() interface runs zero or more UTF-8 encoded, -** semicolon-separate SQL statements passed into its 2nd argument, +** semicolon-separated SQL statements passed into its 2nd argument, ** in the context of the [database connection] passed in as its 1st ** argument. ^If the callback function of the 3rd argument to ** sqlite3_exec() is not NULL, then it is invoked for each result row @@ -403,7 +406,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** result row is NULL then the corresponding string pointer for the ** sqlite3_exec() callback is a NULL pointer. ^The 4th argument to the ** sqlite3_exec() callback is an array of pointers to strings where each -** entry represents the name of corresponding result column as obtained +** entry represents the name of a corresponding result column as obtained ** from [sqlite3_column_name()]. ** ** ^If the 2nd parameter to sqlite3_exec() is a NULL pointer, a pointer @@ -497,6 +500,9 @@ int sqlite3_exec( #define SQLITE_ERROR_MISSING_COLLSEQ (SQLITE_ERROR | (1<<8)) #define SQLITE_ERROR_RETRY (SQLITE_ERROR | (2<<8)) #define SQLITE_ERROR_SNAPSHOT (SQLITE_ERROR | (3<<8)) +#define SQLITE_ERROR_RESERVESIZE (SQLITE_ERROR | (4<<8)) +#define SQLITE_ERROR_KEY (SQLITE_ERROR | (5<<8)) +#define SQLITE_ERROR_UNABLE (SQLITE_ERROR | (6<<8)) #define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) #define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) #define SQLITE_IOERR_WRITE (SQLITE_IOERR | (3<<8)) @@ -531,6 +537,8 @@ int sqlite3_exec( #define SQLITE_IOERR_DATA (SQLITE_IOERR | (32<<8)) #define SQLITE_IOERR_CORRUPTFS (SQLITE_IOERR | (33<<8)) #define SQLITE_IOERR_IN_PAGE (SQLITE_IOERR | (34<<8)) +#define SQLITE_IOERR_BADKEY (SQLITE_IOERR | (35<<8)) +#define SQLITE_IOERR_CODEC (SQLITE_IOERR | (36<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_LOCKED_VTAB (SQLITE_LOCKED | (2<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) @@ -589,7 +597,7 @@ int sqlite3_exec( ** Note in particular that passing the SQLITE_OPEN_EXCLUSIVE flag into ** [sqlite3_open_v2()] does *not* cause the underlying database file ** to be opened using O_EXCL. Passing SQLITE_OPEN_EXCLUSIVE into -** [sqlite3_open_v2()] has historically be a no-op and might become an +** [sqlite3_open_v2()] has historically been a no-op and might become an ** error in future versions of SQLite. */ #define SQLITE_OPEN_READONLY 0x00000001 /* Ok for sqlite3_open_v2() */ @@ -683,7 +691,7 @@ int sqlite3_exec( ** SQLite uses one of these integer values as the second ** argument to calls it makes to the xLock() and xUnlock() methods ** of an [sqlite3_io_methods] object. These values are ordered from -** lest restrictive to most restrictive. +** least restrictive to most restrictive. ** ** The argument to xLock() is always SHARED or higher. The argument to ** xUnlock is either SHARED or NONE. @@ -924,7 +932,7 @@ struct sqlite3_io_methods { ** connection. See also [SQLITE_FCNTL_FILE_POINTER]. ** **
    • [[SQLITE_FCNTL_SYNC_OMITTED]] -** No longer in use. +** The SQLITE_FCNTL_SYNC_OMITTED file-control is no longer used. ** **
    • [[SQLITE_FCNTL_SYNC]] ** The [SQLITE_FCNTL_SYNC] opcode is generated internally by SQLite and @@ -999,7 +1007,7 @@ struct sqlite3_io_methods { ** **
    • [[SQLITE_FCNTL_VFSNAME]] ** ^The [SQLITE_FCNTL_VFSNAME] opcode can be used to obtain the names of -** all [VFSes] in the VFS stack. The names are of all VFS shims and the +** all [VFSes] in the VFS stack. The names of all VFS shims and the ** final bottom-level VFS are written into memory obtained from ** [sqlite3_malloc()] and the result is stored in the char* variable ** that the fourth parameter of [sqlite3_file_control()] points to. @@ -1013,7 +1021,7 @@ struct sqlite3_io_methods { ** ^The [SQLITE_FCNTL_VFS_POINTER] opcode finds a pointer to the top-level ** [VFSes] currently in use. ^(The argument X in ** sqlite3_file_control(db,SQLITE_FCNTL_VFS_POINTER,X) must be -** of type "[sqlite3_vfs] **". This opcodes will set *X +** of type "[sqlite3_vfs] **". This opcode will set *X ** to a pointer to the top-level VFS.)^ ** ^When there are multiple VFS shims in the stack, this opcode finds the ** upper-most shim only. @@ -1203,7 +1211,7 @@ struct sqlite3_io_methods { **
    • [[SQLITE_FCNTL_EXTERNAL_READER]] ** The EXPERIMENTAL [SQLITE_FCNTL_EXTERNAL_READER] opcode is used to detect ** whether or not there is a database client in another process with a wal-mode -** transaction open on the database or not. It is only available on unix.The +** transaction open on the database or not. It is only available on unix. The ** (void*) argument passed with this file-control should be a pointer to a ** value of type (int). The integer value is set to 1 if the database is a wal ** mode database and there exists at least one client in another process that @@ -1221,6 +1229,15 @@ struct sqlite3_io_methods { ** database is not a temp db, then the [SQLITE_FCNTL_RESET_CACHE] file-control ** purges the contents of the in-memory page cache. If there is an open ** transaction, or if the db is a temp-db, this opcode is a no-op, not an error. +** +**
    • [[SQLITE_FCNTL_FILESTAT]] +** The [SQLITE_FCNTL_FILESTAT] opcode returns low-level diagnostic information +** about the [sqlite3_file] objects used access the database and journal files +** for the given schema. The fourth parameter to [sqlite3_file_control()] +** should be an initialized [sqlite3_str] pointer. JSON text describing +** various aspects of the sqlite3_file object is appended to the sqlite3_str. +** The SQLITE_FCNTL_FILESTAT opcode is usually a no-op, unless compile-time +** options are used to enable it. **
    */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -1266,6 +1283,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_RESET_CACHE 42 #define SQLITE_FCNTL_NULL_IO 43 #define SQLITE_FCNTL_BLOCK_ON_CONNECT 44 +#define SQLITE_FCNTL_FILESTAT 45 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE @@ -1628,7 +1646,7 @@ struct sqlite3_vfs { ** SQLite interfaces so that an application usually does not need to ** invoke sqlite3_initialize() directly. For example, [sqlite3_open()] ** calls sqlite3_initialize() so the SQLite library will be automatically -** initialized when [sqlite3_open()] is called if it has not be initialized +** initialized when [sqlite3_open()] is called if it has not been initialized ** already. ^However, if SQLite is compiled with the [SQLITE_OMIT_AUTOINIT] ** compile-time option, then the automatic calls to sqlite3_initialize() ** are omitted and the application must call sqlite3_initialize() directly @@ -1885,21 +1903,21 @@ struct sqlite3_mem_methods { ** The [sqlite3_mem_methods] ** structure is filled with the currently defined memory allocation routines.)^ ** This option can be used to overload the default memory allocation -** routines with a wrapper that simulations memory allocation failure or +** routines with a wrapper that simulates memory allocation failure or ** tracks memory usage, for example. ** ** [[SQLITE_CONFIG_SMALL_MALLOC]]
    SQLITE_CONFIG_SMALL_MALLOC
    -**
    ^The SQLITE_CONFIG_SMALL_MALLOC option takes single argument of +**
    ^The SQLITE_CONFIG_SMALL_MALLOC option takes a single argument of ** type int, interpreted as a boolean, which if true provides a hint to ** SQLite that it should avoid large memory allocations if possible. ** SQLite will run faster if it is free to make large memory allocations, -** but some application might prefer to run slower in exchange for +** but some applications might prefer to run slower in exchange for ** guarantees about memory fragmentation that are possible if large ** allocations are avoided. This hint is normally off. **
    ** ** [[SQLITE_CONFIG_MEMSTATUS]]
    SQLITE_CONFIG_MEMSTATUS
    -**
    ^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int, +**
    ^The SQLITE_CONFIG_MEMSTATUS option takes a single argument of type int, ** interpreted as a boolean, which enables or disables the collection of ** memory allocation statistics. ^(When memory allocation statistics are ** disabled, the following SQLite interfaces become non-operational: @@ -1944,7 +1962,7 @@ struct sqlite3_mem_methods { ** ^If pMem is NULL and N is non-zero, then each database connection ** does an initial bulk allocation for page cache memory ** from [sqlite3_malloc()] sufficient for N cache lines if N is positive or -** of -1024*N bytes if N is negative, . ^If additional +** of -1024*N bytes if N is negative. ^If additional ** page cache memory is needed beyond what is provided by the initial ** allocation, then SQLite goes to [sqlite3_malloc()] separately for each ** additional cache line.
    @@ -1973,7 +1991,7 @@ struct sqlite3_mem_methods { **
    ^(The SQLITE_CONFIG_MUTEX option takes a single argument which is a ** pointer to an instance of the [sqlite3_mutex_methods] structure. ** The argument specifies alternative low-level mutex routines to be used -** in place the mutex routines built into SQLite.)^ ^SQLite makes a copy of +** in place of the mutex routines built into SQLite.)^ ^SQLite makes a copy of ** the content of the [sqlite3_mutex_methods] structure before the call to ** [sqlite3_config()] returns. ^If SQLite is compiled with ** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then @@ -2015,7 +2033,7 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_CONFIG_GETPCACHE2]]
    SQLITE_CONFIG_GETPCACHE2
    **
    ^(The SQLITE_CONFIG_GETPCACHE2 option takes a single argument which -** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies of +** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies off ** the current page cache implementation into that object.)^
    ** ** [[SQLITE_CONFIG_LOG]]
    SQLITE_CONFIG_LOG
    @@ -2032,7 +2050,7 @@ struct sqlite3_mem_methods { ** the logger function is a copy of the first parameter to the corresponding ** [sqlite3_log()] call and is intended to be a [result code] or an ** [extended result code]. ^The third parameter passed to the logger is -** log message after formatting via [sqlite3_snprintf()]. +** a log message after formatting via [sqlite3_snprintf()]. ** The SQLite logging interface is not reentrant; the logger function ** supplied by the application must not invoke any SQLite interface. ** In a multi-threaded application, the application-defined logger @@ -2223,7 +2241,7 @@ struct sqlite3_mem_methods { ** These constants are the available integer configuration options that ** can be passed as the second parameter to the [sqlite3_db_config()] interface. ** -** The [sqlite3_db_config()] interface is a var-args functions. It takes a +** The [sqlite3_db_config()] interface is a var-args function. It takes a ** variable number of parameters, though always at least two. The number of ** parameters passed into sqlite3_db_config() depends on which of these ** constants is given as the second parameter. This documentation page @@ -2335,17 +2353,20 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER]] **
    SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER
    -**
    ^This option is used to enable or disable the -** [fts3_tokenizer()] function which is part of the -** [FTS3] full-text search engine extension. -** There must be two additional arguments. -** The first argument is an integer which is 0 to disable fts3_tokenizer() or -** positive to enable fts3_tokenizer() or negative to leave the setting -** unchanged. -** The second parameter is a pointer to an integer into which -** is written 0 or 1 to indicate whether fts3_tokenizer is disabled or enabled -** following this call. The second parameter may be a NULL pointer, in -** which case the new setting is not reported back.
    +**
    ^This option is used to enable or disable using the +** [fts3_tokenizer()] function - part of the [FTS3] full-text search engine +** extension - without using bound parameters as the parameters. Doing so +** is disabled by default. There must be two additional arguments. The first +** argument is an integer. If it is passed 0, then using fts3_tokenizer() +** without bound parameters is disabled. If it is passed a positive value, +** then calling fts3_tokenizer without bound parameters is enabled. If it +** is passed a negative value, this setting is not modified - this can be +** used to query for the current setting. The second parameter is a pointer +** to an integer into which is written 0 or 1 to indicate the current value +** of this setting (after it is modified, if applicable). The second +** parameter may be a NULL pointer, in which case the value of the setting +** is not reported back. Refer to [FTS3] documentation for further details. +**
    ** ** [[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION]] **
    SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION
    @@ -2357,8 +2378,8 @@ struct sqlite3_mem_methods { ** When the first argument to this interface is 1, then only the C-API is ** enabled and the SQL function remains disabled. If the first argument to ** this interface is 0, then both the C-API and the SQL function are disabled. -** If the first argument is -1, then no changes are made to state of either the -** C-API or the SQL function. +** If the first argument is -1, then no changes are made to the state of either +** the C-API or the SQL function. ** The second parameter is a pointer to an integer into which ** is written 0 or 1 to indicate whether [sqlite3_load_extension()] interface ** is disabled or enabled following this call. The second parameter may @@ -2476,7 +2497,7 @@ struct sqlite3_mem_methods { ** [[SQLITE_DBCONFIG_LEGACY_ALTER_TABLE]] **
    SQLITE_DBCONFIG_LEGACY_ALTER_TABLE
    **
    The SQLITE_DBCONFIG_LEGACY_ALTER_TABLE option activates or deactivates -** the legacy behavior of the [ALTER TABLE RENAME] command such it +** the legacy behavior of the [ALTER TABLE RENAME] command such that it ** behaves as it did prior to [version 3.24.0] (2018-06-04). See the ** "Compatibility Notice" on the [ALTER TABLE RENAME documentation] for ** additional information. This feature can also be turned on and off @@ -2525,7 +2546,7 @@ struct sqlite3_mem_methods { **
    SQLITE_DBCONFIG_LEGACY_FILE_FORMAT
    **
    The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates ** the legacy file format flag. When activated, this flag causes all newly -** created database file to have a schema format version number (the 4-byte +** created database files to have a schema format version number (the 4-byte ** integer found at offset 44 into the database header) of 1. This in turn ** means that the resulting database file will be readable and writable by ** any SQLite version back to 3.0.0 ([dateof:3.0.0]). Without this setting, @@ -2552,7 +2573,7 @@ struct sqlite3_mem_methods { ** the database handle both when the SQL statement is prepared and when it ** is stepped. The flag is set (collection of statistics is enabled) ** by default.

    This option takes two arguments: an integer and a pointer to -** an integer.. The first argument is 1, 0, or -1 to enable, disable, or +** an integer. The first argument is 1, 0, or -1 to enable, disable, or ** leave unchanged the statement scanstatus option. If the second argument ** is not NULL, then the value of the statement scanstatus setting after ** processing the first argument is written into the integer that the second @@ -2595,8 +2616,8 @@ struct sqlite3_mem_methods { **

    The SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE option enables or disables the ** ability of the [ATTACH DATABASE] SQL command to open a database for writing. ** This capability is enabled by default. Applications can disable or -** reenable this capability using the current DBCONFIG option. If the -** the this capability is disabled, the [ATTACH] command will still work, +** reenable this capability using the current DBCONFIG option. If +** this capability is disabled, the [ATTACH] command will still work, ** but the database will be opened read-only. If this option is disabled, ** then the ability to create a new database using [ATTACH] is also disabled, ** regardless of the value of the [SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE] @@ -2630,7 +2651,7 @@ struct sqlite3_mem_methods { ** **

    Most of the SQLITE_DBCONFIG options take two arguments, so that the ** overall call to [sqlite3_db_config()] has a total of four parameters. -** The first argument (the third parameter to sqlite3_db_config()) is a integer. +** The first argument (the third parameter to sqlite3_db_config()) is an integer. ** The second argument is a pointer to an integer. If the first argument is 1, ** then the option becomes enabled. If the first integer argument is 0, then the ** option is disabled. If the first argument is -1, then the option setting @@ -2920,7 +2941,7 @@ int sqlite3_is_interrupted(sqlite3*); ** ^These routines return 0 if the statement is incomplete. ^If a ** memory allocation fails, then SQLITE_NOMEM is returned. ** -** ^These routines do not parse the SQL statements thus +** ^These routines do not parse the SQL statements and thus ** will not detect syntactically incorrect SQL. ** ** ^(If SQLite has not been initialized using [sqlite3_initialize()] prior @@ -3037,7 +3058,7 @@ int sqlite3_busy_timeout(sqlite3*, int ms); ** indefinitely if possible. The results of passing any other negative value ** are undefined. ** -** Internally, each SQLite database handle store two timeout values - the +** Internally, each SQLite database handle stores two timeout values - the ** busy-timeout (used for rollback mode databases, or if the VFS does not ** support blocking locks) and the setlk-timeout (used for blocking locks ** on wal-mode databases). The sqlite3_busy_timeout() method sets both @@ -3067,7 +3088,7 @@ int sqlite3_setlk_timeout(sqlite3*, int ms, int flags); ** This is a legacy interface that is preserved for backwards compatibility. ** Use of this interface is not recommended. ** -** Definition: A result table is memory data structure created by the +** Definition: A result table is a memory data structure created by the ** [sqlite3_get_table()] interface. A result table records the ** complete query results from one or more queries. ** @@ -3210,7 +3231,7 @@ char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** ^Calling sqlite3_free() with a pointer previously returned ** by sqlite3_malloc() or sqlite3_realloc() releases that memory so ** that it might be reused. ^The sqlite3_free() routine is -** a no-op if is called with a NULL pointer. Passing a NULL pointer +** a no-op if it is called with a NULL pointer. Passing a NULL pointer ** to sqlite3_free() is harmless. After being freed, memory ** should neither be read nor written. Even reading previously freed ** memory might result in a segmentation fault or other severe error. @@ -3228,13 +3249,13 @@ char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** sqlite3_free(X). ** ^sqlite3_realloc(X,N) returns a pointer to a memory allocation ** of at least N bytes in size or NULL if insufficient memory is available. -** ^If M is the size of the prior allocation, then min(N,M) bytes -** of the prior allocation are copied into the beginning of buffer returned +** ^If M is the size of the prior allocation, then min(N,M) bytes of the +** prior allocation are copied into the beginning of the buffer returned ** by sqlite3_realloc(X,N) and the prior allocation is freed. ** ^If sqlite3_realloc(X,N) returns NULL and N is positive, then the ** prior allocation is not freed. ** -** ^The sqlite3_realloc64(X,N) interfaces works the same as +** ^The sqlite3_realloc64(X,N) interface works the same as ** sqlite3_realloc(X,N) except that N is a 64-bit unsigned integer instead ** of a 32-bit signed integer. ** @@ -3284,7 +3305,7 @@ sqlite3_uint64 sqlite3_msize(void*); ** was last reset. ^The values returned by [sqlite3_memory_used()] and ** [sqlite3_memory_highwater()] include any overhead ** added by SQLite in its implementation of [sqlite3_malloc()], -** but not overhead added by the any underlying system library +** but not overhead added by any underlying system library ** routines that [sqlite3_malloc()] may call. ** ** ^The memory high-water mark is reset to the current value of @@ -3736,7 +3757,7 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** there is no harm in trying.) ** ** ^(

    [SQLITE_OPEN_SHAREDCACHE]
    -**
    The database is opened [shared cache] enabled, overriding +**
    The database is opened with [shared cache] enabled, overriding ** the default shared cache setting provided by ** [sqlite3_enable_shared_cache()].)^ ** The [use of shared cache mode is discouraged] and hence shared cache @@ -3744,7 +3765,7 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** this option is a no-op. ** ** ^(
    [SQLITE_OPEN_PRIVATECACHE]
    -**
    The database is opened [shared cache] disabled, overriding +**
    The database is opened with [shared cache] disabled, overriding ** the default shared cache setting provided by ** [sqlite3_enable_shared_cache()].)^ ** @@ -4162,7 +4183,7 @@ void sqlite3_free_filename(sqlite3_filename); ** subsequent calls to other SQLite interface functions.)^ ** ** ^The sqlite3_errstr(E) interface returns the English-language text -** that describes the [result code] E, as UTF-8, or NULL if E is not an +** that describes the [result code] E, as UTF-8, or NULL if E is not a ** result code for which a text error message is available. ** ^(Memory to hold the error message string is managed internally ** and must not be freed by the application)^. @@ -4170,7 +4191,7 @@ void sqlite3_free_filename(sqlite3_filename); ** ^If the most recent error references a specific token in the input ** SQL, the sqlite3_error_offset() interface returns the byte offset ** of the start of that token. ^The byte offset returned by -** sqlite3_error_offset() assumes that the input SQL is UTF8. +** sqlite3_error_offset() assumes that the input SQL is UTF-8. ** ^If the most recent error does not reference a specific token in the input ** SQL, then the sqlite3_error_offset() function returns -1. ** @@ -4195,6 +4216,34 @@ const void *sqlite3_errmsg16(sqlite3*); const char *sqlite3_errstr(int); int sqlite3_error_offset(sqlite3 *db); +/* +** CAPI3REF: Set Error Codes And Message +** METHOD: sqlite3 +** +** Set the error code of the database handle passed as the first argument +** to errcode, and the error message to a copy of nul-terminated string +** zErrMsg. If zErrMsg is passed NULL, then the error message is set to +** the default message associated with the supplied error code. Subsequent +** calls to [sqlite3_errcode()] and [sqlite3_errmsg()] and similar will +** return the values set by this routine in place of what was previously +** set by SQLite itself. +** +** This function returns SQLITE_OK if the error code and error message are +** successfully set, SQLITE_NOMEM if an OOM occurs, and SQLITE_MISUSE if +** the database handle is NULL or invalid. +** +** The error code and message set by this routine remains in effect until +** they are changed, either by another call to this routine or until they are +** changed to by SQLite itself to reflect the result of some subsquent +** API call. +** +** This function is intended for use by SQLite extensions or wrappers. The +** idea is that an extension or wrapper can use this routine to set error +** messages and error codes and thus behave more like a core SQLite +** feature from the point of view of an application. +*/ +int sqlite3_set_errmsg(sqlite3 *db, int errcode, const char *zErrMsg); + /* ** CAPI3REF: Prepared Statement Object ** KEYWORDS: {prepared statement} {prepared statements} @@ -4269,8 +4318,8 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** ** These constants define various performance limits ** that can be lowered at run-time using [sqlite3_limit()]. -** The synopsis of the meanings of the various limits is shown below. -** Additional information is available at [limits | Limits in SQLite]. +** A concise description of these limits follows, and additional information +** is available at [limits | Limits in SQLite]. ** **
    ** [[SQLITE_LIMIT_LENGTH]] ^(
    SQLITE_LIMIT_LENGTH
    @@ -4335,7 +4384,7 @@ int sqlite3_limit(sqlite3*, int id, int newVal); /* ** CAPI3REF: Prepare Flags ** -** These constants define various flags that can be passed into +** These constants define various flags that can be passed into the ** "prepFlags" parameter of the [sqlite3_prepare_v3()] and ** [sqlite3_prepare16_v3()] interfaces. ** @@ -4422,7 +4471,7 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** there is a small performance advantage to passing an nByte parameter that ** is the number of bytes in the input string including ** the nul-terminator. -** Note that nByte measure the length of the input in bytes, not +** Note that nByte measures the length of the input in bytes, not ** characters, even for the UTF-16 interfaces. ** ** ^If pzTail is not NULL then *pzTail is made to point to the first byte @@ -4556,7 +4605,7 @@ int sqlite3_prepare16_v3( ** ** ^The sqlite3_expanded_sql() interface returns NULL if insufficient memory ** is available to hold the result, or if the result would exceed the -** the maximum string length determined by the [SQLITE_LIMIT_LENGTH]. +** maximum string length determined by the [SQLITE_LIMIT_LENGTH]. ** ** ^The [SQLITE_TRACE_SIZE_LIMIT] compile-time option limits the size of ** bound parameter expansions. ^The [SQLITE_OMIT_TRACE] compile-time @@ -4744,7 +4793,7 @@ typedef struct sqlite3_value sqlite3_value; ** ** The context in which an SQL function executes is stored in an ** sqlite3_context object. ^A pointer to an sqlite3_context object -** is always first parameter to [application-defined SQL functions]. +** is always the first parameter to [application-defined SQL functions]. ** The application-defined SQL function implementation will pass this ** pointer through into calls to [sqlite3_result_int | sqlite3_result()], ** [sqlite3_aggregate_context()], [sqlite3_user_data()], @@ -4868,9 +4917,11 @@ typedef struct sqlite3_context sqlite3_context; ** associated with the pointer P of type T. ^D is either a NULL pointer or ** a pointer to a destructor function for P. ^SQLite will invoke the ** destructor D with a single argument of P when it is finished using -** P. The T parameter should be a static string, preferably a string -** literal. The sqlite3_bind_pointer() routine is part of the -** [pointer passing interface] added for SQLite 3.20.0. +** P, even if the call to sqlite3_bind_pointer() fails. Due to a +** historical design quirk, results are undefined if D is +** SQLITE_TRANSIENT. The T parameter should be a static string, +** preferably a string literal. The sqlite3_bind_pointer() routine is +** part of the [pointer passing interface] added for SQLite 3.20.0. ** ** ^If any of the sqlite3_bind_*() routines are called with a NULL pointer ** for the [prepared statement] or with a prepared statement for which @@ -5481,7 +5532,7 @@ int sqlite3_column_type(sqlite3_stmt*, int iCol); ** ** ^The sqlite3_finalize() function is called to delete a [prepared statement]. ** ^If the most recent evaluation of the statement encountered no errors -** or if the statement is never been evaluated, then sqlite3_finalize() returns +** or if the statement has never been evaluated, then sqlite3_finalize() returns ** SQLITE_OK. ^If the most recent evaluation of statement S failed, then ** sqlite3_finalize(S) returns the appropriate [error code] or ** [extended error code]. @@ -5713,7 +5764,7 @@ int sqlite3_create_window_function( /* ** CAPI3REF: Text Encodings ** -** These constant define integer codes that represent the various +** These constants define integer codes that represent the various ** text encodings supported by SQLite. */ #define SQLITE_UTF8 1 /* IMP: R-37514-35566 */ @@ -5805,7 +5856,7 @@ int sqlite3_create_window_function( ** result. ** Every function that invokes [sqlite3_result_subtype()] should have this ** property. If it does not, then the call to [sqlite3_result_subtype()] -** might become a no-op if the function is used as term in an +** might become a no-op if the function is used as a term in an ** [expression index]. On the other hand, SQL functions that never invoke ** [sqlite3_result_subtype()] should avoid setting this property, as the ** purpose of this property is to disable certain optimizations that are @@ -5932,7 +5983,7 @@ SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), ** sqlite3_value_nochange(X) interface returns true if and only if ** the column corresponding to X is unchanged by the UPDATE operation ** that the xUpdate method call was invoked to implement and if -** and the prior [xColumn] method call that was invoked to extracted +** the prior [xColumn] method call that was invoked to extract ** the value for that column returned without setting a result (probably ** because it queried [sqlite3_vtab_nochange()] and found that the column ** was unchanging). ^Within an [xUpdate] method, any value for which @@ -6205,6 +6256,7 @@ void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); ** or a NULL pointer if there were no prior calls to ** sqlite3_set_clientdata() with the same values of D and N. ** Names are compared using strcmp() and are thus case sensitive. +** It returns 0 on success and SQLITE_NOMEM on allocation failure. ** ** If P and X are both non-NULL, then the destructor X is invoked with ** argument P on the first of the following occurrences: @@ -6240,7 +6292,7 @@ void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); ** languages or in other circumstances where it might be possible for an ** attacker to invoke them. Any agent that can invoke these interfaces ** can probably also take control of the process. -** +** ** Database connection client data is only available for SQLite ** version 3.44.0 ([dateof:3.44.0]) and later. ** @@ -8881,9 +8933,18 @@ int sqlite3_status64( ** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a ** non-zero [error code] on failure. ** +** ^The sqlite3_db_status64(D,O,C,H,R) routine works exactly the same +** way as sqlite3_db_status(D,O,C,H,R) routine except that the C and H +** parameters are pointer to 64-bit integers (type: sqlite3_int64) instead +** of pointers to 32-bit integers, which allows larger status values +** to be returned. If a status value exceeds 2,147,483,647 then +** sqlite3_db_status() will truncate the value whereas sqlite3_db_status64() +** will return the full value. +** ** See also: [sqlite3_status()] and [sqlite3_stmt_status()]. */ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); +int sqlite3_db_status64(sqlite3*,int,sqlite3_int64*,sqlite3_int64*,int); /* ** CAPI3REF: Status Parameters for database connections @@ -8980,6 +9041,10 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** If an IO or other error occurs while writing a page to disk, the effect ** on subsequent SQLITE_DBSTATUS_CACHE_WRITE requests is undefined.)^ ^The ** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0. +**

    +** ^(There is overlap between the quantities measured by this parameter +** (SQLITE_DBSTATUS_CACHE_WRITE) and SQLITE_DBSTATUS_TEMPBUF_SPILL. +** Resetting one will reduce the other.)^ **

    ** ** [[SQLITE_DBSTATUS_CACHE_SPILL]] ^(
    SQLITE_DBSTATUS_CACHE_SPILL
    @@ -8995,6 +9060,18 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); **
    This parameter returns zero for the current value if and only if ** all foreign key constraints (deferred or immediate) have been ** resolved.)^ ^The highwater mark is always 0. +** +** [[SQLITE_DBSTATUS_TEMPBUF_SPILL] ^(
    SQLITE_DBSTATUS_TEMPBUF_SPILL
    +**
    ^(This parameter returns the number of bytes written to temporary +** files on disk that could have been kept in memory had sufficient memory +** been available. This value includes writes to intermediate tables that +** are part of complex queries, external sorts that spill to disk, and +** writes to TEMP tables.)^ +** ^The highwater mark is always 0. +**

    +** ^(There is overlap between the quantities measured by this parameter +** (SQLITE_DBSTATUS_TEMPBUF_SPILL) and SQLITE_DBSTATUS_CACHE_WRITE. +** Resetting one will reduce the other.)^ **

    ** */ @@ -9011,7 +9088,8 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); #define SQLITE_DBSTATUS_DEFERRED_FKS 10 #define SQLITE_DBSTATUS_CACHE_USED_SHARED 11 #define SQLITE_DBSTATUS_CACHE_SPILL 12 -#define SQLITE_DBSTATUS_MAX 12 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_TEMPBUF_SPILL 13 +#define SQLITE_DBSTATUS_MAX 13 /* Largest defined DBSTATUS */ /* @@ -9776,7 +9854,7 @@ void sqlite3_log(int iErrCode, const char *zFormat, ...); ** is the number of pages currently in the write-ahead log file, ** including those that were just committed. ** -** The callback function should normally return [SQLITE_OK]. ^If an error +** ^The callback function should normally return [SQLITE_OK]. ^If an error ** code is returned, that error will propagate back up through the ** SQLite code base to cause the statement that provoked the callback ** to report an error, though the commit will have still occurred. If the @@ -9784,13 +9862,26 @@ void sqlite3_log(int iErrCode, const char *zFormat, ...); ** that does not correspond to any valid SQLite error code, the results ** are undefined. ** -** A single database handle may have at most a single write-ahead log callback -** registered at one time. ^Calling [sqlite3_wal_hook()] replaces any -** previously registered write-ahead log callback. ^The return value is -** a copy of the third parameter from the previous call, if any, or 0. -** ^Note that the [sqlite3_wal_autocheckpoint()] interface and the -** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and will -** overwrite any prior [sqlite3_wal_hook()] settings. +** ^A single database handle may have at most a single write-ahead log +** callback registered at one time. ^Calling [sqlite3_wal_hook()] +** replaces the default behavior or previously registered write-ahead +** log callback. +** +** ^The return value is a copy of the third parameter from the +** previous call, if any, or 0. +** +** ^The [sqlite3_wal_autocheckpoint()] interface and the +** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and +** will overwrite any prior [sqlite3_wal_hook()] settings. +** +** ^If a write-ahead log callback is set using this function then +** [sqlite3_wal_checkpoint_v2()] or [PRAGMA wal_checkpoint] +** should be invoked periodically to keep the write-ahead log file +** from growing without bound. +** +** ^Passing a NULL pointer for the callback disables automatic +** checkpointing entirely. To re-enable the default behavior, call +** sqlite3_wal_autocheckpoint(db,1000) or use [PRAGMA wal_checkpoint]. */ void *sqlite3_wal_hook( sqlite3*, @@ -9807,7 +9898,7 @@ void *sqlite3_wal_hook( ** to automatically [checkpoint] ** after committing a transaction if there are N or ** more frames in the [write-ahead log] file. ^Passing zero or -** a negative value as the nFrame parameter disables automatic +** a negative value as the N parameter disables automatic ** checkpoints entirely. ** ** ^The callback registered by this function replaces any existing callback @@ -9823,9 +9914,10 @@ void *sqlite3_wal_hook( ** ** ^Every new [database connection] defaults to having the auto-checkpoint ** enabled with a threshold of 1000 or [SQLITE_DEFAULT_WAL_AUTOCHECKPOINT] -** pages. The use of this interface -** is only necessary if the default setting is found to be suboptimal -** for a particular application. +** pages. +** +** ^The use of this interface is only necessary if the default setting +** is found to be suboptimal for a particular application. */ int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); @@ -9890,6 +9982,11 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** ^This mode works the same way as SQLITE_CHECKPOINT_RESTART with the ** addition that it also truncates the log file to zero bytes just prior ** to a successful return. +** +**
    SQLITE_CHECKPOINT_NOOP
    +** ^This mode always checkpoints zero frames. The only reason to invoke +** a NOOP checkpoint is to access the values returned by +** sqlite3_wal_checkpoint_v2() via output parameters *pnLog and *pnCkpt. ** ** ** ^If pnLog is not NULL, then *pnLog is set to the total number of frames in @@ -9960,6 +10057,7 @@ int sqlite3_wal_checkpoint_v2( ** See the [sqlite3_wal_checkpoint_v2()] documentation for details on the ** meaning of each of these checkpoint modes. */ +#define SQLITE_CHECKPOINT_NOOP -1 /* Do no work at all */ #define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ #define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ #define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for readers */ @@ -10787,7 +10885,7 @@ typedef struct sqlite3_snapshot { ** The [sqlite3_snapshot_get()] interface is only available when the ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ -SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( +int sqlite3_snapshot_get( sqlite3 *db, const char *zSchema, sqlite3_snapshot **ppSnapshot @@ -10836,7 +10934,7 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( ** The [sqlite3_snapshot_open()] interface is only available when the ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ -SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( +int sqlite3_snapshot_open( sqlite3 *db, const char *zSchema, sqlite3_snapshot *pSnapshot @@ -10853,7 +10951,7 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( ** The [sqlite3_snapshot_free()] interface is only available when the ** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ -SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); +void sqlite3_snapshot_free(sqlite3_snapshot*); /* ** CAPI3REF: Compare the ages of two snapshot handles. @@ -10880,7 +10978,7 @@ SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); ** This interface is only available if SQLite is compiled with the ** [SQLITE_ENABLE_SNAPSHOT] option. */ -SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( +int sqlite3_snapshot_cmp( sqlite3_snapshot *p1, sqlite3_snapshot *p2 ); @@ -10908,7 +11006,7 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( ** This interface is only available if SQLite is compiled with the ** [SQLITE_ENABLE_SNAPSHOT] option. */ -SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); +int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); /* ** CAPI3REF: Serialize a database @@ -10982,12 +11080,13 @@ unsigned char *sqlite3_serialize( ** ** The sqlite3_deserialize(D,S,P,N,M,F) interface causes the ** [database connection] D to disconnect from database S and then -** reopen S as an in-memory database based on the serialization contained -** in P. The serialized database P is N bytes in size. M is the size of -** the buffer P, which might be larger than N. If M is larger than N, and -** the SQLITE_DESERIALIZE_READONLY bit is not set in F, then SQLite is -** permitted to add content to the in-memory database as long as the total -** size does not exceed M bytes. +** reopen S as an in-memory database based on the serialization +** contained in P. If S is a NULL pointer, the main database is +** used. The serialized database P is N bytes in size. M is the size +** of the buffer P, which might be larger than N. If M is larger than +** N, and the SQLITE_DESERIALIZE_READONLY bit is not set in F, then +** SQLite is permitted to add content to the in-memory database as +** long as the total size does not exceed M bytes. ** ** If the SQLITE_DESERIALIZE_FREEONCLOSE bit is set in F, then SQLite will ** invoke sqlite3_free() on the serialization buffer when the database @@ -11054,6 +11153,54 @@ int sqlite3_deserialize( #define SQLITE_DESERIALIZE_RESIZEABLE 2 /* Resize using sqlite3_realloc64() */ #define SQLITE_DESERIALIZE_READONLY 4 /* Database is read-only */ +/* +** CAPI3REF: Bind array values to the CARRAY table-valued function +** +** The sqlite3_carray_bind(S,I,P,N,F,X) interface binds an array value to +** one of the first argument of the [carray() table-valued function]. The +** S parameter is a pointer to the [prepared statement] that uses the carray() +** functions. I is the parameter index to be bound. P is a pointer to the +** array to be bound, and N is the number of eements in the array. The +** F argument is one of constants [SQLITE_CARRAY_INT32], [SQLITE_CARRAY_INT64], +** [SQLITE_CARRAY_DOUBLE], [SQLITE_CARRAY_TEXT], or [SQLITE_CARRAY_BLOB] to +** indicate the datatype of the array being bound. The X argument is not a +** NULL pointer, then SQLite will invoke the function X on the P parameter +** after it has finished using P, even if the call to +** sqlite3_carray_bind() fails. The special-case finalizer +** SQLITE_TRANSIENT has no effect here. +*/ +int sqlite3_carray_bind( + sqlite3_stmt *pStmt, /* Statement to be bound */ + int i, /* Parameter index */ + void *aData, /* Pointer to array data */ + int nData, /* Number of data elements */ + int mFlags, /* CARRAY flags */ + void (*xDel)(void*) /* Destructor for aData */ +); + +/* +** CAPI3REF: Datatypes for the CARRAY table-valued function +** +** The fifth argument to the [sqlite3_carray_bind()] interface musts be +** one of the following constants, to specify the datatype of the array +** that is being bound into the [carray table-valued function]. +*/ +#define SQLITE_CARRAY_INT32 0 /* Data is 32-bit signed integers */ +#define SQLITE_CARRAY_INT64 1 /* Data is 64-bit signed integers */ +#define SQLITE_CARRAY_DOUBLE 2 /* Data is doubles */ +#define SQLITE_CARRAY_TEXT 3 /* Data is char* */ +#define SQLITE_CARRAY_BLOB 4 /* Data is struct iovec */ + +/* +** Versions of the above #defines that omit the initial SQLITE_, for +** legacy compatibility. +*/ +#define CARRAY_INT32 0 /* Data is 32-bit signed integers */ +#define CARRAY_INT64 1 /* Data is 64-bit signed integers */ +#define CARRAY_DOUBLE 2 /* Data is doubles */ +#define CARRAY_TEXT 3 /* Data is char* */ +#define CARRAY_BLOB 4 /* Data is struct iovec */ + /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index cf775dfbd..5258faaed 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -368,6 +368,10 @@ struct sqlite3_api_routines { int (*set_clientdata)(sqlite3*, const char*, void*, void(*)(void*)); /* Version 3.50.0 and later */ int (*setlk_timeout)(sqlite3*,int,int); + /* Version 3.51.0 and later */ + int (*set_errmsg)(sqlite3*,int,const char*); + int (*db_status64)(sqlite3*,int,sqlite3_int64*,sqlite3_int64*,int); + }; /* @@ -703,6 +707,9 @@ typedef int (*sqlite3_loadext_entry)( #define sqlite3_set_clientdata sqlite3_api->set_clientdata /* Version 3.50.0 and later */ #define sqlite3_setlk_timeout sqlite3_api->setlk_timeout +/* Version 3.51.0 and later */ +#define sqlite3_set_errmsg sqlite3_api->set_errmsg +#define sqlite3_db_status64 sqlite3_api->db_status64 #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 76aef3d12..523bcfb3b 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -765,7 +765,7 @@ ** ourselves. */ #ifndef offsetof -#define offsetof(STRUCTURE,FIELD) ((size_t)((char*)&((STRUCTURE*)0)->FIELD)) +# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) #endif /* @@ -1153,6 +1153,8 @@ extern u32 sqlite3TreeTrace; ** 0x00020000 Transform DISTINCT into GROUP BY ** 0x00040000 SELECT tree dump after all code has been generated ** 0x00080000 NOT NULL strength reduction +** 0x00100000 Pointers are all shown as zero +** 0x00200000 EXISTS-to-JOIN optimization */ /* @@ -1197,6 +1199,7 @@ extern u32 sqlite3WhereTrace; ** 0x00020000 Show WHERE terms returned from whereScanNext() ** 0x00040000 Solver overview messages ** 0x00080000 Star-query heuristic +** 0x00100000 Pointers are all shown as zero */ @@ -1269,7 +1272,7 @@ struct BusyHandler { ** pointer will work here as long as it is distinct from SQLITE_STATIC ** and SQLITE_TRANSIENT. */ -#define SQLITE_DYNAMIC ((sqlite3_destructor_type)sqlite3OomClear) +#define SQLITE_DYNAMIC ((sqlite3_destructor_type)sqlite3RowSetClear) /* ** When SQLITE_OMIT_WSD is defined, it means that the target platform does @@ -1698,7 +1701,7 @@ struct sqlite3 { u8 iDb; /* Which db file is being initialized */ u8 busy; /* TRUE if currently initializing */ unsigned orphanTrigger : 1; /* Last statement is orphaned TEMP trigger */ - unsigned imposterTable : 1; /* Building an imposter table */ + unsigned imposterTable : 2; /* Building an imposter table */ unsigned reopenMemdb : 1; /* ATTACH is really a reopen using MemDB */ const char **azInit; /* "type", "name", and "tbl_name" columns */ } init; @@ -1781,6 +1784,7 @@ struct sqlite3 { i64 nDeferredImmCons; /* Net deferred immediate constraints */ int *pnBytesFreed; /* If not NULL, increment this in DbFree() */ DbClientData *pDbData; /* sqlite3_set_clientdata() content */ + u64 nSpill; /* TEMP content spilled to disk */ #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY /* The following variables are all protected by the STATIC_MAIN ** mutex, not by sqlite3.mutex. They are used by code in notify.c. @@ -1924,6 +1928,7 @@ struct sqlite3 { #define SQLITE_OnePass 0x08000000 /* Single-pass DELETE and UPDATE */ #define SQLITE_OrderBySubq 0x10000000 /* ORDER BY in subquery helps outer */ #define SQLITE_StarQuery 0x20000000 /* Heurists for star queries */ +#define SQLITE_ExistsToJoin 0x40000000 /* The EXISTS-to-JOIN optimization */ #define SQLITE_AllOpts 0xffffffff /* All optimizations */ /* @@ -2162,7 +2167,7 @@ struct FuncDestructor { #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ {nArg, SQLITE_FUNC_BUILTIN|\ SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - pArg, 0, xFunc, 0, 0, 0, #zName, } + pArg, 0, xFunc, 0, 0, 0, #zName, {0} } #define LIKEFUNC(zName, nArg, arg, flags) \ {nArg, SQLITE_FUNC_BUILTIN|SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \ (void *)arg, 0, likeFunc, 0, 0, 0, #zName, {0} } @@ -2490,6 +2495,7 @@ struct Table { #define TF_Ephemeral 0x00004000 /* An ephemeral table */ #define TF_Eponymous 0x00008000 /* An eponymous virtual table */ #define TF_Strict 0x00010000 /* STRICT mode */ +#define TF_Imposter 0x00020000 /* An imposter table */ /* ** Allowed values for Table.eTabType @@ -2645,9 +2651,15 @@ struct FKey { ** argument to sqlite3VdbeKeyCompare and is used to control the ** comparison of the two index keys. ** -** Note that aSortOrder[] and aColl[] have nField+1 slots. There -** are nField slots for the columns of an index then one extra slot -** for the rowid at the end. +** The aSortOrder[] and aColl[] arrays have nAllField slots each. There +** are nKeyField slots for the columns of an index then extra slots +** for the rowid or key at the end. The aSortOrder array is located after +** the aColl[] array. +** +** If SQLITE_ENABLE_PREUPDATE_HOOK is defined, then aSortFlags might be NULL +** to indicate that this object is for use by a preupdate hook. When aSortFlags +** is NULL, then nAllField is uninitialized and no space is allocated for +** aColl[], so those fields may not be used. */ struct KeyInfo { u32 nRef; /* Number of references to this KeyInfo object */ @@ -2659,9 +2671,18 @@ struct KeyInfo { CollSeq *aColl[FLEXARRAY]; /* Collating sequence for each term of the key */ }; -/* The size (in bytes) of a KeyInfo object with up to N fields */ +/* The size (in bytes) of a KeyInfo object with up to N fields. This includes +** the main body of the KeyInfo object and the aColl[] array of N elements, +** but does not count the memory used to hold aSortFlags[]. */ #define SZ_KEYINFO(N) (offsetof(KeyInfo,aColl) + (N)*sizeof(CollSeq*)) +/* The size of a bare KeyInfo with no aColl[] entries */ +#if FLEXARRAY+1 > 1 +# define SZ_KEYINFO_0 offsetof(KeyInfo,aColl) +#else +# define SZ_KEYINFO_0 sizeof(KeyInfo) +#endif + /* ** Allowed bit values for entries in the KeyInfo.aSortFlags[] array. */ @@ -2680,9 +2701,8 @@ struct KeyInfo { ** ** An instance of this object serves as a "key" for doing a search on ** an index b+tree. The goal of the search is to find the entry that -** is closed to the key described by this object. This object might hold -** just a prefix of the key. The number of fields is given by -** pKeyInfo->nField. +** is closest to the key described by this object. This object might hold +** just a prefix of the key. The number of fields is given by nField. ** ** The r1 and r2 fields are the values to return if this key is less than ** or greater than a key in the btree, respectively. These are normally @@ -2692,7 +2712,7 @@ struct KeyInfo { ** The key comparison functions actually return default_rc when they find ** an equals comparison. default_rc can be -1, 0, or +1. If there are ** multiple entries in the b-tree with the same key (when only looking -** at the first pKeyInfo->nFields,) then default_rc can be set to -1 to +** at the first nField elements) then default_rc can be set to -1 to ** cause the search to find the last match, or +1 to cause the search to ** find the first match. ** @@ -2704,8 +2724,8 @@ struct KeyInfo { ** b-tree. */ struct UnpackedRecord { - KeyInfo *pKeyInfo; /* Collation and sort-order information */ - Mem *aMem; /* Values */ + KeyInfo *pKeyInfo; /* Comparison info for the index that is unpacked */ + Mem *aMem; /* Values for columns of the index */ union { char *z; /* Cache of aMem[0].z for vdbeRecordCompareString() */ i64 i; /* Cache of aMem[0].u.i for vdbeRecordCompareInt() */ @@ -3354,6 +3374,7 @@ struct SrcItem { unsigned rowidUsed :1; /* The ROWID of this table is referenced */ unsigned fixedSchema :1; /* Uses u4.pSchema, not u4.zDatabase */ unsigned hadSchema :1; /* Had u4.zDatabase before u4.pSchema */ + unsigned fromExists :1; /* Comes from WHERE EXISTS(...) */ } fg; int iCursor; /* The VDBE cursor number used to access this table */ Bitmask colUsed; /* Bit N set if column N used. Details above for N>62 */ @@ -3641,6 +3662,7 @@ struct Select { #define SF_OrderByReqd 0x8000000 /* The ORDER BY clause may not be omitted */ #define SF_UpdateFrom 0x10000000 /* Query originates with UPDATE FROM */ #define SF_Correlated 0x20000000 /* True if references the outer context */ +#define SF_OnToWhere 0x40000000 /* One or more ON clauses moved to WHERE */ /* True if SrcItem X is a subquery that has SF_NestedFrom */ #define IsNestedFrom(X) \ @@ -3884,6 +3906,7 @@ struct Parse { u8 disableLookaside; /* Number of times lookaside has been disabled */ u8 prepFlags; /* SQLITE_PREPARE_* flags */ u8 withinRJSubrtn; /* Nesting level for RIGHT JOIN body subroutines */ + u8 bHasExists; /* Has a correlated "EXISTS (SELECT ....)" expression */ u8 mSubrtnSig; /* mini Bloom filter on available SubrtnSig.selId */ u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ u8 bReturning; /* Coding a RETURNING trigger */ @@ -4393,6 +4416,7 @@ struct Walker { SrcItem *pSrcItem; /* A single FROM clause item */ DbFixer *pFix; /* See sqlite3FixSelect() */ Mem *aMem; /* See sqlite3BtreeCursorHint() */ + struct CheckOnCtx *pCheckOnCtx; /* See selectCheckOnClauses() */ } u; }; @@ -4880,6 +4904,7 @@ char *sqlite3VMPrintf(sqlite3*,const char*, va_list); void sqlite3ShowWindow(const Window*); void sqlite3ShowWinFunc(const Window*); #endif + void sqlite3ShowBitvec(Bitvec*); #endif void sqlite3SetString(char **, sqlite3*, const char*); @@ -5196,13 +5221,17 @@ void sqlite3RegisterDateTimeFunctions(void); void sqlite3RegisterJsonFunctions(void); void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3*); #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_JSON) - int sqlite3JsonTableFunctions(sqlite3*); + Module *sqlite3JsonVtabRegister(sqlite3*,const char*); #endif int sqlite3SafetyCheckOk(sqlite3*); int sqlite3SafetyCheckSickOrOk(sqlite3*); void sqlite3ChangeCookie(Parse*, int); With *sqlite3WithDup(sqlite3 *db, With *p); +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_CARRAY) + Module *sqlite3CarrayRegister(sqlite3*); +#endif + #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,int); #endif @@ -5423,7 +5452,7 @@ void sqlite3Reindex(Parse*, Token*, Token*); void sqlite3AlterFunctions(void); void sqlite3AlterRenameTable(Parse*, SrcList*, Token*); void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*); -int sqlite3GetToken(const unsigned char *, int *); +i64 sqlite3GetToken(const unsigned char *, int *); void sqlite3NestedParse(Parse*, const char*, ...); void sqlite3ExpirePreparedStatements(sqlite3*, int); void sqlite3CodeRhsOfIN(Parse*, Expr*, int); diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index ec774889b..6b6bb7167 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -196,7 +196,7 @@ ** Maximum number of pages in one database file. ** ** This is really just the default value for the max_page_count pragma. -** This value can be lowered (or raised) at run-time using that the +** This value can be lowered (or raised) at run-time using the ** max_page_count macro. */ #ifndef SQLITE_MAX_PAGE_COUNT diff --git a/src/status.c b/src/status.c index b0a47c7f8..5db67b87b 100644 --- a/src/status.c +++ b/src/status.c @@ -200,23 +200,25 @@ int sqlite3LookasideUsed(sqlite3 *db, int *pHighwater){ /* ** Query status information for a single database connection */ -int sqlite3_db_status( - sqlite3 *db, /* The database connection whose status is desired */ - int op, /* Status verb */ - int *pCurrent, /* Write current value here */ - int *pHighwater, /* Write high-water mark here */ - int resetFlag /* Reset high-water mark if true */ +int sqlite3_db_status64( + sqlite3 *db, /* The database connection whose status is desired */ + int op, /* Status verb */ + sqlite3_int64 *pCurrent, /* Write current value here */ + sqlite3_int64 *pHighwtr, /* Write high-water mark here */ + int resetFlag /* Reset high-water mark if true */ ){ int rc = SQLITE_OK; /* Return code */ #ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) || pCurrent==0|| pHighwater==0 ){ + if( !sqlite3SafetyCheckOk(db) || pCurrent==0|| pHighwtr==0 ){ return SQLITE_MISUSE_BKPT; } #endif sqlite3_mutex_enter(db->mutex); switch( op ){ case SQLITE_DBSTATUS_LOOKASIDE_USED: { - *pCurrent = sqlite3LookasideUsed(db, pHighwater); + int H = 0; + *pCurrent = sqlite3LookasideUsed(db, &H); + *pHighwtr = H; if( resetFlag ){ LookasideSlot *p = db->lookaside.pFree; if( p ){ @@ -247,7 +249,7 @@ int sqlite3_db_status( assert( (op-SQLITE_DBSTATUS_LOOKASIDE_HIT)>=0 ); assert( (op-SQLITE_DBSTATUS_LOOKASIDE_HIT)<3 ); *pCurrent = 0; - *pHighwater = (int)db->lookaside.anStat[op-SQLITE_DBSTATUS_LOOKASIDE_HIT]; + *pHighwtr = db->lookaside.anStat[op-SQLITE_DBSTATUS_LOOKASIDE_HIT]; if( resetFlag ){ db->lookaside.anStat[op - SQLITE_DBSTATUS_LOOKASIDE_HIT] = 0; } @@ -261,7 +263,7 @@ int sqlite3_db_status( */ case SQLITE_DBSTATUS_CACHE_USED_SHARED: case SQLITE_DBSTATUS_CACHE_USED: { - int totalUsed = 0; + sqlite3_int64 totalUsed = 0; int i; sqlite3BtreeEnterAll(db); for(i=0; inDb; i++){ @@ -277,18 +279,18 @@ int sqlite3_db_status( } sqlite3BtreeLeaveAll(db); *pCurrent = totalUsed; - *pHighwater = 0; + *pHighwtr = 0; break; } /* ** *pCurrent gets an accurate estimate of the amount of memory used ** to store the schema for all databases (main, temp, and any ATTACHed - ** databases. *pHighwater is set to zero. + ** databases. *pHighwtr is set to zero. */ case SQLITE_DBSTATUS_SCHEMA_USED: { - int i; /* Used to iterate through schemas */ - int nByte = 0; /* Used to accumulate return value */ + int i; /* Used to iterate through schemas */ + int nByte = 0; /* Used to accumulate return value */ sqlite3BtreeEnterAll(db); db->pnBytesFreed = &nByte; @@ -322,7 +324,7 @@ int sqlite3_db_status( db->lookaside.pEnd = db->lookaside.pTrueEnd; sqlite3BtreeLeaveAll(db); - *pHighwater = 0; + *pHighwtr = 0; *pCurrent = nByte; break; } @@ -330,7 +332,7 @@ int sqlite3_db_status( /* ** *pCurrent gets an accurate estimate of the amount of memory used ** to store all prepared statements. - ** *pHighwater is set to zero. + ** *pHighwtr is set to zero. */ case SQLITE_DBSTATUS_STMT_USED: { struct Vdbe *pVdbe; /* Used to iterate through VMs */ @@ -345,7 +347,7 @@ int sqlite3_db_status( db->lookaside.pEnd = db->lookaside.pTrueEnd; db->pnBytesFreed = 0; - *pHighwater = 0; /* IMP: R-64479-57858 */ + *pHighwtr = 0; /* IMP: R-64479-57858 */ *pCurrent = nByte; break; @@ -353,7 +355,7 @@ int sqlite3_db_status( /* ** Set *pCurrent to the total cache hits or misses encountered by all - ** pagers the database handle is connected to. *pHighwater is always set + ** pagers the database handle is connected to. *pHighwtr is always set ** to zero. */ case SQLITE_DBSTATUS_CACHE_SPILL: @@ -373,19 +375,39 @@ int sqlite3_db_status( sqlite3PagerCacheStat(pPager, op, resetFlag, &nRet); } } - *pHighwater = 0; /* IMP: R-42420-56072 */ + *pHighwtr = 0; /* IMP: R-42420-56072 */ /* IMP: R-54100-20147 */ /* IMP: R-29431-39229 */ - *pCurrent = (int)nRet & 0x7fffffff; + *pCurrent = nRet; + break; + } + + /* Set *pCurrent to the number of bytes that the db database connection + ** has spilled to the filesystem in temporary files that could have been + ** stored in memory, had sufficient memory been available. + ** The *pHighwater is always set to zero. + */ + case SQLITE_DBSTATUS_TEMPBUF_SPILL: { + u64 nRet = 0; + if( db->aDb[1].pBt ){ + Pager *pPager = sqlite3BtreePager(db->aDb[1].pBt); + sqlite3PagerCacheStat(pPager, SQLITE_DBSTATUS_CACHE_WRITE, + resetFlag, &nRet); + nRet *= sqlite3BtreeGetPageSize(db->aDb[1].pBt); + } + nRet += db->nSpill; + if( resetFlag ) db->nSpill = 0; + *pHighwtr = 0; + *pCurrent = nRet; break; } /* Set *pCurrent to non-zero if there are unresolved deferred foreign ** key constraints. Set *pCurrent to zero if all foreign key constraints - ** have been satisfied. The *pHighwater is always set to zero. + ** have been satisfied. The *pHighwtr is always set to zero. */ case SQLITE_DBSTATUS_DEFERRED_FKS: { - *pHighwater = 0; /* IMP: R-11967-56545 */ + *pHighwtr = 0; /* IMP: R-11967-56545 */ *pCurrent = db->nDeferredImmCons>0 || db->nDeferredCons>0; break; } @@ -397,3 +419,28 @@ int sqlite3_db_status( sqlite3_mutex_leave(db->mutex); return rc; } + +/* +** 32-bit variant of sqlite3_db_status64() +*/ +int sqlite3_db_status( + sqlite3 *db, /* The database connection whose status is desired */ + int op, /* Status verb */ + int *pCurrent, /* Write current value here */ + int *pHighwtr, /* Write high-water mark here */ + int resetFlag /* Reset high-water mark if true */ +){ + sqlite3_int64 C = 0, H = 0; + int rc; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || pCurrent==0|| pHighwtr==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + rc = sqlite3_db_status64(db, op, &C, &H, resetFlag); + if( rc==0 ){ + *pCurrent = C & 0x7fffffff; + *pHighwtr = H & 0x7fffffff; + } + return rc; +} diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 8c40b8692..02a4d84e4 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -49,6 +49,10 @@ # define CONST const #elif !defined(Tcl_Size) typedef int Tcl_Size; +# ifndef Tcl_BounceRefCount +# define Tcl_BounceRefCount(X) Tcl_IncrRefCount(X); Tcl_DecrRefCount(X) + /* https://www.tcl-lang.org/man/tcl9.0/TclLib/Object.html */ +# endif #endif /**** End copy of tclsqlite.h ****/ @@ -1084,7 +1088,9 @@ static void tclSqlFunc(sqlite3_context *context, int argc, sqlite3_value**argv){ Tcl_DecrRefCount(pCmd); } - if( rc && rc!=TCL_RETURN ){ + if( TCL_BREAK==rc ){ + sqlite3_result_null(context); + }else if( rc && rc!=TCL_RETURN ){ sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1); }else{ Tcl_Obj *pVar = Tcl_GetObjResult(p->interp); @@ -1102,7 +1108,7 @@ static void tclSqlFunc(sqlite3_context *context, int argc, sqlite3_value**argv){ }else if( (c=='b' && pVar->bytes==0 && strcmp(zType,"boolean")==0 ) || (c=='b' && pVar->bytes==0 && strcmp(zType,"booleanString")==0 ) || (c=='w' && strcmp(zType,"wideInt")==0) - || (c=='i' && strcmp(zType,"int")==0) + || (c=='i' && strcmp(zType,"int")==0) ){ eType = SQLITE_INTEGER; }else if( c=='d' && strcmp(zType,"double")==0 ){ @@ -1616,11 +1622,12 @@ struct DbEvalContext { SqlPreparedStmt *pPreStmt; /* Current statement */ int nCol; /* Number of columns returned by pStmt */ int evalFlags; /* Flags used */ - Tcl_Obj *pArray; /* Name of array variable */ + Tcl_Obj *pVarName; /* Name of target array/dict variable */ Tcl_Obj **apColName; /* Array of column names */ }; #define SQLITE_EVAL_WITHOUTNULLS 0x00001 /* Unset array(*) for NULL */ +#define SQLITE_EVAL_ASDICT 0x00002 /* Use dict instead of array */ /* ** Release any cache of column names currently held as part of @@ -1641,20 +1648,20 @@ static void dbReleaseColumnNames(DbEvalContext *p){ /* ** Initialize a DbEvalContext structure. ** -** If pArray is not NULL, then it contains the name of a Tcl array +** If pVarName is not NULL, then it contains the name of a Tcl array ** variable. The "*" member of this array is set to a list containing ** the names of the columns returned by the statement as part of each ** call to dbEvalStep(), in order from left to right. e.g. if the names ** of the returned columns are a, b and c, it does the equivalent of the ** tcl command: ** -** set ${pArray}(*) {a b c} +** set ${pVarName}(*) {a b c} */ static void dbEvalInit( DbEvalContext *p, /* Pointer to structure to initialize */ SqliteDb *pDb, /* Database handle */ Tcl_Obj *pSql, /* Object containing SQL script */ - Tcl_Obj *pArray, /* Name of Tcl array to set (*) element of */ + Tcl_Obj *pVarName, /* Name of Tcl array to set (*) element of */ int evalFlags /* Flags controlling evaluation */ ){ memset(p, 0, sizeof(DbEvalContext)); @@ -1662,9 +1669,9 @@ static void dbEvalInit( p->zSql = Tcl_GetString(pSql); p->pSql = pSql; Tcl_IncrRefCount(pSql); - if( pArray ){ - p->pArray = pArray; - Tcl_IncrRefCount(pArray); + if( pVarName ){ + p->pVarName = pVarName; + Tcl_IncrRefCount(pVarName); } p->evalFlags = evalFlags; addDatabaseRef(p->pDb); @@ -1687,7 +1694,7 @@ static void dbEvalRowInfo( Tcl_Obj **apColName = 0; /* Array of column names */ p->nCol = nCol = sqlite3_column_count(pStmt); - if( nCol>0 && (papColName || p->pArray) ){ + if( nCol>0 && (papColName || p->pVarName) ){ apColName = (Tcl_Obj**)Tcl_Alloc( sizeof(Tcl_Obj*)*nCol ); for(i=0; iapColName = apColName; } - /* If results are being stored in an array variable, then create - ** the array(*) entry for that array + /* If results are being stored in a variable then create the + ** array(*) or dict(*) entry for that variable. */ - if( p->pArray ){ + if( p->pVarName ){ Tcl_Interp *interp = p->pDb->interp; Tcl_Obj *pColList = Tcl_NewObj(); Tcl_Obj *pStar = Tcl_NewStringObj("*", -1); + Tcl_IncrRefCount(pColList); + Tcl_IncrRefCount(pStar); for(i=0; ipArray, pStar, pColList, 0); + if( 0==(SQLITE_EVAL_ASDICT & p->evalFlags) ){ + Tcl_ObjSetVar2(interp, p->pVarName, pStar, pColList, 0); + }else{ + Tcl_Obj * pDict = Tcl_ObjGetVar2(interp, p->pVarName, NULL, 0); + if( !pDict ){ + pDict = Tcl_NewDictObj(); + }else if( Tcl_IsShared(pDict) ){ + pDict = Tcl_DuplicateObj(pDict); + } + if( Tcl_DictObjPut(interp, pDict, pStar, pColList)==TCL_OK ){ + Tcl_ObjSetVar2(interp, p->pVarName, NULL, pDict, 0); + } + Tcl_BounceRefCount(pDict); + } Tcl_DecrRefCount(pStar); + Tcl_DecrRefCount(pColList); } } @@ -1751,7 +1773,7 @@ static int dbEvalStep(DbEvalContext *p){ if( rcs==SQLITE_ROW ){ return TCL_OK; } - if( p->pArray ){ + if( p->pVarName ){ dbEvalRowInfo(p, 0, 0); } rcs = sqlite3_reset(pStmt); @@ -1802,9 +1824,9 @@ static void dbEvalFinalize(DbEvalContext *p){ dbReleaseStmt(p->pDb, p->pPreStmt, 0); p->pPreStmt = 0; } - if( p->pArray ){ - Tcl_DecrRefCount(p->pArray); - p->pArray = 0; + if( p->pVarName ){ + Tcl_DecrRefCount(p->pVarName); + p->pVarName = 0; } Tcl_DecrRefCount(p->pSql); dbReleaseColumnNames(p); @@ -1879,7 +1901,7 @@ static int DbUseNre(void){ /* ** This function is part of the implementation of the command: ** -** $db eval SQL ?ARRAYNAME? SCRIPT +** $db eval SQL ?TGT-NAME? SCRIPT */ static int SQLITE_TCLAPI DbEvalNextCmd( ClientData data[], /* data[0] is the (DbEvalContext*) */ @@ -1893,8 +1915,8 @@ static int SQLITE_TCLAPI DbEvalNextCmd( ** is a pointer to a Tcl_Obj containing the script to run for each row ** returned by the queries encapsulated in data[0]. */ DbEvalContext *p = (DbEvalContext *)data[0]; - Tcl_Obj *pScript = (Tcl_Obj *)data[1]; - Tcl_Obj *pArray = p->pArray; + Tcl_Obj * const pScript = (Tcl_Obj *)data[1]; + Tcl_Obj * const pVarName = p->pVarName; while( (rc==TCL_OK || rc==TCL_CONTINUE) && TCL_OK==(rc = dbEvalStep(p)) ){ int i; @@ -1902,15 +1924,46 @@ static int SQLITE_TCLAPI DbEvalNextCmd( Tcl_Obj **apColName; dbEvalRowInfo(p, &nCol, &apColName); for(i=0; ievalFlags & SQLITE_EVAL_WITHOUTNULLS)!=0 - && sqlite3_column_type(p->pPreStmt->pStmt, i)==SQLITE_NULL + && sqlite3_column_type(p->pPreStmt->pStmt, i)==SQLITE_NULL ){ - Tcl_UnsetVar2(interp, Tcl_GetString(pArray), - Tcl_GetString(apColName[i]), 0); + /* Remove NULL-containing column from the target container... */ + if( 0==(SQLITE_EVAL_ASDICT & p->evalFlags) ){ + /* Target is an array */ + Tcl_UnsetVar2(interp, Tcl_GetString(pVarName), + Tcl_GetString(apColName[i]), 0); + }else{ + /* Target is a dict */ + Tcl_Obj *pDict = Tcl_ObjGetVar2(interp, pVarName, NULL, 0); + if( pDict ){ + if( Tcl_IsShared(pDict) ){ + pDict = Tcl_DuplicateObj(pDict); + } + if( Tcl_DictObjRemove(interp, pDict, apColName[i])==TCL_OK ){ + Tcl_ObjSetVar2(interp, pVarName, NULL, pDict, 0); + } + Tcl_BounceRefCount(pDict); + } + } + }else if( 0==(SQLITE_EVAL_ASDICT & p->evalFlags) ){ + /* Target is an array: set target(colName) = colValue */ + Tcl_ObjSetVar2(interp, pVarName, apColName[i], + dbEvalColumnValue(p,i), 0); }else{ - Tcl_ObjSetVar2(interp, pArray, apColName[i], dbEvalColumnValue(p,i), 0); + /* Target is a dict: set target(colName) = colValue */ + Tcl_Obj *pDict = Tcl_ObjGetVar2(interp, pVarName, NULL, 0); + if( !pDict ){ + pDict = Tcl_NewDictObj(); + }else if( Tcl_IsShared(pDict) ){ + pDict = Tcl_DuplicateObj(pDict); + } + if( Tcl_DictObjPut(interp, pDict, apColName[i], + dbEvalColumnValue(p,i))==TCL_OK ){ + Tcl_ObjSetVar2(interp, pVarName, NULL, pDict, 0); + } + Tcl_BounceRefCount(pDict); } } @@ -2019,7 +2072,7 @@ static int SQLITE_TCLAPI DbObjCmd( "timeout", "total_changes", "trace", "trace_v2", "transaction", "unlock_notify", "update_hook", "version", "wal_hook", - 0 + 0 }; enum DB_enum { DB_AUTHORIZER, DB_BACKUP, DB_BIND_FALLBACK, @@ -2853,13 +2906,15 @@ static int SQLITE_TCLAPI DbObjCmd( } /* - ** $db eval ?options? $sql ?array? ?{ ...code... }? + ** $db eval ?options? $sql ?varName? ?{ ...code... }? ** - ** The SQL statement in $sql is evaluated. For each row, the values are - ** placed in elements of the array named "array" and ...code... is executed. - ** If "array" and "code" are omitted, then no callback is every invoked. - ** If "array" is an empty string, then the values are placed in variables - ** that have the same name as the fields extracted by the query. + ** The SQL statement in $sql is evaluated. For each row, the values + ** are placed in elements of the array or dict named $varName and + ** ...code... is executed. If $varName and $code are omitted, then + ** no callback is ever invoked. If $varName is an empty string, + ** then the values are placed in variables that have the same name + ** as the fields extracted by the query, and those variables are + ** accessible during the eval of $code. */ case DB_EVAL: { int evalFlags = 0; @@ -2867,8 +2922,9 @@ static int SQLITE_TCLAPI DbObjCmd( while( objc>3 && (zOpt = Tcl_GetString(objv[2]))!=0 && zOpt[0]=='-' ){ if( strcmp(zOpt, "-withoutnulls")==0 ){ evalFlags |= SQLITE_EVAL_WITHOUTNULLS; - } - else{ + }else if( strcmp(zOpt, "-asdict")==0 ){ + evalFlags |= SQLITE_EVAL_ASDICT; + }else{ Tcl_AppendResult(interp, "unknown option: \"", zOpt, "\"", (void*)0); return TCL_ERROR; } @@ -2876,8 +2932,8 @@ static int SQLITE_TCLAPI DbObjCmd( objv++; } if( objc<3 || objc>5 ){ - Tcl_WrongNumArgs(interp, 2, objv, - "?OPTIONS? SQL ?ARRAY-NAME? ?SCRIPT?"); + Tcl_WrongNumArgs(interp, 2, objv, + "?OPTIONS? SQL ?VAR-NAME? ?SCRIPT?"); return TCL_ERROR; } @@ -2903,17 +2959,17 @@ static int SQLITE_TCLAPI DbObjCmd( }else{ ClientData cd2[2]; DbEvalContext *p; - Tcl_Obj *pArray = 0; + Tcl_Obj *pVarName = 0; Tcl_Obj *pScript; if( objc>=5 && *(char *)Tcl_GetString(objv[3]) ){ - pArray = objv[3]; + pVarName = objv[3]; } pScript = objv[objc-1]; Tcl_IncrRefCount(pScript); p = (DbEvalContext *)Tcl_Alloc(sizeof(DbEvalContext)); - dbEvalInit(p, pDb, objv[2], pArray, evalFlags); + dbEvalInit(p, pDb, objv[2], pVarName, evalFlags); cd2[0] = (void *)p; cd2[1] = (void *)pScript; diff --git a/src/tclsqlite.h b/src/tclsqlite.h index a9a126293..f71ec9a7c 100644 --- a/src/tclsqlite.h +++ b/src/tclsqlite.h @@ -32,7 +32,7 @@ /****** Any edits to this file must mirrored in tclsqlite.c ***********/ -/* Compatability between Tcl8.6 and Tcl9.0 */ +/* Compatibility between Tcl8.6 and Tcl9.0 */ #if TCL_MAJOR_VERSION==9 # define CONST const #elif !defined(Tcl_Size) diff --git a/src/test1.c b/src/test1.c index bb2f2d3b9..f89359932 100644 --- a/src/test1.c +++ b/src/test1.c @@ -1060,6 +1060,30 @@ static void shellDtostr( sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT); } +/* +** We need a method for setting the pointer values created by the +** intarray_addr, int64array_addr, doublearray_addr, and textarray_addr +** routines below. The inttoptr(X) SQL function accomplishes +** this. Tcl scripts will bind an array address as an integer X and +** the inttoptr() SQL function will use sqlite3_result_pointer() to +** convert that integer into a pointer usable by carray(). +*/ +static void inttoptrFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + void *p; + sqlite3_int64 i64; + i64 = sqlite3_value_int64(argv[0]); + if( sizeof(i64)==sizeof(p) ){ + memcpy(&p, &i64, sizeof(p)); + }else{ + int i32 = i64 & 0xffffffff; + memcpy(&p, &i32, sizeof(p)); + } + sqlite3_result_pointer(context, p, "carray", 0); +} /* ** Usage: sqlite3_create_function DB @@ -1074,7 +1098,8 @@ static void shellDtostr( ** ** The original motivation for this routine was to be able to call the ** sqlite3_create_function function while a query is in progress in order -** to test the SQLITE_MISUSE detection logic. +** to test the SQLITE_MISUSE detection logic. It is now also used to register +** a bunch of SQL functions that are useful for testing. */ static int SQLITE_TCLAPI test_create_function( void *NotUsed, @@ -1169,6 +1194,10 @@ static int SQLITE_TCLAPI test_create_function( rc = sqlite3_create_function(db, "dtostr", 2, SQLITE_UTF8, 0, shellDtostr, 0, 0); } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "inttoptr", 1, SQLITE_UTF8, 0, + inttoptrFunc, 0, 0); + } #ifndef SQLITE_OMIT_UTF16 /* Use the sqlite3_create_function16() API here. Mainly for fun, but also @@ -3830,7 +3859,7 @@ static int SQLITE_TCLAPI test_intarray_addr( return TCL_OK; } /* -** Usage: intarray_addr INT ... +** Usage: int64array_addr INT ... ** ** Return the address of a C-language array of 32-bit integers. ** @@ -3933,7 +3962,6 @@ static int SQLITE_TCLAPI test_textarray_addr( return TCL_OK; } - /* ** Usage: sqlite3_bind_int64 STMT N VALUE ** @@ -4360,10 +4388,71 @@ static int SQLITE_TCLAPI test_bind_value_from_select( #endif #ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** These two are used by the -malloc option to sqlite3_carray_bind() +*/ +static void *testCarrayAlloc(int n){ + u8 *pRet = (u8*)sqlite3_malloc(n+16); + if( pRet ){ + pRet = &pRet[16]; + } + return (void*)pRet; +} +static void testCarrayFree(void *p){ + if( p ){ + u8 *p2 = (u8*)p; + sqlite3_free(&p2[-16]); + } +} + +static void delIntptr(void *p){ + ckfree(p); +} + +/* +** bind_carray_intptr STMT IPARAM INT0 INT1 INT2... +*/ +static int SQLITE_TCLAPI bind_carray_intptr( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3_stmt *pStmt = 0; + int iVar = 0; + int *aInt = 0; + int nInt = 0; + int ii = 0; + int rc = SQLITE_OK; + + if( objc<3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "STMT"); + return TCL_ERROR; + } + if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[2], &iVar) ) return TCL_ERROR; + nInt = objc - 3; + + aInt = ckalloc((nInt+1) * sizeof(int)); + for(ii=0; ii # include -# ifndef DIRENT -# define DIRENT dirent -# endif #else -# include -# include "test_windirent.h" +# include "windirent.h" # ifndef S_ISREG # define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG) # endif @@ -121,7 +117,7 @@ struct FsdirCsr { char *zDir; /* Buffer containing directory scanned */ DIR *pDir; /* Open directory */ sqlite3_int64 iRowid; - struct DIRENT *pEntry; + struct dirent *pEntry; }; /* @@ -483,9 +479,9 @@ static int fstreeFilter( char aWild[2] = { '\0', '\0' }; #ifdef _WIN32 - const char *zDrive = windirent_getenv("fstreeDrive"); + const char *zDrive = getenv("fstreeDrive"); if( zDrive==0 ){ - zDrive = windirent_getenv("SystemDrive"); + zDrive = getenv("SystemDrive"); } zRoot = sqlite3_mprintf("%s%c", zDrive, '/'); nRoot = sqlite3Strlen30(zRoot); diff --git a/src/test_malloc.c b/src/test_malloc.c index 1c19d896f..68b2c367d 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -1368,6 +1368,7 @@ static int SQLITE_TCLAPI test_db_status( { "DEFERRED_FKS", SQLITE_DBSTATUS_DEFERRED_FKS }, { "CACHE_USED_SHARED", SQLITE_DBSTATUS_CACHE_USED_SHARED }, { "CACHE_SPILL", SQLITE_DBSTATUS_CACHE_SPILL }, + { "TEMPBUF_SPILL", SQLITE_DBSTATUS_TEMPBUF_SPILL }, }; Tcl_Obj *pResult; if( objc!=4 ){ diff --git a/src/test_windirent.c b/src/test_windirent.c deleted file mode 100644 index de4192d7c..000000000 --- a/src/test_windirent.c +++ /dev/null @@ -1,162 +0,0 @@ -/* -** 2015 November 30 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This file contains code to implement most of the opendir() family of -** POSIX functions on Win32 using the MSVCRT. -*/ - -#if defined(_WIN32) && defined(_MSC_VER) -#include "test_windirent.h" - -/* -** Implementation of the POSIX getenv() function using the Win32 API. -** This function is not thread-safe. -*/ -const char *windirent_getenv( - const char *name -){ - static char value[32768]; /* Maximum length, per MSDN */ - DWORD dwSize = sizeof(value) / sizeof(char); /* Size in chars */ - DWORD dwRet; /* Value returned by GetEnvironmentVariableA() */ - - memset(value, 0, sizeof(value)); - dwRet = GetEnvironmentVariableA(name, value, dwSize); - if( dwRet==0 || dwRet>dwSize ){ - /* - ** The function call to GetEnvironmentVariableA() failed -OR- - ** the buffer is not large enough. Either way, return NULL. - */ - return 0; - }else{ - /* - ** The function call to GetEnvironmentVariableA() succeeded - ** -AND- the buffer contains the entire value. - */ - return value; - } -} - -/* -** Implementation of the POSIX opendir() function using the MSVCRT. -*/ -LPDIR opendir( - const char *dirname /* Directory name, UTF8 encoding */ -){ - struct _wfinddata_t data; - LPDIR dirp = (LPDIR)sqlite3_malloc(sizeof(DIR)); - SIZE_T namesize = sizeof(data.name) / sizeof(data.name[0]); - wchar_t *b1; - sqlite3_int64 sz; - - if( dirp==NULL ) return NULL; - memset(dirp, 0, sizeof(DIR)); - - /* TODO: Remove this if Unix-style root paths are not used. */ - if( sqlite3_stricmp(dirname, "/")==0 ){ - dirname = windirent_getenv("SystemDrive"); - } - - memset(&data, 0, sizeof(data)); - sz = strlen(dirname); - b1 = sqlite3_malloc64( (sz+3)*sizeof(b1[0]) ); - if( b1==0 ){ - closedir(dirp); - return NULL; - } - sz = MultiByteToWideChar(CP_UTF8, 0, dirname, sz, b1, sz); - b1[sz++] = '\\'; - b1[sz++] = '*'; - b1[sz] = 0; - if( sz+1>(sqlite3_int64)namesize ){ - closedir(dirp); - sqlite3_free(b1); - return NULL; - } - memcpy(data.name, b1, (sz+1)*sizeof(b1[0])); - sqlite3_free(b1); - dirp->d_handle = _wfindfirst(data.name, &data); - - if( dirp->d_handle==BAD_INTPTR_T ){ - closedir(dirp); - return NULL; - } - - /* TODO: Remove this block to allow hidden and/or system files. */ - if( is_filtered(data) ){ -next: - - memset(&data, 0, sizeof(data)); - if( _wfindnext(dirp->d_handle, &data)==-1 ){ - closedir(dirp); - return NULL; - } - - /* TODO: Remove this block to allow hidden and/or system files. */ - if( is_filtered(data) ) goto next; - } - - dirp->d_first.d_attributes = data.attrib; - WideCharToMultiByte(CP_UTF8, 0, data.name, -1, - dirp->d_first.d_name, DIRENT_NAME_MAX, 0, 0); - return dirp; -} - -/* -** Implementation of the POSIX readdir() function using the MSVCRT. -*/ -LPDIRENT readdir( - LPDIR dirp -){ - struct _wfinddata_t data; - - if( dirp==NULL ) return NULL; - - if( dirp->d_first.d_ino==0 ){ - dirp->d_first.d_ino++; - dirp->d_next.d_ino++; - - return &dirp->d_first; - } - -next: - - memset(&data, 0, sizeof(data)); - if( _wfindnext(dirp->d_handle, &data)==-1 ) return NULL; - - /* TODO: Remove this block to allow hidden and/or system files. */ - if( is_filtered(data) ) goto next; - - dirp->d_next.d_ino++; - dirp->d_next.d_attributes = data.attrib; - WideCharToMultiByte(CP_UTF8, 0, data.name, -1, - dirp->d_next.d_name, DIRENT_NAME_MAX, 0, 0); - return &dirp->d_next; -} - -/* -** Implementation of the POSIX closedir() function using the MSVCRT. -*/ -INT closedir( - LPDIR dirp -){ - INT result = 0; - - if( dirp==NULL ) return EINVAL; - - if( dirp->d_handle!=NULL_INTPTR_T && dirp->d_handle!=BAD_INTPTR_T ){ - result = _findclose(dirp->d_handle); - } - - sqlite3_free(dirp); - return result; -} - -#endif /* defined(WIN32) && defined(_MSC_VER) */ diff --git a/src/test_windirent.h b/src/test_windirent.h deleted file mode 100644 index 527dfaa8f..000000000 --- a/src/test_windirent.h +++ /dev/null @@ -1,158 +0,0 @@ -/* -** 2015 November 30 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This file contains declarations for most of the opendir() family of -** POSIX functions on Win32 using the MSVCRT. -*/ - -#if defined(_WIN32) && defined(_MSC_VER) && !defined(SQLITE_WINDIRENT_H) -#define SQLITE_WINDIRENT_H - -/* -** We need several data types from the Windows SDK header. -*/ - -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif - -#include "windows.h" - -/* -** We need several support functions from the SQLite core. -*/ - -#include "sqlite3.h" - -/* -** We need several things from the ANSI and MSVCRT headers. -*/ - -#include -#include -#include -#include -#include -#include -#include - -/* -** We may need several defines that should have been in "sys/stat.h". -*/ - -#ifndef S_ISREG -#define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG) -#endif - -#ifndef S_ISDIR -#define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) -#endif - -#ifndef S_ISLNK -#define S_ISLNK(mode) (0) -#endif - -/* -** We may need to provide the "mode_t" type. -*/ - -#ifndef MODE_T_DEFINED - #define MODE_T_DEFINED - typedef unsigned short mode_t; -#endif - -/* -** We may need to provide the "ino_t" type. -*/ - -#ifndef INO_T_DEFINED - #define INO_T_DEFINED - typedef unsigned short ino_t; -#endif - -/* -** We need to define "NAME_MAX" if it was not present in "limits.h". -*/ - -#ifndef NAME_MAX -# ifdef FILENAME_MAX -# define NAME_MAX (FILENAME_MAX) -# else -# define NAME_MAX (260) -# endif -# define DIRENT_NAME_MAX (NAME_MAX) -#endif - -/* -** We need to define "NULL_INTPTR_T" and "BAD_INTPTR_T". -*/ - -#ifndef NULL_INTPTR_T -# define NULL_INTPTR_T ((intptr_t)(0)) -#endif - -#ifndef BAD_INTPTR_T -# define BAD_INTPTR_T ((intptr_t)(-1)) -#endif - -/* -** We need to provide the necessary structures and related types. -*/ - -#ifndef DIRENT_DEFINED -#define DIRENT_DEFINED -typedef struct DIRENT DIRENT; -typedef DIRENT *LPDIRENT; -struct DIRENT { - ino_t d_ino; /* Sequence number, do not use. */ - unsigned d_attributes; /* Win32 file attributes. */ - char d_name[NAME_MAX + 1]; /* Name within the directory. */ -}; -#endif - -#ifndef DIR_DEFINED -#define DIR_DEFINED -typedef struct DIR DIR; -typedef DIR *LPDIR; -struct DIR { - intptr_t d_handle; /* Value returned by "_findfirst". */ - DIRENT d_first; /* DIRENT constructed based on "_findfirst". */ - DIRENT d_next; /* DIRENT constructed based on "_findnext". */ -}; -#endif - -/* -** Provide a macro, for use by the implementation, to determine if a -** particular directory entry should be skipped over when searching for -** the next directory entry that should be returned by the readdir(). -*/ - -#ifndef is_filtered -# define is_filtered(a) ((((a).attrib)&_A_HIDDEN) || (((a).attrib)&_A_SYSTEM)) -#endif - -/* -** Provide the function prototype for the POSIX compatible getenv() -** function. This function is not thread-safe. -*/ - -extern const char *windirent_getenv(const char *name); - -/* -** Finally, we can provide the function prototypes for the opendir(), -** readdir(), and closedir() POSIX functions. -*/ - -extern LPDIR opendir(const char *dirname); -extern LPDIRENT readdir(LPDIR dirp); -extern INT closedir(LPDIR dirp); - -#endif /* defined(WIN32) && defined(_MSC_VER) */ diff --git a/src/tokenize.c b/src/tokenize.c index 6f7bab35b..152ada64f 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -270,8 +270,9 @@ static int analyzeFilterKeyword(const unsigned char *z, int lastToken){ ** Return the length (in bytes) of the token that begins at z[0]. ** Store the token type in *tokenType before returning. */ -int sqlite3GetToken(const unsigned char *z, int *tokenType){ - int i, c; +i64 sqlite3GetToken(const unsigned char *z, int *tokenType){ + i64 i; + int c; switch( aiClass[*z] ){ /* Switch on the character-class of the first byte ** of the token. See the comment on the CC_ defines ** above. */ @@ -599,7 +600,7 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ int sqlite3RunParser(Parse *pParse, const char *zSql){ int nErr = 0; /* Number of errors encountered */ void *pEngine; /* The LEMON-generated LALR(1) parser */ - int n = 0; /* Length of the next token token */ + i64 n = 0; /* Length of the next token token */ int tokenType; /* type of the next token */ int lastTokenParsed = -1; /* type of the previous token */ sqlite3 *db = pParse->db; /* The database connection */ @@ -702,13 +703,13 @@ int sqlite3RunParser(Parse *pParse, const char *zSql){ }else if( tokenType!=TK_QNUMBER ){ Token x; x.z = zSql; - x.n = n; + x.n = (u32)n; sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"", &x); break; } } pParse->sLastToken.z = zSql; - pParse->sLastToken.n = n; + pParse->sLastToken.n = (u32)n; sqlite3Parser(pEngine, tokenType, pParse->sLastToken); lastTokenParsed = tokenType; zSql += n; @@ -784,7 +785,7 @@ char *sqlite3Normalize( ){ sqlite3 *db; /* The database connection */ int i; /* Next unread byte of zSql[] */ - int n; /* length of current token */ + i64 n; /* length of current token */ int tokenType; /* type of current token */ int prevType = 0; /* Previous non-whitespace token */ int nParen; /* Number of nested levels of parentheses */ diff --git a/src/treeview.c b/src/treeview.c index 832965924..153fec88d 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -238,9 +238,13 @@ void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc){ n = 0; if( pItem->fg.isSubquery ) n++; if( pItem->fg.isTabFunc ) n++; - if( pItem->fg.isUsing ) n++; + if( pItem->fg.isUsing || pItem->u3.pOn!=0 ) n++; if( pItem->fg.isUsing ){ sqlite3TreeViewIdList(pView, pItem->u3.pUsing, (--n)>0, "USING"); + }else if( pItem->u3.pOn!=0 ){ + sqlite3TreeViewItem(pView, "ON", (--n)>0); + sqlite3TreeViewExpr(pView, pItem->u3.pOn, 0); + sqlite3TreeViewPop(&pView); } if( pItem->fg.isSubquery ){ assert( n==1 ); diff --git a/src/trigger.c b/src/trigger.c index 779da5e5f..799fbe57f 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -1040,7 +1040,10 @@ static void codeReturningTrigger( Returning *pReturning; Select sSelect; SrcList *pFrom; - u8 fromSpace[SZ_SRCLIST_1]; + union { + SrcList sSrc; + u8 fromSpace[SZ_SRCLIST_1]; + } uSrc; assert( v!=0 ); if( !pParse->bReturning ){ @@ -1056,8 +1059,8 @@ static void codeReturningTrigger( return; } memset(&sSelect, 0, sizeof(sSelect)); - pFrom = (SrcList*)fromSpace; - memset(pFrom, 0, SZ_SRCLIST_1); + memset(&uSrc, 0, sizeof(uSrc)); + pFrom = &uSrc.sSrc; sSelect.pEList = sqlite3ExprListDup(db, pReturning->pReturnEL, 0); sSelect.pSrc = pFrom; pFrom->nSrc = 1; diff --git a/src/vacuum.c b/src/vacuum.c index 96d77e5bc..1b4838040 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -195,7 +195,8 @@ SQLITE_NOINLINE int sqlite3RunVacuum( saved_nChange = db->nChange; saved_nTotalChange = db->nTotalChange; saved_mTrace = db->mTrace; - db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_Comments; + db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_Comments + | SQLITE_AttachCreate | SQLITE_AttachWrite; db->mDbFlags |= DBFLAG_PreferBuiltin | DBFLAG_Vacuum; db->flags &= ~(u64)(SQLITE_ForeignKeys | SQLITE_ReverseOrder | SQLITE_Defensive | SQLITE_CountRows); diff --git a/src/vdbe.c b/src/vdbe.c index 29b6f9a65..b5a262e63 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -717,7 +717,7 @@ static u64 filterHash(const Mem *aMem, const Op *pOp){ static SQLITE_NOINLINE int vdbeColumnFromOverflow( VdbeCursor *pC, /* The BTree cursor from which we are reading */ int iCol, /* The column to read */ - int t, /* The serial-type code for the column value */ + u32 t, /* The serial-type code for the column value */ i64 iOffset, /* Offset to the start of the content value */ u32 cacheStatus, /* Current Vdbe.cacheCtr value */ u32 colCacheCtr, /* Current value of the column cache counter */ @@ -792,6 +792,36 @@ static SQLITE_NOINLINE int vdbeColumnFromOverflow( return rc; } +/* +** Send a "statement aborts" message to the error log. +*/ +static SQLITE_NOINLINE void sqlite3VdbeLogAbort( + Vdbe *p, /* The statement that is running at the time of failure */ + int rc, /* Error code */ + Op *pOp, /* Opcode that filed */ + Op *aOp /* All opcodes */ +){ + const char *zSql = p->zSql; /* Original SQL text */ + const char *zPrefix = ""; /* Prefix added to SQL text */ + int pc; /* Opcode address */ + char zXtra[100]; /* Buffer space to store zPrefix */ + + if( p->pFrame ){ + assert( aOp[0].opcode==OP_Init ); + if( aOp[0].p4.z!=0 ){ + assert( aOp[0].p4.z[0]=='-' + && aOp[0].p4.z[1]=='-' + && aOp[0].p4.z[2]==' ' ); + sqlite3_snprintf(sizeof(zXtra), zXtra,"/* %s */ ",aOp[0].p4.z+3); + zPrefix = zXtra; + }else{ + zPrefix = "/* unknown trigger */ "; + } + } + pc = (int)(pOp - aOp); + sqlite3_log(rc, "statement aborts at %d: %s; [%s%s]", + pc, p->zErrMsg, zPrefix, zSql); +} /* ** Return the symbolic name for the data type of a pMem @@ -1317,8 +1347,7 @@ case OP_Halt: { }else{ sqlite3VdbeError(p, "%s", pOp->p4.z); } - pcx = (int)(pOp - aOp); - sqlite3_log(pOp->p1, "abort at %d: %s; [%s]", pcx, p->zErrMsg, p->zSql); + sqlite3VdbeLogAbort(p, pOp->p1, pOp, aOp); } rc = sqlite3VdbeHalt(p); assert( rc==SQLITE_BUSY || rc==SQLITE_OK || rc==SQLITE_ERROR ); @@ -1697,7 +1726,7 @@ case OP_IntCopy: { /* out2 */ ** RETURNING clause. */ case OP_FkCheck: { - if( (rc = sqlite3VdbeCheckFk(p,0))!=SQLITE_OK ){ + if( (rc = sqlite3VdbeCheckFkImmediate(p))!=SQLITE_OK ){ goto abort_due_to_error; } break; @@ -1789,10 +1818,14 @@ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ if( sqlite3VdbeMemExpandBlob(pIn2) ) goto no_mem; flags2 = pIn2->flags & ~MEM_Str; } - nByte = pIn1->n + pIn2->n; + nByte = pIn1->n; + nByte += pIn2->n; if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; } +#if SQLITE_MAX_LENGTH>2147483645 + if( nByte>2147483645 ){ goto too_big; } +#endif if( sqlite3VdbeMemGrow(pOut, (int)nByte+2, pOut==pIn2) ){ goto no_mem; } @@ -2476,6 +2509,7 @@ case OP_Compare: { pKeyInfo = pOp->p4.pKeyInfo; assert( n>0 ); assert( pKeyInfo!=0 ); + assert( pKeyInfo->aSortFlags!=0 ); p1 = pOp->p1; p2 = pOp->p2; #ifdef SQLITE_DEBUG @@ -3238,6 +3272,15 @@ case OP_Column: { /* ncycle */ ** Take the affinities from the Table object in P4. If any value ** cannot be coerced into the correct type, then raise an error. ** +** If P3==0, then omit checking of VIRTUAL columns. +** +** If P3==1, then omit checking of all generated column, both VIRTUAL +** and STORED. +** +** If P3>=2, then only check column number P3-2 in the table (which will +** be a VIRTUAL column) against the value in reg[P1]. In this case, +** P2 will be 1. +** ** This opcode is similar to OP_Affinity except that this opcode ** forces the register type to the Table column type. This is used ** to implement "strict affinity". @@ -3251,8 +3294,8 @@ case OP_Column: { /* ncycle */ ** **
      **
    • P2 should be the number of non-virtual columns in the -** table of P4. -**
    • Table P4 should be a STRICT table. +** table of P4 unless P3>1, in which case P2 will be 1. +**
    • Table P4 is a STRICT table. **
    ** ** If any precondition is false, an assertion fault occurs. @@ -3261,16 +3304,28 @@ case OP_TypeCheck: { Table *pTab; Column *aCol; int i; + int nCol; assert( pOp->p4type==P4_TABLE ); pTab = pOp->p4.pTab; assert( pTab->tabFlags & TF_Strict ); - assert( pTab->nNVCol==pOp->p2 ); + assert( pOp->p3>=0 && pOp->p3nCol+2 ); aCol = pTab->aCol; pIn1 = &aMem[pOp->p1]; - for(i=0; inCol; i++){ - if( aCol[i].colFlags & COLFLAG_GENERATED ){ - if( aCol[i].colFlags & COLFLAG_VIRTUAL ) continue; + if( pOp->p3<2 ){ + assert( pTab->nNVCol==pOp->p2 ); + i = 0; + nCol = pTab->nCol; + }else{ + i = pOp->p3-2; + nCol = i+1; + assert( inCol ); + assert( aCol[i].colFlags & COLFLAG_VIRTUAL ); + assert( pOp->p2==1 ); + } + for(; ip3<2 ){ + if( (aCol[i].colFlags & COLFLAG_VIRTUAL)!=0 ) continue; if( pOp->p3 ){ pIn1++; continue; } } assert( pIn1 < &aMem[pOp->p1+pOp->p2] ); @@ -3592,7 +3647,7 @@ case OP_MakeRecord: { len = (u32)pRec->n; serial_type = (len*2) + 12 + ((pRec->flags & MEM_Str)!=0); if( pRec->flags & MEM_Zero ){ - serial_type += pRec->u.nZero*2; + serial_type += (u32)pRec->u.nZero*2; if( nData ){ if( sqlite3VdbeMemExpandBlob(pRec) ) goto no_mem; len += pRec->u.nZero; @@ -3859,7 +3914,7 @@ case OP_Savepoint: { */ int isTransaction = pSavepoint->pNext==0 && db->isTransactionSavepoint; if( isTransaction && p1==SAVEPOINT_RELEASE ){ - if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ + if( (rc = sqlite3VdbeCheckFkDeferred(p))!=SQLITE_OK ){ goto vdbe_return; } db->autoCommit = 1; @@ -3977,7 +4032,7 @@ case OP_AutoCommit: { "SQL statements in progress"); rc = SQLITE_BUSY; goto abort_due_to_error; - }else if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ + }else if( (rc = sqlite3VdbeCheckFkDeferred(p))!=SQLITE_OK ){ goto vdbe_return; }else{ db->autoCommit = (u8)desiredAutoCommit; @@ -5349,7 +5404,7 @@ case OP_Found: { /* jump, in3, ncycle */ if( rc ) goto no_mem; pIdxKey = sqlite3VdbeAllocUnpackedRecord(pC->pKeyInfo); if( pIdxKey==0 ) goto no_mem; - sqlite3VdbeRecordUnpack(pC->pKeyInfo, r.aMem->n, r.aMem->z, pIdxKey); + sqlite3VdbeRecordUnpack(r.aMem->n, r.aMem->z, pIdxKey); pIdxKey->default_rc = 0; rc = sqlite3BtreeIndexMoveto(pC->uc.pCursor, pIdxKey, &pC->seekResult); sqlite3DbFreeNN(db, pIdxKey); @@ -6347,6 +6402,32 @@ case OP_Rewind: { /* jump0, ncycle */ break; } +/* Opcode: IfEmpty P1 P2 * * * +** Synopsis: if( empty(P1) ) goto P2 +** +** Check to see if the b-tree table that cursor P1 references is empty +** and jump to P2 if it is. +*/ +case OP_IfEmpty: { /* jump */ + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p2>=0 && pOp->p2nOp ); + + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + pCrsr = pC->uc.pCursor; + assert( pCrsr ); + rc = sqlite3BtreeIsEmpty(pCrsr, &res); + if( rc ) goto abort_due_to_error; + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; + break; +} + /* Opcode: Next P1 P2 P3 * P5 ** ** Advance cursor P1 so that it points to the next key/data pair in its @@ -7883,6 +7964,7 @@ case OP_Checkpoint: { || pOp->p2==SQLITE_CHECKPOINT_FULL || pOp->p2==SQLITE_CHECKPOINT_RESTART || pOp->p2==SQLITE_CHECKPOINT_TRUNCATE + || pOp->p2==SQLITE_CHECKPOINT_NOOP ); rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &aRes[1], &aRes[2]); if( rc ){ @@ -8218,7 +8300,14 @@ case OP_VOpen: { /* ncycle */ const sqlite3_module *pModule; assert( p->bIsReader ); - pCur = 0; + pCur = p->apCsr[pOp->p1]; + if( pCur!=0 + && ALWAYS( pCur->eCurType==CURTYPE_VTAB ) + && ALWAYS( pCur->uc.pVCur->pVtab==pOp->p4.pVtab->pVtab ) + ){ + /* This opcode is a no-op if the cursor is already open */ + break; + } pVCur = 0; pVtab = pOp->p4.pVtab->pVtab; if( pVtab==0 || NEVER(pVtab->pModule==0) ){ @@ -9160,8 +9249,7 @@ default: { /* This is really OP_Noop, OP_Explain */ p->rc = rc; sqlite3SystemError(db, rc); testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(rc, "statement aborts at %d: %s; [%s]", - (int)(pOp - aOp), p->zErrMsg, p->zSql); + sqlite3VdbeLogAbort(p, rc, pOp, aOp); if( p->eVdbeState==VDBE_RUN_STATE ) sqlite3VdbeHalt(p); if( rc==SQLITE_IOERR_NOMEM ) sqlite3OomFault(db); if( rc==SQLITE_CORRUPT && db->autoCommit==0 ){ diff --git a/src/vdbe.h b/src/vdbe.h index dc98e270e..28df764bc 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -300,8 +300,11 @@ void sqlite3VdbeSetVarmask(Vdbe*, int); #endif int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); int sqlite3BlobCompare(const Mem*, const Mem*); +#ifdef SQLITE_ENABLE_PERCENTILE + const char *sqlite3VdbeFuncName(const sqlite3_context*); +#endif -void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); +void sqlite3VdbeRecordUnpack(int,const void*,UnpackedRecord*); int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); int sqlite3VdbeRecordCompareWithSkip(int, const void *, UnpackedRecord *, int); UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo*); @@ -314,7 +317,9 @@ int sqlite3VdbeHasSubProgram(Vdbe*); void sqlite3MemSetArrayInt64(sqlite3_value *aMem, int iIdx, i64 val); +#ifndef SQLITE_OMIT_DATETIME_FUNCS int sqlite3NotPureFunc(sqlite3_context*); +#endif #ifdef SQLITE_ENABLE_BYTECODE_VTAB int sqlite3VdbeBytecodeVtabInit(sqlite3*); #endif diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 13262cd4e..8b68c339a 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -288,7 +288,7 @@ struct sqlite3_value { ** MEM_Int, MEM_Real, and MEM_IntReal. ** ** * MEM_Blob|MEM_Zero A blob in Mem.z of length Mem.n plus -** MEM.u.i extra 0x00 bytes at the end. +** Mem.u.nZero extra 0x00 bytes at the end. ** ** * MEM_Int Integer stored in Mem.u.i. ** @@ -557,7 +557,9 @@ struct PreUpdate { Table *pTab; /* Schema object being updated */ Index *pPk; /* PK index if pTab is WITHOUT ROWID */ sqlite3_value **apDflt; /* Array of default values, if required */ - u8 keyinfoSpace[SZ_KEYINFO(0)]; /* Space to hold pKeyinfo[0] content */ + struct { + u8 keyinfoSpace[SZ_KEYINFO_0]; /* Space to hold pKeyinfo[0] content */ + } uKey; }; /* @@ -721,9 +723,11 @@ int sqlite3VdbeCheckMemInvariants(Mem*); #endif #ifndef SQLITE_OMIT_FOREIGN_KEY -int sqlite3VdbeCheckFk(Vdbe *, int); +int sqlite3VdbeCheckFkImmediate(Vdbe*); +int sqlite3VdbeCheckFkDeferred(Vdbe*); #else -# define sqlite3VdbeCheckFk(p,i) 0 +# define sqlite3VdbeCheckFkImmediate(p) 0 +# define sqlite3VdbeCheckFkDeferred(p) 0 #endif #ifdef SQLITE_DEBUG diff --git a/src/vdbeapi.c b/src/vdbeapi.c index ed9549462..1118481d1 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1693,8 +1693,12 @@ static int bindText( if( zData!=0 ){ pVar = &p->aVar[i-1]; rc = sqlite3VdbeMemSetStr(pVar, zData, nData, encoding, xDel); - if( rc==SQLITE_OK && encoding!=0 ){ - rc = sqlite3VdbeChangeEncoding(pVar, ENC(p->db)); + if( rc==SQLITE_OK ){ + if( encoding==0 ){ + pVar->enc = ENC(p->db); + }else{ + rc = sqlite3VdbeChangeEncoding(pVar, ENC(p->db)); + } } if( rc ){ sqlite3Error(p->db, rc); @@ -2163,7 +2167,7 @@ static UnpackedRecord *vdbeUnpackRecord( pRet = sqlite3VdbeAllocUnpackedRecord(pKeyInfo); if( pRet ){ memset(pRet->aMem, 0, sizeof(Mem)*(pKeyInfo->nKeyField+1)); - sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, pRet); + sqlite3VdbeRecordUnpack(nKey, pKey, pRet); } return pRet; } @@ -2192,6 +2196,9 @@ int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ } if( p->pPk ){ iStore = sqlite3TableColumnToIndex(p->pPk, iIdx); + }else if( iIdx >= p->pTab->nCol ){ + rc = SQLITE_MISUSE_BKPT; + goto preupdate_old_out; }else{ iStore = sqlite3TableColumnToStorage(p->pTab, iIdx); } @@ -2347,6 +2354,8 @@ int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ } if( p->pPk && p->op!=SQLITE_UPDATE ){ iStore = sqlite3TableColumnToIndex(p->pPk, iIdx); + }else if( iIdx >= p->pTab->nCol ){ + return SQLITE_MISUSE_BKPT; }else{ iStore = sqlite3TableColumnToStorage(p->pTab, iIdx); } diff --git a/src/vdbeaux.c b/src/vdbeaux.c index a6798e62d..5368c0c42 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2994,10 +2994,12 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ if( 0==sqlite3Strlen30(sqlite3BtreeGetFilename(db->aDb[0].pBt)) || nTrans<=1 ){ - for(i=0; rc==SQLITE_OK && inDb; i++){ - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - rc = sqlite3BtreeCommitPhaseOne(pBt, 0); + if( needXcommit ){ + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( sqlite3BtreeTxnState(pBt)>=SQLITE_TXN_WRITE ){ + rc = sqlite3BtreeCommitPhaseOne(pBt, 0); + } } } @@ -3008,7 +3010,9 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ */ for(i=0; rc==SQLITE_OK && inDb; i++){ Btree *pBt = db->aDb[i].pBt; - if( pBt ){ + int txn = sqlite3BtreeTxnState(pBt); + if( txn!=SQLITE_TXN_NONE ){ + assert( needXcommit || txn==SQLITE_TXN_READ ); rc = sqlite3BtreeCommitPhaseTwo(pBt, 0); } } @@ -3263,28 +3267,31 @@ int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){ /* -** This function is called when a transaction opened by the database +** These functions are called when a transaction opened by the database ** handle associated with the VM passed as an argument is about to be -** committed. If there are outstanding deferred foreign key constraint -** violations, return SQLITE_ERROR. Otherwise, SQLITE_OK. +** committed. If there are outstanding foreign key constraint violations +** return an error code. Otherwise, SQLITE_OK. ** ** If there are outstanding FK violations and this function returns -** SQLITE_ERROR, set the result of the VM to SQLITE_CONSTRAINT_FOREIGNKEY -** and write an error message to it. Then return SQLITE_ERROR. +** non-zero, set the result of the VM to SQLITE_CONSTRAINT_FOREIGNKEY +** and write an error message to it. */ #ifndef SQLITE_OMIT_FOREIGN_KEY -int sqlite3VdbeCheckFk(Vdbe *p, int deferred){ +static SQLITE_NOINLINE int vdbeFkError(Vdbe *p){ + p->rc = SQLITE_CONSTRAINT_FOREIGNKEY; + p->errorAction = OE_Abort; + sqlite3VdbeError(p, "FOREIGN KEY constraint failed"); + if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)==0 ) return SQLITE_ERROR; + return SQLITE_CONSTRAINT_FOREIGNKEY; +} +int sqlite3VdbeCheckFkImmediate(Vdbe *p){ + if( p->nFkConstraint==0 ) return SQLITE_OK; + return vdbeFkError(p); +} +int sqlite3VdbeCheckFkDeferred(Vdbe *p){ sqlite3 *db = p->db; - if( (deferred && (db->nDeferredCons+db->nDeferredImmCons)>0) - || (!deferred && p->nFkConstraint>0) - ){ - p->rc = SQLITE_CONSTRAINT_FOREIGNKEY; - p->errorAction = OE_Abort; - sqlite3VdbeError(p, "FOREIGN KEY constraint failed"); - if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)==0 ) return SQLITE_ERROR; - return SQLITE_CONSTRAINT_FOREIGNKEY; - } - return SQLITE_OK; + if( (db->nDeferredCons+db->nDeferredImmCons)==0 ) return SQLITE_OK; + return vdbeFkError(p); } #endif @@ -3378,7 +3385,7 @@ int sqlite3VdbeHalt(Vdbe *p){ /* Check for immediate foreign key violations. */ if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ - (void)sqlite3VdbeCheckFk(p, 0); + (void)sqlite3VdbeCheckFkImmediate(p); } /* If the auto-commit flag is set and this is the only active writer @@ -3392,7 +3399,7 @@ int sqlite3VdbeHalt(Vdbe *p){ && db->nVdbeWrite==(p->readOnly==0) ){ if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ - rc = sqlite3VdbeCheckFk(p, 1); + rc = sqlite3VdbeCheckFkDeferred(p); if( rc!=SQLITE_OK ){ if( NEVER(p->readOnly) ){ sqlite3VdbeLeave(p); @@ -4202,30 +4209,22 @@ void sqlite3VdbeSerialGet( return; } /* -** This routine is used to allocate sufficient space for an UnpackedRecord -** structure large enough to be used with sqlite3VdbeRecordUnpack() if -** the first argument is a pointer to KeyInfo structure pKeyInfo. -** -** The space is either allocated using sqlite3DbMallocRaw() or from within -** the unaligned buffer passed via the second and third arguments (presumably -** stack space). If the former, then *ppFree is set to a pointer that should -** be eventually freed by the caller using sqlite3DbFree(). Or, if the -** allocation comes from the pSpace/szSpace buffer, *ppFree is set to NULL -** before returning. +** Allocate sufficient space for an UnpackedRecord structure large enough +** to hold a decoded index record for pKeyInfo. ** -** If an OOM error occurs, NULL is returned. +** The space is allocated using sqlite3DbMallocRaw(). If an OOM error +** occurs, NULL is returned. */ UnpackedRecord *sqlite3VdbeAllocUnpackedRecord( KeyInfo *pKeyInfo /* Description of the record */ ){ UnpackedRecord *p; /* Unpacked record to return */ - int nByte; /* Number of bytes required for *p */ + u64 nByte; /* Number of bytes required for *p */ assert( sizeof(UnpackedRecord) + sizeof(Mem)*65536 < 0x7fffffff ); nByte = ROUND8P(sizeof(UnpackedRecord)) + sizeof(Mem)*(pKeyInfo->nKeyField+1); p = (UnpackedRecord *)sqlite3DbMallocRaw(pKeyInfo->db, nByte); if( !p ) return 0; p->aMem = (Mem*)&((char*)p)[ROUND8P(sizeof(UnpackedRecord))]; - assert( pKeyInfo->aSortFlags!=0 ); p->pKeyInfo = pKeyInfo; p->nField = pKeyInfo->nKeyField + 1; return p; @@ -4237,7 +4236,6 @@ UnpackedRecord *sqlite3VdbeAllocUnpackedRecord( ** contents of the decoded record. */ void sqlite3VdbeRecordUnpack( - KeyInfo *pKeyInfo, /* Information about the record format */ int nKey, /* Size of the binary record */ const void *pKey, /* The binary record */ UnpackedRecord *p /* Populate this structure before returning. */ @@ -4248,6 +4246,7 @@ void sqlite3VdbeRecordUnpack( u16 u; /* Unsigned loop counter */ u32 szHdr; Mem *pMem = p->aMem; + KeyInfo *pKeyInfo = p->pKeyInfo; p->default_rc = 0; assert( EIGHT_BYTE_ALIGNMENT(pMem) ); @@ -4265,16 +4264,18 @@ void sqlite3VdbeRecordUnpack( pMem->z = 0; sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); d += sqlite3VdbeSerialTypeLen(serial_type); - pMem++; if( (++u)>=p->nField ) break; + pMem++; } if( d>(u32)nKey && u ){ assert( CORRUPT_DB ); /* In a corrupt record entry, the last pMem might have been set up using ** uninitialized memory. Overwrite its value with NULL, to prevent ** warnings from MSAN. */ - sqlite3VdbeMemSetNull(pMem-1); + sqlite3VdbeMemSetNull(pMem-(unField)); } + testcase( u == pKeyInfo->nKeyField + 1 ); + testcase( u < pKeyInfo->nKeyField + 1 ); assert( u<=pKeyInfo->nKeyField + 1 ); p->nField = u; } @@ -4442,6 +4443,32 @@ static void vdbeAssertFieldCountWithinLimits( ** or positive value if *pMem1 is less than, equal to or greater than ** *pMem2, respectively. Similar in spirit to "rc = (*pMem1) - (*pMem2);". */ +static SQLITE_NOINLINE int vdbeCompareMemStringWithEncodingChange( + const Mem *pMem1, + const Mem *pMem2, + const CollSeq *pColl, + u8 *prcErr /* If an OOM occurs, set to SQLITE_NOMEM */ +){ + int rc; + const void *v1, *v2; + Mem c1; + Mem c2; + sqlite3VdbeMemInit(&c1, pMem1->db, MEM_Null); + sqlite3VdbeMemInit(&c2, pMem1->db, MEM_Null); + sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); + sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); + v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); + v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); + if( (v1==0 || v2==0) ){ + if( prcErr ) *prcErr = SQLITE_NOMEM_BKPT; + rc = 0; + }else{ + rc = pColl->xCmp(pColl->pUser, c1.n, v1, c2.n, v2); + } + sqlite3VdbeMemReleaseMalloc(&c1); + sqlite3VdbeMemReleaseMalloc(&c2); + return rc; +} static int vdbeCompareMemString( const Mem *pMem1, const Mem *pMem2, @@ -4453,25 +4480,7 @@ static int vdbeCompareMemString( ** comparison function directly */ return pColl->xCmp(pColl->pUser,pMem1->n,pMem1->z,pMem2->n,pMem2->z); }else{ - int rc; - const void *v1, *v2; - Mem c1; - Mem c2; - sqlite3VdbeMemInit(&c1, pMem1->db, MEM_Null); - sqlite3VdbeMemInit(&c2, pMem1->db, MEM_Null); - sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); - sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); - v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); - v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); - if( (v1==0 || v2==0) ){ - if( prcErr ) *prcErr = SQLITE_NOMEM_BKPT; - rc = 0; - }else{ - rc = pColl->xCmp(pColl->pUser, c1.n, v1, c2.n, v2); - } - sqlite3VdbeMemReleaseMalloc(&c1); - sqlite3VdbeMemReleaseMalloc(&c2); - return rc; + return vdbeCompareMemStringWithEncodingChange(pMem1,pMem2,pColl,prcErr); } } @@ -5134,6 +5143,7 @@ RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ ** The easiest way to enforce this limit is to consider only records with ** 13 fields or less. If the first field is an integer, the maximum legal ** header size is (12*5 + 1 + 1) bytes. */ + assert( p->pKeyInfo->aSortFlags!=0 ); if( p->pKeyInfo->nAllField<=13 ){ int flags = p->aMem[0].flags; if( p->pKeyInfo->aSortFlags[0] ){ @@ -5383,6 +5393,7 @@ void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ } } +#ifndef SQLITE_OMIT_DATETIME_FUNCS /* ** Cause a function to throw an error if it was call from OP_PureFunc ** rather than OP_Function. @@ -5416,6 +5427,7 @@ int sqlite3NotPureFunc(sqlite3_context *pCtx){ } return 1; } +#endif /* SQLITE_OMIT_DATETIME_FUNCS */ #if defined(SQLITE_ENABLE_CURSOR_HINTS) && defined(SQLITE_DEBUG) /* @@ -5492,7 +5504,6 @@ void sqlite3VdbePreUpdateHook( i64 iKey2; PreUpdate preupdate; const char *zTbl = pTab->zName; - static const u8 fakeSortOrder = 0; #ifdef SQLITE_DEBUG int nRealCol; if( pTab->tabFlags & TF_WithoutRowid ){ @@ -5527,11 +5538,11 @@ void sqlite3VdbePreUpdateHook( preupdate.pCsr = pCsr; preupdate.op = op; preupdate.iNewReg = iReg; - preupdate.pKeyinfo = (KeyInfo*)&preupdate.keyinfoSpace; + preupdate.pKeyinfo = (KeyInfo*)&preupdate.uKey; preupdate.pKeyinfo->db = db; preupdate.pKeyinfo->enc = ENC(db); preupdate.pKeyinfo->nKeyField = pTab->nCol; - preupdate.pKeyinfo->aSortFlags = (u8*)&fakeSortOrder; + preupdate.pKeyinfo->aSortFlags = 0; /* Indicate .aColl, .nAllField uninit */ preupdate.iKey1 = iKey1; preupdate.iKey2 = iKey2; preupdate.pTab = pTab; @@ -5560,3 +5571,14 @@ void sqlite3VdbePreUpdateHook( } } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + +#ifdef SQLITE_ENABLE_PERCENTILE +/* +** Return the name of an SQL function associated with the sqlite3_context. +*/ +const char *sqlite3VdbeFuncName(const sqlite3_context *pCtx){ + assert( pCtx!=0 ); + assert( pCtx->pFunc!=0 ); + return pCtx->pFunc->zName; +} +#endif /* SQLITE_ENABLE_PERCENTILE */ diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 42edcf7de..a15fec6c4 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -385,7 +385,7 @@ static int blobReadWrite( int iOffset, int (*xCall)(BtCursor*, u32, u32, void*) ){ - int rc; + int rc = SQLITE_OK; Incrblob *p = (Incrblob *)pBlob; Vdbe *v; sqlite3 *db; @@ -425,17 +425,32 @@ static int blobReadWrite( ** using the incremental-blob API, this works. For the sessions module ** anyhow. */ - sqlite3_int64 iKey; - iKey = sqlite3BtreeIntegerKey(p->pCsr); - assert( v->apCsr[0]!=0 ); - assert( v->apCsr[0]->eCurType==CURTYPE_BTREE ); - sqlite3VdbePreUpdateHook( - v, v->apCsr[0], SQLITE_DELETE, p->zDb, p->pTab, iKey, -1, p->iCol - ); + if( sqlite3BtreeCursorIsValidNN(p->pCsr)==0 ){ + /* If the cursor is not currently valid, try to reseek it. This + ** always either fails or finds the correct row - the cursor will + ** have been marked permanently CURSOR_INVALID if the open row has + ** been deleted. */ + int bDiff = 0; + rc = sqlite3BtreeCursorRestore(p->pCsr, &bDiff); + assert( bDiff==0 || sqlite3BtreeCursorIsValidNN(p->pCsr)==0 ); + } + if( sqlite3BtreeCursorIsValidNN(p->pCsr) ){ + sqlite3_int64 iKey; + iKey = sqlite3BtreeIntegerKey(p->pCsr); + assert( v->apCsr[0]!=0 ); + assert( v->apCsr[0]->eCurType==CURTYPE_BTREE ); + sqlite3VdbePreUpdateHook( + v, v->apCsr[0], SQLITE_DELETE, p->zDb, p->pTab, iKey, -1, p->iCol + ); + } } + if( rc==SQLITE_OK ){ + rc = xCall(p->pCsr, iOffset+p->iOffset, n, z); + } +#else + rc = xCall(p->pCsr, iOffset+p->iOffset, n, z); #endif - rc = xCall(p->pCsr, iOffset+p->iOffset, n, z); sqlite3BtreeLeaveCursor(p->pCsr); if( rc==SQLITE_ABORT ){ sqlite3VdbeFinalize(v); diff --git a/src/vdbemem.c b/src/vdbemem.c index 6db9e4b1a..2c4d1c568 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1226,6 +1226,7 @@ int sqlite3VdbeMemSetStr( if( sqlite3VdbeMemClearAndResize(pMem, (int)MAX(nAlloc,32)) ){ return SQLITE_NOMEM_BKPT; } + assert( pMem->z!=0 ); memcpy(pMem->z, z, nAlloc); }else{ sqlite3VdbeMemRelease(pMem); diff --git a/src/vdbesort.c b/src/vdbesort.c index 6b1b4cff5..73f330111 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -302,6 +302,7 @@ struct SortSubtask { SorterCompare xCompare; /* Compare function to use */ SorterFile file; /* Temp file for level-0 PMAs */ SorterFile file2; /* Space for other PMAs */ + u64 nSpill; /* Total bytes written by this task */ }; @@ -422,6 +423,7 @@ struct PmaWriter { int iBufEnd; /* Last byte of buffer to write */ i64 iWriteOff; /* Offset of start of buffer in file */ sqlite3_file *pFd; /* File handle to write to */ + u64 nPmaSpill; /* Total number of bytes written */ }; /* @@ -766,7 +768,7 @@ static int vdbeSorterCompareTail( ){ UnpackedRecord *r2 = pTask->pUnpacked; if( *pbKey2Cached==0 ){ - sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2); + sqlite3VdbeRecordUnpack(nKey2, pKey2, r2); *pbKey2Cached = 1; } return sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, r2, 1); @@ -793,7 +795,7 @@ static int vdbeSorterCompare( ){ UnpackedRecord *r2 = pTask->pUnpacked; if( !*pbKey2Cached ){ - sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2); + sqlite3VdbeRecordUnpack(nKey2, pKey2, r2); *pbKey2Cached = 1; } return sqlite3VdbeRecordCompare(nKey1, pKey1, r2); @@ -833,6 +835,7 @@ static int vdbeSorterCompareText( ); } }else{ + assert( pTask->pSorter->pKeyInfo->aSortFlags!=0 ); assert( !(pTask->pSorter->pKeyInfo->aSortFlags[0]&KEYINFO_ORDER_BIGNULL) ); if( pTask->pSorter->pKeyInfo->aSortFlags[0] ){ res = res * -1; @@ -896,6 +899,7 @@ static int vdbeSorterCompareInt( } } + assert( pTask->pSorter->pKeyInfo->aSortFlags!=0 ); if( res==0 ){ if( pTask->pSorter->pKeyInfo->nKeyField>1 ){ res = vdbeSorterCompareTail( @@ -969,7 +973,8 @@ int sqlite3VdbeSorterInit( assert( pCsr->eCurType==CURTYPE_SORTER ); assert( sizeof(KeyInfo) + UMXV(pCsr->pKeyInfo->nKeyField)*sizeof(CollSeq*) < 0x7fffffff ); - szKeyInfo = SZ_KEYINFO(pCsr->pKeyInfo->nKeyField); + assert( pCsr->pKeyInfo->nKeyField<=pCsr->pKeyInfo->nAllField ); + szKeyInfo = SZ_KEYINFO(pCsr->pKeyInfo->nAllField); sz = SZ_VDBESORTER(nWorker+1); pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo); @@ -983,7 +988,12 @@ int sqlite3VdbeSorterInit( pKeyInfo->db = 0; if( nField && nWorker==0 ){ pKeyInfo->nKeyField = nField; + assert( nField<=pCsr->pKeyInfo->nAllField ); } + /* It is OK that pKeyInfo reuses the aSortFlags field from pCsr->pKeyInfo, + ** since the pCsr->pKeyInfo->aSortFlags[] array is invariant and lives + ** longer that pSorter. */ + assert( pKeyInfo->aSortFlags==pCsr->pKeyInfo->aSortFlags ); sqlite3BtreeEnter(pBt); pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(pBt); sqlite3BtreeLeave(pBt); @@ -1272,6 +1282,12 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ assert( pCsr->eCurType==CURTYPE_SORTER ); pSorter = pCsr->uc.pSorter; if( pSorter ){ + /* Increment db->nSpill by the total number of bytes of data written + ** to temp files by this sort operation. */ + int ii; + for(ii=0; iinTask; ii++){ + db->nSpill += pSorter->aTask[ii].nSpill; + } sqlite3VdbeSorterReset(db, pSorter); sqlite3_free(pSorter->list.aMemory); sqlite3DbFree(db, pSorter); @@ -1497,6 +1513,7 @@ static void vdbePmaWriteBlob(PmaWriter *p, u8 *pData, int nData){ &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, p->iWriteOff + p->iBufStart ); + p->nPmaSpill += (p->iBufEnd - p->iBufStart); p->iBufStart = p->iBufEnd = 0; p->iWriteOff += p->nBuffer; } @@ -1513,17 +1530,20 @@ static void vdbePmaWriteBlob(PmaWriter *p, u8 *pData, int nData){ ** required. Otherwise, return an SQLite error code. ** ** Before returning, set *piEof to the offset immediately following the -** last byte written to the file. +** last byte written to the file. Also, increment (*pnSpill) by the total +** number of bytes written to the file. */ -static int vdbePmaWriterFinish(PmaWriter *p, i64 *piEof){ +static int vdbePmaWriterFinish(PmaWriter *p, i64 *piEof, u64 *pnSpill){ int rc; if( p->eFWErr==0 && ALWAYS(p->aBuffer) && p->iBufEnd>p->iBufStart ){ p->eFWErr = sqlite3OsWrite(p->pFd, &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, p->iWriteOff + p->iBufStart ); + p->nPmaSpill += (p->iBufEnd - p->iBufStart); } *piEof = (p->iWriteOff + p->iBufEnd); + *pnSpill += p->nPmaSpill; sqlite3_free(p->aBuffer); rc = p->eFWErr; memset(p, 0, sizeof(PmaWriter)); @@ -1603,7 +1623,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ if( pList->aMemory==0 ) sqlite3_free(p); } pList->pList = p; - rc = vdbePmaWriterFinish(&writer, &pTask->file.iEof); + rc = vdbePmaWriterFinish(&writer, &pTask->file.iEof, &pTask->nSpill); } vdbeSorterWorkDebug(pTask, "exit"); @@ -1917,7 +1937,7 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ rc = vdbeMergeEngineStep(pIncr->pMerger, &dummy); } - rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof); + rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof, &pTask->nSpill); if( rc==SQLITE_OK ) rc = rc2; vdbeSorterPopulateDebug(pTask, "exit"); return rc; @@ -2763,7 +2783,7 @@ int sqlite3VdbeSorterCompare( assert( r2->nField==nKeyCol ); pKey = vdbeSorterRowkey(pSorter, &nKey); - sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, r2); + sqlite3VdbeRecordUnpack(nKey, pKey, r2); for(i=0; iaMem[i].flags & MEM_Null ){ *pRes = -1; diff --git a/src/vdbetrace.c b/src/vdbetrace.c index ae8ad3115..1a59f0e4d 100644 --- a/src/vdbetrace.c +++ b/src/vdbetrace.c @@ -26,10 +26,10 @@ ** a host parameter. If the text contains no host parameters, return ** the total number of bytes in the text. */ -static int findNextHostParameter(const char *zSql, int *pnToken){ +static i64 findNextHostParameter(const char *zSql, i64 *pnToken){ int tokenType; - int nTotal = 0; - int n; + i64 nTotal = 0; + i64 n; *pnToken = 0; while( zSql[0] ){ @@ -76,8 +76,8 @@ char *sqlite3VdbeExpandSql( sqlite3 *db; /* The database connection */ int idx = 0; /* Index of a host parameter */ int nextIndex = 1; /* Index of next ? host parameter */ - int n; /* Length of a token prefix */ - int nToken; /* Length of the parameter token */ + i64 n; /* Length of a token prefix */ + i64 nToken; /* Length of the parameter token */ int i; /* Loop counter */ Mem *pVar; /* Value of a host parameter */ StrAccum out; /* Accumulate the output here */ diff --git a/src/vxworks.h b/src/vxworks.h index e7013c3f6..3a95779d2 100644 --- a/src/vxworks.h +++ b/src/vxworks.h @@ -25,7 +25,9 @@ #define HAVE_UTIME 1 #else /* This is not VxWorks. */ -#define OS_VXWORKS 0 +#ifndef OS_VXWORKS +# define OS_VXWORKS 0 +#endif #define HAVE_FCHOWN 1 #define HAVE_READLINK 1 #define HAVE_LSTAT 1 diff --git a/src/wal.c b/src/wal.c index 41018b584..069852158 100644 --- a/src/wal.c +++ b/src/wal.c @@ -602,7 +602,7 @@ struct WalIterator { /* Size (in bytes) of a WalIterator object suitable for N or fewer segments */ #define SZ_WALITERATOR(N) \ - (offsetof(WalIterator,aSegment)*(N)*sizeof(struct WalSegment)) + (offsetof(WalIterator,aSegment)+(N)*sizeof(struct WalSegment)) /* ** Define the parameters of the hash tables in the wal-index file. There @@ -3488,7 +3488,7 @@ void sqlite3WalEndReadTransaction(Wal *pWal){ assert( pWal->writeLock==0 || pWal->readLock<0 ); #endif if( pWal->readLock>=0 ){ - sqlite3WalEndWriteTransaction(pWal); + (void)sqlite3WalEndWriteTransaction(pWal); walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); pWal->readLock = -1; } @@ -4297,7 +4297,8 @@ int sqlite3WalCheckpoint( /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked ** in the SQLITE_CHECKPOINT_PASSIVE mode. */ - assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); + assert( SQLITE_CHECKPOINT_NOOPSQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); if( pWal->readOnly ) return SQLITE_READONLY; WALTRACE(("WAL%p: checkpoint begins\n", pWal)); @@ -4314,31 +4315,35 @@ int sqlite3WalCheckpoint( ** EVIDENCE-OF: R-53820-33897 Even if there is a busy-handler configured, ** it will not be invoked in this case. */ - rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); - testcase( rc==SQLITE_BUSY ); - testcase( rc!=SQLITE_OK && xBusy2!=0 ); - if( rc==SQLITE_OK ){ - pWal->ckptLock = 1; + if( eMode!=SQLITE_CHECKPOINT_NOOP ){ + rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); + testcase( rc==SQLITE_BUSY ); + testcase( rc!=SQLITE_OK && xBusy2!=0 ); + if( rc==SQLITE_OK ){ + pWal->ckptLock = 1; - /* IMPLEMENTATION-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART and - ** TRUNCATE modes also obtain the exclusive "writer" lock on the database - ** file. - ** - ** EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained - ** immediately, and a busy-handler is configured, it is invoked and the - ** writer lock retried until either the busy-handler returns 0 or the - ** lock is successfully obtained. - */ - if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ - rc = walBusyLock(pWal, xBusy2, pBusyArg, WAL_WRITE_LOCK, 1); - if( rc==SQLITE_OK ){ - pWal->writeLock = 1; - }else if( rc==SQLITE_BUSY ){ - eMode2 = SQLITE_CHECKPOINT_PASSIVE; - xBusy2 = 0; - rc = SQLITE_OK; + /* IMPLEMENTATION-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART + ** and TRUNCATE modes also obtain the exclusive "writer" lock on the + ** database file. + ** + ** EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained + ** immediately, and a busy-handler is configured, it is invoked and the + ** writer lock retried until either the busy-handler returns 0 or the + ** lock is successfully obtained. + */ + if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ + rc = walBusyLock(pWal, xBusy2, pBusyArg, WAL_WRITE_LOCK, 1); + if( rc==SQLITE_OK ){ + pWal->writeLock = 1; + }else if( rc==SQLITE_BUSY ){ + eMode2 = SQLITE_CHECKPOINT_PASSIVE; + xBusy2 = 0; + rc = SQLITE_OK; + } } } + }else{ + rc = SQLITE_OK; } @@ -4352,7 +4357,7 @@ int sqlite3WalCheckpoint( ** immediately and do a partial checkpoint if it cannot obtain it. */ walDisableBlocking(pWal); rc = walIndexReadHdr(pWal, &isChanged); - if( eMode2!=SQLITE_CHECKPOINT_PASSIVE ) (void)walEnableBlocking(pWal); + if( eMode2>SQLITE_CHECKPOINT_PASSIVE ) (void)walEnableBlocking(pWal); if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ sqlite3OsUnfetch(pWal->pDbFd, 0, 0); } @@ -4362,7 +4367,7 @@ int sqlite3WalCheckpoint( if( rc==SQLITE_OK ){ if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ rc = SQLITE_CORRUPT_BKPT; - }else{ + }else if( eMode2!=SQLITE_CHECKPOINT_NOOP ){ rc = walCheckpoint(pWal, db, eMode2, xBusy2, pBusyArg, sync_flags,zBuf); } @@ -4390,7 +4395,7 @@ int sqlite3WalCheckpoint( sqlite3WalDb(pWal, 0); /* Release the locks. */ - sqlite3WalEndWriteTransaction(pWal); + (void)sqlite3WalEndWriteTransaction(pWal); if( pWal->ckptLock ){ walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); pWal->ckptLock = 0; diff --git a/src/where.c b/src/where.c index b0b4de424..b95f6d2a3 100644 --- a/src/where.c +++ b/src/where.c @@ -426,11 +426,11 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ pScan->pWC = pWC; pScan->k = k+1; #ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace & 0x20000 ){ + if( (sqlite3WhereTrace & 0x20000)!=0 && pScan->nEquiv>1 ){ int ii; - sqlite3DebugPrintf("SCAN-TERM %p: nEquiv=%d", - pTerm, pScan->nEquiv); - for(ii=0; iinEquiv; ii++){ + sqlite3DebugPrintf("EQUIVALENT TO {%d:%d} (due to TERM-%d):", + pScan->aiCur[0], pScan->aiColumn[0], pTerm->iTerm); + for(ii=1; iinEquiv; ii++){ sqlite3DebugPrintf(" {%d:%d}", pScan->aiCur[ii], pScan->aiColumn[ii]); } @@ -1201,7 +1201,9 @@ static SQLITE_NOINLINE void constructAutomaticIndex( VdbeCoverage(v); VdbeComment((v, "next row of %s", pSrc->pSTab->zName)); }else{ - addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); + assert( pLevel->addrHalt ); + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind,pLevel->iTabCur,pLevel->addrHalt); + VdbeCoverage(v); } if( pPartial ){ iContinue = sqlite3VdbeMakeLabel(pParse); @@ -1229,11 +1231,14 @@ static SQLITE_NOINLINE void constructAutomaticIndex( pSrc->u4.pSubq->regResult, pLevel->iIdxCur); sqlite3VdbeGoto(v, addrTop); pSrc->fg.viaCoroutine = 0; + sqlite3VdbeJumpHere(v, addrTop); }else{ sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); + if( (pSrc->fg.jointype & JT_LEFT)!=0 ){ + sqlite3VdbeJumpHere(v, addrTop); + } } - sqlite3VdbeJumpHere(v, addrTop); sqlite3ReleaseTempReg(pParse, regRecord); /* Jump here when skipping the initialization */ @@ -2385,6 +2390,7 @@ void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm){ }else{ sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d", pTerm->leftCursor); } + iTerm = pTerm->iTerm = MAX(iTerm,pTerm->iTerm); sqlite3DebugPrintf( "TERM-%-3d %p %s %-12s op=%03x wtFlags=%04x", iTerm, pTerm, zType, zLeft, pTerm->eOperator, pTerm->wtFlags); @@ -3526,6 +3532,7 @@ static int whereLoopAddBtreeIndex( && pProbe->hasStat1!=0 && OptimizationEnabled(db, SQLITE_SkipScan) && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ + && pSrc->fg.fromExists==0 && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK ){ LogEst nIter; @@ -5097,6 +5104,10 @@ static i8 wherePathSatisfiesOrderBy( && ((wctrlFlags&(WHERE_DISTINCTBY|WHERE_SORTBYGROUP))!=WHERE_DISTINCTBY) ){ obSat = obDone; + }else{ + /* No further ORDER BY terms may be matched. So this call should + ** return >=0, not -1. Clear isOrderDistinct to ensure it does so. */ + isOrderDistinct = 0; } break; } @@ -5842,8 +5853,15 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** mxChoice best-so-far paths. ** ** First look for an existing path among best-so-far paths - ** that covers the same set of loops and has the same isOrdered - ** setting as the current path candidate. + ** that: + ** (1) covers the same set of loops, and + ** (2) has a compatible isOrdered value. + ** + ** "Compatible isOrdered value" means either + ** (A) both have isOrdered==-1, or + ** (B) both have isOrder>=0, or + ** (C) ordering does not matter because this is the last round + ** of the solver. ** ** The term "((pTo->isOrdered^isOrdered)&0x80)==0" is equivalent ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range @@ -5852,7 +5870,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ testcase( nTo==0 ); for(jj=0, pTo=aTo; jjmaskLoop==maskNew - && ((pTo->isOrdered^isOrdered)&0x80)==0 + && ( ((pTo->isOrdered^isOrdered)&0x80)==0 || iLoop==nLoop-1 ) ){ testcase( jj==nTo-1 ); break; @@ -6007,11 +6025,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ return SQLITE_ERROR; } - /* Find the lowest cost path. pFrom will be left pointing to that path */ + /* Only one path is available, which is the best path */ + assert( nFrom==1 ); pFrom = aFrom; - for(ii=1; iirCost>aFrom[ii].rCost ) pFrom = &aFrom[ii]; - } + assert( pWInfo->nLevel==nLoop ); /* Load the lowest cost path into pWInfo */ for(iLoop=0; iLoopnLevel; i++){ WhereLoop *p = pWInfo->a[i].pWLoop; if( p==0 ) break; - if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 ) continue; + if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ + /* Treat a vtab scan as similar to a full-table scan */ + break; + } if( (p->wsFlags & (WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_IN))!=0 ){ u8 iTab = p->iTab; WhereLoop *pLoop; @@ -7082,6 +7102,14 @@ WhereInfo *sqlite3WhereBegin( pTab = pTabItem->pSTab; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); pLoop = pLevel->pWLoop; + pLevel->addrBrk = sqlite3VdbeMakeLabel(pParse); + if( ii==0 || (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){ + pLevel->addrHalt = pLevel->addrBrk; + }else if( pWInfo->a[ii-1].pRJ ){ + pLevel->addrHalt = pWInfo->a[ii-1].addrBrk; + }else{ + pLevel->addrHalt = pWInfo->a[ii-1].addrHalt; + } if( (pTab->tabFlags & TF_Ephemeral)!=0 || IsView(pTab) ){ /* Do nothing */ }else @@ -7133,6 +7161,13 @@ WhereInfo *sqlite3WhereBegin( sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0, (const u8*)&pTabItem->colUsed, P4_INT64); #endif + if( ii>=2 + && (pTabItem[0].fg.jointype & (JT_LTORJ|JT_LEFT))==0 + && pLevel->addrHalt==pWInfo->a[0].addrHalt + ){ + sqlite3VdbeAddOp2(v, OP_IfEmpty, pTabItem->iCursor, pWInfo->iBreak); + VdbeCoverage(v); + } }else{ sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); } @@ -7389,6 +7424,9 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2); } #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */ + if( pTabList->a[pLevel->iFrom].fg.fromExists ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+2); + } /* The common case: Advance to the next row */ if( pLevel->addrCont ) sqlite3VdbeResolveLabel(v, pLevel->addrCont); sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3); diff --git a/src/whereInt.h b/src/whereInt.h index 40a720ab9..09e02c8c7 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -75,6 +75,7 @@ struct WhereLevel { int iTabCur; /* The VDBE cursor used to access the table */ int iIdxCur; /* The VDBE cursor used to access pIdx */ int addrBrk; /* Jump here to break out of the loop */ + int addrHalt; /* Abort the query due to empty table or similar */ int addrNxt; /* Jump here to start the next IN combination */ int addrSkip; /* Jump here for next iteration of skip-scan */ int addrCont; /* Jump here to continue with the next loop cycle */ @@ -280,6 +281,9 @@ struct WhereTerm { u8 eMatchOp; /* Op for vtab MATCH/LIKE/GLOB/REGEXP terms */ int iParent; /* Disable pWC->a[iParent] when this term disabled */ int leftCursor; /* Cursor number of X in "X " */ +#ifdef SQLITE_DEBUG + int iTerm; /* Which WhereTerm is this, for debug purposes */ +#endif union { struct { int leftColumn; /* Column number of X in "X " */ diff --git a/src/wherecode.c b/src/wherecode.c index 823b4884e..1efa34a5d 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -110,7 +110,7 @@ static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop){ } /* -** This function sets the P4 value of an existing OP_Explain opcode to +** This function sets the P4 value of an existing OP_Explain opcode to ** text describing the loop in pLevel. If the OP_Explain opcode already has ** a P4 value, it is freed before it is overwritten. */ @@ -126,7 +126,6 @@ void sqlite3WhereAddExplainText( #endif { VdbeOp *pOp = sqlite3VdbeGetOp(pParse->pVdbe, addr); - SrcItem *pItem = &pTabList->a[pLevel->iFrom]; sqlite3 *db = pParse->db; /* Database handle */ int isSearch; /* True for a SEARCH. False for SCAN. */ @@ -149,7 +148,10 @@ void sqlite3WhereAddExplainText( sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); str.printfFlags = SQLITE_PRINTF_INTERNAL; - sqlite3_str_appendf(&str, "%s %S", isSearch ? "SEARCH" : "SCAN", pItem); + sqlite3_str_appendf(&str, "%s %S%s", + isSearch ? "SEARCH" : "SCAN", + pItem, + pItem->fg.fromExists ? " EXISTS" : ""); if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ const char *zFmt = 0; Index *pIdx; @@ -941,7 +943,7 @@ static int codeAllEqualityTerms( testcase( pIdx->aiColumn[j]==XN_EXPR ); VdbeComment((v, "%s", explainIndexColumnName(pIdx, j))); } - } + } /* Evaluate the equality constraints */ @@ -1282,7 +1284,7 @@ static void codeDeferredSeek( assert( iIdxCur>0 ); assert( pIdx->aiColumn[pIdx->nColumn-1]==-1 ); - + pWInfo->bDeferredSeek = 1; sqlite3VdbeAddOp3(v, OP_DeferredSeek, iIdxCur, 0, iCur); if( (pWInfo->wctrlFlags & (WHERE_OR_SUBCLAUSE|WHERE_RIGHT_JOIN)) @@ -1393,6 +1395,7 @@ static SQLITE_NOINLINE void filterPullDown( int addrNxt, /* Jump here to bypass inner loops */ Bitmask notReady /* Loops that are not ready */ ){ + int saved_addrBrk; while( ++iLevel < pWInfo->nLevel ){ WhereLevel *pLevel = &pWInfo->a[iLevel]; WhereLoop *pLoop = pLevel->pWLoop; @@ -1401,7 +1404,7 @@ static SQLITE_NOINLINE void filterPullDown( /* ,--- Because sqlite3ConstructBloomFilter() has will not have set ** vvvvv--' pLevel->regFilter if this were true. */ if( NEVER(pLoop->prereq & notReady) ) continue; - assert( pLevel->addrBrk==0 ); + saved_addrBrk = pLevel->addrBrk; pLevel->addrBrk = addrNxt; if( pLoop->wsFlags & WHERE_IPK ){ WhereTerm *pTerm = pLoop->aLTerm[0]; @@ -1431,19 +1434,19 @@ static SQLITE_NOINLINE void filterPullDown( VdbeCoverage(pParse->pVdbe); } pLevel->regFilter = 0; - pLevel->addrBrk = 0; + pLevel->addrBrk = saved_addrBrk; } } /* -** Loop pLoop is a WHERE_INDEXED level that uses at least one IN(...) -** operator. Return true if level pLoop is guaranteed to visit only one +** Loop pLoop is a WHERE_INDEXED level that uses at least one IN(...) +** operator. Return true if level pLoop is guaranteed to visit only one ** row for each key generated for the index. */ static int whereLoopIsOneRow(WhereLoop *pLoop){ - if( pLoop->u.btree.pIndex->onError - && pLoop->nSkip==0 - && pLoop->u.btree.nEq==pLoop->u.btree.pIndex->nKeyCol + if( pLoop->u.btree.pIndex->onError + && pLoop->nSkip==0 + && pLoop->u.btree.nEq==pLoop->u.btree.pIndex->nKeyCol ){ int ii; for(ii=0; iiu.btree.nEq; ii++){ @@ -1478,7 +1481,6 @@ Bitmask sqlite3WhereCodeOneLoopStart( sqlite3 *db; /* Database connection */ SrcItem *pTabItem; /* FROM clause term being coded */ int addrBrk; /* Jump here to break out of the loop */ - int addrHalt; /* addrBrk for the outermost loop */ int addrCont; /* Jump here to continue with next cycle */ int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ int iReleaseReg = 0; /* Temp register to free before returning */ @@ -1522,7 +1524,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( ** there are no IN operators in the constraints, the "addrNxt" label ** is the same as "addrBrk". */ - addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(pParse); + addrBrk = pLevel->addrNxt = pLevel->addrBrk; addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(pParse); /* If this is the right table of a LEFT OUTER JOIN, allocate and @@ -1538,14 +1540,6 @@ Bitmask sqlite3WhereCodeOneLoopStart( VdbeComment((v, "init LEFT JOIN match flag")); } - /* Compute a safe address to jump to if we discover that the table for - ** this loop is empty and can never contribute content. */ - for(j=iLevel; j>0; j--){ - if( pWInfo->a[j].iLeftJoin ) break; - if( pWInfo->a[j].pRJ ) break; - } - addrHalt = pWInfo->a[j].addrBrk; - /* Special case of a FROM clause subquery implemented as a co-routine */ if( pTabItem->fg.viaCoroutine ){ int regYield; @@ -1784,7 +1778,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( VdbeCoverageIf(v, pX->op==TK_GE); sqlite3ReleaseTempReg(pParse, rTemp); }else{ - sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrHalt); + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, pLevel->addrHalt); VdbeCoverageIf(v, bRev==0); VdbeCoverageIf(v, bRev!=0); } @@ -1824,37 +1818,37 @@ Bitmask sqlite3WhereCodeOneLoopStart( sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL); } }else if( pLoop->wsFlags & WHERE_INDEXED ){ - /* Case 4: A scan using an index. + /* Case 4: Search using an index. ** - ** The WHERE clause may contain zero or more equality - ** terms ("==" or "IN" operators) that refer to the N - ** left-most columns of the index. It may also contain - ** inequality constraints (>, <, >= or <=) on the indexed - ** column that immediately follows the N equalities. Only - ** the right-most column can be an inequality - the rest must - ** use the "==" and "IN" operators. For example, if the - ** index is on (x,y,z), then the following clauses are all - ** optimized: + ** The WHERE clause may contain zero or more equality + ** terms ("==" or "IN" or "IS" operators) that refer to the N + ** left-most columns of the index. It may also contain + ** inequality constraints (>, <, >= or <=) on the indexed + ** column that immediately follows the N equalities. Only + ** the right-most column can be an inequality - the rest must + ** use the "==", "IN", or "IS" operators. For example, if the + ** index is on (x,y,z), then the following clauses are all + ** optimized: ** - ** x=5 - ** x=5 AND y=10 - ** x=5 AND y<10 - ** x=5 AND y>5 AND y<10 - ** x=5 AND y=5 AND z<=10 + ** x=5 + ** x=5 AND y=10 + ** x=5 AND y<10 + ** x=5 AND y>5 AND y<10 + ** x=5 AND y=5 AND z<=10 ** - ** The z<10 term of the following cannot be used, only - ** the x=5 term: + ** The z<10 term of the following cannot be used, only + ** the x=5 term: ** - ** x=5 AND z<10 + ** x=5 AND z<10 ** - ** N may be zero if there are inequality constraints. - ** If there are no inequality constraints, then N is at - ** least one. + ** N may be zero if there are inequality constraints. + ** If there are no inequality constraints, then N is at + ** least one. ** - ** This case is also used when there are no WHERE clause - ** constraints but an index is selected anyway, in order - ** to force the output order to conform to an ORDER BY. - */ + ** This case is also used when there are no WHERE clause + ** constraints but an index is selected anyway, in order + ** to force the output order to conform to an ORDER BY. + */ static const u8 aStartOp[] = { 0, 0, @@ -2008,7 +2002,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( } if( zStartAff ){ updateRangeAffinityStr(pRight, nBtm, &zStartAff[nEq]); - } + } nConstraint += nBtm; testcase( pRangeStart->wtFlags & TERM_VIRTUAL ); if( sqlite3ExprIsVector(pRight)==0 ){ @@ -2210,7 +2204,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( ** a LEFT JOIN: */ assert( (pWInfo->wctrlFlags & (WHERE_OR_SUBCLAUSE|WHERE_RIGHT_JOIN))==0 ); } - + /* Record the instruction used to terminate the loop. */ if( (pLoop->wsFlags & WHERE_ONEROW) || (pLevel->u.in.nIn && regBignull==0 && whereLoopIsOneRow(pLoop)) @@ -2579,7 +2573,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( codeCursorHint(pTabItem, pWInfo, pLevel, 0); pLevel->op = aStep[bRev]; pLevel->p1 = iCur; - pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrHalt); + pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev],iCur,pLevel->addrHalt); VdbeCoverageIf(v, bRev==0); VdbeCoverageIf(v, bRev!=0); pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; @@ -2599,7 +2593,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( ** ** iLoop==1: Code only expressions that are entirely covered by pIdx. ** iLoop==2: Code remaining expressions that do not contain correlated - ** sub-queries. + ** sub-queries. ** iLoop==3: Code all remaining expressions. ** ** An effort is made to skip unnecessary iterations of the loop. @@ -2851,7 +2845,10 @@ SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( WhereLoop *pLoop = pLevel->pWLoop; SrcItem *pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; SrcList *pFrom; - u8 fromSpace[SZ_SRCLIST_1]; + union { + SrcList sSrc; + u8 fromSpace[SZ_SRCLIST_1]; + } uSrc; Bitmask mAll = 0; int k; @@ -2870,7 +2867,7 @@ SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( pSubq = pRight->u4.pSubq; assert( pSubq->pSelect!=0 && pSubq->pSelect->pEList!=0 ); sqlite3VdbeAddOp3( - v, OP_Null, 0, pSubq->regResult, + v, OP_Null, 0, pSubq->regResult, pSubq->regResult + pSubq->pSelect->pEList->nExpr-1 ); } @@ -2895,7 +2892,7 @@ SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( sqlite3ExprDup(pParse->db, pTerm->pExpr, 0)); } } - pFrom = (SrcList*)fromSpace; + pFrom = &uSrc.sSrc; pFrom->nSrc = 1; pFrom->nAlloc = 1; memcpy(&pFrom->a[0], pTabItem, sizeof(SrcItem)); diff --git a/src/whereexpr.c b/src/whereexpr.c index 26bcae71c..0d99ca85e 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -948,7 +948,7 @@ static int termIsEquivalence(Parse *pParse, Expr *pExpr, SrcList *pSrc){ if( ExprHasProperty(pExpr, EP_OuterON) ) return 0; /* (3) */ assert( pSrc!=0 ); if( pExpr->op==TK_IS - && pSrc->nSrc + && pSrc->nSrc>=2 && (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ return 0; /* (4) */ @@ -1124,6 +1124,9 @@ static void exprAnalyze( } assert( pWC->nTerm > idxTerm ); pTerm = &pWC->a[idxTerm]; +#ifdef SQLITE_DEBUG + pTerm->iTerm = idxTerm; +#endif pMaskSet = &pWInfo->sMaskSet; pExpr = pTerm->pExpr; assert( pExpr!=0 ); /* Because malloc() has not failed */ @@ -1167,21 +1170,7 @@ static void exprAnalyze( prereqAll |= x; extraRight = x-1; /* ON clause terms may not be used with an index ** on left table of a LEFT JOIN. Ticket #3015 */ - if( (prereqAll>>1)>=x ){ - sqlite3ErrorMsg(pParse, "ON clause references tables to its right"); - return; - } }else if( (prereqAll>>1)>=x ){ - /* The ON clause of an INNER JOIN references a table to its right. - ** Most other SQL database engines raise an error. But SQLite versions - ** 3.0 through 3.38 just put the ON clause constraint into the WHERE - ** clause and carried on. Beginning with 3.39, raise an error only - ** if there is a RIGHT or FULL JOIN in the query. This makes SQLite - ** more like other systems, and also preserves legacy. */ - if( ALWAYS(pSrc->nSrc>0) && (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ - sqlite3ErrorMsg(pParse, "ON clause references tables to its right"); - return; - } ExprClearProperty(pExpr, EP_InnerON); } } @@ -1538,7 +1527,7 @@ static void exprAnalyze( idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew==0 ); pNewTerm = &pWC->a[idxNew]; - pNewTerm->prereqRight = prereqExpr; + pNewTerm->prereqRight = prereqExpr | extraRight; pNewTerm->leftCursor = pLeft->iTable; pNewTerm->u.x.leftColumn = pLeft->iColumn; pNewTerm->eOperator = WO_AUX; @@ -1649,7 +1638,7 @@ static void whereAddLimitExpr( ** ** 1. The SELECT statement has a LIMIT clause, and ** 2. The SELECT statement is not an aggregate or DISTINCT query, and -** 3. The SELECT statement has exactly one object in its from clause, and +** 3. The SELECT statement has exactly one object in its FROM clause, and ** that object is a virtual table, and ** 4. There are no terms in the WHERE clause that will not be passed ** to the virtual table xBestIndex method. @@ -1686,8 +1675,22 @@ void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Select *p){ ** (leftCursor==iCsr) test below. */ continue; } - if( pWC->a[ii].leftCursor!=iCsr ) return; - if( pWC->a[ii].prereqRight!=0 ) return; + if( pWC->a[ii].leftCursor==iCsr && pWC->a[ii].prereqRight==0 ) continue; + + /* If this term has a parent with exactly one child, and the parent will + ** be passed through to xBestIndex, then this term can be ignored. */ + if( pWC->a[ii].iParent>=0 ){ + WhereTerm *pParent = &pWC->a[ pWC->a[ii].iParent ]; + if( pParent->leftCursor==iCsr + && pParent->prereqRight==0 + && pParent->nChild==1 + ){ + continue; + } + } + + /* This term will not be passed through. Do not add a LIMIT clause. */ + return; } /* Check condition (5). Return early if it is not met. */ diff --git a/src/window.c b/src/window.c index 948b0728a..1f22ab194 100644 --- a/src/window.c +++ b/src/window.c @@ -2553,7 +2553,7 @@ static int windowExprGtZero(Parse *pParse, Expr *pExpr){ ** ** ROWS BETWEEN FOLLOWING AND FOLLOWING ** -** ... loop started by sqlite3WhereBegin() ... +** ... loop started by sqlite3WhereBegin() ... ** if( new partition ){ ** Gosub flush ** } @@ -3071,6 +3071,12 @@ void sqlite3WindowCodeStep( addrBreak2 = windowCodeOp(&s, WINDOW_AGGINVERSE, 0, 1); }else{ assert( pMWin->eEnd==TK_FOLLOWING ); + /* assert( regStart>=0 ); + ** regEnd = regEnd - regStart; + ** regStart = 0; */ + sqlite3VdbeAddOp3(v, OP_Subtract, regStart, regEnd, regEnd); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regStart); + addrStart = sqlite3VdbeCurrentAddr(v); addrBreak1 = windowCodeOp(&s, WINDOW_RETURN_ROW, regEnd, 1); addrBreak2 = windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 1); diff --git a/test/aggorderby.test b/test/aggorderby.test index eed1f83a7..466074815 100644 --- a/test/aggorderby.test +++ b/test/aggorderby.test @@ -158,5 +158,17 @@ do_execsql_test aggorderby-9.3 { SELECT json_group_array(DISTINCT json(x) ORDER BY json(x)) FROM c; } {{[[1,1],[4,4],{"a":3},{"x":2}]}} +#------------------------------------------------------------------------- +reset_db +do_execsql_test aggorderby-10.0 { + CREATE TABLE t1(w, x); + INSERT INTO t1 VALUES(1, 2); +} + +for {set i 0} {$i < 70000} {incr i} { lappend lExpr x } +do_catchsql_test aggorderby-10.1 " + SELECT group_concat(w ORDER BY [join $lExpr ,]) FROM t1 +" {1 {too many terms in ORDER BY clause}} + finish_test diff --git a/test/basexx1.test b/test/basexx1.test index 947a5678f..b34b25ff3 100644 --- a/test/basexx1.test +++ b/test/basexx1.test @@ -39,6 +39,12 @@ do_execsql_test 102 { } {AAECAw== }} +# Buffer size testing +do_execsql_test 102-b { + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c wHERE n<5000) + SELECT sum(length(base64(randomblob(n)))) FROM c; +} {16910656} + # Basic base64 decoding with pad chars do_execsql_test 103 { SELECT hex(base64('AAECAwQF')); @@ -152,4 +158,11 @@ do_execsql_test 117 { SELECT num FROM bs WHERE base85(base85(b))!=b; } {} +do_catchsql_test 118 { + SELECT base64(zeroblob(2000_000_000)) +} {/1.*too big.*/} +do_catchsql_test 119 { + SELECT base85(zeroblob(2000_000_000)) +} {/1.*too big.*/} + finish_test diff --git a/test/bestindexC.test b/test/bestindexC.test index 48f3a2765..8b96a19e6 100644 --- a/test/bestindexC.test +++ b/test/bestindexC.test @@ -349,4 +349,78 @@ do_execsql_test 5.9 { three six seven } +#-------------------------------------------------------------------------- + +reset_db +register_tcl_module db + +proc quote {str} { + return "'[string map {' ''} $str]'" +} + +proc vtab_command {lVal method args} { + switch -- $method { + xConnect { + return "CREATE TABLE t1(a, b, c, d)" + } + + xBestIndex { + set hdl [lindex $args 0] + set clist [$hdl constraints] + + set idx 0 + set idxnum 0 + + foreach c $clist { + array set a $c + if {$a(usable)==0} continue + + if {$a(op)=="limit"} { + set idxnum [$hdl rhs_value $idx 555] + } + + incr idx + } + + return "cost 1000 rows 1000 idxnum $idxnum" + + } + + xFilter { + foreach {idxnum idxstr lArg} $args {} + return [list sql "SELECT 0, $idxnum, $idxnum, $idxnum, $idxnum"] + } + } + + return {} +} + +do_execsql_test 6.0 { + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(2, 2); + CREATE VIRTUAL TABLE x1 USING tcl(vtab_command t1); +} + +do_execsql_test 6.1 { SELECT * FROM x1 LIMIT 50 } {50 50 50 50} + +do_execsql_test 6.2 { SELECT * FROM x1 WHERE b=c LIMIT 5 } {0 0 0 0} + +do_execsql_test 6.3 { + SELECT (SELECT a FROM x1 WHERE t1.x=t1.y LIMIT 10) FROM t1 +} {0} + +do_execsql_test 6.4 { + SELECT (SELECT a FROM x1 WHERE x1.a=1) FROM t1 +} {1} + +do_execsql_test 6.5 { + SELECT (SELECT a FROM x1 WHERE x1.a=1 LIMIT 1) FROM t1 +} {1} + +do_execsql_test 6.6 { + SELECT (SELECT a FROM x1 WHERE x1.a=555 LIMIT 2) FROM t1 +} {555} + finish_test + + diff --git a/test/bestindexE.test b/test/bestindexE.test new file mode 100644 index 000000000..48397fc7b --- /dev/null +++ b/test/bestindexE.test @@ -0,0 +1,130 @@ +# 2025-09-02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix bestindex1 + +ifcapable !vtab { + finish_test + return +} + +register_tcl_module db + +proc pretty_constraint {lCol cons} { + array set C $cons + + set ret "" + if {$C(usable)} { + set OP(eq) = + set ret "[lindex $lCol $C(column)]$OP($C(op))?" + } + + return $ret +} + +proc vtab_command {zName lCol method args} { + switch -- $method { + xConnect { + return "CREATE TABLE $zName ([join $lCol ,]) " + } + + xBestIndex { + set hdl [lindex $args 0] + set conslist [$hdl constraints] + + set ret [list] + foreach cons $conslist { + set pretty [pretty_constraint $lCol $cons] + if {$pretty != ""} { lappend ret $pretty } + } + + lappend ::xBestIndex "$zName: [join $ret { AND }]" + + return "cost 1000 rows 1000 idxnum 555" + } + } + + return {} +} + +proc do_bestindex_test {tn sql lCons} { + set ::xBestIndex [list] + do_execsql_test $tn.1 $sql + uplevel [list do_test $tn.2 [list set ::xBestIndex] [list {*}$lCons]] +} + +proc create_vtab {tname clist} { + set cmd [list vtab_command $tname $clist] + execsql " + CREATE VIRTUAL TABLE $tname USING tcl('$cmd') + " +} + +do_test 1.0 { + create_vtab x1 {a b c} +} {} + +do_bestindex_test 1.1 { + SELECT * FROM x1 WHERE a=? +} {{x1: a=?}} + +do_bestindex_test 1.2 { + SELECT * FROM x1 WHERE a=? AND b=? +} {{x1: a=? AND b=?}} + +#-------------------------------------------------------------------------- +reset_db +register_tcl_module db + +do_test 2.0 { + create_vtab Delivery {id customer} + create_vtab ReturnDelivery {id customer} + create_vtab Customer {oid name} +} {} + +do_bestindex_test 2.1 { + SELECT Delivery.ID, Customer.Name + FROM Delivery LEFT JOIN + Customer ON Delivery.Customer = Customer.OID +} { + {Delivery: } + {Customer: oid=?} +} + +do_bestindex_test 2.2 { + SELECT * FROM + ( + SELECT Delivery.ID, Customer.Name + FROM Delivery LEFT JOIN + Customer ON Delivery.Customer = Customer.OID + + UNION + + SELECT ReturnDelivery.ID, Customer.Name + FROM ReturnDelivery LEFT JOIN + Customer ON ReturnDelivery.Customer = Customer.OID + ) + WHERE ID = 1 +} { + {Delivery: id=?} + {Customer: oid=?} + {ReturnDelivery: id=?} + {Customer: oid=?} +} + + + + +finish_test diff --git a/test/between.test b/test/between.test index 16c3913d1..5e02ef9b2 100644 --- a/test/between.test +++ b/test/between.test @@ -140,4 +140,21 @@ foreach {tn expr res} { do_execsql_test between-2.1.$tn $sql $res } +#------------------------------------------------------------------------- +reset_db +do_execsql_test between-3.0 { + CREATE TABLE t1(x, y); + CREATE INDEX i1 ON t1(x); + INSERT INTO t1 VALUES(4, 4); + CREATE TABLE t2(a, b); +} + +do_execsql_test between-3.1 { + SELECT * FROM t1 LEFT JOIN t2 ON (x BETWEEN 1 AND 3); +} {4 4 {} {}} + +do_execsql_test between-3.2 { + SELECT * FROM t1 LEFT JOIN t2 ON (x BETWEEN 5 AND 7); +} {4 4 {} {}} + finish_test diff --git a/test/bigrow.test b/test/bigrow.test index fa59c3625..83270cb76 100644 --- a/test/bigrow.test +++ b/test/bigrow.test @@ -220,4 +220,15 @@ do_test bigrow-5.99 { execsql {DROP TABLE t1} } {} +if {$SQLITE_MAX_LENGTH > 1200*1000*1000} { + # If SQLITE_MAX_LENGTH is large enough, check that a serial type greater + # than 0x7FFFFFFF does not cause trouble. + set v [string repeat A 1100000000] + do_execsql_test bigrow-6.0 { + CREATE TABLE docs(content TEXT); + INSERT INTO docs VALUES ( $v ); + } + do_execsql_test bigrow-6.1 { SELECT content FROM docs } [list $v] +} + finish_test diff --git a/test/carray01.test b/test/carray01.test index 1af9cb66e..86ea06996 100644 --- a/test/carray01.test +++ b/test/carray01.test @@ -20,7 +20,7 @@ ifcapable !vtab { finish_test return } -load_static_extension db carray +#load_static_extension db carray # Parameter $stmt must be a prepared statement created using # the sqlite3_prepare_v2 command and with parameters fully bound. diff --git a/test/carray02.test b/test/carray02.test new file mode 100644 index 000000000..c75ca4271 --- /dev/null +++ b/test/carray02.test @@ -0,0 +1,162 @@ +# 2025-10-07 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests for CARRAY extension +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix carray02 + +ifcapable !vtab { + finish_test + return +} + +proc run_stmt {stmt} { + set r {} + while {[sqlite3_step $stmt]=="SQLITE_ROW"} { + for {set i 0} {$i<[sqlite3_data_count $stmt]} {incr i} { + lappend r [sqlite3_column_text $stmt $i] + } + } + sqlite3_reset $stmt + return $r +} + +# Bind some text values using sqlite3_carray_bind(). Including a NULL pointer. +# +set STMT [ + sqlite3_prepare_v2 db "SELECT value, value IS NULL FROM carray(?)" -1 TAIL +] +do_test 1.0 { + sqlite3_carray_bind -transient -text $STMT 1 "abc" NULL "def" + run_stmt $STMT +} {abc 0 {} 1 def 0} +sqlite3_finalize $STMT + +# Pass bad values as the "flags" parameter to sqlite3_carray_bind(). +# +set STMT [sqlite3_prepare_v2 db "SELECT value FROM carray(?)" -1 TAIL] +do_test 2.0 { + list [catch {sqlite3_carray_bind -flags 5 -int32 $STMT 1 1 2 3 4} msg] $msg +} {1 {SQL logic error}} +do_test 2.1 { + list [catch {sqlite3_carray_bind -flags -1 -int32 $STMT 1 1 2 3 4} msg] $msg +} {1 {SQL logic error}} +sqlite3_finalize $STMT + +# In each of the following, carray(?) contains integer values 1 to 5, bound +# using sqlite3_carray_bind(). +# +foreach {tn sql res} { + 1 { SELECT value FROM carray(?) WHERE value>2 } {3 4 5} + 2 { + WITH s(i) AS ( VALUES(1) UNION ALL VALUES(2) ) + SELECT i, value FROM s, carray(?) WHERE i=value; + } {1 1 2 2} +} { + do_test 2.2.$tn { + set STMT [sqlite3_prepare_v2 db $sql -1 TAIL] + sqlite3_carray_bind -int32 $STMT 1 1 2 3 4 5 + set r [run_stmt $STMT] + sqlite3_finalize $STMT + set r + } $res +} + +# In each of the following, ? is bound to an int array containing 1 to 5. +# Bound using C code like: +# +# sqlite3_bind_pointer(pStmt, 1, aInt, "carray", SQLITE_TRANSIENT) +# +foreach {tn sql res} { + 1 { SELECT value FROM carray(?, 5) } {1 2 3 4 5} + 2 { SELECT value FROM carray(?, 3, 'int32') } {1 2 3} + 3 { SELECT value, pointer, count, ctype FROM carray(?, 5, 'int32') } + {1 {} 5 int32 2 {} 5 int32 3 {} 5 int32 4 {} 5 int32 5 {} 5 int32} + 4 { SELECT rowid, value FROM carray(?, 5, 'int32') } + {1 1 2 2 3 3 4 4 5 5} +} { + do_test 2.3.$tn { + set STMT [sqlite3_prepare_v2 db $sql -1 TAIL] + bind_carray_intptr $STMT 1 1 2 3 4 5 + set r [run_stmt $STMT] + sqlite3_finalize $STMT + set r + } $res +} + +# In each of the following. Both carray(?1) and carray(?2) contain integer +# values 1 to 5. Bound by sqlite3_carray_bind(). +# +foreach {tn sql res} { + 1 { + SELECT * FROM carray(?1) AS a, carray(?2) AS b + WHERE a.value=b.value + } {1 1 2 2 3 3 4 4 5 5} + + 2 { + SELECT * FROM carray(?1) AS a, carray(?2) AS b + WHERE a.value=b.value AND a.value<3 AND b.value<3 + } {1 1 2 2 3 3} + + 3 { + SELECT * FROM carray(?1) AS a, carray(?2) AS b + WHERE a.value<3 AND b.value<3 AND a.value=b.value + } {1 1 2 2 3 3} + + 4 { + SELECT * FROM carray(?1) AS a, carray(?2, a.value) AS b + WHERE a.value=b.value + } {1 1 2 2 3 3} + +} { + do_test 2.4.$tn { + set STMT [sqlite3_prepare_v2 db { + SELECT * FROM carray(?1) AS a, carray(?2) AS b WHERE a.value=b.value + } -1 TAIL] + sqlite3_carray_bind $STMT 1 1 2 3 4 5 + sqlite3_carray_bind $STMT 2 1 2 3 4 5 + set r [run_stmt $STMT] + sqlite3_finalize $STMT + set r + } {1 1 2 2 3 3 4 4 5 5} +} + +# Test that not binding any pointer, or passing a value that is not a bound +# pointer to carray() produces no rows of output. +# +do_execsql_test 3.0.0 { + SELECT * FROM carray +} {} +do_execsql_test 3.0.1 { + SELECT * FROM carray('0xFFFF', 5) +} {} +do_execsql_test 3.0.2 { + SELECT * FROM carray('0xFFFF') +} {} +do_execsql_test 3.0.3 { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES('0xFFFF'); + SELECT * FROM t1, carray WHERE carray.pointer = t1.x; +} {} + +# Test passing an unknown datatype. +# +do_test 3.1 { + set STMT [sqlite3_prepare_v2 db {SELECT * FROM carray(?, 5, 'apples')} -1 T] + sqlite3_carray_bind $STMT 1 1 2 3 4 5 + sqlite3_step $STMT + list [sqlite3_finalize $STMT] [sqlite3_errmsg db] +} {SQLITE_ERROR {unknown datatype: 'apples'}} + +finish_test diff --git a/test/carrayfault.test b/test/carrayfault.test new file mode 100644 index 000000000..0dd2cd21b --- /dev/null +++ b/test/carrayfault.test @@ -0,0 +1,87 @@ +# 2025-10-07 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix carrayfault + +ifcapable {!carray||!vtab} { + finish_test; return +} + +sqlite3 db test.db + +do_execsql_test 2.0 { + CREATE TABLE t1(a); +} + +set STMT [sqlite3_prepare_v2 db "SELECT value FROM carray(?)" -1 TAIL] + +# Test OOM faults in sqlite3_carray_bind() when binding an integer +# array. +# +foreach {tn mem} { + 1 -static + 2 -transient + 3 -malloc +} { + do_faultsim_test 2.$tn -faults oom* -prep { + } -body { + sqlite3_carray_bind $::mem -int64 $::STMT 1 100 200 300 400 + } -test { + faultsim_test_result {0 {}} {1 {out of memory}} + } +} + +# Test OOM faults in sqlite3_carray_bind() when binding a text array. +# +do_faultsim_test 3 -faults oom* -prep { +} -body { + sqlite3_carray_bind -transient -text $::STMT 1 "hello" "world" +} -test { + faultsim_test_result {0 {}} {1 {initialization of carray failed: }} +} + +sqlite3_finalize $STMT + + +# Test OOM faults when running queries against carray. +# +do_faultsim_test 4 -faults oom* -prep { + set ::STMT [sqlite3_prepare_v2 db "SELECT value FROM carray(?)" -1 TAIL] + sqlite3_carray_bind -transient -text $::STMT 1 "hello" "world" +} -body { + set myres [list] + while { "SQLITE_ROW"==[sqlite3_step $::STMT] } { + lappend myres [sqlite3_column_text $::STMT 1] + } + sqlite3_reset $::STMT +} -test { + faultsim_test_result {0 SQLITE_OK} {0 SQLITE_NOMEM} + sqlite3_finalize $::STMT +} + +# Test OOM faults when preparing queries against carray. +# +do_faultsim_test 5 -faults oom* -prep { + sqlite3 db test.db +} -body { + execsql "SELECT value FROM carray(?)" +} -test { + faultsim_test_result {0 {}} +} + + + +sqlite3_carray_bind + +finish_test diff --git a/test/cksumvfs.test b/test/cksumvfs.test index 8c7bcf551..2bf4d9187 100644 --- a/test/cksumvfs.test +++ b/test/cksumvfs.test @@ -14,20 +14,80 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix cksumvfs -sqlite3_register_cksumvfs + db close +sqlite3_shutdown +#test_sqlite3_log logfunc +sqlite3_initialize + +proc logfunc {args} { + puts "LOG: $args" +} + +sqlite3_register_cksumvfs sqlite3 db test.db + file_control_reservebytes db 8 +execsql { + PRAGMA page_size = 4096; +} set text [db one "SELECT hex(randomblob(5000))"] do_execsql_test 1.0 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b); - INSERT INTO t1 VALUES(1, $text); + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, $text, NULL); } do_execsql_test 1.1 { SELECT * FROM t1; -} [list 1 $text] +} [list 1 $text {}] + +do_execsql_test 1.2 { + DELETE FROM t1; +} + +do_test 1.3 { + execsql BEGIN + for {set ii 1500} {$ii < 10000} {incr ii} { + execsql { INSERT INTO t1 VALUES(NULL, randomblob(5000), randomblob($ii)) } + } + execsql COMMIT +} {} + +do_execsql_test 1.4 { + SELECT count(b) FROM t1 +} {8500} + +do_execsql_test 1.5 { + PRAGMA journal_mode = wal; + DELETE FROM t1; +} {wal} + +do_execsql_test 1.6 { + PRAGMA wal_checkpoint; +} {/0 # #/} + +do_execsql_test 1.7 { + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO t1 SELECT NULL, randomblob(5000), randomblob(i) FROM s; + SELECT count(*) FROM t1; +} {100} + +db_save_and_close +db_restore_and_reopen + +do_execsql_test 1.8 { + SELECT count(*) FROM t1; +} {100} + +db close +sqlite3 db test.db + +do_execsql_test 1.9 { + SELECT count(*) FROM t1; +} {100} finish_test diff --git a/test/date.test b/test/date.test index 8badccf3f..5ff131f73 100644 --- a/test/date.test +++ b/test/date.test @@ -325,9 +325,29 @@ utc_to_local 6.24 {3000-10-30 11:30:00} {3000-10-30 12:00:00} # timezone extension, then the time will already be UTC and subsequent # 'utc' modifiers are no-ops. # -do_execsql_test date-6.25 { +# Forum post 2025-09-17T10:12:14 - The "+00:00" suffix should work like "Z" +# +do_execsql_test date-6.25.1 { SELECT datetime('2000-10-29 12:00Z','utc','utc'); } {{2000-10-29 12:00:00}} +do_execsql_test date-6.25.2 { + SELECT datetime('2000-10-29 12:00 +00:00','utc','utc'); +} {{2000-10-29 12:00:00}} +do_execsql_test date-6.25.3 { + SELECT datetime('2000-10-29 12:00+00:00','utc','utc'); +} {{2000-10-29 12:00:00}} +do_execsql_test date-6.25.4 { + SELECT datetime('2000-10-29 12:00:00+00:00','utc','utc'); +} {{2000-10-29 12:00:00}} +do_execsql_test date-6.25.5 { + SELECT datetime('2000-10-29 12:00 -00:00','utc','utc'); +} {{2000-10-29 12:00:00}} +do_execsql_test date-6.25.6 { + SELECT datetime('2000-10-29 12:00-00:00','utc','utc'); +} {{2000-10-29 12:00:00}} +do_execsql_test date-6.25.7 { + SELECT datetime('2000-10-29 12:00:00-00:00','utc','utc'); +} {{2000-10-29 12:00:00}} do_execsql_test date-6.26 { SELECT datetime('2000-10-29 12:00:00+05:00'); } {{2000-10-29 07:00:00}} diff --git a/test/dblwidth-a.sql b/test/dblwidth-a.sql new file mode 100644 index 000000000..38c219698 --- /dev/null +++ b/test/dblwidth-a.sql @@ -0,0 +1,20 @@ +/* +** Run this script using "sqlite3" to confirm that the command-line +** shell properly handles the output of double-width characters. +** +** https://sqlite.org/forum/forumpost/008ac80276 +*/ +.mode box +CREATE TABLE data(word TEXT, description TEXT); +INSERT INTO data VALUES('〈οὐκέτι〉','Greek without dblwidth <...>'); +.print .mode box +SELECT * FROM data; +.mode table +.print .mode table +SELECT * FROM data; +.mode qbox +.print .mode qbox +SELECT * FROM data; +.mode column +.print .mode column +SELECT * FROM data; diff --git a/test/dbstatus2.test b/test/dbstatus2.test index 5e9ea888a..526d8aa17 100644 --- a/test/dbstatus2.test +++ b/test/dbstatus2.test @@ -41,6 +41,10 @@ proc db_spill {db {reset 0}} { sqlite3_db_status $db CACHE_SPILL $reset } +proc db_temp_spill {db {reset 0}} { + sqlite3_db_status $db TEMPBUF_SPILL $reset +} + do_test 1.1 { db close sqlite3 db test.db @@ -111,5 +115,50 @@ do_execsql_test 3.2 { UPDATE t1 SET b=randomblob(1000); } {delete} do_test 3.3 { db_spill db 0 } {0 8 0} + + +if {$::TEMP_STORE<3} { + do_execsql_test 4.0 { + PRAGMA temp_store = file; + PRAGMA cache_size = -1024; + } {} + + do_test 4.1 { db_temp_spill db 0 } {0 0 0} + + do_execsql_test 4.2 { + CREATE TABLE data(a INTEGER, b BLOB); + + -- Insert 5-6 MB of data. + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<75000 ) + INSERT INTO data SELECT i, hex(randomblob(50)) FROM s; + } + + do_test 4.3 { db_temp_spill db 0 } {0 0 0} + + do_test 4.4 { + execsql { SELECT a, b FROM data ORDER BY a } + set nTmpSpill [lindex [db_temp_spill db 1] 1] + expr {($nTmpSpill>7*1000*1000) && ($nTmpSpill<10*1000*1000)?"ok":$nTmpSpill} + } ok + + # The previous test case reset the status value. + do_test 4.5 { db_temp_spill db 0 } {0 0 0} + + do_test 4.6 { + execsql { CREATE INDEX i1 ON data(a) } + set nTmpSpill [lindex [db_temp_spill db 1] 1] + expr {($nTmpSpill>384*1000) && ($nTmpSpill<768*1000)?"ok":$nTmpSpill} + } ok + + # The previous test case reset the status value. + do_test 4.7 { db_temp_spill db 0 } {0 0 0} + + # Same query as in (4.4). Now does not require temp space. + do_test 4.8 { + execsql { SELECT a, b FROM data ORDER BY a } + db_temp_spill db 0 + } {0 0 0} +} + finish_test diff --git a/test/decimal.test b/test/decimal.test index cf4e06ad9..07510faa1 100644 --- a/test/decimal.test +++ b/test/decimal.test @@ -22,6 +22,9 @@ if {[catch {load_static_extension db decimal} error]} { do_execsql_test 1000 { SELECT decimal(1); } {1} +do_execsql_test 1001 { + SELECT decimal('+0'), decimal('-0'); +} {0 0} do_execsql_test 1010 { SELECT decimal('1.0'); } {1.0} @@ -34,6 +37,12 @@ do_execsql_test 1030 { do_execsql_test 1040 { SELECT decimal('-0001.0'); } {-1.0} +do_execsql_test 1041 { + SELECT decimal('-0000.0'); +} {0.0} +do_execsql_test 1042 { + SELECT decimal(0.0)==decimal(-0.0); +} {1} do_execsql_test 1050 { SELECT decimal('1.0e72'); } {1000000000000000000000000000000000000000000000000000000000000000000000000} @@ -73,6 +82,16 @@ do_execsql_test 2000 { WHERE a.seq=0; } {} +do_execsql_test 2001 { + WITH vx(a,b) AS (VALUES + ('-0','+0'), + ('-000.000','0'), + ('1.2','1.2000') + ) + SELECT *, '|' FROM vx + WHERE decimal_cmp(a,b)!=0 + OR decimal_cmp(b,a)!=0; +} {} do_execsql_test 2010 { SELECT *, '|' FROM t1 AS a, t1 AS b diff --git a/test/e_expr.test b/test/e_expr.test index 6e2b64979..81d2fd172 100644 --- a/test/e_expr.test +++ b/test/e_expr.test @@ -278,6 +278,10 @@ set literals { 13 NULL } foreach op $oplist { + if {$op eq "AND" || $op eq "OR"} { + # These tests do not work with AND and OR due to short-circuit evaluation + continue + } foreach {n1 rhs} $literals { foreach {n2 lhs} $literals { diff --git a/test/eqp.test b/test/eqp.test index 5d2659be7..147b5ceaf 100644 --- a/test/eqp.test +++ b/test/eqp.test @@ -338,8 +338,7 @@ det 3.3.3 { } { QUERY PLAN |--SCAN t1 - `--CORRELATED SCALAR SUBQUERY xxxxxx - `--SCAN t2 + `--SCAN t2 EXISTS } #------------------------------------------------------------------------- diff --git a/test/existsexpr.test b/test/existsexpr.test new file mode 100644 index 000000000..d02f8c5c1 --- /dev/null +++ b/test/existsexpr.test @@ -0,0 +1,432 @@ +# 2024 May 25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set testprefix existsexpr + + +do_execsql_test 1.0 { + CREATE TABLE x1(a, b, PRIMARY KEY(a)) WITHOUT ROWID; + INSERT INTO x1 VALUES(1, 2), (3, 4), (5, 6); + CREATE INDEX x1b ON x1(b); + + CREATE TABLE x2(x, y); + INSERT INTO x2 VALUES(1, 2), (3, 4), (5, 6); +} + +do_execsql_test 1.1 { + SELECT 1 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=5) +} {1} + +do_execsql_test 1.2 { + SELECT * FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=x) +} {1 2 3 4 5 6} + +# With "a=x", the UNIQUE index means the EXIST can be transformed to a join. +# So no "SUBQUERY". With "b=x", the index is not UNIQUE and so there is a +# "SUBQUERY". +do_execsql_test 1.3.1 { + EXPLAIN QUERY PLAN + SELECT * FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=x) +} {~/SUBQUERY/} +do_execsql_test 1.3.2 { + EXPLAIN QUERY PLAN + SELECT * FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE b=x) +} {~/SUBQUERY/} + +do_execsql_test 1.4.1 { + EXPLAIN QUERY PLAN + SELECT * FROM x2 WHERE x=1 AND EXISTS (SELECT 1 FROM x1 WHERE a=x) +} {~/SUBQUERY/} +do_execsql_test 1.4.2 { + EXPLAIN QUERY PLAN + SELECT * FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=x) AND y=2 +} {~/SUBQUERY/} + +do_execsql_test 1.5 { + SELECT count(*) FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=x) +} {3} + +#------------------------------------------------------------------------- +do_execsql_test 2.0 { + CREATE TABLE t1(a, b); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) INSERT INTO t1 SELECT i, i FROM s; + + CREATE TABLE t2(c, d); + WITH s(i) AS ( + SELECT 10 UNION ALL SELECT i+10 FROM s WHERE i<1000 + ) INSERT INTO t2 SELECT i, i FROM s; +} + +do_execsql_test 2.1 { + SELECT count(*) FROM t1; + SELECT count(*) FROM t2; +} {1000 100} + +do_execsql_test 2.2 { + SELECT count(*) FROM t1, t2 WHERE a=c; +} {100} + +do_execsql_test 2.3 { + SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c=a) +} {100} +do_eqp_test 2.4 { + SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c=a) +} {SCAN t1} + +do_execsql_test 2.4.0 { + CREATE UNIQUE INDEX t2c ON t2(c); + CREATE UNIQUE INDEX t1a ON t1(a); +} + +do_eqp_test 2.4.1 { + SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c=a); +} {SCAN t1*t2 EXISTS} +do_execsql_test 2.4.2 { + ANALYZE; +} +do_eqp_test 2.4.3 { + SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c=a); +} {SCAN t1*t2 EXISTS} +do_execsql_test 2.4.4 { + SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c=a); +} {100} + +do_execsql_test 2.5.1 { + EXPLAIN QUERY PLAN + SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE t2.rowid=a); +} {~/SUBQUERY/} + +#------------------------------------------------------------------------- +proc do_subquery_test {tn bSub sql res} { + set r1(0) ~/SUBQUERY/ + set r1(1) /SUBQUERY/ + do_execsql_test $tn.1 "explain query plan $sql" $r1($bSub) + do_execsql_test $tn.2 $sql $res +} + +do_execsql_test 3.0 { + CREATE TABLE y1(a, b, c); + CREATE TABLE y2(x, y, z); + CREATE UNIQUE INDEX y2zy ON y2(z, y); + + INSERT INTO y1 VALUES(1, 1, 1); + INSERT INTO y1 VALUES(2, 2, 2); + INSERT INTO y1 VALUES(3, 3, 3); + INSERT INTO y1 VALUES(4, 4, 4); + + INSERT INTO y2 VALUES(1, 1, 1); + INSERT INTO y2 VALUES(3, 3, 3); +} + +do_subquery_test 3.1 0 { + SELECT * FROM y1 WHERE EXISTS ( + SELECT 1 FROM y2 WHERE z=a AND y=b AND x=z + ) +} { + 1 1 1 3 3 3 +} + +do_subquery_test 3.2 0 { + SELECT * FROM y1 WHERE EXISTS ( + SELECT 1 FROM y2 WHERE z=max(a,b) AND y=min(b,a) AND x=z + ) +} { + 1 1 1 3 3 3 +} + +do_subquery_test 3.3 0 { + SELECT * FROM y1 WHERE EXISTS ( + SELECT 1 FROM y2 WHERE z=max(a,b) AND y=min(b,a) AND c!=3 + ) +} { + 1 1 1 +} + +do_subquery_test 3.4 0 { + SELECT * FROM y1 WHERE EXISTS ( + SELECT 1 FROM y2 WHERE z=max(a,b) AND b=3 + ) +} { + 3 3 3 +} + +do_subquery_test 3.5 0 { + SELECT * FROM y1 WHERE EXISTS ( + SELECT 1 FROM y2 WHERE z=a-1 AND y=a-1 + ) +} { + 2 2 2 + 4 4 4 +} + +do_subquery_test 3.6 0 { + SELECT * FROM y1 WHERE EXISTS ( + SELECT 1 FROM y2 WHERE z=a-1 AND y+1=a + ) +} { + 2 2 2 + 4 4 4 +} + +do_subquery_test 3.7 1 { + SELECT * FROM y1 WHERE EXISTS ( + SELECT count(*) FROM y2 WHERE z=a-1 AND y=a-1 + ) +} { + 1 1 1 + 2 2 2 + 3 3 3 + 4 4 4 +} + +do_subquery_test 3.8 0 { + SELECT * FROM y1 WHERE EXISTS ( SELECT a+1 FROM y2 ) +} { + 1 1 1 + 2 2 2 + 3 3 3 + 4 4 4 +} + +do_subquery_test 3.9 1 { + SELECT * FROM y1 WHERE EXISTS ( + SELECT 1 FROM y2 one, y2 two WHERE one.z=a-1 AND one.y=a-1 + ) +} { + 2 2 2 + 4 4 4 +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE TABLE tx1(a TEXT COLLATE nocase, b TEXT); + CREATE UNIQUE INDEX tx1ab ON tx1(a, b); + + INSERT INTO tx1 VALUES('a', 'a'); + INSERT INTO tx1 VALUES('B', 'b'); + INSERT INTO tx1 VALUES('c', 'c'); + INSERT INTO tx1 VALUES('D', 'd'); + INSERT INTO tx1 VALUES('e', 'e'); + + CREATE TABLE tx2(x, y); + INSERT INTO tx2 VALUES('A', 'a'); + INSERT INTO tx2 VALUES('b', 'b'); + INSERT INTO tx2 VALUES('C', 'c'); + INSERT INTO tx2 VALUES('D', 'd'); +} + +do_subquery_test 4.1 0 { + SELECT * FROM tx2 WHERE EXISTS ( + SELECT 1 FROM tx1 WHERE a=x AND b=y + ) +} { + A a + b b + C c + D d +} + +do_subquery_test 4.1.1 0 { + SELECT * FROM tx2 WHERE EXISTS ( + SELECT 1 FROM tx1 WHERE (a COLLATE nocase)=x AND b=y + ) +} { + A a b b C c D d +} +do_subquery_test 4.1.2 0 { + SELECT * FROM tx2 WHERE EXISTS ( + SELECT 1 FROM tx1 WHERE a=x AND (b COLLATE binary)=y + ) +} { + A a b b C c D d +} +do_subquery_test 4.1.1 0 { + SELECT * FROM tx2 WHERE EXISTS ( + SELECT 1 FROM tx1 WHERE x=(a COLLATE nocase) AND b=y + ) +} { + A a b b C c D d +} +do_subquery_test 4.1.2 0 { + SELECT * FROM tx2 WHERE EXISTS ( + SELECT 1 FROM tx1 WHERE a=x AND y=(b COLLATE binary) + ) +} { + A a b b C c D d +} + +do_subquery_test 4.2 0 { + SELECT * FROM tx2 WHERE EXISTS ( + SELECT 1 FROM tx1 WHERE a=x AND b=y COLLATE nocase + ) +} { + A a + b b + C c + D d +} + +do_execsql_test 4.3 { + DROP INDEX tx1ab; + CREATE UNIQUE INDEX tx1ab ON tx1(a COLLATE binary, b); +} + +do_subquery_test 4.4 0 { + SELECT * FROM tx2 WHERE EXISTS ( + SELECT 1 FROM tx1 WHERE a=x AND b=y + ) +} { + A a + b b + C c + D d +} + +do_subquery_test 4.4 0 { + SELECT * FROM tx2 WHERE EXISTS ( + SELECT 1 FROM tx1 WHERE a=x COLLATE binary AND b=y + ) +} { + D d +} + +do_subquery_test 4.4 1 { + SELECT EXISTS ( SELECT x FROM tx1 ) FROM tx2 +} { + 1 1 1 1 +} + +do_subquery_test 4.4 1 { + SELECT (SELECT EXISTS ( SELECT x FROM tx1 ) WHERE 1) FROM tx2 +} { + 1 1 1 1 +} + +#------------------------------------------------------------------------- +proc cols {s f} { + set lCols [list] + for {set i $s} {$i<=$f} {incr i} { + lappend lCols [format "c%02d" $i] + } + join $lCols ", " +} +proc vals {n val} { + set lVal [list] + for {set i 0} {$i<$n} {incr i} { + lappend lVal $val + } + join $lVal ", " +} +proc exprs {s f} { + set lExpr [list] + for {set i $s} {$i<=$f} {incr i} { + lappend lExpr [format "c%02d = o" $i] + } + join $lExpr " AND " +} + + +do_execsql_test 5.0 " + CREATE TABLE a1( [cols 0 99] ); +" +do_execsql_test 5.1 " + -- 63 column index + CREATE UNIQUE INDEX a1idx1 ON a1( [cols 0 62] ); +" +do_execsql_test 5.2 " + -- 64 column index + CREATE UNIQUE INDEX a1idx2 ON a1( [cols 10 73] ); +" +do_execsql_test 5.2 " + -- 65 column index + CREATE UNIQUE INDEX a1idx3 ON a1( [cols 20 84] ); +" + +do_test 5.3 { + foreach v {1 2 3 4 5 6} { + execsql "INSERT INTO a1 VALUES( [vals 100 $v] )" + } +} {} + +do_execsql_test 5.4 { + CREATE TABLE a2(o); + INSERT INTO a2 VALUES(2), (5); +} + +do_subquery_test 5.5 0 " + SELECT o FROM a2 WHERE EXISTS ( + SELECT 1 FROM a1 WHERE [exprs 0 62] + ) +" { + 2 5 +} + +do_subquery_test 5.6 0 " + SELECT o FROM a2 WHERE EXISTS ( + SELECT 1 FROM a1 WHERE [exprs 10 73] + ) +" { + 2 5 +} + +do_subquery_test 5.7 0 " + SELECT o FROM a2 WHERE EXISTS ( + SELECT 1 FROM a1 WHERE [exprs 20 84] + ) +" { + 2 5 +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE TABLE t1(a, b UNIQUE, c UNIQUE); + CREATE TABLE t2(a INfEGER PRIMARY KEY, b); + CREATE UNIQUE INDEX t2b ON t2(b); +} + +do_catchsql_test 6.1 { + SELECT a FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c COLLATE f = a) +} {1 {no such collation sequence: f}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 7.0 { + CREATE TABLE t1(x); + CREATE TABLE t2(y UNIQUE); + + INSERT INTO t1 VALUES(1), (2); + INSERT INTO t2 VALUES(1), (3); + + SELECT * FROM t1 one LEFT JOIN t1 two ON (one.x=two.x AND EXISTS ( + SELECT 1 FROM t2 WHERE y=one.x + )); +} { + 1 1 + 2 {} +} + +# https://sqlite.org/forum/forumpost/2025-07-23T10:59:14z +reset_db +do_execsql_test 8.0 { + CREATE TABLE t0 (c0 INT); INSERT INTO t0(c0) VALUES (1); + CREATE TABLE t1(c0 INT); INSERT INTO t1(c0) VALUES (2); + SELECT * FROM t1 WHERE EXISTS (SELECT 1 FROM t0 LIMIT 0); +} {} + +finish_test diff --git a/test/existsexpr2.test b/test/existsexpr2.test new file mode 100644 index 000000000..f7644bf80 --- /dev/null +++ b/test/existsexpr2.test @@ -0,0 +1,96 @@ +# 2024 June 14 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set testprefix existsexpr2 + + +do_execsql_test 1.0 { + CREATE TABLE x1(a, b, PRIMARY KEY(a)) WITHOUT ROWID; + INSERT INTO x1 VALUES(1, 2), (3, 4), (5, 6); + CREATE INDEX x1b ON x1(b); + + CREATE TABLE x2(x, y); + INSERT INTO x2 VALUES(1, 2), (3, 4), (5, 6); +} + +do_execsql_test 1.1 { + SELECT * FROM x1 WHERE EXISTS (SELECT 1 FROM x2 WHERE a!=123) +} {1 2 3 4 5 6} + +do_execsql_test 1.2 { + CREATE TABLE x3(u, v); + CREATE INDEX x3u ON x3(u); + INSERT INTO x3 VALUES + (1, 1), (1, 2), (1, 3), + (2, 1), (2, 2), (2, 3); +} + +do_execsql_test 1.3 { + SELECT * FROM x1 WHERE EXISTS ( + SELECT 1 FROM x3 WHERE u IN (1, 2, 3, 4) AND v=b + ); +} { + 1 2 +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(a, b, c); + CREATE INDEX t1ab ON t1(a,b); + + INSERT INTO t1 VALUES + ('abc', 1, 1), + ('abc', 2, 2), + ('abc', 2, 3), + + ('def', 1, 1), + ('def', 2, 2), + ('def', 2, 3); + + CREATE TABLE t2(x, y); + INSERT INTO t2 VALUES(1, 1), (2, 2), (3, 3); + + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1 VALUES('t1','t1ab','10000 5000 2'); + ANALYZE sqlite_master; +} + + +do_execsql_test 2.1 { + SELECT a,b,c FROM t1 WHERE b=2 ORDER BY a +} { + abc 2 2 + abc 2 3 + def 2 2 + def 2 3 +} + +do_execsql_test 2.2 { + SELECT x, y FROM t2 WHERE EXISTS ( + SELECT 1 FROM t1 WHERE b=x + ) +} { + 1 1 + 2 2 +} + + + +finish_test + + diff --git a/test/existsfault.test b/test/existsfault.test new file mode 100644 index 000000000..4b335d84c --- /dev/null +++ b/test/existsfault.test @@ -0,0 +1,49 @@ +# 2024 May 25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl +set testprefix existsfault + +db close +sqlite3_shutdown +sqlite3_config_lookaside 0 0 +sqlite3_initialize +autoinstall_test_functions +sqlite3 db test.db + +do_execsql_test 1.0 { + CREATE TABLE x1(a, b); + INSERT INTO x1 VALUES(1, 2), (3, 4), (5, 6); + CREATE UNIQUE INDEX x1a ON x1(a); + CREATE INDEX x1b ON x1(b); + + CREATE TABLE x2(x, y); + INSERT INTO x2 VALUES(1, 2), (3, 4), (5, 6); +} + +do_faultsim_test 1 -faults oom* -prep { + sqlite3 db test.db + execsql { SELECT * FROM sqlite_schema } +} -body { + execsql { + SELECT count(*) FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=x) AND y!=11 + } +} -test { + faultsim_test_result {0 3} +} + +finish_test + + diff --git a/test/expr.test b/test/expr.test index 71518df69..4f739e2e7 100644 --- a/test/expr.test +++ b/test/expr.test @@ -71,6 +71,9 @@ test_expr expr-1.28 {i1=1, i2=2} {i1=2 AND i2=1} {0} test_expr expr-1.29 {i1=1, i2=2} {i1=1 AND i2=1} {0} test_expr expr-1.30 {i1=1, i2=2} {i1=2 AND i2=2} {0} test_expr expr-1.31 {i1=1, i2=2} {i1==1 OR i2=2} {1} +test_expr expr-1.31b {i1=1} {0 OR 2} {1} +test_expr expr-1.31c {i1=1} {false OR true} {1} +test_expr expr-1.31d {i1=1} {99 OR false} {1} test_expr expr-1.32 {i1=1, i2=2} {i1=2 OR i2=1} {0} test_expr expr-1.33 {i1=1, i2=2} {i1=1 OR i2=1} {1} test_expr expr-1.34 {i1=1, i2=2} {i1=2 OR i2=2} {1} diff --git a/test/fts3atoken2.test b/test/fts3atoken2.test new file mode 100644 index 000000000..f6a2a29ab --- /dev/null +++ b/test/fts3atoken2.test @@ -0,0 +1,106 @@ +# 2025 September 5 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The focus +# of this script is testing the pluggable tokeniser feature of the +# FTS3 module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +set ::testprefix fts3atoken2 + + +reset_db +sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 0 + +# With ENABLE_FTS3_TOKENIZER set to 0: +# +# * It is not possible to get a pointer to a token implementation +# using single arg fts3_tokenize() unless the name of the tokenizer +# is a bound paramter - function should return NULL. +# +# * But it is possible with a bound parameter. +# +do_execsql_test 1.1.1 { + SELECT typeof( fts3_tokenizer('simple') ); +} {null} +set bound "simple" +do_execsql_test 1.1.2 { + SELECT typeof( fts3_tokenizer($bound) ); +} {blob} + +# With ENABLE_FTS3_TOKENIZER set to 0: +# +# * It is not possible to create a token implementation using anything +# other than a bound parameter. +# +# * But it is possible with a bound parameter. +# +set literal [db one {SELECT quote( fts3_tokenizer($bound) )}] +set blob [db one {SELECT fts3_tokenizer($bound) }] + +do_catchsql_test 1.2.1 " + SELECT fts3_tokenizer('mytok', $literal) +" {1 {fts3tokenize disabled}} +do_catchsql_test 1.2.2 { + CREATE VIRTUAL TABLE x1 USING fts3(col, tokenize=mytok); +} {1 {unknown tokenizer: mytok}} +do_catchsql_test 1.2.3 { + SELECT fts3_tokenizer('mytok', $blob) +} {0 {{}}} +do_execsql_test 1.2.4 { + CREATE VIRTUAL TABLE x1 USING fts3(col, tokenize=mytok); +} + +# With ENABLE_FTS3_TOKENIZER set to 1: +# +# * It is possible to get a pointer to a token implementation with either +# a bound parameter or a literal. +# +sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1 +set bound "simple" +do_execsql_test 1.3.1 { + SELECT typeof( fts3_tokenizer('simple') ); +} {blob} +do_execsql_test 1.3.2 { + SELECT typeof( fts3_tokenizer($bound) ); +} {blob} + +# With ENABLE_FTS3_TOKENIZER set to 1: +# +# * It is not possible to create a token implementation using either +# a bound parameter or a literal. +# +set literal [db one {SELECT quote( fts3_tokenizer($bound) )}] +set blob [db one {SELECT fts3_tokenizer($bound) }] + +do_execsql_test 1.4.1 " + SELECT typeof( fts3_tokenizer('mytok2', $literal) ); +" {blob} +do_execsql_test 1.4.2 { + CREATE VIRTUAL TABLE x2 USING fts3(col, tokenize=mytok2); +} +do_execsql_test 1.4.3 { + SELECT typeof( fts3_tokenizer('mytok3', $blob) ); +} {blob} +do_execsql_test 1.4.4 { + CREATE VIRTUAL TABLE x3 USING fts3(col, tokenize=mytok3); +} + +finish_test + diff --git a/test/func9.test b/test/func9.test index 2383b76f6..d24d5f7be 100644 --- a/test/func9.test +++ b/test/func9.test @@ -29,6 +29,15 @@ do_execsql_test func9-130 { do_execsql_test func9-131 { SELECT concat_ws(',',1,2,3,4,'',6,7,8,NULL,9,10,11,12); } {1,2,3,4,,6,7,8,9,10,11,12} +do_execsql_test func9-132 { + SELECT concat_ws(',','',2,3,4,'',6,7,8,NULL,9,10,11,12); +} {,2,3,4,,6,7,8,9,10,11,12} +do_execsql_test func9-133 { + SELECT concat_ws(',',NULL,'',3,4,'',6,7,8,NULL,9,10,11,12); +} {,3,4,,6,7,8,9,10,11,12} +do_execsql_test func9-134 { + SELECT concat_ws(',',NULL,NULL,NULL,'',3,4,'',6,7,8,NULL,9,10,11,12); +} {,3,4,,6,7,8,9,10,11,12} do_execsql_test func9-140 { SELECT concat_ws(NULL,1,2,3,4,5,6,7,8,NULL,9,10,11,12); } {{}} @@ -50,4 +59,8 @@ do_execsql_test func9-220 { SELECT format('%#Q',unistr('G\u00e4ste')); } {'Gäste'} +do_execsql_test func9-300 { + SELECT hex( unistr('\UFFFFFFFF') ) +} {F7BFBFBF} + finish_test diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index 09898d7b3..d8dbf932d 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -171,8 +171,6 @@ static struct GlobalVars { */ extern int sqlite3_vt02_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_randomjson_init(sqlite3*,char**,const sqlite3_api_routines*); -extern int sqlite3_percentile_init(sqlite3*,char**,const sqlite3_api_routines*); - /* ** Print an error message and quit. @@ -541,9 +539,12 @@ static void blobListLoadFromDb( int n = 0; int rc; char *z2; - unsigned char tmp[SZ_BLOB(8)]; + union { + Blob sBlob; + unsigned char tmp[SZ_BLOB(8)]; + } uBlob; - head = (Blob*)tmp; + head = &uBlob.sBlob; if( firstId>0 ){ z2 = sqlite3_mprintf("%s WHERE rowid BETWEEN %d AND %d", zSql, firstId, lastId); @@ -1020,7 +1021,7 @@ extern int sqlite3_dbdata_init(sqlite3*,const char**,void*); ** print the supplied SQL statement to stdout. */ static int recoverSqlCb(void *pCtx, const char *zSql){ - if( eVerbosity>=2 ){ + if( eVerbosity>=2 && zSql ){ printf("%s\n", zSql); } return SQLITE_OK; @@ -1056,11 +1057,14 @@ static int recoverDatabase(sqlite3 *db){ } return rc; } + /* ** Special parameter binding, for testing and debugging purposes. ** -** $int_NNN -> integer value NNN -** $text_TTTT -> floating point value TTT with destructor +** $int_NNN -> integer value NNN +** $text_TTTT -> floating point value TTT with destructor +** $carray_clr -> First argument to carray() for color names +** $carray_primes -> First argument to carray() for prime numbers */ static void bindDebugParameters(sqlite3_stmt *pStmt){ int nVar = sqlite3_bind_parameter_count(pStmt); @@ -1068,6 +1072,24 @@ static void bindDebugParameters(sqlite3_stmt *pStmt){ for(i=1; i<=nVar; i++){ const char *zVar = sqlite3_bind_parameter_name(pStmt, i); if( zVar==0 ) continue; +#ifdef SQLITE_ENABLE_CARRAY + if( strcmp(zVar,"$carray_clr")==0 ){ + static char *azColorNames[] = { + "azure", "black", "blue", "brown", "cyan", "fuchsia", "gold", + "gray", "green", "indigo", "khaki", "lime", "magenta", "maroon", + "navy", "olive", "orange", "pink", "purple", "red", "silver", + "tan", "teal", "violet", "white", "yellow" + }; + sqlite3_carray_bind(pStmt,i,azColorNames,26,SQLITE_CARRAY_TEXT,0); + }else + if( strcmp(zVar,"$carray_primes")==0 ){ + static int aPrimes[] = { + 1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, + 53, 59, 61, 67, 71, 73, 79, 83, 89, 97 + }; + sqlite3_carray_bind(pStmt,i,aPrimes,26,SQLITE_CARRAY_INT32,0); + }else +#endif if( strncmp(zVar, "$int_", 5)==0 ){ sqlite3_bind_int(pStmt, i, atoi(&zVar[5])); }else @@ -1359,7 +1381,6 @@ int runCombinedDbSqlInput( sqlite3_vt02_init(cx.db, 0, 0); /* Activate extensions */ - sqlite3_percentile_init(cx.db, 0, 0); sqlite3_randomjson_init(cx.db, 0, 0); /* Add support for sqlite_dbdata and sqlite_dbptr virtual tables used diff --git a/test/fuzzinvariants.c b/test/fuzzinvariants.c index c8aae6e3d..6a5cfda68 100644 --- a/test/fuzzinvariants.c +++ b/test/fuzzinvariants.c @@ -41,8 +41,10 @@ static void reportInvariantFailed( /* ** Special parameter binding, for testing and debugging purposes. ** -** $int_NNN -> integer value NNN -** $text_TTTT -> floating point value TTT with destructor +** $int_NNN -> integer value NNN +** $text_TTTT -> floating point value TTT with destructor +** $carray_clr -> First argument to carray() for color names +** $carray_primes -> First argument to carray() for prime numbers */ static void bindDebugParameters(sqlite3_stmt *pStmt){ int nVar = sqlite3_bind_parameter_count(pStmt); @@ -50,6 +52,24 @@ static void bindDebugParameters(sqlite3_stmt *pStmt){ for(i=1; i<=nVar; i++){ const char *zVar = sqlite3_bind_parameter_name(pStmt, i); if( zVar==0 ) continue; +#ifdef SQLITE_ENABLE_CARRAY + if( strcmp(zVar,"$carray_clr")==0 ){ + static char *azColorNames[] = { + "azure", "black", "blue", "brown", "cyan", "fuchsia", "gold", + "gray", "green", "indigo", "khaki", "lime", "magenta", "maroon", + "navy", "olive", "orange", "pink", "purple", "red", "silver", + "tan", "teal", "violet", "white", "yellow" + }; + sqlite3_carray_bind(pStmt,i,azColorNames,26,SQLITE_CARRAY_TEXT,0); + }else + if( strcmp(zVar,"$carray_primes")==0 ){ + static int aPrimes[] = { + 1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, + 53, 59, 61, 67, 71, 73, 79, 83, 89, 97 + }; + sqlite3_carray_bind(pStmt,i,aPrimes,26,SQLITE_CARRAY_INT32,0); + }else +#endif if( strncmp(zVar, "$int_", 5)==0 ){ sqlite3_bind_int(pStmt, i, atoi(&zVar[5])); }else @@ -303,6 +323,7 @@ static char *fuzz_invariant_sql(sqlite3_stmt *pStmt, int iCnt){ int bDistinct = 0; int bOrderBy = 0; int nParam = sqlite3_bind_parameter_count(pStmt); + int hasGroupBy = 0; switch( iCnt % 4 ){ case 1: bDistinct = 1; break; @@ -327,6 +348,7 @@ static char *fuzz_invariant_sql(sqlite3_stmt *pStmt, int iCnt){ sqlite3_finalize(pBase); pBase = pStmt; } + hasGroupBy = sqlite3_strlike("%GROUP BY%",zIn,0)==0; bindDebugParameters(pBase); for(i=0; i1 && i+2!=iCnt ) continue; if( zColName==0 ) continue; if( sqlite3_column_type(pStmt, i)==SQLITE_NULL ){ - sqlite3_str_appendf(pTest, " %s \"%w\" ISNULL", zAnd, zColName); + const char *zPlus = hasGroupBy ? "+" : ""; + sqlite3_str_appendf(pTest, " %s %s\"%w\" ISNULL", zAnd, zPlus, zColName); }else{ sqlite3_str_appendf(pTest, " %s \"%w\"=?%d", zAnd, zColName, i+1+nParam); diff --git a/test/hook.test b/test/hook.test index 8638d3a6b..a4256732e 100644 --- a/test/hook.test +++ b/test/hook.test @@ -488,11 +488,21 @@ proc preupdate_hook {args} { set type [lindex $args 0] eval lappend ::preupdate $args if {$type != "INSERT"} { + set x [catch {db preupdate old [db preupdate count]}] + if {!$x} { + lappend "ERROR: sqlite3_preupdate_old() accepted an out-of-bounds\ + column index" + } for {set i 0} {$i < [db preupdate count]} {incr i} { lappend ::preupdate [db preupdate old $i] } } if {$type != "DELETE"} { + set x [catch {db preupdate new [db preupdate count]}] + if {!$x} { + lappend "ERROR: sqlite3_preupdate_old() accepted an out-of-bounds\ + column index" + } for {set i 0} {$i < [db preupdate count]} {incr i} { set rc [catch { db preupdate new $i } v] lappend ::preupdate $v diff --git a/test/ieee754.test b/test/ieee754.test index bd806d2cf..467416dae 100644 --- a/test/ieee754.test +++ b/test/ieee754.test @@ -57,4 +57,16 @@ do_execsql_test ieee754-112 { SELECT ieee754(4503599627370495,973) is null; } {1} +do_execsql_test ieee754-200 { + SELECT ieee754(0.0), hex(ieee754_to_blob(ieee754(0,-1075))); +} {ieee754(0,-1075) 0000000000000000} + +# Special case. -0.0 is rendered as ieee754(-1,-3071). +# +do_execsql_test ieee754-201 { + SELECT ieee754(-0.0), hex(ieee754_to_blob(ieee754(-1,-3071))); +} {ieee754(-1,-3071) 8000000000000000} + + + finish_test diff --git a/test/imposter1.test b/test/imposter1.test index 196767be1..da58266be 100644 --- a/test/imposter1.test +++ b/test/imposter1.test @@ -85,6 +85,7 @@ do_execsql_test imposter-1.2 { # database. # do_execsql_test imposter-1.3 { + PRAGMA writable_schema=on; DELETE FROM xt1 WHERE rowid=5; INSERT INTO xt1(rowid,a,b,c,d) VALUES(99,'hello',1099,2022,NULL); SELECT * FROM chnglog ORDER BY rowid; diff --git a/test/incrblob4.test b/test/incrblob4.test index dbff8eb7d..c9bcee8a3 100644 --- a/test/incrblob4.test +++ b/test/incrblob4.test @@ -106,4 +106,102 @@ do_test 4.4 { } {SQLITE_LOCKED} close $blob +#------------------------------------------------------------------------- + +ifcapable preupdate { + +reset_db +do_execsql_test 5.1 { + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + INSERT INTO t2 VALUES(1000, 'abcdefghijklmnopqrstuvwxyz'); + INSERT INTO t2 VALUES(2000, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'); + INSERT INTO t2 VALUES(3000, 'abcdefghijklmnopqrstuvwxyz'); +} + +do_test 5.2.1 { + execsql BEGIN + set blob [db incrblob t2 b 2000] + seek $blob 0 + puts -nonewline $blob "hello " + flush $blob + execsql ROLLBACK +} {} + +do_test 5.2.2 { + puts -nonewline $blob "world" + set rc [catch { flush $blob } msg] + list $rc [regsub {input/output} $msg {I/O}] +} "1 {error flushing \"$blob\": I/O error}" +catch { close $blob } + +set preupdate_count 0 +proc preupdate {args} { incr ::preupdate_count ; return {} } +db preupdate hook preupdate + +set preupdate_count 0 +do_test 5.3.1 { + execsql BEGIN + set blob [db incrblob t2 b 1000] + seek $blob 0 + puts -nonewline $blob "hello " + flush $blob + execsql ROLLBACK +} {} + +do_test 5.3.2 { + puts -nonewline $blob "world" + set rc [catch { flush $blob } msg] + list $rc [regsub {input/output} $msg {I/O}] +} "1 {error flushing \"$blob\": I/O error}" +catch { close $blob } + +do_test 5.3.3 { + set ::preupdate_count +} {1} + +set preupdate_count 0 +do_test 5.4.1 { + execsql BEGIN + set blob [db incrblob t2 b 1000] + seek $blob 0 + puts -nonewline $blob "hello " + flush $blob + execsql { DELETE FROM t2 WHERE a=3000; } +} {} + +do_test 5.4.2 { + puts -nonewline $blob "world" + list [catch { flush $blob } msg] $msg +} "0 {}" +catch { close $blob } +catchsql { ROLLBACK } + +do_test 5.3.3 { + set ::preupdate_count +} {3} + +set preupdate_count 0 +do_test 5.4.3 { + execsql BEGIN + set blob [db incrblob t2 b 2000] + seek $blob 0 + puts -nonewline $blob "hello " + flush $blob + execsql { UPDATE t2 SET b='abcdefghijklmnopqrstuvwxyz' WHERE a=2000 } +} {} + +do_test 5.4.4 { + puts -nonewline $blob "world" + set rc [catch { flush $blob } msg] + list $rc [regsub {input/output} $msg {I/O}] +} "1 {error flushing \"$blob\": I/O error}" +catch { close $blob } +catchsql { ROLLBACK } + +do_test 5.4.5 { + set ::preupdate_count +} {2} + +} + finish_test diff --git a/test/joinH.test b/test/joinH.test index 339200968..56fa9c7ec 100644 --- a/test/joinH.test +++ b/test/joinH.test @@ -411,4 +411,67 @@ do_execsql_test 15.2 { SELECT lower(x), quote(y) FROM x1 LEFT JOIN x2 USING (x) JOIN x3 USING (x) FULL JOIN t4; } {abc NULL} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 16.0 { + CREATE TABLE t0_a (c0 INT); + CREATE TABLE t0_b (c0 INT); + + CREATE TABLE t2 (c0 INT); + INSERT INTO t2 VALUES (1); +} + +do_execsql_test 16.1 { + SELECT * FROM t2 LEFT JOIN t0_b +} {1 {}} + +do_execsql_test 16.2.1 { + SELECT * FROM (t0_a RIGHT JOIN ( t2 LEFT JOIN t0_b)) +} {{} 1 {}} +do_execsql_test 16.2.2 { + SELECT * FROM (t0_a RIGHT JOIN (SELECT * FROM t2 LEFT JOIN t0_b)) +} {{} 1 {}} + + +do_catchsql_test 16.3.1 { + SELECT * FROM (t0_a RIGHT JOIN ( t2 LEFT JOIN t0_b) USING (c0)); +} {1 {ambiguous column name: c0}} + +do_execsql_test 16.3.2 { + SELECT * FROM (t0_a RIGHT JOIN (SELECT * FROM t2 LEFT JOIN t0_b) USING (c0)); +} {1 {}} + +do_execsql_test 16.4.0 { + CREATE TABLE x0(a TEXT); + CREATE TABLE x1(a TEXT); + CREATE TABLE x2(a TEXT); + + INSERT INTO x1 VALUES('blue'); + INSERT INTO x2 VALUES('red'); +} + +do_catchsql_test 16.4.1 { + SELECT * FROM x0 RIGHT JOIN ( x1, x2) USING (a) +} {1 {ambiguous column name: a}} + +do_execsql_test 16.4.2 { + SELECT * FROM x0 RIGHT JOIN (SELECT * FROM x1, x2) USING (a) +} {blue red} + +reset_db +do_execsql_test 16.5.0 { + CREATE TABLE t0 (c0 INT); + CREATE TABLE t1 (c0 INT); + CREATE TABLE t2 (c0 INT); + + INSERT INTO t1(c0) VALUES (NULL); + INSERT INTO t2 VALUES (1); + + CREATE VIEW v0(c0) AS SELECT 1 AS col_0 FROM t0; +} + +do_catchsql_test 16.5.2 { + SELECT * FROM (t0 NATURAL RIGHT JOIN (t0 FULL JOIN (v0 NATURAL FULL JOIN t2) ON TRUE)) NATURAL FULL JOIN t1; +} {1 {ambiguous column name: c0}} + finish_test diff --git a/test/joinI.test b/test/joinI.test new file mode 100644 index 000000000..577ca4c2c --- /dev/null +++ b/test/joinI.test @@ -0,0 +1,125 @@ +# 2022 May 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix joinI + +do_execsql_test 1.0 { + CREATE TABLE t1(a INT); + CREATE TABLE t2(b INT); + CREATE TABLE t3(c INT); +} + +foreach {tn sql} { + 1 "SELECT * FROM t1 RIGHT JOIN t2 ON t2.b=t3.c CROSS JOIN t3" + 2 "SELECT * FROM t1 RIGHT JOIN t2 ON t2.b=(SELECT t3.c) CROSS JOIN t3" + 3 "SELECT * FROM t1 RIGHT JOIN t2 ON CASE WHEN t2.b THEN t3.c ELSE 1 END CROSS JOIN t3" +} { + do_catchsql_test 1.1.$tn $sql {1 {ON clause references tables to its right}} +} + + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 2.0 { + CREATE TABLE t0(c0 INT, c1 INT); + CREATE TABLE t1 (c0 INT); + + CREATE VIEW v1(c0) AS SELECT t0.c0 FROM t0 NATURAL RIGHT JOIN t1; + CREATE VIEW v2(c0) AS SELECT 0 FROM v1; + + INSERT INTO t0(c0, c1) VALUES (-1, 0); + INSERT INTO t1(c0) VALUES (NULL); + + SELECT * FROM v1 INNER JOIN (v2 CROSS JOIN t0) ON (t0.c0 < t0.c1); +} {{} 0 -1 0} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE TABLE t0(c0, c1); + CREATE TABLE t1(v); + CREATE TABLE t2(w); + CREATE TABLE t3(x); + CREATE TABLE t4(y); + CREATE TABLE t5(z); +} + +do_execsql_test 3.1 { + SELECT 1234 FROM t4 + RIGHT JOIN t5 + CROSS JOIN (t2 CROSS JOIN t0) AS a1 ON (a1.c0 < a1.c1); +} + +do_execsql_test 3.2 { + SELECT 1234 FROM t4 + RIGHT JOIN t5 + CROSS JOIN (t2 CROSS JOIN t1 CROSS JOIN t0) AS a1 ON (a1.c0 < a1.c1); +} + +do_execsql_test 3.3 { + SELECT 5678 FROM t0 RIGHT JOIN t1 ON ( + SELECT 1 FROM t2 RIGHT JOIN t3 ON t2.w + ) CROSS JOIN t4; +} + +do_catchsql_test 3.4 { + SELECT 5678 FROM t0 RIGHT JOIN t1 ON ( + SELECT 1 FROM t2 RIGHT JOIN t3 ON t4.y + ) CROSS JOIN t4; +} {1 {ON clause references tables to its right}} + +do_execsql_test 3.5 { + SELECT 5678 FROM t0 RIGHT JOIN t1 ON ( + SELECT 1 FROM t2 RIGHT JOIN t3 ON 0=t1.v + ) CROSS JOIN t4; +} + +do_catchsql_test 3.6 { + SELECT 5678 FROM t0 RIGHT JOIN t1 ON ( + SELECT 1 FROM t2 RIGHT JOIN t3 ON max(0,t5.z) CROSS JOIN t5 + ) CROSS JOIN t4; +} {1 {ON clause references tables to its right}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE TABLE t1(a); + CREATE TABLE t2(b); + CREATE TABLE t3(c, d); +} + +do_catchsql_test 4.1 { + SELECT c+d AS cd FROM t1 LEFT JOIN t2 ON (cd=5) CROSS JOIN t3; +} {1 {ON clause references tables to its right}} + + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE TABLE parent1(parent1key, child1key, Child2key, child3key); + CREATE TABLE child1 ( child1key NVARCHAR, value NVARCHAR ); + CREATE TABLE child2 ( child2key NVARCHAR, value NVARCHAR ); +} + +do_execsql_test 5.1 { + SELECT parent1.parent1key, child1.value, child2.value + FROM parent1 + LEFT OUTER JOIN child1 ON child1.child1key = parent1.child1key + INNER JOIN child2 ON child2.child2key = parent1.child2key; +} + +finish_test + diff --git a/test/json101.test b/test/json101.test index e22902f86..7582d14a6 100644 --- a/test/json101.test +++ b/test/json101.test @@ -892,15 +892,15 @@ do_execsql_test json101-13.100 { INSERT INTO t2(id,json) VALUES(4,'{"value":4}'); INSERT INTO t2(id,json) VALUES(5,'{"value":5}'); INSERT INTO t2(id,json) VALUES(6,'{"value":6}'); - SELECT * FROM t1 CROSS JOIN t2 + SELECT *, 'NL' FROM t1 CROSS JOIN t2 WHERE EXISTS(SELECT 1 FROM json_each(t1.json,'$.items') AS Z WHERE Z.value==t2.id); -} {1 {{"items":[3,5]}} 3 {{"value":3}} 1 {{"items":[3,5]}} 5 {{"value":5}}} +} {1 {{"items":[3,5]}} 3 {{"value":3}} NL 1 {{"items":[3,5]}} 5 {{"value":5}} NL} do_execsql_test json101-13.110 { - SELECT * FROM t2 CROSS JOIN t1 + SELECT *, 'NL' FROM t2 CROSS JOIN t1 WHERE EXISTS(SELECT 1 FROM json_each(t1.json,'$.items') AS Z WHERE Z.value==t2.id); -} {3 {{"value":3}} 1 {{"items":[3,5]}} 5 {{"value":5}} 1 {{"items":[3,5]}}} +} {3 {{"value":3}} 1 {{"items":[3,5]}} NL 5 {{"value":5}} 1 {{"items":[3,5]}} NL} # 2018-05-16 # Incorrect fullkey output from json_each() diff --git a/test/notnull2.test b/test/notnull2.test index 09161efbd..67d7c26a8 100644 --- a/test/notnull2.test +++ b/test/notnull2.test @@ -66,7 +66,7 @@ do_vmstep_test 1.5.2 { SELECT count(*) FROM t2 WHERE EXISTS( SELECT 1 FROM t1 WHERE t1.a=450 AND t2.c IS NULL ) -} +8000 {0} +} 4000 {0} #------------------------------------------------------------------------- reset_db diff --git a/test/ossfuzz.c b/test/ossfuzz.c index b0156a640..8e80b98ef 100644 --- a/test/ossfuzz.c +++ b/test/ossfuzz.c @@ -155,6 +155,11 @@ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { /* Set a limit on the maximum size of a prepared statement */ sqlite3_limit(cx.db, SQLITE_LIMIT_VDBE_OP, 25000); + /* Set a limit on the maximum LIKE or GLOB pattern length due to + ** https://issues.oss-fuzz.com/issues/453240497. The default is 50K + ** which is causing timeouts in OSS-Fuzz */ + sqlite3_limit(cx.db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH, 250); + /* Limit total memory available to SQLite to 20MB */ sqlite3_hard_heap_limit64(20000000); diff --git a/test/percentile.test b/test/percentile.test index a6a29da34..25096a953 100644 --- a/test/percentile.test +++ b/test/percentile.test @@ -19,7 +19,6 @@ source $testdir/tester.tcl # Basic test of the percentile() function. # do_test percentile-1.0 { - load_static_extension db percentile execsql { CREATE TABLE t1(x); INSERT INTO t1 VALUES(1),(4),(6),(7),(8),(9),(11),(11),(11); diff --git a/test/permutations.test b/test/permutations.test index 87a14ef66..02f482718 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -1119,6 +1119,18 @@ test_suite "maindbname" -prefix "" -description { dbconfig_maindbname_icecube $::dbhandle } +test_suite "win_unc_locking" -prefix "" -description { + Run the "wal*" tests with UNC style single fd locking. +} -files [ + test_set \ + {*}[glob [file join $testdir wal*]] \ + -exclude *fault* *malloc* *crash* *slow* walsetlk.test +] -initialize { + set sqlite3_win_test_unc_locking 1 +} -shutdown { + set sqlite3_win_test_unc_locking 0 +} + # End of tests ############################################################################# diff --git a/test/regexp2.test b/test/regexp2.test index 3e1da9f23..55ce59d05 100644 --- a/test/regexp2.test +++ b/test/regexp2.test @@ -141,4 +141,19 @@ do_execsql_test 4.16 {SELECT ' ' REGEXP '[^a-z]{2}'} {1} do_execsql_test 4.17 {SELECT 'abc' REGEXP '\W{1,1}'} {0} do_execsql_test 4.18 {SELECT 'abc' REGEXP '\W{1}'} {0} +do_execsql_test 5.0 { + SELECT 'abc' REGEXP 'a{1,999}bc'; +} 1 +do_catchsql_test 5.1 { + SELECT 'abc' REGEXP 'a{1,25000}bc'; +} {1 {REGEXP pattern too big}} +do_execsql_test 5.2 { + SELECT 'abc' REGEXP 'a{999}bc'; +} 0 +do_catchsql_test 5.3 { + SELECT 'abc' REGEXP 'a{25000}bc'; +} {1 {REGEXP pattern too big}} + + + finish_test diff --git a/test/reservebytes.test b/test/reservebytes.test new file mode 100644 index 000000000..51b76c723 --- /dev/null +++ b/test/reservebytes.test @@ -0,0 +1,51 @@ +# 2025 August 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix reservebytes + + + +reset_db +file_control_reservebytes db 0 + + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b, c); + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) + INSERT INTO t1 SELECT NULL, i, hex(randomblob(500)) FROM s; +} + +sqlite3 db2 test.db +if {[permutation]=="prepare"} { db2 cache size 0 } + +do_execsql_test -db db2 1.1 { PRAGMA integrity_check } {ok} + +file_control_reservebytes db 8 +do_test 1.2.1 { hexio_read test.db 20 1 } {00} +do_execsql_test -db db2 1.2.2 { PRAGMA integrity_check } {ok} + +do_execsql_test 1.3.2 { VACUUM } +do_execsql_test -db db2 1.3.4 { PRAGMA integrity_check } {ok} +do_test 1.3.5 { hexio_read test.db 20 1 } {08} + +file_control_reservebytes db 16 +do_test 1.4.1 { hexio_read test.db 20 1 } {08} +do_execsql_test 1.4.2 { VACUUM } +do_execsql_test -db db2 1.4.3 { PRAGMA integrity_check } {ok} +do_test 1.4.4 { hexio_read test.db 20 1 } {10} + +finish_test diff --git a/test/rowvalue.test b/test/rowvalue.test index 387780c45..b6286302d 100644 --- a/test/rowvalue.test +++ b/test/rowvalue.test @@ -842,4 +842,17 @@ do_execsql_test 34.5 { ORDER BY t1.id; } {/SEARCH t1 USING COVERING INDEX t1a .a=. AND id>../} +# 2025-08-04 forum post eab63506cf +# +do_execsql_test 35.0 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(c1 TEXT, c2 INTEGER, PRIMARY KEY(c1, c2)); + INSERT INTO t1(c1, c2) VALUES ('a', 1); + SELECT ('a', 1) IN (SELECT c1, c2 from t1); +} 1 +do_execsql_test 35.1 { + SELECT (1, 'a') IN (SELECT c2, c1 from t1); +} 1 + + finish_test diff --git a/test/shell1.test b/test/shell1.test index 8cf11b240..abf214a90 100644 --- a/test/shell1.test +++ b/test/shell1.test @@ -945,6 +945,71 @@ do_test shell1-4.1.6 { } +# DELETE content of sqlite_sequence prior to repopulating, +# but only if the sqlite_sequence table is non-empty. +# Forum: 2024-10-13T17:10:01z and 2025-10-29T19:38:43z +# +do_test shell1-4.1.7 { + db close + forcedelete test2.db + sqlite3 db test2.db + db eval { + CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b); + INSERT INTO t1 VALUES(1,2),(20,21),(15,16); + } + catchcmd test2.db {.dump} +} {0 {PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b); +INSERT INTO t1 VALUES(1,2); +INSERT INTO t1 VALUES(15,16); +INSERT INTO t1 VALUES(20,21); +PRAGMA writable_schema=ON; +CREATE TABLE IF NOT EXISTS sqlite_sequence(name,seq); +DELETE FROM sqlite_sequence; +INSERT INTO sqlite_sequence VALUES('t1',20); +PRAGMA writable_schema=OFF; +COMMIT;}} +do_test shell1-4.1.8 { + db close + forcedelete test2.db + sqlite3 db test2.db + db eval { + CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b); + INSERT INTO t1 VALUES(1,2),(20,21),(15,16); + CREATE TABLE t2(x,y); + INSERT INTO t2 VALUES(99,88); + DROP TABLE t1; + } + catchcmd test2.db {.dump} +} {0 {PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE t2(x,y); +INSERT INTO t2 VALUES(99,88); +COMMIT;}} +do_test shell1-4.1.9 { + db close + forcedelete test2.db + sqlite3 db test2.db + db eval { + CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b); + INSERT INTO t1 VALUES(1,2),(20,21),(15,16); + CREATE TABLE t2(x,y); + INSERT INTO t2 VALUES(99,88); + INSERT INTO sqlite_sequence VALUES('extra',999); + DROP TABLE t1; + } + catchcmd test2.db {.dump} +} {0 {PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE t2(x,y); +INSERT INTO t2 VALUES(99,88); +PRAGMA writable_schema=ON; +CREATE TABLE IF NOT EXISTS sqlite_sequence(name,seq); +DELETE FROM sqlite_sequence; +INSERT INTO sqlite_sequence VALUES('extra',999); +PRAGMA writable_schema=OFF; +COMMIT;}} # Test the output of ".mode insert" # @@ -1304,4 +1369,22 @@ select base64(base64(cast('digity-doo' as blob))), } } {0 digity-doo|digity-doo} +#---------------------------------------------------------------------------- +# Test cases shell1-11.*: +# +do_test shell1-11.1 { + catchcmd :memory: { +.mode list +.header off +select base64(zeroblob(2000000000)); +} +} {/1.*too big.*/} +do_test shell1-11.2 { + catchcmd :memory: { +.mode list +.header off +select base85(zeroblob(2000000000)); +} +} {/1.*too big.*/} + finish_test diff --git a/test/shell2.test b/test/shell2.test index 3f9fec9ef..5f700a9a1 100644 --- a/test/shell2.test +++ b/test/shell2.test @@ -276,5 +276,97 @@ do_test shell2-1.4.12 { .sha3sum}]] } {0 ca08bc02b7e95c7df431a3a4b1cc0f8d8743914793473f55b5558e03} +#------------------------------------------------------------------------- + +foreach {tn hexdump expect} { + 0 { +| size 8192 pagesize 4096 filename my.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 03 00 00 00 02 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 ................ +| 96: 00 2e 8d f8 0d 00 00 00 01 0f df 00 0f df 00 00 ................ +| 4048: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1f ................ +| 4064: 01 06 17 0f 0f 01 2f 74 61 62 6c 65 74 74 02 43 ....../tablett.C +| 4080: 52 45 41 54 45 20 54 41 42 4c 45 20 74 28 78 29 REATE TABLE t(x) +| page 2 offset 4096 +| 0: 0d 00 00 00 02 0f ee 00 0f f7 0f ee 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 07 02 ................ +| 4080: 02 17 77 6f 72 6c 64 07 01 02 17 68 65 6c 6c 6f ..world....hello +| end my.db + } + {0 {}} + + 1 { +| size 2147483647 pagesize 4096 filename my.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 03 00 00 00 02 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 ................ +| 96: 00 2e 8d f8 0d 00 00 00 01 0f df 00 0f df 00 00 ................ +| 4048: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1f ................ +| 4064: 01 06 17 0f 0f 01 2f 74 61 62 6c 65 74 74 02 43 ....../tablett.C +| 4080: 52 45 41 54 45 20 54 41 42 4c 45 20 74 28 78 29 REATE TABLE t(x) +| page 2 offset 4096 +| 0: 0d 00 00 00 02 0f ee 00 0f f7 0f ee 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 07 02 ................ +| 4080: 02 17 77 6f 72 6c 64 07 01 02 17 68 65 6c 6c 6f ..world....hello +| end my.db + } + {1 {Error: out of memory}} + + 2 { +| size 8192 pagesize 4096 filename my.db +| page 1 offset 2147483647 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 03 00 00 00 02 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 ................ +| 96: 00 2e 8d f8 0d 00 00 00 01 0f df 00 0f df 00 00 ................ +| 4048: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1f ................ +| 4064: 01 06 17 0f 0f 01 2f 74 61 62 6c 65 74 74 02 43 ....../tablett.C +| 4080: 52 45 41 54 45 20 54 41 42 4c 45 20 74 28 78 29 REATE TABLE t(x) +| page 2 offset 4096 +| 0: 0d 00 00 00 02 0f ee 00 0f f7 0f ee 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 07 02 ................ +| 4080: 02 17 77 6f 72 6c 64 07 01 02 17 68 65 6c 6c 6f ..world....hello +| end my.db + } + {0 {}} + + 3 { +| size 8192 pagesize 4096 filename my.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 03 00 00 00 02 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 ................ +| 96: 00 2e 8d f8 0d 00 00 00 01 0f df 00 0f df 00 00 ................ +| 4048: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1f ................ +| 4064: 01 06 17 0f 0f 01 2f 74 61 62 6c 65 74 74 02 43 ....../tablett.C +| 4080: 52 45 41 54 45 20 54 41 42 4c 45 20 74 28 78 29 REATE TABLE t(x) +| page 2 offset 4096 +| 2147483647: 0d 00 00 00 02 0f ee 00 0f f7 0f ee 00 00 00 00 ................ +| 4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 07 02 ................ +| 4080: 02 17 77 6f 72 6c 64 07 01 02 17 68 65 6c 6c 6f ..world....hello +| end my.db +} + {0 {}} + +} { + set fd [open dump.txt w] + puts $fd [string trim $hexdump] + close $fd + do_test shell2-2.$tn.1 { + set rc [ catchcmd "" ".open --hexdb dump.txt"] + } $expect +} + finish_test diff --git a/test/shell4.test b/test/shell4.test index 4275911ef..3ced0702e 100644 --- a/test/shell4.test +++ b/test/shell4.test @@ -152,6 +152,6 @@ do_test shell4-4.1 { puts $fd ".read t1.txt" close $fd catchcmd ":memory:" ".read t1.txt" -} {1 {Input nesting limit (25) reached at line 1. Check recursion.}} +} {1 {t1.txt: Input nesting limit (25) reached at line 1. Check recursion.}} finish_test diff --git a/test/speedtest.tcl b/test/speedtest.tcl index 7cd3b5fa1..9cb81c0fc 100755 --- a/test/speedtest.tcl +++ b/test/speedtest.tcl @@ -116,7 +116,7 @@ for {set i 0} {$i<[llength $argv]} {incr i} { continue } if {[string match CC=* $arg]} { - set cc [lrange $arg 3 end] + set cc [string range $arg 3 end] continue } if {[string match *.c $arg]} { diff --git a/test/speedtest1.c b/test/speedtest1.c index 10cd30f1c..a127f62e9 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -159,6 +159,12 @@ static void fatal_error(const char *zMsg, ...){ va_start(ap, zMsg); vfprintf(stderr, zMsg, ap); va_end(ap); +#ifdef SQLITE_SPEEDTEST1_WASM + /* Emscripten complains when exit() is called and anything is left + in the I/O buffers. */ + fflush(stdout); + fflush(stderr); +#endif exit(1); } @@ -566,9 +572,8 @@ char *speedtest1_once(const char *zFormat, ...){ } rc = sqlite3_reset(pStmt); if( rc!=SQLITE_OK ){ - fprintf(stderr, "%s\nError code %d: %s\n", - sqlite3_sql(pStmt), rc, sqlite3_errmsg(g.db)); - exit(1); + fatal_error("%s\nError code %d: %s\n", + sqlite3_sql(pStmt), rc, sqlite3_errmsg(g.db)); } sqlite3_finalize(pStmt); } @@ -660,9 +665,8 @@ void speedtest1_run(void){ sqlite3_prepare_v2(g.db, sqlite3_sql(g.pStmt), -1, &pNew, 0); rc = sqlite3_finalize(g.pStmt); if( rc!=SQLITE_OK ){ - fprintf(stderr, "%s\nError code %d: %s\n", - sqlite3_sql(pNew), rc, sqlite3_errmsg(g.db)); - exit(1); + fatal_error("%s\nError code %d: %s\n", + sqlite3_sql(pNew), rc, sqlite3_errmsg(g.db)); } g.pStmt = pNew; }else @@ -670,9 +674,8 @@ void speedtest1_run(void){ { rc = sqlite3_reset(g.pStmt); if( rc!=SQLITE_OK ){ - fprintf(stderr, "%s\nError code %d: %s\n", - sqlite3_sql(g.pStmt), rc, sqlite3_errmsg(g.db)); - exit(1); + fatal_error("%s\nError code %d: %s\n", + sqlite3_sql(g.pStmt), rc, sqlite3_errmsg(g.db)); } } speedtest1_shrink_memory(); @@ -2146,7 +2149,7 @@ void testset_rtree(int p1, int p2){ } speedtest1_end_test(); } - + n = g.szTest*200; speedtest1_begin_test(120, "%d one-dimensional overlap slice queries", n); speedtest1_prepare("SELECT count(*) FROM rt1 WHERE y1>=?1 AND y0<=?2"); @@ -2175,7 +2178,6 @@ void testset_rtree(int p1, int p2){ } speedtest1_end_test(); } - n = g.szTest*200; speedtest1_begin_test(125, "%d custom geometry callback queries", n); @@ -2994,7 +2996,13 @@ int main(int argc, char **argv){ /* "mix1" is a macro testset: */ static char zMix1Tests[] = - "main,orm/25,cte/20,json,fp/3,parsenumber/25,rtree/10,star,app"; + "main,orm/25,cte/20,json,fp/3,parsenumber/25,rtree/10,star" +#if !defined(SQLITE_SPEEDTEST1_WASM) + ",app" + /* This test misbehaves in WASM builds: sqlite3_open_v2() is + failing to find the db file for reasons not yet understood. */ +#endif + ; #ifdef SQLITE_SPEEDTEST1_WASM /* Resetting all state is important for the WASM build, which may diff --git a/test/strict1.test b/test/strict1.test index fc6438843..c4c086bdb 100644 --- a/test/strict1.test +++ b/test/strict1.test @@ -162,4 +162,101 @@ do_execsql_test strict1-8.2 { SELECT *, '|' FROM t1; } {/5.0 5.0 4.6116\d*e\+18 4.6116\d+e\+18 |/} +# 2025-06-18 https://sqlite.org/forum/forumpost/6caf195248a849e4 +# +# Enforce STRICT table type constraints on STORED generated columns +# +do_execsql_test strict1-9.1 { + CREATE TABLE strict ( + k INTEGER PRIMARY KEY, + c1 REAL AS(if(k=11,1.5, k=12,2, k=13,'x', k=14,x'34', 0.0)) STORED, + c2 INT AS(if(k=21,1.5, k=22,2, k=23,'x', k=24,x'34', 0)) STORED, + c3 TEXT AS(if(k=31,1.5, k=32,2, k=33,'x', k=34,x'34', 'x')) STORED, + c4 BLOB AS(if(k=41,1.5, k=42,2, k=43,'x', k=44,x'34', x'00')) STORED, + c5 ANY AS(if(k=51,1.5, k=52,2, k=53,'x', k=54,x'34', 0)) STORED + ) STRICT; + INSERT INTO strict(k) VALUES(11); + INSERT INTO strict(k) VALUES(12); + INSERT INTO strict(k) VALUES(22); + INSERT INTO strict(k) VALUES(31); + INSERT INTO strict(k) VALUES(32); + INSERT INTO strict(k) VALUES(33); + INSERT INTO strict(k) VALUES(44); + PRAGMA integrity_check; +} {ok} +do_catchsql_test strict1-9.2.13 { + INSERT INTO strict(k) VALUES(13); +} {1 {cannot store TEXT value in REAL column strict.c1}} +do_catchsql_test strict1-9.2.14 { + INSERT INTO strict(k) VALUES(14); +} {1 {cannot store BLOB value in REAL column strict.c1}} +do_catchsql_test strict1-9.2.21 { + INSERT INTO strict(k) VALUES(21); +} {1 {cannot store REAL value in INT column strict.c2}} +do_catchsql_test strict1-9.2.23 { + INSERT INTO strict(k) VALUES(23); +} {1 {cannot store TEXT value in INT column strict.c2}} +do_catchsql_test strict1-9.2.24 { + INSERT INTO strict(k) VALUES(24); +} {1 {cannot store BLOB value in INT column strict.c2}} +do_catchsql_test strict1-9.2.34 { + INSERT INTO strict(k) VALUES(34); +} {1 {cannot store BLOB value in TEXT column strict.c3}} +do_catchsql_test strict1-9.2.41 { + INSERT INTO strict(k) VALUES(41); +} {1 {cannot store REAL value in BLOB column strict.c4}} +do_catchsql_test strict1-9.2.42 { + INSERT INTO strict(k) VALUES(42); +} {1 {cannot store INT value in BLOB column strict.c4}} +do_catchsql_test strict1-9.2.43 { + INSERT INTO strict(k) VALUES(43); +} {1 {cannot store TEXT value in BLOB column strict.c4}} + +do_execsql_test strict1-9.3 { + DROP TABLE strict; + CREATE TABLE strict ( + k INTEGER PRIMARY KEY, + c1 REAL AS(if(k=11,1.5, k=12,2, k=13,'x', k=14,x'34', 0.0)) VIRTUAL, + c2 INT AS(if(k=21,1.5, k=22,2, k=23,'x', k=24,x'34', 0)) VIRTUAL, + c3 TEXT AS(if(k=31,1.5, k=32,2, k=33,'x', k=34,x'34', 'x')) VIRTUAL, + c4 BLOB AS(if(k=41,1.5, k=42,2, k=43,'x', k=44,x'34', x'00')) VIRTUAL, + c5 ANY AS(if(k=51,1.5, k=52,2, k=53,'x', k=54,x'34', 0)) VIRTUAL + ) STRICT; + INSERT INTO strict(k) VALUES(11); + INSERT INTO strict(k) VALUES(12); + INSERT INTO strict(k) VALUES(22); + INSERT INTO strict(k) VALUES(31); + INSERT INTO strict(k) VALUES(32); + INSERT INTO strict(k) VALUES(33); + INSERT INTO strict(k) VALUES(44); + PRAGMA integrity_check; +} {ok} +do_catchsql_test strict1-9.4.13 { + INSERT INTO strict(k) VALUES(13); +} {1 {cannot store TEXT value in REAL column strict.c1}} +do_catchsql_test strict1-9.4.14 { + INSERT INTO strict(k) VALUES(14); +} {1 {cannot store BLOB value in REAL column strict.c1}} +do_catchsql_test strict1-9.4.21 { + INSERT INTO strict(k) VALUES(21); +} {1 {cannot store REAL value in INT column strict.c2}} +do_catchsql_test strict1-9.4.23 { + INSERT INTO strict(k) VALUES(23); +} {1 {cannot store TEXT value in INT column strict.c2}} +do_catchsql_test strict1-9.4.24 { + INSERT INTO strict(k) VALUES(24); +} {1 {cannot store BLOB value in INT column strict.c2}} +do_catchsql_test strict1-9.4.34 { + INSERT INTO strict(k) VALUES(34); +} {1 {cannot store BLOB value in TEXT column strict.c3}} +do_catchsql_test strict1-9.4.41 { + INSERT INTO strict(k) VALUES(41); +} {1 {cannot store REAL value in BLOB column strict.c4}} +do_catchsql_test strict1-9.4.42 { + INSERT INTO strict(k) VALUES(42); +} {1 {cannot store INT value in BLOB column strict.c4}} +do_catchsql_test strict1-9.4.43 { + INSERT INTO strict(k) VALUES(43); +} {1 {cannot store TEXT value in BLOB column strict.c4}} + finish_test diff --git a/test/tabfunc01.test b/test/tabfunc01.test index 0e29c3568..9a2017c46 100644 --- a/test/tabfunc01.test +++ b/test/tabfunc01.test @@ -22,7 +22,6 @@ ifcapable !vtab { return } load_static_extension db series -load_static_extension db carray load_static_extension db remember do_execsql_test tabfunc01-1.1 { @@ -278,6 +277,7 @@ do_execsql_test tabfunc01-600 { do_test tabfunc01-700 { set PTR1 [intarray_addr 5 7 13 17 23] + sqlite3_create_function db db eval { SELECT b FROM t600, carray(inttoptr($PTR1),5) WHERE a=value; } @@ -418,6 +418,66 @@ do_execsql_test tabfunc01-920 { ) LIMIT -1 OFFSET 0; } {1 2 3 4 5 6 7 8 9 10 101 102 103 104} +# Also: https://sqlite.org/forum/forumpost/ce1a06c2c8 +# +do_execsql_test tabfunc01-930 { + SELECT * FROM generate_series(9223372036854775807, + -9223372036854775808, + -9223372036854775808); +} {9223372036854775807 -1} +do_execsql_test tabfunc01-931 { + SELECT * FROM generate_series(9223372036854775807, + -9223372036854775808, + -9223372036854775808) + LIMIT 100 OFFSET 1; +} {-1} +do_execsql_test tabfunc01-932 { + SELECT * FROM generate_series(9223372036854775807, + -9223372036854775808, + -9223372036854775808) + LIMIT 100 OFFSET 2; +} {} +do_execsql_test tabfunc01-933 { + SELECT * FROM generate_series(9223372036854775807, + -9223372036854775808, + -9223372036854775808) + LIMIT 100 OFFSET 9223372036854775807; +} {} +do_execsql_test tabfunc01-940 { + SELECT * FROM generate_series(1,11,2); +} {1 3 5 7 9 11} +do_execsql_test tabfunc01-941 { + SELECT * FROM generate_series(1,11,2) LIMIT 100 OFFSET 4; +} {9 11} +do_execsql_test tabfunc01-942 { + SELECT * FROM generate_series(1,11,2) LIMIT 100 OFFSET 5; +} {11} +do_execsql_test tabfunc01-943 { + SELECT * FROM generate_series(1,11,2) LIMIT 100 OFFSET 6; +} {} +do_execsql_test tabfunc01-944 { + SELECT * FROM generate_series(1,11,2) LIMIT 100 OFFSET 7; +} {} +do_execsql_test tabfunc01-945 { + SELECT * FROM generate_series(1,11,2) LIMIT 100 OFFSET 9223372036854775807; +} {} +do_execsql_test tabfunc01-950 { + SELECT * FROM generate_series(1,11,1) LIMIT 100 OFFSET 10; +} {11} +do_execsql_test tabfunc01-951 { + SELECT * FROM generate_series(1,11,2) LIMIT 100 OFFSET 9223372036854775807; +} {} +do_execsql_test tabfunc01-952 { + SELECT * FROM generate_series(11,1,-1) LIMIT 100 OFFSET 10; +} {1} +do_execsql_test tabfunc01-953 { + SELECT * FROM generate_series(11,1,-1) LIMIT 100 OFFSET 9223372036854775807; +} {} +do_execsql_test tabfunc01-954 { + SELECT * FROM generate_series(1,11,1) LIMIT 3 OFFSET 3; +} {4 5 6} + + #------------------------------------------------------------------------- # Forum post https://sqlite.org/forum/forumpost/e7c3ae1215 # @@ -453,6 +513,163 @@ do_execsql_test 1200 { FROM t1 RIGHT JOIN generate_series(1,3,1) AS t2 USING(value); } {1 1 2 2 3 3} +#-------------------------------------------------------------------------- +# New test cases for generate_series associated with the refactor on branch +# serices-refactor circa 2025-09-27. +# +do_execsql_test 1300 { + SELECT value + FROM generate_series(-9223372036854775808, 9223372036854775807, 2) + WHERE value BETWEEN 1 AND 5; +} {2 4} +do_execsql_test 1301 { + SELECT value + FROM generate_series(-9223372036854775808, 9223372036854775807, 2) + WHERE value BETWEEN 0 AND 6; +} {0 2 4 6} +do_execsql_test 1302 { + SELECT value + FROM generate_series(-9223372036854775808, 9223372036854775807, 2) + WHERE value BETWEEN 0.5 AND 6.25; +} {2 4 6} +do_execsql_test 1303 { + SELECT value + FROM generate_series(-9223372036854775808, 9223372036854775807, 2) + WHERE value BETWEEN 9223372036854775803 AND 9223372036854775807 +} {9223372036854775804 9223372036854775806} +do_execsql_test 1304 { + SELECT value + FROM generate_series(-9223372036854775808, 9223372036854775807, 2) + WHERE value BETWEEN 9223372036854775803 AND 9223372036854775807 + ORDER BY value DESC; +} {9223372036854775806 9223372036854775804} + +do_execsql_test 1310 { + SELECT value FROM generate_series(0) LIMIT 5; +} {0 1 2 3 4} +do_execsql_test 1311 { + SELECT value FROM generate_series(5) LIMIT 5; +} {5 6 7 8 9} +do_execsql_test 1312 { + SELECT value FROM generate_series(0) WHERE stop=6; +} {0 1 2 3 4 5 6} +do_execsql_test 1313 { + SELECT value FROM generate_series(1,10) WHERE step IS NULL; +} {} +do_execsql_test 1314 { + SELECT value FROM generate_series(0xfffffffd); +} {4294967293 4294967294 4294967295} +do_execsql_test 1315 { + SELECT value FROM generate_series(4,3,1); +} {} +do_execsql_test 1316 { + SELECT value FROM generate_series(3,4,-1); +} {} + +do_execsql_test 1320 { + SELECT value FROM generate_series(-9223372036854775808,+9223372036854775807) + WHERE value=9.2234e18; +} {} +do_execsql_test 1321 { + SELECT value FROM generate_series(-9223372036854775808,+9223372036854775807) + WHERE value=-9.2234e18; +} {} +do_execsql_test 1322 { + SELECT value FROM generate_series(-9223372036854775808,+9223372036854775807) + WHERE value>9223372036854775807; +} {} +do_execsql_test 1323 { + SELECT value FROM generate_series(-9223372036854775808,+9223372036854775807) + WHERE value>9223372036854775806; +} {9223372036854775807} +do_execsql_test 1324 { + SELECT value FROM generate_series(-9223372036854775808,+9223372036854775807) + WHERE value>=9223372036854775807; +} {9223372036854775807} + +do_execsql_test 1330 { + SELECT value FROM generate_series(-9223372036854775808,+9223372036854775807) + WHERE value<-9223372036854775808; +} {} +do_execsql_test 1331 { + SELECT value FROM generate_series(-9223372036854775808,+9223372036854775807) + WHERE value<=-9223372036854775808; +} {-9223372036854775808} +do_execsql_test 1332 { + SELECT value FROM generate_series(-9223372036854775808,+9223372036854775807) + WHERE value<-9223372036854775807; +} {-9223372036854775808} +do_execsql_test 1333 { + SELECT value FROM generate_series(-9223372036854775808,+9223372036854775807) + WHERE value BETWEEN 4 AND 1; +} {} + +do_execsql_test 1340 { + SELECT value FROM generate_series(100,0,-10) + WHERE value BETWEEN 33 AND 66; +} {60 50 40} +do_execsql_test 1341 { + SELECT value FROM generate_series(100,0,-10) + WHERE value BETWEEN 33 AND 60; +} {60 50 40} +do_execsql_test 1342 { + SELECT value FROM generate_series(100,0,-10) + WHERE value BETWEEN 33 AND 59; +} {50 40} +do_execsql_test 1343 { + SELECT value + FROM generate_series(-9223372036854760000,-9223372036854775808,-10000); +} {-9223372036854760000 -9223372036854770000} +do_execsql_test 1344 { + SELECT value + FROM generate_series(-9223372036854760000,-9223372036854775808,-10000) + WHERE value < -9223372036854770001; +} {} +do_execsql_test 1345 { + SELECT value + FROM generate_series(9223372036854760000,9223372036854775807,10000); +} {9223372036854760000 9223372036854770000} +do_execsql_test 1346 { + SELECT value + FROM generate_series(9223372036854760000,9223372036854775807,10000) + WHERE value > 9223372036854770001; +} {} + +do_execsql_test 1350 { + SELECT value FROM generate_series(100,0,-10) + WHERE value BETWEEN 33 AND 38; +} {} +do_execsql_test 1351 { + SELECT value FROM generate_series(0,100,+10) + WHERE value BETWEEN 33 AND 38; +} {} + +do_execsql_test 1360 { + SELECT * FROM generate_series(0,-9223372036854775808,-9223372036854775808); +} {0 -9223372036854775808} +do_execsql_test 1361 { + SELECT * FROM generate_series(0,-9223372036854775808,-9223372036854775808) + LIMIT 1 OFFSET 0; +} {0} +do_execsql_test 1362 { + SELECT * FROM generate_series(0,-9223372036854775808,-9223372036854775808) + ORDER BY value ASC; +} {-9223372036854775808 0} +do_execsql_test 1363 { + SELECT * FROM generate_series(0,-9223372036854775808,-9223372036854775808) + ORDER BY value ASC LIMIT 10 OFFSET 1; +} {0} +do_execsql_test 1364 { + SELECT * FROM generate_series(0,-9223372036854775808,-9223372036854775808) + ORDER BY value ASC LIMIT 10 OFFSET 40000000; +} {} + +do_execsql_test 1370 { + SELECT * FROM generate_series(0,0,0); +} {} + + + # Free up memory allocations intarray_addr int64array_addr diff --git a/test/tclsqlite.test b/test/tclsqlite.test index 0758abd82..5f373ea18 100644 --- a/test/tclsqlite.test +++ b/test/tclsqlite.test @@ -9,7 +9,7 @@ # #*********************************************************************** # This file implements regression tests for TCL interface to the -# SQLite library. +# SQLite library. # # Actually, all tests are based on the TCL interface, so the main # interface is pretty well tested. This file contains some addition @@ -121,7 +121,7 @@ ifcapable {complete} { do_test tcl-1.14 { set v [catch {db eval} msg] lappend v $msg -} {1 {wrong # args: should be "db eval ?OPTIONS? SQL ?ARRAY-NAME? ?SCRIPT?"}} +} {1 {wrong # args: should be "db eval ?OPTIONS? SQL ?VAR-NAME? ?SCRIPT?"}} do_test tcl-1.15 { set v [catch {db function} msg] lappend v $msg @@ -359,6 +359,19 @@ do_test tcl-9.3 { execsql {SELECT typeof(ret_int())} } {integer} +proc breakAsNullUdf args { + if {"1" eq [lindex $args 0]} {return -code break} +} +do_test tcl-9.4 { + db function banu breakAsNullUdf + execsql {SELECT typeof(banu()), typeof(banu(1))} +} {text null} +do_test tcl-9.5 { + db nullvalue banunull + db eval {SELECT banu(), banu(1)} +} {{} banunull} + + # Recursive calls to the same user-defined function # ifcapable tclvar { @@ -465,7 +478,7 @@ do_test tcl-10.13 { db eval {SELECT * FROM t4} } {1 2 5 6 7} -# Now test that [db transaction] commands may be nested with +# Now test that [db transaction] commands may be nested with # the expected results. # do_test tcl-10.14 { @@ -475,7 +488,7 @@ do_test tcl-10.14 { INSERT INTO t4 VALUES('one'); } - catch { + catch { db transaction { db eval { INSERT INTO t4 VALUES('two') } db transaction { @@ -674,11 +687,11 @@ do_test tcl-15.5 { } {0} -# 2017-06-26: The --withoutnulls flag to "db eval". +# 2017-06-26: The -withoutnulls flag to "db eval". # -# In the "db eval --withoutnulls SQL ARRAY" form, NULL results cause the -# corresponding array entry to be unset. The default behavior (without -# the -withoutnulls flags) is for the corresponding array value to get +# In the "db eval -withoutnulls SQL TARGET" form, NULL results cause the +# corresponding target entry to be unset. The default behavior (without +# the -withoutnulls flags) is for the corresponding target value to get # the [db nullvalue] string. # catch {db close} @@ -720,64 +733,64 @@ reset_db proc add {a b} { return [expr $a + $b] } proc ret {a} { return $a } -db function add_i -returntype integer add +db function add_i -returntype integer add db function add_r -ret real add -db function add_t -return text add -db function add_b -returntype blob add -db function add_a -returntype any add +db function add_t -return text add +db function add_b -returntype blob add +db function add_a -returntype any add -db function ret_i -returntype int ret +db function ret_i -returntype int ret db function ret_r -returntype real ret -db function ret_t -returntype text ret -db function ret_b -returntype blob ret -db function ret_a -r any ret +db function ret_t -returntype text ret +db function ret_b -returntype blob ret +db function ret_a -r any ret do_execsql_test 17.0 { SELECT quote( add_i(2, 3) ); - SELECT quote( add_r(2, 3) ); - SELECT quote( add_t(2, 3) ); - SELECT quote( add_b(2, 3) ); - SELECT quote( add_a(2, 3) ); + SELECT quote( add_r(2, 3) ); + SELECT quote( add_t(2, 3) ); + SELECT quote( add_b(2, 3) ); + SELECT quote( add_a(2, 3) ); } {5 5.0 '5' X'35' 5} do_execsql_test 17.1 { SELECT quote( add_i(2.2, 3.3) ); - SELECT quote( add_r(2.2, 3.3) ); - SELECT quote( add_t(2.2, 3.3) ); - SELECT quote( add_b(2.2, 3.3) ); - SELECT quote( add_a(2.2, 3.3) ); + SELECT quote( add_r(2.2, 3.3) ); + SELECT quote( add_t(2.2, 3.3) ); + SELECT quote( add_b(2.2, 3.3) ); + SELECT quote( add_a(2.2, 3.3) ); } {5.5 5.5 '5.5' X'352E35' 5.5} do_execsql_test 17.2 { SELECT quote( ret_i(2.5) ); - SELECT quote( ret_r(2.5) ); - SELECT quote( ret_t(2.5) ); - SELECT quote( ret_b(2.5) ); - SELECT quote( ret_a(2.5) ); + SELECT quote( ret_r(2.5) ); + SELECT quote( ret_t(2.5) ); + SELECT quote( ret_b(2.5) ); + SELECT quote( ret_a(2.5) ); } {2.5 2.5 '2.5' X'322E35' 2.5} do_execsql_test 17.3 { SELECT quote( ret_i('2.5') ); - SELECT quote( ret_r('2.5') ); - SELECT quote( ret_t('2.5') ); - SELECT quote( ret_b('2.5') ); - SELECT quote( ret_a('2.5') ); + SELECT quote( ret_r('2.5') ); + SELECT quote( ret_t('2.5') ); + SELECT quote( ret_b('2.5') ); + SELECT quote( ret_a('2.5') ); } {2.5 2.5 '2.5' X'322E35' '2.5'} do_execsql_test 17.4 { SELECT quote( ret_i('abc') ); - SELECT quote( ret_r('abc') ); - SELECT quote( ret_t('abc') ); - SELECT quote( ret_b('abc') ); - SELECT quote( ret_a('abc') ); + SELECT quote( ret_r('abc') ); + SELECT quote( ret_t('abc') ); + SELECT quote( ret_b('abc') ); + SELECT quote( ret_a('abc') ); } {'abc' 'abc' 'abc' X'616263' 'abc'} do_execsql_test 17.5 { SELECT quote( ret_i(X'616263') ); - SELECT quote( ret_r(X'616263') ); - SELECT quote( ret_t(X'616263') ); - SELECT quote( ret_b(X'616263') ); - SELECT quote( ret_a(X'616263') ); + SELECT quote( ret_r(X'616263') ); + SELECT quote( ret_t(X'616263') ); + SELECT quote( ret_b(X'616263') ); + SELECT quote( ret_a(X'616263') ); } {'abc' 'abc' 'abc' X'616263' X'616263'} do_test 17.6.1 { @@ -848,21 +861,70 @@ do_catchsql_test 19.911 { } {1 {invalid command name "bind_fallback_does_not_exist"}} db bind_fallback {} -#------------------------------------------------------------------------- +# 2025-05-05: the -asdict eval flag +# do_test 20.0 { + execsql {CREATE TABLE tad(a,b)} + execsql {INSERT INTO tad(a,b) VALUES('aa','bb'),('AA','BB')} + db eval -asdict { + SELECT a, b FROM tad WHERE 0 + } D {} + set D +} {* {a b}} + +do_test 20.1 { + unset D + set i 0 + set res {} + set colNames {} + db eval -asdict { + SELECT a, b FROM tad ORDER BY a + } D { + dict set D i [incr i] + lappend res $i [dict get $D a] [dict get $D b] + if {1 == $i} { + set colNames [dict get $D *] + } + } + lappend res $colNames + unset D + set res +} {1 AA BB 2 aa bb {a b}} + +do_test 20.2 { + set res {} + db eval -asdict -withoutnulls { + SELECT n, a, b FROM ( + SELECT 1 as n, 'aa' as a, NULL as b + UNION ALL + SELECT 2 as n, NULL as a, 'bb' as b + ) + ORDER BY n + } D { + dict unset D * + lappend res [dict values $D] + } + unset D + execsql {DROP TABLE tad} + set res +} {{1 aa} {2 bb}} + +#------------------------------------------------------------------------- +do_test 21.0 { db transaction { db close } } {} -do_test 20.1 { +do_test 21.1 { sqlite3 db test.db set rc [catch { db eval {SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3} { db close } } msg] list $rc $msg } {1 {invalid command name "db"}} - + + proc closedb {} { db close @@ -874,7 +936,7 @@ sqlite3 db test.db db func closedb closedb db func func1 func1 -do_test 20.2 { +do_test 21.2 { set rc [catch { db eval { SELECT closedb(),func1() UNION ALL SELECT 20,30 UNION ALL SELECT 30,40 @@ -884,9 +946,10 @@ do_test 20.2 { } {0 {10 1 20 30 30 40}} sqlite3 db :memory: -do_test 21.1 { +do_test 22.1 { catch {db eval {SELECT 1 2 3;}} msg db erroroffset } {9} + finish_test diff --git a/test/testloadext.c b/test/testloadext.c new file mode 100644 index 000000000..94cd312e4 --- /dev/null +++ b/test/testloadext.c @@ -0,0 +1,98 @@ +/* +** 2025-10-14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Test the ability of run-time extension loading to use the +** very latest interfaces. +** +** Compile something like this: +** +** Linux: gcc -g -fPIC shared testloadext.c -o testloadext.so +** +** Mac: cc -g -fPIC -dynamiclib testloadext.c -o testloadext.dylib +** +** Win11: cl testloadext.c -link -dll -out:testloadext.dll +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include + +/* +** Implementation of the set_errmsg(CODE,MSG) SQL function. +** +** Raise an error that has numeric code CODE and text message MSG +** using the sqlite3_set_errmsg() API. +*/ +static void seterrmsgfunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db; + char *zRes; + int rc; + assert( argc==2 ); + db = sqlite3_context_db_handle(context); + rc = sqlite3_set_errmsg(db, + sqlite3_value_int(argv[0]), + sqlite3_value_text(argv[1])); + zRes = sqlite3_mprintf("%d %d %s", + rc, sqlite3_errcode(db), sqlite3_errmsg(db)); + sqlite3_result_text64(context, zRes, strlen(zRes), + SQLITE_TRANSIENT, SQLITE_UTF8); + sqlite3_free(zRes); +} + +/* +** Implementation of the tempbuf_spill() SQL function. +** +** Return the value of SQLITE_DBSTATUS_TEMPBUF_SPILL. +*/ +static void tempbuf_spill_func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db; + sqlite3_int64 iHi = 0, iCur = 0; + int rc; + int bReset; + assert( argc==1 ); + bReset = sqlite3_value_int(argv[0]); + db = sqlite3_context_db_handle(context); + (void)sqlite3_db_status64(db, SQLITE_DBSTATUS_TEMPBUF_SPILL, + &iCur, &iHi, bReset); + sqlite3_result_int64(context, iCur); +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_testloadext_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "set_errmsg", 2, + SQLITE_UTF8, + 0, seterrmsgfunc, 0, 0); + if( rc ) return rc; + rc = sqlite3_create_function(db, "tempbuf_spill", 1, + SQLITE_UTF8, + 0, tempbuf_spill_func, 0, 0); + if( rc ) return rc; + return SQLITE_OK; +} diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 0c6982f42..515036368 100755 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -7,6 +7,18 @@ set testdir [file normalize [file dirname $argv0]] set saved $argv set argv [list] source [file join $testdir testrunner_data.tcl] + +# Estimated amount of work required by displaytype, relative to 'tcl' +# +set estwork(tcl) 1 +set estwork(fuzz) 22 +set estwork(bld) 66 +set estwork(make) 102 + +set estworkfile [file join $testdir testrunner_estwork.tcl] +if {[file readable $estworkfile]} { + source $estworkfile +} source [file join $testdir permutations.test] set argv $saved cd $dir @@ -81,7 +93,7 @@ if {[info commands clock_milliseconds]==""} { proc usage {} { set a0 [file tail $::argv0] - puts stderr [string trim [subst -nocommands { + puts [string trim [subst -nocommands { Usage: $a0 ?SWITCHES? ?PERMUTATION? ?PATTERNS? $a0 PERMUTATION FILE @@ -92,9 +104,11 @@ Usage: $a0 script ?-msvc? CONFIG $a0 status ?-d SECS? ?--cls? $a0 halt + $a0 estwork where SWITCHES are: --buildonly Build test exes but do not run tests + --cases DISPLAYNAME Only run test that match DISPLAYNAME --config CONFIGS Only use configs on comma-separate list CONFIGS --dryrun Write what would have happened to testrunner.log --explain Write summary to stdout @@ -226,6 +240,7 @@ set TRG(explain) 0 ;# True for the --explain option set TRG(stopOnError) 0 ;# Stop running at first failure set TRG(stopOnCore) 0 ;# Stop on a core-dump set TRG(fullstatus) 0 ;# Full "status" report while running +set TRG(case) {} ;# Only run cases matching this GLOB pattern switch -nocase -glob -- $tcl_platform(os) { *darwin* { @@ -341,12 +356,14 @@ set TRG(schema) { endtime INTEGER, -- End time span INTEGER, -- Total run-time in milliseconds estwork INTEGER, -- Estimated amount of work + estkey TEXT, -- Key used to compute estwork state TEXT CHECK( state IN ('','ready','running','done','failed','omit','halt') ), ntest INT, -- Number of test cases run nerr INT, -- Number of errors reported svers TEXT, -- Reported SQLite version pltfm TEXT, -- Host platform reported - output TEXT -- test output + output TEXT, -- test output + cwd TEXT -- working directory for test ); CREATE TABLE config( @@ -359,13 +376,6 @@ set TRG(schema) { } #------------------------------------------------------------------------- -# Estimated amount of work required by displaytype, relative to 'tcl' -# -set estwork(tcl) 1 -set estwork(fuzz) 11 -set estwork(bld) 56 -set estwork(make) 97 - #-------------------------------------------------------------------------- # Check if this script is being invoked to run a single file. If so, # run it. @@ -452,6 +462,59 @@ if {[llength $argv]==1 } #-------------------------------------------------------------------------- +#-------------------------------------------------------------------------- +# Check if this is the "estwork" command: +# +# Generate (on standard output) a set of estwork() values based on the lastest +# test case, that can be used to replace the test/testrunner_estwork.tcl file. +# +if {[llength $argv]==1 + && [string compare -nocase estwork [lindex $argv 0]]==0 +} { + sqlite3 mydb $TRG(dbname) + set njob [mydb one {SELECT count(*) FROM jobs WHERE state='done'}] + if {$njob<1000} { + puts "Too few completed jobs to do a work estimate." + puts "Have $njob but not need at least 1000." + mydb close + exit 1 + } + set badjobs [mydb one {SELECT count(*) FROM jobs WHERE state<>'done'}] + if {$badjobs} { + puts "Database contains $badjobs incomplete jobs." + mydb close + exit 1 + } + set half [mydb one {SELECT count(*)/2 FROM jobs WHERE displaytype='tcl'}] + set scale [mydb one {SELECT span FROM jobs WHERE displaytype='tcl' + ORDER BY span LIMIT 1 OFFSET $half}] + mydb eval { + SELECT estkey, CAST(avg(span)/$scale AS INT) AS cost + FROM jobs + GROUP BY estkey + HAVING cost>=2 + } { + set estwork($estkey) $cost + } + set avgtcl [mydb one {SELECT avg(span) FROM jobs WHERE displaytype='tcl'}] + set estwork(tcl) 1 + foreach type {bld fuzz make} { + set avg [mydb one {SELECT avg(span) FROM jobs WHERE displaytype=$type}] + if {$avg!=""} { + set estwork($type) [expr {int($avg/$avgtcl)}] + } + } + mydb close + puts "# Estimated relative cost of various jobs, based on the \"estkey\" field." + puts "# Computed by the \"test/testrunner.tcl estwork\" command." + puts "#" + foreach key [lsort [array names estwork]] { + puts "set [list estwork($key)] $estwork($key)" + } + exit +} +#-------------------------------------------------------------------------- + #-------------------------------------------------------------------------- # Check if this is the "help" command: # @@ -562,8 +625,8 @@ proc show_status {db cls} { set srcdir [file dirname [file dirname $TRG(info_script)]] set line "Running: $S(running) (max: $nJob)" - if {$S(running)>0 && $fin>10} { - set tmleft [expr {($tm/$fin)*($totalw-$fin)}] + if {$S(running)>0 && [set pct [expr {int(($fin*100.0)/$totalw)}]]>=4} { + set tmleft [expr {($tm/double($fin))*($totalw-$fin)}] if {$tmleft<0.02*$tm} { set tmleft [expr {$tm*0.02}] } @@ -571,6 +634,7 @@ proc show_status {db cls} { if {[string length $line]+[string length $etc]<80} { append line $etc } + # append line " $pct%" } puts [format %-79.79s $line] if {$S(running)>0} { @@ -809,10 +873,13 @@ for {set ii 0} {$ii < [llength $argv]} {incr ii} { set TRG(dryrun) 1 } elseif {($n>2 && [string match "$a*" --explain]) || $a=="-e"} { set TRG(explain) 1 - } elseif {($n>2 && [string match "$a*" --omit]) || $a=="-c"} { + } elseif {$n>2 && [string match "$a*" --omit]} { incr ii set TRG(omitconfig) [lindex $argv $ii] - } elseif {($n>2 && [string match "$a*" --fuzzdb])} { + } elseif {$n>2 && [string match "$a*" --cases]} { + incr ii + set TRG(case) [lindex $argv $ii] + } elseif {$n>2 && [string match "$a*" --fuzzdb]} { incr ii set env(FUZZDB) [lindex $argv $ii] } elseif {[string match "$a*" --stop-on-error]} { @@ -931,8 +998,15 @@ proc r_get_next_job {iJob} { set T($iJob) $tm set jobid $job(jobid) + set cwd $job(dirname) + if {$cwd==""} { + set cwd [dirname $iJob] + } + trdb eval { - UPDATE jobs SET starttime=$tm, state='running' WHERE jobid=$jobid + UPDATE jobs + SET starttime=$tm, state='running', cwd=$cwd + WHERE jobid=$jobid } set ret [array get job] @@ -985,12 +1059,31 @@ proc add_job {args} { set state "" if {$A(-depid)==""} { set state ready } set type $A(-displaytype) - set ew $estwork($type) + set displayname $A(-displayname) + switch $type { + tcl { + set ek [file tail [lindex $displayname end]] + } + bld { + set ek [lindex $displayname end] + } + fuzz { + set ek [lrange $displayname 1 2] + } + make { + set ek [lindex $displayname end] + } + } + if {[info exists estwork($ek)]} { + set ew $estwork($ek) + } else { + set ew $estwork($type) + } trdb eval { INSERT INTO jobs( - displaytype, displayname, build, dirname, cmd, depid, priority, estwork, - state + displaytype, displayname, build, dirname, cmd, depid, priority, + estwork, estkey, state ) VALUES ( $type, $A(-displayname), @@ -1000,6 +1093,7 @@ proc add_job {args} { $A(-depid), $A(-priority), $ew, + $ek, $state ) } @@ -1361,6 +1455,31 @@ proc add_jobs_from_cmdline {patternlist} { } } } + + # If the "--case DISPLAYNAME" option appears on the command-line, mark + # all tests other than DISPLAYNAME as 'omit'. + # + if {[info exists TRG(case)] && $TRG(case) ne ""} { + set jid [trdb one { + SELECT jobid FROM jobs WHERE displayname GLOB $TRG(case) + }] + if {$jid eq ""} { + puts "ERROR: No jobs match \"$TRG(case)\"." + puts "The argument to --cases must GLOB match the jobs.displayname column" + puts "of the testrunner.db database." + trdb eval {UPDATE jobs SET state='omit'} + } else { + trdb eval { + WITH RECURSIVE keepers(jid,did) AS ( + SELECT jobid,depid FROM jobs + WHERE displayname GLOB $TRG(case) + UNION + SELECT jobid,depid FROM jobs, keepers WHERE jobid=did + ) + DELETE FROM jobs WHERE jobid NOT IN (SELECT jid FROM keepers); + } + } + } } proc make_new_testset {} { @@ -1586,12 +1705,13 @@ proc progress_report {} { } } set report "[elapsetime $tmms] [join $text { }]" - if {$wdone>0} { - set tmleft [expr {($tmms/$wdone)*($wtotal-$wdone)}] - set etc " ETC [elapsetime $tmleft]" + if {$wdone>0 && [set pct [expr {int(($wdone*100.0)/$wtotal)}]]>=4} { + set tmleft [expr {($tmms/double($wdone))*($wtotal-$wdone)}] + set etc " ETC [elapsetime $tmleft]" if {[string length $report]+[string length $etc]<80} { append report $etc } + # append report " $pct%" } puts -nonewline [format %-79.79s $report]\r flush stdout @@ -1629,6 +1749,7 @@ proc run_testset {} { } close $TRG(log) progress_report + puts "" r_write_db { set tm [clock_milliseconds] @@ -1648,7 +1769,7 @@ proc run_testset {} { } } - puts "\nTest database is $TRG(dbname)" + puts "Test database is $TRG(dbname)" puts "Test log is $TRG(logname)" if {[info exists TRG(FUZZDB)]} { puts "Extra fuzzcheck data taken from $TRG(FUZZDB)" @@ -1705,20 +1826,7 @@ proc explain_layer {indent depid} { puts "${indent}$displayname in $dirname" explain_layer "${indent} " $jobid } elseif {$showtests} { - if {[lindex $displayname end-3] eq "--slice"} { - set M [lindex $displayname end-2] - set N [lindex $displayname end-1] - set tail "[lindex $displayname end] (slice $M/$N)" - } else { - set tail [lindex $displayname end] - } - set e1 [lindex $displayname 1] - if {[string match config=* $e1]} { - set cfg [string range $e1 7 end] - puts "${indent}($cfg) $tail" - } else { - puts "${indent}$tail" - } + puts "${indent}$displayname" } } } diff --git a/test/testrunner_data.tcl b/test/testrunner_data.tcl index 3998bd9cc..e74caee1d 100644 --- a/test/testrunner_data.tcl +++ b/test/testrunner_data.tcl @@ -38,7 +38,7 @@ namespace eval trd { set tcltest(win.Windows-Win32Heap) veryquick set tcltest(win.Windows-Sanitize) veryquick set tcltest(win.Windows-WinRT) veryquick - set tcltest(win.Default) full + set tcltest(win.Default) {full win_unc_locking} # Extra [make xyz] tests that should be run for various builds. # diff --git a/test/testrunner_estwork.tcl b/test/testrunner_estwork.tcl new file mode 100644 index 000000000..c139394a5 --- /dev/null +++ b/test/testrunner_estwork.tcl @@ -0,0 +1,488 @@ +# Estimated relative cost of various jobs, based on the "estkey" field. +# Computed by the "test/testrunner.tcl estwork" command. +# +set estwork((fuzzcheck)) 1291 +set estwork((fuzzcheck-asan)) 2427 +set estwork((fuzzcheck-ubsan)) 2749 +set estwork((sessionfuzz)) 1276 +set estwork((sqlite3)) 1281 +set estwork((testfixture)) 1546 +set estwork(aggerror.test) 5 +set estwork(alter.test) 20 +set estwork(alter3.test) 2 +set estwork(alterauth.test) 5 +set estwork(altercol.test) 5 +set estwork(alterdropcol.test) 17 +set estwork(alterlegacy.test) 2 +set estwork(altertab.test) 34 +set estwork(altertab2.test) 2 +set estwork(altertab3.test) 12 +set estwork(amatch1.test) 40 +set estwork(atof1.test) 146 +set estwork(atomic2.test) 2 +set estwork(auth.test) 2 +set estwork(autoindex1.test) 5 +set estwork(autoindex3.test) 5 +set estwork(autovacuum.test) 15 +set estwork(avfs.test) 16 +set estwork(avtrans.test) 285 +set estwork(backup2.test) 15 +set estwork(bestindex4.test) 12 +set estwork(bigrow.test) 5 +set estwork(bitvec.test) 94 +set estwork(bld) 99 +set estwork(blob.test) 2 +set estwork(boundary1.test) 8 +set estwork(boundary2.test) 16 +set estwork(boundary3.test) 14 +set estwork(btree01.test) 20 +set estwork(busy.test) 2 +set estwork(busy2.test) 288 +set estwork(capi3.test) 5 +set estwork(capi3b.test) 6 +set estwork(capi3c.test) 5 +set estwork(capi3d.test) 8 +set estwork(capi3e.test) 20 +set estwork(changes.test) 11 +set estwork(chunksize.test) 5 +set estwork(closure01.test) 63 +set estwork(collate5.test) 2 +set estwork(conflict.test) 8 +set estwork(conflict2.test) 4 +set estwork(conflict3.test) 8 +set estwork(corrupt2.test) 2 +set estwork(corrupt4.test) 91 +set estwork(corrupt7.test) 5 +set estwork(corrupt8.test) 3 +set estwork(corruptB.test) 4 +set estwork(corruptF.test) 6 +set estwork(count.test) 18 +set estwork(crash8.test) 14 +set estwork(createtab.test) 3 +set estwork(csv01.test) 7 +set estwork(cursorhint.test) 4 +set estwork(date.test) 17 +set estwork(date4.test) 64 +set estwork(date5.test) 3 +set estwork(dbdata.test) 77 +set estwork(dbfuzz001.test) 2 +set estwork(dbstatus.test) 4 +set estwork(decimal.test) 2 +set estwork(delete.test) 4 +set estwork(diskfull.test) 34 +set estwork(e_blobbytes.test) 11 +set estwork(e_createtable.test) 14 +set estwork(e_delete.test) 3 +set estwork(e_droptrigger.test) 6 +set estwork(e_dropview.test) 2 +set estwork(e_expr.test) 251 +set estwork(e_select.test) 5 +set estwork(e_select2.test) 5 +set estwork(e_vacuum.test) 6 +set estwork(e_walauto.test) 51 +set estwork(e_walckpt.test) 6 +set estwork(enc.test) 2 +set estwork(enc3.test) 11 +set estwork(enc4.test) 3 +set estwork(eval.test) 2 +set estwork(exists.test) 4 +set estwork(existsexpr.test) 44 +set estwork(expert1.test) 14 +set estwork(expr.test) 5 +set estwork(filectrl.test) 5 +set estwork(filefmt.test) 6 +set estwork(fkey2.test) 9 +set estwork(fpconv1.test) 40 +set estwork(fts3aa.test) 18 +set estwork(fts3ad.test) 10 +set estwork(fts3ag.test) 6 +set estwork(fts3aj.test) 8 +set estwork(fts3am.test) 5 +set estwork(fts3auto.test) 29 +set estwork(fts3b.test) 41 +set estwork(fts3c.test) 22 +set estwork(fts3conf.test) 3 +set estwork(fts3corrupt4.test) 7 +set estwork(fts3corrupt5.test) 8 +set estwork(fts3corrupt6.test) 13 +set estwork(fts3d.test) 2 +set estwork(fts3defer2.test) 2 +set estwork(fts3expr.test) 10 +set estwork(fts3expr2.test) 8 +set estwork(fts3expr3.test) 145 +set estwork(fts3f.test) 9 +set estwork(fts3first.test) 3 +set estwork(fts3matchinfo.test) 4 +set estwork(fts3misc.test) 54 +set estwork(fts3prefix.test) 4 +set estwork(fts3prefix2.test) 8 +set estwork(fts3query.test) 23 +set estwork(fts3varint.test) 2 +set estwork(fts4aa.test) 34 +set estwork(fts4content.test) 5 +set estwork(fts4incr.test) 19 +set estwork(fts4noti.test) 3 +set estwork(fts4opt.test) 129 +set estwork(fts4unicode.test) 45 +set estwork(fts5aa.test) 442 +set estwork(fts5ab.test) 54 +set estwork(fts5ad.test) 33 +set estwork(fts5ae.test) 3 +set estwork(fts5af.test) 13 +set estwork(fts5ag.test) 13 +set estwork(fts5ah.test) 265 +set estwork(fts5al.test) 2 +set estwork(fts5auto.test) 92 +set estwork(fts5connect.test) 5 +set estwork(fts5content.test) 9 +set estwork(fts5contentless.test) 23 +set estwork(fts5contentless2.test) 189 +set estwork(fts5contentless4.test) 234 +set estwork(fts5contentless5.test) 9 +set estwork(fts5delete.test) 62 +set estwork(fts5doclist.test) 11 +set estwork(fts5expr.test) 10 +set estwork(fts5full.test) 42 +set estwork(fts5hash.test) 17 +set estwork(fts5integrity.test) 84 +set estwork(fts5interrupt.test) 16 +set estwork(fts5locale.test) 5 +set estwork(fts5matchinfo.test) 11 +set estwork(fts5merge.test) 29 +set estwork(fts5merge2.test) 9 +set estwork(fts5misc.test) 8 +set estwork(fts5multiclient.test) 2 +set estwork(fts5optimize.test) 13 +set estwork(fts5optimize2.test) 737 +set estwork(fts5optimize3.test) 280 +set estwork(fts5origintext.test) 144 +set estwork(fts5origintext3.test) 8 +set estwork(fts5origintext4.test) 30 +set estwork(fts5origintext5.test) 462 +set estwork(fts5origintext6.test) 55 +set estwork(fts5porter.test) 57 +set estwork(fts5query.test) 8 +set estwork(fts5restart.test) 10 +set estwork(fts5rowid.test) 35 +set estwork(fts5secure.test) 61 +set estwork(fts5secure3.test) 796 +set estwork(fts5secure4.test) 141 +set estwork(fts5secure5.test) 29 +set estwork(fts5secure6.test) 33 +set estwork(fts5secure7.test) 680 +set estwork(fts5simple.test) 7 +set estwork(fts5simple2.test) 2 +set estwork(fts5synonym.test) 13 +set estwork(fts5synonym2.test) 384 +set estwork(fts5tok2.test) 92 +set estwork(fts5tokenizer.test) 2 +set estwork(fts5tokenizer3.test) 8 +set estwork(fts5trigram.test) 2 +set estwork(fts5unicode2.test) 28 +set estwork(fts5unicode3.test) 72 +set estwork(fts5unindexed.test) 7 +set estwork(fts5update.test) 161 +set estwork(fts5update2.test) 11 +set estwork(fts5vocab.test) 11 +set estwork(fts5vocab2.test) 15 +set estwork(func.test) 36 +set estwork(fuzz) 68 +set estwork(fuzz-oss1.test) 11 +set {estwork(fuzzcheck --slice)} 499 +set {estwork(fuzzcheck fuzzdata1.db)} 301 +set {estwork(fuzzcheck fuzzdata2.db)} 275 +set {estwork(fuzzcheck fuzzdata3.db)} 72 +set {estwork(fuzzcheck fuzzdata4.db)} 30 +set {estwork(fuzzcheck fuzzdata5.db)} 299 +set {estwork(fuzzcheck fuzzdata6.db)} 91 +set {estwork(fuzzcheck fuzzdata7.db)} 201 +set {estwork(fuzzcheck fuzzdata8.db)} 331 +set {estwork(fuzzcheck-asan --slice)} 2037 +set {estwork(fuzzcheck-asan fuzzdata1.db)} 3112 +set {estwork(fuzzcheck-asan fuzzdata2.db)} 8753 +set {estwork(fuzzcheck-asan fuzzdata3.db)} 459 +set {estwork(fuzzcheck-asan fuzzdata4.db)} 123 +set {estwork(fuzzcheck-asan fuzzdata5.db)} 1178 +set {estwork(fuzzcheck-asan fuzzdata6.db)} 1356 +set {estwork(fuzzcheck-asan fuzzdata7.db)} 938 +set {estwork(fuzzcheck-asan fuzzdata8.db)} 1451 +set {estwork(fuzzcheck-ubsan --slice)} 1729 +set {estwork(fuzzcheck-ubsan fuzzdata1.db)} 1180 +set {estwork(fuzzcheck-ubsan fuzzdata2.db)} 876 +set {estwork(fuzzcheck-ubsan fuzzdata3.db)} 306 +set {estwork(fuzzcheck-ubsan fuzzdata4.db)} 95 +set {estwork(fuzzcheck-ubsan fuzzdata5.db)} 1356 +set {estwork(fuzzcheck-ubsan fuzzdata6.db)} 333 +set {estwork(fuzzcheck-ubsan fuzzdata7.db)} 883 +set {estwork(fuzzcheck-ubsan fuzzdata8.db)} 1124 +set estwork(fuzzer1.test) 6 +set estwork(gencol1.test) 3 +set estwork(hook.test) 2 +set estwork(in4.test) 4 +set estwork(in7.test) 6 +set estwork(incrblob2.test) 2 +set estwork(incrblob3.test) 5 +set estwork(incrvacuum.test) 10 +set estwork(incrvacuum2.test) 17 +set estwork(incrvacuum3.test) 17 +set estwork(index.test) 3 +set estwork(index2.test) 8 +set estwork(index4.test) 33 +set estwork(index5.test) 99 +set estwork(index6.test) 2 +set estwork(indexexpr1.test) 2 +set estwork(insert3.test) 5 +set estwork(insert4.test) 2 +set estwork(intarray.test) 8 +set estwork(intck1.test) 34 +set estwork(intck2.test) 37 +set estwork(interrupt.test) 28 +set estwork(io.test) 3 +set estwork(join.test) 3 +set estwork(join3.test) 6 +set estwork(join5.test) 20 +set estwork(join7.test) 2 +set estwork(join8.test) 3 +set estwork(join9.test) 2 +set estwork(joinA.test) 11 +set estwork(joinB.test) 7 +set estwork(joinC.test) 6 +set estwork(joinD.test) 168 +set estwork(json103.test) 6 +set estwork(json106.test) 690 +set estwork(keyword1.test) 3 +set estwork(like2.test) 2 +set estwork(like3.test) 2 +set estwork(limit.test) 3 +set estwork(literal.test) 6 +set estwork(lock.test) 39 +set estwork(lock4.test) 2 +set estwork(lock5.test) 4 +set estwork(make) 102 +set estwork(manydb.test) 12 +set estwork(mem5.test) 2 +set estwork(memdb.test) 9 +set estwork(memdb1.test) 2 +set estwork(memjournal2.test) 164 +set estwork(memsubsys1.test) 8 +set estwork(misc1.test) 6 +set estwork(misc2.test) 6 +set estwork(misc5.test) 11 +set estwork(misc8.test) 5 +set estwork(mmap1.test) 8 +set estwork(mmap2.test) 10 +set estwork(mmapwarm.test) 2 +set estwork(multiplex.test) 30 +set estwork(multiplex2.test) 7 +set estwork(nan.test) 2 +set estwork(notify3.test) 5 +set estwork(orderby1.test) 11 +set estwork(orderby2.test) 2 +set estwork(orderby5.test) 11 +set estwork(orderby6.test) 6 +set estwork(orderby8.test) 20 +set estwork(orderbyA.test) 2 +set estwork(oserror.test) 7 +set estwork(ovfl.test) 11 +set estwork(pager1.test) 81 +set estwork(pager2.test) 129 +set estwork(pagesize.test) 3 +set estwork(percentile.test) 86 +set estwork(pragma.test) 4 +set estwork(pragma4.test) 18 +set estwork(printf.test) 21 +set estwork(printf2.test) 5 +set estwork(quota.test) 14 +set estwork(randexpr1.test) 35 +set estwork(rbu10.test) 16 +set estwork(rbu13.test) 5 +set estwork(rbuexlock.test) 5 +set estwork(rbumisc.test) 2 +set estwork(rbutemplimit.test) 2 +set estwork(readonly.test) 95 +set estwork(recover.test) 3 +set estwork(recover1.test) 7 +set estwork(recovercorrupt3.test) 5 +set estwork(recoverold.test) 7 +set estwork(recoverpgsz.test) 6 +set estwork(recoverrowid.test) 3 +set estwork(returning1.test) 6 +set estwork(rollback2.test) 2 +set estwork(round1.test) 261 +set estwork(rowhash.test) 85 +set estwork(rowid.test) 3 +set estwork(rowvalue.test) 2 +set estwork(rowvalue2.test) 38 +set estwork(rowvalue4.test) 2 +set estwork(rowvalueA.test) 2 +set estwork(rowvaluevtab.test) 2 +set estwork(rtree1.test) 4 +set estwork(rtree2.test) 758 +set estwork(rtree6.test) 6 +set estwork(rtree8.test) 15 +set estwork(rtree9.test) 15 +set estwork(rtreeA.test) 7 +set estwork(rtreeB.test) 7 +set estwork(rtreeE.test) 47 +set estwork(rtreeH.test) 24 +set estwork(rtreecheck.test) 2 +set estwork(rtreedoc.test) 8 +set estwork(rtreedoc3.test) 76 +set estwork(savepoint.test) 10 +set estwork(savepoint2.test) 57 +set estwork(schema2.test) 15 +set estwork(schema3.test) 2 +set estwork(schema5.test) 5 +set estwork(select1.test) 2 +set estwork(select2.test) 17 +set estwork(select3.test) 2 +set estwork(selectA.test) 2 +set estwork(selectB.test) 2 +set estwork(selectG.test) 30 +set estwork(session1.test) 5 +set estwork(session2.test) 33 +set estwork(session5.test) 63 +set estwork(session9.test) 3 +set estwork(sessionG.test) 60 +set estwork(sessionH.test) 18 +set estwork(sessionalter.test) 3 +set estwork(sessionat.test) 2 +set estwork(sessionblob.test) 3 +set {estwork(sessionfuzz sessionfuzz-data1.db)} 5 +set estwork(sessioninvert.test) 2 +set estwork(sessionnoop.test) 2 +set estwork(sessionnoop2.test) 8 +set estwork(sessionrebase.test) 8 +set estwork(shared.test) 7 +set estwork(sharedA.test) 48 +set estwork(shell1.test) 36 +set estwork(shell2.test) 11 +set estwork(shell3.test) 3 +set estwork(shell4.test) 2 +set estwork(shell5.test) 16 +set estwork(shell6.test) 3 +set estwork(shell8.test) 104 +set estwork(shell9.test) 3 +set estwork(shellA.test) 2 +set estwork(shmlock.test) 27 +set estwork(sidedelete.test) 10 +set estwork(skipscan1.test) 7 +set estwork(skipscan2.test) 5 +set estwork(sort.test) 38 +set estwork(sort2.test) 540 +set estwork(sort5.test) 16 +set estwork(spellfix.test) 5 +set estwork(spellfix2.test) 5 +set estwork(spellfix4.test) 11 +set estwork(starschema1.test) 2 +set estwork(strict1.test) 5 +set estwork(swarmvtab.test) 110 +set estwork(swarmvtab3.test) 14 +set estwork(syscall.test) 4 +set estwork(table.test) 62 +set estwork(tableapi.test) 8 +set estwork(tcl) 1 +set estwork(tclsqlite.test) 29 +set estwork(temptable2.test) 274 +set estwork(thread3.test) 21 +set estwork(timediff1.test) 5 +set estwork(tkt-2d1a5c67d.test) 6 +set estwork(tkt-38cb5df375.test) 2 +set estwork(tkt-4dd95f6943.test) 2 +set estwork(tkt-5e10420e8d.test) 5 +set estwork(tkt-6bfb98dfc0.test) 5 +set estwork(tkt-80e031a00f.test) 25 +set estwork(tkt-9d68c883.test) 3 +set estwork(tkt-9f2eb3abac.test) 5 +set estwork(tkt-b1d3a2e531.test) 5 +set estwork(tkt-b72787b1.test) 5 +set estwork(tkt-b75a9ca6b0.test) 8 +set estwork(tkt-d11f09d36e.test) 9 +set estwork(tkt-fc62af4523.test) 10 +set estwork(tkt1435.test) 7 +set estwork(tkt1644.test) 5 +set estwork(tkt1667.test) 20 +set estwork(tkt1873.test) 2 +set estwork(tkt2192.test) 3 +set estwork(tkt2285.test) 19 +set estwork(tkt2332.test) 2 +set estwork(tkt2409.test) 24 +set estwork(tkt3334.test) 5 +set estwork(tkt3357.test) 10 +set estwork(tkt3630.test) 2 +set estwork(tkt3832.test) 8 +set estwork(tkt3838.test) 5 +set estwork(tkt3918.test) 3 +set estwork(tkt4018.test) 48 +set estwork(tokenize.test) 9 +set estwork(tpch01.test) 6 +set estwork(trans.test) 258 +set estwork(trigger2.test) 15 +set estwork(trigger5.test) 2 +set estwork(trigger8.test) 30 +set estwork(triggerA.test) 36 +set estwork(triggerB.test) 4 +set estwork(triggerC.test) 67 +set estwork(triggerD.test) 3 +set estwork(types.test) 2 +set estwork(types2.test) 2 +set estwork(unionall2.test) 9 +set estwork(unionvtab.test) 2 +set estwork(update.test) 7 +set estwork(upsert3.test) 6 +set estwork(upsert5.test) 5 +set estwork(vacuum.test) 2 +set estwork(vacuum5.test) 3 +set estwork(vacuum6.test) 174 +set estwork(vacuummem.test) 132 +set estwork(varint.test) 8 +set estwork(view.test) 2 +set estwork(vtab1.test) 13 +set estwork(vtab6.test) 12 +set estwork(vtabC.test) 33 +set estwork(vtabD.test) 56 +set estwork(vtab_alter.test) 6 +set estwork(wal.test) 38 +set estwork(wal2.test) 7 +set estwork(wal3.test) 196 +set estwork(wal4.test) 70 +set estwork(wal5.test) 22 +set estwork(wal64k.test) 20 +set estwork(wal7.test) 3 +set estwork(wal9.test) 24 +set estwork(walbak.test) 2 +set estwork(walcksum.test) 3 +set estwork(walcrash4.test) 59 +set estwork(waloverwrite.test) 3 +set estwork(walpersist.test) 6 +set estwork(walprotocol2.test) 2 +set estwork(walro2.test) 11 +set estwork(walsetlk.test) 1610 +set estwork(walsetlk3.test) 3 +set estwork(walsetlk_recover.test) 193 +set estwork(walshared.test) 5 +set estwork(walvfs.test) 476 +set estwork(where.test) 24 +set estwork(where3.test) 4 +set estwork(where6.test) 2 +set estwork(where7.test) 20 +set estwork(where8.test) 95 +set estwork(where9.test) 16 +set estwork(whereB.test) 5 +set estwork(whereE.test) 6 +set estwork(whereN.test) 2 +set estwork(window1.test) 5 +set estwork(window2.test) 8 +set estwork(window3.test) 89 +set estwork(window4.test) 3 +set estwork(window5.test) 2 +set estwork(window8.test) 33 +set estwork(windowA.test) 5 +set estwork(windowD.test) 6 +set estwork(with1.test) 114 +set estwork(with2.test) 4 +set estwork(with5.test) 2 +set estwork(withM.test) 4 +set estwork(without_rowid1.test) 2 +set estwork(without_rowid3.test) 9 +set estwork(without_rowid4.test) 6 diff --git a/test/vacuum.test b/test/vacuum.test index 57429c29e..82dd00d09 100644 --- a/test/vacuum.test +++ b/test/vacuum.test @@ -401,4 +401,25 @@ do_test vacuum-10.1 { } {} do_test vacuum-10.2 { execsql VACUUM } {} +# Verify that VACUUM still works if ATTACH is disabled. +# +do_execsql_test vacuum-11.1 { + PRAGMA page_size=1024; + VACUUM; + PRAGMA page_size; +} {1024} +sqlite3_db_config db ATTACH_CREATE 0 +do_execsql_test vacuum-11.2 { + PRAGMA page_size=2048; + VACUUM; + PRAGMA page_size; +} {2048} +sqlite3_db_config db ATTACH_CREATE 1 +sqlite3_db_config db ATTACH_WRITE 0 +do_execsql_test vacuum-11.3 { + PRAGMA page_size=4096; + VACUUM; + PRAGMA page_size; +} {4096} + finish_test diff --git a/test/vtabH.test b/test/vtabH.test index 07704cefb..1496f49b5 100644 --- a/test/vtabH.test +++ b/test/vtabH.test @@ -190,10 +190,10 @@ if {$tcl_platform(platform) ne "windows" || \ lappend res "/$p" } } - set num_root_files [llength $root_files] + set num_root_files [llength $res] do_test 3.1 { sort_files [execsql { - SELECT path FROM fstree WHERE path NOT GLOB '*\$*' LIMIT $num_root_files; + SELECT path FROM fstree WHERE path NOT GLOB '*$*' LIMIT $num_root_files }] true } [sort_files $res true] diff --git a/test/walckptnoop.test b/test/walckptnoop.test new file mode 100644 index 000000000..7ff8e90b8 --- /dev/null +++ b/test/walckptnoop.test @@ -0,0 +1,110 @@ +# 2025 September 5 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA wal_checkpoint = noop" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl +source $testdir/wal_common.tcl + +set testprefix walckptnoop + +ifcapable !wal {finish_test ; return } + +set VAL 123 + +proc myrand {} { + global VAL + + set A 1103515245 + set C 12345 + set M 2147483648 + + set VAL [expr {($A * $VAL + $C) % $M}] + return $VAL +} + +proc myrandomblob {n} { + set l [list] + for {set i 0} {$i < $n} {incr i} { + lappend l [expr [myrand] % 256] + } + binary format c* $l +} + +db func myrandomblob myrandomblob + + +do_execsql_test 1.0 { + PRAGMA page_size=1024; + PRAGMA auto_vacuum=NONE; + PRAGMA secure_delete=OFF; + VACUUM; + CREATE TABLE t1(x INTEGER PRIMARY KEY, y TEXT); + CREATE INDEX i1 ON t1(y); + PRAGMA journal_mode = wal; + + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) + INSERT INTO t1 SELECT NULL, hex(myrandomblob(64)) FROM s; +} {0 wal} + +do_execsql_test 1.1 { + PRAGMA wal_checkpoint = noop; +} {0 298 0} +do_execsql_test 1.2 { + PRAGMA wal_checkpoint = noop; +} {0 298 0} +do_execsql_test 1.3 { + PRAGMA wal_checkpoint = passive; +} {0 298 298} +do_execsql_test 1.4 { + PRAGMA wal_checkpoint = noop; +} {0 298 298} + +db_save_and_close +db_restore_and_reopen +do_execsql_test 1.5 { + PRAGMA wal_checkpoint = noop; +} {0 298 0} + +db close +sqlite3 db test.db +db eval { + PRAGMA auto_vacuum=NONE; + PRAGMA secure_delete=OFF; +} +do_execsql_test 1.6 { + PRAGMA wal_checkpoint = noop; +} {0 0 0} + +do_catchsql_test 1.7 { + BEGIN; + DELETE FROM t1; + PRAGMA wal_checkpoint = noop; +} {1 {database table is locked}} + +do_catchsql_test 1.8 { + COMMIT; + PRAGMA wal_checkpoint = noop; +} {0 {0 5 0}} + +do_execsql_test 1.9 { + PRAGMA journal_mode = delete; + PRAGMA wal_checkpoint = noop; +} {delete 0 -1 -1} + +finish_test diff --git a/test/walthread.test b/test/walthread.test index 8e5df9e58..1a0c35af0 100644 --- a/test/walthread.test +++ b/test/walthread.test @@ -19,6 +19,7 @@ source $testdir/tester.tcl source $testdir/lock_common.tcl if {[run_thread_tests]==0} { finish_test ; return } ifcapable !wal { finish_test ; return } +set DBNAME wt-[expr {int(abs(rand()*100000000))}]-test.db set sqlite_walsummary_mmap_incr 64 @@ -74,7 +75,6 @@ proc lshift {lvar} { # -check SCRIPT Script to run after test. # proc do_thread_test {args} { - set A $args set P(testname) [lshift A] @@ -126,12 +126,12 @@ proc do_thread_test {args} { return } - puts "Running $P(testname) for $P(seconds) seconds..." + puts "Running $P(testname) for $P(seconds) seconds on $::DBNAME" catch { db close } - forcedelete test.db test.db-journal test.db-wal + forcedelete $::DBNAME $::DBNAME-journal $::DBNAME-wal - sqlite3 db test.db + sqlite3 db $::DBNAME eval $P(init) catch { db close } @@ -146,7 +146,7 @@ proc do_thread_test {args} { set E(nthread) $count set E(seconds) $P(seconds) " - set program [string map [list %TEST% $prg %VARS% $vars] { + set program [string map [list %TEST% $prg %VARS% $vars %DB% $::DBNAME] { %VARS% @@ -163,7 +163,7 @@ proc do_thread_test {args} { proc busyhandler {n} { usleep 10 ; return 0 } - sqlite3 db test.db + sqlite3 db %DB% db busy busyhandler db eval { SELECT randomblob($E(pid)*5) } @@ -202,7 +202,7 @@ proc do_thread_test {args} { } puts $report - sqlite3 db test.db + sqlite3 db $::DBNAME set res "" if {[catch $P(check) msg]} { set res $msg } do_test $P(testname).check [list set {} $res] "" @@ -327,16 +327,16 @@ do_thread_test2 walthread-1 -seconds $seconds(walthread-1) -init { # the number of write-transactions performed using a rollback journal. # For example, "192 w, 185 r". # -if {[atomic_batch_write test.db]==0} { +if {[atomic_batch_write $::DBNAME]==0} { do_thread_test2 walthread-2 -seconds $seconds(walthread-2) -init { execsql { CREATE TABLE t1(x INTEGER PRIMARY KEY, y UNIQUE) } - } -thread RB 2 { + } -thread RB 2 [string map [list %DB% $::DBNAME] { db close set nRun 0 set nDel 0 while {[tt_continue]} { - sqlite3 db test.db + sqlite3 db %DB% db busy busyhandler db eval { SELECT * FROM sqlite_master } catch { db eval { PRAGMA journal_mode = DELETE } } @@ -345,8 +345,8 @@ if {[atomic_batch_write test.db]==0} { INSERT INTO t1 VALUES(NULL, randomblob(100+$E(pid))); } incr nRun 1 - incr nDel [file exists test.db-journal] - if {[file exists test.db-journal] + [file exists test.db-wal] != 1} { + incr nDel [file exists %DB%-journal] + if {[file exists %DB%-journal] + [file exists %DB%-wal] != 1} { error "File-system looks bad..." } db eval COMMIT @@ -357,12 +357,12 @@ if {[atomic_batch_write test.db]==0} { list $nRun $nDel set {} "[expr $nRun-$nDel] w, $nDel r" - } -thread WAL 2 { + }] -thread WAL 2 [string map [list %DB% $::DBNAME] { db close set nRun 0 set nDel 0 while {[tt_continue]} { - sqlite3 db test.db + sqlite3 db %DB% db busy busyhandler db eval { SELECT * FROM sqlite_master } catch { db eval { PRAGMA journal_mode = WAL } } @@ -371,8 +371,8 @@ if {[atomic_batch_write test.db]==0} { INSERT INTO t1 VALUES(NULL, randomblob(110+$E(pid))); } incr nRun 1 - incr nDel [file exists test.db-journal] - if {[file exists test.db-journal] + [file exists test.db-wal] != 1} { + incr nDel [file exists %DB%-journal] + if {[file exists %DB%-journal] + [file exists %DB%-wal] != 1} { error "File-system looks bad..." } db eval COMMIT @@ -381,7 +381,7 @@ if {[atomic_batch_write test.db]==0} { db close } set {} "[expr $nRun-$nDel] w, $nDel r" - } + }] } do_thread_test walthread-3 -seconds $seconds(walthread-3) -init { @@ -510,14 +510,14 @@ do_thread_test walthread-5 -seconds $seconds(walthread-5) -init { COMMIT; } - forcecopy test.db-wal bak.db-wal - forcecopy test.db bak.db + forcecopy $::DBNAME-wal $::DBNAME-bak.db-wal + forcecopy $::DBNAME $::DBNAME-bak.db db close - forcecopy bak.db-wal test.db-wal - forcecopy bak.db test.db + forcecopy $::DBNAME-bak.db-wal $::DBNAME-wal + forcecopy $::DBNAME-bak.db $::DBNAME - if {[file size test.db-wal] < [log_file_size [expr 64*1024] 1024]} { + if {[file size $::DBNAME-wal] < [log_file_size [expr 64*1024] 1024]} { error "Somehow failed to create a large log file" } puts "Database with large log file recovered. Now running clients..." @@ -525,5 +525,8 @@ do_thread_test walthread-5 -seconds $seconds(walthread-5) -init { db eval { SELECT count(*) FROM t1 } } unset -nocomplain seconds +catch {db close} +forcedelete $::DBNAME $::DBNAME-journal $::DBNAME-wal $::DBNAME-shm +forcedelete $::DBNAME-bak.db-wal $::DBNAME-bak.db finish_test diff --git a/test/where2.test b/test/where2.test index a38f6e54b..83740bffd 100644 --- a/test/where2.test +++ b/test/where2.test @@ -778,4 +778,20 @@ do_execsql_test where2-14.1 { SELECT x FROM t14a WHERE x NOT IN (SELECT x FROM t14b); } {} +# Demonstrate that the code removed by +# https://sqlite.org/src/info/2025-09-17T17:09:07Z is in fact +# necessary. Answer confirmed by PostgreSQL +# +do_execsql_test where2-15.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a INTEGER PRIMARY KEY,b INT); + INSERT INTO t1 VALUES(12,34),(56,78),(90,12); + DROP TABLE IF EXISTS t2; + CREATE TABLE t2(x INT UNIQUE); + INSERT INTO t2 VALUES(78),(12); + SELECT a,b,quote(x),'|' + FROM t1 LEFT JOIN (SELECT x FROM t2) AS s1 ON t1.b=s1.x + ORDER BY a,EXISTS(SELECT 1 FROM t1 LEFT JOIN (SELECT x AS y FROM t2) AS s2 ON t1.b=s2.y),x; +} {12 34 NULL | 56 78 78 | 90 12 12 |} + finish_test diff --git a/test/window1.test b/test/window1.test index 745755568..db9af62ac 100644 --- a/test/window1.test +++ b/test/window1.test @@ -2394,4 +2394,34 @@ do_execsql_test 78.2 { )) FROM (SELECT 'abc' AS x, 1 AS y); } NULL +#------------------------------------------------------------------------- +# Test that the following queries do not run for a very long time. +# +# https://sqlite.org/forum/forumpost/b1993c858f +# +do_execsql_test 79.0 { + CREATE TABLE t0 (c0 INTEGER ); + INSERT INTO t0 VALUES(1); + INSERT INTO t0 VALUES(2); + INSERT INTO t0 VALUES(3); +} + +do_execsql_test 79.1 { + SELECT COUNT(*) + OVER ( ROWS BETWEEN 0 FOLLOWING AND 100 FOLLOWING) + FROM t0; +} {3 2 1} + +do_execsql_test 79.2 { + SELECT COUNT(*) + OVER ( ORDER BY c0 RANGE BETWEEN 0 FOLLOWING AND 10_000_000_000 FOLLOWING ) + FROM t0; +} {3 2 1} + +do_execsql_test 79.3 { + SELECT sum(c0), COUNT(*) + OVER ( ORDER BY c0 RANGE BETWEEN 0 FOLLOWING AND 10_000_000_000 FOLLOWING ) + FROM t0; +} {6 1} + finish_test diff --git a/test/without_rowid7.test b/test/without_rowid7.test index 56e9fb40b..bf0273d70 100644 --- a/test/without_rowid7.test +++ b/test/without_rowid7.test @@ -55,6 +55,69 @@ do_execsql_test 2.4 { PRAGMA index_info(t3); } {} +#------------------------------------------------------------------------- +reset_db +db collate mysort mysort +db collate mysort2 mysort +proc mysort {a b} { string compare $a $b } +do_execsql_test 3.0 { + CREATE TABLE t1( + a PRIMARY KEY COLLATE mysort, b COLLATE mysort2 + ) WITHOUT ROWID; + INSERT INTO t1 VALUES(1, 2); +} + +db close +sqlite3 db test.db + +do_catchsql_test 3.1.1 { + SELECT * FROM t1 WHERE a=1; +} {1 {no such collation sequence: mysort}} +do_test 3.1.2 { + sqlite3_extended_errcode db +} {SQLITE_ERROR_MISSING_COLLSEQ} + +db collate mysort mysort + +do_catchsql_test 3.2.1 { + CREATE UNIQUE INDEX i1 ON t1(b); +} {1 {no such collation sequence: mysort2}} +do_test 3.2.2 { + sqlite3_extended_errcode db +} {SQLITE_ERROR_MISSING_COLLSEQ} + +db close +sqlite3 db test.db + +do_catchsql_test 3.3.1 { + CREATE UNIQUE INDEX i1 ON t1(1); +} {1 {no such collation sequence: mysort}} +do_test 3.3.2 { + sqlite3_extended_errcode db +} {SQLITE_ERROR_MISSING_COLLSEQ} + +do_test 3.4.1 { + list [catch { + sqlite3_prepare_v3 db "CREATE UNIQUE INDEX i1 ON t1(1)" -1 0 + } msg] $msg +} {1 {(1) no such collation sequence: mysort}} +do_test 3.4.2 { + sqlite3_extended_errcode db +} {SQLITE_ERROR_MISSING_COLLSEQ} + +sqlite3_extended_result_codes db 1 + +do_test 3.5.1 { + list [catch { + sqlite3_prepare_v3 db "CREATE UNIQUE INDEX i1 ON t1(1)" -1 0 + } msg] $msg +} {1 {(257) no such collation sequence: mysort}} +do_test 3.5.2 { + sqlite3_extended_errcode db +} {SQLITE_ERROR_MISSING_COLLSEQ} +do_catchsql_test 3.6 { + SELECT * FROM t1 WHERE a=1; +} {1 {no such collation sequence: mysort}} finish_test diff --git a/test/zipfile.test b/test/zipfile.test index 016a20b42..2b3be6278 100644 --- a/test/zipfile.test +++ b/test/zipfile.test @@ -887,4 +887,18 @@ do_test 19.1 { } {} forcedelete zipfile19.zip +#------------------------------------------------------------------------- +do_catchsql_test 20.0 { + SELECT * FROM zipfile(X'504b050600000000010001004000000000a3e1110000'); +} {1 {zip archive is corrupt}} + +do_catchsql_test 20.1 { + SELECT * FROM zipfile(unhex(' +504b0304140000080000a60d3e5bd42728f602000000020000000500090068 +2e74787455540500012836db682120504b01021e03140000080000a60d3e5b +d42728f602000000020000000500ffff0000000000000000a4810000000068 +2e74787455540500012836db68504b050600000000010001003c0000002e00 +00000000',char(0x0a,0x0d))); +} {1 {zip archive is corrupt}} + finish_test diff --git a/test/zipfile2.test b/test/zipfile2.test index 5277cd58f..8ee90d310 100644 --- a/test/zipfile2.test +++ b/test/zipfile2.test @@ -243,4 +243,63 @@ do_execsql_test 6.3 { SELECT name FROM zip; } {test2} +#------------------------------------------------------------------------- +# Test a crafted corrupt file. +# +proc make_corrupt_file {fname} { + set central_dir_offset 1000 + set lfh_offset 200 + set nFile 60000 + set nExtra 60000 + set nComment 0 + + # Build local file header at lfh_offset + set lfh "" + append lfh [binary format isssssiiiss 0x04034b50 20 0 0 0 0 0 0 0 1 0] + append lfh "A" + + # Build central directory structure (CDS) at central_dir_offset + set cds "" + append cds [binary format issssssiiisssssii 0x02014b50 0 20 0 0 0 0 0 0 0 $nFile $nExtra $nComment 0 0 0 $lfh_offset] + + # Payload following CDS: filename + extra + comment + set payload "" + append payload [string repeat "B" $nFile] + append payload [string repeat "C" $nExtra] + + # EOCD at end + set cdsize [expr {[string length $cds] + [string length $payload]}] + set eocd "" + append eocd [binary format issssiis 0x06054b50 0 0 1 1 $cdsize $central_dir_offset 0] + + # Assemble file + set buf [string repeat "X" $lfh_offset] + append buf $lfh + + set buflen [string length $buf] + if {$central_dir_offset > $buflen} { + append buf [string repeat "\x00" [expr {$central_dir_offset - $buflen}]] + } + + append buf $cds + append buf $payload + append buf $eocd + + # Write to file + set f [open $fname wb] + fconfigure $f -translation binary + puts -nonewline $f $buf + close $f +} + +forcedelete test.zip +make_corrupt_file test.zip +do_execsql_test 7.0 { + DROP TABLE IF EXISTS t1; + CREATE VIRTUAL TABLE t1 USING zipfile('test.zip'); +} +do_execsql_test 7.1 { + SELECT length(name) FROM t1; +} {60000} + finish_test diff --git a/tool/buildtclext.tcl b/tool/buildtclext.tcl index 8e5bef38a..535ed37e7 100644 --- a/tool/buildtclext.tcl +++ b/tool/buildtclext.tcl @@ -17,6 +17,7 @@ Options: --uninstall Uninstall the extension --version-check Check extension version against this source tree --destdir DIR Installation root (used by "make install DESTDIR=...") + --tclConfig.sh FILE Use this tclConfig.sh instead of looking for one Other options are retained and passed through into the compiler.} @@ -29,6 +30,7 @@ set versioncheck 0 set CC {} set OPTS {} set DESTDIR ""; # --destdir "$(DESTDIR)" +set tclConfigSh ""; # --tclConfig.sh FILE for {set ii 0} {$ii<[llength $argv]} {incr ii} { set a0 [lindex $argv $ii] if {$a0=="--install-only"} { @@ -56,6 +58,9 @@ for {set ii 0} {$ii<[llength $argv]} {incr ii} { } elseif {$a0=="--destdir" && $ii+1<[llength $argv]} { incr ii set DESTDIR [lindex $argv $ii] + } elseif {$a0=="--tclConfig.sh" && $ii+1<[llength $argv]} { + incr ii + set tclConfigSh [lindex $argv $ii] } elseif {[string match -* $a0]} { append OPTS " $a0" } else { @@ -88,40 +93,47 @@ if {$tcl_platform(platform) eq "windows"} { } set OUT tclsqlite3.dll } else { - # Figure out the location of the tclConfig.sh file used by the - # tclsh that is executing this script. + # Read the tclConfig.sh file into the $tclConfig variable # - if {[catch { - set LIBDIR [tcl::pkgconfig get libdir,install] - }]} { - puts stderr "$argv0: tclsh does not support tcl::pkgconfig." - exit 1 - } - if {![file exists $LIBDIR]} { - puts stderr "$argv0: cannot find the tclConfig.sh file." - puts stderr "$argv0: tclsh reported library directory \"$LIBDIR\"\ + if {"" eq $tclConfigSh} { + # Figure out the location of the tclConfig.sh file used by the + # tclsh that is executing this script. + # + if {[catch { + set LIBDIR [tcl::pkgconfig get libdir,install] + }]} { + puts stderr "$argv0: tclsh does not support tcl::pkgconfig." + exit 1 + } + if {![file exists $LIBDIR]} { + puts stderr "$argv0: cannot find the tclConfig.sh file." + puts stderr "$argv0: tclsh reported library directory \"$LIBDIR\"\ does not exist." - exit 1 - } - if {![file exists $LIBDIR/tclConfig.sh] - || [file size $LIBDIR/tclConfig.sh]<5000} { - set n1 $LIBDIR/tcl$::tcl_version - if {[file exists $n1/tclConfig.sh] - && [file size $n1/tclConfig.sh]>5000} { - set LIBDIR $n1 - } else { - puts stderr "$argv0: cannot find tclConfig.sh in either $LIBDIR or $n1" exit 1 } + if {![file exists $LIBDIR/tclConfig.sh] + || [file size $LIBDIR/tclConfig.sh]<5000} { + set n1 $LIBDIR/tcl$::tcl_version + if {[file exists $n1/tclConfig.sh] + && [file size $n1/tclConfig.sh]>5000} { + set LIBDIR $n1 + } else { + puts stderr "$argv0: cannot find tclConfig.sh in either $LIBDIR or $n1" + exit 1 + } + } + #puts "using $LIBDIR/tclConfig.sh" + set fd [open $LIBDIR/tclConfig.sh rb] + set tclConfig [read $fd] + close $fd + } else { + # User-provided tclConfig.sh + # + set fd [open $tclConfigSh rb] + set tclConfig [read $fd] + close $fd } - # Read the tclConfig.sh file into the $tclConfig variable - # - #puts "using $LIBDIR/tclConfig.sh" - set fd [open $LIBDIR/tclConfig.sh rb] - set tclConfig [read $fd] - close $fd - # Extract parameter we will need from the tclConfig.sh file # set TCLMAJOR 8 diff --git a/tool/lemon.c b/tool/lemon.c index 795c3a216..324dda0c5 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -2023,10 +2023,10 @@ static char *msort( list = NEXT(list); NEXT(ep) = 0; for(i=0; irp->index*37 + a->dot; + h = a->rp->index*37 + a->dot; return h; } diff --git a/tool/loadfts.c b/tool/loadfts.c index 0000797b8..5ed03b7bd 100644 --- a/tool/loadfts.c +++ b/tool/loadfts.c @@ -145,6 +145,7 @@ static void traverse( for(e=readdir(d); e; e=readdir(d)){ if( strcmp(e->d_name, ".")==0 || strcmp(e->d_name, "..")==0 ) continue; char *zPath = sqlite3_mprintf("%s/%s", zDir, e->d_name); + if( zPath==0 ){ fprintf(stderr, "out of memory\n"); abort(); } if (e->d_type & DT_DIR) { traverse(zPath, pCtx, xCallback); }else{ diff --git a/tool/mkautoconfamal.sh b/tool/mkautoconfamal.sh index b750593c9..3835799a6 100644 --- a/tool/mkautoconfamal.sh +++ b/tool/mkautoconfamal.sh @@ -62,10 +62,6 @@ cp $TOP/main.mk $TMPSPACE cd $TMPSPACE -# Clean up emacs-generated backup files from the target -rm -f ./autosetup/*~ ./autosetup/teaish/*~ -rm -f ./*~ - #if true; then # Clean up *~ files (emacs-generated backups). # This bit is only for use during development of @@ -83,10 +79,14 @@ cat < tea/generic/tclsqlite3.c EOF cat $TOP/src/tclsqlite.c >> tea/generic/tclsqlite3.c -find . -type f -name '*~' -exec rm -f \{} \; -find . -type f -name '#*#' -exec rm -f \{} \; +# Clean up some local remnants from the tarball. +rm -f tea/.env-* # autosetup environment overrides +find . -type f -name '*~' -exec rm -f \{} \; # backup files +find . -type f -name '#*#' -exec rm -f \{} \; # emacs lock files +find . -type f -name '*.o' -exec rm -f \{} \; +find . -type f -name '*.so' -exec rm -f \{} \; -./configure && make dist +./configure && ${MAKE-make} dist tar xzf sqlite-$VERSION.tar.gz mv sqlite-$VERSION $TARBALLNAME tar czf $TARBALLNAME.tar.gz $TARBALLNAME diff --git a/tool/mkctimec.tcl b/tool/mkctimec.tcl index 1c59131b0..64d4a121a 100755 --- a/tool/mkctimec.tcl +++ b/tool/mkctimec.tcl @@ -134,6 +134,7 @@ set boolean_defnil_options { SQLITE_ENABLE_ATOMIC_WRITE SQLITE_ENABLE_BATCH_ATOMIC_WRITE SQLITE_ENABLE_BYTECODE_VTAB + SQLITE_ENABLE_CARRAY SQLITE_ENABLE_COLUMN_METADATA SQLITE_ENABLE_COLUMN_USED_MASK SQLITE_ENABLE_COSTMULT @@ -163,6 +164,7 @@ set boolean_defnil_options { SQLITE_ENABLE_ORDERED_SET_AGGREGATES SQLITE_ENABLE_OFFSET_SQL_FUNC SQLITE_ENABLE_OVERSIZE_CELL_CHECK + SQLITE_ENABLE_PERCENTILE SQLITE_ENABLE_PREUPDATE_HOOK SQLITE_ENABLE_QPSG SQLITE_ENABLE_RBU diff --git a/tool/mkshellc.tcl b/tool/mkshellc.tcl index 85e14f849..2f7a6ea25 100644 --- a/tool/mkshellc.tcl +++ b/tool/mkshellc.tcl @@ -62,7 +62,7 @@ while {1} { if {[regexp {^# *include "sqlite} $lx]} { set lx "/* $lx */" } - if {[regexp {^# *include "test_windirent.h"} $lx]} { + if {[regexp {^# *include "windirent.h"} $lx]} { set lx "/* $lx */" } set lx [string map [list __declspec(dllexport) {}] $lx] diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 7b6f57e42..0c058672f 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -491,6 +491,7 @@ set flist { sqlite3rbu.c dbstat.c dbpage.c + carray.c sqlite3session.c fts5.c stmt.c diff --git a/tool/mksqlite3h.tcl b/tool/mksqlite3h.tcl index 6bbfa8c8b..c661cd7a3 100644 --- a/tool/mksqlite3h.tcl +++ b/tool/mksqlite3h.tcl @@ -79,6 +79,13 @@ set nVersion [eval format "%d%03d%03d" [split $zVersion .]] # Get the source-id # +proc file-content {fn} { + set fd [open $fn rb] + set rv [string trim [read $fd]] + close $fd + return $rv +} + set PWD [pwd] cd $TOP set tmpfile $PWD/tmp-[clock millisec]-[expr {int(rand()*100000000000)}].txt @@ -89,12 +96,31 @@ if {![file exists $mksourceid] && [file exists ${mksourceid}.exe]} { set mksourceid ${mksourceid}.exe } exec $mksourceid manifest > $tmpfile -set fd [open $tmpfile rb] -set zSourceId [string trim [read $fd]] -close $fd +set zSourceId [file-content $tmpfile] file delete -force $tmpfile cd $PWD +# Collect SQLITE_SCM_BRANCH, SQLITE_SCM_TAGS, and SQLITE_SCM_DATETIME +set zVerTime [lindex [lindex [split [file-content $TOP/manifest] "\n"] 1] 1]Z; # D-card +if {![file exists $TOP/manifest.tags]} { + puts stderr "WARNING: building sqlite3.h without manifest.tags, which is generated by the SCM." + puts stderr "This means that we cannot record the tag/branch info. We will continue with " + puts stderr "a placeholder value. To remedy this, run the following command from a " + puts stderr "check-out:\n" + puts stderr " fossil set manifest urt\n" + set zSourceId [string range $zSourceId 0 end-13]-experimental; # Keep SHA3 hash length + set zBranch "unknown" + set zTags "unknown" +} else { + # Read the list of branch/tags from manifest.tags + set content [file-content $TOP/manifest.tags]; + set zTags {} + foreach {x tag} [lassign $content - zBranch] { + if {$tag eq $zBranch} continue + lappend zTags $tag + } +} + # Set up patterns for recognizing API declarations. # set varpattern {^[a-zA-Z][a-zA-Z_0-9 *]+sqlite3_[_a-zA-Z0-9]+(\[|;| =)} @@ -157,6 +183,9 @@ foreach file $filelist { regsub -- --VERS-- $line $zVersion line regsub -- --VERSION-NUMBER-- $line $nVersion line regsub -- --SOURCE-ID-- $line "$zSourceId" line + regsub -- --SCM-BRANCH-- $line "$zBranch" line + regsub -- --SCM-TAGS-- $line "$zTags" line + regsub -- --SCM-DATETIME-- $line "$zVerTime" line if {[regexp $varpattern $line] && ![regexp {^ *typedef} $line]} { set line "SQLITE_API $line" diff --git a/tool/mktoolzip.tcl b/tool/mktoolzip.tcl index c22318441..041bf28cd 100644 --- a/tool/mktoolzip.tcl +++ b/tool/mktoolzip.tcl @@ -12,9 +12,27 @@ # sqlite3_analyzer -- Space analyzer # sqlite3_rsync -- Remote db sync # +# On Windows, add: +# +# sqlite3.def +# sqlite3.dll +# +# Add the --snapshot option to generate a snapshot ZIP archive instead of +# a release ZIP archive. +# +set bSnapshot 0 +for {set i 0} {$i<[llength $argv]} {incr i} { + set a [lindex $argv $i] + if {$a eq "-snapshot" || $a eq "--snapshot"} { + set bSnapshot 1 + continue + } + puts stderr "unknown argument: $a" + exit 1 +} switch $tcl_platform(os) { {Windows NT} { - set OS win32 + set OS win set EXE .exe } Linux { @@ -49,13 +67,39 @@ switch $tcl_platform(machine) { set ARCH unk } } -set in [open [file join [file dirname [file dirname [info script]]] VERSION]] -set vers [read $in] -close $in -scan $vers %d.%d.%d v1 v2 v3 -set v2 [format 3%02d%02d00 $v2 $v3] +if {$bSnapshot} { + set in [open [file join [file dirname [file dirname [info script]]] manifest]] + set manifest [read $in] + close $in + regexp {\nD (.{16})} $manifest all date + regsub -all {[-:T]} $date {} v2 +} else { + set in [open [file join [file dirname [file dirname [info script]]] VERSION]] + set vers [read $in] + close $in + scan $vers %d.%d.%d v1 v2 v3 + set v2 [format 3%02d%02d00 $v2 $v3] +} + set name sqlite-tools-$OS-$ARCH-$v2.zip -set toollist "sqlite3$EXE sqldiff$EXE sqlite3_analyzer$EXE sqlite3_rsync$EXE" -puts "zip $name {*}$toollist" -exec zip $name {*}$toollist -puts "$name: [file size $name] bytes" +set filelist "sqlite3$EXE sqldiff$EXE sqlite3_analyzer$EXE sqlite3_rsync$EXE" +proc make_zip_archive {name filelist} { + file delete -force $name + puts "fossil test-filezip $name $filelist" + if {[catch {exec fossil test-filezip $name {*}$filelist}]} { + puts "^--- Unable. Trying again as:" + puts "zip $name $filelist" + file delete -force $name + exec zip $name {*}$filelist + } + puts "$name: [file size $name] bytes" +} +make_zip_archive $name $filelist + +# On Windows, also try to construct a DLL +# +if {$OS eq "win" && [file exists sqlite3.dll] && [file exists sqlite3.def]} { + set name sqlite-dll-win-$ARCH-$v2.zip + set filelist [list sqlite3.def sqlite3.dll] + make_zip_archive $name $filelist +} diff --git a/tool/mkwinarm64ec.tcl b/tool/mkwinarm64ec.tcl new file mode 100644 index 000000000..388dd47a8 --- /dev/null +++ b/tool/mkwinarm64ec.tcl @@ -0,0 +1,45 @@ +# Script to build ARM64-EC binaries on Windows. +# +# Run from the top-level of a check-out, on a Windows11 ARM machine that +# has Fossil installed, using a command like this: +# +# tclsh tool/mmkwinarm64ec.tcl +# +# Output will be a file named: +# +# sqlite-win-arm64ec-3MMPP00.zip +# +# Where MM is the minor version number and PP is that patch level. +# +puts "Running nmake..." +flush stdout +exec nmake /f Makefile.msc "PLATFORM=ARM64EC" "BCC=cl -nologo -arm64EC" "TCC=cl -nologo -arm64EC -I." "CFLAGS=-nologo -arm64EC -I." clean sqlite3.exe sqldiff.exe sqlite3_rsync.exe sqlite3.dll >@ stdout 2>@ stderr + +proc check-type {file} { + set in [open "| link /dump /headers $file" rb] + set txt [read $in] + close $in + if {![string match {*8664 machine (x64) (ARM64X)*} $txt]} { + puts "$file is not an ARM64-EC binary" + puts "OUTPUT:\n$txt" + flush stdout + exit 1 + } + puts "Confirmed: $file is an ARM64-EC binary" +} +check-type sqlite3.exe +check-type sqldiff.exe +check-type sqlite3_rsync.exe +check-type sqlite3.dll + +set in [open VERSION rb] +set VERSION [read $in] +close $in +regexp {3.(\d+).(\d+)} $VERSION all minor patch +set filename [format sqlite-win-arm64ec-3%02d%02d00.zip $minor $patch] + +puts "Constructing $filename..." +file delete -force $filename +exec fossil test-filezip $filename sqlite3.def sqlite3.dll sqlite3.exe sqldiff.exe sqlite3_rsync.exe >@ stdout 2>@ stderr +puts "$filename: [file size $filename] bytes" +exec fossil test-filezip -l $filename >@ stdout 2>@ stderr diff --git a/tool/showstat4.c b/tool/showstat4.c index b8a12ad63..d1ff4961b 100644 --- a/tool/showstat4.c +++ b/tool/showstat4.c @@ -67,6 +67,7 @@ int main(int argc, char **argv){ "**************\n\n"); sqlite3_free(zIdx); zIdx = sqlite3_mprintf("%s", sqlite3_column_text(pStmt,0)); + if( zIdx==0 ){ fprintf(stderr, "out of memory\n"); abort(); } iRow = 0; } printf("%s sample %d ------------------------------------\n", zIdx, ++iRow); diff --git a/tool/sqlite3_rsync.c b/tool/sqlite3_rsync.c index 84ebf15c5..ad9f132bb 100644 --- a/tool/sqlite3_rsync.c +++ b/tool/sqlite3_rsync.c @@ -20,7 +20,7 @@ #include #include "sqlite3.h" -static const char zUsage[] = +static const char zUsage[] = "sqlite3_rsync ORIGIN REPLICA ?OPTIONS?\n" "\n" "One of ORIGIN or REPLICA is a pathname to a database on the local\n" @@ -566,7 +566,8 @@ int append_escaped_arg(sqlite3_str *pStr, const char *zIn, int isFilename){ */ void add_path_argument(sqlite3_str *pStr){ append_escaped_arg(pStr, - "PATH=$HOME/bin:/usr/local/bin:/opt/homebrew/bin:$PATH", 0); + "PATH=$HOME/bin:/usr/local/bin:/opt/homebrew/bin" + ":/opt/local/bin:$PATH", 0); } /***************************************************************************** @@ -1797,6 +1798,7 @@ static void replicaSide(SQLiteRsync *p){ closeDb(p); break; } + sqlite3_db_config(p->db, SQLITE_DBCONFIG_WRITABLE_SCHEMA, 1, 0); runSql(p, "ATTACH %Q AS 'replica'", p->zReplica); if( p->wrongEncoding ){ p->wrongEncoding = 0; @@ -1852,7 +1854,7 @@ static void replicaSide(SQLiteRsync *p){ nRPage); }else{ runSql(p,"INSERT INTO sendHash VALUES(1,1)"); - subdivideHashRange(p, 2, nRPage); + subdivideHashRange(p, 2, nRPage-1); } sendHashMessages(p, 1, 1); runSql(p, "PRAGMA writable_schema=ON"); @@ -2063,6 +2065,7 @@ int main(int argc, char const * const *argv){ #define cli_opt_val cmdline_option_value(argc, argv, ++i) memset(&ctx, 0, sizeof(ctx)); ctx.iProtocol = PROTOCOL_VERSION; + sqlite3_initialize(); for(i=1; i&1 | grep '^gcc version'` if test "$gccvers" '<' 'gcc version 6' then @@ -22,11 +22,18 @@ fi rm -f sqlite3.c make sqlite3.c -echo '**** No optimizations. Includes FTS4/5, GEOPOLY, JSON1 ***' +echo '**** No optimizations. Includes FTS4/5, GEOPOLY, and others ***' echo '****' $WARNING_OPTS -gcc -c $WARNING_OPTS -std=c89 \ - -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_GEOPOLY \ +gcc -c $WARNING_OPTS -std=c99 \ + -DHAVE_STDINT_H \ + -DSQLITE_ENABLE_FTS4 \ -DSQLITE_ENABLE_FTS5 \ + -DSQLITE_ENABLE_GEOPOLY \ + -DSQLITE_ENABLE_DBSTAT_VTAB \ + -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ + -DSQLITE_ENABLE_MATH_FUNCTIONS_fixme \ + -DSQLITE_ENABLE_STMTVTAB \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ sqlite3.c if test x`uname` = 'xLinux'; then echo '**** Android configuration ******************************' @@ -54,12 +61,15 @@ gcc -c \ fi echo '**** No optimizations. ENABLE_STAT4. THREADSAFE=0 *******' echo '****' $WARNING_OPTS -gcc -c $WARNING_OPTS -std=c89 \ - -ansi -DSQLITE_ENABLE_STAT4 -DSQLITE_THREADSAFE=0 \ +gcc -c $WARNING_OPTS -std=c99 \ + -DSQLITE_ENABLE_STAT4 \ + -DSQLITE_THREADSAFE=0 \ sqlite3.c echo '**** Optimized -O3. Includes FTS4/5, GEOPOLY, JSON1 ******' echo '****' $WARNING_OPTS -gcc -O3 -c $WARNING_OPTS -std=c89 \ - -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_GEOPOLY \ +gcc -O3 -c $WARNING_OPTS -std=c99 \ + -DHAVE_STDINT_H \ + -DSQLITE_ENABLE_FTS4 \ -DSQLITE_ENABLE_FTS5 \ + -DSQLITE_ENABLE_GEOPOLY \ sqlite3.c From b3029dc1460352e38f94f83d3ed82d66a37b07d8 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Tue, 11 Nov 2025 15:22:02 -0500 Subject: [PATCH 45/75] Fixes missing else in pragma handling --- src/sqlcipher.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index c1a3c8879..834abaec6 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -2639,7 +2639,7 @@ int sqlcipher_codec_pragma(sqlite3* db, int iDb, Parse *pParse, const char *zLef char *store_pass_value = sqlite3_mprintf("%d", ctx->store_pass); sqlcipher_vdbe_return_string(pParse, "cipher_store_pass", store_pass_value, P4_DYNAMIC); } - } + } else if( sqlite3_stricmp(zLeft, "cipher_profile")== 0 && zRight ){ char *profile_status = sqlite3_mprintf("%d", sqlcipher_cipher_profile(db, zRight)); sqlcipher_vdbe_return_string(pParse, "cipher_profile", profile_status, P4_DYNAMIC); From 808f05526ca808268c1ee27391b6b56220601f80 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Thu, 20 Nov 2025 13:21:10 -0500 Subject: [PATCH 46/75] Ensures all test databsae handles are closed before delete --- test/sqlcipher-compatibility.test | 4 ++ test/sqlcipher-core.test | 22 +--------- test/sqlcipher-integrity.test | 1 + test/sqlcipher-plaintext-header.test | 2 + test/sqlcipher-zmemory.test | 61 ++++++++++++++++++++++++++++ test/sqlcipher.tcl | 8 ++-- test/sqlcipher.test | 3 +- 7 files changed, 76 insertions(+), 25 deletions(-) create mode 100644 test/sqlcipher-zmemory.test diff --git a/test/sqlcipher-compatibility.test b/test/sqlcipher-compatibility.test index 89a0080cc..656b1c458 100644 --- a/test/sqlcipher-compatibility.test +++ b/test/sqlcipher-compatibility.test @@ -69,6 +69,7 @@ do_test unencrypted-attach { SELECT count(*) FROM t1; } db2 } {ok 1000} +db close db2 close file delete -force test.db file delete -force test2.db @@ -103,6 +104,7 @@ do_test unencrypted-attach-raw-key { SELECT count(*) FROM t1; } db2 } {ok 1000} +db close db2 close file delete -force test.db file delete -force test2.db @@ -1255,6 +1257,7 @@ do_test key-multiple-databases-with-different-keys-using-pragma { attach database 'test.db' as test key 'foobar'; select * from t1; select * from test.t1; + detach database test; } } {ok foo bar baz qux} db close @@ -1291,6 +1294,7 @@ if_built_with_libtomcrypt verify-random-data-alters-file-content { lappend rc [cmpFilesChunked test.db test2.db] lappend rc [cmpFilesChunked test2.db test3.db] } {0 1} +db close file delete -force test.db file delete -force test2.db file delete -force test3.db diff --git a/test/sqlcipher-core.test b/test/sqlcipher-core.test index ee544a634..6538cac09 100644 --- a/test/sqlcipher-core.test +++ b/test/sqlcipher-core.test @@ -821,24 +821,6 @@ do_test cipher-options-before-keys { db close file delete -force test.db -# verify memory security behavior -# initially should report OFF -# then enable, check that it is ON -# try to turn if off, but verify that it -# can't be unset. -do_test verify-memory-security { - sqlite_orig db test.db - execsql { - PRAGMA cipher_memory_security; - PRAGMA cipher_memory_security = ON; - PRAGMA cipher_memory_security; - PRAGMA cipher_memory_security = OFF; - PRAGMA cipher_memory_security; - } -} {0 1 1} -db close -file delete -force test.db - # create two new database files, write to each # and verify that they have different (i.e. random) # salt values @@ -855,10 +837,10 @@ do_test test-random-salt { CREATE TABLE t1(a,b); INSERT INTO t1(a,b) VALUES (1,2); } db2 - db close - db2 close string equal [hexio_read test.db 0 16] [hexio_read test2.db 0 16] } {0} +db close +db2 close file delete -force test.db file delete -force test2.db diff --git a/test/sqlcipher-integrity.test b/test/sqlcipher-integrity.test index fa0949892..dbf903fcd 100644 --- a/test/sqlcipher-integrity.test +++ b/test/sqlcipher-integrity.test @@ -339,6 +339,7 @@ do_test integrity-check-plaintext-header { PRAGMA cipher_integrity_check; }] } {{} 1 {{HMAC verification failed for page 1} {HMAC verification failed for page 2}}} +db close file delete -force test.db # test that changing the key in the middle of database operations does diff --git a/test/sqlcipher-plaintext-header.test b/test/sqlcipher-plaintext-header.test index 81e1a8231..023d41454 100644 --- a/test/sqlcipher-plaintext-header.test +++ b/test/sqlcipher-plaintext-header.test @@ -60,6 +60,7 @@ do_test test-pragma-salt-get { set header [string tolower [hexio_read test.db 0 16]] string equal $header $salt } {1} +db close file delete -force test.db # explicitly set the salt of a new database @@ -84,6 +85,7 @@ do_test test-pragma-salt-set { "] } {01010101010101010101010101010101 {ok 1 01010101010101010101010101010101}} +db close file delete -force test.db diff --git a/test/sqlcipher-zmemory.test b/test/sqlcipher-zmemory.test new file mode 100644 index 000000000..272fccecf --- /dev/null +++ b/test/sqlcipher-zmemory.test @@ -0,0 +1,61 @@ + +# SQLCipher +# codec.test developed by Stephen Lombardo (Zetetic LLC) +# sjlombardo at zetetic dot net +# http://zetetic.net +# +# Copyright (c) 2018, ZETETIC LLC +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the ZETETIC LLC nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# This file implements regression tests for SQLite library. The +# focus of this script is testing code cipher features. +# +# NOTE: tester.tcl has overridden the definition of sqlite3 to +# automatically pass in a key value. Thus tests in this file +# should explicitly close and open db with sqlite_orig in order +# to bypass default key assignment. + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/sqlcipher.tcl + +# verify memory security behavior +# initially should report OFF +# then enable, check that it is ON +# try to turn if off, but verify that it +# can't be unset. +do_test verify-memory-security { + sqlite_orig db test.db + execsql { + PRAGMA cipher_memory_security; + PRAGMA cipher_memory_security = ON; + PRAGMA cipher_memory_security; + PRAGMA cipher_memory_security = OFF; + PRAGMA cipher_memory_security; + } +} {0 1 1} +db close +file delete -force test.db + +finish_test + diff --git a/test/sqlcipher.tcl b/test/sqlcipher.tcl index d058d366b..4752316c6 100644 --- a/test/sqlcipher.tcl +++ b/test/sqlcipher.tcl @@ -34,13 +34,13 @@ # to bypass default key assignment. -file delete -force test.db +set testdir [file dirname $argv0] +set sampleDir [file normalize [file dirname [file dirname $argv0]]]/sqlcipher-resources + +catch {file delete -force test.db} file delete -force test2.db file delete -force test3.db file delete -force test4.db - -set testdir [file dirname $argv0] -set sampleDir [file normalize [file dirname [file dirname $argv0]]]/sqlcipher-resources # If the library is not compiled with has_codec support then # skip all tests in this file. diff --git a/test/sqlcipher.test b/test/sqlcipher.test index 88ab1b5c4..7d008f8d9 100644 --- a/test/sqlcipher.test +++ b/test/sqlcipher.test @@ -58,7 +58,8 @@ test_suite "sqlcipher" -prefix "" -description { sqlcipher-pragmas.test \ sqlcipher-integrity.test \ sqlcipher-codecerror.test \ - sqlcipher-backup.test + sqlcipher-backup.test \ + sqlcipher-zmemory.test ] run_test_suite sqlcipher finish_test From 4128c969d6cf24e2df545d2c70935dfad75ee164 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Mon, 24 Nov 2025 12:13:57 -0500 Subject: [PATCH 47/75] Improves guards against key/rekey/attach misuse per #578 Validate that database handle is not null, database name is valid and database indexes are within bounds. --- src/sqlcipher.c | 51 +++++++++++++++++++++++++++------------- test/sqlcipher-core.test | 14 +++++++++++ 2 files changed, 49 insertions(+), 16 deletions(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 834abaec6..68c335b19 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -3383,8 +3383,8 @@ int sqlcipherCodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) { return SQLITE_MISUSE; } - if(!(db && (pDb = &db->aDb[nDb]))) { - sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: invalid database", __func__); + if(!(db && nDb >= 0 && nDb < db->nDb && (pDb = &db->aDb[nDb]))) { + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: invalid database %p %d", __func__, db, nDb); return SQLITE_MISUSE; } @@ -3485,18 +3485,24 @@ int sqlcipherCodecAttach(sqlite3* db, int nDb, const void *zKey, int nKey) { return rc; } +/* search for the index of the named database by comparing db names. main is + * always 0, temp 1, and other attached databses follow. If the name is + * NULL or empty the main database will be used consistent with sqlite defaults. If + * sqlite3 handle is NULL or the database can't be found by name, return -1 indicating + * an invalid database */ int sqlcipher_find_db_index(sqlite3 *db, const char *zDb) { int db_index; - if(zDb == NULL){ - return 0; - } + + if(!db) return -1; + if(!zDb || sqlite3_stricmp(zDb,"")==0) return 0; + for(db_index = 0; db_index < db->nDb; db_index++) { struct Db *pDb = &db->aDb[db_index]; - if(strcmp(pDb->zDbSName, zDb) == 0) { + if(sqlite3_stricmp(pDb->zDbSName, zDb) == 0) { return db_index; } } - return 0; + return -1; } void sqlite3_activate_see(const char* in) { @@ -3539,8 +3545,16 @@ int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey) { if(db && pKey && nKey) { int db_index = sqlcipher_find_db_index(db, zDb); - struct Db *pDb = &db->aDb[db_index]; + struct Db *pDb = NULL; + + if(!(db_index >= 0 && db_index < db->nDb)) { + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "%s: invalid database zDb=%p", __func__, zDb); + return SQLITE_MISUSE; + } + + pDb = &db->aDb[db_index]; sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlite3_rekey_v2: database zDb=%p db_index:%d", zDb, db_index); + if(pDb->pBt) { codec_ctx *ctx; int rc, page_count; @@ -3603,7 +3617,7 @@ int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey) { return SQLITE_OK; } sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_CORE, "sqlite3_rekey_v2: no key provided for db %s: rekey can't be used to decrypt an encrypted database", zDb); - return SQLITE_ERROR; + return SQLITE_MISUSE; } /* @@ -3616,8 +3630,17 @@ int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey) { * be passed back directly. Otherwise, a "keyspec" consisting of the raw key and salt * will be used instead. */ void sqlcipherCodecGetKey(sqlite3* db, int nDb, void **zKey, int *nKey) { - struct Db *pDb = &db->aDb[nDb]; + struct Db *pDb = NULL; + sqlcipher_log(SQLCIPHER_LOG_DEBUG, SQLCIPHER_LOG_CORE, "sqlcipherCodecGetKey:db=%p, nDb=%d", db, nDb); + + *zKey = NULL; + *nKey = 0; + + if(!(db && nDb >= 0 && nDb < db->nDb)) return; /* invalid database */ + + pDb = &db->aDb[nDb]; + if( pDb->pBt ) { codec_ctx *ctx = (codec_ctx*) sqlcipherPagerGetCodec(sqlite3BtreePager(pDb->pBt)); @@ -3631,9 +3654,6 @@ void sqlcipherCodecGetKey(sqlite3* db, int nDb, void **zKey, int *nKey) { } else { sqlcipher_cipher_ctx_get_keyspec(ctx, ctx->read_ctx, (char**) zKey, nKey); } - } else { - *zKey = NULL; - *nKey = 0; } } } @@ -3754,10 +3774,9 @@ static void sqlcipher_exportFunc(sqlite3_context *context, int argc, sqlite3_val } - /* if the name of the target is not main, but the index returned is zero - there is a mismatch and we should not proceed */ + /* if the target database is not valid, do not proceed. */ targetDb_idx = sqlcipher_find_db_index(db, targetDb); - if(targetDb_idx == 0 && targetDb != NULL && sqlite3_stricmp("main", targetDb) != 0) { + if(targetDb_idx < 0) { rc = SQLITE_ERROR; pzErrMsg = sqlite3_mprintf("unknown database %s", targetDb); goto end_of_export; diff --git a/test/sqlcipher-core.test b/test/sqlcipher-core.test index 6538cac09..6985cac87 100644 --- a/test/sqlcipher-core.test +++ b/test/sqlcipher-core.test @@ -74,6 +74,20 @@ do_test will-open-with-correct-derived-key { db close file delete -force test.db +# run test with sqlite3_key function directly +setup test.db "'testkey'" +do_test will-open-with-sqlite3key { + + sqlite_orig db test.db + sqlite3_key db "testkey" + execsql { + SELECT name FROM sqlite_schema WHERE type='table'; + SELECT * from t1; + } +} {t1 test1 test2} +db close +file delete -force test.db + # set an encryption key (non-hex) and create # temp tables, verify you can read from # sqlite_temp_master From 99c05ab90ae84e92f1123ceadeb97030800b1a7c Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Sun, 30 Nov 2025 09:37:24 -0500 Subject: [PATCH 48/75] Snapshot of upstream SQLite 3.51.1 --- VERSION | 2 +- autoconf/tea/Makefile.in | 5 +- autosetup/teaish/core.tcl | 6 +- ext/fts5/fts5_index.c | 1 + ext/fts5/fts5_vocab.c | 11 + ext/fts5/test/fts5corrupt3.test | 362 +++++++++++++++++++++++++++ ext/fts5/test/fts5vocab2.test | 22 ++ ext/intck/sqlite3intck.c | 1 + ext/wasm/api/sqlite3-api-prologue.js | 6 +- manifest | 59 +++-- manifest.tags | 4 + manifest.uuid | 2 +- src/callback.c | 1 + src/hash.c | 1 + src/main.c | 3 +- src/shell.c.in | 1 + src/sqlite.h.in | 2 +- src/test1.c | 33 +++ src/test_bestindex.c | 119 ++++++++- src/vtab.c | 3 + src/where.c | 18 +- test/bestindexE.test | 63 ++++- test/existsexpr.test | 48 ++++ test/misuse.test | 12 + test/returning1.test | 50 ++++ test/returningfault.test | 51 +++- 26 files changed, 832 insertions(+), 54 deletions(-) create mode 100644 manifest.tags diff --git a/VERSION b/VERSION index 84ba969c4..d1278a467 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.51.0 +3.51.1 diff --git a/autoconf/tea/Makefile.in b/autoconf/tea/Makefile.in index ccf9a7b94..04c8f87f5 100644 --- a/autoconf/tea/Makefile.in +++ b/autoconf/tea/Makefile.in @@ -119,7 +119,7 @@ TCLLIBDIR = @TCLLIBDIR@ # typically come from the ./configure command-line invocation). # CFLAGS.configure = @SH_CFLAGS@ @TEAISH_CFLAGS@ @CFLAGS@ @CPPFLAGS@ $(TCL_INCLUDE_SPEC) -#CFLAGS.configure += -DUSE_TCL_STUBS=1 +CFLAGS.configure += -DUSE_TCL_STUBS=@TEAISH_USE_STUBS@ # # LDFLAGS.configure = LDFLAGS as known at configure-time. @@ -153,6 +153,7 @@ tx.src = @TEAISH_EXT_SRC@ # gets set up via the configure script. # tx.CFLAGS = +tx.CPPFLAGS = # # tx.LDFLAGS is typically set by teaish.make, whereas TEAISH_LDFLAGS @@ -218,7 +219,7 @@ config.log: @TEAISH_TEST_TCL_IN@ # CC variant for compiling Tcl-using sources. # CC.tcl = \ - $(CC) -o $@ $(CFLAGS.configure) $(CFLAGS) $(tx.CFLAGS) + $(CC) -o $@ $(CFLAGS.configure) $(CFLAGS) $(tx.CFLAGS) $(tx.CPPFLAGS) # # CC variant for linking $(tx.src) into an extension DLL. Note that diff --git a/autosetup/teaish/core.tcl b/autosetup/teaish/core.tcl index a4a6b001f..c9abfa062 100644 --- a/autosetup/teaish/core.tcl +++ b/autosetup/teaish/core.tcl @@ -220,7 +220,9 @@ proc teaish-configure-core {} { => {Full pathname of tclsh to use. It is used for trying to find tclConfig.sh. Warning: if its containing dir has multiple tclsh versions, it may select the wrong tclConfig.sh! - Defaults to the $TCLSH environment variable.} + Defaults to the $TCLSH environment variable.} + + tcl-stubs=0 => {Enable use of Tcl stubs library.} # TEA has --with-tclinclude but it appears to only be useful for # building an extension against an uninstalled copy of TCL's own @@ -498,6 +500,8 @@ proc teaish__configure_phase1 {} { } teaish-checks-run -post + define TEAISH_USE_STUBS [opt-bool tcl-stubs] + apply {{} { # Set up "vsatisfies" code for pkgIndex.tcl.in, # _teaish.tester.tcl.in, and for a configure-time check. We would diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index a5a37f758..7e25731ed 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -2074,6 +2074,7 @@ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ while( 1 ){ u64 iDelta = 0; + if( i>=n ) break; if( eDetail==FTS5_DETAIL_NONE ){ /* todo */ if( ipFts5->pConfig->nCol; pCsr->rowid = 0; sqlite3Fts5IterClose(pCsr->pIter); sqlite3Fts5StructureRelease(pCsr->pStruct); @@ -406,6 +411,12 @@ static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){ pCsr->nLeTerm = -1; pCsr->zLeTerm = 0; pCsr->bEof = 0; + pCsr->iCol = 0; + pCsr->iInstPos = 0; + pCsr->iInstOff = 0; + pCsr->colUsed = 0; + memset(pCsr->aCnt, 0, sizeof(i64)*nCol); + memset(pCsr->aDoc, 0, sizeof(i64)*nCol); } /* diff --git a/ext/fts5/test/fts5corrupt3.test b/ext/fts5/test/fts5corrupt3.test index eab4c3c91..20be7c45c 100644 --- a/ext/fts5/test/fts5corrupt3.test +++ b/ext/fts5/test/fts5corrupt3.test @@ -16124,5 +16124,367 @@ do_catchsql_test 83.1 { SELECT * FROM t1('R*R*R*R*R*R*R*R*') WHERE (a,b)<=(current_date,0 BETWEEN 'a'<>11 AND '') ORDER BY rowid DESC; } {/.*fts5: corruption found/} +#------------------------------------------------------------------------- +reset_db +do_test 84.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +.open --hexdb +| size 53248 pagesize 4096 filename c1a.txt.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0d .....@ ........ +| 32: 00 00 00 02 00 00 00 01 00 00 00 09 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36 ...............6 +| 112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 0d 92 00 00 00 00 ...k............ +| 3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74 .........1tablet +| 3488: 32 74 32 0d 43 52 45 41 54 45 20 54 41 42 4c 45 2t2.CREATE TABLE +| 3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61 t2(x)V.......ta +| 3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63 blet1_configt1_c +| 3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42 onfig.CREATE TAB +| 3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b LE 't1_config'(k +| 3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29 PRIMARY KEY, v) +| 3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06 WITHOUT ROWID[. +| 3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64 ..!!...tablet1_d +| 3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65 ocsizet1_docsize +| 3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 .CREATE TABLE 't +| 3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e 1_docsize'(id IN +| 3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 TEGER PRIMARY KE +| 3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21 Y, sz BLOB)^...! +| 3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74 !...tablet1_cont +| 3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52 entt1_content.CR +| 3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 EATE TABLE 't1_c +| 3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 ontent'(id INTEG +| 3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 ER PRIMARY KEY, +| 3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19 c0, c1, c2)i.... +| 3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74 ...-tablet1_idxt +| 3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42 1_idx.CREATE TAB +| 3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69 LE 't1_idx'(segi +| 3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50 d, term, pgno, P +| 3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64 RIMARY KEY(segid +| 3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54 , term)) WITHOUT +| 3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74 ROWIDU........t +| 3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61 ablet1_datat1_da +| 3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20 ta.CREATE TABLE +| 3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54 't1_data'(id INT +| 3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06 , block BLOB)8.. +| 3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52 ...._tablet1t1CR +| 4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42 EATE VIRTUAL TAB +| 4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35 LE t1 USING fts5 +| 4032: 28 61 2c 62 2c 63 29 00 00 00 39 00 00 00 00 00 (a,b,c)...9..... +| page 3 offset 8192 +| 0: 05 00 00 00 02 0f f1 00 00 00 00 0c 0f fb 0f f1 ................ +| 4064: 00 00 0b 01 03 00 1c 81 3a 84 5e 81 3a 81 3a 0a ........:.^.:.:. +| 4080: 0a 00 00 00 0b 84 80 80 80 80 01 00 00 00 0a 0a ................ +| page 4 offset 12288 +| 0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02 ................ +| page 7 offset 24576 +| 0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04 ........version. +| page 10 offset 36864 +| 0: 0d 00 00 00 02 0f e2 00 0f e2 0f ef 00 00 00 00 ................ +| 4064: 00 00 0b 01 03 00 1c 81 3a 84 5e 81 3a 81 3a 0f ........:.^.:.:. +| 4080: 0a 03 00 24 00 00 00 00 01 01 02 00 01 01 01 09 ...$............ +| page 11 offset 40960 +| 0: 0d 00 00 00 01 00 22 00 00 22 00 00 00 00 00 00 ................ +| 32: 00 00 9f 56 84 80 80 80 80 01 04 00 bf 30 00 00 ...V.........0.. +| 48: 0f 58 02 30 30 19 02 05 01 02 05 01 02 05 16 02 .X.00........... +| 64: 05 01 02 05 01 02 05 61 02 05 01 02 05 01 02 05 .......a........ +| 80: 13 02 05 01 02 05 01 02 05 0d 02 03 01 02 03 01 ................ +| 96: 02 03 02 09 78 66 66 66 66 66 66 66 65 81 17 02 ....xfffffffe... +| 112: 05 01 02 05 01 02 05 01 01 31 04 02 04 01 02 04 .........1...... +| 128: 01 02 04 01 02 05 01 02 05 01 02 05 0d 02 06 01 ................ +| 144: 02 06 01 02 06 02 01 30 79 02 04 01 02 04 01 02 .......0y....... +| 160: 04 03 02 30 30 2b 02 05 01 02 05 01 02 05 58 02 ...00+........X. +| 176: 05 01 02 05 01 02 05 01 02 05 01 02 05 01 02 05 ................ +| 192: 16 02 05 01 02 05 01 02 05 05 06 30 30 30 30 30 ...........00000 +| 208: 30 81 0b 02 04 01 02 04 01 02 04 10 02 05 01 02 0............... +| 224: 05 01 02 05 03 02 32 34 76 02 05 01 02 05 01 02 ......24v....... +| 240: 05 02 01 38 07 02 04 01 02 04 01 02 04 01 01 32 ...8...........2 +| 256: 28 02 04 01 02 04 01 02 04 04 02 05 01 02 05 01 (............... +| 272: 02 05 02 01 30 1f 02 05 01 02 05 01 02 05 03 02 ....0........... +| 288: 30 30 10 02 05 01 02 05 01 02 05 6a 02 04 01 02 00.........j.... +| 304: 04 01 02 04 02 08 35 30 30 30 30 30 30 30 81 26 ......50000000.& +| 320: 02 05 01 02 05 01 02 05 01 01 33 07 02 06 01 02 ..........3..... +| 336: 06 01 02 06 81 2c 02 04 01 02 04 01 02 04 02 04 .....,.......... +| 352: 32 37 36 36 81 23 02 05 01 02 05 01 02 05 01 01 2766.#.......... +| 368: 34 13 02 05 01 02 05 01 02 05 02 03 30 39 36 1c 4...........096. +| 384: 02 05 01 02 05 01 02 05 07 02 05 01 02 05 01 02 ................ +| 400: 05 01 03 35 30 30 7f 02 05 01 02 05 01 02 05 04 ...500.......... +| 416: 02 30 30 81 0e 02 06 01 02 06 01 02 06 06 03 30 .00............0 +| 432: 30 30 81 11 02 04 01 02 04 01 02 04 01 05 36 35 00............65 +| 448: 35 33 36 81 1a 02 05 01 02 05 01 02 05 01 04 38 536............8 +| 464: 31 39 32 81 02 02 06 01 02 06 01 02 06 01 05 61 192............a +| 480: 6c 6c 6f 77 01 02 02 01 02 02 01 02 02 02 02 72 llow...........r +| 496: 67 81 08 02 04 01 02 04 01 02 04 02 05 74 6f 6d g............tom +| 512: 69 63 04 02 02 01 02 02 01 02 02 03 06 74 61 63 ic...........tac +| 528: 68 65 64 79 02 03 01 02 03 01 02 03 02 0d 75 74 hedy..........ut +| 544: 6f 63 68 65 63 6b 70 6f 69 6e 74 2b 02 04 01 02 ocheckpoint+.... +| 560: 04 01 02 04 05 06 76 61 63 75 75 6d 0d 02 03 01 ......vacuum.... +| 576: 02 03 01 02 03 01 06 62 69 6e 61 72 79 03 06 01 .......binary... +| 592: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 608: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 624: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 640: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 656: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 672: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 688: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 704: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 720: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 736: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 752: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 768: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 784: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 800: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 816: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 832: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 848: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 864: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 880: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 896: 01 02 02 02 07 79 74 65 63 6f 64 65 37 02 03 01 .....ytecode7... +| 912: 02 03 01 02 03 01 05 63 61 63 68 65 10 02 03 01 .......cache.... +| 928: 02 03 01 01 03 02 04 6c 61 6e 67 07 02 03 01 02 .......lang..... +| 944: 03 01 02 03 02 05 6f 6c 75 6d 6e 7c 02 03 01 02 ......olumn|.... +| 960: 03 01 02 03 03 06 6d 6d 65 6e 74 73 43 02 04 01 ......mmentsC... +| 976: 02 04 01 02 04 04 05 70 69 6c 65 72 07 02 02 01 .......piler.... +| 992: 02 02 01 02 02 05 04 6f 75 6e 64 7f 02 03 01 02 .......ound..... +| 1008: 03 01 02 03 03 03 75 6e 74 81 17 02 04 01 02 04 ......unt....... +| 1024: 01 02 04 02 05 75 72 73 6f 72 3a 02 03 01 02 03 .....ursor:..... +| 1040: 01 02 03 01 06 64 62 70 61 67 65 3d 02 03 01 02 .....dbpage=.... +| 1056: 03 01 02 03 03 04 73 74 61 74 40 02 03 01 02 03 ......stat@..... +| 1072: 01 02 03 02 04 65 62 75 67 0a 02 02 01 02 02 01 .....ebug....... +| 1088: 02 02 03 05 66 61 75 6c 74 0d 02 02 01 02 02 01 ....fault....... +| 1104: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 1120: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 1136: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 1152: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 1168: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 1184: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 1200: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 4f 02 ..............O. +| 1216: 03 01 02 03 01 02 03 03 03 70 74 68 81 05 02 04 .........pth.... +| 1232: 01 02 04 01 02 04 19 02 04 01 02 04 01 02 04 02 ................ +| 1248: 05 69 72 65 63 74 34 02 02 01 02 02 01 02 02 01 .irect4......... +| 1264: 06 65 6e 61 62 6c 65 37 02 02 01 02 02 01 02 02 .enable7........ +| 1280: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 1296: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 1312: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 1328: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 1344: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 1360: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 1376: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 1392: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 1408: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 1424: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 1440: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 1456: 02 01 02 02 02 06 78 70 6c 61 69 6e 43 02 03 01 ......xplainC... +| 1472: 02 03 01 02 03 04 01 72 81 05 02 03 01 02 03 01 .......r........ +| 1488: 02 03 03 07 74 65 6e 73 69 6f 6e 81 2f 02 04 01 ....tension./... +| 1504: 02 04 01 02 04 01 04 66 69 6c 65 13 02 03 01 02 .......file..... +| 1520: 03 01 02 03 02 05 6f 72 6d 61 74 13 02 04 01 02 ......ormat..... +| 1536: 04 01 02 04 02 03 74 73 33 46 02 03 01 02 03 01 ......ts3F...... +| 1552: 02 03 01 02 03 01 02 03 01 02 03 04 01 34 4c 02 .............4L. +| 1568: 03 01 02 03 01 02 03 04 01 35 4f 02 03 01 02 03 .........5O..... +| 1584: 01 02 03 02 03 75 6e 63 5e 02 05 01 02 05 01 02 .....unc^....... +| 1600: 05 05 04 74 69 6f 6e 73 02 05 01 02 05 01 02 05 ...tions........ +| 1616: 13 02 03 01 02 03 01 02 03 09 01 73 55 02 04 01 ...........sU... +| 1632: 02 04 01 02 04 01 07 67 65 6f 70 6f 6c 79 52 02 .......geopolyR. +| 1648: 03 01 02 03 01 02 03 01 05 68 69 6e 74 73 3a 02 .........hints:. +| 1664: 04 01 02 04 01 02 04 02 03 6f 6f 6b 61 02 04 01 .........ooka... +| 1680: 02 04 01 02 04 01 02 69 6e 01 02 04 01 02 04 01 .......in....... +| 1696: 02 04 03 04 69 74 73 7a 1f 02 04 01 02 04 01 02 ....itsz........ +| 1712: 04 03 08 74 72 69 6e 73 69 63 73 04 02 03 01 02 ...trinsics..... +| 1728: 03 01 02 03 01 07 6a 6f 75 72 6e 61 6c 16 02 03 ......journal... +| 1744: 01 02 03 01 02 03 01 06 6c 65 6e 67 74 68 81 0b ........length.. +| 1760: 02 03 01 02 03 01 02 03 01 02 05 01 02 05 01 02 ................ +| 1776: 05 0d 02 04 01 02 04 01 02 04 02 03 69 6b 65 81 ............ike. +| 1792: 0e 02 03 01 02 03 01 02 03 03 03 6d 69 74 16 02 ...........mit.. +| 1808: 05 01 02 05 01 02 05 5e 02 04 01 02 04 01 02 04 .......^........ +| 1824: 02 03 6f 61 64 81 2f 02 03 01 02 03 01 02 03 01 ..oad./......... +| 1840: 06 6d 61 6c 6c 6f 63 76 02 02 01 02 02 01 02 02 .mallocv........ +| 1856: 3a 02 03 01 02 03 01 02 03 03 02 74 68 55 02 03 :..........thU.. +| 1872: 01 02 03 01 02 03 03 01 78 79 02 02 01 02 02 01 ........xy...... +| 1888: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 1904: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 1920: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 1936: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 1952: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 1968: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 1984: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 ................ +| 2000: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 ................ +| 2016: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 ................ +| 2032: 02 02 02 05 65 6d 6f 72 79 81 11 02 03 01 02 03 ....emory....... +| 2048: 01 02 03 04 04 73 79 73 35 58 02 03 01 02 03 01 .....sys5X...... +| 2064: 02 03 02 03 6d 61 70 19 02 03 01 02 03 01 02 03 ....map......... +| 2080: 79 02 03 01 02 03 01 02 03 02 04 75 74 65 78 81 y..........utex. +| 2096: 2c 02 02 01 02 02 01 02 02 01 06 6e 6f 63 61 73 ,..........nocas +| 2112: 65 02 06 01 02 02 03 06 01 02 02 03 06 01 02 02 e............... +| 2128: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 2144: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 2160: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 2176: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 2192: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 2208: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 2224: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 2240: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 2256: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 2272: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 2288: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 2304: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 2320: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 2336: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 2352: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 2368: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 2384: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 2400: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 2416: 02 02 03 06 01 02 02 03 07 72 6d 61 6c 69 7a 65 .........rmalize +| 2432: 5b 02 03 01 02 03 01 02 03 02 05 75 6d 62 65 72 [..........umber +| 2448: 81 23 02 04 01 02 04 01 02 04 01 06 6f 66 66 73 .#..........offs +| 2464: 65 74 5e 02 03 01 02 03 01 02 03 02 03 6d 69 74 et^..........mit +| 2480: 81 2c 02 03 01 02 03 01 02 03 01 02 02 01 02 02 .,.............. +| 2496: 01 02 02 02 01 70 81 26 02 04 01 02 04 01 02 04 .....p.&........ +| 2512: 02 07 76 65 72 66 6c 6f 77 34 02 03 01 02 03 01 ..verflow4...... +| 2528: 02 03 01 04 70 61 67 65 1c 02 03 01 02 03 01 02 ....page........ +| 2544: 03 64 02 04 01 02 04 01 02 04 13 02 03 01 02 03 .d.............. +| 2560: 01 02 03 01 02 03 01 02 03 01 02 03 03 09 72 65 ..............re +| 2576: 6e 74 68 65 73 69 73 49 02 04 01 02 04 01 02 04 nthesisI........ +| 2592: 03 05 74 74 65 72 6e 81 0e 02 04 01 02 04 01 02 ..ttern......... +| 2608: 04 02 05 63 61 63 68 65 1f 02 03 01 02 03 01 02 ...cache........ +| 2624: 03 02 08 72 65 75 70 64 61 74 65 61 02 03 01 02 ...reupdatea.... +| 2640: 03 01 02 03 01 04 72 65 61 64 34 02 04 01 02 04 ......read4..... +| 2656: 01 02 04 03 07 63 75 72 73 69 76 65 22 02 03 01 .....cursive.... +| 2672: 02 03 01 02 03 02 04 6f 77 69 64 01 02 03 01 02 .......owid..... +| 2688: 03 01 02 03 02 04 74 72 65 65 64 02 03 01 02 03 ......treed..... +| 2704: 01 02 03 04 02 69 6d 01 06 01 02 02 03 06 01 02 .....im......... +| 2720: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 2736: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 2752: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 2768: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 2784: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 2800: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 2816: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 2832: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 2848: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 2864: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 2880: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 2896: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 2912: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 2928: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 ................ +| 2944: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 ................ +| 2960: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 ................ +| 2976: 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 ................ +| 2992: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 ................ +| 3008: 01 02 02 03 06 01 02 02 03 06 01 02 02 01 0a 73 ...............s +| 3024: 63 61 6e 73 74 61 74 75 73 70 02 04 01 02 04 01 canstatusp...... +| 3040: 02 04 02 05 65 63 74 6f 72 25 02 03 01 02 03 01 ....ector%...... +| 3056: 02 03 03 04 6c 65 63 74 7f 02 04 01 02 04 01 02 ....lect........ +| 3072: 04 03 05 73 73 69 6f 6e 67 02 03 01 02 03 01 02 ...ssiong....... +| 3088: 03 02 03 69 7a 65 10 02 04 01 02 04 01 02 04 04 ...ize.......... +| 3104: 02 04 01 02 04 01 02 04 01 02 04 01 02 04 01 02 ................ +| 3120: 04 01 02 04 01 02 04 01 02 04 07 02 04 01 02 04 ................ +| 3136: 01 02 04 5b 02 05 01 02 05 01 02 05 10 02 04 01 ...[............ +| 3152: 02 04 01 02 04 04 02 04 01 02 04 01 02 04 02 03 ................ +| 3168: 6f 66 74 76 02 03 01 02 03 01 02 03 02 02 71 6c oftv..........ql +| 3184: 5e 02 04 01 02 04 01 02 04 13 02 04 01 02 04 01 ^............... +| 3200: 02 04 28 02 03 01 02 03 01 02 03 02 04 74 61 74 ..(..........tat +| 3216: 34 6a 02 03 01 02 03 01 02 03 03 02 6d 74 70 02 4j..........mtp. +| 3232: 03 01 02 03 01 02 03 05 04 76 74 61 62 6d 02 03 .........vtabm.. +| 3248: 01 02 03 01 02 03 03 03 6f 72 65 81 35 02 03 01 ........ore.5... +| 3264: 02 03 01 02 03 02 0a 79 6e 63 68 72 6f 6e 6f 75 .......ynchronou +| 3280: 73 28 02 03 01 02 03 01 02 03 04 02 04 01 02 04 s(.............. +| 3296: 01 02 04 03 04 73 74 65 6d 81 32 02 02 01 02 02 .....stem.2..... +| 3312: 01 02 02 01 04 74 65 6d 70 81 35 02 02 01 02 02 .....temp.5..... +| 3328: 01 02 02 02 06 68 72 65 61 64 73 31 02 04 01 02 .....hreads1.... +| 3344: 04 01 02 04 76 02 04 01 02 04 01 02 04 08 03 61 ....v..........a +| 3360: 66 65 81 38 02 02 01 02 02 01 02 02 02 06 72 69 fe.8..........ri +| 3376: 67 67 65 72 81 20 02 03 01 02 03 01 02 03 08 01 gger. .......... +| 3392: 73 22 02 04 01 02 04 01 02 04 01 07 75 6e 6b 6e s...........unkn +| 3408: 6f 77 6e 73 02 03 01 02 03 01 02 03 01 08 76 61 owns..........va +| 3424: 72 69 61 62 6c 65 81 23 02 03 01 02 03 01 02 03 riable.#........ +| 3440: 02 03 64 62 65 81 26 02 03 01 02 03 01 02 03 02 ..dbe.&......... +| 3456: 03 69 65 77 01 02 05 01 02 05 01 02 05 02 03 74 .iew...........t +| 3472: 61 62 37 02 04 01 02 04 01 02 04 04 02 04 01 02 ab7............. +| 3488: 04 01 02 04 01 02 04 01 02 04 01 02 04 01 03 77 ...............w +| 3504: 61 6c 2b 02 03 01 02 03 01 02 03 01 02 03 01 02 al+............. +| 3520: 03 01 02 03 02 05 6f 72 6b 65 72 31 02 03 01 02 ......orker1.... +| 3536: 03 01 02 03 76 02 03 01 02 03 01 02 03 01 01 78 ....v..........x +| 3552: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3568: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3584: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3600: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3616: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3632: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3648: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3664: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3680: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3696: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3712: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3728: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3744: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3760: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3776: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3792: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3808: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3824: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3840: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3856: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3872: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3888: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3904: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3920: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3936: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3952: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3968: 06 01 01 02 01 06 04 30 15 1e 0c 28 1b 0d 0c 15 .......0...(.... +| 3984: 0c 16 14 16 10 0c 17 0e 0e 0f 11 10 10 0e 10 11 ................ +| 4000: 18 11 82 3e 12 10 0f 10 11 10 0f 0f 10 11 0f 0f ...>............ +| 4016: 81 05 18 10 81 45 11 0d 13 0f 10 17 0c 0c 0e 18 .....E.......... +| 4032: 0c 12 10 0e 0d 0f 13 12 24 0f 17 0f 1a 0d 81 1c ........$....... +| 4048: 11 0f 17 10 82 3e 12 11 11 18 0d 12 2a 14 11 10 .....>......*... +| 4064: 13 0f 12 0f 0f 82 3a 15 10 0f 10 4d 0e 1f 0f 0d ......:....M.... +| 4080: 0f 0f 1e 10 10 1a 0f 12 0c 12 14 0f 0e 20 17 19 ............. .. +| page 12 offset 45056 +| 0: 0d 00 00 00 01 0d f4 00 0d f4 00 00 00 00 00 00 ................ +| 3568: 00 00 00 00 84 04 84 80 80 80 80 02 04 00 88 0c ................ +| 3584: 00 07 02 00 01 01 02 56 06 01 01 02 01 06 01 01 .......V........ +| 3600: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3616: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3632: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3648: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3664: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3680: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3696: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3712: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3728: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3744: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3760: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3776: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3792: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3808: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3824: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3840: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3856: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3872: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3888: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3904: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 3920: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 3936: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 3952: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 3968: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 ................ +| 3984: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4000: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| 4016: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 ................ +| 4032: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 ................ +| 4048: 52 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 R............... +| 4064: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 ................ +| 4080: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 ................ +| end c1a.txt.db +}]} {} + +do_catchsql_test 84.1 { + SELECT * FROM t1('R*R*x') ORDER BY rowid DESC; +} {1 {fts5: corruption found reading blob 137438953475 from table "t1"}} + sqlite3_fts5_may_be_corrupt 0 finish_test diff --git a/ext/fts5/test/fts5vocab2.test b/ext/fts5/test/fts5vocab2.test index 7b3c3b0d6..58416a7e9 100644 --- a/ext/fts5/test/fts5vocab2.test +++ b/ext/fts5/test/fts5vocab2.test @@ -305,6 +305,28 @@ do_catchsql_test 6.2 { sqlite3_fts5_may_be_corrupt 0 +#------------------------------------------------------------------------- +reset_db +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b); + CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, col); + + INSERT INTO t1 VALUES('xx', 'xx'); + + CREATE TABLE x1(t); + INSERT INTO x1 VALUES('xx'); + INSERT INTO x1 VALUES('xx'); + + SELECT term, col FROM v1; +} { + xx a xx b +} + +do_execsql_test 7.1 { + SELECT * FROM x1 WHERE 'a'=(SELECT col FROM v1 WHERE term=t) +} {xx xx} + + finish_test diff --git a/ext/intck/sqlite3intck.c b/ext/intck/sqlite3intck.c index ed169a266..5f645fae6 100644 --- a/ext/intck/sqlite3intck.c +++ b/ext/intck/sqlite3intck.c @@ -160,6 +160,7 @@ static char *intckMprintf(sqlite3_intck *p, const char *zFmt, ...){ sqlite3_free(zRet); zRet = 0; } + va_end(ap); return zRet; } diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 57cbe870f..069f3fdb5 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -921,9 +921,11 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( srcTypedArray = new Uint8Array(srcTypedArray); } affirmBindableTypedArray(srcTypedArray); - const heap = wasm.heapForSize(srcTypedArray.constructor); const pRet = wasm.alloc(srcTypedArray.byteLength || 1); - heap.set(srcTypedArray.byteLength ? srcTypedArray : [0], Number(pRet)); + wasm.heapForSize(srcTypedArray.constructor) + .set(srcTypedArray.byteLength ? srcTypedArray : [0], Number(pRet)) + /* Maintenance note: the order of alloc() and heapForSize() calls + is significant: https://sqlite.org/forum/forumpost/05b77273be104532 */; return pRet; }; diff --git a/manifest b/manifest index 5d972e1e2..264ad32c3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.51.0 -D 2025-11-04T19:38:17.314 +C Version\s3.51.1 +D 2025-11-28T17:28:25.933 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -8,7 +8,7 @@ F Makefile.in 3ce07126d7e87c7464301482e161fdae6a51d0a2aa06b200b8f0000ef4d6163b F Makefile.linux-generic bd3e3cacd369821a6241d4ea1967395c962dfe3057e38cb0a435cee0e8b789d0 F Makefile.msc 8dd676302f3165984c046f81af14b6676a334418fa30255efaf439f8033042fa F README.md dae499194b75deed76a13a4a83c82493f2530331882d7dfe5754d63287d3f8f7 -F VERSION 16eddb43056a79c1977427ab7a05f3457c373fa159dcdced8754eb89ce7e06b8 +F VERSION a3971cc92c03ff4c250d95d1ecf61e2433152c21926d8a7dd429b39218a6e4c7 F art/icon-243x273.gif 9750b734f82fdb3dc43127753d5e6fbf3b62c9f4e136c2fbf573b2f57ea87af5 F art/icon-80x90.gif 65509ce3e5f86a9cd64fe7fca2d23954199f31fe44c1e09e208c80fb83d87031 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -22,7 +22,7 @@ F autoconf/Makefile.msc 803aa9e23b6a764751f6a91d0673620eefa46729fb80faf51391120e F autoconf/README.first f1d3876e9a7852c22f275a6f06814e64934cecbc0b5b9617d64849094c1fd136 F autoconf/README.txt b749816b8452b3af994dc6d607394bef3df1736d7e09359f1087de8439a52807 F autoconf/auto.def 3d994f3a9cc9b712dbce92a5708570ddcf3b988141b6eb738f2ed16127a9f0ac -F autoconf/tea/Makefile.in bf6b43eafcd18766d81a8f0085cfc9cb051d8abae9031a8e7c3f5f1246e8f166 +F autoconf/tea/Makefile.in 00e60cf3bf5580f31bfdcf3c914e9ba1831d676948363962de92ce65e5be4431 F autoconf/tea/README.txt 23475876343498ef2b514cc7510e8f1559a17e8e03fbc7a41c1c8a3b89e7b7e3 F autoconf/tea/_teaish.tester.tcl.in 8253b44be88e2e3f21de95a65d3a90c2be8e70b7bdd08a5b80e337ba7402f8f1 F autoconf/tea/auto.def ce95b9450e2fa4ba5dc857e208fe10f4e6f2d737796ac3278aee6079db417529 @@ -50,7 +50,7 @@ F autosetup/proj.tcl 6fc14ef82b19b77a95788ffbcfad7989b4e3cb4ce96a21dcb5cf7312f36 F autosetup/sqlite-config.tcl 5d779fce20c11fde3fe99d157dcd5b5569d729b301141b8dfb8d5aacf9d48cba F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9 F autosetup/teaish/README.txt b40071e6f8506500a2f7f71d5fc69e0bf87b9d7678dd9da1e5b4d0acbf40b1ca -F autosetup/teaish/core.tcl aee092fc71986d1272b835ea7492bb55ffc213a289502e4f14da80cf67b7e3c3 +F autosetup/teaish/core.tcl e014dd95900c7f9a34e8e0f460f47e94841059827bce8b4c49668b0c7ae3f1a0 F autosetup/teaish/feature.tcl 18194fb79a24d30e5bbdeab40999616f39278b53a27525349ded033af2fd73be F autosetup/teaish/tester.tcl 1799514c2652db49561b3386c5242b94534d1663f2cfac861a955e071895fdd0 F configure 9a00b21dfd13757bbfb8d89b30660a89ec1f8f3a79402b8f9f9b6fc475c3303a x @@ -113,7 +113,7 @@ F ext/fts5/fts5_buffer.c f1e6d0324d7c55329d340673befc26681a372a4d36086caa8d1ec7d F ext/fts5/fts5_config.c e7d8dd062b44a66cd77e5a0f74f23a2354cd1f3f8575afb967b2773c3384f7f8 F ext/fts5/fts5_expr.c b8c32da1127bafaf10d6b4768b0dcb92285798524bed2d87a8686f99a8e8d259 F ext/fts5/fts5_hash.c a6266cedd801ab7964fa9e74ebcdda6d30ec6a96107fa24148ec6b7b5b80f6e0 -F ext/fts5/fts5_index.c 8dbda33a9830764167d7697f1c9980c8a6ee74f5decb28206b963222583b8cdd +F ext/fts5/fts5_index.c 5e82963a6691ae519df1e018cc7971272a3da1fd47daca5029af0e9ce24f1a8b F ext/fts5/fts5_main.c 42025174a556257287071e90516d3ab8115daf1dd525a301883544469a260014 F ext/fts5/fts5_storage.c 19bc7c4cbe1e6a2dd9849ef7d84b5ca1fcbf194cefc3e386b901e00e08bf05c2 F ext/fts5/fts5_tcl.c 7fb5a3d3404099075aaa2457307cb459bbc257c0de3dbd52b1e80a5b503e0329 @@ -122,7 +122,7 @@ F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a F ext/fts5/fts5_tokenize.c 49aea8cc400a690a6c4f83c4cedc67f4f8830c6789c4ee343404f62bcaebca7b F ext/fts5/fts5_unicode2.c 536a6dae41d16edadd6a6b58c56e2ebbb133f0dfe757562a2edbcdc9b8362e50 F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d09885d1dadf80 -F ext/fts5/fts5_vocab.c ff0441c4ea165081e8152dec6d29056faa0cdc281a9f218a00e3d7aacc1958bc +F ext/fts5/fts5_vocab.c 23e263ad94ac357cfffd19bd7e001c3f15c4420fb10fa35b5993142127e780e6 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl c5aa7cf7148b6dcffb5b61520ae18212baf169936af734ab265143f59db328fe @@ -162,7 +162,7 @@ F ext/fts5/test/fts5contentless4.test ec34dc69ef474ca9997dae6d91e072906e0e9a5a4b F ext/fts5/test/fts5contentless5.test 38cd0392c730dc7090c550321ce3c24ba4c392bc97308b51a4180e9959dca7b5 F ext/fts5/test/fts5corrupt.test 237fce1c3261bb3a5bec333b0f0dbf5b105ec32627ef14cccbda3cfe13833193 F ext/fts5/test/fts5corrupt2.test 4a03a158c2cb617c9f76d26b35c1ef2534124bc0bbddcea38dfd5b170ebea27b -F ext/fts5/test/fts5corrupt3.test 43d6a836892d79ab738ab89b3b6f4ae46c07ee966193e4b357bbb14e7f81d5da +F ext/fts5/test/fts5corrupt3.test e489b51b6c4ded2a808e0f78bdbe126f6d369de41a59ac2717878e37fc3db0e8 F ext/fts5/test/fts5corrupt4.test dc08d19f5b8943e95a7778a7d8da592042504faf18dd93f68f7d7a0d7d7dd733 F ext/fts5/test/fts5corrupt5.test 73985d4fe6d8f0d5d5c7bcf79ae7c6522c376cd6ad710a0ff2f26e0c2e222abe F ext/fts5/test/fts5corrupt6.test 2d72db743db7b5d9c9a6d0cfef24d799ed1aa5e8192b66c40e871a37ed9eed06 @@ -269,7 +269,7 @@ F ext/fts5/test/fts5update.test b8affd796e45c94a4d19ad5c26606ea06065a0f162a9562d F ext/fts5/test/fts5update2.test c5baa76799ac605ebb8e5e21035db2014b396cef25c903eb96ba39b1d6f9f046 F ext/fts5/test/fts5version.test 44ab35566267b7618c090443de2d9ad84f633df5d20bf72e9bad199ae5fced84 F ext/fts5/test/fts5vocab.test 2a2bdb60d0998fa3124d541b6d30b019504918dc43a6584645b63a24be72f992 -F ext/fts5/test/fts5vocab2.test bbba149c254375d00055930c1a501c9a51e80b0d20bf7b98f3e9fa3b03786373 +F ext/fts5/test/fts5vocab2.test 4265137a3747b27deb1e2e2bde5654120c6de72bfed3238e67806d85af60fc4c F ext/fts5/tool/fts5speed.tcl b0056f91a55b2d1a3684ec05729de92b042e2f85 F ext/fts5/tool/fts5txt2db.tcl c0d43c8590656f8240e622b00957b3a0facc49482411a9fdc2870b45c0c82f9f F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093 @@ -284,7 +284,7 @@ F ext/intck/intck_common.tcl a61fd2697ae55b0a3d89847ca0b590c6e0d8ff64bebb70920d9 F ext/intck/intckbusy.test d5ed4ef85a4b1dc1dee2484bd14a4bb68529659cca743327df0c775f005fa387 F ext/intck/intckcorrupt.test f6c302792326fb3db9dcfc70b554c55369bc4b52882eaaf039cfe0b74c821029 F ext/intck/intckfault.test cff3f75dff74abb3edfcb13f6aa53f6436746ab64b09fe5e2028f051e985efab -F ext/intck/sqlite3intck.c 0d10df36e2b7b438aa80ecd3f5e584d41b747586b038258fe6b407f66b81e7c5 +F ext/intck/sqlite3intck.c b1c8a86f90fc00741d13314db9c58f7e2f92d1d19c5ad1c6904ec83a6bbd5c96 F ext/intck/sqlite3intck.h 2b40c38e7063ab822c974c0bd4aed97dabb579ccfe2e180a4639bb3bbef0f1c9 F ext/intck/test_intck.c 4f9eaadaedccb9df1d26ba41116a0a8e5b0c5556dc3098c8ff68633adcccdea8 F ext/jni/GNUmakefile 8a94e3a1953b88cf117fb2a5380480feada8b4f5316f02572cab425030a720b4 @@ -597,7 +597,7 @@ F ext/wasm/api/pre-js.c-pp.js a876c6399dff29b6fe9e434036beb89889164cc872334e1842 F ext/wasm/api/sqlite3-api-cleanup.js a3d6b9e449aefbb8bba283c2ba9477e2333a0eeb94a7a26b5bf952736f65a6dd F ext/wasm/api/sqlite3-api-glue.c-pp.js d2b8263b3ce0cefc6c5a68d0a4d448a9770eda4bf9d9ded9d7eb0198e4ce4da1 F ext/wasm/api/sqlite3-api-oo1.c-pp.js 31dbfd470c91ffd96d77399b749bab6b69e3ba9074188833f97ac13f087cf07b -F ext/wasm/api/sqlite3-api-prologue.js b5a55ae74efcdcd0aa6a143d59e34137e43ae732f02b563dcab22d735f1599a4 +F ext/wasm/api/sqlite3-api-prologue.js 307583ff39a978c897c4ef4ce53fe231dce5c73dc84785969c81c1ab5960a293 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966 @@ -679,7 +679,7 @@ F src/btree.c cb5b8ceb9baa02a63a2f83dec09c4153e1cfbdf9c2adef5c62c26d2160eeb067 F src/btree.h e823c46d87f63d904d735a24b76146d19f51f04445ea561f71cc3382fd1307f0 F src/btreeInt.h 9c0f9ea5c9b5f4dcaea18111d43efe95f2ac276cd86d770dce10fd99ccc93886 F src/build.c 611e07299d72ff04bbcb9e7109183467e30925d203c3e121ef9bb3cf6876289b -F src/callback.c acae8c8dddda41ee85cfdf19b926eefe830f371069f8aadca3aa39adf5b1c859 +F src/callback.c 3605bbf02bd7ed46c79cd48346db4a32fc51d67624400539c0532f4eead804ad F src/carray.c ff6081a31878fc34df8fa1052a9cbf17ddc22652544dcb3e2326886ed1053b55 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/date.c e19e0cfff9a41bfdd884c655755f6f00bca4c1a22272b56e0dd6667b7ea893a2 @@ -691,7 +691,7 @@ F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f F src/func.c 0b802107498048d3dcac0b757720bcb8506507ce02159e213ab8161458eb293b F src/global.c a19e4b1ca1335f560e9560e590fc13081e21f670643367f99cb9e8f9dc7d615b -F src/hash.c 73934a7f7ab1cb110614a9388cb516893b0cf5b7b69e4fd1a0780ac4ce166be7 +F src/hash.c 03c8c0f4be9e8bcb6de65aa26d34a61d48a9430747084a69f9469fbb00ea52ca F src/hash.h 46b92795a95bfefb210f52f0c316e9d7cdbcdd7e7fcfb0d8be796d3a5767cddf F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 @@ -699,7 +699,7 @@ F src/insert.c dfd311b0ac2d4f6359e62013db67799757f4d2cc56cca5c10f4888acfbbfa3fd F src/json.c fb031340edee159c07ad37dbe668ffe945ed86f525b0eb3822e4a67cbc498a72 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c a3bc9a2522dc3b960e38b7582d1818f6245a49289387c2c7b19f27bfeabf1e81 -F src/main.c ce69a2650e3d359ed6a8a2867ccafb27ac62ce1d39f3120a84ff513320952a6c +F src/main.c 65d11c17890966d271c925c6cc55e3ba50fa08374633cb99c0dee4719a20915a F src/malloc.c 410e570b30c26cc36e3372577df50f7a96ee3eed5b2b161c6b6b48773c650c5e F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2 @@ -736,8 +736,8 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 5616fbcf3b833c7c705b24371828215ad0925d0c0073216c4f153348d5753f0a F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c ba9cd07ffa3277883c1986085f6ddc4320f4d35d5f212ab58df79a7ecc1a576a -F src/shell.c.in 265015aaec4580a0bef50ff570ec8d2813498ee49dcfdf9757e76f3b2a59fc97 -F src/sqlite.h.in f7944026ee89ea348f89aec56372d6d25b6cafc1d89df741278d6917e86326a3 +F src/shell.c.in 223e3703657f5e66c136521a32fc8cc9a7dbbe6b1ade6fd47457e78c38f33e6e +F src/sqlite.h.in c0979f9ac1f5be887397dd2a0bb485636893a81b34d64df85123aae9650c42f2 F src/sqlite3.rc 015537e6ac1eec6c7050e17b616c2ffe6f70fca241835a84a4f0d5937383c479 F src/sqlite3ext.h 7f236ca1b175ffe03316d974ef57df79b3938466c28d2f95caef5e08c57f3a52 F src/sqliteInt.h 88f7fc9ce1630d9a5f7e0a8e1f3287cdc63882fba985c18e7eee1b9f457f59aa @@ -746,7 +746,7 @@ F src/status.c 7565d63a79aa2f326339a24a0461a60096d0bd2bce711fefb50b5c89335f3592 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 F src/tclsqlite.c 3c604c49e6cf4211960a9ddb9505280fd22cde32175f40884c641c0f5a286036 F src/tclsqlite.h 614b3780a62522bc9f8f2b9fb22689e8009958e7aa77e572d0f3149050af348a -F src/test1.c f880ab766eeedf2c063662bd9538b923fd42c4341b7bfc2150a6d93ab8b9341c +F src/test1.c 5d061afe479c7364842e0170be7220dea13389575fa6030d30b3e20bec4e1f75 F src/test2.c 62f0830958f9075692c29c6de51b495ae8969e1bef85f239ffcd9ba5fb44a5ff F src/test3.c 432646f581d8af1bb495e58fc98234380250954f5d5535e507fc785eccc3987a F src/test4.c 0ac87fc13cdb334ab3a71823f99b6c32a6bebe5d603cd6a71d84c823d43a25a0 @@ -756,7 +756,7 @@ F src/test8.c 206d8f3cc73950d252906656e2646b5de0d580b07187b635fcb3edd8c2c5fbc0 F src/test9.c df9ddc7db6ef1b8cf745866ee229090779728bcbe660c7f297d3127ab21d92af F src/test_autoext.c 14d4bbd3d0bd1eec0f6d16b29e28cf1e2d0b020d454835f0721a5f68121ac10f F src/test_backup.c a2bfd90d2ff2511b8635507bdb30fa9b605ade19c16b533066cae3077f5bdb72 -F src/test_bestindex.c 3401bee51665cbf7f9ed2552b5795452a8b86365e4c9ece745b54155a55670c6 +F src/test_bestindex.c a9428931bec06de830b2630f57a7b1f2711761269f04df62b7aa1affcbce15bb F src/test_blob.c 77b994e17f2c87055f44fd96c9a206c5a7155bae2cda2769af60c2f3582f962c F src/test_btree.c 28283787d32b8fa953eb77412ad0de2c9895260e4e5bd5a94b3c7411664f90d5 F src/test_config.c 18aa596d37de1d5968c439fd58ebf38bc4d9c9d1db63621504e241fde375cecd @@ -812,12 +812,12 @@ F src/vdbemem.c 48e562ff27e6386eb8613207ac27d3d98c1f67fdc4775a1ab13759d2c2a1c021 F src/vdbesort.c b69220f4ea9ffea5fdef34d968c60305444eea909252a81933b54c296d9cca70 F src/vdbetrace.c 49e689f751505839742f4a243a1a566e57d5c9eaf0d33bbaa26e2de3febf7b41 F src/vdbevtab.c fc46b9cbd759dc013f0b3724549cc0d71379183c667df3a5988f7e2f1bd485f3 -F src/vtab.c 828221bdbeaaa6d62126ee6d07fd4ec0d09dcaea846f87ad01944d8b7e548859 +F src/vtab.c 5437ce986db2f70e639ce8a3fe68dcdfe64b0f1abb14eaebecdabd5e0766cc68 F src/vxworks.h 9d18819c5235b49c2340a8a4d48195ec5d5afb637b152406de95a9436beeaeab F src/wal.c 505a98fbc599a971d92cb90371cf54546c404cd61e04fd093e7b0c8ff978f9b6 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 -F src/where.c 7d17cd5cb883b2166097957e20c4aab2d0d98e0c1141002ef77b5f6b9deed844 +F src/where.c 1b554a868134cbc9ca2192385403c0b63e5073ff01a6cdd600a846c09f843165 F src/whereInt.h 8d94cb116c9e06205c3d5ac87af065fc044f8cf08bfdccd94b6ea1c1308e65da F src/wherecode.c 71c5c6804b7f882dec8ec858758accae02fcfca13df3cc720f1f258e663ec7c5 F src/whereexpr.c 403a44eeec1a0f0914fccc6a59376b6924bc00ef6728fe6ffce4cf3051b320fc @@ -914,7 +914,7 @@ F test/bestindexA.test e1b5def6b190797cacf008e6815ffb78fb30261999030d60a728d572e F test/bestindexB.test 328b97b69cd1a20928d5997f9ecb04d2e00f1d18e19ab27f9e9adb44d7bc51ce F test/bestindexC.test 95b4a527b1a5d07951d731604a6d4cf7e5a806b39cea0e7819d4c9667e11c3fc F test/bestindexD.test 6a8f6f84990bcf17dfa59652a1f935beddb7afd96f8302830fbc86b0a13df3c3 -F test/bestindexE.test f0c7105d1e7facaa8f5e6c498849cc6dcadd533b35ea9e5e1176e54a9a2d70f1 +F test/bestindexE.test 297f3ea8500a8f3c17d6f78e55bdfee089064c6144ee84a110bd005a03338f49 F test/between.test e7587149796101cbe8d5f8abae8d2a7b87f04d8226610aa1091615005dcf4d54 F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc @@ -1093,7 +1093,7 @@ F test/exclusive.test 7ff63be7503990921838d5c9f77f6e33e68e48ed1a9d48cd28745bf650 F test/exclusive2.test cd70b1d9c6fffd336f9795b711dcc5d9ceba133ad3f7001da3fda63615bdc91e F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac -F test/existsexpr.test 9c4b77c4729281cc2ae63b9b460d0598ce28cc7876135e3e2c21629bbc8d077a +F test/existsexpr.test e4674fb8d610e82540126ed252e6aa48922882e1bcec6522d7e5a44fe715be61 F test/existsexpr2.test dc23e76389eff3d29f6488ff733012a3560cd67ec8cfaecbecd52cced5d5af11 F test/existsfault.test ff41c11f3052c1bbd4f8dd557802310026253d67d7c4e3a180c16d2f0862973e F test/expr.test db981f8a85520e99ae20aab7ad2e9b5b0437ed09159b57ced434c672075d2e61 @@ -1429,7 +1429,7 @@ F test/misc5.test 02fcaf4d42405be02ec975e946270a50b0282dac98c78303ade0d1392839d2 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test d595599972ec0b436985f0f02f243b68500ffc977b9b3194ec66c0866cfddcab F test/misc8.test 08d2380bc435486b12161521f225043ac2be26f02471c2c1ea4cac0b1548edbd -F test/misuse.test 46d42ffdf375833ea5828796e56f84660344f7548659b493059f152f00e66840 +F test/misuse.test 859f37014d9824ca66bd90c36372c08c80c51c9593a7cfa8a31d4f92cd4d5b7f F test/mjournal.test 28a08d5cb5fb5b5702a46e19176e45e964e0800d1f894677169e79f34030e152 F test/mmap1.test 18de3fd7b70a777af6004ca2feecfcdd3d0be17fa04058e808baf530c94b1a1d F test/mmap2.test dba452dc7db91e9df10f70bdd73dc4190c7b8ee7b5133b4684f04277ada0b9ac @@ -1526,8 +1526,8 @@ F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2 F test/reservebytes.test 6163640b5a5120c0dee6591481e673a0fa0bf0d12d4da7513bad692c1a49a162 F test/resetdb.test 54c06f18bc832ac6d6319e5ab23d5c8dd49fdbeec7c696d791682a8006bd5fc3 F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb -F test/returning1.test 212cd4111bb941a60abf608f20250db666c21eb1bc4d49217e96c87ff3ab9d1a -F test/returningfault.test ae4c4b5e8745813287a359d9ccdb9d5c883c2e68afb18fb0767937d5de5692a4 +F test/returning1.test cd32517148948859db214dd814354597dd40e7489259590fac1a4f7bf44deb97 +F test/returningfault.test 5f9649d05680357ab077fa6bad574a3eb3f6e4858a6880b25171be516a4efcda F test/rollback.test 952c4d805bca96adc2be76f621ea22115fe40b330015af36fcc8028c8547fcee F test/rollback2.test 3f3a4e20401825017df7e7671e9f31b6de5fae5620c2b9b49917f52f8c160a8f F test/rollbackfault.test 0e646aeab8840c399cfbfa43daab46fd609cf04a @@ -2171,11 +2171,10 @@ F tool/version-info.c 33d0390ef484b3b1cb685d59362be891ea162123cea181cb8e6d2cf6dd F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ccabbe06d4e0650eecc91296d6a3040ed7fef06b47e4d63393a2367c8c8f55cf -R 9324db4aaf883cfc2983724be7d6c4b2 -T +sym-major-release * +P 40ddaca3fb752425c26570365a9f31820786d21d043c1c0a4b49746ff9bc0782 +R e6faec25512909b375a6de433bcc999d T +sym-release * -T +sym-version-3.51.0 * +T +sym-version-3.51.1 * U drh -Z bd3359fae465c4238136a20bd4edb791 +Z a5cb39bb0c202daa26ad87d8a3e3f169 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.tags b/manifest.tags new file mode 100644 index 000000000..1630d85a0 --- /dev/null +++ b/manifest.tags @@ -0,0 +1,4 @@ +branch branch-3.51 +tag release +tag branch-3.51 +tag version-3.51.1 diff --git a/manifest.uuid b/manifest.uuid index d96096a32..b5f89836b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb2c931ae597f8d00a37574ff67aeed3eced4e5547f9120744ae4bfa8e74527b +281fc0e9afc38674b9b0991943b9e9d1e64c6cbdb133d35f6f5c87ff6af38a88 diff --git a/src/callback.c b/src/callback.c index 6fe21a295..e6418097f 100644 --- a/src/callback.c +++ b/src/callback.c @@ -507,6 +507,7 @@ void sqlite3SchemaClear(void *p){ for(pElem=sqliteHashFirst(&temp2); pElem; pElem=sqliteHashNext(pElem)){ sqlite3DeleteTrigger(&xdb, (Trigger*)sqliteHashData(pElem)); } + sqlite3HashClear(&temp2); sqlite3HashInit(&pSchema->tblHash); for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){ diff --git a/src/hash.c b/src/hash.c index 8cc6c0966..35df65808 100644 --- a/src/hash.c +++ b/src/hash.c @@ -270,3 +270,4 @@ void *sqlite3HashInsert(Hash *pH, const char *pKey, void *data){ insertElement(pH, pH->ht ? &pH->ht[new_elem->h % pH->htsize] : 0, new_elem); return 0; } + diff --git a/src/main.c b/src/main.c index ef1bc6853..6efe538d4 100644 --- a/src/main.c +++ b/src/main.c @@ -1394,6 +1394,7 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ /* Clear the TEMP schema separately and last */ if( db->aDb[1].pSchema ){ sqlite3SchemaClear(db->aDb[1].pSchema); + assert( db->aDb[1].pSchema->trigHash.count==0 ); } sqlite3VtabUnlockList(db); @@ -2722,7 +2723,7 @@ const char *sqlite3_errmsg(sqlite3 *db){ */ int sqlite3_set_errmsg(sqlite3 *db, int errcode, const char *zMsg){ int rc = SQLITE_OK; - if( !sqlite3SafetyCheckSickOrOk(db) ){ + if( !sqlite3SafetyCheckOk(db) ){ return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(db->mutex); diff --git a/src/shell.c.in b/src/shell.c.in index bd44a3251..23aed0de5 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -9261,6 +9261,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } p->showHeader = savedShowHeader; p->shellFlgs = savedShellFlags; + rc = p->nErr>0; }else if( c=='e' && cli_strncmp(azArg[0], "echo", n)==0 ){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index a821ecb30..f6ed48c20 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -10426,7 +10426,7 @@ int sqlite3_vtab_in(sqlite3_index_info*, int iCons, int bHandle); **   ){ **   // do something with pVal **   } -**   if( rc!=SQLITE_OK ){ +**   if( rc!=SQLITE_DONE ){ **   // an error has occurred **   } ** )^ diff --git a/src/test1.c b/src/test1.c index f89359932..f8e83dc42 100644 --- a/src/test1.c +++ b/src/test1.c @@ -4972,6 +4972,37 @@ static int SQLITE_TCLAPI test_errmsg16( return TCL_OK; } +/* +** Usage: sqlite3_set_errmsg DB ERRCODE ERRMSG +*/ +static int SQLITE_TCLAPI test_set_errmsg( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *zDb = 0; + const char *zErr = 0; + int iErr = 0; + sqlite3 *db = 0; + int rc; + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB ERRCODE ERRMSG"); + return TCL_ERROR; + } + zDb = Tcl_GetString(objv[1]); + if( zDb[0] ){ + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[2], &iErr) ) return TCL_ERROR; + zErr = Tcl_GetString(objv[3]); + + rc = sqlite3_set_errmsg(db, iErr, (zErr[0] ? zErr : 0)); + Tcl_SetResult(interp, (char *)t1ErrorName(rc), 0); + return TCL_OK; +} + /* ** Usage: sqlite3_prepare DB sql bytes ?tailvar? ** @@ -8274,6 +8305,7 @@ static int SQLITE_TCLAPI optimization_control( { "balanced-merge", SQLITE_BalancedMerge }, { "propagate-const", SQLITE_PropagateConst }, { "one-pass", SQLITE_OnePass }, + { "exists-to-join", SQLITE_ExistsToJoin }, }; if( objc!=4 ){ @@ -9077,6 +9109,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_errmsg", test_errmsg ,0 }, { "sqlite3_error_offset", test_error_offset ,0 }, { "sqlite3_errmsg16", test_errmsg16 ,0 }, + { "sqlite3_set_errmsg", test_set_errmsg ,0 }, { "sqlite3_open", test_open ,0 }, { "sqlite3_open16", test_open16 ,0 }, { "sqlite3_open_v2", test_open_v2 ,0 }, diff --git a/src/test_bestindex.c b/src/test_bestindex.c index 2f9203d85..f6b5db0fb 100644 --- a/src/test_bestindex.c +++ b/src/test_bestindex.c @@ -101,6 +101,7 @@ typedef struct tcl_vtab tcl_vtab; typedef struct tcl_cursor tcl_cursor; typedef struct TestFindFunction TestFindFunction; +typedef struct TestVtabContext TestVtabContext; /* ** A fs virtual-table object @@ -125,6 +126,10 @@ struct TestFindFunction { TestFindFunction *pNext; }; +struct TestVtabContext { + Tcl_Interp *interp; + Tcl_Obj *pDefault; +}; /* ** Dequote string z in place. @@ -178,25 +183,33 @@ static int tclConnect( sqlite3_vtab **ppVtab, char **pzErr ){ - Tcl_Interp *interp = (Tcl_Interp*)pAux; + TestVtabContext *pCtx = (TestVtabContext*)pAux; + Tcl_Interp *interp = pCtx->interp; tcl_vtab *pTab = 0; char *zCmd = 0; Tcl_Obj *pScript = 0; int rc = SQLITE_OK; - if( argc!=4 ){ + if( argc!=4 && (argc!=3 || pCtx->pDefault==0) ){ *pzErr = sqlite3_mprintf("wrong number of arguments"); return SQLITE_ERROR; } - zCmd = sqlite3_malloc64(strlen(argv[3])+1); + if( argc==4 ){ + zCmd = sqlite3_malloc64(strlen(argv[3])+1); + } pTab = (tcl_vtab*)sqlite3_malloc64(sizeof(tcl_vtab)); - if( zCmd && pTab ){ - memcpy(zCmd, argv[3], strlen(argv[3])+1); - tclDequote(zCmd); + if( (zCmd || argc==3) && pTab ){ memset(pTab, 0, sizeof(tcl_vtab)); - pTab->pCmd = Tcl_NewStringObj(zCmd, -1); + if( zCmd ){ + memcpy(zCmd, argv[3], strlen(argv[3])+1); + tclDequote(zCmd); + pTab->pCmd = Tcl_NewStringObj(zCmd, -1); + }else{ + pTab->pCmd = Tcl_DuplicateObj(pCtx->pDefault); + } + pTab->interp = interp; pTab->db = db; Tcl_IncrRefCount(pTab->pCmd); @@ -208,7 +221,11 @@ static int tclConnect( rc = Tcl_EvalObjEx(interp, pScript, TCL_EVAL_GLOBAL); if( rc!=TCL_OK ){ *pzErr = sqlite3_mprintf("%s", Tcl_GetStringResult(interp)); - rc = SQLITE_ERROR; + if( sqlite3_stricmp(*pzErr, "database schema has changed")==0 ){ + rc = SQLITE_SCHEMA; + }else{ + rc = SQLITE_ERROR; + } }else{ rc = sqlite3_declare_vtab(db, Tcl_GetStringResult(interp)); if( rc!=SQLITE_OK ){ @@ -802,6 +819,39 @@ static int tclFindFunction( return iRet; } +static int tclUpdate( + sqlite3_vtab *tab, + int nArg, + sqlite3_value **apVal, + sqlite3_int64 *piRowid +){ + tcl_vtab *pTab = (tcl_vtab*)tab; + Tcl_Interp *interp = pTab->interp; + Tcl_Obj *pEval = Tcl_DuplicateObj(pTab->pCmd); + Tcl_Obj *pRes = 0; + int rc = TCL_OK; + + Tcl_IncrRefCount(pEval); + Tcl_ListObjAppendElement(interp, pEval, Tcl_NewStringObj("xUpdate",-1)); + + rc = Tcl_EvalObjEx(interp, pEval, TCL_EVAL_GLOBAL); + Tcl_DecrRefCount(pEval); + + if( rc==TCL_OK ){ + Tcl_Obj *pRes = Tcl_GetObjResult(interp); + Tcl_WideInt v; + rc = Tcl_GetWideIntFromObj(interp, pRes, &v); + *piRowid = (sqlite3_int64)v; + } + + if( rc!=TCL_OK ){ + tab->zErrMsg = sqlite3_mprintf("%s", Tcl_GetStringResult(pTab->interp)); + return rc; + } + + return SQLITE_OK; +} + /* ** A virtual table module that provides read-only access to a ** Tcl global variable namespace. @@ -833,12 +883,47 @@ static sqlite3_module tclModule = { 0, /* xShadowName */ 0 /* xIntegrity */ }; +static sqlite3_module tclModuleUpdate = { + 0, /* iVersion */ + tclConnect, + tclConnect, + tclBestIndex, + tclDisconnect, + tclDisconnect, + tclOpen, /* xOpen - open a cursor */ + tclClose, /* xClose - close a cursor */ + tclFilter, /* xFilter - configure scan constraints */ + tclNext, /* xNext - advance a cursor */ + tclEof, /* xEof - check for end of scan */ + tclColumn, /* xColumn - read data */ + tclRowid, /* xRowid - read data */ + tclUpdate, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + tclFindFunction, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; /* ** Decode a pointer to an sqlite3 object. */ extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); +static void delTestVtabCtx(void *p){ + TestVtabContext *pCtx = (TestVtabContext*)p; + if( pCtx->pDefault ){ + Tcl_DecrRefCount(pCtx->pDefault); + } + ckfree(pCtx); +} + /* ** Register the echo virtual table module. */ @@ -849,13 +934,25 @@ static int SQLITE_TCLAPI register_tcl_module( Tcl_Obj *CONST objv[] /* Command arguments */ ){ sqlite3 *db; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB"); + if( objc!=2 && objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB ?DEFAULT-CMD?"); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; #ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3_create_module(db, "tcl", &tclModule, (void *)interp); + { + sqlite3_module *pMod = &tclModule; + TestVtabContext *pCtx = (TestVtabContext*)ckalloc(sizeof(TestVtabContext)); + pCtx->interp = interp; + pCtx->pDefault = 0; + if( objc==3 ){ + pCtx->pDefault = objv[2]; + Tcl_IncrRefCount(pCtx->pDefault); + } + + if( objc==3 ){ pMod = &tclModuleUpdate; } + sqlite3_create_module_v2(db, "tcl", pMod, (void*)pCtx, delTestVtabCtx); + } #endif return TCL_OK; } diff --git a/src/vtab.c b/src/vtab.c index e40f60873..ed4b0afaf 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -1279,9 +1279,12 @@ int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){ addModuleArgument(pParse, pTab, sqlite3DbStrDup(db, pTab->zName)); addModuleArgument(pParse, pTab, 0); addModuleArgument(pParse, pTab, sqlite3DbStrDup(db, pTab->zName)); + db->nSchemaLock++; rc = vtabCallConstructor(db, pTab, pMod, pModule->xConnect, &zErr); + db->nSchemaLock--; if( rc ){ sqlite3ErrorMsg(pParse, "%s", zErr); + pParse->rc = rc; sqlite3DbFree(db, zErr); sqlite3VtabEponymousTableClear(db, pMod); } diff --git a/src/where.c b/src/where.c index b95f6d2a3..6313b87ca 100644 --- a/src/where.c +++ b/src/where.c @@ -7424,8 +7424,22 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2); } #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */ - if( pTabList->a[pLevel->iFrom].fg.fromExists ){ - sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+2); + if( pTabList->a[pLevel->iFrom].fg.fromExists && i==pWInfo->nLevel-1 ){ + /* If the EXISTS-to-JOIN optimization was applied, then the EXISTS + ** loop(s) will be the inner-most loops of the join. There might be + ** multiple EXISTS loops, but they will all be nested, and the join + ** order will not have been changed by the query planner. If the + ** inner-most EXISTS loop sees a single successful row, it should + ** break out of *all* EXISTS loops. But only the inner-most of the + ** nested EXISTS loops should do this breakout. */ + int nOuter = 0; /* Nr of outer EXISTS that this one is nested within */ + while( nOutera[pLevel[-nOuter-1].iFrom].fg.fromExists ) break; + nOuter++; + } + testcase( nOuter>0 ); + sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel[-nOuter].addrBrk); + VdbeComment((v, "EXISTS break")); } /* The common case: Advance to the next row */ if( pLevel->addrCont ) sqlite3VdbeResolveLabel(v, pLevel->addrCont); diff --git a/test/bestindexE.test b/test/bestindexE.test index 48397fc7b..d225d74ae 100644 --- a/test/bestindexE.test +++ b/test/bestindexE.test @@ -13,7 +13,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -set testprefix bestindex1 +set testprefix bestindexE ifcapable !vtab { finish_test @@ -124,7 +124,68 @@ do_bestindex_test 2.2 { {Customer: oid=?} } +#-------------------------------------------------------------------------- +reset_db +register_tcl_module db eponymous_cmd +proc eponymous_cmd {method args} { + switch -- $method { + xConnect { + db eval { SELECT * FROM sqlite_schema } + return "CREATE TABLE t1 (a, b)" + } + xBestIndex { + return "idxnum 555" + } + + xFilter { + return [list sql {SELECT 123, 'A', 'B'}] + } + + xUpdate { + return 123 + } + + } + + return {} +} + +do_execsql_test 3.1.0 { + PRAGMA table_info = tcl +} { + 0 a {} 0 {} 0 1 b {} 0 {} 0 +} +do_execsql_test 3.1.1 { + SELECT rowid, * FROM tcl +} {123 A B} +do_execsql_test 3.1.2 { + INSERT INTO tcl VALUES('i', 'ii') RETURNING *; +} {i ii} +do_test 3.1.3 { + db last_insert_rowid +} {123} + +do_execsql_test 3.1.4 { + CREATE TABLE x1(x); +} + +db close +sqlite3 db test.db +register_tcl_module db eponymous_cmd +sqlite3 db2 test.db + +# Load the schema into connection [db] +do_execsql_test 3.2.1 { SELECT * FROM x1 } + +# Modify the schema on disk +do_execsql_test -db db2 3.2.2 { CREATE TABLE x2(x) } + +# Insert with RETURNING trigger on eponymous table. +do_execsql_test 3.2.3 { + INSERT INTO tcl VALUES('i', 'ii') RETURNING *; +} {i ii} finish_test + diff --git a/test/existsexpr.test b/test/existsexpr.test index d02f8c5c1..38392b07f 100644 --- a/test/existsexpr.test +++ b/test/existsexpr.test @@ -429,4 +429,52 @@ do_execsql_test 8.0 { SELECT * FROM t1 WHERE EXISTS (SELECT 1 FROM t0 LIMIT 0); } {} +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 9.0 { + CREATE TABLE t1(xx); + INSERT INTO t1 VALUES('big string value'); +} {} + +do_execsql_test 9.1 { + PRAGMA automatic_index = off; + CREATE TABLE t2(ii); + INSERT INTO t2 VALUES(100); + INSERT INTO t2 VALUES(200); +} + +do_execsql_test 9.2 { + CREATE TABLE t3(yy); + INSERT INTO t3 VALUES(200); +} + +do_execsql_test 9.3 { + SELECT 1 FROM t2 WHERE EXISTS ( SELECT 1 FROM t3 WHERE yy==t2.ii ) +} {1} + +do_execsql_test 9.4 { + SELECT EXISTS ( + SELECT 1 FROM t2 WHERE EXISTS ( SELECT 1 FROM t3 WHERE yy==t2.ii ) + ) +} {1} + +do_execsql_test 9.5 { + SELECT 1234 WHERE EXISTS ( + SELECT 1 FROM t2 WHERE EXISTS ( SELECT 1 FROM t3 WHERE yy==t2.ii ) + ) +} {1234} + +set Q { + SELECT * FROM t1 WHERE + EXISTS ( + SELECT 1 FROM t2 WHERE EXISTS ( SELECT 1 FROM t3 WHERE yy==t2.ii ) + ) +} + +do_execsql_test 9.5 $Q {{big string value}} +catch { optimization_control db exists-to-join 0 } +db cache flush +do_execsql_test 9.6 $Q {{big string value}} + finish_test diff --git a/test/misuse.test b/test/misuse.test index 2ba61f9eb..640cb5a4d 100644 --- a/test/misuse.test +++ b/test/misuse.test @@ -211,4 +211,16 @@ if {[clang_sanitize_address]==0 && 0} { } {1 {(21) bad parameter or other API misuse}} } +#------------------------------------------------------------------------- +reset_db +do_test misuse-6.0 { + sqlite3_set_errmsg db 1 "an error has occurred" +} {SQLITE_OK} +do_test misuse-6.1 { + sqlite3_errmsg db +} {an error has occurred} +do_test misuse-6.2 { + sqlite3_set_errmsg "" 1 "an error has occurred" +} {SQLITE_MISUSE} + finish_test diff --git a/test/returning1.test b/test/returning1.test index e7be7c65a..9ab646a3b 100644 --- a/test/returning1.test +++ b/test/returning1.test @@ -544,4 +544,54 @@ do_catchsql_test 22.1 { } {1 {no such column: sqlite_master.name}} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 23.0 { + PRAGMA recursive_triggers = 1; + CREATE TABLE t1(x, y); + CREATE TRIGGER t1insert AFTER INSERT ON t1 WHEN new.x<5 BEGIN + INSERT INTO t1 VALUES(new.x+1, new.y); + END; +} + +do_execsql_test 23.1 { + INSERT INTO t1 VALUES(1, 'one') RETURNING *; +} {1 one} + +do_execsql_test 23.2 { + SELECT * FROM t1 +} {1 one 2 one 3 one 4 one 5 one} + +#------------------------------------------------------------------------- +reset_db +ifcapable fts5 { + + do_execsql_test 24.0 { + CREATE VIRTUAL TABLE ft USING fts5(c); + CREATE TABLE t1(x); + INSERT INTO t1 VALUES('x'); + } + + db close + + sqlite3 db test.db + sqlite3 db2 test.db + + do_execsql_test 24.1 { + SELECT * FROM t1 + } {x} + + do_execsql_test -db db2 24.2 { + CREATE TABLE t2(y); + INSERT INTO t2 VALUES('y'); + } {} + + db2 close + + do_execsql_test 24.3 { + INSERT INTO ft VALUES('hello world') RETURNING * + } {{hello world}} +} + + finish_test diff --git a/test/returningfault.test b/test/returningfault.test index 8bf6fbfe0..b0177e6dc 100644 --- a/test/returningfault.test +++ b/test/returningfault.test @@ -14,13 +14,15 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/malloc_common.tcl +set ::testprefix returningfault + do_execsql_test 1.0 { CREATE TABLE t1 (b); } {} faultsim_save_and_close -do_faultsim_test pagerfault-1 -faults oom-t* -prep { +do_faultsim_test 1 -faults oom-t* -prep { faultsim_restore_and_reopen } -body { execsql { @@ -32,5 +34,52 @@ do_faultsim_test pagerfault-1 -faults oom-t* -prep { faultsim_test_result {1 {sub-select returns 5 columns - expected 1}} } +ifcapable vtab { + reset_db + do_execsql_test 2.0 { + CREATE TABLE t1(x); + } + + proc eponymous_cmd {method args} { + switch -- $method { + xConnect { + db eval { SELECT * FROM sqlite_schema } + return "CREATE TABLE t1 (a, b)" + } + + xBestIndex { + return "idxnum 555" + } + + xFilter { + return [list sql {SELECT 123, 'A', 'B'}] + } + + xUpdate { + return 123 + } + + } + + return {} + } + + faultsim_save_and_close + + do_faultsim_test 2 -faults oom* -prep { + faultsim_restore_and_reopen + register_tcl_module db eponymous_cmd + db eval { SELECT * FROM t1 } + sqlite3 db2 test.db + db2 eval { CREATE TABLE t2(y) } + db2 close + } -body { + db eval { + INSERT INTO tcl VALUES('hello', 'world') RETURNING * + } + } -test { + faultsim_test_result {0 {hello world}} {1 {vtable constructor failed: tcl}} + } +} finish_test From c864121b6c35ca03750cab477c146aab9a7b388d Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Tue, 2 Dec 2025 09:57:17 -0500 Subject: [PATCH 49/75] Removes unnecessary shutdown and uri config changes in core tests --- test/sqlcipher-core.test | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/sqlcipher-core.test b/test/sqlcipher-core.test index 6985cac87..9c6e54e53 100644 --- a/test/sqlcipher-core.test +++ b/test/sqlcipher-core.test @@ -957,8 +957,6 @@ file delete -force test.db # close database # open normally providing key via pragma verify # correct key works -sqlite3_shutdown -sqlite3_config_uri 1 do_test uri-key { sqlite_orig db file:test.db?a=a&key=testkey&c=c @@ -996,8 +994,6 @@ do_test uri-key-2 { } {1 {file is not a database}} db close file delete -force test.db -sqlite3_shutdown -sqlite3_config_uri 0 finish_test From 8fd1ef415637eda404933a0fda15ad17cfd24e75 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Tue, 2 Dec 2025 10:46:11 -0500 Subject: [PATCH 50/75] Fixes CHANGELOG.md markdown formatting, typos, and inline code snippets --- CHANGELOG.md | 142 +++++++++++++++++++++++++-------------------------- 1 file changed, 71 insertions(+), 71 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b5ff1965..b6b6b0435 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,30 +5,30 @@ Notable changes to this project are documented in this file. ## [4.11.0] - (October 2025 - [4.11.0 changes]) - Converts log output to UTF-16 when writing to stdout or stderr on Windows -- Fixes scope issues to allow --disable-amalgamation to work properly -- Replaces fortuna seeding mechanism for libtomcrypt with rng_get_bytes() -- Removes CocoaPods support (SQLCipher.podspec.json) +- Fixes scope issues to allow `--disable-amalgamation` to work properly +- Replaces fortuna seeding mechanism for libtomcrypt with `rng_get_bytes()` +- Removes CocoaPods support (`SQLCipher.podspec.json`) - Fixes includes and macros to support non-amalgamated builds -- Fixes check for __has_feature to resolve issue with compilers that don't support it -- Corrects return value from sqlcipher_fprintf -- Fixes use of provider free_ctx +- Fixes check for `__has_feature` to resolve issue with compilers that don't support it +- Corrects return value from `sqlcipher_fprintf` +- Fixes use of provider `free_ctx` - Fixes some compiler warnings ## [4.10.0] - (August 2025 - [4.10.0 changes]) - Updates baseline to SQLite 3.50.4 - Allows compile time override of default log level via `SQLCIPHER_LOG_LEVEL_DEFAULT` macro -- Fixes issue building with `-fstanitize=address` on macOS +- Fixes issue building with `-fsanitize=address` on macOS - Fixes detection of CommonCrypto version on macOS - Improves CommonCrypto version detection on iOS ## [4.9.0] - (May 2025 - [4.9.0 changes]) - Updates baseline to upstream SQLite 3.49.2 - Removes use of static mutex in `sqlcipher_extra_shutdown()` - + ## [4.8.0] - (April 2025 - [4.8.0 changes]) - Fixes regression in `PRAGMA cipher_migrate` where an error would be thrown when migrating a current-version database - Adds selective locking in critical sections of the library for shared cache connections (Note: use of shared cache is still strongly discouraged) -- Standardizes initial private heap size to 48KB to ensure mlock under constrained limits +- Standardizes initial private heap size to 48KB to ensure `mlock` under constrained limits - Removes changes to windows working set sizes - Improvements to logging of memory stats and other cleanup @@ -45,7 +45,7 @@ Notable changes to this project are documented in this file. - Improves error handling in `sqlcipher_export()` and `PRAGMA cipher_migrate` - Allows setting custom compile-time default cryptographic provider via the `SQLCIPHER_CRYPTO_CUSTOM` macro - Removes support for end-of-life OpenSSL versions older than 3.0 -__BREAKING CHANGE__: `SELECT` statements (now also including schema independent queries like `SELECT 1`) cannot be executed on encrypt ed databases prior to setting the database key (behavior inherited from upstream SQLite) +- __BREAKING CHANGE__: `SELECT` statements (now also including schema independent queries like `SELECT 1`) cannot be executed on encrypted databases prior to setting the database key (behavior inherited from upstream SQLite) - __BREAKING CHANGE__: Renames `configure` flag `--enable-tempstore=yes` to `--with-tempstore=yes` for alignment with SQLite (change required for upstream SQLite autosetup) - __BREAKING CHANGE__: Renames default executable and library build outputs from `sqlcipher` and `libsqlcipher` to `sqlite3` and `libsqlite3` (for alignment with SQLite) - __BREAKING CHANGE__: Removes `configure` flag `--with-crypto-lib` (replace with appropriate `-DSQLCIPHER_CRYPTO_*` CFLAG) @@ -72,16 +72,16 @@ __BREAKING CHANGE__: `SELECT` statements (now also including schema independent ## [4.5.7] - (April 2024 - [4.5.7 changes]) - Updates baseline to upstream SQLite 3.45.3 -- Adds "device" logging and profile target using os_log for Apple (and logcat on Android) -- Fixes issues compiling with SQLITE_OMIT_LOG -- fixes malformed man page caused by old merge conflict +- Adds "device" logging and profile target using `os_log` for Apple (and logcat on Android) +- Fixes issues compiling with `SQLITE_OMIT_LOG` +- Fixes malformed man page caused by old merge conflict - Updates podspec for current Xcode versions, improved Swift support, and Privacy Manifest ## [4.5.6] - (January 2024 - [4.5.6 changes]) - Updates baseline to upstream SQLite 3.44.2 -- Improve PRAGMA cipher_integrity check to report expected page size if invalid -- Implement PRAGMA page_size compatibility with PRAGMA cipher_page_size so both will operate properly on encrypted databases -- Updates LICENSE.md with SQLCipher license to avoid ambiguity and remove redundance +- Improves `PRAGMA cipher_integrity_check` to report expected page size if invalid +- Implements `PRAGMA page_size` compatibility with `PRAGMA cipher_page_size` so both will operate properly on encrypted databases +- Updates `LICENSE.md` with SQLCipher license to avoid ambiguity and remove redundancy ## [4.5.5] - (August 2023 - [4.5.5 changes]) - Updates baseline to upstream SQLite 3.42.0 @@ -94,7 +94,7 @@ __BREAKING CHANGE__: `SELECT` statements (now also including schema independent ## [4.5.4] - (April 2023 - [4.5.4 changes]) - Updates baseline to upstream SQLite 3.41.2 - Updates minimum Apple SDK versions in podspec for new Xcode compatibility -- Return runtime OpenSSL version from PRAGMA cipher_provider_version (instead of hardcoded value) +- Return runtime OpenSSL version from `PRAGMA cipher_provider_version` (instead of hardcoded value) - Adds guard against zero block size and crash if cryptographic provider initialization fails - When an ATTACH occurs creating a new encrypted database as the first operation after keying the main database, the new database will have the same salt value. @@ -104,46 +104,46 @@ __BREAKING CHANGE__: `SELECT` statements (now also including schema independent ## [4.5.2] - (August 2022 - [4.5.2 changes]) - Updates source code baseline to upstream SQLite 3.39.2 - Simplifies OpenSSL version conditional code -- Fixes issue where PRAGMA cipher_memory_security could report OFF when it was actually ON -- Fixes fix unfreed OpenSSL allocation when compiled against version 3 +- Fixes issue where `PRAGMA cipher_memory_security` could report OFF when it was actually ON +- Fixes unfreed OpenSSL allocation when compiled against version 3 - Fixes support for building against recent versions of BoringSSL ## [4.5.1] - (March 2022 - [4.5.1 changes]) - Updates source code baseline to upstream SQLite 3.37.2 -- Adds PRAGMA cipher_log and cipher_log_level features to allow logging of TRACE, DEBUG, INFO, WARN, and ERROR messages to stdout, stderr, file, or logcat -- Modifies PRAGMA cipher_profile to use sqlite3_trace_v2 and adds logcat target for Android -- Updates OpenSSL provider to use EVP_MAC API with version 3+ -- Adds new PRAGMA cipher_test_on, cipher_test_off, and cipher_test_rand (available when compiled with -DSQLCIPHER_TEST) to facilitate simulation of error conditions -- Fixes PRAGMA cipher_integrity_check to work properly with databases larger that 2GB -- Fixes missing munlock before free for context internal buffer (thanks to Fedor Indutny) +- Adds `PRAGMA cipher_log` and `PRAGMA cipher_log_level` features to allow logging of TRACE, DEBUG, INFO, WARN, and ERROR messages to stdout, stderr, file, or logcat +- Modifies `PRAGMA cipher_profile` to use `sqlite3_trace_v2` and adds logcat target for Android +- Updates OpenSSL provider to use `EVP_MAC` API with version 3+ +- Adds new `PRAGMA cipher_test_on`, `PRAGMA cipher_test_off`, and `PRAGMA cipher_test_rand` (available when compiled with `-DSQLCIPHER_TEST`) to facilitate simulation of error conditions +- Fixes `PRAGMA cipher_integrity_check` to work properly with databases larger that 2GB +- Fixes missing `munlock` before free for context internal buffer (thanks to Fedor Indutny) ## [4.5.0] - (October 2021 - [4.5.0 changes]) - Updates baseline to upstream SQLite 3.36.0 -- Changes the enhanced memory security feature to be DISABLED by default; once enabled by PRAGMA cipher_memory_security = ON, it can't be turned off for the lifetime of the process -- Changes PRAGMA cipher_migrate to permanently enter an error state if a migration fails +- Changes the enhanced memory security feature to be DISABLED by default; once enabled by `PRAGMA cipher_memory_security = ON`, it can't be turned off for the lifetime of the process +- Changes `PRAGMA cipher_migrate` to permanently enter an error state if a migration fails - Fixes memory locking/unlocking issue with realloc implementation on hardened runtimes when memory security is enabled -- Fixes cipher_migrate to cleanup the temporary database if a migration fails +- Fixes `PRAGMA cipher_migrate` to clean up the temporary database if a migration fails - Removes logging of non-string pointers when compiling with trace level logging ## [4.4.3] - (February 2021 - [4.4.3 changes]) -- Updates baseline to ustream SQLite 3.34.1 -- Fixes sqlcipher_export handling of NULL parameters +- Updates baseline to upstream SQLite 3.34.1 +- Fixes `sqlcipher_export` handling of NULL parameters - Removes randomization of rekey-delete tests to avoid false test failures -- Changes internal usage of sqlite_master to sqlite_schema -- Omits unusued profiling function under certain defines to avoid compiler warnings +- Changes internal usage of `sqlite_master` to `sqlite_schema` +- Omits unused profiling function under certain defines to avoid compiler warnings ## [4.4.2] - (November 2020 - [4.4.2 changes]) -- Improve error handling to resolve potential corruption if an encryption operation failed while operating in WAL mode +- Improves error handling to resolve potential corruption if an encryption operation failed while operating in WAL mode - Changes to OpenSSL library cryptographic provider to reduce initialization complexity -- Adjust cipher_integrity_check to skip locking page to avoid a spurious error report for very large databases +- Adjust `cipher_integrity_check` to skip locking page to avoid a spurious error report for very large databases - Miscellaneous code and comment cleanup ## [4.4.1] - (October 2020 - [4.4.1 changes]) - Updates baseline to upstream SQLite 3.33.0 -- Fixes double-free bug in cipher_default_plaintext_header_size +- Fixes double-free bug in `cipher_default_plaintext_header_size` - Changes SQLCipher tests to use suite runner -- Improvement to cipher_integrity_check tests to minimize false negatives -- Deprecates PRAGMA cipher_store_pass +- Improvement to `cipher_integrity_check` tests to minimize false negatives +- Deprecates `PRAGMA cipher_store_pass` ## [4.4.0] - (May 2020 - [4.4.0 changes]) - Updates baseline to upstream SQLite 3.31.0 @@ -153,40 +153,40 @@ __BREAKING CHANGE__: `SELECT` statements (now also including schema independent ## [4.3.0] - (November 2019 - [4.3.0 changes]) - Updates baseline to upstream SQLite 3.30.1 -- PRAGMA key now returns text result value "ok" after execution +- `PRAGMA key` now returns text result value "ok" after execution - Adjusts backup API so that encrypted to encrypted backups are permitted - Adds NSS crypto provider implementation - Fixes OpenSSL provider compatibility with BoringSSL - Separates memory related traces to reduce verbosity of logging -- Fixes output of PRAGMA cipher_integrity_check on big endian platforms -- Cryptograpic provider interface cleanup +- Fixes output of `PRAGMA cipher_integrity_check` on big endian platforms +- Cryptographic provider interface cleanup - Rework of mutex allocation and management - Resolves miscellaneous build warnings - Force error state at database pager level if SQLCipher initialization fails ## [4.2.0] - (May 2019 - [4.2.0 changes]) -- Adds PRAGMA cipher_integrity_check to perform independent verification of page HMACs +- Adds `PRAGMA cipher_integrity_check` to perform independent verification of page HMACs - Updates baseline to upstream SQLite 3.28.0 -- Improves PRAGMA cipher_migrate to handle keys containing non-terminating zero bytes +- Improves `PRAGMA cipher_migrate` to handle keys containing non-terminating zero bytes ## [4.1.0] - (March 2019 - [4.1.0 changes]) - Defer reading salt from header until key derivation is triggered -- Clarify usage of sqlite3_rekey for plaintext databases in header +- Clarify usage of `sqlite3_rekey` for plaintext databases in header - Normalize attach behavior when key is not yet derived -- Adds PRAGMA cipher_settings to query current database codec settings -- Adds PRAGMA cipher_default_settings to query current default SQLCipher options -- PRAGMA cipher_hmac_pgno is now deprecated -- PRAGMA cipher_hmac_salt_mask is now deprecated -- PRAGMA fast_kdf_iter is now deprecated -- Improve sqlcipher_export routine and restore all database flags -- Clear codec data buffers if a crypographic provider operation fails +- Adds `PRAGMA cipher_settings` to query current database codec settings +- Adds `PRAGMA cipher_default_settings` to query current default SQLCipher options +- `PRAGMA cipher_hmac_pgno` is now deprecated +- `PRAGMA cipher_hmac_salt_mask` is now deprecated +- `PRAGMA fast_kdf_iter` is now deprecated +- Improve `sqlcipher_export` routine and restore all database flags +- Clear codec data buffers if a cryptographic provider operation fails - Disable backup API for encrypted databases (this was previously documented as not-working and non-supported, but will now explicitly error out on initialization) - Updates baseline to upstream SQLite 3.27.2 ## [4.0.1] - (December 2018 - [4.0.1 changes]) - Based on upstream SQLite 3.26.0 (addresses SQLite “Magellan” issue) -- Adds PRAGMA cipher_compatibility and cipher_default_compatibility which take automatcially configure appropriate compatibility settings for the specified SQLCipher major version number -- Filters attach statements with KEY parameters from readline history +- Adds `PRAGMA cipher_compatibility` and `PRAGMA cipher_default_compatibility` which automatically configure appropriate compatibility settings for the specified SQLCipher major version number +- Filters attach statements with `KEY` parameters from readline history - Fixes crash in command line shell with empty input (i.e. ^D) - Fixes warnings when compiled with strict-prototypes @@ -196,32 +196,32 @@ __BREAKING CHANGE__: `SELECT` statements (now also including schema independent - Default PBKDF2 iterations increased to 256,000 (up from 64,000) * - Default KDF algorithm is now PBKDF2-HMAC-SHA512 (from PBKDF2-HMAC-SHA1) * - Default HMAC algorithm is now HMAC-SHA512 (from HMAC-SHA1) * -- PRAGMA cipher is now disabled and no longer supported (after multi-year deprecation) * -- PRAGMA rekey_cipher is now disabled and no longer supported * -- PRAGMA rekey_kdf_iter is now disabled and no longer supported * -- By default all memory allocated internally by SQLite before the memory is wiped before it is freed -- PRAGMA cipher_memory_security: allows full memory wiping to be disabled for performance when the feature is not required -- PRAGMA cipher_kdf_algorithm, cipher_default_kdf_algorithm to control KDF algorithm selection between PBKDF2-HMAC-SHA1, PBKDF2-HMAC-SHA256 and PBKDF2-HMAC-SHA512 -- PRAGMA cipher_hmac_algorithm, cipher_default_hmac_algorithm to control HMAC algorithm selection between HMAC-SHA1, HMAC-SHA256 and PBKDF2-HMAC-SHA512 +- `PRAGMA cipher` is now disabled and no longer supported (after multi-year deprecation) * +- `PRAGMA rekey_cipher` is now disabled and no longer supported * +- `PRAGMA rekey_kdf_iter` is now disabled and no longer supported * +- By default all memory allocated internally by SQLite is wiped before it is freed +- `PRAGMA cipher_memory_security`: allows full memory wiping to be disabled for performance when the feature is not required +- `PRAGMA cipher_kdf_algorithm`, `PRAGMA cipher_default_kdf_algorithm` to control KDF algorithm selection between PBKDF2-HMAC-SHA1, PBKDF2-HMAC-SHA256 and PBKDF2-HMAC-SHA512 +- `PRAGMA cipher_hmac_algorithm`, `PRAGMA cipher_default_hmac_algorithm` to control HMAC algorithm selection between HMAC-SHA1, HMAC-SHA256 and PBKDF2-HMAC-SHA512 - Based on upstream SQLite 3.25.2 -- When compiled with readline support, PRAGMA key and rekey lines will no longer be +- When compiled with readline support, `PRAGMA key` and `PRAGMA rekey` lines will no longer be saved to history -- Adds second optional parameter to sqlcipher_export to specify source database to +- Adds second optional parameter to `sqlcipher_export` to specify source database to support bidirectional exports - Fixes compatibility with LibreSSL 2.7.0+ - Fixes compatibility with OpenSSL 1.1.x -- Simplified and improved performance for PRAGMA cipher_migrate when migrating older database versions +- Simplified and improved performance for `PRAGMA cipher_migrate` when migrating older database versions - Refactoring of SQLCipher tests into separate files by test type -- PRAGMA cipher_plaintext_header_size and cipher_default_plaintext_header_size: allocates a portion of the database header which will not be encrypted to allow identification as a SQLite database -- PRAGMA cipher_salt: retrieve or set the salt value for the database +- `PRAGMA cipher_plaintext_header_size` and `PRAGMA cipher_default_plaintext_header_size`: allocates a portion of the database header which will not be encrypted to allow identification as a SQLite database +- `PRAGMA cipher_salt`: retrieve or set the salt value for the database - Adds Podspec for using tagged versions of SQLCipher -- Define SQLCIPHER_PROFILE_USE_FOPEN for WinXP support +- Define `SQLCIPHER_PROFILE_USE_FOPEN` for WinXP support - Improved error handling for cryptographic providers -- Improved memory handling for PRAGMA commands that return values +- Improved memory handling for `PRAGMA` commands that return values - Improved version reporting to assist with identification of distribution - Major rewrite and simplification of internal codec and pager extension -- Fixes compilation with --disable-amalgamation -- Removes sqlcipher.xcodeproj build support +- Fixes compilation with `--disable-amalgamation` +- Removes `sqlcipher.xcodeproj` build support ## [3.4.2] - (December 2017 - [3.4.2 changes]) ### Added @@ -233,7 +233,7 @@ __BREAKING CHANGE__: `SELECT` statements (now also including schema independent - Remove static modifier for codec password functions - Page alignment for `mlock` - Fix segfault in `sqlcipher_cipher_ctx_cmp` during rekey operation -- Fix `sqlcipher_export` and `cipher_migrate` when tracing API in use +- Fix `sqlcipher_export` and `PRAGMA cipher_migrate` when tracing API in use - Validate codec page size when setting - Guard OpenSSL initialization and cleanup routines - Allow additional linker options to be passed via command line for Windows platforms @@ -276,7 +276,7 @@ __BREAKING CHANGE__: `SELECT` statements (now also including schema independent ### Changed - Merged upstream SQLite 3.8.6 -- Renmed README to README.md +- Renamed `README` to `README.md` ## [3.1.0] - (April 2014 - [3.1.0 changes]) ### Added @@ -299,7 +299,7 @@ __BREAKING CHANGE__: `SELECT` statements (now also including schema independent ### Changed - Merged upstream SQLite 3.8.0.2 -- Remove usage of VirtualLock/Unlock on WinRT and Windows Phone +- Remove usage of `VirtualLock/Unlock` on WinRT and Windows Phone - Ignore HMAC read during Btree file copy - Fix lib naming for pkg-config - Use _v2 version of `sqlite3_key` and `sqlite3_rekey` From 2cd5e4570a5e60a4acf736a00a539ed94d9725b4 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Tue, 2 Dec 2025 10:49:30 -0500 Subject: [PATCH 51/75] Adds new PRAGMA cipher_status PRAGMA cipher_status allows an application to verify that a database is properly setup for encyption (i.e. that operations on the database will be using SQLCipher). It returns a scalar result set "1" if the database has been keyed and is not in an error state. Otherwise, it will return a scalar result set of "0". --- src/sqlcipher.c | 7 ++++ test/sqlcipher-core.test | 79 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 68c335b19..ef9c57dab 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -2626,6 +2626,13 @@ int sqlcipher_codec_pragma(sqlite3* db, int iDb, Parse *pParse, const char *zLef sqlcipher_vdbe_return_string(pParse, "cipher_fips_status", fips_mode_status, P4_DYNAMIC); } } else + if( sqlite3_stricmp(zLeft, "cipher_status")== 0 && !zRight ){ + if(ctx && ctx->error == SQLITE_OK) { + sqlcipher_vdbe_return_string(pParse, "cipher_status", "1", P4_TRANSIENT); + } else { + sqlcipher_vdbe_return_string(pParse, "cipher_status", "0", P4_TRANSIENT); + } + } else if( sqlite3_stricmp(zLeft, "cipher_store_pass")==0 && zRight ) { if(ctx) { char *deprecation = "PRAGMA cipher_store_pass is deprecated, please remove from use"; diff --git a/test/sqlcipher-core.test b/test/sqlcipher-core.test index 9c6e54e53..22179feb7 100644 --- a/test/sqlcipher-core.test +++ b/test/sqlcipher-core.test @@ -995,5 +995,84 @@ do_test uri-key-2 { db close file delete -force test.db +# test cipher_status reports encrypted for database +do_test cipher-status-encrypted { + sqlite_orig db test.db + + execsql { + PRAGMA key = 'test'; + PRAGMA cipher_status; + } + +} {ok 1} +db close +file delete -force test.db + +# test cipher_status reports not-encrypted for plaintext database +do_test cipher-status-plaintext { + sqlite_orig db test.db + + execsql { + PRAGMA cipher_status; + } + +} {0} +db close +file delete -force test.db + +# test cipher_status reports encrypted for attached database +# when main database is plaintext and attached database is +# keyed +do_test cipher-status-attached-encrypted { + sqlite_orig db test.db + + execsql { + PRAGMA cipher_status; + ATTACH DATABASE 'test2.db' AS test2 KEY 'test'; + PRAGMA test2.cipher_status; + DETACH DATABASE test2; + } + +} {0 1} +db close +file delete -force test.db +file delete -force test2.db + +# test cipher_status reports plaintext for attached database +# when main database is keyed and attached database is not +do_test cipher-status-attached-plaintext { + sqlite_orig db test.db + + execsql { + PRAGMA KEY = 'test'; + PRAGMA cipher_status; + ATTACH DATABASE 'test2.db' AS test2 KEY ''; + PRAGMA test2.cipher_status; + DETACH DATABASE test2; + } + +} {ok 1 0} +db close +file delete -force test.db +file delete -force test2.db + +# test cipher_status reports unencrypted if key derivation +# and operation fails +setup test.db "'testkey'" +do_test cipher-status-badkey { + sqlite_orig db test.db + + catchsql { + PRAGMA key = 'test'; + SELECT count(*) FROM sqlite_master; + } + execsql { + PRAGMA cipher_status; + } + +} {0} +db close +file delete -force test.db + finish_test From ab223bd801ec225d1497a077da08777d21d1266d Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Wed, 3 Dec 2025 12:14:16 -0500 Subject: [PATCH 52/75] Updates CHANGELOG.md with relevant changes for 4.12.0 --- CHANGELOG.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6b6b0435..068a09b8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,17 @@ # SQLCipher Change Log Notable changes to this project are documented in this file. -## [4.12.0] - (? 2025 - [4.12.0 changes]) +## [4.12.0] - (December 2025 - [4.12.0 changes]) +- Updates baseline to SQLite 3.51.1 +- Adds `PRAGMA cipher_status` so applications can verify a database handle is using encryption +- Improves guards against key/rekey/attach misuse +- Adds criteria for `PRAGMA cipher_migrate` tests +- Fixes check for `__has_feature` macro to separate it from use +- Fixes CHANGELOG.md markdown formatting, typos, and inline code snippets +- Fixes conditional in SQLCipher pragma handling +- Removes deprecated providers for LibTomCrypt and NSS +- Removes unnecessary shutdown and URI config changes in core tests +- Ensures all test suite database handles are closed before delete ## [4.11.0] - (October 2025 - [4.11.0 changes]) - Converts log output to UTF-16 when writing to stdout or stderr on Windows From 62ff45455238141c8a1d2b7c54c55847f03d1ffc Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Mon, 12 Jan 2026 14:43:45 -0500 Subject: [PATCH 53/75] Corrects encoding for sqlcipher_export function registration --- src/sqlcipher.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index ef9c57dab..160abfaef 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -412,7 +412,7 @@ static void (*const sqlcipher_fini_func)(void) __attribute__((used, section(".fi static void sqlcipher_exportFunc(sqlite3_context*, int, sqlite3_value**); static int sqlcipher_export_init(sqlite3* db, const char** errmsg, const struct sqlite3_api_routines* api) { - sqlite3_create_function_v2(db, "sqlcipher_export", -1, SQLITE_TEXT, 0, sqlcipher_exportFunc, 0, 0, 0); + sqlite3_create_function_v2(db, "sqlcipher_export", -1, SQLITE_UTF8, 0, sqlcipher_exportFunc, 0, 0, 0); return SQLITE_OK; } From da8f6fb68a36b9e740cb342206b4efc239f9f123 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 16 Jan 2026 13:10:05 -0500 Subject: [PATCH 54/75] Updates version number to 4.13.0 --- CHANGELOG.md | 4 ++++ src/sqlcipher.c | 2 +- test/sqlcipher-pragmas.test | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 068a09b8e..d64ce7b7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # SQLCipher Change Log Notable changes to this project are documented in this file. +## [4.13.0] - (? - [4.13.0 changes]) + ## [4.12.0] - (December 2025 - [4.12.0 changes]) - Updates baseline to SQLite 3.51.1 - Adds `PRAGMA cipher_status` so applications can verify a database handle is using encryption @@ -318,6 +320,8 @@ Notable changes to this project are documented in this file. ### Security - Change KDF iteration length from 4,000 to 64,000 +[4.13.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.13.0 +[4.13.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.12.0...v4.13.0 [4.12.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.12.0 [4.12.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.11.0...v4.12.0 [4.11.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.11.0 diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 160abfaef..889e0b725 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -95,7 +95,7 @@ void sqlite3pager_reset(Pager *pPager); #define CIPHER_STR(s) #s #ifndef CIPHER_VERSION_NUMBER -#define CIPHER_VERSION_NUMBER 4.12.0 +#define CIPHER_VERSION_NUMBER 4.13.0 #endif #ifndef CIPHER_VERSION_BUILD diff --git a/test/sqlcipher-pragmas.test b/test/sqlcipher-pragmas.test index 88cf71d5a..2ad946bd0 100644 --- a/test/sqlcipher-pragmas.test +++ b/test/sqlcipher-pragmas.test @@ -46,7 +46,7 @@ do_test verify-pragma-cipher-version { execsql { PRAGMA cipher_version; } -} {{4.12.0 community}} +} {{4.13.0 community}} db close file delete -force test.db From 0d21200fcaa6fb66d43bee4929f9bf26d3c94984 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 16 Jan 2026 13:14:33 -0500 Subject: [PATCH 55/75] Snapshot of upstream SQLite 3.51.2 --- Makefile.msc | 134 ++++++++++++++++++++++++++- VERSION | 2 +- autoconf/Makefile.msc | 4 + ext/fts5/fts5Int.h | 8 +- ext/fts5/fts5_index.c | 2 +- ext/fts5/test/fts5merge.test | 16 ++++ ext/misc/compress.c | 4 +- ext/misc/csv.c | 12 ++- ext/misc/fileio.c | 1 + ext/misc/zipfile.c | 2 +- ext/rtree/rtree.c | 2 +- ext/rtree/rtreeB.test | 10 ++ ext/wasm/api/sqlite3-api-oo1.c-pp.js | 2 +- manifest | 56 +++++------ manifest.tags | 2 +- manifest.uuid | 2 +- src/expr.c | 17 +++- src/os_unix.c | 39 ++++---- src/select.c | 21 ++++- src/shell.c.in | 3 +- src/where.c | 53 ++++++----- test/existsexpr.test | 39 +++++++- test/fuzzcheck.c | 1 + test/subquery2.test | 68 ++++++++++++++ test/zipfile.test | 7 ++ 25 files changed, 415 insertions(+), 92 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 3c075fac3..52807ff7f 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -805,17 +805,21 @@ BCC = $(BCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE !IF $(DEBUG)>1 TCC = $(TCC) -MDd BCC = $(BCC) -MDd +ZLIBCFLAGS = -nologo -MDd -W3 -O2 -Oy- -Zi !ELSE TCC = $(TCC) -MD BCC = $(BCC) -MD +ZLIBCFLAGS = -nologo -MD -W3 -O2 -Oy- -Zi !ENDIF !ELSE !IF $(DEBUG)>1 TCC = $(TCC) -MTd BCC = $(BCC) -MTd +ZLIBCFLAGS = -nologo -MTd -W3 -O2 -Oy- -Zi !ELSE TCC = $(TCC) -MT BCC = $(BCC) -MT +ZLIBCFLAGS = -nologo -MT -W3 -O2 -Oy- -Zi !ENDIF !ENDIF @@ -2416,7 +2420,7 @@ shell.c: $(SHELL_DEP) $(TOP)\tool\mkshellc.tcl $(JIM_TCLSH) $(JIM_TCLSH) $(TOP)\tool\mkshellc.tcl shell.c zlib: - pushd $(ZLIBDIR) && $(MAKE) /f win32\Makefile.msc clean $(ZLIBLIB) && popd + pushd $(ZLIBDIR) && $(MAKE) /f win32\Makefile.msc clean $(ZLIBLIB) CFLAGS="$(ZLIBCFLAGS)" && popd # Rules to build the extension objects. # @@ -2811,6 +2815,134 @@ tcl-env: @echo JIM_TCLSH = $(JIM_TCLSH) @echo VISUALSTUDIOVERSION = $(VISUALSTUDIOVERSION) +env: + @echo ALL_TCL_TARGETS = $(ALL_TCL_TARGETS) + @echo API_ARMOR = $(API_ARMOR) + @echo ASAN = $(ASAN) + @echo BCC = $(BCC) + @echo BUILD_ZLIB = $(BUILD_ZLIB) + @echo CC = $(CC) + @echo CCOPTS = $(CCOPTS) + @echo CHECKER_DEPS = $(CHECKER_DEPS) + @echo CORE_CCONV_OPTS = $(CORE_CCONV_OPTS) + @echo CORE_COMPILE_OPTS = $(CORE_COMPILE_OPTS) + @echo CORE_LINK_DEP = $(CORE_LINK_DEP) + @echo CORE_LINK_OPTS = $(CORE_LINK_OPTS) + @echo CRTLIBPATH = $(CRTLIBPATH) + @echo CSC = $(CSC) + @echo DBFUZZ_COMPILE_OPTS = $(DBFUZZ_COMPILE_OPTS) + @echo DEBUG = $(DEBUG) + @echo DYNAMIC_SHELL = $(DYNAMIC_SHELL) + @echo EXT_FEATURE_FLAGS = $(EXT_FEATURE_FLAGS) + @echo EXTHDR = $(EXTHDR) + @echo EXTRA_SRC = $(EXTRA_SRC) + @echo FOR_UWP = $(FOR_UWP) + @echo FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) + @echo FUZZCHECK_SRC = $(FUZZCHECK_SRC) + @echo FUZZDATA = $(FUZZDATA) + @echo FUZZERSHELL_COMPILE_OPTS = $(FUZZERSHELL_COMPILE_OPTS) + @echo HDR = $(HDR) + @echo ICUDIR = $(ICUDIR) + @echo ICUINCDIR = $(ICUINCDIR) + @echo ICULIBDIR = $(ICULIBDIR) + @echo JIM_TCLSH = $(JIM_TCLSH) + @echo KV_COMPILE_OPTS = $(KV_COMPILE_OPTS) + @echo LDFLAGS = $(LDFLAGS) + @echo LD = $(LD) + @echo LIBICU = $(LIBICU) + @echo LIBOBJ = $(LIBOBJ) + @echo LIBREADLINE = $(LIBREADLINE) + @echo LIBRESOBJS = $(LIBRESOBJS) + @echo LIBTCLPATH = $(LIBTCLPATH) + @echo LIBTCLSTUB = $(LIBTCLSTUB) + @echo LIBTCL = $(LIBTCL) + @echo LTCOMPILE = $(LTCOMPILE) + @echo LTLIB = $(LTLIB) + @echo LTLIBOPTS = $(LTLIBOPTS) + @echo LTLIBPATHS = $(LTLIBPATHS) + @echo LTLIBS = $(LTLIBS) + @echo LTLINK = $(LTLINK) + @echo LTLINKOPTS = $(LTLINKOPTS) + @echo LTRCOMPILE = $(LTRCOMPILE) + @echo MEMDEBUG = $(MEMDEBUG) + @echo MINIMAL_AMALGAMATION = $(MINIMAL_AMALGAMATION) + @echo MPTESTER_COMPILE_OPTS = $(MPTESTER_COMPILE_OPTS) + @echo NCC = $(NCC) + @echo NCRTLIBPATH = $(NCRTLIBPATH) + @echo NLTLIBPATHS = $(NLTLIBPATHS) + @echo NO_LINEMACROS = $(NO_LINEMACROS) + @echo NO_TCL = $(NO_TCL) + @echo NO_WARN = $(NO_WARN) + @echo NSDKLIBPATH = $(NSDKLIBPATH) + @echo NUCRTLIBPATH = $(NUCRTLIBPATH) + @echo OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) + @echo OPTIMIZATIONS = $(OPTIMIZATIONS) + @echo OSSSHELL_SRC = $(OSSSHELL_SRC) + @echo OSTRACE = $(OSTRACE) + @echo RBU = $(RBU) + @echo RCC = $(RCC) + @echo RC = $(RC) + @echo READLINE_FLAGS = $(READLINE_FLAGS) + @echo REQ_FEATURE_FLAGS = $(REQ_FEATURE_FLAGS) + @echo RSYNC_OPT = $(RSYNC_OPT) + @echo RSYNC_SRC = $(RSYNC_SRC) + @echo SESSION = $(SESSION) + @echo SETLK_TIMEOUT = $(SETLK_TIMEOUT) + @echo SHELL_CCONV_OPTS = $(SHELL_CCONV_OPTS) + @echo SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) + @echo SHELL_CORE_DEP = $(SHELL_CORE_DEP) + @echo SHELL_CORE_LIB = $(SHELL_CORE_LIB) + @echo SHELL_CORE_SRC = $(SHELL_CORE_SRC) + @echo SHELL_DEP = $(SHELL_DEP) + @echo SHELL_LINK_OPTS = $(SHELL_LINK_OPTS) + @echo SPLIT_AMALGAMATION = $(SPLIT_AMALGAMATION) + @echo SQLITETCLDECLSH = $(SQLITETCLDECLSH) + @echo SQLITE_TCL_DEP = $(SQLITE_TCL_DEP) + @echo SQLITETCLH = $(SQLITETCLH) + @echo SRC = $(SRC) + @echo STATICALLY_LINK_TCL = $(STATICALLY_LINK_TCL) + @echo ST_COMPILE_OPTS = $(ST_COMPILE_OPTS) + @echo STORELIBPATH = $(STORELIBPATH) + @echo SYMBOLS = $(SYMBOLS) + @echo TCC = $(TCC) + @echo TCLDIR = $(TCLDIR) + @echo TCLINCDIR = $(TCLINCDIR) + @echo TCLLIBDIR = $(TCLLIBDIR) + @echo TCLLIBPATHS = $(TCLLIBPATHS) + @echo TCLLIBS = $(TCLLIBS) + @echo TCLSH_CMD = $(TCLSH_CMD) + @echo TCLSQLITEEX = $(TCLSQLITEEX) + @echo TCLSUFFIX = $(TCLSUFFIX) + @echo TCLVERSION = $(TCLVERSION) + @echo TEST_CCONV_OPTS = $(TEST_CCONV_OPTS) + @echo TESTEXT = $(TESTEXT) + @echo TESTFIXTURE_DEP = $(TESTFIXTURE_DEP) + @echo TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) + @echo TESTFIXTURE_SRC = $(TESTFIXTURE_SRC) + @echo TESTOPTS = $(TESTOPTS) + @echo TESTPROGS = $(TESTPROGS) + @echo TESTSRC = $(TESTSRC) + @echo TLIBS = $(TLIBS) + @echo TOP = $(TOP) + @echo UCRTLIBPATH = $(UCRTLIBPATH) + @echo USE_AMALGAMATION = $(USE_AMALGAMATION) + @echo USE_CRT_DLL = $(USE_CRT_DLL) + @echo USE_FATAL_WARN = $(USE_FATAL_WARN) + @echo USE_FULLWARN = $(USE_FULLWARN) + @echo USE_ICU = $(USE_ICU) + @echo USE_LISTINGS = $(USE_LISTINGS) + @echo USE_NATIVE_LIBPATHS = $(USE_NATIVE_LIBPATHS) + @echo USE_RC = $(USE_RC) + @echo USE_RUNTIME_CHECKS = $(USE_RUNTIME_CHECKS) + @echo USE_SEH = $(USE_SEH) + @echo USE_STDCALL = $(USE_STDCALL) + @echo USE_ZLIB = $(USE_ZLIB) + @echo XCOMPILE = $(XCOMPILE) + @echo ZLIBDIR = $(ZLIBDIR) + @echo ZLIBINCDIR = $(ZLIBINCDIR) + @echo ZLIBLIBDIR = $(ZLIBLIBDIR) + @echo ZLIBLIB = $(ZLIBLIB) + moreclean: clean del /Q $(SQLITE3C) $(SQLITE3H) 2>NUL # <> diff --git a/VERSION b/VERSION index d1278a467..0306a564d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.51.1 +3.51.2 diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index 4f96e3b18..365e1f0fb 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -695,17 +695,21 @@ BCC = $(BCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE !IF $(DEBUG)>1 TCC = $(TCC) -MDd BCC = $(BCC) -MDd +ZLIBCFLAGS = -nologo -MDd -W3 -O2 -Oy- -Zi !ELSE TCC = $(TCC) -MD BCC = $(BCC) -MD +ZLIBCFLAGS = -nologo -MD -W3 -O2 -Oy- -Zi !ENDIF !ELSE !IF $(DEBUG)>1 TCC = $(TCC) -MTd BCC = $(BCC) -MTd +ZLIBCFLAGS = -nologo -MTd -W3 -O2 -Oy- -Zi !ELSE TCC = $(TCC) -MT BCC = $(BCC) -MT +ZLIBCFLAGS = -nologo -MT -W3 -O2 -Oy- -Zi !ENDIF !ENDIF diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index a13a65d3c..d5404535c 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -81,7 +81,13 @@ typedef sqlite3_uint64 u64; # define FLEXARRAY 1 #endif -#endif +#endif /* SQLITE_AMALGAMATION */ + +/* +** Constants for the largest and smallest possible 32-bit signed integers. +*/ +# define LARGEST_INT32 ((int)(0x7fffffff)) +# define SMALLEST_INT32 ((int)((-1) - LARGEST_INT32)) /* Truncate very long tokens to this many bytes. Hard limit is ** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 7e25731ed..acd0570a5 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -5931,7 +5931,7 @@ int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ fts5StructureRelease(pStruct); pStruct = pNew; nMin = 1; - nMerge = nMerge*-1; + nMerge = (nMerge==SMALLEST_INT32 ? LARGEST_INT32 : (nMerge*-1)); } if( pStruct && pStruct->nLevel ){ if( fts5IndexMerge(p, &pStruct, nMerge, nMin) ){ diff --git a/ext/fts5/test/fts5merge.test b/ext/fts5/test/fts5merge.test index c57c21ded..09c18245f 100644 --- a/ext/fts5/test/fts5merge.test +++ b/ext/fts5/test/fts5merge.test @@ -238,6 +238,22 @@ do_execsql_test 6.3 { INSERT INTO g1(g1) VALUES('integrity-check'); } +#-------------------------------------------------------------------------- +# Check that passing -2147483648 as the parameter to a merge command +# does not cause a signed integer overflow error. +# +reset_db +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE f1 USING fts5(a); +} +do_execsql_test 7.1 { + INSERT INTO f1 VALUES('one two three'); + INSERT INTO f1 VALUES('four five six'); + INSERT INTO f1 VALUES('seven eight nine'); +} +do_execsql_test 7.2 { + INSERT INTO f1(f1, rank) VALUES('merge', -2147483648); +} finish_test diff --git a/ext/misc/compress.c b/ext/misc/compress.c index 6b034eb45..48ea5182d 100644 --- a/ext/misc/compress.c +++ b/ext/misc/compress.c @@ -59,7 +59,7 @@ static void compressFunc( pIn = sqlite3_value_blob(argv[0]); nIn = sqlite3_value_bytes(argv[0]); nOut = 13 + nIn + (nIn+999)/1000; - pOut = sqlite3_malloc( nOut+5 ); + pOut = sqlite3_malloc64( nOut+5 ); for(i=4; i>=0; i--){ x[i] = (nIn >> (7*(4-i)))&0x7f; } @@ -98,7 +98,7 @@ static void uncompressFunc( nOut = (nOut<<7) | (pIn[i]&0x7f); if( (pIn[i]&0x80)!=0 ){ i++; break; } } - pOut = sqlite3_malloc( nOut+1 ); + pOut = sqlite3_malloc64( nOut+1 ); rc = uncompress(pOut, &nOut, &pIn[i], nIn-i); if( rc==Z_OK ){ sqlite3_result_blob(context, pOut, nOut, sqlite3_free); diff --git a/ext/misc/csv.c b/ext/misc/csv.c index 8331265aa..1caaaec87 100644 --- a/ext/misc/csv.c +++ b/ext/misc/csv.c @@ -62,6 +62,9 @@ SQLITE_EXTENSION_INIT1 # define CSV_NOINLINE #endif +#ifndef SQLITEINT_H +typedef sqlite3_int64 i64; +#endif /* Max size of the error message in a CsvReader */ #define CSV_MXERR 200 @@ -74,9 +77,9 @@ typedef struct CsvReader CsvReader; struct CsvReader { FILE *in; /* Read the CSV text from this input stream */ char *z; /* Accumulated text for a field */ - int n; /* Number of bytes in z */ - int nAlloc; /* Space allocated for z[] */ - int nLine; /* Current line number */ + i64 n; /* Number of bytes in z */ + i64 nAlloc; /* Space allocated for z[] */ + i64 nLine; /* Current line number */ int bNotFirst; /* True if prior text has been seen */ int cTerm; /* Character that terminated the most recent field */ size_t iIn; /* Next unread character in the input buffer */ @@ -174,7 +177,7 @@ static int csv_getc(CsvReader *p){ ** Return 0 on success and non-zero if there is an OOM error */ static CSV_NOINLINE int csv_resize_and_append(CsvReader *p, char c){ char *zNew; - int nNew = p->nAlloc*2 + 100; + i64 nNew = p->nAlloc*2 + 100; zNew = sqlite3_realloc64(p->z, nNew); if( zNew ){ p->z = zNew; @@ -510,7 +513,6 @@ static int csvtabConnect( # define CSV_DATA (azPValue[1]) # define CSV_SCHEMA (azPValue[2]) - assert( sizeof(azPValue)==sizeof(azParam) ); memset(&sRdr, 0, sizeof(sRdr)); memset(azPValue, 0, sizeof(azPValue)); diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c index d78b14877..6cc2ae008 100644 --- a/ext/misc/fileio.c +++ b/ext/misc/fileio.c @@ -329,6 +329,7 @@ static int fileStat( b1[sz] = 0; rc = _wstat(b1, pStatBuf); if( rc==0 ) statTimesToUtc(zPath, pStatBuf); + sqlite3_free(b1); return rc; #else return stat(zPath, pStatBuf); diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index 58cfba658..2f74906d9 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -875,7 +875,7 @@ static int zipfileGetEntry( ); }else{ aRead = (u8*)&aBlob[iOff + ZIPFILE_CDS_FIXED_SZ]; - if( (iOff + ZIPFILE_LFH_FIXED_SZ + nFile + nExtra)>nBlob ){ + if( (iOff + ZIPFILE_CDS_FIXED_SZ + nFile + nExtra)>nBlob ){ rc = zipfileCorrupt(pzErr); } } diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 8b913ef2d..b3d29283e 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -3775,7 +3775,7 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ if( node.zData==0 ) return; nData = sqlite3_value_bytes(apArg[1]); if( nData<4 ) return; - if( nDatanMem += nReg; if( pExpr->op==TK_SELECT ){ dest.eDest = SRT_Mem; - dest.iSdst = dest.iSDParm; + if( (pSel->selFlags&SF_Distinct) && pSel->pLimit && pSel->pLimit->pRight ){ + /* If there is both a DISTINCT and an OFFSET clause, then allocate + ** a separate dest.iSdst array for sqlite3Select() and other + ** routines to populate. In this case results will be copied over + ** into the dest.iSDParm array only after OFFSET processing. This + ** ensures that in the case where OFFSET excludes all rows, the + ** dest.iSDParm array is not left populated with the contents of the + ** last row visited - it should be all NULLs if all rows were + ** excluded by OFFSET. */ + dest.iSdst = pParse->nMem+1; + pParse->nMem += nReg; + }else{ + dest.iSdst = dest.iSDParm; + } dest.nSdst = nReg; - sqlite3VdbeAddOp3(v, OP_Null, 0, dest.iSDParm, dest.iSDParm+nReg-1); + sqlite3VdbeAddOp3(v, OP_Null, 0, dest.iSDParm, pParse->nMem); VdbeComment((v, "Init subquery result")); }else{ dest.eDest = SRT_Exists; diff --git a/src/os_unix.c b/src/os_unix.c index 6b679c4dc..d73d89924 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -2027,12 +2027,18 @@ static int unixLock(sqlite3_file *id, int eFileLock){ pInode->nLock++; pInode->nShared = 1; } - }else if( (eFileLock==EXCLUSIVE_LOCK && pInode->nShared>1) - || unixIsSharingShmNode(pFile) - ){ + }else if( eFileLock==EXCLUSIVE_LOCK && pInode->nShared>1 ){ /* We are trying for an exclusive lock but another thread in this ** same process is still holding a shared lock. */ rc = SQLITE_BUSY; + }else if( unixIsSharingShmNode(pFile) ){ + /* We are in WAL mode and attempting to delete the SHM and WAL + ** files due to closing the connection or changing out of WAL mode, + ** but another process still holds locks on the SHM file, thus + ** indicating that database locks have been broken, perhaps due + ** to a rogue close(open(dbFile)) or similar. + */ + rc = SQLITE_BUSY; }else{ /* The request was for a RESERVED or EXCLUSIVE lock. It is ** assumed that there is a SHARED or greater lock on the file @@ -4671,26 +4677,21 @@ static int unixFcntlExternalReader(unixFile *pFile, int *piOut){ ** still not a disaster. */ static int unixIsSharingShmNode(unixFile *pFile){ - int rc; unixShmNode *pShmNode; + struct flock lock; if( pFile->pShm==0 ) return 0; if( pFile->ctrlFlags & UNIXFILE_EXCL ) return 0; pShmNode = pFile->pShm->pShmNode; - rc = 1; - unixEnterMutex(); - if( ALWAYS(pShmNode->nRef==1) ){ - struct flock lock; - lock.l_whence = SEEK_SET; - lock.l_start = UNIX_SHM_DMS; - lock.l_len = 1; - lock.l_type = F_WRLCK; - osFcntl(pShmNode->hShm, F_GETLK, &lock); - if( lock.l_type==F_UNLCK ){ - rc = 0; - } - } - unixLeaveMutex(); - return rc; +#if SQLITE_ATOMIC_INTRINSICS + assert( AtomicLoad(&pShmNode->nRef)==1 ); +#endif + memset(&lock, 0, sizeof(lock)); + lock.l_whence = SEEK_SET; + lock.l_start = UNIX_SHM_DMS; + lock.l_len = 1; + lock.l_type = F_WRLCK; + osFcntl(pShmNode->hShm, F_GETLK, &lock); + return (lock.l_type!=F_UNLCK); } /* diff --git a/src/select.c b/src/select.c index bec00ecb9..bc17ecf84 100644 --- a/src/select.c +++ b/src/select.c @@ -1445,9 +1445,14 @@ static void selectInnerLoop( assert( nResultCol<=pDest->nSdst ); pushOntoSorter( pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); + pDest->iSDParm = regResult; }else{ assert( nResultCol==pDest->nSdst ); - assert( regResult==iParm ); + if( regResult!=iParm ){ + /* This occurs in cases where the SELECT had both a DISTINCT and + ** an OFFSET clause. */ + sqlite3VdbeAddOp3(v, OP_Copy, regResult, iParm, nResultCol-1); + } /* The LIMIT clause will jump out of the loop for us */ } break; @@ -7462,12 +7467,24 @@ static SQLITE_NOINLINE void existsToJoin( && (pSub->selFlags & SF_Aggregate)==0 && !pSub->pSrc->a[0].fg.isSubquery && pSub->pLimit==0 + && pSub->pPrior==0 ){ + /* Before combining the sub-select with the parent, renumber the + ** cursor used by the subselect. This is because the EXISTS expression + ** might be a copy of another EXISTS expression from somewhere + ** else in the tree, and in this case it is important that it use + ** a unique cursor number. */ + sqlite3 *db = pParse->db; + int *aCsrMap = sqlite3DbMallocZero(db, (pParse->nTab+2)*sizeof(int)); + if( aCsrMap==0 ) return; + aCsrMap[0] = (pParse->nTab+1); + renumberCursors(pParse, pSub, -1, aCsrMap); + sqlite3DbFree(db, aCsrMap); + memset(pWhere, 0, sizeof(*pWhere)); pWhere->op = TK_INTEGER; pWhere->u.iValue = 1; ExprSetProperty(pWhere, EP_IntValue); - assert( p->pWhere!=0 ); pSub->pSrc->a[0].fg.fromExists = 1; pSub->pSrc->a[0].fg.jointype |= JT_CROSS; diff --git a/src/shell.c.in b/src/shell.c.in index 23aed0de5..bd4483ff7 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -13807,7 +13807,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( nCmd>0 ){ sqlite3_fprintf(stderr,"Error: cannot mix regular SQL or dot-commands" " with \"%s\"\n", z); - return 1; + rc = 1; + goto shell_main_exit; } open_db(&data, OPEN_DB_ZIPFILE); if( z[2] ){ diff --git a/src/where.c b/src/where.c index 6313b87ca..c4f2c5543 100644 --- a/src/where.c +++ b/src/where.c @@ -7380,6 +7380,9 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3 *db = pParse->db; int iEnd = sqlite3VdbeCurrentAddr(v); int nRJ = 0; +#ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT + int addrSeek = 0; +#endif /* Generate loop termination code. */ @@ -7392,7 +7395,10 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ ** the RIGHT JOIN table */ WhereRightJoin *pRJ = pLevel->pRJ; sqlite3VdbeResolveLabel(v, pLevel->addrCont); - pLevel->addrCont = 0; + /* Replace addrCont with a new label that will never be used, just so + ** the subsequent call to resolve pLevel->addrCont will have something + ** to resolve. */ + pLevel->addrCont = sqlite3VdbeMakeLabel(pParse); pRJ->endSubrtn = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_Return, pRJ->regReturn, pRJ->addrSubrtn, 1); VdbeCoverage(v); @@ -7401,7 +7407,6 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ pLoop = pLevel->pWLoop; if( pLevel->op!=OP_Noop ){ #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT - int addrSeek = 0; Index *pIdx; int n; if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED @@ -7424,25 +7429,26 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2); } #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */ - if( pTabList->a[pLevel->iFrom].fg.fromExists && i==pWInfo->nLevel-1 ){ - /* If the EXISTS-to-JOIN optimization was applied, then the EXISTS - ** loop(s) will be the inner-most loops of the join. There might be - ** multiple EXISTS loops, but they will all be nested, and the join - ** order will not have been changed by the query planner. If the - ** inner-most EXISTS loop sees a single successful row, it should - ** break out of *all* EXISTS loops. But only the inner-most of the - ** nested EXISTS loops should do this breakout. */ - int nOuter = 0; /* Nr of outer EXISTS that this one is nested within */ - while( nOutera[pLevel[-nOuter-1].iFrom].fg.fromExists ) break; - nOuter++; - } - testcase( nOuter>0 ); - sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel[-nOuter].addrBrk); - VdbeComment((v, "EXISTS break")); + } + if( pTabList->a[pLevel->iFrom].fg.fromExists && i==pWInfo->nLevel-1 ){ + /* If the EXISTS-to-JOIN optimization was applied, then the EXISTS + ** loop(s) will be the inner-most loops of the join. There might be + ** multiple EXISTS loops, but they will all be nested, and the join + ** order will not have been changed by the query planner. If the + ** inner-most EXISTS loop sees a single successful row, it should + ** break out of *all* EXISTS loops. But only the inner-most of the + ** nested EXISTS loops should do this breakout. */ + int nOuter = 0; /* Nr of outer EXISTS that this one is nested within */ + while( nOutera[pLevel[-nOuter-1].iFrom].fg.fromExists ) break; + nOuter++; } - /* The common case: Advance to the next row */ - if( pLevel->addrCont ) sqlite3VdbeResolveLabel(v, pLevel->addrCont); + testcase( nOuter>0 ); + sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel[-nOuter].addrBrk); + VdbeComment((v, "EXISTS break")); + } + sqlite3VdbeResolveLabel(v, pLevel->addrCont); + if( pLevel->op!=OP_Noop ){ sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3); sqlite3VdbeChangeP5(v, pLevel->p5); VdbeCoverage(v); @@ -7455,10 +7461,11 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ VdbeCoverage(v); } #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT - if( addrSeek ) sqlite3VdbeJumpHere(v, addrSeek); + if( addrSeek ){ + sqlite3VdbeJumpHere(v, addrSeek); + addrSeek = 0; + } #endif - }else if( pLevel->addrCont ){ - sqlite3VdbeResolveLabel(v, pLevel->addrCont); } if( (pLoop->wsFlags & WHERE_IN_ABLE)!=0 && pLevel->u.in.nIn>0 ){ struct InLoop *pIn; diff --git a/test/existsexpr.test b/test/existsexpr.test index 38392b07f..28029359b 100644 --- a/test/existsexpr.test +++ b/test/existsexpr.test @@ -15,7 +15,6 @@ source $testdir/tester.tcl source $testdir/lock_common.tcl set testprefix existsexpr - do_execsql_test 1.0 { CREATE TABLE x1(a, b, PRIMARY KEY(a)) WITHOUT ROWID; INSERT INTO x1 VALUES(1, 2), (3, 4), (5, 6); @@ -477,4 +476,42 @@ catch { optimization_control db exists-to-join 0 } db cache flush do_execsql_test 9.6 $Q {{big string value}} +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 10.0 { + CREATE TABLE t1(a); + CREATE TABLE x1(x); +} + +do_execsql_test 10.1 { + SELECT EXISTS( SELECT 1 FROM t1 ) aaa FROM x1 WHERE aaa AND aaa +} + +do_execsql_test 10.2 { + SELECT + EXISTS( SELECT 1 FROM t1 ) aaa + WHERE ( + SELECT 1 FROM x1 WHERE aaa AND aaa + ); +} + +# https://sqlite.org/forum/forumpost/2026-01-03T14:05:48z +do_execsql_test 11.0 { + CREATE TABLE parent (id TEXT PRIMARY KEY); + CREATE TABLE child_a (id TEXT); + CREATE TABLE child_b (id TEXT); + INSERT INTO parent VALUES ('p1'); + INSERT INTO child_a VALUES ('p1'); +} +do_execsql_test 11.1 { + SELECT count(*), parent.id FROM parent + WHERE EXISTS ( + SELECT 1 FROM child_a WHERE child_a.id = parent.id + UNION + SELECT 1 FROM child_b WHERE child_b.id = parent.id + ) + GROUP BY id; +} {1 p1} + finish_test diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index d8dbf932d..a3377770a 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -1977,6 +1977,7 @@ int main(int argc, char **argv){ int iSliceIdx = 0; /* Only run the piece with this index */ sqlite3_config(SQLITE_CONFIG_URI,1); + sqlite3_config(SQLITE_CONFIG_MEMSTATUS,1); registerOomSimulator(); sqlite3_initialize(); iBegin = timeOfDay(); diff --git a/test/subquery2.test b/test/subquery2.test index 8513dc75c..99a1e903f 100644 --- a/test/subquery2.test +++ b/test/subquery2.test @@ -215,4 +215,72 @@ do_execsql_test 5.1 { SELECT ( SELECT y FROM t2 WHERE x = y ORDER BY y, z) FROM t1; } {ALFKI ANATR} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(1234); +} + +do_execsql_test 6.1 { + SELECT DISTINCT 'string' FROM t1 LIMIT 1 OFFSET 5; +} + +do_execsql_test 6.2 { + SELECT ( + SELECT 'string' FROM t1 LIMIT 1 OFFSET 5 + ); +} {{}} + +do_execsql_test 6.3 { + SELECT ( + SELECT DISTINCT 'string' FROM t1 LIMIT 1 OFFSET 5 + ); +} {{}} + +do_execsql_test 6.4 { + SELECT ( + SELECT DISTINCT 'string' FROM t1 ORDER BY 1 LIMIT 1 OFFSET 5 + ); +} {{}} + +do_execsql_test 6.5 { + SELECT ( + SELECT 'string' FROM t1 ORDER BY 1 LIMIT 1 OFFSET 5 + ); +} {{}} + +do_execsql_test 6.6 { + SELECT (SELECT DISTINCT x, x FROM t1 LIMIT 1 OFFSET 5)==(1234, 1234) +} {{}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 7.0 { + CREATE TABLE t1(x); + CREATE INDEX i1 ON t1(x); + INSERT INTO t1 VALUES(1234); +} + +do_execsql_test 7.1 { + SELECT ( + SELECT DISTINCT 'string' FROM t1 ORDER BY x LIMIT 1 OFFSET 5 + ); +} {{}} + +do_execsql_test 7.2 { + DROP INDEX i1; + CREATE UNIQUE INDEX i1 ON t1(x); +} + +do_execsql_test 7.3 { + SELECT ( + SELECT DISTINCT x FROM t1 ORDER BY 1 LIMIT 1 OFFSET 5 + ); +} {{}} + +do_execsql_test 7.4 { + SELECT (SELECT DISTINCT x FROM t1 ORDER BY +x LIMIT 1 OFFSET 0); +} {1234} + finish_test diff --git a/test/zipfile.test b/test/zipfile.test index 2b3be6278..9bb35ea5d 100644 --- a/test/zipfile.test +++ b/test/zipfile.test @@ -885,7 +885,10 @@ do_test 19.1 { INSERT INTO v0 DEFAULT VALUES; } } {} +db close forcedelete zipfile19.zip +sqlite3 db :memory: +load_static_extension db zipfile #------------------------------------------------------------------------- do_catchsql_test 20.0 { @@ -901,4 +904,8 @@ d42728f602000000020000000500ffff0000000000000000a4810000000068 00000000',char(0x0a,0x0d))); } {1 {zip archive is corrupt}} +# https://sqlite.org/forum/forumpost/2025-12-30T23:57:19z +do_catchsql_test 20.2 { + SELECT * FROM zipfile(unhex('504b0304140000000000000000008b9ed9d30100000001000000010000007841504b01021e03140000000000000000008b9ed9d3010000000100000001001e000000000000000000a4810000000078504b050600000000010001002f000000200000000000')); +} {1 {zip archive is corrupt}} finish_test From 222bdcafad462a1080360de1928cd900a8bccd0a Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Mon, 19 Jan 2026 12:01:34 -0500 Subject: [PATCH 56/75] Updates CHANGELOG.md with relevant changes for 4.13.0 --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d64ce7b7d..015ab3404 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,9 @@ # SQLCipher Change Log Notable changes to this project are documented in this file. -## [4.13.0] - (? - [4.13.0 changes]) +## [4.13.0] - (January 2026 - [4.13.0 changes]) +- Updates baseline to SQLite 3.51.2 +- Corrects encoding for `sqlcipher_export()` function registration ## [4.12.0] - (December 2025 - [4.12.0 changes]) - Updates baseline to SQLite 3.51.1 From 693abfa5f5ee5b60f3645d900053b8d55e491ef6 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Tue, 20 Jan 2026 15:27:19 -0500 Subject: [PATCH 57/75] Updates version number to 4.14.0 --- CHANGELOG.md | 4 ++++ src/sqlcipher.c | 2 +- test/sqlcipher-pragmas.test | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 015ab3404..446d65cae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # SQLCipher Change Log Notable changes to this project are documented in this file. +## [4.14.0] - (? 2026 - [4.14.0 changes]) + ## [4.13.0] - (January 2026 - [4.13.0 changes]) - Updates baseline to SQLite 3.51.2 - Corrects encoding for `sqlcipher_export()` function registration @@ -322,6 +324,8 @@ Notable changes to this project are documented in this file. ### Security - Change KDF iteration length from 4,000 to 64,000 +[4.14.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.14.0 +[4.14.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.13.0...v4.14.0 [4.13.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.13.0 [4.13.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.12.0...v4.13.0 [4.12.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.12.0 diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 889e0b725..bf07153f8 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -95,7 +95,7 @@ void sqlite3pager_reset(Pager *pPager); #define CIPHER_STR(s) #s #ifndef CIPHER_VERSION_NUMBER -#define CIPHER_VERSION_NUMBER 4.13.0 +#define CIPHER_VERSION_NUMBER 4.14.0 #endif #ifndef CIPHER_VERSION_BUILD diff --git a/test/sqlcipher-pragmas.test b/test/sqlcipher-pragmas.test index 2ad946bd0..74c758baf 100644 --- a/test/sqlcipher-pragmas.test +++ b/test/sqlcipher-pragmas.test @@ -46,7 +46,7 @@ do_test verify-pragma-cipher-version { execsql { PRAGMA cipher_version; } -} {{4.13.0 community}} +} {{4.14.0 community}} db close file delete -force test.db From d9175b66da236ad8a0c65e29d7a2b7314baaf0ea Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Sun, 15 Feb 2026 10:39:17 -0500 Subject: [PATCH 58/75] Partial revert of 3607cf68 to restore LibTomCrypt provider --- Makefile.msc | 1 + main.mk | 4 + src/crypto_libtomcrypt.c | 316 +++++++++++++++++++++++++++++++++++++++ src/sqlcipher.c | 4 + tool/mksqlite3c.tcl | 1 + 5 files changed, 326 insertions(+) create mode 100644 src/crypto_libtomcrypt.c diff --git a/Makefile.msc b/Makefile.msc index d3fe57de6..3ef50a338 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1403,6 +1403,7 @@ LIBRESOBJS = SRC00 = \ $(TOP)\src\sqlcipher.c \ $(TOP)\src\crypto_cc.c \ + $(TOP)\src\crypto_libtomcrypt.c \ $(TOP)\src\crypto_openssl.c \ $(TOP)\src\sqlcipher.h \ $(TOP)\src\alter.c \ diff --git a/main.mk b/main.mk index c6bdeafaa..2f02f87fd 100644 --- a/main.mk +++ b/main.mk @@ -534,11 +534,13 @@ clean: clean-sanity-check SQLCIPHER_OBJ = \ sqlcipher.o \ crypto_openssl.o \ + crypto_libtomcrypt.o \ crypto_cc.o SQLCIPHER_SRC = \ $(TOP)/src/sqlcipher.h \ $(TOP)/src/sqlcipher.c \ + $(TOP)/src/crypto_libtomcrypt.c \ $(TOP)/src/crypto_openssl.c \ $(TOP)/src/crypto_cc.c @@ -546,6 +548,8 @@ sqlcipher.o: $(TOP)/src/sqlcipher.c $(DEPS_OBJ_COMMON) $(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/sqlcipher.c crypto_openssl.o: $(TOP)/src/crypto_openssl.c $(DEPS_OBJ_COMMON) $(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/crypto_openssl.c +crypto_libtomcrypt.o: $(TOP)/src/crypto_libtomcrypt.c $(DEPS_OBJ_COMMON) + $(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/crypto_libtomcrypt.c crypto_cc.o: $(TOP)/src/crypto_cc.c $(DEPS_OBJ_COMMON) $(T.cc.sqlite) $(CFLAGS.libsqlite3) -c $(TOP)/src/crypto_cc.c diff --git a/src/crypto_libtomcrypt.c b/src/crypto_libtomcrypt.c new file mode 100644 index 000000000..c8ee378d3 --- /dev/null +++ b/src/crypto_libtomcrypt.c @@ -0,0 +1,316 @@ +/* +** SQLCipher +** http://sqlcipher.net +** +** Copyright (c) 2008 - 2013, ZETETIC LLC +** All rights reserved. +** +** Redistribution and use in source and binary forms, with or without +** modification, are permitted provided that the following conditions are met: +** * Redistributions of source code must retain the above copyright +** notice, this list of conditions and the following disclaimer. +** * Redistributions in binary form must reproduce the above copyright +** notice, this list of conditions and the following disclaimer in the +** documentation and/or other materials provided with the distribution. +** * Neither the name of the ZETETIC LLC nor the +** names of its contributors may be used to endorse or promote products +** derived from this software without specific prior written permission. +** +** THIS SOFTWARE IS PROVIDED BY ZETETIC LLC ''AS IS'' AND ANY +** EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +** WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +** DISCLAIMED. IN NO EVENT SHALL ZETETIC LLC BE LIABLE FOR ANY +** DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +** (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +** LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +** ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +** SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +** +*/ +/* BEGIN SQLCIPHER */ +#ifdef SQLITE_HAS_CODEC +#ifdef SQLCIPHER_CRYPTO_LIBTOMCRYPT +#include "sqliteInt.h" +#include "sqlcipher.h" +#include + +#define FORTUNA_MAX_SZ 32 +static prng_state prng; +static volatile unsigned int ltc_init = 0; +static volatile unsigned int ltc_ref_count = 0; + +#define LTC_CIPHER "rijndael" + +static int sqlcipher_ltc_add_random(void *ctx, const void *buffer, int length) { + int rc = 0; + int data_to_read = length; + int block_sz = data_to_read < FORTUNA_MAX_SZ ? data_to_read : FORTUNA_MAX_SZ; + const unsigned char * data = (const unsigned char *)buffer; + + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_add_random: entering SQLCIPHER_MUTEX_PROVIDER_RAND"); + sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_RAND)); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_add_random: entered SQLCIPHER_MUTEX_PROVIDER_RAND"); + + while(data_to_read > 0){ + rc = fortuna_add_entropy(data, block_sz, &prng); + rc = rc != CRYPT_OK ? SQLITE_ERROR : SQLITE_OK; + if(rc != SQLITE_OK){ + break; + } + data_to_read -= block_sz; + data += block_sz; + block_sz = data_to_read < FORTUNA_MAX_SZ ? data_to_read : FORTUNA_MAX_SZ; + } + fortuna_ready(&prng); + + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_add_random: leaving SQLCIPHER_MUTEX_PROVIDER_RAND"); + sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_RAND)); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_add_random: left SQLCIPHER_MUTEX_PROVIDER_RAND"); + + return rc; +} + +static int sqlcipher_ltc_activate(void *ctx) { + unsigned char random_buffer[FORTUNA_MAX_SZ]; + int bytes = 0; + + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_activate: entering SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); + sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_activate: entered SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); + + sqlcipher_memset(random_buffer, 0, FORTUNA_MAX_SZ); + if(ltc_init == 0) { + if(register_prng(&fortuna_desc) < 0) return SQLITE_ERROR; + if(register_cipher(&rijndael_desc) < 0) return SQLITE_ERROR; + if(register_hash(&sha512_desc) < 0) return SQLITE_ERROR; + if(register_hash(&sha256_desc) < 0) return SQLITE_ERROR; + if(register_hash(&sha1_desc) < 0) return SQLITE_ERROR; + if(fortuna_start(&prng) != CRYPT_OK) { + return SQLITE_ERROR; + } + + ltc_init = 1; + } + ltc_ref_count++; + +#ifndef SQLCIPHER_TEST + bytes = rng_get_bytes(random_buffer, FORTUNA_MAX_SZ, NULL); +#endif + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_PROVIDER, "sqlcipher_ltc_activate: seeded fortuna with %d bytes from rng_get_bytes", bytes); + + if(sqlcipher_ltc_add_random(ctx, random_buffer, FORTUNA_MAX_SZ) != SQLITE_OK) { + return SQLITE_ERROR; + } + sqlcipher_memset(random_buffer, 0, FORTUNA_MAX_SZ); + + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_activate: leaving SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); + sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_activate: left SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); + + return SQLITE_OK; +} + +static int sqlcipher_ltc_deactivate(void *ctx) { + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_deactivate: entering SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); + sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_deactivate: entered SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); + + ltc_ref_count--; + if(ltc_ref_count == 0){ + fortuna_done(&prng); + sqlcipher_memset((void *)&prng, 0, sizeof(prng)); + } + + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_deactivate: leaving SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); + sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_ACTIVATE)); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_deactivate: left SQLCIPHER_MUTEX_PROVIDER_ACTIVATE"); + + return SQLITE_OK; +} + +static const char* sqlcipher_ltc_get_provider_name(void *ctx) { + return "libtomcrypt"; +} + +static const char* sqlcipher_ltc_get_provider_version(void *ctx) { + return SCRYPT; +} + +static int sqlcipher_ltc_random(void *ctx, void *buffer, int length) { + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_random: entering SQLCIPHER_MUTEX_PROVIDER_RAND"); + sqlite3_mutex_enter(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_RAND)); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_random: entered SQLCIPHER_MUTEX_PROVIDER_RAND"); + + fortuna_read(buffer, length, &prng); + + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_random: leaving SQLCIPHER_MUTEX_PROVIDER_RAND"); + sqlite3_mutex_leave(sqlcipher_mutex(SQLCIPHER_MUTEX_PROVIDER_RAND)); + sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_MUTEX, "sqlcipher_ltc_random: left SQLCIPHER_MUTEX_PROVIDER_RAND"); + + return SQLITE_OK; +} + +static int sqlcipher_ltc_hmac( + void *ctx, int algorithm, + const unsigned char *hmac_key, int key_sz, + const unsigned char *in, int in_sz, + const unsigned char *in2, int in2_sz, + unsigned char *out +) { + int rc, hash_idx; + hmac_state hmac; + unsigned long outlen; + switch(algorithm) { + case SQLCIPHER_HMAC_SHA1: + hash_idx = find_hash("sha1"); + break; + case SQLCIPHER_HMAC_SHA256: + hash_idx = find_hash("sha256"); + break; + case SQLCIPHER_HMAC_SHA512: + hash_idx = find_hash("sha512"); + break; + default: + return SQLITE_ERROR; + } + + if(hash_idx < 0) return SQLITE_ERROR; + outlen = hash_descriptor[hash_idx].hashsize; + + if(in == NULL) return SQLITE_ERROR; + if((rc = hmac_init(&hmac, hash_idx, hmac_key, key_sz)) != CRYPT_OK) return SQLITE_ERROR; + if((rc = hmac_process(&hmac, in, in_sz)) != CRYPT_OK) return SQLITE_ERROR; + if(in2 != NULL && (rc = hmac_process(&hmac, in2, in2_sz)) != CRYPT_OK) return SQLITE_ERROR; + if((rc = hmac_done(&hmac, out, &outlen)) != CRYPT_OK) return SQLITE_ERROR; + return SQLITE_OK; +} + +static int sqlcipher_ltc_kdf( + void *ctx, int algorithm, + const unsigned char *pass, int pass_sz, + const unsigned char* salt, int salt_sz, + int workfactor, + int key_sz, unsigned char *key +) { + int rc, hash_idx; + unsigned long outlen = key_sz; + + switch(algorithm) { + case SQLCIPHER_HMAC_SHA1: + hash_idx = find_hash("sha1"); + break; + case SQLCIPHER_HMAC_SHA256: + hash_idx = find_hash("sha256"); + break; + case SQLCIPHER_HMAC_SHA512: + hash_idx = find_hash("sha512"); + break; + default: + return SQLITE_ERROR; + } + if(hash_idx < 0) return SQLITE_ERROR; + + if((rc = pkcs_5_alg2(pass, pass_sz, salt, salt_sz, + workfactor, hash_idx, key, &outlen)) != CRYPT_OK) { + return SQLITE_ERROR; + } + return SQLITE_OK; +} + +static const char* sqlcipher_ltc_get_cipher(void *ctx) { + return "aes-256-cbc"; +} + +static int sqlcipher_ltc_cipher( + void *ctx, int mode, + const unsigned char *key, int key_sz, + const unsigned char *iv, + const unsigned char *in, int in_sz, + unsigned char *out +) { + int rc, cipher_idx; + symmetric_CBC cbc; + + if((cipher_idx = find_cipher(LTC_CIPHER)) == -1) return SQLITE_ERROR; + if((rc = cbc_start(cipher_idx, iv, key, key_sz, 0, &cbc)) != CRYPT_OK) return SQLITE_ERROR; + rc = mode == SQLCIPHER_ENCRYPT ? cbc_encrypt(in, out, in_sz, &cbc) : cbc_decrypt(in, out, in_sz, &cbc); + if(rc != CRYPT_OK) return SQLITE_ERROR; + cbc_done(&cbc); + return SQLITE_OK; +} + +static int sqlcipher_ltc_get_key_sz(void *ctx) { + int cipher_idx = find_cipher(LTC_CIPHER); + return cipher_descriptor[cipher_idx].max_key_length; +} + +static int sqlcipher_ltc_get_iv_sz(void *ctx) { + int cipher_idx = find_cipher(LTC_CIPHER); + return cipher_descriptor[cipher_idx].block_length; +} + +static int sqlcipher_ltc_get_block_sz(void *ctx) { + int cipher_idx = find_cipher(LTC_CIPHER); + return cipher_descriptor[cipher_idx].block_length; +} + +static int sqlcipher_ltc_get_hmac_sz(void *ctx, int algorithm) { + int hash_idx; + switch(algorithm) { + case SQLCIPHER_HMAC_SHA1: + hash_idx = find_hash("sha1"); + break; + case SQLCIPHER_HMAC_SHA256: + hash_idx = find_hash("sha256"); + break; + case SQLCIPHER_HMAC_SHA512: + hash_idx = find_hash("sha512"); + break; + default: + return 0; + } + + if(hash_idx < 0) return 0; + + return hash_descriptor[hash_idx].hashsize; +} + +static int sqlcipher_ltc_ctx_init(void **ctx) { + sqlcipher_ltc_activate(NULL); + return SQLITE_OK; +} + +static int sqlcipher_ltc_ctx_free(void **ctx) { + sqlcipher_ltc_deactivate(&ctx); + return SQLITE_OK; +} + +static int sqlcipher_ltc_fips_status(void *ctx) { + return 0; +} + +int sqlcipher_ltc_setup(sqlcipher_provider *p) { + p->init = NULL; + p->shutdown = NULL; + p->get_provider_name = sqlcipher_ltc_get_provider_name; + p->random = sqlcipher_ltc_random; + p->hmac = sqlcipher_ltc_hmac; + p->kdf = sqlcipher_ltc_kdf; + p->cipher = sqlcipher_ltc_cipher; + p->get_cipher = sqlcipher_ltc_get_cipher; + p->get_key_sz = sqlcipher_ltc_get_key_sz; + p->get_iv_sz = sqlcipher_ltc_get_iv_sz; + p->get_block_sz = sqlcipher_ltc_get_block_sz; + p->get_hmac_sz = sqlcipher_ltc_get_hmac_sz; + p->ctx_init = sqlcipher_ltc_ctx_init; + p->ctx_free = sqlcipher_ltc_ctx_free; + p->add_random = sqlcipher_ltc_add_random; + p->fips_status = sqlcipher_ltc_fips_status; + p->get_provider_version = sqlcipher_ltc_get_provider_version; + return SQLITE_OK; +} + +#endif +#endif +/* END SQLCIPHER */ diff --git a/src/sqlcipher.c b/src/sqlcipher.c index bf07153f8..c93dcf26f 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -84,6 +84,7 @@ void sqlite3pager_reset(Pager *pPager); /* end extensions defined in pager.c */ #if !defined (SQLCIPHER_CRYPTO_CC) \ + && !defined (SQLCIPHER_CRYPTO_LIBTOMCRYPT) \ && !defined (SQLCIPHER_CRYPTO_OPENSSL) \ && !defined (SQLCIPHER_CRYPTO_CUSTOM) #define SQLCIPHER_CRYPTO_OPENSSL @@ -504,6 +505,9 @@ int sqlcipher_extra_init(const char* arg) { #if defined (SQLCIPHER_CRYPTO_CC) extern int sqlcipher_cc_setup(sqlcipher_provider *p); sqlcipher_cc_setup(p); +#elif defined (SQLCIPHER_CRYPTO_LIBTOMCRYPT) + extern int sqlcipher_ltc_setup(sqlcipher_provider *p); + sqlcipher_ltc_setup(p); #elif defined (SQLCIPHER_CRYPTO_OPENSSL) extern int sqlcipher_openssl_setup(sqlcipher_provider *p); sqlcipher_openssl_setup(p); diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 93af23994..c5e45cd5d 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -436,6 +436,7 @@ set flist { memjournal.c sqlcipher.c + crypto_libtomcrypt.c crypto_openssl.c crypto_cc.c From 3c37ad152c5870109be7013108bbfd86c97277ac Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Mon, 23 Feb 2026 16:26:16 -0500 Subject: [PATCH 59/75] Fixes provider specific test detection --- test/sqlcipher.tcl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/sqlcipher.tcl b/test/sqlcipher.tcl index 4752316c6..062a52ffe 100644 --- a/test/sqlcipher.tcl +++ b/test/sqlcipher.tcl @@ -61,10 +61,12 @@ proc setup {file key} { proc get_cipher_provider {} { sqlite_orig db test.db - return [execsql { - PRAGMA key = 'test'; - PRAGMA cipher_provider; - }]; + execsql { + PRAGMA key = 'test'; + } + return [execsql { + PRAGMA cipher_provider; + }]; } proc if_built_with_openssl {name cmd expected} { From 03c64f7b78125c730ff539eb8baba113cfd7e719 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Thu, 26 Feb 2026 10:00:31 -0500 Subject: [PATCH 60/75] Fixes double close of database handle in compat test --- test/sqlcipher-compatibility.test | 1 - 1 file changed, 1 deletion(-) diff --git a/test/sqlcipher-compatibility.test b/test/sqlcipher-compatibility.test index 656b1c458..edaa0f6cc 100644 --- a/test/sqlcipher-compatibility.test +++ b/test/sqlcipher-compatibility.test @@ -1294,7 +1294,6 @@ if_built_with_libtomcrypt verify-random-data-alters-file-content { lappend rc [cmpFilesChunked test.db test2.db] lappend rc [cmpFilesChunked test2.db test3.db] } {0 1} -db close file delete -force test.db file delete -force test2.db file delete -force test3.db From c66691b89097fd9bbda2b16d843f942842d7065b Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Thu, 26 Feb 2026 14:14:35 -0500 Subject: [PATCH 61/75] Always seed libtomcrypt CSPRNG (even for tests) --- src/crypto_libtomcrypt.c | 2 -- test/sqlcipher-compatibility.test | 34 ------------------------------- 2 files changed, 36 deletions(-) diff --git a/src/crypto_libtomcrypt.c b/src/crypto_libtomcrypt.c index c8ee378d3..0ade64b9b 100644 --- a/src/crypto_libtomcrypt.c +++ b/src/crypto_libtomcrypt.c @@ -94,9 +94,7 @@ static int sqlcipher_ltc_activate(void *ctx) { } ltc_ref_count++; -#ifndef SQLCIPHER_TEST bytes = rng_get_bytes(random_buffer, FORTUNA_MAX_SZ, NULL); -#endif sqlcipher_log(SQLCIPHER_LOG_TRACE, SQLCIPHER_LOG_PROVIDER, "sqlcipher_ltc_activate: seeded fortuna with %d bytes from rng_get_bytes", bytes); if(sqlcipher_ltc_add_random(ctx, random_buffer, FORTUNA_MAX_SZ) != SQLITE_OK) { diff --git a/test/sqlcipher-compatibility.test b/test/sqlcipher-compatibility.test index edaa0f6cc..4053d78d9 100644 --- a/test/sqlcipher-compatibility.test +++ b/test/sqlcipher-compatibility.test @@ -1264,40 +1264,6 @@ db close file delete -force test.db file delete -force new.db - -# Requires SQLCipher to be built with -DSQLCIPHER_TEST -if_built_with_libtomcrypt verify-random-data-alters-file-content { - file delete -force test.db - file delete -force test2.db - file delete -force test3.db - set rc {} - - sqlite_orig db test.db - execsql { - PRAGMA key="x'2DD29CA851E7B56E4697B0E1F08507293D761A05CE4D1B628663F411A8086D99'"; - create table t1(a,b); - } - db close - sqlite_orig db test2.db - execsql { - PRAGMA key="x'2DD29CA851E7B56E4697B0E1F08507293D761A05CE4D1B628663F411A8086D99'"; - create table t1(a,b); - } - db close - sqlite_orig db test3.db - execsql { - PRAGMA key="x'2DD29CA851E7B56E4697B0E1F08507293D761A05CE4D1B628663F411A8086D99'"; - PRAGMA cipher_add_random = "x'deadbaad'"; - create table t1(a,b); - } - db close - lappend rc [cmpFilesChunked test.db test2.db] - lappend rc [cmpFilesChunked test2.db test3.db] -} {0 1} -file delete -force test.db -file delete -force test2.db -file delete -force test3.db - do_test can-migrate-with-keys-longer-than-64-characters { sqlite_orig db test.db execsql { From 50ce5b8b383c6369fb436616fd3e583c54d8659e Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Thu, 26 Feb 2026 14:19:26 -0500 Subject: [PATCH 62/75] Modernize libtomcrypt identifiers from rijndael to aes --- src/crypto_libtomcrypt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/crypto_libtomcrypt.c b/src/crypto_libtomcrypt.c index 0ade64b9b..34fde7dcd 100644 --- a/src/crypto_libtomcrypt.c +++ b/src/crypto_libtomcrypt.c @@ -40,7 +40,7 @@ static prng_state prng; static volatile unsigned int ltc_init = 0; static volatile unsigned int ltc_ref_count = 0; -#define LTC_CIPHER "rijndael" +#define LTC_CIPHER "aes" static int sqlcipher_ltc_add_random(void *ctx, const void *buffer, int length) { int rc = 0; @@ -82,7 +82,7 @@ static int sqlcipher_ltc_activate(void *ctx) { sqlcipher_memset(random_buffer, 0, FORTUNA_MAX_SZ); if(ltc_init == 0) { if(register_prng(&fortuna_desc) < 0) return SQLITE_ERROR; - if(register_cipher(&rijndael_desc) < 0) return SQLITE_ERROR; + if(register_cipher(&aes_desc) < 0) return SQLITE_ERROR; if(register_hash(&sha512_desc) < 0) return SQLITE_ERROR; if(register_hash(&sha256_desc) < 0) return SQLITE_ERROR; if(register_hash(&sha1_desc) < 0) return SQLITE_ERROR; From e4a682c03e6224a4a82c7906c390294f2d969f9b Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Thu, 5 Mar 2026 14:25:17 -0500 Subject: [PATCH 63/75] Improves logging and error handling for LibTomCrypt --- src/crypto_libtomcrypt.c | 103 ++++++++++++++++++++++++++++++++------- 1 file changed, 86 insertions(+), 17 deletions(-) diff --git a/src/crypto_libtomcrypt.c b/src/crypto_libtomcrypt.c index 34fde7dcd..3447294a6 100644 --- a/src/crypto_libtomcrypt.c +++ b/src/crypto_libtomcrypt.c @@ -157,8 +157,11 @@ static int sqlcipher_ltc_hmac( unsigned char *out ) { int rc, hash_idx; - hmac_state hmac; unsigned long outlen; + hmac_state *hmac = NULL; + + if(in == NULL) goto error; + switch(algorithm) { case SQLCIPHER_HMAC_SHA1: hash_idx = find_hash("sha1"); @@ -170,18 +173,47 @@ static int sqlcipher_ltc_hmac( hash_idx = find_hash("sha512"); break; default: - return SQLITE_ERROR; + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "%s: unsupported hmac algorithm", __func__); + goto error; + } + + if(hash_idx < 0) { + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "%s: failed find_hash lookup", __func__); + goto error; } - if(hash_idx < 0) return SQLITE_ERROR; outlen = hash_descriptor[hash_idx].hashsize; - if(in == NULL) return SQLITE_ERROR; - if((rc = hmac_init(&hmac, hash_idx, hmac_key, key_sz)) != CRYPT_OK) return SQLITE_ERROR; - if((rc = hmac_process(&hmac, in, in_sz)) != CRYPT_OK) return SQLITE_ERROR; - if(in2 != NULL && (rc = hmac_process(&hmac, in2, in2_sz)) != CRYPT_OK) return SQLITE_ERROR; - if((rc = hmac_done(&hmac, out, &outlen)) != CRYPT_OK) return SQLITE_ERROR; - return SQLITE_OK; + if(!(hmac = sqlcipher_malloc(sizeof(hmac_state)))) { + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "%s: failed to allocate hmac_state", __func__); + goto error; + } + if((rc = hmac_init(hmac, hash_idx, hmac_key, key_sz)) != CRYPT_OK) { + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "%s: hmac_init failed %d", __func__, rc); + goto error; + } + if((rc = hmac_process(hmac, in, in_sz)) != CRYPT_OK) { + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "%s: hmac_process failed %d", __func__, rc); + goto error; + } + if(in2 != NULL && (rc = hmac_process(hmac, in2, in2_sz)) != CRYPT_OK) { + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "%s: hmac_process 2 failed %d", __func__, rc); + goto error; + } + if((rc = hmac_done(hmac, out, &outlen)) != CRYPT_OK) { + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "%s: done failed %d", __func__, rc); + goto error; + } + + rc = SQLITE_OK; + goto cleanup; + +error: + rc = SQLITE_ERROR; + +cleanup: + if(hmac) sqlcipher_free(hmac, sizeof(hmac_state)); + return rc; } static int sqlcipher_ltc_kdf( @@ -205,12 +237,18 @@ static int sqlcipher_ltc_kdf( hash_idx = find_hash("sha512"); break; default: + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "%s: unsupported hmac algorithm", __func__); return SQLITE_ERROR; } - if(hash_idx < 0) return SQLITE_ERROR; + + if(hash_idx < 0) { + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "%s: failed find_hash lookup", __func__); + return SQLITE_ERROR; + } if((rc = pkcs_5_alg2(pass, pass_sz, salt, salt_sz, workfactor, hash_idx, key, &outlen)) != CRYPT_OK) { + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "%s: failed pkc_5_alg2 %d", __func__, rc); return SQLITE_ERROR; } return SQLITE_OK; @@ -228,14 +266,45 @@ static int sqlcipher_ltc_cipher( unsigned char *out ) { int rc, cipher_idx; - symmetric_CBC cbc; + symmetric_CBC *cbc = NULL; - if((cipher_idx = find_cipher(LTC_CIPHER)) == -1) return SQLITE_ERROR; - if((rc = cbc_start(cipher_idx, iv, key, key_sz, 0, &cbc)) != CRYPT_OK) return SQLITE_ERROR; - rc = mode == SQLCIPHER_ENCRYPT ? cbc_encrypt(in, out, in_sz, &cbc) : cbc_decrypt(in, out, in_sz, &cbc); - if(rc != CRYPT_OK) return SQLITE_ERROR; - cbc_done(&cbc); - return SQLITE_OK; + if((cipher_idx = find_cipher(LTC_CIPHER)) == -1) { + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "%s: failed find_cipher lookup", __func__); + goto error; + } + if(!(cbc = sqlcipher_malloc(sizeof(symmetric_CBC)))) { + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "%s: failed to allocate symmetric_CBC", __func__); + goto error; + } + if((rc = cbc_start(cipher_idx, iv, key, key_sz, 0, cbc)) != CRYPT_OK) { + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "%s: failed cbc_start %d", __func__, rc); + goto error; + } + if(mode == SQLCIPHER_ENCRYPT) { + if((rc = cbc_encrypt(in, out, in_sz, cbc)) != CRYPT_OK) { + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "%s: failed cbc_encrypt %d", __func__, rc); + goto error; + } + } else { + if((rc = cbc_decrypt(in, out, in_sz, cbc)) != CRYPT_OK) { + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "%s: failed cbc_decrypt %d", __func__, rc); + goto error; + } + } + if((rc = cbc_done(cbc)) != CRYPT_OK) { + sqlcipher_log(SQLCIPHER_LOG_ERROR, SQLCIPHER_LOG_PROVIDER, "%s: failed cbc_done %d", __func__, rc); + goto error; + } + + rc = SQLITE_OK; + goto cleanup; + +error: + rc = SQLITE_ERROR; + +cleanup: + if(cbc) sqlcipher_free(cbc, sizeof(symmetric_CBC)); + return rc; } static int sqlcipher_ltc_get_key_sz(void *ctx) { From 7b0662cd0733e9064ccd1827d6b6779658d5f913 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo Date: Fri, 6 Mar 2026 21:37:56 -0500 Subject: [PATCH 64/75] Snapshot of upstream SQLite 3.52.0 --- Makefile.msc | 203 +- README.md | 99 +- VERSION | 2 +- autoconf/Makefile.in | 15 +- autoconf/Makefile.msc | 100 +- autosetup/README.md | 10 + autosetup/jimsh0.c | 10 +- autosetup/proj.tcl | 4 +- autosetup/sqlite-config.tcl | 24 +- doc/compile-for-unix.md | 13 +- doc/compile-for-windows.md | 36 + doc/lemon.html | 32 +- doc/testrunner.md | 159 +- ext/expert/expert1.test | 2 +- ext/fts3/fts3.c | 9 +- ext/fts3/fts3Int.h | 9 + ext/fts3/fts3_write.c | 42 +- ext/fts5/fts5_aux.c | 2 +- ext/fts5/fts5_buffer.c | 2 +- ext/fts5/fts5_config.c | 4 +- ext/fts5/fts5_expr.c | 4 +- ext/fts5/fts5_hash.c | 2 +- ext/fts5/fts5_index.c | 74 +- ext/fts5/fts5_main.c | 5 +- ext/fts5/fts5_tcl.c | 8 +- ext/fts5/fts5_test_tok.c | 10 +- ext/fts5/fts5_tokenize.c | 8 +- ext/fts5/fts5_vocab.c | 2 +- ext/fts5/test/fts5corrupt9.test | 129 + ext/fts5/test/fts5integrity.test | 3 - ext/fts5/test/fts5interrupt.test | 17 + ext/intck/sqlite3intck.c | 2 +- ext/jni/README.md | 35 +- ext/jni/src/c/sqlite3-jni.c | 94 +- ext/jni/src/c/sqlite3-jni.h | 4 +- ext/jni/src/org/sqlite/jni/capi/CApi.java | 8 +- ext/jni/src/org/sqlite/jni/capi/Tester1.java | 4 +- .../src/org/sqlite/jni/wrapper1/Sqlite.java | 1 + ext/misc/btreeinfo.c | 6 +- ext/misc/csv.c | 4 +- ext/misc/decimal.c | 43 +- ext/misc/fileio.c | 240 +- ext/misc/fossildelta.c | 10 +- ext/misc/fuzzer.c | 2 +- ext/misc/ieee754.c | 36 +- ext/misc/regexp.c | 28 +- ext/misc/sha1.c | 15 +- ext/misc/sqlite3_stdio.c | 20 +- ext/misc/sqlite3_stdio.h | 3 + ext/misc/tmstmpvfs.c | 1042 +++ ext/misc/vtablog.c | 54 +- ext/misc/zipfile.c | 60 +- ext/qrf/README.md | 762 ++ ext/qrf/dev-notes.md | 14 + ext/qrf/qrf.c | 2983 ++++++ ext/qrf/qrf.h | 200 + ext/rbu/rbu11.test | 28 + ext/rbu/sqlite3rbu.c | 6 +- ext/repair/README.md | 16 - ext/repair/checkfreelist.c | 310 - ext/repair/checkindex.c | 929 -- ext/repair/sqlite3_checker.c.in | 85 - ext/repair/sqlite3_checker.tcl | 264 - ext/repair/test/README.md | 13 - ext/repair/test/checkfreelist01.test | 92 - ext/repair/test/checkindex01.test | 349 - ext/repair/test/test.tcl | 67 - ext/rtree/geopoly.c | 2 +- ext/session/session4.test | 1 + ext/session/sessionC.test | 10 + ext/session/sqlite3session.c | 60 +- ext/session/test_session.c | 66 +- ext/wasm/EXPORTED_FUNCTIONS.fiddle.in | 10 - ext/wasm/GNUmakefile | 483 +- ...S.sqlite3-core => EXPORTED_FUNCTIONS.c-pp} | 90 + .../api/EXPORTED_FUNCTIONS.sqlite3-extras | 68 - ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see | 5 - .../api/EXPORTED_RUNTIME_METHODS.sqlite3-api | 3 - ext/wasm/api/README.md | 84 +- ext/wasm/api/extern-post-js.c-pp.js | 4 +- ext/wasm/api/post-js-footer.js | 69 + ext/wasm/api/post-js-header.js | 10 +- ext/wasm/api/pre-js.c-pp.js | 50 +- ext/wasm/api/sqlite3-api-cleanup.js | 83 - ext/wasm/api/sqlite3-api-glue.c-pp.js | 155 +- ext/wasm/api/sqlite3-api-oo1.c-pp.js | 293 +- ext/wasm/api/sqlite3-api-prologue.js | 385 +- .../api/sqlite3-license-version-header.js | 3 +- ext/wasm/api/sqlite3-opfs-async-proxy.js | 2 +- ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js | 2095 +++++ ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js | 18 +- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 8 +- ext/wasm/api/sqlite3-vtab-helper.c-pp.js | 5 +- ext/wasm/api/sqlite3-wasm.c | 294 +- ext/wasm/api/sqlite3-worker1-promiser.c-pp.js | 26 +- ext/wasm/api/sqlite3-worker1.c-pp.js | 4 +- ext/wasm/c-pp-lite.c | 69 +- ext/wasm/common/SqliteTestUtil.js | 3 +- ext/wasm/common/whwasmutil.js | 67 +- ext/wasm/demo-jsstorage.js | 8 +- ext/wasm/demo-worker1-promiser.c-pp.html | 8 +- ext/wasm/demo-worker1.js | 6 +- ext/wasm/fiddle/fiddle-worker.js | 4 - .../fiddle/{index.html => index.c-pp.html} | 17 +- ext/wasm/index.html | 5 + ext/wasm/jaccwabyt/jaccwabyt.js | 851 +- ext/wasm/jaccwabyt/jaccwabyt.md | 698 +- ext/wasm/mkdist.sh | 2 +- ext/wasm/mkwasmbuilds.c | 109 +- ext/wasm/speedtest1-worker.js | 13 +- ext/wasm/speedtest1.html | 203 +- ext/wasm/tester1-worker.c-pp.html | 16 +- ext/wasm/tester1.c-pp.html | 18 +- ext/wasm/tester1.c-pp.js | 983 +- main.mk | 95 +- make.bat | 2 + manifest | 519 +- manifest.tags | 7 +- manifest.uuid | 2 +- src/alter.c | 779 +- src/attach.c | 2 +- src/auth.c | 2 +- src/btree.c | 30 +- src/build.c | 17 +- src/carray.c | 39 +- src/date.c | 16 +- src/delete.c | 3 +- src/expr.c | 93 +- src/fkey.c | 12 +- src/func.c | 16 +- src/hwtime.h | 41 +- src/json.c | 113 +- src/loadext.c | 47 +- src/main.c | 36 +- src/malloc.c | 24 +- src/mutex.c | 29 +- src/mutex_w32.c | 8 - src/os_kv.c | 266 +- src/os_unix.c | 2 +- src/os_win.c | 436 +- src/os_win.h | 10 +- src/pager.c | 12 +- src/parse.y | 206 +- src/pcache.h | 4 +- src/prepare.c | 3 +- src/printf.c | 21 +- src/resolve.c | 18 +- src/select.c | 721 +- src/shell.c.in | 7958 ++++++++--------- src/sqlite.h.in | 241 +- src/sqlite3ext.h | 11 +- src/sqliteInt.h | 128 +- src/sqliteLimit.h | 50 +- src/tclsqlite.c | 417 +- src/test1.c | 43 +- src/test_bestindex.c | 1 - src/test_config.c | 16 +- src/test_quota.c | 4 - src/tokenize.c | 4 +- src/treeview.c | 8 +- src/trigger.c | 137 +- src/util.c | 665 +- src/vacuum.c | 12 +- src/vdbe.c | 21 +- src/vdbe.h | 2 +- src/vdbeInt.h | 4 +- src/vdbeapi.c | 52 +- src/vdbeaux.c | 2 +- src/vdbemem.c | 201 +- src/wal.c | 131 +- src/where.c | 227 +- src/wherecode.c | 2 +- src/whereexpr.c | 79 +- src/window.c | 6 +- test/alterauth2.test | 51 + test/altercol.test | 13 + test/altercons.test | 442 + test/altercons2.test | 247 + test/alterfault.test | 37 + test/altertab3.test | 48 + test/altertrig.test | 2 +- test/atof2.test | 35 + test/autoindex1.test | 3 +- test/avfs.test | 2 + test/bestindex8.test | 16 +- test/bestindexB.test | 2 +- test/bestindexF.test | 294 + test/carray01.test | 8 + test/collate5.test | 30 +- test/corruptL.test | 2 +- test/cost.test | 2 +- test/dblwidth-a.sql | 38 +- test/distinct2.test | 2 +- test/dotcmd01.sql | 63 + test/e_expr.test | 8 +- test/e_update.test | 14 +- test/e_walckpt.test | 17 +- test/eqp.test | 55 +- test/filectrl.test | 12 +- test/fpconv1.test | 66 +- test/fptest01.sql | 76 + test/fts3comp1.test | 77 + test/fts4content.test | 59 +- test/fts4merge5.test | 3 + test/fuzzcheck.c | 14 +- test/fuzzinvariants.c | 9 +- test/import01.sql | 217 + test/imposter1.sql | 32 + test/insert5.test | 19 +- test/intck01.sql | 23 + test/join.test | 41 + test/joinI.test | 47 + test/json102.test | 21 + test/json103.test | 7 + test/json105.test | 5 + test/json109.test | 72 + test/misc5.test | 2 +- test/modeA.sql | 303 + test/mutex1.test | 26 +- test/notnull2.test | 2 +- test/offset1.test | 28 + test/qrf01.test | 1167 +++ test/qrf02.test | 47 + test/qrf03.test | 176 + test/qrf04.test | 750 ++ test/qrf05.test | 37 + test/qrf06.test | 576 ++ test/recover.test | 2 +- test/regexp1.sql | 32 + test/regexp1.test | 24 + test/rowvalue4.test | 3 +- test/rowvalueA.test | 25 + test/schema.test | 4 +- test/select9.test | 2 +- test/shell1.test | 227 +- test/shell2.test | 92 +- test/shell4.test | 8 +- test/shell5.test | 124 +- test/shell8.test | 40 + test/shellA.test | 233 +- test/shellB.test | 53 + test/speedtest.md | 5 +- test/speedtest.tcl | 2 +- test/tabfunc01.test | 14 + test/tclsqlite.test | 2 +- test/temptrigfault.tes | 120 + test/temptrigger.test | 187 + test/tester.tcl | 5 - test/testrunner.tcl | 128 +- test/testrunner_data.tcl | 8 +- test/testrunner_estwork.tcl | 1 + test/tkt-99378177930f87bd.test | 28 + test/tkt2339.test | 6 +- test/values.test | 6 +- test/vt02.c | 125 +- test/vt100-a.sql | 35 +- test/walckptnoop.test | 6 +- test/walrestart.test | 89 + test/where2.test | 37 + test/whereK.test | 13 + test/win32longpath.test | 4 - test/with1.test | 19 + test/zipfile.test | 6 + test/zipfile2.test | 15 + tool/build-all-msvc.bat | 3 +- tool/dbtotxt.c | 10 +- tool/lemon.c | 63 +- tool/lempar.c | 17 +- tool/mkautoconfamal.sh | 3 +- tool/mkcombo.tcl | 106 + tool/mkkeywordhash.c | 4 +- tool/mkshellc.tcl | 120 +- tool/omittest.tcl | 2 +- tool/showdb.c | 247 +- tool/showtmlog.c | 254 + tool/sqldiff.c | 42 +- tool/sqlite3_rsync.c | 33 +- tool/sqltclsh.c.in | 2 +- tool/winmain.c | 79 + 279 files changed, 26906 insertions(+), 11541 deletions(-) create mode 100644 ext/fts5/test/fts5corrupt9.test create mode 100644 ext/misc/tmstmpvfs.c create mode 100644 ext/qrf/README.md create mode 100644 ext/qrf/dev-notes.md create mode 100644 ext/qrf/qrf.c create mode 100644 ext/qrf/qrf.h delete mode 100644 ext/repair/README.md delete mode 100644 ext/repair/checkfreelist.c delete mode 100644 ext/repair/checkindex.c delete mode 100644 ext/repair/sqlite3_checker.c.in delete mode 100644 ext/repair/sqlite3_checker.tcl delete mode 100644 ext/repair/test/README.md delete mode 100644 ext/repair/test/checkfreelist01.test delete mode 100644 ext/repair/test/checkindex01.test delete mode 100644 ext/repair/test/test.tcl delete mode 100644 ext/wasm/EXPORTED_FUNCTIONS.fiddle.in rename ext/wasm/api/{EXPORTED_FUNCTIONS.sqlite3-core => EXPORTED_FUNCTIONS.c-pp} (61%) delete mode 100644 ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras delete mode 100644 ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see delete mode 100644 ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api delete mode 100644 ext/wasm/api/sqlite3-api-cleanup.js create mode 100644 ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js rename ext/wasm/fiddle/{index.html => index.c-pp.html} (95%) create mode 100644 make.bat create mode 100644 test/altercons.test create mode 100644 test/altercons2.test create mode 100644 test/atof2.test create mode 100644 test/bestindexF.test create mode 100644 test/dotcmd01.sql create mode 100644 test/fptest01.sql create mode 100644 test/import01.sql create mode 100644 test/imposter1.sql create mode 100644 test/intck01.sql create mode 100644 test/json109.test create mode 100644 test/modeA.sql create mode 100644 test/qrf01.test create mode 100644 test/qrf02.test create mode 100644 test/qrf03.test create mode 100644 test/qrf04.test create mode 100644 test/qrf05.test create mode 100644 test/qrf06.test create mode 100644 test/regexp1.sql create mode 100644 test/shellB.test create mode 100644 test/temptrigfault.tes create mode 100644 test/walrestart.test create mode 100644 tool/mkcombo.tcl create mode 100644 tool/showtmlog.c create mode 100644 tool/winmain.c diff --git a/Makefile.msc b/Makefile.msc index 52807ff7f..763616ce9 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -11,6 +11,8 @@ TOP = . # <> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# # Set this non-0 to create and use the SQLite amalgamation file. # !IFNDEF USE_AMALGAMATION @@ -101,13 +103,6 @@ NO_WARN = $(NO_WARN) -wd4210 -wd4232 -wd4244 -wd4305 -wd4306 -wd4702 -wd4706 !ENDIF !ENDIF -# Set this non-0 to use the library paths and other options necessary for -# Windows Phone 8.1. -# -!IFNDEF USE_WP81_OPTS -USE_WP81_OPTS = 0 -!ENDIF - # Set this non-0 to split the SQLite amalgamation file into chunks to # be used for debugging with Visual Studio. # @@ -116,14 +111,8 @@ SPLIT_AMALGAMATION = 0 !ENDIF # <> -# Set this non-0 to have this makefile assume the Tcl shell executable -# (tclsh*.exe) is available in the PATH. By default, this is disabled -# for compatibility with older build environments. This setting only -# applies if TCLSH_CMD is not set manually. +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc # -!IFNDEF USE_TCLSH_IN_PATH -USE_TCLSH_IN_PATH = 0 -!ENDIF # Set this non-0 to use zlib, possibly compiling it from source code. # @@ -186,14 +175,6 @@ USE_NATIVE_LIBPATHS = 0 USE_RC = 1 !ENDIF -# Set this non-0 to compile binaries suitable for the WinRT environment. -# This setting does not apply to any binaries that require Tcl to operate -# properly (i.e. the text fixture, etc). -# -!IFNDEF FOR_WINRT -FOR_WINRT = 0 -!ENDIF - # Set this non-0 to compile binaries suitable for the UWP environment. # This setting does not apply to any binaries that require Tcl to operate # properly (i.e. the text fixture, etc). @@ -209,6 +190,8 @@ FOR_WIN10 = 0 !ENDIF # <> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# # Set this non-0 to skip attempting to look for and/or link with the Tcl # runtime library. # @@ -265,6 +248,8 @@ DEBUG = 0 !ENDIF # <> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# # By default, use --linemacros=1 argument to the mksqlite3c.tcl tool, which # is used to build the amalgamation. This can be turned off to ease debug # of the amalgamation away from the source tree. @@ -360,6 +345,8 @@ SQLITE3EXEPDB = /pdb:sqlite3sh.pdb !ENDIF # <> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# # These are the names of the customized Tcl header files used by various parts # of this makefile when the stdcall calling convention is in use. It is not # used for any other purpose. @@ -641,16 +628,24 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS) !IF "$(PLATFORM)"=="x86" CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall + # <> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# TEST_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl # <> + !ELSE !IFNDEF PLATFORM CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall + # <> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# TEST_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl # <> + !ELSE CORE_CCONV_OPTS = SHELL_CCONV_OPTS = @@ -777,18 +772,6 @@ SHELL_LINK_OPTS = $(SHELL_CORE_LIB) TCC = $(TCC) -FAcs !ENDIF -# When compiling the library for use in the WinRT environment, -# the following compile-time options must be used as well to -# disable use of Win32 APIs that are not available and to enable -# use of Win32 APIs that are specific to Windows 8 and/or WinRT. -# -!IF $(FOR_WINRT)!=0 -TCC = $(TCC) -DSQLITE_OS_WINRT=1 -RCC = $(RCC) -DSQLITE_OS_WINRT=1 -TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP -RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP -!ENDIF - # C compiler options for the Windows 10 platform (needs MSVC 2015). # !IF $(FOR_WIN10)!=0 @@ -801,7 +784,7 @@ BCC = $(BCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE # USE_CRT_DLL option is set to force dynamically linking to the # MSVC runtime library. # -!IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0 +!IF $(USE_CRT_DLL)!=0 !IF $(DEBUG)>1 TCC = $(TCC) -MDd BCC = $(BCC) -MDd @@ -824,6 +807,8 @@ ZLIBCFLAGS = -nologo -MT -W3 -O2 -Oy- -Zi !ENDIF # <> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# # The mksqlite3c.tcl and mksqlite3h.tcl scripts will pull in # any extension header files by default. For non-amalgamation # builds, we need to make sure the compiler can find these. @@ -874,16 +859,6 @@ MKSQLITE3H_ARGS = !ENDIF # <> -# Define -DNDEBUG to compile without debugging (i.e., for production usage) -# Omitting the define will cause extra debugging code to be inserted and -# includes extra comments when "EXPLAIN stmt" is used. -# -!IF $(DEBUG)==0 -TCC = $(TCC) -DNDEBUG -BCC = $(BCC) -DNDEBUG -RCC = $(RCC) -DNDEBUG -!ENDIF - !IF $(DEBUG)>0 || $(API_ARMOR)!=0 || $(FOR_WIN10)!=0 TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1 RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 @@ -949,6 +924,8 @@ TCC = $(TCC) /fsanitize=address !ENDIF # <> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# # The locations of the Tcl header and library files. Also, the library that # non-stubs enabled programs using Tcl must link against. These variables # (TCLINCDIR, TCLLIBDIR, and LIBTCL) may be overridden via the environment @@ -1188,6 +1165,8 @@ BCC = $(BCC) -Zi !ENDIF # <> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# # If zlib support is enabled, add the compiler options for it. # !IF $(USE_ZLIB)!=0 @@ -1240,56 +1219,6 @@ LTLINKOPTS = /NOLOGO LTLIBOPTS = /NOLOGO !ENDIF -# When compiling for use in the WinRT environment, the following -# linker option must be used to mark the executable as runnable -# only in the context of an application container. -# -!IF $(FOR_WINRT)!=0 -LTLINKOPTS = $(LTLINKOPTS) /APPCONTAINER -!IF "$(VISUALSTUDIOVERSION)"=="12.0" || "$(VISUALSTUDIOVERSION)"=="14.0" -!IFNDEF STORELIBPATH -!IF "$(PLATFORM)"=="x86" -STORELIBPATH = $(CRTLIBPATH)\store -!ELSEIF "$(PLATFORM)"=="x64" -STORELIBPATH = $(CRTLIBPATH)\store\amd64 -!ELSEIF "$(PLATFORM)"=="ARM" -STORELIBPATH = $(CRTLIBPATH)\store\arm -!ELSE -STORELIBPATH = $(CRTLIBPATH)\store -!ENDIF -!ENDIF -STORELIBPATH = $(STORELIBPATH:\\=\) -LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(STORELIBPATH)" -!ENDIF -!ENDIF - -# When compiling for Windows Phone 8.1, an extra library path is -# required. -# -!IF $(USE_WP81_OPTS)!=0 -!IFNDEF WP81LIBPATH -!IF "$(PLATFORM)"=="x86" -WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 -!ELSEIF "$(PLATFORM)"=="ARM" -WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\ARM -!ELSE -WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 -!ENDIF -!ENDIF -!ENDIF - -# When compiling for Windows Phone 8.1, some extra linker options -# are also required. -# -!IF $(USE_WP81_OPTS)!=0 -!IFDEF WP81LIBPATH -LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(WP81LIBPATH)" -!ENDIF -LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE -LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib -LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib -!ENDIF - # When compiling for UWP or the Windows 10 platform, some extra linker # options are also required. # @@ -1313,12 +1242,14 @@ LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib # If either debugging or symbols are enabled, enable PDBs. # !IF $(DEBUG)>1 || $(SYMBOLS)!=0 -LDFLAGS = /DEBUG $(LDOPTS) +LDFLAGS = /NODEFAULTLIB:msvcrt /DEBUG $(LDOPTS) !ELSE -LDFLAGS = $(LDOPTS) +LDFLAGS = /NODEFAULTLIB:msvcrt $(LDOPTS) !ENDIF # <> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# # Start with the Tcl related linker options. # !IF $(NO_TCL)==0 @@ -1345,6 +1276,8 @@ LTLIBS = $(LTLIBS) $(LIBICU) ############################################################################### # <> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# # Object files for the SQLite library (non-amalgamation). # LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ @@ -1463,7 +1396,7 @@ SRC01 = \ $(TOP)\src\status.c \ $(TOP)\src\table.c \ $(TOP)\src\threads.c \ - $(TOP)\src\tclsqlite.c \ + tclsqlite-ex.c \ $(TOP)\src\tokenize.c \ $(TOP)\src\treeview.c \ $(TOP)\src\trigger.c \ @@ -1774,6 +1707,7 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_PERCENTILE=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_STRICT_SUBTYPE=1 !ENDIF @@ -1932,8 +1866,8 @@ $(SQLITE3EXE): shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLIT /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) # <> -sqldiff.exe: $(TOP)\tool\sqldiff.c $(TOP)\ext\misc\sqlite3_stdio.h $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) - $(LTLINK) $(NO_WARN) -I$(TOP)\ext\misc $(TOP)\tool\sqldiff.c $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS) +sqldiff.exe: $(TOP)\tool\sqldiff.c $(TOP)\ext\misc\sqlite3_stdio.h $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) $(TOP)\tool\winmain.c + $(LTLINK) $(NO_WARN) -I$(TOP)\ext\misc $(TOP)\tool\sqldiff.c $(TOP)\ext\misc\sqlite3_stdio.c $(TOP)\tool\winmain.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS) dbhash.exe: $(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) @@ -2023,8 +1957,18 @@ sqlite3.c: .target_source sqlite3ext.h sqlite3session.h $(MKSQLITE3C_TOOL) src-v sqlite3-all.c: sqlite3.c $(TOP)\tool\split-sqlite3c.tcl $(JIM_TCLSH) $(JIM_TCLSH) $(TOP)\tool\split-sqlite3c.tcl + +TCLSQLITEEX = \ + $(TOP)\ext\qrf\qrf.h \ + $(TOP)\ext\qrf\qrf.c \ + $(TOP)\src\tclsqlite.c + +tclsqlite-ex.c: $(TCLSQLITEEX) $(TOP)\tool\mkcombo.tcl $(JIM_TCLSH) + $(JIM_TCLSH) $(TOP)\tool\mkcombo.tcl $(TCLSQLITEEX) -o $@ # <> +tclsqlite-ex.c: + # Rule to build the amalgamation # sqlite3.lo: $(SQLITE3C) @@ -2324,11 +2268,11 @@ whereexpr.lo: $(TOP)\src\whereexpr.c $(HDR) window.lo: $(TOP)\src\window.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\window.c -tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP) - $(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c +tclsqlite.lo: tclsqlite-ex.c $(HDR) $(SQLITE_TCL_DEP) + $(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c tclsqlite-ex.c /OUT:tclsqlite.lo -tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP) - $(LTCOMPILE) $(NO_WARN) -DTCLSH -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c +tclsqlite-shell.lo: tclsqlite-ex.c $(HDR) $(SQLITE_TCL_DEP) + $(LTCOMPILE) $(NO_WARN) -DTCLSH -DBUILD_sqlite -I$(TCLINCDIR) -c tclsqlite-ex.c tclsqlite3.exe: tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) $(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS) @@ -2381,6 +2325,8 @@ keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe # Source and header files that shell.c depends on SHELL_DEP = \ $(TOP)\src\shell.c.in \ + $(TOP)\ext\qrf\qrf.c \ + $(TOP)\ext\qrf\qrf.h \ $(TOP)\ext\expert\sqlite3expert.c \ $(TOP)\ext\expert\sqlite3expert.h \ $(TOP)\ext\intck\sqlite3intck.c \ @@ -2544,9 +2490,9 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_STRICT_SUBTYPE=1 TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2) TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C) !IF $(USE_AMALGAMATION)==0 -TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0) +TESTFIXTURE_SRC = $(TESTSRC) tclsqlite-ex.c $(TESTFIXTURE_SRC0) !ELSE -TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1) +TESTFIXTURE_SRC = $(TESTSRC) tclsqlite-ex.c $(TESTFIXTURE_SRC1) !ENDIF !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 @@ -2655,6 +2601,22 @@ devtest: srctree-check sourcetest mdevtest: $(TCLSH_CMD) $(TOP)\test\testrunner.tcl mdevtest +# Rerun test cases that failed on the previous testrunner invocation +# +retest: + $(TCLSH_CMD) $(TOP)\test\testrunner.tcl retest + +# Show all errors from the most reason testrunner invocation +# +errors: + $(TCLSH_CMD) $(TOP)\test\testrunner.tcl errors + +# Show the status of the current testrunner invocation, +# updated every couple of seconds +# +status: + $(TCLSH_CMD) $(TOP)\test\testrunner.tcl status -d 2 + # Validate that various generated files in the source tree # are up-to-date. # @@ -2678,17 +2640,17 @@ smoketest: $(TESTPROGS) @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\main.test $(TESTOPTS) -shelltest: $(TESTPROGS) - .\testfixture.exe $(TOP)\test\permutations.test shell +shelltest: + $(TCLSH_CMD) $(TOP)\test\testrunner.tcl release shell -sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(TOP)\ext\misc\sqlite3_stdio.h $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE_TCL_DEP) +sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) tclsqlite-ex.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(TOP)\ext\misc\sqlite3_stdio.h $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE_TCL_DEP) $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl -DINCLUDE_SQLITE3_C $(TOP)\tool\sqlite3_analyzer.c.in > $@ sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS) $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \ /link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS) -sqltclsh.c: sqlite3.c $(TOP)\src\tclsqlite.c $(TOP)\tool\sqltclsh.tcl $(TOP)\ext\misc\appendvfs.c $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in +sqltclsh.c: sqlite3.c tclsqlite-ex.c $(TOP)\tool\sqltclsh.tcl $(TOP)\ext\misc\appendvfs.c $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in >sqltclsh.c sqltclsh.exe: sqltclsh.c $(SHELL_CORE_DEP) $(LIBRESOBJS) @@ -2698,23 +2660,6 @@ sqltclsh.exe: sqltclsh.c $(SHELL_CORE_DEP) $(LIBRESOBJS) sqlite3_expert.exe: $(SQLITE3C) $(TOP)\ext\expert\sqlite3expert.h $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(LTLINK) $(NO_WARN) $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(SQLITE3C) $(TLIBS) -CHECKER_DEPS =\ - $(TOP)\tool\mkccode.tcl \ - sqlite3.c \ - $(TOP)\src\tclsqlite.c \ - $(TOP)\ext\repair\sqlite3_checker.tcl \ - $(TOP)\ext\repair\checkindex.c \ - $(TOP)\ext\repair\checkfreelist.c \ - $(TOP)\ext\misc\btreeinfo.c \ - $(TOP)\ext\repair\sqlite3_checker.c.in - -sqlite3_checker.c: $(CHECKER_DEPS) - $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\ext\repair\sqlite3_checker.c.in > $@ - -sqlite3_checker.exe: sqlite3_checker.c $(LIBRESOBJS) - $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_checker.c \ - /link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS) - dbdump.exe: $(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) $(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \ /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) @@ -2747,6 +2692,9 @@ showwal.exe: $(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H) showshm.exe: $(TOP)\tool\showshm.c $(LTLINK) $(NO_WARN) $(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS) +showtmlog.exe: $(TOP)\tool\showtmlog.c + $(LTLINK) $(NO_WARN) $(TOP)\tool\showtmlog.c /link $(LDFLAGS) $(LTLINKOPTS) + index_usage.exe: $(TOP)\tool\index_usage.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\tool\index_usage.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) @@ -2938,6 +2886,7 @@ env: @echo USE_STDCALL = $(USE_STDCALL) @echo USE_ZLIB = $(USE_ZLIB) @echo XCOMPILE = $(XCOMPILE) + @echo ZLIBCFLAGS = $(ZLIBCFLAGS) @echo ZLIBDIR = $(ZLIBDIR) @echo ZLIBINCDIR = $(ZLIBINCDIR) @echo ZLIBLIBDIR = $(ZLIBLIBDIR) diff --git a/README.md b/README.md index d44bf5cb7..7ec5042e5 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,9 @@

    SQLite Source Repository

    This repository contains the complete source code for the -[SQLite database engine](https://sqlite.org/), including -many tests. Additional tests and most documentation +[SQLite database engine](https://sqlite.org/) going back +to 2000-05-29. The tree includes many tests and some +documentation, though additional tests and most documentation are managed separately. See the [on-line documentation](https://sqlite.org/) for more information @@ -99,16 +100,16 @@ script found at the root of the source tree. Then run "make". For example: - apt install gcc make tcl-dev ;# Make sure you have all the necessary build tools + apt install gcc make tcl-dev ;# Install the necessary build tools tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite" - mkdir bld ;# Build will occur in a sibling directory + mkdir bld ;# Build happens in a sibling directory cd bld ;# Change to the build directory ../sqlite/configure ;# Run the configure script - make sqlite3 ;# Builds the "sqlite3" command-line tool - make sqlite3.c ;# Build the "amalgamation" source file - make sqldiff ;# Builds the "sqldiff" command-line tool - # Makefile targets below this point require tcl-dev - make tclextension-install ;# Build and install the SQLite TCL extension + make sqlite3 ;# The "sqlite3" command-line tool + make sqlite3.c ;# The "amalgamation" source file + make sqldiff ;# The "sqldiff" command-line tool + #### Targets below require tcl-dev #### + make tclextension-install ;# Install the SQLite TCL extension make devtest ;# Run development tests make releasetest ;# Run full release tests make sqlite3_analyzer ;# Builds the "sqlite3_analyzer" tool @@ -116,14 +117,15 @@ For example: See the makefile for additional targets. For debugging builds, the core developers typically run "configure" with options like this: - ../sqlite/configure --enable-all --enable-debug CFLAGS='-O0 -g' + ../sqlite/configure --all --debug CFLAGS='-O0 -g' For release builds, the core developers usually do: - ../sqlite/configure --enable-all + ../sqlite/configure --all -Almost all makefile targets require a "tclsh" TCL interpreter version 8.6 or -later. The "tclextension-install" target and the test targets that follow +Core deliverables (sqlite3.c, sqlite3) can be built without a TCL, but +many makefile targets require a "tclsh" TCL interpreter version 8.6 +or later. The "tclextension-install" target and the test targets that follow all require TCL development libraries too. ("apt install tcl-dev"). It is helpful, but is not required, to install the SQLite TCL extension (the "tclextension-install" target) prior to running tests. The "releasetest" @@ -133,20 +135,20 @@ On "make" command-lines, one can add "OPTIONS=..." to specify additional compile-time options over and above those set by ./configure. For example, to compile with the SQLITE_OMIT_DEPRECATED compile-time option, one could say: - ./configure --enable-all + ./configure --all make OPTIONS=-DSQLITE_OMIT_DEPRECATED sqlite3 -The configure script uses autoconf 2.61 and libtool. If the configure -script does not work out for you, there is a generic makefile named -"Makefile.linux-gcc" in the top directory of the source tree that you -can copy and edit to suit your needs. Comments on the generic makefile -show what changes are needed. +The configure script uses [autosetup](https://msteveb.github.io/autosetup/). +If the configure script does not work out for you, there is a generic +makefile named "Makefile.linux-gcc" in the top directory of the source tree +that you can copy and edit to suit your needs. Comments on the generic +makefile show what changes are needed. ## Compiling for Windows Using MSVC On Windows, everything can be compiled with MSVC. -You will also need a working installation of TCL if you want to run tests. -TCL is not required if you just want to build SQLite itself. +You will also need a working installation of TCL if you want to run tests, +though TCL is not required if you just want to build SQLite itself. See the [compile-for-windows.md](doc/compile-for-windows.md) document for additional information about how to install MSVC and TCL and configure your build environment. @@ -156,24 +158,25 @@ TCL library, using a command like this: set TCLDIR=c:\Tcl -SQLite uses "tclsh.exe" as part of the build process, and so that -program will need to be somewhere on your %PATH%. SQLite itself -does not contain any TCL code, but it does use TCL to run tests. -You may need to install TCL development -libraries in order to successfully complete some makefile targets. -It is helpful, but is not required, to install the SQLite TCL extension -(the "tclextension-install" target) prior to running tests. - -Build using Makefile.msc. Example: - - nmake /f Makefile.msc sqlite3.exe - nmake /f Makefile.msc sqlite3.c - nmake /f Makefile.msc sqldiff.exe - # Makefile targets below this point require TCL development libraries - nmake /f Makefile.msc tclextension-install - nmake /f Makefile.msc devtest - nmake /f Makefile.msc releasetest - nmake /f Makefile.msc sqlite3_analyzer.exe +SQLite itself does not contain any TCL code, but it does use TCL to run +tests. You may need to install TCL development libraries in order to +successfully complete some makefile targets. It is helpful, but is not +required, to install the SQLite TCL extension (the "tclextension-install" +target) prior to running tests. + +The source tree contains a "make.bat" file that allows the same "make" +commands of Unix to work on Windows. In the following, you can substitute +"nmake /f Makefile.msc" in place of "make", if you prefer to avoid this BAT +file: + + make sqlite3.exe + make sqlite3.c + make sqldiff.exe + #### Targets below require TCL development libraries #### + make tclextension-install + make devtest + make releasetest + make sqlite3_analyzer.exe There are many other makefile targets. See comments in Makefile.msc for details. @@ -181,7 +184,7 @@ details. As with the unix Makefile, the OPTIONS=... argument can be passed on the nmake command-line to enable new compile-time options. For example: - nmake /f Makefile.msc OPTIONS=-DSQLITE_OMIT_DEPRECATED sqlite3.exe + make OPTIONS=-DSQLITE_OMIT_DEPRECATED sqlite3.exe ## Source Tree Map @@ -194,8 +197,7 @@ command-line to enable new compile-time options. For example: * **test/** - This directory and its subdirectories contains code used for testing. Files that end in "`.test`" are TCL scripts that run tests using an augmented TCL interpreter named "testfixture". Use - a command like "`make testfixture`" (unix) or - "`nmake /f Makefile.msc testfixture.exe`" (windows) to build that + a command like "`make testfixture`" to build that augmented TCL interpreter, then run individual tests using commands like "`testfixture test/main.test`". This test/ subdirectory also contains additional C code modules and scripts for other kinds of testing. @@ -377,10 +379,11 @@ implementation. It will not be the easiest library in the world to hack. (and some other test programs too) is built and run when you type "make test". - * **VERSION**, **manifest**, and **manifest.uuid** - These files define - the current SQLite version number. The "VERSION" file is human generated, - but the "manifest" and "manifest.uuid" files are automatically generated - by the [Fossil version control system](https://fossil-scm.org/). + * **VERSION**, **manifest**, **manifest.tags**, and **manifest.uuid** - + These files define the current SQLite version number. The "VERSION" file + is human generated, but the "manifest", "manifest.tags", and + "manifest.uuid" files are automatically generated by the + [Fossil version control system](https://fossil-scm.org/). There are many other source files. Each has a succinct header comment that describes its purpose and role within the larger system. @@ -406,10 +409,6 @@ makefile: > make verify-source -Or on windows: - -> nmake /f Makefile.msc verify-source - Using the makefile to verify source integrity is good for detecting accidental changes to the source tree, but malicious changes could be hidden by also modifying the makefiles. diff --git a/VERSION b/VERSION index 0306a564d..7ac0b0a68 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.51.2 +3.52.0 diff --git a/autoconf/Makefile.in b/autoconf/Makefile.in index c938ffe1b..8e6b358be 100644 --- a/autoconf/Makefile.in +++ b/autoconf/Makefile.in @@ -117,19 +117,6 @@ sqlite_cfg.h: $(AS_AUTO_DEF) # CFLAGS for sqlite3$(T.exe) # SHELL_OPT ?= @OPT_SHELL@ -SHELL_OPT += -DSQLITE_DQS=0 -SHELL_OPT += -DSQLITE_ENABLE_FTS4 -#SHELL_OPT += -DSQLITE_ENABLE_FTS5 -SHELL_OPT += -DSQLITE_ENABLE_RTREE -SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS -SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION -SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB -SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB -SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB -SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB -SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC -SHELL_OPT += -DSQLITE_ENABLE_PERCENTILE -SHELL_OPT += -DSQLITE_STRICT_SUBTYPE=1 # # Library-level feature flags @@ -295,7 +282,7 @@ DIST_FILES := \ README.txt VERSION \ auto.def autosetup configure tea \ sqlite3.h sqlite3.c shell.c sqlite3ext.h \ - Makefile.in Makefile.msc Makefile.fallback \ + Makefile.in Makefile.msc Makefile.fallback make.bat \ sqlite3.rc sqlite3rc.h Replace.cs \ sqlite3.pc.in sqlite3.1 diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index 365e1f0fb..34e41d8aa 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -101,13 +101,6 @@ NO_WARN = $(NO_WARN) -wd4210 -wd4232 -wd4244 -wd4305 -wd4306 -wd4702 -wd4706 !ENDIF !ENDIF -# Set this non-0 to use the library paths and other options necessary for -# Windows Phone 8.1. -# -!IFNDEF USE_WP81_OPTS -USE_WP81_OPTS = 0 -!ENDIF - # Set this non-0 to split the SQLite amalgamation file into chunks to # be used for debugging with Visual Studio. # @@ -156,14 +149,6 @@ USE_NATIVE_LIBPATHS = 0 USE_RC = 1 !ENDIF -# Set this non-0 to compile binaries suitable for the WinRT environment. -# This setting does not apply to any binaries that require Tcl to operate -# properly (i.e. the text fixture, etc). -# -!IFNDEF FOR_WINRT -FOR_WINRT = 0 -!ENDIF - # Set this non-0 to compile binaries suitable for the UWP environment. # This setting does not apply to any binaries that require Tcl to operate # properly (i.e. the text fixture, etc). @@ -563,10 +548,14 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS) !IF "$(PLATFORM)"=="x86" CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall + + !ELSE !IFNDEF PLATFORM CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall + + !ELSE CORE_CCONV_OPTS = SHELL_CCONV_OPTS = @@ -667,18 +656,6 @@ SHELL_LINK_OPTS = $(SHELL_CORE_LIB) TCC = $(TCC) -FAcs !ENDIF -# When compiling the library for use in the WinRT environment, -# the following compile-time options must be used as well to -# disable use of Win32 APIs that are not available and to enable -# use of Win32 APIs that are specific to Windows 8 and/or WinRT. -# -!IF $(FOR_WINRT)!=0 -TCC = $(TCC) -DSQLITE_OS_WINRT=1 -RCC = $(RCC) -DSQLITE_OS_WINRT=1 -TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP -RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP -!ENDIF - # C compiler options for the Windows 10 platform (needs MSVC 2015). # !IF $(FOR_WIN10)!=0 @@ -691,7 +668,7 @@ BCC = $(BCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE # USE_CRT_DLL option is set to force dynamically linking to the # MSVC runtime library. # -!IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0 +!IF $(USE_CRT_DLL)!=0 !IF $(DEBUG)>1 TCC = $(TCC) -MDd BCC = $(BCC) -MDd @@ -714,16 +691,6 @@ ZLIBCFLAGS = -nologo -MT -W3 -O2 -Oy- -Zi !ENDIF -# Define -DNDEBUG to compile without debugging (i.e., for production usage) -# Omitting the define will cause extra debugging code to be inserted and -# includes extra comments when "EXPLAIN stmt" is used. -# -!IF $(DEBUG)==0 -TCC = $(TCC) -DNDEBUG -BCC = $(BCC) -DNDEBUG -RCC = $(RCC) -DNDEBUG -!ENDIF - !IF $(DEBUG)>0 || $(API_ARMOR)!=0 || $(FOR_WIN10)!=0 TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1 RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 @@ -913,56 +880,6 @@ LTLINKOPTS = /NOLOGO LTLIBOPTS = /NOLOGO !ENDIF -# When compiling for use in the WinRT environment, the following -# linker option must be used to mark the executable as runnable -# only in the context of an application container. -# -!IF $(FOR_WINRT)!=0 -LTLINKOPTS = $(LTLINKOPTS) /APPCONTAINER -!IF "$(VISUALSTUDIOVERSION)"=="12.0" || "$(VISUALSTUDIOVERSION)"=="14.0" -!IFNDEF STORELIBPATH -!IF "$(PLATFORM)"=="x86" -STORELIBPATH = $(CRTLIBPATH)\store -!ELSEIF "$(PLATFORM)"=="x64" -STORELIBPATH = $(CRTLIBPATH)\store\amd64 -!ELSEIF "$(PLATFORM)"=="ARM" -STORELIBPATH = $(CRTLIBPATH)\store\arm -!ELSE -STORELIBPATH = $(CRTLIBPATH)\store -!ENDIF -!ENDIF -STORELIBPATH = $(STORELIBPATH:\\=\) -LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(STORELIBPATH)" -!ENDIF -!ENDIF - -# When compiling for Windows Phone 8.1, an extra library path is -# required. -# -!IF $(USE_WP81_OPTS)!=0 -!IFNDEF WP81LIBPATH -!IF "$(PLATFORM)"=="x86" -WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 -!ELSEIF "$(PLATFORM)"=="ARM" -WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\ARM -!ELSE -WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 -!ENDIF -!ENDIF -!ENDIF - -# When compiling for Windows Phone 8.1, some extra linker options -# are also required. -# -!IF $(USE_WP81_OPTS)!=0 -!IFDEF WP81LIBPATH -LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(WP81LIBPATH)" -!ENDIF -LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE -LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib -LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib -!ENDIF - # When compiling for UWP or the Windows 10 platform, some extra linker # options are also required. # @@ -986,9 +903,9 @@ LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib # If either debugging or symbols are enabled, enable PDBs. # !IF $(DEBUG)>1 || $(SYMBOLS)!=0 -LDFLAGS = /DEBUG $(LDOPTS) +LDFLAGS = /NODEFAULTLIB:msvcrt /DEBUG $(LDOPTS) !ELSE -LDFLAGS = $(LDOPTS) +LDFLAGS = /NODEFAULTLIB:msvcrt $(LDOPTS) !ENDIF @@ -1024,6 +941,7 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_PERCENTILE=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_STRICT_SUBTYPE=1 !ENDIF @@ -1072,6 +990,8 @@ $(SQLITE3EXE): shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLIT /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) +tclsqlite-ex.c: + # Rule to build the amalgamation # sqlite3.lo: $(SQLITE3C) diff --git a/autosetup/README.md b/autosetup/README.md index c8da5c643..2560bfab9 100644 --- a/autosetup/README.md +++ b/autosetup/README.md @@ -375,6 +375,11 @@ configure process, and check it in. Patching Autosetup for Project-local Changes ------------------------------------------------------------------------ +The autosetup files require the following patches after updating +from their upstream sources: + +### `--debug` flag + Autosetup reserves the flag name **`--debug`** for its own purposes, and its own special handling of `--enable-...` flags makes `--debug` an alias for `--enable-debug`. As this project has a long history of @@ -388,6 +393,11 @@ If autosetup is upgraded and this patch is _not_ applied the invoking `./configure` will fail loudly because of the declaration of the `debug` flag in `auto.def` - duplicated flags are not permitted. +### Fail on `malloc()` error + +See [check-in 72c8a5b94cdf5d](/info/72c8a5b94cdf5d). + + Branch-specific Customization ======================================================================== diff --git a/autosetup/jimsh0.c b/autosetup/jimsh0.c index 1a6453d0c..0f0a89088 100644 --- a/autosetup/jimsh0.c +++ b/autosetup/jimsh0.c @@ -7409,11 +7409,13 @@ void *JimDefaultAllocator(void *ptr, size_t size) free(ptr); return NULL; } - else if (ptr) { - return realloc(ptr, size); - } else { - return malloc(size); + void *p = realloc(ptr, size); + if( p==0 ){ + fprintf(stderr,"Out of memory\n"); + exit(1); + } + return p; } } diff --git a/autosetup/proj.tcl b/autosetup/proj.tcl index 86f4df44e..caa679ad6 100644 --- a/autosetup/proj.tcl +++ b/autosetup/proj.tcl @@ -1842,7 +1842,7 @@ proc proj-setup-autoreconfig {defName} { } # -# @prop-append-to defineName args... +# @prop-define-append defineName args... # # A proxy for Autosetup's [define-append]. Appends all non-empty $args # to [define-append $defineName]. @@ -1873,7 +1873,7 @@ proc proj-define-append {defineName args} { # but it is technically correct and still relevant on some # environments. # -# See: proj-append-to +# See: proj-define-append # proc proj-define-amend {args} { set defName "" diff --git a/autosetup/sqlite-config.tcl b/autosetup/sqlite-config.tcl index 7c798b31a..fe1b355ab 100644 --- a/autosetup/sqlite-config.tcl +++ b/autosetup/sqlite-config.tcl @@ -557,7 +557,25 @@ proc proc-debug {msg} { } define OPT_FEATURE_FLAGS {} ; # -DSQLITE_OMIT/ENABLE flags. -define OPT_SHELL {} ; # Feature-related CFLAGS for the sqlite3 CLI app +# +# OPT_SHELL = feature-related CFLAGS for the sqlite3 CLI app. The +# list's initial values are defaults which are always applied and not +# affected by --feature-flags. The list is appended to by various +# --feature-flags. +define OPT_SHELL { + -DSQLITE_DQS=0 + -DSQLITE_ENABLE_FTS4 + -DSQLITE_ENABLE_RTREE + -DSQLITE_ENABLE_EXPLAIN_COMMENTS + -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + -DSQLITE_ENABLE_STMTVTAB + -DSQLITE_ENABLE_DBPAGE_VTAB + -DSQLITE_ENABLE_DBSTAT_VTAB + -DSQLITE_ENABLE_BYTECODE_VTAB + -DSQLITE_ENABLE_OFFSET_SQL_FUNC + -DSQLITE_ENABLE_PERCENTILE + -DSQLITE_STRICT_SUBTYPE=1 +} ######################################################################## # Adds $args, if not empty, to OPT_FEATURE_FLAGS. If the first arg is # -shell then it strips that arg and passes the remaining args the @@ -661,6 +679,7 @@ proc sqlite-check-common-system-deps {} { define HAVE_ZLIB 1 define LDFLAGS_ZLIB -lz sqlite-add-shell-opt -DSQLITE_HAVE_ZLIB=1 + sqlite-add-feature-flag -DSQLITE_HAVE_ZLIB=1 } else { define HAVE_ZLIB 0 define LDFLAGS_ZLIB "" @@ -787,7 +806,8 @@ proc sqlite-handle-common-feature-flags {} { sqlite-add-feature-flag -DSQLITE_ENABLE_MEMSYS3 } } - scanstatus -DSQLITE_ENABLE_STMT_SCANSTATUS {} + bytecode-vtab -DSQLITE_ENABLE_BYTECODE_VTAB {} + scanstatus {-DSQLITE_ENABLE_STMT_SCANSTATUS -DSQLITE_ENABLE_BYTECODE_VTAB} {} column-metadata -DSQLITE_ENABLE_COLUMN_METADATA {} dbpage -DSQLITE_ENABLE_DBPAGE_VTAB {} dbstat -DSQLITE_ENABLE_DBSTAT_VTAB {} diff --git a/doc/compile-for-unix.md b/doc/compile-for-unix.md index ce76b97ba..659bdb038 100644 --- a/doc/compile-for-unix.md +++ b/doc/compile-for-unix.md @@ -22,9 +22,10 @@ guidance on building for Windows. or .
  16. Untar the source archive. CD into the "unix/" subfolder of the source tree. -
  17. Run: `mkdir $HOME/local` -
  18. Run: `./configure --prefix=$HOME/local` -
  19. Run: `make install` +
  20. Run:   `mkdir $HOME/local` +
  21. Run:   `./configure --prefix=$HOME/local`
    + SQLite deliverable builds add:   `--static CFLAGS=-Os` +
  22. Run:   `make install`

As of 2024-10-25, TCL is not longer required for many @@ -41,6 +42,12 @@ guidance on building for Windows. You do not need to use --with-tclsh if the tclsh you want to use is the first one on your PATH or if you are building without TCL. + The SQLite developers typically add the +   `--with-linenoise=$HOME/linenoise`   option + to provide command-line editing. "$HOME/linenoise" is a directory + that contains [linenoise](https://github.com/antirez/linenoise) source + code files, `linenoise.c` and `linenoise.h`. + 6. Run the "`Makefile`" makefile with an appropriate target. Examples:

    diff --git a/doc/compile-for-windows.md b/doc/compile-for-windows.md index 0e59c83fe..30536d5fd 100644 --- a/doc/compile-for-windows.md +++ b/doc/compile-for-windows.md @@ -180,3 +180,39 @@ statically linked so that it does not depend on separate DLL: 6. After your executable is built, you can verify that it does not depend on the TCL DLL by running:
    dumpbin /dependents sqlite3_analyzer.exe
    + +## Linking Against ZLIB + +Some feature (such as zip-file support in the CLI) require the ZLIB +compression library. That library is more or less universally available +on unix platforms, but is seldom provided on Windows. You will probably +need to provide it yourself. Here the the steps needed: + + 1. Download the zlib-1.3.1.tar.gz tarball (or a similar version). + Unpack the tarball sources. You can put them wherever you like. + For the purposes of this document, let's assume you put the source + tree in c:\\zlib-64. Note: If you are building for both x64 and + x86, you will need separate builds of ZLIB for each, thus separate + build directories. + + 2. Before building SQLite (as described above) first make these + environment changes. The lead-programmer for SQLite (who writes + these words) has BAT files named "env-x64.bat" and "env-x32.bat" + and "env-arm64.bat" that make these changes, and he runs those + BAT file whenever he starts a new shell. These are the settings + needed: +
    + set USE_ZLIB=1
    + set BUILD_ZLIB=0
    + set ZLIBDIR=c:\\zlib-64 +
    + + 3. Because the settings in step 2 specify "BUILD_ZLIB=0", you will need + to build the library at least once. I recommand: +
    + make clean sqlite3.exe BUILD_ZLIB=1 +
    + + 4. After making the environment changes specified in steps 1 through 3 + above, you then build and test SQLite as you normally would. The + environment variable changes will cause ZLIB to be linked automatically. diff --git a/doc/lemon.html b/doc/lemon.html index 965f305c0..a994b396b 100644 --- a/doc/lemon.html +++ b/doc/lemon.html @@ -696,6 +696,7 @@

    4.4 Special Directives

  • %right
  • %realloc
  • %stack_overflow +
  • %stack_size_limit
  • %stack_size
  • %start_symbol
  • %syntax_error @@ -1203,20 +1204,33 @@

    4.4.25 The %wildcard directive

    The wildcard token is only matched if there are no alternatives.

    -

    4.4.26 The %realloc and %free directives

    +

    4.4.26 The %realloc, %free, and +%stack_size_limit directives

    The %realloc and %free directives defines function -that allocate and free heap memory. The signatures of these functions -should be the same as the realloc() and free() functions from the standard -C library. - -

    If both of these functions are defined -then these functions are used to allocate and free -memory for supplemental parser stack space, if the initial -parse stack space is exceeded. The initial parser stack size +that allocate and free heap memory. The signatures and semantics of +these functions are similar to the realloc() and free() functions from +the standard C library, except that these functions take an extra +parameter at the end that is determined by %extra_context. If +%extra_context is not defined, then the extra argument is 0. The +extra parameter provides the capability to do better error reporting +in the event of a memory allocation error, and/or to use an alternative +private application heap. + +

    If both of these functions are defined then they are used to +allocate and free memory for supplemental parser stack space, if +the initial parse stack space is exceeded. The initial parser stack size is specified by either %stack_size or the -DYYSTACKDEPTH compile-time flag. +

    The %stack_size_limit directive defines a function that returns +the maximum allowed parser stack size. If this diretive does not exist, +no size limit is enforced. This function takes a single argument which +is the %extra_context value or "0" if %extra_context is not defined. +The function should return an integer that is the maximum +number of parser stack entries. If more stack space +than this is needed, the %stack_overflow code is invoked. +

    5.0 Error Processing

    diff --git a/doc/testrunner.md b/doc/testrunner.md index d1696e9d1..90ef4b71f 100644 --- a/doc/testrunner.md +++ b/doc/testrunner.md @@ -4,6 +4,12 @@
    • 1. Overview +
    • 2. Binary Tests
      • 2.1. Organization of Tcl Tests @@ -26,17 +32,40 @@ The testrunner.tcl program is a Tcl script used to run multiple SQLite tests in parallel, thus reducing testing time on multi-core machines. -It supports the following types of tests: +The testrunner.tcl supports running tests that based on `testfixture`, +`sqlite3`, and `fuzzcheck`. + + +## 1.1 Running testrunner.tcl + +The testrunner.tcl script is located in the "test" subdirectory of the +SQLite source tree. So if your shell is current positioned at the top +of the source tree, you would normally run the script using the command: +"test/testrunner.tcl". On Windows, you have to specify the +`tclsh` interpreter command first, like this: +"tclsh test/testrunner.tcl". + +In this document, we will assume that you are on a unix-like OS +(not on Windows) and that your current directory is the root +of the SQLite source tree, and so all invocations of the testrunner.tcl +script will be of the form "test/testrunner.tcl". If you +are in a different directory, then make appropriate adjustments to +the path. On Windows, add the "tclsh" interpreter command +up front. - * Tcl test scripts. + +## 1.2 Run using make - * Fuzzcheck tests, including using an external fuzzcheck database. +The standard Makefiles for SQLite include targets that invoke +testrunner.tcl. So the following commands also run testrunner.tcl: - * Tests run with `make` commands. Examples: - - `make devtest` - - `make releasetest` - - `make sdevtest` - - `make testrunner` + * `make devtest` + * `make releasetest` + * `make sdevtest` + * `make testrunner` + + +## 1.3 Outputs from testrunner.tcl The testrunner.tcl program stores output of all tests and builds run in log file **testrunner.log**, created in the current working directory. @@ -54,17 +83,19 @@ A useful query might be: ``` You can get a summary of errors in a prior run by invoking commands like -these: +those shown below. Note that the testrunner.tcl script can be run directly +on unix systems (including Macs) but you will need to add tclsh +to the front on Windows. ``` - tclsh $(TESTDIR)/testrunner.tcl errors - tclsh $(TESTDIR)/testrunner.tcl errors -v + test/testrunner.tcl errors + test/testrunner.tcl errors -v ``` Running the command: ``` - tclsh $(TESTDIR)/testrunner.tcl status + test/testrunner.tcl status ``` in the directory containing the testrunner.db database runs various queries @@ -73,32 +104,43 @@ A good way to keep and eye on test progress is to run either of the two following commands: ``` - watch tclsh $(TESTDIR)/testrunner.tcl status - tclsh $(TESTDIR)/testrunner.tcl status -d 2 + watch test/testrunner.tcl status + test/testrunner.tcl status -d 2 ``` Both of the commands above accomplish about the same thing, but the second one has the advantage of not requiring "watch" to be installed on your system. -Sometimes testrunner.tcl uses the `testfixture` binary that it is run with -to run tests (see "Binary Tests" below). Sometimes it builds testfixture and +Sometimes testrunner.tcl uses the `testfixture` and `sqlite3` binaries that +are in the directory from which testrunner.tcl is run. +(see "Binary Tests" below). Sometimes it builds testfixture and other binaries in specific configurations to test (see "Source Tests"). + +## 1.4 Built-in help + +Run this command: + +``` + test/testrunner.tcl help +``` + +To get a summary of all of the various command-line options available +with testrunner.tcl + + # 2. Binary Tests The commands described in this section all run various combinations of the Tcl -test scripts using the `testfixture` binary used to run the testrunner.tcl -script (i.e. they do not invoke the compiler to build new binaries, or the -`make` command to run tests that are not Tcl scripts). The procedure to run -these tests is therefore: - - 1. Build the "testfixture" (or "testfixture.exe" for windows) binary using - whatever method seems convenient. - - 2. Test the binary built in step 1 by running testrunner.tcl with it, - perhaps with various options. +test scripts using whatever `testfixture` binary (and maybe also the `sqlite3` +binary, depending on the test) that is found in the directory from which +testrunner.tcl is launched. So typically, one must first run something +like "`make testfixture sqlite3`" before launching binary tests. In other +words, testrunner.tcl does not automatically build the binaries under test +for binary tests. The testrunner.tcl expects the binaries to be available +already. The following sub-sections describe the various options that can be passed to testrunner.tcl to test binary testfixture builds. @@ -140,22 +182,22 @@ are defined in file *testrunner_data.tcl*. To run the "veryquick" test set, use either of the following: ``` - ./testfixture $TESTDIR/testrunner.tcl - ./testfixture $TESTDIR/testrunner.tcl veryquick + test/testrunner.tcl + test/testrunner.tcl veryquick ``` To run the "full" test suite: ``` - ./testfixture $TESTDIR/testrunner.tcl full + test/testrunner.tcl full ``` To run the subset of the "full" test suite for which the test file name matches a specified pattern (e.g. all tests that start with "fts5"), either of: ``` - ./testfixture $TESTDIR/testrunner.tcl fts5% - ./testfixture $TESTDIR/testrunner.tcl 'fts5*' + test/testrunner.tcl fts5% + test/testrunner.tcl 'fts5*' ``` Strictly speaking, for a test to be run the pattern must match the script @@ -167,7 +209,7 @@ characters specified as part of the pattern are transformed to "\*". To run "all" tests (full + permutations): ``` - ./testfixture $TESTDIR/testrunner.tcl all + test/testrunner.tcl all ``` @@ -186,10 +228,15 @@ If there is no permutation, the individual test script may be run with: Or, if the failure occured as part of a permutation: ``` - ./testfixture $TESTDIR/testrunner.tcl $PERMUTATION $PATH_TO_SCRIPT + test/testrunner.tcl $PERMUTATION $PATH_TO_SCRIPT ``` -TODO: An example instead of "$PERMUTATION" and $PATH\_TO\_SCRIPT? +One can also rerun all tests that failed or did not complete +in the previous invocation by typing: + +``` + test/testrunner.tcl retest +``` # 3. Source Code Tests @@ -204,11 +251,9 @@ other tests. The advantages of this are that: * it ensures that tests are always run using binaries created with the same set of compiler options. -The testrunner.tcl commands described in this section may be run using -either a *testfixture* (or testfixture.exe) build, or with any other Tcl -shell that supports SQLite 3.31.1 or newer via "package require sqlite3". - -TODO: ./configure + Makefile.msc build systems. +The testrunner.tcl commands described in this section do not require that +the testfixture and/or sqlite3 binaries be built ahead of time. Those +binaries will be constructed automatically. ## 3.1. Commands to Run SQLite Tests @@ -218,7 +263,7 @@ the `make fuzztest` target once for each of two --enable-all builds - one with debugging enabled and one without: ``` - tclsh $TESTDIR/testrunner.tcl mdevtest + test/testrunner.tcl mdevtest ``` In other words, it is equivalent to running: @@ -227,13 +272,13 @@ In other words, it is equivalent to running: $TOP/configure --enable-all --enable-debug make fuzztest make testfixture - ./testfixture $TOP/test/testrunner.tcl veryquick + $TOP/test/testrunner.tcl veryquick # Then, after removing files created by the tests above: $TOP/configure --enable-all OPTS="-O0" make fuzztest make testfixture - ./testfixture $TOP/test/testrunner.tcl veryquick + $TOP/test/testrunner.tcl veryquick ``` The **sdevtest** command is identical to the mdevtest command, except that the @@ -241,7 +286,7 @@ second of the two builds is a sanitizer build. Specifically, this means that OPTS="-fsanitize=address,undefined" is specified instead of OPTS="-O0": ``` - tclsh $TESTDIR/testrunner.tcl sdevtest + test/testrunner.tcl sdevtest ``` The **release** command runs lots of tests under lots of builds. It runs @@ -250,7 +295,7 @@ on Linux, Windows or OSX. Refer to *testrunner\_data.tcl* for the details of the specific tests run. ``` - tclsh $TESTDIR/testrunner.tcl release + test/testrunner.tcl release ``` As with source code tests, one or more patterns @@ -258,7 +303,7 @@ may be appended to any of the above commands (mdevtest, sdevtest or release). Pattern matching is used for both Tcl tests and fuzz tests. ``` - tclsh $TESTDIR/testrunner.tcl release rtree% + test/testrunner.tcl release rtree% ``` @@ -268,14 +313,14 @@ testrunner.tcl can build a zipvfs-enabled testfixture and use it to run tests from the Zipvfs project with the following command: ``` - tclsh $TESTDIR/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS + test/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS ``` This can be combined with any of "mdevtest", "sdevtest" or "release" to test both SQLite and Zipvfs with a single command: ``` - tclsh $TESTDIR/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS mdevtest + test/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS mdevtest ``` @@ -295,10 +340,10 @@ a dos \*.bat file on windows. For example: ``` # Create a script that recreates build configuration "Device-One" on # Linux or OSX: - tclsh $TESTDIR/testrunner.tcl script Device-One > make.sh + test/testrunner.tcl script Device-One > make.sh # Create a script that recreates build configuration "Have-Not" on Windows: - tclsh $TESTDIR/testrunner.tcl script Have-Not > make.bat + test/testrunner.tcl script Have-Not > make.bat ``` The generated bash or \*.bat file script accepts a single argument - a makefile @@ -321,7 +366,7 @@ Thus, for example, to run a full releasetest including an external dbsqlfuzz database, run a command like one of these: ``` - tclsh test/testrunner.tcl releasetest --fuzzdb ../fuzz/20250415.db + test/testrunner.tcl releasetest --fuzzdb ../fuzz/20250415.db FUZZDB=../fuzz/20250415.db make releasetest nmake /f Makefile.msc FUZZDB=../fuzz/20250415.db releasetest ``` @@ -331,7 +376,7 @@ databases. So if you want to run *only* tests involving the external database, you can use a command something like this: ``` - tclsh test/testrunner.tcl releasetest 20250415 --fuzzdb ../fuzz/20250415.db + test/testrunner.tcl releasetest 20250415 --fuzzdb ../fuzz/20250415.db ``` @@ -345,7 +390,7 @@ required by a test, not to run any actual tests. For example: ``` # Build binaries required by release test. - tclsh $TESTDIR/testrunner.tcl --buildonly release" + test/testrunner.tcl --buildonly release" ``` The **--dryrun** option prevents testrunner.tcl from building any binaries @@ -354,7 +399,7 @@ would normally execute into the testrunner.log file. Example: ``` # Log the shell commmands that make up the mdevtest test. - tclsh $TESTDIR/testrunner.tcl --dryrun mdevtest" + test/testrunner.tcl --dryrun mdevtest" ``` The **--explain** option is similar to --dryrun in that it prevents @@ -364,7 +409,7 @@ summary of all the builds and tests that would have been run. ``` # Show what builds and tests would have been run - tclsh $TESTDIR/testrunner.tcl --explain mdevtest + test/testrunner.tcl --explain mdevtest ``` The **--status** option uses VT100 escape sequences to display the test @@ -380,7 +425,7 @@ When running either binary or source code tests, testrunner.tcl reports the number of jobs it intends to use to stdout. e.g. ``` - $ ./testfixture $TESTDIR/testrunner.tcl + $ test/testrunner.tcl splitting work across 16 jobs ... more output ... ``` @@ -390,7 +435,7 @@ of real cores on the machine. This can be overridden using the "--jobs" (or -j) switch: ``` - $ ./testfixture $TESTDIR/testrunner.tcl --jobs 8 + $ test/testrunner.tcl --jobs 8 splitting work across 8 jobs ... more output ... ``` @@ -400,5 +445,5 @@ running by exucuting the following command from the directory containing the testrunner.log and testrunner.db files: ``` - $ ./testfixture $TESTDIR/testrunner.tcl njob $NEW_NUMBER_OF_JOBS + $ test/testrunner.tcl njob $NEW_NUMBER_OF_JOBS ``` diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test index 0c3b512af..aaea03711 100644 --- a/ext/expert/expert1.test +++ b/ext/expert/expert1.test @@ -90,7 +90,7 @@ foreach {tn setup} { proc do_rec_test {tn sql res} { set res [squish [string trim $res]] set tst [subst -nocommands { - squish [string trim [exec $::CLI test.db ".expert" {$sql;}]] + squish [string trim [exec $::CLI -noinit test.db ".expert" {$sql;}]] }] uplevel [list do_test $tn $tst $res] } diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index f178abafe..368e9b189 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1816,9 +1816,7 @@ static int fts3CursorSeekStmt(Fts3Cursor *pCsr){ zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); if( !zSql ) return SQLITE_NOMEM; p->bLock++; - rc = sqlite3_prepare_v3( - p->db, zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0 - ); + rc = sqlite3Fts3PrepareStmt(p, zSql, 1, 1, &pCsr->pStmt); p->bLock--; sqlite3_free(zSql); } @@ -3393,9 +3391,7 @@ static int fts3FilterMethod( } if( zSql ){ p->bLock++; - rc = sqlite3_prepare_v3( - p->db,zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0 - ); + rc = sqlite3Fts3PrepareStmt(p, zSql, 1, 1, &pCsr->pStmt); p->bLock--; sqlite3_free(zSql); }else{ @@ -4018,6 +4014,7 @@ static int fts3IntegrityMethod( UNUSED_PARAMETER(isQuick); rc = sqlite3Fts3IntegrityCheck(p, &bOk); + assert( pVtab->zErrMsg==0 || rc!=SQLITE_OK ); assert( rc!=SQLITE_CORRUPT_VTAB ); if( rc==SQLITE_ERROR || (rc&0xFF)==SQLITE_CORRUPT ){ *pzErr = sqlite3_mprintf("unable to validate the inverted index for" diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index e98b90a75..556635def 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -601,6 +601,15 @@ int sqlite3Fts3Incrmerge(Fts3Table*,int,int); (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ ) +int sqlite3Fts3PrepareStmt( + Fts3Table *p, /* Prepare for this connection */ + const char *zSql, /* SQL to prepare */ + int bPersist, /* True to set SQLITE_PREPARE_PERSISTENT */ + int bAllowVtab, /* True to omit SQLITE_PREPARE_NO_VTAB */ + sqlite3_stmt **pp /* OUT: Prepared statement */ +); + + /* fts3.c */ void sqlite3Fts3ErrMsg(char**,const char*,...); int sqlite3Fts3PutVarint(char *, sqlite3_int64); diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 19dff31f0..1b8bca70f 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -98,9 +98,9 @@ typedef struct SegmentWriter SegmentWriter; ** incrementally. See function fts3PendingListAppend() for details. */ struct PendingList { - int nData; + sqlite3_int64 nData; char *aData; - int nSpace; + sqlite3_int64 nSpace; sqlite3_int64 iLastDocid; sqlite3_int64 iLastCol; sqlite3_int64 iLastPos; @@ -273,6 +273,24 @@ struct SegmentNode { #define SQL_UPDATE_LEVEL_IDX 38 #define SQL_UPDATE_LEVEL 39 +/* +** Wrapper around sqlite3_prepare_v3() to ensure that SQLITE_PREPARE_FROM_DDL +** is always set. +*/ +int sqlite3Fts3PrepareStmt( + Fts3Table *p, /* Prepare for this connection */ + const char *zSql, /* SQL to prepare */ + int bPersist, /* True to set SQLITE_PREPARE_PERSISTENT */ + int bAllowVtab, /* True to omit SQLITE_PREPARE_NO_VTAB */ + sqlite3_stmt **pp /* OUT: Prepared statement */ +){ + int f = SQLITE_PREPARE_FROM_DDL + |((bAllowVtab==0) ? SQLITE_PREPARE_NO_VTAB : 0) + |(bPersist ? SQLITE_PREPARE_PERSISTENT : 0); + + return sqlite3_prepare_v3(p->db, zSql, -1, f, pp, NULL); +} + /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, @@ -398,12 +416,12 @@ static int fts3SqlStmt( pStmt = p->aStmt[eStmt]; if( !pStmt ){ - int f = SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_NO_VTAB; + int bAllowVtab = 0; char *zSql; if( eStmt==SQL_CONTENT_INSERT ){ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ - f &= ~SQLITE_PREPARE_NO_VTAB; + bAllowVtab = 1; zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist); }else{ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); @@ -411,7 +429,7 @@ static int fts3SqlStmt( if( !zSql ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3_prepare_v3(p->db, zSql, -1, f, &pStmt, NULL); + rc = sqlite3Fts3PrepareStmt(p, zSql, 1, bAllowVtab, &pStmt); sqlite3_free(zSql); assert( rc==SQLITE_OK || pStmt==0 ); p->aStmt[eStmt] = pStmt; @@ -760,7 +778,9 @@ static int fts3PendingTermsAddOne( pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); if( pList ){ - p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); + assert( (i64)pList->nData+(i64)nToken+(i64)sizeof(Fts3HashElem) + <= (i64)p->nPendingData ); + p->nPendingData -= (int)(pList->nData + nToken + sizeof(Fts3HashElem)); } if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ @@ -773,7 +793,9 @@ static int fts3PendingTermsAddOne( } } if( rc==SQLITE_OK ){ - p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + assert( (i64)p->nPendingData + pList->nData + nToken + + sizeof(Fts3HashElem) <= 0x3fffffff ); + p->nPendingData += (int)(pList->nData + nToken + sizeof(Fts3HashElem)); } return rc; } @@ -3574,7 +3596,7 @@ static int fts3DoRebuild(Fts3Table *p){ if( !zSql ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + rc = sqlite3Fts3PrepareStmt(p, zSql, 0, 1, &pStmt); sqlite3_free(zSql); } @@ -5327,7 +5349,7 @@ int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){ if( !zSql ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + rc = sqlite3Fts3PrepareStmt(p, zSql, 0, 1, &pStmt); sqlite3_free(zSql); } @@ -5457,7 +5479,7 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ v = atoi(&zVal[9]); if( v>=24 && v<=p->nPgsz-35 ) p->nNodeSize = v; rc = SQLITE_OK; - }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ + }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 11) ){ v = atoi(&zVal[11]); if( v>=64 && v<=FTS3_MAX_PENDING_DATA ) p->nMaxPendingData = v; rc = SQLITE_OK; diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index 95b33ea31..ee43ca6cc 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -455,7 +455,7 @@ static void fts5SnippetFunction( iBestCol = (iCol>=0 ? iCol : 0); nPhrase = pApi->xPhraseCount(pFts); - aSeen = sqlite3_malloc(nPhrase); + aSeen = sqlite3_malloc64(nPhrase); if( aSeen==0 ){ rc = SQLITE_NOMEM; } diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c index afcd83b6b..d799e34cb 100644 --- a/ext/fts5/fts5_buffer.c +++ b/ext/fts5/fts5_buffer.c @@ -288,7 +288,7 @@ char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){ if( nIn<0 ){ nIn = (int)strlen(pIn); } - zRet = (char*)sqlite3_malloc(nIn+1); + zRet = (char*)sqlite3_malloc64((i64)nIn+1); if( zRet ){ memcpy(zRet, pIn, nIn); zRet[nIn] = '\0'; diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index eea82b046..cea14b500 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -576,7 +576,7 @@ int sqlite3Fts5ConfigParse( sqlite3_int64 nByte; int bUnindexed = 0; /* True if there are one or more UNINDEXED */ - *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config)); + *ppOut = pRet = (Fts5Config*)sqlite3_malloc64(sizeof(Fts5Config)); if( pRet==0 ) return SQLITE_NOMEM; memset(pRet, 0, sizeof(Fts5Config)); pRet->pGlobal = pGlobal; @@ -1123,5 +1123,3 @@ void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){ va_end(ap); } - - diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 352df81f4..8ecaca34f 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -314,7 +314,7 @@ int sqlite3Fts5ExprNew( assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 ); if( sParse.rc==SQLITE_OK ){ - *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr)); + *ppNew = pNew = sqlite3_malloc64(sizeof(Fts5Expr)); if( pNew==0 ){ sParse.rc = SQLITE_NOMEM; sqlite3Fts5ParseNodeFree(sParse.pExpr); @@ -466,7 +466,7 @@ int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){ p2->pRoot = 0; if( sParse.rc==SQLITE_OK ){ - Fts5ExprPhrase **ap = (Fts5ExprPhrase**)sqlite3_realloc( + Fts5ExprPhrase **ap = (Fts5ExprPhrase**)sqlite3_realloc64( p1->apExprPhrase, nPhrase * sizeof(Fts5ExprPhrase*) ); if( ap==0 ){ diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c index a33dec9a9..ba4a030b7 100644 --- a/ext/fts5/fts5_hash.c +++ b/ext/fts5/fts5_hash.c @@ -91,7 +91,7 @@ int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){ int rc = SQLITE_OK; Fts5Hash *pNew; - *ppNew = pNew = (Fts5Hash*)sqlite3_malloc(sizeof(Fts5Hash)); + *ppNew = pNew = (Fts5Hash*)sqlite3_malloc64(sizeof(Fts5Hash)); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index acd0570a5..164d61388 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -2093,7 +2093,7 @@ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ /* If necessary, grow the pIter->aRowidOffset[] array. */ if( iRowidOffset>=pIter->nRowidOffset ){ - int nNew = pIter->nRowidOffset + 8; + i64 nNew = pIter->nRowidOffset + 8; int *aNew = (int*)sqlite3_realloc64(pIter->aRowidOffset,nNew*sizeof(int)); if( aNew==0 ){ p->rc = SQLITE_NOMEM; @@ -5240,7 +5240,7 @@ static void fts5DoSecureDelete( int iSegid = pSeg->pSeg->iSegid; u8 *aPg = pSeg->pLeaf->p; int nPg = pSeg->pLeaf->nn; - int iPgIdx = pSeg->pLeaf->szLeaf; + int iPgIdx = pSeg->pLeaf->szLeaf; /* Offset of page footer */ u64 iDelta = 0; int iNextOff = 0; @@ -5319,7 +5319,7 @@ static void fts5DoSecureDelete( iSOP += fts5GetVarint32(&aPg[iSOP], nPos); } assert_nc( iSOP==pSeg->iLeafOffset ); - iNextOff = pSeg->iLeafOffset + pSeg->nPos; + iNextOff = iSOP + pSeg->nPos; } } @@ -5399,31 +5399,31 @@ static void fts5DoSecureDelete( ** is another term following it on this page. So the subsequent term ** needs to be moved to replace the term associated with the entry ** being removed. */ - int nPrefix = 0; - int nSuffix = 0; - int nPrefix2 = 0; - int nSuffix2 = 0; + u64 nPrefix = 0; + u64 nSuffix = 0; + u64 nPrefix2 = 0; + u64 nSuffix2 = 0; iDelKeyOff = iNextOff; - iNextOff += fts5GetVarint32(&aPg[iNextOff], nPrefix2); - iNextOff += fts5GetVarint32(&aPg[iNextOff], nSuffix2); + iNextOff += fts5GetVarint(&aPg[iNextOff], &nPrefix2); + iNextOff += fts5GetVarint(&aPg[iNextOff], &nSuffix2); if( iKey!=1 ){ - iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nPrefix); + iKeyOff += fts5GetVarint(&aPg[iKeyOff], &nPrefix); } - iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nSuffix); + iKeyOff += fts5GetVarint(&aPg[iKeyOff], &nSuffix); nPrefix = MIN(nPrefix, nPrefix2); nSuffix = (nPrefix2 + nSuffix2) - nPrefix; - if( (iKeyOff+nSuffix)>iPgIdx || (iNextOff+nSuffix2)>iPgIdx ){ + if( (iKeyOff+nSuffix)>(u64)iPgIdx || (iNextOff+nSuffix2)>(u64)iPgIdx ){ FTS5_CORRUPT_IDX(p); }else{ if( iKey!=1 ){ iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix); } iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix); - if( nPrefix2>pSeg->term.n ){ + if( nPrefix2>(u64)pSeg->term.n ){ FTS5_CORRUPT_IDX(p); }else if( nPrefix2>nPrefix ){ memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix); @@ -5454,7 +5454,7 @@ static void fts5DoSecureDelete( u8 *aTermIdx = &pTerm->p[pTerm->szLeaf]; int nTermIdx = pTerm->nn - pTerm->szLeaf; int iTermIdx = 0; - int iTermOff = 0; + i64 iTermOff = 0; while( 1 ){ u32 iVal = 0; @@ -5465,12 +5465,15 @@ static void fts5DoSecureDelete( } nTermIdx = iTermIdx; - memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); - fts5PutU16(&pTerm->p[2], iTermOff); - - fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); - if( nTermIdx==0 ){ - fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); + if( iTermOff>pTerm->szLeaf ){ + FTS5_CORRUPT_IDX(p); + }else{ + memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); + fts5PutU16(&pTerm->p[2], iTermOff); + fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); + if( nTermIdx==0 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); + } } } fts5DataRelease(pTerm); @@ -5493,7 +5496,9 @@ static void fts5DoSecureDelete( int iPrevKeyOut = 0; int iKeyIn = 0; - memmove(&aPg[iOff], &aPg[iNextOff], nMove); + if( nMove>0 ){ + memmove(&aPg[iOff], &aPg[iNextOff], nMove); + } iPgIdx -= nShift; nPg = iPgIdx; fts5PutU16(&aPg[2], iPgIdx); @@ -6413,16 +6418,16 @@ struct Fts5TokenDataMap { ** aMap[] variables. */ struct Fts5TokenDataIter { - int nMapAlloc; /* Allocated size of aMap[] in entries */ - int nMap; /* Number of valid entries in aMap[] */ + i64 nMapAlloc; /* Allocated size of aMap[] in entries */ + i64 nMap; /* Number of valid entries in aMap[] */ Fts5TokenDataMap *aMap; /* Array of (rowid+pos -> token) mappings */ /* The following are used for prefix-queries only. */ Fts5Buffer terms; /* The following are used for other full-token tokendata queries only. */ - int nIter; - int nIterAlloc; + i64 nIter; + i64 nIterAlloc; Fts5PoslistReader *aPoslistReader; int *aPoslistToIter; Fts5Iter *apIter[FLEXARRAY]; @@ -6478,11 +6483,11 @@ static void fts5TokendataIterAppendMap( ){ if( p->rc==SQLITE_OK ){ if( pT->nMap==pT->nMapAlloc ){ - int nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64; - int nAlloc = nNew * sizeof(Fts5TokenDataMap); + i64 nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64; + i64 nAlloc = nNew * sizeof(Fts5TokenDataMap); Fts5TokenDataMap *aNew; - aNew = (Fts5TokenDataMap*)sqlite3_realloc(pT->aMap, nAlloc); + aNew = (Fts5TokenDataMap*)sqlite3_realloc64(pT->aMap, nAlloc); if( aNew==0 ){ p->rc = SQLITE_NOMEM; return; @@ -6508,7 +6513,7 @@ static void fts5TokendataIterAppendMap( */ static void fts5TokendataIterSortMap(Fts5Index *p, Fts5TokenDataIter *pT){ Fts5TokenDataMap *aTmp = 0; - int nByte = pT->nMap * sizeof(Fts5TokenDataMap); + i64 nByte = pT->nMap * sizeof(Fts5TokenDataMap); aTmp = (Fts5TokenDataMap*)sqlite3Fts5MallocZero(&p->rc, nByte); if( aTmp ){ @@ -7042,9 +7047,10 @@ static Fts5TokenDataIter *fts5AppendTokendataIter( if( p->rc==SQLITE_OK ){ if( pIn==0 || pIn->nIter==pIn->nIterAlloc ){ - int nAlloc = pIn ? pIn->nIterAlloc*2 : 16; - int nByte = SZ_FTS5TOKENDATAITER(nAlloc+1); - Fts5TokenDataIter *pNew = (Fts5TokenDataIter*)sqlite3_realloc(pIn, nByte); + i64 nAlloc = pIn ? pIn->nIterAlloc*2 : 16; + i64 nByte = SZ_FTS5TOKENDATAITER(nAlloc+1); + Fts5TokenDataIter *pNew; + pNew = (Fts5TokenDataIter*)sqlite3_realloc64(pIn, nByte); if( pNew==0 ){ p->rc = SQLITE_NOMEM; @@ -7141,8 +7147,8 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){ /* Ensure the token-mapping is large enough */ if( eDetail==FTS5_DETAIL_FULL && pT->nMapAlloc<(pT->nMap + nByte) ){ - int nNew = (pT->nMapAlloc + nByte) * 2; - Fts5TokenDataMap *aNew = (Fts5TokenDataMap*)sqlite3_realloc( + i64 nNew = (pT->nMapAlloc + nByte) * 2; + Fts5TokenDataMap *aNew = (Fts5TokenDataMap*)sqlite3_realloc64( pT->aMap, nNew*sizeof(Fts5TokenDataMap) ); if( aNew==0 ){ diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index f45b9ef90..cf033ab5d 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -631,7 +631,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ return SQLITE_ERROR; } - idxStr = (char*)sqlite3_malloc(pInfo->nConstraint * 8 + 1); + idxStr = (char*)sqlite3_malloc64((i64)pInfo->nConstraint * 8 + 1); if( idxStr==0 ) return SQLITE_NOMEM; pInfo->idxStr = idxStr; pInfo->needToFreeIdxStr = 1; @@ -2081,6 +2081,7 @@ static int fts5UpdateMethod( } update_out: + sqlite3Fts5IndexCloseReader(pTab->p.pIndex); pTab->p.pConfig->pzErrmsg = 0; return rc; } @@ -3762,7 +3763,7 @@ static int fts5Init(sqlite3 *db){ int rc; Fts5Global *pGlobal = 0; - pGlobal = (Fts5Global*)sqlite3_malloc(sizeof(Fts5Global)); + pGlobal = (Fts5Global*)sqlite3_malloc64(sizeof(Fts5Global)); if( pGlobal==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index 25cd5c063..f5d8705ff 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -391,7 +391,7 @@ static int SQLITE_TCLAPI xF5tApi( break; } CASE(12, "xSetAuxdata") { - F5tAuxData *pData = (F5tAuxData*)sqlite3_malloc(sizeof(F5tAuxData)); + F5tAuxData *pData = (F5tAuxData*)sqlite3_malloc64(sizeof(F5tAuxData)); if( pData==0 ){ Tcl_AppendResult(interp, "out of memory", (char*)0); return TCL_ERROR; @@ -780,7 +780,7 @@ static int SQLITE_TCLAPI f5tTokenize( } if( nText>0 ){ - pCopy = sqlite3_malloc(nText); + pCopy = sqlite3_malloc64(nText); if( pCopy==0 ){ tokenizer.xDelete(pTok); Tcl_AppendResult(interp, "error in sqlite3_malloc()", (char*)0); @@ -1420,7 +1420,7 @@ static int f5tOrigintextCreate( void *pTokCtx = 0; int rc = SQLITE_OK; - pTok = (OriginTextTokenizer*)sqlite3_malloc(sizeof(OriginTextTokenizer)); + pTok = (OriginTextTokenizer*)sqlite3_malloc64(sizeof(OriginTextTokenizer)); if( pTok==0 ){ rc = SQLITE_NOMEM; }else if( nArg<1 ){ @@ -1480,7 +1480,7 @@ static int xOriginToken( int nReq = nToken + 1 + (iEnd-iStart); if( nReq>p->nBuf ){ sqlite3_free(p->aBuf); - p->aBuf = sqlite3_malloc(nReq*2); + p->aBuf = sqlite3_malloc64(nReq*2); if( p->aBuf==0 ) return SQLITE_NOMEM; p->nBuf = nReq*2; } diff --git a/ext/fts5/fts5_test_tok.c b/ext/fts5/fts5_test_tok.c index 994d304dc..c77c49de7 100644 --- a/ext/fts5/fts5_test_tok.c +++ b/ext/fts5/fts5_test_tok.c @@ -194,7 +194,7 @@ static int fts5tokConnectMethod( } if( rc==SQLITE_OK ){ - pTab = (Fts5tokTable*)sqlite3_malloc(sizeof(Fts5tokTable)); + pTab = (Fts5tokTable*)sqlite3_malloc64(sizeof(Fts5tokTable)); if( pTab==0 ){ rc = SQLITE_NOMEM; }else{ @@ -275,7 +275,7 @@ static int fts5tokBestIndexMethod( static int fts5tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ Fts5tokCursor *pCsr; - pCsr = (Fts5tokCursor *)sqlite3_malloc(sizeof(Fts5tokCursor)); + pCsr = (Fts5tokCursor *)sqlite3_malloc64(sizeof(Fts5tokCursor)); if( pCsr==0 ){ return SQLITE_NOMEM; } @@ -347,7 +347,7 @@ static int fts5tokCb( if( pCsr->nRow ){ pRow->iPos = pRow[-1].iPos + ((tflags & FTS5_TOKEN_COLOCATED) ? 0 : 1); } - pRow->zToken = sqlite3_malloc(nToken+1); + pRow->zToken = sqlite3_malloc64((sqlite3_int64)nToken+1); if( pRow->zToken==0 ) return SQLITE_NOMEM; memcpy(pRow->zToken, pToken, nToken); pRow->zToken[nToken] = 0; @@ -373,8 +373,8 @@ static int fts5tokFilterMethod( fts5tokResetCursor(pCsr); if( idxNum==1 ){ const char *zByte = (const char *)sqlite3_value_text(apVal[0]); - int nByte = sqlite3_value_bytes(apVal[0]); - pCsr->zInput = sqlite3_malloc(nByte+1); + sqlite3_int64 nByte = sqlite3_value_bytes(apVal[0]); + pCsr->zInput = sqlite3_malloc64(nByte+1); if( pCsr->zInput==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c index b8a113646..990810239 100644 --- a/ext/fts5/fts5_tokenize.c +++ b/ext/fts5/fts5_tokenize.c @@ -72,7 +72,7 @@ static int fts5AsciiCreate( if( nArg%2 ){ rc = SQLITE_ERROR; }else{ - p = sqlite3_malloc(sizeof(AsciiTokenizer)); + p = sqlite3_malloc64(sizeof(AsciiTokenizer)); if( p==0 ){ rc = SQLITE_NOMEM; }else{ @@ -367,7 +367,7 @@ static int fts5UnicodeCreate( if( nArg%2 ){ rc = SQLITE_ERROR; }else{ - p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer)); + p = (Unicode61Tokenizer*)sqlite3_malloc64(sizeof(Unicode61Tokenizer)); if( p ){ const char *zCat = "L* N* Co"; int i; @@ -590,7 +590,7 @@ static int fts5PorterCreate( zBase = azArg[0]; } - pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer)); + pRet = (PorterTokenizer*)sqlite3_malloc64(sizeof(PorterTokenizer)); if( pRet ){ memset(pRet, 0, sizeof(PorterTokenizer)); rc = pApi->xFindTokenizer_v2(pApi, zBase, &pUserdata, &pV2); @@ -1297,7 +1297,7 @@ static int fts5TriCreate( rc = SQLITE_ERROR; }else{ int i; - pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew)); + pNew = (TrigramTokenizer*)sqlite3_malloc64(sizeof(*pNew)); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c index 3a6a968f7..295ace6ba 100644 --- a/ext/fts5/fts5_vocab.c +++ b/ext/fts5/fts5_vocab.c @@ -666,7 +666,7 @@ static int fts5VocabFilterMethod( const char *zCopy = (const char *)sqlite3_value_text(pLe); if( zCopy==0 ) zCopy = ""; pCsr->nLeTerm = sqlite3_value_bytes(pLe); - pCsr->zLeTerm = sqlite3_malloc(pCsr->nLeTerm+1); + pCsr->zLeTerm = sqlite3_malloc64((i64)pCsr->nLeTerm+1); if( pCsr->zLeTerm==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/ext/fts5/test/fts5corrupt9.test b/ext/fts5/test/fts5corrupt9.test new file mode 100644 index 000000000..6cf06f836 --- /dev/null +++ b/ext/fts5/test/fts5corrupt9.test @@ -0,0 +1,129 @@ +# 2026 Jan 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5corrupt9 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +sqlite3_fts5_may_be_corrupt 1 + +sqlite3 db test.db + +set nrows 50 +set repeat 500 +set text [string trim [string repeat "aaa " $repeat]] + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t USING fts5(content); + INSERT INTO t(t, rank) VALUES('secure-delete', 1); +} +do_test 1.1 { + for {set i 0} {$i < $nrows} {incr i} { + db eval "INSERT INTO t(content) VALUES('$text')" + } + db eval "INSERT INTO t(t) VALUES('optimize')" +} {} + +do_test 1.2 { + db eval { SELECT segid, pgno FROM t_idx } {} + set rowid [expr {($segid << 37) + ($pgno >> 1)}] + db eval { + UPDATE t_data + SET block = X'00000009043061616104ffffffff07' + WHERE rowid=$rowid + } +} {} + +# At one point this would segfault due to OOB write. +# +do_catchsql_test 1.3 { + DELETE FROM t WHERE rowid=3 +} {0 {}} + +#------------------------------------------------------------------------- +reset_db + +set nRow 8000 +set zText aaa + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t USING fts5(content, detail=none); + INSERT INTO t(t, rank) VALUES('secure-delete', 1); + BEGIN; +} +do_test 2.1 { + for {set ii 0} {$ii<$nRow} {incr ii} { + execsql { INSERT INTO t(content) VALUES($zText); } + } + execsql { + COMMIT; + INSERT INTO t(t) VALUES('optimize'); + } +} {} + +set hex "00040f7d9f49[string repeat {01} 3958]80" + +do_execsql_test 2.1 " + UPDATE t_data SET block = X'$hex' WHERE rowid=137438953474; +" + +do_execsql_test 2.3 { + DELETE FROM t WHERE rowid=7999 +} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t USING fts5(content, detail=none); + INSERT INTO t(t, rank) VALUES('secure-delete', 1); + INSERT INTO t(content) VALUES('aaa'); + INSERT INTO t(content) VALUES('bbb'); + INSERT INTO t(t) VALUES('optimize'); +} + +do_execsql_test 3.1 { + UPDATE t_data SET block = X'000000100430616161010187ffffff7f0406' WHERE rowid=412316860417; +} + +do_catchsql_test 3.2 { + DELETE FROM t WHERE rowid=1; +} {1 {fts5: corruption in table "t"}} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t USING fts5(content); + INSERT INTO t(t, rank) VALUES('secure-delete', 1); + INSERT INTO t(content) VALUES('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'); + INSERT INTO t(t) VALUES('optimize'); +} + +do_execsql_test 4.1 { + UPDATE t_data SET block = X'00000fce9f4830616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161610487ffffff7f' WHERE rowid=137438953473; + UPDATE t_data SET block = X'0004000801040203' WHERE rowid=137438953474; +} + +do_catchsql_test 4.2 { + DELETE FROM t WHERE rowid = 1; +} {1 {fts5: corruption in table "t"}} + +sqlite3_fts5_may_be_corrupt 0 + +finish_test + diff --git a/ext/fts5/test/fts5integrity.test b/ext/fts5/test/fts5integrity.test index 4bf120c44..9b2720faf 100644 --- a/ext/fts5/test/fts5integrity.test +++ b/ext/fts5/test/fts5integrity.test @@ -379,9 +379,6 @@ do_execsql_test 12.2 { db close sqlite3 db test.db -readonly 1 -explain_i { - PRAGMA integrity_check - } do_execsql_test 12.3 { PRAGMA integrity_check } {ok} diff --git a/ext/fts5/test/fts5interrupt.test b/ext/fts5/test/fts5interrupt.test index 67ef5f7e9..87b232b05 100644 --- a/ext/fts5/test/fts5interrupt.test +++ b/ext/fts5/test/fts5interrupt.test @@ -64,4 +64,21 @@ foreach {tn sql} { } } +#------------------------------------------------------------------------- +# Verify that https://sqlite.org/forum/forumpost/95413eb410 has been +# fixed. +# +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE f1 USING fts5(x); + BEGIN TRANSACTION; + INSERT INTO f1(x) VALUES('abc def ghi'); +} +do_test 2.1 { + sqlite3_interrupt db +} {} +do_execsql_test 2.2 { + ROLLBACK +} + finish_test diff --git a/ext/intck/sqlite3intck.c b/ext/intck/sqlite3intck.c index 5f645fae6..e3fef7763 100644 --- a/ext/intck/sqlite3intck.c +++ b/ext/intck/sqlite3intck.c @@ -319,7 +319,7 @@ static int intckGetToken(const char *z){ char c = z[0]; int iRet = 1; if( c=='\'' || c=='"' || c=='`' ){ - while( 1 ){ + while( z[iRet] ){ if( z[iRet]==c ){ iRet++; if( z[iRet]!=c ) break; diff --git a/ext/jni/README.md b/ext/jni/README.md index 5ad79fce9..0bdbde91e 100644 --- a/ext/jni/README.md +++ b/ext/jni/README.md @@ -13,11 +13,14 @@ Technical support is available in the forum: -> **FOREWARNING:** this subproject is very much in development and - subject to any number of changes. Please do not rely on any - information about its API until this disclaimer is removed. The JNI - bindings released with version 3.43 are a "tech preview." Once - finalized, strong backward compatibility guarantees will apply. +> **FOREWARNING:** the JNI subproject is experimental and subject to + any number of changes. This API is "feature-complete", with only a + few difficult-to-reach corners of the C API not represented here, + but it is not a supported deliverable of the project so does not + have same backward compatibility guarantees which the C APIs + do. That said: the [C-style API](#1to1ish) is especially resistent + to compatibility breakage because it's designed to be as close to + the C API as feasible. Project goals/requirements: @@ -162,11 +165,13 @@ or propagate exceptions and must return error information (if any) via result codes or `null`. The only cases where the C-style APIs may throw is through client-side misuse, e.g. passing in a null where it may cause a `NullPointerException`. The APIs clearly mark function -parameters which should not be null, but does not generally actively -defend itself against such misuse. Some C-style APIs explicitly accept -`null` as a no-op for usability's sake, and some of the JNI APIs -deliberately return an error code, instead of segfaulting, when passed -a `null`. +parameters which should not be null, and it internally uses the +`SQLITE_API_ARMOR` mechanism to help product against such misuse. Some +C-style APIs explicitly accept `null` as a no-op for usability's sake, +and some of the JNI APIs deliberately return an error code, instead of +segfaulting, when passed a `null`. There are no known cases where it +will misuse memory if passed a `null` or out-of-range value from +client code. Client-defined callbacks _must never throw exceptions_ unless _very explicitly documented_ as being throw-safe. Exceptions are generally @@ -194,7 +199,8 @@ Some constructs, when modelled 1-to-1 from C to Java, are unduly clumsy to work with in Java because they try to shoehorn C's way of doing certain things into Java's wildly different ways. The following subsections cover those, starting with a verbose explanation and -demonstration of where such changes are "really necessary"... +demonstration of where such changes are "really necessary" for +usability's sake... ### Custom Collations @@ -286,12 +292,9 @@ binding. The Java API has only one core function-registration function: ```java int sqlite3_create_function(sqlite3 db, String funcName, int nArgs, - int encoding, SQLFunction func); + int flags, SQLFunction func); ``` -> Design question: does the encoding argument serve any purpose in - Java? That's as-yet undetermined. If not, it will be removed. - `SQLFunction` is not used directly, but is instead instantiated via one of its three subclasses: @@ -313,4 +316,4 @@ in-flux nature of this API. Various APIs which accept callbacks, e.g. `sqlite3_trace_v2()` and `sqlite3_update_hook()`, use interfaces similar to those shown above. Despite the changes in signature, the JNI layer makes every effort to -provide the same semantics as the C API documentation suggests. +provide the same semantics as the C API documentation describes. diff --git a/ext/jni/src/c/sqlite3-jni.c b/ext/jni/src/c/sqlite3-jni.c index f130eff04..8cdba9bcf 100644 --- a/ext/jni/src/c/sqlite3-jni.c +++ b/ext/jni/src/c/sqlite3-jni.c @@ -133,11 +133,9 @@ ** Which sqlite3.c we're using needs to be configurable to enable ** building against a custom copy, e.g. the SEE variant. We have to ** include sqlite3.c, as opposed to sqlite3.h, in order to get access -** to some internal details like SQLITE_MAX_... and friends. This -** increases the rebuild time considerably but we need this in order -** to access some internal functionality and keep the to-Java-exported -** values of SQLITE_MAX_... and SQLITE_LIMIT_... in sync with the C -** build. +** to some internal details like SQLITE_MAX_... and friends, and keep +** those consistent with this build. This increases the rebuild time +** considerably, however. */ #ifndef SQLITE_C # define SQLITE_C sqlite3.c @@ -335,7 +333,7 @@ struct S3JniNphOp { const char * const zMember /* Name of member property */; const char * const zTypeSig /* JNI type signature of zMember */; /* - ** klazz is a global ref to the class represented by pRef. + ** klazz is a global ref to the class represented by zName. ** ** According to: ** @@ -999,19 +997,20 @@ static S3JniEnv * S3JniEnv__get(JNIEnv * const env){ ** JNI bindings such as sqlite3_prepare_v2/v3(), and definitely not ** from client code. ** -** Returns err_code. +** Returns err_code _unless_ err_code is 0 and sqlite3_set_errmsg() +** fails with OOM, in which case it may return SQLITE_OOM or fail +** fatally. +** +** This function predates sqlite3_set_errmsg(), which is why it has a +** slightly different interface. Before that function was introduced, +** this code used the SQLite-internal APIs to do this. */ -static int s3jni_db_error(sqlite3* const db, int err_code, - const char * const zMsg){ +static int s3jni_db_error(JNIEnv * env, sqlite3* const db, + int err_code, const char * const zMsg){ if( db!=0 ){ - if( 0==zMsg ){ - sqlite3Error(db, err_code); - }else{ - const int nMsg = sqlite3Strlen30(zMsg); - sqlite3_mutex_enter(sqlite3_db_mutex(db)); - sqlite3ErrorWithMsg(db, err_code, "%.*s", nMsg, zMsg); - sqlite3_mutex_leave(sqlite3_db_mutex(db)); - } + int const rc = sqlite3_set_errmsg(db, err_code, zMsg); + s3jni_oom_fatal(0==rc); + if( rc && !err_code ) err_code=rc; } return err_code; } @@ -1235,11 +1234,11 @@ static int s3jni__db_exception(JNIEnv * const env, sqlite3 * const pDb, char * zMsg; S3JniExceptionClear; zMsg = s3jni_exception_error_msg(env, ex); - s3jni_db_error(pDb, errCode, zMsg ? zMsg : zDfltMsg); + s3jni_db_error(env, pDb, errCode, zMsg ? zMsg : zDfltMsg); sqlite3_free(zMsg); S3JniUnrefLocal(ex); }else if( zDfltMsg ){ - s3jni_db_error(pDb, errCode, zDfltMsg); + s3jni_db_error(env, pDb, errCode, zDfltMsg); } return errCode; } @@ -1956,15 +1955,6 @@ static void S3JniUdf_finalizer(void * s){ S3JniUdf_free(s3jni_env(), (S3JniUdf*)s, 1); } -/* -** Helper for processing args to UDF handlers with signature -** (sqlite3_context*,int,sqlite3_value**). -*/ -typedef struct { - jobject jcx /* sqlite3_context */; - jobjectArray jargv /* sqlite3_value[] */; -} udf_jargs; - /* ** Converts the given (cx, argc, argv) into arguments for the given ** UDF, writing the result (Java wrappers for cx and argv) in the @@ -2006,7 +1996,7 @@ static int udf_args(JNIEnv *env, /* ** Requires that jCx and jArgv are sqlite3_context ** resp. array-of-sqlite3_value values initialized by udf_args(). The -** latter will be 0-and-NULL for UDF types with no arguments. This +** (argc,argv) are (0,NULL) for UDF types with no arguments. This ** function zeroes out the nativePointer member of jCx and each entry ** in jArgv. This is a safety-net precaution to avoid undefined ** behavior if a Java-side UDF holds a reference to its context or one @@ -2099,19 +2089,19 @@ static int udf_xFSI(sqlite3_context* const pCx, int argc, sqlite3_value** const argv, S3JniUdf * const s, jmethodID xMethodID, const char * const zFuncType){ S3JniDeclLocal_env; - udf_jargs args = {0,0}; - int rc = udf_args(env, pCx, argc, argv, &args.jcx, &args.jargv); - + jobject jcx = 0 /* sqlite3_context */; + jobjectArray jargv = 0 /* sqlite3_value[] */; + int rc = udf_args(env, pCx, argc, argv, &jcx, &jargv); if( 0 == rc ){ - (*env)->CallVoidMethod(env, s->jObj, xMethodID, args.jcx, args.jargv); + (*env)->CallVoidMethod(env, s->jObj, xMethodID, jcx, jargv); S3JniIfThrew{ rc = udf_report_exception(env, 'F'==zFuncType[1]/*xFunc*/, pCx, s->zFuncName, zFuncType); } - udf_unargs(env, args.jcx, argc, args.jargv); + udf_unargs(env, jcx, argc, jargv); } - S3JniUnrefLocal(args.jcx); - S3JniUnrefLocal(args.jargv); + S3JniUnrefLocal(jcx); + S3JniUnrefLocal(jargv); return rc; } @@ -3300,7 +3290,7 @@ static jobject s3jni_commit_rollback_hook(int isCommit, JNIEnv * const env, S3JniDb_mutex_enter; ps = S3JniDb_from_jlong(jpDb); if( !ps ){ - s3jni_db_error(ps->pDb, SQLITE_MISUSE, 0); + s3jni_db_error(env, ps->pDb, SQLITE_MISUSE, 0); S3JniDb_mutex_leave; return 0; } @@ -3320,13 +3310,14 @@ static jobject s3jni_commit_rollback_hook(int isCommit, JNIEnv * const env, else sqlite3_rollback_hook(ps->pDb, 0, 0); }else{ jclass const klazz = (*env)->GetObjectClass(env, jHook); - jmethodID const xCallback = (*env)->GetMethodID(env, klazz, "call", - isCommit ? "()I" : "()V"); + jmethodID const xCallback = + (*env)->GetMethodID(env, klazz, "call", + isCommit ? "()I" : "()V"); S3JniUnrefLocal(klazz); S3JniIfThrew { S3JniExceptionReport; S3JniExceptionClear; - s3jni_db_error(ps->pDb, SQLITE_ERROR, + s3jni_db_error(env, ps->pDb, SQLITE_ERROR, "Cannot not find matching call() method in" "hook object."); }else{ @@ -3606,7 +3597,7 @@ S3JniApi(sqlite3_create_collation() sqlite3_create_collation_v2(), (*env)->GetMethodID(env, klazz, "call", "([B[B)I"); S3JniUnrefLocal(klazz); S3JniIfThrew{ - rc = s3jni_db_error(ps->pDb, SQLITE_ERROR, + rc = s3jni_db_error(env, ps->pDb, SQLITE_ERROR, "Could not get call() method from " "CollationCallback object."); }else{ @@ -3645,15 +3636,15 @@ S3JniApi(sqlite3_create_function() sqlite3_create_function_v2() if( !pDb || !jFuncName ){ return SQLITE_MISUSE; - }else if( !encodingTypeIsValid(eTextRep) ){ - return s3jni_db_error(pDb, SQLITE_FORMAT, + }else if( !encodingTypeIsValid(eTextRep & 0x0f) ){ + return s3jni_db_error(env, pDb, SQLITE_FORMAT, "Invalid function encoding option."); } s = S3JniUdf_alloc(env, jFunctor); if( !s ) return SQLITE_NOMEM; if( UDF_UNKNOWN_TYPE==s->type ){ - rc = s3jni_db_error(pDb, SQLITE_MISUSE, + rc = s3jni_db_error(env, pDb, SQLITE_MISUSE, "Cannot unambiguously determine function type."); S3JniUdf_free(env, s, 1); goto error_cleanup; @@ -4012,7 +4003,7 @@ S3JniApi(sqlite3_jni_db_error(), jint, 1jni_1db_1error)( zStr = jStr ? s3jni_jstring_to_utf8( jStr, 0) : NULL; - rc = s3jni_db_error( ps->pDb, (int)jRc, zStr ); + rc = s3jni_db_error(env, ps->pDb, (int)jRc, zStr ); sqlite3_free(zStr); } return rc; @@ -4321,7 +4312,7 @@ static void s3jni_updatepre_hook_impl(void * pState, sqlite3 *pDb, int opId, jTable = jDbName ? s3jni_utf8_to_jstring( zTable, -1) : 0; S3JniIfThrew { S3JniExceptionClear; - s3jni_db_error(ps->pDb, SQLITE_NOMEM, 0); + s3jni_db_error(env, ps->pDb, SQLITE_NOMEM, 0); }else{ assert( hook.jObj ); assert( hook.midCallback ); @@ -4423,7 +4414,7 @@ static jobject s3jni_updatepre_hook(JNIEnv * env, int isPre, jlong jpDb, jobject S3JniUnrefLocal(klazz); S3JniIfThrew { S3JniExceptionClear; - s3jni_db_error(ps->pDb, SQLITE_ERROR, + s3jni_db_error(env, ps->pDb, SQLITE_ERROR, "Cannot not find matching callback on " "(pre)update hook object."); }else{ @@ -4532,7 +4523,7 @@ S3JniApi(sqlite3_progress_handler(),void,1progress_1handler)( S3JniUnrefLocal(klazz); S3JniIfThrew { S3JniExceptionClear; - s3jni_db_error(ps->pDb, SQLITE_ERROR, + s3jni_db_error(env, ps->pDb, SQLITE_ERROR, "Cannot not find matching xCallback() on " "ProgressHandler object."); }else{ @@ -4906,8 +4897,9 @@ S3JniApi(sqlite3_set_authorizer(),jint,1set_1authorizer)( ")I"); S3JniUnrefLocal(klazz); S3JniIfThrew { - rc = s3jni_db_error(ps->pDb, SQLITE_ERROR, - "Error setting up Java parts of authorizer hook."); + rc = s3jni_db_error(env, ps->pDb, SQLITE_ERROR, + "Error setting up Java parts of " + "authorizer hook."); }else{ rc = sqlite3_set_authorizer(ps->pDb, s3jni_xAuth, ps); } @@ -5191,7 +5183,7 @@ S3JniApi(sqlite3_trace_v2(),jint,1trace_1v2)( S3JniUnrefLocal(klazz); S3JniIfThrew { S3JniExceptionClear; - rc = s3jni_db_error(ps->pDb, SQLITE_ERROR, + rc = s3jni_db_error(env, ps->pDb, SQLITE_ERROR, "Cannot not find matching call() on " "TracerCallback object."); }else{ diff --git a/ext/jni/src/c/sqlite3-jni.h b/ext/jni/src/c/sqlite3-jni.h index 81af5cbde..c326fa8ea 100644 --- a/ext/jni/src/c/sqlite3-jni.h +++ b/ext/jni/src/c/sqlite3-jni.h @@ -245,8 +245,10 @@ extern "C" { #define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_USED_SHARED 11L #undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_SPILL #define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_SPILL 12L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_TEMPBUF_SPILL +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_TEMPBUF_SPILL 13L #undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_MAX -#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_MAX 12L +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_MAX 13L #undef org_sqlite_jni_capi_CApi_SQLITE_UTF8 #define org_sqlite_jni_capi_CApi_SQLITE_UTF8 1L #undef org_sqlite_jni_capi_CApi_SQLITE_UTF16LE diff --git a/ext/jni/src/org/sqlite/jni/capi/CApi.java b/ext/jni/src/org/sqlite/jni/capi/CApi.java index 0b840c362..1bdc5300d 100644 --- a/ext/jni/src/org/sqlite/jni/capi/CApi.java +++ b/ext/jni/src/org/sqlite/jni/capi/CApi.java @@ -115,8 +115,9 @@ private static byte[] nulTerminateUtf8(String s){ JNIEnv is not cached, else returns true, but this information is primarily for testing of the JNI bindings and is not information which client-level code can use to make any informed - decisions. Its return type and semantics are not considered - stable and may change at any time. + decisions. The semantics of its return type and value are not + considered stable and may change at any time. i.e. act as if it + returns null. */ public static native boolean sqlite3_java_uncache_thread(); @@ -2585,7 +2586,8 @@ public static int sqlite3_value_type(@NotNull sqlite3_value v){ public static final int SQLITE_DBSTATUS_DEFERRED_FKS = 10; public static final int SQLITE_DBSTATUS_CACHE_USED_SHARED = 11; public static final int SQLITE_DBSTATUS_CACHE_SPILL = 12; - public static final int SQLITE_DBSTATUS_MAX = 12; + public static final int SQLITE_DBSTATUS_TEMPBUF_SPILL = 13; + public static final int SQLITE_DBSTATUS_MAX = 13; // encodings public static final int SQLITE_UTF8 = 1; diff --git a/ext/jni/src/org/sqlite/jni/capi/Tester1.java b/ext/jni/src/org/sqlite/jni/capi/Tester1.java index 9d14c954b..891bdea54 100644 --- a/ext/jni/src/org/sqlite/jni/capi/Tester1.java +++ b/ext/jni/src/org/sqlite/jni/capi/Tester1.java @@ -815,7 +815,9 @@ public void xDestroy(){ }; // Register and use the function... - int rc = sqlite3_create_function(db, "myfunc", -1, SQLITE_UTF8, func); + int rc = sqlite3_create_function(db, "myfunc", -1, + SQLITE_UTF8 | SQLITE_INNOCUOUS, + func); affirm(0 == rc); affirm(0 == xFuncAccum.value); final sqlite3_stmt stmt = prepare(db, "SELECT myfunc(1,2,3)"); diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java b/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java index d259e0ce6..ba2ffd119 100644 --- a/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java +++ b/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java @@ -171,6 +171,7 @@ public final class Sqlite implements AutoCloseable { public static final int DBSTATUS_DEFERRED_FKS = CApi.SQLITE_DBSTATUS_DEFERRED_FKS; public static final int DBSTATUS_CACHE_USED_SHARED = CApi.SQLITE_DBSTATUS_CACHE_USED_SHARED; public static final int DBSTATUS_CACHE_SPILL = CApi.SQLITE_DBSTATUS_CACHE_SPILL; + public static final int DBSTATUS_TEMPBUF_SPILL = CApi.SQLITE_DBSTATUS_TEMPBUF_SPILL; // Limits public static final int LIMIT_LENGTH = CApi.SQLITE_LIMIT_LENGTH; diff --git a/ext/misc/btreeinfo.c b/ext/misc/btreeinfo.c index 9c726f5f1..24645f226 100644 --- a/ext/misc/btreeinfo.c +++ b/ext/misc/btreeinfo.c @@ -306,6 +306,10 @@ static int binfoCompute(sqlite3 *db, int pgno, BinfoCursor *pCsr){ nEntry *= (nCell+1); if( aData[0]==10 || aData[0]==13 ) break; nPage *= (nCell+1); + if( 14+2*(nCell/2)>=pgsz ){ + rc = SQLITE_CORRUPT; + break; + } if( nCell<=1 ){ pgno = get_uint32(aData+8); }else{ @@ -339,7 +343,7 @@ static int binfoColumn( sqlite3 *db = sqlite3_context_db_handle(ctx); int rc = binfoCompute(db, pgno, pCsr); if( rc ){ - pCursor->pVtab->zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + pCursor->pVtab->zErrMsg = sqlite3_mprintf("%s", sqlite3_errstr(rc)); return SQLITE_ERROR; } } diff --git a/ext/misc/csv.c b/ext/misc/csv.c index 1caaaec87..f44a30001 100644 --- a/ext/misc/csv.c +++ b/ext/misc/csv.c @@ -24,8 +24,8 @@ ** schema= parameter, like this: ** ** CREATE VIRTUAL TABLE temp.csv2 USING csv( -** filename = "../http.log", -** schema = "CREATE TABLE x(date,ipaddr,url,referrer,userAgent)" +** filename = '../http.log', +** schema = 'CREATE TABLE x(date,ipaddr,url,referrer,userAgent)' ** ); ** ** Instead of specifying a file, the text of the CSV can be loaded using diff --git a/ext/misc/decimal.c b/ext/misc/decimal.c index f87699f96..be4321ca8 100644 --- a/ext/misc/decimal.c +++ b/ext/misc/decimal.c @@ -291,12 +291,36 @@ static void decimal_result(sqlite3_context *pCtx, Decimal *p){ sqlite3_result_text(pCtx, z, i, sqlite3_free); } +/* +** Round a decimal value to N significant digits. N must be positive. +*/ +static void decimal_round(Decimal *p, int N){ + int i; + int nZero; + if( N<1 ) return; + for(nZero=0; nZeronDigit && p->a[nZero]==0; nZero++){} + N += nZero; + if( p->nDigit<=N ) return; + if( p->a[N]>4 ){ + p->a[N-1]++; + for(i=N-1; i>0 && p->a[i]>9; i--){ + p->a[i] = 0; + p->a[i-1]++; + } + if( p->a[0]>9 ){ + p->a[0] = 1; + p->nFrac--; + } + } + memset(&p->a[N], 0, p->nDigit - N); +} + /* ** Make the given Decimal the result in an format similar to '%+#e'. ** In other words, show exponential notation with leading and trailing ** zeros omitted. */ -static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ +static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p, int N){ char *z; /* The output buffer */ int i; /* Loop counter */ int nZero; /* Number of leading zeros */ @@ -314,7 +338,8 @@ static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ sqlite3_result_null(pCtx); return; } - for(nDigit=p->nDigit; nDigit>0 && p->a[nDigit-1]==0; nDigit--){} + if( N<1 ) N = 0; + for(nDigit=p->nDigit; nDigit>N && p->a[nDigit-1]==0; nDigit--){} for(nZero=0; nZeroa[nZero]==0; nZero++){} nFrac = p->nFrac + (nDigit - p->nDigit); nDigit -= nZero; @@ -677,10 +702,16 @@ static void decimalFunc( sqlite3_value **argv ){ Decimal *p = decimal_new(context, argv[0], 0); - UNUSED_PARAMETER(argc); + int N; + if( argc==2 ){ + N = sqlite3_value_int(argv[1]); + if( N>0 ) decimal_round(p, N); + }else{ + N = 0; + } if( p ){ if( sqlite3_user_data(context)!=0 ){ - decimal_result_sci(context, p); + decimal_result_sci(context, p, N); }else{ decimal_result(context, p); } @@ -850,7 +881,7 @@ static void decimalPow2Func( UNUSED_PARAMETER(argc); if( sqlite3_value_type(argv[0])==SQLITE_INTEGER ){ Decimal *pA = decimalPow2(sqlite3_value_int(argv[0])); - decimal_result_sci(context, pA); + decimal_result_sci(context, pA, 0); decimal_free(pA); } } @@ -871,7 +902,9 @@ int sqlite3_decimal_init( void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } aFunc[] = { { "decimal", 1, 0, decimalFunc }, + { "decimal", 2, 0, decimalFunc }, { "decimal_exp", 1, 1, decimalFunc }, + { "decimal_exp", 2, 1, decimalFunc }, { "decimal_cmp", 2, 0, decimalCmpFunc }, { "decimal_add", 2, 0, decimalAddFunc }, { "decimal_sub", 2, 0, decimalSubFunc }, diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c index 6cc2ae008..51b748291 100644 --- a/ext/misc/fileio.c +++ b/ext/misc/fileio.c @@ -94,12 +94,16 @@ SQLITE_EXTENSION_INIT1 # include # include # define STRUCT_STAT struct stat +# include +# include #else # include "windirent.h" # include # define STRUCT_STAT struct _stat # define chmod(path,mode) fileio_chmod(path,mode) # define mkdir(path,mode) fileio_mkdir(path) + extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*); + extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR); #endif #include #include @@ -131,12 +135,9 @@ SQLITE_EXTENSION_INIT1 */ #if defined(_WIN32) || defined(WIN32) static int fileio_chmod(const char *zPath, int pmode){ - sqlite3_int64 sz = strlen(zPath); - wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(b1[0]) ); int rc; + wchar_t *b1 = sqlite3_win32_utf8_to_unicode(zPath); if( b1==0 ) return -1; - sz = MultiByteToWideChar(CP_UTF8, 0, zPath, sz, b1, sz); - b1[sz] = 0; rc = _wchmod(b1, pmode); sqlite3_free(b1); return rc; @@ -148,12 +149,9 @@ static int fileio_chmod(const char *zPath, int pmode){ */ #if defined(_WIN32) || defined(WIN32) static int fileio_mkdir(const char *zPath){ - sqlite3_int64 sz = strlen(zPath); - wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(b1[0]) ); int rc; + wchar_t *b1 = sqlite3_win32_utf8_to_unicode(zPath); if( b1==0 ) return -1; - sz = MultiByteToWideChar(CP_UTF8, 0, zPath, sz, b1, sz); - b1[sz] = 0; rc = _wmkdir(b1); sqlite3_free(b1); return rc; @@ -266,50 +264,7 @@ static sqlite3_uint64 fileTimeToUnixTime( return (fileIntervals.QuadPart - epochIntervals.QuadPart) / 10000000; } - - -#if defined(FILEIO_WIN32_DLL) && (defined(_WIN32) || defined(WIN32)) -# /* To allow a standalone DLL, use this next replacement function: */ -# undef sqlite3_win32_utf8_to_unicode -# define sqlite3_win32_utf8_to_unicode utf8_to_utf16 -# -LPWSTR utf8_to_utf16(const char *z){ - int nAllot = MultiByteToWideChar(CP_UTF8, 0, z, -1, NULL, 0); - LPWSTR rv = sqlite3_malloc(nAllot * sizeof(WCHAR)); - if( rv!=0 && 0 < MultiByteToWideChar(CP_UTF8, 0, z, -1, rv, nAllot) ) - return rv; - sqlite3_free(rv); - return 0; -} -#endif - -/* -** This function attempts to normalize the time values found in the stat() -** buffer to UTC. This is necessary on Win32, where the runtime library -** appears to return these values as local times. -*/ -static void statTimesToUtc( - const char *zPath, - STRUCT_STAT *pStatBuf -){ - HANDLE hFindFile; - WIN32_FIND_DATAW fd; - LPWSTR zUnicodeName; - extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*); - zUnicodeName = sqlite3_win32_utf8_to_unicode(zPath); - if( zUnicodeName ){ - memset(&fd, 0, sizeof(WIN32_FIND_DATAW)); - hFindFile = FindFirstFileW(zUnicodeName, &fd); - if( hFindFile!=NULL ){ - pStatBuf->st_ctime = (time_t)fileTimeToUnixTime(&fd.ftCreationTime); - pStatBuf->st_atime = (time_t)fileTimeToUnixTime(&fd.ftLastAccessTime); - pStatBuf->st_mtime = (time_t)fileTimeToUnixTime(&fd.ftLastWriteTime); - FindClose(hFindFile); - } - sqlite3_free(zUnicodeName); - } -} -#endif +#endif /* _WIN32 */ /* ** This function is used in place of stat(). On Windows, special handling @@ -321,14 +276,22 @@ static int fileStat( STRUCT_STAT *pStatBuf ){ #if defined(_WIN32) - sqlite3_int64 sz = strlen(zPath); - wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(b1[0]) ); int rc; + wchar_t *b1 = sqlite3_win32_utf8_to_unicode(zPath); if( b1==0 ) return 1; - sz = MultiByteToWideChar(CP_UTF8, 0, zPath, sz, b1, sz); - b1[sz] = 0; rc = _wstat(b1, pStatBuf); - if( rc==0 ) statTimesToUtc(zPath, pStatBuf); + if( rc==0 ){ + HANDLE hFindFile; + WIN32_FIND_DATAW fd; + memset(&fd, 0, sizeof(WIN32_FIND_DATAW)); + hFindFile = FindFirstFileW(b1, &fd); + if( hFindFile!=NULL ){ + pStatBuf->st_ctime = (time_t)fileTimeToUnixTime(&fd.ftCreationTime); + pStatBuf->st_atime = (time_t)fileTimeToUnixTime(&fd.ftLastAccessTime); + pStatBuf->st_mtime = (time_t)fileTimeToUnixTime(&fd.ftLastWriteTime); + FindClose(hFindFile); + } + } sqlite3_free(b1); return rc; #else @@ -460,7 +423,6 @@ static int writeFile( if( mtime>=0 ){ #if defined(_WIN32) -#if !SQLITE_OS_WINRT /* Windows */ FILETIME lastAccess; FILETIME lastWrite; @@ -491,7 +453,6 @@ static int writeFile( }else{ return 1; } -#endif #elif defined(AT_FDCWD) && 0 /* utimensat() is not universally available */ /* Recent unix */ struct timespec times[2]; @@ -1095,6 +1056,154 @@ static int fsdirRegister(sqlite3 *db){ # define fsdirRegister(x) SQLITE_OK #endif +/* +** This version of realpath() works on any system. The string +** returned is held in memory allocated using sqlite3_malloc64(). +** The caller is responsible for calling sqlite3_free(). +*/ +static char *portable_realpath(const char *zPath){ +#if !defined(_WIN32) /* BEGIN unix */ + + char *zOut = 0; /* Result */ + char *z; /* Temporary buffer */ +#if defined(PATH_MAX) + char zBuf[PATH_MAX+1]; /* Space for the temporary buffer */ +#endif + + if( zPath==0 ) return 0; +#if defined(PATH_MAX) + z = realpath(zPath, zBuf); + if( z ){ + zOut = sqlite3_mprintf("%s", zBuf); + } +#endif /* defined(PATH_MAX) */ + if( zOut==0 ){ + /* Try POSIX.1-2008 malloc behavior */ + z = realpath(zPath, NULL); + if( z ){ + zOut = sqlite3_mprintf("%s", z); + free(z); + } + } + return zOut; + +#else /* End UNIX, Begin WINDOWS */ + + wchar_t *zPath16; /* UTF16 translation of zPath */ + char *zOut = 0; /* Result */ + wchar_t *z = 0; /* Temporary buffer */ + + if( zPath==0 ) return 0; + + zPath16 = sqlite3_win32_utf8_to_unicode(zPath); + if( zPath16==0 ) return 0; + z = _wfullpath(NULL, zPath16, 0); + sqlite3_free(zPath16); + if( z ){ + zOut = sqlite3_win32_unicode_to_utf8(z); + free(z); + } + return zOut; + +#endif /* End WINDOWS, Begin common code */ +} + +/* +** SQL function: realpath(X) +** +** Try to convert file or pathname X into its real, absolute pathname. +** Return NULL if unable. +** +** The file or directory X is not required to exist. The answer is formed +** by calling system realpath() on the prefix of X that does exist and +** appending the tail of X that does not (yet) exist. +*/ +static void realpathFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zPath; /* Original input path */ + char *zCopy; /* An editable copy of zPath */ + char *zOut; /* The result */ + char cSep = 0; /* Separator turned into \000 */ + size_t len; /* Prefix length before cSep */ +#ifdef _WIN32 + const int isWin = 1; +#else + const int isWin = 0; +#endif + + (void)argc; + zPath = (const char*)sqlite3_value_text(argv[0]); + if( zPath==0 ) return; + if( zPath[0]==0 ) zPath = "."; + zCopy = sqlite3_mprintf("%s",zPath); + len = strlen(zCopy); + while( len>1 && (zCopy[len-1]=='/' || (isWin && zCopy[len-1]=='\\')) ){ + len--; + } + zCopy[len] = 0; + while( 1 /*exit-by-break*/ ){ + zOut = portable_realpath(zCopy); + zCopy[len] = cSep; + if( zOut ){ + if( cSep ){ + zOut = sqlite3_mprintf("%z%s",zOut,&zCopy[len]); + } + break; + }else{ + size_t i = len-1; + while( i>0 ){ + if( zCopy[i]=='/' || (isWin && zCopy[i]=='\\') ) break; + i--; + } + if( i<=0 ){ + if( zCopy[0]=='/' ){ + zOut = zCopy; + zCopy = 0; + }else if( (zOut = portable_realpath("."))!=0 ){ + zOut = sqlite3_mprintf("%z/%s", zOut, zCopy); + } + break; + } + cSep = zCopy[i]; + zCopy[i] = 0; + len = i; + } + } + sqlite3_free(zCopy); + if( zOut ){ + /* Simplify any "/./" or "/../" that might have snuck into the + ** pathname due to appending of zCopy. We only have to consider + ** unix "/" separators, because the _wfilepath() system call on + ** Windows will have already done this simplification for us. */ + size_t i, j, n; + n = strlen(zOut); + for(i=j=0; i0 && zOut[j-1]!='/' ){ j--; } + if( j>0 ){ j--; } + i += 2; + continue; + } + } + zOut[j++] = zOut[i]; + } + zOut[j] = 0; + + /* Return the result */ + sqlite3_result_text(context, zOut, -1, sqlite3_free); + } +} + + #ifdef _WIN32 __declspec(dllexport) #endif @@ -1121,13 +1230,10 @@ int sqlite3_fileio_init( if( rc==SQLITE_OK ){ rc = fsdirRegister(db); } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "realpath", 1, + SQLITE_UTF8, 0, + realpathFunc, 0, 0); + } return rc; } - -#if defined(FILEIO_WIN32_DLL) && (defined(_WIN32) || defined(WIN32)) -/* To allow a standalone DLL, make test_windirent.c use the same - * redefined SQLite API calls as the above extension code does. - * Just pull in this .c to accomplish this. As a beneficial side - * effect, this extension becomes a single translation unit. */ -# include "test_windirent.c" -#endif diff --git a/ext/misc/fossildelta.c b/ext/misc/fossildelta.c index d24a87700..2dc29b3c3 100644 --- a/ext/misc/fossildelta.c +++ b/ext/misc/fossildelta.c @@ -38,9 +38,11 @@ SQLITE_EXTENSION_INIT1 #ifndef SQLITE_AMALGAMATION /* -** The "u32" type must be an unsigned 32-bit integer. Adjust this +** The "u32" type must be an unsigned 32-bit integer. "u64" is +** an unsigned 64-bit integer. */ typedef unsigned int u32; +typedef sqlite3_uint64 u64; /* ** Must be a 16-bit value @@ -541,8 +543,8 @@ static int delta_apply( int lenDelta, /* Length of the delta */ char *zOut /* Write the output into this preallocated buffer */ ){ - unsigned int limit; - unsigned int total = 0; + sqlite3_uint64 limit; + sqlite3_uint64 total = 0; #ifdef FOSSIL_ENABLE_DELTA_CKSUM_TEST char *zOrigOut = zOut; #endif @@ -570,7 +572,7 @@ static int delta_apply( /* ERROR: copy exceeds output file size */ return -1; } - if( ofst+cnt > lenSrc ){ + if( (u64)ofst+(u64)cnt > (u64)lenSrc ){ /* ERROR: copy extends past end of input */ return -1; } diff --git a/ext/misc/fuzzer.c b/ext/misc/fuzzer.c index e16d005d9..3dcf1d667 100644 --- a/ext/misc/fuzzer.c +++ b/ext/misc/fuzzer.c @@ -617,7 +617,7 @@ static int fuzzerRender( int *pnBuf /* Size of the buffer */ ){ const fuzzer_rule *pRule = pStem->pRule; - int n; /* Size of output term without nul-term */ + sqlite3_int64 n; /* Size of output term without nul-term */ char *z; /* Buffer to assemble output term in */ n = pStem->nBasis + pRule->nTo - pRule->nFrom; diff --git a/ext/misc/ieee754.c b/ext/misc/ieee754.c index 7f1c6d9e1..f551b2265 100644 --- a/ext/misc/ieee754.c +++ b/ext/misc/ieee754.c @@ -179,9 +179,9 @@ static void ieee754func( } if( m<0 ){ + if( m<(-9223372036854775807LL) ) return; isNeg = 1; m = -m; - if( m<0 ) return; }else if( m==0 && e>-1000 && e<1000 ){ sqlite3_result_double(context, 0.0); return; @@ -259,6 +259,38 @@ static void ieee754func_to_blob( } } +/* +** Functions to convert between 64-bit integers and floats. +** +** The bit patterns are copied. The numeric values are different. +*/ +static void ieee754func_from_int( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_INTEGER ){ + double r; + sqlite3_int64 v = sqlite3_value_int64(argv[0]); + memcpy(&r, &v, sizeof(r)); + sqlite3_result_double(context, r); + } +} +static void ieee754func_to_int( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_FLOAT ){ + double r = sqlite3_value_double(argv[0]); + sqlite3_uint64 v; + memcpy(&v, &r, sizeof(v)); + sqlite3_result_int64(context, v); + } +} + /* ** SQL Function: ieee754_inc(r,N) ** @@ -311,6 +343,8 @@ int sqlite3_ieee_init( { "ieee754_exponent", 1, 2, ieee754func }, { "ieee754_to_blob", 1, 0, ieee754func_to_blob }, { "ieee754_from_blob", 1, 0, ieee754func_from_blob }, + { "ieee754_to_int", 1, 0, ieee754func_to_int }, + { "ieee754_from_int", 1, 0, ieee754func_from_int }, { "ieee754_inc", 2, 0, ieee754inc }, }; unsigned int i; diff --git a/ext/misc/regexp.c b/ext/misc/regexp.c index f1babf4ab..e1826caf3 100644 --- a/ext/misc/regexp.c +++ b/ext/misc/regexp.c @@ -32,7 +32,7 @@ ** ^X X occurring at the beginning of the string ** X$ X occurring at the end of the string ** . Match any single character -** \c Character c where c is one of \{}()[]|*+?. +** \c Character c where c is one of \{}()[]|*+?-. ** \c C-language escapes for c in afnrtv. ex: \t or \n ** \uXXXX Where XXXX is exactly 4 hex digits, unicode value XXXX ** \xXX Where XX is exactly 2 hex digits, unicode value XX @@ -417,7 +417,7 @@ static int re_hex(int c, int *pV){ ** return its interpretation. */ static unsigned re_esc_char(ReCompiled *p){ - static const char zEsc[] = "afnrtv\\()*.+?[$^{|}]"; + static const char zEsc[] = "afnrtv\\()*.+?[$^{|}]-"; static const char zTrans[] = "\a\f\n\r\t\v"; int i, v = 0; char c; @@ -740,11 +740,18 @@ static const char *re_compile( } /* -** Compute a reasonable limit on the length of the REGEXP NFA. +** The value of LIMIT_MAX_PATTERN_LENGTH. */ static int re_maxlen(sqlite3_context *context){ sqlite3 *db = sqlite3_context_db_handle(context); - return 75 + sqlite3_limit(db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH,-1)/2; + return sqlite3_limit(db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH,-1); +} + +/* +** Maximum NFA size given a maximum pattern length. +*/ +static int re_maxnfa(int mxlen){ + return 75+mxlen/2; } /* @@ -770,10 +777,17 @@ static void re_sql_func( (void)argc; /* Unused */ pRe = sqlite3_get_auxdata(context, 0); if( pRe==0 ){ + int mxLen = re_maxlen(context); + int nPattern; zPattern = (const char*)sqlite3_value_text(argv[0]); if( zPattern==0 ) return; - zErr = re_compile(&pRe, zPattern, re_maxlen(context), - sqlite3_user_data(context)!=0); + nPattern = sqlite3_value_bytes(argv[0]); + if( nPattern>mxLen ){ + zErr = "REGEXP pattern too big"; + }else{ + zErr = re_compile(&pRe, zPattern, re_maxnfa(mxLen), + sqlite3_user_data(context)!=0); + } if( zErr ){ re_free(pRe); sqlite3_result_error(context, zErr, -1); @@ -839,7 +853,7 @@ static void re_bytecode_func( zPattern = (const char*)sqlite3_value_text(argv[0]); if( zPattern==0 ) return; - zErr = re_compile(&pRe, zPattern, re_maxlen(context), + zErr = re_compile(&pRe, zPattern, re_maxnfa(re_maxlen(context)), sqlite3_user_data(context)!=0); if( zErr ){ re_free(pRe); diff --git a/ext/misc/sha1.c b/ext/misc/sha1.c index 07d797060..02d864955 100644 --- a/ext/misc/sha1.c +++ b/ext/misc/sha1.c @@ -230,13 +230,16 @@ static void hash_finish( *****************************************************************************/ /* -** Implementation of the sha1(X) function. +** Two SQL functions: sha1(X) and sha1b(X). ** -** Return a lower-case hexadecimal rendering of the SHA1 hash of the -** argument X. If X is a BLOB, it is hashed as is. For all other +** sha1(X) returns a lower-case hexadecimal rendering of the SHA1 hash +** of the argument X. If X is a BLOB, it is hashed as is. For all other ** types of input, X is converted into a UTF-8 string and the string -** is hash without the trailing 0x00 terminator. The hash of a NULL +** is hashed without the trailing 0x00 terminator. The hash of a NULL ** value is NULL. +** +** sha1b(X) is the same except that it returns a 20-byte BLOB containing +** the binary hash instead of a hexadecimal string. */ static void sha1Func( sqlite3_context *context, @@ -257,11 +260,13 @@ static void sha1Func( hash_step(&cx, sqlite3_value_text(argv[0]), nByte); } if( sqlite3_user_data(context)!=0 ){ + /* sha1b() - binary result */ hash_finish(&cx, zOut, 1); sqlite3_result_blob(context, zOut, 20, SQLITE_TRANSIENT); }else{ + /* sha1() - hexadecimal text result */ hash_finish(&cx, zOut, 0); - sqlite3_result_blob(context, zOut, 40, SQLITE_TRANSIENT); + sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); } } diff --git a/ext/misc/sqlite3_stdio.c b/ext/misc/sqlite3_stdio.c index c9bceb194..d59757526 100644 --- a/ext/misc/sqlite3_stdio.c +++ b/ext/misc/sqlite3_stdio.c @@ -258,7 +258,7 @@ int sqlite3_fputs(const char *z, FILE *out){ /* -** Work-alike for fprintf() from the standard C library. +** Work-alikes for fprintf() and vfprintf() from the standard C library. */ int sqlite3_fprintf(FILE *out, const char *zFormat, ...){ int rc; @@ -285,6 +285,24 @@ int sqlite3_fprintf(FILE *out, const char *zFormat, ...){ } return rc; } +int sqlite3_vfprintf(FILE *out, const char *zFormat, va_list ap){ + int rc; + if( UseWtextForOutput(out) ){ + /* When writing to the command-prompt in Windows, it is necessary + ** to use _O_WTEXT input mode and write UTF-16 characters. + */ + char *z; + z = sqlite3_vmprintf(zFormat, ap); + sqlite3_fputs(z, out); + rc = (int)strlen(z); + sqlite3_free(z); + }else{ + /* Writing to a file or other destination, just write bytes without + ** any translation. */ + rc = vfprintf(out, zFormat, ap); + } + return rc; +} /* ** Set the mode for an output stream. mode argument is typically _O_BINARY or diff --git a/ext/misc/sqlite3_stdio.h b/ext/misc/sqlite3_stdio.h index dd0eefad0..75368df9f 100644 --- a/ext/misc/sqlite3_stdio.h +++ b/ext/misc/sqlite3_stdio.h @@ -31,6 +31,7 @@ #ifdef _WIN32 /**** Definitions For Windows ****/ #include +#include #include FILE *sqlite3_fopen(const char *zFilename, const char *zMode); @@ -38,6 +39,7 @@ FILE *sqlite3_popen(const char *zCommand, const char *type); char *sqlite3_fgets(char *s, int size, FILE *stream); int sqlite3_fputs(const char *s, FILE *stream); int sqlite3_fprintf(FILE *stream, const char *format, ...); +int sqlite3_vfprintf(FILE *stream, const char *format, va_list); void sqlite3_fsetmode(FILE *stream, int mode); @@ -49,6 +51,7 @@ void sqlite3_fsetmode(FILE *stream, int mode); #define sqlite3_fgets fgets #define sqlite3_fputs fputs #define sqlite3_fprintf fprintf +#define sqlite3_vfprintf vfprintf #define sqlite3_fsetmode(F,X) /*no-op*/ #endif diff --git a/ext/misc/tmstmpvfs.c b/ext/misc/tmstmpvfs.c new file mode 100644 index 000000000..6f1af36f7 --- /dev/null +++ b/ext/misc/tmstmpvfs.c @@ -0,0 +1,1042 @@ +/* +** 2026-01-05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements a VFS shim that writes a timestamp and other tracing +** information into 16 byts of reserved space at the end of each page of the +** database file. +** +** The VFS also tries to generate log-files with names of the form: +** +** $(DATABASE)-tmstmp/$(TIME)-$(PID)-$(ID) +** +** Log files are only generated if directory $(DATABASE)-tmstmp exists. +** The name of each log file is the current ISO8601 time in milliseconds, +** the process ID, and a random 32-bit value (to disambiguate multiple +** connections from the same process) separated by dashes. The log file +** contains 16-bytes records for various events, such as opening or close +** of the database or WAL file, writes to the WAL file, checkpoints, and +** similar. The logfile is only generated if the connection attempts to +** modify the database. There is a separate log file for each open database +** connection. +** +** COMPILING +** +** To build this extension as a separately loaded shared library or +** DLL, use compiler command-lines similar to the following: +** +** (linux) gcc -fPIC -shared tmstmpvfs.c -o tmstmpvfs.so +** (mac) clang -fPIC -dynamiclib tmstmpvfs.c -o tmstmpvfs.dylib +** (windows) cl tmstmpvfs.c -link -dll -out:tmstmpvfs.dll +** +** You may want to add additional compiler options, of course, +** according to the needs of your project. +** +** Another option is to statically link both SQLite and this extension +** into your application. If both this file and "sqlite3.c" are statically +** linked, and if "sqlite3.c" is compiled with an option like: +** +** -DSQLITE_EXTRA_INIT=sqlite3_register_tmstmpvfs +** +** Then SQLite will use the tmstmp VFS by default throughout your +** application. +** +** LOADING +** +** To load this extension as a shared library, you first have to +** bring up a dummy SQLite database connection to use as the argument +** to the sqlite3_load_extension() API call. Then you invoke the +** sqlite3_load_extension() API and shutdown the dummy database +** connection. All subsequent database connections that are opened +** will include this extension. For example: +** +** sqlite3 *db; +** sqlite3_open(":memory:", &db); +** sqlite3_load_extension(db, "./tmstmpvfs"); +** sqlite3_close(db); +** +** Tmstmpvfs is a VFS Shim. When loaded, "tmstmpvfs" becomes the new +** default VFS and it uses the prior default VFS as the next VFS +** down in the stack. This is normally what you want. However, in +** complex situations where multiple VFS shims are being loaded, +** it might be important to ensure that tmstmpvfs is loaded in the +** correct order so that it sequences itself into the default VFS +** Shim stack in the right order. +** +** When running the CLI, you can load this extension at invocation by +** adding a command-line option like this: "--vfs ./tmstmpvfs.so". +** The --vfs option usually specifies the symbolic name of a built-in VFS. +** But if the argument to --vfs is not a built-in VFS but is instead the +** name of a file, the CLI tries to load that file as an extension. Note +** that the full name of the extension file must be provided, including +** the ".so" or ".dylib" or ".dll" suffix. +** +** An application can see if the tmstmpvfs is being used by examining +** the results from SQLITE_FCNTL_VFSNAME (or the .vfsname command in +** the CLI). If the answer include "tmstmp", then this VFS is being +** used. +** +** USING +** +** Open database connections using the sqlite3_open() or +** sqlite3_open_v2() interfaces, as normal. Ordinary database files +** (without a timestamp) will operate normally. +** +** Timestamping only works on databases that have a reserve-bytes +** value of exactly 16. The default value for reserve-bytes is 0. +** Hence, newly created database files will omit the timestamp by +** default. To create a database that includes a timestamp, change +** the reserve-bytes value to 16 by running: +** +** int n = 16; +** sqlite3_file_control(db, 0, SQLITE_FCNTL_RESERVE_BYTES, &n); +** +** If you do this immediately after creating a new database file, +** before anything else has been written into the file, then that +** might be all that you need to do. Otherwise, the API call +** above should be followed by: +** +** sqlite3_exec(db, "VACUUM", 0, 0, 0); +** +** It never hurts to run the VACUUM, even if you don't need it. +** +** From the CLI, use the ".filectrl reserve_bytes 16" command, +** followed by "VACUUM;". +** +** SQLite allows the number of reserve-bytes to be increased, but +** not decreased. If you want to restore the reserve-bytes to 0 +** (to disable tmstmpvfs), the easiest approach is to use VACUUM INTO +** with a URI filename as the argument and include "reserve=0" query +** parameter on the URI. Example: +** +** VACUUM INTO 'file:notimestamps.db?reserve=0'; +** +** Then switch over to using the new database file. The reserve=0 query +** parameter only works on SQLite 3.52.0 and later. +** +** IMPLEMENTATION NOTES +** +** The timestamp information is stored in the last 16 bytes of each page. +** This module only operates if the "bytes of reserved space on each page" +** value at offset 20 the SQLite database header is exactly 16. If +** the reserved-space value is not 16, no timestamp information is added +** to database pages. Some, but not all, logfile entries will be made +** still, but the size of the logs will be greatly reduced. +** +** The timestamp layout is as follows: +** +** bytes 0,1 Zero. Reserved for future expansion +** bytes 2-7 Milliseconds since the Unix Epoch +** bytes 8-11 WAL frame number +** bytes 12 0: WAL write 2: rollback write +** bytes 13-15 Lower 24 bits of Salt-1 +** +** For transactions that occur in rollback mode, only the timestamp +** in bytes 2-7 and byte 12 are non-zero. Byte 12 is set to 2 for +** rollback writes. +** +** The 16-byte tag is added to each database page when the content +** is written into the database file itself. This shim does not make +** any changes to the page as it is written to the WAL file, since +** that would mess up the WAL checksum. +** +** LOGGING +** +** An open database connection that attempts to write to the database +** will create a log file if a directory name $(DATABASE)-tmstmp exists. +** The name of the log file is: +** +** $(TIME)-$(PID)-$(RANDOM) +** +** Where TIME is an ISO 8601 date in milliseconds with no punctuation, +** PID is the process ID, and RANDOM is a 32-bit random number expressed +** as hexadecimal. +** +** The log consists of 16-byte records. Each record consists of five +** unsigned integers: +** +** 1 1 6 4 4 <--- bytes +** op a1 ts a2 a3 +** +** The meanings of the a1-a3 values depend on op. ts is the timestamp +** in milliseconds since the unix epoch (1970-01-01 00:00:00). +** Opcodes are defined by the ELOG_* #defines below. +** +** ELOG_OPEN_DB "Open a connection to the database file" +** op = 0x01 +** a2 = process-ID +** +** ELOG_OPEN_WAL "Open a connection to the -wal file" +** op = 0x02 +** a2 = process-ID +** +** ELOG_WAL_PAGE "New page added to the WAL file" +** op = 0x03 +** a1 = 1 if last page of a txn. 0 otherwise. +** a2 = page number in the DB file +** a3 = frame number in the WAL file +** +** ELOG_DB_PAGE "Database page updated using rollback mode" +** op = 0x04 +** a2 = page number in the DB file +** +** ELOG_CKPT_START "Start of a checkpoint operation" +** op = 0x05 +** +** ELOG_CKPT_PAGE "Page xfer from WAL to database" +** op = 0x06 +** a2 = database page number +** a3 = frame number in the WAL file +** +** ELOG_CKPT_END "Start of a checkpoint operation" +** op = 0x07 +** +** ELOG_WAL_RESET "WAL file header overwritten" +** op = 0x08 +** a3 = Salt1 value +** +** ELOG_CLOSE_WAL "Close the WAL file connection" +** op = 0x0e +** +** ELOG_CLOSE_DB "Close the DB connection" +** op = 0x0f +** +** VIEWING TIMESTAMPS AND LOGS +** +** The command-line utility at tool/showtmlog.c will read and display +** the content of one or more tmstmpvfs.c log files. If all of the +** log files are stored in directory $(DATABASE)-tmstmp, then you can +** view them all using a command like shown below (with an extra "?" +** inserted on the wildcard to avoid closing the C-language comment +** that contains this text): +** +** showtmlog $(DATABASE)-tmstmp/?* +** +** The command-line utility at tools/showdb.c can be used to show the +** timestamps on pages of a database file, using a command like this: +** +** showdb --tmstmp $(DATABASE) pgidx +* +** The command above shows the timestamp and the intended use of every +** pages in the database, in human-readable form. If you also add +** the --csv option to the command above, then the command generates +** a Comma-Separated-Value (CSV) file as output, which contains a +** decoding of the complete timestamp tag on each page of the database. +** This CVS file can be easily imported into another SQLite database +** using a CLI command like the following: +** +** .import --csv '|showdb --tmstmp -csv orig.db pgidx' ts_table +** +** In the command above, the database containing the timestamps is +** "orig.db" and the content is imported into a new table named "ts_table". +** The "ts_table" is created automatically, using the column names found +** in the first line of the CSV file. All columns of the automatically +** created ts_table are of type TEXT. It might make more sense to +** create the table yourself, using more sensible datatypes, like this: +** +** CREATE TABLE ts_table ( +** pgno INT, -- page number +** tm REAL, -- seconds since 1970-01-01 +** frame INT, -- WAL frame number +** flg INT, -- flag (tag byte 12) +** salt INT, -- WAL salt (tag bytes 13-15) +** parent INT, -- Parent page number +** child INT, -- Index of this page in its parent +** ovfl INT, -- Index of this page on the overflow chain +** txt TEXT -- Description of this page +** ); +** +** Then import using: +** +** .import --csv --skip 1 '|showdb --tmstmp --csv orig.db pgidx' ts_table +** +** Note the addition of the "--skip 1" option on ".import" to bypass the +** first line of the CSV file that contains the column names. +** +** Both programs "showdb" and "showtmlog" can be built by running +** "make showtmlog showdb" from the top-level of a recent SQLite +** source tree. +*/ +#if defined(SQLITE_AMALGAMATION) && !defined(SQLITE_TMSTMPVFS_STATIC) +# define SQLITE_TMSTMPVFS_STATIC +#endif +#ifdef SQLITE_TMSTMPVFS_STATIC +# include "sqlite3.h" +#else +# include "sqlite3ext.h" + SQLITE_EXTENSION_INIT1 +#endif +#include +#include +#include + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct sqlite3_vfs TmstmpVfs; +typedef struct TmstmpFile TmstmpFile; +typedef struct TmstmpLog TmstmpLog; + +/* +** Bytes of reserved space used by this extension +*/ +#define TMSTMP_RESERVE 16 + +/* +** The magic number used to identify TmstmpFile objects +*/ +#define TMSTMP_MAGIC 0x2a87b72d + +/* +** Useful datatype abbreviations +*/ +#if !defined(SQLITE_AMALGAMATION) + typedef unsigned char u8; + typedef unsigned int u32; +#endif + +/* +** Current process id +*/ +#if defined(_WIN32) +# include +# define GETPID (u32)GetCurrentProcessId() +#else +# include +# define GETPID (u32)getpid() +#endif + +/* Access to a lower-level VFS that (might) implement dynamic loading, +** access to randomness, etc. +*/ +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) +#define ORIGFILE(p) ((sqlite3_file*)(((TmstmpFile*)(p))+1)) + +/* Information for the tmstmp log file. */ +struct TmstmpLog { + char *zLogname; /* Log filename */ + FILE *log; /* Open log file */ + int n; /* Bytes of a[] used */ + unsigned char a[16*6]; /* Buffered header for the log */ +}; + +/* An open WAL or DB file */ +struct TmstmpFile { + sqlite3_file base; /* IO methods */ + u32 uMagic; /* Magic number for sanity checking */ + u32 salt1; /* Last WAL salt-1 value */ + u32 iFrame; /* Last WAL frame number */ + u32 pgno; /* Current page number */ + u32 pgsz; /* Size of each page, in bytes */ + u8 isWal; /* True if this is a WAL file */ + u8 isDb; /* True if this is a DB file */ + u8 isCommit; /* Last WAL frame header was a transaction commit */ + u8 hasCorrectReserve; /* File has the correct reserve size */ + u8 inCkpt; /* True if in a checkpoint */ + TmstmpLog *pLog; /* Log file */ + TmstmpFile *pPartner; /* DB->WAL or WAL->DB mapping */ + sqlite3_int64 iOfst; /* Offset of last WAL frame header */ + sqlite3_vfs *pSubVfs; /* Underlying VFS */ +}; + +/* +** Event log opcodes +*/ +#define ELOG_OPEN_DB 0x01 +#define ELOG_OPEN_WAL 0x02 +#define ELOG_WAL_PAGE 0x03 +#define ELOG_DB_PAGE 0x04 +#define ELOG_CKPT_START 0x05 +#define ELOG_CKPT_PAGE 0x06 +#define ELOG_CKPT_DONE 0x07 +#define ELOG_WAL_RESET 0x08 +#define ELOG_CLOSE_WAL 0x0e +#define ELOG_CLOSE_DB 0x0f + +/* +** Methods for TmstmpFile +*/ +static int tmstmpClose(sqlite3_file*); +static int tmstmpRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int tmstmpWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int tmstmpTruncate(sqlite3_file*, sqlite3_int64 size); +static int tmstmpSync(sqlite3_file*, int flags); +static int tmstmpFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int tmstmpLock(sqlite3_file*, int); +static int tmstmpUnlock(sqlite3_file*, int); +static int tmstmpCheckReservedLock(sqlite3_file*, int *pResOut); +static int tmstmpFileControl(sqlite3_file*, int op, void *pArg); +static int tmstmpSectorSize(sqlite3_file*); +static int tmstmpDeviceCharacteristics(sqlite3_file*); +static int tmstmpShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); +static int tmstmpShmLock(sqlite3_file*, int offset, int n, int flags); +static void tmstmpShmBarrier(sqlite3_file*); +static int tmstmpShmUnmap(sqlite3_file*, int deleteFlag); +static int tmstmpFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int tmstmpUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for TmstmpVfs +*/ +static int tmstmpOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int tmstmpDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int tmstmpAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int tmstmpFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *tmstmpDlOpen(sqlite3_vfs*, const char *zFilename); +static void tmstmpDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*tmstmpDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void tmstmpDlClose(sqlite3_vfs*, void*); +static int tmstmpRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int tmstmpSleep(sqlite3_vfs*, int microseconds); +static int tmstmpCurrentTime(sqlite3_vfs*, double*); +static int tmstmpGetLastError(sqlite3_vfs*, int, char *); +static int tmstmpCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); +static int tmstmpSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr); +static sqlite3_syscall_ptr tmstmpGetSystemCall(sqlite3_vfs*, const char *z); +static const char *tmstmpNextSystemCall(sqlite3_vfs*, const char *zName); + +static sqlite3_vfs tmstmp_vfs = { + 3, /* iVersion (set when registered) */ + 0, /* szOsFile (set when registered) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "tmstmpvfs", /* zName */ + 0, /* pAppData (set when registered) */ + tmstmpOpen, /* xOpen */ + tmstmpDelete, /* xDelete */ + tmstmpAccess, /* xAccess */ + tmstmpFullPathname, /* xFullPathname */ + tmstmpDlOpen, /* xDlOpen */ + tmstmpDlError, /* xDlError */ + tmstmpDlSym, /* xDlSym */ + tmstmpDlClose, /* xDlClose */ + tmstmpRandomness, /* xRandomness */ + tmstmpSleep, /* xSleep */ + tmstmpCurrentTime, /* xCurrentTime */ + tmstmpGetLastError, /* xGetLastError */ + tmstmpCurrentTimeInt64, /* xCurrentTimeInt64 */ + tmstmpSetSystemCall, /* xSetSystemCall */ + tmstmpGetSystemCall, /* xGetSystemCall */ + tmstmpNextSystemCall /* xNextSystemCall */ +}; + +static const sqlite3_io_methods tmstmp_io_methods = { + 3, /* iVersion */ + tmstmpClose, /* xClose */ + tmstmpRead, /* xRead */ + tmstmpWrite, /* xWrite */ + tmstmpTruncate, /* xTruncate */ + tmstmpSync, /* xSync */ + tmstmpFileSize, /* xFileSize */ + tmstmpLock, /* xLock */ + tmstmpUnlock, /* xUnlock */ + tmstmpCheckReservedLock, /* xCheckReservedLock */ + tmstmpFileControl, /* xFileControl */ + tmstmpSectorSize, /* xSectorSize */ + tmstmpDeviceCharacteristics, /* xDeviceCharacteristics */ + tmstmpShmMap, /* xShmMap */ + tmstmpShmLock, /* xShmLock */ + tmstmpShmBarrier, /* xShmBarrier */ + tmstmpShmUnmap, /* xShmUnmap */ + tmstmpFetch, /* xFetch */ + tmstmpUnfetch /* xUnfetch */ +}; + +/* +** Write a 6-byte millisecond timestamp into aOut[] +*/ +static void tmstmpPutTS(TmstmpFile *p, unsigned char *aOut){ + sqlite3_uint64 tm = 0; + p->pSubVfs->xCurrentTimeInt64(p->pSubVfs, (sqlite3_int64*)&tm); + tm -= 210866760000000LL; + aOut[0] = (tm>>40)&0xff; + aOut[1] = (tm>>32)&0xff; + aOut[2] = (tm>>24)&0xff; + aOut[3] = (tm>>16)&0xff; + aOut[4] = (tm>>8)&0xff; + aOut[5] = tm&0xff; +} + +/* +** Read a 32-bit big-endian unsigned integer and return it. +*/ +static u32 tmstmpGetU32(const unsigned char *a){ + return (a[0]<<24) + (a[1]<<16) + (a[2]<<8) + a[3]; +} + +/* Write a 32-bit integer as big-ending into a[] +*/ +static void tmstmpPutU32(u32 v, unsigned char *a){ + a[0] = (v>>24) & 0xff; + a[1] = (v>>16) & 0xff; + a[2] = (v>>8) & 0xff; + a[3] = v & 0xff; +} + +/* Free a TmstmpLog object */ +static void tmstmpLogFree(TmstmpLog *pLog){ + if( pLog==0 ) return; + if( pLog->log ) fclose(pLog->log); + sqlite3_free(pLog->zLogname); + sqlite3_free(pLog); +} + +/* Flush log content. Open the file if necessary. Return the +** number of errors. */ +static int tmstmpLogFlush(TmstmpFile *p){ + TmstmpLog *pLog = p->pLog; + assert( pLog!=0 ); + if( pLog->log==0 ){ + pLog->log = fopen(pLog->zLogname, "wb"); + if( pLog->log==0 ){ + tmstmpLogFree(pLog); + p->pLog = 0; + return 1; + } + } + (void)fwrite(pLog->a, pLog->n, 1, pLog->log); + fflush(pLog->log); + pLog->n = 0; + return 0; +} + +/* +** Write a record onto the event log +*/ +static void tmstmpEvent( + TmstmpFile *p, + u8 op, + u8 a1, + u32 a2, + u32 a3, + u8 *pTS +){ + unsigned char *a; + TmstmpLog *pLog; + if( p->isWal ){ + p = p->pPartner; + assert( p!=0 ); + assert( p->isDb ); + } + pLog = p->pLog; + if( pLog==0 ) return; + if( pLog->n >= (int)sizeof(pLog->a) ){ + if( tmstmpLogFlush(p) ) return; + } + a = pLog->a + pLog->n; + a[0] = op; + a[1] = a1; + if( pTS ){ + memcpy(a+2, pTS, 6); + }else{ + tmstmpPutTS(p, a+2); + } + tmstmpPutU32(a2, a+8); + tmstmpPutU32(a3, a+12); + pLog->n += 16; + if( pLog->log || (op>=ELOG_WAL_PAGE && op<=ELOG_WAL_RESET) ){ + (void)tmstmpLogFlush(p); + } +} + +/* +** Close a connection +*/ +static int tmstmpClose(sqlite3_file *pFile){ + TmstmpFile *p = (TmstmpFile *)pFile; + if( p->hasCorrectReserve ){ + tmstmpEvent(p, p->isDb ? ELOG_CLOSE_DB : ELOG_CLOSE_WAL, 0, 0, 0, 0); + } + tmstmpLogFree(p->pLog); + if( p->pPartner ){ + assert( p->pPartner->pPartner==p ); + p->pPartner->pPartner = 0; + p->pPartner = 0; + } + pFile = ORIGFILE(pFile); + return pFile->pMethods->xClose(pFile); +} + +/* +** Read bytes from a file +*/ +static int tmstmpRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + int rc; + TmstmpFile *p = (TmstmpFile*)pFile; + pFile = ORIGFILE(pFile); + rc = pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst); + if( rc!=SQLITE_OK ) return rc; + if( p->isDb + && iOfst==0 + && iAmt>=100 + ){ + const unsigned char *a = (unsigned char*)zBuf; + p->hasCorrectReserve = (a[20]==TMSTMP_RESERVE); + p->pgsz = (a[16]<<8) + a[17]; + if( p->pgsz==1 ) p->pgsz = 65536; + if( p->pPartner ){ + p->pPartner->hasCorrectReserve = p->hasCorrectReserve; + p->pPartner->pgsz = p->pgsz; + } + } + if( p->isWal + && p->inCkpt + && iAmt>=512 && iAmt<=65535 && (iAmt&(iAmt-1))==0 + ){ + p->pPartner->iFrame = (iOfst-56)/(p->pgsz+24) + 1; + } + return rc; +} + +/* +** Write data to a tmstmp-file. +*/ +static int tmstmpWrite( + sqlite3_file *pFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + TmstmpFile *p = (TmstmpFile*)pFile; + sqlite3_file *pSub = ORIGFILE(pFile); + if( !p->hasCorrectReserve ){ + /* The database does not have the correct reserve size. No-op */ + }else if( p->isWal ){ + /* Writing into a WAL file */ + if( iAmt==24 ){ + /* A frame header */ + u32 x = 0; + p->iFrame = (iOfst - 32)/(p->pgsz+24)+1; + p->pgno = tmstmpGetU32((const u8*)zBuf); + p->salt1 = tmstmpGetU32(((const u8*)zBuf)+8); + memcpy(&x, ((const u8*)zBuf)+4, 4); + p->isCommit = (x!=0); + p->iOfst = iOfst; + }else if( iAmt>=512 && iOfst==p->iOfst+24 ){ + unsigned char s[TMSTMP_RESERVE]; + memset(s, 0, TMSTMP_RESERVE); + tmstmpPutTS(p, s+2); + tmstmpEvent(p, ELOG_WAL_PAGE, p->isCommit, p->pgno, p->iFrame, s+2); + }else if( iAmt==32 && iOfst==0 ){ + p->salt1 = tmstmpGetU32(((const u8*)zBuf)+16); + tmstmpEvent(p, ELOG_WAL_RESET, 0, 0, p->salt1, 0); + } + }else if( p->inCkpt ){ + unsigned char *s = (unsigned char*)zBuf+iAmt-TMSTMP_RESERVE; + memset(s, 0, TMSTMP_RESERVE); + tmstmpPutTS(p, s+2); + tmstmpPutU32(p->iFrame, s+8); + tmstmpPutU32(p->pPartner->salt1 & 0xffffff, s+12); + assert( p->pgsz>0 ); + tmstmpEvent(p, ELOG_CKPT_PAGE, 0, (iOfst/p->pgsz)+1, p->iFrame, 0); + }else if( p->pPartner==0 ){ + /* Writing into a database in rollback mode */ + unsigned char *s = (unsigned char*)zBuf+iAmt-TMSTMP_RESERVE; + memset(s, 0, TMSTMP_RESERVE); + tmstmpPutTS(p, s+2); + s[12] = 2; + assert( p->pgsz>0 ); + tmstmpEvent(p, ELOG_DB_PAGE, 0, (u32)(iOfst/p->pgsz)+1, 0, s+2); + } + return pSub->pMethods->xWrite(pSub,zBuf,iAmt,iOfst); +} + +/* +** Truncate a tmstmp-file. +*/ +static int tmstmpTruncate(sqlite3_file *pFile, sqlite_int64 size){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xTruncate(pFile, size); +} + +/* +** Sync a tmstmp-file. +*/ +static int tmstmpSync(sqlite3_file *pFile, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSync(pFile, flags); +} + +/* +** Return the current file-size of a tmstmp-file. +*/ +static int tmstmpFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + TmstmpFile *p = (TmstmpFile *)pFile; + pFile = ORIGFILE(p); + return pFile->pMethods->xFileSize(pFile, pSize); +} + +/* +** Lock a tmstmp-file. +*/ +static int tmstmpLock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xLock(pFile, eLock); +} + +/* +** Unlock a tmstmp-file. +*/ +static int tmstmpUnlock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xUnlock(pFile, eLock); +} + +/* +** Check if another file-handle holds a RESERVED lock on a tmstmp-file. +*/ +static int tmstmpCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xCheckReservedLock(pFile, pResOut); +} + +/* +** File control method. For custom operations on a tmstmp-file. +*/ +static int tmstmpFileControl(sqlite3_file *pFile, int op, void *pArg){ + int rc; + TmstmpFile *p = (TmstmpFile*)pFile; + pFile = ORIGFILE(pFile); + rc = pFile->pMethods->xFileControl(pFile, op, pArg); + switch( op ){ + case SQLITE_FCNTL_VFSNAME: { + if( p->hasCorrectReserve && rc==SQLITE_OK ){ + *(char**)pArg = sqlite3_mprintf("tmstmp/%z", *(char**)pArg); + } + break; + } + case SQLITE_FCNTL_CKPT_START: { + p->inCkpt = 1; + assert( p->isDb ); + assert( p->pPartner!=0 ); + p->pPartner->inCkpt = 1; + if( p->hasCorrectReserve ){ + tmstmpEvent(p, ELOG_CKPT_START, 0, 0, 0, 0); + } + rc = SQLITE_OK; + break; + } + case SQLITE_FCNTL_CKPT_DONE: { + p->inCkpt = 0; + assert( p->isDb ); + assert( p->pPartner!=0 ); + p->pPartner->inCkpt = 0; + if( p->hasCorrectReserve ){ + tmstmpEvent(p, ELOG_CKPT_DONE, 0, 0, 0, 0); + } + rc = SQLITE_OK; + break; + } + } + return rc; +} + +/* +** Return the sector-size in bytes for a tmstmp-file. +*/ +static int tmstmpSectorSize(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSectorSize(pFile); +} + +/* +** Return the device characteristic flags supported by a tmstmp-file. +*/ +static int tmstmpDeviceCharacteristics(sqlite3_file *pFile){ + int devchar = 0; + pFile = ORIGFILE(pFile); + devchar = pFile->pMethods->xDeviceCharacteristics(pFile); + return (devchar & ~SQLITE_IOCAP_SUBPAGE_READ); +} + +/* Create a shared memory file mapping */ +static int tmstmpShmMap( + sqlite3_file *pFile, + int iPg, + int pgsz, + int bExtend, + void volatile **pp +){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp); +} + +/* Perform locking on a shared-memory segment */ +static int tmstmpShmLock(sqlite3_file *pFile, int offset, int n, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmLock(pFile,offset,n,flags); +} + +/* Memory barrier operation on shared memory */ +static void tmstmpShmBarrier(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + pFile->pMethods->xShmBarrier(pFile); +} + +/* Unmap a shared memory segment */ +static int tmstmpShmUnmap(sqlite3_file *pFile, int deleteFlag){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmUnmap(pFile,deleteFlag); +} + +/* Fetch a page of a memory-mapped file */ +static int tmstmpFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xFetch(pFile, iOfst, iAmt, pp); +} + +/* Release a memory-mapped page */ +static int tmstmpUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xUnfetch(pFile, iOfst, pPage); +} + + +/* +** Open a tmstmp file handle. +*/ +static int tmstmpOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + TmstmpFile *p, *pDb; + sqlite3_file *pSubFile; + sqlite3_vfs *pSubVfs; + int rc; + + pSubVfs = ORIGVFS(pVfs); + if( (flags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_WAL))==0 ){ + /* If the file is not a persistent database or a WAL file, then + ** bypass the timestamp logic all together */ + return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags); + } + if( (flags & SQLITE_OPEN_WAL)!=0 ){ + pDb = (TmstmpFile*)sqlite3_database_file_object(zName); + if( pDb==0 + || pDb->uMagic!=TMSTMP_MAGIC + || !pDb->isDb + || pDb->pPartner!=0 + ){ + return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags); + } + }else{ + pDb = 0; + } + p = (TmstmpFile*)pFile; + memset(p, 0, sizeof(*p)); + pSubFile = ORIGFILE(pFile); + pFile->pMethods = &tmstmp_io_methods; + p->pSubVfs = pSubVfs; + p->uMagic = TMSTMP_MAGIC; + rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags); + if( rc ) goto tmstmp_open_done; + if( pDb!=0 ){ + p->isWal = 1; + p->pPartner = pDb; + pDb->pPartner = p; + }else{ + u32 r2; + u32 pid; + TmstmpLog *pLog; + sqlite3_uint64 r1; /* Milliseconds since 1970-01-01 */ + sqlite3_uint64 days; /* Days since 1970-01-01 */ + sqlite3_uint64 sod; /* Start of date specified by r1 */ + sqlite3_uint64 z; /* Days since 0000-03-01 */ + sqlite3_uint64 era; /* 400-year era */ + int h; /* hour */ + int m; /* minute */ + int s; /* second */ + int f; /* millisecond */ + int Y; /* year */ + int M; /* month */ + int D; /* day */ + int y; /* year assuming March is first month */ + unsigned int doe; /* day of 400-year era */ + unsigned int yoe; /* year of 400-year era */ + unsigned int doy; /* day of year */ + unsigned int mp; /* month with March==0 */ + + p->isDb = 1; + r1 = 0; + pLog = sqlite3_malloc64( sizeof(TmstmpLog) ); + if( pLog==0 ){ + pSubFile->pMethods->xClose(pSubFile); + rc = SQLITE_NOMEM; + goto tmstmp_open_done; + } + memset(pLog, 0, sizeof(pLog[0])); + p->pLog = pLog; + p->pSubVfs->xCurrentTimeInt64(p->pSubVfs, (sqlite3_int64*)&r1); + r1 -= 210866760000000LL; + days = r1/86400000; + sod = (r1%86400000)/1000; + f = (int)(r1%1000); + + h = sod/3600; + m = (sod%3600)/60; + s = sod%60; + z = days + 719468; + era = z/146097; + doe = (unsigned)(z - era*146097); + yoe = (doe - doe/1460 + doe/36524 - doe/146096)/365; + y = (int)yoe + era*400; + doy = doe - (365*yoe + yoe/4 - yoe/100); + mp = (5*doy + 2)/153; + D = doy - (153*mp + 2)/5 + 1; + M = mp + (mp<10 ? 3 : -9); + Y = y + (M <=2); + sqlite3_randomness(sizeof(r2), &r2); + pid = GETPID; + pLog->zLogname = sqlite3_mprintf( + "%s-tmstmp/%04d%02d%02dT%02d%02d%02d%03d-%08d-%08x", + zName, Y, M, D, h, m, s, f, pid, r2); + } + tmstmpEvent(p, p->isWal ? ELOG_OPEN_WAL : ELOG_OPEN_DB, 0, GETPID, 0, 0); + +tmstmp_open_done: + if( rc ) pFile->pMethods = 0; + return rc; +} + +/* +** All VFS interfaces other than xOpen are passed down into the Sub-VFS. +*/ +static int tmstmpDelete(sqlite3_vfs *p, const char *zName, int syncDir){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xDelete(pSub,zName,syncDir); +} +static int tmstmpAccess(sqlite3_vfs *p, const char *zName, int flags, int *pR){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xAccess(pSub,zName,flags,pR); +} +static int tmstmpFullPathname(sqlite3_vfs*p,const char *zName,int n,char *zOut){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xFullPathname(pSub,zName,n,zOut); +} +static void *tmstmpDlOpen(sqlite3_vfs *p, const char *zFilename){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xDlOpen(pSub,zFilename); +} +static void tmstmpDlError(sqlite3_vfs *p, int nByte, char *zErrMsg){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xDlError(pSub,nByte,zErrMsg); +} +static void(*tmstmpDlSym(sqlite3_vfs *p, void *pDl, const char *zSym))(void){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xDlSym(pSub,pDl,zSym); +} +static void tmstmpDlClose(sqlite3_vfs *p, void *pDl){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xDlClose(pSub,pDl); +} +static int tmstmpRandomness(sqlite3_vfs *p, int nByte, char *zOut){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xRandomness(pSub,nByte,zOut); +} +static int tmstmpSleep(sqlite3_vfs *p, int microseconds){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xSleep(pSub,microseconds); +} +static int tmstmpCurrentTime(sqlite3_vfs *p, double *prNow){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xCurrentTime(pSub,prNow); +} +static int tmstmpGetLastError(sqlite3_vfs *p, int a, char *b){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xGetLastError(pSub,a,b); +} +static int tmstmpCurrentTimeInt64(sqlite3_vfs *p, sqlite3_int64 *piNow){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xCurrentTimeInt64(pSub,piNow); +} +static int tmstmpSetSystemCall(sqlite3_vfs *p, const char *zName, + sqlite3_syscall_ptr x){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xSetSystemCall(pSub,zName,x); +} +static sqlite3_syscall_ptr tmstmpGetSystemCall(sqlite3_vfs *p, const char *z){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xGetSystemCall(pSub,z); +} +static const char *tmstmpNextSystemCall(sqlite3_vfs *p, const char *zName){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xNextSystemCall(pSub,zName); +} + +/* +** Register the tmstmp VFS as the default VFS for the system. +*/ +static int tmstmpRegisterVfs(void){ + int rc = SQLITE_OK; + sqlite3_vfs *pOrig = sqlite3_vfs_find(0); + if( pOrig==0 ) return SQLITE_ERROR; + if( pOrig==&tmstmp_vfs ) return SQLITE_OK; + tmstmp_vfs.iVersion = pOrig->iVersion; + tmstmp_vfs.pAppData = pOrig; + tmstmp_vfs.szOsFile = pOrig->szOsFile + sizeof(TmstmpFile); + rc = sqlite3_vfs_register(&tmstmp_vfs, 1); + return rc; +} + +#if defined(SQLITE_TMSTMPVFS_STATIC) +/* This variant of the initializer runs when the extension is +** statically linked. +*/ +int sqlite3_register_tmstmpvfs(const char *NotUsed){ + (void)NotUsed; + return tmstmpRegisterVfs(); +} +int sqlite3_unregister_tmstmpvfs(void){ + if( sqlite3_vfs_find("tmstmpvfs") ){ + sqlite3_vfs_unregister(&tmstmp_vfs); + } + return SQLITE_OK; +} +#endif /* defined(SQLITE_TMSTMPVFS_STATIC */ + +#if !defined(SQLITE_TMSTMPVFS_STATIC) +/* This variant of the initializer function is used when the +** extension is shared library to be loaded at run-time. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +/* +** This routine is called by sqlite3_load_extension() when the +** extension is first loaded. +***/ +int sqlite3_tmstmpvfs_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* not used */ + (void)db; /* not used */ + rc = tmstmpRegisterVfs(); + if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; + return rc; +} +#endif /* !defined(SQLITE_TMSTMPVFS_STATIC) */ diff --git a/ext/misc/vtablog.c b/ext/misc/vtablog.c index 2b3e30355..1e28495de 100644 --- a/ext/misc/vtablog.c +++ b/ext/misc/vtablog.c @@ -14,6 +14,8 @@ ** on stdout when its key interfaces are called. This is intended for ** interactive analysis and debugging of virtual table interfaces. ** +** HOW TO COMPILE: +** ** To build this extension as a separately loaded shared library or ** DLL, use compiler command-lines similar to the following: ** @@ -21,7 +23,7 @@ ** (mac) clang -fPIC -dynamiclib vtablog.c -o vtablog.dylib ** (windows) cl vtablog.c -link -dll -out:vtablog.dll ** -** Usage example: +** USAGE EXAMPLE: ** ** .load ./vtablog ** CREATE VIRTUAL TABLE temp.log USING vtablog( @@ -29,6 +31,23 @@ ** rows=25 ** ); ** SELECT * FROM log; +** +** ARGUMENTS TO CREATE VIRTUAL TABLE: +** +** In "CREATE VIRTUAL TABLE temp.log AS vtablog(ARGS....)" statement, the +** ARGS argument is a list of key-value pairs that can be any of the +** following. +** +** schema=TEXT Text is a CREATE TABLE statement that defines +** the schema of the new virtual table. +** +** rows=N The table as N rows. +** +** consume_order_by=N If the left-most ORDER BY terms is ASC and +** against column N (where the leftmost column +** is #1) then set the orderByConsumed=1 flag in +** xBestIndex. Or if the left-most ORDER BY is +** DESC and against column -N, do likewise. */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -49,6 +68,8 @@ struct vtablog_vtab { char *zName; /* Table name. argv[2] of xConnect/xCreate */ int nRow; /* Number of rows in the table */ int nCursor; /* Number of cursors created */ + int iConsumeOB; /* Consume the ORDER BY clause if on column N-th + ** and consumeOB=N or consumeOB=(-N) and DESC */ }; /* vtablog_cursor is a subclass of sqlite3_vtab_cursor which will @@ -180,6 +201,7 @@ static int vtablogConnectCreate( int rc; char *zSchema = 0; char *zNRow = 0; + char *zConsumeOB = 0; printf("%s.%s.%s():\n", argv[1], argv[2], isCreate ? "xCreate" : "xConnect"); @@ -203,6 +225,10 @@ static int vtablogConnectCreate( rc = SQLITE_ERROR; goto vtablog_end_connect; } + if( vtablog_string_parameter(pzErr, "consume_order_by", z, &zConsumeOB) ){ + rc = SQLITE_ERROR; + goto vtablog_end_connect; + } } if( zSchema==0 ){ zSchema = sqlite3_mprintf("%s","CREATE TABLE x(a,b);"); @@ -221,6 +247,10 @@ static int vtablogConnectCreate( pNew->nRow = 10; if( zNRow ) pNew->nRow = atoi(zNRow); printf(" nrow = %d\n", pNew->nRow); + if( zConsumeOB ) pNew->iConsumeOB = atoi(zConsumeOB); + if( pNew->iConsumeOB ){ + printf(" consume_order_by = %d\n", pNew->iConsumeOB); + } pNew->zDb = sqlite3_mprintf("%s", argv[1]); pNew->zName = sqlite3_mprintf("%s", argv[2]); } @@ -228,6 +258,7 @@ static int vtablogConnectCreate( vtablog_end_connect: sqlite3_free(zSchema); sqlite3_free(zNRow); + sqlite3_free(zConsumeOB); return rc; } static int vtablogCreate( @@ -514,16 +545,27 @@ static int vtablogBestIndex( } } printf(" nOrderBy: %d\n", p->nOrderBy); - for(i=0; inOrderBy; i++){ - printf(" orderby[%d]: col=%d desc=%d\n", - i, - p->aOrderBy[i].iColumn, - p->aOrderBy[i].desc); + if( p->nOrderBy ){ + for(i=0; inOrderBy; i++){ + printf(" orderby[%d]: col=%d desc=%d\n", + i, + p->aOrderBy[i].iColumn, + p->aOrderBy[i].desc); + } + if( pTab->iConsumeOB ){ + int N = p->aOrderBy[0].iColumn+1; + if( (p->aOrderBy[0].desc && N==-pTab->iConsumeOB) + || (!p->aOrderBy[0].desc && N==pTab->iConsumeOB) + ){ + p->orderByConsumed = 1; + } + } } p->estimatedCost = (double)500; p->estimatedRows = 500; printf(" idxNum=%d\n", p->idxNum); printf(" idxStr=NULL\n"); + printf(" sqlite3_vtab_distinct()=%d\n", sqlite3_vtab_distinct(p)); printf(" orderByConsumed=%d\n", p->orderByConsumed); printf(" estimatedCost=%g\n", p->estimatedCost); printf(" estimatedRows=%lld\n", p->estimatedRows); diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index 2f74906d9..c4862650b 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -393,7 +393,7 @@ static int zipfileConnect( rc = sqlite3_declare_vtab(db, ZIPFILE_SCHEMA); if( rc==SQLITE_OK ){ - pNew = (ZipfileTab*)sqlite3_malloc64((sqlite3_int64)nByte+nFile); + pNew = (ZipfileTab*)sqlite3_malloc64((i64)nByte+nFile); if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, nByte+nFile); pNew->db = db; @@ -539,14 +539,15 @@ static void zipfileCursorErr(ZipfileCsr *pCsr, const char *zFmt, ...){ static int zipfileReadData( FILE *pFile, /* Read from this file */ u8 *aRead, /* Read into this buffer */ - int nRead, /* Number of bytes to read */ + i64 nRead, /* Number of bytes to read */ i64 iOff, /* Offset to read from */ char **pzErrmsg /* OUT: Error message (from sqlite3_malloc) */ ){ size_t n; fseek(pFile, (long)iOff, SEEK_SET); - n = fread(aRead, 1, nRead, pFile); - if( (int)n!=nRead ){ + n = fread(aRead, 1, (long)nRead, pFile); + if( n!=(size_t)nRead ){ + sqlite3_free(*pzErrmsg); *pzErrmsg = sqlite3_mprintf("error in fread()"); return SQLITE_ERROR; } @@ -563,7 +564,7 @@ static int zipfileAppendData( fseek(pTab->pWriteFd, (long)pTab->szCurrent, SEEK_SET); n = fwrite(aWrite, 1, nWrite, pTab->pWriteFd); if( (int)n!=nWrite ){ - pTab->base.zErrMsg = sqlite3_mprintf("error in fwrite()"); + zipfileTableErr(pTab,"error in fwrite()"); return SQLITE_ERROR; } pTab->szCurrent += nWrite; @@ -704,7 +705,12 @@ static int zipfileScanExtra(u8 *aExtra, int nExtra, u32 *pmTime){ u8 *p = aExtra; u8 *pEnd = &aExtra[nExtra]; - while( pcds); if( rc!=SQLITE_OK ){ - *pzErr = sqlite3_mprintf("failed to read CDS at offset %lld", iOff); + zipfileTableErr(pTab, "failed to read CDS at offset %lld", iOff); }else if( aBlob==0 ){ rc = zipfileReadData( pFile, aRead, nExtra+nFile, iOff+ZIPFILE_CDS_FIXED_SZ, pzErr @@ -900,14 +907,15 @@ static int zipfileGetEntry( rc = zipfileReadData(pFile, aRead, szFix, pNew->cds.iOffset, pzErr); }else{ aRead = (u8*)&aBlob[pNew->cds.iOffset]; - if( (pNew->cds.iOffset + ZIPFILE_LFH_FIXED_SZ)>nBlob ){ + if( ((i64)pNew->cds.iOffset + ZIPFILE_LFH_FIXED_SZ)>nBlob ){ rc = zipfileCorrupt(pzErr); } } + memset(&lfh, 0, sizeof(lfh)); if( rc==SQLITE_OK ) rc = zipfileReadLFH(aRead, &lfh); if( rc==SQLITE_OK ){ - pNew->iDataOff = pNew->cds.iOffset + ZIPFILE_LFH_FIXED_SZ; + pNew->iDataOff = (i64)pNew->cds.iOffset + ZIPFILE_LFH_FIXED_SZ; pNew->iDataOff += lfh.nFile + lfh.nExtra; if( aBlob && pNew->cds.szCompressed ){ if( pNew->iDataOff + pNew->cds.szCompressed > nBlob ){ @@ -918,7 +926,7 @@ static int zipfileGetEntry( } } }else{ - *pzErr = sqlite3_mprintf("failed to read LFH at offset %d", + zipfileTableErr(pTab, "failed to read LFH at offset %d", (int)pNew->cds.iOffset ); } @@ -942,7 +950,7 @@ static int zipfileNext(sqlite3_vtab_cursor *cur){ int rc = SQLITE_OK; if( pCsr->pFile ){ - i64 iEof = pCsr->eocd.iOffset + pCsr->eocd.nSize; + i64 iEof = (i64)pCsr->eocd.iOffset + (i64)pCsr->eocd.nSize; zipfileEntryFree(pCsr->pCurrent); pCsr->pCurrent = 0; if( pCsr->iNextOff>=iEof ){ @@ -1008,7 +1016,7 @@ static void zipfileInflate( if( err!=Z_STREAM_END ){ zipfileCtxErrorMsg(pCtx, "inflate() failed (%d)", err); }else{ - sqlite3_result_blob(pCtx, aRes, nOut, zipfileFree); + sqlite3_result_blob(pCtx, aRes, (int)str.total_out, zipfileFree); aRes = 0; } } @@ -1180,12 +1188,12 @@ static int zipfileEof(sqlite3_vtab_cursor *cur){ static int zipfileReadEOCD( ZipfileTab *pTab, /* Return errors here */ const u8 *aBlob, /* Pointer to in-memory file image */ - int nBlob, /* Size of aBlob[] in bytes */ + i64 nBlob, /* Size of aBlob[] in bytes */ FILE *pFile, /* Read from this file if aBlob==0 */ ZipfileEOCD *pEOCD /* Object to populate */ ){ u8 *aRead = pTab->aBuffer; /* Temporary buffer */ - int nRead; /* Bytes to read from file */ + i64 nRead; /* Bytes to read from file */ int rc = SQLITE_OK; memset(pEOCD, 0, sizeof(ZipfileEOCD)); @@ -1206,7 +1214,7 @@ static int zipfileReadEOCD( } if( rc==SQLITE_OK ){ - int i; + i64 i; /* Scan backwards looking for the signature bytes */ for(i=nRead-20; i>=0; i--){ @@ -1217,9 +1225,7 @@ static int zipfileReadEOCD( } } if( i<0 ){ - pTab->base.zErrMsg = sqlite3_mprintf( - "cannot find end of central directory record" - ); + zipfileTableErr(pTab, "cannot find end of central directory record"); return SQLITE_ERROR; } @@ -1264,7 +1270,7 @@ static void zipfileAddEntry( } } -static int zipfileLoadDirectory(ZipfileTab *pTab, const u8 *aBlob, int nBlob){ +static int zipfileLoadDirectory(ZipfileTab *pTab, const u8 *aBlob, i64 nBlob){ ZipfileEOCD eocd; int rc; int i; @@ -1312,7 +1318,7 @@ static int zipfileFilter( }else if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ static const u8 aEmptyBlob = 0; const u8 *aBlob = (const u8*)sqlite3_value_blob(argv[0]); - int nBlob = sqlite3_value_bytes(argv[0]); + i64 nBlob = sqlite3_value_bytes(argv[0]); assert( pTab->pFirstEntry==0 ); if( aBlob==0 ){ aBlob = &aEmptyBlob; @@ -1510,7 +1516,7 @@ static int zipfileBegin(sqlite3_vtab *pVtab){ assert( pTab->pWriteFd==0 ); if( pTab->zFile==0 || pTab->zFile[0]==0 ){ - pTab->base.zErrMsg = sqlite3_mprintf("zipfile: missing filename"); + zipfileTableErr(pTab, "zipfile: missing filename"); return SQLITE_ERROR; } @@ -1520,9 +1526,9 @@ static int zipfileBegin(sqlite3_vtab *pVtab){ ** in main-memory until the transaction is committed. */ pTab->pWriteFd = sqlite3_fopen(pTab->zFile, "ab+"); if( pTab->pWriteFd==0 ){ - pTab->base.zErrMsg = sqlite3_mprintf( - "zipfile: failed to open file %s for writing", pTab->zFile - ); + zipfileTableErr(pTab, + "zipfile: failed to open file %s for writing", pTab->zFile + ); rc = SQLITE_ERROR; }else{ fseek(pTab->pWriteFd, 0, SEEK_END); @@ -1987,7 +1993,7 @@ struct ZipfileCtx { ZipfileBuffer cds; }; -static int zipfileBufferGrow(ZipfileBuffer *pBuf, int nByte){ +static int zipfileBufferGrow(ZipfileBuffer *pBuf, i64 nByte){ if( pBuf->n+nByte>pBuf->nAlloc ){ u8 *aNew; sqlite3_int64 nNew = pBuf->n ? pBuf->n*2 : 512; @@ -2036,7 +2042,7 @@ static void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){ char *zName = 0; /* Path (name) of new entry */ int nName = 0; /* Size of zName in bytes */ char *zFree = 0; /* Free this before returning */ - int nByte; + i64 nByte; memset(&e, 0, sizeof(e)); p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx)); diff --git a/ext/qrf/README.md b/ext/qrf/README.md new file mode 100644 index 000000000..4bb1790a4 --- /dev/null +++ b/ext/qrf/README.md @@ -0,0 +1,762 @@ +# SQLite Query Result Formatting Subsystem + +The "Query Result Formatter" or "QRF" subsystem is a C-language +subroutine that formats the output from an SQLite query for display using +a fix-width font, for example on a terminal window over an SSH connection. +The output format is configurable. The application can request various +table formats, with flexible column widths and alignments, row-oriented +formats, such as CSV and similar, as well as various special purpose formats +like JSON. + +For the first 25 years of SQLite's existance, the +[command-line interface](https://sqlite.org/cli.html) (CLI) +formatted query results using a hodge-podge of routines +that had grown slowly by accretion. The QRF was created +in fall of 2025 to refactor and reorganize this code into +a more usable form. The idea behind QRF is to implement all the +query result formatting capabilities of the CLI in a subroutine +that can be incorporated and reused by other applications. + +## 1.0 Overview Of Operation + +Suppose variable `sqlite3_stmt *pStmt` is a pointer to an SQLite +prepared statement that has been reset and bound and is ready to run. +Then to format the output from this prepared statement, use code +similar to the following: + +> ~~~ +sqlite3_qrf_spec spec; /* Format specification */ +char *zErrMsg; /* Text error message (optional) */ +char *zResult = 0; /* Formatted output written here */ +int rc; /* Result code */ + +memset(&spec, 0, sizeof(spec)); /* Initialize the spec */ +spec.iVersion = 1; /* Version number must be 1 */ +spec.pzOutput = &zResult; /* Write results in variable zResult */ +/* Optionally fill in other settings in spec here, as needed */ +zErrMsg = 0; /* Not required; just being pedantic */ +rc = sqlite3_format_query_result(pStmt, &spec, &zErrMsg); /* Format results */ +if( rc ){ + printf("Error (%d): %s\n", rc, zErrMsg); /* Report an error */ + sqlite3_free(zErrMsg); /* Free the error message text */ +}else{ + printf("%s", zResult); /* Report the results */ +} +sqlite3_free(zResult); /* Free memory used to hold results */ +~~~ + +The `sqlite3_qrf_spec` object describes the desired output format +and where to send the generated output. Most of the work in using +the QRF involves filling out the sqlite3_qrf_spec. + +### 1.1 Using QRF with SQL text + +If you start with SQL text instead of an sqlite3_stmt pointer, and +especially if the SQL text might comprise two or more statements, then +the SQL text needs to be converted into sqlite3_stmt objects separately. +If the original SQL text is in a variable `const char *zSql` and the +database connection is in variable `sqlite3 *db`, then code +similar to the following should work: + +> ~~~ +sqlite3_qrf_spec spec; /* Format specification */ +char *zErrMsg; /* Text error message (optional) */ +char *zResult = 0; /* Formatted output written here */ +sqlite3_stmt *pStmt; /* Next prepared statement */ +int rc; /* Result code */ + +memset(&spec, 0, sizeof(spec)); /* Initialize the spec */ +spec.iVersion = 1; /* Version number must be 1 */ +spec.pzOutput = &zResult; /* Write results in variable zResult */ +/* Optionally fill in other settings in spec here, as needed */ +zErrMsg = 0; /* Not required; just being pedantic */ +while( zSql && zSql[0] ){ + pStmt = 0; /* Not required; just being pedantic */ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); + if( rc!=SQLITE_OK ){ + printf("Error: %s\n", sqlite3_errmsg(db)); + }else{ + rc = sqlite3_format_query_result(pStmt, &spec, &zErrMsg); /* Get results */ + if( rc ){ + printf("Error (%d): %s\n", rc, zErrMsg); /* Report an error */ + sqlite3_free(zErrMsg); /* Free the error message text */ + }else{ + printf("%s", zResult); /* Report the results */ + sqlite3_free(zResult); /* Free memory used to hold results */ + zResult = 0; + } + } + sqlite3_finalize(pStmt); +} +~~~ + + +## 2.0 The `sqlite3_qrf_spec` object + +The `sqlite3_qrf_spec` looks like this: + +> ~~~ +typedef struct sqlite3_qrf_spec sqlite3_qrf_spec; +struct sqlite3_qrf_spec { + unsigned char iVersion; /* Version number of this structure */ + unsigned char eStyle; /* Formatting style. "box", "csv", etc... */ + unsigned char eEsc; /* How to escape control characters in text */ + unsigned char eText; /* Quoting style for text */ + unsigned char eTitle; /* Quating style for the text of column names */ + unsigned char eBlob; /* Quoting style for BLOBs */ + unsigned char bTitles; /* True to show column names */ + unsigned char bWordWrap; /* Try to wrap on word boundaries */ + unsigned char bTextJsonb; /* Render JSONB blobs as JSON text */ + unsigned char eDfltAlign; /* Default alignment, no covered by aAlignment */ + unsigned char eTitleAlign; /* Alignment for column headers */ + unsigned char bSplitColumn; /* Wrap single-column output into many columns */ + unsigned char bBorder; /* Show outer border in Box and Table styles */ + short int nWrap; /* Wrap columns wider than this */ + short int nScreenWidth; /* Maximum overall table width */ + short int nLineLimit; /* Maximum number of lines for any row */ + short int nTitleLimit; /* Maximum number of characters in a title */ + int nCharLimit; /* Maximum number of characters in a cell */ + int nWidth; /* Number of entries in aWidth[] */ + int nAlign; /* Number of entries in aAlignment[] */ + short int *aWidth; /* Column widths */ + unsigned char *aAlign; /* Column alignments */ + char *zColumnSep; /* Alternative column separator */ + char *zRowSep; /* Alternative row separator */ + char *zTableName; /* Output table name */ + char *zNull; /* Rendering of NULL */ + char *(*xRender)(void*,sqlite3_value*); /* Render a value */ + int (*xWrite)(void*,const char*,sqlite3_int64); /* Write output */ + void *pRenderArg; /* First argument to the xRender callback */ + void *pWriteArg; /* First argument to the xWrite callback */ + char **pzOutput; /* Storage location for output string */ + /* Additional fields may be added in the future */ +}; +~~~ + +Do not be alarmed by the complexity of this structure. Everything can +be zeroed except for: + + * `.iVersion` + * One of `.pzOutput` or `.xWrite`. + +You do not need to understand and configure every field of this object +in order to use QRF effectively. Start by zeroing out the whole structure, +then initializing iVersion and one of pzOutput or xWrite. Then maybe +tweak one or two other settings to get the output you want. + +Further detail on the meanings of each of the fields in the +`sqlite3_qrf_spec` object is in the subsequent sections. + +### 2.1 Structure Version Number + +The sqlite3_qrf_spec.iVersion field must be 1. Future enhancements to +the QRF might add new fields to the bottom of the sqlite3_qrf_spec +object. Those new fields will only be accessible if the iVersion is greater +than 1. Thus the iVersion field is used to support upgradability. + +### 2.2 Output Deposition (xWrite and pzOutput) + +The formatted output can either be sent to a callback function +or accumulated into an output buffer in memory obtained +from sqlite3_malloc(). If the sqlite3_qrf_spec.xWrite column is not NULL, +then that function is invoked (using sqlite3_qrf_spec.xWriteArg as its +first argument) to transmit the formatted output. Or, if +sqlite3_qrf_spec.pzOutput points to a pointer to a character, then that +pointer is made to point to memory obtained from sqlite3_malloc() that +contains the complete text of the formatted output. If spec.pzOutput\[0\] +is initially non-NULL, then it is assumed to already point to memory obtained +from sqlite3_malloc(). In that case, the buffer is resized using +sqlite3_realloc() and the new text is appended. + +One of either sqlite3_qrf_spec.xWrite and sqlite3_qrf_spec.pzOutput must be +non-NULL and the other must be NULL. + +The return value from xWrite is an SQLITE result code. The usual return +should be SQLITE_OK. But if for some reason the write fails, a different +value might be returned. + +### 2.3 Output Format + +The sqlite3_qrf_spec.eStyle field is an integer code that defines the +specific output format that will be generated. See [section 4.0](#style) +below for details on the meaning of the various style options. + +Other fields in sqlite3_qrf_spec might be used or might be +ignored, depending on the value of eStyle. + +### 2.4 Show Column Names (bTitles) + +The sqlite3_qrf_spec.bTitles field can be either QRF_SW_Auto, +QRF_SW_On, or QRF_SW_Off. Those three constants also have shorter +alternative spellings: QRF_Auto, QRF_No, and +QRF_Yes. + +> ~~~ +#define QRF_SW_Auto 0 /* Let QRF choose the best value */ +#define QRF_SW_Off 1 /* This setting is forced off */ +#define QRF_SW_On 2 /* This setting is forced on */ +#define QRF_Auto 0 /* Alternate spelling for QRF_SW_Auto and others */ +#define QRF_No 1 /* Alternate spelling for QRF_SW_Off */ +#define QRF_Yes 2 /* Alternate spelling for QRF_SW_On */ +~~~ + +If the value is QRF_Yes, then column names appear in the output. +If the value is QRF_No, column names are omitted. If the +value is QRF_Auto, then an appropriate default is chosen. + +### 2.5 Control Character Escapes (eEsc) + +The sqlite3_qrf_spec.eEsc determines how ASCII control characters are +formatted when displaying TEXT values in the result. These are the allowed +values: + +> ~~~ +#define QRF_ESC_Auto 0 /* Choose the ctrl-char escape automatically */ +#define QRF_ESC_Off 1 /* Do not escape control characters */ +#define QRF_ESC_Ascii 2 /* Unix-style escapes. Ex: U+0007 shows ^G */ +#define QRF_ESC_Symbol 3 /* Unicode escapes. Ex: U+0007 shows U+2407 */ +~~~ + +If the value of eEsc is QRF_ESC_Ascii, then the control character +with value X is displayed as ^Y where Y is X+0x40. Hence, a +backspace character (U+0008) is shown as "^H". + +If eEsc is QRF_ESC_Symbol, then control characters in the range of U+0001 +through U+001f are mapped into U+2401 through U+241f, respectively. + +If the value of eEsc is QRF_ESC_Off, then no translation occurs +and control characters that appear in TEXT strings are transmitted +to the formatted output as-is. This can be dangerous in applications, +since an adversary who can control TEXT values might be able to +inject ANSI cursor movement sequences to hide nefarious values. + +The QRF_ESC_Auto value for eEsc means that the query result formatter +gets to pick whichever control-character encoding it thinks is best for +the situation. This will usually be QRF_ESC_Ascii. + +The TAB (U+0009), LF (U+000a) and CR-LF (U+000d,U+000a) character +sequence are always output literally and are not mapped to alternative +display values, regardless of this setting. + +### 2.6 Display of TEXT values (eText, eTitle) + +The sqlite3_qrf_spec.eText controls how text values are rendered in the +display. sqlite3_qrf_spec.eTitle controls how column names are rendered. +Both fields can have one of the following values: + +> ~~~ +#define QRF_TEXT_Auto 0 /* Choose text encoding automatically */ +#define QRF_TEXT_Plain 1 /* Literal text */ +#define QRF_TEXT_Sql 2 /* Quote as an SQL literal */ +#define QRF_TEXT_Csv 3 /* CSV-style quoting */ +#define QRF_TEXT_Html 4 /* HTML-style quoting */ +#define QRF_TEXT_Tcl 5 /* C/Tcl quoting */ +#define QRF_TEXT_Json 6 /* JSON quoting */ +#define QRF_TEXT_Relaxed 7 /* Relaxed SQL quoting */ +~~~ + +A value of QRF_TEXT_Auto means that the query result formatter will choose +what it thinks will be the best text encoding. + +A value of QRF_TEXT_Plain means that text values appear in the output exactly +as they are found in the database file, with no translation. + +A value of QRF_TEXT_Sql means that text values are escaped so that they +look like SQL literals. That means the value will be surrounded by +single-quotes (U+0027) and any single-quotes contained within the text +will be doubled. + +QRF_TEXT_Relaxed is similar to QRF_TEXT_Sql, except that it automatically +reverts to QRF_TEXT_Plain if the value to be displayed does not contain +special characters and is not easily confused with a NULL or a numeric +value. QRF_TEXT_Relaxed strives to minimize the amount of quoting syntax +while keeping the result unambiguous and easy for humans to read. The +precise rules for when quoting is omitted in QRF_TEXT_Relaxed, and when +it is applied, might be adjusted in future releases. + +A value of QRF_TEXT_Csv means that text values are escaped in accordance +with RFC 4180, which defines Comma-Separated-Value or CSV files. +Text strings that contain no special values appears as-is. Text strings +that contain special values are contained in double-quotes (U+0022) and +any double-quotes within the value are doubled. + +A value of QRF_TEXT_Html means that text values are escaped for use in +HTML. Special characters "<", "&", ">", """, and "'" +are displayed as "&lt;", "&amp;", "&gt;", "&quot;", +and "&#39;", respectively. + +A value of QRF_TEXT_Tcl means that text values are displayed inside of +double-quotes and special characters within the string are escaped using +backslash escape, as in ANSI-C or TCL or Perl or other popular programming +languages. + +A value of QRF_TEXT_Json gives similar results as QRF_TEXT_Tcl except that the +rules are adjusted so that the displayed string is strictly conforming +the JSON specification. + +### 2.7 How to display BLOB values (eBlob and bTextJsonb) + +If the sqlite3_qrf_spec.bTextJsonb flag is QRF_SW_On and if the value to be +displayed is JSONB, then the JSONB is translated into text JSON and the +text is shown according to the sqlite3_qrf_spec.eText setting as +described in the previous section. + +If the bTextJsonb flag is QRF_SW_Off (the usual case) or if the BLOB value to +be displayed is not JSONB, then the sqlite3_qrf_spec.eBlob field determines +how the BLOB value is formatted. The following options are available; + +> ~~~ +#define QRF_BLOB_Auto 0 /* Determine BLOB quoting using eText */ +#define QRF_BLOB_Text 1 /* Display content exactly as it is */ +#define QRF_BLOB_Sql 2 /* Quote as an SQL literal */ +#define QRF_BLOB_Hex 3 /* Hexadecimal representation */ +#define QRF_BLOB_Tcl 4 /* "\000" notation */ +#define QRF_BLOB_Json 5 /* A JSON string */ +#define QRF_BLOB_Size 6 /* Display the blob size only */ +~~~ + +A value of QRF_BLOB_Auto means that display format is selected automatically +by sqlite3_format_query_result() based on eStyle and eText. + +A value of QRF_BLOB_Text means that BLOB values are interpreted as UTF8 +text and are displayed using formatting results set by eEsc and +eText. + +A value of QRF_BLOB_Sql means that BLOB values are shown as SQL BLOB +literals: a prefix "`x'`" following by hexadecimal and ending with a +final "`'`". + +A value of QRF_BLOB_Hex means that BLOB values are shown as +hexadecimal text with no delimiters. + +A value of QRF_BLOB_Tcl means that BLOB values are shown as a +C/Tcl/Perl string literal where every byte is an octal backslash +escape. So a BLOB of `x'052881f3'` would be displayed as +`"\005\050\201\363"`. + +A value of QRF_BLOB_Json is similar to QRF_BLOB_Tcl except that is +uses unicode backslash escapes, since JSON does not understand +the C/Tcl/Perl octal backslash escapes. So the string from the +previous paragraph would be shown as +`"\u0005\u0028\u0081\u00f3"`. + +A value of QRF_BLOB_Size does not show any BLOB content at all. +Instead, it substitutes a text string that says how many bytes +the BLOB contains. + +### 2.8 Maximum size of displayed content (nLineLimit, nCharLimit, nTitleLimit) + +If the sqlite3_qrf_spec.nCharLimit setting is non-zero, then the formatter +will display only the first nCharLimit characters of each value. +Only characters that take up space are counted when enforcing this +limit. Zero-width characters and VT100 escape sequences do not count +toward this limit. The count is in characters, not bytes. When +imposing this limit, the formatter adds the three characters "..." +to the end of the value. Those added characters are not counted +as part of the limit. Very small limits still result in truncation, +but might render a few more characters than the limit. + +If the sqlite3_qrf_spec.nLineLimit setting is non-zero, then the +formatter will only display the first nLineLimit lines of each value. +It does not matter if the value is split because it contains a newline +character, or if it split by wrapping. This setting merely limits +the number of displayed lines. The nLineLimit setting currently only +works for **Box**, **Column**, **Line**, **Markdown**, and **Table** +styles, though that limitation might change in future releases. + +The idea behind both of these settings is to prevent large renderings +when doing a query that (unexpectedly) contains very large text or +blob values: perhaps megabyes of text. + +If the sqlite3_qrf_spec.nTitleLimit is non-zero, then the formatter +attempts to limits the size of column titles to at most nTitleLimit +display characters in width and a single line of text. The nTitleLimit +is useful for queries that have result columns that are scalar +subqueries or complex expressions. If those columns lack an AS +clause, then the name of the column will be a copy of the expression +that defines the column, which in some queries can be hundreds of +characters and multiple lines in length, which can reduce the readability +of tabular displays. An nTitleLimit somewhere in the range of 10 to 20. +can improve readability. The nTitleLimit setting currently only +works for **Box**, **Column**, **Line**, **Markdown**, and **Table** +styles, though that limitation might change in future releases. + +### 2.9 Word Wrapping In Columnar Styles (nWrap, bWordWrap) + +When using columnar formatting modes (QRF_STYLE_Box, QRF_STYLE_Column, +QRF_STYLE_Markdown, or QRF_STYLE_Table), the formatter attempts to limit +the width of any individual column to sqlite3_qrf_spec.nWrap characters +if nWrap is non-zero. A zero value for nWrap means "unlimited". +The nWrap limit might be exceeded if the limit is very small. + +In order to keep individual columns within requested width limits, +it is sometimes necessary to wrap the content for a single row of +a single column across multiple lines. When this +becomes necessary and if the bWordWrap setting is QRF_Yes, then the +formatter attempts to split the content on whitespace or at a word boundary. +If bWordWrap is QRF_No, then the formatter is free to split content +anywhere, including in the middle of a word. + +For narrow columns and wide words, it might sometimes be necessary to split +a column in the middle of a word, even when bWordWrap is QRF_Yes. + +### 2.10 Helping The Output To Fit On The Terminal (nScreenWidth) + +The sqlite3_qrf_spec.nScreenWidth field can be set the number of +characters that will fit on one line on the viewer output device. +This is typically a number like 80 or 132. The formatter will attempt +to reduce the length of output lines, depending on the style, so +that all output fits on that screen. + +A value of zero for nScreenWidth means "unknown" or "no width limit". +When the value is zero, the formatter makes no attempt to keep the +lines of output short. + +The nScreenWidth is a hint to the formatter, not a requirement. +The formatter trieds to keep lines below the nScreenWidth limit, +but it does not guarantee that it will. + +The nScreenWidth field currently only makes a difference in +columnar styles (**Box**, **Column**, **Markdown**, and **Table**) +and in the **Line** style. + +### 2.11 Individual Column Width (nWidth and aWidth) + +The sqlite3_qrf_spec.aWidth field is a pointer to an array of +signed 16-bit integers that control the width of individual columns +in columnar output modes (QRF_STYLE_Box, QRF_STYLE_Column, +QRF_STYLE_Markdown, or QRF_STYLE_Table). The sqlite3_qrf_spec.nWidth +field is the number of integers in the aWidth array. + +If aWidth is a NULL pointer or if nWidth is zero, then the array is +assumed to be all zeros. If nWidth is less then the number of +columns in the output, then zero is used for the width +for all columns past then end of the aWidth array. + +The aWidth array is deliberately an array of 16-bit signed integers. +Only 16 bits are used because no good comes for having very large +column widths. The range if further restricted as follows: + +> ~~~ +#define QRF_MAX_WIDTH 10000 /* Maximum column width */ +#define QRF_MIN_WIDTH 0 /* Minimum column width */ +~~~ + +A width greater than then QRF_MAX_WIDTH is interpreted as QRF_MAX_WIDTH. + +Any aWidth\[\] value of zero means the formatter should use a flexible +width column (limited only by sqlite_qrf_spec.mxWidth) that is just +big enough to hold the largest row. + +For historical compatibility, aWidth\[\] can contain negative values, +down to -QRF_MAX_WIDTH. The column width used is the absolute value +of the number in aWidth\[\]. The only difference is that negative +values cause the default horizontal alignment to be QRF_ALIGN_Right. +The sign of the aWidth\[\] values only affects alignment if the +alignment is not otherwise specified by aAlign\[\] or eDfltAlign. +Again, negative values for aWidth\[\] entries are supported for +backwards compatibility only, and are not recommended for new +applications. + +### 2.12 Alignment (nAlignment, aAlignment, eDfltAlign, eTitleAlign) + +Some cells in a display table might contain a lot of text and thus +be wide, or they might contain newline characters or be wrapped by +width constraints so that they span many rows of text. Other cells +might be narrower and shorter. In columnar formats, the display width +of a cell is the maximum of the widest value in the same column, and the +display height is the height of the tallest value in the same row. +So some cells might be much taller and wider than necessary to hold +their values. + +Alignment determines where smaller values are placed within larger cells. + +The sqlite3_qrf_spec.aAlign field points to an array of unsigned characters +that specifies alignment (both vertical and horizontal) of individual +columns within the table. The sqlite3_qrf_spec.nAlign fields holds +the number of entries in the aAlign\[\] array. + +If sqlite3_qrf_spec.aAlign is a NULL pointer or if sqlite3_qrf_spec.nAlign +is zero, or for columns to the right of what are specified by +sqlite3_qrf_spec.nAlign, the sqlite3_qrf_spec.eDfltAlign value is used +for the alignment. Column names can be (and often are) aligned +differently, as specified by sqlite3_qrf_spec.eTitleAlign. + +Each alignment value specifies both vertical and horizontal alignment. +Horizontal alignment can be left, center, right, or no preference. +Vertical alignment can be top, middle, bottom, or no preference. +Thus there are 16 possible alignment values, as follows: + +> ~~~ +/* +** Horizontal Vertial +** ---------- -------- */ +#define QRF_ALIGN_Auto 0 /* auto auto */ +#define QRF_ALIGN_Left 1 /* left auto */ +#define QRF_ALIGN_Center 2 /* center auto */ +#define QRF_ALIGN_Right 3 /* right auto */ +#define QRF_ALIGN_Top 4 /* auto top */ +#define QRF_ALIGN_NW 5 /* left top */ +#define QRF_ALIGN_N 6 /* center top */ +#define QRF_ALIGN_NE 7 /* right top */ +#define QRF_ALIGN_Middle 8 /* auto middle */ +#define QRF_ALIGN_W 9 /* left middle */ +#define QRF_ALIGN_C 10 /* center middle */ +#define QRF_ALIGN_E 11 /* right middle */ +#define QRF_ALIGN_Bottom 12 /* auto bottom */ +#define QRF_ALIGN_SW 13 /* left bottom */ +#define QRF_ALIGN_S 14 /* center bottom */ +#define QRF_ALIGN_SE 15 /* right bottom */ +~~~ + +Notice how alignment values with an unspecified horizontal +or vertical component can be added to another alignment value +for which that component is specified, to get a fully +specified alignment. For eample: + +> QRF_ALIGN_Center + QRF_ALIGN_Bottom == QRF_ALIGN_S. + +The alignment for column names is always determined by the +eTitleAlign setting. If eTitleAlign is QRF_Auto, then column +names use center-bottom alignment, QRF_ALIGN_W, value 14. +The aAlign\[\] and eDfltAlign settings have no affect on +column names. + +For data in the first nAlign columns, the aAlign\[\] array +entry for that column takes precedence. If either the horizontal +or vertical alignment has an "auto" value for that column or if +a column is beyond the first nAlign entries, then eDfltAlign +is used as a backup. If neither aAlign\[\] nor eDfltAlign +specify a horizontal alignment, then values are right-aligned +(QRF_ALIGN_Right) if they are numeric and left-aligned +(QRF_ALIGN_Left) otherwise. If neither aAlign\[\] nor eDfltAlign +specify a vertical alignment, then values are top-aligned +(QRF_ALIGN_Top). + +*As of 2025-11-08, only horizontal alignment is implemented. +The vertical alignment settings are currently ignored and +the vertical alignment is always QRF_ALIGN_Top.* + +### 2.13 Row and Column Separator Strings + +The sqlite3_qrf_spec.zColumnSep and sqlite3_qrf_spec.zRowSep strings +are alternative column and row separator character sequences. If not +specified (if these pointers are left as NULL) then appropriate defaults +are used. Some output styles have hard-coded column and row separators +and these settings are ignored for those styles. + +### 2.14 The Output Table Name + +The sqlite3_qrf_spec.zTableName value is the name of the output table +when eStyle is QRF_STYLE_Insert. + +### 2.15 The Rendering Of NULL (zNull) + +If a value is NULL then show the NULL using the string +found in sqlite3_qrf_spec.zNull. If zNull is itself a NULL pointer +then NULL values are rendered as an empty string. + +### 2.16 Optional Value Rendering Callback + +If the sqlite3_qrf_spec.xRender field is not NULL, then each +sqlite3_value coming out of the query is first passed to the +xRender function, giving that function an opportunity to render +the results itself, using whatever custom format is desired. +If xRender chooses to render, it should write the rendering +into memory obtained from sqlite3_malloc() and return a pointer +to that memory. The xRender function can decline +to render (for example, based on the sqlite3_value_type() or other +characteristics of the value) in which case it can simply return a +NULL pointer and the usual default rendering will be used instead. + +The sqlite3_format_query_result() function (which calls xRender) +will take responsibility for freeing the string returned by xRender +after it has finished using it. + +The eText, eBlob, and eEsc settings above become no-ops if the xRender +routine returns non-NULL. In other words, the application-supplied +xRender routine is expected to do all of its own quoting and formatting. + +The xRender routine is expected to do character length limiting itself. +So the nCharLimit setting becomes a no-op if xRender is used. However +the nLineLimit setting is still applied. The nTitleLimit setting is +not applicable to xRender because title values come from the +sqlite3_column_name() interface not from sqlite3_column_value(), +and so that names of columns are never processed by xRender. + +## 3.0 The `sqlite3_format_query_result()` Interface + +Invoke the `sqlite3_format_query_result(P,S,E)` interface to run +the prepared statement P and format its results according to the +specification found in S. The sqlite3_format_query_result() function +will return an SQLite result code, usually SQLITE_OK, but perhaps +SQLITE_NOMEM or SQLITE_ERROR or similar. If an error occurs and if +the E parameter is not NULL, then error message text might be written +into *E. Any error message text will be stored in memory obtained +from sqlite3_malloc() and it is the responsibility of the caller to +free that memory by a subsequent call to sqlite3_free(). + + +## 4.0 Output Styles + +The result formatter supports a variety of output styles. The +output style (sometimes called "output mode") is determined by +the eStyle field of the sqlite3_qrf_spec object. The set of +supported output modes might increase in future versions. +The following output modes are currently defined: + +> ~~~ +#define QRF_STYLE_Auto 0 /* Choose a style automatically */ +#define QRF_STYLE_Box 1 /* Unicode box-drawing characters */ +#define QRF_STYLE_Column 2 /* One record per line in neat columns */ +#define QRF_STYLE_Count 3 /* Output only a count of the rows of output */ +#define QRF_STYLE_Csv 4 /* Comma-separated-value */ +#define QRF_STYLE_Eqp 5 /* Format EXPLAIN QUERY PLAN output */ +#define QRF_STYLE_Explain 6 /* EXPLAIN output */ +#define QRF_STYLE_Html 7 /* Generate an XHTML table */ +#define QRF_STYLE_Insert 8 /* Generate SQL "insert" statements */ +#define QRF_STYLE_Json 9 /* Output is a list of JSON objects */ +#define QRF_STYLE_JObject 10 /* Independent JSON objects for each row */ +#define QRF_STYLE_Line 11 /* One column per line. */ +#define QRF_STYLE_List 12 /* One record per line with a separator */ +#define QRF_STYLE_Markdown 13 /* Markdown formatting */ +#define QRF_STYLE_Off 14 /* No query output shown */ +#define QRF_STYLE_Quote 15 /* SQL-quoted, comma-separated */ +#define QRF_STYLE_Stats 16 /* EQP-like output but with performance stats */ +#define QRF_STYLE_StatsEst 17 /* EQP-like output with planner estimates */ +#define QRF_STYLE_StatsVm 18 /* EXPLAIN-like output with performance stats */ +#define QRF_STYLE_Table 19 /* MySQL-style table formatting */ +~~~ + +In the following subsections, these styles will often be referred +to without the "QRF_STYLE_" prefix. + +### 4.1 Default Style (Auto) + +The **Auto** style means QRF gets to choose an appropriate output +style. It will usually choose **Box**, but might also pick one of +**Explain** or **Eqp** if the `sqlite3_stmt_explain()` function +returns 1 or 2, respectively. + +### 4.2 Columnar Styles (Box, Column, Markdown, Table) + +The **Box**, **Column**, **Markdown**, and **Table** +modes are columnar. This means the output is arranged into neat, +uniform-width columns. These styles can use more memory, especially when +the query result has many rows, because they need to load the entire output +into memory first in order to determine how wide to make each column. + +The nWidth, aWidth, and mxWidth fields of the `sqlite3_qrf_spec` object +are used by these styles only, and are ignored by all other styles. +The zRowSep and zColumnSep settings are ignored by these styles. The +bTitles setting is honored by these styles; it defaults to QRF_SW_On. + +The **Box** style uses Unicode box-drawing character to draw a grid +of columns and rows to show the result. The **Table** is the same, +except that it uses ASCII-art rather than Unicode box-drawing characters +to draw the grid. The **Column** arranges the results in neat columns +but does not draw in column or row separator, except that it does draw +lines horizontal lines using "`-`" characters to separate the column names +from the data below. This is very similar to default output styling in +psql. The **Markdown** renders its result in the Markdown table format. + +The **Box** and **Table** styles normally have a border that surrounds +the entire result. However, if sqlite3_qrf_spec.bBorder is QRF_No, then +that border is omitted, saving a little space both horizontally and +vertically. + +#### 4.2.1 Split Column Mode + +If the bSplitColumn field is QRF_Yes, and eStyle is QRF_STYLE_Column, +and bTitles is QRF_No, and nScreenWidth is greater than zero, and if +the query only returns a single column, then a special rendering known +as "Split Column Mode" will be used. In split column mode, instead +of showing all results in one tall column, the content wraps vertically +so that it appears on the screen as multiple columns, as many as will +fit in the available screen width. + +### 4.3 Line-oriented Styles + +The line-oriented styles output each row of result as it is received from +the prepared statement. + +The **List** style is the most familiar line-oriented output format. +The **List** style shows output columns for each row on the +same line, each separated by a single "`|`" character and with lines +terminated by a single newline (\\u000a or \\n). These column +and row separator choices can be overridden using the zColumnSep +and zRowSep fields of the `sqlite3_qrf_spec` structure. The text +formatting is QRF_TEXT_Plain, and BLOB encoding is QRF_BLOB_Text. So +characters appear in the output exactly as they appear in the database. +Except the eEsp mode defaults to `QRF_ESC_On`, so that control +characters are escaped, for safety. + +The **Csv** and **Quote** styles are simply variations on **List** +with hard-coded values for some of the sqlite3_qrf_spec settings: + + +
         QuoteCsv +
        zColumnSep",""," +
        zRowSep"\\n""\\r\\n" +
        zNull"NULL""" +
        eTextQRF_TEXT_SqlQRF_TEXT_Csv +
        eBlobQRF_BLOB_SqlQRF_BLOB_Text +
        + +The **Html** style generates HTML table content, just without +the `..
        ` around the outside. + +The **Insert** style generates a series of SQL "INSERT" statements +that will inserts the data that is output into a table whose name is defined +by the zTableName field of `sqlite3_qrf_spec`. If zTableName is NULL, +then a substitute name is used. + +The **Json** and **JObject** styles generates JSON text for the query result. +The **Json** style produces a JSON array of structures with one +structure per row. **JObject** outputs independent JSON objects, one per +row, with each structure on a separate line all by itself, and not +part of a larger array. In both cases, the labels on the elements of the +JSON objects are taken from the column names of the SQL query. So if +you have an SQL query that has two or more output columns with the same +name, you will end up with JSON structures that have duplicate elements. + +Finally, the **Line** style paints each column of a row on a +separate line with the column name on the left and a "`=`" separating the +column name from its value. A single blank line appears between rows. + +### 4.4 EXPLAIN Styles (Eqp, Explain) + +The **Eqp** and **Explain** styles format output for +EXPLAIN QUERY PLAN and EXPLAIN statements, respectively. If the input +statement is not already an EXPLAIN QUERY PLAN or EXPLAIN statement is +is temporarily converted for the duration of the rendering, but +is converted back before `sqlite3_format_query_result()` returns. + +### 4.5 ScanStatus Styles (Stats, StatsEst, StatsVm) + +The **Stats**, **StatsEst**, and **StatsVm** styles are similar to **Eqp** +and **Explain** except that they include profiling information +from prior executions of the input prepared statement. +These modes only work if SQLite has been compiled with +-DSQLITE_ENABLE_STMT_SCANSTATUS and if the SQLITE_DBCONFIG_STMT_SCANSTATUS +is enabled for the database connection. The **StatsVm** style +also requires the bytecode() virtual table which is enabled using +the -DSQLITE_ENABLE_BYTECODE_VTAB compile-time option. + +### 4.6 Other Styles (Count, Off) + +The **Count** style discards all query results and returns +a count of the number of rows of output at the end. The **Off** +style is completely silent; it generates no output. These corner-case +modes are sometimes useful for debugging. + +### 5.0 Source Code Files + +The SQLite Query Result Formatter is implemented in three source code files: + + * `qrf.c` → The implementation, written in portable C99 + * `qrf.h` → A header file defining interfaces + * `README.md` → This documentation + +To use the SQLite result formatter, include the "`qrf.h`" header file +and link the application against the "`qrf.c`" source file. diff --git a/ext/qrf/dev-notes.md b/ext/qrf/dev-notes.md new file mode 100644 index 000000000..a46aada83 --- /dev/null +++ b/ext/qrf/dev-notes.md @@ -0,0 +1,14 @@ +# Developer Notes + +## Measuring Test Coverage On Linux + +On Mint Linux, as of 2025-12-02: + +> ~~~ +./configure --dev CFLAGS='-O0 -g -fprofile-arcs -ftest-coverage' +make clean testfixture +./testfixture test/qrf*.test +gcov -b -c testfixture-tclsqlite-ex.c +~~~ + +View results in tclsqlite-ex.c.gcov diff --git a/ext/qrf/qrf.c b/ext/qrf/qrf.c new file mode 100644 index 000000000..cacfa1526 --- /dev/null +++ b/ext/qrf/qrf.c @@ -0,0 +1,2983 @@ +/* +** 2025-10-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Implementation of the Result-Format or "qrf" utility library for SQLite. +** See the qrf.md documentation for additional information. +*/ +#ifndef SQLITE_QRF_H +#include "qrf.h" +#endif +#include +#include +#include + +typedef sqlite3_int64 i64; + +/* A single line in the EQP output */ +typedef struct qrfEQPGraphRow qrfEQPGraphRow; +struct qrfEQPGraphRow { + int iEqpId; /* ID for this row */ + int iParentId; /* ID of the parent row */ + qrfEQPGraphRow *pNext; /* Next row in sequence */ + char zText[1]; /* Text to display for this row */ +}; + +/* All EQP output is collected into an instance of the following */ +typedef struct qrfEQPGraph qrfEQPGraph; +struct qrfEQPGraph { + qrfEQPGraphRow *pRow; /* Linked list of all rows of the EQP output */ + qrfEQPGraphRow *pLast; /* Last element of the pRow list */ + int nWidth; /* Width of the graph */ + char zPrefix[400]; /* Graph prefix */ +}; + +/* +** Private state information. Subject to change from one release to the +** next. +*/ +typedef struct Qrf Qrf; +struct Qrf { + sqlite3_stmt *pStmt; /* The statement whose output is to be rendered */ + sqlite3 *db; /* The corresponding database connection */ + sqlite3_stmt *pJTrans; /* JSONB to JSON translator statement */ + char **pzErr; /* Write error message here, if not NULL */ + sqlite3_str *pOut; /* Accumulated output */ + int iErr; /* Error code */ + int nCol; /* Number of output columns */ + int expMode; /* Original sqlite3_stmt_isexplain() plus 1 */ + int mxWidth; /* Screen width */ + int mxHeight; /* nLineLimit */ + union { + struct { /* Content for QRF_STYLE_Line */ + int mxColWth; /* Maximum display width of any column */ + char **azCol; /* Names of output columns (MODE_Line) */ + } sLine; + qrfEQPGraph *pGraph; /* EQP graph (Eqp, Stats, and StatsEst) */ + struct { /* Content for QRF_STYLE_Explain */ + int nIndent; /* Slots allocated for aiIndent */ + int iIndent; /* Current slot */ + int *aiIndent; /* Indentation for each opcode */ + } sExpln; + } u; + sqlite3_int64 nRow; /* Number of rows handled so far */ + int *actualWidth; /* Actual width of each column */ + sqlite3_qrf_spec spec; /* Copy of the original spec */ +}; + +/* +** Data for substitute ctype.h functions. Used for x-platform +** consistency and so that '_' is counted as an alphabetic +** character. +** +** 0x01 - space +** 0x02 - digit +** 0x04 - alphabetic, including '_' +*/ +static const char qrfCType[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; +#define qrfSpace(x) ((qrfCType[(unsigned char)x]&1)!=0) +#define qrfDigit(x) ((qrfCType[(unsigned char)x]&2)!=0) +#define qrfAlpha(x) ((qrfCType[(unsigned char)x]&4)!=0) +#define qrfAlnum(x) ((qrfCType[(unsigned char)x]&6)!=0) + +#ifndef deliberate_fall_through +/* Quiet some compilers about some of our intentional code. */ +# if defined(GCC_VERSION) && GCC_VERSION>=7000000 +# define deliberate_fall_through __attribute__((fallthrough)); +# else +# define deliberate_fall_through +# endif +#endif + +/* +** Set an error code and error message. +*/ +static void qrfError( + Qrf *p, /* Query result state */ + int iCode, /* Error code */ + const char *zFormat, /* Message format (or NULL) */ + ... +){ + p->iErr = iCode; + if( p->pzErr!=0 ){ + sqlite3_free(*p->pzErr); + *p->pzErr = 0; + if( zFormat ){ + va_list ap; + va_start(ap, zFormat); + *p->pzErr = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + } + } +} + +/* +** Out-of-memory error. +*/ +static void qrfOom(Qrf *p){ + qrfError(p, SQLITE_NOMEM, "out of memory"); +} + +/* +** Transfer any error in pStr over into p. +*/ +static void qrfStrErr(Qrf *p, sqlite3_str *pStr){ + int rc = pStr ? sqlite3_str_errcode(pStr) : 0; + if( rc ){ + qrfError(p, rc, sqlite3_errstr(rc)); + } +} + + +/* +** Add a new entry to the EXPLAIN QUERY PLAN data +*/ +static void qrfEqpAppend(Qrf *p, int iEqpId, int p2, const char *zText){ + qrfEQPGraphRow *pNew; + sqlite3_int64 nText; + if( zText==0 ) return; + if( p->u.pGraph==0 ){ + p->u.pGraph = sqlite3_malloc64( sizeof(qrfEQPGraph) ); + if( p->u.pGraph==0 ){ + qrfOom(p); + return; + } + memset(p->u.pGraph, 0, sizeof(qrfEQPGraph) ); + } + nText = strlen(zText); + pNew = sqlite3_malloc64( sizeof(*pNew) + nText ); + if( pNew==0 ){ + qrfOom(p); + return; + } + pNew->iEqpId = iEqpId; + pNew->iParentId = p2; + memcpy(pNew->zText, zText, nText+1); + pNew->pNext = 0; + if( p->u.pGraph->pLast ){ + p->u.pGraph->pLast->pNext = pNew; + }else{ + p->u.pGraph->pRow = pNew; + } + p->u.pGraph->pLast = pNew; +} + +/* +** Free and reset the EXPLAIN QUERY PLAN data that has been collected +** in p->u.pGraph. +*/ +static void qrfEqpReset(Qrf *p){ + qrfEQPGraphRow *pRow, *pNext; + if( p->u.pGraph ){ + for(pRow = p->u.pGraph->pRow; pRow; pRow = pNext){ + pNext = pRow->pNext; + sqlite3_free(pRow); + } + sqlite3_free(p->u.pGraph); + p->u.pGraph = 0; + } +} + +/* Return the next EXPLAIN QUERY PLAN line with iEqpId that occurs after +** pOld, or return the first such line if pOld is NULL +*/ +static qrfEQPGraphRow *qrfEqpNextRow(Qrf *p, int iEqpId, qrfEQPGraphRow *pOld){ + qrfEQPGraphRow *pRow = pOld ? pOld->pNext : p->u.pGraph->pRow; + while( pRow && pRow->iParentId!=iEqpId ) pRow = pRow->pNext; + return pRow; +} + +/* Render a single level of the graph that has iEqpId as its parent. Called +** recursively to render sublevels. +*/ +static void qrfEqpRenderLevel(Qrf *p, int iEqpId){ + qrfEQPGraphRow *pRow, *pNext; + i64 n = strlen(p->u.pGraph->zPrefix); + char *z; + for(pRow = qrfEqpNextRow(p, iEqpId, 0); pRow; pRow = pNext){ + pNext = qrfEqpNextRow(p, iEqpId, pRow); + z = pRow->zText; + sqlite3_str_appendf(p->pOut, "%s%s%s\n", p->u.pGraph->zPrefix, + pNext ? "|--" : "`--", z); + if( n<(i64)sizeof(p->u.pGraph->zPrefix)-7 ){ + memcpy(&p->u.pGraph->zPrefix[n], pNext ? "| " : " ", 4); + qrfEqpRenderLevel(p, pRow->iEqpId); + p->u.pGraph->zPrefix[n] = 0; + } + } +} + +/* +** Render the 64-bit value N in a more human-readable format into +** pOut. +** +** + Only show the first three significant digits. +** + Append suffixes K, M, G, T, P, and E for 1e3, 1e6, ... 1e18 +*/ +static void qrfApproxInt64(sqlite3_str *pOut, i64 N){ + static const char aSuffix[] = { 'K', 'M', 'G', 'T', 'P', 'E' }; + int i; + if( N<0 ){ + N = N==INT64_MIN ? INT64_MAX : -N; + sqlite3_str_append(pOut, "-", 1); + } + if( N<10000 ){ + sqlite3_str_appendf(pOut, "%4lld ", N); + return; + } + for(i=1; i<=18; i++){ + N = (N+5)/10; + if( N<10000 ){ + int n = (int)N; + switch( i%3 ){ + case 0: + sqlite3_str_appendf(pOut, "%d.%02d", n/1000, (n%1000)/10); + break; + case 1: + sqlite3_str_appendf(pOut, "%2d.%d", n/100, (n%100)/10); + break; + case 2: + sqlite3_str_appendf(pOut, "%4d", n/10); + break; + } + sqlite3_str_append(pOut, &aSuffix[i/3], 1); + break; + } + } +} + +/* +** Display and reset the EXPLAIN QUERY PLAN data +*/ +static void qrfEqpRender(Qrf *p, i64 nCycle){ + qrfEQPGraphRow *pRow; + if( p->u.pGraph!=0 && (pRow = p->u.pGraph->pRow)!=0 ){ + if( pRow->zText[0]=='-' ){ + if( pRow->pNext==0 ){ + qrfEqpReset(p); + return; + } + sqlite3_str_appendf(p->pOut, "%s\n", pRow->zText+3); + p->u.pGraph->pRow = pRow->pNext; + sqlite3_free(pRow); + }else if( nCycle>0 ){ + int nSp = p->u.pGraph->nWidth - 2; + if( p->spec.eStyle==QRF_STYLE_StatsEst ){ + sqlite3_str_appendchar(p->pOut, nSp, ' '); + sqlite3_str_appendall(p->pOut, + "Cycles Loops (est) Rows (est)\n"); + sqlite3_str_appendchar(p->pOut, nSp, ' '); + sqlite3_str_appendall(p->pOut, + "---------- ------------ ------------\n"); + }else{ + sqlite3_str_appendchar(p->pOut, nSp, ' '); + sqlite3_str_appendall(p->pOut, + "Cycles Loops Rows \n"); + sqlite3_str_appendchar(p->pOut, nSp, ' '); + sqlite3_str_appendall(p->pOut, + "---------- ----- -----\n"); + } + sqlite3_str_appendall(p->pOut, "QUERY PLAN"); + sqlite3_str_appendchar(p->pOut, nSp - 10, ' '); + qrfApproxInt64(p->pOut, nCycle); + sqlite3_str_appendall(p->pOut, " 100%\n"); + }else{ + sqlite3_str_appendall(p->pOut, "QUERY PLAN\n"); + } + p->u.pGraph->zPrefix[0] = 0; + qrfEqpRenderLevel(p, 0); + qrfEqpReset(p); + } +} + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +/* +** Helper function for qrfExpStats(). +** +*/ +static int qrfStatsHeight(sqlite3_stmt *p, int iEntry){ + int iPid = 0; + int ret = 1; + sqlite3_stmt_scanstatus_v2(p, iEntry, + SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid + ); + while( iPid!=0 ){ + int ii; + for(ii=0; 1; ii++){ + int iId; + int res; + res = sqlite3_stmt_scanstatus_v2(p, ii, + SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iId + ); + if( res ) break; + if( iId==iPid ){ + sqlite3_stmt_scanstatus_v2(p, ii, + SQLITE_SCANSTAT_PARENTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid + ); + } + } + ret++; + } + return ret; +} +#endif /* SQLITE_ENABLE_STMT_SCANSTATUS */ + + +/* +** Generate ".scanstatus est" style of EQP output. +*/ +static void qrfEqpStats(Qrf *p){ +#ifndef SQLITE_ENABLE_STMT_SCANSTATUS + qrfError(p, SQLITE_ERROR, "not available in this build"); +#else + static const int f = SQLITE_SCANSTAT_COMPLEX; + sqlite3_stmt *pS = p->pStmt; + int i = 0; + i64 nTotal = 0; + int nWidth = 0; + int prevPid = -1; /* Previous iPid */ + double rEstCum = 1.0; /* Cumulative row estimate */ + sqlite3_str *pLine = sqlite3_str_new(p->db); + sqlite3_str *pStats = sqlite3_str_new(p->db); + qrfEqpReset(p); + + for(i=0; 1; i++){ + const char *z = 0; + int n = 0; + if( sqlite3_stmt_scanstatus_v2(pS,i,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&z) ){ + break; + } + n = (int)strlen(z) + qrfStatsHeight(pS,i)*3; + if( n>nWidth ) nWidth = n; + } + nWidth += 2; + + sqlite3_stmt_scanstatus_v2(pS,-1, SQLITE_SCANSTAT_NCYCLE, f, (void*)&nTotal); + for(i=0; 1; i++){ + i64 nLoop = 0; + i64 nRow = 0; + i64 nCycle = 0; + int iId = 0; + int iPid = 0; + const char *zo = 0; + const char *zName = 0; + double rEst = 0.0; + + if( sqlite3_stmt_scanstatus_v2(pS,i,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&zo) ){ + break; + } + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_PARENTID,f,(void*)&iPid); + if( iPid!=prevPid ){ + prevPid = iPid; + rEstCum = 1.0; + } + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_EST,f,(void*)&rEst); + rEstCum *= rEst; + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_NLOOP,f,(void*)&nLoop); + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_NVISIT,f,(void*)&nRow); + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_NCYCLE,f,(void*)&nCycle); + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_SELECTID,f,(void*)&iId); + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_NAME,f,(void*)&zName); + + if( nCycle>=0 || nLoop>=0 || nRow>=0 ){ + int nSp = 0; + sqlite3_str_reset(pStats); + if( nCycle>=0 && nTotal>0 ){ + qrfApproxInt64(pStats, nCycle); + sqlite3_str_appendf(pStats, " %3d%%", + ((nCycle*100)+nTotal/2) / nTotal + ); + nSp = 2; + } + if( nLoop>=0 ){ + if( nSp ) sqlite3_str_appendchar(pStats, nSp, ' '); + qrfApproxInt64(pStats, nLoop); + nSp = 2; + if( p->spec.eStyle==QRF_STYLE_StatsEst ){ + sqlite3_str_appendf(pStats, " "); + qrfApproxInt64(pStats, (i64)(rEstCum/rEst)); + } + } + if( nRow>=0 ){ + if( nSp ) sqlite3_str_appendchar(pStats, nSp, ' '); + qrfApproxInt64(pStats, nRow); + nSp = 2; + if( p->spec.eStyle==QRF_STYLE_StatsEst ){ + sqlite3_str_appendf(pStats, " "); + qrfApproxInt64(pStats, (i64)rEstCum); + } + } + sqlite3_str_appendf(pLine, + "% *s %s", -1*(nWidth-qrfStatsHeight(pS,i)*3), zo, + sqlite3_str_value(pStats) + ); + sqlite3_str_reset(pStats); + qrfEqpAppend(p, iId, iPid, sqlite3_str_value(pLine)); + sqlite3_str_reset(pLine); + }else{ + qrfEqpAppend(p, iId, iPid, zo); + } + } + if( p->u.pGraph ) p->u.pGraph->nWidth = nWidth; + qrfStrErr(p, pLine); + sqlite3_free(sqlite3_str_finish(pLine)); + qrfStrErr(p, pStats); + sqlite3_free(sqlite3_str_finish(pStats)); +#endif +} + + +/* +** Reset the prepared statement. +*/ +static void qrfResetStmt(Qrf *p){ + int rc = sqlite3_reset(p->pStmt); + if( rc!=SQLITE_OK && p->iErr==SQLITE_OK ){ + qrfError(p, rc, "%s", sqlite3_errmsg(p->db)); + } +} + +/* +** If xWrite is defined, send all content of pOut to xWrite and +** reset pOut. +*/ +static void qrfWrite(Qrf *p){ + int n; + if( p->spec.xWrite && (n = sqlite3_str_length(p->pOut))>0 ){ + int rc = p->spec.xWrite(p->spec.pWriteArg, + sqlite3_str_value(p->pOut), + (sqlite3_int64)n); + sqlite3_str_reset(p->pOut); + if( rc ){ + qrfError(p, rc, "Failed to write %d bytes of output", n); + } + } +} + +/* Lookup table to estimate the number of columns consumed by a Unicode +** character. +*/ +static const struct { + unsigned char w; /* Width of the character in columns */ + int iFirst; /* First character in a span having this width */ +} aQrfUWidth[] = { + /* {1, 0x00000}, */ + {0, 0x00300}, {1, 0x00370}, {0, 0x00483}, {1, 0x00487}, {0, 0x00488}, + {1, 0x0048a}, {0, 0x00591}, {1, 0x005be}, {0, 0x005bf}, {1, 0x005c0}, + {0, 0x005c1}, {1, 0x005c3}, {0, 0x005c4}, {1, 0x005c6}, {0, 0x005c7}, + {1, 0x005c8}, {0, 0x00600}, {1, 0x00604}, {0, 0x00610}, {1, 0x00616}, + {0, 0x0064b}, {1, 0x0065f}, {0, 0x00670}, {1, 0x00671}, {0, 0x006d6}, + {1, 0x006e5}, {0, 0x006e7}, {1, 0x006e9}, {0, 0x006ea}, {1, 0x006ee}, + {0, 0x0070f}, {1, 0x00710}, {0, 0x00711}, {1, 0x00712}, {0, 0x00730}, + {1, 0x0074b}, {0, 0x007a6}, {1, 0x007b1}, {0, 0x007eb}, {1, 0x007f4}, + {0, 0x00901}, {1, 0x00903}, {0, 0x0093c}, {1, 0x0093d}, {0, 0x00941}, + {1, 0x00949}, {0, 0x0094d}, {1, 0x0094e}, {0, 0x00951}, {1, 0x00955}, + {0, 0x00962}, {1, 0x00964}, {0, 0x00981}, {1, 0x00982}, {0, 0x009bc}, + {1, 0x009bd}, {0, 0x009c1}, {1, 0x009c5}, {0, 0x009cd}, {1, 0x009ce}, + {0, 0x009e2}, {1, 0x009e4}, {0, 0x00a01}, {1, 0x00a03}, {0, 0x00a3c}, + {1, 0x00a3d}, {0, 0x00a41}, {1, 0x00a43}, {0, 0x00a47}, {1, 0x00a49}, + {0, 0x00a4b}, {1, 0x00a4e}, {0, 0x00a70}, {1, 0x00a72}, {0, 0x00a81}, + {1, 0x00a83}, {0, 0x00abc}, {1, 0x00abd}, {0, 0x00ac1}, {1, 0x00ac6}, + {0, 0x00ac7}, {1, 0x00ac9}, {0, 0x00acd}, {1, 0x00ace}, {0, 0x00ae2}, + {1, 0x00ae4}, {0, 0x00b01}, {1, 0x00b02}, {0, 0x00b3c}, {1, 0x00b3d}, + {0, 0x00b3f}, {1, 0x00b40}, {0, 0x00b41}, {1, 0x00b44}, {0, 0x00b4d}, + {1, 0x00b4e}, {0, 0x00b56}, {1, 0x00b57}, {0, 0x00b82}, {1, 0x00b83}, + {0, 0x00bc0}, {1, 0x00bc1}, {0, 0x00bcd}, {1, 0x00bce}, {0, 0x00c3e}, + {1, 0x00c41}, {0, 0x00c46}, {1, 0x00c49}, {0, 0x00c4a}, {1, 0x00c4e}, + {0, 0x00c55}, {1, 0x00c57}, {0, 0x00cbc}, {1, 0x00cbd}, {0, 0x00cbf}, + {1, 0x00cc0}, {0, 0x00cc6}, {1, 0x00cc7}, {0, 0x00ccc}, {1, 0x00cce}, + {0, 0x00ce2}, {1, 0x00ce4}, {0, 0x00d41}, {1, 0x00d44}, {0, 0x00d4d}, + {1, 0x00d4e}, {0, 0x00dca}, {1, 0x00dcb}, {0, 0x00dd2}, {1, 0x00dd5}, + {0, 0x00dd6}, {1, 0x00dd7}, {0, 0x00e31}, {1, 0x00e32}, {0, 0x00e34}, + {1, 0x00e3b}, {0, 0x00e47}, {1, 0x00e4f}, {0, 0x00eb1}, {1, 0x00eb2}, + {0, 0x00eb4}, {1, 0x00eba}, {0, 0x00ebb}, {1, 0x00ebd}, {0, 0x00ec8}, + {1, 0x00ece}, {0, 0x00f18}, {1, 0x00f1a}, {0, 0x00f35}, {1, 0x00f36}, + {0, 0x00f37}, {1, 0x00f38}, {0, 0x00f39}, {1, 0x00f3a}, {0, 0x00f71}, + {1, 0x00f7f}, {0, 0x00f80}, {1, 0x00f85}, {0, 0x00f86}, {1, 0x00f88}, + {0, 0x00f90}, {1, 0x00f98}, {0, 0x00f99}, {1, 0x00fbd}, {0, 0x00fc6}, + {1, 0x00fc7}, {0, 0x0102d}, {1, 0x01031}, {0, 0x01032}, {1, 0x01033}, + {0, 0x01036}, {1, 0x0103b}, {0, 0x01058}, + {1, 0x0105a}, {2, 0x01100}, {0, 0x01160}, {1, 0x01200}, {0, 0x0135f}, + {1, 0x01360}, {0, 0x01712}, {1, 0x01715}, {0, 0x01732}, {1, 0x01735}, + {0, 0x01752}, {1, 0x01754}, {0, 0x01772}, {1, 0x01774}, {0, 0x017b4}, + {1, 0x017b6}, {0, 0x017b7}, {1, 0x017be}, {0, 0x017c6}, {1, 0x017c7}, + {0, 0x017c9}, {1, 0x017d4}, {0, 0x017dd}, {1, 0x017de}, {0, 0x0180b}, + {1, 0x0180e}, {0, 0x018a9}, {1, 0x018aa}, {0, 0x01920}, {1, 0x01923}, + {0, 0x01927}, {1, 0x01929}, {0, 0x01932}, {1, 0x01933}, {0, 0x01939}, + {1, 0x0193c}, {0, 0x01a17}, {1, 0x01a19}, {0, 0x01b00}, {1, 0x01b04}, + {0, 0x01b34}, {1, 0x01b35}, {0, 0x01b36}, {1, 0x01b3b}, {0, 0x01b3c}, + {1, 0x01b3d}, {0, 0x01b42}, {1, 0x01b43}, {0, 0x01b6b}, {1, 0x01b74}, + {0, 0x01dc0}, {1, 0x01dcb}, {0, 0x01dfe}, {1, 0x01e00}, {0, 0x0200b}, + {1, 0x02010}, {0, 0x0202a}, {1, 0x0202f}, {0, 0x02060}, {1, 0x02064}, + {0, 0x0206a}, {1, 0x02070}, {0, 0x020d0}, {1, 0x020f0}, {2, 0x02329}, + {1, 0x0232b}, {2, 0x02e80}, {0, 0x0302a}, {2, 0x03030}, {1, 0x0303f}, + {2, 0x03040}, {0, 0x03099}, {2, 0x0309b}, {1, 0x0a4d0}, {0, 0x0a806}, + {1, 0x0a807}, {0, 0x0a80b}, {1, 0x0a80c}, {0, 0x0a825}, {1, 0x0a827}, + {2, 0x0ac00}, {1, 0x0d7a4}, {2, 0x0f900}, {1, 0x0fb00}, {0, 0x0fb1e}, + {1, 0x0fb1f}, {0, 0x0fe00}, {2, 0x0fe10}, {1, 0x0fe1a}, {0, 0x0fe20}, + {1, 0x0fe24}, {2, 0x0fe30}, {1, 0x0fe70}, {0, 0x0feff}, {2, 0x0ff00}, + {1, 0x0ff61}, {2, 0x0ffe0}, {1, 0x0ffe7}, {0, 0x0fff9}, {1, 0x0fffc}, + {0, 0x10a01}, {1, 0x10a04}, {0, 0x10a05}, {1, 0x10a07}, {0, 0x10a0c}, + {1, 0x10a10}, {0, 0x10a38}, {1, 0x10a3b}, {0, 0x10a3f}, {1, 0x10a40}, + {0, 0x1d167}, {1, 0x1d16a}, {0, 0x1d173}, {1, 0x1d183}, {0, 0x1d185}, + {1, 0x1d18c}, {0, 0x1d1aa}, {1, 0x1d1ae}, {0, 0x1d242}, {1, 0x1d245}, + {2, 0x20000}, {1, 0x2fffe}, {2, 0x30000}, {1, 0x3fffe}, {0, 0xe0001}, + {1, 0xe0002}, {0, 0xe0020}, {1, 0xe0080}, {0, 0xe0100}, {1, 0xe01f0} +}; + +/* +** Return an estimate of the width, in columns, for the single Unicode +** character c. For normal characters, the answer is always 1. But the +** estimate might be 0 or 2 for zero-width and double-width characters. +** +** Different display devices display unicode using different widths. So +** it is impossible to know that true display width with 100% accuracy. +** Inaccuracies in the width estimates might cause columns to be misaligned. +** Unfortunately, there is nothing we can do about that. +*/ +int sqlite3_qrf_wcwidth(int c){ + int iFirst, iLast; + + /* Fast path for common characters */ + if( c<0x300 ) return 1; + + /* The general case */ + iFirst = 0; + iLast = sizeof(aQrfUWidth)/sizeof(aQrfUWidth[0]) - 1; + while( iFirst c ){ + iLast = iMid - 1; + }else{ + return aQrfUWidth[iMid].w; + } + } + if( aQrfUWidth[iLast].iFirst > c ) return aQrfUWidth[iFirst].w; + return aQrfUWidth[iLast].w; +} + +/* +** Compute the value and length of a multi-byte UTF-8 character that +** begins at z[0]. Return the length. Write the Unicode value into *pU. +** +** This routine only works for *multi-byte* UTF-8 characters. It does +** not attempt to detect illegal characters. +*/ +int sqlite3_qrf_decode_utf8(const unsigned char *z, int *pU){ + if( (z[0] & 0xe0)==0xc0 && (z[1] & 0xc0)==0x80 ){ + *pU = ((z[0] & 0x1f)<<6) | (z[1] & 0x3f); + return 2; + } + if( (z[0] & 0xf0)==0xe0 && (z[1] & 0xc0)==0x80 && (z[2] & 0xc0)==0x80 ){ + *pU = ((z[0] & 0x0f)<<12) | ((z[1] & 0x3f)<<6) | (z[2] & 0x3f); + return 3; + } + if( (z[0] & 0xf8)==0xf0 && (z[1] & 0xc0)==0x80 && (z[2] & 0xc0)==0x80 + && (z[3] & 0xc0)==0x80 + ){ + *pU = ((z[0] & 0x0f)<<18) | ((z[1] & 0x3f)<<12) | ((z[2] & 0x3f))<<6 + | (z[3] & 0x3f); + return 4; + } + *pU = 0; + return 1; +} + +/* +** Check to see if z[] is a valid VT100 escape. If it is, then +** return the number of bytes in the escape sequence. Return 0 if +** z[] is not a VT100 escape. +** +** This routine assumes that z[0] is \033 (ESC). +*/ +static int qrfIsVt100(const unsigned char *z){ + int i; + if( z[1]!='[' ) return 0; + i = 2; + while( z[i]>=0x30 && z[i]<=0x3f ){ i++; } + while( z[i]>=0x20 && z[i]<=0x2f ){ i++; } + if( z[i]<0x40 || z[i]>0x7e ) return 0; + return i+1; +} + +/* +** Return the length of a string in display characters. +** +** Most characters of the input string count as 1, including +** multi-byte UTF8 characters. However, zero-width unicode +** characters and VT100 escape sequences count as zero, and +** double-width characters count as two. +** +** The definition of "zero-width" and "double-width" characters +** is not precise. It depends on the output device, to some extent, +** and it varies according to the Unicode version. This routine +** makes the best guess that it can. +*/ +size_t sqlite3_qrf_wcswidth(const char *zIn){ + const unsigned char *z = (const unsigned char*)zIn; + size_t n = 0; + while( *z ){ + if( z[0]<' ' ){ + int k; + if( z[0]=='\033' && (k = qrfIsVt100(z))>0 ){ + z += k; + }else{ + z++; + } + }else if( (0x80&z[0])==0 ){ + n++; + z++; + }else{ + int u = 0; + int len = sqlite3_qrf_decode_utf8(z, &u); + z += len; + n += sqlite3_qrf_wcwidth(u); + } + } + return n; +} + +/* +** Return the display width of the longest line of text +** in the (possibly) multi-line input string zIn[0..nByte]. +** zIn[] is not necessarily zero-terminated. Take +** into account tab characters, zero- and double-width +** characters, CR and NL, and VT100 escape codes. +** +** Write the number of newlines into *pnNL. So, *pnNL will +** return 0 if everything fits on one line, or positive it +** it will need to be split. +*/ +static int qrfDisplayWidth(const char *zIn, sqlite3_int64 nByte, int *pnNL){ + const unsigned char *z; + const unsigned char *zEnd; + int mx = 0; + int n = 0; + int nNL = 0; + if( zIn==0 ) zIn = ""; + z = (const unsigned char*)zIn; + zEnd = &z[nByte]; + while( z0 ){ + z += k; + }else{ + if( z[0]=='\t' ){ + n = (n+8)&~7; + }else if( z[0]=='\n' || z[0]=='\r' ){ + nNL++; + if( n>mx ) mx = n; + n = 0; + } + z++; + } + }else if( (0x80&z[0])==0 ){ + n++; + z++; + }else{ + int u = 0; + int len = sqlite3_qrf_decode_utf8(z, &u); + z += len; + n += sqlite3_qrf_wcwidth(u); + } + } + if( mx>n ) n = mx; + if( pnNL ) *pnNL = nNL; + return n; +} + +/* +** Escape the input string if it is needed and in accordance with +** eEsc, which is either QRF_ESC_Ascii or QRF_ESC_Symbol. +** +** Escaping is needed if the string contains any control characters +** other than \t, \n, and \r\n +** +** If no escaping is needed (the common case) then set *ppOut to NULL +** and return 0. If escaping is needed, write the escaped string into +** memory obtained from sqlite3_malloc64() and make *ppOut point to that +** memory and return 0. If an error occurs, return non-zero. +** +** The caller is responsible for freeing *ppFree if it is non-NULL in order +** to reclaim memory. +*/ +static void qrfEscape( + int eEsc, /* QRF_ESC_Ascii or QRF_ESC_Symbol */ + sqlite3_str *pStr, /* String to be escaped */ + int iStart /* Begin escapding on this byte of pStr */ +){ + sqlite3_int64 i, j; /* Loop counters */ + sqlite3_int64 sz; /* Size of the string prior to escaping */ + sqlite3_int64 nCtrl = 0;/* Number of control characters to escape */ + unsigned char *zIn; /* Text to be escaped */ + unsigned char c; /* A single character of the text */ + unsigned char *zOut; /* Where to write the results */ + + /* Find the text to be escaped */ + zIn = (unsigned char*)sqlite3_str_value(pStr); + if( zIn==0 ) return; + zIn += iStart; + + /* Count the control characters */ + for(i=0; (c = zIn[i])!=0; i++){ + if( c<=0x1f + && c!='\t' + && c!='\n' + && (c!='\r' || zIn[i+1]!='\n') + ){ + nCtrl++; + } + } + if( nCtrl==0 ) return; /* Early out if no control characters */ + + /* Make space to hold the escapes. Copy the original text to the end + ** of the available space. */ + sz = sqlite3_str_length(pStr) - iStart; + if( eEsc==QRF_ESC_Symbol ) nCtrl *= 2; + sqlite3_str_appendchar(pStr, nCtrl, ' '); + zOut = (unsigned char*)sqlite3_str_value(pStr); + if( zOut==0 ) return; + zOut += iStart; + zIn = zOut + nCtrl; + memmove(zIn,zOut,sz); + + /* Convert the control characters */ + for(i=j=0; (c = zIn[i])!=0; i++){ + if( c>0x1f + || c=='\t' + || c=='\n' + || (c=='\r' && zIn[i+1]=='\n') + ){ + continue; + } + if( i>0 ){ + memmove(&zOut[j], zIn, i); + j += i; + } + zIn += i+1; + i = -1; + if( eEsc==QRF_ESC_Symbol ){ + zOut[j++] = 0xe2; + zOut[j++] = 0x90; + zOut[j++] = 0x80+c; + }else{ + zOut[j++] = '^'; + zOut[j++] = 0x40+c; + } + } +} + +/* +** Determine if the string z[] can be shown as plain text. Return true +** if z[] is unambiguously text. Return false if z[] needs to be +** quoted. +** +** All of the following must be true in order for z[] to be relaxable: +** +** (1) z[] does not begin or end with ' or whitespace +** (2) z[] is not the same as the NULL rendering +** (3) z[] does not looks like a numeric literal +*/ +static int qrfRelaxable(Qrf *p, const char *z){ + size_t i, n; + if( z[0]=='\'' || qrfSpace(z[0]) ) return 0; + if( z[0]==0 ){ + return (p->spec.zNull!=0 && p->spec.zNull[0]!=0); + } + n = strlen(z); + if( n==0 || z[n-1]=='\'' || qrfSpace(z[n-1]) ) return 0; + if( p->spec.zNull && strcmp(p->spec.zNull,z)==0 ) return 0; + i = (z[0]=='-' || z[0]=='+'); + if( strcmp(z+i,"Inf")==0 ) return 0; + if( !qrfDigit(z[i]) ) return 1; + i++; + while( qrfDigit(z[i]) ){ i++; } + if( z[i]==0 ) return 0; + if( z[i]=='.' ){ + i++; + while( qrfDigit(z[i]) ){ i++; } + if( z[i]==0 ) return 0; + } + if( z[i]=='e' || z[i]=='E' ){ + i++; + if( z[i]=='+' || z[i]=='-' ){ i++; } + if( !qrfDigit(z[i]) ) return 1; + i++; + while( qrfDigit(z[i]) ){ i++; } + } + return z[i]!=0; +} + +/* +** If a field contains any character identified by a 1 in the following +** array, then the string must be quoted for CSV. +*/ +static const char qrfCsvQuote[] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +}; + +/* +** Encode text appropriately and append it to pOut. +*/ +static void qrfEncodeText(Qrf *p, sqlite3_str *pOut, const char *zTxt){ + int iStart = sqlite3_str_length(pOut); + switch( p->spec.eText ){ + case QRF_TEXT_Relaxed: + if( qrfRelaxable(p, zTxt) ){ + sqlite3_str_appendall(pOut, zTxt); + break; + } + deliberate_fall_through; /* FALLTHRU */ + case QRF_TEXT_Sql: { + if( p->spec.eEsc==QRF_ESC_Off ){ + sqlite3_str_appendf(pOut, "%Q", zTxt); + }else{ + sqlite3_str_appendf(pOut, "%#Q", zTxt); + } + break; + } + case QRF_TEXT_Csv: { + unsigned int i; + for(i=0; zTxt[i]; i++){ + if( qrfCsvQuote[((const unsigned char*)zTxt)[i]] ){ + i = 0; + break; + } + } + if( i==0 || strstr(zTxt, p->spec.zColumnSep)!=0 ){ + sqlite3_str_appendf(pOut, "\"%w\"", zTxt); + }else{ + sqlite3_str_appendall(pOut, zTxt); + } + break; + } + case QRF_TEXT_Html: { + const unsigned char *z = (const unsigned char*)zTxt; + while( *z ){ + unsigned int i = 0; + unsigned char c; + while( (c=z[i])>'>' + || (c && c!='<' && c!='>' && c!='&' && c!='\"' && c!='\'') + ){ + i++; + } + if( i>0 ){ + sqlite3_str_append(pOut, (const char*)z, i); + } + switch( z[i] ){ + case '>': sqlite3_str_append(pOut, "<", 4); break; + case '&': sqlite3_str_append(pOut, "&", 5); break; + case '<': sqlite3_str_append(pOut, "<", 4); break; + case '"': sqlite3_str_append(pOut, """, 6); break; + case '\'': sqlite3_str_append(pOut, "'", 5); break; + default: i--; + } + z += i + 1; + } + break; + } + case QRF_TEXT_Tcl: + case QRF_TEXT_Json: { + const unsigned char *z = (const unsigned char*)zTxt; + sqlite3_str_append(pOut, "\"", 1); + while( *z ){ + unsigned int i; + for(i=0; z[i]>=0x20 && z[i]!='\\' && z[i]!='"'; i++){} + if( i>0 ){ + sqlite3_str_append(pOut, (const char*)z, i); + } + if( z[i]==0 ) break; + switch( z[i] ){ + case '"': sqlite3_str_append(pOut, "\\\"", 2); break; + case '\\': sqlite3_str_append(pOut, "\\\\", 2); break; + case '\b': sqlite3_str_append(pOut, "\\b", 2); break; + case '\f': sqlite3_str_append(pOut, "\\f", 2); break; + case '\n': sqlite3_str_append(pOut, "\\n", 2); break; + case '\r': sqlite3_str_append(pOut, "\\r", 2); break; + case '\t': sqlite3_str_append(pOut, "\\t", 2); break; + default: { + if( p->spec.eText==QRF_TEXT_Json ){ + sqlite3_str_appendf(pOut, "\\u%04x", z[i]); + }else{ + sqlite3_str_appendf(pOut, "\\%03o", z[i]); + } + break; + } + } + z += i + 1; + } + sqlite3_str_append(pOut, "\"", 1); + break; + } + default: { + sqlite3_str_appendall(pOut, zTxt); + break; + } + } + if( p->spec.eEsc!=QRF_ESC_Off ){ + qrfEscape(p->spec.eEsc, pOut, iStart); + } +} + +/* +** Do a quick sanity check to see aBlob[0..nBlob-1] is valid JSONB +** return true if it is and false if it is not. +** +** False positives are possible, but not false negatives. +*/ +static int qrfJsonbQuickCheck(unsigned char *aBlob, int nBlob){ + unsigned char x; /* Payload size half-byte */ + int i; /* Loop counter */ + int n; /* Bytes in the payload size integer */ + sqlite3_uint64 sz; /* value of the payload size integer */ + + if( nBlob==0 ) return 0; + x = aBlob[0]>>4; + if( x<=11 ) return nBlob==(1+x); + n = x<14 ? x-11 : 4*(x-13); + if( nBlob<1+n ) return 0; + sz = aBlob[1]; + for(i=1; ipStmt is known to be a BLOB. Check +** to see if that BLOB is really a JSONB blob. If it is, then translate +** it into a text JSON representation and return a pointer to that text JSON. +** If the BLOB is not JSONB, then return a NULL pointer. +** +** The memory used to hold the JSON text is managed internally by the +** "p" object and is overwritten and/or deallocated upon the next call +** to this routine (with the same p argument) or when the p object is +** finailized. +*/ +static const char *qrfJsonbToJson(Qrf *p, int iCol){ + int nByte; + const void *pBlob; + int rc; + nByte = sqlite3_column_bytes(p->pStmt, iCol); + pBlob = sqlite3_column_blob(p->pStmt, iCol); + if( qrfJsonbQuickCheck((unsigned char*)pBlob, nByte)==0 ){ + return 0; + } + if( p->pJTrans==0 ){ + sqlite3 *db; + rc = sqlite3_open(":memory:",&db); + if( rc ){ + sqlite3_close(db); + return 0; + } + rc = sqlite3_prepare_v2(db, "SELECT json(?1)", -1, &p->pJTrans, 0); + if( rc ){ + sqlite3_finalize(p->pJTrans); + p->pJTrans = 0; + sqlite3_close(db); + return 0; + } + }else{ + sqlite3_reset(p->pJTrans); + } + sqlite3_bind_blob(p->pJTrans, 1, (void*)pBlob, nByte, SQLITE_STATIC); + rc = sqlite3_step(p->pJTrans); + if( rc==SQLITE_ROW ){ + return (const char*)sqlite3_column_text(p->pJTrans, 0); + }else{ + return 0; + } +} + +/* +** Adjust the input string zIn[] such that it is no more than N display +** characters wide. If it is wider than that, then truncate and add +** ellipsis. Or if zIn[] contains a \r or \n, truncate at that point, +** adding ellipsis. Embedded tabs in zIn[] are converted into ordinary +** spaces. +** +** Return this display width of the modified title string. +*/ +static int qrfTitleLimit(char *zIn, int N){ + unsigned char *z = (unsigned char*)zIn; + int n = 0; + unsigned char *zEllipsis = 0; + while( 1 /*exit-by-break*/ ){ + if( z[0]<' ' ){ + int k; + if( z[0]==0 ){ + zEllipsis = 0; + break; + }else if( z[0]=='\033' && (k = qrfIsVt100(z))>0 ){ + z += k; + }else if( z[0]=='\t' ){ + z[0] = ' '; + }else if( z[0]=='\n' || z[0]=='\r' ){ + z[0] = ' '; + }else{ + z++; + } + }else if( (0x80&z[0])==0 ){ + if( n>=(N-3) && zEllipsis==0 ) zEllipsis = z; + if( n==N ){ z[0] = 0; break; } + n++; + z++; + }else{ + int u = 0; + int len = sqlite3_qrf_decode_utf8(z, &u); + if( n+len>(N-3) && zEllipsis==0 ) zEllipsis = z; + if( n+len>N ){ z[0] = 0; break; } + z += len; + n += sqlite3_qrf_wcwidth(u); + } + } + if( zEllipsis && N>=3 ) memcpy(zEllipsis,"...",4); + return n; +} + + +/* +** Render value pVal into pOut +*/ +static void qrfRenderValue(Qrf *p, sqlite3_str *pOut, int iCol){ +#if SQLITE_VERSION_NUMBER>=3052000 + int iStartLen = sqlite3_str_length(pOut); +#endif + if( p->spec.xRender ){ + sqlite3_value *pVal; + char *z; + pVal = sqlite3_value_dup(sqlite3_column_value(p->pStmt,iCol)); + z = p->spec.xRender(p->spec.pRenderArg, pVal); + sqlite3_value_free(pVal); + if( z ){ + sqlite3_str_appendall(pOut, z); + sqlite3_free(z); + return; + } + } + switch( sqlite3_column_type(p->pStmt,iCol) ){ + case SQLITE_INTEGER: { + sqlite3_str_appendf(pOut, "%lld", sqlite3_column_int64(p->pStmt,iCol)); + break; + } + case SQLITE_FLOAT: { + const char *zTxt = (const char*)sqlite3_column_text(p->pStmt,iCol); + sqlite3_str_appendall(pOut, zTxt); + break; + } + case SQLITE_BLOB: { + if( p->spec.bTextJsonb==QRF_Yes ){ + const char *zJson = qrfJsonbToJson(p, iCol); + if( zJson ){ + if( p->spec.eText==QRF_TEXT_Sql ){ + sqlite3_str_append(pOut,"jsonb(",6); + qrfEncodeText(p, pOut, zJson); + sqlite3_str_append(pOut,")",1); + }else{ + qrfEncodeText(p, pOut, zJson); + } + break; + } + } + switch( p->spec.eBlob ){ + case QRF_BLOB_Hex: + case QRF_BLOB_Sql: { + int iStart; + int nBlob = sqlite3_column_bytes(p->pStmt,iCol); + int i, j; + char *zVal; + const unsigned char *a = sqlite3_column_blob(p->pStmt,iCol); + if( p->spec.eBlob==QRF_BLOB_Sql ){ + sqlite3_str_append(pOut, "x'", 2); + } + iStart = sqlite3_str_length(pOut); + sqlite3_str_appendchar(pOut, nBlob, ' '); + sqlite3_str_appendchar(pOut, nBlob, ' '); + if( p->spec.eBlob==QRF_BLOB_Sql ){ + sqlite3_str_appendchar(pOut, 1, '\''); + } + if( sqlite3_str_errcode(pOut) ) return; + zVal = sqlite3_str_value(pOut); + for(i=0, j=iStart; i>4)&0xf]; + zVal[j+1] = "0123456789abcdef"[(c)&0xf]; + } + break; + } + case QRF_BLOB_Tcl: + case QRF_BLOB_Json: { + int iStart; + int nBlob = sqlite3_column_bytes(p->pStmt,iCol); + int i, j; + char *zVal; + const unsigned char *a = sqlite3_column_blob(p->pStmt,iCol); + int szC = p->spec.eBlob==QRF_BLOB_Json ? 6 : 4; + sqlite3_str_append(pOut, "\"", 1); + iStart = sqlite3_str_length(pOut); + for(i=szC; i>0; i--){ + sqlite3_str_appendchar(pOut, nBlob, ' '); + } + sqlite3_str_appendchar(pOut, 1, '"'); + if( sqlite3_str_errcode(pOut) ) return; + zVal = sqlite3_str_value(pOut); + for(i=0, j=iStart; i>6)&3); + zVal[j+2] = '0' + ((c>>3)&7); + zVal[j+3] = '0' + (c&7); + }else{ + zVal[j+1] = 'u'; + zVal[j+2] = '0'; + zVal[j+3] = '0'; + zVal[j+4] = "0123456789abcdef"[(c>>4)&0xf]; + zVal[j+5] = "0123456789abcdef"[(c)&0xf]; + } + } + break; + } + case QRF_BLOB_Size: { + int nBlob = sqlite3_column_bytes(p->pStmt,iCol); + sqlite3_str_appendf(pOut, "(%d-byte blob)", nBlob); + break; + } + default: { + const char *zTxt = (const char*)sqlite3_column_text(p->pStmt,iCol); + qrfEncodeText(p, pOut, zTxt); + } + } + break; + } + case SQLITE_NULL: { + sqlite3_str_appendall(pOut, p->spec.zNull); + break; + } + case SQLITE_TEXT: { + const char *zTxt = (const char*)sqlite3_column_text(p->pStmt,iCol); + qrfEncodeText(p, pOut, zTxt); + break; + } + } +#if SQLITE_VERSION_NUMBER>=3052000 + if( p->spec.nCharLimit>0 + && (sqlite3_str_length(pOut) - iStartLen) > p->spec.nCharLimit + ){ + const unsigned char *z; + int ii = 0, w = 0, limit = p->spec.nCharLimit; + z = (const unsigned char*)sqlite3_str_value(pOut) + iStartLen; + if( limit<4 ) limit = 4; + while( 1 ){ + if( z[ii]<' ' ){ + int k; + if( z[ii]=='\033' && (k = qrfIsVt100(z+ii))>0 ){ + ii += k; + }else if( z[ii]==0 ){ + break; + }else{ + ii++; + } + }else if( (0x80&z[ii])==0 ){ + w++; + if( w>limit ) break; + ii++; + }else{ + int u = 0; + int len = sqlite3_qrf_decode_utf8(&z[ii], &u); + w += sqlite3_qrf_wcwidth(u); + if( w>limit ) break; + ii += len; + } + } + if( w>limit ){ + sqlite3_str_truncate(pOut, iStartLen+ii); + sqlite3_str_append(pOut, "...", 3); + } + } +#endif +} + +/* Trim spaces of the end if pOut +*/ +static void qrfRTrim(sqlite3_str *pOut){ +#if SQLITE_VERSION_NUMBER>=3052000 + int nByte = sqlite3_str_length(pOut); + const char *zOut = sqlite3_str_value(pOut); + while( nByte>0 && zOut[nByte-1]==' ' ){ nByte--; } + sqlite3_str_truncate(pOut, nByte); +#endif +} + +/* +** Store string zUtf to pOut as w characters. If w is negative, +** then right-justify the text. W is the width in display characters, not +** in bytes. Double-width unicode characters count as two characters. +** VT100 escape sequences count as zero. And so forth. +*/ +static void qrfWidthPrint(Qrf *p, sqlite3_str *pOut, int w, const char *zUtf){ + const unsigned char *a = (const unsigned char*)zUtf; + static const int mxW = 10000000; + unsigned char c; + int i = 0; + int n = 0; + int k; + int aw; + (void)p; + if( w<-mxW ){ + w = -mxW; + }else if( w>mxW ){ + w= mxW; + } + aw = w<0 ? -w : w; + if( a==0 ) a = (const unsigned char*)""; + while( (c = a[i])!=0 ){ + if( (c&0xc0)==0xc0 ){ + int u; + int len = sqlite3_qrf_decode_utf8(a+i, &u); + int x = sqlite3_qrf_wcwidth(u); + if( x+n>aw ){ + break; + } + i += len; + n += x; + }else if( c==0x1b && (k = qrfIsVt100(&a[i]))>0 ){ + i += k; + }else if( n>=aw ){ + break; + }else{ + n++; + i++; + } + } + if( n>=aw ){ + sqlite3_str_append(pOut, zUtf, i); + }else if( w<0 ){ + if( aw>n ) sqlite3_str_appendchar(pOut, aw-n, ' '); + sqlite3_str_append(pOut, zUtf, i); + }else{ + sqlite3_str_append(pOut, zUtf, i); + if( aw>n ) sqlite3_str_appendchar(pOut, aw-n, ' '); + } +} + +/* +** (*pz)[] is a line of text that is to be displayed the box or table or +** similar tabular formats. z[] contain newlines or might be too wide +** to fit in the columns so will need to be split into multiple line. +** +** This routine determines: +** +** * How many bytes of z[] should be shown on the current line. +** * How many character positions those bytes will cover. +** * The byte offset to the start of the next line. +*/ +static void qrfWrapLine( + const char *zIn, /* Input text to be displayed */ + int w, /* Column width in characters (not bytes) */ + int bWrap, /* True if we should do word-wrapping */ + int *pnThis, /* OUT: How many bytes of z[] for the current line */ + int *pnWide, /* OUT: How wide is the text of this line */ + int *piNext /* OUT: Offset into z[] to start of the next line */ +){ + int i; /* Input bytes consumed */ + int k; /* Bytes in a VT100 code */ + int n; /* Output column number */ + const unsigned char *z = (const unsigned char*)zIn; + unsigned char c = 0; + + if( z[0]==0 ){ + *pnThis = 0; + *pnWide = 0; + *piNext = 0; + return; + } + n = 0; + for(i=0; n<=w; i++){ + c = z[i]; + if( c>=0xc0 ){ + int u; + int len = sqlite3_qrf_decode_utf8(&z[i], &u); + int wcw = sqlite3_qrf_wcwidth(u); + if( wcw+n>w ) break; + i += len-1; + n += wcw; + continue; + } + if( c>=' ' ){ + if( n==w ) break; + n++; + continue; + } + if( c==0 || c=='\n' ) break; + if( c=='\r' && z[i+1]=='\n' ){ c = z[++i]; break; } + if( c=='\t' ){ + int wcw = 8 - (n&7); + if( n+wcw>w ) break; + n += wcw; + continue; + } + if( c==0x1b && (k = qrfIsVt100(&z[i]))>0 ){ + i += k-1; + }else if( n==w ){ + break; + }else{ + n++; + } + } + if( c==0 ){ + *pnThis = i; + *pnWide = n; + *piNext = i; + return; + } + if( c=='\n' ){ + *pnThis = i; + *pnWide = n; + *piNext = i+1; + return; + } + + /* If we get this far, that means the current line will end at some + ** point that is neither a "\n" or a 0x00. Figure out where that + ** split should occur + */ + if( bWrap && z[i]!=0 && !qrfSpace(z[i]) && qrfAlnum(c)==qrfAlnum(z[i]) ){ + /* Perhaps try to back up to a better place to break the line */ + for(k=i-1; k>=i/2; k--){ + if( qrfSpace(z[k]) ) break; + } + if( k=i/2; k--){ + if( qrfAlnum(z[k-1])!=qrfAlnum(z[k]) && (z[k]&0xc0)!=0x80 ) break; + } + } + if( k>=i/2 ){ + i = k; + n = qrfDisplayWidth((const char*)z, k, 0); + } + } + *pnThis = i; + *pnWide = n; + while( zIn[i]==' ' || zIn[i]=='\t' || zIn[i]=='\r' ){ i++; } + *piNext = i; +} + +/* +** Append nVal bytes of text from zVal onto the end of pOut. +** Convert tab characters in zVal to the appropriate number of +** spaces. +*/ +static void qrfAppendWithTabs( + sqlite3_str *pOut, /* Append text here */ + const char *zVal, /* Text to append */ + int nVal /* Use only the first nVal bytes of zVal[] */ +){ + int i = 0; + unsigned int col = 0; + unsigned char *z = (unsigned char *)zVal; + while( i0 ){ + sqlite3_str_append(pOut, (const char*)z, k); + z += k; + nVal -= k; + }else if( c=='\t' ){ + k = 8 - (col&7); + sqlite3_str_appendchar(pOut, k, ' '); + col += k; + z++; + nVal--; + }else if( c=='\r' && nVal==1 ){ + z++; + nVal--; + }else{ + char zCtrlPik[4]; + col++; + zCtrlPik[0] = 0xe2; + zCtrlPik[1] = 0x90; + zCtrlPik[2] = 0x80+c; + sqlite3_str_append(pOut, zCtrlPik, 3); + z++; + nVal--; + } + }else if( (0x80&c)==0 ){ + i++; + col++; + }else{ + int u = 0; + int len = sqlite3_qrf_decode_utf8(&z[i], &u); + i += len; + col += sqlite3_qrf_wcwidth(u); + } + } + sqlite3_str_append(pOut, (const char*)z, i); +} + +/* +** GCC does not define the offsetof() macro so we'll have to do it +** ourselves. +*/ +#ifndef offsetof +# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) +#endif + +/* +** Data for columnar layout, collected into a single object so +** that it can be more easily passed into subroutines. +*/ +typedef struct qrfColData qrfColData; +struct qrfColData { + Qrf *p; /* The QRF instance */ + int nCol; /* Number of columns in the table */ + unsigned char bMultiRow; /* One or more cells will span multiple lines */ + unsigned char nMargin; /* Width of column margins */ + sqlite3_int64 nRow; /* Number of rows */ + sqlite3_int64 nAlloc; /* Number of cells allocated */ + sqlite3_int64 n; /* Number of cells. nCol*nRow */ + char **az; /* Content of all cells */ + int *aiWth; /* Width of each cell */ + unsigned char *abNum; /* True for each numeric cell */ + struct qrfPerCol { /* Per-column data */ + char *z; /* Cache of text for current row */ + int w; /* Computed width of this column */ + int mxW; /* Maximum natural (unwrapped) width */ + unsigned char e; /* Alignment */ + unsigned char fx; /* Width is fixed */ + unsigned char bNum; /* True if is numeric */ + } *a; /* One per column */ +}; + +/* +** Output horizontally justified text into pOut. The text is the +** first nVal bytes of zVal. Include nWS bytes of whitespace, either +** split between both sides, or on the left, or on the right, depending +** on eAlign. +*/ +static void qrfPrintAligned( + sqlite3_str *pOut, /* Append text here */ + struct qrfPerCol *pCol, /* Information about the text to print */ + int nVal, /* Use only the first nVal bytes of zVal[] */ + int nWS /* Whitespace for horizonal alignment */ +){ + unsigned char eAlign = pCol->e & QRF_ALIGN_HMASK; + if( eAlign==QRF_Auto && pCol->bNum ) eAlign = QRF_ALIGN_Right; + if( eAlign==QRF_ALIGN_Center ){ + /* Center the text */ + sqlite3_str_appendchar(pOut, nWS/2, ' '); + qrfAppendWithTabs(pOut, pCol->z, nVal); + sqlite3_str_appendchar(pOut, nWS - nWS/2, ' '); + }else if( eAlign==QRF_ALIGN_Right ){ + /* Right justify the text */ + sqlite3_str_appendchar(pOut, nWS, ' '); + qrfAppendWithTabs(pOut, pCol->z, nVal); + }else{ + /* Left justify the text */ + qrfAppendWithTabs(pOut, pCol->z, nVal); + sqlite3_str_appendchar(pOut, nWS, ' '); + } +} + +/* +** Free all the memory allocates in the qrfColData object +*/ +static void qrfColDataFree(qrfColData *p){ + sqlite3_int64 i; + for(i=0; in; i++) sqlite3_free(p->az[i]); + sqlite3_free(p->az); + sqlite3_free(p->aiWth); + sqlite3_free(p->abNum); + sqlite3_free(p->a); + memset(p, 0, sizeof(*p)); +} + +/* +** Allocate space for more cells in the qrfColData object. +** Return non-zero if a memory allocation fails. +*/ +static int qrfColDataEnlarge(qrfColData *p){ + char **azData; + int *aiWth; + unsigned char *abNum; + p->nAlloc = 2*p->nAlloc + 10*p->nCol; + azData = sqlite3_realloc64(p->az, p->nAlloc*sizeof(char*)); + if( azData==0 ){ + qrfOom(p->p); + qrfColDataFree(p); + return 1; + } + p->az = azData; + aiWth = sqlite3_realloc64(p->aiWth, p->nAlloc*sizeof(int)); + if( aiWth==0 ){ + qrfOom(p->p); + qrfColDataFree(p); + return 1; + } + p->aiWth = aiWth; + abNum = sqlite3_realloc64(p->abNum, p->nAlloc); + if( abNum==0 ){ + qrfOom(p->p); + qrfColDataFree(p); + return 1; + } + p->abNum = abNum; + return 0; +} + +/* +** Print a markdown or table-style row separator using ascii-art +*/ +static void qrfRowSeparator(sqlite3_str *pOut, qrfColData *p, char cSep){ + int i; + if( p->nCol>0 ){ + int useBorder = p->p->spec.bBorder!=QRF_No; + if( useBorder ){ + sqlite3_str_append(pOut, &cSep, 1); + } + sqlite3_str_appendchar(pOut, p->a[0].w+p->nMargin, '-'); + for(i=1; inCol; i++){ + sqlite3_str_append(pOut, &cSep, 1); + sqlite3_str_appendchar(pOut, p->a[i].w+p->nMargin, '-'); + } + if( useBorder ){ + sqlite3_str_append(pOut, &cSep, 1); + } + } + sqlite3_str_append(pOut, "\n", 1); +} + +/* +** UTF8 box-drawing characters. Imagine box lines like this: +** +** 1 +** | +** 4 --+-- 2 +** | +** 3 +** +** Each box characters has between 2 and 4 of the lines leading from +** the center. The characters are here identified by the numbers of +** their corresponding lines. +*/ +#define BOX_24 "\342\224\200" /* U+2500 --- */ +#define BOX_13 "\342\224\202" /* U+2502 | */ +#define BOX_23 "\342\224\214" /* U+250c ,- */ +#define BOX_34 "\342\224\220" /* U+2510 -, */ +#define BOX_12 "\342\224\224" /* U+2514 '- */ +#define BOX_14 "\342\224\230" /* U+2518 -' */ +#define BOX_123 "\342\224\234" /* U+251c |- */ +#define BOX_134 "\342\224\244" /* U+2524 -| */ +#define BOX_234 "\342\224\254" /* U+252c -,- */ +#define BOX_124 "\342\224\264" /* U+2534 -'- */ +#define BOX_1234 "\342\224\274" /* U+253c -|- */ + +/* Rounded corners: */ +#define BOX_R12 "\342\225\260" /* U+2570 '- */ +#define BOX_R23 "\342\225\255" /* U+256d ,- */ +#define BOX_R34 "\342\225\256" /* U+256e -, */ +#define BOX_R14 "\342\225\257" /* U+256f -' */ + +/* Doubled horizontal lines: */ +#define DBL_24 "\342\225\220" /* U+2550 === */ +#define DBL_123 "\342\225\236" /* U+255e |= */ +#define DBL_134 "\342\225\241" /* U+2561 =| */ +#define DBL_1234 "\342\225\252" /* U+256a =|= */ + +/* Draw horizontal line N characters long using unicode box +** characters +*/ +static void qrfBoxLine(sqlite3_str *pOut, int N, int bDbl){ + const char *azDash[2] = { + BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24, + DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 + };/* 0 1 2 3 4 5 6 7 8 9 */ + const int nDash = 30; + N *= 3; + while( N>nDash ){ + sqlite3_str_append(pOut, azDash[bDbl], nDash); + N -= nDash; + } + sqlite3_str_append(pOut, azDash[bDbl], N); +} + +/* +** Draw a horizontal separator for a QRF_STYLE_Box table. +*/ +static void qrfBoxSeparator( + sqlite3_str *pOut, + qrfColData *p, + const char *zSep1, + const char *zSep2, + const char *zSep3, + int bDbl +){ + int i; + if( p->nCol>0 ){ + int useBorder = p->p->spec.bBorder!=QRF_No; + if( useBorder ){ + sqlite3_str_appendall(pOut, zSep1); + } + qrfBoxLine(pOut, p->a[0].w+p->nMargin, bDbl); + for(i=1; inCol; i++){ + sqlite3_str_appendall(pOut, zSep2); + qrfBoxLine(pOut, p->a[i].w+p->nMargin, bDbl); + } + if( useBorder ){ + sqlite3_str_appendall(pOut, zSep3); + } + } + sqlite3_str_append(pOut, "\n", 1); +} + +/* +** Load into pData the default alignment for the body of a table. +*/ +static void qrfLoadAlignment(qrfColData *pData, Qrf *p){ + sqlite3_int64 i; + for(i=0; inCol; i++){ + pData->a[i].e = p->spec.eDfltAlign; + if( ispec.nAlign ){ + unsigned char ax = p->spec.aAlign[i]; + if( (ax & QRF_ALIGN_HMASK)!=0 ){ + pData->a[i].e = (ax & QRF_ALIGN_HMASK) | + (pData->a[i].e & QRF_ALIGN_VMASK); + } + }else if( ispec.nWidth ){ + if( p->spec.aWidth[i]<0 ){ + pData->a[i].e = QRF_ALIGN_Right | + (pData->a[i].e & QRF_ALIGN_VMASK); + } + } + } +} + +/* +** If the single column in pData->a[] with pData->n entries can be +** laid out as nCol columns with a 2-space gap between each such +** that all columns fit within nSW, then return a pointer to an array +** of integers which is the width of each column from left to right. +** +** If the layout is not possible, return a NULL pointer. +** +** Space to hold the returned array is from sqlite_malloc64(). +*/ +static int *qrfValidLayout( + qrfColData *pData, /* Collected query results */ + Qrf *p, /* On which to report an OOM */ + int nCol, /* Attempt this many columns */ + int nSW /* Screen width */ +){ + int i; /* Loop counter */ + int nr; /* Number of rows */ + int w = 0; /* Width of the current column */ + int t; /* Total width of all columns */ + int *aw; /* Array of individual column widths */ + + aw = sqlite3_malloc64( sizeof(int)*nCol ); + if( aw==0 ){ + qrfOom(p); + return 0; + } + nr = (pData->n + nCol - 1)/nCol; + for(i=0; in; i++){ + if( (i%nr)==0 ){ + if( i>0 ) aw[i/nr-1] = w; + w = pData->aiWth[i]; + }else if( pData->aiWth[i]>w ){ + w = pData->aiWth[i]; + } + } + aw[nCol-1] = w; + for(t=i=0; inSW ){ + sqlite3_free(aw); + return 0; + } + return aw; +} + +/* +** The output is single-column and the bSplitColumn flag is set. +** Check to see if the single-column output can be split into multiple +** columns that appear side-by-side. Adjust pData appropriately. +*/ +static void qrfSplitColumn(qrfColData *pData, Qrf *p){ + int nCol = 1; + int *aw = 0; + char **az = 0; + int *aiWth = 0; + unsigned char *abNum = 0; + int nColNext = 2; + int w; + struct qrfPerCol *a = 0; + sqlite3_int64 nRow = 1; + sqlite3_int64 i; + while( 1/*exit-by-break*/ ){ + int *awNew = qrfValidLayout(pData, p, nColNext, p->spec.nScreenWidth); + if( awNew==0 ) break; + sqlite3_free(aw); + aw = awNew; + nCol = nColNext; + nRow = (pData->n + nCol - 1)/nCol; + if( nRow==1 ) break; + nColNext++; + while( (pData->n + nColNext - 1)/nColNext == nRow ) nColNext++; + } + if( nCol==1 ){ + sqlite3_free(aw); + return; /* Cannot do better than 1 column */ + } + az = sqlite3_malloc64( nRow*nCol*sizeof(char*) ); + if( az==0 ){ + qrfOom(p); + return; + } + aiWth = sqlite3_malloc64( nRow*nCol*sizeof(int) ); + if( aiWth==0 ){ + sqlite3_free(az); + qrfOom(p); + return; + } + a = sqlite3_malloc64( nCol*sizeof(struct qrfPerCol) ); + if( a==0 ){ + sqlite3_free(az); + sqlite3_free(aiWth); + qrfOom(p); + return; + } + abNum = sqlite3_malloc64( nRow*nCol ); + if( abNum==0 ){ + sqlite3_free(az); + sqlite3_free(aiWth); + sqlite3_free(a); + qrfOom(p); + return; + } + for(i=0; in; i++){ + sqlite3_int64 j = (i%nRow)*nCol + (i/nRow); + az[j] = pData->az[i]; + abNum[j]= pData->abNum[i]; + pData->az[i] = 0; + aiWth[j] = pData->aiWth[i]; + } + while( ia[0].e; + } + sqlite3_free(pData->az); + sqlite3_free(pData->aiWth); + sqlite3_free(pData->a); + sqlite3_free(pData->abNum); + sqlite3_free(aw); + pData->az = az; + pData->aiWth = aiWth; + pData->a = a; + pData->abNum = abNum; + pData->nCol = nCol; + pData->n = pData->nAlloc = nRow*nCol; + for(i=w=0; inMargin = (p->spec.nScreenWidth - w)/(nCol - 1); + if( pData->nMargin>5 ) pData->nMargin = 5; +} + +/* +** Adjust the layout for the screen width restriction +*/ +static void qrfRestrictScreenWidth(qrfColData *pData, Qrf *p){ + int sepW; /* Width of all box separators and margins */ + int sumW; /* Total width of data area over all columns */ + int targetW; /* Desired total data area */ + int i; /* Loop counters */ + int nCol; /* Number of columns */ + + pData->nMargin = 2; /* Default to normal margins */ + if( p->spec.nScreenWidth==0 ) return; + if( p->spec.eStyle==QRF_STYLE_Column ){ + sepW = pData->nCol*2 - 2; + }else{ + sepW = pData->nCol*3 + 1; + if( p->spec.bBorder==QRF_No ) sepW -= 2; + } + nCol = pData->nCol; + for(i=sumW=0; ia[i].w; + if( p->spec.nScreenWidth >= sumW+sepW ) return; + + /* First thing to do is reduce the separation between columns */ + pData->nMargin = 0; + if( p->spec.eStyle==QRF_STYLE_Column ){ + sepW = pData->nCol - 1; + }else{ + sepW = pData->nCol + 1; + if( p->spec.bBorder==QRF_No ) sepW -= 2; + } + targetW = p->spec.nScreenWidth - sepW; + +#define MIN_SQUOZE 8 +#define MIN_EX_SQUOZE 16 + /* Reduce the width of the widest eligible column. A column is + ** eligible for narrowing if: + ** + ** * It is not a fixed-width column (a[0].fx is false) + ** * The current width is more than MIN_SQUOZE + ** * Either: + ** + The current width is more then MIN_EX_SQUOZE, or + ** + The current width is more than half the max width (a[].mxW) + ** + ** Keep making reductions until either no more reductions are + ** possible or until the size target is reached. + */ + while( sumW > targetW ){ + int gain, w; + int ix = -1; + int mx = 0; + for(i=0; ia[i].fx==0 + && (w = pData->a[i].w)>mx + && w>MIN_SQUOZE + && (w>MIN_EX_SQUOZE || w*2>pData->a[i].mxW) + ){ + ix = i; + mx = w; + } + } + if( ix<0 ) break; + if( mx>=MIN_SQUOZE*2 ){ + gain = mx/2; + }else{ + gain = mx - MIN_SQUOZE; + } + if( sumW - gain < targetW ){ + gain = sumW - targetW; + } + sumW -= gain; + pData->a[ix].w -= gain; + pData->bMultiRow = 1; + } +} + +/* +** Columnar modes require that the entire query be evaluated first, with +** results written into memory, so that we can compute appropriate column +** widths. +*/ +static void qrfColumnar(Qrf *p){ + sqlite3_int64 i, j; /* Loop counters */ + const char *colSep = 0; /* Column separator text */ + const char *rowSep = 0; /* Row terminator text */ + const char *rowStart = 0; /* Row start text */ + int szColSep, szRowSep, szRowStart; /* Size in bytes of previous 3 */ + int rc; /* Result code */ + int nColumn = p->nCol; /* Number of columns */ + int bWW; /* True to do word-wrap */ + sqlite3_str *pStr; /* Temporary rendering */ + qrfColData data; /* Columnar layout data */ + int bRTrim; /* Trim trailing space */ + + rc = sqlite3_step(p->pStmt); + if( rc!=SQLITE_ROW || nColumn==0 ){ + return; /* No output */ + } + + /* Initialize the data container */ + memset(&data, 0, sizeof(data)); + data.nCol = p->nCol; + data.p = p; + data.a = sqlite3_malloc64( nColumn*sizeof(struct qrfPerCol) ); + if( data.a==0 ){ + qrfOom(p); + return; + } + memset(data.a, 0, nColumn*sizeof(struct qrfPerCol) ); + if( qrfColDataEnlarge(&data) ) return; + assert( data.az!=0 ); + + /* Load the column header names and all cell content into data */ + if( p->spec.bTitles==QRF_Yes ){ + unsigned char saved_eText = p->spec.eText; + p->spec.eText = p->spec.eTitle; + memset(data.abNum, 0, nColumn); + for(i=0; ipStmt,i); + int nNL = 0; + int n, w; + pStr = sqlite3_str_new(p->db); + qrfEncodeText(p, pStr, z ? z : ""); + n = sqlite3_str_length(pStr); + qrfStrErr(p, pStr); + z = data.az[data.n] = sqlite3_str_finish(pStr); + if( p->spec.nTitleLimit ){ + nNL = 0; + data.aiWth[data.n] = w = qrfTitleLimit(data.az[data.n], + p->spec.nTitleLimit ); + }else{ + data.aiWth[data.n] = w = qrfDisplayWidth(z, n, &nNL); + } + data.n++; + if( w>data.a[i].mxW ) data.a[i].mxW = w; + if( nNL ) data.bMultiRow = 1; + } + p->spec.eText = saved_eText; + p->nRow++; + } + do{ + if( data.n+nColumn > data.nAlloc ){ + if( qrfColDataEnlarge(&data) ) return; + } + for(i=0; ipStmt,i); + pStr = sqlite3_str_new(p->db); + qrfRenderValue(p, pStr, i); + n = sqlite3_str_length(pStr); + qrfStrErr(p, pStr); + z = data.az[data.n] = sqlite3_str_finish(pStr); + data.abNum[data.n] = eType==SQLITE_INTEGER || eType==SQLITE_FLOAT; + data.aiWth[data.n] = w = qrfDisplayWidth(z, n, &nNL); + data.n++; + if( w>data.a[i].mxW ) data.a[i].mxW = w; + if( nNL ) data.bMultiRow = 1; + } + p->nRow++; + }while( sqlite3_step(p->pStmt)==SQLITE_ROW && p->iErr==SQLITE_OK ); + if( p->iErr ){ + qrfColDataFree(&data); + return; + } + + /* Compute the width and alignment of every column */ + if( p->spec.bTitles==QRF_No ){ + qrfLoadAlignment(&data, p); + }else{ + unsigned char e; + if( p->spec.eTitleAlign==QRF_Auto ){ + e = QRF_ALIGN_Center; + }else{ + e = p->spec.eTitleAlign; + } + for(i=0; ispec.nWidth ){ + w = p->spec.aWidth[i]; + if( w==(-32768) ){ + w = 0; + if( p->spec.nAlign>i && (p->spec.aAlign[i] & QRF_ALIGN_HMASK)==0 ){ + data.a[i].e |= QRF_ALIGN_Right; + } + }else if( w<0 ){ + w = -w; + if( p->spec.nAlign>i && (p->spec.aAlign[i] & QRF_ALIGN_HMASK)==0 ){ + data.a[i].e |= QRF_ALIGN_Right; + } + } + if( w ) data.a[i].fx = 1; + } + if( w==0 ){ + w = data.a[i].mxW; + if( p->spec.nWrap>0 && w>p->spec.nWrap ){ + w = p->spec.nWrap; + data.bMultiRow = 1; + } + }else if( (data.bMultiRow==0 || w==1) && data.a[i].mxW>w ){ + data.bMultiRow = 1; + if( w==1 ){ + /* If aiWth[j] is 2 or more, then there might be a double-wide + ** character somewhere. So make the column width at least 2. */ + w = 2; + } + } + data.a[i].w = w; + } + + if( nColumn==1 + && data.n>1 + && p->spec.bSplitColumn==QRF_Yes + && p->spec.eStyle==QRF_STYLE_Column + && p->spec.bTitles==QRF_No + && p->spec.nScreenWidth>data.a[0].w+3 + ){ + /* Attempt to convert single-column tables into multi-column by + ** verticle wrapping, if the screen is wide enough and if the + ** bSplitColumn flag is set. */ + qrfSplitColumn(&data, p); + nColumn = data.nCol; + }else{ + /* Adjust the column widths due to screen width restrictions */ + qrfRestrictScreenWidth(&data, p); + } + + /* Draw the line across the top of the table. Also initialize + ** the row boundary and column separator texts. */ + switch( p->spec.eStyle ){ + case QRF_STYLE_Box: + if( data.nMargin ){ + rowStart = BOX_13 " "; + colSep = " " BOX_13 " "; + rowSep = " " BOX_13 "\n"; + }else{ + rowStart = BOX_13; + colSep = BOX_13; + rowSep = BOX_13 "\n"; + } + if( p->spec.bBorder==QRF_No){ + rowStart += 3; + rowSep = "\n"; + }else{ + qrfBoxSeparator(p->pOut, &data, BOX_R23, BOX_234, BOX_R34, 0); + } + break; + case QRF_STYLE_Table: + if( data.nMargin ){ + rowStart = "| "; + colSep = " | "; + rowSep = " |\n"; + }else{ + rowStart = "|"; + colSep = "|"; + rowSep = "|\n"; + } + if( p->spec.bBorder==QRF_No ){ + rowStart += 1; + rowSep = "\n"; + }else{ + qrfRowSeparator(p->pOut, &data, '+'); + } + break; + case QRF_STYLE_Column: { + static const char zSpace[] = " "; + rowStart = ""; + if( data.nMargin<2 ){ + colSep = " "; + }else if( data.nMargin<=5 ){ + colSep = &zSpace[5-data.nMargin]; + }else{ + colSep = zSpace; + } + rowSep = "\n"; + break; + } + default: /*case QRF_STYLE_Markdown:*/ + if( data.nMargin ){ + rowStart = "| "; + colSep = " | "; + rowSep = " |\n"; + }else{ + rowStart = "|"; + colSep = "|"; + rowSep = "|\n"; + } + break; + } + szRowStart = (int)strlen(rowStart); + szRowSep = (int)strlen(rowSep); + szColSep = (int)strlen(colSep); + + bWW = (p->spec.bWordWrap==QRF_Yes && data.bMultiRow); + if( p->spec.eStyle==QRF_STYLE_Column + || (p->spec.bBorder==QRF_No + && (p->spec.eStyle==QRF_STYLE_Box || p->spec.eStyle==QRF_STYLE_Table) + ) + ){ + bRTrim = 1; + }else{ + bRTrim = 0; + } + for(i=0; ipOut)==SQLITE_OK; i+=nColumn){ + int bMore; + int nRow = 0; + + /* Draw a single row of the table. This might be the title line + ** (if there is a title line) or a row in the body of the table. + ** The column number will be j. The row number is i/nColumn. + */ + for(j=0; jpOut, rowStart, szRowStart); + bMore = 0; + for(j=0; jpOut, &data.a[j], nThis, nWS); + data.a[j].z += iNext; + if( data.a[j].z[0]!=0 ){ + bMore = 1; + } + if( jpOut, colSep, szColSep); + }else{ + if( bRTrim ) qrfRTrim(p->pOut); + sqlite3_str_append(p->pOut, rowSep, szRowSep); + } + } + }while( bMore && ++nRow < p->mxHeight ); + if( bMore ){ + /* This row was terminated by nLineLimit. Show ellipsis. */ + sqlite3_str_append(p->pOut, rowStart, szRowStart); + for(j=0; jpOut, data.a[j].w, ' '); + }else{ + int nE = 3; + if( nE>data.a[j].w ) nE = data.a[j].w; + data.a[j].z = "..."; + qrfPrintAligned(p->pOut, &data.a[j], nE, data.a[j].w-nE); + } + if( jpOut, colSep, szColSep); + }else{ + if( bRTrim ) qrfRTrim(p->pOut); + sqlite3_str_append(p->pOut, rowSep, szRowSep); + } + } + } + + /* Draw either (1) the separator between the title line and the body + ** of the table, or (2) separators between individual rows of the table + ** body. isTitleDataSeparator will be true if we are doing (1). + */ + if( (i==0 || data.bMultiRow) && i+nColumnspec.bTitles==QRF_Yes); + if( isTitleDataSeparator ){ + qrfLoadAlignment(&data, p); + } + switch( p->spec.eStyle ){ + case QRF_STYLE_Table: { + if( isTitleDataSeparator || data.bMultiRow ){ + qrfRowSeparator(p->pOut, &data, '+'); + } + break; + } + case QRF_STYLE_Box: { + if( isTitleDataSeparator ){ + qrfBoxSeparator(p->pOut, &data, DBL_123, DBL_1234, DBL_134, 1); + }else if( data.bMultiRow ){ + qrfBoxSeparator(p->pOut, &data, BOX_123, BOX_1234, BOX_134, 0); + } + break; + } + case QRF_STYLE_Markdown: { + if( isTitleDataSeparator ){ + qrfRowSeparator(p->pOut, &data, '|'); + } + break; + } + case QRF_STYLE_Column: { + if( isTitleDataSeparator ){ + for(j=0; jpOut, data.a[j].w, '-'); + if( jpOut, colSep, szColSep); + }else{ + qrfRTrim(p->pOut); + sqlite3_str_append(p->pOut, rowSep, szRowSep); + } + } + }else if( data.bMultiRow ){ + qrfRTrim(p->pOut); + sqlite3_str_append(p->pOut, "\n", 1); + } + break; + } + } + } + } + + /* Draw the line across the bottom of the table */ + if( p->spec.bBorder!=QRF_No ){ + switch( p->spec.eStyle ){ + case QRF_STYLE_Box: + qrfBoxSeparator(p->pOut, &data, BOX_R12, BOX_124, BOX_R14, 0); + break; + case QRF_STYLE_Table: + qrfRowSeparator(p->pOut, &data, '+'); + break; + } + } + qrfWrite(p); + + qrfColDataFree(&data); + return; +} + +/* +** Parameter azArray points to a zero-terminated array of strings. zStr +** points to a single nul-terminated string. Return non-zero if zStr +** is equal, according to strcmp(), to any of the strings in the array. +** Otherwise, return zero. +*/ +static int qrfStringInArray(const char *zStr, const char **azArray){ + int i; + if( zStr==0 ) return 0; + for(i=0; azArray[i]; i++){ + if( 0==strcmp(zStr, azArray[i]) ) return 1; + } + return 0; +} + +/* +** Print out an EXPLAIN with indentation. This is a two-pass algorithm. +** +** On the first pass, we compute aiIndent[iOp] which is the amount of +** indentation to apply to the iOp-th opcode. The output actually occurs +** on the second pass. +** +** The indenting rules are: +** +** * For each "Next", "Prev", "VNext" or "VPrev" instruction, indent +** all opcodes that occur between the p2 jump destination and the opcode +** itself by 2 spaces. +** +** * Do the previous for "Return" instructions for when P2 is positive. +** See tag-20220407a in wherecode.c and vdbe.c. +** +** * For each "Goto", if the jump destination is earlier in the program +** and ends on one of: +** Yield SeekGt SeekLt RowSetRead Rewind +** or if the P1 parameter is one instead of zero, +** then indent all opcodes between the earlier instruction +** and "Goto" by 2 spaces. +*/ +static void qrfExplain(Qrf *p){ + int *abYield = 0; /* abYield[iOp] is rue if opcode iOp is an OP_Yield */ + int *aiIndent = 0; /* Indent the iOp-th opcode by aiIndent[iOp] */ + i64 nAlloc = 0; /* Allocated size of aiIndent[], abYield */ + int nIndent = 0; /* Number of entries in aiIndent[] */ + int iOp; /* Opcode number */ + int i; /* Column loop counter */ + + const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", + "Return", 0 }; + const char *azYield[] = { "Yield", "SeekLT", "SeekGT", "RowSetRead", + "Rewind", 0 }; + const char *azGoto[] = { "Goto", 0 }; + + /* The caller guarantees that the leftmost 4 columns of the statement + ** passed to this function are equivalent to the leftmost 4 columns + ** of EXPLAIN statement output. In practice the statement may be + ** an EXPLAIN, or it may be a query on the bytecode() virtual table. */ + assert( sqlite3_column_count(p->pStmt)>=4 ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(p->pStmt, 0), "addr" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(p->pStmt, 1), "opcode" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(p->pStmt, 2), "p1" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(p->pStmt, 3), "p2" ) ); + + for(iOp=0; SQLITE_ROW==sqlite3_step(p->pStmt) && !p->iErr; iOp++){ + int iAddr = sqlite3_column_int(p->pStmt, 0); + const char *zOp = (const char*)sqlite3_column_text(p->pStmt, 1); + int p1 = sqlite3_column_int(p->pStmt, 2); + int p2 = sqlite3_column_int(p->pStmt, 3); + + /* Assuming that p2 is an instruction address, set variable p2op to the + ** index of that instruction in the aiIndent[] array. p2 and p2op may be + ** different if the current instruction is part of a sub-program generated + ** by an SQL trigger or foreign key. */ + int p2op = (p2 + (iOp-iAddr)); + + /* Grow the aiIndent array as required */ + if( iOp>=nAlloc ){ + nAlloc += 100; + aiIndent = (int*)sqlite3_realloc64(aiIndent, nAlloc*sizeof(int)); + abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int)); + if( aiIndent==0 || abYield==0 ){ + qrfOom(p); + sqlite3_free(aiIndent); + sqlite3_free(abYield); + return; + } + } + + abYield[iOp] = qrfStringInArray(zOp, azYield); + aiIndent[iOp] = 0; + nIndent = iOp+1; + if( qrfStringInArray(zOp, azNext) && p2op>0 ){ + for(i=p2op; ipStmt); + if( p->iErr==SQLITE_OK ){ + static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13}; + static const int aExplainMap[] = {0, 1, 2, 3, 4, 5, 6, 7 }; + static const int aScanExpWidth[] = {4,15, 6, 13, 4, 4, 4, 13, 2, 13}; + static const int aScanExpMap[] = {0, 9, 8, 1, 2, 3, 4, 5, 6, 7 }; + const int *aWidth = aExplainWidth; + const int *aMap = aExplainMap; + int nWidth = sizeof(aExplainWidth)/sizeof(int); + int iIndent = 1; + int nArg = p->nCol; + if( p->spec.eStyle==QRF_STYLE_StatsVm ){ + aWidth = aScanExpWidth; + aMap = aScanExpMap; + nWidth = sizeof(aScanExpWidth)/sizeof(int); + iIndent = 3; + } + if( nArg>nWidth ) nArg = nWidth; + + for(iOp=0; sqlite3_step(p->pStmt)==SQLITE_ROW && !p->iErr; iOp++){ + /* If this is the first row seen, print out the headers */ + if( iOp==0 ){ + for(i=0; ipStmt, aMap[i]); + qrfWidthPrint(p,p->pOut, aWidth[i], zCol); + if( i==nArg-1 ){ + sqlite3_str_append(p->pOut, "\n", 1); + }else{ + sqlite3_str_append(p->pOut, " ", 2); + } + } + for(i=0; ipOut, "%.*c", aWidth[i], '-'); + if( i==nArg-1 ){ + sqlite3_str_append(p->pOut, "\n", 1); + }else{ + sqlite3_str_append(p->pOut, " ", 2); + } + } + } + + for(i=0; ipStmt, aMap[i]); + int len; + if( i==nArg-1 ) w = 0; + if( zVal==0 ) zVal = ""; + len = (int)sqlite3_qrf_wcswidth(zVal); + if( len>w ){ + w = len; + zSep = " "; + } + if( i==iIndent && aiIndent && iOppOut, aiIndent[iOp], ' '); + } + qrfWidthPrint(p, p->pOut, w, zVal); + if( i==nArg-1 ){ + sqlite3_str_append(p->pOut, "\n", 1); + }else{ + sqlite3_str_appendall(p->pOut, zSep); + } + } + p->nRow++; + } + qrfWrite(p); + } + sqlite3_free(aiIndent); +} + +/* +** Do a "scanstatus vm" style EXPLAIN listing on p->pStmt. +** +** p->pStmt is probably not an EXPLAIN query. Instead, construct a +** new query that is a bytecode() rendering of p->pStmt with extra +** columns for the "scanstatus vm" outputs, and run the results of +** that new query through the normal EXPLAIN formatting. +*/ +static void qrfScanStatusVm(Qrf *p){ + sqlite3_stmt *pOrigStmt = p->pStmt; + sqlite3_stmt *pExplain; + int rc; + static const char *zSql = + " SELECT addr, opcode, p1, p2, p3, p4, p5, comment, nexec," + " format('% 6s (%.2f%%)'," + " CASE WHEN ncycle<100_000 THEN ncycle || ' '" + " WHEN ncycle<100_000_000 THEN (ncycle/1_000) || 'K'" + " WHEN ncycle<100_000_000_000 THEN (ncycle/1_000_000) || 'M'" + " ELSE (ncycle/1000_000_000) || 'G' END," + " ncycle*100.0/(sum(ncycle) OVER ())" + " ) AS cycles" + " FROM bytecode(?1)"; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pExplain, 0); + if( rc ){ + qrfError(p, rc, "%s", sqlite3_errmsg(p->db)); + sqlite3_finalize(pExplain); + return; + } + sqlite3_bind_pointer(pExplain, 1, pOrigStmt, "stmt-pointer", 0); + p->pStmt = pExplain; + p->nCol = 10; + qrfExplain(p); + sqlite3_finalize(pExplain); + p->pStmt = pOrigStmt; +} + +/* +** Attempt to determine if identifier zName needs to be quoted, either +** because it contains non-alphanumeric characters, or because it is an +** SQLite keyword. Be conservative in this estimate: When in doubt assume +** that quoting is required. +** +** Return 1 if quoting is required. Return 0 if no quoting is required. +*/ + +static int qrf_need_quote(const char *zName){ + int i; + const unsigned char *z = (const unsigned char*)zName; + if( z==0 ) return 1; + if( !qrfAlpha(z[0]) ) return 1; + for(i=0; z[i]; i++){ + if( !qrfAlnum(z[i]) ) return 1; + } + return sqlite3_keyword_check(zName, i)!=0; +} + +/* +** Helper function for QRF_STYLE_Json and QRF_STYLE_JObject. +** The initial "{" for a JSON object that will contain row content +** has been output. Now output all the content. +*/ +static void qrfOneJsonRow(Qrf *p){ + int i, nItem; + for(nItem=i=0; inCol; i++){ + const char *zCName; + zCName = sqlite3_column_name(p->pStmt, i); + if( nItem>0 ) sqlite3_str_append(p->pOut, ",", 1); + nItem++; + qrfEncodeText(p, p->pOut, zCName); + sqlite3_str_append(p->pOut, ":", 1); + qrfRenderValue(p, p->pOut, i); + } + qrfWrite(p); +} + +/* +** Render a single row of output for non-columnar styles - any +** style that lets us render row by row as the content is received +** from the query. +*/ +static void qrfOneSimpleRow(Qrf *p){ + int i; + switch( p->spec.eStyle ){ + case QRF_STYLE_Off: + case QRF_STYLE_Count: { + /* No-op */ + break; + } + case QRF_STYLE_Json: { + if( p->nRow==0 ){ + sqlite3_str_append(p->pOut, "[{", 2); + }else{ + sqlite3_str_append(p->pOut, "},\n{", 4); + } + qrfOneJsonRow(p); + break; + } + case QRF_STYLE_JObject: { + if( p->nRow==0 ){ + sqlite3_str_append(p->pOut, "{", 1); + }else{ + sqlite3_str_append(p->pOut, "}\n{", 3); + } + qrfOneJsonRow(p); + break; + } + case QRF_STYLE_Html: { + if( p->nRow==0 && p->spec.bTitles==QRF_Yes ){ + sqlite3_str_append(p->pOut, "", 4); + for(i=0; inCol; i++){ + const char *zCName = sqlite3_column_name(p->pStmt, i); + sqlite3_str_append(p->pOut, "\n", 5); + qrfEncodeText(p, p->pOut, zCName); + } + sqlite3_str_append(p->pOut, "\n\n", 7); + } + sqlite3_str_append(p->pOut, "", 4); + for(i=0; inCol; i++){ + sqlite3_str_append(p->pOut, "\n", 5); + qrfRenderValue(p, p->pOut, i); + } + sqlite3_str_append(p->pOut, "\n\n", 7); + qrfWrite(p); + break; + } + case QRF_STYLE_Insert: { + if( qrf_need_quote(p->spec.zTableName) ){ + sqlite3_str_appendf(p->pOut,"INSERT INTO \"%w\"",p->spec.zTableName); + }else{ + sqlite3_str_appendf(p->pOut,"INSERT INTO %s",p->spec.zTableName); + } + if( p->spec.bTitles==QRF_Yes ){ + for(i=0; inCol; i++){ + const char *zCName = sqlite3_column_name(p->pStmt, i); + if( qrf_need_quote(zCName) ){ + sqlite3_str_appendf(p->pOut, "%c\"%w\"", + i==0 ? '(' : ',', zCName); + }else{ + sqlite3_str_appendf(p->pOut, "%c%s", + i==0 ? '(' : ',', zCName); + } + } + sqlite3_str_append(p->pOut, ")", 1); + } + sqlite3_str_append(p->pOut," VALUES(", 8); + for(i=0; inCol; i++){ + if( i>0 ) sqlite3_str_append(p->pOut, ",", 1); + qrfRenderValue(p, p->pOut, i); + } + sqlite3_str_append(p->pOut, ");\n", 3); + qrfWrite(p); + break; + } + case QRF_STYLE_Line: { + sqlite3_str *pVal; + int mxW; + int bWW; + int nSep; + if( p->u.sLine.azCol==0 ){ + p->u.sLine.azCol = sqlite3_malloc64( p->nCol*sizeof(char*) ); + if( p->u.sLine.azCol==0 ){ + qrfOom(p); + break; + } + p->u.sLine.mxColWth = 0; + for(i=0; inCol; i++){ + int sz; + const char *zCName = sqlite3_column_name(p->pStmt, i); + if( zCName==0 ) zCName = "unknown"; + p->u.sLine.azCol[i] = sqlite3_mprintf("%s", zCName); + if( p->spec.nTitleLimit>0 ){ + (void)qrfTitleLimit(p->u.sLine.azCol[i], p->spec.nTitleLimit); + } + sz = (int)sqlite3_qrf_wcswidth(p->u.sLine.azCol[i]); + if( sz > p->u.sLine.mxColWth ) p->u.sLine.mxColWth = sz; + } + } + if( p->nRow ) sqlite3_str_append(p->pOut, "\n", 1); + pVal = sqlite3_str_new(p->db); + nSep = (int)strlen(p->spec.zColumnSep); + mxW = p->mxWidth - (nSep + p->u.sLine.mxColWth); + bWW = p->spec.bWordWrap==QRF_Yes; + for(i=0; inCol; i++){ + const char *zVal; + int cnt = 0; + qrfWidthPrint(p, p->pOut, -p->u.sLine.mxColWth, p->u.sLine.azCol[i]); + sqlite3_str_append(p->pOut, p->spec.zColumnSep, nSep); + qrfRenderValue(p, pVal, i); + zVal = sqlite3_str_value(pVal); + if( zVal==0 ) zVal = ""; + do{ + int nThis, nWide, iNext; + qrfWrapLine(zVal, mxW, bWW, &nThis, &nWide, &iNext); + if( cnt ) sqlite3_str_appendchar(p->pOut,p->u.sLine.mxColWth+2,' '); + cnt++; + if( cnt>p->mxHeight ){ + zVal = "..."; + nThis = iNext = 3; + } + sqlite3_str_append(p->pOut, zVal, nThis); + sqlite3_str_append(p->pOut, "\n", 1); + zVal += iNext; + }while( zVal[0] ); + sqlite3_str_reset(pVal); + } + qrfStrErr(p, pVal); + sqlite3_free(sqlite3_str_finish(pVal)); + qrfWrite(p); + break; + } + case QRF_STYLE_Eqp: { + const char *zEqpLine = (const char*)sqlite3_column_text(p->pStmt,3); + int iEqpId = sqlite3_column_int(p->pStmt, 0); + int iParentId = sqlite3_column_int(p->pStmt, 1); + if( zEqpLine==0 ) zEqpLine = ""; + if( zEqpLine[0]=='-' ) qrfEqpRender(p, 0); + qrfEqpAppend(p, iEqpId, iParentId, zEqpLine); + break; + } + default: { /* QRF_STYLE_List */ + if( p->nRow==0 && p->spec.bTitles==QRF_Yes ){ + int saved_eText = p->spec.eText; + p->spec.eText = p->spec.eTitle; + for(i=0; inCol; i++){ + const char *zCName = sqlite3_column_name(p->pStmt, i); + if( i>0 ) sqlite3_str_appendall(p->pOut, p->spec.zColumnSep); + qrfEncodeText(p, p->pOut, zCName); + } + sqlite3_str_appendall(p->pOut, p->spec.zRowSep); + qrfWrite(p); + p->spec.eText = saved_eText; + } + for(i=0; inCol; i++){ + if( i>0 ) sqlite3_str_appendall(p->pOut, p->spec.zColumnSep); + qrfRenderValue(p, p->pOut, i); + } + sqlite3_str_appendall(p->pOut, p->spec.zRowSep); + qrfWrite(p); + break; + } + } + p->nRow++; +} + +/* +** Initialize the internal Qrf object. +*/ +static void qrfInitialize( + Qrf *p, /* State object to be initialized */ + sqlite3_stmt *pStmt, /* Query whose output to be formatted */ + const sqlite3_qrf_spec *pSpec, /* Format specification */ + char **pzErr /* Write errors here */ +){ + size_t sz; /* Size of pSpec[], based on pSpec->iVersion */ + memset(p, 0, sizeof(*p)); + p->pzErr = pzErr; + if( pSpec->iVersion!=1 ){ + qrfError(p, SQLITE_ERROR, + "unusable sqlite3_qrf_spec.iVersion (%d)", + pSpec->iVersion); + return; + } + p->pStmt = pStmt; + p->db = sqlite3_db_handle(pStmt); + p->pOut = sqlite3_str_new(p->db); + if( p->pOut==0 ){ + qrfOom(p); + return; + } + p->iErr = SQLITE_OK; + p->nCol = sqlite3_column_count(p->pStmt); + p->nRow = 0; + sz = sizeof(sqlite3_qrf_spec); + memcpy(&p->spec, pSpec, sz); + if( p->spec.zNull==0 ) p->spec.zNull = ""; + p->mxWidth = p->spec.nScreenWidth; + if( p->mxWidth<=0 ) p->mxWidth = QRF_MAX_WIDTH; + p->mxHeight = p->spec.nLineLimit; + if( p->mxHeight<=0 ) p->mxHeight = 2147483647; + if( p->spec.eStyle>QRF_STYLE_Table ) p->spec.eStyle = QRF_Auto; + if( p->spec.eEsc>QRF_ESC_Symbol ) p->spec.eEsc = QRF_Auto; + if( p->spec.eText>QRF_TEXT_Relaxed ) p->spec.eText = QRF_Auto; + if( p->spec.eTitle>QRF_TEXT_Relaxed ) p->spec.eTitle = QRF_Auto; + if( p->spec.eBlob>QRF_BLOB_Size ) p->spec.eBlob = QRF_Auto; +qrf_reinit: + switch( p->spec.eStyle ){ + case QRF_Auto: { + switch( sqlite3_stmt_isexplain(pStmt) ){ + case 0: p->spec.eStyle = QRF_STYLE_Box; break; + case 1: p->spec.eStyle = QRF_STYLE_Explain; break; + default: p->spec.eStyle = QRF_STYLE_Eqp; break; + } + goto qrf_reinit; + } + case QRF_STYLE_List: { + if( p->spec.zColumnSep==0 ) p->spec.zColumnSep = "|"; + if( p->spec.zRowSep==0 ) p->spec.zRowSep = "\n"; + break; + } + case QRF_STYLE_JObject: + case QRF_STYLE_Json: { + p->spec.eText = QRF_TEXT_Json; + p->spec.zNull = "null"; + break; + } + case QRF_STYLE_Html: { + p->spec.eText = QRF_TEXT_Html; + p->spec.zNull = "null"; + break; + } + case QRF_STYLE_Insert: { + p->spec.eText = QRF_TEXT_Sql; + p->spec.zNull = "NULL"; + if( p->spec.zTableName==0 || p->spec.zTableName[0]==0 ){ + p->spec.zTableName = "tab"; + } + break; + } + case QRF_STYLE_Line: { + if( p->spec.zColumnSep==0 ){ + p->spec.zColumnSep = ": "; + } + break; + } + case QRF_STYLE_Csv: { + p->spec.eStyle = QRF_STYLE_List; + p->spec.eText = QRF_TEXT_Csv; + p->spec.zColumnSep = ","; + p->spec.zRowSep = "\r\n"; + p->spec.zNull = ""; + break; + } + case QRF_STYLE_Quote: { + p->spec.eText = QRF_TEXT_Sql; + p->spec.zNull = "NULL"; + p->spec.zColumnSep = ","; + p->spec.zRowSep = "\n"; + break; + } + case QRF_STYLE_Eqp: { + int expMode = sqlite3_stmt_isexplain(p->pStmt); + if( expMode!=2 ){ + sqlite3_stmt_explain(p->pStmt, 2); + p->expMode = expMode+1; + } + break; + } + case QRF_STYLE_Explain: { + int expMode = sqlite3_stmt_isexplain(p->pStmt); + if( expMode!=1 ){ + sqlite3_stmt_explain(p->pStmt, 1); + p->expMode = expMode+1; + } + break; + } + } + if( p->spec.eEsc==QRF_Auto ){ + p->spec.eEsc = QRF_ESC_Ascii; + } + if( p->spec.eText==QRF_Auto ){ + p->spec.eText = QRF_TEXT_Plain; + } + if( p->spec.eTitle==QRF_Auto ){ + switch( p->spec.eStyle ){ + case QRF_STYLE_Box: + case QRF_STYLE_Column: + case QRF_STYLE_Table: + p->spec.eTitle = QRF_TEXT_Plain; + break; + default: + p->spec.eTitle = p->spec.eText; + break; + } + } + if( p->spec.eBlob==QRF_Auto ){ + switch( p->spec.eText ){ + case QRF_TEXT_Sql: p->spec.eBlob = QRF_BLOB_Sql; break; + case QRF_TEXT_Csv: p->spec.eBlob = QRF_BLOB_Tcl; break; + case QRF_TEXT_Tcl: p->spec.eBlob = QRF_BLOB_Tcl; break; + case QRF_TEXT_Json: p->spec.eBlob = QRF_BLOB_Json; break; + default: p->spec.eBlob = QRF_BLOB_Text; break; + } + } + if( p->spec.bTitles==QRF_Auto ){ + switch( p->spec.eStyle ){ + case QRF_STYLE_Box: + case QRF_STYLE_Csv: + case QRF_STYLE_Column: + case QRF_STYLE_Table: + case QRF_STYLE_Markdown: + p->spec.bTitles = QRF_Yes; + break; + default: + p->spec.bTitles = QRF_No; + break; + } + } + if( p->spec.bWordWrap==QRF_Auto ){ + p->spec.bWordWrap = QRF_Yes; + } + if( p->spec.bTextJsonb==QRF_Auto ){ + p->spec.bTextJsonb = QRF_No; + } + if( p->spec.zColumnSep==0 ) p->spec.zColumnSep = ","; + if( p->spec.zRowSep==0 ) p->spec.zRowSep = "\n"; +} + +/* +** Finish rendering the results +*/ +static void qrfFinalize(Qrf *p){ + switch( p->spec.eStyle ){ + case QRF_STYLE_Count: { + sqlite3_str_appendf(p->pOut, "%lld\n", p->nRow); + qrfWrite(p); + break; + } + case QRF_STYLE_Json: { + if( p->nRow>0 ){ + sqlite3_str_append(p->pOut, "}]\n", 3); + qrfWrite(p); + } + break; + } + case QRF_STYLE_JObject: { + if( p->nRow>0 ){ + sqlite3_str_append(p->pOut, "}\n", 2); + qrfWrite(p); + } + break; + } + case QRF_STYLE_Line: { + if( p->u.sLine.azCol ){ + int i; + for(i=0; inCol; i++) sqlite3_free(p->u.sLine.azCol[i]); + sqlite3_free(p->u.sLine.azCol); + } + break; + } + case QRF_STYLE_Stats: + case QRF_STYLE_StatsEst: { + i64 nCycle = 0; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + sqlite3_stmt_scanstatus_v2(p->pStmt, -1, SQLITE_SCANSTAT_NCYCLE, + SQLITE_SCANSTAT_COMPLEX, (void*)&nCycle); +#endif + qrfEqpRender(p, nCycle); + qrfWrite(p); + break; + } + case QRF_STYLE_Eqp: { + qrfEqpRender(p, 0); + qrfWrite(p); + break; + } + } + qrfStrErr(p, p->pOut); + if( p->spec.pzOutput ){ + if( p->spec.pzOutput[0] ){ + sqlite3_int64 n, sz; + char *zCombined; + sz = strlen(p->spec.pzOutput[0]); + n = sqlite3_str_length(p->pOut); + zCombined = sqlite3_realloc64(p->spec.pzOutput[0], sz+n+1); + if( zCombined==0 ){ + sqlite3_free(p->spec.pzOutput[0]); + p->spec.pzOutput[0] = 0; + qrfOom(p); + }else{ + p->spec.pzOutput[0] = zCombined; + memcpy(zCombined+sz, sqlite3_str_value(p->pOut), n+1); + } + sqlite3_free(sqlite3_str_finish(p->pOut)); + }else{ + p->spec.pzOutput[0] = sqlite3_str_finish(p->pOut); + } + }else if( p->pOut ){ + sqlite3_free(sqlite3_str_finish(p->pOut)); + } + if( p->expMode>0 ){ + sqlite3_stmt_explain(p->pStmt, p->expMode-1); + } + if( p->actualWidth ){ + sqlite3_free(p->actualWidth); + } + if( p->pJTrans ){ + sqlite3 *db = sqlite3_db_handle(p->pJTrans); + sqlite3_finalize(p->pJTrans); + sqlite3_close(db); + } +} + +/* +** Run the prepared statement pStmt and format the results according +** to the specification provided in pSpec. Return an error code. +** If pzErr is not NULL and if an error occurs, write an error message +** into *pzErr. +*/ +int sqlite3_format_query_result( + sqlite3_stmt *pStmt, /* Statement to evaluate */ + const sqlite3_qrf_spec *pSpec, /* Format specification */ + char **pzErr /* Write error message here */ +){ + Qrf qrf; /* The new Qrf being created */ + + if( pStmt==0 ) return SQLITE_OK; /* No-op */ + if( pSpec==0 ) return SQLITE_MISUSE; + qrfInitialize(&qrf, pStmt, pSpec, pzErr); + switch( qrf.spec.eStyle ){ + case QRF_STYLE_Box: + case QRF_STYLE_Column: + case QRF_STYLE_Markdown: + case QRF_STYLE_Table: { + /* Columnar modes require that the entire query be evaluated and the + ** results stored in memory, so that we can compute column widths */ + qrfColumnar(&qrf); + break; + } + case QRF_STYLE_Explain: { + qrfExplain(&qrf); + break; + } + case QRF_STYLE_StatsVm: { + qrfScanStatusVm(&qrf); + break; + } + case QRF_STYLE_Stats: + case QRF_STYLE_StatsEst: { + qrfEqpStats(&qrf); + break; + } + default: { + /* Non-columnar modes where the output can occur after each row + ** of result is received */ + while( qrf.iErr==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + qrfOneSimpleRow(&qrf); + } + break; + } + } + qrfResetStmt(&qrf); + qrfFinalize(&qrf); + return qrf.iErr; +} diff --git a/ext/qrf/qrf.h b/ext/qrf/qrf.h new file mode 100644 index 000000000..c23ec772f --- /dev/null +++ b/ext/qrf/qrf.h @@ -0,0 +1,200 @@ +/* +** 2025-10-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Header file for the Result-Format or "resfmt" utility library for SQLite. +** See the resfmt.md documentation for additional information. +*/ +#ifndef SQLITE_QRF_H +#define SQLITE_QRF_H +#ifdef __cplusplus +extern "C" { +#endif +#include +#include "sqlite3.h" + +/* +** Specification used by clients to define the output format they want +*/ +typedef struct sqlite3_qrf_spec sqlite3_qrf_spec; +struct sqlite3_qrf_spec { + unsigned char iVersion; /* Version number of this structure */ + unsigned char eStyle; /* Formatting style. "box", "csv", etc... */ + unsigned char eEsc; /* How to escape control characters in text */ + unsigned char eText; /* Quoting style for text */ + unsigned char eTitle; /* Quating style for the text of column names */ + unsigned char eBlob; /* Quoting style for BLOBs */ + unsigned char bTitles; /* True to show column names */ + unsigned char bWordWrap; /* Try to wrap on word boundaries */ + unsigned char bTextJsonb; /* Render JSONB blobs as JSON text */ + unsigned char eDfltAlign; /* Default alignment, no covered by aAlignment */ + unsigned char eTitleAlign; /* Alignment for column headers */ + unsigned char bSplitColumn; /* Wrap single-column output into many columns */ + unsigned char bBorder; /* Show outer border in Box and Table styles */ + short int nWrap; /* Wrap columns wider than this */ + short int nScreenWidth; /* Maximum overall table width */ + short int nLineLimit; /* Maximum number of lines for any row */ + short int nTitleLimit; /* Maximum number of characters in a title */ + int nCharLimit; /* Maximum number of characters in a cell */ + int nWidth; /* Number of entries in aWidth[] */ + int nAlign; /* Number of entries in aAlignment[] */ + short int *aWidth; /* Column widths */ + unsigned char *aAlign; /* Column alignments */ + char *zColumnSep; /* Alternative column separator */ + char *zRowSep; /* Alternative row separator */ + char *zTableName; /* Output table name */ + char *zNull; /* Rendering of NULL */ + char *(*xRender)(void*,sqlite3_value*); /* Render a value */ + int (*xWrite)(void*,const char*,sqlite3_int64); /* Write output */ + void *pRenderArg; /* First argument to the xRender callback */ + void *pWriteArg; /* First argument to the xWrite callback */ + char **pzOutput; /* Storage location for output string */ + /* Additional fields may be added in the future */ +}; + +/* +** Interfaces +*/ +int sqlite3_format_query_result( + sqlite3_stmt *pStmt, /* SQL statement to run */ + const sqlite3_qrf_spec *pSpec, /* Result format specification */ + char **pzErr /* OUT: Write error message here */ +); + +/* +** Range of values for sqlite3_qrf_spec.aWidth[] entries and for +** sqlite3_qrf_spec.mxColWidth and .nScreenWidth +*/ +#define QRF_MAX_WIDTH 10000 +#define QRF_MIN_WIDTH 0 + +/* +** Output styles: +*/ +#define QRF_STYLE_Auto 0 /* Choose a style automatically */ +#define QRF_STYLE_Box 1 /* Unicode box-drawing characters */ +#define QRF_STYLE_Column 2 /* One record per line in neat columns */ +#define QRF_STYLE_Count 3 /* Output only a count of the rows of output */ +#define QRF_STYLE_Csv 4 /* Comma-separated-value */ +#define QRF_STYLE_Eqp 5 /* Format EXPLAIN QUERY PLAN output */ +#define QRF_STYLE_Explain 6 /* EXPLAIN output */ +#define QRF_STYLE_Html 7 /* Generate an XHTML table */ +#define QRF_STYLE_Insert 8 /* Generate SQL "insert" statements */ +#define QRF_STYLE_Json 9 /* Output is a list of JSON objects */ +#define QRF_STYLE_JObject 10 /* Independent JSON objects for each row */ +#define QRF_STYLE_Line 11 /* One column per line. */ +#define QRF_STYLE_List 12 /* One record per line with a separator */ +#define QRF_STYLE_Markdown 13 /* Markdown formatting */ +#define QRF_STYLE_Off 14 /* No query output shown */ +#define QRF_STYLE_Quote 15 /* SQL-quoted, comma-separated */ +#define QRF_STYLE_Stats 16 /* EQP-like output but with performance stats */ +#define QRF_STYLE_StatsEst 17 /* EQP-like output with planner estimates */ +#define QRF_STYLE_StatsVm 18 /* EXPLAIN-like output with performance stats */ +#define QRF_STYLE_Table 19 /* MySQL-style table formatting */ + +/* +** Quoting styles for text. +** Allowed values for sqlite3_qrf_spec.eText +*/ +#define QRF_TEXT_Auto 0 /* Choose text encoding automatically */ +#define QRF_TEXT_Plain 1 /* Literal text */ +#define QRF_TEXT_Sql 2 /* Quote as an SQL literal */ +#define QRF_TEXT_Csv 3 /* CSV-style quoting */ +#define QRF_TEXT_Html 4 /* HTML-style quoting */ +#define QRF_TEXT_Tcl 5 /* C/Tcl quoting */ +#define QRF_TEXT_Json 6 /* JSON quoting */ +#define QRF_TEXT_Relaxed 7 /* Relaxed SQL quoting */ + +/* +** Quoting styles for BLOBs +** Allowed values for sqlite3_qrf_spec.eBlob +*/ +#define QRF_BLOB_Auto 0 /* Determine BLOB quoting using eText */ +#define QRF_BLOB_Text 1 /* Display content exactly as it is */ +#define QRF_BLOB_Sql 2 /* Quote as an SQL literal */ +#define QRF_BLOB_Hex 3 /* Hexadecimal representation */ +#define QRF_BLOB_Tcl 4 /* "\000" notation */ +#define QRF_BLOB_Json 5 /* A JSON string */ +#define QRF_BLOB_Size 6 /* Display the blob size only */ + +/* +** Control-character escape modes. +** Allowed values for sqlite3_qrf_spec.eEsc +*/ +#define QRF_ESC_Auto 0 /* Choose the ctrl-char escape automatically */ +#define QRF_ESC_Off 1 /* Do not escape control characters */ +#define QRF_ESC_Ascii 2 /* Unix-style escapes. Ex: U+0007 shows ^G */ +#define QRF_ESC_Symbol 3 /* Unicode escapes. Ex: U+0007 shows U+2407 */ + +/* +** Allowed values for "boolean" fields, such as "bColumnNames", "bWordWrap", +** and "bTextJsonb". There is an extra "auto" variants so these are actually +** tri-state settings, not booleans. +*/ +#define QRF_SW_Auto 0 /* Let QRF choose the best value */ +#define QRF_SW_Off 1 /* This setting is forced off */ +#define QRF_SW_On 2 /* This setting is forced on */ +#define QRF_Auto 0 /* Alternate spelling for QRF_*_Auto */ +#define QRF_No 1 /* Alternate spelling for QRF_SW_Off */ +#define QRF_Yes 2 /* Alternate spelling for QRF_SW_On */ + +/* +** Possible alignment values alignment settings +** +** Horizontal Vertial +** ---------- -------- */ +#define QRF_ALIGN_Auto 0 /* auto auto */ +#define QRF_ALIGN_Left 1 /* left auto */ +#define QRF_ALIGN_Center 2 /* center auto */ +#define QRF_ALIGN_Right 3 /* right auto */ +#define QRF_ALIGN_Top 4 /* auto top */ +#define QRF_ALIGN_NW 5 /* left top */ +#define QRF_ALIGN_N 6 /* center top */ +#define QRF_ALIGN_NE 7 /* right top */ +#define QRF_ALIGN_Middle 8 /* auto middle */ +#define QRF_ALIGN_W 9 /* left middle */ +#define QRF_ALIGN_C 10 /* center middle */ +#define QRF_ALIGN_E 11 /* right middle */ +#define QRF_ALIGN_Bottom 12 /* auto bottom */ +#define QRF_ALIGN_SW 13 /* left bottom */ +#define QRF_ALIGN_S 14 /* center bottom */ +#define QRF_ALIGN_SE 15 /* right bottom */ +#define QRF_ALIGN_HMASK 3 /* Horizontal alignment mask */ +#define QRF_ALIGN_VMASK 12 /* Vertical alignment mask */ + +/* +** Auxiliary routines contined within this module that might be useful +** in other contexts, and which are therefore exported. +*/ +/* +** Return an estimate of the width, in columns, for the single Unicode +** character c. For normal characters, the answer is always 1. But the +** estimate might be 0 or 2 for zero-width and double-width characters. +** +** Different devices display unicode using different widths. So +** it is impossible to know that true display width with 100% accuracy. +** Inaccuracies in the width estimates might cause columns to be misaligned. +** Unfortunately, there is nothing we can do about that. +*/ +int sqlite3_qrf_wcwidth(int c); + +/* +** Return an estimate of the number of display columns used by the +** string in the argument. The width of individual characters is +** determined as for sqlite3_qrf_wcwidth(). VT100 escape code sequences +** are assigned a width of zero. +*/ +size_t sqlite3_qrf_wcswidth(const char*); + + +#ifdef __cplusplus +} +#endif +#endif /* !defined(SQLITE_QRF_H) */ diff --git a/ext/rbu/rbu11.test b/ext/rbu/rbu11.test index a42163cce..513ab29f6 100644 --- a/ext/rbu/rbu11.test +++ b/ext/rbu/rbu11.test @@ -192,4 +192,32 @@ do_test 4.7.2 { list [catch {rbu close} msg] $msg } {1 {SQLITE_ERROR - rbu_state mismatch error}} +#------------------------------------------------------------------------- +# https://sqlite.org/forum/info/6d0a31e22a435877 +# +reset_db +forcedelete rbu.db + +do_execsql_test 5.0 { + CREATE TABLE t1(a BLOB); + INSERT INTO t1 VALUES(x'41'); +} + +sqlite3 dbRbu rbu.db +dbRbu eval { + CREATE TABLE data_t1(a, rbu_control, rbu_rowid); + INSERT INTO data_t1 VALUES(X'310a313a5a337e7e7e7e7e40302c303b','f',1); +} +dbRbu close + +do_test 5.1 { + sqlite3rbu rbu test.db rbu.db + rbu step +} {SQLITE_ERROR} + +do_test 5.2 { + list [catch {rbu close} msg] $msg +} {1 {SQLITE_ERROR - corrupt fossil delta}} + + finish_test diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 4509986ee..f377d5c30 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -623,7 +623,7 @@ static int rbuDeltaApply( /* ERROR: copy exceeds output file size */ return -1; } - if( (int)(ofst+cnt) > lenSrc ){ + if( (u64)ofst+(u64)cnt > (u64)lenSrc ){ /* ERROR: copy extends past end of input */ return -1; } @@ -2269,8 +2269,8 @@ static char *rbuObjIterGetIndexWhere(sqlite3rbu *p, RbuObjIter *pIter){ /* If necessary, grow the pIter->aIdxCol[] array */ if( iIdxCol==nIdxAlloc ){ - RbuSpan *aIdxCol = (RbuSpan*)sqlite3_realloc( - pIter->aIdxCol, (nIdxAlloc+16)*sizeof(RbuSpan) + RbuSpan *aIdxCol = (RbuSpan*)sqlite3_realloc64( + pIter->aIdxCol, nIdxAlloc*sizeof(RbuSpan) + 16*sizeof(RbuSpan) ); if( aIdxCol==0 ){ rc = SQLITE_NOMEM; diff --git a/ext/repair/README.md b/ext/repair/README.md deleted file mode 100644 index 927ceb7c4..000000000 --- a/ext/repair/README.md +++ /dev/null @@ -1,16 +0,0 @@ -This folder contains extensions and utility programs intended to analyze -live database files, detect problems, and possibly fix them. - -As SQLite is being used on larger and larger databases, database sizes -are growing into the terabyte range. At that size, hardware malfunctions -and/or cosmic rays will occasionally corrupt a database file. Detecting -problems and fixing errors a terabyte-sized databases can take hours or days, -and it is undesirable to take applications that depend on the databases -off-line for such a long time. -The utilities in the folder are intended to provide mechanisms for -detecting and fixing problems in large databases while those databases -are in active use. - -The utilities and extensions in this folder are experimental and under -active development at the time of this writing (2017-10-12). If and when -they stabilize, this README will be updated to reflect that fact. diff --git a/ext/repair/checkfreelist.c b/ext/repair/checkfreelist.c deleted file mode 100644 index d1d3d5407..000000000 --- a/ext/repair/checkfreelist.c +++ /dev/null @@ -1,310 +0,0 @@ -/* -** 2017 October 11 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This module exports a single C function: -** -** int sqlite3_check_freelist(sqlite3 *db, const char *zDb); -** -** This function checks the free-list in database zDb (one of "main", -** "temp", etc.) and reports any errors by invoking the sqlite3_log() -** function. It returns SQLITE_OK if successful, or an SQLite error -** code otherwise. It is not an error if the free-list is corrupted but -** no IO or OOM errors occur. -** -** If this file is compiled and loaded as an SQLite loadable extension, -** it adds an SQL function "checkfreelist" to the database handle, to -** be invoked as follows: -** -** SELECT checkfreelist(); -** -** This function performs the same checks as sqlite3_check_freelist(), -** except that it returns all error messages as a single text value, -** separated by newline characters. If the freelist is not corrupted -** in any way, an empty string is returned. -** -** To compile this module for use as an SQLite loadable extension: -** -** gcc -Os -fPIC -shared checkfreelist.c -o checkfreelist.so -*/ - -#include "sqlite3ext.h" -SQLITE_EXTENSION_INIT1 - -#ifndef SQLITE_AMALGAMATION -# include -# include -# include -# include -# if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) -# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1 -# endif -# if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS) -# define ALWAYS(X) (1) -# define NEVER(X) (0) -# elif !defined(NDEBUG) -# define ALWAYS(X) ((X)?1:(assert(0),0)) -# define NEVER(X) ((X)?(assert(0),1):0) -# else -# define ALWAYS(X) (X) -# define NEVER(X) (X) -# endif - typedef unsigned char u8; - typedef unsigned short u16; - typedef unsigned int u32; -#define get4byte(x) ( \ - ((u32)((x)[0])<<24) + \ - ((u32)((x)[1])<<16) + \ - ((u32)((x)[2])<<8) + \ - ((u32)((x)[3])) \ -) -#endif - -/* -** Execute a single PRAGMA statement and return the integer value returned -** via output parameter (*pnOut). -** -** The SQL statement passed as the third argument should be a printf-style -** format string containing a single "%s" which will be replace by the -** value passed as the second argument. e.g. -** -** sqlGetInteger(db, "main", "PRAGMA %s.page_count", pnOut) -** -** executes "PRAGMA main.page_count" and stores the results in (*pnOut). -*/ -static int sqlGetInteger( - sqlite3 *db, /* Database handle */ - const char *zDb, /* Database name ("main", "temp" etc.) */ - const char *zFmt, /* SQL statement format */ - u32 *pnOut /* OUT: Integer value */ -){ - int rc, rc2; - char *zSql; - sqlite3_stmt *pStmt = 0; - int bOk = 0; - - zSql = sqlite3_mprintf(zFmt, zDb); - if( zSql==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - } - - if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - *pnOut = (u32)sqlite3_column_int(pStmt, 0); - bOk = 1; - } - - rc2 = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) rc = rc2; - if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_ERROR; - return rc; -} - -/* -** Argument zFmt must be a printf-style format string and must be -** followed by its required arguments. If argument pzOut is NULL, -** then the results of printf()ing the format string are passed to -** sqlite3_log(). Otherwise, they are appended to the string -** at (*pzOut). -*/ -static int checkFreelistError(char **pzOut, const char *zFmt, ...){ - int rc = SQLITE_OK; - char *zErr = 0; - va_list ap; - - va_start(ap, zFmt); - zErr = sqlite3_vmprintf(zFmt, ap); - if( zErr==0 ){ - rc = SQLITE_NOMEM; - }else{ - if( pzOut ){ - *pzOut = sqlite3_mprintf("%s%z%s", *pzOut?"\n":"", *pzOut, zErr); - if( *pzOut==0 ) rc = SQLITE_NOMEM; - }else{ - sqlite3_log(SQLITE_ERROR, "checkfreelist: %s", zErr); - } - sqlite3_free(zErr); - } - va_end(ap); - return rc; -} - -static int checkFreelist( - sqlite3 *db, - const char *zDb, - char **pzOut -){ - /* This query returns one row for each page on the free list. Each row has - ** two columns - the page number and page content. */ - const char *zTrunk = - "WITH freelist_trunk(i, d, n) AS (" - "SELECT 1, NULL, sqlite_readint32(data, 32) " - "FROM sqlite_dbpage(:1) WHERE pgno=1 " - "UNION ALL " - "SELECT n, data, sqlite_readint32(data) " - "FROM freelist_trunk, sqlite_dbpage(:1) WHERE pgno=n " - ")" - "SELECT i, d FROM freelist_trunk WHERE i!=1;"; - - int rc, rc2; /* Return code */ - sqlite3_stmt *pTrunk = 0; /* Compilation of zTrunk */ - u32 nPage = 0; /* Number of pages in db */ - u32 nExpected = 0; /* Expected number of free pages */ - u32 nFree = 0; /* Number of pages on free list */ - - if( zDb==0 ) zDb = "main"; - - if( (rc = sqlGetInteger(db, zDb, "PRAGMA %s.page_count", &nPage)) - || (rc = sqlGetInteger(db, zDb, "PRAGMA %s.freelist_count", &nExpected)) - ){ - return rc; - } - - rc = sqlite3_prepare_v2(db, zTrunk, -1, &pTrunk, 0); - if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_text(pTrunk, 1, zDb, -1, SQLITE_STATIC); - while( rc==SQLITE_OK && sqlite3_step(pTrunk)==SQLITE_ROW ){ - u32 i; - u32 iTrunk = (u32)sqlite3_column_int(pTrunk, 0); - const u8 *aData = (const u8*)sqlite3_column_blob(pTrunk, 1); - u32 nData = (u32)sqlite3_column_bytes(pTrunk, 1); - u32 iNext = get4byte(&aData[0]); - u32 nLeaf = get4byte(&aData[4]); - - if( nLeaf>((nData/4)-2-6) ){ - rc = checkFreelistError(pzOut, - "leaf count out of range (%d) on trunk page %d", - (int)nLeaf, (int)iTrunk - ); - nLeaf = (nData/4) - 2 - 6; - } - - nFree += 1+nLeaf; - if( iNext>nPage ){ - rc = checkFreelistError(pzOut, - "trunk page %d is out of range", (int)iNext - ); - } - - for(i=0; rc==SQLITE_OK && inPage ){ - rc = checkFreelistError(pzOut, - "leaf page %d is out of range (child %d of trunk page %d)", - (int)iLeaf, (int)i, (int)iTrunk - ); - } - } - } - - if( rc==SQLITE_OK && nFree!=nExpected ){ - rc = checkFreelistError(pzOut, - "free-list count mismatch: actual=%d header=%d", - (int)nFree, (int)nExpected - ); - } - - rc2 = sqlite3_finalize(pTrunk); - if( rc==SQLITE_OK ) rc = rc2; - return rc; -} - -int sqlite3_check_freelist(sqlite3 *db, const char *zDb){ - return checkFreelist(db, zDb, 0); -} - -static void checkfreelist_function( - sqlite3_context *pCtx, - int nArg, - sqlite3_value **apArg -){ - const char *zDb; - int rc; - char *zOut = 0; - sqlite3 *db = sqlite3_context_db_handle(pCtx); - - assert( nArg==1 ); - zDb = (const char*)sqlite3_value_text(apArg[0]); - rc = checkFreelist(db, zDb, &zOut); - if( rc==SQLITE_OK ){ - sqlite3_result_text(pCtx, zOut?zOut:"ok", -1, SQLITE_TRANSIENT); - }else{ - sqlite3_result_error_code(pCtx, rc); - } - - sqlite3_free(zOut); -} - -/* -** An SQL function invoked as follows: -** -** sqlite_readint32(BLOB) -- Decode 32-bit integer from start of blob -*/ -static void readint_function( - sqlite3_context *pCtx, - int nArg, - sqlite3_value **apArg -){ - const u8 *zBlob; - int nBlob; - int iOff = 0; - u32 iRet = 0; - - if( nArg!=1 && nArg!=2 ){ - sqlite3_result_error( - pCtx, "wrong number of arguments to function sqlite_readint32()", -1 - ); - return; - } - if( nArg==2 ){ - iOff = sqlite3_value_int(apArg[1]); - } - - zBlob = sqlite3_value_blob(apArg[0]); - nBlob = sqlite3_value_bytes(apArg[0]); - - if( nBlob>=(iOff+4) ){ - iRet = get4byte(&zBlob[iOff]); - } - - sqlite3_result_int64(pCtx, (sqlite3_int64)iRet); -} - -/* -** Register the SQL functions. -*/ -static int cflRegister(sqlite3 *db){ - int rc = sqlite3_create_function( - db, "sqlite_readint32", -1, SQLITE_UTF8, 0, readint_function, 0, 0 - ); - if( rc!=SQLITE_OK ) return rc; - rc = sqlite3_create_function( - db, "checkfreelist", 1, SQLITE_UTF8, 0, checkfreelist_function, 0, 0 - ); - return rc; -} - -/* -** Extension load function. -*/ -#ifdef _WIN32 -__declspec(dllexport) -#endif -int sqlite3_checkfreelist_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - SQLITE_EXTENSION_INIT2(pApi); - return cflRegister(db); -} diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c deleted file mode 100644 index ed30357e5..000000000 --- a/ext/repair/checkindex.c +++ /dev/null @@ -1,929 +0,0 @@ -/* -** 2017 October 27 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -*/ - -#include "sqlite3ext.h" -SQLITE_EXTENSION_INIT1 - -/* -** Stuff that is available inside the amalgamation, but which we need to -** declare ourselves if this module is compiled separately. -*/ -#ifndef SQLITE_AMALGAMATION -# include -# include -# include -# include -typedef unsigned char u8; -typedef unsigned short u16; -typedef unsigned int u32; -#define get4byte(x) ( \ - ((u32)((x)[0])<<24) + \ - ((u32)((x)[1])<<16) + \ - ((u32)((x)[2])<<8) + \ - ((u32)((x)[3])) \ -) -#endif - -typedef struct CidxTable CidxTable; -typedef struct CidxCursor CidxCursor; - -struct CidxTable { - sqlite3_vtab base; /* Base class. Must be first */ - sqlite3 *db; -}; - -struct CidxCursor { - sqlite3_vtab_cursor base; /* Base class. Must be first */ - sqlite3_int64 iRowid; /* Row number of the output */ - char *zIdxName; /* Copy of the index_name parameter */ - char *zAfterKey; /* Copy of the after_key parameter */ - sqlite3_stmt *pStmt; /* SQL statement that generates the output */ -}; - -typedef struct CidxColumn CidxColumn; -struct CidxColumn { - char *zExpr; /* Text for indexed expression */ - int bDesc; /* True for DESC columns, otherwise false */ - int bKey; /* Part of index, not PK */ -}; - -typedef struct CidxIndex CidxIndex; -struct CidxIndex { - char *zWhere; /* WHERE clause, if any */ - int nCol; /* Elements in aCol[] array */ - CidxColumn aCol[1]; /* Array of indexed columns */ -}; - -static void *cidxMalloc(int *pRc, int n){ - void *pRet = 0; - assert( n!=0 ); - if( *pRc==SQLITE_OK ){ - pRet = sqlite3_malloc(n); - if( pRet ){ - memset(pRet, 0, n); - }else{ - *pRc = SQLITE_NOMEM; - } - } - return pRet; -} - -static void cidxCursorError(CidxCursor *pCsr, const char *zFmt, ...){ - va_list ap; - va_start(ap, zFmt); - assert( pCsr->base.pVtab->zErrMsg==0 ); - pCsr->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap); - va_end(ap); -} - -/* -** Connect to the incremental_index_check virtual table. -*/ -static int cidxConnect( - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVtab, - char **pzErr -){ - int rc = SQLITE_OK; - CidxTable *pRet; - -#define IIC_ERRMSG 0 -#define IIC_CURRENT_KEY 1 -#define IIC_INDEX_NAME 2 -#define IIC_AFTER_KEY 3 -#define IIC_SCANNER_SQL 4 - rc = sqlite3_declare_vtab(db, - "CREATE TABLE xyz(" - " errmsg TEXT," /* Error message or NULL if everything is ok */ - " current_key TEXT," /* SQLite quote() text of key values */ - " index_name HIDDEN," /* IN: name of the index being scanned */ - " after_key HIDDEN," /* IN: Start scanning after this key */ - " scanner_sql HIDDEN" /* debugging info: SQL used for scanner */ - ")" - ); - pRet = cidxMalloc(&rc, sizeof(CidxTable)); - if( pRet ){ - pRet->db = db; - } - - *ppVtab = (sqlite3_vtab*)pRet; - return rc; -} - -/* -** Disconnect from or destroy an incremental_index_check virtual table. -*/ -static int cidxDisconnect(sqlite3_vtab *pVtab){ - CidxTable *pTab = (CidxTable*)pVtab; - sqlite3_free(pTab); - return SQLITE_OK; -} - -/* -** idxNum and idxStr are not used. There are only three possible plans, -** which are all distinguished by the number of parameters. -** -** No parameters: A degenerate plan. The result is zero rows. -** 1 Parameter: Scan all of the index starting with first entry -** 2 parameters: Scan the index starting after the "after_key". -** -** Provide successively smaller costs for each of these plans to encourage -** the query planner to select the one with the most parameters. -*/ -static int cidxBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pInfo){ - int iIdxName = -1; - int iAfterKey = -1; - int i; - - for(i=0; inConstraint; i++){ - struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; - if( p->usable==0 ) continue; - if( p->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; - - if( p->iColumn==IIC_INDEX_NAME ){ - iIdxName = i; - } - if( p->iColumn==IIC_AFTER_KEY ){ - iAfterKey = i; - } - } - - if( iIdxName<0 ){ - pInfo->estimatedCost = 1000000000.0; - }else{ - pInfo->aConstraintUsage[iIdxName].argvIndex = 1; - pInfo->aConstraintUsage[iIdxName].omit = 1; - if( iAfterKey<0 ){ - pInfo->estimatedCost = 1000000.0; - }else{ - pInfo->aConstraintUsage[iAfterKey].argvIndex = 2; - pInfo->aConstraintUsage[iAfterKey].omit = 1; - pInfo->estimatedCost = 1000.0; - } - } - - return SQLITE_OK; -} - -/* -** Open a new btreeinfo cursor. -*/ -static int cidxOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ - CidxCursor *pRet; - int rc = SQLITE_OK; - - pRet = cidxMalloc(&rc, sizeof(CidxCursor)); - - *ppCursor = (sqlite3_vtab_cursor*)pRet; - return rc; -} - -/* -** Close a btreeinfo cursor. -*/ -static int cidxClose(sqlite3_vtab_cursor *pCursor){ - CidxCursor *pCsr = (CidxCursor*)pCursor; - sqlite3_finalize(pCsr->pStmt); - sqlite3_free(pCsr->zIdxName); - sqlite3_free(pCsr->zAfterKey); - sqlite3_free(pCsr); - return SQLITE_OK; -} - -/* -** Move a btreeinfo cursor to the next entry in the file. -*/ -static int cidxNext(sqlite3_vtab_cursor *pCursor){ - CidxCursor *pCsr = (CidxCursor*)pCursor; - int rc = sqlite3_step(pCsr->pStmt); - if( rc!=SQLITE_ROW ){ - rc = sqlite3_finalize(pCsr->pStmt); - pCsr->pStmt = 0; - if( rc!=SQLITE_OK ){ - sqlite3 *db = ((CidxTable*)pCsr->base.pVtab)->db; - cidxCursorError(pCsr, "Cursor error: %s", sqlite3_errmsg(db)); - } - }else{ - pCsr->iRowid++; - rc = SQLITE_OK; - } - return rc; -} - -/* We have reached EOF if previous sqlite3_step() returned -** anything other than SQLITE_ROW; -*/ -static int cidxEof(sqlite3_vtab_cursor *pCursor){ - CidxCursor *pCsr = (CidxCursor*)pCursor; - return pCsr->pStmt==0; -} - -static char *cidxMprintf(int *pRc, const char *zFmt, ...){ - char *zRet = 0; - va_list ap; - va_start(ap, zFmt); - zRet = sqlite3_vmprintf(zFmt, ap); - if( *pRc==SQLITE_OK ){ - if( zRet==0 ){ - *pRc = SQLITE_NOMEM; - } - }else{ - sqlite3_free(zRet); - zRet = 0; - } - va_end(ap); - return zRet; -} - -static sqlite3_stmt *cidxPrepare( - int *pRc, CidxCursor *pCsr, const char *zFmt, ... -){ - sqlite3_stmt *pRet = 0; - char *zSql; - va_list ap; /* ... printf arguments */ - va_start(ap, zFmt); - - zSql = sqlite3_vmprintf(zFmt, ap); - if( *pRc==SQLITE_OK ){ - if( zSql==0 ){ - *pRc = SQLITE_NOMEM; - }else{ - sqlite3 *db = ((CidxTable*)pCsr->base.pVtab)->db; - *pRc = sqlite3_prepare_v2(db, zSql, -1, &pRet, 0); - if( *pRc!=SQLITE_OK ){ - cidxCursorError(pCsr, "SQL error: %s", sqlite3_errmsg(db)); - } - } - } - sqlite3_free(zSql); - va_end(ap); - - return pRet; -} - -static void cidxFinalize(int *pRc, sqlite3_stmt *pStmt){ - int rc = sqlite3_finalize(pStmt); - if( *pRc==SQLITE_OK ) *pRc = rc; -} - -char *cidxStrdup(int *pRc, const char *zStr){ - char *zRet = 0; - if( *pRc==SQLITE_OK ){ - int n = (int)strlen(zStr); - zRet = cidxMalloc(pRc, n+1); - if( zRet ) memcpy(zRet, zStr, n+1); - } - return zRet; -} - -static void cidxFreeIndex(CidxIndex *pIdx){ - if( pIdx ){ - int i; - for(i=0; inCol; i++){ - sqlite3_free(pIdx->aCol[i].zExpr); - } - sqlite3_free(pIdx->zWhere); - sqlite3_free(pIdx); - } -} - -static int cidx_isspace(char c){ - return c==' ' || c=='\t' || c=='\r' || c=='\n'; -} - -static int cidx_isident(char c){ - return c<0 - || (c>='0' && c<='9') || (c>='a' && c<='z') - || (c>='A' && c<='Z') || c=='_'; -} - -#define CIDX_PARSE_EOF 0 -#define CIDX_PARSE_COMMA 1 /* "," */ -#define CIDX_PARSE_OPEN 2 /* "(" */ -#define CIDX_PARSE_CLOSE 3 /* ")" */ - -/* -** Argument zIn points into the start, middle or end of a CREATE INDEX -** statement. If argument pbDoNotTrim is non-NULL, then this function -** scans the input until it finds EOF, a comma (",") or an open or -** close parenthesis character. It then sets (*pzOut) to point to said -** character and returns a CIDX_PARSE_XXX constant as appropriate. The -** parser is smart enough that special characters inside SQL strings -** or comments are not returned for. -** -** Or, if argument pbDoNotTrim is NULL, then this function sets *pzOut -** to point to the first character of the string that is not whitespace -** or part of an SQL comment and returns CIDX_PARSE_EOF. -** -** Additionally, if pbDoNotTrim is not NULL and the element immediately -** before (*pzOut) is an SQL comment of the form "-- comment", then -** (*pbDoNotTrim) is set before returning. In all other cases it is -** cleared. -*/ -static int cidxFindNext( - const char *zIn, - const char **pzOut, - int *pbDoNotTrim /* OUT: True if prev is -- comment */ -){ - const char *z = zIn; - - while( 1 ){ - while( cidx_isspace(*z) ) z++; - if( z[0]=='-' && z[1]=='-' ){ - z += 2; - while( z[0]!='\n' ){ - if( z[0]=='\0' ) return CIDX_PARSE_EOF; - z++; - } - while( cidx_isspace(*z) ) z++; - if( pbDoNotTrim ) *pbDoNotTrim = 1; - }else - if( z[0]=='/' && z[1]=='*' ){ - z += 2; - while( z[0]!='*' || z[1]!='/' ){ - if( z[1]=='\0' ) return CIDX_PARSE_EOF; - z++; - } - z += 2; - }else{ - *pzOut = z; - if( pbDoNotTrim==0 ) return CIDX_PARSE_EOF; - switch( *z ){ - case '\0': - return CIDX_PARSE_EOF; - case '(': - return CIDX_PARSE_OPEN; - case ')': - return CIDX_PARSE_CLOSE; - case ',': - return CIDX_PARSE_COMMA; - - case '"': - case '\'': - case '`': { - char q = *z; - z++; - while( *z ){ - if( *z==q ){ - z++; - if( *z!=q ) break; - } - z++; - } - break; - } - - case '[': - while( *z++!=']' ); - break; - - default: - z++; - break; - } - *pbDoNotTrim = 0; - } - } - - assert( 0 ); - return -1; -} - -static int cidxParseSQL(CidxCursor *pCsr, CidxIndex *pIdx, const char *zSql){ - const char *z = zSql; - const char *z1; - int e; - int rc = SQLITE_OK; - int nParen = 1; - int bDoNotTrim = 0; - CidxColumn *pCol = pIdx->aCol; - - e = cidxFindNext(z, &z, &bDoNotTrim); - if( e!=CIDX_PARSE_OPEN ) goto parse_error; - z1 = z+1; - z++; - while( nParen>0 ){ - e = cidxFindNext(z, &z, &bDoNotTrim); - if( e==CIDX_PARSE_EOF ) goto parse_error; - if( (e==CIDX_PARSE_COMMA || e==CIDX_PARSE_CLOSE) && nParen==1 ){ - const char *z2 = z; - if( pCol->zExpr ) goto parse_error; - - if( bDoNotTrim==0 ){ - while( cidx_isspace(z[-1]) ) z--; - if( !sqlite3_strnicmp(&z[-3], "asc", 3) && 0==cidx_isident(z[-4]) ){ - z -= 3; - while( cidx_isspace(z[-1]) ) z--; - }else - if( !sqlite3_strnicmp(&z[-4], "desc", 4) && 0==cidx_isident(z[-5]) ){ - z -= 4; - while( cidx_isspace(z[-1]) ) z--; - } - while( cidx_isspace(z1[0]) ) z1++; - } - - pCol->zExpr = cidxMprintf(&rc, "%.*s", z-z1, z1); - pCol++; - z = z1 = z2+1; - } - if( e==CIDX_PARSE_OPEN ) nParen++; - if( e==CIDX_PARSE_CLOSE ) nParen--; - z++; - } - - /* Search for a WHERE clause */ - cidxFindNext(z, &z, 0); - if( 0==sqlite3_strnicmp(z, "where", 5) ){ - pIdx->zWhere = cidxMprintf(&rc, "%s\n", &z[5]); - }else if( z[0]!='\0' ){ - goto parse_error; - } - - return rc; - - parse_error: - cidxCursorError(pCsr, "Parse error in: %s", zSql); - return SQLITE_ERROR; -} - -static int cidxLookupIndex( - CidxCursor *pCsr, /* Cursor object */ - const char *zIdx, /* Name of index to look up */ - CidxIndex **ppIdx, /* OUT: Description of columns */ - char **pzTab /* OUT: Table name */ -){ - int rc = SQLITE_OK; - char *zTab = 0; - CidxIndex *pIdx = 0; - - sqlite3_stmt *pFindTab = 0; - sqlite3_stmt *pInfo = 0; - - /* Find the table for this index. */ - pFindTab = cidxPrepare(&rc, pCsr, - "SELECT tbl_name, sql FROM sqlite_schema WHERE name=%Q AND type='index'", - zIdx - ); - if( rc==SQLITE_OK && sqlite3_step(pFindTab)==SQLITE_ROW ){ - const char *zSql = (const char*)sqlite3_column_text(pFindTab, 1); - zTab = cidxStrdup(&rc, (const char*)sqlite3_column_text(pFindTab, 0)); - - pInfo = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx); - if( rc==SQLITE_OK ){ - int nAlloc = 0; - int iCol = 0; - - while( sqlite3_step(pInfo)==SQLITE_ROW ){ - const char *zName = (const char*)sqlite3_column_text(pInfo, 2); - const char *zColl = (const char*)sqlite3_column_text(pInfo, 4); - CidxColumn *p; - if( zName==0 ) zName = "rowid"; - if( iCol==nAlloc ){ - int nByte = sizeof(CidxIndex) + sizeof(CidxColumn)*(nAlloc+8); - pIdx = (CidxIndex*)sqlite3_realloc(pIdx, nByte); - nAlloc += 8; - } - p = &pIdx->aCol[iCol++]; - p->bDesc = sqlite3_column_int(pInfo, 3); - p->bKey = sqlite3_column_int(pInfo, 5); - if( zSql==0 || p->bKey==0 ){ - p->zExpr = cidxMprintf(&rc, "\"%w\" COLLATE %s",zName,zColl); - }else{ - p->zExpr = 0; - } - pIdx->nCol = iCol; - pIdx->zWhere = 0; - } - cidxFinalize(&rc, pInfo); - } - - if( rc==SQLITE_OK && zSql ){ - rc = cidxParseSQL(pCsr, pIdx, zSql); - } - } - - cidxFinalize(&rc, pFindTab); - if( rc==SQLITE_OK && zTab==0 ){ - rc = SQLITE_ERROR; - } - - if( rc!=SQLITE_OK ){ - sqlite3_free(zTab); - cidxFreeIndex(pIdx); - }else{ - *pzTab = zTab; - *ppIdx = pIdx; - } - - return rc; -} - -static int cidxDecodeAfter( - CidxCursor *pCsr, - int nCol, - const char *zAfterKey, - char ***pazAfter -){ - char **azAfter; - int rc = SQLITE_OK; - int nAfterKey = (int)strlen(zAfterKey); - - azAfter = cidxMalloc(&rc, sizeof(char*)*nCol + nAfterKey+1); - if( rc==SQLITE_OK ){ - int i; - char *zCopy = (char*)&azAfter[nCol]; - char *p = zCopy; - memcpy(zCopy, zAfterKey, nAfterKey+1); - for(i=0; i='0' && *p<='9') - || *p=='.' || *p=='+' || *p=='-' || *p=='e' || *p=='E' - ){ - p++; - } - } - - while( *p==' ' ) p++; - if( *p!=(i==(nCol-1) ? '\0' : ',') ){ - goto parse_error; - } - *p++ = '\0'; - } - } - - *pazAfter = azAfter; - return rc; - - parse_error: - sqlite3_free(azAfter); - *pazAfter = 0; - cidxCursorError(pCsr, "%s", "error parsing after value"); - return SQLITE_ERROR; -} - -static char *cidxWhere( - int *pRc, CidxColumn *aCol, char **azAfter, int iGt, int bLastIsNull -){ - char *zRet = 0; - const char *zSep = ""; - int i; - - for(i=0; i"), - azAfter[iGt] - ); - }else{ - zRet = cidxMprintf(pRc, "%z%s(%s) IS NOT NULL", zRet, zSep,aCol[iGt].zExpr); - } - - return zRet; -} - -#define CIDX_CLIST_ALL 0 -#define CIDX_CLIST_ORDERBY 1 -#define CIDX_CLIST_CURRENT_KEY 2 -#define CIDX_CLIST_SUBWHERE 3 -#define CIDX_CLIST_SUBEXPR 4 - -/* -** This function returns various strings based on the contents of the -** CidxIndex structure and the eType parameter. -*/ -static char *cidxColumnList( - int *pRc, /* IN/OUT: Error code */ - const char *zIdx, - CidxIndex *pIdx, /* Indexed columns */ - int eType /* True to include ASC/DESC */ -){ - char *zRet = 0; - if( *pRc==SQLITE_OK ){ - const char *aDir[2] = {"", " DESC"}; - int i; - const char *zSep = ""; - - for(i=0; inCol; i++){ - CidxColumn *p = &pIdx->aCol[i]; - assert( pIdx->aCol[i].bDesc==0 || pIdx->aCol[i].bDesc==1 ); - switch( eType ){ - - case CIDX_CLIST_ORDERBY: - zRet = cidxMprintf(pRc, "%z%s%d%s", zRet, zSep, i+1, aDir[p->bDesc]); - zSep = ","; - break; - - case CIDX_CLIST_CURRENT_KEY: - zRet = cidxMprintf(pRc, "%z%squote(i%d)", zRet, zSep, i); - zSep = "||','||"; - break; - - case CIDX_CLIST_SUBWHERE: - if( p->bKey==0 ){ - zRet = cidxMprintf(pRc, "%z%s%s IS i.i%d", zRet, - zSep, p->zExpr, i - ); - zSep = " AND "; - } - break; - - case CIDX_CLIST_SUBEXPR: - if( p->bKey==1 ){ - zRet = cidxMprintf(pRc, "%z%s%s IS i.i%d", zRet, - zSep, p->zExpr, i - ); - zSep = " AND "; - } - break; - - default: - assert( eType==CIDX_CLIST_ALL ); - zRet = cidxMprintf(pRc, "%z%s(%s) AS i%d", zRet, zSep, p->zExpr, i); - zSep = ", "; - break; - } - } - } - - return zRet; -} - -/* -** Generate SQL (in memory obtained from sqlite3_malloc()) that will -** continue the index scan for zIdxName starting after zAfterKey. -*/ -int cidxGenerateScanSql( - CidxCursor *pCsr, /* The cursor which needs the new statement */ - const char *zIdxName, /* index to be scanned */ - const char *zAfterKey, /* start after this key, if not NULL */ - char **pzSqlOut /* OUT: Write the generated SQL here */ -){ - int rc; - char *zTab = 0; - char *zCurrentKey = 0; - char *zOrderBy = 0; - char *zSubWhere = 0; - char *zSubExpr = 0; - char *zSrcList = 0; - char **azAfter = 0; - CidxIndex *pIdx = 0; - - *pzSqlOut = 0; - rc = cidxLookupIndex(pCsr, zIdxName, &pIdx, &zTab); - - zOrderBy = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ORDERBY); - zCurrentKey = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_CURRENT_KEY); - zSubWhere = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBWHERE); - zSubExpr = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBEXPR); - zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL); - - if( rc==SQLITE_OK && zAfterKey ){ - rc = cidxDecodeAfter(pCsr, pIdx->nCol, zAfterKey, &azAfter); - } - - if( rc==SQLITE_OK ){ - if( zAfterKey==0 ){ - *pzSqlOut = cidxMprintf(&rc, - "SELECT (SELECT %s FROM %Q AS t WHERE %s), %s " - "FROM (SELECT %s FROM %Q INDEXED BY %Q %s%sORDER BY %s) AS i", - zSubExpr, zTab, zSubWhere, zCurrentKey, - zSrcList, zTab, zIdxName, - (pIdx->zWhere ? "WHERE " : ""), (pIdx->zWhere ? pIdx->zWhere : ""), - zOrderBy - ); - }else{ - const char *zSep = ""; - char *zSql; - int i; - - zSql = cidxMprintf(&rc, - "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (", - zSubExpr, zTab, zSubWhere, zCurrentKey - ); - for(i=pIdx->nCol-1; i>=0; i--){ - int j; - if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue; - for(j=0; j<2; j++){ - char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j); - zSql = cidxMprintf(&rc, "%z" - "%sSELECT * FROM (" - "SELECT %s FROM %Q INDEXED BY %Q WHERE %s%s%z ORDER BY %s" - ")", - zSql, zSep, zSrcList, zTab, zIdxName, - pIdx->zWhere ? pIdx->zWhere : "", - pIdx->zWhere ? " AND " : "", - zWhere, zOrderBy - ); - zSep = " UNION ALL "; - if( pIdx->aCol[i].bDesc==0 ) break; - } - } - *pzSqlOut = cidxMprintf(&rc, "%z) AS i", zSql); - } - } - - sqlite3_free(zTab); - sqlite3_free(zCurrentKey); - sqlite3_free(zOrderBy); - sqlite3_free(zSubWhere); - sqlite3_free(zSubExpr); - sqlite3_free(zSrcList); - cidxFreeIndex(pIdx); - sqlite3_free(azAfter); - return rc; -} - - -/* -** Position a cursor back to the beginning. -*/ -static int cidxFilter( - sqlite3_vtab_cursor *pCursor, - int idxNum, const char *idxStr, - int argc, sqlite3_value **argv -){ - int rc = SQLITE_OK; - CidxCursor *pCsr = (CidxCursor*)pCursor; - const char *zIdxName = 0; - const char *zAfterKey = 0; - - sqlite3_free(pCsr->zIdxName); - pCsr->zIdxName = 0; - sqlite3_free(pCsr->zAfterKey); - pCsr->zAfterKey = 0; - sqlite3_finalize(pCsr->pStmt); - pCsr->pStmt = 0; - - if( argc>0 ){ - zIdxName = (const char*)sqlite3_value_text(argv[0]); - if( argc>1 ){ - zAfterKey = (const char*)sqlite3_value_text(argv[1]); - } - } - - if( zIdxName ){ - char *zSql = 0; - pCsr->zIdxName = sqlite3_mprintf("%s", zIdxName); - pCsr->zAfterKey = zAfterKey ? sqlite3_mprintf("%s", zAfterKey) : 0; - rc = cidxGenerateScanSql(pCsr, zIdxName, zAfterKey, &zSql); - if( zSql ){ - pCsr->pStmt = cidxPrepare(&rc, pCsr, "%z", zSql); - } - } - - if( pCsr->pStmt ){ - assert( rc==SQLITE_OK ); - rc = cidxNext(pCursor); - } - pCsr->iRowid = 1; - return rc; -} - -/* -** Return a column value. -*/ -static int cidxColumn( - sqlite3_vtab_cursor *pCursor, - sqlite3_context *ctx, - int iCol -){ - CidxCursor *pCsr = (CidxCursor*)pCursor; - assert( iCol>=IIC_ERRMSG && iCol<=IIC_SCANNER_SQL ); - switch( iCol ){ - case IIC_ERRMSG: { - const char *zVal = 0; - if( sqlite3_column_type(pCsr->pStmt, 0)==SQLITE_INTEGER ){ - if( sqlite3_column_int(pCsr->pStmt, 0)==0 ){ - zVal = "row data mismatch"; - } - }else{ - zVal = "row missing"; - } - sqlite3_result_text(ctx, zVal, -1, SQLITE_STATIC); - break; - } - case IIC_CURRENT_KEY: { - sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, 1)); - break; - } - case IIC_INDEX_NAME: { - sqlite3_result_text(ctx, pCsr->zIdxName, -1, SQLITE_TRANSIENT); - break; - } - case IIC_AFTER_KEY: { - sqlite3_result_text(ctx, pCsr->zAfterKey, -1, SQLITE_TRANSIENT); - break; - } - case IIC_SCANNER_SQL: { - char *zSql = 0; - cidxGenerateScanSql(pCsr, pCsr->zIdxName, pCsr->zAfterKey, &zSql); - sqlite3_result_text(ctx, zSql, -1, sqlite3_free); - break; - } - } - return SQLITE_OK; -} - -/* Return the ROWID for the sqlite_btreeinfo table */ -static int cidxRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ - CidxCursor *pCsr = (CidxCursor*)pCursor; - *pRowid = pCsr->iRowid; - return SQLITE_OK; -} - -/* -** Register the virtual table modules with the database handle passed -** as the only argument. -*/ -static int ciInit(sqlite3 *db){ - static sqlite3_module cidx_module = { - 0, /* iVersion */ - 0, /* xCreate */ - cidxConnect, /* xConnect */ - cidxBestIndex, /* xBestIndex */ - cidxDisconnect, /* xDisconnect */ - 0, /* xDestroy */ - cidxOpen, /* xOpen - open a cursor */ - cidxClose, /* xClose - close a cursor */ - cidxFilter, /* xFilter - configure scan constraints */ - cidxNext, /* xNext - advance a cursor */ - cidxEof, /* xEof - check for end of scan */ - cidxColumn, /* xColumn - read data */ - cidxRowid, /* xRowid - read data */ - 0, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindMethod */ - 0, /* xRename */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0, /* xRollbackTo */ - 0, /* xShadowName */ - 0 /* xIntegrity */ - }; - return sqlite3_create_module(db, "incremental_index_check", &cidx_module, 0); -} - -/* -** Extension load function. -*/ -#ifdef _WIN32 -__declspec(dllexport) -#endif -int sqlite3_checkindex_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - SQLITE_EXTENSION_INIT2(pApi); - return ciInit(db); -} diff --git a/ext/repair/sqlite3_checker.c.in b/ext/repair/sqlite3_checker.c.in deleted file mode 100644 index 96b15f271..000000000 --- a/ext/repair/sqlite3_checker.c.in +++ /dev/null @@ -1,85 +0,0 @@ -/* -** Read an SQLite database file and analyze its space utilization. Generate -** text on standard output. -*/ -#define TCLSH_INIT_PROC sqlite3_checker_init_proc -#define SQLITE_ENABLE_DBPAGE_VTAB 1 -#undef SQLITE_THREADSAFE -#define SQLITE_THREADSAFE 0 -#undef SQLITE_ENABLE_COLUMN_METADATA -#define SQLITE_OMIT_DECLTYPE 1 -#define SQLITE_OMIT_DEPRECATED 1 -#define SQLITE_OMIT_PROGRESS_CALLBACK 1 -#define SQLITE_OMIT_SHARED_CACHE 1 -#define SQLITE_DEFAULT_MEMSTATUS 0 -#define SQLITE_MAX_EXPR_DEPTH 0 -INCLUDE sqlite3.c -INCLUDE $ROOT/src/tclsqlite.c -INCLUDE $ROOT/ext/misc/btreeinfo.c -INCLUDE $ROOT/ext/repair/checkindex.c -INCLUDE $ROOT/ext/repair/checkfreelist.c - -/* -** Decode a pointer to an sqlite3 object. -*/ -int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb){ - struct SqliteDb *p; - Tcl_CmdInfo cmdInfo; - if( Tcl_GetCommandInfo(interp, zA, &cmdInfo) ){ - p = (struct SqliteDb*)cmdInfo.objClientData; - *ppDb = p->db; - return TCL_OK; - }else{ - *ppDb = 0; - return TCL_ERROR; - } - return TCL_OK; -} - -/* -** sqlite3_imposter db main rootpage {CREATE TABLE...} ;# setup an imposter -** sqlite3_imposter db main ;# rm all imposters -*/ -static int sqlite3_imposter( - void *clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - sqlite3 *db; - const char *zSchema; - int iRoot; - const char *zSql; - - if( objc!=3 && objc!=5 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB SCHEMA [ROOTPAGE SQL]"); - return TCL_ERROR; - } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; - zSchema = Tcl_GetString(objv[2]); - if( objc==3 ){ - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 0, 1); - }else{ - if( Tcl_GetIntFromObj(interp, objv[3], &iRoot) ) return TCL_ERROR; - zSql = Tcl_GetString(objv[4]); - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 1, iRoot); - sqlite3_exec(db, zSql, 0, 0, 0); - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 0, 0); - } - return TCL_OK; -} - -#include - -const char *sqlite3_checker_init_proc(Tcl_Interp *interp){ - Tcl_CreateObjCommand(interp, "sqlite3_imposter", - (Tcl_ObjCmdProc*)sqlite3_imposter, 0, 0); - sqlite3_auto_extension((void(*)(void))sqlite3_btreeinfo_init); - sqlite3_auto_extension((void(*)(void))sqlite3_checkindex_init); - sqlite3_auto_extension((void(*)(void))sqlite3_checkfreelist_init); - return -BEGIN_STRING -INCLUDE $ROOT/ext/repair/sqlite3_checker.tcl -END_STRING -; -} diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl deleted file mode 100644 index 2ae6e15b1..000000000 --- a/ext/repair/sqlite3_checker.tcl +++ /dev/null @@ -1,264 +0,0 @@ -# This TCL script is the main driver script for the sqlite3_checker utility -# program. -# - -# Special case: -# -# sqlite3_checker --test FILENAME ARGS -# -# uses FILENAME in place of this script. -# -if {[lindex $argv 0]=="--test" && [llength $argv]>1} { - set ::argv0 [lindex $argv 1] - set argv [lrange $argv 2 end] - source $argv0 - exit 0 -} - -# Emulate a TCL shell -# -proc tclsh {} { - set line {} - while {![eof stdin]} { - if {$line!=""} { - puts -nonewline "> " - } else { - puts -nonewline "% " - } - flush stdout - append line [gets stdin] - if {[info complete $line]} { - if {[catch {uplevel #0 $line} result]} { - puts stderr "Error: $result" - } elseif {$result!=""} { - puts $result - } - set line {} - } else { - append line \n - } - } -} - -# Do an incremental integrity check of a single index -# -proc check_index {idxname batchsize bTrace} { - set i 0 - set more 1 - set nerr 0 - set pct 00.0 - set max [db one {SELECT nEntry FROM sqlite_btreeinfo('main') - WHERE name=$idxname}] - puts -nonewline "$idxname: $i of $max rows ($pct%)\r" - flush stdout - if {$bTrace} { - set sql {SELECT errmsg, current_key AS key, - CASE WHEN rowid=1 THEN scanner_sql END AS traceOut - FROM incremental_index_check($idxname) - WHERE after_key=$key - LIMIT $batchsize} - } else { - set sql {SELECT errmsg, current_key AS key, NULL AS traceOut - FROM incremental_index_check($idxname) - WHERE after_key=$key - LIMIT $batchsize} - } - while {$more} { - set more 0 - db eval $sql { - set more 1 - if {$errmsg!=""} { - incr nerr - puts "$idxname: key($key): $errmsg" - } elseif {$traceOut!=""} { - puts "$idxname: $traceOut" - } - incr i - - } - set x [format {%.1f} [expr {($i*100.0)/$max}]] - if {$x!=$pct} { - puts -nonewline "$idxname: $i of $max rows ($pct%)\r" - flush stdout - set pct $x - } - } - puts "$idxname: $nerr errors out of $i entries" -} - -# Print a usage message on standard error, then quit. -# -proc usage {} { - set argv0 [file rootname [file tail [info nameofexecutable]]] - puts stderr "Usage: $argv0 OPTIONS database-filename" - puts stderr { -Do sanity checking on a live SQLite3 database file specified by the -"database-filename" argument. - -Options: - - --batchsize N Number of rows to check per transaction - - --freelist Perform a freelist check - - --index NAME Run a check of the index NAME - - --summary Print summary information about the database - - --table NAME Run a check of all indexes for table NAME - - --tclsh Run the built-in TCL interpreter (for debugging) - - --trace (Debugging only:) Output trace information on the scan - - --version Show the version number of SQLite -} - exit 1 -} - -set file_to_analyze {} -append argv {} -set bFreelistCheck 0 -set bSummary 0 -set zIndex {} -set zTable {} -set batchsize 1000 -set bAll 1 -set bTrace 0 -set argc [llength $argv] -for {set i 0} {$i<$argc} {incr i} { - set arg [lindex $argv $i] - if {[regexp {^-+tclsh$} $arg]} { - tclsh - exit 0 - } - if {[regexp {^-+version$} $arg]} { - sqlite3 mem :memory: - puts [mem one {SELECT sqlite_version()||' '||sqlite_source_id()}] - mem close - exit 0 - } - if {[regexp {^-+freelist$} $arg]} { - set bFreelistCheck 1 - set bAll 0 - continue - } - if {[regexp {^-+summary$} $arg]} { - set bSummary 1 - set bAll 0 - continue - } - if {[regexp {^-+trace$} $arg]} { - set bTrace 1 - continue - } - if {[regexp {^-+batchsize$} $arg]} { - incr i - if {$i>=$argc} { - puts stderr "missing argument on $arg" - exit 1 - } - set batchsize [lindex $argv $i] - continue - } - if {[regexp {^-+index$} $arg]} { - incr i - if {$i>=$argc} { - puts stderr "missing argument on $arg" - exit 1 - } - set zIndex [lindex $argv $i] - set bAll 0 - continue - } - if {[regexp {^-+table$} $arg]} { - incr i - if {$i>=$argc} { - puts stderr "missing argument on $arg" - exit 1 - } - set zTable [lindex $argv $i] - set bAll 0 - continue - } - if {[regexp {^-} $arg]} { - puts stderr "Unknown option: $arg" - usage - } - if {$file_to_analyze!=""} { - usage - } else { - set file_to_analyze $arg - } -} -if {$file_to_analyze==""} usage - -# If a TCL script is specified on the command-line, then run that -# script. -# -if {[file extension $file_to_analyze]==".tcl"} { - source $file_to_analyze - exit 0 -} - -set root_filename $file_to_analyze -regexp {^file:(//)?([^?]*)} $file_to_analyze all x1 root_filename -if {![file exists $root_filename]} { - puts stderr "No such file: $root_filename" - exit 1 -} -if {![file readable $root_filename]} { - puts stderr "File is not readable: $root_filename" - exit 1 -} - -if {[catch {sqlite3 db $file_to_analyze} res]} { - puts stderr "Cannot open datababase $root_filename: $res" - exit 1 -} - -if {$bFreelistCheck || $bAll} { - puts -nonewline "freelist-check: " - flush stdout - db eval BEGIN - puts [db one {SELECT checkfreelist('main')}] - db eval END -} -if {$bSummary} { - set scale 0 - set pgsz [db one {PRAGMA page_size}] - db eval {SELECT nPage*$pgsz AS sz, name, tbl_name - FROM sqlite_btreeinfo - WHERE type='index' - ORDER BY 1 DESC, name} { - if {$scale==0} { - if {$sz>10000000} { - set scale 1000000.0 - set unit MB - } else { - set scale 1000.0 - set unit KB - } - } - puts [format {%7.1f %s index %s of table %s} \ - [expr {$sz/$scale}] $unit $name $tbl_name] - } -} -if {$zIndex!=""} { - check_index $zIndex $batchsize $bTrace -} -if {$zTable!=""} { - foreach idx [db eval {SELECT name FROM sqlite_master - WHERE type='index' AND rootpage>0 - AND tbl_name=$zTable}] { - check_index $idx $batchsize $bTrace - } -} -if {$bAll} { - set allidx [db eval {SELECT name FROM sqlite_btreeinfo('main') - WHERE type='index' AND rootpage>0 - ORDER BY nEntry}] - foreach idx $allidx { - check_index $idx $batchsize $bTrace - } -} diff --git a/ext/repair/test/README.md b/ext/repair/test/README.md deleted file mode 100644 index 8cc954adf..000000000 --- a/ext/repair/test/README.md +++ /dev/null @@ -1,13 +0,0 @@ -To run these tests, first build sqlite3_checker: - - -> make sqlite3_checker - - -Then run the "test.tcl" script using: - - -> ./sqlite3_checker --test $path/test.tcl - - -Optionally add the full pathnames of individual *.test modules diff --git a/ext/repair/test/checkfreelist01.test b/ext/repair/test/checkfreelist01.test deleted file mode 100644 index 7e2dd51c3..000000000 --- a/ext/repair/test/checkfreelist01.test +++ /dev/null @@ -1,92 +0,0 @@ -# 2017-10-11 - -set testprefix checkfreelist - -do_execsql_test 1.0 { - PRAGMA page_size=1024; - CREATE TABLE t1(a, b); -} - -do_execsql_test 1.2 { SELECT checkfreelist('main') } {ok} -do_execsql_test 1.3 { - WITH s(i) AS ( - SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10000 - ) - INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s; - DELETE FROM t1 WHERE rowid%3; - PRAGMA freelist_count; -} {6726} - -do_execsql_test 1.4 { SELECT checkfreelist('main') } {ok} -do_execsql_test 1.5 { - WITH freelist_trunk(i, d, n) AS ( - SELECT 1, NULL, sqlite_readint32(data, 32) FROM sqlite_dbpage WHERE pgno=1 - UNION ALL - SELECT n, data, sqlite_readint32(data) - FROM freelist_trunk, sqlite_dbpage WHERE pgno=n - ) - SELECT i FROM freelist_trunk WHERE i!=1; -} { - 10009 9715 9343 8969 8595 8222 7847 7474 7102 6727 6354 5982 5608 5234 - 4860 4487 4112 3740 3367 2992 2619 2247 1872 1499 1125 752 377 5 -} - -do_execsql_test 1.6 { SELECT checkfreelist('main') } {ok} - -proc set_int {blob idx newval} { - binary scan $blob I* ints - lset ints $idx $newval - binary format I* $ints -} -db func set_int set_int - -proc get_int {blob idx} { - binary scan $blob I* ints - lindex $ints $idx -} -db func get_int get_int - -do_execsql_test 1.7 { - BEGIN; - UPDATE sqlite_dbpage - SET data = set_int(data, 1, get_int(data, 1)-1) - WHERE pgno=4860; - SELECT checkfreelist('main'); - ROLLBACK; -} {{free-list count mismatch: actual=6725 header=6726}} - -do_execsql_test 1.8 { - BEGIN; - UPDATE sqlite_dbpage - SET data = set_int(data, 5, (SELECT * FROM pragma_page_count)+1) - WHERE pgno=4860; - SELECT checkfreelist('main'); - ROLLBACK; -} {{leaf page 10092 is out of range (child 3 of trunk page 4860)}} - -do_execsql_test 1.9 { - BEGIN; - UPDATE sqlite_dbpage - SET data = set_int(data, 5, 0) - WHERE pgno=4860; - SELECT checkfreelist('main'); - ROLLBACK; -} {{leaf page 0 is out of range (child 3 of trunk page 4860)}} - -do_execsql_test 1.10 { - BEGIN; - UPDATE sqlite_dbpage - SET data = set_int(data, get_int(data, 1)+1, 0) - WHERE pgno=5; - SELECT checkfreelist('main'); - ROLLBACK; -} {{leaf page 0 is out of range (child 247 of trunk page 5)}} - -do_execsql_test 1.11 { - BEGIN; - UPDATE sqlite_dbpage - SET data = set_int(data, 1, 249) - WHERE pgno=5; - SELECT checkfreelist('main'); - ROLLBACK; -} {{leaf count out of range (249) on trunk page 5}} diff --git a/ext/repair/test/checkindex01.test b/ext/repair/test/checkindex01.test deleted file mode 100644 index 97973aee7..000000000 --- a/ext/repair/test/checkindex01.test +++ /dev/null @@ -1,349 +0,0 @@ -# 2017-10-11 -# -set testprefix checkindex - -do_execsql_test 1.0 { - CREATE TABLE t1(a, b); - CREATE INDEX i1 ON t1(a); - INSERT INTO t1 VALUES('one', 2); - INSERT INTO t1 VALUES('two', 4); - INSERT INTO t1 VALUES('three', 6); - INSERT INTO t1 VALUES('four', 8); - INSERT INTO t1 VALUES('five', 10); - - CREATE INDEX i2 ON t1(a DESC); -} {} - -proc incr_index_check {idx nStep} { - set Q { - SELECT errmsg, current_key FROM incremental_index_check($idx, $after) - LIMIT $nStep - } - - set res [list] - while {1} { - unset -nocomplain current_key - set res1 [db eval $Q] - if {[llength $res1]==0} break - set res [concat $res $res1] - set after [lindex $res end] - } - - return $res -} - -proc do_index_check_test {tn idx res} { - uplevel [list do_execsql_test $tn.1 " - SELECT errmsg, current_key FROM incremental_index_check('$idx'); - " $res] - - uplevel [list do_test $tn.2 "incr_index_check $idx 1" [list {*}$res]] - uplevel [list do_test $tn.3 "incr_index_check $idx 2" [list {*}$res]] - uplevel [list do_test $tn.4 "incr_index_check $idx 5" [list {*}$res]] -} - - -do_execsql_test 1.2.1 { - SELECT rowid, errmsg IS NULL, current_key FROM incremental_index_check('i1'); -} { - 1 1 'five',5 - 2 1 'four',4 - 3 1 'one',1 - 4 1 'three',3 - 5 1 'two',2 -} -do_execsql_test 1.2.2 { - SELECT errmsg IS NULL, current_key, index_name, after_key, scanner_sql - FROM incremental_index_check('i1') LIMIT 1; -} { - 1 - 'five',5 - i1 - {} - {SELECT (SELECT a IS i.i0 FROM 't1' AS t WHERE "rowid" COLLATE BINARY IS i.i1), quote(i0)||','||quote(i1) FROM (SELECT (a) AS i0, ("rowid" COLLATE BINARY) AS i1 FROM 't1' INDEXED BY 'i1' ORDER BY 1,2) AS i} -} - -do_index_check_test 1.3 i1 { - {} 'five',5 - {} 'four',4 - {} 'one',1 - {} 'three',3 - {} 'two',2 -} - -do_index_check_test 1.4 i2 { - {} 'two',2 - {} 'three',3 - {} 'one',1 - {} 'four',4 - {} 'five',5 -} - -do_test 1.5 { - set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t1' }] - sqlite3_imposter db main $tblroot {CREATE TABLE xt1(a,b)} - db eval { - UPDATE xt1 SET a='six' WHERE rowid=3; - DELETE FROM xt1 WHERE rowid = 5; - } - sqlite3_imposter db main -} {} - -do_index_check_test 1.6 i1 { - {row missing} 'five',5 - {} 'four',4 - {} 'one',1 - {row data mismatch} 'three',3 - {} 'two',2 -} - -do_index_check_test 1.7 i2 { - {} 'two',2 - {row data mismatch} 'three',3 - {} 'one',1 - {} 'four',4 - {row missing} 'five',5 -} - -#-------------------------------------------------------------------------- -do_execsql_test 2.0 { - - CREATE TABLE t2(a INTEGER PRIMARY KEY, b, c, d); - - INSERT INTO t2 VALUES(1, NULL, 1, 1); - INSERT INTO t2 VALUES(2, 1, NULL, 1); - INSERT INTO t2 VALUES(3, 1, 1, NULL); - - INSERT INTO t2 VALUES(4, 2, 2, 1); - INSERT INTO t2 VALUES(5, 2, 2, 2); - INSERT INTO t2 VALUES(6, 2, 2, 3); - - INSERT INTO t2 VALUES(7, 2, 2, 1); - INSERT INTO t2 VALUES(8, 2, 2, 2); - INSERT INTO t2 VALUES(9, 2, 2, 3); - - CREATE INDEX i3 ON t2(b, c, d); - CREATE INDEX i4 ON t2(b DESC, c DESC, d DESC); - CREATE INDEX i5 ON t2(d, c DESC, b); -} {} - -do_index_check_test 2.1 i3 { - {} NULL,1,1,1 - {} 1,NULL,1,2 - {} 1,1,NULL,3 - {} 2,2,1,4 - {} 2,2,1,7 - {} 2,2,2,5 - {} 2,2,2,8 - {} 2,2,3,6 - {} 2,2,3,9 -} - -do_index_check_test 2.2 i4 { - {} 2,2,3,6 - {} 2,2,3,9 - {} 2,2,2,5 - {} 2,2,2,8 - {} 2,2,1,4 - {} 2,2,1,7 - {} 1,1,NULL,3 - {} 1,NULL,1,2 - {} NULL,1,1,1 -} - -do_index_check_test 2.3 i5 { - {} NULL,1,1,3 - {} 1,2,2,4 - {} 1,2,2,7 - {} 1,1,NULL,1 - {} 1,NULL,1,2 - {} 2,2,2,5 - {} 2,2,2,8 - {} 3,2,2,6 - {} 3,2,2,9 -} - -#-------------------------------------------------------------------------- -do_execsql_test 3.0 { - - CREATE TABLE t3(w, x, y, z PRIMARY KEY) WITHOUT ROWID; - CREATE INDEX t3wxy ON t3(w, x, y); - CREATE INDEX t3wxy2 ON t3(w DESC, x DESC, y DESC); - - INSERT INTO t3 VALUES(NULL, NULL, NULL, 1); - INSERT INTO t3 VALUES(NULL, NULL, NULL, 2); - INSERT INTO t3 VALUES(NULL, NULL, NULL, 3); - - INSERT INTO t3 VALUES('a', NULL, NULL, 4); - INSERT INTO t3 VALUES('a', NULL, NULL, 5); - INSERT INTO t3 VALUES('a', NULL, NULL, 6); - - INSERT INTO t3 VALUES('a', 'b', NULL, 7); - INSERT INTO t3 VALUES('a', 'b', NULL, 8); - INSERT INTO t3 VALUES('a', 'b', NULL, 9); - -} {} - -do_index_check_test 3.1 t3wxy { - {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3 - {} 'a',NULL,NULL,4 {} 'a',NULL,NULL,5 {} 'a',NULL,NULL,6 - {} 'a','b',NULL,7 {} 'a','b',NULL,8 {} 'a','b',NULL,9 -} -do_index_check_test 3.2 t3wxy2 { - {} 'a','b',NULL,7 {} 'a','b',NULL,8 {} 'a','b',NULL,9 - {} 'a',NULL,NULL,4 {} 'a',NULL,NULL,5 {} 'a',NULL,NULL,6 - {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3 -} - -#-------------------------------------------------------------------------- -# Test with an index that uses non-default collation sequences. -# -do_execsql_test 4.0 { - CREATE TABLE t4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT); - INSERT INTO t4 VALUES(1, 'aaa', 'bbb'); - INSERT INTO t4 VALUES(2, 'AAA', 'CCC'); - INSERT INTO t4 VALUES(3, 'aab', 'ddd'); - INSERT INTO t4 VALUES(4, 'AAB', 'EEE'); - - CREATE INDEX t4cc ON t4(c1 COLLATE nocase, c2 COLLATE nocase); -} - -do_index_check_test 4.1 t4cc { - {} 'aaa','bbb',1 - {} 'AAA','CCC',2 - {} 'aab','ddd',3 - {} 'AAB','EEE',4 -} - -do_test 4.2 { - set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t4' }] - sqlite3_imposter db main $tblroot \ - {CREATE TABLE xt4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT)} - - db eval { - UPDATE xt4 SET c1='hello' WHERE rowid=2; - DELETE FROM xt4 WHERE rowid = 3; - } - sqlite3_imposter db main -} {} - -do_index_check_test 4.3 t4cc { - {} 'aaa','bbb',1 - {row data mismatch} 'AAA','CCC',2 - {row missing} 'aab','ddd',3 - {} 'AAB','EEE',4 -} - -#-------------------------------------------------------------------------- -# Test an index on an expression. -# -do_execsql_test 5.0 { - CREATE TABLE t5(x INTEGER PRIMARY KEY, y TEXT, UNIQUE(y)); - INSERT INTO t5 VALUES(1, '{"x":1, "y":1}'); - INSERT INTO t5 VALUES(2, '{"x":2, "y":2}'); - INSERT INTO t5 VALUES(3, '{"x":3, "y":3}'); - INSERT INTO t5 VALUES(4, '{"w":4, "z":4}'); - INSERT INTO t5 VALUES(5, '{"x":5, "y":5}'); - - CREATE INDEX t5x ON t5( json_extract(y, '$.x') ); - CREATE INDEX t5y ON t5( json_extract(y, '$.y') DESC ); -} - -do_index_check_test 5.1.1 t5x { - {} NULL,4 {} 1,1 {} 2,2 {} 3,3 {} 5,5 -} - -do_index_check_test 5.1.2 t5y { - {} 5,5 {} 3,3 {} 2,2 {} 1,1 {} NULL,4 -} - -do_index_check_test 5.1.3 sqlite_autoindex_t5_1 { - {} {'{"w":4, "z":4}',4} - {} {'{"x":1, "y":1}',1} - {} {'{"x":2, "y":2}',2} - {} {'{"x":3, "y":3}',3} - {} {'{"x":5, "y":5}',5} -} - -do_test 5.2 { - set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t5' }] - sqlite3_imposter db main $tblroot \ - {CREATE TABLE xt5(a INTEGER PRIMARY KEY, c1 TEXT);} - db eval { - UPDATE xt5 SET c1='{"x":22, "y":11}' WHERE rowid=1; - DELETE FROM xt5 WHERE rowid = 4; - } - sqlite3_imposter db main -} {} - -do_index_check_test 5.3.1 t5x { - {row missing} NULL,4 - {row data mismatch} 1,1 - {} 2,2 - {} 3,3 - {} 5,5 -} - -do_index_check_test 5.3.2 sqlite_autoindex_t5_1 { - {row missing} {'{"w":4, "z":4}',4} - {row data mismatch} {'{"x":1, "y":1}',1} - {} {'{"x":2, "y":2}',2} - {} {'{"x":3, "y":3}',3} - {} {'{"x":5, "y":5}',5} -} - -#------------------------------------------------------------------------- -# -do_execsql_test 6.0 { - CREATE TABLE t6(x INTEGER PRIMARY KEY, y, z); - CREATE INDEX t6x1 ON t6(y, /* one,two,three */ z); - CREATE INDEX t6x2 ON t6(z, -- hello,world, - y); - - CREATE INDEX t6x3 ON t6(z -- hello,world - , y); - - INSERT INTO t6 VALUES(1, 2, 3); - INSERT INTO t6 VALUES(4, 5, 6); -} - -do_index_check_test 6.1 t6x1 { - {} 2,3,1 - {} 5,6,4 -} -do_index_check_test 6.2 t6x2 { - {} 3,2,1 - {} 6,5,4 -} -do_index_check_test 6.2 t6x3 { - {} 3,2,1 - {} 6,5,4 -} - -#------------------------------------------------------------------------- -# -do_execsql_test 7.0 { - CREATE TABLE t7(x INTEGER PRIMARY KEY, y, z); - INSERT INTO t7 VALUES(1, 1, 1); - INSERT INTO t7 VALUES(2, 2, 0); - INSERT INTO t7 VALUES(3, 3, 1); - INSERT INTO t7 VALUES(4, 4, 0); - - CREATE INDEX t7i1 ON t7(y) WHERE z=1; - CREATE INDEX t7i2 ON t7(y) /* hello,world */ WHERE z=1; - CREATE INDEX t7i3 ON t7(y) WHERE -- yep - z=1; - CREATE INDEX t7i4 ON t7(y) WHERE z=1 -- yep; -} -do_index_check_test 7.1 t7i1 { - {} 1,1 {} 3,3 -} -do_index_check_test 7.2 t7i2 { - {} 1,1 {} 3,3 -} -do_index_check_test 7.3 t7i3 { - {} 1,1 {} 3,3 -} -do_index_check_test 7.4 t7i4 { - {} 1,1 {} 3,3 -} diff --git a/ext/repair/test/test.tcl b/ext/repair/test/test.tcl deleted file mode 100644 index c073bb73c..000000000 --- a/ext/repair/test/test.tcl +++ /dev/null @@ -1,67 +0,0 @@ -# Run this script using -# -# sqlite3_checker --test $thisscript $testscripts -# -# The $testscripts argument is optional. If omitted, all *.test files -# in the same directory as $thisscript are run. -# -set NTEST 0 -set NERR 0 - - -# Invoke the do_test procedure to run a single test -# -# The $expected parameter is the expected result. The result is the return -# value from the last TCL command in $cmd. -# -# Normally, $expected must match exactly. But if $expected is of the form -# "/regexp/" then regular expression matching is used. If $expected is -# "~/regexp/" then the regular expression must NOT match. If $expected is -# of the form "#/value-list/" then each term in value-list must be numeric -# and must approximately match the corresponding numeric term in $result. -# Values must match within 10%. Or if the $expected term is A..B then the -# $result term must be in between A and B. -# -proc do_test {name cmd expected} { - if {[info exists ::testprefix]} { - set name "$::testprefix$name" - } - - incr ::NTEST - puts -nonewline $name... - flush stdout - - if {[catch {uplevel #0 "$cmd;\n"} result]} { - puts -nonewline $name... - puts "\nError: $result" - incr ::NERR - } else { - set ok [expr {[string compare $result $expected]==0}] - if {!$ok} { - puts "\n! $name expected: \[$expected\]\n! $name got: \[$result\]" - incr ::NERR - } else { - puts " Ok" - } - } - flush stdout -} - -# -# do_execsql_test TESTNAME SQL RES -# -proc do_execsql_test {testname sql {result {}}} { - uplevel [list do_test $testname [list db eval $sql] [list {*}$result]] -} - -if {[llength $argv]==0} { - set dir [file dirname $argv0] - set argv [glob -nocomplain $dir/*.test] -} -foreach testfile $argv { - file delete -force test.db - sqlite3 db test.db - source $testfile - catch {db close} -} -puts "$NERR errors out of $NTEST tests" diff --git a/ext/rtree/geopoly.c b/ext/rtree/geopoly.c index 0ae42e7b7..22166a6f9 100644 --- a/ext/rtree/geopoly.c +++ b/ext/rtree/geopoly.c @@ -200,7 +200,7 @@ static int geopolyParseNumber(GeoParse *p, GeoCoord *pVal){ /* The sqlite3AtoF() routine is much much faster than atof(), if it ** is available */ double r; - (void)sqlite3AtoF((const char*)p->z, &r, j, SQLITE_UTF8); + (void)sqlite3AtoF((const char*)p->z, &r); *pVal = r; #else *pVal = (GeoCoord)atof((const char*)p->z); diff --git a/ext/session/session4.test b/ext/session/session4.test index 55cb76f15..5e44a8eb6 100644 --- a/ext/session/session4.test +++ b/ext/session/session4.test @@ -135,6 +135,7 @@ foreach {tn blob} { 54 540101743400120003001200010000000000000002120002400C000000000002120002400C00000000000050040100000074310017FF0050040100000074310017FF7F00000000000000050100000000000000030100000003001700010000666F7572 55 540101743400120003001200010000000000000002120002400C00000000000050040100000074310017000100010080000001000000020003010100000300170100000003001700010000666F7572 56 5487ffffff7f + 57 54015052494e5446110017120004 } { do_test 2.$tn { set changeset [binary decode hex $blob] diff --git a/ext/session/sessionC.test b/ext/session/sessionC.test index 74370cb79..1997ba5e8 100644 --- a/ext/session/sessionC.test +++ b/ext/session/sessionC.test @@ -192,6 +192,16 @@ do_test 3.3 { } } {1 1 3 3} +#------------------------------------------------------------------------- +# +reset_db +set C [binary format c* 0x54 0x01 0x01 0x00 0x12 0x00 0x05] +do_test 4.0 { + sqlite3changegroup grp + list [catch { grp add $C } msg] $msg +} {1 SQLITE_CORRUPT} +grp delete finish_test + diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index 90fedc6db..5468dff28 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -349,6 +349,20 @@ static int sessionVarintGet(const u8 *aBuf, int *piVal){ return getVarint32(aBuf, *piVal); } +/* +** Read a varint value from buffer aBuf[], size nBuf bytes, into *piVal. +** Return the number of bytes read. +*/ +static int sessionVarintGetSafe(const u8 *aBuf, int nBuf, int *piVal){ + u8 aCopy[5]; + const u8 *aRead = aBuf; + if( nBuf<5 ){ + memcpy(aCopy, aBuf, nBuf); + aRead = aCopy; + } + return getVarint32(aRead, *piVal); +} + /* Load an unaligned and unsigned 32-bit integer */ #define SESSION_UINT32(x) (((u32)(x)[0]<<24)|((x)[1]<<16)|((x)[2]<<8)|(x)[3]) @@ -643,14 +657,10 @@ static unsigned int sessionChangeHash( int isPK = pTab->abPK[i]; if( bPkOnly && isPK==0 ) continue; - /* It is not possible for eType to be SQLITE_NULL here. The session - ** module does not record changes for rows with NULL values stored in - ** primary key columns. */ assert( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT || eType==SQLITE_TEXT || eType==SQLITE_BLOB || eType==SQLITE_NULL || eType==0 ); - assert( !isPK || (eType!=0 && eType!=SQLITE_NULL) ); if( isPK ){ a++; @@ -658,12 +668,16 @@ static unsigned int sessionChangeHash( if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ h = sessionHashAppendI64(h, sessionGetI64(a)); a += 8; - }else{ + }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ int n; a += sessionVarintGet(a, &n); h = sessionHashAppendBlob(h, n, a); a += n; } + /* It should not be possible for eType to be SQLITE_NULL or 0x00 here, + ** as the session module does not record changes for rows with NULL + ** values stored in primary key columns. But a corrupt changesets + ** may contain such a value. */ }else{ a += sessionSerialLen(a); } @@ -3072,10 +3086,13 @@ static int sessionGenerateChangeset( } if( pSession->rc ) return pSession->rc; - rc = sqlite3_exec(pSession->db, "SAVEPOINT changeset", 0, 0, 0); - if( rc!=SQLITE_OK ) return rc; sqlite3_mutex_enter(sqlite3_db_mutex(db)); + rc = sqlite3_exec(pSession->db, "SAVEPOINT changeset", 0, 0, 0); + if( rc!=SQLITE_OK ){ + sqlite3_mutex_leave(sqlite3_db_mutex(db)); + return rc; + } for(pTab=pSession->pTable; rc==SQLITE_OK && pTab; pTab=pTab->pNext){ if( pTab->nEntry ){ @@ -3558,7 +3575,8 @@ static int sessionReadRecord( u8 *aVal = &pIn->aData[pIn->iNext]; if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ int nByte; - pIn->iNext += sessionVarintGet(aVal, &nByte); + int nRem = pIn->nData - pIn->iNext; + pIn->iNext += sessionVarintGetSafe(aVal, nRem, &nByte); rc = sessionInputBuffer(pIn, nByte); if( rc==SQLITE_OK ){ if( nByte<0 || nByte>pIn->nData-pIn->iNext ){ @@ -3611,7 +3629,8 @@ static int sessionChangesetBufferTblhdr(SessionInput *pIn, int *pnByte){ rc = sessionInputBuffer(pIn, 9); if( rc==SQLITE_OK ){ - nRead += sessionVarintGet(&pIn->aData[pIn->iNext + nRead], &nCol); + int nBuf = pIn->nData - pIn->iNext; + nRead += sessionVarintGetSafe(&pIn->aData[pIn->iNext], nBuf, &nCol); /* The hard upper limit for the number of columns in an SQLite ** database table is, according to sqliteLimit.h, 32676. So ** consider any table-header that purports to have more than 65536 @@ -3631,8 +3650,15 @@ static int sessionChangesetBufferTblhdr(SessionInput *pIn, int *pnByte){ while( (pIn->iNext + nRead)nData && pIn->aData[pIn->iNext + nRead] ){ nRead++; } + + /* Break out of the loop if if the nul-terminator byte has been found. + ** Otherwise, read some more input data and keep seeking. If there is + ** no more input data, consider the changeset corrupt. */ if( (pIn->iNext + nRead)nData ) break; rc = sessionInputBuffer(pIn, nRead + 100); + if( rc==SQLITE_OK && (pIn->iNext + nRead)>=pIn->nData ){ + rc = SQLITE_CORRUPT_BKPT; + } } *pnByte = nRead+1; return rc; @@ -3764,10 +3790,10 @@ static int sessionChangesetNextOne( memset(p->apValue, 0, sizeof(sqlite3_value*)*p->nCol*2); } - /* Make sure the buffer contains at least 10 bytes of input data, or all - ** remaining data if there are less than 10 bytes available. This is - ** sufficient either for the 'T' or 'P' byte and the varint that follows - ** it, or for the two single byte values otherwise. */ + /* Make sure the buffer contains at least 2 bytes of input data, or all + ** remaining data if there are less than 2 bytes available. This is + ** sufficient either for the 'T' or 'P' byte that begins a new table, + ** or for the "op" and "bIndirect" single bytes otherwise. */ p->rc = sessionInputBuffer(&p->in, 2); if( p->rc!=SQLITE_OK ) return p->rc; @@ -3797,11 +3823,13 @@ static int sessionChangesetNextOne( return (p->rc = SQLITE_CORRUPT_BKPT); } - p->op = op; - p->bIndirect = p->in.aData[p->in.iNext++]; - if( p->op!=SQLITE_UPDATE && p->op!=SQLITE_DELETE && p->op!=SQLITE_INSERT ){ + if( (op!=SQLITE_UPDATE && op!=SQLITE_DELETE && op!=SQLITE_INSERT) + || (p->in.iNext>=p->in.nData) + ){ return (p->rc = SQLITE_CORRUPT_BKPT); } + p->op = op; + p->bIndirect = p->in.aData[p->in.iNext++]; if( paRec ){ int nVal; /* Number of values to buffer */ diff --git a/ext/session/test_session.c b/ext/session/test_session.c index 6ad5b3774..46a9e3a38 100644 --- a/ext/session/test_session.c +++ b/ext/session/test_session.c @@ -7,6 +7,8 @@ #include #include "tclsqlite.h" +#include + #ifndef SQLITE_AMALGAMATION typedef unsigned char u8; #endif @@ -856,6 +858,21 @@ static int testStreamInput( return SQLITE_OK; } +/* +** This works like Tcl_GetByteArrayFromObj(), except that it returns a buffer +** allocated using malloc() that must be freed by the caller. This is done +** because Tcl's buffers are often padded by a few bytes, which prevents +** small overreads from being detected when tests are run under asan. +*/ +static void *testGetByteArrayFromObj(Tcl_Obj *p, Tcl_Size *pnByte){ + Tcl_Size nByte = 0; + void *aByte = Tcl_GetByteArrayFromObj(p, &nByte); + void *aCopy = malloc(nByte ? (size_t)nByte : 1); + memcpy(aCopy, aByte, (size_t)nByte); + *pnByte = nByte; + return aCopy; +} + static int SQLITE_TCLAPI testSqlite3changesetApply( int iVersion, @@ -920,7 +937,7 @@ static int SQLITE_TCLAPI testSqlite3changesetApply( return TCL_ERROR; } db = *(sqlite3 **)info.objClientData; - pChangeset = (void *)Tcl_GetByteArrayFromObj(objv[2], &nChangeset); + pChangeset = (void *)testGetByteArrayFromObj(objv[2], &nChangeset); ctx.pConflictScript = objv[3]; ctx.pFilterScript = objc==5 ? objv[4] : 0; ctx.interp = interp; @@ -972,6 +989,7 @@ static int SQLITE_TCLAPI testSqlite3changesetApply( } } + free(pChangeset); if( rc!=SQLITE_OK ){ return test_session_error(interp, rc, 0); }else{ @@ -1195,7 +1213,12 @@ static int SQLITE_TCLAPI test_sqlite3session_foreach( pCS = objv[2]; pScript = objv[3]; - pChangeset = (void *)Tcl_GetByteArrayFromObj(pCS, &nChangeset); + /* Take a copy of the changeset into an exact sized buffer allocated + ** using malloc(). The Tcl buffer will be padded by a few bytes, which + ** prevents small overreads from being detected by ASAN when the tests + ** are run. */ + pChangeset = (void*)testGetByteArrayFromObj(pCS, &nChangeset); + sStr.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR); if( isInvert ){ int f = SQLITE_CHANGESETSTART_INVERT; @@ -1216,32 +1239,33 @@ static int SQLITE_TCLAPI test_sqlite3session_foreach( rc = sqlite3changeset_start_strm(&pIter, testStreamInput, (void*)&sStr); } } - if( rc!=SQLITE_OK ){ - return test_session_error(interp, rc, 0); - } - while( SQLITE_ROW==sqlite3changeset_next(pIter) ){ - Tcl_Obj *pVar = 0; /* Tcl value to set $VARNAME to */ - pVar = testIterData(pIter); - Tcl_ObjSetVar2(interp, pVarname, 0, pVar, 0); - rc = Tcl_EvalObjEx(interp, pScript, 0); - if( rc!=TCL_OK && rc!=TCL_CONTINUE ){ - sqlite3changeset_finalize(pIter); - return rc==TCL_BREAK ? TCL_OK : rc; + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==sqlite3changeset_next(pIter) ){ + Tcl_Obj *pVar = 0; /* Tcl value to set $VARNAME to */ + pVar = testIterData(pIter); + Tcl_ObjSetVar2(interp, pVarname, 0, pVar, 0); + rc = Tcl_EvalObjEx(interp, pScript, 0); + if( rc!=TCL_OK && rc!=TCL_CONTINUE ){ + sqlite3changeset_finalize(pIter); + free(pChangeset); + return rc==TCL_BREAK ? TCL_OK : rc; + } } - } - if( isCheckNext ){ - int rc2 = sqlite3changeset_next(pIter); - rc = sqlite3changeset_finalize(pIter); - assert( (rc2==SQLITE_DONE && rc==SQLITE_OK) || rc2==rc ); - }else{ - rc = sqlite3changeset_finalize(pIter); + if( isCheckNext ){ + int rc2 = sqlite3changeset_next(pIter); + rc = sqlite3changeset_finalize(pIter); + assert( (rc2==SQLITE_DONE && rc==SQLITE_OK) || rc2==rc ); + }else{ + rc = sqlite3changeset_finalize(pIter); + } } + + free(pChangeset); if( rc!=SQLITE_OK ){ return test_session_error(interp, rc, 0); } - return TCL_OK; } diff --git a/ext/wasm/EXPORTED_FUNCTIONS.fiddle.in b/ext/wasm/EXPORTED_FUNCTIONS.fiddle.in deleted file mode 100644 index 103704df1..000000000 --- a/ext/wasm/EXPORTED_FUNCTIONS.fiddle.in +++ /dev/null @@ -1,10 +0,0 @@ -_fiddle_db_arg -_fiddle_db_filename -_fiddle_exec -_fiddle_experiment -_fiddle_interrupt -_fiddle_main -_fiddle_reset_db -_fiddle_db_handle -_fiddle_db_vfs -_fiddle_export_db diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 937e16d6e..dc3c2d255 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -1,5 +1,6 @@ -####################################################################### -# This GNU makefile creates the canonical sqlite3 WASM builds. +# +# This GNU makefile creates the canonical sqlite3 WASM builds. Plus some +# others. # # This build assumes a Linux platform and is not intended for # general-purpose client-level use, except for creating builds with @@ -10,28 +11,33 @@ # # default, all = build in dev mode # -# o0, o1, o2, o3, os, oz = full clean/rebuild with the -Ox level indicated -# by the target name. Rebuild is necessary for all components to get -# the desired optimization level. +# o0, o1, o2, o3, os, oz = full clean/rebuild with the -Ox level +# indicated by the target name. A clean rebuild is necessary for +# all components to get the desired optimization level. # # dist = create end user deliverables. Add dist.build=oX to build -# with a specific optimization level, where oX is one of the -# above-listed o? or qo? target names. +# with a specific optimization level, where oX is one of the +# above-listed o? target names. # # snapshot = like dist, but uses a zip file name which clearly -# marks it as a prerelease/snapshot build. +# marks it as a prerelease/snapshot build. # # clean = clean up # # Required tools beyond those needed for the canonical builds: # # - Emscripten SDK: https://emscripten.org/docs/getting_started/downloads.html +# # - The bash shell +# # - GNU make, GNU sed, GNU awk, GNU grep (all in the $PATH and without # a "g" prefix like they have on some non-GNU systems) -# - wasm-strip for release builds: https://github.com/WebAssembly/wabt +# +# - wasm-strip for release builds: https://github.com/WebAssembly/wabt. +# It will build without this but the .wasm files will be huge. +# # - InfoZip for 'dist' zip file -######################################################################## +# default: all MAKEFILE = $(lastword $(MAKEFILE_LIST)) CLEAN_FILES = @@ -80,6 +86,7 @@ emo.strip = 💈 emo.test = 🧪 emo.tool = 🔨 emo.wasm-opt = 🧼 +emo.cleanup = 🧼 # 👷🪄🧮🧫🧽🍿⛽🚧🎱🪚🏆🧼 # @@ -199,7 +206,7 @@ b.mkdir@ = if [ ! -d $(dir $@) ]; then \ # $1 = logtag, $2 = src file(s). $3 = dest dir b.cp = $(call b.mkdir@); \ echo '$(logtag.$(1)) $(emo.disk) $(2) ==> $3'; \ - cp -p $(2) $(3) || exit + cp -f -p $(2) $(3) || exit # # $(call b.c-pp.shcmd,LOGTAG,src,dest,-Dx=y...) @@ -213,9 +220,8 @@ b.cp = $(call b.mkdir@); \ # $4 = optional $(bin.c-pp) flags define b.c-pp.shcmd $(call b.mkdir@); \ -$(call b.echo,$(1),$(emo.disk)$(emo.lock) $(bin.c-pp) $(4) $(if $(loud.if),$(2))); \ -rm -f $(3); \ -$(bin.c-pp) -o $(3) $(4) $(2) || exit; \ +$(call b.echo,$(1),$(emo.disk)$(emo.lock)[$(3)] $(4)); \ +rm -f $(3); $(bin.c-pp) -o $(3) $(4) $(2) || exit; \ chmod -w $(3) endef @@ -227,10 +233,38 @@ endef # Args: as for $(b.c-pp.shcmd). define b.c-pp.target $(3): $$(MAKEFILE_LIST) $$(bin.c-pp) $(2) - @$$(call b.c-pp.shcmd,$(1),$(2),$(3),$(4) $(b.c-pp.target.flags)) + @$$(call b.mkdir@) + @$$(call b.c-pp.shcmd,$(1),$(2),$(3),$(4) $$(b.c-pp.target.flags)) CLEAN_FILES += $(3) endef + +# +# The various -D... values used by *.c-pp.js include: +# +# -Dtarget:es6-module: for all ESM module builds +# +# -Dtarget:node: for node.js builds +# +# -Dtarget:es6-module -Dtarget:es6-bundler-friendly: intended for +# "bundler-friendly" ESM module build. These have some restrictions +# on how URL() objects are constructed in some contexts: URLs which +# refer to files which are part of this project must be referenced +# as string literals so that bundlers' static-analysis tools can +# find those files and include them in their bundles. +# +# -Dtarget:es6-module -Dtarget:node: is intended for use by node.js +# for node.js, as opposed to by node.js on behalf of a +# browser. Mixing -sENVIRONMENT=web and -sENVIRONMENT=node leads to +# ambiguity and confusion on node's part, as it's unable to +# reliably determine whether the target is a browser or node. +# +# To repeat: all node.js builds are 100% untested and unsupported. +# +# Most c-pp.D.X are set via $(bin.mkwb) and X is a build name. +# Those make rules reference c-pp.D.64bit, so it should be defined in +# advance. +# c-pp.D.64bit = -Dbits64 # @@ -248,11 +282,21 @@ c-pp.D.64bit = -Dbits64 # # This is intended to be used in makefile targets which generate an # Emscripten module and where $@ is the module's .js/.mjs file. +# +# This is inherently fragile and has been broken by Emscripten updates +# before. +# +ifeq (1,1) +define b.strip-js-emcc-bindings +echo "$(1) $(emo.garbage) Stripping export wrappers."; \ +sed -i -e '/var _sqlite3.*makeInvalidEarly.*;/d' \ +-e '/assert.*sqlite3.*missing.*;/d' \ +-e '/_sqlite.*createExportWrapper.*;/d' $@ +endef +else b.strip-js-emcc-bindings = \ - sed -i -e '/^.*= \(_sqlite3\|_fiddle\)[^=]*=.*createExportWrapper/d' \ - -e '/^var \(_sqlite3\|_fiddle\)[^=]*=.*makeInvalidEarlyAccess/d' $@ || exit; \ - echo '$(1) $(emo.garbage) (Probably) /createExportWrapper()/d and /makeInvalidEarlyAccess()/d' - + echo '$(1) $(emo.bug) (disabled because it breaks emsdk 4.0.16+)' +endif # # Set up sqlite3.c and sqlite3.h... @@ -271,33 +315,36 @@ b.strip-js-emcc-bindings = \ # $(sqlite3.canonical.c) must point to the sqlite3.c in # the sqlite3 canonical source tree, as that source file # is required for certain utility and test code. +# sqlite3.canonical.c = $(dir.top)/sqlite3.c sqlite3.c ?= $(firstword $(wildcard $(dir.top)/sqlite3-see.c) $(sqlite3.canonical.c)) -sqlite3.h = $(dir $(sqlite3.c))/sqlite3.h +sqlite3.h = $(dir $(sqlite3.c))sqlite3.h # # bin.version-info = binary to output various sqlite3 version info for # embedding in the JS files and in building the distribution zip file. # It must NOT be in $(dir.tmp) because we need it to survive the # cleanup process for the dist build to work properly. +# bin.version-info = ./version-info $(bin.version-info): $(dir.tool)/version-info.c $(sqlite3.h) $(dir.top)/Makefile $(CC) -o $@ -I$(dir $(sqlite3.h)) $(dir.tool)/version-info.c t-version-info: $(bin.version-info) DISTCLEAN_FILES += $(bin.version-info) + # # bin.stripcomments is used for stripping C/C++-style comments from JS # files. The JS files contain large chunks of documentation which we # don't need for all builds. That app's -k flag is of particular # importance here, as it allows us to retain the opening comment # block(s), which contain the license header and version info. +# bin.stripccomments = $(dir.tool)/stripccomments $(bin.stripccomments): $(bin.stripccomments).c $(MAKEFILE) $(CC) -o $@ $< t-stripccomments: $(bin.stripccomments) DISTCLEAN_FILES += $(bin.stripccomments) - ifeq (1,$(MAKING_CLEAN)) SQLITE_C_IS_SEE = 0 else @@ -325,7 +372,7 @@ endif # undefine barebones # relatively new gmake feature, not ubiquitous # -# It's important that sqlite3.h be built to completion before any +# It's important that sqlite3.[ch] be built to completion before any # other parts of the build run, thus we use .NOTPARALLEL to disable # parallel build of that file and its dependants. However, that makes # the whole build non-parallelizable because everything has a dep on @@ -342,8 +389,10 @@ $(sqlite3.h): # $(MAKE) -C $(dir.top) sqlite3.c $(sqlite3.c): $(sqlite3.h) +# # Common options for building sqlite3-wasm.c and speedtest1.c. # Explicit ENABLEs... +# SQLITE_OPT.common = \ -DSQLITE_THREADSAFE=0 \ -DSQLITE_TEMP_STORE=2 \ @@ -360,11 +409,15 @@ SQLITE_OPT.common = \ # removing them from this list will serve only to break the speedtest1 # builds. +# # Currently always needed but TODO is paring tester1.c-pp.js down # to be able to run without this: +# SQLITE_OPT.common += -DSQLITE_WASM_ENABLE_C_TESTS +# # Extra flags for full-featured builds... +# SQLITE_OPT.full-featured = \ -DSQLITE_ENABLE_BYTECODE_VTAB \ -DSQLITE_ENABLE_DBPAGE_VTAB \ @@ -421,13 +474,14 @@ else # -DSQLITE_OMIT_WINDOWFUNC endif +# #SQLITE_OPT += -DSQLITE_DEBUG # Enabling SQLITE_DEBUG will break sqlite3_wasm_vfs_create_file() # (and thus sqlite3_js_vfs_create_file()). Those functions are # deprecated and alternatives are in place, but this crash behavior # can be used to find errant uses of sqlite3_js_vfs_create_file() # in client code. -######################################################################## +# # The following flags are hard-coded into sqlite3-wasm.c and cannot be # modified via the build process: # @@ -436,10 +490,9 @@ endif # SQLITE_OMIT_DEPRECATED # SQLITE_OMIT_UTF16 # SQLITE_OMIT_SHARED_CACHE -######################################################################## - +# -######################################################################## +# # Adding custom C code via sqlite3_wasm_extra_init.c: # # If the canonical build process finds the file @@ -460,7 +513,7 @@ endif # make sqlite3_wasm_extra_init.c=my_custom_stuff.c # # See example_extra_init.c for an example implementation. -######################################################################## +# sqlite3_wasm_extra_init.c ?= $(wildcard sqlite3_wasm_extra_init.c) cflags.wasm_extra_init = ifneq (,$(sqlite3_wasm_extra_init.c)) @@ -481,7 +534,7 @@ endif # WASM_CUSTOM_INSTANTIATE = 1 -######################################################################## +# # $(bin.c-pp): a minimal text file preprocessor. Like C's but much # less so. # @@ -512,6 +565,7 @@ WASM_CUSTOM_INSTANTIATE = 1 # # -D... flags which should be included in all invocations should be # appended to $(b.c-pp.target.flags). +# bin.c-pp = ./c-pp-lite $(bin.c-pp): c-pp-lite.c $(sqlite3.c) $(MAKEFILE) $(CC) -O0 -o $@ c-pp-lite.c $(sqlite3.c) '-DCMPP_DEFAULT_DELIM="//#"' -I$(dir.top) \ @@ -523,7 +577,12 @@ b.c-pp.target.flags ?= ifeq (1,$(SQLITE_C_IS_SEE)) b.c-pp.target.flags += -Denable-see endif +api.oo1 ?= 1 +ifeq (0,$(api.oo1)) + b.c-pp.target.flags += -Domit-oo1 +endif +# # cflags.common = C compiler flags for all builds cflags.common = -I. -I$(dir $(sqlite3.c)) -std=c99 -fPIC # emcc.WASM_BIGINT = 1 for BigInt (C int64) support, else 0. The API @@ -531,13 +590,14 @@ cflags.common = -I. -I$(dir $(sqlite3.c)) -std=c99 -fPIC # _are not tested_ on any regular basis. emcc.WASM_BIGINT ?= 1 emcc.MEMORY64 ?= 0 -######################################################################## +# # https://emscripten.org/docs/tools_reference/settings_reference.html#memory64 # # 64-bit build requires wasm-strip 1.0.36 (maybe 1.0.35, but not # 1.0.34) or will fail to strip with "tables may not be 64-bit". -######################################################################## +# +# # emcc_opt = optimization-related flags. These are primarily used by # the various oX targets. build times for -O levels higher than 0 are # painful at dev-time. @@ -549,14 +609,17 @@ emcc.MEMORY64 ?= 0 # -O2 (which consistently creates the fastest-running deliverables). # Build time suffers greatly compared to -O0, which is why -O0 is the # default. +# ifeq (,$(filter $(OPTIMIZED_TARGETS),$(MAKECMDGOALS))) emcc_opt ?= -O0 else emcc_opt ?= -Oz endif +# # When passing emcc_opt from the CLI, += and re-assignment have no # effect, so emcc_opt+=-g3 doesn't work. So... +# emcc_opt_full = $(emcc_opt) -g3 # ^^^ ALWAYS use -g3. See below for why. # @@ -583,26 +646,26 @@ emcc_opt_full = $(emcc_opt) -g3 # Much practice has demonstrated that -O2 consistently gives the best # runtime speeds, but not by a large enough factor to rule out use of # -Oz when smaller deliverable size is a priority. -######################################################################## +# -######################################################################## +# # EXPORTED_FUNCTIONS.* = files for use with Emscripten's # -sEXPORTED_FUNCTION flag. -EXPORTED_FUNCTIONS.api.core = $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-core -EXPORTED_FUNCTIONS.api.in = $(EXPORTED_FUNCTIONS.api.core) -ifeq (1,$(SQLITE_C_IS_SEE)) - EXPORTED_FUNCTIONS.api.in += $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-see -endif -ifeq (0,$(wasm-bare-bones)) - EXPORTED_FUNCTIONS.api.in += $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-extras -endif +# +EXPORTED_FUNCTIONS.api.in = $(dir.api)/EXPORTED_FUNCTIONS.c-pp EXPORTED_FUNCTIONS.api = $(dir.tmp)/EXPORTED_FUNCTIONS.api -$(EXPORTED_FUNCTIONS.api): $(EXPORTED_FUNCTIONS.api.in) $(sqlite3.c) $(MAKEFILE) - @$(call b.mkdir@) - cat $(EXPORTED_FUNCTIONS.api.in) > $@ +EXPORTED_FUNCTIONS.c-pp.flags = +ifeq (1,$(wasm-bare-bones)) + EXPORTED_FUNCTIONS.c-pp.flags += -Dbare-bones +endif +$(eval $(call b.c-pp.target,filter,\ + $(EXPORTED_FUNCTIONS.api.in),\ + $(EXPORTED_FUNCTIONS.api),\ + $(EXPORTED_FUNCTIONS.c-pp.flags))) -######################################################################## +# # emcc flags for .c/.o/.wasm/.js. +# emcc.flags = ifeq (1,$(emcc.verbose)) emcc.flags += -v @@ -690,9 +753,9 @@ ifeq (,$(emcc.INITIAL_MEMORY.$(emcc.INITIAL_MEMORY))) $(error emcc.INITIAL_MEMORY must be one of: 4, 8, 16, 32, 64, 96, 128 (megabytes)) endif emcc.jsflags += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.$(emcc.INITIAL_MEMORY)) +# # /INITIAL_MEMORY -######################################################################## -#emcc.jsflags += -sMEMORY64=$(emcc.MEMORY64) +# emcc.jsflags += $(emcc.environment) emcc.jsflags += -sSTACK_SIZE=512KB @@ -701,7 +764,8 @@ emcc.jsflags += -sSTACK_SIZE=512KB # VFS, which requires twice that for its xRead() and xWrite() methods. # 2023-03: those methods have since been adapted to use a malloc()'d # buffer. -######################################################################## + +# # $(sqlite3.js.init-func) is the name Emscripten assigns our exported # module init/load function. This symbol name is hard-coded in # $(extern-post-js.js) as well as in numerous docs. @@ -741,7 +805,7 @@ emcc.jsflags += -sLLD_REPORT_UNDEFINED #emcc.jsflags += --experimental-pic --unresolved-symbols=ingore-all --import-undefined #emcc.jsflags += --unresolved-symbols=ignore-all -######################################################################## +# # -sSINGLE_FILE: # https://github.com/emscripten-core/emscripten/blob/main/src/settings.js # @@ -750,12 +814,7 @@ emcc.jsflags += -sLLD_REPORT_UNDEFINED # cannot wasm-strip the binary before it gets encoded into the JS # file. The result is that the generated JS file is, because of the # -g3 debugging info, _huge_. -######################################################################## - - -sqlite3.wasm = $(dir.dout)/sqlite3.wasm -sqlite3-wasm.c = $(dir.api)/sqlite3-wasm.c -sqlite3-wasm.c.in = $(sqlite3-wasm.c) $(sqlite3_wasm_extra_init.c) +# # # b.call.patch-export-default is used by mkwasmbuilds.c and the @@ -798,51 +857,6 @@ if [ x1 = x$(1) ]; then \ fi endef -# -# The various -D... values used by *.c-pp.js include: -# -# -Dtarget:es6-module: for all ESM module builds -# -# -Dtarget:node: for node.js builds -# -# -Dtarget:es6-module -Dtarget:es6-bundler-friendly: intended for -# "bundler-friendly" ESM module build. These have some restrictions -# on how URL() objects are constructed in some contexts: URLs which -# refer to files which are part of this project must be referenced -# as string literals so that bundlers' static-analysis tools can -# find those files and include them in their bundles. -# -# -Dtarget:es6-module -Dtarget:node: is intended for use by node.js -# for node.js, as opposed to by node.js on behalf of a -# browser. Mixing -sENVIRONMENT=web and -sENVIRONMENT=node leads to -# ambiguity and confusion on node's part, as it's unable to -# reliably determine whether the target is a browser or node. -# -# To repeat: all node.js builds are 100% untested and unsupported. -# -######################################################################## - -# -# Inputs/outputs for the sqlite3-api.js family. -# -# sqlite3-api.jses = the list of JS files which make up -# sqlite3-api.js, in the order they need to be assembled. -sqlite3-api.jses = $(sqlite3-license-version.js) -sqlite3-api.jses += $(dir.api)/sqlite3-api-prologue.js -sqlite3-api.jses += $(dir.common)/whwasmutil.js -sqlite3-api.jses += $(dir.jacc)/jaccwabyt.js -sqlite3-api.jses += $(dir.api)/sqlite3-api-glue.c-pp.js -sqlite3-api.jses += $(sqlite3-api-build-version.js) -sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.c-pp.js -sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.c-pp.js -sqlite3-api.jses += $(dir.api)/sqlite3-vfs-helper.c-pp.js -ifeq (0,$(wasm-bare-bones)) - sqlite3-api.jses += $(dir.api)/sqlite3-vtab-helper.c-pp.js -endif -sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js -sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.c-pp.js -sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js - # # $(sqlite3-license-version.js) contains the license header and # in-comment build version info. @@ -852,17 +866,15 @@ sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js # sqlite3-license-version.js = $(dir.tmp)/sqlite3-license-version.js $(sqlite3-license-version.js): $(bin.version-info) \ - $(dir.api)/sqlite3-license-version-header.js - @echo '$(logtag.@) $(emo.disk)'; { \ - $(call b.mkdir@); \ + $(dir.api)/sqlite3-license-version-header.js $(MAKEFILE) + @$(call b.mkdir@); echo '$(logtag.@) $(emo.disk)'; { \ cat $(dir.api)/sqlite3-license-version-header.js || exit $$?; \ - echo '/*'; \ + echo '/* @preserve'; \ echo '** This code was built from sqlite3 version...'; \ echo '**'; \ awk '/define SQLITE_VERSION/{$$1=""; print "**" $$0}' $(sqlite3.h); \ awk '/define SQLITE_SOURCE_ID/{$$1=""; print "**" $$0}' $(sqlite3.h); \ echo '**'; echo '** Emscripten SDK: $(emcc.version)'; \ - echo '**'; \ echo '*/'; \ } > $@ @@ -881,6 +893,34 @@ $(sqlite3-api-build-version.js): $(bin.version-info) $(MAKEFILE) echo '});'; \ } > $@ +# +# Inputs/outputs for the sqlite3-api.js family. +# +# sqlite3-api.jses = the list of JS files which make up +# sqlite3-api.js, in the order they need to be assembled. +sqlite3-api.jses = $(sqlite3-license-version.js) +sqlite3-api.jses += $(dir.api)/sqlite3-api-prologue.js +sqlite3-api.jses += $(sqlite3-api-build-version.js) +sqlite3-api.jses += $(dir.common)/whwasmutil.js +sqlite3-api.jses += $(dir.jacc)/jaccwabyt.js +sqlite3-api.jses += $(dir.api)/sqlite3-api-glue.c-pp.js +sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.c-pp.js +sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.c-pp.js +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-helper.c-pp.js +ifeq (0,$(wasm-bare-bones)) + sqlite3-api.jses += $(dir.api)/sqlite3-vtab-helper.c-pp.js +endif +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-kvvfs.c-pp.js +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.c-pp.js + +# Parallel builds can fail if $(sqlite3-license-version.js) is not +# created early enough, so make all files in $(sqlite-api.jses) except +# for $(sqlite3-license-version.js) depend on +# $(sqlite3-license-version.js). +deps.jses = $(filter-out $(sqlite3-license-version.js),$(sqlite3-api.jses)) +$(deps.jses): $(sqlite3-license-version.js) + # # extern-post-js* and extern-pre-js* are files for use with # Emscripten's --extern-pre-js and --extern-post-js flags. @@ -918,7 +958,7 @@ $(post-js.in.js): $(MKDIR.bld) $(post-jses.js) $(MAKEFILE) done > $@ # -# speedtest1 decls needed before the $(bin.mkws)-generated makefile +# speedtest1 decls needed before the $(bin.mkwb)-generated makefile # is included. # bin.speedtest1 = ../../speedtest1 @@ -957,12 +997,21 @@ endif # /shell.c ######################################################################## +# +# Fiddle-related decls we need before .wasmbuilds is included +# + +fiddle.c.in = $(dir.top)/shell.c $(sqlite3-wasm.c) + EXPORTED_FUNCTIONS.fiddle = $(dir.tmp)/EXPORTED_FUNCTIONS.fiddle -$(EXPORTED_FUNCTIONS.fiddle): $(fiddle.EXPORTED_FUNCTIONS.in) $(MAKEFILE_LIST) - @$(b.mkdir@) - @sort -u $(fiddle.EXPORTED_FUNCTIONS.in) > $@ +$(EXPORTED_FUNCTIONS.fiddle): $(EXPORTED_FUNCTIONS.api.in) \ + $(MAKEFILE_LIST) $(bin.c-pp) + @$(call b.mkdir@) + @$(call b.c-pp.shcmd,fiddle,$(EXPORTED_FUNCTIONS.api.in),\ + $@,$(EXPORTED_FUNCTIONS.c-pp.flags) -Dfiddle) @echo $(logtag.@) $(emo.disk) + emcc.flags.fiddle = \ $(emcc.cflags) $(emcc_opt_full) \ --minify 0 \ @@ -987,26 +1036,23 @@ emcc.flags.fiddle = \ -USQLITE_WASM_BARE_BONES \ -DSQLITE_SHELL_FIDDLE -clean: clean-fiddle -clean-fiddle: - rm -f $(dir.fiddle)/fiddle-module.js \ - $(dir.fiddle)/*.wasm \ - $(dir.fiddle)/sqlite3-opfs-*.js \ - $(dir.fiddle)/*.gz \ - EXPORTED_FUNCTIONS.fiddle - rm -fr $(dir.fiddle-debug) - emcc.flags.fiddle.debug = $(emcc.flags.fiddle) \ -DSQLITE_DEBUG \ -DSQLITE_ENABLE_SELECTTRACE \ -DSQLITE_ENABLE_WHERETRACE -fiddle.EXPORTED_FUNCTIONS.in = \ - EXPORTED_FUNCTIONS.fiddle.in \ - $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-core \ - $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-extras - -fiddle.c.in = $(dir.top)/shell.c $(sqlite3-wasm.c) +clean: clean-fiddle +clean-fiddle: + rm -f $(dir.fiddle)/fiddle-module.js \ + $(dir.fiddle)/*.wasm \ + $(dir.fiddle)/sqlite3-opfs-*.js \ + $(dir.fiddle)/*.gz \ + $(dir.fiddle)/index.html \ + $(EXPORTED_FUNCTIONS.fiddle) + rm -fr $(dir.fiddle-debug) +distclean: distclean-fiddle +distclean-fiddle: + rm -fr $(dir.fiddle)/jqterm # # WASMFS build - unsupported and untested. We used WASMFS @@ -1028,6 +1074,14 @@ cflags.wasmfs = -DSQLITE_ENABLE_WASMFS # end wasmfs (the rest is in mkwasmbuilds.c) # +# +# +# +sqlite3-wasm.c = $(dir.api)/sqlite3-wasm.c +# List of input files for compiling $(sqlite3-wasm.c). That file +# #include's sqlite3.c directly, so it's implicitly includes here. +sqlite3-wasm.c.in = $(sqlite3-wasm.c) $(sqlite3_wasm_extra_init.c) + # # $(bin.mkwb) is used for generating much of the makefile code for the # various wasm builds. It used to be generated in this makefile via a @@ -1083,10 +1137,10 @@ sqlite3.ext.js = define gen-worker1 # $1 = X.ext part of sqlite3-worker1X.ext # $2 = $(c-pp.D.NAME) -$(call b.c-pp.target,filter,$(dir.api)/sqlite3-worker1.c-pp.js,\ - $(dir.dout)/sqlite3-worker1$(1),$(2)) -sqlite3.ext.js += $(dir.dout)/sqlite3-worker1$(1) -all: $(dir.dout)/sqlite3-worker1$(1) +$(call b.c-pp.target,filter,$$(dir.api)/sqlite3-worker1.c-pp.js,\ + $$(dir.dout)/sqlite3-worker1$(1),$(2)) +sqlite3.ext.js += $$(dir.dout)/sqlite3-worker1$(1) +all: $$(dir.dout)/sqlite3-worker1$(1) endef $(eval $(call gen-worker1,.js,$(c-pp.D.vanilla))) @@ -1100,10 +1154,10 @@ $(eval $(call gen-worker1,-bundler-friendly.mjs,$(c-pp.D.bundler))) define gen-promiser # $1 = X.ext part of sqlite3-worker1-promiserX.ext # $2 = $(c-pp.D.NAME) -$(call b.c-pp.target,filter,$(dir.api)/sqlite3-worker1-promiser.c-pp.js,\ - $(dir.dout)/sqlite3-worker1-promiser$(1),$(2)) -sqlite3.ext.js += $(dir.dout)/sqlite3-worker1-promiser$(1) -all: $(dir.dout)/sqlite3-worker1-promiser$(1) +$(call b.c-pp.target,filter,$$(dir.api)/sqlite3-worker1-promiser.c-pp.js,\ + $$(dir.dout)/sqlite3-worker1-promiser$(1),$(2)) +sqlite3.ext.js += $$(dir.dout)/sqlite3-worker1-promiser$(1) +all: $$(dir.dout)/sqlite3-worker1-promiser$(1) endef $(eval $(call gen-promiser,.js,$(c-pp.D.vanilla))) @@ -1145,12 +1199,19 @@ $(dir.dout)/sqlite3-opfs-async-proxy.js: $(dir.api)/sqlite3-opfs-async-proxy.js # we don't otherwise have a great place to attach them such that # they're always copied when we need them. # +# The var $(out.$(B).js) comes from $(bin.mkwb) and $(B) is the name +# of a build set up by that tool, e.g. b-vanilla or b-esm64. +# $(foreach B,$(b.names),$(eval $(out.$(B).js): $(sqlite3.ext.js))) + # # b-all: builds all available js/wasm builds. # $(foreach B,$(b.names),$(eval b-all: $(out.$(B).js))) +#$(foreach B,$(b.names),$(eval pre: $(pre-js.$(B).js))) +$(foreach B,$(b.names),$(eval post: $(post-js.$(B).js))) + # # speedtest1 is our primary benchmarking tool. # @@ -1160,7 +1221,7 @@ $(foreach B,$(b.names),$(eval b-all: $(out.$(B).js))) # These flags get applied via $(bin.mkwb). emcc.speedtest1.common = $(emcc_opt_full) emcc.speedtest1 = -I. -I$(dir $(sqlite3.canonical.c)) -emcc.speedtest1 += -sENVIRONMENT=web +emcc.speedtest1 += -sENVIRONMENT=web,worker emcc.speedtest1 += -sALLOW_MEMORY_GROWTH emcc.speedtest1 += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.32) emcc.speedtest1.common += -sINVOKE_RUN=0 @@ -1199,28 +1260,37 @@ speedtest1.exit-runtime1 = -sEXIT_RUNTIME=1 # -sEXIT_RUNTIME=1 but we need EXIT_RUNTIME=0 for the worker-based app # which runs speedtest1 multiple times. -$(EXPORTED_FUNCTIONS.speedtest1): $(EXPORTED_FUNCTIONS.api.core) - @$(call b.echo,@,$(emo.disk)); \ - $(call b.mkdir@); \ - { echo _wasm_main; cat $(EXPORTED_FUNCTIONS.api.core); } > $@ || exit +$(EXPORTED_FUNCTIONS.speedtest1): $(EXPORTED_FUNCTIONS.api) + @$(call b.mkdir@); $(call b.echo,@,$(emo.disk)); \ + { echo _wasm_main; cat $(EXPORTED_FUNCTIONS.api); } > $@ || exit speedtest1: b-speedtest1 +st: speedtest1 # # Generate 64-bit variants of speedtest1*.{js,html} # +# $1 = input file +# $2 = output file +# +# TODO: preprocess these like we do the rest. +# define gen-st64 $(2): $(1) @$$(call b.echo,speedtest164,$$(emo.disk)$(emo.lock) Creating from $$<) - @rm -f $$@; \ - sed -e 's/$(3)\.js/$(3)-64bit\.js/' < $$< > $$@; \ + rm -f $$@; \ + sed -e 's/speedtest1\.js/speedtest1-64bit\.js/' \ + -e 's/speedtest1-worker\.js/speedtest1-worker-64bit\.js/' \ + < $$< > $$@; \ chmod -w $$@ -b-speedtest164: $(2) +$(2): b-speedtest164 +speedtest1: $(1) $(2) CLEAN_FILES += $(2) endef +speedtest1: b-speedtest164 -$(eval $(call gen-st64,speedtest1.html,speedtest1-64bit.html,speedtest1)) -$(eval $(call gen-st64,speedtest1-worker.html,speedtest1-worker-64bit.html,speedtest1-worker)) -$(eval $(call gen-st64,speedtest1-worker.js,speedtest1-worker-64bit.js,speedtest1-worker)) +$(eval $(call gen-st64,speedtest1.html,speedtest1-64bit.html)) +$(eval $(call gen-st64,speedtest1-worker.html,speedtest1-worker-64bit.html)) +$(eval $(call gen-st64,speedtest1-worker.js,speedtest1-worker-64bit.js)) # end speedtest1.js ######################################################################## @@ -1244,9 +1314,11 @@ $(eval $(call gen-st64,speedtest1-worker.js,speedtest1-worker-64bit.js,speedtest # # To create those, we filter tester1.c-pp.js/html with $(bin.c-pp)... +# # tester1.js variants: +# define gen-tester1.js -# $1 = build name to have dep on +# $1 = build name to have a dep on # $2 = suffix for tester1SUFFIX JS # $3 = $(bin.c-pp) flags $(call b.c-pp.target,test,tester1.c-pp.js,tester1$(2),$(3)) @@ -1255,20 +1327,18 @@ tester1-$(1): tester1$(2) tester1: tester1$(2) endef -$(eval $(call gen-tester1.js,vanilla,.js, \ - $(c-pp.D.vanilla) \ - -Dsqlite3.js=$(dir.dout)/sqlite3.js)) -$(eval $(call gen-tester1.js,vanilla64,-64bit.js, \ - $(c-pp.D.vanilla64) \ - -Dsqlite3.js=$(dir.dout)/sqlite3-64bit.js)) -$(eval $(call gen-tester1.js,esm,.mjs, \ - $(c-pp.D.esm) \ - -Dsqlite3.js=$(dir.dout)/sqlite3.mjs)) -$(eval $(call gen-tester1.js,esm64,-64bit.mjs, \ - $(c-pp.D.esm64) \ - -Dsqlite3.js=$(dir.dout)/sqlite3-64bit.mjs)) +$(eval $(call gen-tester1.js,vanilla,.js,\ + $(c-pp.D.vanilla) -Dsqlite3.js=$(dir.dout)/sqlite3.js)) +$(eval $(call gen-tester1.js,vanilla64,-64bit.js,\ + $(c-pp.D.vanilla64) -Dsqlite3.js=$(dir.dout)/sqlite3-64bit.js)) +$(eval $(call gen-tester1.js,esm,.mjs,\ + $(c-pp.D.esm) -Dsqlite3.js=$(dir.dout)/sqlite3.mjs)) +$(eval $(call gen-tester1.js,esm64,-64bit.mjs,\ + $(c-pp.D.esm64) -Dsqlite3.js=$(dir.dout)/sqlite3-64bit.mjs)) +# # tester1.html variants: +# define gen-tester1.html # $1 = build name to have a dep on # $2 = filename suffix: empty, -64bit, -esm, esm-64bit @@ -1303,9 +1373,11 @@ $(eval $(call gen-tester1.html,esm64,-esm-64bit,\ -Dtester1.js=tester1-64bit.mjs \ -Dsqlite3.js=$(dir.dout)/sqlite3-64bit.mjs)) -# tester1-worker.html variants: -# There is no ESM variant of this file. Instead, that page accepts a -# ?esm URL flag to switch to ESM mode. +# +# tester1-worker.html variants: There is no ESM variant of this +# file. Instead, that page accepts the ?esm URL flag to switch to ESM +# mode. +# $(eval $(call b.c-pp.target,test,tester1-worker.c-pp.html,\ tester1-worker.html,-Dbitness=32)) $(eval $(call b.c-pp.target,test,tester1-worker.c-pp.html,\ @@ -1315,8 +1387,57 @@ tester1-worker.html: tester1.mjs tester1-worker-64bit.html: tester1-64bit.mjs all: tester1 +# # end tester1 -######################################################################## +# + +# +# jquery.terminal support for fiddle: +# +# If a clone of https://github.com/jcubic/jquery.terminal +# is found in $(JQTERM), defaulting to $(HOME)/src/jquery.terminal +# then add jquery.terminal support to fiddle. +# +# To build that package, from its checkout dir: +# +# npm install +# make +# +c-pp.D.fiddle ?= +JQTERM ?= $(HOME)/src/jquery.terminal +dir.jqtermExt = $(firstword $(wildcard $(JQTERM))) +#$(info dir.jqtermExt=$(dir.jqtermExt)) +ifeq (0,$(MAKING_CLEAN)) +ifeq (,$(wildcard $(dir.jqtermExt)/js/jquery.terminal.min.js)) +$(info $(emo.magic) To add jquery.terminal support to fiddle, set JQTERM=/path/to/its/built/checkout) +else +$(info $(emo.magic) jquery.terminal found in $(dir.jqtermExt) - adding it to fiddle. Make sure it is built!) + +dir.jqterm = $(dir.fiddle)/jqterm +$(dir.fiddle)/jqterm/jquery.terminal.bundle.min.js: + @$(call b.mkdir@) + cat $(dir.jqtermExt)/js/jquery-1*.min.js \ + $(dir.jqtermExt)/js/jquery.terminal.min.js > $@ + +$(dir.fiddle)/jqterm/jquery.terminal.min.css: $(dir.jqtermExt)/css/jquery.terminal.min.css + @$(call b.mkdir@) + @$(call b.cp,fiddle,$<,$(dir $@)) + +$(dir.fiddle)/index.html: $(dir.fiddle)/jqterm/jquery.terminal.bundle.min.js \ + $(dir.fiddle)/jqterm/jquery.terminal.min.css +c-pp.D.fiddle += -Djqterm +endif +endif +# ^^^ JQTERM/MAKING_CLEAN + +# +# Generate fiddle/index.html. Must come after JQTERM is handled. +# +$(dir.fiddle)/index.html: $(dir.fiddle)/index.c-pp.html +$(eval $(call b.c-pp.target,fiddle,\ + $(dir.fiddle)/index.c-pp.html,$(dir.fiddle)/index.html,$(c-pp.D.fiddle))) +$(out.fiddle.wasm): $(dir.fiddle)/index.html + # # Convenience rules to rebuild with various -Ox levels. Much @@ -1327,6 +1448,7 @@ all: tester1 # # Achtung: build times with anything higher than -O0 are somewhat # painful, which is why -O0 is the default. +# .PHONY: o0 o1 o2 o3 os oz emcc-opt-extra = #ifeq (1,$(wasm-bare-bones)) @@ -1377,7 +1499,9 @@ push-testing: ssh wasm-testing 'cd $(wasm-testing.dir) && bash .gzip' || \ echo "SSH failed: it's likely that stale content will be served via old gzip files." +# # build everything needed by push-testing with -Oz +# .PHONY: for-testing for-testing: emcc_opt=-Oz for-testing: loud=1 @@ -1404,9 +1528,9 @@ update-docs: exit 127 else wasm.docs.jswasm = $(wasm.docs.home)/jswasm -update-docs: $(bin.stripccomments) $(out.sqlite3.js) $(out.sqlite3.wasm) +update-docs: $(bin.stripccomments) $(out.vanilla.js) $(out.vanilla.wasm) @echo "Copying files to the /wasm docs. Be sure to use an -Oz build for this!"; - cp -p $(sqlite3.wasm) $(wasm.docs.jswasm)/. + cp -p $(out.vanilla.wasm) $(wasm.docs.jswasm)/. $(bin.stripccomments) -k -k < $(out.vanilla.js) \ | sed -e '/^[ \t]*$$/d' > $(wasm.docs.jswasm)/sqlite3.js cp -p demo-123.js demo-123.html demo-123-worker.html $(wasm.docs.home)/. @@ -1462,13 +1586,46 @@ endif dist-name-prefix = sqlite-wasm$(dist-name-extra) .PHONY: dist dist: - ./mkdist.sh $(dist-name-prefix) + $(bin.bash) ./mkdist.sh $(dist-name-prefix) snapshot: - ./mkdist.sh $(dist-name-prefix) --snapshot + $(bin.bash) ./mkdist.sh $(dist-name-prefix) --snapshot endif # ^^^ making dist/snapshot CLEAN_FILES += $(wildcard sqlite-wasm-*.zip) +######################################################################## +# The npm target is specifically for preparing files for the downstream +# https://github.com/sqlite/sqlite-wasm (npm) distribution. +# +# Per agreement with that project's maintainers, these filenames need +# to remain stable. To avoid breakage in their deployment process, any +# changes (like renaming files) which potentially break their deployment +# needs to be communicated to that project via opening a new ticket or +# direct coordination with its maintainers. +# +# This target does a full clean/rebuild so that we can ensure that the +# optimization level is set consistently across all files. +# +npm.bundle.zip = npm-bundle.zip +CLEAN_FILES += $(npm.bundle.zip) +# Distributables which need to be built for npm: +npm_files = $(addprefix $(dir.dout)/, \ +sqlite3-bundler-friendly.mjs \ +sqlite3-opfs-async-proxy.js \ +sqlite3-worker1-bundler-friendly.mjs \ +sqlite3-worker1-promiser.mjs \ +sqlite3-worker1.mjs \ +sqlite3.mjs \ +sqlite3.wasm \ +sqlite3-node.mjs \ +) +npm: $(sqlite3.canonical.c) + @echo "$(emo.cleanup) Forcing a clean rebuild to ensure consistent optimization flags." + $(MAKE) clean + $(MAKE) -e "emcc_opt=-Oz $(emcc-opt-extra)" $(npm_files) + rm -f $(npm.bundle.zip); zip -r $(npm.bundle.zip) $(npm_files) + unzip -l $(npm.bundle.zip) + ######################################################################## # Explanation of, and some commentary on, various emcc build flags # follows. Full docs for these can be found at: diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core b/ext/wasm/api/EXPORTED_FUNCTIONS.c-pp similarity index 61% rename from ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core rename to ext/wasm/api/EXPORTED_FUNCTIONS.c-pp index 506054510..2cdddf1e7 100644 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.c-pp @@ -75,6 +75,7 @@ _sqlite3_limit _sqlite3_malloc _sqlite3_malloc64 _sqlite3_msize +_sqlite3_next_stmt _sqlite3_open _sqlite3_open_v2 _sqlite3_overload_function @@ -155,3 +156,92 @@ _sqlite3_vtab_in_next _sqlite3_vtab_nochange _sqlite3_vtab_on_conflict _sqlite3_vtab_rhs_value +//#if not bare-bones +_sqlite3_column_database_name +_sqlite3_column_origin_name +_sqlite3_column_table_name +_sqlite3_create_module +_sqlite3_create_module_v2 +_sqlite3_create_window_function +_sqlite3_declare_vtab +_sqlite3_drop_modules +_sqlite3_preupdate_blobwrite +_sqlite3_preupdate_count +_sqlite3_preupdate_depth +_sqlite3_preupdate_hook +_sqlite3_preupdate_new +_sqlite3_preupdate_old +_sqlite3_progress_handler +_sqlite3_set_authorizer +_sqlite3_vtab_collation +_sqlite3_vtab_distinct +_sqlite3_vtab_in +_sqlite3_vtab_in_first +_sqlite3_vtab_in_next +_sqlite3_vtab_nochange +_sqlite3_vtab_on_conflict +_sqlite3_vtab_rhs_value +_sqlite3changegroup_add +_sqlite3changegroup_add_strm +_sqlite3changegroup_delete +_sqlite3changegroup_new +_sqlite3changegroup_output +_sqlite3changegroup_output_strm +_sqlite3changeset_apply +_sqlite3changeset_apply_strm +_sqlite3changeset_apply_v2 +_sqlite3changeset_apply_v2_strm +_sqlite3changeset_apply_v3 +_sqlite3changeset_apply_v3_strm +_sqlite3changeset_concat +_sqlite3changeset_concat_strm +_sqlite3changeset_conflict +_sqlite3changeset_finalize +_sqlite3changeset_fk_conflicts +_sqlite3changeset_invert +_sqlite3changeset_invert_strm +_sqlite3changeset_new +_sqlite3changeset_next +_sqlite3changeset_old +_sqlite3changeset_op +_sqlite3changeset_pk +_sqlite3changeset_start +_sqlite3changeset_start_strm +_sqlite3changeset_start_v2 +_sqlite3changeset_start_v2_strm +_sqlite3session_attach +_sqlite3session_changeset +_sqlite3session_changeset_size +_sqlite3session_changeset_strm +_sqlite3session_config +_sqlite3session_create +_sqlite3session_delete +_sqlite3session_diff +_sqlite3session_enable +_sqlite3session_indirect +_sqlite3session_isempty +_sqlite3session_memory_used +_sqlite3session_object_config +_sqlite3session_patchset +_sqlite3session_patchset_strm +_sqlite3session_table_filter +//#endif not bare-bones +//#if enable-see +_sqlite3_key +_sqlite3_key_v2 +_sqlite3_rekey +_sqlite3_rekey_v2 +_sqlite3_activate_see +//#endif enable-see +//#if fiddle +_fiddle_db_arg +_fiddle_db_filename +_fiddle_exec +_fiddle_experiment +_fiddle_interrupt +_fiddle_main +_fiddle_reset_db +_fiddle_db_handle +_fiddle_db_vfs +_fiddle_export_db +//#endif fiddle diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras deleted file mode 100644 index e8304b5f2..000000000 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras +++ /dev/null @@ -1,68 +0,0 @@ -_sqlite3_column_database_name -_sqlite3_column_origin_name -_sqlite3_column_table_name -_sqlite3_create_module -_sqlite3_create_module_v2 -_sqlite3_create_window_function -_sqlite3_declare_vtab -_sqlite3_drop_modules -_sqlite3_preupdate_blobwrite -_sqlite3_preupdate_count -_sqlite3_preupdate_depth -_sqlite3_preupdate_hook -_sqlite3_preupdate_new -_sqlite3_preupdate_old -_sqlite3_progress_handler -_sqlite3_set_authorizer -_sqlite3_vtab_collation -_sqlite3_vtab_distinct -_sqlite3_vtab_in -_sqlite3_vtab_in_first -_sqlite3_vtab_in_next -_sqlite3_vtab_nochange -_sqlite3_vtab_on_conflict -_sqlite3_vtab_rhs_value -_sqlite3changegroup_add -_sqlite3changegroup_add_strm -_sqlite3changegroup_delete -_sqlite3changegroup_new -_sqlite3changegroup_output -_sqlite3changegroup_output_strm -_sqlite3changeset_apply -_sqlite3changeset_apply_strm -_sqlite3changeset_apply_v2 -_sqlite3changeset_apply_v2_strm -_sqlite3changeset_apply_v3 -_sqlite3changeset_apply_v3_strm -_sqlite3changeset_concat -_sqlite3changeset_concat_strm -_sqlite3changeset_conflict -_sqlite3changeset_finalize -_sqlite3changeset_fk_conflicts -_sqlite3changeset_invert -_sqlite3changeset_invert_strm -_sqlite3changeset_new -_sqlite3changeset_next -_sqlite3changeset_old -_sqlite3changeset_op -_sqlite3changeset_pk -_sqlite3changeset_start -_sqlite3changeset_start_strm -_sqlite3changeset_start_v2 -_sqlite3changeset_start_v2_strm -_sqlite3session_attach -_sqlite3session_changeset -_sqlite3session_changeset_size -_sqlite3session_changeset_strm -_sqlite3session_config -_sqlite3session_create -_sqlite3session_delete -_sqlite3session_diff -_sqlite3session_enable -_sqlite3session_indirect -_sqlite3session_isempty -_sqlite3session_memory_used -_sqlite3session_object_config -_sqlite3session_patchset -_sqlite3session_patchset_strm -_sqlite3session_table_filter diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see deleted file mode 100644 index 83f3a97db..000000000 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see +++ /dev/null @@ -1,5 +0,0 @@ -_sqlite3_key -_sqlite3_key_v2 -_sqlite3_rekey -_sqlite3_rekey_v2 -_sqlite3_activate_see diff --git a/ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api b/ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api deleted file mode 100644 index aab1d8bd3..000000000 --- a/ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api +++ /dev/null @@ -1,3 +0,0 @@ -FS -wasmMemory - diff --git a/ext/wasm/api/README.md b/ext/wasm/api/README.md index 3c9669e6b..279d216bf 100644 --- a/ext/wasm/api/README.md +++ b/ext/wasm/api/README.md @@ -23,7 +23,7 @@ this writing, but is not set in stone forever and may change at any time. This doc targets maintainers of this code and those wanting to dive in to the details, not end user. -First off, a pikchr of the proverbial onion: +First off, a [pikchr][] of the proverbial onion: ```pikchr toggle center scale = 0.85 @@ -60,14 +60,14 @@ maintenance point of view. At the center of the onion is `sqlite3-api.js`, which gets generated by concatenating the following files together in their listed order: -- **`sqlite3-api-prologue.js`**\ +- **`sqlite3-api-prologue.js`** Contains the initial bootstrap setup of the sqlite3 API - objects. This is exposed as a function, rather than objects, so that + objects. This is exposed as a bootstrapping function so that the next step can pass in a config object which abstracts away parts of the WASM environment, to facilitate plugging it in to arbitrary WASM toolchains. The bootstrapping function gets removed from the global scope in a later stage of the bootstrapping process. -- **`../common/whwasmutil.js`**\ +- **`../common/whwasmutil.js`** A semi-third-party collection of JS/WASM utility code intended to replace much of the Emscripten glue. The sqlite3 APIs internally use these APIs instead of their Emscripten counterparts, in order to be @@ -77,79 +77,78 @@ by concatenating the following files together in their listed order: toolchains. It is "semi-third-party" in that it was created in order to support this tree but is standalone and maintained together with... -- **`../jaccwabyt/jaccwabyt.js`**\ +- **`../jaccwabyt/jaccwabyt.js`** Another semi-third-party API which creates bindings between JS and C structs, such that changes to the struct state from either JS or C are visible to the other end of the connection. This is also an independent spinoff project, conceived for the sqlite3 project but maintained separately. -- **`sqlite3-api-glue.js`**\ +- **`sqlite3-api-glue.js`** Invokes functionality exposed by the previous two files to flesh out low-level parts of `sqlite3-api-prologue.js`. Most of these pieces involve populating the `sqlite3.capi.wasm` object and creating `sqlite3.capi.sqlite3_...()` bindings. This file also deletes most global-scope symbols the above files create, effectively moving them into the scope being used for initializing the API. -- **`/sqlite3-api-build-version.js`**\ +- **`/sqlite3-api-build-version.js`** Gets created by the build process and populates the `sqlite3.version` object. This part is not critical, but records the version of the library against which this module was built. -- **`sqlite3-api-oo1.js`**\ +- **`sqlite3-api-oo1.js`** Provides a high-level object-oriented wrapper to the lower-level C API, colloquially known as OO API #1. Its API is similar to other high-level sqlite3 JS wrappers and should feel relatively familiar to anyone familiar with such APIs. It is not a "required component" and can be elided from builds which do not want it. -- **`sqlite3-api-worker1.js`**\ +- **`sqlite3-api-worker1.js`** A Worker-thread-based API which uses OO API #1 to provide an interface to a database which can be driven from the main Window thread via the Worker message-passing interface. Like OO API #1, this is an optional component, offering one of any number of potential implementations for such an API. - - **`sqlite3-worker1.js`**\ + - **`sqlite3-worker1.js`** Is not part of the amalgamated sources and is intended to be loaded by a client Worker thread. It loads the sqlite3 module and runs the Worker #1 API which is implemented in `sqlite3-api-worker1.js`. - - **`sqlite3-worker1-promiser.js`**\ + - **`sqlite3-worker1-promiser.js`** Is likewise not part of the amalgamated sources and provides a Promise-based interface into the Worker #1 API. This is a far user-friendlier way to interface with databases running in a Worker thread. -- **`sqlite3-vfs-helper.js`**\ +- **`sqlite3-vfs-helper.c-pp.js`** Installs the `sqlite3.vfs` namespace, which contain helpers for use by downstream code which creates `sqlite3_vfs` implementations. -- **`sqlite3-vtab-helper.js`**\ +- **`sqlite3-vtab-helper.c-pp.js`** Installs the `sqlite3.vtab` namespace, which contain helpers for use by downstream code which creates `sqlite3_module` implementations. -- **`sqlite3-vfs-opfs.c-pp.js`**\ +- **`sqlite3-vfs-opfs.c-pp.js`** is an sqlite3 VFS implementation which supports the [Origin-Private FileSystem (OPFS)][OPFS] as a storage layer to provide persistent storage for database files in a browser. It requires... - - **`sqlite3-opfs-async-proxy.js`**\ + - **`sqlite3-opfs-async-proxy.js`** is the asynchronous backend part of the [OPFS][] proxy. It speaks directly to the (async) OPFS API and channels those results back to its synchronous counterpart. This file, because it must be started in its own Worker, is not part of the amalgamation. -- **`sqlite3-vfs-opfs-sahpool.c-pp.js`**\ +- **`sqlite3-vfs-opfs-sahpool.c-pp.js`** is another sqlite3 VFS supporting the [OPFS][], but uses a completely different approach than the above-listed one. -- **`sqlite3-api-cleanup.js`**\ - The previous files do not immediately extend the library. Instead - they add callback functions to be called during its - bootstrapping. Some also temporarily create global objects in order - to communicate their state to the files which follow them. This file - cleans up any dangling globals and runs the API bootstrapping - process, which is what finally executes the initialization code - installed by the previous files. As of this writing, this code - ensures that the previous files leave no more than a single global - symbol installed - `sqlite3InitModule()`. When adapting the API for - non-Emscripten toolchains, this "should" be the only file, of those - in this list, where changes are needed. The Emscripten-specific - pieces described below may also require counterparts in any as-yet - hypothetical alternative build. +The previous files do not immediately extend the library. Instead they +install a global function `sqlite3ApiBootstrap()`, which downstream +code must call to configure the library for the current JS/WASM +environment. Each file listed above pushes a callback into the +bootstrapping queue, to be called as part of `sqlite3ApiBootstrap()`. +Some files also temporarily create global objects in order to +communicate their state to the files which follow them. Those +get cleaned up vi `post-js-footer.js`, described below. + +Adapting the build for non-Emscripten toolchains essentially requires packaging +the above files, concatated together, into that toolchain's "JS glue" +and, in the final stage of that glue, call `sqlite3ApiBootstrap()` and +return its result to the end user. **Files with the extension `.c-pp.js`** are intended [to be processed with `c-pp`](#c-pp), noting that such preprocessing may be applied @@ -171,23 +170,27 @@ from this file rather than `sqlite3.c`. The following Emscripten-specific files are injected into the build-generated `sqlite3.js` along with `sqlite3-api.js`. -- **`extern-pre-js.js`**\ +- **`extern-pre-js.js`** Emscripten-specific header for Emscripten's `--extern-pre-js` flag. As of this writing, that file is only used for experimentation purposes and holds no code relevant to the production deliverables. -- **`pre-js.c-pp.js`**\ +- **`pre-js.c-pp.js`** Emscripten-specific header for Emscripten's `--pre-js` flag. This file overrides certain Emscripten behavior before Emscripten does most of its work. -- **`post-js-header.js`**\ +- **`post-js-header.js`** Emscripten-specific header for the `--post-js` input. It opens up, but does not close, a function used for initializing the library. -- (**`sqlite3-api.js`** gets sandwiched between these ↑ two - ↓ files.) -- **`post-js-footer.js`**\ +- **`sqlite3-api.js`** gets sandwiched between these ↑ two + ↓ files. +- **`post-js-footer.js`** Emscripten-specific footer for the `--post-js` input. This closes - off the function opened by `post-js-header.js`. -- **`extern-post-js.c-pp.js`**\ + off the function opened by `post-js-header.js`. This file cleans up + any dangling globals and runs `sqlite3ApiBootstrap()`. As of this + writing, this code ensures that the previous files leave no more + than a single global symbol installed - `sqlite3InitModule()`. + +- **`extern-post-js.c-pp.js`** Emscripten-specific header for Emscripten's `--extern-post-js` flag. This file is run in the global scope. It overwrites the Emscripten-installed `sqlite3InitModule()` function with one which @@ -205,9 +208,10 @@ Preprocessing of Source Files Certain files in the build require preprocessing to filter in/out parts which differ between vanilla JS, ES6 Modules, and node.js builds. The preprocessor application itself is in -[`c-pp.c`](/file/ext/wasm/c-pp.c) and the complete technical details -of such preprocessing are maintained in +[`c-pp-lite.c`](/file/ext/wasm/c-pp-lite.c) and the complete technical +details of such preprocessing are maintained in [`GNUMakefile`](/file/ext/wasm/GNUmakefile). [OPFS]: https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system +[pikchr]: https://pikchr.org diff --git a/ext/wasm/api/extern-post-js.c-pp.js b/ext/wasm/api/extern-post-js.c-pp.js index 606e02ae2..cac6e4ab4 100644 --- a/ext/wasm/api/extern-post-js.c-pp.js +++ b/ext/wasm/api/extern-post-js.c-pp.js @@ -26,7 +26,7 @@ const toExportForESM = */ const originalInit = sqlite3InitModule; if(!originalInit){ - throw new Error("Expecting globalThis.sqlite3InitModule to be defined by the Emscripten build."); + throw new Error("Expecting sqlite3InitModule to be defined by the Emscripten build."); } /** We need to add some state which our custom Module.locateFile() @@ -73,6 +73,8 @@ const toExportForESM = const sIM = globalThis.sqlite3InitModule = function ff(...args){ //console.warn("Using replaced sqlite3InitModule()",globalThis.location); + sIMS.emscriptenLocateFile = args[0]?.locateFile /* see pre-js.c-pp.js [tag:locateFile] */; + sIMS.emscriptenInstantiateWasm = args[0]?.instantiateWasm /* see pre-js.c-pp.js [tag:locateFile] */; return originalInit(...args).then((EmscriptenModule)=>{ sIMS.debugModule("sqlite3InitModule() sIMS =",sIMS); sIMS.debugModule("sqlite3InitModule() EmscriptenModule =",EmscriptenModule); diff --git a/ext/wasm/api/post-js-footer.js b/ext/wasm/api/post-js-footer.js index c6a2e1517..f8050ddd3 100644 --- a/ext/wasm/api/post-js-footer.js +++ b/ext/wasm/api/post-js-footer.js @@ -1,3 +1,72 @@ +/* + 2022-07-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file is the tail end of the sqlite3-api.js constellation, + closing the function scope opened by post-js-header.js. + + In terms of amalgamation code placement, this file is appended + immediately after the final sqlite3-api-*.js piece. Those files + cooperate to prepare sqlite3ApiBootstrap() and this file calls it. + It is run within a context which gives it access to Emscripten's + Module object, after sqlite3.wasm is loaded but before + sqlite3ApiBootstrap() has been called. + + Because this code resides (after building) inside the function + installed by post-js-header.js, it has access to state set up by + pre-js.c-pp.js and friends. +*/ +try{ + /* We are in the closing block of Module.runSQLite3PostLoadInit(), so + its arguments are visible here. */ + + /* Config options for sqlite3ApiBootstrap(). */ + const bootstrapConfig = Object.assign( + Object.create(null), + /** The WASM-environment-dependent configuration for sqlite3ApiBootstrap() */ + { + memory: ('undefined'!==typeof wasmMemory) + ? wasmMemory + : EmscriptenModule['wasmMemory'], + exports: ('undefined'!==typeof wasmExports) + ? wasmExports /* emscripten >=3.1.44 */ + : (Object.prototype.hasOwnProperty.call(EmscriptenModule,'wasmExports') + ? EmscriptenModule['wasmExports'] + : EmscriptenModule['asm']/* emscripten <=3.1.43 */) + }, + globalThis.sqlite3ApiBootstrap.defaultConfig, // default options + globalThis.sqlite3ApiConfig || {} // optional client-provided options + ); + + sqlite3InitScriptInfo.debugModule("Bootstrapping lib config", bootstrapConfig); + + /** + For purposes of the Emscripten build, call sqlite3ApiBootstrap(). + Ideally clients should be able to inject their own config here, + but that's not practical in this particular build constellation + because of the order everything happens in. Clients may either + define globalThis.sqlite3ApiConfig or modify + globalThis.sqlite3ApiBootstrap.defaultConfig to tweak the default + configuration used by a no-args call to sqlite3ApiBootstrap(), + but must have first loaded their WASM module in order to be able + to provide the necessary configuration state. + */ + const p = globalThis.sqlite3ApiBootstrap(bootstrapConfig); + delete globalThis.sqlite3ApiBootstrap; + return p /* the eventual result of globalThis.sqlite3InitModule() */; +}catch(e){ + console.error("sqlite3ApiBootstrap() error:",e); + throw e; +} + //console.warn("This is the end of the Module.runSQLite3PostLoadInit handler."); }/*Module.runSQLite3PostLoadInit(...)*/; //console.warn("This is the end of the setup of the (pending) Module.runSQLite3PostLoadInit"); diff --git a/ext/wasm/api/post-js-header.js b/ext/wasm/api/post-js-header.js index cdc6b3a38..670051bd8 100644 --- a/ext/wasm/api/post-js-header.js +++ b/ext/wasm/api/post-js-header.js @@ -3,14 +3,13 @@ post-js.js for use with Emscripten's --post-js flag, so it gets injected in the earliest stages of sqlite3InitModule(). - This function wraps the whole SQLite3 library but does not - bootstrap it. - Running this function will bootstrap the library and return a Promise to the sqlite3 namespace object. + + In the canonical builds, this gets called by extern-post-js.c-pp.js */ -Module.runSQLite3PostLoadInit = function( - sqlite3InitScriptInfo /* populated by extern-post-js.c-pp.js */, +Module.runSQLite3PostLoadInit = async function( + sqlite3InitScriptInfo, EmscriptenModule/*the Emscripten-style module object*/, sqlite3IsUnderTest ){ @@ -35,7 +34,6 @@ Module.runSQLite3PostLoadInit = function( - sqlite3-vtab-helper.c-pp.js => Utilities for virtual table impls - sqlite3-vfs-opfs.c-pp.js => OPFS VFS - sqlite3-vfs-opfs-sahpool.c-pp.js => OPFS SAHPool VFS - - sqlite3-api-cleanup.js => final bootstrapping phase - post-js-footer.js => this file's epilogue And all of that gets sandwiched between extern-pre-js.js and diff --git a/ext/wasm/api/pre-js.c-pp.js b/ext/wasm/api/pre-js.c-pp.js index 8a4a0f9fd..3910cb000 100644 --- a/ext/wasm/api/pre-js.c-pp.js +++ b/ext/wasm/api/pre-js.c-pp.js @@ -14,12 +14,36 @@ itself. i.e. try to keep file-local symbol names obnoxiously collision-resistant. */ +/** + This file was preprocessed using: + +//#@policy error + @c-pp::argv@ +//#@policy off +*/ +//#if unsupported-build +/** + UNSUPPORTED BUILD: + + This SQLite JS build configuration is entirely unsupported! It has + not been tested beyond the ability to compile it. It may not + load. It may not work properly. Only builds _directly_ targeting + browser environments ("vanilla" JS and ESM modules) are supported + and tested. Builds which _indirectly_ target browsers (namely + bundler-friendly builds) are not supported deliverables. +*/ +//#endif +//#if not target:es6-bundler-friendly (function(Module){ const sIMS = globalThis.sqlite3InitModuleState/*from extern-post-js.c-pp.js*/ || Object.assign(Object.create(null),{ - debugModule: ()=>{ - console.warn("globalThis.sqlite3InitModuleState is missing"); + /* In WASMFS builds this file gets loaded once per thread, + but sqlite3InitModuleState is not getting set for the + worker threads? That those workers seem to function fine + despite that is curious. */ + debugModule: function(){ + console.warn("globalThis.sqlite3InitModuleState is missing",arguments); } }); delete globalThis.sqlite3InitModuleState; @@ -47,6 +71,14 @@ approach. */ Module['locateFile'] = function(path, prefix) { + if( this.emscriptenLocateFile instanceof Function ){ + /* [tag:locateFile] Client-overridden impl. We do not support + this but offer it as a back-door which will go away the + moment either Emscripten changes that interface or we manage + to get non-Emscripten builds working. + https://sqlite.org/forum/forumpost/1eec339854c935bd */ + return this.emscriptenLocateFile(path, prefix); + } //#if target:es6-module return new URL(path, import.meta.url).href; //#else @@ -72,8 +104,7 @@ //#endif target:es6-module }.bind(sIMS); -//#if Module.instantiateWasm -//#if not wasmfs +//#if Module.instantiateWasm and not wasmfs and not target:node /** Override Module.instantiateWasm(). @@ -82,8 +113,15 @@ https://github.com/emscripten-core/emscripten/issues/17951 In such builds we must disable this. + + It's disabled in the (unsupported/untested) node builds because + node does not do fetch(). */ Module['instantiateWasm'] = function callee(imports,onSuccess){ + if( this.emscriptenInstantiateWasm instanceof Function ){ + /* See [tag:locateFile]. Same story here */ + return this.emscriptenInstantiateWasm(imports, onSuccess); + } const sims = this; const uri = Module.locateFile( sims.wasmFilename, ( @@ -109,7 +147,7 @@ .then(finalThen) return loadWasm(); }.bind(sIMS); -//#endif not wasmfs -//#endif Module.instantiateWasm +//#endif Module.instantiateWasm and not wasmfs })(Module); +//#endif not target:es6-bundler-friendly /* END FILE: api/pre-js.js. */ diff --git a/ext/wasm/api/sqlite3-api-cleanup.js b/ext/wasm/api/sqlite3-api-cleanup.js deleted file mode 100644 index 223566326..000000000 --- a/ext/wasm/api/sqlite3-api-cleanup.js +++ /dev/null @@ -1,83 +0,0 @@ -/* - 2022-07-22 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - This file is the tail end of the sqlite3-api.js constellation, - intended to be appended after all other sqlite3-api-*.js files so - that it can finalize any setup and clean up any global symbols - temporarily used for setting up the API's various subsystems. - - In Emscripten builds it's run in the context of what amounts to a - Module.postRun handler, though it's no longer actually a postRun - handler because Emscripten 4.0 changed postRun semantics in an - incompatible way. - - In terms of amalgamation code placement, this file is appended - immediately after the final sqlite3-api-*.js piece. Those files - cooperate to prepare sqlite3ApiBootstrap() and this file calls it. - It is run within a context which gives it access to Emscripten's - Module object, after sqlite3.wasm is loaded but before - sqlite3ApiBootstrap() has been called. - - Because this code resides (after building) inside the function - installed by post-js-header.js, it has access to the -*/ -'use strict'; -if( 'undefined' === typeof EmscriptenModule/*from post-js-header.js*/ ){ - console.warn("This is not running in the context of Module.runSQLite3PostLoadInit()"); - throw new Error("sqlite3-api-cleanup.js expects to be running in the "+ - "context of its Emscripten module loader."); -} -try{ - /* Config options for sqlite3ApiBootstrap(). */ - const bootstrapConfig = Object.assign( - Object.create(null), - globalThis.sqlite3ApiBootstrap.defaultConfig, // default options - globalThis.sqlite3ApiConfig || {}, // optional client-provided options - /** The WASM-environment-dependent configuration for sqlite3ApiBootstrap() */ - { - memory: ('undefined'!==typeof wasmMemory) - ? wasmMemory - : EmscriptenModule['wasmMemory'], - exports: ('undefined'!==typeof wasmExports) - ? wasmExports /* emscripten >=3.1.44 */ - : (Object.prototype.hasOwnProperty.call(EmscriptenModule,'wasmExports') - ? EmscriptenModule['wasmExports'] - : EmscriptenModule['asm']/* emscripten <=3.1.43 */) - } - ); - - /** Figure out if this is a 32- or 64-bit WASM build. */ - bootstrapConfig.wasmPtrIR = - 'number'===(typeof bootstrapConfig.exports.sqlite3_libversion()) - ? 'i32' :'i64'; - const sIMS = sqlite3InitScriptInfo; - sIMS.debugModule("Bootstrapping lib config", sIMS); - - /** - For purposes of the Emscripten build, call sqlite3ApiBootstrap(). - Ideally clients should be able to inject their own config here, - but that's not practical in this particular build constellation - because of the order everything happens in. Clients may either - define globalThis.sqlite3ApiConfig or modify - globalThis.sqlite3ApiBootstrap.defaultConfig to tweak the default - configuration used by a no-args call to sqlite3ApiBootstrap(), - but must have first loaded their WASM module in order to be able - to provide the necessary configuration state. - */ - const p = globalThis.sqlite3ApiBootstrap(bootstrapConfig); - delete globalThis.sqlite3ApiBootstrap; - return p /* the eventual result of globalThis.sqlite3InitModule() */; -}catch(e){ - console.error("sqlite3ApiBootstrap() error:",e); - throw e; -} -throw new Error("Maintenance required: this line should never be reached"); diff --git a/ext/wasm/api/sqlite3-api-glue.c-pp.js b/ext/wasm/api/sqlite3-api-glue.c-pp.js index 1c42b0150..d268331a3 100644 --- a/ext/wasm/api/sqlite3-api-glue.c-pp.js +++ b/ext/wasm/api/sqlite3-api-glue.c-pp.js @@ -120,7 +120,11 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_column_double","f64", "sqlite3_stmt*", "int"], ["sqlite3_column_int","int", "sqlite3_stmt*", "int"], ["sqlite3_column_name","string", "sqlite3_stmt*", "int"], +//#define proxy-text-apis=1 +//#if not proxy-text-apis +/* Search this file for tag:proxy-text-apis to see what this is about. */ ["sqlite3_column_text","string", "sqlite3_stmt*", "int"], +//#endif ["sqlite3_column_type","int", "sqlite3_stmt*", "int"], ["sqlite3_column_value","sqlite3_value*", "sqlite3_stmt*", "int"], ["sqlite3_commit_hook", "void*", [ @@ -196,6 +200,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_libversion_number", "int"], ["sqlite3_limit", "int", ["sqlite3*", "int", "int"]], ["sqlite3_malloc", "*","int"], + ["sqlite3_next_stmt", "sqlite3_stmt*", ["sqlite3*","sqlite3_stmt*"]], ["sqlite3_open", "int", "string", "*"], ["sqlite3_open_v2", "int", "string", "*", "int", "string"], /* sqlite3_prepare_v2() and sqlite3_prepare_v3() are handled @@ -317,7 +322,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_value_numeric_type", "int", "sqlite3_value*"], ["sqlite3_value_pointer", "*", "sqlite3_value*", "string:static"], ["sqlite3_value_subtype", "int", "sqlite3_value*"], +//#if not proxy-text-apis ["sqlite3_value_text", "string", "sqlite3_value*"], +//#endif ["sqlite3_value_type", "int", "sqlite3_value*"], ["sqlite3_vfs_find", "*", "string"], ["sqlite3_vfs_register", "int", "sqlite3_vfs*", "int"], @@ -969,10 +976,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ "entry SQLITE_WASM_DEALLOC (=="+capi.SQLITE_WASM_DEALLOC+")."); } const __rcMap = Object.create(null); - for(const t of ['resultCodes']){ - for(const e of Object.entries(wasm.ctype[t])){ - __rcMap[e[1]] = e[0]; - } + for(const e of Object.entries(wasm.ctype['resultCodes'])){ + __rcMap[e[1]] = e[0]; } /** For the given integer, returns the SQLITE_xxx result code as a @@ -984,8 +989,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const notThese = Object.assign(Object.create(null),{ // For each struct to NOT register, map its name to true: WasmTestStruct: true, - /* We unregister the kvvfs VFS from Worker threads below. */ - sqlite3_kvvfs_methods: !util.isUIThread(), /* sqlite3_index_info and friends require int64: */ sqlite3_index_info: !wasm.bigIntEnabled, sqlite3_index_constraint: !wasm.bigIntEnabled, @@ -1654,6 +1657,45 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }/*sqlite3_bind_text/blob()*/ +//#if proxy-text-apis + if(!capi.sqlite3_column_text){ + /*[tag:proxy-text-apis] + As discussed at: + + https://sqlite.org/forum/forumpost/d77281aec2df9ada + + Summary: there are opinions that sqlite3_column_text() and + sqlite3_value_text() should handle strings such that embedded + NULs are retained. This block does that. This block does _not_ + apply that special-case behavior to any number of _other_ + APIs which return C-strings. That discrepancy makes this + block highly arguable, but one can also argue that these two + specific functions can get away with such acrobatics without + it being called voodoo in a pejorative sense. + */ + const argStmt = wasm.xWrap.argAdapter('sqlite3_stmt*'), + argInt = wasm.xWrap.argAdapter('int'), + argValue = wasm.xWrap.argAdapter('sqlite3_value*'), + newStr = + (cstr,n)=>wasm.typedArrayToString(wasm.heap8u(), + Number(cstr), Number(cstr)+n) + capi.sqlite3_column_text = function(stmt, colIndex){ + const a0 = argStmt(stmt), a1 = argInt(colIndex); + const cstr = wasm.exports.sqlite3_column_text(a0, a1); + return cstr + ? newStr(cstr,wasm.exports.sqlite3_column_bytes(a0, a1)) + : null; + }; + capi.sqlite3_value_text = function(val){ + const a0 = argValue(val); + const cstr = wasm.exports.sqlite3_value_text(a0); + return cstr + ? newStr(cstr,wasm.exports.sqlite3_value_bytes(a0)) + : null; + }; + }/*text-return-related bindings*/ +//#endif proxy-text-apis + {/* sqlite3_config() */ /** Wraps a small subset of the C API's sqlite3_config() options. @@ -1740,105 +1782,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }; }/* auto-extension */ - const pKvvfs = capi.sqlite3_vfs_find("kvvfs"); - if( pKvvfs ){/* kvvfs-specific glue */ - if(util.isUIThread()){ - const kvvfsMethods = new capi.sqlite3_kvvfs_methods( - wasm.exports.sqlite3__wasm_kvvfs_methods() - ); - delete capi.sqlite3_kvvfs_methods; - - const kvvfsMakeKey = wasm.exports.sqlite3__wasm_kvvfsMakeKeyOnPstack, - pstack = wasm.pstack; - - const kvvfsStorage = (zClass)=> - ((115/*=='s'*/===wasm.peek(zClass)) - ? sessionStorage : localStorage); - - /** - Implementations for members of the object referred to by - sqlite3__wasm_kvvfs_methods(). We swap out the native - implementations with these, which use localStorage or - sessionStorage for their backing store. - */ - const kvvfsImpls = { - xRead: (zClass, zKey, zBuf, nBuf)=>{ - const stack = pstack.pointer, - astack = wasm.scopedAllocPush(); - try { - const zXKey = kvvfsMakeKey(zClass,zKey); - if(!zXKey) return -3/*OOM*/; - const jKey = wasm.cstrToJs(zXKey); - const jV = kvvfsStorage(zClass).getItem(jKey); - if(!jV) return -1; - const nV = jV.length /* We are relying 100% on v being - ASCII so that jV.length is equal - to the C-string's byte length. */; - if(nBuf<=0) return nV; - else if(1===nBuf){ - wasm.poke(zBuf, 0); - return nV; - } - const zV = wasm.scopedAllocCString(jV); - if(nBuf > nV + 1) nBuf = nV + 1; - wasm.heap8u().copyWithin( - Number(zBuf), Number(zV), wasm.ptr.addn(zV, nBuf,- 1) - ); - wasm.poke(wasm.ptr.add(zBuf, nBuf, -1), 0); - return nBuf - 1; - }catch(e){ - sqlite3.config.error("kvstorageRead()",e); - return -2; - }finally{ - pstack.restore(stack); - wasm.scopedAllocPop(astack); - } - }, - xWrite: (zClass, zKey, zData)=>{ - const stack = pstack.pointer; - try { - const zXKey = kvvfsMakeKey(zClass,zKey); - if(!zXKey) return 1/*OOM*/; - const jKey = wasm.cstrToJs(zXKey); - kvvfsStorage(zClass).setItem(jKey, wasm.cstrToJs(zData)); - return 0; - }catch(e){ - sqlite3.config.error("kvstorageWrite()",e); - return capi.SQLITE_IOERR; - }finally{ - pstack.restore(stack); - } - }, - xDelete: (zClass, zKey)=>{ - const stack = pstack.pointer; - try { - const zXKey = kvvfsMakeKey(zClass,zKey); - if(!zXKey) return 1/*OOM*/; - kvvfsStorage(zClass).removeItem(wasm.cstrToJs(zXKey)); - return 0; - }catch(e){ - sqlite3.config.error("kvstorageDelete()",e); - return capi.SQLITE_IOERR; - }finally{ - pstack.restore(stack); - } - } - }/*kvvfsImpls*/; - for(const k of Object.keys(kvvfsImpls)){ - kvvfsMethods[kvvfsMethods.memberKey(k)] = - wasm.installFunction( - kvvfsMethods.memberSignature(k), - kvvfsImpls[k] - ); - } - }else{ - /* Worker thread: unregister kvvfs to avoid it being used - for anything other than local/sessionStorage. It "can" - be used that way but it's not really intended to be. */ - capi.sqlite3_vfs_unregister(pKvvfs); - } - }/*pKvvfs*/ - /* Warn if client-level code makes use of FuncPtrAdapter. */ wasm.xWrap.FuncPtrAdapter.warnOnUse = true; @@ -1944,7 +1887,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } tgt[memKey] = fProxy; }else{ - const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name)); + const pFunc = wasm.installFunction(fProxy, sigN); tgt[memKey] = pFunc; if(!tgt.ondispose || !tgt.ondispose.__removeFuncList){ tgt.addOnDispose('ondispose.__removeFuncList handler', diff --git a/ext/wasm/api/sqlite3-api-oo1.c-pp.js b/ext/wasm/api/sqlite3-api-oo1.c-pp.js index f7a4e9ebd..9338eef33 100644 --- a/ext/wasm/api/sqlite3-api-oo1.c-pp.js +++ b/ext/wasm/api/sqlite3-api-oo1.c-pp.js @@ -26,6 +26,20 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ the sqlite3 binding if, e.g., the wrapper is in the main thread and the sqlite3 API is in a worker. */ + const outWrapper = function(f){ + return (...args)=>f("sqlite3.oo1:",...args); + }; + + const debug = sqlite3.__isUnderTest + ? outWrapper(console.debug.bind(console)) + : outWrapper(sqlite3.config.debug); + const warn = sqlite3.__isUnderTest + ? outWrapper(console.warn.bind(console)) + : outWrapper(sqlite3.config.warn); + const error = sqlite3.__isUnderTest + ? outWrapper(console.error.bind(console)) + : outWrapper(sqlite3.config.error); + /** In order to keep clients from manipulating, perhaps inadvertently, the underlying pointer values of DB and Stmt @@ -88,7 +102,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ wasm.installFunction('i(ippp)', function(t,c,p,x){ if(capi.SQLITE_TRACE_STMT===t){ // x == SQL, p == sqlite3_stmt* - console.log("SQL TRACE #"+(++this.counter)+' via sqlite3@'+c+':', + console.log("SQL TRACE #"+(++this.counter), + 'via sqlite3@'+c+'['+capi.sqlite3_db_filename(c,null)+']', wasm.cstrToJs(x)); } }.bind({counter: 0})); @@ -213,41 +228,18 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ properties: - `.filename`: the db filename. It may be a special name like ":memory:" - or "". + or "". It may also be a URI-style name. - `.flags`: as documented in the DB constructor. - `.vfs`: as documented in the DB constructor. It also accepts those as the first 3 arguments. + + In non-default builds it may accept additional configuration + options. */ const dbCtorHelper = function ctor(...args){ - if(!ctor._name2vfs){ - /** - Map special filenames which we handle here (instead of in C) - to some helpful metadata... - - As of 2022-09-20, the C API supports the names :localStorage: - and :sessionStorage: for kvvfs. However, C code cannot - determine (without embedded JS code, e.g. via Emscripten's - EM_JS()) whether the kvvfs is legal in the current browser - context (namely the main UI thread). In order to help client - code fail early on, instead of it being delayed until they - try to read or write a kvvfs-backed db, we'll check for those - names here and throw if they're not legal in the current - context. - */ - ctor._name2vfs = Object.create(null); - const isWorkerThread = ('function'===typeof importScripts/*===running in worker thread*/) - ? (n)=>toss3("The VFS for",n,"is only available in the main window thread.") - : false; - ctor._name2vfs[':localStorage:'] = { - vfs: 'kvvfs', filename: isWorkerThread || (()=>'local') - }; - ctor._name2vfs[':sessionStorage:'] = { - vfs: 'kvvfs', filename: isWorkerThread || (()=>'session') - }; - } const opt = ctor.normalizeArgs(...args); //sqlite3.config.debug("DB ctor",opt); let pDb; @@ -269,12 +261,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ sqlite3.config.error("Invalid DB ctor args",opt,arguments); toss3("Invalid arguments for DB constructor:", arguments, "opts:", opt); } - let fnJs = wasm.isPtr(fn) ? wasm.cstrToJs(fn) : fn; - const vfsCheck = ctor._name2vfs[fnJs]; - if(vfsCheck){ - vfsName = vfsCheck.vfs; - fn = fnJs = vfsCheck.filename(fnJs); - } let oflags = 0; if( flagsStr.indexOf('c')>=0 ){ oflags |= capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE; @@ -299,7 +285,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }finally{ wasm.pstack.restore(stack); } - this.filename = fnJs; + this.filename = + /* A poor design choice we have to keep: this.filename may be + in the form "file:....?....". It really should have been + sqlite3_db_filename(pDb) but that discrepancy went too long + unnoticed to be able to change without risk of + breakage. DB.dbFilename() can be used to fetch _just_ the + name part. + */ wasm.isPtr(fn) ? wasm.cstrToJs(fn) : fn; + } __ptrMap.set(this, pDb); __stmtMap.set(this, Object.create(null)); @@ -390,12 +384,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ The given db filename must be resolvable using whatever filesystem layer (virtual or otherwise) is set up for the default - sqlite3 VFS. + sqlite3 VFS or a VFS which can resolve it must be specified. - Note that the special sqlite3 db names ":memory:" and "" - (temporary db) have their normal special meanings here and need - not resolve to real filenames, but "" uses an on-storage - temporary database and requires that the VFS support that. + The special sqlite3 db names ":memory:" and "" (temporary db) + have their normal special meanings here and need not resolve to + real filenames, but "" uses an on-storage temporary database and + requires that the VFS support that. The second argument specifies the open/create mode for the database. It must be string containing a sequence of letters (in @@ -808,6 +802,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ sqlite3_db_filename() value for the given database name, defaulting to "main". The argument may be either a JS string or a pointer to a WASM-allocated C-string. + + this.filename may be in the form of a URI-style string, whereas + the returned string contains only the filename part. */ dbFilename: function(dbName='main'){ return capi.sqlite3_db_filename(affirmDbOpen(this).pointer, dbName); @@ -929,15 +926,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ result set, but only if that statement has any result rows. The callback's "this" is the options object, noting that this function synthesizes one if the caller does not pass one to - exec(). The second argument passed to the callback is always - the current Stmt object, as it's needed if the caller wants to - fetch the column names or some such (noting that they could - also be fetched via `this.columnNames`, if the client provides - the `columnNames` option). If the callback returns a literal - `false` (as opposed to any other falsy value, e.g. an implicit - `undefined` return), any ongoing statement-`step()` iteration - stops without an error. The return value of the callback is - otherwise ignored. + exec(). The first argument passed to the callback is described + below. The second argument is always the current Stmt object, + as it's needed if the caller wants to fetch the column names or + some such (noting that they could also be fetched via + `this.columnNames`, if the client provides the `columnNames` + option). If the callback returns a literal `false` (as opposed + to any other falsy value, e.g. an implicit `undefined` return), + any ongoing statement-`step()` iteration stops without an + error. The return value of the callback is otherwise ignored. ACHTUNG: The callback MUST NOT modify the Stmt object. Calling any of the Stmt.get() variants, Stmt.getColumnName(), or @@ -970,20 +967,23 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ A.3) `'stmt'` causes the current Stmt to be passed to the callback, but this mode will trigger an exception if `resultRows` is an array because appending the transient - statement to the array would be downright unhelpful. + statement to the array would be downright unhelpful. This + option is a legacy feature, retained for backwards + compatibility. The statement object is passed as the second + argument to the callback, as described above. B) An integer, indicating a zero-based column in the result - row. Only that one single value will be passed on. + row. Only that one single value, in JS form, will be passed on. C) A string with a minimum length of 2 and leading character of '$' will fetch the row as an object, extract that one field, - and pass that field's value to the callback. Note that these - keys are case-sensitive so must match the case used in the + and pass that field's value to the callback. These keys are + case-sensitive so must match the case used in the SQL. e.g. `"select a A from t"` with a `rowMode` of `'$A'` would work but `'$a'` would not. A reference to a column not in the result set will trigger an exception on the first row (as - the check is not performed until rows are fetched). Note also - that `$` is a legal identifier character in JS so need not be + the check is not performed until rows are fetched). Note that + `$` is a legal identifier character in JS so need not be quoted. Any other `rowMode` value triggers an exception. @@ -1023,6 +1023,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - `callback` and `resultRows`: permit an array entries with semantics similar to those described for `bind` above. + OTOH, this function already does too much. */ exec: function(/*(sql [,obj]) || (obj)*/){ affirmDbOpen(this); @@ -1046,7 +1047,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /* Optimization: if the SQL is a TypedArray we can save some string conversion costs. */; /* Allocate the two output pointers (ppStmt, pzTail) and heap - space for the SQL (pSql). When prepare_v2() returns, pzTail + space for the SQL (pSql). When prepare_v3() returns, pzTail will point to somewhere in pSql. */ let sqlByteLen = isTA ? arg.sql.byteLength : wasm.jstrlen(arg.sql); const ppStmt = wasm.scopedAlloc( @@ -1058,8 +1059,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const pSqlEnd = wasm.ptr.add(pSql, sqlByteLen); if(isTA) wasm.heap8().set(arg.sql, pSql); else wasm.jstrcpy(arg.sql, wasm.heap8(), pSql, sqlByteLen, false); - wasm.poke(wasm.ptr.add(pSql, sqlByteLen), 0/*NUL terminator*/); - while(pSql && wasm.peek(pSql, 'i8') + wasm.poke8(wasm.ptr.add(pSql, sqlByteLen), 0/*NUL terminator*/); + while(pSql && wasm.peek8(pSql) /* Maintenance reminder:^^^ _must_ be 'i8' or else we will very likely cause an endless loop. What that's doing is checking for a terminating NUL byte. If we @@ -1123,6 +1124,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /* In order to trigger an exception in the INSERT...RETURNING locking scenario: https://sqlite.org/forum/forumpost/36f7a2e7494897df + [tag:insert-returning-reset] */).finalize(); stmt = null; }/*prepare() loop*/ @@ -1139,6 +1141,132 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return arg.returnVal(); }/*exec()*/, +//#if nope + /** + Experimental and untested - do not use. + + Prepares one or more SQL statements, passing each to a callback + for processing. + + It requires an options object with the following properties: + + - "sql": SQL in any format accepted by exec(). + + - "callback" (function): gets passed each prepared statement, + as described below. + + - "asPointer" (bool=false): if true, the callback is passed the + WASM (sqlite3*) pointer instead of a Stmt object. + + - "saveSql" (array): if set, the SQL of each prepared statement + is appended to this array. This can be used without a callback + to split SQL into its component statements. Purely empty + statements (for for which sqlite3_prepare() returns a NULL + sqlite3_stmt, i.e. spaces and comments) are not added to this + list unless... + + - "saveEmpty" (bool=false): If true, empty statements are + retained in opt.saveSql, but their leading/trailing whitespace + is trimmed (as for queries) so they may be empty. + + For each statement in the input SQL: + + 1) If opt.saveSql is set, the SQL is appended to it. + + 2) If callback is set, callback(S) is called, where S is either + a Stmt object (by default) or an (sqlite3*) WASM pointer (if + opt.asPointer is true). If the callback returns a literal true + (as opposed to any other truthy value), ownership of S is + transferred to the callback, otherwise S is reset and finalized + as soon as the callback returns. If the callback throws, S is + unconditionally finalized. + + If neither of opt.saveSql nor opt.callback are set, this + function does nothing more than prepare and finalize each + statement, which will trigger an exception if any of them + contain invalid SQL. + */ + forEachStmt: function(opt){ + affirmDbOpen(this); + opt ??= Object.create(null); + if(!opt.sql){ + return toss3("exec() requires an SQL string."); + } + const sql = util.flexibleString(opt.sql); + const callback = opt.callback; + let stmt, pStmt; + const stack = wasm.scopedAllocPush(); + const saveSql = Array.isArray(opt.saveSql) ? opt.saveSql : undefined; + try{ + const isTA = util.isSQLableTypedArray(opt.sql) + /* Optimization: if the SQL is a TypedArray we can save some string + conversion costs. */; + /* Allocate the two output pointers (ppStmt, pzTail) and heap + space for the SQL (pSql). When prepare_v3() returns, pzTail + will point to somewhere in pSql. */ + let sqlByteLen = isTA ? opt.sql.byteLength : wasm.jstrlen(sql); + const ppStmt = wasm.scopedAlloc( + /* output (sqlite3_stmt**) arg and pzTail */ + (2 * wasm.ptr.size) + (sqlByteLen + 1/* SQL + NUL */) + ); + const pzTail = wasm.ptr.add(ppStmt, wasm.ptr.size) /* final arg to sqlite3_prepare_v2() */; + let pSql = wasm.ptr.add(pzTail, wasm.ptr.size) /* start of the SQL string */; + const pSqlEnd = wasm.ptr.add(pSql, sqlByteLen); + if(isTA) wasm.heap8().set(sql, pSql); + else wasm.jstrcpy(sql, wasm.heap8(), pSql, sqlByteLen, false); + wasm.poke8(wasm.ptr.add(pSql, sqlByteLen), 0/*NUL terminator*/); + while( pSql && wasm.peek8(pSql) ){ + pStmt = stmt = null; + wasm.pokePtr([ppStmt, pzTail], 0); + const zHead = pSql; + DB.checkRc(this, capi.sqlite3_prepare_v3( + this.pointer, pSql, sqlByteLen, 0, ppStmt, pzTail + )); + [pStmt, pSql] = wasm.peekPtr([ppStmt, pzTail]); + sqlByteLen = wasm.ptr.addn(pSqlEnd,-pSql); + if(opt.saveSql){ + if( pStmt ) opt.saveSql.push(capi.sqlite3_sql(pStmt).trim()); + else if( opt.saveEmpty ){ + saveSql.push(wasm.typedArrayToString( + wasm.heap8u(), Number(zHead), + wasm.ptr.addn(zHead, sqlByteLen) + ).trim(/*arguable*/)); + } + } + if(!pStmt) continue; + //sqlite3.config.debug("forEachStmt() pSql =",capi.sqlite3_sql(pStmt)); + if( !opt.callback ){ + capi.sqlite3_finalize(pStmt); + pStmt = null; + continue; + } + stmt = opt.asPointer ? null : new Stmt(this, pStmt, BindTypes); + if( true===callaback(stmt || pStmt) ){ + stmt = pStmt = null /*callback took ownership */; + }else if(stmt){ + pStmt = null; + stmt.reset( + /* See [tag:insert-returning-reset]. The thinking here is + that if the callback didn't throw for this, it + probably should have. + */).finalize(); + stmt = null; + }else{ + const rx = capi.sqlite3_reset(pStmt/*[tag:insert-returning-reset]*/); + capi.sqlite3_finalize(pStmt); + pStmt = null; + DB.checkRc(this, rx); + } + }/*prepare() loop*/ + }finally{ + if(stmt) stmt.finalize(); + else if(pStmt) capi.sqlite3_finalize(pStmt); + wasm.scopedAllocPop(stack); + } + return this; + }/*forEachStmt()*/, +//#endif nope + /** Creates a new UDF (User-Defined Function) which is accessible via SQL code. This function may be called in any of the @@ -2295,55 +2423,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Stmt }/*oo1 object*/; - if(util.isUIThread()){ - /** - Functionally equivalent to DB(storageName,'c','kvvfs') except - that it throws if the given storage name is not one of 'local' - or 'session'. - - As of version 3.46, the argument may optionally be an options - object in the form: - - { - filename: 'session'|'local', - ... etc. (all options supported by the DB ctor) - } - - noting that the 'vfs' option supported by main DB - constructor is ignored here: the vfs is always 'kvvfs'. - */ - sqlite3.oo1.JsStorageDb = function(storageName='session'){ - const opt = dbCtorHelper.normalizeArgs(...arguments); - storageName = opt.filename; - if('session'!==storageName && 'local'!==storageName){ - toss3("JsStorageDb db name must be one of 'session' or 'local'."); - } - opt.vfs = 'kvvfs'; - dbCtorHelper.call(this, opt); - }; - const jdb = sqlite3.oo1.JsStorageDb; - jdb.prototype = Object.create(DB.prototype); - /** Equivalent to sqlite3_js_kvvfs_clear(). */ - jdb.clearStorage = capi.sqlite3_js_kvvfs_clear; - /** - Clears this database instance's storage or throws if this - instance has been closed. Returns the number of - database blocks which were cleaned up. - */ - jdb.prototype.clearStorage = function(){ - return jdb.clearStorage(affirmDbOpen(this).filename); - }; - /** Equivalent to sqlite3_js_kvvfs_size(). */ - jdb.storageSize = capi.sqlite3_js_kvvfs_size; - /** - Returns the _approximate_ number of bytes this database takes - up in its storage or throws if this instance has been closed. - */ - jdb.prototype.storageSize = function(){ - return jdb.storageSize(affirmDbOpen(this).filename); - }; - }/*main-window-only bits*/ - }); //#else /* Built with the omit-oo1 flag. */ diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 069f3fdb5..c53acee76 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -27,13 +27,14 @@ /** sqlite3ApiBootstrap() is the only global symbol persistently exposed by this API. It is intended to be called one time at the - end of the API amalgamation process, passed configuration details - for the current environment, and then optionally be removed from - the global object using `delete globalThis.sqlite3ApiBootstrap`. + end of the API amalgamation process and passed configuration details + for the current environment. This function is not intended for client-level use. It is intended for use in creating bundles configured for specific WASM - environments. + environments. That said, the "sqlite3-api.js" intermediary build + file aims to be suitable for dropping in to custom builds, and it + exposes only this function. This function expects a configuration object, intended to abstract away details specific to any given WASM environment, primarily so @@ -93,9 +94,10 @@ can be replaced with (e.g.) empty functions to squelch all such output. - - `wasmfsOpfsDir`[^1]: Specifies the "mount point" of the OPFS-backed - filesystem in WASMFS-capable builds. - + - `wasmfsOpfsDir`[^1]: Specifies the "mount point" of the + OPFS-backed filesystem in WASMFS-capable builds. This is only + used in WASMFS-capable builds of the library (which the canonical + builds do not include). [^1] = This property may optionally be a function, in which case this function calls that function to fetch the value, @@ -125,7 +127,8 @@ Both sqlite3ApiBootstrap.defaultConfig and globalThis.sqlite3ApiConfig get deleted by sqlite3ApiBootstrap() because any changes to them made after that point would have no - useful effect. + useful effect. This function also deletes itself from globalThis + when it's called. This function returns a Promise to the sqlite3 namespace object, which resolves after the async pieces of the library init are @@ -177,14 +180,6 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( } }); - /** - Eliminate any confusion about whether these config objects may - be used after library initialization by eliminating the outward-facing - objects... - */ - delete globalThis.sqlite3ApiConfig; - delete sqlite3ApiBootstrap.defaultConfig; - /** The main sqlite3 binding API gets installed into this object, mimicking the C API as closely as we can. The numerous members @@ -757,6 +752,11 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( toss: function(...args){throw new Error(args.join(' '))}, toss3, typedArrayPart: wasm.typedArrayPart, + assert: function(arg,msg){ + if( !arg ){ + util.toss("Assertion failed:",msg); + } + }, /** Given a byte array or ArrayBuffer, this function throws if the lead bytes of that buffer do not hold a SQLite3 database header, @@ -796,25 +796,10 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( /** wasm.X properties which are used for configuring the wasm - environment via whwashutil.js. + environment via whwashutil.js. This object gets fleshed out with + a number of WASM-specific utilities, in sqlite3-api-glue.c-pp.js. */ Object.assign(wasm, { - /** - The WASM IR (Intermediate Representation) value for - pointer-type values. If set then it MUST be one of 'i32' or - 'i64' (else an exception will be thrown). If it's not set, it - will default to 'i32'. - */ - pointerIR: config.wasmPtrIR, - - /** - True if BigInt support was enabled via (e.g.) the - Emscripten -sWASM_BIGINT flag, else false. When - enabled, certain 64-bit sqlite3 APIs are enabled which - are not otherwise enabled due to JS/WASM int64 - impedance mismatches. - */ - bigIntEnabled: !!config.bigIntEnabled, /** The symbols exported by the WASM environment. @@ -825,8 +810,9 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( /** When Emscripten compiles with `-sIMPORTED_MEMORY`, it initializes the heap and imports it into wasm, as opposed to - the other way around. In this case, the memory is not - available via this.exports.memory. + the other way around. In this case, the memory is not available + via this.exports.memory so the client must pass it in via + config.memory. */ memory: config.memory || config.exports['memory'] @@ -834,6 +820,29 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( "in either config.exports.memory (exported)", "or config.memory (imported)."), + /** + The WASM pointer size. If set then it MUST be one of 4 or 8 and + it MUST correspond to the WASM environment's pointer size. We + figure out the size by calling some un-JS-wrapped WASM function + which returns a pointer-type value. If that value is a BigInt, + it's 64-bit, else it's 32-bit. The pieces which populate + sqlite3.wasm (whwasmutil.js) can figure this out _if_ they can + allocate, but we have a chicken/egg situation there which makes + it illegal for that code to invoke wasm.dealloc() at the time + it would be needed. So we need to configure it ahead of time + (here) instead. + */ + pointerSize: ('number'===typeof config.exports.sqlite3_libversion()) ? 4 : 8, + + /** + True if BigInt support was enabled via (e.g.) the + Emscripten -sWASM_BIGINT flag, else false. When + enabled, certain 64-bit sqlite3 APIs are enabled which + are not otherwise enabled due to JS/WASM int64 + impedance mismatches. + */ + bigIntEnabled: !!config.bigIntEnabled, + /** WebAssembly.Table object holding the indirect function call table. Defaults to exports.__indirect_function_table. @@ -883,7 +892,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( Like this.alloc.impl(), this.realloc.impl() is a direct binding to the underlying realloc() implementation which does not throw - exceptions, instead returning 0 on allocation error. + exceptions, instead returning 0 (or 0n) on allocation error. */ realloc: undefined/*installed later*/, @@ -949,7 +958,11 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( }; wasm.realloc.impl = wasm.exports[keyRealloc]; wasm.dealloc = function f(m){ - f.impl(wasm.ptr.coerce(m)/*tag:64bit*/); + f.impl(wasm.ptr.coerce(m)/*tag:64bit*/) + /* This coerce() is the reason we have to set wasm.pointerSize before + calling WhWasmUtilInstaller(). If we don't, that code will call + into this very early in its init, before wasm.ptr has been set up, + resulting in a null deref here. */; }; wasm.dealloc.impl = wasm.exports[keyDealloc]; } @@ -1020,18 +1033,18 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( }/*compileOptionUsed()*/; /** - sqlite3.wasm.pstack (pseudo-stack) holds a special-case intended - solely for short-lived, small data. In practice, it's primarily - used to allocate output pointers. It mus not be used for any - memory which needs to outlive the scope in which it's obtained - from pstack. + sqlite3.wasm.pstack (pseudo-stack) holds a special-case allocator + intended solely for short-lived, small data. In practice, it's + primarily used to allocate output pointers. It must not be used + for any memory which needs to outlive the scope in which it's + obtained from pstack. The library guarantees only that a minimum of 2kb are available in this allocator, and it may provide more (it's a build-time value). pstack.quota and pstack.remaining can be used to get the total resp. remaining amount of memory. - It has only a single intended usage: + It has only a single intended usage pattern: ``` const stackPos = pstack.pointer; @@ -1048,13 +1061,11 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( ``` This allocator is much faster than a general-purpose one but is - limited to usage patterns like the one shown above. + limited to usage patterns like the one shown above (which are + pretty common when using sqlite3.capi). - It operates from a static range of memory which lives outside of - space managed by Emscripten's stack-management, so does not - collide with Emscripten-provided stack allocation APIs. The - memory lives in the WASM heap and can be used with routines such - as wasm.poke() and wasm.heap8u().slice(). + The memory lives in the WASM heap and can be used with routines + such as wasm.poke() and wasm.heap8u().slice(). */ wasm.pstack = Object.assign(Object.create(null),{ /** @@ -1128,7 +1139,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( argument: if it's 1, it returns a single pointer value. If it's more than 1, it returns the same as allocChunks(). - When a returned pointers will refer to a 64-bit value, e.g. a + When a returned pointer will refer to a 64-bit value, e.g. a double or int64, and that value must be written or fetched, e.g. using wasm.poke() or wasm.peek(), it is important that the pointer in question be aligned to an 8-byte @@ -1196,6 +1207,9 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( } })/*wasm.pstack properties*/; + /** + Docs: https://sqlite.org/wasm/doc/trunk/api-c-style.md#sqlite3_randomness + */ capi.sqlite3_randomness = (...args)=>{ if(1===args.length && util.isTypedArray(args[0]) @@ -1230,8 +1244,6 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( wasm.exports.sqlite3_randomness(...args); }; - /** State for sqlite3_wasmfs_opfs_dir(). */ - let __wasmfsOpfsDir = undefined; /** If the wasm environment has a WASMFS/OPFS-backed persistent storage directory, its path is returned by this function. If it @@ -1255,7 +1267,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( WASMFS capability requires a custom build. */ capi.sqlite3_wasmfs_opfs_dir = function(){ - if(undefined !== __wasmfsOpfsDir) return __wasmfsOpfsDir; + if(undefined !== this.dir) return this.dir; // If we have no OPFS, there is no persistent dir const pdir = config.wasmfsOpfsDir; if(!pdir @@ -1263,21 +1275,21 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( || !globalThis.FileSystemDirectoryHandle || !globalThis.FileSystemFileHandle || !wasm.exports.sqlite3__wasm_init_wasmfs){ - return __wasmfsOpfsDir = ""; + return this.dir = ""; } try{ if(pdir && 0===wasm.xCallWrapped( 'sqlite3__wasm_init_wasmfs', 'i32', ['string'], pdir )){ - return __wasmfsOpfsDir = pdir; + return this.dir = pdir; }else{ - return __wasmfsOpfsDir = ""; + return this.dir = ""; } }catch(e){ // sqlite3__wasm_init_wasmfs() is not available - return __wasmfsOpfsDir = ""; + return this.dir = ""; } - }; + }.bind(Object.create(null)); /** Returns true if sqlite3.capi.sqlite3_wasmfs_opfs_dir() is a @@ -1333,7 +1345,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( */ capi.sqlite3_js_vfs_list = function(){ const rc = []; - let pVfs = capi.sqlite3_vfs_find(wasm.ptr.coerce(0)); + let pVfs = capi.sqlite3_vfs_find(wasm.ptr.null); while(pVfs){ const oVfs = new capi.sqlite3_vfs(pVfs); rc.push(wasm.cstrToJs(oVfs.$zName)); @@ -1401,7 +1413,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( or not provided, then "main" is assumed. */ capi.sqlite3_js_db_vfs = - (dbPointer, dbName=0)=>util.sqlite3__wasm_db_vfs(dbPointer, dbName); + (dbPointer, dbName=wasm.ptr.null)=>util.sqlite3__wasm_db_vfs(dbPointer, dbName); /** A thin wrapper around capi.sqlite3_aggregate_context() which @@ -1597,86 +1609,6 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( return x===v ? undefined : x; } - if( util.isUIThread() ){ - /* Features specific to the main window thread... */ - - /** - Internal helper for sqlite3_js_kvvfs_clear() and friends. - Its argument should be one of ('local','session',""). - */ - const __kvvfsInfo = function(which){ - const rc = Object.create(null); - rc.prefix = 'kvvfs-'+which; - rc.stores = []; - if('session'===which || ""===which) rc.stores.push(globalThis.sessionStorage); - if('local'===which || ""===which) rc.stores.push(globalThis.localStorage); - return rc; - }; - - /** - Clears all storage used by the kvvfs DB backend, deleting any - DB(s) stored there. Its argument must be either 'session', - 'local', or "". In the first two cases, only sessionStorage - resp. localStorage is cleared. If it's an empty string (the - default) then both are cleared. Only storage keys which match - the pattern used by kvvfs are cleared: any other client-side - data are retained. - - This function is only available in the main window thread. - - Returns the number of entries cleared. - */ - capi.sqlite3_js_kvvfs_clear = function(which=""){ - let rc = 0; - const kvinfo = __kvvfsInfo(which); - kvinfo.stores.forEach((s)=>{ - const toRm = [] /* keys to remove */; - let i; - for( i = 0; i < s.length; ++i ){ - const k = s.key(i); - if(k.startsWith(kvinfo.prefix)) toRm.push(k); - } - toRm.forEach((kk)=>s.removeItem(kk)); - rc += toRm.length; - }); - return rc; - }; - - /** - This routine guesses the approximate amount of - window.localStorage and/or window.sessionStorage in use by the - kvvfs database backend. Its argument must be one of - ('session', 'local', ""). In the first two cases, only - sessionStorage resp. localStorage is counted. If it's an empty - string (the default) then both are counted. Only storage keys - which match the pattern used by kvvfs are counted. The returned - value is the "length" value of every matching key and value, - noting that JavaScript stores each character in 2 bytes. - - Note that the returned size is not authoritative from the - perspective of how much data can fit into localStorage and - sessionStorage, as the precise algorithms for determining - those limits are unspecified and may include per-entry - overhead invisible to clients. - */ - capi.sqlite3_js_kvvfs_size = function(which=""){ - let sz = 0; - const kvinfo = __kvvfsInfo(which); - kvinfo.stores.forEach((s)=>{ - let i; - for(i = 0; i < s.length; ++i){ - const k = s.key(i); - if(k.startsWith(kvinfo.prefix)){ - sz += k.length; - sz += s.getItem(k).length; - } - } - }); - return sz * 2 /* because JS uses 2-byte char encoding */; - }; - - }/* main-window-only bits */ - /** Wraps all known variants of the C-side variadic sqlite3_db_config(). @@ -1953,55 +1885,58 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( return (0===v) ? undefined : capi.sqlite3_value_to_js(v, throwIfCannotConvert); }; - /** - Internal impl of sqlite3_preupdate_new/old_js() and - sqlite3changeset_new/old_js(). - */ - const __newOldValue = function(pObj, iCol, impl){ - impl = capi[impl]; - if(!this.ptr) this.ptr = wasm.allocPtr(); - else wasm.pokePtr(this.ptr, 0); - const rc = impl(pObj, iCol, this.ptr); - if(rc) return SQLite3Error.toss(rc,arguments[2]+"() failed with code "+rc); - const pv = wasm.peekPtr(this.ptr); - return pv ? capi.sqlite3_value_to_js( pv, true ) : undefined; - }.bind(Object.create(null)); - - /** - A wrapper around sqlite3_preupdate_new() which fetches the - sqlite3_value at the given index and returns the result of - passing it to sqlite3_value_to_js(). Throws on error. - */ - capi.sqlite3_preupdate_new_js = - (pDb, iCol)=>__newOldValue(pDb, iCol, 'sqlite3_preupdate_new'); + if( true ){ /* changeset/preupdate additions... */ + /** + Internal impl of sqlite3_preupdate_new/old_js() and + sqlite3changeset_new/old_js(). + */ + const __newOldValue = function(pObj, iCol, impl){ + impl = capi[impl]; + if(!this.ptr) this.ptr = wasm.allocPtr(); + else wasm.pokePtr(this.ptr, 0); + const rc = impl(pObj, iCol, this.ptr); + if(rc) return SQLite3Error.toss(rc,arguments[2]+"() failed with code "+rc); + const pv = wasm.peekPtr(this.ptr); + return pv ? capi.sqlite3_value_to_js( pv, true ) : undefined; + }.bind(Object.create(null)); - /** - The sqlite3_preupdate_old() counterpart of - sqlite3_preupdate_new_js(), with an identical interface. - */ - capi.sqlite3_preupdate_old_js = - (pDb, iCol)=>__newOldValue(pDb, iCol, 'sqlite3_preupdate_old'); + /** + A wrapper around sqlite3_preupdate_new() which fetches the + sqlite3_value at the given index and returns the result of + passing it to sqlite3_value_to_js(). Throws on error. + */ + capi.sqlite3_preupdate_new_js = + (pDb, iCol)=>__newOldValue(pDb, iCol, 'sqlite3_preupdate_new'); - /** - A wrapper around sqlite3changeset_new() which fetches the - sqlite3_value at the given index and returns the result of - passing it to sqlite3_value_to_js(). Throws on error. + /** + The sqlite3_preupdate_old() counterpart of + sqlite3_preupdate_new_js(), with an identical interface. + */ + capi.sqlite3_preupdate_old_js = + (pDb, iCol)=>__newOldValue(pDb, iCol, 'sqlite3_preupdate_old'); - If sqlite3changeset_new() succeeds but has no value to report, - this function returns the undefined value, noting that undefined - is a valid conversion from an `sqlite3_value`, so is unambiguous. - */ - capi.sqlite3changeset_new_js = - (pChangesetIter, iCol) => __newOldValue(pChangesetIter, iCol, - 'sqlite3changeset_new'); + /** + A wrapper around sqlite3changeset_new() which fetches the + sqlite3_value at the given index and returns the result of + passing it to sqlite3_value_to_js(). Throws on error. + + If sqlite3changeset_new() succeeds but has no value to report, + this function returns the undefined value, noting that + undefined is not a valid conversion from an `sqlite3_value`, so + is unambiguous. + */ + capi.sqlite3changeset_new_js = + (pChangesetIter, iCol) => __newOldValue(pChangesetIter, iCol, + 'sqlite3changeset_new'); - /** - The sqlite3changeset_old() counterpart of - sqlite3changeset_new_js(), with an identical interface. - */ - capi.sqlite3changeset_old_js = - (pChangesetIter, iCol)=>__newOldValue(pChangesetIter, iCol, - 'sqlite3changeset_old'); + /** + The sqlite3changeset_old() counterpart of + sqlite3changeset_new_js(), with an identical interface. + */ + capi.sqlite3changeset_old_js = + (pChangesetIter, iCol)=>__newOldValue(pChangesetIter, iCol, + 'sqlite3changeset_old'); + }/*changeset/preupdate additions*/ /* The remainder of the API will be set up in later steps. */ const sqlite3 = { @@ -2013,10 +1948,10 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( config, /** Holds the version info of the sqlite3 source tree from which - the generated sqlite3-api.js gets built. Note that its version - may well differ from that reported by sqlite3_libversion(), but - that should be considered a source file mismatch, as the JS and - WASM files are intended to be built and distributed together. + the generated sqlite3-api.js gets built. Its version may well + differ from that reported by sqlite3_libversion(), but that + should be considered a source file mismatch, as the JS and WASM + files are intended to be built and distributed together. This object is initially a placeholder which gets replaced by a build-generated object. @@ -2041,9 +1976,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( async init will be fatal to the init as a whole, but init routines are themselves welcome to install dummy catch() handlers which are not fatal if their failure should be - considered non-fatal. If called more than once, the second and - subsequent calls are no-ops which return a pre-resolved - Promise. + considered non-fatal. Ideally this function is called as part of the Promise chain which handles the loading and bootstrapping of the API. If not @@ -2060,18 +1993,14 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( */ asyncPostInit: async function ff(){ if(ff.isReady instanceof Promise) return ff.isReady; - let lia = sqlite3ApiBootstrap.initializersAsync; - delete sqlite3ApiBootstrap.initializersAsync; + let lia = this.initializersAsync; + delete this.initializersAsync; const postInit = async ()=>{ if(!sqlite3.__isUnderTest){ /* Delete references to internal-only APIs which are used by some initializers. Retain them when running in test mode so that we can add tests for them. */ delete sqlite3.util; - /* It's conceivable that we might want to expose - StructBinder to client-side code, but it's only useful if - clients build their own sqlite3.wasm which contains their - own C struct types. */ delete sqlite3.StructBinder; } return sqlite3; @@ -2090,7 +2019,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( let p = Promise.resolve(sqlite3); while(lia.length) p = p.then(lia.shift()); return ff.isReady = p.catch(catcher); - }, + }.bind(sqlite3ApiBootstrap), /** scriptInfo ideally gets injected into this object by the infrastructure which assembles the JS/WASM module. It contains @@ -2105,6 +2034,9 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( */ scriptInfo: undefined }; + if( ('undefined'!==typeof sqlite3IsUnderTest/* from post-js-header.js */) ){ + sqlite3.__isUnderTest = !!sqlite3IsUnderTest; + } try{ sqlite3ApiBootstrap.initializers.forEach((f)=>{ f(sqlite3); @@ -2117,16 +2049,34 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( } delete sqlite3ApiBootstrap.initializers; sqlite3ApiBootstrap.sqlite3 = sqlite3; - delete globalThis.sqlite3ApiBootstrap; - delete globalThis.sqlite3ApiConfig; - sqlite3InitScriptInfo.debugModule( - "sqlite3ApiBootstrap() complete", sqlite3 - ); - sqlite3.scriptInfo /* used by some async init code */ = - sqlite3InitScriptInfo /* from post-js-header.js */; - if( (sqlite3.__isUnderTest = sqlite3IsUnderTest /* from post-js-header.js */) ){ - sqlite3.config.emscripten = EmscriptenModule; - const iw = sqlite3InitScriptInfo.instantiateWasm; + if( 'undefined'!==typeof sqlite3InitScriptInfo/* from post-js-header.js */ ){ + sqlite3InitScriptInfo.debugModule( + "sqlite3ApiBootstrap() complete", sqlite3 + ); + sqlite3.scriptInfo + /* Used by some async init code. As of 2025-11-15 this is still + in use by the OPFS VFS for locating its worker. In non-Emscripten + builds, this would need to be injected in somewhere to get + that VFS loading. */ = sqlite3InitScriptInfo; + } + if( sqlite3.__isUnderTest ){ + if( 'undefined'!==typeof EmscriptenModule ){ + sqlite3.config.emscripten = EmscriptenModule; + } + /* + The problem with exposing these pieces (in non-testing runs) via + sqlite3.wasm is that it exposes non-SQLite pieces to the + clients, who may come to expect it to remain. _We_ only have + these data because we've overridden Emscripten's wasm file + loader, and if we lose that capability for some reason then + we'll lose access to this metadata. + + These data are interesting for exploring how the wasm/JS + pieces connect, e.g. for exploring exactly what Emscripten + imports into WASM from its JS glue, but it's not + SQLite-related. + */ + const iw = sqlite3.scriptInfo?.instantiateWasm; if( iw ){ /* Metadata injected by the custom Module.instantiateWasm() in pre-js.c-pp.js. */ @@ -2135,10 +2085,21 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( sqlite3.wasm.imports = iw.imports; } } + + /** + Eliminate any confusion about whether these config objects may + be used after library initialization by eliminating the outward-facing + objects... + */ + delete globalThis.sqlite3ApiConfig; + delete globalThis.sqlite3ApiBootstrap; + delete sqlite3ApiBootstrap.defaultConfig; return sqlite3.asyncPostInit().then((s)=>{ - sqlite3InitScriptInfo.debugModule( - "sqlite3.asyncPostInit() complete", sqlite3 - ); + if( 'undefined'!==typeof sqlite3InitScriptInfo/* from post-js-header.js */ ){ + sqlite3InitScriptInfo.debugModule( + "sqlite3.asyncPostInit() complete", s + ); + } delete s.asyncPostInit; delete s.scriptInfo; delete s.emscripten; diff --git a/ext/wasm/api/sqlite3-license-version-header.js b/ext/wasm/api/sqlite3-license-version-header.js index 482989463..dd32f4666 100644 --- a/ext/wasm/api/sqlite3-license-version-header.js +++ b/ext/wasm/api/sqlite3-license-version-header.js @@ -1,4 +1,5 @@ -/* +/* @preserve +** ** LICENSE for the sqlite3 WebAssembly/JavaScript APIs. ** ** This bundle (typically released as sqlite3.js or sqlite3.mjs) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index e10d0dd50..79fc47393 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -1,4 +1,4 @@ -/* +/* @preserve 2022-09-16 The author disclaims copyright to this source code. In place of a diff --git a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js new file mode 100644 index 000000000..e3fb72287 --- /dev/null +++ b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js @@ -0,0 +1,2095 @@ +/* + 2025-11-21 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file houses the "kvvfs" pieces of the SQLite3 JS API. Most of + kvvfs is implemented in src/os_kv.c and exposed/extended for use + here via sqlite3-wasm.c. + + Main project home page: https://sqlite.org + + Documentation home page: https://sqlite.org/wasm +*/ +//#if omit-kvvfs +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + /* These are JS plumbing, not part of the public API */ + delete sqlite3.capi.sqlite3_kvvfs_methods; + delete sqlite3.capi.KVVfsFile; +} +//#else +//#@policy error +//#savepoint begin +//#define kvvfs-v2-added-in=3.52.0 +/** + kvvfs - the Key/Value VFS - is an SQLite3 VFS which delegates + storage of its pages and metadata to a key-value store. + + It was conceived in order to support JS's localStorage and + sessionStorage objects. Its native implementation uses files as + key/value storage (one file per record) but the JS implementation + replaces a few methods so that it can use the aforementioned + objects as storage. + + It uses a bespoke ASCII encoding to store each db page as a + separate record and stores some metadata, like the db's unencoded + size and its journal, as individual records. + + kvvfs is significantly less efficient than a plain in-memory db but + it also, as a side effect of its design, offers a JSON-friendly + interchange format for exporting and importing databases. + + kvvfs is _not_ designed for heavy db loads. It is relatively + malloc()-heavy, having to de/allocate frequently, and it + spends much of its time converting the raw db pages into and out of + an ASCII encoding. + + But it _does_ work and is "performant enough" for db work of the + scale of a db which will fit within sessionStorage or localStorage + (just 2-3mb). + + "Version 2" extends it to support using Storage-like objects as + backing storage, Storage being the JS class which localStorage and + sessionStorage both derive from. This essentially moves the backing + store from whatever localStorage and sessionStorage use to an + in-memory object. + + This effort is primarily a stepping stone towards eliminating, if + it proves possible, the POSIX I/O API dependencies in SQLite's WASM + builds. That is: if this VFS works properly, it can be set as the + default VFS and we can eliminate the "unix" VFS from the JS/WASM + builds (as opposed to server-wise/WASI builds). That still, as of + 2025-11-23, a ways away, but it's the main driver for version 2 of + kvvfs. + + Version 2 remains compatible with version 1 databases and always + writes localStorage/sessionStorage metadata in the v1 format, so + such dbs can be manipulated freely by either version. For transient + storage objects (new in version 2), the format of its record keys + is simpified, requiring less space than v1 keys by eliding + redundant (in this context) info from the keys. + + Another benefit of v2 is its ability to export dbs into a + JSON-friendly (but not human-friendly) format. + + A potential, as-yet-unproven, benefit, would be the ability to plug + arbitrary Storage-compatible objects in so that clients could, + e.g. asynchronously post updates to db pages to some back-end for + backups. +*/ +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + 'use strict'; + const capi = sqlite3.capi, + sqlite3_kvvfs_methods = capi.sqlite3_kvvfs_methods, + KVVfsFile = capi.KVVfsFile, + pKvvfs = sqlite3.capi.sqlite3_vfs_find("kvvfs") + + /* These are JS plumbing, not part of the public API */ + delete capi.sqlite3_kvvfs_methods; + delete capi.KVVfsFile; + + if( !pKvvfs ) return /* nothing to do */; + if( 0 ){ + /* This working would be our proverbial holy grail, in that it + would allow us to eliminate the current default VFS, which + relies on POSIX I/O APIs. Eliminating that dependency would get + us one giant step closer to creating wasi-sdk builds. */ + capi.sqlite3_vfs_register(pKvvfs, 1); + } + + const util = sqlite3.util, + wasm = sqlite3.wasm, + toss3 = util.toss3, + hop = (o,k)=>Object.prototype.hasOwnProperty.call(o,k); + + const kvvfsMethods = new sqlite3_kvvfs_methods( + /* Wraps the static sqlite3_api_methods singleton */ + wasm.exports.sqlite3__wasm_kvvfs_methods() + ); + util.assert( 32<=kvvfsMethods.$nKeySize, "unexpected kvvfsMethods.$nKeySize: "+kvvfsMethods.$nKeySize); + + /** + Most of the VFS-internal state. + */ + const cache = Object.assign(Object.create(null),{ + /** Regex matching journal file names. */ + rxJournalSuffix: /-journal$/, + /** Frequently-used C-string. */ + zKeyJrnl: wasm.allocCString("jrnl"), + /** Frequently-used C-string. */ + zKeySz: wasm.allocCString("sz"), + /** + The maximum size of a kvvfs record key. It is historically only + 32, a limitation currently retained only because it's convenient to + do so (the underlying code has outgrown the need for the artifically + low limit). + + We cache this value here because the end of this init code will + dispose of kvvfsMethods, invalidating it. + */ + keySize: kvvfsMethods.$nKeySize, + /** + WASM heap memory buffers to optimize out some frequent + allocations. + */ + buffer: Object.assign(Object.create(null),{ + /** + The size of each buffer in this.pool. + + kvvfsMethods.$nBufferSize is slightly larger than the output + space needed for a kvvfs-encoded 64kb db page in a worse-cast + encoding (128kb). It is not suitable for arbitrary buffer + use, only page de/encoding. + */ + n: kvvfsMethods.$nBufferSize, + /** + Map of buffer ids to wasm.alloc()'d pointers of size + this.n. (Re)used by various internals. + + Buffer ids 0 and 1 are used in the API internals. Other + names are used in higher-level APIs. + + See memBuffer() and memBufferFree(). + */ + pool: Object.create(null) + }) + }); + + /** + Returns a (cached) wasm.alloc()'d buffer of cache.buffer.n size, + throwing on OOM. + + We leak this one-time alloc because we've no better option. + sqlite3_vfs does not have a finalizer, so we've no place to hook + in the cleanup. We "could" extend sqlite3_shutdown() to have a + cleanup list for stuff like this but that function is never + used in JS, so it's hardly worth it. + */ + cache.memBuffer = (id=0)=>cache.buffer.pool[id] ??= wasm.alloc(cache.buffer.n); + + /** Frees the buffer with the given id. */ + cache.memBufferFree = (id)=>{ + const b = cache.buffer.pool[id]; + if( b ){ + wasm.dealloc(b); + delete cache.buffer.pool[id]; + } + }; + + const noop = ()=>{}; + const debug = sqlite3.__isUnderTest + ? (...args)=>sqlite3.config.debug?.("kvvfs:", ...args) + : noop; + const warn = (...args)=>sqlite3.config.warn?.("kvvfs:", ...args); + const error = (...args)=>sqlite3.config.error?.("kvvfs:", ...args); + + /** + Implementation of JS's Storage interface for use as backing store + of the kvvfs. Storage is a native class and its constructor + cannot be legally called from JS, making it impossible to + directly subclass Storage. This class implements (only) the + Storage interface, to make it a drop-in replacement for + localStorage/sessionStorage. (Any behavioral discrepancies are to + be considered bugs.) + + This impl simply proxies a plain, prototype-less Object, suitable + for JSON-ing. + + Design note: Storage has a bit of an odd iteration-related + interface as does not (AFAIK) specify specific behavior regarding + modification during traversal. Because of that, this class does + some seemingly unnecessary things with its #keys member, deleting + and recreating it whenever a property index might be invalidated. + */ + class KVVfsStorage { + #map = Object.create(null); + #keys = null; + #size = 0; + + constructor(){ + this.clear(); + } + + #getKeys(){ + return this.#keys ??= Object.keys(this.#map); + } + + key(n){ + if(n < 0 || n >= this.#size) return null; + return this.#getKeys()[n]; + } + + getItem(k){ + return this.#map[k] ?? null; + } + + setItem(k,v){ + if( !(k in this.#map) ){ + ++this.#size; + this.#keys = null; + } + this.#map[k] = ''+v; + } + + removeItem(k){ + if( k in this.#map ){ + delete this.#map[k]; + --this.#size; + this.#keys = null; + } + } + + clear(){ + this.#map = Object.create(null); + this.#keys = null; + this.#size = 0; + } + + get length() { + return this.#size; + } + }/*KVVfsStorage*/; + + /** True if v is the name of one of the special persistant Storage + objects. */ + const kvvfsIsPersistentName = (v)=>'local'===v || 'session'===v; + + /** + Keys in kvvfs have a prefix of "kvvfs-NAME-", where NAME is the + db name. This key is redundant in JS but it's how kvvfs works (it + saves each key to a separate file, so needs a distinct namespace + per data source name). We retain this prefix in 'local' and + 'session' storage for backwards compatibility and so that they + can co-exist with client data in their storage, but we elide them + from "v2" storage, where they're superfluous. + */ + const kvvfsKeyPrefix = (v)=>kvvfsIsPersistentName(v) ? 'kvvfs-'+v+'-' : ''; + + /** + Throws if storage name n (JS string) is not valid for use as a + storage name. Much of this goes back to kvvfs having a fixed + buffer size for its keys, and the storage name needing to be + encoded in the keys for local/session storage. + + The second argument must only be true when called from xOpen() - + it makes names with a "-journal" suffix legal. + */ + const validateStorageName = function(n,mayBeJournal=false){ + if( kvvfsIsPersistentName(n) ) return; + const len = (new Blob([n])).size/*byte length*/; + if( !len ) toss3(capi.SQLITE_MISUSE, "Empty name is not permitted."); + let maxLen = cache.keySize - 1; + if( cache.rxJournalSuffix.test(n) ){ + if( !mayBeJournal ){ + toss3(capi.SQLITE_MISUSE, + "Storage names may not have a '-journal' suffix."); + } + }else if( ['-wal','-shm'].filter(v=>n.endsWith(v)).length ){ + toss3(capi.SQLITE_MISUSE, + "Storage names may not have a -wal or -shm suffix."); + }else{ + maxLen -= 8 /* so we have space for a matching "-journal" suffix */; + } + if( len > maxLen ){ + toss3(capi.SQLITE_RANGE, "Storage name is too long. Limit =", maxLen); + } + let i; + for( i = 0; i < len; ++i ){ + const ch = n.codePointAt(i); + if( ch<32 ){ + toss3(capi.SQLITE_RANGE, + "Illegal character ("+ch+"d) in storage name:",n); + } + } + }; + + /** + Create a new instance of the objects which go into + cache.storagePool, with a refcount of 1. If passed a Storage-like + object as its second argument, it is used for the storage, + otherwise it creates a new KVVfsStorage object. + */ + const newStorageObj = (name,storage=undefined)=>Object.assign(Object.create(null),{ + /** + JS string value of this KVVfsFile::$zClass. i.e. the storage's + name. + */ + jzClass: name, + /** + Refcount. This keeps dbs and journals pointing to the same + storage for the life of both and enables kvvfs to behave more + like a conventional filesystem (a stepping stone towards + downstream API goals). Managed by xOpen() and xClose(). + */ + refc: 1, + /** + If true, this storage will be removed by xClose() or + sqlite3_js_kvvfs_unlink() when refc reaches 0. The others will + persist when refc==0, to give the illusion of real back-end + storage. Managed by xOpen() and sqlite3_js_kvvfs_reserve(). By + default this is false but the delete-on-close=1 flag can be + used to set this to true. + */ + deleteAtRefc0: false, + /** + The backing store. Must implement the Storage interface. + */ + storage: storage || new KVVfsStorage, + /** + The storage prefix used for kvvfs keys. It is + "kvvfs-STORAGENAME-" for local/session storage and an empty + string for other storage. local/session storage must use the + long form (A) for backwards compatibility and (B) so that kvvfs + can coexist with non-db client data in those backends. Neither + (A) nor (B) are concerns for KVVfsStorage objects. + + This prefix mirrors the one generated by os_kv.c's + kvrecordMakeKey() and must stay in sync with that one. + */ + keyPrefix: kvvfsKeyPrefix(name), + /** + KVVfsFile instances currently using this storage. Managed by + xOpen() and xClose(). + */ + files: [], + /** + If set, it's an array of objects with various event + callbacks. See sqlite3_js_kvvfs_listen(). When there are no + listeners, this member is set to undefined (instead of an empty + array) to allow us to more easily optimize out calls to + notifyListeners() for the common case of no listeners. + */ + listeners: undefined + }); + + /** + Public interface for kvvfs v2. The capi.sqlite3_js_kvvfs_...() + routines remain in place for v1. Some members of this class proxy + to those functions but use different default argument values in + some cases. + */ + const kvvfs = sqlite3.kvvfs = Object.create(null); + if( sqlite3.__isUnderTest ){ + /* For inspection via the dev tools console. */ + kvvfs.log = Object.assign(Object.create(null),{ + xOpen: false, + xClose: false, + xWrite: false, + xRead: false, + xSync: false, + xAccess: false, + xFileControl: false, + xRcrdRead: false, + xRcrdWrite: false, + xRcrdDelete: false, + }); + } + + /** + Deletes the cache.storagePool entries for store (a + cache.storagePool entry) and its db/journal counterpart. + */ + const deleteStorage = function(store){ + const other = cache.rxJournalSuffix.test(store.jzClass) + ? store.jzClass.replace(cache.rxJournalSuffix,'') + : store.jzClass+'-journal'; + kvvfs?.log?.xClose + && debug("cleaning up storage handles [", store.jzClass, other,"]",store); + delete cache.storagePool[store.jzClass]; + delete cache.storagePool[other]; + if( !sqlite3.__isUnderTest ){ + /* In test runs, leave these for inspection. If we delete them here, + any prior dumps of them emitted via the console get cleared out + because the console shows live objects instead of call-time + static dumps. */ + delete store.storage; + delete store.refc; + } + }; + + /** + Add both store.jzClass and store.jzClass+"-journal" + to cache,storagePool. + */ + const installStorageAndJournal = (store)=> + cache.storagePool[store.jzClass] = + cache.storagePool[store.jzClass+'-journal'] = store; + + /** + The public name of the current thread's transient storage + object. A storage object with this name gets preinstalled. + */ + const nameOfThisThreadStorage = '.'; + + /** + Map of JS-stringified KVVfsFile::zClass names to + reference-counted Storage objects. These objects are created in + xOpen(). Their refcount is decremented in xClose(), and the + record is destroyed if the refcount reaches 0. We refcount so + that concurrent active xOpen()s on a given name, and within a + given thread, use the same storage object. + */ + cache.storagePool = Object.assign(Object.create(null),{ + /* Start off with mappings for well-known names. */ + [nameOfThisThreadStorage]: newStorageObj(nameOfThisThreadStorage) + }); + + if( globalThis.Storage ){ + /* If available, install local/session storage. */ + if( globalThis.localStorage instanceof globalThis.Storage ){ + cache.storagePool.local = newStorageObj('local', globalThis.localStorage); + } + if( globalThis.sessionStorage instanceof globalThis.Storage ){ + cache.storagePool.session = newStorageObj('session', globalThis.sessionStorage); + } + } + + cache.builtinStorageNames = Object.keys(cache.storagePool); + + const isBuiltinName = (n)=>cache.builtinStorageNames.indexOf(n)>-1; + + /* Add "-journal" twins for each cache.storagePool entry... */ + for(const k of Object.keys(cache.storagePool)){ + /* Journals in kvvfs are are stored as individual records within + their Storage-ish object, named "{storage.keyPrefix}jrnl". We + always map the db and its journal to the same Storage + object. */ + const orig = cache.storagePool[k]; + cache.storagePool[k+'-journal'] = orig; + } + + cache.setError = (e=undefined, dfltErrCode=capi.SQLITE_ERROR)=>{ + if( e ){ + cache.lastError = e; + return (e.resultCode | 0) || dfltErrCode; + } + delete cache.lastError; + return 0; + }; + + cache.popError = ()=>{ + const e = cache.lastError; + delete cache.lastError; + return e; + }; + + /** Exception handler for notifyListeners(). */ + const catchForNotify = (e)=>{ + warn("kvvfs.listener handler threw:",e); + }; + + const kvvfsDecode = wasm.exports.sqlite3__wasm_kvvfs_decode; + const kvvfsEncode = wasm.exports.sqlite3__wasm_kvvfs_encode; + + /** + Listener events and their argument(s) (via the callback(ev) + ev.data member): + + 'open': number of opened handles on this storage. + + 'close': number of opened handles on this storage. + + 'write': key, value + + 'delete': key + + 'sync': true if it's from xSync(), false if it's from + xFileControl(). + + For efficiency's sake, all calls to this function should + be in the form: + + store.listeners && notifyListeners(...); + + Failing to do so will trigger an exceptin in this function (which + will be ignored but may produce a console warning). + */ + const notifyListeners = async function(eventName,store,...args){ + try{ + //cache.rxPageNoSuffix ??= /(\d+)$/; + if( store.keyPrefix && args[0] ){ + args[0] = args[0].replace(store.keyPrefix,''); + } + let u8enc, z0, z1, wcache; + for(const ear of store.listeners){ + const ev = Object.create(null); + ev.storageName = store.jzClass; + ev.type = eventName; + const decodePages = ear.decodePages; + const f = ear.events[eventName]; + if( f ){ + if( !ear.includeJournal && args[0]==='jrnl' ){ + continue; + } + if( 'write'===eventName && ear.decodePages && +args[0]>0 ){ + /* Decode pages to Uint8Array, caching the result in + wcache in case we have more listeners. */ + ev.data = [args[0]]; + if( wcache?.[args[0]] ){ + ev.data[1] = wcache[args[0]]; + continue; + } + u8enc ??= new TextEncoder('utf-8'); + z0 ??= cache.memBuffer(10); + z1 ??= cache.memBuffer(11); + const u = u8enc.encode(args[1]); + const heap = wasm.heap8u(); + heap.set(u, Number(z0)); + heap[wasm.ptr.addn(z0, u.length)] = 0; + const rc = kvvfsDecode(z0, z1, cache.buffer.n); + if( rc>0 ){ + wcache ??= Object.create(null); + wcache[args[0]] + = ev.data[1] + = heap.slice(Number(z1), wasm.ptr.addn(z1,rc)); + }else{ + continue; + } + }else{ + ev.data = args.length + ? ((args.length===1) ? args[0] : args) + : undefined; + } + try{f(ev)?.catch?.(catchForNotify)} + catch(e){ + warn("notifyListeners [",store.jzClass,"]",eventName,e); + } + } + } + }catch(e){ + catchForNotify(e); + } + }/*notifyListeners()*/; + + /** + Returns the storage object mapped to the given string zClass + (C-string pointer or JS string). + */ + const storageForZClass = (zClass)=> + 'string'===typeof zClass + ? cache.storagePool[zClass] + : cache.storagePool[wasm.cstrToJs(zClass)]; + +//#if nope + // fileForDb() works but we don't have a current need for it. + /** + Expects an (sqlite3*). Uses sqlite3_file_control() to extract its + (sqlite3_file*). On success it returns a new KVVfsFile instance + wrapping that pointer, which the caller must eventual call + dispose() on (which won't free the underlying pointer, just the + wrapper). Returns null if no handle is found (which would + indicate either that pDb is not using kvvfs or a severe bug in + its management). + */ + const fileForDb = function(pDb){ + const stack = wasm.pstack.pointer; + try{ + const pOut = wasm.pstack.allocPtr(); + return wasm.exports.sqlite3_file_control( + pDb, wasm.ptr.null, capi.SQLITE_FCNTL_FILE_POINTER, pOut + ) + ? null + : new KVVfsFile(wasm.peekPtr(pOut)); + }finally{ + wasm.pstack.restore(stack); + } + }; + + /** + Expects an object from the storagePool map. The $szPage and + $szDb members of each store.files entry is set to -1 in an attempt + to trigger those values to reload. + */ + const alertFilesToReload = (store)=>{ + try{ + for( const f of store.files ){ + // FIXME: we need to use one of the C APIs for this, maybe an + // fcntl. + f.$szPage = -1; + f.$szDb = -1n + } + }catch(e){ + error("alertFilesToReload()",store,e); + throw e; + } + }; +//#endif nope + + const kvvfsMakeKey = wasm.exports.sqlite3__wasm_kvvfsMakeKey; + /** + Returns a C string from kvvfsMakeKey() OR returns zKey. In the + former case the memory is static, so must be copied before a + second call. zKey MUST be a pointer passed to a VFS/file method, + to allow us to avoid an alloc and/or an snprintf(). It requires + C-string arguments for zClass and zKey. zClass may be NULL but + zKey may not. + */ + const zKeyForStorage = (store, zClass, zKey)=>{ + //debug("zKeyForStorage(",store, wasm.cstrToJs(zClass), wasm.cstrToJs(zKey)); + return (zClass && store.keyPrefix) ? kvvfsMakeKey(zClass, zKey) : zKey; + }; + + const jsKeyForStorage = (store,zClass,zKey)=> + wasm.cstrToJs(zKeyForStorage(store, zClass, zKey)); + + const storageGetDbSize = (store)=>+store.storage.getItem(store.keyPrefix + "sz"); + + /** + sqlite3_file pointers => objects, each of which has: + + .file = KVVfsFile instance + + .jzClass = JS-string form of f.$zClass + + .storage = Storage object. It is shared between a db and its + journal. + */ + const pFileHandles = new Map(); + + /** + Original WASM functions for methods we partially override. + */ + const originalMethods = { + vfs: Object.create(null), + ioDb: Object.create(null), + ioJrnl: Object.create(null) + }; + + /** Returns the appropriate originalMethods[X] instance for the + given a KVVfsFile instance. */ + const originalIoMethods = (kvvfsFile)=> + originalMethods[kvvfsFile.$isJournal ? 'ioJrnl' : 'ioDb']; + + const pVfs = new capi.sqlite3_vfs(kvvfsMethods.$pVfs); + const pIoDb = new capi.sqlite3_io_methods(kvvfsMethods.$pIoDb); + const pIoJrnl = new capi.sqlite3_io_methods(kvvfsMethods.$pIoJrnl); + const recordHandler = + Object.create(null)/** helper for some vfs + routines. Populated later. */; + const kvvfsInternal = Object.assign(Object.create(null),{ + pFileHandles, + cache, + storageForZClass, + KVVfsStorage, + /** + BUG: changing to a page size other than the default, + then vacuuming, corrupts the db. As a workaround, + until this is resolved, we forcibly disable + (pragma page_size=...) changes. + */ + disablePageSizeChange: true + }); + if( kvvfs.log ){ + // this is a test build + kvvfs.internal = kvvfsInternal; + } + + /** + Implementations for members of the object referred to by + sqlite3__wasm_kvvfs_methods(). We swap out some native + implementations with these so that we can use JS Storage for + their backing store. + */ + const methodOverrides = { + + /** + sqlite3_kvvfs_methods's member methods. These perform the + fetching, setting, and removal of storage keys on behalf of + kvvfs. In the native impl these write each db page to a + separate file. This impl stores each db page as a single + record in a Storage object which is mapped to zClass. + + A db's size is stored in a record named kvvfs[-storagename]-sz + and the journal is stored in kvvfs[-storagename]-jrnl. The + [-storagename] part is a remnant of the native impl (so that + it has unique filenames per db) and is only used for + localStorage and sessionStorage. We elide that part (to save + space) from other storage objects but retain it on those two + to avoid invalidating pre-version-2 session/localStorage dbs. + + The interface docs for these methods are in src/os_kv.c's + kvrecordRead(), kvrecordWrite(), and kvrecordDelete(). + */ + recordHandler: { + xRcrdRead: (zClass, zKey, zBuf, nBuf)=>{ + try{ + const jzClass = wasm.cstrToJs(zClass); + const store = storageForZClass(jzClass); + if( !store ) return -1; + const jXKey = jsKeyForStorage(store, zClass, zKey); + kvvfs?.log?.xRcrdRead && warn("xRcrdRead", jzClass, jXKey, nBuf, store ); + const jV = store.storage.getItem(jXKey); + if(null===jV) return -1; + const nV = jV.length /* We are relying 100% on v being + ** ASCII so that jV.length is equal + ** to the C-string's byte length. */; + if( 0 ){ + debug("xRcrdRead", jXKey, store, jV); + } + if(nBuf<=0) return nV; + else if(1===nBuf){ + wasm.poke(zBuf, 0); + return nV; + } + if( nBuf+1{ + try { + const store = storageForZClass(zClass); + const jxKey = jsKeyForStorage(store, zClass, zKey); + const jData = wasm.cstrToJs(zData); + kvvfs?.log?.xRcrdWrite && warn("xRcrdWrite",jxKey, store); + store.storage.setItem(jxKey, jData); + store.listeners && notifyListeners('write', store, jxKey, jData); + return 0; + }catch(e){ + error("kvrecordWrite()",e); + return cache.setError(e, capi.SQLITE_IOERR); + } + }, + + xRcrdDelete: (zClass, zKey)=>{ + try { + const store = storageForZClass(zClass); + const jxKey = jsKeyForStorage(store, zClass, zKey); + kvvfs?.log?.xRcrdDelete && warn("xRcrdDelete",jxKey, store); + store.storage.removeItem(jxKey); + store.listeners && notifyListeners('delete', store, jxKey); + return 0; + }catch(e){ + error("kvrecordDelete()",e); + return cache.setError(e, capi.SQLITE_IOERR); + } + } + }/*recordHandler*/, + + /** + Override certain operations of the underlying sqlite3_vfs and + the two sqlite3_io_methods instances so that we can tie + Storage objects to db names. + */ + vfs:{ + /* sqlite3_kvvfs_methods::pVfs's methods */ + xOpen: function(pProtoVfs,zName,pProtoFile,flags,pOutFlags){ + cache.popError(); + let zToFree /* alloc()'d memory for temp db name */; + if( 0 ){ + /* tester1.js makes it a lot further if we do this. */ + flags |= capi.SQLITE_OPEN_CREATE; + } + try{ + if( !zName ){ + zToFree = wasm.allocCString(""+pProtoFile+"." + +(Math.random() * 100000 | 0)); + zName = zToFree; + } + const jzClass = wasm.cstrToJs(zName); + kvvfs?.log?.xOpen && debug("xOpen",jzClass,"flags =",flags); + validateStorageName(jzClass, true); + if( (flags & (capi.SQLITE_OPEN_MAIN_DB + | capi.SQLITE_OPEN_TEMP_DB + | capi.SQLITE_OPEN_TRANSIENT_DB)) + && cache.rxJournalSuffix.test(jzClass) ){ + toss3(capi.SQLITE_ERROR, + "DB files may not have a '-journal' suffix."); + } + let s = storageForZClass(jzClass); + if( !s && !(flags & capi.SQLITE_OPEN_CREATE) ){ + toss3(capi.SQLITE_ERROR, "Storage not found:", jzClass); + } + const rc = originalMethods.vfs.xOpen(pProtoVfs, zName, pProtoFile, + flags, pOutFlags); + if( rc ) return rc; + let deleteAt0 = !!(capi.SQLITE_OPEN_DELETEONCLOSE & flags); + if(wasm.isPtr(arguments[1]/*original zName*/)){ + if(capi.sqlite3_uri_boolean(zName, "delete-on-close", 0)){ + deleteAt0 = true; + } + } + const f = new KVVfsFile(pProtoFile); + util.assert(f.$zClass, "Missing f.$zClass"); + f.addOnDispose(zToFree); + zToFree = undefined; + //debug("xOpen", jzClass, s); + if( s ){ + ++s.refc; + //no if( true===deleteAt0 ) s.deleteAtRefc0 = true; + s.files.push(f); + wasm.poke32(pOutFlags, flags); + }else{ + wasm.poke32(pOutFlags, flags | capi.SQLITE_OPEN_CREATE); + util.assert( !f.$isJournal, "Opening a journal before its db? "+jzClass ); + /* Map both zName and zName-journal to the same storage. */ + const nm = jzClass.replace(cache.rxJournalSuffix,''); + s = newStorageObj(nm); + installStorageAndJournal(s); + s.files.push(f); + s.deleteAtRefc0 = deleteAt0; + kvvfs?.log?.xOpen + && debug("xOpen installed storage handle [",nm, nm+"-journal","]", s); + } + pFileHandles.set(pProtoFile, {store: s, file: f, jzClass}); + s.listeners && notifyListeners('open', s, s.files.length); + return 0; + }catch(e){ + warn("xOpen:",e); + return cache.setError(e); + }finally{ + zToFree && wasm.dealloc(zToFree); + } + }/*xOpen()*/, + + xDelete: function(pVfs, zName, iSyncFlag){ + cache.popError(); + try{ + const jzName = wasm.cstrToJs(zName); + if( cache.rxJournalSuffix.test(jzName) ){ + recordHandler.xRcrdDelete(zName, cache.zKeyJrnl); + }/* + else: historically not done, but maybe otherwise delete + all db pages from storageForZClass(zName)? + */ + return 0; + }catch(e){ + warn("xDelete",e); + return cache.setError(e); + } + }, + + xAccess: function(pProtoVfs, zPath, flags, pResOut){ + cache.popError(); + try{ + const s = storageForZClass(zPath); + const jzPath = s?.jzClass || wasm.cstrToJs(zPath); + if( kvvfs?.log?.xAccess ){ + debug("xAccess",jzPath,"flags =", + flags,"*pResOut =",wasm.peek32(pResOut), + "store =",s); + } + if( !s ){ + // From the API docs: + /** The xAccess method returns [SQLITE_OK] on success or some + ** non-zero error code if there is an I/O error or if the name of + ** the file given in the second argument is illegal. + */ + // However, returning non-0 from here is fatal, so we don't do that. + try{validateStorageName(jzPath)} + catch(e){ + //warn("xAccess is ignoring name validation failure:",e); + wasm.poke32(pResOut, 0); + return 0; + } + } + if( s ){ + const key = s.keyPrefix+ + (cache.rxJournalSuffix.test(jzPath) ? "jrnl" : "1"); + const res = s.storage.getItem(key) ? 0 : 1; + /* This res value looks completely backwards to me, and + is the opposite of the native kvvfs's impl, but it's + working, whereas reimplementing the native one + faithfully does not. Read the lib-level code of where + this is invoked, my expectation is that we set res to 0 + for not-exists. */ + //warn("access res",jzPath,res); + wasm.poke32(pResOut, res); + }else{ + wasm.poke32(pResOut, 0); + } + return 0; + }catch(e){ + error('xAccess',e); + return cache.setError(e); + } + }, + + xRandomness: function(pVfs, nOut, pOut){ + const heap = wasm.heap8u(); + let i = 0; + const npOut = Number(pOut); + for(; i < nOut; ++i) heap[npOut + i] = (Math.random()*255000) & 0xFF; + return nOut; + }, + + xGetLastError: function(pVfs,nOut,pOut){ + const e = cache.popError(); + debug('xGetLastError',e); + if(e){ + const scope = wasm.scopedAllocPush(); + try{ + const [cMsg, n] = wasm.scopedAllocCString(e.message, true); + wasm.cstrncpy(pOut, cMsg, nOut); + if(n > nOut) wasm.poke8(wasm.ptr.add(pOut,nOut,-1), 0); + debug("set xGetLastError",e.message); + return (e.resultCode | 0) || capi.SQLITE_IOERR; + }catch(e){ + return capi.SQLITE_NOMEM; + }finally{ + wasm.scopedAllocPop(scope); + } + } + return 0; + } + +//#if nope + // these impls work but there's currently no pressing need _not_ use + // the native impls. + xCurrentTime: function(pVfs,pOut){ + wasm.poke64f(pOut, 2440587.5 + (Date.now()/86400000)); + return 0; + }, + + xCurrentTimeInt64: function(pVfs,pOut){ + wasm.poke64(pOut, (2440587.5 * 86400000) + Date.now()); + return 0; + } +//#endif + }/*.vfs*/, + + /** + kvvfs has separate sqlite3_api_methods impls for some of the + methods depending on whether it's a db or journal file. Some + of the methods use shared impls but others are specific to + either db or journal files. + */ + ioDb:{ + /* sqlite3_kvvfs_methods::pIoDb's methods */ + xClose: function(pFile){ + cache.popError(); + try{ + const h = pFileHandles.get(pFile); + kvvfs?.log?.xClose && debug("xClose", pFile, h); + if( h ){ + pFileHandles.delete(pFile); + const s = h.store;//storageForZClass(h.jzClass); + s.files = s.files.filter((v)=>v!==h.file); + if( --s.refc<=0 && s.deleteAtRefc0 ){ + deleteStorage(s); + } + originalMethods.ioDb/*same for journals*/.xClose(pFile); + h.file.dispose(); + s.listeners && notifyListeners('close', s, s.files.length); + }else{ + /* Can happen if xOpen fails */ + } + return 0; + }catch(e){ + error("xClose",e); + return cache.setError(e); + } + }, + + xFileControl: function(pFile, opId, pArg){ + cache.popError(); + try{ + const h = pFileHandles.get(pFile); + util.assert(h, "Missing KVVfsFile handle"); + kvvfs?.log?.xFileControl && debug("xFileControl",h,'op =',opId); + if( opId===capi.SQLITE_FCNTL_PRAGMA + && kvvfsInternal.disablePageSizeChange ){ + /* pArg== length-3 (char**) */ + //const argv = wasm.cArgvToJs(3, pArg); // the easy way + const zName = wasm.peekPtr(wasm.ptr.add(pArg, wasm.ptr.size)); + if( "page_size"===wasm.cstrToJs(zName) ){ + kvvfs?.log?.xFileControl + && debug("xFileControl pragma",wasm.cstrToJs(zName)); + const zVal = wasm.peekPtr(wasm.ptr.add(pArg, 2*wasm.ptr.size)); + if( zVal ){ + /* Without this, pragma page_size=N; followed by a + vacuum breaks the db. With this, it continues + working but does not actually change the page + size. */ + kvvfs?.log?.xFileControl + && warn("xFileControl pragma", h, + "NOT setting page size to", wasm.cstrToJs(zVal)); + h.file.$szPage = -1; + return 0/*corrupts: capi.SQLITE_NOTFOUND*/; + }else if( h.file.$szPage>0 ){ + kvvfs?.log?.xFileControl && + warn("xFileControl", h, "getting page size",h.file.$szPage); + wasm.pokePtr(pArg, wasm.allocCString(""+h.file.$szPage) + /* memory now owned by the library */); + return 0;//capi.SQLITE_NOTFOUND; + } + } + } + const rc = originalMethods.ioDb.xFileControl(pFile, opId, pArg); + if( 0==rc && capi.SQLITE_FCNTL_SYNC===opId ){ + h.store.listeners && notifyListeners('sync', h.store, false); + } + return rc; + }catch(e){ + error("xFileControl",e); + return cache.setError(e); + } + }, + + xSync: function(pFile,flags){ + cache.popError(); + try{ + const h = pFileHandles.get(pFile); + kvvfs?.log?.xSync && debug("xSync", h); + util.assert(h, "Missing KVVfsFile handle"); + const rc = originalMethods.ioDb.xSync(pFile, flags); + if( 0==rc && h.store.listeners ) notifyListeners('sync', h.store, true); + return rc; + }catch(e){ + error("xSync",e); + return cache.setError(e); + } + }, + +//#if not nope + // We override xRead/xWrite only for logging/debugging. They + // should otherwise be disabled (it's faster that way). + xRead: function(pFile,pTgt,n,iOff64){ + cache.popError(); + try{ + if( kvvfs?.log?.xRead ){ + const h = pFileHandles.get(pFile); + util.assert(h, "Missing KVVfsFile handle"); + debug("xRead", n, iOff64, h); + } + return originalMethods.ioDb.xRead(pFile, pTgt, n, iOff64); + }catch(e){ + error("xRead",e); + return cache.setError(e); + } + }, + xWrite: function(pFile,pSrc,n,iOff64){ + cache.popError(); + try{ + if( kvvfs?.log?.xWrite ){ + const h = pFileHandles.get(pFile); + util.assert(h, "Missing KVVfsFile handle"); + debug("xWrite", n, iOff64, h); + } + return originalMethods.ioDb.xWrite(pFile, pSrc, n, iOff64); + }catch(e){ + error("xWrite",e); + return cache.setError(e); + } + }, +//#endif nope + +//#if nope + xTruncate: function(pFile,i64){}, + xFileSize: function(pFile,pi64Out){}, + xLock: function(pFile,iLock){}, + xUnlock: function(pFile,iLock){}, + xCheckReservedLock: function(pFile,piOut){}, + xSectorSize: function(pFile){}, + xDeviceCharacteristics: function(pFile){} +//#endif + }/*.ioDb*/, + + ioJrnl:{ + /* sqlite3_kvvfs_methods::pIoJrnl's methods. Those set to true + are copied as-is from the ioDb objects. Others are specific + to journal files. */ + xClose: true, +//#if nope + xRead: function(pFile,pTgt,n,iOff64){}, + xWrite: function(pFile,pSrc,n,iOff64){}, + xTruncate: function(pFile,i64){}, + xSync: function(pFile,flags){}, + xFileControl: function(pFile, opId, pArg){}, + xFileSize: function(pFile,pi64Out){}, + xLock: true, + xUnlock: true, + xCheckReservedLock: true, + xSectorSize: true, + xDeviceCharacteristics: true +//#endif + }/*.ioJrnl*/ + }/*methodOverrides*/; + + debug("pVfs and friends", pVfs, pIoDb, pIoJrnl, + kvvfsMethods, capi.sqlite3_file.structInfo, + KVVfsFile.structInfo); + try { + util.assert( cache.buffer.n>1024*129, "Heap buffer is not large enough" + /* Native is SQLITE_KVOS_SZ is 133073 as of this writing */ ); + for(const e of Object.entries(methodOverrides.recordHandler)){ + // Overwrite kvvfsMethods's callbacks + const k = e[0], f = e[1]; + recordHandler[k] = f; + if( 0 ){ + // bug: this should work + kvvfsMethods.installMethod(k, f); + }else{ + kvvfsMethods[kvvfsMethods.memberKey(k)] = + wasm.installFunction(kvvfsMethods.memberSignature(k), f); + } + } + for(const e of Object.entries(methodOverrides.vfs)){ + // Overwrite some pVfs entries and stash the original impls + const k = e[0], f = e[1], km = pVfs.memberKey(k), + member = pVfs.structInfo.members[k] + || util.toss("Missing pVfs.structInfo[",k,"]"); + originalMethods.vfs[k] = wasm.functionEntry(pVfs[km]); + pVfs[km] = wasm.installFunction(member.signature, f); + } + for(const e of Object.entries(methodOverrides.ioDb)){ + // Similar treatment for pVfs.$pIoDb a.k.a. pIoDb... + const k = e[0], f = e[1], km = pIoDb.memberKey(k); + originalMethods.ioDb[k] = wasm.functionEntry(pIoDb[km]) + || util.toss("Missing native pIoDb[",km,"]"); + pIoDb[km] = wasm.installFunction(pIoDb.memberSignature(k), f); + } + for(const e of Object.entries(methodOverrides.ioJrnl)){ + // Similar treatment for pVfs.$pIoJrnl a.k.a. pIoJrnl... + const k = e[0], f = e[1], km = pIoJrnl.memberKey(k); + originalMethods.ioJrnl[k] = wasm.functionEntry(pIoJrnl[km]) + || util.toss("Missing native pIoJrnl[",km,"]"); + if( true===f ){ + /* use pIoDb's copy */ + pIoJrnl[km] = pIoDb[km] || util.toss("Missing copied pIoDb[",km,"]"); + }else{ + pIoJrnl[km] = wasm.installFunction(pIoJrnl.memberSignature(k), f); + } + } + }finally{ + kvvfsMethods.dispose(); + pVfs.dispose(); + pIoDb.dispose(); + pIoJrnl.dispose(); + } + + /* + That gets all of the low-level bits out of the way. What follows + are the public API additions. + */ + + /** + Clears all storage used by the kvvfs DB backend, deleting any + DB(s) stored there. + + Its argument must be the name of a kvvfs storage object: + + - 'session' + - 'local' + - '' - see below. + - A transient kvvfs storage object name. + + In the first two cases, only sessionStorage resp. localStorage is + cleared. An empty string resolves to both 'local' and 'session' + storage. + + Returns the number of entries cleared. + + As of kvvfs version 2: + + This API is available in Worker threads but does not have access + to localStorage or sessionStorage in them. Prior versions did not + include this API in Worker threads. + + Differences in this function in version 2: + + - It accepts an arbitrary storage name. In v1 this was a silent + no-op for any names other than ('local','session',''). + + - It throws if a db currently has the storage opened UNLESS the + storage object is localStorage or sessionStorage. That version 1 + did not throw for this case was due to an architectural + limitation which has since been overcome, but removal of + JsStorageDb.prototype.clearStorage() would be a backwards compatibility + break, so this function permits wiping the storage for those two + cases even if they are opened. Use with case. + */ + const sqlite3_js_kvvfs_clear = function callee(which){ + if( ''===which ){ + return callee('local') + callee('session'); + } + const store = storageForZClass(which); + if( !store ) return 0; + if( store.files.length ){ + if( globalThis.localStorage===store.storage + || globalThis.sessionStorage===store.storage ){ + /* backwards compatibility: allow these to be cleared + while opened. */ + }else{ + /* Interestingly, kvvfs recovers just fine when the storage is + wiped, so long as the db is not in use and its schema is + recreated before it's used, but client apps should not have + to be faced with that eventuality mid-query (where it + _will_ cause failures). Therefore we disallow it when + storage handles are opened. Kvvfs version 1 could not + detect this case - see the if() block above. + */ + toss3(capi.SQLITE_ACCESS, + "Cannot clear in-use database storage."); + } + } + const s = store.storage; + const toRm = [] /* keys to remove */; + let i, n = s.length; + //debug("kvvfs_clear",store,s); + for( i = 0; i < n; ++i ){ + const k = s.key(i); + //debug("kvvfs_clear ?",k); + if(!store.keyPrefix || k.startsWith(store.keyPrefix)) toRm.push(k); + } + toRm.forEach((kk)=>s.removeItem(kk)); + //alertFilesToReload(store); + return toRm.length; + }; + + /** + This routine estimates the approximate amount of + storage used by the given kvvfs back-end. + + Its arguments are as documented for sqlite3_js_kvvfs_clear(), + only the operation this performs is different. + + The returned value is twice the "length" value of every matching + key and value, noting that JavaScript stores each character in 2 + bytes. + + The returned size is not authoritative from the perspective of + how much data can fit into localStorage and sessionStorage, as + the precise algorithms for determining those limits are + unspecified and may include per-entry overhead invisible to + clients. + */ + const sqlite3_js_kvvfs_size = function callee(which){ + if( ''===which ){ + return callee('local') + callee('session'); + } + const store = storageForZClass(which); + if( !store ) return 0; + const s = store.storage; + let i, sz = 0; + for(i = 0; i < s.length; ++i){ + const k = s.key(i); + if(!store.keyPrefix || k.startsWith(store.keyPrefix)){ + sz += k.length; + sz += s.getItem(k).length; + } + } + return sz * 2 /* because JS uses 2-byte char encoding */; + }; + + /** + Exports a kvvfs storage object to an object, optionally + JSON-friendly. + + Usages: + + thisfunc(storageName); + thisfunc(options); + + In the latter case, the options object must be an object with + the following properties: + + - "name" (string) required. The storage to export. + + - "decodePages" (bool=false). If true, the .pages result property + holdes Uint8Array objects holding the raw binary-format db + pages. The default is to use kvvfs-encoded string pages + (JSON-friendly). + + - "includeJournal" (bool=false). If true and the db has a current + journal, it is exported as well. (Kvvfs journals are stored as a + single record within the db's storage object.) + + The returned object is structured as follows... + + - "name": the name of the storage. This is 'local' or 'session' + for localStorage resp. sessionStorage, and an arbitrary name for + transient storage. This propery may be changed before passing + this object to sqlite3_js_kvvfs_import() in order to + import into a different storage object. + + - "timestamp": the time this function was called, in Unix + epoch milliseconds. + + - "size": the unencoded db size. + + - "journal": if options.includeJournal is true and this db has a + journal, it is stored as a string here, otherwise this property + is not set. + + - "pages": An array holding the raw encoded db pages in their + proper order. + + Throws if this db is not opened. + + The encoding of the underlying database is not part of this + interface - it is simply passed on as-is. Interested parties are + directed to src/os_kv.c in the SQLite source tree, with the + caveat that that code also does not offer a public interface. + i.e. the encoding is a private implementation detail of kvvfs. + The format may be changed in the future but kvvfs will continue + to support the current form. + + Added in version @kvvfs-v2-added-in@. + */ + const sqlite3_js_kvvfs_export = function callee(...args){ + let opt; + if( 1===args.length && 'object'===typeof args[0] ){ + opt = args[0]; + }else if(args.length){ + opt = Object.assign(Object.create(null),{ + name: args[0], + //decodePages: true + }); + } + const store = opt ? storageForZClass(opt.name) : null; + if( !store ){ + toss3(capi.SQLITE_NOTFOUND, + "There is no kvvfs storage named",opt?.name); + } + //debug("store to export=",store); + const s = store.storage; + const rc = Object.assign(Object.create(null),{ + name: store.jzClass, + timestamp: Date.now(), + pages: [] + }); + const pages = Object.create(null); + let xpages; + const keyPrefix = store.keyPrefix; + const rxTail = keyPrefix + ? /^kvvfs-[^-]+-(\w+)/ /* X... part of kvvfs-NAME-X... */ + : undefined; + let i = 0, n = s.length; + for( ; i < n; ++i ){ + const k = s.key(i); + if( !keyPrefix || k.startsWith(keyPrefix) ){ + let kk = (keyPrefix ? rxTail.exec(k) : undefined)?.[1] ?? k; + switch( kk ){ + case 'jrnl': + if( opt.includeJournal ) rc.journal = s.getItem(k); + break; + case 'sz': + rc.size = +s.getItem(k); + break; + default: + kk = +kk /* coerce to number */; + if( !util.isInt32(kk) || kk<=0 ){ + toss3(capi.SQLITE_RANGE, "Malformed kvvfs key: "+k); + } + if( opt.decodePages ){ + const spg = s.getItem(k), + n = spg.length, + z = cache.memBuffer(0), + zDec = cache.memBuffer(1), + heap = wasm.heap8u()/* MUST be inited last*/; + let i = 0; + for( ; i < n; ++i ){ + heap[wasm.ptr.add(z, i)] = spg.codePointAt(i) & 0xff; + } + heap[wasm.ptr.add(z, i)] = 0; + //debug("Decoding",i,"page bytes"); + const nDec = kvvfsDecode( + z, zDec, cache.buffer.n + ); + //debug("Decoded",nDec,"page bytes"); + pages[kk] = heap.slice(Number(zDec), wasm.ptr.addn(zDec, nDec)); + }else{ + pages[kk] = s.getItem(k); + } + break; + } + } + } + if( opt.decodePages ) cache.memBufferFree(1); + /* Now sort the page numbers and move them into an array. In JS + property keys are always strings, so we have to coerce them to + numbers so we can get them sorted properly for the array. */ + Object.keys(pages).map((v)=>+v).sort().forEach( + (v)=>rc.pages.push(pages[v]) + ); + return rc; + }/* sqlite3_js_kvvfs_export */; + + /** + The counterpart of sqlite3_js_kvvfs_export(). Its + argument must be the result of that function() or + a compatible one. + + This either replaces the contents of an existing transient + storage object or installs one named exp.name, setting + the storage's db contents to that of the exp object. + + Throws on error. Error conditions include: + + - The given storage object is currently opened by any db. + Performing this page-by-page import would invoke undefined + behavior on them. + + - Malformed input object. + + If it throws after starting the import then it clears the storage + before returning, to avoid leaving the db in an undefined + state. It may throw for any of the above-listed conditions before + reaching that step, in which case the db is not modified. If + exp.name refers to a new storage name then if it throws, the name + does not get installed. + + Added in version @kvvfs-v2-added-in@. + */ + const sqlite3_js_kvvfs_import = function(exp, overwrite=false){ + if( !exp?.timestamp + || !exp.name + || undefined===exp.size + || !Array.isArray(exp.pages) ){ + toss3(capi.SQLITE_MISUSE, "Malformed export object."); + }else if( !exp.size + || (exp.size !== (exp.size | 0)) + //|| (exp.size % cache.fixedPageSize) + || exp.size>=0x7fffffff ){ + toss3(capi.SQLITE_RANGE, "Invalid db size: "+exp.size); + } + + validateStorageName(exp.name); + let store = storageForZClass(exp.name); + const isNew = !store; + if( store ){ + if( !overwrite ){ + //warn("Storage exists:",arguments,store); + toss3(capi.SQLITE_ACCESS, + "Storage '"+exp.name+"' already exists and", + "overwrite was not specified."); + }else if( !store.files || !store.jzClass ){ + toss3(capi.SQLITE_ERROR, + "Internal storage object", exp.name,"seems to be malformed."); + }else if( store.files.length ){ + toss3(capi.SQLITE_IOERR_ACCESS, + "Cannot import db storage while it is in use."); + } + sqlite3_js_kvvfs_clear(exp.name); + }else{ + store = newStorageObj(exp.name); + //warn("Installing new storage:",store); + } + //debug("Importing store",store.poolEntry.files.length, store); + //debug("object to import:",exp); + const keyPrefix = kvvfsKeyPrefix(exp.name); + let zEnc; + try{ + /* Force the native KVVfsFile instances to re-read the db + and page size. */; + const s = store.storage; + s.setItem(keyPrefix+'sz', exp.size); + if( exp.journal ) s.setItem(keyPrefix+'jrnl', exp.journal); + if( exp.pages[0] instanceof Uint8Array ){ + /* raw binary pages */ + //debug("pages",exp.pages); + exp.pages.forEach((u,ndx)=>{ + const n = u.length; + if( 0 && cache.fixedPageSize !== n ){ + util.toss3(capi.SQLITE_RANGE,"Unexpected page size:", n); + } + zEnc ??= cache.memBuffer(1); + const zBin = cache.memBuffer(0), + heap = wasm.heap8u()/*MUST be inited last*/; + /* Copy u to the heap and encode the heap copy via C. This + is _presumably_ faster than porting the encoding algo to + JS. */ + heap.set(u, Number(zBin)); + heap[wasm.ptr.addn(zBin,n)] = 0; + const rc = kvvfsEncode(zBin, n, zEnc); + util.assert( rc < cache.buffer.n, + "Impossibly long output - possibly smashed the heap" ); + util.assert( 0===wasm.peek8(wasm.ptr.add(zEnc,rc)), + "Expecting NUL-terminated encoded output" ); + const jenc = wasm.cstrToJs(zEnc); + //debug("(un)encoded page:",u,jenc); + s.setItem(keyPrefix+(ndx+1), jenc); + }); + }else if( exp.pages[0] ){ + /* kvvfs-encoded pages */ + exp.pages.forEach((v,ndx)=>s.setItem(keyPrefix+(ndx+1), v)); + } + if( isNew ) installStorageAndJournal(store); + }catch{ + if( !isNew ){ + try{sqlite3_js_kvvfs_clear(exp.name);}catch(ee){/*ignored*/} + } + }finally{ + if( zEnc ) cache.memBufferFree(1); + } + return this; + }; + + /** + If no kvvfs storage exists with the given name, one is + installed. If one exists, its reference count is increased so + that it won't be freed by the closing of a database or journal + file. + + Throws if the name is not valid for a new storage object. + + Added in version @kvvfs-v2-added-in@. + */ + const sqlite3_js_kvvfs_reserve = function(name){ + let store = storageForZClass(name); + if( store ){ + ++store.refc; + return; + } + validateStorageName(name); + installStorageAndJournal(newStorageObj(name)); + }; + + /** + Conditionally "unlinks" a kvvfs storage object, reducing its + reference count by 1. + + This is a no-op if name ends in "-journal" or refers to a + built-in storage object. + + It will not lower the refcount below the number of + currently-opened db/journal files for the storage (so that it + cannot delete it out from under them). + + If the refcount reaches 0 then the storage object is + removed. + + Returns true if it reduces the refcount, else false. A result of + true does not necessarily mean that the storage unit was removed, + just that its refcount was lowered. Similarly, a result of false + does not mean that the storage is removed - it may still have + opened handles. + + Added in version @kvvfs-v2-added-in@. + */ + const sqlite3_js_kvvfs_unlink = function(name){ + const store = storageForZClass(name); + if( !store + || kvvfsIsPersistentName(store.jzClass) + || isBuiltinName(store.jzClass) + || cache.rxJournalSuffix.test(name) ) return false; + if( store.refc > store.files.length || 0===store.files.length ){ + if( --store.refc<=0 ){ + /* Ignoring deleteAtRefc0 for an explicit unlink */ + deleteStorage(store); + } + return true; + } + return false; + }; + + /** + Adds an event listener to a kvvfs storage object. The idea is + that this can be used to asynchronously back up one kvvfs storage + object to another or another channel entirely. (The caveat in the + latter case is that kvvfs's format is not readily consumable by + downstream code.) + + Its argument must be an object with the following properties: + + - storage: the name of the kvvfs storage object. + + - reserve [=false]: if true, sqlite3_js_kvvfs_reserve() is used + to ensure that the storage exists if it does not already. + If this is false and the storage does not exist then an + exception is thrown. + + - events: an object which may have any of the following + callback function properties: open, close, write, delete. + + - decodePages [=false]: if true, write events will receive each + db page write in the form of a Uint8Array holding the raw binary + db page. The default is to emit the kvvfs-format page because it + requires no extra work, we already have it in hand, and it's + often smaller. It's not great for interchange, though. + + - includeJournal [=false]: if true, writes and deletes of + "jrnl" records are included. If false, no events are sent + for journal updates. + + Passing the same object to sqlite3_js_kvvfs_unlisten() will + remove the listener. + + Each one of the events callbacks will be called asynchronously + when the given storage performs those operations. They may be + asynchronous functions but are not required to be (the events are + fired async either way, but making the event callbacks async may + be advantageous when multiple listeners are involved). All + exceptions, including those via Promises, are ignored but may (or + may not) trigger warning output on the console. + + Each callback gets passed a single object with the following + properties: + + .type = the same as the name of the callback + + .storageName = the name of the storage object + + .data = callback-dependent: + + - 'open' and 'close' get an integer, the number of + currently-opened handles on the storage. + + - 'write' gets a length-two array holding the key and value which + were written. The key is always a string, even if it's a db page + number. For db-page records, the value's type depends on + opt.decodePages. All others, including the journal, are strings. + (The journal, being a kvvfs-specific format, is delivered in + that same JSON-friendly format.) More details below. + + - 'delete' gets the string-type key of the deleted record. + + - 'sync' gets a boolean value: true if it was triggered by db + file's xSync(), false if it was triggered by xFileControl(). The + latter triggers before the xSync() and also triggers if the DB + has PRAGMA SYNCHRONOUS=OFF (in which case xSync() is not + triggered). + + The key/value arguments to 'write', and key argument to 'delete', + are in one of the following forms: + + - 'sz' = the unencoded db size as a string. This specific key is + key is never deleted, so is only ever passed to 'write' events. + + - 'jrnl' = the current db journal as a kvvfs-encoded string. This + journal format is not useful anywhere except in the kvvfs + internals. These events are not fired if opt.includeJournal is + false. + + - '[1-9][0-9]*' (a db page number) = Its type depends on + opt.decodePages. These may be written and deleted in arbitrary + order. + + Design note: JS has StorageEvents but only in the main thread, + which is why the listeners are not based on that. + + Added in version @kvvfs-v2-added-in@. + */ + const sqlite3_js_kvvfs_listen = function(opt){ + if( !opt || 'object'!==typeof opt ){ + toss3(capi.SQLITE_MISUSE, "Expecting a listener object."); + } + let store = storageForZClass(opt.storage); + if( !store ){ + if( opt.storage && opt.reserve ){ + sqlite3_js_kvvfs_reserve(opt.storage); + store = storageForZClass(opt.storage); + util.assert(store, + "Unexpectedly cannot fetch reserved storage " + +opt.storage); + }else{ + toss3(capi.SQLITE_NOTFOUND,"No such storage:",opt.storage); + } + } + if( opt.events ){ + (store.listeners ??= []).push(opt); + } + }; + + /** + Removes the kvvfs event listeners for the given options + object. It must be passed the same object instance which was + passed to sqlite3_js_kvvfs_listen(). + + This has no side effects if opt is invalid or is not a match for + any listeners. + + Return true if it unregisters its argument, else false. + + Added in version @kvvfs-v2-added-in@. + */ + const sqlite3_js_kvvfs_unlisten = function(opt){ + const store = storageForZClass(opt?.storage); + if( store?.listeners && opt.events ){ + const n = store.listeners.length; + store.listeners = store.listeners.filter((v)=>v!==opt); + const rc = n>store.listeners.length; + if( !store.listeners.length ){ + // to speed up downstream checks for listeners + store.listeners = undefined; + } + return rc; + } + return false; + }; + + sqlite3.kvvfs.reserve = sqlite3_js_kvvfs_reserve; + sqlite3.kvvfs.import = sqlite3_js_kvvfs_import; + sqlite3.kvvfs.export = sqlite3_js_kvvfs_export; + sqlite3.kvvfs.unlink = sqlite3_js_kvvfs_unlink; + sqlite3.kvvfs.listen = sqlite3_js_kvvfs_listen; + sqlite3.kvvfs.unlisten = sqlite3_js_kvvfs_unlisten; + sqlite3.kvvfs.exists = (name)=>!!storageForZClass(name); + sqlite3.kvvfs.estimateSize = sqlite3_js_kvvfs_size; + sqlite3.kvvfs.clear = sqlite3_js_kvvfs_clear; + + + if( globalThis.Storage ){ + /** + Prior to version 2, kvvfs was only available in the main + thread. We retain that for the v1 APIs, exposing them only in + the main UI thread. As of version 2, kvvfs is available in all + threads but only via its v2 interface (sqlite3.kvvfs). + + These versions have a default argument value of "" which the v2 + versions lack. + */ + capi.sqlite3_js_kvvfs_size = (which="")=>sqlite3_js_kvvfs_size(which); + capi.sqlite3_js_kvvfs_clear = (which="")=>sqlite3_js_kvvfs_clear(which); + } + +//#if not omit-oo1 + if(sqlite3.oo1?.DB){ + /** + Functionally equivalent to DB(storageName,'c','kvvfs') except + that it throws if the given storage name is not one of 'local' + or 'session'. + + As of version 3.46, the argument may optionally be an options + object in the form: + + { + filename: 'session'|'local', + ... etc. (all options supported by the DB ctor) + } + + noting that the 'vfs' option supported by main DB + constructor is ignored here: the vfs is always 'kvvfs'. + */ + const DB = sqlite3.oo1.DB; + sqlite3.oo1.JsStorageDb = function( + storageName = sqlite3.oo1.JsStorageDb.defaultStorageName + ){ + const opt = DB.dbCtorHelper.normalizeArgs(...arguments); + opt.vfs = 'kvvfs'; + if( 0 ){ + // Current tests rely on these, but that's arguably a bug + if( opt.flags ) opt.flags = 'cw'+opt.flags; + else opt.flags = 'cw'; + } + switch( opt.filename ){ + /* sqlite3_open(), in these builds, recognizes the names + below and performs some magic which we want to bypass + here for sanity's sake. */ + case ":sessionStorage:": opt.filename = 'session'; break; + case ":localStorage:": opt.filename = 'local'; break; + } + const m = /(file:(\/\/)?)([^?]+)/.exec(opt.filename); + validateStorageName( m ? m[3] : opt.filename); + DB.dbCtorHelper.call(this, opt); + }; + sqlite3.oo1.JsStorageDb.defaultStorageName + = cache.storagePool.session ? 'session' : nameOfThisThreadStorage; + const jdb = sqlite3.oo1.JsStorageDb; + jdb.prototype = Object.create(DB.prototype); + jdb.clearStorage = sqlite3_js_kvvfs_clear; + /** + DEPRECATED: the inherited method of this name (as opposed to + the "static" class method) is deprecated with version 2 of + kvvfs. This function will, for backwards comaptibility, + continue to work with localStorage and sessionStorage, but will + throw for all other storage because they are opened. Version 1 + was not capable of recognizing that the storage was opened so + permitted wiping it out at any time, but that was arguably a + bug. + + Clears this database instance's storage or throws if this + instance has been closed. Returns the number of + database pages which were cleaned up. + */ + jdb.prototype.clearStorage = function(){ + return jdb.clearStorage(this.affirmOpen().dbFilename(), true); + }; + /** Equivalent to sqlite3_js_kvvfs_size(). */ + jdb.storageSize = sqlite3_js_kvvfs_size; + /** + Returns the _approximate_ number of bytes this database takes + up in its storage or throws if this instance has been closed. + */ + jdb.prototype.storageSize = function(){ + return jdb.storageSize(this.affirmOpen().dbFilename(), true); + }; + }/*sqlite3.oo1.JsStorageDb*/ +//#endif not omit-oo1 + + if( sqlite3.__isUnderTest && sqlite3.vtab ){ + /** + An eponymous vtab for inspecting the kvvfs state. This is only + intended for use in testing and development, not part of the + public API. + */ + const cols = Object.assign(Object.create(null),{ + rowid: {type: 'INTEGER'}, + name: {type: 'TEXT'}, + nRef: {type: 'INTEGER'}, + nOpen: {type: 'INTEGER'}, + isTransient: {type: 'INTEGER'}, + dbSize: {type: 'INTEGER'} + }); + Object.keys(cols).forEach((v,i)=>cols[v].colId = i); + + const VT = sqlite3.vtab; + const ProtoCursor = Object.assign(Object.create(null),{ + row: function(){ + return cache.storagePool[this.names[this.rowid]]; + } + }); + Object.assign(Object.create(ProtoCursor),{ + rowid: 0, + names: Object.keys(cache.storagePool) + .filter(v=>!cache.rxJournalSuffix.test(v)) + }); + const cursorState = function(cursor, reset){ + const o = (cursor instanceof capi.sqlite3_vtab_cursor) + ? cursor + : VT.xCursor.get(cursor); + if( reset || !o.vTabState ){ + o.vTabState = Object.assign(Object.create(ProtoCursor),{ + rowid: 0, + names: Object.keys(cache.storagePool) + .filter(v=>!cache.rxJournalSuffix.test(v)) + }); + } + return o.vTabState; + }; + + const dbg = 1 ? ()=>{} : (...args)=>debug("vtab",...args); + + const theModule = function f(){ + return f.mod ??= new sqlite3.capi.sqlite3_module().setupModule({ + catchExceptions: true, + methods: { + xConnect: function(pDb, pAux, argc, argv, ppVtab, pzErr){ + dbg("xConnect"); + try{ + const xcol = []; + Object.keys(cols).forEach((k)=>{ + xcol.push(k+" "+cols[k].type); + }); + const rc = capi.sqlite3_declare_vtab( + pDb, "CREATE TABLE ignored("+xcol.join(',')+")" + ); + if(0===rc){ + const t = VT.xVtab.create(ppVtab); + util.assert( + (t === VT.xVtab.get(wasm.peekPtr(ppVtab))), + "output pointer check failed" + ); + } + return rc; + }catch(e){ + return VT.xError('xConnect', e, capi.SQLITE_ERROR); + } + }, + xCreate: wasm.ptr.null, // eponymous only + //xCreate: true, // copy xConnect, i.e. also eponymous only + xDisconnect: function(pVtab){ + dbg("xDisconnect",...arguments); + VT.xVtab.dispose(pVtab); + return 0; + }, + xOpen: function(pVtab, ppCursor){ + dbg("xOpen",...arguments); + VT.xCursor.create(ppCursor); + return 0; + }, + xClose: function(pCursor){ + dbg("xClose",...arguments); + const c = VT.xCursor.unget(pCursor); + delete c.vTabState; + c.dispose(); + return 0; + }, + xNext: function(pCursor){ + dbg("xNext",...arguments); + const c = VT.xCursor.get(pCursor); + ++cursorState(c).rowid; + return 0; + }, + xColumn: function(pCursor, pCtx, iCol){ + dbg("xColumn",...arguments); + //const c = VT.xCursor.get(pCursor); + const st = cursorState(pCursor); + const store = st.row(); + util.assert(store, "Unexpected xColumn call"); + switch(iCol){ + case cols.rowid.colId: + capi.sqlite3_result_int(pCtx, st.rowid); + break; + case cols.name.colId: + capi.sqlite3_result_text(pCtx, store.jzClass, -1, capi.SQLITE_TRANSIENT); + break; + case cols.nRef.colId: + capi.sqlite3_result_int(pCtx, store.refc); + break; + case cols.nOpen.colId: + capi.sqlite3_result_int(pCtx, store.files.length); + break; + case cols.isTransient.colId: + capi.sqlite3_result_int(pCtx, !!store.deleteAtRefc0); + break; + case cols.dbSize.colId: + capi.sqlite3_result_int(pCtx, storageGetDbSize(store)); + break; + default: + capi.sqlite3_result_error(pCtx, "Invalid column id: "+iCol); + return capi.SQLITE_RANGE; + } + return 0; + }, + xRowid: function(pCursor, ppRowid64){ + dbg("xRowid",...arguments); + const st = cursorState(pCursor); + VT.xRowid(ppRowid64, st.rowid); + return 0; + }, + xEof: function(pCursor){ + const st = cursorState(pCursor); + dbg("xEof?="+(!st.row()),...arguments); + return !st.row(); + }, + xFilter: function(pCursor, idxNum, idxCStr, + argc, argv/* [sqlite3_value* ...] */){ + dbg("xFilter",...arguments); + const st = cursorState(pCursor, true); + return 0; + }, + xBestIndex: function(pVtab, pIdxInfo){ + dbg("xBestIndex",...arguments); + //const t = VT.xVtab.get(pVtab); + const pii = new capi.sqlite3_index_info(pIdxInfo); + pii.$estimatedRows = cache.storagePool.size; + pii.$estimatedCost = 1.0; + pii.dispose(); + return 0; + } + } + })/*setupModule*/; + }/*theModule()*/; + + sqlite3.kvvfs.create_module = function(pDb, name="sqlite_kvvfs"){ + return capi.sqlite3_create_module(pDb, name, theModule(), + wasm.ptr.null); + }; + + }/* virtual table */ + +//#if nope + /** + The idea here is a simpler wrapper for listening to kvvfs + changes. Clients would override its onXyz() event methods + instead of providing callbacks for sqlite3.kvvfs.listen(), the + main (only?) benefit of which is that this class would do the + sorting-out and validation of event state before calling the + overloaded callbacks. + */ + kvvfs.Listener = class KvvfsListener { + #store; + #listener; + + constructor(opt){ + this.#listenTo(opt); + } + + #event(ev){ + switch(ev.type){ + case 'open': this.onOpen(ev.data); break; + case 'close': this.onClose(ev.data); break; + case 'sync': this.onSync(ev.data); break; + case 'delete': + switch(ev.data){ + case 'jrnl': break; + default:{ + const n = +ev.data; + util.assert( n>0, "Expecting positive db page number" ); + this.onPageChange(n, null); + break; + } + } + break; + case 'write':{ + const key = ev.data[0], val = ev.data[1]; + switch( key ){ + case 'jrnl': break; + case 'sz':{ + const sz = +val; + util.assert( sz>0, "Expecting a db page number" ); + this.onSizeChange(sz); + break; + } + default: + T.assert( +key>0, "Expecting a positive db page number" ); + this.onPageChange(+key, val); + break; + } + break; + } + } + } + + #listenTo(opt){ + if(this.#listener){ + sqlite3_js_kvvfs_unlisten(this.#listener); + this.#listener = undefined; + } + const eventHandler = async function(ev){this.event(ev)}.bind(this); + const li = Object.assign( + { /* Defaults */ + reserve: false, + includeJournal: false, + decodePages: false, + storage: null + }, + (/*client options*/opt||{}), + {/*hard-coded options*/ + events: Object.assign(Object.create(null),{ + 'open': eventHandler, + 'close': eventHandler, + 'write': eventHandler, + 'delete': eventHandler, + 'sync': eventHandler + }) + } + ); + sqlite3_js_kvvfs_listen(li); + this.#listener = li; + } + + async onSizeChange(sz){} + async onPageChange(pgNo,content/*null for delete*/){} + async onSync(mode/*true=xSync, false=xFileControl*/){} + async onOpen(count){} + async onClose(count){} + }/*KvvfsListener*/; +//#endif nope + +})/*globalThis.sqlite3ApiBootstrap.initializers*/; +//#savepoint rollback +//#endif not omit-kvvfs diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js index 69be338b0..c1fee5e1a 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js @@ -358,7 +358,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ try{ const [cMsg, n] = wasm.scopedAllocCString(e.message, true); wasm.cstrncpy(pOut, cMsg, nOut); - if(n > nOut) wasm.poke8(pOut + nOut - 1, 0); + if(n > nOut) wasm.poke8(wasm.ptr.add(pOut,nOut,-1), 0); }catch(e){ return capi.SQLITE_NOMEM; }finally{ @@ -410,7 +410,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }/*vfsMethods*/; /** - Creates and initializes an sqlite3_vfs instance for an + Creates, initializes, and returns an sqlite3_vfs instance for an OpfsSAHPool. The argument is the VFS's name (JS string). Throws if the VFS name is already registered or if something @@ -1157,8 +1157,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ described at the end of these docs. This function accepts an options object to configure certain - parts but it is only acknowledged for the very first call and - ignored for all subsequent calls. + parts but it is only acknowledged for the very first call for + each distinct name and ignored for all subsequent calls with that + same name. The options, in alphabetical order: @@ -1224,7 +1225,14 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - Paths given to it _must_ be absolute. Relative paths will not be properly recognized. This is arguably a bug but correcting it requires some hoop-jumping in routines which have no business - doing such tricks. + doing such tricks. (2026-01-19 (2.5 years later): the specifics + are lost to history, but this was a side effect of xOpen() + receiving an immutable C-string filename, to which no implicit + "/" can be prefixed without causing a discrepancy between what + the user provided and what the VFS stores. Its conceivable that + that quirk could be glossed over in xFullPathname(), but + regressions when doing so cannot be ruled out, so there are no + current plans to change this behavior.) - It is possible to install multiple instances under different names, each sandboxed from one another inside their own private diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 2b636460d..ffa90ed06 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -16,7 +16,7 @@ asynchronous Origin-Private FileSystem (OPFS) APIs using a second Worker, implemented in sqlite3-opfs-async-proxy.js. This file is intended to be appended to the main sqlite3 JS deliverable somewhere - after sqlite3-api-oo1.js and before sqlite3-api-cleanup.js. + after sqlite3-api-oo1.js. */ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ @@ -589,7 +589,7 @@ const installOpfsVfs = function callee(options){ /** Returns an array of the deserialized state stored by the most - recent serialize() operation (from from this thread or the + recent serialize() operation (from this thread or the counterpart thread), or null if the serialization buffer is empty. If passed a truthy argument, the serialization buffer is cleared after deserialization. @@ -924,7 +924,7 @@ const installOpfsVfs = function callee(options){ fh.filename = zName; fh.sab = new SharedArrayBuffer(state.fileBufferSize); fh.flags = flags; - fh.readOnly = !(sqlite3.SQLITE_OPEN_CREATE & flags) + fh.readOnly = !(capi.SQLITE_OPEN_CREATE & flags) && !!(flags & capi.SQLITE_OPEN_READONLY); const rc = opRun('xOpen', pFile, zName, flags, opfsFlags); if(!rc){ @@ -1441,7 +1441,7 @@ installOpfsVfs.defaultProxyUri = globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ try{ let proxyJs = installOpfsVfs.defaultProxyUri; - if(sqlite3.scriptInfo.sqlite3Dir){ + if( sqlite3?.scriptInfo?.sqlite3Dir ){ installOpfsVfs.defaultProxyUri = sqlite3.scriptInfo.sqlite3Dir + proxyJs; //sqlite3.config.warn("installOpfsVfs.defaultProxyUri =",installOpfsVfs.defaultProxyUri); diff --git a/ext/wasm/api/sqlite3-vtab-helper.c-pp.js b/ext/wasm/api/sqlite3-vtab-helper.c-pp.js index 4c2338fc5..80f4bfac2 100644 --- a/ext/wasm/api/sqlite3-vtab-helper.c-pp.js +++ b/ext/wasm/api/sqlite3-vtab-helper.c-pp.js @@ -172,10 +172,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Works like unget() plus it calls dispose() on the StructType object. */ - dispose: (pCObj)=>{ - const o = __xWrap(pCObj,true); - if(o) o.dispose(); - } + dispose: (pCObj)=>__xWrap(pCObj,true)?.dispose?.() }); }; diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 4d5e9b296..dbfcdb704 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -93,6 +93,18 @@ #undef SQLITE_ENABLE_API_ARMOR #define SQLITE_ENABLE_API_ARMOR 1 +/**********************************************************************/ +/* SQLITE_EXPERIMENTAL_PRAGMA_20251114 */ +/* +** See: +** https://sqlite.org/src/info/e2b3f1a9480a9be3 +** https://github.com/rhashimoto/wa-sqlite/discussions/301 +** +** It is enabled here for the sake of VFS experimentors. +*/ +#undef SQLITE_EXPERIMENTAL_PRAGMA_20251114 +#define SQLITE_EXPERIMENTAL_PRAGMA_20251114 + /**********************************************************************/ /* SQLITE_O... */ #undef SQLITE_OMIT_DEPRECATED @@ -136,8 +148,8 @@ /* ** If SQLITE_WASM_BARE_BONES is defined, undefine most of the ENABLE ** macros. This will, when using the canonical makefile, also elide -** any C functions from the WASM exports which are listed in -** ./EXPORT_FUNCTIONS.sqlite3-extras. +** any C functions from the WASM exports: see +** ./EXPORTED_FUNCTIONS.c-pp. */ #ifdef SQLITE_WASM_BARE_BONES # undef SQLITE_ENABLE_COLUMN_METADATA @@ -217,15 +229,18 @@ ** not by client code, so an argument can be made for reducing their ** visibility by not including them in any build-time export lists. ** -** 2022-09-11: it's not yet _proven_ that this approach works in -** non-Emscripten builds. If not, such builds will need to export -** those using the --export=... wasm-ld flag (or equivalent). As of -** this writing we are tied to Emscripten for various reasons -** and cannot test the library with other build environments. +** 2025-12-01: for use in non-Emscripten builds, we need a more +** invasive macro which explicitly names the export: +** SQLITE_WASM_EXPORT2. */ #define SQLITE_WASM_EXPORT __attribute__((used,visibility("default"))) -// See also: -//__attribute__((export_name("theExportedName"), used, visibility("default"))) +#define SQLITE_WASM_EXPORT_NAMED(X) __attribute__((export_name(#X),used,visibility("default"))) +#define SQLITE_WASM_EXPORT2(RETTYPE,NAME,SIG) SQLITE_WASM_EXPORT_NAMED(NAME) RETTYPE NAME SIG + +#if 1 +/** Increase the kvvfs key size limit from 32. */ +#define KVRECORD_KEY_SZ 128 +#endif /* ** Which sqlite3.c we're using needs to be configurable to enable @@ -249,45 +264,6 @@ #undef INC__STRINGIFY #undef SQLITE_C -#if 0 -/* -** An EXPERIMENT in implementing a stack-based allocator analog to -** Emscripten's stackSave(), stackAlloc(), stackRestore(). -** Unfortunately, this cannot work together with Emscripten because -** Emscripten defines its own native one and we'd stomp on each -** other's memory. Other than that complication, basic tests show it -** to work just fine. -** -** Another option is to malloc() a chunk of our own and call that our -** "stack". -*/ -SQLITE_WASM_EXPORT void * sqlite3__wasm_stack_end(void){ - extern void __heap_base - /* see https://stackoverflow.com/questions/10038964 */; - return &__heap_base; -} -SQLITE_WASM_EXPORT void * sqlite3__wasm_stack_begin(void){ - extern void __data_end; - return &__data_end; -} -static void * pWasmStackPtr = 0; -SQLITE_WASM_EXPORT void * sqlite3__wasm_stack_ptr(void){ - if(!pWasmStackPtr) pWasmStackPtr = sqlite3__wasm_stack_end(); - return pWasmStackPtr; -} -SQLITE_WASM_EXPORT void sqlite3__wasm_stack_restore(void * p){ - pWasmStackPtr = p; -} -SQLITE_WASM_EXPORT void * sqlite3__wasm_stack_alloc(int n){ - if(n<=0) return 0; - n = (n + 7) & ~7 /* align to 8-byte boundary */; - unsigned char * const p = (unsigned char *)sqlite3__wasm_stack_ptr(); - unsigned const char * const b = (unsigned const char *)sqlite3__wasm_stack_begin(); - if(b + n >= p || b + n < b/*overflow*/) return 0; - return pWasmStackPtr = p - n; -} -#endif /* stack allocator experiment */ - /* ** State for the "pseudo-stack" allocator implemented in ** sqlite3__wasm_pstack_xyz(). In order to avoid colliding with @@ -326,7 +302,7 @@ SQLITE_WASM_EXPORT void * sqlite3__wasm_pstack_ptr(void){ */ SQLITE_WASM_EXPORT void sqlite3__wasm_pstack_restore(unsigned char * p){ assert(p>=PStack.pBegin && p<=PStack.pEnd && p>=PStack.pPos); - assert(0==((unsigned long long)p & 0x7)); + assert(0==((unsigned long long)p & 0x7) /* 8-byte aligned */); if(p>=PStack.pBegin && p<=PStack.pEnd /*&& p>=PStack.pPos*/){ PStack.pPos = p; } @@ -336,10 +312,10 @@ SQLITE_WASM_EXPORT void sqlite3__wasm_pstack_restore(unsigned char * p){ ** the memory on success, 0 on error (including a negative n value). n ** is always adjusted to be a multiple of 8 and returned memory is ** always zeroed out before returning (because this keeps the client -** JS code from having to do so, and most uses of the pstack will -** call for doing so). +** JS code from having to do so, and most uses of the pstack call for +** doing so). */ -SQLITE_WASM_EXPORT void * sqlite3__wasm_pstack_alloc(int n){ +SQLITE_WASM_EXPORT2(void *,sqlite3__wasm_pstack_alloc,(int n)){ if( n<=0 ) return 0; n = (n + 7) & ~7 /* align to 8-byte boundary */; if( PStack.pBegin + n > PStack.pPos /*not enough space left*/ @@ -351,7 +327,7 @@ SQLITE_WASM_EXPORT void * sqlite3__wasm_pstack_alloc(int n){ ** Return the number of bytes left which can be ** sqlite3__wasm_pstack_alloc()'d. */ -SQLITE_WASM_EXPORT int sqlite3__wasm_pstack_remaining(void){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_pstack_remaining,(void)){ assert(PStack.pPos >= PStack.pBegin); assert(PStack.pPos <= PStack.pEnd); return (int)(PStack.pPos - PStack.pBegin); @@ -362,7 +338,7 @@ SQLITE_WASM_EXPORT int sqlite3__wasm_pstack_remaining(void){ ** any space which is currently allocated. This value is a ** compile-time constant. */ -SQLITE_WASM_EXPORT int sqlite3__wasm_pstack_quota(void){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_pstack_quota,(void)){ return (int)(PStack.pEnd - PStack.pBegin); } @@ -375,8 +351,7 @@ struct WasmTestStruct { void (*xFunc)(void*); }; typedef struct WasmTestStruct WasmTestStruct; -SQLITE_WASM_EXPORT -void sqlite3__wasm_test_struct(WasmTestStruct * s){ +SQLITE_WASM_EXPORT2(void,sqlite3__wasm_test_struct,(WasmTestStruct * s)){ if(s){ if( 0 ){ /* Do not be alarmed by the small (and odd) pointer values. @@ -416,8 +391,7 @@ void sqlite3__wasm_test_struct(WasmTestStruct * s){ ** fails to compile with "tables may not be 64-bit" but does not tell ** us where it's happening. */ -SQLITE_WASM_EXPORT -const char * sqlite3__wasm_enum_json(void){ +SQLITE_WASM_EXPORT2(const char *,sqlite3__wasm_enum_json,(void)){ static char aBuffer[1024 * 20] = {0} /* where the JSON goes. 2025-09-19: output size=19295, but that can vary slightly from build to build, so a little @@ -620,6 +594,7 @@ const char * sqlite3__wasm_enum_json(void){ DefGroup(encodings) { /* Noting that the wasm binding only aims to support UTF-8. */ DefInt(SQLITE_UTF8); + DefInt(SQLITE_UTF8_ZT); DefInt(SQLITE_UTF16LE); DefInt(SQLITE_UTF16BE); DefInt(SQLITE_UTF16); @@ -1007,13 +982,15 @@ const char * sqlite3__wasm_enum_json(void){ /** ^^^ indirection needed to expand CurrentStruct */ #define StructBinder StructBinder_(CurrentStruct) #define _StructBinder CloseBrace(2) -#define M(MEMBER,SIG) \ - outf("%s\"%s\": " \ - "{\"offset\":%d,\"sizeof\": %d,\"signature\":\"%s\"}", \ - (n++ ? ", " : ""), #MEMBER, \ - (int)offsetof(CurrentStruct,MEMBER), \ - (int)sizeof(((CurrentStruct*)0)->MEMBER), \ - SIG) +#define M3(MEMBER,SIG,READONLY) \ + outf("%s\"%s\": " \ + "{\"offset\":%d,\"sizeof\":%d,\"signature\":\"%s\"%s}", \ + (n++ ? ", " : ""), #MEMBER, \ + (int)offsetof(CurrentStruct,MEMBER), \ + (int)sizeof(((CurrentStruct*)0)->MEMBER), \ + SIG, (READONLY ? ",\"readOnly\":true" : "")) +#define M(MEMBER,SIG) M3(MEMBER,SIG,0) +#define MRO(MEMBER,SIG) M3(MEMBER,SIG,1) nStruct = 0; out(", \"structs\": ["); { @@ -1076,11 +1053,30 @@ const char * sqlite3__wasm_enum_json(void){ #undef CurrentStruct #define CurrentStruct sqlite3_kvvfs_methods + /* From os_kv.c */ + StructBinder { + M(xRcrdRead, "i(sspi)"); + M(xRcrdWrite, "i(sss)"); + M(xRcrdDelete, "i(ss)"); + MRO(nKeySize, "i"); + MRO(nBufferSize, "i"); + M(pVfs, "p"); + M(pIoDb, "p"); + M(pIoJrnl, "p"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct KVVfsFile + /* From os_kv.c */ StructBinder { - M(xRead, "i(sspi)"); - M(xWrite, "i(sss)"); - M(xDelete, "i(ss)"); - M(nKeySize, "i"); + M(base, "p")/*sqlite3_file base*/; + M(zClass, "s"); + M(isJournal, "i"); + M(nJrnl, "i")/*actually unsigned!*/; + M(aJrnl, "p"); + M(szPage, "i"); + M(szDb, "j"); + M(aData, "p"); } _StructBinder; #undef CurrentStruct @@ -1137,7 +1133,13 @@ const char * sqlite3__wasm_enum_json(void){ ** sqlite3_index_info, we have to uplift those into constructs we ** can access by type name. These structs _must_ match their ** in-sqlite3_index_info counterparts byte for byte. - */ + ** + ** 2025-11-21: this uplifing is no longer necessary, as Jaccwabyt + ** can now handle nested structs, but "it ain't broke" so there's + ** no pressing need to rewire this. Also, it's conceivable that + ** rewiring it might break downstream vtab impls, so it shouldn't + ** be rewired. + */ typedef struct { int iColumn; unsigned char op; @@ -1232,6 +1234,8 @@ const char * sqlite3__wasm_enum_json(void){ #undef StructBinder_ #undef StructBinder__ #undef M +#undef MRO +#undef M3 #undef _StructBinder #undef CloseBrace #undef out @@ -1249,8 +1253,7 @@ const char * sqlite3__wasm_enum_json(void){ ** method, SQLITE_MISUSE is returned, else the result of the xDelete() ** call is returned. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_vfs_unlink(sqlite3_vfs *pVfs, const char *zName){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_vfs_unlink,(sqlite3_vfs *pVfs, const char *zName)){ int rc = SQLITE_MISUSE /* ??? */; if( 0==pVfs && 0!=zName ) pVfs = sqlite3_vfs_find(0); if( zName && pVfs && pVfs->xDelete ){ @@ -1267,8 +1270,7 @@ int sqlite3__wasm_vfs_unlink(sqlite3_vfs *pVfs, const char *zName){ ** defaulting to "main" if zDbName is 0. Returns 0 if no db with the ** given name is open. */ -SQLITE_WASM_EXPORT -sqlite3_vfs * sqlite3__wasm_db_vfs(sqlite3 *pDb, const char *zDbName){ +SQLITE_WASM_EXPORT2(sqlite3_vfs *,sqlite3__wasm_db_vfs,(sqlite3 *pDb, const char *zDbName)){ sqlite3_vfs * pVfs = 0; sqlite3_file_control(pDb, zDbName ? zDbName : "main", SQLITE_FCNTL_VFS_POINTER, &pVfs); @@ -1290,8 +1292,7 @@ sqlite3_vfs * sqlite3__wasm_db_vfs(sqlite3 *pDb, const char *zDbName){ ** Returns 0 on success, an SQLITE_xxx code on error. Returns ** SQLITE_MISUSE if pDb is NULL. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_db_reset(sqlite3 *pDb){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_db_reset,(sqlite3 *pDb)){ int rc = SQLITE_MISUSE; if( pDb ){ sqlite3_table_column_metadata(pDb, "main", 0, 0, 0, 0, 0, 0, 0); @@ -1372,10 +1373,10 @@ int sqlite3__wasm_db_export_chunked( sqlite3* pDb, ** If `*pOut` is not NULL, the caller is responsible for passing it to ** sqlite3_free() to free it. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_db_serialize( sqlite3 *pDb, const char *zSchema, - unsigned char **pOut, - sqlite3_int64 *nOut, unsigned int mFlags ){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_db_serialize, + (sqlite3 *pDb, const char *zSchema, + unsigned char **pOut, + sqlite3_int64 *nOut, unsigned int mFlags)){ unsigned char * z; if( !pDb || !pOut ) return SQLITE_MISUSE; if( nOut ) *nOut = 0; @@ -1436,11 +1437,9 @@ int sqlite3__wasm_db_serialize( sqlite3 *pDb, const char *zSchema, ** portability, so that the API can still work in builds where BigInt ** support is disabled or unavailable. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_vfs_create_file( sqlite3_vfs *pVfs, - const char *zFilename, - const unsigned char * pData, - int nData ){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_vfs_create_file, + (sqlite3_vfs *pVfs, const char *zFilename, + const unsigned char * pData, int nData)){ int rc; sqlite3_file *pFile = 0; sqlite3_io_methods const *pIo; @@ -1526,10 +1525,9 @@ int sqlite3__wasm_vfs_create_file( sqlite3_vfs *pVfs, ** zFilename, appends pData bytes to it, and returns 0 on success or ** SQLITE_IOERR on error. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_posix_create_file( const char *zFilename, - const unsigned char * pData, - int nData ){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_posix_create_file, + (const char *zFilename, const unsigned char * pData, + int nData)){ int rc; FILE * pFile = 0; int fileExisted = 0; @@ -1549,22 +1547,30 @@ int sqlite3__wasm_posix_create_file( const char *zFilename, ** This function is NOT part of the sqlite3 public API. It is strictly ** for use by the sqlite project's own JS/WASM bindings. ** -** Allocates sqlite3KvvfsMethods.nKeySize bytes from -** sqlite3__wasm_pstack_alloc() and returns 0 if that allocation fails, -** else it passes that string to kvstorageMakeKey() and returns a -** NUL-terminated pointer to that string. It is up to the caller to -** use sqlite3__wasm_pstack_restore() to free the returned pointer. +** This returns either a pointer to a static buffer or zKeyIn directly +** (if zClass is NULL or empty). */ -SQLITE_WASM_EXPORT -char * sqlite3__wasm_kvvfsMakeKeyOnPstack(const char *zClass, - const char *zKeyIn){ +SQLITE_WASM_EXPORT2(const char *,sqlite3__wasm_kvvfsMakeKey, + (const char *zClass, const char *zKeyIn)){ + static char buf[SQLITE_KVOS_SZ+1] = {0}; assert(sqlite3KvvfsMethods.nKeySize>24); - char *zKeyOut = - (char *)sqlite3__wasm_pstack_alloc(sqlite3KvvfsMethods.nKeySize); - if(zKeyOut){ - kvstorageMakeKey(zClass, zKeyIn, zKeyOut); + if( zClass && *zClass ){ + kvrecordMakeKey(zClass, zKeyIn, buf); + return buf; + }else{ +#if 1 + /* We can return zKeyIn here only because the JS API takes special + ** care with its lifetime.*/ + return zKeyIn; +#else + /* It would be nice to be able to return zKeyIn directly here, but + ** it may have been allocated as part of the automated JS-to-WASM + ** conversions, in which case it will be freed before reaching the + ** caller. */ + sqlite3_snprintf(KVRECORD_KEY_SZ, buf, "%s", zKeyIn); + return buf; +#endif } - return zKeyOut; } /* @@ -1574,8 +1580,7 @@ char * sqlite3__wasm_kvvfsMakeKeyOnPstack(const char *zClass, ** Returns the pointer to the singleton object which holds the kvvfs ** I/O methods and associated state. */ -SQLITE_WASM_EXPORT -sqlite3_kvvfs_methods * sqlite3__wasm_kvvfs_methods(void){ +SQLITE_WASM_EXPORT2(sqlite3_kvvfs_methods *,sqlite3__wasm_kvvfs_methods,(void)){ return &sqlite3KvvfsMethods; } @@ -1590,8 +1595,8 @@ sqlite3_kvvfs_methods * sqlite3__wasm_kvvfs_methods(void){ ** sqlite3_vtab_config(), or SQLITE_MISUSE if the 2nd arg is not a ** valid value. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_vtab_config(sqlite3 *pDb, int op, int arg){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_vtab_config, + (sqlite3 *pDb, int op, int arg)){ switch(op){ case SQLITE_VTAB_DIRECTONLY: case SQLITE_VTAB_INNOCUOUS: @@ -1611,8 +1616,8 @@ int sqlite3__wasm_vtab_config(sqlite3 *pDb, int op, int arg){ ** Wrapper for the variants of sqlite3_db_config() which take ** (int,int*) variadic args. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_db_config_ip(sqlite3 *pDb, int op, int arg1, int* pArg2){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_db_config_ip, + (sqlite3 *pDb, int op, int arg1, int* pArg2)){ switch(op){ case SQLITE_DBCONFIG_ENABLE_FKEY: case SQLITE_DBCONFIG_ENABLE_TRIGGER: @@ -1647,8 +1652,9 @@ int sqlite3__wasm_db_config_ip(sqlite3 *pDb, int op, int arg1, int* pArg2){ ** Wrapper for the variants of sqlite3_db_config() which take ** (void*,int,int) variadic args. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_db_config_pii(sqlite3 *pDb, int op, void * pArg1, int arg2, int arg3){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_db_config_pii, + (sqlite3 *pDb, int op, void * pArg1, int arg2, + int arg3)){ switch(op){ case SQLITE_DBCONFIG_LOOKASIDE: return sqlite3_db_config(pDb, op, pArg1, arg2, arg3); @@ -1663,8 +1669,8 @@ int sqlite3__wasm_db_config_pii(sqlite3 *pDb, int op, void * pArg1, int arg2, in ** Wrapper for the variants of sqlite3_db_config() which take ** (const char *) variadic args. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_db_config_s(sqlite3 *pDb, int op, const char *zArg){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_db_config_s,(sqlite3 *pDb, int op, + const char *zArg)){ switch(op){ case SQLITE_DBCONFIG_MAINDBNAME: return sqlite3_db_config(pDb, op, zArg); @@ -1680,8 +1686,7 @@ int sqlite3__wasm_db_config_s(sqlite3 *pDb, int op, const char *zArg){ ** Binding for combinations of sqlite3_config() arguments which take ** a single integer argument. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_config_i(int op, int arg){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_config_i,(int op, int arg)){ return sqlite3_config(op, arg); } @@ -1692,8 +1697,7 @@ int sqlite3__wasm_config_i(int op, int arg){ ** Binding for combinations of sqlite3_config() arguments which take ** two int arguments. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_config_ii(int op, int arg1, int arg2){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_config_ii,(int op, int arg1, int arg2)){ return sqlite3_config(op, arg1, arg2); } @@ -1704,8 +1708,7 @@ int sqlite3__wasm_config_ii(int op, int arg1, int arg2){ ** Binding for combinations of sqlite3_config() arguments which take ** a single i64 argument. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_config_j(int op, sqlite3_int64 arg){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_config_j,(int op, sqlite3_int64 arg)){ return sqlite3_config(op, arg); } @@ -1717,8 +1720,7 @@ int sqlite3__wasm_config_j(int op, sqlite3_int64 arg){ ** sqlite3_mprintf()'s %Q modifier (if addQuotes is true) or %q (if ** addQuotes is 0). Returns NULL if z is NULL or on OOM. */ -SQLITE_WASM_EXPORT -char * sqlite3__wasm_qfmt_token(char *z, int addQuotes){ +SQLITE_WASM_EXPORT2(char *,sqlite3__wasm_qfmt_token,(char *z, int addQuotes)){ char * rc = 0; if( z ){ rc = addQuotes @@ -1728,6 +1730,21 @@ char * sqlite3__wasm_qfmt_token(char *z, int addQuotes){ return rc; } +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** A WASM wrapper for the interal os_kv.c:kvvfsDecode() for internal +** use by the kvvfs v2 API. +*/ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_kvvfs_decode,(const char *a, char *aOut, int nOut)){ + return kvvfsDecode(a, aOut, nOut); +} +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_kvvfs_encode,(const char *a, int nA, char *aOut)){ + return kvvfsEncode(a, nA, aOut); +} + + #if defined(__EMSCRIPTEN__) && defined(SQLITE_ENABLE_WASMFS) #include #include @@ -1753,8 +1770,7 @@ char * sqlite3__wasm_qfmt_token(char *z, int addQuotes){ ** the virtual FS fails. In builds compiled without SQLITE_ENABLE_WASMFS ** defined, SQLITE_NOTFOUND is returned without side effects. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_init_wasmfs(const char *zMountPoint){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_init_wasmfs,(const char *zMountPoint)){ static backend_t pOpfs = 0; if( !zMountPoint || !*zMountPoint ) zMountPoint = "/opfs"; if( !pOpfs ){ @@ -1773,8 +1789,7 @@ int sqlite3__wasm_init_wasmfs(const char *zMountPoint){ return pOpfs ? 0 : SQLITE_NOMEM; } #else -SQLITE_WASM_EXPORT -int sqlite3__wasm_init_wasmfs(const char *zUnused){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_init_wasmfs,(const char *zUnused)){ //emscripten_console_warn("WASMFS OPFS is not compiled in."); (void)zUnused; return SQLITE_NOTFOUND; @@ -1783,52 +1798,43 @@ int sqlite3__wasm_init_wasmfs(const char *zUnused){ #if SQLITE_WASM_ENABLE_C_TESTS -SQLITE_WASM_EXPORT -int sqlite3__wasm_test_intptr(int * p){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_test_intptr,(int * p)){ return *p = *p * 2; } -SQLITE_WASM_EXPORT -void * sqlite3__wasm_test_voidptr(void * p){ +SQLITE_WASM_EXPORT2(void *,sqlite3__wasm_test_voidptr,(void * p)){ return p; } -SQLITE_WASM_EXPORT -int64_t sqlite3__wasm_test_int64_max(void){ +SQLITE_WASM_EXPORT2(int64_t,sqlite3__wasm_test_int64_max,(void)){ return (int64_t)0x7fffffffffffffff; } -SQLITE_WASM_EXPORT -int64_t sqlite3__wasm_test_int64_min(void){ +SQLITE_WASM_EXPORT2(int64_t,sqlite3__wasm_test_int64_min,(void)){ return ~sqlite3__wasm_test_int64_max(); } -SQLITE_WASM_EXPORT -int64_t sqlite3__wasm_test_int64_times2(int64_t x){ +SQLITE_WASM_EXPORT2(int64_t,sqlite3__wasm_test_int64_times2,(int64_t x)){ return x * 2; } -SQLITE_WASM_EXPORT -void sqlite3__wasm_test_int64_minmax(int64_t * min, int64_t *max){ +SQLITE_WASM_EXPORT2(void,sqlite3__wasm_test_int64_minmax,(int64_t * min, int64_t *max)){ *max = sqlite3__wasm_test_int64_max(); *min = sqlite3__wasm_test_int64_min(); /*printf("minmax: min=%lld, max=%lld\n", *min, *max);*/ } -SQLITE_WASM_EXPORT -int64_t sqlite3__wasm_test_int64ptr(int64_t * p){ +SQLITE_WASM_EXPORT2(int64_t,sqlite3__wasm_test_int64ptr,(int64_t * p)){ /*printf("sqlite3__wasm_test_int64ptr( @%lld = 0x%llx )\n", (int64_t)p, *p);*/ return *p = *p * 2; } -SQLITE_WASM_EXPORT -void sqlite3__wasm_test_stack_overflow(int recurse){ +SQLITE_WASM_EXPORT2(void,sqlite3__wasm_test_stack_overflow,(int recurse)){ if(recurse) sqlite3__wasm_test_stack_overflow(recurse); } /* For testing the 'string:dealloc' whwasmutil.xWrap() conversion. */ -SQLITE_WASM_EXPORT -char * sqlite3__wasm_test_str_hello(int fail){ +SQLITE_WASM_EXPORT2(char *,sqlite3__wasm_test_str_hello,(int fail)){ char * s = fail ? 0 : (char *)sqlite3_malloc(6); if(s){ memcpy(s, "hello", 5); @@ -1942,8 +1948,8 @@ static int sqlite3__wasm_SQLTester_strnotglob(const char *zGlob, const char *z){ return *z==0; } -SQLITE_WASM_EXPORT -int sqlite3__wasm_SQLTester_strglob(const char *zGlob, const char *z){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_SQLTester_strglob, + (const char *zGlob, const char *z)){ return !sqlite3__wasm_SQLTester_strnotglob(zGlob, z); } diff --git a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js index 1a09bf9a6..b282c5e6e 100644 --- a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js +++ b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js @@ -19,10 +19,12 @@ slightly simpler client-side interface than the slightly-lower-level Worker API does. - This script necessarily exposes one global symbol, but clients may - freely `delete` that symbol after calling it. + In non-ESM builds this file necessarily exposes one global symbol, + but clients may freely `delete` that symbol after calling it. */ +//#if not defined target:es6-module 'use strict'; +//#endif /** Configures an sqlite3 Worker API #1 Worker such that it can be manipulated via a Promise-based interface and returns a factory @@ -109,10 +111,12 @@ the callback is called one time for each row of the result set, passed the same worker message format as the worker API emits: - {type:typeString, + { + type:typeString, row:VALUE, rowNumber:1-based-#, - columnNames: array} + columnNames: array + } Where `typeString` is an internally-synthesized message type string used temporarily for worker message dispatching. It can be ignored @@ -123,10 +127,9 @@ callback. At the end of the result set, the same event is fired with - (row=undefined, rowNumber=null) to indicate that - the end of the result set has been reached. Note that the rows - arrive via worker-posted messages, with all the implications - of that. + (row=undefined, rowNumber=null) to indicate that the end of the + result set has been reached. The rows arrive via worker-posted + messages, with all the implications of that. Notable shortcomings: @@ -257,7 +260,9 @@ globalThis.sqlite3Worker1Promiser.defaultConfig = { type: 'module' }); //#elif target:es6-module - return new Worker(new URL("sqlite3-worker1.js", import.meta.url)); + return new Worker(new URL("sqlite3-worker1.mjs", import.meta.url),{ + type: 'module' + }); //#else let theJs = "sqlite3-worker1.js"; if(this.currentScript){ @@ -281,7 +286,7 @@ globalThis.sqlite3Worker1Promiser.defaultConfig = { }) //#endif , - onerror: (...args)=>console.error('worker1 promiser error',...args) + onerror: (...args)=>console.error('sqlite3Worker1Promiser():',...args) }/*defaultConfig*/; /** @@ -343,6 +348,7 @@ globalThis.sqlite3Worker1Promiser.v2.defaultConfig = incompatibility. */ export default sqlite3Worker1Promiser.v2; +delete globalThis.sqlite3Worker1Promiser; //#endif /* target:es6-module */ //#else /* Built with the omit-oo1 flag. */ diff --git a/ext/wasm/api/sqlite3-worker1.c-pp.js b/ext/wasm/api/sqlite3-worker1.c-pp.js index 036c4c6ea..db27c8fc0 100644 --- a/ext/wasm/api/sqlite3-worker1.c-pp.js +++ b/ext/wasm/api/sqlite3-worker1.c-pp.js @@ -33,9 +33,9 @@ directory from which `sqlite3.js` will be loaded. */ //#if target:es6-bundler-friendly -import {default as sqlite3InitModule} from './sqlite3-bundler-friendly.mjs'; +import sqlite3InitModule from './sqlite3-bundler-friendly.mjs'; //#elif target:es6-module - return new Worker(new URL("sqlite3.js", import.meta.url)); +import sqlite3InitModule from './sqlite3.mjs'; //#else "use strict"; { diff --git a/ext/wasm/c-pp-lite.c b/ext/wasm/c-pp-lite.c index 2120c457d..b8d67f6a3 100644 --- a/ext/wasm/c-pp-lite.c +++ b/ext/wasm/c-pp-lite.c @@ -270,9 +270,10 @@ static void db_prepare(sqlite3_stmt **pStmt, const char * zSql, ...); /* ** Opens the given file and processes its contents as c-pp, sending ** all output to the global c-pp output channel. Fails fatally on -** error. +** error. If bRaw is true then the file's contents are passed through +** verbatim, rather than being preprocessed. */ -static void cmpp_process_file(const char * zName); +static void cmpp_process_file(const char * zName, int bRaw); /* ** Operator policy for cmpp_kvp_parse(). @@ -970,7 +971,6 @@ void db_prepare(sqlite3_stmt **pStmt, const char * zSql, ...){ char * z = 0; int n = 0; va_list va; - if(!str) fatal("sqlite3_str_new() failed"); va_start(va, zSql); sqlite3_str_vappendf(str, zSql, va); va_end(va); @@ -2213,12 +2213,23 @@ static void cmpp_kwd_endif(CmppKeyword const * pKw, CmppTokenizer *t){ static void cmpp_kwd_include(CmppKeyword const * pKw, CmppTokenizer *t){ char const * zFile; char * zResolved; + int bRaw = 0 /* -raw flag */; if(CT_skip(t)) return; - else if(t->args.argc!=2){ - cmpp_kwd__err(pKw, t, "Expecting exactly 1 filename argument"); + else if(t->args.argc<2 || t->args.argc>3){ + cmpp_kwd__err(pKw, t, "Expected args: ?-raw? filename"); } - zFile = (const char *)t->args.argv[1]; - if(db_including_has(zFile)){ + if(t->args.argc==2){ + zFile = (const char *)t->args.argv[1]; + }else{ + if( 0==strcmp("-raw", (char*)t->args.argv[1]) ){ + bRaw = 1; + }else{ + cmpp_kwd__err(pKw, t, "Unhandled argument: %s", + t->args.argv[1]); + } + zFile = (const char *)t->args.argv[2]; + } + if(!bRaw && db_including_has(zFile)){ /* Note that different spellings of the same filename ** will elude this check, but that seems okay, as different ** spellings means that we're not re-running the exact same @@ -2229,16 +2240,15 @@ static void cmpp_kwd_include(CmppKeyword const * pKw, CmppTokenizer *t){ } zResolved = db_include_search(zFile); if(zResolved){ - db_including_add(zFile, t->zName, t->token.lineNo); - cmpp_process_file(zResolved); - db_include_rm(zFile); + if( bRaw ) db_including_add(zFile, t->zName, t->token.lineNo); + cmpp_process_file(zResolved, bRaw); + if( bRaw ) db_include_rm(zFile); db_free(zResolved); }else{ cmpp_t__err(t, "file not found: %s", zFile); } } - static void cmpp_dump_defines( FILE * fp, int bIndent ){ sqlite3_stmt * const q = g_stmt(GStmt_defSelAll); while( SQLITE_ROW==sqlite3_step(q) ){ @@ -2392,7 +2402,7 @@ CmppKeyword aKeywords[] = { {S(Endif), 0, TT_Endif, cmpp_kwd_endif}, {S(Error), 0, TT_Error, cmpp_kwd_error}, {S(If), 1, TT_If, cmpp_kwd_if}, - {S(Include), 0, TT_Include, cmpp_kwd_include}, + {S(Include), 1, TT_Include, cmpp_kwd_include}, {S(Pragma), 1, TT_Pragma, cmpp_kwd_pragma}, {S(Savepoint), 1, TT_Savepoint, cmpp_kwd_savepoint}, {S(Stderr), 0, TT_Stderr, cmpp_kwd_stderr}, @@ -2444,14 +2454,25 @@ void cmpp_process_string(const char * zName, g.tok = oldTok; } -void cmpp_process_file(const char * zName){ +void cmpp_process_file(const char * zName, int bRaw){ FileWrapper fw = FileWrapper_empty; FileWrapper_open(&fw, zName, "r"); g_FileWrapper_link(&fw); FileWrapper_slurp(&fw); g_debug(1,("Read %u byte(s) from [%s]\n", fw.nContent, fw.zName)); if( fw.zContent ){ - cmpp_process_string(zName, fw.zContent, fw.nContent); + if( bRaw ){ + FileWrapper fw = FileWrapper_empty; + FileWrapper_open(&fw, zName, "rb"); + g_FileWrapper_link(&fw); + FileWrapper_slurp(&fw); + if( 1!=fwrite(fw.zContent, fw.nContent, 1, g.out.pFile) ){ + fatal("fwrite() failed with errno %d\n", errno); + } + g_FileWrapper_close(&fw); + }else{ + cmpp_process_string(zName, fw.zContent, fw.nContent); + } } g_FileWrapper_close(&fw); } @@ -2580,6 +2601,21 @@ static int arg_is_flag( char const *zFlag, char const *zArg, return 0; } +static void define_argv(int argc, char const * const * argv){ + sqlite3_str * const s = sqlite3_str_new(g.db); + sqlite3_str_append(s, "c-pp::argv=", 11); + for( int i = 0; i < argc; ++i ){ + if( i ) sqlite3_str_appendchar(s, 1, ' '); + sqlite3_str_appendf(s, "%s", argv[i]); + } + char * const z = sqlite3_str_finish(s); + assert(z); + if(z){ + db_define_add(z, NULL); + sqlite3_free(z); + } +} + int main(int argc, char const * const * argv){ int rc = 0; int inclCount = 0; @@ -2617,6 +2653,7 @@ int main(int argc, char const * const * argv){ g.sqlTrace.expandSql = expandMode; } cmpp_initdb(); + define_argv(argc, argv); } for(int i = 1; i < argc; ++i){ int negate = 0; @@ -2674,7 +2711,7 @@ int main(int argc, char const * const * argv){ DOIT { ++nFile; g_out_open; - cmpp_process_file(zVal); + cmpp_process_file(zVal, 0); } } ISFLAG("e"){ @@ -2757,7 +2794,7 @@ int main(int argc, char const * const * argv){ ++inclCount; } FileWrapper_open(&g.out, g.out.zName, "w"); - cmpp_process_file("-"); + cmpp_process_file("-", 0); } } } diff --git a/ext/wasm/common/SqliteTestUtil.js b/ext/wasm/common/SqliteTestUtil.js index 2c17824c5..a817b79f8 100644 --- a/ext/wasm/common/SqliteTestUtil.js +++ b/ext/wasm/common/SqliteTestUtil.js @@ -44,7 +44,8 @@ /** abort() if expr is false. If expr is a function, it is called and its result is evaluated. */ - assert: function f(expr, msg){ + assert: function f(expr, ...msg){ + msg = msg?.join?.(' '); if(!f._){ f._ = ('undefined'===typeof abort ? (msg)=>{throw new Error(msg)} diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js index 1c678f31f..bca05a1ee 100644 --- a/ext/wasm/common/whwasmutil.js +++ b/ext/wasm/common/whwasmutil.js @@ -16,27 +16,35 @@ More specifically: - https://fossil.wanderinghorse.net/r/jaccwabyt/file/common/whwasmutil.js + https://fossil.wanderinghorse.net/r/jaccwabyt/dir/wasmutil and SQLite: https://sqlite.org This file is kept in sync between both of those trees. + + This build was generated using: + + ./c-pp -o js/whwasmutil.js -@policy=error wasmutil/whwasmutil.c-pp.js + + by libcmpp 2.x 2fc4afc31f6505c27b9c34988973a2bd9b157d559247cdd26868ae75632c3a5e @ 2025-11-16 23:03:27.352 UTC */ /** - The primary goal of this function is to replace, where possible, - Emscripten-generated glue code with equivalent utility code which - can be used in arbitrary WASM environments built with toolchains - other than Emscripten. To that end, it populates the given object - with various WASM-specific APIs. These APIs work with both 32- and - 64-bit WASM builds. + The primary goal of this function is to provide JS/WASM utility + code similar to some of that provided by Emscripten-generated + builds, the difference being that this one can be used in arbitrary + WASM environments built with toolchains other than Emscripten. To + that end, it populates the given object with various WASM-specific + APIs. These APIs work with both 32- and 64-bit WASM builds. Forewarning: this API explicitly targets only browser environments. If a given non-browser environment has the capabilities needed for a given feature (e.g. TextEncoder), great, but it does not go out of its way to account for them and does not provide compatibility - crutches for them. + crutches for them. That said: no specific incompatibilities with, + e.g., node.js are known (whereas it is known that some folks + use this with node.js). Intended usage: @@ -217,7 +225,9 @@ newly-created (or config-provided) target. The current approach seemed better at the time. */ -globalThis.WhWasmUtilInstaller = function(target){ +'use strict'; +globalThis.WhWasmUtilInstaller = +function WhWasmUtilInstaller(target){ 'use strict'; if(undefined===target.bigIntEnabled){ target.bigIntEnabled = !!globalThis['BigInt64Array']; @@ -227,6 +237,14 @@ globalThis.WhWasmUtilInstaller = function(target){ all args with a space between each. */ const toss = (...args)=>{throw new Error(args.join(' '))}; + if( !target.pointerSize && !target.pointerIR + && target.alloc && target.dealloc ){ + /* Try to determine the pointer size by allocating. */ + const ptr = target.alloc(1); + target.pointerSize = ('bigint'===typeof ptr ? 8 : 4); + target.dealloc(ptr); + } + /** As of 2025-09-21, this library works with 64-bit WASM modules built with Emscripten's -sMEMORY64=1. @@ -659,12 +677,14 @@ globalThis.WhWasmUtilInstaller = function(target){ const ft = target.functionTable(); const oldLen = __asPtrType(ft.length); let ptr; - while(cache.freeFuncIndexes.length){ - ptr = cache.freeFuncIndexes.pop(); - if(ft.get(ptr)){ /* Table was modified via a different API */ + while( (ptr = cache.freeFuncIndexes.pop()) ){ + if(ft.get(ptr)){ + /* freeFuncIndexes's entry is stale. Table was modified via a + different API */ ptr = null; continue; }else{ + /* This index is free. We'll re-use it. */ break; } } @@ -755,10 +775,10 @@ globalThis.WhWasmUtilInstaller = function(target){ has no side effects and returns undefined. */ target.uninstallFunction = function(ptr){ - if(!ptr && 0!==ptr) return undefined; - const fi = cache.freeFuncIndexes; + if(!ptr && __NullPtr!==ptr) return undefined; + const ft = target.functionTable(); - fi.push(ptr); + cache.freeFuncIndexes.push(ptr); const rc = ft.get(ptr); ft.set(ptr, null); return rc; @@ -996,12 +1016,12 @@ globalThis.WhWasmUtilInstaller = function(target){ target.heap8u(). */ target.cstrlen = function(ptr){ - if(!ptr || !target.isPtr(ptr)) return null; + if(!ptr || !target.isPtr/*64*/(ptr)) return null; ptr = Number(ptr) /*tag:64bit*/; const h = heapWrappers().HEAP8U; let pos = ptr; for( ; h[pos] !== 0; ++pos ){} - return Number(pos - ptr); + return pos - ptr; }; /** Internal helper to use in operations which need to distinguish @@ -2452,7 +2472,8 @@ globalThis.WhWasmUtilInstaller = function(target){ - If `wasmUtilTarget.alloc` is not set and `instance.exports.malloc` is, it installs `wasmUtilTarget.alloc()` and `wasmUtilTarget.dealloc()` - wrappers for the exports `malloc` and `free` functions. + wrappers for the exports' `malloc` and `free` functions + if exports.malloc exists. It returns a function which, when called, initiates loading of the module and returns a Promise. When that Promise resolves, it calls @@ -2475,7 +2496,9 @@ globalThis.WhWasmUtilInstaller = function(target){ Error handling is up to the caller, who may attach a `catch()` call to the promise. */ -globalThis.WhWasmUtilInstaller.yawl = function(config){ +globalThis.WhWasmUtilInstaller +.yawl = function yawl(config){ + 'use strict'; const wfetch = ()=>fetch(config.uri, {credentials: 'same-origin'}); const wui = this; const finalThen = function(arg){ @@ -2500,7 +2523,7 @@ globalThis.WhWasmUtilInstaller.yawl = function(config){ tgt.alloc = function(n){ return exports.malloc(n) || toss("Allocation of",n,"bytes failed."); }; - tgt.dealloc = function(m){exports.free(m)}; + tgt.dealloc = function(m){m && exports.free(m)}; } wui(tgt); } @@ -2519,4 +2542,6 @@ globalThis.WhWasmUtilInstaller.yawl = function(config){ .then(finalThen) ; return loadWasm; -}.bind(globalThis.WhWasmUtilInstaller)/*yawl()*/; +}.bind( +globalThis.WhWasmUtilInstaller +)/*yawl()*/; diff --git a/ext/wasm/demo-jsstorage.js b/ext/wasm/demo-jsstorage.js index 587aa9cc5..e3ab5a9e5 100644 --- a/ext/wasm/demo-jsstorage.js +++ b/ext/wasm/demo-jsstorage.js @@ -16,7 +16,7 @@ */ 'use strict'; (function(){ - const T = self.SqliteTestUtil; + const T = globalThis.SqliteTestUtil; const toss = function(...args){throw new Error(args.join(' '))}; const debug = console.debug.bind(console); const eOutput = document.querySelector('#test-output'); @@ -40,7 +40,7 @@ const error = function(...args){ logHtml('error',...args); }; - + const runTests = function(sqlite3){ const capi = sqlite3.capi, oo = sqlite3.oo1, @@ -51,7 +51,7 @@ error("This build is not kvvfs-capable."); return; } - + const dbStorage = 0 ? 'session' : 'local'; const theStore = 's'===dbStorage[0] ? sessionStorage : localStorage; const db = new oo.JsStorageDb( dbStorage ); @@ -108,7 +108,7 @@ } }; - sqlite3InitModule(self.sqlite3TestModule).then((sqlite3)=>{ + sqlite3InitModule(globalThis.sqlite3TestModule).then((sqlite3)=>{ runTests(sqlite3); }); })(); diff --git a/ext/wasm/demo-worker1-promiser.c-pp.html b/ext/wasm/demo-worker1-promiser.c-pp.html index e0b487bdf..a1005beb9 100644 --- a/ext/wasm/demo-worker1-promiser.c-pp.html +++ b/ext/wasm/demo-worker1-promiser.c-pp.html @@ -6,10 +6,10 @@ -//#if target=es6-module - worker-promise (via ESM) tests +//#if target:es6-module + Worker1-promiser (ESM) tests //#else - worker-promise tests + Worker1-promiser tests //#endif @@ -32,7 +32,7 @@
        -//#if target=es6-module +//#if target:es6-module //#else diff --git a/ext/wasm/demo-worker1.js b/ext/wasm/demo-worker1.js index 1a05cc7ac..348741bf8 100644 --- a/ext/wasm/demo-worker1.js +++ b/ext/wasm/demo-worker1.js @@ -18,7 +18,7 @@ */ 'use strict'; (function(){ - const T = self.SqliteTestUtil; + const T = globalThis.SqliteTestUtil; const SW = new Worker("jswasm/sqlite3-worker1.js"); const DbState = { id: undefined @@ -323,7 +323,7 @@ switch(ev.result){ case 'worker1-ready': log("Message:",ev); - self.sqlite3TestModule.setStatus(null); + globalThis.sqlite3TestModule.setStatus(null); runTests(); return; default: @@ -344,5 +344,5 @@ }; log("Init complete, but async init bits may still be running."); log("Installing Worker into global scope SW for dev purposes."); - self.SW = SW; + globalThis.SW = SW; })(); diff --git a/ext/wasm/fiddle/fiddle-worker.js b/ext/wasm/fiddle/fiddle-worker.js index a5f3e25b7..4b1ea2c53 100644 --- a/ext/wasm/fiddle/fiddle-worker.js +++ b/ext/wasm/fiddle/fiddle-worker.js @@ -175,10 +175,6 @@ "features (e.g. upload) do not yet work with OPFS."); } stdout('\nEnter ".help" for usage hints.'); - this.exec([ // initialization commands... - '.nullvalue NULL', - '.headers on' - ].join('\n')); return true; }, /** diff --git a/ext/wasm/fiddle/index.html b/ext/wasm/fiddle/index.c-pp.html similarity index 95% rename from ext/wasm/fiddle/index.html rename to ext/wasm/fiddle/index.c-pp.html index 378cb3902..1f818286b 100644 --- a/ext/wasm/fiddle/index.html +++ b/ext/wasm/fiddle/index.c-pp.html @@ -5,20 +5,29 @@ SQLite3 Fiddle +//#if jqterm - + + +//#endif -[sqlite3]: https://sqlite.org -[emscripten]: https://emscripten.org -[sgb]: https://wanderinghorse.net/home/stephan/ [appendix-g]: #appendix-g -[StructBinderFactory]: #api-binderfactory -[StructCtors]: #api-structctor -[StructType]: #api-structtype +[BigInt64Array]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt64Array +[c-pp]: https://fossil.wanderinghorse.net/r/c-pp +[Emscripten]: https://emscripten.org +[jaccwabyt.js]: /file/jaccwabyt/jaccwabyt.c-pp.js +[MDN]: https://developer.mozilla.org/docs/Web/API +[sgb]: https://wanderinghorse.net/home/stephan/ +[sqlite3]: https://sqlite.org [StructBinder]: #api-structbinder +[StructBinderFactory]: #api-binderfactory +[StructCtor]: #api-structctor [StructInstance]: #api-structinstance -[^export-func]: In Emscripten, add its name, prefixed with `_`, to the - project's `EXPORT_FUNCTIONS` list. -[BigInt64Array]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt64Array +[StructType]: #api-structtype [TextDecoder]: https://developer.mozilla.org/en-US/docs/Web/API/TextDecoder [TextEncoder]: https://developer.mozilla.org/en-US/docs/Web/API/TextEncoder -[MDN]: https://developer.mozilla.org/docs/Web/API +[WASI-SDK]: https://github.com/WebAssembly/wasi-sdk +[whwasmutil.js]: /file/wasmutil/whwasmutil.c-pp.js + +[^export-func]: In Emscripten, add its name, prefixed with `_`, to the + project's `EXPORT_FUNCTIONS` list. diff --git a/ext/wasm/mkdist.sh b/ext/wasm/mkdist.sh index 84780668b..78b93e5e0 100755 --- a/ext/wasm/mkdist.sh +++ b/ext/wasm/mkdist.sh @@ -52,7 +52,7 @@ for arg in $@; do ;; --snapshot) - snapshotSuffix=$(date +%Y%m%d) + snapshotSuffix=-snapshot-$(date +%Y%m%d) ;; -?|--help) diff --git a/ext/wasm/mkwasmbuilds.c b/ext/wasm/mkwasmbuilds.c index 2730d9d76..37f2967d8 100644 --- a/ext/wasm/mkwasmbuilds.c +++ b/ext/wasm/mkwasmbuilds.c @@ -40,14 +40,22 @@ /* ** Flags for use with BuildDef::flags. ** -** Maintenance reminder: do not combine flags within this enum, -** e.g. F_BUNDLER_FRIENDLY=0x02|F_ESM, as that will lead -** to breakage in some of the flag checks. +** Maintenance reminder: do not combine F_... flags within this enum, +** e.g. F_BUNDLER_FRIENDLY=0x02|F_ESM, as that will lead to breakage +** in some of the flag checks. */ enum BuildDefFlags { /* Indicates an ESM module build. */ F_ESM = 0x01, - /* Indicates a "bundler-friendly" build mode. */ + /* Indicates a "bundler-friendly" build mode. These are untested and + ** unsupported, provided solely for the downstream npm subproject + ** (who is responsible for any testing of these). + ** + ** The only difference beween bundler-friendly and esm builds is + ** that bundlers require static filename strings in a few places due + ** to limitations of bundler tooling, whereas vanilla and JS can + ** both work with dynamic strings. + */ F_BUNDLER_FRIENDLY = 1<<1, /* Indicates that this build is unsupported. Such builds are not ** added to the 'all' target. The unsupported builds exist primarily @@ -57,8 +65,11 @@ enum BuildDefFlags { F_NOT_IN_ALL = 1<<3, /* If it's a 64-bit build. */ F_64BIT = 1<<4, - /* Indicates a node.js-for-node.js build (untested and - ** unsupported). */ + /* Indicates a node.js-for-node.js build. This build is very + ** specificially untested and unsupported, but the downstream npm + ** project makes use of it. None of our JS code is specific to node, + ** but Emscripten's generated sqlite3.js differs between + ** for-the-browser and for-node builds. */ F_NODEJS = 1<<5, /* Indicates a wasmfs build (untested and unsupported). */ F_WASMFS = 1<<6, @@ -71,13 +82,13 @@ enum BuildDefFlags { ** only their JS file and patch their JS to use the WASM file from a ** canonical build which uses that same WASM file. Reusing X.wasm ** that way can only work for builds which are processed identically - ** by Emscripten. For a given set of C flags (as opposed to + ** by Emscripten. For a given set of C flags (as distinct from ** JS-influencing flags), all builds of X.js and Y.js will produce ** identical X.wasm and Y.wasm files. Their JS files may well ** differ, however. */ - CP_JS = 1 << 30, - CP_WASM = 1 << 31, + CP_JS = 1 << 30, /* X.js or X.mjs, depending on F_ESM */ + CP_WASM = 1 << 31, /* X.wasm */ CP_ALL = CP_JS | CP_WASM }; @@ -94,11 +105,10 @@ enum BuildDefFlags { ** final build directory $(dir.dout). ** ** To keep parallel builds from stepping on each other, each distinct -** build goes into its own subdir $(dir.dout.BuildName)[^1], i.e. -** $(dir.dout)/BuildName. Builds which produce deliverables we'd like -** to keep/distribute copy their final results into the build dir -** $(dir.dout). See the notes for the CP_JS enum entry for more -** details on that. +** build goes into its own subdir $(dir.dout.$(BuildDef::zBaseName). +** Builds which produce deliverables we'd like to keep/distribute copy +** their final results into the build dir $(dir.dout). See the notes +** for the CP_JS enum entry for more details on that. ** ** The final result of each build is a pair of JS/WASM files, but ** getting there requires generation of several files, primarily as @@ -116,9 +126,9 @@ enum BuildDefFlags { ** ** --extern-post-js = gets injected immediately after ** sqlite3InitModule(), in the global scope. In this step we replace -** sqlite3InitModule() with a slightly customized, the main purpose of -** which is to (A) give us (not Emscripten) control over the arguments -** it accepts and (B) to run the library bootstrap step. +** sqlite3InitModule() with a slightly customized one, the main +** purpose of which is to (A) give us (not Emscripten) control over +** the arguments it accepts and (B) to run the library bootstrap step. ** ** Then there's sqlite3-api.BuildName.js, which is the entire SQLite3 ** JS API (generated from the list defined in $(sqlite3-api.jses)). It @@ -126,9 +136,7 @@ enum BuildDefFlags { ** ** Each of those inputs has to be generated before passing them on to ** Emscripten so that any build-specific capabilities can get filtered -** in or out (using ./c-pp.c). -** -** [^1]: The legal BuildNames are in this file's BuildDef_map macro. +** in or out (using ./c-pp-lite.c). */ struct BuildDef { /* @@ -143,8 +151,8 @@ struct BuildDef { ** ** The convention for 32- vs 64-bit pairs is to give them similar ** emoji, e.g. a cookie for 32-bit and a donut or cake for 64. - ** Alternately, the same emoji a "64" suffix, excep that that throws - ** off the output alignment in parallel builds ;). + ** Alternately, the same emoji with a "64" suffix, except that that + ** throws off the output alignment in parallel builds ;). */ const char *zEmo; /* @@ -161,13 +169,13 @@ struct BuildDef { const char *zCmppD; /* Extra -D... flags for c-pp */ const char *zEmcc; /* Full flags for emcc. Normally NULL for default. */ const char *zEmccExtra; /* Extra flags for emcc */ - const char *zDeps; /* Extra deps */ + const char *zDeps; /* Extra make target deps */ const char *zEnv; /* emcc -sENVIRONMENT=... value */ /* ** Makefile code "ifeq (...)". If set, this build is enclosed in a ** $zIfCond/endif block. */ - const char *zIfCond; /* makefile "ifeq (...)" or similar */ + const char *zIfCond; int flags; /* Flags from BuildDefFlags */ }; typedef struct BuildDef BuildDef; @@ -183,7 +191,7 @@ typedef struct BuildDef BuildDef; ** logtag.NAME = Used for decorating log output ** ** etc. -***/ +*/ #define BuildDefs_map(E) \ E(vanilla) E(vanilla64) \ E(esm) E(esm64) \ @@ -196,9 +204,8 @@ typedef struct BuildDef BuildDef; ** The set of WASM builds for the library (as opposed to the apps ** (fiddle, speedtest1)). Their order in BuildDefs_map is mostly ** insignificant, but some makefile vars used by some builds are set -** up by prior builds. Because of that, the (sqlite3, vanilla), -** (sqlite3, esm), and (sqlite3, bundler-friendly) builds should be -** defined first (in that order). +** up by prior builds. Because of that, the vanilla, esm, and +** bundler-friendly builds should be defined first (in that order). */ struct BuildDefs { #define E(N) BuildDef N; @@ -284,7 +291,7 @@ const BuildDefs oBuildDefs = { " -DSQLITE_SPEEDTEST1_WASM" " $(SQLITE_OPT)" " -USQLITE_WASM_BARE_BONES" - " -USQLITE_C -DSQLITE_C=$(sqlite3.canonical.c)" + " -USQLITE_C -DSQLITE_C=$(sqlite3.c)" " $(speedtest1.exit-runtime0)" " $(speedtest1.c.in)" " -lm", @@ -312,7 +319,7 @@ const BuildDefs oBuildDefs = { " -DSQLITE_SPEEDTEST1_WASM" " $(SQLITE_OPT)" " -USQLITE_WASM_BARE_BONES" - " -USQLITE_C -DSQLITE_C=$(sqlite3.canonical.c)" + " -USQLITE_C -DSQLITE_C=$(sqlite3.c)" " $(speedtest1.exit-runtime0)" " $(speedtest1.c.in)" " -lm", @@ -345,8 +352,7 @@ const BuildDefs oBuildDefs = { .zEnv = 0, .zDeps = 0, .zIfCond = 0, - .flags = CP_JS | F_BUNDLER_FRIENDLY | F_ESM - //| F_NOT_IN_ALL + .flags = CP_JS | F_BUNDLER_FRIENDLY | F_ESM | F_NOT_IN_ALL }, /* 64-bit bundler-friendly. */ @@ -371,7 +377,7 @@ const BuildDefs oBuildDefs = { .node = { .zEmo = "🍟", .zBaseName = "sqlite3-node", - .zDotWasm = 0, + .zDotWasm = "sqlite3", .zCmppD = "-Dtarget:node $(c-pp.D.bundler)", .zEmcc = 0, .zEmccExtra = 0, @@ -382,21 +388,21 @@ const BuildDefs oBuildDefs = { ** node. */, .zDeps = 0, .zIfCond = 0, - .flags = CP_ALL | F_UNSUPPORTED | F_NODEJS + .flags = CP_JS | F_UNSUPPORTED | F_ESM | F_NODEJS }, /* 64-bit node. */ .node64 = { .zEmo = "🍔", .zBaseName = "sqlite3-node-64bit", - .zDotWasm = 0, + .zDotWasm = "sqlite3-64bit", .zCmppD = "-Dtarget:node $(c-pp.D.bundler)", .zEmcc = 0, .zEmccExtra = 0, .zEnv = "node", .zDeps = 0, .zIfCond = 0, - .flags = CP_ALL | F_UNSUPPORTED | F_NODEJS | F_64BIT + .flags = CP_JS | F_UNSUPPORTED | F_ESM | F_NODEJS | F_64BIT }, /* Entirely unsupported. */ @@ -487,8 +493,8 @@ static void mk_prologue(void){ ** configuration). Comments like "saves nothing" may not be ** technically correct: "nothing" means "some neglible amount." ** - ** Note that performance gains/losses are _not_ taken into - ** account here: only wasm file size. + ** Performance gains or losses are _not_ taken into account + ** here, only wasm file size. */ "--enable-bulk-memory-opt " /* required */ "--all-features " /* required */ @@ -638,7 +644,7 @@ static void mk_pre_post(char const *zBuildName, BuildDef const * pB){ pB->zDotWasm); } ps(""); - pf("\t@$(call b.c-pp.shcmd," + pf("\t@$(call b.mkdir@); $(call b.c-pp.shcmd," "%s," "$(pre-js.in.js)," "$(pre-js.%s.js)," @@ -777,13 +783,14 @@ static void mk_lib_mode(const char *zBuildName, const BuildDef * pB){ zBuildName, zBuildName, zBaseName); pf("dir.dout.%s ?= $(dir.dout)/%s\n", zBuildName, zBuildName); - pf("out.%s.base ?= $(dir.dout.%s)/%s\n", - zBuildName, zBuildName, zBaseName); pf("c-pp.D.%s ?= %s\n", zBuildName, pB->zCmppD ? pB->zCmppD : ""); if( pB->flags & F_64BIT ){ pf("c-pp.D.%s += $(c-pp.D.64bit)\n", zBuildName); } + if( pB->flags & F_UNSUPPORTED ){ + pf("c-pp.D.%s += -Dunsupported-build\n", zBuildName); + } pf("emcc.environment.%s ?= %s\n", zBuildName, pB->zEnv ? pB->zEnv : oBuildDefs.vanilla.zEnv); @@ -914,10 +921,9 @@ static void mk_lib_mode(const char *zBuildName, const BuildDef * pB){ pf("\n%dbit: $(out.%s.js)\n" "$(out.%s.wasm): $(out.%s.js)\n" - "b-%s: $(out.%s.js) $(out.%s.wasm)\n", + "b-%s: $(out.%s.wasm)\n", (F_64BIT & pB->flags) ? 64 : 32, zBuildName, - zBuildName, zBuildName, - zBuildName, zBuildName, zBuildName); + zBuildName, zBuildName, zBuildName, zBuildName); if( CP_JS & pB->flags ){ pf("$(dir.dout)/%s%s: $(out.%s.js)\n", @@ -988,11 +994,18 @@ static void mk_fiddle(void){ pf("$(out.%s.js): $(MAKEFILE_LIST) " "$(EXPORTED_FUNCTIONS.fiddle) " "$(fiddle.c.in) " - "$(pre-post.%s.deps)\n", + "$(pre-post.%s.deps)", zBuildName, zBuildName); + if( isDebug ){ + pf(" $(dir.fiddle)/fiddle-worker.js" + " $(dir.fiddle)/fiddle.js" + " $(dir.fiddle)/index.html"); + } + ps(""); emit_compile_start(zBuildName); - pf("\t$(b.cmd@)$(bin.emcc) -o $@" - " $(emcc.flags.%s)" /* set in fiddle.make */ + pf("\t@$(call b.mkdir@)\n" + "\t$(b.cmd@)$(bin.emcc) -o $@" + " $(emcc.flags.%s)" /* set in GNUmakefile */ " $(pre-post.%s.flags)" " $(fiddle.c.in)" "\n", @@ -1057,7 +1070,7 @@ int main(int argc, char const ** argv){ BuildDefs_map(E) if( 0==strcmp("prologue",zArg) ){ mk_prologue(); }else { - fprintf(stderr,"Unkown build name: %s\n", zArg); + fprintf(stderr,"Unknown build name: %s\n", zArg); rc = 1; break; } diff --git a/ext/wasm/speedtest1-worker.js b/ext/wasm/speedtest1-worker.js index ba11fd163..9355ba93f 100644 --- a/ext/wasm/speedtest1-worker.js +++ b/ext/wasm/speedtest1-worker.js @@ -1,7 +1,7 @@ 'use strict'; (function(){ let speedtestJs = 'speedtest1.js'; - const urlParams = new URL(self.location.href).searchParams; + const urlParams = new URL(globalThis.location.href).searchParams; if(urlParams.has('sqlite3.dir')){ speedtestJs = urlParams.get('sqlite3.dir') + '/' + speedtestJs; } @@ -14,9 +14,9 @@ const wasmfsDir = function f(wasmUtil){ if(undefined !== f._) return f._; const pdir = '/opfs'; - if( !self.FileSystemHandle - || !self.FileSystemDirectoryHandle - || !self.FileSystemFileHandle){ + if( !globalThis.FileSystemHandle + || !globalThis.FileSystemDirectoryHandle + || !globalThis.FileSystemFileHandle){ return f._ = ""; } try{ @@ -92,7 +92,7 @@ } }; - self.onmessage = function(msg){ + globalThis.onmessage = function(msg){ msg = msg.data; switch(msg.type){ case 'run': @@ -111,7 +111,8 @@ setStatus: (text)=>mPost('load-status',text) }; log("Initializing speedtest1 module..."); - self.sqlite3InitModule(EmscriptenModule).then(async (sqlite3)=>{ + globalThis.sqlite3InitModule.__isUnderTest = true; + globalThis.sqlite3InitModule(EmscriptenModule).then(async (sqlite3)=>{ const S = globalThis.S = App.sqlite3 = sqlite3; log("Loaded speedtest1 module. Setting up..."); App.pDir = wasmfsDir(S.wasm); diff --git a/ext/wasm/speedtest1.html b/ext/wasm/speedtest1.html index cce617185..d41f20a4e 100644 --- a/ext/wasm/speedtest1.html +++ b/ext/wasm/speedtest1.html @@ -44,132 +44,137 @@ mounted, else it returns an empty string. */ const wasmfsDir = function f(wasmUtil){ - if(undefined !== f._) return f._; - const pdir = '/persistent'; - if( !self.FileSystemHandle - || !self.FileSystemDirectoryHandle - || !self.FileSystemFileHandle){ - return f._ = ""; - } - try{ - if(0===wasmUtil.xCallWrapped( - 'sqlite3_wasm_init_wasmfs', 'i32', ['string'], pdir - )){ - return f._ = pdir; - }else{ - return f._ = ""; - } - }catch(e){ - // sqlite3_wasm_init_wasmfs() is not available - return f._ = ""; + if(undefined !== f._) return f._; + const pdir = '/persistent'; + if( !self.FileSystemHandle + || !self.FileSystemDirectoryHandle + || !self.FileSystemFileHandle){ + return f._ = ""; + } + try{ + if(0===wasmUtil.xCallWrapped( + 'sqlite3_wasm_init_wasmfs', 'i32', ['string'], pdir + )){ + return f._ = pdir; + }else{ + return f._ = ""; } + }catch(e){ + // sqlite3_wasm_init_wasmfs() is not available + return f._ = ""; + } }; wasmfsDir._ = undefined; const eOut = document.querySelector('#test-output'); const log2 = function(cssClass,...args){ - const ln = document.createElement('div'); - if(cssClass) ln.classList.add(cssClass); - ln.append(document.createTextNode(args.join(' '))); - eOut.append(ln); - //this.e.output.lastElementChild.scrollIntoViewIfNeeded(); + const ln = document.createElement('div'); + if(cssClass) ln.classList.add(cssClass); + ln.append(document.createTextNode(args.join(' '))); + eOut.append(ln); + //this.e.output.lastElementChild.scrollIntoViewIfNeeded(); }; const logList = []; const dumpLogList = function(){ - logList.forEach((v)=>log2('',v)); - logList.length = 0; + logList.forEach((v)=>log2('',v)); + logList.length = 0; }; /* we cannot update DOM while speedtest is running unless we run speedtest in a worker thread. */; const log = (...args)=>{ - console.log(...args); - logList.push(args.join(' ')); + console.log(...args); + logList.push(args.join(' ')); }; const logErr = function(...args){ - console.error(...args); - logList.push('ERROR: '+args.join(' ')); + console.error(...args); + logList.push('ERROR: '+args.join(' ')); }; const runTests = function(sqlite3){ - const capi = sqlite3.capi, wasm = sqlite3.wasm; - //console.debug('sqlite3 =',sqlite3); - const pDir = wasmfsDir(wasm); - if(pDir){ - console.warn("Persistent storage:",pDir); - } - const scope = wasm.scopedAllocPush(); - let dbFile = pDir+"/speedtest1.db"; - const urlParams = new URL(self.location.href).searchParams; - const argv = ["speedtest1"]; - if(urlParams.has('flags')){ - argv.push(...(urlParams.get('flags').split(','))); - } + globalThis.S = sqlite3; + const capi = sqlite3.capi, wasm = sqlite3.wasm; + //console.debug('sqlite3 (global S) =',sqlite3); + const pDir = wasmfsDir(wasm); + if(pDir){ + console.warn("wasmfs storage:",pDir); + } + const scope = wasm.scopedAllocPush(); + let dbFile = pDir+"/speedtest1.db"; + const urlParams = new URL(self.location.href).searchParams; + const argv = ["speedtest1"]; + if(urlParams.has('flags')){ + argv.push(...(urlParams.get('flags').split(','))); + } - let forceSize = 0; - let vfs, pVfs = 0; - if(urlParams.has('vfs')){ - vfs = urlParams.get('vfs'); - pVfs = capi.sqlite3_vfs_find(vfs); - if(!pVfs){ - log2('error',"Unknown VFS:",vfs); - return; - } - argv.push("--vfs", vfs); - log2('',"Using VFS:",vfs); - if('kvvfs' === vfs){ - forceSize = 2 /* >2 is too big as of mid-2025 */; - dbFile = 'session'; - log2('warning',"kvvfs VFS: forcing --size",forceSize, - "and filename '"+dbFile+"'."); - capi.sqlite3_js_kvvfs_clear(dbFile); - } + let forceSize = 0; + let vfs, pVfs = 0; + if(urlParams.has('vfs')){ + vfs = urlParams.get('vfs'); + pVfs = capi.sqlite3_vfs_find(vfs); + if(!pVfs){ + log2('error',"Unknown VFS:",vfs); + return; } - if(forceSize){ - argv.push('--size',forceSize); - }else{ - [ - 'size' - ].forEach(function(k){ - const v = urlParams.get(k); - if(v) argv.push('--'+k, urlParams[k]); - }); + argv.push("--vfs", vfs); + log2('',"Using VFS:",vfs); + if('kvvfs' === vfs){ + forceSize = 5 + /* --size > 2 is too big for local/session storage + as of mid-2025, but kvvfs v2 (late 2025) + lets us use transient storage. */; + dbFile = 'transient'; + log2('warning',"kvvfs VFS: forcing --size",forceSize, + "and filename '"+dbFile+"'."); + capi.sqlite3_js_kvvfs_clear(dbFile); + } + } + if(forceSize){ + argv.push('--size',forceSize); + }else{ + [ + 'size' + ].forEach(function(k){ + const v = urlParams.get(k); + if(v && +v) argv.push('--'+k, urlParams[k]); + }); + } + argv.push( + "--singlethread", + //"--nomutex", + //"--nosync", + //"--memdb", // note that memdb trumps the filename arg + "--nomemstat" + ); + argv.push("--big-transactions"/*important for tests 410 and 510!*/, + dbFile); + console.log("argv =",argv); + // These log messages are not emitted to the UI until after main() returns. Fixing that + // requires moving the main() call and related cleanup into a timeout handler. + if(pDir) wasm.sqlite3_wasm_vfs_unlink(pVfs,dbFile); + log2('',"WASM heap size:",sqlite3.wasm.heap8().byteLength,"bytes"); + log2('',"WASM pointer size:",sqlite3.wasm.ptr.size); + log2('',"Starting native app:\n ",argv.join(' ')); + log2('',"This will take a while and the browser might warn about the runaway JS.", + "Give it time..."); + logList.length = 0; + setTimeout(function(){ + wasm.xCall('wasm_main', argv.length, + wasm.scopedAllocMainArgv(argv)); + wasm.scopedAllocPop(scope); + if('kvvfs'===vfs){ + logList.unshift("KVVFS "+dbFile+" size = "+ + capi.sqlite3_js_kvvfs_size(dbFile)); } - argv.push( - "--singlethread", - //"--nomutex", - //"--nosync", - //"--memdb", // note that memdb trumps the filename arg - "--nomemstat" - ); - argv.push("--big-transactions"/*important for tests 410 and 510!*/, - dbFile); - console.log("argv =",argv); - // These log messages are not emitted to the UI until after main() returns. Fixing that - // requires moving the main() call and related cleanup into a timeout handler. if(pDir) wasm.sqlite3_wasm_vfs_unlink(pVfs,dbFile); + logList.unshift("Done running native main(). Output:"); + dumpLogList(); log2('',"WASM heap size:",sqlite3.wasm.heap8().byteLength,"bytes"); - log2('',"WASM pointer size:",sqlite3.wasm.ptr.size); - log2('',"Starting native app:\n ",argv.join(' ')); - log2('',"This will take a while and the browser might warn about the runaway JS.", - "Give it time..."); - logList.length = 0; - setTimeout(function(){ - wasm.xCall('wasm_main', argv.length, - wasm.scopedAllocMainArgv(argv)); - wasm.scopedAllocPop(scope); - if('kvvfs'===vfs){ - logList.unshift("KVVFS "+dbFile+" size = "+ - capi.sqlite3_js_kvvfs_size(dbFile)); - } - if(pDir) wasm.sqlite3_wasm_vfs_unlink(pVfs,dbFile); - logList.unshift("Done running native main(). Output:"); - dumpLogList(); - log2('',"WASM heap size:",sqlite3.wasm.heap8().byteLength,"bytes"); - }, 50); + }, 50); }/*runTests()*/; self.sqlite3TestModule.print = log; self.sqlite3TestModule.printErr = logErr; + sqlite3InitModule.__isUnderTest = true; sqlite3InitModule(self.sqlite3TestModule).then(runTests); })(); diff --git a/ext/wasm/tester1-worker.c-pp.html b/ext/wasm/tester1-worker.c-pp.html index e461b6cbf..71d827a35 100644 --- a/ext/wasm/tester1-worker.c-pp.html +++ b/ext/wasm/tester1-worker.c-pp.html @@ -13,10 +13,18 @@

        sqlite3 tester #1: Worker thread (@bitness@-bit WASM)

        Variants: - conventional UI thread, - conventional worker, - ESM in UI thread, - ESM worker + conventional UI thread: + (32-bit, + 64-bit), + conventional worker: + (32-bit, + 64-bit), + ESM in UI thread: + (32-bit, + 64-bit), + ESM worker: + (32-bit + 64-bit)
        diff --git a/ext/wasm/tester1.c-pp.html b/ext/wasm/tester1.c-pp.html index 95fe52219..4bb53ee56 100644 --- a/ext/wasm/tester1.c-pp.html +++ b/ext/wasm/tester1.c-pp.html @@ -11,11 +11,19 @@

        -
        Variants: - conventional UI thread, - conventional worker, - ESM in UI thread, - ESM worker +
        Variants (32-bit): + conventional UI thread: + (32-bit, + 64-bit), + conventional worker: + (32-bit, + 64-bit), + ESM in UI thread: + (32-bit, + 64-bit), + ESM worker: + (32-bit + 64-bit)
        diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index f72e0803f..083b5eca4 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -153,6 +153,10 @@ globalThis.sqlite3InitModule = sqlite3InitModule; logClass('error',...args); }; + const debug = (...args)=>{ + console.debug('tester1',...args); + }; + const toss = (...args)=>{ error(...args); throw new Error(args.join(' ')); @@ -210,8 +214,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule; filter.test(error.message) passes. If it's a function, the test passes if filter(error) returns truthy. If it's a string, the test passes if the filter matches the exception message - precisely. In all other cases the test fails, throwing an - Error. + precisely. If filter is a number then it is compared against + the resultCode property of the exception. In all other cases + the test fails, throwing an Error. If it throws, msg is used as the error report unless it's falsy, in which case a default is used. @@ -225,7 +230,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule; if(filter instanceof RegExp) pass = filter.test(err.message); else if(filter instanceof Function) pass = filter(err); else if('string' === typeof filter) pass = (err.message === filter); + else if('number' === typeof filter) pass = (err.resultCode === filter); if(!pass){ + console.error("Filter",filter,"rejected exception",err); throw new Error(msg || ("Filter rejected this exception: <<"+err.message+">>")); } return this; @@ -369,6 +376,88 @@ globalThis.sqlite3InitModule = sqlite3InitModule; clearOnInit: true, initialCapacity: 6 }; + +//#if enable-see + /** + Code consolidator for SEE sanity checks for various VFSes. ctor + is the VFS's oo1.DB-type constructor. ctorOptFunc(bool) is a + function which must return a constructor args object for ctor. It + is passed true if the db needs to be cleaned up/unlinked before + opening it (OPFS) and false if not (how that is done is + VFS-dependent). dbUnlink is a function which is expected to + unlink() the db file if the ctorOpfFunc does not do so when + passed true (kvvfs). + + This function initializes the db described by ctorOptFunc(...), + writes some secret info into it, and re-opens it twice to + confirmi that it can be read with an SEE key and cannot be read + without one. + */ + T.seeBaseCheck = function(ctor, ctorOptFunc, dbUnlink){ + let initDb = true; + const tryKey = function(keyKey, key, expectCount){ + let db; + //console.debug('tryKey()',arguments); + try { + if (initDb) { + const ctoropt = ctorOptFunc(initDb); + initDb = false; + db = new ctor({ + ...ctoropt, + [keyKey]: key + }); + db.exec([ + "drop table if exists t;", + "create table t(a);" + ]); + db.close(); + db = null; + // Ensure that it's actually encrypted... + let err; + try { + db = new ctor(ctorOptFunc(false)); + T.assert(db, 'db opened') /* opening is fine, but... */; + db.exec("select 1 from sqlite_schema"); + console.warn("(should not be reached) sessionStorage =", sessionStorage); + } catch (e) { + err = e; + } finally { + db.close() + db = null; + } + T.assert(err, "Expecting an exception") + .assert(capi.SQLITE_NOTADB == err.resultCode, + "Expecting NOTADB"); + }/*initDb*/ + db = new ctor({ + ...ctorOptFunc(false), + [keyKey]: key + }); + db.exec("insert into t(a) values (1),(2)"); + T.assert(expectCount === db.selectValue('select sum(a) from t')); + } finally { + if (db) db.close(); + } + }; + const hexFoo = new Uint8Array([0x66,0x6f,0x6f]/*=="foo"*/); + dbUnlink(); + tryKey('textkey', 'foo', 3); + T.assert( !initDb ); + tryKey('textkey', 'foo', 6); + dbUnlink(); + initDb = true; + tryKey('key', 'foo', 3); + T.assert( !initDb ); + tryKey('key', hexFoo, 6); + dbUnlink(); + initDb = true; + tryKey('hexkey', hexFoo, 3); + T.assert( !initDb ); + tryKey('hexkey', hexFoo, 6); + dbUnlink(); + }, +//#endif enable-see + //////////////////////////////////////////////////////////////////////// // End of infrastructure setup. Now define the tests... //////////////////////////////////////////////////////////////////////// @@ -431,7 +520,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; assert(wasmCtypes.structs[1/*sqlite3_io_methods*/ ].members.xFileSize.offset>0); [ /* Spot-check a handful of constants to make sure they got installed... */ - 'SQLITE_SCHEMA','SQLITE_NULL','SQLITE_UTF8', + 'SQLITE_SCHEMA','SQLITE_NULL','SQLITE_UTF8','SQLITE_UTF8_ZT', 'SQLITE_STATIC', 'SQLITE_DIRECTONLY', 'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE' ].forEach((k)=>T.assert('number' === typeof capi[k])); @@ -945,7 +1034,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; let ptr = k1.pointer; k1.dispose(); T.assert(undefined === k1.pointer). - mustThrowMatching(()=>{k1.$pP=1}, /disposed instance/); + mustThrowMatching(()=>{k1.$pP=1}, /disposed/); }finally{ k1.dispose(); k2.dispose(); @@ -1549,6 +1638,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule; T.assert(blob instanceof Uint8Array). assert(0x68===blob[0] && 0x69===blob[1]); blob = null; + blob = db.selectValue("select ?1", new Uint8Array([97,0,98,0,99]), + sqlite3.capi.SQLITE_TEXT); + T.assert("a\0b\0c"===blob, "Something is amiss with embedded NULs"); let counter = 0, colNames = []; list.length = 0; db.exec(new TextEncoder('utf-8').encode("SELECT a a, b b FROM t"),{ @@ -2543,7 +2635,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .assert(wasm.isPtr(tmplMod.$xRowid)) .assert(wasm.isPtr(tmplMod.$xCreate)) .assert(tmplMod.$xCreate === tmplMod.$xConnect, - "setup() must make these equivalent and "+ + "setupModule() must make these equivalent and "+ "installMethods() must avoid re-compiling identical functions"); tmplMod.$xCreate = wasm.ptr.null /* make tmplMod eponymous-only */; let rc = capi.sqlite3_create_module( @@ -2794,25 +2886,69 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// T.g('kvvfs') .t({ - name: 'kvvfs is disabled in worker', - predicate: ()=>(isWorker() || "test is only valid in a Worker"), + name: 'kvvfs v1 API availability', test: function(sqlite3){ - T.assert( - !capi.sqlite3_vfs_find('kvvfs'), - "Expecting kvvfs to be unregistered." - ); + const capi = sqlite3.capi; + if( isUIThread() ){ + T.assert( !!capi.sqlite3_js_kvvfs_size ) + .assert( !!capi.sqlite3_js_kvvfs_clear ); + }else{ + /* Historical behaviour retained not for compatibility but + to help avoid some confusion between the v1 and v2 kvvfs + APIs (namely in how the v1 variants handle empty + strings). */ + T.assert( !capi.sqlite3_js_kvvfs_clear ) + .assert( !capi.sqlite3_js_kvvfs_size ); + } + const k = sqlite3.kvvfs; + T.assert( k && 'object'===typeof k ); + for(const n of ['reserve', 'import', 'export', + 'unlink', 'listen', 'unlisten', + 'exists', + 'estimateSize', 'clear'] ){ + T.assert( k[n] instanceof Function ); + } + + if( 0 ){ + const scope = wasm.scopedAllocPush(); + try{ + const pg = [ + "53514C69746520666F726D61742033b20b0101b402020d02d02l01d0", + "4l01jb02b2E91E00Dd011FD3b1FD3dxl2B010617171701377461626C", + "656B767666736B7676667302435245415445205441424C45206B76766673286129" + ].join(''); + const n = pg.length; + const pI = wasm.scopedAlloc( n+1 ); + const nO = 8192 * 2; + const pO = wasm.scopedAlloc( nO ); + const heap = wasm.heap8u(); + let i; + for( i=0; i(globalThis.sessionStorage || "sessionStorage is unavailable"), test: function(sqlite3){ - const filename = this.kvvfsDbFile = 'session'; + const JDb = sqlite3.oo1.JsStorageDb; const pVfs = capi.sqlite3_vfs_find('kvvfs'); T.assert(looksLikePtr(pVfs)); - const JDb = this.JDb = sqlite3.oo1.JsStorageDb; - const unlink = this.kvvfsUnlink = ()=>JDb.clearStorage(this.kvvfsDbFile); + let x = sqlite3.kvvfs.internal.storageForZClass('session'); + T.assert( 0 === x.files.length ) + .assert( globalThis.sessionStorage===x.storage ) + .assert( 'kvvfs-session-' === x.keyPrefix ); + const filename = this.kvvfsDbFile = 'session'; + const unlink = this.kvvfsUnlink = ()=>sqlite3.kvvfs.clear(filename); unlink(); let db = new JDb(filename); try { @@ -2820,87 +2956,533 @@ globalThis.sqlite3InitModule = sqlite3InitModule; 'create table kvvfs(a);', 'insert into kvvfs(a) values(1),(2),(3)' ]); - T.assert(3 === db.selectValue('select count(*) from kvvfs')); + T.assert(3 === db.selectValue('select count(*) from kvvfs')) + .assert( db.storageSize() > 0, "Db size counting is broken" ); db.close(); + db = undefined; db = new JDb(filename); db.exec('insert into kvvfs(a) values(4),(5),(6)'); T.assert(6 === db.selectValue('select count(*) from kvvfs')); }finally{ - db.close(); + if( db ) db.close(); } + //console.debug("sessionStorage",globalThis.sessionStorage); } }/*kvvfs sanity checks*/) -//#if enable-see .t({ - name: 'kvvfs with SEE encryption', - predicate: ()=>(isUIThread() - || "Only available in main thread."), + name: 'transient kvvfs', + //predicate: ()=>false, test: function(sqlite3){ - this.kvvfsUnlink(); - let initDb = true; - const tryKey = function(keyKey, key, expectCount){ - let db; - //console.debug('tryKey()',arguments); - const ctoropt = { - filename: this.kvvfsDbFile - //vfs: 'kvvfs' - //,flags: 'ct' + const filename = '.' /* preinstalled instance */; + const JDb = sqlite3.oo1.JsStorageDb; + const DB = sqlite3.oo1.DB; + T.mustThrowMatching(()=>new JDb(""), capi.SQLITE_MISUSE); + T.mustThrowMatching(()=>{ + new JDb("this\ns an illegal - contains control characters"); + /* We don't have a way to get error strings from xOpen() + to this point? xOpen() does not have a handle to the + db and SQLite is not calling xGetLastError() to fetch + the error string. */ + }, capi.SQLITE_RANGE); + T.mustThrowMatching(()=>{new JDb("foo-journal");}, + capi.SQLITE_MISUSE); + T.mustThrowMatching(()=>{new JDb("foo-wal");}, + capi.SQLITE_MISUSE); + T.mustThrowMatching(()=>{new JDb("foo-shm");}, + capi.SQLITE_MISUSE); + T.mustThrowMatching(()=>{ + new JDb("01234567890123456789"+ + "01234567890123456789"+ + "01234567890123456789"+ + "01234567890123456789"+ + "01234567890123456789"+ + "01234567890123456789"+ + "0"/*too long*/); + }, capi.SQLITE_RANGE); + { + const name = "01234567890123456789012" /* max name length */; + (new JDb(name)).close(); + T.assert( sqlite3.kvvfs.unlink(name) ); + } + + sqlite3.kvvfs.clear(filename); + let db = new JDb(filename); + const sqlSetup = [ + 'create table kvvfs(a);', + 'insert into kvvfs(a) values(1),(2),(3)' + ]; + try { + T.assert( 0===db.storageSize(), "expecting 0 storage size" ); + T.mustThrowMatching(()=>db.clearStorage(), /in-use/); + //db.clearStorage(); + T.assert( 0===db.storageSize(), "expecting 0 storage size" ); + db.exec(sqlSetup); + T.assert( 0db.clearStorage(), /in-use/); + //db.clearStorage(/*wiping everything out from under it*/); + T.assert( 0{ + db.close(); + db = undefined; }; - try { - if (initDb) { - initDb = false; - db = new this.JDb({ - ...ctoropt, - [keyKey]: key - }); - db.exec([ - "drop table if exists t;", - "create table t(a);" - ]); - db.close(); - // Ensure that it's actually encrypted... - let err; - try { - db = new this.JDb(ctoropt); - T.assert(db, 'db opened') /* opening is fine, but... */; - db.exec("select 1 from sqlite_schema"); - console.warn("(should not be reached) sessionStorage =", sessionStorage); - } catch (e) { - err = e; - } finally { - db.close() - } - T.assert(err, "Expecting an exception") - .assert(sqlite3.capi.SQLITE_NOTADB == err.resultCode, - "Expecting NOTADB"); - }/*initDb*/ - //console.debug('tryKey()',arguments); - db = new sqlite3.oo1.DB({ - ...ctoropt, - vfs: 'kvvfs', - [keyKey]: key + T.assert(3 === db.selectValue('select count(*) from kvvfs')); + close(); + + const exportDb = sqlite3.kvvfs.export; + db = new JDb(filename); + db.exec('insert into kvvfs(a) values(4),(5),(6)'); + T.assert(6 === db.selectValue('select count(*) from kvvfs')); + const exp = exportDb({name:filename,includeJournal:true}); + T.assert( filename===exp.name, "Broken export filename" ) + .assert( exp?.size > 0, "Missing db size" ) + .assert( exp?.pages?.length > 0, "Missing db pages" ); + console.debug("kvvfs to Object:",exp); + close(); + + const dbFileRaw = 'file:new-storage?vfs=kvvfs&delete-on-close=1'; + db = new DB({ + filename: dbFileRaw, + //flags: 'ct' + }); + db.exec(sqlSetup); + const dbFilename = db.dbFilename(); + //console.warn("db.dbFilename() =",dbFilename); + T.assert(3 === db.selectValue('select count(*) from kvvfs')); + debug("kvvfs to Object:",exportDb(dbFilename)); + const n = sqlite3.kvvfs.estimateSize( dbFilename ); + T.assert( n>0, "Db size count failed" ); + + if( 1 ){ + // Concurrent open of that same name uses the same storage + const x = new JDb(dbFilename); + T.assert(3 === db.selectValue('select count(*) from kvvfs')); + x.close(); + } + close(); + // When the final instance of a name is closed, the storage + // disappears... + T.mustThrowMatching(function(){ + /* Ensure that 'new-storage' was deleted when its refcount + went to 0, because of its 'transient' flag. By default + the objects are retained, like a filesystem would. */ + let ddb = new JDb(dbFilename); + try{ddb.selectValue('select a from kvvfs')} + finally{ddb.close()} + }, /no such table: kvvfs/); + }finally{ + if( db ) db.close(); + } + } + }/*transient kvvfs*/) + .t({ + name: 'concurrent transient kvvfs', + //predicate: ()=>false, + test: function(sqlite3){ + const filename = 'myStorage'; + const kvvfs = sqlite3.kvvfs; + const DB = sqlite3.oo1.DB; + const JDb = sqlite3.oo1.JsStorageDb; + let db; + let duo; + let q1, q2; + const sqlSetup = [ + 'create table kvvfs(a);', + 'insert into kvvfs(a) values(1),(2),(3)' + ]; + const sqlCount = 'select count(*) from kvvfs'; + + try { + const exportDb = sqlite3.kvvfs.export; + const dbFileRaw = 'file:'+filename+'?vfs=kvvfs&delete-on-close=1'; + sqlite3.kvvfs.clear(filename); + db = new DB(dbFileRaw); + db.exec(sqlSetup); + T.assert(3 === db.selectValue(sqlCount)); + + duo = new JDb(filename); + duo.exec('insert into kvvfs(a) values(4),(5),(6)'); + T.assert(6 === db.selectValue(sqlCount)); + const expOpt = { + name: filename, + decodePages: true + }; + let exp = exportDb(expOpt); + let expectRows = 6; + debug("exported db",exp); + db.close(); + T.assert(expectRows === duo.selectValue(sqlCount)); + duo.close(); + T.mustThrowMatching(function(){ + let ddb = new JDb(filename); + try{ddb.selectValue('select a from kvvfs')} + finally{ddb.close()} + }, /.*no such table: kvvfs.*/); + + T.assert( kvvfs.unlink(filename) ) + .assert( !kvvfs.exists(filename) ); + + const importDb = sqlite3.kvvfs.import; + duo = new JDb(dbFileRaw); + T.mustThrowMatching(()=>importDb(exp,true), /.*in use.*/); + duo.close(); + importDb(exp, true); + duo = new JDb(dbFileRaw); + T.assert(expectRows === duo.selectValue(sqlCount)); + let newCount; + try{ + duo.transaction(()=>{ + duo.exec("insert into kvvfs(a) values(7)"); + newCount = duo.selectValue(sqlCount); + T.assert(false, "rolling back"); }); - db.exec("insert into t(a) values (1),(2)"); - T.assert(expectCount === db.selectValue('select sum(a) from t')); - } finally { - if (db) db.close(); + }catch(e){/*ignored*/} + T.assert(7===newCount, "Unexpected row count before rollback") + .assert(expectRows === duo.selectValue(sqlCount), + "Unexpected row count after rollback"); + duo.close(); + + T.assert( kvvfs.unlink(filename) ) + .assert( !kvvfs.exists(filename) ); + + importDb(exp, true); + db = new JDb({ + filename, + flags: 'c' + /* BUG: without the 'c' flag, the db works until we try to + vacuum, at which point it fails with "read only db". */ + }); + duo = new JDb(filename); + T.assert(expectRows === duo.selectValue(sqlCount)); + const sqlIns1 = "insert into kvvfs(a) values(?)"; + q1 = db.prepare(sqlIns1); + q2 = duo.prepare(sqlIns1); + if( 0 ){ + q1.bind('from q1').stepFinalize(); + ++expectRows; + T.assert(expectRows === duo.selectValue(sqlCount), + "Unexpected record count."); + q2.bind('from q1').stepFinalize(); + ++expectRows; + }else{ + q1.bind('from q1'); + T.assert(capi.SQLITE_DONE===capi.sqlite3_step(q1), + "Unexpected step result"); + ++expectRows; + T.assert(expectRows === duo.selectValue(sqlCount), + "Unexpected record count."); + q2.bind('from q1').step(); + ++expectRows; + } + T.assert(expectRows === db.selectValue(sqlCount), + "Unexpected record count."); + q1.finalize(); + q2.finalize(); + + if( 1 ){ + debug("Begin vacuum/page size test..."); + const defaultPageSize = 1024 * 8 /* build-time default */; + const pageSize = 0 + ? defaultPageSize + : 1024 * 16 /* any valid value other than the default */; + if( 0 ){ + debug("Export before vacuum", exportDb(expOpt)); + debug("page size before vacuum", + db.selectArray( + "select page_size from pragma_page_size()" + )); + } + //kvvfs.log.xFileControl = true; + //kvvfs.log.xAccess = true; + db.exec([ + "BEGIN;", + "insert into kvvfs(a) values(randomblob(16000/*>pg size*/));", + "COMMIT;", + "delete from kvvfs where octet_length(a)>100;", + "pragma page_size="+pageSize+";", + "vacuum;", + "select 1;" + ]); + const expectPageSize = kvvfs.internal.disablePageSizeChange + ? defaultPageSize + : pageSize; + const gotPageSize = db.selectValue( + "select page_size from pragma_page_size()" + ); + T.assert(+gotPageSize === expectPageSize, + "Expecting page size",expectPageSize, + "got",gotPageSize); + T.assert(expectRows === duo.selectValue(sqlCount), + "Unexpected record count."); + kvvfs.log.xAccess = kvvfs.log.xFileControl = false; + T.assert(expectRows === duo.selectValue(sqlCount), + "Unexpected record count."); + exp = exportDb(expOpt); + debug("Exported page-expanded db",exp); + if( 0 ){ + debug("vacuumed export",exp); + } + debug("End vacuum/page size test."); + }else{ + expectRows = 6; } - }.bind(this); - const hexFoo = new Uint8Array([0x66,0x6f,0x6f]/*=="foo"*/); - tryKey('textkey', 'foo', 3); - T.assert( !initDb ); - tryKey('textkey', 'foo', 6); - this.kvvfsUnlink(); - initDb = true; - tryKey('key', 'foo', 3); - T.assert( !initDb ); - tryKey('key', hexFoo, 6); - this.kvvfsUnlink(); - initDb = true; - tryKey('hexkey', hexFoo, 3); - T.assert( !initDb ); - tryKey('hexkey', hexFoo, 6); - this.kvvfsUnlink(); + + db.close(); + duo.close(); + T.assert( kvvfs.unlink(exp.name) ) + .assert( !kvvfs.exists(exp.name) ); + importDb(exp); + T.assert( kvvfs.exists(exp.name) ); + db = new JDb(exp.name); + //debug("column count after export",db.selectValue(sqlCount)); + T.assert(expectRows === db.selectValue(sqlCount), + "Unexpected record count."); + + /* + TODO: more advanced concurrent use tests, e.g. looping + over a query in one connection while writing from + another. Currently that will probably corrupt the db, and + kvvfs's journaling does not support multiple journals per + storage unit. We need to test the locking and fix it as + appropriate. + */ + }finally{ + q1?.finalize?.(); + q2?.finalize?.(); + db?.close?.(); + duo?.close?.(); + } + } + }/*concurrent transient kvvfs*/) + + .t({ + name: 'kvvfs listeners (experiment)', + test: function(sqlite3){ + const kvvfs = sqlite3.kvvfs; + const filename = 'listen'; + let db; + try { + const DB = sqlite3.oo1.DB; + const sqlSetup = [ + 'create table kvvfs(a);', + 'insert into kvvfs(a) values(1),(2),(3)' + ]; + const sqlCount = "select count(*) from kvvfs"; + const sqlSelectSchema = "select * from sqlite_schema"; + const counts = Object.create(null); + const incr = (key)=>counts[key] = 1 + (counts[key] ?? 0); + const pglog = Object.assign(Object.create(null),{ + pages: [], + jrnl: undefined, + size: undefined, + includeJournal: false, + decodePages: true, + exception: new Error("Testing that exceptions from listeners do not interfere") + }); + const toss = ()=>{ + if( pglog.exception ){ + const e = pglog.exception; + delete pglog.exception; + throw e; + } + }; + + const listener = { + storage: filename, + reserve: true, + includeJournal: pglog.includeJournal, + decodePages: pglog.decodePages, + events: { + /** + These may be async but must not be in this case + because we can't test their result without a lot of + hoop-jumping if they are. Kvvfs calls these + asynchronously, though. + */ + 'open': (ev)=>{ + //console.warn('open',ev); + incr(ev.type); + T.assert(filename===ev.storageName) + .assert('number'===typeof ev.data); + }, + 'close': (ev)=>{ + //console.warn('close',ev); + incr(ev.type); + T.assert('number'===typeof ev.data); + toss(); + }, + 'delete': (ev)=>{ + //console.warn('delete',ev); + incr(ev.type); + T.assert('string'===typeof ev.data); + switch(ev.data){ + case 'jrnl': + T.assert(pglog.includeJournal); + pglog.jrnl = null; + break; + default:{ + const n = +ev.data; + T.assert( n>0, "Expecting positive db page number" ); + if( n < pglog.pages.length ){ + pglog.size = undefined; + } + pglog.pages[n] = undefined; + break; + } + } + }, + 'sync': (ev)=>{ + incr(ev.data ? 'xSync' : 'xFileControlSync'); + }, + 'write': (ev)=>{ + //console.warn('write',ev); + incr(ev.type); + T.assert(Array.isArray(ev.data)); + const key = ev.data[0], val = ev.data[1]; + T.assert('string'===typeof key); + switch( key ){ + case 'jrnl': + T.assert(pglog.includeJournal); + pglog.jrnl = val; + break; + case 'sz':{ + const sz = +val; + T.assert( sz>0, "Expecting a db page number" ); + if( sz < pglog.sz ){ + pglog.pages.length = sz / pglog.pages.length; + } + pglog.size = sz; + break; + } + default: + T.assert( +key>0, "Expecting a positive db page number" ); + pglog.pages[+key] = val; + if( pglog.decodePages ){ + T.assert( val instanceof Uint8Array ); + }else{ + T.assert( 'string'===typeof val ); + } + break; + } + } + } + }; + + kvvfs.listen(listener); + const dbFileRaw = 'file:'+filename+'?vfs=kvvfs&delete-on-close=1'; + const expOpt = { + name: filename, + //decodePages: true + }; + db = new DB(dbFileRaw); + db.exec(sqlSetup); + T.assert(db.selectObjects(sqlSelectSchema)?.length>0, + "Unexpected empty schema"); + db.close(); + debug("kvvfs listener counts:",counts); + T.assert( counts.open ); + T.assert( counts.close ); + T.assert( listener.includeJournal ? counts.delete : !counts.delete ); + T.assert( counts.write ); + T.assert( counts.xSync ); + T.assert( counts.xFileControlSync>=counts.xSync ); + T.assert( counts.open===counts.close ); + T.assert( pglog.includeJournal + ? (null===pglog.jrnl) + : (undefined===pglog.jrnl), + "Unexpected pglog.jrnl value: "+pglog.jrnl ); + if( 1 ){ + T.assert(undefined===pglog.pages[0], "Expecting empty slot 0"); + pglog.pages.shift(); + //debug("kvvfs listener pageLog", pglog); + } + const before = JSON.stringify(counts); + T.assert( kvvfs.unlisten(listener) ); + T.assert( !kvvfs.unlisten(listener) ); + db = new DB(dbFileRaw); + T.assert( db.selectObjects(sqlSelectSchema)?.length>0 ); + const exp = kvvfs.export(expOpt); + const expectRows = db.selectValue(sqlCount); + db.exec("delete from kvvfs"); + db.close(); + const after = JSON.stringify(counts); + T.assert( before===after, "Expecting no events after unlistening." ); + if( 0 ){ + exp = kvvfs.export(expOpt); + debug("Post-delete export:",exp); + } + if( 1 ){ + // Replace the storage with the pglog state... + const bogoExp = { + name: filename, + size: pglog.size, + timestamp: Date.now(), + pages: pglog.pages + }; + //debug("exp",exp); + //debug("bogoExp",bogoExp); + kvvfs.import(bogoExp, true); + //debug("Re-exported", kvvfs.export(expOpt)); + db = new DB(dbFileRaw); + // Failing on the next line despite exports looking good + T.assert(db.selectObjects(sqlSelectSchema)?.length>0, + "Empty schema on imported db"); + T.assert(expectRows===db.selectValue(sqlCount)); + db.close(); + } + }finally{ + db?.close?.(); + kvvfs.unlink(filename); + } + } + })/*kvvfs listeners */ + + .t({ + name: 'kvvfs vtab', + predicate: (sqlite3)=>!!sqlite3.kvvfs.create_module, + test: function(sqlite3){ + const kvvfs = sqlite3.kvvfs; + const db = new sqlite3.oo1.DB(); + const db2 = new sqlite3.oo1.DB('file:foo?vfs=kvvfs&delete-on-close=1'); + try{ + kvvfs.create_module(db); + let rc = db.selectObjects("select * from sqlite_kvvfs order by name"); + debug("sqlite_kvvfs vtab:", rc); + const nDb = rc.length; + rc = db.selectObject("select * from sqlite_kvvfs where name='foo'"); + T.assert(rc, "Expecting foo storage record") + .assert('foo'===rc.name, "Unexpected name") + .assert(1===rc.nRef, "Unexpected refcount"); + db2.close(); + rc = db.selectObjects("select * from sqlite_kvvfs"); + T.assert( !rc.filter((o)=>o.name==='foo').length, + "Expecting foo storage to be gone"); + debug("sqlite_kvvfs vtab:", rc); + T.assert( rc.length+1 === nDb, + "Unexpected storage count: got",rc.length,"expected",nDb); + }finally{ + db.close(); + db2.close(); + } + } + })/* kvvfs vtab */ + +//#if enable-see + .t({ + name: 'kvvfs SEE encryption in sessionStorage', + predicate: ()=>(!!globalThis.sessionStorage + || "sessionStorage is not available"), + test: function(sqlite3){ + const JDb = sqlite3.oo1.JsStorageDb; + T.seeBaseCheck(JDb, + (isInit)=>return {filename: "session"}, + ()=>JDb.clearStorage('session')); } })/*kvvfs with SEE*/ //#endif enable-see @@ -2915,7 +3497,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; let countCommit = 0, countRollback = 0;; const db = new sqlite3.oo1.DB(':memory:',1 ? 'c' : 'ct'); let rc = capi.sqlite3_commit_hook(db, (p)=>{ - //console.debug("commit hook",arguments); + //debug("commit hook",arguments); ++countCommit; return (17 == p) ? 0 : capi.SQLITE_ERROR; }, 17); @@ -3315,71 +3897,14 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .t({ name: 'OPFS with SEE encryption', test: function(sqlite3){ - const dbFile = 'file:///sqlite3-see.edb'; - const dbCtor = sqlite3.oo1.OpfsDb; - const hexFoo = new Uint8Array([0x66,0x6f,0x6f]/*=="foo"*/); - let initDb = true; - const tryKey = function(keyKey, key, expectCount){ - let db; - //console.debug('tryKey()',arguments); - const ctoropt = { - filename: dbFile, - flags: 'c' - }; - try { - if (initDb) { - initDb = false; - const opt = { - ...ctoropt, - [keyKey]: key - }; - opt.filename += '?delete-before-open=1'; - db = new dbCtor(opt); - db.exec([ - "drop table if exists t;", - "create table t(a);" - ]); - db.close(); - // Ensure that it's actually encrypted... - let err; - try { - db = new dbCtor(ctoropt); - T.assert(db, 'db opened') /* opening is fine, but... */; - const rv = db.exec({ - sql:"select count(*) from sqlite_schema", - returnValue: 'resultRows' - }); - console.warn("(should not be reached) rv =",rv); - } catch (e) { - err = e; - } finally { - db.close() - } - T.assert(err, "Expecting an exception") - .assert(sqlite3.capi.SQLITE_NOTADB == err.resultCode, - "Expecting NOTADB"); - }/*initDb*/ - db = new dbCtor({ - ...ctoropt, - [keyKey]: key - }); - db.exec("insert into t(a) values (1),(2)"); - T.assert(expectCount === db.selectValue('select sum(a) from t')); - } finally { - if (db) db.close(); - } - }; - tryKey('textkey', 'foo', 3); - T.assert( !initDb ); - tryKey('textkey', 'foo', 6); - initDb = true; - tryKey('key', 'foo', 3); - T.assert( !initDb ); - tryKey('key', hexFoo, 6); - initDb = true; - tryKey('hexkey', hexFoo, 3); - T.assert( !initDb ); - tryKey('hexkey', hexFoo, 6); + T.seeBaseCheck( + sqlite3.oo1.OpfsDb, + function(isInit){ + const opt = {filename: 'file:///sqlite3-see.edb'}; + if( isInit ) opt.filename += '?delete-before-open=1'; + return opt; + }, + ()=>{}); } })/*OPFS with SEE*/ //#endif enable-see @@ -3584,69 +4109,11 @@ globalThis.sqlite3InitModule = sqlite3InitModule; let poolUtil; const P1 = await inst(poolConfig).then(u=>poolUtil = u).catch(catcher); const dbFile = '/sqlite3-see.edb'; - const dbCtor = poolUtil.OpfsSAHPoolDb; - const hexFoo = new Uint8Array([0x66,0x6f,0x6f]/*=="foo"*/); - let initDb = true; - const tryKey = function(keyKey, key, expectCount){ - let db; - //console.debug('tryKey()',arguments); - const ctoropt = { - filename: dbFile, - flags: 'c' - }; - try { - if (initDb) { - initDb = false; - poolUtil.unlink(dbFile); - db = new dbCtor({ - ...ctoropt, - [keyKey]: key - }); - db.exec([ - "drop table if exists t;", - "create table t(a);" - ]); - db.close(); - // Ensure that it's actually encrypted... - let err; - try { - db = new dbCtor(ctoropt); - T.assert(db, 'db opened') /* opening is fine, but... */; - const rv = db.exec({ - sql:"select count(*) from sqlite_schema", - returnValue: 'resultRows' - }); - console.warn("(should not be reached) rv =",rv); - } catch (e) { - err = e; - } finally { - db.close() - } - T.assert(err, "Expecting an exception") - .assert(sqlite3.capi.SQLITE_NOTADB == err.resultCode, - "Expecting NOTADB"); - }/*initDb*/ - db = new dbCtor({ - ...ctoropt, - [keyKey]: key - }); - db.exec("insert into t(a) values (1),(2)"); - T.assert(expectCount === db.selectValue('select sum(a) from t')); - } finally { - if (db) db.close(); - } - }; - tryKey('textkey', 'foo', 3); - T.assert( !initDb ); - tryKey('textkey', 'foo', 6); - initDb = true; - tryKey('key', 'foo', 3); - T.assert( !initDb ); - tryKey('key', hexFoo, 6); - initDb = true; - tryKey('hexkey', hexFoo, 3); - T.assert( !initDb ); - tryKey('hexkey', hexFoo, 6); + T.seeBaseCheck( + poolUtil.OpfsSAHPoolDb, + (isInit)=>{return {filename: dbFile}}, + ()=>poolUtil.unlink(dbFile) + ); poolUtil.removeVfs(); } })/*opfs-sahpool with SEE*/ @@ -3715,44 +4182,54 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .t("Misc. stmt_...", function(sqlite3){ const db = new sqlite3.oo1.DB(); db.exec("create table t(a doggiebiscuits); insert into t(a) values(123)"); - const stmt = db.prepare("select a, a+1 from t"); - T.assert( stmt.isReadOnly() ) - .assert( 0===capi.sqlite3_stmt_isexplain(stmt) ) - .assert( 0===capi.sqlite3_stmt_explain(stmt, 1) ) - .assert( 0!==capi.sqlite3_stmt_isexplain(stmt) ) - .assert( 0===capi.sqlite3_stmt_explain(stmt, 2) ) - .assert( 0!==capi.sqlite3_stmt_isexplain(stmt) ) - .assert( 0===capi.sqlite3_stmt_explain(stmt, 0) ) - .assert( 0===capi.sqlite3_stmt_isexplain(stmt) ); - let n = 0; - while( capi.SQLITE_ROW === capi.sqlite3_step(stmt) ){ - ++n; - T.assert( 0!==capi.sqlite3_stmt_explain(stmt, 1), - "Because stmt is busy" ) - .assert( capi.sqlite3_stmt_busy(stmt) ) - .assert( stmt.isBusy() ) - .assert( 0!==capi.sqlite3_stmt_readonly(stmt) ) - .assert( true===stmt.isReadOnly() ); - const sv = capi.sqlite3_column_value(stmt, 0); - T.assert( 123===capi.sqlite3_value_int(sv) ) - .assert( "doggiebiscuits"===capi.sqlite3_column_decltype(stmt,0) ) - .assert( null===capi.sqlite3_column_decltype(stmt,1) ); - } - T.assert( 1===n ) - .assert( 0===capi.sqlite3_stmt_busy(stmt) ) - .assert( !stmt.isBusy() ); - - if( wasm.exports.sqlite3_column_origin_name ){ - log("Column metadata APIs enabled"); - T.assert( "t" === capi.sqlite3_column_table_name(stmt, 0)) - .assert("a" === capi.sqlite3_column_origin_name(stmt, 0)) - .assert("main" === capi.sqlite3_column_database_name(stmt, 0)) - }else{ - log("Column metadata APIs not enabled"); - } // column metadata APIs - - stmt.finalize(); - db.close(); + let stmt; + try{ + stmt = db.prepare("select a, a+1 from t"); + T.assert( stmt.isReadOnly() ) + .assert( 0===capi.sqlite3_stmt_isexplain(stmt) ) + .assert( 0===capi.sqlite3_stmt_explain(stmt, 1) ) + .assert( 0!==capi.sqlite3_stmt_isexplain(stmt) ) + .assert( 0===capi.sqlite3_stmt_explain(stmt, 2) ) + .assert( 0!==capi.sqlite3_stmt_isexplain(stmt) ) + .assert( 0===capi.sqlite3_stmt_explain(stmt, 0) ) + .assert( 0===capi.sqlite3_stmt_isexplain(stmt) ); + let n = 0; + while( capi.SQLITE_ROW === capi.sqlite3_step(stmt) ){ + ++n; + T.assert( 0!==capi.sqlite3_stmt_explain(stmt, 1), + "Because stmt is busy" ) + .assert( capi.sqlite3_stmt_busy(stmt) ) + .assert( stmt.isBusy() ) + .assert( 0!==capi.sqlite3_stmt_readonly(stmt) ) + .assert( true===stmt.isReadOnly() ); + const sv = capi.sqlite3_column_value(stmt, 0); + T.assert( 123===capi.sqlite3_value_int(sv) ) + .assert( "doggiebiscuits"===capi.sqlite3_column_decltype(stmt,0) ) + .assert( null===capi.sqlite3_column_decltype(stmt,1) ); + } + T.assert( 1===n ) + .assert( 0===capi.sqlite3_stmt_busy(stmt) ) + .assert( !stmt.isBusy() ); + + if( wasm.exports.sqlite3_column_origin_name ){ + log("Column metadata APIs enabled"); + T.assert( "t" === capi.sqlite3_column_table_name(stmt, 0)) + .assert("a" === capi.sqlite3_column_origin_name(stmt, 0)) + .assert("main" === capi.sqlite3_column_database_name(stmt, 0)) + }else{ + log("Column metadata APIs not enabled"); + } // column metadata APIs + stmt.finalize(); + stmt = null; + stmt = db.prepare("select ?1").bind(new Uint8Array([97,0,98,0,99])); + stmt.step(); + const sv = capi.sqlite3_column_value(stmt,0); + T.assert("a\0b\0c"===capi.sqlite3_value_text(sv), + "Expecting NULs to have survived."); + }finally{ + if(stmt) stmt.finalize(); + db.close(); + } }) //////////////////////////////////////////////////////////////////// diff --git a/main.mk b/main.mk index ac0fdb513..46d646d03 100644 --- a/main.mk +++ b/main.mk @@ -652,7 +652,7 @@ SRC = \ $(TOP)/src/sqliteInt.h \ $(TOP)/src/sqliteLimit.h \ $(TOP)/src/table.c \ - $(TOP)/src/tclsqlite.c \ + tclsqlite-ex.c \ $(TOP)/src/threads.c \ $(TOP)/src/tokenize.c \ $(TOP)/src/treeview.c \ @@ -952,25 +952,6 @@ FUZZDATA = \ # TESTOPTS = --verbose=file --output=test-out.txt -# -# Extra compiler options for various shell tools -# -# Note that some of these will only apply when embedding sqlite3.c -# into the shell, as these flags are not otherwise passed on to the -# library. -SHELL_OPT += -DSQLITE_DQS=0 -SHELL_OPT += -DSQLITE_ENABLE_FTS4 -#SHELL_OPT += -DSQLITE_ENABLE_FTS5 -SHELL_OPT += -DSQLITE_ENABLE_RTREE -SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS -SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION -SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB -SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB -SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB -SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB -SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC -SHELL_OPT += -DSQLITE_ENABLE_PERCENTILE -SHELL_OPT += -DSQLITE_STRICT_SUBTYPE=1 FUZZERSHELL_OPT = FUZZCHECK_OPT += -I$(TOP)/test FUZZCHECK_OPT += -I$(TOP)/ext/recover @@ -1428,15 +1409,15 @@ whereexpr.o: $(TOP)/src/whereexpr.c $(DEPS_OBJ_COMMON) window.o: $(TOP)/src/window.c $(DEPS_OBJ_COMMON) $(T.cc.sqlite) -c $(TOP)/src/window.c -tclsqlite.o: $(T.tcl.env.sh) $(TOP)/src/tclsqlite.c $(DEPS_OBJ_COMMON) +tclsqlite.o: $(T.tcl.env.sh) tclsqlite-ex.c $(DEPS_OBJ_COMMON) $(T.compile.tcl) -DUSE_TCL_STUBS=1 $$TCL_INCLUDE_SPEC \ - -c $(TOP)/src/tclsqlite.c + -c tclsqlite-ex.c -o tclsqlite.o -tclsqlite-shell.o: $(T.tcl.env.sh) $(TOP)/src/tclsqlite.c $(DEPS_OBJ_COMMON) - $(T.compile.tcl) -DTCLSH -o $@ -c $(TOP)/src/tclsqlite.c $$TCL_INCLUDE_SPEC +tclsqlite-shell.o: $(T.tcl.env.sh) tclsqlite-ex.c $(DEPS_OBJ_COMMON) + $(T.compile.tcl) -DTCLSH -o $@ -c tclsqlite-ex.c $$TCL_INCLUDE_SPEC -tclsqlite-stubs.o: $(T.tcl.env.sh) $(TOP)/src/tclsqlite.c $(DEPS_OBJ_COMMON) - $(T.compile.tcl) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c $$TCL_INCLUDE_SPEC +tclsqlite-stubs.o: $(T.tcl.env.sh) tclsqlite-ex.c $(DEPS_OBJ_COMMON) + $(T.compile.tcl) -DUSE_TCL_STUBS=1 -o $@ -c tclsqlite-ex.c $$TCL_INCLUDE_SPEC # # STATIC_TCLSQLITE3 = 1 to statically link tclsqlite3, else @@ -1680,11 +1661,19 @@ install-tcl-0 install-tcl-: install-tcl: install-tcl-$(HAVE_TCL) install: install-tcl -tclsqlite3.c: sqlite3.c +TCLSQLITEEX = \ + $(TOP)/ext/qrf/qrf.h \ + $(TOP)/ext/qrf/qrf.c \ + $(TOP)/src/tclsqlite.c + +tclsqlite-ex.c: $(TCLSQLITEEX) $(TOP)/tool/mkcombo.tcl $(B.tclsh) + $(B.tclsh) $(TOP)/tool/mkcombo.tcl $(TCLSQLITEEX) -o $@ + +tclsqlite3.c: sqlite3.c tclsqlite-ex.c echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c cat sqlite3.c >>tclsqlite3.c echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c - cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c + cat tclsqlite-ex.c >>tclsqlite3.c # # $(CFLAGS.tclextension) = CFLAGS for the tclextension* targets. @@ -1801,13 +1790,13 @@ TESTFIXTURE_FLAGS += -DSQLITE_STRICT_SUBTYPE=1 TESTFIXTURE_SRC0 = $(TESTSRC2) $(libsqlite3.LIB) TESTFIXTURE_SRC1 = sqlite3.c -TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c +TESTFIXTURE_SRC = $(TESTSRC) tclsqlite-ex.c TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION)) testfixture$(T.exe): $(T.tcl.env.sh) has_tclsh85 $(TESTFIXTURE_SRC) $(T.link.tcl) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \ -o $@ $(TESTFIXTURE_SRC) \ - $$TCL_LIB_SPEC $$TCL_INCLUDE_SPEC \ + $$TCL_LIB_SPEC $$TCL_INCLUDE_SPEC $$TCL_LIBS \ $(LDFLAGS.libsqlite3) coretestprogs: testfixture$(B.exe) sqlite3$(B.exe) @@ -1874,6 +1863,15 @@ mdevtest: srctree-check has_tclsh85 sdevtest: has_tclsh85 $(TCLSH_CMD) $(TOP)/test/testrunner.tcl sdevtest $(TSTRNNR_OPTS) +retest: has_tclsh85 + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl retest + +errors: + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl errors + +status: + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl status -d 2 + # Like releasetest, except it omits srctree-check and verify-source so # that it can be used on a modified source tree. # @@ -1931,7 +1929,7 @@ shelltest: # sqlite3_analyzer.c.flags.0 = -DINCLUDE_SQLITE3_C=1 sqlite3_analyzer.c.flags.1 = -sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl \ +sqlite3_analyzer.c: sqlite3.c tclsqlite-ex.c $(TOP)/tool/spaceanal.tcl \ $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in $(B.tclsh) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in \ $(sqlite3_analyzer.c.flags.$(LINK_TOOLS_DYNAMICALLY)) \ @@ -1955,7 +1953,7 @@ sqlite3_analyzer$(T.exe): $(T.tcl.env.sh) sqlite3_analyzer.c \ # can cause the $@ to link to an out-of-tree libsqlite3.so, which may # or may not fail or otherwise cause confusion. -sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl \ +sqltclsh.c: sqlite3.c tclsqlite-ex.c $(TOP)/tool/sqltclsh.tcl \ $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl \ $(TOP)/tool/sqltclsh.c.in $(B.tclsh) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c @@ -1973,24 +1971,6 @@ sqlite3_expert$(T.exe): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqli $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(LDFLAGS.libsqlite3) xbin: sqlite3_expert$(T.exe) -CHECKER_DEPS =\ - $(TOP)/tool/mkccode.tcl \ - sqlite3.c \ - $(TOP)/src/tclsqlite.c \ - $(TOP)/ext/repair/sqlite3_checker.tcl \ - $(TOP)/ext/repair/checkindex.c \ - $(TOP)/ext/repair/checkfreelist.c \ - $(TOP)/ext/misc/btreeinfo.c \ - $(TOP)/ext/repair/sqlite3_checker.c.in - -sqlite3_checker.c: $(CHECKER_DEPS) - $(B.tclsh) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@ - -sqlite3_checker$(T.exe): $(T.tcl.env.sh) sqlite3_checker.c - $(T.link.tcl) sqlite3_checker.c -o $@ $$TCL_INCLUDE_SPEC \ - $$TCL_LIB_SPEC $(LDFLAGS.libsqlite3) -xbin: sqlite3_checker$(T.exe) - dbdump$(T.exe): $(TOP)/ext/misc/dbdump.c sqlite3.o $(T.link) -DDBDUMP_STANDALONE -o $@ \ $(TOP)/ext/misc/dbdump.c sqlite3.o $(LDFLAGS.libsqlite3) @@ -2020,6 +2000,10 @@ showshm$(T.exe): $(TOP)/tool/showshm.c $(T.link) -o $@ $(TOP)/tool/showshm.c $(LDFLAGS.configure) xbin: showshm$(T.exe) +showtmlog$(T.exe): $(TOP)/tool/showtmlog.c + $(T.link) -o $@ $(TOP)/tool/showtmlog.c $(LDFLAGS.configure) +xbin: showshm$(T.exe) + index_usage$(T.exe): $(TOP)/tool/index_usage.c sqlite3.o $(T.link) $(SHELL_OPT) -o $@ $(TOP)/tool/index_usage.c sqlite3.o \ $(LDFLAGS.libsqlite3) @@ -2118,16 +2102,19 @@ src-archives: sqlite-amalgamation.zip amalgamation-tarball sqlite-src.zip # Build a ZIP archive containing various command-line tools. # -tool-zip: testfixture$(T.exe) sqlite3$(T.exe) sqldiff$(T.exe) \ +tool-zip: sqlite3$(T.exe) sqldiff$(T.exe) \ sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) $(TOP)/tool/mktoolzip.tcl strip sqlite3$(T.exe) sqldiff$(T.exe) sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) - ./testfixture$(T.exe) $(TOP)/tool/mktoolzip.tcl + $(TCLSH_CMD) $(TOP)/tool/mktoolzip.tcl + snapshot-zip: testfixture$(T.exe) sqlite3$(T.exe) sqldiff$(T.exe) \ sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) $(TOP)/tool/mktoolzip.tcl strip sqlite3$(T.exe) sqldiff$(T.exe) sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) - ./testfixture$(T.exe) $(TOP)/tool/mktoolzip.tcl --snapshot + $(TCLSH_CMD) $(TOP)/tool/mktoolzip.tcl --snapshot + clean-tool-zip: rm -f sqlite-tools-*.zip + clean: clean-tool-zip # @@ -2352,6 +2339,8 @@ mptest: mptester$(T.exe) # Source and header files that shell.c depends on SHELL_DEP = \ $(TOP)/src/shell.c.in \ + $(TOP)/ext/qrf/qrf.c \ + $(TOP)/ext/qrf/qrf.h \ $(TOP)/ext/expert/sqlite3expert.c \ $(TOP)/ext/expert/sqlite3expert.h \ $(TOP)/ext/intck/sqlite3intck.c \ diff --git a/make.bat b/make.bat new file mode 100644 index 000000000..2dc9b61c0 --- /dev/null +++ b/make.bat @@ -0,0 +1,2 @@ +@echo off +nmake /f Makefile.msc %* diff --git a/manifest b/manifest index 7be9b18d3..079aacfd7 100644 --- a/manifest +++ b/manifest @@ -1,14 +1,14 @@ -C Version\s3.51.2 -D 2026-01-09T17:27:48.405 +C Version\s3.52.0 +D 2026-03-06T16:01:44.367 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md 6bc480fc673fb4acbc4094e77edb326267dd460162d7723c7f30bee2d3d9e97d F Makefile.in 3ce07126d7e87c7464301482e161fdae6a51d0a2aa06b200b8f0000ef4d6163b F Makefile.linux-generic bd3e3cacd369821a6241d4ea1967395c962dfe3057e38cb0a435cee0e8b789d0 -F Makefile.msc d4459fad28b388063698cbb7a73bfce8684da998a844a04b21d4b9b10291196a -F README.md dae499194b75deed76a13a4a83c82493f2530331882d7dfe5754d63287d3f8f7 -F VERSION 53fb08d314af314f884da9b33cabad229928aac28b53984a2c38fd4d7dc608ab +F Makefile.msc 174764cb7e80c80f9003c46b3e388d74c68c8c40230208904b3af8fcabee5f4e +F README.md 3fa51fc7ababc32edd175ae8b2986c86d5ea120c1cb1e57c7f7849492d1405ec +F VERSION 74672bfd4c7826c0fc6f84762488a707c52e7d2d94af42ccb0edcc6c74311c41 F art/icon-243x273.gif 9750b734f82fdb3dc43127753d5e6fbf3b62c9f4e136c2fbf573b2f57ea87af5 F art/icon-80x90.gif 65509ce3e5f86a9cd64fe7fca2d23954199f31fe44c1e09e208c80fb83d87031 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -17,8 +17,8 @@ F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F art/sqlite370.svg 40b7e2fe8aac3add5d56dd86ab8d427a4eca5bcb3fe4f8946cb3794e1821d531 F auto.def 44a0d1bf09d78355fc88251ccbf8e64e6341fd89c11de68a01c3645e53a2bade F autoconf/Makefile.fallback 22fe523eb36dfce31e0f6349f782eb084e86a5620b2b0b4f84a2d6133f53f5ac -F autoconf/Makefile.in 118aa2c4d49173672d065fdda19eb8a28642e2c684212d7a626d6db5e6762521 -F autoconf/Makefile.msc 9c1ca648062fd5a4a83ba7590c4422090cccd6399002af0346b7572f086c7483 +F autoconf/Makefile.in efab1f56c7961d301efc030baa4391b9faae38733d94385a3d56b54720a74aba +F autoconf/Makefile.msc e6c596e6e63ea17aa7a97eb8ded18cd984c8dd4203766644fbd57b8fcf720225 F autoconf/README.first f1d3876e9a7852c22f275a6f06814e64934cecbc0b5b9617d64849094c1fd136 F autoconf/README.txt b749816b8452b3af994dc6d607394bef3df1736d7e09359f1087de8439a52807 F autoconf/auto.def 3d994f3a9cc9b712dbce92a5708570ddcf3b988141b6eb738f2ed16127a9f0ac @@ -33,7 +33,7 @@ F autoconf/tea/teaish.tcl 81feb417e718ed75cdd7e2fdf6771f3da80dae97377a90c4d5b62b F autoconf/tea/teaish.test.tcl cfe94e1fb79dd078f650295be59843d470125e0cc3a17a1414c1fb8d77f4aea6 F autosetup/LICENSE 41a26aebdd2cd185d1e2b210f71b7ce234496979f6b35aef2cbf6b80cbed4ce4 F autosetup/README.autosetup a78ff8c4a3d2636a4268736672a74bf14a82f42687fcf0631a70c516075c031e -F autosetup/README.md ce0f95980a687bb861bd830b76bc4b48513567be5cf5ee7004f4f3439ffe3841 +F autosetup/README.md e8503618f6cb31b692383ca662f6eb9ef4345463f5e3c1fc84a5d6863c7058be F autosetup/autosetup b16e44924c197783df67366762dda985b45d49ebc4af15f4054e3ee0e3b65169 x F autosetup/autosetup-config.guess dfa101c5e8220e864d5e9c72a85e87110df60260d36cb951ad0a85d6d9eaa463 x F autosetup/autosetup-config.sub a38fb074d0dece01cf919e9fb534a26011608aa8fa606490864295328526cd73 x @@ -44,10 +44,10 @@ F autosetup/cc-lib.tcl 493c5935b5dd3bf9bd4eca89b07c8b1b1a9356d61783035144e21795f F autosetup/cc-shared.tcl 163eda58c14cd662fd8a504bd2ad8a716ef4db7015dc1de0095d5de8dd601a4b F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e45f F autosetup/find_tclconfig.tcl e64886ffe3b982d4df42cd28ed91fe0b5940c2c5785e126c1821baf61bc86a7e -F autosetup/jimsh0.c a57c16e65dcffc9c76e496757cb3f7fb47e01ecbd1631a0a5e01751fc856f049 +F autosetup/jimsh0.c 916bbdf8023fbda9937afae57d81a853d8c2ea00f2320aa27becbc33574f963d F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba -F autosetup/proj.tcl 6fc14ef82b19b77a95788ffbcfad7989b4e3cb4ce96a21dcb5cf7312f362fba9 -F autosetup/sqlite-config.tcl 5d779fce20c11fde3fe99d157dcd5b5569d729b301141b8dfb8d5aacf9d48cba +F autosetup/proj.tcl ce301197f364f7ce2acabbbd84b43d19e917ec73653157ca134a06f32d322712 +F autosetup/sqlite-config.tcl 7463d59c9c5e86ca286ea16fdab943058beb9346110049eca435154795890f71 F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9 F autosetup/teaish/README.txt b40071e6f8506500a2f7f71d5fc69e0bf87b9d7678dd9da1e5b4d0acbf40b1ca F autosetup/teaish/core.tcl e014dd95900c7f9a34e8e0f460f47e94841059827bce8b4c49668b0c7ae3f1a0 @@ -56,14 +56,14 @@ F autosetup/teaish/tester.tcl 1799514c2652db49561b3386c5242b94534d1663f2cfac861a F configure 9a00b21dfd13757bbfb8d89b30660a89ec1f8f3a79402b8f9f9b6fc475c3303a x F contrib/sqlitecon.tcl eb4c6578e08dd353263958da0dc620f8400b869a50d06e271ab0be85a51a08d3 F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd -F doc/compile-for-unix.md c9dce1ddd4bf0d25efccc5c63eb047e78c01ce06a6ff29c73e0a8af4a0f4adbc -F doc/compile-for-windows.md f9e74d74da88f384edd5809f825035e071608f00f7f39c0e448df7b3982f979c +F doc/compile-for-unix.md c8f05bf9ff8c588c501515eb11642540572203e53d0b5eb5bf60983acdd47643 +F doc/compile-for-windows.md 36601c95fa4070eebfe757684271d17a7c4a586912ba706d0b5e7817e1df54ad F doc/json-enhancements.md e356fc834781f1f1aa22ee300027a270b2c960122468499bf347bb123ce1ea4f F doc/jsonb.md acd77fc3a709f51242655ad7803510c886aa8304202fa9cf2abc5f5c4e9d7ae5 -F doc/lemon.html 89ea833a6f71773ab1a9063fbb7fb9b32147bc0b1057b53ecab94a3b30c0aef5 +F doc/lemon.html 2085fda0a90a94fe92159a79dccc5c30d5a2313524887a31659cd66162f17233 F doc/pager-invariants.txt 83aa3a4724b2d7970cc3f3461f0295c46d4fc19a835a5781cbb35cb52feb0577 F doc/tcl-extension-testing.md b88861804fc1eaf83249f8e206334189b61e150c360e1b80d0dcf91af82354f5 -F doc/testrunner.md 5ee928637e03f136a25fef852c5ed975932e31927bd9b05a574424ae18c31019 +F doc/testrunner.md daffa0ebbbe397a73537ae1b19b3124d489ce5f89dfe570781d1f1ef1809597c F doc/trusted-schema.md 33625008620e879c7bcfbbfa079587612c434fa094d338b08242288d358c3e8a F doc/vdbesort-memory.md 4da2639c14cd24a31e0af694b1a8dd37eaf277aff3867e9a8cc14046bc49df56 F doc/vfs-shm.txt 1a55f3f0e7b6745931b117ba5c9df3640d7a0536f532ef0052563100f4416f86 @@ -71,7 +71,7 @@ F doc/wal-lock.md 7db0cd61e2000b545b78ce89b0c2a9a8dd8d64c097839258ac10d7c5c4156e F ext/README.md 6eb1ac267d917767952ed0ef63f55de003b6a5da433ce1fa389e1a9532e73132 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3 F ext/expert/expert.c d548d603a4cc9e61f446cc179c120c6713511c413f82a4a32b1e1e69d3f086a4 -F ext/expert/expert1.test 1d2da6606623b57bb47064e02140823ce1daecd4cacbf402c73ad3473d7f000c +F ext/expert/expert1.test d9dfbf7fb527cfd43049e30a6238ef02c94484041fa4461ed41acbc6435425d6 F ext/expert/sqlite3expert.c 546010043fbec93544f762de5161b3d553165859e6bd853c4b85c05f93484260 F ext/expert/sqlite3expert.h ca81efc2679a92373a13a3e76a6138d0310e32be53d6c3bfaedabd158ea8969b F ext/expert/test_expert.c c395134bd6d4efa594a7d26578a1cb624c4027b79b4b5fcd44736c5ef1f5f725 @@ -79,9 +79,9 @@ F ext/fts3/README.content b9078d0843a094d86af0d48dffbff13c906702b4c3558012e67b9c F ext/fts3/README.syntax b72477722e9b4fe43f8403227d790a1c94221bfad15c27863a4b36d1052e892b F ext/fts3/README.tokenizers b92bdeb8b46503f0dd301d364efc5ef59ef9fa8e2758b8e742f39fa93a2e422d F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 4f02858ab845a97bedf06e6cc1fba49a81fe5e00a26df68d0ad0f00a5814fa70 +F ext/fts3/fts3.c 6cc7bbc307f27e7b6ee2e1d5ff63ffff4df3b42529dfe00eb34ddded417961b3 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h ed9b8bc5ed5be402069651e49d4855cb849af706cf3fe68548f58a2c21eefc7f +F ext/fts3/fts3Int.h fd6051f7aa4db93e05fdc703ef35faf79f78170419e809139109d7aef28f4170 F ext/fts3/fts3_aux.c 7eab82a9cf0830f6551ba3abfdbe73ed39e322a4d3940ee82fbf723674ecd9f3 F ext/fts3/fts3_expr.c 5c13796638d8192c388777166075cdc8bc4b6712024cd5b72c31acdbefce5984 F ext/fts3/fts3_hash.c d9dba473741445789330c7513d4f65737c92df23c3212784312931641814672a @@ -97,7 +97,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c c1de4ae28356ad98ccb8b2e3388a7fdcce7607b5523738c9afb6275dab765154 F ext/fts3/fts3_unicode.c de426ff05c1c2e7bce161cf6b706638419c3a1d9c2667de9cb9dc0458c18e226 F ext/fts3/fts3_unicode2.c 416eb7e1e81142703520d284b768ca2751d40e31fa912cae24ba74860532bf0f -F ext/fts3/fts3_write.c 2bee1c5828f6401adffd07ca64260aeb79d64138958273a56de8fa5e8759a0c1 +F ext/fts3/fts3_write.c d218b687fb55bce8c9340c6dbb368a10d94647cbe39801d85492d576a4e7da75 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73 F ext/fts3/tool/fts3view.c 413c346399159df81f86c4928b7c4a455caab73bfbc8cd68f950f632e5751674 @@ -108,21 +108,21 @@ F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a0 F ext/fts5/extract_api_docs.tcl 009cf59c77afa86d137b0cca3e3b1a5efbe2264faa2df233f9a7aa8563926d15 F ext/fts5/fts5.h ff5d3cc88b29e41612bfb29eb723e29e38973de62ca75ba3e8f94ccb67f5b5f2 F ext/fts5/fts5Int.h 8d98f8e180fe28d6067e240ed45b9011735d29d5cfb5bac194e1e376baa7c708 -F ext/fts5/fts5_aux.c da4a7a9a11ec15c6df0699d908915a209bcde48f0b04101461316b59f71abffb -F ext/fts5/fts5_buffer.c f1e6d0324d7c55329d340673befc26681a372a4d36086caa8d1ec7d7c53066c7 -F ext/fts5/fts5_config.c e7d8dd062b44a66cd77e5a0f74f23a2354cd1f3f8575afb967b2773c3384f7f8 -F ext/fts5/fts5_expr.c b8c32da1127bafaf10d6b4768b0dcb92285798524bed2d87a8686f99a8e8d259 -F ext/fts5/fts5_hash.c a6266cedd801ab7964fa9e74ebcdda6d30ec6a96107fa24148ec6b7b5b80f6e0 -F ext/fts5/fts5_index.c 4e94cec64da9a61f8763f033fee310d3ce22805e1452fd4190e3f972ec60dfb0 -F ext/fts5/fts5_main.c 42025174a556257287071e90516d3ab8115daf1dd525a301883544469a260014 +F ext/fts5/fts5_aux.c 042da27e97d38071312c111cf18f3cb7983b75ba5e724aa1c3164e61e90f428a +F ext/fts5/fts5_buffer.c dcc3f0352339fe79c9d8abbc1c2009bc3469206467880bf43558447ef4f846fb +F ext/fts5/fts5_config.c bfba970fe1e4eed18ee57c8d51458e226db9a960ddf775c5e50e3d76603a667e +F ext/fts5/fts5_expr.c 71d48e8cf0358deace4949276647d317ff7665db6db09f40b81e2e7fe6664c7c +F ext/fts5/fts5_hash.c d5871df92ce3fa210a650cf419ee916b87c29977e86084d06612edf772bff6f5 +F ext/fts5/fts5_index.c f8cfa37bb7397e5ede20242e4c9cb030bc8b4584ce3f23a5e2495038c0ae64bd +F ext/fts5/fts5_main.c 6889f1373c469d515e792fb3d783c2218e63c560433ebd66edc0f740ab086c1b F ext/fts5/fts5_storage.c 19bc7c4cbe1e6a2dd9849ef7d84b5ca1fcbf194cefc3e386b901e00e08bf05c2 -F ext/fts5/fts5_tcl.c 7fb5a3d3404099075aaa2457307cb459bbc257c0de3dbd52b1e80a5b503e0329 +F ext/fts5/fts5_tcl.c 2be6cc14f9448f720fd4418339cd202961a0801ea9424cb3d9de946f8f5a051c F ext/fts5/fts5_test_mi.c 4308d5658cb1f5eee5998dcbaac7d5bdf7a2ef43c8192ca6e0c843f856ccee26 -F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b -F ext/fts5/fts5_tokenize.c 49aea8cc400a690a6c4f83c4cedc67f4f8830c6789c4ee343404f62bcaebca7b +F ext/fts5/fts5_test_tok.c 6021033bd4f4feffe8579efb6e1f58156ed462256bf99a2acdbd629246529204 +F ext/fts5/fts5_tokenize.c cfc16dde905552fe238c0403670852e75c0330ba508a9fb4836c1f596618561d F ext/fts5/fts5_unicode2.c 536a6dae41d16edadd6a6b58c56e2ebbb133f0dfe757562a2edbcdc9b8362e50 F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d09885d1dadf80 -F ext/fts5/fts5_vocab.c 23e263ad94ac357cfffd19bd7e001c3f15c4420fb10fa35b5993142127e780e6 +F ext/fts5/fts5_vocab.c bebee4aabcd056a44b3731166433cfdecf17ece750c08cb58733216222bd39e2 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl c5aa7cf7148b6dcffb5b61520ae18212baf169936af734ab265143f59db328fe @@ -168,6 +168,7 @@ F ext/fts5/test/fts5corrupt5.test 73985d4fe6d8f0d5d5c7bcf79ae7c6522c376cd6ad710a F ext/fts5/test/fts5corrupt6.test 2d72db743db7b5d9c9a6d0cfef24d799ed1aa5e8192b66c40e871a37ed9eed06 F ext/fts5/test/fts5corrupt7.test 814aab492d7a09abb5bfdd81cc66fc206d7f3868f9a3bae91876e02efc466fb3 F ext/fts5/test/fts5corrupt8.test 0b10750caf8aa23fa1c379ca4caf6130d41454505e4d5315590f4061eedcbe44 +F ext/fts5/test/fts5corrupt9.test 4253b9b59f33effac8b67da72ec34309c738aca2d5e8e2656bfbbd6a489a1dfe F ext/fts5/test/fts5corruptbig.test 9f95b40fa36e292feceab02b2ef06e21878bfa1ac7afefa138aae05518b51774 F ext/fts5/test/fts5delete.test 2a5008f8b1174ef41d1974e606928c20e4f9da77d9f8347aed818994d89cced4 F ext/fts5/test/fts5detail.test 54015e9c43ec4ba542cfb93268abdf280e0300f350efd08ee411284b03595cc4 @@ -198,9 +199,9 @@ F ext/fts5/test/fts5first.test bfd685b96905bf541d99d8644e0a7219d1d833455a08ab64e F ext/fts5/test/fts5full.test 97d263c1072f4a560929cca31e70f65d2ae232610e17e6affcf7e979df59547b F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e F ext/fts5/test/fts5hash.test fd3e0367fbf0b0944d6936fdb22696350f57b9871069c6766251578a103e8a14 -F ext/fts5/test/fts5integrity.test c423ce16fd1ccadcac7fc22f794226b2bb00f5a187c0ab1d9f8502521b1bae05 +F ext/fts5/test/fts5integrity.test 613efcebe16b2d7a4096f03bcfb164f79a000b3354420ceda4a6f3e035090789 F ext/fts5/test/fts5integrity2.test 4c3636615c0201232c44a8105d5cb14fd5499fd0ee3014d7ffd7e83aac76ece8 -F ext/fts5/test/fts5interrupt.test 20d04204d3e341b104c0c24a41596b6393a3a81eba1044c168db0e106f9ac92c +F ext/fts5/test/fts5interrupt.test af7834ac6c2e71c05aea42d92f272eef3655e89b7a14a5620a2cd9de35e2e8ea F ext/fts5/test/fts5join.test 48b7ed36956948c5b8456c8bcaa5b087808d99000675918a43c4f51a925f1514 F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1 F ext/fts5/test/fts5leftjoin.test 1c14b51f4d1344a89e488160882f05a2246dd7e70c5cf077c8fb473e03c66338 @@ -284,14 +285,14 @@ F ext/intck/intck_common.tcl a61fd2697ae55b0a3d89847ca0b590c6e0d8ff64bebb70920d9 F ext/intck/intckbusy.test d5ed4ef85a4b1dc1dee2484bd14a4bb68529659cca743327df0c775f005fa387 F ext/intck/intckcorrupt.test f6c302792326fb3db9dcfc70b554c55369bc4b52882eaaf039cfe0b74c821029 F ext/intck/intckfault.test cff3f75dff74abb3edfcb13f6aa53f6436746ab64b09fe5e2028f051e985efab -F ext/intck/sqlite3intck.c b1c8a86f90fc00741d13314db9c58f7e2f92d1d19c5ad1c6904ec83a6bbd5c96 +F ext/intck/sqlite3intck.c 3c4a166645a1624731f63acd342e24e81e4ffd497116d94a427d72e6cc6caa69 F ext/intck/sqlite3intck.h 2b40c38e7063ab822c974c0bd4aed97dabb579ccfe2e180a4639bb3bbef0f1c9 F ext/intck/test_intck.c 4f9eaadaedccb9df1d26ba41116a0a8e5b0c5556dc3098c8ff68633adcccdea8 F ext/jni/GNUmakefile 8a94e3a1953b88cf117fb2a5380480feada8b4f5316f02572cab425030a720b4 -F ext/jni/README.md e3fbd47c774683539b7fdc95a667eb9cd6e64d8510f3ee327e7fa0c61c8aa787 +F ext/jni/README.md 1479c83dbe26125264a060ee6873531795a7082dbc0d43c4067683371331559f F ext/jni/jar-dist.make 030aaa4ae71dd86e4ec5e7c1e6cd86f9dfa47c4592c070d2e35157e42498e1fa -F ext/jni/src/c/sqlite3-jni.c 3d84a0176af779737ae977ba1c90d2ffe2537b8299c5d9f6552620493f12ac4b -F ext/jni/src/c/sqlite3-jni.h ac180ba9b21978727006c790d3006a95a2402a4c3ec7a0def92707ed89b79945 +F ext/jni/src/c/sqlite3-jni.c 6ccc50b0e98b8ef8fd6a8b837223e1c8ebb92bfe6b976128cc757c26257d994a +F ext/jni/src/c/sqlite3-jni.h df43024cced914c49485633d0f90168689e70577b3b17b0ecbdaf16e4a417bff F ext/jni/src/org/sqlite/jni/annotation/Experimental.java 8603498634e41d0f7c70f661f64e05df64376562ea8f126829fd1e0cdd47e82b F ext/jni/src/org/sqlite/jni/annotation/NotNull.java be6cc3e8e114485822331630097cc0f816377e8503af2fc02f9305ff2b353917 F ext/jni/src/org/sqlite/jni/annotation/Nullable.java 56e3dee1f3f703a545dfdeddc1c3d64d1581172b1ad01ffcae95c18547fafd90 @@ -301,7 +302,7 @@ F ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java 0e28a0df51368c7127e505f F ext/jni/src/org/sqlite/jni/capi/AuthorizerCallback.java c045a5b47e02bb5f1af91973814a905f12048c428a3504fbc5266d1c1be3de5a F ext/jni/src/org/sqlite/jni/capi/AutoExtensionCallback.java 74cc4998a73d6563542ecb90804a3c4f4e828cb4bd69e61226d1a51f4646e759 F ext/jni/src/org/sqlite/jni/capi/BusyHandlerCallback.java 7b8e19810c42b0ad21a04b5d8c804b32ee5905d137148703f16a75b612c380ca -F ext/jni/src/org/sqlite/jni/capi/CApi.java 3d275f5f4fbdbe4fff15f4d42cf5ff559f9a4897e7373fa99f3b1dc9cf7f849c +F ext/jni/src/org/sqlite/jni/capi/CApi.java a9701cbe736b8bef5bc72ae465be0250e137f67bdb5a3ab62497972b6f51572d F ext/jni/src/org/sqlite/jni/capi/CallbackProxy.java 1b3baf5b772f266e8baf8f35f0ddc5bd87fc8c4927ec69115c46fd6fba6701c3 F ext/jni/src/org/sqlite/jni/capi/CollationCallback.java e29bcfc540fdd343e2f5cca4d27235113f2886acb13380686756d5cabdfd065a F ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java 5bfa226a8e7a92e804fd52d6e42b4c7b875fa7a94f8e2c330af8cc244a8920ab @@ -319,7 +320,7 @@ F ext/jni/src/org/sqlite/jni/capi/SQLFunction.java 0d1e9afc9ff8a2adb94a155b72385 F ext/jni/src/org/sqlite/jni/capi/SQLTester.java 3c0babc067d8560627a9ed1b07979f9d4393464e2282c2fca4832052e982c7bc F ext/jni/src/org/sqlite/jni/capi/ScalarFunction.java 93b9700fca4c68075ccab12fe0fbbc76c91cafc9f368e835b9bd7cd7732c8615 F ext/jni/src/org/sqlite/jni/capi/TableColumnMetadata.java 9133bb7685901d2edf07801191284975e33b5583ce09dce1c05202ff91e7bb99 -F ext/jni/src/org/sqlite/jni/capi/Tester1.java 4c3d16fdf6e979f839b2ecdb14d0a0c04bd3d0e41500fc9e8110b588883b140b +F ext/jni/src/org/sqlite/jni/capi/Tester1.java 378d142435d220b20b7ce7343c62a03e853bb8c51e80447ee0f8ac5c37362d9a F ext/jni/src/org/sqlite/jni/capi/TraceV2Callback.java 0a25e117a0daae3394a77f24713e36d7b44c67d6e6d30e9e1d56a63442eef723 F ext/jni/src/org/sqlite/jni/capi/UpdateHookCallback.java c8bdf7848e6599115d601bcc9427ff902cb33129b9be32870ac6808e04b6ae56 F ext/jni/src/org/sqlite/jni/capi/ValueHolder.java 2ce069f3e007fdbbe1f4e507a5a407fc9679da31a0aa40985e6317ed4d5ec7b5 @@ -346,7 +347,7 @@ F ext/jni/src/org/sqlite/jni/test-script-interpreter.md d7987b432870d23f7c72a780 F ext/jni/src/org/sqlite/jni/wrapper1/AggregateFunction.java d5c108b02afd3c63c9e5e53f71f85273c1bfdc461ae526e0a0bb2b25e4df6483 F ext/jni/src/org/sqlite/jni/wrapper1/ScalarFunction.java 326ffba29aab836a6ea189703c3d7fb573305fd93da2d14b0f9e9dcf314c8290 F ext/jni/src/org/sqlite/jni/wrapper1/SqlFunction.java e920f7a031e04975579240d4a07ac5e4a9d0f8de31b0aa7a4be753c98ae596c9 -F ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java c82bc00c1988f86246a89f721d3c41f0d952f33f934aa6677ec87f7ca42519a0 +F ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java ce8aef867e06a21685658eb0173768c355e9e1e2dfdc75f382643fa62c7ee779 F ext/jni/src/org/sqlite/jni/wrapper1/SqliteException.java 982538ddb4c0719ef87dfa664cd137b09890b546029a7477810bd64d4c47ee35 F ext/jni/src/org/sqlite/jni/wrapper1/Tester2.java 08f92d52be2cec28a7b4555479cc54b7ebeeb94985256144eeb727154ec3f85b F ext/jni/src/org/sqlite/jni/wrapper1/ValueHolder.java a84e90c43724a69c2ecebd601bc8e5139f869b7d08cb705c77ef757dacdd0593 @@ -362,20 +363,20 @@ F ext/misc/base64.c 8dc0a08cee11722822858a62625f1b63e5d5f1adac1cf4492d5732b571e3 F ext/misc/base85.c ff54cc676c6ec86231f75ecc86ea45416fcb69751dfb79690d5f5da5f7d39867 F ext/misc/basexx.c 89ad6b76558efbceb627afd5e2ef1d84b2e96d9aaf9b7ecb20e3d00b51be6fcf F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a -F ext/misc/btreeinfo.c 8f5e6da2c82ec2f06ee0216e922370a436dafdbb06ffa7a552203515ff9e7ddf +F ext/misc/btreeinfo.c 13bc9e9f1c13cde370d0e4a6a2683e9f1926a4cead7fb72c71871b11a06d78a1 F ext/misc/cksumvfs.c 9d7d0cf1a8893ac5d48922bfe9f3f217b4a61a6265f559263a02bb2001259913 F ext/misc/closure.c 5559daf1daf742228431db929d1aa86dd535a4224cc634a81d2fd0d1e6ad7839 F ext/misc/completion.c c27b64fdd0943c1b7f152376599814cee2641f7d67a7bb9bd2b957c2a64a5591 F ext/misc/compress.c 8191118b9b73e7796c961790db62d35d9b0fb724b045e005a5713dc9e0795565 -F ext/misc/csv.c d9dab032420d81cdf0abc4b8523711d20a355704f82d958f963c86be48387dd9 +F ext/misc/csv.c e82124eabee0e692d7b90ab8b2c34fadbf7b375279f102567fa06e4da4b771bf F ext/misc/dbdump.c 678f1b9ae2317b4473f65d03132a2482c3f4b08920799ed80feedd2941a06680 -F ext/misc/decimal.c d4883de142f6dcd36eda23da40b55e2b51374e7b01eb54a7173940191389fc5e +F ext/misc/decimal.c e1da22eee70d7e3eaa99a6b761bc03c4d01d7ffa554bf3178b1f1f184932806c F ext/misc/eval.c 04bc9aada78c888394204b4ed996ab834b99726fb59603b0ee3ed6e049755dc1 F ext/misc/explain.c 606100185fb90d6a1eade1ed0414d53503c86820d8956a06e3b0a56291894f2b -F ext/misc/fileio.c d80268a5328fe839062a9d3103ea0fc7cacc6d42605959275675cb37867c84f7 -F ext/misc/fossildelta.c 2fc2dd4f34f478df674887db62586b1071c4cd3c9e73ee40f9ee669670e482d1 -F ext/misc/fuzzer.c 6b231352815304ba60d8e9ec2ee73d4918e74d9b76bda8940ba2b64e8777515e -F ext/misc/ieee754.c 176c061c94857b543313959289cb60cf777c999fd002f82b53d194b95e9f347a +F ext/misc/fileio.c 33165b3cd99f83dcd333a338eb51491f6b01c8d96cb6ae81f96a6a096834e030 +F ext/misc/fossildelta.c 86dfa83f85f7ccd640591d8a5c6865346d0c2ee6a949d78591eceb892f1cbfec +F ext/misc/fuzzer.c 684a4996b523ea89f495b38fd8a14a2ae00695089a88031366a4df6adc2c873b +F ext/misc/ieee754.c 2901d08a586d00a1d3c0fd89e03c57ee9e2b5f013b0daab9e49c7a48a9d5946b F ext/misc/memstat.c 43705d795090efb78c85c736b89251e743c291e23daaa8382fe7a0df2c6a283d F ext/misc/memtrace.c 7c0d115d2ef716ad0ba632c91e05bd119cb16c1aedf3bec9f06196ead2d5537b F ext/misc/mmapwarm.c a81af4aaec00f24f308e2f4c19bf1d88f3ac3ce848c36daa7a4cd38145c4080d @@ -387,21 +388,22 @@ F ext/misc/percentile.c 72e05a21db20a2fa85264b99515941f00ae698824c9db82d7edfbb16 F ext/misc/prefixes.c 82645f79229877afab08c8b08ca1e7fa31921280906b90a61c294e4f540cd2a6 F ext/misc/qpvtab.c fc189e127f68f791af90a487f4460ec91539a716daf45a0c357e963fd47cc06c F ext/misc/randomjson.c ef835fc64289e76ac4873b85fe12f9463a036168d7683cf2b773e36e6262c4ed -F ext/misc/regexp.c f1f7cfe90fc027b33d2b5ae7d6235eecce69c3aca71c9afce56fec62342c8b44 +F ext/misc/regexp.c 69bd45f6931bdc6801c1059b65a3e8b15ba88255e6abe387a34b653ce17e8908 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 F ext/misc/series.c 22c6d8f00cc1b5089b1b37392e9097e9df9a5db53be86daf9a7669d95bb179f4 -F ext/misc/sha1.c cb5002148c2661b5946f34561701e9105e9d339b713ec8ac057fd888b196dcb9 +F ext/misc/sha1.c 3030b5926b34bb09459200e100fae34e48c04077cf175381a7560f72bbf3d9cf F ext/misc/shathree.c fd22d70620f86a0467acfdd3acd8435d5cb54eb1e2d9ff36ae44e389826993df F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c 693c8fd3293087fa821322967a97e59dfa24051e5d2ca7fa85790a4034db6fa4 F ext/misc/sqlar.c a6175790482328171da47095f87608b48a476d4fac78d8a9ff18b03a2454f634 -F ext/misc/sqlite3_stdio.c 0fe5a45bd332b30aef2b68c64edbe69e31e9c42365b0fa79ce95a034bca6fbb0 -F ext/misc/sqlite3_stdio.h f05eaf5e0258f0573910324a789a9586fc360a57678c57a6d63cfaa2245b6176 +F ext/misc/sqlite3_stdio.c e49c07050bf7bdc87866da7583beda236f2f8c462018a34b61785d99cbddedfd +F ext/misc/sqlite3_stdio.h 27a4ecea47e61bc9574ccdf2806f468afe23af2f95028c9b689bfa08ab1ce99f F ext/misc/stmt.c b090086cd6bd6281c21271d38d576eeffe662f0e6b67536352ce32bbaa438321 F ext/misc/stmtrand.c 59cffa5d8e158943ff1ce078956d8e208e8c04e67307e8f249dece2436dcb7fc F ext/misc/templatevtab.c 10f15b165b95423ddef593bc5dcb915ec4eb5e0f1066d585e5435a368b8bc22b +F ext/misc/tmstmpvfs.c 240caad4441328dc52bd2871f48811db46dff858d5598030e389176837a2f4df F ext/misc/totype.c ba11aac3c0b52c685bd25aa4e0f80c41c624fb1cc5ab763250e09ddc762bc3a8 F ext/misc/uint.c 327afc166058acf566f33a15bf47c869d2d3564612644d9ff81a23efc8b36039 F ext/misc/unionvtab.c 716d385256d5fb4beea31b0efede640807e423e85c9784d21d22f0cce010a785 @@ -410,16 +412,20 @@ F ext/misc/uuid.c 5bb2264c1b64d163efa46509544fd7500cb8769cb7c16dd52052da8d961505 F ext/misc/vfslog.c 3932ab932eeb2601dbc4447cb14d445aaa9fbe43b863ef5f014401c3420afd20 F ext/misc/vfsstat.c 0b23c0a69a2b63dc0ef0af44f9c1fc977300c480a1f7a9814500369d8211f56e F ext/misc/vfstrace.c 0e4b8b17ac0675ea90f6d168d8214687e06ca3efbc0060aad4814994d82b41fb -F ext/misc/vtablog.c 2d04386c2f5a3bb93bc9ae978f0b7dcd5a264e126abd640dd6d82aa9067cbd48 +F ext/misc/vtablog.c 402496fb38add7dd2c50f2a0ad20f83a9916ceab48dcd31e62ad621e663c83ac F ext/misc/vtshim.c e5bce24ab8c532f4fdc600148718fe1802cb6ed57417f1c1032d8961f72b0e8f F ext/misc/wholenumber.c 0fa0c082676b7868bf2fa918e911133f2b349bcdceabd1198bba5f65b4fc0668 F ext/misc/windirent.h 02211ce51f3034c675f2dbf4d228194d51b3ee05734678bad5106fff6292e60c -F ext/misc/zipfile.c 71d3fd3155ed5e738473e286e550cf0bcf346cc2fd63646eaf944e7b40531a1b +F ext/misc/zipfile.c c8ee04e1b349270b5df401ad732f5d7c387146e69b33c02fa90322760cc6fee0 F ext/misc/zorder.c bddff2e1b9661a90c95c2a9a9c7ecd8908afab5763256294dd12d609d4664eee +F ext/qrf/README.md e6e0ce2700acf6fd06312b42726a8f08ca240f30e1b122bff87c71c602046352 +F ext/qrf/dev-notes.md e68a6d91ce4c7eb296ef2daadc2bb79c95c317ad15b9fafe40850c67b29c2430 +F ext/qrf/qrf.c cd48c23500c3b129be5e0627ce9d41b5df3c2d715525b00a6ccbd1f30689fb17 +F ext/qrf/qrf.h 2ac14b0aaacf44636d8c81051bfeab4afae50a98fbb2e10ff5aed0c28a87b2b2 F ext/rbu/rbu.c 801450b24eaf14440d8fd20385aacc751d5c9d6123398df41b1b5aa804bf4ce8 F ext/rbu/rbu1.test 25870dd7db7eb5597e2b4d6e29e7a7e095abf332660f67d89959552ce8f8f255 F ext/rbu/rbu10.test 7c22caa32c2ff26983ca8320779a31495a6555737684af7aba3daaf762ef3363 -F ext/rbu/rbu11.test 8584f80ef4be00e6beec4154f638847ffc40b5f2832ffadfbaf558ae40e50cb5 +F ext/rbu/rbu11.test 3b71377c018b05dd39c30ea2feb272a087eb0faeff1b7b4511144f87219c478e F ext/rbu/rbu12.test ec63aa7bfc3c65c1d774bf4357ed731723827d211d9d7cb0efa171bbaeeebaf4 F ext/rbu/rbu13.test 658edbc3325d79252a98b761fde95460e439f80e820ff29e10261e25f870b3b6 F ext/rbu/rbu14.test 05dac607a62f62102f4db92135979a8a4501143638060019aca08c753822cf39 @@ -460,7 +466,7 @@ F ext/rbu/rbuvacuum.test 542561741ff2b262e3694bc6012b44694ee62c545845319a06f3237 F ext/rbu/rbuvacuum2.test 1a9bd41f127be2826de2a65204df9118525a8af8d16e61e6bc63ba3ac0010a23 F ext/rbu/rbuvacuum3.test 3ce42695fdf21aaa3499e857d7d4253bc499ad759bcd6c9362042c13cd37d8de F ext/rbu/rbuvacuum4.test ffccd22f67e2d0b380d2889685742159dfe0d19a3880ca3d2d1d69eefaebb205 -F ext/rbu/sqlite3rbu.c c208f72f20784bf2f39244b6cdf8019724a706e1246be289e7621c42aad54695 +F ext/rbu/sqlite3rbu.c e99400d29d029936075e27495b269a2dcdceae3cf8c86b1d0869b4af487be3ab F ext/rbu/sqlite3rbu.h e3a5bf21e09ca93ce4e8740e00d6a853e90a697968ec0ea98f40826938bdb68e F ext/rbu/test_rbu.c 8b6e64e486c28c41ef29f6f4ea6be7b3091958987812784904f5e903f6b56418 F ext/recover/dbdata.c 10d3c56968a9af6853722a47280805ad1564714d79ea45ac6f7da14bb57fd137 @@ -482,17 +488,8 @@ F ext/recover/recoversql.test e66d01f95302a223bcd3fd42b5ee58dc2b53d70afa90b0d00e F ext/recover/sqlite3recover.c 56c216332ea91233d6d820d429f3384adbec9ecedda67aa98186b691d427cc57 F ext/recover/sqlite3recover.h 011c799f02deb70ab685916f6f538e6bb32c4e0025e79bfd0e24ff9c74820959 F ext/recover/test_recover.c 3d0fb1df7823f5bc22a0b93955034d16a2dfa2eb1e443e9a0123a77f120599a3 -F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 -F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 -F ext/repair/checkindex.c 7639b4f8928f82c10b950169e60cc45a7f6798df0b299771d17bef025736f657 -F ext/repair/sqlite3_checker.c.in 445118c5f7fea958b36fba1b2c464283e60ed4842039ddee3265f1698115ebf7 -F ext/repair/sqlite3_checker.tcl a9a2caa9660567257c177a91124d8c0dccdfa341e25c51e6da7f1fd9e601eafa -F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e -F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc78249442da72ff3f8297398a69 -F ext/repair/test/checkindex01.test b530f141413b587c9eb78ff734de6bb79bc3515c335096108c12c01bddbadcec -F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c F ext/rtree/README 734aa36238bcd2dee91db5dba107d5fcbdb02396612811377a8ad50f1272b1c1 -F ext/rtree/geopoly.c f0573d5109fdc658a180db0db6eec86ab2a1cf5ce58ec66cbf3356167ea757eb +F ext/rtree/geopoly.c bd1971479184d559499ff3087c37f2823977d7b0ec80916141ae66f70345c88d F ext/rtree/rtree.c 9331997a76b88a9bc04e156bdfd6e2fe35c0aa93bc338ebc6aa0ae470fe4a852 F ext/rtree/rtree.h 4a690463901cb5e6127cf05eb8e642f127012fd5003830dbc974eca5802d9412 F ext/rtree/rtree1.test e0608db762b2aadca0ecb6f97396cf66244490adc3ba88f2a292b27be3e1da3e @@ -535,14 +532,14 @@ F ext/session/changesetfuzz1.test 15b629004e58d5ffcc852e6842a603775bb64b1ce51254 F ext/session/session1.test 5d2502922d38a1579076863827342379a1609ca6bae78c40691a2be1ed1be2aa x F ext/session/session2.test ee83bb973b9ce17ccce4db931cdcdae65eb40bbb22089b2fe6aa4f6be3b9303f F ext/session/session3.test 2cc1629cfb880243aec1a7251145e07b78411d851b39b2aa1390704550db8e6a -F ext/session/session4.test 823f6f018fcbb8dacf61e2960f8b3b848d492b094f8b495eae1d9407d9ab7219 +F ext/session/session4.test ad0ddaaddb9a99dac433d83fc6674aae2af072b8f57e63a6b3f2d76f1a66e98c F ext/session/session5.test 716bc6fafd625ce60dfa62ae128971628c1a1169 F ext/session/session6.test 35279f2ec45448cd2e24a61688219dc6cf7871757716063acf4a8b5455e1e926 F ext/session/session8.test 326f3273abf9d5d2d7d559eee8f5994c4ea74a5d935562454605e6607ee29904 F ext/session/session9.test 0c4a8fbe7a5031f50855f020f3408e1f07fd7859f1daa1629eadcec3422072d6 F ext/session/sessionA.test 1feeab0b8e03527f08f2f1defb442da25480138f F ext/session/sessionB.test c4fb7f8a688787111606e123a555f18ee04f65bb9f2a4bb2aa71d55ce4e6d02c -F ext/session/sessionC.test f8a5508bc059ae646e5ec9bdbca66ad24bc92fe99fda5790ac57e1f59fce2fdf +F ext/session/sessionC.test c3fade0a460d898fa42e9077b88e45c0d24ead3150268e145c8e19aeafc24ba1 F ext/session/sessionD.test 470ff917dc849e2eb78142ade63aaabd729d773833cff0ff01bca0eda68a21ce F ext/session/sessionE.test b2010949c9d7415306f64e3c2072ddabc4b8250c98478d3c0c4d064bce83111d F ext/session/sessionF.test d37ed800881e742c208df443537bf29aa49fd56eac520d0f0c6df3e6320f3401 @@ -572,11 +569,10 @@ F ext/session/sessionrowid.test 85187c2f1b38861a5844868126f69f9ec62223a03449a98a F ext/session/sessionsize.test 8fcf4685993c3dbaa46a24183940ab9f5aa9ed0d23e5fb63bfffbdb56134b795 F ext/session/sessionstat1.test 5e718d5888c0c49bbb33a7a4f816366db85f59f6a4f97544a806421b85dc2dec F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009ecef8511f4cf3fc -F ext/session/sqlite3session.c b3de195ce668cace9b324599bf6255a70290cbfb5451e826e946f3aee6e64c54 +F ext/session/sqlite3session.c 6ebd02be470f36d41c4bd78927f39d507b62051ba025eacaed9936c769902a07 F ext/session/sqlite3session.h 7404723606074fcb2afdc6b72c206072cdb2b7d8ba097ca1559174a80bc26f7a -F ext/session/test_session.c 8766b5973a6323934cb51248f621c3dc87ad2a98f023c3cc280d79e7d78d36fb -F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 3dc01e673c456d3b752674c9407276e8fef35dec1d304b3cc1de362f019b2a09 +F ext/session/test_session.c 190110e3bd9463717248dec1272b44fe9943e57b7646d0b4200dcf11e4dccee6 +F ext/wasm/GNUmakefile 79236447d750609aa6beda30feec1314180c5462a493ad94214122887232bfd4 F ext/wasm/README-dist.txt f01081a850ce38a56706af6b481e3a7878e24e42b314cfcd4b129f0f8427066a F ext/wasm/README.md 2e87804e12c98f1d194b7a06162a88441d33bb443efcfe00dc6565a780d2f259 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff @@ -584,71 +580,68 @@ F ext/wasm/SQLTester/SQLTester.mjs 6b3c52ed36a5573ca4883176f326332a8d4c0cecf5efd F ext/wasm/SQLTester/SQLTester.run.mjs 57f2adb33f43f2784abbf8026c1bfd049d8013af1998e7dcb8b50c89ffc332e0 F ext/wasm/SQLTester/index.html 64f3435084c7d6139b08d1f2a713828a73f68de2ae6a3112cbb5980d991ba06f F ext/wasm/SQLTester/touint8array.c 2d5ece04ec1393a6a60c4bf96385bda5e1a10ad49f3038b96460fc5e5aa7e536 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core ef34398a903d0a2425fbbfbd4ed2cd596daea55b8515e2617c8dc7ad7c0767dd -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras 9eae68943ce91ab145892b31370819c2103525240eb72e0fce53c498b8d8275a -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b -F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 -F ext/wasm/api/README.md f4c0d67caaee21a77b8938c30b5f79667bfc9d0c95d01b51df77ea35ee773884 -F ext/wasm/api/extern-post-js.c-pp.js 205f55aacfc62c580985db5c790300779de3876a76a5c7e1bfb13e71c8b4506b +F ext/wasm/api/EXPORTED_FUNCTIONS.c-pp 7ba933e8f1290cc65459dd371c0c9a031d96bdf14d7a2244fa761d9775117b90 +F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b47244781a7e81 +F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 -F ext/wasm/api/post-js-footer.js 5bd7170b5e8ce7b62102702bbcf47ef7b3b49cd56ed40c043fd990aa715b74ee -F ext/wasm/api/post-js-header.js 79d078aec33d93b640a19c574b504d88bb2446432f38e2fbb3bb8e36da436e70 -F ext/wasm/api/pre-js.c-pp.js a876c6399dff29b6fe9e434036beb89889164cc872334e184291723ecc7cb072 -F ext/wasm/api/sqlite3-api-cleanup.js a3d6b9e449aefbb8bba283c2ba9477e2333a0eeb94a7a26b5bf952736f65a6dd -F ext/wasm/api/sqlite3-api-glue.c-pp.js d2b8263b3ce0cefc6c5a68d0a4d448a9770eda4bf9d9ded9d7eb0198e4ce4da1 -F ext/wasm/api/sqlite3-api-oo1.c-pp.js c4260f3fdc553c56ee530c20cc1119029067b503f0d6d7b472705536cb45aa1d -F ext/wasm/api/sqlite3-api-prologue.js 307583ff39a978c897c4ef4ce53fe231dce5c73dc84785969c81c1ab5960a293 +F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b +F ext/wasm/api/post-js-header.js d24bd0d065f3489c8b78ddf3ead6321e5d047187a162cd503c41700e03dd1f06 +F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f +F ext/wasm/api/sqlite3-api-glue.c-pp.js 9b33e3ee467791dec4fd1b444b12a8545dfbb6c8b28ac651c7bdc7661a3b5a5c +F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c41db98a22b534c15e543cfa +F ext/wasm/api/sqlite3-api-prologue.js 1fefd40ab21e3dbf46f43b6fafb07f13eb13cc157a884f7c1134caf631ddb3f2 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 -F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966 +F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc +F ext/wasm/api/sqlite3-opfs-async-proxy.js 92d6d327a862f1627ff3e88e60fdfea9def06ad539d98929ba46490e64372736 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 26cb41d5a62f46a106b6371eb00fef02de3cdbfaa51338ba087a45f53028e0d0 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 418c33fe284739564daab3c7a7a88882fdd3c99137497900f98eddec1e409af5 -F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 9097074724172e31e56ce20ccd7482259cf72a76124213cbc9469d757676da86 -F ext/wasm/api/sqlite3-wasm.c dd7fc1d535281f0d5d2732bb1b662d1d403a762f07b63c2ea5663053377b2804 -F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bda1c75bd674a92a0e27cc2f3d46dbbf21e422413f8046814515a0bd7409328a -F ext/wasm/api/sqlite3-worker1.c-pp.js 802d69ead8c38dc1be52c83afbfc77e757da8a91a2e159e7ed3ecda8b8dba2e7 -F ext/wasm/c-pp-lite.c 8fa0148e73782a86274db688c4730e2962cd675af329490493adddaf3322f16f -F ext/wasm/common/SqliteTestUtil.js 7adaeffef757d8708418dc9190f72df22367b531831775804b31598b44f6aa51 +F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 2ccf4322f42063aefc150972943e750c77f7926b866f1639d40eec05df075b6e +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3 +F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 +F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 +F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js aa9715f661fb700459a5a6cb1c32a4d6a770723b47aa9ac0e16c2cf87d622a66 +F ext/wasm/api/sqlite3-worker1.c-pp.js bd0655687090e3b1657268a6a9cacde1ea2a734079d194e16dbbed9083e51b38 +F ext/wasm/c-pp-lite.c f38254fba42561728c2e4764a7ba8d68700091e7c2f4418112868c0daba16783 +F ext/wasm/common/SqliteTestUtil.js dae753b95e72248c4395d8de8359e0d055cd9928488e8dd84aef89e46d23b32e F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15 F ext/wasm/common/testing.css e97549bab24126c24e0daabfe2de9bb478fb0a69fdb2ddd0a73a992c091aad6f -F ext/wasm/common/whwasmutil.js 0d539324097fc83b953e9844267359ba0fd02286caa784ea2f597ced279ea640 +F ext/wasm/common/whwasmutil.js 831f07a0d9bb61713164871370811432e96d0f813806a4d2c783d3c77c2373a0 F ext/wasm/config.make.in c424ae1cc3c89274520ad312509d36c4daa34a3fce5d0c688e5f8f4365e1049a F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 F ext/wasm/demo-123.js c7b3cca50c55841c381a9ca4f9396e5bbdc6114273d0b10a43e378e32e7be5bf F ext/wasm/demo-jsstorage.html 409c4be4af5f207fb2877160724b91b33ea36a3cd8c204e8da1acb828ffe588e -F ext/wasm/demo-jsstorage.js 42131ddfa18e817d0e39ac63745e9ea31553980a5ebd2222e04d4fac60c19837 -F ext/wasm/demo-worker1-promiser.c-pp.html 635cf90685805e21772a5f7a35d1ace80f98a9ef7c42ff04d7a125ddca7e5db8 +F ext/wasm/demo-jsstorage.js 467cb4126ff679ebcdb112d100d073af26b9808d0a0b52d66a40e28f59c5099b +F ext/wasm/demo-worker1-promiser.c-pp.html f73b0b98457e7fdad40d8353cb9b2919391da180f49549a86f3d58b4e5a010eb F ext/wasm/demo-worker1-promiser.c-pp.js f40ec65810048e368896be71461028bd10de01e24277208c59266edf23bb9f52 F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d -F ext/wasm/demo-worker1.js 08720227e98fa5b44761cf6e219269cee3e9dd0421d8d91459535da776950314 +F ext/wasm/demo-worker1.js fdfa90aa9d6b402bfed802cf1595fe4da6cc834ac38c8ff854bf1ee01f5ff9bb F ext/wasm/example_extra_init.c 2347cd69d19d839ef4e5e77b7855103a7fe3ef2af86f2e8c95839afd8b05862f -F ext/wasm/fiddle/fiddle-worker.js 7798af02e672e088ff192716f80626c8895e19301a65b8af6d5d12b2d13d2451 +F ext/wasm/fiddle/fiddle-worker.js 6c72acac2d381480bc9f5eb538e3f2faf2c1f72dd4fcbd05d3b409818a9a8fd5 F ext/wasm/fiddle/fiddle.js 84fd75967e0af8b69d3dd849818342227d0f81d13db92e0dcbc63649b31a4893 -F ext/wasm/fiddle/index.html a27b8127ef9ecf19612da93b2a6a73bdb3777b5c56b5450bb7200a94bc108ff9 +F ext/wasm/fiddle/index.c-pp.html 72c7e5517217960b3809648429ea396a7cbad0ffb2c92f6a2f5703abecb27317 F ext/wasm/index-dist.html db23748044e286773f2768eec287669501703b5d5f72755e8db73607dc54d290 -F ext/wasm/index.html 54e27db740695ab2cb296e02d42c4c66b3f11b65797340d19fa6590f5b287da1 -F ext/wasm/jaccwabyt/jaccwabyt.js bbac67bc7a79dca34afe6215fd16b27768d84e22273507206f888c117e2ede7d -F ext/wasm/jaccwabyt/jaccwabyt.md 167fc0b624c9bc2c477846e336de9403842d81b1a24fc4d3b24317cb9eba734f -F ext/wasm/mkdist.sh 64d53f469c823ed311f6696f69cec9093f745e467334b34f5ceabdf9de3c5b28 x -F ext/wasm/mkwasmbuilds.c 1b53c4d2a1350c19a96a8cdfbda6a39baea9d2142bfe0cbef0ccb0e898787f47 +F ext/wasm/index.html 475bc283338749db4e3fbf24cf3f5aa020cc85a1fffb780d400a915fcb5f1756 +F ext/wasm/jaccwabyt/jaccwabyt.js 4e2b797dc170851c9c530c3567679f4aa509eec0fab73b466d945b00b356574b +F ext/wasm/jaccwabyt/jaccwabyt.md 6aa90fa1a973d0ad10d077088bea163b241d8470c75eafdef87620a1de1dea41 +F ext/wasm/mkdist.sh f8883b077a2ca47cf92e6f0ce305fbf72ca648c3501810125056c4b09c2d5554 x +F ext/wasm/mkwasmbuilds.c 0e9198eb90acae4bcf57cf62d7186f6af5aaac02efdb075a1aded33614b3805a F ext/wasm/module-symbols.html e54f42112e0aac2a31f850ab33e7f2630a2ea4f63496f484a12469a2501e07e2 F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96 F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d F ext/wasm/speedtest1-wasmfs.mjs 60dd5842f6d2a70a6d0bef12633a11491bde6984aff75a37c2040980d8cbf36a F ext/wasm/speedtest1-worker.html 068d4190f304fa1c34e6501a1b3a4c32fe8d8dac93c2d0f53d667a1cb386eedc -F ext/wasm/speedtest1-worker.js 958a2d3c710bf8e82567277f656193a0248216db99a3c2c86966124b84309efb -F ext/wasm/speedtest1.html c90d63dfa795f0cb1ad188de587be9024b1ff73b4adc5fdf7efc0d781be94d03 +F ext/wasm/speedtest1-worker.js 8acad67bfd6aeeb799bd5ae007ea32af85a082a287d8877c5a10adf4bd7efd89 +F ext/wasm/speedtest1.html f32c66997eb0b036c4546e6302cd0673157912661df0b290ab65816f713feac6 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555e685bce3da8c3f F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c -F ext/wasm/tester1-worker.c-pp.html 883881eeac14eeeecc8ff22acf9fe0f18a97cacb48be08ebb0bae891ceded584 -F ext/wasm/tester1.c-pp.html 949920126dcf477925d8d540093d9cc374d3ab4c4ddee920c1dcadcf37917306 -F ext/wasm/tester1.c-pp.js 2b014884dadf28928fabcb688746ca87145673eef75e154486505a266203fc15 +F ext/wasm/tester1-worker.c-pp.html d0032241d0b24d996cf1c4dd0dde364189693af9b5c986e48af7d3d720fcd244 +F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb +F ext/wasm/tester1.c-pp.js 6b946cd6d4da130dbae4a401057716d27117ca02cad2ea8c29ae9c46c675d618 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -658,7 +651,8 @@ F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca F ext/wasm/tests/opfs/sahpool/sahpool-pausing.js f264925cfc82155de38cecb3d204c36e0f6991460fff0cb7c15079454679a4e2 F ext/wasm/tests/opfs/sahpool/sahpool-worker.js bd25a43fc2ab2d1bafd8f2854ad3943ef673f7c3be03e95ecf1612ff6e8e2a61 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk 00dd631c66c1f7922b2d691631163899eff1c3d1da780ff37a62f8d997b368e1 +F main.mk e1a03e9206f6a042a9147035915cb944e9242d570779bc3ccd7ed6a39df10cae +F make.bat a136fd0b1c93e89854a86d5f4edcf0386d211e5d5ec2434480f6eea436c7420c F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -668,39 +662,39 @@ F mptest/multiwrite01.test dab5c5f8f9534971efce679152c5146da265222d F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 1b9c24374a85dfc7eb8fa7c4266ee0db4f9609cceecfc5481cd8307e5af04366 F sqlite3.pc.in e6dee284fba59ef500092fdc1843df3be8433323a3733c91da96690a50a5b398 -F src/alter.c fc7bbbeb9e89c7124bf5772ce474b333b7bdc18d6e080763211a40fde69fb1da +F src/alter.c fc36b19273ffe364aeb4d00ba04bda8798ad7a67fec7a035ee8ee56272e1bdbe F src/analyze.c 03bcfc083fc0cccaa9ded93604e1d4244ea245c17285d463ef6a60425fcb247d -F src/attach.c 9af61b63b10ee702b1594ecd24fb8cea0839cfdb6addee52fba26fa879f5db9d -F src/auth.c 54ab9c6c5803b47c0d45b76ce27eff22a03b4b1f767c5945a3a4eb13aa4c78dc +F src/attach.c 7cf07d4fa42b9fc8662237c60c40b730326c30aa90ae5fffc0b18b2d726ebf61 +F src/auth.c ebec42df26b34a62b6750d30d9c2c03554a1c522020182476f7729a439fef04f F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 F src/bitvec.c e242d4496774dfc88fa278177dd23b607dce369ccafb3f61b41638eea2c9b399 F src/btmutex.c 30dada73a819a1ef5b7583786370dce1842e12e1ad941e4d05ac29695528daea -F src/btree.c cb5b8ceb9baa02a63a2f83dec09c4153e1cfbdf9c2adef5c62c26d2160eeb067 +F src/btree.c b744bf69d520534751c742cababe7ad28c3892f1e3a75242e75a20bca15a834a F src/btree.h e823c46d87f63d904d735a24b76146d19f51f04445ea561f71cc3382fd1307f0 F src/btreeInt.h 9c0f9ea5c9b5f4dcaea18111d43efe95f2ac276cd86d770dce10fd99ccc93886 -F src/build.c 611e07299d72ff04bbcb9e7109183467e30925d203c3e121ef9bb3cf6876289b +F src/build.c b993e4adef4c4cdfd7abf62e2676c467bb1923f25f40c3c7ab2a7bfbace3de7f F src/callback.c 3605bbf02bd7ed46c79cd48346db4a32fc51d67624400539c0532f4eead804ad -F src/carray.c ff6081a31878fc34df8fa1052a9cbf17ddc22652544dcb3e2326886ed1053b55 +F src/carray.c 3efe3982d5fb323334c29328a4e189ccaef6b95612a6084ad5fa124fd5db1179 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e -F src/date.c e19e0cfff9a41bfdd884c655755f6f00bca4c1a22272b56e0dd6667b7ea893a2 +F src/date.c 61e92f1f7e2e88e1cd91e91dc69eb2b2854e7877254470f9fabd776bfac922b8 F src/dbpage.c c9ea81c11727f27e02874611e92773e68e2a90a875ef2404b084564c235fd91f F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c -F src/delete.c 03a77ba20e54f0f42ebd8eddf15411ed6bdb06a2c472ac4b6b336521bf7cea42 -F src/expr.c 28b1cc3d2f147cc888703d5482f9581f17656d02abfa331c34370cb3350776be +F src/delete.c 901499bed747c3b4b2be45be1abe912ba50a3f6a40ba88cc006ccf279f2d0e97 +F src/expr.c 8c3b23cb35f43c2d0570c1058b9a269e561e769e09c81ba192992c95022c1939 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 -F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f -F src/func.c 0b802107498048d3dcac0b757720bcb8506507ce02159e213ab8161458eb293b +F src/fkey.c fb0f74c57d19a2d3f113f3476826919d68feda7ff334abfdb479a9a6353b9fcd +F src/func.c 6e7de3551ae0f8205006e5109f025223246edd20186d54d90746dee7c1c5c093 F src/global.c a19e4b1ca1335f560e9560e590fc13081e21f670643367f99cb9e8f9dc7d615b F src/hash.c 03c8c0f4be9e8bcb6de65aa26d34a61d48a9430747084a69f9469fbb00ea52ca F src/hash.h 46b92795a95bfefb210f52f0c316e9d7cdbcdd7e7fcfb0d8be796d3a5767cddf -F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 +F src/hwtime.h 21c2cf1f736e7b97502c3674d0c386db3f06870d6f10d0cf8174e2a4b8cb726e F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c dfd311b0ac2d4f6359e62013db67799757f4d2cc56cca5c10f4888acfbbfa3fd -F src/json.c fb031340edee159c07ad37dbe668ffe945ed86f525b0eb3822e4a67cbc498a72 +F src/json.c 8b6341a419150b28530cc21e3951b2238c35cdc312f11b2ca29017fe4b1dedc0 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa -F src/loadext.c a3bc9a2522dc3b960e38b7582d1818f6245a49289387c2c7b19f27bfeabf1e81 -F src/main.c 65d11c17890966d271c925c6cc55e3ba50fa08374633cb99c0dee4719a20915a -F src/malloc.c 410e570b30c26cc36e3372577df50f7a96ee3eed5b2b161c6b6b48773c650c5e +F src/loadext.c 56a542244fbefc739a2ef57fac007c16b2aefdb4377f584e9547db2ce3e071f9 +F src/main.c 31a13302193fbd51279c7e69cdfa0320d0de7629f9151e0964c1d320e8bdd7a4 +F src/malloc.c 422f7e0498e1c9ef967f06283b6f2c0b16db6b905d8e06f6dbc8baaa3e4e6c5a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2 F src/mem2.c c8bfc9446fd0798bddd495eb5d9dbafa7d4b7287d8c22d50a83ac9daa26d8a75 @@ -709,44 +703,44 @@ F src/mem5.c b7da5c10a726aacacc9ad7cdcb0667deec643e117591cc69cf9b4b9e7f3e96ff F src/memdb.c a3feb427cdd4036ea2db0ba56d152f14c8212ca760ccb05fb7aa49ff6b897df3 F src/memjournal.c c283c6c95d940eb9dc70f1863eef3ee40382dbd35e5a1108026e7817c206e8a0 F src/msvc.h 80b35f95d93bf996ccb3e498535255f2ef1118c78764719a7cd15ab4106ccac9 -F src/mutex.c 06bcd9c3dbf2d9b21fcd182606c00fafb9bfe0287983c8e17acd13d2c81a2fa9 +F src/mutex.c 00b8cee206a67fd764d001f3a148494331d8d0b3b9c3974ecd69ff29bb444462 F src/mutex.h a7b2293c48db5f27007c3bdb21d438873637d12658f5a0bf8ad025bb96803c4a F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 F src/mutex_unix.c f7ee5a2061a4c11815a2bf4fc0e2bfa6fb8d9dc89390eb613ca0cec32fc9a3d1 -F src/mutex_w32.c 28f8d480387db5b2ef5248705dd4e19db0cfc12c3ba426695a7d2c45c48e6885 +F src/mutex_w32.c e1d317d29cb623667d43de94714264d1e1871cc4bb39fa67dd17048e8138c739 F src/notify.c 57c2d1a2805d6dee32acd5d250d928ab94e02d76369ae057dee7d445fd64e878 F src/os.c 509452169d5ea739723e213b8e2481cf0e587f0e88579a912d200db5269f5f6d F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06 -F src/os_kv.c fb7ba8d6204197357f1eb7e1c7450d09c10043bf7e99aba602f4aa46b8fb11a3 +F src/os_kv.c e7d96727db5b67e39d590a68cc61c86daf4c093c36c011a09ebfb521182ec28d F src/os_setup.h 8efc64eda6a6c2f221387eefc2e7e45fd5a3d5c8337a7a83519ba4fbd2957ae2 -F src/os_unix.c dcf7988ddbdd68619b821c9a722f9377abb46f1d26c9279eb5a50402fd43d749 -F src/os_win.c a89b501fc195085c7d6c9eec7f5bd782625e94bb2a96b000f4d009703df1083f -F src/os_win.h 4c247cdb6d407c75186c94a1e84d5a22cbae4adcec93fcae8d2bc1f956fd1f19 -F src/pager.c cd562b878ea1b44d021ba199abc9d3b54f6b3347500a9fed03f66d6000620945 +F src/os_unix.c fa5e09b4df35ad845440cad67b86908cfe1fd4c28c51915f82e23633d1992bf4 +F src/os_win.c 0d553b6e8b92c8eb85e7f1b4a8036fe8638c8b32c9ad8d9d72a861c10f81b4c5 +F src/os_win.h 5e168adf482484327195d10f9c3bce3520f598e04e07ffe62c9c5a8067c1037b +F src/pager.c fe34fd22ec251436985d7b6ebdd05bf238a17901c2cb23d3d28974dd2361a912 F src/pager.h 6137149346e6c8a3ddc1eeb40aee46381e9bc8b0fcc6dda8a1efde993c2275b8 -F src/parse.y 619c3e92a54686c5e47923688c4b9bf7ec534a4690db5677acc28b299c403250 +F src/parse.y 3b784d6083380a950e3b1b32ce5ddd303e8c7c209d8ab788df2c62aaf9ee8eb3 F src/pcache.c 588cc3c5ccaaadde689ed35ce5c5c891a1f7b1f4d1f56f6cf0143b74d8ee6484 -F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 +F src/pcache.h 092b758d2c5e4dabb30eae46d8dfad77c0f70b16bf3ff1943f7a232b0fe0d4ba F src/pcache1.c 131ca0daf4e66b4608d2945ae76d6ed90de3f60539afbd5ef9ec65667a5f2fcd F src/pragma.c ecec75795c1821520266e4f93fa8840cce48979af532db06f085e36a7813860f -F src/prepare.c 2af0b5c1ec787c8eebd21baa9d79caf4a4dc3a18e76ce2edbf2027d706bca37a -F src/printf.c 7297c2aeed4d90d80c5ba82920d9e57b7bfad04b3466be1d7e042db382fe296e +F src/prepare.c f6a6e28a281bd1d1da12f47d370a81af46159b40f73bf7fa0b276b664f9c8b7d +F src/printf.c 9cff219dba73b1aa9a8113e83e962f03f7bea8b6eb51cefb25bc468d5a69fb2d F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c -F src/resolve.c 5616fbcf3b833c7c705b24371828215ad0925d0c0073216c4f153348d5753f0a +F src/resolve.c 928ff887f2a7c64275182060d94d06fdddbe32226c569781cf7e7edc6f58d7fd F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c fa67ce1dd60744c315791085897c1571ffe5554bf3e0410942b5fc9d7e0a4b56 -F src/shell.c.in 2c7e751795f38bb1855c35b556419cab5b8ba22e0f6758f5a629338065d6b79f -F src/sqlite.h.in c0979f9ac1f5be887397dd2a0bb485636893a81b34d64df85123aae9650c42f2 +F src/select.c ffe199f025a0dd74670d2a77232bdea364a4d7b36f32c64a6572d39ba6a11576 +F src/shell.c.in d4e9ce266ca8f7364da6e86df011f8655beeb5f0d074d624215a2d8ce220a0ad +F src/sqlite.h.in 1f853f1d836af3e5a0b451521041d05658988a45f6978aaae08286e483fee5ac F src/sqlite3.rc 015537e6ac1eec6c7050e17b616c2ffe6f70fca241835a84a4f0d5937383c479 -F src/sqlite3ext.h 7f236ca1b175ffe03316d974ef57df79b3938466c28d2f95caef5e08c57f3a52 -F src/sqliteInt.h 88f7fc9ce1630d9a5f7e0a8e1f3287cdc63882fba985c18e7eee1b9f457f59aa -F src/sqliteLimit.h fe70bd8983e5d317a264f2ea97473b359faf3ebb0827877a76813f5cf0cdc364 +F src/sqlite3ext.h 1b7a0ee438bb5c2896d0609c537e917d8057b3340f6ad004d2de44f03e3d3cca +F src/sqliteInt.h 1c7f23ab9d6efdf3dc434880b6320f158937284f6e2cebd2a024def0c749cb04 +F src/sqliteLimit.h 904a3f520362c7065c18165aaabd504fb13cc1b76cb411f38bd41ac219e4af1e F src/status.c 7565d63a79aa2f326339a24a0461a60096d0bd2bce711fefb50b5c89335f3592 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 -F src/tclsqlite.c 3c604c49e6cf4211960a9ddb9505280fd22cde32175f40884c641c0f5a286036 +F src/tclsqlite.c 85b5a20df96016e5d1d8fdc68c8a4c279c5b93e2049b77cd806c2cc50b9d8c56 F src/tclsqlite.h 614b3780a62522bc9f8f2b9fb22689e8009958e7aa77e572d0f3149050af348a -F src/test1.c 5d061afe479c7364842e0170be7220dea13389575fa6030d30b3e20bec4e1f75 +F src/test1.c 3e3b013f59ffcb57dce00c90d55907072d71d4e970cb0a590cb261efe11bae9c F src/test2.c 62f0830958f9075692c29c6de51b495ae8969e1bef85f239ffcd9ba5fb44a5ff F src/test3.c 432646f581d8af1bb495e58fc98234380250954f5d5535e507fc785eccc3987a F src/test4.c 0ac87fc13cdb334ab3a71823f99b6c32a6bebe5d603cd6a71d84c823d43a25a0 @@ -756,10 +750,10 @@ F src/test8.c 206d8f3cc73950d252906656e2646b5de0d580b07187b635fcb3edd8c2c5fbc0 F src/test9.c df9ddc7db6ef1b8cf745866ee229090779728bcbe660c7f297d3127ab21d92af F src/test_autoext.c 14d4bbd3d0bd1eec0f6d16b29e28cf1e2d0b020d454835f0721a5f68121ac10f F src/test_backup.c a2bfd90d2ff2511b8635507bdb30fa9b605ade19c16b533066cae3077f5bdb72 -F src/test_bestindex.c a9428931bec06de830b2630f57a7b1f2711761269f04df62b7aa1affcbce15bb +F src/test_bestindex.c d75fad21369d80910238032bcf8d9ca1f2bffda13c1ceec63bfbb7f704448b15 F src/test_blob.c 77b994e17f2c87055f44fd96c9a206c5a7155bae2cda2769af60c2f3582f962c F src/test_btree.c 28283787d32b8fa953eb77412ad0de2c9895260e4e5bd5a94b3c7411664f90d5 -F src/test_config.c 18aa596d37de1d5968c439fd58ebf38bc4d9c9d1db63621504e241fde375cecd +F src/test_config.c e02566c2c4ee2916324ce17123a798b47663cead2de546cfbd71d8cddb46bb26 F src/test_delete.c d0e8f6dc55cfc98a7c27c057fb88d512260564bf0b611482656c68b8f7f401ed F src/test_demovfs.c 3efa2adf4f21e10d95521721687d5ca047aea91fa62dd8cc22ac9e5a9c942383 F src/test_devsym.c 649434ed34d0b03fbd5a6b42df80f0f9a7e53f94dd1710aad5dd8831e91c4e86 @@ -779,7 +773,7 @@ F src/test_mutex.c dacae6790956c0d4e705aaed2090227792e291b0496cccd688e9994c1e21f F src/test_onefile.c f31e52e891c5fef6709b9fcef54ce660648a34172423a9cbdf4cbce3ba0049f4 F src/test_osinst.c 269039d9c0820a02ee928014c30860d57ee757ecda54df42e463d0ca1377b835 F src/test_pcache.c 496da3f7e2ca66aefbc36bbf22138b1eff43ba0dff175c228b760fa020a37bd0 -F src/test_quota.c 180e87437250bed7e17e4e61c106730939e39fec9be73d28961f27f579a92078 +F src/test_quota.c 5bb44452b9c6c248bb3c82d2466a20915aa6d12801f6c1784b6499aaa04d9811 F src/test_quota.h 2a8ad1952d1d2ca9af0ce0465e56e6c023b5e15d F src/test_rtree.c d844d746a3cc027247318b970025a927f14772339c991f40e7911583ea5ed0d9 F src/test_schema.c b06d3ddc3edc173c143878f3edb869dd200d57d918ae2f38820534f9a5e3d7d9 @@ -794,34 +788,34 @@ F src/test_vfs.c b4135c1308516adf0dfd494e6d6c33114e03732be899eace0502919b674586b F src/test_window.c 6d80e11fba89a1796525e6f0048ff0c7789aa2c6b0b11c80827dc1437bd8ea72 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c cb3294cf23c11106b50d9af6998a6c1bf389b52e15b17698c9fab97bbaa9b37f -F src/treeview.c 3ce7ac9835d2d70cc1c868b01b747ae8a062322e155701e58e3d62ca79aada7a -F src/trigger.c d5cf2541ff048f30b6a0507eb3d1ec4e695c53584e3b2298a5bf248714fe185e +F src/tokenize.c f297bbf02037639e7a93b37d9c6e4415b3de1273395ee8fa8183e741e1e7fb72 +F src/treeview.c feaa59f14db4f7b5aacca9c5ad5aeb562c1f98262c1ffd74371f4186ade91fc5 +F src/trigger.c 4bf3bfb3851d165e4404a9f9e69357345f3f7103378c07e07139fdd8aeb7bd20 F src/update.c 3e5e7ff66fa19ebe4d1b113d480639a24cc1175adbefabbd1a948a07f28e37cf F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1 F src/utf.c 7267c3fb9e2467020507601af3354c2446c61f444387e094c779dccd5ca62165 -F src/util.c 36fb1150062957280777655976f3f9a75db236cb8207a0770ceae8d5ec17fcd3 -F src/vacuum.c 1bacdd0a81d2b5dc1c508fbf0d938c89fa78dd8d5b46ec92686d44030d4f4789 -F src/vdbe.c b44c366e83412d3b8c190feb1f029b7d02e1bd69252a57b32f195107f0d03964 -F src/vdbe.h be33bd7b17f2ec92939642416030491508c51071f6c14e27cd195983fec56b63 -F src/vdbeInt.h 2aaeb6df2938b181b4700a9328688a3986f2bba71e8b96f6a80671316618fa49 -F src/vdbeapi.c 869a0da5d855495055f4d35c6ada582f64ce995ce14b26ff9d336274d497266c -F src/vdbeaux.c 908d8a191aed444b2e4c920159249127f3ff67b94c56a16fad1dfdf9c7488f20 +F src/util.c eccfa8b3b414bb64c6543421c9fd10e5f07e103baae36427a273a9131527694c +F src/vacuum.c d3d35d8ae893d419ade5fa196d761a83bddcbb62137a1a157ae751ef38b26e82 +F src/vdbe.c 5328c99dd256ee8132383565a86e253543a85daccfd7477c52f20bac6b385a7f +F src/vdbe.h 966d0677a540b7ea6549b7c4e1312fc0d830fce3a235a58c801f2cc31cf5ecf9 +F src/vdbeInt.h 42488247a80cd9d300627833c6c85ace067ae5011a99e7614e2358130d62feea +F src/vdbeapi.c 6cdcbe5c7afa754c998e73d2d5d2805556268362914b952811bdfb9c78a37cf1 +F src/vdbeaux.c 396d38a62a357b807eabae0cae441fc89d2767a57ab08026b7072bf7aa2dd00c F src/vdbeblob.c b3f0640db9642fbdc88bd6ebcc83d6009514cafc98f062f675f2c8d505d82692 -F src/vdbemem.c 48e562ff27e6386eb8613207ac27d3d98c1f67fdc4775a1ab13759d2c2a1c021 +F src/vdbemem.c 317ec5e870ddb16951b606c9fe8be22baef22ecbe46f58fdefc259662238afb7 F src/vdbesort.c b69220f4ea9ffea5fdef34d968c60305444eea909252a81933b54c296d9cca70 F src/vdbetrace.c 49e689f751505839742f4a243a1a566e57d5c9eaf0d33bbaa26e2de3febf7b41 F src/vdbevtab.c fc46b9cbd759dc013f0b3724549cc0d71379183c667df3a5988f7e2f1bd485f3 F src/vtab.c 5437ce986db2f70e639ce8a3fe68dcdfe64b0f1abb14eaebecdabd5e0766cc68 F src/vxworks.h 9d18819c5235b49c2340a8a4d48195ec5d5afb637b152406de95a9436beeaeab -F src/wal.c 505a98fbc599a971d92cb90371cf54546c404cd61e04fd093e7b0c8ff978f9b6 +F src/wal.c 88d94fd15a75f6eda831fa32d1148a267ea37bf0a4b69829a73dfde06244b08f F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 -F src/where.c 287324fe73a0ae8e55b3be89bb2fe4148e3a8394e1e2f10ed2113713a037d8a3 +F src/where.c 9f09ee7b260010138d5f9fb5f195b98051119eae3096a99d72ff16c83230f4af F src/whereInt.h 8d94cb116c9e06205c3d5ac87af065fc044f8cf08bfdccd94b6ea1c1308e65da -F src/wherecode.c 71c5c6804b7f882dec8ec858758accae02fcfca13df3cc720f1f258e663ec7c5 -F src/whereexpr.c 403a44eeec1a0f0914fccc6a59376b6924bc00ef6728fe6ffce4cf3051b320fc -F src/window.c 538195bbc75bb924e18e368fbd4ed731a3fe3f901351b44f6466ec486f53affe +F src/wherecode.c 783ecd30061c875c919a5163e4b55f9a0eccdaf7c9b17ad2908a1668a8766bc4 +F src/whereexpr.c e9f7185fba366d9365aa7a97329609e4cf00b3dd0400d069fbaa5187350c17c6 +F src/window.c c0a38cd32473e8e8e7bc435039f914a36ca42465506dc491c65870c01ddac9fb F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test 4d7a34d328e58ca2a2d78fd76c27614a41ca7ddf4312ded9c68c04f430b3b47d F test/affinity3.test 9b7d1133e11d5edd7805573c4ab6f3ba73b0b74a1f280d5b130d4bf3506a93ff @@ -836,12 +830,14 @@ F test/alter2.test 7e3d26ab409df52df887b366a63902c3429b935c41cb962fd58ffc25784f2 F test/alter3.test dcdd5f850f30656a45a0f05e41abfb52b74bbf6ccba165d0f7adf6b0116e4fd6 F test/alter4.test 37cafe164067a6590a0ee4cec780bddbbaa33dc50b11542dcfbe0e65626494fd F test/alterauth.test 63442ba61ceb0c1eeb63aac1f4f5cebfa509d352276059d27106ae256bafc959 -F test/alterauth2.test 48967abae0494d9a300d1c92473d99fcb66edfcc23579c89322f033f49410adc -F test/altercol.test b43fb5725332f4cf8cff0280605202c1672e808281accea60a066d2ccc5129e5 +F test/alterauth2.test 4b74fa8f184f4736497317feb587b65759eb87d87acfe3a8ef433d4d18bb002b +F test/altercol.test 3661c432aacb42bc2198dd4611bbb9c3b09fc73251b59edda046109103b8ac00 +F test/altercons.test ea18def4a0f26b9066da56095c9c480df705df4d02e4ae151708fae76f7e3884 +F test/altercons2.test 3c1f58312817df43aeada3b1827fdc3ce3fc50c6f49a95ef62cf4cbbae8583a0 F test/altercorrupt.test 2e1d705342cf9d7de884518ddbb053fd52d7e60d2b8869b7b63b2fda68435c12 F test/alterdropcol.test a653a3945f964d26845ec0cd0a8e74189f46de3119a984c5bc45457da392612e F test/alterdropcol2.test 527fce683b200d620f560f666c44ae33e22728e990a10a48a543280dfd4b4d41 -F test/alterfault.test 289067108947bedca27534edd4ff251bcd298cf84402d7b24eaa3749305418c6 +F test/alterfault.test 2bb3103954ea60f2e2777b1ae12e79ec3e1fd278f2b1398ad316c68835a62898 F test/alterlegacy.test f38c6d06cda39e1f7b955bbce57f2e3ef5b7cb566d3d1234502093e228c15811 F test/altermalloc.test 167a47de41b5c638f5f5c6efb59784002b196fff70f98d9b4ed3cd74a3fb80c9 F test/altermalloc2.test 17fb3724c4b004c469c27dc4ef181608aa644555fbd3f3236767584f73747c81 @@ -849,8 +845,8 @@ F test/altermalloc3.test 8040e486368403f2fdd6fc3998258b499bd4cc2f3ddbb5f8f874cd4 F test/alterqf.test 8ec03d776de9c391daa0078ea8f838903bdcfb11dfae4ba3576b48436834ccba F test/altertab.test 8a2712f9076da5012a002d0b5cc0a421398a5bf61c25bab41b77c427586a7a27 F test/altertab2.test 0889ba0700cc1cdb7bc7d25975aa61fece34f621de963d0886e2395716b38576 -F test/altertab3.test 471b8898d10bbc6488db9c23dc76811f405de6707d2d342b1b8b6fd1f13cd3c8 -F test/altertrig.test aacc980b657354fe2d3d4d3a004f07d04ccc1a93e5ef82d68a79088c274ddc6b +F test/altertab3.test 575e771e2f02b13eb98798dc92eabacd187d6dbcf596e70f11d699b0b6b5d0b2 +F test/altertrig.test b1590647076add5a47aea0f2236c609ca0bc8a7a2462463edd3e5882c7894802 F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 2fb21d7d64748636384e6cb8998dbf83968caf644c07fcb4f76c18f2e7ede94b F test/analyze3.test c5156cef33f04b90a6b9e9d5d0bbc273a0fb44147d4508407bf1080811e2c6c8 @@ -867,6 +863,7 @@ F test/analyzeF.test 40b5cc3ad7b10e81020d7ca86f1417647ecfae7477cfd88acc5aa7ae106 F test/analyzeG.test 623be33038c49648872746c8dd8b23b5792c08fef173c55e82f1b12fca259852 F test/analyzer1.test b6a624ec0af92eec209e1328465b66937c8fdf2fb442a3fa45321ddb3700f4aa F test/atof1.test bd21c4a0e718ab1470de07a2a79f2544d7903be34feebcc80de04beee4807b00 +F test/atof2.test 12912add57230495450e2fc94cb8ad1c9f3277f8843a3bc27079cae45c9782a1 F test/atomic.test 065a453dde33c77ff586d91ccaa6ed419829d492dbb1a5694b8a09f3f9d7d061 F test/atomic2.test b6863b4aa552543874f80b42fb3063f1c8c2e3d8e56b6562f00a3cc347b5c1da F test/atrc.c c388fac43dbba05c804432a7135ae688b32e8f25818e9994ffba4b64cf60c27c @@ -880,7 +877,7 @@ F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1 F test/auth3.test 76d20a7fa136d63bcfcf8bcb65c0b1455ed71078d81f22bcd0550d3eb18594ab F test/autoanalyze1.test b9cc3f32a990fa56669b668d237c6d53e983554ae80c0604992e18869a0b2dec F test/autoinc.test 9df9930966dbe92c55ef37a4d89112cfd537be0d0596d397177c12db9e581be0 -F test/autoindex1.test 65931519206bbec71948b11e125af0656435a0937973fe5fed70d776a712911f +F test/autoindex1.test 2523a76f30734742c3f4d948d0cbf3b6627f775e7833814f425a2e289ba58b22 F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df F test/autoindex3.test ca502c8050166ac6107a7b4fe4e951f4d3270a23a958af02b14f1b962b83c4b6 F test/autoindex4.test 3c2105e9172920e26f950ba3c5823e4972190e022c1e6f260ba476b0af24c593 @@ -888,7 +885,7 @@ F test/autoindex5.test 3fb938cbf4e7f3896563ce04e2a24b0bc653fc6245b4bf3268cd7b20f F test/autovacuum.test 00671369bbf96c6a49989a9425f5b78b94075d6a4b031e5e00000c2c32f365df F test/autovacuum2.test 76f7eb4fe6a6bf6d33a196a7141dba98886d2fb53a268d7feca285d5da4759d7 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 -F test/avfs.test 76f59743dc1f5fa533840d1818b420fe1ee45e21c0fd6bbac7942ba677903128 +F test/avfs.test 95bb8d04f8edad6dc9e600221d103f7e2cc3da398af84df215a3a819e560c45c F test/avtrans.test 7a6eae44763293024b137b53ff824d8500d754dbae060a8d940afbacfc1d4a15 F test/backcompat.test f2431465ed668f09fc3f6998e56e893a1506ccea6e8b6f409f085f759f431b48 F test/backup.test 3b08fd4af69f0fa786931103a31f4542b184aba16e239e5f22b18c3c2476697f @@ -908,13 +905,14 @@ F test/bestindex4.test 3039894f2dad50f3a68443dffad1b44c9b067ac03870102df1ce3d9a4 F test/bestindex5.test a0c90b2dad7836e80a01379e200e5f8ec9476d49b349af02c0dbff2fb75dc98d F test/bestindex6.test 16942535b551273f3ad9df8d7cc4b7f22b1fcd8882714358859eb049a6f99dd4 F test/bestindex7.test f094c669a6400777f4d2ddc3ed28e39169f1adb5be3d59b55f22ccf8c414b71e -F test/bestindex8.test b63a4f171a2c83d481bb14c431a8b72e85d27b2ffdaa0435a95d58ca941678f9 +F test/bestindex8.test 4d8b1e8f30a7f405988ce4dbcc2b95c0775f0bed9ec08e0291a07e2f35f7e653 F test/bestindex9.test 1a4b93db117fd8abe74ae9be982f86aa72f01e60cd4ac541e6ede39673a451a0 F test/bestindexA.test e1b5def6b190797cacf008e6815ffb78fb30261999030d60a728d572eef44c7f -F test/bestindexB.test 328b97b69cd1a20928d5997f9ecb04d2e00f1d18e19ab27f9e9adb44d7bc51ce +F test/bestindexB.test 14db2f66ec9cc5064a74996033b74e5eec0fd2f3a327fbe34ff18de67e9d2671 F test/bestindexC.test 95b4a527b1a5d07951d731604a6d4cf7e5a806b39cea0e7819d4c9667e11c3fc F test/bestindexD.test 6a8f6f84990bcf17dfa59652a1f935beddb7afd96f8302830fbc86b0a13df3c3 F test/bestindexE.test 297f3ea8500a8f3c17d6f78e55bdfee089064c6144ee84a110bd005a03338f49 +F test/bestindexF.test 4e53d606cbde40a2254aa016d500c5b71766a4065b8541202d195a3d9fe11b1c F test/between.test e7587149796101cbe8d5f8abae8d2a7b87f04d8226610aa1091615005dcf4d54 F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc @@ -949,7 +947,7 @@ F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 F test/capi3c.test 31d3a6778f2d06f2d9222bd7660c41a516d1518a059b069e96ebbeadb5a490f7 F test/capi3d.test 8b778794af891b0dca3d900bd345fbc8ebd2aa2aae425a9dccdd10d5233dfbde F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe -F test/carray01.test 49e2aedfdf2c715bc002d2773cdc1217166679639542c79c8aa4115f06421407 +F test/carray01.test 17c1cf8287862b15dda949dba626fd5fee5c58471dcc1cae0341471c2ae7da01 F test/carray02.test 9d070b54f24a34d1f3b3c552ba34db0375a9d1c4219067416fb07d1595987c9d F test/carrayfault.test 108a7d83904fc267c448e27c13b2a857c700bd6ddaa2f1e2518be718b159cb6b F test/cast.test a2a3b32df86e3c0601ffa2e9f028a18796305d251801efea807092dbf374a040 @@ -967,7 +965,7 @@ F test/collate1.test 0890fa372753b59eba53832d37328af815f6b8e4b16761823180eeb62c8 F test/collate2.test 471c6f74573382b89b0f8b88a05256faa52f7964f9e4799e76708a3b1ece6ba4 F test/collate3.test 89defc49983ddfbf0a0555aca8c0521a676f56a5 F test/collate4.test c953715fb498b87163e3e73dd94356bff1f317bd -F test/collate5.test b1dfeff239ea69ee9225832553f423d37a6184eb730cee06f6846ab4e3c6dbef +F test/collate5.test 42daaf7799b04221206a219fb3c0f9efeede03e760f9562b8c0114b8df183fe3 F test/collate6.test 8be65a182abaac8011a622131486dafb8076e907 F test/collate7.test 8ec29d98f3ee4ccebce6e16ce3863fb6b8c7b868 F test/collate8.test cd9b3d3f999b8520ffaa7cc1647061fc5bab1334 @@ -1001,10 +999,10 @@ F test/corruptH.test 79801d97ec5c2f9f3c87739aa1ec2eb786f96454 F test/corruptI.test 9d8cbf6214e492abe9e822e759b9751ae336cec0a6fe3ff3b37bfbd8ff9c22ca F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4 F test/corruptK.test ac13504593d89d69690d45479547616ed12644d42b5cb7eeb2e759a76fc23dcb -F test/corruptL.test 652fc8ac0763a6fd3eb28b951d481924167b2d9936083bcc68253b2274a0c8fe +F test/corruptL.test f15de2b4729c0851ea89916a26766b094d74bac79f9f9f2b0191935aa3b344c9 F test/corruptM.test 7d574320e08c1b36caa3e47262061f186367d593a7e305d35f15289cc2c3e067 F test/corruptN.test a034bb217bebd8d007625dfb078e76ec3d53515052dbceb68bd47b2c27674d5c -F test/cost.test cc434a026b1e9d0d98137a147e24e5daf1b1ad09e9ff7da63b34c83ddd136d92 +F test/cost.test 3786cd1cc6d1ab416004a1e39387fb0db0c8e259f46b0bce62cbdc328f2c55a0 F test/count.test cd4bd531066e8d77ef8fe1e3fc8253d042072e117ccab214b290cf83f1602249 F test/countofview.test 4088e461a10ee33e69803c177a69aa1d7bba81a9ffc2df66d76465a22ca7fdfc F test/coveridxscan.test f35c7208dedc4f98e471c569df64c0f95a49f6e072d8dc7c8f99bdee2697de1b @@ -1035,7 +1033,7 @@ F test/dbfuzz.c fc566102f72c8af84ae8077b4faf7f056c571e6fa7a32e98b66e42b7505f47b6 F test/dbfuzz001.test 6c9a4622029d69dc38926f115864b055cb2f39badd25ec22cbfb130c8ba8e9c3 F test/dbfuzz2-seed1.db e6225c6f3d7b63f9c5b6867146a5f329d997ab105bee64644dc2b3a2f2aebaee F test/dbfuzz2.c 4b3c12de4d98b1b2d908ab03d217d4619e47c8b23d5e67f8a6f2b1bdee7cae23 -F test/dblwidth-a.sql eb4141518610e52f931a55a984310075e98dc31eee5a28ae806b1e35377be85a +F test/dblwidth-a.sql 59dd59aa78ce8fd8ab631a3816516831f4e947b143039257e6fe132c3cea4171 F test/dbpage.test 63fab1eb026bada121107e53436fa749bbf83281dc9dea17af422f7a5c0f289f F test/dbpagefault.test ea39de2ca86041a9c6df1135645180a76d0a8da93ac159e2fafe38e39636530b F test/dbstatus.test 4a4221a883025ffd39696b3d1b3910b928fb097d77e671351acb35f3aed42759 @@ -1052,8 +1050,9 @@ F test/descidx2.test a0ba347037ff3b811f4c6ceca5fd0f9d5d72e74e59f2d9de346a9d2f6ad F test/descidx3.test 953c831df7ea219c73826dfbf2f6ee02d95040725aa88ccb4fa43d1a1999b926 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 691c9e850b0d0b56b66e7e235453198cb4cf0760e324b7403d3c5abbeab0a014 -F test/distinct2.test 4d6316b6487a0aa5a90bee111575c957e2a5ba5a9be9156febe9533ce78876e8 +F test/distinct2.test a6af6a90b2c1eea64c3cc87ea7f8feb832053f7276fe3c212abacf646de4762a F test/distinctagg.test 40d7169ae5846caaf62c6e307d2ca3c333daf9b6f7cde888956a339a97afe85f +F test/dotcmd01.sql 0388a778912ed08436ae5c80e03389d8bd347fa724611193257a18c69692019d F test/e_blobbytes.test 4c01dfe4f12087b92b20705a3fdfded45dc4ed16d5a211fed4e1d2786ba68a52 F test/e_blobclose.test 692fc02a058476c2222a63d97e3f3b2b809c1842e5525ded7f854d540ac2e075 F test/e_blobopen.test 29f6055ee453b8e679fe9570c4d3acfedbef821622c5dad16875148c5952ef50 @@ -1063,7 +1062,7 @@ F test/e_createtable.test 31b9bcb6ac8876bc7ec342d86d9c231a84c62b442093a6651dfd0f F test/e_delete.test ab39084f26ae1f033c940b70ebdbbd523dc4962e F test/e_droptrigger.test 235c610f8bf8ec44513e222b9085c7e49fad65ad0c1975ac2577109dd06fd8fa F test/e_dropview.test 74e405df7fa0f762e0c9445b166fe03955856532e2bb234c372f7c51228d75e7 -F test/e_expr.test 0a1e175caddc78b27306647cb4ce2362c55790190f8cdd178b75fd6262eb8f76 +F test/e_expr.test 9bdb347b78b9f4eff9153ea97797facc179a821898588471a70808b4471a69b0 F test/e_fkey.test feeba6238aeff9d809fb6236b351da8df4ae9bda89e088e54526b31a0cbfeec5 F test/e_fts3.test 17ba7c373aba4d4f5696ba147ee23fd1a1ef70782af050e03e262ca187c5ee07 F test/e_insert.test f02f7f17852b2163732c6611d193f84fc67bc641fb4882c77a464076e5eba80e @@ -1072,19 +1071,19 @@ F test/e_resolve.test a61751c368b109db73df0f20fc75fb47e166b1d8 F test/e_select.test 327a15f14068bbd6f647cedc67210f8680fcb2f05e481a0a855fccd2abfa1292 F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f F test/e_totalchanges.test c927f7499dc3aa28b9b556b7d6d115a2f0fe41f012b128d16bf1f3b30e9b41e4 -F test/e_update.test f46c2554d915c9197548681e8d8c33a267e84528 +F test/e_update.test 9f8bb82b8760ac66f2c9c2aadb78a418bd639d22f041d712570c9db56f20afda F test/e_uri.test 86564382132d9c453845eeb5293c7e375487b625900ab56c181a0464908417d8 F test/e_vacuum.test 89fc48e8beee2f9dfd6de1fbb2edea6542dae9121dc0fbe6313764169e742104 F test/e_wal.test db7c33642711cf3c7959714b5f012aca08cacfa78da0382f95e849eb3ba66aa4 F test/e_walauto.test 248af31e73c98df23476a22bdb815524c9dc3ba8 -F test/e_walckpt.test 28c371a6bb5e5fe7f31679c1df1763a19d19e8a0 +F test/e_walckpt.test 16e7d006e8687654ee59e7ad5a6d285ba23f0fe0eeb87f790afd6bc9cf1d1924 F test/e_walhook.test 01b494287ba9e60b70f6ebf3c6c62e0ffe01788e344a4846b08e5de0b344cb66 F test/emptytable.test a38110becbdfa6325cd65cb588dca658cd885f62 F test/enc.test b5503a87b31cea8a5084c6e447383f9ca08933bd2f29d97b6b6201081b2343eb F test/enc2.test 872afe58db772e7dfa1ad8e0759f8cc820e9efc8172d460fae83023101c2e435 F test/enc3.test 55ef64416d72975c66167310a51dc9fc544ba3ae4858b8d5ab22f4cb6500b087 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 -F test/eqp.test 746db9fe11629a0d00328e1721cc2a2e4726d574b677ab14de35fd914f54cc82 +F test/eqp.test 1d653fe8d2612cd6764e5ea2f16dcf5f13a9f50448b9233bb1573804bccd7579 F test/eqp2.test 6e8996148de88f0e7670491e92e712a2920a369b4406f21a27c3c9b6a46b68dd F test/errmsg.test eae9f091eb39ce7e20305de45d8e5d115b68fa856fba4ea6757b6ca3705ff7f9 F test/errofst1.test 6da78363739ba8991f498396ab331b5d64e7ab5c4172c12b5884683ef523ac53 @@ -1104,7 +1103,7 @@ F test/extension01.test 5de412c66276105901c370770175003381fdcb0c4da7054fa43cf4a3 F test/external_reader.test 6fdec43eeca23eb32faad1e95a4d1abc402bc8b3db70df12d6fc08a637f4a2b5 F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79 F test/fallocate.test 37a62e396a68eeede8f8d2ecf23573a80faceb630788d314d0a073d862616717 -F test/filectrl.test 4b720117388cf6766d0b798e2dddd785953f8f371633b0c0084d2f34cf72336a +F test/filectrl.test 3344987f143f3cb9914895dc63d9e9518a535ef1b1438d22caf3ba2fa76cc6da F test/filefmt.test f393e80c4b8d493b7a7f8f3809a8425bbf4292af1f5140f01cb1427798a2bbd4 F test/filter1.test 590f8ba9a0cd0823b80d89ac75c5ce72276189cef9225d2436adaf1ee87f3727 F test/filter2.tcl 44e525497ce07382915f01bd29ffd0fa49dab3adb87253b5e5103ba8f93393e8 @@ -1123,7 +1122,8 @@ F test/fordelete.test ba98f14446b310f9c9d935b97ec748753d0144a28b356ba30d1f4f6958 F test/fork-test.c 9ac2e6423a1d38df3d6be0e8ac15608b545de21e2b19d9d876254c5931b63edb F test/format4.test eeae341953db8b6bda7f549044797c3278a6cc345d11ada81471671b654f8ef4 F test/fp-speed-1.c b37de94eba034e1703668816225f54510ec60fb0685406608cc707afe6b8234d -F test/fpconv1.test d5d8aa0c427533006c112fb1957cdd1ea68c1d0709470dabb9ca02c2e4c06ad8 +F test/fpconv1.test 63f352682fa65601a326563ad633086df6ab194e6ed5e7366786f38a525a7fd7 +F test/fptest01.sql 210562ad8d5a7895f26273dd3be56561a41bcb51d78a28a337af0f1ceaa3bb8d F test/fts-9fd058691.test 78b887e30ae6816df0e1fed6259de4b5a64ad33c F test/fts3.test 672a040ea57036fb4b6fdc09027c18d7d24ab654 F test/fts3_common.tcl dffad248f9ce090800e272017d2898005c28ee6314fc1dd5550643a02666907a @@ -1149,7 +1149,7 @@ F test/fts3aux1.test 1880eaa75c586cd10f53080479a2b819b3915ae7ce55c4e0ba8f1fe05ac F test/fts3aux2.test 2459e7fa3e22734aed237d1e2ae192f5541c4d8b218956ad2d90754977bf907f F test/fts3b.test c15c4a9d04e210d0be67e54ce6a87b927168fbf9c1e3faec8c1a732c366fd491 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 -F test/fts3comp1.test a0f5b16a2df44dd0b15751787130af2183167c0c +F test/fts3comp1.test 73a53ada3d25bf242c4b2a24cfe9d39e658be56cfa74754279b9e6db776ed7ce F test/fts3conf.test c9cd45433b6787d48a43e84949aa2eb8b3b3d242bac7276731c1476290d31f29 F test/fts3corrupt.test 6732477c5ace050c5758a40a8b5706c8c0cccd416b9c558e0e15224805a40e57 F test/fts3corrupt2.test e318f0676e5e78d5a4b702637e2bb25265954c08a1b1e4aaf93c7880bb0c67d0 @@ -1199,7 +1199,7 @@ F test/fts3tok_err.test 52273cd193b9036282f7bacb43da78c6be87418d F test/fts3varint.test 0b84a3fd4eba8a39f3687523804d18f3b322e6d4539a55bf342079c3614f2ada F test/fts4aa.test 0e6bfd6a81695a39b23e448dda25d864e63dda75bde6949c45ddc95426c6c3f5 F test/fts4check.test f0ea5e5581951d8ef7a341eea14486daf6c5f516a2f3273b0d5e8cb8a6cd3bd2 -F test/fts4content.test 73bbb123420d2c46ef2fb3b24761e9acdb78b0877179d3a5d7d57aada08066f6 +F test/fts4content.test 7f441866207f5b1e76e0f18bde5d9925d1ee8f60388054613dd14a29a79f0bc4 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 F test/fts4growth.test 289833c34ad45a5e6e6133b53b6a71647231fb89d36ddcb8d9c87211b6721d7f F test/fts4growth2.test 13ad4e76451af6e6906c95cdc725d01b00044269 @@ -1211,7 +1211,7 @@ F test/fts4merge.test 57d093660a5093ae6e9fbd2d17592a88b45bbd66db2703c4b640b28828 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test 8d9ccb4a3d41c4c617a149d6c4b13ad02de797d0 F test/fts4merge4.test 66fce89934cd9508cbdc67de486558c34912ffb2e8ffe5c9a1bbb9b8a4408ba7 -F test/fts4merge5.test 69932d85cda8a1c4dcfb742865900ed8fbda51724b8cf9a45bbe226dfd06c596 +F test/fts4merge5.test 987af90c930e8555f74ab994f597431caec7f8defc52de7718655c32da07af9e F test/fts4min.test 1c11e4bde16674a0c795953509cbc3731a7d9cbd1ddc7f35467bf39d632d749f F test/fts4noti.test d5d933705b1b1516b67a5e3f8e514ecb19c6522fb3357bb744776d48427c2292 F test/fts4onepass.test d69ddc4ee3415e40b0c5d1d0408488a87614d4f63ba9c44f3e52db541d6b7cc7 @@ -1238,7 +1238,7 @@ F test/fuzz3.test 70ba57260364b83e964707b9d4b5625284239768ab907dd387c740c0370ce3 F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c 34a025386f84d818cd3343e69e9d9083091af83153e226d71d4e1c126b5f1dd0 +F test/fuzzcheck.c 9096506277f33cc242eb59743c409c81306492b6ebb84571198f864e536ebe22 F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -1250,7 +1250,7 @@ F test/fuzzdata8.db 8f34ae00d8d5d4747dd80983cf46161065e4f78324dcff3c893506ff8db3 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test f64c4aef4c9e9edf1d6dc0d3f1e65dcc81e67c996403c88d14f09b74807a42bc -F test/fuzzinvariants.c 3ddfec7f5b970b018f1a982532de905cf180e0c1e48cd653be9365d3e6177625 +F test/fuzzinvariants.c 6768bcd03290776cd982624729d2abee2e89e6aba62b4a2b839a98332725a167 F test/gcfault.test 4ea410ac161e685f17b19e1f606f58514a2850e806c65b846d05f60d436c5b0d F test/gencol1.test ceb3163b59cb77f4ad57ae4f01a143ce36b06fdd6a8dab1149235db89979ffd8 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 @@ -1261,6 +1261,8 @@ F test/hook.test 2d89bf9480646feb8093be3a58ea502d6521906779ed960de31dd9c4502c054 F test/hook2.test b9ff3b8c6519fb67f33192f1afe86e7782ee4ac8 F test/icu.test 8da7d52cd9722c82f33b0466ed915460cb03c23a38f18a9a2d3ff97da9a4a8c0 F test/ieee754.test 0d3ab84ab2069c9994c833a7cd820ee6037f0cf888e206a4a7fc05f735d5790a +F test/import01.sql 11a5f8325b8b04c66fe489d30558d462411432adf750cef1c0f7b06564691a8c +F test/imposter1.sql fc5ad0945bb19622688c7a1cd7dfd1cefa4b013bac9e2628c22b03c7309f021f F test/imposter1.test 5a20b2cdeb53e65fc57cdb10a33750bd4ef6259909eaf1972253b9e79f7a3fb2 F test/in.test edf979bff3244b9e47849e2b43886631354c8213791f42da92216f08012141af F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 @@ -1300,11 +1302,12 @@ F test/insert.test 97cfb30b83ca1622b9422a1e4c4831b4cb767cf5d654660945036d1e72067 F test/insert2.test 4d14b8f1b810a41995f6286b64a6943215d52208 F test/insert3.test 1b7db95a03ad9c5013fdf7d6722b6cd66ee55e30 F test/insert4.test 2bf81535a990c969665d66db51fcf76c23499b39893b5109f413d1de4ad34cd3 -F test/insert5.test 394f96728d1258f406fe5f5aeb0aaf29487c39a6 +F test/insert5.test 79f6b6efd0d3db5f4e3ff442300b7d9e7185adb345b29aacc3ea5a9c58ab9beb F test/insertfault.test ac63d14ea3b49c573673a572f4014b9117383a03e497c58f308b5c776e4a7f74 F test/instr.test 67ba309e9697c24a304e98a7c8f372456177dd4e32237d2a305e1e05f7bb79c2 F test/instrfault.test 95e28efade652e6d51ae11b377088fe523a581a07ec428009e152a4dd0e0f44c F test/intarray.test bb976b0b3df0ebb6a2eddfb61768280440e672beba5460ed49679ea984ccf440 +F test/intck01.sql f2d88bf41cdd64f2ed8c3d4f357cf520f017aa2986999ab9a62eb6506ef18106 F test/interrupt.test ac1ef50ec9ab8e4f0e17c47629f82539d4b22558904e321ed5abea2e6187da7a F test/interrupt2.test e4408ca770a6feafbadb0801e54a0dcd1a8d108d F test/intpkey.test 7d54711acf553cdd641a40e9c6cfc2bf1a76070074940c1b126442517054320f @@ -1317,7 +1320,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 5984da7bf74b6540aa356f2ab0c6ae68a6d12039a3d798a9ac6a100abc17d520 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/istrue.test e7f285bb70282625c258e866ce6337d4c762922f5a300e1b50f958aef6e7d9c9 -F test/join.test 2fcfd84640cfd9ff48f31b4b0d370c4d5498c355ae4384544668ca54d37ae186 +F test/join.test c706b382ed09ddc89eee7ad0ffd08d862655b0abc292a690d41d995c18c17b3f F test/join2.test f59d63264fb24784ae9c3bc9d867eb569cd6d442da5660f8852effe5c1938c27 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 @@ -1333,7 +1336,7 @@ F test/joinD.test 2ce62e7353a0702ca5e70008faf319c1d4686aa19fba34275c6d1da0e960be F test/joinE.test d5d182f3812771e2c0d97c9dcf5dbe4c41c8e21c82560e59358731c4a3981d6b F test/joinF.test 53dd66158806823ea680dd7543b5406af151b5aafa5cd06a7f3231cd94938127 F test/joinH.test 1d2fc3190be68525fd9ce749b9468c40ba2930181e52fb5ee6f836051b38effb -F test/joinI.test fc7d24a2b1e444979b83bd92c30ebb975cebb5b9eae4442ce94969bd8d083053 +F test/joinI.test 249802b168ce96d8d57943ef9abafc1e36e28d91829f68bc2b6e87f2b4d33241 F test/journal1.test bc61a4228db11bffca118bd358ba4b868524bf080f3532749de6c539656e20fa F test/journal2.test 9dac6b4ba0ca79c3b21446bbae993a462c2397c4 F test/journal3.test e5aeff93a7776cf644dbc48dec277655cff80a1cd24689036abc87869b120ea6 @@ -1346,13 +1349,14 @@ F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb9 F test/json/json-speed-check.sh 7d5898808ce7542762318306ae6075a30f5e7ee115c4a409f487e123afe91d88 x F test/json/jsonb-q1.txt 1e180fe6491efab307e318b22879e3a736ac9a96539bbde7911a13ee5b33abc7 F test/json101.test cf53254f0f0c1399a01b21fc58fee0e63a12a556be91b9ee9faccdb8b82c083c -F test/json102.test 9b2e5ada10845ff84853b3feaae2ce51ce7145ae458f74c6a6cecc6ef6ee3ae1 -F test/json103.test 355746a6b66aa438f214b4fae454b13068fad2444b5f693e0d538ad1c059b264 +F test/json102.test ea5c9811e408e115c8fc539548deef431fda4924c23cacd79dd4b783f4449f07 +F test/json103.test e626d109cd0bdb8282ec9bf755af3befa50e3e03a255362fc53433d31e1d66d4 F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 -F test/json105.test 043838b56e68f3252a0dcf5be1689016f6f3f05056f8dcfcdc9d074f4d932988 +F test/json105.test 9900caa21888289873bc6c14f5ee41213d28ac9b7ca3395f8afb73d540e80f66 F test/json106.test 4aed3afd16549045d198a8d9cea00deea96e1f2ecf55864dce96cac558b8abef F test/json107.test 59054e815c8f6b67d634d44ace421cf975828fb5651c4460aa66015c8e19d562 F test/json108.test 0a5f1e2d4b35a1bc33052563d2a5ede03052e2099e58cb424547656c898e0f49 +F test/json109.test 441cea5d73c24a1a34d284101740dfae5a082237c048c8a66b03aeebe5e3643e F test/json501.test b95e2d14988b682a5cadf079dd6162f0f85fb74cd59c6b1f1624110104a974eb F test/json502.test 4ef68e4f272dfb083d4cbceb4e9e51d67ec1186a185e0c13637c50a4dc2f9796 F test/jsonb01.test f4cdfb4cf5a0c940091b17675ed9583f45add0c938f07d65b0de0e19d3a9a101 @@ -1425,7 +1429,7 @@ F test/misc1.test e3e36262aff1bd9b8b9bf1eeb3af04adb3fc1e23f0a92dbff708bba9e939ac F test/misc2.test a1a3573cc02662becd967766021d6f16c54684d56df5f227481c7ef0d9df0bd0 F test/misc3.test 651b88bca19b8ff6a7b6af73dae00c3fd5b3ea5bee0c0d1d91abd4c4b4748718 F test/misc4.test 10cd6addb2fa9093df4751a1b92b50440175dd5468a6ec84d0386e78f087db0e -F test/misc5.test 02fcaf4d42405be02ec975e946270a50b0282dac98c78303ade0d1392839d2b8 +F test/misc5.test 0a5d7604e197f10ee471280bfcaaf8229f9d8e2eebfef2c8853222cbc1ea9cd5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test d595599972ec0b436985f0f02f243b68500ffc977b9b3194ec66c0866cfddcab F test/misc8.test 08d2380bc435486b12161521f225043ac2be26f02471c2c1ea4cac0b1548edbd @@ -1438,11 +1442,12 @@ F test/mmap4.test 2e2b4e32555b58da15176e6fe750f17c9dcf7f93 F test/mmapcorrupt.test 470fb44fe92e99c1d23701d156f8c17865f5b027063c9119dcfdb842791f4465 F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3 F test/mmapwarm.test 2272005969cd17a910077bd5082f70bc1fefad9a875afec7fc9af483898ecaf3 +F test/modeA.sql 3f2b5a7ce7074a52b2b7ec07b07dc1a08edba19e40bce9b4d65d3965413bbea3 F test/multiplex.test d74c034e52805f6de8cc5432cef8c9eb774bb64ec29b83a22effc8ca4dac1f08 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test fac575e0b1b852025575a6a8357701d80933e98b5d2fe6d35ddaa68f92f6a1f7 F test/multiplex4.test e8ae4c4bd70606a5727743241f13b5701990abe4 -F test/mutex1.test 42cb5e244c3a77bb0ef2b967e06fa5e7ba7d32d90a9b20bed98f6f5ede185a25 +F test/mutex1.test 2cdc320a3320521d73b8090a04a2245c1e625e5f90672882517bf5fedcec8f13 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test 73ea63ab43668313e2f8cc9ef9e9a966672c7934f3ce76926fbe991235d07d91 F test/nockpt.test 3db354270fc63b6871eebd40285d4c55324fb27be629c958adbff6d7fcaa8e14 @@ -1452,13 +1457,13 @@ F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161 F test/notify3.test 796c7b7157f55c93b4e672b724e9c923a6fc6aa72ac419379a623e2350472e22 F test/notnull.test a37b663d5bb728d66fc182016613fb8e4a0a4bbf3d75b8876a7527f7d4ed3f18 -F test/notnull2.test 5b7dd6e82c409b2d011ad6acf19ae4bf0816a9c69ccf600b529d7405d7c49874 +F test/notnull2.test c2c7b670fb8fa6ffe5f9cc08af88864fbb8237e28b56ad528e8dee921019c5fe F test/notnullfault.test fc4bb7845582a2b3db376001ef49118393b1b11abe0d24adb03db057ee2b73d5 F test/null.test b7ff206a1c60fe01aa2abd33ef9ea83c93727d993ca8a613de86e925c9f2bc6f F test/nulls1.test 7a5e4346ee4285034100b4cd20e6784f16a9d6c927e44ecdf10034086bbee9c9 F test/numcast.test 5d126f7f581432e86a90d1e35cac625164aec4a1 F test/numindex1.test 20a5450d4b056e48cd5db30e659f13347a099823 -F test/offset1.test 72cca52482cbd5bc687cfa67aa2566c859081b5a353fd2f9da9bbd3914dea1ef +F test/offset1.test c21e67d2d5ae8ed310243fbe84fc2f0dca49e9ffed3ea89110c0d5914c0de620 F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 F test/optfuzz-db01.c 9f2fa80b8f84ebbf1f2e8b13421a4e0477fe300f6686fbd76cac1d2db66e0fdc F test/optfuzz-db01.txt 21f6bdeadc701cf11528276e2a55c70bfcb846ba42df327f979bd9e7b6ce7041 @@ -1507,6 +1512,12 @@ F test/printf2.test 3f55c1871a5a65507416076f6eb97e738d5210aeda7595a74ee895f2224c F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc F test/pushdown.test 46a626ef1c0ca79b85296ff2e078b9da20a50e9b804b38f441590c3987580ddd +F test/qrf01.test abc3e558a75ae2678a3172051b39960dc6fd4b298b6d594afa50939759f4037f +F test/qrf02.test 39b4afdc000bedccdafc0aecf17638df67a67aaa2d2942865ae6abcc48ba0e92 +F test/qrf03.test e7efe46d204671726b4707585126cd78d107368de4a7d0c7b8d5157cdd8624ed +F test/qrf04.test 0894692c998d2401dcc33449c02051b503ecce0c94217be54fb007c82d2d1379 +F test/qrf05.test 8ade5bfa7ef0b448e531687203fa8ae9ef41f1d7e4c11d5ba0c4846af75b13d5 +F test/qrf06.test cd7d0f0e2904904ab88141630a8fff5718ef7e3cc23e5a9c519cf29bb0919d89 F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 F test/quickcheck.test a4b7e878cd97e46108291c409b0bf8214f29e18fddd68a42bc5c1375ad1fb80a @@ -1519,8 +1530,9 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 F test/readonly.test 0d307c335b3421898cfe64a783a376138aa003849b6bff61ee2d21e805bc0051 -F test/recover.test c76d05f33f0271fba0f0752170e03b0ab5952dc61dcea7ab3ba40df03c4c42de -F test/regexp1.test 8f2a8bc1569666e29a4cee6c1a666cd224eb6d50e2470d1dc1df995170f3e0f1 +F test/recover.test 643139b911ac880a1e881d7621f02cfb546b608b8f2494d7d26fd5ed103b1ceb +F test/regexp1.sql de2b5b33b16b664d655b41e780f2efca38de3e5559fc254b4c9783ff0bea96b0 +F test/regexp1.test 0023eae4073265641b826a70d81ba34d4dd66ad71871a5b4a1b7cf500d5c0c51 F test/regexp2.test 64f9726b2ddc71aea06725fcad53231833d038d58b936d49083ace658b370a13 F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2118d F test/reservebytes.test 6163640b5a5120c0dee6591481e673a0fa0bf0d12d4da7513bad692c1a49a162 @@ -1538,13 +1550,13 @@ F test/rowid.test d27191b5ce794c05bf61081e8b2c546a1844c1641321dcaf7fb785234256cc F test/rowvalue.test 93474d8e1c496e970bdcc3a7f54ac835adda667d2fd971957b4bce0c0b11707b F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b F test/rowvalue3.test 103e9a224ca0548dd0d67e439f39c5dd16de4200221a333927372408c025324c -F test/rowvalue4.test bac9326d1e886656650f67c0ec484eb5f452244a8209c6af508e9a862ace08ed +F test/rowvalue4.test 6e160977d44ee715e142f63ec0e339586c61f12bbbffacee369b1cdc0b7390f0 F test/rowvalue5.test 00740304ea6a53a8704640c7405690f0045d5d2a6b4b04dde7bccc14c3068ea7 F test/rowvalue6.test d19b54feb604d5601f8614b15e214e0774c01087 F test/rowvalue7.test 06ec0aca725bf683313d03793aa2943bc7f45a901848c7056a9665b769c8fc38 F test/rowvalue8.test 5900eddad9e2c3c2e26f1a95f74aafc1232ee5e0 F test/rowvalue9.test 7499a8fd7ca3a3f0e19d94e135355439aa2b596f86b775ca8de79672da2ca378 -F test/rowvalueA.test be8d6ad8b476eb24c151bb20bfd487e0d50c5e99618b7b0e656035069d2fc2cf +F test/rowvalueA.test 1c5ed13f3b0641452ae35e6488d6ecc16cefce99f2adf7c07c513530e2aac6b7 F test/rowvaluefault.test 963ae9cdaed30a85a29668dd514e639f3556cae903ee9f172ea972d511c54fff F test/rowvaluevtab.test cd9747bb3f308086944c07968f547ad6b05022e698d80b9ffbdfe09ce0b8da6f F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 @@ -1558,7 +1570,7 @@ F test/savepoint7.test 24c69af86d750c80d51cf6500fde9270717f2b6e5658f055b5e75af75 F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 F test/scanstatus.test b249328caf4d317e71058006872b8012598a5fa045b30bf24a81eeff650ab49e F test/scanstatus2.test d85d17f2b0b4c013dde95232f7beab749f11f0ef847f5ecffb9486d2f5ecf9f9 -F test/schema.test 5dd11c96ba64744de955315d2e4f8992e447533690153b93377dffb2a5ef5431 +F test/schema.test e615575f2d756df4629596523f11d9322384ecf9f980e58c774cff80ff041c33 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 8ed4ae66e082cdd8b1b1f22d8549e1e7a0db4527a8e6ee8b6193053ee1e5c9ce F test/schema4.test 3b26c9fa916abb6dadf894137adcf41b7796f7b9 @@ -1576,7 +1588,7 @@ F test/select5.test 8afc5e5dcdebc2be54472e73ebd9cd1adef1225fd15d37a1c62f969159f3 F test/select6.test da91e61d26b8dea4b61e4a862088dd6ab19998f7be22a16a5b0cfe806e597639 F test/select7.test b825420da8a0b5722fdb77f3369f6396a3d198c46e8787eb26ff9425d4ac9d27 F test/select8.test 8c8f5ae43894c891efc5755ed905467d1d67ad5d -F test/select9.test f7586b207ce2304ab80dc93d3146469a28fd4403621dd3a82d06644563d3c812 +F test/select9.test 108ceff733f31698fef41eb9a0c332f150c54e98be534ee38019a19943f3f5ae F test/selectA.test 1da8ce3884c326e11d2855baffb76436b0d7e044404af8a2a70d1399a4ff7e29 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 F test/selectC.test 38c530b0cc5728b793c3c11f52b52c70290d39822224acd39011c89c1853bd31 @@ -1600,16 +1612,17 @@ F test/sharedA.test 64bdd21216dda2c6a3bd3475348ccdc108160f34682c97f2f51c19fc0e21 F test/sharedB.test 1a84863d7a2204e0d42f2e1606577c5e92e4473fa37ea0f5bdf829e4bf8ee707 F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 -F test/shell1.test ebe953d64c937ad42a0f33170ac0d2d2568faae26813fc7a95203756446d54aa -F test/shell2.test ab23f01ea2347e4b72bb2399af7ee82aa00f9c059141749f7c4064abca5ad728 +F test/shell1.test 2d658ceee13d9e4361d04d0ea16340ad17784ddf378fb6e9ca6d49c682cb4bae +F test/shell2.test dc541d2681503e55466a24d35a4cbf8ca5b90b8fcdef37fc4db07373a67d31d3 F test/shell3.test 603b448e917537cf77be0f265c05c6f63bc677c63a533c8e96aae923b56f4a0e -F test/shell4.test 03593fa7908a55f255916ffeda707cdf55680c777736e3da62b1d78cde0d684d -F test/shell5.test d17e7927ab8b7f720efbdd9b5d05fceb6c3c56c25917901b315400214bf24ef4 +F test/shell4.test e25580a792b7b54560c3a76b6968bd8189261f38979fe28e6bc6312c5db280db +F test/shell5.test a9cd2c8b62e125049ef500937674f47dd6787f0157ac0515aa554044a4dc3ea9 F test/shell6.test e3b883b61d4916b6906678a35f9d19054861123ad91b856461e0a456273bdbb8 F test/shell7.test 43fd8e511c533bab5232e95c7b4be93b243451709e89582600d4b6e67693d5c3 -F test/shell8.test 641cf21a99c59404c24e3062923734951c4099a6b6b6520de00cf7a1249ee871 +F test/shell8.test 38c9e4d7e85d2a3ecfacaa9f6cda4f7a81bf4fffb5f3f37f9cd76827c6883192 F test/shell9.test 8742a5b390cdcef6369f5aa223e415aa4255a4129ef249b177887dc635a87209 -F test/shellA.test 4ecff8b7b2c0122ba8174abfbcc4b0f59e44d80f2a911068f8cd4cfc6661032d +F test/shellA.test 05cdaafa1f79913654487ce3aefa038d4106245d58f52e02faf506140a76d480 +F test/shellB.test 7123d231158588401f332bf278754687b83ba5fc5b352ec8679fb19edfb4cc0a F test/shmlock.test 9f1f729a7fe2c46c88b156af819ac9b72c0714ac6f7246638a73c5752b5fd13c F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 @@ -1643,8 +1656,8 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 377a0c48e5a92e0b11c1c5ebb1bc9d83a7312c922bc0cb05970ef5d6a96d1f0c -F test/speedtest.md ee958457ae1b729d9715ae33c0320600000bf1d9ddea1a88dcf79f56729d6fad -F test/speedtest.tcl 6b66974d833d35a63d0e9ec344e0ffa92fbbfac83e173556f700a61cb3be96fc x +F test/speedtest.md ea0c85ebe0ecff8b45ba6cdb26e694871f469009a5a29dcfe634b055f05ab241 +F test/speedtest.tcl b06f6321ef90bb68f18f7b0e430e25203d9da79b80f8926986a0d5f21ac485fb x F test/speedtest1.c 6c01252e66f46de0b6b8d5316e03521e2151782104f3608c10262aa5dce85721 F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 @@ -1679,23 +1692,24 @@ F test/sync.test a619e407ede58a7b6e3e44375328628559fc9695a9c24c47cb5690a866b0031 F test/sync2.test 06152269ed73128782c450c355988fe8dd794d305833af75e1a5e79edd4dae47 F test/syscall.test a067468b43b8cb2305e9f9fe414e5f40c875bb5d2cba5f00b8154396e95fcf37 F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 -F test/tabfunc01.test 56eeae736217204bb1d9f9ef38340d48058f809b64249217cf77ff4ba600cc21 +F test/tabfunc01.test cfa96a9a235c39fb0cae69928b989b28bfec108f62d2533486f76e32dcedfdfb F test/table.test e87294bf1c80bfd7792142b84ab32ea5beb4f3f71e535d7fb263a6b2068377bf F test/tableapi.test e37c33e6be2276e3a96bb54b00eea7f321277115d10e5b30fdb52a112b432750 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 -F test/tclsqlite.test 3f697424cfc1cdc9c076ec0cadb0e700f059400a3e3ce134b7d856fc9f880e1c +F test/tclsqlite.test 5d6c73bfe7006c85e2f7fb7db8638b521eb2043d5451aaacdac4851eab895443 F test/tempdb.test 4cdaa23ddd8acb4d79cbb1b68ccdfd09b0537aaba909ca69a876157c2a2cbd08 F test/tempdb2.test 353864e96fd3ae2f70773d0ffbf8b1fe48589b02c2ec05013b540879410c3440 F test/tempfault.test 0c0d349c9a99bf5f374655742577f8712c647900 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d1631311a16 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 -F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl 463ae33b8bf75ac77451df19bd65e7c415c2e9891227c7c9e657d0a2d8e1074a +F test/temptrigfault.tes fc5918e64f3867156fefe7cfca9d8e1f495134a5229b2b511b0dc11c07f2eab4 +F test/temptrigger.test a00f258ed8d21a0e8fd4f322f15e8cfb5cef2e43655670e07a753e3fb4769d61 +F test/tester.tcl 2d943f60200e0a36bcd3f1f0baf181a751cd3604ef6b6bd4c8dc39b4e8a53116 F test/testloadext.c 862b848783eaed9985fbce46c65cd214664376b549fae252b364d5d1ef350a27 -F test/testrunner.tcl 60d7efa1816c5dfc37df3e3454b94b9042c0c8c50b27ae296d4a797cd309ace6 x -F test/testrunner_data.tcl c507a9afa911c03446ed90442ffd4a98aca02882c3d51bd1177c24795674def8 -F test/testrunner_estwork.tcl 7927a84327259a32854926f68a75292e33a61e7e052fdbfcb01f18696c99c724 +F test/testrunner.tcl 78d67079fc39caf2af3fd9d4c30bdac78dae7ec50b9fc802835e7a5189581e07 x +F test/testrunner_data.tcl 078e251983c8fc573567125147655f68132210f226c92922daf21fb913779717 +F test/testrunner_estwork.tcl 81e2ae10238f50540f42fbf2d94913052a99bfb494b69e546506323f195dcff9 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1744,7 +1758,7 @@ F test/tkt-8454a207b9.test ead80b7a01438ca1436cee029694a96c821346cf1e24f06de12f8 F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356 F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 -F test/tkt-99378177930f87bd.test 9d6cff39b50d062c813ae1cb0ebbd1b7acf81ecc23ae5d5215e5bb05667dc137 +F test/tkt-99378177930f87bd.test 1ee631d155f0d51a4547e9405ef35a3a9a32977352a37a10bcbbacc5e38356ad F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667 F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6 F test/tkt-9f2eb3abac.test cb6123ac695a08b4454c3792fbe85108f67fabf8 @@ -1792,7 +1806,7 @@ F test/tkt2213.test a9702175601a57b61aba095a233b001d6f362474 F test/tkt2251.test 5aab8c7898cd2df2a68fe19289cc29e8f5cf8c82 F test/tkt2285.test cca17be61cf600b397188e77e7143844d2b977e9 F test/tkt2332.test fc955609b958ca86dfa102832243370a0cc84070 -F test/tkt2339.test 73bd17818924cd2ac442e5fd9916b58565739450 +F test/tkt2339.test bad48bd064594aa7b4de23f6d59a72b0b0c4175fd917f4b66907584732d41652 F test/tkt2391.test ab7a11be7402da8b51a5be603425367aa0684567 F test/tkt2409.test be0d60e7d283f639dccea4b0b5e1cd3a4851fb5b F test/tkt2450.test 77ed94863f2049c1420288ddfea2d41e5e0971d6 @@ -1926,15 +1940,15 @@ F test/vacuum4.test 7ea76b769fffeb41f925303b04cbcf5a5bbeabe55e4c60ae754ff24eeeb7 F test/vacuum5.test 263b144d537e92ad8e9ca8a73cc6e1583f41cfd0dda9432b87f7806174a2f48c F test/vacuum6.test b137b04bf3392d3f5c3b8fda0ce85a6775a70ca112f6559f74ff52dc9ce042fd F test/vacuummem.test 4b30f5b95a9ff86e9d5c20741e50a898b2dc10b0962a3211571eb165357003fb -F test/values.test 0eda08a6ce6545f1ab012dff4cc72a7dd0fee2510f42444136bb2b2b5ed84bc0 +F test/values.test 0e037c50789ac2a308746567d07b53b2f6026c1bb3a435d1b099424600e64caf F test/valuesfault.test 2ef23ed965e3bd08e268cdc38a0d11653390ddbbe1e8e2e98d16f55edd30f6e8 F test/varint.test bbce22cda8fc4d135bcc2b589574be8410614e62 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 F test/view.test 3c23d7a068e9e4a0c4e6907498042772adea725f0630c3d9638ffd4e5a08b92b F test/view2.test db32c8138b5b556f610b35dfddd38c5a58a292f07fda5281eedb0851b2672679 F test/view3.test ad8a8290ee2b55ff6ce66c9ef1ce3f1e47926273a3814e1c425293e128a95456 -F test/vt02.c 5b44ac67b1a283fedecf2d6e2ceda61e7a157f01d44dcb4490dcb1e87d057060 -F test/vt100-a.sql 631eeab18c5adb531bab79aecf64eee3934b42c75a309ee395c814717a6a7651 +F test/vt02.c c2faf56d74470d569cd00741acb3f1719ee95d668f84ef58acc3872635789680 +F test/vt100-a.sql a3e188a118ca78c08b41681a4db6d0f353e554ceb33f1573b1872d16e2d30596 F test/vtab1.test 09a72330d0f31eda2ffaa828b06a6b917fb86250ee72de0301570af725774c07 F test/vtab2.test 14d4ab26cee13ba6cf5c5601b158e4f57552d3b055cdd9406cf7f711e9c84082 F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e @@ -1975,7 +1989,7 @@ F test/wal_common.tcl 204d1721ac13c5e0c7fae6380315b5ab7f4e8423f580d826c5e9df1995 F test/walbak.test 018d4e5a3d45c6298d11b99f09a8ef6876527946 F test/walbig.test f437473a16cfb314867c6b5d1dbcd519e73e3434 F test/walblock.test 6bb472e82730e7e4e81395e907a01d8cfc2bd9e1f01f8a9184ca572e2955a4bf -F test/walckptnoop.test b13a2c3140f2c913cfd422d9a224544757d04b8b14ab4c267ab9910467c0b9be +F test/walckptnoop.test 5f6123750f40cb86633a7e014f9fb805d0eb494b811840086dc72e554e68c7c1 F test/walcksum.test 50e204500eed9c691b6045e467bb2923f49aa93d8adf315e2be135fdb202c1c2 F test/walcrash.test 21038858cc552077b0522f50b0fa87e38139306a F test/walcrash2.test a0edab4e5390f03b99a790de89aad15d6ec70b36 @@ -1990,6 +2004,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8d78a1ec91299163451417b451a2bac3481f8eb9f455b1ca507a6625c927ca6e F test/walprotocol.test 1b3f922125e341703f6e946d77fdc564d38fb3e07a9385cfdc6c99cac1ecf878 F test/walprotocol2.test 7d3b6b4bf0b12f8007121b1e6ef714bc99101fb3b48e46371df1db868eebc131 +F test/walrestart.test 5168c0c2414d1971d8dec949c1070a0144cf15402361ba0d0e6a8054f5598a64 F test/walro.test 78a84bc0fdae1385c06b017215c426b6845734d6a5a3ac75c918dd9b801b1b9d F test/walro2.test 33955a6fd874dd9724005e17f77fef89d334b3171454a1256fe4941a96766cdc F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68 @@ -2004,7 +2019,7 @@ F test/walslow.test 0c51843836c9dcf40a5ac05aa781bfb977b396ee2c872d92bd48b79d5dd9 F test/walthread.test d562f51a61191ccfab64940df7aa1cef87c902fa5ab742590ef7f859dfe6a44b F test/walvfs.test e1a6ad0f3c78e98b55c3d5f0889cf366cc0d0a1cb2bccb44ac9ec67384adc4a1 F test/where.test 5087c72d26fd075a1644c8512be9fe18de9bf2d2b0754f7fd9b74a1c6540c4fc -F test/where2.test 52237a8cb27ebbf6583469429bb5733d1b94ac37d09c3dbd0f487952ed0ab3f8 +F test/where2.test 1dbff4ab847068a52b927a0c1a7cf7faed4c8cae081fbcdac9b052a3a209aefa F test/where3.test 4ccb156ae33de86414a52775a6f590a9d60ba2cbc7a93a24fa331b7bcf5b6030 F test/where4.test 4a371bfcc607f41d233701bdec33ac2972908ba8 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 @@ -2022,7 +2037,7 @@ F test/whereG.test 875d020ac0a47828b31e36c54f1bf0cf81c9ea43b257bc21286eca1fe9a48 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/whereI.test c4bb7e2ca56d49bd8ab5c7bd085b8b83e353922b46904d68aefb3c7468643581 F test/whereJ.test fc05e374cc9f2dc204148d6c06822c380ad388895fe97a6d335b94a26a08aecf -F test/whereK.test 0270ab7f04ba5436fb9156d31d642a1c82727f4c4bfe5ba90d435c78cf44684a +F test/whereK.test 4fb96b078f2ecedc467fa53177787378ff659539e415a4256cae7ae4e2a804b2 F test/whereL.test cb115604cc9bd61acbc99a1f1df0eb1ea7a7875a77fef25ba9282f01d10283e1 F test/whereM.test 0dbc9998783458ddcf3cc078ca7c2951d8b2677d472ecf0028f449ed327c0250 F test/whereN.test 63a3584b71acfb6963416de82f26c6b1644abc5ca6080c76546b9246734c8803 @@ -2034,7 +2049,7 @@ F test/wherelimit3.test 22d73e046870cf8bbe15573eda6b432b07ebe64a88711f9f849c6b36 F test/widetab1.test c296a98e123762de79917350e45fa33fdf88577a2571eb3a64c8bf7e44ef74d1 F test/win32heap.test 1ec2ce646aee491ec23bfcdfd005b33c79f13bf91467966f374a76ffe7c7e85f F test/win32lock.test e56d7a9b6cf9d5f3867c2dd19ff36c5326881e4038c6867610ecb3a9868ea4eb -F test/win32longpath.test 0f9837039b306735c13521c5f25b6ed42937b600dace58e28a3d2f8baf429b6a +F test/win32longpath.test 2641e3a5dbb59f49456f6caf78c0acd6ec7dbba27cb56363bab8fbfe93995caa F test/win32nolock.test 95854dc0206b8a95e4aee15a76acc082767b38f079b2e24676aed6cbb0f32798 F test/window1.test b46d28b9698559e66aa4adafd8074b940faee498bf0c4fbdb62548bfcccc67e7 F test/window2.tcl 492c125fa550cda1dd3555768a2303b3effbeceee215293adf8871efc25f1476 @@ -2059,7 +2074,7 @@ F test/windowerr.tcl f5acd6fbc210d7b5546c0e879d157888455cd4a17a1d3f28f07c1c8a387 F test/windowerr.test a8b752402109c15aa1c5efe1b93ccb0ce1ef84fa964ae1cd6684dd0b3cc1819b F test/windowfault.test 15094c1529424e62f798bc679e3fe9dfab6e8ba2f7dfe8c923b6248c31660a7c F test/windowpushd.test c420e2265f0e09a0e798d0513a660d71b51602088d81b3dbd038918ee1339dcc -F test/with1.test 1ee171d7c306ab8b0771f3511d870f56c735607729836585bbceb1fc2f47e0b1 +F test/with1.test 31db84788e0429885b63995149fab57d32e26196b752a3a926249ae74c0adddd F test/with2.test 181674a6cc86a601ca2ac052741cdfad5b529e07e870435d2f6cdb92d589ff17 F test/with3.test e30369ea27aa27eb1bda4c5e510c8a9f782c8afd2ab99d1a02b8a7f25a5d3e65 F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205 @@ -2078,13 +2093,13 @@ F test/writecrash.test 13520af28f376bfc8c0bcd130efc1fff20bb165198e8b94cf153f1f75 F test/zeroblob.test 7b74cefc7b281dfa2b07cd237987fbe94b4a2037a7771e9e83f2d5f608b1d99e F test/zeroblobfault.test 861d8191a0d944dfebb3cb4d2c5b4e46a5a119eaec5a63dd996c2389f8063441 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc -F test/zipfile.test c52db63e31a66ae4245affa3e4e65e302442a87e5fd5f2ad29060bc849a83480 -F test/zipfile2.test a577e0775e32ef8972e7d5e9a45bc071a5ae061b5b965a08c9c4b709ad036a25 +F test/zipfile.test a3fcfc43115e4226fdddadd43bdf31c8ca805ad08dad435634f1633d8f5840d9 +F test/zipfile2.test 21afaffcf4f7769df38bf16e4a9c4dfa6ba1b0f5b695f844ec61fafb92db0db7 F test/zipfilefault.test 44d4d7a7f7cca7521d569d7f71026b241d65a6b1757aa409c1a168827edbbc2c F tool/GetFile.cs 47852aa0d806fe47ed1ac5138bdce7f000fe87aaa7f28107d0cb1e26682aeb44 F tool/GetTclKit.bat d84033c6a93dfe735d247f48ba00292a1cc284dcf69963e5e672444e04534bbf F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 -F tool/build-all-msvc.bat 1960a7a3e5d8176c4329e31476f6e3dfa9543675355fa9020a569f4452628458 x +F tool/build-all-msvc.bat 1ee9dbadcc07fc23268025854e97b392bcbad72376b47aee7b22f3797a4f2c87 x F tool/build-shell.sh 369c4b171cc877ad974fef691e4da782b4c1e99fe8f4361316c735f64d49280f F tool/buildtclext.tcl d09b753d7858314104eeaf5f4def85d35784470279809e47a633f142226f2b3f F tool/cg_anno.tcl c1f875f5a4c9caca3d59937b16aff716f8b1883935f1b4c9ae23124705bc8099 x @@ -2093,7 +2108,7 @@ F tool/cktclsh.sh 6075eef9c6b9ba4b38fef2ca2a66d25f2311bd3c610498d18a9b01f861629c F tool/cp.tcl 9a0d663ad45828de13763ee7ca0200f31f56c6d742cf104a56ae80e027c242d8 F tool/custom.txt 24ed55e71c5edae0067ba159bbf09240d58b160331f7716e95816cd3aa0ba5c4 F tool/dbhash.c 5da0c61032d23d74f2ab84ffc5740f0e8abec94f2c45c0b4306be7eb3ae96df0 -F tool/dbtotxt.c ca48d34eaca6d6b6e4bd6a7be2b72caf34475869054240244c60fa7e69a518d6 +F tool/dbtotxt.c cfeb957571735af345f253ba8417256031fa0dddf79468eefad184262d17211e F tool/dbtotxt.md c9a57af8739957ef36d2cfad5c4b1443ff3688ed33e4901ee200c8b651f43f3c F tool/emcc.sh.in 41a049468c8155433e37e656ba5bae063a000768b1d627025f277732c4e7c4a4 F tool/enlargedb.c 3e8b2612b985cfa7e3e8800031ee191b43ae80de96abb5abbd5eada62651ee21 @@ -2106,24 +2121,25 @@ F tool/genfkey.README e550911fa984c8255ebed2ef97824125d83806eb5232582700de949edf F tool/genfkey.test b6afd7b825d797a1e1274f519ab5695373552ecad5cd373530c63533638a5a4f F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/index_usage.c f62a0c701b2c7ff2f3e21d206f093c123f222dbf07136a10ffd1ca15a5c706c5 -F tool/lemon.c 8f6c122e5727cb0e5f302b8efc91489b1947a8d98206d7a1b1cfc0ed685b6e7c -F tool/lempar.c bdffd3b233a4e4e78056c9c01fadd2bb3fe902435abde3bce3d769fdf0d5cca2 +F tool/lemon.c 3fdc16b23f1ea0c91c049b518fc3f75c71843dbfe2b447fcb3cd92d9e4f219f8 +F tool/lempar.c b57e1780bf8098dd4a9a5bba537f994276ea825a420f6165153e5894dc2dfb07 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c 63412f9790e5e8538fbde0b4f6db154aaaf80f7a10a01e3c94d14b773a8dd5a6 F tool/logest.c c34e5944318415de513d29a6098df247a9618c96d83c38d4abd88641fe46e669 F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439 F tool/merge-test.tcl de76b62f2de2a92d4c1ca4f976bce0aea6899e0229e250479b229b2a1914b176 F tool/mkamalzip.tcl 8aa5ebe7973c8b8774062d34e15fea9815c4cc2ceea3a9b184695f005910876a -F tool/mkautoconfamal.sh 647dada5e34c466bef62a4408e1c99a7e5e1922805479dd57944f33f9803f2f8 +F tool/mkautoconfamal.sh 06fbe090b81c24e592c1f22b404334f805ba74d482a9260f2ac81e6f3d3386d8 F tool/mkccode.tcl c42a8f8cf78f92e83795d5447460dbce7aaf78a3bbf9082f1507dc71a3665f3c x +F tool/mkcombo.tcl 2a5189b219c4a495e1ff7fc980bd568d3cfb82ae9d50c84e77f7a161e96fc132 F tool/mkctimec.tcl 3fb5cad05922f5da61262cb6bcd5868a34e94a49ca8833ae2d7796e7df075576 x -F tool/mkkeywordhash.c 6b0be901c47f9ad42215fc995eb2f4384ac49213b1fba395102ec3e999acf559 +F tool/mkkeywordhash.c 82d5af1d0e677900739fba59155cddac172d8c712c2d91ab73d6e6bcb30060f0 F tool/mkmsvcmin.tcl d76c45efda1cce2d4005bcea7b8a22bb752e3256009f331120fb4fecb14ebb7a F tool/mkopcodec.tcl 33d20791e191df43209b77d37f0ff0904620b28465cca6990cf8d60da61a07ef F tool/mkopcodeh.tcl 2b4e6967a670ef21bf53a164964c35c6163277d002a4c6f56fa231d68c88d023 F tool/mkopts.tcl 680f785fdb09729fd9ac50632413da4eadbdf9071535e3f26d03795828ab07fa F tool/mkpragmatab.tcl 3801ce32f8c55fe63a3b279f231fb26c2c1a2ea9a09d2dd599239d87a609acec -F tool/mkshellc.tcl bab0a72a68384181a5706712dfdf6815f6526446d4e8aacace2de5e80cda91b2 +F tool/mkshellc.tcl da6918b128e928a8f0d663519e14829153e59465bd5eb596442e99fa10a411b7 F tool/mksourceid.c 36aa8020014aed0836fd13c51d6dc9219b0df1761d6b5f58ff5b616211b079b9 F tool/mksqlite3c-noext.tcl 351c55256213154cabb051a3c870ef9f4487de905015141ae50dc7578a901b84 F tool/mksqlite3c.tcl 7a268139158e5deef27a370bc2f8db6ccf100c1ad7ac5e5b23743c0fd354f609 @@ -2135,26 +2151,27 @@ F tool/mkvsix.tcl 67b40996a50f985a573278eea32fc5a5eb6110bdf14d33f1d8086e48c69e54 F tool/mkwinarm64ec.tcl 171f79234fa53552a129b360356df5599fdab15239caffb3d29c571292728033 F tool/offsets.c 8ed2b344d33f06e71366a9b93ccedaa38c096cc1dbd4c3c26ad08c6115285845 F tool/omittest-msvc.tcl d6b8f501ac1d7798c4126065030f89812379012cad98a1735d6d7221492abc08 -F tool/omittest.tcl bec70ef0e16255c8d9eb06ecd7edf823c07a60a836186cdbce3528fb34b67995 +F tool/omittest.tcl 436b7072e00e25e9b77145a9f67aa8e0eeabd186168827435fd03f8f981aac32 F tool/opcodesum.tcl 740ed206ba8c5040018988129abbf3089a0ccf4a F tool/pagesig.c f98909b4168d9cac11a2de7f031adea0e2f3131faa7515a72807c03ec58eafeb F tool/replace.tcl 511c61acfe563dfb58675efb4628bb158a13d48ff8322123ac447e9d25a82d9a F tool/restore_jrnl.tcl 1079ecba47cc82fa82115b81c1f68097ab1f956f357ee8da5fc4b2589af6bd98 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 -F tool/showdb.c 3956d71e5193162609a60e8c9edfcf09274c00cfea2b1d221261427adb2b5cca +F tool/showdb.c 1faa3661d2d634f206c76794cb21d89d3ea9082d07d5e983be0f025e40f21320 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818 F tool/showlocks.c 9cc5e66d4ebbf2d194f39db2527ece92077e86ae627ddd233ee48e16e8142564 F tool/showshm.c a0ab6ec32dd1f11218ca2a4018f8fb875b59414801ab8ceed8b2e69b7b45a809 F tool/showstat4.c b706fcbc4cd1a6e4a73ac32549afc4b460479d650402d64b23e8d813516e8de4 +F tool/showtmlog.c 2e9da6c4b4767113a0ad5ddabd4337ea100d38ff9c7fee260f9ccdefb2ffdc23 F tool/showwal.c 11eca547980a066b081f512636151233350ac679f29ecf4ebfce7f4530230b3d F tool/soak1.tcl a3892082ed1079671565c044e93b55c3c7f38829aedf53cc597c65d23ffdaddf F tool/spaceanal.tcl 1f83962090a6b60e1d7bf92495d643e622bef9fe82ea3f2d22350dcbce9a12d0 F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 4969fd642dad0ea483e4e104163021d92baf98f6a8eac981fe48525f9b873430 -F tool/sqldiff.c 134be7866be19f8beb32043d5aea5657f01aaeae2df8d33d758ff722c78666b9 +F tool/sqldiff.c 847edc1e0d1e1feb652d3d6128e504456deaf254ab9ad3e7cebd4317d2037182 F tool/sqlite3_analyzer.c.in 14f02cb5ec3c264cd6107d1f1dad77092b1cf440fc196c30b69ae87b56a1a43b -F tool/sqlite3_rsync.c d0e58a1e49fe2192c3ee0b697aed182d502bebfe5b4b406ba6b2baa52a04ecbe -F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 +F tool/sqlite3_rsync.c f510a8b230e1c5b0f62842acd0e94ff15d2f77a00ae782f7d20f9e39919fa19b +F tool/sqltclsh.c.in c103c6fc7d42bce611f9d4596774d60b7ef3d0b291a1f58c9e6184e458b89296 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 F tool/src-verify.c 6c655d9a8d6b30f3648fc78a79bf3838ed68f8543869d380c43ea9f17b3b8501 F tool/srcck1.c 559e703c6cca1d70398bdba1d7f91036c1a71adf718a1aaa6401a562ccaed154 @@ -2171,10 +2188,12 @@ F tool/version-info.c 33d0390ef484b3b1cb685d59362be891ea162123cea181cb8e6d2cf6dd F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 10e11b9c539a8be50fd93bdf7cf5afe97d9757ce8577cac58426a1b218063e47 -R ec4161915fdee7c9c786931b31f64669 +F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c +P 5fa49c4d592778fb82c4e25c77cf0442d3dc23cc7f8d91d25952c722af866930 +R b4a00b0f2cf0b157145ad751c36ae742 +T +sym-major-release * T +sym-release * -T +sym-version-3.51.2 * +T +sym-version-3.52.0 * U drh -Z 6e3a819551f499011e2275a586e8d608 +Z ee8b8689818d3dfbd1a3f7d6f78a36ce # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.tags b/manifest.tags index c644e06f9..e0b8607c1 100644 --- a/manifest.tags +++ b/manifest.tags @@ -1,4 +1,5 @@ -branch branch-3.51 +branch trunk +tag trunk tag release -tag branch-3.51 -tag version-3.51.2 +tag major-release +tag version-3.52.0 diff --git a/manifest.uuid b/manifest.uuid index 75ccd2e60..55fe12eaa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b270f8339eb13b504d0b2ba154ebca966b7dde08e40c3ed7d559749818cb2075 +557aeb43869d3585137b17690cb3b64f7de6921774daae9e56403c3717dceab6 diff --git a/src/alter.c b/src/alter.c index a7255e75e..fb5a37935 100644 --- a/src/alter.c +++ b/src/alter.c @@ -491,7 +491,7 @@ void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ /* Look up the table being altered. */ assert( pParse->pNewTable==0 ); assert( sqlite3BtreeHoldsAllMutexes(db) ); - if( db->mallocFailed ) goto exit_begin_add_column; + if( NEVER(db->mallocFailed) ) goto exit_begin_add_column; pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); if( !pTab ) goto exit_begin_add_column; @@ -563,7 +563,7 @@ void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ ** Or, if pTab is not a view or virtual table, zero is returned. */ #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) -static int isRealTable(Parse *pParse, Table *pTab, int bDrop){ +static int isRealTable(Parse *pParse, Table *pTab, int iOp){ const char *zType = 0; #ifndef SQLITE_OMIT_VIEW if( IsView(pTab) ){ @@ -576,9 +576,12 @@ static int isRealTable(Parse *pParse, Table *pTab, int bDrop){ } #endif if( zType ){ + const char *azMsg[] = { + "rename columns of", "drop column from", "edit constraints of" + }; + assert( iOp>=0 && iOpzName + azMsg[iOp], zType, pTab->zName ); return 1; } @@ -1049,6 +1052,25 @@ static RenameToken *renameColumnTokenNext(RenameCtx *pCtx){ return pBest; } +/* +** Set the error message of the context passed as the first argument to +** the result of formatting zFmt using printf() style formatting. +*/ +static void errorMPrintf(sqlite3_context *pCtx, const char *zFmt, ...){ + sqlite3 *db = sqlite3_context_db_handle(pCtx); + char *zErr = 0; + va_list ap; + va_start(ap, zFmt); + zErr = sqlite3VMPrintf(db, zFmt, ap); + va_end(ap); + if( zErr ){ + sqlite3_result_error(pCtx, zErr, -1); + sqlite3DbFree(db, zErr); + }else{ + sqlite3_result_error_nomem(pCtx); + } +} + /* ** An error occurred while parsing or otherwise processing a database ** object (either pParse->pNewTable, pNewIndex or pNewTrigger) as part of an @@ -1346,8 +1368,8 @@ static int renameResolveTrigger(Parse *pParse){ sqlite3SelectPrep(pParse, pStep->pSelect, &sNC); if( pParse->nErr ) rc = pParse->rc; } - if( rc==SQLITE_OK && pStep->zTarget ){ - SrcList *pSrc = sqlite3TriggerStepSrc(pParse, pStep); + if( rc==SQLITE_OK && pStep->pSrc ){ + SrcList *pSrc = sqlite3SrcListDup(db, pStep->pSrc, 0); if( pSrc ){ Select *pSel = sqlite3SelectNew( pParse, pStep->pExprList, pSrc, 0, 0, 0, 0, 0, 0 @@ -1375,10 +1397,10 @@ static int renameResolveTrigger(Parse *pParse){ pSel->pSrc = 0; sqlite3SelectDelete(db, pSel); } - if( pStep->pFrom ){ + if( ALWAYS(pStep->pSrc) ){ int i; - for(i=0; ipFrom->nSrc && rc==SQLITE_OK; i++){ - SrcItem *p = &pStep->pFrom->a[i]; + for(i=0; ipSrc->nSrc && rc==SQLITE_OK; i++){ + SrcItem *p = &pStep->pSrc->a[i]; if( p->fg.isSubquery ){ assert( p->u4.pSubq!=0 ); sqlite3SelectPrep(pParse, p->u4.pSubq->pSelect, 0); @@ -1447,13 +1469,13 @@ static void renameWalkTrigger(Walker *pWalker, Trigger *pTrigger){ sqlite3WalkExpr(pWalker, pUpsert->pUpsertWhere); sqlite3WalkExpr(pWalker, pUpsert->pUpsertTargetWhere); } - if( pStep->pFrom ){ + if( pStep->pSrc ){ int i; - SrcList *pFrom = pStep->pFrom; - for(i=0; inSrc; i++){ - if( pFrom->a[i].fg.isSubquery ){ - assert( pFrom->a[i].u4.pSubq!=0 ); - sqlite3WalkSelect(pWalker, pFrom->a[i].u4.pSubq->pSelect); + SrcList *pSrc = pStep->pSrc; + for(i=0; inSrc; i++){ + if( pSrc->a[i].fg.isSubquery ){ + assert( pSrc->a[i].u4.pSubq!=0 ); + sqlite3WalkSelect(pWalker, pSrc->a[i].u4.pSubq->pSelect); } } } @@ -1624,8 +1646,8 @@ static void renameColumnFunc( if( rc!=SQLITE_OK ) goto renameColumnFunc_done; for(pStep=sParse.pNewTrigger->step_list; pStep; pStep=pStep->pNext){ - if( pStep->zTarget ){ - Table *pTarget = sqlite3LocateTable(&sParse, 0, pStep->zTarget, zDb); + if( pStep->pSrc ){ + Table *pTarget = sqlite3LocateTableItem(&sParse, 0, &pStep->pSrc->a[0]); if( pTarget==pTab ){ if( pStep->pUpsert ){ ExprList *pUpsertSet = pStep->pUpsert->pUpsertSet; @@ -1637,7 +1659,6 @@ static void renameColumnFunc( } } - /* Find tokens to edit in UPDATE OF clause */ if( sParse.pTriggerTab==pTab ){ renameColumnIdlistNames(&sParse, &sCtx,sParse.pNewTrigger->pColumns,zOld); @@ -1839,13 +1860,10 @@ static void renameTableFunc( if( rc==SQLITE_OK ){ renameWalkTrigger(&sWalker, pTrigger); for(pStep=pTrigger->step_list; pStep; pStep=pStep->pNext){ - if( pStep->zTarget && 0==sqlite3_stricmp(pStep->zTarget, zOld) ){ - renameTokenFind(&sParse, &sCtx, pStep->zTarget); - } - if( pStep->pFrom ){ + if( pStep->pSrc ){ int i; - for(i=0; ipFrom->nSrc; i++){ - SrcItem *pItem = &pStep->pFrom->a[i]; + for(i=0; ipSrc->nSrc; i++){ + SrcItem *pItem = &pStep->pSrc->a[i]; if( 0==sqlite3_stricmp(pItem->zName, zOld) ){ renameTokenFind(&sParse, &sCtx, pItem->zName); } @@ -2092,6 +2110,57 @@ static void renameTableTest( #endif } + +/* +** Return the number of bytes until the end of the next non-whitespace and +** non-comment token. For the purpose of this function, a "(" token includes +** all of the bytes through and including the matching ")", or until the +** first illegal token, whichever comes first. +** +** Write the token type into *piToken. +** +** The value returned is the number of bytes in the token itself plus +** the number of bytes of leading whitespace and comments skipped plus +** all bytes through the next matching ")" if the token is TK_LP. +** +** Example: (Note: '.' used in place of '*' in the example z[] text) +** +** ,--------- *piToken := TK_RP +** v +** z[] = " /.comment./ --comment\n (two three four) five" +** | | +** |<-------------------------------------->| +** | +** `--- return value +*/ +static int getConstraintToken(const u8 *z, int *piToken){ + int iOff = 0; + int t = 0; + do { + iOff += sqlite3GetToken(&z[iOff], &t); + }while( t==TK_SPACE || t==TK_COMMENT ); + + *piToken = t; + + if( t==TK_LP ){ + int nNest = 1; + while( nNest>0 ){ + iOff += sqlite3GetToken(&z[iOff], &t); + if( t==TK_LP ){ + nNest++; + }else if( t==TK_RP ){ + t = TK_LP; + nNest--; + }else if( t==TK_ILLEGAL ){ + break; + } + } + } + + *piToken = t; + return iOff; +} + /* ** The implementation of internal UDF sqlite_drop_column(). ** @@ -2136,15 +2205,24 @@ static void dropColumnFunc( goto drop_column_done; } - pCol = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol].zCnName); if( iColnCol-1 ){ RenameToken *pEnd; + pCol = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol].zCnName); pEnd = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol+1].zCnName); zEnd = (const char*)pEnd->t.z; }else{ + int eTok; assert( IsOrdinaryTable(pTab) ); + assert( iCol!=0 ); + /* Point pCol->t.z at the "," immediately preceding the definition of + ** the column being dropped. To do this, start at the name of the + ** previous column, and tokenize until the next ",". */ + pCol = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol-1].zCnName); + do { + pCol->t.z += getConstraintToken((const u8*)pCol->t.z, &eTok); + }while( eTok!=TK_COMMA ); + pCol->t.z--; zEnd = (const char*)&zSql[pTab->u.tab.addColOffset]; - while( ALWAYS(pCol->t.z[0]!=0) && pCol->t.z[0]!=',' ) pCol->t.z--; } zNew = sqlite3MPrintf(db, "%.*s%s", pCol->t.z-zSql, zSql, zEnd); @@ -2313,6 +2391,651 @@ void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const Token *pName){ sqlite3SrcListDelete(db, pSrc); } +/* +** Return the number of bytes of leading whitespace/comments in string z[]. +*/ +static int getWhitespace(const u8 *z){ + int nRet = 0; + while( 1 ){ + int t = 0; + int n = sqlite3GetToken(&z[nRet], &t); + if( t!=TK_SPACE && t!=TK_COMMENT ) break; + nRet += n; + } + return nRet; +} + + +/* +** Argument z points into the body of a constraint - specifically the +** second token of the constraint definition. For a named constraint, +** z points to the first token past the CONSTRAINT keyword. For an +** unnamed NOT NULL constraint, z points to the first byte past the NOT +** keyword. +** +** Return the number of bytes until the end of the constraint. +*/ +static int getConstraint(const u8 *z){ + int iOff = 0; + int t = 0; + + /* Now, the current constraint proceeds until the next occurence of one + ** of the following tokens: + ** + ** CONSTRAINT, PRIMARY, NOT, UNIQUE, CHECK, DEFAULT, + ** COLLATE, REFERENCES, FOREIGN, GENERATED, AS, RP, or COMMA + ** + ** Also exit the loop if ILLEGAL turns up. + */ + while( 1 ){ + int n = getConstraintToken(&z[iOff], &t); + if( t==TK_CONSTRAINT || t==TK_PRIMARY || t==TK_NOT || t==TK_UNIQUE + || t==TK_CHECK || t==TK_DEFAULT || t==TK_COLLATE || t==TK_REFERENCES + || t==TK_FOREIGN || t==TK_RP || t==TK_COMMA || t==TK_ILLEGAL + || t==TK_AS || t==TK_GENERATED + ){ + break; + } + iOff += n; + } + + return iOff; +} + +/* +** Compare two constraint names. +** +** Summary: *pRes := zQuote != zCmp +** +** Details: +** Compare the (possibly quoted) constraint name zQuote[0..nQuote-1] +** against zCmp[]. Write zero into *pRes if they are the same and +** non-zero if they differ. Normally return SQLITE_OK, except if there +** is an OOM, set the OOM error condition on ctx and return SQLITE_NOMEM. +*/ +static int quotedCompare( + sqlite3_context *ctx, /* Function context on which to report errors */ + int t, /* Token type */ + const u8 *zQuote, /* Possibly quoted text. Not zero-terminated. */ + int nQuote, /* Length of zQuote in bytes */ + const u8 *zCmp, /* Zero-terminated, unquoted name to compare against */ + int *pRes /* OUT: Set to 0 if equal, non-zero if unequal */ +){ + char *zCopy = 0; /* De-quoted, zero-terminated copy of zQuote[] */ + + if( t==TK_ILLEGAL ){ + *pRes = 1; + return SQLITE_OK; + } + zCopy = sqlite3MallocZero(nQuote+1); + if( zCopy==0 ){ + sqlite3_result_error_nomem(ctx); + return SQLITE_NOMEM_BKPT; + } + memcpy(zCopy, zQuote, nQuote); + sqlite3Dequote(zCopy); + *pRes = sqlite3_stricmp((const char*)zCopy, (const char*)zCmp); + sqlite3_free(zCopy); + return SQLITE_OK; +} + +/* +** zSql[] is a CREATE TABLE statement, supposedly. Find the offset +** into zSql[] of the first character past the first "(" and write +** that offset into *piOff and return SQLITE_OK. Or, if not found, +** set the SQLITE_CORRUPT error code and return SQLITE_ERROR. +*/ +static int skipCreateTable(sqlite3_context *ctx, const u8 *zSql, int *piOff){ + int iOff = 0; + + if( zSql==0 ) return SQLITE_ERROR; + + /* Jump past the "CREATE TABLE" bit. */ + while( 1 ){ + int t = 0; + iOff += sqlite3GetToken(&zSql[iOff], &t); + if( t==TK_LP ) break; + if( t==TK_ILLEGAL ){ + sqlite3_result_error_code(ctx, SQLITE_CORRUPT_BKPT); + return SQLITE_ERROR; + } + } + + *piOff = iOff; + return SQLITE_OK; +} + +/* +** Internal SQL function sqlite3_drop_constraint(): Given an input +** CREATE TABLE statement, return a revised CREATE TABLE statement +** with a constraint removed. Two forms, depending on the datatype +** of argv[2]: +** +** sqlite_drop_constraint(SQL, INT) -- Omit NOT NULL from the INT-th column +** sqlite_drop_constraint(SQL, TEXT) -- OMIT constraint with name TEXT +** +** In the first case, the left-most column is 0. +*/ +static void dropConstraintFunc( + sqlite3_context *ctx, + int NotUsed, + sqlite3_value **argv +){ + const u8 *zSql = sqlite3_value_text(argv[0]); + const u8 *zCons = 0; + int iNotNull = -1; + int ii; + int iOff = 0; + int iStart = 0; + int iEnd = 0; + char *zNew = 0; + int t = 0; + sqlite3 *db; + UNUSED_PARAMETER(NotUsed); + + if( zSql==0 ) return; + + /* Jump past the "CREATE TABLE" bit. */ + if( skipCreateTable(ctx, zSql, &iOff) ) return; + + if( sqlite3_value_type(argv[1])==SQLITE_INTEGER ){ + iNotNull = sqlite3_value_int(argv[1]); + }else{ + zCons = sqlite3_value_text(argv[1]); + } + + /* Search for the named constraint within column definitions. */ + for(ii=0; iEnd==0; ii++){ + + /* Now parse the column or table constraint definition. Search + ** for the token CONSTRAINT if this is a DROP CONSTRAINT command, or + ** NOT in the right column if this is a DROP NOT NULL. */ + while( 1 ){ + iStart = iOff; + iOff += getConstraintToken(&zSql[iOff], &t); + if( t==TK_CONSTRAINT && (zCons || iNotNull==ii) ){ + /* Check if this is the constraint we are searching for. */ + int nTok = 0; + int cmp = 1; + + /* Skip past any whitespace. */ + iOff += getWhitespace(&zSql[iOff]); + + /* Compare the next token - which may be quoted - with the name of + ** the constraint being dropped. */ + nTok = getConstraintToken(&zSql[iOff], &t); + if( zCons ){ + if( quotedCompare(ctx, t, &zSql[iOff], nTok, zCons, &cmp) ) return; + } + iOff += nTok; + + /* The next token is usually the first token of the constraint + ** definition. This is enough to tell the type of the constraint - + ** TK_NOT means it is a NOT NULL, TK_CHECK a CHECK constraint etc. + ** + ** There is also the chance that the next token is TK_CONSTRAINT + ** (or TK_DEFAULT or TK_COLLATE), for example if a table has been + ** created as follows: + ** + ** CREATE TABLE t1(cols, CONSTRAINT one CONSTRAINT two NOT NULL); + ** + ** In this case, allow the "CONSTRAINT one" bit to be dropped by + ** this command if that is what is requested, or to advance to + ** the next iteration of the loop with &zSql[iOff] still pointing + ** to the CONSTRAINT keyword. */ + nTok = getConstraintToken(&zSql[iOff], &t); + if( t==TK_CONSTRAINT || t==TK_DEFAULT || t==TK_COLLATE + || t==TK_COMMA || t==TK_RP || t==TK_GENERATED || t==TK_AS + ){ + t = TK_CHECK; + }else{ + iOff += nTok; + iOff += getConstraint(&zSql[iOff]); + } + + if( cmp==0 || (iNotNull>=0 && t==TK_NOT) ){ + if( t!=TK_NOT && t!=TK_CHECK ){ + errorMPrintf(ctx, "constraint may not be dropped: %s", zCons); + return; + } + iEnd = iOff; + break; + } + + }else if( t==TK_NOT && iNotNull==ii ){ + iEnd = iOff + getConstraint(&zSql[iOff]); + break; + }else if( t==TK_RP || t==TK_ILLEGAL ){ + iEnd = -1; + break; + }else if( t==TK_COMMA ){ + break; + } + } + } + + /* If the constraint has not been found it is an error. */ + if( iEnd<=0 ){ + if( zCons ){ + errorMPrintf(ctx, "no such constraint: %s", zCons); + }else{ + /* SQLite follows postgres in that a DROP NOT NULL on a column that is + ** not NOT NULL is not an error. So just return the original SQL here. */ + sqlite3_result_text(ctx, (const char*)zSql, -1, SQLITE_TRANSIENT); + } + }else{ + + /* Figure out if an extra space should be inserted after the constraint + ** is removed. And if an additional comma preceding the constraint + ** should be removed. */ + const char *zSpace = " "; + iEnd += getWhitespace(&zSql[iEnd]); + sqlite3GetToken(&zSql[iEnd], &t); + if( t==TK_RP || t==TK_COMMA ){ + zSpace = ""; + if( zSql[iStart-1]==',' ) iStart--; + } + + db = sqlite3_context_db_handle(ctx); + zNew = sqlite3MPrintf(db, "%.*s%s%s", iStart, zSql, zSpace, &zSql[iEnd]); + sqlite3_result_text(ctx, zNew, -1, SQLITE_DYNAMIC); + } +} + +/* +** Internal SQL function: +** +** sqlite_add_constraint(SQL, CONSTRAINT-TEXT, ICOL) +** +** SQL is a CREATE TABLE statement. Return a modified version of +** SQL that adds CONSTRAINT-TEXT at the end of the ICOL-th column +** definition. (The left-most column defintion is 0.) +*/ +static void addConstraintFunc( + sqlite3_context *ctx, + int NotUsed, + sqlite3_value **argv +){ + const u8 *zSql = sqlite3_value_text(argv[0]); + const char *zCons = (const char*)sqlite3_value_text(argv[1]); + int iCol = sqlite3_value_int(argv[2]); + int iOff = 0; + int ii; + char *zNew = 0; + int t = 0; + sqlite3 *db; + UNUSED_PARAMETER(NotUsed); + + if( skipCreateTable(ctx, zSql, &iOff) ) return; + + for(ii=0; ii<=iCol || (iCol<0 && t!=TK_RP); ii++){ + iOff += getConstraintToken(&zSql[iOff], &t); + while( 1 ){ + int nTok = getConstraintToken(&zSql[iOff], &t); + if( t==TK_COMMA || t==TK_RP ) break; + if( t==TK_ILLEGAL ){ + sqlite3_result_error_code(ctx, SQLITE_CORRUPT_BKPT); + return; + } + iOff += nTok; + } + } + + iOff += getWhitespace(&zSql[iOff]); + + db = sqlite3_context_db_handle(ctx); + if( iCol<0 ){ + zNew = sqlite3MPrintf(db, "%.*s, %s%s", iOff, zSql, zCons, &zSql[iOff]); + }else{ + zNew = sqlite3MPrintf(db, "%.*s %s%s", iOff, zSql, zCons, &zSql[iOff]); + } + sqlite3_result_text(ctx, zNew, -1, SQLITE_DYNAMIC); +} + +/* +** Find a column named pCol in table pTab. If successful, set output +** parameter *piCol to the index of the column in the table and return +** SQLITE_OK. Otherwise, set *piCol to -1 and return an SQLite error +** code. +*/ +static int alterFindCol(Parse *pParse, Table *pTab, Token *pCol, int *piCol){ + sqlite3 *db = pParse->db; + char *zName = sqlite3NameFromToken(db, pCol); + int rc = SQLITE_NOMEM; + int iCol = -1; + + if( zName ){ + iCol = sqlite3ColumnIndex(pTab, zName); + if( iCol<0 ){ + sqlite3ErrorMsg(pParse, "no such column: %s", zName); + rc = SQLITE_ERROR; + }else{ + rc = SQLITE_OK; + } + } + +#ifndef SQLITE_OMIT_AUTHORIZATION + if( rc==SQLITE_OK ){ + const char *zDb = db->aDb[sqlite3SchemaToIndex(db, pTab->pSchema)].zDbSName; + const char *zCol = pTab->aCol[iCol].zCnName; + if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, zCol) ){ + pTab = 0; + } + } +#endif + + sqlite3DbFree(db, zName); + *piCol = iCol; + return rc; +} + + +/* +** Find the table named by the first entry in source list pSrc. If successful, +** return a pointer to the Table structure and set output variable (*pzDb) +** to point to the name of the database containin the table (i.e. "main", +** "temp" or the name of an attached database). +** +** If the table cannot be located, return NULL. The value of the two output +** parameters is undefined in this case. +*/ +static Table *alterFindTable( + Parse *pParse, /* Parsing context */ + SrcList *pSrc, /* Name of the table to look for */ + int *piDb, /* OUT: write the iDb here */ + const char **pzDb, /* OUT: write name of schema here */ + int bAuth /* Do ALTER TABLE authorization checks if true */ +){ + sqlite3 *db = pParse->db; + Table *pTab = 0; + assert( sqlite3BtreeHoldsAllMutexes(db) ); + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); + if( pTab ){ + int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + *pzDb = db->aDb[iDb].zDbSName; + *piDb = iDb; + + if( SQLITE_OK!=isRealTable(pParse, pTab, 2) + || SQLITE_OK!=isAlterableTable(pParse, pTab) + ){ + pTab = 0; + } + } +#ifndef SQLITE_OMIT_AUTHORIZATION + if( pTab && bAuth ){ + if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, *pzDb, pTab->zName, 0) ){ + pTab = 0; + } + } +#endif + sqlite3SrcListDelete(db, pSrc); + return pTab; +} + +/* +** Generate bytecode for one of: +** +** (1) ALTER TABLE pSrc DROP CONSTRAINT pCons +** (2) ALTER TABLE pSrc ALTER pCol DROP NOT NULL +** +** One of pCons and pCol must be NULL and the other non-null. +*/ +void sqlite3AlterDropConstraint( + Parse *pParse, /* Parsing context */ + SrcList *pSrc, /* The table being altered */ + Token *pCons, /* Name of the constraint to drop */ + Token *pCol /* Name of the column from which to remove the NOT NULL */ +){ + sqlite3 *db = pParse->db; + Table *pTab = 0; + int iDb = 0; + const char *zDb = 0; + char *zArg = 0; + + assert( (pCol==0)!=(pCons==0) ); + assert( pSrc->nSrc==1 ); + pTab = alterFindTable(pParse, pSrc, &iDb, &zDb, pCons!=0); + if( !pTab ) return; + + if( pCons ){ + zArg = sqlite3MPrintf(db, "%.*Q", pCons->n, pCons->z); + }else{ + int iCol; + if( alterFindCol(pParse, pTab, pCol, &iCol) ) return; + zArg = sqlite3MPrintf(db, "%d", iCol); + } + + /* Edit the SQL for the named table. */ + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE " SET " + "sql = sqlite_drop_constraint(sql, %s) " + "WHERE type='table' AND tbl_name=%Q COLLATE nocase" + , zDb, zArg, pTab->zName + ); + sqlite3DbFree(db, zArg); + + /* Finally, reload the database schema. */ + renameReloadSchema(pParse, iDb, INITFLAG_AlterDropCons); +} + +/* +** The implementation of SQL function sqlite_fail(MSG). This takes a single +** argument, and returns it as an error message with the error code set to +** SQLITE_CONSTRAINT. +*/ +static void failConstraintFunc( + sqlite3_context *ctx, + int NotUsed, + sqlite3_value **argv +){ + const char *zText = (const char*)sqlite3_value_text(argv[0]); + int err = sqlite3_value_int(argv[1]); + (void)NotUsed; + sqlite3_result_error(ctx, zText, -1); + sqlite3_result_error_code(ctx, err); +} + +/* +** Buffer pCons, which is nCons bytes in size, contains the text of a +** NOT NULL or CHECK constraint that will be inserted into a CREATE TABLE +** statement. If successful, this function returns the size of the buffer in +** bytes not including any trailing whitespace or "--" style comments. Or, +** if an OOM occurs, it returns 0 and sets db->mallocFailed to true. +** +** C-style comments at the end are preserved. "--" style comments are +** removed because the comment terminator might be \000, and we are about +** to insert the pCons[] text into the middle of a larger string, and that +** will have the effect of removing the comment terminator and messing up +** the syntax. +*/ +static int alterRtrimConstraint( + sqlite3 *db, /* used to record OOM error */ + const char *pCons, /* Buffer containing constraint */ + int nCons /* Size of pCons in bytes */ +){ + u8 *zTmp = (u8*)sqlite3MPrintf(db, "%.*s", nCons, pCons); + int iOff = 0; + int iEnd = 0; + + if( zTmp==0 ) return 0; + + while( 1 ){ + int t = 0; + int nToken = sqlite3GetToken(&zTmp[iOff], &t); + if( t==TK_ILLEGAL ) break; + if( t!=TK_SPACE && (t!=TK_COMMENT || zTmp[iOff]!='-') ){ + iEnd = iOff+nToken; + } + iOff += nToken; + } + + sqlite3DbFree(db, zTmp); + return iEnd; +} + +/* +** Prepare a statement of the form: +** +** ALTER TABLE pSrc ALTER pCol SET NOT NULL +*/ +void sqlite3AlterSetNotNull( + Parse *pParse, /* Parsing context */ + SrcList *pSrc, /* Name of the table being altered */ + Token *pCol, /* Name of the column to add a NOT NULL constraint to */ + Token *pFirst /* The NOT token of the NOT NULL constraint text */ +){ + Table *pTab = 0; + int iCol = 0; + int iDb = 0; + const char *zDb = 0; + const char *pCons = 0; + int nCons = 0; + + /* Look up the table being altered. */ + assert( pSrc->nSrc==1 ); + pTab = alterFindTable(pParse, pSrc, &iDb, &zDb, 0); + if( !pTab ) return; + + /* Find the column being altered. */ + if( alterFindCol(pParse, pTab, pCol, &iCol) ){ + return; + } + + /* Find the length in bytes of the constraint definition */ + pCons = pFirst->z; + nCons = alterRtrimConstraint(pParse->db, pCons, pParse->sLastToken.z - pCons); + + /* Search for a constraint violation. Throw an exception if one is found. */ + sqlite3NestedParse(pParse, + "SELECT sqlite_fail('constraint failed', %d) " + "FROM %Q.%Q AS x WHERE x.%.*s IS NULL", + SQLITE_CONSTRAINT, zDb, pTab->zName, (int)pCol->n, pCol->z + ); + + /* Edit the SQL for the named table. */ + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE " SET " + "sql = sqlite_add_constraint(sqlite_drop_constraint(sql, %d), %.*Q, %d) " + "WHERE type='table' AND tbl_name=%Q COLLATE nocase" + , zDb, iCol, nCons, pCons, iCol, pTab->zName + ); + + /* Finally, reload the database schema. */ + renameReloadSchema(pParse, iDb, INITFLAG_AlterDropCons); +} + +/* +** Implementation of internal SQL function: +** +** sqlite_find_constraint(SQL, CONSTRAINT-NAME) +** +** This function returns true if the SQL passed as the first argument is a +** CREATE TABLE that contains a constraint with the name CONSTRAINT-NAME, +** or false otherwise. +*/ +static void findConstraintFunc( + sqlite3_context *ctx, + int NotUsed, + sqlite3_value **argv +){ + const u8 *zSql = 0; + const u8 *zCons = 0; + int iOff = 0; + int t = 0; + + (void)NotUsed; + zSql = sqlite3_value_text(argv[0]); + zCons = sqlite3_value_text(argv[1]); + + if( zSql==0 || zCons==0 ) return; + while( t!=TK_LP && t!=TK_ILLEGAL ){ + iOff += sqlite3GetToken(&zSql[iOff], &t); + } + + while( 1 ){ + iOff += getConstraintToken(&zSql[iOff], &t); + if( t==TK_CONSTRAINT ){ + int nTok = 0; + int cmp = 0; + iOff += getWhitespace(&zSql[iOff]); + nTok = getConstraintToken(&zSql[iOff], &t); + if( quotedCompare(ctx, t, &zSql[iOff], nTok, zCons, &cmp) ) return; + if( cmp==0 ){ + sqlite3_result_int(ctx, 1); + return; + } + }else if( t==TK_ILLEGAL ){ + break; + } + } + + sqlite3_result_int(ctx, 0); +} + +/* +** Generate bytecode to implement: +** +** ALTER TABLE pSrc ADD [CONSTRAINT pName] CHECK(pExpr) +** +** Any "ON CONFLICT" text that occurs after the "CHECK(...)", up +** until pParse->sLastToken, is included as part of the new constraint. +*/ +void sqlite3AlterAddConstraint( + Parse *pParse, /* Parse context */ + SrcList *pSrc, /* Table to add constraint to */ + Token *pFirst, /* First token of new constraint */ + Token *pName, /* Name of new constraint. NULL if name omitted. */ + const char *pExpr, /* Text of CHECK expression */ + int nExpr /* Size of pExpr in bytes */ +){ + Table *pTab = 0; /* Table identified by pSrc */ + int iDb = 0; /* Which schema does pTab live in */ + const char *zDb = 0; /* Name of the schema in which pTab lives */ + const char *pCons = 0; /* Text of the constraint */ + int nCons; /* Bytes of text to use from pCons[] */ + + /* Look up the table being altered. */ + assert( pSrc->nSrc==1 ); + pTab = alterFindTable(pParse, pSrc, &iDb, &zDb, 1); + if( !pTab ) return; + + /* If this new constraint has a name, check that it is not a duplicate of + ** an existing constraint. It is an error if it is. */ + if( pName ){ + char *zName = sqlite3NameFromToken(pParse->db, pName); + + sqlite3NestedParse(pParse, + "SELECT sqlite_fail('constraint %q already exists', %d) " + "FROM \"%w\"." LEGACY_SCHEMA_TABLE " " + "WHERE type='table' AND tbl_name=%Q COLLATE nocase " + "AND sqlite_find_constraint(sql, %Q)", + zName, SQLITE_ERROR, zDb, pTab->zName, zName + ); + sqlite3DbFree(pParse->db, zName); + } + + /* Search for a constraint violation. Throw an exception if one is found. */ + sqlite3NestedParse(pParse, + "SELECT sqlite_fail('constraint failed', %d) " + "FROM %Q.%Q WHERE (%.*s) IS NOT TRUE", + SQLITE_CONSTRAINT, zDb, pTab->zName, nExpr, pExpr + ); + + /* Edit the SQL for the named table. */ + pCons = pFirst->z; + nCons = alterRtrimConstraint(pParse->db, pCons, pParse->sLastToken.z - pCons); + + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE " SET " + "sql = sqlite_add_constraint(sql, %.*Q, -1) " + "WHERE type='table' AND tbl_name=%Q COLLATE nocase" + , zDb, nCons, pCons, pTab->zName + ); + + /* Finally, reload the database schema. */ + renameReloadSchema(pParse, iDb, INITFLAG_AlterDropCons); +} + /* ** Register built-in functions used to help implement ALTER TABLE */ @@ -2323,6 +3046,10 @@ void sqlite3AlterFunctions(void){ INTERNAL_FUNCTION(sqlite_rename_test, 7, renameTableTest), INTERNAL_FUNCTION(sqlite_drop_column, 3, dropColumnFunc), INTERNAL_FUNCTION(sqlite_rename_quotefix,2, renameQuotefixFunc), + INTERNAL_FUNCTION(sqlite_drop_constraint,2, dropConstraintFunc), + INTERNAL_FUNCTION(sqlite_fail, 2, failConstraintFunc), + INTERNAL_FUNCTION(sqlite_add_constraint, 3, addConstraintFunc), + INTERNAL_FUNCTION(sqlite_find_constraint,2, findConstraintFunc), }; sqlite3InsertBuiltinFuncs(aAlterTableFuncs, ArraySize(aAlterTableFuncs)); } diff --git a/src/attach.c b/src/attach.c index 085e1b0ec..f27c1e6be 100644 --- a/src/attach.c +++ b/src/attach.c @@ -596,7 +596,7 @@ int sqlite3FixTriggerStep( if( sqlite3WalkSelect(&pFix->w, pStep->pSelect) || sqlite3WalkExpr(&pFix->w, pStep->pWhere) || sqlite3WalkExprList(&pFix->w, pStep->pExprList) - || sqlite3FixSrcList(pFix, pStep->pFrom) + || sqlite3FixSrcList(pFix, pStep->pSrc) ){ return 1; } diff --git a/src/auth.c b/src/auth.c index 9ec2e7d04..1088f844a 100644 --- a/src/auth.c +++ b/src/auth.c @@ -78,7 +78,7 @@ int sqlite3_set_authorizer( sqlite3_mutex_enter(db->mutex); db->xAuth = (sqlite3_xauth)xAuth; db->pAuthArg = pArg; - if( db->xAuth ) sqlite3ExpirePreparedStatements(db, 1); + sqlite3ExpirePreparedStatements(db, 1); sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } diff --git a/src/btree.c b/src/btree.c index a931b0d12..7e73c7fd7 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3673,6 +3673,30 @@ static SQLITE_NOINLINE int btreeBeginTrans( } #endif +#ifdef SQLITE_EXPERIMENTAL_PRAGMA_20251114 + /* If both a read and write transaction will be opened by this call, + ** then issue a file-control as if the following pragma command had + ** been evaluated: + ** + ** PRAGMA experimental_pragma_20251114 = 1|2 + ** + ** where the RHS is "1" if wrflag is 1 (RESERVED lock), or "2" if wrflag + ** is 2 (EXCLUSIVE lock). Ignore any result or error returned by the VFS. + ** + ** WARNING: This code will likely remain part of SQLite only temporarily - + ** it exists to allow users to experiment with certain types of blocking + ** locks in custom VFS implementations. It MAY BE REMOVED AT ANY TIME. */ + if( pBt->pPage1==0 && wrflag ){ + sqlite3_file *fd = sqlite3PagerFile(pPager); + char *aFcntl[3] = {0,0,0}; + aFcntl[1] = "experimental_pragma_20251114"; + assert( wrflag==1 || wrflag==2 ); + aFcntl[2] = (wrflag==1 ? "1" : "2"); + sqlite3OsFileControlHint(fd, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); + sqlite3_free(aFcntl[0]); + } +#endif + /* Call lockBtree() until either pBt->pPage1 is populated or ** lockBtree() returns something other than SQLITE_OK. lockBtree() ** may return SQLITE_OK but leave pBt->pPage1 set to 0 if after @@ -5120,7 +5144,7 @@ static int accessPayload( getCellInfo(pCur); aPayload = pCur->info.pPayload; - assert( offset+amt <= pCur->info.nPayload ); + assert( (u64)offset+(u64)amt <= (u64)pCur->info.nPayload ); assert( aPayload > pPage->aData ); if( (uptr)(aPayload - pPage->aData) > (pBt->usableSize - pCur->info.nLocal) ){ @@ -5677,7 +5701,7 @@ int sqlite3BtreeIsEmpty(BtCursor *pCur, int *pRes){ assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - if( pCur->eState==CURSOR_VALID ){ + if( NEVER(pCur->eState==CURSOR_VALID) ){ *pRes = 0; return SQLITE_OK; } @@ -9758,7 +9782,7 @@ int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ }while( rc==SQLITE_OK && nOut>0 ); if( rc==SQLITE_OK && nRem>0 && ALWAYS(pPgnoOut) ){ - Pgno pgnoNew; + Pgno pgnoNew = 0; /* Prevent harmless static-analyzer warning */ MemPage *pNew = 0; rc = allocateBtreePage(pBt, &pNew, &pgnoNew, 0, 0); put4byte(pPgnoOut, pgnoNew); diff --git a/src/build.c b/src/build.c index de890c2e9..c45194145 100644 --- a/src/build.c +++ b/src/build.c @@ -486,6 +486,7 @@ Table *sqlite3LocateTableItem( const char *zDb; if( p->fg.fixedSchema ){ int iDb = sqlite3SchemaToIndex(pParse->db, p->u4.pSchema); + assert( iDb>=0 && iDbdb->nDb ); zDb = pParse->db->aDb[iDb].zDbSName; }else{ assert( !p->fg.isSubquery ); @@ -2059,8 +2060,8 @@ void sqlite3ChangeCookie(Parse *pParse, int iDb){ ** The estimate is conservative. It might be larger that what is ** really needed. */ -static int identLength(const char *z){ - int n; +static i64 identLength(const char *z){ + i64 n; for(n=0; *z; n++, z++){ if( *z=='"' ){ n++; } } @@ -2570,13 +2571,14 @@ void sqlite3MarkAllShadowTablesOf(sqlite3 *db, Table *pTab){ ** restored to its original value prior to this routine returning. */ int sqlite3ShadowTableName(sqlite3 *db, const char *zName){ - char *zTail; /* Pointer to the last "_" in zName */ + const char *zTail; /* Pointer to the last "_" in zName */ Table *pTab; /* Table that zName is a shadow of */ + char *zCopy; zTail = strrchr(zName, '_'); if( zTail==0 ) return 0; - *zTail = 0; - pTab = sqlite3FindTable(db, zName, 0); - *zTail = '_'; + zCopy = sqlite3DbStrNDup(db, zName, (int)(zTail-zName)); + pTab = zCopy ? sqlite3FindTable(db, zCopy, 0) : 0; + sqlite3DbFree(db, zCopy); if( pTab==0 ) return 0; if( !IsVirtual(pTab) ) return 0; return sqlite3IsShadowTableOf(db, pTab, zName); @@ -2729,6 +2731,7 @@ void sqlite3EndTable( convertToWithoutRowidTable(pParse, p); } iDb = sqlite3SchemaToIndex(db, p->pSchema); + assert( iDb>=0 && iDb<=db->nDb ); #ifndef SQLITE_OMIT_CHECK /* Resolve names in all CHECK constraint expressions. @@ -3024,6 +3027,7 @@ void sqlite3CreateView( sqlite3TwoPartName(pParse, pName1, pName2, &pName); iDb = sqlite3SchemaToIndex(db, p->pSchema); + assert( iDb>=0 && iDbnDb ); sqlite3FixInit(&sFix, pParse, iDb, "view", pName); if( sqlite3FixSelect(&sFix, pSelect) ) goto create_view_fail; @@ -4620,6 +4624,7 @@ void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists){ goto exit_drop_index; } iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); + assert( iDb>=0 && iDbnDb ); #ifndef SQLITE_OMIT_AUTHORIZATION { int code = SQLITE_DROP_INDEX; diff --git a/src/carray.c b/src/carray.c index 154d107dd..ff0691a85 100644 --- a/src/carray.c +++ b/src/carray.c @@ -79,6 +79,7 @@ struct carray_bind { int nData; /* Number of elements */ int mFlags; /* Control flags */ void (*xDel)(void*); /* Destructor for aData */ + void *pDel; /* Alternative argument to xDel() */ }; @@ -411,7 +412,7 @@ static sqlite3_module carrayModule = { static void carrayBindDel(void *pPtr){ carray_bind *p = (carray_bind*)pPtr; if( p->xDel!=SQLITE_STATIC ){ - p->xDel(p->aData); + p->xDel(p->pDel); } sqlite3_free(p); } @@ -419,14 +420,26 @@ static void carrayBindDel(void *pPtr){ /* ** Invoke this interface in order to bind to the single-argument ** version of CARRAY(). +** +** pStmt The prepared statement to which to bind +** idx The index of the parameter of pStmt to which to bind +** aData The data to be bound +** nData The number of elements in aData +** mFlags One of SQLITE_CARRAY_xxxx indicating datatype of aData +** xDestroy Destructor for pDestroy or aData if pDestroy==NULL. +** pDestroy Invoke xDestroy on this pointer if not NULL +** +** The destructor is called pDestroy if pDestroy!=NULL, or against +** aData if pDestroy==NULL. */ -SQLITE_API int sqlite3_carray_bind( +SQLITE_API int sqlite3_carray_bind_v2( sqlite3_stmt *pStmt, int idx, void *aData, int nData, int mFlags, - void (*xDestroy)(void*) + void (*xDestroy)(void*), + void *pDestroy ){ carray_bind *pNew = 0; int i; @@ -503,20 +516,38 @@ SQLITE_API int sqlite3_carray_bind( memcpy(pNew->aData, aData, sz); } pNew->xDel = sqlite3_free; + pNew->pDel = pNew->aData; }else{ pNew->aData = aData; pNew->xDel = xDestroy; + pNew->pDel = pDestroy; } return sqlite3_bind_pointer(pStmt, idx, pNew, "carray-bind", carrayBindDel); carray_bind_error: if( xDestroy!=SQLITE_STATIC && xDestroy!=SQLITE_TRANSIENT ){ - xDestroy(aData); + xDestroy(pDestroy); } sqlite3_free(pNew); return rc; } +/* +** Invoke this interface in order to bind to the single-argument +** version of CARRAY(). Same as sqlite3_carray_bind_v2() with the +** pDestroy parameter set to NULL. +*/ +SQLITE_API int sqlite3_carray_bind( + sqlite3_stmt *pStmt, + int idx, + void *aData, + int nData, + int mFlags, + void (*xDestroy)(void*) +){ + return sqlite3_carray_bind_v2(pStmt,idx,aData,nData,mFlags,xDestroy,aData); +} + /* ** Invoke this routine to register the carray() function. */ diff --git a/src/date.c b/src/date.c index 5e7ae6f1f..58a7ce544 100644 --- a/src/date.c +++ b/src/date.c @@ -429,7 +429,7 @@ static int parseDateOrTime( return 0; }else if( sqlite3StrICmp(zDate,"now")==0 && sqlite3NotPureFunc(context) ){ return setDateTimeToCurrent(context, p); - }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8)>0 ){ + }else if( sqlite3AtoF(zDate, &r)>0 ){ setRawDateNumber(p, r); return 0; }else if( (sqlite3StrICmp(zDate,"subsec")==0 @@ -875,7 +875,7 @@ static int parseModifier( ** date is already on the appropriate weekday, this is a no-op. */ if( sqlite3_strnicmp(z, "weekday ", 8)==0 - && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8)>0 + && sqlite3AtoF(&z[8], &r)>0 && r>=0.0 && r<7.0 && (n=(int)r)==r ){ sqlite3_int64 Z; computeYMD_HMS(p); @@ -946,9 +946,11 @@ static int parseModifier( case '8': case '9': { double rRounder; - int i; + int i, rx; int Y,M,D,h,m,x; const char *z2 = z; + char *zCopy; + sqlite3 *db = sqlite3_context_db_handle(pCtx); char z0 = z[0]; for(n=1; z[n]; n++){ if( z[n]==':' ) break; @@ -958,7 +960,11 @@ static int parseModifier( if( n==6 && getDigits(&z[1], "50f", &Y)==1 ) break; } } - if( sqlite3AtoF(z, &r, n, SQLITE_UTF8)<=0 ){ + zCopy = sqlite3DbStrNDup(db, z, n); + if( zCopy==0 ) break; + rx = sqlite3AtoF(zCopy, &r)<=0; + sqlite3DbFree(db, zCopy); + if( rx ){ assert( rc==1 ); break; } @@ -1778,7 +1784,7 @@ static void datedebugFunc( char *zJson; zJson = sqlite3_mprintf( "{iJD:%lld,Y:%d,M:%d,D:%d,h:%d,m:%d,tz:%d," - "s:%.3f,validJD:%d,validYMS:%d,validHMS:%d," + "s:%.3f,validJD:%d,validYMD:%d,validHMS:%d," "nFloor:%d,rawS:%d,isError:%d,useSubsec:%d," "isUtc:%d,isLocal:%d}", x.iJD, x.Y, x.M, x.D, x.h, x.m, x.tz, diff --git a/src/delete.c b/src/delete.c index 8fac7c2f3..d4d2337c7 100644 --- a/src/delete.c +++ b/src/delete.c @@ -86,7 +86,7 @@ static int vtabIsReadOnly(Parse *pParse, Table *pTab){ ** * Only allow DELETE, INSERT, or UPDATE of non-SQLITE_VTAB_INNOCUOUS ** virtual tables if PRAGMA trusted_schema=ON. */ - if( pParse->pToplevel!=0 + if( (pParse->pToplevel!=0 || (pParse->prepFlags & SQLITE_PREPARE_FROM_DDL)) && pTab->u.vtab.p->eVtabRisk > ((pParse->db->flags & SQLITE_TrustedSchema)!=0) ){ @@ -924,7 +924,6 @@ void sqlite3GenerateRowIndexDelete( &iPartIdxLabel, pPrior, r1); sqlite3VdbeAddOp3(v, OP_IdxDelete, iIdxCur+i, r1, pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); - sqlite3VdbeChangeP5(v, 1); /* Cause IdxDelete to error if no entry found */ sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); pPrior = pIdx; } diff --git a/src/expr.c b/src/expr.c index fdc05366c..d486e48e3 100644 --- a/src/expr.c +++ b/src/expr.c @@ -935,34 +935,22 @@ Expr *sqlite3ExprAlloc( int dequote /* True to dequote */ ){ Expr *pNew; - int nExtra = 0; - int iValue = 0; + int nExtra = pToken ? pToken->n+1 : 0; assert( db!=0 ); - if( pToken ){ - if( op!=TK_INTEGER || pToken->z==0 - || sqlite3GetInt32(pToken->z, &iValue)==0 ){ - nExtra = pToken->n+1; /* tag-20240227-a */ - assert( iValue>=0 ); - } - } pNew = sqlite3DbMallocRawNN(db, sizeof(Expr)+nExtra); if( pNew ){ memset(pNew, 0, sizeof(Expr)); pNew->op = (u8)op; pNew->iAgg = -1; - if( pToken ){ - if( nExtra==0 ){ - pNew->flags |= EP_IntValue|EP_Leaf|(iValue?EP_IsTrue:EP_IsFalse); - pNew->u.iValue = iValue; - }else{ - pNew->u.zToken = (char*)&pNew[1]; - assert( pToken->z!=0 || pToken->n==0 ); - if( pToken->n ) memcpy(pNew->u.zToken, pToken->z, pToken->n); - pNew->u.zToken[pToken->n] = 0; - if( dequote && sqlite3Isquote(pNew->u.zToken[0]) ){ - sqlite3DequoteExpr(pNew); - } + if( nExtra ){ + assert( pToken!=0 ); + pNew->u.zToken = (char*)&pNew[1]; + assert( pToken->z!=0 || pToken->n==0 ); + if( pToken->n ) memcpy(pNew->u.zToken, pToken->z, pToken->n); + pNew->u.zToken[pToken->n] = 0; + if( dequote && sqlite3Isquote(pNew->u.zToken[0]) ){ + sqlite3DequoteExpr(pNew); } } #if SQLITE_MAX_EXPR_DEPTH>0 @@ -987,6 +975,24 @@ Expr *sqlite3Expr( return sqlite3ExprAlloc(db, op, &x, 0); } +/* +** Allocate an expression for a 32-bit signed integer literal. +*/ +Expr *sqlite3ExprInt32(sqlite3 *db, int iVal){ + Expr *pNew = sqlite3DbMallocRawNN(db, sizeof(Expr)); + if( pNew ){ + memset(pNew, 0, sizeof(Expr)); + pNew->op = TK_INTEGER; + pNew->iAgg = -1; + pNew->flags = EP_IntValue|EP_Leaf|(iVal?EP_IsTrue:EP_IsFalse); + pNew->u.iValue = iVal; +#if SQLITE_MAX_EXPR_DEPTH>0 + pNew->nHeight = 1; +#endif + } + return pNew; +} + /* ** Attach subtrees pLeft and pRight to the Expr node pRoot. ** @@ -1149,7 +1155,7 @@ Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ ){ sqlite3ExprDeferredDelete(pParse, pLeft); sqlite3ExprDeferredDelete(pParse, pRight); - return sqlite3Expr(db, TK_INTEGER, "0"); + return sqlite3ExprInt32(db, 0); }else{ return sqlite3PExpr(pParse, TK_AND, pLeft, pRight); } @@ -1274,7 +1280,9 @@ void sqlite3ExprFunctionUsable( ){ assert( !IN_RENAME_OBJECT ); assert( (pDef->funcFlags & (SQLITE_FUNC_DIRECT|SQLITE_FUNC_UNSAFE))!=0 ); - if( ExprHasProperty(pExpr, EP_FromDDL) ){ + if( ExprHasProperty(pExpr, EP_FromDDL) + || pParse->prepFlags & SQLITE_PREPARE_FROM_DDL + ){ if( (pDef->funcFlags & SQLITE_FUNC_DIRECT)!=0 || (pParse->db->flags & SQLITE_TrustedSchema)==0 ){ @@ -1970,9 +1978,7 @@ Select *sqlite3SelectDup(sqlite3 *db, const Select *pDup, int flags){ pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); pNew->iLimit = 0; pNew->iOffset = 0; - pNew->selFlags = p->selFlags & ~(u32)SF_UsesEphemeral; - pNew->addrOpenEphm[0] = -1; - pNew->addrOpenEphm[1] = -1; + pNew->selFlags = p->selFlags; pNew->nSelectRow = p->nSelectRow; pNew->pWith = sqlite3WithDup(db, p->pWith); #ifndef SQLITE_OMIT_WINDOWFUNC @@ -2624,7 +2630,7 @@ static int exprIsConst(Parse *pParse, Expr *p, int initFlag){ /* ** Walk an expression tree. Return non-zero if the expression is constant -** and 0 if it involves variables or function calls. +** or return zero if the expression involves variables or function calls. ** ** For the purposes of this function, a double-quoted string (ex: "abc") ** is considered a variable but a single-quoted string (ex: 'abc') is @@ -3414,6 +3420,7 @@ int sqlite3FindInIndex( */ u32 savedNQueryLoop = pParse->nQueryLoop; int rMayHaveNull = 0; + int bloomOk = (inFlags & IN_INDEX_MEMBERSHIP)!=0; eType = IN_INDEX_EPH; if( inFlags & IN_INDEX_LOOP ){ pParse->nQueryLoop = 0; @@ -3421,7 +3428,13 @@ int sqlite3FindInIndex( *prRhsHasNull = rMayHaveNull = ++pParse->nMem; } assert( pX->op==TK_IN ); - sqlite3CodeRhsOfIN(pParse, pX, iTab); + if( !bloomOk + && ExprUseXSelect(pX) + && (pX->x.pSelect->selFlags & SF_ClonedRhsIn)!=0 + ){ + bloomOk = 1; + } + sqlite3CodeRhsOfIN(pParse, pX, iTab, bloomOk); if( rMayHaveNull ){ sqlite3SetHasNullFlag(v, iTab, rMayHaveNull); } @@ -3579,7 +3592,8 @@ static int findCompatibleInRhsSubrtn( void sqlite3CodeRhsOfIN( Parse *pParse, /* Parsing context */ Expr *pExpr, /* The IN operator */ - int iTab /* Use this cursor number */ + int iTab, /* Use this cursor number */ + int allowBloom /* True to allow the use of a Bloom filter */ ){ int addrOnce = 0; /* Address of the OP_Once instruction at top */ int addr; /* Address of OP_OpenEphemeral instruction */ @@ -3701,7 +3715,10 @@ void sqlite3CodeRhsOfIN( sqlite3SelectDestInit(&dest, SRT_Set, iTab); dest.zAffSdst = exprINAffinity(pParse, pExpr); pSelect->iLimit = 0; - if( addrOnce && OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){ + if( addrOnce + && allowBloom + && OptimizationEnabled(pParse->db, SQLITE_BloomFilter) + ){ int regBloom = ++pParse->nMem; addrBloom = sqlite3VdbeAddOp2(v, OP_Blob, 10000, regBloom); VdbeComment((v, "Bloom filter")); @@ -3922,7 +3939,7 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ || (pLeft->u.iValue!=1 && pLeft->u.iValue!=0) ){ sqlite3 *db = pParse->db; - pLimit = sqlite3Expr(db, TK_INTEGER, "0"); + pLimit = sqlite3ExprInt32(db, 0); if( pLimit ){ pLimit->affExpr = SQLITE_AFF_NUMERIC; pLimit = sqlite3PExpr(pParse, TK_NE, @@ -3933,7 +3950,7 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ } }else{ /* If there is no pre-existing limit add a limit of 1 */ - pLimit = sqlite3Expr(pParse->db, TK_INTEGER, "1"); + pLimit = sqlite3ExprInt32(pParse->db, 1); pSel->pLimit = sqlite3PExpr(pParse, TK_LIMIT, pLimit, 0); } pSel->iLimit = 0; @@ -4194,8 +4211,9 @@ static void sqlite3ExprCodeIN( if( ExprHasProperty(pExpr, EP_Subrtn) ){ const VdbeOp *pOp = sqlite3VdbeGetOp(v, pExpr->y.sub.iAddr); assert( pOp->opcode==OP_Once || pParse->nErr ); - if( pOp->opcode==OP_Once && pOp->p3>0 ){ /* tag-202407032019 */ - assert( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ); + if( pOp->p3>0 ){ /* tag-202407032019 */ + assert( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) + || pParse->nErr ); sqlite3VdbeAddOp4Int(v, OP_Filter, pOp->p3, destIfFalse, rLhs, nVector); VdbeCoverage(v); } @@ -4285,7 +4303,7 @@ static void sqlite3ExprCodeIN( static void codeReal(Vdbe *v, const char *z, int negateFlag, int iMem){ if( ALWAYS(z!=0) ){ double value; - sqlite3AtoF(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); + sqlite3AtoF(z, &value); assert( !sqlite3IsNaN(value) ); /* The new AtoF never returns NaN */ if( negateFlag ) value = -value; sqlite3VdbeAddOp4Dup8(v, OP_Real, 0, iMem, 0, (u8*)&value, P4_REAL); @@ -7386,7 +7404,10 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ if( pIEpr==0 ) break; if( NEVER(!ExprUseYTab(pExpr)) ) break; for(i=0; inSrc; i++){ - if( pSrcList->a[0].iCursor==pIEpr->iDataCur ) break; + if( pSrcList->a[i].iCursor==pIEpr->iDataCur ){ + testcase( i>0 ); + break; + } } if( i>=pSrcList->nSrc ) break; if( NEVER(pExpr->pAggInfo!=0) ) break; /* Resolved by outer context */ diff --git a/src/fkey.c b/src/fkey.c index f1117a884..59edd8810 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -688,6 +688,7 @@ FKey *sqlite3FkReferences(Table *pTab){ static void fkTriggerDelete(sqlite3 *dbMem, Trigger *p){ if( p ){ TriggerStep *pStep = p->step_list; + sqlite3SrcListDelete(dbMem, pStep->pSrc); sqlite3ExprDelete(dbMem, pStep->pWhere); sqlite3ExprListDelete(dbMem, pStep->pExprList); sqlite3SelectDelete(dbMem, pStep->pSelect); @@ -907,6 +908,7 @@ void sqlite3FkCheck( if( !IsOrdinaryTable(pTab) ) return; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDb>=00 && iDbnDb ); zDb = db->aDb[iDb].zDbSName; /* Loop through all the foreign key constraints for which pTab is the @@ -1354,14 +1356,14 @@ static Trigger *fkActionTrigger( pTrigger = (Trigger *)sqlite3DbMallocZero(db, sizeof(Trigger) + /* struct Trigger */ - sizeof(TriggerStep) + /* Single step in trigger program */ - nFrom + 1 /* Space for pStep->zTarget */ + sizeof(TriggerStep) /* Single step in trigger program */ ); if( pTrigger ){ pStep = pTrigger->step_list = (TriggerStep *)&pTrigger[1]; - pStep->zTarget = (char *)&pStep[1]; - memcpy((char *)pStep->zTarget, zFrom, nFrom); - + pStep->pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); + if( pStep->pSrc ){ + pStep->pSrc->a[0].zName = sqlite3DbStrNDup(db, zFrom, nFrom); + } pStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); pStep->pExprList = sqlite3ExprListDup(db, pList, EXPRDUP_REDUCE); pStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); diff --git a/src/func.c b/src/func.c index 6dac7195a..d9d8f59ad 100644 --- a/src/func.c +++ b/src/func.c @@ -466,7 +466,7 @@ static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ sqlite3_result_error_nomem(context); return; } - sqlite3AtoF(zBuf, &r, sqlite3Strlen30(zBuf), SQLITE_UTF8); + sqlite3AtoF(zBuf, &r); sqlite3_free(zBuf); } sqlite3_result_double(context, r); @@ -1104,7 +1104,7 @@ void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue, int bEscape){ sqlite3_str_appendf(pStr, "%!0.15g", r1); zVal = sqlite3_str_value(pStr); if( zVal ){ - sqlite3AtoF(zVal, &r2, pStr->nChar, SQLITE_UTF8); + sqlite3AtoF(zVal, &r2); if( r1!=r2 ){ sqlite3_str_reset(pStr); sqlite3_str_appendf(pStr, "%!0.20e", r1); @@ -1201,7 +1201,7 @@ static void unistrFunc( } i = j = 0; while( ifuncFlags |= flags; pDef->funcFlags &= ~SQLITE_FUNC_UNSAFE; } diff --git a/src/hwtime.h b/src/hwtime.h index f808fa40e..cf637edaf 100644 --- a/src/hwtime.h +++ b/src/hwtime.h @@ -16,17 +16,19 @@ #ifndef SQLITE_HWTIME_H #define SQLITE_HWTIME_H -/* -** The following routine only works on Pentium-class (or newer) processors. -** It uses the RDTSC opcode to read the cycle count value out of the -** processor and returns that value. This can be used for high-res -** profiling. -*/ -#if !defined(__STRICT_ANSI__) && \ - (defined(__GNUC__) || defined(_MSC_VER)) && \ - (defined(i386) || defined(__i386__) || defined(_M_IX86)) +#if defined(_MSC_VER) && defined(_WIN32) + + #include "windows.h" + #include - #if defined(__GNUC__) + __inline sqlite3_uint64 sqlite3Hwtime(void){ + LARGE_INTEGER tm; + QueryPerformanceCounter(&tm); + return (sqlite3_uint64)tm.QuadPart; + } + +#elif !defined(__STRICT_ANSI__) && defined(__GNUC__) && \ + (defined(i386) || defined(__i386__) || defined(_M_IX86)) __inline__ sqlite_uint64 sqlite3Hwtime(void){ unsigned int lo, hi; @@ -34,17 +36,6 @@ return (sqlite_uint64)hi << 32 | lo; } - #elif defined(_MSC_VER) - - __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ - __asm { - rdtsc - ret ; return value at EDX:EAX - } - } - - #endif - #elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) __inline__ sqlite_uint64 sqlite3Hwtime(void){ @@ -52,6 +43,14 @@ __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); return (sqlite_uint64)hi << 32 | lo; } + +#elif !defined(__STRICT_ANSI__) && defined(__GNUC__) && defined(__aarch64__) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + sqlite3_uint64 cnt; + __asm__ __volatile__ ("mrs %0, cntvct_el0" : "=r" (cnt)); + return cnt; + } #elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) diff --git a/src/json.c b/src/json.c index d0d3c53a2..795d3ed73 100644 --- a/src/json.c +++ b/src/json.c @@ -328,7 +328,10 @@ struct JsonString { #define JSON_SQL 0x02 /* Result is always SQL */ #define JSON_ABPATH 0x03 /* Allow abbreviated JSON path specs */ #define JSON_ISSET 0x04 /* json_set(), not json_insert() */ -#define JSON_BLOB 0x08 /* Use the BLOB output format */ +#define JSON_AINS 0x08 /* json_array_insert(), not json_insert() */ +#define JSON_BLOB 0x10 /* Use the BLOB output format */ + +#define JSON_INSERT_TYPE(X) (((X)&0xC)>>2) /* A parsed JSON value. Lifecycle: @@ -374,6 +377,7 @@ struct JsonParse { #define JEDIT_REPL 2 /* Overwrite if exists */ #define JEDIT_INS 3 /* Insert if not exists */ #define JEDIT_SET 4 /* Insert or overwrite */ +#define JEDIT_AINS 5 /* array_insert() */ /* ** Maximum nesting depth of JSON for this implementation. @@ -2870,7 +2874,8 @@ static int jsonLabelCompare( */ #define JSON_LOOKUP_ERROR 0xffffffff #define JSON_LOOKUP_NOTFOUND 0xfffffffe -#define JSON_LOOKUP_PATHERROR 0xfffffffd +#define JSON_LOOKUP_NOTARRAY 0xfffffffd +#define JSON_LOOKUP_PATHERROR 0xfffffffc #define JSON_LOOKUP_ISERROR(x) ((x)>=JSON_LOOKUP_PATHERROR) /* Forward declaration */ @@ -2899,7 +2904,7 @@ static u32 jsonLookupStep(JsonParse*,u32,const char*,u32); static u32 jsonCreateEditSubstructure( JsonParse *pParse, /* The original JSONB that is being edited */ JsonParse *pIns, /* Populate this with the blob data to insert */ - const char *zTail /* Tail of the path that determins substructure */ + const char *zTail /* Tail of the path that determines substructure */ ){ static const u8 emptyObject[] = { JSONB_ARRAY, JSONB_OBJECT }; int rc; @@ -2934,9 +2939,9 @@ static u32 jsonCreateEditSubstructure( ** Return one of the JSON_LOOKUP error codes if problems are seen. ** ** This routine will also modify the blob. If pParse->eEdit is one of -** JEDIT_DEL, JEDIT_REPL, JEDIT_INS, or JEDIT_SET, then changes might be -** made to the selected value. If an edit is performed, then the return -** value does not necessarily point to the select element. If an edit +** JEDIT_DEL, JEDIT_REPL, JEDIT_INS, JEDIT_SET, or JEDIT_AINS, then changes +** might be made to the selected value. If an edit is performed, then the +** return value does not necessarily point to the select element. If an edit ** is performed, the return value is only useful for detecting error ** conditions. */ @@ -2962,6 +2967,13 @@ static u32 jsonLookupStep( jsonBlobEdit(pParse, iRoot, sz, 0, 0); }else if( pParse->eEdit==JEDIT_INS ){ /* Already exists, so json_insert() is a no-op */ + }else if( pParse->eEdit==JEDIT_AINS ){ + /* json_array_insert() */ + if( zPath[-1]!=']' ){ + return JSON_LOOKUP_NOTARRAY; + }else{ + jsonBlobEdit(pParse, iRoot, 0, pParse->aIns, pParse->nIns); + } }else{ /* json_set() or json_replace() */ jsonBlobEdit(pParse, iRoot, sz, pParse->aIns, pParse->nIns); @@ -3033,6 +3045,10 @@ static u32 jsonLookupStep( JsonParse ix; /* Header of the label to be inserted */ testcase( pParse->eEdit==JEDIT_INS ); testcase( pParse->eEdit==JEDIT_SET ); + testcase( pParse->eEdit==JEDIT_AINS ); + if( pParse->eEdit==JEDIT_AINS && sqlite3_strglob("*]",&zPath[i])!=0 ){ + return JSON_LOOKUP_NOTARRAY; + } memset(&ix, 0, sizeof(ix)); ix.db = pParse->db; jsonBlobAppendNode(&ix, rawKey?JSONB_TEXTRAW:JSONB_TEXT5, nKey, 0); @@ -3060,28 +3076,32 @@ static u32 jsonLookupStep( return rc; } }else if( zPath[0]=='[' ){ + u64 kk = 0; x = pParse->aBlob[iRoot] & 0x0f; if( x!=JSONB_ARRAY ) return JSON_LOOKUP_NOTFOUND; n = jsonbPayloadSize(pParse, iRoot, &sz); - k = 0; i = 1; while( sqlite3Isdigit(zPath[i]) ){ - k = k*10 + zPath[i] - '0'; + if( kk<0xffffffff ) kk = kk*10 + zPath[i] - '0'; + /* ^^^^^^^^^^--- Allow kk to be bigger than any JSON array so that + ** we get NOTFOUND instead of PATHERROR, without overflowing kk. */ i++; } if( i<2 || zPath[i]!=']' ){ if( zPath[1]=='#' ){ - k = jsonbArrayCount(pParse, iRoot); + kk = jsonbArrayCount(pParse, iRoot); i = 2; if( zPath[2]=='-' && sqlite3Isdigit(zPath[3]) ){ - unsigned int nn = 0; + u64 nn = 0; i = 3; do{ - nn = nn*10 + zPath[i] - '0'; + if( nn<0xffffffff ) nn = nn*10 + zPath[i] - '0'; + /* ^^^^^^^^^^--- Allow nn to be bigger than any JSON array to + ** get NOTFOUND instead of PATHERROR, without overflowing nn. */ i++; }while( sqlite3Isdigit(zPath[i]) ); - if( nn>k ) return JSON_LOOKUP_NOTFOUND; - k -= nn; + if( nn>kk ) return JSON_LOOKUP_NOTFOUND; + kk -= nn; } if( zPath[i]!=']' ){ return JSON_LOOKUP_PATHERROR; @@ -3093,21 +3113,22 @@ static u32 jsonLookupStep( j = iRoot+n; iEnd = j+sz; while( jdelta ) jsonAfterEditSizeAdjust(pParse, iRoot); return rc; } - k--; + kk--; n = jsonbPayloadSize(pParse, j, &sz); if( n==0 ) return JSON_LOOKUP_ERROR; j += n+sz; } if( j>iEnd ) return JSON_LOOKUP_ERROR; - if( k>0 ) return JSON_LOOKUP_NOTFOUND; + if( kk>0 ) return JSON_LOOKUP_NOTFOUND; if( pParse->eEdit>=JEDIT_INS ){ JsonParse v; testcase( pParse->eEdit==JEDIT_INS ); + testcase( pParse->eEdit==JEDIT_AINS ); testcase( pParse->eEdit==JEDIT_SET ); rc = jsonCreateEditSubstructure(pParse, &v, &zPath[i+1]); if( !JSON_LOOKUP_ISERROR(rc) @@ -3245,7 +3266,7 @@ static void jsonReturnFromBlob( to_double: z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); if( z==0 ) goto returnfromblob_oom; - rc = sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); + rc = sqlite3AtoF(z, &r); sqlite3DbFree(db, z); if( rc<=0 ) goto returnfromblob_malformed; sqlite3_result_double(pCtx, r); @@ -3432,9 +3453,15 @@ static int jsonFunctionArgToBlob( */ static char *jsonBadPathError( sqlite3_context *ctx, /* The function call containing the error */ - const char *zPath /* The path with the problem */ + const char *zPath, /* The path with the problem */ + int rc /* Maybe JSON_LOOKUP_NOTARRAY */ ){ - char *zMsg = sqlite3_mprintf("bad JSON path: %Q", zPath); + char *zMsg; + if( rc==(int)JSON_LOOKUP_NOTARRAY ){ + zMsg = sqlite3_mprintf("not an array element: %Q", zPath); + }else{ + zMsg = sqlite3_mprintf("bad JSON path: %Q", zPath); + } if( ctx==0 ) return zMsg; if( zMsg ){ sqlite3_result_error(ctx, zMsg, -1); @@ -3451,13 +3478,13 @@ static char *jsonBadPathError( ** and return the result. ** ** The specific operation is determined by eEdit, which can be one -** of JEDIT_INS, JEDIT_REPL, or JEDIT_SET. +** of JEDIT_INS, JEDIT_REPL, JEDIT_SET, or JEDIT_AINS. */ static void jsonInsertIntoBlob( sqlite3_context *ctx, int argc, sqlite3_value **argv, - int eEdit /* JEDIT_INS, JEDIT_REPL, or JEDIT_SET */ + int eEdit /* JEDIT_INS, JEDIT_REPL, JEDIT_SET, JEDIT_AINS */ ){ int i; u32 rc = 0; @@ -3509,7 +3536,7 @@ static void jsonInsertIntoBlob( if( rc==JSON_LOOKUP_ERROR ){ sqlite3_result_error(ctx, "malformed JSON", -1); }else{ - jsonBadPathError(ctx, zPath); + jsonBadPathError(ctx, zPath, rc); } return; } @@ -3951,7 +3978,7 @@ static void jsonArrayLengthFunc( if( i==JSON_LOOKUP_NOTFOUND ){ /* no-op */ }else if( i==JSON_LOOKUP_PATHERROR ){ - jsonBadPathError(ctx, zPath); + jsonBadPathError(ctx, zPath, 0); }else{ sqlite3_result_error(ctx, "malformed JSON", -1); } @@ -4056,7 +4083,7 @@ static void jsonExtractFunc( j = jsonLookupStep(p, 0, jx.zBuf, 0); jsonStringReset(&jx); }else{ - jsonBadPathError(ctx, zPath); + jsonBadPathError(ctx, zPath, 0); goto json_extract_error; } if( jnBlob ){ @@ -4091,7 +4118,7 @@ static void jsonExtractFunc( sqlite3_result_error(ctx, "malformed JSON", -1); goto json_extract_error; }else{ - jsonBadPathError(ctx, zPath); + jsonBadPathError(ctx, zPath, 0); goto json_extract_error; } } @@ -4420,7 +4447,7 @@ static void jsonRemoveFunc( if( rc==JSON_LOOKUP_NOTFOUND ){ continue; /* No-op */ }else if( rc==JSON_LOOKUP_PATHERROR ){ - jsonBadPathError(ctx, zPath); + jsonBadPathError(ctx, zPath, rc); }else{ sqlite3_result_error(ctx, "malformed JSON", -1); } @@ -4432,7 +4459,7 @@ static void jsonRemoveFunc( return; json_remove_patherror: - jsonBadPathError(ctx, zPath); + jsonBadPathError(ctx, zPath, 0); json_remove_done: jsonParseFree(p); @@ -4476,16 +4503,18 @@ static void jsonSetFunc( int argc, sqlite3_value **argv ){ - int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); - int bIsSet = (flags&JSON_ISSET)!=0; + int eInsType = JSON_INSERT_TYPE(flags); + static const char *azInsType[] = { "insert", "set", "array_insert" }; + static const u8 aEditType[] = { JEDIT_INS, JEDIT_SET, JEDIT_AINS }; if( argc<1 ) return; + assert( eInsType>=0 && eInsType<=2 ); if( (argc&1)==0 ) { - jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); + jsonWrongNumArgs(ctx, azInsType[eInsType]); return; } - jsonInsertIntoBlob(ctx, argc, argv, bIsSet ? JEDIT_SET : JEDIT_INS); + jsonInsertIntoBlob(ctx, argc, argv, aEditType[eInsType]); } /* @@ -4510,7 +4539,7 @@ static void jsonTypeFunc( zPath = (const char*)sqlite3_value_text(argv[1]); if( zPath==0 ) goto json_type_done; if( zPath[0]!='$' ){ - jsonBadPathError(ctx, zPath); + jsonBadPathError(ctx, zPath, 0); goto json_type_done; } i = jsonLookupStep(p, 0, zPath+1, 0); @@ -4518,7 +4547,7 @@ static void jsonTypeFunc( if( i==JSON_LOOKUP_NOTFOUND ){ /* no-op */ }else if( i==JSON_LOOKUP_PATHERROR ){ - jsonBadPathError(ctx, zPath); + jsonBadPathError(ctx, zPath, 0); }else{ sqlite3_result_error(ctx, "malformed JSON", -1); } @@ -4774,12 +4803,11 @@ static void jsonArrayStep( } static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ JsonString *pStr; + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); if( pStr ){ - int flags; pStr->pCtx = ctx; jsonAppendChar(pStr, ']'); - flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); if( pStr->eErr ){ jsonReturnString(pStr, 0, 0); return; @@ -4800,6 +4828,9 @@ static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); jsonStringTrimOneChar(pStr); } + }else if( flags & JSON_BLOB ){ + static const u8 emptyArray = 0x0b; + sqlite3_result_blob(ctx, &emptyArray, 1, SQLITE_STATIC); }else{ sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC); } @@ -4896,12 +4927,11 @@ static void jsonObjectStep( } static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ JsonString *pStr; + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); if( pStr ){ - int flags; jsonAppendChar(pStr, '}'); pStr->pCtx = ctx; - flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); if( pStr->eErr ){ jsonReturnString(pStr, 0, 0); return; @@ -4922,6 +4952,9 @@ static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); jsonStringTrimOneChar(pStr); } + }else if( flags & JSON_BLOB ){ + static const unsigned char emptyObject = 0x0c; + sqlite3_result_blob(ctx, &emptyObject, 1, SQLITE_STATIC); }else{ sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC); } @@ -5422,7 +5455,7 @@ static int jsonEachFilter( if( zRoot==0 ) return SQLITE_OK; if( zRoot[0]!='$' ){ sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot); + cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot, 0); jsonEachCursorReset(p); return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; } @@ -5440,7 +5473,7 @@ static int jsonEachFilter( return SQLITE_OK; } sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot); + cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot, 0); jsonEachCursorReset(p); return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; } @@ -5530,6 +5563,8 @@ void sqlite3RegisterJsonFunctions(void){ JFUNCTION(jsonb, 1,1,0, 0,1,0, jsonRemoveFunc), JFUNCTION(json_array, -1,0,1, 1,0,0, jsonArrayFunc), JFUNCTION(jsonb_array, -1,0,1, 1,1,0, jsonArrayFunc), + JFUNCTION(json_array_insert, -1,1,1, 1,0,JSON_AINS, jsonSetFunc), + JFUNCTION(jsonb_array_insert,-1,1,0, 1,1,JSON_AINS, jsonSetFunc), JFUNCTION(json_array_length, 1,1,0, 0,0,0, jsonArrayLengthFunc), JFUNCTION(json_array_length, 2,1,0, 0,0,0, jsonArrayLengthFunc), JFUNCTION(json_error_position,1,1,0, 0,0,0, jsonErrorFunc), diff --git a/src/loadext.c b/src/loadext.c index c5177715e..55325edd7 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -522,7 +522,17 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_setlk_timeout, /* Version 3.51.0 and later */ sqlite3_set_errmsg, - sqlite3_db_status64 + sqlite3_db_status64, + /* Version 3.52.0 and later */ + sqlite3_str_truncate, + sqlite3_str_free, +#ifdef SQLITE_ENABLE_CARRAY + sqlite3_carray_bind, + sqlite3_carray_bind_v2 +#else + 0, + 0 +#endif }; /* True if x is the directory separator character @@ -624,33 +634,42 @@ static int sqlite3LoadExtension( ** entry point name "sqlite3_extension_init" was not found, then ** construct an entry point name "sqlite3_X_init" where the X is ** replaced by the lowercase value of every ASCII alphabetic - ** character in the filename after the last "/" upto the first ".", - ** and eliding the first three characters if they are "lib". + ** character in the filename after the last "/" up to the first ".", + ** and skipping the first three characters if they are "lib". ** Examples: ** ** /usr/local/lib/libExample5.4.3.so ==> sqlite3_example_init ** C:/lib/mathfuncs.dll ==> sqlite3_mathfuncs_init + ** + ** If that still finds no entry point, repeat a second time but this + ** time include both alphabetic and numeric characters up to the first + ** ".". Example: + ** + ** /usr/local/lib/libExample5.4.3.so ==> sqlite3_example5_init */ if( xInit==0 && zProc==0 ){ int iFile, iEntry, c; int ncFile = sqlite3Strlen30(zFile); + int cnt = 0; zAltEntry = sqlite3_malloc64(ncFile+30); if( zAltEntry==0 ){ sqlite3OsDlClose(pVfs, handle); return SQLITE_NOMEM_BKPT; } - memcpy(zAltEntry, "sqlite3_", 8); - for(iFile=ncFile-1; iFile>=0 && !DirSep(zFile[iFile]); iFile--){} - iFile++; - if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; - for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ - if( sqlite3Isalpha(c) ){ - zAltEntry[iEntry++] = (char)sqlite3UpperToLower[(unsigned)c]; + do{ + memcpy(zAltEntry, "sqlite3_", 8); + for(iFile=ncFile-1; iFile>=0 && !DirSep(zFile[iFile]); iFile--){} + iFile++; + if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; + for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ + if( sqlite3Isalpha(c) || (cnt && sqlite3Isdigit(c)) ){ + zAltEntry[iEntry++] = (char)sqlite3UpperToLower[(unsigned)c]; + } } - } - memcpy(zAltEntry+iEntry, "_init", 6); - zEntry = zAltEntry; - xInit = (sqlite3_loadext_entry)sqlite3OsDlSym(pVfs, handle, zEntry); + memcpy(zAltEntry+iEntry, "_init", 6); + zEntry = zAltEntry; + xInit = (sqlite3_loadext_entry)sqlite3OsDlSym(pVfs, handle, zEntry); + }while( xInit==0 && (++cnt)<2 ); } if( xInit==0 ){ if( pzErrMsg ){ diff --git a/src/main.c b/src/main.c index 6efe538d4..b44ac8dca 100644 --- a/src/main.c +++ b/src/main.c @@ -971,6 +971,14 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){ rc = setupLookaside(db, pBuf, sz, cnt); break; } + case SQLITE_DBCONFIG_FP_DIGITS: { + int nIn = va_arg(ap, int); + int *pOut = va_arg(ap, int*); + if( nIn>3 && nIn<24 ) db->nFpDigit = (u8)nIn; + if( pOut ) *pOut = db->nFpDigit; + rc = SQLITE_OK; + break; + } default: { static const struct { int op; /* The opcode */ @@ -2526,6 +2534,9 @@ void *sqlite3_wal_hook( sqlite3_mutex_leave(db->mutex); return pRet; #else + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(xCallback); + UNUSED_PARAMETER(pArg); return 0; #endif } @@ -2541,6 +2552,11 @@ int sqlite3_wal_checkpoint_v2( int *pnCkpt /* OUT: Total number of frames checkpointed */ ){ #ifdef SQLITE_OMIT_WAL + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(zDb); + UNUSED_PARAMETER(eMode); + UNUSED_PARAMETER(pnLog); + UNUSED_PARAMETER(pnCkpt); return SQLITE_OK; #else int rc; /* Return code */ @@ -2554,11 +2570,12 @@ int sqlite3_wal_checkpoint_v2( if( pnLog ) *pnLog = -1; if( pnCkpt ) *pnCkpt = -1; + assert( SQLITE_CHECKPOINT_NOOP==-1 ); assert( SQLITE_CHECKPOINT_PASSIVE==0 ); assert( SQLITE_CHECKPOINT_FULL==1 ); assert( SQLITE_CHECKPOINT_RESTART==2 ); assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); - if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ + if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint ** mode: */ return SQLITE_MISUSE_BKPT; @@ -2922,6 +2939,7 @@ static const int aHardLimit[] = { SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */ SQLITE_MAX_TRIGGER_DEPTH, SQLITE_MAX_WORKER_THREADS, + SQLITE_MAX_PARSER_DEPTH, }; /* @@ -2936,6 +2954,9 @@ static const int aHardLimit[] = { #if SQLITE_MAX_SQL_LENGTH>SQLITE_MAX_LENGTH # error SQLITE_MAX_SQL_LENGTH must not be greater than SQLITE_MAX_LENGTH #endif +#if SQLITE_MAX_SQL_LENGTH>2147482624 /* 1024 less than 2^31 */ +# error SQLITE_MAX_SQL_LENGTH must not be greater than 2147482624 +#endif #if SQLITE_MAX_COMPOUND_SELECT<2 # error SQLITE_MAX_COMPOUND_SELECT must be at least 2 #endif @@ -2991,6 +3012,7 @@ int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ assert( aHardLimit[SQLITE_LIMIT_SQL_LENGTH]==SQLITE_MAX_SQL_LENGTH ); assert( aHardLimit[SQLITE_LIMIT_COLUMN]==SQLITE_MAX_COLUMN ); assert( aHardLimit[SQLITE_LIMIT_EXPR_DEPTH]==SQLITE_MAX_EXPR_DEPTH ); + assert( aHardLimit[SQLITE_LIMIT_PARSER_DEPTH]==SQLITE_MAX_PARSER_DEPTH ); assert( aHardLimit[SQLITE_LIMIT_COMPOUND_SELECT]==SQLITE_MAX_COMPOUND_SELECT); assert( aHardLimit[SQLITE_LIMIT_VDBE_OP]==SQLITE_MAX_VDBE_OP ); assert( aHardLimit[SQLITE_LIMIT_FUNCTION_ARG]==SQLITE_MAX_FUNCTION_ARG ); @@ -3000,7 +3022,7 @@ int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS ); - assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-1) ); + assert( SQLITE_LIMIT_PARSER_DEPTH==(SQLITE_N_LIMIT-1) ); if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ @@ -3364,7 +3386,7 @@ static int openDatabase( db = sqlite3MallocZero( sizeof(sqlite3) ); if( db==0 ) goto opendb_out; if( isThreadsafe -#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS +#if defined(SQLITE_THREAD_MISUSE_WARNINGS) || sqlite3GlobalConfig.bCoreMutex #endif ){ @@ -3385,6 +3407,7 @@ static int openDatabase( db->aDb = db->aDbStatic; db->lookaside.bDisable = 1; db->lookaside.sz = 0; + db->nFpDigit = 17; assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); @@ -3830,6 +3853,12 @@ int sqlite3_collation_needed16( */ void *sqlite3_get_clientdata(sqlite3 *db, const char *zName){ DbClientData *p; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zName || !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); for(p=db->pDbData; p; p=p->pNext){ if( strcmp(p->zName, zName)==0 ){ @@ -4903,6 +4932,7 @@ const char *sqlite3_filename_journal(const char *zFilename){ } const char *sqlite3_filename_wal(const char *zFilename){ #ifdef SQLITE_OMIT_WAL + UNUSED_PARAMETER(zFilename); return 0; #else zFilename = sqlite3_filename_journal(zFilename); diff --git a/src/malloc.c b/src/malloc.c index 9a635e716..942934644 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -289,27 +289,6 @@ static void mallocWithAlarm(int n, void **pp){ *pp = p; } -/* -** Maximum size of any single memory allocation. -** -** This is not a limit on the total amount of memory used. This is -** a limit on the size parameter to sqlite3_malloc() and sqlite3_realloc(). -** -** The upper bound is slightly less than 2GiB: 0x7ffffeff == 2,147,483,391 -** This provides a 256-byte safety margin for defense against 32-bit -** signed integer overflow bugs when computing memory allocation sizes. -** Paranoid applications might want to reduce the maximum allocation size -** further for an even larger safety margin. 0x3fffffff or 0x0fffffff -** or even smaller would be reasonable upper bounds on the size of a memory -** allocations for most applications. -*/ -#ifndef SQLITE_MAX_ALLOCATION_SIZE -# define SQLITE_MAX_ALLOCATION_SIZE 2147483391 -#endif -#if SQLITE_MAX_ALLOCATION_SIZE>2147483391 -# error Maximum size for SQLITE_MAX_ALLOCATION_SIZE is 2147483391 -#endif - /* ** Allocate memory. This routine is like sqlite3_malloc() except that it ** assumes the memory subsystem has already been initialized. @@ -533,8 +512,7 @@ void *sqlite3Realloc(void *pOld, u64 nBytes){ sqlite3_free(pOld); /* IMP: R-26507-47431 */ return 0; } - if( nBytes>=0x7fffff00 ){ - /* The 0x7ffff00 limit term is explained in comments on sqlite3Malloc() */ + if( nBytes>SQLITE_MAX_ALLOCATION_SIZE ){ return 0; } nOld = sqlite3MallocSize(pOld); diff --git a/src/mutex.c b/src/mutex.c index 62e09cb4f..21b47e511 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -27,23 +27,28 @@ static SQLITE_WSD int mutexIsInit = 0; #ifndef SQLITE_MUTEX_OMIT -#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS +#ifdef SQLITE_THREAD_MISUSE_WARNINGS /* -** This block (enclosed by SQLITE_ENABLE_MULTITHREADED_CHECKS) contains +** This block (enclosed by SQLITE_THREAD_MISUSE_WARNINGS) contains ** the implementation of a wrapper around the system default mutex ** implementation (sqlite3DefaultMutex()). ** ** Most calls are passed directly through to the underlying default ** mutex implementation. Except, if a mutex is configured by calling ** sqlite3MutexWarnOnContention() on it, then if contention is ever -** encountered within xMutexEnter() a warning is emitted via sqlite3_log(). +** encountered within xMutexEnter() then a warning is emitted via +** sqlite3_log(). Furthermore, if SQLITE_THREAD_MISUSE_ABORT is +** defined then abort() is called after the sqlite3_log() warning. ** -** This type of mutex is used as the database handle mutex when testing -** apps that usually use SQLITE_CONFIG_MULTITHREAD mode. +** This type of mutex is used on the database handle mutex when testing +** apps that usually use SQLITE_CONFIG_MULTITHREAD mode. A failure +** indicates that the app ought to be using SQLITE_OPEN_FULLMUTEX or +** similar because it is trying to use the same database handle from +** two different connections at the same time. */ /* -** Type for all mutexes used when SQLITE_ENABLE_MULTITHREADED_CHECKS +** Type for all mutexes used when SQLITE_THREAD_MISUSE_WARNINGS ** is defined. Variable CheckMutex.mutex is a pointer to the real mutex ** allocated by the system mutex implementation. Variable iType is usually set ** to the type of mutex requested - SQLITE_MUTEX_RECURSIVE, SQLITE_MUTEX_FAST @@ -79,11 +84,12 @@ static int checkMutexNotheld(sqlite3_mutex *p){ */ static int checkMutexInit(void){ pGlobalMutexMethods = sqlite3DefaultMutex(); - return SQLITE_OK; + return pGlobalMutexMethods->xMutexInit(); } static int checkMutexEnd(void){ + int rc = pGlobalMutexMethods->xMutexEnd(); pGlobalMutexMethods = 0; - return SQLITE_OK; + return rc; } /* @@ -160,6 +166,9 @@ static void checkMutexEnter(sqlite3_mutex *p){ sqlite3_log(SQLITE_MISUSE, "illegal multi-threaded access to database connection" ); +#if SQLITE_THREAD_MISUSE_ABORT + abort(); +#endif } pGlobalMutexMethods->xMutexEnter(pCheck->mutex); } @@ -211,7 +220,7 @@ void sqlite3MutexWarnOnContention(sqlite3_mutex *p){ pCheck->iType = SQLITE_MUTEX_WARNONCONTENTION; } } -#endif /* ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS */ +#endif /* ifdef SQLITE_THREAD_MISUSE_WARNINGS */ /* ** Initialize the mutex system. @@ -228,7 +237,7 @@ int sqlite3MutexInit(void){ sqlite3_mutex_methods *pTo = &sqlite3GlobalConfig.mutex; if( sqlite3GlobalConfig.bCoreMutex ){ -#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS +#ifdef SQLITE_THREAD_MISUSE_WARNINGS pFrom = multiThreadedCheckMutex(); #else pFrom = sqlite3DefaultMutex(); diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 7eb5b50be..20e41b161 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -129,11 +129,7 @@ static int winMutexInit(void){ if( InterlockedCompareExchange(&winMutex_lock, 1, 0)==0 ){ int i; for(i=0; itrace = 1; #endif #endif -#if SQLITE_OS_WINRT - InitializeCriticalSectionEx(&p->mutex, 0, 0); -#else InitializeCriticalSection(&p->mutex); -#endif } break; } diff --git a/src/os_kv.c b/src/os_kv.c index c2d1f9b7a..1fd1c8e8c 100644 --- a/src/os_kv.c +++ b/src/os_kv.c @@ -21,7 +21,7 @@ ** Debugging logic */ -/* SQLITE_KV_TRACE() is used for tracing calls to kvstorage routines. */ +/* SQLITE_KV_TRACE() is used for tracing calls to kvrecord routines. */ #if 0 #define SQLITE_KV_TRACE(X) printf X #else @@ -35,7 +35,6 @@ #define SQLITE_KV_LOG(X) #endif - /* ** Forward declaration of objects used by this VFS implementation */ @@ -43,6 +42,11 @@ typedef struct KVVfsFile KVVfsFile; /* A single open file. There are only two files represented by this ** VFS - the database and the rollback journal. +** +** Maintenance reminder: if this struct changes in any way, the JSON +** rendering of its structure must be updated in +** sqlite3-wasm.c:sqlite3__wasm_enum_json(). There are no binary +** compatibility concerns, so it does not need an iVersion member. */ struct KVVfsFile { sqlite3_file base; /* IO methods */ @@ -92,7 +96,7 @@ static int kvvfsCurrentTime(sqlite3_vfs*, double*); static int kvvfsCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); static sqlite3_vfs sqlite3OsKvvfsObject = { - 1, /* iVersion */ + 2, /* iVersion */ sizeof(KVVfsFile), /* szOsFile */ 1024, /* mxPathname */ 0, /* pNext */ @@ -168,23 +172,37 @@ static sqlite3_io_methods kvvfs_jrnl_io_methods = { /* Forward declarations for the low-level storage engine */ -static int kvstorageWrite(const char*, const char *zKey, const char *zData); -static int kvstorageDelete(const char*, const char *zKey); -static int kvstorageRead(const char*, const char *zKey, char *zBuf, int nBuf); -#define KVSTORAGE_KEY_SZ 32 +#ifndef SQLITE_WASM +/* In WASM builds these are implemented in JS. */ +static int kvrecordWrite(const char*, const char *zKey, const char *zData); +static int kvrecordDelete(const char*, const char *zKey); +static int kvrecordRead(const char*, const char *zKey, char *zBuf, int nBuf); +#endif +#ifndef KVRECORD_KEY_SZ +#define KVRECORD_KEY_SZ 32 +#endif /* Expand the key name with an appropriate prefix and put the result ** in zKeyOut[]. The zKeyOut[] buffer is assumed to hold at least -** KVSTORAGE_KEY_SZ bytes. +** KVRECORD_KEY_SZ bytes. */ -static void kvstorageMakeKey( +static void kvrecordMakeKey( const char *zClass, const char *zKeyIn, char *zKeyOut ){ - sqlite3_snprintf(KVSTORAGE_KEY_SZ, zKeyOut, "kvvfs-%s-%s", zClass, zKeyIn); + assert( zKeyIn ); + assert( zKeyOut ); + assert( zClass ); + sqlite3_snprintf(KVRECORD_KEY_SZ, zKeyOut, "kvvfs-%s-%s", + zClass, zKeyIn); } +#ifndef SQLITE_WASM +/* In WASM builds do not define APIs which use fopen(), fwrite(), +** and the like because those APIs are a portability issue for +** WASM. +*/ /* Write content into a key. zClass is the particular namespace of the ** underlying key/value store to use - either "local" or "session". ** @@ -192,14 +210,14 @@ static void kvstorageMakeKey( ** ** Return the number of errors. */ -static int kvstorageWrite( +static int kvrecordWrite( const char *zClass, const char *zKey, const char *zData ){ FILE *fd; - char zXKey[KVSTORAGE_KEY_SZ]; - kvstorageMakeKey(zClass, zKey, zXKey); + char zXKey[KVRECORD_KEY_SZ]; + kvrecordMakeKey(zClass, zKey, zXKey); fd = fopen(zXKey, "wb"); if( fd ){ SQLITE_KV_TRACE(("KVVFS-WRITE %-15s (%d) %.50s%s\n", zXKey, @@ -217,9 +235,9 @@ static int kvstorageWrite( ** namespace given by zClass. If the key does not previously exist, ** this routine is a no-op. */ -static int kvstorageDelete(const char *zClass, const char *zKey){ - char zXKey[KVSTORAGE_KEY_SZ]; - kvstorageMakeKey(zClass, zKey, zXKey); +static int kvrecordDelete(const char *zClass, const char *zKey){ + char zXKey[KVRECORD_KEY_SZ]; + kvrecordMakeKey(zClass, zKey, zXKey); unlink(zXKey); SQLITE_KV_TRACE(("KVVFS-DELETE %-15s\n", zXKey)); return 0; @@ -240,7 +258,7 @@ static int kvstorageDelete(const char *zClass, const char *zKey){ ** zero-terminates zBuf at zBuf[0] and returns the size of the data ** without reading it. */ -static int kvstorageRead( +static int kvrecordRead( const char *zClass, const char *zKey, char *zBuf, @@ -248,8 +266,8 @@ static int kvstorageRead( ){ FILE *fd; struct stat buf; - char zXKey[KVSTORAGE_KEY_SZ]; - kvstorageMakeKey(zClass, zKey, zXKey); + char zXKey[KVRECORD_KEY_SZ]; + kvrecordMakeKey(zClass, zKey, zXKey); if( access(zXKey, R_OK)!=0 || stat(zXKey, &buf)!=0 || !S_ISREG(buf.st_mode) @@ -281,6 +299,8 @@ static int kvstorageRead( return (int)n; } } +#endif /* #ifndef SQLITE_WASM */ + /* ** An internal level of indirection which enables us to replace the @@ -288,17 +308,27 @@ static int kvstorageRead( ** Maintenance reminder: if this struct changes in any way, the JSON ** rendering of its structure must be updated in ** sqlite3-wasm.c:sqlite3__wasm_enum_json(). There are no binary -** compatibility concerns, so it does not need an iVersion -** member. +** compatibility concerns, so it does not need an iVersion member. */ typedef struct sqlite3_kvvfs_methods sqlite3_kvvfs_methods; struct sqlite3_kvvfs_methods { - int (*xRead)(const char *zClass, const char *zKey, char *zBuf, int nBuf); - int (*xWrite)(const char *zClass, const char *zKey, const char *zData); - int (*xDelete)(const char *zClass, const char *zKey); + int (*xRcrdRead)(const char*, const char *zKey, char *zBuf, int nBuf); + int (*xRcrdWrite)(const char*, const char *zKey, const char *zData); + int (*xRcrdDelete)(const char*, const char *zKey); const int nKeySize; + const int nBufferSize; +#ifndef SQLITE_WASM +# define MAYBE_CONST const +#else +# define MAYBE_CONST +#endif + MAYBE_CONST sqlite3_vfs * pVfs; + MAYBE_CONST sqlite3_io_methods *pIoDb; + MAYBE_CONST sqlite3_io_methods *pIoJrnl; +#undef MAYBE_CONST }; + /* ** This object holds the kvvfs I/O methods which may be swapped out ** for JavaScript-side implementations in WASM builds. In such builds @@ -313,10 +343,20 @@ struct sqlite3_kvvfs_methods { const #endif sqlite3_kvvfs_methods sqlite3KvvfsMethods = { -kvstorageRead, -kvstorageWrite, -kvstorageDelete, -KVSTORAGE_KEY_SZ +#ifndef SQLITE_WASM + .xRcrdRead = kvrecordRead, + .xRcrdWrite = kvrecordWrite, + .xRcrdDelete = kvrecordDelete, +#else + .xRcrdRead = 0, + .xRcrdWrite = 0, + .xRcrdDelete = 0, +#endif + .nKeySize = KVRECORD_KEY_SZ, + .nBufferSize = SQLITE_KVOS_SZ, + .pVfs = &sqlite3OsKvvfsObject, + .pIoDb = &kvvfs_db_io_methods, + .pIoJrnl = &kvvfs_jrnl_io_methods }; /****** Utility subroutines ************************************************/ @@ -343,7 +383,10 @@ KVSTORAGE_KEY_SZ ** of hexadecimal and base-26 numbers, it is always clear where ** one stops and the next begins. */ -static int kvvfsEncode(const char *aData, int nData, char *aOut){ +#ifndef SQLITE_WASM +static +#endif +int kvvfsEncode(const char *aData, int nData, char *aOut){ int i, j; const unsigned char *a = (const unsigned char*)aData; for(i=j=0; izClass, "sz", zData, sizeof(zData)-1); + sqlite3KvvfsMethods.xRcrdRead(pFile->zClass, "sz", zData, + sizeof(zData)-1); return strtoll(zData, 0, 0); } static int kvvfsWriteFileSize(KVVfsFile *pFile, sqlite3_int64 sz){ char zData[50]; sqlite3_snprintf(sizeof(zData), zData, "%lld", sz); - return sqlite3KvvfsMethods.xWrite(pFile->zClass, "sz", zData); + return sqlite3KvvfsMethods.xRcrdWrite(pFile->zClass, "sz", zData); } /****** sqlite3_io_methods methods ******************************************/ @@ -491,10 +539,13 @@ static int kvvfsWriteFileSize(KVVfsFile *pFile, sqlite3_int64 sz){ static int kvvfsClose(sqlite3_file *pProtoFile){ KVVfsFile *pFile = (KVVfsFile *)pProtoFile; - SQLITE_KV_LOG(("xClose %s %s\n", pFile->zClass, + SQLITE_KV_LOG(("xClose %s %s\n", pFile->zClass, pFile->isJournal ? "journal" : "db")); sqlite3_free(pFile->aJrnl); sqlite3_free(pFile->aData); +#ifdef SQLITE_WASM + memset(pFile, 0, sizeof(*pFile)); +#endif return SQLITE_OK; } @@ -503,24 +554,30 @@ static int kvvfsClose(sqlite3_file *pProtoFile){ */ static int kvvfsReadJrnl( sqlite3_file *pProtoFile, - void *zBuf, - int iAmt, + void *zBuf, + int iAmt, sqlite_int64 iOfst ){ KVVfsFile *pFile = (KVVfsFile*)pProtoFile; assert( pFile->isJournal ); SQLITE_KV_LOG(("xRead('%s-journal',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); if( pFile->aJrnl==0 ){ - int szTxt = kvstorageRead(pFile->zClass, "jrnl", 0, 0); + int rc; + int szTxt = sqlite3KvvfsMethods.xRcrdRead(pFile->zClass, "jrnl", + 0, 0); char *aTxt; if( szTxt<=4 ){ return SQLITE_IOERR; } aTxt = sqlite3_malloc64( szTxt+1 ); if( aTxt==0 ) return SQLITE_NOMEM; - kvstorageRead(pFile->zClass, "jrnl", aTxt, szTxt+1); - kvvfsDecodeJournal(pFile, aTxt, szTxt); + rc = sqlite3KvvfsMethods.xRcrdRead(pFile->zClass, "jrnl", + aTxt, szTxt+1); + if( rc>=0 ){ + kvvfsDecodeJournal(pFile, aTxt, szTxt); + } sqlite3_free(aTxt); + if( rc ) return rc; if( pFile->aJrnl==0 ) return SQLITE_IOERR; } if( iOfst+iAmt>pFile->nJrnl ){ @@ -535,8 +592,8 @@ static int kvvfsReadJrnl( */ static int kvvfsReadDb( sqlite3_file *pProtoFile, - void *zBuf, - int iAmt, + void *zBuf, + int iAmt, sqlite_int64 iOfst ){ KVVfsFile *pFile = (KVVfsFile*)pProtoFile; @@ -560,8 +617,8 @@ static int kvvfsReadDb( pgno = 1; } sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); - got = sqlite3KvvfsMethods.xRead(pFile->zClass, zKey, - aData, SQLITE_KVOS_SZ-1); + got = sqlite3KvvfsMethods.xRcrdRead(pFile->zClass, zKey, + aData, SQLITE_KVOS_SZ-1); if( got<0 ){ n = 0; }else{ @@ -593,8 +650,8 @@ static int kvvfsReadDb( */ static int kvvfsWriteJrnl( sqlite3_file *pProtoFile, - const void *zBuf, - int iAmt, + const void *zBuf, + int iAmt, sqlite_int64 iOfst ){ KVVfsFile *pFile = (KVVfsFile*)pProtoFile; @@ -621,14 +678,15 @@ static int kvvfsWriteJrnl( */ static int kvvfsWriteDb( sqlite3_file *pProtoFile, - const void *zBuf, - int iAmt, + const void *zBuf, + int iAmt, sqlite_int64 iOfst ){ KVVfsFile *pFile = (KVVfsFile*)pProtoFile; unsigned int pgno; char zKey[30]; char *aData = pFile->aData; + int rc; SQLITE_KV_LOG(("xWrite('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); assert( iAmt>=512 && iAmt<=65536 ); assert( (iAmt & (iAmt-1))==0 ); @@ -637,13 +695,13 @@ static int kvvfsWriteDb( pgno = 1 + iOfst/iAmt; sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); kvvfsEncode(zBuf, iAmt, aData); - if( sqlite3KvvfsMethods.xWrite(pFile->zClass, zKey, aData) ){ - return SQLITE_IOERR; - } - if( iOfst+iAmt > pFile->szDb ){ - pFile->szDb = iOfst + iAmt; + rc = sqlite3KvvfsMethods.xRcrdWrite(pFile->zClass, zKey, aData); + if( 0==rc ){ + if( iOfst+iAmt > pFile->szDb ){ + pFile->szDb = iOfst + iAmt; + } } - return SQLITE_OK; + return rc; } /* @@ -653,7 +711,7 @@ static int kvvfsTruncateJrnl(sqlite3_file *pProtoFile, sqlite_int64 size){ KVVfsFile *pFile = (KVVfsFile *)pProtoFile; SQLITE_KV_LOG(("xTruncate('%s-journal',%lld)\n", pFile->zClass, size)); assert( size==0 ); - sqlite3KvvfsMethods.xDelete(pFile->zClass, "jrnl"); + sqlite3KvvfsMethods.xRcrdDelete(pFile->zClass, "jrnl"); sqlite3_free(pFile->aJrnl); pFile->aJrnl = 0; pFile->nJrnl = 0; @@ -662,7 +720,7 @@ static int kvvfsTruncateJrnl(sqlite3_file *pProtoFile, sqlite_int64 size){ static int kvvfsTruncateDb(sqlite3_file *pProtoFile, sqlite_int64 size){ KVVfsFile *pFile = (KVVfsFile *)pProtoFile; if( pFile->szDb>size - && pFile->szPage>0 + && pFile->szPage>0 && (size % pFile->szPage)==0 ){ char zKey[50]; @@ -672,7 +730,7 @@ static int kvvfsTruncateDb(sqlite3_file *pProtoFile, sqlite_int64 size){ pgnoMax = 2 + pFile->szDb/pFile->szPage; while( pgno<=pgnoMax ){ sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); - sqlite3KvvfsMethods.xDelete(pFile->zClass, zKey); + sqlite3KvvfsMethods.xRcrdDelete(pFile->zClass, zKey); pgno++; } pFile->szDb = size; @@ -704,7 +762,7 @@ static int kvvfsSyncJrnl(sqlite3_file *pProtoFile, int flags){ }while( n>0 ); zOut[i++] = ' '; kvvfsEncode(pFile->aJrnl, pFile->nJrnl, &zOut[i]); - i = sqlite3KvvfsMethods.xWrite(pFile->zClass, "jrnl", zOut); + i = sqlite3KvvfsMethods.xRcrdWrite(pFile->zClass, "jrnl", zOut); sqlite3_free(zOut); return i ? SQLITE_IOERR : SQLITE_OK; } @@ -818,33 +876,32 @@ static int kvvfsOpen( KVVfsFile *pFile = (KVVfsFile*)pProtoFile; if( zName==0 ) zName = ""; SQLITE_KV_LOG(("xOpen(\"%s\")\n", zName)); - if( strcmp(zName, "local")==0 - || strcmp(zName, "session")==0 - ){ - pFile->isJournal = 0; - pFile->base.pMethods = &kvvfs_db_io_methods; - }else - if( strcmp(zName, "local-journal")==0 - || strcmp(zName, "session-journal")==0 - ){ + assert(!pFile->zClass); + assert(!pFile->aData); + assert(!pFile->aJrnl); + assert(!pFile->nJrnl); + assert(!pFile->base.pMethods); + pFile->szPage = -1; + pFile->szDb = -1; + if( 0==sqlite3_strglob("*-journal", zName) ){ pFile->isJournal = 1; pFile->base.pMethods = &kvvfs_jrnl_io_methods; + if( 0==strcmp("session-journal",zName) ){ + pFile->zClass = "session"; + }else if( 0==strcmp("local-journal",zName) ){ + pFile->zClass = "local"; + } }else{ - return SQLITE_CANTOPEN; + pFile->isJournal = 0; + pFile->base.pMethods = &kvvfs_db_io_methods; } - if( zName[0]=='s' ){ - pFile->zClass = "session"; - }else{ - pFile->zClass = "local"; + if( !pFile->zClass ){ + pFile->zClass = zName; } pFile->aData = sqlite3_malloc64(SQLITE_KVOS_SZ); if( pFile->aData==0 ){ return SQLITE_NOMEM; } - pFile->aJrnl = 0; - pFile->nJrnl = 0; - pFile->szPage = -1; - pFile->szDb = -1; return SQLITE_OK; } @@ -854,13 +911,17 @@ static int kvvfsOpen( ** returning. */ static int kvvfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + int rc /* The JS impl can fail with OOM in argument conversion */; if( strcmp(zPath, "local-journal")==0 ){ - sqlite3KvvfsMethods.xDelete("local", "jrnl"); + rc = sqlite3KvvfsMethods.xRcrdDelete("local", "jrnl"); }else if( strcmp(zPath, "session-journal")==0 ){ - sqlite3KvvfsMethods.xDelete("session", "jrnl"); + rc = sqlite3KvvfsMethods.xRcrdDelete("session", "jrnl"); } - return SQLITE_OK; + else{ + rc = 0; + } + return rc; } /* @@ -868,27 +929,48 @@ static int kvvfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ ** is available, or false otherwise. */ static int kvvfsAccess( - sqlite3_vfs *pProtoVfs, - const char *zPath, - int flags, + sqlite3_vfs *pProtoVfs, + const char *zPath, + int flags, int *pResOut ){ SQLITE_KV_LOG(("xAccess(\"%s\")\n", zPath)); +#if 0 && defined(SQLITE_WASM) + /* + ** This is not having the desired effect in the JS bindings. + ** It's ostensibly the same logic as the #else block, but + ** it's not behaving that way. + ** + ** In JS we map all zPaths to Storage objects, and -journal files + ** are mapped to the storage for the main db (which is is exactly + ** what the mapping of "local-journal" -> "local" is doing). + */ + const char *zKey = (0==sqlite3_strglob("*-journal", zPath)) + ? "jrnl" : "sz"; + *pResOut = + sqlite3KvvfsMethods.xRcrdRead(zPath, zKey, 0, 0)>0; +#else if( strcmp(zPath, "local-journal")==0 ){ - *pResOut = sqlite3KvvfsMethods.xRead("local", "jrnl", 0, 0)>0; + *pResOut = + sqlite3KvvfsMethods.xRcrdRead("local", "jrnl", 0, 0)>0; }else if( strcmp(zPath, "session-journal")==0 ){ - *pResOut = sqlite3KvvfsMethods.xRead("session", "jrnl", 0, 0)>0; + *pResOut = + sqlite3KvvfsMethods.xRcrdRead("session", "jrnl", 0, 0)>0; }else if( strcmp(zPath, "local")==0 ){ - *pResOut = sqlite3KvvfsMethods.xRead("local", "sz", 0, 0)>0; + *pResOut = + sqlite3KvvfsMethods.xRcrdRead("local", "sz", 0, 0)>0; }else if( strcmp(zPath, "session")==0 ){ - *pResOut = sqlite3KvvfsMethods.xRead("session", "sz", 0, 0)>0; + *pResOut = + sqlite3KvvfsMethods.xRcrdRead("session", "sz", 0, 0)>0; }else { *pResOut = 0; } + /*all current JS tests avoid triggering: assert( *pResOut == 0 ); */ +#endif SQLITE_KV_LOG(("xAccess returns %d\n",*pResOut)); return SQLITE_OK; } @@ -899,9 +981,9 @@ static int kvvfsAccess( ** of at least (INST_MAX_PATHNAME+1) bytes. */ static int kvvfsFullPathname( - sqlite3_vfs *pVfs, - const char *zPath, - int nOut, + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, char *zOut ){ size_t nPath; @@ -924,7 +1006,7 @@ static void *kvvfsDlOpen(sqlite3_vfs *pVfs, const char *zPath){ } /* -** Populate the buffer pointed to by zBufOut with nByte bytes of +** Populate the buffer pointed to by zBufOut with nByte bytes of ** random data. */ static int kvvfsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ @@ -933,7 +1015,7 @@ static int kvvfsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ } /* -** Sleep for nMicro microseconds. Return the number of microseconds +** Sleep for nMicro microseconds. Return the number of microseconds ** actually slept. */ static int kvvfsSleep(sqlite3_vfs *pVfs, int nMicro){ @@ -961,7 +1043,7 @@ static int kvvfsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ #endif /* SQLITE_OS_KV || SQLITE_OS_UNIX */ #if SQLITE_OS_KV -/* +/* ** This routine is called initialize the KV-vfs as the default VFS. */ int sqlite3_os_init(void){ diff --git a/src/os_unix.c b/src/os_unix.c index d73d89924..2f75829c8 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5190,7 +5190,7 @@ static int unixShmMap( } /* Map the requested memory region into this processes address space. */ - apNew = (char **)sqlite3_realloc( + apNew = (char **)sqlite3_realloc64( pShmNode->apRegion, nReqRegion*sizeof(char *) ); if( !apNew ){ diff --git a/src/os_win.c b/src/os_win.c index a6b25f2e8..7583ecc1f 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -43,7 +43,7 @@ ** Are most of the Win32 ANSI APIs available (i.e. with certain exceptions ** based on the sub-platform)? */ -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(SQLITE_WIN32_NO_ANSI) +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_ANSI) # define SQLITE_WIN32_HAS_ANSI #endif @@ -51,7 +51,7 @@ ** Are most of the Win32 Unicode APIs available (i.e. with certain exceptions ** based on the sub-platform)? */ -#if (SQLITE_OS_WINCE || SQLITE_OS_WINNT || SQLITE_OS_WINRT) && \ +#if (SQLITE_OS_WINCE || SQLITE_OS_WINNT) && \ !defined(SQLITE_WIN32_NO_WIDE) # define SQLITE_WIN32_HAS_WIDE #endif @@ -190,16 +190,7 @@ */ #if SQLITE_WIN32_FILEMAPPING_API && \ (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) -/* -** Two of the file mapping APIs are different under WinRT. Figure out which -** set we need. -*/ -#if SQLITE_OS_WINRT -WINBASEAPI HANDLE WINAPI CreateFileMappingFromApp(HANDLE, \ - LPSECURITY_ATTRIBUTES, ULONG, ULONG64, LPCWSTR); -WINBASEAPI LPVOID WINAPI MapViewOfFileFromApp(HANDLE, ULONG, ULONG64, SIZE_T); -#else #if defined(SQLITE_WIN32_HAS_ANSI) WINBASEAPI HANDLE WINAPI CreateFileMappingA(HANDLE, LPSECURITY_ATTRIBUTES, \ DWORD, DWORD, DWORD, LPCSTR); @@ -211,7 +202,6 @@ WINBASEAPI HANDLE WINAPI CreateFileMappingW(HANDLE, LPSECURITY_ATTRIBUTES, \ #endif /* defined(SQLITE_WIN32_HAS_WIDE) */ WINBASEAPI LPVOID WINAPI MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); -#endif /* SQLITE_OS_WINRT */ /* ** These file mapping APIs are common to both Win32 and WinRT. @@ -502,7 +492,7 @@ static LONG SQLITE_WIN32_VOLATILE sqlite3_os_type = 0; ** This function is not available on Windows CE or WinRT. */ -#if SQLITE_OS_WINCE || SQLITE_OS_WINRT +#if SQLITE_OS_WINCE # define osAreFileApisANSI() 1 #endif @@ -517,7 +507,7 @@ static struct win_syscall { sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ sqlite3_syscall_ptr pDefault; /* Default value */ } aSyscall[] = { -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE { "AreFileApisANSI", (SYSCALL)AreFileApisANSI, 0 }, #else { "AreFileApisANSI", (SYSCALL)0, 0 }, @@ -556,7 +546,7 @@ static struct win_syscall { #define osCreateFileA ((HANDLE(WINAPI*)(LPCSTR,DWORD,DWORD, \ LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[4].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if defined(SQLITE_WIN32_HAS_WIDE) { "CreateFileW", (SYSCALL)CreateFileW, 0 }, #else { "CreateFileW", (SYSCALL)0, 0 }, @@ -565,7 +555,7 @@ static struct win_syscall { #define osCreateFileW ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD, \ LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[5].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_ANSI) && \ +#if defined(SQLITE_WIN32_HAS_ANSI) && \ (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) && \ SQLITE_WIN32_CREATEFILEMAPPINGA { "CreateFileMappingA", (SYSCALL)CreateFileMappingA, 0 }, @@ -576,8 +566,8 @@ static struct win_syscall { #define osCreateFileMappingA ((HANDLE(WINAPI*)(HANDLE,LPSECURITY_ATTRIBUTES, \ DWORD,DWORD,DWORD,LPCSTR))aSyscall[6].pCurrent) -#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ - (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) +#if (SQLITE_OS_WINCE || defined(SQLITE_WIN32_HAS_WIDE)) && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) { "CreateFileMappingW", (SYSCALL)CreateFileMappingW, 0 }, #else { "CreateFileMappingW", (SYSCALL)0, 0 }, @@ -586,7 +576,7 @@ static struct win_syscall { #define osCreateFileMappingW ((HANDLE(WINAPI*)(HANDLE,LPSECURITY_ATTRIBUTES, \ DWORD,DWORD,DWORD,LPCWSTR))aSyscall[7].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if defined(SQLITE_WIN32_HAS_WIDE) { "CreateMutexW", (SYSCALL)CreateMutexW, 0 }, #else { "CreateMutexW", (SYSCALL)0, 0 }, @@ -672,7 +662,7 @@ static struct win_syscall { #define osGetDiskFreeSpaceA ((BOOL(WINAPI*)(LPCSTR,LPDWORD,LPDWORD,LPDWORD, \ LPDWORD))aSyscall[18].pCurrent) -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_WIDE) { "GetDiskFreeSpaceW", (SYSCALL)GetDiskFreeSpaceW, 0 }, #else { "GetDiskFreeSpaceW", (SYSCALL)0, 0 }, @@ -689,7 +679,7 @@ static struct win_syscall { #define osGetFileAttributesA ((DWORD(WINAPI*)(LPCSTR))aSyscall[20].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if defined(SQLITE_WIN32_HAS_WIDE) { "GetFileAttributesW", (SYSCALL)GetFileAttributesW, 0 }, #else { "GetFileAttributesW", (SYSCALL)0, 0 }, @@ -706,11 +696,7 @@ static struct win_syscall { #define osGetFileAttributesExW ((BOOL(WINAPI*)(LPCWSTR,GET_FILEEX_INFO_LEVELS, \ LPVOID))aSyscall[22].pCurrent) -#if !SQLITE_OS_WINRT { "GetFileSize", (SYSCALL)GetFileSize, 0 }, -#else - { "GetFileSize", (SYSCALL)0, 0 }, -#endif #define osGetFileSize ((DWORD(WINAPI*)(HANDLE,LPDWORD))aSyscall[23].pCurrent) @@ -723,7 +709,7 @@ static struct win_syscall { #define osGetFullPathNameA ((DWORD(WINAPI*)(LPCSTR,DWORD,LPSTR, \ LPSTR*))aSyscall[24].pCurrent) -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_WIDE) { "GetFullPathNameW", (SYSCALL)GetFullPathNameW, 0 }, #else { "GetFullPathNameW", (SYSCALL)0, 0 }, @@ -758,16 +744,10 @@ static struct win_syscall { #define osGetProcAddressA ((FARPROC(WINAPI*)(HMODULE, \ LPCSTR))aSyscall[27].pCurrent) -#if !SQLITE_OS_WINRT { "GetSystemInfo", (SYSCALL)GetSystemInfo, 0 }, -#else - { "GetSystemInfo", (SYSCALL)0, 0 }, -#endif - #define osGetSystemInfo ((VOID(WINAPI*)(LPSYSTEM_INFO))aSyscall[28].pCurrent) { "GetSystemTime", (SYSCALL)GetSystemTime, 0 }, - #define osGetSystemTime ((VOID(WINAPI*)(LPSYSTEMTIME))aSyscall[29].pCurrent) #if !SQLITE_OS_WINCE @@ -787,7 +767,7 @@ static struct win_syscall { #define osGetTempPathA ((DWORD(WINAPI*)(DWORD,LPSTR))aSyscall[31].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if defined(SQLITE_WIN32_HAS_WIDE) { "GetTempPathW", (SYSCALL)GetTempPathW, 0 }, #else { "GetTempPathW", (SYSCALL)0, 0 }, @@ -795,11 +775,7 @@ static struct win_syscall { #define osGetTempPathW ((DWORD(WINAPI*)(DWORD,LPWSTR))aSyscall[32].pCurrent) -#if !SQLITE_OS_WINRT { "GetTickCount", (SYSCALL)GetTickCount, 0 }, -#else - { "GetTickCount", (SYSCALL)0, 0 }, -#endif #define osGetTickCount ((DWORD(WINAPI*)(VOID))aSyscall[33].pCurrent) @@ -812,7 +788,7 @@ static struct win_syscall { #define osGetVersionExA ((BOOL(WINAPI*)( \ LPOSVERSIONINFOA))aSyscall[34].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ +#if defined(SQLITE_WIN32_HAS_WIDE) && \ SQLITE_WIN32_GETVERSIONEX { "GetVersionExW", (SYSCALL)GetVersionExW, 0 }, #else @@ -827,20 +803,12 @@ static struct win_syscall { #define osHeapAlloc ((LPVOID(WINAPI*)(HANDLE,DWORD, \ SIZE_T))aSyscall[36].pCurrent) -#if !SQLITE_OS_WINRT { "HeapCreate", (SYSCALL)HeapCreate, 0 }, -#else - { "HeapCreate", (SYSCALL)0, 0 }, -#endif #define osHeapCreate ((HANDLE(WINAPI*)(DWORD,SIZE_T, \ SIZE_T))aSyscall[37].pCurrent) -#if !SQLITE_OS_WINRT { "HeapDestroy", (SYSCALL)HeapDestroy, 0 }, -#else - { "HeapDestroy", (SYSCALL)0, 0 }, -#endif #define osHeapDestroy ((BOOL(WINAPI*)(HANDLE))aSyscall[38].pCurrent) @@ -858,16 +826,12 @@ static struct win_syscall { #define osHeapSize ((SIZE_T(WINAPI*)(HANDLE,DWORD, \ LPCVOID))aSyscall[41].pCurrent) -#if !SQLITE_OS_WINRT { "HeapValidate", (SYSCALL)HeapValidate, 0 }, -#else - { "HeapValidate", (SYSCALL)0, 0 }, -#endif #define osHeapValidate ((BOOL(WINAPI*)(HANDLE,DWORD, \ LPCVOID))aSyscall[42].pCurrent) -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE { "HeapCompact", (SYSCALL)HeapCompact, 0 }, #else { "HeapCompact", (SYSCALL)0, 0 }, @@ -883,7 +847,7 @@ static struct win_syscall { #define osLoadLibraryA ((HMODULE(WINAPI*)(LPCSTR))aSyscall[44].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ +#if defined(SQLITE_WIN32_HAS_WIDE) && \ !defined(SQLITE_OMIT_LOAD_EXTENSION) { "LoadLibraryW", (SYSCALL)LoadLibraryW, 0 }, #else @@ -892,15 +856,11 @@ static struct win_syscall { #define osLoadLibraryW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[45].pCurrent) -#if !SQLITE_OS_WINRT { "LocalFree", (SYSCALL)LocalFree, 0 }, -#else - { "LocalFree", (SYSCALL)0, 0 }, -#endif #define osLocalFree ((HLOCAL(WINAPI*)(HLOCAL))aSyscall[46].pCurrent) -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE { "LockFile", (SYSCALL)LockFile, 0 }, #else { "LockFile", (SYSCALL)0, 0 }, @@ -922,8 +882,7 @@ static struct win_syscall { LPOVERLAPPED))aSyscall[48].pCurrent) #endif -#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && \ - (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) +#if SQLITE_OS_WINCE || !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 { "MapViewOfFile", (SYSCALL)MapViewOfFile, 0 }, #else { "MapViewOfFile", (SYSCALL)0, 0 }, @@ -951,20 +910,12 @@ static struct win_syscall { #define osSetEndOfFile ((BOOL(WINAPI*)(HANDLE))aSyscall[53].pCurrent) -#if !SQLITE_OS_WINRT { "SetFilePointer", (SYSCALL)SetFilePointer, 0 }, -#else - { "SetFilePointer", (SYSCALL)0, 0 }, -#endif #define osSetFilePointer ((DWORD(WINAPI*)(HANDLE,LONG,PLONG, \ DWORD))aSyscall[54].pCurrent) -#if !SQLITE_OS_WINRT { "Sleep", (SYSCALL)Sleep, 0 }, -#else - { "Sleep", (SYSCALL)0, 0 }, -#endif #define osSleep ((VOID(WINAPI*)(DWORD))aSyscall[55].pCurrent) @@ -973,7 +924,7 @@ static struct win_syscall { #define osSystemTimeToFileTime ((BOOL(WINAPI*)(const SYSTEMTIME*, \ LPFILETIME))aSyscall[56].pCurrent) -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE { "UnlockFile", (SYSCALL)UnlockFile, 0 }, #else { "UnlockFile", (SYSCALL)0, 0 }, @@ -1011,15 +962,6 @@ static struct win_syscall { #define osWriteFile ((BOOL(WINAPI*)(HANDLE,LPCVOID,DWORD,LPDWORD, \ LPOVERLAPPED))aSyscall[61].pCurrent) -#if SQLITE_OS_WINRT - { "CreateEventExW", (SYSCALL)CreateEventExW, 0 }, -#else - { "CreateEventExW", (SYSCALL)0, 0 }, -#endif - -#define osCreateEventExW ((HANDLE(WINAPI*)(LPSECURITY_ATTRIBUTES,LPCWSTR, \ - DWORD,DWORD))aSyscall[62].pCurrent) - /* ** For WaitForSingleObject(), MSDN says: ** @@ -1029,7 +971,7 @@ static struct win_syscall { { "WaitForSingleObject", (SYSCALL)WaitForSingleObject, 0 }, #define osWaitForSingleObject ((DWORD(WINAPI*)(HANDLE, \ - DWORD))aSyscall[63].pCurrent) + DWORD))aSyscall[62].pCurrent) #if !SQLITE_OS_WINCE { "WaitForSingleObjectEx", (SYSCALL)WaitForSingleObjectEx, 0 }, @@ -1038,69 +980,12 @@ static struct win_syscall { #endif #define osWaitForSingleObjectEx ((DWORD(WINAPI*)(HANDLE,DWORD, \ - BOOL))aSyscall[64].pCurrent) - -#if SQLITE_OS_WINRT - { "SetFilePointerEx", (SYSCALL)SetFilePointerEx, 0 }, -#else - { "SetFilePointerEx", (SYSCALL)0, 0 }, -#endif - -#define osSetFilePointerEx ((BOOL(WINAPI*)(HANDLE,LARGE_INTEGER, \ - PLARGE_INTEGER,DWORD))aSyscall[65].pCurrent) + BOOL))aSyscall[63].pCurrent) -#if SQLITE_OS_WINRT - { "GetFileInformationByHandleEx", (SYSCALL)GetFileInformationByHandleEx, 0 }, -#else - { "GetFileInformationByHandleEx", (SYSCALL)0, 0 }, -#endif - -#define osGetFileInformationByHandleEx ((BOOL(WINAPI*)(HANDLE, \ - FILE_INFO_BY_HANDLE_CLASS,LPVOID,DWORD))aSyscall[66].pCurrent) - -#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) - { "MapViewOfFileFromApp", (SYSCALL)MapViewOfFileFromApp, 0 }, -#else - { "MapViewOfFileFromApp", (SYSCALL)0, 0 }, -#endif - -#define osMapViewOfFileFromApp ((LPVOID(WINAPI*)(HANDLE,ULONG,ULONG64, \ - SIZE_T))aSyscall[67].pCurrent) - -#if SQLITE_OS_WINRT - { "CreateFile2", (SYSCALL)CreateFile2, 0 }, -#else - { "CreateFile2", (SYSCALL)0, 0 }, -#endif - -#define osCreateFile2 ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD,DWORD, \ - LPCREATEFILE2_EXTENDED_PARAMETERS))aSyscall[68].pCurrent) - -#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_LOAD_EXTENSION) - { "LoadPackagedLibrary", (SYSCALL)LoadPackagedLibrary, 0 }, -#else - { "LoadPackagedLibrary", (SYSCALL)0, 0 }, -#endif - -#define osLoadPackagedLibrary ((HMODULE(WINAPI*)(LPCWSTR, \ - DWORD))aSyscall[69].pCurrent) - -#if SQLITE_OS_WINRT - { "GetTickCount64", (SYSCALL)GetTickCount64, 0 }, -#else - { "GetTickCount64", (SYSCALL)0, 0 }, -#endif - -#define osGetTickCount64 ((ULONGLONG(WINAPI*)(VOID))aSyscall[70].pCurrent) - -#if SQLITE_OS_WINRT { "GetNativeSystemInfo", (SYSCALL)GetNativeSystemInfo, 0 }, -#else - { "GetNativeSystemInfo", (SYSCALL)0, 0 }, -#endif #define osGetNativeSystemInfo ((VOID(WINAPI*)( \ - LPSYSTEM_INFO))aSyscall[71].pCurrent) + LPSYSTEM_INFO))aSyscall[64].pCurrent) #if defined(SQLITE_WIN32_HAS_ANSI) { "OutputDebugStringA", (SYSCALL)OutputDebugStringA, 0 }, @@ -1108,7 +993,7 @@ static struct win_syscall { { "OutputDebugStringA", (SYSCALL)0, 0 }, #endif -#define osOutputDebugStringA ((VOID(WINAPI*)(LPCSTR))aSyscall[72].pCurrent) +#define osOutputDebugStringA ((VOID(WINAPI*)(LPCSTR))aSyscall[65].pCurrent) #if defined(SQLITE_WIN32_HAS_WIDE) { "OutputDebugStringW", (SYSCALL)OutputDebugStringW, 0 }, @@ -1116,20 +1001,11 @@ static struct win_syscall { { "OutputDebugStringW", (SYSCALL)0, 0 }, #endif -#define osOutputDebugStringW ((VOID(WINAPI*)(LPCWSTR))aSyscall[73].pCurrent) +#define osOutputDebugStringW ((VOID(WINAPI*)(LPCWSTR))aSyscall[66].pCurrent) { "GetProcessHeap", (SYSCALL)GetProcessHeap, 0 }, -#define osGetProcessHeap ((HANDLE(WINAPI*)(VOID))aSyscall[74].pCurrent) - -#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) - { "CreateFileMappingFromApp", (SYSCALL)CreateFileMappingFromApp, 0 }, -#else - { "CreateFileMappingFromApp", (SYSCALL)0, 0 }, -#endif - -#define osCreateFileMappingFromApp ((HANDLE(WINAPI*)(HANDLE, \ - LPSECURITY_ATTRIBUTES,ULONG,ULONG64,LPCWSTR))aSyscall[75].pCurrent) +#define osGetProcessHeap ((HANDLE(WINAPI*)(VOID))aSyscall[67].pCurrent) /* ** NOTE: On some sub-platforms, the InterlockedCompareExchange "function" @@ -1144,25 +1020,25 @@ static struct win_syscall { { "InterlockedCompareExchange", (SYSCALL)InterlockedCompareExchange, 0 }, #define osInterlockedCompareExchange ((LONG(WINAPI*)(LONG \ - SQLITE_WIN32_VOLATILE*, LONG,LONG))aSyscall[76].pCurrent) + SQLITE_WIN32_VOLATILE*, LONG,LONG))aSyscall[68].pCurrent) #endif /* defined(InterlockedCompareExchange) */ -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID +#if !SQLITE_OS_WINCE && SQLITE_WIN32_USE_UUID { "UuidCreate", (SYSCALL)UuidCreate, 0 }, #else { "UuidCreate", (SYSCALL)0, 0 }, #endif -#define osUuidCreate ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[77].pCurrent) +#define osUuidCreate ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[69].pCurrent) -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID +#if !SQLITE_OS_WINCE && SQLITE_WIN32_USE_UUID { "UuidCreateSequential", (SYSCALL)UuidCreateSequential, 0 }, #else { "UuidCreateSequential", (SYSCALL)0, 0 }, #endif #define osUuidCreateSequential \ - ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[78].pCurrent) + ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[70].pCurrent) #if !defined(SQLITE_NO_SYNC) && SQLITE_MAX_MMAP_SIZE>0 { "FlushViewOfFile", (SYSCALL)FlushViewOfFile, 0 }, @@ -1171,7 +1047,7 @@ static struct win_syscall { #endif #define osFlushViewOfFile \ - ((BOOL(WINAPI*)(LPCVOID,SIZE_T))aSyscall[79].pCurrent) + ((BOOL(WINAPI*)(LPCVOID,SIZE_T))aSyscall[71].pCurrent) /* ** If SQLITE_ENABLE_SETLK_TIMEOUT is defined, we require CreateEvent() @@ -1188,7 +1064,7 @@ static struct win_syscall { #define osCreateEvent ( \ (HANDLE(WINAPI*) (LPSECURITY_ATTRIBUTES,BOOL,BOOL,LPCSTR)) \ - aSyscall[80].pCurrent \ + aSyscall[72].pCurrent \ ) /* @@ -1205,7 +1081,7 @@ static struct win_syscall { { "CancelIo", (SYSCALL)0, 0 }, #endif -#define osCancelIo ((BOOL(WINAPI*)(HANDLE))aSyscall[81].pCurrent) +#define osCancelIo ((BOOL(WINAPI*)(HANDLE))aSyscall[73].pCurrent) #if defined(SQLITE_WIN32_HAS_WIDE) && defined(_WIN32) { "GetModuleHandleW", (SYSCALL)GetModuleHandleW, 0 }, @@ -1213,7 +1089,7 @@ static struct win_syscall { { "GetModuleHandleW", (SYSCALL)0, 0 }, #endif -#define osGetModuleHandleW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[82].pCurrent) +#define osGetModuleHandleW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[74].pCurrent) #ifndef _WIN32 { "getenv", (SYSCALL)getenv, 0 }, @@ -1221,7 +1097,7 @@ static struct win_syscall { { "getenv", (SYSCALL)0, 0 }, #endif -#define osGetenv ((const char *(*)(const char *))aSyscall[83].pCurrent) +#define osGetenv ((const char *(*)(const char *))aSyscall[75].pCurrent) #ifndef _WIN32 { "getcwd", (SYSCALL)getcwd, 0 }, @@ -1229,7 +1105,7 @@ static struct win_syscall { { "getcwd", (SYSCALL)0, 0 }, #endif -#define osGetcwd ((char*(*)(char*,size_t))aSyscall[84].pCurrent) +#define osGetcwd ((char*(*)(char*,size_t))aSyscall[76].pCurrent) #ifndef _WIN32 { "readlink", (SYSCALL)readlink, 0 }, @@ -1237,7 +1113,7 @@ static struct win_syscall { { "readlink", (SYSCALL)0, 0 }, #endif -#define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[85].pCurrent) +#define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[77].pCurrent) #ifndef _WIN32 { "lstat", (SYSCALL)lstat, 0 }, @@ -1245,7 +1121,7 @@ static struct win_syscall { { "lstat", (SYSCALL)0, 0 }, #endif -#define osLstat ((int(*)(const char*,struct stat*))aSyscall[86].pCurrent) +#define osLstat ((int(*)(const char*,struct stat*))aSyscall[78].pCurrent) #ifndef _WIN32 { "__errno", (SYSCALL)__errno, 0 }, @@ -1253,7 +1129,7 @@ static struct win_syscall { { "__errno", (SYSCALL)0, 0 }, #endif -#define osErrno (*((int*(*)(void))aSyscall[87].pCurrent)()) +#define osErrno (*((int*(*)(void))aSyscall[79].pCurrent)()) #ifndef _WIN32 { "cygwin_conv_path", (SYSCALL)cygwin_conv_path, 0 }, @@ -1262,7 +1138,7 @@ static struct win_syscall { #endif #define osCygwin_conv_path ((size_t(*)(unsigned int, \ - const void *, void *, size_t))aSyscall[88].pCurrent) + const void *, void *, size_t))aSyscall[80].pCurrent) }; /* End of the overrideable system calls */ @@ -1366,10 +1242,10 @@ int sqlite3_win32_compact_heap(LPUINT pnLargest){ hHeap = winMemGetHeap(); assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); #endif -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE if( (nLargest=osHeapCompact(hHeap, SQLITE_WIN32_HEAP_FLAGS))==0 ){ DWORD lastErrno = osGetLastError(); if( lastErrno==NO_ERROR ){ @@ -1482,28 +1358,11 @@ void sqlite3_win32_write_debug(const char *zBuf, int nBuf){ } #endif /* _WIN32 */ -/* -** The following routine suspends the current thread for at least ms -** milliseconds. This is equivalent to the Win32 Sleep() interface. -*/ -#if SQLITE_OS_WINRT -static HANDLE sleepObj = NULL; -#endif - void sqlite3_win32_sleep(DWORD milliseconds){ -#if SQLITE_OS_WINRT - if ( sleepObj==NULL ){ - sleepObj = osCreateEventExW(NULL, NULL, CREATE_EVENT_MANUAL_RESET, - SYNCHRONIZE); - } - assert( sleepObj!=NULL ); - osWaitForSingleObjectEx(sleepObj, milliseconds, FALSE); -#else osSleep(milliseconds); -#endif } -#if SQLITE_MAX_WORKER_THREADS>0 && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && \ +#if SQLITE_MAX_WORKER_THREADS>0 && !SQLITE_OS_WINCE && \ SQLITE_THREADSAFE>0 DWORD sqlite3Win32Wait(HANDLE hObject){ DWORD rc; @@ -1527,7 +1386,7 @@ DWORD sqlite3Win32Wait(HANDLE hObject){ #if !SQLITE_WIN32_GETVERSIONEX # define osIsNT() (1) -#elif SQLITE_OS_WINCE || SQLITE_OS_WINRT || !defined(SQLITE_WIN32_HAS_ANSI) +#elif SQLITE_OS_WINCE || !defined(SQLITE_WIN32_HAS_ANSI) # define osIsNT() (1) #elif !defined(SQLITE_WIN32_HAS_WIDE) # define osIsNT() (0) @@ -1540,13 +1399,7 @@ DWORD sqlite3Win32Wait(HANDLE hObject){ ** based on the NT kernel. */ int sqlite3_win32_is_nt(void){ -#if SQLITE_OS_WINRT - /* - ** NOTE: The WinRT sub-platform is always assumed to be based on the NT - ** kernel. - */ - return 1; -#elif SQLITE_WIN32_GETVERSIONEX +#if SQLITE_WIN32_GETVERSIONEX if( osInterlockedCompareExchange(&sqlite3_os_type, 0, 0)==0 ){ #if defined(SQLITE_WIN32_HAS_ANSI) OSVERSIONINFOA sInfo; @@ -1588,7 +1441,7 @@ static void *winMemMalloc(int nBytes){ hHeap = winMemGetHeap(); assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); #endif assert( nBytes>=0 ); @@ -1610,7 +1463,7 @@ static void winMemFree(void *pPrior){ hHeap = winMemGetHeap(); assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ); #endif if( !pPrior ) return; /* Passing NULL to HeapFree is undefined. */ @@ -1631,7 +1484,7 @@ static void *winMemRealloc(void *pPrior, int nBytes){ hHeap = winMemGetHeap(); assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ); #endif assert( nBytes>=0 ); @@ -1659,7 +1512,7 @@ static int winMemSize(void *p){ hHeap = winMemGetHeap(); assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, p) ); #endif if( !p ) return 0; @@ -1689,7 +1542,7 @@ static int winMemInit(void *pAppData){ assert( pWinMemData->magic1==WINMEM_MAGIC1 ); assert( pWinMemData->magic2==WINMEM_MAGIC2 ); -#if !SQLITE_OS_WINRT && SQLITE_WIN32_HEAP_CREATE +#if SQLITE_WIN32_HEAP_CREATE if( !pWinMemData->hHeap ){ DWORD dwInitialSize = SQLITE_WIN32_HEAP_INIT_SIZE; DWORD dwMaximumSize = (DWORD)sqlite3GlobalConfig.nHeap; @@ -1722,7 +1575,7 @@ static int winMemInit(void *pAppData){ #endif assert( pWinMemData->hHeap!=0 ); assert( pWinMemData->hHeap!=INVALID_HANDLE_VALUE ); -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(pWinMemData->hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); #endif return SQLITE_OK; @@ -1740,7 +1593,7 @@ static void winMemShutdown(void *pAppData){ if( pWinMemData->hHeap ){ assert( pWinMemData->hHeap!=INVALID_HANDLE_VALUE ); -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(pWinMemData->hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); #endif if( pWinMemData->bOwned ){ @@ -2121,17 +1974,6 @@ static int winGetLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ char *zOut = 0; if( osIsNT() ){ -#if SQLITE_OS_WINRT - WCHAR zTempWide[SQLITE_WIN32_MAX_ERRMSG_CHARS+1]; - dwLen = osFormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - lastErrno, - 0, - zTempWide, - SQLITE_WIN32_MAX_ERRMSG_CHARS, - 0); -#else LPWSTR zTempWide = NULL; dwLen = osFormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | @@ -2142,16 +1984,13 @@ static int winGetLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ (LPWSTR) &zTempWide, 0, 0); -#endif if( dwLen > 0 ){ /* allocate a buffer and convert to UTF8 */ sqlite3BeginBenignMalloc(); zOut = winUnicodeToUtf8(zTempWide); sqlite3EndBenignMalloc(); -#if !SQLITE_OS_WINRT /* free the system buffer allocated by FormatMessage */ osLocalFree(zTempWide); -#endif } } #ifdef SQLITE_WIN32_HAS_ANSI @@ -2812,7 +2651,6 @@ static int winHandleUnlock(HANDLE h, int iOff, int nByte){ static int winHandleSeek(HANDLE h, sqlite3_int64 iOffset){ int rc = SQLITE_OK; /* Return value */ -#if !SQLITE_OS_WINRT LONG upperBits; /* Most sig. 32 bits of new offset */ LONG lowerBits; /* Least sig. 32 bits of new offset */ DWORD dwRet; /* Value returned by SetFilePointer() */ @@ -2834,20 +2672,7 @@ static int winHandleSeek(HANDLE h, sqlite3_int64 iOffset){ rc = SQLITE_IOERR_SEEK; } } -#else - /* This implementation works for WinRT. */ - LARGE_INTEGER x; /* The new offset */ - BOOL bRet; /* Value returned by SetFilePointerEx() */ - - x.QuadPart = iOffset; - bRet = osSetFilePointerEx(h, x, 0, FILE_BEGIN); - - if(!bRet){ - rc = SQLITE_IOERR_SEEK; - } -#endif - - OSTRACE(("SEEK file=%p, offset=%lld rc=%s\n", h, iOffset, sqlite3ErrName(rc))); + OSTRACE(("SEEK file=%p, offset=%lld rc=%s\n", h, iOffset,sqlite3ErrName(rc))); return rc; } @@ -3148,17 +2973,6 @@ static int winHandleTruncate(HANDLE h, sqlite3_int64 nByte){ */ static int winHandleSize(HANDLE h, sqlite3_int64 *pnByte){ int rc = SQLITE_OK; - -#if SQLITE_OS_WINRT - FILE_STANDARD_INFO info; - BOOL b; - b = osGetFileInformationByHandleEx(h, FileStandardInfo, &info, sizeof(info)); - if( b ){ - *pnByte = info.EndOfFile.QuadPart; - }else{ - rc = SQLITE_IOERR_FSTAT; - } -#else DWORD upperBits = 0; DWORD lowerBits = 0; @@ -3168,8 +2982,6 @@ static int winHandleSize(HANDLE h, sqlite3_int64 *pnByte){ if( lowerBits==INVALID_FILE_SIZE && osGetLastError()!=NO_ERROR ){ rc = SQLITE_IOERR_FSTAT; } -#endif - return rc; } @@ -3368,20 +3180,6 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ assert( pSize!=0 ); SimulateIOError(return SQLITE_IOERR_FSTAT); OSTRACE(("SIZE file=%p, pSize=%p\n", pFile->h, pSize)); - -#if SQLITE_OS_WINRT - { - FILE_STANDARD_INFO info; - if( osGetFileInformationByHandleEx(pFile->h, FileStandardInfo, - &info, sizeof(info)) ){ - *pSize = info.EndOfFile.QuadPart; - }else{ - pFile->lastErrno = osGetLastError(); - rc = winLogError(SQLITE_IOERR_FSTAT, pFile->lastErrno, - "winFileSize", pFile->zPath); - } - } -#else { DWORD upperBits; DWORD lowerBits; @@ -3396,7 +3194,6 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ "winFileSize", pFile->zPath); } } -#endif OSTRACE(("SIZE file=%p, pSize=%p, *pSize=%lld, rc=%s\n", pFile->h, pSize, *pSize, sqlite3ErrName(rc))); return rc; @@ -4358,20 +4155,6 @@ static int winHandleOpen( ** TODO: retry-on-ioerr. */ if( osIsNT() ){ -#if SQLITE_OS_WINRT - CREATEFILE2_EXTENDED_PARAMETERS extendedParameters; - memset(&extendedParameters, 0, sizeof(extendedParameters)); - extendedParameters.dwSize = sizeof(extendedParameters); - extendedParameters.dwFileAttributes = FILE_ATTRIBUTE_NORMAL; - extendedParameters.dwFileFlags = flag_overlapped; - extendedParameters.dwSecurityQosFlags = SECURITY_ANONYMOUS; - h = osCreateFile2((LPCWSTR)zConverted, - (GENERIC_READ | (bReadonly ? 0 : GENERIC_WRITE)),/* dwDesiredAccess */ - FILE_SHARE_READ | FILE_SHARE_WRITE, /* dwShareMode */ - OPEN_ALWAYS, /* dwCreationDisposition */ - &extendedParameters - ); -#else h = osCreateFileW((LPCWSTR)zConverted, /* lpFileName */ (GENERIC_READ | (bReadonly ? 0 : GENERIC_WRITE)), /* dwDesiredAccess */ FILE_SHARE_READ | FILE_SHARE_WRITE, /* dwShareMode */ @@ -4380,7 +4163,6 @@ static int winHandleOpen( FILE_ATTRIBUTE_NORMAL|flag_overlapped, NULL ); -#endif }else{ /* Due to pre-processor directives earlier in this file, ** SQLITE_WIN32_HAS_ANSI is always defined if osIsNT() is false. */ @@ -4848,9 +4630,7 @@ static int winShmMap( HANDLE hMap = NULL; /* file-mapping handle */ void *pMap = 0; /* Mapped memory region */ -#if SQLITE_OS_WINRT - hMap = osCreateFileMappingFromApp(hShared, NULL, protect, nByte, NULL); -#elif defined(SQLITE_WIN32_HAS_WIDE) +#if defined(SQLITE_WIN32_HAS_WIDE) hMap = osCreateFileMappingW(hShared, NULL, protect, 0, nByte, NULL); #elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA hMap = osCreateFileMappingA(hShared, NULL, protect, 0, nByte, NULL); @@ -4862,15 +4642,9 @@ static int winShmMap( if( hMap ){ int iOffset = pShmNode->nRegion*szRegion; int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; -#if SQLITE_OS_WINRT - pMap = osMapViewOfFileFromApp(hMap, flags, - iOffset - iOffsetShift, szRegion + iOffsetShift - ); -#else pMap = osMapViewOfFile(hMap, flags, 0, iOffset - iOffsetShift, szRegion + iOffsetShift ); -#endif OSTRACE(("SHM-MAP-MAP pid=%lu, region=%d, offset=%d, size=%d, rc=%s\n", osGetCurrentProcessId(), pShmNode->nRegion, iOffset, szRegion, pMap ? "ok" : "failed")); @@ -5003,9 +4777,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ flags |= FILE_MAP_WRITE; } #endif -#if SQLITE_OS_WINRT - pFd->hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); -#elif defined(SQLITE_WIN32_HAS_WIDE) +#if defined(SQLITE_WIN32_HAS_WIDE) pFd->hMap = osCreateFileMappingW(pFd->h, NULL, protect, (DWORD)((nMap>>32) & 0xffffffff), (DWORD)(nMap & 0xffffffff), NULL); @@ -5025,11 +4797,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ } assert( (nMap % winSysInfo.dwPageSize)==0 ); assert( sizeof(SIZE_T)==sizeof(sqlite3_int64) || nMap<=0xffffffff ); -#if SQLITE_OS_WINRT - pNew = osMapViewOfFileFromApp(pFd->hMap, flags, 0, (SIZE_T)nMap); -#else pNew = osMapViewOfFile(pFd->hMap, flags, 0, 0, (SIZE_T)nMap); -#endif if( pNew==NULL ){ osCloseHandle(pFd->hMap); pFd->hMap = NULL; @@ -5364,7 +5132,6 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ } #endif -#if !SQLITE_OS_WINRT && defined(_WIN32) else if( osIsNT() ){ char *zMulti; LPWSTR zWidePath = sqlite3MallocZero( nMax*sizeof(WCHAR) ); @@ -5418,7 +5185,6 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ } } #endif /* SQLITE_WIN32_HAS_ANSI */ -#endif /* !SQLITE_OS_WINRT */ /* ** Check to make sure the temporary directory ends with an appropriate @@ -5593,13 +5359,6 @@ static int winOpen( memset(pFile, 0, sizeof(winFile)); pFile->h = INVALID_HANDLE_VALUE; -#if SQLITE_OS_WINRT - if( !zUtf8Name && !sqlite3_temp_directory ){ - sqlite3_log(SQLITE_ERROR, - "sqlite3_temp_directory variable should be set for WinRT"); - } -#endif - /* If the second argument to this function is NULL, generate a ** temporary file name to use */ @@ -5682,31 +5441,6 @@ static int winOpen( #endif if( osIsNT() ){ -#if SQLITE_OS_WINRT - CREATEFILE2_EXTENDED_PARAMETERS extendedParameters; - extendedParameters.dwSize = sizeof(CREATEFILE2_EXTENDED_PARAMETERS); - extendedParameters.dwFileAttributes = - dwFlagsAndAttributes & FILE_ATTRIBUTE_MASK; - extendedParameters.dwFileFlags = dwFlagsAndAttributes & FILE_FLAG_MASK; - extendedParameters.dwSecurityQosFlags = SECURITY_ANONYMOUS; - extendedParameters.lpSecurityAttributes = NULL; - extendedParameters.hTemplateFile = NULL; - do{ - h = osCreateFile2((LPCWSTR)zConverted, - dwDesiredAccess, - dwShareMode, - dwCreationDisposition, - &extendedParameters); - if( h!=INVALID_HANDLE_VALUE ) break; - if( isReadWrite ){ - int rc2; - sqlite3BeginBenignMalloc(); - rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ|NORETRY, &isRO); - sqlite3EndBenignMalloc(); - if( rc2==SQLITE_OK && isRO ) break; - } - }while( winRetryIoerr(&cnt, &lastErrno) ); -#else do{ h = osCreateFileW((LPCWSTR)zConverted, dwDesiredAccess, @@ -5723,7 +5457,6 @@ static int winOpen( if( rc2==SQLITE_OK && isRO ) break; } }while( winRetryIoerr(&cnt, &lastErrno) ); -#endif } #ifdef SQLITE_WIN32_HAS_ANSI else{ @@ -5860,25 +5593,7 @@ static int winDelete( } if( osIsNT() ){ do { -#if SQLITE_OS_WINRT - WIN32_FILE_ATTRIBUTE_DATA sAttrData; - memset(&sAttrData, 0, sizeof(sAttrData)); - if ( osGetFileAttributesExW(zConverted, GetFileExInfoStandard, - &sAttrData) ){ - attr = sAttrData.dwFileAttributes; - }else{ - lastErrno = osGetLastError(); - if( lastErrno==ERROR_FILE_NOT_FOUND - || lastErrno==ERROR_PATH_NOT_FOUND ){ - rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ - }else{ - rc = SQLITE_ERROR; - } - break; - } -#else attr = osGetFileAttributesW(zConverted); -#endif if ( attr==INVALID_FILE_ATTRIBUTES ){ lastErrno = osGetLastError(); if( lastErrno==ERROR_FILE_NOT_FOUND @@ -6001,6 +5716,7 @@ static int winAccess( attr = sAttrData.dwFileAttributes; } }else{ + if( noRetry ) lastErrno = osGetLastError(); winLogIoerr(cnt, __LINE__); if( lastErrno!=ERROR_FILE_NOT_FOUND && lastErrno!=ERROR_PATH_NOT_FOUND ){ sqlite3_free(zConverted); @@ -6169,7 +5885,7 @@ static int winFullPathnameNoMutex( int nFull, /* Size of output buffer in bytes */ char *zFull /* Output buffer */ ){ -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE int nByte; void *zConverted; char *zOut; @@ -6258,7 +5974,7 @@ static int winFullPathnameNoMutex( } #endif /* __CYGWIN__ */ -#if (SQLITE_OS_WINCE || SQLITE_OS_WINRT) && defined(_WIN32) +#if SQLITE_OS_WINCE && defined(_WIN32) SimulateIOError( return SQLITE_ERROR ); /* WinCE has no concept of a relative pathname, or so I am told. */ /* WinRT has no way to convert a relative path to an absolute one. */ @@ -6277,7 +5993,7 @@ static int winFullPathnameNoMutex( return SQLITE_OK; #endif -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE #if defined(_WIN32) /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this @@ -6409,11 +6125,7 @@ static void *winDlOpen(sqlite3_vfs *pVfs, const char *zFilename){ return 0; } if( osIsNT() ){ -#if SQLITE_OS_WINRT - h = osLoadPackagedLibrary((LPCWSTR)zConverted, 0); -#else h = osLoadLibraryW((LPCWSTR)zConverted); -#endif } #ifdef SQLITE_WIN32_HAS_ANSI else{ @@ -6495,23 +6207,16 @@ static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ DWORD pid = osGetCurrentProcessId(); xorMemory(&e, (unsigned char*)&pid, sizeof(DWORD)); } -#if SQLITE_OS_WINRT - { - ULONGLONG cnt = osGetTickCount64(); - xorMemory(&e, (unsigned char*)&cnt, sizeof(ULONGLONG)); - } -#else { DWORD cnt = osGetTickCount(); xorMemory(&e, (unsigned char*)&cnt, sizeof(DWORD)); } -#endif /* SQLITE_OS_WINRT */ { LARGE_INTEGER i; osQueryPerformanceCounter(&i); xorMemory(&e, (unsigned char*)&i, sizeof(LARGE_INTEGER)); } -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID +#if !SQLITE_OS_WINCE && SQLITE_WIN32_USE_UUID { UUID id; memset(&id, 0, sizeof(UUID)); @@ -6521,7 +6226,7 @@ static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ osUuidCreateSequential(&id); xorMemory(&e, (unsigned char*)&id, sizeof(UUID)); } -#endif /* !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID */ +#endif /* !SQLITE_OS_WINCE && SQLITE_WIN32_USE_UUID */ return e.nXor>nBuf ? nBuf : e.nXor; #endif /* defined(SQLITE_TEST) || defined(SQLITE_OMIT_RANDOMNESS) */ } @@ -6752,15 +6457,16 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==89 ); + assert( ArraySize(aSyscall)==81 ); + assert( strcmp(aSyscall[0].zName,"AreFileApisANSI")==0 ); + assert( strcmp(aSyscall[20].zName,"GetFileAttributesA")==0 ); + assert( strcmp(aSyscall[40].zName,"HeapReAlloc")==0 ); + assert( strcmp(aSyscall[60].zName,"WideCharToMultiByte")==0 ); + assert( strcmp(aSyscall[80].zName,"cygwin_conv_path")==0 ); /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); -#if SQLITE_OS_WINRT - osGetNativeSystemInfo(&winSysInfo); -#else osGetSystemInfo(&winSysInfo); -#endif assert( winSysInfo.dwAllocationGranularity>0 ); assert( winSysInfo.dwPageSize>0 ); @@ -6784,17 +6490,9 @@ int sqlite3_os_init(void){ } int sqlite3_os_end(void){ -#if SQLITE_OS_WINRT - if( sleepObj!=NULL ){ - osCloseHandle(sleepObj); - sleepObj = NULL; - } -#endif - #ifndef SQLITE_OMIT_WAL winBigLock = 0; #endif - return SQLITE_OK; } diff --git a/src/os_win.h b/src/os_win.h index a0845f003..696486c19 100644 --- a/src/os_win.h +++ b/src/os_win.h @@ -58,14 +58,6 @@ # define SQLITE_OS_WINCE 0 #endif -/* -** Determine if we are dealing with WinRT, which provides only a subset of -** the full Win32 API. -*/ -#if !defined(SQLITE_OS_WINRT) -# define SQLITE_OS_WINRT 0 -#endif - /* ** For WinCE, some API function parameters do not appear to be declared as ** volatile. @@ -80,7 +72,7 @@ ** For some Windows sub-platforms, the _beginthreadex() / _endthreadex() ** functions are not available (e.g. those not using MSVC, Cygwin, etc). */ -#if SQLITE_OS_WIN && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && \ +#if SQLITE_OS_WIN && !SQLITE_OS_WINCE && \ SQLITE_THREADSAFE>0 && !defined(__CYGWIN__) # define SQLITE_OS_WIN_THREADS 1 #else diff --git a/src/pager.c b/src/pager.c index cbaef186a..61b391d6b 100644 --- a/src/pager.c +++ b/src/pager.c @@ -813,6 +813,8 @@ int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){ (void)sqlite3WalFindFrame(pPager->pWal, pgno, &iRead); if( iRead ) return 0; /* Case (4) */ } +#else + UNUSED_PARAMETER(pgno); #endif assert( pPager->fd->pMethods->xDeviceCharacteristics!=0 ); if( (pPager->fd->pMethods->xDeviceCharacteristics(pPager->fd) @@ -1233,17 +1235,17 @@ static int jrnlBufferSize(Pager *pPager){ */ #ifdef SQLITE_CHECK_PAGES /* -** Return a 32-bit hash of the page data for pPage. +** Return a 64-bit hash of the page data for pPage. */ -static u32 pager_datahash(int nByte, unsigned char *pData){ - u32 hash = 0; +static u64 pager_datahash(int nByte, unsigned char *pData){ + u64 hash = 0; int i; for(i=0; ipPager->pageSize, (unsigned char *)pPage->pData); } static void pager_set_pagehash(PgHdr *pPage){ @@ -4192,6 +4194,8 @@ int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize,a); pPager->pWal = 0; } +#else + UNUSED_PARAMETER(db); #endif pager_reset(pPager); if( MEMDB ){ diff --git a/src/parse.y b/src/parse.y index 617eb7303..f5a6bed14 100644 --- a/src/parse.y +++ b/src/parse.y @@ -21,9 +21,11 @@ */ } -// Function used to enlarge the parser stack, if needed -%realloc parserStackRealloc -%free sqlite3_free +// Setup for the parser stack +%stack_size 50 // Initial stack size +%stack_size_limit parserStackSizeLimit // Function returning max stack size +%realloc parserStackRealloc // realloc() for the stack +%free parserStackFree // free() for the stack // All token codes are small integers with #defines that begin with "TK_" %token_prefix TK_ @@ -49,7 +51,7 @@ } } %stack_overflow { - sqlite3OomFault(pParse->db); + if( pParse->nErr==0 ) sqlite3ErrorMsg(pParse, "Recursion limit"); } // The name of the generated procedure that implements the parser @@ -583,8 +585,23 @@ cmd ::= select(X). { ** sqlite3_realloc() that includes a call to sqlite3FaultSim() to facilitate ** testing. */ - static void *parserStackRealloc(void *pOld, sqlite3_uint64 newSize){ - return sqlite3FaultSim(700) ? 0 : sqlite3_realloc(pOld, newSize); + static void *parserStackRealloc( + void *pOld, /* Prior allocation */ + sqlite3_uint64 newSize, /* Requested new alloation size */ + Parse *pParse /* Parsing context */ + ){ + void *p = sqlite3FaultSim(700) ? 0 : sqlite3_realloc(pOld, newSize); + if( p==0 ) sqlite3OomFault(pParse->db); + return p; + } + static void parserStackFree(void *pOld, Parse *pParse){ + (void)pParse; + sqlite3_free(pOld); + } + + /* Return an integer that is the maximum allowed stack size */ + static int parserStackSizeLimit(Parse *pParse){ + return pParse->db->aLimit[SQLITE_LIMIT_PARSER_DEPTH]; } } @@ -816,19 +833,36 @@ fullname(A) ::= nm(X) DOT nm(Y). { %type xfullname {SrcList*} %destructor xfullname {sqlite3SrcListDelete(pParse->db, $$);} -xfullname(A) ::= nm(X). - {A = sqlite3SrcListAppend(pParse,0,&X,0); /*A-overwrites-X*/} -xfullname(A) ::= nm(X) DOT nm(Y). - {A = sqlite3SrcListAppend(pParse,0,&X,&Y); /*A-overwrites-X*/} -xfullname(A) ::= nm(X) DOT nm(Y) AS nm(Z). { - A = sqlite3SrcListAppend(pParse,0,&X,&Y); /*A-overwrites-X*/ - if( A ) A->a[0].zAlias = sqlite3NameFromToken(pParse->db, &Z); +xfullname(A) ::= nm(X). { + A = sqlite3SrcListAppend(pParse,0,&X,0); + if( IN_RENAME_OBJECT && A ) sqlite3RenameTokenMap(pParse, A->a[0].zName, &X); } -xfullname(A) ::= nm(X) AS nm(Z). { - A = sqlite3SrcListAppend(pParse,0,&X,0); /*A-overwrites-X*/ - if( A ) A->a[0].zAlias = sqlite3NameFromToken(pParse->db, &Z); +xfullname(A) ::= nm(X) DOT nm(Y). { + A = sqlite3SrcListAppend(pParse,0,&X,&Y); + if( IN_RENAME_OBJECT && A ) sqlite3RenameTokenMap(pParse, A->a[0].zName, &Y); +} +xfullname(A) ::= nm(X) AS nm(Z). { + A = sqlite3SrcListAppend(pParse,0,&X,0); + if( A ){ + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, A->a[0].zName, &X); + }else{ + A->a[0].zAlias = sqlite3NameFromToken(pParse->db, &Z); + } + } +} +xfullname(A) ::= nm(X) DOT nm(Y) AS nm(Z). { + A = sqlite3SrcListAppend(pParse,0,&X,&Y); + if( A ){ + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, A->a[0].zName, &Y); + }else{ + A->a[0].zAlias = sqlite3NameFromToken(pParse->db, &Z); + } + } } + %type joinop {int} joinop(X) ::= COMMA|JOIN. { X = JT_INNER; } joinop(X) ::= JOIN_KW(A) JOIN. @@ -1159,7 +1193,12 @@ expr(A) ::= nm(X) DOT nm(Y) DOT nm(Z). { term(A) ::= NULL|FLOAT|BLOB(X). {A=tokenExpr(pParse,@X,X); /*A-overwrites-X*/} term(A) ::= STRING(X). {A=tokenExpr(pParse,@X,X); /*A-overwrites-X*/} term(A) ::= INTEGER(X). { - A = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &X, 1); + int iValue; + if( sqlite3GetInt32(X.z, &iValue)==0 ){ + A = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &X, 0); + }else{ + A = sqlite3ExprInt32(pParse->db, iValue); + } if( A ) A->w.iOfst = (int)(X.z - pParse->zTail); } expr(A) ::= VARIABLE(X). { @@ -1341,43 +1380,67 @@ expr(A) ::= expr(A) likeop(OP) expr(Y) ESCAPE expr(E). [LIKE_KW] { if( A ) A->flags |= EP_InfixFunc; } -expr(A) ::= expr(A) ISNULL|NOTNULL(E). {A = sqlite3PExpr(pParse,@E,A,0);} -expr(A) ::= expr(A) NOT NULL. {A = sqlite3PExpr(pParse,TK_NOTNULL,A,0);} - %include { - /* A routine to convert a binary TK_IS or TK_ISNOT expression into a - ** unary TK_ISNULL or TK_NOTNULL expression. */ - static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){ - sqlite3 *db = pParse->db; - if( pA && pY && pY->op==TK_NULL && !IN_RENAME_OBJECT ){ - pA->op = (u8)op; - sqlite3ExprDelete(db, pA->pRight); - pA->pRight = 0; + /* Create a TK_ISNULL or TK_NOTNULL expression, perhaps optimized to + ** to TK_TRUEFALSE, if possible */ + static Expr *sqlite3PExprIsNull( + Parse *pParse, /* Parsing context */ + int op, /* TK_ISNULL or TK_NOTNULL */ + Expr *pLeft /* Operand */ + ){ + Expr *p = pLeft; + assert( op==TK_ISNULL || op==TK_NOTNULL ); + assert( pLeft!=0 ); + while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ + p = p->pLeft; + assert( p!=0 ); + } + switch( p->op ){ + case TK_INTEGER: + case TK_STRING: + case TK_FLOAT: + case TK_BLOB: + sqlite3ExprDeferredDelete(pParse, pLeft); + return sqlite3ExprInt32(pParse->db, op==TK_NOTNULL); + default: + break; } + return sqlite3PExpr(pParse, op, pLeft, 0); + } + + /* Create a TK_IS or TK_ISNOT operator, perhaps optimized to + ** TK_ISNULL or TK_NOTNULL or TK_TRUEFALSE. */ + static Expr *sqlite3PExprIs( + Parse *pParse, /* Parsing context */ + int op, /* TK_IS or TK_ISNOT */ + Expr *pLeft, /* Left operand */ + Expr *pRight /* Right operand */ + ){ + if( pRight && pRight->op==TK_NULL ){ + sqlite3ExprDeferredDelete(pParse, pRight); + return sqlite3PExprIsNull(pParse, op==TK_IS ? TK_ISNULL : TK_NOTNULL, pLeft); + } + return sqlite3PExpr(pParse, op, pLeft, pRight); } } -// expr1 IS expr2 -// expr1 IS NOT expr2 +expr(A) ::= expr(A) ISNULL|NOTNULL(E). {A = sqlite3PExprIsNull(pParse,@E,A);} +expr(A) ::= expr(A) NOT NULL. {A = sqlite3PExprIsNull(pParse,TK_NOTNULL,A);} + +// expr1 IS expr2 same as expr1 IS NOT DISTINCT FROM expr2 +// expr1 IS NOT expr2 same as expr1 IS DISTINCT FROM expr2 // -// If expr2 is NULL then code as TK_ISNULL or TK_NOTNULL. If expr2 -// is any other expression, code as TK_IS or TK_ISNOT. -// expr(A) ::= expr(A) IS expr(Y). { - A = sqlite3PExpr(pParse,TK_IS,A,Y); - binaryToUnaryIfNull(pParse, Y, A, TK_ISNULL); + A = sqlite3PExprIs(pParse, TK_IS, A, Y); } expr(A) ::= expr(A) IS NOT expr(Y). { - A = sqlite3PExpr(pParse,TK_ISNOT,A,Y); - binaryToUnaryIfNull(pParse, Y, A, TK_NOTNULL); + A = sqlite3PExprIs(pParse, TK_ISNOT, A, Y); } expr(A) ::= expr(A) IS NOT DISTINCT FROM expr(Y). { - A = sqlite3PExpr(pParse,TK_IS,A,Y); - binaryToUnaryIfNull(pParse, Y, A, TK_ISNULL); + A = sqlite3PExprIs(pParse, TK_IS, A, Y); } expr(A) ::= expr(A) IS DISTINCT FROM expr(Y). { - A = sqlite3PExpr(pParse,TK_ISNOT,A,Y); - binaryToUnaryIfNull(pParse, Y, A, TK_NOTNULL); + A = sqlite3PExprIs(pParse, TK_ISNOT, A, Y); } expr(A) ::= NOT(B) expr(X). @@ -1707,28 +1770,13 @@ when_clause(A) ::= WHEN expr(X). { A = X; } %type trigger_cmd_list {TriggerStep*} %destructor trigger_cmd_list {sqlite3DeleteTriggerStep(pParse->db, $$);} trigger_cmd_list(A) ::= trigger_cmd_list(A) trigger_cmd(X) SEMI. { - assert( A!=0 ); A->pLast->pNext = X; A->pLast = X; } trigger_cmd_list(A) ::= trigger_cmd(A) SEMI. { - assert( A!=0 ); A->pLast = A; } -// Disallow qualified table names on INSERT, UPDATE, and DELETE statements -// within a trigger. The table to INSERT, UPDATE, or DELETE is always in -// the same database as the table that the trigger fires on. -// -%type trnm {Token} -trnm(A) ::= nm(A). -trnm(A) ::= nm DOT nm(X). { - A = X; - sqlite3ErrorMsg(pParse, - "qualified table names are not allowed on INSERT, UPDATE, and DELETE " - "statements within triggers"); -} - // Disallow the INDEX BY and NOT INDEXED clauses on UPDATE and DELETE // statements within triggers. We make a specific error message for this // since it is an exception to the default grammar rules. @@ -1751,17 +1799,17 @@ tridxby ::= NOT INDEXED. { %destructor trigger_cmd {sqlite3DeleteTriggerStep(pParse->db, $$);} // UPDATE trigger_cmd(A) ::= - UPDATE(B) orconf(R) trnm(X) tridxby SET setlist(Y) from(F) where_opt(Z) scanpt(E). - {A = sqlite3TriggerUpdateStep(pParse, &X, F, Y, Z, R, B.z, E);} + UPDATE(B) orconf(R) xfullname(X) tridxby SET setlist(Y) from(F) where_opt(Z) scanpt(E). + {A = sqlite3TriggerUpdateStep(pParse, X, F, Y, Z, R, B.z, E);} // INSERT trigger_cmd(A) ::= scanpt(B) insert_cmd(R) INTO - trnm(X) idlist_opt(F) select(S) upsert(U) scanpt(Z). { - A = sqlite3TriggerInsertStep(pParse,&X,F,S,R,U,B,Z);/*A-overwrites-R*/ + xfullname(X) idlist_opt(F) select(S) upsert(U) scanpt(Z). { + A = sqlite3TriggerInsertStep(pParse,X,F,S,R,U,B,Z);/*A-overwrites-R*/ } // DELETE -trigger_cmd(A) ::= DELETE(B) FROM trnm(X) tridxby where_opt(Y) scanpt(E). - {A = sqlite3TriggerDeleteStep(pParse, &X, Y, B.z, E);} +trigger_cmd(A) ::= DELETE(B) FROM xfullname(X) tridxby where_opt(Y) scanpt(E). + {A = sqlite3TriggerDeleteStep(pParse, X, Y, B.z, E);} // SELECT trigger_cmd(A) ::= scanpt(B) select(X) scanpt(E). @@ -1831,22 +1879,42 @@ cmd ::= ANALYZE nm(X) dbnm(Y). {sqlite3Analyze(pParse, &X, &Y);} cmd ::= ALTER TABLE fullname(X) RENAME TO nm(Z). { sqlite3AlterRenameTable(pParse,X,&Z); } -cmd ::= ALTER TABLE add_column_fullname - ADD kwcolumn_opt columnname(Y) carglist. { + +// The ALTER TABLE ADD COLUMN command. This is broken into two sections so +// that sqlite3AlterBeginAddColumn() is called before parsing the various +// constraints and so on (carglist) attached to the new column definition. +cmd ::= alter_add(Y) carglist. { Y.n = (int)(pParse->sLastToken.z-Y.z) + pParse->sLastToken.n; sqlite3AlterFinishAddColumn(pParse, &Y); } -cmd ::= ALTER TABLE fullname(X) DROP kwcolumn_opt nm(Y). { - sqlite3AlterDropColumn(pParse, X, &Y); -} - -add_column_fullname ::= fullname(X). { +alter_add(A) ::= ALTER TABLE fullname(X) ADD kwcolumn_opt nm(Y) typetoken(Z). { disableLookaside(pParse); sqlite3AlterBeginAddColumn(pParse, X); + sqlite3AddColumn(pParse, Y, Z); + A = Y; +} + +cmd ::= ALTER TABLE fullname(X) DROP kwcolumn_opt nm(Y). { + sqlite3AlterDropColumn(pParse, X, &Y); } cmd ::= ALTER TABLE fullname(X) RENAME kwcolumn_opt nm(Y) TO nm(Z). { sqlite3AlterRenameColumn(pParse, X, &Y, &Z); } +cmd ::= ALTER TABLE fullname(X) DROP CONSTRAINT nm(Y). { + sqlite3AlterDropConstraint(pParse, X, &Y, 0); +} +cmd ::= ALTER TABLE fullname(X) ALTER kwcolumn_opt nm(Y) DROP NOT NULL. { + sqlite3AlterDropConstraint(pParse, X, 0, &Y); +} +cmd ::= ALTER TABLE fullname(X) ALTER kwcolumn_opt nm(Y) SET NOT(Z) NULL onconf. { + sqlite3AlterSetNotNull(pParse, X, &Y, &Z); +} +cmd ::= ALTER TABLE fullname(X) ADD CONSTRAINT(Y) nm(Z) CHECK LP(A) expr RP(B) onconf. { + sqlite3AlterAddConstraint(pParse, X, &Y, &Z, A.z+1, (B.z-A.z-1)); +} +cmd ::= ALTER TABLE fullname(X) ADD CHECK(Y) LP(A) expr RP(B) onconf. { + sqlite3AlterAddConstraint(pParse, X, &Y, 0, A.z+1, (B.z-A.z-1)); +} kwcolumn_opt ::= . kwcolumn_opt ::= COLUMNKW. diff --git a/src/pcache.h b/src/pcache.h index f945dab1a..dafb59390 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -29,10 +29,10 @@ struct PgHdr { PCache *pCache; /* PRIVATE: Cache that owns this page */ PgHdr *pDirty; /* Transient list of dirty sorted by pgno */ Pager *pPager; /* The pager this page is part of */ - Pgno pgno; /* Page number for this page */ #ifdef SQLITE_CHECK_PAGES - u32 pageHash; /* Hash of page content */ + u64 pageHash; /* Hash of page content */ #endif + Pgno pgno; /* Page number for this page */ u16 flags; /* PGHDR flags defined below */ /********************************************************************** diff --git a/src/prepare.c b/src/prepare.c index 539360b74..be9e496f1 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -33,7 +33,8 @@ static void corruptSchema( static const char *azAlterType[] = { "rename", "drop column", - "add column" + "add column", + "drop constraint" }; *pData->pzErrMsg = sqlite3MPrintf(db, "error in %s %s after %s: %s", azObj[0], azObj[1], diff --git a/src/printf.c b/src/printf.c index f75ed3b8a..d9f3c229d 100644 --- a/src/printf.c +++ b/src/printf.c @@ -549,7 +549,7 @@ void sqlite3_str_vappendf( }else{ iRound = precision+1; } - sqlite3FpDecode(&s, realvalue, iRound, flag_altform2 ? 26 : 16); + sqlite3FpDecode(&s, realvalue, iRound, flag_altform2 ? 20 : 16); if( s.isSpecial ){ if( s.isSpecial==2 ){ bufpt = flag_zeropad ? "null" : "NaN"; @@ -1228,6 +1228,14 @@ int sqlite3_str_length(sqlite3_str *p){ return p ? p->nChar : 0; } +/* Truncate the text of the string to be no more than N bytes. */ +void sqlite3_str_truncate(sqlite3_str *p, int N){ + if( p!=0 && N>=0 && (u32)NnChar ){ + p->nChar = N; + p->zText[p->nChar] = 0; + } +} + /* Return the current value for p */ char *sqlite3_str_value(sqlite3_str *p){ if( p==0 || p->nChar==0 ) return 0; @@ -1248,6 +1256,17 @@ void sqlite3_str_reset(StrAccum *p){ p->zText = 0; } +/* +** Destroy a dynamically allocate sqlite3_str object and all +** of its content, all in one call. +*/ +void sqlite3_str_free(sqlite3_str *p){ + if( p ){ + sqlite3_str_reset(p); + sqlite3_free(p); + } +} + /* ** Initialize a string accumulator. ** diff --git a/src/resolve.c b/src/resolve.c index 16c193ca2..5b6d6c9c5 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -936,7 +936,7 @@ static int exprProbability(Expr *p){ double r = -1.0; if( p->op!=TK_FLOAT ) return -1; assert( !ExprHasProperty(p, EP_IntValue) ); - sqlite3AtoF(p->u.zToken, &r, sqlite3Strlen30(p->u.zToken), SQLITE_UTF8); + sqlite3AtoF(p->u.zToken, &r); assert( r>=0.0 ); if( r>1.0 ) return -1; return (int)(r*134217728.0); @@ -1656,10 +1656,8 @@ static int resolveCompoundOrderBy( /* Convert the ORDER BY term into an integer column number iCol, ** taking care to preserve the COLLATE clause if it exists. */ if( !IN_RENAME_OBJECT ){ - Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); + Expr *pNew = sqlite3ExprInt32(db, iCol); if( pNew==0 ) return 1; - pNew->flags |= EP_IntValue; - pNew->u.iValue = iCol; if( pItem->pExpr==pE ){ pItem->pExpr = pNew; }else{ @@ -2013,10 +2011,6 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ } #endif - /* The ORDER BY and GROUP BY clauses may not refer to terms in - ** outer queries - */ - sNC.pNext = 0; sNC.ncFlags |= NC_AllowAgg|NC_AllowWin; /* If this is a converted compound query, move the ORDER BY clause from @@ -2079,6 +2073,14 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ return WRC_Abort; } + /* If the SELECT statement contains ON clauses that were moved into + ** the WHERE clause, go through and verify that none of the terms + ** in the ON clauses reference tables to the right of the ON clause. */ + if( (p->selFlags & SF_OnToWhere) ){ + sqlite3SelectCheckOnClauses(pParse, p); + if( pParse->nErr ) return WRC_Abort; + } + /* Advance to the next term of the compound */ p = p->pPrior; diff --git a/src/select.c b/src/select.c index bc17ecf84..e8e9f36a8 100644 --- a/src/select.c +++ b/src/select.c @@ -21,7 +21,7 @@ */ typedef struct DistinctCtx DistinctCtx; struct DistinctCtx { - u8 isTnct; /* 0: Not distinct. 1: DISTICT 2: DISTINCT and ORDER BY */ + u8 isTnct; /* 0: Not distinct. 1: DISTINCT 2: DISTINCT and ORDER BY */ u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */ int tabTnct; /* Ephemeral table used for DISTINCT processing */ int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */ @@ -151,8 +151,6 @@ Select *sqlite3SelectNew( pNew->iLimit = 0; pNew->iOffset = 0; pNew->selId = ++pParse->nSelect; - pNew->addrOpenEphm[0] = -1; - pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = 0; if( pSrc==0 ) pSrc = sqlite3DbMallocZero(pParse->db, SZ_SRCLIST_1); pNew->pSrc = pSrc; @@ -661,6 +659,10 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ pRight->fg.isOn = 1; p->selFlags |= SF_OnToWhere; } + + if( IsVirtual(pRightTab) && joinType==EP_OuterON && pRight->u1.pFuncArg ){ + p->selFlags |= SF_OnToWhere; + } } return 0; } @@ -1300,29 +1302,6 @@ static void selectInnerLoop( } switch( eDest ){ - /* In this mode, write each query result to the key of the temporary - ** table iParm. - */ -#ifndef SQLITE_OMIT_COMPOUND_SELECT - case SRT_Union: { - int r1; - r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); - sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol); - sqlite3ReleaseTempReg(pParse, r1); - break; - } - - /* Construct a record from the query result, but instead of - ** saving that record, use it as a key to delete elements from - ** the temporary table iParm. - */ - case SRT_Except: { - sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nResultCol); - break; - } -#endif /* SQLITE_OMIT_COMPOUND_SELECT */ - /* Store the result as data using a unique key. */ case SRT_Fifo: @@ -2435,8 +2414,8 @@ void sqlite3SubqueryColumnTypes( } } if( zType ){ - const i64 k = sqlite3Strlen30(zType); - n = sqlite3Strlen30(pCol->zCnName); + const i64 k = strlen(zType); + n = strlen(pCol->zCnName); pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+k+2); pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL); if( pCol->zCnName ){ @@ -2609,9 +2588,9 @@ static CollSeq *multiSelectCollSeq(Parse *pParse, Select *p, int iCol){ ** function is responsible for ensuring that this structure is eventually ** freed. */ -static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){ +static KeyInfo *multiSelectByMergeKeyInfo(Parse *pParse, Select *p, int nExtra){ ExprList *pOrderBy = p->pOrderBy; - int nOrderBy = ALWAYS(pOrderBy!=0) ? pOrderBy->nExpr : 0; + int nOrderBy = (pOrderBy!=0) ? pOrderBy->nExpr : 0; sqlite3 *db = pParse->db; KeyInfo *pRet = sqlite3KeyInfoAlloc(db, nOrderBy+nExtra, 1); if( pRet ){ @@ -2744,7 +2723,7 @@ static void generateWithRecursiveQuery( regCurrent = ++pParse->nMem; sqlite3VdbeAddOp3(v, OP_OpenPseudo, iCurrent, regCurrent, nCol); if( pOrderBy ){ - KeyInfo *pKeyInfo = multiSelectOrderByKeyInfo(pParse, p, 1); + KeyInfo *pKeyInfo = multiSelectByMergeKeyInfo(pParse, p, 1); sqlite3VdbeAddOp4(v, OP_OpenEphemeral, iQueue, pOrderBy->nExpr+2, 0, (char*)pKeyInfo, P4_KEYINFO); destQueue.pOrderBy = pOrderBy; @@ -2753,8 +2732,28 @@ static void generateWithRecursiveQuery( } VdbeComment((v, "Queue table")); if( iDistinct ){ - p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iDistinct, 0); - p->selFlags |= SF_UsesEphemeral; + /* Generate an ephemeral table used to enforce distinctness on the + ** output of the recursive part of the CTE. + */ + KeyInfo *pKeyInfo; /* Collating sequence for the result set */ + CollSeq **apColl; /* For looping through pKeyInfo->aColl[] */ + + assert( p->pNext==0 ); + assert( p->pEList!=0 ); + nCol = p->pEList->nExpr; + pKeyInfo = sqlite3KeyInfoAlloc(pParse->db, nCol, 1); + if( pKeyInfo ){ + for(i=0, apColl=pKeyInfo->aColl; idb->pDfltColl; + } + } + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, iDistinct, nCol, 0, + (void*)pKeyInfo, P4_KEYINFO); + }else{ + assert( pParse->nErr>0 ); + } } /* Detach the ORDER BY clause from the compound SELECT */ @@ -2829,7 +2828,7 @@ static void generateWithRecursiveQuery( #endif /* SQLITE_OMIT_CTE */ /* Forward references */ -static int multiSelectOrderBy( +static int multiSelectByMerge( Parse *pParse, /* Parsing context */ Select *p, /* The right-most of SELECTs to be coded */ SelectDest *pDest /* What to do with query results */ @@ -2978,12 +2977,26 @@ static int multiSelect( generateWithRecursiveQuery(pParse, p, &dest); }else #endif - - /* Compound SELECTs that have an ORDER BY clause are handled separately. - */ if( p->pOrderBy ){ - return multiSelectOrderBy(pParse, p, pDest); + /* If the compound has an ORDER BY clause, then always use the merge + ** algorithm. */ + return multiSelectByMerge(pParse, p, pDest); + }else if( p->op!=TK_ALL ){ + /* If the compound is EXCEPT, INTERSECT, or UNION (anything other than + ** UNION ALL) then also always use the merge algorithm. However, the + ** multiSelectByMerge() routine requires that the compound have an + ** ORDER BY clause, and it doesn't right now. So invent one first. */ + Expr *pOne = sqlite3ExprInt32(db, 1); + p->pOrderBy = sqlite3ExprListAppend(pParse, 0, pOne); + if( pParse->nErr ) goto multi_select_end; + assert( p->pOrderBy!=0 ); + p->pOrderBy->a[0].u.x.iOrderByCol = 1; + return multiSelectByMerge(pParse, p, pDest); }else{ + /* For a UNION ALL compound without ORDER BY, simply run the left + ** query, then run the right query */ + int addr = 0; + int nLimit = 0; /* Initialize to suppress harmless compiler warning */ #ifndef SQLITE_OMIT_EXPLAIN if( pPrior->pPrior==0 ){ @@ -2991,300 +3004,49 @@ static int multiSelect( ExplainQueryPlan((pParse, 1, "LEFT-MOST SUBQUERY")); } #endif - - /* Generate code for the left and right SELECT statements. - */ - switch( p->op ){ - case TK_ALL: { - int addr = 0; - int nLimit = 0; /* Initialize to suppress harmless compiler warning */ - assert( !pPrior->pLimit ); - pPrior->iLimit = p->iLimit; - pPrior->iOffset = p->iOffset; - pPrior->pLimit = p->pLimit; - TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL left...\n")); - rc = sqlite3Select(pParse, pPrior, &dest); - pPrior->pLimit = 0; - if( rc ){ - goto multi_select_end; - } - p->pPrior = 0; - p->iLimit = pPrior->iLimit; - p->iOffset = pPrior->iOffset; - if( p->iLimit ){ - addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v); - VdbeComment((v, "Jump ahead if LIMIT reached")); - if( p->iOffset ){ - sqlite3VdbeAddOp3(v, OP_OffsetLimit, - p->iLimit, p->iOffset+1, p->iOffset); - } - } - ExplainQueryPlan((pParse, 1, "UNION ALL")); - TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL right...\n")); - rc = sqlite3Select(pParse, p, &dest); - testcase( rc!=SQLITE_OK ); - pDelete = p->pPrior; - p->pPrior = pPrior; - p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); - if( p->pLimit - && sqlite3ExprIsInteger(p->pLimit->pLeft, &nLimit, pParse) - && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) - ){ - p->nSelectRow = sqlite3LogEst((u64)nLimit); - } - if( addr ){ - sqlite3VdbeJumpHere(v, addr); - } - break; - } - case TK_EXCEPT: - case TK_UNION: { - int unionTab; /* Cursor number of the temp table holding result */ - u8 op = 0; /* One of the SRT_ operations to apply to self */ - int priorOp; /* The SRT_ operation to apply to prior selects */ - Expr *pLimit; /* Saved values of p->nLimit */ - int addr; - int emptyBypass = 0; /* IfEmpty opcode to bypass RHS */ - SelectDest uniondest; - - - testcase( p->op==TK_EXCEPT ); - testcase( p->op==TK_UNION ); - priorOp = SRT_Union; - if( dest.eDest==priorOp ){ - /* We can reuse a temporary table generated by a SELECT to our - ** right. - */ - assert( p->pLimit==0 ); /* Not allowed on leftward elements */ - unionTab = dest.iSDParm; - }else{ - /* We will need to create our own temporary table to hold the - ** intermediate results. - */ - unionTab = pParse->nTab++; - assert( p->pOrderBy==0 ); - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, unionTab, 0); - assert( p->addrOpenEphm[0] == -1 ); - p->addrOpenEphm[0] = addr; - findRightmost(p)->selFlags |= SF_UsesEphemeral; - assert( p->pEList ); - } - - - /* Code the SELECT statements to our left - */ - assert( !pPrior->pOrderBy ); - sqlite3SelectDestInit(&uniondest, priorOp, unionTab); - TREETRACE(0x200, pParse, p, ("multiSelect EXCEPT/UNION left...\n")); - rc = sqlite3Select(pParse, pPrior, &uniondest); - if( rc ){ - goto multi_select_end; - } - - /* Code the current SELECT statement - */ - if( p->op==TK_EXCEPT ){ - op = SRT_Except; - emptyBypass = sqlite3VdbeAddOp1(v, OP_IfEmpty, unionTab); - VdbeCoverage(v); - }else{ - assert( p->op==TK_UNION ); - op = SRT_Union; - } - p->pPrior = 0; - pLimit = p->pLimit; - p->pLimit = 0; - uniondest.eDest = op; - ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", - sqlite3SelectOpName(p->op))); - TREETRACE(0x200, pParse, p, ("multiSelect EXCEPT/UNION right...\n")); - rc = sqlite3Select(pParse, p, &uniondest); - testcase( rc!=SQLITE_OK ); - assert( p->pOrderBy==0 ); - pDelete = p->pPrior; - p->pPrior = pPrior; - p->pOrderBy = 0; - if( p->op==TK_UNION ){ - p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); - } - if( emptyBypass ) sqlite3VdbeJumpHere(v, emptyBypass); - sqlite3ExprDelete(db, p->pLimit); - p->pLimit = pLimit; - p->iLimit = 0; - p->iOffset = 0; - - /* Convert the data in the temporary table into whatever form - ** it is that we currently need. - */ - assert( unionTab==dest.iSDParm || dest.eDest!=priorOp ); - assert( p->pEList || db->mallocFailed ); - if( dest.eDest!=priorOp && db->mallocFailed==0 ){ - int iCont, iBreak, iStart; - iBreak = sqlite3VdbeMakeLabel(pParse); - iCont = sqlite3VdbeMakeLabel(pParse); - computeLimitRegisters(pParse, p, iBreak); - sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v); - iStart = sqlite3VdbeCurrentAddr(v); - selectInnerLoop(pParse, p, unionTab, - 0, 0, &dest, iCont, iBreak); - sqlite3VdbeResolveLabel(v, iCont); - sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); VdbeCoverage(v); - sqlite3VdbeResolveLabel(v, iBreak); - sqlite3VdbeAddOp2(v, OP_Close, unionTab, 0); - } - break; - } - default: assert( p->op==TK_INTERSECT ); { - int tab1, tab2; - int iCont, iBreak, iStart; - Expr *pLimit; - int addr, iLimit, iOffset; - SelectDest intersectdest; - int r1; - int emptyBypass; - - /* INTERSECT is different from the others since it requires - ** two temporary tables. Hence it has its own case. Begin - ** by allocating the tables we will need. - */ - tab1 = pParse->nTab++; - tab2 = pParse->nTab++; - assert( p->pOrderBy==0 ); - - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab1, 0); - assert( p->addrOpenEphm[0] == -1 ); - p->addrOpenEphm[0] = addr; - findRightmost(p)->selFlags |= SF_UsesEphemeral; - assert( p->pEList ); - - /* Code the SELECTs to our left into temporary table "tab1". - */ - sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1); - TREETRACE(0x400, pParse, p, ("multiSelect INTERSECT left...\n")); - rc = sqlite3Select(pParse, pPrior, &intersectdest); - if( rc ){ - goto multi_select_end; - } - - /* Initialize LIMIT counters before checking to see if the LHS - ** is empty, in case the jump is taken */ - iBreak = sqlite3VdbeMakeLabel(pParse); - computeLimitRegisters(pParse, p, iBreak); - emptyBypass = sqlite3VdbeAddOp1(v, OP_IfEmpty, tab1); VdbeCoverage(v); - - /* Code the current SELECT into temporary table "tab2" - */ - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab2, 0); - assert( p->addrOpenEphm[1] == -1 ); - p->addrOpenEphm[1] = addr; - - /* Disable prior SELECTs and the LIMIT counters during the computation - ** of the RHS select */ - pLimit = p->pLimit; - iLimit = p->iLimit; - iOffset = p->iOffset; - p->pPrior = 0; - p->pLimit = 0; - p->iLimit = 0; - p->iOffset = 0; - - intersectdest.iSDParm = tab2; - ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", - sqlite3SelectOpName(p->op))); - TREETRACE(0x400, pParse, p, ("multiSelect INTERSECT right...\n")); - rc = sqlite3Select(pParse, p, &intersectdest); - testcase( rc!=SQLITE_OK ); - pDelete = p->pPrior; - p->pPrior = pPrior; - if( p->nSelectRow>pPrior->nSelectRow ){ - p->nSelectRow = pPrior->nSelectRow; - } - sqlite3ExprDelete(db, p->pLimit); - - /* Reinstate the LIMIT counters prior to running the final intersect */ - p->pLimit = pLimit; - p->iLimit = iLimit; - p->iOffset = iOffset; - - /* Generate code to take the intersection of the two temporary - ** tables. - */ - if( rc ) break; - assert( p->pEList ); - sqlite3VdbeAddOp1(v, OP_Rewind, tab1); - r1 = sqlite3GetTempReg(pParse); - iStart = sqlite3VdbeAddOp2(v, OP_RowData, tab1, r1); - iCont = sqlite3VdbeMakeLabel(pParse); - sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); - VdbeCoverage(v); - sqlite3ReleaseTempReg(pParse, r1); - selectInnerLoop(pParse, p, tab1, - 0, 0, &dest, iCont, iBreak); - sqlite3VdbeResolveLabel(v, iCont); - sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v); - sqlite3VdbeResolveLabel(v, iBreak); - sqlite3VdbeAddOp2(v, OP_Close, tab2, 0); - sqlite3VdbeJumpHere(v, emptyBypass); - sqlite3VdbeAddOp2(v, OP_Close, tab1, 0); - break; - } - } - - #ifndef SQLITE_OMIT_EXPLAIN - if( p->pNext==0 ){ - ExplainQueryPlanPop(pParse); - } - #endif - } - if( pParse->nErr ) goto multi_select_end; - - /* Compute collating sequences used by - ** temporary tables needed to implement the compound select. - ** Attach the KeyInfo structure to all temporary tables. - ** - ** This section is run by the right-most SELECT statement only. - ** SELECT statements to the left always skip this part. The right-most - ** SELECT might also skip this part if it has no ORDER BY clause and - ** no temp tables are required. - */ - if( p->selFlags & SF_UsesEphemeral ){ - int i; /* Loop counter */ - KeyInfo *pKeyInfo; /* Collating sequence for the result set */ - Select *pLoop; /* For looping through SELECT statements */ - CollSeq **apColl; /* For looping through pKeyInfo->aColl[] */ - int nCol; /* Number of columns in result set */ - - assert( p->pNext==0 ); - assert( p->pEList!=0 ); - nCol = p->pEList->nExpr; - pKeyInfo = sqlite3KeyInfoAlloc(db, nCol, 1); - if( !pKeyInfo ){ - rc = SQLITE_NOMEM_BKPT; + assert( !pPrior->pLimit ); + pPrior->iLimit = p->iLimit; + pPrior->iOffset = p->iOffset; + pPrior->pLimit = sqlite3ExprDup(db, p->pLimit, 0); + TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL left...\n")); + rc = sqlite3Select(pParse, pPrior, &dest); + sqlite3ExprDelete(db, pPrior->pLimit); + pPrior->pLimit = 0; + if( rc ){ goto multi_select_end; } - for(i=0, apColl=pKeyInfo->aColl; ipDfltColl; - } + p->pPrior = 0; + p->iLimit = pPrior->iLimit; + p->iOffset = pPrior->iOffset; + if( p->iLimit ){ + addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v); + VdbeComment((v, "Jump ahead if LIMIT reached")); + if( p->iOffset ){ + sqlite3VdbeAddOp3(v, OP_OffsetLimit, + p->iLimit, p->iOffset+1, p->iOffset); + } + } + ExplainQueryPlan((pParse, 1, "UNION ALL")); + TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL right...\n")); + rc = sqlite3Select(pParse, p, &dest); + testcase( rc!=SQLITE_OK ); + pDelete = p->pPrior; + p->pPrior = pPrior; + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); + if( p->pLimit + && sqlite3ExprIsInteger(p->pLimit->pLeft, &nLimit, pParse) + && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) + ){ + p->nSelectRow = sqlite3LogEst((u64)nLimit); } - - for(pLoop=p; pLoop; pLoop=pLoop->pPrior){ - for(i=0; i<2; i++){ - int addr = pLoop->addrOpenEphm[i]; - if( addr<0 ){ - /* If [0] is unused then [1] is also unused. So we can - ** always safely abort as soon as the first unused slot is found */ - assert( pLoop->addrOpenEphm[1]<0 ); - break; - } - sqlite3VdbeChangeP2(v, addr, nCol); - sqlite3VdbeChangeP4(v, addr, (char*)sqlite3KeyInfoRef(pKeyInfo), - P4_KEYINFO); - pLoop->addrOpenEphm[i] = -1; - } + if( addr ){ + sqlite3VdbeJumpHere(v, addr); + } +#ifndef SQLITE_OMIT_EXPLAIN + if( p->pNext==0 ){ + ExplainQueryPlanPop(pParse); } - sqlite3KeyInfoUnref(pKeyInfo); +#endif } multi_select_end: @@ -3316,8 +3078,8 @@ void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p){ ** Code an output subroutine for a coroutine implementation of a ** SELECT statement. ** -** The data to be output is contained in pIn->iSdst. There are -** pIn->nSdst columns to be output. pDest is where the output should +** The data to be output is contained in an array of pIn->nSdst registers +** starting at register pIn->iSdst. pDest is where the output should ** be sent. ** ** regReturn is the number of the register holding the subroutine @@ -3346,6 +3108,8 @@ static int generateOutputSubroutine( int iContinue; int addr; + assert( pIn->eDest==SRT_Coroutine ); + addr = sqlite3VdbeCurrentAddr(v); iContinue = sqlite3VdbeMakeLabel(pParse); @@ -3367,23 +3131,60 @@ static int generateOutputSubroutine( */ codeOffset(v, p->iOffset, iContinue); - assert( pDest->eDest!=SRT_Exists ); - assert( pDest->eDest!=SRT_Table ); switch( pDest->eDest ){ /* Store the result as data using a unique key. */ + case SRT_Fifo: + case SRT_DistFifo: + case SRT_Table: case SRT_EphemTab: { int r1 = sqlite3GetTempReg(pParse); int r2 = sqlite3GetTempReg(pParse); + int iParm = pDest->iSDParm; + testcase( pDest->eDest==SRT_Table ); + testcase( pDest->eDest==SRT_EphemTab ); + testcase( pDest->eDest==SRT_Fifo ); + testcase( pDest->eDest==SRT_DistFifo ); sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r1); - sqlite3VdbeAddOp2(v, OP_NewRowid, pDest->iSDParm, r2); - sqlite3VdbeAddOp3(v, OP_Insert, pDest->iSDParm, r1, r2); +#if !defined(SQLITE_ENABLE_NULL_TRIM) && defined(SQLITE_DEBUG) + /* A destination of SRT_Table and a non-zero iSDParm2 parameter means + ** that this is an "UPDATE ... FROM" on a virtual table or view. In this + ** case set the p5 parameter of the OP_MakeRecord to OPFLAG_NOCHNG_MAGIC. + ** This does not affect operation in any way - it just allows MakeRecord + ** to process OPFLAG_NOCHANGE values without an assert() failing. */ + if( pDest->eDest==SRT_Table && pDest->iSDParm2 ){ + sqlite3VdbeChangeP5(v, OPFLAG_NOCHNG_MAGIC); + } +#endif +#ifndef SQLITE_OMIT_CTE + if( pDest->eDest==SRT_DistFifo ){ + /* If the destination is DistFifo, then cursor (iParm+1) is open + ** on an ephemeral index that is used to enforce uniqueness on the + ** total result. At this point, we are processing the setup portion + ** of the recursive CTE using the merge algorithm, so the results are + ** guaranteed to be unique anyhow. But we still need to populate the + ** (iParm+1) cursor for use by the subsequent recursive phase. + */ + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm+1, r1, + pIn->iSdst, pIn->nSdst); + } +#endif + sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); + sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, r2); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3ReleaseTempReg(pParse, r2); sqlite3ReleaseTempReg(pParse, r1); break; } + /* If any row exist in the result set, record that fact and abort. + */ + case SRT_Exists: { + sqlite3VdbeAddOp2(v, OP_Integer, 1, pDest->iSDParm); + /* The LIMIT clause will terminate the loop for us */ + break; + } + #ifndef SQLITE_OMIT_SUBQUERY /* If we are creating a set for an "expr IN (SELECT ...)". */ @@ -3430,9 +3231,51 @@ static int generateOutputSubroutine( break; } +#ifndef SQLITE_OMIT_CTE + /* Write the results into a priority queue that is order according to + ** pDest->pOrderBy (in pSO). pDest->iSDParm (in iParm) is the cursor for an + ** index with pSO->nExpr+2 columns. Build a key using pSO for the first + ** pSO->nExpr columns, then make sure all keys are unique by adding a + ** final OP_Sequence column. The last column is the record as a blob. + */ + case SRT_DistQueue: + case SRT_Queue: { + int nKey; + int r1, r2, r3, ii; + ExprList *pSO; + int iParm = pDest->iSDParm; + pSO = pDest->pOrderBy; + assert( pSO ); + nKey = pSO->nExpr; + r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3GetTempRange(pParse, nKey+2); + r3 = r2+nKey+1; + + sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r3); + if( pDest->eDest==SRT_DistQueue ){ + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r3); + } + for(ii=0; iiiSdst + pSO->a[ii].u.x.iOrderByCol - 1, + r2+ii); + } + sqlite3VdbeAddOp2(v, OP_Sequence, iParm, r2+nKey); + sqlite3VdbeAddOp3(v, OP_MakeRecord, r2, nKey+2, r1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, r2, nKey+2); + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempRange(pParse, r2, nKey+2); + break; + } +#endif /* SQLITE_OMIT_CTE */ + + /* Ignore the output */ + case SRT_Discard: { + break; + } + /* If none of the above, then the result destination must be - ** SRT_Output. This routine is never called with any other - ** destination other than the ones handled above or SRT_Output. + ** SRT_Output. ** ** For SRT_Output, results are stored in a sequence of registers. ** Then the OP_ResultRow opcode is used to cause sqlite3_step() to @@ -3460,8 +3303,9 @@ static int generateOutputSubroutine( } /* -** Alternative compound select code generator for cases when there -** is an ORDER BY clause. +** Generate code for a compound SELECT statement using a merge +** algorithm. The compound must have an ORDER BY clause for this +** to work. ** ** We assume a query of the following form: ** @@ -3478,7 +3322,7 @@ static int generateOutputSubroutine( ** ** outB: Move the output of the selectB coroutine into the output ** of the compound query. (Only generated for UNION and -** UNION ALL. EXCEPT and INSERTSECT never output a row that +** UNION ALL. EXCEPT and INTERSECT never output a row that ** appears only in B.) ** ** AltB: Called when there is data from both coroutines and Au.x.iOrderByCol==i ) break; } if( j==nOrderBy ){ - Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); + Expr *pNew = sqlite3ExprInt32(db, i); if( pNew==0 ) return SQLITE_NOMEM_BKPT; - pNew->flags |= EP_IntValue; - pNew->u.iValue = i; p->pOrderBy = pOrderBy = sqlite3ExprListAppend(pParse, pOrderBy, pNew); if( pOrderBy ) pOrderBy->a[nOrderBy++].u.x.iOrderByCol = (u16)i; } @@ -3628,26 +3468,29 @@ static int multiSelectOrderBy( } /* Compute the comparison permutation and keyinfo that is used with - ** the permutation used to determine if the next - ** row of results comes from selectA or selectB. Also add explicit - ** collations to the ORDER BY clause terms so that when the subqueries - ** to the right and the left are evaluated, they use the correct - ** collation. + ** the permutation to determine if the next row of results comes + ** from selectA or selectB. Also add literal collations to the + ** ORDER BY clause terms so that when selectA and selectB are + ** evaluated, they use the correct collation. */ aPermute = sqlite3DbMallocRawNN(db, sizeof(u32)*(nOrderBy + 1)); if( aPermute ){ struct ExprList_item *pItem; + int bKeep = 0; aPermute[0] = nOrderBy; for(i=1, pItem=pOrderBy->a; i<=nOrderBy; i++, pItem++){ assert( pItem!=0 ); assert( pItem->u.x.iOrderByCol>0 ); assert( pItem->u.x.iOrderByCol<=p->pEList->nExpr ); aPermute[i] = pItem->u.x.iOrderByCol - 1; + if( aPermute[i]!=(u32)i-1 ) bKeep = 1; + } + if( bKeep==0 ){ + sqlite3DbFreeNN(db, aPermute); + aPermute = 0; } - pKeyMerge = multiSelectOrderByKeyInfo(pParse, p, 1); - }else{ - pKeyMerge = 0; } + pKeyMerge = multiSelectByMergeKeyInfo(pParse, p, 1); /* Allocate a range of temporary registers and the KeyInfo needed ** for the logic that removes duplicate result rows when the @@ -3726,7 +3569,7 @@ static int multiSelectOrderBy( */ addrSelectA = sqlite3VdbeCurrentAddr(v) + 1; addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); - VdbeComment((v, "left SELECT")); + VdbeComment((v, "SUBR: next-A")); pPrior->iLimit = regLimitA; ExplainQueryPlan((pParse, 1, "LEFT")); sqlite3Select(pParse, pPrior, &destA); @@ -3738,7 +3581,7 @@ static int multiSelectOrderBy( */ addrSelectB = sqlite3VdbeCurrentAddr(v) + 1; addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB); - VdbeComment((v, "right SELECT")); + VdbeComment((v, "SUBR: next-B")); savedLimit = p->iLimit; savedOffset = p->iOffset; p->iLimit = regLimitB; @@ -3752,7 +3595,7 @@ static int multiSelectOrderBy( /* Generate a subroutine that outputs the current row of the A ** select as the next output row of the compound select. */ - VdbeNoopComment((v, "Output routine for A")); + VdbeNoopComment((v, "SUBR: out-A")); addrOutA = generateOutputSubroutine(pParse, p, &destA, pDest, regOutA, regPrev, pKeyDup, labelEnd); @@ -3761,7 +3604,7 @@ static int multiSelectOrderBy( ** select as the next output row of the compound select. */ if( op==TK_ALL || op==TK_UNION ){ - VdbeNoopComment((v, "Output routine for B")); + VdbeNoopComment((v, "SUBR: out-B")); addrOutB = generateOutputSubroutine(pParse, p, &destB, pDest, regOutB, regPrev, pKeyDup, labelEnd); @@ -3774,10 +3617,12 @@ static int multiSelectOrderBy( if( op==TK_EXCEPT || op==TK_INTERSECT ){ addrEofA_noB = addrEofA = labelEnd; }else{ - VdbeNoopComment((v, "eof-A subroutine")); + VdbeNoopComment((v, "SUBR: eof-A")); addrEofA = sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); + VdbeComment((v, "out-B")); addrEofA_noB = sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, labelEnd); VdbeCoverage(v); + VdbeComment((v, "next-B")); sqlite3VdbeGoto(v, addrEofA); p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); } @@ -3789,17 +3634,20 @@ static int multiSelectOrderBy( addrEofB = addrEofA; if( p->nSelectRow > pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; }else{ - VdbeNoopComment((v, "eof-B subroutine")); + VdbeNoopComment((v, "SUBR: eof-B")); addrEofB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); + VdbeComment((v, "out-A")); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, labelEnd); VdbeCoverage(v); + VdbeComment((v, "next-A")); sqlite3VdbeGoto(v, addrEofB); } /* Generate code to handle the case of AB */ - VdbeNoopComment((v, "A-gt-B subroutine")); addrAgtB = sqlite3VdbeCurrentAddr(v); if( op==TK_ALL || op==TK_UNION ){ sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); + VdbeComment((v, "out-B")); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); + VdbeComment((v, "next-B")); + sqlite3VdbeGoto(v, labelCmpr); + }else{ + addrAgtB++; /* Just do next-B. Might as well use the next-B call + ** in the next code block */ } - sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); - sqlite3VdbeGoto(v, labelCmpr); /* This code runs once to initialize everything. */ sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA_noB); VdbeCoverage(v); + VdbeComment((v, "next-A")); + /* v--- Also the A>B case for EXCEPT and INTERSECT */ sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); + VdbeComment((v, "next-B")); /* Implement the main merge loop */ + if( aPermute!=0 ){ + sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY); + } sqlite3VdbeResolveLabel(v, labelCmpr); - sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY); sqlite3VdbeAddOp4(v, OP_Compare, destA.iSdst, destB.iSdst, nOrderBy, (char*)pKeyMerge, P4_KEYINFO); - sqlite3VdbeChangeP5(v, OPFLAG_PERMUTE); - sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); VdbeCoverage(v); + if( aPermute!=0 ){ + sqlite3VdbeChangeP5(v, OPFLAG_PERMUTE); + } + sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); + VdbeCoverageIf(v, op==TK_ALL); + VdbeCoverageIf(v, op==TK_UNION); + VdbeCoverageIf(v, op==TK_EXCEPT); + VdbeCoverageIf(v, op==TK_INTERSECT); /* Jump to the this point in order to terminate the query. */ @@ -4756,7 +4616,7 @@ static int flattenSubquery( } pSubitem->fg.jointype |= jointype; - /* Now begin substituting subquery result set expressions for + /* Begin substituting subquery result set expressions for ** references to the iParent in the outer query. ** ** Example: @@ -4768,7 +4628,7 @@ static int flattenSubquery( ** We look at every expression in the outer query and every place we see ** "a" we substitute "x*3" and every place we see "b" we substitute "y+10". */ - if( pSub->pOrderBy && (pParent->selFlags & SF_NoopOrderBy)==0 ){ + if( pSub->pOrderBy ){ /* At this point, any non-zero iOrderByCol values indicate that the ** ORDER BY column expression is identical to the iOrderByCol'th ** expression returned by SELECT statement pSub. Since these values @@ -4776,9 +4636,9 @@ static int flattenSubquery( ** zero them before transferring the ORDER BY clause. ** ** Not doing this may cause an error if a subsequent call to this - ** function attempts to flatten a compound sub-query into pParent - ** (the only way this can happen is if the compound sub-query is - ** currently part of pSub->pSrc). See ticket [d11a6e908f]. */ + ** function attempts to flatten a compound sub-query into pParent. + ** See ticket [d11a6e908f]. + */ ExprList *pOrderBy = pSub->pOrderBy; for(i=0; inExpr; i++){ pOrderBy->a[i].u.x.iOrderByCol = 0; @@ -5396,6 +5256,16 @@ static int pushDownWhereTerms( x.pEList = pSubq->pEList; x.pCList = findLeftmostExprlist(pSubq); pNew = substExpr(&x, pNew); + assert( pNew!=0 || pParse->nErr!=0 ); + if( pParse->nErr==0 && pNew->op==TK_IN && ExprUseXSelect(pNew) ){ + assert( pNew->x.pSelect!=0 ); + pNew->x.pSelect->selFlags |= SF_ClonedRhsIn; + assert( pWhere!=0 ); + assert( pWhere->op==TK_IN ); + assert( ExprUseXSelect(pWhere) ); + assert( pWhere->x.pSelect!=0 ); + pWhere->x.pSelect->selFlags |= SF_ClonedRhsIn; + } #ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pWin && 0==pushDownWindowCheck(pParse, pSubq, pNew) ){ /* Restriction 6c has prevented push-down in this case */ @@ -5630,14 +5500,14 @@ int sqlite3IndexedByLookup(Parse *pParse, SrcItem *pFrom){ ** SELECT * FROM (SELECT ... FROM t1 EXCEPT SELECT ... FROM t2) ** ORDER BY ... COLLATE ... ** -** This transformation is necessary because the multiSelectOrderBy() routine +** This transformation is necessary because the multiSelectByMerge() routine ** above that generates the code for a compound SELECT with an ORDER BY clause ** uses a merge algorithm that requires the same collating sequence on the ** result columns as on the ORDER BY clause. See ticket ** http://sqlite.org/src/info/6709574d2a ** ** This transformation is only needed for EXCEPT, INTERSECT, and UNION. -** The UNION ALL operator works fine with multiSelectOrderBy() even when +** The UNION ALL operator works fine with multiSelectByMerge() even when ** there are COLLATE terms in the ORDER BY. */ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ @@ -6183,7 +6053,7 @@ static int selectExpander(Walker *pWalker, Select *p){ } #ifndef SQLITE_OMIT_VIRTUALTABLE else if( ALWAYS(IsVirtual(pTab)) - && pFrom->fg.fromDDL + && (pFrom->fg.fromDDL || (pParse->prepFlags & SQLITE_PREPARE_FROM_DDL)) && ALWAYS(pTab->u.vtab.p!=0) && pTab->u.vtab.p->eVtabRisk > ((db->flags & SQLITE_TrustedSchema)!=0) ){ @@ -7136,7 +7006,7 @@ static int havingToWhereExprCb(Walker *pWalker, Expr *pExpr){ && pExpr->pAggInfo==0 ){ sqlite3 *db = pWalker->pParse->db; - Expr *pNew = sqlite3Expr(db, TK_INTEGER, "1"); + Expr *pNew = sqlite3ExprInt32(db, 1); if( pNew ){ Expr *pWhere = pS->pWhere; SWAP(Expr, *pNew, *pExpr); @@ -7487,7 +7357,6 @@ static SQLITE_NOINLINE void existsToJoin( ExprSetProperty(pWhere, EP_IntValue); assert( p->pWhere!=0 ); pSub->pSrc->a[0].fg.fromExists = 1; - pSub->pSrc->a[0].fg.jointype |= JT_CROSS; p->pSrc = sqlite3SrcListAppendList(pParse, p->pSrc, pSub->pSrc); if( pSubWhere ){ p->pWhere = sqlite3PExpr(pParse, TK_AND, p->pWhere, pSubWhere); @@ -7515,6 +7384,7 @@ typedef struct CheckOnCtx CheckOnCtx; struct CheckOnCtx { SrcList *pSrc; /* SrcList for this context */ int iJoin; /* Cursor numbers must be =< than this */ + int bFuncArg; /* True for table-function arg */ CheckOnCtx *pParent; /* Parent context */ }; @@ -7566,7 +7436,9 @@ static int selectCheckOnClausesExpr(Walker *pWalker, Expr *pExpr){ if( iTab>=pSrc->a[0].iCursor && iTab<=pSrc->a[pSrc->nSrc-1].iCursor ){ if( pCtx->iJoin && iTab>pCtx->iJoin ){ sqlite3ErrorMsg(pWalker->pParse, - "ON clause references tables to its right"); + "%s references tables to its right", + (pCtx->bFuncArg ? "table-function argument" : "ON clause") + ); return WRC_Abort; } break; @@ -7601,9 +7473,10 @@ static int selectCheckOnClausesSelect(Walker *pWalker, Select *pSelect){ ** Check all ON clauses in pSelect to verify that they do not reference ** columns to the right. */ -static void selectCheckOnClauses(Parse *pParse, Select *pSelect){ +void sqlite3SelectCheckOnClauses(Parse *pParse, Select *pSelect){ Walker w; CheckOnCtx sCtx; + int ii; assert( pSelect->selFlags & SF_OnToWhere ); assert( pSelect->pSrc!=0 && pSelect->pSrc->nSrc>=2 ); memset(&w, 0, sizeof(w)); @@ -7613,8 +7486,46 @@ static void selectCheckOnClauses(Parse *pParse, Select *pSelect){ w.u.pCheckOnCtx = &sCtx; memset(&sCtx, 0, sizeof(sCtx)); sCtx.pSrc = pSelect->pSrc; - sqlite3WalkExprNN(&w, pSelect->pWhere); + sqlite3WalkExpr(&w, pSelect->pWhere); pSelect->selFlags &= ~SF_OnToWhere; + + /* Check for any table-function args that are attached to virtual tables + ** on the RHS of an outer join. They are subject to the same constraints + ** as ON clauses. */ + sCtx.bFuncArg = 1; + for(ii=0; iipSrc->nSrc; ii++){ + SrcItem *pItem = &pSelect->pSrc->a[ii]; + if( pItem->fg.isTabFunc + && (pItem->fg.jointype & JT_OUTER) + ){ + sCtx.iJoin = pItem->iCursor; + sqlite3WalkExprList(&w, pItem->u1.pFuncArg); + } + } +} + +/* +** If p2 exists and p1 and p2 have the same number of terms, then change +** every term of p1 to have the same sort order as p2 and return true. +** +** If p2 is NULL or p1 and p2 are different lengths, then make no changes +** and return false. +** +** p1 must be non-NULL. +*/ +static int sqlite3CopySortOrder(ExprList *p1, ExprList *p2){ + assert( p1 ); + if( p2 && p1->nExpr==p2->nExpr ){ + int ii; + for(ii=0; iinExpr; ii++){ + u8 sortFlags; + sortFlags = p2->a[ii].fg.sortFlags & KEYINFO_ORDER_DESC; + p1->a[ii].fg.sortFlags = sortFlags; + } + return 1; + }else{ + return 0; + } } /* @@ -7712,8 +7623,7 @@ int sqlite3Select( assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_Queue ); if( IgnorableDistinct(pDest) ){ - assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union || - pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard || + assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Discard || pDest->eDest==SRT_DistQueue || pDest->eDest==SRT_DistFifo ); /* All of these destinations are also able to ignore the ORDER BY clause */ if( p->pOrderBy ){ @@ -7729,7 +7639,6 @@ int sqlite3Select( p->pOrderBy = 0; } p->selFlags &= ~(u32)SF_Distinct; - p->selFlags |= SF_NoopOrderBy; } sqlite3SelectPrep(pParse, p, 0); if( pParse->nErr ){ @@ -7744,18 +7653,6 @@ int sqlite3Select( } #endif - /* If the SELECT statement contains ON clauses that were moved into - ** the WHERE clause, go through and verify that none of the terms - ** in the ON clauses reference tables to the right of the ON clause. - ** Do this now, after name resolution, but before query flattening - */ - if( p->selFlags & SF_OnToWhere ){ - selectCheckOnClauses(pParse, p); - if( pParse->nErr ){ - goto select_end; - } - } - /* If the SF_UFSrcCheck flag is set, then this function is being called ** as part of populating the temp table for an UPDATE...FROM statement. ** In this case, it is an error if the target object (pSrc->a[0]) name @@ -8269,7 +8166,8 @@ int sqlite3Select( ** BY and DISTINCT, and an index or separate temp-table for the other. */ if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct - && sqlite3ExprListCompare(sSort.pOrderBy, pEList, -1)==0 + && sqlite3CopySortOrder(pEList, sSort.pOrderBy) + && sqlite3ExprListCompare(pEList, sSort.pOrderBy, -1)==0 && OptimizationEnabled(db, SQLITE_GroupByOrder) #ifndef SQLITE_OMIT_WINDOWFUNC && p->pWin==0 @@ -8483,21 +8381,10 @@ int sqlite3Select( ** but not actually sorted. Either way, record the fact that the ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp ** variable. */ - if( sSort.pOrderBy && pGroupBy->nExpr==sSort.pOrderBy->nExpr ){ - int ii; - /* The GROUP BY processing doesn't care whether rows are delivered in - ** ASC or DESC order - only that each group is returned contiguously. - ** So set the ASC/DESC flags in the GROUP BY to match those in the - ** ORDER BY to maximize the chances of rows being delivered in an - ** order that makes the ORDER BY redundant. */ - for(ii=0; iinExpr; ii++){ - u8 sortFlags; - sortFlags = sSort.pOrderBy->a[ii].fg.sortFlags & KEYINFO_ORDER_DESC; - pGroupBy->a[ii].fg.sortFlags = sortFlags; - } - if( sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 ){ - orderByGrp = 1; - } + if( sqlite3CopySortOrder(pGroupBy, sSort.pOrderBy) + && sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 + ){ + orderByGrp = 1; } }else{ assert( 0==sqlite3LogEst(1) ); diff --git a/src/shell.c.in b/src/shell.c.in index bd4483ff7..ef30194fa 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file contains code to implement the "sqlite" command line +** This file contains code to implement the "sqlite3" command line ** utility for accessing SQLite databases. */ #if (defined(_WIN32) || defined(WIN32)) && !defined(_CRT_SECURE_NO_WARNINGS) @@ -31,14 +31,6 @@ typedef unsigned short int u16; # include SHELL_STRINGIFY(SQLITE_CUSTOM_INCLUDE) #endif -/* -** Determine if we are dealing with WinRT, which provides only a subset of -** the full Win32 API. -*/ -#if !defined(SQLITE_OS_WINRT) -# define SQLITE_OS_WINRT 0 -#endif - /* ** If SQLITE_SHELL_FIDDLE is defined then the shell is modified ** somewhat for use as a WASM module in a web browser. This flag @@ -47,6 +39,10 @@ typedef unsigned short int u16; ** and this build mode rewires the user input subsystem to account for ** that. */ +#if defined(SQLITE_SHELL_FIDDLE) +# undef SQLITE_OMIT_LOAD_EXTENSION +# define SQLITE_OMIT_LOAD_EXTENSION 1 +#endif /* ** Warning pragmas copied from msvc.h in the core. @@ -108,6 +104,7 @@ typedef unsigned char u8; #include #ifndef _WIN32 # include +# include #endif #if !defined(_WIN32) && !defined(WIN32) @@ -177,9 +174,6 @@ typedef unsigned char u8; #endif #if defined(_WIN32) || defined(WIN32) -# if SQLITE_OS_WINRT -# define SQLITE_OMIT_POPEN 1 -# else # include # include # define isatty(h) _isatty(h) @@ -194,7 +188,6 @@ typedef unsigned char u8; # endif # undef pclose # define pclose _pclose -# endif #else /* Make sure isatty() has a prototype. */ extern int isatty(int); @@ -225,9 +218,6 @@ typedef unsigned char u8; #define IsAlpha(X) isalpha((unsigned char)X) #if defined(_WIN32) || defined(WIN32) -#if SQLITE_OS_WINRT -#include -#endif #undef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #include @@ -239,6 +229,8 @@ extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); INCLUDE ../ext/misc/sqlite3_stdio.h INCLUDE ../ext/misc/sqlite3_stdio.c +INCLUDE ../ext/qrf/qrf.h +INCLUDE ../ext/qrf/qrf.c /* Use console I/O package as a direct INCLUDE. */ #define SQLITE_INTERNAL_LINKAGE static @@ -252,11 +244,67 @@ INCLUDE ../ext/misc/sqlite3_stdio.c # define SQLITE_CIO_NO_FLUSH #endif -#define eputz(z) sqlite3_fputs(z,stderr) -#define sputz(fp,z) sqlite3_fputs(z,fp) +/* +** Output routines that are able to redirect to memory rather than +** doing actually I/O. +** Works like. +** -------------- +** cli_printf(FILE*, const char*, ...); fprintf() +** cli_puts(const char*, FILE*); fputs() +** cli_vprintf(FILE*, const char*, va_list); vfprintf() +** +** These are just thin wrappers with the following added semantics: +** If the file-scope variable cli_output_capture is not NULL, and +** if the FILE* argument is stdout or stderr, then rather than +** writing to stdout/stdout, append the text to the cli_output_capture +** variable. +** +** The cli_exit(int) routine works like exit() except that it +** first dumps any capture output to stdout. +*/ +static sqlite3_str *cli_output_capture = 0; +static int cli_printf(FILE *out, const char *zFormat, ...){ + va_list ap; + int rc; + va_start(ap,zFormat); + if( cli_output_capture && (out==stdout || out==stderr) ){ + sqlite3_str_vappendf(cli_output_capture, zFormat, ap); + rc = 1; + }else{ + rc = sqlite3_vfprintf(out, zFormat, ap); + } + va_end(ap); + return rc; +} +static int cli_puts(const char *zText, FILE *out){ + if( cli_output_capture && (out==stdout || out==stderr) ){ + sqlite3_str_appendall(cli_output_capture, zText); + return 1; + } + return sqlite3_fputs(zText, out); +} +#if 0 /* Not currently used - available if we need it later */ +static int cli_vprintf(FILE *out, const char *zFormat, va_list ap){ + if( cli_output_capture && (out==stdout || out==stderr) ){ + sqlite3_str_vappendf(cli_output_capture, zFormat, ap); + return 1; + }else{ + return sqlite3_vfprintf(out, zFormat, ap); + } +} +#endif +static void cli_exit(int rc){ + if( cli_output_capture ){ + char *z = sqlite3_str_finish(cli_output_capture); + sqlite3_fputs(z, stdout); + fflush(stdout); + } + exit(rc); +} + -/* True if the timer is enabled */ -static int enableTimer = 0; +#define eputz(z) cli_puts(z,stderr) +#define sputz(fp,z) cli_puts(z,fp) /* A version of strcmp() that works with NULL values */ static int cli_strcmp(const char *a, const char *b){ @@ -274,7 +322,7 @@ static int cli_strncmp(const char *a, const char *b, size_t n){ ** Unix epoch (1970-01-01T00:00:00Z) */ static sqlite3_int64 timeOfDay(void){ -#if defined(_WIN64) +#if defined(_WIN64) && _WIN32_WINNT >= _WIN32_WINNT_WIN8 sqlite3_uint64 t; FILETIME tm; GetSystemTimePreciseAsFileTime(&tm); @@ -302,152 +350,6 @@ static sqlite3_int64 timeOfDay(void){ #endif } -#if !defined(_WIN32) && !defined(WIN32) && !defined(__minux) -#include -#include - -/* VxWorks does not support getrusage() as far as we can determine */ -#if defined(_WRS_KERNEL) || defined(__RTP__) -struct rusage { - struct timeval ru_utime; /* user CPU time used */ - struct timeval ru_stime; /* system CPU time used */ -}; -#define getrusage(A,B) memset(B,0,sizeof(*B)) -#endif - - -/* Saved resource information for the beginning of an operation */ -static struct rusage sBegin; /* CPU time at start */ -static sqlite3_int64 iBegin; /* Wall-clock time at start */ - -/* -** Begin timing an operation -*/ -static void beginTimer(void){ - if( enableTimer ){ - getrusage(RUSAGE_SELF, &sBegin); - iBegin = timeOfDay(); - } -} - -/* Return the difference of two time_structs in seconds */ -static double timeDiff(struct timeval *pStart, struct timeval *pEnd){ - return (pEnd->tv_usec - pStart->tv_usec)*0.000001 + - (double)(pEnd->tv_sec - pStart->tv_sec); -} - -/* -** Print the timing results. -*/ -static void endTimer(FILE *out){ - if( enableTimer ){ - sqlite3_int64 iEnd = timeOfDay(); - struct rusage sEnd; - getrusage(RUSAGE_SELF, &sEnd); - sqlite3_fprintf(out, "Run Time: real %.6f user %.6f sys %.6f\n", - (iEnd - iBegin)*0.000001, - timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), - timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); - } -} - -#define BEGIN_TIMER beginTimer() -#define END_TIMER(X) endTimer(X) -#define HAS_TIMER 1 - -#elif (defined(_WIN32) || defined(WIN32)) - -/* Saved resource information for the beginning of an operation */ -static HANDLE hProcess; -static FILETIME ftKernelBegin; -static FILETIME ftUserBegin; -static sqlite3_int64 ftWallBegin; -typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME, - LPFILETIME, LPFILETIME); -static GETPROCTIMES getProcessTimesAddr = NULL; - -/* -** Check to see if we have timer support. Return 1 if necessary -** support found (or found previously). -*/ -static int hasTimer(void){ - if( getProcessTimesAddr ){ - return 1; - } else { -#if !SQLITE_OS_WINRT - /* GetProcessTimes() isn't supported in WIN95 and some other Windows - ** versions. See if the version we are running on has it, and if it - ** does, save off a pointer to it and the current process handle. - */ - hProcess = GetCurrentProcess(); - if( hProcess ){ - HINSTANCE hinstLib = LoadLibrary(TEXT("Kernel32.dll")); - if( NULL != hinstLib ){ - getProcessTimesAddr = - (GETPROCTIMES) GetProcAddress(hinstLib, "GetProcessTimes"); - if( NULL != getProcessTimesAddr ){ - return 1; - } - FreeLibrary(hinstLib); - } - } -#endif - } - return 0; -} - -/* -** Begin timing an operation -*/ -static void beginTimer(void){ - if( enableTimer && getProcessTimesAddr ){ - FILETIME ftCreation, ftExit; - getProcessTimesAddr(hProcess,&ftCreation,&ftExit, - &ftKernelBegin,&ftUserBegin); - ftWallBegin = timeOfDay(); - } -} - -/* Return the difference of two FILETIME structs in seconds */ -static double timeDiff(FILETIME *pStart, FILETIME *pEnd){ - sqlite_int64 i64Start = *((sqlite_int64 *) pStart); - sqlite_int64 i64End = *((sqlite_int64 *) pEnd); - return (double) ((i64End - i64Start) / 10000000.0); -} - -/* -** Print the timing results. -*/ -static void endTimer(FILE *out){ - if( enableTimer && getProcessTimesAddr){ - FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; - sqlite3_int64 ftWallEnd = timeOfDay(); - getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); -#ifdef _WIN64 - /* microsecond precision on 64-bit windows */ - sqlite3_fprintf(out, "Run Time: real %.6f user %f sys %f\n", - (ftWallEnd - ftWallBegin)*0.000001, - timeDiff(&ftUserBegin, &ftUserEnd), - timeDiff(&ftKernelBegin, &ftKernelEnd)); -#else - /* millisecond precisino on 32-bit windows */ - sqlite3_fprintf(out, "Run Time: real %.3f user %.3f sys %.3f\n", - (ftWallEnd - ftWallBegin)*0.000001, - timeDiff(&ftUserBegin, &ftUserEnd), - timeDiff(&ftKernelBegin, &ftKernelEnd)); -#endif - } -} - -#define BEGIN_TIMER beginTimer() -#define END_TIMER(X) endTimer(X) -#define HAS_TIMER hasTimer() - -#else -#define BEGIN_TIMER -#define END_TIMER(X) /*no-op*/ -#define HAS_TIMER 0 -#endif /* ** Used to prevent warnings about unused parameters @@ -472,12 +374,17 @@ static int bail_on_error = 0; static int stdin_is_interactive = 1; /* -** On Windows systems we need to know if standard output is a console -** in order to show that UTF-16 translation is done in the sign-on -** banner. The following variable is true if it is the console. +** Treat stdout like a TTY if true. */ static int stdout_is_console = 1; +/* +** Use this value as the width of the output device. Or, figure it +** out at runtime if the value is negative. Or use a default width +** if this value is zero. +*/ +static int stdout_tty_width = -1; + /* ** The following is the open SQLite database. We make a pointer ** to this database a static variable so that it can be accessed @@ -506,6 +413,14 @@ static char mainPrompt[PROMPT_LEN_MAX]; /* Continuation prompt. default: " ...> " */ static char continuePrompt[PROMPT_LEN_MAX]; +/* +** Write I/O traces to the following stream. +*/ +#ifdef SQLITE_ENABLE_IOTRACE +static FILE *iotrace = 0; +#endif + + /* This is variant of the standard-library strncpy() routine with the ** one change that the destination string is always zero-terminated, even ** if there is no zero-terminator in the first n-1 characters of the source @@ -615,7 +530,7 @@ static char *dynamicContinuePrompt(void){ /* Indicate out-of-memory and exit. */ static void shell_out_of_memory(void){ eputz("Error: out of memory\n"); - exit(1); + cli_exit(1); } /* Check a pointer to see if it is NULL. If it is NULL, exit with an @@ -625,13 +540,6 @@ static void shell_check_oom(const void *p){ if( p==0 ) shell_out_of_memory(); } -/* -** Write I/O traces to the following stream. -*/ -#ifdef SQLITE_ENABLE_IOTRACE -static FILE *iotrace = 0; -#endif - /* ** This routine works like printf in that its first argument is a ** format string and subsequent arguments are values to be substituted @@ -646,349 +554,67 @@ static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){ va_start(ap, zFormat); z = sqlite3_vmprintf(zFormat, ap); va_end(ap); - sqlite3_fprintf(iotrace, "%s", z); + cli_printf(iotrace, "%s", z); sqlite3_free(z); } #endif -/* Lookup table to estimate the number of columns consumed by a Unicode -** character. -*/ -static const struct { - unsigned char w; /* Width of the character in columns */ - int iFirst; /* First character in a span having this width */ -} aUWidth[] = { - /* {1, 0x00000}, */ - {0, 0x00300}, {1, 0x00370}, {0, 0x00483}, {1, 0x00487}, {0, 0x00488}, - {1, 0x0048a}, {0, 0x00591}, {1, 0x005be}, {0, 0x005bf}, {1, 0x005c0}, - {0, 0x005c1}, {1, 0x005c3}, {0, 0x005c4}, {1, 0x005c6}, {0, 0x005c7}, - {1, 0x005c8}, {0, 0x00600}, {1, 0x00604}, {0, 0x00610}, {1, 0x00616}, - {0, 0x0064b}, {1, 0x0065f}, {0, 0x00670}, {1, 0x00671}, {0, 0x006d6}, - {1, 0x006e5}, {0, 0x006e7}, {1, 0x006e9}, {0, 0x006ea}, {1, 0x006ee}, - {0, 0x0070f}, {1, 0x00710}, {0, 0x00711}, {1, 0x00712}, {0, 0x00730}, - {1, 0x0074b}, {0, 0x007a6}, {1, 0x007b1}, {0, 0x007eb}, {1, 0x007f4}, - {0, 0x00901}, {1, 0x00903}, {0, 0x0093c}, {1, 0x0093d}, {0, 0x00941}, - {1, 0x00949}, {0, 0x0094d}, {1, 0x0094e}, {0, 0x00951}, {1, 0x00955}, - {0, 0x00962}, {1, 0x00964}, {0, 0x00981}, {1, 0x00982}, {0, 0x009bc}, - {1, 0x009bd}, {0, 0x009c1}, {1, 0x009c5}, {0, 0x009cd}, {1, 0x009ce}, - {0, 0x009e2}, {1, 0x009e4}, {0, 0x00a01}, {1, 0x00a03}, {0, 0x00a3c}, - {1, 0x00a3d}, {0, 0x00a41}, {1, 0x00a43}, {0, 0x00a47}, {1, 0x00a49}, - {0, 0x00a4b}, {1, 0x00a4e}, {0, 0x00a70}, {1, 0x00a72}, {0, 0x00a81}, - {1, 0x00a83}, {0, 0x00abc}, {1, 0x00abd}, {0, 0x00ac1}, {1, 0x00ac6}, - {0, 0x00ac7}, {1, 0x00ac9}, {0, 0x00acd}, {1, 0x00ace}, {0, 0x00ae2}, - {1, 0x00ae4}, {0, 0x00b01}, {1, 0x00b02}, {0, 0x00b3c}, {1, 0x00b3d}, - {0, 0x00b3f}, {1, 0x00b40}, {0, 0x00b41}, {1, 0x00b44}, {0, 0x00b4d}, - {1, 0x00b4e}, {0, 0x00b56}, {1, 0x00b57}, {0, 0x00b82}, {1, 0x00b83}, - {0, 0x00bc0}, {1, 0x00bc1}, {0, 0x00bcd}, {1, 0x00bce}, {0, 0x00c3e}, - {1, 0x00c41}, {0, 0x00c46}, {1, 0x00c49}, {0, 0x00c4a}, {1, 0x00c4e}, - {0, 0x00c55}, {1, 0x00c57}, {0, 0x00cbc}, {1, 0x00cbd}, {0, 0x00cbf}, - {1, 0x00cc0}, {0, 0x00cc6}, {1, 0x00cc7}, {0, 0x00ccc}, {1, 0x00cce}, - {0, 0x00ce2}, {1, 0x00ce4}, {0, 0x00d41}, {1, 0x00d44}, {0, 0x00d4d}, - {1, 0x00d4e}, {0, 0x00dca}, {1, 0x00dcb}, {0, 0x00dd2}, {1, 0x00dd5}, - {0, 0x00dd6}, {1, 0x00dd7}, {0, 0x00e31}, {1, 0x00e32}, {0, 0x00e34}, - {1, 0x00e3b}, {0, 0x00e47}, {1, 0x00e4f}, {0, 0x00eb1}, {1, 0x00eb2}, - {0, 0x00eb4}, {1, 0x00eba}, {0, 0x00ebb}, {1, 0x00ebd}, {0, 0x00ec8}, - {1, 0x00ece}, {0, 0x00f18}, {1, 0x00f1a}, {0, 0x00f35}, {1, 0x00f36}, - {0, 0x00f37}, {1, 0x00f38}, {0, 0x00f39}, {1, 0x00f3a}, {0, 0x00f71}, - {1, 0x00f7f}, {0, 0x00f80}, {1, 0x00f85}, {0, 0x00f86}, {1, 0x00f88}, - {0, 0x00f90}, {1, 0x00f98}, {0, 0x00f99}, {1, 0x00fbd}, {0, 0x00fc6}, - {1, 0x00fc7}, {0, 0x0102d}, {1, 0x01031}, {0, 0x01032}, {1, 0x01033}, - {0, 0x01036}, {1, 0x01038}, {0, 0x01039}, {1, 0x0103a}, {0, 0x01058}, - {1, 0x0105a}, {2, 0x01100}, {0, 0x01160}, {1, 0x01200}, {0, 0x0135f}, - {1, 0x01360}, {0, 0x01712}, {1, 0x01715}, {0, 0x01732}, {1, 0x01735}, - {0, 0x01752}, {1, 0x01754}, {0, 0x01772}, {1, 0x01774}, {0, 0x017b4}, - {1, 0x017b6}, {0, 0x017b7}, {1, 0x017be}, {0, 0x017c6}, {1, 0x017c7}, - {0, 0x017c9}, {1, 0x017d4}, {0, 0x017dd}, {1, 0x017de}, {0, 0x0180b}, - {1, 0x0180e}, {0, 0x018a9}, {1, 0x018aa}, {0, 0x01920}, {1, 0x01923}, - {0, 0x01927}, {1, 0x01929}, {0, 0x01932}, {1, 0x01933}, {0, 0x01939}, - {1, 0x0193c}, {0, 0x01a17}, {1, 0x01a19}, {0, 0x01b00}, {1, 0x01b04}, - {0, 0x01b34}, {1, 0x01b35}, {0, 0x01b36}, {1, 0x01b3b}, {0, 0x01b3c}, - {1, 0x01b3d}, {0, 0x01b42}, {1, 0x01b43}, {0, 0x01b6b}, {1, 0x01b74}, - {0, 0x01dc0}, {1, 0x01dcb}, {0, 0x01dfe}, {1, 0x01e00}, {0, 0x0200b}, - {1, 0x02010}, {0, 0x0202a}, {1, 0x0202f}, {0, 0x02060}, {1, 0x02064}, - {0, 0x0206a}, {1, 0x02070}, {0, 0x020d0}, {1, 0x020f0}, {2, 0x02329}, - {1, 0x0232b}, {2, 0x02e80}, {0, 0x0302a}, {2, 0x03030}, {1, 0x0303f}, - {2, 0x03040}, {0, 0x03099}, {2, 0x0309b}, {1, 0x0a4d0}, {0, 0x0a806}, - {1, 0x0a807}, {0, 0x0a80b}, {1, 0x0a80c}, {0, 0x0a825}, {1, 0x0a827}, - {2, 0x0ac00}, {1, 0x0d7a4}, {2, 0x0f900}, {1, 0x0fb00}, {0, 0x0fb1e}, - {1, 0x0fb1f}, {0, 0x0fe00}, {2, 0x0fe10}, {1, 0x0fe1a}, {0, 0x0fe20}, - {1, 0x0fe24}, {2, 0x0fe30}, {1, 0x0fe70}, {0, 0x0feff}, {2, 0x0ff00}, - {1, 0x0ff61}, {2, 0x0ffe0}, {1, 0x0ffe7}, {0, 0x0fff9}, {1, 0x0fffc}, - {0, 0x10a01}, {1, 0x10a04}, {0, 0x10a05}, {1, 0x10a07}, {0, 0x10a0c}, - {1, 0x10a10}, {0, 0x10a38}, {1, 0x10a3b}, {0, 0x10a3f}, {1, 0x10a40}, - {0, 0x1d167}, {1, 0x1d16a}, {0, 0x1d173}, {1, 0x1d183}, {0, 0x1d185}, - {1, 0x1d18c}, {0, 0x1d1aa}, {1, 0x1d1ae}, {0, 0x1d242}, {1, 0x1d245}, - {2, 0x20000}, {1, 0x2fffe}, {2, 0x30000}, {1, 0x3fffe}, {0, 0xe0001}, - {1, 0xe0002}, {0, 0xe0020}, {1, 0xe0080}, {0, 0xe0100}, {1, 0xe01f0} -}; - -/* -** Return an estimate of the width, in columns, for the single Unicode -** character c. For normal characters, the answer is always 1. But the -** estimate might be 0 or 2 for zero-width and double-width characters. -** -** Different display devices display unicode using different widths. So -** it is impossible to know that true display width with 100% accuracy. -** Inaccuracies in the width estimates might cause columns to be misaligned. -** Unfortunately, there is nothing we can do about that. -*/ -int cli_wcwidth(int c){ - int iFirst, iLast; - - /* Fast path for common characters */ - if( c<=0x300 ) return 1; - - /* The general case */ - iFirst = 0; - iLast = sizeof(aUWidth)/sizeof(aUWidth[0]) - 1; - while( iFirst c ){ - iLast = iMid - 1; - }else{ - return aUWidth[iMid].w; - } - } - if( aUWidth[iLast].iFirst > c ) return aUWidth[iFirst].w; - return aUWidth[iLast].w; -} - /* -** Compute the value and length of a multi-byte UTF-8 character that -** begins at z[0]. Return the length. Write the Unicode value into *pU. -** -** This routine only works for *multi-byte* UTF-8 characters. +** Compute a string length that is limited to what can be stored in +** lower 30 bits of a 32-bit signed integer. */ -static int decodeUtf8(const unsigned char *z, int *pU){ - if( (z[0] & 0xe0)==0xc0 && (z[1] & 0xc0)==0x80 ){ - *pU = ((z[0] & 0x1f)<<6) | (z[1] & 0x3f); - return 2; - } - if( (z[0] & 0xf0)==0xe0 && (z[1] & 0xc0)==0x80 && (z[2] & 0xc0)==0x80 ){ - *pU = ((z[0] & 0x0f)<<12) | ((z[1] & 0x3f)<<6) | (z[2] & 0x3f); - return 3; - } - if( (z[0] & 0xf8)==0xf0 && (z[1] & 0xc0)==0x80 && (z[2] & 0xc0)==0x80 - && (z[3] & 0xc0)==0x80 - ){ - *pU = ((z[0] & 0x0f)<<18) | ((z[1] & 0x3f)<<12) | ((z[2] & 0x3f))<<6 - | (z[3] & 0x3f); - return 4; - } - *pU = 0; - return 1; +static int strlen30(const char *z){ + size_t n; + if( z==0 ) return 0; + n = strlen(z); + return n>0x3fffffff ? 0x3fffffff : (int)n; } - -#if 0 /* NOT USED */ /* -** Return the width, in display columns, of a UTF-8 string. -** -** Each normal character counts as 1. Zero-width characters count -** as zero, and double-width characters count as 2. +** Return open FILE * if zFile exists, can be opened for read +** and is an ordinary file or a character stream source. +** Otherwise return 0. */ -int cli_wcswidth(const char *z){ - const unsigned char *a = (const unsigned char*)z; - int n = 0; - int i = 0; - unsigned char c; - while( (c = a[i])!=0 ){ - if( c>=0xc0 ){ - int u; - int len = decodeUtf8(&a[i], &u); - i += len; - n += cli_wcwidth(u); - }else if( c>=' ' ){ - n++; - i++; - }else{ - i++; - } +static FILE * openChrSource(const char *zFile){ +#if defined(_WIN32) || defined(WIN32) + struct __stat64 x = {0}; +# define STAT_CHR_SRC(mode) ((mode & (_S_IFCHR|_S_IFIFO|_S_IFREG))!=0) + /* On Windows, open first, then check the stream nature. This order + ** is necessary because _stat() and sibs, when checking a named pipe, + ** effectively break the pipe as its supplier sees it. */ + FILE *rv = sqlite3_fopen(zFile, "rb"); + if( rv==0 ) return 0; + if( _fstat64(_fileno(rv), &x) != 0 + || !STAT_CHR_SRC(x.st_mode)){ + fclose(rv); + rv = 0; + } + return rv; +#else + struct stat x = {0}; + int rc = stat(zFile, &x); +# define STAT_CHR_SRC(mode) (S_ISREG(mode)||S_ISFIFO(mode)||S_ISCHR(mode)) + if( rc!=0 ) return 0; + if( STAT_CHR_SRC(x.st_mode) ){ + return sqlite3_fopen(zFile, "rb"); + }else{ + return 0; } - return n; -} #endif - -/* -** Check to see if z[] is a valid VT100 escape. If it is, then -** return the number of bytes in the escape sequence. Return 0 if -** z[] is not a VT100 escape. -** -** This routine assumes that z[0] is \033 (ESC). -*/ -static int isVt100(const unsigned char *z){ - int i; - if( z[1]!='[' ) return 0; - i = 2; - while( z[i]>=0x30 && z[i]<=0x3f ){ i++; } - while( z[i]>=0x20 && z[i]<=0x2f ){ i++; } - if( z[i]<0x40 || z[i]>0x7e ) return 0; - return i+1; +#undef STAT_CHR_SRC } /* -** Output string zUtf to stdout as w characters. If w is negative, -** then right-justify the text. W is the width in UTF-8 characters, not -** in bytes. This is different from the %*.*s specification in printf -** since with %*.*s the width is measured in bytes, not characters. -** -** Take into account zero-width and double-width Unicode characters. -** In other words, a zero-width character does not count toward the -** the w limit. A double-width character counts as two. +** This routine reads a line of text from FILE in, stores +** the text in memory obtained from malloc() and returns a pointer +** to the text. NULL is returned at end of file, or if malloc() +** fails, or if the length of the line is longer than about a gigabyte. ** -** w should normally be a small number. A couple hundred at most. This -** routine caps w at 100 million to avoid integer overflow issues. +** If zLine is not NULL then it is a malloced buffer returned from +** a previous call to this routine that may be reused. */ -static void utf8_width_print(FILE *out, int w, const char *zUtf){ - const unsigned char *a = (const unsigned char*)zUtf; - static const int mxW = 10000000; - unsigned char c; - int i = 0; - int n = 0; - int k; - int aw; - if( w<-mxW ){ - w = -mxW; - }else if( w>mxW ){ - w= mxW; - } - aw = w<0 ? -w : w; - if( zUtf==0 ) zUtf = ""; - while( (c = a[i])!=0 ){ - if( (c&0xc0)==0xc0 ){ - int u; - int len = decodeUtf8(a+i, &u); - int x = cli_wcwidth(u); - if( x+n>aw ){ - break; - } - i += len; - n += x; - }else if( c==0x1b && (k = isVt100(&a[i]))>0 ){ - i += k; - }else if( n>=aw ){ - break; - }else{ - n++; - i++; - } - } - if( n>=aw ){ - sqlite3_fprintf(out, "%.*s", i, zUtf); - }else if( w<0 ){ - sqlite3_fprintf(out, "%*s%s", aw-n, "", zUtf); - }else{ - sqlite3_fprintf(out, "%s%*s", zUtf, aw-n, ""); - } -} - - -/* -** Determines if a string is a number of not. -*/ -static int isNumber(const char *z, int *realnum){ - if( *z=='-' || *z=='+' ) z++; - if( !IsDigit(*z) ){ - return 0; - } - z++; - if( realnum ) *realnum = 0; - while( IsDigit(*z) ){ z++; } - if( *z=='.' ){ - z++; - if( !IsDigit(*z) ) return 0; - while( IsDigit(*z) ){ z++; } - if( realnum ) *realnum = 1; - } - if( *z=='e' || *z=='E' ){ - z++; - if( *z=='+' || *z=='-' ) z++; - if( !IsDigit(*z) ) return 0; - while( IsDigit(*z) ){ z++; } - if( realnum ) *realnum = 1; - } - return *z==0; -} - -/* -** Compute a string length that is limited to what can be stored in -** lower 30 bits of a 32-bit signed integer. -*/ -static int strlen30(const char *z){ - const char *z2 = z; - while( *z2 ){ z2++; } - return 0x3fffffff & (int)(z2 - z); -} - -/* -** Return the length of a string in characters. Multibyte UTF8 characters -** count as a single character for single-width characters, or as two -** characters for double-width characters. -*/ -static int strlenChar(const char *z){ - int n = 0; - while( *z ){ - if( (0x80&z[0])==0 ){ - n++; - z++; - }else{ - int u = 0; - int len = decodeUtf8((const u8*)z, &u); - z += len; - n += cli_wcwidth(u); - } - } - return n; -} - -/* -** Return open FILE * if zFile exists, can be opened for read -** and is an ordinary file or a character stream source. -** Otherwise return 0. -*/ -static FILE * openChrSource(const char *zFile){ -#if defined(_WIN32) || defined(WIN32) - struct __stat64 x = {0}; -# define STAT_CHR_SRC(mode) ((mode & (_S_IFCHR|_S_IFIFO|_S_IFREG))!=0) - /* On Windows, open first, then check the stream nature. This order - ** is necessary because _stat() and sibs, when checking a named pipe, - ** effectively break the pipe as its supplier sees it. */ - FILE *rv = sqlite3_fopen(zFile, "rb"); - if( rv==0 ) return 0; - if( _fstat64(_fileno(rv), &x) != 0 - || !STAT_CHR_SRC(x.st_mode)){ - fclose(rv); - rv = 0; - } - return rv; -#else - struct stat x = {0}; - int rc = stat(zFile, &x); -# define STAT_CHR_SRC(mode) (S_ISREG(mode)||S_ISFIFO(mode)||S_ISCHR(mode)) - if( rc!=0 ) return 0; - if( STAT_CHR_SRC(x.st_mode) ){ - return sqlite3_fopen(zFile, "rb"); - }else{ - return 0; - } -#endif -#undef STAT_CHR_SRC -} - -/* -** This routine reads a line of text from FILE in, stores -** the text in memory obtained from malloc() and returns a pointer -** to the text. NULL is returned at end of file, or if malloc() -** fails, or if the length of the line is longer than about a gigabyte. -** -** If zLine is not NULL then it is a malloced buffer returned from -** a previous call to this routine that may be reused. -*/ -static char *local_getline(char *zLine, FILE *in){ - int nLine = zLine==0 ? 0 : 100; +static char *local_getline(char *zLine, FILE *in){ + int nLine = zLine==0 ? 0 : 100; int n = 0; while( 1 ){ @@ -1305,7 +931,7 @@ static void shellDtostr( char z[400]; if( n<1 ) n = 1; if( n>350 ) n = 350; - sqlite3_snprintf(sizeof(z), z, "%#+.*e", n, r); + sprintf(z, "%#+.*e", n, r); sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT); } @@ -1463,31 +1089,24 @@ struct ExpertInfo { }; #endif -/* A single line in the EQP output */ -typedef struct EQPGraphRow EQPGraphRow; -struct EQPGraphRow { - int iEqpId; /* ID for this row */ - int iParentId; /* ID of the parent row */ - EQPGraphRow *pNext; /* Next row in sequence */ - char zText[1]; /* Text to display for this row */ -}; +/* All the parameters that determine how to render query results. +*/ +typedef struct Mode { + u8 autoExplain; /* Automatically turn on .explain mode */ + u8 autoEQP; /* Run EXPLAIN QUERY PLAN prior to each SQL stmt */ + u8 autoEQPtrace; /* autoEQP is in trace mode */ + u8 scanstatsOn; /* True to display scan stats before each finalize */ + u8 bAutoScreenWidth; /* Using the TTY to determine screen width */ + u8 mFlags; /* MFLG_ECHO, MFLG_CRLF, etc. */ + u8 eMode; /* One of the MODE_ values */ + sqlite3_qrf_spec spec; /* Spec to be passed into QRF */ +} Mode; -/* All EQP output is collected into an instance of the following */ -typedef struct EQPGraph EQPGraph; -struct EQPGraph { - EQPGraphRow *pRow; /* Linked list of all rows of the EQP output */ - EQPGraphRow *pLast; /* Last element of the pRow list */ - char zPrefix[100]; /* Graph prefix */ -}; +/* Flags for Mode.mFlags */ +#define MFLG_ECHO 0x01 /* Echo inputs to output */ +#define MFLG_CRLF 0x02 /* Use CR/LF output line endings */ +#define MFLG_HDR 0x04 /* .header used to change headers on/off */ -/* Parameters affecting columnar mode result display (defaulting together) */ -typedef struct ColModeOpts { - int iWrap; /* In columnar modes, wrap lines reaching this limit */ - u8 bQuote; /* Quote results for .mode box and table */ - u8 bWordWrap; /* In columnar modes, wrap at word boundaries */ -} ColModeOpts; -#define ColModeOpts_default { 60, 0, 0 } -#define ColModeOpts_default_qbox { 60, 1, 0 } /* ** State information about the database connection is contained in an @@ -1496,11 +1115,6 @@ typedef struct ColModeOpts { typedef struct ShellState ShellState; struct ShellState { sqlite3 *db; /* The database */ - u8 autoExplain; /* Automatically turn on .explain mode */ - u8 autoEQP; /* Run EXPLAIN QUERY PLAN prior to each SQL stmt */ - u8 autoEQPtest; /* autoEQP is in test mode */ - u8 autoEQPtrace; /* autoEQP is in trace mode */ - u8 scanstatsOn; /* True to display scan stats before each finalize */ u8 openMode; /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */ u8 doXdgOpen; /* Invoke start/open/xdg-open in output_reset() */ u8 nEqpLevel; /* Depth of the EQP output graph */ @@ -1508,48 +1122,44 @@ struct ShellState { u8 bSafeMode; /* True to prohibit unsafe operations */ u8 bSafeModePersist; /* The long-term value of bSafeMode */ u8 eRestoreState; /* See comments above doAutoDetectRestore() */ - u8 crlfMode; /* Do NL-to-CRLF translations when enabled (maybe) */ - u8 eEscMode; /* Escape mode for text output */ - ColModeOpts cmOpts; /* Option values affecting columnar mode output */ unsigned statsOn; /* True to display memory stats before each finalize */ unsigned mEqpLines; /* Mask of vertical lines in the EQP output graph */ + u8 nPopOutput; /* Revert .output settings when reaching zero */ + u8 nPopMode; /* Revert .mode settings when reaching zero */ + u8 enableTimer; /* Enable the timer. 2: permanently 1: only once */ int inputNesting; /* Track nesting level of .read and other redirects */ - int outCount; /* Revert to stdout when reaching zero */ - int cnt; /* Number of records displayed so far */ + double prevTimer; /* Last reported timer value */ + double tmProgress; /* --timeout option for .progress */ i64 lineno; /* Line number of last line read from in */ + const char *zInFile; /* Name of the input file */ int openFlags; /* Additional flags to open. (SQLITE_OPEN_NOFOLLOW) */ FILE *in; /* Read commands from this stream */ FILE *out; /* Write results here */ FILE *traceOut; /* Output for sqlite3_trace() */ int nErr; /* Number of errors seen */ - int mode; /* An output mode setting */ - int modePrior; /* Saved mode */ - int cMode; /* temporary output mode for the current query */ - int normalMode; /* Output mode before ".explain on" */ int writableSchema; /* True if PRAGMA writable_schema=ON */ - int showHeader; /* True to show column names in List or Column mode */ int nCheck; /* Number of ".check" commands run */ unsigned nProgress; /* Number of progress callbacks encountered */ unsigned mxProgress; /* Maximum progress callbacks before failing */ unsigned flgProgress; /* Flags for the progress callback */ unsigned shellFlgs; /* Various flags */ - unsigned priorShFlgs; /* Saved copy of flags */ + unsigned nTestRun; /* Number of test cases run */ + unsigned nTestErr; /* Number of test cases that failed */ sqlite3_int64 szMax; /* --maxsize argument to .open */ char *zDestTable; /* Name of destination table when MODE_Insert */ char *zTempFile; /* Temporary file that might need deleting */ + char *zErrPrefix; /* Alternative error message prefix */ char zTestcase[30]; /* Name of current test case */ - char colSeparator[20]; /* Column separator character for several modes */ - char rowSeparator[20]; /* Row separator character for MODE_Ascii */ - char colSepPrior[20]; /* Saved column separator */ - char rowSepPrior[20]; /* Saved row separator */ - int *colWidth; /* Requested width of each column in columnar modes */ - int *actualWidth; /* Actual width of each column */ - int nWidth; /* Number of slots in colWidth[] and actualWidth[] */ - char nullValue[20]; /* The text to print when a NULL comes back from - ** the database */ char outfile[FILENAME_MAX]; /* Filename for *out */ sqlite3_stmt *pStmt; /* Current statement if any. */ FILE *pLog; /* Write log output here */ + Mode mode; /* Current display mode */ + Mode modePrior; /* Backup */ + struct SavedMode { /* Ability to define custom mode configurations */ + char *zTag; /* Name of this saved mode */ + Mode mode; /* The saved mode */ + } *aSavedModes; /* Array of saved .mode settings. system malloc() */ + int nSavedModes; /* Number of saved .mode settings */ struct AuxDb { /* Storage space for auxiliary database connections */ sqlite3 *db; /* Connection pointer */ const char *zDbFilename; /* Filename used to open the connection */ @@ -1560,14 +1170,19 @@ struct ShellState { #endif } aAuxDb[5], /* Array of all database connections */ *pAuxDb; /* Currently active database connection */ - int *aiIndent; /* Array of indents used in MODE_Explain */ - int nIndent; /* Size of array aiIndent[] */ - int iIndent; /* Index of current op in aiIndent[] */ char *zNonce; /* Nonce for temporary safe-mode escapes */ - EQPGraph sGraph; /* Information for the graphical EXPLAIN QUERY PLAN */ #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) ExpertInfo expert; /* Valid if previous command was ".expert OPT..." */ #endif + struct DotCmdLine { /* Info about arguments to a dot-command */ + const char *zOrig; /* Original text of the dot-command */ + char *zCopy; /* Copy of zOrig, from malloc() */ + int nAlloc; /* Size of allocates for arrays below */ + int nArg; /* Number of argument slots actually used */ + char **azArg; /* Pointer to each argument, dequoted */ + int *aiOfst; /* Offset into zOrig[] for start of each arg */ + char *abQuot; /* True if the argment was originally quoted */ + } dot; #ifdef SQLITE_SHELL_FIDDLE struct { const char * zInput; /* Input string from wasm/JS proxy */ @@ -1582,7 +1197,7 @@ static ShellState shellState; #endif -/* Allowed values for ShellState.autoEQP +/* Allowed values for ShellState.mode.autoEQP */ #define AUTOEQP_off 0 /* Automatic EXPLAIN QUERY PLAN is off */ #define AUTOEQP_on 1 /* Automatic EQP is on */ @@ -1610,15 +1225,13 @@ static ShellState shellState; ** callback limit is reached, and for each ** top-level SQL statement */ #define SHELL_PROGRESS_ONCE 0x04 /* Cancel the --limit after firing once */ +#define SHELL_PROGRESS_TMOUT 0x08 /* Stop after tmProgress seconds */ -/* Allowed values for ShellState.eEscMode. The default value should -** be 0, so to change the default, reorder the names. +/* Names of values for Mode.spec.eEsc and Mode.spec.eText */ -#define SHELL_ESC_ASCII 0 /* Substitute ^Y for X where Y=X+0x40 */ -#define SHELL_ESC_SYMBOL 1 /* Substitute U+2400 graphics */ -#define SHELL_ESC_OFF 2 /* Send characters verbatim */ - -static const char *shell_EscModeNames[] = { "ascii", "symbol", "off" }; +static const char *qrfEscNames[] = { "auto", "off", "ascii", "symbol" }; +static const char *qrfQuoteNames[] = + { "off","off","sql","hex","csv","tcl","json","relaxed"}; /* ** These are the allowed shellFlgs values @@ -1627,10 +1240,8 @@ static const char *shell_EscModeNames[] = { "ascii", "symbol", "off" }; #define SHFLG_Lookaside 0x00000002 /* Lookaside memory is used */ #define SHFLG_Backslash 0x00000004 /* The --backslash option is used */ #define SHFLG_PreserveRowid 0x00000008 /* .dump preserves rowid values */ -#define SHFLG_Newlines 0x00000010 /* .dump --newline flag */ +#define SHFLG_NoErrLineno 0x00000010 /* Omit line numbers from error msgs */ #define SHFLG_CountChanges 0x00000020 /* .changes setting */ -#define SHFLG_Echo 0x00000040 /* .echo on/off, or --echo setting */ -#define SHFLG_HeaderSet 0x00000080 /* showHeader has been specified */ #define SHFLG_DumpDataOnly 0x00000100 /* .dump show data only */ #define SHFLG_DumpNoSys 0x00000200 /* .dump omits system tables */ #define SHFLG_TestingMode 0x00000400 /* allow unsafe testing features */ @@ -1643,54 +1254,107 @@ static const char *shell_EscModeNames[] = { "ascii", "symbol", "off" }; #define ShellClearFlag(P,X) ((P)->shellFlgs&=(~(X))) /* -** These are the allowed modes. -*/ -#define MODE_Line 0 /* One column per line. Blank line between records */ -#define MODE_Column 1 /* One record per line in neat columns */ -#define MODE_List 2 /* One record per line with a separator */ -#define MODE_Semi 3 /* Same as MODE_List but append ";" to each line */ -#define MODE_Html 4 /* Generate an XHTML table */ -#define MODE_Insert 5 /* Generate SQL "insert" statements */ -#define MODE_Quote 6 /* Quote values as for SQL */ -#define MODE_Tcl 7 /* Generate ANSI-C or TCL quoted elements */ -#define MODE_Csv 8 /* Quote strings, numbers are plain */ -#define MODE_Explain 9 /* Like MODE_Column, but do not truncate data */ -#define MODE_Ascii 10 /* Use ASCII unit and record separators (0x1F/0x1E) */ -#define MODE_Pretty 11 /* Pretty-print schemas */ -#define MODE_EQP 12 /* Converts EXPLAIN QUERY PLAN output into a graph */ -#define MODE_Json 13 /* Output JSON */ -#define MODE_Markdown 14 /* Markdown formatting */ -#define MODE_Table 15 /* MySQL-style table formatting */ -#define MODE_Box 16 /* Unicode box-drawing characters */ -#define MODE_Count 17 /* Output only a count of the rows of output */ -#define MODE_Off 18 /* No query output shown */ -#define MODE_ScanExp 19 /* Like MODE_Explain, but for ".scanstats vm" */ -#define MODE_Www 20 /* Full web-page output */ - -static const char *modeDescr[] = { - "line", - "column", - "list", - "semi", - "html", - "insert", - "quote", - "tcl", - "csv", - "explain", - "ascii", - "prettyprint", - "eqp", - "json", - "markdown", - "table", - "box", - "count", - "off", - "scanexp", - "www", +** These are the allowed values for Mode.eMode. There is a lot of overlap +** between these values and the Mode.spec.eStyle values, but they are not +** one-to-one, and thus need to be tracked separately. +*/ +#define MODE_Ascii 0 /* Use ASCII unit and record separators (0x1F/0x1E) */ +#define MODE_Box 1 /* Unicode box-drawing characters */ +#define MODE_C 2 /* Comma-separated list of C-strings */ +#define MODE_Column 3 /* One record per line in neat columns */ +#define MODE_Count 4 /* Output only a count of the rows of output */ +#define MODE_Csv 5 /* Quote strings, numbers are plain */ +#define MODE_Html 6 /* Generate an XHTML table */ +#define MODE_Insert 7 /* Generate SQL "insert" statements */ +#define MODE_JAtom 8 /* Comma-separated list of JSON atoms */ +#define MODE_JObject 9 /* One JSON object per row */ +#define MODE_Json 10 /* Output JSON */ +#define MODE_Line 11 /* One column per line. Blank line between records */ +#define MODE_List 12 /* One record per line with a separator */ +#define MODE_Markdown 13 /* Markdown formatting */ +#define MODE_Off 14 /* No query output shown */ +#define MODE_Psql 15 /* Similar to psql */ +#define MODE_QBox 16 /* BOX with SQL-quoted content */ +#define MODE_Quote 17 /* Quote values as for SQL */ +#define MODE_Split 18 /* Split-column mode */ +#define MODE_Table 19 /* MySQL-style table formatting */ +#define MODE_Tabs 20 /* Tab-separated values */ +#define MODE_Tcl 21 /* Space-separated list of TCL strings */ +#define MODE_Www 22 /* Full web-page output */ + +#define MODE_BUILTIN 22 /* Maximum built-in mode */ +#define MODE_BATCH 50 /* Default mode for batch processing */ +#define MODE_TTY 51 /* Default mode for interactive processing */ +#define MODE_USER 75 /* First user-defined mode */ +#define MODE_N_USER 25 /* Maximum number of user-defined modes */ + +/* +** Information about built-in display modes +*/ +typedef struct ModeInfo ModeInfo; +struct ModeInfo { + char zName[9]; /* Symbolic name of the mode */ + unsigned char eCSep; /* Column separator */ + unsigned char eRSep; /* Row separator */ + unsigned char eNull; /* Null representation */ + unsigned char eText; /* Default text encoding */ + unsigned char eHdr; /* Default header encoding. */ + unsigned char eBlob; /* Default blob encoding. */ + unsigned char bHdr; /* Show headers by default. 0: n/a, 1: no 2: yes */ + unsigned char eStyle; /* Underlying QRF style */ + unsigned char eCx; /* 0: other, 1: line, 2: columnar */ + unsigned char mFlg; /* Flags. 1=border-off 2=split-column */ }; +/* String constants used by built-in modes */ +static const char *aModeStr[] = + /* 0 1 2 3 4 5 6 7 8 */ + { 0, "\n", "|", " ", ",", "\r\n", "\036", "\037", "\t", + "", "NULL", "null", "\"\"", ": ", }; + /* 9 10 11 12 13 */ + +static const ModeInfo aModeInfo[] = { +/* zName eCSep eRSep eNull eText eHdr eBlob bHdr eStyle eCx mFlg */ + { "ascii", 7, 6, 9, 1, 1, 0, 1, 12, 0, 0 }, + { "box", 0, 0, 9, 1, 1, 0, 2, 1, 2, 0 }, + { "c", 4, 1, 10, 5, 5, 4, 1, 12, 0, 0 }, + { "column", 0, 0, 9, 1, 1, 0, 2, 2, 2, 0 }, + { "count", 0, 0, 0, 0, 0, 0, 0, 3, 0, 0 }, + { "csv", 4, 5, 9, 3, 3, 0, 1, 12, 0, 0 }, + { "html", 0, 0, 9, 4, 4, 0, 2, 7, 0, 0 }, + { "insert", 0, 0, 10, 2, 2, 0, 1, 8, 0, 0 }, + { "jatom", 4, 1, 11, 6, 6, 0, 1, 12, 0, 0 }, + { "jobject", 0, 1, 11, 6, 6, 0, 0, 10, 0, 0 }, + { "json", 0, 0, 11, 6, 6, 0, 0, 9, 0, 0 }, + { "line", 13, 1, 9, 1, 1, 0, 0, 11, 1, 0 }, + { "list", 2, 1, 9, 1, 1, 0, 1, 12, 0, 0 }, + { "markdown", 0, 0, 9, 1, 1, 0, 2, 13, 2, 0 }, + { "off", 0, 0, 0, 0, 0, 0, 0, 14, 0, 0 }, + { "psql", 0, 0, 9, 1, 1, 0, 2, 19, 2, 1 }, + { "qbox", 0, 0, 10, 2, 1, 0, 2, 1, 2, 0 }, + { "quote", 4, 1, 10, 2, 2, 0, 1, 12, 0, 0 }, + { "split", 0, 0, 9, 1, 1, 0, 1, 2, 2, 2 }, + { "table", 0, 0, 9, 1, 1, 0, 2, 19, 2, 0 }, + { "tabs", 8, 1, 9, 3, 3, 0, 1, 12, 0, 0 }, + { "tcl", 3, 1, 12, 5, 5, 4, 1, 12, 0, 0 }, + { "www", 0, 0, 9, 4, 4, 0, 2, 7, 0, 0 } +}; /* | / / | / / | | \ + ** | / / | / / | | \_ 2: columnar + ** Index into aModeStr[] | / / | | 1: line + ** | / / | | 0: other + ** | / / | \ + ** text encoding |/ | show | \ + ** v-------------------' | hdrs? | The QRF style + ** 0: n/a blob | v-----' + ** 1: plain v_---------' 0: n/a + ** 2: sql 0: auto 1: no + ** 3: csv 1: as-text 2: yes + ** 4: html 2: sql + ** 5: c 3: hex + ** 6: json 4: c + ** 5: json + ** 6: size + ******************************************************************/ /* ** These are the column/row/line separators used by the various ** import/export modes. @@ -1704,111 +1368,532 @@ static const char *modeDescr[] = { #define SEP_Unit "\x1F" #define SEP_Record "\x1E" +/* +** Default values for the various QRF limits +*/ +#ifndef DFLT_CHAR_LIMIT +# define DFLT_CHAR_LIMIT 300 +#endif +#ifndef DFLT_LINE_LIMIT +# define DFLT_LINE_LIMIT 5 +#endif +#ifndef DFLT_TITLE_LIMIT +# define DFLT_TITLE_LIMIT 20 +#endif + /* ** Limit input nesting via .read or any other input redirect. ** It's not too expensive, so a generous allowance can be made. */ #define MAX_INPUT_NESTING 25 +/************************* BEGIN PERFORMANCE TIMER *****************************/ +#if !defined(_WIN32) && !defined(WIN32) && !defined(__minux) +#include +#include +/* VxWorks does not support getrusage() as far as we can determine */ +#if defined(_WRS_KERNEL) || defined(__RTP__) +struct rusage { + struct timeval ru_utime; /* user CPU time used */ + struct timeval ru_stime; /* system CPU time used */ +}; +#define getrusage(A,B) memset(B,0,sizeof(*B)) +#endif + +/* Saved resource information for the beginning of an operation */ +static struct rusage sBegin; /* CPU time at start */ +static sqlite3_int64 iBegin; /* Wall-clock time at start */ + /* -** A callback for the sqlite3_log() interface. +** Begin timing an operation */ -static void shellLog(void *pArg, int iErrCode, const char *zMsg){ - ShellState *p = (ShellState*)pArg; - if( p->pLog==0 ) return; - sqlite3_fprintf(p->pLog, "(%d) %s\n", iErrCode, zMsg); - fflush(p->pLog); +static void beginTimer(ShellState *p){ + if( p->enableTimer || (p->flgProgress & SHELL_PROGRESS_TMOUT)!=0 ){ + getrusage(RUSAGE_SELF, &sBegin); + iBegin = timeOfDay(); + } +} + +/* Return the difference of two time_structs in seconds */ +static double timeDiff(struct timeval *pStart, struct timeval *pEnd){ + return (pEnd->tv_usec - pStart->tv_usec)*0.000001 + + (double)(pEnd->tv_sec - pStart->tv_sec); } +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK +/* Return the time since the start of the timer in +** seconds. */ +static double elapseTime(ShellState *NotUsed){ + (void)NotUsed; + if( iBegin==0 ) return 0.0; + return (timeOfDay() - iBegin)*0.000001; +} +#endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ + /* -** SQL function: shell_putsnl(X) -** -** Write the text X to the screen (or whatever output is being directed) -** adding a newline at the end, and then return X. +** Print the timing results. */ -static void shellPutsFunc( - sqlite3_context *pCtx, - int nVal, - sqlite3_value **apVal -){ - ShellState *p = (ShellState*)sqlite3_user_data(pCtx); - (void)nVal; - sqlite3_fprintf(p->out, "%s\n", sqlite3_value_text(apVal[0])); - sqlite3_result_value(pCtx, apVal[0]); +static void endTimer(ShellState *p){ + if( p->enableTimer ){ + sqlite3_int64 iEnd = timeOfDay(); + struct rusage sEnd; + getrusage(RUSAGE_SELF, &sEnd); + p->prevTimer = (iEnd - iBegin)*0.000001; + cli_printf(p->out, "Run Time: real %.6f user %.6f sys %.6f\n", + p->prevTimer, + timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), + timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); + if( p->enableTimer==1 ) p->enableTimer = 0; + iBegin = 0; + } } +#define BEGIN_TIMER(X) beginTimer(X) +#define END_TIMER(X) endTimer(X) +#define ELAPSE_TIME(X) elapseTime(X) +#define HAS_TIMER 1 + +#elif (defined(_WIN32) || defined(WIN32)) + +/* Saved resource information for the beginning of an operation */ +static HANDLE hProcess; +static FILETIME ftKernelBegin; +static FILETIME ftUserBegin; +static sqlite3_int64 ftWallBegin; +typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME, + LPFILETIME, LPFILETIME); +static GETPROCTIMES getProcessTimesAddr = NULL; + /* -** If in safe mode, print an error message described by the arguments -** and exit immediately. +** Check to see if we have timer support. Return 1 if necessary +** support found (or found previously). */ -static void failIfSafeMode( - ShellState *p, - const char *zErrMsg, - ... -){ - if( p->bSafeMode ){ - va_list ap; - char *zMsg; - va_start(ap, zErrMsg); - zMsg = sqlite3_vmprintf(zErrMsg, ap); - va_end(ap); - sqlite3_fprintf(stderr, "line %lld: %s\n", p->lineno, zMsg); - exit(1); +static int hasTimer(void){ + if( getProcessTimesAddr ){ + return 1; + } else { + /* GetProcessTimes() isn't supported in WIN95 and some other Windows + ** versions. See if the version we are running on has it, and if it + ** does, save off a pointer to it and the current process handle. + */ + hProcess = GetCurrentProcess(); + if( hProcess ){ + HINSTANCE hinstLib = LoadLibrary(TEXT("Kernel32.dll")); + if( NULL != hinstLib ){ + getProcessTimesAddr = + (GETPROCTIMES) GetProcAddress(hinstLib, "GetProcessTimes"); + if( NULL != getProcessTimesAddr ){ + return 1; + } + FreeLibrary(hinstLib); + } + } } + return 0; } /* -** SQL function: edit(VALUE) -** edit(VALUE,EDITOR) -** -** These steps: -** -** (1) Write VALUE into a temporary file. -** (2) Run program EDITOR on that temporary file. -** (3) Read the temporary file back and return its content as the result. -** (4) Delete the temporary file -** -** If the EDITOR argument is omitted, use the value in the VISUAL -** environment variable. If still there is no EDITOR, through an error. -** -** Also throw an error if the EDITOR program returns a non-zero exit code. +** Begin timing an operation */ -#ifndef SQLITE_NOHAVE_SYSTEM -static void editFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const char *zEditor; - char *zTempFile = 0; - sqlite3 *db; - char *zCmd = 0; - int bBin; - int rc; - int hasCRLF = 0; - FILE *f = 0; - sqlite3_int64 sz; - sqlite3_int64 x; - unsigned char *p = 0; - - if( argc==2 ){ - zEditor = (const char*)sqlite3_value_text(argv[1]); - }else{ - zEditor = getenv("VISUAL"); - } - if( zEditor==0 ){ - sqlite3_result_error(context, "no editor for edit()", -1); - return; - } - if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ - sqlite3_result_error(context, "NULL input to edit()", -1); - return; +static void beginTimer(ShellState *p){ + if( (p->enableTimer || (p->flgProgress & SHELL_PROGRESS_TMOUT)!=0) + && getProcessTimesAddr + ){ + FILETIME ftCreation, ftExit; + getProcessTimesAddr(hProcess,&ftCreation,&ftExit, + &ftKernelBegin,&ftUserBegin); + ftWallBegin = timeOfDay(); } - db = sqlite3_context_db_handle(context); - zTempFile = 0; - sqlite3_file_control(db, 0, SQLITE_FCNTL_TEMPFILENAME, &zTempFile); - if( zTempFile==0 ){ - sqlite3_uint64 r = 0; +} + +/* Return the difference of two FILETIME structs in seconds */ +static double timeDiff(FILETIME *pStart, FILETIME *pEnd){ + sqlite_int64 i64Start = *((sqlite_int64 *) pStart); + sqlite_int64 i64End = *((sqlite_int64 *) pEnd); + return (double) ((i64End - i64Start) / 10000000.0); +} + +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK +/* Return the time since the start of the timer in +** seconds. */ +static double elapseTime(ShellState *NotUsed){ + (void)NotUsed; + if( ftWallBegin==0 ) return 0.0; + return (timeOfDay() - ftWallBegin)*0.000001; +} +#endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ + +/* +** Print the timing results. +*/ +static void endTimer(ShellState *p){ + if( p->enableTimer && getProcessTimesAddr){ + FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; + sqlite3_int64 ftWallEnd = timeOfDay(); + getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); + p->prevTimer = (ftWallEnd - ftWallBegin)*0.000001; +#ifdef _WIN64 + /* microsecond precision on 64-bit windows */ + cli_printf(p->out, "Run Time: real %.6f user %f sys %f\n", + p->prevTimer, + timeDiff(&ftUserBegin, &ftUserEnd), + timeDiff(&ftKernelBegin, &ftKernelEnd)); +#else + /* millisecond precisino on 32-bit windows */ + cli_printf(p->out, "Run Time: real %.3f user %.3f sys %.3f\n", + p->prevTimer, + timeDiff(&ftUserBegin, &ftUserEnd), + timeDiff(&ftKernelBegin, &ftKernelEnd)); +#endif + if( p->enableTimer==1 ) p->enableTimer = 0; + ftWallBegin = 0; + } +} + +#define BEGIN_TIMER(X) beginTimer(X) +#define ELAPSE_TIME(X) elapseTime(X) +#define END_TIMER(X) endTimer(X) +#define HAS_TIMER hasTimer() + +#else +#define BEGIN_TIMER(X) /* no-op */ +#define ELAPSE_TIME(X) 0.0 +#define END_TIMER(X) /*no-op*/ +#define HAS_TIMER 0 +#endif +/************************* END PERFORMANCE TIMER ******************************/ + +/* +** Clear a display mode, freeing any allocated memory that it +** contains. +*/ +static void modeFree(Mode *p){ + u8 autoExplain = p->autoExplain; + free(p->spec.aWidth); + free(p->spec.aAlign); + free(p->spec.zColumnSep); + free(p->spec.zRowSep); + free(p->spec.zTableName); + free(p->spec.zNull); + memset(p, 0, sizeof(*p)); + p->spec.iVersion = 1; + p->autoExplain = autoExplain; +} + +/* +** Duplicate Mode pSrc into pDest. pDest is assumed to be +** uninitialized prior to invoking this routine. +*/ +static void modeDup(Mode *pDest, Mode *pSrc){ + memcpy(pDest, pSrc, sizeof(*pDest)); + if( pDest->spec.aWidth ){ + size_t sz = sizeof(pSrc->spec.aWidth[0]) * pSrc->spec.nWidth; + pDest->spec.aWidth = malloc( sz ); + if( pDest->spec.aWidth ){ + memcpy(pDest->spec.aWidth, pSrc->spec.aWidth, sz); + }else{ + pDest->spec.nWidth = 0; + } + } + if( pDest->spec.aAlign ){ + size_t sz = sizeof(pSrc->spec.aAlign[0]) * pSrc->spec.nAlign; + pDest->spec.aAlign = malloc( sz ); + if( pDest->spec.aAlign ){ + memcpy(pDest->spec.aAlign, pSrc->spec.aAlign, sz); + }else{ + pDest->spec.nAlign = 0; + } + } + if( pDest->spec.zColumnSep ){ + pDest->spec.zColumnSep = strdup(pSrc->spec.zColumnSep); + } + if( pDest->spec.zRowSep ){ + pDest->spec.zRowSep = strdup(pSrc->spec.zRowSep); + } + if( pDest->spec.zTableName ){ + pDest->spec.zTableName = strdup(pSrc->spec.zTableName); + } + if( pDest->spec.zNull ){ + pDest->spec.zNull = strdup(pSrc->spec.zNull); + } +} + +/* +** Set a string value to a copy of the zNew string in memory +** obtained from system malloc(). +*/ +static void modeSetStr(char **az, const char *zNew){ + free(*az); + if( zNew==0 ){ + *az = 0; + }else{ + size_t n = strlen(zNew); + *az = malloc( n+1 ); + if( *az ){ + memcpy(*az, zNew, n+1 ); + } + } +} + +/* +** Change the mode to eMode +*/ +static void modeChange(ShellState *p, unsigned char eMode){ + const ModeInfo *pI; + if( eModemode; + pI = &aModeInfo[eMode]; + pM->eMode = eMode; + if( pI->eCSep ) modeSetStr(&pM->spec.zColumnSep, aModeStr[pI->eCSep]); + if( pI->eRSep ) modeSetStr(&pM->spec.zRowSep, aModeStr[pI->eRSep]); + if( pI->eNull ) modeSetStr(&pM->spec.zNull, aModeStr[pI->eNull]); + pM->spec.eText = pI->eText; + pM->spec.eBlob = pI->eBlob; + if( (pM->mFlags & MFLG_HDR)==0 ){ + pM->spec.bTitles = pI->bHdr; + } + pM->spec.eTitle = pI->eHdr; + if( pI->mFlg & 0x01 ){ + pM->spec.bBorder = QRF_No; + }else{ + pM->spec.bBorder = QRF_Auto; + } + if( pI->mFlg & 0x02 ){ + pM->spec.bSplitColumn = QRF_Yes; + pM->bAutoScreenWidth = 1; + }else{ + pM->spec.bSplitColumn = QRF_No; + } + }else if( eMode>=MODE_USER && eMode-MODE_USERnSavedModes ){ + modeFree(&p->mode); + modeDup(&p->mode, &p->aSavedModes[eMode-MODE_USER].mode); + }else if( eMode==MODE_BATCH ){ + u8 mFlags = p->mode.mFlags; + modeFree(&p->mode); + modeChange(p, MODE_List); + p->mode.mFlags = mFlags; + }else if( eMode==MODE_TTY ){ + u8 mFlags = p->mode.mFlags; + modeFree(&p->mode); + modeChange(p, MODE_QBox); + p->mode.bAutoScreenWidth = 1; + p->mode.spec.eText = QRF_TEXT_Relaxed; + p->mode.spec.nCharLimit = DFLT_CHAR_LIMIT; + p->mode.spec.nLineLimit = DFLT_LINE_LIMIT; + p->mode.spec.bTextJsonb = QRF_Yes; + p->mode.spec.nTitleLimit = DFLT_TITLE_LIMIT; + p->mode.mFlags = mFlags; + } +} + +/* +** Set the mode to the default. It assumed that the mode has +** already been freed and zeroed prior to calling this routine. +*/ +static void modeDefault(ShellState *p){ + p->mode.spec.iVersion = 1; + p->mode.autoExplain = 1; + if( stdin_is_interactive || stdout_is_console ){ + modeChange(p, MODE_TTY); + }else{ + modeChange(p, MODE_BATCH); + } +} + +/* +** Find the number of a display mode given its name. Return -1 if +** the name does not match any mode. +** +** Saved modes are also searched if p!=NULL. The number returned +** for a saved mode is the index into the p->aSavedModes[] array +** plus MODE_USER. +** +** Two special mode names are also available: "batch" and "tty". +** evaluate to the default mode for batch operation and interactive +** operation on a TTY, respectively. +*/ +static int modeFind(ShellState *p, const char *zName){ + int i; + for(i=0; inSavedModes; i++){ + if( cli_strcmp(p->aSavedModes[i].zTag,zName)==0 ) return i+MODE_USER; + } + if( strcmp(zName,"batch")==0 ) return MODE_BATCH; + if( strcmp(zName,"tty")==0 ) return MODE_TTY; + return -1; +} + +/* +** Save or restore the current output mode +*/ +static void modePush(ShellState *p){ + if( p->nPopMode==0 ){ + modeFree(&p->modePrior); + modeDup(&p->modePrior,&p->mode); + } +} +static void modePop(ShellState *p){ + if( p->modePrior.spec.iVersion>0 ){ + modeFree(&p->mode); + p->mode = p->modePrior; + memset(&p->modePrior, 0, sizeof(p->modePrior)); + } +} + + +/* +** A callback for the sqlite3_log() interface. +*/ +static void shellLog(void *pArg, int iErrCode, const char *zMsg){ + ShellState *p = (ShellState*)pArg; + if( p->pLog==0 ) return; + cli_printf(p->pLog, "(%d) %s\n", iErrCode, zMsg); + fflush(p->pLog); +} + +/* +** SQL function: shell_putsnl(X) +** +** Write the text X to the screen (or whatever output is being directed) +** adding a newline at the end, and then return X. +*/ +static void shellPutsFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + ShellState *p = (ShellState*)sqlite3_user_data(pCtx); + (void)nVal; + cli_printf(p->out, "%s\n", sqlite3_value_text(apVal[0])); + sqlite3_result_value(pCtx, apVal[0]); +} + +/* +** Compute the name of the location of an input error in memory +** obtained from sqlite3_malloc(). +*/ +static char *shellErrorLocation(ShellState *p){ + char *zLoc; + if( p->zErrPrefix ){ + zLoc = sqlite3_mprintf("%s", p->zErrPrefix); + }else if( p->zInFile==0 || strcmp(p->zInFile,"")==0){ + zLoc = sqlite3_mprintf("line %lld:", p->lineno); + }else{ + zLoc = sqlite3_mprintf("%s:%lld:", p->zInFile, p->lineno); + } + return zLoc; +} + +/* +** If in safe mode, print an error message described by the arguments +** and exit immediately. +*/ +static void failIfSafeMode( + ShellState *p, + const char *zErrMsg, + ... +){ + if( p->bSafeMode ){ + va_list ap; + char *zMsg; + char *zLoc = shellErrorLocation(p); + va_start(ap, zErrMsg); + zMsg = sqlite3_vmprintf(zErrMsg, ap); + va_end(ap); + cli_printf(stderr, "%s %s\n", zLoc, zMsg); + cli_exit(1); + } +} + +/* +** Issue an error message from a dot-command. +*/ +static void dotCmdError( + ShellState *p, /* Shell state */ + int iArg, /* Index of argument on which error occurred */ + const char *zBrief, /* Brief (<20 character) error description */ + const char *zDetail, /* Error details */ + ... +){ + FILE *out = stderr; + char *zLoc = shellErrorLocation(p); + if( zBrief!=0 && iArg>=0 && iArgdot.nArg ){ + int i = p->dot.aiOfst[iArg]; + int nPrompt = strlen30(zBrief) + 5; + cli_printf(out, "%s %s\n", zLoc, p->dot.zOrig); + if( i > nPrompt ){ + cli_printf(out, "%s %*s%s ---^\n", zLoc, 1+i-nPrompt, "", zBrief); + }else{ + cli_printf(out, "%s %*s^--- %s\n", zLoc, i, "", zBrief); + } + } + if( zDetail ){ + char *zMsg; + va_list ap; + va_start(ap, zDetail); + zMsg = sqlite3_vmprintf(zDetail,ap); + va_end(ap); + cli_printf(out,"%s %s\n", zLoc, zMsg); + sqlite3_free(zMsg); + } + sqlite3_free(zLoc); +} + + +/* +** SQL function: edit(VALUE) +** edit(VALUE,EDITOR) +** +** These steps: +** +** (1) Write VALUE into a temporary file. +** (2) Run program EDITOR on that temporary file. +** (3) Read the temporary file back and return its content as the result. +** (4) Delete the temporary file +** +** If the EDITOR argument is omitted, use the value in the VISUAL +** environment variable. If still there is no EDITOR, through an error. +** +** Also throw an error if the EDITOR program returns a non-zero exit code. +*/ +#ifndef SQLITE_NOHAVE_SYSTEM +static void editFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zEditor; + char *zTempFile = 0; + sqlite3 *db; + char *zCmd = 0; + int bBin; + int rc; + int hasCRLF = 0; + FILE *f = 0; + sqlite3_int64 sz; + sqlite3_int64 x; + unsigned char *p = 0; + + if( argc==2 ){ + zEditor = (const char*)sqlite3_value_text(argv[1]); + }else{ + zEditor = getenv("VISUAL"); + } + if( zEditor==0 ){ + sqlite3_result_error(context, "no editor for edit()", -1); + return; + } + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + sqlite3_result_error(context, "NULL input to edit()", -1); + return; + } + db = sqlite3_context_db_handle(context); + zTempFile = 0; + sqlite3_file_control(db, 0, SQLITE_FCNTL_TEMPFILENAME, &zTempFile); + if( zTempFile==0 ){ + sqlite3_uint64 r = 0; sqlite3_randomness(sizeof(r), &r); zTempFile = sqlite3_mprintf("temp%llx", r); if( zTempFile==0 ){ @@ -1901,28 +1986,12 @@ edit_func_end: } #endif /* SQLITE_NOHAVE_SYSTEM */ -/* -** Save or restore the current output mode -*/ -static void outputModePush(ShellState *p){ - p->modePrior = p->mode; - p->priorShFlgs = p->shellFlgs; - memcpy(p->colSepPrior, p->colSeparator, sizeof(p->colSeparator)); - memcpy(p->rowSepPrior, p->rowSeparator, sizeof(p->rowSeparator)); -} -static void outputModePop(ShellState *p){ - p->mode = p->modePrior; - p->shellFlgs = p->priorShFlgs; - memcpy(p->colSeparator, p->colSepPrior, sizeof(p->colSeparator)); - memcpy(p->rowSeparator, p->rowSepPrior, sizeof(p->rowSeparator)); -} - /* ** Set output mode to text or binary for Windows. */ static void setCrlfMode(ShellState *p){ #ifdef _WIN32 - if( p->crlfMode ){ + if( p->mode.mFlags & MFLG_CRLF ){ sqlite3_fsetmode(p->out, _O_TEXT); }else{ sqlite3_fsetmode(p->out, _O_BINARY); @@ -1932,126 +2001,6 @@ static void setCrlfMode(ShellState *p){ #endif } -/* -** Output the given string as a hex-encoded blob (eg. X'1234' ) -*/ -static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){ - int i; - unsigned char *aBlob = (unsigned char*)pBlob; - - char *zStr = sqlite3_malloc64((i64)nBlob*2 + 1); - shell_check_oom(zStr); - - for(i=0; i> 4) ]; - zStr[i*2+1] = aHex[ (aBlob[i] & 0x0F) ]; - } - zStr[i*2] = '\0'; - - sqlite3_fprintf(out, "X'%s'", zStr); - sqlite3_free(zStr); -} - -/* -** Output the given string as a quoted string using SQL quoting conventions: -** -** (1) Single quotes (') within the string are doubled -** (2) The while string is enclosed in '...' -** (3) Control characters other than \n, \t, and \r\n are escaped -** using \u00XX notation and if such substitutions occur, -** the whole string is enclosed in unistr('...') instead of '...'. -** -** Step (3) is omitted if the control-character escape mode is OFF. -** -** See also: output_quoted_escaped_string() which does the same except -** that it does not make exceptions for \n, \t, and \r\n in step (3). -*/ -static void output_quoted_string(ShellState *p, const char *zInX){ - int i; - int needUnistr = 0; - int needDblQuote = 0; - const unsigned char *z = (const unsigned char*)zInX; - unsigned char c; - FILE *out = p->out; - sqlite3_fsetmode(out, _O_BINARY); - if( z==0 ) return; - for(i=0; (c = z[i])!=0; i++){ - if( c=='\'' ){ needDblQuote = 1; } - if( c>0x1f ) continue; - if( c=='\t' || c=='\n' ) continue; - if( c=='\r' && z[i+1]=='\n' ) continue; - needUnistr = 1; - break; - } - if( (needDblQuote==0 && needUnistr==0) - || (needDblQuote==0 && p->eEscMode==SHELL_ESC_OFF) - ){ - sqlite3_fprintf(out, "'%s'",z); - }else if( p->eEscMode==SHELL_ESC_OFF ){ - char *zEncoded = sqlite3_mprintf("%Q", z); - sqlite3_fputs(zEncoded, out); - sqlite3_free(zEncoded); - }else{ - if( needUnistr ){ - sqlite3_fputs("unistr('", out); - }else{ - sqlite3_fputs("'", out); - } - while( *z ){ - for(i=0; (c = z[i])!=0; i++){ - if( c=='\'' ) break; - if( c>0x1f ) continue; - if( c=='\t' || c=='\n' ) continue; - if( c=='\r' && z[i+1]=='\n' ) continue; - break; - } - if( i ){ - sqlite3_fprintf(out, "%.*s", i, z); - z += i; - } - if( c==0 ) break; - if( c=='\'' ){ - sqlite3_fputs("''", out); - }else{ - sqlite3_fprintf(out, "\\u%04x", c); - } - z++; - } - if( needUnistr ){ - sqlite3_fputs("')", out); - }else{ - sqlite3_fputs("'", out); - } - } - setCrlfMode(p); -} - -/* -** Output the given string as a quoted string using SQL quoting conventions. -** Additionallly , escape the "\n" and "\r" characters so that they do not -** get corrupted by end-of-line translation facilities in some operating -** systems. -** -** This is like output_quoted_string() but with the addition of the \r\n -** escape mechanism. -*/ -static void output_quoted_escaped_string(ShellState *p, const char *z){ - char *zEscaped; - sqlite3_fsetmode(p->out, _O_BINARY); - if( p->eEscMode==SHELL_ESC_OFF ){ - zEscaped = sqlite3_mprintf("%Q", z); - }else{ - zEscaped = sqlite3_mprintf("%#Q", z); - } - sqlite3_fputs(zEscaped, p->out); - sqlite3_free(zEscaped); - setCrlfMode(p); -} - /* ** Find earliest of chars within s specified in zAny. ** With ns == ~0, is like strpbrk(s,zAny) and s must be 0-terminated. @@ -2115,13 +2064,14 @@ static void output_c_string(FILE *out, const char *z){ static const char *zDQBSRO = "\"\\\x7f"; /* double-quote, backslash, rubout */ char ace[3] = "\\?"; char cbsSay; - sqlite3_fputs(zq, out); + cli_puts(zq, out); + if( z==0 ) z = ""; while( *z!=0 ){ const char *pcDQBSRO = anyOfInStr(z, zDQBSRO, ~(size_t)0); const char *pcPast = zSkipValidUtf8(z, INT_MAX, ctrlMask); const char *pcEnd = (pcDQBSRO && pcDQBSRO < pcPast)? pcDQBSRO : pcPast; if( pcEnd > z ){ - sqlite3_fprintf(out, "%.*s", (int)(pcEnd-z), z); + cli_printf(out, "%.*s", (int)(pcEnd-z), z); } if( (c = *pcEnd)==0 ) break; ++pcEnd; @@ -2137,250 +2087,106 @@ static void output_c_string(FILE *out, const char *z){ } if( cbsSay ){ ace[1] = cbsSay; - sqlite3_fputs(ace, out); + cli_puts(ace, out); }else if( !isprint(c&0xff) ){ - sqlite3_fprintf(out, "\\%03o", c&0xff); + cli_printf(out, "\\%03o", c&0xff); }else{ ace[1] = (char)c; - sqlite3_fputs(ace+1, out); + cli_puts(ace+1, out); } z = pcEnd; } - sqlite3_fputs(zq, out); + cli_puts(zq, out); } -/* -** Output the given string as quoted according to JSON quoting rules. +/* Encode input string z[] as a C-language string literal and +** append it to the sqlite3_str. If z is NULL render and empty string. */ -static void output_json_string(FILE *out, const char *z, i64 n){ - unsigned char c; +static void append_c_string(sqlite3_str *out, const char *z){ + char c; static const char *zq = "\""; static long ctrlMask = ~0L; - static const char *zDQBS = "\"\\"; - const char *pcLimit; + static const char *zDQBSRO = "\"\\\x7f"; /* double-quote, backslash, rubout */ char ace[3] = "\\?"; char cbsSay; - if( z==0 ) z = ""; - pcLimit = z + ((n<0)? strlen(z) : (size_t)n); - sqlite3_fputs(zq, out); - while( z < pcLimit ){ - const char *pcDQBS = anyOfInStr(z, zDQBS, pcLimit-z); - const char *pcPast = zSkipValidUtf8(z, (int)(pcLimit-z), ctrlMask); - const char *pcEnd = (pcDQBS && pcDQBS < pcPast)? pcDQBS : pcPast; + sqlite3_str_appendall(out,zq); + while( *z!=0 ){ + const char *pcDQBSRO = anyOfInStr(z, zDQBSRO, ~(size_t)0); + const char *pcPast = zSkipValidUtf8(z, INT_MAX, ctrlMask); + const char *pcEnd = (pcDQBSRO && pcDQBSRO < pcPast)? pcDQBSRO : pcPast; if( pcEnd > z ){ - sqlite3_fprintf(out, "%.*s", (int)(pcEnd-z), z); - z = pcEnd; + sqlite3_str_appendf(out, "%.*s", (int)(pcEnd-z), z); } - if( z >= pcLimit ) break; - c = (unsigned char)*(z++); + if( (c = *pcEnd)==0 ) break; + ++pcEnd; switch( c ){ - case '"': case '\\': + case '\\': case '"': cbsSay = (char)c; break; - case '\b': cbsSay = 'b'; break; - case '\f': cbsSay = 'f'; break; + case '\t': cbsSay = 't'; break; case '\n': cbsSay = 'n'; break; case '\r': cbsSay = 'r'; break; - case '\t': cbsSay = 't'; break; + case '\f': cbsSay = 'f'; break; default: cbsSay = 0; break; } if( cbsSay ){ ace[1] = cbsSay; - sqlite3_fputs(ace, out); - }else if( c<=0x1f || c>=0x7f ){ - sqlite3_fprintf(out, "\\u%04x", c); + sqlite3_str_appendall(out,ace); + }else if( !isprint(c&0xff) ){ + sqlite3_str_appendf(out, "\\%03o", c&0xff); }else{ ace[1] = (char)c; - sqlite3_fputs(ace+1, out); - } - } - sqlite3_fputs(zq, out); -} - -/* -** Escape the input string if it is needed and in accordance with -** eEscMode. -** -** Escaping is needed if the string contains any control characters -** other than \t, \n, and \r\n -** -** If no escaping is needed (the common case) then set *ppFree to NULL -** and return the original string. If escaping is needed, write the -** escaped string into memory obtained from sqlite3_malloc64() or the -** equivalent, and return the new string and set *ppFree to the new string -** as well. -** -** The caller is responsible for freeing *ppFree if it is non-NULL in order -** to reclaim memory. -*/ -static const char *escapeOutput( - ShellState *p, - const char *zInX, - char **ppFree -){ - i64 i, j; - i64 nCtrl = 0; - unsigned char *zIn; - unsigned char c; - unsigned char *zOut; - - - /* No escaping if disabled */ - if( p->eEscMode==SHELL_ESC_OFF ){ - *ppFree = 0; - return zInX; - } - - /* Count the number of control characters in the string. */ - zIn = (unsigned char*)zInX; - for(i=0; (c = zIn[i])!=0; i++){ - if( c<=0x1f - && c!='\t' - && c!='\n' - && (c!='\r' || zIn[i+1]!='\n') - ){ - nCtrl++; - } - } - if( nCtrl==0 ){ - *ppFree = 0; - return zInX; - } - if( p->eEscMode==SHELL_ESC_SYMBOL ) nCtrl *= 2; - zOut = sqlite3_malloc64( i + nCtrl + 1 ); - shell_check_oom(zOut); - for(i=j=0; (c = zIn[i])!=0; i++){ - if( c>0x1f - || c=='\t' - || c=='\n' - || (c=='\r' && zIn[i+1]=='\n') - ){ - continue; - } - if( i>0 ){ - memcpy(&zOut[j], zIn, i); - j += i; - } - zIn += i+1; - i = -1; - switch( p->eEscMode ){ - case SHELL_ESC_SYMBOL: - zOut[j++] = 0xe2; - zOut[j++] = 0x90; - zOut[j++] = 0x80+c; - break; - case SHELL_ESC_ASCII: - zOut[j++] = '^'; - zOut[j++] = 0x40+c; - break; + sqlite3_str_appendall(out, ace+1); } + z = pcEnd; } - if( i>0 ){ - memcpy(&zOut[j], zIn, i); - j += i; - } - zOut[j] = 0; - *ppFree = (char*)zOut; - return (char*)zOut; + sqlite3_str_appendall(out, zq); } /* -** Output the given string with characters that are special to -** HTML escaped. +** This routine runs when the user presses Ctrl-C */ -static void output_html_string(FILE *out, const char *z){ - int i; - if( z==0 ) z = ""; - while( *z ){ - for(i=0; z[i] - && z[i]!='<' - && z[i]!='&' - && z[i]!='>' - && z[i]!='\"' - && z[i]!='\''; - i++){} - if( i>0 ){ - sqlite3_fprintf(out, "%.*s",i,z); - } - if( z[i]=='<' ){ - sqlite3_fputs("<", out); - }else if( z[i]=='&' ){ - sqlite3_fputs("&", out); - }else if( z[i]=='>' ){ - sqlite3_fputs(">", out); - }else if( z[i]=='\"' ){ - sqlite3_fputs(""", out); - }else if( z[i]=='\'' ){ - sqlite3_fputs("'", out); - }else{ - break; - } - z += i + 1; - } +static void interrupt_handler(int NotUsed){ + UNUSED_PARAMETER(NotUsed); + if( ++seenInterrupt>1 ) cli_exit(1); + if( globalDb ) sqlite3_interrupt(globalDb); } -/* -** If a field contains any character identified by a 1 in the following -** array, then the string must be quoted for CSV. -*/ -static const char needCsvQuote[] = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -}; - -/* -** Output a single term of CSV. Actually, p->colSeparator is used for -** the separator, which may or may not be a comma. p->nullValue is -** the null value. Strings are quoted if necessary. The separator -** is only issued if bSep is true. +/* Try to determine the screen width. Use the default if unable. */ -static void output_csv(ShellState *p, const char *z, int bSep){ - if( z==0 ){ - sqlite3_fprintf(p->out, "%s",p->nullValue); +int shellScreenWidth(void){ + if( stdout_tty_width>0 ){ + return stdout_tty_width; }else{ - unsigned i; - for(i=0; z[i]; i++){ - if( needCsvQuote[((unsigned char*)z)[i]] ){ - i = 0; - break; - } +#if defined(TIOCGSIZE) + struct ttysize ts; + if( ioctl(STDIN_FILENO, TIOCGSIZE, &ts)>=0 + || ioctl(STDOUT_FILENO, TIOCGSIZE, &ts)>=0 + || ioctl(STDERR_FILENO, TIOCGSIZE, &ts)>=0 + ){ + return ts.ts_cols; } - if( i==0 || strstr(z, p->colSeparator)!=0 ){ - char *zQuoted = sqlite3_mprintf("\"%w\"", z); - shell_check_oom(zQuoted); - sqlite3_fputs(zQuoted, p->out); - sqlite3_free(zQuoted); - }else{ - sqlite3_fputs(z, p->out); +#elif defined(TIOCGWINSZ) + struct winsize ws; + if( ioctl(STDIN_FILENO, TIOCGWINSZ, &ws)>=0 + || ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws)>=0 + || ioctl(STDERR_FILENO, TIOCGWINSZ, &ws)>=0 + ){ + return ws.ws_col; } +#elif defined(_WIN32) + CONSOLE_SCREEN_BUFFER_INFO csbi; + if( GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi) + || GetConsoleScreenBufferInfo(GetStdHandle(STD_ERROR_HANDLE), &csbi) + || GetConsoleScreenBufferInfo(GetStdHandle(STD_INPUT_HANDLE), &csbi) + ){ + return csbi.srWindow.Right - csbi.srWindow.Left + 1; + } +#endif +#define DEFAULT_SCREEN_WIDTH 80 + return DEFAULT_SCREEN_WIDTH; } - if( bSep ){ - sqlite3_fputs(p->colSeparator, p->out); - } -} - -/* -** This routine runs when the user presses Ctrl-C -*/ -static void interrupt_handler(int NotUsed){ - UNUSED_PARAMETER(NotUsed); - if( ++seenInterrupt>1 ) exit(1); - if( globalDb ) sqlite3_interrupt(globalDb); } #if (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE) @@ -2416,6 +2222,7 @@ static int safeModeAuth( "fts3_tokenizer", "load_extension", "readfile", + "realpath", "writefile", "zipfile", "zipfile_cds", @@ -2478,23 +2285,23 @@ static int shellAuth( az[1] = zA2; az[2] = zA3; az[3] = zA4; - sqlite3_fprintf(p->out, "authorizer: %s", azAction[op]); + cli_printf(p->out, "authorizer: %s", azAction[op]); for(i=0; i<4; i++){ - sqlite3_fputs(" ", p->out); + cli_puts(" ", p->out); if( az[i] ){ output_c_string(p->out, az[i]); }else{ - sqlite3_fputs("NULL", p->out); + cli_puts("NULL", p->out); } } - sqlite3_fputs("\n", p->out); + cli_puts("\n", p->out); if( p->bSafeMode ) (void)safeModeAuth(pClientData, op, zA1, zA2, zA3, zA4); return SQLITE_OK; } #endif /* -** Print a schema statement. Part of MODE_Semi and MODE_Pretty output. +** Print a schema statement. This is helper routine to dump_callbac(). ** ** This routine converts some CREATE TABLE statements for shadow tables ** in FTS3/4/5 into CREATE TABLE IF NOT EXISTS statements. @@ -2525,18 +2332,12 @@ static void printSchemaLine(FILE *out, const char *z, const char *zTail){ } } if( sqlite3_strglob("CREATE TABLE ['\"]*", z)==0 ){ - sqlite3_fprintf(out, "CREATE TABLE IF NOT EXISTS %s%s", z+13, zTail); + cli_printf(out, "CREATE TABLE IF NOT EXISTS %s%s", z+13, zTail); }else{ - sqlite3_fprintf(out, "%s%s", z, zTail); + cli_printf(out, "%s%s", z, zTail); } sqlite3_free(zToFree); } -static void printSchemaLineN(FILE *out, char *z, int n, const char *zTail){ - char c = z[n]; - z[n] = 0; - printSchemaLine(out, z, zTail); - z[n] = c; -} /* ** Return true if string z[] has nothing but whitespace and comments to the @@ -2554,95 +2355,130 @@ static int wsToEol(const char *z){ } /* -** Add a new entry to the EXPLAIN QUERY PLAN data -*/ -static void eqp_append(ShellState *p, int iEqpId, int p2, const char *zText){ - EQPGraphRow *pNew; - i64 nText; - if( zText==0 ) return; - nText = strlen(zText); - if( p->autoEQPtest ){ - sqlite3_fprintf(p->out, "%d,%d,%s\n", iEqpId, p2, zText); - } - pNew = sqlite3_malloc64( sizeof(*pNew) + nText ); - shell_check_oom(pNew); - pNew->iEqpId = iEqpId; - pNew->iParentId = p2; - memcpy(pNew->zText, zText, nText+1); - pNew->pNext = 0; - if( p->sGraph.pLast ){ - p->sGraph.pLast->pNext = pNew; - }else{ - p->sGraph.pRow = pNew; - } - p->sGraph.pLast = pNew; -} - -/* -** Free and reset the EXPLAIN QUERY PLAN data that has been collected -** in p->sGraph. +** SQL Function: shell_format_schema(SQL,FLAGS) +** +** This function is internally by the CLI to assist with the +** ".schema", ".fullschema", and ".dump" commands. The first +** argument is the value from sqlite_schema.sql. The value returned +** is a modification of the input that can actually be run as SQL +** to recreate the schema object. +** +** When FLAGS is zero, the only changes is to append ";". If the +** 0x01 bit of FLAGS is set, then transformations are made to implement +** ".schema --indent". */ -static void eqp_reset(ShellState *p){ - EQPGraphRow *pRow, *pNext; - for(pRow = p->sGraph.pRow; pRow; pRow = pNext){ - pNext = pRow->pNext; - sqlite3_free(pRow); +static void shellFormatSchema( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + int flags; /* Value of 2nd parameter */ + const char *zSql; /* Value of 1st parameter */ + int nSql; /* Bytes of text in zSql[] */ + sqlite3_str *pOut; /* Output buffer */ + char *z; /* Writable copy of zSql */ + int i, j; /* Loop counters */ + int nParen = 0; + char cEnd = 0; + char c; + int nLine = 0; + int isIndex; + int isWhere = 0; + + assert( nVal==2 ); + pOut = sqlite3_str_new(sqlite3_context_db_handle(pCtx)); + nSql = sqlite3_value_bytes(apVal[0]); + zSql = (const char*)sqlite3_value_text(apVal[0]); + if( zSql==0 || zSql[0]==0 ) goto shellFormatSchema_finish; + flags = sqlite3_value_int(apVal[1]); + if( (flags & 0x01)==0 ){ + sqlite3_str_append(pOut, zSql, nSql); + sqlite3_str_append(pOut, ";", 1); + goto shellFormatSchema_finish; + } + if( sqlite3_strlike("CREATE VIEW%", zSql, 0)==0 + || sqlite3_strlike("CREATE TRIG%", zSql, 0)==0 + ){ + sqlite3_str_append(pOut, zSql, nSql); + sqlite3_str_append(pOut, ";", 1); + goto shellFormatSchema_finish; } - memset(&p->sGraph, 0, sizeof(p->sGraph)); -} - -/* Return the next EXPLAIN QUERY PLAN line with iEqpId that occurs after -** pOld, or return the first such line if pOld is NULL -*/ -static EQPGraphRow *eqp_next_row(ShellState *p, int iEqpId, EQPGraphRow *pOld){ - EQPGraphRow *pRow = pOld ? pOld->pNext : p->sGraph.pRow; - while( pRow && pRow->iParentId!=iEqpId ) pRow = pRow->pNext; - return pRow; -} - -/* Render a single level of the graph that has iEqpId as its parent. Called -** recursively to render sublevels. -*/ -static void eqp_render_level(ShellState *p, int iEqpId){ - EQPGraphRow *pRow, *pNext; - i64 n = strlen(p->sGraph.zPrefix); - char *z; - for(pRow = eqp_next_row(p, iEqpId, 0); pRow; pRow = pNext){ - pNext = eqp_next_row(p, iEqpId, pRow); - z = pRow->zText; - sqlite3_fprintf(p->out, "%s%s%s\n", p->sGraph.zPrefix, - pNext ? "|--" : "`--", z); - if( n<(i64)sizeof(p->sGraph.zPrefix)-7 ){ - memcpy(&p->sGraph.zPrefix[n], pNext ? "| " : " ", 4); - eqp_render_level(p, pRow->iEqpId); - p->sGraph.zPrefix[n] = 0; - } + isIndex = sqlite3_strlike("CREATE INDEX%", zSql, 0)==0 + || sqlite3_strlike("CREATE UNIQUE INDEX%", zSql, 0)==0; + z = sqlite3_mprintf("%s", zSql); + if( z==0 ){ + sqlite3_str_free(pOut); + sqlite3_result_error_nomem(pCtx); + return; } -} - -/* -** Display and reset the EXPLAIN QUERY PLAN data -*/ -static void eqp_render(ShellState *p, i64 nCycle){ - EQPGraphRow *pRow = p->sGraph.pRow; - if( pRow ){ - if( pRow->zText[0]=='-' ){ - if( pRow->pNext==0 ){ - eqp_reset(p); - return; + j = 0; + for(i=0; IsSpace(z[i]); i++){} + for(; (c = z[i])!=0; i++){ + if( IsSpace(c) ){ + if( z[j-1]=='\r' ) z[j-1] = '\n'; + if( IsSpace(z[j-1]) || z[j-1]=='(' ) continue; + }else if( (c=='(' || c==')') && j>0 && IsSpace(z[j-1]) ){ + j--; + } + z[j++] = c; + } + while( j>0 && IsSpace(z[j-1]) ){ j--; } + z[j] = 0; + if( strlen30(z)>=79 ){ + for(i=j=0; (c = z[i])!=0; i++){ /* Copy from z[i] back to z[j] */ + if( c==cEnd ){ + cEnd = 0; + }else if( cEnd!=0){ + /* No-op */ + }else if( c=='"' || c=='\'' || c=='`' ){ + cEnd = c; + }else if( c=='[' ){ + cEnd = ']'; + }else if( c=='-' && z[i+1]=='-' ){ + cEnd = '\n'; + }else if( c=='(' ){ + nParen++; + }else if( c==')' ){ + nParen--; + if( nLine>0 && nParen==0 && j>0 && !isWhere ){ + sqlite3_str_append(pOut, z, j); + sqlite3_str_append(pOut, "\n", 1); + j = 0; + } + }else if( (c=='w' || c=='W') + && nParen==0 && isIndex + && sqlite3_strnicmp("WHERE",&z[i],5)==0 + && !IsAlnum(z[i+5]) && z[i+5]!='_' ){ + isWhere = 1; + }else if( isWhere && (c=='A' || c=='a') + && nParen==0 + && sqlite3_strnicmp("AND",&z[i],3)==0 + && !IsAlnum(z[i+3]) && z[i+3]!='_' ){ + sqlite3_str_append(pOut, z, j); + sqlite3_str_append(pOut, "\n ", 5); + j = 0; + } + z[j++] = c; + if( nParen==1 && cEnd==0 + && (c=='(' || c=='\n' || (c==',' && !wsToEol(z+i+1))) + && !isWhere + ){ + if( c=='\n' ) j--; + sqlite3_str_append(pOut, z, j); + sqlite3_str_append(pOut, "\n ", 3); + j = 0; + nLine++; + while( IsSpace(z[i+1]) ){ i++; } } - sqlite3_fprintf(p->out, "%s\n", pRow->zText+3); - p->sGraph.pRow = pRow->pNext; - sqlite3_free(pRow); - }else if( nCycle>0 ){ - sqlite3_fprintf(p->out, "QUERY PLAN (cycles=%lld [100%%])\n", nCycle); - }else{ - sqlite3_fputs("QUERY PLAN\n", p->out); } - p->sGraph.zPrefix[0] = 0; - eqp_render_level(p, 0); - eqp_reset(p); + z[j] = 0; } + sqlite3_str_appendall(pOut, z); + sqlite3_str_append(pOut, ";", 1); + sqlite3_free(z); + +shellFormatSchema_finish: + sqlite3_result_text(pCtx, sqlite3_str_finish(pOut), -1, sqlite3_free); } #ifndef SQLITE_OMIT_PROGRESS_CALLBACK @@ -2652,493 +2488,26 @@ static void eqp_render(ShellState *p, i64 nCycle){ static int progress_handler(void *pClientData) { ShellState *p = (ShellState*)pClientData; p->nProgress++; + if( (p->flgProgress & SHELL_PROGRESS_TMOUT)!=0 + && ELAPSE_TIME(p)>=p->tmProgress + ){ + cli_printf(p->out, "Progress timeout after %.6f seconds\n", + ELAPSE_TIME(p)); + return 1; + } if( p->nProgress>=p->mxProgress && p->mxProgress>0 ){ - sqlite3_fprintf(p->out, "Progress limit reached (%u)\n", p->nProgress); + cli_printf(p->out, "Progress limit reached (%u)\n", p->nProgress); if( p->flgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0; if( p->flgProgress & SHELL_PROGRESS_ONCE ) p->mxProgress = 0; return 1; } if( (p->flgProgress & SHELL_PROGRESS_QUIET)==0 ){ - sqlite3_fprintf(p->out, "Progress %u\n", p->nProgress); + cli_printf(p->out, "Progress %u\n", p->nProgress); } return 0; } #endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ -/* -** Print N dashes -*/ -static void print_dashes(FILE *out, int N){ - const char zDash[] = "--------------------------------------------------"; - const int nDash = sizeof(zDash) - 1; - while( N>nDash ){ - sqlite3_fputs(zDash, out); - N -= nDash; - } - sqlite3_fprintf(out, "%.*s", N, zDash); -} - -/* -** Print a markdown or table-style row separator using ascii-art -*/ -static void print_row_separator( - ShellState *p, - int nArg, - const char *zSep -){ - int i; - if( nArg>0 ){ - sqlite3_fputs(zSep, p->out); - print_dashes(p->out, p->actualWidth[0]+2); - for(i=1; iout); - print_dashes(p->out, p->actualWidth[i]+2); - } - sqlite3_fputs(zSep, p->out); - } - sqlite3_fputs("\n", p->out); -} - -/* -** This is the callback routine that the shell -** invokes for each row of a query result. -*/ -static int shell_callback( - void *pArg, - int nArg, /* Number of result columns */ - char **azArg, /* Text of each result column */ - char **azCol, /* Column names */ - int *aiType /* Column types. Might be NULL */ -){ - int i; - ShellState *p = (ShellState*)pArg; - - if( azArg==0 ) return 0; - switch( p->cMode ){ - case MODE_Count: - case MODE_Off: { - break; - } - case MODE_Line: { - int w = 5; - if( azArg==0 ) break; - for(i=0; iw ) w = len; - } - if( p->cnt++>0 ) sqlite3_fputs(p->rowSeparator, p->out); - for(i=0; inullValue, &pFree); - sqlite3_fprintf(p->out, "%*s = %s%s", w, azCol[i], - pDisplay, p->rowSeparator); - if( pFree ) sqlite3_free(pFree); - } - break; - } - case MODE_ScanExp: - case MODE_Explain: { - static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13}; - static const int aExplainMap[] = {0, 1, 2, 3, 4, 5, 6, 7 }; - static const int aScanExpWidth[] = {4, 15, 6, 13, 4, 4, 4, 13, 2, 13}; - static const int aScanExpMap[] = {0, 9, 8, 1, 2, 3, 4, 5, 6, 7 }; - - const int *aWidth = aExplainWidth; - const int *aMap = aExplainMap; - int nWidth = ArraySize(aExplainWidth); - int iIndent = 1; - - if( p->cMode==MODE_ScanExp ){ - aWidth = aScanExpWidth; - aMap = aScanExpMap; - nWidth = ArraySize(aScanExpWidth); - iIndent = 3; - } - if( nArg>nWidth ) nArg = nWidth; - - /* If this is the first row seen, print out the headers */ - if( p->cnt++==0 ){ - for(i=0; iout, aWidth[i], azCol[ aMap[i] ]); - sqlite3_fputs(i==nArg-1 ? "\n" : " ", p->out); - } - for(i=0; iout, aWidth[i]); - sqlite3_fputs(i==nArg-1 ? "\n" : " ", p->out); - } - } - - /* If there is no data, exit early. */ - if( azArg==0 ) break; - - for(i=0; iw ){ - w = strlenChar(zVal); - zSep = " "; - } - if( i==iIndent && p->aiIndent && p->pStmt ){ - if( p->iIndentnIndent ){ - sqlite3_fprintf(p->out, "%*.s", p->aiIndent[p->iIndent], ""); - } - p->iIndent++; - } - utf8_width_print(p->out, w, zVal ? zVal : p->nullValue); - sqlite3_fputs(i==nArg-1 ? "\n" : zSep, p->out); - } - break; - } - case MODE_Semi: { /* .schema and .fullschema output */ - printSchemaLine(p->out, azArg[0], ";\n"); - break; - } - case MODE_Pretty: { /* .schema and .fullschema with --indent */ - char *z; - int j; - int nParen = 0; - char cEnd = 0; - char c; - int nLine = 0; - int isIndex; - int isWhere = 0; - assert( nArg==1 ); - if( azArg[0]==0 ) break; - if( sqlite3_strlike("CREATE VIEW%", azArg[0], 0)==0 - || sqlite3_strlike("CREATE TRIG%", azArg[0], 0)==0 - ){ - sqlite3_fprintf(p->out, "%s;\n", azArg[0]); - break; - } - isIndex = sqlite3_strlike("CREATE INDEX%", azArg[0], 0)==0 - || sqlite3_strlike("CREATE UNIQUE INDEX%", azArg[0], 0)==0; - z = sqlite3_mprintf("%s", azArg[0]); - shell_check_oom(z); - j = 0; - for(i=0; IsSpace(z[i]); i++){} - for(; (c = z[i])!=0; i++){ - if( IsSpace(c) ){ - if( z[j-1]=='\r' ) z[j-1] = '\n'; - if( IsSpace(z[j-1]) || z[j-1]=='(' ) continue; - }else if( (c=='(' || c==')') && j>0 && IsSpace(z[j-1]) ){ - j--; - } - z[j++] = c; - } - while( j>0 && IsSpace(z[j-1]) ){ j--; } - z[j] = 0; - if( strlen30(z)>=79 ){ - for(i=j=0; (c = z[i])!=0; i++){ /* Copy from z[i] back to z[j] */ - if( c==cEnd ){ - cEnd = 0; - }else if( c=='"' || c=='\'' || c=='`' ){ - cEnd = c; - }else if( c=='[' ){ - cEnd = ']'; - }else if( c=='-' && z[i+1]=='-' ){ - cEnd = '\n'; - }else if( c=='(' ){ - nParen++; - }else if( c==')' ){ - nParen--; - if( nLine>0 && nParen==0 && j>0 && !isWhere ){ - printSchemaLineN(p->out, z, j, "\n"); - j = 0; - } - }else if( (c=='w' || c=='W') - && nParen==0 && isIndex - && sqlite3_strnicmp("WHERE",&z[i],5)==0 - && !IsAlnum(z[i+5]) && z[i+5]!='_' ){ - isWhere = 1; - }else if( isWhere && (c=='A' || c=='a') - && nParen==0 - && sqlite3_strnicmp("AND",&z[i],3)==0 - && !IsAlnum(z[i+3]) && z[i+3]!='_' ){ - printSchemaLineN(p->out, z, j, "\n "); - j = 0; - } - z[j++] = c; - if( nParen==1 && cEnd==0 - && (c=='(' || c=='\n' || (c==',' && !wsToEol(z+i+1))) - && !isWhere - ){ - if( c=='\n' ) j--; - printSchemaLineN(p->out, z, j, "\n "); - j = 0; - nLine++; - while( IsSpace(z[i+1]) ){ i++; } - } - } - z[j] = 0; - } - printSchemaLine(p->out, z, ";\n"); - sqlite3_free(z); - break; - } - case MODE_List: { - if( p->cnt++==0 && p->showHeader ){ - for(i=0; iout, "%s%s", zOut, - i==nArg-1 ? p->rowSeparator : p->colSeparator); - if( pFree ) sqlite3_free(pFree); - } - } - if( azArg==0 ) break; - for(i=0; inullValue; - zOut = escapeOutput(p, z, &pFree); - sqlite3_fputs(zOut, p->out); - if( pFree ) sqlite3_free(pFree); - sqlite3_fputs((icolSeparator : p->rowSeparator, p->out); - } - break; - } - case MODE_Www: - case MODE_Html: { - if( p->cnt==0 && p->cMode==MODE_Www ){ - sqlite3_fputs( - "\n" - "\n" - ,p->out - ); - } - if( p->cnt==0 && (p->showHeader || p->cMode==MODE_Www) ){ - sqlite3_fputs("", p->out); - for(i=0; i", p->out); - output_html_string(p->out, azCol[i]); - sqlite3_fputs("\n", p->out); - } - sqlite3_fputs("\n", p->out); - } - p->cnt++; - if( azArg==0 ) break; - sqlite3_fputs("", p->out); - for(i=0; i", p->out); - output_html_string(p->out, azArg[i] ? azArg[i] : p->nullValue); - sqlite3_fputs("\n", p->out); - } - sqlite3_fputs("\n", p->out); - break; - } - case MODE_Tcl: { - if( p->cnt++==0 && p->showHeader ){ - for(i=0; iout, azCol[i] ? azCol[i] : ""); - if(icolSeparator, p->out); - } - sqlite3_fputs(p->rowSeparator, p->out); - } - if( azArg==0 ) break; - for(i=0; iout, azArg[i] ? azArg[i] : p->nullValue); - if(icolSeparator, p->out); - } - sqlite3_fputs(p->rowSeparator, p->out); - break; - } - case MODE_Csv: { - sqlite3_fsetmode(p->out, _O_BINARY); - if( p->cnt++==0 && p->showHeader ){ - for(i=0; irowSeparator, p->out); - } - if( nArg>0 ){ - for(i=0; irowSeparator, p->out); - } - setCrlfMode(p); - break; - } - case MODE_Insert: { - if( azArg==0 ) break; - sqlite3_fprintf(p->out, "INSERT INTO %s",p->zDestTable); - if( p->showHeader ){ - sqlite3_fputs("(", p->out); - for(i=0; i0 ) sqlite3_fputs(",", p->out); - if( quoteChar(azCol[i]) ){ - char *z = sqlite3_mprintf("\"%w\"", azCol[i]); - shell_check_oom(z); - sqlite3_fputs(z, p->out); - sqlite3_free(z); - }else{ - sqlite3_fprintf(p->out, "%s", azCol[i]); - } - } - sqlite3_fputs(")", p->out); - } - p->cnt++; - for(i=0; i0 ? "," : " VALUES(", p->out); - if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ - sqlite3_fputs("NULL", p->out); - }else if( aiType && aiType[i]==SQLITE_TEXT ){ - if( ShellHasFlag(p, SHFLG_Newlines) ){ - output_quoted_string(p, azArg[i]); - }else{ - output_quoted_escaped_string(p, azArg[i]); - } - }else if( aiType && aiType[i]==SQLITE_INTEGER ){ - sqlite3_fputs(azArg[i], p->out); - }else if( aiType && aiType[i]==SQLITE_FLOAT ){ - char z[50]; - double r = sqlite3_column_double(p->pStmt, i); - sqlite3_uint64 ur; - memcpy(&ur,&r,sizeof(r)); - if( ur==0x7ff0000000000000LL ){ - sqlite3_fputs("9.0e+999", p->out); - }else if( ur==0xfff0000000000000LL ){ - sqlite3_fputs("-9.0e+999", p->out); - }else{ - sqlite3_int64 ir = (sqlite3_int64)r; - if( r==(double)ir ){ - sqlite3_snprintf(50,z,"%lld.0", ir); - }else{ - sqlite3_snprintf(50,z,"%!.20g", r); - } - sqlite3_fputs(z, p->out); - } - }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ - const void *pBlob = sqlite3_column_blob(p->pStmt, i); - int nBlob = sqlite3_column_bytes(p->pStmt, i); - output_hex_blob(p->out, pBlob, nBlob); - }else if( isNumber(azArg[i], 0) ){ - sqlite3_fputs(azArg[i], p->out); - }else if( ShellHasFlag(p, SHFLG_Newlines) ){ - output_quoted_string(p, azArg[i]); - }else{ - output_quoted_escaped_string(p, azArg[i]); - } - } - sqlite3_fputs(");\n", p->out); - break; - } - case MODE_Json: { - if( azArg==0 ) break; - if( p->cnt==0 ){ - sqlite3_fputs("[{", p->out); - }else{ - sqlite3_fputs(",\n{", p->out); - } - p->cnt++; - for(i=0; iout, azCol[i], -1); - sqlite3_fputs(":", p->out); - if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ - sqlite3_fputs("null", p->out); - }else if( aiType && aiType[i]==SQLITE_FLOAT ){ - char z[50]; - double r = sqlite3_column_double(p->pStmt, i); - sqlite3_uint64 ur; - memcpy(&ur,&r,sizeof(r)); - if( ur==0x7ff0000000000000LL ){ - sqlite3_fputs("9.0e+999", p->out); - }else if( ur==0xfff0000000000000LL ){ - sqlite3_fputs("-9.0e+999", p->out); - }else{ - sqlite3_snprintf(50,z,"%!.20g", r); - sqlite3_fputs(z, p->out); - } - }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ - const void *pBlob = sqlite3_column_blob(p->pStmt, i); - int nBlob = sqlite3_column_bytes(p->pStmt, i); - output_json_string(p->out, pBlob, nBlob); - }else if( aiType && aiType[i]==SQLITE_TEXT ){ - output_json_string(p->out, azArg[i], -1); - }else{ - sqlite3_fputs(azArg[i], p->out); - } - if( iout); - } - } - sqlite3_fputs("}", p->out); - break; - } - case MODE_Quote: { - if( azArg==0 ) break; - if( p->cnt==0 && p->showHeader ){ - for(i=0; i0 ) sqlite3_fputs(p->colSeparator, p->out); - output_quoted_string(p, azCol[i]); - } - sqlite3_fputs(p->rowSeparator, p->out); - } - p->cnt++; - for(i=0; i0 ) sqlite3_fputs(p->colSeparator, p->out); - if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ - sqlite3_fputs("NULL", p->out); - }else if( aiType && aiType[i]==SQLITE_TEXT ){ - output_quoted_string(p, azArg[i]); - }else if( aiType && aiType[i]==SQLITE_INTEGER ){ - sqlite3_fputs(azArg[i], p->out); - }else if( aiType && aiType[i]==SQLITE_FLOAT ){ - char z[50]; - double r = sqlite3_column_double(p->pStmt, i); - sqlite3_snprintf(50,z,"%!.20g", r); - sqlite3_fputs(z, p->out); - }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ - const void *pBlob = sqlite3_column_blob(p->pStmt, i); - int nBlob = sqlite3_column_bytes(p->pStmt, i); - output_hex_blob(p->out, pBlob, nBlob); - }else if( isNumber(azArg[i], 0) ){ - sqlite3_fputs(azArg[i], p->out); - }else{ - output_quoted_string(p, azArg[i]); - } - } - sqlite3_fputs(p->rowSeparator, p->out); - break; - } - case MODE_Ascii: { - if( p->cnt++==0 && p->showHeader ){ - for(i=0; i0 ) sqlite3_fputs(p->colSeparator, p->out); - sqlite3_fputs(azCol[i] ? azCol[i] : "", p->out); - } - sqlite3_fputs(p->rowSeparator, p->out); - } - if( azArg==0 ) break; - for(i=0; i0 ) sqlite3_fputs(p->colSeparator, p->out); - sqlite3_fputs(azArg[i] ? azArg[i] : p->nullValue, p->out); - } - sqlite3_fputs(p->rowSeparator, p->out); - break; - } - case MODE_EQP: { - eqp_append(p, atoi(azArg[0]), atoi(azArg[1]), azArg[3]); - break; - } - } - return 0; -} - -/* -** This is the callback routine that the SQLite library -** invokes for each row of a query result. -*/ -static int callback(void *pArg, int nArg, char **azArg, char **azCol){ - /* since we don't have type info, call the shell_callback with a NULL value */ - return shell_callback(pArg, nArg, azArg, azCol, NULL); -} - /* ** This is the callback routine from sqlite3_exec() that appends all ** output onto the end of a ShellText object. @@ -3198,7 +2567,7 @@ static void createSelftestTable(ShellState *p){ "DROP TABLE [_shell$self];" ,0,0,&zErrMsg); if( zErrMsg ){ - sqlite3_fprintf(stderr, "SELFTEST initialization failure: %s\n", zErrMsg); + cli_printf(stderr, "SELFTEST initialization failure: %s\n", zErrMsg); sqlite3_free(zErrMsg); } sqlite3_exec(p->db, "RELEASE selftest_init",0,0,0); @@ -3216,11 +2585,7 @@ static void set_table_name(ShellState *p, const char *zName){ p->zDestTable = 0; } if( zName==0 ) return; - if( quoteChar(zName) ){ - p->zDestTable = sqlite3_mprintf("\"%w\"", zName); - }else{ - p->zDestTable = sqlite3_mprintf("%s", zName); - } + p->zDestTable = sqlite3_mprintf("%s", zName); shell_check_oom(p->zDestTable); } @@ -3290,7 +2655,7 @@ static int run_table_dump_query( rc = sqlite3_prepare_v2(p->db, zSelect, -1, &pSelect, 0); if( rc!=SQLITE_OK || !pSelect ){ char *zContext = shell_error_context(zSelect, p->db); - sqlite3_fprintf(p->out, "/**** ERROR: (%d) %s *****/\n%s", + cli_printf(p->out, "/**** ERROR: (%d) %s *****/\n%s", rc, sqlite3_errmsg(p->db), zContext); sqlite3_free(zContext); if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; @@ -3300,22 +2665,22 @@ static int run_table_dump_query( nResult = sqlite3_column_count(pSelect); while( rc==SQLITE_ROW ){ z = (const char*)sqlite3_column_text(pSelect, 0); - sqlite3_fprintf(p->out, "%s", z); + cli_printf(p->out, "%s", z); for(i=1; iout, ",%s", sqlite3_column_text(pSelect, i)); + cli_printf(p->out, ",%s", sqlite3_column_text(pSelect, i)); } if( z==0 ) z = ""; while( z[0] && (z[0]!='-' || z[1]!='-') ) z++; if( z[0] ){ - sqlite3_fputs("\n;\n", p->out); + cli_puts("\n;\n", p->out); }else{ - sqlite3_fputs(";\n", p->out); + cli_puts(";\n", p->out); } rc = sqlite3_step(pSelect); } rc = sqlite3_finalize(pSelect); if( rc!=SQLITE_OK ){ - sqlite3_fprintf(p->out, "/**** ERROR: (%d) %s *****/\n", + cli_printf(p->out, "/**** ERROR: (%d) %s *****/\n", rc, sqlite3_errmsg(p->db)); if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; } @@ -3375,7 +2740,7 @@ static void displayLinuxIoStats(FILE *out){ for(i=0; ipStmt; char z[100]; nCol = sqlite3_column_count(pStmt); - sqlite3_fprintf(out, "%-36s %d\n", "Number of output columns:", nCol); + cli_printf(out, "%-36s %d\n", "Number of output columns:", nCol); for(i=0; istatsOn==3 ){ if( pArg->pStmt ){ iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP,bReset); - sqlite3_fprintf(out, "VM-steps: %d\n", iCur); + cli_printf(out, "VM-steps: %d\n", iCur); } return 0; } @@ -3481,55 +2846,55 @@ static int display_stats( iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, &iCur, &iHiwtr, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Lookaside Slots Used: %d (max %d)\n", iCur, iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT, &iCur, &iHiwtr, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Successful lookaside attempts: %d\n", iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, &iCur, &iHiwtr, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Lookaside failures due to size: %d\n", iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, &iCur, &iHiwtr, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Lookaside failures due to OOM: %d\n", iHiwtr); } iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Pager Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1); - sqlite3_fprintf(out, + cli_printf(out, "Page cache hits: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1); - sqlite3_fprintf(out, + cli_printf(out, "Page cache misses: %d\n", iCur); iHiwtr64 = iCur64 = -1; sqlite3_db_status64(db, SQLITE_DBSTATUS_TEMPBUF_SPILL, &iCur64, &iHiwtr64, 0); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1); - sqlite3_fprintf(out, + cli_printf(out, "Page cache writes: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_SPILL, &iCur, &iHiwtr, 1); - sqlite3_fprintf(out, + cli_printf(out, "Page cache spills: %d\n", iCur); - sqlite3_fprintf(out, + cli_printf(out, "Temporary data spilled to disk: %lld\n", iCur64); sqlite3_db_status64(db, SQLITE_DBSTATUS_TEMPBUF_SPILL, &iCur64, &iHiwtr64, 1); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Schema Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_STMT_USED, &iCur, &iHiwtr, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Statement Heap/Lookaside Usage: %d bytes\n", iCur); } @@ -3537,33 +2902,33 @@ static int display_stats( int iHit, iMiss; iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Fullscan Steps: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_SORT, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Sort Operations: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX,bReset); - sqlite3_fprintf(out, + cli_printf(out, "Autoindex Inserts: %d\n", iCur); iHit = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_HIT, bReset); iMiss = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_MISS, bReset); if( iHit || iMiss ){ - sqlite3_fprintf(out, + cli_printf(out, "Bloom filter bypass taken: %d/%d\n", iHit, iHit+iMiss); } iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Virtual Machine Steps: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_REPREPARE,bReset); - sqlite3_fprintf(out, + cli_printf(out, "Reprepare operations: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_RUN, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Number of times run: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_MEMUSED, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Memory used by prepared stmt: %d\n", iCur); } @@ -3576,278 +2941,17 @@ static int display_stats( return 0; } - -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS -static int scanStatsHeight(sqlite3_stmt *p, int iEntry){ - int iPid = 0; - int ret = 1; - sqlite3_stmt_scanstatus_v2(p, iEntry, - SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid - ); - while( iPid!=0 ){ - int ii; - for(ii=0; 1; ii++){ - int iId; - int res; - res = sqlite3_stmt_scanstatus_v2(p, ii, - SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iId - ); - if( res ) break; - if( iId==iPid ){ - sqlite3_stmt_scanstatus_v2(p, ii, - SQLITE_SCANSTAT_PARENTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid - ); - } - } - ret++; - } - return ret; -} -#endif - -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS -static void display_explain_scanstats( - sqlite3 *db, /* Database to query */ - ShellState *pArg /* Pointer to ShellState */ -){ - static const int f = SQLITE_SCANSTAT_COMPLEX; - sqlite3_stmt *p = pArg->pStmt; - int ii = 0; - i64 nTotal = 0; - int nWidth = 0; - eqp_reset(pArg); - - for(ii=0; 1; ii++){ - const char *z = 0; - int n = 0; - if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&z) ){ - break; - } - n = (int)strlen(z) + scanStatsHeight(p, ii)*3; - if( n>nWidth ) nWidth = n; - } - nWidth += 4; - - sqlite3_stmt_scanstatus_v2(p, -1, SQLITE_SCANSTAT_NCYCLE, f, (void*)&nTotal); - for(ii=0; 1; ii++){ - i64 nLoop = 0; - i64 nRow = 0; - i64 nCycle = 0; - int iId = 0; - int iPid = 0; - const char *zo = 0; - const char *zName = 0; - char *zText = 0; - double rEst = 0.0; - - if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&zo) ){ - break; - } - sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_EST,f,(void*)&rEst); - sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NLOOP,f,(void*)&nLoop); - sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NVISIT,f,(void*)&nRow); - sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NCYCLE,f,(void*)&nCycle); - sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_SELECTID,f,(void*)&iId); - sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_PARENTID,f,(void*)&iPid); - sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NAME,f,(void*)&zName); - - zText = sqlite3_mprintf("%s", zo); - if( nCycle>=0 || nLoop>=0 || nRow>=0 ){ - char *z = 0; - if( nCycle>=0 && nTotal>0 ){ - z = sqlite3_mprintf("%zcycles=%lld [%d%%]", z, - nCycle, ((nCycle*100)+nTotal/2) / nTotal - ); - } - if( nLoop>=0 ){ - z = sqlite3_mprintf("%z%sloops=%lld", z, z ? " " : "", nLoop); - } - if( nRow>=0 ){ - z = sqlite3_mprintf("%z%srows=%lld", z, z ? " " : "", nRow); - } - - if( zName && pArg->scanstatsOn>1 ){ - double rpl = (double)nRow / (double)nLoop; - z = sqlite3_mprintf("%z rpl=%.1f est=%.1f", z, rpl, rEst); - } - - zText = sqlite3_mprintf( - "% *z (%z)", -1*(nWidth-scanStatsHeight(p, ii)*3), zText, z - ); - } - - eqp_append(pArg, iId, iPid, zText); - sqlite3_free(zText); - } - - eqp_render(pArg, nTotal); -} -#endif - - -/* -** Parameter azArray points to a zero-terminated array of strings. zStr -** points to a single nul-terminated string. Return non-zero if zStr -** is equal, according to strcmp(), to any of the strings in the array. -** Otherwise, return zero. -*/ -static int str_in_array(const char *zStr, const char **azArray){ - int i; - for(i=0; azArray[i]; i++){ - if( 0==cli_strcmp(zStr, azArray[i]) ) return 1; - } - return 0; -} - -/* -** If compiled statement pSql appears to be an EXPLAIN statement, allocate -** and populate the ShellState.aiIndent[] array with the number of -** spaces each opcode should be indented before it is output. -** -** The indenting rules are: -** -** * For each "Next", "Prev", "VNext" or "VPrev" instruction, indent -** all opcodes that occur between the p2 jump destination and the opcode -** itself by 2 spaces. -** -** * Do the previous for "Return" instructions for when P2 is positive. -** See tag-20220407a in wherecode.c and vdbe.c. -** -** * For each "Goto", if the jump destination is earlier in the program -** and ends on one of: -** Yield SeekGt SeekLt RowSetRead Rewind -** or if the P1 parameter is one instead of zero, -** then indent all opcodes between the earlier instruction -** and "Goto" by 2 spaces. -*/ -static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ - int *abYield = 0; /* True if op is an OP_Yield */ - int nAlloc = 0; /* Allocated size of p->aiIndent[], abYield */ - int iOp; /* Index of operation in p->aiIndent[] */ - - const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", - "Return", 0 }; - const char *azYield[] = { "Yield", "SeekLT", "SeekGT", "RowSetRead", - "Rewind", 0 }; - const char *azGoto[] = { "Goto", 0 }; - - /* The caller guarantees that the leftmost 4 columns of the statement - ** passed to this function are equivalent to the leftmost 4 columns - ** of EXPLAIN statement output. In practice the statement may be - ** an EXPLAIN, or it may be a query on the bytecode() virtual table. */ - assert( sqlite3_column_count(pSql)>=4 ); - assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 0), "addr" ) ); - assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 1), "opcode" ) ); - assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 2), "p1" ) ); - assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 3), "p2" ) ); - - for(iOp=0; SQLITE_ROW==sqlite3_step(pSql); iOp++){ - int i; - int iAddr = sqlite3_column_int(pSql, 0); - const char *zOp = (const char*)sqlite3_column_text(pSql, 1); - int p1 = sqlite3_column_int(pSql, 2); - int p2 = sqlite3_column_int(pSql, 3); - - /* Assuming that p2 is an instruction address, set variable p2op to the - ** index of that instruction in the aiIndent[] array. p2 and p2op may be - ** different if the current instruction is part of a sub-program generated - ** by an SQL trigger or foreign key. */ - int p2op = (p2 + (iOp-iAddr)); - - /* Grow the p->aiIndent array as required */ - if( iOp>=nAlloc ){ - nAlloc += 100; - p->aiIndent = (int*)sqlite3_realloc64(p->aiIndent, nAlloc*sizeof(int)); - shell_check_oom(p->aiIndent); - abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int)); - shell_check_oom(abYield); - } - - abYield[iOp] = str_in_array(zOp, azYield); - p->aiIndent[iOp] = 0; - p->nIndent = iOp+1; - if( str_in_array(zOp, azNext) && p2op>0 ){ - for(i=p2op; iaiIndent[i] += 2; - } - if( str_in_array(zOp, azGoto) && p2opaiIndent[i] += 2; - } - } - - p->iIndent = 0; - sqlite3_free(abYield); - sqlite3_reset(pSql); -} - -/* -** Free the array allocated by explain_data_prepare(). -*/ -static void explain_data_delete(ShellState *p){ - sqlite3_free(p->aiIndent); - p->aiIndent = 0; - p->nIndent = 0; - p->iIndent = 0; -} - -static void exec_prepared_stmt(ShellState*, sqlite3_stmt*); - -/* -** Display scan stats. -*/ -static void display_scanstats( - sqlite3 *db, /* Database to query */ - ShellState *pArg /* Pointer to ShellState */ -){ -#ifndef SQLITE_ENABLE_STMT_SCANSTATUS - UNUSED_PARAMETER(db); - UNUSED_PARAMETER(pArg); -#else - if( pArg->scanstatsOn==3 ){ - const char *zSql = - " SELECT addr, opcode, p1, p2, p3, p4, p5, comment, nexec," - " format('% 6s (%.2f%%)'," - " CASE WHEN ncycle<100_000 THEN ncycle || ' '" - " WHEN ncycle<100_000_000 THEN (ncycle/1_000) || 'K'" - " WHEN ncycle<100_000_000_000 THEN (ncycle/1_000_000) || 'M'" - " ELSE (ncycle/1000_000_000) || 'G' END," - " ncycle*100.0/(sum(ncycle) OVER ())" - " ) AS cycles" - " FROM bytecode(?)"; - - int rc = SQLITE_OK; - sqlite3_stmt *pStmt = 0; - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_stmt *pSave = pArg->pStmt; - pArg->pStmt = pStmt; - sqlite3_bind_pointer(pStmt, 1, pSave, "stmt-pointer", 0); - - pArg->cnt = 0; - pArg->cMode = MODE_ScanExp; - explain_data_prepare(pArg, pStmt); - exec_prepared_stmt(pArg, pStmt); - explain_data_delete(pArg); - - sqlite3_finalize(pStmt); - pArg->pStmt = pSave; - } - }else{ - display_explain_scanstats(db, pArg); - } -#endif -} - -/* -** Disable and restore .wheretrace and .treetrace/.selecttrace settings. -*/ -static unsigned int savedSelectTrace; -static unsigned int savedWhereTrace; -static void disable_debug_trace_modes(void){ - unsigned int zero = 0; - sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 0, &savedSelectTrace); - sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &zero); - sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 2, &savedWhereTrace); - sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &zero); +/* +** Disable and restore .wheretrace and .treetrace/.selecttrace settings. +*/ +static unsigned int savedSelectTrace; +static unsigned int savedWhereTrace; +static void disable_debug_trace_modes(void){ + unsigned int zero = 0; + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 0, &savedSelectTrace); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &zero); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 2, &savedWhereTrace); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &zero); } static void restore_debug_trace_modes(void){ sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &savedSelectTrace); @@ -3928,6 +3032,8 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ memcpy(zBuf, &zVar[6], szVar-5); sqlite3_bind_text64(pStmt, i, zBuf, szVar-6, sqlite3_free, SQLITE_UTF8); } + }else if( strcmp(zVar, "$TIMER")==0 ){ + sqlite3_bind_double(pStmt, i, pArg->prevTimer); #ifdef SQLITE_ENABLE_CARRAY }else if( strncmp(zVar, "$carray_", 8)==0 ){ static char *azColorNames[] = { @@ -3966,580 +3072,6 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ sqlite3_finalize(pQ); } -/* -** UTF8 box-drawing characters. Imagine box lines like this: -** -** 1 -** | -** 4 --+-- 2 -** | -** 3 -** -** Each box characters has between 2 and 4 of the lines leading from -** the center. The characters are here identified by the numbers of -** their corresponding lines. -*/ -#define BOX_24 "\342\224\200" /* U+2500 --- */ -#define BOX_13 "\342\224\202" /* U+2502 | */ -#define BOX_23 "\342\224\214" /* U+250c ,- */ -#define BOX_34 "\342\224\220" /* U+2510 -, */ -#define BOX_12 "\342\224\224" /* U+2514 '- */ -#define BOX_14 "\342\224\230" /* U+2518 -' */ -#define BOX_123 "\342\224\234" /* U+251c |- */ -#define BOX_134 "\342\224\244" /* U+2524 -| */ -#define BOX_234 "\342\224\254" /* U+252c -,- */ -#define BOX_124 "\342\224\264" /* U+2534 -'- */ -#define BOX_1234 "\342\224\274" /* U+253c -|- */ - -/* Draw horizontal line N characters long using unicode box -** characters -*/ -static void print_box_line(FILE *out, int N){ - const char zDash[] = - BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 - BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24; - const int nDash = sizeof(zDash) - 1; - N *= 3; - while( N>nDash ){ - sqlite3_fputs(zDash, out); - N -= nDash; - } - sqlite3_fprintf(out, "%.*s", N, zDash); -} - -/* -** Draw a horizontal separator for a MODE_Box table. -*/ -static void print_box_row_separator( - ShellState *p, - int nArg, - const char *zSep1, - const char *zSep2, - const char *zSep3 -){ - int i; - if( nArg>0 ){ - sqlite3_fputs(zSep1, p->out); - print_box_line(p->out, p->actualWidth[0]+2); - for(i=1; iout); - print_box_line(p->out, p->actualWidth[i]+2); - } - sqlite3_fputs(zSep3, p->out); - } - sqlite3_fputs("\n", p->out); -} - -/* -** z[] is a line of text that is to be displayed the .mode box or table or -** similar tabular formats. z[] might contain control characters such -** as \n, \t, \f, or \r. -** -** Compute characters to display on the first line of z[]. Stop at the -** first \r, \n, or \f. Expand \t into spaces. Return a copy (obtained -** from malloc()) of that first line, which caller should free sometime. -** Write anything to display on the next line into *pzTail. If this is -** the last line, write a NULL into *pzTail. (*pzTail is not allocated.) -*/ -static char *translateForDisplayAndDup( - ShellState *p, /* To access current settings */ - const unsigned char *z, /* Input text to be transformed */ - const unsigned char **pzTail, /* OUT: Tail of the input for next line */ - int mxWidth, /* Max width. 0 means no limit */ - u8 bWordWrap /* If true, avoid breaking mid-word */ -){ - int i; /* Input bytes consumed */ - int j; /* Output bytes generated */ - int k; /* Input bytes to be displayed */ - int n; /* Output column number */ - unsigned char *zOut; /* Output text */ - - if( z==0 ){ - *pzTail = 0; - return 0; - } - if( mxWidth<0 ) mxWidth = -mxWidth; - if( mxWidth==0 ) mxWidth = 1000000; - i = j = n = 0; - while( n=0xc0 ){ - int u; - int len = decodeUtf8(&z[i], &u); - i += len; - j += len; - n += cli_wcwidth(u); - continue; - } - if( c>=' ' ){ - n++; - i++; - j++; - continue; - } - if( c==0 || c=='\n' || (c=='\r' && z[i+1]=='\n') ) break; - if( c=='\t' ){ - do{ - n++; - j++; - }while( (n&7)!=0 && neEscMode==SHELL_ESC_OFF && (k = isVt100(&z[i]))>0 ){ - i += k; - j += k; - }else{ - n++; - j += 3; - i++; - } - } - if( n>=mxWidth && bWordWrap ){ - /* Perhaps try to back up to a better place to break the line */ - for(k=i; k>i/2; k--){ - if( IsSpace(z[k-1]) ) break; - } - if( k<=i/2 ){ - for(k=i; k>i/2; k--){ - if( IsAlnum(z[k-1])!=IsAlnum(z[k]) && (z[k]&0xc0)!=0x80 ) break; - } - } - if( k<=i/2 ){ - k = i; - }else{ - i = k; - while( z[i]==' ' ) i++; - } - }else{ - k = i; - } - if( n>=mxWidth && z[i]>=' ' ){ - *pzTail = &z[i]; - }else if( z[i]=='\r' && z[i+1]=='\n' ){ - *pzTail = z[i+2] ? &z[i+2] : 0; - }else if( z[i]==0 || z[i+1]==0 ){ - *pzTail = 0; - }else{ - *pzTail = &z[i+1]; - } - zOut = malloc( j+1 ); - shell_check_oom(zOut); - i = j = n = 0; - while( i=0xc0 ){ - int u; - int len = decodeUtf8(&z[i], &u); - do{ zOut[j++] = z[i++]; }while( (--len)>0 ); - n += cli_wcwidth(u); - continue; - } - if( c>=' ' ){ - n++; - zOut[j++] = z[i++]; - continue; - } - if( c==0 ) break; - if( z[i]=='\t' ){ - do{ - n++; - zOut[j++] = ' '; - }while( (n&7)!=0 && neEscMode ){ - case SHELL_ESC_SYMBOL: - zOut[j++] = 0xe2; - zOut[j++] = 0x90; - zOut[j++] = 0x80 + c; - break; - case SHELL_ESC_ASCII: - zOut[j++] = '^'; - zOut[j++] = 0x40 + c; - break; - case SHELL_ESC_OFF: { - int nn; - if( c==0x1b && (nn = isVt100(&z[i]))>0 ){ - memcpy(&zOut[j], &z[i], nn); - j += nn; - i += nn - 1; - }else{ - zOut[j++] = c; - } - break; - } - } - i++; - } - zOut[j] = 0; - return (char*)zOut; -} - -/* Return true if the text string z[] contains characters that need -** unistr() escaping. -*/ -static int needUnistr(const unsigned char *z){ - unsigned char c; - if( z==0 ) return 0; - while( (c = *z)>0x1f || c=='\t' || c=='\n' || (c=='\r' && z[1]=='\n') ){ z++; } - return c!=0; -} - -/* Extract the value of the i-th current column for pStmt as an SQL literal -** value. Memory is obtained from sqlite3_malloc64() and must be freed by -** the caller. -*/ -static char *quoted_column(sqlite3_stmt *pStmt, int i){ - switch( sqlite3_column_type(pStmt, i) ){ - case SQLITE_NULL: { - return sqlite3_mprintf("NULL"); - } - case SQLITE_INTEGER: - case SQLITE_FLOAT: { - return sqlite3_mprintf("%s",sqlite3_column_text(pStmt,i)); - } - case SQLITE_TEXT: { - const unsigned char *zText = sqlite3_column_text(pStmt,i); - return sqlite3_mprintf(needUnistr(zText)?"%#Q":"%Q",zText); - } - case SQLITE_BLOB: { - int j; - sqlite3_str *pStr = sqlite3_str_new(0); - const unsigned char *a = sqlite3_column_blob(pStmt,i); - int n = sqlite3_column_bytes(pStmt,i); - sqlite3_str_append(pStr, "x'", 2); - for(j=0; jcmOpts.bWordWrap; - const char *zEmpty = ""; - const char *zShowNull = p->nullValue; - - rc = sqlite3_step(pStmt); - if( rc!=SQLITE_ROW ) return; - nColumn = sqlite3_column_count(pStmt); - if( nColumn==0 ) goto columnar_end; - nAlloc = nColumn*4; - if( nAlloc<=0 ) nAlloc = 1; - azData = sqlite3_malloc64( nAlloc*sizeof(char*) ); - shell_check_oom(azData); - azNextLine = sqlite3_malloc64( nColumn*sizeof(char*) ); - shell_check_oom(azNextLine); - memset((void*)azNextLine, 0, nColumn*sizeof(char*) ); - if( p->cmOpts.bQuote ){ - azQuoted = sqlite3_malloc64( nColumn*sizeof(char*) ); - shell_check_oom(azQuoted); - memset(azQuoted, 0, nColumn*sizeof(char*) ); - } - abRowDiv = sqlite3_malloc64( nAlloc/nColumn ); - shell_check_oom(abRowDiv); - if( nColumn>p->nWidth ){ - p->colWidth = realloc(p->colWidth, (nColumn+1)*2*sizeof(int)); - shell_check_oom(p->colWidth); - for(i=p->nWidth; icolWidth[i] = 0; - p->nWidth = nColumn; - p->actualWidth = &p->colWidth[nColumn]; - } - memset(p->actualWidth, 0, nColumn*sizeof(int)); - for(i=0; icolWidth[i]; - if( w<0 ) w = -w; - p->actualWidth[i] = w; - } - for(i=0; icolWidth[i]; - if( wx==0 ){ - wx = p->cmOpts.iWrap; - } - if( wx<0 ) wx = -wx; - uz = (const unsigned char*)sqlite3_column_name(pStmt,i); - if( uz==0 ) uz = (u8*)""; - azData[i] = translateForDisplayAndDup(p, uz, &zNotUsed, wx, bw); - } - do{ - int useNextLine = bNextLine; - bNextLine = 0; - if( (nRow+2)*nColumn >= nAlloc ){ - nAlloc *= 2; - azData = sqlite3_realloc64(azData, nAlloc*sizeof(char*)); - shell_check_oom(azData); - abRowDiv = sqlite3_realloc64(abRowDiv, nAlloc/nColumn); - shell_check_oom(abRowDiv); - } - abRowDiv[nRow] = 1; - nRow++; - for(i=0; icolWidth[i]; - if( wx==0 ){ - wx = p->cmOpts.iWrap; - } - if( wx<0 ) wx = -wx; - if( useNextLine ){ - uz = azNextLine[i]; - if( uz==0 ) uz = (u8*)zEmpty; - }else if( p->cmOpts.bQuote ){ - assert( azQuoted!=0 ); - sqlite3_free(azQuoted[i]); - azQuoted[i] = quoted_column(pStmt,i); - uz = (const unsigned char*)azQuoted[i]; - }else{ - uz = (const unsigned char*)sqlite3_column_text(pStmt,i); - if( uz==0 ) uz = (u8*)zShowNull; - } - azData[nRow*nColumn + i] - = translateForDisplayAndDup(p, uz, &azNextLine[i], wx, bw); - if( azNextLine[i] ){ - bNextLine = 1; - abRowDiv[nRow-1] = 0; - bMultiLineRowExists = 1; - } - } - }while( bNextLine || sqlite3_step(pStmt)==SQLITE_ROW ); - nTotal = nColumn*(nRow+1); - for(i=0; ip->actualWidth[j] ) p->actualWidth[j] = n; - } - if( seenInterrupt ) goto columnar_end; - switch( p->cMode ){ - case MODE_Column: { - colSep = " "; - rowSep = "\n"; - if( p->showHeader ){ - for(i=0; iactualWidth[i]; - if( p->colWidth[i]<0 ) w = -w; - utf8_width_print(p->out, w, azData[i]); - sqlite3_fputs(i==nColumn-1?"\n":" ", p->out); - } - for(i=0; iout, p->actualWidth[i]); - sqlite3_fputs(i==nColumn-1?"\n":" ", p->out); - } - } - break; - } - case MODE_Table: { - colSep = " | "; - rowSep = " |\n"; - print_row_separator(p, nColumn, "+"); - sqlite3_fputs("| ", p->out); - for(i=0; iactualWidth[i]; - n = strlenChar(azData[i]); - sqlite3_fprintf(p->out, "%*s%s%*s", (w-n)/2, "", - azData[i], (w-n+1)/2, ""); - sqlite3_fputs(i==nColumn-1?" |\n":" | ", p->out); - } - print_row_separator(p, nColumn, "+"); - break; - } - case MODE_Markdown: { - colSep = " | "; - rowSep = " |\n"; - sqlite3_fputs("| ", p->out); - for(i=0; iactualWidth[i]; - n = strlenChar(azData[i]); - sqlite3_fprintf(p->out, "%*s%s%*s", (w-n)/2, "", - azData[i], (w-n+1)/2, ""); - sqlite3_fputs(i==nColumn-1?" |\n":" | ", p->out); - } - print_row_separator(p, nColumn, "|"); - break; - } - case MODE_Box: { - colSep = " " BOX_13 " "; - rowSep = " " BOX_13 "\n"; - print_box_row_separator(p, nColumn, BOX_23, BOX_234, BOX_34); - sqlite3_fputs(BOX_13 " ", p->out); - for(i=0; iactualWidth[i]; - n = strlenChar(azData[i]); - sqlite3_fprintf(p->out, "%*s%s%*s%s", - (w-n)/2, "", azData[i], (w-n+1)/2, "", - i==nColumn-1?" "BOX_13"\n":" "BOX_13" "); - } - print_box_row_separator(p, nColumn, BOX_123, BOX_1234, BOX_134); - break; - } - } - for(i=nColumn, j=0; icMode!=MODE_Column ){ - sqlite3_fputs(p->cMode==MODE_Box?BOX_13" ":"| ", p->out); - } - z = azData[i]; - if( z==0 ) z = p->nullValue; - w = p->actualWidth[j]; - if( p->colWidth[j]<0 ) w = -w; - utf8_width_print(p->out, w, z); - if( j==nColumn-1 ){ - sqlite3_fputs(rowSep, p->out); - if( bMultiLineRowExists && abRowDiv[i/nColumn-1] && i+1cMode==MODE_Table ){ - print_row_separator(p, nColumn, "+"); - }else if( p->cMode==MODE_Box ){ - print_box_row_separator(p, nColumn, BOX_123, BOX_1234, BOX_134); - }else if( p->cMode==MODE_Column ){ - sqlite3_fputs("\n", p->out); - } - } - j = -1; - if( seenInterrupt ) goto columnar_end; - }else{ - sqlite3_fputs(colSep, p->out); - } - } - if( p->cMode==MODE_Table ){ - print_row_separator(p, nColumn, "+"); - }else if( p->cMode==MODE_Box ){ - print_box_row_separator(p, nColumn, BOX_12, BOX_124, BOX_14); - } -columnar_end: - if( seenInterrupt ){ - sqlite3_fputs("Interrupt\n", p->out); - } - nData = (nRow+1)*nColumn; - for(i=0; icMode==MODE_Column - || pArg->cMode==MODE_Table - || pArg->cMode==MODE_Box - || pArg->cMode==MODE_Markdown - ){ - exec_prepared_stmt_columnar(pArg, pStmt); - return; - } - - /* perform the first step. this will tell us if we - ** have a result set or not and how wide it is. - */ - rc = sqlite3_step(pStmt); - /* if we have a result set... */ - if( SQLITE_ROW == rc ){ - /* allocate space for col name ptr, value ptr, and type */ - int nCol = sqlite3_column_count(pStmt); - void *pData = sqlite3_malloc64(3*nCol*sizeof(const char*) + 1); - if( !pData ){ - shell_out_of_memory(); - }else{ - char **azCols = (char **)pData; /* Names of result columns */ - char **azVals = &azCols[nCol]; /* Results */ - int *aiTypes = (int *)&azVals[nCol]; /* Result types */ - int i, x; - assert(sizeof(int) <= sizeof(char *)); - /* save off ptrs to column names */ - for(i=0; icMode==MODE_Insert || pArg->cMode==MODE_Quote) - ){ - azVals[i] = ""; - }else{ - azVals[i] = (char*)sqlite3_column_text(pStmt, i); - } - if( !azVals[i] && (aiTypes[i]!=SQLITE_NULL) ){ - rc = SQLITE_NOMEM; - break; /* from for */ - } - } /* end for */ - - /* if data and types extracted successfully... */ - if( SQLITE_ROW == rc ){ - /* call the supplied callback with the result row data */ - if( shell_callback(pArg, nCol, azVals, azCols, aiTypes) ){ - rc = SQLITE_ABORT; - }else{ - rc = sqlite3_step(pStmt); - } - } - } while( SQLITE_ROW == rc ); - sqlite3_free(pData); - if( pArg->cMode==MODE_Json ){ - sqlite3_fputs("]\n", pArg->out); - }else if( pArg->cMode==MODE_Www ){ - sqlite3_fputs("
        \n
        \n", pArg->out);
        -      }else if( pArg->cMode==MODE_Count ){
        -        char zBuf[200];
        -        sqlite3_snprintf(sizeof(zBuf), zBuf, "%llu row%s\n",
        -                         nRow, nRow!=1 ? "s" : "");
        -        printf("%s", zBuf);
        -      }
        -    }
        -  }
        -}
        -
         #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION)
         /*
         ** This function is called to process SQL if the previous shell command
        @@ -4591,8 +3123,8 @@ static int expertFinish(
         
               if( bVerbose ){
                 const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES);
        -        sqlite3_fputs("-- Candidates -----------------------------\n", out);
        -        sqlite3_fprintf(out, "%s\n", zCand);
        +        cli_puts("-- Candidates -----------------------------\n", out);
        +        cli_printf(out, "%s\n", zCand);
               }
               for(i=0; i=2 && 0==cli_strncmp(z, "-sample", n) ){
               if( i==(nArg-1) ){
        -        sqlite3_fprintf(stderr, "option requires an argument: %s\n", z);
        +        cli_printf(stderr, "option requires an argument: %s\n", z);
                 rc = SQLITE_ERROR;
               }else{
                 iSample = (int)integerValue(azArg[++i]);
                 if( iSample<0 || iSample>100 ){
        -          sqlite3_fprintf(stderr,"value out of range: %s\n", azArg[i]);
        +          cli_printf(stderr,"value out of range: %s\n", azArg[i]);
                   rc = SQLITE_ERROR;
                 }
               }
             }
             else{
        -      sqlite3_fprintf(stderr,"unknown option: %s\n", z);
        +      cli_printf(stderr,"unknown option: %s\n", z);
               rc = SQLITE_ERROR;
             }
           }
        @@ -4659,7 +3191,7 @@ static int expertDotCommand(
           if( rc==SQLITE_OK ){
             pState->expert.pExpert = sqlite3_expert_new(pState->db, &zErr);
             if( pState->expert.pExpert==0 ){
        -      sqlite3_fprintf(stderr,
        +      cli_printf(stderr,
                   "sqlite3_expert_new: %s\n", zErr ? zErr : "out of memory");
               rc = SQLITE_ERROR;
             }else{
        @@ -4674,6 +3206,15 @@ static int expertDotCommand(
         }
         #endif /* !SQLITE_OMIT_VIRTUALTABLE && !SQLITE_OMIT_AUTHORIZATION */
         
        +/*
        +** QRF write callback
        +*/
        +static int shellWriteQR(void *pX, const char *z, sqlite3_int64 n){
        +  ShellState *pArg = (ShellState*)pX;
        +  cli_printf(pArg->out, "%.*s", (int)n, z);
        +  return SQLITE_OK;
        +}
        +
         /*
         ** Execute a statement or set of statements.  Print
         ** any result rows/columns depending on the current mode
        @@ -4693,10 +3234,31 @@ static int shell_exec(
           int rc2;
           const char *zLeftover;          /* Tail of unprocessed SQL */
           sqlite3 *db = pArg->db;
        +  unsigned char eStyle;
        +  sqlite3_qrf_spec spec;
         
           if( pzErrMsg ){
             *pzErrMsg = NULL;
           }
        +  memcpy(&spec, &pArg->mode.spec, sizeof(spec));
        +  spec.xWrite = shellWriteQR;
        +  spec.pWriteArg = (void*)pArg;
        +  if( pArg->mode.eMode==MODE_Insert && ShellHasFlag(pArg, SHFLG_PreserveRowid) ){
        +    spec.bTitles = QRF_SW_On;
        +  }
        +  assert( pArg->mode.eMode>=0 && pArg->mode.eModemode.eMode].eStyle;
        +  if( pArg->mode.bAutoScreenWidth ){
        +    spec.nScreenWidth = shellScreenWidth();
        +  }
        +  if( spec.eBlob==QRF_BLOB_Auto ){
        +    switch( spec.eText ){
        +      case QRF_TEXT_Relaxed: /* fall through */
        +      case QRF_TEXT_Sql:  spec.eBlob = QRF_BLOB_Sql;   break;
        +      case QRF_TEXT_Json: spec.eBlob = QRF_BLOB_Json;  break;
        +      default:            spec.eBlob = QRF_BLOB_Text;  break;
        +    }
        +  }
         
         #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION)
           if( pArg->expert.pExpert ){
        @@ -4705,7 +3267,7 @@ static int shell_exec(
           }
         #endif
         
        -  while( zSql[0] && (SQLITE_OK == rc) ){
        +  while( zSql && zSql[0] && (SQLITE_OK == rc) ){
             static const char *zStmtSql;
             rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover);
             if( SQLITE_OK != rc ){
        @@ -4713,6 +3275,7 @@ static int shell_exec(
                 *pzErrMsg = save_err_msg(db, "in prepare", rc, zSql);
               }
             }else{
        +      int isExplain;
               if( !pStmt ){
                 /* this happens for a comment or white-space */
                 zSql = zLeftover;
        @@ -4723,80 +3286,58 @@ static int shell_exec(
               if( zStmtSql==0 ) zStmtSql = "";
               while( IsSpace(zStmtSql[0]) ) zStmtSql++;
         
        -      /* save off the prepared statement handle and reset row count */
        +      /* save off the prepared statement handle */
               if( pArg ){
                 pArg->pStmt = pStmt;
        -        pArg->cnt = 0;
               }
        -
        +     
               /* Show the EXPLAIN QUERY PLAN if .eqp is on */
        -      if( pArg && pArg->autoEQP && sqlite3_stmt_isexplain(pStmt)==0 ){
        -        sqlite3_stmt *pExplain;
        +      isExplain = sqlite3_stmt_isexplain(pStmt);
        +      if( pArg && pArg->mode.autoEQP && isExplain==0 && pArg->dot.nArg==0 ){
                 int triggerEQP = 0;
        +        u8 savedEnableTimer = pArg->enableTimer;
        +        pArg->enableTimer = 0;
                 disable_debug_trace_modes();
                 sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, -1, &triggerEQP);
        -        if( pArg->autoEQP>=AUTOEQP_trigger ){
        +        if( pArg->mode.autoEQP>=AUTOEQP_trigger ){
                   sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 1, 0);
                 }
        -        pExplain = pStmt;
        -        sqlite3_reset(pExplain);
        -        rc = sqlite3_stmt_explain(pExplain, 2);
        -        if( rc==SQLITE_OK ){
        -          bind_prepared_stmt(pArg, pExplain);
        -          while( sqlite3_step(pExplain)==SQLITE_ROW ){
        -            const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3);
        -            int iEqpId = sqlite3_column_int(pExplain, 0);
        -            int iParentId = sqlite3_column_int(pExplain, 1);
        -            if( zEQPLine==0 ) zEQPLine = "";
        -            if( zEQPLine[0]=='-' ) eqp_render(pArg, 0);
        -            eqp_append(pArg, iEqpId, iParentId, zEQPLine);
        -          }
        -          eqp_render(pArg, 0);
        -        }
        -        if( pArg->autoEQP>=AUTOEQP_full ){
        -          /* Also do an EXPLAIN for ".eqp full" mode */
        -          sqlite3_reset(pExplain);
        -          rc = sqlite3_stmt_explain(pExplain, 1);
        -          if( rc==SQLITE_OK ){
        -            pArg->cMode = MODE_Explain;
        -            assert( sqlite3_stmt_isexplain(pExplain)==1 );
        -            bind_prepared_stmt(pArg, pExplain);
        -            explain_data_prepare(pArg, pExplain);
        -            exec_prepared_stmt(pArg, pExplain);
        -            explain_data_delete(pArg);
        -          }
        +        sqlite3_reset(pStmt);
        +        spec.eStyle = QRF_STYLE_Auto;
        +        sqlite3_stmt_explain(pStmt, 2);
        +        sqlite3_format_query_result(pStmt, &spec, 0);
        +        if( pArg->mode.autoEQP>=AUTOEQP_full ){
        +          sqlite3_reset(pStmt);
        +          sqlite3_stmt_explain(pStmt, 1);
        +          sqlite3_format_query_result(pStmt, &spec, 0);
                 }
        -        if( pArg->autoEQP>=AUTOEQP_trigger && triggerEQP==0 ){
        +
        +        if( pArg->mode.autoEQP>=AUTOEQP_trigger && triggerEQP==0 ){
                   sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 0, 0);
                 }
                 sqlite3_reset(pStmt);
                 sqlite3_stmt_explain(pStmt, 0);
                 restore_debug_trace_modes();
        -      }
        -
        -      if( pArg ){
        -        int bIsExplain = (sqlite3_stmt_isexplain(pStmt)==1);
        -        pArg->cMode = pArg->mode;
        -        if( pArg->autoExplain ){
        -          if( bIsExplain ){
        -            pArg->cMode = MODE_Explain;
        -          }
        -          if( sqlite3_stmt_isexplain(pStmt)==2 ){
        -            pArg->cMode = MODE_EQP;
        -          }
        -        }
        -
        -        /* If the shell is currently in ".explain" mode, gather the extra
        -        ** data required to add indents to the output.*/
        -        if( pArg->cMode==MODE_Explain && bIsExplain ){
        -          explain_data_prepare(pArg, pStmt);
        -        }
        +        pArg->enableTimer = savedEnableTimer;
               }
         
               bind_prepared_stmt(pArg, pStmt);
        -      exec_prepared_stmt(pArg, pStmt);
        -      explain_data_delete(pArg);
        -      eqp_render(pArg, 0);
        +      if( isExplain && pArg->mode.autoExplain ){
        +        spec.eStyle = isExplain==1 ? QRF_STYLE_Explain : QRF_STYLE_Eqp;
        +        sqlite3_format_query_result(pStmt, &spec, pzErrMsg);
        +      }else if( pArg->mode.eMode==MODE_Www ){
        +        cli_printf(pArg->out,
        +              "
        \n" + "\n"); + spec.eStyle = QRF_STYLE_Html; + sqlite3_format_query_result(pStmt, &spec, pzErrMsg); + cli_printf(pArg->out, + "
        \n" + "
        ");
        +      }else{
        +        spec.eStyle = eStyle;
        +        sqlite3_format_query_result(pStmt, &spec, pzErrMsg);
        +      }
         
               /* print usage stats if stats on */
               if( pArg && pArg->statsOn ){
        @@ -4804,8 +3345,19 @@ static int shell_exec(
               }
         
               /* print loop-counters if required */
        -      if( pArg && pArg->scanstatsOn ){
        -        display_scanstats(db, pArg);
        +      if( pArg && pArg->mode.scanstatsOn ){
        +        char *zErr = 0;
        +        switch( pArg->mode.scanstatsOn ){
        +          case 1:   spec.eStyle = QRF_STYLE_Stats;     break;
        +          case 2:   spec.eStyle = QRF_STYLE_StatsEst;  break;
        +          default:  spec.eStyle = QRF_STYLE_StatsVm;   break;
        +        }
        +        sqlite3_reset(pStmt);
        +        rc = sqlite3_format_query_result(pStmt, &spec, &zErr);
        +        if( rc ){
        +          cli_printf(stderr, "Stats query failed: %s\n", zErr);
        +          sqlite3_free(zErr);
        +        }          
               }
         
               /* Finalize the statement just executed. If this fails, save a
        @@ -5001,14 +3553,14 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
             */
             if( db_int(p->db, "SELECT count(*) FROM sqlite_sequence")>0 ){
               if( !p->writableSchema ){
        -        sqlite3_fputs("PRAGMA writable_schema=ON;\n", p->out);
        +        cli_puts("PRAGMA writable_schema=ON;\n", p->out);
                 p->writableSchema = 1;
               }
        -      sqlite3_fputs("CREATE TABLE IF NOT EXISTS sqlite_sequence(name,seq);\n"
        +      cli_puts("CREATE TABLE IF NOT EXISTS sqlite_sequence(name,seq);\n"
                             "DELETE FROM sqlite_sequence;\n", p->out);
             }
           }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 && !noSys ){
        -    if( !dataOnly ) sqlite3_fputs("ANALYZE sqlite_schema;\n", p->out);
        +    if( !dataOnly ) cli_puts("ANALYZE sqlite_schema;\n", p->out);
           }else if( cli_strncmp(zTable, "sqlite_", 7)==0 ){
             return 0;
           }else if( dataOnly ){
        @@ -5016,7 +3568,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
           }else if( cli_strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){
             char *zIns;
             if( !p->writableSchema ){
        -      sqlite3_fputs("PRAGMA writable_schema=ON;\n", p->out);
        +      cli_puts("PRAGMA writable_schema=ON;\n", p->out);
               p->writableSchema = 1;
             }
             zIns = sqlite3_mprintf(
        @@ -5024,7 +3576,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
                "VALUES('table','%q','%q',0,'%q');",
                zTable, zTable, zSql);
             shell_check_oom(zIns);
        -    sqlite3_fprintf(p->out, "%s\n", zIns);
        +    cli_printf(p->out, "%s\n", zIns);
             sqlite3_free(zIns);
             return 0;
           }else{
        @@ -5036,8 +3588,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
             ShellText sTable;
             char **azCol;
             int i;
        -    char *savedDestTable;
        -    int savedMode;
        +    Mode savedMode;
         
             azCol = tableColumnList(p, zTable);
             if( azCol==0 ){
        @@ -5080,18 +3631,20 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){
             appendText(&sSelect, " FROM ", 0);
             appendText(&sSelect, zTable, quoteChar(zTable));
         
        -    savedDestTable = p->zDestTable;
        +
             savedMode = p->mode;
        -    p->zDestTable = sTable.zTxt;
        -    p->mode = p->cMode = MODE_Insert;
        +    p->mode.spec.zTableName = (char*)zTable;
        +    p->mode.eMode = MODE_Insert;
        +    p->mode.spec.eText = QRF_TEXT_Sql;
        +    p->mode.spec.eBlob = QRF_BLOB_Sql;
        +    p->mode.spec.bTitles = QRF_No;
             rc = shell_exec(p, sSelect.zTxt, 0);
             if( (rc&0xff)==SQLITE_CORRUPT ){
        -      sqlite3_fputs("/****** CORRUPTION ERROR *******/\n", p->out);
        +      cli_puts("/****** CORRUPTION ERROR *******/\n", p->out);
               toggleSelectOrder(p->db);
               shell_exec(p, sSelect.zTxt, 0);
               toggleSelectOrder(p->db);
             }
        -    p->zDestTable = savedDestTable;
             p->mode = savedMode;
             freeText(&sTable);
             freeText(&sSelect);
        @@ -5117,9 +3670,9 @@ static int run_schema_dump_query(
           if( rc==SQLITE_CORRUPT ){
             char *zQ2;
             int len = strlen30(zQuery);
        -    sqlite3_fputs("/****** CORRUPTION ERROR *******/\n", p->out);
        +    cli_puts("/****** CORRUPTION ERROR *******/\n", p->out);
             if( zErr ){
        -      sqlite3_fprintf(p->out, "/****** %s ******/\n", zErr);
        +      cli_printf(p->out, "/****** %s ******/\n", zErr);
               sqlite3_free(zErr);
               zErr = 0;
             }
        @@ -5128,7 +3681,7 @@ static int run_schema_dump_query(
             sqlite3_snprintf(len+100, zQ2, "%s ORDER BY rowid DESC", zQuery);
             rc = sqlite3_exec(p->db, zQ2, dump_callback, p, &zErr);
             if( rc ){
        -      sqlite3_fprintf(p->out, "/****** ERROR: %s ******/\n", zErr);
        +      cli_printf(p->out, "/****** ERROR: %s ******/\n", zErr);
             }else{
               rc = SQLITE_CORRUPT;
             }
        @@ -5186,8 +3739,8 @@ static const char *(azHelp[]) = {
           ".cd DIRECTORY            Change the working directory to DIRECTORY",
         #endif
           ".changes on|off          Show number of rows changed by SQL",
        +  ".check OPTIONS ...       Verify the results of a .testcase",
         #ifndef SQLITE_SHELL_FIDDLE
        -  ".check GLOB              Fail if output since .testcase does not match",
           ".clone NEWDB             Clone data into NEWDB from the existing database",
         #endif
           ".connection [close] [#]  Open or close an auxiliary database connection",
        @@ -5229,23 +3782,10 @@ static const char *(azHelp[]) = {
           "   --schema SCHEMA         Use SCHEMA instead of \"main\"",
           "   --help                  Show CMD details",
           ".fullschema ?--indent?   Show schema and the content of sqlite_stat tables",
        -  ".headers on|off          Turn display of headers on or off",
        +  ",headers on|off          Turn display of headers on or off",
           ".help ?-all? ?PATTERN?   Show help text for PATTERN",
         #ifndef SQLITE_SHELL_FIDDLE
           ".import FILE TABLE       Import data from FILE into TABLE",
        -  "   Options:",
        -  "     --ascii               Use \\037 and \\036 as column and row separators",
        -  "     --csv                 Use , and \\n as column and row separators",
        -  "     --skip N              Skip the first N rows of input",
        -  "     --schema S            Target table to be S.TABLE",
        -  "     -v                    \"Verbose\" - increase auxiliary output",
        -  "   Notes:",
        -  "     *  If TABLE does not exist, it is created.  The first row of input",
        -  "        determines the column names.",
        -  "     *  If neither --csv or --ascii are used, the input mode is derived",
        -  "        from the \".mode\" output mode",
        -  "     *  If FILE begins with \"|\" then it is a command that generates the",
        -  "        input text.",
         #endif
         #ifndef SQLITE_OMIT_TEST_CONTROL
           ".imposter INDEX TABLE    Create imposter table TABLE on index INDEX",
        @@ -5270,42 +3810,12 @@ static const char *(azHelp[]) = {
           ".log on|off              Turn logging on or off.",
         #endif
           ".mode ?MODE? ?OPTIONS?   Set output mode",
        -  "   MODE is one of:",
        -  "     ascii       Columns/rows delimited by 0x1F and 0x1E",
        -  "     box         Tables using unicode box-drawing characters",
        -  "     csv         Comma-separated values",
        -  "     column      Output in columns.  (See .width)",
        -  "     html        HTML  code",
        -  "     insert      SQL insert statements for TABLE",
        -  "     json        Results in a JSON array",
        -  "     line        One value per line",
        -  "     list        Values delimited by \"|\"",
        -  "     markdown    Markdown table format",
        -  "     qbox        Shorthand for \"box --wrap 60 --quote\"",
        -  "     quote       Escape answers as for SQL",
        -  "     table       ASCII-art table",
        -  "     tabs        Tab-separated values",
        -  "     tcl         TCL list elements",
        -  "   OPTIONS: (for columnar modes or insert mode):",
        -  "     --escape T     ctrl-char escape; T is one of: symbol, ascii, off",
        -  "     --wrap N       Wrap output lines to no longer than N characters",
        -  "     --wordwrap B   Wrap or not at word boundaries per B (on/off)",
        -  "     --ww           Shorthand for \"--wordwrap 1\"",
        -  "     --quote        Quote output text as SQL literals",
        -  "     --noquote      Do not quote output text",
        -  "     TABLE          The name of SQL table used for \"insert\" mode",
         #ifndef SQLITE_SHELL_FIDDLE
           ".nonce STRING            Suspend safe mode for one command if nonce matches",
         #endif
           ".nullvalue STRING        Use STRING in place of NULL values",
         #ifndef SQLITE_SHELL_FIDDLE
           ".once ?OPTIONS? ?FILE?   Output for the next SQL command only to FILE",
        -  "     If FILE begins with '|' then open as a pipe",
        -  "       --bom    Put a UTF8 byte-order mark at the beginning",
        -  "       -e       Send output to the system text editor",
        -  "       --plain  Use text/plain output instead of HTML for -w option",
        -  "       -w       Send output as HTML to a web browser (same as \".www\")",
        -  "       -x       Send output as CSV to a spreadsheet (same as \".excel\")",
           /* Note that .open is (partially) available in WASM builds but is
           ** currently only intended to be used by the fiddle tool, not
           ** end users, so is "undocumented." */
        @@ -5331,14 +3841,6 @@ static const char *(azHelp[]) = {
           "        --zip           FILE is a ZIP archive",
         #ifndef SQLITE_SHELL_FIDDLE
           ".output ?FILE?           Send output to FILE or stdout if FILE is omitted",
        -  "   If FILE begins with '|' then open it as a pipe.",
        -  "   If FILE is 'off' then output is disabled.",
        -  "   Options:",
        -  "     --bom                 Prefix output with a UTF8 byte-order mark",
        -  "     -e                    Send output to the system text editor",
        -  "     --plain               Use text/plain for -w option",
        -  "     -w                    Send output to a web browser",
        -  "     -x                    Send output as CSV to a spreadsheet",
         #endif
           ".parameter CMD ...       Manage SQL parameter bindings",
           "   clear                   Erase all bindings",
        @@ -5354,6 +3856,7 @@ static const char *(azHelp[]) = {
           "   --once                    Do no more than one progress interrupt",
           "   --quiet|-q                No output except at interrupts",
           "   --reset                   Reset the count for each input and interrupt",
        +  "   --timeout S               Halt after running for S seconds",
         #endif
           ".prompt MAIN CONTINUE    Replace the standard prompts",
         #ifndef SQLITE_SHELL_FIDDLE
        @@ -5381,7 +3884,7 @@ static const char *(azHelp[]) = {
           "    Options:",
           "       --init               Create a new SELFTEST table",
           "       -v                   Verbose output",
        -  ".separator COL ?ROW?     Change the column and row separators",
        +  ",separator COL ?ROW?     Change the column and row separators",
         #if defined(SQLITE_ENABLE_SESSION)
           ".session ?NAME? CMD ...  Create or control sessions",
           "   Subcommands:",
        @@ -5408,7 +3911,7 @@ static const char *(azHelp[]) = {
         #if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE)
           ".shell CMD ARGS...       Run CMD ARGS... in a system shell",
         #endif
        -  ".show                    Show the current values for various settings",
        +  ",show                    Show the current values for various settings",
           ".stats ?ARG?             Show stats or turn stats on or off",
           "   off                      Turn off automatic stat display",
           "   on                       Turn on automatic stat display",
        @@ -5418,13 +3921,11 @@ static const char *(azHelp[]) = {
           ".system CMD ARGS...      Run CMD ARGS... in a system shell",
         #endif
           ".tables ?TABLE?          List names of tables matching LIKE pattern TABLE",
        -#ifndef SQLITE_SHELL_FIDDLE
        -  ",testcase NAME           Begin redirecting output to 'testcase-out.txt'",
        -#endif
        +  ".testcase NAME           Begin a test case.",
           ",testctrl CMD ...        Run various sqlite3_test_control() operations",
           "                           Run \".testctrl\" with no arguments for details",
           ".timeout MS              Try opening locked tables for MS milliseconds",
        -  ".timer on|off            Turn SQL timer on or off",
        +  ".timer on|off|once       Turn SQL timer on or off.",
         #ifndef SQLITE_OMIT_TRACE
           ".trace ?OPTIONS?         Output each SQL statement as it is run",
           "    FILE                    Send output to FILE",
        @@ -5449,7 +3950,7 @@ static const char *(azHelp[]) = {
           ".vfsinfo ?AUX?           Information about the top-level VFS",
           ".vfslist                 List all available VFSes",
           ".vfsname ?AUX?           Print the name of the VFS stack",
        -  ".width NUM1 NUM2 ...     Set minimum column widths for columnar output",
        +  ",width NUM1 NUM2 ...     Set minimum column widths for columnar output",
           "     Negative values right-justify",
         #ifndef SQLITE_SHELL_FIDDLE
           ".www                     Display output of the next command in web browser",
        @@ -5457,13 +3958,26 @@ static const char *(azHelp[]) = {
         #endif
         };
         
        +INSERT-USAGE-TEXT-HERE
        +
         /*
        -** Output help text for commands that match zPattern.
        -**
        -**    *   If zPattern is NULL, then show all documented commands, but
        -**        only give a one-line summary of each.
        -**
        -**    *   If zPattern is "-a" or "-all" or "--all" then show all help text
        +** Return a pointer to usage text for zCmd, or NULL if none exists.
        +*/
        +static const char *findUsage(const char *zCmd){
        +  int i;
        +  for(i=0; i return false
        +**    * If the size of the file is not a multiple of 512 -> return false
        +**    * If sqlite3_open() fails -> return false
        +**    * if sqlite3_prepare() or sqlite3_step() fails -> return false
        +**    * Otherwise -> return true
        +*/
        +static int isDatabaseFile(const char *zFile, int openFlags){
        +  sqlite3 *db = 0;
        +  sqlite3_stmt *pStmt = 0;
        +  int rc;
        +  sqlite3_int64 sz = fileSize(zFile);
        +  if( sz<512 || (sz%512)!=0 ) return 0;
        +  if( sqlite3_open_v2(zFile, &db, openFlags, 0)==SQLITE_OK
        +   && sqlite3_prepare_v2(db,"SELECT count(*) FROM sqlite_schema",-1,&pStmt,0)
        +           ==SQLITE_OK
        +   && sqlite3_step(pStmt)==SQLITE_ROW
        +  ){
        +    rc = 1;
        +  }else{
        +    rc = 0;
        +  }
        +  sqlite3_finalize(pStmt);
        +  sqlite3_close(db);
        +  return rc;
        +}
        +
         /*
         ** Try to deduce the type of file for zName based on its content.  Return
         ** one of the SHELL_OPEN_* constants.
        @@ -5670,20 +4240,12 @@ static int session_filter(void *pCtx, const char *zTab){
         int deduceDatabaseType(const char *zName, int dfltZip, int openFlags){
           FILE *f;
           size_t n;
        -  sqlite3 *db = 0;
        -  sqlite3_stmt *pStmt = 0;
           int rc = SHELL_OPEN_UNSPEC;
           char zBuf[100];
           if( access(zName,0)!=0 ) goto database_type_by_name;
        -  if( sqlite3_open_v2(zName, &db, openFlags, 0)==SQLITE_OK
        -   && sqlite3_prepare_v2(db,"SELECT count(*) FROM sqlite_schema",-1,&pStmt,0)
        -           ==SQLITE_OK
        -   && sqlite3_step(pStmt)==SQLITE_ROW
        -  ){
        +  if( isDatabaseFile(zName, openFlags) ){
             rc = SHELL_OPEN_NORMAL;
           }
        -  sqlite3_finalize(pStmt);
        -  sqlite3_close(db);
           if( rc==SHELL_OPEN_NORMAL ) return SHELL_OPEN_NORMAL;
           f = sqlite3_fopen(zName, "rb");
           if( f==0 ) goto database_type_by_name;
        @@ -5718,6 +4280,35 @@ database_type_by_name:
           return rc;
         }
         
        +/*
        +** If the text in z[] is the name of a readable file and that file appears
        +** to contain SQL text and/or dot-commands, then return true.  If z[] is
        +** not a file, or if the file is unreadable, or if the file is a database
        +** or anything else that is not SQL text and dot-commands, then return false.
        +**
        +** If the bLeaveUninit flag is set, then be sure to leave SQLite in an
        +** uninitialized state.  This means invoking sqlite3_shutdown() after any
        +** SQLite API is used.
        +**
        +** Some amount of guesswork is involved in this decision.
        +*/
        +static int isScriptFile(const char *z, int bLeaveUninit){
        +  sqlite3_int64 sz = fileSize(z);
        +  if( sz<=0 ) return 0;
        +  if( (sz%512)==0 ){
        +    int rc;
        +    sqlite3_initialize();
        +    rc = isDatabaseFile(z, SQLITE_OPEN_READONLY);
        +    if( bLeaveUninit ){
        +      sqlite3_shutdown();
        +    }
        +    if( rc ) return 0;  /* Is a database */
        +  }
        +  if( sqlite3_strlike("%.sql",z,0)==0 ) return 1;
        +  if( sqlite3_strlike("%.txt",z,0)==0 ) return 1;
        +  return 0;
        +}
        +
         #ifndef SQLITE_OMIT_DESERIALIZE
         /*
         ** Reconstruct an in-memory database using the output from the "dbtotxt"
        @@ -5739,7 +4330,7 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){
           if( zDbFilename ){
             in = sqlite3_fopen(zDbFilename, "r");
             if( in==0 ){
        -      sqlite3_fprintf(stderr,"cannot open \"%s\" for reading\n", zDbFilename);
        +      cli_printf(stderr,"cannot open \"%s\" for reading\n", zDbFilename);
               return 0;
             }
             nLine = 0;
        @@ -5755,7 +4346,7 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){
           if( rc!=2 ) goto readHexDb_error;
           if( n<0 ) goto readHexDb_error;
           if( pgsz<512 || pgsz>65536 || (pgsz & (pgsz-1))!=0 ){
        -    sqlite3_fputs("invalid pagesize\n", stderr);
        +    cli_puts("invalid pagesize\n", stderr);
             goto readHexDb_error;
           }
           sz = ((i64)n+pgsz-1)&~(pgsz-1); /* Round up to nearest multiple of pgsz */
        @@ -5766,7 +4357,7 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){
             int j = 0;                    /* Page number from "| page" line */
             int k = 0;                    /* Offset from "| page" line */
             if( nLine>=2000000000 ){
        -      sqlite3_fprintf(stderr, "input too big\n");
        +      cli_printf(stderr, "input too big\n");
               goto readHexDb_error;
             }
             rc = sscanf(zLine, "| page %d offset %d", &j, &k);
        @@ -5807,7 +4398,7 @@ readHexDb_error:
             p->lineno = nLine;
           }
           sqlite3_free(a);
        -  sqlite3_fprintf(stderr,"Error on line %lld of --hexdb input\n", nLine);
        +  cli_printf(stderr,"Error on line %lld of --hexdb input\n", nLine);
           return 0;
         }
         #endif /* SQLITE_OMIT_DESERIALIZE */
        @@ -5912,19 +4503,19 @@ static void open_db(ShellState *p, int openFlags){
               }
             }
             if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){
        -      sqlite3_fprintf(stderr,"Error: unable to open database \"%s\": %s\n",
        +      cli_printf(stderr,"Error: unable to open database \"%s\": %s\n",
                     zDbFilename, sqlite3_errmsg(p->db));
               if( (openFlags & OPEN_DB_KEEPALIVE)==0 ){
        -        exit(1);
        +        cli_exit(1);
               }
               sqlite3_close(p->db);
               sqlite3_open(":memory:", &p->db);
               if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){
        -        sqlite3_fputs("Also: unable to open substitute in-memory database.\n",
        +        cli_puts("Also: unable to open substitute in-memory database.\n",
                               stderr);
        -        exit(1);
        +        cli_exit(1);
               }else{
        -        sqlite3_fprintf(stderr,
        +        cli_printf(stderr,
                       "Notice: using substitute in-memory database instead of \"%s\"\n",
                       zDbFilename);
               }
        @@ -6002,6 +4593,8 @@ static void open_db(ShellState *p, int openFlags){
                                     shellModuleSchema, 0, 0);
             sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p,
                                     shellPutsFunc, 0, 0);
        +    sqlite3_create_function(p->db, "shell_format_schema", 2, SQLITE_UTF8, p,
        +                            shellFormatSchema, 0, 0);
             sqlite3_create_function(p->db, "usleep",1,SQLITE_UTF8,0,
                                     shellUSleepFunc, 0, 0);
         #ifndef SQLITE_NOHAVE_SYSTEM
        @@ -6036,7 +4629,7 @@ static void open_db(ShellState *p, int openFlags){
                            SQLITE_DESERIALIZE_RESIZEABLE |
                            SQLITE_DESERIALIZE_FREEONCLOSE);
               if( rc ){
        -        sqlite3_fprintf(stderr,"Error: sqlite3_deserialize() returns %d\n", rc);
        +        cli_printf(stderr,"Error: sqlite3_deserialize() returns %d\n", rc);
               }
               if( p->szMax>0 ){
                 sqlite3_file_control(p->db, "main", SQLITE_FCNTL_SIZE_LIMIT, &p->szMax);
        @@ -6051,7 +4644,7 @@ static void open_db(ShellState *p, int openFlags){
             }
         #endif
             sqlite3_db_config(
        -        p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->scanstatsOn, (int*)0
        +        p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->mode.scanstatsOn, (int*)0
             );
           }
         }
        @@ -6062,7 +4655,7 @@ static void open_db(ShellState *p, int openFlags){
         void close_db(sqlite3 *db){
           int rc = sqlite3_close(db);
           if( rc ){
        -    sqlite3_fprintf(stderr,
        +    cli_printf(stderr,
                 "Error: sqlite3_close() returns %d: %s\n", rc, sqlite3_errmsg(db));
           }
         }
        @@ -6235,7 +4828,7 @@ static int booleanValue(const char *zArg){
           if( sqlite3_stricmp(zArg, "off")==0 || sqlite3_stricmp(zArg,"no")==0 ){
             return 0;
           }
        -  sqlite3_fprintf(stderr,
        +  cli_printf(stderr,
                "ERROR: Not a boolean value: \"%s\". Assuming \"no\".\n", zArg);
           return 0;
         }
        @@ -6263,18 +4856,18 @@ static void output_file_close(FILE *f){
         ** recognized and do the right thing.  NULL is returned if the output
         ** filename is "off".
         */
        -static FILE *output_file_open(const char *zFile){
        +static FILE *output_file_open(ShellState *p, const char *zFile){
           FILE *f;
           if( cli_strcmp(zFile,"stdout")==0 ){
             f = stdout;
           }else if( cli_strcmp(zFile, "stderr")==0 ){
             f = stderr;
        -  }else if( cli_strcmp(zFile, "off")==0 ){
        +  }else if( cli_strcmp(zFile, "off")==0 || p->bSafeMode ){
             f = 0;
           }else{
             f = sqlite3_fopen(zFile, "w");
             if( f==0 ){
        -      sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zFile);
        +      cli_printf(stderr,"Error: cannot open \"%s\"\n", zFile);
             }
           }
           return f;
        @@ -6327,12 +4920,12 @@ static int sql_trace_callback(
           switch( mType ){
             case SQLITE_TRACE_ROW:
             case SQLITE_TRACE_STMT: {
        -      sqlite3_fprintf(p->traceOut, "%.*s;\n", (int)nSql, zSql);
        +      cli_printf(p->traceOut, "%.*s;\n", (int)nSql, zSql);
               break;
             }
             case SQLITE_TRACE_PROFILE: {
               sqlite3_int64 nNanosec = pX ? *(sqlite3_int64*)pX : 0;
        -      sqlite3_fprintf(p->traceOut,
        +      cli_printf(p->traceOut,
                               "%.*s; -- %lld ns\n", (int)nSql, zSql, nNanosec);
               break;
             }
        @@ -6361,7 +4954,9 @@ struct ImportCtx {
           const char *zFile;  /* Name of the input file */
           FILE *in;           /* Read the CSV text from this input stream */
           int (SQLITE_CDECL *xCloser)(FILE*);      /* Func to close in */
        +  char *zIn;          /* Input text */
           char *z;            /* Accumulated text for a field */
        +  i64 nUsed;          /* Bytes of zIn[] used so far */
           i64 n;              /* Number of bytes in z */
           i64 nAlloc;         /* Space allocated for z[] */
           int nLine;          /* Current line number */
        @@ -6371,6 +4966,8 @@ struct ImportCtx {
           int cTerm;          /* Character that terminated the most recent field */
           int cColSep;        /* The column separator character.  (Usually ",") */
           int cRowSep;        /* The row separator character.  (Usually "\n") */
        +  int cQEscape;       /* Escape character with "...".  0 for none */
        +  int cUQEscape;      /* Escape character not with "...".  0 for none */
         };
         
         /* Clean up resourced used by an ImportCtx */
        @@ -6381,9 +4978,28 @@ static void import_cleanup(ImportCtx *p){
           }
           sqlite3_free(p->z);
           p->z = 0;
        +  if( p->zIn ){
        +    sqlite3_free(p->zIn);
        +    p->zIn = 0;
        +  }
        +}
        +
        +/* Read a single character of the .import input text.  Return EOF
        +** at end-of-file.
        +*/
        +static int import_getc(ImportCtx *p){
        +  if( p->in ){
        +    return fgetc(p->in);
        +  }else if( p->zIn && p->zIn[p->nUsed]!=0 ){
        +    return p->zIn[p->nUsed++];
        +  }else{
        +    return EOF;
        +  }
         }
         
        -/* Append a single byte to z[] */
        +/* Append a single byte to the field value begin constructed
        +** in the p->z[] buffer
        +*/
         static void import_append_char(ImportCtx *p, int c){
           if( p->n+1>=p->nAlloc ){
             p->nAlloc += p->nAlloc + 100;
        @@ -6399,8 +5015,8 @@ static void import_append_char(ImportCtx *p, int c){
         **   +  Input comes from p->in.
         **   +  Store results in p->z of length p->n.  Space to hold p->z comes
         **      from sqlite3_malloc64().
        -**   +  Use p->cSep as the column separator.  The default is ",".
        -**   +  Use p->rSep as the row separator.  The default is "\n".
        +**   +  Use p->cColSep as the column separator.  The default is ",".
        +**   +  Use p->cRowSep as the row separator.  The default is "\n".
         **   +  Keep track of the line number in p->nLine.
         **   +  Store the character that terminates the field in p->cTerm.  Store
         **      EOF on end-of-file.
        @@ -6411,7 +5027,7 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){
           int cSep = (u8)p->cColSep;
           int rSep = (u8)p->cRowSep;
           p->n = 0;
        -  c = fgetc(p->in);
        +  c = import_getc(p);
           if( c==EOF || seenInterrupt ){
             p->cTerm = EOF;
             return 0;
        @@ -6420,10 +5036,17 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){
             int pc, ppc;
             int startLine = p->nLine;
             int cQuote = c;
        +    int cEsc = (u8)p->cQEscape;
             pc = ppc = 0;
             while( 1 ){
        -      c = fgetc(p->in);
        +      c = import_getc(p);
               if( c==rSep ) p->nLine++;
        +      if( c==cEsc && cEsc!=0 ){
        +        c = import_getc(p);
        +        import_append_char(p, c);
        +        ppc = pc = 0;
        +        continue;
        +      }
               if( c==cQuote ){
                 if( pc==cQuote ){
                   pc = 0;
        @@ -6440,11 +5063,11 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){
                 break;
               }
               if( pc==cQuote && c!='\r' ){
        -        sqlite3_fprintf(stderr,"%s:%d: unescaped %c character\n", 
        -                        p->zFile, p->nLine, cQuote);
        +        cli_printf(stderr,"%s:%d: unescaped %c character\n", 
        +                   p->zFile, p->nLine, cQuote);
               }
               if( c==EOF ){
        -        sqlite3_fprintf(stderr,"%s:%d: unterminated %c-quoted field\n",
        +        cli_printf(stderr,"%s:%d: unterminated %c-quoted field\n",
                       p->zFile, startLine, cQuote);
                 p->cTerm = c;
                 break;
        @@ -6456,12 +5079,13 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){
           }else{
             /* If this is the first field being parsed and it begins with the
             ** UTF-8 BOM  (0xEF BB BF) then skip the BOM */
        +    int cEsc = p->cUQEscape;
             if( (c&0xff)==0xef && p->bNotFirst==0 ){
               import_append_char(p, c);
        -      c = fgetc(p->in);
        +      c = import_getc(p);
               if( (c&0xff)==0xbb ){
                 import_append_char(p, c);
        -        c = fgetc(p->in);
        +        c = import_getc(p);
                 if( (c&0xff)==0xbf ){
                   p->bNotFirst = 1;
                   p->n = 0;
        @@ -6470,8 +5094,9 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){
               }
             }
             while( c!=EOF && c!=cSep && c!=rSep ){
        +      if( c==cEsc && cEsc!=0 ) c = import_getc(p);
               import_append_char(p, c);
        -      c = fgetc(p->in);
        +      c = import_getc(p);
             }
             if( c==rSep ){
               p->nLine++;
        @@ -6489,8 +5114,8 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){
         **   +  Input comes from p->in.
         **   +  Store results in p->z of length p->n.  Space to hold p->z comes
         **      from sqlite3_malloc64().
        -**   +  Use p->cSep as the column separator.  The default is "\x1F".
        -**   +  Use p->rSep as the row separator.  The default is "\x1E".
        +**   +  Use p->cColSep as the column separator.  The default is "\x1F".
        +**   +  Use p->cRowSep as the row separator.  The default is "\x1E".
         **   +  Keep track of the row number in p->nLine.
         **   +  Store the character that terminates the field in p->cTerm.  Store
         **      EOF on end-of-file.
        @@ -6501,14 +5126,14 @@ static char *SQLITE_CDECL ascii_read_one_field(ImportCtx *p){
           int cSep = (u8)p->cColSep;
           int rSep = (u8)p->cRowSep;
           p->n = 0;
        -  c = fgetc(p->in);
        +  c = import_getc(p);
           if( c==EOF || seenInterrupt ){
             p->cTerm = EOF;
             return 0;
           }
           while( c!=EOF && c!=cSep && c!=rSep ){
             import_append_char(p, c);
        -    c = fgetc(p->in);
        +    c = import_getc(p);
           }
           if( c==rSep ){
             p->nLine++;
        @@ -6543,7 +5168,7 @@ static void tryToCloneData(
           shell_check_oom(zQuery);
           rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
           if( rc ){
        -    sqlite3_fprintf(stderr,"Error %d: %s on [%s]\n",
        +    cli_printf(stderr,"Error %d: %s on [%s]\n",
                   sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery);
             goto end_data_xfer;
           }
        @@ -6560,7 +5185,7 @@ static void tryToCloneData(
           memcpy(zInsert+i, ");", 3);
           rc = sqlite3_prepare_v2(newDb, zInsert, -1, &pInsert, 0);
           if( rc ){
        -    sqlite3_fprintf(stderr,"Error %d: %s on [%s]\n",
        +    cli_printf(stderr,"Error %d: %s on [%s]\n",
                   sqlite3_extended_errcode(newDb), sqlite3_errmsg(newDb), zInsert);
             goto end_data_xfer;
           }
        @@ -6596,7 +5221,7 @@ static void tryToCloneData(
               } /* End for */
               rc = sqlite3_step(pInsert);
               if( rc!=SQLITE_OK && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){
        -        sqlite3_fprintf(stderr,"Error %d: %s\n",
        +        cli_printf(stderr,"Error %d: %s\n",
                       sqlite3_extended_errcode(newDb), sqlite3_errmsg(newDb));
               }
               sqlite3_reset(pInsert);
        @@ -6614,7 +5239,7 @@ static void tryToCloneData(
             shell_check_oom(zQuery);
             rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
             if( rc ){
        -      sqlite3_fprintf(stderr,"Warning: cannot step \"%s\" backwards", zTable);
        +      cli_printf(stderr,"Warning: cannot step \"%s\" backwards", zTable);
               break;
             }
           } /* End for(k=0...) */
        @@ -6651,7 +5276,7 @@ static void tryToCloneSchema(
           shell_check_oom(zQuery);
           rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
           if( rc ){
        -    sqlite3_fprintf(stderr,
        +    cli_printf(stderr,
                   "Error: (%d) %s on [%s]\n", sqlite3_extended_errcode(p->db),
                   sqlite3_errmsg(p->db), zQuery);
             goto end_schema_xfer;
        @@ -6661,10 +5286,10 @@ static void tryToCloneSchema(
             zSql = sqlite3_column_text(pQuery, 1);
             if( zName==0 || zSql==0 ) continue;
             if( sqlite3_stricmp((char*)zName, "sqlite_sequence")!=0 ){
        -      sqlite3_fprintf(stdout, "%s... ", zName); fflush(stdout);
        +      cli_printf(stdout, "%s... ", zName); fflush(stdout);
               sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg);
               if( zErrMsg ){
        -        sqlite3_fprintf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql);
        +        cli_printf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql);
                 sqlite3_free(zErrMsg);
                 zErrMsg = 0;
               }
        @@ -6682,7 +5307,7 @@ static void tryToCloneSchema(
             shell_check_oom(zQuery);
             rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0);
             if( rc ){
        -      sqlite3_fprintf(stderr,"Error: (%d) %s on [%s]\n",
        +      cli_printf(stderr,"Error: (%d) %s on [%s]\n",
                     sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery);
               goto end_schema_xfer;
             }
        @@ -6691,10 +5316,10 @@ static void tryToCloneSchema(
               zSql = sqlite3_column_text(pQuery, 1);
               if( zName==0 || zSql==0 ) continue;
               if( sqlite3_stricmp((char*)zName, "sqlite_sequence")==0 ) continue;
        -      sqlite3_fprintf(stdout, "%s... ", zName); fflush(stdout);
        +      cli_printf(stdout, "%s... ", zName); fflush(stdout);
               sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg);
               if( zErrMsg ){
        -        sqlite3_fprintf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql);
        +        cli_printf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql);
                 sqlite3_free(zErrMsg);
                 zErrMsg = 0;
               }
        @@ -6718,12 +5343,12 @@ static void tryToClone(ShellState *p, const char *zNewDb){
           int rc;
           sqlite3 *newDb = 0;
           if( access(zNewDb,0)==0 ){
        -    sqlite3_fprintf(stderr,"File \"%s\" already exists.\n", zNewDb);
        +    cli_printf(stderr,"File \"%s\" already exists.\n", zNewDb);
             return;
           }
           rc = sqlite3_open(zNewDb, &newDb);
           if( rc ){
        -    sqlite3_fprintf(stderr,
        +    cli_printf(stderr,
                 "Cannot create output database: %s\n", sqlite3_errmsg(newDb));
           }else{
             sqlite3_exec(p->db, "PRAGMA writable_schema=ON;", 0, 0, 0);
        @@ -6742,12 +5367,12 @@ static void tryToClone(ShellState *p, const char *zNewDb){
         */
         static void output_redir(ShellState *p, FILE *pfNew){
           if( p->out != stdout ){
        -    sqlite3_fputs("Output already redirected.\n", stderr);
        +    cli_puts("Output already redirected.\n", stderr);
           }else{
             p->out = pfNew;
             setCrlfMode(p);
        -    if( p->mode==MODE_Www ){
        -      sqlite3_fputs(
        +    if( p->mode.eMode==MODE_Www ){
        +      cli_puts(
                 "\n"
                 "
        \n",
                 p->out
        @@ -6769,8 +5394,8 @@ static void output_reset(ShellState *p){
             pclose(p->out);
         #endif
           }else{
        -    if( p->mode==MODE_Www ){
        -      sqlite3_fputs("
        \n", p->out); + if( p->mode.eMode==MODE_Www ){ + cli_puts("\n", p->out); } output_file_close(p->out); #ifndef SQLITE_NOHAVE_SYSTEM @@ -6786,7 +5411,7 @@ static void output_reset(ShellState *p){ char *zCmd; zCmd = sqlite3_mprintf("%s %s", zXdgOpenCmd, p->zTempFile); if( system(zCmd) ){ - sqlite3_fprintf(stderr,"Failed: [%s]\n", zCmd); + cli_printf(stderr,"Failed: [%s]\n", zCmd); }else{ /* Give the start/open/xdg-open command some time to get ** going before we continue, and potential delete the @@ -6794,7 +5419,7 @@ static void output_reset(ShellState *p){ sqlite3_sleep(2000); } sqlite3_free(zCmd); - outputModePop(p); + modePop(p); p->doXdgOpen = 0; } #endif /* !defined(SQLITE_NOHAVE_SYSTEM) */ @@ -6802,6 +5427,10 @@ static void output_reset(ShellState *p){ p->outfile[0] = 0; p->out = stdout; setCrlfMode(p); + if( cli_output_capture ){ + sqlite3_str_free(cli_output_capture); + cli_output_capture = 0; + } } #else # define output_redir(SS,pfO) @@ -6886,7 +5515,7 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ "SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1", -1, &pStmt, 0); if( rc ){ - sqlite3_fprintf(stderr,"error: %s\n", sqlite3_errmsg(p->db)); + cli_printf(stderr,"error: %s\n", sqlite3_errmsg(p->db)); sqlite3_finalize(pStmt); return 1; } @@ -6899,28 +5528,28 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ memcpy(aHdr, pb, 100); sqlite3_finalize(pStmt); }else{ - sqlite3_fputs("unable to read database header\n", stderr); + cli_puts("unable to read database header\n", stderr); sqlite3_finalize(pStmt); return 1; } i = get2byteInt(aHdr+16); if( i==1 ) i = 65536; - sqlite3_fprintf(p->out, "%-20s %d\n", "database page size:", i); - sqlite3_fprintf(p->out, "%-20s %d\n", "write format:", aHdr[18]); - sqlite3_fprintf(p->out, "%-20s %d\n", "read format:", aHdr[19]); - sqlite3_fprintf(p->out, "%-20s %d\n", "reserved bytes:", aHdr[20]); + cli_printf(p->out, "%-20s %d\n", "database page size:", i); + cli_printf(p->out, "%-20s %d\n", "write format:", aHdr[18]); + cli_printf(p->out, "%-20s %d\n", "read format:", aHdr[19]); + cli_printf(p->out, "%-20s %d\n", "reserved bytes:", aHdr[20]); for(i=0; iout, "%-20s %u", aField[i].zName, val); + cli_printf(p->out, "%-20s %u", aField[i].zName, val); switch( ofst ){ case 56: { - if( val==1 ) sqlite3_fputs(" (utf8)", p->out); - if( val==2 ) sqlite3_fputs(" (utf16le)", p->out); - if( val==3 ) sqlite3_fputs(" (utf16be)", p->out); + if( val==1 ) cli_puts(" (utf8)", p->out); + if( val==2 ) cli_puts(" (utf16le)", p->out); + if( val==3 ) cli_puts(" (utf16be)", p->out); } } - sqlite3_fputs("\n", p->out); + cli_puts("\n", p->out); } if( zDb==0 ){ zSchemaTab = sqlite3_mprintf("main.sqlite_schema"); @@ -6931,11 +5560,11 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ } for(i=0; idb, aQuery[i].zSql, zSchemaTab); - sqlite3_fprintf(p->out, "%-20s %d\n", aQuery[i].zName, val); + cli_printf(p->out, "%-20s %d\n", aQuery[i].zName, val); } sqlite3_free(zSchemaTab); sqlite3_file_control(p->db, zDb, SQLITE_FCNTL_DATA_VERSION, &iDataVersion); - sqlite3_fprintf(p->out, "%-20s %u\n", "data version", iDataVersion); + cli_printf(p->out, "%-20s %u\n", "data version", iDataVersion); return 0; } #endif /* SQLITE_SHELL_HAVE_RECOVER */ @@ -6995,7 +5624,7 @@ static int shell_dbtotxt_command(ShellState *p, int nArg, char **azArg){ } zName = strdup(zTail); shell_check_oom(zName); - sqlite3_fprintf(p->out, "| size %lld pagesize %d filename %s\n", + cli_printf(p->out, "| size %lld pagesize %d filename %s\n", nPage*pgSz, pgSz, zName); sqlite3_finalize(pStmt); pStmt = 0; @@ -7011,27 +5640,27 @@ static int shell_dbtotxt_command(ShellState *p, int nArg, char **azArg){ for(j=0; j<16 && aLine[j]==0; j++){} if( j==16 ) continue; if( !seenPageLabel ){ - sqlite3_fprintf(p->out, "| page %lld offset %lld\n",pgno,(pgno-1)*pgSz); + cli_printf(p->out, "| page %lld offset %lld\n",pgno,(pgno-1)*pgSz); seenPageLabel = 1; } - sqlite3_fprintf(p->out, "| %5d:", i); - for(j=0; j<16; j++) sqlite3_fprintf(p->out, " %02x", aLine[j]); - sqlite3_fprintf(p->out, " "); + cli_printf(p->out, "| %5d:", i); + for(j=0; j<16; j++) cli_printf(p->out, " %02x", aLine[j]); + cli_printf(p->out, " "); for(j=0; j<16; j++){ unsigned char c = (unsigned char)aLine[j]; - sqlite3_fprintf(p->out, "%c", bShow[c]); + cli_printf(p->out, "%c", bShow[c]); } - sqlite3_fprintf(p->out, "\n"); + cli_printf(p->out, "\n"); } } sqlite3_finalize(pStmt); - sqlite3_fprintf(p->out, "| end %s\n", zName); + cli_printf(p->out, "| end %s\n", zName); free(zName); return 0; dbtotxt_error: if( rc ){ - sqlite3_fprintf(stderr, "ERROR: %s\n", sqlite3_errmsg(p->db)); + cli_printf(stderr, "ERROR: %s\n", sqlite3_errmsg(p->db)); } sqlite3_finalize(pStmt); free(zName); @@ -7042,7 +5671,7 @@ dbtotxt_error: ** Print the given string as an error message. */ static void shellEmitError(const char *zErr){ - sqlite3_fprintf(stderr,"Error: %s\n", zErr); + cli_printf(stderr,"Error: %s\n", zErr); } /* ** Print the current sqlite3_errmsg() value to stderr and return 1. @@ -7225,39 +5854,42 @@ static void clearTempFile(ShellState *p){ p->zTempFile = 0; } +/* Forward reference */ +static char *find_home_dir(int clearFlag); + /* ** Create a new temp file name with the given suffix. +** +** Because the classic temp folders like /tmp are no longer +** accessible to web browsers, for security reasons, create the +** temp file in the user's home directory. */ static void newTempFile(ShellState *p, const char *zSuffix){ - clearTempFile(p); - sqlite3_free(p->zTempFile); - p->zTempFile = 0; - if( p->db ){ - sqlite3_file_control(p->db, 0, SQLITE_FCNTL_TEMPFILENAME, &p->zTempFile); - } - if( p->zTempFile==0 ){ - /* If p->db is an in-memory database then the TEMPFILENAME file-control - ** will not work and we will need to fallback to guessing */ - char *zTemp; - sqlite3_uint64 r; - sqlite3_randomness(sizeof(r), &r); - zTemp = getenv("TEMP"); - if( zTemp==0 ) zTemp = getenv("TMP"); - if( zTemp==0 ){ + char *zHome; /* Home directory */ + int i; /* Loop counter */ + sqlite3_uint64 r = 0; /* Integer with 64 bits of randomness */ + char zRand[32]; /* Text string with 160 bits of randomness */ #ifdef _WIN32 - zTemp = "\\tmp"; + const char cDirSep = '\\'; #else - zTemp = "/tmp"; + const char cDirSep = '/'; #endif - } - p->zTempFile = sqlite3_mprintf("%s/temp%llx.%s", zTemp, r, zSuffix); - }else{ - p->zTempFile = sqlite3_mprintf("%z.%s", p->zTempFile, zSuffix); + + for(i=0; i<31; i++){ + if( (i%12)==0 ) sqlite3_randomness(sizeof(r),&r); + zRand[i] = "0123456789abcdefghijklmnopqrstuvwxyz"[r%36]; + r /= 36; } + zRand[i] = 0; + clearTempFile(p); + sqlite3_free(p->zTempFile); + p->zTempFile = 0; + zHome = find_home_dir(0); + p->zTempFile = sqlite3_mprintf("%s%ctemp-%s.%s", + zHome,cDirSep,zRand,zSuffix); shell_check_oom(p->zTempFile); } - /* ** The implementation of SQL scalar function fkey_collate_clause(), used ** by the ".lint fkey-indexes" command. This scalar function is always @@ -7402,7 +6034,7 @@ static int lintFkeyIndexes( zIndent = " "; } else{ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Usage: %s %s ?-verbose? ?-groupbyparent?\n", azArg[0], azArg[1]); return SQLITE_ERROR; } @@ -7447,22 +6079,22 @@ static int lintFkeyIndexes( if( rc!=SQLITE_OK ) break; if( res<0 ){ - sqlite3_fputs("Error: internal error", stderr); + cli_puts("Error: internal error", stderr); break; }else{ if( bGroupByParent && (bVerbose || res==0) && (zPrev==0 || sqlite3_stricmp(zParent, zPrev)) ){ - sqlite3_fprintf(out, "-- Parent table %s\n", zParent); + cli_printf(out, "-- Parent table %s\n", zParent); sqlite3_free(zPrev); zPrev = sqlite3_mprintf("%s", zParent); } if( res==0 ){ - sqlite3_fprintf(out, "%s%s --> %s\n", zIndent, zCI, zTarget); + cli_printf(out, "%s%s --> %s\n", zIndent, zCI, zTarget); }else if( bVerbose ){ - sqlite3_fprintf(out, + cli_printf(out, "%s/* no extra indexes required for %s -> %s */\n", zIndent, zFrom, zTarget ); @@ -7472,16 +6104,16 @@ static int lintFkeyIndexes( sqlite3_free(zPrev); if( rc!=SQLITE_OK ){ - sqlite3_fprintf(stderr,"%s\n", sqlite3_errmsg(db)); + cli_printf(stderr,"%s\n", sqlite3_errmsg(db)); } rc2 = sqlite3_finalize(pSql); if( rc==SQLITE_OK && rc2!=SQLITE_OK ){ rc = rc2; - sqlite3_fprintf(stderr,"%s\n", sqlite3_errmsg(db)); + cli_printf(stderr,"%s\n", sqlite3_errmsg(db)); } }else{ - sqlite3_fprintf(stderr,"%s\n", sqlite3_errmsg(db)); + cli_printf(stderr,"%s\n", sqlite3_errmsg(db)); } return rc; @@ -7501,9 +6133,9 @@ static int lintDotCommand( return lintFkeyIndexes(pState, azArg, nArg); usage: - sqlite3_fprintf(stderr,"Usage %s sub-command ?switches...?\n", azArg[0]); - sqlite3_fprintf(stderr, "Where sub-commands are:\n"); - sqlite3_fprintf(stderr, " fkey-indexes\n"); + cli_printf(stderr,"Usage %s sub-command ?switches...?\n", azArg[0]); + cli_printf(stderr, "Where sub-commands are:\n"); + cli_printf(stderr, " fkey-indexes\n"); return SQLITE_ERROR; } @@ -7517,7 +6149,7 @@ static void shellPrepare( if( *pRc==SQLITE_OK ){ int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); if( rc!=SQLITE_OK ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "sql error: %s (%d)\n", sqlite3_errmsg(db), sqlite3_errcode(db)); *pRc = rc; } @@ -7562,7 +6194,7 @@ static void shellFinalize( int rc = sqlite3_finalize(pStmt); if( *pRc==SQLITE_OK ){ if( rc!=SQLITE_OK ){ - sqlite3_fprintf(stderr,"SQL error: %s\n", sqlite3_errmsg(db)); + cli_printf(stderr,"SQL error: %s\n", sqlite3_errmsg(db)); } *pRc = rc; } @@ -7584,7 +6216,7 @@ void shellReset( if( *pRc==SQLITE_OK ){ if( rc!=SQLITE_OK ){ sqlite3 *db = sqlite3_db_handle(pStmt); - sqlite3_fprintf(stderr,"SQL error: %s\n", sqlite3_errmsg(db)); + cli_printf(stderr,"SQL error: %s\n", sqlite3_errmsg(db)); } *pRc = rc; } @@ -7637,9 +6269,9 @@ static int arErrorMsg(ArCommand *pAr, const char *zFmt, ...){ va_end(ap); shellEmitError(z); if( pAr->fromCmdLine ){ - sqlite3_fputs("Use \"-A\" for more help\n", stderr); + cli_puts("Use \"-A\" for more help\n", stderr); }else{ - sqlite3_fputs("Use \".archive --help\" for more help\n", stderr); + cli_puts("Use \".archive --help\" for more help\n", stderr); } sqlite3_free(z); return SQLITE_ERROR; @@ -7739,7 +6371,7 @@ static int arParseCommand( struct ArSwitch *pEnd = &aSwitch[nSwitch]; if( nArg<=1 ){ - sqlite3_fprintf(stderr, "Wrong number of arguments. Usage:\n"); + cli_printf(stderr, "Wrong number of arguments. Usage:\n"); return arUsage(stderr); }else{ char *z = azArg[1]; @@ -7845,7 +6477,7 @@ static int arParseCommand( } } if( pAr->eCmd==0 ){ - sqlite3_fprintf(stderr, "Required argument missing. Usage:\n"); + cli_printf(stderr, "Required argument missing. Usage:\n"); return arUsage(stderr); } return SQLITE_OK; @@ -7888,7 +6520,7 @@ static int arCheckEntries(ArCommand *pAr){ } shellReset(&rc, pTest); if( rc==SQLITE_OK && bOk==0 ){ - sqlite3_fprintf(stderr,"not found in archive: %s\n", z); + cli_printf(stderr,"not found in archive: %s\n", z); rc = SQLITE_ERROR; } } @@ -7971,15 +6603,15 @@ static int arListCommand(ArCommand *pAr){ shellPreparePrintf(pAr->db, &rc, &pSql, zSql, azCols[pAr->bVerbose], pAr->zSrcTable, zWhere); if( pAr->bDryRun ){ - sqlite3_fprintf(pAr->out, "%s\n", sqlite3_sql(pSql)); + cli_printf(pAr->out, "%s\n", sqlite3_sql(pSql)); }else{ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ if( pAr->bVerbose ){ - sqlite3_fprintf(pAr->out, "%s % 10d %s %s\n", + cli_printf(pAr->out, "%s % 10d %s %s\n", sqlite3_column_text(pSql, 0), sqlite3_column_int(pSql, 1), sqlite3_column_text(pSql, 2),sqlite3_column_text(pSql, 3)); }else{ - sqlite3_fprintf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0)); + cli_printf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0)); } } } @@ -8006,7 +6638,7 @@ static int arRemoveCommand(ArCommand *pAr){ zSql = sqlite3_mprintf("DELETE FROM %s WHERE %s;", pAr->zSrcTable, zWhere); if( pAr->bDryRun ){ - sqlite3_fprintf(pAr->out, "%s\n", zSql); + cli_printf(pAr->out, "%s\n", zSql); }else{ char *zErr = 0; rc = sqlite3_exec(pAr->db, "SAVEPOINT ar;", 0, 0, 0); @@ -8019,7 +6651,7 @@ static int arRemoveCommand(ArCommand *pAr){ } } if( zErr ){ - sqlite3_fprintf(stdout, "ERROR: %s\n", zErr); /* stdout? */ + cli_printf(stdout, "ERROR: %s\n", zErr); /* stdout? */ sqlite3_free(zErr); } } @@ -8034,11 +6666,15 @@ static int arRemoveCommand(ArCommand *pAr){ */ static int arExtractCommand(ArCommand *pAr){ const char *zSql1 = - "SELECT " - " ($dir || name)," - " writefile(($dir || name), %s, mode, mtime) " - "FROM %s WHERE (%s) AND (data IS NULL OR $dirOnly = 0)" - " AND name NOT GLOB '*..[/\\]*'"; + "WITH dest(dpath,dlen) AS (SELECT realpath($dir),length(realpath($dir)))\n" + "SELECT ($dir || name),\n" + " CASE WHEN $dryrun THEN 0\n" + " ELSE writefile($dir||name, %s, mode, mtime) END\n" + " FROM dest CROSS JOIN %s\n" + " WHERE (%s)\n" + " AND (data IS NULL OR $pass==0)\n" /* Dirs both passes */ + " AND dpath=substr(realpath($dir||name),1,dlen)\n" /* No escapes */ + " AND name NOT GLOB '*..[/\\]*'\n"; /* No /../ in paths */ const char *azExtraArg[] = { "sqlar_uncompress(data, sz)", @@ -8073,24 +6709,28 @@ static int arExtractCommand(ArCommand *pAr){ if( rc==SQLITE_OK ){ j = sqlite3_bind_parameter_index(pSql, "$dir"); sqlite3_bind_text(pSql, j, zDir, -1, SQLITE_STATIC); - - /* Run the SELECT statement twice. The first time, writefile() is called - ** for all archive members that should be extracted. The second time, - ** only for the directories. This is because the timestamps for - ** extracted directories must be reset after they are populated (as - ** populating them changes the timestamp). */ + j = sqlite3_bind_parameter_index(pSql, "$dryrun"); + sqlite3_bind_int(pSql, j, pAr->bDryRun); + + /* Run the SELECT statement twice + ** (0) writefile() all files and directories + ** (1) writefile() for directory again + ** The second pass is so that the timestamps for extracted directories + ** will be reset to the value in the archive, since populating them + ** in the first pass will have changed the timestamp. */ for(i=0; i<2; i++){ - j = sqlite3_bind_parameter_index(pSql, "$dirOnly"); + j = sqlite3_bind_parameter_index(pSql, "$pass"); sqlite3_bind_int(pSql, j, i); if( pAr->bDryRun ){ - sqlite3_fprintf(pAr->out, "%s\n", sqlite3_sql(pSql)); - }else{ - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ - if( i==0 && pAr->bVerbose ){ - sqlite3_fprintf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0)); - } + cli_printf(pAr->out, "%s\n", sqlite3_sql(pSql)); + if( pAr->bVerbose==0 ) break; + } + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ + if( i==0 && pAr->bVerbose ){ + cli_printf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0)); } } + if( pAr->bDryRun ) break; shellReset(&rc, pSql); } shellFinalize(&rc, pSql); @@ -8107,13 +6747,13 @@ static int arExtractCommand(ArCommand *pAr){ static int arExecSql(ArCommand *pAr, const char *zSql){ int rc; if( pAr->bDryRun ){ - sqlite3_fprintf(pAr->out, "%s\n", zSql); + cli_printf(pAr->out, "%s\n", zSql); rc = SQLITE_OK; }else{ char *zErr = 0; rc = sqlite3_exec(pAr->db, zSql, 0, 0, &zErr); if( zErr ){ - sqlite3_fprintf(stdout, "ERROR: %s\n", zErr); + cli_printf(stdout, "ERROR: %s\n", zErr); sqlite3_free(zErr); } } @@ -8289,13 +6929,13 @@ static int arDotCommand( } cmd.db = 0; if( cmd.bDryRun ){ - sqlite3_fprintf(cmd.out, "-- open database '%s'%s\n", cmd.zFile, + cli_printf(cmd.out, "-- open database '%s'%s\n", cmd.zFile, eDbType==SHELL_OPEN_APPENDVFS ? " using 'apndvfs'" : ""); } rc = sqlite3_open_v2(cmd.zFile, &cmd.db, flags, eDbType==SHELL_OPEN_APPENDVFS ? "apndvfs" : 0); if( rc!=SQLITE_OK ){ - sqlite3_fprintf(stderr, "cannot open file: %s (%s)\n", + cli_printf(stderr, "cannot open file: %s (%s)\n", cmd.zFile, sqlite3_errmsg(cmd.db)); goto end_ar_command; } @@ -8309,7 +6949,7 @@ static int arDotCommand( if( cmd.eCmd!=AR_CMD_CREATE && sqlite3_table_column_metadata(cmd.db,0,"sqlar","name",0,0,0,0,0) ){ - sqlite3_fprintf(stderr, "database does not contain an 'sqlar' table\n"); + cli_printf(stderr, "database does not contain an 'sqlar' table\n"); rc = SQLITE_ERROR; goto end_ar_command; } @@ -8367,7 +7007,7 @@ end_ar_command: */ static int recoverSqlCb(void *pCtx, const char *zSql){ ShellState *pState = (ShellState*)pCtx; - sqlite3_fprintf(pState->out, "%s;\n", zSql); + cli_printf(pState->out, "%s;\n", zSql); return SQLITE_OK; } @@ -8410,7 +7050,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ bRowids = 0; } else{ - sqlite3_fprintf(stderr,"unexpected option: %s\n", azArg[i]); + cli_printf(stderr,"unexpected option: %s\n", azArg[i]); showHelp(pState->out, azArg[0]); return 1; } @@ -8420,17 +7060,19 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ pState->db, "main", recoverSqlCb, (void*)pState ); - sqlite3_recover_config(p, 789, (void*)zRecoveryDb); /* Debug use only */ + if( !pState->bSafeMode ){ + sqlite3_recover_config(p, 789, (void*)zRecoveryDb); /* Debug use only */ + } sqlite3_recover_config(p, SQLITE_RECOVER_LOST_AND_FOUND, (void*)zLAF); sqlite3_recover_config(p, SQLITE_RECOVER_ROWIDS, (void*)&bRowids); sqlite3_recover_config(p, SQLITE_RECOVER_FREELIST_CORRUPT,(void*)&bFreelist); - sqlite3_fprintf(pState->out, ".dbconfig defensive off\n"); + cli_printf(pState->out, ".dbconfig defensive off\n"); sqlite3_recover_run(p); if( sqlite3_recover_errcode(p)!=SQLITE_OK ){ const char *zErr = sqlite3_recover_errmsg(p); int errCode = sqlite3_recover_errcode(p); - sqlite3_fprintf(stderr,"sql error: %s (%d)\n", zErr, errCode); + cli_printf(stderr,"sql error: %s (%d)\n", zErr, errCode); } rc = sqlite3_recover_finish(p); return rc; @@ -8452,7 +7094,7 @@ static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){ while( SQLITE_OK==sqlite3_intck_step(p) ){ const char *zMsg = sqlite3_intck_message(p); if( zMsg ){ - sqlite3_fprintf(pState->out, "%s\n", zMsg); + cli_printf(pState->out, "%s\n", zMsg); nError++; } nStep++; @@ -8462,11 +7104,11 @@ static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){ } rc = sqlite3_intck_error(p, &zErr); if( zErr ){ - sqlite3_fprintf(stderr,"%s\n", zErr); + cli_printf(stderr,"%s\n", zErr); } sqlite3_intck_close(p); - sqlite3_fprintf(pState->out, "%lld steps, %lld errors\n", nStep, nError); + cli_printf(pState->out, "%lld steps, %lld errors\n", nStep, nError); } return rc; @@ -8489,7 +7131,7 @@ static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){ #define rc_err_oom_die(rc) \ if( rc==SQLITE_NOMEM ) shell_check_oom(0); \ else if(!(rc==SQLITE_OK||rc==SQLITE_DONE)) \ - sqlite3_fprintf(stderr,"E:%d\n",rc), assert(0) + cli_printf(stderr,"E:%d\n",rc), assert(0) #else static void rc_err_oom_die(int rc){ if( rc==SQLITE_NOMEM ) shell_check_oom(0); @@ -8604,7 +7246,7 @@ SELECT CASE WHEN (nc < 10) THEN 1 WHEN (nc < 100) THEN 2 \ SELECT\ '('||x'0a'\ || group_concat(\ - cname||' TEXT',\ + cname||' ANY',\ ','||iif((cpos-1)%4>0, ' ', x'0a'||' '))\ ||')' AS ColsSpec \ FROM (\ @@ -8670,90 +7312,1648 @@ FROM (\ if( rc==SQLITE_ROW ){ zColsSpec = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); }else{ - zColsSpec = 0; + zColsSpec = 0; + } + if( pzRenamed!=0 ){ + if( !hasDupes ) *pzRenamed = 0; + else{ + sqlite3_finalize(pStmt); + if( SQLITE_OK==sqlite3_prepare_v2(*pDb, zRenamesDone, -1, &pStmt, 0) + && SQLITE_ROW==sqlite3_step(pStmt) ){ + *pzRenamed = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); + }else + *pzRenamed = 0; + } + } + sqlite3_finalize(pStmt); + sqlite3_close(*pDb); + *pDb = 0; + return zColsSpec; + } +} + +/* +** Check if the sqlite_schema table contains one or more virtual tables. If +** parameter zLike is not NULL, then it is an SQL expression that the +** sqlite_schema row must also match. If one or more such rows are found, +** print the following warning to the output: +** +** WARNING: Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled +*/ +static int outputDumpWarning(ShellState *p, const char *zLike){ + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + shellPreparePrintf(p->db, &rc, &pStmt, + "SELECT 1 FROM sqlite_schema o WHERE " + "sql LIKE 'CREATE VIRTUAL TABLE%%' AND %s", zLike ? zLike : "true" + ); + if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + cli_puts("/* WARNING: " + "Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled */\n", + p->out + ); + } + shellFinalize(&rc, pStmt); + return rc; +} + +/* +** Fault-Simulator state and logic. +*/ +static struct { + int iId; /* ID that triggers a simulated fault. -1 means "any" */ + int iErr; /* The error code to return on a fault */ + int iCnt; /* Trigger the fault only if iCnt is already zero */ + int iInterval; /* Reset iCnt to this value after each fault */ + int eVerbose; /* When to print output */ + int nHit; /* Number of hits seen so far */ + int nRepeat; /* Turn off after this many hits. 0 for never */ + int nSkip; /* Skip this many before first fault */ +} faultsim_state = {-1, 0, 0, 0, 0, 0, 0, 0}; + +/* +** This is the fault-sim callback +*/ +static int faultsim_callback(int iArg){ + if( faultsim_state.iId>0 && faultsim_state.iId!=iArg ){ + return SQLITE_OK; + } + if( faultsim_state.iCnt ){ + if( faultsim_state.iCnt>0 ) faultsim_state.iCnt--; + if( faultsim_state.eVerbose>=2 ){ + cli_printf(stdout, + "FAULT-SIM id=%d no-fault (cnt=%d)\n", iArg, faultsim_state.iCnt); + } + return SQLITE_OK; + } + if( faultsim_state.eVerbose>=1 ){ + cli_printf(stdout, + "FAULT-SIM id=%d returns %d\n", iArg, faultsim_state.iErr); + } + faultsim_state.iCnt = faultsim_state.iInterval; + faultsim_state.nHit++; + if( faultsim_state.nRepeat>0 && faultsim_state.nRepeat<=faultsim_state.nHit ){ + faultsim_state.iCnt = -1; + } + return faultsim_state.iErr; +} + +/* +** pickStr(zArg, &zErr, zS1, zS2, ..., ""); +** +** Try to match zArg against zS1, zS2, and so forth until the first +** emptry string. Return the index of the match or -1 if none is found. +** If no match is found, and &zErr is not NULL, then write into +** zErr a message describing the valid choices. +*/ +static int pickStr(const char *zArg, char **pzErr, ...){ + int i, n; + const char *z; + sqlite3_str *pMsg; + va_list ap; + va_start(ap, pzErr); + i = 0; + while( (z = va_arg(ap,const char*))!=0 && z[0]!=0 ){ + if( cli_strcmp(zArg, z)==0 ) return i; + i++; + } + va_end(ap); + if( pzErr==0 ) return -1; + n = i; + pMsg = sqlite3_str_new(0); + va_start(ap, pzErr); + sqlite3_str_appendall(pMsg, "should be"); + i = 0; + while( (z = va_arg(ap, const char*))!=0 && z[0]!=0 ){ + if( i==n-1 ){ + sqlite3_str_append(pMsg,", or",4); + }else if( i>0 ){ + sqlite3_str_append(pMsg, ",", 1); + } + sqlite3_str_appendf(pMsg, " %s", z); + i++; + } + va_end(ap); + *pzErr = sqlite3_str_finish(pMsg); + return -1; +} + +/* +** DOT-COMMAND: .import +** +** USAGE: .import [OPTIONS] FILE TABLE +** +** Import CSV or similar text from FILE into TABLE. If TABLE does +** not exist, it is created using the first row of FILE as the column +** names. If FILE begins with "|" then it is a command that is run +** and the output from the command is used as the input data. If +** FILE begins with "<<" followed by a label, then content is read from +** the script until the first line that matches the label. +** +** The content of FILE is interpreted using RFC-4180 ("CSV") quoting +** rules unless the current mode is "ascii" or "tabs" or unless one +** the --ascii option is used. +** +** The column and row separators must be single ASCII characters. If +** multiple characters or a Unicode character are specified for the +** separators, then only the first byte of the separator is used. Except, +** if the row separator is \n and the mode is not --ascii, then \r\n is +** understood as a row separator too. +** +** Options: +** --ascii Do not use RFC-4180 quoting. Use \037 and \036 +** as column and row separators on input, unless other +** delimiters are specified using --colsep and/or --rowsep +** --colsep CHAR Use CHAR as the column separator. +** --csv Input is standard RFC-4180 CSV. +** --esc CHAR Use CHAR as an escape character in unquoted CSV inputs. +** --qesc CHAR Use CHAR as an escape character in quoted CSV inputs. +** --rowsep CHAR Use CHAR as the row separator. +** --schema S When creating TABLE, put it in schema S +** --skip N Ignore the first N rows of input +** -v Verbose mode +*/ +static int dotCmdImport(ShellState *p){ + int nArg = p->dot.nArg; /* Number of arguments */ + char **azArg = p->dot.azArg;/* Argument list */ + char *zTable = 0; /* Insert data into this table */ + char *zSchema = 0; /* Schema of zTable */ + char *zFile = 0; /* Name of file to extra content from */ + sqlite3_stmt *pStmt = NULL; /* A statement */ + int nCol; /* Number of columns in the table */ + i64 nByte; /* Number of bytes in an SQL string */ + int i, j; /* Loop counters */ + int needCommit; /* True to COMMIT or ROLLBACK at end */ + char *zSql = 0; /* An SQL statement */ + ImportCtx sCtx; /* Reader context */ + char *(SQLITE_CDECL *xRead)(ImportCtx*); /* Func to read one value */ + int eVerbose = 0; /* Larger for more console output */ + i64 nSkip = 0; /* Initial lines to skip */ + i64 iLineOffset = 0; /* Offset to the first line of input */ + char *zCreate = 0; /* CREATE TABLE statement text */ + int rc; /* Result code */ + + failIfSafeMode(p, "cannot run .import in safe mode"); + memset(&sCtx, 0, sizeof(sCtx)); + if( p->mode.eMode==MODE_Ascii ){ + xRead = ascii_read_one_field; + }else{ + xRead = csv_read_one_field; + } + for(i=1; imode.spec.zColumnSep && p->mode.spec.zColumnSep[0]!=0 ){ + sCtx.cColSep = p->mode.spec.zColumnSep[0]; + }else{ + sCtx.cColSep = ','; + } + } + if( (sCtx.cColSep & 0x80)!=0 ){ + eputz("Error: .import column separator must be ASCII\n"); + return 1; + } + if( sCtx.cRowSep==0 ){ + if( p->mode.spec.zRowSep && p->mode.spec.zRowSep[0]!=0 ){ + sCtx.cRowSep = p->mode.spec.zRowSep[0]; + }else{ + sCtx.cRowSep = '\n'; + } + } + if( sCtx.cRowSep=='\r' && xRead!=ascii_read_one_field ){ + sCtx.cRowSep = '\n'; + } + if( (sCtx.cRowSep & 0x80)!=0 ){ + eputz("Error: .import row separator must be ASCII\n"); + return 1; + } + sCtx.zFile = zFile; + sCtx.nLine = 1; + if( sCtx.zFile[0]=='|' ){ +#ifdef SQLITE_OMIT_POPEN + eputz("Error: pipes are not supported in this OS\n"); + return 1; +#else + sCtx.in = sqlite3_popen(sCtx.zFile+1, "r"); + sCtx.zFile = ""; + sCtx.xCloser = pclose; +#endif + }else if( sCtx.zFile[0]=='<' && sCtx.zFile[1]=='<' && sCtx.zFile[2]!=0 ){ + /* Input text comes from subsequent lines of script until the zFile + ** delimiter */ + int nEndMark = strlen30(zFile)-2; + char *zEndMark = &zFile[2]; + sqlite3_str *pContent = sqlite3_str_new(p->db); + int ckEnd = 1; + i64 iStart = p->lineno; + char zLine[2000]; + sCtx.zFile = p->zInFile; + sCtx.nLine = p->lineno+1; + iLineOffset = p->lineno; + while( sqlite3_fgets(zLine,sizeof(zLine),p->in) ){ + if( ckEnd && cli_strncmp(zLine,zEndMark,nEndMark)==0 ){ + ckEnd = 2; + if( strchr(zLine,'\n') ) p->lineno++; + break; + } + if( strchr(zLine,'\n') ){ + p->lineno++; + ckEnd = 1; + }else{ + ckEnd = 0; + } + sqlite3_str_appendall(pContent, zLine); + } + sCtx.zIn = sqlite3_str_finish(pContent); + if( sCtx.zIn==0 ){ + sCtx.zIn = sqlite3_mprintf(""); + } + if( ckEnd<2 ){ + i64 savedLn = p->lineno; + p->lineno = iStart; + dotCmdError(p, 0, 0,"Content terminator \"%s\" not found.",zEndMark); + p->lineno = savedLn; + import_cleanup(&sCtx); + return 1; + } + }else{ + sCtx.in = sqlite3_fopen(sCtx.zFile, "rb"); + sCtx.xCloser = fclose; + } + if( sCtx.in==0 && sCtx.zIn==0 ){ + dotCmdError(p, 0, 0, "cannot open \"%s\"", zFile); + import_cleanup(&sCtx); + return 1; + } + if( eVerbose>=1 ){ + char zSep[2]; + zSep[1] = 0; + zSep[0] = sCtx.cColSep; + cli_puts("Column separator ", p->out); + output_c_string(p->out, zSep); + cli_puts(", row separator ", p->out); + zSep[0] = sCtx.cRowSep; + output_c_string(p->out, zSep); + cli_puts("\n", p->out); + } + sCtx.z = sqlite3_malloc64(120); + if( sCtx.z==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + /* Below, resources must be freed before exit. */ + while( nSkip>0 ){ + nSkip--; + while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){} + } + import_append_char(&sCtx, 0); /* To ensure sCtx.z is allocated */ + if( sqlite3_table_column_metadata(p->db, zSchema, zTable,0,0,0,0,0,0) + && 0==db_int(p->db, "SELECT count(*) FROM \"%w\".sqlite_schema" + " WHERE name=%Q AND type='view'", + zSchema ? zSchema : "main", zTable) + ){ + /* Table does not exist. Create it. */ + sqlite3 *dbCols = 0; + char *zRenames = 0; + char *zColDefs; + zCreate = sqlite3_mprintf("CREATE TABLE \"%w\".\"%w\"", + zSchema ? zSchema : "main", zTable); + while( xRead(&sCtx) ){ + zAutoColumn(sCtx.z, &dbCols, 0); + if( sCtx.cTerm!=sCtx.cColSep ) break; + } + zColDefs = zAutoColumn(0, &dbCols, &zRenames); + if( zRenames!=0 ){ + cli_printf((stdin_is_interactive && p->in==stdin)? p->out : stderr, + "Columns renamed during .import %s due to duplicates:\n" + "%s\n", sCtx.zFile, zRenames); + sqlite3_free(zRenames); + } + assert(dbCols==0); + if( zColDefs==0 ){ + cli_printf(stderr,"%s: empty file\n", sCtx.zFile); + import_cleanup(&sCtx); + sqlite3_free(zCreate); + return 1; + } + zCreate = sqlite3_mprintf("%z%z\n", zCreate, zColDefs); + if( zCreate==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + if( eVerbose>=1 ){ + cli_printf(p->out, "%s\n", zCreate); + } + rc = sqlite3_exec(p->db, zCreate, 0, 0, 0); + if( rc ){ + cli_printf(stderr, + "%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db)); + } + sqlite3_free(zCreate); + zCreate = 0; + if( rc ){ + import_cleanup(&sCtx); + return 1; + } + } + zSql = sqlite3_mprintf("SELECT count(*) FROM pragma_table_info(%Q,%Q);", + zTable, zSchema); + if( zSql==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + zSql = 0; + if( rc ){ + if (pStmt) sqlite3_finalize(pStmt); + shellDatabaseError(p->db); + import_cleanup(&sCtx); + return 1; + } + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + nCol = sqlite3_column_int(pStmt, 0); + }else{ + nCol = 0; + } + sqlite3_finalize(pStmt); + pStmt = 0; + if( nCol==0 ) return 0; /* no columns, no error */ + + nByte = 64 /* space for "INSERT INTO", "VALUES(", ")\0" */ + + (zSchema ? strlen(zSchema)*2 + 2: 0) /* Quoted schema name */ + + strlen(zTable)*2 + 2 /* Quoted table name */ + + nCol*2; /* Space for ",?" for each column */ + zSql = sqlite3_malloc64( nByte ); + if( zSql==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + if( zSchema ){ + sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\".\"%w\" VALUES(?", + zSchema, zTable); + }else{ + sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); + } + j = strlen30(zSql); + for(i=1; i=2 ){ + cli_printf(p->out, "Insert using: %s\n", zSql); + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + zSql = 0; + if( rc ){ + shellDatabaseError(p->db); + if (pStmt) sqlite3_finalize(pStmt); + import_cleanup(&sCtx); + return 1; + } + needCommit = sqlite3_get_autocommit(p->db); + if( needCommit ) sqlite3_exec(p->db, "BEGIN", 0, 0, 0); + do{ + int startLine = sCtx.nLine; + for(i=0; imode.eMode==MODE_Ascii && (z==0 || z[0]==0) && i==0 ) break; + /* + ** For CSV mode, per RFC 4180, accept EOF in lieu of final + ** record terminator but only for last field of multi-field row. + ** (If there are too few fields, it's not valid CSV anyway.) + */ + if( z==0 && (xRead==csv_read_one_field) && i==nCol-1 && i>0 ){ + z = ""; + } + sqlite3_bind_text(pStmt, i+1, z, -1, SQLITE_TRANSIENT); + if( i=nCol ){ + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + cli_printf(stderr,"%s:%d: INSERT failed: %s\n", + sCtx.zFile, startLine, sqlite3_errmsg(p->db)); + sCtx.nErr++; + if( bail_on_error ) break; + }else{ + sCtx.nRow++; + } + } + }while( sCtx.cTerm!=EOF ); + + import_cleanup(&sCtx); + sqlite3_finalize(pStmt); + if( needCommit ) sqlite3_exec(p->db, "COMMIT", 0, 0, 0); + if( eVerbose>0 ){ + cli_printf(p->out, + "Added %d rows with %d errors using %d lines of input\n", + sCtx.nRow, sCtx.nErr, sCtx.nLine-1-iLineOffset); + } + return sCtx.nErr ? 1 : 0; +} + + +/* +** This function computes what to show the user about the configured +** titles (or column-names). Output is an integer between 0 and 3: +** +** 0: The titles do not matter. Never show anything. +** 1: Show "--titles off" +** 2: Show "--titles on" +** 3: Show "--title VALUE" where VALUE is an encoding method +** to use, one of: plain sql csv html tcl json +** +** Inputs are: +** +** spec.bTitles (bT) Whether or not to show the titles +** spec.eTitle (eT) The actual encoding to be used for titles +** ModeInfo.bHdr (bH) Default value for spec.bTitles +** ModeInfo.eHdr (eH) Default value for spec.eTitle +** bAll Whether the -v option is used +*/ +static int modeTitleDsply(ShellState *p, int bAll){ + int eMode = p->mode.eMode; + const ModeInfo *pI = &aModeInfo[eMode]; + int bT = p->mode.spec.bTitles; + int eT = p->mode.spec.eTitle; + int bH = pI->bHdr; + int eH = pI->eHdr; + + /* Variable "v" is the truth table that will determine the answer + ** + ** Actual encoding is different from default + ** vvvvvvvv */ + sqlite3_uint64 v = 0x0133013311220102; + /* ^^^^ ^^^^ + ** Upper 2-byte groups for when ON/OFF disagrees with + ** the default. */ + + if( bH==0 ) return 0; /* Header not appliable. Ex: off, count */ + + if( eT==0 ) eT = eH; /* Fill in missing spec.eTitle */ + if( bT==0 ) bT = bH; /* Fill in missing spec.bTitles */ + + if( eT!=eH ) v >>= 32; /* Encoding disagree in upper 4-bytes */ + if( bT!=bH ) v >>= 16; /* ON/OFF disagree in upper 2-byte pairs */ + if( bT<2 ) v >>= 8; /* ON in even bytes, OFF in odd bytes (1st byte 0) */ + if( !bAll ) v >>= 4; /* bAll values are in the lower half-byte */ + + return v & 3; /* Return the selected truth-table entry */ +} + +/* +** DOT-COMMAND: .mode +** +** USAGE: .mode [MODE] [OPTIONS] +** +** Change the output mode to MODE and/or apply OPTIONS to the output mode. +** Arguments are processed from left to right. If no arguments, show the +** current output mode and relevant options. +** +** Options: +** --align STRING Set the alignment of text in columnar modes +** String consists of characters 'L', 'C', 'R' +** meaning "left", "centered", and "right", with +** one letter per column starting from the left. +** Unspecified alignment defaults to 'L'. +** --blob-quote ARG ARG can be "auto", "text", "sql", "hex", "tcl", +** "json", or "size". Default is "auto". +** --border on|off Show outer border on "box" and "table" modes. +** --charlimit N Set the maximum number of output characters to +** show for any single SQL value to N. Longer values +** truncated. Zero means "no limit". +** --colsep STRING Use STRING as the column separator +** --escape ESC Enable/disable escaping of control characters +** found in the output. ESC can be "off", "ascii", +** or "symbol". +** --linelimit N Set the maximum number of output lines to show for +** any single SQL value to N. Longer values are +** truncated. Zero means "no limit". Only works +** in "line" mode and in columnar modes. +** --limits L,C,T Shorthand for "--linelimit L --charlimit C +** --titlelimit T". The ",T" can be omitted in which +** case the --titlelimit is unchanged. The argument +** can also be "off" to mean "0,0,0" or "on" to +** mean "5,300,20". +** --list List available modes +** --null STRING Render SQL NULL values as the given string +** --once Setting changes to the right are reverted after +** the next SQL command. +** --quote ARG Enable/disable quoting of text. ARG can be +** "off", "on", "sql", "relaxed", "csv", "html", +** "tcl", or "json". "off" means show the text as-is. +** "on" is an alias for "sql". +** --reset Changes all mode settings back to their default. +** --rowsep STRING Use STRING as the row separator +** --sw|--screenwidth N Declare the screen width of the output device +** to be N characters. An attempt may be made to +** wrap output text to fit within this limit. Zero +** means "no limit". Or N can be "auto" to set the +** width automatically. +** --tablename NAME Set the name of the table for "insert" mode. +** --tag NAME Save mode to the left as NAME. +** --textjsonb BOOLEAN If enabled, JSONB text is displayed as text JSON. +** --title ARG Whether or not to show column headers, and if so +** how to encode them. ARG can be "off", "on", +** "sql", "csv", "html", "tcl", or "json". +** --titlelimit N Limit the length of column titles to N characters. +** -v|--verbose Verbose output +** --widths LIST Set the columns widths for columnar modes. The +** argument is a list of integers, one for each +** column. A "0" width means use a dynamic width +** based on the actual width of data. If there are +** fewer entries in LIST than columns, "0" is used +** for the unspecified widths. +** --wordwrap BOOLEAN Enable/disable word wrapping +** --wrap N Wrap columns wider than N characters +** --ww Shorthand for "--wordwrap on" +*/ +static int dotCmdMode(ShellState *p){ + int nArg = p->dot.nArg; /* Number of arguments */ + char **azArg = p->dot.azArg;/* Argument list */ + int eMode = -1; /* New mode value, or -1 for none */ + int iMode = -1; /* Index of the argument that is the mode name */ + int i; /* Loop counter */ + int k; /* Misc index variable */ + int chng = 0; /* True if anything has changed */ + int bAll = 0; /* Show all details of the mode */ + + for(i=1; i=0 + && eMode!=MODE_Www + ){ + iMode = i; + modeChange(p, eMode); + /* (Legacy) If the mode is MODE_Insert and the next argument + ** is not an option, then the next argument must be the table + ** name. + */ + if( i+1mode.spec.zTableName, azArg[i]); + } + chng = 1; + }else if( optionMatch(z,"align") ){ + char *zAlign; + int nAlign; + int nErr = 0; + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + i++; + zAlign = azArg[i]; + nAlign = 0x3fff & strlen(zAlign); + free(p->mode.spec.aAlign); + p->mode.spec.aAlign = malloc(nAlign); + shell_check_oom(p->mode.spec.aAlign); + for(k=0; kmode.spec.aAlign[k] = c; + } + p->mode.spec.nAlign = nAlign; + chng = 1; + if( nErr ){ + dotCmdError(p, i, "bad alignment string", + "Should contain only characters L, C, and R."); + return 1; + } + }else if( pickStr(z,0,"-blob","-blob-quote","")>=0 ){ + if( (++i)>=nArg ){ + dotCmdError(p, i-1, "missing argument", 0); + return 1; + } + k = pickStr(azArg[i], 0, + "auto", "text", "sql", "hex", "tcl", "json", "size", ""); + /* 0 1 2 3 4 5 6 + ** Must match QRF_BLOB_xxxx values. See also tag-20251124a */ + if( k>=0 ){ + p->mode.spec.eBlob = k & 0xff; + } + chng = 1; + }else if( optionMatch(z,"border") ){ + if( (++i)>=nArg ){ + dotCmdError(p, i-1, "missing argument", 0); + return 1; + } + k = pickStr(azArg[i], 0, "auto", "off", "on", ""); + if( k>=0 ){ + p->mode.spec.bBorder = k & 0x3; + } + chng = 1; + }else if( 0<=(k=pickStr(z,0,"-charlimit","-linelimit","-titlelimit","")) ){ + int w; /* 0 1 */ + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + w = integerValue(azArg[++i]); + switch( k ){ + case 0: p->mode.spec.nCharLimit = w; break; + case 1: p->mode.spec.nLineLimit = w; break; + default: p->mode.spec.nTitleLimit = w; break; + } + chng = 1; + }else if( 0<=(k=pickStr(z,0,"-tablename","-rowsep","-colsep","-null","")) ){ + /* 0 1 2 3 */ + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + i++; + switch( k ){ + case 0: modeSetStr(&p->mode.spec.zTableName, azArg[i]); break; + case 1: modeSetStr(&p->mode.spec.zRowSep, azArg[i]); break; + case 2: modeSetStr(&p->mode.spec.zColumnSep, azArg[i]); break; + case 3: modeSetStr(&p->mode.spec.zNull, azArg[i]); break; + } + chng = 1; + }else if( optionMatch(z,"escape") ){ + /* See similar code at tag-20250224-1 */ + char *zErr = 0; + if( (++i)>=nArg ){ + dotCmdError(p, i-1, "missing argument", 0); + return 1; + } /* 0 1 2 <-- One less than QRF_ESC_ */ + k = pickStr(azArg[i],&zErr,"off","ascii","symbol",""); + if( k<0 ){ + dotCmdError(p, i, "unknown escape type", "%s", zErr); + sqlite3_free(zErr); + return 1; + } + p->mode.spec.eEsc = k+1; + chng = 1; + }else if( optionMatch(z,"limits") ){ + if( (++i)>=nArg ){ + dotCmdError(p, i-1, "missing argument", 0); + return 1; + } + k = pickStr(azArg[i],0,"on","off",""); + if( k==0 ){ + p->mode.spec.nLineLimit = DFLT_LINE_LIMIT; + p->mode.spec.nCharLimit = DFLT_CHAR_LIMIT; + p->mode.spec.nTitleLimit = DFLT_TITLE_LIMIT; + }else if( k==1 ){ + p->mode.spec.nLineLimit = 0; + p->mode.spec.nCharLimit = 0; + p->mode.spec.nTitleLimit = 0; + }else{ + int L, C, T = 0; + int nNum = sscanf(azArg[i], "%d,%d,%d", &L, &C, &T); + if( nNum<2 || L<0 || C<0 || T<0){ + dotCmdError(p, i, "bad argument", "Should be \"L,C,T\" where L, C" + " and T are unsigned integers"); + return 1; + } + p->mode.spec.nLineLimit = L; + p->mode.spec.nCharLimit = C; + if( nNum==3 ) p->mode.spec.nTitleLimit = T; + } + chng = 1; + }else if( optionMatch(z,"list") ){ + int ii; + cli_puts("available modes:", p->out); + for(ii=0; iiout, " %s", aModeInfo[ii].zName); + } + for(ii=0; iinSavedModes; ii++){ + cli_printf(p->out, " %s", p->aSavedModes[ii].zTag); + } + cli_puts(" batch tty\n", p->out); + chng = 1; /* Not really a change, but we still want to suppress the + ** "current mode" output */ + }else if( optionMatch(z,"once") ){ + p->nPopMode = 0; + modePush(p); + p->nPopMode = 1; + }else if( optionMatch(z,"noquote") ){ + /* (undocumented legacy) --noquote always turns quoting off */ + p->mode.spec.eText = QRF_TEXT_Plain; + p->mode.spec.eBlob = QRF_BLOB_Auto; + chng = 1; + }else if( optionMatch(z,"quote") ){ + if( i+10 || strcmp(azArg[i+1],"off")==0 || modeFind(p, azArg[i+1])<0) + ){ + /* --quote is followed by an argument other that is not an option + ** or a mode name. See it must be a boolean or a keyword to describe + ** how to set quoting. */ + i++; + if( (k = pickStr(azArg[i],0,"no","yes","0","1",""))>=0 ){ + k &= 1; /* 0 for "off". 1 for "on". */ + }else{ + char *zErr = 0; + k = pickStr(azArg[i],&zErr, + "off","on","sql","csv","html","tcl","json","relaxed",""); + /* 0 1 2 3 4 5 6 7 */ + if( k<0 ){ + dotCmdError(p, i, "unknown", "%z", zErr); + return 1; + } + } + }else{ + /* (Legacy) no following boolean argument. Turn quoting on */ + k = 1; + } + switch( k ){ + case 1: /* on */ + modeSetStr(&p->mode.spec.zNull, "NULL"); + /* Fall through */ + case 2: /* sql */ + p->mode.spec.eText = QRF_TEXT_Sql; + break; + case 3: /* csv */ + p->mode.spec.eText = QRF_TEXT_Csv; + break; + case 4: /* html */ + p->mode.spec.eText = QRF_TEXT_Html; + break; + case 5: /* tcl */ + p->mode.spec.eText = QRF_TEXT_Tcl; + break; + case 6: /* json */ + p->mode.spec.eText = QRF_TEXT_Json; + break; + case 7: /* relaxed */ + p->mode.spec.eText = QRF_TEXT_Relaxed; + break; + default: /* off */ + p->mode.spec.eText = QRF_TEXT_Plain; + break; + } + chng = 1; + }else if( optionMatch(z,"reset") ){ + int saved_eMode = p->mode.eMode; + modeFree(&p->mode); + modeChange(p, saved_eMode); + }else if( optionMatch(z,"screenwidth") || optionMatch(z,"sw") ){ + if( (++i)>=nArg ){ + dotCmdError(p, i-1, "missing argument", 0); + return 1; + } + k = pickStr(azArg[i],0,"off","auto",""); + if( k==0 ){ + p->mode.bAutoScreenWidth = 0; + p->mode.spec.nScreenWidth = 0; + }else if( k==1 ){ + p->mode.bAutoScreenWidth = 1; + }else{ + i64 w = integerValue(azArg[i]); + p->mode.bAutoScreenWidth = 0; + if( w<0 ) w = 0; + if( w>QRF_MAX_WIDTH ) w = QRF_MAX_WIDTH; + p->mode.spec.nScreenWidth = w; + } + chng = 1; + }else if( optionMatch(z,"tag") ){ + size_t nByte; + int n; + const char *zTag; + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + zTag = azArg[++i]; + if( modeFind(p, zTag)>=0 ){ + dotCmdError(p, i, "mode already exists", 0); + return 1; + } + if( p->nSavedModes > MODE_N_USER ){ + dotCmdError(p, i-1, "cannot add more modes", 0); + return 1; + } + n = p->nSavedModes++; + nByte = sizeof(p->aSavedModes[0]); + nByte *= n+1; + p->aSavedModes = realloc( p->aSavedModes, nByte ); + shell_check_oom(p->aSavedModes); + p->aSavedModes[n].zTag = strdup(zTag); + shell_check_oom(p->aSavedModes[n].zTag); + modeDup(&p->aSavedModes[n].mode, &p->mode); + chng = 1; + }else if( optionMatch(z,"textjsonb") ){ + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + p->mode.spec.bTextJsonb = booleanValue(azArg[++i]) ? QRF_Yes : QRF_No; + chng = 1; + }else if( optionMatch(z,"titles") || optionMatch(z,"title") ){ + char *zErr = 0; + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + k = pickStr(azArg[++i],&zErr, + "off","on","plain","sql","csv","html","tcl","json",""); + /* 0 1 2 3 4 5 6 7 */ + if( k<0 ){ + dotCmdError(p, i, "bad --titles value","%z", zErr); + return 1; + } + p->mode.spec.bTitles = k>=1 ? QRF_Yes : QRF_No; + p->mode.mFlags &= ~MFLG_HDR; + p->mode.spec.eTitle = k>1 ? k-1 : aModeInfo[p->mode.eMode].eHdr; + chng = 1; + }else if( optionMatch(z,"widths") || optionMatch(z,"width") ){ + int nWidth = 0; + short int *aWidth; + const char *zW; + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + zW = azArg[++i]; + /* Every width value takes at least 2 bytes in the input string to + ** specify, so strlen(zW) bytes should be plenty of space to hold the + ** result. */ + aWidth = malloc( strlen(zW) ); + while( IsSpace(zW[0]) ) zW++; + while( zW[0] ){ + int w = 0; + int nDigit = 0; + k = zW[0]=='-' && IsDigit(zW[1]); + while( IsDigit(zW[k]) ){ + w = w*10 + zW[k] - '0'; + if( w>QRF_MAX_WIDTH ){ + dotCmdError(p,i+1,"width too big", + "Maximum column width is %d", QRF_MAX_WIDTH); + free(aWidth); + return 1; + } + nDigit++; + k++; + } + if( nDigit==0 ){ + dotCmdError(p,i+1,"syntax error", + "should be a comma-separated list if integers"); + free(aWidth); + return 1; + } + if( zW[0]=='-' ) w = -w; + aWidth[nWidth++] = w; + zW += k; + if( zW[0]==',' ) zW++; + while( IsSpace(zW[0]) ) zW++; + } + free(p->mode.spec.aWidth); + p->mode.spec.aWidth = aWidth; + p->mode.spec.nWidth = nWidth; + chng = 1; + }else if( optionMatch(z,"wrap") ){ + int w; + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + w = integerValue(azArg[++i]); + if( w<(-QRF_MAX_WIDTH) ) w = -QRF_MAX_WIDTH; + if( w>QRF_MAX_WIDTH ) w = QRF_MAX_WIDTH; + p->mode.spec.nWrap = w; + chng = 1; + }else if( optionMatch(z,"ww") ){ + p->mode.spec.bWordWrap = QRF_Yes; + chng = 1; + }else if( optionMatch(z,"wordwrap") ){ + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + p->mode.spec.bWordWrap = (u8)booleanValue(azArg[++i]) ? QRF_Yes : QRF_No; + chng = 1; + }else if( optionMatch(z,"v") || optionMatch(z,"verbose") ){ + bAll = 1; + }else if( z[0]=='-' ){ + dotCmdError(p, i, "bad option", "Use \".help .mode\" for more info"); + return 1; + }else{ + dotCmdError(p, i, iMode>0?"bad argument":"unknown mode", + "Use \".help .mode\" for more info"); + return 1; + } + } + if( !chng || bAll ){ + const ModeInfo *pI = aModeInfo + p->mode.eMode; + sqlite3_str *pDesc = sqlite3_str_new(p->db); + char *zDesc; + const char *zSetting; + + if( p->nPopMode ) sqlite3_str_appendall(pDesc, "--once "); + sqlite3_str_appendall(pDesc,pI->zName); + if( bAll || (p->mode.spec.nAlign && pI->eCx==2) ){ + int ii; + sqlite3_str_appendall(pDesc, " --align \""); + for(ii=0; iimode.spec.nAlign; ii++){ + unsigned char a = p->mode.spec.aAlign[ii]; + sqlite3_str_appendchar(pDesc, 1, "LLCR"[a&3]); + } + sqlite3_str_append(pDesc, "\"", 1); + } + if( bAll + || (p->mode.spec.bBorder==QRF_No) != ((pI->mFlg&1)!=0) + ){ + sqlite3_str_appendf(pDesc," --border %s", + p->mode.spec.bBorder==QRF_No ? "off" : "on"); + } + if( bAll || p->mode.spec.eBlob!=QRF_BLOB_Auto ){ + const char *azBQuote[] = + { "auto", "text", "sql", "hex", "tcl", "json", "size" }; + /* 0 1 2 3 4 5 6 + ** Must match QRF_BLOB_xxxx values. See all instances of tag-20251124a */ + u8 e = p->mode.spec.eBlob; + sqlite3_str_appendf(pDesc, " --blob-quote %s", azBQuote[e]); + } + zSetting = aModeStr[pI->eCSep]; + if( bAll || (zSetting && cli_strcmp(zSetting,p->mode.spec.zColumnSep)!=0) ){ + sqlite3_str_appendf(pDesc, " --colsep "); + append_c_string(pDesc, p->mode.spec.zColumnSep); + } + if( bAll || p->mode.spec.eEsc!=QRF_Auto ){ + sqlite3_str_appendf(pDesc, " --escape %s",qrfEscNames[p->mode.spec.eEsc]); + } + if( bAll + || (p->mode.spec.nLineLimit>0 && pI->eCx>0) + || p->mode.spec.nCharLimit>0 + || (p->mode.spec.nTitleLimit>0 && pI->eCx>0) + ){ + if( p->mode.spec.nLineLimit==0 + && p->mode.spec.nCharLimit==0 + && p->mode.spec.nTitleLimit==0 + ){ + sqlite3_str_appendf(pDesc, " --limits off"); + }else if( p->mode.spec.nLineLimit==DFLT_LINE_LIMIT + && p->mode.spec.nCharLimit==DFLT_CHAR_LIMIT + && p->mode.spec.nTitleLimit==DFLT_TITLE_LIMIT + ){ + sqlite3_str_appendf(pDesc, " --limits on"); + }else{ + sqlite3_str_appendf(pDesc, " --limits %d,%d,%d", + p->mode.spec.nLineLimit, p->mode.spec.nCharLimit, + p->mode.spec.nTitleLimit); + } + } + zSetting = aModeStr[pI->eNull]; + if( bAll || (zSetting && cli_strcmp(zSetting,p->mode.spec.zNull)!=0) ){ + sqlite3_str_appendf(pDesc, " --null "); + append_c_string(pDesc, p->mode.spec.zNull); + } + if( bAll + || (pI->eText!=p->mode.spec.eText && (pI->eText>1 || p->mode.spec.eText>1)) + ){ + sqlite3_str_appendf(pDesc," --quote %s",qrfQuoteNames[p->mode.spec.eText]); + } + zSetting = aModeStr[pI->eRSep]; + if( bAll || (zSetting && cli_strcmp(zSetting,p->mode.spec.zRowSep)!=0) ){ + sqlite3_str_appendf(pDesc, " --rowsep "); + append_c_string(pDesc, p->mode.spec.zRowSep); + } + if( bAll + || (pI->eCx && (p->mode.spec.nScreenWidth>0 || p->mode.bAutoScreenWidth)) + ){ + if( p->mode.bAutoScreenWidth ){ + sqlite3_str_appendall(pDesc, " --sw auto"); + }else{ + sqlite3_str_appendf(pDesc," --sw %d", + p->mode.spec.nScreenWidth); + } + } + if( bAll || p->mode.eMode==MODE_Insert ){ + sqlite3_str_appendf(pDesc," --tablename "); + append_c_string(pDesc, p->mode.spec.zTableName); + } + if( bAll || p->mode.spec.bTextJsonb ){ + sqlite3_str_appendf(pDesc," --textjsonb %s", + p->mode.spec.bTextJsonb==QRF_Yes ? "on" : "off"); + } + k = modeTitleDsply(p, bAll); + if( k==1 ){ + sqlite3_str_appendall(pDesc, " --titles off"); + }else if( k==2 ){ + sqlite3_str_appendall(pDesc, " --titles on"); + }else if( k==3 ){ + static const char *azTitle[] = + { "plain", "sql", "csv", "html", "tcl", "json"}; + sqlite3_str_appendf(pDesc, " --titles %s", + azTitle[p->mode.spec.eTitle-1]); + } + if( p->mode.spec.nWidth>0 && (bAll || pI->eCx==2) ){ + int ii; + const char *zSep = " --widths "; + for(ii=0; iimode.spec.nWidth; ii++){ + sqlite3_str_appendf(pDesc, "%s%d", zSep, (int)p->mode.spec.aWidth[ii]); + zSep = ","; + } + }else if( bAll ){ + sqlite3_str_appendall(pDesc, " --widths \"\""); + } + if( bAll || (pI->eCx>0 && p->mode.spec.bWordWrap) ){ + if( bAll ){ + sqlite3_str_appendf(pDesc, " --wordwrap %s", + p->mode.spec.bWordWrap==QRF_Yes ? "on" : "off"); + } + if( p->mode.spec.nWrap ){ + sqlite3_str_appendf(pDesc, " --wrap %d", p->mode.spec.nWrap); + } + if( !bAll ) sqlite3_str_append(pDesc, " --ww", 5); + } + zDesc = sqlite3_str_finish(pDesc); + cli_printf(p->out, ".mode %s\n", zDesc); + fflush(p->out); + sqlite3_free(zDesc); + } + return 0; +} + +/* +** DOT-COMMAND: .output +** USAGE: .output [OPTIONS] [FILE] +** +** Begin redirecting output to FILE. Or if FILE is omitted, revert +** to sending output to the console. If FILE begins with "|" then +** the remainder of file is taken as a pipe and output is directed +** into that pipe. If FILE is "memory" then output is captured in an +** internal memory buffer. If FILE is "off" then output is redirected +** into /dev/null or the equivalent. +** +** Options: +** --bom Prepend a byte-order mark to the output +** -e Accumulate output in a temporary text file then +** launch a text editor when the redirection ends. +** --error-prefix X Use X as the left-margin prefix for error messages. +** Set to an empty string to restore the default. +** --keep Keep redirecting output to its current destination. +** Use this option in combination with --show or +** with --error-prefix when you do not want to stop +** a current redirection. +** --plain Use plain text rather than HTML tables with -w +** --show Show output text captured by .testcase or by +** redirecting to "memory". +** -w Show the output in a web browser. Output is +** written into a temporary HTML file until the +** redirect ends, then the web browser is launched. +** Query results are shown as HTML tables, unless +** the --plain is used too. +** -x Show the output in a spreadsheet. Output is +** written to a temp file as CSV then the spreadsheet +** is launched when +** +** DOT-COMMAND: .once +** USAGE: .once [OPTIONS] FILE ... +** +** Write the output for the next line of SQL or the next dot-command into +** FILE. If FILE begins with "|" then it is a program into which output +** is written. The FILE argument should be omitted if one of the -e, -w, +** or -x options is used. +** +** Options: +** -e Capture output into a temporary file then bring up +** a text editor on that temporary file. +** --plain Use plain text rather than HTML tables with -w +** -w Capture output into an HTML file then bring up that +** file in a web browser +** -x Show the output in a spreadsheet. Output is +** written to a temp file as CSV then the spreadsheet +** is launched when +** +** DOT-COMMAND: .excel +** Shorthand for ".once -x" +** +** DOT-COMMAND: .www [--plain] +** Shorthand for ".once -w" or ".once --plain -w" +*/ +static int dotCmdOutput(ShellState *p){ + int nArg = p->dot.nArg; /* Number of arguments */ + char **azArg = p->dot.azArg; /* Text of the arguments */ + char *zFile = 0; /* The FILE argument */ + int i; /* Loop counter */ + int eMode = 0; /* 0: .outout/.once, 'x'=.excel, 'w'=.www */ + int bOnce = 0; /* 0: .output, 1: .once, 2: .excel/.www */ + int bPlain = 0; /* --plain option */ + int bKeep = 0; /* Keep redirecting */ + static const char *zBomUtf8 = "\357\273\277"; + const char *zBom = 0; + char c = azArg[0][0]; + int n = strlen30(azArg[0]); + + failIfSafeMode(p, "cannot run .%s in safe mode", azArg[0]); + if( c=='e' ){ + eMode = 'x'; + bOnce = 2; + }else if( c=='w' ){ + eMode = 'w'; + bOnce = 2; + }else if( n>=2 && cli_strncmp(azArg[0],"once",n)==0 ){ + bOnce = 1; + } + for(i=1; i=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + free(p->zErrPrefix); + i++; + p->zErrPrefix = azArg[i][0]==0 ? 0 : strdup(azArg[i]); + }else{ + dotCmdError(p, i, "unknown option", 0); + sqlite3_free(zFile); + return 1; + } + }else if( zFile==0 && eMode==0 ){ + if( bKeep ){ + dotCmdError(p, i, "incompatible with prior options",0); + goto dotCmdOutput_error; + } + if( cli_strcmp(z, "memory")==0 && bOnce ){ + dotCmdError(p, 0, "cannot redirect to \"memory\"", 0); + goto dotCmdOutput_error; + } + if( cli_strcmp(z, "off")==0 ){ +#ifdef _WIN32 + zFile = sqlite3_mprintf("nul"); +#else + zFile = sqlite3_mprintf("/dev/null"); +#endif + }else{ + zFile = sqlite3_mprintf("%s", z); + } + if( zFile && zFile[0]=='|' ){ + while( i+1nPopOutput = 2; + }else{ + p->nPopOutput = 0; + } + if( !bKeep ) output_reset(p); +#ifndef SQLITE_NOHAVE_SYSTEM + if( eMode=='e' || eMode=='x' || eMode=='w' ){ + p->doXdgOpen = 1; + modePush(p); + if( eMode=='x' ){ + /* spreadsheet mode. Output as CSV. */ + newTempFile(p, "csv"); + p->mode.mFlags &= ~MFLG_ECHO; + p->mode.eMode = MODE_Csv; + modeSetStr(&p->mode.spec.zColumnSep, SEP_Comma); + modeSetStr(&p->mode.spec.zRowSep, SEP_CrLf); +#ifdef _WIN32 + zBom = zBomUtf8; /* Always include the BOM on Windows, as Excel does + ** not work without it. */ +#endif + }else if( eMode=='w' ){ + /* web-browser mode. */ + newTempFile(p, "html"); + if( !bPlain ) p->mode.eMode = MODE_Www; + }else{ + /* text editor mode */ + newTempFile(p, "txt"); + } + sqlite3_free(zFile); + zFile = sqlite3_mprintf("%s", p->zTempFile); + } +#endif /* SQLITE_NOHAVE_SYSTEM */ + if( !bKeep ) shell_check_oom(zFile); + if( bKeep ){ + /* no-op */ + }else if( cli_strcmp(zFile,"memory")==0 ){ + if( cli_output_capture ){ + sqlite3_str_free(cli_output_capture); + } + cli_output_capture = sqlite3_str_new(0); + }else if( zFile[0]=='|' ){ +#ifdef SQLITE_OMIT_POPEN + eputz("Error: pipes are not supported in this OS\n"); + output_redir(p, stdout); + goto dotCmdOutput_error; +#else + FILE *pfPipe = sqlite3_popen(zFile + 1, "w"); + if( pfPipe==0 ){ + assert( stderr!=NULL ); + cli_printf(stderr,"Error: cannot open pipe \"%s\"\n", zFile + 1); + goto dotCmdOutput_error; + }else{ + output_redir(p, pfPipe); + if( zBom ) cli_puts(zBom, pfPipe); + sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); + } +#endif + }else{ + FILE *pfFile = output_file_open(p, zFile); + if( pfFile==0 ){ + if( cli_strcmp(zFile,"off")!=0 ){ + assert( stderr!=NULL ); + cli_printf(stderr,"Error: cannot write to \"%s\"\n", zFile); + } + goto dotCmdOutput_error; + } else { + output_redir(p, pfFile); + if( zBom ) cli_puts(zBom, pfFile); + if( bPlain && eMode=='w' ){ + cli_puts( + "\n\n\n", + pfFile + ); + } + sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); + } + } + sqlite3_free(zFile); + return 0; + +dotCmdOutput_error: + sqlite3_free(zFile); + return 1; +} + +/* +** DOT-COMMAND: .check +** USAGE: .check [OPTIONS] PATTERN +** +** Verify results of commands since the most recent .testcase command. +** Restore output to the console, unless --keep is used. +** +** If PATTERN starts with "<<ENDMARK" then the actual pattern is taken from +** subsequent lines of text up to the first line that begins with ENDMARK. +** All pattern lines and the ENDMARK are discarded. +** +** Options: +** --exact Do an exact comparison including leading and +** trailing whitespace. +** --glob Treat PATTERN as a GLOB +** --keep Do not reset the testcase. More .check commands +** will follow. +** --notglob Output should not match PATTERN +** --show Write testcase output to the screen, for debugging. +*/ +static int dotCmdCheck(ShellState *p){ + int nArg = p->dot.nArg; /* Number of arguments */ + char **azArg = p->dot.azArg; /* Text of the arguments */ + int i; /* Loop counter */ + int k; /* Result of pickStr() */ + char *zTest; /* Textcase result */ + int bKeep = 0; /* --keep option */ + char *zCheck = 0; /* PATTERN argument */ + char *zPattern = 0; /* Actual test pattern */ + int eCheck = 0; /* 1: --glob, 2: --notglob, 3: --exact */ + int isOk; /* True if results are OK */ + sqlite3_int64 iStart = p->lineno; /* Line number of .check statement */ + + if( p->zTestcase[0]==0 ){ + dotCmdError(p, 0, "no .testcase is active", 0); + return 1; + } + for(i=1; i<nArg; i++){ + char *z = azArg[i]; + if( z[0]=='-' && z[1]=='-' && z[2]!=0 ) z++; + if( cli_strcmp(z,"-keep")==0 ){ + bKeep = 1; + }else if( cli_strcmp(z,"-show")==0 ){ + if( cli_output_capture ){ + sqlite3_fprintf(stdout, "%s", sqlite3_str_value(cli_output_capture)); + } + bKeep = 1; + }else if( z[0]=='-' + && (k = pickStr(&z[1],0,"glob","notglob","exact",""))>=0 + ){ + if( eCheck && eCheck!=k+1 ){ + dotCmdError(p, i, "incompatible with prior options",0); + return 1; + } + eCheck = k+1; + }else if( zCheck ){ + dotCmdError(p, i, "unknown option", 0); + return 1; + }else{ + zCheck = azArg[i]; } - if( pzRenamed!=0 ){ - if( !hasDupes ) *pzRenamed = 0; - else{ - sqlite3_finalize(pStmt); - if( SQLITE_OK==sqlite3_prepare_v2(*pDb, zRenamesDone, -1, &pStmt, 0) - && SQLITE_ROW==sqlite3_step(pStmt) ){ - *pzRenamed = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); - }else - *pzRenamed = 0; - } + } + if( zCheck==0 ){ + dotCmdError(p, 0, "no PATTERN specified", 0); + return 1; + } + if( cli_output_capture && sqlite3_str_length(cli_output_capture) ){ + zTest = sqlite3_str_value(cli_output_capture); + shell_check_oom(zTest); + }else{ + zTest = ""; + } + p->nTestRun++; + if( zCheck[0]=='<' && zCheck[1]=='<' && zCheck[2]!=0 ){ + int nCheck = strlen30(zCheck); + sqlite3_str *pPattern = sqlite3_str_new(p->db); + char zLine[2000]; + while( sqlite3_fgets(zLine,sizeof(zLine),p->in) ){ + if( strchr(zLine,'\n') ) p->lineno++; + if( cli_strncmp(&zCheck[2],zLine,nCheck-2)==0 ) break; + sqlite3_str_appendall(pPattern, zLine); + } + zPattern = sqlite3_str_finish(pPattern); + if( zPattern==0 ){ + zPattern = sqlite3_mprintf(""); + } + }else{ + zPattern = zCheck; + } + shell_check_oom(zPattern); + switch( eCheck ){ + case 1: { + char *zGlob = sqlite3_mprintf("*%s*", zPattern); + isOk = testcase_glob(zGlob, zTest)!=0; + sqlite3_free(zGlob); + break; + } + case 2: { + char *zGlob = sqlite3_mprintf("*%s*", zPattern); + isOk = testcase_glob(zGlob, zTest)==0; + sqlite3_free(zGlob); + break; + } + case 3: { + isOk = cli_strcmp(zTest,zPattern)==0; + break; + } + default: { + /* Skip leading and trailing \n and \r on both pattern and test output */ + const char *z1 = zPattern; + const char *z2 = zTest; + size_t n1, n2; + while( z1[0]=='\n' || z1[0]=='\r' ) z1++; + n1 = strlen(z1); + while( n1>0 && (z1[n1-1]=='\n' || z1[n1-1]=='\r') ) n1--; + while( z2[0]=='\n' || z2[0]=='\r' ) z2++; + n2 = strlen(z2); + while( n2>0 && (z2[n2-1]=='\n' || z2[n2-1]=='\r') ) n2--; + isOk = n1==n2 && memcmp(z1,z2,n1)==0; + break; } - sqlite3_finalize(pStmt); - sqlite3_close(*pDb); - *pDb = 0; - return zColsSpec; } + if( !isOk ){ + sqlite3_fprintf(stderr, + "%s:%lld: .check failed for testcase %s\n", + p->zInFile, iStart, p->zTestcase); + p->nTestErr++; + sqlite3_fprintf(stderr, "Expected: [%s]\n", zPattern); + sqlite3_fprintf(stderr, "Got: [%s]\n", zTest); + } + if( zPattern!=zCheck ){ + sqlite3_free(zPattern); + } + if( !bKeep ){ + output_reset(p); + p->zTestcase[0] = 0; + } + return 0; } /* -** Check if the sqlite_schema table contains one or more virtual tables. If -** parameter zLike is not NULL, then it is an SQL expression that the -** sqlite_schema row must also match. If one or more such rows are found, -** print the following warning to the output: +** DOT-COMMAND: .testcase +** USAGE: .testcase [OPTIONS] NAME ** -** WARNING: Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled +** Start a new test case identified by NAME. All output +** through the next ".check" command is captured for comparison. See the +** ".check" commandn for additional informatioon. +** +** Options: +** --error-prefix TEXT Change error message prefix text to TEXT */ -static int outputDumpWarning(ShellState *p, const char *zLike){ - int rc = SQLITE_OK; - sqlite3_stmt *pStmt = 0; - shellPreparePrintf(p->db, &rc, &pStmt, - "SELECT 1 FROM sqlite_schema o WHERE " - "sql LIKE 'CREATE VIRTUAL TABLE%%' AND %s", zLike ? zLike : "true" - ); - if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ - sqlite3_fputs("/* WARNING: " - "Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled */\n", - p->out - ); +static int dotCmdTestcase(ShellState *p){ + int nArg = p->dot.nArg; /* Number of arguments */ + char **azArg = p->dot.azArg; /* Text of the arguments */ + int i; /* Loop counter */ + const char *zName = 0; /* Testcase name */ + + for(i=1; i<nArg; i++){ + char *z = azArg[i]; + if( z[0]=='-' && z[1]=='-' && z[2]!=0 ) z++; + if( optionMatch(z,"error-prefix") ){ + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + free(p->zErrPrefix); + i++; + p->zErrPrefix = azArg[i][0]==0 ? 0 : strdup(azArg[i]); + }else if( zName ){ + dotCmdError(p, i, "unknown option", 0); + return 1; + }else{ + zName = azArg[i]; + } } - shellFinalize(&rc, pStmt); - return rc; + output_reset(p); + if( cli_output_capture ){ + sqlite3_str_free(cli_output_capture); + } + cli_output_capture = sqlite3_str_new(0); + if( zName ){ + sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "%s", zName); + }else{ + sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "%s:%lld", + p->zInFile, p->lineno); + } + return 0; } /* -** Fault-Simulator state and logic. +** Enlarge the space allocated in p->dot so that it can hold more +** than nArg parsed command-line arguments. */ -static struct { - int iId; /* ID that triggers a simulated fault. -1 means "any" */ - int iErr; /* The error code to return on a fault */ - int iCnt; /* Trigger the fault only if iCnt is already zero */ - int iInterval; /* Reset iCnt to this value after each fault */ - int eVerbose; /* When to print output */ - int nHit; /* Number of hits seen so far */ - int nRepeat; /* Turn off after this many hits. 0 for never */ - int nSkip; /* Skip this many before first fault */ -} faultsim_state = {-1, 0, 0, 0, 0, 0, 0, 0}; +static void parseDotRealloc(ShellState *p, int nArg){ + p->dot.nAlloc = nArg+22; + p->dot.azArg = realloc(p->dot.azArg,p->dot.nAlloc*sizeof(char*)); + shell_check_oom(p->dot.azArg); + p->dot.aiOfst = realloc(p->dot.aiOfst,p->dot.nAlloc*sizeof(int)); + shell_check_oom(p->dot.aiOfst); + p->dot.abQuot = realloc(p->dot.abQuot,p->dot.nAlloc); + shell_check_oom(p->dot.abQuot); +} + /* -** This is the fault-sim callback +** Parse input line zLine up into individual arguments. Retain the +** parse in the p->dot substructure. */ -static int faultsim_callback(int iArg){ - if( faultsim_state.iId>0 && faultsim_state.iId!=iArg ){ - return SQLITE_OK; - } - if( faultsim_state.iCnt ){ - if( faultsim_state.iCnt>0 ) faultsim_state.iCnt--; - if( faultsim_state.eVerbose>=2 ){ - sqlite3_fprintf(stdout, - "FAULT-SIM id=%d no-fault (cnt=%d)\n", iArg, faultsim_state.iCnt); +static void parseDotCmdArgs(const char *zLine, ShellState *p){ + char *z; + int h = 1; + int nArg = 0; + size_t szLine; + + p->dot.zOrig = zLine; + free(p->dot.zCopy); + z = p->dot.zCopy = strdup(zLine); + shell_check_oom(z); + szLine = strlen(z); + while( szLine>0 && IsSpace(z[szLine-1]) ) szLine--; + if( szLine>0 && z[szLine-1]==';' ){ + szLine--; + while( szLine>0 && IsSpace(z[szLine-1]) ) szLine--; + } + z[szLine] = 0; + parseDotRealloc(p, 2); + while( z[h] ){ + while( IsSpace(z[h]) ){ h++; } + if( z[h]==0 ) break; + if( nArg+2>p->dot.nAlloc ){ + parseDotRealloc(p, nArg); + } + if( z[h]=='\'' || z[h]=='"' ){ + int delim = z[h++]; + p->dot.abQuot[nArg] = 1; + p->dot.azArg[nArg] = &z[h]; + p->dot.aiOfst[nArg] = h; + while( z[h] && z[h]!=delim ){ + if( z[h]=='\\' && delim=='"' && z[h+1]!=0 ) h++; + h++; + } + if( z[h]==delim ){ + z[h++] = 0; + } + if( delim=='"' ) resolve_backslashes(p->dot.azArg[nArg]); + }else{ + p->dot.abQuot[nArg] = 0; + p->dot.azArg[nArg] = &z[h]; + p->dot.aiOfst[nArg] = h; + while( z[h] && !IsSpace(z[h]) ){ h++; } + if( z[h] ) z[h++] = 0; } - return SQLITE_OK; - } - if( faultsim_state.eVerbose>=1 ){ - sqlite3_fprintf(stdout, - "FAULT-SIM id=%d returns %d\n", iArg, faultsim_state.iErr); - } - faultsim_state.iCnt = faultsim_state.iInterval; - faultsim_state.nHit++; - if( faultsim_state.nRepeat>0 && faultsim_state.nRepeat<=faultsim_state.nHit ){ - faultsim_state.iCnt = -1; + nArg++; } - return faultsim_state.iErr; + p->dot.nArg = nArg; + p->dot.azArg[nArg] = 0; } /* @@ -8762,12 +8962,11 @@ static int faultsim_callback(int iArg){ ** ** Return 1 on error, 2 to exit, and 0 otherwise. */ -static int do_meta_command(char *zLine, ShellState *p){ - int h = 1; - int nArg = 0; +static int do_meta_command(const char *zLine, ShellState *p){ + int nArg; int n, c; int rc = 0; - char *azArg[52]; + char **azArg; #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) if( p->expert.pExpert ){ @@ -8775,29 +8974,11 @@ static int do_meta_command(char *zLine, ShellState *p){ } #endif - /* Parse the input line into tokens. + /* Parse the input line into tokens stored in p->dot. */ - while( zLine[h] && nArg<ArraySize(azArg)-1 ){ - while( IsSpace(zLine[h]) ){ h++; } - if( zLine[h]==0 ) break; - if( zLine[h]=='\'' || zLine[h]=='"' ){ - int delim = zLine[h++]; - azArg[nArg++] = &zLine[h]; - while( zLine[h] && zLine[h]!=delim ){ - if( zLine[h]=='\\' && delim=='"' && zLine[h+1]!=0 ) h++; - h++; - } - if( zLine[h]==delim ){ - zLine[h++] = 0; - } - if( delim=='"' ) resolve_backslashes(azArg[nArg-1]); - }else{ - azArg[nArg++] = &zLine[h]; - while( zLine[h] && !IsSpace(zLine[h]) ){ h++; } - if( zLine[h] ) zLine[h++] = 0; - } - } - azArg[nArg] = 0; + parseDotCmdArgs(zLine, p); + nArg = p->dot.nArg; + azArg = p->dot.azArg; /* Process the input line. */ @@ -8809,7 +8990,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #ifndef SQLITE_OMIT_AUTHORIZATION if( c=='a' && cli_strncmp(azArg[0], "auth", n)==0 ){ if( nArg!=2 ){ - sqlite3_fprintf(stderr, "Usage: .auth ON|OFF\n"); + cli_printf(stderr, "Usage: .auth ON|OFF\n"); rc = 1; goto meta_command_exit; } @@ -8856,7 +9037,7 @@ static int do_meta_command(char *zLine, ShellState *p){ bAsync = 1; }else { - sqlite3_fprintf(stderr,"unknown option: %s\n", azArg[j]); + dotCmdError(p, j, "unknown option", "should be -append or -async"); return 1; } }else if( zDestFile==0 ){ @@ -8865,19 +9046,19 @@ static int do_meta_command(char *zLine, ShellState *p){ zDb = zDestFile; zDestFile = azArg[j]; }else{ - sqlite3_fprintf(stderr, "Usage: .backup ?DB? ?OPTIONS? FILENAME\n"); + cli_printf(stderr, "Usage: .backup ?DB? ?OPTIONS? FILENAME\n"); return 1; } } if( zDestFile==0 ){ - sqlite3_fprintf(stderr, "missing FILENAME argument on .backup\n"); + cli_printf(stderr, "missing FILENAME argument on .backup\n"); return 1; } if( zDb==0 ) zDb = "main"; rc = sqlite3_open_v2(zDestFile, &pDest, SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, zVfs); if( rc!=SQLITE_OK ){ - sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zDestFile); + cli_printf(stderr,"Error: cannot open \"%s\"\n", zDestFile); close_db(pDest); return 1; } @@ -8938,7 +9119,7 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = chdir(azArg[1]); #endif if( rc ){ - sqlite3_fprintf(stderr,"Cannot change to directory \"%s\"\n", azArg[1]); + cli_printf(stderr,"Cannot change to directory \"%s\"\n", azArg[1]); rc = 1; } }else{ @@ -8957,31 +9138,13 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else -#ifndef SQLITE_SHELL_FIDDLE /* Cancel output redirection, if it is currently set (by .testcase) ** Then read the content of the testcase-out.txt file and compare against ** azArg[1]. If there are differences, report an error and exit. */ if( c=='c' && n>=3 && cli_strncmp(azArg[0], "check", n)==0 ){ - char *zRes = 0; - output_reset(p); - if( nArg!=2 ){ - eputz("Usage: .check GLOB-PATTERN\n"); - rc = 2; - }else if( (zRes = readFile("testcase-out.txt", 0))==0 ){ - rc = 2; - }else if( testcase_glob(azArg[1],zRes)==0 ){ - sqlite3_fprintf(stderr, - "testcase-%s FAILED\n Expected: [%s]\n Got: [%s]\n", - p->zTestcase, azArg[1], zRes); - rc = 1; - }else{ - sqlite3_fprintf(p->out, "testcase-%s ok\n", p->zTestcase); - p->nCheck++; - } - sqlite3_free(zRes); + rc = dotCmdCheck(p); }else -#endif /* !defined(SQLITE_SHELL_FIDDLE) */ #ifndef SQLITE_SHELL_FIDDLE if( c=='c' && cli_strncmp(azArg[0], "clone", n)==0 ){ @@ -9009,9 +9172,9 @@ static int do_meta_command(char *zLine, ShellState *p){ zFile = "(temporary-file)"; } if( p->pAuxDb == &p->aAuxDb[i] ){ - sqlite3_fprintf(stdout, "ACTIVE %d: %s\n", i, zFile); + cli_printf(stdout, "ACTIVE %d: %s\n", i, zFile); }else if( p->aAuxDb[i].db!=0 ){ - sqlite3_fprintf(stdout, " %d: %s\n", i, zFile); + cli_printf(stdout, " %d: %s\n", i, zFile); } } }else if( nArg==2 && IsDigit(azArg[1][0]) && azArg[1][1]==0 ){ @@ -9047,12 +9210,17 @@ static int do_meta_command(char *zLine, ShellState *p){ ){ if( nArg==2 ){ #ifdef _WIN32 - p->crlfMode = booleanValue(azArg[1]); + if( booleanValue(azArg[1]) ){ + p->mode.mFlags |= MFLG_CRLF; + }else{ + p->mode.mFlags &= ~MFLG_CRLF; + } #else - p->crlfMode = 0; + p->mode.mFlags &= ~MFLG_CRLF; #endif } - sqlite3_fprintf(stderr, "crlf is %s\n", p->crlfMode ? "ON" : "OFF"); + cli_printf(stderr, "crlf is %s\n", + (p->mode.mFlags & MFLG_CRLF)!=0 ? "ON" : "OFF"); }else if( c=='d' && n>1 && cli_strncmp(azArg[0], "databases", n)==0 ){ @@ -9082,7 +9250,7 @@ static int do_meta_command(char *zLine, ShellState *p){ int eTxn = sqlite3_txn_state(p->db, azName[i*2]); int bRdonly = sqlite3_db_readonly(p->db, azName[i*2]); const char *z = azName[i*2+1]; - sqlite3_fprintf(p->out, "%s: %s %s%s\n", + cli_printf(p->out, "%s: %s %s%s\n", azName[i*2], z && z[0] ? z : "\"\"", bRdonly ? "r/o" : "r/w", eTxn==SQLITE_TXN_NONE ? "" : eTxn==SQLITE_TXN_READ ? " read-txn" : " write-txn"); @@ -9108,6 +9276,7 @@ static int do_meta_command(char *zLine, ShellState *p){ { "enable_trigger", SQLITE_DBCONFIG_ENABLE_TRIGGER }, { "enable_view", SQLITE_DBCONFIG_ENABLE_VIEW }, { "fts3_tokenizer", SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER }, + { "fp_digits", SQLITE_DBCONFIG_FP_DIGITS }, { "legacy_alter_table", SQLITE_DBCONFIG_LEGACY_ALTER_TABLE }, { "legacy_file_format", SQLITE_DBCONFIG_LEGACY_FILE_FORMAT }, { "load_extension", SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION }, @@ -9124,16 +9293,24 @@ static int do_meta_command(char *zLine, ShellState *p){ for(ii=0; ii<ArraySize(aDbConfig); ii++){ if( nArg>1 && cli_strcmp(azArg[1], aDbConfig[ii].zName)!=0 ) continue; if( nArg>=3 ){ - sqlite3_db_config(p->db, aDbConfig[ii].op, booleanValue(azArg[2]), 0); + if( aDbConfig[ii].op==SQLITE_DBCONFIG_FP_DIGITS ){ + sqlite3_db_config(p->db, aDbConfig[ii].op, atoi(azArg[2]), 0); + }else{ + sqlite3_db_config(p->db, aDbConfig[ii].op, booleanValue(azArg[2]), 0); + } } sqlite3_db_config(p->db, aDbConfig[ii].op, -1, &v); - sqlite3_fprintf(p->out, "%19s %s\n", - aDbConfig[ii].zName, v ? "on" : "off"); + if( aDbConfig[ii].op==SQLITE_DBCONFIG_FP_DIGITS ){ + cli_printf(p->out, "%19s %d\n", aDbConfig[ii].zName, v); + }else{ + cli_printf(p->out, "%19s %s\n", + aDbConfig[ii].zName, v ? "on" : "off"); + } if( nArg>1 ) break; } if( nArg>1 && ii==ArraySize(aDbConfig) ){ - sqlite3_fprintf(stderr,"Error: unknown dbconfig \"%s\"\n", azArg[1]); - eputz("Enter \".dbconfig\" with no arguments for a list\n"); + dotCmdError(p, 1, "unknown dbconfig", + "Enter \".dbconfig\" with no arguments for a list"); } }else @@ -9152,19 +9329,19 @@ static int do_meta_command(char *zLine, ShellState *p){ char *zLike = 0; char *zSql; int i; - int savedShowHeader = p->showHeader; int savedShellFlags = p->shellFlgs; + Mode saved_mode; ShellClearFlag(p, - SHFLG_PreserveRowid|SHFLG_Newlines|SHFLG_Echo - |SHFLG_DumpDataOnly|SHFLG_DumpNoSys); + SHFLG_PreserveRowid|SHFLG_DumpDataOnly|SHFLG_DumpNoSys); for(i=1; i<nArg; i++){ if( azArg[i][0]=='-' ){ const char *z = azArg[i]+1; if( z[0]=='-' ) z++; if( cli_strcmp(z,"preserve-rowids")==0 ){ #ifdef SQLITE_OMIT_VIRTUALTABLE - eputz("The --preserve-rowids option is not compatible" - " with SQLITE_OMIT_VIRTUALTABLE\n"); + dotCmdError(p, i, "unable", + "The --preserve-rowids option is not compatible" + " with SQLITE_OMIT_VIRTUALTABLE"); rc = 1; sqlite3_free(zLike); goto meta_command_exit; @@ -9173,7 +9350,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif }else if( cli_strcmp(z,"newlines")==0 ){ - ShellSetFlag(p, SHFLG_Newlines); + /*ShellSetFlag(p, SHFLG_Newlines);*/ }else if( cli_strcmp(z,"data-only")==0 ){ ShellSetFlag(p, SHFLG_DumpDataOnly); @@ -9182,8 +9359,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ShellSetFlag(p, SHFLG_DumpNoSys); }else { - sqlite3_fprintf(stderr, - "Unknown option \"%s\" on \".dump\"\n", azArg[i]); + dotCmdError(p, i, "unknown option", 0); rc = 1; sqlite3_free(zLike); goto meta_command_exit; @@ -9213,16 +9389,17 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); + modeDup(&saved_mode, &p->mode); outputDumpWarning(p, zLike); if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){ /* When playing back a "dump", the content might appear in an order ** which causes immediate foreign key constraints to be violated. ** So disable foreign-key constraint enforcement to prevent problems. */ - sqlite3_fputs("PRAGMA foreign_keys=OFF;\n", p->out); - sqlite3_fputs("BEGIN TRANSACTION;\n", p->out); + cli_puts("PRAGMA foreign_keys=OFF;\n", p->out); + cli_puts("BEGIN TRANSACTION;\n", p->out); } p->writableSchema = 0; - p->showHeader = 0; + p->mode.spec.bTitles = QRF_No; /* Set writable_schema=ON since doing so forces SQLite to initialize ** as much of the schema as it can even if the sqlite_schema table is ** corrupt. */ @@ -9251,22 +9428,27 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_free(zLike); if( p->writableSchema ){ - sqlite3_fputs("PRAGMA writable_schema=OFF;\n", p->out); + cli_puts("PRAGMA writable_schema=OFF;\n", p->out); p->writableSchema = 0; } sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0); sqlite3_exec(p->db, "RELEASE dump;", 0, 0, 0); if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){ - sqlite3_fputs(p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n", p->out); + cli_puts(p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n", p->out); } - p->showHeader = savedShowHeader; p->shellFlgs = savedShellFlags; + modeFree(&p->mode); + p->mode = saved_mode; rc = p->nErr>0; }else if( c=='e' && cli_strncmp(azArg[0], "echo", n)==0 ){ if( nArg==2 ){ - setOrClearFlag(p, SHFLG_Echo, azArg[1]); + if( booleanValue(azArg[1]) ){ + p->mode.mFlags |= MFLG_ECHO; + }else{ + p->mode.mFlags &= ~MFLG_ECHO; + } }else{ eputz("Usage: .echo on|off\n"); rc = 1; @@ -9280,28 +9462,24 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='e' && cli_strncmp(azArg[0], "eqp", n)==0 ){ if( nArg==2 ){ - p->autoEQPtest = 0; - if( p->autoEQPtrace ){ + if( p->mode.autoEQPtrace ){ if( p->db ) sqlite3_exec(p->db, "PRAGMA vdbe_trace=OFF;", 0, 0, 0); - p->autoEQPtrace = 0; + p->mode.autoEQPtrace = 0; } if( cli_strcmp(azArg[1],"full")==0 ){ - p->autoEQP = AUTOEQP_full; + p->mode.autoEQP = AUTOEQP_full; }else if( cli_strcmp(azArg[1],"trigger")==0 ){ - p->autoEQP = AUTOEQP_trigger; + p->mode.autoEQP = AUTOEQP_trigger; #ifdef SQLITE_DEBUG - }else if( cli_strcmp(azArg[1],"test")==0 ){ - p->autoEQP = AUTOEQP_on; - p->autoEQPtest = 1; }else if( cli_strcmp(azArg[1],"trace")==0 ){ - p->autoEQP = AUTOEQP_full; - p->autoEQPtrace = 1; + p->mode.autoEQP = AUTOEQP_full; + p->mode.autoEQPtrace = 1; open_db(p, 0); sqlite3_exec(p->db, "SELECT name FROM sqlite_schema LIMIT 1", 0, 0, 0); sqlite3_exec(p->db, "PRAGMA vdbe_trace=ON;", 0, 0, 0); #endif }else{ - p->autoEQP = (u8)booleanValue(azArg[1]); + p->mode.autoEQP = (u8)booleanValue(azArg[1]); } }else{ eputz("Usage: .eqp off|on|trace|trigger|full\n"); @@ -9311,7 +9489,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #ifndef SQLITE_SHELL_FIDDLE if( c=='e' && cli_strncmp(azArg[0], "exit", n)==0 ){ - if( nArg>1 && (rc = (int)integerValue(azArg[1]))!=0 ) exit(rc); + if( nArg>1 && (rc = (int)integerValue(azArg[1]))!=0 ) cli_exit(rc); rc = 2; }else #endif @@ -9319,31 +9497,19 @@ static int do_meta_command(char *zLine, ShellState *p){ /* The ".explain" command is automatic now. It is largely pointless. It ** retained purely for backwards compatibility */ if( c=='e' && cli_strncmp(azArg[0], "explain", n)==0 ){ - int val = 1; if( nArg>=2 ){ if( cli_strcmp(azArg[1],"auto")==0 ){ - val = 99; + p->mode.autoExplain = 1; }else{ - val = booleanValue(azArg[1]); + p->mode.autoExplain = booleanValue(azArg[1]); } } - if( val==1 && p->mode!=MODE_Explain ){ - p->normalMode = p->mode; - p->mode = MODE_Explain; - p->autoExplain = 0; - }else if( val==0 ){ - if( p->mode==MODE_Explain ) p->mode = p->normalMode; - p->autoExplain = 0; - }else if( val==99 ){ - if( p->mode==MODE_Explain ) p->mode = p->normalMode; - p->autoExplain = 1; - } }else #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) if( c=='e' && cli_strncmp(azArg[0], "expert", n)==0 ){ if( p->bSafeMode ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Cannot run experimental commands such as \"%s\" in safe mode\n", azArg[0]); rc = 1; @@ -9401,9 +9567,9 @@ static int do_meta_command(char *zLine, ShellState *p){ /* --help lists all file-controls */ if( cli_strcmp(zCmd,"help")==0 ){ - sqlite3_fputs("Available file-controls:\n", p->out); + cli_puts("Available file-controls:\n", p->out); for(i=0; i<ArraySize(aCtrl); i++){ - sqlite3_fprintf(p->out, + cli_printf(p->out, " .filectrl %s %s\n", aCtrl[i].zCtrlName, aCtrl[i].zUsage); } rc = 1; @@ -9419,7 +9585,7 @@ static int do_meta_command(char *zLine, ShellState *p){ filectrl = aCtrl[i].ctrlCode; iCtrl = i; }else{ - sqlite3_fprintf(stderr,"Error: ambiguous file-control: \"%s\"\n" + cli_printf(stderr,"Error: ambiguous file-control: \"%s\"\n" "Use \".filectrl --help\" for help\n", zCmd); rc = 1; goto meta_command_exit; @@ -9427,7 +9593,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } } if( filectrl<0 ){ - sqlite3_fprintf(stderr,"Error: unknown file-control: %s\n" + cli_printf(stderr,"Error: unknown file-control: %s\n" "Use \".filectrl --help\" for help\n", zCmd); }else{ switch(filectrl){ @@ -9471,7 +9637,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nArg!=2 ) break; sqlite3_file_control(p->db, zSchema, filectrl, &z); if( z ){ - sqlite3_fprintf(p->out, "%s\n", z); + cli_printf(p->out, "%s\n", z); sqlite3_free(z); } isOk = 2; @@ -9485,31 +9651,30 @@ static int do_meta_command(char *zLine, ShellState *p){ } x = -1; sqlite3_file_control(p->db, zSchema, filectrl, &x); - sqlite3_fprintf(p->out, "%d\n", x); + cli_printf(p->out, "%d\n", x); isOk = 2; break; } } } if( isOk==0 && iCtrl>=0 ){ - sqlite3_fprintf(p->out, "Usage: .filectrl %s %s\n", + cli_printf(p->out, "Usage: .filectrl %s %s\n", zCmd, aCtrl[iCtrl].zUsage); rc = 1; }else if( isOk==1 ){ char zBuf[100]; sqlite3_snprintf(sizeof(zBuf), zBuf, "%lld", iRes); - sqlite3_fprintf(p->out, "%s\n", zBuf); + cli_printf(p->out, "%s\n", zBuf); } }else if( c=='f' && cli_strncmp(azArg[0], "fullschema", n)==0 ){ ShellState data; int doStats = 0; - memcpy(&data, p, sizeof(data)); - data.showHeader = 0; - data.cMode = data.mode = MODE_Semi; + int hasStat[5]; + int flgs = 0; + char *zSql; if( nArg==2 && optionMatch(azArg[1], "indent") ){ - data.cMode = data.mode = MODE_Pretty; nArg = 1; } if( nArg!=1 ){ @@ -9518,385 +9683,82 @@ static int do_meta_command(char *zLine, ShellState *p){ goto meta_command_exit; } open_db(p, 0); - rc = sqlite3_exec(p->db, - "SELECT sql FROM" + zSql = sqlite3_mprintf( + "SELECT shell_format_schema(sql,%d) FROM" " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" " FROM sqlite_schema UNION ALL" " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_schema) " "WHERE type!='meta' AND sql NOTNULL" - " AND name NOT LIKE 'sqlite__%' ESCAPE '_' " - "ORDER BY x", - callback, &data, 0 - ); + " AND name NOT LIKE 'sqlite__%%' ESCAPE '_' " + "ORDER BY x", flgs); + memcpy(&data, p, sizeof(data)); + data.mode.spec.bTitles = QRF_No; + data.mode.eMode = MODE_List; + data.mode.spec.eText = QRF_TEXT_Plain; + data.mode.spec.nCharLimit = 0; + data.mode.spec.zRowSep = "\n"; + rc = shell_exec(&data,zSql,0); + sqlite3_free(zSql); if( rc==SQLITE_OK ){ sqlite3_stmt *pStmt; + memset(hasStat, 0, sizeof(hasStat)); rc = sqlite3_prepare_v2(p->db, - "SELECT rowid FROM sqlite_schema" + "SELECT substr(name,12,1) FROM sqlite_schema" " WHERE name GLOB 'sqlite_stat[134]'", -1, &pStmt, 0); if( rc==SQLITE_OK ){ - doStats = sqlite3_step(pStmt)==SQLITE_ROW; - sqlite3_finalize(pStmt); - } - } - if( doStats==0 ){ - sqlite3_fputs("/* No STAT tables available */\n", p->out); - }else{ - sqlite3_fputs("ANALYZE sqlite_schema;\n", p->out); - data.cMode = data.mode = MODE_Insert; - data.zDestTable = "sqlite_stat1"; - shell_exec(&data, "SELECT * FROM sqlite_stat1", 0); - data.zDestTable = "sqlite_stat4"; - shell_exec(&data, "SELECT * FROM sqlite_stat4", 0); - sqlite3_fputs("ANALYZE sqlite_schema;\n", p->out); - } - }else - - if( c=='h' && cli_strncmp(azArg[0], "headers", n)==0 ){ - if( nArg==2 ){ - p->showHeader = booleanValue(azArg[1]); - p->shellFlgs |= SHFLG_HeaderSet; - }else{ - eputz("Usage: .headers on|off\n"); - rc = 1; - } - }else - - if( c=='h' && cli_strncmp(azArg[0], "help", n)==0 ){ - if( nArg>=2 ){ - n = showHelp(p->out, azArg[1]); - if( n==0 ){ - sqlite3_fprintf(p->out, "Nothing matches '%s'\n", azArg[1]); - } - }else{ - showHelp(p->out, 0); - } - }else - -#ifndef SQLITE_SHELL_FIDDLE - if( c=='i' && cli_strncmp(azArg[0], "import", n)==0 ){ - char *zTable = 0; /* Insert data into this table */ - char *zSchema = 0; /* Schema of zTable */ - char *zFile = 0; /* Name of file to extra content from */ - sqlite3_stmt *pStmt = NULL; /* A statement */ - int nCol; /* Number of columns in the table */ - i64 nByte; /* Number of bytes in an SQL string */ - int i, j; /* Loop counters */ - int needCommit; /* True to COMMIT or ROLLBACK at end */ - int nSep; /* Number of bytes in p->colSeparator[] */ - char *zSql = 0; /* An SQL statement */ - ImportCtx sCtx; /* Reader context */ - char *(SQLITE_CDECL *xRead)(ImportCtx*); /* Func to read one value */ - int eVerbose = 0; /* Larger for more console output */ - i64 nSkip = 0; /* Initial lines to skip */ - int useOutputMode = 1; /* Use output mode to determine separators */ - char *zCreate = 0; /* CREATE TABLE statement text */ - - failIfSafeMode(p, "cannot run .import in safe mode"); - memset(&sCtx, 0, sizeof(sCtx)); - if( p->mode==MODE_Ascii ){ - xRead = ascii_read_one_field; - }else{ - xRead = csv_read_one_field; - } - rc = 1; - for(i=1; i<nArg; i++){ - char *z = azArg[i]; - if( z[0]=='-' && z[1]=='-' ) z++; - if( z[0]!='-' ){ - if( zFile==0 ){ - zFile = z; - }else if( zTable==0 ){ - zTable = z; - }else{ - sqlite3_fprintf(p->out, "ERROR: extra argument: \"%s\". Usage:\n",z); - showHelp(p->out, "import"); - goto meta_command_exit; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + int k = sqlite3_column_int(pStmt,0); + assert( k==1 || k==3 || k==4 ); + hasStat[k] = 1; + doStats = 1; } - }else if( cli_strcmp(z,"-v")==0 ){ - eVerbose++; - }else if( cli_strcmp(z,"-schema")==0 && i<nArg-1 ){ - zSchema = azArg[++i]; - }else if( cli_strcmp(z,"-skip")==0 && i<nArg-1 ){ - nSkip = integerValue(azArg[++i]); - }else if( cli_strcmp(z,"-ascii")==0 ){ - sCtx.cColSep = SEP_Unit[0]; - sCtx.cRowSep = SEP_Record[0]; - xRead = ascii_read_one_field; - useOutputMode = 0; - }else if( cli_strcmp(z,"-csv")==0 ){ - sCtx.cColSep = ','; - sCtx.cRowSep = '\n'; - xRead = csv_read_one_field; - useOutputMode = 0; - }else{ - sqlite3_fprintf(p->out, "ERROR: unknown option: \"%s\". Usage:\n", z); - showHelp(p->out, "import"); - goto meta_command_exit; - } - } - if( zTable==0 ){ - sqlite3_fprintf(p->out, "ERROR: missing %s argument. Usage:\n", - zFile==0 ? "FILE" : "TABLE"); - showHelp(p->out, "import"); - goto meta_command_exit; - } - seenInterrupt = 0; - open_db(p, 0); - if( useOutputMode ){ - /* If neither the --csv or --ascii options are specified, then set - ** the column and row separator characters from the output mode. */ - nSep = strlen30(p->colSeparator); - if( nSep==0 ){ - eputz("Error: non-null column separator required for import\n"); - goto meta_command_exit; - } - if( nSep>1 ){ - eputz("Error: multi-character column separators not allowed" - " for import\n"); - goto meta_command_exit; - } - nSep = strlen30(p->rowSeparator); - if( nSep==0 ){ - eputz("Error: non-null row separator required for import\n"); - goto meta_command_exit; - } - if( nSep==2 && p->mode==MODE_Csv - && cli_strcmp(p->rowSeparator,SEP_CrLf)==0 - ){ - /* When importing CSV (only), if the row separator is set to the - ** default output row separator, change it to the default input - ** row separator. This avoids having to maintain different input - ** and output row separators. */ - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - nSep = strlen30(p->rowSeparator); - } - if( nSep>1 ){ - eputz("Error: multi-character row separators not allowed" - " for import\n"); - goto meta_command_exit; - } - sCtx.cColSep = (u8)p->colSeparator[0]; - sCtx.cRowSep = (u8)p->rowSeparator[0]; - } - sCtx.zFile = zFile; - sCtx.nLine = 1; - if( sCtx.zFile[0]=='|' ){ -#ifdef SQLITE_OMIT_POPEN - eputz("Error: pipes are not supported in this OS\n"); - goto meta_command_exit; -#else - sCtx.in = sqlite3_popen(sCtx.zFile+1, "r"); - sCtx.zFile = "<pipe>"; - sCtx.xCloser = pclose; -#endif - }else{ - sCtx.in = sqlite3_fopen(sCtx.zFile, "rb"); - sCtx.xCloser = fclose; - } - if( sCtx.in==0 ){ - sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zFile); - goto meta_command_exit; - } - if( eVerbose>=2 || (eVerbose>=1 && useOutputMode) ){ - char zSep[2]; - zSep[1] = 0; - zSep[0] = sCtx.cColSep; - sqlite3_fputs("Column separator ", p->out); - output_c_string(p->out, zSep); - sqlite3_fputs(", row separator ", p->out); - zSep[0] = sCtx.cRowSep; - output_c_string(p->out, zSep); - sqlite3_fputs("\n", p->out); - } - sCtx.z = sqlite3_malloc64(120); - if( sCtx.z==0 ){ - import_cleanup(&sCtx); - shell_out_of_memory(); - } - /* Below, resources must be freed before exit. */ - while( nSkip>0 ){ - nSkip--; - while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){} - } - import_append_char(&sCtx, 0); /* To ensure sCtx.z is allocated */ - if( sqlite3_table_column_metadata(p->db, zSchema, zTable,0,0,0,0,0,0) - && 0==db_int(p->db, "SELECT count(*) FROM \"%w\".sqlite_schema" - " WHERE name=%Q AND type='view'", - zSchema ? zSchema : "main", zTable) - ){ - /* Table does not exist. Create it. */ - sqlite3 *dbCols = 0; - char *zRenames = 0; - char *zColDefs; - zCreate = sqlite3_mprintf("CREATE TABLE \"%w\".\"%w\"", - zSchema ? zSchema : "main", zTable); - while( xRead(&sCtx) ){ - zAutoColumn(sCtx.z, &dbCols, 0); - if( sCtx.cTerm!=sCtx.cColSep ) break; - } - zColDefs = zAutoColumn(0, &dbCols, &zRenames); - if( zRenames!=0 ){ - sqlite3_fprintf((stdin_is_interactive && p->in==stdin)? p->out : stderr, - "Columns renamed during .import %s due to duplicates:\n" - "%s\n", sCtx.zFile, zRenames); - sqlite3_free(zRenames); - } - assert(dbCols==0); - if( zColDefs==0 ){ - sqlite3_fprintf(stderr,"%s: empty file\n", sCtx.zFile); - import_cleanup(&sCtx); - rc = 1; - sqlite3_free(zCreate); - goto meta_command_exit; - } - zCreate = sqlite3_mprintf("%z%z\n", zCreate, zColDefs); - if( zCreate==0 ){ - import_cleanup(&sCtx); - shell_out_of_memory(); - } - if( eVerbose>=1 ){ - sqlite3_fprintf(p->out, "%s\n", zCreate); - } - rc = sqlite3_exec(p->db, zCreate, 0, 0, 0); - if( rc ){ - sqlite3_fprintf(stderr, - "%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db)); - } - sqlite3_free(zCreate); - zCreate = 0; - if( rc ){ - import_cleanup(&sCtx); - rc = 1; - goto meta_command_exit; } + sqlite3_finalize(pStmt); } - zSql = sqlite3_mprintf("SELECT count(*) FROM pragma_table_info(%Q,%Q);", - zTable, zSchema); - if( zSql==0 ){ - import_cleanup(&sCtx); - shell_out_of_memory(); - } - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - zSql = 0; - if( rc ){ - if (pStmt) sqlite3_finalize(pStmt); - shellDatabaseError(p->db); - import_cleanup(&sCtx); - rc = 1; - goto meta_command_exit; - } - if( sqlite3_step(pStmt)==SQLITE_ROW ){ - nCol = sqlite3_column_int(pStmt, 0); - }else{ - nCol = 0; - } - sqlite3_finalize(pStmt); - pStmt = 0; - if( nCol==0 ) return 0; /* no columns, no error */ - - nByte = 64 /* space for "INSERT INTO", "VALUES(", ")\0" */ - + (zSchema ? strlen(zSchema)*2 + 2: 0) /* Quoted schema name */ - + strlen(zTable)*2 + 2 /* Quoted table name */ - + nCol*2; /* Space for ",?" for each column */ - zSql = sqlite3_malloc64( nByte ); - if( zSql==0 ){ - import_cleanup(&sCtx); - shell_out_of_memory(); - } - if( zSchema ){ - sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\".\"%w\" VALUES(?", - zSchema, zTable); - }else{ - sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); - } - j = strlen30(zSql); - for(i=1; i<nCol; i++){ - zSql[j++] = ','; - zSql[j++] = '?'; - } - zSql[j++] = ')'; - zSql[j] = 0; - assert( j<nByte ); - if( eVerbose>=2 ){ - sqlite3_fprintf(p->out, "Insert using: %s\n", zSql); - } - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - zSql = 0; - if( rc ){ - shellDatabaseError(p->db); - if (pStmt) sqlite3_finalize(pStmt); - import_cleanup(&sCtx); - rc = 1; - goto meta_command_exit; - } - needCommit = sqlite3_get_autocommit(p->db); - if( needCommit ) sqlite3_exec(p->db, "BEGIN", 0, 0, 0); - do{ - int startLine = sCtx.nLine; - for(i=0; i<nCol; i++){ - char *z = xRead(&sCtx); - /* - ** Did we reach end-of-file before finding any columns? - ** If so, stop instead of NULL filling the remaining columns. - */ - if( z==0 && i==0 ) break; - /* - ** Did we reach end-of-file OR end-of-line before finding any - ** columns in ASCII mode? If so, stop instead of NULL filling - ** the remaining columns. - */ - if( p->mode==MODE_Ascii && (z==0 || z[0]==0) && i==0 ) break; - /* - ** For CSV mode, per RFC 4180, accept EOF in lieu of final - ** record terminator but only for last field of multi-field row. - ** (If there are too few fields, it's not valid CSV anyway.) - */ - if( z==0 && (xRead==csv_read_one_field) && i==nCol-1 && i>0 ){ - z = ""; - } - sqlite3_bind_text(pStmt, i+1, z, -1, SQLITE_TRANSIENT); - if( i<nCol-1 && sCtx.cTerm!=sCtx.cColSep ){ - sqlite3_fprintf(stderr,"%s:%d: expected %d columns but found %d" - " - filling the rest with NULL\n", - sCtx.zFile, startLine, nCol, i+1); - i += 2; - while( i<=nCol ){ sqlite3_bind_null(pStmt, i); i++; } - } + if( doStats==0 ){ + cli_puts("/* No STAT tables available */\n", p->out); + }else{ + cli_puts("ANALYZE sqlite_schema;\n", p->out); + data.mode.eMode = MODE_Insert; + if( hasStat[1] ){ + data.mode.spec.zTableName = "sqlite_stat1"; + shell_exec(&data, "SELECT * FROM sqlite_stat1", 0); } - if( sCtx.cTerm==sCtx.cColSep ){ - do{ - xRead(&sCtx); - i++; - }while( sCtx.cTerm==sCtx.cColSep ); - sqlite3_fprintf(stderr, - "%s:%d: expected %d columns but found %d - extras ignored\n", - sCtx.zFile, startLine, nCol, i); - } - if( i>=nCol ){ - sqlite3_step(pStmt); - rc = sqlite3_reset(pStmt); - if( rc!=SQLITE_OK ){ - sqlite3_fprintf(stderr,"%s:%d: INSERT failed: %s\n", - sCtx.zFile, startLine, sqlite3_errmsg(p->db)); - sCtx.nErr++; - }else{ - sCtx.nRow++; - } + if( hasStat[4] ){ + data.mode.spec.zTableName = "sqlite_stat4"; + shell_exec(&data, "SELECT * FROM sqlite_stat4", 0); } - }while( sCtx.cTerm!=EOF ); + cli_puts("ANALYZE sqlite_schema;\n", p->out); + } + }else - import_cleanup(&sCtx); - sqlite3_finalize(pStmt); - if( needCommit ) sqlite3_exec(p->db, "COMMIT", 0, 0, 0); - if( eVerbose>0 ){ - sqlite3_fprintf(p->out, - "Added %d rows with %d errors using %d lines of input\n", - sCtx.nRow, sCtx.nErr, sCtx.nLine-1); + if( c=='h' && cli_strncmp(azArg[0], "headers", n)==0 ){ + if( nArg==2 ){ + p->mode.spec.bTitles = booleanValue(azArg[1]) ? QRF_Yes : QRF_No; + p->mode.mFlags |= MFLG_HDR; + p->mode.spec.eTitle = aModeInfo[p->mode.eMode].eHdr; + }else{ + eputz("Usage: .headers on|off\n"); + rc = 1; + } + }else + + if( c=='h' && cli_strncmp(azArg[0], "help", n)==0 ){ + if( nArg>=2 ){ + n = showHelp(p->out, azArg[1]); + if( n==0 ){ + cli_printf(p->out, "Nothing matches '%s'\n", azArg[1]); + } + }else{ + showHelp(p->out, 0); } }else + +#ifndef SQLITE_SHELL_FIDDLE + if( c=='i' && cli_strncmp(azArg[0], "import", n)==0 ){ + rc = dotCmdImport(p); + }else #endif /* !defined(SQLITE_SHELL_FIDDLE) */ #ifndef SQLITE_UNTESTABLE @@ -9928,10 +9790,10 @@ static int do_meta_command(char *zLine, ShellState *p){ } zSql = sqlite3_mprintf( "SELECT rootpage, 0 FROM sqlite_schema" - " WHERE name='%q' AND type='index'" + " WHERE type='index' AND lower(name)=lower('%q')" "UNION ALL " "SELECT rootpage, 1 FROM sqlite_schema" - " WHERE name='%q' AND type='table'" + " WHERE type='table' AND lower(name)=lower('%q')" " AND sql LIKE '%%without%%rowid%%'", azArg[1], azArg[1] ); @@ -9969,7 +9831,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_finalize(pStmt); if( i==0 || tnum==0 ){ - sqlite3_fprintf(stderr,"no such index: \"%s\"\n", azArg[1]); + cli_printf(stderr,"no such index: \"%s\"\n", azArg[1]); rc = 1; sqlite3_free(zCollist); goto meta_command_exit; @@ -9984,13 +9846,13 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = sqlite3_exec(p->db, zSql, 0, 0, 0); sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 0); if( rc ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Error in [%s]: %s\n", zSql, sqlite3_errmsg(p->db)); }else{ - sqlite3_fprintf(stdout, "%s;\n", zSql); + cli_printf(stdout, "%s;\n", zSql); } }else{ - sqlite3_fprintf(stderr,"SQLITE_TESTCTRL_IMPOSTER returns %d\n", rc); + cli_printf(stderr,"SQLITE_TESTCTRL_IMPOSTER returns %d\n", rc); rc = 1; } sqlite3_free(zSql); @@ -10004,7 +9866,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( iArg==0 ) iArg = -1; } if( (nArg!=1 && nArg!=2) || iArg<0 ){ - sqlite3_fprintf(stderr,"%s","Usage: .intck STEPS_PER_UNLOCK\n"); + cli_printf(stderr,"%s","Usage: .intck STEPS_PER_UNLOCK\n"); rc = 1; goto meta_command_exit; } @@ -10025,7 +9887,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else{ iotrace = sqlite3_fopen(azArg[1], "w"); if( iotrace==0 ){ - sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); + cli_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); sqlite3IoTrace = 0; rc = 1; }else{ @@ -10044,6 +9906,7 @@ static int do_meta_command(char *zLine, ShellState *p){ { "sql_length", SQLITE_LIMIT_SQL_LENGTH }, { "column", SQLITE_LIMIT_COLUMN }, { "expr_depth", SQLITE_LIMIT_EXPR_DEPTH }, + { "parser_depth", SQLITE_LIMIT_PARSER_DEPTH }, { "compound_select", SQLITE_LIMIT_COMPOUND_SELECT }, { "vdbe_op", SQLITE_LIMIT_VDBE_OP }, { "function_arg", SQLITE_LIMIT_FUNCTION_ARG }, @@ -10057,7 +9920,7 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); if( nArg==1 ){ for(i=0; i<ArraySize(aLimit); i++){ - sqlite3_fprintf(stdout, "%20s %d\n", aLimit[i].zLimitName, + cli_printf(stdout, "%20s %d\n", aLimit[i].zLimitName, sqlite3_limit(p->db, aLimit[i].limitCode, -1)); } }else if( nArg>3 ){ @@ -10072,14 +9935,14 @@ static int do_meta_command(char *zLine, ShellState *p){ if( iLimit<0 ){ iLimit = i; }else{ - sqlite3_fprintf(stderr,"ambiguous limit: \"%s\"\n", azArg[1]); + cli_printf(stderr,"ambiguous limit: \"%s\"\n", azArg[1]); rc = 1; goto meta_command_exit; } } } if( iLimit<0 ){ - sqlite3_fprintf(stderr,"unknown limit: \"%s\"\n" + cli_printf(stderr,"unknown limit: \"%s\"\n" "enter \".limits\" with no arguments for a list.\n", azArg[1]); rc = 1; @@ -10088,9 +9951,10 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nArg==3 ){ sqlite3_limit(p->db, aLimit[iLimit].limitCode, (int)integerValue(azArg[2])); + }else{ + cli_printf(stdout, "%20s %d\n", aLimit[iLimit].zLimitName, + sqlite3_limit(p->db, aLimit[iLimit].limitCode, -1)); } - sqlite3_fprintf(stdout, "%20s %d\n", aLimit[iLimit].zLimitName, - sqlite3_limit(p->db, aLimit[iLimit].limitCode, -1)); } }else @@ -10138,174 +10002,12 @@ static int do_meta_command(char *zLine, ShellState *p){ } output_file_close(p->pLog); if( cli_strcmp(zFile,"on")==0 ) zFile = "stdout"; - p->pLog = output_file_open(zFile); + p->pLog = output_file_open(p, zFile); } }else if( c=='m' && cli_strncmp(azArg[0], "mode", n)==0 ){ - const char *zMode = 0; - const char *zTabname = 0; - int i, n2; - int chng = 0; /* 0x01: change to cmopts. 0x02: Any other change */ - ColModeOpts cmOpts = ColModeOpts_default; - for(i=1; i<nArg; i++){ - const char *z = azArg[i]; - if( optionMatch(z,"wrap") && i+1<nArg ){ - cmOpts.iWrap = integerValue(azArg[++i]); - chng |= 1; - }else if( optionMatch(z,"ww") ){ - cmOpts.bWordWrap = 1; - chng |= 1; - }else if( optionMatch(z,"wordwrap") && i+1<nArg ){ - cmOpts.bWordWrap = (u8)booleanValue(azArg[++i]); - chng |= 1; - }else if( optionMatch(z,"quote") ){ - cmOpts.bQuote = 1; - chng |= 1; - }else if( optionMatch(z,"noquote") ){ - cmOpts.bQuote = 0; - chng |= 1; - }else if( optionMatch(z,"escape") && i+1<nArg ){ - /* See similar code at tag-20250224-1 */ - const char *zEsc = azArg[++i]; - int k; - for(k=0; k<ArraySize(shell_EscModeNames); k++){ - if( sqlite3_stricmp(zEsc,shell_EscModeNames[k])==0 ){ - p->eEscMode = k; - chng |= 2; - break; - } - } - if( k>=ArraySize(shell_EscModeNames) ){ - sqlite3_fprintf(stderr, "unknown control character escape mode \"%s\"" - " - choices:", zEsc); - for(k=0; k<ArraySize(shell_EscModeNames); k++){ - sqlite3_fprintf(stderr, " %s", shell_EscModeNames[k]); - } - sqlite3_fprintf(stderr, "\n"); - rc = 1; - goto meta_command_exit; - } - }else if( zMode==0 ){ - zMode = z; - /* Apply defaults for qbox pseudo-mode. If that - * overwrites already-set values, user was informed of this. - */ - chng |= 1; - if( cli_strcmp(z, "qbox")==0 ){ - ColModeOpts cmo = ColModeOpts_default_qbox; - zMode = "box"; - cmOpts = cmo; - } - }else if( zTabname==0 ){ - zTabname = z; - }else if( z[0]=='-' ){ - sqlite3_fprintf(stderr,"unknown option: %s\n", z); - eputz("options:\n" - " --escape MODE\n" - " --noquote\n" - " --quote\n" - " --wordwrap on/off\n" - " --wrap N\n" - " --ww\n"); - rc = 1; - goto meta_command_exit; - }else{ - sqlite3_fprintf(stderr,"extra argument: \"%s\"\n", z); - rc = 1; - goto meta_command_exit; - } - } - if( !chng ){ - if( p->mode==MODE_Column - || (p->mode>=MODE_Markdown && p->mode<=MODE_Box) - ){ - sqlite3_fprintf(p->out, - "current output mode: %s --wrap %d --wordwrap %s " - "--%squote --escape %s\n", - modeDescr[p->mode], p->cmOpts.iWrap, - p->cmOpts.bWordWrap ? "on" : "off", - p->cmOpts.bQuote ? "" : "no", - shell_EscModeNames[p->eEscMode] - ); - }else{ - sqlite3_fprintf(p->out, - "current output mode: %s --escape %s\n", - modeDescr[p->mode], - shell_EscModeNames[p->eEscMode] - ); - } - } - if( zMode==0 ){ - zMode = modeDescr[p->mode]; - if( (chng&1)==0 ) cmOpts = p->cmOpts; - } - n2 = strlen30(zMode); - if( cli_strncmp(zMode,"lines",n2)==0 ){ - p->mode = MODE_Line; - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( cli_strncmp(zMode,"columns",n2)==0 ){ - p->mode = MODE_Column; - if( (p->shellFlgs & SHFLG_HeaderSet)==0 ){ - p->showHeader = 1; - } - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - p->cmOpts = cmOpts; - }else if( cli_strncmp(zMode,"list",n2)==0 ){ - p->mode = MODE_List; - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Column); - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( cli_strncmp(zMode,"html",n2)==0 ){ - p->mode = MODE_Html; - }else if( cli_strncmp(zMode,"tcl",n2)==0 ){ - p->mode = MODE_Tcl; - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Space); - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( cli_strncmp(zMode,"csv",n2)==0 ){ - p->mode = MODE_Csv; - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf); - }else if( cli_strncmp(zMode,"tabs",n2)==0 ){ - p->mode = MODE_List; - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Tab); - }else if( cli_strncmp(zMode,"insert",n2)==0 ){ - p->mode = MODE_Insert; - set_table_name(p, zTabname ? zTabname : "table"); - if( p->eEscMode==SHELL_ESC_OFF ){ - ShellSetFlag(p, SHFLG_Newlines); - }else{ - ShellClearFlag(p, SHFLG_Newlines); - } - }else if( cli_strncmp(zMode,"quote",n2)==0 ){ - p->mode = MODE_Quote; - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( cli_strncmp(zMode,"ascii",n2)==0 ){ - p->mode = MODE_Ascii; - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Unit); - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Record); - }else if( cli_strncmp(zMode,"markdown",n2)==0 ){ - p->mode = MODE_Markdown; - p->cmOpts = cmOpts; - }else if( cli_strncmp(zMode,"table",n2)==0 ){ - p->mode = MODE_Table; - p->cmOpts = cmOpts; - }else if( cli_strncmp(zMode,"box",n2)==0 ){ - p->mode = MODE_Box; - p->cmOpts = cmOpts; - }else if( cli_strncmp(zMode,"count",n2)==0 ){ - p->mode = MODE_Count; - }else if( cli_strncmp(zMode,"off",n2)==0 ){ - p->mode = MODE_Off; - }else if( cli_strncmp(zMode,"json",n2)==0 ){ - p->mode = MODE_Json; - }else{ - eputz("Error: mode should be one of: " - "ascii box column csv html insert json line list markdown " - "qbox quote table tabs tcl\n"); - rc = 1; - } - p->cMode = p->mode; + rc = dotCmdMode(p); }else #ifndef SQLITE_SHELL_FIDDLE @@ -10314,9 +10016,9 @@ static int do_meta_command(char *zLine, ShellState *p){ eputz("Usage: .nonce NONCE\n"); rc = 1; }else if( p->zNonce==0 || cli_strcmp(azArg[1],p->zNonce)!=0 ){ - sqlite3_fprintf(stderr,"line %lld: incorrect nonce: \"%s\"\n", + cli_printf(stderr,"line %lld: incorrect nonce: \"%s\"\n", p->lineno, azArg[1]); - exit(1); + cli_exit(1); }else{ p->bSafeMode = 0; return 0; /* Return immediately to bypass the safe mode reset @@ -10327,8 +10029,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='n' && cli_strncmp(azArg[0], "nullvalue", n)==0 ){ if( nArg==2 ){ - sqlite3_snprintf(sizeof(p->nullValue), p->nullValue, - "%.*s", (int)ArraySize(p->nullValue)-1, azArg[1]); + modeSetStr(&p->mode.spec.zNull, azArg[1]); }else{ eputz("Usage: .nullvalue STRING\n"); rc = 1; @@ -10343,6 +10044,8 @@ static int do_meta_command(char *zLine, ShellState *p){ int openMode = SHELL_OPEN_UNSPEC; int openFlags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; + if( p->bSafeMode ) openFlags = SQLITE_OPEN_READONLY; + /* Check for command-line arguments */ for(iName=1; iName<nArg; iName++){ const char *z = azArg[iName]; @@ -10350,10 +10053,10 @@ static int do_meta_command(char *zLine, ShellState *p){ if( optionMatch(z,"new") ){ newFlag = 1; #ifdef SQLITE_HAVE_ZLIB - }else if( optionMatch(z, "zip") ){ + }else if( optionMatch(z, "zip") && !p->bSafeMode ){ openMode = SHELL_OPEN_ZIPFILE; #endif - }else if( optionMatch(z, "append") ){ + }else if( optionMatch(z, "append") && !p->bSafeMode ){ openMode = SHELL_OPEN_APPENDVFS; }else if( optionMatch(z, "readonly") ){ openFlags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); @@ -10377,11 +10080,11 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif /* !SQLITE_SHELL_FIDDLE */ if( z[0]=='-' ){ - sqlite3_fprintf(stderr,"unknown option: %s\n", z); + cli_printf(stderr,"unknown option: %s\n", z); rc = 1; goto meta_command_exit; }else if( zFN ){ - sqlite3_fprintf(stderr,"extra argument: \"%s\"\n", z); + cli_printf(stderr,"extra argument: \"%s\"\n", z); rc = 1; goto meta_command_exit; }else{ @@ -10432,7 +10135,7 @@ static int do_meta_command(char *zLine, ShellState *p){ p->pAuxDb->zDbFilename = zNewFilename; open_db(p, OPEN_DB_KEEPALIVE); if( p->db==0 ){ - sqlite3_fprintf(stderr,"Error: cannot open '%s'\n", zNewFilename); + cli_printf(stderr,"Error: cannot open '%s'\n", zNewFilename); sqlite3_free(zNewFilename); }else{ p->pAuxDb->zFreeOnClose = zNewFilename; @@ -10452,145 +10155,7 @@ static int do_meta_command(char *zLine, ShellState *p){ || (c=='e' && n==5 && cli_strcmp(azArg[0],"excel")==0) || (c=='w' && n==3 && cli_strcmp(azArg[0],"www")==0) ){ - char *zFile = 0; - int i; - int eMode = 0; /* 0: .outout/.once, 'x'=.excel, 'w'=.www */ - int bOnce = 0; /* 0: .output, 1: .once, 2: .excel/.www */ - int bPlain = 0; /* --plain option */ - static const char *zBomUtf8 = "\357\273\277"; - const char *zBom = 0; - - failIfSafeMode(p, "cannot run .%s in safe mode", azArg[0]); - if( c=='e' ){ - eMode = 'x'; - bOnce = 2; - }else if( c=='w' ){ - eMode = 'w'; - bOnce = 2; - }else if( cli_strncmp(azArg[0],"once",n)==0 ){ - bOnce = 1; - } - for(i=1; i<nArg; i++){ - char *z = azArg[i]; - if( z[0]=='-' ){ - if( z[1]=='-' ) z++; - if( cli_strcmp(z,"-bom")==0 ){ - zBom = zBomUtf8; - }else if( cli_strcmp(z,"-plain")==0 ){ - bPlain = 1; - }else if( c=='o' && cli_strcmp(z,"-x")==0 ){ - eMode = 'x'; /* spreadsheet */ - }else if( c=='o' && cli_strcmp(z,"-e")==0 ){ - eMode = 'e'; /* text editor */ - }else if( c=='o' && cli_strcmp(z,"-w")==0 ){ - eMode = 'w'; /* Web browser */ - }else{ - sqlite3_fprintf(p->out, - "ERROR: unknown option: \"%s\". Usage:\n", azArg[i]); - showHelp(p->out, azArg[0]); - rc = 1; - sqlite3_free(zFile); - goto meta_command_exit; - } - }else if( zFile==0 && eMode==0 ){ - if( cli_strcmp(z, "off")==0 ){ -#ifdef _WIN32 - zFile = sqlite3_mprintf("nul"); -#else - zFile = sqlite3_mprintf("/dev/null"); -#endif - }else{ - zFile = sqlite3_mprintf("%s", z); - } - if( zFile && zFile[0]=='|' ){ - while( i+1<nArg ) zFile = sqlite3_mprintf("%z %s", zFile, azArg[++i]); - break; - } - }else{ - sqlite3_fprintf(p->out, - "ERROR: extra parameter: \"%s\". Usage:\n", azArg[i]); - showHelp(p->out, azArg[0]); - rc = 1; - sqlite3_free(zFile); - goto meta_command_exit; - } - } - if( zFile==0 ){ - zFile = sqlite3_mprintf("stdout"); - } - shell_check_oom(zFile); - if( bOnce ){ - p->outCount = 2; - }else{ - p->outCount = 0; - } - output_reset(p); -#ifndef SQLITE_NOHAVE_SYSTEM - if( eMode=='e' || eMode=='x' || eMode=='w' ){ - p->doXdgOpen = 1; - outputModePush(p); - if( eMode=='x' ){ - /* spreadsheet mode. Output as CSV. */ - newTempFile(p, "csv"); - ShellClearFlag(p, SHFLG_Echo); - p->mode = MODE_Csv; - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf); -#ifdef _WIN32 - zBom = zBomUtf8; /* Always include the BOM on Windows, as Excel does - ** not work without it. */ -#endif - }else if( eMode=='w' ){ - /* web-browser mode. */ - newTempFile(p, "html"); - if( !bPlain ) p->mode = MODE_Www; - }else{ - /* text editor mode */ - newTempFile(p, "txt"); - } - sqlite3_free(zFile); - zFile = sqlite3_mprintf("%s", p->zTempFile); - } -#endif /* SQLITE_NOHAVE_SYSTEM */ - shell_check_oom(zFile); - if( zFile[0]=='|' ){ -#ifdef SQLITE_OMIT_POPEN - eputz("Error: pipes are not supported in this OS\n"); - rc = 1; - output_redir(p, stdout); -#else - FILE *pfPipe = sqlite3_popen(zFile + 1, "w"); - if( pfPipe==0 ){ - assert( stderr!=NULL ); - sqlite3_fprintf(stderr,"Error: cannot open pipe \"%s\"\n", zFile + 1); - rc = 1; - }else{ - output_redir(p, pfPipe); - if( zBom ) sqlite3_fputs(zBom, pfPipe); - sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); - } -#endif - }else{ - FILE *pfFile = output_file_open(zFile); - if( pfFile==0 ){ - if( cli_strcmp(zFile,"off")!=0 ){ - assert( stderr!=NULL ); - sqlite3_fprintf(stderr,"Error: cannot write to \"%s\"\n", zFile); - } - rc = 1; - } else { - output_redir(p, pfFile); - if( zBom ) sqlite3_fputs(zBom, pfFile); - if( bPlain && eMode=='w' ){ - sqlite3_fputs( - "<!DOCTYPE html>\n<BODY>\n<PLAINTEXT>\n", - pfFile - ); - } - sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); - } - } - sqlite3_free(zFile); + rc = dotCmdOutput(p); }else #endif /* !defined(SQLITE_SHELL_FIDDLE) */ @@ -10627,7 +10192,7 @@ static int do_meta_command(char *zLine, ShellState *p){ "SELECT key, quote(value) " "FROM temp.sqlite_parameters;", -1, &pStmt, 0); while( rx==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ - sqlite3_fprintf(p->out, + cli_printf(p->out, "%-*s %s\n", len, sqlite3_column_text(pStmt,0), sqlite3_column_text(pStmt,1)); } @@ -10673,7 +10238,7 @@ static int do_meta_command(char *zLine, ShellState *p){ rx = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); if( rx!=SQLITE_OK ){ - sqlite3_fprintf(p->out, "Error: %s\n", sqlite3_errmsg(p->db)); + cli_printf(p->out, "Error: %s\n", sqlite3_errmsg(p->db)); sqlite3_finalize(pStmt); pStmt = 0; rc = 1; @@ -10703,10 +10268,10 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='p' && n>=3 && cli_strncmp(azArg[0], "print", n)==0 ){ int i; for(i=1; i<nArg; i++){ - if( i>1 ) sqlite3_fputs(" ", p->out); - sqlite3_fputs(azArg[i], p->out); + if( i>1 ) cli_puts(" ", p->out); + cli_puts(azArg[i], p->out); } - sqlite3_fputs("\n", p->out); + cli_puts("\n", p->out); }else #ifndef SQLITE_OMIT_PROGRESS_CALLBACK @@ -10733,6 +10298,19 @@ static int do_meta_command(char *zLine, ShellState *p){ p->flgProgress |= SHELL_PROGRESS_ONCE; continue; } + if( cli_strcmp(z,"timeout")==0 ){ + if( i==nArg-1 ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + i++; + p->tmProgress = atof(azArg[i]); + if( p->tmProgress>0.0 ){ + p->flgProgress = SHELL_PROGRESS_QUIET|SHELL_PROGRESS_TMOUT; + if( nn==0 ) nn = 100; + } + continue; + } if( cli_strcmp(z,"limit")==0 ){ if( i+1>=nArg ){ eputz("Error: missing argument on --limit\n"); @@ -10743,7 +10321,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } continue; } - sqlite3_fprintf(stderr,"Error: unknown option: \"%s\"\n", azArg[i]); + cli_printf(stderr,"Error: unknown option: \"%s\"\n", azArg[i]); rc = 1; goto meta_command_exit; }else{ @@ -10787,7 +10365,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #else p->in = sqlite3_popen(azArg[1]+1, "r"); if( p->in==0 ){ - sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); + cli_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); rc = 1; }else{ rc = process_input(p, "<pipe>"); @@ -10795,10 +10373,12 @@ static int do_meta_command(char *zLine, ShellState *p){ } #endif }else if( (p->in = openChrSource(azArg[1]))==0 ){ - sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); + cli_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); rc = 1; }else{ - rc = process_input(p, azArg[1]); + char *zFilename = strdup(azArg[1]); + rc = process_input(p, zFilename); + free(zFilename); fclose(p->in); } p->in = inSaved; @@ -10828,7 +10408,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } rc = sqlite3_open(zSrcFile, &pSrc); if( rc!=SQLITE_OK ){ - sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zSrcFile); + cli_printf(stderr,"Error: cannot open \"%s\"\n", zSrcFile); close_db(pSrc); return 1; } @@ -10866,21 +10446,21 @@ static int do_meta_command(char *zLine, ShellState *p){ ){ if( nArg==2 ){ if( cli_strcmp(azArg[1], "vm")==0 ){ - p->scanstatsOn = 3; + p->mode.scanstatsOn = 3; }else if( cli_strcmp(azArg[1], "est")==0 ){ - p->scanstatsOn = 2; + p->mode.scanstatsOn = 2; }else{ - p->scanstatsOn = (u8)booleanValue(azArg[1]); + p->mode.scanstatsOn = (u8)booleanValue(azArg[1]); } open_db(p, 0); sqlite3_db_config( - p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->scanstatsOn, (int*)0 + p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->mode.scanstatsOn, (int*)0 ); #if !defined(SQLITE_ENABLE_STMT_SCANSTATUS) eputz("Warning: .scanstats not available in this build.\n"); #elif !defined(SQLITE_ENABLE_BYTECODE_VTAB) - if( p->scanstatsOn==3 ){ + if( p->mode.scanstatsOn==3 ){ eputz("Warning: \".scanstats vm\" not available in this build.\n"); } #endif @@ -10891,7 +10471,6 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( c=='s' && cli_strncmp(azArg[0], "schema", n)==0 ){ - ShellText sSelect; ShellState data; char *zErrMsg = 0; const char *zDiv = "("; @@ -10899,22 +10478,27 @@ static int do_meta_command(char *zLine, ShellState *p){ int iSchema = 0; int bDebug = 0; int bNoSystemTabs = 0; + int bIndent = 0; int ii; - + sqlite3_str *pSql; + sqlite3_stmt *pStmt = 0; + open_db(p, 0); memcpy(&data, p, sizeof(data)); - data.showHeader = 0; - data.cMode = data.mode = MODE_Semi; - initText(&sSelect); + data.mode.spec.bTitles = QRF_No; + data.mode.eMode = MODE_List; + data.mode.spec.eText = QRF_TEXT_Plain; + data.mode.spec.nCharLimit = 0; + data.mode.spec.zRowSep = "\n"; for(ii=1; ii<nArg; ii++){ if( optionMatch(azArg[ii],"indent") ){ - data.cMode = data.mode = MODE_Pretty; + bIndent = 1; }else if( optionMatch(azArg[ii],"debug") ){ bDebug = 1; }else if( optionMatch(azArg[ii],"nosys") ){ bNoSystemTabs = 1; }else if( azArg[ii][0]=='-' ){ - sqlite3_fprintf(stderr,"Unknown option: \"%s\"\n", azArg[ii]); + cli_printf(stderr,"Unknown option: \"%s\"\n", azArg[ii]); rc = 1; goto meta_command_exit; }else if( zName==0 ){ @@ -10931,96 +10515,85 @@ static int do_meta_command(char *zLine, ShellState *p){ || sqlite3_strlike(zName,"sqlite_temp_master", '\\')==0 || sqlite3_strlike(zName,"sqlite_temp_schema", '\\')==0; if( isSchema ){ - char *new_argv[2], *new_colv[2]; - new_argv[0] = sqlite3_mprintf( - "CREATE TABLE %s (\n" + cli_printf(p->out, + "CREATE TABLE %ssqlite_schema (\n" " type text,\n" " name text,\n" " tbl_name text,\n" " rootpage integer,\n" " sql text\n" - ")", zName); - shell_check_oom(new_argv[0]); - new_argv[1] = 0; - new_colv[0] = "sql"; - new_colv[1] = 0; - callback(&data, 1, new_argv, new_colv); - sqlite3_free(new_argv[0]); + ");\n", + sqlite3_strlike("sqlite_t%",zName,0)==0 ? "temp." : "" + ); } } - if( zDiv ){ - sqlite3_stmt *pStmt = 0; - rc = sqlite3_prepare_v2(p->db, "SELECT name FROM pragma_database_list", - -1, &pStmt, 0); - if( rc ){ - shellDatabaseError(p->db); - sqlite3_finalize(pStmt); - rc = 1; - goto meta_command_exit; - } - appendText(&sSelect, "SELECT sql FROM", 0); - iSchema = 0; - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - const char *zDb = (const char*)sqlite3_column_text(pStmt, 0); - char zScNum[30]; - sqlite3_snprintf(sizeof(zScNum), zScNum, "%d", ++iSchema); - appendText(&sSelect, zDiv, 0); - zDiv = " UNION ALL "; - appendText(&sSelect, "SELECT shell_add_schema(sql,", 0); - if( sqlite3_stricmp(zDb, "main")!=0 ){ - appendText(&sSelect, zDb, '\''); - }else{ - appendText(&sSelect, "NULL", 0); - } - appendText(&sSelect, ",name) AS sql, type, tbl_name, name, rowid,", 0); - appendText(&sSelect, zScNum, 0); - appendText(&sSelect, " AS snum, ", 0); - appendText(&sSelect, zDb, '\''); - appendText(&sSelect, " AS sname FROM ", 0); - appendText(&sSelect, zDb, quoteChar(zDb)); - appendText(&sSelect, ".sqlite_schema", 0); - } + rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); + if( rc ){ + shellDatabaseError(p->db); sqlite3_finalize(pStmt); -#ifndef SQLITE_OMIT_INTROSPECTION_PRAGMAS - if( zName ){ - appendText(&sSelect, - " UNION ALL SELECT shell_module_schema(name)," - " 'table', name, name, name, 9e+99, 'main' FROM pragma_module_list", - 0); - } -#endif - appendText(&sSelect, ") WHERE ", 0); - if( zName ){ - char *zQarg = sqlite3_mprintf("%Q", zName); - int bGlob; - shell_check_oom(zQarg); - bGlob = strchr(zName, '*') != 0 || strchr(zName, '?') != 0 || - strchr(zName, '[') != 0; - if( strchr(zName, '.') ){ - appendText(&sSelect, "lower(printf('%s.%s',sname,tbl_name))", 0); - }else{ - appendText(&sSelect, "lower(tbl_name)", 0); - } - appendText(&sSelect, bGlob ? " GLOB " : " LIKE ", 0); - appendText(&sSelect, zQarg, 0); - if( !bGlob ){ - appendText(&sSelect, " ESCAPE '\\' ", 0); - } - appendText(&sSelect, " AND ", 0); - sqlite3_free(zQarg); + + rc = 1; + goto meta_command_exit; + } + pSql = sqlite3_str_new(p->db); + sqlite3_str_appendf(pSql, "SELECT sql FROM", 0); + iSchema = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zDb = (const char*)sqlite3_column_text(pStmt, 1); + char zScNum[30]; + sqlite3_snprintf(sizeof(zScNum), zScNum, "%d", ++iSchema); + sqlite3_str_appendall(pSql, zDiv); + zDiv = " UNION ALL "; + if( sqlite3_stricmp(zDb, "main")==0 ){ + sqlite3_str_appendf(pSql, + "SELECT shell_format_schema(shell_add_schema(sql,NULL,name),%d)", + bIndent); + }else{ + sqlite3_str_appendf(pSql, + "SELECT shell_format_schema(shell_add_schema(sql,%Q,name),%d)", + zDb, bIndent); } - if( bNoSystemTabs ){ - appendText(&sSelect, "name NOT LIKE 'sqlite__%%' ESCAPE '_' AND ", 0); + sqlite3_str_appendf(pSql, + " AS sql, type, tbl_name, name, rowid, %d AS snum, %Q as sname", + ++iSchema, zDb); + sqlite3_str_appendf(pSql," FROM \"%w\".sqlite_schema", zDb); + } + sqlite3_finalize(pStmt); +#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) \ + && !defined(SQLITE_OMIT_VIRTUALTABLE) + if( zName ){ + sqlite3_str_appendall(pSql, + " UNION ALL SELECT shell_module_schema(name)," + " 'table', name, name, name, 9e+99, 'main' FROM pragma_module_list"); + } +#endif + sqlite3_str_appendf(pSql, ") WHERE ", 0); + if( zName ){ + int bGlob; + bGlob = strchr(zName, '*') != 0 || strchr(zName, '?') != 0 || + strchr(zName, '[') != 0; + if( strchr(zName, '.') ){ + sqlite3_str_appendall(pSql, "lower(format('%%s.%%s',sname,tbl_name))"); + }else{ + sqlite3_str_appendall(pSql, "lower(tbl_name)"); } - appendText(&sSelect, "sql IS NOT NULL" - " ORDER BY snum, rowid", 0); - if( bDebug ){ - sqlite3_fprintf(p->out, "SQL: %s;\n", sSelect.zTxt); + if( bGlob ){ + sqlite3_str_appendf(pSql, " GLOB %Q AND ", zName); }else{ - rc = sqlite3_exec(p->db, sSelect.zTxt, callback, &data, &zErrMsg); + sqlite3_str_appendf(pSql, " LIKE %Q ESCAPE '\\' AND ", zName); } - freeText(&sSelect); } + if( bNoSystemTabs ){ + sqlite3_str_appendf(pSql, " name NOT LIKE 'sqlite__%%' ESCAPE '_' AND "); + } + sqlite3_str_appendf(pSql, "sql IS NOT NULL ORDER BY snum, rowid"); + if( bDebug ){ + cli_printf(p->out, "SQL: %s;\n", sqlite3_str_value(pSql)); + }else{ + rc = shell_exec(&data, sqlite3_str_value(pSql), &zErrMsg); + } + sqlite3_str_free(pSql); + if( zErrMsg ){ shellEmitError(zErrMsg); sqlite3_free(zErrMsg); @@ -11076,7 +10649,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else{ rc = sqlite3session_attach(pSession->p, azCmd[1]); if( rc ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "ERROR: sqlite3session_attach() returns %d\n",rc); rc = 0; } @@ -11096,7 +10669,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( pSession->p==0 ) goto session_not_open; out = sqlite3_fopen(azCmd[1], "wb"); if( out==0 ){ - sqlite3_fprintf(stderr,"ERROR: cannot open \"%s\" for writing\n", + cli_printf(stderr,"ERROR: cannot open \"%s\" for writing\n", azCmd[1]); }else{ int szChng; @@ -11107,12 +10680,12 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = sqlite3session_patchset(pSession->p, &szChng, &pChng); } if( rc ){ - sqlite3_fprintf(stdout, "Error: error code %d\n", rc); + cli_printf(stdout, "Error: error code %d\n", rc); rc = 0; } if( pChng && fwrite(pChng, szChng, 1, out)!=1 ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "ERROR: Failed to write entire %d-byte output\n", szChng); } sqlite3_free(pChng); @@ -11140,7 +10713,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); if( pAuxDb->nSession ){ ii = sqlite3session_enable(pSession->p, ii); - sqlite3_fprintf(p->out, + cli_printf(p->out, "session %s enable flag = %d\n", pSession->zName, ii); } }else @@ -11177,7 +10750,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); if( pAuxDb->nSession ){ ii = sqlite3session_indirect(pSession->p, ii); - sqlite3_fprintf(p->out, + cli_printf(p->out, "session %s indirect flag = %d\n", pSession->zName, ii); } }else @@ -11190,7 +10763,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nCmd!=1 ) goto session_syntax_error; if( pAuxDb->nSession ){ ii = sqlite3session_isempty(pSession->p); - sqlite3_fprintf(p->out, + cli_printf(p->out, "session %s isempty flag = %d\n", pSession->zName, ii); } }else @@ -11200,7 +10773,7 @@ static int do_meta_command(char *zLine, ShellState *p){ */ if( cli_strcmp(azCmd[0],"list")==0 ){ for(i=0; i<pAuxDb->nSession; i++){ - sqlite3_fprintf(p->out, "%d %s\n", i, pAuxDb->aSession[i].zName); + cli_printf(p->out, "%d %s\n", i, pAuxDb->aSession[i].zName); } }else @@ -11215,19 +10788,19 @@ static int do_meta_command(char *zLine, ShellState *p){ if( zName[0]==0 ) goto session_syntax_error; for(i=0; i<pAuxDb->nSession; i++){ if( cli_strcmp(pAuxDb->aSession[i].zName,zName)==0 ){ - sqlite3_fprintf(stderr,"Session \"%s\" already exists\n", zName); + cli_printf(stderr,"Session \"%s\" already exists\n", zName); goto meta_command_exit; } } if( pAuxDb->nSession>=ArraySize(pAuxDb->aSession) ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Maximum of %d sessions\n", ArraySize(pAuxDb->aSession)); goto meta_command_exit; } pSession = &pAuxDb->aSession[pAuxDb->nSession]; rc = sqlite3session_create(p->db, azCmd[1], &pSession->p); if( rc ){ - sqlite3_fprintf(stderr,"Cannot open session: error code=%d\n", rc); + cli_printf(stderr,"Cannot open session: error code=%d\n", rc); rc = 0; goto meta_command_exit; } @@ -11251,7 +10824,7 @@ static int do_meta_command(char *zLine, ShellState *p){ int i, v; for(i=1; i<nArg; i++){ v = booleanValue(azArg[i]); - sqlite3_fprintf(p->out, "%s: %d 0x%x\n", azArg[i], v, v); + cli_printf(p->out, "%s: %d 0x%x\n", azArg[i], v, v); } } if( cli_strncmp(azArg[0]+9, "integer", n-9)==0 ){ @@ -11260,7 +10833,7 @@ static int do_meta_command(char *zLine, ShellState *p){ char zBuf[200]; v = integerValue(azArg[i]); sqlite3_snprintf(sizeof(zBuf),zBuf,"%s: %lld 0x%llx\n", azArg[i],v,v); - sqlite3_fputs(zBuf, p->out); + cli_puts(zBuf, p->out); } } }else @@ -11287,9 +10860,9 @@ static int do_meta_command(char *zLine, ShellState *p){ bVerbose++; }else { - sqlite3_fprintf(stderr, + cli_printf(stderr, "Unknown option \"%s\" on \"%s\"\n", azArg[i], azArg[0]); - sqlite3_fputs("Should be one of: --init -v\n", stderr); + cli_puts("Should be one of: --init -v\n", stderr); rc = 1; goto meta_command_exit; } @@ -11334,10 +10907,10 @@ static int do_meta_command(char *zLine, ShellState *p){ if( zAns==0 ) continue; k = 0; if( bVerbose>0 ){ - sqlite3_fprintf(stdout, "%d: %s %s\n", tno, zOp, zSql); + cli_printf(stdout, "%d: %s %s\n", tno, zOp, zSql); } if( cli_strcmp(zOp,"memo")==0 ){ - sqlite3_fprintf(p->out, "%s\n", zSql); + cli_printf(p->out, "%s\n", zSql); }else if( cli_strcmp(zOp,"run")==0 ){ char *zErrMsg = 0; @@ -11346,22 +10919,22 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = sqlite3_exec(p->db, zSql, captureOutputCallback, &str, &zErrMsg); nTest++; if( bVerbose ){ - sqlite3_fprintf(p->out, "Result: %s\n", str.zTxt); + cli_printf(p->out, "Result: %s\n", str.zTxt); } if( rc || zErrMsg ){ nErr++; rc = 1; - sqlite3_fprintf(p->out, "%d: error-code-%d: %s\n", tno, rc,zErrMsg); + cli_printf(p->out, "%d: error-code-%d: %s\n", tno, rc,zErrMsg); sqlite3_free(zErrMsg); }else if( cli_strcmp(zAns,str.zTxt)!=0 ){ nErr++; rc = 1; - sqlite3_fprintf(p->out, "%d: Expected: [%s]\n", tno, zAns); - sqlite3_fprintf(p->out, "%d: Got: [%s]\n", tno, str.zTxt); + cli_printf(p->out, "%d: Expected: [%s]\n", tno, zAns); + cli_printf(p->out, "%d: Got: [%s]\n", tno, str.zTxt); } } else{ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Unknown operation \"%s\" on selftest line %d\n", zOp, tno); rc = 1; break; @@ -11370,7 +10943,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_finalize(pStmt); } /* End loop over k */ freeText(&str); - sqlite3_fprintf(p->out, "%d errors out of %d tests\n", nErr, nTest); + cli_printf(p->out, "%d errors out of %d tests\n", nErr, nTest); }else if( c=='s' && cli_strncmp(azArg[0], "separator", n)==0 ){ @@ -11379,12 +10952,10 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = 1; } if( nArg>=2 ){ - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, - "%.*s", (int)ArraySize(p->colSeparator)-1, azArg[1]); + modeSetStr(&p->mode.spec.zColumnSep, azArg[1]); } if( nArg>=3 ){ - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, - "%.*s", (int)ArraySize(p->rowSeparator)-1, azArg[2]); + modeSetStr(&p->mode.spec.zRowSep,azArg[2]); } }else @@ -11418,7 +10989,7 @@ static int do_meta_command(char *zLine, ShellState *p){ bDebug = 1; }else { - sqlite3_fprintf(stderr, + cli_printf(stderr, "Unknown option \"%s\" on \"%s\"\n", azArg[i], azArg[0]); showHelp(p->out, azArg[0]); rc = 1; @@ -11497,7 +11068,7 @@ static int do_meta_command(char *zLine, ShellState *p){ freeText(&sQuery); freeText(&sSql); if( bDebug ){ - sqlite3_fprintf(p->out, "%s\n", zSql); + cli_printf(p->out, "%s\n", zSql); }else{ shell_exec(p, zSql, 0); } @@ -11527,7 +11098,7 @@ static int do_meta_command(char *zLine, ShellState *p){ "' OR ') as query, tname from tabcols group by tname)" , zRevText); shell_check_oom(zRevText); - if( bDebug ) sqlite3_fprintf(p->out, "%s\n", zRevText); + if( bDebug ) cli_printf(p->out, "%s\n", zRevText); lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0); if( lrc!=SQLITE_OK ){ /* assert(lrc==SQLITE_NOMEM); // might also be SQLITE_ERROR if the @@ -11540,7 +11111,7 @@ static int do_meta_command(char *zLine, ShellState *p){ const char *zGenQuery = (char*)sqlite3_column_text(pStmt,0); sqlite3_stmt *pCheckStmt; lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0); - if( bDebug ) sqlite3_fprintf(p->out, "%s\n", zGenQuery); + if( bDebug ) cli_printf(p->out, "%s\n", zGenQuery); if( lrc!=SQLITE_OK ){ rc = 1; }else{ @@ -11548,7 +11119,7 @@ static int do_meta_command(char *zLine, ShellState *p){ double countIrreversible = sqlite3_column_double(pCheckStmt, 0); if( countIrreversible>0 ){ int sz = (int)(countIrreversible + 0.5); - sqlite3_fprintf(stderr, + cli_printf(stderr, "Digest includes %d invalidly encoded text field%s.\n", sz, (sz>1)? "s": ""); } @@ -11587,7 +11158,7 @@ static int do_meta_command(char *zLine, ShellState *p){ x = zCmd!=0 ? system(zCmd) : 1; /*consoleRenewSetup();*/ sqlite3_free(zCmd); - if( x ) sqlite3_fprintf(stderr,"System command returns %d\n", x); + if( x ) cli_printf(stderr,"System command returns %d\n", x); }else #endif /* !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) */ @@ -11600,48 +11171,51 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = 1; goto meta_command_exit; } - sqlite3_fprintf(p->out, "%12.12s: %s\n","echo", - azBool[ShellHasFlag(p, SHFLG_Echo)]); - sqlite3_fprintf(p->out, "%12.12s: %s\n","eqp", azBool[p->autoEQP&3]); - sqlite3_fprintf(p->out, "%12.12s: %s\n","explain", - p->mode==MODE_Explain ? "on" : p->autoExplain ? "auto" : "off"); - sqlite3_fprintf(p->out, "%12.12s: %s\n","headers", - azBool[p->showHeader!=0]); - if( p->mode==MODE_Column - || (p->mode>=MODE_Markdown && p->mode<=MODE_Box) + cli_printf(p->out, "%12.12s: %s\n","echo", + azBool[(p->mode.mFlags & MFLG_ECHO)!=0]); + cli_printf(p->out, "%12.12s: %s\n","eqp", azBool[p->mode.autoEQP&3]); + cli_printf(p->out, "%12.12s: %s\n","explain", + p->mode.autoExplain ? "auto" : "off"); + cli_printf(p->out, "%12.12s: %s\n","headers", + azBool[p->mode.spec.bTitles==QRF_Yes]); + if( p->mode.spec.eStyle==QRF_STYLE_Column + || p->mode.spec.eStyle==QRF_STYLE_Box + || p->mode.spec.eStyle==QRF_STYLE_Table + || p->mode.spec.eStyle==QRF_STYLE_Markdown ){ - sqlite3_fprintf(p->out, + cli_printf(p->out, "%12.12s: %s --wrap %d --wordwrap %s --%squote\n", "mode", - modeDescr[p->mode], p->cmOpts.iWrap, - p->cmOpts.bWordWrap ? "on" : "off", - p->cmOpts.bQuote ? "" : "no"); + aModeInfo[p->mode.eMode].zName, p->mode.spec.nWrap, + p->mode.spec.bWordWrap==QRF_Yes ? "on" : "off", + p->mode.spec.eText==QRF_TEXT_Sql ? "" : "no"); }else{ - sqlite3_fprintf(p->out, "%12.12s: %s\n","mode", modeDescr[p->mode]); + cli_printf(p->out, "%12.12s: %s\n","mode", + aModeInfo[p->mode.eMode].zName); } - sqlite3_fprintf(p->out, "%12.12s: ", "nullvalue"); - output_c_string(p->out, p->nullValue); - sqlite3_fputs("\n", p->out); - sqlite3_fprintf(p->out, "%12.12s: %s\n","output", + cli_printf(p->out, "%12.12s: ", "nullvalue"); + output_c_string(p->out, p->mode.spec.zNull); + cli_puts("\n", p->out); + cli_printf(p->out, "%12.12s: %s\n","output", strlen30(p->outfile) ? p->outfile : "stdout"); - sqlite3_fprintf(p->out, "%12.12s: ", "colseparator"); - output_c_string(p->out, p->colSeparator); - sqlite3_fputs("\n", p->out); - sqlite3_fprintf(p->out, "%12.12s: ", "rowseparator"); - output_c_string(p->out, p->rowSeparator); - sqlite3_fputs("\n", p->out); + cli_printf(p->out, "%12.12s: ", "colseparator"); + output_c_string(p->out, p->mode.spec.zColumnSep); + cli_puts("\n", p->out); + cli_printf(p->out, "%12.12s: ", "rowseparator"); + output_c_string(p->out, p->mode.spec.zRowSep); + cli_puts("\n", p->out); switch( p->statsOn ){ case 0: zOut = "off"; break; default: zOut = "on"; break; case 2: zOut = "stmt"; break; case 3: zOut = "vmstep"; break; } - sqlite3_fprintf(p->out, "%12.12s: %s\n","stats", zOut); - sqlite3_fprintf(p->out, "%12.12s: ", "width"); - for (i=0;i<p->nWidth;i++) { - sqlite3_fprintf(p->out, "%d ", p->colWidth[i]); + cli_printf(p->out, "%12.12s: %s\n","stats", zOut); + cli_printf(p->out, "%12.12s: ", "width"); + for(i=0; i<p->mode.spec.nWidth; i++){ + cli_printf(p->out, "%d ", (int)p->mode.spec.aWidth[i]); } - sqlite3_fputs("\n", p->out); - sqlite3_fprintf(p->out, "%12.12s: %s\n", "filename", + cli_puts("\n", p->out); + cli_printf(p->out, "%12.12s: %s\n", "filename", p->pAuxDb->zDbFilename ? p->pAuxDb->zDbFilename : ""); }else @@ -11667,11 +11241,9 @@ static int do_meta_command(char *zLine, ShellState *p){ || cli_strncmp(azArg[0], "indexes", n)==0) ) ){ sqlite3_stmt *pStmt; - char **azResult; - int nRow, nAlloc; - int ii; - ShellText s; - initText(&s); + sqlite3_str *pSql; + const char *zPattern = nArg>1 ? azArg[1] : 0; + open_db(p, 0); rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); if( rc ){ @@ -11688,103 +11260,53 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_finalize(pStmt); goto meta_command_exit; } - for(ii=0; sqlite3_step(pStmt)==SQLITE_ROW; ii++){ + pSql = sqlite3_str_new(p->db); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ const char *zDbName = (const char*)sqlite3_column_text(pStmt, 1); if( zDbName==0 ) continue; - if( s.zTxt && s.zTxt[0] ) appendText(&s, " UNION ALL ", 0); + if( sqlite3_str_length(pSql) ){ + sqlite3_str_appendall(pSql, " UNION ALL "); + } if( sqlite3_stricmp(zDbName, "main")==0 ){ - appendText(&s, "SELECT name FROM ", 0); + sqlite3_str_appendall(pSql, "SELECT name FROM "); }else{ - appendText(&s, "SELECT ", 0); - appendText(&s, zDbName, '\''); - appendText(&s, "||'.'||name FROM ", 0); + sqlite3_str_appendf(pSql, "SELECT %Q||'.'||name FROM ", zDbName); } - appendText(&s, zDbName, '"'); - appendText(&s, ".sqlite_schema ", 0); + sqlite3_str_appendf(pSql, "\"%w\".sqlite_schema", zDbName); if( c=='t' ){ - appendText(&s," WHERE type IN ('table','view')" - " AND name NOT LIKE 'sqlite__%' ESCAPE '_'" - " AND name LIKE ?1", 0); + sqlite3_str_appendf(pSql, + " WHERE type IN ('table','view')" + " AND name NOT LIKE 'sqlite__%%' ESCAPE '_'" + ); + if( zPattern ){ + sqlite3_str_appendf(pSql," AND name LIKE %Q", zPattern); + } }else{ - appendText(&s," WHERE type='index'" - " AND tbl_name LIKE ?1", 0); + sqlite3_str_appendf(pSql, " WHERE type='index'"); + if( zPattern ){ + sqlite3_str_appendf(pSql," AND tbl_name LIKE %Q", zPattern); + } } } rc = sqlite3_finalize(pStmt); if( rc==SQLITE_OK ){ - appendText(&s, " ORDER BY 1", 0); - rc = sqlite3_prepare_v2(p->db, s.zTxt, -1, &pStmt, 0); - } - freeText(&s); - if( rc ) return shellDatabaseError(p->db); - - /* Run the SQL statement prepared by the above block. Store the results - ** as an array of nul-terminated strings in azResult[]. */ - nRow = nAlloc = 0; - azResult = 0; - if( nArg>1 ){ - sqlite3_bind_text(pStmt, 1, azArg[1], -1, SQLITE_TRANSIENT); - }else{ - sqlite3_bind_text(pStmt, 1, "%", -1, SQLITE_STATIC); - } - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - if( nRow>=nAlloc ){ - char **azNew; - sqlite3_int64 n2 = 2*(sqlite3_int64)nAlloc + 10; - azNew = sqlite3_realloc64(azResult, sizeof(azResult[0])*n2); - shell_check_oom(azNew); - nAlloc = (int)n2; - azResult = azNew; - } - azResult[nRow] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); - shell_check_oom(azResult[nRow]); - nRow++; - } - if( sqlite3_finalize(pStmt)!=SQLITE_OK ){ - rc = shellDatabaseError(p->db); - } - - /* Pretty-print the contents of array azResult[] to the output */ - if( rc==0 && nRow>0 ){ - int len, maxlen = 0; - int i, j; - int nPrintCol, nPrintRow; - for(i=0; i<nRow; i++){ - len = strlen30(azResult[i]); - if( len>maxlen ) maxlen = len; - } - nPrintCol = 80/(maxlen+2); - if( nPrintCol<1 ) nPrintCol = 1; - nPrintRow = (nRow + nPrintCol - 1)/nPrintCol; - for(i=0; i<nPrintRow; i++){ - for(j=i; j<nRow; j+=nPrintRow){ - char *zSp = j<nPrintRow ? "" : " "; - sqlite3_fprintf(p->out, - "%s%-*s", zSp, maxlen, azResult[j] ? azResult[j]:""); - } - sqlite3_fputs("\n", p->out); - } + sqlite3_str_appendall(pSql, " ORDER BY 1"); } - for(ii=0; ii<nRow; ii++) sqlite3_free(azResult[ii]); - sqlite3_free(azResult); + /* Run the SQL statement in "split" mode. */ + modePush(p); + modeChange(p, MODE_Split); + shell_exec(p, sqlite3_str_value(pSql), 0); + sqlite3_str_free(pSql); + modePop(p); + if( rc ) return shellDatabaseError(p->db); }else -#ifndef SQLITE_SHELL_FIDDLE - /* Begin redirecting output to the file "testcase-out.txt" */ + /* Set the p->zTestcase name and begin redirecting output into + ** the cli_output_capture sqlite3_str */ if( c=='t' && cli_strcmp(azArg[0],"testcase")==0 ){ - output_reset(p); - p->out = output_file_open("testcase-out.txt"); - if( p->out==0 ){ - eputz("Error: cannot open 'testcase-out.txt'\n"); - } - if( nArg>=2 ){ - sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "%s", azArg[1]); - }else{ - sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "?"); - } + rc = dotCmdTestcase(p); }else -#endif /* !defined(SQLITE_SHELL_FIDDLE) */ #ifndef SQLITE_UNTESTABLE if( c=='t' && n>=8 && cli_strncmp(azArg[0], "testctrl", n)==0 ){ @@ -11837,10 +11359,10 @@ static int do_meta_command(char *zLine, ShellState *p){ /* --help lists all test-controls */ if( cli_strcmp(zCmd,"help")==0 ){ - sqlite3_fputs("Available test-controls:\n", p->out); + cli_puts("Available test-controls:\n", p->out); for(i=0; i<ArraySize(aCtrl); i++){ if( aCtrl[i].unSafe && !ShellHasFlag(p,SHFLG_TestingMode) ) continue; - sqlite3_fprintf(p->out, " .testctrl %s %s\n", + cli_printf(p->out, " .testctrl %s %s\n", aCtrl[i].zCtrlName, aCtrl[i].zUsage); } rc = 1; @@ -11857,7 +11379,7 @@ static int do_meta_command(char *zLine, ShellState *p){ testctrl = aCtrl[i].ctrlCode; iCtrl = i; }else{ - sqlite3_fprintf(stderr,"Error: ambiguous test-control: \"%s\"\n" + cli_printf(stderr,"Error: ambiguous test-control: \"%s\"\n" "Use \".testctrl --help\" for help\n", zCmd); rc = 1; goto meta_command_exit; @@ -11865,7 +11387,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } } if( testctrl<0 ){ - sqlite3_fprintf(stderr,"Error: unknown test-control: %s\n" + cli_printf(stderr,"Error: unknown test-control: %s\n" "Use \".testctrl --help\" for help\n", zCmd); }else{ switch(testctrl){ @@ -11945,13 +11467,13 @@ static int do_meta_command(char *zLine, ShellState *p){ if( sqlite3_stricmp(zLabel, aLabel[jj].zLabel)==0 ) break; } if( jj>=ArraySize(aLabel) ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Error: no such optimization: \"%s\"\n", zLabel); - sqlite3_fputs("Should be one of:", stderr); + cli_puts("Should be one of:", stderr); for(jj=0; jj<ArraySize(aLabel); jj++){ - sqlite3_fprintf(stderr," %s", aLabel[jj].zLabel); + cli_printf(stderr," %s", aLabel[jj].zLabel); } - sqlite3_fputs("\n", stderr); + cli_puts("\n", stderr); rc = 1; goto meta_command_exit; } @@ -11969,23 +11491,23 @@ static int do_meta_command(char *zLine, ShellState *p){ if( m & newOpt ) nOff++; } if( nOff<12 ){ - sqlite3_fputs("+All", p->out); + cli_puts("+All", p->out); for(ii=0; ii<ArraySize(aLabel); ii++){ if( !aLabel[ii].bDsply ) continue; if( (newOpt & aLabel[ii].mask)!=0 ){ - sqlite3_fprintf(p->out, " -%s", aLabel[ii].zLabel); + cli_printf(p->out, " -%s", aLabel[ii].zLabel); } } }else{ - sqlite3_fputs("-All", p->out); + cli_puts("-All", p->out); for(ii=0; ii<ArraySize(aLabel); ii++){ if( !aLabel[ii].bDsply ) continue; if( (newOpt & aLabel[ii].mask)==0 ){ - sqlite3_fprintf(p->out, " +%s", aLabel[ii].zLabel); + cli_printf(p->out, " +%s", aLabel[ii].zLabel); } } } - sqlite3_fputs("\n", p->out); + cli_puts("\n", p->out); rc2 = isOk = 3; break; } @@ -12025,7 +11547,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3 *db; if( ii==0 && cli_strcmp(azArg[2],"random")==0 ){ sqlite3_randomness(sizeof(ii),&ii); - sqlite3_fprintf(stdout, "-- random seed: %d\n", ii); + cli_printf(stdout, "-- random seed: %d\n", ii); } if( nArg==3 ){ db = 0; @@ -12078,7 +11600,7 @@ static int do_meta_command(char *zLine, ShellState *p){ case SQLITE_TESTCTRL_SEEK_COUNT: { u64 x = 0; rc2 = sqlite3_test_control(testctrl, p->db, &x); - sqlite3_fprintf(p->out, "%llu\n", x); + cli_printf(p->out, "%llu\n", x); isOk = 3; break; } @@ -12109,11 +11631,11 @@ static int do_meta_command(char *zLine, ShellState *p){ int val = 0; rc2 = sqlite3_test_control(testctrl, -id, &val); if( rc2!=SQLITE_OK ) break; - if( id>1 ) sqlite3_fputs(" ", p->out); - sqlite3_fprintf(p->out, "%d: %d", id, val); + if( id>1 ) cli_puts(" ", p->out); + cli_printf(p->out, "%d: %d", id, val); id++; } - if( id>1 ) sqlite3_fputs("\n", p->out); + if( id>1 ) cli_puts("\n", p->out); isOk = 3; } break; @@ -12152,7 +11674,7 @@ static int do_meta_command(char *zLine, ShellState *p){ int ii, jj, x; int *aOp; if( nArg!=4 ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "ERROR - should be: \".testctrl bitvec_test SIZE INT-ARRAY\"\n" ); rc = 1; @@ -12175,7 +11697,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } aOp[jj] = x; x = sqlite3_test_control(testctrl, iSize, aOp); - sqlite3_fprintf(p->out, "result: %d\n", x); + cli_printf(p->out, "result: %d\n", x); free(aOp); break; } @@ -12198,21 +11720,21 @@ static int do_meta_command(char *zLine, ShellState *p){ faultsim_state.nHit = 0; sqlite3_test_control(testctrl, faultsim_callback); }else if( cli_strcmp(z,"status")==0 ){ - sqlite3_fprintf(p->out, "faultsim.iId: %d\n", + cli_printf(p->out, "faultsim.iId: %d\n", faultsim_state.iId); - sqlite3_fprintf(p->out, "faultsim.iErr: %d\n", + cli_printf(p->out, "faultsim.iErr: %d\n", faultsim_state.iErr); - sqlite3_fprintf(p->out, "faultsim.iCnt: %d\n", + cli_printf(p->out, "faultsim.iCnt: %d\n", faultsim_state.iCnt); - sqlite3_fprintf(p->out, "faultsim.nHit: %d\n", + cli_printf(p->out, "faultsim.nHit: %d\n", faultsim_state.nHit); - sqlite3_fprintf(p->out, "faultsim.iInterval: %d\n", + cli_printf(p->out, "faultsim.iInterval: %d\n", faultsim_state.iInterval); - sqlite3_fprintf(p->out, "faultsim.eVerbose: %d\n", + cli_printf(p->out, "faultsim.eVerbose: %d\n", faultsim_state.eVerbose); - sqlite3_fprintf(p->out, "faultsim.nRepeat: %d\n", + cli_printf(p->out, "faultsim.nRepeat: %d\n", faultsim_state.nRepeat); - sqlite3_fprintf(p->out, "faultsim.nSkip: %d\n", + cli_printf(p->out, "faultsim.nSkip: %d\n", faultsim_state.nSkip); }else if( cli_strcmp(z,"-v")==0 ){ if( faultsim_state.eVerbose<2 ) faultsim_state.eVerbose++; @@ -12231,7 +11753,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( cli_strcmp(z,"-?")==0 || sqlite3_strglob("*help*",z)==0){ bShowHelp = 1; }else{ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Unrecognized fault_install argument: \"%s\"\n", azArg[kk]); rc = 1; @@ -12240,7 +11762,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } } if( bShowHelp ){ - sqlite3_fputs( + cli_puts( "Usage: .testctrl fault_install ARGS\n" "Possible arguments:\n" " off Disable faultsim\n" @@ -12262,13 +11784,13 @@ static int do_meta_command(char *zLine, ShellState *p){ } } if( isOk==0 && iCtrl>=0 ){ - sqlite3_fprintf(p->out, + cli_printf(p->out, "Usage: .testctrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage); rc = 1; }else if( isOk==1 ){ - sqlite3_fprintf(p->out, "%d\n", rc2); + cli_printf(p->out, "%d\n", rc2); }else if( isOk==2 ){ - sqlite3_fprintf(p->out, "0x%08x\n", rc2); + cli_printf(p->out, "0x%08x\n", rc2); } }else #endif /* !defined(SQLITE_UNTESTABLE) */ @@ -12280,13 +11802,17 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='t' && n>=5 && cli_strncmp(azArg[0], "timer", n)==0 ){ if( nArg==2 ){ - enableTimer = booleanValue(azArg[1]); - if( enableTimer && !HAS_TIMER ){ + if( cli_strcmp(azArg[1],"once")==0 ){ + p->enableTimer = 1; + }else{ + p->enableTimer = 2*booleanValue(azArg[1]); + } + if( p->enableTimer && !HAS_TIMER ){ eputz("Error: timer not available on this system.\n"); - enableTimer = 0; + p->enableTimer = 0; } }else{ - eputz("Usage: .timer on|off\n"); + eputz("Usage: .timer on|off|once\n"); rc = 1; } }else @@ -12323,13 +11849,13 @@ static int do_meta_command(char *zLine, ShellState *p){ mType |= SQLITE_TRACE_CLOSE; } else { - sqlite3_fprintf(stderr,"Unknown option \"%s\" on \".trace\"\n", z); + cli_printf(stderr,"Unknown option \"%s\" on \".trace\"\n", z); rc = 1; goto meta_command_exit; } }else{ output_file_close(p->traceOut); - p->traceOut = output_file_open(z); + p->traceOut = output_file_open(p, z); } } if( p->traceOut==0 ){ @@ -12368,21 +11894,21 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='v' && cli_strncmp(azArg[0], "version", n)==0 ){ char *zPtrSz = sizeof(void*)==8 ? "64-bit" : "32-bit"; - sqlite3_fprintf(p->out, "SQLite %s %s\n" /*extra-version-info*/, + cli_printf(p->out, "SQLite %s %s\n" /*extra-version-info*/, sqlite3_libversion(), sqlite3_sourceid()); #if SQLITE_HAVE_ZLIB - sqlite3_fprintf(p->out, "zlib version %s\n", zlibVersion()); + cli_printf(p->out, "zlib version %s\n", zlibVersion()); #endif #define CTIMEOPT_VAL_(opt) #opt #define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt) #if defined(__clang__) && defined(__clang_major__) - sqlite3_fprintf(p->out, "clang-" CTIMEOPT_VAL(__clang_major__) "." + cli_printf(p->out, "clang-" CTIMEOPT_VAL(__clang_major__) "." CTIMEOPT_VAL(__clang_minor__) "." CTIMEOPT_VAL(__clang_patchlevel__) " (%s)\n", zPtrSz); #elif defined(_MSC_VER) - sqlite3_fprintf(p->out, "msvc-" CTIMEOPT_VAL(_MSC_VER) " (%s)\n", zPtrSz); + cli_printf(p->out, "msvc-" CTIMEOPT_VAL(_MSC_VER) " (%s)\n", zPtrSz); #elif defined(__GNUC__) && defined(__VERSION__) - sqlite3_fprintf(p->out, "gcc-" __VERSION__ " (%s)\n", zPtrSz); + cli_printf(p->out, "gcc-" __VERSION__ " (%s)\n", zPtrSz); #endif }else @@ -12392,10 +11918,10 @@ static int do_meta_command(char *zLine, ShellState *p){ if( p->db ){ sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFS_POINTER, &pVfs); if( pVfs ){ - sqlite3_fprintf(p->out, "vfs.zName = \"%s\"\n", pVfs->zName); - sqlite3_fprintf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); - sqlite3_fprintf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); - sqlite3_fprintf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); + cli_printf(p->out, "vfs.zName = \"%s\"\n", pVfs->zName); + cli_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); + cli_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); + cli_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); } } }else @@ -12407,13 +11933,13 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_file_control(p->db, "main", SQLITE_FCNTL_VFS_POINTER, &pCurrent); } for(pVfs=sqlite3_vfs_find(0); pVfs; pVfs=pVfs->pNext){ - sqlite3_fprintf(p->out, "vfs.zName = \"%s\"%s\n", pVfs->zName, + cli_printf(p->out, "vfs.zName = \"%s\"%s\n", pVfs->zName, pVfs==pCurrent ? " <--- CURRENT" : ""); - sqlite3_fprintf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); - sqlite3_fprintf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); - sqlite3_fprintf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); + cli_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); + cli_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); + cli_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); if( pVfs->pNext ){ - sqlite3_fputs("-----------------------------------\n", p->out); + cli_puts("-----------------------------------\n", p->out); } } }else @@ -12424,7 +11950,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( p->db ){ sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFSNAME, &zVfsName); if( zVfsName ){ - sqlite3_fprintf(p->out, "%s\n", zVfsName); + cli_printf(p->out, "%s\n", zVfsName); sqlite3_free(zVfsName); } } @@ -12437,31 +11963,31 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='w' && cli_strncmp(azArg[0], "width", n)==0 ){ int j; - assert( nArg<=ArraySize(azArg) ); - p->nWidth = nArg-1; - p->colWidth = realloc(p->colWidth, (p->nWidth+1)*sizeof(int)*2); - if( p->colWidth==0 && p->nWidth>0 ) shell_out_of_memory(); - if( p->nWidth ) p->actualWidth = &p->colWidth[p->nWidth]; + p->mode.spec.nWidth = nArg-1; + p->mode.spec.aWidth = realloc(p->mode.spec.aWidth, + (p->mode.spec.nWidth+1)*sizeof(short int)); + shell_check_oom(p->mode.spec.aWidth); for(j=1; j<nArg; j++){ i64 w = integerValue(azArg[j]); - if( w < -30000 ) w = -30000; - if( w > +30000 ) w = +30000; - p->colWidth[j-1] = (int)w; + if( w < -QRF_MAX_WIDTH ) w = -QRF_MAX_WIDTH; + if( w > QRF_MAX_WIDTH ) w = QRF_MAX_WIDTH; + p->mode.spec.aWidth[j-1] = (short int)w; } }else { - sqlite3_fprintf(stderr,"Error: unknown command or invalid arguments: " + cli_printf(stderr,"Error: unknown command or invalid arguments: " " \"%s\". Enter \".help\" for help\n", azArg[0]); rc = 1; } meta_command_exit: - if( p->outCount ){ - p->outCount--; - if( p->outCount==0 ) output_reset(p); + if( p->nPopOutput ){ + p->nPopOutput--; + if( p->nPopOutput==0 ) output_reset(p); } p->bSafeMode = p->bSafeModePersist; + p->dot.nArg = 0; return rc; } @@ -12698,9 +12224,9 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ open_db(p, 0); if( ShellHasFlag(p,SHFLG_Backslash) ) resolve_backslashes(zSql); if( p->flgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0; - BEGIN_TIMER; + BEGIN_TIMER(p); rc = shell_exec(p, zSql, &zErrMsg); - END_TIMER(p->out); + END_TIMER(p); if( rc || zErrMsg ){ char zPrefix[100]; const char *zErrorTail; @@ -12724,7 +12250,7 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ }else{ sqlite3_snprintf(sizeof(zPrefix), zPrefix, "%s:", zErrorType); } - sqlite3_fprintf(stderr,"%s %s\n", zPrefix, zErrorTail); + cli_printf(stderr,"%s %s\n", zPrefix, zErrorTail); sqlite3_free(zErrMsg); zErrMsg = 0; return 1; @@ -12733,7 +12259,7 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ sqlite3_snprintf(sizeof(zLineBuf), zLineBuf, "changes: %lld total_changes: %lld", sqlite3_changes64(p->db), sqlite3_total_changes64(p->db)); - sqlite3_fprintf(p->out, "%s\n", zLineBuf); + cli_printf(p->out, "%s\n", zLineBuf); } if( doAutoDetectRestore(p, zSql) ) return 1; @@ -12741,8 +12267,8 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ } static void echo_group_input(ShellState *p, const char *zDo){ - if( ShellHasFlag(p, SHFLG_Echo) ){ - sqlite3_fprintf(p->out, "%s\n", zDo); + if( p->mode.mFlags & MFLG_ECHO ){ + cli_printf(p->out, "%s\n", zDo); fflush(p->out); } } @@ -12799,14 +12325,19 @@ static int process_input(ShellState *p, const char *zSrc){ int errCnt = 0; /* Number of errors seen */ i64 startline = 0; /* Line number for start of current input */ QuickScanState qss = QSS_Start; /* Accumulated line status (so far) */ + const char *saved_zInFile; /* Prior value of p->zInFile */ + i64 saved_lineno; /* Prior value of p->lineno */ if( p->inputNesting==MAX_INPUT_NESTING ){ /* This will be more informative in a later version. */ - sqlite3_fprintf(stderr,"%s: Input nesting limit (%d) reached at line %lld." + cli_printf(stderr,"%s: Input nesting limit (%d) reached at line %lld." " Check recursion.\n", zSrc, MAX_INPUT_NESTING, p->lineno); return 1; } ++p->inputNesting; + saved_zInFile = p->zInFile; + p->zInFile = zSrc; + saved_lineno = p->lineno; p->lineno = 0; CONTINUE_PROMPT_RESET; while( errCnt==0 || !bail_on_error || (p->in==0 && stdin_is_interactive) ){ @@ -12814,7 +12345,7 @@ static int process_input(ShellState *p, const char *zSrc){ zLine = one_input_line(p->in, zLine, nSql>0); if( zLine==0 ){ /* End of input */ - if( p->in==0 && stdin_is_interactive ) sqlite3_fputs("\n", p->out); + if( p->in==0 && stdin_is_interactive ) cli_puts("\n", p->out); break; } if( seenInterrupt ){ @@ -12871,7 +12402,7 @@ static int process_input(ShellState *p, const char *zSrc){ if( nSql>0x7fff0000 ){ char zSize[100]; sqlite3_snprintf(sizeof(zSize),zSize,"%,lld",nSql); - sqlite3_fprintf(stderr, "%s:%lld: Input SQL is too big: %s bytes\n", + cli_printf(stderr, "%s:%lld: Input SQL is too big: %s bytes\n", zSrc, startline, zSize); nSql = 0; errCnt++; @@ -12881,12 +12412,16 @@ static int process_input(ShellState *p, const char *zSrc){ errCnt += runOneSqlLine(p, zSql, p->in, startline); CONTINUE_PROMPT_RESET; nSql = 0; - if( p->outCount ){ + if( p->nPopOutput ){ output_reset(p); - p->outCount = 0; + p->nPopOutput = 0; }else{ clearTempFile(p); } + if( p->nPopMode ){ + modePop(p); + p->nPopMode = 0; + } p->bSafeMode = p->bSafeModePersist; qss = QSS_Start; }else if( nSql && QSS_PLAINWHITE(qss) ){ @@ -12904,6 +12439,8 @@ static int process_input(ShellState *p, const char *zSrc){ free(zSql); free(zLine); --p->inputNesting; + p->zInFile = saved_zInFile; + p->lineno = saved_lineno; return errCnt>0; } @@ -13064,13 +12601,13 @@ static void process_sqliterc( p->in = sqliterc ? sqlite3_fopen(sqliterc,"rb") : 0; if( p->in ){ if( stdin_is_interactive ){ - sqlite3_fprintf(stderr,"-- Loading resources from %s\n", sqliterc); + cli_printf(stderr,"-- Loading resources from %s\n", sqliterc); } - if( process_input(p, sqliterc) && bail_on_error ) exit(1); + if( process_input(p, sqliterc) && bail_on_error ) cli_exit(1); fclose(p->in); }else if( sqliterc_override!=0 ){ - sqlite3_fprintf(stderr,"cannot open: \"%s\"\n", sqliterc); - if( bail_on_error ) exit(1); + cli_printf(stderr,"cannot open: \"%s\"\n", sqliterc); + if( bail_on_error ) cli_exit(1); } p->in = inSaved; p->lineno = savedLineno; @@ -13092,8 +12629,8 @@ static const char zOptions[] = " -bail stop after hitting an error\n" " -batch force batch I/O\n" " -box set output mode to 'box'\n" - " -column set output mode to 'column'\n" " -cmd COMMAND run \"COMMAND\" before reading stdin\n" + " -column set output mode to 'column'\n" " -csv set output mode to 'csv'\n" #if !defined(SQLITE_OMIT_DESERIALIZE) " -deserialize open the database using sqlite3_deserialize()\n" @@ -13124,6 +12661,7 @@ static const char zOptions[] = #endif " -newline SEP set output row separator. Default: '\\n'\n" " -nofollow refuse to open symbolic links to database files\n" + " -noinit Do not read the ~/.sqliterc file at startup\n" " -nonce STRING set the safe-mode escape nonce\n" " -no-rowid-in-view Disable rowid-in-view using sqlite3_config()\n" " -nullvalue TEXT set text string for NULL values. Default ''\n" @@ -13132,6 +12670,7 @@ static const char zOptions[] = " -quote set output mode to 'quote'\n" " -readonly open the database read-only\n" " -safe enable safe-mode\n" + " -screenwidth N use N as the default screenwidth \n" " -separator SEP set output column separator. Default: '|'\n" #ifdef SQLITE_ENABLE_SORTER_REFERENCES " -sorterref SIZE sorter references threshold size\n" @@ -13148,11 +12687,11 @@ static const char zOptions[] = #endif ; static void usage(int showDetail){ - sqlite3_fprintf(stderr,"Usage: %s [OPTIONS] [FILENAME [SQL...]]\n" + cli_printf(stderr,"Usage: %s [OPTIONS] [FILENAME [SQL...]]\n" "FILENAME is the name of an SQLite database. A new database is created\n" "if the file does not previously exist. Defaults to :memory:.\n", Argv0); if( showDetail ){ - sqlite3_fprintf(stderr,"OPTIONS include:\n%s", zOptions); + cli_printf(stderr,"OPTIONS include:\n%s", zOptions); }else{ eputz("Use the -help option for additional information\n"); } @@ -13173,19 +12712,11 @@ static void verify_uninitialized(void){ /* ** Initialize the state information in data */ -static void main_init(ShellState *data) { - memset(data, 0, sizeof(*data)); - data->normalMode = data->cMode = data->mode = MODE_List; - data->autoExplain = 1; -#ifdef _WIN32 - data->crlfMode = 1; -#endif - data->pAuxDb = &data->aAuxDb[0]; - memcpy(data->colSeparator,SEP_Column, 2); - memcpy(data->rowSeparator,SEP_Row, 2); - data->showHeader = 0; - data->shellFlgs = SHFLG_Lookaside; - sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); +static void main_init(ShellState *p) { + memset(p, 0, sizeof(*p)); + p->pAuxDb = &p->aAuxDb[0]; + p->shellFlgs = SHFLG_Lookaside; + sqlite3_config(SQLITE_CONFIG_LOG, shellLog, p); #if !defined(SQLITE_SHELL_FIDDLE) verify_uninitialized(); #endif @@ -13200,22 +12731,18 @@ static void main_init(ShellState *data) { */ #if defined(_WIN32) || defined(WIN32) static void printBold(const char *zText){ -#if !SQLITE_OS_WINRT HANDLE out = GetStdHandle(STD_OUTPUT_HANDLE); CONSOLE_SCREEN_BUFFER_INFO defaultScreenInfo; GetConsoleScreenBufferInfo(out, &defaultScreenInfo); SetConsoleTextAttribute(out, FOREGROUND_RED|FOREGROUND_INTENSITY ); -#endif sputz(stdout, zText); -#if !SQLITE_OS_WINRT SetConsoleTextAttribute(out, defaultScreenInfo.wAttributes); -#endif } #else static void printBold(const char *zText){ - sqlite3_fprintf(stdout, "\033[1m%s\033[0m", zText); + cli_printf(stdout, "\033[1m%s\033[0m", zText); } #endif @@ -13225,9 +12752,9 @@ static void printBold(const char *zText){ */ static char *cmdline_option_value(int argc, char **argv, int i){ if( i==argc ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "%s: Error: missing argument to %s\n", argv[0], argv[argc-1]); - exit(1); + cli_exit(1); } return argv[i]; } @@ -13240,30 +12767,66 @@ static void sayAbnormalExit(void){ */ static int vfstraceOut(const char *z, void *pArg){ ShellState *p = (ShellState*)pArg; - sqlite3_fputs(z, p->out); + cli_puts(z, p->out); fflush(p->out); return 1; } -#ifndef SQLITE_SHELL_IS_UTF8 -# if (defined(_WIN32) || defined(WIN32)) \ - && (defined(_MSC_VER) || (defined(UNICODE) && defined(__GNUC__))) -# define SQLITE_SHELL_IS_UTF8 (0) -# else -# define SQLITE_SHELL_IS_UTF8 (1) -# endif -#endif - +/* Alternative name to the entry point for Fiddle */ #ifdef SQLITE_SHELL_FIDDLE # define main fiddle_main #endif -#if SQLITE_SHELL_IS_UTF8 -int SQLITE_CDECL main(int argc, char **argv){ -#else +/* Use the wmain() entry point on Windows. Translate arguments to +** UTF8, then invoke the traditional main() entry point which is +** renamed using a #define to utf8_main() . +*/ +#if defined(_WIN32) && !defined(main) +# define main utf8_main /* Rename entry point to utf_main() */ +int SQLITE_CDECL utf8_main(int,char**); /* Forward declaration */ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ - char **argv; -#endif + int rc, i; + char **argv = malloc( sizeof(char*) * (argc+1) ); + char **orig = argv; + if( argv==0 ){ + fprintf(stderr, "malloc failed\n"); + exit(1); + } + for(i=0; i<argc; i++){ + int nByte = WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, 0, 0, 0, 0); + if( nByte==0 ){ + argv[i] = 0; + }else{ + argv[i] = malloc( nByte ); + if( argv[i]==0 ){ + fprintf(stderr, "malloc failed\n"); + exit(1); + } + nByte = WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, argv[i],nByte,0,0); + if( nByte==0 ){ + free(argv[i]); + argv[i] = 0; + } + } + } + argv[argc] = 0; + rc = utf8_main(argc, argv); + for(i=0; i<argc; i++) free(orig[i]); + free(argv); + return rc; +} +#endif /* WIN32 */ + +/* +** This is the main entry point for the process. Everything starts here. +** +** The "main" identifier may have been #defined to something else: +** +** utf8_main On Windows +** fiddle_main In Fiddle +** sqlite3_shell Other projects that use shell.c as a subroutine +*/ +int SQLITE_CDECL main(int argc, char **argv){ #ifdef SQLITE_DEBUG sqlite3_int64 mem_main_enter = 0; #endif @@ -13278,15 +12841,13 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ int rc = 0; int warnInmemoryDb = 0; int readStdin = 1; + int noInit = 0; /* Do not read ~/.sqliterc if true */ int nCmd = 0; int nOptsEnd = argc; int bEnableVfstrace = 0; char **azCmd = 0; + int *aiCmd = 0; const char *zVfs = 0; /* Value of -vfs command-line option */ -#if !SQLITE_SHELL_IS_UTF8 - char **argvToFree = 0; - int argcToFree = 0; -#endif setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */ #ifdef SQLITE_SHELL_FIDDLE @@ -13305,7 +12866,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( getenv("SQLITE_DEBUG_BREAK") ){ if( isatty(0) && isatty(2) ){ char zLine[100]; - sqlite3_fprintf(stderr, + cli_printf(stderr, "attach debugger to process %d and press ENTER to continue...", GETPID()); if( sqlite3_fgets(zLine, sizeof(zLine), stdin)!=0 @@ -13315,11 +12876,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } }else{ #if defined(_WIN32) || defined(WIN32) -#if SQLITE_OS_WINRT - __debugbreak(); -#else DebugBreak(); -#endif #elif defined(SIGTRAP) raise(SIGTRAP); #endif @@ -13337,7 +12894,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #if USE_SYSTEM_SQLITE+0!=1 if( cli_strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n", sqlite3_sourceid(), SQLITE_SOURCE_ID); exit(1); @@ -13345,32 +12902,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #endif main_init(&data); - /* On Windows, we must translate command-line arguments into UTF-8. - ** The SQLite memory allocator subsystem has to be enabled in order to - ** do this. But we want to run an sqlite3_shutdown() afterwards so that - ** subsequent sqlite3_config() calls will work. So copy all results into - ** memory that does not come from the SQLite memory allocator. - */ -#if !SQLITE_SHELL_IS_UTF8 - sqlite3_initialize(); - argvToFree = malloc(sizeof(argv[0])*argc*2); - shell_check_oom(argvToFree); - argcToFree = argc; - argv = argvToFree + argc; - for(i=0; i<argc; i++){ - char *z = sqlite3_win32_unicode_to_utf8(wargv[i]); - i64 n; - shell_check_oom(z); - n = strlen(z); - argv[i] = malloc( n+1 ); - shell_check_oom(argv[i]); - memcpy(argv[i], z, n+1); - argvToFree[i] = argv[i]; - sqlite3_free(z); - } - sqlite3_shutdown(); -#endif - assert( argc>=1 && argv && argv[0] ); Argv0 = argv[0]; @@ -13386,7 +12917,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } #endif - /* Do an initial pass through the command-line argument to locate + /* Do an initial pass through the command-line arguments to locate ** the name of the database file, the name of the initialization file, ** the size of the alternative malloc heap, options affecting commands ** or SQL run from the command line, and the first command to execute. @@ -13398,16 +12929,20 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ char *z; z = argv[i]; if( z[0]!='-' || i>nOptsEnd ){ - if( data.aAuxDb->zDbFilename==0 ){ + if( data.aAuxDb->zDbFilename==0 && !isScriptFile(z,1) ){ data.aAuxDb->zDbFilename = z; }else{ /* Excess arguments are interpreted as SQL (or dot-commands) and ** mean that nothing is read from stdin */ readStdin = 0; + stdin_is_interactive = 0; nCmd++; azCmd = realloc(azCmd, sizeof(azCmd[0])*nCmd); shell_check_oom(azCmd); + aiCmd = realloc(aiCmd, sizeof(aiCmd[0])*nCmd); + shell_check_oom(azCmd); azCmd[nCmd-1] = z; + aiCmd[nCmd-1] = i; } continue; } @@ -13430,6 +12965,15 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ ** we do the actual processing of arguments later in a second pass. */ stdin_is_interactive = 0; + stdout_is_console = 0; + modeChange(&data, MODE_BATCH); + }else if( cli_strcmp(z,"-screenwidth")==0 ){ + int n = atoi(cmdline_option_value(argc, argv, ++i)); + if( n<2 ){ + sqlite3_fprintf(stderr,"minimum --screenwidth is 2\n"); + exit(1); + } + stdout_tty_width = n; }else if( cli_strcmp(z,"-utf8")==0 ){ }else if( cli_strcmp(z,"-no-utf8")==0 ){ }else if( cli_strcmp(z,"-no-rowid-in-view")==0 ){ @@ -13463,7 +13007,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ int szHdr = 0; sqlite3_config(SQLITE_CONFIG_PCACHE_HDRSZ, &szHdr); sz += szHdr; - sqlite3_fprintf(stdout, "Page cache size increased to %d to accommodate" + cli_printf(stdout, "Page cache size increased to %d to accommodate" " the %d-byte headers\n", (int)sz, szHdr); } verify_uninitialized(); @@ -13524,6 +13068,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ data.openFlags |= SQLITE_OPEN_READONLY; }else if( cli_strcmp(z,"-nofollow")==0 ){ data.openFlags |= SQLITE_OPEN_NOFOLLOW; + }else if( cli_strcmp(z,"-noinit")==0 ){ + noInit = 1; }else if( cli_strcmp(z,"-exclusive")==0 ){ /* UNDOCUMENTED */ data.openFlags |= SQLITE_OPEN_EXCLUSIVE; }else if( cli_strcmp(z,"-ifexists")==0 ){ @@ -13551,6 +13097,16 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( cli_strcmp(z,"-escape")==0 && i+1<argc ){ /* skip over the argument */ i++; + }else if( cli_strcmp(z,"-test-argv")==0 ){ + /* Undocumented test option. Print the values in argv[] and exit. + ** Use this to verify that any translation of the argv[], for example + ** on Windows that receives wargv[] from the OS and must convert + ** to UTF8 prior to calling this routine. */ + int kk; + for(kk=0; kk<argc; kk++){ + sqlite3_fprintf(stdout,"argv[%d] = \"%s\"\n", kk, argv[kk]); + } + return 0; } } #ifndef SQLITE_SHELL_FIDDLE @@ -13577,8 +13133,27 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ sqlite3_vfs *pVfs = sqlite3_vfs_find(zVfs); if( pVfs ){ sqlite3_vfs_register(pVfs, 1); - }else{ - sqlite3_fprintf(stderr,"no such VFS: \"%s\"\n", zVfs); + } +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) + else if( access(zVfs,0)==0 ){ + /* If the VFS name is not the name of an existing VFS, but it is + ** the name of a file, then try to load that file as an extension. + ** Presumably the extension implements the desired VFS. */ + sqlite3 *db = 0; + char *zErr = 0; + sqlite3_open(":memory:", &db); + sqlite3_enable_load_extension(db, 1); + rc = sqlite3_load_extension(db, zVfs, 0, &zErr); + sqlite3_close(db); + if( (rc&0xff)!=SQLITE_OK ){ + cli_printf(stderr, "could not load extension VFS \"%s\": %s\n", + zVfs, zErr); + exit(1); + } + } +#endif + else{ + cli_printf(stderr,"no such VFS: \"%s\"\n", zVfs); exit(1); } } @@ -13588,9 +13163,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ data.pAuxDb->zDbFilename = ":memory:"; warnInmemoryDb = argc==1; #else - sqlite3_fprintf(stderr, + cli_printf(stderr, "%s: Error: no database filename specified\n", Argv0); - return 1; + rc = 1; + goto shell_main_exit; #endif } data.out = stdout; @@ -13600,6 +13176,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #ifndef SQLITE_SHELL_FIDDLE sqlite3_appendvfs_init(0,0,0); #endif + modeDefault(&data); /* Go ahead and open the database file if it already exists. If the ** file does not exist, delay opening it. This prevents empty database @@ -13614,9 +13191,9 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ ** is given on the command line, look for a file named ~/.sqliterc and ** try to process it. */ - process_sqliterc(&data,zInitFile); + if( !noInit ) process_sqliterc(&data,zInitFile); - /* Make a second pass through the command-line argument and set + /* Make a second pass through the command-line arguments and set ** options. This second pass is delayed until after the initialization ** file is processed so that the command-line arguments will override ** settings in the initialization file. @@ -13628,45 +13205,44 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( cli_strcmp(z,"-init")==0 ){ i++; }else if( cli_strcmp(z,"-html")==0 ){ - data.mode = MODE_Html; + modeChange(&data, MODE_Html); }else if( cli_strcmp(z,"-list")==0 ){ - data.mode = MODE_List; + modeChange(&data, MODE_List); }else if( cli_strcmp(z,"-quote")==0 ){ - data.mode = MODE_Quote; - sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator, SEP_Comma); - sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator, SEP_Row); + modeChange(&data, MODE_Quote); }else if( cli_strcmp(z,"-line")==0 ){ - data.mode = MODE_Line; + modeChange(&data, MODE_Line); }else if( cli_strcmp(z,"-column")==0 ){ - data.mode = MODE_Column; + modeChange(&data, MODE_Column); }else if( cli_strcmp(z,"-json")==0 ){ - data.mode = MODE_Json; + modeChange(&data, MODE_Json); }else if( cli_strcmp(z,"-markdown")==0 ){ - data.mode = MODE_Markdown; + modeChange(&data, MODE_Markdown); }else if( cli_strcmp(z,"-table")==0 ){ - data.mode = MODE_Table; + modeChange(&data, MODE_Table); + }else if( cli_strcmp(z,"-psql")==0 ){ + modeChange(&data, MODE_Psql); }else if( cli_strcmp(z,"-box")==0 ){ - data.mode = MODE_Box; + modeChange(&data, MODE_Box); }else if( cli_strcmp(z,"-csv")==0 ){ - data.mode = MODE_Csv; - memcpy(data.colSeparator,",",2); + modeChange(&data, MODE_Csv); }else if( cli_strcmp(z,"-escape")==0 && i+1<argc ){ /* See similar code at tag-20250224-1 */ const char *zEsc = argv[++i]; int k; - for(k=0; k<ArraySize(shell_EscModeNames); k++){ - if( sqlite3_stricmp(zEsc,shell_EscModeNames[k])==0 ){ - data.eEscMode = k; + for(k=0; k<ArraySize(qrfEscNames); k++){ + if( sqlite3_stricmp(zEsc,qrfEscNames[k])==0 ){ + data.mode.spec.eEsc = k; break; } } - if( k>=ArraySize(shell_EscModeNames) ){ - sqlite3_fprintf(stderr, "unknown control character escape mode \"%s\"" + if( k>=ArraySize(qrfEscNames) ){ + cli_printf(stderr, "unknown control character escape mode \"%s\"" " - choices:", zEsc); - for(k=0; k<ArraySize(shell_EscModeNames); k++){ - sqlite3_fprintf(stderr, " %s", shell_EscModeNames[k]); + for(k=0; k<ArraySize(qrfEscNames); k++){ + cli_printf(stderr, " %s", qrfEscNames[k]); } - sqlite3_fprintf(stderr, "\n"); + cli_printf(stderr, "\n"); exit(1); } #ifdef SQLITE_HAVE_ZLIB @@ -13686,44 +13262,40 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ data.openFlags |= SQLITE_OPEN_READONLY; }else if( cli_strcmp(z,"-nofollow")==0 ){ data.openFlags |= SQLITE_OPEN_NOFOLLOW; + }else if( cli_strcmp(z,"-noinit")==0 ){ + /* No-op */ }else if( cli_strcmp(z,"-exclusive")==0 ){ /* UNDOCUMENTED */ data.openFlags |= SQLITE_OPEN_EXCLUSIVE; }else if( cli_strcmp(z,"-ifexists")==0 ){ data.openFlags &= ~(SQLITE_OPEN_CREATE); if( data.openFlags==0 ) data.openFlags = SQLITE_OPEN_READWRITE; }else if( cli_strcmp(z,"-ascii")==0 ){ - data.mode = MODE_Ascii; - sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,SEP_Unit); - sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator,SEP_Record); + modeChange(&data, MODE_Ascii); }else if( cli_strcmp(z,"-tabs")==0 ){ - data.mode = MODE_List; - sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,SEP_Tab); - sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator,SEP_Row); + modeChange(&data, MODE_Tabs); }else if( cli_strcmp(z,"-separator")==0 ){ - sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator, - "%s",cmdline_option_value(argc,argv,++i)); + modeSetStr(&data.mode.spec.zColumnSep, + cmdline_option_value(argc,argv,++i)); }else if( cli_strcmp(z,"-newline")==0 ){ - sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator, - "%s",cmdline_option_value(argc,argv,++i)); + modeSetStr(&data.mode.spec.zRowSep, + cmdline_option_value(argc,argv,++i)); }else if( cli_strcmp(z,"-nullvalue")==0 ){ - sqlite3_snprintf(sizeof(data.nullValue), data.nullValue, - "%s",cmdline_option_value(argc,argv,++i)); + modeSetStr(&data.mode.spec.zNull, + cmdline_option_value(argc,argv,++i)); }else if( cli_strcmp(z,"-header")==0 ){ - data.showHeader = 1; - ShellSetFlag(&data, SHFLG_HeaderSet); + data.mode.spec.bTitles = QRF_Yes; }else if( cli_strcmp(z,"-noheader")==0 ){ - data.showHeader = 0; - ShellSetFlag(&data, SHFLG_HeaderSet); + data.mode.spec.bTitles = QRF_No; }else if( cli_strcmp(z,"-echo")==0 ){ - ShellSetFlag(&data, SHFLG_Echo); + data.mode.mFlags |= MFLG_ECHO; }else if( cli_strcmp(z,"-eqp")==0 ){ - data.autoEQP = AUTOEQP_on; + data.mode.autoEQP = AUTOEQP_on; }else if( cli_strcmp(z,"-eqpfull")==0 ){ - data.autoEQP = AUTOEQP_full; + data.mode.autoEQP = AUTOEQP_full; }else if( cli_strcmp(z,"-stats")==0 ){ data.statsOn = 1; }else if( cli_strcmp(z,"-scanstats")==0 ){ - data.scanstatsOn = 1; + data.mode.scanstatsOn = 1; }else if( cli_strcmp(z,"-backslash")==0 ){ /* Undocumented command-line option: -backslash ** Causes C-style backslash escapes to be evaluated in SQL statements @@ -13734,9 +13306,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( cli_strcmp(z,"-bail")==0 ){ /* No-op. The bail_on_error flag should already be set. */ }else if( cli_strcmp(z,"-version")==0 ){ - sqlite3_fprintf(stdout, "%s %s (%d-bit)\n", + cli_printf(stdout, "%s %s (%d-bit)\n", sqlite3_libversion(), sqlite3_sourceid(), 8*(int)sizeof(char*)); - return 0; + rc = 0; + goto shell_main_exit; }else if( cli_strcmp(z,"-interactive")==0 ){ /* Need to check for interactive override here to so that it can ** affect console setup (for Windows only) and testing thereof. @@ -13744,6 +13317,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ stdin_is_interactive = 1; }else if( cli_strcmp(z,"-batch")==0 ){ /* already handled */ + }else if( cli_strcmp(z,"-screenwidth")==0 ){ + i++; }else if( cli_strcmp(z,"-utf8")==0 ){ /* already handled */ }else if( cli_strcmp(z,"-no-utf8")==0 ){ @@ -13789,23 +13364,26 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ z = cmdline_option_value(argc,argv,++i); if( z[0]=='.' ){ rc = do_meta_command(z, &data); - if( rc && bail_on_error ) return rc==2 ? 0 : rc; + if( rc && (bail_on_error || rc==2) ){ + if( rc==2 ) rc = 0; + goto shell_main_exit; + } }else{ open_db(&data, 0); rc = shell_exec(&data, z, &zErrMsg); if( zErrMsg!=0 ){ shellEmitError(zErrMsg); sqlite3_free(zErrMsg); - if( bail_on_error ) return rc!=0 ? rc : 1; + if( !rc ) rc = 1; }else if( rc!=0 ){ - sqlite3_fprintf(stderr,"Error: unable to process SQL \"%s\"\n", z); - if( bail_on_error ) return rc; + cli_printf(stderr,"Error: unable to process SQL \"%s\"\n", z); } + if( bail_on_error ) goto shell_main_exit; } #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) }else if( cli_strncmp(z, "-A", 2)==0 ){ if( nCmd>0 ){ - sqlite3_fprintf(stderr,"Error: cannot mix regular SQL or dot-commands" + cli_printf(stderr,"Error: cannot mix regular SQL or dot-commands" " with \"%s\"\n", z); rc = 1; goto shell_main_exit; @@ -13818,6 +13396,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ arDotCommand(&data, 1, argv+i, argc-i); } readStdin = 0; + stdin_is_interactive = 0; break; #endif }else if( cli_strcmp(z,"-safe")==0 ){ @@ -13825,11 +13404,11 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( cli_strcmp(z,"-unsafe-testing")==0 ){ /* Acted upon in first pass. */ }else{ - sqlite3_fprintf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); + cli_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); eputz("Use -help for a list of options.\n"); - return 1; + rc = 1; + goto shell_main_exit; } - data.cMode = data.mode; } if( !readStdin ){ @@ -13839,8 +13418,26 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ */ for(i=0; i<nCmd; i++){ echo_group_input(&data, azCmd[i]); - if( azCmd[i][0]=='.' ){ + if( isScriptFile(azCmd[i],0) ){ + FILE *inSaved = data.in; + i64 savedLineno = data.lineno; + int res = 1; + if( (data.in = openChrSource(azCmd[i]))!=0 ){ + res = process_input(&data, azCmd[i]); + fclose(data.in); + } + data.in = inSaved; + data.lineno = savedLineno; + if( res ) i = nCmd; + }else if( azCmd[i][0]=='.' ){ + char *zErrCtx = malloc( 64 ); + shell_check_oom(zErrCtx); + sqlite3_snprintf(64,zErrCtx,"argv[%i]:",aiCmd[i]); + data.zInFile = "<cmdline>"; + data.zErrPrefix = zErrCtx; rc = do_meta_command(azCmd[i], &data); + free(data.zErrPrefix); + data.zErrPrefix = 0; if( rc ){ if( rc==2 ) rc = 0; goto shell_main_exit; @@ -13852,13 +13449,23 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( zErrMsg!=0 ){ shellEmitError(zErrMsg); }else{ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Error: unable to process SQL: %s\n", azCmd[i]); } sqlite3_free(zErrMsg); if( rc==0 ) rc = 1; goto shell_main_exit; } + if( data.nPopMode ){ + modePop(&data); + data.nPopMode = 0; + } + } + if( data.nPopOutput ){ + output_reset(&data); + data.nPopOutput = 0; + }else{ + clearTempFile(&data); } } }else{ @@ -13867,7 +13474,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( stdin_is_interactive ){ char *zHome; char *zHistory; - sqlite3_fprintf(stdout, + cli_printf(stdout, "SQLite version %s %.19s\n" /*extra-version-info*/ "Enter \".help\" for usage hints.\n", sqlite3_libversion(), sqlite3_sourceid()); @@ -13920,6 +13527,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #endif shell_main_exit: free(azCmd); + free(aiCmd); set_table_name(&data, 0); if( data.db ){ session_close_all(&data, -1); @@ -13936,12 +13544,28 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ output_reset(&data); data.doXdgOpen = 0; clearTempFile(&data); -#if !SQLITE_SHELL_IS_UTF8 - for(i=0; i<argcToFree; i++) free(argvToFree[i]); - free(argvToFree); -#endif - free(data.colWidth); + modeFree(&data.mode); + if( data.nSavedModes ){ + int ii; + for(ii=0; ii<data.nSavedModes; ii++){ + modeFree(&data.aSavedModes[ii].mode); + free(data.aSavedModes[ii].zTag); + } + free(data.aSavedModes); + } + free(data.zErrPrefix); free(data.zNonce); + free(data.dot.zCopy); + free(data.dot.azArg); + free(data.dot.aiOfst); + free(data.dot.abQuot); + if( data.nTestRun ){ + sqlite3_fprintf(stdout, "%d test%s run with %d error%s\n", + data.nTestRun, data.nTestRun==1 ? "" : "s", + data.nTestErr, data.nTestErr==1 ? "" : "s"); + fflush(stdout); + rc = data.nTestErr>0; + } /* Clear the global data structure so that valgrind will detect memory ** leaks */ memset(&data, 0, sizeof(data)); @@ -13950,7 +13574,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } #ifdef SQLITE_DEBUG if( sqlite3_memory_used()>mem_main_enter ){ - sqlite3_fprintf(stderr,"Memory leaked: %u bytes\n", + cli_printf(stderr,"Memory leaked: %u bytes\n", (unsigned int)(sqlite3_memory_used()-mem_main_enter)); } #endif @@ -13990,7 +13614,7 @@ sqlite3_vfs * fiddle_db_vfs(const char *zDbName){ /* Only for emcc experimentation purposes. */ sqlite3 * fiddle_db_arg(sqlite3 *arg){ - sqlite3_fprintf(stdout, "fiddle_db_arg(%p)\n", (const void*)arg); + cli_printf(stdout, "fiddle_db_arg(%p)\n", (const void*)arg); return arg; } @@ -14027,7 +13651,7 @@ void fiddle_reset_db(void){ ** Resolve problem reported in ** https://sqlite.org/forum/forumpost/0b41a25d65 */ - sqlite3_fputs("Rolling back in-progress transaction.\n", stdout); + cli_puts("Rolling back in-progress transaction.\n", stdout); sqlite3_exec(globalDb,"ROLLBACK", 0, 0, 0); } rc = sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index f6ed48c20..b6773e1d9 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1490,7 +1490,7 @@ typedef const char *sqlite3_filename; ** greater and the function pointer is not NULL) and will fall back ** to xCurrentTime() if xCurrentTimeInt64() is unavailable. ** -** ^The xSetSystemCall(), xGetSystemCall(), and xNestSystemCall() interfaces +** ^The xSetSystemCall(), xGetSystemCall(), and xNextSystemCall() interfaces ** are not used by the SQLite core. These optional interfaces are provided ** by some VFSes to facilitate testing of the VFS code. By overriding ** system calls with functions under its control, a test program can @@ -2567,12 +2567,15 @@ struct sqlite3_mem_methods { ** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]] ** <dt>SQLITE_DBCONFIG_STMT_SCANSTATUS</dt> ** <dd>The SQLITE_DBCONFIG_STMT_SCANSTATUS option is only useful in -** SQLITE_ENABLE_STMT_SCANSTATUS builds. In this case, it sets or clears -** a flag that enables collection of the sqlite3_stmt_scanstatus_v2() -** statistics. For statistics to be collected, the flag must be set on -** the database handle both when the SQL statement is prepared and when it -** is stepped. The flag is set (collection of statistics is enabled) -** by default. <p>This option takes two arguments: an integer and a pointer to +** [SQLITE_ENABLE_STMT_SCANSTATUS] builds. In this case, it sets or clears +** a flag that enables collection of run-time performance statistics +** used by [sqlite3_stmt_scanstatus_v2()] and the [nexec and ncycle] +** columns of the [bytecode virtual table]. +** For statistics to be collected, the flag must be set on +** the database handle both when the SQL statement is +** [sqlite3_prepare|prepared] and when it is [sqlite3_step|stepped]. +** The flag is set (collection of statistics is enabled) by default. +** <p>This option takes two arguments: an integer and a pointer to ** an integer. The first argument is 1, 0, or -1 to enable, disable, or ** leave unchanged the statement scanstatus option. If the second argument ** is not NULL, then the value of the statement scanstatus setting after @@ -2645,6 +2648,22 @@ struct sqlite3_mem_methods { ** comments are allowed in SQL text after processing the first argument. ** </dd> ** +** [[SQLITE_DBCONFIG_FP_DIGITS]] +** <dt>SQLITE_DBCONFIG_FP_DIGITS</dt> +** <dd>The SQLITE_DBCONFIG_FP_DIGITS setting is a small integer that determines +** the number of significant digits that SQLite will attempt to preserve when +** converting floating point numbers (IEEE 754 "doubles") into text. The +** default value 17, as of SQLite version 3.52.0. The value was 15 in all +** prior versions.<p> +** This option takes two arguments which are an integer and a pointer +** to an integer. The first argument is a small integer, between 3 and 23, or +** zero. The FP_DIGITS setting is changed to that small integer, or left +** altered if the first argument is zero or out of range. The second argument +** is a pointer to an integer. If the pointer is not NULL, then the value of +** the FP_DIGITS setting, after possibly being modified by the first +** arguments, is written into the integer to which the second argument points. +** </dd> +** ** </dl> ** ** [[DBCONFIG arguments]] <h3>Arguments To SQLITE_DBCONFIG Options</h3> @@ -2662,9 +2681,10 @@ struct sqlite3_mem_methods { ** the first argument. ** ** <p>While most SQLITE_DBCONFIG options use the argument format -** described in the previous paragraph, the [SQLITE_DBCONFIG_MAINDBNAME] -** and [SQLITE_DBCONFIG_LOOKASIDE] options are different. See the -** documentation of those exceptional options for details. +** described in the previous paragraph, the [SQLITE_DBCONFIG_MAINDBNAME], +** [SQLITE_DBCONFIG_LOOKASIDE], and [SQLITE_DBCONFIG_FP_DIGITS] options +** are different. See the documentation of those exceptional options for +** details. */ #define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ #define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ @@ -2689,7 +2709,8 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE 1020 /* int int* */ #define SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE 1021 /* int int* */ #define SQLITE_DBCONFIG_ENABLE_COMMENTS 1022 /* int int* */ -#define SQLITE_DBCONFIG_MAX 1022 /* Largest DBCONFIG */ +#define SQLITE_DBCONFIG_FP_DIGITS 1023 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1023 /* Largest DBCONFIG */ /* ** CAPI3REF: Enable Or Disable Extended Result Codes @@ -4171,6 +4192,7 @@ void sqlite3_free_filename(sqlite3_filename); ** <li> sqlite3_errmsg() ** <li> sqlite3_errmsg16() ** <li> sqlite3_error_offset() +** <li> sqlite3_db_handle() ** </ul> ** ** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language @@ -4217,7 +4239,7 @@ const char *sqlite3_errstr(int); int sqlite3_error_offset(sqlite3 *db); /* -** CAPI3REF: Set Error Codes And Message +** CAPI3REF: Set Error Code And Message ** METHOD: sqlite3 ** ** Set the error code of the database handle passed as the first argument @@ -4336,6 +4358,10 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** [[SQLITE_LIMIT_EXPR_DEPTH]] ^(<dt>SQLITE_LIMIT_EXPR_DEPTH</dt> ** <dd>The maximum depth of the parse tree on any expression.</dd>)^ ** +** [[SQLITE_LIMIT_PARSER_DEPTH]] ^(<dt>SQLITE_LIMIT_PARSER_DEPTH</dt> +** <dd>The maximum depth of the LALR(1) parser stack used to analyze +** input SQL statements.</dd>)^ +** ** [[SQLITE_LIMIT_COMPOUND_SELECT]] ^(<dt>SQLITE_LIMIT_COMPOUND_SELECT</dt> ** <dd>The maximum number of terms in a compound SELECT statement.</dd>)^ ** @@ -4380,6 +4406,7 @@ int sqlite3_limit(sqlite3*, int id, int newVal); #define SQLITE_LIMIT_VARIABLE_NUMBER 9 #define SQLITE_LIMIT_TRIGGER_DEPTH 10 #define SQLITE_LIMIT_WORKER_THREADS 11 +#define SQLITE_LIMIT_PARSER_DEPTH 12 /* ** CAPI3REF: Prepare Flags @@ -4424,12 +4451,29 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** fails, the sqlite3_prepare_v3() call returns the same error indications ** with or without this flag; it just omits the call to [sqlite3_log()] that ** logs the error. +** +** [[SQLITE_PREPARE_FROM_DDL]] <dt>SQLITE_PREPARE_FROM_DDL</dt> +** <dd>The SQLITE_PREPARE_FROM_DDL flag causes the SQL compiler to enforce +** security constraints that would otherwise only be enforced when parsing +** the database schema. In other words, the SQLITE_PREPARE_FROM_DDL flag +** causes the SQL compiler to treat the SQL statement being prepared as if +** it had come from an attacker. When SQLITE_PREPARE_FROM_DDL is used and +** [SQLITE_DBCONFIG_TRUSTED_SCHEMA] is off, SQL functions may only be called +** if they are tagged with [SQLITE_INNOCUOUS] and virtual tables may only +** be used if they are tagged with [SQLITE_VTAB_INNOCUOUS]. Best practice +** is to use the SQLITE_PREPARE_FROM_DDL option when preparing any SQL that +** is derived from parts of the database schema. In particular, virtual +** table implementations that run SQL statements that are derived from +** arguments to their CREATE VIRTUAL TABLE statement should always use +** [sqlite3_prepare_v3()] and set the SQLITE_PREPARE_FROM_DDL flag to +** prevent bypass of the [SQLITE_DBCONFIG_TRUSTED_SCHEMA] security checks. ** </dl> */ #define SQLITE_PREPARE_PERSISTENT 0x01 #define SQLITE_PREPARE_NORMALIZE 0x02 #define SQLITE_PREPARE_NO_VTAB 0x04 #define SQLITE_PREPARE_DONT_LOG 0x10 +#define SQLITE_PREPARE_FROM_DDL 0x20 /* ** CAPI3REF: Compiling An SQL Statement @@ -4443,8 +4487,9 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** ** The preferred routine to use is [sqlite3_prepare_v2()]. The ** [sqlite3_prepare()] interface is legacy and should be avoided. -** [sqlite3_prepare_v3()] has an extra "prepFlags" option that is used -** for special purposes. +** [sqlite3_prepare_v3()] has an extra +** [SQLITE_PREPARE_FROM_DDL|"prepFlags" option] that is some times +** needed for special purpose or to pass along security restrictions. ** ** The use of the UTF-8 interfaces is preferred, as SQLite currently ** does all parsing using UTF-8. The UTF-16 interfaces are provided @@ -4849,8 +4894,8 @@ typedef struct sqlite3_context sqlite3_context; ** it should be a pointer to well-formed UTF16 text. ** ^If the third parameter to sqlite3_bind_text64() is not NULL, then ** it should be a pointer to a well-formed unicode string that is -** either UTF8 if the sixth parameter is SQLITE_UTF8, or UTF16 -** otherwise. +** either UTF8 if the sixth parameter is SQLITE_UTF8 or SQLITE_UTF8_ZT, +** or UTF16 otherwise. ** ** [[byte-order determination rules]] ^The byte-order of ** UTF16 input text is determined by the byte-order mark (BOM, U+FEFF) @@ -4896,10 +4941,15 @@ typedef struct sqlite3_context sqlite3_context; ** object and pointer to it must remain valid until then. ^SQLite will then ** manage the lifetime of its private copy. ** -** ^The sixth argument to sqlite3_bind_text64() must be one of -** [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE] -** to specify the encoding of the text in the third parameter. If -** the sixth argument to sqlite3_bind_text64() is not one of the +** ^The sixth argument (the E argument) +** to sqlite3_bind_text64(S,K,Z,N,D,E) must be one of +** [SQLITE_UTF8], [SQLITE_UTF8_ZT], [SQLITE_UTF16], [SQLITE_UTF16BE], +** or [SQLITE_UTF16LE] to specify the encoding of the text in the +** third parameter, Z. The special value [SQLITE_UTF8_ZT] means that the +** string argument is both UTF-8 encoded and is zero-terminated. In other +** words, SQLITE_UTF8_ZT means that the Z array is allocated to hold at +** least N+1 bytes and that the Z&#91;N&#93; byte is zero. If +** the E argument to sqlite3_bind_text64(S,K,Z,N,D,E) is not one of the ** allowed values shown above, or if the text encoding is different ** from the encoding specified by the sixth parameter, then the behavior ** is undefined. @@ -5766,6 +5816,51 @@ int sqlite3_create_window_function( ** ** These constants define integer codes that represent the various ** text encodings supported by SQLite. +** +** <dl> +** [[SQLITE_UTF8]] <dt>SQLITE_UTF8</dt><dd>Text is encoding as UTF-8</dd> +** +** [[SQLITE_UTF16LE]] <dt>SQLITE_UTF16LE</dt><dd>Text is encoding as UTF-16 +** with each code point being expressed "little endian" - the least significant +** byte first. This is the usual encoding, for example on Windows.</dd> +** +** [[SQLITE_UTF16BE]] <dt>SQLITE_UTF16BE</dt><dd>Text is encoding as UTF-16 +** with each code point being expressed "big endian" - the most significant +** byte first. This encoding is less common, but is still sometimes seen, +** specially on older systems. +** +** [[SQLITE_UTF16]] <dt>SQLITE_UTF16</dt><dd>Text is encoding as UTF-16 +** with each code point being expressed either little endian or as big +** endian, according to the native endianness of the host computer. +** +** [[SQLITE_ANY]] <dt>SQLITE_ANY</dt><dd>This encoding value may only be used +** to declare the preferred text for [application-defined SQL functions] +** created using [sqlite3_create_function()] and similar. If the preferred +** encoding (the 4th parameter to sqlite3_create_function() - the eTextRep +** parameter) is SQLITE_ANY, that indicates that the function does not have +** a preference regarding the text encoding of its parameters and can take +** any text encoding that the SQLite core find convenient to supply. This +** option is deprecated. Please do not use it in new applications. +** +** [[SQLITE_UTF16_ALIGNED]] <dt>SQLITE_UTF16_ALIGNED</dt><dd>This encoding +** value may be used as the 3rd parameter (the eTextRep parameter) to +** [sqlite3_create_collation()] and similar. This encoding value means +** that the application-defined collating sequence created expects its +** input strings to be in UTF16 in native byte order, and that the start +** of the strings must be aligned to a 2-byte boundary. +** +** [[SQLITE_UTF8_ZT]] <dt>SQLITE_UTF8_ZT</dt><dd>This option can only be +** used to specify the text encoding to strings input to [sqlite3_result_text64()] +** and [sqlite3_bind_text64()]. It means that the input string (call it "z") +** is UTF-8 encoded and that it is zero-terminated. If the length parameter +** (call it "n") is non-negative, this encoding option means that the caller +** guarantees that z array contains at least n+1 bytes and that the z&#91;n&#93; +** byte has a value of zero. +** This option gives the same output as SQLITE_UTF8, but can be more efficient +** by avoiding the need to make a copy of the input string, in some cases. +** However, if z is allocated to hold fewer than n+1 bytes or if the +** z&#91;n&#93; byte is not zero, undefined behavior may result. +** </dl> */ #define SQLITE_UTF8 1 /* IMP: R-37514-35566 */ #define SQLITE_UTF16LE 2 /* IMP: R-03371-37637 */ @@ -5773,6 +5868,7 @@ int sqlite3_create_window_function( #define SQLITE_UTF16 4 /* Use native byte order */ #define SQLITE_ANY 5 /* Deprecated */ #define SQLITE_UTF16_ALIGNED 8 /* sqlite3_create_collation only */ +#define SQLITE_UTF8_ZT 16 /* Zero-terminated UTF8 */ /* ** CAPI3REF: Function Flags @@ -6278,10 +6374,14 @@ void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); ** ** There is no limit (other than available memory) on the number of different ** client data pointers (with different names) that can be attached to a -** single database connection. However, the implementation is optimized -** for the case of having only one or two different client data names. -** Applications and wrapper libraries are discouraged from using more than -** one client data name each. +** single database connection. However, the current implementation stores +** the content on a linked list. Insert and retrieval performance will +** be proportional to the number of entries. The design use case, and +** the use case for which the implementation is optimized, is +** that an application will store only small number of client data names, +** typically just one or two. This interface is not intended to be a +** generalized key/value store for thousands or millions of keys. It +** will work for that, but performance might be disappointing. ** ** There is no way to enumerate the client data pointers ** associated with a database connection. The N parameter can be thought @@ -6389,10 +6489,14 @@ typedef void (*sqlite3_destructor_type)(void*); ** set the return value of the application-defined function to be ** a text string which is represented as UTF-8, UTF-16 native byte order, ** UTF-16 little endian, or UTF-16 big endian, respectively. -** ^The sqlite3_result_text64() interface sets the return value of an +** ^The sqlite3_result_text64(C,Z,N,D,E) interface sets the return value of an ** application-defined function to be a text string in an encoding -** specified by the fifth (and last) parameter, which must be one -** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]. +** specified the E parameter, which must be one +** of [SQLITE_UTF8], [SQLITE_UTF8_ZT], [SQLITE_UTF16], [SQLITE_UTF16BE], +** or [SQLITE_UTF16LE]. ^The special value [SQLITE_UTF8_ZT] means that +** the result text is both UTF-8 and zero-terminated. In other words, +** SQLITE_UTF8_ZT means that the Z array holds at least N+1 byes and that +** the Z&#91;N&#93; is zero. ** ^SQLite takes the text result from the application from ** the 2nd parameter of the sqlite3_result_text* interfaces. ** ^If the 3rd parameter to any of the sqlite3_result_text* interfaces @@ -6479,7 +6583,7 @@ void sqlite3_result_int(sqlite3_context*, int); void sqlite3_result_int64(sqlite3_context*, sqlite3_int64); void sqlite3_result_null(sqlite3_context*); void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); -void sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64, +void sqlite3_result_text64(sqlite3_context*, const char *z, sqlite3_uint64 n, void(*)(void*), unsigned char encoding); void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); @@ -7418,7 +7522,7 @@ int sqlite3_table_column_metadata( ** ^The sqlite3_load_extension() interface attempts to load an ** [SQLite extension] library contained in the file zFile. If ** the file cannot be loaded directly, attempts are made to load -** with various operating-system specific extensions added. +** with various operating-system specific filename extensions added. ** So for example, if "samplelib" cannot be loaded, then names like ** "samplelib.so" or "samplelib.dylib" or "samplelib.dll" might ** be tried also. @@ -7426,10 +7530,10 @@ int sqlite3_table_column_metadata( ** ^The entry point is zProc. ** ^(zProc may be 0, in which case SQLite will try to come up with an ** entry point name on its own. It first tries "sqlite3_extension_init". -** If that does not work, it constructs a name "sqlite3_X_init" where -** X consists of the lower-case equivalent of all ASCII alphabetic -** characters in the filename from the last "/" to the first following -** "." and omitting any initial "lib".)^ +** If that does not work, it tries names of the form "sqlite3_X_init" +** where X consists of the lower-case equivalent of all ASCII alphabetic +** characters or all ASCII alphanumeric characters in the filename from +** the last "/" to the first following "." and omitting any initial "lib".)^ ** ^The sqlite3_load_extension() interface returns ** [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. ** ^If an error occurs and pzErrMsg is not 0, then the @@ -8722,17 +8826,22 @@ sqlite3_str *sqlite3_str_new(sqlite3*); ** pass the returned value to [sqlite3_free()] to avoid a memory leak. ** ^The [sqlite3_str_finish(X)] interface may return a NULL pointer if any ** errors were encountered during construction of the string. ^The -** [sqlite3_str_finish(X)] interface will also return a NULL pointer if the +** [sqlite3_str_finish(X)] interface might also return a NULL pointer if the ** string in [sqlite3_str] object X is zero bytes long. +** +** ^The [sqlite3_str_free(X)] interface destroys both the sqlite3_str object +** X and the string content it contains. Calling sqlite3_str_free(X) is +** the equivalent of calling [sqlite3_free](sqlite3_str_finish(X)). */ char *sqlite3_str_finish(sqlite3_str*); +void sqlite3_str_free(sqlite3_str*); /* ** CAPI3REF: Add Content To A Dynamic String ** METHOD: sqlite3_str ** -** These interfaces add content to an sqlite3_str object previously obtained -** from [sqlite3_str_new()]. +** These interfaces add or remove content to an sqlite3_str object +** previously obtained from [sqlite3_str_new()]. ** ** ^The [sqlite3_str_appendf(X,F,...)] and ** [sqlite3_str_vappendf(X,F,V)] interfaces uses the [built-in printf] @@ -8755,6 +8864,10 @@ char *sqlite3_str_finish(sqlite3_str*); ** ^The [sqlite3_str_reset(X)] method resets the string under construction ** inside [sqlite3_str] object X back to zero bytes in length. ** +** ^The [sqlite3_str_truncate(X,N)] method changes the length of the string +** under construction to be N bytes are less. This routine is a no-op if +** N is negative or if the string is already N bytes or smaller in size. +** ** These methods do not return a result code. ^If an error occurs, that fact ** is recorded in the [sqlite3_str] object and can be recovered by a ** subsequent call to [sqlite3_str_errcode(X)]. @@ -8765,6 +8878,7 @@ void sqlite3_str_append(sqlite3_str*, const char *zIn, int N); void sqlite3_str_appendall(sqlite3_str*, const char *zIn); void sqlite3_str_appendchar(sqlite3_str*, int N, char C); void sqlite3_str_reset(sqlite3_str*); +void sqlite3_str_truncate(sqlite3_str*,int N); /* ** CAPI3REF: Status Of A Dynamic String @@ -10598,9 +10712,9 @@ int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); ** a variable pointed to by the "pOut" parameter. ** ** The "flags" parameter must be passed a mask of flags. At present only -** one flag is defined - SQLITE_SCANSTAT_COMPLEX. If SQLITE_SCANSTAT_COMPLEX +** one flag is defined - [SQLITE_SCANSTAT_COMPLEX]. If SQLITE_SCANSTAT_COMPLEX ** is specified, then status information is available for all elements -** of a query plan that are reported by "EXPLAIN QUERY PLAN" output. If +** of a query plan that are reported by "[EXPLAIN QUERY PLAN]" output. If ** SQLITE_SCANSTAT_COMPLEX is not specified, then only query plan elements ** that correspond to query loops (the "SCAN..." and "SEARCH..." elements of ** the EXPLAIN QUERY PLAN output) are available. Invoking API @@ -10614,7 +10728,8 @@ int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); ** elements used to implement the statement - a non-zero value is returned and ** the variable that pOut points to is unchanged. ** -** See also: [sqlite3_stmt_scanstatus_reset()] +** See also: [sqlite3_stmt_scanstatus_reset()] and the +** [nexec and ncycle] columnes of the [bytecode virtual table]. */ int sqlite3_stmt_scanstatus( sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ @@ -11156,19 +11271,41 @@ int sqlite3_deserialize( /* ** CAPI3REF: Bind array values to the CARRAY table-valued function ** -** The sqlite3_carray_bind(S,I,P,N,F,X) interface binds an array value to -** one of the first argument of the [carray() table-valued function]. The -** S parameter is a pointer to the [prepared statement] that uses the carray() -** functions. I is the parameter index to be bound. P is a pointer to the -** array to be bound, and N is the number of eements in the array. The -** F argument is one of constants [SQLITE_CARRAY_INT32], [SQLITE_CARRAY_INT64], -** [SQLITE_CARRAY_DOUBLE], [SQLITE_CARRAY_TEXT], or [SQLITE_CARRAY_BLOB] to -** indicate the datatype of the array being bound. The X argument is not a -** NULL pointer, then SQLite will invoke the function X on the P parameter -** after it has finished using P, even if the call to -** sqlite3_carray_bind() fails. The special-case finalizer -** SQLITE_TRANSIENT has no effect here. -*/ +** The sqlite3_carray_bind_v2(S,I,P,N,F,X,D) interface binds an array value to +** parameter that is the first argument of the [carray() table-valued function]. +** The S parameter is a pointer to the [prepared statement] that uses the carray() +** functions. I is the parameter index to be bound. I must be the index of the +** parameter that is the first argument to the carray() table-valued function. +** P is a pointer to the array to be bound, and N is the number of elements in +** the array. The F argument is one of constants [SQLITE_CARRAY_INT32], +** [SQLITE_CARRAY_INT64], [SQLITE_CARRAY_DOUBLE], [SQLITE_CARRAY_TEXT], +** or [SQLITE_CARRAY_BLOB] to indicate the datatype of the array P. +** +** If the X argument is not a NULL pointer or one of the special +** values [SQLITE_STATIC] or [SQLITE_TRANSIENT], then SQLite will invoke +** the function X with argument D when it is finished using the data in P. +** The call to X(D) is a destructor for the array P. The destructor X(D) +** is invoked even if the call to sqlite3_carray_bind() fails. If the X +** parameter is the special-case value [SQLITE_STATIC], then SQLite assumes +** that the data static and the destructor is never invoked. If the X +** parameter is the special-case value [SQLITE_TRANSIENT], then +** sqlite3_carray_bind_v2() makes its own private copy of the data prior +** to returning and never invokes the destructor X. +** +** The sqlite3_carray_bind() function works the same as sqlite_carray_bind_v2() +** with a D parameter set to P. In other words, +** sqlite3_carray_bind(S,I,P,N,F,X) is same as +** sqlite3_carray_bind(S,I,P,N,F,X,P). +*/ +int sqlite3_carray_bind_v2( + sqlite3_stmt *pStmt, /* Statement to be bound */ + int i, /* Parameter index */ + void *aData, /* Pointer to array data */ + int nData, /* Number of data elements */ + int mFlags, /* CARRAY flags */ + void (*xDel)(void*), /* Destructor for aData */ + void *pDel /* Optional argument to xDel() */ +); int sqlite3_carray_bind( sqlite3_stmt *pStmt, /* Statement to be bound */ int i, /* Parameter index */ diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index 5258faaed..cad1a2a00 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -371,7 +371,11 @@ struct sqlite3_api_routines { /* Version 3.51.0 and later */ int (*set_errmsg)(sqlite3*,int,const char*); int (*db_status64)(sqlite3*,int,sqlite3_int64*,sqlite3_int64*,int); - + /* Version 3.52.0 and later */ + void (*str_truncate)(sqlite3_str*,int); + void (*str_free)(sqlite3_str*); + int (*carray_bind)(sqlite3_stmt*,int,void*,int,int,void(*)(void*)); + int (*carray_bind_v2)(sqlite3_stmt*,int,void*,int,int,void(*)(void*),void*); }; /* @@ -710,6 +714,11 @@ typedef int (*sqlite3_loadext_entry)( /* Version 3.51.0 and later */ #define sqlite3_set_errmsg sqlite3_api->set_errmsg #define sqlite3_db_status64 sqlite3_api->db_status64 +/* Version 3.52.0 and later */ +#define sqlite3_str_truncate sqlite3_api->str_truncate +#define sqlite3_str_free sqlite3_api->str_free +#define sqlite3_carray_bind sqlite3_api->carray_bind +#define sqlite3_carray_bind_v2 sqlite3_api->carray_bind_v2 #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 523bcfb3b..c4a876861 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -661,6 +661,7 @@ # define float sqlite_int64 # define fabs(X) ((X)<0?-(X):(X)) # define sqlite3IsOverflow(X) 0 +# define INFINITY (9223372036854775807LL) # ifndef SQLITE_BIG_DBL # define SQLITE_BIG_DBL (((sqlite3_int64)1)<<50) # endif @@ -1070,6 +1071,7 @@ typedef INT16_TYPE LogEst; #else # define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&7)==0) #endif +#define TWO_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&1)==0) /* ** Disable MMAP on platforms where it is known to not work @@ -1537,7 +1539,7 @@ struct Schema { ** The number of different kinds of things that can be limited ** using the sqlite3_limit() interface. */ -#define SQLITE_N_LIMIT (SQLITE_LIMIT_WORKER_THREADS+1) +#define SQLITE_N_LIMIT (SQLITE_LIMIT_PARSER_DEPTH+1) /* ** Lookaside malloc is a set of fixed-size buffers that can be used @@ -1691,6 +1693,7 @@ struct sqlite3 { u8 noSharedCache; /* True if no shared-cache backends */ u8 nSqlExec; /* Number of pending OP_SqlExec opcodes */ u8 eOpenState; /* Current condition of the connection */ + u8 nFpDigit; /* Significant digits to keep on double->text */ int nextPagesize; /* Pagesize after VACUUM if >0 */ i64 nChange; /* Value returned by sqlite3_changes() */ i64 nTotalChange; /* Value returned by sqlite3_total_changes() */ @@ -3585,19 +3588,6 @@ struct Upsert { /* ** An instance of the following structure contains all information ** needed to generate code for a single SELECT statement. -** -** See the header comment on the computeLimitRegisters() routine for a -** detailed description of the meaning of the iLimit and iOffset fields. -** -** addrOpenEphm[] entries contain the address of OP_OpenEphemeral opcodes. -** These addresses must be stored so that we can go back and fill in -** the P4_KEYINFO and P2 parameters later. Neither the KeyInfo nor -** the number of columns in P2 can be computed at the same time -** as the OP_OpenEphm instruction is coded because not -** enough information about the compound query is known at that point. -** The KeyInfo for addrOpenTran[0] and [1] contains collating sequences -** for the result set. The KeyInfo for addrOpenEphm[2] contains collating -** sequences for the ORDER BY clause. */ struct Select { u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ @@ -3605,7 +3595,6 @@ struct Select { u32 selFlags; /* Various SF_* values */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ u32 selId; /* Unique identifier number for this SELECT */ - int addrOpenEphm[2]; /* OP_OpenEphem opcodes related to this select */ ExprList *pEList; /* The fields of the result */ SrcList *pSrc; /* The FROM clause */ Expr *pWhere; /* The WHERE clause */ @@ -3637,7 +3626,7 @@ struct Select { #define SF_Resolved 0x0000004 /* Identifiers have been resolved */ #define SF_Aggregate 0x0000008 /* Contains agg functions or a GROUP BY */ #define SF_HasAgg 0x0000010 /* Contains aggregate functions */ -#define SF_UsesEphemeral 0x0000020 /* Uses the OpenEphemeral opcode */ +#define SF_ClonedRhsIn 0x0000020 /* Cloned RHS of an IN operator */ #define SF_Expanded 0x0000040 /* sqlite3SelectExpand() called on this */ #define SF_HasTypeInfo 0x0000080 /* FROM subqueries have Table metadata */ #define SF_Compound 0x0000100 /* Part of a compound query */ @@ -3647,14 +3636,14 @@ struct Select { #define SF_MinMaxAgg 0x0001000 /* Aggregate containing min() or max() */ #define SF_Recursive 0x0002000 /* The recursive part of a recursive CTE */ #define SF_FixedLimit 0x0004000 /* nSelectRow set by a constant LIMIT */ -#define SF_MaybeConvert 0x0008000 /* Need convertCompoundSelectToSubquery() */ +/* 0x0008000 // available for reuse */ #define SF_Converted 0x0010000 /* By convertCompoundSelectToSubquery() */ #define SF_IncludeHidden 0x0020000 /* Include hidden columns in output */ #define SF_ComplexResult 0x0040000 /* Result contains subquery or function */ #define SF_WhereBegin 0x0080000 /* Really a WhereBegin() call. Debug Only */ #define SF_WinRewrite 0x0100000 /* Window function rewrite accomplished */ #define SF_View 0x0200000 /* SELECT statement is a view */ -#define SF_NoopOrderBy 0x0400000 /* ORDER BY is ignored for this query */ +/* 0x0400000 // available for reuse */ #define SF_UFSrcCheck 0x0800000 /* Check pSrc as required by UPDATE...FROM */ #define SF_PushDown 0x1000000 /* Modified by WHERE-clause push-down opt */ #define SF_MultiPart 0x2000000 /* Has multiple incompatible PARTITIONs */ @@ -3674,11 +3663,6 @@ struct Select { ** by one of the following macros. The "SRT" prefix means "SELECT Result ** Type". ** -** SRT_Union Store results as a key in a temporary index -** identified by pDest->iSDParm. -** -** SRT_Except Remove results from the temporary index pDest->iSDParm. -** ** SRT_Exists Store a 1 in memory cell pDest->iSDParm if the result ** set is not empty. ** @@ -3742,30 +3726,28 @@ struct Select { ** table. (pDest->iSDParm) is the number of key columns in ** each index record in this case. */ -#define SRT_Union 1 /* Store result as keys in an index */ -#define SRT_Except 2 /* Remove result from a UNION index */ -#define SRT_Exists 3 /* Store 1 if the result is not empty */ -#define SRT_Discard 4 /* Do not save the results anywhere */ -#define SRT_DistFifo 5 /* Like SRT_Fifo, but unique results only */ -#define SRT_DistQueue 6 /* Like SRT_Queue, but unique results only */ +#define SRT_Exists 1 /* Store 1 if the result is not empty */ +#define SRT_Discard 2 /* Do not save the results anywhere */ +#define SRT_DistFifo 3 /* Like SRT_Fifo, but unique results only */ +#define SRT_DistQueue 4 /* Like SRT_Queue, but unique results only */ /* The DISTINCT clause is ignored for all of the above. Not that ** IgnorableDistinct() implies IgnorableOrderby() */ #define IgnorableDistinct(X) ((X->eDest)<=SRT_DistQueue) -#define SRT_Queue 7 /* Store result in an queue */ -#define SRT_Fifo 8 /* Store result as data with an automatic rowid */ +#define SRT_Queue 5 /* Store result in an queue */ +#define SRT_Fifo 6 /* Store result as data with an automatic rowid */ /* The ORDER BY clause is ignored for all of the above */ #define IgnorableOrderby(X) ((X->eDest)<=SRT_Fifo) -#define SRT_Output 9 /* Output each row of result */ -#define SRT_Mem 10 /* Store result in a memory cell */ -#define SRT_Set 11 /* Store results as keys in an index */ -#define SRT_EphemTab 12 /* Create transient tab and store like SRT_Table */ -#define SRT_Coroutine 13 /* Generate a single row of result */ -#define SRT_Table 14 /* Store result as data with an automatic rowid */ -#define SRT_Upfrom 15 /* Store result as data with rowid */ +#define SRT_Output 7 /* Output each row of result */ +#define SRT_Mem 8 /* Store result in a memory cell */ +#define SRT_Set 9 /* Store results as keys in an index */ +#define SRT_EphemTab 10 /* Create transient tab and store like SRT_Table */ +#define SRT_Coroutine 11 /* Generate a single row of result */ +#define SRT_Table 12 /* Store result as data with an automatic rowid */ +#define SRT_Upfrom 13 /* Store result as data with rowid */ /* ** An instance of this object describes where to put of the results of @@ -3901,17 +3883,12 @@ struct Parse { u8 nested; /* Number of nested calls to the parser/code generator */ u8 nTempReg; /* Number of temporary registers in aTempReg[] */ u8 isMultiWrite; /* True if statement may modify/insert multiple rows */ - u8 mayAbort; /* True if statement may throw an ABORT exception */ - u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ u8 disableLookaside; /* Number of times lookaside has been disabled */ u8 prepFlags; /* SQLITE_PREPARE_* flags */ u8 withinRJSubrtn; /* Nesting level for RIGHT JOIN body subroutines */ - u8 bHasExists; /* Has a correlated "EXISTS (SELECT ....)" expression */ u8 mSubrtnSig; /* mini Bloom filter on available SubrtnSig.selId */ u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ - u8 bReturning; /* Coding a RETURNING trigger */ u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ - u8 disableTriggers; /* True to disable triggers */ #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) u8 earlyCleanup; /* OOM inside sqlite3ParserAddCleanup() */ #endif @@ -3920,10 +3897,15 @@ struct Parse { u8 isCreate; /* CREATE TABLE, INDEX, or VIEW (but not TRIGGER) ** and ALTER TABLE ADD COLUMN. */ #endif - bft colNamesSet :1; /* TRUE after OP_ColumnName has been issued to pVdbe */ - bft bHasWith :1; /* True if statement contains WITH */ - bft okConstFactor :1; /* OK to factor out constants */ - bft checkSchema :1; /* Causes schema cookie check after an error */ + bft disableTriggers:1; /* True to disable triggers */ + bft mayAbort :1; /* True if statement may throw an ABORT exception */ + bft hasCompound :1; /* Need to invoke convertCompoundSelectToSubquery() */ + bft bReturning :1; /* Coding a RETURNING trigger */ + bft bHasExists :1; /* Has a correlated "EXISTS (SELECT ....)" expression */ + bft colNamesSet :1; /* TRUE after OP_ColumnName has been issued to pVdbe */ + bft bHasWith :1; /* True if statement contains WITH */ + bft okConstFactor:1; /* OK to factor out constants */ + bft checkSchema :1; /* Causes schema cookie check after an error */ int nRangeReg; /* Size of the temporary register block */ int iRangeReg; /* First register in temporary register block */ int nErr; /* Number of errors seen */ @@ -4152,19 +4134,19 @@ struct Trigger { ** orconf -> stores the ON CONFLICT algorithm ** pSelect -> The content to be inserted - either a SELECT statement or ** a VALUES clause. -** zTarget -> Dequoted name of the table to insert into. +** pSrc -> Table to insert into. ** pIdList -> If this is an INSERT INTO ... (<column-names>) VALUES ... ** statement, then this stores the column-names to be ** inserted into. ** pUpsert -> The ON CONFLICT clauses for an Upsert ** ** (op == TK_DELETE) -** zTarget -> Dequoted name of the table to delete from. +** pSrc -> Table to delete from ** pWhere -> The WHERE clause of the DELETE statement if one is specified. ** Otherwise NULL. ** ** (op == TK_UPDATE) -** zTarget -> Dequoted name of the table to update. +** pSrc -> Table to update, followed by any FROM clause tables. ** pWhere -> The WHERE clause of the UPDATE statement if one is specified. ** Otherwise NULL. ** pExprList -> A list of the columns to update and the expressions to update @@ -4184,8 +4166,7 @@ struct TriggerStep { u8 orconf; /* OE_Rollback etc. */ Trigger *pTrig; /* The trigger that this step is a part of */ Select *pSelect; /* SELECT statement or RHS of INSERT INTO SELECT ... */ - char *zTarget; /* Target table for DELETE, UPDATE, INSERT */ - SrcList *pFrom; /* FROM clause for UPDATE statement (if any) */ + SrcList *pSrc; /* Table to insert/update/delete */ Expr *pWhere; /* The WHERE clause for DELETE or UPDATE steps */ ExprList *pExprList; /* SET clause for UPDATE, or RETURNING clause */ IdList *pIdList; /* Column names for INSERT */ @@ -4268,10 +4249,11 @@ typedef struct { /* ** Allowed values for mInitFlags */ -#define INITFLAG_AlterMask 0x0003 /* Types of ALTER */ +#define INITFLAG_AlterMask 0x0007 /* Types of ALTER */ #define INITFLAG_AlterRename 0x0001 /* Reparse after a RENAME */ #define INITFLAG_AlterDrop 0x0002 /* Reparse after a DROP COLUMN */ #define INITFLAG_AlterAdd 0x0003 /* Reparse after an ADD COLUMN */ +#define INITFLAG_AlterDropCons 0x0004 /* Reparse after an ADD COLUMN */ /* Tuning parameters are set using SQLITE_TESTCTRL_TUNE and are controlled ** on debug-builds of the CLI using ".testctrl tune ID VALUE". Tuning @@ -4401,6 +4383,7 @@ struct Walker { NameContext *pNC; /* Naming context */ int n; /* A counter */ int iCur; /* A cursor number */ + int sz; /* String literal length */ SrcList *pSrcList; /* FROM clause */ struct CCurHint *pCCurHint; /* Used by codeCursorHint() */ struct RefSrcList *pRefSrcList; /* sqlite3ReferencesSrcList() */ @@ -4805,7 +4788,20 @@ int sqlite3LookasideUsed(sqlite3*,int*); sqlite3_mutex *sqlite3Pcache1Mutex(void); sqlite3_mutex *sqlite3MallocMutex(void); -#if defined(SQLITE_ENABLE_MULTITHREADED_CHECKS) && !defined(SQLITE_MUTEX_OMIT) + +/* The SQLITE_THREAD_MISUSE_WARNINGS compile-time option used to be called +** SQLITE_ENABLE_MULTITHREADED_CHECKS. Keep that older macro for backwards +** compatibility, at least for a while... */ +#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS +# define SQLITE_THREAD_MISUSE_WARNINGS 1 +#endif + +/* SQLITE_THREAD_MISUSE_ABORT implies SQLITE_THREAD_MISUSE_WARNINGS */ +#ifdef SQLITE_THREAD_MISUSE_ABORT +# define SQLITE_THREAD_MISUSE_WARNINGS 1 +#endif + +#if defined(SQLITE_THREAD_MISUSE_WARNINGS) && !defined(SQLITE_MUTEX_OMIT) void sqlite3MutexWarnOnContention(sqlite3_mutex*); #else # define sqlite3MutexWarnOnContention(x) @@ -4839,12 +4835,12 @@ struct PrintfArguments { ** value into an approximate decimal representation. */ struct FpDecode { - char sign; /* '+' or '-' */ - char isSpecial; /* 1: Infinity 2: NaN */ int n; /* Significant digits in the decode */ int iDP; /* Location of the decimal point */ char *z; /* Start of significant digits */ - char zBuf[24]; /* Storage for significant digits */ + char zBuf[20]; /* Storage for significant digits */ + char sign; /* '+' or '-' */ + char isSpecial; /* 1: Infinity 2: NaN */ }; void sqlite3FpDecode(FpDecode*,double,int,int); @@ -4933,6 +4929,7 @@ int sqlite3NoTempsInRange(Parse*,int,int); #endif Expr *sqlite3ExprAlloc(sqlite3*,int,const Token*,int); Expr *sqlite3Expr(sqlite3*,int,const char*); +Expr *sqlite3ExprInt32(sqlite3*,int); void sqlite3ExprAttachSubtrees(sqlite3*,Expr*,Expr*,Expr*); Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*); void sqlite3PExprAddSelect(Parse*, Expr*, Select*); @@ -5086,6 +5083,7 @@ Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*, Expr*,ExprList*,u32,Expr*); void sqlite3SelectDelete(sqlite3*, Select*); void sqlite3SelectDeleteGeneric(sqlite3*,void*); +void sqlite3SelectCheckOnClauses(Parse *pParse, Select *pSelect); Table *sqlite3SrcListLookup(Parse*, SrcList*); int sqlite3IsReadOnly(Parse*, Table*, Trigger*); void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int); @@ -5183,6 +5181,7 @@ int sqlite3ExprContainsSubquery(Expr*); int sqlite3ExprIsInteger(const Expr*, int*, Parse*); int sqlite3ExprCanBeNull(const Expr*); int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); +int sqlite3ExprIsLikeOperator(const Expr*); int sqlite3IsRowid(const char*); const char *sqlite3RowidAlias(Table *pTab); void sqlite3GenerateRowDelete( @@ -5251,17 +5250,16 @@ void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,int); void sqlite3DeleteTriggerStep(sqlite3*, TriggerStep*); TriggerStep *sqlite3TriggerSelectStep(sqlite3*,Select*, const char*,const char*); - TriggerStep *sqlite3TriggerInsertStep(Parse*,Token*, IdList*, + TriggerStep *sqlite3TriggerInsertStep(Parse*,SrcList*, IdList*, Select*,u8,Upsert*, const char*,const char*); - TriggerStep *sqlite3TriggerUpdateStep(Parse*,Token*,SrcList*,ExprList*, + TriggerStep *sqlite3TriggerUpdateStep(Parse*,SrcList*,SrcList*,ExprList*, Expr*, u8, const char*,const char*); - TriggerStep *sqlite3TriggerDeleteStep(Parse*,Token*, Expr*, + TriggerStep *sqlite3TriggerDeleteStep(Parse*,SrcList*, Expr*, const char*,const char*); void sqlite3DeleteTrigger(sqlite3*, Trigger*); void sqlite3UnlinkAndDeleteTrigger(sqlite3*,int,const char*); u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Table*,int); - SrcList *sqlite3TriggerStepSrc(Parse*, TriggerStep*); # define sqlite3ParseToplevel(p) ((p)->pToplevel ? (p)->pToplevel : (p)) # define sqlite3IsToplevel(p) ((p)->pToplevel==0) #else @@ -5275,7 +5273,6 @@ void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,int); # define sqlite3ParseToplevel(p) p # define sqlite3IsToplevel(p) 1 # define sqlite3TriggerColmask(A,B,C,D,E,F,G) 0 -# define sqlite3TriggerStepSrc(A,B) 0 #endif int sqlite3JoinType(Parse*, Token*, Token*, Token*); @@ -5308,7 +5305,7 @@ int sqlite3FixTriggerStep(DbFixer*, TriggerStep*); int sqlite3RealSameAsInt(double,sqlite3_int64); i64 sqlite3RealToI64(double); int sqlite3Int64ToText(i64,char*); -int sqlite3AtoF(const char *z, double*, int, u8); +int sqlite3AtoF(const char *z, double*); int sqlite3GetInt32(const char *, int*); int sqlite3GetUInt32(const char*, u32*); int sqlite3Atoi(const char*); @@ -5452,10 +5449,13 @@ void sqlite3Reindex(Parse*, Token*, Token*); void sqlite3AlterFunctions(void); void sqlite3AlterRenameTable(Parse*, SrcList*, Token*); void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*); +void sqlite3AlterDropConstraint(Parse*,SrcList*,Token*,Token*); +void sqlite3AlterAddConstraint(Parse*,SrcList*,Token*,Token*,const char*,int); +void sqlite3AlterSetNotNull(Parse*, SrcList*, Token*, Token*); i64 sqlite3GetToken(const unsigned char *, int *); void sqlite3NestedParse(Parse*, const char*, ...); void sqlite3ExpirePreparedStatements(sqlite3*, int); -void sqlite3CodeRhsOfIN(Parse*, Expr*, int); +void sqlite3CodeRhsOfIN(Parse*, Expr*, int, int); int sqlite3CodeSubselect(Parse*, Expr*); void sqlite3SelectPrep(Parse*, Select*, NameContext*); int sqlite3ExpandSubquery(Parse*, SrcItem*); diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index 6b6bb7167..ecbd0858e 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -25,6 +25,27 @@ #endif #define SQLITE_MIN_LENGTH 30 /* Minimum value for the length limit */ +/* +** Maximum size of any single memory allocation. +** +** This is not a limit on the total amount of memory used. This is +** a limit on the size parameter to sqlite3_malloc() and sqlite3_realloc(). +** +** The upper bound is slightly less than 2GiB: 0x7ffffeff == 2,147,483,391 +** This provides a 256-byte safety margin for defense against 32-bit +** signed integer overflow bugs when computing memory allocation sizes. +** Paranoid applications might want to reduce the maximum allocation size +** further for an even larger safety margin. 0x3fffffff or 0x0fffffff +** or even smaller would be reasonable upper bounds on the size of a memory +** allocations for most applications. +*/ +#ifndef SQLITE_MAX_ALLOCATION_SIZE +# define SQLITE_MAX_ALLOCATION_SIZE 2147483391 +#endif +#if SQLITE_MAX_ALLOCATION_SIZE>2147483391 +# error Maximum size for SQLITE_MAX_ALLOCATION_SIZE is 2147483391 +#endif + /* ** This is the maximum number of ** @@ -60,21 +81,42 @@ ** It used to be the case that setting this value to zero would ** turn the limit off. That is no longer true. It is not possible ** to turn this limit off. +** +** The hard limit is the largest possible 32-bit signed integer less +** 1024, or 2147482624. */ #ifndef SQLITE_MAX_SQL_LENGTH # define SQLITE_MAX_SQL_LENGTH 1000000000 #endif /* -** The maximum depth of an expression tree. This is limited to -** some extent by SQLITE_MAX_SQL_LENGTH. But sometime you might -** want to place more severe limits on the complexity of an -** expression. A value of 0 means that there is no limit. +** The maximum depth of an expression tree. The expression tree depth +** is also limited indirectly by SQLITE_MAX_SQL_LENGTH and by +** SQLITE_MAX_PARSER_DEPTH. Reducing the maximum complexity of +** expressions can help prevent excess memory usage by hostile SQL. +** +** A value of 0 for this compile-time option causes all expression +** depth limiting code to be omitted. */ #ifndef SQLITE_MAX_EXPR_DEPTH # define SQLITE_MAX_EXPR_DEPTH 1000 #endif +/* +** The maximum depth of the LALR(1) stack used in the parser that +** interprets SQL inputs. The parser stack depth can also be limited +** indirectly by SQLITE_MAX_SQL_LENGTH. Limiting the parser stack +** depth can help prevent excess memory usage and excess CPU stack +** usage when processing hostile SQL. +** +** Prior to version 3.45.0 (2024-01-15), the parser stack was +** hard-coded to 100 entries, and that worked fine for almost all +** applications. So the upper bound on this limit need not be large. +*/ +#ifndef SQLITE_MAX_PARSER_DEPTH +# define SQLITE_MAX_PARSER_DEPTH 2500 +#endif + /* ** The maximum number of terms in a compound SELECT statement. ** The code generator for compound SELECT statements does one diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 02a4d84e4..2c7918926 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -124,6 +124,15 @@ /* Forward declaration */ typedef struct SqliteDb SqliteDb; +/* Add -DSQLITE_ENABLE_QRF_IN_TCL to add the Query Result Formatter (QRF) +** into the build of the TCL extension, when building using separate +** source files. The QRF is included automatically when building from +** the tclsqlite3.c amalgamation. +*/ +#if defined(SQLITE_ENABLE_QRF_IN_TCL) +#include "qrf.h" +#endif + /* ** New SQL functions can be created as TCL scripts. Each such function ** is described by an instance of the following structure. @@ -2035,6 +2044,367 @@ static void DbHookCmd( sqlite3_wal_hook(db, (pDb->pWalHook?DbWalHandler:0), pDb); } +/* +** Implementation of the "db format" command. +** +** Based on provided options, format the results of the SQL statement(s) +** provided into human-readable form using the Query Result Formatter (QRF) +** and return the resuling text. +** +** Syntax: db format OPTIONS SQL +** +** OPTIONS may be: +** +** -style ("auto"|"box"|"column"|...) Output style +** -esc ("auto"|"off"|"ascii"|"symbol") How to deal with ctrl chars +** -text ("auto"|"off"|"sql"|"csv"|...) How to escape TEXT values +** -title ("auto"|"off"|"sql"|...|"off") How to escape column names +** -blob ("auto"|"text"|"sql"|...) How to escape BLOB values +** -wordwrap ("auto"|"off"|"on") Try to wrap at word boundry? +** -textjsonb ("auto"|"off"|"on") Auto-convert JSONB to text? +** -splitcolumn ("auto"|"off"|"on") Enable split-column mode +** -defaultalign ("auto"|"left"|...) Default alignment +** -titalalign ("auto"|"left"|"right"|...) Default column name alignment +** -border ("auto"|"off"|"on") Border for box and table styles +** -wrap NUMBER Max width of any single column +** -screenwidth NUMBER Width of the display TTY +** -linelimit NUMBER Max lines for any cell +** -charlimit NUMBER Content truncated to this size +** -titlelimit NUMBER Max width of column titles +** -align LIST-OF-ALIGNMENT Alignment of columns +** -widths LIST-OF-NUMBERS Widths for individual columns +** -columnsep TEXT Column separator text +** -rowsep TEXT Row separator text +** -tablename TEXT Table name for style "insert" +** -null TEXT Text for NULL values +** +** A mapping from TCL "format" command options to sqlite3_qrf_spec fields +** is below. Use this to reference the QRF documentation: +** +** TCL Option spec field +** ---------- ---------- +** -style eStyle +** -esc eEsc +** -text eText +** -title eTitle, bTitle +** -blob eBlob +** -wordwrap bWordWrap +** -textjsonb bTextJsonb +** -splitcolumn bSplitColumn +** -defaultalign eDfltAlign +** -titlealign eTitleAlign +** -border bBorder +** -wrap nWrap +** -screenwidth nScreenWidth +** -linelimit nLineLimit +** -charlimit nCharLimit +** -titlelimit nTitleLimit +** -align nAlign, aAlign +** -widths nWidth, aWidth +** -columnsep zColumnSep +** -rowsep zRowSep +** -tablename zTableName +** -null zNull +*/ +static int dbQrf(SqliteDb *pDb, int objc, Tcl_Obj *const*objv){ +#ifndef SQLITE_QRF_H + Tcl_SetResult(pDb->interp, "QRF not available in this build", TCL_VOLATILE); + return TCL_ERROR; +#else + char *zResult = 0; /* Result to be returned */ + const char *zSql = 0; /* SQL to run */ + int i; /* Loop counter */ + int rc; /* Result code */ + sqlite3_qrf_spec qrf; /* Formatting spec */ + static const char *azAlign[] = { + "auto", "bottom", "c", + "center", "e", "left", + "middle", "n", "ne", + "nw", "right", "s", + "se", "sw", "top", + "w", 0 + }; + static const unsigned char aAlignMap[] = { + QRF_ALIGN_Auto, QRF_ALIGN_Bottom, QRF_ALIGN_C, + QRF_ALIGN_Center, QRF_ALIGN_E, QRF_ALIGN_Left, + QRF_ALIGN_Middle, QRF_ALIGN_N, QRF_ALIGN_NE, + QRF_ALIGN_NW, QRF_ALIGN_Right, QRF_ALIGN_S, + QRF_ALIGN_SE, QRF_ALIGN_SW, QRF_ALIGN_Top, + QRF_ALIGN_W + }; + + memset(&qrf, 0, sizeof(qrf)); + qrf.iVersion = 1; + qrf.pzOutput = &zResult; + for(i=2; i<objc; i++){ + const char *zArg = Tcl_GetString(objv[i]); + const char *azBool[] = { "auto", "yes", "no", "on", "off", 0 }; + const unsigned char aBoolMap[] = { 0, 2, 1, 2, 1 }; + if( zArg[0]!='-' ){ + if( zSql ){ + Tcl_AppendResult(pDb->interp, "unknown argument: ", zArg, (char*)0); + rc = TCL_ERROR; + goto format_failed; + } + zSql = zArg; + }else if( i==objc-1 ){ + Tcl_AppendResult(pDb->interp, "option has no argument: ", zArg, (char*)0); + rc = TCL_ERROR; + goto format_failed; + }else if( strcmp(zArg,"-style")==0 ){ + static const char *azStyles[] = { + "auto", "box", "column", + "count", "csv", "eqp", + "explain", "html", "insert", + "jobject", "json", "line", + "list", "markdown", "quote", + "stats", "stats-est", "stats-vm", + "table", 0 + }; + static unsigned char aStyleMap[] = { + QRF_STYLE_Auto, QRF_STYLE_Box, QRF_STYLE_Column, + QRF_STYLE_Count, QRF_STYLE_Csv, QRF_STYLE_Eqp, + QRF_STYLE_Explain, QRF_STYLE_Html, QRF_STYLE_Insert, + QRF_STYLE_JObject, QRF_STYLE_Json, QRF_STYLE_Line, + QRF_STYLE_List, QRF_STYLE_Markdown, QRF_STYLE_Quote, + QRF_STYLE_Stats, QRF_STYLE_StatsEst, QRF_STYLE_StatsVm, + QRF_STYLE_Table + }; + int style; + rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azStyles, + "format style (-style)", 0, &style); + if( rc ) goto format_failed; + qrf.eStyle = aStyleMap[style]; + i++; + }else if( strcmp(zArg,"-esc")==0 ){ + static const char *azEsc[] = { + "ascii", "auto", "off", "symbol", 0 + }; + static unsigned char aEscMap[] = { + QRF_ESC_Ascii, QRF_ESC_Auto, QRF_ESC_Off, QRF_ESC_Symbol + }; + int esc; + rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azEsc, + "control character escape (-esc)", 0, &esc); + if( rc ) goto format_failed; + qrf.eEsc = aEscMap[esc]; + i++; + }else if( strcmp(zArg,"-text")==0 || strcmp(zArg, "-title")==0 ){ + /* NB: --title can be "off" or "on but --text may not be. Thus we put + ** the "off" and "on" choices first and start the search on the + ** thrid element of the array when processing --text */ + static const char *azText[] = { "off", "on", + "auto", "csv", "html", + "json", "plain", "relaxed", + "sql", "tcl", 0 + }; + static unsigned char aTextMap[] = { + QRF_TEXT_Auto, QRF_TEXT_Csv, QRF_TEXT_Html, + QRF_TEXT_Json, QRF_TEXT_Plain, QRF_TEXT_Relaxed, + QRF_TEXT_Sql, QRF_TEXT_Tcl + }; + int txt; + int k = zArg[2]=='e'; + rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], &azText[k*2], zArg, + 0, &txt); + if( rc ) goto format_failed; + if( k ){ + qrf.eText = aTextMap[txt]; + }else if( txt<=1 ){ + qrf.bTitles = txt ? QRF_Yes : QRF_No; + qrf.eTitle = QRF_TEXT_Auto; + }else{ + qrf.bTitles = QRF_Yes; + qrf.eTitle = aTextMap[txt-2]; + } + i++; + }else if( strcmp(zArg,"-blob")==0 ){ + static const char *azBlob[] = { + "auto", "hex", "json", + "tcl", "text", "sql", + "size", 0 + }; + static unsigned char aBlobMap[] = { + QRF_BLOB_Auto, QRF_BLOB_Hex, QRF_BLOB_Json, + QRF_BLOB_Tcl, QRF_BLOB_Text, QRF_BLOB_Sql, + QRF_BLOB_Size + }; + int blob; + rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azBlob, + "BLOB encoding (-blob)", 0, &blob); + if( rc ) goto format_failed; + qrf.eBlob = aBlobMap[blob]; + i++; + }else if( strcmp(zArg,"-wordwrap")==0 ){ + int v = 0; + rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azBool, + "-wordwrap", 0, &v); + if( rc ) goto format_failed; + qrf.bWordWrap = aBoolMap[v]; + i++; + }else if( strcmp(zArg,"-textjsonb")==0 + || strcmp(zArg,"-splitcolumn")==0 + || strcmp(zArg,"-border")==0 + ){ + int v = 0; + rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azBool, + zArg, 0, &v); + if( rc ) goto format_failed; + if( zArg[1]=='t' ){ + qrf.bTextJsonb = aBoolMap[v]; + }else if( zArg[1]=='b' ){ + qrf.bBorder = aBoolMap[v]; + }else{ + qrf.bSplitColumn = aBoolMap[v]; + } + i++; + }else if( strcmp(zArg,"-defaultalign")==0 || strcmp(zArg,"-titlealign")==0){ + int ax = 0; + rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azAlign, + zArg[1]=='d' ? "default alignment (-defaultalign)" : + "title alignment (-titlealign)", + 0, &ax); + if( rc ) goto format_failed; + if( zArg[1]=='d' ){ + qrf.eDfltAlign = aAlignMap[ax]; + }else{ + qrf.eTitleAlign = aAlignMap[ax]; + } + i++; + }else if( strcmp(zArg,"-wrap")==0 + || strcmp(zArg,"-screenwidth")==0 + || strcmp(zArg,"-linelimit")==0 + || strcmp(zArg,"-titlelimit")==0 + ){ + int v = 0; + rc = Tcl_GetIntFromObj(pDb->interp, objv[i+1], &v); + if( rc ) goto format_failed; + if( v<QRF_MIN_WIDTH ){ + v = QRF_MIN_WIDTH; + }else if( v>QRF_MAX_WIDTH ){ + v = QRF_MAX_WIDTH; + } + if( zArg[1]=='w' ){ + qrf.nWrap = v; + }else if( zArg[1]=='s' ){ + qrf.nScreenWidth = v; + }else if( zArg[1]=='t' ){ + qrf.nTitleLimit = v; + }else{ + qrf.nLineLimit = v; + } + i++; + }else if( strcmp(zArg,"-charlimit")==0 ){ + int v = 0; + rc = Tcl_GetIntFromObj(pDb->interp, objv[i+1], &v); + if( rc ) goto format_failed; + if( v<0 ) v = 0; + qrf.nCharLimit = v; + i++; + }else if( strcmp(zArg,"-align")==0 ){ + Tcl_Size n = 0; + int jj; + rc = Tcl_ListObjLength(pDb->interp, objv[i+1], &n); + if( rc ) goto format_failed; + sqlite3_free(qrf.aAlign); + qrf.aAlign = sqlite3_malloc64( (n+1)*sizeof(qrf.aAlign[0]) ); + if( qrf.aAlign==0 ){ + Tcl_AppendResult(pDb->interp, "out of memory", (char*)0); + rc = TCL_ERROR; + goto format_failed; + } + memset(qrf.aAlign, 0, (n+1)*sizeof(qrf.aAlign[0])); + qrf.nAlign = n; + for(jj=0; jj<n; jj++){ + int x; + Tcl_Obj *pTerm; + rc = Tcl_ListObjIndex(pDb->interp, objv[i+1], jj, &pTerm); + if( rc ) goto format_failed; + rc = Tcl_GetIndexFromObj(pDb->interp, pTerm, azAlign, + "column alignment (-align)", 0, &x); + if( rc ) goto format_failed; + qrf.aAlign[jj] = aAlignMap[x]; + } + i++; + }else if( strcmp(zArg,"-widths")==0 ){ + Tcl_Size n = 0; + int jj; + rc = Tcl_ListObjLength(pDb->interp, objv[i+1], &n); + if( rc ) goto format_failed; + sqlite3_free(qrf.aWidth); + qrf.aWidth = sqlite3_malloc64( (n+1)*sizeof(qrf.aWidth[0]) ); + if( qrf.aWidth==0 ){ + Tcl_AppendResult(pDb->interp, "out of memory", (char*)0); + rc = TCL_ERROR; + goto format_failed; + } + memset(qrf.aWidth, 0, (n+1)*sizeof(qrf.aWidth[0])); + qrf.nWidth = n; + for(jj=0; jj<n; jj++){ + Tcl_Obj *pTerm; + int v; + rc = Tcl_ListObjIndex(pDb->interp, objv[i+1], jj, &pTerm); + if( rc ) goto format_failed; + rc = Tcl_GetIntFromObj(pDb->interp, pTerm, &v); + if( v<(-QRF_MAX_WIDTH) ){ + v = -QRF_MAX_WIDTH; + }else if( v>QRF_MAX_WIDTH ){ + v = QRF_MAX_WIDTH; + } + qrf.aWidth[jj] = (short int)v; + } + i++; + }else if( strcmp(zArg,"-columnsep")==0 ){ + qrf.zColumnSep = Tcl_GetString(objv[i+1]); + i++; + }else if( strcmp(zArg,"-rowsep")==0 ){ + qrf.zRowSep = Tcl_GetString(objv[i+1]); + i++; + }else if( strcmp(zArg,"-tablename")==0 ){ + qrf.zTableName = Tcl_GetString(objv[i+1]); + i++; + }else if( strcmp(zArg,"-null")==0 ){ + qrf.zNull = Tcl_GetString(objv[i+1]); + i++; + }else if( strcmp(zArg,"-version")==0 ){ + /* Undocumented. Testing use only */ + qrf.iVersion = atoi(Tcl_GetString(objv[i+1])); + i++; + }else{ + Tcl_AppendResult(pDb->interp, "unknown option: ", zArg, (char*)0); + rc = TCL_ERROR; + goto format_failed; + } + } + while( zSql && zSql[0] ){ + SqlPreparedStmt *pStmt = 0; /* Next statement to run */ + char *zErr = 0; /* Error message from QRF */ + + rc = dbPrepareAndBind(pDb, zSql, &zSql, &pStmt); + if( rc ) goto format_failed; + if( pStmt==0 ) continue; + rc = sqlite3_format_query_result(pStmt->pStmt, &qrf, &zErr); + dbReleaseStmt(pDb, pStmt, 0); + if( rc ){ + Tcl_SetResult(pDb->interp, zErr, TCL_VOLATILE); + sqlite3_free(zErr); + rc = TCL_ERROR; + goto format_failed; + } + } + Tcl_SetResult(pDb->interp, zResult, TCL_VOLATILE); + rc = TCL_OK; + /* Fall through...*/ + +format_failed: + sqlite3_free(qrf.aWidth); + sqlite3_free(qrf.aAlign); + sqlite3_free(zResult); + return rc; + +#endif +} + /* ** The "sqlite" command below creates a new Tcl command for each ** connection it opens to an SQLite database. This routine is invoked @@ -2064,15 +2434,15 @@ static int SQLITE_TCLAPI DbObjCmd( "commit_hook", "complete", "config", "copy", "deserialize", "enable_load_extension", "errorcode", "erroroffset", "eval", - "exists", "function", "incrblob", - "interrupt", "last_insert_rowid", "nullvalue", - "onecolumn", "preupdate", "profile", - "progress", "rekey", "restore", - "rollback_hook", "serialize", "status", - "timeout", "total_changes", "trace", - "trace_v2", "transaction", "unlock_notify", - "update_hook", "version", "wal_hook", - 0 + "exists", "format", "function", + "incrblob", "interrupt", "last_insert_rowid", + "nullvalue", "onecolumn", "preupdate", + "profile", "progress", "rekey", + "restore", "rollback_hook", "serialize", + "status", "timeout", "total_changes", + "trace", "trace_v2", "transaction", + "unlock_notify", "update_hook", "version", + "wal_hook", 0 }; enum DB_enum { DB_AUTHORIZER, DB_BACKUP, DB_BIND_FALLBACK, @@ -2081,14 +2451,15 @@ static int SQLITE_TCLAPI DbObjCmd( DB_COMMIT_HOOK, DB_COMPLETE, DB_CONFIG, DB_COPY, DB_DESERIALIZE, DB_ENABLE_LOAD_EXTENSION, DB_ERRORCODE, DB_ERROROFFSET, DB_EVAL, - DB_EXISTS, DB_FUNCTION, DB_INCRBLOB, - DB_INTERRUPT, DB_LAST_INSERT_ROWID, DB_NULLVALUE, - DB_ONECOLUMN, DB_PREUPDATE, DB_PROFILE, - DB_PROGRESS, DB_REKEY, DB_RESTORE, - DB_ROLLBACK_HOOK, DB_SERIALIZE, DB_STATUS, - DB_TIMEOUT, DB_TOTAL_CHANGES, DB_TRACE, - DB_TRACE_V2, DB_TRANSACTION, DB_UNLOCK_NOTIFY, - DB_UPDATE_HOOK, DB_VERSION, DB_WAL_HOOK, + DB_EXISTS, DB_FORMAT, DB_FUNCTION, + DB_INCRBLOB, DB_INTERRUPT, DB_LAST_INSERT_ROWID, + DB_NULLVALUE, DB_ONECOLUMN, DB_PREUPDATE, + DB_PROFILE, DB_PROGRESS, DB_REKEY, + DB_RESTORE, DB_ROLLBACK_HOOK, DB_SERIALIZE, + DB_STATUS, DB_TIMEOUT, DB_TOTAL_CHANGES, + DB_TRACE, DB_TRACE_V2, DB_TRANSACTION, + DB_UNLOCK_NOTIFY, DB_UPDATE_HOOK, DB_VERSION, + DB_WAL_HOOK }; /* don't leave trailing commas on DB_enum, it confuses the AIX xlc compiler */ @@ -2978,6 +3349,18 @@ static int SQLITE_TCLAPI DbObjCmd( break; } + /* + ** $db format [OPTIONS] SQL + ** + ** Run the SQL statement(s) given as the final argument. Use the + ** Query Result Formatter extension of SQLite to format the output as + ** text and return that text. + */ + case DB_FORMAT: { + rc = dbQrf(pDb, objc, objv); + break; + } + /* ** $db function NAME [OPTIONS] SCRIPT ** diff --git a/src/test1.c b/src/test1.c index f8e83dc42..3ca5c837a 100644 --- a/src/test1.c +++ b/src/test1.c @@ -4411,7 +4411,7 @@ static void delIntptr(void *p){ } /* -** bind_carray_intptr STMT IPARAM INT0 INT1 INT2... +** bind_carray_intptr STMT IPARAM INT-0 INT-1 INT-2... */ static int SQLITE_TCLAPI bind_carray_intptr( void * clientData, @@ -4455,6 +4455,7 @@ static int SQLITE_TCLAPI bind_carray_intptr( ** -malloc ** -transient ** -static +** -v2 ** -int32 ** -int64 ** -double @@ -4477,6 +4478,7 @@ static int SQLITE_TCLAPI test_carray_bind( void *aData = 0; int isTransient = 0; int isStatic = 0; + int isV2 = 0; int isMalloc = 0; /* True to use custom xDel function */ int idx; int i, j; @@ -4509,16 +4511,22 @@ static int SQLITE_TCLAPI test_carray_bind( const char *z = Tcl_GetString(objv[i]); if( strcmp(z, "-transient")==0 ){ isTransient = 1; + isStatic = isMalloc = 0; xDel = SQLITE_TRANSIENT; }else if( strcmp(z, "-static")==0 ){ isStatic = 1; + isMalloc = isTransient = 0; xDel = SQLITE_STATIC; }else if( strcmp(z, "-malloc")==0 ){ isMalloc = 1; + isStatic = isTransient = 0; xDel = testCarrayFree; }else + if( strcmp(z, "-v2")==0 ){ + isV2 = 1; + }else if( strcmp(z, "-int32")==0 ){ eType = 0; /* CARRAY_INT32 */ }else @@ -4687,7 +4695,20 @@ static int SQLITE_TCLAPI test_carray_bind( if( rc==SQLITE_OK ){ if( mFlagsOverride==0 ) mFlagsOverride = eType; - rc = sqlite3_carray_bind(pStmt, idx, aData, nData, mFlagsOverride, xDel); + if( isV2 ){ + void *pDel; + if( xDel==testCarrayFree ){ + u8 *p2 = (u8*)aData; + pDel = (void*)&p2[-16]; + xDel = sqlite3_free; + }else{ + pDel = aData; + } + rc = sqlite3_carray_bind_v2(pStmt, idx, aData, nData, mFlagsOverride, + xDel, pDel); + }else{ + rc = sqlite3_carray_bind(pStmt, idx, aData, nData, mFlagsOverride, xDel); + } } if( isTransient ){ if( eType==3 && aData ){ @@ -7370,6 +7391,7 @@ static int SQLITE_TCLAPI test_limit( { "SQLITE_LIMIT_SQL_LENGTH", SQLITE_LIMIT_SQL_LENGTH }, { "SQLITE_LIMIT_COLUMN", SQLITE_LIMIT_COLUMN }, { "SQLITE_LIMIT_EXPR_DEPTH", SQLITE_LIMIT_EXPR_DEPTH }, + { "SQLITE_LIMIT_PARSER_DEPTH", SQLITE_LIMIT_PARSER_DEPTH }, { "SQLITE_LIMIT_COMPOUND_SELECT", SQLITE_LIMIT_COMPOUND_SELECT }, { "SQLITE_LIMIT_VDBE_OP", SQLITE_LIMIT_VDBE_OP }, { "SQLITE_LIMIT_FUNCTION_ARG", SQLITE_LIMIT_FUNCTION_ARG }, @@ -7381,7 +7403,7 @@ static int SQLITE_TCLAPI test_limit( /* Out of range test cases */ { "SQLITE_LIMIT_TOOSMALL", -1, }, - { "SQLITE_LIMIT_TOOBIG", SQLITE_LIMIT_WORKER_THREADS+1 }, + { "SQLITE_LIMIT_TOOBIG", SQLITE_LIMIT_PARSER_DEPTH+1 }, }; int i, id = 0; int val; @@ -7675,7 +7697,8 @@ static int SQLITE_TCLAPI test_wal_checkpoint_v2( int nCkpt = -555; Tcl_Obj *pRet; - const char * aMode[] = { "passive", "full", "restart", "truncate", 0 }; + const char * aMode[] = {"noop", "passive", "full", "restart", "truncate", 0}; + assert( SQLITE_CHECKPOINT_NOOP==-1 ); assert( SQLITE_CHECKPOINT_PASSIVE==0 ); assert( SQLITE_CHECKPOINT_FULL==1 ); assert( SQLITE_CHECKPOINT_RESTART==2 ); @@ -7689,12 +7712,15 @@ static int SQLITE_TCLAPI test_wal_checkpoint_v2( if( objc==4 ){ zDb = Tcl_GetString(objv[3]); } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) || ( - TCL_OK!=Tcl_GetIntFromObj(0, objv[2], &eMode) - && TCL_OK!=Tcl_GetIndexFromObj(interp, objv[2], aMode, "mode", 0, &eMode) - )){ + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ return TCL_ERROR; } + if( TCL_OK!=Tcl_GetIntFromObj(0, objv[2], &eMode) ){ + if( TCL_OK!=Tcl_GetIndexFromObj(interp, objv[2], aMode, "mode", 0,&eMode) ){ + return TCL_ERROR; + } + eMode = eMode - 1; + } rc = sqlite3_wal_checkpoint_v2(db, zDb, eMode, &nLog, &nCkpt); if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ @@ -8618,6 +8644,7 @@ static int SQLITE_TCLAPI test_sqlite3_db_config( { "ATTACH_CREATE", SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE }, { "ATTACH_WRITE", SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE }, { "COMMENTS", SQLITE_DBCONFIG_ENABLE_COMMENTS }, + { "FP_DIGITS", SQLITE_DBCONFIG_FP_DIGITS }, }; int i; int v = 0; diff --git a/src/test_bestindex.c b/src/test_bestindex.c index f6b5db0fb..963abfec0 100644 --- a/src/test_bestindex.c +++ b/src/test_bestindex.c @@ -828,7 +828,6 @@ static int tclUpdate( tcl_vtab *pTab = (tcl_vtab*)tab; Tcl_Interp *interp = pTab->interp; Tcl_Obj *pEval = Tcl_DuplicateObj(pTab->pCmd); - Tcl_Obj *pRes = 0; int rc = TCL_OK; Tcl_IncrRefCount(pEval); diff --git a/src/test_config.c b/src/test_config.c index 3dbef3c9a..bebf8625a 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -82,7 +82,7 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options","configslower","1.0",TCL_GLOBAL_ONLY); #endif -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE Tcl_SetVar2(interp, "sqlite_options", "curdir", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "curdir", "0", TCL_GLOBAL_ONLY); @@ -94,12 +94,6 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "win32malloc", "0", TCL_GLOBAL_ONLY); #endif -#if defined(SQLITE_OS_WINRT) && SQLITE_OS_WINRT - Tcl_SetVar2(interp, "sqlite_options", "winrt", "1", TCL_GLOBAL_ONLY); -#else - Tcl_SetVar2(interp, "sqlite_options", "winrt", "0", TCL_GLOBAL_ONLY); -#endif - #ifdef SQLITE_DEBUG Tcl_SetVar2(interp, "sqlite_options", "debug", "1", TCL_GLOBAL_ONLY); #else @@ -685,6 +679,14 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); Tcl_SetVar2(interp, "sqlite_options", "trace", "1", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_THREAD_MISUSE_WARNINGS + Tcl_SetVar2(interp, "sqlite_options", "thread_misuse_warnings", + "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "thread_misuse_warnings", + "0", TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_OMIT_TRIGGER Tcl_SetVar2(interp, "sqlite_options", "trigger", "0", TCL_GLOBAL_ONLY); #else diff --git a/src/test_quota.c b/src/test_quota.c index d2f9cddd1..3eeacc7e7 100644 --- a/src/test_quota.c +++ b/src/test_quota.c @@ -387,11 +387,7 @@ static char *quota_utf8_to_mbcs(const char *zUtf8){ zTmpWide = (LPWSTR)sqlite3_malloc( (nWide+1)*sizeof(zTmpWide[0]) ); if( zTmpWide==0 ) return 0; MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, zTmpWide, nWide); -#ifdef SQLITE_OS_WINRT - codepage = CP_ACP; -#else codepage = AreFileApisANSI() ? CP_ACP : CP_OEMCP; -#endif nMbcs = WideCharToMultiByte(codepage, 0, zTmpWide, nWide, 0, 0, 0, 0); zMbcs = nMbcs ? (char*)sqlite3_malloc( nMbcs+1 ) : 0; if( zMbcs ){ diff --git a/src/tokenize.c b/src/tokenize.c index 152ada64f..884d1acb8 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -508,7 +508,7 @@ i64 sqlite3GetToken(const unsigned char *z, int *tokenType){ } case CC_DOLLAR: case CC_VARALPHA: { - int n = 0; + i64 n = 0; testcase( z[0]=='$' ); testcase( z[0]=='@' ); testcase( z[0]==':' ); testcase( z[0]=='#' ); *tokenType = TK_VARIABLE; @@ -604,7 +604,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql){ int tokenType; /* type of the next token */ int lastTokenParsed = -1; /* type of the previous token */ sqlite3 *db = pParse->db; /* The database connection */ - int mxSqlLen; /* Max length of an SQL string */ + i64 mxSqlLen; /* Max length of an SQL string */ Parse *pParentParse = 0; /* Outer parse context, if any */ #ifdef sqlite3Parser_ENGINEALWAYSONSTACK yyParser sEngine; /* Space to hold the Lemon-generated Parser object */ diff --git a/src/treeview.c b/src/treeview.c index 153fec88d..b482e1581 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -1300,7 +1300,13 @@ void sqlite3TreeViewTrigger( void sqlite3ShowExpr(const Expr *p){ sqlite3TreeViewExpr(0,p,0); } void sqlite3ShowExprList(const ExprList *p){ sqlite3TreeViewExprList(0,p,0,0);} void sqlite3ShowIdList(const IdList *p){ sqlite3TreeViewIdList(0,p,0,0); } -void sqlite3ShowSrcList(const SrcList *p){ sqlite3TreeViewSrcList(0,p); } +void sqlite3ShowSrcList(const SrcList *p){ + TreeView *pView = 0; + sqlite3TreeViewPush(&pView, 0); + sqlite3TreeViewLine(pView, "SRCLIST"); + sqlite3TreeViewSrcList(pView,p); + sqlite3TreeViewPop(&pView); +} void sqlite3ShowSelect(const Select *p){ sqlite3TreeViewSelect(0,p,0); } void sqlite3ShowWith(const With *p){ sqlite3TreeViewWith(0,p,0); } void sqlite3ShowUpsert(const Upsert *p){ sqlite3TreeViewUpsert(0,p,0); } diff --git a/src/trigger.c b/src/trigger.c index 799fbe57f..4f9068ad8 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -26,7 +26,7 @@ void sqlite3DeleteTriggerStep(sqlite3 *db, TriggerStep *pTriggerStep){ sqlite3SelectDelete(db, pTmp->pSelect); sqlite3IdListDelete(db, pTmp->pIdList); sqlite3UpsertDelete(db, pTmp->pUpsert); - sqlite3SrcListDelete(db, pTmp->pFrom); + sqlite3SrcListDelete(db, pTmp->pSrc); sqlite3DbFree(db, pTmp->zSpan); sqlite3DbFree(db, pTmp); @@ -215,11 +215,16 @@ void sqlite3BeginTrigger( } } + /* NB: The SQLITE_ALLOW_TRIGGERS_ON_SYSTEM_TABLES compile-time option is + ** experimental and unsupported. Do not use it unless understand the + ** implications and you cannot get by without this capability. */ +#if !defined(SQLITE_ALLOW_TRIGGERS_ON_SYSTEM_TABLES) /* Experimental */ /* Do not create a trigger on a system table */ if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ sqlite3ErrorMsg(pParse, "cannot create trigger on system table"); goto trigger_cleanup; } +#endif /* INSTEAD of triggers are only for views and views only support INSTEAD ** of triggers. @@ -331,6 +336,7 @@ void sqlite3FinishTrigger( if( NEVER(pParse->nErr) || !pTrig ) goto triggerfinish_cleanup; zName = pTrig->zName; iDb = sqlite3SchemaToIndex(pParse->db, pTrig->pSchema); + assert( iDb>=00 && iDb<db->nDb ); pTrig->step_list = pStepList; while( pStepList ){ pStepList->pTrig = pTrig; @@ -365,12 +371,12 @@ void sqlite3FinishTrigger( if( sqlite3ReadOnlyShadowTables(db) ){ TriggerStep *pStep; for(pStep=pTrig->step_list; pStep; pStep=pStep->pNext){ - if( pStep->zTarget!=0 - && sqlite3ShadowTableName(db, pStep->zTarget) + if( pStep->pSrc!=0 + && sqlite3ShadowTableName(db, pStep->pSrc->a[0].zName) ){ sqlite3ErrorMsg(pParse, "trigger \"%s\" may not write to shadow table \"%s\"", - pTrig->zName, pStep->zTarget); + pTrig->zName, pStep->pSrc->a[0].zName); goto triggerfinish_cleanup; } } @@ -461,26 +467,39 @@ TriggerStep *sqlite3TriggerSelectStep( static TriggerStep *triggerStepAllocate( Parse *pParse, /* Parser context */ u8 op, /* Trigger opcode */ - Token *pName, /* The target name */ + SrcList *pTabList, /* Target table */ const char *zStart, /* Start of SQL text */ const char *zEnd /* End of SQL text */ ){ + Trigger *pNew = pParse->pNewTrigger; sqlite3 *db = pParse->db; - TriggerStep *pTriggerStep; + TriggerStep *pTriggerStep = 0; - if( pParse->nErr ) return 0; - pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep) + pName->n + 1); - if( pTriggerStep ){ - char *z = (char*)&pTriggerStep[1]; - memcpy(z, pName->z, pName->n); - sqlite3Dequote(z); - pTriggerStep->zTarget = z; - pTriggerStep->op = op; - pTriggerStep->zSpan = triggerSpanDup(db, zStart, zEnd); - if( IN_RENAME_OBJECT ){ - sqlite3RenameTokenMap(pParse, pTriggerStep->zTarget, pName); + if( pParse->nErr==0 ){ + if( pNew + && pNew->pSchema!=db->aDb[1].pSchema + && pTabList->a[0].u4.zDatabase + ){ + sqlite3ErrorMsg(pParse, + "qualified table names are not allowed on INSERT, UPDATE, and DELETE " + "statements within triggers"); + }else{ + pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep)); + if( pTriggerStep ){ + pTriggerStep->pSrc = sqlite3SrcListDup(db, pTabList, EXPRDUP_REDUCE); + pTriggerStep->op = op; + pTriggerStep->zSpan = triggerSpanDup(db, zStart, zEnd); + if( pTriggerStep->pSrc && IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, + pTriggerStep->pSrc->a[0].zName, + pTabList->a[0].zName + ); + } + } } } + + sqlite3SrcListDelete(db, pTabList); return pTriggerStep; } @@ -493,7 +512,7 @@ static TriggerStep *triggerStepAllocate( */ TriggerStep *sqlite3TriggerInsertStep( Parse *pParse, /* Parser */ - Token *pTableName, /* Name of the table into which we insert */ + SrcList *pTabList, /* Table to INSERT into */ IdList *pColumn, /* List of columns in pTableName to insert into */ Select *pSelect, /* A SELECT statement that supplies values */ u8 orconf, /* The conflict algorithm (OE_Abort, OE_Replace, etc.) */ @@ -506,7 +525,7 @@ TriggerStep *sqlite3TriggerInsertStep( assert(pSelect != 0 || db->mallocFailed); - pTriggerStep = triggerStepAllocate(pParse, TK_INSERT, pTableName,zStart,zEnd); + pTriggerStep = triggerStepAllocate(pParse, TK_INSERT, pTabList, zStart, zEnd); if( pTriggerStep ){ if( IN_RENAME_OBJECT ){ pTriggerStep->pSelect = pSelect; @@ -538,7 +557,7 @@ TriggerStep *sqlite3TriggerInsertStep( */ TriggerStep *sqlite3TriggerUpdateStep( Parse *pParse, /* Parser */ - Token *pTableName, /* Name of the table to be updated */ + SrcList *pTabList, /* Name of the table to be updated */ SrcList *pFrom, /* FROM clause for an UPDATE-FROM, or NULL */ ExprList *pEList, /* The SET clause: list of column and new values */ Expr *pWhere, /* The WHERE clause */ @@ -549,21 +568,36 @@ TriggerStep *sqlite3TriggerUpdateStep( sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; - pTriggerStep = triggerStepAllocate(pParse, TK_UPDATE, pTableName,zStart,zEnd); + pTriggerStep = triggerStepAllocate(pParse, TK_UPDATE, pTabList, zStart, zEnd); if( pTriggerStep ){ + SrcList *pFromDup = 0; if( IN_RENAME_OBJECT ){ pTriggerStep->pExprList = pEList; pTriggerStep->pWhere = pWhere; - pTriggerStep->pFrom = pFrom; + pFromDup = pFrom; pEList = 0; pWhere = 0; pFrom = 0; }else{ pTriggerStep->pExprList = sqlite3ExprListDup(db, pEList, EXPRDUP_REDUCE); pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); - pTriggerStep->pFrom = sqlite3SrcListDup(db, pFrom, EXPRDUP_REDUCE); + pFromDup = sqlite3SrcListDup(db, pFrom, EXPRDUP_REDUCE); } pTriggerStep->orconf = orconf; + + if( pFromDup && !IN_RENAME_OBJECT){ + Select *pSub; + Token as = {0, 0}; + pSub = sqlite3SelectNew(pParse, 0, pFromDup, 0,0,0,0, SF_NestedFrom, 0); + pFromDup = sqlite3SrcListAppendFromTerm(pParse, 0, 0, 0, &as, pSub ,0); + } + if( pFromDup && pTriggerStep->pSrc ){ + pTriggerStep->pSrc = sqlite3SrcListAppendList( + pParse, pTriggerStep->pSrc, pFromDup + ); + }else{ + sqlite3SrcListDelete(db, pFromDup); + } } sqlite3ExprListDelete(db, pEList); sqlite3ExprDelete(db, pWhere); @@ -578,7 +612,7 @@ TriggerStep *sqlite3TriggerUpdateStep( */ TriggerStep *sqlite3TriggerDeleteStep( Parse *pParse, /* Parser */ - Token *pTableName, /* The table from which rows are deleted */ + SrcList *pTabList, /* The table from which rows are deleted */ Expr *pWhere, /* The WHERE clause */ const char *zStart, /* Start of SQL text */ const char *zEnd /* End of SQL text */ @@ -586,7 +620,7 @@ TriggerStep *sqlite3TriggerDeleteStep( sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; - pTriggerStep = triggerStepAllocate(pParse, TK_DELETE, pTableName,zStart,zEnd); + pTriggerStep = triggerStepAllocate(pParse, TK_DELETE, pTabList, zStart, zEnd); if( pTriggerStep ){ if( IN_RENAME_OBJECT ){ pTriggerStep->pWhere = pWhere; @@ -786,6 +820,7 @@ static SQLITE_NOINLINE Trigger *triggersReallyExist( p = pList; if( (pParse->db->flags & SQLITE_EnableTrigger)==0 && pTab->pTrigger!=0 + && sqlite3SchemaToIndex(pParse->db, pTab->pTrigger->pSchema)!=1 ){ /* The SQLITE_DBCONFIG_ENABLE_TRIGGER setting is off. That means that ** only TEMP triggers are allowed. Truncate the pList so that it @@ -848,52 +883,6 @@ Trigger *sqlite3TriggersExist( return triggersReallyExist(pParse,pTab,op,pChanges,pMask); } -/* -** Convert the pStep->zTarget string into a SrcList and return a pointer -** to that SrcList. -** -** This routine adds a specific database name, if needed, to the target when -** forming the SrcList. This prevents a trigger in one database from -** referring to a target in another database. An exception is when the -** trigger is in TEMP in which case it can refer to any other database it -** wants. -*/ -SrcList *sqlite3TriggerStepSrc( - Parse *pParse, /* The parsing context */ - TriggerStep *pStep /* The trigger containing the target token */ -){ - sqlite3 *db = pParse->db; - SrcList *pSrc; /* SrcList to be returned */ - char *zName = sqlite3DbStrDup(db, pStep->zTarget); - pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); - assert( pSrc==0 || pSrc->nSrc==1 ); - assert( zName || pSrc==0 ); - if( pSrc ){ - Schema *pSchema = pStep->pTrig->pSchema; - pSrc->a[0].zName = zName; - if( pSchema!=db->aDb[1].pSchema ){ - assert( pSrc->a[0].fg.fixedSchema || pSrc->a[0].u4.zDatabase==0 ); - pSrc->a[0].u4.pSchema = pSchema; - pSrc->a[0].fg.fixedSchema = 1; - } - if( pStep->pFrom ){ - SrcList *pDup = sqlite3SrcListDup(db, pStep->pFrom, 0); - if( pDup && pDup->nSrc>1 && !IN_RENAME_OBJECT ){ - Select *pSubquery; - Token as; - pSubquery = sqlite3SelectNew(pParse,0,pDup,0,0,0,0,SF_NestedFrom,0); - as.n = 0; - as.z = 0; - pDup = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&as,pSubquery,0); - } - pSrc = sqlite3SrcListAppendList(pParse, pSrc, pDup); - } - }else{ - sqlite3DbFree(db, zName); - } - return pSrc; -} - /* ** Return true if the pExpr term from the RETURNING clause argument ** list is of the form "*". Raise an error if the terms if of the @@ -1159,7 +1148,7 @@ static int codeTriggerProgram( switch( pStep->op ){ case TK_UPDATE: { sqlite3Update(pParse, - sqlite3TriggerStepSrc(pParse, pStep), + sqlite3SrcListDup(db, pStep->pSrc, 0), sqlite3ExprListDup(db, pStep->pExprList, 0), sqlite3ExprDup(db, pStep->pWhere, 0), pParse->eOrconf, 0, 0, 0 @@ -1169,7 +1158,7 @@ static int codeTriggerProgram( } case TK_INSERT: { sqlite3Insert(pParse, - sqlite3TriggerStepSrc(pParse, pStep), + sqlite3SrcListDup(db, pStep->pSrc, 0), sqlite3SelectDup(db, pStep->pSelect, 0), sqlite3IdListDup(db, pStep->pIdList), pParse->eOrconf, @@ -1180,7 +1169,7 @@ static int codeTriggerProgram( } case TK_DELETE: { sqlite3DeleteFrom(pParse, - sqlite3TriggerStepSrc(pParse, pStep), + sqlite3SrcListDup(db, pStep->pSrc, 0), sqlite3ExprDup(db, pStep->pWhere, 0), 0, 0 ); sqlite3VdbeAddOp0(v, OP_ResetCount); diff --git a/src/util.c b/src/util.c index 8e4fd516e..071029173 100644 --- a/src/util.c +++ b/src/util.c @@ -458,48 +458,262 @@ u8 sqlite3StrIHash(const char *z){ return h; } -/* Double-Double multiplication. (x[0],x[1]) *= (y,yy) +/* +** Two inputs are multiplied to get a 128-bit result. Return +** the high-order 64 bits of that result. +*/ +static u64 sqlite3Multiply128(u64 a, u64 b){ +#if (defined(__GNUC__) || defined(__clang__)) \ + && (defined(__x86_64__) || defined(__aarch64__) || defined(__riscv)) + return ((__uint128_t)a * b) >> 64; +#elif defined(_MSC_VER) && defined(_M_X64) + return __umulh(a, b); +#else + u64 a1 = (u32)a; + u64 a2 = a >> 32; + u64 b1 = (u32)b; + u64 b2 = b >> 32; + u64 p0 = a1 * b1; + u64 p1 = a1 * b2; + u64 p2 = a2 * b1; + u64 p3 = a2 * b2; + u64 carry = ((p0 >> 32) + (u32)p1 + (u32)p2) >> 32; + return p3 + (p1 >> 32) + (p2 >> 32) + carry; +#endif +} + +/* +** Return a u64 with the N-th bit set. +*/ +#define U64_BIT(N) (((u64)1)<<(N)) + +/* +** Range of powers of 10 that we need to deal with when converting +** IEEE754 doubles to and from decimal. +*/ +#define POWERSOF10_FIRST (-348) +#define POWERSOF10_LAST (+347) + +/* +** For any p between -348 and +347, return the integer part of +** +** pow(10,p) * pow(2,63-pow10to2(p)) +** +** Or, in other words, for any p in range, return the most significant +** 64 bits of pow(10,p). The pow(10,p) value is shifted left or right, +** as appropriate so the most significant 64 bits fit exactly into a +** 64-bit unsigned integer. ** -** Reference: -** T. J. Dekker, "A Floating-Point Technique for Extending the -** Available Precision". 1971-07-26. +** Algorithm: +** +** (1) For p between 0 and 26, return the value directly from the aBase[] +** lookup table. +** +** (2) For p outside the range 0 to 26, use aScale[] for the initial value +** then refine that result (if necessary) by a single multiplication +** against aBase[]. */ -static void dekkerMul2(volatile double *x, double y, double yy){ - /* - ** The "volatile" keywords on parameter x[] and on local variables - ** below are needed force intermediate results to be truncated to - ** binary64 rather than be carried around in an extended-precision - ** format. The truncation is necessary for the Dekker algorithm to - ** work. Intel x86 floating point might omit the truncation without - ** the use of volatile. - */ - volatile double tx, ty, p, q, c, cc; - double hx, hy; - u64 m; - memcpy(&m, (void*)&x[0], 8); - m &= 0xfffffffffc000000LL; - memcpy(&hx, &m, 8); - tx = x[0] - hx; - memcpy(&m, &y, 8); - m &= 0xfffffffffc000000LL; - memcpy(&hy, &m, 8); - ty = y - hy; - p = hx*hy; - q = hx*ty + tx*hy; - c = p+q; - cc = p - c + q + tx*ty; - cc = x[0]*yy + x[1]*y + cc; - x[0] = c + cc; - x[1] = c - x[0]; - x[1] += cc; +static u64 powerOfTen(int p){ + static const u64 aBase[] = { + 0x8000000000000000LLU, /* 0: 1.0e+0 << 63 */ + 0xa000000000000000LLU, /* 1: 1.0e+1 << 60 */ + 0xc800000000000000LLU, /* 2: 1.0e+2 << 57 */ + 0xfa00000000000000LLU, /* 3: 1.0e+3 << 54 */ + 0x9c40000000000000LLU, /* 4: 1.0e+4 << 50 */ + 0xc350000000000000LLU, /* 5: 1.0e+5 << 47 */ + 0xf424000000000000LLU, /* 6: 1.0e+6 << 44 */ + 0x9896800000000000LLU, /* 7: 1.0e+7 << 40 */ + 0xbebc200000000000LLU, /* 8: 1.0e+8 << 37 */ + 0xee6b280000000000LLU, /* 9: 1.0e+9 << 34 */ + 0x9502f90000000000LLU, /* 10: 1.0e+10 << 30 */ + 0xba43b74000000000LLU, /* 11: 1.0e+11 << 27 */ + 0xe8d4a51000000000LLU, /* 12: 1.0e+12 << 24 */ + 0x9184e72a00000000LLU, /* 13: 1.0e+13 << 20 */ + 0xb5e620f480000000LLU, /* 14: 1.0e+14 << 17 */ + 0xe35fa931a0000000LLU, /* 15: 1.0e+15 << 14 */ + 0x8e1bc9bf04000000LLU, /* 16: 1.0e+16 << 10 */ + 0xb1a2bc2ec5000000LLU, /* 17: 1.0e+17 << 7 */ + 0xde0b6b3a76400000LLU, /* 18: 1.0e+18 << 4 */ + 0x8ac7230489e80000LLU, /* 19: 1.0e+19 >> 0 */ + 0xad78ebc5ac620000LLU, /* 20: 1.0e+20 >> 3 */ + 0xd8d726b7177a8000LLU, /* 21: 1.0e+21 >> 6 */ + 0x878678326eac9000LLU, /* 22: 1.0e+22 >> 10 */ + 0xa968163f0a57b400LLU, /* 23: 1.0e+23 >> 13 */ + 0xd3c21bcecceda100LLU, /* 24: 1.0e+24 >> 16 */ + 0x84595161401484a0LLU, /* 25: 1.0e+25 >> 20 */ + 0xa56fa5b99019a5c8LLU, /* 26: 1.0e+26 >> 23 */ + }; + static const u64 aScale[] = { + 0x8049a4ac0c5811aeLLU, /* 0: 1.0e-351 << 1229 */ + 0xcf42894a5dce35eaLLU, /* 1: 1.0e-324 << 1140 */ + 0xa76c582338ed2622LLU, /* 2: 1.0e-297 << 1050 */ + 0x873e4f75e2224e68LLU, /* 3: 1.0e-270 << 960 */ + 0xda7f5bf590966849LLU, /* 4: 1.0e-243 << 871 */ + 0xb080392cc4349dedLLU, /* 5: 1.0e-216 << 781 */ + 0x8e938662882af53eLLU, /* 6: 1.0e-189 << 691 */ + 0xe65829b3046b0afaLLU, /* 7: 1.0e-162 << 602 */ + 0xba121a4650e4ddecLLU, /* 8: 1.0e-135 << 512 */ + 0x964e858c91ba2655LLU, /* 9: 1.0e-108 << 422 */ + 0xf2d56790ab41c2a3LLU, /* 10: 1.0e-81 << 333 */ + 0xc428d05aa4751e4dLLU, /* 11: 1.0e-54 << 243 */ + 0x9e74d1b791e07e48LLU, /* 12: 1.0e-27 << 153 */ + 0x8000000000000000LLU, /* 13: 1.0e+0 << 63 */ + 0xcecb8f27f4200f3aLLU, /* 14: 1.0e+27 >> 26 */ + 0xa70c3c40a64e6c52LLU, /* 15: 1.0e+54 >> 116 */ + 0x86f0ac99b4e8dafdLLU, /* 16: 1.0e+81 >> 206 */ + 0xda01ee641a708deaLLU, /* 17: 1.0e+108 >> 295 */ + 0xb01ae745b101e9e4LLU, /* 18: 1.0e+135 >> 385 */ + 0x8e41ade9fbebc27dLLU, /* 19: 1.0e+162 >> 475 */ + 0xe5d3ef282a242e82LLU, /* 20: 1.0e+189 >> 564 */ + 0xb9a74a0637ce2ee1LLU, /* 21: 1.0e+216 >> 654 */ + 0x95f83d0a1fb69cd9LLU, /* 22: 1.0e+243 >> 744 */ + 0xf24a01a73cf2dcd0LLU, /* 23: 1.0e+270 >> 833 */ + 0xc3b8358109e84f07LLU, /* 24: 1.0e+297 >> 923 */ + 0x9e19db92b4e31ba9LLU, /* 25: 1.0e+324 >> 1013 */ + }; + int g, n; + u64 x, y; + + assert( p>=POWERSOF10_FIRST && p<=POWERSOF10_LAST ); + if( p<0 ){ + g = p/27; + n = p%27; + if( n ){ + g--; + n += 27; + } + }else if( p<27 ){ + return aBase[p]; + }else{ + g = p/27; + n = p%27; + } + y = aScale[g+13]; + if( n==0 ){ + return y; + } + x = sqlite3Multiply128(aBase[n],y); + if( (U64_BIT(63) & x)==0 ){ + x = (x<<1)|1; + } + return x; +} + +/* +** pow10to2(x) computes floor(log2(pow(10,x))). +** pow2to10(y) computes floor(log10(pow(2,y))). +** +** Conceptually, pow10to2(p) converts a base-10 exponent p into +** a corresponding base-2 exponent, and pow2to10(e) converts a base-2 +** exponent into a base-10 exponent. +** +** The conversions are based on the observation that: +** +** ln(10.0)/ln(2.0) == 108853/32768 (approximately) +** ln(2.0)/ln(10.0) == 78913/262144 (approximately) +** +** These ratios are approximate, but they are accurate to 5 digits, +** which is close enough for the usage here. Right-shift is used +** for division so that rounding of negative numbers happens in the +** right direction. +*/ +static int pwr10to2(int p){ return (p*108853) >> 15; } +static int pwr2to10(int p){ return (p*78913) >> 18; } + +/* +** Count leading zeros for a 64-bit unsigned integer. +*/ +static int countLeadingZeros(u64 m){ +#if defined(__GNUC__) || defined(__clang__) + return __builtin_clzll(m); +#else + int n = 0; + if( m <= 0x00000000ffffffffULL) { n += 32; m <<= 32; } + if( m <= 0x0000ffffffffffffULL) { n += 16; m <<= 16; } + if( m <= 0x00ffffffffffffffULL) { n += 8; m <<= 8; } + if( m <= 0x0fffffffffffffffULL) { n += 4; m <<= 4; } + if( m <= 0x3fffffffffffffffULL) { n += 2; m <<= 2; } + if( m <= 0x7fffffffffffffffULL) { n += 1; } + return n; +#endif +} + +/* +** Given m and e, which represent a quantity r == m*pow(2,e), +** return values *pD and *pP such that r == (*pD)*pow(10,*pP), +** approximately. *pD should contain at least n significant digits. +** +** The input m is required to have its highest bit set. In other words, +** m should be left-shifted, and e decremented, to maximize the value of m. +*/ +static void sqlite3Fp2Convert10(u64 m, int e, int n, u64 *pD, int *pP){ + int p; + u64 h; + assert( n>=1 && n<=18 ); + p = n - 1 - pwr2to10(e+63); + h = sqlite3Multiply128(m, powerOfTen(p)); + assert( -(e + pwr10to2(p) + 2) >= 0 ); + assert( -(e + pwr10to2(p) + 1) <= 63 ); + if( n==18 ){ + h >>= -(e + pwr10to2(p) + 2); + *pD = (h + ((h<<1)&2))>>1; + }else{ + *pD = h >> -(e + pwr10to2(p) + 1); + } + *pP = -p; +} + +/* +** Return an IEEE754 floating point value that approximates d*pow(10,p). +*/ +static double sqlite3Fp10Convert2(u64 d, int p){ + u64 out; + int e1; + int lz; + int lp; + int x; + u64 h; + double r; + assert( (d & U64_BIT(63))==0 ); + assert( d!=0 ); + if( p<POWERSOF10_FIRST ){ + return 0.0; + } + if( p>POWERSOF10_LAST ){ + return INFINITY; + } + lz = countLeadingZeros(d); + lp = pwr10to2(p); + e1 = lz - (lp + 11); + if( e1>1074 ){ + if( e1>=1130 ) return 0.0; + e1 = 1074; + } + h = sqlite3Multiply128(d<<lz, powerOfTen(p)); + x = lz - (e1 + lp + 3); + assert( x >= 0 ); + assert( x <= 63 ); + out = h >> x; + if( out >= U64_BIT(55)-2 ){ + out >>= 1; + e1--; + } + if( e1<=(-972) ){ + return INFINITY; + } + out = (out + 2) >> 2; + if( (out & U64_BIT(52))!=0 ){ + out = (out & ~U64_BIT(52)) | ((u64)(1075-e1)<<52); + } + memcpy(&r, &out, 8); + return r; } /* ** The string z[] is an text representation of a real number. ** Convert this string to a double and write it into *pResult. ** -** The string z[] is length bytes in length (bytes, not characters) and -** uses the encoding enc. The string is not necessarily zero-terminated. +** z[] must be UTF-8 and zero-terminated. ** ** Return TRUE if the result is a valid real number (or integer) and FALSE ** if the string is empty or contains extraneous text. More specifically @@ -526,198 +740,131 @@ static void dekkerMul2(volatile double *x, double y, double yy){ #if defined(_MSC_VER) #pragma warning(disable : 4756) #endif -int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ +int sqlite3AtoF(const char *z, double *pResult){ #ifndef SQLITE_OMIT_FLOATING_POINT - int incr; - const char *zEnd; /* sign * significand * (10 ^ (esign * exponent)) */ - int sign = 1; /* sign of significand */ - u64 s = 0; /* significand */ - int d = 0; /* adjust exponent for shifting decimal point */ - int esign = 1; /* sign of exponent */ - int e = 0; /* exponent */ - int eValid = 1; /* True exponent is either not used or is well-formed */ + int neg = 0; /* True for a negative value */ + u64 s = 0; /* mantissa */ + int d = 0; /* Value is s * pow(10,d) */ int nDigit = 0; /* Number of digits processed */ - int eType = 1; /* 1: pure integer, 2+: fractional -1 or less: bad UTF16 */ - u64 s2; /* round-tripped significand */ - double rr[2]; + int eType = 1; /* 1: pure integer, 2+: fractional */ - assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); *pResult = 0.0; /* Default return value, in case of an error */ - if( length==0 ) return 0; - - if( enc==SQLITE_UTF8 ){ - incr = 1; - zEnd = z + length; - }else{ - int i; - incr = 2; - length &= ~1; - assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); - testcase( enc==SQLITE_UTF16LE ); - testcase( enc==SQLITE_UTF16BE ); - for(i=3-enc; i<length && z[i]==0; i+=2){} - if( i<length ) eType = -100; - zEnd = &z[i^1]; - z += (enc&1); - } /* skip leading spaces */ - while( z<zEnd && sqlite3Isspace(*z) ) z+=incr; - if( z>=zEnd ) return 0; + while( sqlite3Isspace(*z) ) z++; /* get sign of significand */ if( *z=='-' ){ - sign = -1; - z+=incr; + neg = 1; + z++; }else if( *z=='+' ){ - z+=incr; + z++; } /* copy max significant digits to significand */ - while( z<zEnd && sqlite3Isdigit(*z) ){ + while( sqlite3Isdigit(*z) ){ s = s*10 + (*z - '0'); - z+=incr; nDigit++; - if( s>=((LARGEST_UINT64-9)/10) ){ + z++; nDigit++; + if( s>=((LARGEST_INT64-9)/10) ){ /* skip non-significant significand digits ** (increase exponent by d to shift decimal left) */ - while( z<zEnd && sqlite3Isdigit(*z) ){ z+=incr; d++; } + while( sqlite3Isdigit(*z) ){ z++; d++; } } } - if( z>=zEnd ) goto do_atof_calc; /* if decimal point is present */ if( *z=='.' ){ - z+=incr; + z++; eType++; /* copy digits from after decimal to significand ** (decrease exponent by d to shift decimal right) */ - while( z<zEnd && sqlite3Isdigit(*z) ){ - if( s<((LARGEST_UINT64-9)/10) ){ + while( sqlite3Isdigit(*z) ){ + if( s<((LARGEST_INT64-9)/10) ){ s = s*10 + (*z - '0'); d--; nDigit++; } - z+=incr; + z++; } } - if( z>=zEnd ) goto do_atof_calc; /* if exponent is present */ if( *z=='e' || *z=='E' ){ - z+=incr; - eValid = 0; + int esign = 1; /* sign of exponent */ + z++; eType++; - /* This branch is needed to avoid a (harmless) buffer overread. The - ** special comment alerts the mutation tester that the correct answer - ** is obtained even if the branch is omitted */ - if( z>=zEnd ) goto do_atof_calc; /*PREVENTS-HARMLESS-OVERREAD*/ - /* get sign of exponent */ if( *z=='-' ){ esign = -1; - z+=incr; + z++; }else if( *z=='+' ){ - z+=incr; + z++; } /* copy digits to exponent */ - while( z<zEnd && sqlite3Isdigit(*z) ){ - e = e<10000 ? (e*10 + (*z - '0')) : 10000; - z+=incr; - eValid = 1; + if( sqlite3Isdigit(*z) ){ + int exp = *z - '0'; + z++; + while( sqlite3Isdigit(*z) ){ + exp = exp<10000 ? (exp*10 + (*z - '0')) : 10000; + z++; + } + d += esign*exp; + }else{ + eType = -1; } } /* skip trailing spaces */ - while( z<zEnd && sqlite3Isspace(*z) ) z+=incr; + while( sqlite3Isspace(*z) ) z++; -do_atof_calc: /* Zero is a special case */ if( s==0 ){ - *pResult = sign<0 ? -0.0 : +0.0; - goto atof_return; - } - - /* adjust exponent by d, and update sign */ - e = (e*esign) + d; - - /* Try to adjust the exponent to make it smaller */ - while( e>0 && s<((LARGEST_UINT64-0x7ff)/10) ){ - s *= 10; - e--; - } - while( e<0 && (s%10)==0 ){ - s /= 10; - e++; - } - - rr[0] = (double)s; - assert( sizeof(s2)==sizeof(rr[0]) ); -#ifdef SQLITE_DEBUG - rr[1] = 18446744073709549568.0; - memcpy(&s2, &rr[1], sizeof(s2)); - assert( s2==0x43efffffffffffffLL ); -#endif - /* Largest double that can be safely converted to u64 - ** vvvvvvvvvvvvvvvvvvvvvv */ - if( rr[0]<=18446744073709549568.0 ){ - s2 = (u64)rr[0]; - rr[1] = s>=s2 ? (double)(s - s2) : -(double)(s2 - s); - }else{ - rr[1] = 0.0; - } - assert( rr[1]<=1.0e-10*rr[0] ); /* Equal only when rr[0]==0.0 */ - - if( e>0 ){ - while( e>=100 ){ - e -= 100; - dekkerMul2(rr, 1.0e+100, -1.5902891109759918046e+83); - } - while( e>=10 ){ - e -= 10; - dekkerMul2(rr, 1.0e+10, 0.0); - } - while( e>=1 ){ - e -= 1; - dekkerMul2(rr, 1.0e+01, 0.0); - } + *pResult = neg ? -0.0 : +0.0; }else{ - while( e<=-100 ){ - e += 100; - dekkerMul2(rr, 1.0e-100, -1.99918998026028836196e-117); - } - while( e<=-10 ){ - e += 10; - dekkerMul2(rr, 1.0e-10, -3.6432197315497741579e-27); - } - while( e<=-1 ){ - e += 1; - dekkerMul2(rr, 1.0e-01, -5.5511151231257827021e-18); - } + *pResult = sqlite3Fp10Convert2(s,d); + if( neg ) *pResult = -*pResult; + assert( !sqlite3IsNaN(*pResult) ); } - *pResult = rr[0]+rr[1]; - if( sqlite3IsNaN(*pResult) ) *pResult = 1e300*1e300; - if( sign<0 ) *pResult = -*pResult; - assert( !sqlite3IsNaN(*pResult) ); -atof_return: /* return true if number and no extra non-whitespace characters after */ - if( z==zEnd && nDigit>0 && eValid && eType>0 ){ + if( z[0]==0 && nDigit>0 ){ return eType; - }else if( eType>=2 && (eType==3 || eValid) && nDigit>0 ){ + }else if( eType>=2 && nDigit>0 ){ return -1; }else{ return 0; } #else - return !sqlite3Atoi64(z, pResult, length, enc); + return !sqlite3Atoi64(z, pResult, strlen(z), SQLITE_UTF8); #endif /* SQLITE_OMIT_FLOATING_POINT */ } #if defined(_MSC_VER) #pragma warning(default : 4756) #endif +/* +** Digit pairs used to convert a U64 or I64 into text, two digits +** at a time. +*/ +static const union { + char a[201]; + short int forceAlignment; +} sqlite3DigitPairs = { + "00010203040506070809" + "10111213141516171819" + "20212223242526272829" + "30313233343536373839" + "40414243444546474849" + "50515253545556575859" + "60616263646566676869" + "70717273747576777879" + "80818283848586878889" + "90919293949596979899" +}; + + /* ** Render an signed 64-bit integer as text. Store the result in zOut[] and ** return the length of the string that was stored, in bytes. The value @@ -729,23 +876,35 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ int sqlite3Int64ToText(i64 v, char *zOut){ int i; u64 x; - char zTemp[22]; - if( v<0 ){ - x = (v==SMALLEST_INT64) ? ((u64)1)<<63 : (u64)-v; - }else{ + union { + char a[23]; + u16 forceAlignment; + } u; + if( v>0 ){ x = v; + }else if( v==0 ){ + zOut[0] = '0'; + zOut[1] = 0; + return 1; + }else{ + x = (v==SMALLEST_INT64) ? ((u64)1)<<63 : (u64)-v; } - i = sizeof(zTemp)-2; - zTemp[sizeof(zTemp)-1] = 0; - while( 1 /*exit-by-break*/ ){ - zTemp[i] = (x%10) + '0'; - x = x/10; - if( x==0 ) break; - i--; - }; - if( v<0 ) zTemp[--i] = '-'; - memcpy(zOut, &zTemp[i], sizeof(zTemp)-i); - return sizeof(zTemp)-1-i; + i = sizeof(u.a)-1; + u.a[i] = 0; + while( x>=10 ){ + int kk = (x%100)*2; + assert( TWO_BYTE_ALIGNMENT(&sqlite3DigitPairs.a[kk]) ); + assert( TWO_BYTE_ALIGNMENT(&u.a[i-2]) ); + *(u16*)(&u.a[i-2]) = *(u16*)&sqlite3DigitPairs.a[kk]; + i -= 2; + x /= 100; + } + if( x ){ + u.a[--i] = x + '0'; + } + if( v<0 ) u.a[--i] = '-'; + memcpy(zOut, &u.a[i], sizeof(u.a)-i); + return sizeof(u.a)-1-i; } /* @@ -1002,7 +1161,7 @@ int sqlite3Atoi(const char *z){ ** representation. ** ** If iRound<=0 then round to -iRound significant digits to the -** the left of the decimal point, or to a maximum of mxRound total +** the right of the decimal point, or to a maximum of mxRound total ** significant digits. ** ** If iRound>0 round to min(iRound,mxRound) significant digits total. @@ -1015,13 +1174,14 @@ int sqlite3Atoi(const char *z){ ** The p->z[] array is *not* zero-terminated. */ void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRound){ - int i; - u64 v; - int e, exp = 0; - double rr[2]; + int i; /* Index into zBuf[] where to put next character */ + int n; /* Number of digits */ + u64 v; /* mantissa */ + int e, exp = 0; /* Base-2 and base-10 exponent */ + char *zBuf; /* Local alias for p->zBuf */ + char *z; /* Local alias for p->z */ p->isSpecial = 0; - p->z = p->zBuf; assert( mxRound>0 ); /* Convert negative numbers to positive. Deal with Infinity, 0.0, and @@ -1039,78 +1199,94 @@ void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRound){ p->sign = '+'; } memcpy(&v,&r,8); - e = v>>52; - if( (e&0x7ff)==0x7ff ){ + e = (v>>52)&0x7ff; + if( e==0x7ff ){ p->isSpecial = 1 + (v!=0x7ff0000000000000LL); p->n = 0; p->iDP = 0; + p->z = p->zBuf; return; } - - /* Multiply r by powers of ten until it lands somewhere in between - ** 1.0e+19 and 1.0e+17. - ** - ** Use Dekker-style double-double computation to increase the - ** precision. - ** - ** The error terms on constants like 1.0e+100 computed using the - ** decimal extension, for example as follows: - ** - ** SELECT decimal_exp(decimal_sub('1.0e+100',decimal(1.0e+100))); - */ - rr[0] = r; - rr[1] = 0.0; - if( rr[0]>9.223372036854774784e+18 ){ - while( rr[0]>9.223372036854774784e+118 ){ - exp += 100; - dekkerMul2(rr, 1.0e-100, -1.99918998026028836196e-117); - } - while( rr[0]>9.223372036854774784e+28 ){ - exp += 10; - dekkerMul2(rr, 1.0e-10, -3.6432197315497741579e-27); - } - while( rr[0]>9.223372036854774784e+18 ){ - exp += 1; - dekkerMul2(rr, 1.0e-01, -5.5511151231257827021e-18); - } + v &= 0x000fffffffffffffULL; + if( e==0 ){ + int nn = countLeadingZeros(v); + v <<= nn; + e = -1074 - nn; }else{ - while( rr[0]<9.223372036854774784e-83 ){ - exp -= 100; - dekkerMul2(rr, 1.0e+100, -1.5902891109759918046e+83); - } - while( rr[0]<9.223372036854774784e+07 ){ - exp -= 10; - dekkerMul2(rr, 1.0e+10, 0.0); - } - while( rr[0]<9.22337203685477478e+17 ){ - exp -= 1; - dekkerMul2(rr, 1.0e+01, 0.0); - } + v = (v<<11) | U64_BIT(63); + e -= 1086; } - v = rr[1]<0.0 ? (u64)rr[0]-(u64)(-rr[1]) : (u64)rr[0]+(u64)rr[1]; + sqlite3Fp2Convert10(v, e, (iRound<=0||iRound>=18)?18:iRound+1, &v, &exp); - /* Extract significant digits. */ + /* Extract significant digits, start at the right-most slot in p->zBuf + ** and working back to the right. "i" keeps track of the next slot in + ** which to store a digit. */ i = sizeof(p->zBuf)-1; + zBuf = p->zBuf; assert( v>0 ); - while( v ){ p->zBuf[i--] = (v%10) + '0'; v /= 10; } + while( v>=10 ){ + int kk = (v%100)*2; + assert( TWO_BYTE_ALIGNMENT(&sqlite3DigitPairs.a[kk]) ); + assert( TWO_BYTE_ALIGNMENT(&zBuf[i-1]) ); + *(u16*)(&zBuf[i-1]) = *(u16*)&sqlite3DigitPairs.a[kk]; + i -= 2; + v /= 100; + } + if( v ){ + assert( v<10 ); + zBuf[i--] = v + '0'; + } assert( i>=0 && i<sizeof(p->zBuf)-1 ); - p->n = sizeof(p->zBuf) - 1 - i; - assert( p->n>0 ); - assert( p->n<sizeof(p->zBuf) ); - p->iDP = p->n + exp; + n = sizeof(p->zBuf) - 1 - i; /* Total number of digits extracted */ + assert( n>0 ); + assert( n<sizeof(p->zBuf) ); + testcase( n==sizeof(p->zBuf)-1 ); + p->iDP = n + exp; if( iRound<=0 ){ iRound = p->iDP - iRound; - if( iRound==0 && p->zBuf[i+1]>='5' ){ + if( iRound==0 && zBuf[i+1]>='5' ){ iRound = 1; - p->zBuf[i--] = '0'; - p->n++; + zBuf[i--] = '0'; + n++; p->iDP++; } } - if( iRound>0 && (iRound<p->n || p->n>mxRound) ){ - char *z = &p->zBuf[i+1]; + z = &zBuf[i+1]; /* z points to the first digit */ + if( iRound>0 && (iRound<n || n>mxRound) ){ if( iRound>mxRound ) iRound = mxRound; - p->n = iRound; + if( iRound==17 ){ + /* If the precision is exactly 17, which only happens with the "!" + ** flag (ex: "%!.17g") then try to reduce the precision if that + ** yields text that will round-trip to the original floating-point. + ** value. Thus, for exaple, 49.47 will render as 49.47, rather than + ** as 49.469999999999999. */ + if( z[15]=='9' && z[14]=='9' ){ + int jj, kk; + u64 v2; + for(jj=14; jj>0 && z[jj-1]=='9'; jj--){} + if( jj==0 ){ + v2 = 1; + }else{ + v2 = z[0] - '0'; + for(kk=1; kk<jj; kk++) v2 = (v2*10) + z[kk] - '0'; + v2++; + } + if( r==sqlite3Fp10Convert2(v2, exp + n - jj) ){ + iRound = jj+1; + } + }else if( p->iDP>=n || (z[15]=='0' && z[14]=='0' && z[13]=='0') ){ + int jj, kk; + u64 v2; + assert( z[0]!='0' ); + for(jj=14; z[jj-1]=='0'; jj--){} + v2 = z[0] - '0'; + for(kk=1; kk<jj; kk++) v2 = (v2*10) + z[kk] - '0'; + if( r==sqlite3Fp10Convert2(v2, exp + n - jj) ){ + iRound = jj+1; + } + } + } + n = iRound; if( z[iRound]>='5' ){ int j = iRound-1; while( 1 /*exit-by-break*/ ){ @@ -1118,8 +1294,9 @@ void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRound){ if( z[j]<='9' ) break; z[j] = '0'; if( j==0 ){ - p->z[i--] = '1'; - p->n++; + z--; + z[0] = '1'; + n++; p->iDP++; break; }else{ @@ -1128,13 +1305,13 @@ void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRound){ } } } - p->z = &p->zBuf[i+1]; - assert( i+p->n < sizeof(p->zBuf) ); - assert( p->n>0 ); - while( p->z[p->n-1]=='0' ){ - p->n--; - assert( p->n>0 ); + assert( n>0 ); + while( z[n-1]=='0' ){ + n--; + assert( n>0 ); } + p->n = n; + p->z = z; } /* diff --git a/src/vacuum.c b/src/vacuum.c index 1b4838040..70e62e1ef 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -230,9 +230,11 @@ SQLITE_NOINLINE int sqlite3RunVacuum( pDb = &db->aDb[nDb]; assert( strcmp(pDb->zDbSName,zDbVacuum)==0 ); pTemp = pDb->pBt; + nRes = sqlite3BtreeGetRequestedReserve(pMain); if( pOut ){ sqlite3_file *id = sqlite3PagerFile(sqlite3BtreePager(pTemp)); i64 sz = 0; + const char *zFilename; if( id->pMethods!=0 && (sqlite3OsFileSize(id, &sz)!=SQLITE_OK || sz>0) ){ rc = SQLITE_ERROR; sqlite3SetString(pzErrMsg, db, "output file already exists"); @@ -244,8 +246,16 @@ SQLITE_NOINLINE int sqlite3RunVacuum( ** they are for the database being vacuumed, except that PAGER_CACHESPILL ** is always set. */ pgflags = db->aDb[iDb].safety_level | (db->flags & PAGER_FLAGS_MASK); + + /* If the VACUUM INTO target file is a URI filename and if the + ** "reserve=N" query parameter is present, reset the reserve to the + ** amount specified, if the amount is within range */ + zFilename = sqlite3BtreeGetFilename(pTemp); + if( ALWAYS(zFilename) ){ + int nNew = (int)sqlite3_uri_int64(zFilename, "reserve", nRes); + if( nNew>=0 && nNew<=255 ) nRes = nNew; + } } - nRes = sqlite3BtreeGetRequestedReserve(pMain); sqlite3BtreeSetCacheSize(pTemp, db->aDb[iDb].pSchema->cache_size); sqlite3BtreeSetSpillSize(pTemp, sqlite3BtreeSetSpillSize(pMain,0)); diff --git a/src/vdbe.c b/src/vdbe.c index b5a262e63..e2e98eb5f 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -353,10 +353,9 @@ static int alsoAnInt(Mem *pRec, double rValue, i64 *piValue){ */ static void applyNumericAffinity(Mem *pRec, int bTryForInt){ double rValue; - u8 enc = pRec->enc; int rc; assert( (pRec->flags & (MEM_Str|MEM_Int|MEM_Real|MEM_IntReal))==MEM_Str ); - rc = sqlite3AtoF(pRec->z, &rValue, pRec->n, enc); + rValue = sqlite3MemRealValueRC(pRec, &rc); if( rc<=0 ) return; if( rc==1 && alsoAnInt(pRec, rValue, &pRec->u.i) ){ pRec->flags |= MEM_Int; @@ -438,7 +437,10 @@ int sqlite3_value_numeric_type(sqlite3_value *pVal){ int eType = sqlite3_value_type(pVal); if( eType==SQLITE_TEXT ){ Mem *pMem = (Mem*)pVal; + assert( pMem->db!=0 ); + sqlite3_mutex_enter(pMem->db->mutex); applyNumericAffinity(pMem, 0); + sqlite3_mutex_leave(pMem->db->mutex); eType = sqlite3_value_type(pVal); } return eType; @@ -471,7 +473,7 @@ static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ pMem->u.i = 0; return MEM_Int; } - rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); + pMem->u.r = sqlite3MemRealValueRC(pMem, &rc); if( rc<=0 ){ if( rc==0 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1 ){ pMem->u.i = ix; @@ -6620,20 +6622,15 @@ case OP_SorterInsert: { /* in2 */ break; } -/* Opcode: IdxDelete P1 P2 P3 * P5 +/* Opcode: IdxDelete P1 P2 P3 * * ** Synopsis: key=r[P2@P3] ** ** The content of P3 registers starting at register P2 form ** an unpacked index key. This opcode removes that entry from the ** index opened by cursor P1. ** -** If P5 is not zero, then raise an SQLITE_CORRUPT_INDEX error -** if no matching index entry is found. This happens when running -** an UPDATE or DELETE statement and the index entry to be updated -** or deleted is not found. For some uses of IdxDelete -** (example: the EXCEPT operator) it does not matter that no matching -** entry is found. For those cases, P5 is zero. Also, do not raise -** this (self-correcting and non-critical) error if in writable_schema mode. +** Raise an SQLITE_CORRUPT_INDEX error if no matching index entry is found +** and not in writable_schema mode. */ case OP_IdxDelete: { VdbeCursor *pC; @@ -6659,7 +6656,7 @@ case OP_IdxDelete: { if( res==0 ){ rc = sqlite3BtreeDelete(pCrsr, BTREE_AUXDELETE); if( rc ) goto abort_due_to_error; - }else if( pOp->p5 && !sqlite3WritableSchema(db) ){ + }else if( !sqlite3WritableSchema(db) ){ rc = sqlite3ReportError(SQLITE_CORRUPT_INDEX, __LINE__, "index corruption"); goto abort_due_to_error; } diff --git a/src/vdbe.h b/src/vdbe.h index 28df764bc..a2905eae4 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -186,7 +186,7 @@ typedef struct VdbeOpList VdbeOpList; ** Additional non-public SQLITE_PREPARE_* flags */ #define SQLITE_PREPARE_SAVESQL 0x80 /* Preserve SQL text */ -#define SQLITE_PREPARE_MASK 0x1f /* Mask of public flags */ +#define SQLITE_PREPARE_MASK 0x3f /* Mask of public flags */ /* ** Prototypes for the VDBE interface. See comments on the implementation diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 8b68c339a..320721d06 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -630,6 +630,7 @@ void sqlite3VdbeMemShallowCopy(Mem*, const Mem*, int); void sqlite3VdbeMemMove(Mem*, Mem*); int sqlite3VdbeMemNulTerminate(Mem*); int sqlite3VdbeMemSetStr(Mem*, const char*, i64, u8, void(*)(void*)); +int sqlite3VdbeMemSetText(Mem*, const char*, i64, void(*)(void*)); void sqlite3VdbeMemSetInt64(Mem*, i64); #ifdef SQLITE_OMIT_FLOATING_POINT # define sqlite3VdbeMemSetDouble sqlite3VdbeMemSetInt64 @@ -648,13 +649,14 @@ int sqlite3VdbeMemSetZeroBlob(Mem*,int); int sqlite3VdbeMemIsRowSet(const Mem*); #endif int sqlite3VdbeMemSetRowSet(Mem*); -void sqlite3VdbeMemZeroTerminateIfAble(Mem*); +int sqlite3VdbeMemZeroTerminateIfAble(Mem*); int sqlite3VdbeMemMakeWriteable(Mem*); int sqlite3VdbeMemStringify(Mem*, u8, u8); int sqlite3IntFloatCompare(i64,double); i64 sqlite3VdbeIntValue(const Mem*); int sqlite3VdbeMemIntegerify(Mem*); double sqlite3VdbeRealValue(Mem*); +SQLITE_NOINLINE double sqlite3MemRealValueRC(Mem*, int*); int sqlite3VdbeBooleanValue(Mem*, int ifNull); void sqlite3VdbeIntegerAffinity(Mem*); int sqlite3VdbeMemRealify(Mem*); diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 1118481d1..9fd4715ce 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -392,7 +392,23 @@ static void setResultStrOrError( void (*xDel)(void*) /* Destructor function */ ){ Mem *pOut = pCtx->pOut; - int rc = sqlite3VdbeMemSetStr(pOut, z, n, enc, xDel); + int rc; + if( enc==SQLITE_UTF8 ){ + rc = sqlite3VdbeMemSetText(pOut, z, n, xDel); + }else if( enc==SQLITE_UTF8_ZT ){ + /* It is usually considered improper to assert() on an input. However, + ** the following assert() is checking for inputs that are documented + ** to result in undefined behavior. */ + assert( z==0 + || n<0 + || n>pOut->db->aLimit[SQLITE_LIMIT_LENGTH] + || z[n]==0 + ); + rc = sqlite3VdbeMemSetText(pOut, z, n, xDel); + pOut->flags |= MEM_Term; + }else{ + rc = sqlite3VdbeMemSetStr(pOut, z, n, enc, xDel); + } if( rc ){ if( rc==SQLITE_TOOBIG ){ sqlite3_result_error_toobig(pCtx); @@ -585,7 +601,7 @@ void sqlite3_result_text64( #endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); assert( xDel!=SQLITE_DYNAMIC ); - if( enc!=SQLITE_UTF8 ){ + if( enc!=SQLITE_UTF8 && enc!=SQLITE_UTF8_ZT ){ if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; n &= ~(u64)1; } @@ -736,6 +752,8 @@ static int doWalCallbacks(sqlite3 *db){ } } } +#else + UNUSED_PARAMETER(db); #endif return rc; } @@ -1043,7 +1061,7 @@ static int valueFromValueList( Mem sMem; /* Raw content of current row */ memset(&sMem, 0, sizeof(sMem)); sz = sqlite3BtreePayloadSize(pRhs->pCsr); - rc = sqlite3VdbeMemFromBtreeZeroOffset(pRhs->pCsr,(int)sz,&sMem); + rc = sqlite3VdbeMemFromBtreeZeroOffset(pRhs->pCsr,sz,&sMem); if( rc==SQLITE_OK ){ u8 *zBuf = (u8*)sMem.z; u32 iSerial; @@ -1692,13 +1710,25 @@ static int bindText( assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */ if( zData!=0 ){ pVar = &p->aVar[i-1]; - rc = sqlite3VdbeMemSetStr(pVar, zData, nData, encoding, xDel); - if( rc==SQLITE_OK ){ - if( encoding==0 ){ - pVar->enc = ENC(p->db); - }else{ - rc = sqlite3VdbeChangeEncoding(pVar, ENC(p->db)); - } + if( encoding==SQLITE_UTF8 ){ + rc = sqlite3VdbeMemSetText(pVar, zData, nData, xDel); + }else if( encoding==SQLITE_UTF8_ZT ){ + /* It is usually consider improper to assert() on an input. + ** However, the following assert() is checking for inputs + ** that are documented to result in undefined behavior. */ + assert( zData==0 + || nData<0 + || nData>pVar->db->aLimit[SQLITE_LIMIT_LENGTH] + || ((u8*)zData)[nData]==0 + ); + rc = sqlite3VdbeMemSetText(pVar, zData, nData, xDel); + pVar->flags |= MEM_Term; + }else{ + rc = sqlite3VdbeMemSetStr(pVar, zData, nData, encoding, xDel); + if( encoding==0 ) pVar->enc = ENC(p->db); + } + if( rc==SQLITE_OK && encoding!=0 ){ + rc = sqlite3VdbeChangeEncoding(pVar, ENC(p->db)); } if( rc ){ sqlite3Error(p->db, rc); @@ -1810,7 +1840,7 @@ int sqlite3_bind_text64( unsigned char enc ){ assert( xDel!=SQLITE_DYNAMIC ); - if( enc!=SQLITE_UTF8 ){ + if( enc!=SQLITE_UTF8 && enc!=SQLITE_UTF8_ZT ){ if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; nData &= ~(u64)1; } diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 5368c0c42..603e85ddf 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2901,7 +2901,7 @@ int sqlite3VdbeSetColName( } assert( p->aColName!=0 ); pColName = &(p->aColName[idx+var*p->nResAlloc]); - rc = sqlite3VdbeMemSetStr(pColName, zName, -1, SQLITE_UTF8, xDel); + rc = sqlite3VdbeMemSetText(pColName, zName, -1, xDel); assert( rc!=0 || !zName || (pColName->flags&MEM_Term)!=0 ); return rc; } diff --git a/src/vdbemem.c b/src/vdbemem.c index 2c4d1c568..5689cb755 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -107,21 +107,27 @@ static void vdbeMemRenderNum(int sz, char *zBuf, Mem *p){ StrAccum acc; assert( p->flags & (MEM_Int|MEM_Real|MEM_IntReal) ); assert( sz>22 ); - if( p->flags & MEM_Int ){ -#if GCC_VERSION>=7000000 - /* Work-around for GCC bug - ** https://gcc.gnu.org/bugzilla/show_bug.cgi?id=96270 */ + if( p->flags & (MEM_Int|MEM_IntReal) ){ +#if GCC_VERSION>=7000000 && GCC_VERSION<15000000 && defined(__i386__) + /* Work-around for GCC bug or bugs: + ** https://gcc.gnu.org/bugzilla/show_bug.cgi?id=96270 + ** https://gcc.gnu.org/bugzilla/show_bug.cgi?id=114659 + ** The problem appears to be fixed in GCC 15 */ i64 x; - assert( (p->flags&MEM_Int)*2==sizeof(x) ); - memcpy(&x, (char*)&p->u, (p->flags&MEM_Int)*2); + assert( (MEM_Str&~p->flags)*4==sizeof(x) ); + memcpy(&x, (char*)&p->u, (MEM_Str&~p->flags)*4); p->n = sqlite3Int64ToText(x, zBuf); #else p->n = sqlite3Int64ToText(p->u.i, zBuf); #endif + if( p->flags & MEM_IntReal ){ + memcpy(zBuf+p->n,".0", 3); + p->n += 2; + } }else{ sqlite3StrAccumInit(&acc, 0, zBuf, sz, 0); - sqlite3_str_appendf(&acc, "%!.15g", - (p->flags & MEM_IntReal)!=0 ? (double)p->u.i : p->u.r); + sqlite3_str_appendf(&acc, "%!.*g", + (p->db ? p->db->nFpDigit : 17), p->u.r); assert( acc.zText==zBuf && acc.mxAlloc<=0 ); zBuf[acc.nChar] = 0; /* Fast version of sqlite3StrAccumFinish(&acc) */ p->n = acc.nChar; @@ -170,6 +176,9 @@ int sqlite3VdbeMemValidStrRep(Mem *p){ assert( p->enc==SQLITE_UTF8 || p->z[((p->n+1)&~1)+1]==0 ); } if( (p->flags & (MEM_Int|MEM_Real|MEM_IntReal))==0 ) return 1; + if( p->db==0 ){ + return 1; /* db->nFpDigit required to validate p->z[] */ + } memcpy(&tmp, p, sizeof(tmp)); vdbeMemRenderNum(sizeof(zBuf), zBuf, &tmp); z = p->z; @@ -320,13 +329,16 @@ int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ ** ** This is an optimization. Correct operation continues even if ** this routine is a no-op. +** +** Return true if the strig is zero-terminated after this routine is +** called and false if it is not. */ -void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ +int sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ if( (pMem->flags & (MEM_Str|MEM_Term|MEM_Ephem|MEM_Static))!=MEM_Str ){ /* pMem must be a string, and it cannot be an ephemeral or static string */ - return; + return 0; } - if( pMem->enc!=SQLITE_UTF8 ) return; + if( pMem->enc!=SQLITE_UTF8 ) return 0; assert( pMem->z!=0 ); if( pMem->flags & MEM_Dyn ){ if( pMem->xDel==sqlite3_free @@ -334,18 +346,19 @@ void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ ){ pMem->z[pMem->n] = 0; pMem->flags |= MEM_Term; - return; + return 1; } if( pMem->xDel==sqlite3RCStrUnref ){ /* Blindly assume that all RCStr objects are zero-terminated */ pMem->flags |= MEM_Term; - return; + return 1; } }else if( pMem->szMalloc >= pMem->n+1 ){ pMem->z[pMem->n] = 0; pMem->flags |= MEM_Term; - return; + return 1; } + return 0; } /* @@ -643,18 +656,70 @@ i64 sqlite3VdbeIntValue(const Mem *pMem){ } } +/* +** Invoke sqlite3AtoF() on the text value of pMem and return the +** double result. If sqlite3AtoF() returns an error code, write +** that code into *pRC if (*pRC)!=NULL. +** +** The caller must ensure that pMem->db!=0 and that pMem is in +** mode MEM_Str or MEM_Blob. +*/ +SQLITE_NOINLINE double sqlite3MemRealValueRC(Mem *pMem, int *pRC){ + double val = (double)0; + int rc = 0; + assert( pMem->db!=0 ); + assert( pMem->flags & (MEM_Str|MEM_Blob) ); + if( pMem->z==0 ){ + /* no-op */ + }else if( pMem->enc==SQLITE_UTF8 + && ((pMem->flags & MEM_Term)!=0 || sqlite3VdbeMemZeroTerminateIfAble(pMem)) + ){ + rc = sqlite3AtoF(pMem->z, &val); + }else if( pMem->n==0 ){ + /* no-op */ + }else if( pMem->enc==SQLITE_UTF8 ){ + char *zCopy = sqlite3DbStrNDup(pMem->db, pMem->z, pMem->n); + if( zCopy ){ + rc = sqlite3AtoF(zCopy, &val); + sqlite3DbFree(pMem->db, zCopy); + } + }else{ + int n, i, j; + char *zCopy; + const char *z; + + n = pMem->n & ~1; + zCopy = sqlite3DbMallocRaw(pMem->db, n/2 + 2); + if( zCopy ){ + z = pMem->z; + if( pMem->enc==SQLITE_UTF16LE ){ + for(i=j=0; i<n-1; i+=2, j++){ + zCopy[j] = z[i]; + if( z[i+1]!=0 ) break; + } + }else{ + for(i=j=0; i<n-1; i+=2, j++){ + if( z[i]!=0 ) break; + zCopy[j] = z[i+1]; + } + } + assert( j<=n/2 ); + zCopy[j] = 0; + rc = sqlite3AtoF(zCopy, &val); + if( i<n ) rc = -100; + sqlite3DbFree(pMem->db, zCopy); + } + } + if( pRC ) *pRC = rc; + return val; +} + /* ** Return the best representation of pMem that we can get into a ** double. If pMem is already a double or an integer, return its ** value. If it is a string or blob, try to convert it to a double. ** If it is a NULL, return 0.0. */ -static SQLITE_NOINLINE double memRealValue(Mem *pMem){ - /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ - double val = (double)0; - sqlite3AtoF(pMem->z, &val, pMem->n, pMem->enc); - return val; -} double sqlite3VdbeRealValue(Mem *pMem){ assert( pMem!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); @@ -665,7 +730,7 @@ double sqlite3VdbeRealValue(Mem *pMem){ testcase( pMem->flags & MEM_IntReal ); return (double)pMem->u.i; }else if( pMem->flags & (MEM_Str|MEM_Blob) ){ - return memRealValue(pMem); + return sqlite3MemRealValueRC(pMem, 0); }else{ /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ return (double)0; @@ -789,7 +854,7 @@ int sqlite3VdbeMemNumerify(Mem *pMem){ sqlite3_int64 ix; assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); + pMem->u.r = sqlite3MemRealValueRC(pMem, &rc); if( ((rc==0 || rc==1) && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1) || sqlite3RealSameAsInt(pMem->u.r, (ix = sqlite3RealToI64(pMem->u.r))) ){ @@ -1254,6 +1319,84 @@ int sqlite3VdbeMemSetStr( return SQLITE_OK; } +/* Like sqlite3VdbeMemSetStr() except: +** +** enc is always SQLITE_UTF8 +** pMem->db is always non-NULL +*/ +int sqlite3VdbeMemSetText( + Mem *pMem, /* Memory cell to set to string value */ + const char *z, /* String pointer */ + i64 n, /* Bytes in string, or negative */ + void (*xDel)(void*) /* Destructor function */ +){ + i64 nByte = n; /* New value for pMem->n */ + u16 flags; + + assert( pMem!=0 ); + assert( pMem->db!=0 ); + assert( sqlite3_mutex_held(pMem->db->mutex) ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); + + /* If z is a NULL pointer, set pMem to contain an SQL NULL. */ + if( !z ){ + sqlite3VdbeMemSetNull(pMem); + return SQLITE_OK; + } + + if( nByte<0 ){ + nByte = strlen(z); + flags = MEM_Str|MEM_Term; + }else{ + flags = MEM_Str; + } + if( nByte>(i64)pMem->db->aLimit[SQLITE_LIMIT_LENGTH] ){ + if( xDel && xDel!=SQLITE_TRANSIENT ){ + if( xDel==SQLITE_DYNAMIC ){ + sqlite3DbFree(pMem->db, (void*)z); + }else{ + xDel((void*)z); + } + } + sqlite3VdbeMemSetNull(pMem); + return sqlite3ErrorToParser(pMem->db, SQLITE_TOOBIG); + } + + /* The following block sets the new values of Mem.z and Mem.xDel. It + ** also sets a flag in local variable "flags" to indicate the memory + ** management (one of MEM_Dyn or MEM_Static). + */ + if( xDel==SQLITE_TRANSIENT ){ + i64 nAlloc = nByte + 1; + testcase( nAlloc==31 ); + testcase( nAlloc==32 ); + if( sqlite3VdbeMemClearAndResize(pMem, (int)MAX(nAlloc,32)) ){ + return SQLITE_NOMEM_BKPT; + } + assert( pMem->z!=0 ); + memcpy(pMem->z, z, nByte); + pMem->z[nByte] = 0; + }else{ + sqlite3VdbeMemRelease(pMem); + pMem->z = (char *)z; + if( xDel==SQLITE_DYNAMIC ){ + pMem->zMalloc = pMem->z; + pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); + pMem->xDel = 0; + }else if( xDel==SQLITE_STATIC ){ + pMem->xDel = xDel; + flags |= MEM_Static; + }else{ + pMem->xDel = xDel; + flags |= MEM_Dyn; + } + } + pMem->flags = flags; + pMem->n = (int)(nByte & 0x7fffffff); + pMem->enc = SQLITE_UTF8; + return SQLITE_OK; +} + /* ** Move data out of a btree key or data field and into a Mem structure. ** The data is payload from the entry that pCur is currently pointing @@ -1277,7 +1420,12 @@ int sqlite3VdbeMemFromBtree( ){ int rc; pMem->flags = MEM_Null; - if( sqlite3BtreeMaxRecordSize(pCur)<offset+amt ){ + testcase( amt==SQLITE_MAX_ALLOCATION_SIZE-1 ); + testcase( amt==SQLITE_MAX_ALLOCATION_SIZE ); + if( amt>=SQLITE_MAX_ALLOCATION_SIZE ){ + return SQLITE_NOMEM_BKPT; + } + if( (u64)amt + (u64)offset > (u64)sqlite3BtreeMaxRecordSize(pCur) ){ return SQLITE_CORRUPT_BKPT; } if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt+1)) ){ @@ -1677,7 +1825,7 @@ static int valueFromExpr( if( affinity==SQLITE_AFF_BLOB ){ if( op==TK_FLOAT ){ assert( pVal && pVal->z && pVal->flags==(MEM_Str|MEM_Term) ); - sqlite3AtoF(pVal->z, &pVal->u.r, pVal->n, SQLITE_UTF8); + sqlite3AtoF(pVal->z, &pVal->u.r); pVal->flags = MEM_Real; }else if( op==TK_INTEGER ){ /* This case is required by -9223372036854775808 and other strings @@ -1945,6 +2093,11 @@ int sqlite3Stat4ValueFromExpr( ** ** If *ppVal is initially NULL then the caller is responsible for ** ensuring that the value written into *ppVal is eventually freed. +** +** If the buffer does not contain a well-formed record, this routine may +** read several bytes past the end of the buffer. Callers must therefore +** ensure that any buffer which may contain a corrupt record is padded +** with at least 8 bytes of addressable memory. */ int sqlite3Stat4Column( sqlite3 *db, /* Database handle */ diff --git a/src/wal.c b/src/wal.c index 069852158..7f7bee626 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2254,68 +2254,82 @@ static int walCheckpoint( && (rc = walBusyLock(pWal,xBusy,pBusyArg,WAL_READ_LOCK(0),1))==SQLITE_OK ){ u32 nBackfill = pInfo->nBackfill; - pInfo->nBackfillAttempted = mxSafeFrame; SEH_INJECT_FAULT; - - /* Sync the WAL to disk */ - rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags)); - - /* If the database may grow as a result of this checkpoint, hint - ** about the eventual size of the db file to the VFS layer. - */ - if( rc==SQLITE_OK ){ - i64 nReq = ((i64)mxPage * szPage); - i64 nSize; /* Current size of database file */ - sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_START, 0); - rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); - if( rc==SQLITE_OK && nSize<nReq ){ - if( (nSize+65536+(i64)pWal->hdr.mxFrame*szPage)<nReq ){ - /* If the size of the final database is larger than the current - ** database plus the amount of data in the wal file, plus the - ** maximum size of the pending-byte page (65536 bytes), then - ** must be corruption somewhere. */ - rc = SQLITE_CORRUPT_BKPT; - }else{ - sqlite3OsFileControlHint(pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT,&nReq); + WalIndexHdr *pLive = (WalIndexHdr*)walIndexHdr(pWal); + + /* Now that read-lock slot 0 is locked, check that the wal has not been + ** wrapped since the header was read for this checkpoint. If it was, then + ** there was no work to do anyway. In this case the + ** (pInfo->nBackfill<pWal->hdr.mxFrame) test above only passed because + ** pInfo->nBackfill had already been set to 0 by the writer that wrapped + ** the wal file. It would also be dangerous to proceed, as there may be + ** fewer than pWal->hdr.mxFrame valid frames in the wal file. */ + int bChg = memcmp(pLive->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt)); + if( 0==bChg ){ + pInfo->nBackfillAttempted = mxSafeFrame; SEH_INJECT_FAULT; + + /* Sync the WAL to disk */ + rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags)); + + /* If the database may grow as a result of this checkpoint, hint + ** about the eventual size of the db file to the VFS layer. + */ + if( rc==SQLITE_OK ){ + i64 nReq = ((i64)mxPage * szPage); + i64 nSize; /* Current size of database file */ + sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_START, 0); + rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); + if( rc==SQLITE_OK && nSize<nReq ){ + if( (nSize+65536+(i64)pWal->hdr.mxFrame*szPage)<nReq ){ + /* If the size of the final database is larger than the current + ** database plus the amount of data in the wal file, plus the + ** maximum size of the pending-byte page (65536 bytes), then + ** must be corruption somewhere. */ + rc = SQLITE_CORRUPT_BKPT; + }else{ + sqlite3OsFileControlHint( + pWal->pDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); + } } + } - - } - - /* Iterate through the contents of the WAL, copying data to the db file */ - while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ - i64 iOffset; - assert( walFramePgno(pWal, iFrame)==iDbpage ); - SEH_INJECT_FAULT; - if( AtomicLoad(&db->u1.isInterrupted) ){ - rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT; - break; - } - if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ){ - continue; - } - iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE; - /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ - rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset); - if( rc!=SQLITE_OK ) break; - iOffset = (iDbpage-1)*(i64)szPage; - testcase( IS_BIG_INT(iOffset) ); - rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); - if( rc!=SQLITE_OK ) break; - } - sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_DONE, 0); - - /* If work was actually accomplished... */ - if( rc==SQLITE_OK ){ - if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ - i64 szDb = pWal->hdr.nPage*(i64)szPage; - testcase( IS_BIG_INT(szDb) ); - rc = sqlite3OsTruncate(pWal->pDbFd, szDb); - if( rc==SQLITE_OK ){ - rc = sqlite3OsSync(pWal->pDbFd, CKPT_SYNC_FLAGS(sync_flags)); + + /* Iterate through the contents of the WAL, copying data to the + ** db file */ + while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ + i64 iOffset; + assert( walFramePgno(pWal, iFrame)==iDbpage ); + SEH_INJECT_FAULT; + if( AtomicLoad(&db->u1.isInterrupted) ){ + rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT; + break; } + if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ){ + continue; + } + iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE; + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ + rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset); + if( rc!=SQLITE_OK ) break; + iOffset = (iDbpage-1)*(i64)szPage; + testcase( IS_BIG_INT(iOffset) ); + rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); + if( rc!=SQLITE_OK ) break; } + sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_DONE, 0); + + /* If work was actually accomplished... */ if( rc==SQLITE_OK ){ - AtomicStore(&pInfo->nBackfill, mxSafeFrame); SEH_INJECT_FAULT; + if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ + i64 szDb = pWal->hdr.nPage*(i64)szPage; + testcase( IS_BIG_INT(szDb) ); + rc = sqlite3OsTruncate(pWal->pDbFd, szDb); + if( rc==SQLITE_OK ){ + rc = sqlite3OsSync(pWal->pDbFd, CKPT_SYNC_FLAGS(sync_flags)); + } + } + if( rc==SQLITE_OK ){ + AtomicStore(&pInfo->nBackfill, mxSafeFrame); SEH_INJECT_FAULT; + } } } @@ -4362,9 +4376,10 @@ int sqlite3WalCheckpoint( sqlite3OsUnfetch(pWal->pDbFd, 0, 0); } } - + /* Copy data from the log to the database file. */ if( rc==SQLITE_OK ){ + sqlite3FaultSim(660); if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ rc = SQLITE_CORRUPT_BKPT; }else if( eMode2!=SQLITE_CHECKPOINT_NOOP ){ diff --git a/src/where.c b/src/where.c index c4f2c5543..2ef2ce0be 100644 --- a/src/where.c +++ b/src/where.c @@ -1514,11 +1514,14 @@ static sqlite3_index_info *allocateIndexInfo( break; } if( i==n ){ + int bSortByGroup = (pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0; nOrderBy = n; if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) && !pSrc->fg.rowidUsed ){ - eDistinct = 2 + ((pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0); + eDistinct = 2 + bSortByGroup; }else if( pWInfo->wctrlFlags & WHERE_GROUPBY ){ - eDistinct = 1; + eDistinct = 1 - bSortByGroup; + }else if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ + eDistinct = 3; } } } @@ -2929,6 +2932,67 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ return rc; } +/* +** Callback for estLikePatternLength(). +** +** If this node is a string literal that is longer pWalker->sz, then set +** pWalker->sz to the byte length of that string literal. +** +** pWalker->eCode indicates how to count characters: +** +** eCode==0 Count as a GLOB pattern +** eCode==1 Count as a LIKE pattern +*/ +static int exprNodePatternLengthEst(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_STRING ){ + int sz = 0; /* Pattern size in bytes */ + u8 *z = (u8*)pExpr->u.zToken; /* The pattern */ + u8 c; /* Next character of the pattern */ + u8 c1, c2, c3; /* Wildcards */ + if( pWalker->eCode ){ + c1 = '%'; + c2 = '_'; + c3 = 0; + }else{ + c1 = '*'; + c2 = '?'; + c3 = '['; + } + while( (c = *(z++))!=0 ){ + if( c==c3 ){ + if( *z ) z++; + while( *z && *z!=']' ) z++; + }else if( c!=c1 && c!=c2 ){ + sz++; + } + } + if( sz>pWalker->u.sz ) pWalker->u.sz = sz; + } + return WRC_Continue; +} + +/* +** Return the length of the longest string literal in the given +** expression. +** +** eCode indicates how to count characters: +** +** eCode==0 Count as a GLOB pattern +** eCode==1 Count as a LIKE pattern +*/ +static int estLikePatternLength(Expr *p, u16 eCode){ + Walker w; + w.u.sz = 0; + w.eCode = eCode; + w.xExprCallback = exprNodePatternLengthEst; + w.xSelectCallback = sqlite3SelectWalkFail; +#ifdef SQLITE_DEBUG + w.xSelectCallback2 = sqlite3SelectWalkAssert2; +#endif + sqlite3WalkExpr(&w, p); + return w.u.sz; +} + /* ** Adjust the WhereLoop.nOut value downward to account for terms of the ** WHERE clause that reference the loop but which are not used by an @@ -2957,6 +3021,13 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** "x" column is boolean or else -1 or 0 or 1 is a common default value ** on the "x" column and so in that case only cap the output row estimate ** at 1/2 instead of 1/4. +** +** Heuristic 3: If there is a LIKE or GLOB (or REGEXP or MATCH) operator +** with a large constant pattern, then reduce the size of the search +** space according to the length of the pattern, under the theory that +** longer patterns are less likely to match. This heuristic was added +** to give better output-row count estimates when preparing queries for +** the Join-Order Benchmarks. See forum thread 2026-01-30T09:57:54z */ static void whereLoopOutputAdjust( WhereClause *pWC, /* The WHERE clause */ @@ -3006,13 +3077,14 @@ static void whereLoopOutputAdjust( }else{ /* In the absence of explicit truth probabilities, use heuristics to ** guess a reasonable truth probability. */ + Expr *pOpExpr = pTerm->pExpr; pLoop->nOut--; if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && (pTerm->wtFlags & TERM_HIGHTRUTH)==0 /* tag-20200224-1 */ ){ - Expr *pRight = pTerm->pExpr->pRight; + Expr *pRight = pOpExpr->pRight; int k = 0; - testcase( pTerm->pExpr->op==TK_IS ); + testcase( pOpExpr->op==TK_IS ); if( sqlite3ExprIsInteger(pRight, &k, 0) && k>=(-1) && k<=1 ){ k = 10; }else{ @@ -3022,6 +3094,23 @@ static void whereLoopOutputAdjust( pTerm->wtFlags |= TERM_HEURTRUTH; iReduce = k; } + }else + if( ExprHasProperty(pOpExpr, EP_InfixFunc) + && pOpExpr->op==TK_FUNCTION + ){ + int eOp; + assert( ExprUseXList(pOpExpr) ); + assert( pOpExpr->x.pList->nExpr>=2 ); + eOp = sqlite3ExprIsLikeOperator(pOpExpr); + if( ALWAYS(eOp>0) ){ + int szPattern; + Expr *pRHS = pOpExpr->x.pList->a[0].pExpr; + eOp = eOp==SQLITE_INDEX_CONSTRAINT_LIKE; + szPattern = estLikePatternLength(pRHS, eOp); + if( szPattern>0 ){ + pLoop->nOut -= szPattern*2; + } + } } } } @@ -3093,7 +3182,7 @@ static int whereRangeVectorLen( idxaff = sqlite3TableColumnAffinity(pIdx->pTable, pLhs->iColumn); if( aff!=idxaff ) break; - pColl = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); + pColl = sqlite3ExprCompareCollSeq(pParse, pTerm->pExpr); if( pColl==0 ) break; if( sqlite3StrICmp(pColl->zName, pIdx->azColl[i+nEq]) ) break; } @@ -3482,6 +3571,7 @@ static int whereLoopAddBtreeIndex( pNew->rRun += nInMul + nIn; pNew->nOut += nInMul + nIn; whereLoopOutputAdjust(pBuilder->pWC, pNew, rSize); + if( pSrc->fg.fromExists ) pNew->nOut = 0; rc = whereLoopInsert(pBuilder, pNew); if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ @@ -4078,6 +4168,8 @@ static int whereLoopAddBtree( if( pSrc->fg.isSubquery ){ if( pSrc->fg.viaCoroutine ) pNew->wsFlags |= WHERE_COROUTINE; pNew->u.btree.pOrderBy = pSrc->u4.pSubq->pSelect->pOrderBy; + }else if( pSrc->fg.fromExists ){ + pNew->nOut = 0; } rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; @@ -4180,6 +4272,7 @@ static int whereLoopAddBtree( ** positioned to the correct row during the right-join no-match ** loop. */ }else{ + if( pSrc->fg.fromExists ) pNew->nOut = 0; rc = whereLoopInsert(pBuilder, pNew); } pNew->nOut = rSize; @@ -4842,7 +4935,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ sqlite3 *db = pWInfo->pParse->db; int rc = SQLITE_OK; int bFirstPastRJ = 0; - int hasRightJoin = 0; + int hasRightCrossJoin = 0; WhereLoop *pNew; @@ -4869,15 +4962,34 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ ** prevents the right operand of a RIGHT JOIN from being swapped with ** other elements even further to the right. ** - ** The JT_LTORJ case and the hasRightJoin flag work together to - ** prevent FROM-clause terms from moving from the right side of - ** a LEFT JOIN over to the left side of that join if the LEFT JOIN - ** is itself on the left side of a RIGHT JOIN. + ** The hasRightCrossJoin flag prevent FROM-clause terms from moving + ** from the right side of a LEFT JOIN or CROSS JOIN over to the + ** left side of that same join. This is a required restriction in + ** the case of LEFT JOIN - an incorrect answer may results if it is + ** not enforced. This restriction is not required for CROSS JOIN. + ** It is provided merely as a means of controlling join order, under + ** the theory that no real-world queries that care about performance + ** actually use the CROSS JOIN syntax. */ - if( pItem->fg.jointype & JT_LTORJ ) hasRightJoin = 1; + if( pItem->fg.jointype & (JT_LTORJ|JT_CROSS) ){ + testcase( pItem->fg.jointype & JT_LTORJ ); + testcase( pItem->fg.jointype & JT_CROSS ); + hasRightCrossJoin = 1; + } mPrereq |= mPrior; bFirstPastRJ = (pItem->fg.jointype & JT_RIGHT)!=0; - }else if( !hasRightJoin ){ + }else if( pItem->fg.fromExists ){ + /* joins that result from the EXISTS-to-JOIN optimization should not + ** be moved to the left of any of their dependencies */ + WhereClause *pWC = &pWInfo->sWC; + WhereTerm *pTerm; + int i; + for(i=pWC->nBase, pTerm=pWC->a; i>0; i--, pTerm++){ + if( (pNew->maskSelf & pTerm->prereqAll)!=0 ){ + mPrereq |= (pTerm->prereqAll & (pNew->maskSelf-1)); + } + } + }else if( !hasRightCrossJoin ){ mPrereq = 0; } #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -5100,9 +5212,7 @@ static i8 wherePathSatisfiesOrderBy( pLoop = pLast; } if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ - if( pLoop->u.vtab.isOrdered - && ((wctrlFlags&(WHERE_DISTINCTBY|WHERE_SORTBYGROUP))!=WHERE_DISTINCTBY) - ){ + if( pLoop->u.vtab.isOrdered && pWInfo->pOrderBy==pOrderBy ){ obSat = obDone; }else{ /* No further ORDER BY terms may be matched. So this call should @@ -5478,12 +5588,21 @@ static LogEst whereSortingCost( ** 12 otherwise ** ** For the purposes of this heuristic, a star-query is defined as a query -** with a large central table that is joined using an INNER JOIN, -** not CROSS or OUTER JOINs, against four or more smaller tables. -** The central table is called the "fact" table. The smaller tables -** that get joined are "dimension tables". Also, any table that is -** self-joined cannot be a dimension table; we assume that dimension -** tables may only be joined against fact tables. +** with a central "fact" table that is joined against multiple +** "dimension" tables, subject to the following constraints: +** +** (aa) Only a five-way or larger join is considered for this +** optimization. If there are fewer than four terms in the FROM +** clause, this heuristic does not apply. +** +** (bb) The join between the fact table and the dimension tables must +** be an INNER join. CROSS and OUTER JOINs do not qualify. +** +** (cc) A table must have 3 or more dimension tables in order to be +** considered a fact table. (Was 4 prior to 2026-02-10.) +** +** (dd) A table that is a self-join cannot be a dimension table. +** Dimension tables are joined against fact tables. ** ** SIDE EFFECT: (and really the whole point of this subroutine) ** @@ -5536,7 +5655,7 @@ static int computeMxChoice(WhereInfo *pWInfo){ } #endif /* SQLITE_DEBUG */ - if( nLoop>=5 + if( nLoop>=4 /* Constraint (aa) */ && !pWInfo->bStarDone && OptimizationEnabled(pWInfo->pParse->db, SQLITE_StarQuery) ){ @@ -5548,7 +5667,7 @@ static int computeMxChoice(WhereInfo *pWInfo){ pWInfo->bStarDone = 1; /* Only do this computation once */ - /* Look for fact tables with four or more dimensions where the + /* Look for fact tables with three or more dimensions where the ** dimension tables are not separately from the fact tables by an outer ** or cross join. Adjust cost weights if found. */ @@ -5565,18 +5684,17 @@ static int computeMxChoice(WhereInfo *pWInfo){ if( (pFactTab->fg.jointype & (JT_OUTER|JT_CROSS))!=0 ){ /* If the candidate fact-table is the right table of an outer join ** restrict the search for dimension-tables to be tables to the right - ** of the fact-table. */ - if( iFromIdx+4 > nLoop ) break; /* Impossible to reach nDep>=4 */ + ** of the fact-table. Constraint (bb) */ + if( iFromIdx+3 > nLoop ){ + break; /* ^-- Impossible to reach nDep>=2 - Constraint (cc) */ + } while( pStart && pStart->iTab<=iFromIdx ){ pStart = pStart->pNextLoop; } } for(pWLoop=pStart; pWLoop; pWLoop=pWLoop->pNextLoop){ if( (aFromTabs[pWLoop->iTab].fg.jointype & (JT_OUTER|JT_CROSS))!=0 ){ - /* Fact-tables and dimension-tables cannot be separated by an - ** outer join (at least for the definition of fact- and dimension- - ** used by this heuristic). */ - break; + break; /* Constraint (bb) */ } if( (pWLoop->prereq & m)!=0 /* pWInfo depends on iFromIdx */ && (pWLoop->maskSelf & mSeen)==0 /* pWInfo not already a dependency */ @@ -5590,7 +5708,9 @@ static int computeMxChoice(WhereInfo *pWInfo){ } } } - if( nDep<=3 ) continue; + if( nDep<=2 ){ + continue; /* Constraint (cc) */ + } /* If we reach this point, it means that pFactTab is a fact table ** with four or more dimensions connected by inner joins. Proceed @@ -5603,6 +5723,23 @@ static int computeMxChoice(WhereInfo *pWInfo){ pWLoop->rStarDelta = 0; } } +#endif +#ifdef WHERETRACE_ENABLED /* 0x80000 */ + if( sqlite3WhereTrace & 0x80000 ){ + Bitmask mShow = mSeen; + sqlite3DebugPrintf("Fact table %s(%d), dimensions:", + pFactTab->zAlias ? pFactTab->zAlias : pFactTab->pSTab->zName, + iFromIdx); + for(pWLoop=pStart; pWLoop; pWLoop=pWLoop->pNextLoop){ + if( mShow & pWLoop->maskSelf ){ + SrcItem *pDim = aFromTabs + pWLoop->iTab; + mShow &= ~pWLoop->maskSelf; + sqlite3DebugPrintf(" %s(%d)", + pDim->zAlias ? pDim->zAlias: pDim->pSTab->zName, pWLoop->iTab); + } + } + sqlite3DebugPrintf("\n"); + } #endif pWInfo->bStarUsed = 1; @@ -5626,10 +5763,8 @@ static int computeMxChoice(WhereInfo *pWInfo){ if( sqlite3WhereTrace & 0x80000 ){ SrcItem *pDim = aFromTabs + pWLoop->iTab; sqlite3DebugPrintf( - "Increase SCAN cost of dimension %s(%d) of fact %s(%d) to %d\n", - pDim->zAlias ? pDim->zAlias: pDim->pSTab->zName, pWLoop->iTab, - pFactTab->zAlias ? pFactTab->zAlias : pFactTab->pSTab->zName, - iFromIdx, mxRun + "Increase SCAN cost of %s to %d\n", + pDim->zAlias ? pDim->zAlias: pDim->pSTab->zName, mxRun ); } pWLoop->rStarDelta = mxRun - pWLoop->rRun; @@ -6443,6 +6578,7 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin( for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){ if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){ pTerm->wtFlags |= TERM_CODED; + pTerm->prereqAll = 0; } } if( i!=pWInfo->nLevel-1 ){ @@ -7430,14 +7566,15 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ } #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */ } - if( pTabList->a[pLevel->iFrom].fg.fromExists && i==pWInfo->nLevel-1 ){ - /* If the EXISTS-to-JOIN optimization was applied, then the EXISTS - ** loop(s) will be the inner-most loops of the join. There might be - ** multiple EXISTS loops, but they will all be nested, and the join - ** order will not have been changed by the query planner. If the - ** inner-most EXISTS loop sees a single successful row, it should - ** break out of *all* EXISTS loops. But only the inner-most of the - ** nested EXISTS loops should do this breakout. */ + if( pTabList->a[pLevel->iFrom].fg.fromExists + && (i==pWInfo->nLevel-1 + || pTabList->a[pWInfo->a[i+1].iFrom].fg.fromExists==0) + ){ + /* This is an EXISTS-to-JOIN optimization which is either the + ** inner-most loop, or the inner-most of a group of nested + ** EXISTS-to-JOIN optimization loops. If this loop sees a successful + ** row, it should break out of itself as well as other EXISTS-to-JOIN + ** loops in which is is directly nested. */ int nOuter = 0; /* Nr of outer EXISTS that this one is nested within */ while( nOuter<i ){ if( !pTabList->a[pLevel[-nOuter-1].iFrom].fg.fromExists ) break; @@ -7445,7 +7582,11 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ } testcase( nOuter>0 ); sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel[-nOuter].addrBrk); - VdbeComment((v, "EXISTS break")); + if( nOuter ){ + VdbeComment((v, "EXISTS break %d..%d", i-nOuter, i)); + }else{ + VdbeComment((v, "EXISTS break %d", i)); + } } sqlite3VdbeResolveLabel(v, pLevel->addrCont); if( pLevel->op!=OP_Noop ){ diff --git a/src/wherecode.c b/src/wherecode.c index 1efa34a5d..31d7990ad 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -1573,7 +1573,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( if( SMASKBIT32(j) & pLoop->u.vtab.mHandleIn ){ int iTab = pParse->nTab++; int iCache = ++pParse->nMem; - sqlite3CodeRhsOfIN(pParse, pTerm->pExpr, iTab); + sqlite3CodeRhsOfIN(pParse, pTerm->pExpr, iTab, 0); sqlite3VdbeAddOp3(v, OP_VInitIn, iTab, iTarget, iCache); }else{ codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); diff --git a/src/whereexpr.c b/src/whereexpr.c index 0d99ca85e..74bf624c8 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -293,13 +293,14 @@ static int isLikeOrGlob( ){ int isNum; double rDummy; - isNum = sqlite3AtoF(zNew, &rDummy, iTo, SQLITE_UTF8); + assert( zNew[iTo]==0 ); + isNum = sqlite3AtoF(zNew, &rDummy); if( isNum<=0 ){ if( iTo==1 && zNew[0]=='-' ){ isNum = +1; }else{ zNew[iTo-1]++; - isNum = sqlite3AtoF(zNew, &rDummy, iTo, SQLITE_UTF8); + isNum = sqlite3AtoF(zNew, &rDummy); zNew[iTo-1]--; } } @@ -342,6 +343,34 @@ static int isLikeOrGlob( } #endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ +/* +** If pExpr is one of "like", "glob", "match", or "regexp", then +** return the corresponding SQLITE_INDEX_CONSTRAINT_xxxx value. +** If not, return 0. +** +** pExpr is guaranteed to be a TK_FUNCTION. +*/ +int sqlite3ExprIsLikeOperator(const Expr *pExpr){ + static const struct { + const char *zOp; + unsigned char eOp; + } aOp[] = { + { "match", SQLITE_INDEX_CONSTRAINT_MATCH }, + { "glob", SQLITE_INDEX_CONSTRAINT_GLOB }, + { "like", SQLITE_INDEX_CONSTRAINT_LIKE }, + { "regexp", SQLITE_INDEX_CONSTRAINT_REGEXP } + }; + int i; + assert( pExpr->op==TK_FUNCTION ); + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + for(i=0; i<ArraySize(aOp); i++){ + if( sqlite3StrICmp(pExpr->u.zToken, aOp[i].zOp)==0 ){ + return aOp[i].eOp; + } + } + return 0; +} + #ifndef SQLITE_OMIT_VIRTUALTABLE /* @@ -378,15 +407,6 @@ static int isAuxiliaryVtabOperator( Expr **ppRight /* Expression to left of MATCH/op2 */ ){ if( pExpr->op==TK_FUNCTION ){ - static const struct Op2 { - const char *zOp; - unsigned char eOp2; - } aOp[] = { - { "match", SQLITE_INDEX_CONSTRAINT_MATCH }, - { "glob", SQLITE_INDEX_CONSTRAINT_GLOB }, - { "like", SQLITE_INDEX_CONSTRAINT_LIKE }, - { "regexp", SQLITE_INDEX_CONSTRAINT_REGEXP } - }; ExprList *pList; Expr *pCol; /* Column reference */ int i; @@ -406,16 +426,11 @@ static int isAuxiliaryVtabOperator( */ pCol = pList->a[1].pExpr; assert( pCol->op!=TK_COLUMN || (ExprUseYTab(pCol) && pCol->y.pTab!=0) ); - if( ExprIsVtab(pCol) ){ - for(i=0; i<ArraySize(aOp); i++){ - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - if( sqlite3StrICmp(pExpr->u.zToken, aOp[i].zOp)==0 ){ - *peOp2 = aOp[i].eOp2; - *ppRight = pList->a[0].pExpr; - *ppLeft = pCol; - return 1; - } - } + if( ExprIsVtab(pCol) && (i = sqlite3ExprIsLikeOperator(pExpr))!=0 ){ + *peOp2 = i; + *ppRight = pList->a[0].pExpr; + *ppLeft = pCol; + return 1; } /* We can also match against the first column of overloaded @@ -549,16 +564,22 @@ static void whereCombineDisjuncts( Expr *pNew; /* New virtual expression */ int op; /* Operator for the combined expression */ int idxNew; /* Index in pWC of the next virtual term */ + Expr *pA, *pB; /* Expressions associated with pOne and pTwo */ if( (pOne->wtFlags | pTwo->wtFlags) & TERM_VNULL ) return; if( (pOne->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; if( (pTwo->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; if( (eOp & (WO_EQ|WO_LT|WO_LE))!=eOp && (eOp & (WO_EQ|WO_GT|WO_GE))!=eOp ) return; - assert( pOne->pExpr->pLeft!=0 && pOne->pExpr->pRight!=0 ); - assert( pTwo->pExpr->pLeft!=0 && pTwo->pExpr->pRight!=0 ); - if( sqlite3ExprCompare(0,pOne->pExpr->pLeft, pTwo->pExpr->pLeft, -1) ) return; - if( sqlite3ExprCompare(0,pOne->pExpr->pRight, pTwo->pExpr->pRight,-1) )return; + pA = pOne->pExpr; + pB = pTwo->pExpr; + assert( pA->pLeft!=0 && pA->pRight!=0 ); + assert( pB->pLeft!=0 && pB->pRight!=0 ); + if( sqlite3ExprCompare(0,pA->pLeft, pB->pLeft, -1) ) return; + if( sqlite3ExprCompare(0,pA->pRight, pB->pRight,-1) ) return; + if( ExprHasProperty(pA,EP_Commuted)!=ExprHasProperty(pB,EP_Commuted) ){ + return; + } /* If we reach this point, it means the two subterms can be combined */ if( (eOp & (eOp-1))!=0 ){ if( eOp & (WO_LT|WO_LE) ){ @@ -569,7 +590,7 @@ static void whereCombineDisjuncts( } } db = pWC->pWInfo->pParse->db; - pNew = sqlite3ExprDup(db, pOne->pExpr, 0); + pNew = sqlite3ExprDup(db, pA, 0); if( pNew==0 ) return; for(op=TK_EQ; eOp!=(WO_EQ<<(op-TK_EQ)); op++){ assert( op<TK_GE ); } pNew->op = op; @@ -1609,13 +1630,11 @@ static void whereAddLimitExpr( int iVal = 0; if( sqlite3ExprIsInteger(pExpr, &iVal, pParse) && iVal>=0 ){ - Expr *pVal = sqlite3Expr(db, TK_INTEGER, 0); + Expr *pVal = sqlite3ExprInt32(db, iVal); if( pVal==0 ) return; - ExprSetProperty(pVal, EP_IntValue); - pVal->u.iValue = iVal; pNew = sqlite3PExpr(pParse, TK_MATCH, 0, pVal); }else{ - Expr *pVal = sqlite3Expr(db, TK_REGISTER, 0); + Expr *pVal = sqlite3ExprAlloc(db, TK_REGISTER, 0, 0); if( pVal==0 ) return; pVal->iTable = iReg; pNew = sqlite3PExpr(pParse, TK_MATCH, 0, pVal); diff --git a/src/window.c b/src/window.c index 1f22ab194..ea2781864 100644 --- a/src/window.c +++ b/src/window.c @@ -717,7 +717,7 @@ void sqlite3WindowUpdate( pWin->eEnd = aUp[i].eEnd; pWin->eExclude = 0; if( pWin->eStart==TK_FOLLOWING ){ - pWin->pStart = sqlite3Expr(db, TK_INTEGER, "1"); + pWin->pStart = sqlite3ExprInt32(db, 1); } break; } @@ -1062,9 +1062,7 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){ ** keep everything legal in this case. */ if( pSublist==0 ){ - pSublist = sqlite3ExprListAppend(pParse, 0, - sqlite3Expr(db, TK_INTEGER, "0") - ); + pSublist = sqlite3ExprListAppend(pParse, 0, sqlite3ExprInt32(db, 0)); } pSub = sqlite3SelectNew( diff --git a/test/alterauth2.test b/test/alterauth2.test index 6f9242d36..260b635c3 100644 --- a/test/alterauth2.test +++ b/test/alterauth2.test @@ -116,4 +116,55 @@ do_auth_test 1.3 { {SQLITE_UPDATE sqlite_temp_master sql temp {}} } +do_auth_test 1.4 { + ALTER TABLE t2 ALTER COLUMN b SET NOT NULL; +} { + {SQLITE_ALTER_TABLE main t2 b {}} + {SQLITE_FUNCTION {} sqlite_add_constraint {} {}} + {SQLITE_FUNCTION {} sqlite_drop_constraint {} {}} + {SQLITE_FUNCTION {} sqlite_fail {} {}} + {SQLITE_READ sqlite_master sql main {}} + {SQLITE_READ sqlite_master tbl_name main {}} + {SQLITE_READ sqlite_master type main {}} + {SQLITE_READ t2 b main {}} + {SQLITE_SELECT {} {} {} {}} + {SQLITE_UPDATE sqlite_master sql main {}} +} +do_auth_test 1.5 { + ALTER TABLE t2 ALTER COLUMN 'b' DROP NOT NULL; +} { + {SQLITE_ALTER_TABLE main t2 b {}} + {SQLITE_FUNCTION {} sqlite_drop_constraint {} {}} + {SQLITE_READ sqlite_master sql main {}} + {SQLITE_READ sqlite_master tbl_name main {}} + {SQLITE_READ sqlite_master type main {}} + {SQLITE_UPDATE sqlite_master sql main {}} +} + +do_auth_test 1.6 { + ALTER TABLE t2 ADD CONSTRAINT abc CHECK (aaa>b) +} { + {SQLITE_ALTER_TABLE main t2 {} {}} + {SQLITE_FUNCTION {} sqlite_add_constraint {} {}} + {SQLITE_FUNCTION {} sqlite_fail {} {}} + {SQLITE_FUNCTION {} sqlite_find_constraint {} {}} + {SQLITE_READ sqlite_master sql main {}} + {SQLITE_READ sqlite_master tbl_name main {}} + {SQLITE_READ sqlite_master type main {}} + {SQLITE_READ t2 aaa main {}} + {SQLITE_READ t2 b main {}} + {SQLITE_SELECT {} {} {} {}} + {SQLITE_UPDATE sqlite_master sql main {}} +} +do_auth_test 1.7 { + ALTER TABLE t2 DROP CONSTRAINT abc; +} { + {SQLITE_ALTER_TABLE main t2 {} {}} + {SQLITE_FUNCTION {} sqlite_drop_constraint {} {}} + {SQLITE_READ sqlite_master sql main {}} + {SQLITE_READ sqlite_master tbl_name main {}} + {SQLITE_READ sqlite_master type main {}} + {SQLITE_UPDATE sqlite_master sql main {}} +} + finish_test diff --git a/test/altercol.test b/test/altercol.test index 5f7de57a4..d94e0529c 100644 --- a/test/altercol.test +++ b/test/altercol.test @@ -946,4 +946,17 @@ do_execsql_test 23.20 { ROLLBACK; } {t4new} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 24.0 { + CREATE TABLE t1(a PRIMARY KEY, b); + CREATE TABLE t2(a, b, c); + INSERT INTO t2 VALUES(1, 1, 1); +} + +do_catchsql_test 24.1 { + PRAGMA foreign_keys = 1; + ALTER TABLE t2 ADD COLUMN d REFERENCES t1 DEFAULT 123; +} {1 {Cannot add a REFERENCES column with non-NULL default value}} + finish_test diff --git a/test/altercons.test b/test/altercons.test new file mode 100644 index 000000000..fecdf858b --- /dev/null +++ b/test/altercons.test @@ -0,0 +1,442 @@ +# 2025 September 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix altercons + +# If SQLITE_OMIT_ALTERTABLE is defined, omit this file. +ifcapable !altertable { + finish_test + return +} + +foreach {tn before after} { + 1 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b)) } + { CREATE TABLE t1(a, b) } + + 2 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b) NOT NULL) } + { CREATE TABLE t1(a, b NOT NULL) } + + 3 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b)NOT NULL) } + { CREATE TABLE t1(a, b NOT NULL) } + + 3 { CREATE TABLE t1(a, b NOT NULL CONSTRAINT abc CHECK(t1.a != t1.b)); } + { CREATE TABLE t1(a, b NOT NULL) } + + 4 { CREATE TABLE t1(a, b, CONSTRAINT abc CHECK(t1.a != t1.b)) } + { CREATE TABLE t1(a, b) } + + 5 { CREATE TABLE t1(a, b, CONSTRAINT abc CHECK(t1.a != t1.b), PRIMARY KEY(a))} + { CREATE TABLE t1(a, b, PRIMARY KEY(a)) } + + 6 { CREATE TABLE t1(a, b,CONSTRAINT abc CHECK(t1.a != t1.b),PRIMARY KEY(a))} + { CREATE TABLE t1(a, b,PRIMARY KEY(a)) } + + 7 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b) CONSTRAINT def UNIQUE) } + { CREATE TABLE t1(a, b CONSTRAINT def UNIQUE) } + + 8 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b) CHECK (123)) } + { CREATE TABLE t1(a, b CHECK (123)) } + + 9 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b) DEFAULT NULL) } + { CREATE TABLE t1(a, b DEFAULT NULL) } + + 10 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b) COLLATE nocase) } + { CREATE TABLE t1(a, b COLLATE nocase) } + + 11 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b) REFERENCES t2) } + { CREATE TABLE t1(a, b REFERENCES t2) } + + 12 { CREATE TABLE t1(a, b, c, CONSTRAINT one CONSTRAINT abc CHECK(a!=b) CONSTRAINT three) } + { CREATE TABLE t1(a, b, c, CONSTRAINT one CONSTRAINT three) } + + 13 { CREATE TABLE t1(a, b, c, CONSTRAINT abc CONSTRAINT one CHECK(a!=b) CONSTRAINT three) } + { CREATE TABLE t1(a, b, c, CONSTRAINT one CHECK(a!=b) CONSTRAINT three) } + + 14 { CREATE TABLE t1(a, b, c, CONSTRAINT abc) } + { CREATE TABLE t1(a, b, c) } + + 15 { CREATE TABLE t1(a, b, c, + CONSTRAINT abc, CHECK( a!=b )) } + { CREATE TABLE t1(a, b, c, CHECK( a!=b )) } + + 16 { CREATE TABLE t1(a, b, c, CONSTRAINT abc /* hello */ CHECK( a!=b )) } + { CREATE TABLE t1(a, b, c) } + + 17 { CREATE TABLE t1(a, b, c, /* world */ CONSTRAINT abc CHECK( a!=b )) } + { CREATE TABLE t1(a, b, c) } + + 18 { CREATE TABLE t1(a, b, c -- comment + CONSTRAINT abc NOT NULL + ) } + { CREATE TABLE t1(a, b, c) } + + 19 { CREATE TABLE t1(a, b, c, -- comment + CONSTRAINT abc CHECK (a>b) CONSTRAINT two + ) } + { CREATE TABLE t1(a, b, c, CONSTRAINT two + ) } + + 20 { CREATE TABLE t1(a, b, c, CONSTRAINT one CONSTRAINT abc CHECK (a>b)CONSTRAINT two) } + { CREATE TABLE t1(a, b, c, CONSTRAINT one CONSTRAINT two) } + + 21 { CREATE TABLE t1(a, b, c CONSTRAINT abc AS (b+1)) } + { CREATE TABLE t1(a, b, c AS (b+1)) } + + 22 { CREATE TABLE t1(a, b, c CONSTRAINT abc GENERATED ALWAYS AS (b+1) STORED) } + { CREATE TABLE t1(a, b, c GENERATED ALWAYS AS (b+1) STORED) } +} { + reset_db + + do_execsql_test 1.$tn.0 $before + + do_execsql_test 1.$tn.1 { + ALTER TABLE t1 DROP CONSTRAINT abc; + } {} + + do_execsql_test 1.$tn.2 { + SELECT sql FROM sqlite_schema WHERE name='t1' + } [list [string trim $after]] +} + +#------------------------------------------------------------------------- + +do_execsql_test 2.0 { + CREATE TABLE t2(x, y CONSTRAINT ccc UNIQUE); +} +do_catchsql_test 2.1 { + ALTER TABLE t2 DROP CONSTRAINT ccc +} {1 {constraint may not be dropped: ccc}} +do_catchsql_test 2.2 { + ALTER TABLE t2 DROP CONSTRAINT ddd +} {1 {no such constraint: ddd}} + +#------------------------------------------------------------------------- +reset_db +foreach {tn col before after} { + 1 a { CREATE TABLE t1(a NOT NULL, b) } + { CREATE TABLE t1(a, b) } + + 2 a { CREATE TABLE t1(a NOT NULL ON CONFLICT FAIL, b) } + { CREATE TABLE t1(a, b) } + + 3 a { CREATE TABLE t1(a NOT NULL ON CONFLICT FAIL UNIQUE, b) } + { CREATE TABLE t1(a UNIQUE, b) } + + 4 b { CREATE TABLE t1(a NOT NULL ON CONFLICT FAIL UNIQUE, b) } + { CREATE TABLE t1(a NOT NULL ON CONFLICT FAIL UNIQUE, b) } + + 5 a { CREATE TABLE t1(a CHECK(a<b) NOT NULL, b) } + { CREATE TABLE t1(a CHECK(a<b), b) } + + 6 a { CREATE TABLE t1(a CHECK(a<b) CONSTRAINT nn NOT NULL, b) } + { CREATE TABLE t1(a CHECK(a<b), b) } + + 7 b { CREATE TABLE t1(a, b NOT NULL PRIMARY KEY) } + { CREATE TABLE t1(a, b PRIMARY KEY) } + + 8 b { CREATE TABLE t1(a, b CHECK ((b+a) IS NOT NULL) NOT NULL PRIMARY KEY) } + { CREATE TABLE t1(a, b CHECK ((b+a) IS NOT NULL) PRIMARY KEY) } + + 9 b { CREATE TABLE t1(a, b CONSTRAINT nn CHECK (b IS NOT NULL) NOT NULL) } + { CREATE TABLE t1(a, b CONSTRAINT nn CHECK (b IS NOT NULL)) } + + 10 b { CREATE TABLE t1(a, b NOT NULL AS (a+1)) } + { CREATE TABLE t1(a, b AS (a+1)) } + + 11 b { CREATE TABLE t1(a, b NOT NULL GENERATED ALWAYS AS (a+1)) } + { CREATE TABLE t1(a, b GENERATED ALWAYS AS (a+1)) } +} { + reset_db + + do_execsql_test 3.$tn.0 $before + + do_execsql_test 3.$tn.1 " + ALTER TABLE t1 ALTER COLUMN $col DROP NOT NULL + " + + do_execsql_test 3.$tn.2 { + SELECT sql FROM sqlite_schema WHERE name='t1' + } [list [string trim $after]] +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE TABLE t2(x, y CONSTRAINT ccc UNIQUE); +} +do_execsql_test 4.1 { + ALTER TABLE t2 ALTER x DROP NOT NULL; + ALTER TABLE t2 ALTER x DROP NOT NULL; + ALTER TABLE t2 ALTER x DROP NOT NULL; +} {} + +#------------------------------------------------------------------------- +# +reset_db + +do_execsql_test 5.1 { + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + INSERT INTO t3 VALUES(1000, NULL); +} + +do_catchsql_test 5.2.1 { + ALTER TABLE t3 ALTER b SET NOT NULL +} {1 {constraint failed}} + +do_test 5.2.2 { + sqlite3_errcode db +} {SQLITE_CONSTRAINT} + +foreach {tn before alter after} { + 1 { CREATE TABLE t1(a, b) } + { ALTER TABLE t1 ALTER a SET NOT NULL } + { CREATE TABLE t1(a NOT NULL, b) } + + 2 { CREATE TABLE t1(a, b) } + { ALTER TABLE t1 ALTER a SET NOT NULL ON CONFLICT FAIL } + { CREATE TABLE t1(a NOT NULL ON CONFLICT FAIL, b) } + + 3 { CREATE TABLE t1(a, b) } + { ALTER TABLE t1 ALTER a SET NOT NULL ON CONFLICT fail; } + { CREATE TABLE t1(a NOT NULL ON CONFLICT fail, b) } + + 4 { CREATE TABLE t1(a, b) } + { ALTER TABLE t1 ALTER b SET NOT NULL ON CONFLICT IGNORE ; } + { CREATE TABLE t1(a, b NOT NULL ON CONFLICT IGNORE) } + + 5 { CREATE TABLE t1(a, 'a b c' VARCHAR(10), UNIQUE(a)) } + { ALTER TABLE t1 ALTER 'a b c' SET NOT NULL } + { CREATE TABLE t1(a, 'a b c' VARCHAR(10) NOT NULL, UNIQUE(a)) } +} { + reset_db + do_execsql_test 5.3.$tn.1 $before + do_execsql_test 5.3.$tn.2 $alter + do_execsql_test 5.3.$tn.3 { + SELECT sql FROM sqlite_schema WHERE name='t1'; + } [list [string trim $after]] +} + +do_execsql_test 5.4.1 { + CREATE TABLE x1(a, b, c); +} +do_catchsql_test 5.4.2 { + ALTER TABLE x1 ALTER d SET NOT NULL; +} {1 {no such column: d}} +do_catchsql_test 5.4.3 { + ALTER TABLE x2 ALTER c SET NOT NULL; +} {1 {no such table: x2}} +do_catchsql_test 5.4.4 { + ALTER TABLE temp.x1 ALTER c SET NOT NULL; +} {1 {no such table: temp.x1}} + +#------------------------------------------------------------------------- +# +reset_db + +do_execsql_test 6.1 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); +} + +do_catchsql_test 6.2.1 { + ALTER TABLE t1 ADD CONSTRAINT nn CHECK (c!=6); +} {1 {constraint failed}} +do_execsql_test 6.2.2 { + DELETE FROM t1 WHERE c=6; + ALTER TABLE t1 ADD CONSTRAINT nn CHECK (c!=6); +} {} +do_catchsql_test 6.2.3 { + INSERT INTO t1 VALUES(4, 5, 6); +} {1 {CHECK constraint failed: nn}} + +foreach {tn before alter after} { + 1 { CREATE TABLE t1(a, b) } + { ALTER TABLE t1 ADD CONSTRAINT nn CHECK (a>=0) } + { CREATE TABLE t1(a, b, CONSTRAINT nn CHECK (a>=0)) } + + 2 { CREATE TABLE t1(a, b ) } + { ALTER TABLE t1 ADD CONSTRAINT nn CHECK (a>=0) } + { CREATE TABLE t1(a, b , CONSTRAINT nn CHECK (a>=0)) } + + 3 { CREATE TABLE t1(a, b ) } + { ALTER TABLE t1 ADD CHECK (a>=0) } + { CREATE TABLE t1(a, b , CHECK (a>=0)) } +} { + reset_db + do_execsql_test 6.3.$tn.1 $before + do_execsql_test 6.3.$tn.2 $alter + do_execsql_test 6.3.$tn.3 { + SELECT sql FROM sqlite_schema WHERE type='table'; + } [list [string trim $after]] +} + +do_execsql_test 6.4.1 { + CREATE TABLE b1(a, b, CONSTRAINT abc CHECK (a!=2)); +} +do_catchsql_test 6.4.2 { + ALTER TABLE b1 ADD CONSTRAINT abc CHECK (a!=3); +} {1 {constraint abc already exists}} +do_execsql_test 6.4.1 { + SELECT sql FROM sqlite_schema WHERE tbl_name='b1' +} {{CREATE TABLE b1(a, b, CONSTRAINT abc CHECK (a!=2))}} + +do_execsql_test 6.5 { + CREATE TABLE abc(x,y); +} + +do_catchsql_test 6.6 { + ALTER TABLE abc ADD CHECK (z>=0); +} {1 {no such column: z}} + +#------------------------------------------------------------------------- +# Try attaching a NOT NULL to a generated column. +# +reset_db +do_execsql_test 7.0 { + CREATE TABLE x1(a, b AS (a+1)); + INSERT INTO x1 VALUES(1), (2), (3), (NULL); +} + +do_catchsql_test 7.1 { + ALTER TABLE x1 ALTER b SET NOT NULL; +} {1 {constraint failed}} + +do_catchsql_test 7.2 { + DELETE FROM x1 WHERE b IS NULL; + ALTER TABLE x1 ALTER b SET NOT NULL; +} {0 {}} + +do_execsql_test 7.3 { + SELECT b FROM x1 +} {2 3 4} + +do_catchsql_test 7.4 { + ALTER TABLE x1 ALTER rowid SET NOT NULL; +} {1 {no such column: rowid}} + +do_execsql_test 7.5 { + CREATE VIEW v1 AS SELECT a, b FROM x1; +} +do_catchsql_test 7.6 { + ALTER TABLE v1 RENAME a TO c; +} {1 {cannot rename columns of view "v1"}} +do_catchsql_test 7.7 { + ALTER TABLE v1 ALTER a SET NOT NULL; +} {1 {cannot edit constraints of view "v1"}} +do_catchsql_test 7.8 { + ALTER TABLE sqlite_schema ALTER sql SET NOT NULL; +} {1 {table sqlite_master may not be altered}} +do_catchsql_test 7.9 { + ALTER TABLE v1 ALTER a DROP NOT NULL +} {1 {cannot edit constraints of view "v1"}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 8.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b NOT NULL, c CHECK (c!=555), d); + INSERT INTO t1 VALUES(1, 1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3, 3); +} + +do_execsql_test 8.1.1 { + ALTER TABLE t1 ALTER a SET NOT NULL; + ALTER TABLE t1 ALTER b SET NOT NULL; + ALTER TABLE t1 ALTER c SET NOT NULL; + ALTER TABLE t1 ALTER d SET NOT NULL; +} + +do_execsql_test 8.1.2 { + SELECT sql FROM sqlite_schema WHERE tbl_name = 't1' +} {{CREATE TABLE t1(a INTEGER PRIMARY KEY NOT NULL, b NOT NULL, c CHECK (c!=555) NOT NULL, d NOT NULL)}} + +do_execsql_test 8.1.3 { + SELECT * FROM t1 WHERE a=2; +} {2 2 2 2} + +do_execsql_test 8.2.1 { + ALTER TABLE t1 ALTER a DROP NOT NULL; + ALTER TABLE t1 ALTER b DROP NOT NULL; + ALTER TABLE t1 ALTER c DROP NOT NULL; + ALTER TABLE t1 ALTER d DROP NOT NULL; +} + +do_execsql_test 8.2.2 { + SELECT sql FROM sqlite_schema WHERE tbl_name = 't1' +} {{CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c CHECK (c!=555), d)}} + +do_execsql_test 8.2.3 { + SELECT * FROM t1 WHERE a=3; +} {3 3 3 3} + +#------------------------------------------------------------------------- +reset_db +forcedelete test.db2 +do_execsql_test 9.0 { + CREATE TABLE t1(x, y, z); + ATTACH 'test.db2' AS aux; + CREATE TABLE aux.t1(x, y, z); + INSERT INTO aux.t1 VALUES(1, 1, 1); + INSERT INTO aux.t1 VALUES(2, 2, 2); + INSERT INTO aux.t1 VALUES(3, 3, NULL); + + CREATE TABLE aux.t2(x, y, z); +} + +do_catchsql_test 9.1.1 { + ALTER TABLE aux.t1 ALTER COLUMN z SET NOT NULL +} {1 {constraint failed}} +do_execsql_test 9.1.2 { + UPDATE aux.t1 SET z=x; + ALTER TABLE aux.t1 ALTER COLUMN z SET NOT NULL; + SELECT sql FROM aux.sqlite_schema WHERE name='t1'; +} {{CREATE TABLE t1(x, y, z NOT NULL)}} +do_execsql_test 9.1.3 { + ALTER TABLE aux.t1 ALTER z DROP NOT NULL; + SELECT sql FROM aux.sqlite_schema WHERE name='t1'; +} {{CREATE TABLE t1(x, y, z)}} +do_execsql_test 9.1.4 { + ALTER TABLE t2 ALTER x SET NOT NULL; + SELECT sql FROM aux.sqlite_schema WHERE name='t2'; +} {{CREATE TABLE t2(x NOT NULL, y, z)}} +do_execsql_test 9.1.5 { + ALTER TABLE t2 ALTER x DROP NOT NULL; + SELECT sql FROM aux.sqlite_schema WHERE name='t2'; +} {{CREATE TABLE t2(x, y, z)}} + +do_catchsql_test 9.2.1 { + ALTER TABLE aux.t1 ADD CONSTRAINT bill CHECK (y!=2); +} {1 {constraint failed}} +do_execsql_test 9.2.2 { + UPDATE aux.t1 SET y=4 WHERE y=2; + ALTER TABLE aux.t1 ADD CONSTRAINT bill CHECK (y!=2); + SELECT sql FROM aux.sqlite_schema WHERE name='t1'; +} {{CREATE TABLE t1(x, y, z, CONSTRAINT bill CHECK (y!=2))}} +do_execsql_test 9.2.3 { + ALTER TABLE aux.t1 DROP CONSTRAINT bill; + SELECT sql FROM aux.sqlite_schema WHERE name='t1'; +} {{CREATE TABLE t1(x, y, z)}} +do_execsql_test 9.2.4 { + ALTER TABLE t2 ADD CONSTRAINT william CHECK (z!=''); + SELECT sql FROM aux.sqlite_schema WHERE name='t2'; +} {{CREATE TABLE t2(x, y, z, CONSTRAINT william CHECK (z!=''))}} +do_execsql_test 9.2.5 { + ALTER TABLE t2 DROP CONSTRAINT william; + SELECT sql FROM aux.sqlite_schema WHERE name='t2'; +} {{CREATE TABLE t2(x, y, z)}} + +finish_test + diff --git a/test/altercons2.test b/test/altercons2.test new file mode 100644 index 000000000..a5bbaf6fc --- /dev/null +++ b/test/altercons2.test @@ -0,0 +1,247 @@ +# 2025 September 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix altercons2 + +# If SQLITE_OMIT_ALTERTABLE is defined, omit this file. +ifcapable !altertable { + finish_test + return +} + +foreach {tn newsql alter res final} { + 1 "CREATE TABLE t1(a, b" + "ALTER TABLE t1 ALTER c SET NOT NULL" + {1 {database disk image is malformed}} + "CREATE TABLE t1(a, b" + + 2 "CREATE TABLE t1(a, b, " + "ALTER TABLE t1 ALTER c DROP NOT NULL" + {0 {}} + "CREATE TABLE t1(a, b, " + + 3 "CREATE TABLE t1(a, b, CHECK( ..." + "ALTER TABLE t1 ALTER c DROP NOT NULL" + {0 {}} + "CREATE TABLE t1(a, b, CHECK( ..." + + 4 "CREATE TABLE t1(a, b, c NOT NULL" + "ALTER TABLE t1 ALTER c DROP NOT NULL" + {0 {}} + "CREATE TABLE t1(a, b, c " + + 5 "CREATE TABLE" + "ALTER TABLE t1 ALTER c DROP NOT NULL" + {1 {database disk image is malformed}} + "CREATE TABLE" + + 6 "CREATE TABLE" + "ALTER TABLE t1 ADD CONSTRAINT nn CHECK (a!=0)" + {1 {database disk image is malformed}} + "CREATE TABLE" + +} { + reset_db + do_execsql_test 1.$tn.0 { + CREATE TABLE t1(a, b, c NOT NULL, CONSTRAINT xyz CHECK( a!=0 )); + } + do_execsql_test 1.$tn.1 { + PRAGMA writable_schema = 1; + UPDATE sqlite_schema SET sql = $::newsql + } + do_catchsql_test 1.$tn.2 $alter $res + + do_execsql_test 1.$tn.3 { + SELECT sql FROM sqlite_schema WHERE name='t1' + } [list $final] +} + +#------------------------------------------------------------------------- + +reset_db +proc xAuth {t args} { + if {$t=="SQLITE_ALTER_TABLE"} { + return "SQLITE_DENY" + } + return "SQLITE_OK" +} +sqlite3 db test.db +db auth xAuth + +do_execsql_test 2.0 { + CREATE TABLE x1(a PRIMARY KEY, b CHECK(a!=b) NOT NULL, c); +} + +do_catchsql_test 2.1.1 { + ALTER TABLE x1 ADD CONSTRAINT ccc CHECK (a!='a') +} {1 {not authorized}} +do_execsql_test 2.1.2 { + SELECT sql FROM sqlite_schema WHERE name='x1' +} {{CREATE TABLE x1(a PRIMARY KEY, b CHECK(a!=b) NOT NULL, c)}} + +do_catchsql_test 2.2.1 { + ALTER TABLE x1 ALTER c SET NOT NULL +} {1 {not authorized}} +do_execsql_test 2.2.2 { + SELECT sql FROM sqlite_schema WHERE name='x1' +} {{CREATE TABLE x1(a PRIMARY KEY, b CHECK(a!=b) NOT NULL, c)}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(x); +} +do_execsql_test 3.1 { + ALTER TABLE t1 ALTER x SET NOT NULL; +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE TABLE abc(a, b, c, CONSTRAINT one CONSTRAINT two CHECK (b!=c)); +} +do_execsql_test 4.1 { + ALTER TABLE abc DROP CONSTRAINT one +} +do_execsql_test 4.2 { + SELECT sql FROM sqlite_schema +} { + {CREATE TABLE abc(a, b, c, CONSTRAINT two CHECK (b!=c))} +} + +#------------------------------------------------------------------------- +reset_db + +# The columns must come before the table constraints in a CREATE TABLE +# statement. This is useful, as it means the DROP CONSTRAINT code does +# not have to handle the constraint immediately following the '(' at +# the start of the column-list. +do_catchsql_test 5.0 { + CREATE TABLE abc(a, b, c, CONSTRAINT two CHECK (b!=c), d) +} {1 {near "d": syntax error}} +do_catchsql_test 5.1 { + CREATE TABLE def(CONSTRAINT abc CHECK( b!=c ), a, b, c); +} {1 {near "CONSTRAINT": syntax error}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE TABLE abc(a, b CONSTRAINT two COLLATE nocase CHECK (a!=b), c CONSTRAINT one DEFAULT 'abc'); +} + +do_execsql_test 6.1 { + ALTER TABLE abc DROP CONSTRAINT one; + ALTER TABLE abc DROP CONSTRAINT two; +} + +do_execsql_test 6.2 { + SELECT sql FROM sqlite_schema +} { + {CREATE TABLE abc(a, b COLLATE nocase CHECK (a!=b), c DEFAULT 'abc')} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 7.0 { + CREATE TABLE abc(a, b, c, CONSTRAINT one CHECK (a>b) FOREIGN KEY(a) REFERENCES abc); +} +do_execsql_test 7.1 { + ALTER TABLE abc DROP CONSTRAINT one +} +do_execsql_test 7.2 { + SELECT sql FROM sqlite_schema +} { + {CREATE TABLE abc(a, b, c, FOREIGN KEY(a) REFERENCES abc)} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 8.0 { + CREATE TABLE abc(a, b, c, CONSTRAINT one FOREIGN KEY(a) REFERENCES abc); +} +do_catchsql_test 8.1 { + ALTER TABLE abc DROP CONSTRAINT one +} {1 {constraint may not be dropped: one}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 9.0 { + CREATE TABLE abc(a, b NOT NULL AS (a+1)) +} +do_execsql_test 9.1 { + ALTER TABLE abc ALTER b DROP NOT NULL; + SELECT sql FROM sqlite_schema; +} {{CREATE TABLE abc(a, b AS (a+1))}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 10.0 { + CREATE TABLE abc(a, b GENERATED ALWAYS AS (a+1)); + INSERT INTO abc VALUES(1), (2); + SELECT * FROM abc; +} {1 2 2 3} + +do_execsql_test 10.1 { + ALTER TABLE abc ALTER b SET NOT NULL; +} +do_catchsql_test 10.2 { + INSERT INTO abc VALUES(NULL); +} {1 {NOT NULL constraint failed: abc.b}} +do_execsql_test 10.3 { + INSERT INTO abc VALUES(3); + ALTER TABLE abc ALTER COLUMN b DROP NOT NULL; +} +do_execsql_test 10.4 { + INSERT INTO abc VALUES(NULL); +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 11.0 { + CREATE TABLE t1(a, b, c); +} + +do_execsql_test 11.1.1 { + ALTER TABLE t1 ADD CONSTRAINT c1 CHECK(a=b) --comment + ; +} + +do_execsql_test 11.1.2 {ALTER TABLE t1 ADD CONSTRAINT c2 CHECK(a=b) --comment} + +do_execsql_test 11.1.3 { + SELECT sql FROM sqlite_schema; +} { + {CREATE TABLE t1(a, b, c, CONSTRAINT c1 CHECK(a=b), CONSTRAINT c2 CHECK(a=b))} +} + +do_execsql_test 11.2.1 { + CREATE TABLE t2(a, b); +} +do_execsql_test 11.2.2 {ALTER TABLE t2 ALTER b SET NOT NULL --new cons} +do_execsql_test 11.2.3 { + SELECT sql FROM sqlite_schema WHERE name='t2'; +} { + {CREATE TABLE t2(a, b NOT NULL)} +} +do_execsql_test 11.2.3 { + ALTER TABLE t2 ALTER b DROP NOT NULL; + SELECT sql FROM sqlite_schema WHERE name='t2'; +} { + {CREATE TABLE t2(a, b)} +} +do_execsql_test 11.2.2 {ALTER TABLE t2 ALTER b SET NOT NULL --new cons +; +} +finish_test + diff --git a/test/alterfault.test b/test/alterfault.test index b6b42973e..c9c0d77ba 100644 --- a/test/alterfault.test +++ b/test/alterfault.test @@ -23,6 +23,9 @@ ifcapable !altertable { do_execsql_test 1.0 { CREATE TABLE t1(a); + CREATE TEMP TRIGGER tr1 AFTER INSERT ON t1 BEGIN + SELECT 123; + END; } faultsim_save_and_close @@ -36,6 +39,40 @@ do_faultsim_test 1.1 -faults oom* -prep { faultsim_test_result {0 {}} } +reset_db +do_execsql_test 2.0 { + CREATE TABLE x1(d, e CONSTRAINT abc NOT NULL, f); +} +faultsim_save_and_close + +foreach {tn sql} { + 1 { ALTER TABLE x1 ADD CHECK (d!=1) } + 2 { ALTER TABLE x1 ADD CONSTRAINT xyz CHECK (f>d+e); } + 3 { ALTER TABLE x1 DROP CONSTRAINT abc } + 4 { ALTER TABLE x1 ALTER f SET NOT NULL } + 5 { ALTER TABLE x1 ALTER e DROP NOT NULL } +} { + do_faultsim_test 2.$tn -faults oom* -prep { + faultsim_restore_and_reopen + } -body { + execsql $::sql + } -test { + faultsim_test_result {0 {}} + } +} +# Test an OOM when returning an error. +# +do_faultsim_test 2.e -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + ALTER TABLE x1 DROP CONSTRAINT nosuchconstraint + } +} -test { + faultsim_test_result \ + {1 {no such constraint: nosuchconstraint}} \ + {1 {SQL logic error}} +} finish_test diff --git a/test/altertab3.test b/test/altertab3.test index 92060fb41..36e08c769 100644 --- a/test/altertab3.test +++ b/test/altertab3.test @@ -778,4 +778,52 @@ do_execsql_test 31.2 { SELECT rr FROM t1 LIMIT 1 } {5.0} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 32.1.0 { + CREATE TABLE t1( + a INT, + b INT, + -- comment with comma + c INT + ); +} +do_execsql_test 32.1.1 { + ALTER TABLE t1 DROP COLUMN c; +} +do_execsql_test 32.1.2 { + SELECT sql FROM sqlite_schema +} {{CREATE TABLE t1( + a INT, + b INT)}} + +reset_db +do_execsql_test 32.2.0 { + CREATE TABLE t1( + a INT, + b INT, + -- comment with, comma + c INT + ); +} +do_execsql_test 32.2.1 { + ALTER TABLE t1 DROP COLUMN c; +} +do_execsql_test 32.2.2 { + SELECT sql FROM sqlite_schema +} {{CREATE TABLE t1( + a INT, + b INT)}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 33.1 { + CREATE TABLE x1(a TEXT, b INTEGER, c CHECK(c!=0)); +} + +do_execsql_test 33.2 { + ALTER TABLE x1 DROP COLUMN b; + SELECT sql FROM sqlite_schema; +} {{CREATE TABLE x1(a TEXT, c CHECK(c!=0))}} + finish_test diff --git a/test/altertrig.test b/test/altertrig.test index 556dc3fea..223feaf8f 100644 --- a/test/altertrig.test +++ b/test/altertrig.test @@ -41,7 +41,7 @@ do_execsql_test 1.0 { CREATE TABLE t4(a); CREATE TRIGGER r1 INSERT ON t1 BEGIN - UPDATE t1 SET d='xyz' FROM t2, t3; + UPDATE t1 SET d='xyz' FROM t2, t3; END; } diff --git a/test/atof2.test b/test/atof2.test new file mode 100644 index 000000000..5a68d1352 --- /dev/null +++ b/test/atof2.test @@ -0,0 +1,35 @@ +# 2026-02-20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests of the sqlite3AtoF() function. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# Rounding cases: +# +do_execsql_test atof2-1.0 { + SELECT format('%g',192.496475); +} 192.496 +do_execsql_test atof2-1.1 { + SELECT format('%g',192.496501); +} 192.497 + +load_static_extension db ieee754 +do_execsql_test atof2-2.1 { + SELECT format('%!.30f',ieee754_inc(100.0,-1)); +} 99.9999999999999858 +do_execsql_test atof2-2.2 { + SELECT format('%!.30f',ieee754_inc(100.0,-2)); +} 99.9999999999999716 + +finish_test diff --git a/test/autoindex1.test b/test/autoindex1.test index 1c8ce007f..be41d702c 100644 --- a/test/autoindex1.test +++ b/test/autoindex1.test @@ -185,8 +185,7 @@ do_eqp_test autoindex1-500.1 { QUERY PLAN |--SEARCH t501 USING INTEGER PRIMARY KEY (rowid=?) `--LIST SUBQUERY xxxxxx - |--SCAN t502 - `--CREATE BLOOM FILTER + `--SCAN t502 } do_eqp_test autoindex1-501 { SELECT b FROM t501 diff --git a/test/avfs.test b/test/avfs.test index ffd6b309f..1503e8613 100644 --- a/test/avfs.test +++ b/test/avfs.test @@ -332,9 +332,11 @@ do_test 4.3 { set ofd [open $shdo w] if {$::cliDoesAr} { puts $ofd ".ar -u $shdo" + puts $ofd ".mode list" puts $ofd "select count(*) from sqlar where name = '$shdo';" } else { puts $ofd "insert into sqlar values (1);" + puts $ofd ".mode list" puts $ofd "select count(*) from sqlar;" } puts $ofd ".q" diff --git a/test/bestindex8.test b/test/bestindex8.test index 3ed7f6703..43d9cf6af 100644 --- a/test/bestindex8.test +++ b/test/bestindex8.test @@ -80,17 +80,17 @@ do_execsql_test 1.0 { foreach {tn sql bDistinct idxinsert bConsumed res} { 1 "SELECT a, b FROM vt1" 0 0 0 {a b c d a b c d} - 2 "SELECT DISTINCT a, b FROM vt1" 2 1 1 {a b c d} - 3 "SELECT DISTINCT a FROM vt1" 2 1 1 {a c} + 2 "SELECT DISTINCT a, b FROM vt1" 2 0 1 {a b c d} + 3 "SELECT DISTINCT a FROM vt1" 2 0 1 {a c} 4 "SELECT DISTINCT b FROM vt1" 2 1 0 {b d} - 5 "SELECT DISTINCT b FROM vt1 ORDER BY a" 0 1 1 {b d} - 6 "SELECT DISTINCT t0.c0 FROM vt1, t0 ORDER BY vt1.a" 0 1 1 {1 0} + 5 "SELECT DISTINCT b FROM vt1 ORDER BY a" 3 1 1 {b d} + 6 "SELECT DISTINCT t0.c0 FROM vt1, t0 ORDER BY vt1.a" 3 1 1 {1 0} 7 "SELECT DISTINCT a, b FROM vt1 ORDER BY a, b" 3 0 1 {a b c d} - 8 "SELECT DISTINCT a, b FROM vt1 ORDER BY a" 0 1 1 {a b c d} - 9 "SELECT DISTINCT a FROM vt1 ORDER BY a, b" 0 1 1 {a c} + 8 "SELECT DISTINCT a, b FROM vt1 ORDER BY a" 3 1 1 {a b c d} + 9 "SELECT DISTINCT a FROM vt1 ORDER BY a, b" 3 1 1 {a c} - 10 "SELECT DISTINCT a, b FROM vt1 WHERE b='b'" 2 1 1 {a b} - 11 "SELECT DISTINCT a, b FROM vt1 WHERE +b='b'" 2 1 1 {a b} + 10 "SELECT DISTINCT a, b FROM vt1 WHERE b='b'" 2 0 1 {a b} + 11 "SELECT DISTINCT a, b FROM vt1 WHERE +b='b'" 2 0 1 {a b} } { set ::lBestIndexDistinct "" set ::lOrderByConsumed 0 diff --git a/test/bestindexB.test b/test/bestindexB.test index b50e74fee..5850e35bd 100644 --- a/test/bestindexB.test +++ b/test/bestindexB.test @@ -34,7 +34,7 @@ proc vtab_command {method args} { set orderby [$hdl orderby] if {[info exists ::xbestindex_sql]} { - explain_i $::xbestindex_sql + # explain_i $::xbestindex_sql set ::xbestindex_res [ execsql $::xbestindex_sql ] } diff --git a/test/bestindexF.test b/test/bestindexF.test new file mode 100644 index 000000000..4f49f610d --- /dev/null +++ b/test/bestindexF.test @@ -0,0 +1,294 @@ +# 2025-12-15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix bestindexF + +ifcapable !vtab { + finish_test + return +} + + +proc vtab_command {method args} { + switch -- $method { + xConnect { + return "CREATE TABLE t1(a, b, c)" + } + + xBestIndex { + set hdl [lindex $args 0] + set ::vtab_orderby [$hdl orderby] + set ::vtab_distinct [$hdl distinct] + + if {$::vtab_orderby == "{column 0 desc 0} {column 1 desc 0}" + || $::vtab_orderby == "{column 0 desc 0}" + } { + return [list orderby 1] + } + + return "" + } + + xFilter { + set sql { + SELECT 1, 1, 'a', 555 + UNION ALL + SELECT 2, 1, 'a', NULL + UNION ALL + SELECT 3, 1, 'b', 'text' + UNION ALL + SELECT 4, 2, 'a', 3.14 + UNION ALL + SELECT 5, 2, 'b', 0 + } + return [list sql $sql] + } + } + + return {} +} + +register_tcl_module db + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING tcl(vtab_command) +} + +proc uses_idxinsert {sql} { + return [expr [lsearch [db eval "explain $sql"] IdxInsert]>=0] +} +proc do_idxinsert_test {tn sql res} { + set uses [uses_idxinsert $sql] + uplevel [list do_execsql_test $tn "SELECT $uses ; $sql" $res] +} + +do_idxinsert_test 1.1.1 { + SELECT DISTINCT a, b FROM t1 +} {0 1 a 1 b 2 a 2 b} + +do_test 1.1.2 { + list $::vtab_distinct $::vtab_orderby +} {2 {{column 0 desc 0} {column 1 desc 0}}} + +do_execsql_test 1.3 { + CREATE TABLE t0(c0); + INSERT INTO t0 VALUES(0); + INSERT INTO t0 VALUES(1); +} + +do_idxinsert_test 1.4.1 { + SELECT DISTINCT t0.c0 FROM t1, t0 ORDER BY t1.a; +} {1 0 1} + +do_test 1.4.2 { + list $::vtab_distinct $::vtab_orderby +} {3 {{column 0 desc 0}}} + +#------------------------------------------------------------------------- +# +reset_db +proc vtab_command {method args} { + switch -- $method { + xConnect { + return "CREATE TABLE t1(a, b, c)" + } + + xBestIndex { + set hdl [lindex $args 0] + set ::vtab_orderby [$hdl orderby] + set ::vtab_distinct [$hdl distinct] + + # Set idxNum to 1 if DISTINCT is to be used in xFilter. + # + set idxStr [list ""] + if {$::vtab_distinct==2 || $::vtab_distinct==3} { + set idxStr [list DISTINCT] + } + + set orderby 0 + if {$::vtab_orderby == "{column 0 desc 1}" + || $::vtab_orderby == "{column 0 desc 0}" + } { + set orderby 1 + if {$::vtab_distinct==1 || $::vtab_distinct==2} { + lappend idxStr "ORDER BY ((a+2)%5)" + } else { + set sort "ORDER BY a" + if {$::vtab_orderby == "{column 0 desc 1}"} { + append sort " DESC" + } + lappend idxStr $sort + } + } else { + lappend idxStr "" + } + + return [list orderby $orderby idxstr $idxStr] + return "" + } + + xFilter { + set idxstr [lindex $args 1] + + set distinct [lindex $idxstr 0] + set orderby [lindex $idxstr 1] + set sql " + SELECT $distinct 0, a, b FROM real_t1 $orderby + " + return [list sql $sql] + } + } + + return {} +} + +do_execsql_test 2.0 { + CREATE TABLE real_t1(a, b); + + INSERT INTO real_t1 VALUES (1, 'a'); + INSERT INTO real_t1 VALUES (2, 'a'); + INSERT INTO real_t1 VALUES (1, 'a'); + + INSERT INTO real_t1 VALUES (2, 'b'); + INSERT INTO real_t1 VALUES (1, 'b'); + INSERT INTO real_t1 VALUES (2, 'b'); + + INSERT INTO real_t1 VALUES (3, 'a'); + INSERT INTO real_t1 VALUES (4, 'b'); + INSERT INTO real_t1 VALUES (3, 'a'); + + INSERT INTO real_t1 VALUES (4, 'b'); + INSERT INTO real_t1 VALUES (3, 'a'); + INSERT INTO real_t1 VALUES (4, 'b'); +} + +register_tcl_module db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING tcl(vtab_command) +} + +do_execsql_test 2.1 { + SELECT a, b FROM t1 +} { + 1 a 2 a 1 a + 2 b 1 b 2 b + 3 a 4 b 3 a + 4 b 3 a 4 b +} + +# This is like do_execsql_test, except one value is prepended to the +# expected result - the P4 (idxStr) of the VFilter opcode. It is an error +# if $sql generates two or more VFilter instructions. +# +proc do_vtabsorter_test {tn sql expect} { + set vm [db eval "EXPLAIN $sql"] + + set ii [lsearch $vm VFilter] + set ::res [lindex $vm [expr $ii+4]] + + set ::idx [expr [lsearch $vm IdxInsert]>=0] + + set iSort [lsearch $vm SorterSort] + if {$iSort>=0} { + error "query is using sorter" + } + + uplevel [list do_test $tn.0 { set ::idx } [lindex $expect 0]] + uplevel [list do_test $tn.1 { set ::res } [lindex $expect 1]] + uplevel [list do_execsql_test $tn.2 $sql [lrange $expect 2 end]] +} + +do_vtabsorter_test 2.2 { + SELECT a, b FROM t1 +} { 0 "{} {}" + 1 a 2 a 1 a + 2 b 1 b 2 b + 3 a 4 b 3 a + 4 b 3 a 4 b +} + +do_vtabsorter_test 2.3 { + SELECT DISTINCT a FROM t1 +} { 0 "DISTINCT {ORDER BY ((a+2)%5)}" + 3 4 1 2 +} + +do_vtabsorter_test 2.4 { + SELECT DISTINCT a FROM t1 ORDER BY a +} { 0 "DISTINCT {ORDER BY a}" + 1 2 3 4 +} + +do_vtabsorter_test 2.5 { + SELECT DISTINCT a FROM t1 ORDER BY a DESC +} { 0 "DISTINCT {ORDER BY a DESC}" + 4 3 2 1 +} + +do_vtabsorter_test 2.6 { + SELECT a FROM t1 ORDER BY a +} { 0 "{} {ORDER BY a}" + 1 1 1 + 2 2 2 + 3 3 3 + 4 4 4 +} + +do_vtabsorter_test 2.7 { + SELECT a FROM t1 ORDER BY a DESC +} { 0 "{} {ORDER BY a DESC}" + 4 4 4 + 3 3 3 + 2 2 2 + 1 1 1 +} + +do_vtabsorter_test 2.8 { + SELECT a, count(*) FROM t1 GROUP BY a ORDER BY a +} { 0 "{} {ORDER BY a}" + 1 3 + 2 3 + 3 3 + 4 3 +} + +do_vtabsorter_test 2.9 { + SELECT a, count(*) FROM t1 GROUP BY a ORDER BY a DESC +} { 0 "{} {ORDER BY a DESC}" + 4 3 + 3 3 + 2 3 + 1 3 +} + +do_vtabsorter_test 2.10 { + SELECT a, count(*) FROM t1 GROUP BY a +} { 0 "{} {ORDER BY ((a+2)%5)}" + 3 3 + 4 3 + 1 3 + 2 3 +} + +do_vtabsorter_test 2.11 { + SELECT DISTINCT a, count(*) FROM t1 GROUP BY a +} { 1 "{} {ORDER BY ((a+2)%5)}" + 3 3 + 4 3 + 1 3 + 2 3 +} + +finish_test + diff --git a/test/carray01.test b/test/carray01.test index 86ea06996..b17a481e1 100644 --- a/test/carray01.test +++ b/test/carray01.test @@ -51,6 +51,10 @@ do_test 101 { run_stmt $STMT } {1} do_test 102 { + sqlite3_carray_bind -v2 -malloc $STMT 3 1 2 3 4 5 6 7 + run_stmt $STMT +} {1} +do_test 103 { set STMT2 [sqlite3_prepare_v2 db { SELECT DISTINCT typeof(value) FROM carray(?3)} -1] sqlite3_carray_bind $STMT2 3 1 2 3 4 5 6 7 @@ -124,6 +128,10 @@ do_test 160 { sqlite3_carray_bind -double $STMT 3 1 2 3 4 5 6 7 run_stmt $STMT } {1} +do_test 161 { + sqlite3_carray_bind -double -v2 $STMT 3 1 2 3 4 5 6 7 + run_stmt $STMT +} {1} do_test 170 { sqlite3_carray_bind -text -static $STMT 3 1 2 3 4 6 7 run_stmt $STMT diff --git a/test/collate5.test b/test/collate5.test index 71d4efe25..b474c39d5 100644 --- a/test/collate5.test +++ b/test/collate5.test @@ -122,9 +122,9 @@ do_test collate5-2.0 { } {} do_test collate5-2.1.1 { - execsql { + string toupper [execsql { SELECT a FROM collate5t1 UNION select a FROM collate5t2; - } + }] } {A B N} do_test collate5-2.1.2 { execsql { @@ -132,10 +132,10 @@ do_test collate5-2.1.2 { } } {A B N a b n} do_test collate5-2.1.3 { - execsql { + string tolower [execsql { SELECT a, b FROM collate5t1 UNION select a, b FROM collate5t2; - } -} {A Apple A apple B Banana b banana N {}} + }] +} {a apple a apple b banana b banana n {}} do_test collate5-2.1.4 { execsql { SELECT a, b FROM collate5t2 UNION select a, b FROM collate5t1; @@ -143,9 +143,9 @@ do_test collate5-2.1.4 { } {A Apple B banana N {} a apple b banana n {}} do_test collate5-2.2.1 { - execsql { + string toupper [execsql { SELECT a FROM collate5t1 EXCEPT select a FROM collate5t2; - } + }] } {N} do_test collate5-2.2.2 { execsql { @@ -153,10 +153,10 @@ do_test collate5-2.2.2 { } } {A a} do_test collate5-2.2.3 { - execsql { + string tolower [execsql { SELECT a, b FROM collate5t1 EXCEPT select a, b FROM collate5t2; - } -} {A Apple N {}} + }] +} {a apple n {}} do_test collate5-2.2.4 { execsql { SELECT a, b FROM collate5t2 EXCEPT select a, b FROM collate5t1 @@ -165,9 +165,9 @@ do_test collate5-2.2.4 { } {A apple a apple} do_test collate5-2.3.1 { - execsql { + string toupper [execsql { SELECT a FROM collate5t1 INTERSECT select a FROM collate5t2; - } + }] } {A B} do_test collate5-2.3.2 { execsql { @@ -175,10 +175,10 @@ do_test collate5-2.3.2 { } } {B b} do_test collate5-2.3.3 { - execsql { + string tolower [execsql { SELECT a, b FROM collate5t1 INTERSECT select a, b FROM collate5t2; - } -} {a apple B banana} + }] +} {a apple b banana} do_test collate5-2.3.4 { execsql { SELECT a, b FROM collate5t2 INTERSECT select a, b FROM collate5t1; diff --git a/test/corruptL.test b/test/corruptL.test index 52adf6fd7..8e3dd6012 100644 --- a/test/corruptL.test +++ b/test/corruptL.test @@ -234,7 +234,7 @@ do_execsql_test 2.1 { do_catchsql_test 2.2 { SELECT b,c FROM t1 ORDER BY a; -} {1 {database disk image is malformed}} +} {1 {out of memory}} #------------------------------------------------------------------------- reset_db diff --git a/test/cost.test b/test/cost.test index 6106caba8..5ef84c2b2 100644 --- a/test/cost.test +++ b/test/cost.test @@ -203,8 +203,8 @@ do_eqp_test 8.2 { } { QUERY PLAN |--SCAN track - |--SEARCH album USING INTEGER PRIMARY KEY (rowid=?) |--SEARCH composer USING INTEGER PRIMARY KEY (rowid=?) + |--SEARCH album USING INTEGER PRIMARY KEY (rowid=?) `--USE TEMP B-TREE FOR DISTINCT } diff --git a/test/dblwidth-a.sql b/test/dblwidth-a.sql index 38c219698..bcd60359f 100644 --- a/test/dblwidth-a.sql +++ b/test/dblwidth-a.sql @@ -1,20 +1,50 @@ +#!sqlite3 /* ** Run this script using "sqlite3" to confirm that the command-line ** shell properly handles the output of double-width characters. ** ** https://sqlite.org/forum/forumpost/008ac80276 */ +.testcase 100 .mode box CREATE TABLE data(word TEXT, description TEXT); INSERT INTO data VALUES('〈οὐκέτι〉','Greek without dblwidth <...>'); -.print .mode box SELECT * FROM data; +.check <<END +╭────────────┬──────────────────────────────╮ +│ word │ description │ +╞════════════╪══════════════════════════════╡ +│ 〈οὐκέτι〉 │ Greek without dblwidth <...> │ +╰────────────┴──────────────────────────────╯ +END + +.testcase 200 .mode table -.print .mode table SELECT * FROM data; +.check <<END ++------------+------------------------------+ +| word | description | ++------------+------------------------------+ +| 〈οὐκέτι〉 | Greek without dblwidth <...> | ++------------+------------------------------+ +END + +.testcase 300 .mode qbox -.print .mode qbox SELECT * FROM data; +.check <<END +╭──────────────┬────────────────────────────────╮ +│ word │ description │ +╞══════════════╪════════════════════════════════╡ +│ '〈οὐκέτι〉' │ 'Greek without dblwidth <...>' │ +╰──────────────┴────────────────────────────────╯ +END + +.testcase 400 .mode column -.print .mode column SELECT * FROM data; +.check <<END + word description +---------- ---------------------------- +〈οὐκέτι〉 Greek without dblwidth <...> +END diff --git a/test/distinct2.test b/test/distinct2.test index 980b0b1e3..5de4d3094 100644 --- a/test/distinct2.test +++ b/test/distinct2.test @@ -311,7 +311,7 @@ do_execsql_test 4010 { } do_execsql_test 4020 { SELECT b FROM t1 UNION SELECT 1; -} {1 { }} +} {1 {}} #------------------------------------------------------------------------- # diff --git a/test/dotcmd01.sql b/test/dotcmd01.sql new file mode 100644 index 000000000..751d32659 --- /dev/null +++ b/test/dotcmd01.sql @@ -0,0 +1,63 @@ +#!sqlite3 +# +# 2026-01-30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Miscellaneous tests for dot-commands +# +# ./sqlite3 test/dotcmd01.sql +# + +.testcase setup +.open :memory: +.mode tty +.check '' + +# The ".eqp on" setting does not affect the output from .fullschema +# and similar. +# +.testctrl opt -Stat4 +.testcase 100 +CREATE TABLE t1(a,b,c); +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<300) + INSERT INTO t1(a,b,c) + SELECT n%10, n%30, n%100 FROM c; +CREATE INDEX t1a ON t1(a); +CREATE INDEX t1b ON t1(b); +ANALYZE; +.eqp on +.fullschema +.check <<END +CREATE TABLE t1(a,b,c); +CREATE INDEX t1a ON t1(a); +CREATE INDEX t1b ON t1(b); +ANALYZE sqlite_schema; +INSERT INTO sqlite_stat1 VALUES('t1','t1b','300 10'); +INSERT INTO sqlite_stat1 VALUES('t1','t1a','300 30'); +ANALYZE sqlite_schema; +END + +.testcase 110 +.schema +.check <<END +CREATE TABLE t1(a,b,c); +CREATE INDEX t1a ON t1(a); +CREATE INDEX t1b ON t1(b); +CREATE TABLE sqlite_stat1(tbl,idx,stat); +END + +.testcase 120 +.tables +.check --glob "t1" + +.testcase 130 +.indexes +.check --glob t1a*t1b diff --git a/test/e_expr.test b/test/e_expr.test index 81d2fd172..5c0bfb0c1 100644 --- a/test/e_expr.test +++ b/test/e_expr.test @@ -1744,10 +1744,10 @@ do_execsql_test e_expr-32.2.8 { integer 9223372036854775807 \ integer 9223372036854775807 \ integer 9223372036854775807 \ - real 9.22337203685478e+18 \ - real 9.22337203685478e+18 \ - real 9.22337203685478e+18 \ - real 9.22337203685478e+18 \ + real 9.2233720368547758e+18 \ + real 9.2233720368547758e+18 \ + real 9.2233720368547758e+18 \ + real 9.2233720368547758e+18 \ integer -5 \ integer -5 \ ] diff --git a/test/e_update.test b/test/e_update.test index a13b059b3..2b3341dc7 100644 --- a/test/e_update.test +++ b/test/e_update.test @@ -325,6 +325,8 @@ foreach {tn sql error ac data } { # EVIDENCE-OF: R-43190-62442 In other words, the schema-name. prefix on # the table name of the UPDATE is not allowed within triggers. # +# Update: Unless the trigger is in the temp schema. +# do_update_tests e_update-2.1 -error { qualified table names are not allowed on INSERT, UPDATE, and DELETE statements within triggers } { @@ -339,12 +341,14 @@ do_update_tests e_update-2.1 -error { UPDATE aux.t1 SET a=1, b=2; END; } {} +} - 3 { - CREATE TRIGGER tr1 AFTER DELETE ON t4 BEGIN - UPDATE main.t1 SET a=1, b=2; - END; - } {} +# Qualified table name is allowed as t4 is a temp table. +do_execsql_test e_update-2.1.3 { + CREATE TRIGGER tr1 AFTER DELETE ON t4 BEGIN + UPDATE main.t1 SET a=1, b=2; + END; + DROP TRIGGER tr1; } # EVIDENCE-OF: R-06085-13761 Unless the table to which the trigger is diff --git a/test/e_walckpt.test b/test/e_walckpt.test index 3b1f3b015..4aae52a78 100644 --- a/test/e_walckpt.test +++ b/test/e_walckpt.test @@ -605,14 +605,15 @@ foreach {tn script} { sqlite3 db test.db foreach {tn mode res} { 0 -1001 {1 {SQLITE_MISUSE - not an error}} - 1 -1 {1 {SQLITE_MISUSE - not an error}} - 2 0 {0 {0 -1 -1}} - 3 1 {0 {0 -1 -1}} - 4 2 {0 {0 -1 -1}} - 5 3 {0 {0 -1 -1}} - 6 4 {1 {SQLITE_MISUSE - not an error}} - 7 114 {1 {SQLITE_MISUSE - not an error}} - 8 1000000 {1 {SQLITE_MISUSE - not an error}} + 1 -2 {1 {SQLITE_MISUSE - not an error}} + 2 -1 {0 {0 -1 -1}} + 3 0 {0 {0 -1 -1}} + 4 1 {0 {0 -1 -1}} + 5 2 {0 {0 -1 -1}} + 6 3 {0 {0 -1 -1}} + 7 4 {1 {SQLITE_MISUSE - not an error}} + 8 114 {1 {SQLITE_MISUSE - not an error}} + 9 1000000 {1 {SQLITE_MISUSE - not an error}} } { do_test 4.$tn { list [catch "wal_checkpoint_v2 db $mode" msg] $msg diff --git a/test/eqp.test b/test/eqp.test index 147b5ceaf..d2bdc49e3 100644 --- a/test/eqp.test +++ b/test/eqp.test @@ -124,10 +124,10 @@ do_eqp_test 1.8 { } { QUERY PLAN |--CO-ROUTINE (subquery-xxxxxx) - | `--COMPOUND QUERY - | |--LEFT-MOST SUBQUERY + | `--MERGE (UNION) + | |--LEFT | | `--SCAN CONSTANT ROW - | `--UNION USING TEMP B-TREE + | `--RIGHT | `--SCAN CONSTANT ROW |--SCAN (subquery-xxxxxx) `--SCAN t3 @@ -137,24 +137,26 @@ do_eqp_test 1.9 { } { QUERY PLAN |--CO-ROUTINE abc - | `--COMPOUND QUERY - | |--LEFT-MOST SUBQUERY + | `--MERGE (EXCEPT) + | |--LEFT | | `--SCAN CONSTANT ROW - | `--EXCEPT USING TEMP B-TREE - | `--SCAN t3 - |--SCAN abc - `--SCAN t3 + | `--RIGHT + | |--SCAN t3 + | `--USE TEMP B-TREE FOR ORDER BY + |--SCAN t3 + `--SCAN abc } do_eqp_test 1.10 { SELECT * FROM t3 JOIN (SELECT 1 INTERSECT SELECT a FROM t3 LIMIT 17) AS abc } { QUERY PLAN |--CO-ROUTINE abc - | `--COMPOUND QUERY - | |--LEFT-MOST SUBQUERY + | `--MERGE (INTERSECT) + | |--LEFT | | `--SCAN CONSTANT ROW - | `--INTERSECT USING TEMP B-TREE - | `--SCAN t3 + | `--RIGHT + | |--SCAN t3 + | `--USE TEMP B-TREE FOR ORDER BY |--SCAN abc `--SCAN t3 } @@ -455,10 +457,11 @@ do_eqp_test 4.3.1 { SELECT x FROM t1 UNION SELECT x FROM t2 } { QUERY PLAN - `--COMPOUND QUERY - |--LEFT-MOST SUBQUERY - | `--SCAN t1 - `--UNION USING TEMP B-TREE + `--MERGE (UNION) + |--LEFT + | |--SCAN t1 + | `--USE TEMP B-TREE FOR ORDER BY + `--RIGHT `--SCAN t2 USING COVERING INDEX t2i1 } @@ -466,13 +469,17 @@ do_eqp_test 4.3.2 { SELECT x FROM t1 UNION SELECT x FROM t2 UNION SELECT x FROM t1 } { QUERY PLAN - `--COMPOUND QUERY - |--LEFT-MOST SUBQUERY - | `--SCAN t1 - |--UNION USING TEMP B-TREE - | `--SCAN t2 USING COVERING INDEX t2i1 - `--UNION USING TEMP B-TREE - `--SCAN t1 + `--MERGE (UNION) + |--LEFT + | `--MERGE (UNION) + | |--LEFT + | | |--SCAN t1 + | | `--USE TEMP B-TREE FOR ORDER BY + | `--RIGHT + | `--SCAN t2 USING COVERING INDEX t2i1 + `--RIGHT + |--SCAN t1 + `--USE TEMP B-TREE FOR ORDER BY } do_eqp_test 4.3.3 { SELECT x FROM t1 UNION SELECT x FROM t2 UNION SELECT x FROM t1 ORDER BY 1 diff --git a/test/filectrl.test b/test/filectrl.test index 9b1a1c758..eea9a4773 100644 --- a/test/filectrl.test +++ b/test/filectrl.test @@ -36,13 +36,11 @@ do_test filectrl-1.5 { sqlite3 db test_control_lockproxy.db file_control_lockproxy_test db [get_pwd] } {} -ifcapable !winrt { - do_test filectrl-1.6 { - sqlite3 db test.db - set fn [file_control_tempfilename db] - set fn - } {/etilqs_/} -} +do_test filectrl-1.6 { + sqlite3 db test.db + set fn [file_control_tempfilename db] + set fn +} {/etilqs_/} db close forcedelete .test_control_lockproxy.db-conch test.proxy forcedelete test.db test2.db diff --git a/test/fpconv1.test b/test/fpconv1.test index 195fdf990..a93489907 100644 --- a/test/fpconv1.test +++ b/test/fpconv1.test @@ -17,28 +17,82 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl + +# Unusual rendering cases: +# +do_execsql_test fpconv1-1.0 { + SELECT 1.23 - 2.34; +} {-1.1099999999999999} +# ^--- Not -1.11 as you would expect. -1.11 has a different bit pattern + +do_execsql_test fpconv1-1.1 { + SELECT 1.23 * 2.34; +} {2.8781999999999996} +# ^--- Not 2.8782 as you would expect. 2.8782 has a different bit pattern + +# Change significant digits to 15 and get a different result. +sqlite3_db_config db FP_DIGITS 15 +do_execsql_test fpconv1-1.2 { + SELECT 1.23 - 2.34; +} {-1.11} +do_execsql_test fpconv1-1.3 { + SELECT 1.23 * 2.34; +} {2.8782} +sqlite3_db_config db FP_DIGITS 17 + + if {[catch {load_static_extension db decimal} error]} { puts "Skipping decimal tests, hit load error: $error" finish_test; return } sqlite3_create_function db -do_execsql_test fpconv1-1.0 { +do_execsql_test fpconv1-2.0 { WITH RECURSIVE /* Number of random floating-point values to try. - ** On a circa 2016 x64 linux box, this test runs at - ** about 80000 cases per second -------------------vvvvvv */ - c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100000), + ** On a circa 2021 Ryzen 5950X running Mint Linux, and + ** compiled with -O0 -DSQLITE_DEBUG, this test runs at + ** about 150000 cases per second ------------------vvvvvvv */ + c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<500_000), fp(y) AS MATERIALIZED ( SELECT CAST( format('%+d.%019d0e%+03d', random()%10,abs(random()),random()%200) AS real) FROM c ) SELECT y FROM fp - WHERE -log10(abs(decimal_sub(dtostr(y,24),format('%!.24e',y))/y))<15.0; + WHERE -log10(abs(decimal_sub(dtostr(y,24),format('%!.24e',y))/y))<17.0; /* Number of digits of accuracy required -------^^^^ */ } {} # ^---- Expect a empty set as the result. The output is all tested numbers -# that fail to preserve at least 15 significant digits of accuracy. +# that fail to preserve at least 16 significant digits of accuracy. + +######################################################################## +# Random test to ensure that double -> text -> double conversions +# round-trip exactly. +# + +load_static_extension db ieee754 + +do_execsql_test fpconv1-3.0 { + WITH RECURSIVE + c(x,s) AS MATERIALIZED (VALUES(1,random()&0xffefffffffffffff) + UNION ALL + SELECT x+1,random()&0xffefffffffffffff + FROM c WHERE x<1_000_000), + fp(y,s) AS ( + SELECT ieee754_from_int(s),s FROM c + ), + fp2(yt,y,s) AS ( + SELECT y||'', y, s FROM fp + ) + SELECT format('%#016x',s) aS 'orig-hex', + format('%#016x',ieee754_to_int(CAST(yt AS real))) AS 'full-roundtrip', + yt AS 'rendered-as', + decimal_exp(yt,30) AS 'text-decimal', + decimal_exp(ieee754_from_int(s),30) AS 'float-decimal' + FROM fp2 + WHERE ieee754_to_int(CAST(yt AS real))<>s; +} {} +# ^---- Values that fail to round-trip will be reported finish_test diff --git a/test/fptest01.sql b/test/fptest01.sql new file mode 100644 index 000000000..6221760cf --- /dev/null +++ b/test/fptest01.sql @@ -0,0 +1,76 @@ +#!sqlite3 +# +# 2026-03-01 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Floating-point to text conversions +# + +# Verify that binary64 -> text -> binary64 conversions round-trip +# successfully for 98,256 different edge-case binary64 values. The +# query result is all cases that do not round-trip without change, +# and so the query result should be an empty set. +# +.testcase 100 +.mode list +WITH + i1(i) AS (VALUES(0) UNION ALL SELECT i+1 FROM i1 WHERE i<15), + i2(j) AS (VALUES(0) UNION ALL SELECT j+1 FROM i2 WHERE j<0x7fe), + i3(k) AS (VALUES(0x0000000000000000), + (0x000ffffffffffff0), + (0x0008080808080800)), + fpint(n) AS (SELECT (j<<52)+i+k FROM i2, i1, i3), + fp(n,r) AS (SELECT n, ieee754_from_int(n) FROM fpint) +SELECT n, r FROM fp WHERE r<>(0.0 + (r||'')); +.check '' + +# Another batch of 106,444 edge cases: All postiive floating point +# values that have only a single bit set in the mantissa part of the +# number. +# +.testcase 110 +WITH + i1(i) AS (VALUES(0) UNION ALL SELECT i+1 FROM i1 WHERE i<51), + i2(j) AS (VALUES(0) UNION ALL SELECT j+1 FROM i2 WHERE j<0x7fe), + fpint(n) AS (SELECT (j<<52)+(1<<i) FROM i2, i1), + fp(n,r) AS (SELECT n, ieee754_from_int(n) FROM fpint) +SELECT n, r FROM fp WHERE r<>(0.0 + (r||'')); +.check '' + +# Verify that text -> binary64 conversions agree with system strtod(). +# for 98,256 different edge-cases. +# +.testcase 200 +.mode list +WITH + i1(i) AS (VALUES(0) UNION ALL SELECT i+1 FROM i1 WHERE i<15), + i2(j) AS (VALUES(0) UNION ALL SELECT j+1 FROM i2 WHERE j<0x7fe), + i3(k) AS (VALUES(0x0000000000000000), + (0x000ffffffffffff0), + (0x0008080808080800)), + fpint(n) AS (SELECT (j<<52)+i+k FROM i2, i1, i3), + fp(r) AS (SELECT ieee754_from_int(n) FROM fpint) +SELECT r FROM fp WHERE r<>strtod(r||''); +.check '' + + +# Another batch of 106,444 edge cases: All postiive floating point +# values that have only a single bit set in the mantissa part of the +# number. +# +.testcase 210 +WITH + i1(i) AS (VALUES(0) UNION ALL SELECT i+1 FROM i1 WHERE i<51), + i2(j) AS (VALUES(0) UNION ALL SELECT j+1 FROM i2 WHERE j<0x7fe), + fpint(n) AS (SELECT (j<<52)+(1<<i) FROM i2, i1), + fp(r) AS (SELECT ieee754_from_int(n) FROM fpint) +SELECT r FROM fp WHERE r<>strtod(r||''); +.check '' diff --git a/test/fts3comp1.test b/test/fts3comp1.test index 9f13aaa64..b5077a355 100644 --- a/test/fts3comp1.test +++ b/test/fts3comp1.test @@ -112,4 +112,81 @@ do_catchsql_test 2.2 { CREATE VIRTUAL TABLE t2 USING fts4(x, uncompress=unzip) } {1 {missing compress parameter in fts4 constructor}} +#-------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + PRAGMA trusted_schema = OFF; +} + +set ::myfunc_invoked 0 +proc myfunc {data} { + incr ::myfunc_invoked + return $data +} +db func myfunc myfunc + +do_execsql_test 3.1 { + CREATE VIEW v1 AS SELECT myfunc('xyz'); +} + +do_catchsql_test 3.2 { + SELECT * FROM v1 +} {1 {unsafe use of myfunc()}} + +do_execsql_test 3.3 { + CREATE VIRTUAL TABLE f1 USING fts4(x, compress=myfunc, uncompress=myfunc); +} + +do_catchsql_test 3.4 { + INSERT INTO f1(rowid, x) VALUES(123, 'one two three'); +} {1 {SQL logic error}} + +do_test 3.5 { + set ::myfunc_invoked +} {0} + +do_execsql_test 3.6.1 { + CREATE TABLE t1(x); + CREATE TABLE t2(y); + + CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN + INSERT INTO t2 VALUES( myfunc(new.x) ); + END; +} + +do_catchsql_test 3.6.2 { + INSERT INTO t1 VALUES('hello world'); +} {1 {unsafe use of myfunc()}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE v1 USING fts4(x, compress=comp, uncompress=uncomp); +} + +proc comp {data} { return $data } +proc uncomp {data} { return $data } + +db func comp comp +db func uncomp uncomp + +do_catchsql_test 4.1 { + INSERT INTO v1 VALUES('one two three'); +} {0 {}} + +db close +sqlite3 db test.db +db func comp -directonly comp + +do_catchsql_test 4.2 { + INSERT INTO v1 VALUES('one two three'); +} {1 {SQL logic error}} + +db func uncomp -directonly uncomp + +do_catchsql_test 4.3 { + SELECT * FROM v1 +} {1 {SQL logic error}} + + finish_test diff --git a/test/fts4content.test b/test/fts4content.test index 980586ea3..8268e734a 100644 --- a/test/fts4content.test +++ b/test/fts4content.test @@ -638,7 +638,6 @@ do_catchsql_test 11.1 { # Check that an fts4 table cannot be its own content table. # reset_db -breakpoint do_execsql_test 12.1.1 { CREATE VIRTUAL TABLE t1 USING fts4(a, content=t1 ); INSERT INTO t1(rowid, a) VALUES(1, 'abc'); @@ -669,6 +668,64 @@ do_catchsql_test 12.2.4 { SELECT count(*) FROM t1; } {1 {SQL logic error}} +#--------------------------------------------------------------------------- +# Check that an fts4 table cannot read from an unsafe vtab in a non-trusted +# schema. +reset_db +do_execsql_test 13.0 { + PRAGMA trusted_schema = off; + CREATE VIRTUAL TABLE t1 USING fts4(data, content=sqlite_dbpage); +} + +do_catchsql_test 13.1 { + INSERT INTO t1(t1) VALUES('rebuild'); +} {1 {SQL logic error}} + +proc vtab_command {method args} { + switch -- $method { + xConnect { + return "CREATE TABLE t1(a)" + } + + xBestIndex { + return "" + } + + xFilter { + return [list sql {SELECT 1, 123}] + } + + xUpdate { + return 123 + } + } + + return {} +} + +register_tcl_module db xyz + +do_execsql_test 13.2.0 { + CREATE VIRTUAL TABLE aa USING tcl(vtab_command); +} + +do_execsql_test 13.2.1 { + INSERT INTO aa VALUES('one two three'); +} + +do_test 13.2.2 { + set ::stmt [sqlite3_prepare_v3 db \ + "INSERT INTO aa VALUES('one two three');" -1 0x00 + ] + sqlite3_finalize $::stmt +} {SQLITE_OK} +do_test 13.2.2 { + list [catch { + set ::stmt [sqlite3_prepare_v3 db \ + "INSERT INTO aa VALUES('one two three');" -1 0x20 + ] + } msg] $msg +} {1 {(1) unsafe use of virtual table "aa"}} finish_test diff --git a/test/fts4merge5.test b/test/fts4merge5.test index 1fad778b9..90870ca34 100644 --- a/test/fts4merge5.test +++ b/test/fts4merge5.test @@ -53,6 +53,9 @@ for {set tn 1} {1} {incr tn} { } } +do_catchsql_test 1.5 { + INSERT INTO x1(x1) VALUES('maxpendinAB64'); +} {1 {SQL logic error}} finish_test diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index a3377770a..f056d2d93 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -1954,6 +1954,7 @@ int main(int argc, char **argv){ char **azSrcDb = 0; /* Array of source database names */ int iSrcDb; /* Loop over all source databases */ int nTest = 0; /* Total number of tests performed */ + int nSliceSkip = 0; /* Skipped due to --slice */ char *zDbName = ""; /* Appreviated name of a source database */ const char *zFailCode = 0; /* Value of the TEST_FAILURE env variable */ int cellSzCkFlag = 0; /* --cell-size-check */ @@ -2520,6 +2521,7 @@ int main(int argc, char **argv){ if( isDbSql(pSql->a, pSql->sz) ){ if( iSliceSz>0 && (nTest%iSliceSz)!=iSliceIdx ){ nTest++; + nSliceSkip++; continue; } sqlite3_snprintf(sizeof(g.zTestName), g.zTestName, "sqlid=%d",pSql->id); @@ -2582,6 +2584,7 @@ int main(int argc, char **argv){ const char *zVfs = "inmem"; if( iSliceSz>0 && (nTest%iSliceSz)!=iSliceIdx ){ nTest++; + nSliceSkip++; continue; } sqlite3_snprintf(sizeof(g.zTestName), g.zTestName, "sqlid=%d,dbid=%d", @@ -2693,12 +2696,13 @@ int main(int argc, char **argv){ if( briefFlag ){ sqlite3_int64 iElapse = timeOfDay() - iBegin; if( iSliceSz>0 ){ - printf("%s %s: slice %d/%d of %d tests, %d.%03d seconds\n", - pathTail(argv[0]), pathTail(g.zDbFile), - iSliceIdx, iSliceSz, nTest, - (int)(iElapse/1000), (int)(iElapse%1000)); + printf( + "%s %s: 0 errors out of %d tests, slice %d/%d, %d.%03d seconds\n", + pathTail(argv[0]), pathTail(g.zDbFile), + nTest - nSliceSkip, iSliceIdx, iSliceSz, + (int)(iElapse/1000), (int)(iElapse%1000)); }else{ - printf("%s %s: 0 errors, %d tests, %d.%03d seconds\n", + printf("%s %s: 0 errors out of %d tests, %d.%03d seconds\n", pathTail(argv[0]), pathTail(g.zDbFile), nTest, (int)(iElapse/1000), (int)(iElapse%1000)); } diff --git a/test/fuzzinvariants.c b/test/fuzzinvariants.c index 6a5cfda68..9341b2056 100644 --- a/test/fuzzinvariants.c +++ b/test/fuzzinvariants.c @@ -261,9 +261,12 @@ int fuzz_invariant( sqlite3_finalize(pCk); /* Invariants do not necessarily work if there are virtual tables - ** involved in the query */ - rc = sqlite3_prepare_v2(db, - "SELECT 1 FROM bytecode(?1) WHERE opcode='VOpen'", -1, &pCk, 0); + ** or scalar subqueries involved in the query */ + rc = sqlite3_prepare_v2(db, + "SELECT 1 FROM bytecode(?1)" + " WHERE opcode='VOpen' OR" + " (opcode='Explain' AND p4 GLOB 'SCALAR SUBQUERY*')", + -1, &pCk, 0); if( rc==SQLITE_OK ){ if( eVerbosity>=2 ){ char *zSql = sqlite3_expanded_sql(pCk); diff --git a/test/import01.sql b/test/import01.sql new file mode 100644 index 000000000..6e029b8b2 --- /dev/null +++ b/test/import01.sql @@ -0,0 +1,217 @@ +#!sqlite3 +# +# 2025-12-28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the ".import" command of the CLI. +# To run these tests: +# +# ./sqlite3 test/import01.sql +# + +.testcase setup +.open :memory: +.mode tty +.check '' + +.testcase 100 +CREATE TABLE t1(a,b,c); +.import -csv <<END t1 +111,222,333 +abc,def,ghi +END +SELECT * FROM t1; +.check <<END +╭───────┬───────┬───────╮ +│ a │ b │ c │ +╞═══════╪═══════╪═══════╡ +│ '111' │ '222' │ '333' │ +│ abc │ def │ ghi │ +╰───────┴───────┴───────╯ +END + +.testcase 110 +DELETE FROM t1; +.import -colsep ";" <<END t1 +this;is a;test +one;two;three +END +SELECT * FROM t1; +.check <<END +╭──────┬──────┬───────╮ +│ a │ b │ c │ +╞══════╪══════╪═══════╡ +│ this │ is a │ test │ +│ one │ two │ three │ +╰──────┴──────┴───────╯ +END + +.testcase 120 +DELETE FROM t1; +.import -colsep "," -rowsep ';' <<END t1 +this,is a,test;one,two,three; +END +SELECT * FROM t1; +.check <<END +╭──────┬──────┬───────╮ +│ a │ b │ c │ +╞══════╪══════╪═══════╡ +│ this │ is a │ test │ +│ one │ two │ three │ +╰──────┴──────┴───────╯ +END + +.testcase 130 +DELETE FROM t1; +.import -csv -colsep "," -rowsep "\n" <<END t1 +this,"is a","test ""with quotes""" +"second",,"line" +END +SELECT * FROM t1; +.check <<END +╭────────┬──────┬────────────────────╮ +│ a │ b │ c │ +╞════════╪══════╪════════════════════╡ +│ this │ is a │ test "with quotes" │ +│ second │ │ line │ +╰────────┴──────┴────────────────────╯ +END +.testcase 131 +DELETE FROM t1; +.import -ascii -colsep "," -rowsep "\n" <<END t1 +this,"is a","test ""with quotes""" +"second",,"line" +END +SELECT * FROM t1; +.check <<END +╭──────────┬────────┬────────────────────────╮ +│ a │ b │ c │ +╞══════════╪════════╪════════════════════════╡ +│ this │ "is a" │ "test ""with quotes""" │ +│ "second" │ │ "line" │ +╰──────────┴────────┴────────────────────────╯ +END + +.testcase 140 +DROP TABLE t1; +.import -csv <<END t1 +"abc","def","xy z" +"This","is","a" +"test","...", +END +SELECT * FROM t1; +.check <<END +╭──────┬─────┬──────╮ +│ abc │ def │ xy z │ +╞══════╪═════╪══════╡ +│ This │ is │ a │ +│ test │ ... │ │ +╰──────┴─────┴──────╯ +END +.testcase 141 +SELECT sql FROM sqlite_schema WHERE name='t1'; +.check <<END +╭───────────────────────────────────╮ +│ sql │ +╞═══════════════════════════════════╡ +│ CREATE TABLE "t1"( │ +│ "abc" ANY, "def" ANY, "xy z" ANY) │ +╰───────────────────────────────────╯ +END + +.testcase 150 +DROP TABLE t1; +.import -csv -v <<END t1 +"abc","def","xy z" +"This","is","a" +"test","...", +END +SELECT * FROM t1; +.check <<END +Column separator ",", row separator "\n" +CREATE TABLE "main"."t1"( +"abc" ANY, "def" ANY, "xy z" ANY) + +Added 2 rows with 0 errors using 3 lines of input +╭──────┬─────┬──────╮ +│ abc │ def │ xy z │ +╞══════╪═════╪══════╡ +│ This │ is │ a │ +│ test │ ... │ │ +╰──────┴─────┴──────╯ +END + +.testcase 160 +DROP TABLE t1; +.import -csv -schema TEMP <<END t2 +"x" +"abcdef" +END +SELECT * FROM t2; +.tables +.check <<END +╭────────╮ +│ x │ +╞════════╡ +│ abcdef │ +╰────────╯ +temp.t2 +END + +.testcase 170 +.import -csv -skip 2 <<END t3 +a,b,c +d,e,f +g,h,i +j,k,l +m,n,o +END +SELECT * FROM t3; +.check <<END +╭───┬───┬───╮ +│ g │ h │ i │ +╞═══╪═══╪═══╡ +│ j │ k │ l │ +│ m │ n │ o │ +╰───┴───┴───╯ +END + +.testcase 180 +DELETE FROM t3; +.import -csv -skip 7 <<END t3 +a,b,c +d,e,f +g,h,i +j,k,l +m,n,o +END +SELECT * FROM t3; +.check <<END +END + +.testcase 200 --error-prefix ERROR: +.import -csv +.check 'ERROR: Missing FILE argument' +.testcase 201 +.import -csv file1.csv +.check 'ERROR: Missing TABLE argument' +.testcase --error-prefix test-error: 202 +.import -csvxyzzy file1.csv +.check <<END +test-error: .import -csvxyzzy file1.csv +test-error: ^--- unknown option +END +.testcase 203 +.import -csv file1.csv t4 -colsep +.check <<END +test-error: .import -csv file1.csv t4 -colsep +test-error: missing argument ---^ +END diff --git a/test/imposter1.sql b/test/imposter1.sql new file mode 100644 index 000000000..9604b43f4 --- /dev/null +++ b/test/imposter1.sql @@ -0,0 +1,32 @@ +#!sqlite3 +# +# 2025-12-16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the .imposter command. +# +.mode box -reset +.testcase 100 +CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c INT); +INSERT INTO t1 VALUES(1,'two',3),(4,'five',6); +CREATE INDEX t1bc ON t1(b,c); +.imposter T1BC x1 +----------^^^^--- Different case that the original +SELECT * FROM x1; +.check <<END +CREATE TABLE "x1"("b","c","_ROWID_",PRIMARY KEY("b","c","_ROWID_"))WITHOUT ROWID; +╭──────┬───┬─────────╮ +│ b │ c │ _ROWID_ │ +╞══════╪═══╪═════════╡ +│ five │ 6 │ 4 │ +│ two │ 3 │ 1 │ +╰──────┴───┴─────────╯ +END diff --git a/test/insert5.test b/test/insert5.test index 1e58902e0..3ae99a5d2 100644 --- a/test/insert5.test +++ b/test/insert5.test @@ -96,22 +96,19 @@ do_test insert5-2.8 { } } {1} -# UPDATE: Using a column from the outer query (main.id) in the GROUP BY -# or ORDER BY of a sub-query is no longer supported. -# -# do_test insert5-2.9 { -# uses_temp_table { -# INSERT INTO b -# SELECT * FROM main -# WHERE id > 10 AND (SELECT count(*) FROM v2 GROUP BY main.id) -# } -# } {} do_test insert5-2.9 { catchsql { INSERT INTO b SELECT * FROM main WHERE id > 10 AND (SELECT count(*) FROM v2 GROUP BY main.id) } -} {1 {no such column: main.id}} +} {0 {}} +do_execsql_test insert5-2.10 { + CREATE TABLE t1(a INT); + INSERT INTO t1 VALUES(2); + CREATE TABLE t2(c INT, d INT); + INSERT INTO t2 VALUES(3,4),(10,NULL); + SELECT (SELECT c FROM t2 ORDER BY coalesce(d,a) LIMIT 1) FROM t1; +} {10} finish_test diff --git a/test/intck01.sql b/test/intck01.sql new file mode 100644 index 000000000..b1996aeeb --- /dev/null +++ b/test/intck01.sql @@ -0,0 +1,23 @@ +#!sqlite3 +# +# 2026-03-01 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Bug report sqlite.org/forum/forumpost/efc9bc9cb3 +# +.testcase 100 +.mode quote +.intck 1 +SELECT parse_create_index('CREATE IDEX i ON t("x',0); +.check <<END +1 steps, 0 errors +NULL +END diff --git a/test/join.test b/test/join.test index b33a7560a..a1ce7da0c 100644 --- a/test/join.test +++ b/test/join.test @@ -1369,4 +1369,45 @@ do_execsql_test join-32.3 { INNER JOIN t2 ON +y IS z; } {NULL NULL 123 NULL} +# 2025-12-24 https://sqlite.org/forum/forumpost/11a53f2bad +# +# Chained omit-noop-join optimization +# +reset_db +db null NULL +do_execsql_test join-33.1 { + CREATE TABLE t1(a1 INTEGER PRIMARY KEY, b1); + CREATE TABLE t2(a2 INTEGER PRIMARY KEY, b2); + CREATE TABLE t3(a3 INTEGER PRIMARY KEY, b3); + CREATE TABLE t4(a4 INTEGER PRIMARY KEY, b4); + INSERT INTO t1 VALUES(1,11),(2,12),(3,13), (5,15); + INSERT INTO t2 VALUES(1,21), (3,23),(4,24),(5,25); + INSERT INTO t3 VALUES (2,32),(3,33), (5,35); + INSERT INTO t4 VALUES(1,41),(2,42), (4,44),(5,45); + CREATE VIEW vchain AS + SELECT a1, b1, b2, b3, b4 + FROM t1 LEFT JOIN t2 ON a1=a2 + LEFT JOIN t3 ON a2=a3 + LEFT JOIN t4 ON a3=a4; +} +do_execsql_test join-33.2 { + SELECT a1 FROM vchain ORDER BY a1; +} {1 2 3 5} +do_eqp_test join-33.2-eqp { + SELECT a1 FROM vchain ORDER BY a1; +} { + QUERY PLAN + `--SCAN t1 +} +do_execsql_test join-33.3 { + SELECT a1, b2 FROM vchain ORDER BY a1; +} {1 21 2 NULL 3 23 5 25} +do_eqp_test join-33.3-eqp { + SELECT a1, b2 FROM vchain ORDER BY a1; +} { + QUERY PLAN + |--SCAN t1 + `--SEARCH t2 USING INTEGER PRIMARY KEY (rowid=?) LEFT-JOIN +} + finish_test diff --git a/test/joinI.test b/test/joinI.test index 577ca4c2c..3390afc6e 100644 --- a/test/joinI.test +++ b/test/joinI.test @@ -121,5 +121,52 @@ do_execsql_test 5.1 { INNER JOIN child2 ON child2.child2key = parent1.child2key; } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE TABLE t5(a, d); + CREATE TABLE t6(a, e); + INSERT INTO t5 VALUES(1, 'red'); + INSERT INTO t6 VALUES(0, 1000); + + CREATE TABLE t7(x); + CREATE TABLE t8(y); +} + +do_catchsql_test 6.1 { + SELECT * FROM t6 CROSS JOIN (t7 RIGHT JOIN t8 ON (t6.a)); +} {1 {no such column: t6.a}} + +do_catchsql_test 6.4 { + SELECT * FROM t7 RIGHT JOIN t8 ON (t6.a) CROSS JOIN t6; +} {1 {ON clause references tables to its right}} + +do_catchsql_test 6.5 { + SELECT * FROM + (SELECT * FROM t7 RIGHT JOIN t8 ON (t6.a) CROSS JOIN t6); +} {1 {ON clause references tables to its right}} + +do_catchsql_test 6.6 { + SELECT *, NOT EXISTS ( + SELECT * FROM t7 RIGHT JOIN t8 ON (t6.a) CROSS JOIN t6 + ) FROM t5; +} {1 {ON clause references tables to its right}} + +do_catchsql_test 6.7 { + SELECT *, NOT EXISTS ( + SELECT 1 + EXCEPT + SELECT 11 FROM t7 RIGHT JOIN t8 ON (t6.a) CROSS JOIN t6 + ) FROM t5; +} {1 {ON clause references tables to its right}} + +do_catchsql_test 6.8 { + SELECT *, NOT EXISTS ( + SELECT 11 FROM t7 RIGHT JOIN t8 ON (t6.a) CROSS JOIN t6 + EXCEPT + SELECT 1 + ) FROM t5; +} {1 {ON clause references tables to its right}} + finish_test diff --git a/test/json102.test b/test/json102.test index 54a0e1e0e..54f66a83f 100644 --- a/test/json102.test +++ b/test/json102.test @@ -375,6 +375,27 @@ do_execsql_test json102-440-3 { do_execsql_test json102-440-4 { SELECT json(jsonb_remove(jsonb('[0,1,2,3,4]'),'$[2]')); } {{[0,1,3,4]}} +do_execsql_test json102-445-1 { + SELECT json_remove('[0,1,2,3,4]','$[5]'); +} {{[0,1,2,3,4]}} +do_execsql_test json102-445-2 { + SELECT json_remove('[0,1,2,3,4]','$[6]'); +} {{[0,1,2,3,4]}} +do_execsql_test json102-445-3 { + SELECT json_remove('[0,1,2,3,4]','$[4294967295]'); +} {{[0,1,2,3,4]}} +do_execsql_test json102-445-4 { + SELECT json_remove('[0,1,2,3,4]','$[4294967296]'); +} {{[0,1,2,3,4]}} +do_execsql_test json102-445-5 { + SELECT json_remove('[0,1,2,3,4]','$[4294967297]'); +} {{[0,1,2,3,4]}} +do_execsql_test json102-445-6 { + SELECT json_remove('[0,1,2,3,4]','$[42949672950]'); +} {{[0,1,2,3,4]}} +do_execsql_test json102-445-7 { + SELECT json_remove('[0,1,2,3,4]','$[42949672960]'); +} {{[0,1,2,3,4]}} do_execsql_test json102-450 { SELECT json_remove('[0,1,2,3,4]','$[2]','$[0]'); } {{[1,3,4]}} diff --git a/test/json103.test b/test/json103.test index f94217ac1..9eadf29f8 100644 --- a/test/json103.test +++ b/test/json103.test @@ -27,6 +27,9 @@ do_execsql_test json103-100 { do_catchsql_test json103-101 { SELECT json_group_array(a) FROM t1; } {1 {JSON cannot hold BLOB values}} +do_execsql_test json103-102 { + SELECT quote(jsonb_group_array(a)) FROM t1 WHERE a<0 AND typeof(a)!='blob'; +} {X'0B'} do_execsql_test json103-110 { SELECT json_group_array(a) FROM t1 WHERE rowid BETWEEN 31 AND 39; @@ -45,6 +48,10 @@ do_execsql_test json103-200 { do_catchsql_test json103-201 { SELECT json_group_object(c,a) FROM t1; } {1 {JSON cannot hold BLOB values}} +do_execsql_test json103-202 { + SELECT quote(jsonb_group_object(c,a)) FROM t1 WHERE a<0 AND typeof(a)!='blob'; +} {X'0C'} + do_execsql_test json103-210 { SELECT json_group_object(c,a) FROM t1 diff --git a/test/json105.test b/test/json105.test index 509db94e1..4a5572cf0 100644 --- a/test/json105.test +++ b/test/json105.test @@ -29,6 +29,11 @@ json_extract_test 30 {'$.b[#-2]'} {'[2,3]'} json_extract_test 31 {'$.b[#-02]'} {'[2,3]'} json_extract_test 40 {'$.b[#-3]'} 1 json_extract_test 50 {'$.b[#-4]'} NULL +json_extract_test 51 {'$.b[#-4296967295]'} NULL +json_extract_test 52 {'$.b[#-4296967296]'} NULL +json_extract_test 53 {'$.b[#-4296967297]'} NULL +json_extract_test 54 {'$.b[#-42969672950]'} NULL +json_extract_test 55 {'$.b[#-42969672960]'} NULL json_extract_test 60 {'$.b[#-2][#-1]'} 3 json_extract_test 70 {'$.b[0]','$.b[#-1]'} {'[1,4]'} diff --git a/test/json109.test b/test/json109.test new file mode 100644 index 000000000..1631a1f0f --- /dev/null +++ b/test/json109.test @@ -0,0 +1,72 @@ +# 2026-01-17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for json_array_insert(). +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix json109 + +do_execsql_test 1.1 { + SELECT json_array_insert('[1,2,3]','$[0]',999,'$[0]',888); +} {{[888,999,1,2,3]}} +do_execsql_test 1.2 { + SELECT json_array_insert('[1,2,3]','$[0]',999,'$[#]',888); +} {{[999,1,2,3,888]}} +do_execsql_test 1.3 { + SELECT json_array_insert('[1,2,3]','$[1]',888); +} {{[1,888,2,3]}} +do_execsql_test 1.4 { + SELECT json_array_insert('[1,2,3]','$[2]',888); +} {{[1,2,888,3]}} +do_execsql_test 1.5 { + SELECT json_array_insert('[1,2,3]','$[3]',888); +} {{[1,2,3,888]}} +do_execsql_test 1.6 { + SELECT json_array_insert('[1,2,3]','$[#-1]',888); +} {{[1,2,888,3]}} +do_execsql_test 1.7 { + SELECT json_array_insert('[1,2,3]','$[#-2]',888); +} {{[1,888,2,3]}} +do_execsql_test 1.8 { + SELECT json_array_insert('[1,2,3]','$[#-3]',888); +} {{[888,1,2,3]}} +do_execsql_test 1.9 { + SELECT json_array_insert('[1,2,3]','$[#-4]',888); +} {{[1,2,3]}} + +do_catchsql_test 2.1 { + SELECT json_array_insert('{a:[1,2,3]}','$.a',888); +} {1 {not an array element: '$.a'}} +do_catchsql_test 2.2 { + SELECT json_array_insert('{a:[1,2,3]}','$.b',888); +} {1 {not an array element: '$.b'}} +do_catchsql_test 2.3 { + SELECT json_array_insert('{a:[1,2,3]}','$.b[0]',888); +} {0 {{{"a":[1,2,3],"b":[888]}}}} +do_catchsql_test 2.4 { + SELECT json_array_insert('{a:[1,2,3]}','$.b.c.d[0]',888); +} {0 {{{"a":[1,2,3],"b":{"c":{"d":[888]}}}}}} +do_catchsql_test 2.5 { + SELECT json_array_insert('{a:[1,2,3]}','$.b.c.d[0',888); +} {1 {not an array element: '$.b.c.d[0'}} +do_catchsql_test 2.6 { + SELECT json_array_insert('{a:[1,2,3]}','$.b.c.d',888); +} {1 {not an array element: '$.b.c.d'}} +do_catchsql_test 2.7 { + SELECT json_array_insert('{a:[1,2,3]}','$[0]',888); +} {0 {{{"a":[1,2,3]}}}} +do_catchsql_test 2.8 { + SELECT json_array_insert('{a:[1,2,3]}','$.b[0]',888,'$.a[1]','999','$.c',0); +} {1 {not an array element: '$.c'}} + +finish_test diff --git a/test/misc5.test b/test/misc5.test index 43ee2781a..80b8d3c67 100644 --- a/test/misc5.test +++ b/test/misc5.test @@ -595,7 +595,7 @@ do_test misc5-7.1.2 { } append sql "0$tail); SELECT * FROM t1;" catchsql $sql -} {0 900} +} {1 {Recursion limit}} # Parser stack overflow is silently ignored when it occurs while parsing the diff --git a/test/modeA.sql b/test/modeA.sql new file mode 100644 index 000000000..4e62093b2 --- /dev/null +++ b/test/modeA.sql @@ -0,0 +1,303 @@ +#!sqlite3 +# +# 2025-11-12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the ".mode" command of the CLI. +# To run these tests: +# +# ./sqlite3 test/modeA.sql +# +# +.open :memory: +CREATE TABLE t1(a,b,c,d,e); +INSERT INTO t1 VALUES(1,2.5,'three',x'4444',NULL); +INSERT INTO t1 SELECT b,c,d,e,a FROM t1; +INSERT INTO t1 SELECT d,e,a,b,c FROM t1; +.mode box + +.testcase 100 +SELECT * FROM t1; +.check <<END +╭─────┬───────┬───────┬───────┬───────╮ +│ a │ b │ c │ d │ e │ +╞═════╪═══════╪═══════╪═══════╪═══════╡ +│ 1 │ 2.5 │ three │ DD │ │ +│ 2.5 │ three │ DD │ │ 1 │ +│ DD │ │ 1 │ 2.5 │ three │ +│ │ 1 │ 2.5 │ three │ DD │ +╰─────┴───────┴───────┴───────┴───────╯ +END + +.testcase 110 +.mode --null xyz +SELECT * FROM t1; +.check <<END +╭─────┬───────┬───────┬───────┬───────╮ +│ a │ b │ c │ d │ e │ +╞═════╪═══════╪═══════╪═══════╪═══════╡ +│ 1 │ 2.5 │ three │ DD │ xyz │ +│ 2.5 │ three │ DD │ xyz │ 1 │ +│ DD │ xyz │ 1 │ 2.5 │ three │ +│ xyz │ 1 │ 2.5 │ three │ DD │ +╰─────┴───────┴───────┴───────┴───────╯ +END + +# Default output mode is qbox --quote relaxed +# +.mode tty --wrap 10 +CREATE TABLE t2(a,b,c,d); +INSERT INTO t2 VALUES(1,2.5,'three',x'4444'); +INSERT INTO t2 VALUES('The quick fox jumps over the lazy brown dog',2,3,4); +INSERT INTO t2 VALUES('10','', -1.25,NULL); +INSERT INTO t2 VALUES('a,b,c','"Double-Quoted"','-1.25','NULL'); +.testcase 120 +SELECT * FROM t2; +.check <<END +╭────────────┬────────────┬─────────┬─────────╮ +│ a │ b │ c │ d │ +╞════════════╪════════════╪═════════╪═════════╡ +│ 1 │ 2.5 │ three │ x'4444' │ +├────────────┼────────────┼─────────┼─────────┤ +│ The quick │ 2 │ 3 │ 4 │ +│ fox jumps │ │ │ │ +│ over the │ │ │ │ +│ lazy brown │ │ │ │ +│ dog │ │ │ │ +├────────────┼────────────┼─────────┼─────────┤ +│ '10' │ │ -1.25 │ NULL │ +├────────────┼────────────┼─────────┼─────────┤ +│ a,b,c │ "Double- │ '-1.25' │ 'NULL' │ +│ │ Quoted" │ │ │ +╰────────────┴────────────┴─────────┴─────────╯ +END +.testcase 130 +.mode +.check <<END +.mode qbox --limits on --quote relaxed --sw auto --textjsonb on +END +.testcase 140 +.mode -v +.check <<END +.mode qbox --align "" --border on --blob-quote auto --colsep "" --escape auto --limits on --null "NULL" --quote relaxed --rowsep "" --sw auto --tablename "" --textjsonb on --titles on --widths "" --wordwrap off --wrap 10 +END +.testcase 150 --error-prefix "Error:" +.mode foo +.check <<END +Error: .mode foo +Error: ^--- unknown mode +Error: Use ".help .mode" for more info +END + +.testcase 160 +.mode --null xyzzy -v +.output -glob ' --null "xyzzy"' +.testcase 170 +.mode -null abcde -v +.output -glob ' --null "abcde"' + +# Test cases for the ".explain off" command +.mode box -reset +.testcase 180 +EXPLAIN SELECT * FROM t1; +.output --notglob *────* --keep +.output --notglob "* id │ parent │ notused │ detail *" --keep +.output --glob "* Init *" +.testcase 190 +EXPLAIN QUERY PLAN SELECT * FROM t1; +.output --glob "*`--SCAN *" +.explain off +.testcase 200 +EXPLAIN SELECT * FROM t1; +.output --glob *────* +.testcase 210 +EXPLAIN QUERY PLAN SELECT * FROM t1; +.output --glob "* id │ parent │ notused │ detail *" +.explain auto + +# Test cases for limit settings in the .mode command. +.testcase 300 +.mode box --reset +.mode +.check <<END +.mode box +END +.testcase 310 +.mode --limits 5,300,20 +.mode +.check <<END +.mode box --limits on +END +.testcase 320 +.mode --limits 5,300,19 +.mode +.check <<END +.mode box --limits 5,300,19 +END +.testcase 330 +.mode --limits 0,0,0 +.mode -v +.check <<END +.mode box --align "" --border on --blob-quote auto --colsep "" --escape auto --limits off --null "" --quote off --rowsep "" --sw 0 --tablename "" --textjsonb off --titles on --widths "" --wordwrap off +END + +.testcase 400 +.mode --linelimit 123 +.mode +.check <<END +.mode box --limits 123,0,0 +END + +.testcase 410 +.mode --linelimit 0 -charlimit 123 +.mode +.check <<END +.mode box --limits 0,123,0 +END + +.testcase 420 +.mode --charlimit 0 -titlelimit 123 +.mode +.check <<END +.mode box --limits 0,0,123 +END + +.testcase 430 +.mode list +.mode +.check <<END +.mode list +END + +.testcase 440 +.mode -limits 0,123,0 +.mode +.check <<END +.mode list --limits 0,123,0 +END + +.testcase 450 +.mode -limits 123,0,0 +.mode +.check <<END +.mode list +END + +# --titlelimit functionality +# +.testcase 500 +.mode line --limits off --titlelimit 20 +SELECT a AS 'abcdefghijklmnopqrstuvwxyz', b FROM t2 WHERE c=3; +.check <<END +abcdefghijklmnopq...: The quick fox jumps over the lazy brown dog + b: 2 +END +.testcase 510 +.mode line --titlelimit 10 +SELECT a AS 'abcdefghijklmnopqrstuvwxyz', b FROM t2 WHERE c=3; +.check <<END +abcdefg...: The quick fox jumps over the lazy brown dog + b: 2 +END +.testcase 520 +.mode line --titlelimit 2 +SELECT a AS 'abcdefghijklmnopqrstuvwxyz', b FROM t2 WHERE c=3; +.check <<END +ab: The quick fox jumps over the lazy brown dog + b: 2 +END +.testcase 530 +.mode line --titlelimit 4 +SELECT a AS 'abcd', b FROM t2 WHERE c=3; +.check <<END +abcd: The quick fox jumps over the lazy brown dog + b: 2 +END +.testcase 540 +.mode line --titlelimit 3 +SELECT a AS 'abcd', b FROM t2 WHERE c=3; +.check <<END +...: The quick fox jumps over the lazy brown dog + b: 2 +END + +# https://sqlite.org/forum/forumpost/2025-12-31T19:14:24z +# +# For legacy compatibility, ".header" settings are not changed +# by ".mode" unless the --title or --reset option is used on .mode. +# +.testcase 600 +DROP TABLE IF EXISTS t1; +CREATE TABLE t1(a,b,c); +INSERT INTO t1 VALUES(1,2,3); +.header on +.mode csv +SELECT * FROM t1; +.check --glob a,b,c* + +.testcase 610 +.mode csv -reset +SELECT * FROM t1; +.check 1,2,3 + +.testcase 620 +.mode tty +.mode csv +.header on +SELECT * FROM t1; +.check --glob a,b,c* + +.testcase 630 +.mode tty +.mode csv --title on +SELECT * FROM t1; +.check --glob a,b,c* +.testcase 631 +.mode tty +.mode csv --title off +SELECT * FROM t1; +.check 1,2,3 + +# Verification of claims about .insert mode in the climode.html +# documentation. +.testcase 700 +CREATE TABLE tbl1(one,two); +INSERT INTO tbl1 VALUES('hello!',10),('goodbye',20); +.mode insert new_table +SELECT * FROM tbl1; +.check <<END +INSERT INTO new_table VALUES('hello!',10); +INSERT INTO new_table VALUES('goodbye',20); +END +.testcase 710 +.mode insert new_table --titles on +SELECT * FROM tbl1; +.check <<END +INSERT INTO new_table(one,two) VALUES('hello!',10); +INSERT INTO new_table(one,two) VALUES('goodbye',20); +END +.testcase 720 +.mode insert new_table --titles off +SELECT * FROM tbl1; +.check <<END +INSERT INTO new_table VALUES('hello!',10); +INSERT INTO new_table VALUES('goodbye',20); +END + +# QRF reports an error if the string is too big. +# +.testcase 800 +.mode box +.limit length 1000 +WITH c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<100) +SELECT hex(randomblob(100)) c; +.check -glob "*: string or blob too big" +.limit length 10000000 diff --git a/test/mutex1.test b/test/mutex1.test index cb189a7a8..de291f4c9 100644 --- a/test/mutex1.test +++ b/test/mutex1.test @@ -115,6 +115,10 @@ ifcapable threadsafe1&&shared_cache { } } { + ifcapable thread_misuse_warnings { + if {$mode ne "serialized"} continue + } + # For journal_mode=memory, the static_prng mutex is not required. This # is because the header of an in-memory journal does not contain # any random bytes, and so no call to sqlite3_randomness() is made. @@ -177,16 +181,18 @@ ifcapable threadsafe1&&shared_cache { # Open and use a connection in "nomutex" mode. Test that no recursive # mutexes are obtained. - do_test mutex1.3.1 { - catch {db close} - clear_mutex_counters - sqlite3 db test.db -nomutex 1 - execsql { SELECT * FROM abc } - } {1 2 3 1 2 3 1 2 3} - do_test mutex1.3.2 { - mutex_counters counters - set counters(recursive) - } {0} + ifcapable !thread_misuse_warnings { + do_test mutex1.3.1 { + catch {db close} + clear_mutex_counters + sqlite3 db test.db -nomutex 1 + execsql { SELECT * FROM abc } + } {1 2 3 1 2 3 1 2 3} + do_test mutex1.3.2 { + mutex_counters counters + set counters(recursive) + } {0} + } } # Test the sqlite3_db_mutex() function. diff --git a/test/notnull2.test b/test/notnull2.test index 67d7c26a8..f49a13b56 100644 --- a/test/notnull2.test +++ b/test/notnull2.test @@ -66,7 +66,7 @@ do_vmstep_test 1.5.2 { SELECT count(*) FROM t2 WHERE EXISTS( SELECT 1 FROM t1 WHERE t1.a=450 AND t2.c IS NULL ) -} 4000 {0} +} 5000 {0} #------------------------------------------------------------------------- reset_db diff --git a/test/offset1.test b/test/offset1.test index 5b04bd836..fb68f0d02 100644 --- a/test/offset1.test +++ b/test/offset1.test @@ -190,6 +190,20 @@ do_execsql_test offset1-2.0 { ORDER BY salary asc); } {} do_execsql_test offset1-2.1 { + SELECT * FROM v ORDER BY +id; +} { + 11 Diane London hr 70 + 12 Bob London hr 78 + 21 Emma London it 84 + 22 Grace Berlin it 90 + 23 Henry London it 104 + 24 Irene Berlin it 104 + 25 Frank Berlin it 120 + 31 Cindy Berlin sales 96 + 32 Dave London sales 96 + 33 Alice Berlin sales 100 +} +do_execsql_test offset1-2.2 { SELECT * FROM v LIMIT 5 OFFSET 2; } { 22 Grace Berlin it 90 @@ -198,5 +212,19 @@ do_execsql_test offset1-2.1 { 11 Diane London hr 70 33 Alice Berlin sales 100 } +do_execsql_test offset1-2.3 { + SELECT * FROM v LIMIT 3 OFFSET 6; +} { + 33 Alice Berlin sales 100 + 23 Henry London it 104 + 24 Irene Berlin it 104 +} +do_execsql_test offset1-2.4 { + SELECT * FROM v LIMIT 3 OFFSET 1; +} { + 32 Dave London sales 96 + 22 Grace Berlin it 90 + 21 Emma London it 84 +} finish_test diff --git a/test/qrf01.test b/test/qrf01.test new file mode 100644 index 000000000..3ae027957 --- /dev/null +++ b/test/qrf01.test @@ -0,0 +1,1167 @@ +# 2025-11-05 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the Query Result Formatter (QRF) +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix qrf01 + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1,2.5,'three'),(x'424c4f42',NULL,'Ἀμήν'); +} + +do_test 1.10 { + set result "\n[db format {SELECT * FROM t1}]" +} { +╭──────┬─────┬───────╮ +│ a │ b │ c │ +╞══════╪═════╪═══════╡ +│ 1 │ 2.5 │ three │ +│ BLOB │ │ Ἀμήν │ +╰──────┴─────┴───────╯ +} +do_test 1.11a { + set result "\n[db format -title off {SELECT * FROM t1}]" +} { +╭──────┬─────┬───────╮ +│ 1 │ 2.5 │ three │ +│ BLOB │ │ Ἀμήν │ +╰──────┴─────┴───────╯ +} +do_test 1.11b { + set result "\n[db format -text sql {SELECT * FROM t1}]" +} { +╭─────────────┬─────┬─────────╮ +│ a │ b │ c │ +╞═════════════╪═════╪═════════╡ +│ 1 │ 2.5 │ 'three' │ +│ x'424c4f42' │ │ 'Ἀμήν' │ +╰─────────────┴─────┴─────────╯ +} +do_test 1.11c { + set result "\n[db format -text sql -border off {SELECT * FROM t1}]" +} { + a │ b │ c +═════════════╪═════╪═════════ + 1 │ 2.5 │ 'three' + x'424c4f42' │ │ 'Ἀμήν' +} +do_test 1.11d { + set result "\n[db format -text relaxed -blob sql -border off \ + {SELECT * FROM t1}]" +} { + a │ b │ c +═════════════╪═════╪═══════ + 1 │ 2.5 │ three + x'424c4f42' │ │ Ἀμήν +} +do_test 1.12 { + set result "\n[db format -text csv {SELECT * FROM t1}]" +} { +╭────────────────────┬─────┬────────╮ +│ a │ b │ c │ +╞════════════════════╪═════╪════════╡ +│ 1 │ 2.5 │ three │ +│ "\102\114\117\102" │ │ "Ἀμήν" │ +╰────────────────────┴─────┴────────╯ +} +do_test 1.13 { + set result "\n[db format -text csv -blob hex {SELECT * FROM t1}]" +} { +╭──────────┬─────┬────────╮ +│ a │ b │ c │ +╞══════════╪═════╪════════╡ +│ 1 │ 2.5 │ three │ +│ 424c4f42 │ │ "Ἀμήν" │ +╰──────────┴─────┴────────╯ +} +do_test 1.14 { + catch {db format -text unk -blob hex {SELECT * FROM t1}} res + set res +} {bad -text "unk": must be auto, csv, html, json, plain, relaxed, sql, or tcl} +do_test 1.15 { + catch {db format -text sql -blob unk {SELECT * FROM t1}} res + set res +} {bad BLOB encoding (-blob) "unk": must be auto, hex, json, tcl, text, sql, or size} +do_test 1.16 { + catch {db format -text sql -style unk {SELECT * FROM t1}} res + set res +} {bad format style (-style) "unk": must be auto, box, column, count, csv, eqp, explain, html, insert, jobject, json, line, list, markdown, quote, stats, stats-est, stats-vm, or table} + + +do_test 1.20 { + set result "\n[db format -style box {SELECT * FROM t1}]" +} { +╭──────┬─────┬───────╮ +│ a │ b │ c │ +╞══════╪═════╪═══════╡ +│ 1 │ 2.5 │ three │ +│ BLOB │ │ Ἀμήν │ +╰──────┴─────┴───────╯ +} + +do_test 1.30 { + set result "\n[db format -style table {SELECT * FROM t1}]" +} { ++------+-----+-------+ +| a | b | c | ++------+-----+-------+ +| 1 | 2.5 | three | +| BLOB | | Ἀμήν | ++------+-----+-------+ +} +do_test 1.31 { + set result "\n[db format -style table -title off {SELECT * FROM t1}]" +} { ++------+-----+-------+ +| 1 | 2.5 | three | +| BLOB | | Ἀμήν | ++------+-----+-------+ +} +do_test 1.32 { + set result "\n[db format -style table -border off {SELECT * FROM t1}]" +} { + a | b | c +------+-----+------- + 1 | 2.5 | three + BLOB | | Ἀμήν +} +do_test 1.33 { + set result "\n[db format -style table -border off \ + -screenwidth 15 \ + {SELECT * FROM t1}]" +} { + a | b | c +----+---+----- + 1|2.5|three +BLOB| |Ἀμήν +} +do_test 1.34 { + set result "\n[db format -style box -border off \ + -screenwidth 30 \ + {SELECT * FROM t1}]" +} { + a │ b │ c +══════╪═════╪═══════ + 1 │ 2.5 │ three + BLOB │ │ Ἀμήν +} +do_test 1.35 { + set result "\n[db format -style box -border off \ + -screenwidth 15 \ + {SELECT * FROM t1}]" +} { + a │ b │ c +════╪═══╪═════ + 1│2.5│three +BLOB│ │Ἀμήν +} + +do_test 1.40 { + set result "\n[db format -style column {SELECT * FROM t1}]" +} { +a b c +---- --- ----- +1 2.5 three +BLOB Ἀμήν +} +do_test 1.41 { + set result "\n[db format -style column -title off {SELECT * FROM t1}]" +} { +1 2.5 three +BLOB Ἀμήν +} + +do_test 1.50 { + db format -style count {SELECT * FROM t1} +} 2 + +do_test 1.60a { + db format -style list -columnsep , -rowsep \r\n -text csv -blob tcl {SELECT * FROM t1} +} "1,2.5,three\r\n\"\\102\\114\\117\\102\",,\"Ἀμήν\"\r\n" +do_test 1.60b { + db format -style csv -columnsep xyz -rowsep pqr -text sql -blob sql {SELECT * FROM t1} +} "1,2.5,three\r\nx'424c4f42',,\"Ἀμήν\"\r\n" +do_test 1.61a { + db format -style list -columnsep , -rowsep \r\n -text csv -title auto -blob tcl {SELECT * FROM t1} +} "a,b,c\r\n1,2.5,three\r\n\"\\102\\114\\117\\102\",,\"Ἀμήν\"\r\n" +do_test 1.61b { + db format -style csv -title auto -blob tcl {SELECT * FROM t1} +} "a,b,c\r\n1,2.5,three\r\n\"\\102\\114\\117\\102\",,\"Ἀμήν\"\r\n" +do_test 1.62a { + db format -style list -columnsep , -rowsep \r\n -text csv -title csv -blob tcl {SELECT a AS 'a x y', b, c FROM t1} +} "\"a x y\",b,c\r\n1,2.5,three\r\n\"\\102\\114\\117\\102\",,\"Ἀμήν\"\r\n" +do_test 1.62b { + db format -style csv -title csv -blob tcl {SELECT a AS 'a x y', b, c FROM t1} +} "\"a x y\",b,c\r\n1,2.5,three\r\n\"\\102\\114\\117\\102\",,\"Ἀμήν\"\r\n" + +do_test 1.70 { + set result "\n[db format -style html {SELECT * FROM t1}]" +} { +<TR> +<TD>1 +<TD>2.5 +<TD>three +</TR> +<TR> +<TD>BLOB +<TD>null +<TD>Ἀμήν +</TR> +} +do_test 1.71 { + set result "\n[db format -style html -title auto {SELECT * FROM t1}]" +} { +<TR> +<TH>a +<TH>b +<TH>c +</TR> +<TR> +<TD>1 +<TD>2.5 +<TD>three +</TR> +<TR> +<TD>BLOB +<TD>null +<TD>Ἀμήν +</TR> +} + +do_test 1.80 { + set result "\n[db format -style insert {SELECT * FROM t1}]" +} { +INSERT INTO tab VALUES(1,2.5,'three'); +INSERT INTO tab VALUES(x'424c4f42',NULL,'Ἀμήν'); +} +do_test 1.81 { + set result "\n[db format -style insert -tablename t1 {SELECT * FROM t1}]" +} { +INSERT INTO t1 VALUES(1,2.5,'three'); +INSERT INTO t1 VALUES(x'424c4f42',NULL,'Ἀμήν'); +} +do_test 1.82 { + set result "\n[db format -style insert -tablename t1 -title auto \ + {SELECT * FROM t1}]" +} { +INSERT INTO t1(a,b,c) VALUES(1,2.5,'three'); +INSERT INTO t1(a,b,c) VALUES(x'424c4f42',NULL,'Ἀμήν'); +} +do_test 1.83 { + set result "\n[db format -style insert -tablename drop -title on \ + {SELECT a AS "a-b", b, c AS "123" FROM t1}]" +} { +INSERT INTO "drop"("a-b",b,"123") VALUES(1,2.5,'three'); +INSERT INTO "drop"("a-b",b,"123") VALUES(x'424c4f42',NULL,'Ἀμήν'); +} + +do_test 1.90 { + set result "\n[db format -style json {SELECT * FROM t1}]" +} { +[{"a":1,"b":2.5,"c":"three"}, +{"a":"\u0042\u004c\u004f\u0042","b":null,"c":"Ἀμήν"}] +} +do_test 1.91 { + set result "\n[db format -style jobject {SELECT * FROM t1}]" +} { +{"a":1,"b":2.5,"c":"three"} +{"a":"\u0042\u004c\u004f\u0042","b":null,"c":"Ἀμήν"} +} +do_test 1.92 { + set result "\n[db format -style jobject {SELECT *, unistr('abc\u000a123\u000d\u000axyz') AS xyz FROM t1}]" +} { +{"a":1,"b":2.5,"c":"three","xyz":"abc\n123\r\nxyz"} +{"a":"\u0042\u004c\u004f\u0042","b":null,"c":"Ἀμήν","xyz":"abc\n123\r\nxyz"} +} + +do_test 1.100 { + set result "\n[db format -style line {SELECT * FROM t1}]" +} { +a: 1 +b: 2.5 +c: three + +a: BLOB +b: +c: Ἀμήν +} +do_test 1.101 { + set result "\n[db format -style line -null (NULL) {SELECT * FROM t1}]" +} { +a: 1 +b: 2.5 +c: three + +a: BLOB +b: (NULL) +c: Ἀμήν +} +do_test 1.102 { + set result "\n[db format -style line -null (NULL) -columnsep { = } \ + -text sql {SELECT * FROM t1}]" +} { +a = 1 +b = 2.5 +c = 'three' + +a = x'424c4f42' +b = (NULL) +c = 'Ἀμήν' +} + +do_test 1.110 { + set result "\n[db format -style list {SELECT * FROM t1}]" +} { +1|2.5|three +BLOB||Ἀμήν +} +do_test 1.111 { + set result "\n[db format -style list -title on {SELECT * FROM t1}]" +} { +a|b|c +1|2.5|three +BLOB||Ἀμήν +} +do_test 1.112 { + set result "\n[db format -style list -title on -text sql -null NULL \ + -title plain {SELECT * FROM t1}]" +} { +a|b|c +1|2.5|'three' +x'424c4f42'|NULL|'Ἀμήν' +} +do_test 1.118 { + set rc [catch {db format -style list -title unk {SELECT * FROM t1}} res] + lappend rc $res +} {1 {bad -title "unk": must be off, on, auto, csv, html, json, plain, relaxed, sql, or tcl}} + + +do_test 1.120 { + set result "\n[db format -style markdown {SELECT * FROM t1}]" +} { +| a | b | c | +|------|-----|-------| +| 1 | 2.5 | three | +| BLOB | | Ἀμήν | +} +do_test 1.121 { + set result "\n[db format -style markdown -title off {SELECT * FROM t1}]" +} { +| 1 | 2.5 | three | +| BLOB | | Ἀμήν | +} + +do_test 1.130 { + set result "\n[db format -style quote {SELECT * FROM t1}]" +} { +1,2.5,'three' +x'424c4f42',NULL,'Ἀμήν' +} +do_test 1.131 { + set result "\n[db format -style quote -title on {SELECT * FROM t1}]" +} { +'a','b','c' +1,2.5,'three' +x'424c4f42',NULL,'Ἀμήν' +} + + +do_execsql_test 2.0 { + DELETE FROM t1; + INSERT INTO t1 VALUES(1,2,'The quick fox jumps over the lazy brown dog.'); +} +do_test 2.1 { + set result "\n[db format -widths {5 -5 19} -wordwrap on \ + {SELECT * FROM t1}]" +} { +╭───────┬───────┬─────────────────────╮ +│ a │ b │ c │ +╞═══════╪═══════╪═════════════════════╡ +│ 1 │ 2 │ The quick fox jumps │ +│ │ │ over the lazy brown │ +│ │ │ dog. │ +╰───────┴───────┴─────────────────────╯ +} +do_test 2.2 { + set result "\n[db format -widths {5 -5 19} -wordwrap off \ + {SELECT * FROM t1}]" +} { +╭───────┬───────┬─────────────────────╮ +│ a │ b │ c │ +╞═══════╪═══════╪═════════════════════╡ +│ 1 │ 2 │ The quick fox jumps │ +│ │ │ over the lazy brown │ +│ │ │ dog. │ +╰───────┴───────┴─────────────────────╯ +} +do_test 2.3 { + set result "\n[db format -widths {5 -5 18} -wordwrap on \ + {SELECT * FROM t1}]" +} { +╭───────┬───────┬────────────────────╮ +│ a │ b │ c │ +╞═══════╪═══════╪════════════════════╡ +│ 1 │ 2 │ The quick fox │ +│ │ │ jumps over the │ +│ │ │ lazy brown dog. │ +╰───────┴───────┴────────────────────╯ +} +do_test 2.4 { + set result "\n[db format -widths {5 -5 -18} -wordwrap on \ + {SELECT * FROM t1}]" +} { +╭───────┬───────┬────────────────────╮ +│ a │ b │ c │ +╞═══════╪═══════╪════════════════════╡ +│ 1 │ 2 │ The quick fox │ +│ │ │ jumps over the │ +│ │ │ lazy brown dog. │ +╰───────┴───────┴────────────────────╯ +} +do_test 2.5 { + set result "\n[db format -widths {5 -5 19} -wordwrap off \ + {SELECT * FROM t1}]" +} { +╭───────┬───────┬─────────────────────╮ +│ a │ b │ c │ +╞═══════╪═══════╪═════════════════════╡ +│ 1 │ 2 │ The quick fox jumps │ +│ │ │ over the lazy brown │ +│ │ │ dog. │ +╰───────┴───────┴─────────────────────╯ +} +do_test 2.6 { + set result "\n[db format -widths {5 -5 18} -wordwrap off \ + {SELECT * FROM t1}]" +} { +╭───────┬───────┬────────────────────╮ +│ a │ b │ c │ +╞═══════╪═══════╪════════════════════╡ +│ 1 │ 2 │ The quick fox jump │ +│ │ │ s over the lazy br │ +│ │ │ own dog. │ +╰───────┴───────┴────────────────────╯ +} +do_test 2.7 { + set result "\n[db format -widths {5 5 18} -wordwrap yes \ + -align {left center right} -titlealign right \ + {SELECT * FROM t1}]" +} { +╭───────┬───────┬────────────────────╮ +│ a │ b │ c │ +╞═══════╪═══════╪════════════════════╡ +│ 1 │ 2 │ The quick fox │ +│ │ │ jumps over the │ +│ │ │ lazy brown dog. │ +╰───────┴───────┴────────────────────╯ +} +do_test 2.8 { + set result "\n[db format -widths {5 8 11} -wordwrap yes \ + -align {auto auto center} -titlealign left \ + -defaultalign right \ + {SELECT * FROM t1}]" +} { +╭───────┬──────────┬─────────────╮ +│ a │ b │ c │ +╞═══════╪══════════╪═════════════╡ +│ 1 │ 2 │ The quick │ +│ │ │ fox jumps │ +│ │ │ over the │ +│ │ │ lazy brown │ +│ │ │ dog. │ +╰───────┴──────────┴─────────────╯ +} +do_test 2.9 { + catch {db format -align {auto xyz 123} {SELECT * FROM t1}} res + set res +} {bad column alignment (-align) "xyz": must be auto, bottom, c, center, e, left, middle, n, ne, nw, right, s, se, sw, top, or w} +do_test 2.10 { + catch {db format -defaultalign xyz {SELECT * FROM t1}} res + set res +} {bad default alignment (-defaultalign) "xyz": must be auto, bottom, c, center, e, left, middle, n, ne, nw, right, s, se, sw, top, or w} +do_test 2.11 { + catch {db format -titlealign xyz {SELECT * FROM t1}} res + set res +} {bad title alignment (-titlealign) "xyz": must be auto, bottom, c, center, e, left, middle, n, ne, nw, right, s, se, sw, top, or w} + + +do_execsql_test 2.30 { + UPDATE t1 SET c='Η γρήγορη αλεπού πηδάει πάνω από το τεμπέλικο καφέ σκυλί'; + SELECT hex(c) FROM t1; +} {CE9720CEB3CF81CEAECEB3CEBFCF81CEB720CEB1CEBBCEB5CF80CEBFCF8D20CF80CEB7CEB4CEACCEB5CEB920CF80CEACCEBDCF8920CEB1CF80CF8C20CF84CEBF20CF84CEB5CEBCCF80CEADCEBBCEB9CEBACEBF20CEBACEB1CF86CEAD20CF83CEBACF85CEBBCEAF} +do_test 2.31 { + set result "\n[db format -widths {5 -5 18} -wordwrap on \ + {SELECT * FROM t1}]" +} { +╭───────┬───────┬────────────────────╮ +│ a │ b │ c │ +╞═══════╪═══════╪════════════════════╡ +│ 1 │ 2 │ Η γρήγορη αλεπού │ +│ │ │ πηδάει πάνω από το │ +│ │ │ τεμπέλικο καφέ │ +│ │ │ σκυλί │ +╰───────┴───────┴────────────────────╯ +} +do_test 2.32 { + set result "\n[db format -widths {5 5 18} -align {left center center} -wordwrap on \ + {SELECT * FROM t1}]" +} { +╭───────┬───────┬────────────────────╮ +│ a │ b │ c │ +╞═══════╪═══════╪════════════════════╡ +│ 1 │ 2 │ Η γρήγορη αλεπού │ +│ │ │ πηδάει πάνω από το │ +│ │ │ τεμπέλικο καφέ │ +│ │ │ σκυλί │ +╰───────┴───────┴────────────────────╯ +} + + +do_execsql_test 3.0 { + DELETE FROM t1; + INSERT INTO t1 VALUES(1,2,unistr('abc\u001b[1;31m123\u001b[0mxyz')); +} +do_test 3.1 { + set result "\n[db format {SELECT * FROM t1}]" +} { +╭───┬───┬────────────────────────╮ +│ a │ b │ c │ +╞═══╪═══╪════════════════════════╡ +│ 1 │ 2 │ abc^[[1;31m123^[[0mxyz │ +╰───┴───┴────────────────────────╯ +} +do_test 3.2 { + set result "\n[db format -esc off {SELECT * FROM t1}]" + string map [list \033 X] $result +} { +╭───┬───┬───────────╮ +│ a │ b │ c │ +╞═══╪═══╪═══════════╡ +│ 1 │ 2 │ abcX[1;31m123X[0mxyz │ +╰───┴───┴───────────╯ +} +do_test 3.3 { + set result "\n[db format -esc symbol {SELECT * FROM t1}]" +} { +╭───┬───┬──────────────────────╮ +│ a │ b │ c │ +╞═══╪═══╪══════════════════════╡ +│ 1 │ 2 │ abc␛[1;31m123␛[0mxyz │ +╰───┴───┴──────────────────────╯ +} +do_test 3.4 { + set result "\n[db format -esc ascii {SELECT * FROM t1}]" +} { +╭───┬───┬────────────────────────╮ +│ a │ b │ c │ +╞═══╪═══╪════════════════════════╡ +│ 1 │ 2 │ abc^[[1;31m123^[[0mxyz │ +╰───┴───┴────────────────────────╯ +} +do_test 3.5 { + catch {db format -esc unk {SELECT * FROM t1}} res + set res +} {bad control character escape (-esc) "unk": must be ascii, auto, off, or symbol} + +do_execsql_test 4.0 { + DELETE FROM t1; + INSERT INTO t1 VALUES(json('{a:5,b:6}'), jsonb('{c:1,d:2}'), 99); +} +do_test 4.1 { + set result "\n[db format -text sql {SELECT * FROM t1}]" +} { +╭─────────────────┬───────────────────────┬────╮ +│ a │ b │ c │ +╞═════════════════╪═══════════════════════╪════╡ +│ '{"a":5,"b":6}' │ x'8c1763133117641332' │ 99 │ +╰─────────────────┴───────────────────────┴────╯ +} +do_test 4.2 { + set result "\n[db format -text sql -textjsonb on {SELECT * FROM t1}]" +} { +╭─────────────────┬────────────────────────┬────╮ +│ a │ b │ c │ +╞═════════════════╪════════════════════════╪════╡ +│ '{"a":5,"b":6}' │ jsonb('{"c":1,"d":2}') │ 99 │ +╰─────────────────┴────────────────────────┴────╯ +} +do_test 4.3 { + set result "\n[db format -text plain -textjsonb on -wrap 11 \ + {SELECT a AS json, b AS jsonb, c AS num FROM t1}]" +} { +╭─────────────┬─────────────┬─────╮ +│ json │ jsonb │ num │ +╞═════════════╪═════════════╪═════╡ +│ {"a":5,"b": │ {"c":1,"d": │ 99 │ +│ 6} │ 2} │ │ +╰─────────────┴─────────────┴─────╯ +} + +do_execsql_test 5.0 { + DROP TABLE t1; + CREATE TABLE t1(name, mtime, value); + INSERT INTO t1 VALUES + ('entry-one',1708791504,zeroblob(300)), + (unistr('one\u000atwo\u000athree'),1333206973,NULL), + ('sample-jsonb',1333101221,jsonb('{ + "alpha":53.11688723, + "beta":"qrfWidthPrint(p, p->pOut, -p->u.sLine.mxColWth);", + "zeta":[15,null,1333206973,"fd8ffe000104a46494600010101"]}')); +} +do_test 5.1 { + set sql {SELECT name, mtime, datetime(mtime,'unixepoch') AS time, + value FROM t1 ORDER BY mtime} + set result "\n[db format -style line -screenwidth 60 -blob sql \ + -text sql -wordwrap off -linelimit 77 \ + -columnsep { = } $sql]" +} { + name = 'sample-jsonb' +mtime = 1333101221 + time = '2012-03-30 09:53:41' +value = x'cc7c57616c706861b535332e31313638383732334762657461 + c73071726657696474685072696e7428702c20702d3e704f7574 + 2c202d702d3e752e734c696e652e6d78436f6c577468293b477a + 657461cb2c23313500a331333333323036393733c71b66643866 + 6665303030313034613436343934363030303130313031' + + name = unistr('one\u000atwo\u000athree') +mtime = 1333206973 + time = '2012-03-31 15:16:13' +value = + + name = 'entry-one' +mtime = 1708791504 + time = '2024-02-24 16:18:24' +value = x'00000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000' +} +do_test 5.2a { + set sql {SELECT name, mtime, datetime(mtime,'unixepoch') AS time, + value FROM t1 ORDER BY mtime} + set result "\n[db format -style line -screenwidth 60 -blob sql \ + -text plain -esc off -textjsonb yes -columnsep { = }\ + -wordwrap yes -linelimit 3 $sql]" +} { + name = sample-jsonb +mtime = 1333101221 + time = 2012-03-30 09:53:41 +value = {"alpha":53.11688723,"beta":"qrfWidthPrint(p, + p->pOut, -p->u.sLine.mxColWth);","zeta":[15,null, + 1333206973,"fd8ffe000104a46494600010101"]} + + name = one + two + three +mtime = 1333206973 + time = 2012-03-31 15:16:13 +value = + + name = entry-one +mtime = 1708791504 + time = 2024-02-24 16:18:24 +value = x'00000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + ... +} +set sqlnolabel "SELECT name, mtime, datetime(mtime,'unixepoch'),\ + value FROM t1 ORDER BY mtime" +do_test 5.2b { + set result "\n[db format -style line -screenwidth 60 -blob sql \ + -text plain -esc off -textjsonb no -titlelimit 12 \ + -wordwrap yes -linelimit 3 $sqlnolabel ]" +} { + name: sample-jsonb + mtime: 1333101221 +datetime(...: 2012-03-30 09:53:41 + value: x'cc7c57616c706861b535332e31313638383732334762 + 657461c73071726657696474685072696e7428702c2070 + 2d3e704f75742c202d702d3e752e734c696e652e6d7843 + ... + + name: one + two + three + mtime: 1333206973 +datetime(...: 2012-03-31 15:16:13 + value: + + name: entry-one + mtime: 1708791504 +datetime(...: 2024-02-24 16:18:24 + value: x'00000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000 + ... +} +set sql "SELECT name, mtime, datetime(mtime,'unixepoch') AS time,\ + value FROM t1 ORDER BY mtime" +do_test 5.3a { + set result "\n[db format -style box -widths {0 10 10 14}\ + -align {left right right center} \ + -blob sql \ + -text plain -esc off -textjsonb no \ + -wordwrap yes -linelimit 2 $sql]" +} { +╭──────────────┬────────────┬────────────┬────────────────╮ +│ name │ mtime │ time │ value │ +╞══════════════╪════════════╪════════════╪════════════════╡ +│ sample-jsonb │ 1333101221 │ 2012-03-30 │ x'cc7c57616c70 │ +│ │ │ 09:53:41 │ 6861b535332e31 │ +│ │ │ │ ... │ +├──────────────┼────────────┼────────────┼────────────────┤ +│ one │ 1333206973 │ 2012-03-31 │ │ +│ two │ │ 15:16:13 │ │ +│ ... │ │ │ │ +├──────────────┼────────────┼────────────┼────────────────┤ +│ entry-one │ 1708791504 │ 2024-02-24 │ x'000000000000 │ +│ │ │ 16:18:24 │ 00000000000000 │ +│ │ │ │ ... │ +╰──────────────┴────────────┴────────────┴────────────────╯ +} +do_test 5.3b { + set result "\n[db format -style box -widths {0 10 0 14} \ + -align {left right right center} \ + -blob sql -titlelimit 12 \ + -text plain -esc off -textjsonb no \ + -wordwrap yes -linelimit 2 $sqlnolabel ]" +} { +╭──────────────┬────────────┬─────────────────────┬────────────────╮ +│ name │ mtime │ datetime(... │ value │ +╞══════════════╪════════════╪═════════════════════╪════════════════╡ +│ sample-jsonb │ 1333101221 │ 2012-03-30 09:53:41 │ x'cc7c57616c70 │ +│ │ │ │ 6861b535332e31 │ +│ │ │ │ ... │ +├──────────────┼────────────┼─────────────────────┼────────────────┤ +│ one │ 1333206973 │ 2012-03-31 15:16:13 │ │ +│ two │ │ │ │ +│ ... │ │ │ │ +├──────────────┼────────────┼─────────────────────┼────────────────┤ +│ entry-one │ 1708791504 │ 2024-02-24 16:18:24 │ x'000000000000 │ +│ │ │ │ 00000000000000 │ +│ │ │ │ ... │ +╰──────────────┴────────────┴─────────────────────┴────────────────╯ +} +do_test 5.3c { + set result "\n[db format -style table -widths {0 10 10 14}\ + -align {center right right right} \ + -blob sql \ + -text plain -esc off -textjsonb no \ + -wordwrap yes -linelimit 2 $sql]" +} { ++--------------+------------+------------+----------------+ +| name | mtime | time | value | ++--------------+------------+------------+----------------+ +| sample-jsonb | 1333101221 | 2012-03-30 | x'cc7c57616c70 | +| | | 09:53:41 | 6861b535332e31 | +| | | | ... | ++--------------+------------+------------+----------------+ +| one | 1333206973 | 2012-03-31 | | +| two | | 15:16:13 | | +| ... | | | | ++--------------+------------+------------+----------------+ +| entry-one | 1708791504 | 2024-02-24 | x'000000000000 | +| | | 16:18:24 | 00000000000000 | +| | | | ... | ++--------------+------------+------------+----------------+ +} +do_test 5.3c { + set result "\n[db format -style column -widths {0 10 10 14}\ + -align {center right right right} \ + -blob sql \ + -text plain -esc off -textjsonb no \ + -wordwrap yes -linelimit 2 $sql]" +} { + name mtime time value +------------ ---------- ---------- -------------- +sample-jsonb 1333101221 2012-03-30 x'cc7c57616c70 + 09:53:41 6861b535332e31 + ... + + one 1333206973 2012-03-31 + two 15:16:13 + ... + + entry-one 1708791504 2024-02-24 x'000000000000 + 16:18:24 00000000000000 + ... +} +do_test 5.4 { + db eval { + CREATE TABLE t2(a,b,c,d,e); + WITH v(x) AS (SELECT 'abcdefghijklmnopqrstuvwxyz') + INSERT INTO t2 SELECT x,x,x,x,x FROM v; + } + set sql {SELECT char(0x61,0xa,0x62,0xa,0x63,0xa,0x64) a, + mtime b, mtime c, mtime d, mtime e FROM t1} + set result "\n[db format -style box -widths {1 2 3 4 5}\ + -linelimit 3 -wordwrap off {SELECT *, 'x' AS x FROM t2}]" +} { +╭────┬────┬─────┬──────┬───────┬───╮ +│ a │ b │ c │ d │ e │ x │ +╞════╪════╪═════╪══════╪═══════╪═══╡ +│ ab │ ab │ abc │ abcd │ abcde │ x │ +│ cd │ cd │ def │ efgh │ fghij │ │ +│ ef │ ef │ ghi │ ijkl │ klmno │ │ +│ .. │ .. │ ... │ ... │ ... │ │ +╰────┴────┴─────┴──────┴───────┴───╯ +} + +do_execsql_test 6.0 { + DELETE FROM t2; + INSERT INTO t2 VALUES + (1, 2.5, 'three', x'342028666f757229', null); +} +do_test 6.1a { + set result "\n[db format -style list -null NULL \ + -text tcl -columnsep , \ + {SELECT * FROM t2}]" +} { +1,2.5,"three","\064\040\050\146\157\165\162\051",NULL +} + +do_execsql_test 7.0 { + CREATE TABLE t7(a,b); + INSERT INTO t7 VALUES('abcdefghijklmnop', + 'abcぁdefかghiのjklはmnop'); +} +do_test 7.1 { + set result "\n[db format -style list -charlimit 13 \ + {SELECT * FROM t7}]" +} { +abcdefghijklm...|abcぁdefかghi... +} +do_test 7.2 { + set result "\n[db format -style list -charlimit 14 \ + {SELECT * FROM t7}]" +} { +abcdefghijklmn...|abcぁdefかghi... +} +do_test 7.3 { + set result "\n[db format -style list -charlimit 15 \ + {SELECT * FROM t7}]" +} { +abcdefghijklmno...|abcぁdefかghiの... +} +do_test 7.4 { + set result "\n[db format -style list -charlimit 16 \ + {SELECT * FROM t7}]" +} { +abcdefghijklmnop|abcぁdefかghiのj... +} + +do_test 8.0 { + set result "\n[db format -style table { + WITH c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<10) + SELECT 'aaa' AS 'a', format('%.*c',n,'b')||char(9)||'xx' AS x FROM c}]" +} { ++-----+--------------------+ +| a | x | ++-----+--------------------+ +| aaa | b xx | +| aaa | bb xx | +| aaa | bbb xx | +| aaa | bbbb xx | +| aaa | bbbbb xx | +| aaa | bbbbbb xx | +| aaa | bbbbbbb xx | +| aaa | bbbbbbbb xx | +| aaa | bbbbbbbbb xx | +| aaa | bbbbbbbbbb xx | ++-----+--------------------+ +} +do_test 8.1 { + set result "\n[db format -style table { + WITH c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<10) + SELECT 'aaaa' AS 'a', format('%.*c',n,'b')||char(9)||'xx' AS x FROM c}]" +} { ++------+--------------------+ +| a | x | ++------+--------------------+ +| aaaa | b xx | +| aaaa | bb xx | +| aaaa | bbb xx | +| aaaa | bbbb xx | +| aaaa | bbbbb xx | +| aaaa | bbbbbb xx | +| aaaa | bbbbbbb xx | +| aaaa | bbbbbbbb xx | +| aaaa | bbbbbbbbb xx | +| aaaa | bbbbbbbbbb xx | ++------+--------------------+ +} +do_test 8.3 { + set result "\n[db format -style table -esc off { + WITH c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<15) + SELECT 'aaa' AS 'a',n, 'xx'||char(n)||'yy' AS xy FROM c + WHERE n NOT IN (8,10,13,14)}]" +} { ++-----+----+------------+ +| a | n | xy | ++-----+----+------------+ +| aaa | 1 | xx␁yy | +| aaa | 2 | xx␂yy | +| aaa | 3 | xx␃yy | +| aaa | 4 | xx␄yy | +| aaa | 5 | xx␅yy | +| aaa | 6 | xx␆yy | +| aaa | 7 | xx␇yy | +| aaa | 9 | xx yy | +| aaa | 11 | xx␋yy | +| aaa | 12 | xx␌yy | +| aaa | 15 | xx␏yy | ++-----+----+------------+ +} +do_test 8.4 { + set result "\n[db format -style table -esc off { + WITH c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<15) + SELECT 'aaa' AS 'a',n, 'xx'||char(n)||'yy'||char(9)||'zz' AS xyz FROM c + WHERE n NOT IN (8,10,13,14)}]" +} { ++-----+----+--------------------+ +| a | n | xyz | ++-----+----+--------------------+ +| aaa | 1 | xx␁yy zz | +| aaa | 2 | xx␂yy zz | +| aaa | 3 | xx␃yy zz | +| aaa | 4 | xx␄yy zz | +| aaa | 5 | xx␅yy zz | +| aaa | 6 | xx␆yy zz | +| aaa | 7 | xx␇yy zz | +| aaa | 9 | xx yy zz | +| aaa | 11 | xx␋yy zz | +| aaa | 12 | xx␌yy zz | +| aaa | 15 | xx␏yy zz | ++-----+----+--------------------+ +} + +do_test 9.1 { + db eval { + CREATE TABLE t9(x); + INSERT INTO t9 VALUES + (x'4331323334'), + (x'c30431323334'), + (x'd3000431323334'), + (x'e30000000431323334'), + (x'f3000000000000000431323334'); + } + db format -style list -text plain -rowsep , -textjsonb on \ + {SELECT * FROM t9} +} {1234,1234,1234,1234,1234,} +do_test 9.2 { + db format -style list -text sql -rowsep , -textjsonb on \ + {SELECT * FROM t9} +} {jsonb('1234'),jsonb('1234'),jsonb('1234'),jsonb('1234'),jsonb('1234'),} +do_test 9.3 { + db format -style json {SELECT * FROM t9 WHERE rowid<0} +} {} +do_test 9.4 { + db format -style jobject {SELECT * FROM t9 WHERE rowid<0} +} {} + +do_test 10.1 { + db eval { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(x); + INSERT INTO t1(x) VALUES + ('alice'), + ('bob'), + ('cinderella-cinderella'), + ('daniel'), + ('emma'), + ('fred'), + ('gertrude'), + ('harold'), + ('ingrid'), + ('jake'), + ('lisa'), + ('mike'), + ('nina'), + ('octavian'), + ('paula'), + ('quintus'), + ('rita'), + ('sam'), + ('tammy'), + ('ulysses'), + ('violet'), + ('william'), + ('xanthippe'), + ('yates'), + ('zoe'); + } + set result "\n[db format -style column -title off -screenwidth 41 -splitcolumn on \ + {SELECT x FROM t1}]" +} { +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +} +do_test 10.2 { + set result "\n[db format -style column -title off -screenwidth 42 -splitcolumn on \ + {SELECT x FROM t1}]" +} { +alice jake tammy +bob lisa ulysses +cinderella-cinderella mike violet +daniel nina william +emma octavian xanthippe +fred paula yates +gertrude quintus zoe +harold rita +ingrid sam +} +do_test 10.3 { + set result "\n[db format -style column -title off -screenwidth 51 -splitcolumn on \ + {SELECT x FROM t1}]" +} { +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +} +do_test 10.4 { + set result "\n[db format -style column -title off -screenwidth 61 -splitcolumn on \ + {SELECT x FROM t1}]" +} { +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +} +do_test 10.5 { + set result "\n[db format -style column -title off -screenwidth 74 -splitcolumn on \ + {SELECT x FROM t1}]" +} { +alice emma ingrid nina rita violet zoe +bob fred jake octavian sam william +cinderella-cinderella gertrude lisa paula tammy xanthippe +daniel harold mike quintus ulysses yates +} + +do_test 11.1 { + set result "\n[db format -style table -blob size {SELECT randomblob(1234)}]" +} { ++------------------+ +| randomblob(1234) | ++------------------+ +| (1234-byte blob) | ++------------------+ +} + +do_test 12.1 { + set result "\n[db format -style box -text html \ + {SELECT 'abc','','xyz'}]" +} { +╭───────┬────┬───────╮ +│ 'abc' │ '' │ 'xyz' │ +╞═══════╪════╪═══════╡ +│ abc │ │ xyz │ +╰───────┴────┴───────╯ +} + +# Tests for "relaxed" quoting +# +do_test 13.2 { + db eval { + CREATE TABLE t13(a,b); + INSERT INTO t13(a,b) VALUES + (1,'NULL'), + (0,'-NULL-'), + (0,''), + (1,'''abcde'), + (1,'abcde'''), + (0,'abcde'), + (1,' abcde'), + (1,'abcde '), + (1,'+0'), + (1,'-0'), + (1,'012345'), + (0,'012xyz345'), + (1,'0123.45'), + (0,'12.34.56'), + (0,'12.3e'), + (1,'12.3e+123'), + (1,'12.3e-34'), + (1,'12.3E56'), + (1,'12E56'), + (0,'12.5E5.6'), + (0,'12.5e+'), + (0,'12.5e-'), + (1,'+Inf'),(1,'-Inf'),(1,'Inf'); + } + set result \n[db format -style box -text relaxed -null NULL \ + -align {center left} \ + {SELECT if(a,'yes','') AS 'quoted?', b AS string + FROM t13 ORDER BY rowid}] +} { +╭─────────┬─────────────╮ +│ quoted? │ string │ +╞═════════╪═════════════╡ +│ yes │ 'NULL' │ +│ │ -NULL- │ +│ │ │ +│ yes │ '''abcde' │ +│ yes │ 'abcde''' │ +│ │ abcde │ +│ yes │ ' abcde' │ +│ yes │ 'abcde ' │ +│ yes │ '+0' │ +│ yes │ '-0' │ +│ yes │ '012345' │ +│ │ 012xyz345 │ +│ yes │ '0123.45' │ +│ │ 12.34.56 │ +│ │ 12.3e │ +│ yes │ '12.3e+123' │ +│ yes │ '12.3e-34' │ +│ yes │ '12.3E56' │ +│ yes │ '12E56' │ +│ │ 12.5E5.6 │ +│ │ 12.5e+ │ +│ │ 12.5e- │ +│ yes │ '+Inf' │ +│ yes │ '-Inf' │ +│ yes │ 'Inf' │ +╰─────────┴─────────────╯ +} + +db close + +finish_test diff --git a/test/qrf02.test b/test/qrf02.test new file mode 100644 index 000000000..07e1568f7 --- /dev/null +++ b/test/qrf02.test @@ -0,0 +1,47 @@ +# 2025-11-05 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the Query Result Formatter (QRF) +# +# These tests are for EXPLAIN and EXPLAIN QUERY PLAN formatting, the +# output of which can change when enhancments are made to the query +# planner. So expect to have to modify the expected results of these +# test cases in the future. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix qrf02 + +do_execsql_test 1.0 { + CREATE TABLE t1(a); + INSERT INTO t1 VALUES(1); +} + +set result [db format {EXPLAIN SELECT * FROM t1}] +do_test 1.10 { + set result +} {/*addr opcode p1 p2 p3 p4 p5 comment +---- ------------- ---- ---- ---- ------------- -- ------------- +0 Init */} +regsub -all {\d+} $result {N} result2 +do_test 1.11 { + set result2 +} "/.*\nN Rewind .*\nN Column .*/" + +do_test 1.20 { + set result "\n[db format {EXPLAIN QUERY PLAN SELECT * FROM t1}]" +} { +QUERY PLAN +`--SCAN t1 +} + +finish_test diff --git a/test/qrf03.test b/test/qrf03.test new file mode 100644 index 000000000..c0457df7f --- /dev/null +++ b/test/qrf03.test @@ -0,0 +1,176 @@ +# 2025-11-15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the Query Result Formatter (QRF) +# +# Format narrowing due to nScreenWidth +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix qrf03 + +do_execsql_test 1.0 { + CREATE TABLE mlink( + mid INTEGER, + fid INTEGER, + pmid INTEGER, + pid INTEGER, + fnid INTEGER REFERENCES filename, + pfnid INTEGER, + mperm INTEGER, + isaux BOOLEAN DEFAULT 0 + ); + INSERT INTO mlink VALUES(28775,28774,28773,28706,1,0,0,0); + INSERT INTO mlink VALUES(28773,28706,28770,28685,1,0,0,0); + INSERT INTO mlink VALUES(28770,28736,28769,28695,2,0,0,0); + INSERT INTO mlink VALUES(28770,28697,28769,28698,3,0,0,0); + INSERT INTO mlink VALUES(28767,28768,28759,28746,4,0,0,0); + CREATE TABLE event( + type TEXT, + mtime DATETIME, + objid INTEGER PRIMARY KEY, + tagid INTEGER, + uid INTEGER REFERENCES user, + bgcolor TEXT, + euser TEXT, + user TEXT, + ecomment TEXT, + comment TEXT, + brief TEXT, + omtime DATETIME + ); + INSERT INTO event VALUES('ci',2460994.978048461023,126223,NULL,NULL,NULL,NULL,'drh',NULL,unistr('Data structure improvements on columnar layout. Prep work for getting\u000acolumnar layouts to respond to nScreenWidth.'),NULL,2460994.978048461023); + INSERT INTO event VALUES('ci',2460994.836955601816,126218,NULL,NULL,NULL,NULL,'stephan',NULL,'API doc typo fix.',NULL,2460994.836955601816); + INSERT INTO event VALUES('ci',2460994.88823369192,126212,NULL,NULL,NULL,NULL,'stephan',NULL,'Move sqlite3-api-cleanup.js into post-js-footer.js to remove the final direct Emscripten dependency from the intermediary build product sqlite3-api.js (the whole library, waiting to be bootstrapped). This is partly in response to [forum:4b7d45433731d2e0|forum post 4b7d45433731d2e0], which demonstrates a potential use case for a standalone sqlite3-api.js. This is a build/doc change, not a functional one.',NULL,2460994.88823369192); + INSERT INTO event VALUES('ci',2460994.516081551089,126211,NULL,NULL,NULL,NULL,'drh',NULL,unistr('Improve columnar layout in QRF so that it correctly deals with control\u000acharacters, and especially tabs.'),NULL,2460994.516081551089); + INSERT INTO event VALUES('ci',2460994.409343171865,126208,NULL,NULL,NULL,NULL,'drh',NULL,'Make use of the new sqlite3_str_free() interface in the CLI.',NULL,2460994.409343171865); +} + +do_test 1.10 { + set x "\n[db format -style box -screenwidth 68 \ + {SELECT * FROM mlink ORDER BY rowid}]" + +} { +╭───────┬───────┬───────┬───────┬──────┬───────┬───────┬───────╮ +│ mid │ fid │ pmid │ pid │ fnid │ pfnid │ mperm │ isaux │ +╞═══════╪═══════╪═══════╪═══════╪══════╪═══════╪═══════╪═══════╡ +│ 28775 │ 28774 │ 28773 │ 28706 │ 1 │ 0 │ 0 │ 0 │ +│ 28773 │ 28706 │ 28770 │ 28685 │ 1 │ 0 │ 0 │ 0 │ +│ 28770 │ 28736 │ 28769 │ 28695 │ 2 │ 0 │ 0 │ 0 │ +│ 28770 │ 28697 │ 28769 │ 28698 │ 3 │ 0 │ 0 │ 0 │ +│ 28767 │ 28768 │ 28759 │ 28746 │ 4 │ 0 │ 0 │ 0 │ +╰───────┴───────┴───────┴───────┴──────┴───────┴───────┴───────╯ +} +do_test 1.11 { + set x "\n[db format -style box -screenwidth 52 \ + {SELECT * FROM mlink ORDER BY rowid}]" + +} { +╭─────┬─────┬─────┬─────┬────┬─────┬─────┬─────╮ +│ mid │ fid │pmid │ pid │fnid│pfnid│mperm│isaux│ +╞═════╪═════╪═════╪═════╪════╪═════╪═════╪═════╡ +│28775│28774│28773│28706│ 1│ 0│ 0│ 0│ +│28773│28706│28770│28685│ 1│ 0│ 0│ 0│ +│28770│28736│28769│28695│ 2│ 0│ 0│ 0│ +│28770│28697│28769│28698│ 3│ 0│ 0│ 0│ +│28767│28768│28759│28746│ 4│ 0│ 0│ 0│ +╰─────┴─────┴─────┴─────┴────┴─────┴─────┴─────╯ +} + +do_test 1.20 { + set x "\n[db format -style table -screenwidth 68 \ + {SELECT * FROM mlink ORDER BY rowid}]" + +} { ++-------+-------+-------+-------+------+-------+-------+-------+ +| mid | fid | pmid | pid | fnid | pfnid | mperm | isaux | ++-------+-------+-------+-------+------+-------+-------+-------+ +| 28775 | 28774 | 28773 | 28706 | 1 | 0 | 0 | 0 | +| 28773 | 28706 | 28770 | 28685 | 1 | 0 | 0 | 0 | +| 28770 | 28736 | 28769 | 28695 | 2 | 0 | 0 | 0 | +| 28770 | 28697 | 28769 | 28698 | 3 | 0 | 0 | 0 | +| 28767 | 28768 | 28759 | 28746 | 4 | 0 | 0 | 0 | ++-------+-------+-------+-------+------+-------+-------+-------+ +} +do_test 1.21 { + set x "\n[db format -style table -screenwidth 52 \ + {SELECT * FROM mlink ORDER BY rowid}]" + +} { ++-----+-----+-----+-----+----+-----+-----+-----+ +| mid | fid |pmid | pid |fnid|pfnid|mperm|isaux| ++-----+-----+-----+-----+----+-----+-----+-----+ +|28775|28774|28773|28706| 1| 0| 0| 0| +|28773|28706|28770|28685| 1| 0| 0| 0| +|28770|28736|28769|28695| 2| 0| 0| 0| +|28770|28697|28769|28698| 3| 0| 0| 0| +|28767|28768|28759|28746| 4| 0| 0| 0| ++-----+-----+-----+-----+----+-----+-----+-----+ +} + +do_test 1.30 { + set x "\n[db format -style markdown -screenwidth 68 \ + {SELECT * FROM mlink ORDER BY rowid}]" + +} { +| mid | fid | pmid | pid | fnid | pfnid | mperm | isaux | +|-------|-------|-------|-------|------|-------|-------|-------| +| 28775 | 28774 | 28773 | 28706 | 1 | 0 | 0 | 0 | +| 28773 | 28706 | 28770 | 28685 | 1 | 0 | 0 | 0 | +| 28770 | 28736 | 28769 | 28695 | 2 | 0 | 0 | 0 | +| 28770 | 28697 | 28769 | 28698 | 3 | 0 | 0 | 0 | +| 28767 | 28768 | 28759 | 28746 | 4 | 0 | 0 | 0 | +} +do_test 1.31 { + set x "\n[db format -style markdown -screenwidth 52 \ + {SELECT * FROM mlink ORDER BY rowid}]" + +} { +| mid | fid |pmid | pid |fnid|pfnid|mperm|isaux| +|-----|-----|-----|-----|----|-----|-----|-----| +|28775|28774|28773|28706| 1| 0| 0| 0| +|28773|28706|28770|28685| 1| 0| 0| 0| +|28770|28736|28769|28695| 2| 0| 0| 0| +|28770|28697|28769|28698| 3| 0| 0| 0| +|28767|28768|28759|28746| 4| 0| 0| 0| +} + +do_test 1.40 { + set x "\n[db format -style column -screenwidth 68 \ + {SELECT * FROM mlink ORDER BY rowid}]" + +} { + mid fid pmid pid fnid pfnid mperm isaux +----- ----- ----- ----- ---- ----- ----- ----- +28775 28774 28773 28706 1 0 0 0 +28773 28706 28770 28685 1 0 0 0 +28770 28736 28769 28695 2 0 0 0 +28770 28697 28769 28698 3 0 0 0 +28767 28768 28759 28746 4 0 0 0 +} +do_test 1.41 { + set x "\n[db format -style column -screenwidth 52 \ + {SELECT * FROM mlink ORDER BY rowid}]" + +} { + mid fid pmid pid fnid pfnid mperm isaux +----- ----- ----- ----- ---- ----- ----- ----- +28775 28774 28773 28706 1 0 0 0 +28773 28706 28770 28685 1 0 0 0 +28770 28736 28769 28695 2 0 0 0 +28770 28697 28769 28698 3 0 0 0 +28767 28768 28759 28746 4 0 0 0 +} + + + +finish_test diff --git a/test/qrf04.test b/test/qrf04.test new file mode 100644 index 000000000..0b231d921 --- /dev/null +++ b/test/qrf04.test @@ -0,0 +1,750 @@ +# 2025-11-23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the Query Result Formatter (QRF), and especially +# the bSplitColumn feature. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix qrf01 + +# The expected output from test 1.1. The "do_test" procedure normally +# ignores differences in whitespace, but whitespace is important for +# this test, so we have to do the comparison ourselves. +# +set expected { +<---- 22 ----> +alice +bob +cinderella-cinderella +daniel +emma +fred +gertrude +harold +ingrid +jake +lisa +mike +nina +octavian +paula +quintus +rita +sam +tammy +ulysses +violet +william +xanthippe +yates +zoe +<---- 23 ----> +alice +bob +cinderella-cinderella +daniel +emma +fred +gertrude +harold +ingrid +jake +lisa +mike +nina +octavian +paula +quintus +rita +sam +tammy +ulysses +violet +william +xanthippe +yates +zoe +<---- 24 ----> +alice +bob +cinderella-cinderella +daniel +emma +fred +gertrude +harold +ingrid +jake +lisa +mike +nina +octavian +paula +quintus +rita +sam +tammy +ulysses +violet +william +xanthippe +yates +zoe +<---- 25 ----> +alice +bob +cinderella-cinderella +daniel +emma +fred +gertrude +harold +ingrid +jake +lisa +mike +nina +octavian +paula +quintus +rita +sam +tammy +ulysses +violet +william +xanthippe +yates +zoe +<---- 26 ----> +alice +bob +cinderella-cinderella +daniel +emma +fred +gertrude +harold +ingrid +jake +lisa +mike +nina +octavian +paula +quintus +rita +sam +tammy +ulysses +violet +william +xanthippe +yates +zoe +<---- 27 ----> +alice +bob +cinderella-cinderella +daniel +emma +fred +gertrude +harold +ingrid +jake +lisa +mike +nina +octavian +paula +quintus +rita +sam +tammy +ulysses +violet +william +xanthippe +yates +zoe +<---- 28 ----> +alice +bob +cinderella-cinderella +daniel +emma +fred +gertrude +harold +ingrid +jake +lisa +mike +nina +octavian +paula +quintus +rita +sam +tammy +ulysses +violet +william +xanthippe +yates +zoe +<---- 29 ----> +alice +bob +cinderella-cinderella +daniel +emma +fred +gertrude +harold +ingrid +jake +lisa +mike +nina +octavian +paula +quintus +rita +sam +tammy +ulysses +violet +william +xanthippe +yates +zoe +<---- 30 ----> +alice +bob +cinderella-cinderella +daniel +emma +fred +gertrude +harold +ingrid +jake +lisa +mike +nina +octavian +paula +quintus +rita +sam +tammy +ulysses +violet +william +xanthippe +yates +zoe +<---- 31 ----> +alice +bob +cinderella-cinderella +daniel +emma +fred +gertrude +harold +ingrid +jake +lisa +mike +nina +octavian +paula +quintus +rita +sam +tammy +ulysses +violet +william +xanthippe +yates +zoe +<---- 32 ----> +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +<---- 33 ----> +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +<---- 34 ----> +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +<---- 35 ----> +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +<---- 36 ----> +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +<---- 37 ----> +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +<---- 38 ----> +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +<---- 39 ----> +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +<---- 40 ----> +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +<---- 41 ----> +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +<---- 42 ----> +alice jake tammy +bob lisa ulysses +cinderella-cinderella mike violet +daniel nina william +emma octavian xanthippe +fred paula yates +gertrude quintus zoe +harold rita +ingrid sam +<---- 43 ----> +alice jake tammy +bob lisa ulysses +cinderella-cinderella mike violet +daniel nina william +emma octavian xanthippe +fred paula yates +gertrude quintus zoe +harold rita +ingrid sam +<---- 44 ----> +alice jake tammy +bob lisa ulysses +cinderella-cinderella mike violet +daniel nina william +emma octavian xanthippe +fred paula yates +gertrude quintus zoe +harold rita +ingrid sam +<---- 45 ----> +alice jake tammy +bob lisa ulysses +cinderella-cinderella mike violet +daniel nina william +emma octavian xanthippe +fred paula yates +gertrude quintus zoe +harold rita +ingrid sam +<---- 46 ----> +alice jake tammy +bob lisa ulysses +cinderella-cinderella mike violet +daniel nina william +emma octavian xanthippe +fred paula yates +gertrude quintus zoe +harold rita +ingrid sam +<---- 47 ----> +alice jake tammy +bob lisa ulysses +cinderella-cinderella mike violet +daniel nina william +emma octavian xanthippe +fred paula yates +gertrude quintus zoe +harold rita +ingrid sam +<---- 48 ----> +alice jake tammy +bob lisa ulysses +cinderella-cinderella mike violet +daniel nina william +emma octavian xanthippe +fred paula yates +gertrude quintus zoe +harold rita +ingrid sam +<---- 49 ----> +alice jake tammy +bob lisa ulysses +cinderella-cinderella mike violet +daniel nina william +emma octavian xanthippe +fred paula yates +gertrude quintus zoe +harold rita +ingrid sam +<---- 50 ----> +alice jake tammy +bob lisa ulysses +cinderella-cinderella mike violet +daniel nina william +emma octavian xanthippe +fred paula yates +gertrude quintus zoe +harold rita +ingrid sam +<---- 51 ----> +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +<---- 52 ----> +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +<---- 53 ----> +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +<---- 54 ----> +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +<---- 55 ----> +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +<---- 56 ----> +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +<---- 57 ----> +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +<---- 58 ----> +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +<---- 59 ----> +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +<---- 60 ----> +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +<---- 61 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 62 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 63 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 64 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 65 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 66 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 67 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 68 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 69 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 70 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 71 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 72 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 73 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 74 ----> +alice emma ingrid nina rita violet zoe +bob fred jake octavian sam william +cinderella-cinderella gertrude lisa paula tammy xanthippe +daniel harold mike quintus ulysses yates +<---- 75 ----> +alice emma ingrid nina rita violet zoe +bob fred jake octavian sam william +cinderella-cinderella gertrude lisa paula tammy xanthippe +daniel harold mike quintus ulysses yates +<---- 76 ----> +alice emma ingrid nina rita violet zoe +bob fred jake octavian sam william +cinderella-cinderella gertrude lisa paula tammy xanthippe +daniel harold mike quintus ulysses yates +<---- 77 ----> +alice emma ingrid nina rita violet zoe +bob fred jake octavian sam william +cinderella-cinderella gertrude lisa paula tammy xanthippe +daniel harold mike quintus ulysses yates +<---- 78 ----> +alice emma ingrid nina rita violet zoe +bob fred jake octavian sam william +cinderella-cinderella gertrude lisa paula tammy xanthippe +daniel harold mike quintus ulysses yates +<---- 79 ----> +alice emma ingrid nina rita violet zoe +bob fred jake octavian sam william +cinderella-cinderella gertrude lisa paula tammy xanthippe +daniel harold mike quintus ulysses yates +<---- 80 ----> +alice emma ingrid nina rita violet zoe +bob fred jake octavian sam william +cinderella-cinderella gertrude lisa paula tammy xanthippe +daniel harold mike quintus ulysses yates +} + +do_test 1.0 { + db eval { + CREATE TABLE t1(x); + INSERT INTO t1(x) VALUES + ('alice'), + ('bob'), + ('cinderella-cinderella'), + ('daniel'), + ('emma'), + ('fred'), + ('gertrude'), + ('harold'), + ('ingrid'), + ('jake'), + ('lisa'), + ('mike'), + ('nina'), + ('octavian'), + ('paula'), + ('quintus'), + ('rita'), + ('sam'), + ('tammy'), + ('ulysses'), + ('violet'), + ('william'), + ('xanthippe'), + ('yates'), + ('zoe'); + } + set res \n + for {set i 22} {$i<=80} {incr i} { + set sp [expr {$i-13}] + append res [format "<----%*s%3d%*s---->\n" \ + [expr {$sp/2}] {} $i [expr {$sp-$sp/2}] {}] + append res [db format -style column -title off \ + -screenwidth $i -splitcolumn on \ + {SELECT x FROM t1 ORDER BY x ASC}] + } + expr {$res eq $::expected} +} {1} diff --git a/test/qrf05.test b/test/qrf05.test new file mode 100644 index 000000000..0d5a4d7f9 --- /dev/null +++ b/test/qrf05.test @@ -0,0 +1,37 @@ +# 2025-12-02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the Query Result Formatter (QRF) +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix qrf05 + +do_execsql_test 1.0 { + CREATE TABLE t1(a INT NOT NULL); +} +do_test 1.1 { + set rc [catch {db format -style list \ + {INSERT INTO t1 VALUES(123) RETURNING *}} msg] + list $rc [string trim $msg] +} {0 123} +do_test 1.2 { + set rc [catch {db format -style list \ + {INSERT INTO t1 VALUES(NULL) RETURNING *}} msg] + list $rc [string trim $msg] +} {1 {NOT NULL constraint failed: t1.a}} +do_test 1.3 { + set rc [catch {db format -version 99 {SELECT * FROM t1}} msg] + list $rc [string trim $msg] +} {1 {unusable sqlite3_qrf_spec.iVersion (99)}} + +finish_test diff --git a/test/qrf06.test b/test/qrf06.test new file mode 100644 index 000000000..5fa62c26f --- /dev/null +++ b/test/qrf06.test @@ -0,0 +1,576 @@ +# 2025-12-02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the Query Result Formatter (QRF), and especially +# the sqlite3_qrf_wcwidth() function and its utilization. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix qrf06 + +# Data +db eval { + BEGIN TRANSACTION; + CREATE TABLE language(name TEXT); + INSERT INTO language(name) VALUES + ('العربية'), + ('Deutsch'), + ('English'), + ('Español'), + ('فارسی'), + ('Français'), + ('Italiano'), + ('مصرى'), + ('Nederlands'), + ('日本語'), + ('Polski'), + ('Português'), + ('Sinugboanong Binisaya'), + ('Svenska'), + ('Українська'), + ('Tiếng Việt'), + ('Winaray'), + ('中文'), + ('Русский'), + ('Afrikaans'), + ('Shqip'), + ('Asturianu'), + ('Azərbaycanca'), + ('Български'), + ('閩南語 / Bân-lâm-gú'), + ('বাংলা'), + ('Беларуская'), + ('Català'), + ('Čeština'), + ('Cymraeg'), + ('Dansk'), + ('Eesti'), + ('Ελληνικά'), + ('Esperanto'), + ('Euskara'), + ('Galego'), + ('한국어'), + ('Հայերեն'), + ('हिन्दी'), + ('Hrvatski'), + ('Bahasa Indonesia'), + ('עברית'), + ('ქართული'), + ('Ladin'), + ('Latina'), + ('Latviešu'), + ('Lietuvių'), + ('Magyar'), + ('Македонски'), + ('Malagasy'), + ('मराठी'), + ('Bahasa Melayu'), + ('Bahaso Minangkabau'), + ('မြန်မာဘာသာ'), + ('Norskbokmålnynorsk'), + ('Нохчийн'), + ('Oʻzbekcha / Ўзбекча'), + ('Қазақша / Qazaqşa / قازاقشا'), + ('Română'), + ('Simple English'), + ('Slovenčina'), + ('Slovenščina'), + ('Српски / Srpski'), + ('Srpskohrvatski / Српскохрватски'), + ('Suomi'), + ('Kiswahili'), + ('தமிழ்'), + ('Татарча / Tatarça'), + ('తెలుగు'), + ('ภาษาไทย'), + ('Тоҷикӣ'), + ('تۆرکجه'), + ('Türkçe'), + ('اردو'), + ('粵語'), + ('Bahsa Acèh'), + ('Alemannisch'), + ('አማርኛ'), + ('Aragonés'), + ('Արեւմտահայերէն'), + ('Bahasa Hulontalo'), + ('Basa Bali'), + ('Bahasa Banjar'), + ('Basa Banyumasan'), + ('Башҡортса'), + ('Беларуская (тарашкевіца)'), + ('Bikol Central'), + ('বিষ্ণুপ্রিয়া মণিপুরী'), + ('Boarisch'), + ('Bosanski'), + ('Brezhoneg'), + ('Чӑвашла'), + ('Dagbanli'), + ('الدارجة'), + ('Diné Bizaad'), + ('Emigliàn–Rumagnòl'), + ('Fiji Hindi'), + ('Føroyskt'), + ('Frysk'), + ('Fulfulde'), + ('Gaeilge'), + ('Gàidhlig'), + ('گیلکی'), + ('ગુજરાતી'), + ('Hak-kâ-ngî / 客家語'), + ('Hausa'), + ('Hornjoserbsce'), + ('Ido'), + ('Igbo'), + ('Ilokano'), + ('Interlingua'), + ('Interlingue'), + ('Ирон'), + ('Íslenska'), + ('Jawa'), + ('ಕನ್ನಡ'), + ('Kapampangan'), + ('ភាសាខ្មែរ'), + ('Kotava'), + ('Kreyòl Ayisyen'), + ('Kurdî / كوردی'), + ('کوردیی ناوەندی'), + ('Кыргызча'), + ('Кырык мары'), + ('Lëtzebuergesch'), + ('Lìgure'), + ('Limburgs'), + ('Lombard'), + ('मैथिली'), + ('മലയാളം'), + ('მარგალური'), + ('مازِرونی'), + ('Mìng-dĕ̤ng-ngṳ̄ / 閩東語'), + ('Монгол'), + ('Napulitano'), + ('नेपाल भाषा'), + ('Nordfriisk'), + ('Occitan'), + ('Олык марий'), + ('ଓଡି଼ଆ'), + ('অসমীযা়'), + ('ਪੰਜਾਬੀ'), + ('پنجابی (شاہ مکھی)'), + ('پښتو'), + ('Piemontèis'), + ('Plattdüütsch'), + ('Qaraqalpaqsha'), + ('Qırımtatarca'), + ('Runa Simi'), + ('Русиньскый'), + ('संस्कृतम्'), + ('ᱥᱟᱱᱛᱟᱲᱤ'), + ('سرائیکی'), + ('Саха Тыла'), + ('Scots'), + ('ChiShona'), + ('Sicilianu'), + ('සිංහල'), + ('سنڌي'), + ('Ślůnski'), + ('Basa Sunda'), + ('Taclḥit'), + ('Tagalog'), + ('ၽႃႇသႃႇတႆး'), + ('ⵜⴰⵎⴰⵣⵉⵖⵜ ⵜⴰⵏⴰⵡⴰⵢⵜ'), + ('tolışi'), + ('chiTumbuka'), + ('Basa Ugi'), + ('Vèneto'), + ('Volapük'), + ('Walon'), + ('文言'), + ('吴语'), + ('ייִדיש'), + ('Yorùbá'), + ('Zazaki'), + ('žemaitėška'), + ('isiZulu'), + ('नेपाली'), + ('ꯃꯤꯇꯩ ꯂꯣꯟ'), + ('Dzhudezmo / לאדינו'), + ('Адыгэбзэ'), + ('Ænglisc'), + ('Anarâškielâ'), + ('अंगिका'), + ('Аԥсшәа'), + ('armãneashti'), + ('Arpitan'), + ('atikamekw'), + ('ܐܬܘܪܝܐ'), + ('Avañe’ẽ'), + ('Авар'), + ('Aymar'), + ('Batak Toba'), + ('Betawi'), + ('भोजपुरी'), + ('Bislama'), + ('བོད་ཡིག'), + ('Буряад'), + ('Chavacano de Zamboanga'), + ('Chichewa'), + ('Corsu'), + ('Vahcuengh / 話僮'), + ('Dagaare'), + ('Davvisámegiella'), + ('Deitsch'), + ('ދިވެހިބަސް'), + ('Dolnoserbski'), + ('Dusun Bundu-liwan'), + ('Эрзянь'), + ('Estremeñu'), + ('Eʋegbe'), + ('Farefare'), + ('Fɔ̀ngbè'), + ('Furlan'), + ('Gaelg'), + ('Gagauz'), + ('ГӀалгӀай'), + ('Ghanaian Pidgin'), + ('Gĩkũyũ'), + ('赣语 / 贛語'), + ('Gungbe'), + ('Хальмг'), + ('ʻŌlelo Hawaiʻi'), + ('Ikinyarwanda'), + ('Jaku Iban'), + ('Kabɩyɛ'), + ('Yerwa Kanuri'), + ('Kaszëbsczi'), + ('Kernewek'), + ('Коми'), + ('Перем коми'), + ('Kongo'), + ('कोंकणी / Konknni'), + ('كٲشُر'), + ('Kriyòl Gwiyannen'), + ('Kumoring'), + ('Kʋsaal'), + ('ພາສາລາວ'), + ('Лакку'), + ('Latgaļu'), + ('Лезги'), + ('Li Niha'), + ('Lingála'), + ('Lingua Franca Nova'), + ('livvinkarjala'), + ('lojban'), + ('Luganda'), + ('Madhurâ'), + ('Malti'), + ('Mandailing'), + ('Māori'), + ('Mfantse'), + ('Mirandés'), + ('Мокшень'), + ('ဘာသာ မန်'), + ('Moore'), + ('ߒߞߏ'), + ('Na Vosa Vaka-Viti'), + ('Nāhuatlahtōlli'), + ('Naijá'), + ('Nedersaksisch'), + ('Nouormand / Normaund'), + ('Novial'), + ('Afaan Oromoo'), + ('ပအိုဝ်ႏဘာႏသာႏ'), + ('Pangasinán'), + ('Pangcah'), + ('Papiamentu'), + ('Patois'), + ('Pfälzisch'), + ('Picard'), + ('Къарачай–малкъар'), + ('Ripoarisch'), + ('Rumantsch'), + ('Sakizaya'), + ('Gagana Sāmoa'), + ('Sardu'), + ('Seediq'), + ('Seeltersk'), + ('Sesotho'), + ('Sesotho sa Leboa'), + ('Setswana'), + ('ꠍꠤꠟꠐꠤ'), + ('Словѣ́ньскъ / ⰔⰎⰑⰂⰡⰐⰠⰔⰍⰟ'), + ('Soomaaliga'), + ('Sranantongo'), + ('SiSwati'), + ('Reo tahiti'), + ('Taqbaylit'), + ('Tarandíne'), + ('Tayal'), + ('Tetun'), + ('Tok Pisin'), + ('faka Tonga'), + ('Türkmençe'), + ('Twi'), + ('Tyap'), + ('Тыва дыл'), + ('Удмурт'), + ('ئۇيغۇرچه'), + ('Vepsän'), + ('võro'), + ('West-Vlams'), + ('Wolof'), + ('isiXhosa'), + ('Zeêuws'), + ('алтай тил'), + ('अवधी'), + ('डोटेली'), + ('ತುಳು'), + ('ရခိုင်'), + ('Bajau Sama'), + ('Bamanankan'), + ('Chamoru'), + ('རྫོང་ཁ'), + ('𐌲𐌿𐍄𐌹𐍃𐌺x'), + ('Igala'), + ('ᐃᓄᒃᑎᑐᑦ / Inuktitut'), + ('Iñupiak'), + ('isiNdebele seSewula'), + ('Kalaallisut'), + ('Nupe'), + ('Obolo'), + ('पालि'), + ('pinayuanan'), + ('Ποντιακά'), + ('romani čhib'), + ('Ikirundi'), + ('руски'), + ('Sängö'), + ('ᥖᥭᥰᥖᥬᥳᥑᥨᥒᥰ'), + ('ትግርኛ'), + ('Thuɔŋjäŋ'), + ('ᏣᎳᎩ'), + ('Tsėhesenėstsestotse'), + ('Xitsonga'), + ('Tshivenḓa'), + ('Wayuunaiki'), + ('адыгабзэ'); + COMMIT; +} + +do_test 1.2 { + set res \n[db format -style box { + SELECT name, rowid AS id FROM language + WHERE length(name)=2 + ORDER BY name + }] + set exp { +╭──────┬─────╮ +│ name │ id │ +╞══════╪═════╡ +│ 中文 │ 18 │ +│ 吴语 │ 173 │ +│ 文言 │ 172 │ +│ 粵語 │ 75 │ +╰──────┴─────╯ +} + if {$res ne $exp} { + puts [list $res] + puts [list $exp] + } + string compare $res $exp +} {0} + +do_test 1.3 { + set res \n[db format -style box { + SELECT name, rowid AS id FROM language + WHERE length(name)=3 + ORDER BY name + }] + set exp { +╭────────┬─────╮ +│ name │ id │ +╞════════╪═════╡ +│ Ido │ 108 │ +│ Twi │ 297 │ +│ ߒߞߏ │ 258 │ +│ ᏣᎳᎩ │ 335 │ +│ 日本語 │ 10 │ +│ 한국어 │ 37 │ +╰────────┴─────╯ +} + if {$res ne $exp} { + puts [list $res] + puts [list $exp] + } + string compare $res $exp +} {0} + +do_test 1.4 { + set res \n[db format -style box { + SELECT name, rowid AS id FROM language + WHERE length(name)=4 + ORDER BY name + }] + set exp { +╭──────┬─────╮ +│ name │ id │ +╞══════╪═════╡ +│ Igbo │ 109 │ +│ Jawa │ 115 │ +│ Nupe │ 323 │ +│ Tyap │ 298 │ +│ võro │ 303 │ +│ Авар │ 192 │ +│ Ирон │ 113 │ +│ Коми │ 231 │ +│ اردو │ 74 │ +│ سنڌي │ 159 │ +│ مصرى │ 8 │ +│ پښتو │ 144 │ +│ अवधी │ 309 │ +│ पालि │ 325 │ +│ ತುಳು │ 311 │ +│ ትግርኛ │ 333 │ +│ አማርኛ │ 78 │ +╰──────┴─────╯ +} + if {$res ne $exp} { + puts [list $res] + puts [list $exp] + } + string compare $res $exp +} {0} + +do_test 1.5 { + set res \n[db format -style box { + SELECT name, rowid AS id FROM language + WHERE length(name)=5 + ORDER BY name + }] + set exp { +╭───────┬─────╮ +│ name │ id │ +╞═══════╪═════╡ +│ Aymar │ 193 │ +│ Corsu │ 202 │ +│ Dansk │ 31 │ +│ Eesti │ 32 │ +│ Frysk │ 99 │ +│ Gaelg │ 216 │ +│ Hausa │ 106 │ +│ Igala │ 318 │ +│ Kongo │ 233 │ +│ Ladin │ 44 │ +│ Malti │ 250 │ +│ Moore │ 257 │ +│ Māori │ 252 │ +│ Naijá │ 261 │ +│ Obolo │ 324 │ +│ Sardu │ 278 │ +│ Scots │ 155 │ +│ Shqip │ 21 │ +│ Suomi │ 65 │ +│ Sängö │ 331 │ +│ Tayal │ 292 │ +│ Tetun │ 293 │ +│ Walon │ 171 │ +│ Wolof │ 305 │ +│ Лакку │ 240 │ +│ Лезги │ 242 │ +│ руски │ 330 │ +│ עברית │ 42 │ +│ فارسی │ 5 │ +│ كٲشُر │ 235 │ +│ گیلکی │ 103 │ +│ मराठी │ 51 │ +│ বাংলা │ 26 │ +│ ଓଡି଼ଆ │ 140 │ +│ தமிழ் │ 67 │ +│ ಕನ್ನಡ │ 116 │ +│ සිංහල │ 158 │ +│ ꠍꠤꠟꠐꠤ │ 284 │ +╰───────┴─────╯ +} + if {$res ne $exp} { + puts [list $res] + puts [list $exp] + } + string compare $res $exp +} {0} + +do_test 1.6 { + set res \n[db format -style box { + SELECT name, rowid AS id FROM language + WHERE length(name)=6 + ORDER BY name + }] + set exp { +╭────────┬─────╮ +│ name │ id │ +╞════════╪═════╡ +│ Betawi │ 195 │ +│ Català │ 28 │ +│ Eʋegbe │ 212 │ +│ Furlan │ 215 │ +│ Gagauz │ 217 │ +│ Galego │ 36 │ +│ Gungbe │ 222 │ +│ Gĩkũyũ │ 220 │ +│ Kabɩyɛ │ 227 │ +│ Kotava │ 119 │ +│ Kʋsaal │ 238 │ +│ Latina │ 45 │ +│ Lìgure │ 126 │ +│ Magyar │ 48 │ +│ Novial │ 264 │ +│ Patois │ 270 │ +│ Picard │ 272 │ +│ Polski │ 11 │ +│ Română │ 59 │ +│ Seediq │ 279 │ +│ Türkçe │ 73 │ +│ Vepsän │ 302 │ +│ Vèneto │ 169 │ +│ Yorùbá │ 175 │ +│ Zazaki │ 176 │ +│ Zeêuws │ 307 │ +│ lojban │ 247 │ +│ tolışi │ 166 │ +│ Аԥсшәа │ 186 │ +│ Буряад │ 199 │ +│ Монгол │ 134 │ +│ Тоҷикӣ │ 71 │ +│ Удмурт │ 300 │ +│ Хальмг │ 223 │ +│ Эрзянь │ 210 │ +│ ייִדיש │ 174 │ +│ تۆرکجه │ 72 │ +│ ܐܬܘܪܝܐ │ 190 │ +│ अंगिका │ 185 │ +│ डोटेली │ 310 │ +│ नेपाली │ 179 │ +│ मैथिली │ 129 │ +│ हिन्दी │ 39 │ +│ ਪੰਜਾਬੀ │ 142 │ +│ తెలుగు │ 69 │ +│ മലയാളം │ 130 │ +│ རྫོང་ཁ │ 316 │ +│ ရခိုင် │ 312 │ +╰────────┴─────╯ +} + if {$res ne $exp} { + puts [list $res] + puts [list $exp] + } + string compare $res $exp +} {0} + +finish_test diff --git a/test/recover.test b/test/recover.test index ad6b7298d..7035e407c 100644 --- a/test/recover.test +++ b/test/recover.test @@ -42,7 +42,7 @@ proc compare_dbs {db1 db2} { proc recover_with_opts {opts} { set cmd ".recover $opts" - set fd [open [list |$::CLI test.db $cmd]] + set fd [open [list |$::CLI -noinit test.db $cmd]] fconfigure $fd -translation binary set sql [read $fd] close $fd diff --git a/test/regexp1.sql b/test/regexp1.sql new file mode 100644 index 000000000..c1938885a --- /dev/null +++ b/test/regexp1.sql @@ -0,0 +1,32 @@ +#!sqlite3 +# +# 2025-12-16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the oversized patterns in the REGEXP extension found +# at ext/misc/regexp.c. +# +.mode list +.testcase 100 +-- 0 1 2 3 4 +-- 123456789 123456789 123456789 123456789 123 +SELECT 'abcdefg' REGEXP '((((((((((((((((((abcdefg))))))))))))))))))'; +.check 1 + +.testcase 110 +.limit like_pattern_length 42 +SELECT 'abcdefg' REGEXP '((((((((((((((((((abcdefg))))))))))))))))))'; +.check -glob "Error near line #: REGEXP pattern too big*" + +.testcase 120 +.limit like_pattern_length 43 +SELECT 'abcdefg' REGEXP '((((((((((((((((((abcdefg))))))))))))))))))'; +.check 1 diff --git a/test/regexp1.test b/test/regexp1.test index 0401b13d7..fb123284b 100644 --- a/test/regexp1.test +++ b/test/regexp1.test @@ -331,5 +331,29 @@ do_execsql_test regexp1-7.12 { SELECT char(0x61,0x10ffff,0x62) REGEXP char(0x10ffff); } 1 +do_execsql_test regexp1-8.0 { + CREATE TABLE t2(a); + INSERT INTO t2 VALUES('abc-def'); + SELECT length(a) FROM t2; +} {7} + +do_execsql_test regexp1-8.1 { + SELECT rowid FROM t2 WHERE a REGEXP '[1-5]'; +} {} +do_execsql_test regexp1-8.2 { + SELECT rowid FROM t2 WHERE a REGEXP '[1\-5]'; +} {1} +do_execsql_test regexp1-8.3 { + SELECT rowid FROM t2 WHERE a REGEXP '[x\-]'; +} {1} +do_catchsql_test regexp1-8.4 { + SELECT rowid FROM t2 WHERE a REGEXP '[x-]'; +} {1 {unclosed '['}} +do_execsql_test regexp1-8.5 { + SELECT rowid FROM t2 WHERE a REGEXP '-'; +} {1} +do_execsql_test regexp1-8.6 { + SELECT rowid FROM t2 WHERE a REGEXP '\-'; +} {1} finish_test diff --git a/test/rowvalue4.test b/test/rowvalue4.test index 1ef5fc292..5e02f0fc2 100644 --- a/test/rowvalue4.test +++ b/test/rowvalue4.test @@ -236,8 +236,7 @@ do_eqp_test 5.1 { QUERY PLAN |--SEARCH d2 USING INDEX d2ab (a=? AND b=?) |--LIST SUBQUERY xxxxxx - | |--SCAN d1 - | `--CREATE BLOOM FILTER + | `--SCAN d1 `--LIST SUBQUERY xxxxxx |--SCAN d1 `--CREATE BLOOM FILTER diff --git a/test/rowvalueA.test b/test/rowvalueA.test index 8760c2c39..16429f985 100644 --- a/test/rowvalueA.test +++ b/test/rowvalueA.test @@ -73,4 +73,29 @@ do_catchsql_test 2.3 { SELECT 2 IN ( (1, 2), (3, 4), (5, 6) ) } {1 {row value misused}} +#------------------------------------------------------------------------- +# Test the fix for forum post https://sqlite.org/forum/forumpost/6ceca07fc3 +# +do_execsql_test 3.0 { + CREATE TABLE x2 (x, y); + INSERT INTO x2 VALUES (1234, 'abc'); + + CREATE TABLE x1 (a, b PRIMARY KEY COLLATE NOCASE) WITHOUT ROWID; + INSERT INTO x1 VALUES (1234, 'ABCD'); +} + +do_execsql_test 3.1 { + SELECT * FROM x2 CROSS JOIN x1 WHERE (1234, x2.y) > (x1.a, x1.b); +} {1234 abc 1234 ABCD} + +do_execsql_test 3.2 { + CREATE INDEX x1a ON x1(a); +} + +do_execsql_test 3.3 { + SELECT * FROM x2 CROSS JOIN x1 WHERE (1234, x2.y) > (x1.a, x1.b); +} {1234 abc 1234 ABCD} + + + finish_test diff --git a/test/schema.test b/test/schema.test index c7daef20b..a6564293b 100644 --- a/test/schema.test +++ b/test/schema.test @@ -227,10 +227,10 @@ ifcapable auth { set ::STMT [sqlite3_prepare $::DB {SELECT * FROM sqlite_master} -1 TAIL] db auth {} sqlite3_step $::STMT - } {SQLITE_ROW} + } {SQLITE_ERROR} do_test schema-8.12 { sqlite3_finalize $::STMT - } {SQLITE_OK} + } {SQLITE_SCHEMA} } diff --git a/test/select9.test b/test/select9.test index bbed8e18f..bef56d83f 100644 --- a/test/select9.test +++ b/test/select9.test @@ -406,7 +406,7 @@ do_test select9-4.4 { do_test select9-4.5 { execsql { CREATE VIEW v1 AS SELECT a FROM t1 UNION SELECT d FROM t2 } cksort { SELECT a FROM v1 ORDER BY 1 LIMIT 5 } -} {1 2 3 4 5 sort} +} {1 2 3 4 5 nosort} do_test select9-4.X { execsql { DROP INDEX i1; diff --git a/test/shell1.test b/test/shell1.test index abf214a90..1d111d616 100644 --- a/test/shell1.test +++ b/test/shell1.test @@ -216,10 +216,14 @@ do_test shell1-2.2.4 { } {0 {}} do_test shell1-2.2.5 { catchcmd "test.db" ".mode \"insert FOO" -} {1 {Error: mode should be one of: ascii box column csv html insert json line list markdown qbox quote table tabs tcl}} +} {1 {line 1: .mode "insert FOO +line 1: ^--- unknown mode +line 1: Use ".help .mode" for more info}} do_test shell1-2.2.6 { catchcmd "test.db" ".mode \'insert FOO" -} {1 {Error: mode should be one of: ascii box column csv html insert json line list markdown qbox quote table tabs tcl}} +} {1 {line 1: .mode 'insert FOO +line 1: ^--- unknown mode +line 1: Use ".help .mode" for more info}} # check multiple tokens, and quoted tokens do_test shell1-2.3.1 { @@ -247,7 +251,9 @@ do_test shell1-2.3.7 { # check quoted args are unquoted do_test shell1-2.4.1 { catchcmd "test.db" ".mode FOO" -} {1 {Error: mode should be one of: ascii box column csv html insert json line list markdown qbox quote table tabs tcl}} +} {1 {line 1: .mode FOO +line 1: ^--- unknown mode +line 1: Use ".help .mode" for more info}} do_test shell1-2.4.2 { catchcmd "test.db" ".mode csv" } {0 {}} @@ -296,9 +302,7 @@ do_test shell1-3.2.4 { catchcmd "test.db" ".bail OFF BAD" } {1 {Usage: .bail on|off}} -# This test will not work on winrt, as winrt has no concept of the absolute -# paths that the test expects in the result. It uses relative paths only. -ifcapable vtab&&!winrt { +ifcapable vtab { # .databases List names and files of attached databases do_test shell1-3.3.1 { catchcmd "-csv test.db" ".databases" @@ -364,7 +368,6 @@ do_test shell1-3.7.4 { catchcmd "test.db" ".explain OFF BAD" } {0 {}} - # .header(s) ON|OFF Turn display of headers on or off do_test shell1-3.9.1 { catchcmd "test.db" ".header" @@ -400,7 +403,7 @@ do_test shell1-3.10.1 { # look for a few of the possible help commands list [regexp {.help} $res] \ [regexp {.quit} $res] \ - [regexp {.show} $res] + [regexp {.mode} $res] } {1 1 1} do_test shell1-3.10.2 { # we allow .help to take extra args (it is help after all) @@ -408,20 +411,24 @@ do_test shell1-3.10.2 { # look for a few of the possible help commands list [regexp {.help} $res] \ [regexp {.quit} $res] \ - [regexp {.show} $res] + [regexp {.mode} $res] } {1 1 1} # .import FILE TABLE Import data from FILE into TABLE do_test shell1-3.11.1 { catchcmd "test.db" ".import" -} {/1 .ERROR: missing FILE argument.*/} +} {/1 .line 1: Missing FILE argument.*/} do_test shell1-3.11.2 { catchcmd "test.db" ".import FOO" -} {/1 .ERROR: missing TABLE argument.*/} +} {/1 .line 1: Missing TABLE argument.*/} do_test shell1-3.11.3 { # too many arguments catchcmd "test.db" ".import FOO BAR BAD" -} {/1 .ERROR: extra argument: "BAD".*./} +} {1 {line 1: .import FOO BAR BAD +line 1: ^--- unknown argument}} +do_test shell1-3.11.4 { + catchcmd "test.db" ".import <<END t1\na,b,c\n1,2,3" +} {1 {line 1: Content terminator "END" not found.}} # .indexes ?TABLE? Show names of all indexes # If TABLE specified, only show indexes for tables @@ -451,11 +458,13 @@ do_test shell1-3.12.3 { # tabs Tab-separated values # tcl TCL list elements do_test shell1-3.13.1 { - catchcmd "test.db" ".mode" -} {0 {current output mode: list --escape ascii}} + catchcmd "test.db" ".mode batch\n.mode" +} {0 {.mode list}} do_test shell1-3.13.2 { catchcmd "test.db" ".mode FOO" -} {1 {Error: mode should be one of: ascii box column csv html insert json line list markdown qbox quote table tabs tcl}} +} {1 {line 1: .mode FOO +line 1: ^--- unknown mode +line 1: Use ".help .mode" for more info}} do_test shell1-3.13.3 { catchcmd "test.db" ".mode csv" } {0 {}} @@ -503,7 +512,7 @@ do_test shell1-3.15.1 { .print x" } {0 x} do_test shell1-3.15.2 { - catchcmd "test.db" ".output FOO + catchcmd "test.db" ".mode batch\n.output FOO .print x .output SELECT readfile('FOO');" @@ -512,17 +521,8 @@ SELECT readfile('FOO');" do_test shell1-3.15.3 { # too many arguments catchcmd "test.db" ".output FOO BAD" -} {1 {ERROR: extra parameter: "BAD". Usage: -.output ?FILE? Send output to FILE or stdout if FILE is omitted - If FILE begins with '|' then open it as a pipe. - If FILE is 'off' then output is disabled. - Options: - --bom Prefix output with a UTF8 byte-order mark - -e Send output to the system text editor - --plain Use text/plain for -w option - -w Send output to a web browser - -x Send output as CSV to a spreadsheet -child process exited abnormally}} +} {1 {line 1: .output FOO BAD +line 1: ^--- surplus argument}} # .output stdout Send output to the screen do_test shell1-3.16.1 { @@ -531,17 +531,8 @@ do_test shell1-3.16.1 { do_test shell1-3.16.2 { # too many arguments catchcmd "test.db" ".output stdout BAD" -} {1 {ERROR: extra parameter: "BAD". Usage: -.output ?FILE? Send output to FILE or stdout if FILE is omitted - If FILE begins with '|' then open it as a pipe. - If FILE is 'off' then output is disabled. - Options: - --bom Prefix output with a UTF8 byte-order mark - -e Send output to the system text editor - --plain Use text/plain for -w option - -w Send output to a web browser - -x Send output as CSV to a spreadsheet -child process exited abnormally}} +} {1 {line 1: .output stdout BAD +line 1: ^--- surplus argument}} # .prompt MAIN CONTINUE Replace the standard prompts do_test shell1-3.17.1 { @@ -625,6 +616,20 @@ CREATE VIEW v1 AS SELECT y+1 FROM v2 catch {db eval {DROP VIEW v1; DROP VIEW v2; DROP TABLE t1;}} } +do_test shell1-3.21.5 { + exec {*}$CLI -noinit test.db \ + {CREATE TABLE t2(a INTEGER PRIMARY KEY, b BLOB DEFAULT(jsonb('[]')),c TEXT NOT NULL)STRICT;} \ + {.schema -indent t2} +} {CREATE TABLE t2( + a INTEGER PRIMARY KEY, + b BLOB DEFAULT(jsonb('[]')), + c TEXT NOT NULL +)STRICT;} +do_test shell1-3.21.6 { + exec {*}$CLI -noinit test.db \ + {DROP TABLE t2;} \ + {.schema -indent t2} +} {} # .separator STRING Change column separator used by output and .import do_test shell1-3.22.1 { @@ -643,7 +648,7 @@ do_test shell1-3.22.4 { # .show Show the current values for various settings do_test shell1-3.23.1 { - set res [catchcmd "test.db" ".show"] + set res [catchcmd "test.db" ".mode batch\n.show"] list [regexp {echo:} $res] \ [regexp {explain:} $res] \ [regexp {headers:} $res] \ @@ -679,7 +684,7 @@ do_test shell1-3.23b.4 { # Adverse interaction between .stats and .eqp # do_test shell1-3.23b.5 { - catchcmd "test.db" [string map {"\n " "\n"} { + catchcmd "test.db" [string map {"\n " "\n"} {.mode batch CREATE TEMP TABLE t1(x); INSERT INTO t1 VALUES(1),(2); .stats on @@ -741,30 +746,27 @@ do_test shell1-3.26.5 { do_test shell1-3.26.6 { catchcmd "test.db" ".mode column\n.header off\n.width -10 10\nSELECT 'abcdefg', 123456;" # this should be treated the same as a '1' width for col 1 and 2 -} {0 { abcdefg 123456 }} +} {0 { abcdefg 123456}} # .timer ON|OFF Turn the CPU timer measurement on or off do_test shell1-3.27.1 { catchcmd "test.db" ".timer" -} {1 {Usage: .timer on|off}} -ifcapable !winrt { - # No timer support on winrt. - do_test shell1-3.27.2 { - catchcmd "test.db" ".timer ON" - } {0 {}} -} +} {1 {Usage: .timer on|off|once}} +do_test shell1-3.27.2 { + catchcmd "test.db" ".timer ON" +} {0 {}} do_test shell1-3.27.3 { catchcmd "test.db" ".timer OFF" } {0 {}} do_test shell1-3.27.4 { # too many arguments catchcmd "test.db" ".timer OFF BAD" -} {1 {Usage: .timer on|off}} +} {1 {Usage: .timer on|off|once}} -do_test shell1-3-28.1 { +do_test shell1-3.28.1 { catchcmd test.db \ - ".log stdout\nSELECT coalesce(sqlite_log(123,'hello'),'456');" + ".mode batch\n.log stdout\nSELECT coalesce(sqlite_log(123,'hello'),'456');" } "0 {(123) hello\n456}" do_test shell1-3-29.1 { @@ -803,14 +805,14 @@ INSERT INTO t1 VALUES(''); INSERT INTO t1 VALUES(1); INSERT INTO t1 VALUES(2.25); INSERT INTO t1 VALUES('hello'); -INSERT INTO t1 VALUES(X'807f'); +INSERT INTO t1 VALUES(x'807f'); CREATE TABLE t3(x,y); INSERT INTO t3 VALUES(1,NULL); INSERT INTO t3 VALUES(2,''); INSERT INTO t3 VALUES(3,1); INSERT INTO t3 VALUES(4,2.25); INSERT INTO t3 VALUES(5,'hello'); -INSERT INTO t3 VALUES(6,X'807f'); +INSERT INTO t3 VALUES(6,x'807f'); COMMIT;}} @@ -828,14 +830,14 @@ INSERT INTO t1(rowid,x) VALUES(2,''); INSERT INTO t1(rowid,x) VALUES(3,1); INSERT INTO t1(rowid,x) VALUES(4,2.25); INSERT INTO t1(rowid,x) VALUES(5,'hello'); -INSERT INTO t1(rowid,x) VALUES(6,X'807f'); +INSERT INTO t1(rowid,x) VALUES(6,x'807f'); CREATE TABLE t3(x,y); INSERT INTO t3(rowid,x,y) VALUES(1,1,NULL); INSERT INTO t3(rowid,x,y) VALUES(2,2,''); INSERT INTO t3(rowid,x,y) VALUES(3,3,1); INSERT INTO t3(rowid,x,y) VALUES(4,4,2.25); INSERT INTO t3(rowid,x,y) VALUES(5,5,'hello'); -INSERT INTO t3(rowid,x,y) VALUES(6,6,X'807f'); +INSERT INTO t3(rowid,x,y) VALUES(6,6,x'807f'); COMMIT;}} # If the table contains an INTEGER PRIMARY KEY, do not record a separate @@ -854,12 +856,12 @@ do_test shell1-4.1.2 { } {0 {PRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE t1(x INTEGER PRIMARY KEY, y); -INSERT INTO t1 VALUES(1,NULL); -INSERT INTO t1 VALUES(2,''); -INSERT INTO t1 VALUES(3,1); -INSERT INTO t1 VALUES(4,2.25); -INSERT INTO t1 VALUES(5,'hello'); -INSERT INTO t1 VALUES(6,X'807f'); +INSERT INTO t1(x,y) VALUES(1,NULL); +INSERT INTO t1(x,y) VALUES(2,''); +INSERT INTO t1(x,y) VALUES(3,1); +INSERT INTO t1(x,y) VALUES(4,2.25); +INSERT INTO t1(x,y) VALUES(5,'hello'); +INSERT INTO t1(x,y) VALUES(6,x'807f'); COMMIT;}} # Verify that the table named [table] is correctly quoted and that @@ -883,7 +885,7 @@ INSERT INTO "table"(rowid,x,y) VALUES(2,12,''); INSERT INTO "table"(rowid,x,y) VALUES(3,23,1); INSERT INTO "table"(rowid,x,y) VALUES(4,34,2.25); INSERT INTO "table"(rowid,x,y) VALUES(5,45,'hello'); -INSERT INTO "table"(rowid,x,y) VALUES(6,56,X'807f'); +INSERT INTO "table"(rowid,x,y) VALUES(6,56,x'807f'); COMMIT;}} # Do not record rowids for a WITHOUT ROWID table. Also check correct quoting @@ -902,12 +904,12 @@ do_test shell1-4.1.4 { } {0 {PRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE [ta<>ble](x INTEGER PRIMARY KEY, y) WITHOUT ROWID; -INSERT INTO "ta<>ble" VALUES(1,NULL); -INSERT INTO "ta<>ble" VALUES(12,''); -INSERT INTO "ta<>ble" VALUES(23,1); -INSERT INTO "ta<>ble" VALUES(34,2.25); -INSERT INTO "ta<>ble" VALUES(45,'hello'); -INSERT INTO "ta<>ble" VALUES(56,X'807f'); +INSERT INTO "ta<>ble"(x,y) VALUES(1,NULL); +INSERT INTO "ta<>ble"(x,y) VALUES(12,''); +INSERT INTO "ta<>ble"(x,y) VALUES(23,1); +INSERT INTO "ta<>ble"(x,y) VALUES(34,2.25); +INSERT INTO "ta<>ble"(x,y) VALUES(45,'hello'); +INSERT INTO "ta<>ble"(x,y) VALUES(56,x'807f'); COMMIT;}} # Do not record rowids if the rowid is inaccessible @@ -924,9 +926,9 @@ do_test shell1-4.1.5 { } {0 {PRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE t1(_ROWID_,rowid,oid); -INSERT INTO t1 VALUES(1,NULL,'alpha'); -INSERT INTO t1 VALUES(12,'',99); -INSERT INTO t1 VALUES(23,1,X'b0b1b2'); +INSERT INTO t1(_ROWID_,rowid,oid) VALUES(1,NULL,'alpha'); +INSERT INTO t1(_ROWID_,rowid,oid) VALUES(12,'',99); +INSERT INTO t1(_ROWID_,rowid,oid) VALUES(23,1,x'b0b1b2'); COMMIT;}} } else { @@ -941,7 +943,7 @@ do_test shell1-4.1.6 { (4,2.25), (5,'hello'), (6,x'807f'); } catchcmd test2.db {.dump --preserve-rowids} -} {1 {The --preserve-rowids option is not compatible with SQLITE_OMIT_VIRTUALTABLE}} +} {/.* --preserve-rowids option is not compatible with SQLITE_OMIT_VIRTUALTABLE/} } @@ -1020,7 +1022,7 @@ INSERT INTO t1 VALUES(''); INSERT INTO t1 VALUES(1); INSERT INTO t1 VALUES(2.25); INSERT INTO t1 VALUES('hello'); -INSERT INTO t1 VALUES(X'807f');}} +INSERT INTO t1 VALUES(x'807f');}} # Test the output of ".mode insert" with headers # @@ -1031,7 +1033,7 @@ INSERT INTO t1(x) VALUES(''); INSERT INTO t1(x) VALUES(1); INSERT INTO t1(x) VALUES(2.25); INSERT INTO t1(x) VALUES('hello'); -INSERT INTO t1(x) VALUES(X'807f');}} +INSERT INTO t1(x) VALUES(x'807f');}} # Test the output of ".mode insert" # @@ -1042,7 +1044,7 @@ INSERT INTO t3 VALUES(2,''); INSERT INTO t3 VALUES(3,1); INSERT INTO t3 VALUES(4,2.25); INSERT INTO t3 VALUES(5,'hello'); -INSERT INTO t3 VALUES(6,X'807f');}} +INSERT INTO t3 VALUES(6,x'807f');}} # Test the output of ".mode insert" with headers # @@ -1053,7 +1055,7 @@ INSERT INTO t3(x,y) VALUES(2,''); INSERT INTO t3(x,y) VALUES(3,1); INSERT INTO t3(x,y) VALUES(4,2.25); INSERT INTO t3(x,y) VALUES(5,'hello'); -INSERT INTO t3(x,y) VALUES(6,X'807f');}} +INSERT INTO t3(x,y) VALUES(6,x'807f');}} # Test the output of ".mode tcl" # @@ -1069,8 +1071,8 @@ do_test shell1-4.3 { catchcmd test.db ".mode tcl\nselect * from t1;" } {0 {"" "" -"1" -"2.25" +1 +2.25 "hello" "\200\177"}} @@ -1083,15 +1085,15 @@ do_test shell1-4.4 { } catchcmd test.db ".mode tcl\nselect * from t2;" } {0 {"" "" -"1" "2.25" +1 2.25 "hello" "\200\177"}} # Test the output of ".mode tcl" with ".nullvalue" # do_test shell1-4.5 { catchcmd test.db ".mode tcl\n.nullvalue NULL\nselect * from t2;" -} {0 {"NULL" "" -"1" "2.25" +} {0 {NULL "" +1 2.25 "hello" "\200\177"}} # Test the output of ".mode tcl" with Tcl reserved characters @@ -1115,10 +1117,11 @@ do_test shell1-4.6 { # do_test shell1-4.7 { catchcmd test.db ".mode quote\nselect x'0123456789ABCDEF';" -} {0 X'0123456789abcdef'} +} {0 x'0123456789abcdef'} # Test using arbitrary byte data with the shell via standard input/output. # +if 0 { # Causes a valgrind error in TCL. Seems to be a TCL problem. do_test shell1-5.0 { # # NOTE: Skip NUL byte because it appears to be incompatible with command @@ -1185,6 +1188,7 @@ do_test shell1-5.0 { } } } {} +} # These test cases do not work on MinGW if 0 { @@ -1274,7 +1278,7 @@ do_test shell1-7.1.7 { # information. # do_test shell1-8.1 { - catchcmd ":memory:" { + catchcmd ":memory:" {.mode batch -- The pow2 table will hold all the necessary powers of two. CREATE TABLE pow2(x INTEGER PRIMARY KEY, v TEXT); WITH RECURSIVE c(x,v) AS ( @@ -1300,20 +1304,20 @@ do_test_with_ansi_output shell1-8.2 { .mode box SELECT ieee754(47.49) AS x; } -} {0 {┌───────────────────────────────┐ +} {0 {╭───────────────────────────────╮ │ x │ -├───────────────────────────────┤ +╞═══════════════════════════════╡ │ ieee754(6683623321994527,-47) │ -└───────────────────────────────┘}} +╰───────────────────────────────╯}} do_test_with_ansi_output shell1-8.3 { catchcmd ":memory: --box" { select ieee754(6683623321994527,-47) as x; } -} {0 {┌───────┐ +} {0 {╭───────╮ │ x │ -├───────┤ +╞═══════╡ │ 47.49 │ -└───────┘}} +╰───────╯}} do_test shell1-8.4 { catchcmd ":memory: --table" {SELECT ieee754_mantissa(47.49) AS M, ieee754_exponent(47.49) AS E;} } {0 {+------------------+-----+ @@ -1321,35 +1325,42 @@ do_test shell1-8.4 { +------------------+-----+ | 6683623321994527 | -47 | +------------------+-----+}} +do_test shell1-8.4b { + catchcmd ":memory: --psql" \ + {SELECT ieee754_mantissa(47.49) AS M, ieee754_exponent(47.49) AS E;} +} {0 { M | E +------------------+----- + 6683623321994527 | -47}} do_test_with_ansi_output shell1-8.5 { catchcmd ":memory: --box" { create table t(a text, b int); insert into t values ('too long for one line', 1), ('shorter', NULL); .header on +.mode box --wordwrap off .width 10 10 .nullvalue NADA select * from t;} -} {0 {┌────────────┬────────────┐ +} {0 {╭────────────┬────────────╮ │ a │ b │ -├────────────┼────────────┤ -│ too long f │ 1 │ +╞════════════╪════════════╡ +│ too long f │ 1 │ │ or one lin │ │ │ e │ │ ├────────────┼────────────┤ │ shorter │ NADA │ -└────────────┴────────────┘}} +╰────────────┴────────────╯}} #---------------------------------------------------------------------------- # Test cases shell1-9.*: Basic test that "dot" commands and SQL intermix ok. # do_test shell1-9.1 { catchcmd :memory: { -.mode csv +.mode csv --rowsep "\n" /* x */ select 1,2; --x -- .nada ; -.mode csv +.mode csv --rowsep "\n" --x select 2,1; select 3,4; } @@ -1387,4 +1398,30 @@ select base85(zeroblob(2000000000)); } } {/1.*too big.*/} +#---------------------------------------------------------------------------- +# As of 2025-11-17, the default mode is: +# +# qbox --screenwidth auto --linelimit 5 --charlimit 300 --textjsonb on +# +do_test shell1-12.1 { + catchcmd :memory: {.mode tty -quote sql +.print +SELECT jsonb(1234) AS x;} +} {0 { +╭───────────────╮ +│ x │ +╞═══════════════╡ +│ jsonb('1234') │ +╰───────────────╯}} +do_test shell1-12.2 { + catchcmd :memory: {.mode box --textjsonb on +.print +SELECT jsonb(1234) AS x;} +} {0 { +╭──────╮ +│ x │ +╞══════╡ +│ 1234 │ +╰──────╯}} + finish_test diff --git a/test/shell2.test b/test/shell2.test index 5f700a9a1..7141c4d49 100644 --- a/test/shell2.test +++ b/test/shell2.test @@ -44,7 +44,7 @@ do_test shell2-1.1.1 { # Shell silently ignores extra parameters. # Ticket [f5cb008a65]. do_test shell2-1.2.1 { - catchcmdex {:memory: "select+3" "select+4"} + catchcmdex {:memory: -list "select+3" "select+4"} } {0 {3 4 }} @@ -64,7 +64,7 @@ do_test shell2-1.3 { UPDATE OR REPLACE t5 SET a = 4 WHERE a = 1; } -} {1 {Runtime error near line 9: too many levels of trigger recursion}} +} {1 {Error near line 9: too many levels of trigger recursion}} @@ -75,7 +75,8 @@ do_test shell2-1.3 { # NB. whitespace is important do_test shell2-1.4.1 { forcedelete foo.db - catchcmd "foo.db" {CREATE TABLE foo(a); + catchcmd "foo.db" {.mode batch +CREATE TABLE foo(a); INSERT INTO foo(a) VALUES(1); SELECT * FROM foo;} } {0 1} @@ -96,7 +97,9 @@ SELECT * FROM foo; # NB. whitespace is important do_test shell2-1.4.3 { forcedelete foo.db - catchcmd "foo.db" {.echo ON + catchcmd "foo.db" { +.mode batch +.echo ON CREATE TABLE foo(a); INSERT INTO foo(a) VALUES(1); SELECT * FROM foo;} @@ -110,7 +113,9 @@ SELECT * FROM foo; # NB. whitespace is important do_test shell2-1.4.4 { forcedelete foo.db - catchcmd "foo.db" {.echo ON + catchcmd "foo.db" { +.mode batch +.echo ON CREATE TABLE foo(a); .echo OFF INSERT INTO foo(a) VALUES(1); @@ -124,7 +129,9 @@ SELECT * FROM foo;} # NB. whitespace is important do_test shell2-1.4.5 { forcedelete foo.db - catchcmdex "foo.db" {.echo ON + catchcmdex "foo.db" { +.mode batch +.echo ON CREATE TABLE foo1(a); INSERT INTO foo1(a) VALUES(1); CREATE TABLE foo2(b); @@ -153,7 +160,9 @@ SELECT * FROM foo1; SELECT * FROM foo2; # NB. whitespace is important do_test shell2-1.4.6 { forcedelete foo.db - catchcmdex "foo.db" {.echo ON + catchcmdex "foo.db" { +.mode batch +.echo ON .headers ON CREATE TABLE foo1(a); INSERT INTO foo1(a) VALUES(1); @@ -208,6 +217,7 @@ do_test shell2-1.4.9 { do_test shell2-1.4.9 { forcedelete clone.db set res [catchcmd :memory: [string trim { +.mode batch CREATE TABLE t(id INTEGER PRIMARY KEY AUTOINCREMENT); INSERT INTO t VALUES (1),(2); .clone clone.db @@ -222,6 +232,7 @@ ifcapable vtab { # See overflow report at https://sqlite.org/forum/forumpost/5d34ce5280 do_test shell2-1.4.10 { set res [catchcmd :memory: [string trim { + .mode batch SELECT * FROM generate_series(9223372036854775807,9223372036854775807,1); SELECT * FROM generate_series(9223372036854775807,9223372036854775807,-1); SELECT avg(value),min(value),max(value) FROM generate_series( @@ -248,6 +259,65 @@ do_test shell2-1.4.10 { 0 1 2}} +do_test shell2-1.4.10b { + set res [catchcmd :memory: [string trim { + .mode tty +.print + SELECT * FROM generate_series(9223372036854775807,9223372036854775807,1); + SELECT * FROM generate_series(9223372036854775807,9223372036854775807,-1); + SELECT avg(value),min(value),max(value) FROM generate_series( + -9223372036854775808,9223372036854775807,1085102592571150095); + SELECT * FROM generate_series(-9223372036854775808,9223372036854775807, + 9223372036854775807); + SELECT value FROM generate_series(-4611686018427387904, + 4611686018427387904, 4611686018427387904) ORDER BY value DESC; + SELECT * FROM generate_series(0,-2,-1); + SELECT * FROM generate_series(0,-2); + SELECT * FROM generate_series(0,2) LIMIT 3;}]] +} {0 { +╭─────────────────────╮ +│ value │ +╞═════════════════════╡ +│ 9223372036854775807 │ +╰─────────────────────╯ +╭─────────────────────╮ +│ value │ +╞═════════════════════╡ +│ 9223372036854775807 │ +╰─────────────────────╯ +╭────────────┬──────────────────────┬─────────────────────╮ +│ avg(value) │ min(value) │ max(value) │ +╞════════════╪══════════════════════╪═════════════════════╡ +│ -0.5 │ -9223372036854775808 │ 9223372036854775807 │ +╰────────────┴──────────────────────┴─────────────────────╯ +╭──────────────────────╮ +│ value │ +╞══════════════════════╡ +│ -9223372036854775808 │ +│ -1 │ +│ 9223372036854775806 │ +╰──────────────────────╯ +╭──────────────────────╮ +│ value │ +╞══════════════════════╡ +│ 4611686018427387904 │ +│ 0 │ +│ -4611686018427387904 │ +╰──────────────────────╯ +╭───────╮ +│ value │ +╞═══════╡ +│ 0 │ +│ -1 │ +│ -2 │ +╰───────╯ +╭───────╮ +│ value │ +╞═══════╡ +│ 0 │ +│ 1 │ +│ 2 │ +╰───────╯}} } ;# ifcapable vtab ifcapable vtab { @@ -260,17 +330,17 @@ do_test shell2-1.4.11 { close $df set res [catchcmd :memory: [string trim { CREATE TABLE t(line text); -.mode ascii -.separator "\377" "\n" +.mode ascii -colsep "\377" -rowsep "\n" .import dummy.csv t SELECT count(*) FROM t;}]] -} {0 1} +} {1 {0 +Error: .import column separator must be ASCII}} } ;# ifcapable vtab # Bug from forum post 7cbe081746dd3803 # Keywords as column names were producing an error message. do_test shell2-1.4.12 { - set res [catchcmd :memory: [string trim { + set res [catchcmd :memory: [string trim {.mode batch CREATE TABLE "group"("order" text); INSERT INTO "group" VALUES ('ABC'); .sha3sum}]] diff --git a/test/shell4.test b/test/shell4.test index 3ced0702e..3614909c7 100644 --- a/test/shell4.test +++ b/test/shell4.test @@ -136,15 +136,15 @@ SELECT * FROM t1;} do_test shell4-3.1 { set fd [open t1.txt wb] - puts $fd "SELECT 'squirrel';" + puts $fd ".mode list\nSELECT 'squirrel';" close $fd - exec $::CLI_ONLY :memory: --interactive ".read t1.txt" + exec $::CLI_ONLY --noinit :memory: --interactive ".read t1.txt" } {squirrel} do_test_with_ansi_output shell4-3.2 { set fd [open t1.txt wb] - puts $fd "SELECT 'pound: \302\243';" + puts $fd ".mode list\nSELECT 'pound: \302\243';" close $fd - exec $::CLI_ONLY :memory: --interactive ".read t1.txt" + exec $::CLI_ONLY --noinit :memory: --interactive ".read t1.txt" } {pound: £} do_test shell4-4.1 { diff --git a/test/shell5.test b/test/shell5.test index 70a2298bc..559dc3ce7 100644 --- a/test/shell5.test +++ b/test/shell5.test @@ -33,14 +33,15 @@ forcedelete test.db test.db-journal test.db-wal # .import FILE TABLE Import data from FILE into TABLE do_test shell5-1.1.1 { catchcmd "test.db" ".import" -} {/1 .ERROR: missing FILE argument.*/} +} {/1 .line 1: Missing FILE argument.*/} do_test shell5-1.1.2 { catchcmd "test.db" ".import FOO" -} {/1 .ERROR: missing TABLE argument.*/} +} {/1 .line 1: Missing TABLE argument.*/} do_test shell5-1.1.3 { # too many arguments catchcmd "test.db" ".import FOO BAR BAD" -} {/1 .ERROR: extra argument.*/} +} {1 {line 1: .import FOO BAR BAD +line 1: ^--- unknown argument}} # .separator STRING Change separator used by output mode and .import do_test shell5-1.2.1 { @@ -59,13 +60,13 @@ do_test shell5-1.2.4 { # column separator should default to "|" do_test shell5-1.3.1.1 { - set res [catchcmd "test.db" ".show"] + set res [catchcmd "test.db" ".mode list\n.show"] list [regexp {colseparator: \"\|\"} $res] } {1} # row separator should default to "\n" do_test shell5-1.3.1.2 { - set res [catchcmd "test.db" ".show"] + set res [catchcmd "test.db" ".mode list\n.show"] list [regexp {rowseparator: \"\\n\"} $res] } {1} @@ -82,7 +83,7 @@ do_test shell5-1.4.1 { forcedelete FOO set res [catchcmd "test.db" {CREATE TABLE t1(a, b); .import FOO t1}] -} {1 {Error: cannot open "FOO"}} +} {1 {line 2: cannot open "FOO"}} # the remainder of these test cases require virtual tables. # @@ -97,7 +98,9 @@ do_test shell5-1.4.2 { forcedelete shell5.csv set in [open shell5.csv w] close $in - set res [catchcmd ":memory:" {ATTACH 'test.db' AS test; + set res [catchcmd ":memory:" { +.mode list +ATTACH 'test.db' AS test; .import -schema test shell5.csv t1 SELECT COUNT(*) FROM test.t1;}] } {0 0} @@ -116,7 +119,7 @@ do_test shell5-1.4.4 { set in [open shell5.csv w] puts $in "1|2|3" close $in - set res [catchcmd ":memory:" {ATTACH 'test.db' AS test; + set res [catchcmd ":memory: --list" {ATTACH 'test.db' AS test; .import --schema test shell5.csv t1}] } {1 {shell5.csv:1: expected 2 columns but found 3 - extras ignored}} @@ -125,7 +128,7 @@ do_test shell5-1.4.5 { set in [open shell5.csv w] puts $in "1|2" close $in - set res [catchcmd "test.db" {DELETE FROM t1; + set res [catchcmd "test.db -list" {DELETE FROM t1; .import shell5.csv t1 SELECT COUNT(*) FROM t1;}] } {0 1} @@ -138,7 +141,9 @@ do_test shell5-1.4.6 { puts $in "2|3" puts $in "3|4" close $in - set res [catchcmd ":memory:" {ATTACH 'test.db' AS test; + set res [catchcmd ":memory:" { +.mode list +ATTACH 'test.db' AS test; .import -schema test shell5.csv t1 SELECT COUNT(*) FROM test.t1;}] } {0 3} @@ -148,7 +153,9 @@ do_test shell5-1.4.7 { set in [open shell5.csv w] puts $in "4,5" close $in - set res [catchcmd ":memory:" {ATTACH 'test.db' AS test; + set res [catchcmd ":memory:" { +.mode list +ATTACH 'test.db' AS test; .separator , .import --schema test shell5.csv t1 SELECT COUNT(*) FROM test.t1;}] @@ -159,12 +166,13 @@ do_test shell5-1.4.8.1 { set in [open shell5.csv w] puts $in "5|Now is the time for all good men to come to the aid of their country." close $in - set res [catchcmd "test.db" {.import shell5.csv t1 + set res [catchcmd "test.db" {.mode list +.import shell5.csv t1 SELECT COUNT(*) FROM t1;}] } {0 5} do_test shell5-1.4.8.2 { - catchcmd "test.db" {SELECT b FROM t1 WHERE a='5';} + catchcmd "test.db -list" {SELECT b FROM t1 WHERE a='5';} } {0 {Now is the time for all good men to come to the aid of their country.}} # import file with 1 row, 2 columns, quoted text data @@ -174,12 +182,12 @@ do_test shell5-1.4.9.1 { set in [open shell5.csv w] puts $in "6|'Now is the time for all good men to come to the aid of their country.'" close $in - set res [catchcmd "test.db" {.import shell5.csv t1 + set res [catchcmd "test.db -list" {.import shell5.csv t1 SELECT COUNT(*) FROM t1;}] } {0 6} do_test shell5-1.4.9.2 { - catchcmd "test.db" {SELECT b FROM t1 WHERE a='6';} + catchcmd "test.db -list" {SELECT b FROM t1 WHERE a='6';} } {0 {'Now is the time for all good men to come to the aid of their country.'}} # import file with 1 row, 2 columns, quoted text data @@ -187,12 +195,12 @@ do_test shell5-1.4.10.1 { set in [open shell5.csv w] puts $in "7|\"Now is the time for all good men to come to the aid of their country.\"" close $in - set res [catchcmd "test.db" {.import shell5.csv t1 + set res [catchcmd "test.db -list" {.import shell5.csv t1 SELECT COUNT(*) FROM t1;}] } {0 7} do_test shell5-1.4.10.2 { - catchcmd "test.db" {SELECT b FROM t1 WHERE a='7';} + catchcmd "test.db -list" {SELECT b FROM t1 WHERE a='7';} } {0 {Now is the time for all good men to come to the aid of their country.}} # import file with 2 rows, 2 columns and an initial BOM @@ -203,7 +211,7 @@ do_test shell5-1.4.11 { puts $in "2|3" puts $in "4|5" close $in - set res [catchcmd "test.db" {CREATE TABLE t2(x INT, y INT); + set res [catchcmd "test.db -list" {CREATE TABLE t2(x INT, y INT); .import shell5.csv t2 .mode quote .header on @@ -218,7 +226,7 @@ do_test shell5-1.4.12 { puts $in "\xef\xbb\xbf\"two\"|3" puts $in "4|5" close $in - set res [catchcmd "test.db" {DELETE FROM t2; + set res [catchcmd "test.db -list" {DELETE FROM t2; .import shell5.csv t2 .mode quote .header on @@ -232,7 +240,7 @@ do_test shell5-1.5.1 { set in [open shell5.csv w] puts $in "8|$str" close $in - set res [catchcmd "test.db" {.import shell5.csv t1 + set res [catchcmd "test.db -list" {.import shell5.csv t1 SELECT length(b) FROM t1 WHERE a='8';}] } {0 999} @@ -252,7 +260,7 @@ do_test shell5-1.6.1 { set in [open shell5.csv w] puts $in $data close $in - set res [catchcmd "test.db" {DROP TABLE IF EXISTS t2; + set res [catchcmd "test.db -list" {DROP TABLE IF EXISTS t2; .import shell5.csv t2 SELECT COUNT(*) FROM t2;}] } {0 1} @@ -268,6 +276,7 @@ do_test shell5-1.7.1 { close $in set res [catchcmd "test.db" {.mode csv .import shell5.csv t3 +.mode quote SELECT COUNT(*) FROM t3;}] } [list 0 $rows] @@ -329,6 +338,24 @@ do_test shell5-1.10 { db eval {SELECT hex(c) FROM t1 ORDER BY rowid} } {636F6C756D6E33 783320220D0A64617461222033 783320220A64617461222033} +# The --escape option +# +do_test shell5-1.10.1 { + set out [open shell5.csv w] + fconfigure $out -translation lf + puts $out {column1,column2,column3,column4} + puts $out "x1,x2%\"x3,\"x3\\\"data\\\"3\",x4" + close $out + db close + forcedelete test.db + catchcmd test.db { + CREATE TABLE t1(a,b,c,d); +.import --csv --qesc \\ --esc % shell5.csv t1 + } + sqlite3 db test.db + db eval {SELECT b, c FROM t1 ORDER BY rowid} +} {column2 column3 x2\"x3 x3\"data\"3} + # Blank last column with \r\n line endings. do_test shell5-1.11 { set out [open shell5.csv w] @@ -495,9 +522,10 @@ do_test shell5-4.4 { CREATE TEMP TABLE t8(a, b, c); .import shell5.csv t8 .nullvalue # +.mode quote SELECT * FROM temp.t8 }] -} {0 1,2,3} +} {0 '1','2','3'} #---------------------------------------------------------------------------- # Tests for the shell automatic column rename. @@ -513,7 +541,7 @@ do_test shell5-5.1 { close $out forcedelete test.db catchcmd test.db {.import -csv shell5.csv t1 -.mode line +.mode line --colsep ' = ' SELECT * FROM t1;} } {1 { ? = 0 x_02 = x2 @@ -529,7 +557,7 @@ Columns renamed during .import shell5.csv due to duplicates: "z" to "z_05", "z" to "z_08"}} -do_test shell5-5.1 { +do_test shell5-5.1b { set out [open shell5.csv w] fconfigure $out -translation lf puts $out {"COW","cow","CoW","cOw"} @@ -539,10 +567,10 @@ do_test shell5-5.1 { catchcmd test.db {.import -csv shell5.csv t1 .mode line SELECT * FROM t1;} -} {1 {COW_1 = uuu -cow_2 = lll -CoW_3 = ulu -cOw_4 = lul +} {1 {COW_1: uuu +cow_2: lll +CoW_3: ulu +cOw_4: lul Columns renamed during .import shell5.csv due to duplicates: "COW" to "COW_1", "cow" to "cow_2", @@ -561,7 +589,7 @@ do_test_with_ansi_output shell5-6.1 { close $out forcedelete test.db catchcmd test.db {.import -csv shell5.csv t1 -.mode line +.mode line --colsep " = " SELECT * FROM t1;} } {0 { あい = 1 うえお = 2}} @@ -576,8 +604,8 @@ do_test_with_ansi_output shell5-6.2 { catchcmd test.db {.import -csv shell5.csv t1 .mode line SELECT * FROM t1;} -} {0 { 1 = あい - 2 = うえお}} +} {0 {1: あい +2: うえお}} # 2024-03-11 https://sqlite.org/forum/forumpost/ca014d7358 # Import into a table that contains computed columns. @@ -588,7 +616,7 @@ do_test shell5-7.1 { puts $out {aaa|bbb} close $out forcedelete test.db - catchcmd :memory: {CREATE TABLE t1(a TEXT, b TEXT, c AS (a||b)); + catchcmd ":memory: -list" {CREATE TABLE t1(a TEXT, b TEXT, c AS (a||b)); .import shell5.csv t1 SELECT * FROM t1;} } {0 aaa|bbb|aaabbb} @@ -605,4 +633,36 @@ do_test shell5-8.1 { catchcmd :memory: {.import --csv shell5.csv '""""""""""""""""""""""""""""""""""""""""""""""'} } {0 {}} + +# 2025-12-29 https://sqlite.org/forum/forumpost/6c1c0e213d +# .import honor .bail +# +do_test shell5-9.1 { + catchcmd ":memory:" { + CREATE TABLE t1(a,b,c INT CHECK(c<>5)); +.bail on +.import -csv <<END t1 +1,2,3 +"a","b","c" +3,4,5 +"q","r","s" +END +SELECT * FROM t1;} +} {1 {<stdin>:7: INSERT failed: CHECK constraint failed: c<>5}} +do_test shell5-9.2 { + catchcmd ":memory:" { + CREATE TABLE t1(a,b,c INT CHECK(c<>5)); +.bail off +.import -csv <<END t1 +1,2,3 +"a","b","c" +3,4,5 +"q","r","s" +END +SELECT * FROM t1;} +} {1 {1|2|3 +a|b|c +q|r|s +<stdin>:7: INSERT failed: CHECK constraint failed: c<>5}} + finish_test diff --git a/test/shell8.test b/test/shell8.test index e55539636..40579a599 100644 --- a/test/shell8.test +++ b/test/shell8.test @@ -217,6 +217,46 @@ if {$tcl_platform(platform)=="unix"} { do_test 3.3 { catchcmd shell8.db {.ar -x} } {0 {}} + + # Test defenses against using symlinks to write outside + # of the destination directory. See forum thread at + # sqlite.org/forum/forumpost/2026-02-21T11:04:36z + # + forcedelete shell8.db + forcedelete ar1 + forcedelete ar2 + forcedelete ar3 + file mkdir ar2 + file mkdir ar3 + set pwd [pwd] + sqlite3 db shell8.db + db eval { + CREATE TABLE sqlar( + name TEXT PRIMARY KEY, -- name of the file + mode INT, -- access permissions + mtime INT, -- last modification time + sz INT, -- original file size + data BLOB -- compressed content + ); + INSERT INTO sqlar VALUES + ('abc',33188,0,-1,'content for abc'), + ('escape',40960,0,-1,$pwd||'/ar3'), + ('escape/def',33188,0,-1,'content for escape/def'), + ('ghi',33188,0,-1,'content for ghi'); + } + do_test 3.4.1 { + catchcmd shell8.db {.ar -x --directory ar2} + lsort [glob -tails -directory ar2 -nocomplain *] + } {abc escape ghi} + do_test 3.4.2 { + lsort [glob -tails -directory ar3 -nocomplain *] + } {} + # ^^--- An extraction into ar2 should not leak any files into ar3 + + forcedelete shell8.db + forcedelete ar2 + forcedelete ar3 + } finish_test diff --git a/test/shellA.test b/test/shellA.test index f3959d428..3b28c921c 100644 --- a/test/shellA.test +++ b/test/shellA.test @@ -36,11 +36,11 @@ do_execsql_test shellA-1.0 { # and that our calls to the CLI are working. # do_test_with_ansi_output shellA-1.2 { - exec {*}$CLI test.db {.mode box --escape symbol} {SELECT * FROM t1;} + exec {*}$CLI -noinit test.db {.mode box -quote off --escape symbol} {SELECT * FROM t1;} } { -┌───┬──────────────────────────┐ +╭───┬──────────────────────────╮ │ a │ x │ -├───┼──────────────────────────┤ +╞═══╪══════════════════════════╡ │ 1 │ line with ' single quote │ ├───┼──────────────────────────┤ │ 2 │ ␛[31mVT-100 codes␛[0m │ @@ -57,39 +57,39 @@ do_test_with_ansi_output shellA-1.2 { │ 7 │ carriage␍return │ ├───┼──────────────────────────┤ │ 8 │ last line │ -└───┴──────────────────────────┘ +╰───┴──────────────────────────╯ } # ".mode list" # do_test shellA-1.3 { - exec {*}$CLI test.db {SELECT x FROM t1 WHERE a=2;} + exec {*}$CLI -noinit -list test.db {SELECT x FROM t1 WHERE a=2;} } { ^[[31mVT-100 codes^[[0m } do_test_with_ansi_output shellA-1.4 { - exec {*}$CLI test.db --escape symbol {SELECT x FROM t1 WHERE a=2;} + exec {*}$CLI -noinit -list test.db --escape symbol {SELECT x FROM t1 WHERE a=2;} } { ␛[31mVT-100 codes␛[0m } do_test shellA-1.5 { - exec {*}$CLI test.db --escape ascii {SELECT x FROM t1 WHERE a=2;} + exec {*}$CLI -noinit -list test.db --escape ascii {SELECT x FROM t1 WHERE a=2;} } { ^[[31mVT-100 codes^[[0m } do_test_with_ansi_output shellA-1.6 { - exec {*}$CLI test.db {.mode list --escape symbol} {SELECT x FROM t1 WHERE a=2;} + exec {*}$CLI -noinit test.db {.mode list --escape symbol} {SELECT x FROM t1 WHERE a=2;} } { ␛[31mVT-100 codes␛[0m } do_test shellA-1.7 { - exec {*}$CLI test.db {.mode list --escape ascii} {SELECT x FROM t1 WHERE a=2;} + exec {*}$CLI -noinit test.db {.mode list --escape ascii} {SELECT x FROM t1 WHERE a=2;} } { ^[[31mVT-100 codes^[[0m } do_test shellA-1.8 { file delete -force out.txt - exec {*}$CLI test.db {.mode list --escape off} {SELECT x FROM t1 WHERE a=7;} \ + exec {*}$CLI -noinit test.db {.mode list --escape off} {SELECT x FROM t1 WHERE a=7;} \ >out.txt set fd [open out.txt rb] set res [read $fd] @@ -98,92 +98,93 @@ do_test shellA-1.8 { } "carriage\rreturn" do_test shellA-1.9 { set rc [catch { - exec {*}$CLI test.db {.mode test --escape xyz} + exec {*}$CLI -noinit test.db {.mode test --escape xyz} } msg] lappend rc $msg -} {1 {unknown control character escape mode "xyz" - choices: ascii symbol off}} +} {1 {argv[3]: .mode test --escape xyz +argv[3]: ^--- unknown mode +argv[3]: Use ".help .mode" for more info}} do_test shellA-1.10 { set rc [catch { - exec {*}$CLI --escape abc test.db .q + exec {*}$CLI --noinit --escape abc test.db .q } msg] lappend rc $msg -} {1 {unknown control character escape mode "abc" - choices: ascii symbol off}} +} {1 {unknown control character escape mode "abc" - choices: auto off ascii symbol}} # ".mode quote" # do_test shellA-2.1 { - exec {*}$CLI test.db --quote {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} + exec {*}$CLI -noinit test.db --quote {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} } { 1,'line with '' single quote' 2,unistr('\u001b[31mVT-100 codes\u001b[0m') -6,'new -line' +6,unistr('new\u000aline') 7,unistr('carriage\u000dreturn') 8,'last line' } do_test shellA-2.2 { - exec {*}$CLI test.db --quote {.mode} -} {current output mode: quote --escape ascii} + exec {*}$CLI -noinit test.db --quote {.mode -v} +} {/*.mode quote* --escape auto*/} do_test shellA-2.3 { - exec {*}$CLI test.db --quote --escape SYMBOL {.mode} -} {current output mode: quote --escape symbol} + exec {*}$CLI -noinit test.db --quote --escape SYMBOL {.mode} +} {.mode quote --escape symbol} do_test shellA-2.4 { - exec {*}$CLI test.db --quote --escape OFF {.mode} -} {current output mode: quote --escape off} + exec {*}$CLI -noinit test.db --quote --escape OFF {.mode} +} {.mode quote --escape off} # ".mode line" # do_test_with_ansi_output shellA-3.1 { - exec {*}$CLI test.db --line --escape symbol \ + exec {*}$CLI -noinit test.db --line --escape symbol \ {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} } { - a = 1 - x = line with ' single quote + a: 1 + x: line with ' single quote - a = 2 - x = ␛[31mVT-100 codes␛[0m + a: 2 + x: ␛[31mVT-100 codes␛[0m - a = 6 - x = new -line + a: 6 + x: new + line - a = 7 - x = carriage␍return + a: 7 + x: carriage␍return - a = 8 - x = last line + a: 8 + x: last line } do_test shellA-3.2 { - exec {*}$CLI test.db --line --escape ascii \ + exec {*}$CLI -noinit test.db --line --escape ascii \ {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} } { - a = 1 - x = line with ' single quote + a: 1 + x: line with ' single quote - a = 2 - x = ^[[31mVT-100 codes^[[0m + a: 2 + x: ^[[31mVT-100 codes^[[0m - a = 6 - x = new -line + a: 6 + x: new + line - a = 7 - x = carriage^Mreturn + a: 7 + x: carriage^Mreturn - a = 8 - x = last line + a: 8 + x: last line } # ".mode box" # do_test_with_ansi_output shellA-4.1 { - exec {*}$CLI test.db --box --escape ascii \ + exec {*}$CLI -noinit test.db --box --escape ascii \ {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} } { -┌───┬──────────────────────────┐ +╭───┬──────────────────────────╮ │ a │ x │ -├───┼──────────────────────────┤ +╞═══╪══════════════════════════╡ │ 1 │ line with ' single quote │ ├───┼──────────────────────────┤ │ 2 │ ^[[31mVT-100 codes^[[0m │ @@ -194,31 +195,56 @@ do_test_with_ansi_output shellA-4.1 { │ 7 │ carriage^Mreturn │ ├───┼──────────────────────────┤ │ 8 │ last line │ -└───┴──────────────────────────┘ +╰───┴──────────────────────────╯ +} +do_test_with_ansi_output shellA-4.1b { + exec {*}$CLI -noinit test.db --box --escape ascii \ + {.mode -border off} \ + {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} +} { + a │ x +═══╪══════════════════════════ + 1 │ line with ' single quote +───┼────────────────────────── + 2 │ ^[[31mVT-100 codes^[[0m +───┼────────────────────────── + 6 │ new + │ line +───┼────────────────────────── + 7 │ carriage^Mreturn +───┼────────────────────────── + 8 │ last line } do_test_with_ansi_output shellA-4.2 { - exec {*}$CLI test.db {.mode qbox} {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} + exec {*}$CLI -noinit test.db {.mode qbox} {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} } { -┌───┬───────────────────────────────────────────┐ +╭───┬───────────────────────────────────────────╮ │ a │ x │ -├───┼───────────────────────────────────────────┤ +╞═══╪═══════════════════════════════════════════╡ │ 1 │ 'line with '' single quote' │ -├───┼───────────────────────────────────────────┤ │ 2 │ unistr('\u001b[31mVT-100 codes\u001b[0m') │ -├───┼───────────────────────────────────────────┤ -│ 6 │ 'new │ -│ │ line' │ -├───┼───────────────────────────────────────────┤ +│ 6 │ unistr('new\u000aline') │ │ 7 │ unistr('carriage\u000dreturn') │ -├───┼───────────────────────────────────────────┤ │ 8 │ 'last line' │ -└───┴───────────────────────────────────────────┘ +╰───┴───────────────────────────────────────────╯ +} +do_test_with_ansi_output shellA-4.2b { + exec {*}$CLI -noinit test.db {.mode qbox -border off} \ + {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} +} { + a │ x +═══╪═══════════════════════════════════════════ + 1 │ 'line with '' single quote' + 2 │ unistr('\u001b[31mVT-100 codes\u001b[0m') + 6 │ unistr('new\u000aline') + 7 │ unistr('carriage\u000dreturn') + 8 │ 'last line' } # ".mode insert" # do_test shellA-5.1 { - exec {*}$CLI test.db {.mode insert t1 --escape ascii} \ + exec {*}$CLI -noinit test.db {.mode insert t1 --escape ascii} \ {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} } { INSERT INTO t1 VALUES(1,'line with '' single quote'); @@ -228,7 +254,7 @@ INSERT INTO t1 VALUES(7,unistr('carriage\u000dreturn')); INSERT INTO t1 VALUES(8,'last line'); } do_test shellA-5.2 { - exec {*}$CLI test.db {.mode insert t1 --escape symbol} \ + exec {*}$CLI -noinit test.db {.mode insert t1 --escape symbol} \ {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} } { INSERT INTO t1 VALUES(1,'line with '' single quote'); @@ -239,7 +265,7 @@ INSERT INTO t1 VALUES(8,'last line'); } do_test shellA-5.3 { file delete -force out.txt - exec {*}$CLI test.db {.mode insert t1 --escape off} \ + exec {*}$CLI -noinit test.db {.mode insert t1 --escape off} \ {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} >out.txt set fd [open out.txt rb] set res [read $fd] @@ -254,4 +280,83 @@ INSERT INTO t1 VALUES(7,'carriage\rreturn'); INSERT INTO t1 VALUES(8,'last line'); " +# ".mode split" +# +do_test shellA-6.1 { + db eval { + CREATE TABLE t2(x); + INSERT INTO t2(x) VALUES + ('one'), ('two'), ('three'), ('four'), ('five'), + ('six'), ('seven'), ('eight'), ('nine'), ('ten'), + ('eleven'), ('twelve'), ('thirteen'), ('fourteen'); + } + exec {*}$CLI -noinit test.db \ + {.print} \ + {.mode split -screenwidth 30} \ + {SELECT x FROM t2} +} { +one five nine thirteen +two six ten fourteen +three seven eleven +four eight twelve} +# 3456789 123456789 123456789 + +do_test shellA-6.2 { + exec {*}$CLI -noinit test.db \ + {.print} \ + {.mode split -screenwidth 30} \ + {SELECT x FROM t2} \ + {.mode column -titles off} \ + {SELECT x FROM t2} +} { +one five nine thirteen +two six ten fourteen +three seven eleven +four eight twelve +one +two +three +four +five +six +seven +eight +nine +ten +eleven +twelve +thirteen +fourteen} + +do_test shellA-6.3 { + exec {*}$CLI -noinit test.db \ + {.print} \ + {.mode table} \ + {.mode --once split -screenwidth 30} \ + {SELECT x FROM t2} \ + {SELECT x FROM t2} +} { +one five nine thirteen +two six ten fourteen +three seven eleven +four eight twelve ++----------+ +| x | ++----------+ +| one | +| two | +| three | +| four | +| five | +| six | +| seven | +| eight | +| nine | +| ten | +| eleven | +| twelve | +| thirteen | +| fourteen | ++----------+} + finish_test diff --git a/test/shellB.test b/test/shellB.test new file mode 100644 index 000000000..d98a77cb5 --- /dev/null +++ b/test/shellB.test @@ -0,0 +1,53 @@ +# 2025-11-12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# TESTRUNNER: shell +# +# Test cases for the command-line shell using the newly renovated +# ".testcase" and ".check" commands. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set CLI [test_cli_invocation] + +# Run an instance of the CLI on the file $name. +# Capture the number of test cases and the number of +# errors and increment the counts. +# +proc do_clitest {name} { + set mapping [list <NAME> $::testdir/$name <CLI> $::CLI] + set script [string map $mapping { + catch {exec <CLI> :memory: ".read <NAME>" 2>@stdout} res + set ntest 0 + set nerr 999 + regexp {(\d+) tests? run with (\d+) errors?} $res all ntest nerr + set_test_counter count [expr {[set_test_counter count]+$ntest-1}] + set_test_counter errors [expr {[set_test_counter errors]+$nerr}] + if {$nerr==0} {set res "error count: 0"} + set res + }] + # puts $script + do_test shellB-$name $script {error count: 0} +} + +do_clitest modeA.sql +do_clitest dblwidth-a.sql +do_clitest vt100-a.sql +do_clitest regexp1.sql +do_clitest imposter1.sql +do_clitest dotcmd01.sql +ifcapable vtab { + do_clitest import01.sql + do_clitest intck01.sql +} +do_clitest fptest01.sql + +finish_test diff --git a/test/speedtest.md b/test/speedtest.md index 135e562ae..98cba93ff 100644 --- a/test/speedtest.md +++ b/test/speedtest.md @@ -8,8 +8,9 @@ You will need: * valgrind * tclsh * A script or program named "open" that brings up *.txt files in an - editor for viewing. (Macs provide this by default. You'll need to - come up with your own on Linux and Windows.) + editor for viewing. (Macs provide this by default. On Linux it's + called xdg-open and some distributions symlink it to "open". You'll + need to come up with your own on Windows.) * An SQLite source tree The procedure described in this document is not the only way to make diff --git a/test/speedtest.tcl b/test/speedtest.tcl index 9cb81c0fc..26bc27530 100755 --- a/test/speedtest.tcl +++ b/test/speedtest.tcl @@ -27,7 +27,7 @@ Other options include: --lookaside N SZ Lookahead uses N slots of SZ bytes each. --osmalloc Use the OS native malloc() instead of MEMSYS5 --pagesize N Use N as the page size. - --quiet | -q "Quite". Put results in file but don't pop up editor + --quiet | -q "Quiet". Put results in file but don't pop up editor --size N Change the test size. 100 means 100%. Default: 5. --testset TEST Specify the specific testset to use. The default is "mix1". Other options include: "main", "json", diff --git a/test/tabfunc01.test b/test/tabfunc01.test index 9a2017c46..60f546ce4 100644 --- a/test/tabfunc01.test +++ b/test/tabfunc01.test @@ -668,6 +668,20 @@ do_execsql_test 1370 { SELECT * FROM generate_series(0,0,0); } {} +reset_db +load_static_extension db series +do_execsql_test 1400 { + CREATE TABLE t1(x); + CREATE TABLE t2(y); +} +do_catchsql_test 1410 { + SELECT x, y, value + FROM (t1 RIGHT JOIN generate_series(t2.y,5) AS value) JOIN t2; +} {1 {table-function argument references tables to its right}} +do_catchsql_test 1420 { + SELECT x, y, value + FROM t2 JOIN (t1 RIGHT JOIN generate_series(t2.y,5) AS value) +} {1 {no such column: t2.y}} # Free up memory allocations diff --git a/test/tclsqlite.test b/test/tclsqlite.test index 5f373ea18..6cababad3 100644 --- a/test/tclsqlite.test +++ b/test/tclsqlite.test @@ -42,7 +42,7 @@ do_test tcl-1.1.1 { do_test tcl-1.2 { set v [catch {db bogus} msg] lappend v $msg -} {1 {bad option "bogus": must be authorizer, backup, bind_fallback, busy, cache, changes, close, collate, collation_needed, commit_hook, complete, config, copy, deserialize, enable_load_extension, errorcode, erroroffset, eval, exists, function, incrblob, interrupt, last_insert_rowid, nullvalue, onecolumn, preupdate, profile, progress, rekey, restore, rollback_hook, serialize, status, timeout, total_changes, trace, trace_v2, transaction, unlock_notify, update_hook, version, or wal_hook}} +} {1 {bad option "bogus": must be authorizer, backup, bind_fallback, busy, cache, changes, close, collate, collation_needed, commit_hook, complete, config, copy, deserialize, enable_load_extension, errorcode, erroroffset, eval, exists, format, function, incrblob, interrupt, last_insert_rowid, nullvalue, onecolumn, preupdate, profile, progress, rekey, restore, rollback_hook, serialize, status, timeout, total_changes, trace, trace_v2, transaction, unlock_notify, update_hook, version, or wal_hook}} do_test tcl-1.2.1 { set v [catch {db cache bogus} msg] lappend v $msg diff --git a/test/temptrigfault.tes b/test/temptrigfault.tes new file mode 100644 index 000000000..4b124e1f7 --- /dev/null +++ b/test/temptrigfault.tes @@ -0,0 +1,120 @@ +# 2025 November 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix temptrigfault + +forcedelete test.db2 +do_execsql_test 1.0 { + CREATE TABLE t1(x, y); + ATTACH 'test.db2' AS aux; + CREATE TABLE aux.t1(x, y); +} + +do_faultsim_test 1.1 -faults oom* -prep { +} -body { + execsql { + CREATE TEMP TRIGGER tmptrig AFTER INSERT ON t1 BEGIN + INSERT INTO aux.t1 VALUES(new.x, new.y); + END; + } +} -test { + faultsim_test_result {0 {}} + catchsql { DROP TRIGGER tmptrig } +} + +do_execsql_test 2.0 { + CREATE TEMP TRIGGER tmptrig AFTER INSERT ON t1 BEGIN + INSERT INTO aux.t1 VALUES(new.x, new.y); + END; +} + +do_faultsim_test 2 -faults oom* -prep { +} -body { + execsql { + INSERT INTO t1 VALUES('x', 'y'); + } +} -test { + faultsim_test_result {0 {}} +} + +do_execsql_test 3.0.1 { + SELECT * FROM t1; + DELETE FROM t1; + DELETE FROM aux.t1; +} [db eval {SELECT * FROM aux.t1}] + +do_execsql_test 3.0.2 { + CREATE TEMP TRIGGER tmptrig2 AFTER INSERT ON aux.t1 BEGIN + INSERT INTO t1 VALUES(new.x||'2', new.y||'2'); + END; +} + +do_faultsim_test 3 -faults oom* -prep { +} -body { + execsql { + INSERT INTO aux.t1 VALUES('aaa', 'bbb'); + } +} -test { + faultsim_test_result {0 {}} +} + +proc repeatlist {list n} { + set ret [list] + for {set i 0} {$i < $n} {incr i} { + set ret [concat $ret $list] + } + set ret +} + +do_execsql_test 3.x.1 { + SELECT * FROM main.t1; +} [repeatlist {aaa2 bbb2} 5] + +do_execsql_test 3.x.2 { + SELECT * FROM aux.t1; +} [repeatlist {aaa bbb aaa2 bbb2} 5] + +faultsim_save_and_close +do_faultsim_test 4 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { ATTACH 'test.db2' AS aux; } +} -body { + execsql { + CREATE TEMP TRIGGER xyz AFTER DELETE ON main.t1 BEGIN + DELETE FROM aux.t1 WHERE rowid=old.rowid; + END; + + DELETE FROM t1 WHERE rowid=2; + } +} -test { + faultsim_test_result {0 {}} {1 {unable to open a temporary database file for storing temporary tables}} +} + +faultsim_save_and_close +do_faultsim_test 5 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { ATTACH 'test.db2' AS aux; } +} -body { + execsql { + CREATE TEMP TRIGGER xyz AFTER UPDATE ON aux.t1 BEGIN + UPDATE main.t1 SET x=new.x, y=new.y WHERE rowid=new.rowid; + END; + UPDATE aux.t1 SET x=x||x WHERE rowid=1+abs(random() % 5); + } +} -test { + faultsim_test_result {0 {}} {1 {unable to open a temporary database file for storing temporary tables}} +} + + +finish_test diff --git a/test/temptrigger.test b/test/temptrigger.test index e4277adf6..74390f3b8 100644 --- a/test/temptrigger.test +++ b/test/temptrigger.test @@ -276,4 +276,191 @@ do_catchsql_test 6.3 { } {1 error} db2 close +#------------------------------------------------------------------------- +reset_db +forcedelete test.db2 + +do_execsql_test 7.0 { + CREATE TABLE m1(a, b); + ATTACH 'test.db2' AS aux; + CREATE TABLE aux.a1(c, d); +} + +do_execsql_test 7.1 { + CREATE TEMP TRIGGER tr1 AFTER INSERT ON m1 BEGIN + INSERT INTO a1 VALUES(new.a, new.b); + END; + + INSERT INTO m1 VALUES(5, 6); + SELECT * FROM aux.a1; +} {5 6} + +do_execsql_test 7.2 { + CREATE TABLE a1(e, f); + INSERT INTO m1 VALUES(7, 8); +} + +do_execsql_test 7.3.1 { SELECT * FROM main.a1 } {7 8} +do_execsql_test 7.3.2 { SELECT * FROM aux.a1 } {5 6} + +do_execsql_test 7.4 { + DROP TRIGGER tr1; + CREATE TEMP TRIGGER tr1 AFTER INSERT ON m1 BEGIN + INSERT INTO a1 SELECT d, c FROM aux.a1; + END; + + DELETE FROM aux.a1; + DELETE FROM main.a1; + INSERT INTO aux.a1 VALUES('hello', 'world'); +} + +do_execsql_test 7.5 { + INSERT INTO m1 VALUES(9, 10); + SELECT * FROM main.a1; +} {world hello} + +do_catchsql_test 7.6 { + DROP TRIGGER tr1; + CREATE TRIGGER tr1 AFTER INSERT ON m1 BEGIN + INSERT INTO a1 SELECT d, c FROM aux.a1; + END; +} {1 {trigger tr1 cannot reference objects in database aux}} + +#------------------------------------------------------------------------- +# Check that temp triggers may INSERT/UPDATE/DELETE to fully qualified +# table names. +reset_db +forcedelete {*}[glob -nocomplain *mj*] +forcedelete test.db2 +do_execsql_test 8.0 { + ATTACH 'test.db2' AS aux; + CREATE TABLE t1(a, b); + CREATE TABLE t2(c, d); + CREATE TABLE aux.t1(e, f); + CREATE TABLE aux.t2(g, h); +} + +do_catchsql_test 8.1.1 { + CREATE TRIGGER tr1 AFTER INSERT ON t2 BEGIN + INSERT INTO aux.t1 VALUES(new.c, new.d); + END; +} {1 {qualified table names are not allowed on INSERT, UPDATE, and DELETE statements within triggers}} + +do_execsql_test 8.1.2 { + CREATE TEMP TRIGGER tr1 AFTER INSERT ON t2 BEGIN + INSERT INTO aux.t1 VALUES(new.c, new.d); + END; + + INSERT INTO main.t2 VALUES('x', 'y'); + SELECT * FROM aux.t1; +} {x y} + +do_execsql_test 8.1.3 { SELECT * FROM t1 } {} + +do_catchsql_test 8.2.1 { + CREATE TRIGGER aux.tr2 AFTER UPDATE ON aux.t1 BEGIN + UPDATE main.t2 SET c=new.e, d=new.f; + END; +} {1 {qualified table names are not allowed on INSERT, UPDATE, and DELETE statements within triggers}} + +do_execsql_test 8.2.2 { + CREATE TEMP TRIGGER tr2 AFTER UPDATE ON aux.t1 BEGIN + UPDATE main.t2 SET c=new.e, d=new.f; + END; + + UPDATE aux.t1 SET e=1, f=2; + SELECT * FROM t2; +} {1 2} + +do_execsql_test 8.2.3 { SELECT * FROM aux.t2 } {} + +do_catchsql_test 8.3.1 { + CREATE TRIGGER tr3 AFTER DELETE ON t2 BEGIN + DELETE FROM aux.t1; + END; +} {1 {qualified table names are not allowed on INSERT, UPDATE, and DELETE statements within triggers}} + +do_execsql_test 8.3.2 { + INSERT INTO main.t1 VALUES('a', 'b'); + CREATE TEMP TRIGGER tr3 AFTER DELETE ON t2 BEGIN + DELETE FROM aux.t1; + END; + + DELETE FROM main.t2; + SELECT * FROM aux.t1; +} {} + +do_execsql_test 8.3.3 { SELECT * FROM t1 } {a b} + +#------------------------------------------------------------------------- +reset_db +set nDb 8 +do_test 9.0 { + for {set ii 0} {$ii < $nDb} {incr ii} { + db eval "ATTACH ':memory:' AS db$ii" + db eval "CREATE TABLE db$ii.tbl(a, b, c)" + } + + for {set ii 0} {$ii < ($nDb-1)} {incr ii} { + set jj [expr $ii+1] + db eval " + CREATE TEMP TRIGGER tr$ii AFTER INSERT ON db$ii.tbl BEGIN + INSERT INTO db$jj.tbl VALUES(new.b, new.c, new.a); + END; + " + } +} {} + +do_execsql_test 9.1 { INSERT INTO db0.tbl VALUES('a', 'b', 'c'); } +do_execsql_test 9.1.1 { SELECT * FROM db0.tbl } {a b c} +do_execsql_test 9.1.2 { SELECT * FROM db1.tbl } {b c a} +do_execsql_test 9.1.3 { SELECT * FROM db2.tbl } {c a b} +do_execsql_test 9.1.1 { SELECT * FROM db3.tbl } {a b c} +do_execsql_test 9.1.2 { SELECT * FROM db4.tbl } {b c a} +do_execsql_test 9.1.3 { SELECT * FROM db5.tbl } {c a b} +do_execsql_test 9.1.1 { SELECT * FROM db6.tbl } {a b c} +do_execsql_test 9.1.2 { SELECT * FROM db7.tbl } {b c a} + +do_test 9.2 { + for {set ii 0} {$ii < ($nDb-1)} {incr ii} { + set jj [expr $ii+1] + db eval " + CREATE TEMP TRIGGER tru$ii AFTER UPDATE ON db$ii.tbl BEGIN + UPDATE db$jj.tbl SET a=new.b, b=new.c, c=new.a; + END; + " + } +} {} + +do_execsql_test 9.3 { UPDATE db0.tbl SET a=1, b=2, c=3 } +do_execsql_test 9.3.1 { SELECT * FROM db0.tbl } {1 2 3} +do_execsql_test 9.3.2 { SELECT * FROM db1.tbl } {2 3 1} +do_execsql_test 9.3.3 { SELECT * FROM db2.tbl } {3 1 2} +do_execsql_test 9.3.1 { SELECT * FROM db3.tbl } {1 2 3} +do_execsql_test 9.3.2 { SELECT * FROM db4.tbl } {2 3 1} +do_execsql_test 9.3.3 { SELECT * FROM db5.tbl } {3 1 2} +do_execsql_test 9.3.1 { SELECT * FROM db6.tbl } {1 2 3} +do_execsql_test 9.3.2 { SELECT * FROM db7.tbl } {2 3 1} + +do_test 9.4 { + for {set ii 0} {$ii < ($nDb-1)} {incr ii} { + set jj [expr $ii+1] + db eval " + CREATE TEMP TRIGGER trd$ii BEFORE DELETE ON db$ii.tbl BEGIN + DELETE FROM db$jj.tbl; + END; + " + } +} {} + +do_execsql_test 9.5 { DELETE FROM db0.tbl } +do_execsql_test 9.5.1 { SELECT * FROM db0.tbl } {} +do_execsql_test 9.5.2 { SELECT * FROM db1.tbl } {} +do_execsql_test 9.5.3 { SELECT * FROM db2.tbl } {} +do_execsql_test 9.5.1 { SELECT * FROM db3.tbl } {} +do_execsql_test 9.5.2 { SELECT * FROM db4.tbl } {} +do_execsql_test 9.5.3 { SELECT * FROM db5.tbl } {} +do_execsql_test 9.5.1 { SELECT * FROM db6.tbl } {} +do_execsql_test 9.5.2 { SELECT * FROM db7.tbl } {} + finish_test diff --git a/test/tester.tcl b/test/tester.tcl index 3fad39668..856df5421 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -1785,11 +1785,6 @@ proc crashsql {args} { # cfSync(), which can be different then what TCL uses by # default, so here we force it to the "nativename" format. set cfile [string map {\\ \\\\} [file nativename [file join [get_pwd] $crashfile]]] - ifcapable winrt { - # Except on winrt. Winrt has no way to transform a relative path into - # an absolute one, so it just uses the relative paths. - set cfile $crashfile - } set f [open crash.tcl w] puts $f "sqlite3_initialize ; sqlite3_shutdown" diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 515036368..1f81690f2 100755 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -101,6 +101,7 @@ Usage: $a0 help $a0 joblist ?PATTERN? $a0 njob ?NJOB? + $a0 retest $a0 script ?-msvc? CONFIG $a0 status ?-d SECS? ?--cls? $a0 halt @@ -120,20 +121,21 @@ Usage: --stop-on-error Stop running after any reported error --zipvfs ZIPVFSDIR ZIPVFS source directory -Special values for PERMUTATION that work with plain tclsh: +Special values for PERMUTATION include: - list - show all allowed PERMUTATION arguments. + list - show allowed PERMUTATION arguments. mdevtest - tests recommended prior to normal development check-ins. + devtest - alias for "mdevtest" release - full release test with various builds. sdevtest - like mdevtest but using ASAN and UBSAN. - -Other PERMUTATION arguments must be run using testfixture, not tclsh: - all - all tcl test scripts, plus a subset of test scripts rerun with various permutations. full - all tcl test scripts. veryquick - a fast subset of the tcl test scripts. This is the default. +The interpreter that runs this script can be an ordinary "tclsh" as long +as "package require sqlite3" works, or it can be "testfixture". + If no PATTERN arguments are present, all tests specified by the PERMUTATION are run. Otherwise, each pattern is interpreted as a glob pattern. Only those tcl tests for which the final component of the filename matches at @@ -167,6 +169,9 @@ only the parts that contain the error messages. The --summary option just shows the jobs that failed. If PATTERN are provided, the error information is only provided for jobs that match PATTERN. +The "retest" command reruns tests that failed or were never completed +by a prior invocation of testrunner.tcl. + Full documentation here: https://sqlite.org/src/doc/trunk/doc/testrunner.md }]] @@ -212,7 +217,8 @@ proc default_njob {} { if {$nCore<=2} { set nHelper 1 } else { - set nHelper [expr int($nCore*0.5)] + set nHelper [expr int($nCore*0.8)] + if {$nHelper>20} {set nHelper 20} } return $nHelper } @@ -297,6 +303,8 @@ switch -nocase -glob -- $tcl_platform(os) { error "cannot determine platform!" } } +set TRG(testfixture-fullpath) [file join $dir $TRG(testfixture)] +set TRG(interp) [info nameofexec] #------------------------------------------------------------------------- #------------------------------------------------------------------------- @@ -706,9 +714,18 @@ if {[llength $argv]>=1 } } - if {![file readable $TRG(dbname)]} { - puts "Database missing: $TRG(dbname)" - exit + set once 1 + while {![file readable $TRG(dbname)]} { + if {$delay==0} { + puts "Database missing: $TRG(dbname)" + exit + } + if {$once} { + set once 0 + puts "Waiting for testing to start...." + flush stdout + } + after [expr {$delay*1000}] } sqlite3 mydb $TRG(dbname) mydb timeout 2000 @@ -1148,7 +1165,7 @@ proc add_tcl_jobs {build config patternlist {shelldepid ""}} { set testrunner_tcl [file normalize [info script]] if {$build==""} { - set testfixture [info nameofexec] + set testfixture $TRG(interp) } else { set testfixture [file join [lindex $build 1] $TRG(testfixture)] } @@ -1269,6 +1286,26 @@ proc add_fuzztest_jobs {buildname patternlist} { set subcmd [lrange $interpreter 1 end] set interpreter [lindex $interpreter 0] + # For fuzzcheck-asan and fuzzcheck-ubsan, break up some + # fuzzdata files into multiple slices, for improved + # concurrency. + # + if {[string match *fuzzcheck-*san $interpreter]} { + set newscripts {} + foreach s $scripts { + if {[string match {*fuzzdata[12].db} $s] + && ![string match slice $s]} { + set N 6 + for {set i 0} {$i<$N} {incr i} { + lappend newscripts [list --slice $i $N $s] + } + } else { + lappend newscripts $s + } + } + set scripts $newscripts + } + if {[string match fuzzcheck* $interpreter] && [info exists env(FUZZDB)] && [file readable $env(FUZZDB)] @@ -1359,14 +1396,30 @@ proc add_devtest_jobs {lBld patternlist} { } } -# Check to ensure that the interpreter is a full-blown "testfixture" -# build and not just a "tclsh". If this is not the case, issue an -# error message and exit. +# Check to ensure that TRG(interp) is a full-blown "testfixture" and +# not just a "tclsh". +# +# The value of TRG(interp) defaults to whatever interpreter is running +# this script, which might be either tclsh or testfixture. If tclsh is +# running this script, change $TRG(interp) to be an instance of testfixture. +# If no testfixture exists in the directory from which this script is run, +# attempt to build one. +# +# Do not return unless $TRG(interp) is a valid testfixture. If unable +# to find and/or construct one, abort with an error message. # proc must_be_testfixture {} { + global TRG if {[lsearch [info commands] sqlite3_soft_heap_limit]<0} { - puts "Use testfixture, not tclsh, for these arguments." - exit 1 + if {![file exec $TRG(testfixture-fullpath)]} { + puts "make testfixture" + catch {exec make testfixture >@stdout 2>@stderr} + } + if {![file exec $TRG(testfixture-fullpath)]} { + puts "Requires testfixture, and I was unable to build it." + exit 1 + } + set TRG(interp) $TRG(testfixture-fullpath) } } @@ -1441,11 +1494,15 @@ proc add_jobs_from_cmdline {patternlist} { list { set allperm [array names ::testspec] - lappend allperm all mdevtest sdevtest release list + lappend allperm all devtest mdevtest sdevtest release list puts "Allowed values for the PERMUTATION argument: [lsort $allperm]" exit 0 } + retest { + # no-op + } + default { must_be_testfixture if {[info exists ::testspec($first)]} { @@ -1482,6 +1539,8 @@ proc add_jobs_from_cmdline {patternlist} { } } +# Initializer, or reinitialize, the testrunner.db database file. +# proc make_new_testset {} { global TRG @@ -1525,7 +1584,8 @@ proc mark_job_as_finished {jobid output state endtm} { SET output=$output, state=$state, endtime=$endtm, span=$endtm-starttime, ntest=$ntest, nerr=$nerr, svers=$svers, pltfm=$pltfm WHERE jobid=$jobid; - UPDATE jobs SET state=$childstate WHERE depid=$jobid AND state!='halt'; + UPDATE jobs SET state=$childstate + WHERE depid=$jobid AND state!='halt' AND state!='done'; UPDATE config SET value=value+$nerr WHERE name='nfail'; UPDATE config SET value=value+$ntest WHERE name='ntest'; } @@ -1597,7 +1657,7 @@ proc launch_another_job {iJob} { global O global T - set testfixture [info nameofexec] + set testfixture $TRG(interp) set script $TRG(info_script) set O($iJob) "" @@ -1798,6 +1858,27 @@ proc run_testset {} { } +# If the argument is "retest", simply rerun all tests from the previous +# run that are marked as one of "ready", "running", "failed", or "omit" +# plus redo any build of dependencies those tests. +# +proc handle_retest {} { + set cnt 0 + if {[catch {trdb exists {SELECT jobid FROM jobs}} cnt] || $cnt==0} { + puts "No test available to rerun" + exit 1 + } + trdb eval {UPDATE jobs SET state='ready' + WHERE state IN ('running','failed','omit')} + for {set kk 0} {$kk<2} {incr kk} { + trdb eval { + UPDATE jobs SET state='ready' + WHERE jobid IN (SELECT depid FROM jobs WHERE state='ready'); + UPDATE jobs SET state='' WHERE state='ready' AND depid<>''; + } + } +} + # Handle the --buildonly option, if it was specified. # proc handle_buildonly {} { @@ -1836,14 +1917,21 @@ proc explain_tests {} { sqlite3 trdb $TRG(dbname) trdb timeout $TRG(timeout) -set tm [lindex [time { make_new_testset }] 0] +if {[llength $TRG(patternlist)]==1 && $TRG(patternlist) eq "retest"} { + set tm 0 + handle_retest +} else { + set tm [lindex [time { make_new_testset }] 0] +} if {$TRG(explain)} { explain_tests } else { if {$TRG(nJob)>1} { puts "splitting work across $TRG(nJob) cores" } - puts "built testset in [expr $tm/1000]ms.." + if {$tm>0} { + puts "built testset in [expr $tm/1000]ms.." + } handle_buildonly run_testset } diff --git a/test/testrunner_data.tcl b/test/testrunner_data.tcl index e74caee1d..4daee0274 100644 --- a/test/testrunner_data.tcl +++ b/test/testrunner_data.tcl @@ -37,7 +37,6 @@ namespace eval trd { set tcltest(win.Windows-Memdebug) veryquick set tcltest(win.Windows-Win32Heap) veryquick set tcltest(win.Windows-Sanitize) veryquick - set tcltest(win.Windows-WinRT) veryquick set tcltest(win.Default) {full win_unc_locking} # Extra [make xyz] tests that should be run for various builds. @@ -193,6 +192,7 @@ namespace eval trd { -DSQLITE_ENABLE_HIDDEN_COLUMNS -DSQLITE_MAX_ATTACHED=125 -DSQLITE_MUTATION_TEST + -DSQLITE_THREAD_MISUSE_ABORT --enable-fts5 } set build(Debug-Two) { @@ -364,12 +364,6 @@ namespace eval trd { set build(Windows-Sanitize) { ASAN=1 } - - set build(Windows-WinRT) { - FOR_WINRT=1 - ENABLE_SETLK=1 - -DSQLITE_TEMP_STORE=3 - } } diff --git a/test/testrunner_estwork.tcl b/test/testrunner_estwork.tcl index c139394a5..e02eb22dc 100644 --- a/test/testrunner_estwork.tcl +++ b/test/testrunner_estwork.tcl @@ -364,6 +364,7 @@ set estwork(shell6.test) 3 set estwork(shell8.test) 104 set estwork(shell9.test) 3 set estwork(shellA.test) 2 +set estwork(shellB.test) 2 set estwork(shmlock.test) 27 set estwork(sidedelete.test) 10 set estwork(skipscan1.test) 7 diff --git a/test/tkt-99378177930f87bd.test b/test/tkt-99378177930f87bd.test index ba9fdc702..495867280 100644 --- a/test/tkt-99378177930f87bd.test +++ b/test/tkt-99378177930f87bd.test @@ -33,6 +33,8 @@ do_execsql_test tkt-99378-100 { (2, '{"x":2}', 4, 5), (3, '{"x":1}', 6, 7); CREATE INDEX t1x ON t1(d, a, b->>'x', c); + CREATE TABLE t2(y); + INSERT INTO t2(y) VALUES(9); } {} do_execsql_test tkt-99378-110 { SELECT a, @@ -48,6 +50,20 @@ do_execsql_test tkt-99378-110 { 2 2 1 26 22 3 1 1 6 6 } +do_execsql_test tkt-99378-111 { + SELECT if(a,a,y), + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t2 CROSS JOIN t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} # The proof that the index on the expression is being used is in the # fact that the byte code contains no "Function" opcodes. In other words, @@ -65,6 +81,17 @@ do_execsql_test tkt-99378-120 { WHERE d BETWEEN 0 and 10 GROUP BY a; } {~/Function/} +do_execsql_test tkt-99378-121 { + EXPLAIN + SELECT if(a,a,y), + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t2 CROSS JOIN t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} {~/Function/} do_execsql_test tkt-99378-130 { @@ -182,6 +209,7 @@ do_execsql_test tkt-99378-310 { # do_execsql_test tkt-99378-400 { DROP TABLE t1; + DROP TABLE t2; CREATE TABLE t0(w); INSERT INTO t0(w) VALUES(1); CREATE TABLE t1(x); diff --git a/test/tkt2339.test b/test/tkt2339.test index 41acd377c..4bbedb828 100644 --- a/test/tkt2339.test +++ b/test/tkt2339.test @@ -47,12 +47,12 @@ do_test tkt2339.2 { } } {4 3 14 13} do_test tkt2339.3 { - execsql { + lsort -integer [execsql { SELECT * FROM (SELECT * FROM t1 ORDER BY num DESC) UNION ALL SELECT * FROM (SELECT * FROM t2 ORDER BY num DESC LIMIT 2) - } -} {4 3 2 1 14 13} + }] +} {1 2 3 4 13 14} do_test tkt2339.4 { execsql { SELECT * FROM (SELECT * FROM t1 ORDER BY num DESC LIMIT 2) diff --git a/test/values.test b/test/values.test index c3c52ceb1..58e764ce6 100644 --- a/test/values.test +++ b/test/values.test @@ -21,9 +21,9 @@ do_execsql_test 1.0 { } -explain_i { - INSERT INTO x1(a, b, c) VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4); -} +#explain_i { +# INSERT INTO x1(a, b, c) VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4); +#} do_execsql_test 1.1.1 { INSERT INTO x1 VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4); } diff --git a/test/vt02.c b/test/vt02.c index 06fade096..e08d649c1 100644 --- a/test/vt02.c +++ b/test/vt02.c @@ -45,8 +45,29 @@ ** a=... AND b=... AND c=... ** a=... AND b=... AND c=... AND d=... ** -** Various ORDER BY constraints are also recognized and consumed. The -** OFFSET constraint is recognized and consumed. +** The table will also recognize IN constraints on column D using the +** sqlite3_vtab_in_first() and sqlite3_vtab_in_next() interfaces: +** +** a=... AND d IN (...) +** a=... AND b=... AND d IN (...) +** a=... AND b=... AND c=... AND d IN (...) +** +** Various ORDER BY constraints are also recognized and consumed. +** +** ORDER BY x +** ORDER BY a +** ORDER BY a, b +** ORDER BY a, b, c +** ORDER BY a, b, c, d +** +** The above also work if every term is DESC rather than ASC. However, +** the orderByConsumed is not set if there are a mixture of ASC and DESC +** terms in the ORDER BY clause. +** +** The sqlite3_vtab_distinct() interface is used to recognize DISTINCT +** and GROUP BY constraints and consume the corresponding sorting requirements. +** +** The OFFSET constraint is recognized and consumed. ** ** ## TABLE-VALUED FUNCTION ** @@ -90,6 +111,13 @@ ** vector of results sent to xFilter. Only the first ** few are used, as required by idxNum. ** +** 0x80 If sqlite3_vtab_distinct() says that duplicate rows +** may be omitted (values 2 or 3), then maybe omit +** some but not all of the duplicate rows. +** +** 0x100 Do not omit any duplicate rows even if +** sqlite3_vtab_distinct() says that is ok to do. +** ** Because these flags take effect during xBestIndex, the RHS of the ** flag= constraint must be accessible. In other words, the RHS of flag= ** needs to be an integer literal, not another column of a join or a @@ -136,14 +164,12 @@ ** ## COMPILING AND RUNNING ** ** This file can also be compiled separately as a loadable extension -** for SQLite (as long as the -DTH3_VERSION is not defined). To compile as a -** loadable extension do his: -** -** gcc -Wall -g -shared -fPIC -I. -DSQLITE_DEBUG vt02.c -o vt02.so +** for SQLite (as long as the -DTH3_VERSION is not defined). To compile as a +** loadable extension do something like this: ** -** Or on Windows: -** -** cl vt02.c -link -dll -out:vt02.dll +** (linux) gcc -shared -fPIC -I. vt02.c -o vt02.so +** (mac) clang -dynamiclib -fPIC -I. vt02.c -o vt02.dylib +** (windows) cl vt02.c -link -dll -out:vt02.dll ** ** Then load into the CLI using: ** @@ -167,6 +193,7 @@ ** 2x increment by 100 ** 3x increment by 1000 ** 1xx Use offset provided by argv[N] +** 1xxx Reverse output order (to implement ORDER BY ... DESC) */ #ifndef TH3_VERSION /* These bits for separate compilation as a loadable extension, only */ @@ -195,6 +222,8 @@ struct vt02_vtab { #define VT02_NO_OFFSET 0x0004 /* Omit the offset optimization */ #define VT02_ALLOC_IDXSTR 0x0008 /* Alloate an idxStr */ #define VT02_BAD_IDXNUM 0x0010 /* Generate an invalid idxNum */ +#define VT02_PARTIAL_DEDUP 0x0080 /* Omit some but not all duplicate rows */ +#define VT02_NO_DEDUP 0x0100 /* Include all duplicate rows */ /* ** A cursor @@ -203,6 +232,7 @@ struct vt02_cur { sqlite3_vtab_cursor parent; /* Base class. Must be first */ sqlite3_int64 i; /* Current entry */ sqlite3_int64 iEof; /* Indicate EOF when reaching this value */ + sqlite3_int64 iMin; /* EOF if dropping below this value */ int iIncr; /* Amount by which to increment */ unsigned int mD; /* Mask of allowed D-column values */ }; @@ -292,7 +322,7 @@ static int vt02Close(sqlite3_vtab_cursor *pCursor){ */ static int vt02Eof(sqlite3_vtab_cursor *pCursor){ vt02_cur *pCur = (vt02_cur*)pCursor; - return pCur->i<0 || pCur->i>=pCur->iEof; + return pCur->i<pCur->iMin || pCur->i>=pCur->iEof; } /* Advance the cursor to the next row in the table @@ -301,8 +331,8 @@ static int vt02Next(sqlite3_vtab_cursor *pCursor){ vt02_cur *pCur = (vt02_cur*)pCursor; do{ pCur->i += pCur->iIncr; - if( pCur->i<0 ) pCur->i = pCur->iEof; - }while( (pCur->mD & (1<<(pCur->i%10)))==0 && pCur->i<pCur->iEof ); + if( pCur->i<pCur->iMin || pCur->i>=pCur->iEof ) break; + }while( (pCur->mD & (1<<(pCur->i%10)))==0 ); return SQLITE_OK; } @@ -324,6 +354,7 @@ static int vt02Next(sqlite3_vtab_cursor *pCursor){ ** 2x increment by 100 ** 3x increment by 1000 ** 1xx Use offset provided by argv[N] +** 1xxx Output rows in reverse order */ static int vt02Filter( sqlite3_vtab_cursor *pCursor, /* The cursor to rewind */ @@ -334,11 +365,17 @@ static int vt02Filter( ){ vt02_cur *pCur = (vt02_cur*)pCursor; /* The vt02 cursor */ int bUseOffset = 0; /* True to use OFFSET value */ + int bReverse = 0; /* Output rows in reverse order */ int iArg = 0; /* argv[] values used so far */ int iOrigIdxNum = idxNum; /* Original value for idxNum */ pCur->iIncr = 1; + pCur->iMin = 0; pCur->mD = 0x3ff; + if( idxNum>=1000 ){ + bReverse = 1; + idxNum -= 1000; + } if( idxNum>=100 ){ bUseOffset = 1; idxNum -= 100; @@ -416,9 +453,17 @@ static int vt02Filter( }else{ goto vt02_bad_idxnum; } + if( bReverse ){ + sqlite3_int64 x; + x = pCur->i + ((pCur->iEof - pCur->i)/pCur->iIncr)*pCur->iIncr; + if( x>=pCur->iEof ) x -= pCur->iIncr; + pCur->iIncr = -pCur->iIncr; + pCur->iMin = pCur->i; + pCur->i = x; + } if( bUseOffset ){ int nSkip = sqlite3_value_int(argv[iArg]); - while( nSkip-- > 0 && pCur->i<pCur->iEof ) vt02Next(pCursor); + while( nSkip-- > 0 && !vt02Eof(pCursor) ) vt02Next(pCursor); } return SQLITE_OK; @@ -838,35 +883,52 @@ static int vt02BestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ ** the same answer. */ if( pInfo->nOrderBy>0 && (flags & VT02_NO_SORT_OPT)==0 ){ + int eDistinct = sqlite3_vtab_distinct(pInfo); if( pInfo->idxNum==1 ){ /* There will only be one row of output. So it is always sorted. */ pInfo->orderByConsumed = 1; }else - if( pInfo->aOrderBy[0].iColumn<=0 - && pInfo->aOrderBy[0].desc==0 - ){ - /* First column of order by is X ascending */ + if( pInfo->aOrderBy[0].iColumn<=0 ){ + /* First column of order by is X */ + if( pInfo->aOrderBy[0].desc ){ + pInfo->idxNum += 1000; /* Reverse output order */ + } pInfo->orderByConsumed = 1; }else - if( sqlite3_vtab_distinct(pInfo)>=1 ){ + if( eDistinct>=1 ){ unsigned int x = 0; + int nDesc = 0; + int nAsc = 0; for(i=0; i<pInfo->nOrderBy; i++){ int iCol = pInfo->aOrderBy[i].iColumn; if( iCol<0 ) iCol = 0; + if( pInfo->aOrderBy[i].desc ){ + nDesc++; + }else{ + nAsc++; + } x |= 1<<iCol; } - if( sqlite3_vtab_distinct(pInfo)==2 ){ + if( nDesc>0 && nAsc>0 ){ + if( eDistinct!=1 ) eDistinct = -999; /* Never set orderByConsumed */ + }else if( nAsc==0 ){ + pInfo->idxNum += 1000; /* Reverse output order */ + } + if( eDistinct>=2 && (flags & VT02_NO_DEDUP)!=0 ){ + eDistinct = 1; + } + if( eDistinct>=2 ){ /* DISTINCT or (DISTINCT and ORDER BY) */ if( x==0x02 ){ /* DISTINCT A */ - pInfo->idxNum += 30; + pInfo->idxNum += (flags & VT02_PARTIAL_DEDUP)!=0 ? 20 : 30; pInfo->orderByConsumed = 1; }else if( x==0x06 ){ /* DISTINCT A,B */ - pInfo->idxNum += 20; + pInfo->idxNum += (flags & VT02_PARTIAL_DEDUP)!=0 ? 10 : 20; pInfo->orderByConsumed = 1; }else if( x==0x0e ){ /* DISTINCT A,B,C */ - pInfo->idxNum += 10; + pInfo->idxNum += (flags & VT02_PARTIAL_DEDUP)!=0 ? 0 : 10; pInfo->orderByConsumed = 1; }else if( x & 0x01 ){ /* DISTINCT X */ @@ -875,7 +937,7 @@ static int vt02BestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ /* DISTINCT A,B,C,D */ pInfo->orderByConsumed = 1; } - }else{ + }else if( eDistinct==1 ){ /* GROUP BY */ if( x==0x02 ){ /* GROUP BY A */ pInfo->orderByConsumed = 1; @@ -893,6 +955,21 @@ static int vt02BestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ pInfo->orderByConsumed = 1; } } + }else{ + int nDesc = 0; + int nAsc = 0; + for(i=0; i<pInfo->nOrderBy; i++){ + if( pInfo->aOrderBy[i].iColumn!=i+1 ) break; + if( pInfo->aOrderBy[i].desc ){ + nDesc++; + }else{ + nAsc++; + } + } + if( i==pInfo->nOrderBy && (nDesc==0 || nAsc==0) ){ + pInfo->orderByConsumed = 1; + if( nDesc ) pInfo->idxNum += 1000; + } } } @@ -901,7 +978,7 @@ static int vt02BestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ pInfo->needToFreeIdxStr = 1; } if( flags & VT02_BAD_IDXNUM ){ - pInfo->idxNum += 1000; + pInfo->idxNum += 10000; } if( iOffset>=0 ){ diff --git a/test/vt100-a.sql b/test/vt100-a.sql index a0d3f46be..f141f637f 100644 --- a/test/vt100-a.sql +++ b/test/vt100-a.sql @@ -9,11 +9,40 @@ INSERT INTO t1 VALUES ('one','twotwotwo','thirty-three'), (unistr('\u001b[91mRED\u001b[0m'),'fourfour','fifty-five'), ('six','seven','eighty-eight'); -.print With -escape off +.testcase 100 SELECT * FROM t1; +.check <<END +╭─────┬───────────┬──────────────╮ +│ a │ b │ c │ +╞═════╪═══════════╪══════════════╡ +│ one │ twotwotwo │ thirty-three │ +│ RED │ fourfour │ fifty-five │ +│ six │ seven │ eighty-eight │ +╰─────┴───────────┴──────────────╯ +END + .mode box -escape ascii -.print With -escape ascii +.testcase 200 SELECT * FROM t1; +.check <<END +╭────────────────┬───────────┬──────────────╮ +│ a │ b │ c │ +╞════════════════╪═══════════╪══════════════╡ +│ one │ twotwotwo │ thirty-three │ +│ ^[[91mRED^[[0m │ fourfour │ fifty-five │ +│ six │ seven │ eighty-eight │ +╰────────────────┴───────────┴──────────────╯ +END + +.testcase 300 .mode box -escape symbol -.print With -escape symbol SELECT * FROM t1; +.check <<END +╭──────────────┬───────────┬──────────────╮ +│ a │ b │ c │ +╞══════════════╪═══════════╪══════════════╡ +│ one │ twotwotwo │ thirty-three │ +│ ␛[91mRED␛[0m │ fourfour │ fifty-five │ +│ six │ seven │ eighty-eight │ +╰──────────────┴───────────┴──────────────╯ +END diff --git a/test/walckptnoop.test b/test/walckptnoop.test index 7ff8e90b8..89055316f 100644 --- a/test/walckptnoop.test +++ b/test/walckptnoop.test @@ -102,7 +102,11 @@ do_catchsql_test 1.8 { PRAGMA wal_checkpoint = noop; } {0 {0 5 0}} -do_execsql_test 1.9 { +do_test 1.9 { + sqlite3_wal_checkpoint_v2 db noop +} {0 5 0} + +do_execsql_test 1.10 { PRAGMA journal_mode = delete; PRAGMA wal_checkpoint = noop; } {delete 0 -1 -1} diff --git a/test/walrestart.test b/test/walrestart.test new file mode 100644 index 000000000..cf27a4098 --- /dev/null +++ b/test/walrestart.test @@ -0,0 +1,89 @@ +# 2026-03-03 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing a race condition in WAL restart. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +if {$::tcl_platform(platform) ne "unix"} { + # This test only works on unix + finish_test + return +} +set testprefix walrestart + +if {[permutation]=="memsubsys1"} { + # memsubsys1 configures a very small page-cache, which causes different + # numbers of frames to be written to the wal file for some transactions, + # which causes some of the tests in this file to fail. + finish_test + return +} + +db close +sqlite3_shutdown + +proc faultsim {n} { return 0 } +sqlite3_test_control_fault_install faultsim + +# Populate database. Create a large wal file and checkpoint it. +# +reset_db +do_execsql_test 1.0 { + PRAGMA auto_vacuum = 0; + PRAGMA journal_mode = wal; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<20 + ) + INSERT INTO t1 SELECT NULL, randomblob(600) FROM s; + CREATE INDEX i1 ON t1(b); + PRAGMA wal_checkpoint; +} {wal 0 49 49} +do_execsql_test 1.1 { + UPDATE t1 SET b=randomblob(600); + PRAGMA wal_checkpoint; +} {0 45 45} + +# We have a completely checkpointed wal file on disk. mxFrame=$large, +# nBackfill=$large. Do another checkpoint with [db]. This time, after [db] +# reads mxFrame but before it reads nBackfill, write to the db such +# that 0 < mxFrame < large. +# +proc faultsim {n} { + if {$n==660} { + db2 eval { UPDATE t1 SET b=randomblob(600) WHERE a<5 } + } + return 0 +} +sqlite3 db2 test.db +do_execsql_test 1.2 { + PRAGMA wal_checkpoint; +} {0 45 0} + +# Now write another big update to the wal file and checkpoint it. +# +do_execsql_test -db db2 1.3 { + UPDATE t1 SET b=randomblob(600); +} +proc faultsim {n} { return 0 } +do_execsql_test 1.4 { + PRAGMA wal_checkpoint; +} {/0 5. 5./} + +do_catchsql_test 1.5 { + PRAGMA integrity_check +} {0 ok} + +sqlite3_test_control_fault_install + +finish_test diff --git a/test/where2.test b/test/where2.test index 83740bffd..6045cb117 100644 --- a/test/where2.test +++ b/test/where2.test @@ -794,4 +794,41 @@ do_execsql_test where2-15.1 { ORDER BY a,EXISTS(SELECT 1 FROM t1 LEFT JOIN (SELECT x AS y FROM t2) AS s2 ON t1.b=s2.y),x; } {12 34 NULL | 56 78 78 | 90 12 12 |} +# Demonstrate that CROSS JOIN is a join reordering barrier. +# +reset_db +do_execsql_test where2-16.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b INT); + CREATE TABLE t2(c INTEGER PRIMARY KEY, d INT); + CREATE TABLE t3(e INTEGER PRIMARY KEY, f INT); + CREATE TABLE t4(g INTEGER PRIMARY KEY, h INT); +} + +# Here the query planner wants to move t4 forward so that it is in front of +# t1. Ensure that does not happen. +do_execsql_test where2-16.2 { + EXPLAIN QUERY PLAN + SELECT * + FROM t1, t2 CROSS JOIN t3, t4 + WHERE t4.g=1 + AND t1.a=t4.h + AND t2.c=t1.b + AND t3.e=t2.d; +} {~/.* t4 .* t[12] .*/} + +# In this case the planner wants to move t1 to come after t4. Ensure that +# does not happen. +do_execsql_test where2-16.2 { + EXPLAIN QUERY PLAN + SELECT * + FROM t1, t2 CROSS JOIN t3, t4 + WHERE t2.c=1 + AND t3.e=t2.d + AND t4.g=t3.f + AND t1.a=t4.h; +} {~/.* t[34] .* t1 .*/} + + + + finish_test diff --git a/test/whereK.test b/test/whereK.test index 060d470ff..995c08371 100644 --- a/test/whereK.test +++ b/test/whereK.test @@ -69,4 +69,17 @@ do_execsql_test 1.5eqp { ORDER BY +a; } {/SEARCH t1 USING INDEX t1bc/} +# https://sqlite.org/forum/forumpost/2026-01-16T11:35:28Z +do_execsql_test 2.1 { + DROP TABLE t1; + CREATE TABLE t0(x COLLATE NOCASE); + CREATE INDEX t0x ON t0(x); + CREATE TABLE t1(y); + INSERT INTO t0 VALUES('a'); + INSERT INTO t1 VALUES('AB'); + SELECT count(*) FROM t0, t1 WHERE (y BETWEEN 1 AND x) OR (x>=y AND x); + SELECT count(*) FROM t0, t1 WHERE (x>=y AND x) OR (y BETWEEN 1 AND x); +} {1 1} + + finish_test diff --git a/test/win32longpath.test b/test/win32longpath.test index 2545c55a1..43905f26e 100644 --- a/test/win32longpath.test +++ b/test/win32longpath.test @@ -115,11 +115,7 @@ do_test 1.6 { db3 close -# winrt platforms do not handle paths with unix-style '/' directory separators. -# set lUri [list 1a 1b 1c 1d 1e 1f] -ifcapable winrt { set lUri [list 1a 1c 1e] } - foreach tn $lUri { sqlite3 db3 $uri($tn) -vfs win32-longpath -uri 1 -translatefilename 0 diff --git a/test/with1.test b/test/with1.test index 5ddf9dce0..c87082583 100644 --- a/test/with1.test +++ b/test/with1.test @@ -1237,4 +1237,23 @@ do_execsql_test 27.1 { SELECT k, cte_map, main_map, '|' FROM log ORDER BY k; } {1 cte1 main1 | 2 cte2 main2 |} +# forum post https://sqlite.org/forum/forumpost/2026-03-04T05:06:26Z +# +db null NULL +do_execsql_test 28.1 { + DROP TABLE t1; + CREATE TABLE t1(x INTEGER PRIMARY KEY); + INSERT INTO t1 VALUES(1),(4),(999); + SELECT ( + WITH RECURSIVE t2(y) AS ( + SELECT 4 + UNION + SELECT NULL + UNION + SELECT y+1 FROM t2 WHERE y=4 ORDER BY 1 + ) + SELECT 1 FROM t2 WHERE y=x + ) FROM t1; +} {NULL 1 NULL} + finish_test diff --git a/test/zipfile.test b/test/zipfile.test index 9bb35ea5d..f57170724 100644 --- a/test/zipfile.test +++ b/test/zipfile.test @@ -908,4 +908,10 @@ d42728f602000000020000000500ffff0000000000000000a4810000000068 do_catchsql_test 20.2 { SELECT * FROM zipfile(unhex('504b0304140000000000000000008b9ed9d30100000001000000010000007841504b01021e03140000000000000000008b9ed9d3010000000100000001001e000000000000000000a4810000000078504b050600000000010001002f000000200000000000')); } {1 {zip archive is corrupt}} + +# https://sqlite.org/forum/forumpost/2026-01-13T00:09:06Z +do_catchsql_test 21.0 { + SELECT * FROM zipfile(X'504B03040A0000000000000000000000000000000000000000000100000078504B010200000A0000000000000000000000000000000000000000000100000000000000000000000000E2FFFFFF78504B050600000000010001002F0000001F0000000000'); +} {1 {failed to read LFH at offset -30}} + finish_test diff --git a/test/zipfile2.test b/test/zipfile2.test index 8ee90d310..c1498872a 100644 --- a/test/zipfile2.test +++ b/test/zipfile2.test @@ -302,4 +302,19 @@ do_execsql_test 7.1 { SELECT length(name) FROM t1; } {60000} + +# https://sqlite.org/forum/forumpost/721a05d2c5 +# +if {[catch { load_static_extension db fileio }]==0} { + forcedelete test.zip + set fd [open test.zip wb] + fconfigure $fd -translation binary + puts -nonewline $fd [db one {SELECT X'504b0506000000000100010030000000160000000000504b01021400140000000000000000000000000000000000000000000100010000000000000000000000000000006100'}] + close $fd + + do_catchsql_test 8.0 { + SELECT name,sz FROM zipfile(readfile('test.zip')); + } {1 {failed to read LFH at offset 0}} +} + finish_test diff --git a/tool/build-all-msvc.bat b/tool/build-all-msvc.bat index 83d660deb..ae0d0e5b6 100755 --- a/tool/build-all-msvc.bat +++ b/tool/build-all-msvc.bat @@ -105,7 +105,6 @@ REM When set, these values are expanded and passed to the NMAKE command line, REM after its other arguments. These may be used to specify additional NMAKE REM options, for example: REM -REM SET NMAKE_ARGS=FOR_WINRT=1 REM SET NMAKE_ARGS_DEBUG=MEMDEBUG=1 REM SET NMAKE_ARGS_RETAIL=WIN32HEAP=1 REM @@ -861,4 +860,4 @@ GOTO no_errors GOTO end_of_file :end_of_file -%__ECHO% EXIT /B %ERRORLEVEL% +%__ECHO% EXIT /B %ERRORLEVEL% diff --git a/tool/dbtotxt.c b/tool/dbtotxt.c index fbd6e3d51..ed347840a 100644 --- a/tool/dbtotxt.c +++ b/tool/dbtotxt.c @@ -1,6 +1,12 @@ /* -** Copyright 2008 D. Richard Hipp and Hipp, Wyrick & Company, Inc. -** All Rights Reserved +** 2018-12-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** ****************************************************************************** ** diff --git a/tool/lemon.c b/tool/lemon.c index 324dda0c5..9071e7719 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -494,6 +494,7 @@ struct lemon { char *filename; /* Name of the input file */ char *outname; /* Name of the current output file */ char *tokenprefix; /* A prefix added to token names in the .h file */ + char *stackSizeLimit; /* Function to return the stack size limit */ char *reallocFunc; /* Function to use to allocate stack space */ char *freeFunc; /* Function to use to free stack space */ int nconflict; /* Number of parsing conflicts */ @@ -2638,6 +2639,9 @@ static void parseonetoken(struct pstate *psp) }else if( strcmp(x,"default_type")==0 ){ psp->declargslot = &(psp->gp->vartype); psp->insertLineMacro = 0; + }else if( strcmp(x,"stack_size_limit")==0 ){ + psp->declargslot = &(psp->gp->stackSizeLimit); + psp->insertLineMacro = 0; }else if( strcmp(x,"realloc")==0 ){ psp->declargslot = &(psp->gp->reallocFunc); psp->insertLineMacro = 0; @@ -3715,7 +3719,7 @@ PRIVATE int compute_action(struct lemon *lemp, struct action *ap) return act; } -#define LINESIZE 1000 +#define LINESIZE 10000 /* The next cluster of routines are for reading the template file ** and writing the results to the generated parser */ /* The first function transfers data from "in" to "out" until @@ -3750,12 +3754,9 @@ PRIVATE void tplt_xfer(char *name, FILE *in, FILE *out, int *lineno) /* Skip forward past the header of the template file to the first "%%" */ -PRIVATE void tplt_skip_header(FILE *in, int *lineno) -{ +PRIVATE void tplt_skip_header(FILE *in){ char line[LINESIZE]; - while( fgets(line,LINESIZE,in) && (line[0]!='%' || line[1]!='%') ){ - (*lineno)++; - } + while( fgets(line,LINESIZE,in) && (line[0]!='%' || line[1]!='%') ){} } /* The next function finds the template file and opens it, returning @@ -3825,12 +3826,14 @@ PRIVATE void tplt_linedir(FILE *out, int lineno, char *filename) filename++; } fprintf(out,"\"\n"); + fflush(out); } /* Print a string to the file and keep the linenumber up to date */ PRIVATE void tplt_print(FILE *out, struct lemon *lemp, char *str, int *lineno) { if( str==0 ) return; + fflush(out); while( *str ){ putc(*str,out); if( *str=='\n' ) (*lineno)++; @@ -3843,6 +3846,7 @@ PRIVATE void tplt_print(FILE *out, struct lemon *lemp, char *str, int *lineno) if (!lemp->nolinenosflag) { (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); } + fflush(out); return; } @@ -4407,6 +4411,13 @@ static void writeRuleText(FILE *out, struct rule *rp){ } } +/* +** Return true if the string is not NULL and not empty. +*/ +static int notnull(const char *z){ + return z && z[0]; +} + /* Generate C source code for the parser */ void ReportTable( @@ -4414,7 +4425,7 @@ void ReportTable( int mhflag, /* Output in makeheaders format if true */ int sqlFlag /* Generate the *.sql file too */ ){ - FILE *out, *in, *sql; + FILE *out, *in; int lineno; struct state *stp; struct action *ap; @@ -4439,18 +4450,10 @@ void ReportTable( in = tplt_open(lemp); if( in==0 ) return; - out = file_open(lemp,".c","wb"); - if( out==0 ){ - fclose(in); - return; - } - if( sqlFlag==0 ){ - sql = 0; - }else{ - sql = file_open(lemp, ".sql", "wb"); + if( sqlFlag ){ + FILE *sql = file_open(lemp, ".sql", "wb"); if( sql==0 ){ fclose(in); - fclose(out); return; } fprintf(sql, @@ -4515,6 +4518,12 @@ void ReportTable( } } fprintf(sql, "COMMIT;\n"); + fclose(sql); + } + out = file_open(lemp,".c","wb"); + if( out==0 ){ + fclose(in); + return; } lineno = 1; @@ -4543,7 +4552,7 @@ void ReportTable( } } if( lemp->include[0]=='/' ){ - tplt_skip_header(in,&lineno); + tplt_skip_header(in); }else{ tplt_xfer(lemp->name,in,out,&lineno); } @@ -4563,7 +4572,7 @@ void ReportTable( if( mhflag ){ fprintf(out,"#if INTERFACE\n"); lineno++; }else{ - fprintf(out,"#ifndef %s%s\n", prefix, lemp->symbols[1]->name); + fprintf(out,"#ifndef %s%s\n", prefix, lemp->symbols[1]->name); lineno++; } for(i=1; i<lemp->nterminal; i++){ fprintf(out,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i); @@ -4612,25 +4621,33 @@ void ReportTable( fprintf(out,"#define %sARG_FETCH\n",name); lineno++; fprintf(out,"#define %sARG_STORE\n",name); lineno++; } + fprintf(out, "#undef YYREALLOC\n"); lineno++; if( lemp->reallocFunc ){ fprintf(out,"#define YYREALLOC %s\n", lemp->reallocFunc); lineno++; }else{ fprintf(out,"#define YYREALLOC realloc\n"); lineno++; } + fprintf(out, "#undef YYFREE\n"); lineno++; if( lemp->freeFunc ){ fprintf(out,"#define YYFREE %s\n", lemp->freeFunc); lineno++; }else{ fprintf(out,"#define YYFREE free\n"); lineno++; } + fprintf(out, "#undef YYDYNSTACK\n"); lineno++; if( lemp->reallocFunc && lemp->freeFunc ){ fprintf(out,"#define YYDYNSTACK 1\n"); lineno++; }else{ fprintf(out,"#define YYDYNSTACK 0\n"); lineno++; } - if( lemp->ctx && lemp->ctx[0] ){ + fprintf(out, "#undef YYSIZELIMIT\n"); lineno++; + if( notnull(lemp->ctx) ){ i = lemonStrlen(lemp->ctx); while( i>=1 && ISSPACE(lemp->ctx[i-1]) ) i--; while( i>=1 && (ISALNUM(lemp->ctx[i-1]) || lemp->ctx[i-1]=='_') ) i--; + if( notnull(lemp->stackSizeLimit) ){ + fprintf(out,"#define YYSIZELIMIT %s\n", lemp->stackSizeLimit); lineno++; + } + fprintf(out,"#define %sCTX(P) ((P)->%s)\n",name,&lemp->ctx[i]); lineno++; fprintf(out,"#define %sCTX_SDECL %s;\n",name,lemp->ctx); lineno++; fprintf(out,"#define %sCTX_PDECL ,%s\n",name,lemp->ctx); lineno++; fprintf(out,"#define %sCTX_PARAM ,%s\n",name,&lemp->ctx[i]); lineno++; @@ -4639,6 +4656,7 @@ void ReportTable( fprintf(out,"#define %sCTX_STORE yypParser->%s=%s;\n", name,&lemp->ctx[i],&lemp->ctx[i]); lineno++; }else{ + fprintf(out,"#define %sCTX(P) 0\n",name); lineno++; fprintf(out,"#define %sCTX_SDECL\n",name); lineno++; fprintf(out,"#define %sCTX_PDECL\n",name); lineno++; fprintf(out,"#define %sCTX_PARAM\n",name); lineno++; @@ -4648,10 +4666,13 @@ void ReportTable( if( mhflag ){ fprintf(out,"#endif\n"); lineno++; } + fprintf(out, "#undef YYERRORSYMBOL\n"); lineno++; + fprintf(out, "#undef YYERRSYMDT\n"); lineno++; if( lemp->errsym && lemp->errsym->useCnt ){ fprintf(out,"#define YYERRORSYMBOL %d\n",lemp->errsym->index); lineno++; fprintf(out,"#define YYERRSYMDT yy%d\n",lemp->errsym->dtnum); lineno++; } + fprintf(out,"#undef YYFALLBACK\n"); lineno++; if( lemp->has_fallback ){ fprintf(out,"#define YYFALLBACK 1\n"); lineno++; } @@ -5003,7 +5024,6 @@ void ReportTable( sp2->destLineno = -1; /* Avoid emitting this destructor again */ } } - emit_destructor_code(out,lemp->symbols[i],lemp,&lineno); fprintf(out," break;\n"); lineno++; } @@ -5103,7 +5123,6 @@ void ReportTable( acttab_free(pActtab); fclose(in); fclose(out); - if( sql ) fclose(sql); return; } diff --git a/tool/lempar.c b/tool/lempar.c index 74314efea..7b654e650 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -299,15 +299,24 @@ static int yyGrowStack(yyParser *p){ int newSize; int idx; yyStackEntry *pNew; +#ifdef YYSIZELIMIT + int nLimit = YYSIZELIMIT(ParseCTX(p)); +#endif newSize = oldSize*2 + 100; +#ifdef YYSIZELIMIT + if( newSize>nLimit ){ + newSize = nLimit; + if( newSize<=oldSize ) return 1; + } +#endif idx = (int)(p->yytos - p->yystack); if( p->yystack==p->yystk0 ){ - pNew = YYREALLOC(0, newSize*sizeof(pNew[0])); + pNew = YYREALLOC(0, newSize*sizeof(pNew[0]), ParseCTX(p)); if( pNew==0 ) return 1; memcpy(pNew, p->yystack, oldSize*sizeof(pNew[0])); }else{ - pNew = YYREALLOC(p->yystack, newSize*sizeof(pNew[0])); + pNew = YYREALLOC(p->yystack, newSize*sizeof(pNew[0]), ParseCTX(p)); if( pNew==0 ) return 1; } p->yystack = pNew; @@ -459,7 +468,9 @@ void ParseFinalize(void *p){ } #if YYGROWABLESTACK - if( pParser->yystack!=pParser->yystk0 ) YYFREE(pParser->yystack); + if( pParser->yystack!=pParser->yystk0 ){ + YYFREE(pParser->yystack, ParseCTX(pParser)); + } #endif } diff --git a/tool/mkautoconfamal.sh b/tool/mkautoconfamal.sh index 3835799a6..002a3b8ee 100644 --- a/tool/mkautoconfamal.sh +++ b/tool/mkautoconfamal.sh @@ -59,7 +59,8 @@ cp $TOP/src/sqlite3.rc $TMPSPACE cp $TOP/tool/Replace.cs $TMPSPACE cp $TOP/VERSION $TMPSPACE cp $TOP/main.mk $TMPSPACE - +cp $TOP/make.bat $TMPSPACE +tree $TMPSPACE cd $TMPSPACE #if true; then diff --git a/tool/mkcombo.tcl b/tool/mkcombo.tcl new file mode 100644 index 000000000..71368ec41 --- /dev/null +++ b/tool/mkcombo.tcl @@ -0,0 +1,106 @@ +#!/usr/bin/tclsh +# +# Use this script to combine multiple source code files into a single +# file. Example: +# +# tclsh mkcombo.tcl file1.c file2.c file3.c -o file123.c +# + +set help {Usage: tclsh mkcombo.tcl [OPTIONS] [FILELIST] + where OPTIONS is zero or more of the following with these effects: + --linemacros=? => Emit #line directives into output or not. (? = 1 or 0) + --o FILE => write to alternative output file named FILE + --help => See this. +} + +set linemacros 0 +set fname {} +set src [list] + + +for {set i 0} {$i<[llength $argv]} {incr i} { + set x [lindex $argv $i] + if {[regexp {^-?-linemacros(?:=([01]))?$} $x ma ulm]} { + if {$ulm == ""} {set ulm 1} + set linemacros $ulm + } elseif {[regexp {^-o$} $x]} { + incr i + if {$i==[llength $argv]} { + error "No argument following $x" + } + set fname [lindex $argv $i] + } elseif {[regexp {^-?-((help)|\?)$} $x]} { + puts $help + exit 0 + } elseif {[regexp {^-?-} $x]} { + error "unknown command-line option: $x" + } else { + lappend src $x + } +} + +# Open the output file and write a header comment at the beginning +# of the file. +# +if {![info exists fname]} { + set fname sqlite3.c + if {$enable_recover} { set fname sqlite3r.c } +} +set out [open $fname wb] + +# Return a string consisting of N "*" characters. +# +proc star N { + set r {} + for {set i 0} {$i<$N} {incr i} {append r *} + return $r +} + +# Force the output to use unix line endings, even on Windows. +fconfigure $out -translation binary +puts $out "/[star 78]" +puts $out {** The following is an amalgamation of these source code files:} +puts $out {**} +foreach s $src { + regsub {^.*/(src|ext)/} $s {\1/} s2 + puts $out "** $s2" +} +puts $out {**} +puts $out "[star 78]/" + +# Insert a comment into the code +# +proc section_comment {text} { + global out s78 + set n [string length $text] + set nstar [expr {60 - $n}] + puts $out "/************** $text [star $nstar]/" +} + +# Read the source file named $filename and write it into the +# sqlite3.c output file. The only transformation is the trimming +# of EOL whitespace. +# +proc copy_file_verbatim {filename} { + global out + set in [open $filename rb] + set tail [file tail $filename] + section_comment "Begin file $tail" + while {![eof $in]} { + set line [string trimright [gets $in]] + puts $out $line + } + section_comment "End of $tail" +} +set taillist "" +foreach file $src { + copy_file_verbatim $file + append taillist ", [file tail $file]" +} + +set taillist "End of the amalgamation of [string range $taillist 2 end]" +set n [string length $taillist] +set ns [expr {(75-$n)/2}] +if {$ns<3} {set ns 3} +puts $out "/[star $ns] $taillist [star $ns]/" +close $out diff --git a/tool/mkkeywordhash.c b/tool/mkkeywordhash.c index 188c0a29a..d6d54a5a4 100644 --- a/tool/mkkeywordhash.c +++ b/tool/mkkeywordhash.c @@ -668,8 +668,8 @@ int main(int argc, char **argv){ printf("/* Check to see if z[0..n-1] is a keyword. If it is, write the\n"); printf("** parser symbol code for that keyword into *pType. Always\n"); printf("** return the integer n (the length of the token). */\n"); - printf("static int keywordCode(const char *z, int n, int *pType){\n"); - printf(" int i, j;\n"); + printf("static i64 keywordCode(const char *z, i64 n, int *pType){\n"); + printf(" i64 i, j;\n"); printf(" const char *zKW;\n"); printf(" assert( n>=2 );\n"); printf(" i = ((charMap(z[0])*%d) %c", HASH_C0, HASH_CC); diff --git a/tool/mkshellc.tcl b/tool/mkshellc.tcl index 2f7a6ea25..45452621b 100644 --- a/tool/mkshellc.tcl +++ b/tool/mkshellc.tcl @@ -13,28 +13,90 @@ set topdir [file dir [file dir [file normal $argv0]]] set out stdout fconfigure stdout -translation binary if {[lindex $argv 0]!=""} { - set out [open [lindex $argv 0] wb] + set output_file [lindex $argv 0] + file delete -force $output_file + set out [open $output_file wb] +} else { + set output_file {} } -puts $out {/* DO NOT EDIT! -** This file is automatically generated by the script in the canonical -** SQLite source tree at tool/mkshellc.tcl. That script combines source -** code from various constituent source files of SQLite into this single -** "shell.c" file used to implement the SQLite command-line shell. -** -** Most of the code found below comes from the "src/shell.c.in" file in -** the canonical SQLite source tree. That main file contains "INCLUDE" -** lines that specify other files in the canonical source tree that are -** inserted to getnerate this complete program source file. -** -** The code from multiple files is combined into this single "shell.c" -** source file to help make the command-line program easier to compile. -** -** To modify this program, get a copy of the canonical SQLite source tree, -** edit the src/shell.c.in" and/or some of the other files that are included -** by "src/shell.c.in", then rerun the tool/mkshellc.tcl script. -*/} + +############################## FIRST PASS ################################ +# Read through the shell.c.in source file to gather information. Do not +# yet generate any code +# set in [open $topdir/src/shell.c.in] fconfigure $in -translation binary +set allSource(src/shell.c.in) 1 +set inUsage 0 +set dotcmd {} +while {1} { + set lx [gets $in] + if {[eof $in]} break; + if {[regexp {^INCLUDE } $lx]} { + set cfile [lindex $lx 1] + if {[string match ../* $cfile]} { + set xfile [string range $cfile 3 end] + } else { + set xfile "src/$cfile" + } + set allSource($xfile) 1 + } elseif {[regexp {^\*\* USAGE:\s+([^\s]+)} $lx all dotcmd]} { + set inUsage 1 + set details [string trim [string range $lx 2 end]] + } elseif {$inUsage} { + if {![regexp {^\*\*} $lx] || [regexp { DOT-COMMAND: } $lx]} { + set inUsage 0 + set Usage($dotcmd) [string trim $details] + } else { + append details \n + append details [string range [string trimright $lx] 3 end] + } + } +} + +# Generate dot-command usage text based on the data accumulated in +# the Usage() array. +# +proc generate_usage {out} { + global Usage + puts $out "/**************************************************************" + puts $out "** \"Usage\" help text automatically generated from comments */" + puts $out "static const struct \173" + puts $out " const char *zCmd; /* Name of the dot-command */" + puts $out " const char *zUsage; /* Documentation */" + puts $out "\175 aUsage\[\] = \173" + foreach dotcmd [array names Usage] { + puts $out " \173 \"$dotcmd\"," + foreach line [split $Usage($dotcmd) \n] { + set x [string map [list \\ \\\\ \" \\\"] $line] + puts $out "\"$x\\n\"" + } + puts $out " \175," + } + puts $out "\175;" +} +# generate_usage stderr + +###### SECOND PASS ####### +# Make a second pass through shell.c.in to generate the the final +# output, based on data gathered during the first pass. +# + +puts $out {/* +** This is the amalgamated source code to the "sqlite3" or "sqlite3.exe" +** command-line shell (CLI) for SQLite. This file is automatically +** generated by the tool/mkshellc.tcl script from the following sources: +**} +foreach fn [lsort [array names allSource]] { + puts $out "** $fn" +} +puts $out {** +** To modify this program, get a copy of the canonical SQLite source tree, +** edit the src/shell.c.in file and/or some of the other files that are +** listed above, then rerun the command "make shell.c". +*/} +seek $in 0 start +puts $out "/************************* Begin src/shell.c.in ******************/" proc omit_redundant_typedefs {line} { global typedef_seen if {[regexp {^typedef .* ([a-zA-Z0-9_]+);} $line all typename]} { @@ -53,9 +115,14 @@ while {1} { incr iLine if {[regexp {^INCLUDE } $lx]} { set cfile [lindex $lx 1] - puts $out "/************************* Begin $cfile ******************/" -# puts $out "#line 1 \"$cfile\"" - set in2 [open $topdir/src/$cfile] + if {[string match ../* $cfile]} { + set xfile [string range $cfile 3 end] + } else { + set xfile "src/$cfile" + } + puts $out "/************************* Begin $xfile ******************/" +# puts $out "#line 1 \"$xfile\"" + set in2 [open $topdir/$xfile] fconfigure $in2 -translation binary while {![eof $in2]} { set lx [omit_redundant_typedefs [gets $in2]] @@ -69,11 +136,14 @@ while {1} { puts $out $lx } close $in2 - puts $out "/************************* End $cfile ********************/" + puts $out "/************************* End $xfile ********************/" # puts $out "#line [expr $iLine+1] \"shell.c.in\"" - continue + } elseif {[regexp {^INSERT-USAGE-TEXT-HERE} $lx]} { + generate_usage $out + } else { + puts $out $lx } - puts $out $lx } +puts $out "/************************* End src/shell.c.in ******************/" close $in close $out diff --git a/tool/omittest.tcl b/tool/omittest.tcl index 0452a4c6f..03c9220cd 100644 --- a/tool/omittest.tcl +++ b/tool/omittest.tcl @@ -123,7 +123,6 @@ set CompileOptionsToTest { SQLITE_ENABLE_MEMSYS SQLITE_ENABLE_MODULE_COMMENTS SQLITE_ENABLE_MULTIPLEX - SQLITE_ENABLE_MULTITHREADED_CHECKS SQLITE_ENABLE_NORMALIZE SQLITE_ENABLE_NULL_TRIM SQLITE_ENABLE_OFFSET_SQL_FUNC @@ -152,6 +151,7 @@ set CompileOptionsToTest { SQLITE_ENABLE_VFSTRACE SQLITE_ENABLE_WHERETRACE SQLITE_ENABLE_ZIPVFS + SQLITE_THREAD_MISUSE_WARNINGS } # Parse command-line options. diff --git a/tool/showdb.c b/tool/showdb.c index f0bd9737c..84813b839 100644 --- a/tool/showdb.c +++ b/tool/showdb.c @@ -8,6 +8,7 @@ #include <sys/types.h> #include <sys/stat.h> #include <fcntl.h> +#include <stdint.h> #if !defined(_MSC_VER) #include <unistd.h> @@ -28,13 +29,21 @@ typedef sqlite3_uint64 u64; /* unsigned 64-bit */ static struct GlobalData { i64 pagesize; /* Size of a database page */ + i64 usablesize; /* pagesize-nRes */ int dbfd; /* File descriptor for reading the DB */ u32 mxPage; /* Last page number */ + u32 nRes; /* Amount of reserve space */ int perLine; /* HEX elements to print per line */ int bRaw; /* True to access db file via OS APIs */ + int bCSV; /* CSV output for "pgidx" */ + int bTmstmp; /* Interpret tmstmpvfs tags on "pgidx" */ sqlite3_file *pFd; /* File descriptor for non-raw mode */ sqlite3 *pDb; /* Database handle that owns pFd */ -} g = {1024, -1, 0, 16, 0, 0, 0}; + char **zPageUse; /* Use for each page */ + struct TmstmpTag { + unsigned char a[16]; /* tmstmpvfs tag for each page */ + } *aPageTag; +} g = {4096, 4096, -1, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0}; /* ** Convert the var-int format into i64. Return the number of bytes @@ -142,11 +151,12 @@ static void fileClose(){ static unsigned char *fileRead(sqlite3_int64 ofst, int nByte){ unsigned char *aData; int got; + int rc; aData = sqlite3_malloc64(32+(i64)nByte); if( aData==0 ) out_of_memory(); memset(aData, 0, nByte+32); if( g.bRaw==0 ){ - int rc = g.pFd->pMethods->xRead(g.pFd, (void*)aData, nByte, ofst); + rc = g.pFd->pMethods->xRead(g.pFd, (void*)aData, nByte, ofst); if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ fprintf(stderr, "error in xRead() - %d\n", rc); exit(1); @@ -154,7 +164,21 @@ static unsigned char *fileRead(sqlite3_int64 ofst, int nByte){ }else{ lseek(g.dbfd, (long)ofst, SEEK_SET); got = read(g.dbfd, aData, nByte); - if( got>0 && got<nByte ) memset(aData+got, 0, nByte-got); + if( got==nByte ){ + rc = SQLITE_OK; + }else if( got>0 && got<nByte ){ + memset(aData+got, 0, nByte-got); + rc = SQLITE_IOERR_SHORT_READ; + }else{ + memset(aData,0,nByte); + rc = SQLITE_IOERR; + } + } + if( g.aPageTag && nByte==(int)g.pagesize ){ + unsigned int pgno = (unsigned int)(ofst/g.pagesize) + 1; + if( pgno>0 && pgno<=g.mxPage ){ + memcpy(g.aPageTag[pgno].a, &aData[nByte-16], 16); + } } return aData; } @@ -382,14 +406,14 @@ static i64 localPayload(i64 nPayload, char cType){ i64 nLocal; if( cType==13 ){ /* Table leaf */ - maxLocal = g.pagesize-35; - minLocal = (g.pagesize-12)*32/255-23; + maxLocal = g.usablesize-35; + minLocal = (g.usablesize-12)*32/255-23; }else{ - maxLocal = (g.pagesize-12)*64/255-23; - minLocal = (g.pagesize-12)*32/255-23; + maxLocal = (g.usablesize-12)*64/255-23; + minLocal = (g.usablesize-12)*32/255-23; } if( nPayload>maxLocal ){ - surplus = minLocal + (nPayload-minLocal)%(g.pagesize-4); + surplus = minLocal + (nPayload-minLocal)%(g.usablesize-4); if( surplus<=maxLocal ){ nLocal = surplus; }else{ @@ -751,7 +775,7 @@ static void decode_trunk_page( print_decode_line(a, 4, 4, "Number of entries on this page"); if( detail ){ n = decodeInt32(&a[4]); - for(i=0; i<n && i<g.pagesize/4; i++){ + for(i=0; i<n && i<g.usablesize/4; i++){ u32 x = decodeInt32(&a[8+4*i]); char zIdx[13]; sprintf(zIdx, "[%d]", i); @@ -769,11 +793,6 @@ static void decode_trunk_page( } } -/* -** A short text comment on the use of each page. -*/ -static char **zPageUse; - /* ** Add a comment on the use of a page. */ @@ -790,13 +809,13 @@ static void page_usage_msg(u32 pgno, const char *zFormat, ...){ sqlite3_free(zMsg); return; } - if( zPageUse[pgno]!=0 ){ + if( g.zPageUse[pgno]!=0 ){ printf("ERROR: page %d used multiple times:\n", pgno); - printf("ERROR: previous: %s\n", zPageUse[pgno]); + printf("ERROR: previous: %s\n", g.zPageUse[pgno]); printf("ERROR: current: %s\n", zMsg); - sqlite3_free(zPageUse[pgno]); + sqlite3_free(g.zPageUse[pgno]); } - zPageUse[pgno] = zMsg; + g.zPageUse[pgno] = zMsg; } /* @@ -837,7 +856,7 @@ static void page_usage_cell( while( ovfl && (cnt++)<g.mxPage ){ page_usage_msg(ovfl, "overflow %d from cell %d of page %u", cnt, cellno, pgno); - a = fileRead((ovfl-1)*(sqlite3_int64)g.pagesize, 4); + a = fileRead((ovfl-1)*(sqlite3_int64)g.pagesize, g.pagesize); ovfl = decodeInt32(a); sqlite3_free(a); } @@ -916,12 +935,12 @@ static void page_usage_btree( u32 ofst; cellidx = cellstart + i*2; - if( cellidx+1 >= g.pagesize ){ + if( cellidx+1 >= g.usablesize ){ printf("ERROR: page %d too many cells (%d)\n", pgno, nCell); break; } ofst = a[cellidx]*256 + a[cellidx+1]; - if( ofst<cellidx+2 || ofst+4>=g.pagesize ){ + if( ofst<cellidx+2 || ofst+4>=g.usablesize ){ printf("ERROR: page %d cell %d out of bounds\n", pgno, i); continue; } @@ -959,9 +978,9 @@ static void page_usage_freelist(u32 pgno){ a = fileRead((pgno-1)*g.pagesize, g.pagesize); iNext = decodeInt32(a); n = decodeInt32(a+4); - if( n>(g.pagesize - 8)/4 ){ + if( n>(g.usablesize - 8)/4 ){ printf("ERROR: page %d too many freelist entries (%d)\n", pgno, n); - n = (g.pagesize - 8)/4; + n = (g.usablesize - 8)/4; } for(i=0; i<n; i++){ int child = decodeInt32(a + (i*4+8)); @@ -990,6 +1009,63 @@ static void page_usage_ptrmap(u8 *a){ } } +/* +** The six bytes at a[] are a big-endian unsigned integer which is the +** number of milliseconds since 1970. Decode that value into an ISO 8601 +** date/time string stored in static space and return a pointer to that +** string. +*/ +static const char *decodeTimestamp(const unsigned char *a){ + uint64_t ms; /* Milliseconds since 1970 */ + uint64_t days; /* Days since 1970-01-01 */ + uint64_t sod; /* Start of date specified by ms */ + uint64_t z; /* Days since 0000-03-01 */ + uint64_t era; /* 400-year era */ + int i; /* Loop counter */ + int h; /* hour */ + int m; /* minute */ + int s; /* second */ + int f; /* millisecond */ + int Y; /* year */ + int M; /* month */ + int D; /* day */ + int y; /* year assuming March is first month */ + unsigned int doe; /* day of 400-year era */ + unsigned int yoe; /* year of 400-year era */ + unsigned int doy; /* day of year */ + unsigned int mp; /* month with March==0 */ + static char zOut[50]; /* Return results here */ + + for(ms=0, i=0; i<=5; i++) ms = (ms<<8) + a[i]; + if( ms==0 ){ + return " "; + }else if( ms>4102444800000LL ){ /* 2100-01-01 */ + /* YYYY-MM-DD HH:MM:SS.SSS */ + return " (bad date) "; + } + days = ms/86400000; + sod = (ms%86400000)/1000; + f = (int)(ms%1000); + + h = sod/3600; + m = (sod%3600)/60; + s = sod%60; + z = days + 719468; + era = z/146097; + doe = (unsigned)(z - era*146097); + yoe = (doe - doe/1460 + doe/36524 - doe/146096)/365; + y = (int)yoe + era*400; + doy = doe - (365*yoe + yoe/4 - yoe/100); + mp = (5*doy + 2)/153; + D = doy - (153*mp + 2)/5 + 1; + M = mp + (mp<10 ? 3 : -9); + Y = y + (M <=2); + snprintf(zOut, sizeof(zOut), + "%04d-%02d-%02d %02d:%02d:%02d.%03d", + Y, M, D, h, m, s, f); + return zOut; +} + /* ** Try to figure out how every page in the database file is being used. */ @@ -1010,14 +1086,22 @@ static void page_usage_report(const char *zPrg, const char *zDbName){ /* Open the database file */ db = openDatabase(zPrg, zDbName); - /* Set up global variables zPageUse[] and g.mxPage to record page + /* Set up global variables g.zPageUse[] and g.mxPage to record page ** usages */ - zPageUse = sqlite3_malloc64( sizeof(zPageUse[0])*(g.mxPage+1) ); - if( zPageUse==0 ) out_of_memory(); - memset(zPageUse, 0, sizeof(zPageUse[0])*(g.mxPage+1)); + g.zPageUse = sqlite3_malloc64( sizeof(g.zPageUse[0])*(g.mxPage+1) ); + if( g.zPageUse==0 ) out_of_memory(); + memset(g.zPageUse, 0, sizeof(g.zPageUse[0])*(g.mxPage+1)); /* Discover the usage of each page */ a = fileRead(0, 100); + if( g.bTmstmp && a[20]==16 ){ + g.aPageTag = sqlite3_malloc64( sizeof(struct TmstmpTag)*(g.mxPage+1) ); + if( g.aPageTag==0 ) out_of_memory(); + memset(g.aPageTag, 0, sizeof(struct TmstmpTag)*(g.mxPage+1) ); + }else{ + g.bTmstmp = 0; + g.aPageTag = 0; + } page_usage_freelist(decodeInt32(a+32)); page_usage_ptrmap(a); sqlite3_free(a); @@ -1042,15 +1126,68 @@ static void page_usage_report(const char *zPrg, const char *zDbName){ sqlite3_close(db); /* Print the report and free memory used */ + if( g.bCSV ){ + if( g.bTmstmp ){ + printf("pgno,tm,frame,flg,salt,parent,child,ovfl,txt\r\n"); + }else{ + printf("pgno,parent,child,ovfl,txt\r\n"); + } + } for(i=1; i<=g.mxPage; i++){ - if( zPageUse[i]==0 ) page_usage_btree(i, -1, 0, 0); - printf("%5u: %s\n", i, zPageUse[i] ? zPageUse[i] : "???"); + if( g.zPageUse[i]==0 ){ + g.zPageUse[i] = sqlite3_mprintf("???"); + if( g.zPageUse[i]==0 ) continue; + } + if( g.bCSV ){ + const char *z = g.zPageUse[i]; + const char *s; + printf("%u,", i); + if( g.bTmstmp ){ + const unsigned char *a = g.aPageTag[i].a; + sqlite3_uint64 tm = 0; + unsigned int x; + int k; + for(k=2; k<=7; k++) tm = (tm<<8)+a[k]; + printf("%llu.%03u,", tm/1000, (unsigned int)(tm%1000)); + for(x=0, k=8; k<=11; k++) x = (x<<8)+a[k]; + printf("%u,", x); + printf("%u,", a[12]); + for(x=0, k=13; k<=15; k++) x = (x<<8)+a[k]; + printf("%u,", x); + } + if( (s = strstr(z, " of page "))!=0 ){ + printf("%d,", atoi(s+9)); + }else if( (s = strstr(z, " of trunk page "))!=0 ){ + printf("%d,", atoi(s+15)); + }else{ + printf("0,"); + } + if( (s = strstr(z, "], child "))!=0 ){ + printf("%d,", atoi(s+9)); + }else if( (s = strstr(z, " from cell "))!=0 ){ + printf("%d,", atoi(s+12)); + }else{ + printf("-1,"); + } + if( strncmp(z,"overflow ", 9)==0 ){ + printf("%d,", atoi(z+9)); + }else{ + printf("-1,"); + } + printf("\"%s\"\r\n", z); + }else if( g.bTmstmp ){ + printf("%5u: %s %s\n", i, + decodeTimestamp(&g.aPageTag[i].a[2]), + g.zPageUse[i]); + }else{ + printf("%5u: %s\n", i, g.zPageUse[i]); + } } for(i=1; i<=g.mxPage; i++){ - sqlite3_free(zPageUse[i]); + sqlite3_free(g.zPageUse[i]); } - sqlite3_free(zPageUse); - zPageUse = 0; + sqlite3_free(g.zPageUse); + g.zPageUse = 0; } /* @@ -1083,7 +1220,7 @@ static void ptrmap_coverage_report(const char *zDbName){ for(pgno=2; pgno<=g.mxPage; pgno += perPage+1){ printf("%5llu: PTRMAP page covering %llu..%llu\n", pgno, pgno+1, pgno+perPage); - a = fileRead((pgno-1)*g.pagesize, usable); + a = fileRead((pgno-1)*g.pagesize, g.pagesize); for(i=0; i+5<=usable; i+=5){ const char *zType; u32 iFrom = decodeInt32(&a[i+1]); @@ -1115,7 +1252,7 @@ static void ptrmap_coverage_report(const char *zDbName){ ** Check the range validity for a page number. Print an error and ** exit if the page is out of range. */ -static void checkPageValidity(int iPage){ +static void checkPageValidity(unsigned int iPage){ if( iPage<1 || iPage>g.mxPage ){ fprintf(stderr, "Invalid page number %d: valid range is 1..%d\n", iPage, g.mxPage); @@ -1130,7 +1267,9 @@ static void usage(const char *argv0){ fprintf(stderr, "Usage %s ?--uri? FILENAME ?args...?\n\n", argv0); fprintf(stderr, "switches:\n" + " --csv CSV output for \"pgidx\"\n" " --raw Read db file directly, bypassing SQLite VFS\n" + " --tmstmp Interpret tmstmpvfs tags\n" "args:\n" " dbheader Show database header\n" " pgidx Index of how each page is used\n" @@ -1154,14 +1293,28 @@ int main(int argc, char **argv){ char **azArg = argv; int nArg = argc; - /* Check for the "--uri" or "-uri" switch. */ - if( nArg>1 ){ - if( sqlite3_stricmp("-raw", azArg[1])==0 - || sqlite3_stricmp("--raw", azArg[1])==0 - ){ + /* Check for the switches. */ + while( nArg>1 && azArg[1][0]=='-' ){ + const char *z = azArg[1]; + if( z[1]=='-' && z[2]!=0 ) z++; + if( sqlite3_stricmp("-raw", z)==0 ){ g.bRaw = 1; azArg++; nArg--; + }else + if( strcmp("-csv", z)==0 ){ + g.bCSV = 1; + azArg++; + nArg--; + }else + if( strcmp("-tmstmp", z)==0 ){ + g.bTmstmp = 1; + azArg++; + nArg--; + }else + { + usage(zPrg); + exit(1); } } @@ -1173,15 +1326,19 @@ int main(int argc, char **argv){ fileOpen(zPrg, azArg[1]); szFile = fileGetsize(); - zPgSz = fileRead(16, 2); - g.pagesize = zPgSz[0]*256 + zPgSz[1]*65536; - if( g.pagesize==0 ) g.pagesize = 1024; + zPgSz = fileRead(0, 24); + g.pagesize = zPgSz[16]*256 + zPgSz[17]*65536; + if( g.pagesize==0 ) g.pagesize = 4096; + g.nRes = zPgSz[20]; + g.usablesize = g.pagesize - g.nRes; sqlite3_free(zPgSz); - - printf("Pagesize: %d\n", (int)g.pagesize); g.mxPage = (u32)((szFile+g.pagesize-1)/g.pagesize); - printf("Available pages: 1..%u\n", g.mxPage); + if( !g.bCSV ){ + printf("Pagesize: %d\n", (int)g.pagesize); + if( g.nRes ) printf("Useable-size: %d\n", (int)g.usablesize); + printf("Available pages: 1..%u\n", g.mxPage); + } if( nArg==2 ){ u32 i; for(i=1; i<=g.mxPage; i++) print_page(i); diff --git a/tool/showtmlog.c b/tool/showtmlog.c new file mode 100644 index 000000000..4c35b777b --- /dev/null +++ b/tool/showtmlog.c @@ -0,0 +1,254 @@ +/* +** A utility program to decode tmstmpvfs log files. +*/ +#include <stdio.h> +#include <assert.h> +#include <string.h> +#include <stdint.h> +#include <stdlib.h> + +/* +** The six bytes at a[] are a big-endian unsigned integer which is the +** number of milliseconds since 1970. Decode that value into an ISO 8601 +** date/time string stored in static space and return a pointer to that +** string. +*/ +static const char *decodeTimestamp(const unsigned char *a){ + uint64_t ms; /* Milliseconds since 1970 */ + uint64_t days; /* Days since 1970-01-01 */ + uint64_t sod; /* Start of date specified by ms */ + uint64_t z; /* Days since 0000-03-01 */ + uint64_t era; /* 400-year era */ + int i; /* Loop counter */ + int h; /* hour */ + int m; /* minute */ + int s; /* second */ + int f; /* millisecond */ + int Y; /* year */ + int M; /* month */ + int D; /* day */ + int y; /* year assuming March is first month */ + unsigned int doe; /* day of 400-year era */ + unsigned int yoe; /* year of 400-year era */ + unsigned int doy; /* day of year */ + unsigned int mp; /* month with March==0 */ + static char zOut[50]; /* Return results here */ + + for(ms=0, i=0; i<=5; i++) ms = (ms<<8) + a[i]; + if( ms==0 ){ + return " "; + }else if( ms>4102444800000LL ){ /* 2100-01-01 */ + /* YYYY-MM-DD HH:MM:SS.SSS */ + return " (bad date) "; + } + days = ms/86400000; + sod = (ms%86400000)/1000; + f = (int)(ms%1000); + + h = sod/3600; + m = (sod%3600)/60; + s = sod%60; + z = days + 719468; + era = z/146097; + doe = (unsigned)(z - era*146097); + yoe = (doe - doe/1460 + doe/36524 - doe/146096)/365; + y = (int)yoe + era*400; + doy = doe - (365*yoe + yoe/4 - yoe/100); + mp = (5*doy + 2)/153; + D = doy - (153*mp + 2)/5 + 1; + M = mp + (mp<10 ? 3 : -9); + Y = y + (M <=2); + snprintf(zOut, sizeof(zOut), + "%04d-%02d-%02d %02d:%02d:%02d.%03d", + Y, M, D, h, m, s, f); + return zOut; +} + +/* +** Render a single 16-byte tmstmpvfs log record as a line to a CSV file. +** +** Columns: tmstmp,fileno,op,pid,pgno,frame,salt,txn +*/ +static void renderCSV(int iFile, unsigned char *a){ + unsigned int a2, a3; + int j; + uint64_t ms; + + for(ms=0, j=2; j<=7; j++) ms = (ms<<8) + a[j]; + printf("%u.%03u,%d,", (unsigned int)(ms/1000), (unsigned)(ms%1000), iFile); + for(a2=0, j=8; j<=11; j++) a2 = (a2<<8)+a[j]; + for(a3=0, j=12; j<=15; j++) a3 = (a3<<8)+a[j]; + switch( a[0] ){ + case 0x01: { + printf("\"open-db\",%u,,,,\r\n",a2); + break; + } + case 0x02: { + printf("\"open-wal\",%u,,,,\r\n", a2); + break; + } + case 0x03: { + printf("\"wal-page\",,%u,%u,,%d\r\n", a2, a3, a[1]); + break; + } + case 0x04: { + printf("\"db-page\",,%u,,,\r\n", a2); + break; + } + case 0x05: { + printf("\"ckpt-start\",,,,,\r\n"); + break; + } + case 0x06: { + printf("\"ckpt-page\",,%u,%u,,\r\n", a2, a3); + break; + } + case 0x07: { + printf("\"ckpt-end\",,,,,\r\n"); + break; + } + case 0x08: { + printf("\"wal-reset\",,,,%u,\r\n", a3); + break; + } + case 0x0e: { + printf("\"close-wal\",,,,,\r\n"); + break; + } + case 0x0f: { + printf("\"close-db\",,,,,\r\n"); + break; + } + default: { + printf("\"invalid-record\",,,,,\r\n"); + break; + } + } +} + +/* +** Render a single 16-byte tmstmpvfs log record as human-readable text +** on stdout. +*/ +static void renderText(unsigned char *a){ + unsigned int a2, a3; + int j; + + printf("%s ", decodeTimestamp(a+2)); + for(a2=0, j=8; j<=11; j++) a2 = (a2<<8)+a[j]; + for(a3=0, j=12; j<=15; j++) a3 = (a3<<8)+a[j]; + switch( a[0] ){ + case 0x01: { + printf("open-db pid %u\n", a2); + break; + } + case 0x02: { + printf("open-wal pid %u\n", a2); + break; + } + case 0x03: { + printf("wal-page pgno %-8u frame %-8u%s\n", a2, a3, + a[1]==1 ? " txn" : ""); + break; + } + case 0x04: { + printf("db-page pgno %-8u\n", a2); + break; + } + case 0x05: { + printf("ckpt-start\n"); + break; + } + case 0x06: { + printf("ckpt-page pgno %-8u frame %-8u\n", a2, a3); + break; + } + case 0x07: { + printf("ckpt-end\n"); + break; + } + case 0x08: { + printf("wal-reset salt1 0x%08x\n", a3); + break; + } + case 0x0e: { + printf("close-wal\n"); + break; + } + case 0x0f: { + printf("close-db\n"); + break; + } + default: { + printf("invalid-record\n"); + break; + } + } +} + +static void usage(const char *argv0){ + printf("Usage: %s [--csv] LOGFILE ...\n", argv0); + printf("Decode one or more tmstmpvfs log files and display the results\n" + "on stdout. Render as CSV if the --csv option is used.\n"); +} + +int main(int argc, char **argv){ + int i; + FILE *in; + unsigned char a[16]; + int bCSV = 0; + const char *z; + int nFile = 0; + int iFile; + for(i=1; i<argc; i++){ + z = argv[i]; + if( z[0]=='-' ){ + if( z[1]=='-' ) z++; + if( strcmp(z,"-csv")==0 ){ + bCSV = 1; + }else + if( strcmp(z,"-help")==0 || strcmp(z,"-?")==0 ){ + usage(argv[0]); + return 0; + }else + { + printf("unknown command-line option: \"%s\"\n", + argv[i]); + usage(argv[0]); + return 1; + } + }else{ + nFile++; + } + } + if( nFile==0 ){ + usage(argv[0]); + return 1; + } + iFile = 0; + if( bCSV ){ + printf("tmstmp,fileno,op,pid,pgno,frame,salt,txn\r\n"); + } + for(i=1; i<argc; i++){ + z = argv[i]; + if( z[0]=='-' ) continue; + in = fopen(z, "rb"); + if( in==0 ){ + printf("%s: can't open\n", z); + continue; + } + iFile++; + if( nFile>1 && !bCSV ){ + printf("*** %s ***\n", z); + } + while( 16==fread(a, 1, 16, in) ){ + if( bCSV ){ + renderCSV(iFile, a); + }else{ + renderText(a); + } + } + fclose(in); + } + return 0; +} diff --git a/tool/sqldiff.c b/tool/sqldiff.c index 44bf488f8..d27a62e14 100644 --- a/tool/sqldiff.c +++ b/tool/sqldiff.c @@ -28,6 +28,9 @@ #include "sqlite3.h" #include "sqlite3_stdio.h" +typedef sqlite3_int64 i64; +typedef sqlite3_uint64 u64; + /* ** All global variables are gathered into the "g" singleton. */ @@ -202,12 +205,12 @@ static char **columnNames( int *pbRowid /* OUT: True if PK is an implicit rowid */ ){ char **az = 0; /* List of column names to be returned */ - int naz = 0; /* Number of entries in az[] */ + i64 naz = 0; /* Number of entries in az[] */ sqlite3_stmt *pStmt; /* SQL statement being run */ char *zPkIdxName = 0; /* Name of the PRIMARY KEY index */ int truePk = 0; /* PRAGMA table_info identifies the PK to use */ - int nPK = 0; /* Number of PRIMARY KEY columns */ - int i, j; /* Loop counters */ + i64 nPK = 0; /* Number of PRIMARY KEY columns */ + i64 i, j; /* Loop counters */ if( g.bSchemaPK==0 ){ /* Normal case: Figure out what the true primary key is for the table. @@ -271,7 +274,7 @@ static char **columnNames( } *pnPKey = nPK; naz = nPK; - az = sqlite3_malloc( sizeof(char*)*(nPK+1) ); + az = sqlite3_malloc64( sizeof(char*)*(nPK+1) ); if( az==0 ) runtimeError("out of memory"); memset(az, 0, sizeof(char*)*(nPK+1)); if( g.bSchemaCompare ){ @@ -288,7 +291,7 @@ static char **columnNames( || !(strcmp(sid,"rootpage")==0 ||strcmp(sid,"name")==0 ||strcmp(sid,"type")==0)){ - az = sqlite3_realloc(az, sizeof(char*)*(naz+2) ); + az = sqlite3_realloc64(az, sizeof(char*)*(naz+2) ); if( az==0 ) runtimeError("out of memory"); az[naz++] = sid; } @@ -954,7 +957,7 @@ static int rbuDeltaCreate( unsigned int i, base; char *zOrigDelta = zDelta; hash h; - int nHash; /* Number of hash table entries */ + i64 nHash; /* Number of hash table entries */ int *landmark; /* Primary hash table */ int *collide; /* Collision chain */ int lastRead = -1; /* Last byte of zSrc read by a COPY command */ @@ -982,7 +985,7 @@ static int rbuDeltaCreate( ** source file. */ nHash = lenSrc/NHASH; - collide = sqlite3_malloc( nHash*2*sizeof(int) ); + collide = sqlite3_malloc64( nHash*2*sizeof(int) ); landmark = &collide[nHash]; memset(landmark, -1, nHash*sizeof(int)); memset(collide, -1, nHash*sizeof(int)); @@ -1286,9 +1289,9 @@ static void rbudiff_one_table(const char *zTab, FILE *out){ } }else{ char *zOtaControl; - int nOtaControl = sqlite3_column_bytes(pStmt, nCol); + i64 nOtaControl = sqlite3_column_bytes(pStmt, nCol); - zOtaControl = (char*)sqlite3_malloc(nOtaControl+1); + zOtaControl = (char*)sqlite3_malloc64(nOtaControl+1); memcpy(zOtaControl, sqlite3_column_text(pStmt, nCol), nOtaControl+1); for(i=0; i<nCol; i++){ @@ -1300,11 +1303,11 @@ static void rbudiff_one_table(const char *zTab, FILE *out){ const char *aSrc = sqlite3_column_blob(pStmt, nCol+1+i); int nSrc = sqlite3_column_bytes(pStmt, nCol+1+i); const char *aFinal = sqlite3_column_blob(pStmt, i); - int nFinal = sqlite3_column_bytes(pStmt, i); + i64 nFinal = sqlite3_column_bytes(pStmt, i); char *aDelta; int nDelta; - aDelta = sqlite3_malloc(nFinal + 60); + aDelta = sqlite3_malloc64(nFinal + 60); nDelta = rbuDeltaCreate(aSrc, nSrc, aFinal, nFinal, aDelta); if( nDelta<nFinal ){ int j; @@ -1549,10 +1552,10 @@ static void changeset_one_table(const char *zTab, FILE *out){ sqlite3_stmt *pStmt; /* SQL statment */ char *zId = safeId(zTab); /* Escaped name of the table */ char **azCol = 0; /* List of escaped column names */ - int nCol = 0; /* Number of columns */ + i64 nCol = 0; /* Number of columns */ int *aiFlg = 0; /* 0 if column is not part of PK */ int *aiPk = 0; /* Column numbers for each PK column */ - int nPk = 0; /* Number of PRIMARY KEY columns */ + i64 nPk = 0; /* Number of PRIMARY KEY columns */ sqlite3_str *pSql; /* SQL for the diff query */ int i, k; /* Loop counters */ const char *zSep; /* List separator */ @@ -1564,16 +1567,16 @@ static void changeset_one_table(const char *zTab, FILE *out){ pStmt = db_prepare("PRAGMA main.table_info=%Q", zTab); while( SQLITE_ROW==sqlite3_step(pStmt) ){ nCol++; - azCol = sqlite3_realloc(azCol, sizeof(char*)*nCol); + azCol = sqlite3_realloc64(azCol, sizeof(char*)*nCol); if( azCol==0 ) runtimeError("out of memory"); - aiFlg = sqlite3_realloc(aiFlg, sizeof(int)*nCol); + aiFlg = sqlite3_realloc64(aiFlg, sizeof(int)*nCol); if( aiFlg==0 ) runtimeError("out of memory"); azCol[nCol-1] = safeId((const char*)sqlite3_column_text(pStmt,1)); aiFlg[nCol-1] = i = sqlite3_column_int(pStmt,5); if( i>0 ){ if( i>nPk ){ nPk = i; - aiPk = sqlite3_realloc(aiPk, sizeof(int)*nPk); + aiPk = sqlite3_realloc64(aiPk, sizeof(int)*nPk); if( aiPk==0 ) runtimeError("out of memory"); } aiPk[i-1] = nCol-1; @@ -1896,6 +1899,11 @@ static void showHelp(void){ ); } +/* work-around the Microsoft "WorstFit" bug */ +#ifdef _WIN32 +#define main utf8_main +#endif + int main(int argc, char **argv){ const char *zDb1 = 0; const char *zDb2 = 0; @@ -1908,7 +1916,7 @@ int main(int argc, char **argv){ FILE *out = stdout; void (*xDiff)(const char*,FILE*) = diff_one_table; #ifndef SQLITE_OMIT_LOAD_EXTENSION - int nExt = 0; + i64 nExt = 0; char **azExt = 0; #endif int useTransaction = 0; diff --git a/tool/sqlite3_rsync.c b/tool/sqlite3_rsync.c index ad9f132bb..b10224b2f 100644 --- a/tool/sqlite3_rsync.c +++ b/tool/sqlite3_rsync.c @@ -32,6 +32,7 @@ static const char zUsage[] = "\n" " --exe PATH Name of the sqlite3_rsync program on the remote side\n" " --help Show this help screen\n" + " -p|--port PORT Run SSH over TCP port PORT instead of the default 22\n" " --protocol N Use sync protocol version N.\n" " --ssh PATH Name of the SSH program used to reach the remote side\n" " -v Verbose. Multiple v's for increasing output\n" @@ -324,15 +325,29 @@ static int popen2( ** Close the connection to a child process previously created using ** popen2(). */ -static void pclose2(FILE *pIn, FILE *pOut, int childPid){ +static int pclose2(FILE *pIn, FILE *pOut, int childPid){ #ifdef _WIN32 /* Not implemented, yet */ fclose(pIn); fclose(pOut); + return 0; #else + int wp, rc = 0; fclose(pIn); fclose(pOut); - while( waitpid(0, 0, WNOHANG)>0 ) {} + do{ + wp = waitpid(0, &rc, WNOHANG); + if( wp>0 ){ + if( WIFEXITED(rc) ){ + rc = WEXITSTATUS(rc); + }else if( WIFSIGNALED(rc) ){ + rc = WTERMSIG(rc); + }else{ + rc = 0/*???*/; + } + } + } while( wp>0 ); + return rc; #endif } /***************************************************************************** @@ -2054,6 +2069,7 @@ int main(int argc, char const * const *argv){ FILE *pOut = 0; int childPid = 0; const char *zSsh = "ssh"; + int iPort = 0; const char *zExe = "sqlite3_rsync"; char *zCmd = 0; sqlite3_int64 tmStart; @@ -2094,6 +2110,15 @@ int main(int argc, char const * const *argv){ zSsh = cli_opt_val; continue; } + if( strcmp(z, "-port")==0 || strcmp(z, "-p")==0 ){ + const char *zPort = cli_opt_val; + iPort = atoi(zPort); + if( iPort<1 || iPort>65535 ){ + fprintf(stderr, "invalid TCP port number: \"%s\"\n", zPort); + return 1; + } + continue; + } if( strcmp(z, "-exe")==0 ){ zExe = cli_opt_val; continue; @@ -2235,6 +2260,7 @@ int main(int argc, char const * const *argv){ for(iRetry=0; 1 /*exit-via-break*/; iRetry++){ sqlite3_str *pStr = sqlite3_str_new(0); append_escaped_arg(pStr, zSsh, 1); + if( iPort>0 ) sqlite3_str_appendf(pStr, " -p %d", iPort); sqlite3_str_appendf(pStr, " -e none"); append_escaped_arg(pStr, ctx.zOrigin, 0); if( iRetry ) add_path_argument(pStr); @@ -2283,6 +2309,7 @@ int main(int argc, char const * const *argv){ for(iRetry=0; 1 /*exit-by-break*/; iRetry++){ sqlite3_str *pStr = sqlite3_str_new(0); append_escaped_arg(pStr, zSsh, 1); + if( iPort>0 ) sqlite3_str_appendf(pStr, " -p %d", iPort); sqlite3_str_appendf(pStr, " -e none"); append_escaped_arg(pStr, ctx.zReplica, 0); if( iRetry==1 ) add_path_argument(pStr); @@ -2349,7 +2376,7 @@ int main(int argc, char const * const *argv){ } originSide(&ctx); } - pclose2(ctx.pIn, ctx.pOut, childPid); + ctx.nErr += !!pclose2(ctx.pIn, ctx.pOut, childPid); if( ctx.pLog ) fclose(ctx.pLog); tmEnd = currentTime(); tmElapse = tmEnd - tmStart; /* Elapse time in milliseconds */ diff --git a/tool/sqltclsh.c.in b/tool/sqltclsh.c.in index da354ee93..3e4d38b2d 100644 --- a/tool/sqltclsh.c.in +++ b/tool/sqltclsh.c.in @@ -33,7 +33,7 @@ INCLUDE $ROOT/ext/misc/appendvfs.c INCLUDE $ROOT/ext/misc/zipfile.c INCLUDE $ROOT/ext/misc/sqlar.c #endif -INCLUDE $ROOT/src/tclsqlite.c +INCLUDE tclsqlite-ex.c const char *sqlite3_tclapp_init_proc(Tcl_Interp *interp){ (void)interp; diff --git a/tool/winmain.c b/tool/winmain.c new file mode 100644 index 000000000..72ae00d5c --- /dev/null +++ b/tool/winmain.c @@ -0,0 +1,79 @@ +/* +** 2025-12-26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements a substitute process entry point for windows +** that correctly interprets command-line arguments as UTF-8. This is +** a work-around for the (unfixed) Windows bug known as "WorstFit" and +** described at: +** +** * https://news.ycombinator.com/item?id=42647101 +** +** USAGE: +** +** If you have a command-line program coded to C-language standard in which +** the entry point is: +** +** int main(int argc, char **argv){...} +** +** That does not work on Windows if there are non-ASCII characters on the +** command line. Programs are expected to use an alternative entry point: +** +** int wmain(int wargc, wchar_t **wargv){...} +** +** This file implements _wmain() but then converts all of the wargv elements +** to UTF-8 and passes them off to utf8_main(). +** +** So, a command-line tool that implements the standard entry point can be +** modified by adding the following lines just prior to main(): +** +** #ifdef _WIN32 +** #define main utf8_main +** #endif +** +** Then, include this file in the list of files that implement the program, +** and the program will work correctly, even on Windows. +*/ +#include <windows.h> +#include <stdio.h> + +extern int utf8_main(int,char**); +int wmain(int argc, wchar_t **wargv){ + int rc, i; + char **argv = malloc( sizeof(char*) * (argc+1) ); + char **orig = argv; + if( argv==0 ){ + fprintf(stderr, "malloc failed\n"); + exit(1); + } + for(i=0; i<argc; i++){ + int nByte = WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, 0, 0, 0, 0); + if( nByte==0 ){ + argv[i] = 0; + }else{ + argv[i] = malloc( nByte ); + if( argv[i]==0 ){ + fprintf(stderr, "malloc failed\n"); + exit(1); + } + nByte = WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, argv[i],nByte,0,0); + if( nByte==0 ){ + free(argv[i]); + argv[i] = 0; + } + } + } + argv[argc] = 0; + rc = utf8_main(argc, argv); + for(i=0; i<argc; i++) free(orig[i]); + free(argv); + return rc; +} From b3444ee185afd513840d568c51a82f46d172ee13 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo <sjlombardo@zetetic.net> Date: Fri, 6 Mar 2026 21:59:35 -0500 Subject: [PATCH 65/75] Updates CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 446d65cae..51fcfd558 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ Notable changes to this project are documented in this file. ## [4.14.0] - (? 2026 - [4.14.0 changes]) +- Updates baseline to SQLite 3.52.0 +- Restores and improves upon LibTomCrypto provder +- Minor test improvements ## [4.13.0] - (January 2026 - [4.13.0 changes]) - Updates baseline to SQLite 3.51.2 From 2b30a9fdc1d348cf45e4ae252308cfb753e1c7ed Mon Sep 17 00:00:00 2001 From: Stephen Lombardo <sjlombardo@zetetic.net> Date: Fri, 13 Mar 2026 09:24:56 -0400 Subject: [PATCH 66/75] Snapshot of upstream SQLite 3.51.3 --- Makefile.msc | 203 +- README.md | 99 +- VERSION | 2 +- autoconf/Makefile.in | 15 +- autoconf/Makefile.msc | 100 +- autosetup/README.md | 10 - autosetup/jimsh0.c | 10 +- autosetup/proj.tcl | 4 +- autosetup/sqlite-config.tcl | 24 +- doc/compile-for-unix.md | 13 +- doc/compile-for-windows.md | 36 - doc/lemon.html | 32 +- doc/testrunner.md | 159 +- ext/expert/expert1.test | 2 +- ext/fts3/fts3.c | 9 +- ext/fts3/fts3Int.h | 9 - ext/fts3/fts3_write.c | 42 +- ext/fts5/fts5_aux.c | 2 +- ext/fts5/fts5_buffer.c | 2 +- ext/fts5/fts5_config.c | 4 +- ext/fts5/fts5_expr.c | 4 +- ext/fts5/fts5_hash.c | 2 +- ext/fts5/fts5_index.c | 70 +- ext/fts5/fts5_main.c | 5 +- ext/fts5/fts5_tcl.c | 8 +- ext/fts5/fts5_test_tok.c | 10 +- ext/fts5/fts5_tokenize.c | 8 +- ext/fts5/fts5_vocab.c | 2 +- ext/fts5/test/fts5corrupt9.test | 69 - ext/fts5/test/fts5integrity.test | 3 + ext/fts5/test/fts5interrupt.test | 17 - ext/intck/sqlite3intck.c | 2 +- ext/jni/README.md | 35 +- ext/jni/src/c/sqlite3-jni.c | 94 +- ext/jni/src/c/sqlite3-jni.h | 4 +- ext/jni/src/org/sqlite/jni/capi/CApi.java | 8 +- ext/jni/src/org/sqlite/jni/capi/Tester1.java | 4 +- .../src/org/sqlite/jni/wrapper1/Sqlite.java | 1 - ext/misc/btreeinfo.c | 6 +- ext/misc/csv.c | 4 +- ext/misc/decimal.c | 43 +- ext/misc/fileio.c | 240 +- ext/misc/fuzzer.c | 2 +- ext/misc/ieee754.c | 36 +- ext/misc/regexp.c | 28 +- ext/misc/sha1.c | 15 +- ext/misc/sqlite3_stdio.c | 20 +- ext/misc/sqlite3_stdio.h | 3 - ext/misc/tmstmpvfs.c | 1042 --- ext/misc/vtablog.c | 54 +- ext/misc/zipfile.c | 7 +- ext/qrf/README.md | 762 -- ext/qrf/dev-notes.md | 14 - ext/qrf/qrf.c | 2983 ------ ext/qrf/qrf.h | 200 - ext/rbu/rbu11.test | 28 - ext/rbu/sqlite3rbu.c | 4 +- ext/repair/README.md | 16 + ext/repair/checkfreelist.c | 310 + ext/repair/checkindex.c | 929 ++ ext/repair/sqlite3_checker.c.in | 85 + ext/repair/sqlite3_checker.tcl | 264 + ext/repair/test/README.md | 13 + ext/repair/test/checkfreelist01.test | 92 + ext/repair/test/checkindex01.test | 349 + ext/repair/test/test.tcl | 67 + ext/rtree/geopoly.c | 2 +- ext/session/session4.test | 1 - ext/session/sessionC.test | 10 - ext/session/sqlite3session.c | 60 +- ext/session/test_session.c | 66 +- ext/wasm/EXPORTED_FUNCTIONS.fiddle.in | 10 + ext/wasm/GNUmakefile | 483 +- ...S.c-pp => EXPORTED_FUNCTIONS.sqlite3-core} | 90 - .../api/EXPORTED_FUNCTIONS.sqlite3-extras | 68 + ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see | 5 + .../api/EXPORTED_RUNTIME_METHODS.sqlite3-api | 3 + ext/wasm/api/README.md | 84 +- ext/wasm/api/extern-post-js.c-pp.js | 4 +- ext/wasm/api/post-js-footer.js | 69 - ext/wasm/api/post-js-header.js | 10 +- ext/wasm/api/pre-js.c-pp.js | 50 +- ext/wasm/api/sqlite3-api-cleanup.js | 83 + ext/wasm/api/sqlite3-api-glue.c-pp.js | 155 +- ext/wasm/api/sqlite3-api-oo1.c-pp.js | 293 +- ext/wasm/api/sqlite3-api-prologue.js | 385 +- .../api/sqlite3-license-version-header.js | 3 +- ext/wasm/api/sqlite3-opfs-async-proxy.js | 2 +- ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js | 2095 ----- ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js | 18 +- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 8 +- ext/wasm/api/sqlite3-vtab-helper.c-pp.js | 5 +- ext/wasm/api/sqlite3-wasm.c | 294 +- ext/wasm/api/sqlite3-worker1-promiser.c-pp.js | 26 +- ext/wasm/api/sqlite3-worker1.c-pp.js | 4 +- ext/wasm/c-pp-lite.c | 69 +- ext/wasm/common/SqliteTestUtil.js | 3 +- ext/wasm/common/whwasmutil.js | 67 +- ext/wasm/demo-jsstorage.js | 8 +- ext/wasm/demo-worker1-promiser.c-pp.html | 8 +- ext/wasm/demo-worker1.js | 6 +- ext/wasm/fiddle/fiddle-worker.js | 4 + .../fiddle/{index.c-pp.html => index.html} | 17 +- ext/wasm/index.html | 5 - ext/wasm/jaccwabyt/jaccwabyt.js | 851 +- ext/wasm/jaccwabyt/jaccwabyt.md | 698 +- ext/wasm/mkdist.sh | 2 +- ext/wasm/mkwasmbuilds.c | 109 +- ext/wasm/speedtest1-worker.js | 13 +- ext/wasm/speedtest1.html | 203 +- ext/wasm/tester1-worker.c-pp.html | 16 +- ext/wasm/tester1.c-pp.html | 18 +- ext/wasm/tester1.c-pp.js | 983 +- main.mk | 95 +- make.bat | 2 - manifest | 511 +- manifest.tags | 7 +- manifest.uuid | 2 +- src/alter.c | 779 +- src/attach.c | 2 +- src/auth.c | 2 +- src/btree.c | 28 +- src/build.c | 17 +- src/carray.c | 39 +- src/date.c | 16 +- src/delete.c | 3 +- src/expr.c | 93 +- src/fkey.c | 12 +- src/func.c | 16 +- src/hwtime.h | 41 +- src/json.c | 113 +- src/loadext.c | 47 +- src/main.c | 36 +- src/mutex.c | 29 +- src/mutex_w32.c | 8 + src/os_kv.c | 266 +- src/os_unix.c | 2 +- src/os_win.c | 436 +- src/os_win.h | 10 +- src/pager.c | 12 +- src/parse.y | 206 +- src/pcache.h | 4 +- src/prepare.c | 3 +- src/printf.c | 21 +- src/resolve.c | 10 +- src/select.c | 707 +- src/shell.c.in | 7966 +++++++++-------- src/sqlite.h.in | 241 +- src/sqlite3ext.h | 11 +- src/sqliteInt.h | 127 +- src/sqliteLimit.h | 29 +- src/tclsqlite.c | 417 +- src/test1.c | 43 +- src/test_bestindex.c | 1 + src/test_config.c | 16 +- src/test_quota.c | 4 + src/tokenize.c | 4 +- src/treeview.c | 8 +- src/trigger.c | 137 +- src/util.c | 665 +- src/vacuum.c | 12 +- src/vdbe.c | 21 +- src/vdbe.h | 2 +- src/vdbeInt.h | 4 +- src/vdbeapi.c | 50 +- src/vdbeaux.c | 2 +- src/vdbemem.c | 194 +- src/where.c | 227 +- src/wherecode.c | 11 +- src/whereexpr.c | 79 +- src/window.c | 6 +- test/alterauth2.test | 51 - test/altercol.test | 13 - test/altercons.test | 442 - test/altercons2.test | 247 - test/alterfault.test | 37 - test/altertab3.test | 48 - test/altertrig.test | 2 +- test/atof2.test | 35 - test/autoindex1.test | 3 +- test/avfs.test | 2 - test/bestindex8.test | 16 +- test/bestindexB.test | 2 +- test/bestindexF.test | 294 - test/carray01.test | 8 - test/collate5.test | 30 +- test/cost.test | 2 +- test/dblwidth-a.sql | 38 +- test/distinct2.test | 2 +- test/dotcmd01.sql | 63 - test/e_expr.test | 8 +- test/e_update.test | 14 +- test/e_walckpt.test | 17 +- test/eqp.test | 55 +- test/filectrl.test | 12 +- test/fpconv1.test | 66 +- test/fptest01.sql | 76 - test/fts3comp1.test | 77 - test/fts4content.test | 59 +- test/fts4merge5.test | 3 - test/fuzzcheck.c | 14 +- test/fuzzinvariants.c | 9 +- test/import01.sql | 217 - test/imposter1.sql | 32 - test/insert5.test | 19 +- test/intck01.sql | 23 - test/join.test | 41 - test/joinI.test | 43 + test/json102.test | 21 - test/json103.test | 7 - test/json105.test | 5 - test/json109.test | 72 - test/misc5.test | 2 +- test/modeA.sql | 303 - test/mutex1.test | 26 +- test/notnull2.test | 2 +- test/offset1.test | 28 - test/qrf01.test | 1167 --- test/qrf02.test | 47 - test/qrf03.test | 176 - test/qrf04.test | 750 -- test/qrf05.test | 37 - test/qrf06.test | 576 -- test/recover.test | 2 +- test/regexp1.sql | 32 - test/regexp1.test | 24 - test/rowvalue4.test | 3 +- test/rowvalueA.test | 25 - test/schema.test | 4 +- test/select9.test | 2 +- test/shell1.test | 227 +- test/shell2.test | 92 +- test/shell4.test | 8 +- test/shell5.test | 124 +- test/shell8.test | 40 - test/shellA.test | 233 +- test/shellB.test | 53 - test/speedtest.md | 5 +- test/speedtest.tcl | 2 +- test/tabfunc01.test | 14 - test/tclsqlite.test | 2 +- test/temptrigfault.tes | 120 - test/temptrigger.test | 187 - test/tester.tcl | 5 + test/testrunner.tcl | 128 +- test/testrunner_data.tcl | 8 +- test/testrunner_estwork.tcl | 1 - test/tkt-99378177930f87bd.test | 28 - test/tkt2339.test | 6 +- test/values.test | 6 +- test/vt02.c | 125 +- test/vt100-a.sql | 35 +- test/walckptnoop.test | 6 +- test/walrestart.test | 4 +- test/where2.test | 37 - test/whereK.test | 13 - test/win32longpath.test | 4 + test/with1.test | 19 - test/zipfile2.test | 15 - tool/build-all-msvc.bat | 3 +- tool/dbtotxt.c | 10 +- tool/lemon.c | 63 +- tool/lempar.c | 17 +- tool/mkautoconfamal.sh | 3 +- tool/mkcombo.tcl | 106 - tool/mkkeywordhash.c | 4 +- tool/mkshellc.tcl | 120 +- tool/omittest.tcl | 2 +- tool/showdb.c | 247 +- tool/showtmlog.c | 254 - tool/sqldiff.c | 42 +- tool/sqlite3_rsync.c | 33 +- tool/sqltclsh.c.in | 2 +- tool/winmain.c | 79 - 274 files changed, 11464 insertions(+), 26556 deletions(-) delete mode 100644 ext/misc/tmstmpvfs.c delete mode 100644 ext/qrf/README.md delete mode 100644 ext/qrf/dev-notes.md delete mode 100644 ext/qrf/qrf.c delete mode 100644 ext/qrf/qrf.h create mode 100644 ext/repair/README.md create mode 100644 ext/repair/checkfreelist.c create mode 100644 ext/repair/checkindex.c create mode 100644 ext/repair/sqlite3_checker.c.in create mode 100644 ext/repair/sqlite3_checker.tcl create mode 100644 ext/repair/test/README.md create mode 100644 ext/repair/test/checkfreelist01.test create mode 100644 ext/repair/test/checkindex01.test create mode 100644 ext/repair/test/test.tcl create mode 100644 ext/wasm/EXPORTED_FUNCTIONS.fiddle.in rename ext/wasm/api/{EXPORTED_FUNCTIONS.c-pp => EXPORTED_FUNCTIONS.sqlite3-core} (61%) create mode 100644 ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras create mode 100644 ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see create mode 100644 ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api create mode 100644 ext/wasm/api/sqlite3-api-cleanup.js delete mode 100644 ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js rename ext/wasm/fiddle/{index.c-pp.html => index.html} (95%) delete mode 100644 make.bat delete mode 100644 test/altercons.test delete mode 100644 test/altercons2.test delete mode 100644 test/atof2.test delete mode 100644 test/bestindexF.test delete mode 100644 test/dotcmd01.sql delete mode 100644 test/fptest01.sql delete mode 100644 test/import01.sql delete mode 100644 test/imposter1.sql delete mode 100644 test/intck01.sql delete mode 100644 test/json109.test delete mode 100644 test/modeA.sql delete mode 100644 test/qrf01.test delete mode 100644 test/qrf02.test delete mode 100644 test/qrf03.test delete mode 100644 test/qrf04.test delete mode 100644 test/qrf05.test delete mode 100644 test/qrf06.test delete mode 100644 test/regexp1.sql delete mode 100644 test/shellB.test delete mode 100644 test/temptrigfault.tes delete mode 100644 tool/mkcombo.tcl delete mode 100644 tool/showtmlog.c delete mode 100644 tool/winmain.c diff --git a/Makefile.msc b/Makefile.msc index 763616ce9..52807ff7f 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -11,8 +11,6 @@ TOP = . # <<mark>> -# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc -# # Set this non-0 to create and use the SQLite amalgamation file. # !IFNDEF USE_AMALGAMATION @@ -103,6 +101,13 @@ NO_WARN = $(NO_WARN) -wd4210 -wd4232 -wd4244 -wd4305 -wd4306 -wd4702 -wd4706 !ENDIF !ENDIF +# Set this non-0 to use the library paths and other options necessary for +# Windows Phone 8.1. +# +!IFNDEF USE_WP81_OPTS +USE_WP81_OPTS = 0 +!ENDIF + # Set this non-0 to split the SQLite amalgamation file into chunks to # be used for debugging with Visual Studio. # @@ -111,8 +116,14 @@ SPLIT_AMALGAMATION = 0 !ENDIF # <<mark>> -# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# Set this non-0 to have this makefile assume the Tcl shell executable +# (tclsh*.exe) is available in the PATH. By default, this is disabled +# for compatibility with older build environments. This setting only +# applies if TCLSH_CMD is not set manually. # +!IFNDEF USE_TCLSH_IN_PATH +USE_TCLSH_IN_PATH = 0 +!ENDIF # Set this non-0 to use zlib, possibly compiling it from source code. # @@ -175,6 +186,14 @@ USE_NATIVE_LIBPATHS = 0 USE_RC = 1 !ENDIF +# Set this non-0 to compile binaries suitable for the WinRT environment. +# This setting does not apply to any binaries that require Tcl to operate +# properly (i.e. the text fixture, etc). +# +!IFNDEF FOR_WINRT +FOR_WINRT = 0 +!ENDIF + # Set this non-0 to compile binaries suitable for the UWP environment. # This setting does not apply to any binaries that require Tcl to operate # properly (i.e. the text fixture, etc). @@ -190,8 +209,6 @@ FOR_WIN10 = 0 !ENDIF # <<mark>> -# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc -# # Set this non-0 to skip attempting to look for and/or link with the Tcl # runtime library. # @@ -248,8 +265,6 @@ DEBUG = 0 !ENDIF # <<mark>> -# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc -# # By default, use --linemacros=1 argument to the mksqlite3c.tcl tool, which # is used to build the amalgamation. This can be turned off to ease debug # of the amalgamation away from the source tree. @@ -345,8 +360,6 @@ SQLITE3EXEPDB = /pdb:sqlite3sh.pdb !ENDIF # <<mark>> -# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc -# # These are the names of the customized Tcl header files used by various parts # of this makefile when the stdcall calling convention is in use. It is not # used for any other purpose. @@ -628,24 +641,16 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS) !IF "$(PLATFORM)"=="x86" CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall - # <<mark>> -# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc -# TEST_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl # <</mark>> - !ELSE !IFNDEF PLATFORM CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall - # <<mark>> -# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc -# TEST_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl # <</mark>> - !ELSE CORE_CCONV_OPTS = SHELL_CCONV_OPTS = @@ -772,6 +777,18 @@ SHELL_LINK_OPTS = $(SHELL_CORE_LIB) TCC = $(TCC) -FAcs !ENDIF +# When compiling the library for use in the WinRT environment, +# the following compile-time options must be used as well to +# disable use of Win32 APIs that are not available and to enable +# use of Win32 APIs that are specific to Windows 8 and/or WinRT. +# +!IF $(FOR_WINRT)!=0 +TCC = $(TCC) -DSQLITE_OS_WINRT=1 +RCC = $(RCC) -DSQLITE_OS_WINRT=1 +TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP +RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP +!ENDIF + # C compiler options for the Windows 10 platform (needs MSVC 2015). # !IF $(FOR_WIN10)!=0 @@ -784,7 +801,7 @@ BCC = $(BCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE # USE_CRT_DLL option is set to force dynamically linking to the # MSVC runtime library. # -!IF $(USE_CRT_DLL)!=0 +!IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0 !IF $(DEBUG)>1 TCC = $(TCC) -MDd BCC = $(BCC) -MDd @@ -807,8 +824,6 @@ ZLIBCFLAGS = -nologo -MT -W3 -O2 -Oy- -Zi !ENDIF # <<mark>> -# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc -# # The mksqlite3c.tcl and mksqlite3h.tcl scripts will pull in # any extension header files by default. For non-amalgamation # builds, we need to make sure the compiler can find these. @@ -859,6 +874,16 @@ MKSQLITE3H_ARGS = !ENDIF # <</mark>> +# Define -DNDEBUG to compile without debugging (i.e., for production usage) +# Omitting the define will cause extra debugging code to be inserted and +# includes extra comments when "EXPLAIN stmt" is used. +# +!IF $(DEBUG)==0 +TCC = $(TCC) -DNDEBUG +BCC = $(BCC) -DNDEBUG +RCC = $(RCC) -DNDEBUG +!ENDIF + !IF $(DEBUG)>0 || $(API_ARMOR)!=0 || $(FOR_WIN10)!=0 TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1 RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 @@ -924,8 +949,6 @@ TCC = $(TCC) /fsanitize=address !ENDIF # <<mark>> -# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc -# # The locations of the Tcl header and library files. Also, the library that # non-stubs enabled programs using Tcl must link against. These variables # (TCLINCDIR, TCLLIBDIR, and LIBTCL) may be overridden via the environment @@ -1165,8 +1188,6 @@ BCC = $(BCC) -Zi !ENDIF # <<mark>> -# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc -# # If zlib support is enabled, add the compiler options for it. # !IF $(USE_ZLIB)!=0 @@ -1219,6 +1240,56 @@ LTLINKOPTS = /NOLOGO LTLIBOPTS = /NOLOGO !ENDIF +# When compiling for use in the WinRT environment, the following +# linker option must be used to mark the executable as runnable +# only in the context of an application container. +# +!IF $(FOR_WINRT)!=0 +LTLINKOPTS = $(LTLINKOPTS) /APPCONTAINER +!IF "$(VISUALSTUDIOVERSION)"=="12.0" || "$(VISUALSTUDIOVERSION)"=="14.0" +!IFNDEF STORELIBPATH +!IF "$(PLATFORM)"=="x86" +STORELIBPATH = $(CRTLIBPATH)\store +!ELSEIF "$(PLATFORM)"=="x64" +STORELIBPATH = $(CRTLIBPATH)\store\amd64 +!ELSEIF "$(PLATFORM)"=="ARM" +STORELIBPATH = $(CRTLIBPATH)\store\arm +!ELSE +STORELIBPATH = $(CRTLIBPATH)\store +!ENDIF +!ENDIF +STORELIBPATH = $(STORELIBPATH:\\=\) +LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(STORELIBPATH)" +!ENDIF +!ENDIF + +# When compiling for Windows Phone 8.1, an extra library path is +# required. +# +!IF $(USE_WP81_OPTS)!=0 +!IFNDEF WP81LIBPATH +!IF "$(PLATFORM)"=="x86" +WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 +!ELSEIF "$(PLATFORM)"=="ARM" +WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\ARM +!ELSE +WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 +!ENDIF +!ENDIF +!ENDIF + +# When compiling for Windows Phone 8.1, some extra linker options +# are also required. +# +!IF $(USE_WP81_OPTS)!=0 +!IFDEF WP81LIBPATH +LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(WP81LIBPATH)" +!ENDIF +LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE +LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib +!ENDIF + # When compiling for UWP or the Windows 10 platform, some extra linker # options are also required. # @@ -1242,14 +1313,12 @@ LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib # If either debugging or symbols are enabled, enable PDBs. # !IF $(DEBUG)>1 || $(SYMBOLS)!=0 -LDFLAGS = /NODEFAULTLIB:msvcrt /DEBUG $(LDOPTS) +LDFLAGS = /DEBUG $(LDOPTS) !ELSE -LDFLAGS = /NODEFAULTLIB:msvcrt $(LDOPTS) +LDFLAGS = $(LDOPTS) !ENDIF # <<mark>> -# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc -# # Start with the Tcl related linker options. # !IF $(NO_TCL)==0 @@ -1276,8 +1345,6 @@ LTLIBS = $(LTLIBS) $(LIBICU) ############################################################################### # <<mark>> -# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc -# # Object files for the SQLite library (non-amalgamation). # LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ @@ -1396,7 +1463,7 @@ SRC01 = \ $(TOP)\src\status.c \ $(TOP)\src\table.c \ $(TOP)\src\threads.c \ - tclsqlite-ex.c \ + $(TOP)\src\tclsqlite.c \ $(TOP)\src\tokenize.c \ $(TOP)\src\treeview.c \ $(TOP)\src\trigger.c \ @@ -1707,7 +1774,6 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_PERCENTILE=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1 -SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_STRICT_SUBTYPE=1 !ENDIF @@ -1866,8 +1932,8 @@ $(SQLITE3EXE): shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLIT /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) # <<mark>> -sqldiff.exe: $(TOP)\tool\sqldiff.c $(TOP)\ext\misc\sqlite3_stdio.h $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) $(TOP)\tool\winmain.c - $(LTLINK) $(NO_WARN) -I$(TOP)\ext\misc $(TOP)\tool\sqldiff.c $(TOP)\ext\misc\sqlite3_stdio.c $(TOP)\tool\winmain.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS) +sqldiff.exe: $(TOP)\tool\sqldiff.c $(TOP)\ext\misc\sqlite3_stdio.h $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) + $(LTLINK) $(NO_WARN) -I$(TOP)\ext\misc $(TOP)\tool\sqldiff.c $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS) dbhash.exe: $(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) @@ -1957,18 +2023,8 @@ sqlite3.c: .target_source sqlite3ext.h sqlite3session.h $(MKSQLITE3C_TOOL) src-v sqlite3-all.c: sqlite3.c $(TOP)\tool\split-sqlite3c.tcl $(JIM_TCLSH) $(JIM_TCLSH) $(TOP)\tool\split-sqlite3c.tcl - -TCLSQLITEEX = \ - $(TOP)\ext\qrf\qrf.h \ - $(TOP)\ext\qrf\qrf.c \ - $(TOP)\src\tclsqlite.c - -tclsqlite-ex.c: $(TCLSQLITEEX) $(TOP)\tool\mkcombo.tcl $(JIM_TCLSH) - $(JIM_TCLSH) $(TOP)\tool\mkcombo.tcl $(TCLSQLITEEX) -o $@ # <</mark>> -tclsqlite-ex.c: - # Rule to build the amalgamation # sqlite3.lo: $(SQLITE3C) @@ -2268,11 +2324,11 @@ whereexpr.lo: $(TOP)\src\whereexpr.c $(HDR) window.lo: $(TOP)\src\window.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\window.c -tclsqlite.lo: tclsqlite-ex.c $(HDR) $(SQLITE_TCL_DEP) - $(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c tclsqlite-ex.c /OUT:tclsqlite.lo +tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP) + $(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c -tclsqlite-shell.lo: tclsqlite-ex.c $(HDR) $(SQLITE_TCL_DEP) - $(LTCOMPILE) $(NO_WARN) -DTCLSH -DBUILD_sqlite -I$(TCLINCDIR) -c tclsqlite-ex.c +tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP) + $(LTCOMPILE) $(NO_WARN) -DTCLSH -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c tclsqlite3.exe: tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) $(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS) @@ -2325,8 +2381,6 @@ keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe # Source and header files that shell.c depends on SHELL_DEP = \ $(TOP)\src\shell.c.in \ - $(TOP)\ext\qrf\qrf.c \ - $(TOP)\ext\qrf\qrf.h \ $(TOP)\ext\expert\sqlite3expert.c \ $(TOP)\ext\expert\sqlite3expert.h \ $(TOP)\ext\intck\sqlite3intck.c \ @@ -2490,9 +2544,9 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_STRICT_SUBTYPE=1 TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2) TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C) !IF $(USE_AMALGAMATION)==0 -TESTFIXTURE_SRC = $(TESTSRC) tclsqlite-ex.c $(TESTFIXTURE_SRC0) +TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0) !ELSE -TESTFIXTURE_SRC = $(TESTSRC) tclsqlite-ex.c $(TESTFIXTURE_SRC1) +TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1) !ENDIF !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 @@ -2601,22 +2655,6 @@ devtest: srctree-check sourcetest mdevtest: $(TCLSH_CMD) $(TOP)\test\testrunner.tcl mdevtest -# Rerun test cases that failed on the previous testrunner invocation -# -retest: - $(TCLSH_CMD) $(TOP)\test\testrunner.tcl retest - -# Show all errors from the most reason testrunner invocation -# -errors: - $(TCLSH_CMD) $(TOP)\test\testrunner.tcl errors - -# Show the status of the current testrunner invocation, -# updated every couple of seconds -# -status: - $(TCLSH_CMD) $(TOP)\test\testrunner.tcl status -d 2 - # Validate that various generated files in the source tree # are up-to-date. # @@ -2640,17 +2678,17 @@ smoketest: $(TESTPROGS) @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\main.test $(TESTOPTS) -shelltest: - $(TCLSH_CMD) $(TOP)\test\testrunner.tcl release shell +shelltest: $(TESTPROGS) + .\testfixture.exe $(TOP)\test\permutations.test shell -sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) tclsqlite-ex.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(TOP)\ext\misc\sqlite3_stdio.h $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE_TCL_DEP) +sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(TOP)\ext\misc\sqlite3_stdio.h $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE_TCL_DEP) $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl -DINCLUDE_SQLITE3_C $(TOP)\tool\sqlite3_analyzer.c.in > $@ sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS) $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \ /link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS) -sqltclsh.c: sqlite3.c tclsqlite-ex.c $(TOP)\tool\sqltclsh.tcl $(TOP)\ext\misc\appendvfs.c $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in +sqltclsh.c: sqlite3.c $(TOP)\src\tclsqlite.c $(TOP)\tool\sqltclsh.tcl $(TOP)\ext\misc\appendvfs.c $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in >sqltclsh.c sqltclsh.exe: sqltclsh.c $(SHELL_CORE_DEP) $(LIBRESOBJS) @@ -2660,6 +2698,23 @@ sqltclsh.exe: sqltclsh.c $(SHELL_CORE_DEP) $(LIBRESOBJS) sqlite3_expert.exe: $(SQLITE3C) $(TOP)\ext\expert\sqlite3expert.h $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(LTLINK) $(NO_WARN) $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(SQLITE3C) $(TLIBS) +CHECKER_DEPS =\ + $(TOP)\tool\mkccode.tcl \ + sqlite3.c \ + $(TOP)\src\tclsqlite.c \ + $(TOP)\ext\repair\sqlite3_checker.tcl \ + $(TOP)\ext\repair\checkindex.c \ + $(TOP)\ext\repair\checkfreelist.c \ + $(TOP)\ext\misc\btreeinfo.c \ + $(TOP)\ext\repair\sqlite3_checker.c.in + +sqlite3_checker.c: $(CHECKER_DEPS) + $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\ext\repair\sqlite3_checker.c.in > $@ + +sqlite3_checker.exe: sqlite3_checker.c $(LIBRESOBJS) + $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_checker.c \ + /link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS) + dbdump.exe: $(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) $(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \ /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) @@ -2692,9 +2747,6 @@ showwal.exe: $(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H) showshm.exe: $(TOP)\tool\showshm.c $(LTLINK) $(NO_WARN) $(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS) -showtmlog.exe: $(TOP)\tool\showtmlog.c - $(LTLINK) $(NO_WARN) $(TOP)\tool\showtmlog.c /link $(LDFLAGS) $(LTLINKOPTS) - index_usage.exe: $(TOP)\tool\index_usage.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\tool\index_usage.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) @@ -2886,7 +2938,6 @@ env: @echo USE_STDCALL = $(USE_STDCALL) @echo USE_ZLIB = $(USE_ZLIB) @echo XCOMPILE = $(XCOMPILE) - @echo ZLIBCFLAGS = $(ZLIBCFLAGS) @echo ZLIBDIR = $(ZLIBDIR) @echo ZLIBINCDIR = $(ZLIBINCDIR) @echo ZLIBLIBDIR = $(ZLIBLIBDIR) diff --git a/README.md b/README.md index 7ec5042e5..d44bf5cb7 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,8 @@ <h1 align="center">SQLite Source Repository</h1> This repository contains the complete source code for the -[SQLite database engine](https://sqlite.org/) going back -to 2000-05-29. The tree includes many tests and some -documentation, though additional tests and most documentation +[SQLite database engine](https://sqlite.org/), including +many tests. Additional tests and most documentation are managed separately. See the [on-line documentation](https://sqlite.org/) for more information @@ -100,16 +99,16 @@ script found at the root of the source tree. Then run "make". For example: - apt install gcc make tcl-dev ;# Install the necessary build tools + apt install gcc make tcl-dev ;# Make sure you have all the necessary build tools tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite" - mkdir bld ;# Build happens in a sibling directory + mkdir bld ;# Build will occur in a sibling directory cd bld ;# Change to the build directory ../sqlite/configure ;# Run the configure script - make sqlite3 ;# The "sqlite3" command-line tool - make sqlite3.c ;# The "amalgamation" source file - make sqldiff ;# The "sqldiff" command-line tool - #### Targets below require tcl-dev #### - make tclextension-install ;# Install the SQLite TCL extension + make sqlite3 ;# Builds the "sqlite3" command-line tool + make sqlite3.c ;# Build the "amalgamation" source file + make sqldiff ;# Builds the "sqldiff" command-line tool + # Makefile targets below this point require tcl-dev + make tclextension-install ;# Build and install the SQLite TCL extension make devtest ;# Run development tests make releasetest ;# Run full release tests make sqlite3_analyzer ;# Builds the "sqlite3_analyzer" tool @@ -117,15 +116,14 @@ For example: See the makefile for additional targets. For debugging builds, the core developers typically run "configure" with options like this: - ../sqlite/configure --all --debug CFLAGS='-O0 -g' + ../sqlite/configure --enable-all --enable-debug CFLAGS='-O0 -g' For release builds, the core developers usually do: - ../sqlite/configure --all + ../sqlite/configure --enable-all -Core deliverables (sqlite3.c, sqlite3) can be built without a TCL, but -many makefile targets require a "tclsh" TCL interpreter version 8.6 -or later. The "tclextension-install" target and the test targets that follow +Almost all makefile targets require a "tclsh" TCL interpreter version 8.6 or +later. The "tclextension-install" target and the test targets that follow all require TCL development libraries too. ("apt install tcl-dev"). It is helpful, but is not required, to install the SQLite TCL extension (the "tclextension-install" target) prior to running tests. The "releasetest" @@ -135,20 +133,20 @@ On "make" command-lines, one can add "OPTIONS=..." to specify additional compile-time options over and above those set by ./configure. For example, to compile with the SQLITE_OMIT_DEPRECATED compile-time option, one could say: - ./configure --all + ./configure --enable-all make OPTIONS=-DSQLITE_OMIT_DEPRECATED sqlite3 -The configure script uses [autosetup](https://msteveb.github.io/autosetup/). -If the configure script does not work out for you, there is a generic -makefile named "Makefile.linux-gcc" in the top directory of the source tree -that you can copy and edit to suit your needs. Comments on the generic -makefile show what changes are needed. +The configure script uses autoconf 2.61 and libtool. If the configure +script does not work out for you, there is a generic makefile named +"Makefile.linux-gcc" in the top directory of the source tree that you +can copy and edit to suit your needs. Comments on the generic makefile +show what changes are needed. ## Compiling for Windows Using MSVC On Windows, everything can be compiled with MSVC. -You will also need a working installation of TCL if you want to run tests, -though TCL is not required if you just want to build SQLite itself. +You will also need a working installation of TCL if you want to run tests. +TCL is not required if you just want to build SQLite itself. See the [compile-for-windows.md](doc/compile-for-windows.md) document for additional information about how to install MSVC and TCL and configure your build environment. @@ -158,25 +156,24 @@ TCL library, using a command like this: set TCLDIR=c:\Tcl -SQLite itself does not contain any TCL code, but it does use TCL to run -tests. You may need to install TCL development libraries in order to -successfully complete some makefile targets. It is helpful, but is not -required, to install the SQLite TCL extension (the "tclextension-install" -target) prior to running tests. - -The source tree contains a "make.bat" file that allows the same "make" -commands of Unix to work on Windows. In the following, you can substitute -"nmake /f Makefile.msc" in place of "make", if you prefer to avoid this BAT -file: - - make sqlite3.exe - make sqlite3.c - make sqldiff.exe - #### Targets below require TCL development libraries #### - make tclextension-install - make devtest - make releasetest - make sqlite3_analyzer.exe +SQLite uses "tclsh.exe" as part of the build process, and so that +program will need to be somewhere on your %PATH%. SQLite itself +does not contain any TCL code, but it does use TCL to run tests. +You may need to install TCL development +libraries in order to successfully complete some makefile targets. +It is helpful, but is not required, to install the SQLite TCL extension +(the "tclextension-install" target) prior to running tests. + +Build using Makefile.msc. Example: + + nmake /f Makefile.msc sqlite3.exe + nmake /f Makefile.msc sqlite3.c + nmake /f Makefile.msc sqldiff.exe + # Makefile targets below this point require TCL development libraries + nmake /f Makefile.msc tclextension-install + nmake /f Makefile.msc devtest + nmake /f Makefile.msc releasetest + nmake /f Makefile.msc sqlite3_analyzer.exe There are many other makefile targets. See comments in Makefile.msc for details. @@ -184,7 +181,7 @@ details. As with the unix Makefile, the OPTIONS=... argument can be passed on the nmake command-line to enable new compile-time options. For example: - make OPTIONS=-DSQLITE_OMIT_DEPRECATED sqlite3.exe + nmake /f Makefile.msc OPTIONS=-DSQLITE_OMIT_DEPRECATED sqlite3.exe ## Source Tree Map @@ -197,7 +194,8 @@ command-line to enable new compile-time options. For example: * **test/** - This directory and its subdirectories contains code used for testing. Files that end in "`.test`" are TCL scripts that run tests using an augmented TCL interpreter named "testfixture". Use - a command like "`make testfixture`" to build that + a command like "`make testfixture`" (unix) or + "`nmake /f Makefile.msc testfixture.exe`" (windows) to build that augmented TCL interpreter, then run individual tests using commands like "`testfixture test/main.test`". This test/ subdirectory also contains additional C code modules and scripts for other kinds of testing. @@ -379,11 +377,10 @@ implementation. It will not be the easiest library in the world to hack. (and some other test programs too) is built and run when you type "make test". - * **VERSION**, **manifest**, **manifest.tags**, and **manifest.uuid** - - These files define the current SQLite version number. The "VERSION" file - is human generated, but the "manifest", "manifest.tags", and - "manifest.uuid" files are automatically generated by the - [Fossil version control system](https://fossil-scm.org/). + * **VERSION**, **manifest**, and **manifest.uuid** - These files define + the current SQLite version number. The "VERSION" file is human generated, + but the "manifest" and "manifest.uuid" files are automatically generated + by the [Fossil version control system](https://fossil-scm.org/). There are many other source files. Each has a succinct header comment that describes its purpose and role within the larger system. @@ -409,6 +406,10 @@ makefile: > make verify-source +Or on windows: + +> nmake /f Makefile.msc verify-source + Using the makefile to verify source integrity is good for detecting accidental changes to the source tree, but malicious changes could be hidden by also modifying the makefiles. diff --git a/VERSION b/VERSION index 7ac0b0a68..f17126c72 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.52.0 +3.51.3 diff --git a/autoconf/Makefile.in b/autoconf/Makefile.in index 8e6b358be..c938ffe1b 100644 --- a/autoconf/Makefile.in +++ b/autoconf/Makefile.in @@ -117,6 +117,19 @@ sqlite_cfg.h: $(AS_AUTO_DEF) # CFLAGS for sqlite3$(T.exe) # SHELL_OPT ?= @OPT_SHELL@ +SHELL_OPT += -DSQLITE_DQS=0 +SHELL_OPT += -DSQLITE_ENABLE_FTS4 +#SHELL_OPT += -DSQLITE_ENABLE_FTS5 +SHELL_OPT += -DSQLITE_ENABLE_RTREE +SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS +SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION +SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB +SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB +SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB +SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB +SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC +SHELL_OPT += -DSQLITE_ENABLE_PERCENTILE +SHELL_OPT += -DSQLITE_STRICT_SUBTYPE=1 # # Library-level feature flags @@ -282,7 +295,7 @@ DIST_FILES := \ README.txt VERSION \ auto.def autosetup configure tea \ sqlite3.h sqlite3.c shell.c sqlite3ext.h \ - Makefile.in Makefile.msc Makefile.fallback make.bat \ + Makefile.in Makefile.msc Makefile.fallback \ sqlite3.rc sqlite3rc.h Replace.cs \ sqlite3.pc.in sqlite3.1 diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index 34e41d8aa..365e1f0fb 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -101,6 +101,13 @@ NO_WARN = $(NO_WARN) -wd4210 -wd4232 -wd4244 -wd4305 -wd4306 -wd4702 -wd4706 !ENDIF !ENDIF +# Set this non-0 to use the library paths and other options necessary for +# Windows Phone 8.1. +# +!IFNDEF USE_WP81_OPTS +USE_WP81_OPTS = 0 +!ENDIF + # Set this non-0 to split the SQLite amalgamation file into chunks to # be used for debugging with Visual Studio. # @@ -149,6 +156,14 @@ USE_NATIVE_LIBPATHS = 0 USE_RC = 1 !ENDIF +# Set this non-0 to compile binaries suitable for the WinRT environment. +# This setting does not apply to any binaries that require Tcl to operate +# properly (i.e. the text fixture, etc). +# +!IFNDEF FOR_WINRT +FOR_WINRT = 0 +!ENDIF + # Set this non-0 to compile binaries suitable for the UWP environment. # This setting does not apply to any binaries that require Tcl to operate # properly (i.e. the text fixture, etc). @@ -548,14 +563,10 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS) !IF "$(PLATFORM)"=="x86" CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall - - !ELSE !IFNDEF PLATFORM CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall - - !ELSE CORE_CCONV_OPTS = SHELL_CCONV_OPTS = @@ -656,6 +667,18 @@ SHELL_LINK_OPTS = $(SHELL_CORE_LIB) TCC = $(TCC) -FAcs !ENDIF +# When compiling the library for use in the WinRT environment, +# the following compile-time options must be used as well to +# disable use of Win32 APIs that are not available and to enable +# use of Win32 APIs that are specific to Windows 8 and/or WinRT. +# +!IF $(FOR_WINRT)!=0 +TCC = $(TCC) -DSQLITE_OS_WINRT=1 +RCC = $(RCC) -DSQLITE_OS_WINRT=1 +TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP +RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP +!ENDIF + # C compiler options for the Windows 10 platform (needs MSVC 2015). # !IF $(FOR_WIN10)!=0 @@ -668,7 +691,7 @@ BCC = $(BCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE # USE_CRT_DLL option is set to force dynamically linking to the # MSVC runtime library. # -!IF $(USE_CRT_DLL)!=0 +!IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0 !IF $(DEBUG)>1 TCC = $(TCC) -MDd BCC = $(BCC) -MDd @@ -691,6 +714,16 @@ ZLIBCFLAGS = -nologo -MT -W3 -O2 -Oy- -Zi !ENDIF +# Define -DNDEBUG to compile without debugging (i.e., for production usage) +# Omitting the define will cause extra debugging code to be inserted and +# includes extra comments when "EXPLAIN stmt" is used. +# +!IF $(DEBUG)==0 +TCC = $(TCC) -DNDEBUG +BCC = $(BCC) -DNDEBUG +RCC = $(RCC) -DNDEBUG +!ENDIF + !IF $(DEBUG)>0 || $(API_ARMOR)!=0 || $(FOR_WIN10)!=0 TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1 RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 @@ -880,6 +913,56 @@ LTLINKOPTS = /NOLOGO LTLIBOPTS = /NOLOGO !ENDIF +# When compiling for use in the WinRT environment, the following +# linker option must be used to mark the executable as runnable +# only in the context of an application container. +# +!IF $(FOR_WINRT)!=0 +LTLINKOPTS = $(LTLINKOPTS) /APPCONTAINER +!IF "$(VISUALSTUDIOVERSION)"=="12.0" || "$(VISUALSTUDIOVERSION)"=="14.0" +!IFNDEF STORELIBPATH +!IF "$(PLATFORM)"=="x86" +STORELIBPATH = $(CRTLIBPATH)\store +!ELSEIF "$(PLATFORM)"=="x64" +STORELIBPATH = $(CRTLIBPATH)\store\amd64 +!ELSEIF "$(PLATFORM)"=="ARM" +STORELIBPATH = $(CRTLIBPATH)\store\arm +!ELSE +STORELIBPATH = $(CRTLIBPATH)\store +!ENDIF +!ENDIF +STORELIBPATH = $(STORELIBPATH:\\=\) +LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(STORELIBPATH)" +!ENDIF +!ENDIF + +# When compiling for Windows Phone 8.1, an extra library path is +# required. +# +!IF $(USE_WP81_OPTS)!=0 +!IFNDEF WP81LIBPATH +!IF "$(PLATFORM)"=="x86" +WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 +!ELSEIF "$(PLATFORM)"=="ARM" +WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\ARM +!ELSE +WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 +!ENDIF +!ENDIF +!ENDIF + +# When compiling for Windows Phone 8.1, some extra linker options +# are also required. +# +!IF $(USE_WP81_OPTS)!=0 +!IFDEF WP81LIBPATH +LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(WP81LIBPATH)" +!ENDIF +LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE +LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib +LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib +!ENDIF + # When compiling for UWP or the Windows 10 platform, some extra linker # options are also required. # @@ -903,9 +986,9 @@ LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib # If either debugging or symbols are enabled, enable PDBs. # !IF $(DEBUG)>1 || $(SYMBOLS)!=0 -LDFLAGS = /NODEFAULTLIB:msvcrt /DEBUG $(LDOPTS) +LDFLAGS = /DEBUG $(LDOPTS) !ELSE -LDFLAGS = /NODEFAULTLIB:msvcrt $(LDOPTS) +LDFLAGS = $(LDOPTS) !ENDIF @@ -941,7 +1024,6 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_PERCENTILE=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1 -SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_STRICT_SUBTYPE=1 !ENDIF @@ -990,8 +1072,6 @@ $(SQLITE3EXE): shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLIT /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) -tclsqlite-ex.c: - # Rule to build the amalgamation # sqlite3.lo: $(SQLITE3C) diff --git a/autosetup/README.md b/autosetup/README.md index 2560bfab9..c8da5c643 100644 --- a/autosetup/README.md +++ b/autosetup/README.md @@ -375,11 +375,6 @@ configure process, and check it in. Patching Autosetup for Project-local Changes ------------------------------------------------------------------------ -The autosetup files require the following patches after updating -from their upstream sources: - -### `--debug` flag - Autosetup reserves the flag name **`--debug`** for its own purposes, and its own special handling of `--enable-...` flags makes `--debug` an alias for `--enable-debug`. As this project has a long history of @@ -393,11 +388,6 @@ If autosetup is upgraded and this patch is _not_ applied the invoking `./configure` will fail loudly because of the declaration of the `debug` flag in `auto.def` - duplicated flags are not permitted. -### Fail on `malloc()` error - -See [check-in 72c8a5b94cdf5d](/info/72c8a5b94cdf5d). - - <a name="branch-customization"></a> Branch-specific Customization ======================================================================== diff --git a/autosetup/jimsh0.c b/autosetup/jimsh0.c index 0f0a89088..1a6453d0c 100644 --- a/autosetup/jimsh0.c +++ b/autosetup/jimsh0.c @@ -7409,13 +7409,11 @@ void *JimDefaultAllocator(void *ptr, size_t size) free(ptr); return NULL; } + else if (ptr) { + return realloc(ptr, size); + } else { - void *p = realloc(ptr, size); - if( p==0 ){ - fprintf(stderr,"Out of memory\n"); - exit(1); - } - return p; + return malloc(size); } } diff --git a/autosetup/proj.tcl b/autosetup/proj.tcl index caa679ad6..86f4df44e 100644 --- a/autosetup/proj.tcl +++ b/autosetup/proj.tcl @@ -1842,7 +1842,7 @@ proc proj-setup-autoreconfig {defName} { } # -# @prop-define-append defineName args... +# @prop-append-to defineName args... # # A proxy for Autosetup's [define-append]. Appends all non-empty $args # to [define-append $defineName]. @@ -1873,7 +1873,7 @@ proc proj-define-append {defineName args} { # but it is technically correct and still relevant on some # environments. # -# See: proj-define-append +# See: proj-append-to # proc proj-define-amend {args} { set defName "" diff --git a/autosetup/sqlite-config.tcl b/autosetup/sqlite-config.tcl index fe1b355ab..7c798b31a 100644 --- a/autosetup/sqlite-config.tcl +++ b/autosetup/sqlite-config.tcl @@ -557,25 +557,7 @@ proc proc-debug {msg} { } define OPT_FEATURE_FLAGS {} ; # -DSQLITE_OMIT/ENABLE flags. -# -# OPT_SHELL = feature-related CFLAGS for the sqlite3 CLI app. The -# list's initial values are defaults which are always applied and not -# affected by --feature-flags. The list is appended to by various -# --feature-flags. -define OPT_SHELL { - -DSQLITE_DQS=0 - -DSQLITE_ENABLE_FTS4 - -DSQLITE_ENABLE_RTREE - -DSQLITE_ENABLE_EXPLAIN_COMMENTS - -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION - -DSQLITE_ENABLE_STMTVTAB - -DSQLITE_ENABLE_DBPAGE_VTAB - -DSQLITE_ENABLE_DBSTAT_VTAB - -DSQLITE_ENABLE_BYTECODE_VTAB - -DSQLITE_ENABLE_OFFSET_SQL_FUNC - -DSQLITE_ENABLE_PERCENTILE - -DSQLITE_STRICT_SUBTYPE=1 -} +define OPT_SHELL {} ; # Feature-related CFLAGS for the sqlite3 CLI app ######################################################################## # Adds $args, if not empty, to OPT_FEATURE_FLAGS. If the first arg is # -shell then it strips that arg and passes the remaining args the @@ -679,7 +661,6 @@ proc sqlite-check-common-system-deps {} { define HAVE_ZLIB 1 define LDFLAGS_ZLIB -lz sqlite-add-shell-opt -DSQLITE_HAVE_ZLIB=1 - sqlite-add-feature-flag -DSQLITE_HAVE_ZLIB=1 } else { define HAVE_ZLIB 0 define LDFLAGS_ZLIB "" @@ -806,8 +787,7 @@ proc sqlite-handle-common-feature-flags {} { sqlite-add-feature-flag -DSQLITE_ENABLE_MEMSYS3 } } - bytecode-vtab -DSQLITE_ENABLE_BYTECODE_VTAB {} - scanstatus {-DSQLITE_ENABLE_STMT_SCANSTATUS -DSQLITE_ENABLE_BYTECODE_VTAB} {} + scanstatus -DSQLITE_ENABLE_STMT_SCANSTATUS {} column-metadata -DSQLITE_ENABLE_COLUMN_METADATA {} dbpage -DSQLITE_ENABLE_DBPAGE_VTAB {} dbstat -DSQLITE_ENABLE_DBSTAT_VTAB {} diff --git a/doc/compile-for-unix.md b/doc/compile-for-unix.md index 659bdb038..ce76b97ba 100644 --- a/doc/compile-for-unix.md +++ b/doc/compile-for-unix.md @@ -22,10 +22,9 @@ guidance on building for Windows. or <https://sqlite.org/tmp/tcl9.0.0.tar.gz>. <li>Untar the source archive. CD into the "unix/" subfolder of the source tree. - <li>Run: &ensp; `mkdir $HOME/local` - <li>Run: &ensp; `./configure --prefix=$HOME/local`<br> - SQLite deliverable builds add: &emsp; `--static CFLAGS=-Os` - <li>Run: &ensp; `make install` + <li>Run: `mkdir $HOME/local` + <li>Run: `./configure --prefix=$HOME/local` + <li>Run: `make install` </ol> <p> As of 2024-10-25, TCL is not longer required for many @@ -42,12 +41,6 @@ guidance on building for Windows. You do not need to use --with-tclsh if the tclsh you want to use is the first one on your PATH or if you are building without TCL. - The SQLite developers typically add the - &ensp; `--with-linenoise=$HOME/linenoise` &ensp; option - to provide command-line editing. "$HOME/linenoise" is a directory - that contains [linenoise](https://github.com/antirez/linenoise) source - code files, `linenoise.c` and `linenoise.h`. - 6. Run the "`Makefile`" makefile with an appropriate target. Examples: <ul> diff --git a/doc/compile-for-windows.md b/doc/compile-for-windows.md index 30536d5fd..0e59c83fe 100644 --- a/doc/compile-for-windows.md +++ b/doc/compile-for-windows.md @@ -180,39 +180,3 @@ statically linked so that it does not depend on separate DLL: 6. After your executable is built, you can verify that it does not depend on the TCL DLL by running: <blockquote><pre>dumpbin /dependents sqlite3_analyzer.exe</pre></blockquote> - -## Linking Against ZLIB - -Some feature (such as zip-file support in the CLI) require the ZLIB -compression library. That library is more or less universally available -on unix platforms, but is seldom provided on Windows. You will probably -need to provide it yourself. Here the the steps needed: - - 1. Download the zlib-1.3.1.tar.gz tarball (or a similar version). - Unpack the tarball sources. You can put them wherever you like. - For the purposes of this document, let's assume you put the source - tree in c:\\zlib-64. Note: If you are building for both x64 and - x86, you will need separate builds of ZLIB for each, thus separate - build directories. - - 2. Before building SQLite (as described above) first make these - environment changes. The lead-programmer for SQLite (who writes - these words) has BAT files named "env-x64.bat" and "env-x32.bat" - and "env-arm64.bat" that make these changes, and he runs those - BAT file whenever he starts a new shell. These are the settings - needed: - <blockquote> - set USE_ZLIB=1<br> - set BUILD_ZLIB=0<br> - set ZLIBDIR=c:\\zlib-64 - </blockquote> - - 3. Because the settings in step 2 specify "BUILD_ZLIB=0", you will need - to build the library at least once. I recommand: - <blockquote> - make clean sqlite3.exe BUILD_ZLIB=1 - </blockquote> - - 4. After making the environment changes specified in steps 1 through 3 - above, you then build and test SQLite as you normally would. The - environment variable changes will cause ZLIB to be linked automatically. diff --git a/doc/lemon.html b/doc/lemon.html index a994b396b..965f305c0 100644 --- a/doc/lemon.html +++ b/doc/lemon.html @@ -696,7 +696,6 @@ <h3>4.4 Special Directives</h3> <li><tt><a href='#pright'>%right</a></tt> <li><tt><a href='#reallc'>%realloc</a></tt> <li><tt><a href='#stack_overflow'>%stack_overflow</a></tt> -<li><tt><a href='#reallc'>%stack_size_limit</a></tt> <li><tt><a href='#stack_size'>%stack_size</a></tt> <li><tt><a href='#start_symbol'>%start_symbol</a></tt> <li><tt><a href='#syntax_error'>%syntax_error</a></tt> @@ -1204,33 +1203,20 @@ <h4>4.4.25 The <tt>%wildcard</tt> directive</h4> The wildcard token is only matched if there are no alternatives.</p> <a id='reallc'></a> -<h4>4.4.26 The <tt>%realloc</tt>, <tt>%free</tt>, and -<tt>%stack_size_limit</tt> directives</h4> +<h4>4.4.26 The <tt>%realloc</tt> and <tt>%free</tt> directives</h4> <p>The <tt>%realloc</tt> and <tt>%free</tt> directives defines function -that allocate and free heap memory. The signatures and semantics of -these functions are similar to the realloc() and free() functions from -the standard C library, except that these functions take an extra -parameter at the end that is determined by %extra_context. If -%extra_context is not defined, then the extra argument is 0. The -extra parameter provides the capability to do better error reporting -in the event of a memory allocation error, and/or to use an alternative -private application heap. - -<p>If both of these functions are defined then they are used to -allocate and free memory for supplemental parser stack space, if -the initial parse stack space is exceeded. The initial parser stack size +that allocate and free heap memory. The signatures of these functions +should be the same as the realloc() and free() functions from the standard +C library. + +<p>If both of these functions are defined +then these functions are used to allocate and free +memory for supplemental parser stack space, if the initial +parse stack space is exceeded. The initial parser stack size is specified by either <tt>%stack_size</tt> or the -DYYSTACKDEPTH compile-time flag. -<p>The <tt>%stack_size_limit</tt> directive defines a function that returns -the maximum allowed parser stack size. If this diretive does not exist, -no size limit is enforced. This function takes a single argument which -is the %extra_context value or "0" if %extra_context is not defined. -The function should return an integer that is the maximum -number of parser stack entries. If more stack space -than this is needed, the %stack_overflow code is invoked. - <a id='errors'></a> <h2>5.0 Error Processing</h2> diff --git a/doc/testrunner.md b/doc/testrunner.md index 90ef4b71f..d1696e9d1 100644 --- a/doc/testrunner.md +++ b/doc/testrunner.md @@ -4,12 +4,6 @@ <ul type=none> <li> 1. <a href=#overview>Overview</a> -<ul type=none> - <li> 1.1. <a href=#runtr>Running testrunner.tcl</a> - <li> 1.2. <a href=#runviamake>Run using "make"</a> - <li> 1.3. <a href=#outputs>Outputs from testrunner.tcl</a> - <li> 1.4. <a href=#help>Built-in help</a> -</ul> <li> 2. <a href=#binary_tests>Binary Tests</a> <ul type=none> <li> 2.1. <a href=#organization_tests>Organization of Tcl Tests</a> @@ -32,40 +26,17 @@ The testrunner.tcl program is a Tcl script used to run multiple SQLite tests in parallel, thus reducing testing time on multi-core machines. -The testrunner.tcl supports running tests that based on `testfixture`, -`sqlite3`, and `fuzzcheck`. - -<a name="runtr"></a> -## 1.1 Running testrunner.tcl - -The testrunner.tcl script is located in the "test" subdirectory of the -SQLite source tree. So if your shell is current positioned at the top -of the source tree, you would normally run the script using the command: -"<tt>test/testrunner.tcl</tt>". On Windows, you have to specify the -`tclsh` interpreter command first, like this: -"<tt>tclsh&nbsp;test/testrunner.tcl</tt>". - -In this document, we will assume that you are on a unix-like OS -(not on Windows) and that your current directory is the root -of the SQLite source tree, and so all invocations of the testrunner.tcl -script will be of the form "<tt>test/testrunner.tcl</tt>". If you -are in a different directory, then make appropriate adjustments to -the path. On Windows, add the "<tt>tclsh</tt>" interpreter command -up front. +It supports the following types of tests: -<a name="runviamake"></a> -## 1.2 Run using make + * Tcl test scripts. -The standard Makefiles for SQLite include targets that invoke -testrunner.tcl. So the following commands also run testrunner.tcl: + * Fuzzcheck tests, including using an external fuzzcheck database. - * `make devtest` - * `make releasetest` - * `make sdevtest` - * `make testrunner` - -<a name="outputs"></a> -## 1.3 Outputs from testrunner.tcl + * Tests run with `make` commands. Examples: + - `make devtest` + - `make releasetest` + - `make sdevtest` + - `make testrunner` The testrunner.tcl program stores output of all tests and builds run in log file **testrunner.log**, created in the current working directory. @@ -83,19 +54,17 @@ A useful query might be: ``` You can get a summary of errors in a prior run by invoking commands like -those shown below. Note that the testrunner.tcl script can be run directly -on unix systems (including Macs) but you will need to add <tt>tclsh</tt> -to the front on Windows. +these: ``` - test/testrunner.tcl errors - test/testrunner.tcl errors -v + tclsh $(TESTDIR)/testrunner.tcl errors + tclsh $(TESTDIR)/testrunner.tcl errors -v ``` Running the command: ``` - test/testrunner.tcl status + tclsh $(TESTDIR)/testrunner.tcl status ``` in the directory containing the testrunner.db database runs various queries @@ -104,43 +73,32 @@ A good way to keep and eye on test progress is to run either of the two following commands: ``` - watch test/testrunner.tcl status - test/testrunner.tcl status -d 2 + watch tclsh $(TESTDIR)/testrunner.tcl status + tclsh $(TESTDIR)/testrunner.tcl status -d 2 ``` Both of the commands above accomplish about the same thing, but the second one has the advantage of not requiring "watch" to be installed on your system. -Sometimes testrunner.tcl uses the `testfixture` and `sqlite3` binaries that -are in the directory from which testrunner.tcl is run. -(see "Binary Tests" below). Sometimes it builds testfixture and +Sometimes testrunner.tcl uses the `testfixture` binary that it is run with +to run tests (see "Binary Tests" below). Sometimes it builds testfixture and other binaries in specific configurations to test (see "Source Tests"). -<a name=help></a> -## 1.4 Built-in help - -Run this command: - -``` - test/testrunner.tcl help -``` - -To get a summary of all of the various command-line options available -with testrunner.tcl - - <a name=binary_tests></a> # 2. Binary Tests The commands described in this section all run various combinations of the Tcl -test scripts using whatever `testfixture` binary (and maybe also the `sqlite3` -binary, depending on the test) that is found in the directory from which -testrunner.tcl is launched. So typically, one must first run something -like "`make testfixture sqlite3`" before launching binary tests. In other -words, testrunner.tcl does not automatically build the binaries under test -for binary tests. The testrunner.tcl expects the binaries to be available -already. +test scripts using the `testfixture` binary used to run the testrunner.tcl +script (i.e. they do not invoke the compiler to build new binaries, or the +`make` command to run tests that are not Tcl scripts). The procedure to run +these tests is therefore: + + 1. Build the "testfixture" (or "testfixture.exe" for windows) binary using + whatever method seems convenient. + + 2. Test the binary built in step 1 by running testrunner.tcl with it, + perhaps with various options. The following sub-sections describe the various options that can be passed to testrunner.tcl to test binary testfixture builds. @@ -182,22 +140,22 @@ are defined in file *testrunner_data.tcl*. To run the "veryquick" test set, use either of the following: ``` - test/testrunner.tcl - test/testrunner.tcl veryquick + ./testfixture $TESTDIR/testrunner.tcl + ./testfixture $TESTDIR/testrunner.tcl veryquick ``` To run the "full" test suite: ``` - test/testrunner.tcl full + ./testfixture $TESTDIR/testrunner.tcl full ``` To run the subset of the "full" test suite for which the test file name matches a specified pattern (e.g. all tests that start with "fts5"), either of: ``` - test/testrunner.tcl fts5% - test/testrunner.tcl 'fts5*' + ./testfixture $TESTDIR/testrunner.tcl fts5% + ./testfixture $TESTDIR/testrunner.tcl 'fts5*' ``` Strictly speaking, for a test to be run the pattern must match the script @@ -209,7 +167,7 @@ characters specified as part of the pattern are transformed to "\*". To run "all" tests (full + permutations): ``` - test/testrunner.tcl all + ./testfixture $TESTDIR/testrunner.tcl all ``` <a name=binary_test_failures></a> @@ -228,15 +186,10 @@ If there is no permutation, the individual test script may be run with: Or, if the failure occured as part of a permutation: ``` - test/testrunner.tcl $PERMUTATION $PATH_TO_SCRIPT + ./testfixture $TESTDIR/testrunner.tcl $PERMUTATION $PATH_TO_SCRIPT ``` -One can also rerun all tests that failed or did not complete -in the previous invocation by typing: - -``` - test/testrunner.tcl retest -``` +TODO: An example instead of "$PERMUTATION" and $PATH\_TO\_SCRIPT? <a name=source_code_tests></a> # 3. Source Code Tests @@ -251,9 +204,11 @@ other tests. The advantages of this are that: * it ensures that tests are always run using binaries created with the same set of compiler options. -The testrunner.tcl commands described in this section do not require that -the testfixture and/or sqlite3 binaries be built ahead of time. Those -binaries will be constructed automatically. +The testrunner.tcl commands described in this section may be run using +either a *testfixture* (or testfixture.exe) build, or with any other Tcl +shell that supports SQLite 3.31.1 or newer via "package require sqlite3". + +TODO: ./configure + Makefile.msc build systems. <a name=commands_to_run_tests></a> ## 3.1. Commands to Run SQLite Tests @@ -263,7 +218,7 @@ the `make fuzztest` target once for each of two --enable-all builds - one with debugging enabled and one without: ``` - test/testrunner.tcl mdevtest + tclsh $TESTDIR/testrunner.tcl mdevtest ``` In other words, it is equivalent to running: @@ -272,13 +227,13 @@ In other words, it is equivalent to running: $TOP/configure --enable-all --enable-debug make fuzztest make testfixture - $TOP/test/testrunner.tcl veryquick + ./testfixture $TOP/test/testrunner.tcl veryquick # Then, after removing files created by the tests above: $TOP/configure --enable-all OPTS="-O0" make fuzztest make testfixture - $TOP/test/testrunner.tcl veryquick + ./testfixture $TOP/test/testrunner.tcl veryquick ``` The **sdevtest** command is identical to the mdevtest command, except that the @@ -286,7 +241,7 @@ second of the two builds is a sanitizer build. Specifically, this means that OPTS="-fsanitize=address,undefined" is specified instead of OPTS="-O0": ``` - test/testrunner.tcl sdevtest + tclsh $TESTDIR/testrunner.tcl sdevtest ``` The **release** command runs lots of tests under lots of builds. It runs @@ -295,7 +250,7 @@ on Linux, Windows or OSX. Refer to *testrunner\_data.tcl* for the details of the specific tests run. ``` - test/testrunner.tcl release + tclsh $TESTDIR/testrunner.tcl release ``` As with <a href=#source code tests>source code tests</a>, one or more patterns @@ -303,7 +258,7 @@ may be appended to any of the above commands (mdevtest, sdevtest or release). Pattern matching is used for both Tcl tests and fuzz tests. ``` - test/testrunner.tcl release rtree% + tclsh $TESTDIR/testrunner.tcl release rtree% ``` <a name=zipvfs_tests></a> @@ -313,14 +268,14 @@ testrunner.tcl can build a zipvfs-enabled testfixture and use it to run tests from the Zipvfs project with the following command: ``` - test/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS + tclsh $TESTDIR/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS ``` This can be combined with any of "mdevtest", "sdevtest" or "release" to test both SQLite and Zipvfs with a single command: ``` - test/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS mdevtest + tclsh $TESTDIR/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS mdevtest ``` <a name=source_code_test_failures></a> @@ -340,10 +295,10 @@ a dos \*.bat file on windows. For example: ``` # Create a script that recreates build configuration "Device-One" on # Linux or OSX: - test/testrunner.tcl script Device-One > make.sh + tclsh $TESTDIR/testrunner.tcl script Device-One > make.sh # Create a script that recreates build configuration "Have-Not" on Windows: - test/testrunner.tcl script Have-Not > make.bat + tclsh $TESTDIR/testrunner.tcl script Have-Not > make.bat ``` The generated bash or \*.bat file script accepts a single argument - a makefile @@ -366,7 +321,7 @@ Thus, for example, to run a full releasetest including an external dbsqlfuzz database, run a command like one of these: ``` - test/testrunner.tcl releasetest --fuzzdb ../fuzz/20250415.db + tclsh test/testrunner.tcl releasetest --fuzzdb ../fuzz/20250415.db FUZZDB=../fuzz/20250415.db make releasetest nmake /f Makefile.msc FUZZDB=../fuzz/20250415.db releasetest ``` @@ -376,7 +331,7 @@ databases. So if you want to run *only* tests involving the external database, you can use a command something like this: ``` - test/testrunner.tcl releasetest 20250415 --fuzzdb ../fuzz/20250415.db + tclsh test/testrunner.tcl releasetest 20250415 --fuzzdb ../fuzz/20250415.db ``` <a name=testrunner_options></a> @@ -390,7 +345,7 @@ required by a test, not to run any actual tests. For example: ``` # Build binaries required by release test. - test/testrunner.tcl --buildonly release" + tclsh $TESTDIR/testrunner.tcl --buildonly release" ``` The **--dryrun** option prevents testrunner.tcl from building any binaries @@ -399,7 +354,7 @@ would normally execute into the testrunner.log file. Example: ``` # Log the shell commmands that make up the mdevtest test. - test/testrunner.tcl --dryrun mdevtest" + tclsh $TESTDIR/testrunner.tcl --dryrun mdevtest" ``` The **--explain** option is similar to --dryrun in that it prevents @@ -409,7 +364,7 @@ summary of all the builds and tests that would have been run. ``` # Show what builds and tests would have been run - test/testrunner.tcl --explain mdevtest + tclsh $TESTDIR/testrunner.tcl --explain mdevtest ``` The **--status** option uses VT100 escape sequences to display the test @@ -425,7 +380,7 @@ When running either binary or source code tests, testrunner.tcl reports the number of jobs it intends to use to stdout. e.g. ``` - $ test/testrunner.tcl + $ ./testfixture $TESTDIR/testrunner.tcl splitting work across 16 jobs ... more output ... ``` @@ -435,7 +390,7 @@ of real cores on the machine. This can be overridden using the "--jobs" (or -j) switch: ``` - $ test/testrunner.tcl --jobs 8 + $ ./testfixture $TESTDIR/testrunner.tcl --jobs 8 splitting work across 8 jobs ... more output ... ``` @@ -445,5 +400,5 @@ running by exucuting the following command from the directory containing the testrunner.log and testrunner.db files: ``` - $ test/testrunner.tcl njob $NEW_NUMBER_OF_JOBS + $ ./testfixture $TESTDIR/testrunner.tcl njob $NEW_NUMBER_OF_JOBS ``` diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test index aaea03711..0c3b512af 100644 --- a/ext/expert/expert1.test +++ b/ext/expert/expert1.test @@ -90,7 +90,7 @@ foreach {tn setup} { proc do_rec_test {tn sql res} { set res [squish [string trim $res]] set tst [subst -nocommands { - squish [string trim [exec $::CLI -noinit test.db ".expert" {$sql;}]] + squish [string trim [exec $::CLI test.db ".expert" {$sql;}]] }] uplevel [list do_test $tn $tst $res] } diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 368e9b189..f178abafe 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1816,7 +1816,9 @@ static int fts3CursorSeekStmt(Fts3Cursor *pCsr){ zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); if( !zSql ) return SQLITE_NOMEM; p->bLock++; - rc = sqlite3Fts3PrepareStmt(p, zSql, 1, 1, &pCsr->pStmt); + rc = sqlite3_prepare_v3( + p->db, zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0 + ); p->bLock--; sqlite3_free(zSql); } @@ -3391,7 +3393,9 @@ static int fts3FilterMethod( } if( zSql ){ p->bLock++; - rc = sqlite3Fts3PrepareStmt(p, zSql, 1, 1, &pCsr->pStmt); + rc = sqlite3_prepare_v3( + p->db,zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0 + ); p->bLock--; sqlite3_free(zSql); }else{ @@ -4014,7 +4018,6 @@ static int fts3IntegrityMethod( UNUSED_PARAMETER(isQuick); rc = sqlite3Fts3IntegrityCheck(p, &bOk); - assert( pVtab->zErrMsg==0 || rc!=SQLITE_OK ); assert( rc!=SQLITE_CORRUPT_VTAB ); if( rc==SQLITE_ERROR || (rc&0xFF)==SQLITE_CORRUPT ){ *pzErr = sqlite3_mprintf("unable to validate the inverted index for" diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 556635def..e98b90a75 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -601,15 +601,6 @@ int sqlite3Fts3Incrmerge(Fts3Table*,int,int); (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ ) -int sqlite3Fts3PrepareStmt( - Fts3Table *p, /* Prepare for this connection */ - const char *zSql, /* SQL to prepare */ - int bPersist, /* True to set SQLITE_PREPARE_PERSISTENT */ - int bAllowVtab, /* True to omit SQLITE_PREPARE_NO_VTAB */ - sqlite3_stmt **pp /* OUT: Prepared statement */ -); - - /* fts3.c */ void sqlite3Fts3ErrMsg(char**,const char*,...); int sqlite3Fts3PutVarint(char *, sqlite3_int64); diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 1b8bca70f..19dff31f0 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -98,9 +98,9 @@ typedef struct SegmentWriter SegmentWriter; ** incrementally. See function fts3PendingListAppend() for details. */ struct PendingList { - sqlite3_int64 nData; + int nData; char *aData; - sqlite3_int64 nSpace; + int nSpace; sqlite3_int64 iLastDocid; sqlite3_int64 iLastCol; sqlite3_int64 iLastPos; @@ -273,24 +273,6 @@ struct SegmentNode { #define SQL_UPDATE_LEVEL_IDX 38 #define SQL_UPDATE_LEVEL 39 -/* -** Wrapper around sqlite3_prepare_v3() to ensure that SQLITE_PREPARE_FROM_DDL -** is always set. -*/ -int sqlite3Fts3PrepareStmt( - Fts3Table *p, /* Prepare for this connection */ - const char *zSql, /* SQL to prepare */ - int bPersist, /* True to set SQLITE_PREPARE_PERSISTENT */ - int bAllowVtab, /* True to omit SQLITE_PREPARE_NO_VTAB */ - sqlite3_stmt **pp /* OUT: Prepared statement */ -){ - int f = SQLITE_PREPARE_FROM_DDL - |((bAllowVtab==0) ? SQLITE_PREPARE_NO_VTAB : 0) - |(bPersist ? SQLITE_PREPARE_PERSISTENT : 0); - - return sqlite3_prepare_v3(p->db, zSql, -1, f, pp, NULL); -} - /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, @@ -416,12 +398,12 @@ static int fts3SqlStmt( pStmt = p->aStmt[eStmt]; if( !pStmt ){ - int bAllowVtab = 0; + int f = SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_NO_VTAB; char *zSql; if( eStmt==SQL_CONTENT_INSERT ){ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ - bAllowVtab = 1; + f &= ~SQLITE_PREPARE_NO_VTAB; zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist); }else{ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); @@ -429,7 +411,7 @@ static int fts3SqlStmt( if( !zSql ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3Fts3PrepareStmt(p, zSql, 1, bAllowVtab, &pStmt); + rc = sqlite3_prepare_v3(p->db, zSql, -1, f, &pStmt, NULL); sqlite3_free(zSql); assert( rc==SQLITE_OK || pStmt==0 ); p->aStmt[eStmt] = pStmt; @@ -778,9 +760,7 @@ static int fts3PendingTermsAddOne( pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); if( pList ){ - assert( (i64)pList->nData+(i64)nToken+(i64)sizeof(Fts3HashElem) - <= (i64)p->nPendingData ); - p->nPendingData -= (int)(pList->nData + nToken + sizeof(Fts3HashElem)); + p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); } if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ @@ -793,9 +773,7 @@ static int fts3PendingTermsAddOne( } } if( rc==SQLITE_OK ){ - assert( (i64)p->nPendingData + pList->nData + nToken - + sizeof(Fts3HashElem) <= 0x3fffffff ); - p->nPendingData += (int)(pList->nData + nToken + sizeof(Fts3HashElem)); + p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); } return rc; } @@ -3596,7 +3574,7 @@ static int fts3DoRebuild(Fts3Table *p){ if( !zSql ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3Fts3PrepareStmt(p, zSql, 0, 1, &pStmt); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); } @@ -5349,7 +5327,7 @@ int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){ if( !zSql ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3Fts3PrepareStmt(p, zSql, 0, 1, &pStmt); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); } @@ -5479,7 +5457,7 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ v = atoi(&zVal[9]); if( v>=24 && v<=p->nPgsz-35 ) p->nNodeSize = v; rc = SQLITE_OK; - }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 11) ){ + }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ v = atoi(&zVal[11]); if( v>=64 && v<=FTS3_MAX_PENDING_DATA ) p->nMaxPendingData = v; rc = SQLITE_OK; diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index ee43ca6cc..95b33ea31 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -455,7 +455,7 @@ static void fts5SnippetFunction( iBestCol = (iCol>=0 ? iCol : 0); nPhrase = pApi->xPhraseCount(pFts); - aSeen = sqlite3_malloc64(nPhrase); + aSeen = sqlite3_malloc(nPhrase); if( aSeen==0 ){ rc = SQLITE_NOMEM; } diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c index d799e34cb..afcd83b6b 100644 --- a/ext/fts5/fts5_buffer.c +++ b/ext/fts5/fts5_buffer.c @@ -288,7 +288,7 @@ char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){ if( nIn<0 ){ nIn = (int)strlen(pIn); } - zRet = (char*)sqlite3_malloc64((i64)nIn+1); + zRet = (char*)sqlite3_malloc(nIn+1); if( zRet ){ memcpy(zRet, pIn, nIn); zRet[nIn] = '\0'; diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index cea14b500..eea82b046 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -576,7 +576,7 @@ int sqlite3Fts5ConfigParse( sqlite3_int64 nByte; int bUnindexed = 0; /* True if there are one or more UNINDEXED */ - *ppOut = pRet = (Fts5Config*)sqlite3_malloc64(sizeof(Fts5Config)); + *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config)); if( pRet==0 ) return SQLITE_NOMEM; memset(pRet, 0, sizeof(Fts5Config)); pRet->pGlobal = pGlobal; @@ -1123,3 +1123,5 @@ void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){ va_end(ap); } + + diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 8ecaca34f..352df81f4 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -314,7 +314,7 @@ int sqlite3Fts5ExprNew( assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 ); if( sParse.rc==SQLITE_OK ){ - *ppNew = pNew = sqlite3_malloc64(sizeof(Fts5Expr)); + *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr)); if( pNew==0 ){ sParse.rc = SQLITE_NOMEM; sqlite3Fts5ParseNodeFree(sParse.pExpr); @@ -466,7 +466,7 @@ int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){ p2->pRoot = 0; if( sParse.rc==SQLITE_OK ){ - Fts5ExprPhrase **ap = (Fts5ExprPhrase**)sqlite3_realloc64( + Fts5ExprPhrase **ap = (Fts5ExprPhrase**)sqlite3_realloc( p1->apExprPhrase, nPhrase * sizeof(Fts5ExprPhrase*) ); if( ap==0 ){ diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c index ba4a030b7..a33dec9a9 100644 --- a/ext/fts5/fts5_hash.c +++ b/ext/fts5/fts5_hash.c @@ -91,7 +91,7 @@ int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){ int rc = SQLITE_OK; Fts5Hash *pNew; - *ppNew = pNew = (Fts5Hash*)sqlite3_malloc64(sizeof(Fts5Hash)); + *ppNew = pNew = (Fts5Hash*)sqlite3_malloc(sizeof(Fts5Hash)); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 164d61388..b10df893f 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -2093,7 +2093,7 @@ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ /* If necessary, grow the pIter->aRowidOffset[] array. */ if( iRowidOffset>=pIter->nRowidOffset ){ - i64 nNew = pIter->nRowidOffset + 8; + int nNew = pIter->nRowidOffset + 8; int *aNew = (int*)sqlite3_realloc64(pIter->aRowidOffset,nNew*sizeof(int)); if( aNew==0 ){ p->rc = SQLITE_NOMEM; @@ -5399,31 +5399,31 @@ static void fts5DoSecureDelete( ** is another term following it on this page. So the subsequent term ** needs to be moved to replace the term associated with the entry ** being removed. */ - u64 nPrefix = 0; - u64 nSuffix = 0; - u64 nPrefix2 = 0; - u64 nSuffix2 = 0; + int nPrefix = 0; + int nSuffix = 0; + int nPrefix2 = 0; + int nSuffix2 = 0; iDelKeyOff = iNextOff; - iNextOff += fts5GetVarint(&aPg[iNextOff], &nPrefix2); - iNextOff += fts5GetVarint(&aPg[iNextOff], &nSuffix2); + iNextOff += fts5GetVarint32(&aPg[iNextOff], nPrefix2); + iNextOff += fts5GetVarint32(&aPg[iNextOff], nSuffix2); if( iKey!=1 ){ - iKeyOff += fts5GetVarint(&aPg[iKeyOff], &nPrefix); + iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nPrefix); } - iKeyOff += fts5GetVarint(&aPg[iKeyOff], &nSuffix); + iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nSuffix); nPrefix = MIN(nPrefix, nPrefix2); nSuffix = (nPrefix2 + nSuffix2) - nPrefix; - if( (iKeyOff+nSuffix)>(u64)iPgIdx || (iNextOff+nSuffix2)>(u64)iPgIdx ){ + if( (iKeyOff+nSuffix)>iPgIdx || (iNextOff+nSuffix2)>iPgIdx ){ FTS5_CORRUPT_IDX(p); }else{ if( iKey!=1 ){ iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix); } iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix); - if( nPrefix2>(u64)pSeg->term.n ){ + if( nPrefix2>pSeg->term.n ){ FTS5_CORRUPT_IDX(p); }else if( nPrefix2>nPrefix ){ memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix); @@ -5454,7 +5454,7 @@ static void fts5DoSecureDelete( u8 *aTermIdx = &pTerm->p[pTerm->szLeaf]; int nTermIdx = pTerm->nn - pTerm->szLeaf; int iTermIdx = 0; - i64 iTermOff = 0; + int iTermOff = 0; while( 1 ){ u32 iVal = 0; @@ -5465,15 +5465,12 @@ static void fts5DoSecureDelete( } nTermIdx = iTermIdx; - if( iTermOff>pTerm->szLeaf ){ - FTS5_CORRUPT_IDX(p); - }else{ - memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); - fts5PutU16(&pTerm->p[2], iTermOff); - fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); - if( nTermIdx==0 ){ - fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); - } + memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); + fts5PutU16(&pTerm->p[2], iTermOff); + + fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); + if( nTermIdx==0 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); } } fts5DataRelease(pTerm); @@ -5496,9 +5493,7 @@ static void fts5DoSecureDelete( int iPrevKeyOut = 0; int iKeyIn = 0; - if( nMove>0 ){ - memmove(&aPg[iOff], &aPg[iNextOff], nMove); - } + memmove(&aPg[iOff], &aPg[iNextOff], nMove); iPgIdx -= nShift; nPg = iPgIdx; fts5PutU16(&aPg[2], iPgIdx); @@ -6418,16 +6413,16 @@ struct Fts5TokenDataMap { ** aMap[] variables. */ struct Fts5TokenDataIter { - i64 nMapAlloc; /* Allocated size of aMap[] in entries */ - i64 nMap; /* Number of valid entries in aMap[] */ + int nMapAlloc; /* Allocated size of aMap[] in entries */ + int nMap; /* Number of valid entries in aMap[] */ Fts5TokenDataMap *aMap; /* Array of (rowid+pos -> token) mappings */ /* The following are used for prefix-queries only. */ Fts5Buffer terms; /* The following are used for other full-token tokendata queries only. */ - i64 nIter; - i64 nIterAlloc; + int nIter; + int nIterAlloc; Fts5PoslistReader *aPoslistReader; int *aPoslistToIter; Fts5Iter *apIter[FLEXARRAY]; @@ -6483,11 +6478,11 @@ static void fts5TokendataIterAppendMap( ){ if( p->rc==SQLITE_OK ){ if( pT->nMap==pT->nMapAlloc ){ - i64 nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64; - i64 nAlloc = nNew * sizeof(Fts5TokenDataMap); + int nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64; + int nAlloc = nNew * sizeof(Fts5TokenDataMap); Fts5TokenDataMap *aNew; - aNew = (Fts5TokenDataMap*)sqlite3_realloc64(pT->aMap, nAlloc); + aNew = (Fts5TokenDataMap*)sqlite3_realloc(pT->aMap, nAlloc); if( aNew==0 ){ p->rc = SQLITE_NOMEM; return; @@ -6513,7 +6508,7 @@ static void fts5TokendataIterAppendMap( */ static void fts5TokendataIterSortMap(Fts5Index *p, Fts5TokenDataIter *pT){ Fts5TokenDataMap *aTmp = 0; - i64 nByte = pT->nMap * sizeof(Fts5TokenDataMap); + int nByte = pT->nMap * sizeof(Fts5TokenDataMap); aTmp = (Fts5TokenDataMap*)sqlite3Fts5MallocZero(&p->rc, nByte); if( aTmp ){ @@ -7047,10 +7042,9 @@ static Fts5TokenDataIter *fts5AppendTokendataIter( if( p->rc==SQLITE_OK ){ if( pIn==0 || pIn->nIter==pIn->nIterAlloc ){ - i64 nAlloc = pIn ? pIn->nIterAlloc*2 : 16; - i64 nByte = SZ_FTS5TOKENDATAITER(nAlloc+1); - Fts5TokenDataIter *pNew; - pNew = (Fts5TokenDataIter*)sqlite3_realloc64(pIn, nByte); + int nAlloc = pIn ? pIn->nIterAlloc*2 : 16; + int nByte = SZ_FTS5TOKENDATAITER(nAlloc+1); + Fts5TokenDataIter *pNew = (Fts5TokenDataIter*)sqlite3_realloc(pIn, nByte); if( pNew==0 ){ p->rc = SQLITE_NOMEM; @@ -7147,8 +7141,8 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){ /* Ensure the token-mapping is large enough */ if( eDetail==FTS5_DETAIL_FULL && pT->nMapAlloc<(pT->nMap + nByte) ){ - i64 nNew = (pT->nMapAlloc + nByte) * 2; - Fts5TokenDataMap *aNew = (Fts5TokenDataMap*)sqlite3_realloc64( + int nNew = (pT->nMapAlloc + nByte) * 2; + Fts5TokenDataMap *aNew = (Fts5TokenDataMap*)sqlite3_realloc( pT->aMap, nNew*sizeof(Fts5TokenDataMap) ); if( aNew==0 ){ diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index cf033ab5d..f45b9ef90 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -631,7 +631,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ return SQLITE_ERROR; } - idxStr = (char*)sqlite3_malloc64((i64)pInfo->nConstraint * 8 + 1); + idxStr = (char*)sqlite3_malloc(pInfo->nConstraint * 8 + 1); if( idxStr==0 ) return SQLITE_NOMEM; pInfo->idxStr = idxStr; pInfo->needToFreeIdxStr = 1; @@ -2081,7 +2081,6 @@ static int fts5UpdateMethod( } update_out: - sqlite3Fts5IndexCloseReader(pTab->p.pIndex); pTab->p.pConfig->pzErrmsg = 0; return rc; } @@ -3763,7 +3762,7 @@ static int fts5Init(sqlite3 *db){ int rc; Fts5Global *pGlobal = 0; - pGlobal = (Fts5Global*)sqlite3_malloc64(sizeof(Fts5Global)); + pGlobal = (Fts5Global*)sqlite3_malloc(sizeof(Fts5Global)); if( pGlobal==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index f5d8705ff..25cd5c063 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -391,7 +391,7 @@ static int SQLITE_TCLAPI xF5tApi( break; } CASE(12, "xSetAuxdata") { - F5tAuxData *pData = (F5tAuxData*)sqlite3_malloc64(sizeof(F5tAuxData)); + F5tAuxData *pData = (F5tAuxData*)sqlite3_malloc(sizeof(F5tAuxData)); if( pData==0 ){ Tcl_AppendResult(interp, "out of memory", (char*)0); return TCL_ERROR; @@ -780,7 +780,7 @@ static int SQLITE_TCLAPI f5tTokenize( } if( nText>0 ){ - pCopy = sqlite3_malloc64(nText); + pCopy = sqlite3_malloc(nText); if( pCopy==0 ){ tokenizer.xDelete(pTok); Tcl_AppendResult(interp, "error in sqlite3_malloc()", (char*)0); @@ -1420,7 +1420,7 @@ static int f5tOrigintextCreate( void *pTokCtx = 0; int rc = SQLITE_OK; - pTok = (OriginTextTokenizer*)sqlite3_malloc64(sizeof(OriginTextTokenizer)); + pTok = (OriginTextTokenizer*)sqlite3_malloc(sizeof(OriginTextTokenizer)); if( pTok==0 ){ rc = SQLITE_NOMEM; }else if( nArg<1 ){ @@ -1480,7 +1480,7 @@ static int xOriginToken( int nReq = nToken + 1 + (iEnd-iStart); if( nReq>p->nBuf ){ sqlite3_free(p->aBuf); - p->aBuf = sqlite3_malloc64(nReq*2); + p->aBuf = sqlite3_malloc(nReq*2); if( p->aBuf==0 ) return SQLITE_NOMEM; p->nBuf = nReq*2; } diff --git a/ext/fts5/fts5_test_tok.c b/ext/fts5/fts5_test_tok.c index c77c49de7..994d304dc 100644 --- a/ext/fts5/fts5_test_tok.c +++ b/ext/fts5/fts5_test_tok.c @@ -194,7 +194,7 @@ static int fts5tokConnectMethod( } if( rc==SQLITE_OK ){ - pTab = (Fts5tokTable*)sqlite3_malloc64(sizeof(Fts5tokTable)); + pTab = (Fts5tokTable*)sqlite3_malloc(sizeof(Fts5tokTable)); if( pTab==0 ){ rc = SQLITE_NOMEM; }else{ @@ -275,7 +275,7 @@ static int fts5tokBestIndexMethod( static int fts5tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ Fts5tokCursor *pCsr; - pCsr = (Fts5tokCursor *)sqlite3_malloc64(sizeof(Fts5tokCursor)); + pCsr = (Fts5tokCursor *)sqlite3_malloc(sizeof(Fts5tokCursor)); if( pCsr==0 ){ return SQLITE_NOMEM; } @@ -347,7 +347,7 @@ static int fts5tokCb( if( pCsr->nRow ){ pRow->iPos = pRow[-1].iPos + ((tflags & FTS5_TOKEN_COLOCATED) ? 0 : 1); } - pRow->zToken = sqlite3_malloc64((sqlite3_int64)nToken+1); + pRow->zToken = sqlite3_malloc(nToken+1); if( pRow->zToken==0 ) return SQLITE_NOMEM; memcpy(pRow->zToken, pToken, nToken); pRow->zToken[nToken] = 0; @@ -373,8 +373,8 @@ static int fts5tokFilterMethod( fts5tokResetCursor(pCsr); if( idxNum==1 ){ const char *zByte = (const char *)sqlite3_value_text(apVal[0]); - sqlite3_int64 nByte = sqlite3_value_bytes(apVal[0]); - pCsr->zInput = sqlite3_malloc64(nByte+1); + int nByte = sqlite3_value_bytes(apVal[0]); + pCsr->zInput = sqlite3_malloc(nByte+1); if( pCsr->zInput==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c index 990810239..b8a113646 100644 --- a/ext/fts5/fts5_tokenize.c +++ b/ext/fts5/fts5_tokenize.c @@ -72,7 +72,7 @@ static int fts5AsciiCreate( if( nArg%2 ){ rc = SQLITE_ERROR; }else{ - p = sqlite3_malloc64(sizeof(AsciiTokenizer)); + p = sqlite3_malloc(sizeof(AsciiTokenizer)); if( p==0 ){ rc = SQLITE_NOMEM; }else{ @@ -367,7 +367,7 @@ static int fts5UnicodeCreate( if( nArg%2 ){ rc = SQLITE_ERROR; }else{ - p = (Unicode61Tokenizer*)sqlite3_malloc64(sizeof(Unicode61Tokenizer)); + p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer)); if( p ){ const char *zCat = "L* N* Co"; int i; @@ -590,7 +590,7 @@ static int fts5PorterCreate( zBase = azArg[0]; } - pRet = (PorterTokenizer*)sqlite3_malloc64(sizeof(PorterTokenizer)); + pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer)); if( pRet ){ memset(pRet, 0, sizeof(PorterTokenizer)); rc = pApi->xFindTokenizer_v2(pApi, zBase, &pUserdata, &pV2); @@ -1297,7 +1297,7 @@ static int fts5TriCreate( rc = SQLITE_ERROR; }else{ int i; - pNew = (TrigramTokenizer*)sqlite3_malloc64(sizeof(*pNew)); + pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew)); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c index 295ace6ba..3a6a968f7 100644 --- a/ext/fts5/fts5_vocab.c +++ b/ext/fts5/fts5_vocab.c @@ -666,7 +666,7 @@ static int fts5VocabFilterMethod( const char *zCopy = (const char *)sqlite3_value_text(pLe); if( zCopy==0 ) zCopy = ""; pCsr->nLeTerm = sqlite3_value_bytes(pLe); - pCsr->zLeTerm = sqlite3_malloc64((i64)pCsr->nLeTerm+1); + pCsr->zLeTerm = sqlite3_malloc(pCsr->nLeTerm+1); if( pCsr->zLeTerm==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/ext/fts5/test/fts5corrupt9.test b/ext/fts5/test/fts5corrupt9.test index 6cf06f836..102a4135b 100644 --- a/ext/fts5/test/fts5corrupt9.test +++ b/ext/fts5/test/fts5corrupt9.test @@ -54,75 +54,6 @@ do_catchsql_test 1.3 { DELETE FROM t WHERE rowid=3 } {0 {}} -#------------------------------------------------------------------------- -reset_db - -set nRow 8000 -set zText aaa - -do_execsql_test 2.0 { - CREATE VIRTUAL TABLE t USING fts5(content, detail=none); - INSERT INTO t(t, rank) VALUES('secure-delete', 1); - BEGIN; -} -do_test 2.1 { - for {set ii 0} {$ii<$nRow} {incr ii} { - execsql { INSERT INTO t(content) VALUES($zText); } - } - execsql { - COMMIT; - INSERT INTO t(t) VALUES('optimize'); - } -} {} - -set hex "00040f7d9f49[string repeat {01} 3958]80" - -do_execsql_test 2.1 " - UPDATE t_data SET block = X'$hex' WHERE rowid=137438953474; -" - -do_execsql_test 2.3 { - DELETE FROM t WHERE rowid=7999 -} - -#------------------------------------------------------------------------- -reset_db - -do_execsql_test 3.0 { - CREATE VIRTUAL TABLE t USING fts5(content, detail=none); - INSERT INTO t(t, rank) VALUES('secure-delete', 1); - INSERT INTO t(content) VALUES('aaa'); - INSERT INTO t(content) VALUES('bbb'); - INSERT INTO t(t) VALUES('optimize'); -} - -do_execsql_test 3.1 { - UPDATE t_data SET block = X'000000100430616161010187ffffff7f0406' WHERE rowid=412316860417; -} - -do_catchsql_test 3.2 { - DELETE FROM t WHERE rowid=1; -} {1 {fts5: corruption in table "t"}} - -#------------------------------------------------------------------------- -reset_db - -do_execsql_test 4.0 { - CREATE VIRTUAL TABLE t USING fts5(content); - INSERT INTO t(t, rank) VALUES('secure-delete', 1); - INSERT INTO t(content) VALUES('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'); - INSERT INTO t(t) VALUES('optimize'); -} - -do_execsql_test 4.1 { - UPDATE t_data SET block = X'00000fce9f4830616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161610487ffffff7f' WHERE rowid=137438953473; - UPDATE t_data SET block = X'0004000801040203' WHERE rowid=137438953474; -} - -do_catchsql_test 4.2 { - DELETE FROM t WHERE rowid = 1; -} {1 {fts5: corruption in table "t"}} - sqlite3_fts5_may_be_corrupt 0 finish_test diff --git a/ext/fts5/test/fts5integrity.test b/ext/fts5/test/fts5integrity.test index 9b2720faf..4bf120c44 100644 --- a/ext/fts5/test/fts5integrity.test +++ b/ext/fts5/test/fts5integrity.test @@ -379,6 +379,9 @@ do_execsql_test 12.2 { db close sqlite3 db test.db -readonly 1 +explain_i { + PRAGMA integrity_check + } do_execsql_test 12.3 { PRAGMA integrity_check } {ok} diff --git a/ext/fts5/test/fts5interrupt.test b/ext/fts5/test/fts5interrupt.test index 87b232b05..67ef5f7e9 100644 --- a/ext/fts5/test/fts5interrupt.test +++ b/ext/fts5/test/fts5interrupt.test @@ -64,21 +64,4 @@ foreach {tn sql} { } } -#------------------------------------------------------------------------- -# Verify that https://sqlite.org/forum/forumpost/95413eb410 has been -# fixed. -# -reset_db -do_execsql_test 2.0 { - CREATE VIRTUAL TABLE f1 USING fts5(x); - BEGIN TRANSACTION; - INSERT INTO f1(x) VALUES('abc def ghi'); -} -do_test 2.1 { - sqlite3_interrupt db -} {} -do_execsql_test 2.2 { - ROLLBACK -} - finish_test diff --git a/ext/intck/sqlite3intck.c b/ext/intck/sqlite3intck.c index e3fef7763..5f645fae6 100644 --- a/ext/intck/sqlite3intck.c +++ b/ext/intck/sqlite3intck.c @@ -319,7 +319,7 @@ static int intckGetToken(const char *z){ char c = z[0]; int iRet = 1; if( c=='\'' || c=='"' || c=='`' ){ - while( z[iRet] ){ + while( 1 ){ if( z[iRet]==c ){ iRet++; if( z[iRet]!=c ) break; diff --git a/ext/jni/README.md b/ext/jni/README.md index 0bdbde91e..5ad79fce9 100644 --- a/ext/jni/README.md +++ b/ext/jni/README.md @@ -13,14 +13,11 @@ Technical support is available in the forum: <https://sqlite.org/forum> -> **FOREWARNING:** the JNI subproject is experimental and subject to - any number of changes. This API is "feature-complete", with only a - few difficult-to-reach corners of the C API not represented here, - but it is not a supported deliverable of the project so does not - have same backward compatibility guarantees which the C APIs - do. That said: the [C-style API](#1to1ish) is especially resistent - to compatibility breakage because it's designed to be as close to - the C API as feasible. +> **FOREWARNING:** this subproject is very much in development and + subject to any number of changes. Please do not rely on any + information about its API until this disclaimer is removed. The JNI + bindings released with version 3.43 are a "tech preview." Once + finalized, strong backward compatibility guarantees will apply. Project goals/requirements: @@ -165,13 +162,11 @@ or propagate exceptions and must return error information (if any) via result codes or `null`. The only cases where the C-style APIs may throw is through client-side misuse, e.g. passing in a null where it may cause a `NullPointerException`. The APIs clearly mark function -parameters which should not be null, and it internally uses the -`SQLITE_API_ARMOR` mechanism to help product against such misuse. Some -C-style APIs explicitly accept `null` as a no-op for usability's sake, -and some of the JNI APIs deliberately return an error code, instead of -segfaulting, when passed a `null`. There are no known cases where it -will misuse memory if passed a `null` or out-of-range value from -client code. +parameters which should not be null, but does not generally actively +defend itself against such misuse. Some C-style APIs explicitly accept +`null` as a no-op for usability's sake, and some of the JNI APIs +deliberately return an error code, instead of segfaulting, when passed +a `null`. Client-defined callbacks _must never throw exceptions_ unless _very explicitly documented_ as being throw-safe. Exceptions are generally @@ -199,8 +194,7 @@ Some constructs, when modelled 1-to-1 from C to Java, are unduly clumsy to work with in Java because they try to shoehorn C's way of doing certain things into Java's wildly different ways. The following subsections cover those, starting with a verbose explanation and -demonstration of where such changes are "really necessary" for -usability's sake... +demonstration of where such changes are "really necessary"... ### Custom Collations @@ -292,9 +286,12 @@ binding. The Java API has only one core function-registration function: ```java int sqlite3_create_function(sqlite3 db, String funcName, int nArgs, - int flags, SQLFunction func); + int encoding, SQLFunction func); ``` +> Design question: does the encoding argument serve any purpose in + Java? That's as-yet undetermined. If not, it will be removed. + `SQLFunction` is not used directly, but is instead instantiated via one of its three subclasses: @@ -316,4 +313,4 @@ in-flux nature of this API. Various APIs which accept callbacks, e.g. `sqlite3_trace_v2()` and `sqlite3_update_hook()`, use interfaces similar to those shown above. Despite the changes in signature, the JNI layer makes every effort to -provide the same semantics as the C API documentation describes. +provide the same semantics as the C API documentation suggests. diff --git a/ext/jni/src/c/sqlite3-jni.c b/ext/jni/src/c/sqlite3-jni.c index 8cdba9bcf..f130eff04 100644 --- a/ext/jni/src/c/sqlite3-jni.c +++ b/ext/jni/src/c/sqlite3-jni.c @@ -133,9 +133,11 @@ ** Which sqlite3.c we're using needs to be configurable to enable ** building against a custom copy, e.g. the SEE variant. We have to ** include sqlite3.c, as opposed to sqlite3.h, in order to get access -** to some internal details like SQLITE_MAX_... and friends, and keep -** those consistent with this build. This increases the rebuild time -** considerably, however. +** to some internal details like SQLITE_MAX_... and friends. This +** increases the rebuild time considerably but we need this in order +** to access some internal functionality and keep the to-Java-exported +** values of SQLITE_MAX_... and SQLITE_LIMIT_... in sync with the C +** build. */ #ifndef SQLITE_C # define SQLITE_C sqlite3.c @@ -333,7 +335,7 @@ struct S3JniNphOp { const char * const zMember /* Name of member property */; const char * const zTypeSig /* JNI type signature of zMember */; /* - ** klazz is a global ref to the class represented by zName. + ** klazz is a global ref to the class represented by pRef. ** ** According to: ** @@ -997,20 +999,19 @@ static S3JniEnv * S3JniEnv__get(JNIEnv * const env){ ** JNI bindings such as sqlite3_prepare_v2/v3(), and definitely not ** from client code. ** -** Returns err_code _unless_ err_code is 0 and sqlite3_set_errmsg() -** fails with OOM, in which case it may return SQLITE_OOM or fail -** fatally. -** -** This function predates sqlite3_set_errmsg(), which is why it has a -** slightly different interface. Before that function was introduced, -** this code used the SQLite-internal APIs to do this. +** Returns err_code. */ -static int s3jni_db_error(JNIEnv * env, sqlite3* const db, - int err_code, const char * const zMsg){ +static int s3jni_db_error(sqlite3* const db, int err_code, + const char * const zMsg){ if( db!=0 ){ - int const rc = sqlite3_set_errmsg(db, err_code, zMsg); - s3jni_oom_fatal(0==rc); - if( rc && !err_code ) err_code=rc; + if( 0==zMsg ){ + sqlite3Error(db, err_code); + }else{ + const int nMsg = sqlite3Strlen30(zMsg); + sqlite3_mutex_enter(sqlite3_db_mutex(db)); + sqlite3ErrorWithMsg(db, err_code, "%.*s", nMsg, zMsg); + sqlite3_mutex_leave(sqlite3_db_mutex(db)); + } } return err_code; } @@ -1234,11 +1235,11 @@ static int s3jni__db_exception(JNIEnv * const env, sqlite3 * const pDb, char * zMsg; S3JniExceptionClear; zMsg = s3jni_exception_error_msg(env, ex); - s3jni_db_error(env, pDb, errCode, zMsg ? zMsg : zDfltMsg); + s3jni_db_error(pDb, errCode, zMsg ? zMsg : zDfltMsg); sqlite3_free(zMsg); S3JniUnrefLocal(ex); }else if( zDfltMsg ){ - s3jni_db_error(env, pDb, errCode, zDfltMsg); + s3jni_db_error(pDb, errCode, zDfltMsg); } return errCode; } @@ -1955,6 +1956,15 @@ static void S3JniUdf_finalizer(void * s){ S3JniUdf_free(s3jni_env(), (S3JniUdf*)s, 1); } +/* +** Helper for processing args to UDF handlers with signature +** (sqlite3_context*,int,sqlite3_value**). +*/ +typedef struct { + jobject jcx /* sqlite3_context */; + jobjectArray jargv /* sqlite3_value[] */; +} udf_jargs; + /* ** Converts the given (cx, argc, argv) into arguments for the given ** UDF, writing the result (Java wrappers for cx and argv) in the @@ -1996,7 +2006,7 @@ static int udf_args(JNIEnv *env, /* ** Requires that jCx and jArgv are sqlite3_context ** resp. array-of-sqlite3_value values initialized by udf_args(). The -** (argc,argv) are (0,NULL) for UDF types with no arguments. This +** latter will be 0-and-NULL for UDF types with no arguments. This ** function zeroes out the nativePointer member of jCx and each entry ** in jArgv. This is a safety-net precaution to avoid undefined ** behavior if a Java-side UDF holds a reference to its context or one @@ -2089,19 +2099,19 @@ static int udf_xFSI(sqlite3_context* const pCx, int argc, sqlite3_value** const argv, S3JniUdf * const s, jmethodID xMethodID, const char * const zFuncType){ S3JniDeclLocal_env; - jobject jcx = 0 /* sqlite3_context */; - jobjectArray jargv = 0 /* sqlite3_value[] */; - int rc = udf_args(env, pCx, argc, argv, &jcx, &jargv); + udf_jargs args = {0,0}; + int rc = udf_args(env, pCx, argc, argv, &args.jcx, &args.jargv); + if( 0 == rc ){ - (*env)->CallVoidMethod(env, s->jObj, xMethodID, jcx, jargv); + (*env)->CallVoidMethod(env, s->jObj, xMethodID, args.jcx, args.jargv); S3JniIfThrew{ rc = udf_report_exception(env, 'F'==zFuncType[1]/*xFunc*/, pCx, s->zFuncName, zFuncType); } - udf_unargs(env, jcx, argc, jargv); + udf_unargs(env, args.jcx, argc, args.jargv); } - S3JniUnrefLocal(jcx); - S3JniUnrefLocal(jargv); + S3JniUnrefLocal(args.jcx); + S3JniUnrefLocal(args.jargv); return rc; } @@ -3290,7 +3300,7 @@ static jobject s3jni_commit_rollback_hook(int isCommit, JNIEnv * const env, S3JniDb_mutex_enter; ps = S3JniDb_from_jlong(jpDb); if( !ps ){ - s3jni_db_error(env, ps->pDb, SQLITE_MISUSE, 0); + s3jni_db_error(ps->pDb, SQLITE_MISUSE, 0); S3JniDb_mutex_leave; return 0; } @@ -3310,14 +3320,13 @@ static jobject s3jni_commit_rollback_hook(int isCommit, JNIEnv * const env, else sqlite3_rollback_hook(ps->pDb, 0, 0); }else{ jclass const klazz = (*env)->GetObjectClass(env, jHook); - jmethodID const xCallback = - (*env)->GetMethodID(env, klazz, "call", - isCommit ? "()I" : "()V"); + jmethodID const xCallback = (*env)->GetMethodID(env, klazz, "call", + isCommit ? "()I" : "()V"); S3JniUnrefLocal(klazz); S3JniIfThrew { S3JniExceptionReport; S3JniExceptionClear; - s3jni_db_error(env, ps->pDb, SQLITE_ERROR, + s3jni_db_error(ps->pDb, SQLITE_ERROR, "Cannot not find matching call() method in" "hook object."); }else{ @@ -3597,7 +3606,7 @@ S3JniApi(sqlite3_create_collation() sqlite3_create_collation_v2(), (*env)->GetMethodID(env, klazz, "call", "([B[B)I"); S3JniUnrefLocal(klazz); S3JniIfThrew{ - rc = s3jni_db_error(env, ps->pDb, SQLITE_ERROR, + rc = s3jni_db_error(ps->pDb, SQLITE_ERROR, "Could not get call() method from " "CollationCallback object."); }else{ @@ -3636,15 +3645,15 @@ S3JniApi(sqlite3_create_function() sqlite3_create_function_v2() if( !pDb || !jFuncName ){ return SQLITE_MISUSE; - }else if( !encodingTypeIsValid(eTextRep & 0x0f) ){ - return s3jni_db_error(env, pDb, SQLITE_FORMAT, + }else if( !encodingTypeIsValid(eTextRep) ){ + return s3jni_db_error(pDb, SQLITE_FORMAT, "Invalid function encoding option."); } s = S3JniUdf_alloc(env, jFunctor); if( !s ) return SQLITE_NOMEM; if( UDF_UNKNOWN_TYPE==s->type ){ - rc = s3jni_db_error(env, pDb, SQLITE_MISUSE, + rc = s3jni_db_error(pDb, SQLITE_MISUSE, "Cannot unambiguously determine function type."); S3JniUdf_free(env, s, 1); goto error_cleanup; @@ -4003,7 +4012,7 @@ S3JniApi(sqlite3_jni_db_error(), jint, 1jni_1db_1error)( zStr = jStr ? s3jni_jstring_to_utf8( jStr, 0) : NULL; - rc = s3jni_db_error(env, ps->pDb, (int)jRc, zStr ); + rc = s3jni_db_error( ps->pDb, (int)jRc, zStr ); sqlite3_free(zStr); } return rc; @@ -4312,7 +4321,7 @@ static void s3jni_updatepre_hook_impl(void * pState, sqlite3 *pDb, int opId, jTable = jDbName ? s3jni_utf8_to_jstring( zTable, -1) : 0; S3JniIfThrew { S3JniExceptionClear; - s3jni_db_error(env, ps->pDb, SQLITE_NOMEM, 0); + s3jni_db_error(ps->pDb, SQLITE_NOMEM, 0); }else{ assert( hook.jObj ); assert( hook.midCallback ); @@ -4414,7 +4423,7 @@ static jobject s3jni_updatepre_hook(JNIEnv * env, int isPre, jlong jpDb, jobject S3JniUnrefLocal(klazz); S3JniIfThrew { S3JniExceptionClear; - s3jni_db_error(env, ps->pDb, SQLITE_ERROR, + s3jni_db_error(ps->pDb, SQLITE_ERROR, "Cannot not find matching callback on " "(pre)update hook object."); }else{ @@ -4523,7 +4532,7 @@ S3JniApi(sqlite3_progress_handler(),void,1progress_1handler)( S3JniUnrefLocal(klazz); S3JniIfThrew { S3JniExceptionClear; - s3jni_db_error(env, ps->pDb, SQLITE_ERROR, + s3jni_db_error(ps->pDb, SQLITE_ERROR, "Cannot not find matching xCallback() on " "ProgressHandler object."); }else{ @@ -4897,9 +4906,8 @@ S3JniApi(sqlite3_set_authorizer(),jint,1set_1authorizer)( ")I"); S3JniUnrefLocal(klazz); S3JniIfThrew { - rc = s3jni_db_error(env, ps->pDb, SQLITE_ERROR, - "Error setting up Java parts of " - "authorizer hook."); + rc = s3jni_db_error(ps->pDb, SQLITE_ERROR, + "Error setting up Java parts of authorizer hook."); }else{ rc = sqlite3_set_authorizer(ps->pDb, s3jni_xAuth, ps); } @@ -5183,7 +5191,7 @@ S3JniApi(sqlite3_trace_v2(),jint,1trace_1v2)( S3JniUnrefLocal(klazz); S3JniIfThrew { S3JniExceptionClear; - rc = s3jni_db_error(env, ps->pDb, SQLITE_ERROR, + rc = s3jni_db_error(ps->pDb, SQLITE_ERROR, "Cannot not find matching call() on " "TracerCallback object."); }else{ diff --git a/ext/jni/src/c/sqlite3-jni.h b/ext/jni/src/c/sqlite3-jni.h index c326fa8ea..81af5cbde 100644 --- a/ext/jni/src/c/sqlite3-jni.h +++ b/ext/jni/src/c/sqlite3-jni.h @@ -245,10 +245,8 @@ extern "C" { #define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_USED_SHARED 11L #undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_SPILL #define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_SPILL 12L -#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_TEMPBUF_SPILL -#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_TEMPBUF_SPILL 13L #undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_MAX -#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_MAX 13L +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_MAX 12L #undef org_sqlite_jni_capi_CApi_SQLITE_UTF8 #define org_sqlite_jni_capi_CApi_SQLITE_UTF8 1L #undef org_sqlite_jni_capi_CApi_SQLITE_UTF16LE diff --git a/ext/jni/src/org/sqlite/jni/capi/CApi.java b/ext/jni/src/org/sqlite/jni/capi/CApi.java index 1bdc5300d..0b840c362 100644 --- a/ext/jni/src/org/sqlite/jni/capi/CApi.java +++ b/ext/jni/src/org/sqlite/jni/capi/CApi.java @@ -115,9 +115,8 @@ private static byte[] nulTerminateUtf8(String s){ JNIEnv is not cached, else returns true, but this information is primarily for testing of the JNI bindings and is not information which client-level code can use to make any informed - decisions. The semantics of its return type and value are not - considered stable and may change at any time. i.e. act as if it - returns null. + decisions. Its return type and semantics are not considered + stable and may change at any time. */ public static native boolean sqlite3_java_uncache_thread(); @@ -2586,8 +2585,7 @@ public static int sqlite3_value_type(@NotNull sqlite3_value v){ public static final int SQLITE_DBSTATUS_DEFERRED_FKS = 10; public static final int SQLITE_DBSTATUS_CACHE_USED_SHARED = 11; public static final int SQLITE_DBSTATUS_CACHE_SPILL = 12; - public static final int SQLITE_DBSTATUS_TEMPBUF_SPILL = 13; - public static final int SQLITE_DBSTATUS_MAX = 13; + public static final int SQLITE_DBSTATUS_MAX = 12; // encodings public static final int SQLITE_UTF8 = 1; diff --git a/ext/jni/src/org/sqlite/jni/capi/Tester1.java b/ext/jni/src/org/sqlite/jni/capi/Tester1.java index 891bdea54..9d14c954b 100644 --- a/ext/jni/src/org/sqlite/jni/capi/Tester1.java +++ b/ext/jni/src/org/sqlite/jni/capi/Tester1.java @@ -815,9 +815,7 @@ public void xDestroy(){ }; // Register and use the function... - int rc = sqlite3_create_function(db, "myfunc", -1, - SQLITE_UTF8 | SQLITE_INNOCUOUS, - func); + int rc = sqlite3_create_function(db, "myfunc", -1, SQLITE_UTF8, func); affirm(0 == rc); affirm(0 == xFuncAccum.value); final sqlite3_stmt stmt = prepare(db, "SELECT myfunc(1,2,3)"); diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java b/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java index ba2ffd119..d259e0ce6 100644 --- a/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java +++ b/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java @@ -171,7 +171,6 @@ public final class Sqlite implements AutoCloseable { public static final int DBSTATUS_DEFERRED_FKS = CApi.SQLITE_DBSTATUS_DEFERRED_FKS; public static final int DBSTATUS_CACHE_USED_SHARED = CApi.SQLITE_DBSTATUS_CACHE_USED_SHARED; public static final int DBSTATUS_CACHE_SPILL = CApi.SQLITE_DBSTATUS_CACHE_SPILL; - public static final int DBSTATUS_TEMPBUF_SPILL = CApi.SQLITE_DBSTATUS_TEMPBUF_SPILL; // Limits public static final int LIMIT_LENGTH = CApi.SQLITE_LIMIT_LENGTH; diff --git a/ext/misc/btreeinfo.c b/ext/misc/btreeinfo.c index 24645f226..9c726f5f1 100644 --- a/ext/misc/btreeinfo.c +++ b/ext/misc/btreeinfo.c @@ -306,10 +306,6 @@ static int binfoCompute(sqlite3 *db, int pgno, BinfoCursor *pCsr){ nEntry *= (nCell+1); if( aData[0]==10 || aData[0]==13 ) break; nPage *= (nCell+1); - if( 14+2*(nCell/2)>=pgsz ){ - rc = SQLITE_CORRUPT; - break; - } if( nCell<=1 ){ pgno = get_uint32(aData+8); }else{ @@ -343,7 +339,7 @@ static int binfoColumn( sqlite3 *db = sqlite3_context_db_handle(ctx); int rc = binfoCompute(db, pgno, pCsr); if( rc ){ - pCursor->pVtab->zErrMsg = sqlite3_mprintf("%s", sqlite3_errstr(rc)); + pCursor->pVtab->zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); return SQLITE_ERROR; } } diff --git a/ext/misc/csv.c b/ext/misc/csv.c index f44a30001..1caaaec87 100644 --- a/ext/misc/csv.c +++ b/ext/misc/csv.c @@ -24,8 +24,8 @@ ** schema= parameter, like this: ** ** CREATE VIRTUAL TABLE temp.csv2 USING csv( -** filename = '../http.log', -** schema = 'CREATE TABLE x(date,ipaddr,url,referrer,userAgent)' +** filename = "../http.log", +** schema = "CREATE TABLE x(date,ipaddr,url,referrer,userAgent)" ** ); ** ** Instead of specifying a file, the text of the CSV can be loaded using diff --git a/ext/misc/decimal.c b/ext/misc/decimal.c index be4321ca8..f87699f96 100644 --- a/ext/misc/decimal.c +++ b/ext/misc/decimal.c @@ -291,36 +291,12 @@ static void decimal_result(sqlite3_context *pCtx, Decimal *p){ sqlite3_result_text(pCtx, z, i, sqlite3_free); } -/* -** Round a decimal value to N significant digits. N must be positive. -*/ -static void decimal_round(Decimal *p, int N){ - int i; - int nZero; - if( N<1 ) return; - for(nZero=0; nZero<p->nDigit && p->a[nZero]==0; nZero++){} - N += nZero; - if( p->nDigit<=N ) return; - if( p->a[N]>4 ){ - p->a[N-1]++; - for(i=N-1; i>0 && p->a[i]>9; i--){ - p->a[i] = 0; - p->a[i-1]++; - } - if( p->a[0]>9 ){ - p->a[0] = 1; - p->nFrac--; - } - } - memset(&p->a[N], 0, p->nDigit - N); -} - /* ** Make the given Decimal the result in an format similar to '%+#e'. ** In other words, show exponential notation with leading and trailing ** zeros omitted. */ -static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p, int N){ +static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ char *z; /* The output buffer */ int i; /* Loop counter */ int nZero; /* Number of leading zeros */ @@ -338,8 +314,7 @@ static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p, int N){ sqlite3_result_null(pCtx); return; } - if( N<1 ) N = 0; - for(nDigit=p->nDigit; nDigit>N && p->a[nDigit-1]==0; nDigit--){} + for(nDigit=p->nDigit; nDigit>0 && p->a[nDigit-1]==0; nDigit--){} for(nZero=0; nZero<nDigit && p->a[nZero]==0; nZero++){} nFrac = p->nFrac + (nDigit - p->nDigit); nDigit -= nZero; @@ -702,16 +677,10 @@ static void decimalFunc( sqlite3_value **argv ){ Decimal *p = decimal_new(context, argv[0], 0); - int N; - if( argc==2 ){ - N = sqlite3_value_int(argv[1]); - if( N>0 ) decimal_round(p, N); - }else{ - N = 0; - } + UNUSED_PARAMETER(argc); if( p ){ if( sqlite3_user_data(context)!=0 ){ - decimal_result_sci(context, p, N); + decimal_result_sci(context, p); }else{ decimal_result(context, p); } @@ -881,7 +850,7 @@ static void decimalPow2Func( UNUSED_PARAMETER(argc); if( sqlite3_value_type(argv[0])==SQLITE_INTEGER ){ Decimal *pA = decimalPow2(sqlite3_value_int(argv[0])); - decimal_result_sci(context, pA, 0); + decimal_result_sci(context, pA); decimal_free(pA); } } @@ -902,9 +871,7 @@ int sqlite3_decimal_init( void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } aFunc[] = { { "decimal", 1, 0, decimalFunc }, - { "decimal", 2, 0, decimalFunc }, { "decimal_exp", 1, 1, decimalFunc }, - { "decimal_exp", 2, 1, decimalFunc }, { "decimal_cmp", 2, 0, decimalCmpFunc }, { "decimal_add", 2, 0, decimalAddFunc }, { "decimal_sub", 2, 0, decimalSubFunc }, diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c index 51b748291..6cc2ae008 100644 --- a/ext/misc/fileio.c +++ b/ext/misc/fileio.c @@ -94,16 +94,12 @@ SQLITE_EXTENSION_INIT1 # include <utime.h> # include <sys/time.h> # define STRUCT_STAT struct stat -# include <limits.h> -# include <stdlib.h> #else # include "windirent.h" # include <direct.h> # define STRUCT_STAT struct _stat # define chmod(path,mode) fileio_chmod(path,mode) # define mkdir(path,mode) fileio_mkdir(path) - extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*); - extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR); #endif #include <time.h> #include <errno.h> @@ -135,9 +131,12 @@ SQLITE_EXTENSION_INIT1 */ #if defined(_WIN32) || defined(WIN32) static int fileio_chmod(const char *zPath, int pmode){ + sqlite3_int64 sz = strlen(zPath); + wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(b1[0]) ); int rc; - wchar_t *b1 = sqlite3_win32_utf8_to_unicode(zPath); if( b1==0 ) return -1; + sz = MultiByteToWideChar(CP_UTF8, 0, zPath, sz, b1, sz); + b1[sz] = 0; rc = _wchmod(b1, pmode); sqlite3_free(b1); return rc; @@ -149,9 +148,12 @@ static int fileio_chmod(const char *zPath, int pmode){ */ #if defined(_WIN32) || defined(WIN32) static int fileio_mkdir(const char *zPath){ + sqlite3_int64 sz = strlen(zPath); + wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(b1[0]) ); int rc; - wchar_t *b1 = sqlite3_win32_utf8_to_unicode(zPath); if( b1==0 ) return -1; + sz = MultiByteToWideChar(CP_UTF8, 0, zPath, sz, b1, sz); + b1[sz] = 0; rc = _wmkdir(b1); sqlite3_free(b1); return rc; @@ -264,34 +266,69 @@ static sqlite3_uint64 fileTimeToUnixTime( return (fileIntervals.QuadPart - epochIntervals.QuadPart) / 10000000; } -#endif /* _WIN32 */ + + +#if defined(FILEIO_WIN32_DLL) && (defined(_WIN32) || defined(WIN32)) +# /* To allow a standalone DLL, use this next replacement function: */ +# undef sqlite3_win32_utf8_to_unicode +# define sqlite3_win32_utf8_to_unicode utf8_to_utf16 +# +LPWSTR utf8_to_utf16(const char *z){ + int nAllot = MultiByteToWideChar(CP_UTF8, 0, z, -1, NULL, 0); + LPWSTR rv = sqlite3_malloc(nAllot * sizeof(WCHAR)); + if( rv!=0 && 0 < MultiByteToWideChar(CP_UTF8, 0, z, -1, rv, nAllot) ) + return rv; + sqlite3_free(rv); + return 0; +} +#endif /* -** This function is used in place of stat(). On Windows, special handling -** is required in order for the included time to be returned as UTC. On all -** other systems, this function simply calls stat(). +** This function attempts to normalize the time values found in the stat() +** buffer to UTC. This is necessary on Win32, where the runtime library +** appears to return these values as local times. */ -static int fileStat( +static void statTimesToUtc( const char *zPath, STRUCT_STAT *pStatBuf ){ -#if defined(_WIN32) - int rc; - wchar_t *b1 = sqlite3_win32_utf8_to_unicode(zPath); - if( b1==0 ) return 1; - rc = _wstat(b1, pStatBuf); - if( rc==0 ){ - HANDLE hFindFile; - WIN32_FIND_DATAW fd; + HANDLE hFindFile; + WIN32_FIND_DATAW fd; + LPWSTR zUnicodeName; + extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*); + zUnicodeName = sqlite3_win32_utf8_to_unicode(zPath); + if( zUnicodeName ){ memset(&fd, 0, sizeof(WIN32_FIND_DATAW)); - hFindFile = FindFirstFileW(b1, &fd); + hFindFile = FindFirstFileW(zUnicodeName, &fd); if( hFindFile!=NULL ){ pStatBuf->st_ctime = (time_t)fileTimeToUnixTime(&fd.ftCreationTime); pStatBuf->st_atime = (time_t)fileTimeToUnixTime(&fd.ftLastAccessTime); pStatBuf->st_mtime = (time_t)fileTimeToUnixTime(&fd.ftLastWriteTime); FindClose(hFindFile); } + sqlite3_free(zUnicodeName); } +} +#endif + +/* +** This function is used in place of stat(). On Windows, special handling +** is required in order for the included time to be returned as UTC. On all +** other systems, this function simply calls stat(). +*/ +static int fileStat( + const char *zPath, + STRUCT_STAT *pStatBuf +){ +#if defined(_WIN32) + sqlite3_int64 sz = strlen(zPath); + wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(b1[0]) ); + int rc; + if( b1==0 ) return 1; + sz = MultiByteToWideChar(CP_UTF8, 0, zPath, sz, b1, sz); + b1[sz] = 0; + rc = _wstat(b1, pStatBuf); + if( rc==0 ) statTimesToUtc(zPath, pStatBuf); sqlite3_free(b1); return rc; #else @@ -423,6 +460,7 @@ static int writeFile( if( mtime>=0 ){ #if defined(_WIN32) +#if !SQLITE_OS_WINRT /* Windows */ FILETIME lastAccess; FILETIME lastWrite; @@ -453,6 +491,7 @@ static int writeFile( }else{ return 1; } +#endif #elif defined(AT_FDCWD) && 0 /* utimensat() is not universally available */ /* Recent unix */ struct timespec times[2]; @@ -1056,154 +1095,6 @@ static int fsdirRegister(sqlite3 *db){ # define fsdirRegister(x) SQLITE_OK #endif -/* -** This version of realpath() works on any system. The string -** returned is held in memory allocated using sqlite3_malloc64(). -** The caller is responsible for calling sqlite3_free(). -*/ -static char *portable_realpath(const char *zPath){ -#if !defined(_WIN32) /* BEGIN unix */ - - char *zOut = 0; /* Result */ - char *z; /* Temporary buffer */ -#if defined(PATH_MAX) - char zBuf[PATH_MAX+1]; /* Space for the temporary buffer */ -#endif - - if( zPath==0 ) return 0; -#if defined(PATH_MAX) - z = realpath(zPath, zBuf); - if( z ){ - zOut = sqlite3_mprintf("%s", zBuf); - } -#endif /* defined(PATH_MAX) */ - if( zOut==0 ){ - /* Try POSIX.1-2008 malloc behavior */ - z = realpath(zPath, NULL); - if( z ){ - zOut = sqlite3_mprintf("%s", z); - free(z); - } - } - return zOut; - -#else /* End UNIX, Begin WINDOWS */ - - wchar_t *zPath16; /* UTF16 translation of zPath */ - char *zOut = 0; /* Result */ - wchar_t *z = 0; /* Temporary buffer */ - - if( zPath==0 ) return 0; - - zPath16 = sqlite3_win32_utf8_to_unicode(zPath); - if( zPath16==0 ) return 0; - z = _wfullpath(NULL, zPath16, 0); - sqlite3_free(zPath16); - if( z ){ - zOut = sqlite3_win32_unicode_to_utf8(z); - free(z); - } - return zOut; - -#endif /* End WINDOWS, Begin common code */ -} - -/* -** SQL function: realpath(X) -** -** Try to convert file or pathname X into its real, absolute pathname. -** Return NULL if unable. -** -** The file or directory X is not required to exist. The answer is formed -** by calling system realpath() on the prefix of X that does exist and -** appending the tail of X that does not (yet) exist. -*/ -static void realpathFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const char *zPath; /* Original input path */ - char *zCopy; /* An editable copy of zPath */ - char *zOut; /* The result */ - char cSep = 0; /* Separator turned into \000 */ - size_t len; /* Prefix length before cSep */ -#ifdef _WIN32 - const int isWin = 1; -#else - const int isWin = 0; -#endif - - (void)argc; - zPath = (const char*)sqlite3_value_text(argv[0]); - if( zPath==0 ) return; - if( zPath[0]==0 ) zPath = "."; - zCopy = sqlite3_mprintf("%s",zPath); - len = strlen(zCopy); - while( len>1 && (zCopy[len-1]=='/' || (isWin && zCopy[len-1]=='\\')) ){ - len--; - } - zCopy[len] = 0; - while( 1 /*exit-by-break*/ ){ - zOut = portable_realpath(zCopy); - zCopy[len] = cSep; - if( zOut ){ - if( cSep ){ - zOut = sqlite3_mprintf("%z%s",zOut,&zCopy[len]); - } - break; - }else{ - size_t i = len-1; - while( i>0 ){ - if( zCopy[i]=='/' || (isWin && zCopy[i]=='\\') ) break; - i--; - } - if( i<=0 ){ - if( zCopy[0]=='/' ){ - zOut = zCopy; - zCopy = 0; - }else if( (zOut = portable_realpath("."))!=0 ){ - zOut = sqlite3_mprintf("%z/%s", zOut, zCopy); - } - break; - } - cSep = zCopy[i]; - zCopy[i] = 0; - len = i; - } - } - sqlite3_free(zCopy); - if( zOut ){ - /* Simplify any "/./" or "/../" that might have snuck into the - ** pathname due to appending of zCopy. We only have to consider - ** unix "/" separators, because the _wfilepath() system call on - ** Windows will have already done this simplification for us. */ - size_t i, j, n; - n = strlen(zOut); - for(i=j=0; i<n; i++){ - if( zOut[i]=='/' ){ - if( zOut[i+1]=='/' ) continue; - if( zOut[i+1]=='.' && i+2<n && zOut[i+2]=='/' ){ - i += 1; - continue; - } - if( zOut[i+1]=='.' && i+3<n && zOut[i+2]=='.' && zOut[i+3]=='/' ){ - while( j>0 && zOut[j-1]!='/' ){ j--; } - if( j>0 ){ j--; } - i += 2; - continue; - } - } - zOut[j++] = zOut[i]; - } - zOut[j] = 0; - - /* Return the result */ - sqlite3_result_text(context, zOut, -1, sqlite3_free); - } -} - - #ifdef _WIN32 __declspec(dllexport) #endif @@ -1230,10 +1121,13 @@ int sqlite3_fileio_init( if( rc==SQLITE_OK ){ rc = fsdirRegister(db); } - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "realpath", 1, - SQLITE_UTF8, 0, - realpathFunc, 0, 0); - } return rc; } + +#if defined(FILEIO_WIN32_DLL) && (defined(_WIN32) || defined(WIN32)) +/* To allow a standalone DLL, make test_windirent.c use the same + * redefined SQLite API calls as the above extension code does. + * Just pull in this .c to accomplish this. As a beneficial side + * effect, this extension becomes a single translation unit. */ +# include "test_windirent.c" +#endif diff --git a/ext/misc/fuzzer.c b/ext/misc/fuzzer.c index 3dcf1d667..e16d005d9 100644 --- a/ext/misc/fuzzer.c +++ b/ext/misc/fuzzer.c @@ -617,7 +617,7 @@ static int fuzzerRender( int *pnBuf /* Size of the buffer */ ){ const fuzzer_rule *pRule = pStem->pRule; - sqlite3_int64 n; /* Size of output term without nul-term */ + int n; /* Size of output term without nul-term */ char *z; /* Buffer to assemble output term in */ n = pStem->nBasis + pRule->nTo - pRule->nFrom; diff --git a/ext/misc/ieee754.c b/ext/misc/ieee754.c index f551b2265..7f1c6d9e1 100644 --- a/ext/misc/ieee754.c +++ b/ext/misc/ieee754.c @@ -179,9 +179,9 @@ static void ieee754func( } if( m<0 ){ - if( m<(-9223372036854775807LL) ) return; isNeg = 1; m = -m; + if( m<0 ) return; }else if( m==0 && e>-1000 && e<1000 ){ sqlite3_result_double(context, 0.0); return; @@ -259,38 +259,6 @@ static void ieee754func_to_blob( } } -/* -** Functions to convert between 64-bit integers and floats. -** -** The bit patterns are copied. The numeric values are different. -*/ -static void ieee754func_from_int( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - UNUSED_PARAMETER(argc); - if( sqlite3_value_type(argv[0])==SQLITE_INTEGER ){ - double r; - sqlite3_int64 v = sqlite3_value_int64(argv[0]); - memcpy(&r, &v, sizeof(r)); - sqlite3_result_double(context, r); - } -} -static void ieee754func_to_int( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - UNUSED_PARAMETER(argc); - if( sqlite3_value_type(argv[0])==SQLITE_FLOAT ){ - double r = sqlite3_value_double(argv[0]); - sqlite3_uint64 v; - memcpy(&v, &r, sizeof(v)); - sqlite3_result_int64(context, v); - } -} - /* ** SQL Function: ieee754_inc(r,N) ** @@ -343,8 +311,6 @@ int sqlite3_ieee_init( { "ieee754_exponent", 1, 2, ieee754func }, { "ieee754_to_blob", 1, 0, ieee754func_to_blob }, { "ieee754_from_blob", 1, 0, ieee754func_from_blob }, - { "ieee754_to_int", 1, 0, ieee754func_to_int }, - { "ieee754_from_int", 1, 0, ieee754func_from_int }, { "ieee754_inc", 2, 0, ieee754inc }, }; unsigned int i; diff --git a/ext/misc/regexp.c b/ext/misc/regexp.c index e1826caf3..f1babf4ab 100644 --- a/ext/misc/regexp.c +++ b/ext/misc/regexp.c @@ -32,7 +32,7 @@ ** ^X X occurring at the beginning of the string ** X$ X occurring at the end of the string ** . Match any single character -** \c Character c where c is one of \{}()[]|*+?-. +** \c Character c where c is one of \{}()[]|*+?. ** \c C-language escapes for c in afnrtv. ex: \t or \n ** \uXXXX Where XXXX is exactly 4 hex digits, unicode value XXXX ** \xXX Where XX is exactly 2 hex digits, unicode value XX @@ -417,7 +417,7 @@ static int re_hex(int c, int *pV){ ** return its interpretation. */ static unsigned re_esc_char(ReCompiled *p){ - static const char zEsc[] = "afnrtv\\()*.+?[$^{|}]-"; + static const char zEsc[] = "afnrtv\\()*.+?[$^{|}]"; static const char zTrans[] = "\a\f\n\r\t\v"; int i, v = 0; char c; @@ -740,18 +740,11 @@ static const char *re_compile( } /* -** The value of LIMIT_MAX_PATTERN_LENGTH. +** Compute a reasonable limit on the length of the REGEXP NFA. */ static int re_maxlen(sqlite3_context *context){ sqlite3 *db = sqlite3_context_db_handle(context); - return sqlite3_limit(db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH,-1); -} - -/* -** Maximum NFA size given a maximum pattern length. -*/ -static int re_maxnfa(int mxlen){ - return 75+mxlen/2; + return 75 + sqlite3_limit(db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH,-1)/2; } /* @@ -777,17 +770,10 @@ static void re_sql_func( (void)argc; /* Unused */ pRe = sqlite3_get_auxdata(context, 0); if( pRe==0 ){ - int mxLen = re_maxlen(context); - int nPattern; zPattern = (const char*)sqlite3_value_text(argv[0]); if( zPattern==0 ) return; - nPattern = sqlite3_value_bytes(argv[0]); - if( nPattern>mxLen ){ - zErr = "REGEXP pattern too big"; - }else{ - zErr = re_compile(&pRe, zPattern, re_maxnfa(mxLen), - sqlite3_user_data(context)!=0); - } + zErr = re_compile(&pRe, zPattern, re_maxlen(context), + sqlite3_user_data(context)!=0); if( zErr ){ re_free(pRe); sqlite3_result_error(context, zErr, -1); @@ -853,7 +839,7 @@ static void re_bytecode_func( zPattern = (const char*)sqlite3_value_text(argv[0]); if( zPattern==0 ) return; - zErr = re_compile(&pRe, zPattern, re_maxnfa(re_maxlen(context)), + zErr = re_compile(&pRe, zPattern, re_maxlen(context), sqlite3_user_data(context)!=0); if( zErr ){ re_free(pRe); diff --git a/ext/misc/sha1.c b/ext/misc/sha1.c index 02d864955..07d797060 100644 --- a/ext/misc/sha1.c +++ b/ext/misc/sha1.c @@ -230,16 +230,13 @@ static void hash_finish( *****************************************************************************/ /* -** Two SQL functions: sha1(X) and sha1b(X). +** Implementation of the sha1(X) function. ** -** sha1(X) returns a lower-case hexadecimal rendering of the SHA1 hash -** of the argument X. If X is a BLOB, it is hashed as is. For all other +** Return a lower-case hexadecimal rendering of the SHA1 hash of the +** argument X. If X is a BLOB, it is hashed as is. For all other ** types of input, X is converted into a UTF-8 string and the string -** is hashed without the trailing 0x00 terminator. The hash of a NULL +** is hash without the trailing 0x00 terminator. The hash of a NULL ** value is NULL. -** -** sha1b(X) is the same except that it returns a 20-byte BLOB containing -** the binary hash instead of a hexadecimal string. */ static void sha1Func( sqlite3_context *context, @@ -260,13 +257,11 @@ static void sha1Func( hash_step(&cx, sqlite3_value_text(argv[0]), nByte); } if( sqlite3_user_data(context)!=0 ){ - /* sha1b() - binary result */ hash_finish(&cx, zOut, 1); sqlite3_result_blob(context, zOut, 20, SQLITE_TRANSIENT); }else{ - /* sha1() - hexadecimal text result */ hash_finish(&cx, zOut, 0); - sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); + sqlite3_result_blob(context, zOut, 40, SQLITE_TRANSIENT); } } diff --git a/ext/misc/sqlite3_stdio.c b/ext/misc/sqlite3_stdio.c index d59757526..c9bceb194 100644 --- a/ext/misc/sqlite3_stdio.c +++ b/ext/misc/sqlite3_stdio.c @@ -258,7 +258,7 @@ int sqlite3_fputs(const char *z, FILE *out){ /* -** Work-alikes for fprintf() and vfprintf() from the standard C library. +** Work-alike for fprintf() from the standard C library. */ int sqlite3_fprintf(FILE *out, const char *zFormat, ...){ int rc; @@ -285,24 +285,6 @@ int sqlite3_fprintf(FILE *out, const char *zFormat, ...){ } return rc; } -int sqlite3_vfprintf(FILE *out, const char *zFormat, va_list ap){ - int rc; - if( UseWtextForOutput(out) ){ - /* When writing to the command-prompt in Windows, it is necessary - ** to use _O_WTEXT input mode and write UTF-16 characters. - */ - char *z; - z = sqlite3_vmprintf(zFormat, ap); - sqlite3_fputs(z, out); - rc = (int)strlen(z); - sqlite3_free(z); - }else{ - /* Writing to a file or other destination, just write bytes without - ** any translation. */ - rc = vfprintf(out, zFormat, ap); - } - return rc; -} /* ** Set the mode for an output stream. mode argument is typically _O_BINARY or diff --git a/ext/misc/sqlite3_stdio.h b/ext/misc/sqlite3_stdio.h index 75368df9f..dd0eefad0 100644 --- a/ext/misc/sqlite3_stdio.h +++ b/ext/misc/sqlite3_stdio.h @@ -31,7 +31,6 @@ #ifdef _WIN32 /**** Definitions For Windows ****/ #include <stdio.h> -#include <stdarg.h> #include <windows.h> FILE *sqlite3_fopen(const char *zFilename, const char *zMode); @@ -39,7 +38,6 @@ FILE *sqlite3_popen(const char *zCommand, const char *type); char *sqlite3_fgets(char *s, int size, FILE *stream); int sqlite3_fputs(const char *s, FILE *stream); int sqlite3_fprintf(FILE *stream, const char *format, ...); -int sqlite3_vfprintf(FILE *stream, const char *format, va_list); void sqlite3_fsetmode(FILE *stream, int mode); @@ -51,7 +49,6 @@ void sqlite3_fsetmode(FILE *stream, int mode); #define sqlite3_fgets fgets #define sqlite3_fputs fputs #define sqlite3_fprintf fprintf -#define sqlite3_vfprintf vfprintf #define sqlite3_fsetmode(F,X) /*no-op*/ #endif diff --git a/ext/misc/tmstmpvfs.c b/ext/misc/tmstmpvfs.c deleted file mode 100644 index 6f1af36f7..000000000 --- a/ext/misc/tmstmpvfs.c +++ /dev/null @@ -1,1042 +0,0 @@ -/* -** 2026-01-05 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file implements a VFS shim that writes a timestamp and other tracing -** information into 16 byts of reserved space at the end of each page of the -** database file. -** -** The VFS also tries to generate log-files with names of the form: -** -** $(DATABASE)-tmstmp/$(TIME)-$(PID)-$(ID) -** -** Log files are only generated if directory $(DATABASE)-tmstmp exists. -** The name of each log file is the current ISO8601 time in milliseconds, -** the process ID, and a random 32-bit value (to disambiguate multiple -** connections from the same process) separated by dashes. The log file -** contains 16-bytes records for various events, such as opening or close -** of the database or WAL file, writes to the WAL file, checkpoints, and -** similar. The logfile is only generated if the connection attempts to -** modify the database. There is a separate log file for each open database -** connection. -** -** COMPILING -** -** To build this extension as a separately loaded shared library or -** DLL, use compiler command-lines similar to the following: -** -** (linux) gcc -fPIC -shared tmstmpvfs.c -o tmstmpvfs.so -** (mac) clang -fPIC -dynamiclib tmstmpvfs.c -o tmstmpvfs.dylib -** (windows) cl tmstmpvfs.c -link -dll -out:tmstmpvfs.dll -** -** You may want to add additional compiler options, of course, -** according to the needs of your project. -** -** Another option is to statically link both SQLite and this extension -** into your application. If both this file and "sqlite3.c" are statically -** linked, and if "sqlite3.c" is compiled with an option like: -** -** -DSQLITE_EXTRA_INIT=sqlite3_register_tmstmpvfs -** -** Then SQLite will use the tmstmp VFS by default throughout your -** application. -** -** LOADING -** -** To load this extension as a shared library, you first have to -** bring up a dummy SQLite database connection to use as the argument -** to the sqlite3_load_extension() API call. Then you invoke the -** sqlite3_load_extension() API and shutdown the dummy database -** connection. All subsequent database connections that are opened -** will include this extension. For example: -** -** sqlite3 *db; -** sqlite3_open(":memory:", &db); -** sqlite3_load_extension(db, "./tmstmpvfs"); -** sqlite3_close(db); -** -** Tmstmpvfs is a VFS Shim. When loaded, "tmstmpvfs" becomes the new -** default VFS and it uses the prior default VFS as the next VFS -** down in the stack. This is normally what you want. However, in -** complex situations where multiple VFS shims are being loaded, -** it might be important to ensure that tmstmpvfs is loaded in the -** correct order so that it sequences itself into the default VFS -** Shim stack in the right order. -** -** When running the CLI, you can load this extension at invocation by -** adding a command-line option like this: "--vfs ./tmstmpvfs.so". -** The --vfs option usually specifies the symbolic name of a built-in VFS. -** But if the argument to --vfs is not a built-in VFS but is instead the -** name of a file, the CLI tries to load that file as an extension. Note -** that the full name of the extension file must be provided, including -** the ".so" or ".dylib" or ".dll" suffix. -** -** An application can see if the tmstmpvfs is being used by examining -** the results from SQLITE_FCNTL_VFSNAME (or the .vfsname command in -** the CLI). If the answer include "tmstmp", then this VFS is being -** used. -** -** USING -** -** Open database connections using the sqlite3_open() or -** sqlite3_open_v2() interfaces, as normal. Ordinary database files -** (without a timestamp) will operate normally. -** -** Timestamping only works on databases that have a reserve-bytes -** value of exactly 16. The default value for reserve-bytes is 0. -** Hence, newly created database files will omit the timestamp by -** default. To create a database that includes a timestamp, change -** the reserve-bytes value to 16 by running: -** -** int n = 16; -** sqlite3_file_control(db, 0, SQLITE_FCNTL_RESERVE_BYTES, &n); -** -** If you do this immediately after creating a new database file, -** before anything else has been written into the file, then that -** might be all that you need to do. Otherwise, the API call -** above should be followed by: -** -** sqlite3_exec(db, "VACUUM", 0, 0, 0); -** -** It never hurts to run the VACUUM, even if you don't need it. -** -** From the CLI, use the ".filectrl reserve_bytes 16" command, -** followed by "VACUUM;". -** -** SQLite allows the number of reserve-bytes to be increased, but -** not decreased. If you want to restore the reserve-bytes to 0 -** (to disable tmstmpvfs), the easiest approach is to use VACUUM INTO -** with a URI filename as the argument and include "reserve=0" query -** parameter on the URI. Example: -** -** VACUUM INTO 'file:notimestamps.db?reserve=0'; -** -** Then switch over to using the new database file. The reserve=0 query -** parameter only works on SQLite 3.52.0 and later. -** -** IMPLEMENTATION NOTES -** -** The timestamp information is stored in the last 16 bytes of each page. -** This module only operates if the "bytes of reserved space on each page" -** value at offset 20 the SQLite database header is exactly 16. If -** the reserved-space value is not 16, no timestamp information is added -** to database pages. Some, but not all, logfile entries will be made -** still, but the size of the logs will be greatly reduced. -** -** The timestamp layout is as follows: -** -** bytes 0,1 Zero. Reserved for future expansion -** bytes 2-7 Milliseconds since the Unix Epoch -** bytes 8-11 WAL frame number -** bytes 12 0: WAL write 2: rollback write -** bytes 13-15 Lower 24 bits of Salt-1 -** -** For transactions that occur in rollback mode, only the timestamp -** in bytes 2-7 and byte 12 are non-zero. Byte 12 is set to 2 for -** rollback writes. -** -** The 16-byte tag is added to each database page when the content -** is written into the database file itself. This shim does not make -** any changes to the page as it is written to the WAL file, since -** that would mess up the WAL checksum. -** -** LOGGING -** -** An open database connection that attempts to write to the database -** will create a log file if a directory name $(DATABASE)-tmstmp exists. -** The name of the log file is: -** -** $(TIME)-$(PID)-$(RANDOM) -** -** Where TIME is an ISO 8601 date in milliseconds with no punctuation, -** PID is the process ID, and RANDOM is a 32-bit random number expressed -** as hexadecimal. -** -** The log consists of 16-byte records. Each record consists of five -** unsigned integers: -** -** 1 1 6 4 4 <--- bytes -** op a1 ts a2 a3 -** -** The meanings of the a1-a3 values depend on op. ts is the timestamp -** in milliseconds since the unix epoch (1970-01-01 00:00:00). -** Opcodes are defined by the ELOG_* #defines below. -** -** ELOG_OPEN_DB "Open a connection to the database file" -** op = 0x01 -** a2 = process-ID -** -** ELOG_OPEN_WAL "Open a connection to the -wal file" -** op = 0x02 -** a2 = process-ID -** -** ELOG_WAL_PAGE "New page added to the WAL file" -** op = 0x03 -** a1 = 1 if last page of a txn. 0 otherwise. -** a2 = page number in the DB file -** a3 = frame number in the WAL file -** -** ELOG_DB_PAGE "Database page updated using rollback mode" -** op = 0x04 -** a2 = page number in the DB file -** -** ELOG_CKPT_START "Start of a checkpoint operation" -** op = 0x05 -** -** ELOG_CKPT_PAGE "Page xfer from WAL to database" -** op = 0x06 -** a2 = database page number -** a3 = frame number in the WAL file -** -** ELOG_CKPT_END "Start of a checkpoint operation" -** op = 0x07 -** -** ELOG_WAL_RESET "WAL file header overwritten" -** op = 0x08 -** a3 = Salt1 value -** -** ELOG_CLOSE_WAL "Close the WAL file connection" -** op = 0x0e -** -** ELOG_CLOSE_DB "Close the DB connection" -** op = 0x0f -** -** VIEWING TIMESTAMPS AND LOGS -** -** The command-line utility at tool/showtmlog.c will read and display -** the content of one or more tmstmpvfs.c log files. If all of the -** log files are stored in directory $(DATABASE)-tmstmp, then you can -** view them all using a command like shown below (with an extra "?" -** inserted on the wildcard to avoid closing the C-language comment -** that contains this text): -** -** showtmlog $(DATABASE)-tmstmp/?* -** -** The command-line utility at tools/showdb.c can be used to show the -** timestamps on pages of a database file, using a command like this: -** -** showdb --tmstmp $(DATABASE) pgidx -* -** The command above shows the timestamp and the intended use of every -** pages in the database, in human-readable form. If you also add -** the --csv option to the command above, then the command generates -** a Comma-Separated-Value (CSV) file as output, which contains a -** decoding of the complete timestamp tag on each page of the database. -** This CVS file can be easily imported into another SQLite database -** using a CLI command like the following: -** -** .import --csv '|showdb --tmstmp -csv orig.db pgidx' ts_table -** -** In the command above, the database containing the timestamps is -** "orig.db" and the content is imported into a new table named "ts_table". -** The "ts_table" is created automatically, using the column names found -** in the first line of the CSV file. All columns of the automatically -** created ts_table are of type TEXT. It might make more sense to -** create the table yourself, using more sensible datatypes, like this: -** -** CREATE TABLE ts_table ( -** pgno INT, -- page number -** tm REAL, -- seconds since 1970-01-01 -** frame INT, -- WAL frame number -** flg INT, -- flag (tag byte 12) -** salt INT, -- WAL salt (tag bytes 13-15) -** parent INT, -- Parent page number -** child INT, -- Index of this page in its parent -** ovfl INT, -- Index of this page on the overflow chain -** txt TEXT -- Description of this page -** ); -** -** Then import using: -** -** .import --csv --skip 1 '|showdb --tmstmp --csv orig.db pgidx' ts_table -** -** Note the addition of the "--skip 1" option on ".import" to bypass the -** first line of the CSV file that contains the column names. -** -** Both programs "showdb" and "showtmlog" can be built by running -** "make showtmlog showdb" from the top-level of a recent SQLite -** source tree. -*/ -#if defined(SQLITE_AMALGAMATION) && !defined(SQLITE_TMSTMPVFS_STATIC) -# define SQLITE_TMSTMPVFS_STATIC -#endif -#ifdef SQLITE_TMSTMPVFS_STATIC -# include "sqlite3.h" -#else -# include "sqlite3ext.h" - SQLITE_EXTENSION_INIT1 -#endif -#include <string.h> -#include <assert.h> -#include <stdio.h> - -/* -** Forward declaration of objects used by this utility -*/ -typedef struct sqlite3_vfs TmstmpVfs; -typedef struct TmstmpFile TmstmpFile; -typedef struct TmstmpLog TmstmpLog; - -/* -** Bytes of reserved space used by this extension -*/ -#define TMSTMP_RESERVE 16 - -/* -** The magic number used to identify TmstmpFile objects -*/ -#define TMSTMP_MAGIC 0x2a87b72d - -/* -** Useful datatype abbreviations -*/ -#if !defined(SQLITE_AMALGAMATION) - typedef unsigned char u8; - typedef unsigned int u32; -#endif - -/* -** Current process id -*/ -#if defined(_WIN32) -# include <windows.h> -# define GETPID (u32)GetCurrentProcessId() -#else -# include <unistd.h> -# define GETPID (u32)getpid() -#endif - -/* Access to a lower-level VFS that (might) implement dynamic loading, -** access to randomness, etc. -*/ -#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) -#define ORIGFILE(p) ((sqlite3_file*)(((TmstmpFile*)(p))+1)) - -/* Information for the tmstmp log file. */ -struct TmstmpLog { - char *zLogname; /* Log filename */ - FILE *log; /* Open log file */ - int n; /* Bytes of a[] used */ - unsigned char a[16*6]; /* Buffered header for the log */ -}; - -/* An open WAL or DB file */ -struct TmstmpFile { - sqlite3_file base; /* IO methods */ - u32 uMagic; /* Magic number for sanity checking */ - u32 salt1; /* Last WAL salt-1 value */ - u32 iFrame; /* Last WAL frame number */ - u32 pgno; /* Current page number */ - u32 pgsz; /* Size of each page, in bytes */ - u8 isWal; /* True if this is a WAL file */ - u8 isDb; /* True if this is a DB file */ - u8 isCommit; /* Last WAL frame header was a transaction commit */ - u8 hasCorrectReserve; /* File has the correct reserve size */ - u8 inCkpt; /* True if in a checkpoint */ - TmstmpLog *pLog; /* Log file */ - TmstmpFile *pPartner; /* DB->WAL or WAL->DB mapping */ - sqlite3_int64 iOfst; /* Offset of last WAL frame header */ - sqlite3_vfs *pSubVfs; /* Underlying VFS */ -}; - -/* -** Event log opcodes -*/ -#define ELOG_OPEN_DB 0x01 -#define ELOG_OPEN_WAL 0x02 -#define ELOG_WAL_PAGE 0x03 -#define ELOG_DB_PAGE 0x04 -#define ELOG_CKPT_START 0x05 -#define ELOG_CKPT_PAGE 0x06 -#define ELOG_CKPT_DONE 0x07 -#define ELOG_WAL_RESET 0x08 -#define ELOG_CLOSE_WAL 0x0e -#define ELOG_CLOSE_DB 0x0f - -/* -** Methods for TmstmpFile -*/ -static int tmstmpClose(sqlite3_file*); -static int tmstmpRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); -static int tmstmpWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); -static int tmstmpTruncate(sqlite3_file*, sqlite3_int64 size); -static int tmstmpSync(sqlite3_file*, int flags); -static int tmstmpFileSize(sqlite3_file*, sqlite3_int64 *pSize); -static int tmstmpLock(sqlite3_file*, int); -static int tmstmpUnlock(sqlite3_file*, int); -static int tmstmpCheckReservedLock(sqlite3_file*, int *pResOut); -static int tmstmpFileControl(sqlite3_file*, int op, void *pArg); -static int tmstmpSectorSize(sqlite3_file*); -static int tmstmpDeviceCharacteristics(sqlite3_file*); -static int tmstmpShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); -static int tmstmpShmLock(sqlite3_file*, int offset, int n, int flags); -static void tmstmpShmBarrier(sqlite3_file*); -static int tmstmpShmUnmap(sqlite3_file*, int deleteFlag); -static int tmstmpFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); -static int tmstmpUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); - -/* -** Methods for TmstmpVfs -*/ -static int tmstmpOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); -static int tmstmpDelete(sqlite3_vfs*, const char *zName, int syncDir); -static int tmstmpAccess(sqlite3_vfs*, const char *zName, int flags, int *); -static int tmstmpFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); -static void *tmstmpDlOpen(sqlite3_vfs*, const char *zFilename); -static void tmstmpDlError(sqlite3_vfs*, int nByte, char *zErrMsg); -static void (*tmstmpDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); -static void tmstmpDlClose(sqlite3_vfs*, void*); -static int tmstmpRandomness(sqlite3_vfs*, int nByte, char *zOut); -static int tmstmpSleep(sqlite3_vfs*, int microseconds); -static int tmstmpCurrentTime(sqlite3_vfs*, double*); -static int tmstmpGetLastError(sqlite3_vfs*, int, char *); -static int tmstmpCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); -static int tmstmpSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr); -static sqlite3_syscall_ptr tmstmpGetSystemCall(sqlite3_vfs*, const char *z); -static const char *tmstmpNextSystemCall(sqlite3_vfs*, const char *zName); - -static sqlite3_vfs tmstmp_vfs = { - 3, /* iVersion (set when registered) */ - 0, /* szOsFile (set when registered) */ - 1024, /* mxPathname */ - 0, /* pNext */ - "tmstmpvfs", /* zName */ - 0, /* pAppData (set when registered) */ - tmstmpOpen, /* xOpen */ - tmstmpDelete, /* xDelete */ - tmstmpAccess, /* xAccess */ - tmstmpFullPathname, /* xFullPathname */ - tmstmpDlOpen, /* xDlOpen */ - tmstmpDlError, /* xDlError */ - tmstmpDlSym, /* xDlSym */ - tmstmpDlClose, /* xDlClose */ - tmstmpRandomness, /* xRandomness */ - tmstmpSleep, /* xSleep */ - tmstmpCurrentTime, /* xCurrentTime */ - tmstmpGetLastError, /* xGetLastError */ - tmstmpCurrentTimeInt64, /* xCurrentTimeInt64 */ - tmstmpSetSystemCall, /* xSetSystemCall */ - tmstmpGetSystemCall, /* xGetSystemCall */ - tmstmpNextSystemCall /* xNextSystemCall */ -}; - -static const sqlite3_io_methods tmstmp_io_methods = { - 3, /* iVersion */ - tmstmpClose, /* xClose */ - tmstmpRead, /* xRead */ - tmstmpWrite, /* xWrite */ - tmstmpTruncate, /* xTruncate */ - tmstmpSync, /* xSync */ - tmstmpFileSize, /* xFileSize */ - tmstmpLock, /* xLock */ - tmstmpUnlock, /* xUnlock */ - tmstmpCheckReservedLock, /* xCheckReservedLock */ - tmstmpFileControl, /* xFileControl */ - tmstmpSectorSize, /* xSectorSize */ - tmstmpDeviceCharacteristics, /* xDeviceCharacteristics */ - tmstmpShmMap, /* xShmMap */ - tmstmpShmLock, /* xShmLock */ - tmstmpShmBarrier, /* xShmBarrier */ - tmstmpShmUnmap, /* xShmUnmap */ - tmstmpFetch, /* xFetch */ - tmstmpUnfetch /* xUnfetch */ -}; - -/* -** Write a 6-byte millisecond timestamp into aOut[] -*/ -static void tmstmpPutTS(TmstmpFile *p, unsigned char *aOut){ - sqlite3_uint64 tm = 0; - p->pSubVfs->xCurrentTimeInt64(p->pSubVfs, (sqlite3_int64*)&tm); - tm -= 210866760000000LL; - aOut[0] = (tm>>40)&0xff; - aOut[1] = (tm>>32)&0xff; - aOut[2] = (tm>>24)&0xff; - aOut[3] = (tm>>16)&0xff; - aOut[4] = (tm>>8)&0xff; - aOut[5] = tm&0xff; -} - -/* -** Read a 32-bit big-endian unsigned integer and return it. -*/ -static u32 tmstmpGetU32(const unsigned char *a){ - return (a[0]<<24) + (a[1]<<16) + (a[2]<<8) + a[3]; -} - -/* Write a 32-bit integer as big-ending into a[] -*/ -static void tmstmpPutU32(u32 v, unsigned char *a){ - a[0] = (v>>24) & 0xff; - a[1] = (v>>16) & 0xff; - a[2] = (v>>8) & 0xff; - a[3] = v & 0xff; -} - -/* Free a TmstmpLog object */ -static void tmstmpLogFree(TmstmpLog *pLog){ - if( pLog==0 ) return; - if( pLog->log ) fclose(pLog->log); - sqlite3_free(pLog->zLogname); - sqlite3_free(pLog); -} - -/* Flush log content. Open the file if necessary. Return the -** number of errors. */ -static int tmstmpLogFlush(TmstmpFile *p){ - TmstmpLog *pLog = p->pLog; - assert( pLog!=0 ); - if( pLog->log==0 ){ - pLog->log = fopen(pLog->zLogname, "wb"); - if( pLog->log==0 ){ - tmstmpLogFree(pLog); - p->pLog = 0; - return 1; - } - } - (void)fwrite(pLog->a, pLog->n, 1, pLog->log); - fflush(pLog->log); - pLog->n = 0; - return 0; -} - -/* -** Write a record onto the event log -*/ -static void tmstmpEvent( - TmstmpFile *p, - u8 op, - u8 a1, - u32 a2, - u32 a3, - u8 *pTS -){ - unsigned char *a; - TmstmpLog *pLog; - if( p->isWal ){ - p = p->pPartner; - assert( p!=0 ); - assert( p->isDb ); - } - pLog = p->pLog; - if( pLog==0 ) return; - if( pLog->n >= (int)sizeof(pLog->a) ){ - if( tmstmpLogFlush(p) ) return; - } - a = pLog->a + pLog->n; - a[0] = op; - a[1] = a1; - if( pTS ){ - memcpy(a+2, pTS, 6); - }else{ - tmstmpPutTS(p, a+2); - } - tmstmpPutU32(a2, a+8); - tmstmpPutU32(a3, a+12); - pLog->n += 16; - if( pLog->log || (op>=ELOG_WAL_PAGE && op<=ELOG_WAL_RESET) ){ - (void)tmstmpLogFlush(p); - } -} - -/* -** Close a connection -*/ -static int tmstmpClose(sqlite3_file *pFile){ - TmstmpFile *p = (TmstmpFile *)pFile; - if( p->hasCorrectReserve ){ - tmstmpEvent(p, p->isDb ? ELOG_CLOSE_DB : ELOG_CLOSE_WAL, 0, 0, 0, 0); - } - tmstmpLogFree(p->pLog); - if( p->pPartner ){ - assert( p->pPartner->pPartner==p ); - p->pPartner->pPartner = 0; - p->pPartner = 0; - } - pFile = ORIGFILE(pFile); - return pFile->pMethods->xClose(pFile); -} - -/* -** Read bytes from a file -*/ -static int tmstmpRead( - sqlite3_file *pFile, - void *zBuf, - int iAmt, - sqlite_int64 iOfst -){ - int rc; - TmstmpFile *p = (TmstmpFile*)pFile; - pFile = ORIGFILE(pFile); - rc = pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst); - if( rc!=SQLITE_OK ) return rc; - if( p->isDb - && iOfst==0 - && iAmt>=100 - ){ - const unsigned char *a = (unsigned char*)zBuf; - p->hasCorrectReserve = (a[20]==TMSTMP_RESERVE); - p->pgsz = (a[16]<<8) + a[17]; - if( p->pgsz==1 ) p->pgsz = 65536; - if( p->pPartner ){ - p->pPartner->hasCorrectReserve = p->hasCorrectReserve; - p->pPartner->pgsz = p->pgsz; - } - } - if( p->isWal - && p->inCkpt - && iAmt>=512 && iAmt<=65535 && (iAmt&(iAmt-1))==0 - ){ - p->pPartner->iFrame = (iOfst-56)/(p->pgsz+24) + 1; - } - return rc; -} - -/* -** Write data to a tmstmp-file. -*/ -static int tmstmpWrite( - sqlite3_file *pFile, - const void *zBuf, - int iAmt, - sqlite_int64 iOfst -){ - TmstmpFile *p = (TmstmpFile*)pFile; - sqlite3_file *pSub = ORIGFILE(pFile); - if( !p->hasCorrectReserve ){ - /* The database does not have the correct reserve size. No-op */ - }else if( p->isWal ){ - /* Writing into a WAL file */ - if( iAmt==24 ){ - /* A frame header */ - u32 x = 0; - p->iFrame = (iOfst - 32)/(p->pgsz+24)+1; - p->pgno = tmstmpGetU32((const u8*)zBuf); - p->salt1 = tmstmpGetU32(((const u8*)zBuf)+8); - memcpy(&x, ((const u8*)zBuf)+4, 4); - p->isCommit = (x!=0); - p->iOfst = iOfst; - }else if( iAmt>=512 && iOfst==p->iOfst+24 ){ - unsigned char s[TMSTMP_RESERVE]; - memset(s, 0, TMSTMP_RESERVE); - tmstmpPutTS(p, s+2); - tmstmpEvent(p, ELOG_WAL_PAGE, p->isCommit, p->pgno, p->iFrame, s+2); - }else if( iAmt==32 && iOfst==0 ){ - p->salt1 = tmstmpGetU32(((const u8*)zBuf)+16); - tmstmpEvent(p, ELOG_WAL_RESET, 0, 0, p->salt1, 0); - } - }else if( p->inCkpt ){ - unsigned char *s = (unsigned char*)zBuf+iAmt-TMSTMP_RESERVE; - memset(s, 0, TMSTMP_RESERVE); - tmstmpPutTS(p, s+2); - tmstmpPutU32(p->iFrame, s+8); - tmstmpPutU32(p->pPartner->salt1 & 0xffffff, s+12); - assert( p->pgsz>0 ); - tmstmpEvent(p, ELOG_CKPT_PAGE, 0, (iOfst/p->pgsz)+1, p->iFrame, 0); - }else if( p->pPartner==0 ){ - /* Writing into a database in rollback mode */ - unsigned char *s = (unsigned char*)zBuf+iAmt-TMSTMP_RESERVE; - memset(s, 0, TMSTMP_RESERVE); - tmstmpPutTS(p, s+2); - s[12] = 2; - assert( p->pgsz>0 ); - tmstmpEvent(p, ELOG_DB_PAGE, 0, (u32)(iOfst/p->pgsz)+1, 0, s+2); - } - return pSub->pMethods->xWrite(pSub,zBuf,iAmt,iOfst); -} - -/* -** Truncate a tmstmp-file. -*/ -static int tmstmpTruncate(sqlite3_file *pFile, sqlite_int64 size){ - pFile = ORIGFILE(pFile); - return pFile->pMethods->xTruncate(pFile, size); -} - -/* -** Sync a tmstmp-file. -*/ -static int tmstmpSync(sqlite3_file *pFile, int flags){ - pFile = ORIGFILE(pFile); - return pFile->pMethods->xSync(pFile, flags); -} - -/* -** Return the current file-size of a tmstmp-file. -*/ -static int tmstmpFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ - TmstmpFile *p = (TmstmpFile *)pFile; - pFile = ORIGFILE(p); - return pFile->pMethods->xFileSize(pFile, pSize); -} - -/* -** Lock a tmstmp-file. -*/ -static int tmstmpLock(sqlite3_file *pFile, int eLock){ - pFile = ORIGFILE(pFile); - return pFile->pMethods->xLock(pFile, eLock); -} - -/* -** Unlock a tmstmp-file. -*/ -static int tmstmpUnlock(sqlite3_file *pFile, int eLock){ - pFile = ORIGFILE(pFile); - return pFile->pMethods->xUnlock(pFile, eLock); -} - -/* -** Check if another file-handle holds a RESERVED lock on a tmstmp-file. -*/ -static int tmstmpCheckReservedLock(sqlite3_file *pFile, int *pResOut){ - pFile = ORIGFILE(pFile); - return pFile->pMethods->xCheckReservedLock(pFile, pResOut); -} - -/* -** File control method. For custom operations on a tmstmp-file. -*/ -static int tmstmpFileControl(sqlite3_file *pFile, int op, void *pArg){ - int rc; - TmstmpFile *p = (TmstmpFile*)pFile; - pFile = ORIGFILE(pFile); - rc = pFile->pMethods->xFileControl(pFile, op, pArg); - switch( op ){ - case SQLITE_FCNTL_VFSNAME: { - if( p->hasCorrectReserve && rc==SQLITE_OK ){ - *(char**)pArg = sqlite3_mprintf("tmstmp/%z", *(char**)pArg); - } - break; - } - case SQLITE_FCNTL_CKPT_START: { - p->inCkpt = 1; - assert( p->isDb ); - assert( p->pPartner!=0 ); - p->pPartner->inCkpt = 1; - if( p->hasCorrectReserve ){ - tmstmpEvent(p, ELOG_CKPT_START, 0, 0, 0, 0); - } - rc = SQLITE_OK; - break; - } - case SQLITE_FCNTL_CKPT_DONE: { - p->inCkpt = 0; - assert( p->isDb ); - assert( p->pPartner!=0 ); - p->pPartner->inCkpt = 0; - if( p->hasCorrectReserve ){ - tmstmpEvent(p, ELOG_CKPT_DONE, 0, 0, 0, 0); - } - rc = SQLITE_OK; - break; - } - } - return rc; -} - -/* -** Return the sector-size in bytes for a tmstmp-file. -*/ -static int tmstmpSectorSize(sqlite3_file *pFile){ - pFile = ORIGFILE(pFile); - return pFile->pMethods->xSectorSize(pFile); -} - -/* -** Return the device characteristic flags supported by a tmstmp-file. -*/ -static int tmstmpDeviceCharacteristics(sqlite3_file *pFile){ - int devchar = 0; - pFile = ORIGFILE(pFile); - devchar = pFile->pMethods->xDeviceCharacteristics(pFile); - return (devchar & ~SQLITE_IOCAP_SUBPAGE_READ); -} - -/* Create a shared memory file mapping */ -static int tmstmpShmMap( - sqlite3_file *pFile, - int iPg, - int pgsz, - int bExtend, - void volatile **pp -){ - pFile = ORIGFILE(pFile); - return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp); -} - -/* Perform locking on a shared-memory segment */ -static int tmstmpShmLock(sqlite3_file *pFile, int offset, int n, int flags){ - pFile = ORIGFILE(pFile); - return pFile->pMethods->xShmLock(pFile,offset,n,flags); -} - -/* Memory barrier operation on shared memory */ -static void tmstmpShmBarrier(sqlite3_file *pFile){ - pFile = ORIGFILE(pFile); - pFile->pMethods->xShmBarrier(pFile); -} - -/* Unmap a shared memory segment */ -static int tmstmpShmUnmap(sqlite3_file *pFile, int deleteFlag){ - pFile = ORIGFILE(pFile); - return pFile->pMethods->xShmUnmap(pFile,deleteFlag); -} - -/* Fetch a page of a memory-mapped file */ -static int tmstmpFetch( - sqlite3_file *pFile, - sqlite3_int64 iOfst, - int iAmt, - void **pp -){ - pFile = ORIGFILE(pFile); - return pFile->pMethods->xFetch(pFile, iOfst, iAmt, pp); -} - -/* Release a memory-mapped page */ -static int tmstmpUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ - pFile = ORIGFILE(pFile); - return pFile->pMethods->xUnfetch(pFile, iOfst, pPage); -} - - -/* -** Open a tmstmp file handle. -*/ -static int tmstmpOpen( - sqlite3_vfs *pVfs, - const char *zName, - sqlite3_file *pFile, - int flags, - int *pOutFlags -){ - TmstmpFile *p, *pDb; - sqlite3_file *pSubFile; - sqlite3_vfs *pSubVfs; - int rc; - - pSubVfs = ORIGVFS(pVfs); - if( (flags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_WAL))==0 ){ - /* If the file is not a persistent database or a WAL file, then - ** bypass the timestamp logic all together */ - return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags); - } - if( (flags & SQLITE_OPEN_WAL)!=0 ){ - pDb = (TmstmpFile*)sqlite3_database_file_object(zName); - if( pDb==0 - || pDb->uMagic!=TMSTMP_MAGIC - || !pDb->isDb - || pDb->pPartner!=0 - ){ - return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags); - } - }else{ - pDb = 0; - } - p = (TmstmpFile*)pFile; - memset(p, 0, sizeof(*p)); - pSubFile = ORIGFILE(pFile); - pFile->pMethods = &tmstmp_io_methods; - p->pSubVfs = pSubVfs; - p->uMagic = TMSTMP_MAGIC; - rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags); - if( rc ) goto tmstmp_open_done; - if( pDb!=0 ){ - p->isWal = 1; - p->pPartner = pDb; - pDb->pPartner = p; - }else{ - u32 r2; - u32 pid; - TmstmpLog *pLog; - sqlite3_uint64 r1; /* Milliseconds since 1970-01-01 */ - sqlite3_uint64 days; /* Days since 1970-01-01 */ - sqlite3_uint64 sod; /* Start of date specified by r1 */ - sqlite3_uint64 z; /* Days since 0000-03-01 */ - sqlite3_uint64 era; /* 400-year era */ - int h; /* hour */ - int m; /* minute */ - int s; /* second */ - int f; /* millisecond */ - int Y; /* year */ - int M; /* month */ - int D; /* day */ - int y; /* year assuming March is first month */ - unsigned int doe; /* day of 400-year era */ - unsigned int yoe; /* year of 400-year era */ - unsigned int doy; /* day of year */ - unsigned int mp; /* month with March==0 */ - - p->isDb = 1; - r1 = 0; - pLog = sqlite3_malloc64( sizeof(TmstmpLog) ); - if( pLog==0 ){ - pSubFile->pMethods->xClose(pSubFile); - rc = SQLITE_NOMEM; - goto tmstmp_open_done; - } - memset(pLog, 0, sizeof(pLog[0])); - p->pLog = pLog; - p->pSubVfs->xCurrentTimeInt64(p->pSubVfs, (sqlite3_int64*)&r1); - r1 -= 210866760000000LL; - days = r1/86400000; - sod = (r1%86400000)/1000; - f = (int)(r1%1000); - - h = sod/3600; - m = (sod%3600)/60; - s = sod%60; - z = days + 719468; - era = z/146097; - doe = (unsigned)(z - era*146097); - yoe = (doe - doe/1460 + doe/36524 - doe/146096)/365; - y = (int)yoe + era*400; - doy = doe - (365*yoe + yoe/4 - yoe/100); - mp = (5*doy + 2)/153; - D = doy - (153*mp + 2)/5 + 1; - M = mp + (mp<10 ? 3 : -9); - Y = y + (M <=2); - sqlite3_randomness(sizeof(r2), &r2); - pid = GETPID; - pLog->zLogname = sqlite3_mprintf( - "%s-tmstmp/%04d%02d%02dT%02d%02d%02d%03d-%08d-%08x", - zName, Y, M, D, h, m, s, f, pid, r2); - } - tmstmpEvent(p, p->isWal ? ELOG_OPEN_WAL : ELOG_OPEN_DB, 0, GETPID, 0, 0); - -tmstmp_open_done: - if( rc ) pFile->pMethods = 0; - return rc; -} - -/* -** All VFS interfaces other than xOpen are passed down into the Sub-VFS. -*/ -static int tmstmpDelete(sqlite3_vfs *p, const char *zName, int syncDir){ - sqlite3_vfs *pSub = ORIGVFS(p); - return pSub->xDelete(pSub,zName,syncDir); -} -static int tmstmpAccess(sqlite3_vfs *p, const char *zName, int flags, int *pR){ - sqlite3_vfs *pSub = ORIGVFS(p); - return pSub->xAccess(pSub,zName,flags,pR); -} -static int tmstmpFullPathname(sqlite3_vfs*p,const char *zName,int n,char *zOut){ - sqlite3_vfs *pSub = ORIGVFS(p); - return pSub->xFullPathname(pSub,zName,n,zOut); -} -static void *tmstmpDlOpen(sqlite3_vfs *p, const char *zFilename){ - sqlite3_vfs *pSub = ORIGVFS(p); - return pSub->xDlOpen(pSub,zFilename); -} -static void tmstmpDlError(sqlite3_vfs *p, int nByte, char *zErrMsg){ - sqlite3_vfs *pSub = ORIGVFS(p); - return pSub->xDlError(pSub,nByte,zErrMsg); -} -static void(*tmstmpDlSym(sqlite3_vfs *p, void *pDl, const char *zSym))(void){ - sqlite3_vfs *pSub = ORIGVFS(p); - return pSub->xDlSym(pSub,pDl,zSym); -} -static void tmstmpDlClose(sqlite3_vfs *p, void *pDl){ - sqlite3_vfs *pSub = ORIGVFS(p); - return pSub->xDlClose(pSub,pDl); -} -static int tmstmpRandomness(sqlite3_vfs *p, int nByte, char *zOut){ - sqlite3_vfs *pSub = ORIGVFS(p); - return pSub->xRandomness(pSub,nByte,zOut); -} -static int tmstmpSleep(sqlite3_vfs *p, int microseconds){ - sqlite3_vfs *pSub = ORIGVFS(p); - return pSub->xSleep(pSub,microseconds); -} -static int tmstmpCurrentTime(sqlite3_vfs *p, double *prNow){ - sqlite3_vfs *pSub = ORIGVFS(p); - return pSub->xCurrentTime(pSub,prNow); -} -static int tmstmpGetLastError(sqlite3_vfs *p, int a, char *b){ - sqlite3_vfs *pSub = ORIGVFS(p); - return pSub->xGetLastError(pSub,a,b); -} -static int tmstmpCurrentTimeInt64(sqlite3_vfs *p, sqlite3_int64 *piNow){ - sqlite3_vfs *pSub = ORIGVFS(p); - return pSub->xCurrentTimeInt64(pSub,piNow); -} -static int tmstmpSetSystemCall(sqlite3_vfs *p, const char *zName, - sqlite3_syscall_ptr x){ - sqlite3_vfs *pSub = ORIGVFS(p); - return pSub->xSetSystemCall(pSub,zName,x); -} -static sqlite3_syscall_ptr tmstmpGetSystemCall(sqlite3_vfs *p, const char *z){ - sqlite3_vfs *pSub = ORIGVFS(p); - return pSub->xGetSystemCall(pSub,z); -} -static const char *tmstmpNextSystemCall(sqlite3_vfs *p, const char *zName){ - sqlite3_vfs *pSub = ORIGVFS(p); - return pSub->xNextSystemCall(pSub,zName); -} - -/* -** Register the tmstmp VFS as the default VFS for the system. -*/ -static int tmstmpRegisterVfs(void){ - int rc = SQLITE_OK; - sqlite3_vfs *pOrig = sqlite3_vfs_find(0); - if( pOrig==0 ) return SQLITE_ERROR; - if( pOrig==&tmstmp_vfs ) return SQLITE_OK; - tmstmp_vfs.iVersion = pOrig->iVersion; - tmstmp_vfs.pAppData = pOrig; - tmstmp_vfs.szOsFile = pOrig->szOsFile + sizeof(TmstmpFile); - rc = sqlite3_vfs_register(&tmstmp_vfs, 1); - return rc; -} - -#if defined(SQLITE_TMSTMPVFS_STATIC) -/* This variant of the initializer runs when the extension is -** statically linked. -*/ -int sqlite3_register_tmstmpvfs(const char *NotUsed){ - (void)NotUsed; - return tmstmpRegisterVfs(); -} -int sqlite3_unregister_tmstmpvfs(void){ - if( sqlite3_vfs_find("tmstmpvfs") ){ - sqlite3_vfs_unregister(&tmstmp_vfs); - } - return SQLITE_OK; -} -#endif /* defined(SQLITE_TMSTMPVFS_STATIC */ - -#if !defined(SQLITE_TMSTMPVFS_STATIC) -/* This variant of the initializer function is used when the -** extension is shared library to be loaded at run-time. -*/ -#ifdef _WIN32 -__declspec(dllexport) -#endif -/* -** This routine is called by sqlite3_load_extension() when the -** extension is first loaded. -***/ -int sqlite3_tmstmpvfs_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - int rc; - SQLITE_EXTENSION_INIT2(pApi); - (void)pzErrMsg; /* not used */ - (void)db; /* not used */ - rc = tmstmpRegisterVfs(); - if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; - return rc; -} -#endif /* !defined(SQLITE_TMSTMPVFS_STATIC) */ diff --git a/ext/misc/vtablog.c b/ext/misc/vtablog.c index 1e28495de..2b3e30355 100644 --- a/ext/misc/vtablog.c +++ b/ext/misc/vtablog.c @@ -14,8 +14,6 @@ ** on stdout when its key interfaces are called. This is intended for ** interactive analysis and debugging of virtual table interfaces. ** -** HOW TO COMPILE: -** ** To build this extension as a separately loaded shared library or ** DLL, use compiler command-lines similar to the following: ** @@ -23,7 +21,7 @@ ** (mac) clang -fPIC -dynamiclib vtablog.c -o vtablog.dylib ** (windows) cl vtablog.c -link -dll -out:vtablog.dll ** -** USAGE EXAMPLE: +** Usage example: ** ** .load ./vtablog ** CREATE VIRTUAL TABLE temp.log USING vtablog( @@ -31,23 +29,6 @@ ** rows=25 ** ); ** SELECT * FROM log; -** -** ARGUMENTS TO CREATE VIRTUAL TABLE: -** -** In "CREATE VIRTUAL TABLE temp.log AS vtablog(ARGS....)" statement, the -** ARGS argument is a list of key-value pairs that can be any of the -** following. -** -** schema=TEXT Text is a CREATE TABLE statement that defines -** the schema of the new virtual table. -** -** rows=N The table as N rows. -** -** consume_order_by=N If the left-most ORDER BY terms is ASC and -** against column N (where the leftmost column -** is #1) then set the orderByConsumed=1 flag in -** xBestIndex. Or if the left-most ORDER BY is -** DESC and against column -N, do likewise. */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -68,8 +49,6 @@ struct vtablog_vtab { char *zName; /* Table name. argv[2] of xConnect/xCreate */ int nRow; /* Number of rows in the table */ int nCursor; /* Number of cursors created */ - int iConsumeOB; /* Consume the ORDER BY clause if on column N-th - ** and consumeOB=N or consumeOB=(-N) and DESC */ }; /* vtablog_cursor is a subclass of sqlite3_vtab_cursor which will @@ -201,7 +180,6 @@ static int vtablogConnectCreate( int rc; char *zSchema = 0; char *zNRow = 0; - char *zConsumeOB = 0; printf("%s.%s.%s():\n", argv[1], argv[2], isCreate ? "xCreate" : "xConnect"); @@ -225,10 +203,6 @@ static int vtablogConnectCreate( rc = SQLITE_ERROR; goto vtablog_end_connect; } - if( vtablog_string_parameter(pzErr, "consume_order_by", z, &zConsumeOB) ){ - rc = SQLITE_ERROR; - goto vtablog_end_connect; - } } if( zSchema==0 ){ zSchema = sqlite3_mprintf("%s","CREATE TABLE x(a,b);"); @@ -247,10 +221,6 @@ static int vtablogConnectCreate( pNew->nRow = 10; if( zNRow ) pNew->nRow = atoi(zNRow); printf(" nrow = %d\n", pNew->nRow); - if( zConsumeOB ) pNew->iConsumeOB = atoi(zConsumeOB); - if( pNew->iConsumeOB ){ - printf(" consume_order_by = %d\n", pNew->iConsumeOB); - } pNew->zDb = sqlite3_mprintf("%s", argv[1]); pNew->zName = sqlite3_mprintf("%s", argv[2]); } @@ -258,7 +228,6 @@ static int vtablogConnectCreate( vtablog_end_connect: sqlite3_free(zSchema); sqlite3_free(zNRow); - sqlite3_free(zConsumeOB); return rc; } static int vtablogCreate( @@ -545,27 +514,16 @@ static int vtablogBestIndex( } } printf(" nOrderBy: %d\n", p->nOrderBy); - if( p->nOrderBy ){ - for(i=0; i<p->nOrderBy; i++){ - printf(" orderby[%d]: col=%d desc=%d\n", - i, - p->aOrderBy[i].iColumn, - p->aOrderBy[i].desc); - } - if( pTab->iConsumeOB ){ - int N = p->aOrderBy[0].iColumn+1; - if( (p->aOrderBy[0].desc && N==-pTab->iConsumeOB) - || (!p->aOrderBy[0].desc && N==pTab->iConsumeOB) - ){ - p->orderByConsumed = 1; - } - } + for(i=0; i<p->nOrderBy; i++){ + printf(" orderby[%d]: col=%d desc=%d\n", + i, + p->aOrderBy[i].iColumn, + p->aOrderBy[i].desc); } p->estimatedCost = (double)500; p->estimatedRows = 500; printf(" idxNum=%d\n", p->idxNum); printf(" idxStr=NULL\n"); - printf(" sqlite3_vtab_distinct()=%d\n", sqlite3_vtab_distinct(p)); printf(" orderByConsumed=%d\n", p->orderByConsumed); printf(" estimatedCost=%g\n", p->estimatedCost); printf(" estimatedRows=%lld\n", p->estimatedRows); diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index c4862650b..086b058cc 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -705,12 +705,7 @@ static int zipfileScanExtra(u8 *aExtra, int nExtra, u32 *pmTime){ u8 *p = aExtra; u8 *pEnd = &aExtra[nExtra]; - /* Stop when there are less than 9 bytes left to scan in the buffer. This - ** is because the timestamp field requires exactly 9 bytes - 4 bytes of - ** header fields and 5 bytes of data. If there are less than 9 bytes - ** remaining, either it is some other field or else the extra data - ** is corrupt. Either way, do not process it. */ - while( p+(2*sizeof(u16) + 1 + sizeof(u32))<=pEnd ){ + while( p<pEnd ){ u16 id = zipfileRead16(p); u16 nByte = zipfileRead16(p); diff --git a/ext/qrf/README.md b/ext/qrf/README.md deleted file mode 100644 index 4bb1790a4..000000000 --- a/ext/qrf/README.md +++ /dev/null @@ -1,762 +0,0 @@ -# SQLite Query Result Formatting Subsystem - -The "Query Result Formatter" or "QRF" subsystem is a C-language -subroutine that formats the output from an SQLite query for display using -a fix-width font, for example on a terminal window over an SSH connection. -The output format is configurable. The application can request various -table formats, with flexible column widths and alignments, row-oriented -formats, such as CSV and similar, as well as various special purpose formats -like JSON. - -For the first 25 years of SQLite's existance, the -[command-line interface](https://sqlite.org/cli.html) (CLI) -formatted query results using a hodge-podge of routines -that had grown slowly by accretion. The QRF was created -in fall of 2025 to refactor and reorganize this code into -a more usable form. The idea behind QRF is to implement all the -query result formatting capabilities of the CLI in a subroutine -that can be incorporated and reused by other applications. - -## 1.0 Overview Of Operation - -Suppose variable `sqlite3_stmt *pStmt` is a pointer to an SQLite -prepared statement that has been reset and bound and is ready to run. -Then to format the output from this prepared statement, use code -similar to the following: - -> ~~~ -sqlite3_qrf_spec spec; /* Format specification */ -char *zErrMsg; /* Text error message (optional) */ -char *zResult = 0; /* Formatted output written here */ -int rc; /* Result code */ - -memset(&spec, 0, sizeof(spec)); /* Initialize the spec */ -spec.iVersion = 1; /* Version number must be 1 */ -spec.pzOutput = &zResult; /* Write results in variable zResult */ -/* Optionally fill in other settings in spec here, as needed */ -zErrMsg = 0; /* Not required; just being pedantic */ -rc = sqlite3_format_query_result(pStmt, &spec, &zErrMsg); /* Format results */ -if( rc ){ - printf("Error (%d): %s\n", rc, zErrMsg); /* Report an error */ - sqlite3_free(zErrMsg); /* Free the error message text */ -}else{ - printf("%s", zResult); /* Report the results */ -} -sqlite3_free(zResult); /* Free memory used to hold results */ -~~~ - -The `sqlite3_qrf_spec` object describes the desired output format -and where to send the generated output. Most of the work in using -the QRF involves filling out the sqlite3_qrf_spec. - -### 1.1 Using QRF with SQL text - -If you start with SQL text instead of an sqlite3_stmt pointer, and -especially if the SQL text might comprise two or more statements, then -the SQL text needs to be converted into sqlite3_stmt objects separately. -If the original SQL text is in a variable `const char *zSql` and the -database connection is in variable `sqlite3 *db`, then code -similar to the following should work: - -> ~~~ -sqlite3_qrf_spec spec; /* Format specification */ -char *zErrMsg; /* Text error message (optional) */ -char *zResult = 0; /* Formatted output written here */ -sqlite3_stmt *pStmt; /* Next prepared statement */ -int rc; /* Result code */ - -memset(&spec, 0, sizeof(spec)); /* Initialize the spec */ -spec.iVersion = 1; /* Version number must be 1 */ -spec.pzOutput = &zResult; /* Write results in variable zResult */ -/* Optionally fill in other settings in spec here, as needed */ -zErrMsg = 0; /* Not required; just being pedantic */ -while( zSql && zSql[0] ){ - pStmt = 0; /* Not required; just being pedantic */ - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); - if( rc!=SQLITE_OK ){ - printf("Error: %s\n", sqlite3_errmsg(db)); - }else{ - rc = sqlite3_format_query_result(pStmt, &spec, &zErrMsg); /* Get results */ - if( rc ){ - printf("Error (%d): %s\n", rc, zErrMsg); /* Report an error */ - sqlite3_free(zErrMsg); /* Free the error message text */ - }else{ - printf("%s", zResult); /* Report the results */ - sqlite3_free(zResult); /* Free memory used to hold results */ - zResult = 0; - } - } - sqlite3_finalize(pStmt); -} -~~~ - -<a id="spec"></a> -## 2.0 The `sqlite3_qrf_spec` object - -The `sqlite3_qrf_spec` looks like this: - -> ~~~ -typedef struct sqlite3_qrf_spec sqlite3_qrf_spec; -struct sqlite3_qrf_spec { - unsigned char iVersion; /* Version number of this structure */ - unsigned char eStyle; /* Formatting style. "box", "csv", etc... */ - unsigned char eEsc; /* How to escape control characters in text */ - unsigned char eText; /* Quoting style for text */ - unsigned char eTitle; /* Quating style for the text of column names */ - unsigned char eBlob; /* Quoting style for BLOBs */ - unsigned char bTitles; /* True to show column names */ - unsigned char bWordWrap; /* Try to wrap on word boundaries */ - unsigned char bTextJsonb; /* Render JSONB blobs as JSON text */ - unsigned char eDfltAlign; /* Default alignment, no covered by aAlignment */ - unsigned char eTitleAlign; /* Alignment for column headers */ - unsigned char bSplitColumn; /* Wrap single-column output into many columns */ - unsigned char bBorder; /* Show outer border in Box and Table styles */ - short int nWrap; /* Wrap columns wider than this */ - short int nScreenWidth; /* Maximum overall table width */ - short int nLineLimit; /* Maximum number of lines for any row */ - short int nTitleLimit; /* Maximum number of characters in a title */ - int nCharLimit; /* Maximum number of characters in a cell */ - int nWidth; /* Number of entries in aWidth[] */ - int nAlign; /* Number of entries in aAlignment[] */ - short int *aWidth; /* Column widths */ - unsigned char *aAlign; /* Column alignments */ - char *zColumnSep; /* Alternative column separator */ - char *zRowSep; /* Alternative row separator */ - char *zTableName; /* Output table name */ - char *zNull; /* Rendering of NULL */ - char *(*xRender)(void*,sqlite3_value*); /* Render a value */ - int (*xWrite)(void*,const char*,sqlite3_int64); /* Write output */ - void *pRenderArg; /* First argument to the xRender callback */ - void *pWriteArg; /* First argument to the xWrite callback */ - char **pzOutput; /* Storage location for output string */ - /* Additional fields may be added in the future */ -}; -~~~ - -Do not be alarmed by the complexity of this structure. Everything can -be zeroed except for: - - * `.iVersion` - * One of `.pzOutput` or `.xWrite`. - -You do not need to understand and configure every field of this object -in order to use QRF effectively. Start by zeroing out the whole structure, -then initializing iVersion and one of pzOutput or xWrite. Then maybe -tweak one or two other settings to get the output you want. - -Further detail on the meanings of each of the fields in the -`sqlite3_qrf_spec` object is in the subsequent sections. - -### 2.1 Structure Version Number - -The sqlite3_qrf_spec.iVersion field must be 1. Future enhancements to -the QRF might add new fields to the bottom of the sqlite3_qrf_spec -object. Those new fields will only be accessible if the iVersion is greater -than 1. Thus the iVersion field is used to support upgradability. - -### 2.2 Output Deposition (xWrite and pzOutput) - -The formatted output can either be sent to a callback function -or accumulated into an output buffer in memory obtained -from sqlite3_malloc(). If the sqlite3_qrf_spec.xWrite column is not NULL, -then that function is invoked (using sqlite3_qrf_spec.xWriteArg as its -first argument) to transmit the formatted output. Or, if -sqlite3_qrf_spec.pzOutput points to a pointer to a character, then that -pointer is made to point to memory obtained from sqlite3_malloc() that -contains the complete text of the formatted output. If spec.pzOutput\[0\] -is initially non-NULL, then it is assumed to already point to memory obtained -from sqlite3_malloc(). In that case, the buffer is resized using -sqlite3_realloc() and the new text is appended. - -One of either sqlite3_qrf_spec.xWrite and sqlite3_qrf_spec.pzOutput must be -non-NULL and the other must be NULL. - -The return value from xWrite is an SQLITE result code. The usual return -should be SQLITE_OK. But if for some reason the write fails, a different -value might be returned. - -### 2.3 Output Format - -The sqlite3_qrf_spec.eStyle field is an integer code that defines the -specific output format that will be generated. See [section 4.0](#style) -below for details on the meaning of the various style options. - -Other fields in sqlite3_qrf_spec might be used or might be -ignored, depending on the value of eStyle. - -### 2.4 Show Column Names (bTitles) - -The sqlite3_qrf_spec.bTitles field can be either QRF_SW_Auto, -QRF_SW_On, or QRF_SW_Off. Those three constants also have shorter -alternative spellings: QRF_Auto, QRF_No, and -QRF_Yes. - -> ~~~ -#define QRF_SW_Auto 0 /* Let QRF choose the best value */ -#define QRF_SW_Off 1 /* This setting is forced off */ -#define QRF_SW_On 2 /* This setting is forced on */ -#define QRF_Auto 0 /* Alternate spelling for QRF_SW_Auto and others */ -#define QRF_No 1 /* Alternate spelling for QRF_SW_Off */ -#define QRF_Yes 2 /* Alternate spelling for QRF_SW_On */ -~~~ - -If the value is QRF_Yes, then column names appear in the output. -If the value is QRF_No, column names are omitted. If the -value is QRF_Auto, then an appropriate default is chosen. - -### 2.5 Control Character Escapes (eEsc) - -The sqlite3_qrf_spec.eEsc determines how ASCII control characters are -formatted when displaying TEXT values in the result. These are the allowed -values: - -> ~~~ -#define QRF_ESC_Auto 0 /* Choose the ctrl-char escape automatically */ -#define QRF_ESC_Off 1 /* Do not escape control characters */ -#define QRF_ESC_Ascii 2 /* Unix-style escapes. Ex: U+0007 shows ^G */ -#define QRF_ESC_Symbol 3 /* Unicode escapes. Ex: U+0007 shows U+2407 */ -~~~ - -If the value of eEsc is QRF_ESC_Ascii, then the control character -with value X is displayed as ^Y where Y is X+0x40. Hence, a -backspace character (U+0008) is shown as "^H". - -If eEsc is QRF_ESC_Symbol, then control characters in the range of U+0001 -through U+001f are mapped into U+2401 through U+241f, respectively. - -If the value of eEsc is QRF_ESC_Off, then no translation occurs -and control characters that appear in TEXT strings are transmitted -to the formatted output as-is. This can be dangerous in applications, -since an adversary who can control TEXT values might be able to -inject ANSI cursor movement sequences to hide nefarious values. - -The QRF_ESC_Auto value for eEsc means that the query result formatter -gets to pick whichever control-character encoding it thinks is best for -the situation. This will usually be QRF_ESC_Ascii. - -The TAB (U+0009), LF (U+000a) and CR-LF (U+000d,U+000a) character -sequence are always output literally and are not mapped to alternative -display values, regardless of this setting. - -### 2.6 Display of TEXT values (eText, eTitle) - -The sqlite3_qrf_spec.eText controls how text values are rendered in the -display. sqlite3_qrf_spec.eTitle controls how column names are rendered. -Both fields can have one of the following values: - -> ~~~ -#define QRF_TEXT_Auto 0 /* Choose text encoding automatically */ -#define QRF_TEXT_Plain 1 /* Literal text */ -#define QRF_TEXT_Sql 2 /* Quote as an SQL literal */ -#define QRF_TEXT_Csv 3 /* CSV-style quoting */ -#define QRF_TEXT_Html 4 /* HTML-style quoting */ -#define QRF_TEXT_Tcl 5 /* C/Tcl quoting */ -#define QRF_TEXT_Json 6 /* JSON quoting */ -#define QRF_TEXT_Relaxed 7 /* Relaxed SQL quoting */ -~~~ - -A value of QRF_TEXT_Auto means that the query result formatter will choose -what it thinks will be the best text encoding. - -A value of QRF_TEXT_Plain means that text values appear in the output exactly -as they are found in the database file, with no translation. - -A value of QRF_TEXT_Sql means that text values are escaped so that they -look like SQL literals. That means the value will be surrounded by -single-quotes (U+0027) and any single-quotes contained within the text -will be doubled. - -QRF_TEXT_Relaxed is similar to QRF_TEXT_Sql, except that it automatically -reverts to QRF_TEXT_Plain if the value to be displayed does not contain -special characters and is not easily confused with a NULL or a numeric -value. QRF_TEXT_Relaxed strives to minimize the amount of quoting syntax -while keeping the result unambiguous and easy for humans to read. The -precise rules for when quoting is omitted in QRF_TEXT_Relaxed, and when -it is applied, might be adjusted in future releases. - -A value of QRF_TEXT_Csv means that text values are escaped in accordance -with RFC&nbsp;4180, which defines Comma-Separated-Value or CSV files. -Text strings that contain no special values appears as-is. Text strings -that contain special values are contained in double-quotes (U+0022) and -any double-quotes within the value are doubled. - -A value of QRF_TEXT_Html means that text values are escaped for use in -HTML. Special characters "&lt;", "&amp;", "&gt;", "&quot;", and "&#39;" -are displayed as "&amp;lt;", "&amp;amp;", "&amp;gt;", "&amp;quot;", -and "&amp;#39;", respectively. - -A value of QRF_TEXT_Tcl means that text values are displayed inside of -double-quotes and special characters within the string are escaped using -backslash escape, as in ANSI-C or TCL or Perl or other popular programming -languages. - -A value of QRF_TEXT_Json gives similar results as QRF_TEXT_Tcl except that the -rules are adjusted so that the displayed string is strictly conforming -the JSON specification. - -### 2.7 How to display BLOB values (eBlob and bTextJsonb) - -If the sqlite3_qrf_spec.bTextJsonb flag is QRF_SW_On and if the value to be -displayed is JSONB, then the JSONB is translated into text JSON and the -text is shown according to the sqlite3_qrf_spec.eText setting as -described in the previous section. - -If the bTextJsonb flag is QRF_SW_Off (the usual case) or if the BLOB value to -be displayed is not JSONB, then the sqlite3_qrf_spec.eBlob field determines -how the BLOB value is formatted. The following options are available; - -> ~~~ -#define QRF_BLOB_Auto 0 /* Determine BLOB quoting using eText */ -#define QRF_BLOB_Text 1 /* Display content exactly as it is */ -#define QRF_BLOB_Sql 2 /* Quote as an SQL literal */ -#define QRF_BLOB_Hex 3 /* Hexadecimal representation */ -#define QRF_BLOB_Tcl 4 /* "\000" notation */ -#define QRF_BLOB_Json 5 /* A JSON string */ -#define QRF_BLOB_Size 6 /* Display the blob size only */ -~~~ - -A value of QRF_BLOB_Auto means that display format is selected automatically -by sqlite3_format_query_result() based on eStyle and eText. - -A value of QRF_BLOB_Text means that BLOB values are interpreted as UTF8 -text and are displayed using formatting results set by eEsc and -eText. - -A value of QRF_BLOB_Sql means that BLOB values are shown as SQL BLOB -literals: a prefix "`x'`" following by hexadecimal and ending with a -final "`'`". - -A value of QRF_BLOB_Hex means that BLOB values are shown as -hexadecimal text with no delimiters. - -A value of QRF_BLOB_Tcl means that BLOB values are shown as a -C/Tcl/Perl string literal where every byte is an octal backslash -escape. So a BLOB of `x'052881f3'` would be displayed as -`"\005\050\201\363"`. - -A value of QRF_BLOB_Json is similar to QRF_BLOB_Tcl except that is -uses unicode backslash escapes, since JSON does not understand -the C/Tcl/Perl octal backslash escapes. So the string from the -previous paragraph would be shown as -`"\u0005\u0028\u0081\u00f3"`. - -A value of QRF_BLOB_Size does not show any BLOB content at all. -Instead, it substitutes a text string that says how many bytes -the BLOB contains. - -### 2.8 Maximum size of displayed content (nLineLimit, nCharLimit, nTitleLimit) - -If the sqlite3_qrf_spec.nCharLimit setting is non-zero, then the formatter -will display only the first nCharLimit characters of each value. -Only characters that take up space are counted when enforcing this -limit. Zero-width characters and VT100 escape sequences do not count -toward this limit. The count is in characters, not bytes. When -imposing this limit, the formatter adds the three characters "..." -to the end of the value. Those added characters are not counted -as part of the limit. Very small limits still result in truncation, -but might render a few more characters than the limit. - -If the sqlite3_qrf_spec.nLineLimit setting is non-zero, then the -formatter will only display the first nLineLimit lines of each value. -It does not matter if the value is split because it contains a newline -character, or if it split by wrapping. This setting merely limits -the number of displayed lines. The nLineLimit setting currently only -works for **Box**, **Column**, **Line**, **Markdown**, and **Table** -styles, though that limitation might change in future releases. - -The idea behind both of these settings is to prevent large renderings -when doing a query that (unexpectedly) contains very large text or -blob values: perhaps megabyes of text. - -If the sqlite3_qrf_spec.nTitleLimit is non-zero, then the formatter -attempts to limits the size of column titles to at most nTitleLimit -display characters in width and a single line of text. The nTitleLimit -is useful for queries that have result columns that are scalar -subqueries or complex expressions. If those columns lack an AS -clause, then the name of the column will be a copy of the expression -that defines the column, which in some queries can be hundreds of -characters and multiple lines in length, which can reduce the readability -of tabular displays. An nTitleLimit somewhere in the range of 10 to 20. -can improve readability. The nTitleLimit setting currently only -works for **Box**, **Column**, **Line**, **Markdown**, and **Table** -styles, though that limitation might change in future releases. - -### 2.9 Word Wrapping In Columnar Styles (nWrap, bWordWrap) - -When using columnar formatting modes (QRF_STYLE_Box, QRF_STYLE_Column, -QRF_STYLE_Markdown, or QRF_STYLE_Table), the formatter attempts to limit -the width of any individual column to sqlite3_qrf_spec.nWrap characters -if nWrap is non-zero. A zero value for nWrap means "unlimited". -The nWrap limit might be exceeded if the limit is very small. - -In order to keep individual columns within requested width limits, -it is sometimes necessary to wrap the content for a single row of -a single column across multiple lines. When this -becomes necessary and if the bWordWrap setting is QRF_Yes, then the -formatter attempts to split the content on whitespace or at a word boundary. -If bWordWrap is QRF_No, then the formatter is free to split content -anywhere, including in the middle of a word. - -For narrow columns and wide words, it might sometimes be necessary to split -a column in the middle of a word, even when bWordWrap is QRF_Yes. - -### 2.10 Helping The Output To Fit On The Terminal (nScreenWidth) - -The sqlite3_qrf_spec.nScreenWidth field can be set the number of -characters that will fit on one line on the viewer output device. -This is typically a number like 80 or 132. The formatter will attempt -to reduce the length of output lines, depending on the style, so -that all output fits on that screen. - -A value of zero for nScreenWidth means "unknown" or "no width limit". -When the value is zero, the formatter makes no attempt to keep the -lines of output short. - -The nScreenWidth is a hint to the formatter, not a requirement. -The formatter trieds to keep lines below the nScreenWidth limit, -but it does not guarantee that it will. - -The nScreenWidth field currently only makes a difference in -columnar styles (**Box**, **Column**, **Markdown**, and **Table**) -and in the **Line** style. - -### 2.11 Individual Column Width (nWidth and aWidth) - -The sqlite3_qrf_spec.aWidth field is a pointer to an array of -signed 16-bit integers that control the width of individual columns -in columnar output modes (QRF_STYLE_Box, QRF_STYLE_Column, -QRF_STYLE_Markdown, or QRF_STYLE_Table). The sqlite3_qrf_spec.nWidth -field is the number of integers in the aWidth array. - -If aWidth is a NULL pointer or if nWidth is zero, then the array is -assumed to be all zeros. If nWidth is less then the number of -columns in the output, then zero is used for the width -for all columns past then end of the aWidth array. - -The aWidth array is deliberately an array of 16-bit signed integers. -Only 16 bits are used because no good comes for having very large -column widths. The range if further restricted as follows: - -> ~~~ -#define QRF_MAX_WIDTH 10000 /* Maximum column width */ -#define QRF_MIN_WIDTH 0 /* Minimum column width */ -~~~ - -A width greater than then QRF_MAX_WIDTH is interpreted as QRF_MAX_WIDTH. - -Any aWidth\[\] value of zero means the formatter should use a flexible -width column (limited only by sqlite_qrf_spec.mxWidth) that is just -big enough to hold the largest row. - -For historical compatibility, aWidth\[\] can contain negative values, -down to -QRF_MAX_WIDTH. The column width used is the absolute value -of the number in aWidth\[\]. The only difference is that negative -values cause the default horizontal alignment to be QRF_ALIGN_Right. -The sign of the aWidth\[\] values only affects alignment if the -alignment is not otherwise specified by aAlign\[\] or eDfltAlign. -Again, negative values for aWidth\[\] entries are supported for -backwards compatibility only, and are not recommended for new -applications. - -### 2.12 Alignment (nAlignment, aAlignment, eDfltAlign, eTitleAlign) - -Some cells in a display table might contain a lot of text and thus -be wide, or they might contain newline characters or be wrapped by -width constraints so that they span many rows of text. Other cells -might be narrower and shorter. In columnar formats, the display width -of a cell is the maximum of the widest value in the same column, and the -display height is the height of the tallest value in the same row. -So some cells might be much taller and wider than necessary to hold -their values. - -Alignment determines where smaller values are placed within larger cells. - -The sqlite3_qrf_spec.aAlign field points to an array of unsigned characters -that specifies alignment (both vertical and horizontal) of individual -columns within the table. The sqlite3_qrf_spec.nAlign fields holds -the number of entries in the aAlign\[\] array. - -If sqlite3_qrf_spec.aAlign is a NULL pointer or if sqlite3_qrf_spec.nAlign -is zero, or for columns to the right of what are specified by -sqlite3_qrf_spec.nAlign, the sqlite3_qrf_spec.eDfltAlign value is used -for the alignment. Column names can be (and often are) aligned -differently, as specified by sqlite3_qrf_spec.eTitleAlign. - -Each alignment value specifies both vertical and horizontal alignment. -Horizontal alignment can be left, center, right, or no preference. -Vertical alignment can be top, middle, bottom, or no preference. -Thus there are 16 possible alignment values, as follows: - -> ~~~ -/* -** Horizontal Vertial -** ---------- -------- */ -#define QRF_ALIGN_Auto 0 /* auto auto */ -#define QRF_ALIGN_Left 1 /* left auto */ -#define QRF_ALIGN_Center 2 /* center auto */ -#define QRF_ALIGN_Right 3 /* right auto */ -#define QRF_ALIGN_Top 4 /* auto top */ -#define QRF_ALIGN_NW 5 /* left top */ -#define QRF_ALIGN_N 6 /* center top */ -#define QRF_ALIGN_NE 7 /* right top */ -#define QRF_ALIGN_Middle 8 /* auto middle */ -#define QRF_ALIGN_W 9 /* left middle */ -#define QRF_ALIGN_C 10 /* center middle */ -#define QRF_ALIGN_E 11 /* right middle */ -#define QRF_ALIGN_Bottom 12 /* auto bottom */ -#define QRF_ALIGN_SW 13 /* left bottom */ -#define QRF_ALIGN_S 14 /* center bottom */ -#define QRF_ALIGN_SE 15 /* right bottom */ -~~~ - -Notice how alignment values with an unspecified horizontal -or vertical component can be added to another alignment value -for which that component is specified, to get a fully -specified alignment. For eample: - -> QRF_ALIGN_Center + QRF_ALIGN_Bottom == QRF_ALIGN_S. - -The alignment for column names is always determined by the -eTitleAlign setting. If eTitleAlign is QRF_Auto, then column -names use center-bottom alignment, QRF_ALIGN_W, value 14. -The aAlign\[\] and eDfltAlign settings have no affect on -column names. - -For data in the first nAlign columns, the aAlign\[\] array -entry for that column takes precedence. If either the horizontal -or vertical alignment has an "auto" value for that column or if -a column is beyond the first nAlign entries, then eDfltAlign -is used as a backup. If neither aAlign\[\] nor eDfltAlign -specify a horizontal alignment, then values are right-aligned -(QRF_ALIGN_Right) if they are numeric and left-aligned -(QRF_ALIGN_Left) otherwise. If neither aAlign\[\] nor eDfltAlign -specify a vertical alignment, then values are top-aligned -(QRF_ALIGN_Top). - -*As of 2025-11-08, only horizontal alignment is implemented. -The vertical alignment settings are currently ignored and -the vertical alignment is always QRF_ALIGN_Top.* - -### 2.13 Row and Column Separator Strings - -The sqlite3_qrf_spec.zColumnSep and sqlite3_qrf_spec.zRowSep strings -are alternative column and row separator character sequences. If not -specified (if these pointers are left as NULL) then appropriate defaults -are used. Some output styles have hard-coded column and row separators -and these settings are ignored for those styles. - -### 2.14 The Output Table Name - -The sqlite3_qrf_spec.zTableName value is the name of the output table -when eStyle is QRF_STYLE_Insert. - -### 2.15 The Rendering Of NULL (zNull) - -If a value is NULL then show the NULL using the string -found in sqlite3_qrf_spec.zNull. If zNull is itself a NULL pointer -then NULL values are rendered as an empty string. - -### 2.16 Optional Value Rendering Callback - -If the sqlite3_qrf_spec.xRender field is not NULL, then each -sqlite3_value coming out of the query is first passed to the -xRender function, giving that function an opportunity to render -the results itself, using whatever custom format is desired. -If xRender chooses to render, it should write the rendering -into memory obtained from sqlite3_malloc() and return a pointer -to that memory. The xRender function can decline -to render (for example, based on the sqlite3_value_type() or other -characteristics of the value) in which case it can simply return a -NULL pointer and the usual default rendering will be used instead. - -The sqlite3_format_query_result() function (which calls xRender) -will take responsibility for freeing the string returned by xRender -after it has finished using it. - -The eText, eBlob, and eEsc settings above become no-ops if the xRender -routine returns non-NULL. In other words, the application-supplied -xRender routine is expected to do all of its own quoting and formatting. - -The xRender routine is expected to do character length limiting itself. -So the nCharLimit setting becomes a no-op if xRender is used. However -the nLineLimit setting is still applied. The nTitleLimit setting is -not applicable to xRender because title values come from the -sqlite3_column_name() interface not from sqlite3_column_value(), -and so that names of columns are never processed by xRender. - -## 3.0 The `sqlite3_format_query_result()` Interface - -Invoke the `sqlite3_format_query_result(P,S,E)` interface to run -the prepared statement P and format its results according to the -specification found in S. The sqlite3_format_query_result() function -will return an SQLite result code, usually SQLITE_OK, but perhaps -SQLITE_NOMEM or SQLITE_ERROR or similar. If an error occurs and if -the E parameter is not NULL, then error message text might be written -into *E. Any error message text will be stored in memory obtained -from sqlite3_malloc() and it is the responsibility of the caller to -free that memory by a subsequent call to sqlite3_free(). - -<a id="style"></a> -## 4.0 Output Styles - -The result formatter supports a variety of output styles. The -output style (sometimes called "output mode") is determined by -the eStyle field of the sqlite3_qrf_spec object. The set of -supported output modes might increase in future versions. -The following output modes are currently defined: - -> ~~~ -#define QRF_STYLE_Auto 0 /* Choose a style automatically */ -#define QRF_STYLE_Box 1 /* Unicode box-drawing characters */ -#define QRF_STYLE_Column 2 /* One record per line in neat columns */ -#define QRF_STYLE_Count 3 /* Output only a count of the rows of output */ -#define QRF_STYLE_Csv 4 /* Comma-separated-value */ -#define QRF_STYLE_Eqp 5 /* Format EXPLAIN QUERY PLAN output */ -#define QRF_STYLE_Explain 6 /* EXPLAIN output */ -#define QRF_STYLE_Html 7 /* Generate an XHTML table */ -#define QRF_STYLE_Insert 8 /* Generate SQL "insert" statements */ -#define QRF_STYLE_Json 9 /* Output is a list of JSON objects */ -#define QRF_STYLE_JObject 10 /* Independent JSON objects for each row */ -#define QRF_STYLE_Line 11 /* One column per line. */ -#define QRF_STYLE_List 12 /* One record per line with a separator */ -#define QRF_STYLE_Markdown 13 /* Markdown formatting */ -#define QRF_STYLE_Off 14 /* No query output shown */ -#define QRF_STYLE_Quote 15 /* SQL-quoted, comma-separated */ -#define QRF_STYLE_Stats 16 /* EQP-like output but with performance stats */ -#define QRF_STYLE_StatsEst 17 /* EQP-like output with planner estimates */ -#define QRF_STYLE_StatsVm 18 /* EXPLAIN-like output with performance stats */ -#define QRF_STYLE_Table 19 /* MySQL-style table formatting */ -~~~ - -In the following subsections, these styles will often be referred -to without the "QRF_STYLE_" prefix. - -### 4.1 Default Style (Auto) - -The **Auto** style means QRF gets to choose an appropriate output -style. It will usually choose **Box**, but might also pick one of -**Explain** or **Eqp** if the `sqlite3_stmt_explain()` function -returns 1 or 2, respectively. - -### 4.2 Columnar Styles (Box, Column, Markdown, Table) - -The **Box**, **Column**, **Markdown**, and **Table** -modes are columnar. This means the output is arranged into neat, -uniform-width columns. These styles can use more memory, especially when -the query result has many rows, because they need to load the entire output -into memory first in order to determine how wide to make each column. - -The nWidth, aWidth, and mxWidth fields of the `sqlite3_qrf_spec` object -are used by these styles only, and are ignored by all other styles. -The zRowSep and zColumnSep settings are ignored by these styles. The -bTitles setting is honored by these styles; it defaults to QRF_SW_On. - -The **Box** style uses Unicode box-drawing character to draw a grid -of columns and rows to show the result. The **Table** is the same, -except that it uses ASCII-art rather than Unicode box-drawing characters -to draw the grid. The **Column** arranges the results in neat columns -but does not draw in column or row separator, except that it does draw -lines horizontal lines using "`-`" characters to separate the column names -from the data below. This is very similar to default output styling in -psql. The **Markdown** renders its result in the Markdown table format. - -The **Box** and **Table** styles normally have a border that surrounds -the entire result. However, if sqlite3_qrf_spec.bBorder is QRF_No, then -that border is omitted, saving a little space both horizontally and -vertically. - -#### 4.2.1 Split Column Mode - -If the bSplitColumn field is QRF_Yes, and eStyle is QRF_STYLE_Column, -and bTitles is QRF_No, and nScreenWidth is greater than zero, and if -the query only returns a single column, then a special rendering known -as "Split Column Mode" will be used. In split column mode, instead -of showing all results in one tall column, the content wraps vertically -so that it appears on the screen as multiple columns, as many as will -fit in the available screen width. - -### 4.3 Line-oriented Styles - -The line-oriented styles output each row of result as it is received from -the prepared statement. - -The **List** style is the most familiar line-oriented output format. -The **List** style shows output columns for each row on the -same line, each separated by a single "`|`" character and with lines -terminated by a single newline (\\u000a or \\n). These column -and row separator choices can be overridden using the zColumnSep -and zRowSep fields of the `sqlite3_qrf_spec` structure. The text -formatting is QRF_TEXT_Plain, and BLOB encoding is QRF_BLOB_Text. So -characters appear in the output exactly as they appear in the database. -Except the eEsp mode defaults to `QRF_ESC_On`, so that control -characters are escaped, for safety. - -The **Csv** and **Quote** styles are simply variations on **List** -with hard-coded values for some of the sqlite3_qrf_spec settings: - -<table border=1 cellpadding=2 cellspacing=0> -<tr><th>&nbsp;<th>Quote<th>Csv -<tr><td>zColumnSep<td>","<td>"," -<tr><td>zRowSep<td>"\\n"<td>"\\r\\n" -<tr><td>zNull<td>"NULL"<td>"" -<tr><td>eText<td>QRF_TEXT_Sql<td>QRF_TEXT_Csv -<tr><td>eBlob<td>QRF_BLOB_Sql<td>QRF_BLOB_Text -</table> - -The **Html** style generates HTML table content, just without -the `<TABLE>..</TABLE>` around the outside. - -The **Insert** style generates a series of SQL "INSERT" statements -that will inserts the data that is output into a table whose name is defined -by the zTableName field of `sqlite3_qrf_spec`. If zTableName is NULL, -then a substitute name is used. - -The **Json** and **JObject** styles generates JSON text for the query result. -The **Json** style produces a JSON array of structures with one -structure per row. **JObject** outputs independent JSON objects, one per -row, with each structure on a separate line all by itself, and not -part of a larger array. In both cases, the labels on the elements of the -JSON objects are taken from the column names of the SQL query. So if -you have an SQL query that has two or more output columns with the same -name, you will end up with JSON structures that have duplicate elements. - -Finally, the **Line** style paints each column of a row on a -separate line with the column name on the left and a "`=`" separating the -column name from its value. A single blank line appears between rows. - -### 4.4 EXPLAIN Styles (Eqp, Explain) - -The **Eqp** and **Explain** styles format output for -EXPLAIN QUERY PLAN and EXPLAIN statements, respectively. If the input -statement is not already an EXPLAIN QUERY PLAN or EXPLAIN statement is -is temporarily converted for the duration of the rendering, but -is converted back before `sqlite3_format_query_result()` returns. - -### 4.5 ScanStatus Styles (Stats, StatsEst, StatsVm) - -The **Stats**, **StatsEst**, and **StatsVm** styles are similar to **Eqp** -and **Explain** except that they include profiling information -from prior executions of the input prepared statement. -These modes only work if SQLite has been compiled with --DSQLITE_ENABLE_STMT_SCANSTATUS and if the SQLITE_DBCONFIG_STMT_SCANSTATUS -is enabled for the database connection. The **StatsVm** style -also requires the bytecode() virtual table which is enabled using -the -DSQLITE_ENABLE_BYTECODE_VTAB compile-time option. - -### 4.6 Other Styles (Count, Off) - -The **Count** style discards all query results and returns -a count of the number of rows of output at the end. The **Off** -style is completely silent; it generates no output. These corner-case -modes are sometimes useful for debugging. - -### 5.0 Source Code Files - -The SQLite Query Result Formatter is implemented in three source code files: - - * `qrf.c` &rarr; The implementation, written in portable C99 - * `qrf.h` &rarr; A header file defining interfaces - * `README.md` &rarr; This documentation - -To use the SQLite result formatter, include the "`qrf.h`" header file -and link the application against the "`qrf.c`" source file. diff --git a/ext/qrf/dev-notes.md b/ext/qrf/dev-notes.md deleted file mode 100644 index a46aada83..000000000 --- a/ext/qrf/dev-notes.md +++ /dev/null @@ -1,14 +0,0 @@ -# Developer Notes - -## Measuring Test Coverage On Linux - -On Mint Linux, as of 2025-12-02: - -> ~~~ -./configure --dev CFLAGS='-O0 -g -fprofile-arcs -ftest-coverage' -make clean testfixture -./testfixture test/qrf*.test -gcov -b -c testfixture-tclsqlite-ex.c -~~~ - -View results in tclsqlite-ex.c.gcov diff --git a/ext/qrf/qrf.c b/ext/qrf/qrf.c deleted file mode 100644 index cacfa1526..000000000 --- a/ext/qrf/qrf.c +++ /dev/null @@ -1,2983 +0,0 @@ -/* -** 2025-10-20 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** Implementation of the Result-Format or "qrf" utility library for SQLite. -** See the qrf.md documentation for additional information. -*/ -#ifndef SQLITE_QRF_H -#include "qrf.h" -#endif -#include <string.h> -#include <assert.h> -#include <stdint.h> - -typedef sqlite3_int64 i64; - -/* A single line in the EQP output */ -typedef struct qrfEQPGraphRow qrfEQPGraphRow; -struct qrfEQPGraphRow { - int iEqpId; /* ID for this row */ - int iParentId; /* ID of the parent row */ - qrfEQPGraphRow *pNext; /* Next row in sequence */ - char zText[1]; /* Text to display for this row */ -}; - -/* All EQP output is collected into an instance of the following */ -typedef struct qrfEQPGraph qrfEQPGraph; -struct qrfEQPGraph { - qrfEQPGraphRow *pRow; /* Linked list of all rows of the EQP output */ - qrfEQPGraphRow *pLast; /* Last element of the pRow list */ - int nWidth; /* Width of the graph */ - char zPrefix[400]; /* Graph prefix */ -}; - -/* -** Private state information. Subject to change from one release to the -** next. -*/ -typedef struct Qrf Qrf; -struct Qrf { - sqlite3_stmt *pStmt; /* The statement whose output is to be rendered */ - sqlite3 *db; /* The corresponding database connection */ - sqlite3_stmt *pJTrans; /* JSONB to JSON translator statement */ - char **pzErr; /* Write error message here, if not NULL */ - sqlite3_str *pOut; /* Accumulated output */ - int iErr; /* Error code */ - int nCol; /* Number of output columns */ - int expMode; /* Original sqlite3_stmt_isexplain() plus 1 */ - int mxWidth; /* Screen width */ - int mxHeight; /* nLineLimit */ - union { - struct { /* Content for QRF_STYLE_Line */ - int mxColWth; /* Maximum display width of any column */ - char **azCol; /* Names of output columns (MODE_Line) */ - } sLine; - qrfEQPGraph *pGraph; /* EQP graph (Eqp, Stats, and StatsEst) */ - struct { /* Content for QRF_STYLE_Explain */ - int nIndent; /* Slots allocated for aiIndent */ - int iIndent; /* Current slot */ - int *aiIndent; /* Indentation for each opcode */ - } sExpln; - } u; - sqlite3_int64 nRow; /* Number of rows handled so far */ - int *actualWidth; /* Actual width of each column */ - sqlite3_qrf_spec spec; /* Copy of the original spec */ -}; - -/* -** Data for substitute ctype.h functions. Used for x-platform -** consistency and so that '_' is counted as an alphabetic -** character. -** -** 0x01 - space -** 0x02 - digit -** 0x04 - alphabetic, including '_' -*/ -static const char qrfCType[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, - 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, - 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 -}; -#define qrfSpace(x) ((qrfCType[(unsigned char)x]&1)!=0) -#define qrfDigit(x) ((qrfCType[(unsigned char)x]&2)!=0) -#define qrfAlpha(x) ((qrfCType[(unsigned char)x]&4)!=0) -#define qrfAlnum(x) ((qrfCType[(unsigned char)x]&6)!=0) - -#ifndef deliberate_fall_through -/* Quiet some compilers about some of our intentional code. */ -# if defined(GCC_VERSION) && GCC_VERSION>=7000000 -# define deliberate_fall_through __attribute__((fallthrough)); -# else -# define deliberate_fall_through -# endif -#endif - -/* -** Set an error code and error message. -*/ -static void qrfError( - Qrf *p, /* Query result state */ - int iCode, /* Error code */ - const char *zFormat, /* Message format (or NULL) */ - ... -){ - p->iErr = iCode; - if( p->pzErr!=0 ){ - sqlite3_free(*p->pzErr); - *p->pzErr = 0; - if( zFormat ){ - va_list ap; - va_start(ap, zFormat); - *p->pzErr = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - } - } -} - -/* -** Out-of-memory error. -*/ -static void qrfOom(Qrf *p){ - qrfError(p, SQLITE_NOMEM, "out of memory"); -} - -/* -** Transfer any error in pStr over into p. -*/ -static void qrfStrErr(Qrf *p, sqlite3_str *pStr){ - int rc = pStr ? sqlite3_str_errcode(pStr) : 0; - if( rc ){ - qrfError(p, rc, sqlite3_errstr(rc)); - } -} - - -/* -** Add a new entry to the EXPLAIN QUERY PLAN data -*/ -static void qrfEqpAppend(Qrf *p, int iEqpId, int p2, const char *zText){ - qrfEQPGraphRow *pNew; - sqlite3_int64 nText; - if( zText==0 ) return; - if( p->u.pGraph==0 ){ - p->u.pGraph = sqlite3_malloc64( sizeof(qrfEQPGraph) ); - if( p->u.pGraph==0 ){ - qrfOom(p); - return; - } - memset(p->u.pGraph, 0, sizeof(qrfEQPGraph) ); - } - nText = strlen(zText); - pNew = sqlite3_malloc64( sizeof(*pNew) + nText ); - if( pNew==0 ){ - qrfOom(p); - return; - } - pNew->iEqpId = iEqpId; - pNew->iParentId = p2; - memcpy(pNew->zText, zText, nText+1); - pNew->pNext = 0; - if( p->u.pGraph->pLast ){ - p->u.pGraph->pLast->pNext = pNew; - }else{ - p->u.pGraph->pRow = pNew; - } - p->u.pGraph->pLast = pNew; -} - -/* -** Free and reset the EXPLAIN QUERY PLAN data that has been collected -** in p->u.pGraph. -*/ -static void qrfEqpReset(Qrf *p){ - qrfEQPGraphRow *pRow, *pNext; - if( p->u.pGraph ){ - for(pRow = p->u.pGraph->pRow; pRow; pRow = pNext){ - pNext = pRow->pNext; - sqlite3_free(pRow); - } - sqlite3_free(p->u.pGraph); - p->u.pGraph = 0; - } -} - -/* Return the next EXPLAIN QUERY PLAN line with iEqpId that occurs after -** pOld, or return the first such line if pOld is NULL -*/ -static qrfEQPGraphRow *qrfEqpNextRow(Qrf *p, int iEqpId, qrfEQPGraphRow *pOld){ - qrfEQPGraphRow *pRow = pOld ? pOld->pNext : p->u.pGraph->pRow; - while( pRow && pRow->iParentId!=iEqpId ) pRow = pRow->pNext; - return pRow; -} - -/* Render a single level of the graph that has iEqpId as its parent. Called -** recursively to render sublevels. -*/ -static void qrfEqpRenderLevel(Qrf *p, int iEqpId){ - qrfEQPGraphRow *pRow, *pNext; - i64 n = strlen(p->u.pGraph->zPrefix); - char *z; - for(pRow = qrfEqpNextRow(p, iEqpId, 0); pRow; pRow = pNext){ - pNext = qrfEqpNextRow(p, iEqpId, pRow); - z = pRow->zText; - sqlite3_str_appendf(p->pOut, "%s%s%s\n", p->u.pGraph->zPrefix, - pNext ? "|--" : "`--", z); - if( n<(i64)sizeof(p->u.pGraph->zPrefix)-7 ){ - memcpy(&p->u.pGraph->zPrefix[n], pNext ? "| " : " ", 4); - qrfEqpRenderLevel(p, pRow->iEqpId); - p->u.pGraph->zPrefix[n] = 0; - } - } -} - -/* -** Render the 64-bit value N in a more human-readable format into -** pOut. -** -** + Only show the first three significant digits. -** + Append suffixes K, M, G, T, P, and E for 1e3, 1e6, ... 1e18 -*/ -static void qrfApproxInt64(sqlite3_str *pOut, i64 N){ - static const char aSuffix[] = { 'K', 'M', 'G', 'T', 'P', 'E' }; - int i; - if( N<0 ){ - N = N==INT64_MIN ? INT64_MAX : -N; - sqlite3_str_append(pOut, "-", 1); - } - if( N<10000 ){ - sqlite3_str_appendf(pOut, "%4lld ", N); - return; - } - for(i=1; i<=18; i++){ - N = (N+5)/10; - if( N<10000 ){ - int n = (int)N; - switch( i%3 ){ - case 0: - sqlite3_str_appendf(pOut, "%d.%02d", n/1000, (n%1000)/10); - break; - case 1: - sqlite3_str_appendf(pOut, "%2d.%d", n/100, (n%100)/10); - break; - case 2: - sqlite3_str_appendf(pOut, "%4d", n/10); - break; - } - sqlite3_str_append(pOut, &aSuffix[i/3], 1); - break; - } - } -} - -/* -** Display and reset the EXPLAIN QUERY PLAN data -*/ -static void qrfEqpRender(Qrf *p, i64 nCycle){ - qrfEQPGraphRow *pRow; - if( p->u.pGraph!=0 && (pRow = p->u.pGraph->pRow)!=0 ){ - if( pRow->zText[0]=='-' ){ - if( pRow->pNext==0 ){ - qrfEqpReset(p); - return; - } - sqlite3_str_appendf(p->pOut, "%s\n", pRow->zText+3); - p->u.pGraph->pRow = pRow->pNext; - sqlite3_free(pRow); - }else if( nCycle>0 ){ - int nSp = p->u.pGraph->nWidth - 2; - if( p->spec.eStyle==QRF_STYLE_StatsEst ){ - sqlite3_str_appendchar(p->pOut, nSp, ' '); - sqlite3_str_appendall(p->pOut, - "Cycles Loops (est) Rows (est)\n"); - sqlite3_str_appendchar(p->pOut, nSp, ' '); - sqlite3_str_appendall(p->pOut, - "---------- ------------ ------------\n"); - }else{ - sqlite3_str_appendchar(p->pOut, nSp, ' '); - sqlite3_str_appendall(p->pOut, - "Cycles Loops Rows \n"); - sqlite3_str_appendchar(p->pOut, nSp, ' '); - sqlite3_str_appendall(p->pOut, - "---------- ----- -----\n"); - } - sqlite3_str_appendall(p->pOut, "QUERY PLAN"); - sqlite3_str_appendchar(p->pOut, nSp - 10, ' '); - qrfApproxInt64(p->pOut, nCycle); - sqlite3_str_appendall(p->pOut, " 100%\n"); - }else{ - sqlite3_str_appendall(p->pOut, "QUERY PLAN\n"); - } - p->u.pGraph->zPrefix[0] = 0; - qrfEqpRenderLevel(p, 0); - qrfEqpReset(p); - } -} - -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS -/* -** Helper function for qrfExpStats(). -** -*/ -static int qrfStatsHeight(sqlite3_stmt *p, int iEntry){ - int iPid = 0; - int ret = 1; - sqlite3_stmt_scanstatus_v2(p, iEntry, - SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid - ); - while( iPid!=0 ){ - int ii; - for(ii=0; 1; ii++){ - int iId; - int res; - res = sqlite3_stmt_scanstatus_v2(p, ii, - SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iId - ); - if( res ) break; - if( iId==iPid ){ - sqlite3_stmt_scanstatus_v2(p, ii, - SQLITE_SCANSTAT_PARENTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid - ); - } - } - ret++; - } - return ret; -} -#endif /* SQLITE_ENABLE_STMT_SCANSTATUS */ - - -/* -** Generate ".scanstatus est" style of EQP output. -*/ -static void qrfEqpStats(Qrf *p){ -#ifndef SQLITE_ENABLE_STMT_SCANSTATUS - qrfError(p, SQLITE_ERROR, "not available in this build"); -#else - static const int f = SQLITE_SCANSTAT_COMPLEX; - sqlite3_stmt *pS = p->pStmt; - int i = 0; - i64 nTotal = 0; - int nWidth = 0; - int prevPid = -1; /* Previous iPid */ - double rEstCum = 1.0; /* Cumulative row estimate */ - sqlite3_str *pLine = sqlite3_str_new(p->db); - sqlite3_str *pStats = sqlite3_str_new(p->db); - qrfEqpReset(p); - - for(i=0; 1; i++){ - const char *z = 0; - int n = 0; - if( sqlite3_stmt_scanstatus_v2(pS,i,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&z) ){ - break; - } - n = (int)strlen(z) + qrfStatsHeight(pS,i)*3; - if( n>nWidth ) nWidth = n; - } - nWidth += 2; - - sqlite3_stmt_scanstatus_v2(pS,-1, SQLITE_SCANSTAT_NCYCLE, f, (void*)&nTotal); - for(i=0; 1; i++){ - i64 nLoop = 0; - i64 nRow = 0; - i64 nCycle = 0; - int iId = 0; - int iPid = 0; - const char *zo = 0; - const char *zName = 0; - double rEst = 0.0; - - if( sqlite3_stmt_scanstatus_v2(pS,i,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&zo) ){ - break; - } - sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_PARENTID,f,(void*)&iPid); - if( iPid!=prevPid ){ - prevPid = iPid; - rEstCum = 1.0; - } - sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_EST,f,(void*)&rEst); - rEstCum *= rEst; - sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_NLOOP,f,(void*)&nLoop); - sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_NVISIT,f,(void*)&nRow); - sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_NCYCLE,f,(void*)&nCycle); - sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_SELECTID,f,(void*)&iId); - sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_NAME,f,(void*)&zName); - - if( nCycle>=0 || nLoop>=0 || nRow>=0 ){ - int nSp = 0; - sqlite3_str_reset(pStats); - if( nCycle>=0 && nTotal>0 ){ - qrfApproxInt64(pStats, nCycle); - sqlite3_str_appendf(pStats, " %3d%%", - ((nCycle*100)+nTotal/2) / nTotal - ); - nSp = 2; - } - if( nLoop>=0 ){ - if( nSp ) sqlite3_str_appendchar(pStats, nSp, ' '); - qrfApproxInt64(pStats, nLoop); - nSp = 2; - if( p->spec.eStyle==QRF_STYLE_StatsEst ){ - sqlite3_str_appendf(pStats, " "); - qrfApproxInt64(pStats, (i64)(rEstCum/rEst)); - } - } - if( nRow>=0 ){ - if( nSp ) sqlite3_str_appendchar(pStats, nSp, ' '); - qrfApproxInt64(pStats, nRow); - nSp = 2; - if( p->spec.eStyle==QRF_STYLE_StatsEst ){ - sqlite3_str_appendf(pStats, " "); - qrfApproxInt64(pStats, (i64)rEstCum); - } - } - sqlite3_str_appendf(pLine, - "% *s %s", -1*(nWidth-qrfStatsHeight(pS,i)*3), zo, - sqlite3_str_value(pStats) - ); - sqlite3_str_reset(pStats); - qrfEqpAppend(p, iId, iPid, sqlite3_str_value(pLine)); - sqlite3_str_reset(pLine); - }else{ - qrfEqpAppend(p, iId, iPid, zo); - } - } - if( p->u.pGraph ) p->u.pGraph->nWidth = nWidth; - qrfStrErr(p, pLine); - sqlite3_free(sqlite3_str_finish(pLine)); - qrfStrErr(p, pStats); - sqlite3_free(sqlite3_str_finish(pStats)); -#endif -} - - -/* -** Reset the prepared statement. -*/ -static void qrfResetStmt(Qrf *p){ - int rc = sqlite3_reset(p->pStmt); - if( rc!=SQLITE_OK && p->iErr==SQLITE_OK ){ - qrfError(p, rc, "%s", sqlite3_errmsg(p->db)); - } -} - -/* -** If xWrite is defined, send all content of pOut to xWrite and -** reset pOut. -*/ -static void qrfWrite(Qrf *p){ - int n; - if( p->spec.xWrite && (n = sqlite3_str_length(p->pOut))>0 ){ - int rc = p->spec.xWrite(p->spec.pWriteArg, - sqlite3_str_value(p->pOut), - (sqlite3_int64)n); - sqlite3_str_reset(p->pOut); - if( rc ){ - qrfError(p, rc, "Failed to write %d bytes of output", n); - } - } -} - -/* Lookup table to estimate the number of columns consumed by a Unicode -** character. -*/ -static const struct { - unsigned char w; /* Width of the character in columns */ - int iFirst; /* First character in a span having this width */ -} aQrfUWidth[] = { - /* {1, 0x00000}, */ - {0, 0x00300}, {1, 0x00370}, {0, 0x00483}, {1, 0x00487}, {0, 0x00488}, - {1, 0x0048a}, {0, 0x00591}, {1, 0x005be}, {0, 0x005bf}, {1, 0x005c0}, - {0, 0x005c1}, {1, 0x005c3}, {0, 0x005c4}, {1, 0x005c6}, {0, 0x005c7}, - {1, 0x005c8}, {0, 0x00600}, {1, 0x00604}, {0, 0x00610}, {1, 0x00616}, - {0, 0x0064b}, {1, 0x0065f}, {0, 0x00670}, {1, 0x00671}, {0, 0x006d6}, - {1, 0x006e5}, {0, 0x006e7}, {1, 0x006e9}, {0, 0x006ea}, {1, 0x006ee}, - {0, 0x0070f}, {1, 0x00710}, {0, 0x00711}, {1, 0x00712}, {0, 0x00730}, - {1, 0x0074b}, {0, 0x007a6}, {1, 0x007b1}, {0, 0x007eb}, {1, 0x007f4}, - {0, 0x00901}, {1, 0x00903}, {0, 0x0093c}, {1, 0x0093d}, {0, 0x00941}, - {1, 0x00949}, {0, 0x0094d}, {1, 0x0094e}, {0, 0x00951}, {1, 0x00955}, - {0, 0x00962}, {1, 0x00964}, {0, 0x00981}, {1, 0x00982}, {0, 0x009bc}, - {1, 0x009bd}, {0, 0x009c1}, {1, 0x009c5}, {0, 0x009cd}, {1, 0x009ce}, - {0, 0x009e2}, {1, 0x009e4}, {0, 0x00a01}, {1, 0x00a03}, {0, 0x00a3c}, - {1, 0x00a3d}, {0, 0x00a41}, {1, 0x00a43}, {0, 0x00a47}, {1, 0x00a49}, - {0, 0x00a4b}, {1, 0x00a4e}, {0, 0x00a70}, {1, 0x00a72}, {0, 0x00a81}, - {1, 0x00a83}, {0, 0x00abc}, {1, 0x00abd}, {0, 0x00ac1}, {1, 0x00ac6}, - {0, 0x00ac7}, {1, 0x00ac9}, {0, 0x00acd}, {1, 0x00ace}, {0, 0x00ae2}, - {1, 0x00ae4}, {0, 0x00b01}, {1, 0x00b02}, {0, 0x00b3c}, {1, 0x00b3d}, - {0, 0x00b3f}, {1, 0x00b40}, {0, 0x00b41}, {1, 0x00b44}, {0, 0x00b4d}, - {1, 0x00b4e}, {0, 0x00b56}, {1, 0x00b57}, {0, 0x00b82}, {1, 0x00b83}, - {0, 0x00bc0}, {1, 0x00bc1}, {0, 0x00bcd}, {1, 0x00bce}, {0, 0x00c3e}, - {1, 0x00c41}, {0, 0x00c46}, {1, 0x00c49}, {0, 0x00c4a}, {1, 0x00c4e}, - {0, 0x00c55}, {1, 0x00c57}, {0, 0x00cbc}, {1, 0x00cbd}, {0, 0x00cbf}, - {1, 0x00cc0}, {0, 0x00cc6}, {1, 0x00cc7}, {0, 0x00ccc}, {1, 0x00cce}, - {0, 0x00ce2}, {1, 0x00ce4}, {0, 0x00d41}, {1, 0x00d44}, {0, 0x00d4d}, - {1, 0x00d4e}, {0, 0x00dca}, {1, 0x00dcb}, {0, 0x00dd2}, {1, 0x00dd5}, - {0, 0x00dd6}, {1, 0x00dd7}, {0, 0x00e31}, {1, 0x00e32}, {0, 0x00e34}, - {1, 0x00e3b}, {0, 0x00e47}, {1, 0x00e4f}, {0, 0x00eb1}, {1, 0x00eb2}, - {0, 0x00eb4}, {1, 0x00eba}, {0, 0x00ebb}, {1, 0x00ebd}, {0, 0x00ec8}, - {1, 0x00ece}, {0, 0x00f18}, {1, 0x00f1a}, {0, 0x00f35}, {1, 0x00f36}, - {0, 0x00f37}, {1, 0x00f38}, {0, 0x00f39}, {1, 0x00f3a}, {0, 0x00f71}, - {1, 0x00f7f}, {0, 0x00f80}, {1, 0x00f85}, {0, 0x00f86}, {1, 0x00f88}, - {0, 0x00f90}, {1, 0x00f98}, {0, 0x00f99}, {1, 0x00fbd}, {0, 0x00fc6}, - {1, 0x00fc7}, {0, 0x0102d}, {1, 0x01031}, {0, 0x01032}, {1, 0x01033}, - {0, 0x01036}, {1, 0x0103b}, {0, 0x01058}, - {1, 0x0105a}, {2, 0x01100}, {0, 0x01160}, {1, 0x01200}, {0, 0x0135f}, - {1, 0x01360}, {0, 0x01712}, {1, 0x01715}, {0, 0x01732}, {1, 0x01735}, - {0, 0x01752}, {1, 0x01754}, {0, 0x01772}, {1, 0x01774}, {0, 0x017b4}, - {1, 0x017b6}, {0, 0x017b7}, {1, 0x017be}, {0, 0x017c6}, {1, 0x017c7}, - {0, 0x017c9}, {1, 0x017d4}, {0, 0x017dd}, {1, 0x017de}, {0, 0x0180b}, - {1, 0x0180e}, {0, 0x018a9}, {1, 0x018aa}, {0, 0x01920}, {1, 0x01923}, - {0, 0x01927}, {1, 0x01929}, {0, 0x01932}, {1, 0x01933}, {0, 0x01939}, - {1, 0x0193c}, {0, 0x01a17}, {1, 0x01a19}, {0, 0x01b00}, {1, 0x01b04}, - {0, 0x01b34}, {1, 0x01b35}, {0, 0x01b36}, {1, 0x01b3b}, {0, 0x01b3c}, - {1, 0x01b3d}, {0, 0x01b42}, {1, 0x01b43}, {0, 0x01b6b}, {1, 0x01b74}, - {0, 0x01dc0}, {1, 0x01dcb}, {0, 0x01dfe}, {1, 0x01e00}, {0, 0x0200b}, - {1, 0x02010}, {0, 0x0202a}, {1, 0x0202f}, {0, 0x02060}, {1, 0x02064}, - {0, 0x0206a}, {1, 0x02070}, {0, 0x020d0}, {1, 0x020f0}, {2, 0x02329}, - {1, 0x0232b}, {2, 0x02e80}, {0, 0x0302a}, {2, 0x03030}, {1, 0x0303f}, - {2, 0x03040}, {0, 0x03099}, {2, 0x0309b}, {1, 0x0a4d0}, {0, 0x0a806}, - {1, 0x0a807}, {0, 0x0a80b}, {1, 0x0a80c}, {0, 0x0a825}, {1, 0x0a827}, - {2, 0x0ac00}, {1, 0x0d7a4}, {2, 0x0f900}, {1, 0x0fb00}, {0, 0x0fb1e}, - {1, 0x0fb1f}, {0, 0x0fe00}, {2, 0x0fe10}, {1, 0x0fe1a}, {0, 0x0fe20}, - {1, 0x0fe24}, {2, 0x0fe30}, {1, 0x0fe70}, {0, 0x0feff}, {2, 0x0ff00}, - {1, 0x0ff61}, {2, 0x0ffe0}, {1, 0x0ffe7}, {0, 0x0fff9}, {1, 0x0fffc}, - {0, 0x10a01}, {1, 0x10a04}, {0, 0x10a05}, {1, 0x10a07}, {0, 0x10a0c}, - {1, 0x10a10}, {0, 0x10a38}, {1, 0x10a3b}, {0, 0x10a3f}, {1, 0x10a40}, - {0, 0x1d167}, {1, 0x1d16a}, {0, 0x1d173}, {1, 0x1d183}, {0, 0x1d185}, - {1, 0x1d18c}, {0, 0x1d1aa}, {1, 0x1d1ae}, {0, 0x1d242}, {1, 0x1d245}, - {2, 0x20000}, {1, 0x2fffe}, {2, 0x30000}, {1, 0x3fffe}, {0, 0xe0001}, - {1, 0xe0002}, {0, 0xe0020}, {1, 0xe0080}, {0, 0xe0100}, {1, 0xe01f0} -}; - -/* -** Return an estimate of the width, in columns, for the single Unicode -** character c. For normal characters, the answer is always 1. But the -** estimate might be 0 or 2 for zero-width and double-width characters. -** -** Different display devices display unicode using different widths. So -** it is impossible to know that true display width with 100% accuracy. -** Inaccuracies in the width estimates might cause columns to be misaligned. -** Unfortunately, there is nothing we can do about that. -*/ -int sqlite3_qrf_wcwidth(int c){ - int iFirst, iLast; - - /* Fast path for common characters */ - if( c<0x300 ) return 1; - - /* The general case */ - iFirst = 0; - iLast = sizeof(aQrfUWidth)/sizeof(aQrfUWidth[0]) - 1; - while( iFirst<iLast-1 ){ - int iMid = (iFirst+iLast)/2; - int cMid = aQrfUWidth[iMid].iFirst; - if( cMid < c ){ - iFirst = iMid; - }else if( cMid > c ){ - iLast = iMid - 1; - }else{ - return aQrfUWidth[iMid].w; - } - } - if( aQrfUWidth[iLast].iFirst > c ) return aQrfUWidth[iFirst].w; - return aQrfUWidth[iLast].w; -} - -/* -** Compute the value and length of a multi-byte UTF-8 character that -** begins at z[0]. Return the length. Write the Unicode value into *pU. -** -** This routine only works for *multi-byte* UTF-8 characters. It does -** not attempt to detect illegal characters. -*/ -int sqlite3_qrf_decode_utf8(const unsigned char *z, int *pU){ - if( (z[0] & 0xe0)==0xc0 && (z[1] & 0xc0)==0x80 ){ - *pU = ((z[0] & 0x1f)<<6) | (z[1] & 0x3f); - return 2; - } - if( (z[0] & 0xf0)==0xe0 && (z[1] & 0xc0)==0x80 && (z[2] & 0xc0)==0x80 ){ - *pU = ((z[0] & 0x0f)<<12) | ((z[1] & 0x3f)<<6) | (z[2] & 0x3f); - return 3; - } - if( (z[0] & 0xf8)==0xf0 && (z[1] & 0xc0)==0x80 && (z[2] & 0xc0)==0x80 - && (z[3] & 0xc0)==0x80 - ){ - *pU = ((z[0] & 0x0f)<<18) | ((z[1] & 0x3f)<<12) | ((z[2] & 0x3f))<<6 - | (z[3] & 0x3f); - return 4; - } - *pU = 0; - return 1; -} - -/* -** Check to see if z[] is a valid VT100 escape. If it is, then -** return the number of bytes in the escape sequence. Return 0 if -** z[] is not a VT100 escape. -** -** This routine assumes that z[0] is \033 (ESC). -*/ -static int qrfIsVt100(const unsigned char *z){ - int i; - if( z[1]!='[' ) return 0; - i = 2; - while( z[i]>=0x30 && z[i]<=0x3f ){ i++; } - while( z[i]>=0x20 && z[i]<=0x2f ){ i++; } - if( z[i]<0x40 || z[i]>0x7e ) return 0; - return i+1; -} - -/* -** Return the length of a string in display characters. -** -** Most characters of the input string count as 1, including -** multi-byte UTF8 characters. However, zero-width unicode -** characters and VT100 escape sequences count as zero, and -** double-width characters count as two. -** -** The definition of "zero-width" and "double-width" characters -** is not precise. It depends on the output device, to some extent, -** and it varies according to the Unicode version. This routine -** makes the best guess that it can. -*/ -size_t sqlite3_qrf_wcswidth(const char *zIn){ - const unsigned char *z = (const unsigned char*)zIn; - size_t n = 0; - while( *z ){ - if( z[0]<' ' ){ - int k; - if( z[0]=='\033' && (k = qrfIsVt100(z))>0 ){ - z += k; - }else{ - z++; - } - }else if( (0x80&z[0])==0 ){ - n++; - z++; - }else{ - int u = 0; - int len = sqlite3_qrf_decode_utf8(z, &u); - z += len; - n += sqlite3_qrf_wcwidth(u); - } - } - return n; -} - -/* -** Return the display width of the longest line of text -** in the (possibly) multi-line input string zIn[0..nByte]. -** zIn[] is not necessarily zero-terminated. Take -** into account tab characters, zero- and double-width -** characters, CR and NL, and VT100 escape codes. -** -** Write the number of newlines into *pnNL. So, *pnNL will -** return 0 if everything fits on one line, or positive it -** it will need to be split. -*/ -static int qrfDisplayWidth(const char *zIn, sqlite3_int64 nByte, int *pnNL){ - const unsigned char *z; - const unsigned char *zEnd; - int mx = 0; - int n = 0; - int nNL = 0; - if( zIn==0 ) zIn = ""; - z = (const unsigned char*)zIn; - zEnd = &z[nByte]; - while( z<zEnd ){ - if( z[0]<' ' ){ - int k; - if( z[0]=='\033' && (k = qrfIsVt100(z))>0 ){ - z += k; - }else{ - if( z[0]=='\t' ){ - n = (n+8)&~7; - }else if( z[0]=='\n' || z[0]=='\r' ){ - nNL++; - if( n>mx ) mx = n; - n = 0; - } - z++; - } - }else if( (0x80&z[0])==0 ){ - n++; - z++; - }else{ - int u = 0; - int len = sqlite3_qrf_decode_utf8(z, &u); - z += len; - n += sqlite3_qrf_wcwidth(u); - } - } - if( mx>n ) n = mx; - if( pnNL ) *pnNL = nNL; - return n; -} - -/* -** Escape the input string if it is needed and in accordance with -** eEsc, which is either QRF_ESC_Ascii or QRF_ESC_Symbol. -** -** Escaping is needed if the string contains any control characters -** other than \t, \n, and \r\n -** -** If no escaping is needed (the common case) then set *ppOut to NULL -** and return 0. If escaping is needed, write the escaped string into -** memory obtained from sqlite3_malloc64() and make *ppOut point to that -** memory and return 0. If an error occurs, return non-zero. -** -** The caller is responsible for freeing *ppFree if it is non-NULL in order -** to reclaim memory. -*/ -static void qrfEscape( - int eEsc, /* QRF_ESC_Ascii or QRF_ESC_Symbol */ - sqlite3_str *pStr, /* String to be escaped */ - int iStart /* Begin escapding on this byte of pStr */ -){ - sqlite3_int64 i, j; /* Loop counters */ - sqlite3_int64 sz; /* Size of the string prior to escaping */ - sqlite3_int64 nCtrl = 0;/* Number of control characters to escape */ - unsigned char *zIn; /* Text to be escaped */ - unsigned char c; /* A single character of the text */ - unsigned char *zOut; /* Where to write the results */ - - /* Find the text to be escaped */ - zIn = (unsigned char*)sqlite3_str_value(pStr); - if( zIn==0 ) return; - zIn += iStart; - - /* Count the control characters */ - for(i=0; (c = zIn[i])!=0; i++){ - if( c<=0x1f - && c!='\t' - && c!='\n' - && (c!='\r' || zIn[i+1]!='\n') - ){ - nCtrl++; - } - } - if( nCtrl==0 ) return; /* Early out if no control characters */ - - /* Make space to hold the escapes. Copy the original text to the end - ** of the available space. */ - sz = sqlite3_str_length(pStr) - iStart; - if( eEsc==QRF_ESC_Symbol ) nCtrl *= 2; - sqlite3_str_appendchar(pStr, nCtrl, ' '); - zOut = (unsigned char*)sqlite3_str_value(pStr); - if( zOut==0 ) return; - zOut += iStart; - zIn = zOut + nCtrl; - memmove(zIn,zOut,sz); - - /* Convert the control characters */ - for(i=j=0; (c = zIn[i])!=0; i++){ - if( c>0x1f - || c=='\t' - || c=='\n' - || (c=='\r' && zIn[i+1]=='\n') - ){ - continue; - } - if( i>0 ){ - memmove(&zOut[j], zIn, i); - j += i; - } - zIn += i+1; - i = -1; - if( eEsc==QRF_ESC_Symbol ){ - zOut[j++] = 0xe2; - zOut[j++] = 0x90; - zOut[j++] = 0x80+c; - }else{ - zOut[j++] = '^'; - zOut[j++] = 0x40+c; - } - } -} - -/* -** Determine if the string z[] can be shown as plain text. Return true -** if z[] is unambiguously text. Return false if z[] needs to be -** quoted. -** -** All of the following must be true in order for z[] to be relaxable: -** -** (1) z[] does not begin or end with ' or whitespace -** (2) z[] is not the same as the NULL rendering -** (3) z[] does not looks like a numeric literal -*/ -static int qrfRelaxable(Qrf *p, const char *z){ - size_t i, n; - if( z[0]=='\'' || qrfSpace(z[0]) ) return 0; - if( z[0]==0 ){ - return (p->spec.zNull!=0 && p->spec.zNull[0]!=0); - } - n = strlen(z); - if( n==0 || z[n-1]=='\'' || qrfSpace(z[n-1]) ) return 0; - if( p->spec.zNull && strcmp(p->spec.zNull,z)==0 ) return 0; - i = (z[0]=='-' || z[0]=='+'); - if( strcmp(z+i,"Inf")==0 ) return 0; - if( !qrfDigit(z[i]) ) return 1; - i++; - while( qrfDigit(z[i]) ){ i++; } - if( z[i]==0 ) return 0; - if( z[i]=='.' ){ - i++; - while( qrfDigit(z[i]) ){ i++; } - if( z[i]==0 ) return 0; - } - if( z[i]=='e' || z[i]=='E' ){ - i++; - if( z[i]=='+' || z[i]=='-' ){ i++; } - if( !qrfDigit(z[i]) ) return 1; - i++; - while( qrfDigit(z[i]) ){ i++; } - } - return z[i]!=0; -} - -/* -** If a field contains any character identified by a 1 in the following -** array, then the string must be quoted for CSV. -*/ -static const char qrfCsvQuote[] = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -}; - -/* -** Encode text appropriately and append it to pOut. -*/ -static void qrfEncodeText(Qrf *p, sqlite3_str *pOut, const char *zTxt){ - int iStart = sqlite3_str_length(pOut); - switch( p->spec.eText ){ - case QRF_TEXT_Relaxed: - if( qrfRelaxable(p, zTxt) ){ - sqlite3_str_appendall(pOut, zTxt); - break; - } - deliberate_fall_through; /* FALLTHRU */ - case QRF_TEXT_Sql: { - if( p->spec.eEsc==QRF_ESC_Off ){ - sqlite3_str_appendf(pOut, "%Q", zTxt); - }else{ - sqlite3_str_appendf(pOut, "%#Q", zTxt); - } - break; - } - case QRF_TEXT_Csv: { - unsigned int i; - for(i=0; zTxt[i]; i++){ - if( qrfCsvQuote[((const unsigned char*)zTxt)[i]] ){ - i = 0; - break; - } - } - if( i==0 || strstr(zTxt, p->spec.zColumnSep)!=0 ){ - sqlite3_str_appendf(pOut, "\"%w\"", zTxt); - }else{ - sqlite3_str_appendall(pOut, zTxt); - } - break; - } - case QRF_TEXT_Html: { - const unsigned char *z = (const unsigned char*)zTxt; - while( *z ){ - unsigned int i = 0; - unsigned char c; - while( (c=z[i])>'>' - || (c && c!='<' && c!='>' && c!='&' && c!='\"' && c!='\'') - ){ - i++; - } - if( i>0 ){ - sqlite3_str_append(pOut, (const char*)z, i); - } - switch( z[i] ){ - case '>': sqlite3_str_append(pOut, "&lt;", 4); break; - case '&': sqlite3_str_append(pOut, "&amp;", 5); break; - case '<': sqlite3_str_append(pOut, "&lt;", 4); break; - case '"': sqlite3_str_append(pOut, "&quot;", 6); break; - case '\'': sqlite3_str_append(pOut, "&#39;", 5); break; - default: i--; - } - z += i + 1; - } - break; - } - case QRF_TEXT_Tcl: - case QRF_TEXT_Json: { - const unsigned char *z = (const unsigned char*)zTxt; - sqlite3_str_append(pOut, "\"", 1); - while( *z ){ - unsigned int i; - for(i=0; z[i]>=0x20 && z[i]!='\\' && z[i]!='"'; i++){} - if( i>0 ){ - sqlite3_str_append(pOut, (const char*)z, i); - } - if( z[i]==0 ) break; - switch( z[i] ){ - case '"': sqlite3_str_append(pOut, "\\\"", 2); break; - case '\\': sqlite3_str_append(pOut, "\\\\", 2); break; - case '\b': sqlite3_str_append(pOut, "\\b", 2); break; - case '\f': sqlite3_str_append(pOut, "\\f", 2); break; - case '\n': sqlite3_str_append(pOut, "\\n", 2); break; - case '\r': sqlite3_str_append(pOut, "\\r", 2); break; - case '\t': sqlite3_str_append(pOut, "\\t", 2); break; - default: { - if( p->spec.eText==QRF_TEXT_Json ){ - sqlite3_str_appendf(pOut, "\\u%04x", z[i]); - }else{ - sqlite3_str_appendf(pOut, "\\%03o", z[i]); - } - break; - } - } - z += i + 1; - } - sqlite3_str_append(pOut, "\"", 1); - break; - } - default: { - sqlite3_str_appendall(pOut, zTxt); - break; - } - } - if( p->spec.eEsc!=QRF_ESC_Off ){ - qrfEscape(p->spec.eEsc, pOut, iStart); - } -} - -/* -** Do a quick sanity check to see aBlob[0..nBlob-1] is valid JSONB -** return true if it is and false if it is not. -** -** False positives are possible, but not false negatives. -*/ -static int qrfJsonbQuickCheck(unsigned char *aBlob, int nBlob){ - unsigned char x; /* Payload size half-byte */ - int i; /* Loop counter */ - int n; /* Bytes in the payload size integer */ - sqlite3_uint64 sz; /* value of the payload size integer */ - - if( nBlob==0 ) return 0; - x = aBlob[0]>>4; - if( x<=11 ) return nBlob==(1+x); - n = x<14 ? x-11 : 4*(x-13); - if( nBlob<1+n ) return 0; - sz = aBlob[1]; - for(i=1; i<n; i++) sz = (sz<<8) + aBlob[i+1]; - return sz+n+1==(sqlite3_uint64)nBlob; -} - -/* -** The current iCol-th column of p->pStmt is known to be a BLOB. Check -** to see if that BLOB is really a JSONB blob. If it is, then translate -** it into a text JSON representation and return a pointer to that text JSON. -** If the BLOB is not JSONB, then return a NULL pointer. -** -** The memory used to hold the JSON text is managed internally by the -** "p" object and is overwritten and/or deallocated upon the next call -** to this routine (with the same p argument) or when the p object is -** finailized. -*/ -static const char *qrfJsonbToJson(Qrf *p, int iCol){ - int nByte; - const void *pBlob; - int rc; - nByte = sqlite3_column_bytes(p->pStmt, iCol); - pBlob = sqlite3_column_blob(p->pStmt, iCol); - if( qrfJsonbQuickCheck((unsigned char*)pBlob, nByte)==0 ){ - return 0; - } - if( p->pJTrans==0 ){ - sqlite3 *db; - rc = sqlite3_open(":memory:",&db); - if( rc ){ - sqlite3_close(db); - return 0; - } - rc = sqlite3_prepare_v2(db, "SELECT json(?1)", -1, &p->pJTrans, 0); - if( rc ){ - sqlite3_finalize(p->pJTrans); - p->pJTrans = 0; - sqlite3_close(db); - return 0; - } - }else{ - sqlite3_reset(p->pJTrans); - } - sqlite3_bind_blob(p->pJTrans, 1, (void*)pBlob, nByte, SQLITE_STATIC); - rc = sqlite3_step(p->pJTrans); - if( rc==SQLITE_ROW ){ - return (const char*)sqlite3_column_text(p->pJTrans, 0); - }else{ - return 0; - } -} - -/* -** Adjust the input string zIn[] such that it is no more than N display -** characters wide. If it is wider than that, then truncate and add -** ellipsis. Or if zIn[] contains a \r or \n, truncate at that point, -** adding ellipsis. Embedded tabs in zIn[] are converted into ordinary -** spaces. -** -** Return this display width of the modified title string. -*/ -static int qrfTitleLimit(char *zIn, int N){ - unsigned char *z = (unsigned char*)zIn; - int n = 0; - unsigned char *zEllipsis = 0; - while( 1 /*exit-by-break*/ ){ - if( z[0]<' ' ){ - int k; - if( z[0]==0 ){ - zEllipsis = 0; - break; - }else if( z[0]=='\033' && (k = qrfIsVt100(z))>0 ){ - z += k; - }else if( z[0]=='\t' ){ - z[0] = ' '; - }else if( z[0]=='\n' || z[0]=='\r' ){ - z[0] = ' '; - }else{ - z++; - } - }else if( (0x80&z[0])==0 ){ - if( n>=(N-3) && zEllipsis==0 ) zEllipsis = z; - if( n==N ){ z[0] = 0; break; } - n++; - z++; - }else{ - int u = 0; - int len = sqlite3_qrf_decode_utf8(z, &u); - if( n+len>(N-3) && zEllipsis==0 ) zEllipsis = z; - if( n+len>N ){ z[0] = 0; break; } - z += len; - n += sqlite3_qrf_wcwidth(u); - } - } - if( zEllipsis && N>=3 ) memcpy(zEllipsis,"...",4); - return n; -} - - -/* -** Render value pVal into pOut -*/ -static void qrfRenderValue(Qrf *p, sqlite3_str *pOut, int iCol){ -#if SQLITE_VERSION_NUMBER>=3052000 - int iStartLen = sqlite3_str_length(pOut); -#endif - if( p->spec.xRender ){ - sqlite3_value *pVal; - char *z; - pVal = sqlite3_value_dup(sqlite3_column_value(p->pStmt,iCol)); - z = p->spec.xRender(p->spec.pRenderArg, pVal); - sqlite3_value_free(pVal); - if( z ){ - sqlite3_str_appendall(pOut, z); - sqlite3_free(z); - return; - } - } - switch( sqlite3_column_type(p->pStmt,iCol) ){ - case SQLITE_INTEGER: { - sqlite3_str_appendf(pOut, "%lld", sqlite3_column_int64(p->pStmt,iCol)); - break; - } - case SQLITE_FLOAT: { - const char *zTxt = (const char*)sqlite3_column_text(p->pStmt,iCol); - sqlite3_str_appendall(pOut, zTxt); - break; - } - case SQLITE_BLOB: { - if( p->spec.bTextJsonb==QRF_Yes ){ - const char *zJson = qrfJsonbToJson(p, iCol); - if( zJson ){ - if( p->spec.eText==QRF_TEXT_Sql ){ - sqlite3_str_append(pOut,"jsonb(",6); - qrfEncodeText(p, pOut, zJson); - sqlite3_str_append(pOut,")",1); - }else{ - qrfEncodeText(p, pOut, zJson); - } - break; - } - } - switch( p->spec.eBlob ){ - case QRF_BLOB_Hex: - case QRF_BLOB_Sql: { - int iStart; - int nBlob = sqlite3_column_bytes(p->pStmt,iCol); - int i, j; - char *zVal; - const unsigned char *a = sqlite3_column_blob(p->pStmt,iCol); - if( p->spec.eBlob==QRF_BLOB_Sql ){ - sqlite3_str_append(pOut, "x'", 2); - } - iStart = sqlite3_str_length(pOut); - sqlite3_str_appendchar(pOut, nBlob, ' '); - sqlite3_str_appendchar(pOut, nBlob, ' '); - if( p->spec.eBlob==QRF_BLOB_Sql ){ - sqlite3_str_appendchar(pOut, 1, '\''); - } - if( sqlite3_str_errcode(pOut) ) return; - zVal = sqlite3_str_value(pOut); - for(i=0, j=iStart; i<nBlob; i++, j+=2){ - unsigned char c = a[i]; - zVal[j] = "0123456789abcdef"[(c>>4)&0xf]; - zVal[j+1] = "0123456789abcdef"[(c)&0xf]; - } - break; - } - case QRF_BLOB_Tcl: - case QRF_BLOB_Json: { - int iStart; - int nBlob = sqlite3_column_bytes(p->pStmt,iCol); - int i, j; - char *zVal; - const unsigned char *a = sqlite3_column_blob(p->pStmt,iCol); - int szC = p->spec.eBlob==QRF_BLOB_Json ? 6 : 4; - sqlite3_str_append(pOut, "\"", 1); - iStart = sqlite3_str_length(pOut); - for(i=szC; i>0; i--){ - sqlite3_str_appendchar(pOut, nBlob, ' '); - } - sqlite3_str_appendchar(pOut, 1, '"'); - if( sqlite3_str_errcode(pOut) ) return; - zVal = sqlite3_str_value(pOut); - for(i=0, j=iStart; i<nBlob; i++, j+=szC){ - unsigned char c = a[i]; - zVal[j] = '\\'; - if( szC==4 ){ - zVal[j+1] = '0' + ((c>>6)&3); - zVal[j+2] = '0' + ((c>>3)&7); - zVal[j+3] = '0' + (c&7); - }else{ - zVal[j+1] = 'u'; - zVal[j+2] = '0'; - zVal[j+3] = '0'; - zVal[j+4] = "0123456789abcdef"[(c>>4)&0xf]; - zVal[j+5] = "0123456789abcdef"[(c)&0xf]; - } - } - break; - } - case QRF_BLOB_Size: { - int nBlob = sqlite3_column_bytes(p->pStmt,iCol); - sqlite3_str_appendf(pOut, "(%d-byte blob)", nBlob); - break; - } - default: { - const char *zTxt = (const char*)sqlite3_column_text(p->pStmt,iCol); - qrfEncodeText(p, pOut, zTxt); - } - } - break; - } - case SQLITE_NULL: { - sqlite3_str_appendall(pOut, p->spec.zNull); - break; - } - case SQLITE_TEXT: { - const char *zTxt = (const char*)sqlite3_column_text(p->pStmt,iCol); - qrfEncodeText(p, pOut, zTxt); - break; - } - } -#if SQLITE_VERSION_NUMBER>=3052000 - if( p->spec.nCharLimit>0 - && (sqlite3_str_length(pOut) - iStartLen) > p->spec.nCharLimit - ){ - const unsigned char *z; - int ii = 0, w = 0, limit = p->spec.nCharLimit; - z = (const unsigned char*)sqlite3_str_value(pOut) + iStartLen; - if( limit<4 ) limit = 4; - while( 1 ){ - if( z[ii]<' ' ){ - int k; - if( z[ii]=='\033' && (k = qrfIsVt100(z+ii))>0 ){ - ii += k; - }else if( z[ii]==0 ){ - break; - }else{ - ii++; - } - }else if( (0x80&z[ii])==0 ){ - w++; - if( w>limit ) break; - ii++; - }else{ - int u = 0; - int len = sqlite3_qrf_decode_utf8(&z[ii], &u); - w += sqlite3_qrf_wcwidth(u); - if( w>limit ) break; - ii += len; - } - } - if( w>limit ){ - sqlite3_str_truncate(pOut, iStartLen+ii); - sqlite3_str_append(pOut, "...", 3); - } - } -#endif -} - -/* Trim spaces of the end if pOut -*/ -static void qrfRTrim(sqlite3_str *pOut){ -#if SQLITE_VERSION_NUMBER>=3052000 - int nByte = sqlite3_str_length(pOut); - const char *zOut = sqlite3_str_value(pOut); - while( nByte>0 && zOut[nByte-1]==' ' ){ nByte--; } - sqlite3_str_truncate(pOut, nByte); -#endif -} - -/* -** Store string zUtf to pOut as w characters. If w is negative, -** then right-justify the text. W is the width in display characters, not -** in bytes. Double-width unicode characters count as two characters. -** VT100 escape sequences count as zero. And so forth. -*/ -static void qrfWidthPrint(Qrf *p, sqlite3_str *pOut, int w, const char *zUtf){ - const unsigned char *a = (const unsigned char*)zUtf; - static const int mxW = 10000000; - unsigned char c; - int i = 0; - int n = 0; - int k; - int aw; - (void)p; - if( w<-mxW ){ - w = -mxW; - }else if( w>mxW ){ - w= mxW; - } - aw = w<0 ? -w : w; - if( a==0 ) a = (const unsigned char*)""; - while( (c = a[i])!=0 ){ - if( (c&0xc0)==0xc0 ){ - int u; - int len = sqlite3_qrf_decode_utf8(a+i, &u); - int x = sqlite3_qrf_wcwidth(u); - if( x+n>aw ){ - break; - } - i += len; - n += x; - }else if( c==0x1b && (k = qrfIsVt100(&a[i]))>0 ){ - i += k; - }else if( n>=aw ){ - break; - }else{ - n++; - i++; - } - } - if( n>=aw ){ - sqlite3_str_append(pOut, zUtf, i); - }else if( w<0 ){ - if( aw>n ) sqlite3_str_appendchar(pOut, aw-n, ' '); - sqlite3_str_append(pOut, zUtf, i); - }else{ - sqlite3_str_append(pOut, zUtf, i); - if( aw>n ) sqlite3_str_appendchar(pOut, aw-n, ' '); - } -} - -/* -** (*pz)[] is a line of text that is to be displayed the box or table or -** similar tabular formats. z[] contain newlines or might be too wide -** to fit in the columns so will need to be split into multiple line. -** -** This routine determines: -** -** * How many bytes of z[] should be shown on the current line. -** * How many character positions those bytes will cover. -** * The byte offset to the start of the next line. -*/ -static void qrfWrapLine( - const char *zIn, /* Input text to be displayed */ - int w, /* Column width in characters (not bytes) */ - int bWrap, /* True if we should do word-wrapping */ - int *pnThis, /* OUT: How many bytes of z[] for the current line */ - int *pnWide, /* OUT: How wide is the text of this line */ - int *piNext /* OUT: Offset into z[] to start of the next line */ -){ - int i; /* Input bytes consumed */ - int k; /* Bytes in a VT100 code */ - int n; /* Output column number */ - const unsigned char *z = (const unsigned char*)zIn; - unsigned char c = 0; - - if( z[0]==0 ){ - *pnThis = 0; - *pnWide = 0; - *piNext = 0; - return; - } - n = 0; - for(i=0; n<=w; i++){ - c = z[i]; - if( c>=0xc0 ){ - int u; - int len = sqlite3_qrf_decode_utf8(&z[i], &u); - int wcw = sqlite3_qrf_wcwidth(u); - if( wcw+n>w ) break; - i += len-1; - n += wcw; - continue; - } - if( c>=' ' ){ - if( n==w ) break; - n++; - continue; - } - if( c==0 || c=='\n' ) break; - if( c=='\r' && z[i+1]=='\n' ){ c = z[++i]; break; } - if( c=='\t' ){ - int wcw = 8 - (n&7); - if( n+wcw>w ) break; - n += wcw; - continue; - } - if( c==0x1b && (k = qrfIsVt100(&z[i]))>0 ){ - i += k-1; - }else if( n==w ){ - break; - }else{ - n++; - } - } - if( c==0 ){ - *pnThis = i; - *pnWide = n; - *piNext = i; - return; - } - if( c=='\n' ){ - *pnThis = i; - *pnWide = n; - *piNext = i+1; - return; - } - - /* If we get this far, that means the current line will end at some - ** point that is neither a "\n" or a 0x00. Figure out where that - ** split should occur - */ - if( bWrap && z[i]!=0 && !qrfSpace(z[i]) && qrfAlnum(c)==qrfAlnum(z[i]) ){ - /* Perhaps try to back up to a better place to break the line */ - for(k=i-1; k>=i/2; k--){ - if( qrfSpace(z[k]) ) break; - } - if( k<i/2 ){ - for(k=i; k>=i/2; k--){ - if( qrfAlnum(z[k-1])!=qrfAlnum(z[k]) && (z[k]&0xc0)!=0x80 ) break; - } - } - if( k>=i/2 ){ - i = k; - n = qrfDisplayWidth((const char*)z, k, 0); - } - } - *pnThis = i; - *pnWide = n; - while( zIn[i]==' ' || zIn[i]=='\t' || zIn[i]=='\r' ){ i++; } - *piNext = i; -} - -/* -** Append nVal bytes of text from zVal onto the end of pOut. -** Convert tab characters in zVal to the appropriate number of -** spaces. -*/ -static void qrfAppendWithTabs( - sqlite3_str *pOut, /* Append text here */ - const char *zVal, /* Text to append */ - int nVal /* Use only the first nVal bytes of zVal[] */ -){ - int i = 0; - unsigned int col = 0; - unsigned char *z = (unsigned char *)zVal; - while( i<nVal ){ - unsigned char c = z[i]; - if( c<' ' ){ - int k; - sqlite3_str_append(pOut, (const char*)z, i); - nVal -= i; - z += i; - i = 0; - if( c=='\033' && (k = qrfIsVt100(z))>0 ){ - sqlite3_str_append(pOut, (const char*)z, k); - z += k; - nVal -= k; - }else if( c=='\t' ){ - k = 8 - (col&7); - sqlite3_str_appendchar(pOut, k, ' '); - col += k; - z++; - nVal--; - }else if( c=='\r' && nVal==1 ){ - z++; - nVal--; - }else{ - char zCtrlPik[4]; - col++; - zCtrlPik[0] = 0xe2; - zCtrlPik[1] = 0x90; - zCtrlPik[2] = 0x80+c; - sqlite3_str_append(pOut, zCtrlPik, 3); - z++; - nVal--; - } - }else if( (0x80&c)==0 ){ - i++; - col++; - }else{ - int u = 0; - int len = sqlite3_qrf_decode_utf8(&z[i], &u); - i += len; - col += sqlite3_qrf_wcwidth(u); - } - } - sqlite3_str_append(pOut, (const char*)z, i); -} - -/* -** GCC does not define the offsetof() macro so we'll have to do it -** ourselves. -*/ -#ifndef offsetof -# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) -#endif - -/* -** Data for columnar layout, collected into a single object so -** that it can be more easily passed into subroutines. -*/ -typedef struct qrfColData qrfColData; -struct qrfColData { - Qrf *p; /* The QRF instance */ - int nCol; /* Number of columns in the table */ - unsigned char bMultiRow; /* One or more cells will span multiple lines */ - unsigned char nMargin; /* Width of column margins */ - sqlite3_int64 nRow; /* Number of rows */ - sqlite3_int64 nAlloc; /* Number of cells allocated */ - sqlite3_int64 n; /* Number of cells. nCol*nRow */ - char **az; /* Content of all cells */ - int *aiWth; /* Width of each cell */ - unsigned char *abNum; /* True for each numeric cell */ - struct qrfPerCol { /* Per-column data */ - char *z; /* Cache of text for current row */ - int w; /* Computed width of this column */ - int mxW; /* Maximum natural (unwrapped) width */ - unsigned char e; /* Alignment */ - unsigned char fx; /* Width is fixed */ - unsigned char bNum; /* True if is numeric */ - } *a; /* One per column */ -}; - -/* -** Output horizontally justified text into pOut. The text is the -** first nVal bytes of zVal. Include nWS bytes of whitespace, either -** split between both sides, or on the left, or on the right, depending -** on eAlign. -*/ -static void qrfPrintAligned( - sqlite3_str *pOut, /* Append text here */ - struct qrfPerCol *pCol, /* Information about the text to print */ - int nVal, /* Use only the first nVal bytes of zVal[] */ - int nWS /* Whitespace for horizonal alignment */ -){ - unsigned char eAlign = pCol->e & QRF_ALIGN_HMASK; - if( eAlign==QRF_Auto && pCol->bNum ) eAlign = QRF_ALIGN_Right; - if( eAlign==QRF_ALIGN_Center ){ - /* Center the text */ - sqlite3_str_appendchar(pOut, nWS/2, ' '); - qrfAppendWithTabs(pOut, pCol->z, nVal); - sqlite3_str_appendchar(pOut, nWS - nWS/2, ' '); - }else if( eAlign==QRF_ALIGN_Right ){ - /* Right justify the text */ - sqlite3_str_appendchar(pOut, nWS, ' '); - qrfAppendWithTabs(pOut, pCol->z, nVal); - }else{ - /* Left justify the text */ - qrfAppendWithTabs(pOut, pCol->z, nVal); - sqlite3_str_appendchar(pOut, nWS, ' '); - } -} - -/* -** Free all the memory allocates in the qrfColData object -*/ -static void qrfColDataFree(qrfColData *p){ - sqlite3_int64 i; - for(i=0; i<p->n; i++) sqlite3_free(p->az[i]); - sqlite3_free(p->az); - sqlite3_free(p->aiWth); - sqlite3_free(p->abNum); - sqlite3_free(p->a); - memset(p, 0, sizeof(*p)); -} - -/* -** Allocate space for more cells in the qrfColData object. -** Return non-zero if a memory allocation fails. -*/ -static int qrfColDataEnlarge(qrfColData *p){ - char **azData; - int *aiWth; - unsigned char *abNum; - p->nAlloc = 2*p->nAlloc + 10*p->nCol; - azData = sqlite3_realloc64(p->az, p->nAlloc*sizeof(char*)); - if( azData==0 ){ - qrfOom(p->p); - qrfColDataFree(p); - return 1; - } - p->az = azData; - aiWth = sqlite3_realloc64(p->aiWth, p->nAlloc*sizeof(int)); - if( aiWth==0 ){ - qrfOom(p->p); - qrfColDataFree(p); - return 1; - } - p->aiWth = aiWth; - abNum = sqlite3_realloc64(p->abNum, p->nAlloc); - if( abNum==0 ){ - qrfOom(p->p); - qrfColDataFree(p); - return 1; - } - p->abNum = abNum; - return 0; -} - -/* -** Print a markdown or table-style row separator using ascii-art -*/ -static void qrfRowSeparator(sqlite3_str *pOut, qrfColData *p, char cSep){ - int i; - if( p->nCol>0 ){ - int useBorder = p->p->spec.bBorder!=QRF_No; - if( useBorder ){ - sqlite3_str_append(pOut, &cSep, 1); - } - sqlite3_str_appendchar(pOut, p->a[0].w+p->nMargin, '-'); - for(i=1; i<p->nCol; i++){ - sqlite3_str_append(pOut, &cSep, 1); - sqlite3_str_appendchar(pOut, p->a[i].w+p->nMargin, '-'); - } - if( useBorder ){ - sqlite3_str_append(pOut, &cSep, 1); - } - } - sqlite3_str_append(pOut, "\n", 1); -} - -/* -** UTF8 box-drawing characters. Imagine box lines like this: -** -** 1 -** | -** 4 --+-- 2 -** | -** 3 -** -** Each box characters has between 2 and 4 of the lines leading from -** the center. The characters are here identified by the numbers of -** their corresponding lines. -*/ -#define BOX_24 "\342\224\200" /* U+2500 --- */ -#define BOX_13 "\342\224\202" /* U+2502 | */ -#define BOX_23 "\342\224\214" /* U+250c ,- */ -#define BOX_34 "\342\224\220" /* U+2510 -, */ -#define BOX_12 "\342\224\224" /* U+2514 '- */ -#define BOX_14 "\342\224\230" /* U+2518 -' */ -#define BOX_123 "\342\224\234" /* U+251c |- */ -#define BOX_134 "\342\224\244" /* U+2524 -| */ -#define BOX_234 "\342\224\254" /* U+252c -,- */ -#define BOX_124 "\342\224\264" /* U+2534 -'- */ -#define BOX_1234 "\342\224\274" /* U+253c -|- */ - -/* Rounded corners: */ -#define BOX_R12 "\342\225\260" /* U+2570 '- */ -#define BOX_R23 "\342\225\255" /* U+256d ,- */ -#define BOX_R34 "\342\225\256" /* U+256e -, */ -#define BOX_R14 "\342\225\257" /* U+256f -' */ - -/* Doubled horizontal lines: */ -#define DBL_24 "\342\225\220" /* U+2550 === */ -#define DBL_123 "\342\225\236" /* U+255e |= */ -#define DBL_134 "\342\225\241" /* U+2561 =| */ -#define DBL_1234 "\342\225\252" /* U+256a =|= */ - -/* Draw horizontal line N characters long using unicode box -** characters -*/ -static void qrfBoxLine(sqlite3_str *pOut, int N, int bDbl){ - const char *azDash[2] = { - BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24, - DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 - };/* 0 1 2 3 4 5 6 7 8 9 */ - const int nDash = 30; - N *= 3; - while( N>nDash ){ - sqlite3_str_append(pOut, azDash[bDbl], nDash); - N -= nDash; - } - sqlite3_str_append(pOut, azDash[bDbl], N); -} - -/* -** Draw a horizontal separator for a QRF_STYLE_Box table. -*/ -static void qrfBoxSeparator( - sqlite3_str *pOut, - qrfColData *p, - const char *zSep1, - const char *zSep2, - const char *zSep3, - int bDbl -){ - int i; - if( p->nCol>0 ){ - int useBorder = p->p->spec.bBorder!=QRF_No; - if( useBorder ){ - sqlite3_str_appendall(pOut, zSep1); - } - qrfBoxLine(pOut, p->a[0].w+p->nMargin, bDbl); - for(i=1; i<p->nCol; i++){ - sqlite3_str_appendall(pOut, zSep2); - qrfBoxLine(pOut, p->a[i].w+p->nMargin, bDbl); - } - if( useBorder ){ - sqlite3_str_appendall(pOut, zSep3); - } - } - sqlite3_str_append(pOut, "\n", 1); -} - -/* -** Load into pData the default alignment for the body of a table. -*/ -static void qrfLoadAlignment(qrfColData *pData, Qrf *p){ - sqlite3_int64 i; - for(i=0; i<pData->nCol; i++){ - pData->a[i].e = p->spec.eDfltAlign; - if( i<p->spec.nAlign ){ - unsigned char ax = p->spec.aAlign[i]; - if( (ax & QRF_ALIGN_HMASK)!=0 ){ - pData->a[i].e = (ax & QRF_ALIGN_HMASK) | - (pData->a[i].e & QRF_ALIGN_VMASK); - } - }else if( i<p->spec.nWidth ){ - if( p->spec.aWidth[i]<0 ){ - pData->a[i].e = QRF_ALIGN_Right | - (pData->a[i].e & QRF_ALIGN_VMASK); - } - } - } -} - -/* -** If the single column in pData->a[] with pData->n entries can be -** laid out as nCol columns with a 2-space gap between each such -** that all columns fit within nSW, then return a pointer to an array -** of integers which is the width of each column from left to right. -** -** If the layout is not possible, return a NULL pointer. -** -** Space to hold the returned array is from sqlite_malloc64(). -*/ -static int *qrfValidLayout( - qrfColData *pData, /* Collected query results */ - Qrf *p, /* On which to report an OOM */ - int nCol, /* Attempt this many columns */ - int nSW /* Screen width */ -){ - int i; /* Loop counter */ - int nr; /* Number of rows */ - int w = 0; /* Width of the current column */ - int t; /* Total width of all columns */ - int *aw; /* Array of individual column widths */ - - aw = sqlite3_malloc64( sizeof(int)*nCol ); - if( aw==0 ){ - qrfOom(p); - return 0; - } - nr = (pData->n + nCol - 1)/nCol; - for(i=0; i<pData->n; i++){ - if( (i%nr)==0 ){ - if( i>0 ) aw[i/nr-1] = w; - w = pData->aiWth[i]; - }else if( pData->aiWth[i]>w ){ - w = pData->aiWth[i]; - } - } - aw[nCol-1] = w; - for(t=i=0; i<nCol; i++) t += aw[i]; - t += 2*(nCol-1); - if( t>nSW ){ - sqlite3_free(aw); - return 0; - } - return aw; -} - -/* -** The output is single-column and the bSplitColumn flag is set. -** Check to see if the single-column output can be split into multiple -** columns that appear side-by-side. Adjust pData appropriately. -*/ -static void qrfSplitColumn(qrfColData *pData, Qrf *p){ - int nCol = 1; - int *aw = 0; - char **az = 0; - int *aiWth = 0; - unsigned char *abNum = 0; - int nColNext = 2; - int w; - struct qrfPerCol *a = 0; - sqlite3_int64 nRow = 1; - sqlite3_int64 i; - while( 1/*exit-by-break*/ ){ - int *awNew = qrfValidLayout(pData, p, nColNext, p->spec.nScreenWidth); - if( awNew==0 ) break; - sqlite3_free(aw); - aw = awNew; - nCol = nColNext; - nRow = (pData->n + nCol - 1)/nCol; - if( nRow==1 ) break; - nColNext++; - while( (pData->n + nColNext - 1)/nColNext == nRow ) nColNext++; - } - if( nCol==1 ){ - sqlite3_free(aw); - return; /* Cannot do better than 1 column */ - } - az = sqlite3_malloc64( nRow*nCol*sizeof(char*) ); - if( az==0 ){ - qrfOom(p); - return; - } - aiWth = sqlite3_malloc64( nRow*nCol*sizeof(int) ); - if( aiWth==0 ){ - sqlite3_free(az); - qrfOom(p); - return; - } - a = sqlite3_malloc64( nCol*sizeof(struct qrfPerCol) ); - if( a==0 ){ - sqlite3_free(az); - sqlite3_free(aiWth); - qrfOom(p); - return; - } - abNum = sqlite3_malloc64( nRow*nCol ); - if( abNum==0 ){ - sqlite3_free(az); - sqlite3_free(aiWth); - sqlite3_free(a); - qrfOom(p); - return; - } - for(i=0; i<pData->n; i++){ - sqlite3_int64 j = (i%nRow)*nCol + (i/nRow); - az[j] = pData->az[i]; - abNum[j]= pData->abNum[i]; - pData->az[i] = 0; - aiWth[j] = pData->aiWth[i]; - } - while( i<nRow*nCol ){ - sqlite3_int64 j = (i%nRow)*nCol + (i/nRow); - az[j] = sqlite3_mprintf(""); - if( az[j]==0 ) qrfOom(p); - aiWth[j] = 0; - abNum[j] = 0; - i++; - } - for(i=0; i<nCol; i++){ - a[i].fx = a[i].mxW = a[i].w = aw[i]; - a[i].e = pData->a[0].e; - } - sqlite3_free(pData->az); - sqlite3_free(pData->aiWth); - sqlite3_free(pData->a); - sqlite3_free(pData->abNum); - sqlite3_free(aw); - pData->az = az; - pData->aiWth = aiWth; - pData->a = a; - pData->abNum = abNum; - pData->nCol = nCol; - pData->n = pData->nAlloc = nRow*nCol; - for(i=w=0; i<nCol; i++) w += a[i].w; - pData->nMargin = (p->spec.nScreenWidth - w)/(nCol - 1); - if( pData->nMargin>5 ) pData->nMargin = 5; -} - -/* -** Adjust the layout for the screen width restriction -*/ -static void qrfRestrictScreenWidth(qrfColData *pData, Qrf *p){ - int sepW; /* Width of all box separators and margins */ - int sumW; /* Total width of data area over all columns */ - int targetW; /* Desired total data area */ - int i; /* Loop counters */ - int nCol; /* Number of columns */ - - pData->nMargin = 2; /* Default to normal margins */ - if( p->spec.nScreenWidth==0 ) return; - if( p->spec.eStyle==QRF_STYLE_Column ){ - sepW = pData->nCol*2 - 2; - }else{ - sepW = pData->nCol*3 + 1; - if( p->spec.bBorder==QRF_No ) sepW -= 2; - } - nCol = pData->nCol; - for(i=sumW=0; i<nCol; i++) sumW += pData->a[i].w; - if( p->spec.nScreenWidth >= sumW+sepW ) return; - - /* First thing to do is reduce the separation between columns */ - pData->nMargin = 0; - if( p->spec.eStyle==QRF_STYLE_Column ){ - sepW = pData->nCol - 1; - }else{ - sepW = pData->nCol + 1; - if( p->spec.bBorder==QRF_No ) sepW -= 2; - } - targetW = p->spec.nScreenWidth - sepW; - -#define MIN_SQUOZE 8 -#define MIN_EX_SQUOZE 16 - /* Reduce the width of the widest eligible column. A column is - ** eligible for narrowing if: - ** - ** * It is not a fixed-width column (a[0].fx is false) - ** * The current width is more than MIN_SQUOZE - ** * Either: - ** + The current width is more then MIN_EX_SQUOZE, or - ** + The current width is more than half the max width (a[].mxW) - ** - ** Keep making reductions until either no more reductions are - ** possible or until the size target is reached. - */ - while( sumW > targetW ){ - int gain, w; - int ix = -1; - int mx = 0; - for(i=0; i<nCol; i++){ - if( pData->a[i].fx==0 - && (w = pData->a[i].w)>mx - && w>MIN_SQUOZE - && (w>MIN_EX_SQUOZE || w*2>pData->a[i].mxW) - ){ - ix = i; - mx = w; - } - } - if( ix<0 ) break; - if( mx>=MIN_SQUOZE*2 ){ - gain = mx/2; - }else{ - gain = mx - MIN_SQUOZE; - } - if( sumW - gain < targetW ){ - gain = sumW - targetW; - } - sumW -= gain; - pData->a[ix].w -= gain; - pData->bMultiRow = 1; - } -} - -/* -** Columnar modes require that the entire query be evaluated first, with -** results written into memory, so that we can compute appropriate column -** widths. -*/ -static void qrfColumnar(Qrf *p){ - sqlite3_int64 i, j; /* Loop counters */ - const char *colSep = 0; /* Column separator text */ - const char *rowSep = 0; /* Row terminator text */ - const char *rowStart = 0; /* Row start text */ - int szColSep, szRowSep, szRowStart; /* Size in bytes of previous 3 */ - int rc; /* Result code */ - int nColumn = p->nCol; /* Number of columns */ - int bWW; /* True to do word-wrap */ - sqlite3_str *pStr; /* Temporary rendering */ - qrfColData data; /* Columnar layout data */ - int bRTrim; /* Trim trailing space */ - - rc = sqlite3_step(p->pStmt); - if( rc!=SQLITE_ROW || nColumn==0 ){ - return; /* No output */ - } - - /* Initialize the data container */ - memset(&data, 0, sizeof(data)); - data.nCol = p->nCol; - data.p = p; - data.a = sqlite3_malloc64( nColumn*sizeof(struct qrfPerCol) ); - if( data.a==0 ){ - qrfOom(p); - return; - } - memset(data.a, 0, nColumn*sizeof(struct qrfPerCol) ); - if( qrfColDataEnlarge(&data) ) return; - assert( data.az!=0 ); - - /* Load the column header names and all cell content into data */ - if( p->spec.bTitles==QRF_Yes ){ - unsigned char saved_eText = p->spec.eText; - p->spec.eText = p->spec.eTitle; - memset(data.abNum, 0, nColumn); - for(i=0; i<nColumn; i++){ - const char *z = (const char*)sqlite3_column_name(p->pStmt,i); - int nNL = 0; - int n, w; - pStr = sqlite3_str_new(p->db); - qrfEncodeText(p, pStr, z ? z : ""); - n = sqlite3_str_length(pStr); - qrfStrErr(p, pStr); - z = data.az[data.n] = sqlite3_str_finish(pStr); - if( p->spec.nTitleLimit ){ - nNL = 0; - data.aiWth[data.n] = w = qrfTitleLimit(data.az[data.n], - p->spec.nTitleLimit ); - }else{ - data.aiWth[data.n] = w = qrfDisplayWidth(z, n, &nNL); - } - data.n++; - if( w>data.a[i].mxW ) data.a[i].mxW = w; - if( nNL ) data.bMultiRow = 1; - } - p->spec.eText = saved_eText; - p->nRow++; - } - do{ - if( data.n+nColumn > data.nAlloc ){ - if( qrfColDataEnlarge(&data) ) return; - } - for(i=0; i<nColumn; i++){ - char *z; - int nNL = 0; - int n, w; - int eType = sqlite3_column_type(p->pStmt,i); - pStr = sqlite3_str_new(p->db); - qrfRenderValue(p, pStr, i); - n = sqlite3_str_length(pStr); - qrfStrErr(p, pStr); - z = data.az[data.n] = sqlite3_str_finish(pStr); - data.abNum[data.n] = eType==SQLITE_INTEGER || eType==SQLITE_FLOAT; - data.aiWth[data.n] = w = qrfDisplayWidth(z, n, &nNL); - data.n++; - if( w>data.a[i].mxW ) data.a[i].mxW = w; - if( nNL ) data.bMultiRow = 1; - } - p->nRow++; - }while( sqlite3_step(p->pStmt)==SQLITE_ROW && p->iErr==SQLITE_OK ); - if( p->iErr ){ - qrfColDataFree(&data); - return; - } - - /* Compute the width and alignment of every column */ - if( p->spec.bTitles==QRF_No ){ - qrfLoadAlignment(&data, p); - }else{ - unsigned char e; - if( p->spec.eTitleAlign==QRF_Auto ){ - e = QRF_ALIGN_Center; - }else{ - e = p->spec.eTitleAlign; - } - for(i=0; i<nColumn; i++) data.a[i].e = e; - } - - for(i=0; i<nColumn; i++){ - int w = 0; - if( i<p->spec.nWidth ){ - w = p->spec.aWidth[i]; - if( w==(-32768) ){ - w = 0; - if( p->spec.nAlign>i && (p->spec.aAlign[i] & QRF_ALIGN_HMASK)==0 ){ - data.a[i].e |= QRF_ALIGN_Right; - } - }else if( w<0 ){ - w = -w; - if( p->spec.nAlign>i && (p->spec.aAlign[i] & QRF_ALIGN_HMASK)==0 ){ - data.a[i].e |= QRF_ALIGN_Right; - } - } - if( w ) data.a[i].fx = 1; - } - if( w==0 ){ - w = data.a[i].mxW; - if( p->spec.nWrap>0 && w>p->spec.nWrap ){ - w = p->spec.nWrap; - data.bMultiRow = 1; - } - }else if( (data.bMultiRow==0 || w==1) && data.a[i].mxW>w ){ - data.bMultiRow = 1; - if( w==1 ){ - /* If aiWth[j] is 2 or more, then there might be a double-wide - ** character somewhere. So make the column width at least 2. */ - w = 2; - } - } - data.a[i].w = w; - } - - if( nColumn==1 - && data.n>1 - && p->spec.bSplitColumn==QRF_Yes - && p->spec.eStyle==QRF_STYLE_Column - && p->spec.bTitles==QRF_No - && p->spec.nScreenWidth>data.a[0].w+3 - ){ - /* Attempt to convert single-column tables into multi-column by - ** verticle wrapping, if the screen is wide enough and if the - ** bSplitColumn flag is set. */ - qrfSplitColumn(&data, p); - nColumn = data.nCol; - }else{ - /* Adjust the column widths due to screen width restrictions */ - qrfRestrictScreenWidth(&data, p); - } - - /* Draw the line across the top of the table. Also initialize - ** the row boundary and column separator texts. */ - switch( p->spec.eStyle ){ - case QRF_STYLE_Box: - if( data.nMargin ){ - rowStart = BOX_13 " "; - colSep = " " BOX_13 " "; - rowSep = " " BOX_13 "\n"; - }else{ - rowStart = BOX_13; - colSep = BOX_13; - rowSep = BOX_13 "\n"; - } - if( p->spec.bBorder==QRF_No){ - rowStart += 3; - rowSep = "\n"; - }else{ - qrfBoxSeparator(p->pOut, &data, BOX_R23, BOX_234, BOX_R34, 0); - } - break; - case QRF_STYLE_Table: - if( data.nMargin ){ - rowStart = "| "; - colSep = " | "; - rowSep = " |\n"; - }else{ - rowStart = "|"; - colSep = "|"; - rowSep = "|\n"; - } - if( p->spec.bBorder==QRF_No ){ - rowStart += 1; - rowSep = "\n"; - }else{ - qrfRowSeparator(p->pOut, &data, '+'); - } - break; - case QRF_STYLE_Column: { - static const char zSpace[] = " "; - rowStart = ""; - if( data.nMargin<2 ){ - colSep = " "; - }else if( data.nMargin<=5 ){ - colSep = &zSpace[5-data.nMargin]; - }else{ - colSep = zSpace; - } - rowSep = "\n"; - break; - } - default: /*case QRF_STYLE_Markdown:*/ - if( data.nMargin ){ - rowStart = "| "; - colSep = " | "; - rowSep = " |\n"; - }else{ - rowStart = "|"; - colSep = "|"; - rowSep = "|\n"; - } - break; - } - szRowStart = (int)strlen(rowStart); - szRowSep = (int)strlen(rowSep); - szColSep = (int)strlen(colSep); - - bWW = (p->spec.bWordWrap==QRF_Yes && data.bMultiRow); - if( p->spec.eStyle==QRF_STYLE_Column - || (p->spec.bBorder==QRF_No - && (p->spec.eStyle==QRF_STYLE_Box || p->spec.eStyle==QRF_STYLE_Table) - ) - ){ - bRTrim = 1; - }else{ - bRTrim = 0; - } - for(i=0; i<data.n && sqlite3_str_errcode(p->pOut)==SQLITE_OK; i+=nColumn){ - int bMore; - int nRow = 0; - - /* Draw a single row of the table. This might be the title line - ** (if there is a title line) or a row in the body of the table. - ** The column number will be j. The row number is i/nColumn. - */ - for(j=0; j<nColumn; j++){ - data.a[j].z = data.az[i+j]; - if( data.a[j].z==0 ) data.a[j].z = ""; - data.a[j].bNum = data.abNum[i+j]; - } - do{ - sqlite3_str_append(p->pOut, rowStart, szRowStart); - bMore = 0; - for(j=0; j<nColumn; j++){ - int nThis = 0; - int nWide = 0; - int iNext = 0; - int nWS; - qrfWrapLine(data.a[j].z, data.a[j].w, bWW, &nThis, &nWide, &iNext); - nWS = data.a[j].w - nWide; - qrfPrintAligned(p->pOut, &data.a[j], nThis, nWS); - data.a[j].z += iNext; - if( data.a[j].z[0]!=0 ){ - bMore = 1; - } - if( j<nColumn-1 ){ - sqlite3_str_append(p->pOut, colSep, szColSep); - }else{ - if( bRTrim ) qrfRTrim(p->pOut); - sqlite3_str_append(p->pOut, rowSep, szRowSep); - } - } - }while( bMore && ++nRow < p->mxHeight ); - if( bMore ){ - /* This row was terminated by nLineLimit. Show ellipsis. */ - sqlite3_str_append(p->pOut, rowStart, szRowStart); - for(j=0; j<nColumn; j++){ - if( data.a[j].z[0]==0 ){ - sqlite3_str_appendchar(p->pOut, data.a[j].w, ' '); - }else{ - int nE = 3; - if( nE>data.a[j].w ) nE = data.a[j].w; - data.a[j].z = "..."; - qrfPrintAligned(p->pOut, &data.a[j], nE, data.a[j].w-nE); - } - if( j<nColumn-1 ){ - sqlite3_str_append(p->pOut, colSep, szColSep); - }else{ - if( bRTrim ) qrfRTrim(p->pOut); - sqlite3_str_append(p->pOut, rowSep, szRowSep); - } - } - } - - /* Draw either (1) the separator between the title line and the body - ** of the table, or (2) separators between individual rows of the table - ** body. isTitleDataSeparator will be true if we are doing (1). - */ - if( (i==0 || data.bMultiRow) && i+nColumn<data.n ){ - int isTitleDataSeparator = (i==0 && p->spec.bTitles==QRF_Yes); - if( isTitleDataSeparator ){ - qrfLoadAlignment(&data, p); - } - switch( p->spec.eStyle ){ - case QRF_STYLE_Table: { - if( isTitleDataSeparator || data.bMultiRow ){ - qrfRowSeparator(p->pOut, &data, '+'); - } - break; - } - case QRF_STYLE_Box: { - if( isTitleDataSeparator ){ - qrfBoxSeparator(p->pOut, &data, DBL_123, DBL_1234, DBL_134, 1); - }else if( data.bMultiRow ){ - qrfBoxSeparator(p->pOut, &data, BOX_123, BOX_1234, BOX_134, 0); - } - break; - } - case QRF_STYLE_Markdown: { - if( isTitleDataSeparator ){ - qrfRowSeparator(p->pOut, &data, '|'); - } - break; - } - case QRF_STYLE_Column: { - if( isTitleDataSeparator ){ - for(j=0; j<nColumn; j++){ - sqlite3_str_appendchar(p->pOut, data.a[j].w, '-'); - if( j<nColumn-1 ){ - sqlite3_str_append(p->pOut, colSep, szColSep); - }else{ - qrfRTrim(p->pOut); - sqlite3_str_append(p->pOut, rowSep, szRowSep); - } - } - }else if( data.bMultiRow ){ - qrfRTrim(p->pOut); - sqlite3_str_append(p->pOut, "\n", 1); - } - break; - } - } - } - } - - /* Draw the line across the bottom of the table */ - if( p->spec.bBorder!=QRF_No ){ - switch( p->spec.eStyle ){ - case QRF_STYLE_Box: - qrfBoxSeparator(p->pOut, &data, BOX_R12, BOX_124, BOX_R14, 0); - break; - case QRF_STYLE_Table: - qrfRowSeparator(p->pOut, &data, '+'); - break; - } - } - qrfWrite(p); - - qrfColDataFree(&data); - return; -} - -/* -** Parameter azArray points to a zero-terminated array of strings. zStr -** points to a single nul-terminated string. Return non-zero if zStr -** is equal, according to strcmp(), to any of the strings in the array. -** Otherwise, return zero. -*/ -static int qrfStringInArray(const char *zStr, const char **azArray){ - int i; - if( zStr==0 ) return 0; - for(i=0; azArray[i]; i++){ - if( 0==strcmp(zStr, azArray[i]) ) return 1; - } - return 0; -} - -/* -** Print out an EXPLAIN with indentation. This is a two-pass algorithm. -** -** On the first pass, we compute aiIndent[iOp] which is the amount of -** indentation to apply to the iOp-th opcode. The output actually occurs -** on the second pass. -** -** The indenting rules are: -** -** * For each "Next", "Prev", "VNext" or "VPrev" instruction, indent -** all opcodes that occur between the p2 jump destination and the opcode -** itself by 2 spaces. -** -** * Do the previous for "Return" instructions for when P2 is positive. -** See tag-20220407a in wherecode.c and vdbe.c. -** -** * For each "Goto", if the jump destination is earlier in the program -** and ends on one of: -** Yield SeekGt SeekLt RowSetRead Rewind -** or if the P1 parameter is one instead of zero, -** then indent all opcodes between the earlier instruction -** and "Goto" by 2 spaces. -*/ -static void qrfExplain(Qrf *p){ - int *abYield = 0; /* abYield[iOp] is rue if opcode iOp is an OP_Yield */ - int *aiIndent = 0; /* Indent the iOp-th opcode by aiIndent[iOp] */ - i64 nAlloc = 0; /* Allocated size of aiIndent[], abYield */ - int nIndent = 0; /* Number of entries in aiIndent[] */ - int iOp; /* Opcode number */ - int i; /* Column loop counter */ - - const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", - "Return", 0 }; - const char *azYield[] = { "Yield", "SeekLT", "SeekGT", "RowSetRead", - "Rewind", 0 }; - const char *azGoto[] = { "Goto", 0 }; - - /* The caller guarantees that the leftmost 4 columns of the statement - ** passed to this function are equivalent to the leftmost 4 columns - ** of EXPLAIN statement output. In practice the statement may be - ** an EXPLAIN, or it may be a query on the bytecode() virtual table. */ - assert( sqlite3_column_count(p->pStmt)>=4 ); - assert( 0==sqlite3_stricmp( sqlite3_column_name(p->pStmt, 0), "addr" ) ); - assert( 0==sqlite3_stricmp( sqlite3_column_name(p->pStmt, 1), "opcode" ) ); - assert( 0==sqlite3_stricmp( sqlite3_column_name(p->pStmt, 2), "p1" ) ); - assert( 0==sqlite3_stricmp( sqlite3_column_name(p->pStmt, 3), "p2" ) ); - - for(iOp=0; SQLITE_ROW==sqlite3_step(p->pStmt) && !p->iErr; iOp++){ - int iAddr = sqlite3_column_int(p->pStmt, 0); - const char *zOp = (const char*)sqlite3_column_text(p->pStmt, 1); - int p1 = sqlite3_column_int(p->pStmt, 2); - int p2 = sqlite3_column_int(p->pStmt, 3); - - /* Assuming that p2 is an instruction address, set variable p2op to the - ** index of that instruction in the aiIndent[] array. p2 and p2op may be - ** different if the current instruction is part of a sub-program generated - ** by an SQL trigger or foreign key. */ - int p2op = (p2 + (iOp-iAddr)); - - /* Grow the aiIndent array as required */ - if( iOp>=nAlloc ){ - nAlloc += 100; - aiIndent = (int*)sqlite3_realloc64(aiIndent, nAlloc*sizeof(int)); - abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int)); - if( aiIndent==0 || abYield==0 ){ - qrfOom(p); - sqlite3_free(aiIndent); - sqlite3_free(abYield); - return; - } - } - - abYield[iOp] = qrfStringInArray(zOp, azYield); - aiIndent[iOp] = 0; - nIndent = iOp+1; - if( qrfStringInArray(zOp, azNext) && p2op>0 ){ - for(i=p2op; i<iOp; i++) aiIndent[i] += 2; - } - if( qrfStringInArray(zOp, azGoto) && p2op<iOp && (abYield[p2op] || p1) ){ - for(i=p2op; i<iOp; i++) aiIndent[i] += 2; - } - } - sqlite3_free(abYield); - - /* Second pass. Actually generate output */ - sqlite3_reset(p->pStmt); - if( p->iErr==SQLITE_OK ){ - static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13}; - static const int aExplainMap[] = {0, 1, 2, 3, 4, 5, 6, 7 }; - static const int aScanExpWidth[] = {4,15, 6, 13, 4, 4, 4, 13, 2, 13}; - static const int aScanExpMap[] = {0, 9, 8, 1, 2, 3, 4, 5, 6, 7 }; - const int *aWidth = aExplainWidth; - const int *aMap = aExplainMap; - int nWidth = sizeof(aExplainWidth)/sizeof(int); - int iIndent = 1; - int nArg = p->nCol; - if( p->spec.eStyle==QRF_STYLE_StatsVm ){ - aWidth = aScanExpWidth; - aMap = aScanExpMap; - nWidth = sizeof(aScanExpWidth)/sizeof(int); - iIndent = 3; - } - if( nArg>nWidth ) nArg = nWidth; - - for(iOp=0; sqlite3_step(p->pStmt)==SQLITE_ROW && !p->iErr; iOp++){ - /* If this is the first row seen, print out the headers */ - if( iOp==0 ){ - for(i=0; i<nArg; i++){ - const char *zCol = sqlite3_column_name(p->pStmt, aMap[i]); - qrfWidthPrint(p,p->pOut, aWidth[i], zCol); - if( i==nArg-1 ){ - sqlite3_str_append(p->pOut, "\n", 1); - }else{ - sqlite3_str_append(p->pOut, " ", 2); - } - } - for(i=0; i<nArg; i++){ - sqlite3_str_appendf(p->pOut, "%.*c", aWidth[i], '-'); - if( i==nArg-1 ){ - sqlite3_str_append(p->pOut, "\n", 1); - }else{ - sqlite3_str_append(p->pOut, " ", 2); - } - } - } - - for(i=0; i<nArg; i++){ - const char *zSep = " "; - int w = aWidth[i]; - const char *zVal = (const char*)sqlite3_column_text(p->pStmt, aMap[i]); - int len; - if( i==nArg-1 ) w = 0; - if( zVal==0 ) zVal = ""; - len = (int)sqlite3_qrf_wcswidth(zVal); - if( len>w ){ - w = len; - zSep = " "; - } - if( i==iIndent && aiIndent && iOp<nIndent ){ - sqlite3_str_appendchar(p->pOut, aiIndent[iOp], ' '); - } - qrfWidthPrint(p, p->pOut, w, zVal); - if( i==nArg-1 ){ - sqlite3_str_append(p->pOut, "\n", 1); - }else{ - sqlite3_str_appendall(p->pOut, zSep); - } - } - p->nRow++; - } - qrfWrite(p); - } - sqlite3_free(aiIndent); -} - -/* -** Do a "scanstatus vm" style EXPLAIN listing on p->pStmt. -** -** p->pStmt is probably not an EXPLAIN query. Instead, construct a -** new query that is a bytecode() rendering of p->pStmt with extra -** columns for the "scanstatus vm" outputs, and run the results of -** that new query through the normal EXPLAIN formatting. -*/ -static void qrfScanStatusVm(Qrf *p){ - sqlite3_stmt *pOrigStmt = p->pStmt; - sqlite3_stmt *pExplain; - int rc; - static const char *zSql = - " SELECT addr, opcode, p1, p2, p3, p4, p5, comment, nexec," - " format('% 6s (%.2f%%)'," - " CASE WHEN ncycle<100_000 THEN ncycle || ' '" - " WHEN ncycle<100_000_000 THEN (ncycle/1_000) || 'K'" - " WHEN ncycle<100_000_000_000 THEN (ncycle/1_000_000) || 'M'" - " ELSE (ncycle/1000_000_000) || 'G' END," - " ncycle*100.0/(sum(ncycle) OVER ())" - " ) AS cycles" - " FROM bytecode(?1)"; - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pExplain, 0); - if( rc ){ - qrfError(p, rc, "%s", sqlite3_errmsg(p->db)); - sqlite3_finalize(pExplain); - return; - } - sqlite3_bind_pointer(pExplain, 1, pOrigStmt, "stmt-pointer", 0); - p->pStmt = pExplain; - p->nCol = 10; - qrfExplain(p); - sqlite3_finalize(pExplain); - p->pStmt = pOrigStmt; -} - -/* -** Attempt to determine if identifier zName needs to be quoted, either -** because it contains non-alphanumeric characters, or because it is an -** SQLite keyword. Be conservative in this estimate: When in doubt assume -** that quoting is required. -** -** Return 1 if quoting is required. Return 0 if no quoting is required. -*/ - -static int qrf_need_quote(const char *zName){ - int i; - const unsigned char *z = (const unsigned char*)zName; - if( z==0 ) return 1; - if( !qrfAlpha(z[0]) ) return 1; - for(i=0; z[i]; i++){ - if( !qrfAlnum(z[i]) ) return 1; - } - return sqlite3_keyword_check(zName, i)!=0; -} - -/* -** Helper function for QRF_STYLE_Json and QRF_STYLE_JObject. -** The initial "{" for a JSON object that will contain row content -** has been output. Now output all the content. -*/ -static void qrfOneJsonRow(Qrf *p){ - int i, nItem; - for(nItem=i=0; i<p->nCol; i++){ - const char *zCName; - zCName = sqlite3_column_name(p->pStmt, i); - if( nItem>0 ) sqlite3_str_append(p->pOut, ",", 1); - nItem++; - qrfEncodeText(p, p->pOut, zCName); - sqlite3_str_append(p->pOut, ":", 1); - qrfRenderValue(p, p->pOut, i); - } - qrfWrite(p); -} - -/* -** Render a single row of output for non-columnar styles - any -** style that lets us render row by row as the content is received -** from the query. -*/ -static void qrfOneSimpleRow(Qrf *p){ - int i; - switch( p->spec.eStyle ){ - case QRF_STYLE_Off: - case QRF_STYLE_Count: { - /* No-op */ - break; - } - case QRF_STYLE_Json: { - if( p->nRow==0 ){ - sqlite3_str_append(p->pOut, "[{", 2); - }else{ - sqlite3_str_append(p->pOut, "},\n{", 4); - } - qrfOneJsonRow(p); - break; - } - case QRF_STYLE_JObject: { - if( p->nRow==0 ){ - sqlite3_str_append(p->pOut, "{", 1); - }else{ - sqlite3_str_append(p->pOut, "}\n{", 3); - } - qrfOneJsonRow(p); - break; - } - case QRF_STYLE_Html: { - if( p->nRow==0 && p->spec.bTitles==QRF_Yes ){ - sqlite3_str_append(p->pOut, "<TR>", 4); - for(i=0; i<p->nCol; i++){ - const char *zCName = sqlite3_column_name(p->pStmt, i); - sqlite3_str_append(p->pOut, "\n<TH>", 5); - qrfEncodeText(p, p->pOut, zCName); - } - sqlite3_str_append(p->pOut, "\n</TR>\n", 7); - } - sqlite3_str_append(p->pOut, "<TR>", 4); - for(i=0; i<p->nCol; i++){ - sqlite3_str_append(p->pOut, "\n<TD>", 5); - qrfRenderValue(p, p->pOut, i); - } - sqlite3_str_append(p->pOut, "\n</TR>\n", 7); - qrfWrite(p); - break; - } - case QRF_STYLE_Insert: { - if( qrf_need_quote(p->spec.zTableName) ){ - sqlite3_str_appendf(p->pOut,"INSERT INTO \"%w\"",p->spec.zTableName); - }else{ - sqlite3_str_appendf(p->pOut,"INSERT INTO %s",p->spec.zTableName); - } - if( p->spec.bTitles==QRF_Yes ){ - for(i=0; i<p->nCol; i++){ - const char *zCName = sqlite3_column_name(p->pStmt, i); - if( qrf_need_quote(zCName) ){ - sqlite3_str_appendf(p->pOut, "%c\"%w\"", - i==0 ? '(' : ',', zCName); - }else{ - sqlite3_str_appendf(p->pOut, "%c%s", - i==0 ? '(' : ',', zCName); - } - } - sqlite3_str_append(p->pOut, ")", 1); - } - sqlite3_str_append(p->pOut," VALUES(", 8); - for(i=0; i<p->nCol; i++){ - if( i>0 ) sqlite3_str_append(p->pOut, ",", 1); - qrfRenderValue(p, p->pOut, i); - } - sqlite3_str_append(p->pOut, ");\n", 3); - qrfWrite(p); - break; - } - case QRF_STYLE_Line: { - sqlite3_str *pVal; - int mxW; - int bWW; - int nSep; - if( p->u.sLine.azCol==0 ){ - p->u.sLine.azCol = sqlite3_malloc64( p->nCol*sizeof(char*) ); - if( p->u.sLine.azCol==0 ){ - qrfOom(p); - break; - } - p->u.sLine.mxColWth = 0; - for(i=0; i<p->nCol; i++){ - int sz; - const char *zCName = sqlite3_column_name(p->pStmt, i); - if( zCName==0 ) zCName = "unknown"; - p->u.sLine.azCol[i] = sqlite3_mprintf("%s", zCName); - if( p->spec.nTitleLimit>0 ){ - (void)qrfTitleLimit(p->u.sLine.azCol[i], p->spec.nTitleLimit); - } - sz = (int)sqlite3_qrf_wcswidth(p->u.sLine.azCol[i]); - if( sz > p->u.sLine.mxColWth ) p->u.sLine.mxColWth = sz; - } - } - if( p->nRow ) sqlite3_str_append(p->pOut, "\n", 1); - pVal = sqlite3_str_new(p->db); - nSep = (int)strlen(p->spec.zColumnSep); - mxW = p->mxWidth - (nSep + p->u.sLine.mxColWth); - bWW = p->spec.bWordWrap==QRF_Yes; - for(i=0; i<p->nCol; i++){ - const char *zVal; - int cnt = 0; - qrfWidthPrint(p, p->pOut, -p->u.sLine.mxColWth, p->u.sLine.azCol[i]); - sqlite3_str_append(p->pOut, p->spec.zColumnSep, nSep); - qrfRenderValue(p, pVal, i); - zVal = sqlite3_str_value(pVal); - if( zVal==0 ) zVal = ""; - do{ - int nThis, nWide, iNext; - qrfWrapLine(zVal, mxW, bWW, &nThis, &nWide, &iNext); - if( cnt ) sqlite3_str_appendchar(p->pOut,p->u.sLine.mxColWth+2,' '); - cnt++; - if( cnt>p->mxHeight ){ - zVal = "..."; - nThis = iNext = 3; - } - sqlite3_str_append(p->pOut, zVal, nThis); - sqlite3_str_append(p->pOut, "\n", 1); - zVal += iNext; - }while( zVal[0] ); - sqlite3_str_reset(pVal); - } - qrfStrErr(p, pVal); - sqlite3_free(sqlite3_str_finish(pVal)); - qrfWrite(p); - break; - } - case QRF_STYLE_Eqp: { - const char *zEqpLine = (const char*)sqlite3_column_text(p->pStmt,3); - int iEqpId = sqlite3_column_int(p->pStmt, 0); - int iParentId = sqlite3_column_int(p->pStmt, 1); - if( zEqpLine==0 ) zEqpLine = ""; - if( zEqpLine[0]=='-' ) qrfEqpRender(p, 0); - qrfEqpAppend(p, iEqpId, iParentId, zEqpLine); - break; - } - default: { /* QRF_STYLE_List */ - if( p->nRow==0 && p->spec.bTitles==QRF_Yes ){ - int saved_eText = p->spec.eText; - p->spec.eText = p->spec.eTitle; - for(i=0; i<p->nCol; i++){ - const char *zCName = sqlite3_column_name(p->pStmt, i); - if( i>0 ) sqlite3_str_appendall(p->pOut, p->spec.zColumnSep); - qrfEncodeText(p, p->pOut, zCName); - } - sqlite3_str_appendall(p->pOut, p->spec.zRowSep); - qrfWrite(p); - p->spec.eText = saved_eText; - } - for(i=0; i<p->nCol; i++){ - if( i>0 ) sqlite3_str_appendall(p->pOut, p->spec.zColumnSep); - qrfRenderValue(p, p->pOut, i); - } - sqlite3_str_appendall(p->pOut, p->spec.zRowSep); - qrfWrite(p); - break; - } - } - p->nRow++; -} - -/* -** Initialize the internal Qrf object. -*/ -static void qrfInitialize( - Qrf *p, /* State object to be initialized */ - sqlite3_stmt *pStmt, /* Query whose output to be formatted */ - const sqlite3_qrf_spec *pSpec, /* Format specification */ - char **pzErr /* Write errors here */ -){ - size_t sz; /* Size of pSpec[], based on pSpec->iVersion */ - memset(p, 0, sizeof(*p)); - p->pzErr = pzErr; - if( pSpec->iVersion!=1 ){ - qrfError(p, SQLITE_ERROR, - "unusable sqlite3_qrf_spec.iVersion (%d)", - pSpec->iVersion); - return; - } - p->pStmt = pStmt; - p->db = sqlite3_db_handle(pStmt); - p->pOut = sqlite3_str_new(p->db); - if( p->pOut==0 ){ - qrfOom(p); - return; - } - p->iErr = SQLITE_OK; - p->nCol = sqlite3_column_count(p->pStmt); - p->nRow = 0; - sz = sizeof(sqlite3_qrf_spec); - memcpy(&p->spec, pSpec, sz); - if( p->spec.zNull==0 ) p->spec.zNull = ""; - p->mxWidth = p->spec.nScreenWidth; - if( p->mxWidth<=0 ) p->mxWidth = QRF_MAX_WIDTH; - p->mxHeight = p->spec.nLineLimit; - if( p->mxHeight<=0 ) p->mxHeight = 2147483647; - if( p->spec.eStyle>QRF_STYLE_Table ) p->spec.eStyle = QRF_Auto; - if( p->spec.eEsc>QRF_ESC_Symbol ) p->spec.eEsc = QRF_Auto; - if( p->spec.eText>QRF_TEXT_Relaxed ) p->spec.eText = QRF_Auto; - if( p->spec.eTitle>QRF_TEXT_Relaxed ) p->spec.eTitle = QRF_Auto; - if( p->spec.eBlob>QRF_BLOB_Size ) p->spec.eBlob = QRF_Auto; -qrf_reinit: - switch( p->spec.eStyle ){ - case QRF_Auto: { - switch( sqlite3_stmt_isexplain(pStmt) ){ - case 0: p->spec.eStyle = QRF_STYLE_Box; break; - case 1: p->spec.eStyle = QRF_STYLE_Explain; break; - default: p->spec.eStyle = QRF_STYLE_Eqp; break; - } - goto qrf_reinit; - } - case QRF_STYLE_List: { - if( p->spec.zColumnSep==0 ) p->spec.zColumnSep = "|"; - if( p->spec.zRowSep==0 ) p->spec.zRowSep = "\n"; - break; - } - case QRF_STYLE_JObject: - case QRF_STYLE_Json: { - p->spec.eText = QRF_TEXT_Json; - p->spec.zNull = "null"; - break; - } - case QRF_STYLE_Html: { - p->spec.eText = QRF_TEXT_Html; - p->spec.zNull = "null"; - break; - } - case QRF_STYLE_Insert: { - p->spec.eText = QRF_TEXT_Sql; - p->spec.zNull = "NULL"; - if( p->spec.zTableName==0 || p->spec.zTableName[0]==0 ){ - p->spec.zTableName = "tab"; - } - break; - } - case QRF_STYLE_Line: { - if( p->spec.zColumnSep==0 ){ - p->spec.zColumnSep = ": "; - } - break; - } - case QRF_STYLE_Csv: { - p->spec.eStyle = QRF_STYLE_List; - p->spec.eText = QRF_TEXT_Csv; - p->spec.zColumnSep = ","; - p->spec.zRowSep = "\r\n"; - p->spec.zNull = ""; - break; - } - case QRF_STYLE_Quote: { - p->spec.eText = QRF_TEXT_Sql; - p->spec.zNull = "NULL"; - p->spec.zColumnSep = ","; - p->spec.zRowSep = "\n"; - break; - } - case QRF_STYLE_Eqp: { - int expMode = sqlite3_stmt_isexplain(p->pStmt); - if( expMode!=2 ){ - sqlite3_stmt_explain(p->pStmt, 2); - p->expMode = expMode+1; - } - break; - } - case QRF_STYLE_Explain: { - int expMode = sqlite3_stmt_isexplain(p->pStmt); - if( expMode!=1 ){ - sqlite3_stmt_explain(p->pStmt, 1); - p->expMode = expMode+1; - } - break; - } - } - if( p->spec.eEsc==QRF_Auto ){ - p->spec.eEsc = QRF_ESC_Ascii; - } - if( p->spec.eText==QRF_Auto ){ - p->spec.eText = QRF_TEXT_Plain; - } - if( p->spec.eTitle==QRF_Auto ){ - switch( p->spec.eStyle ){ - case QRF_STYLE_Box: - case QRF_STYLE_Column: - case QRF_STYLE_Table: - p->spec.eTitle = QRF_TEXT_Plain; - break; - default: - p->spec.eTitle = p->spec.eText; - break; - } - } - if( p->spec.eBlob==QRF_Auto ){ - switch( p->spec.eText ){ - case QRF_TEXT_Sql: p->spec.eBlob = QRF_BLOB_Sql; break; - case QRF_TEXT_Csv: p->spec.eBlob = QRF_BLOB_Tcl; break; - case QRF_TEXT_Tcl: p->spec.eBlob = QRF_BLOB_Tcl; break; - case QRF_TEXT_Json: p->spec.eBlob = QRF_BLOB_Json; break; - default: p->spec.eBlob = QRF_BLOB_Text; break; - } - } - if( p->spec.bTitles==QRF_Auto ){ - switch( p->spec.eStyle ){ - case QRF_STYLE_Box: - case QRF_STYLE_Csv: - case QRF_STYLE_Column: - case QRF_STYLE_Table: - case QRF_STYLE_Markdown: - p->spec.bTitles = QRF_Yes; - break; - default: - p->spec.bTitles = QRF_No; - break; - } - } - if( p->spec.bWordWrap==QRF_Auto ){ - p->spec.bWordWrap = QRF_Yes; - } - if( p->spec.bTextJsonb==QRF_Auto ){ - p->spec.bTextJsonb = QRF_No; - } - if( p->spec.zColumnSep==0 ) p->spec.zColumnSep = ","; - if( p->spec.zRowSep==0 ) p->spec.zRowSep = "\n"; -} - -/* -** Finish rendering the results -*/ -static void qrfFinalize(Qrf *p){ - switch( p->spec.eStyle ){ - case QRF_STYLE_Count: { - sqlite3_str_appendf(p->pOut, "%lld\n", p->nRow); - qrfWrite(p); - break; - } - case QRF_STYLE_Json: { - if( p->nRow>0 ){ - sqlite3_str_append(p->pOut, "}]\n", 3); - qrfWrite(p); - } - break; - } - case QRF_STYLE_JObject: { - if( p->nRow>0 ){ - sqlite3_str_append(p->pOut, "}\n", 2); - qrfWrite(p); - } - break; - } - case QRF_STYLE_Line: { - if( p->u.sLine.azCol ){ - int i; - for(i=0; i<p->nCol; i++) sqlite3_free(p->u.sLine.azCol[i]); - sqlite3_free(p->u.sLine.azCol); - } - break; - } - case QRF_STYLE_Stats: - case QRF_STYLE_StatsEst: { - i64 nCycle = 0; -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - sqlite3_stmt_scanstatus_v2(p->pStmt, -1, SQLITE_SCANSTAT_NCYCLE, - SQLITE_SCANSTAT_COMPLEX, (void*)&nCycle); -#endif - qrfEqpRender(p, nCycle); - qrfWrite(p); - break; - } - case QRF_STYLE_Eqp: { - qrfEqpRender(p, 0); - qrfWrite(p); - break; - } - } - qrfStrErr(p, p->pOut); - if( p->spec.pzOutput ){ - if( p->spec.pzOutput[0] ){ - sqlite3_int64 n, sz; - char *zCombined; - sz = strlen(p->spec.pzOutput[0]); - n = sqlite3_str_length(p->pOut); - zCombined = sqlite3_realloc64(p->spec.pzOutput[0], sz+n+1); - if( zCombined==0 ){ - sqlite3_free(p->spec.pzOutput[0]); - p->spec.pzOutput[0] = 0; - qrfOom(p); - }else{ - p->spec.pzOutput[0] = zCombined; - memcpy(zCombined+sz, sqlite3_str_value(p->pOut), n+1); - } - sqlite3_free(sqlite3_str_finish(p->pOut)); - }else{ - p->spec.pzOutput[0] = sqlite3_str_finish(p->pOut); - } - }else if( p->pOut ){ - sqlite3_free(sqlite3_str_finish(p->pOut)); - } - if( p->expMode>0 ){ - sqlite3_stmt_explain(p->pStmt, p->expMode-1); - } - if( p->actualWidth ){ - sqlite3_free(p->actualWidth); - } - if( p->pJTrans ){ - sqlite3 *db = sqlite3_db_handle(p->pJTrans); - sqlite3_finalize(p->pJTrans); - sqlite3_close(db); - } -} - -/* -** Run the prepared statement pStmt and format the results according -** to the specification provided in pSpec. Return an error code. -** If pzErr is not NULL and if an error occurs, write an error message -** into *pzErr. -*/ -int sqlite3_format_query_result( - sqlite3_stmt *pStmt, /* Statement to evaluate */ - const sqlite3_qrf_spec *pSpec, /* Format specification */ - char **pzErr /* Write error message here */ -){ - Qrf qrf; /* The new Qrf being created */ - - if( pStmt==0 ) return SQLITE_OK; /* No-op */ - if( pSpec==0 ) return SQLITE_MISUSE; - qrfInitialize(&qrf, pStmt, pSpec, pzErr); - switch( qrf.spec.eStyle ){ - case QRF_STYLE_Box: - case QRF_STYLE_Column: - case QRF_STYLE_Markdown: - case QRF_STYLE_Table: { - /* Columnar modes require that the entire query be evaluated and the - ** results stored in memory, so that we can compute column widths */ - qrfColumnar(&qrf); - break; - } - case QRF_STYLE_Explain: { - qrfExplain(&qrf); - break; - } - case QRF_STYLE_StatsVm: { - qrfScanStatusVm(&qrf); - break; - } - case QRF_STYLE_Stats: - case QRF_STYLE_StatsEst: { - qrfEqpStats(&qrf); - break; - } - default: { - /* Non-columnar modes where the output can occur after each row - ** of result is received */ - while( qrf.iErr==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ - qrfOneSimpleRow(&qrf); - } - break; - } - } - qrfResetStmt(&qrf); - qrfFinalize(&qrf); - return qrf.iErr; -} diff --git a/ext/qrf/qrf.h b/ext/qrf/qrf.h deleted file mode 100644 index c23ec772f..000000000 --- a/ext/qrf/qrf.h +++ /dev/null @@ -1,200 +0,0 @@ -/* -** 2025-10-20 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** Header file for the Result-Format or "resfmt" utility library for SQLite. -** See the resfmt.md documentation for additional information. -*/ -#ifndef SQLITE_QRF_H -#define SQLITE_QRF_H -#ifdef __cplusplus -extern "C" { -#endif -#include <stdlib.h> -#include "sqlite3.h" - -/* -** Specification used by clients to define the output format they want -*/ -typedef struct sqlite3_qrf_spec sqlite3_qrf_spec; -struct sqlite3_qrf_spec { - unsigned char iVersion; /* Version number of this structure */ - unsigned char eStyle; /* Formatting style. "box", "csv", etc... */ - unsigned char eEsc; /* How to escape control characters in text */ - unsigned char eText; /* Quoting style for text */ - unsigned char eTitle; /* Quating style for the text of column names */ - unsigned char eBlob; /* Quoting style for BLOBs */ - unsigned char bTitles; /* True to show column names */ - unsigned char bWordWrap; /* Try to wrap on word boundaries */ - unsigned char bTextJsonb; /* Render JSONB blobs as JSON text */ - unsigned char eDfltAlign; /* Default alignment, no covered by aAlignment */ - unsigned char eTitleAlign; /* Alignment for column headers */ - unsigned char bSplitColumn; /* Wrap single-column output into many columns */ - unsigned char bBorder; /* Show outer border in Box and Table styles */ - short int nWrap; /* Wrap columns wider than this */ - short int nScreenWidth; /* Maximum overall table width */ - short int nLineLimit; /* Maximum number of lines for any row */ - short int nTitleLimit; /* Maximum number of characters in a title */ - int nCharLimit; /* Maximum number of characters in a cell */ - int nWidth; /* Number of entries in aWidth[] */ - int nAlign; /* Number of entries in aAlignment[] */ - short int *aWidth; /* Column widths */ - unsigned char *aAlign; /* Column alignments */ - char *zColumnSep; /* Alternative column separator */ - char *zRowSep; /* Alternative row separator */ - char *zTableName; /* Output table name */ - char *zNull; /* Rendering of NULL */ - char *(*xRender)(void*,sqlite3_value*); /* Render a value */ - int (*xWrite)(void*,const char*,sqlite3_int64); /* Write output */ - void *pRenderArg; /* First argument to the xRender callback */ - void *pWriteArg; /* First argument to the xWrite callback */ - char **pzOutput; /* Storage location for output string */ - /* Additional fields may be added in the future */ -}; - -/* -** Interfaces -*/ -int sqlite3_format_query_result( - sqlite3_stmt *pStmt, /* SQL statement to run */ - const sqlite3_qrf_spec *pSpec, /* Result format specification */ - char **pzErr /* OUT: Write error message here */ -); - -/* -** Range of values for sqlite3_qrf_spec.aWidth[] entries and for -** sqlite3_qrf_spec.mxColWidth and .nScreenWidth -*/ -#define QRF_MAX_WIDTH 10000 -#define QRF_MIN_WIDTH 0 - -/* -** Output styles: -*/ -#define QRF_STYLE_Auto 0 /* Choose a style automatically */ -#define QRF_STYLE_Box 1 /* Unicode box-drawing characters */ -#define QRF_STYLE_Column 2 /* One record per line in neat columns */ -#define QRF_STYLE_Count 3 /* Output only a count of the rows of output */ -#define QRF_STYLE_Csv 4 /* Comma-separated-value */ -#define QRF_STYLE_Eqp 5 /* Format EXPLAIN QUERY PLAN output */ -#define QRF_STYLE_Explain 6 /* EXPLAIN output */ -#define QRF_STYLE_Html 7 /* Generate an XHTML table */ -#define QRF_STYLE_Insert 8 /* Generate SQL "insert" statements */ -#define QRF_STYLE_Json 9 /* Output is a list of JSON objects */ -#define QRF_STYLE_JObject 10 /* Independent JSON objects for each row */ -#define QRF_STYLE_Line 11 /* One column per line. */ -#define QRF_STYLE_List 12 /* One record per line with a separator */ -#define QRF_STYLE_Markdown 13 /* Markdown formatting */ -#define QRF_STYLE_Off 14 /* No query output shown */ -#define QRF_STYLE_Quote 15 /* SQL-quoted, comma-separated */ -#define QRF_STYLE_Stats 16 /* EQP-like output but with performance stats */ -#define QRF_STYLE_StatsEst 17 /* EQP-like output with planner estimates */ -#define QRF_STYLE_StatsVm 18 /* EXPLAIN-like output with performance stats */ -#define QRF_STYLE_Table 19 /* MySQL-style table formatting */ - -/* -** Quoting styles for text. -** Allowed values for sqlite3_qrf_spec.eText -*/ -#define QRF_TEXT_Auto 0 /* Choose text encoding automatically */ -#define QRF_TEXT_Plain 1 /* Literal text */ -#define QRF_TEXT_Sql 2 /* Quote as an SQL literal */ -#define QRF_TEXT_Csv 3 /* CSV-style quoting */ -#define QRF_TEXT_Html 4 /* HTML-style quoting */ -#define QRF_TEXT_Tcl 5 /* C/Tcl quoting */ -#define QRF_TEXT_Json 6 /* JSON quoting */ -#define QRF_TEXT_Relaxed 7 /* Relaxed SQL quoting */ - -/* -** Quoting styles for BLOBs -** Allowed values for sqlite3_qrf_spec.eBlob -*/ -#define QRF_BLOB_Auto 0 /* Determine BLOB quoting using eText */ -#define QRF_BLOB_Text 1 /* Display content exactly as it is */ -#define QRF_BLOB_Sql 2 /* Quote as an SQL literal */ -#define QRF_BLOB_Hex 3 /* Hexadecimal representation */ -#define QRF_BLOB_Tcl 4 /* "\000" notation */ -#define QRF_BLOB_Json 5 /* A JSON string */ -#define QRF_BLOB_Size 6 /* Display the blob size only */ - -/* -** Control-character escape modes. -** Allowed values for sqlite3_qrf_spec.eEsc -*/ -#define QRF_ESC_Auto 0 /* Choose the ctrl-char escape automatically */ -#define QRF_ESC_Off 1 /* Do not escape control characters */ -#define QRF_ESC_Ascii 2 /* Unix-style escapes. Ex: U+0007 shows ^G */ -#define QRF_ESC_Symbol 3 /* Unicode escapes. Ex: U+0007 shows U+2407 */ - -/* -** Allowed values for "boolean" fields, such as "bColumnNames", "bWordWrap", -** and "bTextJsonb". There is an extra "auto" variants so these are actually -** tri-state settings, not booleans. -*/ -#define QRF_SW_Auto 0 /* Let QRF choose the best value */ -#define QRF_SW_Off 1 /* This setting is forced off */ -#define QRF_SW_On 2 /* This setting is forced on */ -#define QRF_Auto 0 /* Alternate spelling for QRF_*_Auto */ -#define QRF_No 1 /* Alternate spelling for QRF_SW_Off */ -#define QRF_Yes 2 /* Alternate spelling for QRF_SW_On */ - -/* -** Possible alignment values alignment settings -** -** Horizontal Vertial -** ---------- -------- */ -#define QRF_ALIGN_Auto 0 /* auto auto */ -#define QRF_ALIGN_Left 1 /* left auto */ -#define QRF_ALIGN_Center 2 /* center auto */ -#define QRF_ALIGN_Right 3 /* right auto */ -#define QRF_ALIGN_Top 4 /* auto top */ -#define QRF_ALIGN_NW 5 /* left top */ -#define QRF_ALIGN_N 6 /* center top */ -#define QRF_ALIGN_NE 7 /* right top */ -#define QRF_ALIGN_Middle 8 /* auto middle */ -#define QRF_ALIGN_W 9 /* left middle */ -#define QRF_ALIGN_C 10 /* center middle */ -#define QRF_ALIGN_E 11 /* right middle */ -#define QRF_ALIGN_Bottom 12 /* auto bottom */ -#define QRF_ALIGN_SW 13 /* left bottom */ -#define QRF_ALIGN_S 14 /* center bottom */ -#define QRF_ALIGN_SE 15 /* right bottom */ -#define QRF_ALIGN_HMASK 3 /* Horizontal alignment mask */ -#define QRF_ALIGN_VMASK 12 /* Vertical alignment mask */ - -/* -** Auxiliary routines contined within this module that might be useful -** in other contexts, and which are therefore exported. -*/ -/* -** Return an estimate of the width, in columns, for the single Unicode -** character c. For normal characters, the answer is always 1. But the -** estimate might be 0 or 2 for zero-width and double-width characters. -** -** Different devices display unicode using different widths. So -** it is impossible to know that true display width with 100% accuracy. -** Inaccuracies in the width estimates might cause columns to be misaligned. -** Unfortunately, there is nothing we can do about that. -*/ -int sqlite3_qrf_wcwidth(int c); - -/* -** Return an estimate of the number of display columns used by the -** string in the argument. The width of individual characters is -** determined as for sqlite3_qrf_wcwidth(). VT100 escape code sequences -** are assigned a width of zero. -*/ -size_t sqlite3_qrf_wcswidth(const char*); - - -#ifdef __cplusplus -} -#endif -#endif /* !defined(SQLITE_QRF_H) */ diff --git a/ext/rbu/rbu11.test b/ext/rbu/rbu11.test index 513ab29f6..a42163cce 100644 --- a/ext/rbu/rbu11.test +++ b/ext/rbu/rbu11.test @@ -192,32 +192,4 @@ do_test 4.7.2 { list [catch {rbu close} msg] $msg } {1 {SQLITE_ERROR - rbu_state mismatch error}} -#------------------------------------------------------------------------- -# https://sqlite.org/forum/info/6d0a31e22a435877 -# -reset_db -forcedelete rbu.db - -do_execsql_test 5.0 { - CREATE TABLE t1(a BLOB); - INSERT INTO t1 VALUES(x'41'); -} - -sqlite3 dbRbu rbu.db -dbRbu eval { - CREATE TABLE data_t1(a, rbu_control, rbu_rowid); - INSERT INTO data_t1 VALUES(X'310a313a5a337e7e7e7e7e40302c303b','f',1); -} -dbRbu close - -do_test 5.1 { - sqlite3rbu rbu test.db rbu.db - rbu step -} {SQLITE_ERROR} - -do_test 5.2 { - list [catch {rbu close} msg] $msg -} {1 {SQLITE_ERROR - corrupt fossil delta}} - - finish_test diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index f377d5c30..e3bcd5fc7 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -2269,8 +2269,8 @@ static char *rbuObjIterGetIndexWhere(sqlite3rbu *p, RbuObjIter *pIter){ /* If necessary, grow the pIter->aIdxCol[] array */ if( iIdxCol==nIdxAlloc ){ - RbuSpan *aIdxCol = (RbuSpan*)sqlite3_realloc64( - pIter->aIdxCol, nIdxAlloc*sizeof(RbuSpan) + 16*sizeof(RbuSpan) + RbuSpan *aIdxCol = (RbuSpan*)sqlite3_realloc( + pIter->aIdxCol, (nIdxAlloc+16)*sizeof(RbuSpan) ); if( aIdxCol==0 ){ rc = SQLITE_NOMEM; diff --git a/ext/repair/README.md b/ext/repair/README.md new file mode 100644 index 000000000..927ceb7c4 --- /dev/null +++ b/ext/repair/README.md @@ -0,0 +1,16 @@ +This folder contains extensions and utility programs intended to analyze +live database files, detect problems, and possibly fix them. + +As SQLite is being used on larger and larger databases, database sizes +are growing into the terabyte range. At that size, hardware malfunctions +and/or cosmic rays will occasionally corrupt a database file. Detecting +problems and fixing errors a terabyte-sized databases can take hours or days, +and it is undesirable to take applications that depend on the databases +off-line for such a long time. +The utilities in the folder are intended to provide mechanisms for +detecting and fixing problems in large databases while those databases +are in active use. + +The utilities and extensions in this folder are experimental and under +active development at the time of this writing (2017-10-12). If and when +they stabilize, this README will be updated to reflect that fact. diff --git a/ext/repair/checkfreelist.c b/ext/repair/checkfreelist.c new file mode 100644 index 000000000..d1d3d5407 --- /dev/null +++ b/ext/repair/checkfreelist.c @@ -0,0 +1,310 @@ +/* +** 2017 October 11 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This module exports a single C function: +** +** int sqlite3_check_freelist(sqlite3 *db, const char *zDb); +** +** This function checks the free-list in database zDb (one of "main", +** "temp", etc.) and reports any errors by invoking the sqlite3_log() +** function. It returns SQLITE_OK if successful, or an SQLite error +** code otherwise. It is not an error if the free-list is corrupted but +** no IO or OOM errors occur. +** +** If this file is compiled and loaded as an SQLite loadable extension, +** it adds an SQL function "checkfreelist" to the database handle, to +** be invoked as follows: +** +** SELECT checkfreelist(<database-name>); +** +** This function performs the same checks as sqlite3_check_freelist(), +** except that it returns all error messages as a single text value, +** separated by newline characters. If the freelist is not corrupted +** in any way, an empty string is returned. +** +** To compile this module for use as an SQLite loadable extension: +** +** gcc -Os -fPIC -shared checkfreelist.c -o checkfreelist.so +*/ + +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 + +#ifndef SQLITE_AMALGAMATION +# include <string.h> +# include <stdio.h> +# include <stdlib.h> +# include <assert.h> +# if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) +# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1 +# endif +# if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS) +# define ALWAYS(X) (1) +# define NEVER(X) (0) +# elif !defined(NDEBUG) +# define ALWAYS(X) ((X)?1:(assert(0),0)) +# define NEVER(X) ((X)?(assert(0),1):0) +# else +# define ALWAYS(X) (X) +# define NEVER(X) (X) +# endif + typedef unsigned char u8; + typedef unsigned short u16; + typedef unsigned int u32; +#define get4byte(x) ( \ + ((u32)((x)[0])<<24) + \ + ((u32)((x)[1])<<16) + \ + ((u32)((x)[2])<<8) + \ + ((u32)((x)[3])) \ +) +#endif + +/* +** Execute a single PRAGMA statement and return the integer value returned +** via output parameter (*pnOut). +** +** The SQL statement passed as the third argument should be a printf-style +** format string containing a single "%s" which will be replace by the +** value passed as the second argument. e.g. +** +** sqlGetInteger(db, "main", "PRAGMA %s.page_count", pnOut) +** +** executes "PRAGMA main.page_count" and stores the results in (*pnOut). +*/ +static int sqlGetInteger( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Database name ("main", "temp" etc.) */ + const char *zFmt, /* SQL statement format */ + u32 *pnOut /* OUT: Integer value */ +){ + int rc, rc2; + char *zSql; + sqlite3_stmt *pStmt = 0; + int bOk = 0; + + zSql = sqlite3_mprintf(zFmt, zDb); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + *pnOut = (u32)sqlite3_column_int(pStmt, 0); + bOk = 1; + } + + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_ERROR; + return rc; +} + +/* +** Argument zFmt must be a printf-style format string and must be +** followed by its required arguments. If argument pzOut is NULL, +** then the results of printf()ing the format string are passed to +** sqlite3_log(). Otherwise, they are appended to the string +** at (*pzOut). +*/ +static int checkFreelistError(char **pzOut, const char *zFmt, ...){ + int rc = SQLITE_OK; + char *zErr = 0; + va_list ap; + + va_start(ap, zFmt); + zErr = sqlite3_vmprintf(zFmt, ap); + if( zErr==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( pzOut ){ + *pzOut = sqlite3_mprintf("%s%z%s", *pzOut?"\n":"", *pzOut, zErr); + if( *pzOut==0 ) rc = SQLITE_NOMEM; + }else{ + sqlite3_log(SQLITE_ERROR, "checkfreelist: %s", zErr); + } + sqlite3_free(zErr); + } + va_end(ap); + return rc; +} + +static int checkFreelist( + sqlite3 *db, + const char *zDb, + char **pzOut +){ + /* This query returns one row for each page on the free list. Each row has + ** two columns - the page number and page content. */ + const char *zTrunk = + "WITH freelist_trunk(i, d, n) AS (" + "SELECT 1, NULL, sqlite_readint32(data, 32) " + "FROM sqlite_dbpage(:1) WHERE pgno=1 " + "UNION ALL " + "SELECT n, data, sqlite_readint32(data) " + "FROM freelist_trunk, sqlite_dbpage(:1) WHERE pgno=n " + ")" + "SELECT i, d FROM freelist_trunk WHERE i!=1;"; + + int rc, rc2; /* Return code */ + sqlite3_stmt *pTrunk = 0; /* Compilation of zTrunk */ + u32 nPage = 0; /* Number of pages in db */ + u32 nExpected = 0; /* Expected number of free pages */ + u32 nFree = 0; /* Number of pages on free list */ + + if( zDb==0 ) zDb = "main"; + + if( (rc = sqlGetInteger(db, zDb, "PRAGMA %s.page_count", &nPage)) + || (rc = sqlGetInteger(db, zDb, "PRAGMA %s.freelist_count", &nExpected)) + ){ + return rc; + } + + rc = sqlite3_prepare_v2(db, zTrunk, -1, &pTrunk, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_text(pTrunk, 1, zDb, -1, SQLITE_STATIC); + while( rc==SQLITE_OK && sqlite3_step(pTrunk)==SQLITE_ROW ){ + u32 i; + u32 iTrunk = (u32)sqlite3_column_int(pTrunk, 0); + const u8 *aData = (const u8*)sqlite3_column_blob(pTrunk, 1); + u32 nData = (u32)sqlite3_column_bytes(pTrunk, 1); + u32 iNext = get4byte(&aData[0]); + u32 nLeaf = get4byte(&aData[4]); + + if( nLeaf>((nData/4)-2-6) ){ + rc = checkFreelistError(pzOut, + "leaf count out of range (%d) on trunk page %d", + (int)nLeaf, (int)iTrunk + ); + nLeaf = (nData/4) - 2 - 6; + } + + nFree += 1+nLeaf; + if( iNext>nPage ){ + rc = checkFreelistError(pzOut, + "trunk page %d is out of range", (int)iNext + ); + } + + for(i=0; rc==SQLITE_OK && i<nLeaf; i++){ + u32 iLeaf = get4byte(&aData[8 + 4*i]); + if( iLeaf==0 || iLeaf>nPage ){ + rc = checkFreelistError(pzOut, + "leaf page %d is out of range (child %d of trunk page %d)", + (int)iLeaf, (int)i, (int)iTrunk + ); + } + } + } + + if( rc==SQLITE_OK && nFree!=nExpected ){ + rc = checkFreelistError(pzOut, + "free-list count mismatch: actual=%d header=%d", + (int)nFree, (int)nExpected + ); + } + + rc2 = sqlite3_finalize(pTrunk); + if( rc==SQLITE_OK ) rc = rc2; + return rc; +} + +int sqlite3_check_freelist(sqlite3 *db, const char *zDb){ + return checkFreelist(db, zDb, 0); +} + +static void checkfreelist_function( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + const char *zDb; + int rc; + char *zOut = 0; + sqlite3 *db = sqlite3_context_db_handle(pCtx); + + assert( nArg==1 ); + zDb = (const char*)sqlite3_value_text(apArg[0]); + rc = checkFreelist(db, zDb, &zOut); + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, zOut?zOut:"ok", -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + + sqlite3_free(zOut); +} + +/* +** An SQL function invoked as follows: +** +** sqlite_readint32(BLOB) -- Decode 32-bit integer from start of blob +*/ +static void readint_function( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + const u8 *zBlob; + int nBlob; + int iOff = 0; + u32 iRet = 0; + + if( nArg!=1 && nArg!=2 ){ + sqlite3_result_error( + pCtx, "wrong number of arguments to function sqlite_readint32()", -1 + ); + return; + } + if( nArg==2 ){ + iOff = sqlite3_value_int(apArg[1]); + } + + zBlob = sqlite3_value_blob(apArg[0]); + nBlob = sqlite3_value_bytes(apArg[0]); + + if( nBlob>=(iOff+4) ){ + iRet = get4byte(&zBlob[iOff]); + } + + sqlite3_result_int64(pCtx, (sqlite3_int64)iRet); +} + +/* +** Register the SQL functions. +*/ +static int cflRegister(sqlite3 *db){ + int rc = sqlite3_create_function( + db, "sqlite_readint32", -1, SQLITE_UTF8, 0, readint_function, 0, 0 + ); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3_create_function( + db, "checkfreelist", 1, SQLITE_UTF8, 0, checkfreelist_function, 0, 0 + ); + return rc; +} + +/* +** Extension load function. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_checkfreelist_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + return cflRegister(db); +} diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c new file mode 100644 index 000000000..ed30357e5 --- /dev/null +++ b/ext/repair/checkindex.c @@ -0,0 +1,929 @@ +/* +** 2017 October 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 + +/* +** Stuff that is available inside the amalgamation, but which we need to +** declare ourselves if this module is compiled separately. +*/ +#ifndef SQLITE_AMALGAMATION +# include <string.h> +# include <stdio.h> +# include <stdlib.h> +# include <assert.h> +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned int u32; +#define get4byte(x) ( \ + ((u32)((x)[0])<<24) + \ + ((u32)((x)[1])<<16) + \ + ((u32)((x)[2])<<8) + \ + ((u32)((x)[3])) \ +) +#endif + +typedef struct CidxTable CidxTable; +typedef struct CidxCursor CidxCursor; + +struct CidxTable { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; +}; + +struct CidxCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + sqlite3_int64 iRowid; /* Row number of the output */ + char *zIdxName; /* Copy of the index_name parameter */ + char *zAfterKey; /* Copy of the after_key parameter */ + sqlite3_stmt *pStmt; /* SQL statement that generates the output */ +}; + +typedef struct CidxColumn CidxColumn; +struct CidxColumn { + char *zExpr; /* Text for indexed expression */ + int bDesc; /* True for DESC columns, otherwise false */ + int bKey; /* Part of index, not PK */ +}; + +typedef struct CidxIndex CidxIndex; +struct CidxIndex { + char *zWhere; /* WHERE clause, if any */ + int nCol; /* Elements in aCol[] array */ + CidxColumn aCol[1]; /* Array of indexed columns */ +}; + +static void *cidxMalloc(int *pRc, int n){ + void *pRet = 0; + assert( n!=0 ); + if( *pRc==SQLITE_OK ){ + pRet = sqlite3_malloc(n); + if( pRet ){ + memset(pRet, 0, n); + }else{ + *pRc = SQLITE_NOMEM; + } + } + return pRet; +} + +static void cidxCursorError(CidxCursor *pCsr, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + assert( pCsr->base.pVtab->zErrMsg==0 ); + pCsr->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap); + va_end(ap); +} + +/* +** Connect to the incremental_index_check virtual table. +*/ +static int cidxConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + int rc = SQLITE_OK; + CidxTable *pRet; + +#define IIC_ERRMSG 0 +#define IIC_CURRENT_KEY 1 +#define IIC_INDEX_NAME 2 +#define IIC_AFTER_KEY 3 +#define IIC_SCANNER_SQL 4 + rc = sqlite3_declare_vtab(db, + "CREATE TABLE xyz(" + " errmsg TEXT," /* Error message or NULL if everything is ok */ + " current_key TEXT," /* SQLite quote() text of key values */ + " index_name HIDDEN," /* IN: name of the index being scanned */ + " after_key HIDDEN," /* IN: Start scanning after this key */ + " scanner_sql HIDDEN" /* debugging info: SQL used for scanner */ + ")" + ); + pRet = cidxMalloc(&rc, sizeof(CidxTable)); + if( pRet ){ + pRet->db = db; + } + + *ppVtab = (sqlite3_vtab*)pRet; + return rc; +} + +/* +** Disconnect from or destroy an incremental_index_check virtual table. +*/ +static int cidxDisconnect(sqlite3_vtab *pVtab){ + CidxTable *pTab = (CidxTable*)pVtab; + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** idxNum and idxStr are not used. There are only three possible plans, +** which are all distinguished by the number of parameters. +** +** No parameters: A degenerate plan. The result is zero rows. +** 1 Parameter: Scan all of the index starting with first entry +** 2 parameters: Scan the index starting after the "after_key". +** +** Provide successively smaller costs for each of these plans to encourage +** the query planner to select the one with the most parameters. +*/ +static int cidxBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pInfo){ + int iIdxName = -1; + int iAfterKey = -1; + int i; + + for(i=0; i<pInfo->nConstraint; i++){ + struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; + if( p->usable==0 ) continue; + if( p->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + + if( p->iColumn==IIC_INDEX_NAME ){ + iIdxName = i; + } + if( p->iColumn==IIC_AFTER_KEY ){ + iAfterKey = i; + } + } + + if( iIdxName<0 ){ + pInfo->estimatedCost = 1000000000.0; + }else{ + pInfo->aConstraintUsage[iIdxName].argvIndex = 1; + pInfo->aConstraintUsage[iIdxName].omit = 1; + if( iAfterKey<0 ){ + pInfo->estimatedCost = 1000000.0; + }else{ + pInfo->aConstraintUsage[iAfterKey].argvIndex = 2; + pInfo->aConstraintUsage[iAfterKey].omit = 1; + pInfo->estimatedCost = 1000.0; + } + } + + return SQLITE_OK; +} + +/* +** Open a new btreeinfo cursor. +*/ +static int cidxOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + CidxCursor *pRet; + int rc = SQLITE_OK; + + pRet = cidxMalloc(&rc, sizeof(CidxCursor)); + + *ppCursor = (sqlite3_vtab_cursor*)pRet; + return rc; +} + +/* +** Close a btreeinfo cursor. +*/ +static int cidxClose(sqlite3_vtab_cursor *pCursor){ + CidxCursor *pCsr = (CidxCursor*)pCursor; + sqlite3_finalize(pCsr->pStmt); + sqlite3_free(pCsr->zIdxName); + sqlite3_free(pCsr->zAfterKey); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Move a btreeinfo cursor to the next entry in the file. +*/ +static int cidxNext(sqlite3_vtab_cursor *pCursor){ + CidxCursor *pCsr = (CidxCursor*)pCursor; + int rc = sqlite3_step(pCsr->pStmt); + if( rc!=SQLITE_ROW ){ + rc = sqlite3_finalize(pCsr->pStmt); + pCsr->pStmt = 0; + if( rc!=SQLITE_OK ){ + sqlite3 *db = ((CidxTable*)pCsr->base.pVtab)->db; + cidxCursorError(pCsr, "Cursor error: %s", sqlite3_errmsg(db)); + } + }else{ + pCsr->iRowid++; + rc = SQLITE_OK; + } + return rc; +} + +/* We have reached EOF if previous sqlite3_step() returned +** anything other than SQLITE_ROW; +*/ +static int cidxEof(sqlite3_vtab_cursor *pCursor){ + CidxCursor *pCsr = (CidxCursor*)pCursor; + return pCsr->pStmt==0; +} + +static char *cidxMprintf(int *pRc, const char *zFmt, ...){ + char *zRet = 0; + va_list ap; + va_start(ap, zFmt); + zRet = sqlite3_vmprintf(zFmt, ap); + if( *pRc==SQLITE_OK ){ + if( zRet==0 ){ + *pRc = SQLITE_NOMEM; + } + }else{ + sqlite3_free(zRet); + zRet = 0; + } + va_end(ap); + return zRet; +} + +static sqlite3_stmt *cidxPrepare( + int *pRc, CidxCursor *pCsr, const char *zFmt, ... +){ + sqlite3_stmt *pRet = 0; + char *zSql; + va_list ap; /* ... printf arguments */ + va_start(ap, zFmt); + + zSql = sqlite3_vmprintf(zFmt, ap); + if( *pRc==SQLITE_OK ){ + if( zSql==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + sqlite3 *db = ((CidxTable*)pCsr->base.pVtab)->db; + *pRc = sqlite3_prepare_v2(db, zSql, -1, &pRet, 0); + if( *pRc!=SQLITE_OK ){ + cidxCursorError(pCsr, "SQL error: %s", sqlite3_errmsg(db)); + } + } + } + sqlite3_free(zSql); + va_end(ap); + + return pRet; +} + +static void cidxFinalize(int *pRc, sqlite3_stmt *pStmt){ + int rc = sqlite3_finalize(pStmt); + if( *pRc==SQLITE_OK ) *pRc = rc; +} + +char *cidxStrdup(int *pRc, const char *zStr){ + char *zRet = 0; + if( *pRc==SQLITE_OK ){ + int n = (int)strlen(zStr); + zRet = cidxMalloc(pRc, n+1); + if( zRet ) memcpy(zRet, zStr, n+1); + } + return zRet; +} + +static void cidxFreeIndex(CidxIndex *pIdx){ + if( pIdx ){ + int i; + for(i=0; i<pIdx->nCol; i++){ + sqlite3_free(pIdx->aCol[i].zExpr); + } + sqlite3_free(pIdx->zWhere); + sqlite3_free(pIdx); + } +} + +static int cidx_isspace(char c){ + return c==' ' || c=='\t' || c=='\r' || c=='\n'; +} + +static int cidx_isident(char c){ + return c<0 + || (c>='0' && c<='9') || (c>='a' && c<='z') + || (c>='A' && c<='Z') || c=='_'; +} + +#define CIDX_PARSE_EOF 0 +#define CIDX_PARSE_COMMA 1 /* "," */ +#define CIDX_PARSE_OPEN 2 /* "(" */ +#define CIDX_PARSE_CLOSE 3 /* ")" */ + +/* +** Argument zIn points into the start, middle or end of a CREATE INDEX +** statement. If argument pbDoNotTrim is non-NULL, then this function +** scans the input until it finds EOF, a comma (",") or an open or +** close parenthesis character. It then sets (*pzOut) to point to said +** character and returns a CIDX_PARSE_XXX constant as appropriate. The +** parser is smart enough that special characters inside SQL strings +** or comments are not returned for. +** +** Or, if argument pbDoNotTrim is NULL, then this function sets *pzOut +** to point to the first character of the string that is not whitespace +** or part of an SQL comment and returns CIDX_PARSE_EOF. +** +** Additionally, if pbDoNotTrim is not NULL and the element immediately +** before (*pzOut) is an SQL comment of the form "-- comment", then +** (*pbDoNotTrim) is set before returning. In all other cases it is +** cleared. +*/ +static int cidxFindNext( + const char *zIn, + const char **pzOut, + int *pbDoNotTrim /* OUT: True if prev is -- comment */ +){ + const char *z = zIn; + + while( 1 ){ + while( cidx_isspace(*z) ) z++; + if( z[0]=='-' && z[1]=='-' ){ + z += 2; + while( z[0]!='\n' ){ + if( z[0]=='\0' ) return CIDX_PARSE_EOF; + z++; + } + while( cidx_isspace(*z) ) z++; + if( pbDoNotTrim ) *pbDoNotTrim = 1; + }else + if( z[0]=='/' && z[1]=='*' ){ + z += 2; + while( z[0]!='*' || z[1]!='/' ){ + if( z[1]=='\0' ) return CIDX_PARSE_EOF; + z++; + } + z += 2; + }else{ + *pzOut = z; + if( pbDoNotTrim==0 ) return CIDX_PARSE_EOF; + switch( *z ){ + case '\0': + return CIDX_PARSE_EOF; + case '(': + return CIDX_PARSE_OPEN; + case ')': + return CIDX_PARSE_CLOSE; + case ',': + return CIDX_PARSE_COMMA; + + case '"': + case '\'': + case '`': { + char q = *z; + z++; + while( *z ){ + if( *z==q ){ + z++; + if( *z!=q ) break; + } + z++; + } + break; + } + + case '[': + while( *z++!=']' ); + break; + + default: + z++; + break; + } + *pbDoNotTrim = 0; + } + } + + assert( 0 ); + return -1; +} + +static int cidxParseSQL(CidxCursor *pCsr, CidxIndex *pIdx, const char *zSql){ + const char *z = zSql; + const char *z1; + int e; + int rc = SQLITE_OK; + int nParen = 1; + int bDoNotTrim = 0; + CidxColumn *pCol = pIdx->aCol; + + e = cidxFindNext(z, &z, &bDoNotTrim); + if( e!=CIDX_PARSE_OPEN ) goto parse_error; + z1 = z+1; + z++; + while( nParen>0 ){ + e = cidxFindNext(z, &z, &bDoNotTrim); + if( e==CIDX_PARSE_EOF ) goto parse_error; + if( (e==CIDX_PARSE_COMMA || e==CIDX_PARSE_CLOSE) && nParen==1 ){ + const char *z2 = z; + if( pCol->zExpr ) goto parse_error; + + if( bDoNotTrim==0 ){ + while( cidx_isspace(z[-1]) ) z--; + if( !sqlite3_strnicmp(&z[-3], "asc", 3) && 0==cidx_isident(z[-4]) ){ + z -= 3; + while( cidx_isspace(z[-1]) ) z--; + }else + if( !sqlite3_strnicmp(&z[-4], "desc", 4) && 0==cidx_isident(z[-5]) ){ + z -= 4; + while( cidx_isspace(z[-1]) ) z--; + } + while( cidx_isspace(z1[0]) ) z1++; + } + + pCol->zExpr = cidxMprintf(&rc, "%.*s", z-z1, z1); + pCol++; + z = z1 = z2+1; + } + if( e==CIDX_PARSE_OPEN ) nParen++; + if( e==CIDX_PARSE_CLOSE ) nParen--; + z++; + } + + /* Search for a WHERE clause */ + cidxFindNext(z, &z, 0); + if( 0==sqlite3_strnicmp(z, "where", 5) ){ + pIdx->zWhere = cidxMprintf(&rc, "%s\n", &z[5]); + }else if( z[0]!='\0' ){ + goto parse_error; + } + + return rc; + + parse_error: + cidxCursorError(pCsr, "Parse error in: %s", zSql); + return SQLITE_ERROR; +} + +static int cidxLookupIndex( + CidxCursor *pCsr, /* Cursor object */ + const char *zIdx, /* Name of index to look up */ + CidxIndex **ppIdx, /* OUT: Description of columns */ + char **pzTab /* OUT: Table name */ +){ + int rc = SQLITE_OK; + char *zTab = 0; + CidxIndex *pIdx = 0; + + sqlite3_stmt *pFindTab = 0; + sqlite3_stmt *pInfo = 0; + + /* Find the table for this index. */ + pFindTab = cidxPrepare(&rc, pCsr, + "SELECT tbl_name, sql FROM sqlite_schema WHERE name=%Q AND type='index'", + zIdx + ); + if( rc==SQLITE_OK && sqlite3_step(pFindTab)==SQLITE_ROW ){ + const char *zSql = (const char*)sqlite3_column_text(pFindTab, 1); + zTab = cidxStrdup(&rc, (const char*)sqlite3_column_text(pFindTab, 0)); + + pInfo = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx); + if( rc==SQLITE_OK ){ + int nAlloc = 0; + int iCol = 0; + + while( sqlite3_step(pInfo)==SQLITE_ROW ){ + const char *zName = (const char*)sqlite3_column_text(pInfo, 2); + const char *zColl = (const char*)sqlite3_column_text(pInfo, 4); + CidxColumn *p; + if( zName==0 ) zName = "rowid"; + if( iCol==nAlloc ){ + int nByte = sizeof(CidxIndex) + sizeof(CidxColumn)*(nAlloc+8); + pIdx = (CidxIndex*)sqlite3_realloc(pIdx, nByte); + nAlloc += 8; + } + p = &pIdx->aCol[iCol++]; + p->bDesc = sqlite3_column_int(pInfo, 3); + p->bKey = sqlite3_column_int(pInfo, 5); + if( zSql==0 || p->bKey==0 ){ + p->zExpr = cidxMprintf(&rc, "\"%w\" COLLATE %s",zName,zColl); + }else{ + p->zExpr = 0; + } + pIdx->nCol = iCol; + pIdx->zWhere = 0; + } + cidxFinalize(&rc, pInfo); + } + + if( rc==SQLITE_OK && zSql ){ + rc = cidxParseSQL(pCsr, pIdx, zSql); + } + } + + cidxFinalize(&rc, pFindTab); + if( rc==SQLITE_OK && zTab==0 ){ + rc = SQLITE_ERROR; + } + + if( rc!=SQLITE_OK ){ + sqlite3_free(zTab); + cidxFreeIndex(pIdx); + }else{ + *pzTab = zTab; + *ppIdx = pIdx; + } + + return rc; +} + +static int cidxDecodeAfter( + CidxCursor *pCsr, + int nCol, + const char *zAfterKey, + char ***pazAfter +){ + char **azAfter; + int rc = SQLITE_OK; + int nAfterKey = (int)strlen(zAfterKey); + + azAfter = cidxMalloc(&rc, sizeof(char*)*nCol + nAfterKey+1); + if( rc==SQLITE_OK ){ + int i; + char *zCopy = (char*)&azAfter[nCol]; + char *p = zCopy; + memcpy(zCopy, zAfterKey, nAfterKey+1); + for(i=0; i<nCol; i++){ + while( *p==' ' ) p++; + + /* Check NULL values */ + if( *p=='N' ){ + if( memcmp(p, "NULL", 4) ) goto parse_error; + p += 4; + } + + /* Check strings and blob literals */ + else if( *p=='X' || *p=='\'' ){ + azAfter[i] = p; + if( *p=='X' ) p++; + if( *p!='\'' ) goto parse_error; + p++; + while( 1 ){ + if( *p=='\0' ) goto parse_error; + if( *p=='\'' ){ + p++; + if( *p!='\'' ) break; + } + p++; + } + } + + /* Check numbers */ + else{ + azAfter[i] = p; + while( (*p>='0' && *p<='9') + || *p=='.' || *p=='+' || *p=='-' || *p=='e' || *p=='E' + ){ + p++; + } + } + + while( *p==' ' ) p++; + if( *p!=(i==(nCol-1) ? '\0' : ',') ){ + goto parse_error; + } + *p++ = '\0'; + } + } + + *pazAfter = azAfter; + return rc; + + parse_error: + sqlite3_free(azAfter); + *pazAfter = 0; + cidxCursorError(pCsr, "%s", "error parsing after value"); + return SQLITE_ERROR; +} + +static char *cidxWhere( + int *pRc, CidxColumn *aCol, char **azAfter, int iGt, int bLastIsNull +){ + char *zRet = 0; + const char *zSep = ""; + int i; + + for(i=0; i<iGt; i++){ + zRet = cidxMprintf(pRc, "%z%s(%s) IS %s", zRet, + zSep, aCol[i].zExpr, (azAfter[i] ? azAfter[i] : "NULL") + ); + zSep = " AND "; + } + + if( bLastIsNull ){ + zRet = cidxMprintf(pRc, "%z%s(%s) IS NULL", zRet, zSep, aCol[iGt].zExpr); + } + else if( azAfter[iGt] ){ + zRet = cidxMprintf(pRc, "%z%s(%s) %s %s", zRet, + zSep, aCol[iGt].zExpr, (aCol[iGt].bDesc ? "<" : ">"), + azAfter[iGt] + ); + }else{ + zRet = cidxMprintf(pRc, "%z%s(%s) IS NOT NULL", zRet, zSep,aCol[iGt].zExpr); + } + + return zRet; +} + +#define CIDX_CLIST_ALL 0 +#define CIDX_CLIST_ORDERBY 1 +#define CIDX_CLIST_CURRENT_KEY 2 +#define CIDX_CLIST_SUBWHERE 3 +#define CIDX_CLIST_SUBEXPR 4 + +/* +** This function returns various strings based on the contents of the +** CidxIndex structure and the eType parameter. +*/ +static char *cidxColumnList( + int *pRc, /* IN/OUT: Error code */ + const char *zIdx, + CidxIndex *pIdx, /* Indexed columns */ + int eType /* True to include ASC/DESC */ +){ + char *zRet = 0; + if( *pRc==SQLITE_OK ){ + const char *aDir[2] = {"", " DESC"}; + int i; + const char *zSep = ""; + + for(i=0; i<pIdx->nCol; i++){ + CidxColumn *p = &pIdx->aCol[i]; + assert( pIdx->aCol[i].bDesc==0 || pIdx->aCol[i].bDesc==1 ); + switch( eType ){ + + case CIDX_CLIST_ORDERBY: + zRet = cidxMprintf(pRc, "%z%s%d%s", zRet, zSep, i+1, aDir[p->bDesc]); + zSep = ","; + break; + + case CIDX_CLIST_CURRENT_KEY: + zRet = cidxMprintf(pRc, "%z%squote(i%d)", zRet, zSep, i); + zSep = "||','||"; + break; + + case CIDX_CLIST_SUBWHERE: + if( p->bKey==0 ){ + zRet = cidxMprintf(pRc, "%z%s%s IS i.i%d", zRet, + zSep, p->zExpr, i + ); + zSep = " AND "; + } + break; + + case CIDX_CLIST_SUBEXPR: + if( p->bKey==1 ){ + zRet = cidxMprintf(pRc, "%z%s%s IS i.i%d", zRet, + zSep, p->zExpr, i + ); + zSep = " AND "; + } + break; + + default: + assert( eType==CIDX_CLIST_ALL ); + zRet = cidxMprintf(pRc, "%z%s(%s) AS i%d", zRet, zSep, p->zExpr, i); + zSep = ", "; + break; + } + } + } + + return zRet; +} + +/* +** Generate SQL (in memory obtained from sqlite3_malloc()) that will +** continue the index scan for zIdxName starting after zAfterKey. +*/ +int cidxGenerateScanSql( + CidxCursor *pCsr, /* The cursor which needs the new statement */ + const char *zIdxName, /* index to be scanned */ + const char *zAfterKey, /* start after this key, if not NULL */ + char **pzSqlOut /* OUT: Write the generated SQL here */ +){ + int rc; + char *zTab = 0; + char *zCurrentKey = 0; + char *zOrderBy = 0; + char *zSubWhere = 0; + char *zSubExpr = 0; + char *zSrcList = 0; + char **azAfter = 0; + CidxIndex *pIdx = 0; + + *pzSqlOut = 0; + rc = cidxLookupIndex(pCsr, zIdxName, &pIdx, &zTab); + + zOrderBy = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ORDERBY); + zCurrentKey = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_CURRENT_KEY); + zSubWhere = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBWHERE); + zSubExpr = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBEXPR); + zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL); + + if( rc==SQLITE_OK && zAfterKey ){ + rc = cidxDecodeAfter(pCsr, pIdx->nCol, zAfterKey, &azAfter); + } + + if( rc==SQLITE_OK ){ + if( zAfterKey==0 ){ + *pzSqlOut = cidxMprintf(&rc, + "SELECT (SELECT %s FROM %Q AS t WHERE %s), %s " + "FROM (SELECT %s FROM %Q INDEXED BY %Q %s%sORDER BY %s) AS i", + zSubExpr, zTab, zSubWhere, zCurrentKey, + zSrcList, zTab, zIdxName, + (pIdx->zWhere ? "WHERE " : ""), (pIdx->zWhere ? pIdx->zWhere : ""), + zOrderBy + ); + }else{ + const char *zSep = ""; + char *zSql; + int i; + + zSql = cidxMprintf(&rc, + "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (", + zSubExpr, zTab, zSubWhere, zCurrentKey + ); + for(i=pIdx->nCol-1; i>=0; i--){ + int j; + if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue; + for(j=0; j<2; j++){ + char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j); + zSql = cidxMprintf(&rc, "%z" + "%sSELECT * FROM (" + "SELECT %s FROM %Q INDEXED BY %Q WHERE %s%s%z ORDER BY %s" + ")", + zSql, zSep, zSrcList, zTab, zIdxName, + pIdx->zWhere ? pIdx->zWhere : "", + pIdx->zWhere ? " AND " : "", + zWhere, zOrderBy + ); + zSep = " UNION ALL "; + if( pIdx->aCol[i].bDesc==0 ) break; + } + } + *pzSqlOut = cidxMprintf(&rc, "%z) AS i", zSql); + } + } + + sqlite3_free(zTab); + sqlite3_free(zCurrentKey); + sqlite3_free(zOrderBy); + sqlite3_free(zSubWhere); + sqlite3_free(zSubExpr); + sqlite3_free(zSrcList); + cidxFreeIndex(pIdx); + sqlite3_free(azAfter); + return rc; +} + + +/* +** Position a cursor back to the beginning. +*/ +static int cidxFilter( + sqlite3_vtab_cursor *pCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + int rc = SQLITE_OK; + CidxCursor *pCsr = (CidxCursor*)pCursor; + const char *zIdxName = 0; + const char *zAfterKey = 0; + + sqlite3_free(pCsr->zIdxName); + pCsr->zIdxName = 0; + sqlite3_free(pCsr->zAfterKey); + pCsr->zAfterKey = 0; + sqlite3_finalize(pCsr->pStmt); + pCsr->pStmt = 0; + + if( argc>0 ){ + zIdxName = (const char*)sqlite3_value_text(argv[0]); + if( argc>1 ){ + zAfterKey = (const char*)sqlite3_value_text(argv[1]); + } + } + + if( zIdxName ){ + char *zSql = 0; + pCsr->zIdxName = sqlite3_mprintf("%s", zIdxName); + pCsr->zAfterKey = zAfterKey ? sqlite3_mprintf("%s", zAfterKey) : 0; + rc = cidxGenerateScanSql(pCsr, zIdxName, zAfterKey, &zSql); + if( zSql ){ + pCsr->pStmt = cidxPrepare(&rc, pCsr, "%z", zSql); + } + } + + if( pCsr->pStmt ){ + assert( rc==SQLITE_OK ); + rc = cidxNext(pCursor); + } + pCsr->iRowid = 1; + return rc; +} + +/* +** Return a column value. +*/ +static int cidxColumn( + sqlite3_vtab_cursor *pCursor, + sqlite3_context *ctx, + int iCol +){ + CidxCursor *pCsr = (CidxCursor*)pCursor; + assert( iCol>=IIC_ERRMSG && iCol<=IIC_SCANNER_SQL ); + switch( iCol ){ + case IIC_ERRMSG: { + const char *zVal = 0; + if( sqlite3_column_type(pCsr->pStmt, 0)==SQLITE_INTEGER ){ + if( sqlite3_column_int(pCsr->pStmt, 0)==0 ){ + zVal = "row data mismatch"; + } + }else{ + zVal = "row missing"; + } + sqlite3_result_text(ctx, zVal, -1, SQLITE_STATIC); + break; + } + case IIC_CURRENT_KEY: { + sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, 1)); + break; + } + case IIC_INDEX_NAME: { + sqlite3_result_text(ctx, pCsr->zIdxName, -1, SQLITE_TRANSIENT); + break; + } + case IIC_AFTER_KEY: { + sqlite3_result_text(ctx, pCsr->zAfterKey, -1, SQLITE_TRANSIENT); + break; + } + case IIC_SCANNER_SQL: { + char *zSql = 0; + cidxGenerateScanSql(pCsr, pCsr->zIdxName, pCsr->zAfterKey, &zSql); + sqlite3_result_text(ctx, zSql, -1, sqlite3_free); + break; + } + } + return SQLITE_OK; +} + +/* Return the ROWID for the sqlite_btreeinfo table */ +static int cidxRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + CidxCursor *pCsr = (CidxCursor*)pCursor; + *pRowid = pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the virtual table modules with the database handle passed +** as the only argument. +*/ +static int ciInit(sqlite3 *db){ + static sqlite3_module cidx_module = { + 0, /* iVersion */ + 0, /* xCreate */ + cidxConnect, /* xConnect */ + cidxBestIndex, /* xBestIndex */ + cidxDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + cidxOpen, /* xOpen - open a cursor */ + cidxClose, /* xClose - close a cursor */ + cidxFilter, /* xFilter - configure scan constraints */ + cidxNext, /* xNext - advance a cursor */ + cidxEof, /* xEof - check for end of scan */ + cidxColumn, /* xColumn - read data */ + cidxRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ + }; + return sqlite3_create_module(db, "incremental_index_check", &cidx_module, 0); +} + +/* +** Extension load function. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_checkindex_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + return ciInit(db); +} diff --git a/ext/repair/sqlite3_checker.c.in b/ext/repair/sqlite3_checker.c.in new file mode 100644 index 000000000..96b15f271 --- /dev/null +++ b/ext/repair/sqlite3_checker.c.in @@ -0,0 +1,85 @@ +/* +** Read an SQLite database file and analyze its space utilization. Generate +** text on standard output. +*/ +#define TCLSH_INIT_PROC sqlite3_checker_init_proc +#define SQLITE_ENABLE_DBPAGE_VTAB 1 +#undef SQLITE_THREADSAFE +#define SQLITE_THREADSAFE 0 +#undef SQLITE_ENABLE_COLUMN_METADATA +#define SQLITE_OMIT_DECLTYPE 1 +#define SQLITE_OMIT_DEPRECATED 1 +#define SQLITE_OMIT_PROGRESS_CALLBACK 1 +#define SQLITE_OMIT_SHARED_CACHE 1 +#define SQLITE_DEFAULT_MEMSTATUS 0 +#define SQLITE_MAX_EXPR_DEPTH 0 +INCLUDE sqlite3.c +INCLUDE $ROOT/src/tclsqlite.c +INCLUDE $ROOT/ext/misc/btreeinfo.c +INCLUDE $ROOT/ext/repair/checkindex.c +INCLUDE $ROOT/ext/repair/checkfreelist.c + +/* +** Decode a pointer to an sqlite3 object. +*/ +int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb){ + struct SqliteDb *p; + Tcl_CmdInfo cmdInfo; + if( Tcl_GetCommandInfo(interp, zA, &cmdInfo) ){ + p = (struct SqliteDb*)cmdInfo.objClientData; + *ppDb = p->db; + return TCL_OK; + }else{ + *ppDb = 0; + return TCL_ERROR; + } + return TCL_OK; +} + +/* +** sqlite3_imposter db main rootpage {CREATE TABLE...} ;# setup an imposter +** sqlite3_imposter db main ;# rm all imposters +*/ +static int sqlite3_imposter( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; + const char *zSchema; + int iRoot; + const char *zSql; + + if( objc!=3 && objc!=5 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB SCHEMA [ROOTPAGE SQL]"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zSchema = Tcl_GetString(objv[2]); + if( objc==3 ){ + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 0, 1); + }else{ + if( Tcl_GetIntFromObj(interp, objv[3], &iRoot) ) return TCL_ERROR; + zSql = Tcl_GetString(objv[4]); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 1, iRoot); + sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 0, 0); + } + return TCL_OK; +} + +#include <stdio.h> + +const char *sqlite3_checker_init_proc(Tcl_Interp *interp){ + Tcl_CreateObjCommand(interp, "sqlite3_imposter", + (Tcl_ObjCmdProc*)sqlite3_imposter, 0, 0); + sqlite3_auto_extension((void(*)(void))sqlite3_btreeinfo_init); + sqlite3_auto_extension((void(*)(void))sqlite3_checkindex_init); + sqlite3_auto_extension((void(*)(void))sqlite3_checkfreelist_init); + return +BEGIN_STRING +INCLUDE $ROOT/ext/repair/sqlite3_checker.tcl +END_STRING +; +} diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl new file mode 100644 index 000000000..2ae6e15b1 --- /dev/null +++ b/ext/repair/sqlite3_checker.tcl @@ -0,0 +1,264 @@ +# This TCL script is the main driver script for the sqlite3_checker utility +# program. +# + +# Special case: +# +# sqlite3_checker --test FILENAME ARGS +# +# uses FILENAME in place of this script. +# +if {[lindex $argv 0]=="--test" && [llength $argv]>1} { + set ::argv0 [lindex $argv 1] + set argv [lrange $argv 2 end] + source $argv0 + exit 0 +} + +# Emulate a TCL shell +# +proc tclsh {} { + set line {} + while {![eof stdin]} { + if {$line!=""} { + puts -nonewline "> " + } else { + puts -nonewline "% " + } + flush stdout + append line [gets stdin] + if {[info complete $line]} { + if {[catch {uplevel #0 $line} result]} { + puts stderr "Error: $result" + } elseif {$result!=""} { + puts $result + } + set line {} + } else { + append line \n + } + } +} + +# Do an incremental integrity check of a single index +# +proc check_index {idxname batchsize bTrace} { + set i 0 + set more 1 + set nerr 0 + set pct 00.0 + set max [db one {SELECT nEntry FROM sqlite_btreeinfo('main') + WHERE name=$idxname}] + puts -nonewline "$idxname: $i of $max rows ($pct%)\r" + flush stdout + if {$bTrace} { + set sql {SELECT errmsg, current_key AS key, + CASE WHEN rowid=1 THEN scanner_sql END AS traceOut + FROM incremental_index_check($idxname) + WHERE after_key=$key + LIMIT $batchsize} + } else { + set sql {SELECT errmsg, current_key AS key, NULL AS traceOut + FROM incremental_index_check($idxname) + WHERE after_key=$key + LIMIT $batchsize} + } + while {$more} { + set more 0 + db eval $sql { + set more 1 + if {$errmsg!=""} { + incr nerr + puts "$idxname: key($key): $errmsg" + } elseif {$traceOut!=""} { + puts "$idxname: $traceOut" + } + incr i + + } + set x [format {%.1f} [expr {($i*100.0)/$max}]] + if {$x!=$pct} { + puts -nonewline "$idxname: $i of $max rows ($pct%)\r" + flush stdout + set pct $x + } + } + puts "$idxname: $nerr errors out of $i entries" +} + +# Print a usage message on standard error, then quit. +# +proc usage {} { + set argv0 [file rootname [file tail [info nameofexecutable]]] + puts stderr "Usage: $argv0 OPTIONS database-filename" + puts stderr { +Do sanity checking on a live SQLite3 database file specified by the +"database-filename" argument. + +Options: + + --batchsize N Number of rows to check per transaction + + --freelist Perform a freelist check + + --index NAME Run a check of the index NAME + + --summary Print summary information about the database + + --table NAME Run a check of all indexes for table NAME + + --tclsh Run the built-in TCL interpreter (for debugging) + + --trace (Debugging only:) Output trace information on the scan + + --version Show the version number of SQLite +} + exit 1 +} + +set file_to_analyze {} +append argv {} +set bFreelistCheck 0 +set bSummary 0 +set zIndex {} +set zTable {} +set batchsize 1000 +set bAll 1 +set bTrace 0 +set argc [llength $argv] +for {set i 0} {$i<$argc} {incr i} { + set arg [lindex $argv $i] + if {[regexp {^-+tclsh$} $arg]} { + tclsh + exit 0 + } + if {[regexp {^-+version$} $arg]} { + sqlite3 mem :memory: + puts [mem one {SELECT sqlite_version()||' '||sqlite_source_id()}] + mem close + exit 0 + } + if {[regexp {^-+freelist$} $arg]} { + set bFreelistCheck 1 + set bAll 0 + continue + } + if {[regexp {^-+summary$} $arg]} { + set bSummary 1 + set bAll 0 + continue + } + if {[regexp {^-+trace$} $arg]} { + set bTrace 1 + continue + } + if {[regexp {^-+batchsize$} $arg]} { + incr i + if {$i>=$argc} { + puts stderr "missing argument on $arg" + exit 1 + } + set batchsize [lindex $argv $i] + continue + } + if {[regexp {^-+index$} $arg]} { + incr i + if {$i>=$argc} { + puts stderr "missing argument on $arg" + exit 1 + } + set zIndex [lindex $argv $i] + set bAll 0 + continue + } + if {[regexp {^-+table$} $arg]} { + incr i + if {$i>=$argc} { + puts stderr "missing argument on $arg" + exit 1 + } + set zTable [lindex $argv $i] + set bAll 0 + continue + } + if {[regexp {^-} $arg]} { + puts stderr "Unknown option: $arg" + usage + } + if {$file_to_analyze!=""} { + usage + } else { + set file_to_analyze $arg + } +} +if {$file_to_analyze==""} usage + +# If a TCL script is specified on the command-line, then run that +# script. +# +if {[file extension $file_to_analyze]==".tcl"} { + source $file_to_analyze + exit 0 +} + +set root_filename $file_to_analyze +regexp {^file:(//)?([^?]*)} $file_to_analyze all x1 root_filename +if {![file exists $root_filename]} { + puts stderr "No such file: $root_filename" + exit 1 +} +if {![file readable $root_filename]} { + puts stderr "File is not readable: $root_filename" + exit 1 +} + +if {[catch {sqlite3 db $file_to_analyze} res]} { + puts stderr "Cannot open datababase $root_filename: $res" + exit 1 +} + +if {$bFreelistCheck || $bAll} { + puts -nonewline "freelist-check: " + flush stdout + db eval BEGIN + puts [db one {SELECT checkfreelist('main')}] + db eval END +} +if {$bSummary} { + set scale 0 + set pgsz [db one {PRAGMA page_size}] + db eval {SELECT nPage*$pgsz AS sz, name, tbl_name + FROM sqlite_btreeinfo + WHERE type='index' + ORDER BY 1 DESC, name} { + if {$scale==0} { + if {$sz>10000000} { + set scale 1000000.0 + set unit MB + } else { + set scale 1000.0 + set unit KB + } + } + puts [format {%7.1f %s index %s of table %s} \ + [expr {$sz/$scale}] $unit $name $tbl_name] + } +} +if {$zIndex!=""} { + check_index $zIndex $batchsize $bTrace +} +if {$zTable!=""} { + foreach idx [db eval {SELECT name FROM sqlite_master + WHERE type='index' AND rootpage>0 + AND tbl_name=$zTable}] { + check_index $idx $batchsize $bTrace + } +} +if {$bAll} { + set allidx [db eval {SELECT name FROM sqlite_btreeinfo('main') + WHERE type='index' AND rootpage>0 + ORDER BY nEntry}] + foreach idx $allidx { + check_index $idx $batchsize $bTrace + } +} diff --git a/ext/repair/test/README.md b/ext/repair/test/README.md new file mode 100644 index 000000000..8cc954adf --- /dev/null +++ b/ext/repair/test/README.md @@ -0,0 +1,13 @@ +To run these tests, first build sqlite3_checker: + + +> make sqlite3_checker + + +Then run the "test.tcl" script using: + + +> ./sqlite3_checker --test $path/test.tcl + + +Optionally add the full pathnames of individual *.test modules diff --git a/ext/repair/test/checkfreelist01.test b/ext/repair/test/checkfreelist01.test new file mode 100644 index 000000000..7e2dd51c3 --- /dev/null +++ b/ext/repair/test/checkfreelist01.test @@ -0,0 +1,92 @@ +# 2017-10-11 + +set testprefix checkfreelist + +do_execsql_test 1.0 { + PRAGMA page_size=1024; + CREATE TABLE t1(a, b); +} + +do_execsql_test 1.2 { SELECT checkfreelist('main') } {ok} +do_execsql_test 1.3 { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10000 + ) + INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s; + DELETE FROM t1 WHERE rowid%3; + PRAGMA freelist_count; +} {6726} + +do_execsql_test 1.4 { SELECT checkfreelist('main') } {ok} +do_execsql_test 1.5 { + WITH freelist_trunk(i, d, n) AS ( + SELECT 1, NULL, sqlite_readint32(data, 32) FROM sqlite_dbpage WHERE pgno=1 + UNION ALL + SELECT n, data, sqlite_readint32(data) + FROM freelist_trunk, sqlite_dbpage WHERE pgno=n + ) + SELECT i FROM freelist_trunk WHERE i!=1; +} { + 10009 9715 9343 8969 8595 8222 7847 7474 7102 6727 6354 5982 5608 5234 + 4860 4487 4112 3740 3367 2992 2619 2247 1872 1499 1125 752 377 5 +} + +do_execsql_test 1.6 { SELECT checkfreelist('main') } {ok} + +proc set_int {blob idx newval} { + binary scan $blob I* ints + lset ints $idx $newval + binary format I* $ints +} +db func set_int set_int + +proc get_int {blob idx} { + binary scan $blob I* ints + lindex $ints $idx +} +db func get_int get_int + +do_execsql_test 1.7 { + BEGIN; + UPDATE sqlite_dbpage + SET data = set_int(data, 1, get_int(data, 1)-1) + WHERE pgno=4860; + SELECT checkfreelist('main'); + ROLLBACK; +} {{free-list count mismatch: actual=6725 header=6726}} + +do_execsql_test 1.8 { + BEGIN; + UPDATE sqlite_dbpage + SET data = set_int(data, 5, (SELECT * FROM pragma_page_count)+1) + WHERE pgno=4860; + SELECT checkfreelist('main'); + ROLLBACK; +} {{leaf page 10092 is out of range (child 3 of trunk page 4860)}} + +do_execsql_test 1.9 { + BEGIN; + UPDATE sqlite_dbpage + SET data = set_int(data, 5, 0) + WHERE pgno=4860; + SELECT checkfreelist('main'); + ROLLBACK; +} {{leaf page 0 is out of range (child 3 of trunk page 4860)}} + +do_execsql_test 1.10 { + BEGIN; + UPDATE sqlite_dbpage + SET data = set_int(data, get_int(data, 1)+1, 0) + WHERE pgno=5; + SELECT checkfreelist('main'); + ROLLBACK; +} {{leaf page 0 is out of range (child 247 of trunk page 5)}} + +do_execsql_test 1.11 { + BEGIN; + UPDATE sqlite_dbpage + SET data = set_int(data, 1, 249) + WHERE pgno=5; + SELECT checkfreelist('main'); + ROLLBACK; +} {{leaf count out of range (249) on trunk page 5}} diff --git a/ext/repair/test/checkindex01.test b/ext/repair/test/checkindex01.test new file mode 100644 index 000000000..97973aee7 --- /dev/null +++ b/ext/repair/test/checkindex01.test @@ -0,0 +1,349 @@ +# 2017-10-11 +# +set testprefix checkindex + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); + INSERT INTO t1 VALUES('one', 2); + INSERT INTO t1 VALUES('two', 4); + INSERT INTO t1 VALUES('three', 6); + INSERT INTO t1 VALUES('four', 8); + INSERT INTO t1 VALUES('five', 10); + + CREATE INDEX i2 ON t1(a DESC); +} {} + +proc incr_index_check {idx nStep} { + set Q { + SELECT errmsg, current_key FROM incremental_index_check($idx, $after) + LIMIT $nStep + } + + set res [list] + while {1} { + unset -nocomplain current_key + set res1 [db eval $Q] + if {[llength $res1]==0} break + set res [concat $res $res1] + set after [lindex $res end] + } + + return $res +} + +proc do_index_check_test {tn idx res} { + uplevel [list do_execsql_test $tn.1 " + SELECT errmsg, current_key FROM incremental_index_check('$idx'); + " $res] + + uplevel [list do_test $tn.2 "incr_index_check $idx 1" [list {*}$res]] + uplevel [list do_test $tn.3 "incr_index_check $idx 2" [list {*}$res]] + uplevel [list do_test $tn.4 "incr_index_check $idx 5" [list {*}$res]] +} + + +do_execsql_test 1.2.1 { + SELECT rowid, errmsg IS NULL, current_key FROM incremental_index_check('i1'); +} { + 1 1 'five',5 + 2 1 'four',4 + 3 1 'one',1 + 4 1 'three',3 + 5 1 'two',2 +} +do_execsql_test 1.2.2 { + SELECT errmsg IS NULL, current_key, index_name, after_key, scanner_sql + FROM incremental_index_check('i1') LIMIT 1; +} { + 1 + 'five',5 + i1 + {} + {SELECT (SELECT a IS i.i0 FROM 't1' AS t WHERE "rowid" COLLATE BINARY IS i.i1), quote(i0)||','||quote(i1) FROM (SELECT (a) AS i0, ("rowid" COLLATE BINARY) AS i1 FROM 't1' INDEXED BY 'i1' ORDER BY 1,2) AS i} +} + +do_index_check_test 1.3 i1 { + {} 'five',5 + {} 'four',4 + {} 'one',1 + {} 'three',3 + {} 'two',2 +} + +do_index_check_test 1.4 i2 { + {} 'two',2 + {} 'three',3 + {} 'one',1 + {} 'four',4 + {} 'five',5 +} + +do_test 1.5 { + set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t1' }] + sqlite3_imposter db main $tblroot {CREATE TABLE xt1(a,b)} + db eval { + UPDATE xt1 SET a='six' WHERE rowid=3; + DELETE FROM xt1 WHERE rowid = 5; + } + sqlite3_imposter db main +} {} + +do_index_check_test 1.6 i1 { + {row missing} 'five',5 + {} 'four',4 + {} 'one',1 + {row data mismatch} 'three',3 + {} 'two',2 +} + +do_index_check_test 1.7 i2 { + {} 'two',2 + {row data mismatch} 'three',3 + {} 'one',1 + {} 'four',4 + {row missing} 'five',5 +} + +#-------------------------------------------------------------------------- +do_execsql_test 2.0 { + + CREATE TABLE t2(a INTEGER PRIMARY KEY, b, c, d); + + INSERT INTO t2 VALUES(1, NULL, 1, 1); + INSERT INTO t2 VALUES(2, 1, NULL, 1); + INSERT INTO t2 VALUES(3, 1, 1, NULL); + + INSERT INTO t2 VALUES(4, 2, 2, 1); + INSERT INTO t2 VALUES(5, 2, 2, 2); + INSERT INTO t2 VALUES(6, 2, 2, 3); + + INSERT INTO t2 VALUES(7, 2, 2, 1); + INSERT INTO t2 VALUES(8, 2, 2, 2); + INSERT INTO t2 VALUES(9, 2, 2, 3); + + CREATE INDEX i3 ON t2(b, c, d); + CREATE INDEX i4 ON t2(b DESC, c DESC, d DESC); + CREATE INDEX i5 ON t2(d, c DESC, b); +} {} + +do_index_check_test 2.1 i3 { + {} NULL,1,1,1 + {} 1,NULL,1,2 + {} 1,1,NULL,3 + {} 2,2,1,4 + {} 2,2,1,7 + {} 2,2,2,5 + {} 2,2,2,8 + {} 2,2,3,6 + {} 2,2,3,9 +} + +do_index_check_test 2.2 i4 { + {} 2,2,3,6 + {} 2,2,3,9 + {} 2,2,2,5 + {} 2,2,2,8 + {} 2,2,1,4 + {} 2,2,1,7 + {} 1,1,NULL,3 + {} 1,NULL,1,2 + {} NULL,1,1,1 +} + +do_index_check_test 2.3 i5 { + {} NULL,1,1,3 + {} 1,2,2,4 + {} 1,2,2,7 + {} 1,1,NULL,1 + {} 1,NULL,1,2 + {} 2,2,2,5 + {} 2,2,2,8 + {} 3,2,2,6 + {} 3,2,2,9 +} + +#-------------------------------------------------------------------------- +do_execsql_test 3.0 { + + CREATE TABLE t3(w, x, y, z PRIMARY KEY) WITHOUT ROWID; + CREATE INDEX t3wxy ON t3(w, x, y); + CREATE INDEX t3wxy2 ON t3(w DESC, x DESC, y DESC); + + INSERT INTO t3 VALUES(NULL, NULL, NULL, 1); + INSERT INTO t3 VALUES(NULL, NULL, NULL, 2); + INSERT INTO t3 VALUES(NULL, NULL, NULL, 3); + + INSERT INTO t3 VALUES('a', NULL, NULL, 4); + INSERT INTO t3 VALUES('a', NULL, NULL, 5); + INSERT INTO t3 VALUES('a', NULL, NULL, 6); + + INSERT INTO t3 VALUES('a', 'b', NULL, 7); + INSERT INTO t3 VALUES('a', 'b', NULL, 8); + INSERT INTO t3 VALUES('a', 'b', NULL, 9); + +} {} + +do_index_check_test 3.1 t3wxy { + {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3 + {} 'a',NULL,NULL,4 {} 'a',NULL,NULL,5 {} 'a',NULL,NULL,6 + {} 'a','b',NULL,7 {} 'a','b',NULL,8 {} 'a','b',NULL,9 +} +do_index_check_test 3.2 t3wxy2 { + {} 'a','b',NULL,7 {} 'a','b',NULL,8 {} 'a','b',NULL,9 + {} 'a',NULL,NULL,4 {} 'a',NULL,NULL,5 {} 'a',NULL,NULL,6 + {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3 +} + +#-------------------------------------------------------------------------- +# Test with an index that uses non-default collation sequences. +# +do_execsql_test 4.0 { + CREATE TABLE t4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT); + INSERT INTO t4 VALUES(1, 'aaa', 'bbb'); + INSERT INTO t4 VALUES(2, 'AAA', 'CCC'); + INSERT INTO t4 VALUES(3, 'aab', 'ddd'); + INSERT INTO t4 VALUES(4, 'AAB', 'EEE'); + + CREATE INDEX t4cc ON t4(c1 COLLATE nocase, c2 COLLATE nocase); +} + +do_index_check_test 4.1 t4cc { + {} 'aaa','bbb',1 + {} 'AAA','CCC',2 + {} 'aab','ddd',3 + {} 'AAB','EEE',4 +} + +do_test 4.2 { + set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t4' }] + sqlite3_imposter db main $tblroot \ + {CREATE TABLE xt4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT)} + + db eval { + UPDATE xt4 SET c1='hello' WHERE rowid=2; + DELETE FROM xt4 WHERE rowid = 3; + } + sqlite3_imposter db main +} {} + +do_index_check_test 4.3 t4cc { + {} 'aaa','bbb',1 + {row data mismatch} 'AAA','CCC',2 + {row missing} 'aab','ddd',3 + {} 'AAB','EEE',4 +} + +#-------------------------------------------------------------------------- +# Test an index on an expression. +# +do_execsql_test 5.0 { + CREATE TABLE t5(x INTEGER PRIMARY KEY, y TEXT, UNIQUE(y)); + INSERT INTO t5 VALUES(1, '{"x":1, "y":1}'); + INSERT INTO t5 VALUES(2, '{"x":2, "y":2}'); + INSERT INTO t5 VALUES(3, '{"x":3, "y":3}'); + INSERT INTO t5 VALUES(4, '{"w":4, "z":4}'); + INSERT INTO t5 VALUES(5, '{"x":5, "y":5}'); + + CREATE INDEX t5x ON t5( json_extract(y, '$.x') ); + CREATE INDEX t5y ON t5( json_extract(y, '$.y') DESC ); +} + +do_index_check_test 5.1.1 t5x { + {} NULL,4 {} 1,1 {} 2,2 {} 3,3 {} 5,5 +} + +do_index_check_test 5.1.2 t5y { + {} 5,5 {} 3,3 {} 2,2 {} 1,1 {} NULL,4 +} + +do_index_check_test 5.1.3 sqlite_autoindex_t5_1 { + {} {'{"w":4, "z":4}',4} + {} {'{"x":1, "y":1}',1} + {} {'{"x":2, "y":2}',2} + {} {'{"x":3, "y":3}',3} + {} {'{"x":5, "y":5}',5} +} + +do_test 5.2 { + set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t5' }] + sqlite3_imposter db main $tblroot \ + {CREATE TABLE xt5(a INTEGER PRIMARY KEY, c1 TEXT);} + db eval { + UPDATE xt5 SET c1='{"x":22, "y":11}' WHERE rowid=1; + DELETE FROM xt5 WHERE rowid = 4; + } + sqlite3_imposter db main +} {} + +do_index_check_test 5.3.1 t5x { + {row missing} NULL,4 + {row data mismatch} 1,1 + {} 2,2 + {} 3,3 + {} 5,5 +} + +do_index_check_test 5.3.2 sqlite_autoindex_t5_1 { + {row missing} {'{"w":4, "z":4}',4} + {row data mismatch} {'{"x":1, "y":1}',1} + {} {'{"x":2, "y":2}',2} + {} {'{"x":3, "y":3}',3} + {} {'{"x":5, "y":5}',5} +} + +#------------------------------------------------------------------------- +# +do_execsql_test 6.0 { + CREATE TABLE t6(x INTEGER PRIMARY KEY, y, z); + CREATE INDEX t6x1 ON t6(y, /* one,two,three */ z); + CREATE INDEX t6x2 ON t6(z, -- hello,world, + y); + + CREATE INDEX t6x3 ON t6(z -- hello,world + , y); + + INSERT INTO t6 VALUES(1, 2, 3); + INSERT INTO t6 VALUES(4, 5, 6); +} + +do_index_check_test 6.1 t6x1 { + {} 2,3,1 + {} 5,6,4 +} +do_index_check_test 6.2 t6x2 { + {} 3,2,1 + {} 6,5,4 +} +do_index_check_test 6.2 t6x3 { + {} 3,2,1 + {} 6,5,4 +} + +#------------------------------------------------------------------------- +# +do_execsql_test 7.0 { + CREATE TABLE t7(x INTEGER PRIMARY KEY, y, z); + INSERT INTO t7 VALUES(1, 1, 1); + INSERT INTO t7 VALUES(2, 2, 0); + INSERT INTO t7 VALUES(3, 3, 1); + INSERT INTO t7 VALUES(4, 4, 0); + + CREATE INDEX t7i1 ON t7(y) WHERE z=1; + CREATE INDEX t7i2 ON t7(y) /* hello,world */ WHERE z=1; + CREATE INDEX t7i3 ON t7(y) WHERE -- yep + z=1; + CREATE INDEX t7i4 ON t7(y) WHERE z=1 -- yep; +} +do_index_check_test 7.1 t7i1 { + {} 1,1 {} 3,3 +} +do_index_check_test 7.2 t7i2 { + {} 1,1 {} 3,3 +} +do_index_check_test 7.3 t7i3 { + {} 1,1 {} 3,3 +} +do_index_check_test 7.4 t7i4 { + {} 1,1 {} 3,3 +} diff --git a/ext/repair/test/test.tcl b/ext/repair/test/test.tcl new file mode 100644 index 000000000..c073bb73c --- /dev/null +++ b/ext/repair/test/test.tcl @@ -0,0 +1,67 @@ +# Run this script using +# +# sqlite3_checker --test $thisscript $testscripts +# +# The $testscripts argument is optional. If omitted, all *.test files +# in the same directory as $thisscript are run. +# +set NTEST 0 +set NERR 0 + + +# Invoke the do_test procedure to run a single test +# +# The $expected parameter is the expected result. The result is the return +# value from the last TCL command in $cmd. +# +# Normally, $expected must match exactly. But if $expected is of the form +# "/regexp/" then regular expression matching is used. If $expected is +# "~/regexp/" then the regular expression must NOT match. If $expected is +# of the form "#/value-list/" then each term in value-list must be numeric +# and must approximately match the corresponding numeric term in $result. +# Values must match within 10%. Or if the $expected term is A..B then the +# $result term must be in between A and B. +# +proc do_test {name cmd expected} { + if {[info exists ::testprefix]} { + set name "$::testprefix$name" + } + + incr ::NTEST + puts -nonewline $name... + flush stdout + + if {[catch {uplevel #0 "$cmd;\n"} result]} { + puts -nonewline $name... + puts "\nError: $result" + incr ::NERR + } else { + set ok [expr {[string compare $result $expected]==0}] + if {!$ok} { + puts "\n! $name expected: \[$expected\]\n! $name got: \[$result\]" + incr ::NERR + } else { + puts " Ok" + } + } + flush stdout +} + +# +# do_execsql_test TESTNAME SQL RES +# +proc do_execsql_test {testname sql {result {}}} { + uplevel [list do_test $testname [list db eval $sql] [list {*}$result]] +} + +if {[llength $argv]==0} { + set dir [file dirname $argv0] + set argv [glob -nocomplain $dir/*.test] +} +foreach testfile $argv { + file delete -force test.db + sqlite3 db test.db + source $testfile + catch {db close} +} +puts "$NERR errors out of $NTEST tests" diff --git a/ext/rtree/geopoly.c b/ext/rtree/geopoly.c index 22166a6f9..0ae42e7b7 100644 --- a/ext/rtree/geopoly.c +++ b/ext/rtree/geopoly.c @@ -200,7 +200,7 @@ static int geopolyParseNumber(GeoParse *p, GeoCoord *pVal){ /* The sqlite3AtoF() routine is much much faster than atof(), if it ** is available */ double r; - (void)sqlite3AtoF((const char*)p->z, &r); + (void)sqlite3AtoF((const char*)p->z, &r, j, SQLITE_UTF8); *pVal = r; #else *pVal = (GeoCoord)atof((const char*)p->z); diff --git a/ext/session/session4.test b/ext/session/session4.test index 5e44a8eb6..55cb76f15 100644 --- a/ext/session/session4.test +++ b/ext/session/session4.test @@ -135,7 +135,6 @@ foreach {tn blob} { 54 540101743400120003001200010000000000000002120002400C000000000002120002400C00000000000050040100000074310017FF0050040100000074310017FF7F00000000000000050100000000000000030100000003001700010000666F7572 55 540101743400120003001200010000000000000002120002400C00000000000050040100000074310017000100010080000001000000020003010100000300170100000003001700010000666F7572 56 5487ffffff7f - 57 54015052494e5446110017120004 } { do_test 2.$tn { set changeset [binary decode hex $blob] diff --git a/ext/session/sessionC.test b/ext/session/sessionC.test index 1997ba5e8..74370cb79 100644 --- a/ext/session/sessionC.test +++ b/ext/session/sessionC.test @@ -192,16 +192,6 @@ do_test 3.3 { } } {1 1 3 3} -#------------------------------------------------------------------------- -# -reset_db -set C [binary format c* 0x54 0x01 0x01 0x00 0x12 0x00 0x05] -do_test 4.0 { - sqlite3changegroup grp - list [catch { grp add $C } msg] $msg -} {1 SQLITE_CORRUPT} -grp delete finish_test - diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index 5468dff28..90fedc6db 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -349,20 +349,6 @@ static int sessionVarintGet(const u8 *aBuf, int *piVal){ return getVarint32(aBuf, *piVal); } -/* -** Read a varint value from buffer aBuf[], size nBuf bytes, into *piVal. -** Return the number of bytes read. -*/ -static int sessionVarintGetSafe(const u8 *aBuf, int nBuf, int *piVal){ - u8 aCopy[5]; - const u8 *aRead = aBuf; - if( nBuf<5 ){ - memcpy(aCopy, aBuf, nBuf); - aRead = aCopy; - } - return getVarint32(aRead, *piVal); -} - /* Load an unaligned and unsigned 32-bit integer */ #define SESSION_UINT32(x) (((u32)(x)[0]<<24)|((x)[1]<<16)|((x)[2]<<8)|(x)[3]) @@ -657,10 +643,14 @@ static unsigned int sessionChangeHash( int isPK = pTab->abPK[i]; if( bPkOnly && isPK==0 ) continue; + /* It is not possible for eType to be SQLITE_NULL here. The session + ** module does not record changes for rows with NULL values stored in + ** primary key columns. */ assert( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT || eType==SQLITE_TEXT || eType==SQLITE_BLOB || eType==SQLITE_NULL || eType==0 ); + assert( !isPK || (eType!=0 && eType!=SQLITE_NULL) ); if( isPK ){ a++; @@ -668,16 +658,12 @@ static unsigned int sessionChangeHash( if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ h = sessionHashAppendI64(h, sessionGetI64(a)); a += 8; - }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ + }else{ int n; a += sessionVarintGet(a, &n); h = sessionHashAppendBlob(h, n, a); a += n; } - /* It should not be possible for eType to be SQLITE_NULL or 0x00 here, - ** as the session module does not record changes for rows with NULL - ** values stored in primary key columns. But a corrupt changesets - ** may contain such a value. */ }else{ a += sessionSerialLen(a); } @@ -3086,13 +3072,10 @@ static int sessionGenerateChangeset( } if( pSession->rc ) return pSession->rc; + rc = sqlite3_exec(pSession->db, "SAVEPOINT changeset", 0, 0, 0); + if( rc!=SQLITE_OK ) return rc; sqlite3_mutex_enter(sqlite3_db_mutex(db)); - rc = sqlite3_exec(pSession->db, "SAVEPOINT changeset", 0, 0, 0); - if( rc!=SQLITE_OK ){ - sqlite3_mutex_leave(sqlite3_db_mutex(db)); - return rc; - } for(pTab=pSession->pTable; rc==SQLITE_OK && pTab; pTab=pTab->pNext){ if( pTab->nEntry ){ @@ -3575,8 +3558,7 @@ static int sessionReadRecord( u8 *aVal = &pIn->aData[pIn->iNext]; if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ int nByte; - int nRem = pIn->nData - pIn->iNext; - pIn->iNext += sessionVarintGetSafe(aVal, nRem, &nByte); + pIn->iNext += sessionVarintGet(aVal, &nByte); rc = sessionInputBuffer(pIn, nByte); if( rc==SQLITE_OK ){ if( nByte<0 || nByte>pIn->nData-pIn->iNext ){ @@ -3629,8 +3611,7 @@ static int sessionChangesetBufferTblhdr(SessionInput *pIn, int *pnByte){ rc = sessionInputBuffer(pIn, 9); if( rc==SQLITE_OK ){ - int nBuf = pIn->nData - pIn->iNext; - nRead += sessionVarintGetSafe(&pIn->aData[pIn->iNext], nBuf, &nCol); + nRead += sessionVarintGet(&pIn->aData[pIn->iNext + nRead], &nCol); /* The hard upper limit for the number of columns in an SQLite ** database table is, according to sqliteLimit.h, 32676. So ** consider any table-header that purports to have more than 65536 @@ -3650,15 +3631,8 @@ static int sessionChangesetBufferTblhdr(SessionInput *pIn, int *pnByte){ while( (pIn->iNext + nRead)<pIn->nData && pIn->aData[pIn->iNext + nRead] ){ nRead++; } - - /* Break out of the loop if if the nul-terminator byte has been found. - ** Otherwise, read some more input data and keep seeking. If there is - ** no more input data, consider the changeset corrupt. */ if( (pIn->iNext + nRead)<pIn->nData ) break; rc = sessionInputBuffer(pIn, nRead + 100); - if( rc==SQLITE_OK && (pIn->iNext + nRead)>=pIn->nData ){ - rc = SQLITE_CORRUPT_BKPT; - } } *pnByte = nRead+1; return rc; @@ -3790,10 +3764,10 @@ static int sessionChangesetNextOne( memset(p->apValue, 0, sizeof(sqlite3_value*)*p->nCol*2); } - /* Make sure the buffer contains at least 2 bytes of input data, or all - ** remaining data if there are less than 2 bytes available. This is - ** sufficient either for the 'T' or 'P' byte that begins a new table, - ** or for the "op" and "bIndirect" single bytes otherwise. */ + /* Make sure the buffer contains at least 10 bytes of input data, or all + ** remaining data if there are less than 10 bytes available. This is + ** sufficient either for the 'T' or 'P' byte and the varint that follows + ** it, or for the two single byte values otherwise. */ p->rc = sessionInputBuffer(&p->in, 2); if( p->rc!=SQLITE_OK ) return p->rc; @@ -3823,13 +3797,11 @@ static int sessionChangesetNextOne( return (p->rc = SQLITE_CORRUPT_BKPT); } - if( (op!=SQLITE_UPDATE && op!=SQLITE_DELETE && op!=SQLITE_INSERT) - || (p->in.iNext>=p->in.nData) - ){ - return (p->rc = SQLITE_CORRUPT_BKPT); - } p->op = op; p->bIndirect = p->in.aData[p->in.iNext++]; + if( p->op!=SQLITE_UPDATE && p->op!=SQLITE_DELETE && p->op!=SQLITE_INSERT ){ + return (p->rc = SQLITE_CORRUPT_BKPT); + } if( paRec ){ int nVal; /* Number of values to buffer */ diff --git a/ext/session/test_session.c b/ext/session/test_session.c index 46a9e3a38..6ad5b3774 100644 --- a/ext/session/test_session.c +++ b/ext/session/test_session.c @@ -7,8 +7,6 @@ #include <string.h> #include "tclsqlite.h" -#include <stdlib.h> - #ifndef SQLITE_AMALGAMATION typedef unsigned char u8; #endif @@ -858,21 +856,6 @@ static int testStreamInput( return SQLITE_OK; } -/* -** This works like Tcl_GetByteArrayFromObj(), except that it returns a buffer -** allocated using malloc() that must be freed by the caller. This is done -** because Tcl's buffers are often padded by a few bytes, which prevents -** small overreads from being detected when tests are run under asan. -*/ -static void *testGetByteArrayFromObj(Tcl_Obj *p, Tcl_Size *pnByte){ - Tcl_Size nByte = 0; - void *aByte = Tcl_GetByteArrayFromObj(p, &nByte); - void *aCopy = malloc(nByte ? (size_t)nByte : 1); - memcpy(aCopy, aByte, (size_t)nByte); - *pnByte = nByte; - return aCopy; -} - static int SQLITE_TCLAPI testSqlite3changesetApply( int iVersion, @@ -937,7 +920,7 @@ static int SQLITE_TCLAPI testSqlite3changesetApply( return TCL_ERROR; } db = *(sqlite3 **)info.objClientData; - pChangeset = (void *)testGetByteArrayFromObj(objv[2], &nChangeset); + pChangeset = (void *)Tcl_GetByteArrayFromObj(objv[2], &nChangeset); ctx.pConflictScript = objv[3]; ctx.pFilterScript = objc==5 ? objv[4] : 0; ctx.interp = interp; @@ -989,7 +972,6 @@ static int SQLITE_TCLAPI testSqlite3changesetApply( } } - free(pChangeset); if( rc!=SQLITE_OK ){ return test_session_error(interp, rc, 0); }else{ @@ -1213,12 +1195,7 @@ static int SQLITE_TCLAPI test_sqlite3session_foreach( pCS = objv[2]; pScript = objv[3]; - /* Take a copy of the changeset into an exact sized buffer allocated - ** using malloc(). The Tcl buffer will be padded by a few bytes, which - ** prevents small overreads from being detected by ASAN when the tests - ** are run. */ - pChangeset = (void*)testGetByteArrayFromObj(pCS, &nChangeset); - + pChangeset = (void *)Tcl_GetByteArrayFromObj(pCS, &nChangeset); sStr.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR); if( isInvert ){ int f = SQLITE_CHANGESETSTART_INVERT; @@ -1239,33 +1216,32 @@ static int SQLITE_TCLAPI test_sqlite3session_foreach( rc = sqlite3changeset_start_strm(&pIter, testStreamInput, (void*)&sStr); } } + if( rc!=SQLITE_OK ){ + return test_session_error(interp, rc, 0); + } - if( rc==SQLITE_OK ){ - while( SQLITE_ROW==sqlite3changeset_next(pIter) ){ - Tcl_Obj *pVar = 0; /* Tcl value to set $VARNAME to */ - pVar = testIterData(pIter); - Tcl_ObjSetVar2(interp, pVarname, 0, pVar, 0); - rc = Tcl_EvalObjEx(interp, pScript, 0); - if( rc!=TCL_OK && rc!=TCL_CONTINUE ){ - sqlite3changeset_finalize(pIter); - free(pChangeset); - return rc==TCL_BREAK ? TCL_OK : rc; - } - } - - if( isCheckNext ){ - int rc2 = sqlite3changeset_next(pIter); - rc = sqlite3changeset_finalize(pIter); - assert( (rc2==SQLITE_DONE && rc==SQLITE_OK) || rc2==rc ); - }else{ - rc = sqlite3changeset_finalize(pIter); + while( SQLITE_ROW==sqlite3changeset_next(pIter) ){ + Tcl_Obj *pVar = 0; /* Tcl value to set $VARNAME to */ + pVar = testIterData(pIter); + Tcl_ObjSetVar2(interp, pVarname, 0, pVar, 0); + rc = Tcl_EvalObjEx(interp, pScript, 0); + if( rc!=TCL_OK && rc!=TCL_CONTINUE ){ + sqlite3changeset_finalize(pIter); + return rc==TCL_BREAK ? TCL_OK : rc; } } - free(pChangeset); + if( isCheckNext ){ + int rc2 = sqlite3changeset_next(pIter); + rc = sqlite3changeset_finalize(pIter); + assert( (rc2==SQLITE_DONE && rc==SQLITE_OK) || rc2==rc ); + }else{ + rc = sqlite3changeset_finalize(pIter); + } if( rc!=SQLITE_OK ){ return test_session_error(interp, rc, 0); } + return TCL_OK; } diff --git a/ext/wasm/EXPORTED_FUNCTIONS.fiddle.in b/ext/wasm/EXPORTED_FUNCTIONS.fiddle.in new file mode 100644 index 000000000..103704df1 --- /dev/null +++ b/ext/wasm/EXPORTED_FUNCTIONS.fiddle.in @@ -0,0 +1,10 @@ +_fiddle_db_arg +_fiddle_db_filename +_fiddle_exec +_fiddle_experiment +_fiddle_interrupt +_fiddle_main +_fiddle_reset_db +_fiddle_db_handle +_fiddle_db_vfs +_fiddle_export_db diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index dc3c2d255..937e16d6e 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -1,6 +1,5 @@ -# -# This GNU makefile creates the canonical sqlite3 WASM builds. Plus some -# others. +####################################################################### +# This GNU makefile creates the canonical sqlite3 WASM builds. # # This build assumes a Linux platform and is not intended for # general-purpose client-level use, except for creating builds with @@ -11,33 +10,28 @@ # # default, all = build in dev mode # -# o0, o1, o2, o3, os, oz = full clean/rebuild with the -Ox level -# indicated by the target name. A clean rebuild is necessary for -# all components to get the desired optimization level. +# o0, o1, o2, o3, os, oz = full clean/rebuild with the -Ox level indicated +# by the target name. Rebuild is necessary for all components to get +# the desired optimization level. # # dist = create end user deliverables. Add dist.build=oX to build -# with a specific optimization level, where oX is one of the -# above-listed o? target names. +# with a specific optimization level, where oX is one of the +# above-listed o? or qo? target names. # # snapshot = like dist, but uses a zip file name which clearly -# marks it as a prerelease/snapshot build. +# marks it as a prerelease/snapshot build. # # clean = clean up # # Required tools beyond those needed for the canonical builds: # # - Emscripten SDK: https://emscripten.org/docs/getting_started/downloads.html -# # - The bash shell -# # - GNU make, GNU sed, GNU awk, GNU grep (all in the $PATH and without # a "g" prefix like they have on some non-GNU systems) -# -# - wasm-strip for release builds: https://github.com/WebAssembly/wabt. -# It will build without this but the .wasm files will be huge. -# +# - wasm-strip for release builds: https://github.com/WebAssembly/wabt # - InfoZip for 'dist' zip file -# +######################################################################## default: all MAKEFILE = $(lastword $(MAKEFILE_LIST)) CLEAN_FILES = @@ -86,7 +80,6 @@ emo.strip = 💈 emo.test = 🧪 emo.tool = 🔨 emo.wasm-opt = 🧼 -emo.cleanup = 🧼 # 👷🪄🧮🧫🧽🍿⛽🚧🎱🪚🏆🧼 # @@ -206,7 +199,7 @@ b.mkdir@ = if [ ! -d $(dir $@) ]; then \ # $1 = logtag, $2 = src file(s). $3 = dest dir b.cp = $(call b.mkdir@); \ echo '$(logtag.$(1)) $(emo.disk) $(2) ==> $3'; \ - cp -f -p $(2) $(3) || exit + cp -p $(2) $(3) || exit # # $(call b.c-pp.shcmd,LOGTAG,src,dest,-Dx=y...) @@ -220,8 +213,9 @@ b.cp = $(call b.mkdir@); \ # $4 = optional $(bin.c-pp) flags define b.c-pp.shcmd $(call b.mkdir@); \ -$(call b.echo,$(1),$(emo.disk)$(emo.lock)[$(3)] $(4)); \ -rm -f $(3); $(bin.c-pp) -o $(3) $(4) $(2) || exit; \ +$(call b.echo,$(1),$(emo.disk)$(emo.lock) $(bin.c-pp) $(4) $(if $(loud.if),$(2))); \ +rm -f $(3); \ +$(bin.c-pp) -o $(3) $(4) $(2) || exit; \ chmod -w $(3) endef @@ -233,38 +227,10 @@ endef # Args: as for $(b.c-pp.shcmd). define b.c-pp.target $(3): $$(MAKEFILE_LIST) $$(bin.c-pp) $(2) - @$$(call b.mkdir@) - @$$(call b.c-pp.shcmd,$(1),$(2),$(3),$(4) $$(b.c-pp.target.flags)) + @$$(call b.c-pp.shcmd,$(1),$(2),$(3),$(4) $(b.c-pp.target.flags)) CLEAN_FILES += $(3) endef - -# -# The various -D... values used by *.c-pp.js include: -# -# -Dtarget:es6-module: for all ESM module builds -# -# -Dtarget:node: for node.js builds -# -# -Dtarget:es6-module -Dtarget:es6-bundler-friendly: intended for -# "bundler-friendly" ESM module build. These have some restrictions -# on how URL() objects are constructed in some contexts: URLs which -# refer to files which are part of this project must be referenced -# as string literals so that bundlers' static-analysis tools can -# find those files and include them in their bundles. -# -# -Dtarget:es6-module -Dtarget:node: is intended for use by node.js -# for node.js, as opposed to by node.js on behalf of a -# browser. Mixing -sENVIRONMENT=web and -sENVIRONMENT=node leads to -# ambiguity and confusion on node's part, as it's unable to -# reliably determine whether the target is a browser or node. -# -# To repeat: all node.js builds are 100% untested and unsupported. -# -# Most c-pp.D.X are set via $(bin.mkwb) and X is a build name. -# Those make rules reference c-pp.D.64bit, so it should be defined in -# advance. -# c-pp.D.64bit = -Dbits64 # @@ -282,21 +248,11 @@ c-pp.D.64bit = -Dbits64 # # This is intended to be used in makefile targets which generate an # Emscripten module and where $@ is the module's .js/.mjs file. -# -# This is inherently fragile and has been broken by Emscripten updates -# before. -# -ifeq (1,1) -define b.strip-js-emcc-bindings -echo "$(1) $(emo.garbage) Stripping export wrappers."; \ -sed -i -e '/var _sqlite3.*makeInvalidEarly.*;/d' \ --e '/assert.*sqlite3.*missing.*;/d' \ --e '/_sqlite.*createExportWrapper.*;/d' $@ -endef -else b.strip-js-emcc-bindings = \ - echo '$(1) $(emo.bug) (disabled because it breaks emsdk 4.0.16+)' -endif + sed -i -e '/^.*= \(_sqlite3\|_fiddle\)[^=]*=.*createExportWrapper/d' \ + -e '/^var \(_sqlite3\|_fiddle\)[^=]*=.*makeInvalidEarlyAccess/d' $@ || exit; \ + echo '$(1) $(emo.garbage) (Probably) /createExportWrapper()/d and /makeInvalidEarlyAccess()/d' + # # Set up sqlite3.c and sqlite3.h... @@ -315,36 +271,33 @@ endif # $(sqlite3.canonical.c) must point to the sqlite3.c in # the sqlite3 canonical source tree, as that source file # is required for certain utility and test code. -# sqlite3.canonical.c = $(dir.top)/sqlite3.c sqlite3.c ?= $(firstword $(wildcard $(dir.top)/sqlite3-see.c) $(sqlite3.canonical.c)) -sqlite3.h = $(dir $(sqlite3.c))sqlite3.h +sqlite3.h = $(dir $(sqlite3.c))/sqlite3.h # # bin.version-info = binary to output various sqlite3 version info for # embedding in the JS files and in building the distribution zip file. # It must NOT be in $(dir.tmp) because we need it to survive the # cleanup process for the dist build to work properly. -# bin.version-info = ./version-info $(bin.version-info): $(dir.tool)/version-info.c $(sqlite3.h) $(dir.top)/Makefile $(CC) -o $@ -I$(dir $(sqlite3.h)) $(dir.tool)/version-info.c t-version-info: $(bin.version-info) DISTCLEAN_FILES += $(bin.version-info) - # # bin.stripcomments is used for stripping C/C++-style comments from JS # files. The JS files contain large chunks of documentation which we # don't need for all builds. That app's -k flag is of particular # importance here, as it allows us to retain the opening comment # block(s), which contain the license header and version info. -# bin.stripccomments = $(dir.tool)/stripccomments $(bin.stripccomments): $(bin.stripccomments).c $(MAKEFILE) $(CC) -o $@ $< t-stripccomments: $(bin.stripccomments) DISTCLEAN_FILES += $(bin.stripccomments) + ifeq (1,$(MAKING_CLEAN)) SQLITE_C_IS_SEE = 0 else @@ -372,7 +325,7 @@ endif # undefine barebones # relatively new gmake feature, not ubiquitous # -# It's important that sqlite3.[ch] be built to completion before any +# It's important that sqlite3.h be built to completion before any # other parts of the build run, thus we use .NOTPARALLEL to disable # parallel build of that file and its dependants. However, that makes # the whole build non-parallelizable because everything has a dep on @@ -389,10 +342,8 @@ $(sqlite3.h): # $(MAKE) -C $(dir.top) sqlite3.c $(sqlite3.c): $(sqlite3.h) -# # Common options for building sqlite3-wasm.c and speedtest1.c. # Explicit ENABLEs... -# SQLITE_OPT.common = \ -DSQLITE_THREADSAFE=0 \ -DSQLITE_TEMP_STORE=2 \ @@ -409,15 +360,11 @@ SQLITE_OPT.common = \ # removing them from this list will serve only to break the speedtest1 # builds. -# # Currently always needed but TODO is paring tester1.c-pp.js down # to be able to run without this: -# SQLITE_OPT.common += -DSQLITE_WASM_ENABLE_C_TESTS -# # Extra flags for full-featured builds... -# SQLITE_OPT.full-featured = \ -DSQLITE_ENABLE_BYTECODE_VTAB \ -DSQLITE_ENABLE_DBPAGE_VTAB \ @@ -474,14 +421,13 @@ else # -DSQLITE_OMIT_WINDOWFUNC endif -# #SQLITE_OPT += -DSQLITE_DEBUG # Enabling SQLITE_DEBUG will break sqlite3_wasm_vfs_create_file() # (and thus sqlite3_js_vfs_create_file()). Those functions are # deprecated and alternatives are in place, but this crash behavior # can be used to find errant uses of sqlite3_js_vfs_create_file() # in client code. -# +######################################################################## # The following flags are hard-coded into sqlite3-wasm.c and cannot be # modified via the build process: # @@ -490,9 +436,10 @@ endif # SQLITE_OMIT_DEPRECATED # SQLITE_OMIT_UTF16 # SQLITE_OMIT_SHARED_CACHE -# +######################################################################## -# + +######################################################################## # Adding custom C code via sqlite3_wasm_extra_init.c: # # If the canonical build process finds the file @@ -513,7 +460,7 @@ endif # make sqlite3_wasm_extra_init.c=my_custom_stuff.c # # See example_extra_init.c for an example implementation. -# +######################################################################## sqlite3_wasm_extra_init.c ?= $(wildcard sqlite3_wasm_extra_init.c) cflags.wasm_extra_init = ifneq (,$(sqlite3_wasm_extra_init.c)) @@ -534,7 +481,7 @@ endif # WASM_CUSTOM_INSTANTIATE = 1 -# +######################################################################## # $(bin.c-pp): a minimal text file preprocessor. Like C's but much # less so. # @@ -565,7 +512,6 @@ WASM_CUSTOM_INSTANTIATE = 1 # # -D... flags which should be included in all invocations should be # appended to $(b.c-pp.target.flags). -# bin.c-pp = ./c-pp-lite $(bin.c-pp): c-pp-lite.c $(sqlite3.c) $(MAKEFILE) $(CC) -O0 -o $@ c-pp-lite.c $(sqlite3.c) '-DCMPP_DEFAULT_DELIM="//#"' -I$(dir.top) \ @@ -577,12 +523,7 @@ b.c-pp.target.flags ?= ifeq (1,$(SQLITE_C_IS_SEE)) b.c-pp.target.flags += -Denable-see endif -api.oo1 ?= 1 -ifeq (0,$(api.oo1)) - b.c-pp.target.flags += -Domit-oo1 -endif -# # cflags.common = C compiler flags for all builds cflags.common = -I. -I$(dir $(sqlite3.c)) -std=c99 -fPIC # emcc.WASM_BIGINT = 1 for BigInt (C int64) support, else 0. The API @@ -590,14 +531,13 @@ cflags.common = -I. -I$(dir $(sqlite3.c)) -std=c99 -fPIC # _are not tested_ on any regular basis. emcc.WASM_BIGINT ?= 1 emcc.MEMORY64 ?= 0 -# +######################################################################## # https://emscripten.org/docs/tools_reference/settings_reference.html#memory64 # # 64-bit build requires wasm-strip 1.0.36 (maybe 1.0.35, but not # 1.0.34) or will fail to strip with "tables may not be 64-bit". -# +######################################################################## -# # emcc_opt = optimization-related flags. These are primarily used by # the various oX targets. build times for -O levels higher than 0 are # painful at dev-time. @@ -609,17 +549,14 @@ emcc.MEMORY64 ?= 0 # -O2 (which consistently creates the fastest-running deliverables). # Build time suffers greatly compared to -O0, which is why -O0 is the # default. -# ifeq (,$(filter $(OPTIMIZED_TARGETS),$(MAKECMDGOALS))) emcc_opt ?= -O0 else emcc_opt ?= -Oz endif -# # When passing emcc_opt from the CLI, += and re-assignment have no # effect, so emcc_opt+=-g3 doesn't work. So... -# emcc_opt_full = $(emcc_opt) -g3 # ^^^ ALWAYS use -g3. See below for why. # @@ -646,26 +583,26 @@ emcc_opt_full = $(emcc_opt) -g3 # Much practice has demonstrated that -O2 consistently gives the best # runtime speeds, but not by a large enough factor to rule out use of # -Oz when smaller deliverable size is a priority. -# +######################################################################## -# +######################################################################## # EXPORTED_FUNCTIONS.* = files for use with Emscripten's # -sEXPORTED_FUNCTION flag. -# -EXPORTED_FUNCTIONS.api.in = $(dir.api)/EXPORTED_FUNCTIONS.c-pp -EXPORTED_FUNCTIONS.api = $(dir.tmp)/EXPORTED_FUNCTIONS.api -EXPORTED_FUNCTIONS.c-pp.flags = -ifeq (1,$(wasm-bare-bones)) - EXPORTED_FUNCTIONS.c-pp.flags += -Dbare-bones +EXPORTED_FUNCTIONS.api.core = $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-core +EXPORTED_FUNCTIONS.api.in = $(EXPORTED_FUNCTIONS.api.core) +ifeq (1,$(SQLITE_C_IS_SEE)) + EXPORTED_FUNCTIONS.api.in += $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-see endif -$(eval $(call b.c-pp.target,filter,\ - $(EXPORTED_FUNCTIONS.api.in),\ - $(EXPORTED_FUNCTIONS.api),\ - $(EXPORTED_FUNCTIONS.c-pp.flags))) +ifeq (0,$(wasm-bare-bones)) + EXPORTED_FUNCTIONS.api.in += $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-extras +endif +EXPORTED_FUNCTIONS.api = $(dir.tmp)/EXPORTED_FUNCTIONS.api +$(EXPORTED_FUNCTIONS.api): $(EXPORTED_FUNCTIONS.api.in) $(sqlite3.c) $(MAKEFILE) + @$(call b.mkdir@) + cat $(EXPORTED_FUNCTIONS.api.in) > $@ -# +######################################################################## # emcc flags for .c/.o/.wasm/.js. -# emcc.flags = ifeq (1,$(emcc.verbose)) emcc.flags += -v @@ -753,9 +690,9 @@ ifeq (,$(emcc.INITIAL_MEMORY.$(emcc.INITIAL_MEMORY))) $(error emcc.INITIAL_MEMORY must be one of: 4, 8, 16, 32, 64, 96, 128 (megabytes)) endif emcc.jsflags += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.$(emcc.INITIAL_MEMORY)) -# # /INITIAL_MEMORY -# +######################################################################## +#emcc.jsflags += -sMEMORY64=$(emcc.MEMORY64) emcc.jsflags += $(emcc.environment) emcc.jsflags += -sSTACK_SIZE=512KB @@ -764,8 +701,7 @@ emcc.jsflags += -sSTACK_SIZE=512KB # VFS, which requires twice that for its xRead() and xWrite() methods. # 2023-03: those methods have since been adapted to use a malloc()'d # buffer. - -# +######################################################################## # $(sqlite3.js.init-func) is the name Emscripten assigns our exported # module init/load function. This symbol name is hard-coded in # $(extern-post-js.js) as well as in numerous docs. @@ -805,7 +741,7 @@ emcc.jsflags += -sLLD_REPORT_UNDEFINED #emcc.jsflags += --experimental-pic --unresolved-symbols=ingore-all --import-undefined #emcc.jsflags += --unresolved-symbols=ignore-all -# +######################################################################## # -sSINGLE_FILE: # https://github.com/emscripten-core/emscripten/blob/main/src/settings.js # @@ -814,7 +750,12 @@ emcc.jsflags += -sLLD_REPORT_UNDEFINED # cannot wasm-strip the binary before it gets encoded into the JS # file. The result is that the generated JS file is, because of the # -g3 debugging info, _huge_. -# +######################################################################## + + +sqlite3.wasm = $(dir.dout)/sqlite3.wasm +sqlite3-wasm.c = $(dir.api)/sqlite3-wasm.c +sqlite3-wasm.c.in = $(sqlite3-wasm.c) $(sqlite3_wasm_extra_init.c) # # b.call.patch-export-default is used by mkwasmbuilds.c and the @@ -857,6 +798,51 @@ if [ x1 = x$(1) ]; then \ fi endef +# +# The various -D... values used by *.c-pp.js include: +# +# -Dtarget:es6-module: for all ESM module builds +# +# -Dtarget:node: for node.js builds +# +# -Dtarget:es6-module -Dtarget:es6-bundler-friendly: intended for +# "bundler-friendly" ESM module build. These have some restrictions +# on how URL() objects are constructed in some contexts: URLs which +# refer to files which are part of this project must be referenced +# as string literals so that bundlers' static-analysis tools can +# find those files and include them in their bundles. +# +# -Dtarget:es6-module -Dtarget:node: is intended for use by node.js +# for node.js, as opposed to by node.js on behalf of a +# browser. Mixing -sENVIRONMENT=web and -sENVIRONMENT=node leads to +# ambiguity and confusion on node's part, as it's unable to +# reliably determine whether the target is a browser or node. +# +# To repeat: all node.js builds are 100% untested and unsupported. +# +######################################################################## + +# +# Inputs/outputs for the sqlite3-api.js family. +# +# sqlite3-api.jses = the list of JS files which make up +# sqlite3-api.js, in the order they need to be assembled. +sqlite3-api.jses = $(sqlite3-license-version.js) +sqlite3-api.jses += $(dir.api)/sqlite3-api-prologue.js +sqlite3-api.jses += $(dir.common)/whwasmutil.js +sqlite3-api.jses += $(dir.jacc)/jaccwabyt.js +sqlite3-api.jses += $(dir.api)/sqlite3-api-glue.c-pp.js +sqlite3-api.jses += $(sqlite3-api-build-version.js) +sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.c-pp.js +sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.c-pp.js +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-helper.c-pp.js +ifeq (0,$(wasm-bare-bones)) + sqlite3-api.jses += $(dir.api)/sqlite3-vtab-helper.c-pp.js +endif +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.c-pp.js +sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js + # # $(sqlite3-license-version.js) contains the license header and # in-comment build version info. @@ -866,15 +852,17 @@ endef # sqlite3-license-version.js = $(dir.tmp)/sqlite3-license-version.js $(sqlite3-license-version.js): $(bin.version-info) \ - $(dir.api)/sqlite3-license-version-header.js $(MAKEFILE) - @$(call b.mkdir@); echo '$(logtag.@) $(emo.disk)'; { \ + $(dir.api)/sqlite3-license-version-header.js + @echo '$(logtag.@) $(emo.disk)'; { \ + $(call b.mkdir@); \ cat $(dir.api)/sqlite3-license-version-header.js || exit $$?; \ - echo '/* @preserve'; \ + echo '/*'; \ echo '** This code was built from sqlite3 version...'; \ echo '**'; \ awk '/define SQLITE_VERSION/{$$1=""; print "**" $$0}' $(sqlite3.h); \ awk '/define SQLITE_SOURCE_ID/{$$1=""; print "**" $$0}' $(sqlite3.h); \ echo '**'; echo '** Emscripten SDK: $(emcc.version)'; \ + echo '**'; \ echo '*/'; \ } > $@ @@ -893,34 +881,6 @@ $(sqlite3-api-build-version.js): $(bin.version-info) $(MAKEFILE) echo '});'; \ } > $@ -# -# Inputs/outputs for the sqlite3-api.js family. -# -# sqlite3-api.jses = the list of JS files which make up -# sqlite3-api.js, in the order they need to be assembled. -sqlite3-api.jses = $(sqlite3-license-version.js) -sqlite3-api.jses += $(dir.api)/sqlite3-api-prologue.js -sqlite3-api.jses += $(sqlite3-api-build-version.js) -sqlite3-api.jses += $(dir.common)/whwasmutil.js -sqlite3-api.jses += $(dir.jacc)/jaccwabyt.js -sqlite3-api.jses += $(dir.api)/sqlite3-api-glue.c-pp.js -sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.c-pp.js -sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.c-pp.js -sqlite3-api.jses += $(dir.api)/sqlite3-vfs-helper.c-pp.js -ifeq (0,$(wasm-bare-bones)) - sqlite3-api.jses += $(dir.api)/sqlite3-vtab-helper.c-pp.js -endif -sqlite3-api.jses += $(dir.api)/sqlite3-vfs-kvvfs.c-pp.js -sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js -sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.c-pp.js - -# Parallel builds can fail if $(sqlite3-license-version.js) is not -# created early enough, so make all files in $(sqlite-api.jses) except -# for $(sqlite3-license-version.js) depend on -# $(sqlite3-license-version.js). -deps.jses = $(filter-out $(sqlite3-license-version.js),$(sqlite3-api.jses)) -$(deps.jses): $(sqlite3-license-version.js) - # # extern-post-js* and extern-pre-js* are files for use with # Emscripten's --extern-pre-js and --extern-post-js flags. @@ -958,7 +918,7 @@ $(post-js.in.js): $(MKDIR.bld) $(post-jses.js) $(MAKEFILE) done > $@ # -# speedtest1 decls needed before the $(bin.mkwb)-generated makefile +# speedtest1 decls needed before the $(bin.mkws)-generated makefile # is included. # bin.speedtest1 = ../../speedtest1 @@ -997,21 +957,12 @@ endif # /shell.c ######################################################################## -# -# Fiddle-related decls we need before .wasmbuilds is included -# - -fiddle.c.in = $(dir.top)/shell.c $(sqlite3-wasm.c) - EXPORTED_FUNCTIONS.fiddle = $(dir.tmp)/EXPORTED_FUNCTIONS.fiddle -$(EXPORTED_FUNCTIONS.fiddle): $(EXPORTED_FUNCTIONS.api.in) \ - $(MAKEFILE_LIST) $(bin.c-pp) - @$(call b.mkdir@) - @$(call b.c-pp.shcmd,fiddle,$(EXPORTED_FUNCTIONS.api.in),\ - $@,$(EXPORTED_FUNCTIONS.c-pp.flags) -Dfiddle) +$(EXPORTED_FUNCTIONS.fiddle): $(fiddle.EXPORTED_FUNCTIONS.in) $(MAKEFILE_LIST) + @$(b.mkdir@) + @sort -u $(fiddle.EXPORTED_FUNCTIONS.in) > $@ @echo $(logtag.@) $(emo.disk) - emcc.flags.fiddle = \ $(emcc.cflags) $(emcc_opt_full) \ --minify 0 \ @@ -1036,23 +987,26 @@ emcc.flags.fiddle = \ -USQLITE_WASM_BARE_BONES \ -DSQLITE_SHELL_FIDDLE +clean: clean-fiddle +clean-fiddle: + rm -f $(dir.fiddle)/fiddle-module.js \ + $(dir.fiddle)/*.wasm \ + $(dir.fiddle)/sqlite3-opfs-*.js \ + $(dir.fiddle)/*.gz \ + EXPORTED_FUNCTIONS.fiddle + rm -fr $(dir.fiddle-debug) + emcc.flags.fiddle.debug = $(emcc.flags.fiddle) \ -DSQLITE_DEBUG \ -DSQLITE_ENABLE_SELECTTRACE \ -DSQLITE_ENABLE_WHERETRACE -clean: clean-fiddle -clean-fiddle: - rm -f $(dir.fiddle)/fiddle-module.js \ - $(dir.fiddle)/*.wasm \ - $(dir.fiddle)/sqlite3-opfs-*.js \ - $(dir.fiddle)/*.gz \ - $(dir.fiddle)/index.html \ - $(EXPORTED_FUNCTIONS.fiddle) - rm -fr $(dir.fiddle-debug) -distclean: distclean-fiddle -distclean-fiddle: - rm -fr $(dir.fiddle)/jqterm +fiddle.EXPORTED_FUNCTIONS.in = \ + EXPORTED_FUNCTIONS.fiddle.in \ + $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-core \ + $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-extras + +fiddle.c.in = $(dir.top)/shell.c $(sqlite3-wasm.c) # # WASMFS build - unsupported and untested. We used WASMFS @@ -1074,14 +1028,6 @@ cflags.wasmfs = -DSQLITE_ENABLE_WASMFS # end wasmfs (the rest is in mkwasmbuilds.c) # -# -# -# -sqlite3-wasm.c = $(dir.api)/sqlite3-wasm.c -# List of input files for compiling $(sqlite3-wasm.c). That file -# #include's sqlite3.c directly, so it's implicitly includes here. -sqlite3-wasm.c.in = $(sqlite3-wasm.c) $(sqlite3_wasm_extra_init.c) - # # $(bin.mkwb) is used for generating much of the makefile code for the # various wasm builds. It used to be generated in this makefile via a @@ -1137,10 +1083,10 @@ sqlite3.ext.js = define gen-worker1 # $1 = X.ext part of sqlite3-worker1X.ext # $2 = $(c-pp.D.NAME) -$(call b.c-pp.target,filter,$$(dir.api)/sqlite3-worker1.c-pp.js,\ - $$(dir.dout)/sqlite3-worker1$(1),$(2)) -sqlite3.ext.js += $$(dir.dout)/sqlite3-worker1$(1) -all: $$(dir.dout)/sqlite3-worker1$(1) +$(call b.c-pp.target,filter,$(dir.api)/sqlite3-worker1.c-pp.js,\ + $(dir.dout)/sqlite3-worker1$(1),$(2)) +sqlite3.ext.js += $(dir.dout)/sqlite3-worker1$(1) +all: $(dir.dout)/sqlite3-worker1$(1) endef $(eval $(call gen-worker1,.js,$(c-pp.D.vanilla))) @@ -1154,10 +1100,10 @@ $(eval $(call gen-worker1,-bundler-friendly.mjs,$(c-pp.D.bundler))) define gen-promiser # $1 = X.ext part of sqlite3-worker1-promiserX.ext # $2 = $(c-pp.D.NAME) -$(call b.c-pp.target,filter,$$(dir.api)/sqlite3-worker1-promiser.c-pp.js,\ - $$(dir.dout)/sqlite3-worker1-promiser$(1),$(2)) -sqlite3.ext.js += $$(dir.dout)/sqlite3-worker1-promiser$(1) -all: $$(dir.dout)/sqlite3-worker1-promiser$(1) +$(call b.c-pp.target,filter,$(dir.api)/sqlite3-worker1-promiser.c-pp.js,\ + $(dir.dout)/sqlite3-worker1-promiser$(1),$(2)) +sqlite3.ext.js += $(dir.dout)/sqlite3-worker1-promiser$(1) +all: $(dir.dout)/sqlite3-worker1-promiser$(1) endef $(eval $(call gen-promiser,.js,$(c-pp.D.vanilla))) @@ -1199,19 +1145,12 @@ $(dir.dout)/sqlite3-opfs-async-proxy.js: $(dir.api)/sqlite3-opfs-async-proxy.js # we don't otherwise have a great place to attach them such that # they're always copied when we need them. # -# The var $(out.$(B).js) comes from $(bin.mkwb) and $(B) is the name -# of a build set up by that tool, e.g. b-vanilla or b-esm64. -# $(foreach B,$(b.names),$(eval $(out.$(B).js): $(sqlite3.ext.js))) - # # b-all: builds all available js/wasm builds. # $(foreach B,$(b.names),$(eval b-all: $(out.$(B).js))) -#$(foreach B,$(b.names),$(eval pre: $(pre-js.$(B).js))) -$(foreach B,$(b.names),$(eval post: $(post-js.$(B).js))) - # # speedtest1 is our primary benchmarking tool. # @@ -1221,7 +1160,7 @@ $(foreach B,$(b.names),$(eval post: $(post-js.$(B).js))) # These flags get applied via $(bin.mkwb). emcc.speedtest1.common = $(emcc_opt_full) emcc.speedtest1 = -I. -I$(dir $(sqlite3.canonical.c)) -emcc.speedtest1 += -sENVIRONMENT=web,worker +emcc.speedtest1 += -sENVIRONMENT=web emcc.speedtest1 += -sALLOW_MEMORY_GROWTH emcc.speedtest1 += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.32) emcc.speedtest1.common += -sINVOKE_RUN=0 @@ -1260,37 +1199,28 @@ speedtest1.exit-runtime1 = -sEXIT_RUNTIME=1 # -sEXIT_RUNTIME=1 but we need EXIT_RUNTIME=0 for the worker-based app # which runs speedtest1 multiple times. -$(EXPORTED_FUNCTIONS.speedtest1): $(EXPORTED_FUNCTIONS.api) - @$(call b.mkdir@); $(call b.echo,@,$(emo.disk)); \ - { echo _wasm_main; cat $(EXPORTED_FUNCTIONS.api); } > $@ || exit +$(EXPORTED_FUNCTIONS.speedtest1): $(EXPORTED_FUNCTIONS.api.core) + @$(call b.echo,@,$(emo.disk)); \ + $(call b.mkdir@); \ + { echo _wasm_main; cat $(EXPORTED_FUNCTIONS.api.core); } > $@ || exit speedtest1: b-speedtest1 -st: speedtest1 # # Generate 64-bit variants of speedtest1*.{js,html} # -# $1 = input file -# $2 = output file -# -# TODO: preprocess these like we do the rest. -# define gen-st64 $(2): $(1) @$$(call b.echo,speedtest164,$$(emo.disk)$(emo.lock) Creating from $$<) - rm -f $$@; \ - sed -e 's/speedtest1\.js/speedtest1-64bit\.js/' \ - -e 's/speedtest1-worker\.js/speedtest1-worker-64bit\.js/' \ - < $$< > $$@; \ + @rm -f $$@; \ + sed -e 's/$(3)\.js/$(3)-64bit\.js/' < $$< > $$@; \ chmod -w $$@ -$(2): b-speedtest164 -speedtest1: $(1) $(2) +b-speedtest164: $(2) CLEAN_FILES += $(2) endef -speedtest1: b-speedtest164 -$(eval $(call gen-st64,speedtest1.html,speedtest1-64bit.html)) -$(eval $(call gen-st64,speedtest1-worker.html,speedtest1-worker-64bit.html)) -$(eval $(call gen-st64,speedtest1-worker.js,speedtest1-worker-64bit.js)) +$(eval $(call gen-st64,speedtest1.html,speedtest1-64bit.html,speedtest1)) +$(eval $(call gen-st64,speedtest1-worker.html,speedtest1-worker-64bit.html,speedtest1-worker)) +$(eval $(call gen-st64,speedtest1-worker.js,speedtest1-worker-64bit.js,speedtest1-worker)) # end speedtest1.js ######################################################################## @@ -1314,11 +1244,9 @@ $(eval $(call gen-st64,speedtest1-worker.js,speedtest1-worker-64bit.js)) # # To create those, we filter tester1.c-pp.js/html with $(bin.c-pp)... -# # tester1.js variants: -# define gen-tester1.js -# $1 = build name to have a dep on +# $1 = build name to have dep on # $2 = suffix for tester1SUFFIX JS # $3 = $(bin.c-pp) flags $(call b.c-pp.target,test,tester1.c-pp.js,tester1$(2),$(3)) @@ -1327,18 +1255,20 @@ tester1-$(1): tester1$(2) tester1: tester1$(2) endef -$(eval $(call gen-tester1.js,vanilla,.js,\ - $(c-pp.D.vanilla) -Dsqlite3.js=$(dir.dout)/sqlite3.js)) -$(eval $(call gen-tester1.js,vanilla64,-64bit.js,\ - $(c-pp.D.vanilla64) -Dsqlite3.js=$(dir.dout)/sqlite3-64bit.js)) -$(eval $(call gen-tester1.js,esm,.mjs,\ - $(c-pp.D.esm) -Dsqlite3.js=$(dir.dout)/sqlite3.mjs)) -$(eval $(call gen-tester1.js,esm64,-64bit.mjs,\ - $(c-pp.D.esm64) -Dsqlite3.js=$(dir.dout)/sqlite3-64bit.mjs)) +$(eval $(call gen-tester1.js,vanilla,.js, \ + $(c-pp.D.vanilla) \ + -Dsqlite3.js=$(dir.dout)/sqlite3.js)) +$(eval $(call gen-tester1.js,vanilla64,-64bit.js, \ + $(c-pp.D.vanilla64) \ + -Dsqlite3.js=$(dir.dout)/sqlite3-64bit.js)) +$(eval $(call gen-tester1.js,esm,.mjs, \ + $(c-pp.D.esm) \ + -Dsqlite3.js=$(dir.dout)/sqlite3.mjs)) +$(eval $(call gen-tester1.js,esm64,-64bit.mjs, \ + $(c-pp.D.esm64) \ + -Dsqlite3.js=$(dir.dout)/sqlite3-64bit.mjs)) -# # tester1.html variants: -# define gen-tester1.html # $1 = build name to have a dep on # $2 = filename suffix: empty, -64bit, -esm, esm-64bit @@ -1373,11 +1303,9 @@ $(eval $(call gen-tester1.html,esm64,-esm-64bit,\ -Dtester1.js=tester1-64bit.mjs \ -Dsqlite3.js=$(dir.dout)/sqlite3-64bit.mjs)) -# -# tester1-worker.html variants: There is no ESM variant of this -# file. Instead, that page accepts the ?esm URL flag to switch to ESM -# mode. -# +# tester1-worker.html variants: +# There is no ESM variant of this file. Instead, that page accepts a +# ?esm URL flag to switch to ESM mode. $(eval $(call b.c-pp.target,test,tester1-worker.c-pp.html,\ tester1-worker.html,-Dbitness=32)) $(eval $(call b.c-pp.target,test,tester1-worker.c-pp.html,\ @@ -1387,57 +1315,8 @@ tester1-worker.html: tester1.mjs tester1-worker-64bit.html: tester1-64bit.mjs all: tester1 -# # end tester1 -# - -# -# jquery.terminal support for fiddle: -# -# If a clone of https://github.com/jcubic/jquery.terminal -# is found in $(JQTERM), defaulting to $(HOME)/src/jquery.terminal -# then add jquery.terminal support to fiddle. -# -# To build that package, from its checkout dir: -# -# npm install -# make -# -c-pp.D.fiddle ?= -JQTERM ?= $(HOME)/src/jquery.terminal -dir.jqtermExt = $(firstword $(wildcard $(JQTERM))) -#$(info dir.jqtermExt=$(dir.jqtermExt)) -ifeq (0,$(MAKING_CLEAN)) -ifeq (,$(wildcard $(dir.jqtermExt)/js/jquery.terminal.min.js)) -$(info $(emo.magic) To add jquery.terminal support to fiddle, set JQTERM=/path/to/its/built/checkout) -else -$(info $(emo.magic) jquery.terminal found in $(dir.jqtermExt) - adding it to fiddle. Make sure it is built!) - -dir.jqterm = $(dir.fiddle)/jqterm -$(dir.fiddle)/jqterm/jquery.terminal.bundle.min.js: - @$(call b.mkdir@) - cat $(dir.jqtermExt)/js/jquery-1*.min.js \ - $(dir.jqtermExt)/js/jquery.terminal.min.js > $@ - -$(dir.fiddle)/jqterm/jquery.terminal.min.css: $(dir.jqtermExt)/css/jquery.terminal.min.css - @$(call b.mkdir@) - @$(call b.cp,fiddle,$<,$(dir $@)) - -$(dir.fiddle)/index.html: $(dir.fiddle)/jqterm/jquery.terminal.bundle.min.js \ - $(dir.fiddle)/jqterm/jquery.terminal.min.css -c-pp.D.fiddle += -Djqterm -endif -endif -# ^^^ JQTERM/MAKING_CLEAN - -# -# Generate fiddle/index.html. Must come after JQTERM is handled. -# -$(dir.fiddle)/index.html: $(dir.fiddle)/index.c-pp.html -$(eval $(call b.c-pp.target,fiddle,\ - $(dir.fiddle)/index.c-pp.html,$(dir.fiddle)/index.html,$(c-pp.D.fiddle))) -$(out.fiddle.wasm): $(dir.fiddle)/index.html - +######################################################################## # # Convenience rules to rebuild with various -Ox levels. Much @@ -1448,7 +1327,6 @@ $(out.fiddle.wasm): $(dir.fiddle)/index.html # # Achtung: build times with anything higher than -O0 are somewhat # painful, which is why -O0 is the default. -# .PHONY: o0 o1 o2 o3 os oz emcc-opt-extra = #ifeq (1,$(wasm-bare-bones)) @@ -1499,9 +1377,7 @@ push-testing: ssh wasm-testing 'cd $(wasm-testing.dir) && bash .gzip' || \ echo "SSH failed: it's likely that stale content will be served via old gzip files." -# # build everything needed by push-testing with -Oz -# .PHONY: for-testing for-testing: emcc_opt=-Oz for-testing: loud=1 @@ -1528,9 +1404,9 @@ update-docs: exit 127 else wasm.docs.jswasm = $(wasm.docs.home)/jswasm -update-docs: $(bin.stripccomments) $(out.vanilla.js) $(out.vanilla.wasm) +update-docs: $(bin.stripccomments) $(out.sqlite3.js) $(out.sqlite3.wasm) @echo "Copying files to the /wasm docs. Be sure to use an -Oz build for this!"; - cp -p $(out.vanilla.wasm) $(wasm.docs.jswasm)/. + cp -p $(sqlite3.wasm) $(wasm.docs.jswasm)/. $(bin.stripccomments) -k -k < $(out.vanilla.js) \ | sed -e '/^[ \t]*$$/d' > $(wasm.docs.jswasm)/sqlite3.js cp -p demo-123.js demo-123.html demo-123-worker.html $(wasm.docs.home)/. @@ -1586,46 +1462,13 @@ endif dist-name-prefix = sqlite-wasm$(dist-name-extra) .PHONY: dist dist: - $(bin.bash) ./mkdist.sh $(dist-name-prefix) + ./mkdist.sh $(dist-name-prefix) snapshot: - $(bin.bash) ./mkdist.sh $(dist-name-prefix) --snapshot + ./mkdist.sh $(dist-name-prefix) --snapshot endif # ^^^ making dist/snapshot CLEAN_FILES += $(wildcard sqlite-wasm-*.zip) -######################################################################## -# The npm target is specifically for preparing files for the downstream -# https://github.com/sqlite/sqlite-wasm (npm) distribution. -# -# Per agreement with that project's maintainers, these filenames need -# to remain stable. To avoid breakage in their deployment process, any -# changes (like renaming files) which potentially break their deployment -# needs to be communicated to that project via opening a new ticket or -# direct coordination with its maintainers. -# -# This target does a full clean/rebuild so that we can ensure that the -# optimization level is set consistently across all files. -# -npm.bundle.zip = npm-bundle.zip -CLEAN_FILES += $(npm.bundle.zip) -# Distributables which need to be built for npm: -npm_files = $(addprefix $(dir.dout)/, \ -sqlite3-bundler-friendly.mjs \ -sqlite3-opfs-async-proxy.js \ -sqlite3-worker1-bundler-friendly.mjs \ -sqlite3-worker1-promiser.mjs \ -sqlite3-worker1.mjs \ -sqlite3.mjs \ -sqlite3.wasm \ -sqlite3-node.mjs \ -) -npm: $(sqlite3.canonical.c) - @echo "$(emo.cleanup) Forcing a clean rebuild to ensure consistent optimization flags." - $(MAKE) clean - $(MAKE) -e "emcc_opt=-Oz $(emcc-opt-extra)" $(npm_files) - rm -f $(npm.bundle.zip); zip -r $(npm.bundle.zip) $(npm_files) - unzip -l $(npm.bundle.zip) - ######################################################################## # Explanation of, and some commentary on, various emcc build flags # follows. Full docs for these can be found at: diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.c-pp b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core similarity index 61% rename from ext/wasm/api/EXPORTED_FUNCTIONS.c-pp rename to ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core index 2cdddf1e7..506054510 100644 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.c-pp +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core @@ -75,7 +75,6 @@ _sqlite3_limit _sqlite3_malloc _sqlite3_malloc64 _sqlite3_msize -_sqlite3_next_stmt _sqlite3_open _sqlite3_open_v2 _sqlite3_overload_function @@ -156,92 +155,3 @@ _sqlite3_vtab_in_next _sqlite3_vtab_nochange _sqlite3_vtab_on_conflict _sqlite3_vtab_rhs_value -//#if not bare-bones -_sqlite3_column_database_name -_sqlite3_column_origin_name -_sqlite3_column_table_name -_sqlite3_create_module -_sqlite3_create_module_v2 -_sqlite3_create_window_function -_sqlite3_declare_vtab -_sqlite3_drop_modules -_sqlite3_preupdate_blobwrite -_sqlite3_preupdate_count -_sqlite3_preupdate_depth -_sqlite3_preupdate_hook -_sqlite3_preupdate_new -_sqlite3_preupdate_old -_sqlite3_progress_handler -_sqlite3_set_authorizer -_sqlite3_vtab_collation -_sqlite3_vtab_distinct -_sqlite3_vtab_in -_sqlite3_vtab_in_first -_sqlite3_vtab_in_next -_sqlite3_vtab_nochange -_sqlite3_vtab_on_conflict -_sqlite3_vtab_rhs_value -_sqlite3changegroup_add -_sqlite3changegroup_add_strm -_sqlite3changegroup_delete -_sqlite3changegroup_new -_sqlite3changegroup_output -_sqlite3changegroup_output_strm -_sqlite3changeset_apply -_sqlite3changeset_apply_strm -_sqlite3changeset_apply_v2 -_sqlite3changeset_apply_v2_strm -_sqlite3changeset_apply_v3 -_sqlite3changeset_apply_v3_strm -_sqlite3changeset_concat -_sqlite3changeset_concat_strm -_sqlite3changeset_conflict -_sqlite3changeset_finalize -_sqlite3changeset_fk_conflicts -_sqlite3changeset_invert -_sqlite3changeset_invert_strm -_sqlite3changeset_new -_sqlite3changeset_next -_sqlite3changeset_old -_sqlite3changeset_op -_sqlite3changeset_pk -_sqlite3changeset_start -_sqlite3changeset_start_strm -_sqlite3changeset_start_v2 -_sqlite3changeset_start_v2_strm -_sqlite3session_attach -_sqlite3session_changeset -_sqlite3session_changeset_size -_sqlite3session_changeset_strm -_sqlite3session_config -_sqlite3session_create -_sqlite3session_delete -_sqlite3session_diff -_sqlite3session_enable -_sqlite3session_indirect -_sqlite3session_isempty -_sqlite3session_memory_used -_sqlite3session_object_config -_sqlite3session_patchset -_sqlite3session_patchset_strm -_sqlite3session_table_filter -//#endif not bare-bones -//#if enable-see -_sqlite3_key -_sqlite3_key_v2 -_sqlite3_rekey -_sqlite3_rekey_v2 -_sqlite3_activate_see -//#endif enable-see -//#if fiddle -_fiddle_db_arg -_fiddle_db_filename -_fiddle_exec -_fiddle_experiment -_fiddle_interrupt -_fiddle_main -_fiddle_reset_db -_fiddle_db_handle -_fiddle_db_vfs -_fiddle_export_db -//#endif fiddle diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras new file mode 100644 index 000000000..e8304b5f2 --- /dev/null +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras @@ -0,0 +1,68 @@ +_sqlite3_column_database_name +_sqlite3_column_origin_name +_sqlite3_column_table_name +_sqlite3_create_module +_sqlite3_create_module_v2 +_sqlite3_create_window_function +_sqlite3_declare_vtab +_sqlite3_drop_modules +_sqlite3_preupdate_blobwrite +_sqlite3_preupdate_count +_sqlite3_preupdate_depth +_sqlite3_preupdate_hook +_sqlite3_preupdate_new +_sqlite3_preupdate_old +_sqlite3_progress_handler +_sqlite3_set_authorizer +_sqlite3_vtab_collation +_sqlite3_vtab_distinct +_sqlite3_vtab_in +_sqlite3_vtab_in_first +_sqlite3_vtab_in_next +_sqlite3_vtab_nochange +_sqlite3_vtab_on_conflict +_sqlite3_vtab_rhs_value +_sqlite3changegroup_add +_sqlite3changegroup_add_strm +_sqlite3changegroup_delete +_sqlite3changegroup_new +_sqlite3changegroup_output +_sqlite3changegroup_output_strm +_sqlite3changeset_apply +_sqlite3changeset_apply_strm +_sqlite3changeset_apply_v2 +_sqlite3changeset_apply_v2_strm +_sqlite3changeset_apply_v3 +_sqlite3changeset_apply_v3_strm +_sqlite3changeset_concat +_sqlite3changeset_concat_strm +_sqlite3changeset_conflict +_sqlite3changeset_finalize +_sqlite3changeset_fk_conflicts +_sqlite3changeset_invert +_sqlite3changeset_invert_strm +_sqlite3changeset_new +_sqlite3changeset_next +_sqlite3changeset_old +_sqlite3changeset_op +_sqlite3changeset_pk +_sqlite3changeset_start +_sqlite3changeset_start_strm +_sqlite3changeset_start_v2 +_sqlite3changeset_start_v2_strm +_sqlite3session_attach +_sqlite3session_changeset +_sqlite3session_changeset_size +_sqlite3session_changeset_strm +_sqlite3session_config +_sqlite3session_create +_sqlite3session_delete +_sqlite3session_diff +_sqlite3session_enable +_sqlite3session_indirect +_sqlite3session_isempty +_sqlite3session_memory_used +_sqlite3session_object_config +_sqlite3session_patchset +_sqlite3session_patchset_strm +_sqlite3session_table_filter diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see new file mode 100644 index 000000000..83f3a97db --- /dev/null +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see @@ -0,0 +1,5 @@ +_sqlite3_key +_sqlite3_key_v2 +_sqlite3_rekey +_sqlite3_rekey_v2 +_sqlite3_activate_see diff --git a/ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api b/ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api new file mode 100644 index 000000000..aab1d8bd3 --- /dev/null +++ b/ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api @@ -0,0 +1,3 @@ +FS +wasmMemory + diff --git a/ext/wasm/api/README.md b/ext/wasm/api/README.md index 279d216bf..3c9669e6b 100644 --- a/ext/wasm/api/README.md +++ b/ext/wasm/api/README.md @@ -23,7 +23,7 @@ this writing, but is not set in stone forever and may change at any time. This doc targets maintainers of this code and those wanting to dive in to the details, not end user. -First off, a [pikchr][] of the proverbial onion: +First off, a pikchr of the proverbial onion: ```pikchr toggle center scale = 0.85 @@ -60,14 +60,14 @@ maintenance point of view. At the center of the onion is `sqlite3-api.js`, which gets generated by concatenating the following files together in their listed order: -- **`sqlite3-api-prologue.js`** +- **`sqlite3-api-prologue.js`**\ Contains the initial bootstrap setup of the sqlite3 API - objects. This is exposed as a bootstrapping function so that + objects. This is exposed as a function, rather than objects, so that the next step can pass in a config object which abstracts away parts of the WASM environment, to facilitate plugging it in to arbitrary WASM toolchains. The bootstrapping function gets removed from the global scope in a later stage of the bootstrapping process. -- **`../common/whwasmutil.js`** +- **`../common/whwasmutil.js`**\ A semi-third-party collection of JS/WASM utility code intended to replace much of the Emscripten glue. The sqlite3 APIs internally use these APIs instead of their Emscripten counterparts, in order to be @@ -77,78 +77,79 @@ by concatenating the following files together in their listed order: toolchains. It is "semi-third-party" in that it was created in order to support this tree but is standalone and maintained together with... -- **`../jaccwabyt/jaccwabyt.js`** +- **`../jaccwabyt/jaccwabyt.js`**\ Another semi-third-party API which creates bindings between JS and C structs, such that changes to the struct state from either JS or C are visible to the other end of the connection. This is also an independent spinoff project, conceived for the sqlite3 project but maintained separately. -- **`sqlite3-api-glue.js`** +- **`sqlite3-api-glue.js`**\ Invokes functionality exposed by the previous two files to flesh out low-level parts of `sqlite3-api-prologue.js`. Most of these pieces involve populating the `sqlite3.capi.wasm` object and creating `sqlite3.capi.sqlite3_...()` bindings. This file also deletes most global-scope symbols the above files create, effectively moving them into the scope being used for initializing the API. -- **`<build>/sqlite3-api-build-version.js`** +- **`<build>/sqlite3-api-build-version.js`**\ Gets created by the build process and populates the `sqlite3.version` object. This part is not critical, but records the version of the library against which this module was built. -- **`sqlite3-api-oo1.js`** +- **`sqlite3-api-oo1.js`**\ Provides a high-level object-oriented wrapper to the lower-level C API, colloquially known as OO API #1. Its API is similar to other high-level sqlite3 JS wrappers and should feel relatively familiar to anyone familiar with such APIs. It is not a "required component" and can be elided from builds which do not want it. -- **`sqlite3-api-worker1.js`** +- **`sqlite3-api-worker1.js`**\ A Worker-thread-based API which uses OO API #1 to provide an interface to a database which can be driven from the main Window thread via the Worker message-passing interface. Like OO API #1, this is an optional component, offering one of any number of potential implementations for such an API. - - **`sqlite3-worker1.js`** + - **`sqlite3-worker1.js`**\ Is not part of the amalgamated sources and is intended to be loaded by a client Worker thread. It loads the sqlite3 module and runs the Worker #1 API which is implemented in `sqlite3-api-worker1.js`. - - **`sqlite3-worker1-promiser.js`** + - **`sqlite3-worker1-promiser.js`**\ Is likewise not part of the amalgamated sources and provides a Promise-based interface into the Worker #1 API. This is a far user-friendlier way to interface with databases running in a Worker thread. -- **`sqlite3-vfs-helper.c-pp.js`** +- **`sqlite3-vfs-helper.js`**\ Installs the `sqlite3.vfs` namespace, which contain helpers for use by downstream code which creates `sqlite3_vfs` implementations. -- **`sqlite3-vtab-helper.c-pp.js`** +- **`sqlite3-vtab-helper.js`**\ Installs the `sqlite3.vtab` namespace, which contain helpers for use by downstream code which creates `sqlite3_module` implementations. -- **`sqlite3-vfs-opfs.c-pp.js`** +- **`sqlite3-vfs-opfs.c-pp.js`**\ is an sqlite3 VFS implementation which supports the [Origin-Private FileSystem (OPFS)][OPFS] as a storage layer to provide persistent storage for database files in a browser. It requires... - - **`sqlite3-opfs-async-proxy.js`** + - **`sqlite3-opfs-async-proxy.js`**\ is the asynchronous backend part of the [OPFS][] proxy. It speaks directly to the (async) OPFS API and channels those results back to its synchronous counterpart. This file, because it must be started in its own Worker, is not part of the amalgamation. -- **`sqlite3-vfs-opfs-sahpool.c-pp.js`** +- **`sqlite3-vfs-opfs-sahpool.c-pp.js`**\ is another sqlite3 VFS supporting the [OPFS][], but uses a completely different approach than the above-listed one. +- **`sqlite3-api-cleanup.js`**\ + The previous files do not immediately extend the library. Instead + they add callback functions to be called during its + bootstrapping. Some also temporarily create global objects in order + to communicate their state to the files which follow them. This file + cleans up any dangling globals and runs the API bootstrapping + process, which is what finally executes the initialization code + installed by the previous files. As of this writing, this code + ensures that the previous files leave no more than a single global + symbol installed - `sqlite3InitModule()`. When adapting the API for + non-Emscripten toolchains, this "should" be the only file, of those + in this list, where changes are needed. The Emscripten-specific + pieces described below may also require counterparts in any as-yet + hypothetical alternative build. -The previous files do not immediately extend the library. Instead they -install a global function `sqlite3ApiBootstrap()`, which downstream -code must call to configure the library for the current JS/WASM -environment. Each file listed above pushes a callback into the -bootstrapping queue, to be called as part of `sqlite3ApiBootstrap()`. -Some files also temporarily create global objects in order to -communicate their state to the files which follow them. Those -get cleaned up vi `post-js-footer.js`, described below. - -Adapting the build for non-Emscripten toolchains essentially requires packaging -the above files, concatated together, into that toolchain's "JS glue" -and, in the final stage of that glue, call `sqlite3ApiBootstrap()` and -return its result to the end user. **Files with the extension `.c-pp.js`** are intended [to be processed with `c-pp`](#c-pp), noting that such preprocessing may be applied @@ -170,27 +171,23 @@ from this file rather than `sqlite3.c`. The following Emscripten-specific files are injected into the build-generated `sqlite3.js` along with `sqlite3-api.js`. -- **`extern-pre-js.js`** +- **`extern-pre-js.js`**\ Emscripten-specific header for Emscripten's `--extern-pre-js` flag. As of this writing, that file is only used for experimentation purposes and holds no code relevant to the production deliverables. -- **`pre-js.c-pp.js`** +- **`pre-js.c-pp.js`**\ Emscripten-specific header for Emscripten's `--pre-js` flag. This file overrides certain Emscripten behavior before Emscripten does most of its work. -- **`post-js-header.js`** +- **`post-js-header.js`**\ Emscripten-specific header for the `--post-js` input. It opens up, but does not close, a function used for initializing the library. -- **`sqlite3-api.js`** gets sandwiched between these &uarr; two - &darr; files. -- **`post-js-footer.js`** +- (**`sqlite3-api.js`** gets sandwiched between these &uarr; two + &darr; files.) +- **`post-js-footer.js`**\ Emscripten-specific footer for the `--post-js` input. This closes - off the function opened by `post-js-header.js`. This file cleans up - any dangling globals and runs `sqlite3ApiBootstrap()`. As of this - writing, this code ensures that the previous files leave no more - than a single global symbol installed - `sqlite3InitModule()`. - -- **`extern-post-js.c-pp.js`** + off the function opened by `post-js-header.js`. +- **`extern-post-js.c-pp.js`**\ Emscripten-specific header for Emscripten's `--extern-post-js` flag. This file is run in the global scope. It overwrites the Emscripten-installed `sqlite3InitModule()` function with one which @@ -208,10 +205,9 @@ Preprocessing of Source Files Certain files in the build require preprocessing to filter in/out parts which differ between vanilla JS, ES6 Modules, and node.js builds. The preprocessor application itself is in -[`c-pp-lite.c`](/file/ext/wasm/c-pp-lite.c) and the complete technical -details of such preprocessing are maintained in +[`c-pp.c`](/file/ext/wasm/c-pp.c) and the complete technical details +of such preprocessing are maintained in [`GNUMakefile`](/file/ext/wasm/GNUmakefile). [OPFS]: https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system -[pikchr]: https://pikchr.org diff --git a/ext/wasm/api/extern-post-js.c-pp.js b/ext/wasm/api/extern-post-js.c-pp.js index cac6e4ab4..606e02ae2 100644 --- a/ext/wasm/api/extern-post-js.c-pp.js +++ b/ext/wasm/api/extern-post-js.c-pp.js @@ -26,7 +26,7 @@ const toExportForESM = */ const originalInit = sqlite3InitModule; if(!originalInit){ - throw new Error("Expecting sqlite3InitModule to be defined by the Emscripten build."); + throw new Error("Expecting globalThis.sqlite3InitModule to be defined by the Emscripten build."); } /** We need to add some state which our custom Module.locateFile() @@ -73,8 +73,6 @@ const toExportForESM = const sIM = globalThis.sqlite3InitModule = function ff(...args){ //console.warn("Using replaced sqlite3InitModule()",globalThis.location); - sIMS.emscriptenLocateFile = args[0]?.locateFile /* see pre-js.c-pp.js [tag:locateFile] */; - sIMS.emscriptenInstantiateWasm = args[0]?.instantiateWasm /* see pre-js.c-pp.js [tag:locateFile] */; return originalInit(...args).then((EmscriptenModule)=>{ sIMS.debugModule("sqlite3InitModule() sIMS =",sIMS); sIMS.debugModule("sqlite3InitModule() EmscriptenModule =",EmscriptenModule); diff --git a/ext/wasm/api/post-js-footer.js b/ext/wasm/api/post-js-footer.js index f8050ddd3..c6a2e1517 100644 --- a/ext/wasm/api/post-js-footer.js +++ b/ext/wasm/api/post-js-footer.js @@ -1,72 +1,3 @@ -/* - 2022-07-22 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - This file is the tail end of the sqlite3-api.js constellation, - closing the function scope opened by post-js-header.js. - - In terms of amalgamation code placement, this file is appended - immediately after the final sqlite3-api-*.js piece. Those files - cooperate to prepare sqlite3ApiBootstrap() and this file calls it. - It is run within a context which gives it access to Emscripten's - Module object, after sqlite3.wasm is loaded but before - sqlite3ApiBootstrap() has been called. - - Because this code resides (after building) inside the function - installed by post-js-header.js, it has access to state set up by - pre-js.c-pp.js and friends. -*/ -try{ - /* We are in the closing block of Module.runSQLite3PostLoadInit(), so - its arguments are visible here. */ - - /* Config options for sqlite3ApiBootstrap(). */ - const bootstrapConfig = Object.assign( - Object.create(null), - /** The WASM-environment-dependent configuration for sqlite3ApiBootstrap() */ - { - memory: ('undefined'!==typeof wasmMemory) - ? wasmMemory - : EmscriptenModule['wasmMemory'], - exports: ('undefined'!==typeof wasmExports) - ? wasmExports /* emscripten >=3.1.44 */ - : (Object.prototype.hasOwnProperty.call(EmscriptenModule,'wasmExports') - ? EmscriptenModule['wasmExports'] - : EmscriptenModule['asm']/* emscripten <=3.1.43 */) - }, - globalThis.sqlite3ApiBootstrap.defaultConfig, // default options - globalThis.sqlite3ApiConfig || {} // optional client-provided options - ); - - sqlite3InitScriptInfo.debugModule("Bootstrapping lib config", bootstrapConfig); - - /** - For purposes of the Emscripten build, call sqlite3ApiBootstrap(). - Ideally clients should be able to inject their own config here, - but that's not practical in this particular build constellation - because of the order everything happens in. Clients may either - define globalThis.sqlite3ApiConfig or modify - globalThis.sqlite3ApiBootstrap.defaultConfig to tweak the default - configuration used by a no-args call to sqlite3ApiBootstrap(), - but must have first loaded their WASM module in order to be able - to provide the necessary configuration state. - */ - const p = globalThis.sqlite3ApiBootstrap(bootstrapConfig); - delete globalThis.sqlite3ApiBootstrap; - return p /* the eventual result of globalThis.sqlite3InitModule() */; -}catch(e){ - console.error("sqlite3ApiBootstrap() error:",e); - throw e; -} - //console.warn("This is the end of the Module.runSQLite3PostLoadInit handler."); }/*Module.runSQLite3PostLoadInit(...)*/; //console.warn("This is the end of the setup of the (pending) Module.runSQLite3PostLoadInit"); diff --git a/ext/wasm/api/post-js-header.js b/ext/wasm/api/post-js-header.js index 670051bd8..cdc6b3a38 100644 --- a/ext/wasm/api/post-js-header.js +++ b/ext/wasm/api/post-js-header.js @@ -3,13 +3,14 @@ post-js.js for use with Emscripten's --post-js flag, so it gets injected in the earliest stages of sqlite3InitModule(). + This function wraps the whole SQLite3 library but does not + bootstrap it. + Running this function will bootstrap the library and return a Promise to the sqlite3 namespace object. - - In the canonical builds, this gets called by extern-post-js.c-pp.js */ -Module.runSQLite3PostLoadInit = async function( - sqlite3InitScriptInfo, +Module.runSQLite3PostLoadInit = function( + sqlite3InitScriptInfo /* populated by extern-post-js.c-pp.js */, EmscriptenModule/*the Emscripten-style module object*/, sqlite3IsUnderTest ){ @@ -34,6 +35,7 @@ Module.runSQLite3PostLoadInit = async function( - sqlite3-vtab-helper.c-pp.js => Utilities for virtual table impls - sqlite3-vfs-opfs.c-pp.js => OPFS VFS - sqlite3-vfs-opfs-sahpool.c-pp.js => OPFS SAHPool VFS + - sqlite3-api-cleanup.js => final bootstrapping phase - post-js-footer.js => this file's epilogue And all of that gets sandwiched between extern-pre-js.js and diff --git a/ext/wasm/api/pre-js.c-pp.js b/ext/wasm/api/pre-js.c-pp.js index 3910cb000..8a4a0f9fd 100644 --- a/ext/wasm/api/pre-js.c-pp.js +++ b/ext/wasm/api/pre-js.c-pp.js @@ -14,36 +14,12 @@ itself. i.e. try to keep file-local symbol names obnoxiously collision-resistant. */ -/** - This file was preprocessed using: - -//#@policy error - @c-pp::argv@ -//#@policy off -*/ -//#if unsupported-build -/** - UNSUPPORTED BUILD: - - This SQLite JS build configuration is entirely unsupported! It has - not been tested beyond the ability to compile it. It may not - load. It may not work properly. Only builds _directly_ targeting - browser environments ("vanilla" JS and ESM modules) are supported - and tested. Builds which _indirectly_ target browsers (namely - bundler-friendly builds) are not supported deliverables. -*/ -//#endif -//#if not target:es6-bundler-friendly (function(Module){ const sIMS = globalThis.sqlite3InitModuleState/*from extern-post-js.c-pp.js*/ || Object.assign(Object.create(null),{ - /* In WASMFS builds this file gets loaded once per thread, - but sqlite3InitModuleState is not getting set for the - worker threads? That those workers seem to function fine - despite that is curious. */ - debugModule: function(){ - console.warn("globalThis.sqlite3InitModuleState is missing",arguments); + debugModule: ()=>{ + console.warn("globalThis.sqlite3InitModuleState is missing"); } }); delete globalThis.sqlite3InitModuleState; @@ -71,14 +47,6 @@ approach. */ Module['locateFile'] = function(path, prefix) { - if( this.emscriptenLocateFile instanceof Function ){ - /* [tag:locateFile] Client-overridden impl. We do not support - this but offer it as a back-door which will go away the - moment either Emscripten changes that interface or we manage - to get non-Emscripten builds working. - https://sqlite.org/forum/forumpost/1eec339854c935bd */ - return this.emscriptenLocateFile(path, prefix); - } //#if target:es6-module return new URL(path, import.meta.url).href; //#else @@ -104,7 +72,8 @@ //#endif target:es6-module }.bind(sIMS); -//#if Module.instantiateWasm and not wasmfs and not target:node +//#if Module.instantiateWasm +//#if not wasmfs /** Override Module.instantiateWasm(). @@ -113,15 +82,8 @@ https://github.com/emscripten-core/emscripten/issues/17951 In such builds we must disable this. - - It's disabled in the (unsupported/untested) node builds because - node does not do fetch(). */ Module['instantiateWasm'] = function callee(imports,onSuccess){ - if( this.emscriptenInstantiateWasm instanceof Function ){ - /* See [tag:locateFile]. Same story here */ - return this.emscriptenInstantiateWasm(imports, onSuccess); - } const sims = this; const uri = Module.locateFile( sims.wasmFilename, ( @@ -147,7 +109,7 @@ .then(finalThen) return loadWasm(); }.bind(sIMS); -//#endif Module.instantiateWasm and not wasmfs +//#endif not wasmfs +//#endif Module.instantiateWasm })(Module); -//#endif not target:es6-bundler-friendly /* END FILE: api/pre-js.js. */ diff --git a/ext/wasm/api/sqlite3-api-cleanup.js b/ext/wasm/api/sqlite3-api-cleanup.js new file mode 100644 index 000000000..223566326 --- /dev/null +++ b/ext/wasm/api/sqlite3-api-cleanup.js @@ -0,0 +1,83 @@ +/* + 2022-07-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file is the tail end of the sqlite3-api.js constellation, + intended to be appended after all other sqlite3-api-*.js files so + that it can finalize any setup and clean up any global symbols + temporarily used for setting up the API's various subsystems. + + In Emscripten builds it's run in the context of what amounts to a + Module.postRun handler, though it's no longer actually a postRun + handler because Emscripten 4.0 changed postRun semantics in an + incompatible way. + + In terms of amalgamation code placement, this file is appended + immediately after the final sqlite3-api-*.js piece. Those files + cooperate to prepare sqlite3ApiBootstrap() and this file calls it. + It is run within a context which gives it access to Emscripten's + Module object, after sqlite3.wasm is loaded but before + sqlite3ApiBootstrap() has been called. + + Because this code resides (after building) inside the function + installed by post-js-header.js, it has access to the +*/ +'use strict'; +if( 'undefined' === typeof EmscriptenModule/*from post-js-header.js*/ ){ + console.warn("This is not running in the context of Module.runSQLite3PostLoadInit()"); + throw new Error("sqlite3-api-cleanup.js expects to be running in the "+ + "context of its Emscripten module loader."); +} +try{ + /* Config options for sqlite3ApiBootstrap(). */ + const bootstrapConfig = Object.assign( + Object.create(null), + globalThis.sqlite3ApiBootstrap.defaultConfig, // default options + globalThis.sqlite3ApiConfig || {}, // optional client-provided options + /** The WASM-environment-dependent configuration for sqlite3ApiBootstrap() */ + { + memory: ('undefined'!==typeof wasmMemory) + ? wasmMemory + : EmscriptenModule['wasmMemory'], + exports: ('undefined'!==typeof wasmExports) + ? wasmExports /* emscripten >=3.1.44 */ + : (Object.prototype.hasOwnProperty.call(EmscriptenModule,'wasmExports') + ? EmscriptenModule['wasmExports'] + : EmscriptenModule['asm']/* emscripten <=3.1.43 */) + } + ); + + /** Figure out if this is a 32- or 64-bit WASM build. */ + bootstrapConfig.wasmPtrIR = + 'number'===(typeof bootstrapConfig.exports.sqlite3_libversion()) + ? 'i32' :'i64'; + const sIMS = sqlite3InitScriptInfo; + sIMS.debugModule("Bootstrapping lib config", sIMS); + + /** + For purposes of the Emscripten build, call sqlite3ApiBootstrap(). + Ideally clients should be able to inject their own config here, + but that's not practical in this particular build constellation + because of the order everything happens in. Clients may either + define globalThis.sqlite3ApiConfig or modify + globalThis.sqlite3ApiBootstrap.defaultConfig to tweak the default + configuration used by a no-args call to sqlite3ApiBootstrap(), + but must have first loaded their WASM module in order to be able + to provide the necessary configuration state. + */ + const p = globalThis.sqlite3ApiBootstrap(bootstrapConfig); + delete globalThis.sqlite3ApiBootstrap; + return p /* the eventual result of globalThis.sqlite3InitModule() */; +}catch(e){ + console.error("sqlite3ApiBootstrap() error:",e); + throw e; +} +throw new Error("Maintenance required: this line should never be reached"); diff --git a/ext/wasm/api/sqlite3-api-glue.c-pp.js b/ext/wasm/api/sqlite3-api-glue.c-pp.js index d268331a3..1c42b0150 100644 --- a/ext/wasm/api/sqlite3-api-glue.c-pp.js +++ b/ext/wasm/api/sqlite3-api-glue.c-pp.js @@ -120,11 +120,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_column_double","f64", "sqlite3_stmt*", "int"], ["sqlite3_column_int","int", "sqlite3_stmt*", "int"], ["sqlite3_column_name","string", "sqlite3_stmt*", "int"], -//#define proxy-text-apis=1 -//#if not proxy-text-apis -/* Search this file for tag:proxy-text-apis to see what this is about. */ ["sqlite3_column_text","string", "sqlite3_stmt*", "int"], -//#endif ["sqlite3_column_type","int", "sqlite3_stmt*", "int"], ["sqlite3_column_value","sqlite3_value*", "sqlite3_stmt*", "int"], ["sqlite3_commit_hook", "void*", [ @@ -200,7 +196,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_libversion_number", "int"], ["sqlite3_limit", "int", ["sqlite3*", "int", "int"]], ["sqlite3_malloc", "*","int"], - ["sqlite3_next_stmt", "sqlite3_stmt*", ["sqlite3*","sqlite3_stmt*"]], ["sqlite3_open", "int", "string", "*"], ["sqlite3_open_v2", "int", "string", "*", "int", "string"], /* sqlite3_prepare_v2() and sqlite3_prepare_v3() are handled @@ -322,9 +317,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_value_numeric_type", "int", "sqlite3_value*"], ["sqlite3_value_pointer", "*", "sqlite3_value*", "string:static"], ["sqlite3_value_subtype", "int", "sqlite3_value*"], -//#if not proxy-text-apis ["sqlite3_value_text", "string", "sqlite3_value*"], -//#endif ["sqlite3_value_type", "int", "sqlite3_value*"], ["sqlite3_vfs_find", "*", "string"], ["sqlite3_vfs_register", "int", "sqlite3_vfs*", "int"], @@ -976,8 +969,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ "entry SQLITE_WASM_DEALLOC (=="+capi.SQLITE_WASM_DEALLOC+")."); } const __rcMap = Object.create(null); - for(const e of Object.entries(wasm.ctype['resultCodes'])){ - __rcMap[e[1]] = e[0]; + for(const t of ['resultCodes']){ + for(const e of Object.entries(wasm.ctype[t])){ + __rcMap[e[1]] = e[0]; + } } /** For the given integer, returns the SQLITE_xxx result code as a @@ -989,6 +984,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const notThese = Object.assign(Object.create(null),{ // For each struct to NOT register, map its name to true: WasmTestStruct: true, + /* We unregister the kvvfs VFS from Worker threads below. */ + sqlite3_kvvfs_methods: !util.isUIThread(), /* sqlite3_index_info and friends require int64: */ sqlite3_index_info: !wasm.bigIntEnabled, sqlite3_index_constraint: !wasm.bigIntEnabled, @@ -1657,45 +1654,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }/*sqlite3_bind_text/blob()*/ -//#if proxy-text-apis - if(!capi.sqlite3_column_text){ - /*[tag:proxy-text-apis] - As discussed at: - - https://sqlite.org/forum/forumpost/d77281aec2df9ada - - Summary: there are opinions that sqlite3_column_text() and - sqlite3_value_text() should handle strings such that embedded - NULs are retained. This block does that. This block does _not_ - apply that special-case behavior to any number of _other_ - APIs which return C-strings. That discrepancy makes this - block highly arguable, but one can also argue that these two - specific functions can get away with such acrobatics without - it being called voodoo in a pejorative sense. - */ - const argStmt = wasm.xWrap.argAdapter('sqlite3_stmt*'), - argInt = wasm.xWrap.argAdapter('int'), - argValue = wasm.xWrap.argAdapter('sqlite3_value*'), - newStr = - (cstr,n)=>wasm.typedArrayToString(wasm.heap8u(), - Number(cstr), Number(cstr)+n) - capi.sqlite3_column_text = function(stmt, colIndex){ - const a0 = argStmt(stmt), a1 = argInt(colIndex); - const cstr = wasm.exports.sqlite3_column_text(a0, a1); - return cstr - ? newStr(cstr,wasm.exports.sqlite3_column_bytes(a0, a1)) - : null; - }; - capi.sqlite3_value_text = function(val){ - const a0 = argValue(val); - const cstr = wasm.exports.sqlite3_value_text(a0); - return cstr - ? newStr(cstr,wasm.exports.sqlite3_value_bytes(a0)) - : null; - }; - }/*text-return-related bindings*/ -//#endif proxy-text-apis - {/* sqlite3_config() */ /** Wraps a small subset of the C API's sqlite3_config() options. @@ -1782,6 +1740,105 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }; }/* auto-extension */ + const pKvvfs = capi.sqlite3_vfs_find("kvvfs"); + if( pKvvfs ){/* kvvfs-specific glue */ + if(util.isUIThread()){ + const kvvfsMethods = new capi.sqlite3_kvvfs_methods( + wasm.exports.sqlite3__wasm_kvvfs_methods() + ); + delete capi.sqlite3_kvvfs_methods; + + const kvvfsMakeKey = wasm.exports.sqlite3__wasm_kvvfsMakeKeyOnPstack, + pstack = wasm.pstack; + + const kvvfsStorage = (zClass)=> + ((115/*=='s'*/===wasm.peek(zClass)) + ? sessionStorage : localStorage); + + /** + Implementations for members of the object referred to by + sqlite3__wasm_kvvfs_methods(). We swap out the native + implementations with these, which use localStorage or + sessionStorage for their backing store. + */ + const kvvfsImpls = { + xRead: (zClass, zKey, zBuf, nBuf)=>{ + const stack = pstack.pointer, + astack = wasm.scopedAllocPush(); + try { + const zXKey = kvvfsMakeKey(zClass,zKey); + if(!zXKey) return -3/*OOM*/; + const jKey = wasm.cstrToJs(zXKey); + const jV = kvvfsStorage(zClass).getItem(jKey); + if(!jV) return -1; + const nV = jV.length /* We are relying 100% on v being + ASCII so that jV.length is equal + to the C-string's byte length. */; + if(nBuf<=0) return nV; + else if(1===nBuf){ + wasm.poke(zBuf, 0); + return nV; + } + const zV = wasm.scopedAllocCString(jV); + if(nBuf > nV + 1) nBuf = nV + 1; + wasm.heap8u().copyWithin( + Number(zBuf), Number(zV), wasm.ptr.addn(zV, nBuf,- 1) + ); + wasm.poke(wasm.ptr.add(zBuf, nBuf, -1), 0); + return nBuf - 1; + }catch(e){ + sqlite3.config.error("kvstorageRead()",e); + return -2; + }finally{ + pstack.restore(stack); + wasm.scopedAllocPop(astack); + } + }, + xWrite: (zClass, zKey, zData)=>{ + const stack = pstack.pointer; + try { + const zXKey = kvvfsMakeKey(zClass,zKey); + if(!zXKey) return 1/*OOM*/; + const jKey = wasm.cstrToJs(zXKey); + kvvfsStorage(zClass).setItem(jKey, wasm.cstrToJs(zData)); + return 0; + }catch(e){ + sqlite3.config.error("kvstorageWrite()",e); + return capi.SQLITE_IOERR; + }finally{ + pstack.restore(stack); + } + }, + xDelete: (zClass, zKey)=>{ + const stack = pstack.pointer; + try { + const zXKey = kvvfsMakeKey(zClass,zKey); + if(!zXKey) return 1/*OOM*/; + kvvfsStorage(zClass).removeItem(wasm.cstrToJs(zXKey)); + return 0; + }catch(e){ + sqlite3.config.error("kvstorageDelete()",e); + return capi.SQLITE_IOERR; + }finally{ + pstack.restore(stack); + } + } + }/*kvvfsImpls*/; + for(const k of Object.keys(kvvfsImpls)){ + kvvfsMethods[kvvfsMethods.memberKey(k)] = + wasm.installFunction( + kvvfsMethods.memberSignature(k), + kvvfsImpls[k] + ); + } + }else{ + /* Worker thread: unregister kvvfs to avoid it being used + for anything other than local/sessionStorage. It "can" + be used that way but it's not really intended to be. */ + capi.sqlite3_vfs_unregister(pKvvfs); + } + }/*pKvvfs*/ + /* Warn if client-level code makes use of FuncPtrAdapter. */ wasm.xWrap.FuncPtrAdapter.warnOnUse = true; @@ -1887,7 +1944,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } tgt[memKey] = fProxy; }else{ - const pFunc = wasm.installFunction(fProxy, sigN); + const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name)); tgt[memKey] = pFunc; if(!tgt.ondispose || !tgt.ondispose.__removeFuncList){ tgt.addOnDispose('ondispose.__removeFuncList handler', diff --git a/ext/wasm/api/sqlite3-api-oo1.c-pp.js b/ext/wasm/api/sqlite3-api-oo1.c-pp.js index 9338eef33..f7a4e9ebd 100644 --- a/ext/wasm/api/sqlite3-api-oo1.c-pp.js +++ b/ext/wasm/api/sqlite3-api-oo1.c-pp.js @@ -26,20 +26,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ the sqlite3 binding if, e.g., the wrapper is in the main thread and the sqlite3 API is in a worker. */ - const outWrapper = function(f){ - return (...args)=>f("sqlite3.oo1:",...args); - }; - - const debug = sqlite3.__isUnderTest - ? outWrapper(console.debug.bind(console)) - : outWrapper(sqlite3.config.debug); - const warn = sqlite3.__isUnderTest - ? outWrapper(console.warn.bind(console)) - : outWrapper(sqlite3.config.warn); - const error = sqlite3.__isUnderTest - ? outWrapper(console.error.bind(console)) - : outWrapper(sqlite3.config.error); - /** In order to keep clients from manipulating, perhaps inadvertently, the underlying pointer values of DB and Stmt @@ -102,8 +88,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ wasm.installFunction('i(ippp)', function(t,c,p,x){ if(capi.SQLITE_TRACE_STMT===t){ // x == SQL, p == sqlite3_stmt* - console.log("SQL TRACE #"+(++this.counter), - 'via sqlite3@'+c+'['+capi.sqlite3_db_filename(c,null)+']', + console.log("SQL TRACE #"+(++this.counter)+' via sqlite3@'+c+':', wasm.cstrToJs(x)); } }.bind({counter: 0})); @@ -228,18 +213,41 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ properties: - `.filename`: the db filename. It may be a special name like ":memory:" - or "". It may also be a URI-style name. + or "". - `.flags`: as documented in the DB constructor. - `.vfs`: as documented in the DB constructor. It also accepts those as the first 3 arguments. - - In non-default builds it may accept additional configuration - options. */ const dbCtorHelper = function ctor(...args){ + if(!ctor._name2vfs){ + /** + Map special filenames which we handle here (instead of in C) + to some helpful metadata... + + As of 2022-09-20, the C API supports the names :localStorage: + and :sessionStorage: for kvvfs. However, C code cannot + determine (without embedded JS code, e.g. via Emscripten's + EM_JS()) whether the kvvfs is legal in the current browser + context (namely the main UI thread). In order to help client + code fail early on, instead of it being delayed until they + try to read or write a kvvfs-backed db, we'll check for those + names here and throw if they're not legal in the current + context. + */ + ctor._name2vfs = Object.create(null); + const isWorkerThread = ('function'===typeof importScripts/*===running in worker thread*/) + ? (n)=>toss3("The VFS for",n,"is only available in the main window thread.") + : false; + ctor._name2vfs[':localStorage:'] = { + vfs: 'kvvfs', filename: isWorkerThread || (()=>'local') + }; + ctor._name2vfs[':sessionStorage:'] = { + vfs: 'kvvfs', filename: isWorkerThread || (()=>'session') + }; + } const opt = ctor.normalizeArgs(...args); //sqlite3.config.debug("DB ctor",opt); let pDb; @@ -261,6 +269,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ sqlite3.config.error("Invalid DB ctor args",opt,arguments); toss3("Invalid arguments for DB constructor:", arguments, "opts:", opt); } + let fnJs = wasm.isPtr(fn) ? wasm.cstrToJs(fn) : fn; + const vfsCheck = ctor._name2vfs[fnJs]; + if(vfsCheck){ + vfsName = vfsCheck.vfs; + fn = fnJs = vfsCheck.filename(fnJs); + } let oflags = 0; if( flagsStr.indexOf('c')>=0 ){ oflags |= capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE; @@ -285,15 +299,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }finally{ wasm.pstack.restore(stack); } - this.filename = - /* A poor design choice we have to keep: this.filename may be - in the form "file:....?....". It really should have been - sqlite3_db_filename(pDb) but that discrepancy went too long - unnoticed to be able to change without risk of - breakage. DB.dbFilename() can be used to fetch _just_ the - name part. - */ wasm.isPtr(fn) ? wasm.cstrToJs(fn) : fn; - + this.filename = fnJs; } __ptrMap.set(this, pDb); __stmtMap.set(this, Object.create(null)); @@ -384,12 +390,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ The given db filename must be resolvable using whatever filesystem layer (virtual or otherwise) is set up for the default - sqlite3 VFS or a VFS which can resolve it must be specified. + sqlite3 VFS. - The special sqlite3 db names ":memory:" and "" (temporary db) - have their normal special meanings here and need not resolve to - real filenames, but "" uses an on-storage temporary database and - requires that the VFS support that. + Note that the special sqlite3 db names ":memory:" and "" + (temporary db) have their normal special meanings here and need + not resolve to real filenames, but "" uses an on-storage + temporary database and requires that the VFS support that. The second argument specifies the open/create mode for the database. It must be string containing a sequence of letters (in @@ -802,9 +808,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ sqlite3_db_filename() value for the given database name, defaulting to "main". The argument may be either a JS string or a pointer to a WASM-allocated C-string. - - this.filename may be in the form of a URI-style string, whereas - the returned string contains only the filename part. */ dbFilename: function(dbName='main'){ return capi.sqlite3_db_filename(affirmDbOpen(this).pointer, dbName); @@ -926,15 +929,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ result set, but only if that statement has any result rows. The callback's "this" is the options object, noting that this function synthesizes one if the caller does not pass one to - exec(). The first argument passed to the callback is described - below. The second argument is always the current Stmt object, - as it's needed if the caller wants to fetch the column names or - some such (noting that they could also be fetched via - `this.columnNames`, if the client provides the `columnNames` - option). If the callback returns a literal `false` (as opposed - to any other falsy value, e.g. an implicit `undefined` return), - any ongoing statement-`step()` iteration stops without an - error. The return value of the callback is otherwise ignored. + exec(). The second argument passed to the callback is always + the current Stmt object, as it's needed if the caller wants to + fetch the column names or some such (noting that they could + also be fetched via `this.columnNames`, if the client provides + the `columnNames` option). If the callback returns a literal + `false` (as opposed to any other falsy value, e.g. an implicit + `undefined` return), any ongoing statement-`step()` iteration + stops without an error. The return value of the callback is + otherwise ignored. ACHTUNG: The callback MUST NOT modify the Stmt object. Calling any of the Stmt.get() variants, Stmt.getColumnName(), or @@ -967,23 +970,20 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ A.3) `'stmt'` causes the current Stmt to be passed to the callback, but this mode will trigger an exception if `resultRows` is an array because appending the transient - statement to the array would be downright unhelpful. This - option is a legacy feature, retained for backwards - compatibility. The statement object is passed as the second - argument to the callback, as described above. + statement to the array would be downright unhelpful. B) An integer, indicating a zero-based column in the result - row. Only that one single value, in JS form, will be passed on. + row. Only that one single value will be passed on. C) A string with a minimum length of 2 and leading character of '$' will fetch the row as an object, extract that one field, - and pass that field's value to the callback. These keys are - case-sensitive so must match the case used in the + and pass that field's value to the callback. Note that these + keys are case-sensitive so must match the case used in the SQL. e.g. `"select a A from t"` with a `rowMode` of `'$A'` would work but `'$a'` would not. A reference to a column not in the result set will trigger an exception on the first row (as - the check is not performed until rows are fetched). Note that - `$` is a legal identifier character in JS so need not be + the check is not performed until rows are fetched). Note also + that `$` is a legal identifier character in JS so need not be quoted. Any other `rowMode` value triggers an exception. @@ -1023,7 +1023,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - `callback` and `resultRows`: permit an array entries with semantics similar to those described for `bind` above. - OTOH, this function already does too much. */ exec: function(/*(sql [,obj]) || (obj)*/){ affirmDbOpen(this); @@ -1047,7 +1046,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /* Optimization: if the SQL is a TypedArray we can save some string conversion costs. */; /* Allocate the two output pointers (ppStmt, pzTail) and heap - space for the SQL (pSql). When prepare_v3() returns, pzTail + space for the SQL (pSql). When prepare_v2() returns, pzTail will point to somewhere in pSql. */ let sqlByteLen = isTA ? arg.sql.byteLength : wasm.jstrlen(arg.sql); const ppStmt = wasm.scopedAlloc( @@ -1059,8 +1058,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const pSqlEnd = wasm.ptr.add(pSql, sqlByteLen); if(isTA) wasm.heap8().set(arg.sql, pSql); else wasm.jstrcpy(arg.sql, wasm.heap8(), pSql, sqlByteLen, false); - wasm.poke8(wasm.ptr.add(pSql, sqlByteLen), 0/*NUL terminator*/); - while(pSql && wasm.peek8(pSql) + wasm.poke(wasm.ptr.add(pSql, sqlByteLen), 0/*NUL terminator*/); + while(pSql && wasm.peek(pSql, 'i8') /* Maintenance reminder:^^^ _must_ be 'i8' or else we will very likely cause an endless loop. What that's doing is checking for a terminating NUL byte. If we @@ -1124,7 +1123,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /* In order to trigger an exception in the INSERT...RETURNING locking scenario: https://sqlite.org/forum/forumpost/36f7a2e7494897df - [tag:insert-returning-reset] */).finalize(); stmt = null; }/*prepare() loop*/ @@ -1141,132 +1139,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return arg.returnVal(); }/*exec()*/, -//#if nope - /** - Experimental and untested - do not use. - - Prepares one or more SQL statements, passing each to a callback - for processing. - - It requires an options object with the following properties: - - - "sql": SQL in any format accepted by exec(). - - - "callback" (function): gets passed each prepared statement, - as described below. - - - "asPointer" (bool=false): if true, the callback is passed the - WASM (sqlite3*) pointer instead of a Stmt object. - - - "saveSql" (array): if set, the SQL of each prepared statement - is appended to this array. This can be used without a callback - to split SQL into its component statements. Purely empty - statements (for for which sqlite3_prepare() returns a NULL - sqlite3_stmt, i.e. spaces and comments) are not added to this - list unless... - - - "saveEmpty" (bool=false): If true, empty statements are - retained in opt.saveSql, but their leading/trailing whitespace - is trimmed (as for queries) so they may be empty. - - For each statement in the input SQL: - - 1) If opt.saveSql is set, the SQL is appended to it. - - 2) If callback is set, callback(S) is called, where S is either - a Stmt object (by default) or an (sqlite3*) WASM pointer (if - opt.asPointer is true). If the callback returns a literal true - (as opposed to any other truthy value), ownership of S is - transferred to the callback, otherwise S is reset and finalized - as soon as the callback returns. If the callback throws, S is - unconditionally finalized. - - If neither of opt.saveSql nor opt.callback are set, this - function does nothing more than prepare and finalize each - statement, which will trigger an exception if any of them - contain invalid SQL. - */ - forEachStmt: function(opt){ - affirmDbOpen(this); - opt ??= Object.create(null); - if(!opt.sql){ - return toss3("exec() requires an SQL string."); - } - const sql = util.flexibleString(opt.sql); - const callback = opt.callback; - let stmt, pStmt; - const stack = wasm.scopedAllocPush(); - const saveSql = Array.isArray(opt.saveSql) ? opt.saveSql : undefined; - try{ - const isTA = util.isSQLableTypedArray(opt.sql) - /* Optimization: if the SQL is a TypedArray we can save some string - conversion costs. */; - /* Allocate the two output pointers (ppStmt, pzTail) and heap - space for the SQL (pSql). When prepare_v3() returns, pzTail - will point to somewhere in pSql. */ - let sqlByteLen = isTA ? opt.sql.byteLength : wasm.jstrlen(sql); - const ppStmt = wasm.scopedAlloc( - /* output (sqlite3_stmt**) arg and pzTail */ - (2 * wasm.ptr.size) + (sqlByteLen + 1/* SQL + NUL */) - ); - const pzTail = wasm.ptr.add(ppStmt, wasm.ptr.size) /* final arg to sqlite3_prepare_v2() */; - let pSql = wasm.ptr.add(pzTail, wasm.ptr.size) /* start of the SQL string */; - const pSqlEnd = wasm.ptr.add(pSql, sqlByteLen); - if(isTA) wasm.heap8().set(sql, pSql); - else wasm.jstrcpy(sql, wasm.heap8(), pSql, sqlByteLen, false); - wasm.poke8(wasm.ptr.add(pSql, sqlByteLen), 0/*NUL terminator*/); - while( pSql && wasm.peek8(pSql) ){ - pStmt = stmt = null; - wasm.pokePtr([ppStmt, pzTail], 0); - const zHead = pSql; - DB.checkRc(this, capi.sqlite3_prepare_v3( - this.pointer, pSql, sqlByteLen, 0, ppStmt, pzTail - )); - [pStmt, pSql] = wasm.peekPtr([ppStmt, pzTail]); - sqlByteLen = wasm.ptr.addn(pSqlEnd,-pSql); - if(opt.saveSql){ - if( pStmt ) opt.saveSql.push(capi.sqlite3_sql(pStmt).trim()); - else if( opt.saveEmpty ){ - saveSql.push(wasm.typedArrayToString( - wasm.heap8u(), Number(zHead), - wasm.ptr.addn(zHead, sqlByteLen) - ).trim(/*arguable*/)); - } - } - if(!pStmt) continue; - //sqlite3.config.debug("forEachStmt() pSql =",capi.sqlite3_sql(pStmt)); - if( !opt.callback ){ - capi.sqlite3_finalize(pStmt); - pStmt = null; - continue; - } - stmt = opt.asPointer ? null : new Stmt(this, pStmt, BindTypes); - if( true===callaback(stmt || pStmt) ){ - stmt = pStmt = null /*callback took ownership */; - }else if(stmt){ - pStmt = null; - stmt.reset( - /* See [tag:insert-returning-reset]. The thinking here is - that if the callback didn't throw for this, it - probably should have. - */).finalize(); - stmt = null; - }else{ - const rx = capi.sqlite3_reset(pStmt/*[tag:insert-returning-reset]*/); - capi.sqlite3_finalize(pStmt); - pStmt = null; - DB.checkRc(this, rx); - } - }/*prepare() loop*/ - }finally{ - if(stmt) stmt.finalize(); - else if(pStmt) capi.sqlite3_finalize(pStmt); - wasm.scopedAllocPop(stack); - } - return this; - }/*forEachStmt()*/, -//#endif nope - /** Creates a new UDF (User-Defined Function) which is accessible via SQL code. This function may be called in any of the @@ -2423,6 +2295,55 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Stmt }/*oo1 object*/; + if(util.isUIThread()){ + /** + Functionally equivalent to DB(storageName,'c','kvvfs') except + that it throws if the given storage name is not one of 'local' + or 'session'. + + As of version 3.46, the argument may optionally be an options + object in the form: + + { + filename: 'session'|'local', + ... etc. (all options supported by the DB ctor) + } + + noting that the 'vfs' option supported by main DB + constructor is ignored here: the vfs is always 'kvvfs'. + */ + sqlite3.oo1.JsStorageDb = function(storageName='session'){ + const opt = dbCtorHelper.normalizeArgs(...arguments); + storageName = opt.filename; + if('session'!==storageName && 'local'!==storageName){ + toss3("JsStorageDb db name must be one of 'session' or 'local'."); + } + opt.vfs = 'kvvfs'; + dbCtorHelper.call(this, opt); + }; + const jdb = sqlite3.oo1.JsStorageDb; + jdb.prototype = Object.create(DB.prototype); + /** Equivalent to sqlite3_js_kvvfs_clear(). */ + jdb.clearStorage = capi.sqlite3_js_kvvfs_clear; + /** + Clears this database instance's storage or throws if this + instance has been closed. Returns the number of + database blocks which were cleaned up. + */ + jdb.prototype.clearStorage = function(){ + return jdb.clearStorage(affirmDbOpen(this).filename); + }; + /** Equivalent to sqlite3_js_kvvfs_size(). */ + jdb.storageSize = capi.sqlite3_js_kvvfs_size; + /** + Returns the _approximate_ number of bytes this database takes + up in its storage or throws if this instance has been closed. + */ + jdb.prototype.storageSize = function(){ + return jdb.storageSize(affirmDbOpen(this).filename); + }; + }/*main-window-only bits*/ + }); //#else /* Built with the omit-oo1 flag. */ diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index c53acee76..069f3fdb5 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -27,14 +27,13 @@ /** sqlite3ApiBootstrap() is the only global symbol persistently exposed by this API. It is intended to be called one time at the - end of the API amalgamation process and passed configuration details - for the current environment. + end of the API amalgamation process, passed configuration details + for the current environment, and then optionally be removed from + the global object using `delete globalThis.sqlite3ApiBootstrap`. This function is not intended for client-level use. It is intended for use in creating bundles configured for specific WASM - environments. That said, the "sqlite3-api.js" intermediary build - file aims to be suitable for dropping in to custom builds, and it - exposes only this function. + environments. This function expects a configuration object, intended to abstract away details specific to any given WASM environment, primarily so @@ -94,10 +93,9 @@ can be replaced with (e.g.) empty functions to squelch all such output. - - `wasmfsOpfsDir`[^1]: Specifies the "mount point" of the - OPFS-backed filesystem in WASMFS-capable builds. This is only - used in WASMFS-capable builds of the library (which the canonical - builds do not include). + - `wasmfsOpfsDir`[^1]: Specifies the "mount point" of the OPFS-backed + filesystem in WASMFS-capable builds. + [^1] = This property may optionally be a function, in which case this function calls that function to fetch the value, @@ -127,8 +125,7 @@ Both sqlite3ApiBootstrap.defaultConfig and globalThis.sqlite3ApiConfig get deleted by sqlite3ApiBootstrap() because any changes to them made after that point would have no - useful effect. This function also deletes itself from globalThis - when it's called. + useful effect. This function returns a Promise to the sqlite3 namespace object, which resolves after the async pieces of the library init are @@ -180,6 +177,14 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( } }); + /** + Eliminate any confusion about whether these config objects may + be used after library initialization by eliminating the outward-facing + objects... + */ + delete globalThis.sqlite3ApiConfig; + delete sqlite3ApiBootstrap.defaultConfig; + /** The main sqlite3 binding API gets installed into this object, mimicking the C API as closely as we can. The numerous members @@ -752,11 +757,6 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( toss: function(...args){throw new Error(args.join(' '))}, toss3, typedArrayPart: wasm.typedArrayPart, - assert: function(arg,msg){ - if( !arg ){ - util.toss("Assertion failed:",msg); - } - }, /** Given a byte array or ArrayBuffer, this function throws if the lead bytes of that buffer do not hold a SQLite3 database header, @@ -796,10 +796,25 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( /** wasm.X properties which are used for configuring the wasm - environment via whwashutil.js. This object gets fleshed out with - a number of WASM-specific utilities, in sqlite3-api-glue.c-pp.js. + environment via whwashutil.js. */ Object.assign(wasm, { + /** + The WASM IR (Intermediate Representation) value for + pointer-type values. If set then it MUST be one of 'i32' or + 'i64' (else an exception will be thrown). If it's not set, it + will default to 'i32'. + */ + pointerIR: config.wasmPtrIR, + + /** + True if BigInt support was enabled via (e.g.) the + Emscripten -sWASM_BIGINT flag, else false. When + enabled, certain 64-bit sqlite3 APIs are enabled which + are not otherwise enabled due to JS/WASM int64 + impedance mismatches. + */ + bigIntEnabled: !!config.bigIntEnabled, /** The symbols exported by the WASM environment. @@ -810,9 +825,8 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( /** When Emscripten compiles with `-sIMPORTED_MEMORY`, it initializes the heap and imports it into wasm, as opposed to - the other way around. In this case, the memory is not available - via this.exports.memory so the client must pass it in via - config.memory. + the other way around. In this case, the memory is not + available via this.exports.memory. */ memory: config.memory || config.exports['memory'] @@ -820,29 +834,6 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( "in either config.exports.memory (exported)", "or config.memory (imported)."), - /** - The WASM pointer size. If set then it MUST be one of 4 or 8 and - it MUST correspond to the WASM environment's pointer size. We - figure out the size by calling some un-JS-wrapped WASM function - which returns a pointer-type value. If that value is a BigInt, - it's 64-bit, else it's 32-bit. The pieces which populate - sqlite3.wasm (whwasmutil.js) can figure this out _if_ they can - allocate, but we have a chicken/egg situation there which makes - it illegal for that code to invoke wasm.dealloc() at the time - it would be needed. So we need to configure it ahead of time - (here) instead. - */ - pointerSize: ('number'===typeof config.exports.sqlite3_libversion()) ? 4 : 8, - - /** - True if BigInt support was enabled via (e.g.) the - Emscripten -sWASM_BIGINT flag, else false. When - enabled, certain 64-bit sqlite3 APIs are enabled which - are not otherwise enabled due to JS/WASM int64 - impedance mismatches. - */ - bigIntEnabled: !!config.bigIntEnabled, - /** WebAssembly.Table object holding the indirect function call table. Defaults to exports.__indirect_function_table. @@ -892,7 +883,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( Like this.alloc.impl(), this.realloc.impl() is a direct binding to the underlying realloc() implementation which does not throw - exceptions, instead returning 0 (or 0n) on allocation error. + exceptions, instead returning 0 on allocation error. */ realloc: undefined/*installed later*/, @@ -958,11 +949,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( }; wasm.realloc.impl = wasm.exports[keyRealloc]; wasm.dealloc = function f(m){ - f.impl(wasm.ptr.coerce(m)/*tag:64bit*/) - /* This coerce() is the reason we have to set wasm.pointerSize before - calling WhWasmUtilInstaller(). If we don't, that code will call - into this very early in its init, before wasm.ptr has been set up, - resulting in a null deref here. */; + f.impl(wasm.ptr.coerce(m)/*tag:64bit*/); }; wasm.dealloc.impl = wasm.exports[keyDealloc]; } @@ -1033,18 +1020,18 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( }/*compileOptionUsed()*/; /** - sqlite3.wasm.pstack (pseudo-stack) holds a special-case allocator - intended solely for short-lived, small data. In practice, it's - primarily used to allocate output pointers. It must not be used - for any memory which needs to outlive the scope in which it's - obtained from pstack. + sqlite3.wasm.pstack (pseudo-stack) holds a special-case intended + solely for short-lived, small data. In practice, it's primarily + used to allocate output pointers. It mus not be used for any + memory which needs to outlive the scope in which it's obtained + from pstack. The library guarantees only that a minimum of 2kb are available in this allocator, and it may provide more (it's a build-time value). pstack.quota and pstack.remaining can be used to get the total resp. remaining amount of memory. - It has only a single intended usage pattern: + It has only a single intended usage: ``` const stackPos = pstack.pointer; @@ -1061,11 +1048,13 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( ``` This allocator is much faster than a general-purpose one but is - limited to usage patterns like the one shown above (which are - pretty common when using sqlite3.capi). + limited to usage patterns like the one shown above. - The memory lives in the WASM heap and can be used with routines - such as wasm.poke() and wasm.heap8u().slice(). + It operates from a static range of memory which lives outside of + space managed by Emscripten's stack-management, so does not + collide with Emscripten-provided stack allocation APIs. The + memory lives in the WASM heap and can be used with routines such + as wasm.poke() and wasm.heap8u().slice(). */ wasm.pstack = Object.assign(Object.create(null),{ /** @@ -1139,7 +1128,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( argument: if it's 1, it returns a single pointer value. If it's more than 1, it returns the same as allocChunks(). - When a returned pointer will refer to a 64-bit value, e.g. a + When a returned pointers will refer to a 64-bit value, e.g. a double or int64, and that value must be written or fetched, e.g. using wasm.poke() or wasm.peek(), it is important that the pointer in question be aligned to an 8-byte @@ -1207,9 +1196,6 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( } })/*wasm.pstack properties*/; - /** - Docs: https://sqlite.org/wasm/doc/trunk/api-c-style.md#sqlite3_randomness - */ capi.sqlite3_randomness = (...args)=>{ if(1===args.length && util.isTypedArray(args[0]) @@ -1244,6 +1230,8 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( wasm.exports.sqlite3_randomness(...args); }; + /** State for sqlite3_wasmfs_opfs_dir(). */ + let __wasmfsOpfsDir = undefined; /** If the wasm environment has a WASMFS/OPFS-backed persistent storage directory, its path is returned by this function. If it @@ -1267,7 +1255,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( WASMFS capability requires a custom build. */ capi.sqlite3_wasmfs_opfs_dir = function(){ - if(undefined !== this.dir) return this.dir; + if(undefined !== __wasmfsOpfsDir) return __wasmfsOpfsDir; // If we have no OPFS, there is no persistent dir const pdir = config.wasmfsOpfsDir; if(!pdir @@ -1275,21 +1263,21 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( || !globalThis.FileSystemDirectoryHandle || !globalThis.FileSystemFileHandle || !wasm.exports.sqlite3__wasm_init_wasmfs){ - return this.dir = ""; + return __wasmfsOpfsDir = ""; } try{ if(pdir && 0===wasm.xCallWrapped( 'sqlite3__wasm_init_wasmfs', 'i32', ['string'], pdir )){ - return this.dir = pdir; + return __wasmfsOpfsDir = pdir; }else{ - return this.dir = ""; + return __wasmfsOpfsDir = ""; } }catch(e){ // sqlite3__wasm_init_wasmfs() is not available - return this.dir = ""; + return __wasmfsOpfsDir = ""; } - }.bind(Object.create(null)); + }; /** Returns true if sqlite3.capi.sqlite3_wasmfs_opfs_dir() is a @@ -1345,7 +1333,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( */ capi.sqlite3_js_vfs_list = function(){ const rc = []; - let pVfs = capi.sqlite3_vfs_find(wasm.ptr.null); + let pVfs = capi.sqlite3_vfs_find(wasm.ptr.coerce(0)); while(pVfs){ const oVfs = new capi.sqlite3_vfs(pVfs); rc.push(wasm.cstrToJs(oVfs.$zName)); @@ -1413,7 +1401,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( or not provided, then "main" is assumed. */ capi.sqlite3_js_db_vfs = - (dbPointer, dbName=wasm.ptr.null)=>util.sqlite3__wasm_db_vfs(dbPointer, dbName); + (dbPointer, dbName=0)=>util.sqlite3__wasm_db_vfs(dbPointer, dbName); /** A thin wrapper around capi.sqlite3_aggregate_context() which @@ -1609,6 +1597,86 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( return x===v ? undefined : x; } + if( util.isUIThread() ){ + /* Features specific to the main window thread... */ + + /** + Internal helper for sqlite3_js_kvvfs_clear() and friends. + Its argument should be one of ('local','session',""). + */ + const __kvvfsInfo = function(which){ + const rc = Object.create(null); + rc.prefix = 'kvvfs-'+which; + rc.stores = []; + if('session'===which || ""===which) rc.stores.push(globalThis.sessionStorage); + if('local'===which || ""===which) rc.stores.push(globalThis.localStorage); + return rc; + }; + + /** + Clears all storage used by the kvvfs DB backend, deleting any + DB(s) stored there. Its argument must be either 'session', + 'local', or "". In the first two cases, only sessionStorage + resp. localStorage is cleared. If it's an empty string (the + default) then both are cleared. Only storage keys which match + the pattern used by kvvfs are cleared: any other client-side + data are retained. + + This function is only available in the main window thread. + + Returns the number of entries cleared. + */ + capi.sqlite3_js_kvvfs_clear = function(which=""){ + let rc = 0; + const kvinfo = __kvvfsInfo(which); + kvinfo.stores.forEach((s)=>{ + const toRm = [] /* keys to remove */; + let i; + for( i = 0; i < s.length; ++i ){ + const k = s.key(i); + if(k.startsWith(kvinfo.prefix)) toRm.push(k); + } + toRm.forEach((kk)=>s.removeItem(kk)); + rc += toRm.length; + }); + return rc; + }; + + /** + This routine guesses the approximate amount of + window.localStorage and/or window.sessionStorage in use by the + kvvfs database backend. Its argument must be one of + ('session', 'local', ""). In the first two cases, only + sessionStorage resp. localStorage is counted. If it's an empty + string (the default) then both are counted. Only storage keys + which match the pattern used by kvvfs are counted. The returned + value is the "length" value of every matching key and value, + noting that JavaScript stores each character in 2 bytes. + + Note that the returned size is not authoritative from the + perspective of how much data can fit into localStorage and + sessionStorage, as the precise algorithms for determining + those limits are unspecified and may include per-entry + overhead invisible to clients. + */ + capi.sqlite3_js_kvvfs_size = function(which=""){ + let sz = 0; + const kvinfo = __kvvfsInfo(which); + kvinfo.stores.forEach((s)=>{ + let i; + for(i = 0; i < s.length; ++i){ + const k = s.key(i); + if(k.startsWith(kvinfo.prefix)){ + sz += k.length; + sz += s.getItem(k).length; + } + } + }); + return sz * 2 /* because JS uses 2-byte char encoding */; + }; + + }/* main-window-only bits */ + /** Wraps all known variants of the C-side variadic sqlite3_db_config(). @@ -1885,58 +1953,55 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( return (0===v) ? undefined : capi.sqlite3_value_to_js(v, throwIfCannotConvert); }; - if( true ){ /* changeset/preupdate additions... */ - /** - Internal impl of sqlite3_preupdate_new/old_js() and - sqlite3changeset_new/old_js(). - */ - const __newOldValue = function(pObj, iCol, impl){ - impl = capi[impl]; - if(!this.ptr) this.ptr = wasm.allocPtr(); - else wasm.pokePtr(this.ptr, 0); - const rc = impl(pObj, iCol, this.ptr); - if(rc) return SQLite3Error.toss(rc,arguments[2]+"() failed with code "+rc); - const pv = wasm.peekPtr(this.ptr); - return pv ? capi.sqlite3_value_to_js( pv, true ) : undefined; - }.bind(Object.create(null)); + /** + Internal impl of sqlite3_preupdate_new/old_js() and + sqlite3changeset_new/old_js(). + */ + const __newOldValue = function(pObj, iCol, impl){ + impl = capi[impl]; + if(!this.ptr) this.ptr = wasm.allocPtr(); + else wasm.pokePtr(this.ptr, 0); + const rc = impl(pObj, iCol, this.ptr); + if(rc) return SQLite3Error.toss(rc,arguments[2]+"() failed with code "+rc); + const pv = wasm.peekPtr(this.ptr); + return pv ? capi.sqlite3_value_to_js( pv, true ) : undefined; + }.bind(Object.create(null)); - /** - A wrapper around sqlite3_preupdate_new() which fetches the - sqlite3_value at the given index and returns the result of - passing it to sqlite3_value_to_js(). Throws on error. - */ - capi.sqlite3_preupdate_new_js = - (pDb, iCol)=>__newOldValue(pDb, iCol, 'sqlite3_preupdate_new'); + /** + A wrapper around sqlite3_preupdate_new() which fetches the + sqlite3_value at the given index and returns the result of + passing it to sqlite3_value_to_js(). Throws on error. + */ + capi.sqlite3_preupdate_new_js = + (pDb, iCol)=>__newOldValue(pDb, iCol, 'sqlite3_preupdate_new'); - /** - The sqlite3_preupdate_old() counterpart of - sqlite3_preupdate_new_js(), with an identical interface. - */ - capi.sqlite3_preupdate_old_js = - (pDb, iCol)=>__newOldValue(pDb, iCol, 'sqlite3_preupdate_old'); + /** + The sqlite3_preupdate_old() counterpart of + sqlite3_preupdate_new_js(), with an identical interface. + */ + capi.sqlite3_preupdate_old_js = + (pDb, iCol)=>__newOldValue(pDb, iCol, 'sqlite3_preupdate_old'); - /** - A wrapper around sqlite3changeset_new() which fetches the - sqlite3_value at the given index and returns the result of - passing it to sqlite3_value_to_js(). Throws on error. - - If sqlite3changeset_new() succeeds but has no value to report, - this function returns the undefined value, noting that - undefined is not a valid conversion from an `sqlite3_value`, so - is unambiguous. - */ - capi.sqlite3changeset_new_js = - (pChangesetIter, iCol) => __newOldValue(pChangesetIter, iCol, - 'sqlite3changeset_new'); + /** + A wrapper around sqlite3changeset_new() which fetches the + sqlite3_value at the given index and returns the result of + passing it to sqlite3_value_to_js(). Throws on error. - /** - The sqlite3changeset_old() counterpart of - sqlite3changeset_new_js(), with an identical interface. - */ - capi.sqlite3changeset_old_js = - (pChangesetIter, iCol)=>__newOldValue(pChangesetIter, iCol, - 'sqlite3changeset_old'); - }/*changeset/preupdate additions*/ + If sqlite3changeset_new() succeeds but has no value to report, + this function returns the undefined value, noting that undefined + is a valid conversion from an `sqlite3_value`, so is unambiguous. + */ + capi.sqlite3changeset_new_js = + (pChangesetIter, iCol) => __newOldValue(pChangesetIter, iCol, + 'sqlite3changeset_new'); + + /** + The sqlite3changeset_old() counterpart of + sqlite3changeset_new_js(), with an identical interface. + */ + capi.sqlite3changeset_old_js = + (pChangesetIter, iCol)=>__newOldValue(pChangesetIter, iCol, + 'sqlite3changeset_old'); /* The remainder of the API will be set up in later steps. */ const sqlite3 = { @@ -1948,10 +2013,10 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( config, /** Holds the version info of the sqlite3 source tree from which - the generated sqlite3-api.js gets built. Its version may well - differ from that reported by sqlite3_libversion(), but that - should be considered a source file mismatch, as the JS and WASM - files are intended to be built and distributed together. + the generated sqlite3-api.js gets built. Note that its version + may well differ from that reported by sqlite3_libversion(), but + that should be considered a source file mismatch, as the JS and + WASM files are intended to be built and distributed together. This object is initially a placeholder which gets replaced by a build-generated object. @@ -1976,7 +2041,9 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( async init will be fatal to the init as a whole, but init routines are themselves welcome to install dummy catch() handlers which are not fatal if their failure should be - considered non-fatal. + considered non-fatal. If called more than once, the second and + subsequent calls are no-ops which return a pre-resolved + Promise. Ideally this function is called as part of the Promise chain which handles the loading and bootstrapping of the API. If not @@ -1993,14 +2060,18 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( */ asyncPostInit: async function ff(){ if(ff.isReady instanceof Promise) return ff.isReady; - let lia = this.initializersAsync; - delete this.initializersAsync; + let lia = sqlite3ApiBootstrap.initializersAsync; + delete sqlite3ApiBootstrap.initializersAsync; const postInit = async ()=>{ if(!sqlite3.__isUnderTest){ /* Delete references to internal-only APIs which are used by some initializers. Retain them when running in test mode so that we can add tests for them. */ delete sqlite3.util; + /* It's conceivable that we might want to expose + StructBinder to client-side code, but it's only useful if + clients build their own sqlite3.wasm which contains their + own C struct types. */ delete sqlite3.StructBinder; } return sqlite3; @@ -2019,7 +2090,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( let p = Promise.resolve(sqlite3); while(lia.length) p = p.then(lia.shift()); return ff.isReady = p.catch(catcher); - }.bind(sqlite3ApiBootstrap), + }, /** scriptInfo ideally gets injected into this object by the infrastructure which assembles the JS/WASM module. It contains @@ -2034,9 +2105,6 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( */ scriptInfo: undefined }; - if( ('undefined'!==typeof sqlite3IsUnderTest/* from post-js-header.js */) ){ - sqlite3.__isUnderTest = !!sqlite3IsUnderTest; - } try{ sqlite3ApiBootstrap.initializers.forEach((f)=>{ f(sqlite3); @@ -2049,34 +2117,16 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( } delete sqlite3ApiBootstrap.initializers; sqlite3ApiBootstrap.sqlite3 = sqlite3; - if( 'undefined'!==typeof sqlite3InitScriptInfo/* from post-js-header.js */ ){ - sqlite3InitScriptInfo.debugModule( - "sqlite3ApiBootstrap() complete", sqlite3 - ); - sqlite3.scriptInfo - /* Used by some async init code. As of 2025-11-15 this is still - in use by the OPFS VFS for locating its worker. In non-Emscripten - builds, this would need to be injected in somewhere to get - that VFS loading. */ = sqlite3InitScriptInfo; - } - if( sqlite3.__isUnderTest ){ - if( 'undefined'!==typeof EmscriptenModule ){ - sqlite3.config.emscripten = EmscriptenModule; - } - /* - The problem with exposing these pieces (in non-testing runs) via - sqlite3.wasm is that it exposes non-SQLite pieces to the - clients, who may come to expect it to remain. _We_ only have - these data because we've overridden Emscripten's wasm file - loader, and if we lose that capability for some reason then - we'll lose access to this metadata. - - These data are interesting for exploring how the wasm/JS - pieces connect, e.g. for exploring exactly what Emscripten - imports into WASM from its JS glue, but it's not - SQLite-related. - */ - const iw = sqlite3.scriptInfo?.instantiateWasm; + delete globalThis.sqlite3ApiBootstrap; + delete globalThis.sqlite3ApiConfig; + sqlite3InitScriptInfo.debugModule( + "sqlite3ApiBootstrap() complete", sqlite3 + ); + sqlite3.scriptInfo /* used by some async init code */ = + sqlite3InitScriptInfo /* from post-js-header.js */; + if( (sqlite3.__isUnderTest = sqlite3IsUnderTest /* from post-js-header.js */) ){ + sqlite3.config.emscripten = EmscriptenModule; + const iw = sqlite3InitScriptInfo.instantiateWasm; if( iw ){ /* Metadata injected by the custom Module.instantiateWasm() in pre-js.c-pp.js. */ @@ -2085,21 +2135,10 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( sqlite3.wasm.imports = iw.imports; } } - - /** - Eliminate any confusion about whether these config objects may - be used after library initialization by eliminating the outward-facing - objects... - */ - delete globalThis.sqlite3ApiConfig; - delete globalThis.sqlite3ApiBootstrap; - delete sqlite3ApiBootstrap.defaultConfig; return sqlite3.asyncPostInit().then((s)=>{ - if( 'undefined'!==typeof sqlite3InitScriptInfo/* from post-js-header.js */ ){ - sqlite3InitScriptInfo.debugModule( - "sqlite3.asyncPostInit() complete", s - ); - } + sqlite3InitScriptInfo.debugModule( + "sqlite3.asyncPostInit() complete", sqlite3 + ); delete s.asyncPostInit; delete s.scriptInfo; delete s.emscripten; diff --git a/ext/wasm/api/sqlite3-license-version-header.js b/ext/wasm/api/sqlite3-license-version-header.js index dd32f4666..482989463 100644 --- a/ext/wasm/api/sqlite3-license-version-header.js +++ b/ext/wasm/api/sqlite3-license-version-header.js @@ -1,5 +1,4 @@ -/* @preserve -** +/* ** LICENSE for the sqlite3 WebAssembly/JavaScript APIs. ** ** This bundle (typically released as sqlite3.js or sqlite3.mjs) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index 79fc47393..e10d0dd50 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -1,4 +1,4 @@ -/* @preserve +/* 2022-09-16 The author disclaims copyright to this source code. In place of a diff --git a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js deleted file mode 100644 index e3fb72287..000000000 --- a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js +++ /dev/null @@ -1,2095 +0,0 @@ -/* - 2025-11-21 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - This file houses the "kvvfs" pieces of the SQLite3 JS API. Most of - kvvfs is implemented in src/os_kv.c and exposed/extended for use - here via sqlite3-wasm.c. - - Main project home page: https://sqlite.org - - Documentation home page: https://sqlite.org/wasm -*/ -//#if omit-kvvfs -globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - /* These are JS plumbing, not part of the public API */ - delete sqlite3.capi.sqlite3_kvvfs_methods; - delete sqlite3.capi.KVVfsFile; -} -//#else -//#@policy error -//#savepoint begin -//#define kvvfs-v2-added-in=3.52.0 -/** - kvvfs - the Key/Value VFS - is an SQLite3 VFS which delegates - storage of its pages and metadata to a key-value store. - - It was conceived in order to support JS's localStorage and - sessionStorage objects. Its native implementation uses files as - key/value storage (one file per record) but the JS implementation - replaces a few methods so that it can use the aforementioned - objects as storage. - - It uses a bespoke ASCII encoding to store each db page as a - separate record and stores some metadata, like the db's unencoded - size and its journal, as individual records. - - kvvfs is significantly less efficient than a plain in-memory db but - it also, as a side effect of its design, offers a JSON-friendly - interchange format for exporting and importing databases. - - kvvfs is _not_ designed for heavy db loads. It is relatively - malloc()-heavy, having to de/allocate frequently, and it - spends much of its time converting the raw db pages into and out of - an ASCII encoding. - - But it _does_ work and is "performant enough" for db work of the - scale of a db which will fit within sessionStorage or localStorage - (just 2-3mb). - - "Version 2" extends it to support using Storage-like objects as - backing storage, Storage being the JS class which localStorage and - sessionStorage both derive from. This essentially moves the backing - store from whatever localStorage and sessionStorage use to an - in-memory object. - - This effort is primarily a stepping stone towards eliminating, if - it proves possible, the POSIX I/O API dependencies in SQLite's WASM - builds. That is: if this VFS works properly, it can be set as the - default VFS and we can eliminate the "unix" VFS from the JS/WASM - builds (as opposed to server-wise/WASI builds). That still, as of - 2025-11-23, a ways away, but it's the main driver for version 2 of - kvvfs. - - Version 2 remains compatible with version 1 databases and always - writes localStorage/sessionStorage metadata in the v1 format, so - such dbs can be manipulated freely by either version. For transient - storage objects (new in version 2), the format of its record keys - is simpified, requiring less space than v1 keys by eliding - redundant (in this context) info from the keys. - - Another benefit of v2 is its ability to export dbs into a - JSON-friendly (but not human-friendly) format. - - A potential, as-yet-unproven, benefit, would be the ability to plug - arbitrary Storage-compatible objects in so that clients could, - e.g. asynchronously post updates to db pages to some back-end for - backups. -*/ -globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - 'use strict'; - const capi = sqlite3.capi, - sqlite3_kvvfs_methods = capi.sqlite3_kvvfs_methods, - KVVfsFile = capi.KVVfsFile, - pKvvfs = sqlite3.capi.sqlite3_vfs_find("kvvfs") - - /* These are JS plumbing, not part of the public API */ - delete capi.sqlite3_kvvfs_methods; - delete capi.KVVfsFile; - - if( !pKvvfs ) return /* nothing to do */; - if( 0 ){ - /* This working would be our proverbial holy grail, in that it - would allow us to eliminate the current default VFS, which - relies on POSIX I/O APIs. Eliminating that dependency would get - us one giant step closer to creating wasi-sdk builds. */ - capi.sqlite3_vfs_register(pKvvfs, 1); - } - - const util = sqlite3.util, - wasm = sqlite3.wasm, - toss3 = util.toss3, - hop = (o,k)=>Object.prototype.hasOwnProperty.call(o,k); - - const kvvfsMethods = new sqlite3_kvvfs_methods( - /* Wraps the static sqlite3_api_methods singleton */ - wasm.exports.sqlite3__wasm_kvvfs_methods() - ); - util.assert( 32<=kvvfsMethods.$nKeySize, "unexpected kvvfsMethods.$nKeySize: "+kvvfsMethods.$nKeySize); - - /** - Most of the VFS-internal state. - */ - const cache = Object.assign(Object.create(null),{ - /** Regex matching journal file names. */ - rxJournalSuffix: /-journal$/, - /** Frequently-used C-string. */ - zKeyJrnl: wasm.allocCString("jrnl"), - /** Frequently-used C-string. */ - zKeySz: wasm.allocCString("sz"), - /** - The maximum size of a kvvfs record key. It is historically only - 32, a limitation currently retained only because it's convenient to - do so (the underlying code has outgrown the need for the artifically - low limit). - - We cache this value here because the end of this init code will - dispose of kvvfsMethods, invalidating it. - */ - keySize: kvvfsMethods.$nKeySize, - /** - WASM heap memory buffers to optimize out some frequent - allocations. - */ - buffer: Object.assign(Object.create(null),{ - /** - The size of each buffer in this.pool. - - kvvfsMethods.$nBufferSize is slightly larger than the output - space needed for a kvvfs-encoded 64kb db page in a worse-cast - encoding (128kb). It is not suitable for arbitrary buffer - use, only page de/encoding. - */ - n: kvvfsMethods.$nBufferSize, - /** - Map of buffer ids to wasm.alloc()'d pointers of size - this.n. (Re)used by various internals. - - Buffer ids 0 and 1 are used in the API internals. Other - names are used in higher-level APIs. - - See memBuffer() and memBufferFree(). - */ - pool: Object.create(null) - }) - }); - - /** - Returns a (cached) wasm.alloc()'d buffer of cache.buffer.n size, - throwing on OOM. - - We leak this one-time alloc because we've no better option. - sqlite3_vfs does not have a finalizer, so we've no place to hook - in the cleanup. We "could" extend sqlite3_shutdown() to have a - cleanup list for stuff like this but that function is never - used in JS, so it's hardly worth it. - */ - cache.memBuffer = (id=0)=>cache.buffer.pool[id] ??= wasm.alloc(cache.buffer.n); - - /** Frees the buffer with the given id. */ - cache.memBufferFree = (id)=>{ - const b = cache.buffer.pool[id]; - if( b ){ - wasm.dealloc(b); - delete cache.buffer.pool[id]; - } - }; - - const noop = ()=>{}; - const debug = sqlite3.__isUnderTest - ? (...args)=>sqlite3.config.debug?.("kvvfs:", ...args) - : noop; - const warn = (...args)=>sqlite3.config.warn?.("kvvfs:", ...args); - const error = (...args)=>sqlite3.config.error?.("kvvfs:", ...args); - - /** - Implementation of JS's Storage interface for use as backing store - of the kvvfs. Storage is a native class and its constructor - cannot be legally called from JS, making it impossible to - directly subclass Storage. This class implements (only) the - Storage interface, to make it a drop-in replacement for - localStorage/sessionStorage. (Any behavioral discrepancies are to - be considered bugs.) - - This impl simply proxies a plain, prototype-less Object, suitable - for JSON-ing. - - Design note: Storage has a bit of an odd iteration-related - interface as does not (AFAIK) specify specific behavior regarding - modification during traversal. Because of that, this class does - some seemingly unnecessary things with its #keys member, deleting - and recreating it whenever a property index might be invalidated. - */ - class KVVfsStorage { - #map = Object.create(null); - #keys = null; - #size = 0; - - constructor(){ - this.clear(); - } - - #getKeys(){ - return this.#keys ??= Object.keys(this.#map); - } - - key(n){ - if(n < 0 || n >= this.#size) return null; - return this.#getKeys()[n]; - } - - getItem(k){ - return this.#map[k] ?? null; - } - - setItem(k,v){ - if( !(k in this.#map) ){ - ++this.#size; - this.#keys = null; - } - this.#map[k] = ''+v; - } - - removeItem(k){ - if( k in this.#map ){ - delete this.#map[k]; - --this.#size; - this.#keys = null; - } - } - - clear(){ - this.#map = Object.create(null); - this.#keys = null; - this.#size = 0; - } - - get length() { - return this.#size; - } - }/*KVVfsStorage*/; - - /** True if v is the name of one of the special persistant Storage - objects. */ - const kvvfsIsPersistentName = (v)=>'local'===v || 'session'===v; - - /** - Keys in kvvfs have a prefix of "kvvfs-NAME-", where NAME is the - db name. This key is redundant in JS but it's how kvvfs works (it - saves each key to a separate file, so needs a distinct namespace - per data source name). We retain this prefix in 'local' and - 'session' storage for backwards compatibility and so that they - can co-exist with client data in their storage, but we elide them - from "v2" storage, where they're superfluous. - */ - const kvvfsKeyPrefix = (v)=>kvvfsIsPersistentName(v) ? 'kvvfs-'+v+'-' : ''; - - /** - Throws if storage name n (JS string) is not valid for use as a - storage name. Much of this goes back to kvvfs having a fixed - buffer size for its keys, and the storage name needing to be - encoded in the keys for local/session storage. - - The second argument must only be true when called from xOpen() - - it makes names with a "-journal" suffix legal. - */ - const validateStorageName = function(n,mayBeJournal=false){ - if( kvvfsIsPersistentName(n) ) return; - const len = (new Blob([n])).size/*byte length*/; - if( !len ) toss3(capi.SQLITE_MISUSE, "Empty name is not permitted."); - let maxLen = cache.keySize - 1; - if( cache.rxJournalSuffix.test(n) ){ - if( !mayBeJournal ){ - toss3(capi.SQLITE_MISUSE, - "Storage names may not have a '-journal' suffix."); - } - }else if( ['-wal','-shm'].filter(v=>n.endsWith(v)).length ){ - toss3(capi.SQLITE_MISUSE, - "Storage names may not have a -wal or -shm suffix."); - }else{ - maxLen -= 8 /* so we have space for a matching "-journal" suffix */; - } - if( len > maxLen ){ - toss3(capi.SQLITE_RANGE, "Storage name is too long. Limit =", maxLen); - } - let i; - for( i = 0; i < len; ++i ){ - const ch = n.codePointAt(i); - if( ch<32 ){ - toss3(capi.SQLITE_RANGE, - "Illegal character ("+ch+"d) in storage name:",n); - } - } - }; - - /** - Create a new instance of the objects which go into - cache.storagePool, with a refcount of 1. If passed a Storage-like - object as its second argument, it is used for the storage, - otherwise it creates a new KVVfsStorage object. - */ - const newStorageObj = (name,storage=undefined)=>Object.assign(Object.create(null),{ - /** - JS string value of this KVVfsFile::$zClass. i.e. the storage's - name. - */ - jzClass: name, - /** - Refcount. This keeps dbs and journals pointing to the same - storage for the life of both and enables kvvfs to behave more - like a conventional filesystem (a stepping stone towards - downstream API goals). Managed by xOpen() and xClose(). - */ - refc: 1, - /** - If true, this storage will be removed by xClose() or - sqlite3_js_kvvfs_unlink() when refc reaches 0. The others will - persist when refc==0, to give the illusion of real back-end - storage. Managed by xOpen() and sqlite3_js_kvvfs_reserve(). By - default this is false but the delete-on-close=1 flag can be - used to set this to true. - */ - deleteAtRefc0: false, - /** - The backing store. Must implement the Storage interface. - */ - storage: storage || new KVVfsStorage, - /** - The storage prefix used for kvvfs keys. It is - "kvvfs-STORAGENAME-" for local/session storage and an empty - string for other storage. local/session storage must use the - long form (A) for backwards compatibility and (B) so that kvvfs - can coexist with non-db client data in those backends. Neither - (A) nor (B) are concerns for KVVfsStorage objects. - - This prefix mirrors the one generated by os_kv.c's - kvrecordMakeKey() and must stay in sync with that one. - */ - keyPrefix: kvvfsKeyPrefix(name), - /** - KVVfsFile instances currently using this storage. Managed by - xOpen() and xClose(). - */ - files: [], - /** - If set, it's an array of objects with various event - callbacks. See sqlite3_js_kvvfs_listen(). When there are no - listeners, this member is set to undefined (instead of an empty - array) to allow us to more easily optimize out calls to - notifyListeners() for the common case of no listeners. - */ - listeners: undefined - }); - - /** - Public interface for kvvfs v2. The capi.sqlite3_js_kvvfs_...() - routines remain in place for v1. Some members of this class proxy - to those functions but use different default argument values in - some cases. - */ - const kvvfs = sqlite3.kvvfs = Object.create(null); - if( sqlite3.__isUnderTest ){ - /* For inspection via the dev tools console. */ - kvvfs.log = Object.assign(Object.create(null),{ - xOpen: false, - xClose: false, - xWrite: false, - xRead: false, - xSync: false, - xAccess: false, - xFileControl: false, - xRcrdRead: false, - xRcrdWrite: false, - xRcrdDelete: false, - }); - } - - /** - Deletes the cache.storagePool entries for store (a - cache.storagePool entry) and its db/journal counterpart. - */ - const deleteStorage = function(store){ - const other = cache.rxJournalSuffix.test(store.jzClass) - ? store.jzClass.replace(cache.rxJournalSuffix,'') - : store.jzClass+'-journal'; - kvvfs?.log?.xClose - && debug("cleaning up storage handles [", store.jzClass, other,"]",store); - delete cache.storagePool[store.jzClass]; - delete cache.storagePool[other]; - if( !sqlite3.__isUnderTest ){ - /* In test runs, leave these for inspection. If we delete them here, - any prior dumps of them emitted via the console get cleared out - because the console shows live objects instead of call-time - static dumps. */ - delete store.storage; - delete store.refc; - } - }; - - /** - Add both store.jzClass and store.jzClass+"-journal" - to cache,storagePool. - */ - const installStorageAndJournal = (store)=> - cache.storagePool[store.jzClass] = - cache.storagePool[store.jzClass+'-journal'] = store; - - /** - The public name of the current thread's transient storage - object. A storage object with this name gets preinstalled. - */ - const nameOfThisThreadStorage = '.'; - - /** - Map of JS-stringified KVVfsFile::zClass names to - reference-counted Storage objects. These objects are created in - xOpen(). Their refcount is decremented in xClose(), and the - record is destroyed if the refcount reaches 0. We refcount so - that concurrent active xOpen()s on a given name, and within a - given thread, use the same storage object. - */ - cache.storagePool = Object.assign(Object.create(null),{ - /* Start off with mappings for well-known names. */ - [nameOfThisThreadStorage]: newStorageObj(nameOfThisThreadStorage) - }); - - if( globalThis.Storage ){ - /* If available, install local/session storage. */ - if( globalThis.localStorage instanceof globalThis.Storage ){ - cache.storagePool.local = newStorageObj('local', globalThis.localStorage); - } - if( globalThis.sessionStorage instanceof globalThis.Storage ){ - cache.storagePool.session = newStorageObj('session', globalThis.sessionStorage); - } - } - - cache.builtinStorageNames = Object.keys(cache.storagePool); - - const isBuiltinName = (n)=>cache.builtinStorageNames.indexOf(n)>-1; - - /* Add "-journal" twins for each cache.storagePool entry... */ - for(const k of Object.keys(cache.storagePool)){ - /* Journals in kvvfs are are stored as individual records within - their Storage-ish object, named "{storage.keyPrefix}jrnl". We - always map the db and its journal to the same Storage - object. */ - const orig = cache.storagePool[k]; - cache.storagePool[k+'-journal'] = orig; - } - - cache.setError = (e=undefined, dfltErrCode=capi.SQLITE_ERROR)=>{ - if( e ){ - cache.lastError = e; - return (e.resultCode | 0) || dfltErrCode; - } - delete cache.lastError; - return 0; - }; - - cache.popError = ()=>{ - const e = cache.lastError; - delete cache.lastError; - return e; - }; - - /** Exception handler for notifyListeners(). */ - const catchForNotify = (e)=>{ - warn("kvvfs.listener handler threw:",e); - }; - - const kvvfsDecode = wasm.exports.sqlite3__wasm_kvvfs_decode; - const kvvfsEncode = wasm.exports.sqlite3__wasm_kvvfs_encode; - - /** - Listener events and their argument(s) (via the callback(ev) - ev.data member): - - 'open': number of opened handles on this storage. - - 'close': number of opened handles on this storage. - - 'write': key, value - - 'delete': key - - 'sync': true if it's from xSync(), false if it's from - xFileControl(). - - For efficiency's sake, all calls to this function should - be in the form: - - store.listeners && notifyListeners(...); - - Failing to do so will trigger an exceptin in this function (which - will be ignored but may produce a console warning). - */ - const notifyListeners = async function(eventName,store,...args){ - try{ - //cache.rxPageNoSuffix ??= /(\d+)$/; - if( store.keyPrefix && args[0] ){ - args[0] = args[0].replace(store.keyPrefix,''); - } - let u8enc, z0, z1, wcache; - for(const ear of store.listeners){ - const ev = Object.create(null); - ev.storageName = store.jzClass; - ev.type = eventName; - const decodePages = ear.decodePages; - const f = ear.events[eventName]; - if( f ){ - if( !ear.includeJournal && args[0]==='jrnl' ){ - continue; - } - if( 'write'===eventName && ear.decodePages && +args[0]>0 ){ - /* Decode pages to Uint8Array, caching the result in - wcache in case we have more listeners. */ - ev.data = [args[0]]; - if( wcache?.[args[0]] ){ - ev.data[1] = wcache[args[0]]; - continue; - } - u8enc ??= new TextEncoder('utf-8'); - z0 ??= cache.memBuffer(10); - z1 ??= cache.memBuffer(11); - const u = u8enc.encode(args[1]); - const heap = wasm.heap8u(); - heap.set(u, Number(z0)); - heap[wasm.ptr.addn(z0, u.length)] = 0; - const rc = kvvfsDecode(z0, z1, cache.buffer.n); - if( rc>0 ){ - wcache ??= Object.create(null); - wcache[args[0]] - = ev.data[1] - = heap.slice(Number(z1), wasm.ptr.addn(z1,rc)); - }else{ - continue; - } - }else{ - ev.data = args.length - ? ((args.length===1) ? args[0] : args) - : undefined; - } - try{f(ev)?.catch?.(catchForNotify)} - catch(e){ - warn("notifyListeners [",store.jzClass,"]",eventName,e); - } - } - } - }catch(e){ - catchForNotify(e); - } - }/*notifyListeners()*/; - - /** - Returns the storage object mapped to the given string zClass - (C-string pointer or JS string). - */ - const storageForZClass = (zClass)=> - 'string'===typeof zClass - ? cache.storagePool[zClass] - : cache.storagePool[wasm.cstrToJs(zClass)]; - -//#if nope - // fileForDb() works but we don't have a current need for it. - /** - Expects an (sqlite3*). Uses sqlite3_file_control() to extract its - (sqlite3_file*). On success it returns a new KVVfsFile instance - wrapping that pointer, which the caller must eventual call - dispose() on (which won't free the underlying pointer, just the - wrapper). Returns null if no handle is found (which would - indicate either that pDb is not using kvvfs or a severe bug in - its management). - */ - const fileForDb = function(pDb){ - const stack = wasm.pstack.pointer; - try{ - const pOut = wasm.pstack.allocPtr(); - return wasm.exports.sqlite3_file_control( - pDb, wasm.ptr.null, capi.SQLITE_FCNTL_FILE_POINTER, pOut - ) - ? null - : new KVVfsFile(wasm.peekPtr(pOut)); - }finally{ - wasm.pstack.restore(stack); - } - }; - - /** - Expects an object from the storagePool map. The $szPage and - $szDb members of each store.files entry is set to -1 in an attempt - to trigger those values to reload. - */ - const alertFilesToReload = (store)=>{ - try{ - for( const f of store.files ){ - // FIXME: we need to use one of the C APIs for this, maybe an - // fcntl. - f.$szPage = -1; - f.$szDb = -1n - } - }catch(e){ - error("alertFilesToReload()",store,e); - throw e; - } - }; -//#endif nope - - const kvvfsMakeKey = wasm.exports.sqlite3__wasm_kvvfsMakeKey; - /** - Returns a C string from kvvfsMakeKey() OR returns zKey. In the - former case the memory is static, so must be copied before a - second call. zKey MUST be a pointer passed to a VFS/file method, - to allow us to avoid an alloc and/or an snprintf(). It requires - C-string arguments for zClass and zKey. zClass may be NULL but - zKey may not. - */ - const zKeyForStorage = (store, zClass, zKey)=>{ - //debug("zKeyForStorage(",store, wasm.cstrToJs(zClass), wasm.cstrToJs(zKey)); - return (zClass && store.keyPrefix) ? kvvfsMakeKey(zClass, zKey) : zKey; - }; - - const jsKeyForStorage = (store,zClass,zKey)=> - wasm.cstrToJs(zKeyForStorage(store, zClass, zKey)); - - const storageGetDbSize = (store)=>+store.storage.getItem(store.keyPrefix + "sz"); - - /** - sqlite3_file pointers => objects, each of which has: - - .file = KVVfsFile instance - - .jzClass = JS-string form of f.$zClass - - .storage = Storage object. It is shared between a db and its - journal. - */ - const pFileHandles = new Map(); - - /** - Original WASM functions for methods we partially override. - */ - const originalMethods = { - vfs: Object.create(null), - ioDb: Object.create(null), - ioJrnl: Object.create(null) - }; - - /** Returns the appropriate originalMethods[X] instance for the - given a KVVfsFile instance. */ - const originalIoMethods = (kvvfsFile)=> - originalMethods[kvvfsFile.$isJournal ? 'ioJrnl' : 'ioDb']; - - const pVfs = new capi.sqlite3_vfs(kvvfsMethods.$pVfs); - const pIoDb = new capi.sqlite3_io_methods(kvvfsMethods.$pIoDb); - const pIoJrnl = new capi.sqlite3_io_methods(kvvfsMethods.$pIoJrnl); - const recordHandler = - Object.create(null)/** helper for some vfs - routines. Populated later. */; - const kvvfsInternal = Object.assign(Object.create(null),{ - pFileHandles, - cache, - storageForZClass, - KVVfsStorage, - /** - BUG: changing to a page size other than the default, - then vacuuming, corrupts the db. As a workaround, - until this is resolved, we forcibly disable - (pragma page_size=...) changes. - */ - disablePageSizeChange: true - }); - if( kvvfs.log ){ - // this is a test build - kvvfs.internal = kvvfsInternal; - } - - /** - Implementations for members of the object referred to by - sqlite3__wasm_kvvfs_methods(). We swap out some native - implementations with these so that we can use JS Storage for - their backing store. - */ - const methodOverrides = { - - /** - sqlite3_kvvfs_methods's member methods. These perform the - fetching, setting, and removal of storage keys on behalf of - kvvfs. In the native impl these write each db page to a - separate file. This impl stores each db page as a single - record in a Storage object which is mapped to zClass. - - A db's size is stored in a record named kvvfs[-storagename]-sz - and the journal is stored in kvvfs[-storagename]-jrnl. The - [-storagename] part is a remnant of the native impl (so that - it has unique filenames per db) and is only used for - localStorage and sessionStorage. We elide that part (to save - space) from other storage objects but retain it on those two - to avoid invalidating pre-version-2 session/localStorage dbs. - - The interface docs for these methods are in src/os_kv.c's - kvrecordRead(), kvrecordWrite(), and kvrecordDelete(). - */ - recordHandler: { - xRcrdRead: (zClass, zKey, zBuf, nBuf)=>{ - try{ - const jzClass = wasm.cstrToJs(zClass); - const store = storageForZClass(jzClass); - if( !store ) return -1; - const jXKey = jsKeyForStorage(store, zClass, zKey); - kvvfs?.log?.xRcrdRead && warn("xRcrdRead", jzClass, jXKey, nBuf, store ); - const jV = store.storage.getItem(jXKey); - if(null===jV) return -1; - const nV = jV.length /* We are relying 100% on v being - ** ASCII so that jV.length is equal - ** to the C-string's byte length. */; - if( 0 ){ - debug("xRcrdRead", jXKey, store, jV); - } - if(nBuf<=0) return nV; - else if(1===nBuf){ - wasm.poke(zBuf, 0); - return nV; - } - if( nBuf+1<nV ){ - toss3(capi.SQLITE_RANGE, - "xRcrdRead()",jzClass,jXKey, - "input buffer is too small: need", - nV,"but have",nBuf); - } - if( 0 ){ - debug("xRcrdRead", nBuf, zClass, wasm.cstrToJs(zClass), - wasm.cstrToJs(zKey), nV, jV, store); - } - const zV = cache.memBuffer(0); - //if( !zV ) return -3 /*OOM*/; - const heap = wasm.heap8(); - let i; - for(i = 0; i < nV; ++i){ - heap[wasm.ptr.add(zV,i)] = jV.codePointAt(i) & 0xFF; - } - heap.copyWithin( - Number(zBuf), Number(zV), wasm.ptr.addn(zV, i) - ); - heap[wasm.ptr.add(zBuf, nV)] = 0; - return nBuf; - }catch(e){ - error("kvrecordRead()",e); - cache.setError(e); - return -2; - } - }, - - xRcrdWrite: (zClass, zKey, zData)=>{ - try { - const store = storageForZClass(zClass); - const jxKey = jsKeyForStorage(store, zClass, zKey); - const jData = wasm.cstrToJs(zData); - kvvfs?.log?.xRcrdWrite && warn("xRcrdWrite",jxKey, store); - store.storage.setItem(jxKey, jData); - store.listeners && notifyListeners('write', store, jxKey, jData); - return 0; - }catch(e){ - error("kvrecordWrite()",e); - return cache.setError(e, capi.SQLITE_IOERR); - } - }, - - xRcrdDelete: (zClass, zKey)=>{ - try { - const store = storageForZClass(zClass); - const jxKey = jsKeyForStorage(store, zClass, zKey); - kvvfs?.log?.xRcrdDelete && warn("xRcrdDelete",jxKey, store); - store.storage.removeItem(jxKey); - store.listeners && notifyListeners('delete', store, jxKey); - return 0; - }catch(e){ - error("kvrecordDelete()",e); - return cache.setError(e, capi.SQLITE_IOERR); - } - } - }/*recordHandler*/, - - /** - Override certain operations of the underlying sqlite3_vfs and - the two sqlite3_io_methods instances so that we can tie - Storage objects to db names. - */ - vfs:{ - /* sqlite3_kvvfs_methods::pVfs's methods */ - xOpen: function(pProtoVfs,zName,pProtoFile,flags,pOutFlags){ - cache.popError(); - let zToFree /* alloc()'d memory for temp db name */; - if( 0 ){ - /* tester1.js makes it a lot further if we do this. */ - flags |= capi.SQLITE_OPEN_CREATE; - } - try{ - if( !zName ){ - zToFree = wasm.allocCString(""+pProtoFile+"." - +(Math.random() * 100000 | 0)); - zName = zToFree; - } - const jzClass = wasm.cstrToJs(zName); - kvvfs?.log?.xOpen && debug("xOpen",jzClass,"flags =",flags); - validateStorageName(jzClass, true); - if( (flags & (capi.SQLITE_OPEN_MAIN_DB - | capi.SQLITE_OPEN_TEMP_DB - | capi.SQLITE_OPEN_TRANSIENT_DB)) - && cache.rxJournalSuffix.test(jzClass) ){ - toss3(capi.SQLITE_ERROR, - "DB files may not have a '-journal' suffix."); - } - let s = storageForZClass(jzClass); - if( !s && !(flags & capi.SQLITE_OPEN_CREATE) ){ - toss3(capi.SQLITE_ERROR, "Storage not found:", jzClass); - } - const rc = originalMethods.vfs.xOpen(pProtoVfs, zName, pProtoFile, - flags, pOutFlags); - if( rc ) return rc; - let deleteAt0 = !!(capi.SQLITE_OPEN_DELETEONCLOSE & flags); - if(wasm.isPtr(arguments[1]/*original zName*/)){ - if(capi.sqlite3_uri_boolean(zName, "delete-on-close", 0)){ - deleteAt0 = true; - } - } - const f = new KVVfsFile(pProtoFile); - util.assert(f.$zClass, "Missing f.$zClass"); - f.addOnDispose(zToFree); - zToFree = undefined; - //debug("xOpen", jzClass, s); - if( s ){ - ++s.refc; - //no if( true===deleteAt0 ) s.deleteAtRefc0 = true; - s.files.push(f); - wasm.poke32(pOutFlags, flags); - }else{ - wasm.poke32(pOutFlags, flags | capi.SQLITE_OPEN_CREATE); - util.assert( !f.$isJournal, "Opening a journal before its db? "+jzClass ); - /* Map both zName and zName-journal to the same storage. */ - const nm = jzClass.replace(cache.rxJournalSuffix,''); - s = newStorageObj(nm); - installStorageAndJournal(s); - s.files.push(f); - s.deleteAtRefc0 = deleteAt0; - kvvfs?.log?.xOpen - && debug("xOpen installed storage handle [",nm, nm+"-journal","]", s); - } - pFileHandles.set(pProtoFile, {store: s, file: f, jzClass}); - s.listeners && notifyListeners('open', s, s.files.length); - return 0; - }catch(e){ - warn("xOpen:",e); - return cache.setError(e); - }finally{ - zToFree && wasm.dealloc(zToFree); - } - }/*xOpen()*/, - - xDelete: function(pVfs, zName, iSyncFlag){ - cache.popError(); - try{ - const jzName = wasm.cstrToJs(zName); - if( cache.rxJournalSuffix.test(jzName) ){ - recordHandler.xRcrdDelete(zName, cache.zKeyJrnl); - }/* - else: historically not done, but maybe otherwise delete - all db pages from storageForZClass(zName)? - */ - return 0; - }catch(e){ - warn("xDelete",e); - return cache.setError(e); - } - }, - - xAccess: function(pProtoVfs, zPath, flags, pResOut){ - cache.popError(); - try{ - const s = storageForZClass(zPath); - const jzPath = s?.jzClass || wasm.cstrToJs(zPath); - if( kvvfs?.log?.xAccess ){ - debug("xAccess",jzPath,"flags =", - flags,"*pResOut =",wasm.peek32(pResOut), - "store =",s); - } - if( !s ){ - // From the API docs: - /** The xAccess method returns [SQLITE_OK] on success or some - ** non-zero error code if there is an I/O error or if the name of - ** the file given in the second argument is illegal. - */ - // However, returning non-0 from here is fatal, so we don't do that. - try{validateStorageName(jzPath)} - catch(e){ - //warn("xAccess is ignoring name validation failure:",e); - wasm.poke32(pResOut, 0); - return 0; - } - } - if( s ){ - const key = s.keyPrefix+ - (cache.rxJournalSuffix.test(jzPath) ? "jrnl" : "1"); - const res = s.storage.getItem(key) ? 0 : 1; - /* This res value looks completely backwards to me, and - is the opposite of the native kvvfs's impl, but it's - working, whereas reimplementing the native one - faithfully does not. Read the lib-level code of where - this is invoked, my expectation is that we set res to 0 - for not-exists. */ - //warn("access res",jzPath,res); - wasm.poke32(pResOut, res); - }else{ - wasm.poke32(pResOut, 0); - } - return 0; - }catch(e){ - error('xAccess',e); - return cache.setError(e); - } - }, - - xRandomness: function(pVfs, nOut, pOut){ - const heap = wasm.heap8u(); - let i = 0; - const npOut = Number(pOut); - for(; i < nOut; ++i) heap[npOut + i] = (Math.random()*255000) & 0xFF; - return nOut; - }, - - xGetLastError: function(pVfs,nOut,pOut){ - const e = cache.popError(); - debug('xGetLastError',e); - if(e){ - const scope = wasm.scopedAllocPush(); - try{ - const [cMsg, n] = wasm.scopedAllocCString(e.message, true); - wasm.cstrncpy(pOut, cMsg, nOut); - if(n > nOut) wasm.poke8(wasm.ptr.add(pOut,nOut,-1), 0); - debug("set xGetLastError",e.message); - return (e.resultCode | 0) || capi.SQLITE_IOERR; - }catch(e){ - return capi.SQLITE_NOMEM; - }finally{ - wasm.scopedAllocPop(scope); - } - } - return 0; - } - -//#if nope - // these impls work but there's currently no pressing need _not_ use - // the native impls. - xCurrentTime: function(pVfs,pOut){ - wasm.poke64f(pOut, 2440587.5 + (Date.now()/86400000)); - return 0; - }, - - xCurrentTimeInt64: function(pVfs,pOut){ - wasm.poke64(pOut, (2440587.5 * 86400000) + Date.now()); - return 0; - } -//#endif - }/*.vfs*/, - - /** - kvvfs has separate sqlite3_api_methods impls for some of the - methods depending on whether it's a db or journal file. Some - of the methods use shared impls but others are specific to - either db or journal files. - */ - ioDb:{ - /* sqlite3_kvvfs_methods::pIoDb's methods */ - xClose: function(pFile){ - cache.popError(); - try{ - const h = pFileHandles.get(pFile); - kvvfs?.log?.xClose && debug("xClose", pFile, h); - if( h ){ - pFileHandles.delete(pFile); - const s = h.store;//storageForZClass(h.jzClass); - s.files = s.files.filter((v)=>v!==h.file); - if( --s.refc<=0 && s.deleteAtRefc0 ){ - deleteStorage(s); - } - originalMethods.ioDb/*same for journals*/.xClose(pFile); - h.file.dispose(); - s.listeners && notifyListeners('close', s, s.files.length); - }else{ - /* Can happen if xOpen fails */ - } - return 0; - }catch(e){ - error("xClose",e); - return cache.setError(e); - } - }, - - xFileControl: function(pFile, opId, pArg){ - cache.popError(); - try{ - const h = pFileHandles.get(pFile); - util.assert(h, "Missing KVVfsFile handle"); - kvvfs?.log?.xFileControl && debug("xFileControl",h,'op =',opId); - if( opId===capi.SQLITE_FCNTL_PRAGMA - && kvvfsInternal.disablePageSizeChange ){ - /* pArg== length-3 (char**) */ - //const argv = wasm.cArgvToJs(3, pArg); // the easy way - const zName = wasm.peekPtr(wasm.ptr.add(pArg, wasm.ptr.size)); - if( "page_size"===wasm.cstrToJs(zName) ){ - kvvfs?.log?.xFileControl - && debug("xFileControl pragma",wasm.cstrToJs(zName)); - const zVal = wasm.peekPtr(wasm.ptr.add(pArg, 2*wasm.ptr.size)); - if( zVal ){ - /* Without this, pragma page_size=N; followed by a - vacuum breaks the db. With this, it continues - working but does not actually change the page - size. */ - kvvfs?.log?.xFileControl - && warn("xFileControl pragma", h, - "NOT setting page size to", wasm.cstrToJs(zVal)); - h.file.$szPage = -1; - return 0/*corrupts: capi.SQLITE_NOTFOUND*/; - }else if( h.file.$szPage>0 ){ - kvvfs?.log?.xFileControl && - warn("xFileControl", h, "getting page size",h.file.$szPage); - wasm.pokePtr(pArg, wasm.allocCString(""+h.file.$szPage) - /* memory now owned by the library */); - return 0;//capi.SQLITE_NOTFOUND; - } - } - } - const rc = originalMethods.ioDb.xFileControl(pFile, opId, pArg); - if( 0==rc && capi.SQLITE_FCNTL_SYNC===opId ){ - h.store.listeners && notifyListeners('sync', h.store, false); - } - return rc; - }catch(e){ - error("xFileControl",e); - return cache.setError(e); - } - }, - - xSync: function(pFile,flags){ - cache.popError(); - try{ - const h = pFileHandles.get(pFile); - kvvfs?.log?.xSync && debug("xSync", h); - util.assert(h, "Missing KVVfsFile handle"); - const rc = originalMethods.ioDb.xSync(pFile, flags); - if( 0==rc && h.store.listeners ) notifyListeners('sync', h.store, true); - return rc; - }catch(e){ - error("xSync",e); - return cache.setError(e); - } - }, - -//#if not nope - // We override xRead/xWrite only for logging/debugging. They - // should otherwise be disabled (it's faster that way). - xRead: function(pFile,pTgt,n,iOff64){ - cache.popError(); - try{ - if( kvvfs?.log?.xRead ){ - const h = pFileHandles.get(pFile); - util.assert(h, "Missing KVVfsFile handle"); - debug("xRead", n, iOff64, h); - } - return originalMethods.ioDb.xRead(pFile, pTgt, n, iOff64); - }catch(e){ - error("xRead",e); - return cache.setError(e); - } - }, - xWrite: function(pFile,pSrc,n,iOff64){ - cache.popError(); - try{ - if( kvvfs?.log?.xWrite ){ - const h = pFileHandles.get(pFile); - util.assert(h, "Missing KVVfsFile handle"); - debug("xWrite", n, iOff64, h); - } - return originalMethods.ioDb.xWrite(pFile, pSrc, n, iOff64); - }catch(e){ - error("xWrite",e); - return cache.setError(e); - } - }, -//#endif nope - -//#if nope - xTruncate: function(pFile,i64){}, - xFileSize: function(pFile,pi64Out){}, - xLock: function(pFile,iLock){}, - xUnlock: function(pFile,iLock){}, - xCheckReservedLock: function(pFile,piOut){}, - xSectorSize: function(pFile){}, - xDeviceCharacteristics: function(pFile){} -//#endif - }/*.ioDb*/, - - ioJrnl:{ - /* sqlite3_kvvfs_methods::pIoJrnl's methods. Those set to true - are copied as-is from the ioDb objects. Others are specific - to journal files. */ - xClose: true, -//#if nope - xRead: function(pFile,pTgt,n,iOff64){}, - xWrite: function(pFile,pSrc,n,iOff64){}, - xTruncate: function(pFile,i64){}, - xSync: function(pFile,flags){}, - xFileControl: function(pFile, opId, pArg){}, - xFileSize: function(pFile,pi64Out){}, - xLock: true, - xUnlock: true, - xCheckReservedLock: true, - xSectorSize: true, - xDeviceCharacteristics: true -//#endif - }/*.ioJrnl*/ - }/*methodOverrides*/; - - debug("pVfs and friends", pVfs, pIoDb, pIoJrnl, - kvvfsMethods, capi.sqlite3_file.structInfo, - KVVfsFile.structInfo); - try { - util.assert( cache.buffer.n>1024*129, "Heap buffer is not large enough" - /* Native is SQLITE_KVOS_SZ is 133073 as of this writing */ ); - for(const e of Object.entries(methodOverrides.recordHandler)){ - // Overwrite kvvfsMethods's callbacks - const k = e[0], f = e[1]; - recordHandler[k] = f; - if( 0 ){ - // bug: this should work - kvvfsMethods.installMethod(k, f); - }else{ - kvvfsMethods[kvvfsMethods.memberKey(k)] = - wasm.installFunction(kvvfsMethods.memberSignature(k), f); - } - } - for(const e of Object.entries(methodOverrides.vfs)){ - // Overwrite some pVfs entries and stash the original impls - const k = e[0], f = e[1], km = pVfs.memberKey(k), - member = pVfs.structInfo.members[k] - || util.toss("Missing pVfs.structInfo[",k,"]"); - originalMethods.vfs[k] = wasm.functionEntry(pVfs[km]); - pVfs[km] = wasm.installFunction(member.signature, f); - } - for(const e of Object.entries(methodOverrides.ioDb)){ - // Similar treatment for pVfs.$pIoDb a.k.a. pIoDb... - const k = e[0], f = e[1], km = pIoDb.memberKey(k); - originalMethods.ioDb[k] = wasm.functionEntry(pIoDb[km]) - || util.toss("Missing native pIoDb[",km,"]"); - pIoDb[km] = wasm.installFunction(pIoDb.memberSignature(k), f); - } - for(const e of Object.entries(methodOverrides.ioJrnl)){ - // Similar treatment for pVfs.$pIoJrnl a.k.a. pIoJrnl... - const k = e[0], f = e[1], km = pIoJrnl.memberKey(k); - originalMethods.ioJrnl[k] = wasm.functionEntry(pIoJrnl[km]) - || util.toss("Missing native pIoJrnl[",km,"]"); - if( true===f ){ - /* use pIoDb's copy */ - pIoJrnl[km] = pIoDb[km] || util.toss("Missing copied pIoDb[",km,"]"); - }else{ - pIoJrnl[km] = wasm.installFunction(pIoJrnl.memberSignature(k), f); - } - } - }finally{ - kvvfsMethods.dispose(); - pVfs.dispose(); - pIoDb.dispose(); - pIoJrnl.dispose(); - } - - /* - That gets all of the low-level bits out of the way. What follows - are the public API additions. - */ - - /** - Clears all storage used by the kvvfs DB backend, deleting any - DB(s) stored there. - - Its argument must be the name of a kvvfs storage object: - - - 'session' - - 'local' - - '' - see below. - - A transient kvvfs storage object name. - - In the first two cases, only sessionStorage resp. localStorage is - cleared. An empty string resolves to both 'local' and 'session' - storage. - - Returns the number of entries cleared. - - As of kvvfs version 2: - - This API is available in Worker threads but does not have access - to localStorage or sessionStorage in them. Prior versions did not - include this API in Worker threads. - - Differences in this function in version 2: - - - It accepts an arbitrary storage name. In v1 this was a silent - no-op for any names other than ('local','session',''). - - - It throws if a db currently has the storage opened UNLESS the - storage object is localStorage or sessionStorage. That version 1 - did not throw for this case was due to an architectural - limitation which has since been overcome, but removal of - JsStorageDb.prototype.clearStorage() would be a backwards compatibility - break, so this function permits wiping the storage for those two - cases even if they are opened. Use with case. - */ - const sqlite3_js_kvvfs_clear = function callee(which){ - if( ''===which ){ - return callee('local') + callee('session'); - } - const store = storageForZClass(which); - if( !store ) return 0; - if( store.files.length ){ - if( globalThis.localStorage===store.storage - || globalThis.sessionStorage===store.storage ){ - /* backwards compatibility: allow these to be cleared - while opened. */ - }else{ - /* Interestingly, kvvfs recovers just fine when the storage is - wiped, so long as the db is not in use and its schema is - recreated before it's used, but client apps should not have - to be faced with that eventuality mid-query (where it - _will_ cause failures). Therefore we disallow it when - storage handles are opened. Kvvfs version 1 could not - detect this case - see the if() block above. - */ - toss3(capi.SQLITE_ACCESS, - "Cannot clear in-use database storage."); - } - } - const s = store.storage; - const toRm = [] /* keys to remove */; - let i, n = s.length; - //debug("kvvfs_clear",store,s); - for( i = 0; i < n; ++i ){ - const k = s.key(i); - //debug("kvvfs_clear ?",k); - if(!store.keyPrefix || k.startsWith(store.keyPrefix)) toRm.push(k); - } - toRm.forEach((kk)=>s.removeItem(kk)); - //alertFilesToReload(store); - return toRm.length; - }; - - /** - This routine estimates the approximate amount of - storage used by the given kvvfs back-end. - - Its arguments are as documented for sqlite3_js_kvvfs_clear(), - only the operation this performs is different. - - The returned value is twice the "length" value of every matching - key and value, noting that JavaScript stores each character in 2 - bytes. - - The returned size is not authoritative from the perspective of - how much data can fit into localStorage and sessionStorage, as - the precise algorithms for determining those limits are - unspecified and may include per-entry overhead invisible to - clients. - */ - const sqlite3_js_kvvfs_size = function callee(which){ - if( ''===which ){ - return callee('local') + callee('session'); - } - const store = storageForZClass(which); - if( !store ) return 0; - const s = store.storage; - let i, sz = 0; - for(i = 0; i < s.length; ++i){ - const k = s.key(i); - if(!store.keyPrefix || k.startsWith(store.keyPrefix)){ - sz += k.length; - sz += s.getItem(k).length; - } - } - return sz * 2 /* because JS uses 2-byte char encoding */; - }; - - /** - Exports a kvvfs storage object to an object, optionally - JSON-friendly. - - Usages: - - thisfunc(storageName); - thisfunc(options); - - In the latter case, the options object must be an object with - the following properties: - - - "name" (string) required. The storage to export. - - - "decodePages" (bool=false). If true, the .pages result property - holdes Uint8Array objects holding the raw binary-format db - pages. The default is to use kvvfs-encoded string pages - (JSON-friendly). - - - "includeJournal" (bool=false). If true and the db has a current - journal, it is exported as well. (Kvvfs journals are stored as a - single record within the db's storage object.) - - The returned object is structured as follows... - - - "name": the name of the storage. This is 'local' or 'session' - for localStorage resp. sessionStorage, and an arbitrary name for - transient storage. This propery may be changed before passing - this object to sqlite3_js_kvvfs_import() in order to - import into a different storage object. - - - "timestamp": the time this function was called, in Unix - epoch milliseconds. - - - "size": the unencoded db size. - - - "journal": if options.includeJournal is true and this db has a - journal, it is stored as a string here, otherwise this property - is not set. - - - "pages": An array holding the raw encoded db pages in their - proper order. - - Throws if this db is not opened. - - The encoding of the underlying database is not part of this - interface - it is simply passed on as-is. Interested parties are - directed to src/os_kv.c in the SQLite source tree, with the - caveat that that code also does not offer a public interface. - i.e. the encoding is a private implementation detail of kvvfs. - The format may be changed in the future but kvvfs will continue - to support the current form. - - Added in version @kvvfs-v2-added-in@. - */ - const sqlite3_js_kvvfs_export = function callee(...args){ - let opt; - if( 1===args.length && 'object'===typeof args[0] ){ - opt = args[0]; - }else if(args.length){ - opt = Object.assign(Object.create(null),{ - name: args[0], - //decodePages: true - }); - } - const store = opt ? storageForZClass(opt.name) : null; - if( !store ){ - toss3(capi.SQLITE_NOTFOUND, - "There is no kvvfs storage named",opt?.name); - } - //debug("store to export=",store); - const s = store.storage; - const rc = Object.assign(Object.create(null),{ - name: store.jzClass, - timestamp: Date.now(), - pages: [] - }); - const pages = Object.create(null); - let xpages; - const keyPrefix = store.keyPrefix; - const rxTail = keyPrefix - ? /^kvvfs-[^-]+-(\w+)/ /* X... part of kvvfs-NAME-X... */ - : undefined; - let i = 0, n = s.length; - for( ; i < n; ++i ){ - const k = s.key(i); - if( !keyPrefix || k.startsWith(keyPrefix) ){ - let kk = (keyPrefix ? rxTail.exec(k) : undefined)?.[1] ?? k; - switch( kk ){ - case 'jrnl': - if( opt.includeJournal ) rc.journal = s.getItem(k); - break; - case 'sz': - rc.size = +s.getItem(k); - break; - default: - kk = +kk /* coerce to number */; - if( !util.isInt32(kk) || kk<=0 ){ - toss3(capi.SQLITE_RANGE, "Malformed kvvfs key: "+k); - } - if( opt.decodePages ){ - const spg = s.getItem(k), - n = spg.length, - z = cache.memBuffer(0), - zDec = cache.memBuffer(1), - heap = wasm.heap8u()/* MUST be inited last*/; - let i = 0; - for( ; i < n; ++i ){ - heap[wasm.ptr.add(z, i)] = spg.codePointAt(i) & 0xff; - } - heap[wasm.ptr.add(z, i)] = 0; - //debug("Decoding",i,"page bytes"); - const nDec = kvvfsDecode( - z, zDec, cache.buffer.n - ); - //debug("Decoded",nDec,"page bytes"); - pages[kk] = heap.slice(Number(zDec), wasm.ptr.addn(zDec, nDec)); - }else{ - pages[kk] = s.getItem(k); - } - break; - } - } - } - if( opt.decodePages ) cache.memBufferFree(1); - /* Now sort the page numbers and move them into an array. In JS - property keys are always strings, so we have to coerce them to - numbers so we can get them sorted properly for the array. */ - Object.keys(pages).map((v)=>+v).sort().forEach( - (v)=>rc.pages.push(pages[v]) - ); - return rc; - }/* sqlite3_js_kvvfs_export */; - - /** - The counterpart of sqlite3_js_kvvfs_export(). Its - argument must be the result of that function() or - a compatible one. - - This either replaces the contents of an existing transient - storage object or installs one named exp.name, setting - the storage's db contents to that of the exp object. - - Throws on error. Error conditions include: - - - The given storage object is currently opened by any db. - Performing this page-by-page import would invoke undefined - behavior on them. - - - Malformed input object. - - If it throws after starting the import then it clears the storage - before returning, to avoid leaving the db in an undefined - state. It may throw for any of the above-listed conditions before - reaching that step, in which case the db is not modified. If - exp.name refers to a new storage name then if it throws, the name - does not get installed. - - Added in version @kvvfs-v2-added-in@. - */ - const sqlite3_js_kvvfs_import = function(exp, overwrite=false){ - if( !exp?.timestamp - || !exp.name - || undefined===exp.size - || !Array.isArray(exp.pages) ){ - toss3(capi.SQLITE_MISUSE, "Malformed export object."); - }else if( !exp.size - || (exp.size !== (exp.size | 0)) - //|| (exp.size % cache.fixedPageSize) - || exp.size>=0x7fffffff ){ - toss3(capi.SQLITE_RANGE, "Invalid db size: "+exp.size); - } - - validateStorageName(exp.name); - let store = storageForZClass(exp.name); - const isNew = !store; - if( store ){ - if( !overwrite ){ - //warn("Storage exists:",arguments,store); - toss3(capi.SQLITE_ACCESS, - "Storage '"+exp.name+"' already exists and", - "overwrite was not specified."); - }else if( !store.files || !store.jzClass ){ - toss3(capi.SQLITE_ERROR, - "Internal storage object", exp.name,"seems to be malformed."); - }else if( store.files.length ){ - toss3(capi.SQLITE_IOERR_ACCESS, - "Cannot import db storage while it is in use."); - } - sqlite3_js_kvvfs_clear(exp.name); - }else{ - store = newStorageObj(exp.name); - //warn("Installing new storage:",store); - } - //debug("Importing store",store.poolEntry.files.length, store); - //debug("object to import:",exp); - const keyPrefix = kvvfsKeyPrefix(exp.name); - let zEnc; - try{ - /* Force the native KVVfsFile instances to re-read the db - and page size. */; - const s = store.storage; - s.setItem(keyPrefix+'sz', exp.size); - if( exp.journal ) s.setItem(keyPrefix+'jrnl', exp.journal); - if( exp.pages[0] instanceof Uint8Array ){ - /* raw binary pages */ - //debug("pages",exp.pages); - exp.pages.forEach((u,ndx)=>{ - const n = u.length; - if( 0 && cache.fixedPageSize !== n ){ - util.toss3(capi.SQLITE_RANGE,"Unexpected page size:", n); - } - zEnc ??= cache.memBuffer(1); - const zBin = cache.memBuffer(0), - heap = wasm.heap8u()/*MUST be inited last*/; - /* Copy u to the heap and encode the heap copy via C. This - is _presumably_ faster than porting the encoding algo to - JS. */ - heap.set(u, Number(zBin)); - heap[wasm.ptr.addn(zBin,n)] = 0; - const rc = kvvfsEncode(zBin, n, zEnc); - util.assert( rc < cache.buffer.n, - "Impossibly long output - possibly smashed the heap" ); - util.assert( 0===wasm.peek8(wasm.ptr.add(zEnc,rc)), - "Expecting NUL-terminated encoded output" ); - const jenc = wasm.cstrToJs(zEnc); - //debug("(un)encoded page:",u,jenc); - s.setItem(keyPrefix+(ndx+1), jenc); - }); - }else if( exp.pages[0] ){ - /* kvvfs-encoded pages */ - exp.pages.forEach((v,ndx)=>s.setItem(keyPrefix+(ndx+1), v)); - } - if( isNew ) installStorageAndJournal(store); - }catch{ - if( !isNew ){ - try{sqlite3_js_kvvfs_clear(exp.name);}catch(ee){/*ignored*/} - } - }finally{ - if( zEnc ) cache.memBufferFree(1); - } - return this; - }; - - /** - If no kvvfs storage exists with the given name, one is - installed. If one exists, its reference count is increased so - that it won't be freed by the closing of a database or journal - file. - - Throws if the name is not valid for a new storage object. - - Added in version @kvvfs-v2-added-in@. - */ - const sqlite3_js_kvvfs_reserve = function(name){ - let store = storageForZClass(name); - if( store ){ - ++store.refc; - return; - } - validateStorageName(name); - installStorageAndJournal(newStorageObj(name)); - }; - - /** - Conditionally "unlinks" a kvvfs storage object, reducing its - reference count by 1. - - This is a no-op if name ends in "-journal" or refers to a - built-in storage object. - - It will not lower the refcount below the number of - currently-opened db/journal files for the storage (so that it - cannot delete it out from under them). - - If the refcount reaches 0 then the storage object is - removed. - - Returns true if it reduces the refcount, else false. A result of - true does not necessarily mean that the storage unit was removed, - just that its refcount was lowered. Similarly, a result of false - does not mean that the storage is removed - it may still have - opened handles. - - Added in version @kvvfs-v2-added-in@. - */ - const sqlite3_js_kvvfs_unlink = function(name){ - const store = storageForZClass(name); - if( !store - || kvvfsIsPersistentName(store.jzClass) - || isBuiltinName(store.jzClass) - || cache.rxJournalSuffix.test(name) ) return false; - if( store.refc > store.files.length || 0===store.files.length ){ - if( --store.refc<=0 ){ - /* Ignoring deleteAtRefc0 for an explicit unlink */ - deleteStorage(store); - } - return true; - } - return false; - }; - - /** - Adds an event listener to a kvvfs storage object. The idea is - that this can be used to asynchronously back up one kvvfs storage - object to another or another channel entirely. (The caveat in the - latter case is that kvvfs's format is not readily consumable by - downstream code.) - - Its argument must be an object with the following properties: - - - storage: the name of the kvvfs storage object. - - - reserve [=false]: if true, sqlite3_js_kvvfs_reserve() is used - to ensure that the storage exists if it does not already. - If this is false and the storage does not exist then an - exception is thrown. - - - events: an object which may have any of the following - callback function properties: open, close, write, delete. - - - decodePages [=false]: if true, write events will receive each - db page write in the form of a Uint8Array holding the raw binary - db page. The default is to emit the kvvfs-format page because it - requires no extra work, we already have it in hand, and it's - often smaller. It's not great for interchange, though. - - - includeJournal [=false]: if true, writes and deletes of - "jrnl" records are included. If false, no events are sent - for journal updates. - - Passing the same object to sqlite3_js_kvvfs_unlisten() will - remove the listener. - - Each one of the events callbacks will be called asynchronously - when the given storage performs those operations. They may be - asynchronous functions but are not required to be (the events are - fired async either way, but making the event callbacks async may - be advantageous when multiple listeners are involved). All - exceptions, including those via Promises, are ignored but may (or - may not) trigger warning output on the console. - - Each callback gets passed a single object with the following - properties: - - .type = the same as the name of the callback - - .storageName = the name of the storage object - - .data = callback-dependent: - - - 'open' and 'close' get an integer, the number of - currently-opened handles on the storage. - - - 'write' gets a length-two array holding the key and value which - were written. The key is always a string, even if it's a db page - number. For db-page records, the value's type depends on - opt.decodePages. All others, including the journal, are strings. - (The journal, being a kvvfs-specific format, is delivered in - that same JSON-friendly format.) More details below. - - - 'delete' gets the string-type key of the deleted record. - - - 'sync' gets a boolean value: true if it was triggered by db - file's xSync(), false if it was triggered by xFileControl(). The - latter triggers before the xSync() and also triggers if the DB - has PRAGMA SYNCHRONOUS=OFF (in which case xSync() is not - triggered). - - The key/value arguments to 'write', and key argument to 'delete', - are in one of the following forms: - - - 'sz' = the unencoded db size as a string. This specific key is - key is never deleted, so is only ever passed to 'write' events. - - - 'jrnl' = the current db journal as a kvvfs-encoded string. This - journal format is not useful anywhere except in the kvvfs - internals. These events are not fired if opt.includeJournal is - false. - - - '[1-9][0-9]*' (a db page number) = Its type depends on - opt.decodePages. These may be written and deleted in arbitrary - order. - - Design note: JS has StorageEvents but only in the main thread, - which is why the listeners are not based on that. - - Added in version @kvvfs-v2-added-in@. - */ - const sqlite3_js_kvvfs_listen = function(opt){ - if( !opt || 'object'!==typeof opt ){ - toss3(capi.SQLITE_MISUSE, "Expecting a listener object."); - } - let store = storageForZClass(opt.storage); - if( !store ){ - if( opt.storage && opt.reserve ){ - sqlite3_js_kvvfs_reserve(opt.storage); - store = storageForZClass(opt.storage); - util.assert(store, - "Unexpectedly cannot fetch reserved storage " - +opt.storage); - }else{ - toss3(capi.SQLITE_NOTFOUND,"No such storage:",opt.storage); - } - } - if( opt.events ){ - (store.listeners ??= []).push(opt); - } - }; - - /** - Removes the kvvfs event listeners for the given options - object. It must be passed the same object instance which was - passed to sqlite3_js_kvvfs_listen(). - - This has no side effects if opt is invalid or is not a match for - any listeners. - - Return true if it unregisters its argument, else false. - - Added in version @kvvfs-v2-added-in@. - */ - const sqlite3_js_kvvfs_unlisten = function(opt){ - const store = storageForZClass(opt?.storage); - if( store?.listeners && opt.events ){ - const n = store.listeners.length; - store.listeners = store.listeners.filter((v)=>v!==opt); - const rc = n>store.listeners.length; - if( !store.listeners.length ){ - // to speed up downstream checks for listeners - store.listeners = undefined; - } - return rc; - } - return false; - }; - - sqlite3.kvvfs.reserve = sqlite3_js_kvvfs_reserve; - sqlite3.kvvfs.import = sqlite3_js_kvvfs_import; - sqlite3.kvvfs.export = sqlite3_js_kvvfs_export; - sqlite3.kvvfs.unlink = sqlite3_js_kvvfs_unlink; - sqlite3.kvvfs.listen = sqlite3_js_kvvfs_listen; - sqlite3.kvvfs.unlisten = sqlite3_js_kvvfs_unlisten; - sqlite3.kvvfs.exists = (name)=>!!storageForZClass(name); - sqlite3.kvvfs.estimateSize = sqlite3_js_kvvfs_size; - sqlite3.kvvfs.clear = sqlite3_js_kvvfs_clear; - - - if( globalThis.Storage ){ - /** - Prior to version 2, kvvfs was only available in the main - thread. We retain that for the v1 APIs, exposing them only in - the main UI thread. As of version 2, kvvfs is available in all - threads but only via its v2 interface (sqlite3.kvvfs). - - These versions have a default argument value of "" which the v2 - versions lack. - */ - capi.sqlite3_js_kvvfs_size = (which="")=>sqlite3_js_kvvfs_size(which); - capi.sqlite3_js_kvvfs_clear = (which="")=>sqlite3_js_kvvfs_clear(which); - } - -//#if not omit-oo1 - if(sqlite3.oo1?.DB){ - /** - Functionally equivalent to DB(storageName,'c','kvvfs') except - that it throws if the given storage name is not one of 'local' - or 'session'. - - As of version 3.46, the argument may optionally be an options - object in the form: - - { - filename: 'session'|'local', - ... etc. (all options supported by the DB ctor) - } - - noting that the 'vfs' option supported by main DB - constructor is ignored here: the vfs is always 'kvvfs'. - */ - const DB = sqlite3.oo1.DB; - sqlite3.oo1.JsStorageDb = function( - storageName = sqlite3.oo1.JsStorageDb.defaultStorageName - ){ - const opt = DB.dbCtorHelper.normalizeArgs(...arguments); - opt.vfs = 'kvvfs'; - if( 0 ){ - // Current tests rely on these, but that's arguably a bug - if( opt.flags ) opt.flags = 'cw'+opt.flags; - else opt.flags = 'cw'; - } - switch( opt.filename ){ - /* sqlite3_open(), in these builds, recognizes the names - below and performs some magic which we want to bypass - here for sanity's sake. */ - case ":sessionStorage:": opt.filename = 'session'; break; - case ":localStorage:": opt.filename = 'local'; break; - } - const m = /(file:(\/\/)?)([^?]+)/.exec(opt.filename); - validateStorageName( m ? m[3] : opt.filename); - DB.dbCtorHelper.call(this, opt); - }; - sqlite3.oo1.JsStorageDb.defaultStorageName - = cache.storagePool.session ? 'session' : nameOfThisThreadStorage; - const jdb = sqlite3.oo1.JsStorageDb; - jdb.prototype = Object.create(DB.prototype); - jdb.clearStorage = sqlite3_js_kvvfs_clear; - /** - DEPRECATED: the inherited method of this name (as opposed to - the "static" class method) is deprecated with version 2 of - kvvfs. This function will, for backwards comaptibility, - continue to work with localStorage and sessionStorage, but will - throw for all other storage because they are opened. Version 1 - was not capable of recognizing that the storage was opened so - permitted wiping it out at any time, but that was arguably a - bug. - - Clears this database instance's storage or throws if this - instance has been closed. Returns the number of - database pages which were cleaned up. - */ - jdb.prototype.clearStorage = function(){ - return jdb.clearStorage(this.affirmOpen().dbFilename(), true); - }; - /** Equivalent to sqlite3_js_kvvfs_size(). */ - jdb.storageSize = sqlite3_js_kvvfs_size; - /** - Returns the _approximate_ number of bytes this database takes - up in its storage or throws if this instance has been closed. - */ - jdb.prototype.storageSize = function(){ - return jdb.storageSize(this.affirmOpen().dbFilename(), true); - }; - }/*sqlite3.oo1.JsStorageDb*/ -//#endif not omit-oo1 - - if( sqlite3.__isUnderTest && sqlite3.vtab ){ - /** - An eponymous vtab for inspecting the kvvfs state. This is only - intended for use in testing and development, not part of the - public API. - */ - const cols = Object.assign(Object.create(null),{ - rowid: {type: 'INTEGER'}, - name: {type: 'TEXT'}, - nRef: {type: 'INTEGER'}, - nOpen: {type: 'INTEGER'}, - isTransient: {type: 'INTEGER'}, - dbSize: {type: 'INTEGER'} - }); - Object.keys(cols).forEach((v,i)=>cols[v].colId = i); - - const VT = sqlite3.vtab; - const ProtoCursor = Object.assign(Object.create(null),{ - row: function(){ - return cache.storagePool[this.names[this.rowid]]; - } - }); - Object.assign(Object.create(ProtoCursor),{ - rowid: 0, - names: Object.keys(cache.storagePool) - .filter(v=>!cache.rxJournalSuffix.test(v)) - }); - const cursorState = function(cursor, reset){ - const o = (cursor instanceof capi.sqlite3_vtab_cursor) - ? cursor - : VT.xCursor.get(cursor); - if( reset || !o.vTabState ){ - o.vTabState = Object.assign(Object.create(ProtoCursor),{ - rowid: 0, - names: Object.keys(cache.storagePool) - .filter(v=>!cache.rxJournalSuffix.test(v)) - }); - } - return o.vTabState; - }; - - const dbg = 1 ? ()=>{} : (...args)=>debug("vtab",...args); - - const theModule = function f(){ - return f.mod ??= new sqlite3.capi.sqlite3_module().setupModule({ - catchExceptions: true, - methods: { - xConnect: function(pDb, pAux, argc, argv, ppVtab, pzErr){ - dbg("xConnect"); - try{ - const xcol = []; - Object.keys(cols).forEach((k)=>{ - xcol.push(k+" "+cols[k].type); - }); - const rc = capi.sqlite3_declare_vtab( - pDb, "CREATE TABLE ignored("+xcol.join(',')+")" - ); - if(0===rc){ - const t = VT.xVtab.create(ppVtab); - util.assert( - (t === VT.xVtab.get(wasm.peekPtr(ppVtab))), - "output pointer check failed" - ); - } - return rc; - }catch(e){ - return VT.xError('xConnect', e, capi.SQLITE_ERROR); - } - }, - xCreate: wasm.ptr.null, // eponymous only - //xCreate: true, // copy xConnect, i.e. also eponymous only - xDisconnect: function(pVtab){ - dbg("xDisconnect",...arguments); - VT.xVtab.dispose(pVtab); - return 0; - }, - xOpen: function(pVtab, ppCursor){ - dbg("xOpen",...arguments); - VT.xCursor.create(ppCursor); - return 0; - }, - xClose: function(pCursor){ - dbg("xClose",...arguments); - const c = VT.xCursor.unget(pCursor); - delete c.vTabState; - c.dispose(); - return 0; - }, - xNext: function(pCursor){ - dbg("xNext",...arguments); - const c = VT.xCursor.get(pCursor); - ++cursorState(c).rowid; - return 0; - }, - xColumn: function(pCursor, pCtx, iCol){ - dbg("xColumn",...arguments); - //const c = VT.xCursor.get(pCursor); - const st = cursorState(pCursor); - const store = st.row(); - util.assert(store, "Unexpected xColumn call"); - switch(iCol){ - case cols.rowid.colId: - capi.sqlite3_result_int(pCtx, st.rowid); - break; - case cols.name.colId: - capi.sqlite3_result_text(pCtx, store.jzClass, -1, capi.SQLITE_TRANSIENT); - break; - case cols.nRef.colId: - capi.sqlite3_result_int(pCtx, store.refc); - break; - case cols.nOpen.colId: - capi.sqlite3_result_int(pCtx, store.files.length); - break; - case cols.isTransient.colId: - capi.sqlite3_result_int(pCtx, !!store.deleteAtRefc0); - break; - case cols.dbSize.colId: - capi.sqlite3_result_int(pCtx, storageGetDbSize(store)); - break; - default: - capi.sqlite3_result_error(pCtx, "Invalid column id: "+iCol); - return capi.SQLITE_RANGE; - } - return 0; - }, - xRowid: function(pCursor, ppRowid64){ - dbg("xRowid",...arguments); - const st = cursorState(pCursor); - VT.xRowid(ppRowid64, st.rowid); - return 0; - }, - xEof: function(pCursor){ - const st = cursorState(pCursor); - dbg("xEof?="+(!st.row()),...arguments); - return !st.row(); - }, - xFilter: function(pCursor, idxNum, idxCStr, - argc, argv/* [sqlite3_value* ...] */){ - dbg("xFilter",...arguments); - const st = cursorState(pCursor, true); - return 0; - }, - xBestIndex: function(pVtab, pIdxInfo){ - dbg("xBestIndex",...arguments); - //const t = VT.xVtab.get(pVtab); - const pii = new capi.sqlite3_index_info(pIdxInfo); - pii.$estimatedRows = cache.storagePool.size; - pii.$estimatedCost = 1.0; - pii.dispose(); - return 0; - } - } - })/*setupModule*/; - }/*theModule()*/; - - sqlite3.kvvfs.create_module = function(pDb, name="sqlite_kvvfs"){ - return capi.sqlite3_create_module(pDb, name, theModule(), - wasm.ptr.null); - }; - - }/* virtual table */ - -//#if nope - /** - The idea here is a simpler wrapper for listening to kvvfs - changes. Clients would override its onXyz() event methods - instead of providing callbacks for sqlite3.kvvfs.listen(), the - main (only?) benefit of which is that this class would do the - sorting-out and validation of event state before calling the - overloaded callbacks. - */ - kvvfs.Listener = class KvvfsListener { - #store; - #listener; - - constructor(opt){ - this.#listenTo(opt); - } - - #event(ev){ - switch(ev.type){ - case 'open': this.onOpen(ev.data); break; - case 'close': this.onClose(ev.data); break; - case 'sync': this.onSync(ev.data); break; - case 'delete': - switch(ev.data){ - case 'jrnl': break; - default:{ - const n = +ev.data; - util.assert( n>0, "Expecting positive db page number" ); - this.onPageChange(n, null); - break; - } - } - break; - case 'write':{ - const key = ev.data[0], val = ev.data[1]; - switch( key ){ - case 'jrnl': break; - case 'sz':{ - const sz = +val; - util.assert( sz>0, "Expecting a db page number" ); - this.onSizeChange(sz); - break; - } - default: - T.assert( +key>0, "Expecting a positive db page number" ); - this.onPageChange(+key, val); - break; - } - break; - } - } - } - - #listenTo(opt){ - if(this.#listener){ - sqlite3_js_kvvfs_unlisten(this.#listener); - this.#listener = undefined; - } - const eventHandler = async function(ev){this.event(ev)}.bind(this); - const li = Object.assign( - { /* Defaults */ - reserve: false, - includeJournal: false, - decodePages: false, - storage: null - }, - (/*client options*/opt||{}), - {/*hard-coded options*/ - events: Object.assign(Object.create(null),{ - 'open': eventHandler, - 'close': eventHandler, - 'write': eventHandler, - 'delete': eventHandler, - 'sync': eventHandler - }) - } - ); - sqlite3_js_kvvfs_listen(li); - this.#listener = li; - } - - async onSizeChange(sz){} - async onPageChange(pgNo,content/*null for delete*/){} - async onSync(mode/*true=xSync, false=xFileControl*/){} - async onOpen(count){} - async onClose(count){} - }/*KvvfsListener*/; -//#endif nope - -})/*globalThis.sqlite3ApiBootstrap.initializers*/; -//#savepoint rollback -//#endif not omit-kvvfs diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js index c1fee5e1a..69be338b0 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js @@ -358,7 +358,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ try{ const [cMsg, n] = wasm.scopedAllocCString(e.message, true); wasm.cstrncpy(pOut, cMsg, nOut); - if(n > nOut) wasm.poke8(wasm.ptr.add(pOut,nOut,-1), 0); + if(n > nOut) wasm.poke8(pOut + nOut - 1, 0); }catch(e){ return capi.SQLITE_NOMEM; }finally{ @@ -410,7 +410,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }/*vfsMethods*/; /** - Creates, initializes, and returns an sqlite3_vfs instance for an + Creates and initializes an sqlite3_vfs instance for an OpfsSAHPool. The argument is the VFS's name (JS string). Throws if the VFS name is already registered or if something @@ -1157,9 +1157,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ described at the end of these docs. This function accepts an options object to configure certain - parts but it is only acknowledged for the very first call for - each distinct name and ignored for all subsequent calls with that - same name. + parts but it is only acknowledged for the very first call and + ignored for all subsequent calls. The options, in alphabetical order: @@ -1225,14 +1224,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - Paths given to it _must_ be absolute. Relative paths will not be properly recognized. This is arguably a bug but correcting it requires some hoop-jumping in routines which have no business - doing such tricks. (2026-01-19 (2.5 years later): the specifics - are lost to history, but this was a side effect of xOpen() - receiving an immutable C-string filename, to which no implicit - "/" can be prefixed without causing a discrepancy between what - the user provided and what the VFS stores. Its conceivable that - that quirk could be glossed over in xFullPathname(), but - regressions when doing so cannot be ruled out, so there are no - current plans to change this behavior.) + doing such tricks. - It is possible to install multiple instances under different names, each sandboxed from one another inside their own private diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index ffa90ed06..2b636460d 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -16,7 +16,7 @@ asynchronous Origin-Private FileSystem (OPFS) APIs using a second Worker, implemented in sqlite3-opfs-async-proxy.js. This file is intended to be appended to the main sqlite3 JS deliverable somewhere - after sqlite3-api-oo1.js. + after sqlite3-api-oo1.js and before sqlite3-api-cleanup.js. */ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ @@ -589,7 +589,7 @@ const installOpfsVfs = function callee(options){ /** Returns an array of the deserialized state stored by the most - recent serialize() operation (from this thread or the + recent serialize() operation (from from this thread or the counterpart thread), or null if the serialization buffer is empty. If passed a truthy argument, the serialization buffer is cleared after deserialization. @@ -924,7 +924,7 @@ const installOpfsVfs = function callee(options){ fh.filename = zName; fh.sab = new SharedArrayBuffer(state.fileBufferSize); fh.flags = flags; - fh.readOnly = !(capi.SQLITE_OPEN_CREATE & flags) + fh.readOnly = !(sqlite3.SQLITE_OPEN_CREATE & flags) && !!(flags & capi.SQLITE_OPEN_READONLY); const rc = opRun('xOpen', pFile, zName, flags, opfsFlags); if(!rc){ @@ -1441,7 +1441,7 @@ installOpfsVfs.defaultProxyUri = globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ try{ let proxyJs = installOpfsVfs.defaultProxyUri; - if( sqlite3?.scriptInfo?.sqlite3Dir ){ + if(sqlite3.scriptInfo.sqlite3Dir){ installOpfsVfs.defaultProxyUri = sqlite3.scriptInfo.sqlite3Dir + proxyJs; //sqlite3.config.warn("installOpfsVfs.defaultProxyUri =",installOpfsVfs.defaultProxyUri); diff --git a/ext/wasm/api/sqlite3-vtab-helper.c-pp.js b/ext/wasm/api/sqlite3-vtab-helper.c-pp.js index 80f4bfac2..4c2338fc5 100644 --- a/ext/wasm/api/sqlite3-vtab-helper.c-pp.js +++ b/ext/wasm/api/sqlite3-vtab-helper.c-pp.js @@ -172,7 +172,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Works like unget() plus it calls dispose() on the StructType object. */ - dispose: (pCObj)=>__xWrap(pCObj,true)?.dispose?.() + dispose: (pCObj)=>{ + const o = __xWrap(pCObj,true); + if(o) o.dispose(); + } }); }; diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index dbfcdb704..4d5e9b296 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -93,18 +93,6 @@ #undef SQLITE_ENABLE_API_ARMOR #define SQLITE_ENABLE_API_ARMOR 1 -/**********************************************************************/ -/* SQLITE_EXPERIMENTAL_PRAGMA_20251114 */ -/* -** See: -** https://sqlite.org/src/info/e2b3f1a9480a9be3 -** https://github.com/rhashimoto/wa-sqlite/discussions/301 -** -** It is enabled here for the sake of VFS experimentors. -*/ -#undef SQLITE_EXPERIMENTAL_PRAGMA_20251114 -#define SQLITE_EXPERIMENTAL_PRAGMA_20251114 - /**********************************************************************/ /* SQLITE_O... */ #undef SQLITE_OMIT_DEPRECATED @@ -148,8 +136,8 @@ /* ** If SQLITE_WASM_BARE_BONES is defined, undefine most of the ENABLE ** macros. This will, when using the canonical makefile, also elide -** any C functions from the WASM exports: see -** ./EXPORTED_FUNCTIONS.c-pp. +** any C functions from the WASM exports which are listed in +** ./EXPORT_FUNCTIONS.sqlite3-extras. */ #ifdef SQLITE_WASM_BARE_BONES # undef SQLITE_ENABLE_COLUMN_METADATA @@ -229,18 +217,15 @@ ** not by client code, so an argument can be made for reducing their ** visibility by not including them in any build-time export lists. ** -** 2025-12-01: for use in non-Emscripten builds, we need a more -** invasive macro which explicitly names the export: -** SQLITE_WASM_EXPORT2. +** 2022-09-11: it's not yet _proven_ that this approach works in +** non-Emscripten builds. If not, such builds will need to export +** those using the --export=... wasm-ld flag (or equivalent). As of +** this writing we are tied to Emscripten for various reasons +** and cannot test the library with other build environments. */ #define SQLITE_WASM_EXPORT __attribute__((used,visibility("default"))) -#define SQLITE_WASM_EXPORT_NAMED(X) __attribute__((export_name(#X),used,visibility("default"))) -#define SQLITE_WASM_EXPORT2(RETTYPE,NAME,SIG) SQLITE_WASM_EXPORT_NAMED(NAME) RETTYPE NAME SIG - -#if 1 -/** Increase the kvvfs key size limit from 32. */ -#define KVRECORD_KEY_SZ 128 -#endif +// See also: +//__attribute__((export_name("theExportedName"), used, visibility("default"))) /* ** Which sqlite3.c we're using needs to be configurable to enable @@ -264,6 +249,45 @@ #undef INC__STRINGIFY #undef SQLITE_C +#if 0 +/* +** An EXPERIMENT in implementing a stack-based allocator analog to +** Emscripten's stackSave(), stackAlloc(), stackRestore(). +** Unfortunately, this cannot work together with Emscripten because +** Emscripten defines its own native one and we'd stomp on each +** other's memory. Other than that complication, basic tests show it +** to work just fine. +** +** Another option is to malloc() a chunk of our own and call that our +** "stack". +*/ +SQLITE_WASM_EXPORT void * sqlite3__wasm_stack_end(void){ + extern void __heap_base + /* see https://stackoverflow.com/questions/10038964 */; + return &__heap_base; +} +SQLITE_WASM_EXPORT void * sqlite3__wasm_stack_begin(void){ + extern void __data_end; + return &__data_end; +} +static void * pWasmStackPtr = 0; +SQLITE_WASM_EXPORT void * sqlite3__wasm_stack_ptr(void){ + if(!pWasmStackPtr) pWasmStackPtr = sqlite3__wasm_stack_end(); + return pWasmStackPtr; +} +SQLITE_WASM_EXPORT void sqlite3__wasm_stack_restore(void * p){ + pWasmStackPtr = p; +} +SQLITE_WASM_EXPORT void * sqlite3__wasm_stack_alloc(int n){ + if(n<=0) return 0; + n = (n + 7) & ~7 /* align to 8-byte boundary */; + unsigned char * const p = (unsigned char *)sqlite3__wasm_stack_ptr(); + unsigned const char * const b = (unsigned const char *)sqlite3__wasm_stack_begin(); + if(b + n >= p || b + n < b/*overflow*/) return 0; + return pWasmStackPtr = p - n; +} +#endif /* stack allocator experiment */ + /* ** State for the "pseudo-stack" allocator implemented in ** sqlite3__wasm_pstack_xyz(). In order to avoid colliding with @@ -302,7 +326,7 @@ SQLITE_WASM_EXPORT void * sqlite3__wasm_pstack_ptr(void){ */ SQLITE_WASM_EXPORT void sqlite3__wasm_pstack_restore(unsigned char * p){ assert(p>=PStack.pBegin && p<=PStack.pEnd && p>=PStack.pPos); - assert(0==((unsigned long long)p & 0x7) /* 8-byte aligned */); + assert(0==((unsigned long long)p & 0x7)); if(p>=PStack.pBegin && p<=PStack.pEnd /*&& p>=PStack.pPos*/){ PStack.pPos = p; } @@ -312,10 +336,10 @@ SQLITE_WASM_EXPORT void sqlite3__wasm_pstack_restore(unsigned char * p){ ** the memory on success, 0 on error (including a negative n value). n ** is always adjusted to be a multiple of 8 and returned memory is ** always zeroed out before returning (because this keeps the client -** JS code from having to do so, and most uses of the pstack call for -** doing so). +** JS code from having to do so, and most uses of the pstack will +** call for doing so). */ -SQLITE_WASM_EXPORT2(void *,sqlite3__wasm_pstack_alloc,(int n)){ +SQLITE_WASM_EXPORT void * sqlite3__wasm_pstack_alloc(int n){ if( n<=0 ) return 0; n = (n + 7) & ~7 /* align to 8-byte boundary */; if( PStack.pBegin + n > PStack.pPos /*not enough space left*/ @@ -327,7 +351,7 @@ SQLITE_WASM_EXPORT2(void *,sqlite3__wasm_pstack_alloc,(int n)){ ** Return the number of bytes left which can be ** sqlite3__wasm_pstack_alloc()'d. */ -SQLITE_WASM_EXPORT2(int,sqlite3__wasm_pstack_remaining,(void)){ +SQLITE_WASM_EXPORT int sqlite3__wasm_pstack_remaining(void){ assert(PStack.pPos >= PStack.pBegin); assert(PStack.pPos <= PStack.pEnd); return (int)(PStack.pPos - PStack.pBegin); @@ -338,7 +362,7 @@ SQLITE_WASM_EXPORT2(int,sqlite3__wasm_pstack_remaining,(void)){ ** any space which is currently allocated. This value is a ** compile-time constant. */ -SQLITE_WASM_EXPORT2(int,sqlite3__wasm_pstack_quota,(void)){ +SQLITE_WASM_EXPORT int sqlite3__wasm_pstack_quota(void){ return (int)(PStack.pEnd - PStack.pBegin); } @@ -351,7 +375,8 @@ struct WasmTestStruct { void (*xFunc)(void*); }; typedef struct WasmTestStruct WasmTestStruct; -SQLITE_WASM_EXPORT2(void,sqlite3__wasm_test_struct,(WasmTestStruct * s)){ +SQLITE_WASM_EXPORT +void sqlite3__wasm_test_struct(WasmTestStruct * s){ if(s){ if( 0 ){ /* Do not be alarmed by the small (and odd) pointer values. @@ -391,7 +416,8 @@ SQLITE_WASM_EXPORT2(void,sqlite3__wasm_test_struct,(WasmTestStruct * s)){ ** fails to compile with "tables may not be 64-bit" but does not tell ** us where it's happening. */ -SQLITE_WASM_EXPORT2(const char *,sqlite3__wasm_enum_json,(void)){ +SQLITE_WASM_EXPORT +const char * sqlite3__wasm_enum_json(void){ static char aBuffer[1024 * 20] = {0} /* where the JSON goes. 2025-09-19: output size=19295, but that can vary slightly from build to build, so a little @@ -594,7 +620,6 @@ SQLITE_WASM_EXPORT2(const char *,sqlite3__wasm_enum_json,(void)){ DefGroup(encodings) { /* Noting that the wasm binding only aims to support UTF-8. */ DefInt(SQLITE_UTF8); - DefInt(SQLITE_UTF8_ZT); DefInt(SQLITE_UTF16LE); DefInt(SQLITE_UTF16BE); DefInt(SQLITE_UTF16); @@ -982,15 +1007,13 @@ SQLITE_WASM_EXPORT2(const char *,sqlite3__wasm_enum_json,(void)){ /** ^^^ indirection needed to expand CurrentStruct */ #define StructBinder StructBinder_(CurrentStruct) #define _StructBinder CloseBrace(2) -#define M3(MEMBER,SIG,READONLY) \ - outf("%s\"%s\": " \ - "{\"offset\":%d,\"sizeof\":%d,\"signature\":\"%s\"%s}", \ - (n++ ? ", " : ""), #MEMBER, \ - (int)offsetof(CurrentStruct,MEMBER), \ - (int)sizeof(((CurrentStruct*)0)->MEMBER), \ - SIG, (READONLY ? ",\"readOnly\":true" : "")) -#define M(MEMBER,SIG) M3(MEMBER,SIG,0) -#define MRO(MEMBER,SIG) M3(MEMBER,SIG,1) +#define M(MEMBER,SIG) \ + outf("%s\"%s\": " \ + "{\"offset\":%d,\"sizeof\": %d,\"signature\":\"%s\"}", \ + (n++ ? ", " : ""), #MEMBER, \ + (int)offsetof(CurrentStruct,MEMBER), \ + (int)sizeof(((CurrentStruct*)0)->MEMBER), \ + SIG) nStruct = 0; out(", \"structs\": ["); { @@ -1053,30 +1076,11 @@ SQLITE_WASM_EXPORT2(const char *,sqlite3__wasm_enum_json,(void)){ #undef CurrentStruct #define CurrentStruct sqlite3_kvvfs_methods - /* From os_kv.c */ - StructBinder { - M(xRcrdRead, "i(sspi)"); - M(xRcrdWrite, "i(sss)"); - M(xRcrdDelete, "i(ss)"); - MRO(nKeySize, "i"); - MRO(nBufferSize, "i"); - M(pVfs, "p"); - M(pIoDb, "p"); - M(pIoJrnl, "p"); - } _StructBinder; -#undef CurrentStruct - -#define CurrentStruct KVVfsFile - /* From os_kv.c */ StructBinder { - M(base, "p")/*sqlite3_file base*/; - M(zClass, "s"); - M(isJournal, "i"); - M(nJrnl, "i")/*actually unsigned!*/; - M(aJrnl, "p"); - M(szPage, "i"); - M(szDb, "j"); - M(aData, "p"); + M(xRead, "i(sspi)"); + M(xWrite, "i(sss)"); + M(xDelete, "i(ss)"); + M(nKeySize, "i"); } _StructBinder; #undef CurrentStruct @@ -1133,13 +1137,7 @@ SQLITE_WASM_EXPORT2(const char *,sqlite3__wasm_enum_json,(void)){ ** sqlite3_index_info, we have to uplift those into constructs we ** can access by type name. These structs _must_ match their ** in-sqlite3_index_info counterparts byte for byte. - ** - ** 2025-11-21: this uplifing is no longer necessary, as Jaccwabyt - ** can now handle nested structs, but "it ain't broke" so there's - ** no pressing need to rewire this. Also, it's conceivable that - ** rewiring it might break downstream vtab impls, so it shouldn't - ** be rewired. - */ + */ typedef struct { int iColumn; unsigned char op; @@ -1234,8 +1232,6 @@ SQLITE_WASM_EXPORT2(const char *,sqlite3__wasm_enum_json,(void)){ #undef StructBinder_ #undef StructBinder__ #undef M -#undef MRO -#undef M3 #undef _StructBinder #undef CloseBrace #undef out @@ -1253,7 +1249,8 @@ SQLITE_WASM_EXPORT2(const char *,sqlite3__wasm_enum_json,(void)){ ** method, SQLITE_MISUSE is returned, else the result of the xDelete() ** call is returned. */ -SQLITE_WASM_EXPORT2(int,sqlite3__wasm_vfs_unlink,(sqlite3_vfs *pVfs, const char *zName)){ +SQLITE_WASM_EXPORT +int sqlite3__wasm_vfs_unlink(sqlite3_vfs *pVfs, const char *zName){ int rc = SQLITE_MISUSE /* ??? */; if( 0==pVfs && 0!=zName ) pVfs = sqlite3_vfs_find(0); if( zName && pVfs && pVfs->xDelete ){ @@ -1270,7 +1267,8 @@ SQLITE_WASM_EXPORT2(int,sqlite3__wasm_vfs_unlink,(sqlite3_vfs *pVfs, const char ** defaulting to "main" if zDbName is 0. Returns 0 if no db with the ** given name is open. */ -SQLITE_WASM_EXPORT2(sqlite3_vfs *,sqlite3__wasm_db_vfs,(sqlite3 *pDb, const char *zDbName)){ +SQLITE_WASM_EXPORT +sqlite3_vfs * sqlite3__wasm_db_vfs(sqlite3 *pDb, const char *zDbName){ sqlite3_vfs * pVfs = 0; sqlite3_file_control(pDb, zDbName ? zDbName : "main", SQLITE_FCNTL_VFS_POINTER, &pVfs); @@ -1292,7 +1290,8 @@ SQLITE_WASM_EXPORT2(sqlite3_vfs *,sqlite3__wasm_db_vfs,(sqlite3 *pDb, const char ** Returns 0 on success, an SQLITE_xxx code on error. Returns ** SQLITE_MISUSE if pDb is NULL. */ -SQLITE_WASM_EXPORT2(int,sqlite3__wasm_db_reset,(sqlite3 *pDb)){ +SQLITE_WASM_EXPORT +int sqlite3__wasm_db_reset(sqlite3 *pDb){ int rc = SQLITE_MISUSE; if( pDb ){ sqlite3_table_column_metadata(pDb, "main", 0, 0, 0, 0, 0, 0, 0); @@ -1373,10 +1372,10 @@ int sqlite3__wasm_db_export_chunked( sqlite3* pDb, ** If `*pOut` is not NULL, the caller is responsible for passing it to ** sqlite3_free() to free it. */ -SQLITE_WASM_EXPORT2(int,sqlite3__wasm_db_serialize, - (sqlite3 *pDb, const char *zSchema, - unsigned char **pOut, - sqlite3_int64 *nOut, unsigned int mFlags)){ +SQLITE_WASM_EXPORT +int sqlite3__wasm_db_serialize( sqlite3 *pDb, const char *zSchema, + unsigned char **pOut, + sqlite3_int64 *nOut, unsigned int mFlags ){ unsigned char * z; if( !pDb || !pOut ) return SQLITE_MISUSE; if( nOut ) *nOut = 0; @@ -1437,9 +1436,11 @@ SQLITE_WASM_EXPORT2(int,sqlite3__wasm_db_serialize, ** portability, so that the API can still work in builds where BigInt ** support is disabled or unavailable. */ -SQLITE_WASM_EXPORT2(int,sqlite3__wasm_vfs_create_file, - (sqlite3_vfs *pVfs, const char *zFilename, - const unsigned char * pData, int nData)){ +SQLITE_WASM_EXPORT +int sqlite3__wasm_vfs_create_file( sqlite3_vfs *pVfs, + const char *zFilename, + const unsigned char * pData, + int nData ){ int rc; sqlite3_file *pFile = 0; sqlite3_io_methods const *pIo; @@ -1525,9 +1526,10 @@ SQLITE_WASM_EXPORT2(int,sqlite3__wasm_vfs_create_file, ** zFilename, appends pData bytes to it, and returns 0 on success or ** SQLITE_IOERR on error. */ -SQLITE_WASM_EXPORT2(int,sqlite3__wasm_posix_create_file, - (const char *zFilename, const unsigned char * pData, - int nData)){ +SQLITE_WASM_EXPORT +int sqlite3__wasm_posix_create_file( const char *zFilename, + const unsigned char * pData, + int nData ){ int rc; FILE * pFile = 0; int fileExisted = 0; @@ -1547,30 +1549,22 @@ SQLITE_WASM_EXPORT2(int,sqlite3__wasm_posix_create_file, ** This function is NOT part of the sqlite3 public API. It is strictly ** for use by the sqlite project's own JS/WASM bindings. ** -** This returns either a pointer to a static buffer or zKeyIn directly -** (if zClass is NULL or empty). +** Allocates sqlite3KvvfsMethods.nKeySize bytes from +** sqlite3__wasm_pstack_alloc() and returns 0 if that allocation fails, +** else it passes that string to kvstorageMakeKey() and returns a +** NUL-terminated pointer to that string. It is up to the caller to +** use sqlite3__wasm_pstack_restore() to free the returned pointer. */ -SQLITE_WASM_EXPORT2(const char *,sqlite3__wasm_kvvfsMakeKey, - (const char *zClass, const char *zKeyIn)){ - static char buf[SQLITE_KVOS_SZ+1] = {0}; +SQLITE_WASM_EXPORT +char * sqlite3__wasm_kvvfsMakeKeyOnPstack(const char *zClass, + const char *zKeyIn){ assert(sqlite3KvvfsMethods.nKeySize>24); - if( zClass && *zClass ){ - kvrecordMakeKey(zClass, zKeyIn, buf); - return buf; - }else{ -#if 1 - /* We can return zKeyIn here only because the JS API takes special - ** care with its lifetime.*/ - return zKeyIn; -#else - /* It would be nice to be able to return zKeyIn directly here, but - ** it may have been allocated as part of the automated JS-to-WASM - ** conversions, in which case it will be freed before reaching the - ** caller. */ - sqlite3_snprintf(KVRECORD_KEY_SZ, buf, "%s", zKeyIn); - return buf; -#endif + char *zKeyOut = + (char *)sqlite3__wasm_pstack_alloc(sqlite3KvvfsMethods.nKeySize); + if(zKeyOut){ + kvstorageMakeKey(zClass, zKeyIn, zKeyOut); } + return zKeyOut; } /* @@ -1580,7 +1574,8 @@ SQLITE_WASM_EXPORT2(const char *,sqlite3__wasm_kvvfsMakeKey, ** Returns the pointer to the singleton object which holds the kvvfs ** I/O methods and associated state. */ -SQLITE_WASM_EXPORT2(sqlite3_kvvfs_methods *,sqlite3__wasm_kvvfs_methods,(void)){ +SQLITE_WASM_EXPORT +sqlite3_kvvfs_methods * sqlite3__wasm_kvvfs_methods(void){ return &sqlite3KvvfsMethods; } @@ -1595,8 +1590,8 @@ SQLITE_WASM_EXPORT2(sqlite3_kvvfs_methods *,sqlite3__wasm_kvvfs_methods,(void)){ ** sqlite3_vtab_config(), or SQLITE_MISUSE if the 2nd arg is not a ** valid value. */ -SQLITE_WASM_EXPORT2(int,sqlite3__wasm_vtab_config, - (sqlite3 *pDb, int op, int arg)){ +SQLITE_WASM_EXPORT +int sqlite3__wasm_vtab_config(sqlite3 *pDb, int op, int arg){ switch(op){ case SQLITE_VTAB_DIRECTONLY: case SQLITE_VTAB_INNOCUOUS: @@ -1616,8 +1611,8 @@ SQLITE_WASM_EXPORT2(int,sqlite3__wasm_vtab_config, ** Wrapper for the variants of sqlite3_db_config() which take ** (int,int*) variadic args. */ -SQLITE_WASM_EXPORT2(int,sqlite3__wasm_db_config_ip, - (sqlite3 *pDb, int op, int arg1, int* pArg2)){ +SQLITE_WASM_EXPORT +int sqlite3__wasm_db_config_ip(sqlite3 *pDb, int op, int arg1, int* pArg2){ switch(op){ case SQLITE_DBCONFIG_ENABLE_FKEY: case SQLITE_DBCONFIG_ENABLE_TRIGGER: @@ -1652,9 +1647,8 @@ SQLITE_WASM_EXPORT2(int,sqlite3__wasm_db_config_ip, ** Wrapper for the variants of sqlite3_db_config() which take ** (void*,int,int) variadic args. */ -SQLITE_WASM_EXPORT2(int,sqlite3__wasm_db_config_pii, - (sqlite3 *pDb, int op, void * pArg1, int arg2, - int arg3)){ +SQLITE_WASM_EXPORT +int sqlite3__wasm_db_config_pii(sqlite3 *pDb, int op, void * pArg1, int arg2, int arg3){ switch(op){ case SQLITE_DBCONFIG_LOOKASIDE: return sqlite3_db_config(pDb, op, pArg1, arg2, arg3); @@ -1669,8 +1663,8 @@ SQLITE_WASM_EXPORT2(int,sqlite3__wasm_db_config_pii, ** Wrapper for the variants of sqlite3_db_config() which take ** (const char *) variadic args. */ -SQLITE_WASM_EXPORT2(int,sqlite3__wasm_db_config_s,(sqlite3 *pDb, int op, - const char *zArg)){ +SQLITE_WASM_EXPORT +int sqlite3__wasm_db_config_s(sqlite3 *pDb, int op, const char *zArg){ switch(op){ case SQLITE_DBCONFIG_MAINDBNAME: return sqlite3_db_config(pDb, op, zArg); @@ -1686,7 +1680,8 @@ SQLITE_WASM_EXPORT2(int,sqlite3__wasm_db_config_s,(sqlite3 *pDb, int op, ** Binding for combinations of sqlite3_config() arguments which take ** a single integer argument. */ -SQLITE_WASM_EXPORT2(int,sqlite3__wasm_config_i,(int op, int arg)){ +SQLITE_WASM_EXPORT +int sqlite3__wasm_config_i(int op, int arg){ return sqlite3_config(op, arg); } @@ -1697,7 +1692,8 @@ SQLITE_WASM_EXPORT2(int,sqlite3__wasm_config_i,(int op, int arg)){ ** Binding for combinations of sqlite3_config() arguments which take ** two int arguments. */ -SQLITE_WASM_EXPORT2(int,sqlite3__wasm_config_ii,(int op, int arg1, int arg2)){ +SQLITE_WASM_EXPORT +int sqlite3__wasm_config_ii(int op, int arg1, int arg2){ return sqlite3_config(op, arg1, arg2); } @@ -1708,7 +1704,8 @@ SQLITE_WASM_EXPORT2(int,sqlite3__wasm_config_ii,(int op, int arg1, int arg2)){ ** Binding for combinations of sqlite3_config() arguments which take ** a single i64 argument. */ -SQLITE_WASM_EXPORT2(int,sqlite3__wasm_config_j,(int op, sqlite3_int64 arg)){ +SQLITE_WASM_EXPORT +int sqlite3__wasm_config_j(int op, sqlite3_int64 arg){ return sqlite3_config(op, arg); } @@ -1720,7 +1717,8 @@ SQLITE_WASM_EXPORT2(int,sqlite3__wasm_config_j,(int op, sqlite3_int64 arg)){ ** sqlite3_mprintf()'s %Q modifier (if addQuotes is true) or %q (if ** addQuotes is 0). Returns NULL if z is NULL or on OOM. */ -SQLITE_WASM_EXPORT2(char *,sqlite3__wasm_qfmt_token,(char *z, int addQuotes)){ +SQLITE_WASM_EXPORT +char * sqlite3__wasm_qfmt_token(char *z, int addQuotes){ char * rc = 0; if( z ){ rc = addQuotes @@ -1730,21 +1728,6 @@ SQLITE_WASM_EXPORT2(char *,sqlite3__wasm_qfmt_token,(char *z, int addQuotes)){ return rc; } -/* -** This function is NOT part of the sqlite3 public API. It is strictly -** for use by the sqlite project's own JS/WASM bindings. -** -** A WASM wrapper for the interal os_kv.c:kvvfsDecode() for internal -** use by the kvvfs v2 API. -*/ -SQLITE_WASM_EXPORT2(int,sqlite3__wasm_kvvfs_decode,(const char *a, char *aOut, int nOut)){ - return kvvfsDecode(a, aOut, nOut); -} -SQLITE_WASM_EXPORT2(int,sqlite3__wasm_kvvfs_encode,(const char *a, int nA, char *aOut)){ - return kvvfsEncode(a, nA, aOut); -} - - #if defined(__EMSCRIPTEN__) && defined(SQLITE_ENABLE_WASMFS) #include <emscripten/console.h> #include <emscripten/wasmfs.h> @@ -1770,7 +1753,8 @@ SQLITE_WASM_EXPORT2(int,sqlite3__wasm_kvvfs_encode,(const char *a, int nA, char ** the virtual FS fails. In builds compiled without SQLITE_ENABLE_WASMFS ** defined, SQLITE_NOTFOUND is returned without side effects. */ -SQLITE_WASM_EXPORT2(int,sqlite3__wasm_init_wasmfs,(const char *zMountPoint)){ +SQLITE_WASM_EXPORT +int sqlite3__wasm_init_wasmfs(const char *zMountPoint){ static backend_t pOpfs = 0; if( !zMountPoint || !*zMountPoint ) zMountPoint = "/opfs"; if( !pOpfs ){ @@ -1789,7 +1773,8 @@ SQLITE_WASM_EXPORT2(int,sqlite3__wasm_init_wasmfs,(const char *zMountPoint)){ return pOpfs ? 0 : SQLITE_NOMEM; } #else -SQLITE_WASM_EXPORT2(int,sqlite3__wasm_init_wasmfs,(const char *zUnused)){ +SQLITE_WASM_EXPORT +int sqlite3__wasm_init_wasmfs(const char *zUnused){ //emscripten_console_warn("WASMFS OPFS is not compiled in."); (void)zUnused; return SQLITE_NOTFOUND; @@ -1798,43 +1783,52 @@ SQLITE_WASM_EXPORT2(int,sqlite3__wasm_init_wasmfs,(const char *zUnused)){ #if SQLITE_WASM_ENABLE_C_TESTS -SQLITE_WASM_EXPORT2(int,sqlite3__wasm_test_intptr,(int * p)){ +SQLITE_WASM_EXPORT +int sqlite3__wasm_test_intptr(int * p){ return *p = *p * 2; } -SQLITE_WASM_EXPORT2(void *,sqlite3__wasm_test_voidptr,(void * p)){ +SQLITE_WASM_EXPORT +void * sqlite3__wasm_test_voidptr(void * p){ return p; } -SQLITE_WASM_EXPORT2(int64_t,sqlite3__wasm_test_int64_max,(void)){ +SQLITE_WASM_EXPORT +int64_t sqlite3__wasm_test_int64_max(void){ return (int64_t)0x7fffffffffffffff; } -SQLITE_WASM_EXPORT2(int64_t,sqlite3__wasm_test_int64_min,(void)){ +SQLITE_WASM_EXPORT +int64_t sqlite3__wasm_test_int64_min(void){ return ~sqlite3__wasm_test_int64_max(); } -SQLITE_WASM_EXPORT2(int64_t,sqlite3__wasm_test_int64_times2,(int64_t x)){ +SQLITE_WASM_EXPORT +int64_t sqlite3__wasm_test_int64_times2(int64_t x){ return x * 2; } -SQLITE_WASM_EXPORT2(void,sqlite3__wasm_test_int64_minmax,(int64_t * min, int64_t *max)){ +SQLITE_WASM_EXPORT +void sqlite3__wasm_test_int64_minmax(int64_t * min, int64_t *max){ *max = sqlite3__wasm_test_int64_max(); *min = sqlite3__wasm_test_int64_min(); /*printf("minmax: min=%lld, max=%lld\n", *min, *max);*/ } -SQLITE_WASM_EXPORT2(int64_t,sqlite3__wasm_test_int64ptr,(int64_t * p)){ +SQLITE_WASM_EXPORT +int64_t sqlite3__wasm_test_int64ptr(int64_t * p){ /*printf("sqlite3__wasm_test_int64ptr( @%lld = 0x%llx )\n", (int64_t)p, *p);*/ return *p = *p * 2; } -SQLITE_WASM_EXPORT2(void,sqlite3__wasm_test_stack_overflow,(int recurse)){ +SQLITE_WASM_EXPORT +void sqlite3__wasm_test_stack_overflow(int recurse){ if(recurse) sqlite3__wasm_test_stack_overflow(recurse); } /* For testing the 'string:dealloc' whwasmutil.xWrap() conversion. */ -SQLITE_WASM_EXPORT2(char *,sqlite3__wasm_test_str_hello,(int fail)){ +SQLITE_WASM_EXPORT +char * sqlite3__wasm_test_str_hello(int fail){ char * s = fail ? 0 : (char *)sqlite3_malloc(6); if(s){ memcpy(s, "hello", 5); @@ -1948,8 +1942,8 @@ static int sqlite3__wasm_SQLTester_strnotglob(const char *zGlob, const char *z){ return *z==0; } -SQLITE_WASM_EXPORT2(int,sqlite3__wasm_SQLTester_strglob, - (const char *zGlob, const char *z)){ +SQLITE_WASM_EXPORT +int sqlite3__wasm_SQLTester_strglob(const char *zGlob, const char *z){ return !sqlite3__wasm_SQLTester_strnotglob(zGlob, z); } diff --git a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js index b282c5e6e..1a09bf9a6 100644 --- a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js +++ b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js @@ -19,12 +19,10 @@ slightly simpler client-side interface than the slightly-lower-level Worker API does. - In non-ESM builds this file necessarily exposes one global symbol, - but clients may freely `delete` that symbol after calling it. + This script necessarily exposes one global symbol, but clients may + freely `delete` that symbol after calling it. */ -//#if not defined target:es6-module 'use strict'; -//#endif /** Configures an sqlite3 Worker API #1 Worker such that it can be manipulated via a Promise-based interface and returns a factory @@ -111,12 +109,10 @@ the callback is called one time for each row of the result set, passed the same worker message format as the worker API emits: - { - type:typeString, + {type:typeString, row:VALUE, rowNumber:1-based-#, - columnNames: array - } + columnNames: array} Where `typeString` is an internally-synthesized message type string used temporarily for worker message dispatching. It can be ignored @@ -127,9 +123,10 @@ callback. At the end of the result set, the same event is fired with - (row=undefined, rowNumber=null) to indicate that the end of the - result set has been reached. The rows arrive via worker-posted - messages, with all the implications of that. + (row=undefined, rowNumber=null) to indicate that + the end of the result set has been reached. Note that the rows + arrive via worker-posted messages, with all the implications + of that. Notable shortcomings: @@ -260,9 +257,7 @@ globalThis.sqlite3Worker1Promiser.defaultConfig = { type: 'module' }); //#elif target:es6-module - return new Worker(new URL("sqlite3-worker1.mjs", import.meta.url),{ - type: 'module' - }); + return new Worker(new URL("sqlite3-worker1.js", import.meta.url)); //#else let theJs = "sqlite3-worker1.js"; if(this.currentScript){ @@ -286,7 +281,7 @@ globalThis.sqlite3Worker1Promiser.defaultConfig = { }) //#endif , - onerror: (...args)=>console.error('sqlite3Worker1Promiser():',...args) + onerror: (...args)=>console.error('worker1 promiser error',...args) }/*defaultConfig*/; /** @@ -348,7 +343,6 @@ globalThis.sqlite3Worker1Promiser.v2.defaultConfig = incompatibility. */ export default sqlite3Worker1Promiser.v2; -delete globalThis.sqlite3Worker1Promiser; //#endif /* target:es6-module */ //#else /* Built with the omit-oo1 flag. */ diff --git a/ext/wasm/api/sqlite3-worker1.c-pp.js b/ext/wasm/api/sqlite3-worker1.c-pp.js index db27c8fc0..036c4c6ea 100644 --- a/ext/wasm/api/sqlite3-worker1.c-pp.js +++ b/ext/wasm/api/sqlite3-worker1.c-pp.js @@ -33,9 +33,9 @@ directory from which `sqlite3.js` will be loaded. */ //#if target:es6-bundler-friendly -import sqlite3InitModule from './sqlite3-bundler-friendly.mjs'; +import {default as sqlite3InitModule} from './sqlite3-bundler-friendly.mjs'; //#elif target:es6-module -import sqlite3InitModule from './sqlite3.mjs'; + return new Worker(new URL("sqlite3.js", import.meta.url)); //#else "use strict"; { diff --git a/ext/wasm/c-pp-lite.c b/ext/wasm/c-pp-lite.c index b8d67f6a3..2120c457d 100644 --- a/ext/wasm/c-pp-lite.c +++ b/ext/wasm/c-pp-lite.c @@ -270,10 +270,9 @@ static void db_prepare(sqlite3_stmt **pStmt, const char * zSql, ...); /* ** Opens the given file and processes its contents as c-pp, sending ** all output to the global c-pp output channel. Fails fatally on -** error. If bRaw is true then the file's contents are passed through -** verbatim, rather than being preprocessed. +** error. */ -static void cmpp_process_file(const char * zName, int bRaw); +static void cmpp_process_file(const char * zName); /* ** Operator policy for cmpp_kvp_parse(). @@ -971,6 +970,7 @@ void db_prepare(sqlite3_stmt **pStmt, const char * zSql, ...){ char * z = 0; int n = 0; va_list va; + if(!str) fatal("sqlite3_str_new() failed"); va_start(va, zSql); sqlite3_str_vappendf(str, zSql, va); va_end(va); @@ -2213,23 +2213,12 @@ static void cmpp_kwd_endif(CmppKeyword const * pKw, CmppTokenizer *t){ static void cmpp_kwd_include(CmppKeyword const * pKw, CmppTokenizer *t){ char const * zFile; char * zResolved; - int bRaw = 0 /* -raw flag */; if(CT_skip(t)) return; - else if(t->args.argc<2 || t->args.argc>3){ - cmpp_kwd__err(pKw, t, "Expected args: ?-raw? filename"); - } - if(t->args.argc==2){ - zFile = (const char *)t->args.argv[1]; - }else{ - if( 0==strcmp("-raw", (char*)t->args.argv[1]) ){ - bRaw = 1; - }else{ - cmpp_kwd__err(pKw, t, "Unhandled argument: %s", - t->args.argv[1]); - } - zFile = (const char *)t->args.argv[2]; + else if(t->args.argc!=2){ + cmpp_kwd__err(pKw, t, "Expecting exactly 1 filename argument"); } - if(!bRaw && db_including_has(zFile)){ + zFile = (const char *)t->args.argv[1]; + if(db_including_has(zFile)){ /* Note that different spellings of the same filename ** will elude this check, but that seems okay, as different ** spellings means that we're not re-running the exact same @@ -2240,15 +2229,16 @@ static void cmpp_kwd_include(CmppKeyword const * pKw, CmppTokenizer *t){ } zResolved = db_include_search(zFile); if(zResolved){ - if( bRaw ) db_including_add(zFile, t->zName, t->token.lineNo); - cmpp_process_file(zResolved, bRaw); - if( bRaw ) db_include_rm(zFile); + db_including_add(zFile, t->zName, t->token.lineNo); + cmpp_process_file(zResolved); + db_include_rm(zFile); db_free(zResolved); }else{ cmpp_t__err(t, "file not found: %s", zFile); } } + static void cmpp_dump_defines( FILE * fp, int bIndent ){ sqlite3_stmt * const q = g_stmt(GStmt_defSelAll); while( SQLITE_ROW==sqlite3_step(q) ){ @@ -2402,7 +2392,7 @@ CmppKeyword aKeywords[] = { {S(Endif), 0, TT_Endif, cmpp_kwd_endif}, {S(Error), 0, TT_Error, cmpp_kwd_error}, {S(If), 1, TT_If, cmpp_kwd_if}, - {S(Include), 1, TT_Include, cmpp_kwd_include}, + {S(Include), 0, TT_Include, cmpp_kwd_include}, {S(Pragma), 1, TT_Pragma, cmpp_kwd_pragma}, {S(Savepoint), 1, TT_Savepoint, cmpp_kwd_savepoint}, {S(Stderr), 0, TT_Stderr, cmpp_kwd_stderr}, @@ -2454,25 +2444,14 @@ void cmpp_process_string(const char * zName, g.tok = oldTok; } -void cmpp_process_file(const char * zName, int bRaw){ +void cmpp_process_file(const char * zName){ FileWrapper fw = FileWrapper_empty; FileWrapper_open(&fw, zName, "r"); g_FileWrapper_link(&fw); FileWrapper_slurp(&fw); g_debug(1,("Read %u byte(s) from [%s]\n", fw.nContent, fw.zName)); if( fw.zContent ){ - if( bRaw ){ - FileWrapper fw = FileWrapper_empty; - FileWrapper_open(&fw, zName, "rb"); - g_FileWrapper_link(&fw); - FileWrapper_slurp(&fw); - if( 1!=fwrite(fw.zContent, fw.nContent, 1, g.out.pFile) ){ - fatal("fwrite() failed with errno %d\n", errno); - } - g_FileWrapper_close(&fw); - }else{ - cmpp_process_string(zName, fw.zContent, fw.nContent); - } + cmpp_process_string(zName, fw.zContent, fw.nContent); } g_FileWrapper_close(&fw); } @@ -2601,21 +2580,6 @@ static int arg_is_flag( char const *zFlag, char const *zArg, return 0; } -static void define_argv(int argc, char const * const * argv){ - sqlite3_str * const s = sqlite3_str_new(g.db); - sqlite3_str_append(s, "c-pp::argv=", 11); - for( int i = 0; i < argc; ++i ){ - if( i ) sqlite3_str_appendchar(s, 1, ' '); - sqlite3_str_appendf(s, "%s", argv[i]); - } - char * const z = sqlite3_str_finish(s); - assert(z); - if(z){ - db_define_add(z, NULL); - sqlite3_free(z); - } -} - int main(int argc, char const * const * argv){ int rc = 0; int inclCount = 0; @@ -2653,7 +2617,6 @@ int main(int argc, char const * const * argv){ g.sqlTrace.expandSql = expandMode; } cmpp_initdb(); - define_argv(argc, argv); } for(int i = 1; i < argc; ++i){ int negate = 0; @@ -2711,7 +2674,7 @@ int main(int argc, char const * const * argv){ DOIT { ++nFile; g_out_open; - cmpp_process_file(zVal, 0); + cmpp_process_file(zVal); } } ISFLAG("e"){ @@ -2794,7 +2757,7 @@ int main(int argc, char const * const * argv){ ++inclCount; } FileWrapper_open(&g.out, g.out.zName, "w"); - cmpp_process_file("-", 0); + cmpp_process_file("-"); } } } diff --git a/ext/wasm/common/SqliteTestUtil.js b/ext/wasm/common/SqliteTestUtil.js index a817b79f8..2c17824c5 100644 --- a/ext/wasm/common/SqliteTestUtil.js +++ b/ext/wasm/common/SqliteTestUtil.js @@ -44,8 +44,7 @@ /** abort() if expr is false. If expr is a function, it is called and its result is evaluated. */ - assert: function f(expr, ...msg){ - msg = msg?.join?.(' '); + assert: function f(expr, msg){ if(!f._){ f._ = ('undefined'===typeof abort ? (msg)=>{throw new Error(msg)} diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js index bca05a1ee..1c678f31f 100644 --- a/ext/wasm/common/whwasmutil.js +++ b/ext/wasm/common/whwasmutil.js @@ -16,35 +16,27 @@ More specifically: - https://fossil.wanderinghorse.net/r/jaccwabyt/dir/wasmutil + https://fossil.wanderinghorse.net/r/jaccwabyt/file/common/whwasmutil.js and SQLite: https://sqlite.org This file is kept in sync between both of those trees. - - This build was generated using: - - ./c-pp -o js/whwasmutil.js -@policy=error wasmutil/whwasmutil.c-pp.js - - by libcmpp 2.x 2fc4afc31f6505c27b9c34988973a2bd9b157d559247cdd26868ae75632c3a5e @ 2025-11-16 23:03:27.352 UTC */ /** - The primary goal of this function is to provide JS/WASM utility - code similar to some of that provided by Emscripten-generated - builds, the difference being that this one can be used in arbitrary - WASM environments built with toolchains other than Emscripten. To - that end, it populates the given object with various WASM-specific - APIs. These APIs work with both 32- and 64-bit WASM builds. + The primary goal of this function is to replace, where possible, + Emscripten-generated glue code with equivalent utility code which + can be used in arbitrary WASM environments built with toolchains + other than Emscripten. To that end, it populates the given object + with various WASM-specific APIs. These APIs work with both 32- and + 64-bit WASM builds. Forewarning: this API explicitly targets only browser environments. If a given non-browser environment has the capabilities needed for a given feature (e.g. TextEncoder), great, but it does not go out of its way to account for them and does not provide compatibility - crutches for them. That said: no specific incompatibilities with, - e.g., node.js are known (whereas it is known that some folks - use this with node.js). + crutches for them. Intended usage: @@ -225,9 +217,7 @@ newly-created (or config-provided) target. The current approach seemed better at the time. */ -'use strict'; -globalThis.WhWasmUtilInstaller = -function WhWasmUtilInstaller(target){ +globalThis.WhWasmUtilInstaller = function(target){ 'use strict'; if(undefined===target.bigIntEnabled){ target.bigIntEnabled = !!globalThis['BigInt64Array']; @@ -237,14 +227,6 @@ function WhWasmUtilInstaller(target){ all args with a space between each. */ const toss = (...args)=>{throw new Error(args.join(' '))}; - if( !target.pointerSize && !target.pointerIR - && target.alloc && target.dealloc ){ - /* Try to determine the pointer size by allocating. */ - const ptr = target.alloc(1); - target.pointerSize = ('bigint'===typeof ptr ? 8 : 4); - target.dealloc(ptr); - } - /** As of 2025-09-21, this library works with 64-bit WASM modules built with Emscripten's -sMEMORY64=1. @@ -677,14 +659,12 @@ function WhWasmUtilInstaller(target){ const ft = target.functionTable(); const oldLen = __asPtrType(ft.length); let ptr; - while( (ptr = cache.freeFuncIndexes.pop()) ){ - if(ft.get(ptr)){ - /* freeFuncIndexes's entry is stale. Table was modified via a - different API */ + while(cache.freeFuncIndexes.length){ + ptr = cache.freeFuncIndexes.pop(); + if(ft.get(ptr)){ /* Table was modified via a different API */ ptr = null; continue; }else{ - /* This index is free. We'll re-use it. */ break; } } @@ -775,10 +755,10 @@ function WhWasmUtilInstaller(target){ has no side effects and returns undefined. */ target.uninstallFunction = function(ptr){ - if(!ptr && __NullPtr!==ptr) return undefined; - + if(!ptr && 0!==ptr) return undefined; + const fi = cache.freeFuncIndexes; const ft = target.functionTable(); - cache.freeFuncIndexes.push(ptr); + fi.push(ptr); const rc = ft.get(ptr); ft.set(ptr, null); return rc; @@ -1016,12 +996,12 @@ function WhWasmUtilInstaller(target){ target.heap8u(). */ target.cstrlen = function(ptr){ - if(!ptr || !target.isPtr/*64*/(ptr)) return null; + if(!ptr || !target.isPtr(ptr)) return null; ptr = Number(ptr) /*tag:64bit*/; const h = heapWrappers().HEAP8U; let pos = ptr; for( ; h[pos] !== 0; ++pos ){} - return pos - ptr; + return Number(pos - ptr); }; /** Internal helper to use in operations which need to distinguish @@ -2472,8 +2452,7 @@ function WhWasmUtilInstaller(target){ - If `wasmUtilTarget.alloc` is not set and `instance.exports.malloc` is, it installs `wasmUtilTarget.alloc()` and `wasmUtilTarget.dealloc()` - wrappers for the exports' `malloc` and `free` functions - if exports.malloc exists. + wrappers for the exports `malloc` and `free` functions. It returns a function which, when called, initiates loading of the module and returns a Promise. When that Promise resolves, it calls @@ -2496,9 +2475,7 @@ function WhWasmUtilInstaller(target){ Error handling is up to the caller, who may attach a `catch()` call to the promise. */ -globalThis.WhWasmUtilInstaller -.yawl = function yawl(config){ - 'use strict'; +globalThis.WhWasmUtilInstaller.yawl = function(config){ const wfetch = ()=>fetch(config.uri, {credentials: 'same-origin'}); const wui = this; const finalThen = function(arg){ @@ -2523,7 +2500,7 @@ globalThis.WhWasmUtilInstaller tgt.alloc = function(n){ return exports.malloc(n) || toss("Allocation of",n,"bytes failed."); }; - tgt.dealloc = function(m){m && exports.free(m)}; + tgt.dealloc = function(m){exports.free(m)}; } wui(tgt); } @@ -2542,6 +2519,4 @@ globalThis.WhWasmUtilInstaller .then(finalThen) ; return loadWasm; -}.bind( -globalThis.WhWasmUtilInstaller -)/*yawl()*/; +}.bind(globalThis.WhWasmUtilInstaller)/*yawl()*/; diff --git a/ext/wasm/demo-jsstorage.js b/ext/wasm/demo-jsstorage.js index e3ab5a9e5..587aa9cc5 100644 --- a/ext/wasm/demo-jsstorage.js +++ b/ext/wasm/demo-jsstorage.js @@ -16,7 +16,7 @@ */ 'use strict'; (function(){ - const T = globalThis.SqliteTestUtil; + const T = self.SqliteTestUtil; const toss = function(...args){throw new Error(args.join(' '))}; const debug = console.debug.bind(console); const eOutput = document.querySelector('#test-output'); @@ -40,7 +40,7 @@ const error = function(...args){ logHtml('error',...args); }; - + const runTests = function(sqlite3){ const capi = sqlite3.capi, oo = sqlite3.oo1, @@ -51,7 +51,7 @@ error("This build is not kvvfs-capable."); return; } - + const dbStorage = 0 ? 'session' : 'local'; const theStore = 's'===dbStorage[0] ? sessionStorage : localStorage; const db = new oo.JsStorageDb( dbStorage ); @@ -108,7 +108,7 @@ } }; - sqlite3InitModule(globalThis.sqlite3TestModule).then((sqlite3)=>{ + sqlite3InitModule(self.sqlite3TestModule).then((sqlite3)=>{ runTests(sqlite3); }); })(); diff --git a/ext/wasm/demo-worker1-promiser.c-pp.html b/ext/wasm/demo-worker1-promiser.c-pp.html index a1005beb9..e0b487bdf 100644 --- a/ext/wasm/demo-worker1-promiser.c-pp.html +++ b/ext/wasm/demo-worker1-promiser.c-pp.html @@ -6,10 +6,10 @@ <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon"> <link rel="stylesheet" href="common/emscripten.css"/> <link rel="stylesheet" href="common/testing.css"/> -//#if target:es6-module - <title>Worker1-promiser (ESM) tests</title> +//#if target=es6-module + <title>worker-promise (via ESM) tests</title> //#else - <title>Worker1-promiser tests</title> + <title>worker-promise tests</title> //#endif </head> <body> @@ -32,7 +32,7 @@ <hr> <div id='test-output'></div> <script src="common/SqliteTestUtil.js"></script> -//#if target:es6-module +//#if target=es6-module <script src="demo-worker1-promiser.mjs" type="module"></script> //#else <script src="jswasm/sqlite3-worker1-promiser.js"></script> diff --git a/ext/wasm/demo-worker1.js b/ext/wasm/demo-worker1.js index 348741bf8..1a05cc7ac 100644 --- a/ext/wasm/demo-worker1.js +++ b/ext/wasm/demo-worker1.js @@ -18,7 +18,7 @@ */ 'use strict'; (function(){ - const T = globalThis.SqliteTestUtil; + const T = self.SqliteTestUtil; const SW = new Worker("jswasm/sqlite3-worker1.js"); const DbState = { id: undefined @@ -323,7 +323,7 @@ switch(ev.result){ case 'worker1-ready': log("Message:",ev); - globalThis.sqlite3TestModule.setStatus(null); + self.sqlite3TestModule.setStatus(null); runTests(); return; default: @@ -344,5 +344,5 @@ }; log("Init complete, but async init bits may still be running."); log("Installing Worker into global scope SW for dev purposes."); - globalThis.SW = SW; + self.SW = SW; })(); diff --git a/ext/wasm/fiddle/fiddle-worker.js b/ext/wasm/fiddle/fiddle-worker.js index 4b1ea2c53..a5f3e25b7 100644 --- a/ext/wasm/fiddle/fiddle-worker.js +++ b/ext/wasm/fiddle/fiddle-worker.js @@ -175,6 +175,10 @@ "features (e.g. upload) do not yet work with OPFS."); } stdout('\nEnter ".help" for usage hints.'); + this.exec([ // initialization commands... + '.nullvalue NULL', + '.headers on' + ].join('\n')); return true; }, /** diff --git a/ext/wasm/fiddle/index.c-pp.html b/ext/wasm/fiddle/index.html similarity index 95% rename from ext/wasm/fiddle/index.c-pp.html rename to ext/wasm/fiddle/index.html index 1f818286b..378cb3902 100644 --- a/ext/wasm/fiddle/index.c-pp.html +++ b/ext/wasm/fiddle/index.html @@ -5,29 +5,20 @@ <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> <title>SQLite3 Fiddle</title> <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon"> -//#if jqterm <!-- To add a terminal-style view using jquery.terminal[^1], uncomment the following two HTML lines and ensure that these files are on the web server. - jquery.terminal.bundle.min.js is a concatenation of jquery.min.js from + jquery-bundle.min.js is a concatenation of jquery.min.js from [^2] and jquery.terminal.min.js from [^1]. - jquery.terminal.min.css is from [^1]. Alterntely, jquery-VERSION.min.js - can be found in jquery.terminal's source tree. - - Fiddle automatically enables support for this if it's installed. - - In the canonical build process, if the JQTERM env var is set - to a dir containing a clone of [^1] then this block gets - enabled and a copy of [^1] is integrated automatically. + jquery.terminal.min.css is from [^1]. [^1]: https://github.com/jcubic/jquery.terminal [^2]: https://jquery.com --> - <script src="jqterm/jquery.terminal.bundle.min.js"></script> - <link rel="stylesheet" href="jqterm/jquery.terminal.min.css"> -//#endif + <!--script src="jqterm/jqterm-bundle.min.js"></script> + <link rel="stylesheet" href="jqterm/jquery.terminal.min.css"--> <style> /* The following styles are for app-level use. */ :root { diff --git a/ext/wasm/index.html b/ext/wasm/index.html index fd65dbc33..55e4cdb75 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -105,11 +105,6 @@ (<a href='speedtest1-worker.html?size=15'>32-bit</a>, <a href='speedtest1-worker-64bit.html?size=15'>64-bit</a>): an interactive Worker-thread variant of speedtest1.</li> - <li>speedtest1-worker?vfs=kvvfs - (<a href='speedtest1-worker.html?vfs=kvvfs&size=10'>32-bit</a>, - <a href='speedtest1-worker-64bit.html?vfs=kvvfs&size=10'>64-bit</a>): - speedtest1-worker with the - kvvfs VFS preselected and configured for a moderate workload.</li> <li>speedtest1-worker?vfs=opfs (<a href='speedtest1-worker.html?vfs=opfs&size=10'>32-bit</a>, <a href='speedtest1-worker-64bit.html?vfs=opfs&size=10'>64-bit</a>): diff --git a/ext/wasm/jaccwabyt/jaccwabyt.js b/ext/wasm/jaccwabyt/jaccwabyt.js index 6fa86c341..8ea08e213 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.js +++ b/ext/wasm/jaccwabyt/jaccwabyt.js @@ -16,17 +16,9 @@ Project homes: - https://fossil.wanderinghorse.net/r/jaccwabyt - https://sqlite.org/src/dir/ext/wasm/jaccwabyt - - This build was generated using: - - ./c-pp -o js/jaccwabyt.js -@policy=error jaccwabyt/jaccwabyt.c-pp.js - - by libcmpp 2.x 2fc4afc31f6505c27b9c34988973a2bd9b157d559247cdd26868ae75632c3a5e @ 2025-11-16 23:03:27.352 UTC */ 'use strict'; -globalThis.Jaccwabyt = -function StructBinderFactory(config){ - 'use strict'; +globalThis.Jaccwabyt = function StructBinderFactory(config){ /* ^^^^ it is recommended that clients move that object into wherever they'd like to have it and delete the globalThis-held copy. This API does not require the global reference - it is simply installed @@ -38,55 +30,35 @@ function StructBinderFactory(config){ all args with a space between each. */ const toss = (...args)=>{throw new Error(args.join(' '))}; - { - let h = config.heap; - if( h instanceof WebAssembly.Memory ){ - h = function(){return new Uint8Array(this.buffer)}.bind(h); - }else if( !(h instanceof Function) ){ - //console.warn("The bothersome StructBinderFactory config:",config); - toss("config.heap must be WebAssembly.Memory instance or", - "a function which returns one."); - } - config.heap = h; + if(!(config.heap instanceof WebAssembly.Memory) + && !(config.heap instanceof Function)){ + toss("config.heap must be WebAssembly.Memory instance or a function."); } ['alloc','dealloc'].forEach(function(k){ (config[k] instanceof Function) || toss("Config option '"+k+"' must be a function."); }); + const __heap = config.heap; const SBF = StructBinderFactory; - const heap = config.heap, + const heap = __heap ? __heap : ()=>new Uint8Array(__heap.buffer), alloc = config.alloc, dealloc = config.dealloc, - realloc = (config.realloc || function(){ - toss("This StructBinderFactory was configured without realloc()"); - /* We can't know the original memory's size from here unless - we internally proxy alloc()/dealloc() to track all - pointers (not going to happen), so we can't fall back to - doing alloc()/copy/dealloc(). */ - }), log = config.log || console.debug.bind(console), memberPrefix = (config.memberPrefix || ""), memberSuffix = (config.memberSuffix || ""), BigInt = globalThis['BigInt'], BigInt64Array = globalThis['BigInt64Array'], - bigIntEnabled = config.bigIntEnabled ?? !!BigInt64Array; - - //console.warn("config",config); - let ptr; - const ptrSize = config.pointerSize - || config.ptrSize/*deprecated*/ - || ('bigint'===typeof (ptr = alloc(1)) ? 8 : 4); - const ptrIR = config.pointerIR/*deprecated*/ + bigIntEnabled = config.bigIntEnabled ?? !!BigInt64Array, + ptrIR = config.pointerIR || config.ptrIR/*deprecated*/ - || (4===ptrSize ? 'i32' : 'i64'); - if( ptr ){ - dealloc(ptr); - ptr = undefined; - } - //console.warn("ptrIR =",ptrIR,"ptrSize =",ptrSize); + || 'i32', + /* Undocumented (on purpose) config options: */ + ptrSize = config.ptrSize/*deprecated*/ + || ('i32'===ptrIR ? 4 : 8) + ; - if(ptrSize!==4 && ptrSize!==8) toss("Invalid pointer size:",ptrSize); if(ptrIR!=='i32' && ptrIR!=='i64') toss("Invalid pointer representation:",ptrIR); + if(ptrSize!==4 && ptrSize!==8) toss("Invalid pointer size:",ptrSize); /** Either BigInt or, if !bigIntEnabled, a function which throws complaining that BigInt is not enabled. */ @@ -114,10 +86,6 @@ function StructBinderFactory(config){ return rc; }; - const __ptrAddSelf = function(...args){ - return __ptrAdd(this.pointer,...args); - }; - if(!SBF.debugFlags){ SBF.__makeDebugFlags = function(deriveFrom=null){ /* This is disgustingly overengineered. :/ */ @@ -147,12 +115,12 @@ function StructBinderFactory(config){ SBF.debugFlags = SBF.__makeDebugFlags(); }/*static init*/ - const isLittleEndian = true || (function() { + const isLittleEndian = (function() { const buffer = new ArrayBuffer(2); new DataView(buffer).setInt16(0, 256, true /* littleEndian */); // Int16Array uses the platform's endianness. return new Int16Array(buffer)[0] === 256; - })() /* WASM is, by definition, Little Endian */; + })(); /** Some terms used in the internal docs: @@ -168,14 +136,13 @@ function StructBinderFactory(config){ /** True if SIG s looks like a function signature, else false. */ const isFuncSig = (s)=>'('===s[1]; - /** True if SIG s is-a pointer-type signature. */ - const isPtrSig = (s)=>'p'===s || 'P'===s || 's'===s; + /** True if SIG s is-a pointer signature. */ + const isPtrSig = (s)=>'p'===s || 'P'===s; const isAutoPtrSig = (s)=>'P'===s /*EXPERIMENTAL*/; /** Returns p if SIG s is a function SIG, else returns s[0]. */ - const sigLetter = (s)=>s ? (isFuncSig(s) ? 'p' : s[0]) : undefined; - - /** Returns the WASM IR form of the letter at SIG s[0]. Throws for - an unknown SIG. */ + const sigLetter = (s)=>isFuncSig(s) ? 'p' : s[0]; + /** Returns the WASM IR form of the Emscripten-conventional letter + at SIG s[0]. Throws for an unknown SIG. */ const sigIR = function(s){ switch(sigLetter(s)){ case 'c': case 'C': return 'i8'; @@ -188,23 +155,8 @@ function StructBinderFactory(config){ toss("Unhandled signature IR:",s); }; - /** Returns the WASM sizeof of the letter at SIG s[0]. Throws for an - unknown SIG. */ - const sigSize = function(s){ - switch(sigLetter(s)){ - case 'c': case 'C': return 1; - case 'i': return 4; - case 'p': case 'P': case 's': return ptrSize; - case 'j': return 8; - case 'f': return 4; - case 'd': return 8; - } - toss("Unhandled signature sizeof:",s); - }; - const affirmBigIntArray = BigInt64Array ? ()=>true : ()=>toss('BigInt64Array is not available.'); - /** Returns the name of a DataView getter method corresponding to the given SIG. */ const sigDVGetter = function(s){ @@ -225,7 +177,6 @@ function StructBinderFactory(config){ } toss("Unhandled DataView getter for signature:",s); }; - /** Returns the name of a DataView setter method corresponding to the given SIG. */ const sigDVSetter = function(s){ @@ -277,44 +228,15 @@ function StructBinderFactory(config){ /** In order to completely hide StructBinder-bound struct pointers from JS code, we store them in a scope-local WeakMap which maps - the struct-bound objects to an object with their metadata: - - { - .p = the native pointer, - .o = self (for an eventual reverse-mapping), - .xb = extra bytes allocated for p, - .zod = zeroOnDispose, - .ownsPointer = true if this object owns p - } - - The .p data are accessible via obj.pointer, which is gated behind - a property interceptor, but are not exposed anywhere else in the - public API. + the struct-bound objects to their WASM pointers. The pointers are + accessible via boundObject.pointer, which is gated behind a + property interceptor, but are not exposed anywhere else in the + object. */ - const getInstanceHandle = function f(obj, create=true){ - let ii = f.map.get(obj); - if( !ii && create ){ - f.map.set(obj, (ii=f.create(obj))); - } - return ii; - }; - getInstanceHandle.map = new WeakMap; - getInstanceHandle.create = (forObj)=>{ - return Object.assign(Object.create(null),{ - o: forObj, - p: undefined/*native ptr*/, - ownsPointer: false, - zod: false/*zeroOnDispose*/, - xb: 0/*extraBytes*/ - }); - }; + const __instancePointerMap = new WeakMap(); - /** - Remove the getInstanceHandle() mapping for obj. - */ - const rmInstanceHandle = (obj)=>getInstanceHandle.map.delete(obj) - /* If/when we have a reverse map of ptr-to-objects, we need to - clean that here. */; + /** Property name for the pointer-is-external marker. */ + const xPtrPropName = '(pointer-is-external)'; const __isPtr32 = (ptr)=>('number'===typeof ptr && (ptr===(ptr|0)) && ptr>=0); const __isPtr64 = (ptr)=>( @@ -327,199 +249,83 @@ function StructBinderFactory(config){ */ const __isPtr = (4===ptrSize) ? __isPtr32 : __isPtr64; - const __isNonNullPtr = (v)=>__isPtr(v) && (v>0); - - /** Frees the obj.pointer memory (a.k.a. m), handles obj.ondispose, - and unmaps obj from its native resources. */ + /** Frees the obj.pointer memory and clears the pointer + property. */ const __freeStruct = function(ctor, obj, m){ - const ii = getInstanceHandle(obj, false); - if( !ii ) return; - rmInstanceHandle(obj); - if( !m && !(m = ii.p) ){ - console.warn("Cannot(?) happen: __freeStruct() found no instanceInfo"); - return; - } - if(Array.isArray(obj.ondispose)){ - let x; - while((x = obj.ondispose.pop())){ - try{ - if(x instanceof Function) x.call(obj); - else if(x instanceof StructType) x.dispose(); - else if(__isPtr(x)) dealloc(x); - // else ignore. Strings are permitted to annotate entries - // to assist in debugging. - }catch(e){ + if(!m) m = __instancePointerMap.get(obj); + if(m) { + __instancePointerMap.delete(obj); + if(Array.isArray(obj.ondispose)){ + let x; + while((x = obj.ondispose.shift())){ + try{ + if(x instanceof Function) x.call(obj); + else if(x instanceof StructType) x.dispose(); + else if(__isPtr(x)) dealloc(x); + // else ignore. Strings are permitted to annotate entries + // to assist in debugging. + }catch(e){ + console.warn("ondispose() for",ctor.structName,'@', + m,'threw. NOT propagating it.',e); + } + } + }else if(obj.ondispose instanceof Function){ + try{obj.ondispose()} + catch(e){ + /*do not rethrow: destructors must not throw*/ console.warn("ondispose() for",ctor.structName,'@', m,'threw. NOT propagating it.',e); } } - }else if(obj.ondispose instanceof Function){ - try{obj.ondispose()} - catch(e){ - /*do not rethrow: destructors must not throw*/ - console.warn("ondispose() for",ctor.structName,'@', - m,'threw. NOT propagating it.',e); - } - } - delete obj.ondispose; - if(ctor.debugFlags.__flags.dealloc){ - log("debug.dealloc:",(ii.ownsPointer?"":"EXTERNAL"), - ctor.structName,"instance:", - ctor.structInfo.sizeof,"bytes @"+m); - } - if(ii.ownsPointer){ - if( ii.zod || ctor.structInfo.zeroOnDispose ){ - heap().fill(0, Number(m), - Number(m) + ctor.structInfo.sizeof + ii.xb); + delete obj.ondispose; + if(ctor.debugFlags.__flags.dealloc){ + log("debug.dealloc:",(obj[xPtrPropName]?"EXTERNAL":""), + ctor.structName,"instance:", + ctor.structInfo.sizeof,"bytes @"+m); } - dealloc(m); + if(!obj[xPtrPropName]) dealloc(m); } }; - /** Returns a skeleton for a read-only, non-iterable property - * descriptor. */ - const rop0 = ()=>{return {configurable: false, writable: false, - iterable: false}}; - /** Returns a skeleton for a read-only property accessor wrapping value v. */ - const rop = (v)=>{return {...rop0(), value: v}}; + const rop = (v)=>{return {configurable: false, writable: false, + iterable: false, value: v}}; /** Allocates obj's memory buffer based on the size defined in ctor.structInfo.sizeof. */ - const __allocStruct = function f(ctor, obj, xm){ - let opt; - const checkPtr = (ptr)=>{ - __isNonNullPtr(ptr) || - toss("Invalid pointer value",arguments[0],"for",ctor.structName,"constructor."); - }; - if( arguments.length>=3 ){ - if( xm && ('object'===typeof xm) ){ - opt = xm; - xm = opt?.wrap; - }else{ - checkPtr(xm); - opt = {wrap: xm}; - } - }else{ - opt = {} - } - - const fill = !xm /* true if we need to zero the memory */; - let nAlloc = 0; - let ownsPointer = false; - if(xm){ - /* Externally-allocated memory. */ - checkPtr(xm); - ownsPointer = !!opt?.takeOwnership; - }else{ - const nX = opt?.extraBytes ?? 0; - if( nX<0 || (nX!==(nX|0)) ){ - toss("Invalid extraBytes value:",opt?.extraBytes); - } - nAlloc = ctor.structInfo.sizeof + nX; - xm = alloc(nAlloc) - || toss("Allocation of",ctor.structName,"structure failed."); - ownsPointer = true; + const __allocStruct = function(ctor, obj, m){ + let fill = !m; + if(m) Object.defineProperty(obj, xPtrPropName, rop(m)); + else{ + m = alloc(ctor.structInfo.sizeof); + if(!m) toss("Allocation of",ctor.structName,"structure failed."); } try { - if( opt?.debugFlags ){ - /* specifically undocumented */ - obj.debugFlags(opt.debugFlags); - } - if(ctor./*prototype.???*/debugFlags.__flags.alloc){ + if(ctor.debugFlags.__flags.alloc){ log("debug.alloc:",(fill?"":"EXTERNAL"), ctor.structName,"instance:", - ctor.structInfo.sizeof,"bytes @"+xm); + ctor.structInfo.sizeof,"bytes @"+m); } if(fill){ - heap().fill(0, Number(xm), Number(xm) + nAlloc); - } - const ii = getInstanceHandle(obj); - ii.p = xm; - ii.ownsPointer = ownsPointer; - ii.xb = nAlloc ? (nAlloc-ctor.structInfo.sizeof) : 0; - ii.zod = !!opt?.zeroOnDispose; - if( opt?.ondispose && opt.ondispose!==xm ){ - obj.addOnDispose( opt.ondispose ); + heap().fill(0, Number(m), Number(m) + ctor.structInfo.sizeof); } + __instancePointerMap.set(obj, m); }catch(e){ - __freeStruct(ctor, obj, xm); + __freeStruct(ctor, obj, m); throw e; } }; - - /** True if sig looks like an emscripten/jaccwabyt - type signature, else false. */ - const looksLikeASig = function f(sig){ - f.rxSig1 ??= /^[ipPsjfdcC]$/; - f.rxSig2 ??= /^[vipPsjfdcC]\([ipPsjfdcC]*\)$/; - return f.rxSig1.test(sig) || f.rxSig2.test(sig); - }; - - /** Returns a pair of adaptor maps (objects) in a length-3 - array specific to the given object. */ - const __adaptorsFor = function(who){ - let x = this.get(who); - if( !x ){ - x = [ Object.create(null), Object.create(null), Object.create(null) ]; - this.set(who, x); - } - return x; - }.bind(new WeakMap); - - /** Code de-duplifier for __adaptGet(), __adaptSet(), and - __adaptStruct(). */ - const __adaptor = function(who, which, key, proxy){ - const a = __adaptorsFor(who)[which]; - if(3===arguments.length) return a[key]; - if( proxy ) return a[key] = proxy; - return delete a[key]; - }; - - const noopAdapter = (x)=>x; - - // StructBinder::adaptGet() - const __adaptGet = function(key, ...args){ - return __adaptor(this, 0, key, ...args); - }; - - const __affirmNotASig = function(ctx,key){ - looksLikeASig(key) && - toss(ctx,"(",key,") collides with a data type signature."); - }; - - // StructBinder::adaptSet() - const __adaptSet = function(key, ...args){ - __affirmNotASig('Setter adaptor',key); - return __adaptor(this, 1, key, ...args); - }; - - // StructBinder::adaptStruct() - const __adaptStruct = function(key, ...args){ - __affirmNotASig('Struct adaptor',key); - return __adaptor(this, 2, key, ...args); - }; - - /** - An internal counterpart of __adaptStruct(). If key is-a string, - uses __adaptor(who) to fetch the struct-adaptor entry for key, - else key is assumed to be a struct description object. If it - resolves to an object, that's returned, else an exception is - thrown. - */ - const __adaptStruct2 = function(who,key){ - const si = ('string'===typeof key) - ? __adaptor(who, 2, key) : key; - if( 'object'!==typeof si ){ - toss("Invalid struct mapping object. Arg =",key,JSON.stringify(si)); - } - return si; + /** Gets installed as the memoryDump() method of all structs. */ + const __memoryDump = function(){ + const p = this.pointer; + return p + ? new Uint8Array(heap().slice(Number(p), Number(p) + this.structInfo.sizeof)) + : null; }; const __memberKey = (k)=>memberPrefix + k + memberSuffix; const __memberKeyProp = rop(__memberKey); - //const __adaptGetProp = rop(__adaptGet); /** Looks up a struct member in structInfo.members. Throws if found @@ -536,8 +342,7 @@ function StructBinderFactory(config){ if(v.key===memberName){ m = v; break; } } if(!m && tossIfNotFound){ - toss(sPropName(structInfo.name || structInfo.structName, memberName), - 'is not a mapped struct member.'); + toss(sPropName(structInfo.name,memberName),'is not a mapped struct member.'); } } return m; @@ -554,6 +359,15 @@ function StructBinderFactory(config){ return emscriptenFormat ? f._(m.signature) : m.signature; }; + const __ptrPropDescriptor = { + configurable: false, enumerable: false, + get: function(){return __instancePointerMap.get(this)}, + set: ()=>toss("Cannot assign the 'pointer' property of a struct.") + // Reminder: leaving `set` undefined makes assignments + // to the property _silently_ do nothing. Current unit tests + // rely on it throwing, though. + }; + /** Impl of X.memberKeys() for StructType and struct ctors. */ const __structMemberKeys = rop(function(){ const a = []; @@ -687,13 +501,13 @@ function StructBinderFactory(config){ Prototype for all StructFactory instances (the constructors returned from StructBinder). */ - const StructType = function StructType(structName, structInfo){ - if(arguments[2]!==rop/*internal sentinel value*/){ + const StructType = function ctor(structName, structInfo){ + if(arguments[2]!==rop){ toss("Do not call the StructType constructor", "from client-level code."); } Object.defineProperties(this,{ - //isA: rop((v)=>v instanceof StructType), + //isA: rop((v)=>v instanceof ctor), structName: rop(structName), structInfo: rop(structInfo) }); @@ -719,31 +533,8 @@ function StructBinderFactory(config){ memberSignature: rop(function(memberName, emscriptenFormat=false){ return __memberSignature(this, memberName, emscriptenFormat); }), - memoryDump: rop(function(){ - const p = this.pointer; - return p - ? new Uint8Array(heap().slice(Number(p), Number(p) + this.structInfo.sizeof)) - : null; - }), - extraBytes: { - configurable: false, enumerable: false, - get: function(){return getInstanceHandle(this, false)?.xb ?? 0;} - }, - zeroOnDispose: { - configurable: false, enumerable: false, - get: function(){ - return getInstanceHandle(this, false)?.zod - ?? !!this.structInfo.zeroOnDispose; - } - }, - pointer: { - configurable: false, enumerable: false, - get: function(){return getInstanceHandle(this, false)?.p}, - set: ()=>toss("Cannot assign the 'pointer' property of a struct.") - // Reminder: leaving `set` undefined makes assignments - // to the property _silently_ do nothing. Current unit tests - // rely on it throwing, though. - }, + memoryDump: rop(__memoryDump), + pointer: __ptrPropDescriptor, setMemberCString: rop(function(memberName, str){ return __setMemberCString(this, memberName, str); }) @@ -762,435 +553,183 @@ function StructBinderFactory(config){ Object.defineProperties(StructType, { allocCString: rop(__allocCString), isA: rop((v)=>v instanceof StructType), - hasExternalPointer: rop((v)=>{ - const ii = getInstanceHandle(v, false); - return !!(ii?.p && !ii?.ownsPointer); - }), + hasExternalPointer: rop((v)=>(v instanceof StructType) && !!v[xPtrPropName]), memberKey: __memberKeyProp - //ptrAdd = rop(__ptrAdd) no b/c one might think that it adds based on this.pointer. }); /** - If struct description object si has a getter proxy, return it (a - function), else return undefined. - */ - const memberGetterProxy = function(si){ - return si.get || (si.adaptGet - ? StructBinder.adaptGet(si.adaptGet) - : undefined); - }; - - /** - If struct description object si has a setter proxy, return it (a - function), else return undefined. - */ - const memberSetterProxy = function(si){ - return si.set || (si.adaptSet - ? StructBinder.adaptSet(si.adaptSet) - : undefined); - }; - - /** - To be called by makeMemberWrapper() when si has a 'members' - member, i.e. is an embedded struct. This function sets up that - struct like any other and also sets up property accessor for - ctor.memberKey(name) which returns an instance of that new - StructType when the member is accessed. That instance wraps the - memory of the member's part of the containing C struct instance. - - That is, if struct Foo has member bar which is an inner struct - then: - - const f = new Foo; - const b = f.bar; - assert( b is-a StructType object ); - assert( b.pointer === f.b.pointer ); - - b will be disposed of when f() is. Calling b.dispose() will not - do any permanent harm, as the wrapper object will be recreated - when accessing f.bar, pointing to the same memory in f. - - The si.zeroOnDispose flag has no effect on embedded structs because - they wrap "external" memory, so do not own it, and are thus never - freed, as such. + Pass this a StructBinder-generated prototype, and the struct + member description object. It will define property accessors for + proto[memberKey] which read from/write to memory in + this.pointer. It modifies descr to make certain downstream + operations much simpler. */ - const makeMemberStructWrapper = function callee(ctor, name, si){ - /** - Where we store inner-struct member proxies. Keys are a - combination of the parent object's pointer address and the - property's name. The values are StructType instances. - */ - const __innerStructs = (callee.innerStructs ??= new Map()); - const key = ctor.memberKey(name); - if( undefined!==si.signature ){ - toss("'signature' cannot be used on an embedded struct (", - ctor.structName,".",key,")."); - } - if( memberSetterProxy(si) ){ - toss("'set' and 'adaptSet' are not permitted for nested struct members."); - } - //console.warn("si =",ctor.structName, name, JSON.stringify(si,' ')); - si.structName ??= ctor.structName+'::'+name; - si.key = key; - si.name = name; - si.constructor = this.call(this, si.structName, si); - //console.warn("si.constructor =",si.constructor); - //console.warn("si =",si,"ctor=",ctor); - const getterProxy = memberGetterProxy(si); - const prop = Object.assign(Object.create(null),{ - configurable: false, - enumerable: false, - set: __propThrowOnSet(ctor/*not si.constructor*/.structName, key), - get: function(){ - const dbg = this.debugFlags.__flags; - const p = this.pointer; - const k = p+'.'+key; - let s = __innerStructs.get(k); - if(dbg.getter){ log("debug.getter: k =",k); } - if( !s ){ - s = new si.constructor(__ptrAdd(p, si.offset)); - __innerStructs.set(k, s); - this.addOnDispose(()=>s.dispose()); - s.addOnDispose(()=>__innerStructs.delete(k)); - //console.warn("Created",k,"proxy"); - } - if(getterProxy) s = getterProxy.apply(this,[s,key]); - if(dbg.getter) log("debug.getter: result =",s); - return s; - } - }); - Object.defineProperty(ctor.prototype, key, prop); - }/*makeMemberStructWrapper()*/; - - /** - This is where most of the magic happens. - - Pass this a StructBinderImpl-generated constructor, a member - property name, and the struct member description object. It will - define property accessors for proto[memberKey] which read - from/write to memory in this.pointer. It modifies si to make - certain downstream operations simpler. - */ - const makeMemberWrapper = function f(ctor, name, si){ - si = __adaptStruct2(this, si); - if( si.members ){ - return makeMemberStructWrapper.call(this, ctor, name, si); - } - - if(!f.cache){ - /* Cache all available getters/setters/set-wrappers for - direct reuse in each accessor function. */ - f.cache = {getters: {}, setters: {}, sw:{}}; + const makeMemberWrapper = function f(ctor,name, descr){ + if(!f._){ + /*cache all available getters/setters/set-wrappers for + direct reuse in each accessor function. */ + f._ = {getters: {}, setters: {}, sw:{}}; const a = ['i','c','C','p','P','s','f','d','v()']; if(bigIntEnabled) a.push('j'); a.forEach(function(v){ - f.cache.getters[v] = sigDVGetter(v) /* DataView[MethodName] values for GETTERS */; - f.cache.setters[v] = sigDVSetter(v) /* DataView[MethodName] values for SETTERS */; - f.cache.sw[v] = sigDVSetWrapper(v) /* BigInt or Number ctor to wrap around values + //const ir = sigIR(v); + f._.getters[v] = sigDVGetter(v) /* DataView[MethodName] values for GETTERS */; + f._.setters[v] = sigDVSetter(v) /* DataView[MethodName] values for SETTERS */; + f._.sw[v] = sigDVSetWrapper(v) /* BigInt or Number ctor to wrap around values for conversion */; }); + const rxSig1 = /^[ipPsjfdcC]$/, + rxSig2 = /^[vipPsjfdcC]\([ipPsjfdcC]*\)$/; f.sigCheck = function(obj, name, key,sig){ if(Object.prototype.hasOwnProperty.call(obj, key)){ toss(obj.structName,'already has a property named',key+'.'); } - looksLikeASig(sig) + rxSig1.test(sig) || rxSig2.test(sig) || toss("Malformed signature for", sPropName(obj.structName,name)+":",sig); }; } const key = ctor.memberKey(name); - f.sigCheck(ctor.prototype, name, key, si.signature); - si.key = key; - si.name = name; - const sigGlyph = sigLetter(si.signature); - const xPropName = sPropName(ctor.structName,key); - const dbg = ctor.debugFlags.__flags; + f.sigCheck(ctor.prototype, name, key, descr.signature); + descr.key = key; + descr.name = name; + const sigGlyph = sigLetter(descr.signature); + const xPropName = sPropName(ctor.prototype.structName,key); + const dbg = ctor.prototype.debugFlags.__flags; /* - TODO?: set prototype of si to an object which can set/fetch + TODO?: set prototype of descr to an object which can set/fetch its preferred representation, e.g. conversion to string or mapped function. Advantage: we can avoid doing that via if/else if/else in the get/set methods. */ - const getterProxy = memberGetterProxy(si); const prop = Object.create(null); prop.configurable = false; prop.enumerable = false; prop.get = function(){ - /** - This getter proxy reads its value from the appropriate pointer - address in the heap. It knows where and how much to read based on - this.pointer, si.offset, and si.sizeof. - */ if(dbg.getter){ - log("debug.getter:",f.cache.getters[sigGlyph],"for", sigIR(sigGlyph), - xPropName,'@', this.pointer,'+',si.offset,'sz',si.sizeof); + log("debug.getter:",f._.getters[sigGlyph],"for", sigIR(sigGlyph), + xPropName,'@', this.pointer,'+',descr.offset,'sz',descr.sizeof); } let rc = ( - new DataView(heap().buffer, Number(this.pointer) + si.offset, si.sizeof) - )[f.cache.getters[sigGlyph]](0, isLittleEndian); - - if(getterProxy) rc = getterProxy.apply(this,[key,rc]); + new DataView(heap().buffer, Number(this.pointer) + descr.offset, descr.sizeof) + )[f._.getters[sigGlyph]](0, isLittleEndian); if(dbg.getter) log("debug.getter:",xPropName,"result =",rc); return rc; }; - if(si.readOnly){ + if(descr.readOnly){ prop.set = __propThrowOnSet(ctor.prototype.structName,key); }else{ - const setterProxy = memberSetterProxy(si); prop.set = function(v){ - /** - The converse of prop.get(), this encodes v into the appropriate - spot in the WASM heap. - */ if(dbg.setter){ - log("debug.setter:",f.cache.setters[sigGlyph],"for", sigIR(sigGlyph), - xPropName,'@', this.pointer,'+',si.offset,'sz',si.sizeof, v); + log("debug.setter:",f._.setters[sigGlyph],"for", sigIR(sigGlyph), + xPropName,'@', this.pointer,'+',descr.offset,'sz',descr.sizeof, v); } if(!this.pointer){ - toss("Cannot set native property on a disposed", - this.structName,"instance."); + toss("Cannot set struct property on disposed instance."); } - if( setterProxy ) v = setterProxy.apply(this,[key,v]); - if( null===v || undefined===v ) v = __NullPtr; - else if( isPtrSig(si.signature) && !__isPtr(v) ){ - if(isAutoPtrSig(si.signature) && (v instanceof StructType)){ - // It's a struct instance: store its pointer value + if(null===v) v = __NullPtr; + else while(!__isPtr(v)){ + if(isAutoPtrSig(descr.signature) && (v instanceof StructType)){ + // It's a struct instance: let's store its pointer value! v = v.pointer || __NullPtr; if(dbg.setter) log("debug.setter:",xPropName,"resolved to",v); - }else{ - toss("Invalid value for pointer-type",xPropName+'.'); + break; } + toss("Invalid value for pointer-type",xPropName+'.'); } ( - new DataView(heap().buffer, Number(this.pointer) + si.offset, - si.sizeof) - )[f.cache.setters[sigGlyph]](0, f.cache.sw[sigGlyph](v), isLittleEndian); + new DataView(heap().buffer, Number(this.pointer) + descr.offset, + descr.sizeof) + )[f._.setters[sigGlyph]](0, f._.sw[sigGlyph](v), isLittleEndian); }; } Object.defineProperty(ctor.prototype, key, prop); - }/*makeMemberWrapper()*/; + }/*makeMemberWrapper*/; /** The main factory function which will be returned to the - caller. The third argument is structly for internal use. - - This level of indirection is to avoid that clients can pass a - third argument to this, as that's only for internal use. - - internalOpt options: - - - None right now. This is for potential use in recursion. - - Usages: - - StructBinder(string, obj [,optObj]); - StructBinder(obj); + caller. */ - const StructBinderImpl = function StructBinderImpl( - structName, si, opt = Object.create(null) - ){ - /** - StructCtor is the eventual return value of this function. We - need to populate this early on so that we can do some trickery - in feeding it through recursion. - - Uses: - - // heap-allocated: - const x = new StructCtor; - // externally-managed memory: - const y = new StructCtor( aPtrToACompatibleCStruct ); - - or, more recently: - - const z = new StructCtor({ - extraBytes: [int=0] extra bytes to allocate after the struct - - wrap: [aPtrToACompatibleCStruct=undefined]. If provided, this - instance waps, but does not (by default) own the memory, else - a new instance is allocated from the WASM heap. - - ownsPointer: true if this object takes over ownership of - wrap. - - zeroOnDispose: [bool=StructCtor.structInfo.zeroOnDispose] - - autoCalcSizeOffset: [bool=false] Automatically calculate - sizeof an offset. This is fine for pure-JS structs (which - probably aren't useful beyond testing of Jaccwabyt) but it's - dangerous to use with actual WASM objects because we cannot - be guaranteed to have the same memory layout as an ostensibly - matching C struct. This applies recursively to all children - of the struct description. - - // TODO? Per-instance overrides of the struct-level flags? - - get: (k,v)=>v, - set: (k,v)=>v, - adaptGet: string, - adaptSet: string - - // That wouldn't fit really well right now, apparently. - }); - - */ - const StructCtor = function StructCtor(arg){ - //console.warn("opt",opt,arguments[0]); - if(!(this instanceof StructCtor)){ - toss("The",structName,"constructor may only be called via 'new'."); - } - __allocStruct(StructCtor, this, ...arguments); - }; - const self = this; - /** - "Convert" struct description x to a struct description, if - needed. This expands adaptStruct() mappings and transforms - {memberName:signatureString} signature syntax to object form. - */ - const ads = (x)=>{ - //console.warn("looksLikeASig(",x,") =",looksLikeASig(x)); - return (('string'===typeof x) && looksLikeASig(x)) - ? {signature: x} : __adaptStruct2(self,x); - }; + const StructBinder = function StructBinder(structName, structInfo){ if(1===arguments.length){ - si = ads(structName); - structName = si.structName || si.name; - }else if(2===arguments.length){ - si = ads(si); - si.name ??= structName; - }else{ - si = ads(si); + structInfo = structName; + structName = structInfo.name; + }else if(!structInfo.name){ + structInfo.name = structName; } - structName ??= si.structName; - //console.warn("arguments =",JSON.stringify(arguments)); - structName ??= opt.structName; - if( !structName ) toss("One of 'name' or 'structName' are required."); - if( si.adapt ){ - /* Install adaptGet(), adaptSet(), and adaptStruct() proxies. */ - Object.keys(si.adapt.struct||{}).forEach((k)=>{ - __adaptStruct.call(StructBinderImpl, k, si.adapt.struct[k]); - }); - Object.keys(si.adapt.set||{}).forEach((k)=>{ - __adaptSet.call(StructBinderImpl, k, si.adapt.set[k]); - }); - Object.keys(si.adapt.get||{}).forEach((k)=>{ - __adaptGet.call(StructBinderImpl, k, si.adapt.get[k]); - }); - } - if(!si.members && !si.sizeof){ - si.sizeof = sigSize(si.signature); + if(!structName) toss("Struct name is required."); + let lastMember = false; + Object.keys(structInfo.members).forEach((k)=>{ + // Sanity checks of sizeof/offset info... + const m = structInfo.members[k]; + if(!m.sizeof) toss(structName,"member",k,"is missing sizeof."); + else if(m.sizeof===1){ + (m.signature === 'c' || m.signature === 'C') || + toss("Unexpected sizeof==1 member", + sPropName(structInfo.name,k), + "with signature",m.signature); + }else{ + // sizes and offsets of size-1 members may be odd values, but + // others may not. + if(0!==(m.sizeof%4)){ + console.warn("Invalid struct member description =",m,"from",structInfo); + toss(structName,"member",k,"sizeof is not aligned. sizeof="+m.sizeof); + } + if(0!==(m.offset%4)){ + console.warn("Invalid struct member description =",m,"from",structInfo); + toss(structName,"member",k,"offset is not aligned. offset="+m.offset); + } + } + if(!lastMember || lastMember.offset < m.offset) lastMember = m; + }); + if(!lastMember) toss("No member property descriptions found."); + else if(structInfo.sizeof < lastMember.offset+lastMember.sizeof){ + toss("Invalid struct config:",structName, + "max member offset ("+lastMember.offset+") ", + "extends past end of struct (sizeof="+structInfo.sizeof+")."); } - const debugFlags = rop(SBF.__makeDebugFlags(StructBinder.debugFlags)); + /** Constructor for the StructCtor. */ + const zeroAsPtr = __asPtrType(0); + const StructCtor = function StructCtor(externalMemory){ + externalMemory = __asPtrType(externalMemory); + //console.warn("externalMemory",externalMemory,arguments[0]); + if(!(this instanceof StructCtor)){ + toss("The",structName,"constructor may only be called via 'new'."); + }else if(arguments.length){ + if(Number.isNaN(externalMemory) || externalMemory<=zeroAsPtr){ + toss("Invalid pointer value",arguments[0],"for",structName,"constructor."); + } + __allocStruct(StructCtor, this, externalMemory); + }else{ + __allocStruct(StructCtor, this); + } + }; Object.defineProperties(StructCtor,{ debugFlags: debugFlags, isA: rop((v)=>v instanceof StructCtor), memberKey: __memberKeyProp, memberKeys: __structMemberKeys, - //methodInfoForKey: rop(function(mKey){/*???*/}), - structInfo: rop(si), - structName: rop(structName), - ptrAdd: rop(__ptrAdd) + methodInfoForKey: rop(function(mKey){ + }), + structInfo: rop(structInfo), + structName: rop(structName) }); - StructCtor.prototype = new StructType(structName, si, rop); + StructCtor.prototype = new StructType(structName, structInfo, rop); Object.defineProperties(StructCtor.prototype,{ debugFlags: debugFlags, constructor: rop(StructCtor) /*if we assign StructCtor.prototype and don't do - this then StructCtor!==instance.constructor*/, - ptrAdd: rop(__ptrAddSelf) + this then StructCtor!==instance.constructor!*/ }); - let lastMember = false; - let offset = 0; - const autoCalc = !!si.autoCalcSizeOffset; - //console.warn(structName,"si =",si); - if( !autoCalc ){ - if( !si.sizeof ){ - toss(structName,"description is missing its sizeof property."); - } - /*if( undefined===si.offset ){ - toss(structName,"description is missing its offset property."); - }*/ - si.offset ??= 0; - }else{ - si.offset ??= 0; - } - Object.keys(si.members || {}).forEach((k)=>{ - // Sanity checks of sizeof/offset info... - let m = ads(si.members[k]); - if(!m.members && !m.sizeof){ - /* ^^^^ fixme: we need to recursively collect all sizeofs - before updating that. */ - m.sizeof = sigSize(m.signature); - if(!m.sizeof){ - toss(sPropName(structName,k), "is missing a sizeof property.",m); - } - } - if( undefined===m.offset ){ - if( autoCalc ) m.offset = offset; - else{ - toss(sPropName(structName,k),"is missing its offset.", - JSON.stringify(m)); - } - /* A missing offset on the initial child is okay (it's always - zero), but we don't know for sure that the members are - their natural order, so we don't know, at this point, which - one is "first". */ - } - si.members[k] = m /* in case ads() resolved it to something else */; - if(!lastMember || lastMember.offset < m.offset) lastMember = m; - const oldAutoCalc = !!m.autoCalc; - if( autoCalc ) m.autoCalcSizeOffset = true; - makeMemberWrapper.call(self, StructCtor, k, m); - if( oldAutoCalc ) m.autoCalcSizeOffset = true; - else delete m.autoCalcSizeOffset; - offset += m.sizeof; - //console.warn("offset",sPropName(structName,k),offset); - }); - - if( !lastMember ) toss("No member property descriptions found."); - if( !si.sizeof ) si.sizeof = offset; - if(si.sizeof===1){ - (si.signature === 'c' || si.signature === 'C') || - toss("Unexpected sizeof==1 member", - sPropName(structName,k), - "with signature",si.signature); - }else{ - // sizes and offsets of size-1 members may be odd values, but - // others may not. - if(0!==(si.sizeof%4)){ - console.warn("Invalid struct member description",si); - toss(structName,"sizeof is not aligned. sizeof="+si.sizeof); - } - if(0!==(si.offset%4)){ - console.warn("Invalid struct member description",si); - toss(structName,"offset is not aligned. offset="+si.offset); - } - } - if( si.sizeof < offset ){ - console.warn("Suspect struct description:",si,"offset =",offset); - toss("Mismatch in the calculated vs. the provided sizeof/offset info.", - "Expected sizeof",offset,"but got",si.sizeof,"for",si); - /* It is legal for the native struct to be larger, so long as - we're pointing to all the right offsets for the members - exposed here. */ - } - delete si.autoCalcSizeOffset; + Object.keys(structInfo.members).forEach( + (name)=>makeMemberWrapper(StructCtor, name, structInfo.members[name]) + ); return StructCtor; - }/*StructBinderImpl*/; - - const StructBinder = function StructBinder(structName, structInfo){ - return (1==arguments.length) - ? StructBinderImpl.call(StructBinder, structName) - : StructBinderImpl.call(StructBinder, structName, structInfo); }; StructBinder.StructType = StructType; StructBinder.config = config; StructBinder.allocCString = __allocCString; - StructBinder.adaptGet = __adaptGet; - StructBinder.adaptSet = __adaptSet; - StructBinder.adaptStruct = __adaptStruct; - StructBinder.ptrAdd = __ptrAdd; if(!StructBinder.debugFlags){ StructBinder.debugFlags = SBF.__makeDebugFlags(SBF.debugFlags); } diff --git a/ext/wasm/jaccwabyt/jaccwabyt.md b/ext/wasm/jaccwabyt/jaccwabyt.md index 5c30268e8..5ec3151d5 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.md +++ b/ext/wasm/jaccwabyt/jaccwabyt.md @@ -12,35 +12,6 @@ friction. (If that means nothing to you, neither will the rest of this page!) -To the best of its creator's fallible knowledge, Jaccwabyt is the only -library of its kind (as of 2025-11). Aside from wrapping existing -structs, e.g. to integrate "legacy" C code into JS/WASM, it can also -model C structs without requiring a native C counterpart, a feature -which probably has only exceedingly obscure uses in JS-side -implementations for native callbacks. - -How it works: - -- The client provides a JSON-friendly description of a C struct, - describing the names, sizes, and offsets of each member. -- Pass that description to a factory function to create - a JS constructor for that C struct. -- That constructor allocates a block of heap memory of the C struct's - size and maps it to the new JS-side struct instance. Each instance - inherits property interceptors for each struct member, such that - fetching the C struct's members reads directly from the struct's - memory and setting them writes to that memory. Similarly, these - objects can be provided with memory constructed elsewhere, e.g. - a struct pointer returned from a WASM function, and can proxy - that memory via the struct's interface. -- Clients eventually call the `dispose()` method to free the - instance's heap memory and disassociate the JS instance with its - WASM-side resources. - -Easy peasy! - -**Build instructions**: [see Appendix B](#appendix-b). - **Browser compatibility**: this library requires a _recent_ browser and makes no attempt whatsoever to accommodate "older" or lesser-capable ones, where "recent," _very roughly_, means released in @@ -54,10 +25,7 @@ are based solely on feature compatibility tables provided at **Non-browser compatibility**: this code does not target non-browser JS engines and is completely untested on them. That said, it "might -work". These JS APIs do not use the DOM API in any way, so are not -specifically tied to a browser, but they _are_ fully untested in such -environments. This code is known to work with both [Emscripten][] builds -and [WASI-SDK][] SDK builds (at of this writing, 2025-11-08). +work". **64-bit WASM:** as of 2025-09-21 this API supports 64-bit WASM builds but it has to be configured for it (see [](#api-binderfactory) for @@ -89,7 +57,7 @@ project was spawned: ----- -<a id='overview'></a> +<a name='overview'></a> Table of Contents ============================================================ @@ -104,18 +72,16 @@ Table of Contents - APIs - [Struct Binder Factory](#api-binderfactory) - [Struct Binder](#api-structbinder) - - [Struct Description Objects](#struct-descr) -- [Struct Type](#api-structtype) + - [Struct Type](#api-structtype) - [Struct Constructors](#api-structctor) - [Struct Protypes](#api-structprototype) - [Struct Instances](#api-structinstance) - Appendices - [Appendix A: Limitations, TODOs, etc.](#appendix-a) - - [Appendix B: Build](#appendix-b) - [Appendix D: Debug Info](#appendix-d) - [Appendix G: Generating Struct Descriptions](#appendix-g) -<a id='overview'></a> +<a name='overview'></a> Overview ============================================================ @@ -148,16 +114,15 @@ Portability notes: because it is the most widespread WASM toolchain, but this code is specifically designed to be usable in arbitrary WASM environments. It abstracts away a few Emscripten-specific features into - configurable options. The build tree supports both [Emscripten][] - and [WASI-SDK][] to demonstrate that it has no dependencies on - either. + configurable options. Similarly, the build tree requires Emscripten + but Jaccwabyt does not have any hard Emscripten dependencies. - This code is encapsulated into a single JavaScript function. It should be trivial to copy/paste into arbitrary WASM/JS-using projects. - The source tree includes C code, but only for testing and - demonstration purposes. It is not a core distributable. + demonstration purposes. It is not part of the core distributable. -<a id='architecture'></a> +<a name='architecture'></a> Architecture ------------------------------------------------------------ @@ -191,10 +156,9 @@ Its major classes and functions are: an appropriate configuration, to generate a single... - **[StructBinder][]** is a factory function which converts an arbitrary number struct descriptions into... -- **[StructType][]** are [constructors][StructCtor], one per struct +- **[StructTypes][StructCtors]** are constructors, one per struct description, which inherit from - **[`StructBinder.StructType`][StructType]** and are used to - instantiate... + **[`StructBinder.StructType`][StructType]** and are used to instantiate... - **[Struct instances][StructInstance]** are objects representing individual instances of generated struct types. @@ -203,7 +167,7 @@ need only one. Each StructBinder is effectively a separate namespace for struct creation. -<a id='creating-binding'></a> +<a name='creating-binding'></a> Creating and Binding Structs ============================================================ @@ -226,7 +190,7 @@ essentially boils down to: Detailed instructions for each of those steps follows... -<a id='step-1'></a> +<a name='step-1'></a> Step 1: Configure Jaccwabyt for the Environment ------------------------------------------------------------ @@ -244,7 +208,7 @@ const MyBinder = StructBinderFactory({ a Uint8Array or Int8Array view of the WASM memory, alloc: function(howMuchMemory){...}, dealloc: function(pointerToFree){...}, - pointerSize: 4 or 8 // WASM pointer size + pointerIR: 'i32' or 'i64' // WASM pointer type - default = 'i32' }); ``` @@ -270,7 +234,7 @@ a conventional Emscripten setup, that config might simply look like: The StructBinder factory function returns a function which can then be used to create bindings for our structs. -<a id='step-2'></a> +<a name='step-2'></a> Step 2: Create a Struct Description ------------------------------------------------------------ @@ -303,9 +267,100 @@ Its JSON description looks like: } ``` -This is described in more detail in [][StructBinder]. +These data _must_ match up with the C-side definition of the struct +(if any). See [Appendix G][appendix-g] for one way to easily generate +these from C code. + +Each entry in the `members` object maps the member's name to +its low-level layout: + +- `offset`: the byte offset from the start of the struct, as reported + by C's `offsetof()` feature. +- `sizeof`: as reported by C's `sizeof()`. +- `signature`: described below. +- `readOnly`: optional. If set to true, the binding layer will + throw if JS code tries to set that property. + +The order of the `members` entries is not important: their memory +layout is determined by their `offset` and `sizeof` members. The +`name` property is technically optional, but one of the steps in the +binding process requires that either it be passed an explicit name or +there be one in the struct description. The names of the `members` +entries need not match their C counterparts. Project conventions may +call for giving them different names in the JS side and the +[StructBinderFactory][] can be configured to automatically add a +prefix and/or suffix to their names. + +Nested structs are as-yet unsupported by this tool. -<a id='step-2-pvsp'></a> +Struct member "signatures" describe the data types of the members and +are an extended variant of the format used by Emscripten's +`addFunction()`. A signature for a non-function-pointer member, or +function pointer member which is to be modelled as an opaque pointer, +is a single letter. A signature for a function pointer may also be +modelled as a series of letters describing the call signature. The +supported letters are: + +- **`v`** = `void` (only used as return type for function pointer members) +- **`i`** = `int32` (4 bytes) +- **`j`** = `int64` (8 bytes) is only really usable if this code is built + with BigInt support (e.g. using the Emscripten `-sWASM_BIGINT` build + flag). Without that, this API may throw when encountering the `j` + signature entry. +- **`f`** = `float` (4 bytes) +- **`d`** = `double` (8 bytes) +- **`c`** = `int8` (1 byte) char - see notes below! +- **`C`** = `uint8` (1 byte) unsigned char - see notes below! +- **`p`** = `int32` (see notes below!) +- **`P`** = Like `p` but with extra handling. Described below. +- **`s`** = like `int32` but is a _hint_ that it's a pointer to a + string so that _some_ (very limited) contexts may treat it as such, + noting that such algorithms must, for lack of information to the + contrary, assume both that the encoding is UTF-8 and that the + pointer's member is NUL-terminated. If that is _not_ the case for a + given string member, do not use `s`: use `i` or `p` instead and do + any string handling yourself. + +Noting that: + +- **All of these types are numeric**. Attempting to set any + struct-bound property to a non-numeric value will trigger an + exception except in cases explicitly noted otherwise. +- **"Char" types**: WASM does not define an `int8` type, nor does its + JS representation distinguish between signed and unsigned. This API + treats `c` as `int8` and `C` as `uint8` for purposes of getting and + setting values when using the `DataView` class. It is _not_ + recommended that client code use these types in new WASM-capable + code, but they were added for the sake of binding some immutable + legacy code to WASM. + +> Sidebar: Emscripten's public docs do not mention `p`, but their +generated code includes `p` as an alias for `i`, presumably to mean +"pointer". Though `i` is legal for pointer types in the signature, `p` +is more descriptive, so this framework encourages the use of `p` for +pointer-type members. Using `p` for pointers also helps future-proof +the signatures against the eventuality that WASM eventually supports +64-bit pointers. Note that sometimes `p` _really_ means a +pointer-to-pointer. We simply have to be aware of when we need to deal +with pointers and pointers-to-pointers in JS code. + +> Trivia: this API treates `p` as distinctly different from `i` in +some contexts, so its use is encouraged for pointer types. + +Signatures in the form `x(...)` denote function-pointer members and +`x` denotes non-function members. Functions with no arguments use the +form `x()`. For function-type signatures, the strings are formulated +such that they can be passed to Emscripten's `addFunction()` after +stripping out the `(` and `)` characters. For good measure, to match +the public Emscripten docs, `p`, `c`, and `C`, should also be replaced +with `i`. In JavaScript that might look like: + +> +``` +signature.replace(/[^vipPsjfdcC]/g,'').replace(/[pPscC]/g,'i'); +``` + +<a name='step-2-pvsp'></a> ### `P` vs `p` in Method Signatures *This support is experimental and subject to change.* @@ -324,7 +379,7 @@ stored in `myStruct.x`. If `y` is neither a pointer nor a or `P` is used). -<a id='step-3'></a> +<a name='step-3'></a> Step 3: Binding the Struct ------------------------------------------------------------ @@ -347,7 +402,7 @@ simplify certain later operations. If that is not desired, then feed it a copy of the original, e.g. by passing it `JSON.parse(JSON.stringify(structDefinition))`. -<a id='step-4'></a> +<a name='step-4'></a> Step 4: Creating, Using, and Destroying Struct Instances ------------------------------------------------------------ @@ -406,11 +461,11 @@ Now that we have struct instances, there are a number of things we can do with them, as covered in the rest of this document. -<a id='api'></a> +<a name='api'></a> API Reference ============================================================ -<a id='api-binderfactory'></a> +<a name='api-binderfactory'></a> API: Binder Factory ------------------------------------------------------------ @@ -425,14 +480,14 @@ Function StructBinderFactory(object configOptions); It returns a function which these docs refer to as a [StructBinder][] (covered in the next section). It throws on error. -The binder factory supports the following options in its configuration -object argument: +The binder factory supports the following options in its +configuration object argument: -- `pointerSize` (Added 2025-11-15 to replace `pointerIR`) - Optionally specify the WASM pointer size of 4 (32-bit) or 8 - (64-bit). Any other truthy value triggers an exception. If - `pointerSize` is not set then it will guess the size by `alloc()`ing - one byte, checking the result type, and `dealloc()`ing it. +- `pointerIR` (Added 2025-09-21) + Optionally specify the WASM pointer size with the string `'i32'` or + `'i64'`, defaulting to the former. When using with 64-bit WASM + builds, this must be set to `'i64'` by the client. Any other value + triggers an exception. - `heap` Must be either a `WebAssembly.Memory` instance representing the WASM @@ -443,24 +498,21 @@ object argument: for the WASM heap to grow at runtime. - `alloc` - Must be a function semantically compatible with C's - `malloc(3)`. That is, it is passed the number of bytes to allocate - and it returns a pointer. On allocation failure it may either return - 0 or throw an exception. This API will throw an exception if - allocation fails or will propagate whatever exception the allocator - throws. The allocator _must_ use the same heap as the `heap` config - option. + Must be a function semantically compatible with Emscripten's + `Module._malloc()`. That is, it is passed the number of bytes to + allocate and it returns a pointer. On allocation failure it may + either return 0 or throw an exception. This API will throw an + exception if allocation fails or will propagate whatever exception + the allocator throws. The allocator _must_ use the same heap as the + `heap` config option. - `dealloc` - Must be a function semantically compatible with C's `free(3)`. That - is, it takes a pointer returned from `alloc()` and releases that - memory. It must never throw and must accept a value of 0/null to - mean "do nothing". - -- `realloc` - Optional but required for (eventual (and optional) realloc support - of structs. If set, it must be a function semantically compatible - with C's `realloc()`. See `alloc`, above, for other requirements. + Must be a function semantically compatible with Emscripten's + `Module._free()`. That is, it takes a pointer returned from + `alloc()` and releases that memory. It must never throw and must + accept a value of 0/null to mean "do nothing" (noting that 0 is + _technically_ a legal memory address in WASM, but that seems like a + design flaw). - `bigIntEnabled` (bool=true if BigInt64Array is available, else false) If true, the WASM bits this code is used with must have been @@ -490,13 +542,13 @@ object argument: (like `console.debug` does). See [Appendix D](#appendix-d) for info about enabling debugging output. -<a id='api-structbinder'></a> +<a name='api-structbinder'></a> API: Struct Binder ------------------------------------------------------------ Struct Binders are factories which are created by the [StructBinderFactory][]. A given Struct Binder can process any number -of distinct structs. In a typical setup, an app will have only one +of distinct structs. In a typical setup, an app will have ony one shared Binder Factory and one Struct Binder. Struct Binders which are created via different [StructBinderFactory][] calls are unrelated to each other, sharing no state except, perhaps, indirectly via @@ -516,7 +568,7 @@ The returned object is a constructor for instances of the struct described by its argument(s), each of which derives from a separate [StructType][] instance. -StructBinder has the following members: +The Struct Binder has the following members: - `allocCString(str)` Allocates a new UTF-8-encoded, NUL-terminated copy of the given JS @@ -531,221 +583,7 @@ StructBinder has the following members: any of its "significant" configuration values may have undefined results. -- `adaptGet(key [,func])` - Gets or sets a "get adaptor" by name - an arbitrary client-defined - string, e.g. `"to-js-string"`. Struct description objects may have - their `adaptGet` property set to the name of a mapped getter to - behave exactly as if that struct description had set the given - function as its `get` property. This offers a JSON-friendly way of - storing adaptor mappings, with the caveat that the adaptors need to - be defined _somewhere_ outside of JSON (typically it should be done - immediately after creating the StructBinder). - -- `adaptSet(key [,func])` - The "set" counterpart of `adaptGet`. - -- `ptrAdd(...)` - Coerces all of its arguments to the WASM pointer type, adds them - together, and returns a result of that same type. This is a - workaround for mixed-BigInt/Number pointer math being illegal in JS. - -The `structDescription` argument is described in detail in the -following section. - -<a id='struct-descr'></a> -### Struct Description Object - -C structs are described in a JSON-friendly format: - -> -```json -{ - "name": "MyStruct", - "sizeof": 16, - "members": { - "member1": {"offset": 0,"sizeof": 4,"signature": "i"}, - "member2": {"offset": 4,"sizeof": 4,"signature": "p"}, - "member3": {"offset": 8,"sizeof": 8,"signature": "j"} - } -} -``` - -Forewarning: these data _must_ match up with the C-side definition of -the struct (if any). See [Appendix G][appendix-g] for one way to -easily generate these from C code. - -Every struct must have a `sizeof`. (Though we _could_ calculate it -based on the list of members, we don't. Actually, we do, then we throw -if the values don't match up.) The `name` is required as well but it -may optionally be passed as the first argument to -`StructBinder(structName,structDescription)`. the `name` property -represents the member's symbolic name, typically its native name. - -Abstractly speaking, a struct description in an object with the -properties `sizeof`, `offset`, and either `signature` _or_ -`member`. `offset` is optional only in the top-most object of a struct -description. Every sub-object (a.k.a. member description object) -requires the `offset` property. - -Member description objects are those in the `members` property: - -`"members": {"memberName": {...member description...}, ...}` - -A struct description which has its own `members` object represents a -nested struct, with an identical description syntax to that of a -top-level struct except that nested structs require an `offset` -property. - -Each entry in a struct/member description object maps the member's -name to its low-level layout and other metadata: - -- `offset` - The byte offset from the start of the struct, as reported by C's - `offsetof()` feature. For nested structs's members, this value is - relative to the nested struct, not the parent struct. -- `sizeof` - As reported by C's `sizeof()`. -- `signature` - A type-id signature for this member. Described below. -- `readOnly [=false]` - Optional boolean. If set to true, the binding layer will throw if JS - code tries to set that property. -- `zeroOnDispose [=false]` - If true, then the library will zero out the memory of instances of - this struct when their `dispose()` method is called. Because - `StructType.dispose()` does not free instances which wrap - externally-provided memory, those instances are not wiped when - disposed (doing so would likely interfere with other users of that - memory). (There is no need for a `zeroOnAlloc` counterpart because - newly-allocated instances are always zero-filled for sanity's - sake.) -- `members` - This object describes the individual struct members, mapping their - names to a member description object. `members` gets processed - recursively. Any member with its own `members` property is a - nested-struct, and the property accessor for such members will - return an instance of the distinct StructType which wraps that - member's native memory. - Nested-struct members cannot be assigned over. `signature` is - illegal if `members` is set. -- `get` - Optional function. When fetching this member, this getter is passed - `(K,V)`, where `K` is the struct member's key and `V` is the - _native_ value. `get()`'s return value becomes the value of the - property access operation. This enables custom "native-to-JS" - conversions. If the member is a nested struct, the value passed to - the getter is a StructType proxy object which provides access to its - own members, a subset of the parent object's memory. In the context - of the getter call, "this" is the object upon which the get is being - performed. -- `set` - Optional function. When setting this member, this setter is passed - `(K,V)`, where `K` is the struct member's key and `V` is the _JS_ - value the property is being assigned to. The `set()` return value is - assigned to the _native_ struct member. Thus `set()` _must_ return - an appropriate numeric value and can perform "JS-to-native" - conversions. `set` is not currently legal for nested struct values, - but it is on their own non-nested-struct members. In the context of - the setter call, "this" is the object upon which the set is being - performed. -- `adaptGet` and `adaptSet` - JSON-friendly variants of `get` and `set`. Each may be assigned a - string value, and each such string must be mapped with - `StructBinder.adaptGet(key,func)` - resp. `StructBinder.adaptSet(key,func)`. With that in place, these - behave like `get` resp. `set`. -- `structName` - Optional descriptive name, possibly distinct from the `name`, - primarily used for nested structs. The intent is that this be some - form of the struct type's name, optionally with leading parts of - this object is a nested struct. -- `name` - Is usually optional, and is always optional in `members` entries - because their name is conveniently derived from their containing - object. `name` must be provided only for the top-most struct. The - intent is that `name` maps to the member's property name and that - `structName` optionally be set for nested structs (it will be - derived from the name if it's not set). - -The order of the `members` entries is not important: their memory -layout is determined by their `offset` and `sizeof` members. The -`name` property is technically optional, but one of the steps in the -binding process requires that either it be passed an explicit name or -there be one in the struct description. The names of the `members` -entries need not match their C counterparts. Project conventions may -call for giving them different names in the JS side and the -[StructBinderFactory][] can be configured to automatically add a -prefix and/or suffix to their names. - -Struct member "signatures" describe the data types of the members and -are an extended variant of the format used by Emscripten's -`addFunction()`. A signature for a non-function-pointer member, or -function pointer member which is to be modelled as an opaque pointer, -is a single letter. A signature for a function pointer may also be -modelled as a series of letters describing the call signature. The -supported letters are: - -- **`v`** = `void` (only used as return type for function pointer members) -- **`i`** = `int32` (4 bytes) -- **`j`** = `int64` (8 bytes) is only really usable if this code is built - with BigInt support (e.g. using the Emscripten `-sWASM_BIGINT` build - flag). Without that, this API may throw when encountering the `j` - signature entry. -- **`f`** = `float` (4 bytes) -- **`d`** = `double` (8 bytes) -- **`c`** = `int8` (1 byte) char - see notes below! -- **`C`** = `uint8` (1 byte) unsigned char - see notes below! -- **`p`** = `int32` (see notes below!) -- **`P`** = Like `p` but with extra handling. Described below. -- **`s`** = like `int32` but is a _hint_ that it's a pointer to a - string so that _some_ (very limited) contexts may treat it as such, - noting that such algorithms must, for lack of information to the - contrary, assume both that the encoding is UTF-8 and that the - pointer's member is NUL-terminated. If that is _not_ the case for a - given string member, do not use `s`: use `i` or `p` instead and do - any string handling yourself. - -Noting that: - -- **All of these types are numeric**. Attempting to set any - struct-bound property to a non-numeric value will trigger an - exception except in cases explicitly noted otherwise. -- **"Char" types**: WASM does not define an `int8` type, nor does its - JS representation distinguish between signed and unsigned. This API - treats `c` as `int8` and `C` as `uint8` for purposes of getting and - setting values when using the `DataView` class. It is _not_ - recommended that client code use these types in new WASM-capable - code, but they were added for the sake of binding some immutable - legacy code to WASM. - -> Sidebar: Emscripten's public docs do not mention `p`, but their -generated code includes `p` as an alias for `i`, presumably to mean -"pointer". Though `i` is legal for pointer types in the signature, `p` -is more descriptive, so this framework encourages the use of `p` for -pointer-type members. Using `p` for pointers also helps future-proof -the signatures against the eventuality that WASM eventually supports -64-bit pointers. Note that sometimes `p` _really_ means a -pointer-to-pointer. We simply have to be aware of when we need to deal -with pointers and pointers-to-pointers in JS code. - -> Trivia: this API treates `p` as distinctly different from `i` in -some contexts, so its use is encouraged for pointer types. - -Signatures in the form `x(...)` denote function-pointer members and -`x` denotes non-function members. Functions with no arguments use the -form `x()`. For function-type signatures, the strings are formulated -such that they can be passed to Emscripten's `addFunction()` after -stripping out the `(` and `)` characters. For good measure, to match -the public Emscripten docs, `p`, `c`, and `C`, should also be replaced -with `i`. In JavaScript that might look like: - -> -``` -signature.replace(/[^vipPsjfdcC]/g,'').replace(/[pPscC]/g,'i'); -``` - - -<a id='api-structtype'></a> +<a name='api-structtype'></a> API: Struct Type ------------------------------------------------------------ @@ -760,7 +598,7 @@ config options. The StructType constructor cannot be called from client code. It is only called by the [StructBinder][]-generated -[constructors][StructCtor]. The `StructBinder.StructType` object +[constructors][StructCtors]. The `StructBinder.StructType` object has the following "static" properties (^Which are accessible from individual instances via `theInstance.constructor`.): @@ -770,8 +608,7 @@ individual instances via `theInstance.constructor`.): a function-typed `ondispose` property, this call replaces it with an array and moves that function into the array. In all other cases, `ondispose` is assumed to be an array and the argument(s) is/are - appended to it. Returns `this`. See `dispose()`, below, for where - this applies. + appended to it. Returns `this`. - `allocCString(str)` Identical to the [StructBinder][] method of the same name. @@ -779,7 +616,7 @@ individual instances via `theInstance.constructor`.): - `hasExternalPointer(object)` Returns true if the given object's `pointer` member refers to an "external" object. That is the case when a pointer is passed to a - [struct's constructor][StructCtor]. If true, the memory is owned by + [struct's constructor][StructCtors]. If true, the memory is owned by someone other than the object and must outlive the object. - `isA(value)` @@ -796,7 +633,7 @@ individual instances via `theInstance.constructor`.): The base StructType prototype has the following members, all of which are inherited by [struct instances](#api-structinstance) and may only -legally be used with concrete struct instances unless noted otherwise: +legally be called on concrete struct instances unless noted otherwise: - `dispose()` Frees, if appropriate, the WASM-allocated memory which is allocated @@ -804,11 +641,11 @@ legally be used with concrete struct instances unless noted otherwise: cleans up the object, a leak in the WASM heap memory pool will result. When `dispose()` is called, if the object has a property named `ondispose` then it is treated as follows: - - If it is a function, it is called with the struct object as its - `this`. That method must not throw - if it does, the exception - will be ignored. + - If it is a function, it is called with the struct object as its `this`. + That method must not throw - if it does, the exception will be + ignored. - If it is an array, it may contain functions, pointers, other - [StructType][] instances, and/or JS strings. If an entry is a + [StructType] instances, and/or JS strings. If an entry is a function, it is called as described above. If it's a number, it's assumed to be a pointer and is passed to the `dealloc()` function configured for the parent [StructBinder][]. If it's a @@ -818,12 +655,7 @@ legally be used with concrete struct instances unless noted otherwise: supported primarily for use as debugging information. - Some struct APIs will manipulate the `ondispose` member, creating it as an array or converting it from a function to array as - needed. Most simply, `addOnDispose()` is used to manipulate the - on-dispose data. - -- `extraBytes` (integer, read-only) - If this instance was allocated with the `extraBytes` option, this is - that value, else it is 0. + needed. - `lookupMember(memberName,throwIfNotFound=true)` Given the name of a mapped struct member, it returns the member @@ -876,19 +708,6 @@ legally be used with concrete struct instances unless noted otherwise: the struct will invalidate older serialized data and (B) serializing member pointers is useless. -- `pointer` (number, read-only) - A read-only numeric property which is the "pointer" returned by the - configured allocator when this object is constructed. After - `dispose()` (inherited from [StructType][]) is called, this property - has the `undefined` value. When calling C-side code which takes a - pointer to a struct of this type, simply pass it `myStruct.pointer`. - Whether this member is of type Number or BigInt depends on whether - the WASM environment is 32-bit (Number) or 64-bit (BigInt). - -- `ptrAdd(args...)` - Equivalent to [StructBinder][]`.ptrAdd(this.pointer, args...)` - or [StructCtor][]`.ptrAdd(this.pointer, args...)`. - - `setMemberCString(memberName,str)` Uses `StructType.allocCString()` to allocate a new C-style string, assign it to the given member, and add the new string to this @@ -906,13 +725,9 @@ legally be used with concrete struct instances unless noted otherwise: from JS be kept to a minimum or that the relationship be one-way: let C manage the strings and only fetch them from JS using, e.g., `memberToJsString()`. + -- `zeroOnDispose` (bool, read-only) - True if this instance or its prototype were configured with - the `zeroOnDispose` flag. - - -<a id='api-structctor'></a> +<a name='api-structctor'></a> API: Struct Constructors ------------------------------------------------------------ @@ -926,90 +741,29 @@ const x = new MyStruct; ``` Normally they should be passed no arguments, but they optionally -accept a single argument: a WASM heap pointer address of memory which -the object will use for storage. It does _not_ take over ownership of -that memory and that memory must remain valid for at least as long as -this struct instance. This is used, for example, to proxy -static/shared C-side instances or those which we simply want to -get access to via this higher-level API for a while: +accept a single argument: a WASM heap pointer address of memory +which the object will use for storage. It does _not_ take over +ownership of that memory and that memory must be valid at +for least as long as this struct instance. This is used, for example, +to proxy static/shared C-side instances: > ``` const x = new MyStruct( someCFuncWhichReturnsAMyStructPointer() ); ... -x.dispose(); // does NOT free or zero the memory +x.dispose(); // does NOT free the memory ``` The JS-side construct does not own the memory in that case and has no way of knowing when the C-side struct is destroyed. Results are specifically undefined if the JS-side struct is used after the C-side -struct's member is freed. However, if the client has allocated that -memory themselves and wants to transfer ownership of it to the -instance, that can be done with: - -> -``` -x.addOnDispose( thePtr ); -``` - -Which will free `thePtr` when `x.dispose()` is called. - -As of 2025-11-10, a third option is available: +struct's member is freed. -> -``` -const x = new MyStruct({ - wrap: ptr, // as per MyStruct(ptr) - takeOwnership: bool, // If true, take ownership of the wrap ptr. - zeroOnDispose: bool , // if true, overrides MyStruct.structInfo.zeroOnDispose - extraBytes: int // bytes to alloc after the end of the struct -}); -``` - -- If `wrap` is set then (A) it must be at least - `MyStruct.structInfo.sizeof` of memory owned by the caller, (B) it - _must_ have been allocated using the same allocator as Jaccwabyt is - configured for, and (C) ownership of it is transfered to the new - object. `zeroOnDispose` and `extraBytes` are disregarded if `wrap` - is set. A falsy `wrap` value is equivalent to not providing one and - a non-integer value, or a number less than 0, triggers an error. - -- If `takeOwnership` is truthy then this object takes over ownership - of the `wrap` pointer (if any). This flag is otherwise ignored. - -- If `zeroOnDispose` or `extraBytes` are are used without `wrap`, each - which is used is set as a read-only propperty on the resulting - `MyStruct` object, except that `zeroOnDispose` is only recorded if - it is truthy and `extraBytes` is only recorded if it is not 0 (a - negative value, or non-integer, triggers an exception). - -- `ondispose`: if set it is passed to the new object's - `addOnDispose()` before the constructor returns, so may have any - value legal for that method. Results are undefined if `ondispose` - and `wrap` have the same pointer value (because `wrap` will already - be cleaned up via `dispose()`, as if it had been passed to - `addOnDispose()`). - -In the case of `extraBytes`, a pointer to the tail of the memory can -be obtained by adding `theInstance.pointer` to -`theInstance.structInfo.sizeof`, with the caveat that `pointer` may be -a BigInt value (on 64-bit WASM), `sizeof` will be a Number and, -spoiler alert, `(BigInt(1) + Number(1))` is not legal. Thus this API -adds a small convenience method to work around that portability issue: - -> -``` -const ptrTail = MyStruct.ptrAdd(theInstance.pointer, theInstance.extraBytes); -``` - -`typeof ptrTail` is `'bigint'` in 64-bit builds and `'number'` in -32-bit. i.e. it's the same type as `theInstance.pointer`. -Equivalently, the inherited `MyStruct` instance method with the same -name adds an instance's own `pointer` value to its arguments: - -``` -const ptrTail = theInstance.ptrAdd(theInstance.extraBytes); -``` +> Potential TODO: add a way of passing ownership of the C-side struct +to the JS-side object. e.g. maybe simply pass `true` as the second +argument to tell the constructor to take over ownership. Currently the +pointer can be taken over using something like +`myStruct.ondispose=[myStruct.pointer]` immediately after creation. These constructors have the following "static" members: @@ -1022,10 +776,6 @@ These constructors have the following "static" members: - `memberKeys(string)` Works exactly as documented for [StructType][]. -- `ptrAdd(args...)` - Equivalent to [StructBinder][]`.ptrAdd(args...)`. The [_inherited_ - method with the same name][StructType] behaves differently. - - `structInfo` The structure description passed to [StructBinder][] when this constructor was generated. @@ -1033,14 +783,14 @@ These constructors have the following "static" members: - `structName` The structure name passed to [StructBinder][] when this constructor was generated. + - -<a id='api-structprototype'></a> +<a name='api-structprototype'></a> API: Struct Prototypes ------------------------------------------------------------ The prototypes of structs created via [the constructors described in -the previous section][StructCtor] are each a struct-type-specific +the previous section][StructCtors] are each a struct-type-specific instance of [StructType][] and add the following struct-type-specific properties to the mix: @@ -1052,57 +802,78 @@ properties to the mix: The name of the struct, as it was given to the [StructBinder][] which created this class. -<a id='api-structinstance'></a> +<a name='api-structinstance'></a> API: Struct Instances ------------------------------------------------------------------------ Instances of structs created via [the constructors described -above][StructCtor]. Each inherits all of the methods and properties -from their constructor's prototype. +above][StructCtors] each have the following instance-specific state in +common: + +- `pointer` + A read-only numeric property which is the "pointer" returned by the + configured allocator when this object is constructed. After + `dispose()` (inherited from [StructType][]) is called, this property + has the `undefined` value. When calling C-side code which takes a + pointer to a struct of this type, simply pass it `myStruct.pointer`. -<a id='appendices'></a> +<a name='appendices'></a> Appendices ============================================================ -<a id='appendix-a'></a> +<a name='appendix-a'></a> Appendix A: Limitations, TODOs, and Non-TODOs ------------------------------------------------------------ - This library only supports the basic set of member types supported - by WASM: numbers (which includes pointers). - -- Binding JS functions to struct instances, such that C can see and - call JS-defined functions, is not as transparent as it really could - be. [The WhWasmUtil API][whwasmutil.js], and - standalone subproject co-developed with Jaccwabyt, provides such a - binding mechanism. There is some overlap between the two APIs and - they arguably belong bundled together, but doing so would more than - triple Jaccwabyt's size. (That said, the only known Jaccwabyt - deployment uses both APIs, so maybe it's time to merge them (he says - on 2025-11-10). As of this writing, jaccwabyt.js is 38k, half of - which is comments/docs, whereas whwasmutil.js is 100kb and 75% - docs). - -<a id='appendix-b'></a> -Appendix B: Build ------------------------------------------------------------------------- - -In order to support both vanilla JS and ESM (ES6 module) builds from a -single source, this project uses [a preprocessor][c-pp], which requires -only a C compiler: - -> $ make - -The makefile requires GNU make, not BSD or POSIX make. - -The resulting files are in the `js/` subdirectory, in both "vanilla" -JS and ESM formats. - -This tree [includes all of the requisite sources](/dir/tool) and -requires no out-of-tree dependencies beyond the system's libc. - - -<a id='appendix-d'></a> + by WASM: numbers (which includes pointers). Nested structs are not + handled except that a member may be a _pointer_ to such a + struct. Whether or not it ever will depends entirely on whether its + developer ever needs that support. Conversion of strings between + JS and C requires infrastructure specific to each WASM environment + and is not directly supported by this library. + +- Binding functions to struct instances, such that C can see and call + JS-defined functions, is not as transparent as it really could be, + due to [shortcomings in the Emscripten + `addFunction()`/`removeFunction()` + interfaces](https://github.com/emscripten-core/emscripten/issues/17323). Until + a replacement for that API can be written, this support will be + quite limited. It _is_ possible to bind a JS-defined function to a + C-side function pointer and call that function from C. What's + missing is easier-to-use/more transparent support for doing so. + - In the meantime, a [standalone + subproject](/file/common/whwasmutil.js) of Jaccwabyt provides such a + binding mechanism, but integrating it directly with Jaccwabyt would + not only more than double its size but somehow feels inappropriate, so + experimentation is in order for how to offer that capability via + completely optional [StructBinderFactory][] config options. + +- It "might be interesting" to move access of the C-bound members into + a sub-object. e.g., from JS they might be accessed via + `myStructInstance.s.structMember`. The main advantage is that it would + eliminate any potential confusion about which members are part of + the C struct and which exist purely in JS. "The problem" with that + is that it requires internally mapping the `s` member back to the + object which contains it, which makes the whole thing more costly + and adds one more moving part which can break. Even so, it's + something to try out one rainy day. Maybe even make it optional and + make the `s` name configurable via the [StructBinderFactory][] + options. (Over-engineering is an arguably bad habit of mine.) + +- It "might be interesting" to offer (de)serialization support. It + would be very limited, e.g. we can't serialize arbitrary pointers in + any meaningful way, but "might" be useful for structs which contain + only numeric or C-string state. As it is, it's easy enough for + client code to write wrappers for that and handle the members in + ways appropriate to their apps. Any impl provided in this library + would have the shortcoming that it may inadvertently serialize + pointers (since they're just integers), resulting in potential chaos + after deserialization. Perhaps the struct description can be + extended to tag specific members as serializable and how to + serialize them. + +<a name='appendix-d'></a> Appendix D: Debug Info ------------------------------------------------------------ @@ -1123,7 +894,7 @@ client code: [StructType][]. -<a id='appendix-g'></a> +<a name='appendix-g'></a> Appendix G: Generating Struct Descriptions From C ------------------------------------------------------------ @@ -1293,23 +1064,18 @@ div.content h3::before { div.content h3 {border-left-width: 2.5em} </style> -[appendix-g]: #appendix-g -[BigInt64Array]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt64Array -[c-pp]: https://fossil.wanderinghorse.net/r/c-pp -[Emscripten]: https://emscripten.org -[jaccwabyt.js]: /file/jaccwabyt/jaccwabyt.c-pp.js -[MDN]: https://developer.mozilla.org/docs/Web/API -[sgb]: https://wanderinghorse.net/home/stephan/ [sqlite3]: https://sqlite.org -[StructBinder]: #api-structbinder +[emscripten]: https://emscripten.org +[sgb]: https://wanderinghorse.net/home/stephan/ +[appendix-g]: #appendix-g [StructBinderFactory]: #api-binderfactory -[StructCtor]: #api-structctor -[StructInstance]: #api-structinstance +[StructCtors]: #api-structctor [StructType]: #api-structtype -[TextDecoder]: https://developer.mozilla.org/en-US/docs/Web/API/TextDecoder -[TextEncoder]: https://developer.mozilla.org/en-US/docs/Web/API/TextEncoder -[WASI-SDK]: https://github.com/WebAssembly/wasi-sdk -[whwasmutil.js]: /file/wasmutil/whwasmutil.c-pp.js - +[StructBinder]: #api-structbinder +[StructInstance]: #api-structinstance [^export-func]: In Emscripten, add its name, prefixed with `_`, to the project's `EXPORT_FUNCTIONS` list. +[BigInt64Array]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt64Array +[TextDecoder]: https://developer.mozilla.org/en-US/docs/Web/API/TextDecoder +[TextEncoder]: https://developer.mozilla.org/en-US/docs/Web/API/TextEncoder +[MDN]: https://developer.mozilla.org/docs/Web/API diff --git a/ext/wasm/mkdist.sh b/ext/wasm/mkdist.sh index 78b93e5e0..84780668b 100755 --- a/ext/wasm/mkdist.sh +++ b/ext/wasm/mkdist.sh @@ -52,7 +52,7 @@ for arg in $@; do ;; --snapshot) - snapshotSuffix=-snapshot-$(date +%Y%m%d) + snapshotSuffix=$(date +%Y%m%d) ;; -?|--help) diff --git a/ext/wasm/mkwasmbuilds.c b/ext/wasm/mkwasmbuilds.c index 37f2967d8..2730d9d76 100644 --- a/ext/wasm/mkwasmbuilds.c +++ b/ext/wasm/mkwasmbuilds.c @@ -40,22 +40,14 @@ /* ** Flags for use with BuildDef::flags. ** -** Maintenance reminder: do not combine F_... flags within this enum, -** e.g. F_BUNDLER_FRIENDLY=0x02|F_ESM, as that will lead to breakage -** in some of the flag checks. +** Maintenance reminder: do not combine flags within this enum, +** e.g. F_BUNDLER_FRIENDLY=0x02|F_ESM, as that will lead +** to breakage in some of the flag checks. */ enum BuildDefFlags { /* Indicates an ESM module build. */ F_ESM = 0x01, - /* Indicates a "bundler-friendly" build mode. These are untested and - ** unsupported, provided solely for the downstream npm subproject - ** (who is responsible for any testing of these). - ** - ** The only difference beween bundler-friendly and esm builds is - ** that bundlers require static filename strings in a few places due - ** to limitations of bundler tooling, whereas vanilla and JS can - ** both work with dynamic strings. - */ + /* Indicates a "bundler-friendly" build mode. */ F_BUNDLER_FRIENDLY = 1<<1, /* Indicates that this build is unsupported. Such builds are not ** added to the 'all' target. The unsupported builds exist primarily @@ -65,11 +57,8 @@ enum BuildDefFlags { F_NOT_IN_ALL = 1<<3, /* If it's a 64-bit build. */ F_64BIT = 1<<4, - /* Indicates a node.js-for-node.js build. This build is very - ** specificially untested and unsupported, but the downstream npm - ** project makes use of it. None of our JS code is specific to node, - ** but Emscripten's generated sqlite3.js differs between - ** for-the-browser and for-node builds. */ + /* Indicates a node.js-for-node.js build (untested and + ** unsupported). */ F_NODEJS = 1<<5, /* Indicates a wasmfs build (untested and unsupported). */ F_WASMFS = 1<<6, @@ -82,13 +71,13 @@ enum BuildDefFlags { ** only their JS file and patch their JS to use the WASM file from a ** canonical build which uses that same WASM file. Reusing X.wasm ** that way can only work for builds which are processed identically - ** by Emscripten. For a given set of C flags (as distinct from + ** by Emscripten. For a given set of C flags (as opposed to ** JS-influencing flags), all builds of X.js and Y.js will produce ** identical X.wasm and Y.wasm files. Their JS files may well ** differ, however. */ - CP_JS = 1 << 30, /* X.js or X.mjs, depending on F_ESM */ - CP_WASM = 1 << 31, /* X.wasm */ + CP_JS = 1 << 30, + CP_WASM = 1 << 31, CP_ALL = CP_JS | CP_WASM }; @@ -105,10 +94,11 @@ enum BuildDefFlags { ** final build directory $(dir.dout). ** ** To keep parallel builds from stepping on each other, each distinct -** build goes into its own subdir $(dir.dout.$(BuildDef::zBaseName). -** Builds which produce deliverables we'd like to keep/distribute copy -** their final results into the build dir $(dir.dout). See the notes -** for the CP_JS enum entry for more details on that. +** build goes into its own subdir $(dir.dout.BuildName)[^1], i.e. +** $(dir.dout)/BuildName. Builds which produce deliverables we'd like +** to keep/distribute copy their final results into the build dir +** $(dir.dout). See the notes for the CP_JS enum entry for more +** details on that. ** ** The final result of each build is a pair of JS/WASM files, but ** getting there requires generation of several files, primarily as @@ -126,9 +116,9 @@ enum BuildDefFlags { ** ** --extern-post-js = gets injected immediately after ** sqlite3InitModule(), in the global scope. In this step we replace -** sqlite3InitModule() with a slightly customized one, the main -** purpose of which is to (A) give us (not Emscripten) control over -** the arguments it accepts and (B) to run the library bootstrap step. +** sqlite3InitModule() with a slightly customized, the main purpose of +** which is to (A) give us (not Emscripten) control over the arguments +** it accepts and (B) to run the library bootstrap step. ** ** Then there's sqlite3-api.BuildName.js, which is the entire SQLite3 ** JS API (generated from the list defined in $(sqlite3-api.jses)). It @@ -136,7 +126,9 @@ enum BuildDefFlags { ** ** Each of those inputs has to be generated before passing them on to ** Emscripten so that any build-specific capabilities can get filtered -** in or out (using ./c-pp-lite.c). +** in or out (using ./c-pp.c). +** +** [^1]: The legal BuildNames are in this file's BuildDef_map macro. */ struct BuildDef { /* @@ -151,8 +143,8 @@ struct BuildDef { ** ** The convention for 32- vs 64-bit pairs is to give them similar ** emoji, e.g. a cookie for 32-bit and a donut or cake for 64. - ** Alternately, the same emoji with a "64" suffix, except that that - ** throws off the output alignment in parallel builds ;). + ** Alternately, the same emoji a "64" suffix, excep that that throws + ** off the output alignment in parallel builds ;). */ const char *zEmo; /* @@ -169,13 +161,13 @@ struct BuildDef { const char *zCmppD; /* Extra -D... flags for c-pp */ const char *zEmcc; /* Full flags for emcc. Normally NULL for default. */ const char *zEmccExtra; /* Extra flags for emcc */ - const char *zDeps; /* Extra make target deps */ + const char *zDeps; /* Extra deps */ const char *zEnv; /* emcc -sENVIRONMENT=... value */ /* ** Makefile code "ifeq (...)". If set, this build is enclosed in a ** $zIfCond/endif block. */ - const char *zIfCond; + const char *zIfCond; /* makefile "ifeq (...)" or similar */ int flags; /* Flags from BuildDefFlags */ }; typedef struct BuildDef BuildDef; @@ -191,7 +183,7 @@ typedef struct BuildDef BuildDef; ** logtag.NAME = Used for decorating log output ** ** etc. -*/ +***/ #define BuildDefs_map(E) \ E(vanilla) E(vanilla64) \ E(esm) E(esm64) \ @@ -204,8 +196,9 @@ typedef struct BuildDef BuildDef; ** The set of WASM builds for the library (as opposed to the apps ** (fiddle, speedtest1)). Their order in BuildDefs_map is mostly ** insignificant, but some makefile vars used by some builds are set -** up by prior builds. Because of that, the vanilla, esm, and -** bundler-friendly builds should be defined first (in that order). +** up by prior builds. Because of that, the (sqlite3, vanilla), +** (sqlite3, esm), and (sqlite3, bundler-friendly) builds should be +** defined first (in that order). */ struct BuildDefs { #define E(N) BuildDef N; @@ -291,7 +284,7 @@ const BuildDefs oBuildDefs = { " -DSQLITE_SPEEDTEST1_WASM" " $(SQLITE_OPT)" " -USQLITE_WASM_BARE_BONES" - " -USQLITE_C -DSQLITE_C=$(sqlite3.c)" + " -USQLITE_C -DSQLITE_C=$(sqlite3.canonical.c)" " $(speedtest1.exit-runtime0)" " $(speedtest1.c.in)" " -lm", @@ -319,7 +312,7 @@ const BuildDefs oBuildDefs = { " -DSQLITE_SPEEDTEST1_WASM" " $(SQLITE_OPT)" " -USQLITE_WASM_BARE_BONES" - " -USQLITE_C -DSQLITE_C=$(sqlite3.c)" + " -USQLITE_C -DSQLITE_C=$(sqlite3.canonical.c)" " $(speedtest1.exit-runtime0)" " $(speedtest1.c.in)" " -lm", @@ -352,7 +345,8 @@ const BuildDefs oBuildDefs = { .zEnv = 0, .zDeps = 0, .zIfCond = 0, - .flags = CP_JS | F_BUNDLER_FRIENDLY | F_ESM | F_NOT_IN_ALL + .flags = CP_JS | F_BUNDLER_FRIENDLY | F_ESM + //| F_NOT_IN_ALL }, /* 64-bit bundler-friendly. */ @@ -377,7 +371,7 @@ const BuildDefs oBuildDefs = { .node = { .zEmo = "🍟", .zBaseName = "sqlite3-node", - .zDotWasm = "sqlite3", + .zDotWasm = 0, .zCmppD = "-Dtarget:node $(c-pp.D.bundler)", .zEmcc = 0, .zEmccExtra = 0, @@ -388,21 +382,21 @@ const BuildDefs oBuildDefs = { ** node. */, .zDeps = 0, .zIfCond = 0, - .flags = CP_JS | F_UNSUPPORTED | F_ESM | F_NODEJS + .flags = CP_ALL | F_UNSUPPORTED | F_NODEJS }, /* 64-bit node. */ .node64 = { .zEmo = "🍔", .zBaseName = "sqlite3-node-64bit", - .zDotWasm = "sqlite3-64bit", + .zDotWasm = 0, .zCmppD = "-Dtarget:node $(c-pp.D.bundler)", .zEmcc = 0, .zEmccExtra = 0, .zEnv = "node", .zDeps = 0, .zIfCond = 0, - .flags = CP_JS | F_UNSUPPORTED | F_ESM | F_NODEJS | F_64BIT + .flags = CP_ALL | F_UNSUPPORTED | F_NODEJS | F_64BIT }, /* Entirely unsupported. */ @@ -493,8 +487,8 @@ static void mk_prologue(void){ ** configuration). Comments like "saves nothing" may not be ** technically correct: "nothing" means "some neglible amount." ** - ** Performance gains or losses are _not_ taken into account - ** here, only wasm file size. + ** Note that performance gains/losses are _not_ taken into + ** account here: only wasm file size. */ "--enable-bulk-memory-opt " /* required */ "--all-features " /* required */ @@ -644,7 +638,7 @@ static void mk_pre_post(char const *zBuildName, BuildDef const * pB){ pB->zDotWasm); } ps(""); - pf("\t@$(call b.mkdir@); $(call b.c-pp.shcmd," + pf("\t@$(call b.c-pp.shcmd," "%s," "$(pre-js.in.js)," "$(pre-js.%s.js)," @@ -783,14 +777,13 @@ static void mk_lib_mode(const char *zBuildName, const BuildDef * pB){ zBuildName, zBuildName, zBaseName); pf("dir.dout.%s ?= $(dir.dout)/%s\n", zBuildName, zBuildName); + pf("out.%s.base ?= $(dir.dout.%s)/%s\n", + zBuildName, zBuildName, zBaseName); pf("c-pp.D.%s ?= %s\n", zBuildName, pB->zCmppD ? pB->zCmppD : ""); if( pB->flags & F_64BIT ){ pf("c-pp.D.%s += $(c-pp.D.64bit)\n", zBuildName); } - if( pB->flags & F_UNSUPPORTED ){ - pf("c-pp.D.%s += -Dunsupported-build\n", zBuildName); - } pf("emcc.environment.%s ?= %s\n", zBuildName, pB->zEnv ? pB->zEnv : oBuildDefs.vanilla.zEnv); @@ -921,9 +914,10 @@ static void mk_lib_mode(const char *zBuildName, const BuildDef * pB){ pf("\n%dbit: $(out.%s.js)\n" "$(out.%s.wasm): $(out.%s.js)\n" - "b-%s: $(out.%s.wasm)\n", + "b-%s: $(out.%s.js) $(out.%s.wasm)\n", (F_64BIT & pB->flags) ? 64 : 32, zBuildName, - zBuildName, zBuildName, zBuildName, zBuildName); + zBuildName, zBuildName, + zBuildName, zBuildName, zBuildName); if( CP_JS & pB->flags ){ pf("$(dir.dout)/%s%s: $(out.%s.js)\n", @@ -994,18 +988,11 @@ static void mk_fiddle(void){ pf("$(out.%s.js): $(MAKEFILE_LIST) " "$(EXPORTED_FUNCTIONS.fiddle) " "$(fiddle.c.in) " - "$(pre-post.%s.deps)", + "$(pre-post.%s.deps)\n", zBuildName, zBuildName); - if( isDebug ){ - pf(" $(dir.fiddle)/fiddle-worker.js" - " $(dir.fiddle)/fiddle.js" - " $(dir.fiddle)/index.html"); - } - ps(""); emit_compile_start(zBuildName); - pf("\t@$(call b.mkdir@)\n" - "\t$(b.cmd@)$(bin.emcc) -o $@" - " $(emcc.flags.%s)" /* set in GNUmakefile */ + pf("\t$(b.cmd@)$(bin.emcc) -o $@" + " $(emcc.flags.%s)" /* set in fiddle.make */ " $(pre-post.%s.flags)" " $(fiddle.c.in)" "\n", @@ -1070,7 +1057,7 @@ int main(int argc, char const ** argv){ BuildDefs_map(E) if( 0==strcmp("prologue",zArg) ){ mk_prologue(); }else { - fprintf(stderr,"Unknown build name: %s\n", zArg); + fprintf(stderr,"Unkown build name: %s\n", zArg); rc = 1; break; } diff --git a/ext/wasm/speedtest1-worker.js b/ext/wasm/speedtest1-worker.js index 9355ba93f..ba11fd163 100644 --- a/ext/wasm/speedtest1-worker.js +++ b/ext/wasm/speedtest1-worker.js @@ -1,7 +1,7 @@ 'use strict'; (function(){ let speedtestJs = 'speedtest1.js'; - const urlParams = new URL(globalThis.location.href).searchParams; + const urlParams = new URL(self.location.href).searchParams; if(urlParams.has('sqlite3.dir')){ speedtestJs = urlParams.get('sqlite3.dir') + '/' + speedtestJs; } @@ -14,9 +14,9 @@ const wasmfsDir = function f(wasmUtil){ if(undefined !== f._) return f._; const pdir = '/opfs'; - if( !globalThis.FileSystemHandle - || !globalThis.FileSystemDirectoryHandle - || !globalThis.FileSystemFileHandle){ + if( !self.FileSystemHandle + || !self.FileSystemDirectoryHandle + || !self.FileSystemFileHandle){ return f._ = ""; } try{ @@ -92,7 +92,7 @@ } }; - globalThis.onmessage = function(msg){ + self.onmessage = function(msg){ msg = msg.data; switch(msg.type){ case 'run': @@ -111,8 +111,7 @@ setStatus: (text)=>mPost('load-status',text) }; log("Initializing speedtest1 module..."); - globalThis.sqlite3InitModule.__isUnderTest = true; - globalThis.sqlite3InitModule(EmscriptenModule).then(async (sqlite3)=>{ + self.sqlite3InitModule(EmscriptenModule).then(async (sqlite3)=>{ const S = globalThis.S = App.sqlite3 = sqlite3; log("Loaded speedtest1 module. Setting up..."); App.pDir = wasmfsDir(S.wasm); diff --git a/ext/wasm/speedtest1.html b/ext/wasm/speedtest1.html index d41f20a4e..cce617185 100644 --- a/ext/wasm/speedtest1.html +++ b/ext/wasm/speedtest1.html @@ -44,137 +44,132 @@ mounted, else it returns an empty string. */ const wasmfsDir = function f(wasmUtil){ - if(undefined !== f._) return f._; - const pdir = '/persistent'; - if( !self.FileSystemHandle - || !self.FileSystemDirectoryHandle - || !self.FileSystemFileHandle){ - return f._ = ""; - } - try{ - if(0===wasmUtil.xCallWrapped( - 'sqlite3_wasm_init_wasmfs', 'i32', ['string'], pdir - )){ - return f._ = pdir; - }else{ - return f._ = ""; + if(undefined !== f._) return f._; + const pdir = '/persistent'; + if( !self.FileSystemHandle + || !self.FileSystemDirectoryHandle + || !self.FileSystemFileHandle){ + return f._ = ""; + } + try{ + if(0===wasmUtil.xCallWrapped( + 'sqlite3_wasm_init_wasmfs', 'i32', ['string'], pdir + )){ + return f._ = pdir; + }else{ + return f._ = ""; + } + }catch(e){ + // sqlite3_wasm_init_wasmfs() is not available + return f._ = ""; } - }catch(e){ - // sqlite3_wasm_init_wasmfs() is not available - return f._ = ""; - } }; wasmfsDir._ = undefined; const eOut = document.querySelector('#test-output'); const log2 = function(cssClass,...args){ - const ln = document.createElement('div'); - if(cssClass) ln.classList.add(cssClass); - ln.append(document.createTextNode(args.join(' '))); - eOut.append(ln); - //this.e.output.lastElementChild.scrollIntoViewIfNeeded(); + const ln = document.createElement('div'); + if(cssClass) ln.classList.add(cssClass); + ln.append(document.createTextNode(args.join(' '))); + eOut.append(ln); + //this.e.output.lastElementChild.scrollIntoViewIfNeeded(); }; const logList = []; const dumpLogList = function(){ - logList.forEach((v)=>log2('',v)); - logList.length = 0; + logList.forEach((v)=>log2('',v)); + logList.length = 0; }; /* we cannot update DOM while speedtest is running unless we run speedtest in a worker thread. */; const log = (...args)=>{ - console.log(...args); - logList.push(args.join(' ')); + console.log(...args); + logList.push(args.join(' ')); }; const logErr = function(...args){ - console.error(...args); - logList.push('ERROR: '+args.join(' ')); + console.error(...args); + logList.push('ERROR: '+args.join(' ')); }; const runTests = function(sqlite3){ - globalThis.S = sqlite3; - const capi = sqlite3.capi, wasm = sqlite3.wasm; - //console.debug('sqlite3 (global S) =',sqlite3); - const pDir = wasmfsDir(wasm); - if(pDir){ - console.warn("wasmfs storage:",pDir); - } - const scope = wasm.scopedAllocPush(); - let dbFile = pDir+"/speedtest1.db"; - const urlParams = new URL(self.location.href).searchParams; - const argv = ["speedtest1"]; - if(urlParams.has('flags')){ - argv.push(...(urlParams.get('flags').split(','))); - } - - let forceSize = 0; - let vfs, pVfs = 0; - if(urlParams.has('vfs')){ - vfs = urlParams.get('vfs'); - pVfs = capi.sqlite3_vfs_find(vfs); - if(!pVfs){ - log2('error',"Unknown VFS:",vfs); - return; + const capi = sqlite3.capi, wasm = sqlite3.wasm; + //console.debug('sqlite3 =',sqlite3); + const pDir = wasmfsDir(wasm); + if(pDir){ + console.warn("Persistent storage:",pDir); } - argv.push("--vfs", vfs); - log2('',"Using VFS:",vfs); - if('kvvfs' === vfs){ - forceSize = 5 - /* --size > 2 is too big for local/session storage - as of mid-2025, but kvvfs v2 (late 2025) - lets us use transient storage. */; - dbFile = 'transient'; - log2('warning',"kvvfs VFS: forcing --size",forceSize, - "and filename '"+dbFile+"'."); - capi.sqlite3_js_kvvfs_clear(dbFile); + const scope = wasm.scopedAllocPush(); + let dbFile = pDir+"/speedtest1.db"; + const urlParams = new URL(self.location.href).searchParams; + const argv = ["speedtest1"]; + if(urlParams.has('flags')){ + argv.push(...(urlParams.get('flags').split(','))); } - } - if(forceSize){ - argv.push('--size',forceSize); - }else{ - [ - 'size' - ].forEach(function(k){ - const v = urlParams.get(k); - if(v && +v) argv.push('--'+k, urlParams[k]); - }); - } - argv.push( - "--singlethread", - //"--nomutex", - //"--nosync", - //"--memdb", // note that memdb trumps the filename arg - "--nomemstat" - ); - argv.push("--big-transactions"/*important for tests 410 and 510!*/, - dbFile); - console.log("argv =",argv); - // These log messages are not emitted to the UI until after main() returns. Fixing that - // requires moving the main() call and related cleanup into a timeout handler. - if(pDir) wasm.sqlite3_wasm_vfs_unlink(pVfs,dbFile); - log2('',"WASM heap size:",sqlite3.wasm.heap8().byteLength,"bytes"); - log2('',"WASM pointer size:",sqlite3.wasm.ptr.size); - log2('',"Starting native app:\n ",argv.join(' ')); - log2('',"This will take a while and the browser might warn about the runaway JS.", - "Give it time..."); - logList.length = 0; - setTimeout(function(){ - wasm.xCall('wasm_main', argv.length, - wasm.scopedAllocMainArgv(argv)); - wasm.scopedAllocPop(scope); - if('kvvfs'===vfs){ - logList.unshift("KVVFS "+dbFile+" size = "+ - capi.sqlite3_js_kvvfs_size(dbFile)); + + let forceSize = 0; + let vfs, pVfs = 0; + if(urlParams.has('vfs')){ + vfs = urlParams.get('vfs'); + pVfs = capi.sqlite3_vfs_find(vfs); + if(!pVfs){ + log2('error',"Unknown VFS:",vfs); + return; + } + argv.push("--vfs", vfs); + log2('',"Using VFS:",vfs); + if('kvvfs' === vfs){ + forceSize = 2 /* >2 is too big as of mid-2025 */; + dbFile = 'session'; + log2('warning',"kvvfs VFS: forcing --size",forceSize, + "and filename '"+dbFile+"'."); + capi.sqlite3_js_kvvfs_clear(dbFile); + } + } + if(forceSize){ + argv.push('--size',forceSize); + }else{ + [ + 'size' + ].forEach(function(k){ + const v = urlParams.get(k); + if(v) argv.push('--'+k, urlParams[k]); + }); } + argv.push( + "--singlethread", + //"--nomutex", + //"--nosync", + //"--memdb", // note that memdb trumps the filename arg + "--nomemstat" + ); + argv.push("--big-transactions"/*important for tests 410 and 510!*/, + dbFile); + console.log("argv =",argv); + // These log messages are not emitted to the UI until after main() returns. Fixing that + // requires moving the main() call and related cleanup into a timeout handler. if(pDir) wasm.sqlite3_wasm_vfs_unlink(pVfs,dbFile); - logList.unshift("Done running native main(). Output:"); - dumpLogList(); log2('',"WASM heap size:",sqlite3.wasm.heap8().byteLength,"bytes"); - }, 50); + log2('',"WASM pointer size:",sqlite3.wasm.ptr.size); + log2('',"Starting native app:\n ",argv.join(' ')); + log2('',"This will take a while and the browser might warn about the runaway JS.", + "Give it time..."); + logList.length = 0; + setTimeout(function(){ + wasm.xCall('wasm_main', argv.length, + wasm.scopedAllocMainArgv(argv)); + wasm.scopedAllocPop(scope); + if('kvvfs'===vfs){ + logList.unshift("KVVFS "+dbFile+" size = "+ + capi.sqlite3_js_kvvfs_size(dbFile)); + } + if(pDir) wasm.sqlite3_wasm_vfs_unlink(pVfs,dbFile); + logList.unshift("Done running native main(). Output:"); + dumpLogList(); + log2('',"WASM heap size:",sqlite3.wasm.heap8().byteLength,"bytes"); + }, 50); }/*runTests()*/; self.sqlite3TestModule.print = log; self.sqlite3TestModule.printErr = logErr; - sqlite3InitModule.__isUnderTest = true; sqlite3InitModule(self.sqlite3TestModule).then(runTests); })();</script> </body> diff --git a/ext/wasm/tester1-worker.c-pp.html b/ext/wasm/tester1-worker.c-pp.html index 71d827a35..e461b6cbf 100644 --- a/ext/wasm/tester1-worker.c-pp.html +++ b/ext/wasm/tester1-worker.c-pp.html @@ -13,18 +13,10 @@ <body> <h1 id='color-target'>sqlite3 tester #1: Worker thread (@bitness@-bit WASM)</h1> <div>Variants: - conventional UI thread: - (<a href='tester1.html' target='tester1.html'>32-bit</a>, - <a href='tester1-64bit.html' target='tester1-64bit.html'>64-bit</a>), - conventional worker: - (<a href='tester1-worker.html' target='tester1-worker.html'>32-bit</a>, - <a href='tester1-worker-64bit.html' target='tester1-worker-64bit.html'>64-bit</a>), - ESM in UI thread: - (<a href='tester1-esm.html' target='tester1-esm.html'>32-bit</a>, - <a href='tester1-esm-64bit.html' target='tester1-esm-64bit.html'>64-bit</a>), - ESM worker:</a> - (<a href='tester1-worker.html?esm' target='tester1-worker.html?esm'>32-bit</a> - <a href='tester1-worker-64bit.html?esm' target='tester1-worker-64bit.html?esm'>64-bit</a>) + <a href='tester1.html' target='tester1.html'>conventional UI thread</a>, + <a href='tester1-worker.html' target='tester1-worker.html'>conventional worker</a>, + <a href='tester1-esm.html' target='tester1-esm.html'>ESM in UI thread</a>, + <a href='tester1-worker.html?esm' target='tester1-worker.html?esm'>ESM worker</a> </div> <div class='input-wrapper'> <input type='checkbox' id='cb-log-reverse'> diff --git a/ext/wasm/tester1.c-pp.html b/ext/wasm/tester1.c-pp.html index 4bb53ee56..95fe52219 100644 --- a/ext/wasm/tester1.c-pp.html +++ b/ext/wasm/tester1.c-pp.html @@ -11,19 +11,11 @@ <style></style> </head> <body><h1 id='color-target'></h1> - <div>Variants (32-bit): - conventional UI thread: - (<a href='tester1.html' target='tester1.html'>32-bit</a>, - <a href='tester1-64bit.html' target='tester1-64bit.html'>64-bit</a>), - conventional worker: - (<a href='tester1-worker.html' target='tester1-worker.html'>32-bit</a>, - <a href='tester1-worker-64bit.html' target='tester1-worker-64bit.html'>64-bit</a>), - ESM in UI thread: - (<a href='tester1-esm.html' target='tester1-esm.html'>32-bit</a>, - <a href='tester1-esm-64bit.html' target='tester1-esm-64bit.html'>64-bit</a>), - ESM worker:</a> - (<a href='tester1-worker.html?esm' target='tester1-worker.html?esm'>32-bit</a> - <a href='tester1-worker-64bit.html?esm' target='tester1-worker-64bit.html?esm'>64-bit</a>) + <div>Variants: + <a href='tester1.html' target='tester1.html'>conventional UI thread</a>, + <a href='tester1-worker.html' target='tester1-worker.html'>conventional worker</a>, + <a href='tester1-esm.html' target='tester1-esm.html'>ESM in UI thread</a>, + <a href='tester1-worker.html?esm' target='tester1-worker.html?esm'>ESM worker</a> </div> <div class='input-wrapper'> <input type='checkbox' id='cb-log-reverse'> diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 083b5eca4..f72e0803f 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -153,10 +153,6 @@ globalThis.sqlite3InitModule = sqlite3InitModule; logClass('error',...args); }; - const debug = (...args)=>{ - console.debug('tester1',...args); - }; - const toss = (...args)=>{ error(...args); throw new Error(args.join(' ')); @@ -214,9 +210,8 @@ globalThis.sqlite3InitModule = sqlite3InitModule; filter.test(error.message) passes. If it's a function, the test passes if filter(error) returns truthy. If it's a string, the test passes if the filter matches the exception message - precisely. If filter is a number then it is compared against - the resultCode property of the exception. In all other cases - the test fails, throwing an Error. + precisely. In all other cases the test fails, throwing an + Error. If it throws, msg is used as the error report unless it's falsy, in which case a default is used. @@ -230,9 +225,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; if(filter instanceof RegExp) pass = filter.test(err.message); else if(filter instanceof Function) pass = filter(err); else if('string' === typeof filter) pass = (err.message === filter); - else if('number' === typeof filter) pass = (err.resultCode === filter); if(!pass){ - console.error("Filter",filter,"rejected exception",err); throw new Error(msg || ("Filter rejected this exception: <<"+err.message+">>")); } return this; @@ -376,88 +369,6 @@ globalThis.sqlite3InitModule = sqlite3InitModule; clearOnInit: true, initialCapacity: 6 }; - -//#if enable-see - /** - Code consolidator for SEE sanity checks for various VFSes. ctor - is the VFS's oo1.DB-type constructor. ctorOptFunc(bool) is a - function which must return a constructor args object for ctor. It - is passed true if the db needs to be cleaned up/unlinked before - opening it (OPFS) and false if not (how that is done is - VFS-dependent). dbUnlink is a function which is expected to - unlink() the db file if the ctorOpfFunc does not do so when - passed true (kvvfs). - - This function initializes the db described by ctorOptFunc(...), - writes some secret info into it, and re-opens it twice to - confirmi that it can be read with an SEE key and cannot be read - without one. - */ - T.seeBaseCheck = function(ctor, ctorOptFunc, dbUnlink){ - let initDb = true; - const tryKey = function(keyKey, key, expectCount){ - let db; - //console.debug('tryKey()',arguments); - try { - if (initDb) { - const ctoropt = ctorOptFunc(initDb); - initDb = false; - db = new ctor({ - ...ctoropt, - [keyKey]: key - }); - db.exec([ - "drop table if exists t;", - "create table t(a);" - ]); - db.close(); - db = null; - // Ensure that it's actually encrypted... - let err; - try { - db = new ctor(ctorOptFunc(false)); - T.assert(db, 'db opened') /* opening is fine, but... */; - db.exec("select 1 from sqlite_schema"); - console.warn("(should not be reached) sessionStorage =", sessionStorage); - } catch (e) { - err = e; - } finally { - db.close() - db = null; - } - T.assert(err, "Expecting an exception") - .assert(capi.SQLITE_NOTADB == err.resultCode, - "Expecting NOTADB"); - }/*initDb*/ - db = new ctor({ - ...ctorOptFunc(false), - [keyKey]: key - }); - db.exec("insert into t(a) values (1),(2)"); - T.assert(expectCount === db.selectValue('select sum(a) from t')); - } finally { - if (db) db.close(); - } - }; - const hexFoo = new Uint8Array([0x66,0x6f,0x6f]/*=="foo"*/); - dbUnlink(); - tryKey('textkey', 'foo', 3); - T.assert( !initDb ); - tryKey('textkey', 'foo', 6); - dbUnlink(); - initDb = true; - tryKey('key', 'foo', 3); - T.assert( !initDb ); - tryKey('key', hexFoo, 6); - dbUnlink(); - initDb = true; - tryKey('hexkey', hexFoo, 3); - T.assert( !initDb ); - tryKey('hexkey', hexFoo, 6); - dbUnlink(); - }, -//#endif enable-see - //////////////////////////////////////////////////////////////////////// // End of infrastructure setup. Now define the tests... //////////////////////////////////////////////////////////////////////// @@ -520,7 +431,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; assert(wasmCtypes.structs[1/*sqlite3_io_methods*/ ].members.xFileSize.offset>0); [ /* Spot-check a handful of constants to make sure they got installed... */ - 'SQLITE_SCHEMA','SQLITE_NULL','SQLITE_UTF8','SQLITE_UTF8_ZT', + 'SQLITE_SCHEMA','SQLITE_NULL','SQLITE_UTF8', 'SQLITE_STATIC', 'SQLITE_DIRECTONLY', 'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE' ].forEach((k)=>T.assert('number' === typeof capi[k])); @@ -1034,7 +945,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; let ptr = k1.pointer; k1.dispose(); T.assert(undefined === k1.pointer). - mustThrowMatching(()=>{k1.$pP=1}, /disposed/); + mustThrowMatching(()=>{k1.$pP=1}, /disposed instance/); }finally{ k1.dispose(); k2.dispose(); @@ -1638,9 +1549,6 @@ globalThis.sqlite3InitModule = sqlite3InitModule; T.assert(blob instanceof Uint8Array). assert(0x68===blob[0] && 0x69===blob[1]); blob = null; - blob = db.selectValue("select ?1", new Uint8Array([97,0,98,0,99]), - sqlite3.capi.SQLITE_TEXT); - T.assert("a\0b\0c"===blob, "Something is amiss with embedded NULs"); let counter = 0, colNames = []; list.length = 0; db.exec(new TextEncoder('utf-8').encode("SELECT a a, b b FROM t"),{ @@ -2635,7 +2543,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .assert(wasm.isPtr(tmplMod.$xRowid)) .assert(wasm.isPtr(tmplMod.$xCreate)) .assert(tmplMod.$xCreate === tmplMod.$xConnect, - "setupModule() must make these equivalent and "+ + "setup() must make these equivalent and "+ "installMethods() must avoid re-compiling identical functions"); tmplMod.$xCreate = wasm.ptr.null /* make tmplMod eponymous-only */; let rc = capi.sqlite3_create_module( @@ -2886,69 +2794,25 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// T.g('kvvfs') .t({ - name: 'kvvfs v1 API availability', + name: 'kvvfs is disabled in worker', + predicate: ()=>(isWorker() || "test is only valid in a Worker"), test: function(sqlite3){ - const capi = sqlite3.capi; - if( isUIThread() ){ - T.assert( !!capi.sqlite3_js_kvvfs_size ) - .assert( !!capi.sqlite3_js_kvvfs_clear ); - }else{ - /* Historical behaviour retained not for compatibility but - to help avoid some confusion between the v1 and v2 kvvfs - APIs (namely in how the v1 variants handle empty - strings). */ - T.assert( !capi.sqlite3_js_kvvfs_clear ) - .assert( !capi.sqlite3_js_kvvfs_size ); - } - const k = sqlite3.kvvfs; - T.assert( k && 'object'===typeof k ); - for(const n of ['reserve', 'import', 'export', - 'unlink', 'listen', 'unlisten', - 'exists', - 'estimateSize', 'clear'] ){ - T.assert( k[n] instanceof Function ); - } - - if( 0 ){ - const scope = wasm.scopedAllocPush(); - try{ - const pg = [ - "53514C69746520666F726D61742033b20b0101b402020d02d02l01d0", - "4l01jb02b2E91E00Dd011FD3b1FD3dxl2B010617171701377461626C", - "656B767666736B7676667302435245415445205441424C45206B76766673286129" - ].join(''); - const n = pg.length; - const pI = wasm.scopedAlloc( n+1 ); - const nO = 8192 * 2; - const pO = wasm.scopedAlloc( nO ); - const heap = wasm.heap8u(); - let i; - for( i=0; i<n; ++i ){ - heap[wasm.ptr.add(pI, i)] = pg.codePointAt(i) & 0xff; - } - heap[wasm.ptr.add(pI, i)] = 0; - const rc = wasm.exports.sqlite3__wasm_kvvfs_decode(pI, pO, nO); - const u = heap.slice(pO, wasm.ptr.add(pO,rc)); - debug("decode rc=", rc, u); - }finally{ - wasm.scopedAllocPop(scope); - } - } + T.assert( + !capi.sqlite3_vfs_find('kvvfs'), + "Expecting kvvfs to be unregistered." + ); } - }/*kvvfs API availability*/) + }) .t({ - name: 'kvvfs sessionStorage', - predicate: ()=>(globalThis.sessionStorage || "sessionStorage is unavailable"), + name: 'kvvfs in main thread', + predicate: ()=>(isUIThread() + || "local/sessionStorage are unavailable in a Worker"), test: function(sqlite3){ - const JDb = sqlite3.oo1.JsStorageDb; + const filename = this.kvvfsDbFile = 'session'; const pVfs = capi.sqlite3_vfs_find('kvvfs'); T.assert(looksLikePtr(pVfs)); - let x = sqlite3.kvvfs.internal.storageForZClass('session'); - T.assert( 0 === x.files.length ) - .assert( globalThis.sessionStorage===x.storage ) - .assert( 'kvvfs-session-' === x.keyPrefix ); - const filename = this.kvvfsDbFile = 'session'; - const unlink = this.kvvfsUnlink = ()=>sqlite3.kvvfs.clear(filename); + const JDb = this.JDb = sqlite3.oo1.JsStorageDb; + const unlink = this.kvvfsUnlink = ()=>JDb.clearStorage(this.kvvfsDbFile); unlink(); let db = new JDb(filename); try { @@ -2956,533 +2820,87 @@ globalThis.sqlite3InitModule = sqlite3InitModule; 'create table kvvfs(a);', 'insert into kvvfs(a) values(1),(2),(3)' ]); - T.assert(3 === db.selectValue('select count(*) from kvvfs')) - .assert( db.storageSize() > 0, "Db size counting is broken" ); - db.close(); - db = undefined; - db = new JDb(filename); - db.exec('insert into kvvfs(a) values(4),(5),(6)'); - T.assert(6 === db.selectValue('select count(*) from kvvfs')); - }finally{ - if( db ) db.close(); - } - //console.debug("sessionStorage",globalThis.sessionStorage); - } - }/*kvvfs sanity checks*/) - .t({ - name: 'transient kvvfs', - //predicate: ()=>false, - test: function(sqlite3){ - const filename = '.' /* preinstalled instance */; - const JDb = sqlite3.oo1.JsStorageDb; - const DB = sqlite3.oo1.DB; - T.mustThrowMatching(()=>new JDb(""), capi.SQLITE_MISUSE); - T.mustThrowMatching(()=>{ - new JDb("this\ns an illegal - contains control characters"); - /* We don't have a way to get error strings from xOpen() - to this point? xOpen() does not have a handle to the - db and SQLite is not calling xGetLastError() to fetch - the error string. */ - }, capi.SQLITE_RANGE); - T.mustThrowMatching(()=>{new JDb("foo-journal");}, - capi.SQLITE_MISUSE); - T.mustThrowMatching(()=>{new JDb("foo-wal");}, - capi.SQLITE_MISUSE); - T.mustThrowMatching(()=>{new JDb("foo-shm");}, - capi.SQLITE_MISUSE); - T.mustThrowMatching(()=>{ - new JDb("01234567890123456789"+ - "01234567890123456789"+ - "01234567890123456789"+ - "01234567890123456789"+ - "01234567890123456789"+ - "01234567890123456789"+ - "0"/*too long*/); - }, capi.SQLITE_RANGE); - { - const name = "01234567890123456789012" /* max name length */; - (new JDb(name)).close(); - T.assert( sqlite3.kvvfs.unlink(name) ); - } - - sqlite3.kvvfs.clear(filename); - let db = new JDb(filename); - const sqlSetup = [ - 'create table kvvfs(a);', - 'insert into kvvfs(a) values(1),(2),(3)' - ]; - try { - T.assert( 0===db.storageSize(), "expecting 0 storage size" ); - T.mustThrowMatching(()=>db.clearStorage(), /in-use/); - //db.clearStorage(); - T.assert( 0===db.storageSize(), "expecting 0 storage size" ); - db.exec(sqlSetup); - T.assert( 0<db.storageSize(), "expecting non-0 db size" ); - T.mustThrowMatching(()=>db.clearStorage(), /in-use/); - //db.clearStorage(/*wiping everything out from under it*/); - T.assert( 0<db.storageSize(), "expecting non-0 storage size" ); - //db.exec(sqlSetup/*that actually worked*/); - /* Clearing the storage out from under the db does actually - work so long as we re-initialize it before reading. - It is tentatively disallowed for sanity's sake rather - than it not working. - */ - T.assert( 0<db.storageSize(), "expecting non-0 db size" ); - const close = ()=>{ - db.close(); - db = undefined; - }; T.assert(3 === db.selectValue('select count(*) from kvvfs')); - close(); - - const exportDb = sqlite3.kvvfs.export; + db.close(); db = new JDb(filename); db.exec('insert into kvvfs(a) values(4),(5),(6)'); T.assert(6 === db.selectValue('select count(*) from kvvfs')); - const exp = exportDb({name:filename,includeJournal:true}); - T.assert( filename===exp.name, "Broken export filename" ) - .assert( exp?.size > 0, "Missing db size" ) - .assert( exp?.pages?.length > 0, "Missing db pages" ); - console.debug("kvvfs to Object:",exp); - close(); - - const dbFileRaw = 'file:new-storage?vfs=kvvfs&delete-on-close=1'; - db = new DB({ - filename: dbFileRaw, - //flags: 'ct' - }); - db.exec(sqlSetup); - const dbFilename = db.dbFilename(); - //console.warn("db.dbFilename() =",dbFilename); - T.assert(3 === db.selectValue('select count(*) from kvvfs')); - debug("kvvfs to Object:",exportDb(dbFilename)); - const n = sqlite3.kvvfs.estimateSize( dbFilename ); - T.assert( n>0, "Db size count failed" ); - - if( 1 ){ - // Concurrent open of that same name uses the same storage - const x = new JDb(dbFilename); - T.assert(3 === db.selectValue('select count(*) from kvvfs')); - x.close(); - } - close(); - // When the final instance of a name is closed, the storage - // disappears... - T.mustThrowMatching(function(){ - /* Ensure that 'new-storage' was deleted when its refcount - went to 0, because of its 'transient' flag. By default - the objects are retained, like a filesystem would. */ - let ddb = new JDb(dbFilename); - try{ddb.selectValue('select a from kvvfs')} - finally{ddb.close()} - }, /no such table: kvvfs/); }finally{ - if( db ) db.close(); - } - } - }/*transient kvvfs*/) - .t({ - name: 'concurrent transient kvvfs', - //predicate: ()=>false, - test: function(sqlite3){ - const filename = 'myStorage'; - const kvvfs = sqlite3.kvvfs; - const DB = sqlite3.oo1.DB; - const JDb = sqlite3.oo1.JsStorageDb; - let db; - let duo; - let q1, q2; - const sqlSetup = [ - 'create table kvvfs(a);', - 'insert into kvvfs(a) values(1),(2),(3)' - ]; - const sqlCount = 'select count(*) from kvvfs'; - - try { - const exportDb = sqlite3.kvvfs.export; - const dbFileRaw = 'file:'+filename+'?vfs=kvvfs&delete-on-close=1'; - sqlite3.kvvfs.clear(filename); - db = new DB(dbFileRaw); - db.exec(sqlSetup); - T.assert(3 === db.selectValue(sqlCount)); - - duo = new JDb(filename); - duo.exec('insert into kvvfs(a) values(4),(5),(6)'); - T.assert(6 === db.selectValue(sqlCount)); - const expOpt = { - name: filename, - decodePages: true - }; - let exp = exportDb(expOpt); - let expectRows = 6; - debug("exported db",exp); db.close(); - T.assert(expectRows === duo.selectValue(sqlCount)); - duo.close(); - T.mustThrowMatching(function(){ - let ddb = new JDb(filename); - try{ddb.selectValue('select a from kvvfs')} - finally{ddb.close()} - }, /.*no such table: kvvfs.*/); - - T.assert( kvvfs.unlink(filename) ) - .assert( !kvvfs.exists(filename) ); - - const importDb = sqlite3.kvvfs.import; - duo = new JDb(dbFileRaw); - T.mustThrowMatching(()=>importDb(exp,true), /.*in use.*/); - duo.close(); - importDb(exp, true); - duo = new JDb(dbFileRaw); - T.assert(expectRows === duo.selectValue(sqlCount)); - let newCount; - try{ - duo.transaction(()=>{ - duo.exec("insert into kvvfs(a) values(7)"); - newCount = duo.selectValue(sqlCount); - T.assert(false, "rolling back"); - }); - }catch(e){/*ignored*/} - T.assert(7===newCount, "Unexpected row count before rollback") - .assert(expectRows === duo.selectValue(sqlCount), - "Unexpected row count after rollback"); - duo.close(); - - T.assert( kvvfs.unlink(filename) ) - .assert( !kvvfs.exists(filename) ); - - importDb(exp, true); - db = new JDb({ - filename, - flags: 'c' - /* BUG: without the 'c' flag, the db works until we try to - vacuum, at which point it fails with "read only db". */ - }); - duo = new JDb(filename); - T.assert(expectRows === duo.selectValue(sqlCount)); - const sqlIns1 = "insert into kvvfs(a) values(?)"; - q1 = db.prepare(sqlIns1); - q2 = duo.prepare(sqlIns1); - if( 0 ){ - q1.bind('from q1').stepFinalize(); - ++expectRows; - T.assert(expectRows === duo.selectValue(sqlCount), - "Unexpected record count."); - q2.bind('from q1').stepFinalize(); - ++expectRows; - }else{ - q1.bind('from q1'); - T.assert(capi.SQLITE_DONE===capi.sqlite3_step(q1), - "Unexpected step result"); - ++expectRows; - T.assert(expectRows === duo.selectValue(sqlCount), - "Unexpected record count."); - q2.bind('from q1').step(); - ++expectRows; - } - T.assert(expectRows === db.selectValue(sqlCount), - "Unexpected record count."); - q1.finalize(); - q2.finalize(); - - if( 1 ){ - debug("Begin vacuum/page size test..."); - const defaultPageSize = 1024 * 8 /* build-time default */; - const pageSize = 0 - ? defaultPageSize - : 1024 * 16 /* any valid value other than the default */; - if( 0 ){ - debug("Export before vacuum", exportDb(expOpt)); - debug("page size before vacuum", - db.selectArray( - "select page_size from pragma_page_size()" - )); - } - //kvvfs.log.xFileControl = true; - //kvvfs.log.xAccess = true; - db.exec([ - "BEGIN;", - "insert into kvvfs(a) values(randomblob(16000/*>pg size*/));", - "COMMIT;", - "delete from kvvfs where octet_length(a)>100;", - "pragma page_size="+pageSize+";", - "vacuum;", - "select 1;" - ]); - const expectPageSize = kvvfs.internal.disablePageSizeChange - ? defaultPageSize - : pageSize; - const gotPageSize = db.selectValue( - "select page_size from pragma_page_size()" - ); - T.assert(+gotPageSize === expectPageSize, - "Expecting page size",expectPageSize, - "got",gotPageSize); - T.assert(expectRows === duo.selectValue(sqlCount), - "Unexpected record count."); - kvvfs.log.xAccess = kvvfs.log.xFileControl = false; - T.assert(expectRows === duo.selectValue(sqlCount), - "Unexpected record count."); - exp = exportDb(expOpt); - debug("Exported page-expanded db",exp); - if( 0 ){ - debug("vacuumed export",exp); - } - debug("End vacuum/page size test."); - }else{ - expectRows = 6; - } - - db.close(); - duo.close(); - T.assert( kvvfs.unlink(exp.name) ) - .assert( !kvvfs.exists(exp.name) ); - importDb(exp); - T.assert( kvvfs.exists(exp.name) ); - db = new JDb(exp.name); - //debug("column count after export",db.selectValue(sqlCount)); - T.assert(expectRows === db.selectValue(sqlCount), - "Unexpected record count."); - - /* - TODO: more advanced concurrent use tests, e.g. looping - over a query in one connection while writing from - another. Currently that will probably corrupt the db, and - kvvfs's journaling does not support multiple journals per - storage unit. We need to test the locking and fix it as - appropriate. - */ - }finally{ - q1?.finalize?.(); - q2?.finalize?.(); - db?.close?.(); - duo?.close?.(); } } - }/*concurrent transient kvvfs*/) - + }/*kvvfs sanity checks*/) +//#if enable-see .t({ - name: 'kvvfs listeners (experiment)', + name: 'kvvfs with SEE encryption', + predicate: ()=>(isUIThread() + || "Only available in main thread."), test: function(sqlite3){ - const kvvfs = sqlite3.kvvfs; - const filename = 'listen'; - let db; - try { - const DB = sqlite3.oo1.DB; - const sqlSetup = [ - 'create table kvvfs(a);', - 'insert into kvvfs(a) values(1),(2),(3)' - ]; - const sqlCount = "select count(*) from kvvfs"; - const sqlSelectSchema = "select * from sqlite_schema"; - const counts = Object.create(null); - const incr = (key)=>counts[key] = 1 + (counts[key] ?? 0); - const pglog = Object.assign(Object.create(null),{ - pages: [], - jrnl: undefined, - size: undefined, - includeJournal: false, - decodePages: true, - exception: new Error("Testing that exceptions from listeners do not interfere") - }); - const toss = ()=>{ - if( pglog.exception ){ - const e = pglog.exception; - delete pglog.exception; - throw e; - } + this.kvvfsUnlink(); + let initDb = true; + const tryKey = function(keyKey, key, expectCount){ + let db; + //console.debug('tryKey()',arguments); + const ctoropt = { + filename: this.kvvfsDbFile + //vfs: 'kvvfs' + //,flags: 'ct' }; - - const listener = { - storage: filename, - reserve: true, - includeJournal: pglog.includeJournal, - decodePages: pglog.decodePages, - events: { - /** - These may be async but must not be in this case - because we can't test their result without a lot of - hoop-jumping if they are. Kvvfs calls these - asynchronously, though. - */ - 'open': (ev)=>{ - //console.warn('open',ev); - incr(ev.type); - T.assert(filename===ev.storageName) - .assert('number'===typeof ev.data); - }, - 'close': (ev)=>{ - //console.warn('close',ev); - incr(ev.type); - T.assert('number'===typeof ev.data); - toss(); - }, - 'delete': (ev)=>{ - //console.warn('delete',ev); - incr(ev.type); - T.assert('string'===typeof ev.data); - switch(ev.data){ - case 'jrnl': - T.assert(pglog.includeJournal); - pglog.jrnl = null; - break; - default:{ - const n = +ev.data; - T.assert( n>0, "Expecting positive db page number" ); - if( n < pglog.pages.length ){ - pglog.size = undefined; - } - pglog.pages[n] = undefined; - break; - } - } - }, - 'sync': (ev)=>{ - incr(ev.data ? 'xSync' : 'xFileControlSync'); - }, - 'write': (ev)=>{ - //console.warn('write',ev); - incr(ev.type); - T.assert(Array.isArray(ev.data)); - const key = ev.data[0], val = ev.data[1]; - T.assert('string'===typeof key); - switch( key ){ - case 'jrnl': - T.assert(pglog.includeJournal); - pglog.jrnl = val; - break; - case 'sz':{ - const sz = +val; - T.assert( sz>0, "Expecting a db page number" ); - if( sz < pglog.sz ){ - pglog.pages.length = sz / pglog.pages.length; - } - pglog.size = sz; - break; - } - default: - T.assert( +key>0, "Expecting a positive db page number" ); - pglog.pages[+key] = val; - if( pglog.decodePages ){ - T.assert( val instanceof Uint8Array ); - }else{ - T.assert( 'string'===typeof val ); - } - break; - } + try { + if (initDb) { + initDb = false; + db = new this.JDb({ + ...ctoropt, + [keyKey]: key + }); + db.exec([ + "drop table if exists t;", + "create table t(a);" + ]); + db.close(); + // Ensure that it's actually encrypted... + let err; + try { + db = new this.JDb(ctoropt); + T.assert(db, 'db opened') /* opening is fine, but... */; + db.exec("select 1 from sqlite_schema"); + console.warn("(should not be reached) sessionStorage =", sessionStorage); + } catch (e) { + err = e; + } finally { + db.close() } - } - }; - - kvvfs.listen(listener); - const dbFileRaw = 'file:'+filename+'?vfs=kvvfs&delete-on-close=1'; - const expOpt = { - name: filename, - //decodePages: true - }; - db = new DB(dbFileRaw); - db.exec(sqlSetup); - T.assert(db.selectObjects(sqlSelectSchema)?.length>0, - "Unexpected empty schema"); - db.close(); - debug("kvvfs listener counts:",counts); - T.assert( counts.open ); - T.assert( counts.close ); - T.assert( listener.includeJournal ? counts.delete : !counts.delete ); - T.assert( counts.write ); - T.assert( counts.xSync ); - T.assert( counts.xFileControlSync>=counts.xSync ); - T.assert( counts.open===counts.close ); - T.assert( pglog.includeJournal - ? (null===pglog.jrnl) - : (undefined===pglog.jrnl), - "Unexpected pglog.jrnl value: "+pglog.jrnl ); - if( 1 ){ - T.assert(undefined===pglog.pages[0], "Expecting empty slot 0"); - pglog.pages.shift(); - //debug("kvvfs listener pageLog", pglog); - } - const before = JSON.stringify(counts); - T.assert( kvvfs.unlisten(listener) ); - T.assert( !kvvfs.unlisten(listener) ); - db = new DB(dbFileRaw); - T.assert( db.selectObjects(sqlSelectSchema)?.length>0 ); - const exp = kvvfs.export(expOpt); - const expectRows = db.selectValue(sqlCount); - db.exec("delete from kvvfs"); - db.close(); - const after = JSON.stringify(counts); - T.assert( before===after, "Expecting no events after unlistening." ); - if( 0 ){ - exp = kvvfs.export(expOpt); - debug("Post-delete export:",exp); - } - if( 1 ){ - // Replace the storage with the pglog state... - const bogoExp = { - name: filename, - size: pglog.size, - timestamp: Date.now(), - pages: pglog.pages - }; - //debug("exp",exp); - //debug("bogoExp",bogoExp); - kvvfs.import(bogoExp, true); - //debug("Re-exported", kvvfs.export(expOpt)); - db = new DB(dbFileRaw); - // Failing on the next line despite exports looking good - T.assert(db.selectObjects(sqlSelectSchema)?.length>0, - "Empty schema on imported db"); - T.assert(expectRows===db.selectValue(sqlCount)); - db.close(); + T.assert(err, "Expecting an exception") + .assert(sqlite3.capi.SQLITE_NOTADB == err.resultCode, + "Expecting NOTADB"); + }/*initDb*/ + //console.debug('tryKey()',arguments); + db = new sqlite3.oo1.DB({ + ...ctoropt, + vfs: 'kvvfs', + [keyKey]: key + }); + db.exec("insert into t(a) values (1),(2)"); + T.assert(expectCount === db.selectValue('select sum(a) from t')); + } finally { + if (db) db.close(); } - }finally{ - db?.close?.(); - kvvfs.unlink(filename); - } - } - })/*kvvfs listeners */ - - .t({ - name: 'kvvfs vtab', - predicate: (sqlite3)=>!!sqlite3.kvvfs.create_module, - test: function(sqlite3){ - const kvvfs = sqlite3.kvvfs; - const db = new sqlite3.oo1.DB(); - const db2 = new sqlite3.oo1.DB('file:foo?vfs=kvvfs&delete-on-close=1'); - try{ - kvvfs.create_module(db); - let rc = db.selectObjects("select * from sqlite_kvvfs order by name"); - debug("sqlite_kvvfs vtab:", rc); - const nDb = rc.length; - rc = db.selectObject("select * from sqlite_kvvfs where name='foo'"); - T.assert(rc, "Expecting foo storage record") - .assert('foo'===rc.name, "Unexpected name") - .assert(1===rc.nRef, "Unexpected refcount"); - db2.close(); - rc = db.selectObjects("select * from sqlite_kvvfs"); - T.assert( !rc.filter((o)=>o.name==='foo').length, - "Expecting foo storage to be gone"); - debug("sqlite_kvvfs vtab:", rc); - T.assert( rc.length+1 === nDb, - "Unexpected storage count: got",rc.length,"expected",nDb); - }finally{ - db.close(); - db2.close(); - } - } - })/* kvvfs vtab */ - -//#if enable-see - .t({ - name: 'kvvfs SEE encryption in sessionStorage', - predicate: ()=>(!!globalThis.sessionStorage - || "sessionStorage is not available"), - test: function(sqlite3){ - const JDb = sqlite3.oo1.JsStorageDb; - T.seeBaseCheck(JDb, - (isInit)=>return {filename: "session"}, - ()=>JDb.clearStorage('session')); + }.bind(this); + const hexFoo = new Uint8Array([0x66,0x6f,0x6f]/*=="foo"*/); + tryKey('textkey', 'foo', 3); + T.assert( !initDb ); + tryKey('textkey', 'foo', 6); + this.kvvfsUnlink(); + initDb = true; + tryKey('key', 'foo', 3); + T.assert( !initDb ); + tryKey('key', hexFoo, 6); + this.kvvfsUnlink(); + initDb = true; + tryKey('hexkey', hexFoo, 3); + T.assert( !initDb ); + tryKey('hexkey', hexFoo, 6); + this.kvvfsUnlink(); } })/*kvvfs with SEE*/ //#endif enable-see @@ -3497,7 +2915,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; let countCommit = 0, countRollback = 0;; const db = new sqlite3.oo1.DB(':memory:',1 ? 'c' : 'ct'); let rc = capi.sqlite3_commit_hook(db, (p)=>{ - //debug("commit hook",arguments); + //console.debug("commit hook",arguments); ++countCommit; return (17 == p) ? 0 : capi.SQLITE_ERROR; }, 17); @@ -3897,14 +3315,71 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .t({ name: 'OPFS with SEE encryption', test: function(sqlite3){ - T.seeBaseCheck( - sqlite3.oo1.OpfsDb, - function(isInit){ - const opt = {filename: 'file:///sqlite3-see.edb'}; - if( isInit ) opt.filename += '?delete-before-open=1'; - return opt; - }, - ()=>{}); + const dbFile = 'file:///sqlite3-see.edb'; + const dbCtor = sqlite3.oo1.OpfsDb; + const hexFoo = new Uint8Array([0x66,0x6f,0x6f]/*=="foo"*/); + let initDb = true; + const tryKey = function(keyKey, key, expectCount){ + let db; + //console.debug('tryKey()',arguments); + const ctoropt = { + filename: dbFile, + flags: 'c' + }; + try { + if (initDb) { + initDb = false; + const opt = { + ...ctoropt, + [keyKey]: key + }; + opt.filename += '?delete-before-open=1'; + db = new dbCtor(opt); + db.exec([ + "drop table if exists t;", + "create table t(a);" + ]); + db.close(); + // Ensure that it's actually encrypted... + let err; + try { + db = new dbCtor(ctoropt); + T.assert(db, 'db opened') /* opening is fine, but... */; + const rv = db.exec({ + sql:"select count(*) from sqlite_schema", + returnValue: 'resultRows' + }); + console.warn("(should not be reached) rv =",rv); + } catch (e) { + err = e; + } finally { + db.close() + } + T.assert(err, "Expecting an exception") + .assert(sqlite3.capi.SQLITE_NOTADB == err.resultCode, + "Expecting NOTADB"); + }/*initDb*/ + db = new dbCtor({ + ...ctoropt, + [keyKey]: key + }); + db.exec("insert into t(a) values (1),(2)"); + T.assert(expectCount === db.selectValue('select sum(a) from t')); + } finally { + if (db) db.close(); + } + }; + tryKey('textkey', 'foo', 3); + T.assert( !initDb ); + tryKey('textkey', 'foo', 6); + initDb = true; + tryKey('key', 'foo', 3); + T.assert( !initDb ); + tryKey('key', hexFoo, 6); + initDb = true; + tryKey('hexkey', hexFoo, 3); + T.assert( !initDb ); + tryKey('hexkey', hexFoo, 6); } })/*OPFS with SEE*/ //#endif enable-see @@ -4109,11 +3584,69 @@ globalThis.sqlite3InitModule = sqlite3InitModule; let poolUtil; const P1 = await inst(poolConfig).then(u=>poolUtil = u).catch(catcher); const dbFile = '/sqlite3-see.edb'; - T.seeBaseCheck( - poolUtil.OpfsSAHPoolDb, - (isInit)=>{return {filename: dbFile}}, - ()=>poolUtil.unlink(dbFile) - ); + const dbCtor = poolUtil.OpfsSAHPoolDb; + const hexFoo = new Uint8Array([0x66,0x6f,0x6f]/*=="foo"*/); + let initDb = true; + const tryKey = function(keyKey, key, expectCount){ + let db; + //console.debug('tryKey()',arguments); + const ctoropt = { + filename: dbFile, + flags: 'c' + }; + try { + if (initDb) { + initDb = false; + poolUtil.unlink(dbFile); + db = new dbCtor({ + ...ctoropt, + [keyKey]: key + }); + db.exec([ + "drop table if exists t;", + "create table t(a);" + ]); + db.close(); + // Ensure that it's actually encrypted... + let err; + try { + db = new dbCtor(ctoropt); + T.assert(db, 'db opened') /* opening is fine, but... */; + const rv = db.exec({ + sql:"select count(*) from sqlite_schema", + returnValue: 'resultRows' + }); + console.warn("(should not be reached) rv =",rv); + } catch (e) { + err = e; + } finally { + db.close() + } + T.assert(err, "Expecting an exception") + .assert(sqlite3.capi.SQLITE_NOTADB == err.resultCode, + "Expecting NOTADB"); + }/*initDb*/ + db = new dbCtor({ + ...ctoropt, + [keyKey]: key + }); + db.exec("insert into t(a) values (1),(2)"); + T.assert(expectCount === db.selectValue('select sum(a) from t')); + } finally { + if (db) db.close(); + } + }; + tryKey('textkey', 'foo', 3); + T.assert( !initDb ); + tryKey('textkey', 'foo', 6); + initDb = true; + tryKey('key', 'foo', 3); + T.assert( !initDb ); + tryKey('key', hexFoo, 6); + initDb = true; + tryKey('hexkey', hexFoo, 3); + T.assert( !initDb ); + tryKey('hexkey', hexFoo, 6); poolUtil.removeVfs(); } })/*opfs-sahpool with SEE*/ @@ -4182,54 +3715,44 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .t("Misc. stmt_...", function(sqlite3){ const db = new sqlite3.oo1.DB(); db.exec("create table t(a doggiebiscuits); insert into t(a) values(123)"); - let stmt; - try{ - stmt = db.prepare("select a, a+1 from t"); - T.assert( stmt.isReadOnly() ) - .assert( 0===capi.sqlite3_stmt_isexplain(stmt) ) - .assert( 0===capi.sqlite3_stmt_explain(stmt, 1) ) - .assert( 0!==capi.sqlite3_stmt_isexplain(stmt) ) - .assert( 0===capi.sqlite3_stmt_explain(stmt, 2) ) - .assert( 0!==capi.sqlite3_stmt_isexplain(stmt) ) - .assert( 0===capi.sqlite3_stmt_explain(stmt, 0) ) - .assert( 0===capi.sqlite3_stmt_isexplain(stmt) ); - let n = 0; - while( capi.SQLITE_ROW === capi.sqlite3_step(stmt) ){ - ++n; - T.assert( 0!==capi.sqlite3_stmt_explain(stmt, 1), - "Because stmt is busy" ) - .assert( capi.sqlite3_stmt_busy(stmt) ) - .assert( stmt.isBusy() ) - .assert( 0!==capi.sqlite3_stmt_readonly(stmt) ) - .assert( true===stmt.isReadOnly() ); - const sv = capi.sqlite3_column_value(stmt, 0); - T.assert( 123===capi.sqlite3_value_int(sv) ) - .assert( "doggiebiscuits"===capi.sqlite3_column_decltype(stmt,0) ) - .assert( null===capi.sqlite3_column_decltype(stmt,1) ); - } - T.assert( 1===n ) - .assert( 0===capi.sqlite3_stmt_busy(stmt) ) - .assert( !stmt.isBusy() ); - - if( wasm.exports.sqlite3_column_origin_name ){ - log("Column metadata APIs enabled"); - T.assert( "t" === capi.sqlite3_column_table_name(stmt, 0)) - .assert("a" === capi.sqlite3_column_origin_name(stmt, 0)) - .assert("main" === capi.sqlite3_column_database_name(stmt, 0)) - }else{ - log("Column metadata APIs not enabled"); - } // column metadata APIs - stmt.finalize(); - stmt = null; - stmt = db.prepare("select ?1").bind(new Uint8Array([97,0,98,0,99])); - stmt.step(); - const sv = capi.sqlite3_column_value(stmt,0); - T.assert("a\0b\0c"===capi.sqlite3_value_text(sv), - "Expecting NULs to have survived."); - }finally{ - if(stmt) stmt.finalize(); - db.close(); - } + const stmt = db.prepare("select a, a+1 from t"); + T.assert( stmt.isReadOnly() ) + .assert( 0===capi.sqlite3_stmt_isexplain(stmt) ) + .assert( 0===capi.sqlite3_stmt_explain(stmt, 1) ) + .assert( 0!==capi.sqlite3_stmt_isexplain(stmt) ) + .assert( 0===capi.sqlite3_stmt_explain(stmt, 2) ) + .assert( 0!==capi.sqlite3_stmt_isexplain(stmt) ) + .assert( 0===capi.sqlite3_stmt_explain(stmt, 0) ) + .assert( 0===capi.sqlite3_stmt_isexplain(stmt) ); + let n = 0; + while( capi.SQLITE_ROW === capi.sqlite3_step(stmt) ){ + ++n; + T.assert( 0!==capi.sqlite3_stmt_explain(stmt, 1), + "Because stmt is busy" ) + .assert( capi.sqlite3_stmt_busy(stmt) ) + .assert( stmt.isBusy() ) + .assert( 0!==capi.sqlite3_stmt_readonly(stmt) ) + .assert( true===stmt.isReadOnly() ); + const sv = capi.sqlite3_column_value(stmt, 0); + T.assert( 123===capi.sqlite3_value_int(sv) ) + .assert( "doggiebiscuits"===capi.sqlite3_column_decltype(stmt,0) ) + .assert( null===capi.sqlite3_column_decltype(stmt,1) ); + } + T.assert( 1===n ) + .assert( 0===capi.sqlite3_stmt_busy(stmt) ) + .assert( !stmt.isBusy() ); + + if( wasm.exports.sqlite3_column_origin_name ){ + log("Column metadata APIs enabled"); + T.assert( "t" === capi.sqlite3_column_table_name(stmt, 0)) + .assert("a" === capi.sqlite3_column_origin_name(stmt, 0)) + .assert("main" === capi.sqlite3_column_database_name(stmt, 0)) + }else{ + log("Column metadata APIs not enabled"); + } // column metadata APIs + + stmt.finalize(); + db.close(); }) //////////////////////////////////////////////////////////////////// diff --git a/main.mk b/main.mk index 46d646d03..ac0fdb513 100644 --- a/main.mk +++ b/main.mk @@ -652,7 +652,7 @@ SRC = \ $(TOP)/src/sqliteInt.h \ $(TOP)/src/sqliteLimit.h \ $(TOP)/src/table.c \ - tclsqlite-ex.c \ + $(TOP)/src/tclsqlite.c \ $(TOP)/src/threads.c \ $(TOP)/src/tokenize.c \ $(TOP)/src/treeview.c \ @@ -952,6 +952,25 @@ FUZZDATA = \ # TESTOPTS = --verbose=file --output=test-out.txt +# +# Extra compiler options for various shell tools +# +# Note that some of these will only apply when embedding sqlite3.c +# into the shell, as these flags are not otherwise passed on to the +# library. +SHELL_OPT += -DSQLITE_DQS=0 +SHELL_OPT += -DSQLITE_ENABLE_FTS4 +#SHELL_OPT += -DSQLITE_ENABLE_FTS5 +SHELL_OPT += -DSQLITE_ENABLE_RTREE +SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS +SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION +SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB +SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB +SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB +SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB +SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC +SHELL_OPT += -DSQLITE_ENABLE_PERCENTILE +SHELL_OPT += -DSQLITE_STRICT_SUBTYPE=1 FUZZERSHELL_OPT = FUZZCHECK_OPT += -I$(TOP)/test FUZZCHECK_OPT += -I$(TOP)/ext/recover @@ -1409,15 +1428,15 @@ whereexpr.o: $(TOP)/src/whereexpr.c $(DEPS_OBJ_COMMON) window.o: $(TOP)/src/window.c $(DEPS_OBJ_COMMON) $(T.cc.sqlite) -c $(TOP)/src/window.c -tclsqlite.o: $(T.tcl.env.sh) tclsqlite-ex.c $(DEPS_OBJ_COMMON) +tclsqlite.o: $(T.tcl.env.sh) $(TOP)/src/tclsqlite.c $(DEPS_OBJ_COMMON) $(T.compile.tcl) -DUSE_TCL_STUBS=1 $$TCL_INCLUDE_SPEC \ - -c tclsqlite-ex.c -o tclsqlite.o + -c $(TOP)/src/tclsqlite.c -tclsqlite-shell.o: $(T.tcl.env.sh) tclsqlite-ex.c $(DEPS_OBJ_COMMON) - $(T.compile.tcl) -DTCLSH -o $@ -c tclsqlite-ex.c $$TCL_INCLUDE_SPEC +tclsqlite-shell.o: $(T.tcl.env.sh) $(TOP)/src/tclsqlite.c $(DEPS_OBJ_COMMON) + $(T.compile.tcl) -DTCLSH -o $@ -c $(TOP)/src/tclsqlite.c $$TCL_INCLUDE_SPEC -tclsqlite-stubs.o: $(T.tcl.env.sh) tclsqlite-ex.c $(DEPS_OBJ_COMMON) - $(T.compile.tcl) -DUSE_TCL_STUBS=1 -o $@ -c tclsqlite-ex.c $$TCL_INCLUDE_SPEC +tclsqlite-stubs.o: $(T.tcl.env.sh) $(TOP)/src/tclsqlite.c $(DEPS_OBJ_COMMON) + $(T.compile.tcl) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c $$TCL_INCLUDE_SPEC # # STATIC_TCLSQLITE3 = 1 to statically link tclsqlite3, else @@ -1661,19 +1680,11 @@ install-tcl-0 install-tcl-: install-tcl: install-tcl-$(HAVE_TCL) install: install-tcl -TCLSQLITEEX = \ - $(TOP)/ext/qrf/qrf.h \ - $(TOP)/ext/qrf/qrf.c \ - $(TOP)/src/tclsqlite.c - -tclsqlite-ex.c: $(TCLSQLITEEX) $(TOP)/tool/mkcombo.tcl $(B.tclsh) - $(B.tclsh) $(TOP)/tool/mkcombo.tcl $(TCLSQLITEEX) -o $@ - -tclsqlite3.c: sqlite3.c tclsqlite-ex.c +tclsqlite3.c: sqlite3.c echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c cat sqlite3.c >>tclsqlite3.c echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c - cat tclsqlite-ex.c >>tclsqlite3.c + cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c # # $(CFLAGS.tclextension) = CFLAGS for the tclextension* targets. @@ -1790,13 +1801,13 @@ TESTFIXTURE_FLAGS += -DSQLITE_STRICT_SUBTYPE=1 TESTFIXTURE_SRC0 = $(TESTSRC2) $(libsqlite3.LIB) TESTFIXTURE_SRC1 = sqlite3.c -TESTFIXTURE_SRC = $(TESTSRC) tclsqlite-ex.c +TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION)) testfixture$(T.exe): $(T.tcl.env.sh) has_tclsh85 $(TESTFIXTURE_SRC) $(T.link.tcl) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \ -o $@ $(TESTFIXTURE_SRC) \ - $$TCL_LIB_SPEC $$TCL_INCLUDE_SPEC $$TCL_LIBS \ + $$TCL_LIB_SPEC $$TCL_INCLUDE_SPEC \ $(LDFLAGS.libsqlite3) coretestprogs: testfixture$(B.exe) sqlite3$(B.exe) @@ -1863,15 +1874,6 @@ mdevtest: srctree-check has_tclsh85 sdevtest: has_tclsh85 $(TCLSH_CMD) $(TOP)/test/testrunner.tcl sdevtest $(TSTRNNR_OPTS) -retest: has_tclsh85 - $(TCLSH_CMD) $(TOP)/test/testrunner.tcl retest - -errors: - $(TCLSH_CMD) $(TOP)/test/testrunner.tcl errors - -status: - $(TCLSH_CMD) $(TOP)/test/testrunner.tcl status -d 2 - # Like releasetest, except it omits srctree-check and verify-source so # that it can be used on a modified source tree. # @@ -1929,7 +1931,7 @@ shelltest: # sqlite3_analyzer.c.flags.0 = -DINCLUDE_SQLITE3_C=1 sqlite3_analyzer.c.flags.1 = -sqlite3_analyzer.c: sqlite3.c tclsqlite-ex.c $(TOP)/tool/spaceanal.tcl \ +sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl \ $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in $(B.tclsh) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in \ $(sqlite3_analyzer.c.flags.$(LINK_TOOLS_DYNAMICALLY)) \ @@ -1953,7 +1955,7 @@ sqlite3_analyzer$(T.exe): $(T.tcl.env.sh) sqlite3_analyzer.c \ # can cause the $@ to link to an out-of-tree libsqlite3.so, which may # or may not fail or otherwise cause confusion. -sqltclsh.c: sqlite3.c tclsqlite-ex.c $(TOP)/tool/sqltclsh.tcl \ +sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl \ $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl \ $(TOP)/tool/sqltclsh.c.in $(B.tclsh) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c @@ -1971,6 +1973,24 @@ sqlite3_expert$(T.exe): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqli $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(LDFLAGS.libsqlite3) xbin: sqlite3_expert$(T.exe) +CHECKER_DEPS =\ + $(TOP)/tool/mkccode.tcl \ + sqlite3.c \ + $(TOP)/src/tclsqlite.c \ + $(TOP)/ext/repair/sqlite3_checker.tcl \ + $(TOP)/ext/repair/checkindex.c \ + $(TOP)/ext/repair/checkfreelist.c \ + $(TOP)/ext/misc/btreeinfo.c \ + $(TOP)/ext/repair/sqlite3_checker.c.in + +sqlite3_checker.c: $(CHECKER_DEPS) + $(B.tclsh) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@ + +sqlite3_checker$(T.exe): $(T.tcl.env.sh) sqlite3_checker.c + $(T.link.tcl) sqlite3_checker.c -o $@ $$TCL_INCLUDE_SPEC \ + $$TCL_LIB_SPEC $(LDFLAGS.libsqlite3) +xbin: sqlite3_checker$(T.exe) + dbdump$(T.exe): $(TOP)/ext/misc/dbdump.c sqlite3.o $(T.link) -DDBDUMP_STANDALONE -o $@ \ $(TOP)/ext/misc/dbdump.c sqlite3.o $(LDFLAGS.libsqlite3) @@ -2000,10 +2020,6 @@ showshm$(T.exe): $(TOP)/tool/showshm.c $(T.link) -o $@ $(TOP)/tool/showshm.c $(LDFLAGS.configure) xbin: showshm$(T.exe) -showtmlog$(T.exe): $(TOP)/tool/showtmlog.c - $(T.link) -o $@ $(TOP)/tool/showtmlog.c $(LDFLAGS.configure) -xbin: showshm$(T.exe) - index_usage$(T.exe): $(TOP)/tool/index_usage.c sqlite3.o $(T.link) $(SHELL_OPT) -o $@ $(TOP)/tool/index_usage.c sqlite3.o \ $(LDFLAGS.libsqlite3) @@ -2102,19 +2118,16 @@ src-archives: sqlite-amalgamation.zip amalgamation-tarball sqlite-src.zip # Build a ZIP archive containing various command-line tools. # -tool-zip: sqlite3$(T.exe) sqldiff$(T.exe) \ +tool-zip: testfixture$(T.exe) sqlite3$(T.exe) sqldiff$(T.exe) \ sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) $(TOP)/tool/mktoolzip.tcl strip sqlite3$(T.exe) sqldiff$(T.exe) sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) - $(TCLSH_CMD) $(TOP)/tool/mktoolzip.tcl - + ./testfixture$(T.exe) $(TOP)/tool/mktoolzip.tcl snapshot-zip: testfixture$(T.exe) sqlite3$(T.exe) sqldiff$(T.exe) \ sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) $(TOP)/tool/mktoolzip.tcl strip sqlite3$(T.exe) sqldiff$(T.exe) sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) - $(TCLSH_CMD) $(TOP)/tool/mktoolzip.tcl --snapshot - + ./testfixture$(T.exe) $(TOP)/tool/mktoolzip.tcl --snapshot clean-tool-zip: rm -f sqlite-tools-*.zip - clean: clean-tool-zip # @@ -2339,8 +2352,6 @@ mptest: mptester$(T.exe) # Source and header files that shell.c depends on SHELL_DEP = \ $(TOP)/src/shell.c.in \ - $(TOP)/ext/qrf/qrf.c \ - $(TOP)/ext/qrf/qrf.h \ $(TOP)/ext/expert/sqlite3expert.c \ $(TOP)/ext/expert/sqlite3expert.h \ $(TOP)/ext/intck/sqlite3intck.c \ diff --git a/make.bat b/make.bat deleted file mode 100644 index 2dc9b61c0..000000000 --- a/make.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off -nmake /f Makefile.msc %* diff --git a/manifest b/manifest index 079aacfd7..648b118a6 100644 --- a/manifest +++ b/manifest @@ -1,14 +1,14 @@ -C Version\s3.52.0 -D 2026-03-06T16:01:44.367 +C Version\s3.51.3 +D 2026-03-13T10:38:09.694 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md 6bc480fc673fb4acbc4094e77edb326267dd460162d7723c7f30bee2d3d9e97d F Makefile.in 3ce07126d7e87c7464301482e161fdae6a51d0a2aa06b200b8f0000ef4d6163b F Makefile.linux-generic bd3e3cacd369821a6241d4ea1967395c962dfe3057e38cb0a435cee0e8b789d0 -F Makefile.msc 174764cb7e80c80f9003c46b3e388d74c68c8c40230208904b3af8fcabee5f4e -F README.md 3fa51fc7ababc32edd175ae8b2986c86d5ea120c1cb1e57c7f7849492d1405ec -F VERSION 74672bfd4c7826c0fc6f84762488a707c52e7d2d94af42ccb0edcc6c74311c41 +F Makefile.msc d4459fad28b388063698cbb7a73bfce8684da998a844a04b21d4b9b10291196a +F README.md dae499194b75deed76a13a4a83c82493f2530331882d7dfe5754d63287d3f8f7 +F VERSION 04eea51aa150c240c9a4dbd740393c575e73b04428482dc2b14ca6b98967b53c F art/icon-243x273.gif 9750b734f82fdb3dc43127753d5e6fbf3b62c9f4e136c2fbf573b2f57ea87af5 F art/icon-80x90.gif 65509ce3e5f86a9cd64fe7fca2d23954199f31fe44c1e09e208c80fb83d87031 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -17,8 +17,8 @@ F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F art/sqlite370.svg 40b7e2fe8aac3add5d56dd86ab8d427a4eca5bcb3fe4f8946cb3794e1821d531 F auto.def 44a0d1bf09d78355fc88251ccbf8e64e6341fd89c11de68a01c3645e53a2bade F autoconf/Makefile.fallback 22fe523eb36dfce31e0f6349f782eb084e86a5620b2b0b4f84a2d6133f53f5ac -F autoconf/Makefile.in efab1f56c7961d301efc030baa4391b9faae38733d94385a3d56b54720a74aba -F autoconf/Makefile.msc e6c596e6e63ea17aa7a97eb8ded18cd984c8dd4203766644fbd57b8fcf720225 +F autoconf/Makefile.in 118aa2c4d49173672d065fdda19eb8a28642e2c684212d7a626d6db5e6762521 +F autoconf/Makefile.msc 9c1ca648062fd5a4a83ba7590c4422090cccd6399002af0346b7572f086c7483 F autoconf/README.first f1d3876e9a7852c22f275a6f06814e64934cecbc0b5b9617d64849094c1fd136 F autoconf/README.txt b749816b8452b3af994dc6d607394bef3df1736d7e09359f1087de8439a52807 F autoconf/auto.def 3d994f3a9cc9b712dbce92a5708570ddcf3b988141b6eb738f2ed16127a9f0ac @@ -33,7 +33,7 @@ F autoconf/tea/teaish.tcl 81feb417e718ed75cdd7e2fdf6771f3da80dae97377a90c4d5b62b F autoconf/tea/teaish.test.tcl cfe94e1fb79dd078f650295be59843d470125e0cc3a17a1414c1fb8d77f4aea6 F autosetup/LICENSE 41a26aebdd2cd185d1e2b210f71b7ce234496979f6b35aef2cbf6b80cbed4ce4 F autosetup/README.autosetup a78ff8c4a3d2636a4268736672a74bf14a82f42687fcf0631a70c516075c031e -F autosetup/README.md e8503618f6cb31b692383ca662f6eb9ef4345463f5e3c1fc84a5d6863c7058be +F autosetup/README.md ce0f95980a687bb861bd830b76bc4b48513567be5cf5ee7004f4f3439ffe3841 F autosetup/autosetup b16e44924c197783df67366762dda985b45d49ebc4af15f4054e3ee0e3b65169 x F autosetup/autosetup-config.guess dfa101c5e8220e864d5e9c72a85e87110df60260d36cb951ad0a85d6d9eaa463 x F autosetup/autosetup-config.sub a38fb074d0dece01cf919e9fb534a26011608aa8fa606490864295328526cd73 x @@ -44,10 +44,10 @@ F autosetup/cc-lib.tcl 493c5935b5dd3bf9bd4eca89b07c8b1b1a9356d61783035144e21795f F autosetup/cc-shared.tcl 163eda58c14cd662fd8a504bd2ad8a716ef4db7015dc1de0095d5de8dd601a4b F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e45f F autosetup/find_tclconfig.tcl e64886ffe3b982d4df42cd28ed91fe0b5940c2c5785e126c1821baf61bc86a7e -F autosetup/jimsh0.c 916bbdf8023fbda9937afae57d81a853d8c2ea00f2320aa27becbc33574f963d +F autosetup/jimsh0.c a57c16e65dcffc9c76e496757cb3f7fb47e01ecbd1631a0a5e01751fc856f049 F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba -F autosetup/proj.tcl ce301197f364f7ce2acabbbd84b43d19e917ec73653157ca134a06f32d322712 -F autosetup/sqlite-config.tcl 7463d59c9c5e86ca286ea16fdab943058beb9346110049eca435154795890f71 +F autosetup/proj.tcl 6fc14ef82b19b77a95788ffbcfad7989b4e3cb4ce96a21dcb5cf7312f362fba9 +F autosetup/sqlite-config.tcl 5d779fce20c11fde3fe99d157dcd5b5569d729b301141b8dfb8d5aacf9d48cba F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9 F autosetup/teaish/README.txt b40071e6f8506500a2f7f71d5fc69e0bf87b9d7678dd9da1e5b4d0acbf40b1ca F autosetup/teaish/core.tcl e014dd95900c7f9a34e8e0f460f47e94841059827bce8b4c49668b0c7ae3f1a0 @@ -56,14 +56,14 @@ F autosetup/teaish/tester.tcl 1799514c2652db49561b3386c5242b94534d1663f2cfac861a F configure 9a00b21dfd13757bbfb8d89b30660a89ec1f8f3a79402b8f9f9b6fc475c3303a x F contrib/sqlitecon.tcl eb4c6578e08dd353263958da0dc620f8400b869a50d06e271ab0be85a51a08d3 F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd -F doc/compile-for-unix.md c8f05bf9ff8c588c501515eb11642540572203e53d0b5eb5bf60983acdd47643 -F doc/compile-for-windows.md 36601c95fa4070eebfe757684271d17a7c4a586912ba706d0b5e7817e1df54ad +F doc/compile-for-unix.md c9dce1ddd4bf0d25efccc5c63eb047e78c01ce06a6ff29c73e0a8af4a0f4adbc +F doc/compile-for-windows.md f9e74d74da88f384edd5809f825035e071608f00f7f39c0e448df7b3982f979c F doc/json-enhancements.md e356fc834781f1f1aa22ee300027a270b2c960122468499bf347bb123ce1ea4f F doc/jsonb.md acd77fc3a709f51242655ad7803510c886aa8304202fa9cf2abc5f5c4e9d7ae5 -F doc/lemon.html 2085fda0a90a94fe92159a79dccc5c30d5a2313524887a31659cd66162f17233 +F doc/lemon.html 89ea833a6f71773ab1a9063fbb7fb9b32147bc0b1057b53ecab94a3b30c0aef5 F doc/pager-invariants.txt 83aa3a4724b2d7970cc3f3461f0295c46d4fc19a835a5781cbb35cb52feb0577 F doc/tcl-extension-testing.md b88861804fc1eaf83249f8e206334189b61e150c360e1b80d0dcf91af82354f5 -F doc/testrunner.md daffa0ebbbe397a73537ae1b19b3124d489ce5f89dfe570781d1f1ef1809597c +F doc/testrunner.md 5ee928637e03f136a25fef852c5ed975932e31927bd9b05a574424ae18c31019 F doc/trusted-schema.md 33625008620e879c7bcfbbfa079587612c434fa094d338b08242288d358c3e8a F doc/vdbesort-memory.md 4da2639c14cd24a31e0af694b1a8dd37eaf277aff3867e9a8cc14046bc49df56 F doc/vfs-shm.txt 1a55f3f0e7b6745931b117ba5c9df3640d7a0536f532ef0052563100f4416f86 @@ -71,7 +71,7 @@ F doc/wal-lock.md 7db0cd61e2000b545b78ce89b0c2a9a8dd8d64c097839258ac10d7c5c4156e F ext/README.md 6eb1ac267d917767952ed0ef63f55de003b6a5da433ce1fa389e1a9532e73132 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3 F ext/expert/expert.c d548d603a4cc9e61f446cc179c120c6713511c413f82a4a32b1e1e69d3f086a4 -F ext/expert/expert1.test d9dfbf7fb527cfd43049e30a6238ef02c94484041fa4461ed41acbc6435425d6 +F ext/expert/expert1.test 1d2da6606623b57bb47064e02140823ce1daecd4cacbf402c73ad3473d7f000c F ext/expert/sqlite3expert.c 546010043fbec93544f762de5161b3d553165859e6bd853c4b85c05f93484260 F ext/expert/sqlite3expert.h ca81efc2679a92373a13a3e76a6138d0310e32be53d6c3bfaedabd158ea8969b F ext/expert/test_expert.c c395134bd6d4efa594a7d26578a1cb624c4027b79b4b5fcd44736c5ef1f5f725 @@ -79,9 +79,9 @@ F ext/fts3/README.content b9078d0843a094d86af0d48dffbff13c906702b4c3558012e67b9c F ext/fts3/README.syntax b72477722e9b4fe43f8403227d790a1c94221bfad15c27863a4b36d1052e892b F ext/fts3/README.tokenizers b92bdeb8b46503f0dd301d364efc5ef59ef9fa8e2758b8e742f39fa93a2e422d F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 6cc7bbc307f27e7b6ee2e1d5ff63ffff4df3b42529dfe00eb34ddded417961b3 +F ext/fts3/fts3.c 4f02858ab845a97bedf06e6cc1fba49a81fe5e00a26df68d0ad0f00a5814fa70 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h fd6051f7aa4db93e05fdc703ef35faf79f78170419e809139109d7aef28f4170 +F ext/fts3/fts3Int.h ed9b8bc5ed5be402069651e49d4855cb849af706cf3fe68548f58a2c21eefc7f F ext/fts3/fts3_aux.c 7eab82a9cf0830f6551ba3abfdbe73ed39e322a4d3940ee82fbf723674ecd9f3 F ext/fts3/fts3_expr.c 5c13796638d8192c388777166075cdc8bc4b6712024cd5b72c31acdbefce5984 F ext/fts3/fts3_hash.c d9dba473741445789330c7513d4f65737c92df23c3212784312931641814672a @@ -97,7 +97,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c c1de4ae28356ad98ccb8b2e3388a7fdcce7607b5523738c9afb6275dab765154 F ext/fts3/fts3_unicode.c de426ff05c1c2e7bce161cf6b706638419c3a1d9c2667de9cb9dc0458c18e226 F ext/fts3/fts3_unicode2.c 416eb7e1e81142703520d284b768ca2751d40e31fa912cae24ba74860532bf0f -F ext/fts3/fts3_write.c d218b687fb55bce8c9340c6dbb368a10d94647cbe39801d85492d576a4e7da75 +F ext/fts3/fts3_write.c 2bee1c5828f6401adffd07ca64260aeb79d64138958273a56de8fa5e8759a0c1 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73 F ext/fts3/tool/fts3view.c 413c346399159df81f86c4928b7c4a455caab73bfbc8cd68f950f632e5751674 @@ -108,21 +108,21 @@ F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a0 F ext/fts5/extract_api_docs.tcl 009cf59c77afa86d137b0cca3e3b1a5efbe2264faa2df233f9a7aa8563926d15 F ext/fts5/fts5.h ff5d3cc88b29e41612bfb29eb723e29e38973de62ca75ba3e8f94ccb67f5b5f2 F ext/fts5/fts5Int.h 8d98f8e180fe28d6067e240ed45b9011735d29d5cfb5bac194e1e376baa7c708 -F ext/fts5/fts5_aux.c 042da27e97d38071312c111cf18f3cb7983b75ba5e724aa1c3164e61e90f428a -F ext/fts5/fts5_buffer.c dcc3f0352339fe79c9d8abbc1c2009bc3469206467880bf43558447ef4f846fb -F ext/fts5/fts5_config.c bfba970fe1e4eed18ee57c8d51458e226db9a960ddf775c5e50e3d76603a667e -F ext/fts5/fts5_expr.c 71d48e8cf0358deace4949276647d317ff7665db6db09f40b81e2e7fe6664c7c -F ext/fts5/fts5_hash.c d5871df92ce3fa210a650cf419ee916b87c29977e86084d06612edf772bff6f5 -F ext/fts5/fts5_index.c f8cfa37bb7397e5ede20242e4c9cb030bc8b4584ce3f23a5e2495038c0ae64bd -F ext/fts5/fts5_main.c 6889f1373c469d515e792fb3d783c2218e63c560433ebd66edc0f740ab086c1b +F ext/fts5/fts5_aux.c da4a7a9a11ec15c6df0699d908915a209bcde48f0b04101461316b59f71abffb +F ext/fts5/fts5_buffer.c f1e6d0324d7c55329d340673befc26681a372a4d36086caa8d1ec7d7c53066c7 +F ext/fts5/fts5_config.c e7d8dd062b44a66cd77e5a0f74f23a2354cd1f3f8575afb967b2773c3384f7f8 +F ext/fts5/fts5_expr.c b8c32da1127bafaf10d6b4768b0dcb92285798524bed2d87a8686f99a8e8d259 +F ext/fts5/fts5_hash.c a6266cedd801ab7964fa9e74ebcdda6d30ec6a96107fa24148ec6b7b5b80f6e0 +F ext/fts5/fts5_index.c f0562b4adb9dc2d56addcb8833edab50817725032b1cbd46335c0b32d7f1525d +F ext/fts5/fts5_main.c 42025174a556257287071e90516d3ab8115daf1dd525a301883544469a260014 F ext/fts5/fts5_storage.c 19bc7c4cbe1e6a2dd9849ef7d84b5ca1fcbf194cefc3e386b901e00e08bf05c2 -F ext/fts5/fts5_tcl.c 2be6cc14f9448f720fd4418339cd202961a0801ea9424cb3d9de946f8f5a051c +F ext/fts5/fts5_tcl.c 7fb5a3d3404099075aaa2457307cb459bbc257c0de3dbd52b1e80a5b503e0329 F ext/fts5/fts5_test_mi.c 4308d5658cb1f5eee5998dcbaac7d5bdf7a2ef43c8192ca6e0c843f856ccee26 -F ext/fts5/fts5_test_tok.c 6021033bd4f4feffe8579efb6e1f58156ed462256bf99a2acdbd629246529204 -F ext/fts5/fts5_tokenize.c cfc16dde905552fe238c0403670852e75c0330ba508a9fb4836c1f596618561d +F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b +F ext/fts5/fts5_tokenize.c 49aea8cc400a690a6c4f83c4cedc67f4f8830c6789c4ee343404f62bcaebca7b F ext/fts5/fts5_unicode2.c 536a6dae41d16edadd6a6b58c56e2ebbb133f0dfe757562a2edbcdc9b8362e50 F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d09885d1dadf80 -F ext/fts5/fts5_vocab.c bebee4aabcd056a44b3731166433cfdecf17ece750c08cb58733216222bd39e2 +F ext/fts5/fts5_vocab.c 23e263ad94ac357cfffd19bd7e001c3f15c4420fb10fa35b5993142127e780e6 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl c5aa7cf7148b6dcffb5b61520ae18212baf169936af734ab265143f59db328fe @@ -168,7 +168,7 @@ F ext/fts5/test/fts5corrupt5.test 73985d4fe6d8f0d5d5c7bcf79ae7c6522c376cd6ad710a F ext/fts5/test/fts5corrupt6.test 2d72db743db7b5d9c9a6d0cfef24d799ed1aa5e8192b66c40e871a37ed9eed06 F ext/fts5/test/fts5corrupt7.test 814aab492d7a09abb5bfdd81cc66fc206d7f3868f9a3bae91876e02efc466fb3 F ext/fts5/test/fts5corrupt8.test 0b10750caf8aa23fa1c379ca4caf6130d41454505e4d5315590f4061eedcbe44 -F ext/fts5/test/fts5corrupt9.test 4253b9b59f33effac8b67da72ec34309c738aca2d5e8e2656bfbbd6a489a1dfe +F ext/fts5/test/fts5corrupt9.test 65a3a64f0e930fcc37e0389ae26224a4faba3a2c938d2f8eb4811a45b5b495fe F ext/fts5/test/fts5corruptbig.test 9f95b40fa36e292feceab02b2ef06e21878bfa1ac7afefa138aae05518b51774 F ext/fts5/test/fts5delete.test 2a5008f8b1174ef41d1974e606928c20e4f9da77d9f8347aed818994d89cced4 F ext/fts5/test/fts5detail.test 54015e9c43ec4ba542cfb93268abdf280e0300f350efd08ee411284b03595cc4 @@ -199,9 +199,9 @@ F ext/fts5/test/fts5first.test bfd685b96905bf541d99d8644e0a7219d1d833455a08ab64e F ext/fts5/test/fts5full.test 97d263c1072f4a560929cca31e70f65d2ae232610e17e6affcf7e979df59547b F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e F ext/fts5/test/fts5hash.test fd3e0367fbf0b0944d6936fdb22696350f57b9871069c6766251578a103e8a14 -F ext/fts5/test/fts5integrity.test 613efcebe16b2d7a4096f03bcfb164f79a000b3354420ceda4a6f3e035090789 +F ext/fts5/test/fts5integrity.test c423ce16fd1ccadcac7fc22f794226b2bb00f5a187c0ab1d9f8502521b1bae05 F ext/fts5/test/fts5integrity2.test 4c3636615c0201232c44a8105d5cb14fd5499fd0ee3014d7ffd7e83aac76ece8 -F ext/fts5/test/fts5interrupt.test af7834ac6c2e71c05aea42d92f272eef3655e89b7a14a5620a2cd9de35e2e8ea +F ext/fts5/test/fts5interrupt.test 20d04204d3e341b104c0c24a41596b6393a3a81eba1044c168db0e106f9ac92c F ext/fts5/test/fts5join.test 48b7ed36956948c5b8456c8bcaa5b087808d99000675918a43c4f51a925f1514 F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1 F ext/fts5/test/fts5leftjoin.test 1c14b51f4d1344a89e488160882f05a2246dd7e70c5cf077c8fb473e03c66338 @@ -285,14 +285,14 @@ F ext/intck/intck_common.tcl a61fd2697ae55b0a3d89847ca0b590c6e0d8ff64bebb70920d9 F ext/intck/intckbusy.test d5ed4ef85a4b1dc1dee2484bd14a4bb68529659cca743327df0c775f005fa387 F ext/intck/intckcorrupt.test f6c302792326fb3db9dcfc70b554c55369bc4b52882eaaf039cfe0b74c821029 F ext/intck/intckfault.test cff3f75dff74abb3edfcb13f6aa53f6436746ab64b09fe5e2028f051e985efab -F ext/intck/sqlite3intck.c 3c4a166645a1624731f63acd342e24e81e4ffd497116d94a427d72e6cc6caa69 +F ext/intck/sqlite3intck.c b1c8a86f90fc00741d13314db9c58f7e2f92d1d19c5ad1c6904ec83a6bbd5c96 F ext/intck/sqlite3intck.h 2b40c38e7063ab822c974c0bd4aed97dabb579ccfe2e180a4639bb3bbef0f1c9 F ext/intck/test_intck.c 4f9eaadaedccb9df1d26ba41116a0a8e5b0c5556dc3098c8ff68633adcccdea8 F ext/jni/GNUmakefile 8a94e3a1953b88cf117fb2a5380480feada8b4f5316f02572cab425030a720b4 -F ext/jni/README.md 1479c83dbe26125264a060ee6873531795a7082dbc0d43c4067683371331559f +F ext/jni/README.md e3fbd47c774683539b7fdc95a667eb9cd6e64d8510f3ee327e7fa0c61c8aa787 F ext/jni/jar-dist.make 030aaa4ae71dd86e4ec5e7c1e6cd86f9dfa47c4592c070d2e35157e42498e1fa -F ext/jni/src/c/sqlite3-jni.c 6ccc50b0e98b8ef8fd6a8b837223e1c8ebb92bfe6b976128cc757c26257d994a -F ext/jni/src/c/sqlite3-jni.h df43024cced914c49485633d0f90168689e70577b3b17b0ecbdaf16e4a417bff +F ext/jni/src/c/sqlite3-jni.c 3d84a0176af779737ae977ba1c90d2ffe2537b8299c5d9f6552620493f12ac4b +F ext/jni/src/c/sqlite3-jni.h ac180ba9b21978727006c790d3006a95a2402a4c3ec7a0def92707ed89b79945 F ext/jni/src/org/sqlite/jni/annotation/Experimental.java 8603498634e41d0f7c70f661f64e05df64376562ea8f126829fd1e0cdd47e82b F ext/jni/src/org/sqlite/jni/annotation/NotNull.java be6cc3e8e114485822331630097cc0f816377e8503af2fc02f9305ff2b353917 F ext/jni/src/org/sqlite/jni/annotation/Nullable.java 56e3dee1f3f703a545dfdeddc1c3d64d1581172b1ad01ffcae95c18547fafd90 @@ -302,7 +302,7 @@ F ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java 0e28a0df51368c7127e505f F ext/jni/src/org/sqlite/jni/capi/AuthorizerCallback.java c045a5b47e02bb5f1af91973814a905f12048c428a3504fbc5266d1c1be3de5a F ext/jni/src/org/sqlite/jni/capi/AutoExtensionCallback.java 74cc4998a73d6563542ecb90804a3c4f4e828cb4bd69e61226d1a51f4646e759 F ext/jni/src/org/sqlite/jni/capi/BusyHandlerCallback.java 7b8e19810c42b0ad21a04b5d8c804b32ee5905d137148703f16a75b612c380ca -F ext/jni/src/org/sqlite/jni/capi/CApi.java a9701cbe736b8bef5bc72ae465be0250e137f67bdb5a3ab62497972b6f51572d +F ext/jni/src/org/sqlite/jni/capi/CApi.java 3d275f5f4fbdbe4fff15f4d42cf5ff559f9a4897e7373fa99f3b1dc9cf7f849c F ext/jni/src/org/sqlite/jni/capi/CallbackProxy.java 1b3baf5b772f266e8baf8f35f0ddc5bd87fc8c4927ec69115c46fd6fba6701c3 F ext/jni/src/org/sqlite/jni/capi/CollationCallback.java e29bcfc540fdd343e2f5cca4d27235113f2886acb13380686756d5cabdfd065a F ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java 5bfa226a8e7a92e804fd52d6e42b4c7b875fa7a94f8e2c330af8cc244a8920ab @@ -320,7 +320,7 @@ F ext/jni/src/org/sqlite/jni/capi/SQLFunction.java 0d1e9afc9ff8a2adb94a155b72385 F ext/jni/src/org/sqlite/jni/capi/SQLTester.java 3c0babc067d8560627a9ed1b07979f9d4393464e2282c2fca4832052e982c7bc F ext/jni/src/org/sqlite/jni/capi/ScalarFunction.java 93b9700fca4c68075ccab12fe0fbbc76c91cafc9f368e835b9bd7cd7732c8615 F ext/jni/src/org/sqlite/jni/capi/TableColumnMetadata.java 9133bb7685901d2edf07801191284975e33b5583ce09dce1c05202ff91e7bb99 -F ext/jni/src/org/sqlite/jni/capi/Tester1.java 378d142435d220b20b7ce7343c62a03e853bb8c51e80447ee0f8ac5c37362d9a +F ext/jni/src/org/sqlite/jni/capi/Tester1.java 4c3d16fdf6e979f839b2ecdb14d0a0c04bd3d0e41500fc9e8110b588883b140b F ext/jni/src/org/sqlite/jni/capi/TraceV2Callback.java 0a25e117a0daae3394a77f24713e36d7b44c67d6e6d30e9e1d56a63442eef723 F ext/jni/src/org/sqlite/jni/capi/UpdateHookCallback.java c8bdf7848e6599115d601bcc9427ff902cb33129b9be32870ac6808e04b6ae56 F ext/jni/src/org/sqlite/jni/capi/ValueHolder.java 2ce069f3e007fdbbe1f4e507a5a407fc9679da31a0aa40985e6317ed4d5ec7b5 @@ -347,7 +347,7 @@ F ext/jni/src/org/sqlite/jni/test-script-interpreter.md d7987b432870d23f7c72a780 F ext/jni/src/org/sqlite/jni/wrapper1/AggregateFunction.java d5c108b02afd3c63c9e5e53f71f85273c1bfdc461ae526e0a0bb2b25e4df6483 F ext/jni/src/org/sqlite/jni/wrapper1/ScalarFunction.java 326ffba29aab836a6ea189703c3d7fb573305fd93da2d14b0f9e9dcf314c8290 F ext/jni/src/org/sqlite/jni/wrapper1/SqlFunction.java e920f7a031e04975579240d4a07ac5e4a9d0f8de31b0aa7a4be753c98ae596c9 -F ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java ce8aef867e06a21685658eb0173768c355e9e1e2dfdc75f382643fa62c7ee779 +F ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java c82bc00c1988f86246a89f721d3c41f0d952f33f934aa6677ec87f7ca42519a0 F ext/jni/src/org/sqlite/jni/wrapper1/SqliteException.java 982538ddb4c0719ef87dfa664cd137b09890b546029a7477810bd64d4c47ee35 F ext/jni/src/org/sqlite/jni/wrapper1/Tester2.java 08f92d52be2cec28a7b4555479cc54b7ebeeb94985256144eeb727154ec3f85b F ext/jni/src/org/sqlite/jni/wrapper1/ValueHolder.java a84e90c43724a69c2ecebd601bc8e5139f869b7d08cb705c77ef757dacdd0593 @@ -363,20 +363,20 @@ F ext/misc/base64.c 8dc0a08cee11722822858a62625f1b63e5d5f1adac1cf4492d5732b571e3 F ext/misc/base85.c ff54cc676c6ec86231f75ecc86ea45416fcb69751dfb79690d5f5da5f7d39867 F ext/misc/basexx.c 89ad6b76558efbceb627afd5e2ef1d84b2e96d9aaf9b7ecb20e3d00b51be6fcf F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a -F ext/misc/btreeinfo.c 13bc9e9f1c13cde370d0e4a6a2683e9f1926a4cead7fb72c71871b11a06d78a1 +F ext/misc/btreeinfo.c 8f5e6da2c82ec2f06ee0216e922370a436dafdbb06ffa7a552203515ff9e7ddf F ext/misc/cksumvfs.c 9d7d0cf1a8893ac5d48922bfe9f3f217b4a61a6265f559263a02bb2001259913 F ext/misc/closure.c 5559daf1daf742228431db929d1aa86dd535a4224cc634a81d2fd0d1e6ad7839 F ext/misc/completion.c c27b64fdd0943c1b7f152376599814cee2641f7d67a7bb9bd2b957c2a64a5591 F ext/misc/compress.c 8191118b9b73e7796c961790db62d35d9b0fb724b045e005a5713dc9e0795565 -F ext/misc/csv.c e82124eabee0e692d7b90ab8b2c34fadbf7b375279f102567fa06e4da4b771bf +F ext/misc/csv.c d9dab032420d81cdf0abc4b8523711d20a355704f82d958f963c86be48387dd9 F ext/misc/dbdump.c 678f1b9ae2317b4473f65d03132a2482c3f4b08920799ed80feedd2941a06680 -F ext/misc/decimal.c e1da22eee70d7e3eaa99a6b761bc03c4d01d7ffa554bf3178b1f1f184932806c +F ext/misc/decimal.c d4883de142f6dcd36eda23da40b55e2b51374e7b01eb54a7173940191389fc5e F ext/misc/eval.c 04bc9aada78c888394204b4ed996ab834b99726fb59603b0ee3ed6e049755dc1 F ext/misc/explain.c 606100185fb90d6a1eade1ed0414d53503c86820d8956a06e3b0a56291894f2b -F ext/misc/fileio.c 33165b3cd99f83dcd333a338eb51491f6b01c8d96cb6ae81f96a6a096834e030 +F ext/misc/fileio.c d80268a5328fe839062a9d3103ea0fc7cacc6d42605959275675cb37867c84f7 F ext/misc/fossildelta.c 86dfa83f85f7ccd640591d8a5c6865346d0c2ee6a949d78591eceb892f1cbfec -F ext/misc/fuzzer.c 684a4996b523ea89f495b38fd8a14a2ae00695089a88031366a4df6adc2c873b -F ext/misc/ieee754.c 2901d08a586d00a1d3c0fd89e03c57ee9e2b5f013b0daab9e49c7a48a9d5946b +F ext/misc/fuzzer.c 6b231352815304ba60d8e9ec2ee73d4918e74d9b76bda8940ba2b64e8777515e +F ext/misc/ieee754.c 176c061c94857b543313959289cb60cf777c999fd002f82b53d194b95e9f347a F ext/misc/memstat.c 43705d795090efb78c85c736b89251e743c291e23daaa8382fe7a0df2c6a283d F ext/misc/memtrace.c 7c0d115d2ef716ad0ba632c91e05bd119cb16c1aedf3bec9f06196ead2d5537b F ext/misc/mmapwarm.c a81af4aaec00f24f308e2f4c19bf1d88f3ac3ce848c36daa7a4cd38145c4080d @@ -388,22 +388,21 @@ F ext/misc/percentile.c 72e05a21db20a2fa85264b99515941f00ae698824c9db82d7edfbb16 F ext/misc/prefixes.c 82645f79229877afab08c8b08ca1e7fa31921280906b90a61c294e4f540cd2a6 F ext/misc/qpvtab.c fc189e127f68f791af90a487f4460ec91539a716daf45a0c357e963fd47cc06c F ext/misc/randomjson.c ef835fc64289e76ac4873b85fe12f9463a036168d7683cf2b773e36e6262c4ed -F ext/misc/regexp.c 69bd45f6931bdc6801c1059b65a3e8b15ba88255e6abe387a34b653ce17e8908 +F ext/misc/regexp.c f1f7cfe90fc027b33d2b5ae7d6235eecce69c3aca71c9afce56fec62342c8b44 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 F ext/misc/series.c 22c6d8f00cc1b5089b1b37392e9097e9df9a5db53be86daf9a7669d95bb179f4 -F ext/misc/sha1.c 3030b5926b34bb09459200e100fae34e48c04077cf175381a7560f72bbf3d9cf +F ext/misc/sha1.c cb5002148c2661b5946f34561701e9105e9d339b713ec8ac057fd888b196dcb9 F ext/misc/shathree.c fd22d70620f86a0467acfdd3acd8435d5cb54eb1e2d9ff36ae44e389826993df F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c 693c8fd3293087fa821322967a97e59dfa24051e5d2ca7fa85790a4034db6fa4 F ext/misc/sqlar.c a6175790482328171da47095f87608b48a476d4fac78d8a9ff18b03a2454f634 -F ext/misc/sqlite3_stdio.c e49c07050bf7bdc87866da7583beda236f2f8c462018a34b61785d99cbddedfd -F ext/misc/sqlite3_stdio.h 27a4ecea47e61bc9574ccdf2806f468afe23af2f95028c9b689bfa08ab1ce99f +F ext/misc/sqlite3_stdio.c 0fe5a45bd332b30aef2b68c64edbe69e31e9c42365b0fa79ce95a034bca6fbb0 +F ext/misc/sqlite3_stdio.h f05eaf5e0258f0573910324a789a9586fc360a57678c57a6d63cfaa2245b6176 F ext/misc/stmt.c b090086cd6bd6281c21271d38d576eeffe662f0e6b67536352ce32bbaa438321 F ext/misc/stmtrand.c 59cffa5d8e158943ff1ce078956d8e208e8c04e67307e8f249dece2436dcb7fc F ext/misc/templatevtab.c 10f15b165b95423ddef593bc5dcb915ec4eb5e0f1066d585e5435a368b8bc22b -F ext/misc/tmstmpvfs.c 240caad4441328dc52bd2871f48811db46dff858d5598030e389176837a2f4df F ext/misc/totype.c ba11aac3c0b52c685bd25aa4e0f80c41c624fb1cc5ab763250e09ddc762bc3a8 F ext/misc/uint.c 327afc166058acf566f33a15bf47c869d2d3564612644d9ff81a23efc8b36039 F ext/misc/unionvtab.c 716d385256d5fb4beea31b0efede640807e423e85c9784d21d22f0cce010a785 @@ -412,20 +411,16 @@ F ext/misc/uuid.c 5bb2264c1b64d163efa46509544fd7500cb8769cb7c16dd52052da8d961505 F ext/misc/vfslog.c 3932ab932eeb2601dbc4447cb14d445aaa9fbe43b863ef5f014401c3420afd20 F ext/misc/vfsstat.c 0b23c0a69a2b63dc0ef0af44f9c1fc977300c480a1f7a9814500369d8211f56e F ext/misc/vfstrace.c 0e4b8b17ac0675ea90f6d168d8214687e06ca3efbc0060aad4814994d82b41fb -F ext/misc/vtablog.c 402496fb38add7dd2c50f2a0ad20f83a9916ceab48dcd31e62ad621e663c83ac +F ext/misc/vtablog.c 2d04386c2f5a3bb93bc9ae978f0b7dcd5a264e126abd640dd6d82aa9067cbd48 F ext/misc/vtshim.c e5bce24ab8c532f4fdc600148718fe1802cb6ed57417f1c1032d8961f72b0e8f F ext/misc/wholenumber.c 0fa0c082676b7868bf2fa918e911133f2b349bcdceabd1198bba5f65b4fc0668 F ext/misc/windirent.h 02211ce51f3034c675f2dbf4d228194d51b3ee05734678bad5106fff6292e60c -F ext/misc/zipfile.c c8ee04e1b349270b5df401ad732f5d7c387146e69b33c02fa90322760cc6fee0 +F ext/misc/zipfile.c 837591f0505d21f7f7937ea046c9b0fc594f7fa3ca00c2bd54ffa1c94bfccd63 F ext/misc/zorder.c bddff2e1b9661a90c95c2a9a9c7ecd8908afab5763256294dd12d609d4664eee -F ext/qrf/README.md e6e0ce2700acf6fd06312b42726a8f08ca240f30e1b122bff87c71c602046352 -F ext/qrf/dev-notes.md e68a6d91ce4c7eb296ef2daadc2bb79c95c317ad15b9fafe40850c67b29c2430 -F ext/qrf/qrf.c cd48c23500c3b129be5e0627ce9d41b5df3c2d715525b00a6ccbd1f30689fb17 -F ext/qrf/qrf.h 2ac14b0aaacf44636d8c81051bfeab4afae50a98fbb2e10ff5aed0c28a87b2b2 F ext/rbu/rbu.c 801450b24eaf14440d8fd20385aacc751d5c9d6123398df41b1b5aa804bf4ce8 F ext/rbu/rbu1.test 25870dd7db7eb5597e2b4d6e29e7a7e095abf332660f67d89959552ce8f8f255 F ext/rbu/rbu10.test 7c22caa32c2ff26983ca8320779a31495a6555737684af7aba3daaf762ef3363 -F ext/rbu/rbu11.test 3b71377c018b05dd39c30ea2feb272a087eb0faeff1b7b4511144f87219c478e +F ext/rbu/rbu11.test 8584f80ef4be00e6beec4154f638847ffc40b5f2832ffadfbaf558ae40e50cb5 F ext/rbu/rbu12.test ec63aa7bfc3c65c1d774bf4357ed731723827d211d9d7cb0efa171bbaeeebaf4 F ext/rbu/rbu13.test 658edbc3325d79252a98b761fde95460e439f80e820ff29e10261e25f870b3b6 F ext/rbu/rbu14.test 05dac607a62f62102f4db92135979a8a4501143638060019aca08c753822cf39 @@ -466,7 +461,7 @@ F ext/rbu/rbuvacuum.test 542561741ff2b262e3694bc6012b44694ee62c545845319a06f3237 F ext/rbu/rbuvacuum2.test 1a9bd41f127be2826de2a65204df9118525a8af8d16e61e6bc63ba3ac0010a23 F ext/rbu/rbuvacuum3.test 3ce42695fdf21aaa3499e857d7d4253bc499ad759bcd6c9362042c13cd37d8de F ext/rbu/rbuvacuum4.test ffccd22f67e2d0b380d2889685742159dfe0d19a3880ca3d2d1d69eefaebb205 -F ext/rbu/sqlite3rbu.c e99400d29d029936075e27495b269a2dcdceae3cf8c86b1d0869b4af487be3ab +F ext/rbu/sqlite3rbu.c 3fb2390575b261c365d3f6fea61ff15e74d5d89e373f2a2bfa4d80c24321e793 F ext/rbu/sqlite3rbu.h e3a5bf21e09ca93ce4e8740e00d6a853e90a697968ec0ea98f40826938bdb68e F ext/rbu/test_rbu.c 8b6e64e486c28c41ef29f6f4ea6be7b3091958987812784904f5e903f6b56418 F ext/recover/dbdata.c 10d3c56968a9af6853722a47280805ad1564714d79ea45ac6f7da14bb57fd137 @@ -488,8 +483,17 @@ F ext/recover/recoversql.test e66d01f95302a223bcd3fd42b5ee58dc2b53d70afa90b0d00e F ext/recover/sqlite3recover.c 56c216332ea91233d6d820d429f3384adbec9ecedda67aa98186b691d427cc57 F ext/recover/sqlite3recover.h 011c799f02deb70ab685916f6f538e6bb32c4e0025e79bfd0e24ff9c74820959 F ext/recover/test_recover.c 3d0fb1df7823f5bc22a0b93955034d16a2dfa2eb1e443e9a0123a77f120599a3 +F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 +F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 +F ext/repair/checkindex.c 7639b4f8928f82c10b950169e60cc45a7f6798df0b299771d17bef025736f657 +F ext/repair/sqlite3_checker.c.in 445118c5f7fea958b36fba1b2c464283e60ed4842039ddee3265f1698115ebf7 +F ext/repair/sqlite3_checker.tcl a9a2caa9660567257c177a91124d8c0dccdfa341e25c51e6da7f1fd9e601eafa +F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e +F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc78249442da72ff3f8297398a69 +F ext/repair/test/checkindex01.test b530f141413b587c9eb78ff734de6bb79bc3515c335096108c12c01bddbadcec +F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c F ext/rtree/README 734aa36238bcd2dee91db5dba107d5fcbdb02396612811377a8ad50f1272b1c1 -F ext/rtree/geopoly.c bd1971479184d559499ff3087c37f2823977d7b0ec80916141ae66f70345c88d +F ext/rtree/geopoly.c f0573d5109fdc658a180db0db6eec86ab2a1cf5ce58ec66cbf3356167ea757eb F ext/rtree/rtree.c 9331997a76b88a9bc04e156bdfd6e2fe35c0aa93bc338ebc6aa0ae470fe4a852 F ext/rtree/rtree.h 4a690463901cb5e6127cf05eb8e642f127012fd5003830dbc974eca5802d9412 F ext/rtree/rtree1.test e0608db762b2aadca0ecb6f97396cf66244490adc3ba88f2a292b27be3e1da3e @@ -532,14 +536,14 @@ F ext/session/changesetfuzz1.test 15b629004e58d5ffcc852e6842a603775bb64b1ce51254 F ext/session/session1.test 5d2502922d38a1579076863827342379a1609ca6bae78c40691a2be1ed1be2aa x F ext/session/session2.test ee83bb973b9ce17ccce4db931cdcdae65eb40bbb22089b2fe6aa4f6be3b9303f F ext/session/session3.test 2cc1629cfb880243aec1a7251145e07b78411d851b39b2aa1390704550db8e6a -F ext/session/session4.test ad0ddaaddb9a99dac433d83fc6674aae2af072b8f57e63a6b3f2d76f1a66e98c +F ext/session/session4.test 823f6f018fcbb8dacf61e2960f8b3b848d492b094f8b495eae1d9407d9ab7219 F ext/session/session5.test 716bc6fafd625ce60dfa62ae128971628c1a1169 F ext/session/session6.test 35279f2ec45448cd2e24a61688219dc6cf7871757716063acf4a8b5455e1e926 F ext/session/session8.test 326f3273abf9d5d2d7d559eee8f5994c4ea74a5d935562454605e6607ee29904 F ext/session/session9.test 0c4a8fbe7a5031f50855f020f3408e1f07fd7859f1daa1629eadcec3422072d6 F ext/session/sessionA.test 1feeab0b8e03527f08f2f1defb442da25480138f F ext/session/sessionB.test c4fb7f8a688787111606e123a555f18ee04f65bb9f2a4bb2aa71d55ce4e6d02c -F ext/session/sessionC.test c3fade0a460d898fa42e9077b88e45c0d24ead3150268e145c8e19aeafc24ba1 +F ext/session/sessionC.test f8a5508bc059ae646e5ec9bdbca66ad24bc92fe99fda5790ac57e1f59fce2fdf F ext/session/sessionD.test 470ff917dc849e2eb78142ade63aaabd729d773833cff0ff01bca0eda68a21ce F ext/session/sessionE.test b2010949c9d7415306f64e3c2072ddabc4b8250c98478d3c0c4d064bce83111d F ext/session/sessionF.test d37ed800881e742c208df443537bf29aa49fd56eac520d0f0c6df3e6320f3401 @@ -569,10 +573,11 @@ F ext/session/sessionrowid.test 85187c2f1b38861a5844868126f69f9ec62223a03449a98a F ext/session/sessionsize.test 8fcf4685993c3dbaa46a24183940ab9f5aa9ed0d23e5fb63bfffbdb56134b795 F ext/session/sessionstat1.test 5e718d5888c0c49bbb33a7a4f816366db85f59f6a4f97544a806421b85dc2dec F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009ecef8511f4cf3fc -F ext/session/sqlite3session.c 6ebd02be470f36d41c4bd78927f39d507b62051ba025eacaed9936c769902a07 +F ext/session/sqlite3session.c b3de195ce668cace9b324599bf6255a70290cbfb5451e826e946f3aee6e64c54 F ext/session/sqlite3session.h 7404723606074fcb2afdc6b72c206072cdb2b7d8ba097ca1559174a80bc26f7a -F ext/session/test_session.c 190110e3bd9463717248dec1272b44fe9943e57b7646d0b4200dcf11e4dccee6 -F ext/wasm/GNUmakefile 79236447d750609aa6beda30feec1314180c5462a493ad94214122887232bfd4 +F ext/session/test_session.c 8766b5973a6323934cb51248f621c3dc87ad2a98f023c3cc280d79e7d78d36fb +F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c +F ext/wasm/GNUmakefile 3dc01e673c456d3b752674c9407276e8fef35dec1d304b3cc1de362f019b2a09 F ext/wasm/README-dist.txt f01081a850ce38a56706af6b481e3a7878e24e42b314cfcd4b129f0f8427066a F ext/wasm/README.md 2e87804e12c98f1d194b7a06162a88441d33bb443efcfe00dc6565a780d2f259 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff @@ -580,68 +585,71 @@ F ext/wasm/SQLTester/SQLTester.mjs 6b3c52ed36a5573ca4883176f326332a8d4c0cecf5efd F ext/wasm/SQLTester/SQLTester.run.mjs 57f2adb33f43f2784abbf8026c1bfd049d8013af1998e7dcb8b50c89ffc332e0 F ext/wasm/SQLTester/index.html 64f3435084c7d6139b08d1f2a713828a73f68de2ae6a3112cbb5980d991ba06f F ext/wasm/SQLTester/touint8array.c 2d5ece04ec1393a6a60c4bf96385bda5e1a10ad49f3038b96460fc5e5aa7e536 -F ext/wasm/api/EXPORTED_FUNCTIONS.c-pp 7ba933e8f1290cc65459dd371c0c9a031d96bdf14d7a2244fa761d9775117b90 -F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b47244781a7e81 -F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core ef34398a903d0a2425fbbfbd4ed2cd596daea55b8515e2617c8dc7ad7c0767dd +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras 9eae68943ce91ab145892b31370819c2103525240eb72e0fce53c498b8d8275a +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b +F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 +F ext/wasm/api/README.md f4c0d67caaee21a77b8938c30b5f79667bfc9d0c95d01b51df77ea35ee773884 +F ext/wasm/api/extern-post-js.c-pp.js 205f55aacfc62c580985db5c790300779de3876a76a5c7e1bfb13e71c8b4506b F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 -F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b -F ext/wasm/api/post-js-header.js d24bd0d065f3489c8b78ddf3ead6321e5d047187a162cd503c41700e03dd1f06 -F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f -F ext/wasm/api/sqlite3-api-glue.c-pp.js 9b33e3ee467791dec4fd1b444b12a8545dfbb6c8b28ac651c7bdc7661a3b5a5c -F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c41db98a22b534c15e543cfa -F ext/wasm/api/sqlite3-api-prologue.js 1fefd40ab21e3dbf46f43b6fafb07f13eb13cc157a884f7c1134caf631ddb3f2 +F ext/wasm/api/post-js-footer.js 5bd7170b5e8ce7b62102702bbcf47ef7b3b49cd56ed40c043fd990aa715b74ee +F ext/wasm/api/post-js-header.js 79d078aec33d93b640a19c574b504d88bb2446432f38e2fbb3bb8e36da436e70 +F ext/wasm/api/pre-js.c-pp.js a876c6399dff29b6fe9e434036beb89889164cc872334e184291723ecc7cb072 +F ext/wasm/api/sqlite3-api-cleanup.js a3d6b9e449aefbb8bba283c2ba9477e2333a0eeb94a7a26b5bf952736f65a6dd +F ext/wasm/api/sqlite3-api-glue.c-pp.js d2b8263b3ce0cefc6c5a68d0a4d448a9770eda4bf9d9ded9d7eb0198e4ce4da1 +F ext/wasm/api/sqlite3-api-oo1.c-pp.js c4260f3fdc553c56ee530c20cc1119029067b503f0d6d7b472705536cb45aa1d +F ext/wasm/api/sqlite3-api-prologue.js 307583ff39a978c897c4ef4ce53fe231dce5c73dc84785969c81c1ab5960a293 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 -F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.js 92d6d327a862f1627ff3e88e60fdfea9def06ad539d98929ba46490e64372736 +F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 +F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 2ccf4322f42063aefc150972943e750c77f7926b866f1639d40eec05df075b6e -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3 -F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 -F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 -F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js aa9715f661fb700459a5a6cb1c32a4d6a770723b47aa9ac0e16c2cf87d622a66 -F ext/wasm/api/sqlite3-worker1.c-pp.js bd0655687090e3b1657268a6a9cacde1ea2a734079d194e16dbbed9083e51b38 -F ext/wasm/c-pp-lite.c f38254fba42561728c2e4764a7ba8d68700091e7c2f4418112868c0daba16783 -F ext/wasm/common/SqliteTestUtil.js dae753b95e72248c4395d8de8359e0d055cd9928488e8dd84aef89e46d23b32e +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 26cb41d5a62f46a106b6371eb00fef02de3cdbfaa51338ba087a45f53028e0d0 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 418c33fe284739564daab3c7a7a88882fdd3c99137497900f98eddec1e409af5 +F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 9097074724172e31e56ce20ccd7482259cf72a76124213cbc9469d757676da86 +F ext/wasm/api/sqlite3-wasm.c dd7fc1d535281f0d5d2732bb1b662d1d403a762f07b63c2ea5663053377b2804 +F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bda1c75bd674a92a0e27cc2f3d46dbbf21e422413f8046814515a0bd7409328a +F ext/wasm/api/sqlite3-worker1.c-pp.js 802d69ead8c38dc1be52c83afbfc77e757da8a91a2e159e7ed3ecda8b8dba2e7 +F ext/wasm/c-pp-lite.c 8fa0148e73782a86274db688c4730e2962cd675af329490493adddaf3322f16f +F ext/wasm/common/SqliteTestUtil.js 7adaeffef757d8708418dc9190f72df22367b531831775804b31598b44f6aa51 F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15 F ext/wasm/common/testing.css e97549bab24126c24e0daabfe2de9bb478fb0a69fdb2ddd0a73a992c091aad6f -F ext/wasm/common/whwasmutil.js 831f07a0d9bb61713164871370811432e96d0f813806a4d2c783d3c77c2373a0 +F ext/wasm/common/whwasmutil.js 0d539324097fc83b953e9844267359ba0fd02286caa784ea2f597ced279ea640 F ext/wasm/config.make.in c424ae1cc3c89274520ad312509d36c4daa34a3fce5d0c688e5f8f4365e1049a F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 F ext/wasm/demo-123.js c7b3cca50c55841c381a9ca4f9396e5bbdc6114273d0b10a43e378e32e7be5bf F ext/wasm/demo-jsstorage.html 409c4be4af5f207fb2877160724b91b33ea36a3cd8c204e8da1acb828ffe588e -F ext/wasm/demo-jsstorage.js 467cb4126ff679ebcdb112d100d073af26b9808d0a0b52d66a40e28f59c5099b -F ext/wasm/demo-worker1-promiser.c-pp.html f73b0b98457e7fdad40d8353cb9b2919391da180f49549a86f3d58b4e5a010eb +F ext/wasm/demo-jsstorage.js 42131ddfa18e817d0e39ac63745e9ea31553980a5ebd2222e04d4fac60c19837 +F ext/wasm/demo-worker1-promiser.c-pp.html 635cf90685805e21772a5f7a35d1ace80f98a9ef7c42ff04d7a125ddca7e5db8 F ext/wasm/demo-worker1-promiser.c-pp.js f40ec65810048e368896be71461028bd10de01e24277208c59266edf23bb9f52 F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d -F ext/wasm/demo-worker1.js fdfa90aa9d6b402bfed802cf1595fe4da6cc834ac38c8ff854bf1ee01f5ff9bb +F ext/wasm/demo-worker1.js 08720227e98fa5b44761cf6e219269cee3e9dd0421d8d91459535da776950314 F ext/wasm/example_extra_init.c 2347cd69d19d839ef4e5e77b7855103a7fe3ef2af86f2e8c95839afd8b05862f -F ext/wasm/fiddle/fiddle-worker.js 6c72acac2d381480bc9f5eb538e3f2faf2c1f72dd4fcbd05d3b409818a9a8fd5 +F ext/wasm/fiddle/fiddle-worker.js 7798af02e672e088ff192716f80626c8895e19301a65b8af6d5d12b2d13d2451 F ext/wasm/fiddle/fiddle.js 84fd75967e0af8b69d3dd849818342227d0f81d13db92e0dcbc63649b31a4893 -F ext/wasm/fiddle/index.c-pp.html 72c7e5517217960b3809648429ea396a7cbad0ffb2c92f6a2f5703abecb27317 +F ext/wasm/fiddle/index.html a27b8127ef9ecf19612da93b2a6a73bdb3777b5c56b5450bb7200a94bc108ff9 F ext/wasm/index-dist.html db23748044e286773f2768eec287669501703b5d5f72755e8db73607dc54d290 -F ext/wasm/index.html 475bc283338749db4e3fbf24cf3f5aa020cc85a1fffb780d400a915fcb5f1756 -F ext/wasm/jaccwabyt/jaccwabyt.js 4e2b797dc170851c9c530c3567679f4aa509eec0fab73b466d945b00b356574b -F ext/wasm/jaccwabyt/jaccwabyt.md 6aa90fa1a973d0ad10d077088bea163b241d8470c75eafdef87620a1de1dea41 -F ext/wasm/mkdist.sh f8883b077a2ca47cf92e6f0ce305fbf72ca648c3501810125056c4b09c2d5554 x -F ext/wasm/mkwasmbuilds.c 0e9198eb90acae4bcf57cf62d7186f6af5aaac02efdb075a1aded33614b3805a +F ext/wasm/index.html 54e27db740695ab2cb296e02d42c4c66b3f11b65797340d19fa6590f5b287da1 +F ext/wasm/jaccwabyt/jaccwabyt.js bbac67bc7a79dca34afe6215fd16b27768d84e22273507206f888c117e2ede7d +F ext/wasm/jaccwabyt/jaccwabyt.md 167fc0b624c9bc2c477846e336de9403842d81b1a24fc4d3b24317cb9eba734f +F ext/wasm/mkdist.sh 64d53f469c823ed311f6696f69cec9093f745e467334b34f5ceabdf9de3c5b28 x +F ext/wasm/mkwasmbuilds.c 1b53c4d2a1350c19a96a8cdfbda6a39baea9d2142bfe0cbef0ccb0e898787f47 F ext/wasm/module-symbols.html e54f42112e0aac2a31f850ab33e7f2630a2ea4f63496f484a12469a2501e07e2 F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96 F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d F ext/wasm/speedtest1-wasmfs.mjs 60dd5842f6d2a70a6d0bef12633a11491bde6984aff75a37c2040980d8cbf36a F ext/wasm/speedtest1-worker.html 068d4190f304fa1c34e6501a1b3a4c32fe8d8dac93c2d0f53d667a1cb386eedc -F ext/wasm/speedtest1-worker.js 8acad67bfd6aeeb799bd5ae007ea32af85a082a287d8877c5a10adf4bd7efd89 -F ext/wasm/speedtest1.html f32c66997eb0b036c4546e6302cd0673157912661df0b290ab65816f713feac6 +F ext/wasm/speedtest1-worker.js 958a2d3c710bf8e82567277f656193a0248216db99a3c2c86966124b84309efb +F ext/wasm/speedtest1.html c90d63dfa795f0cb1ad188de587be9024b1ff73b4adc5fdf7efc0d781be94d03 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555e685bce3da8c3f F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c -F ext/wasm/tester1-worker.c-pp.html d0032241d0b24d996cf1c4dd0dde364189693af9b5c986e48af7d3d720fcd244 -F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3d5669ea724ddb -F ext/wasm/tester1.c-pp.js 6b946cd6d4da130dbae4a401057716d27117ca02cad2ea8c29ae9c46c675d618 +F ext/wasm/tester1-worker.c-pp.html 883881eeac14eeeecc8ff22acf9fe0f18a97cacb48be08ebb0bae891ceded584 +F ext/wasm/tester1.c-pp.html 949920126dcf477925d8d540093d9cc374d3ab4c4ddee920c1dcadcf37917306 +F ext/wasm/tester1.c-pp.js 2b014884dadf28928fabcb688746ca87145673eef75e154486505a266203fc15 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -651,8 +659,7 @@ F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca F ext/wasm/tests/opfs/sahpool/sahpool-pausing.js f264925cfc82155de38cecb3d204c36e0f6991460fff0cb7c15079454679a4e2 F ext/wasm/tests/opfs/sahpool/sahpool-worker.js bd25a43fc2ab2d1bafd8f2854ad3943ef673f7c3be03e95ecf1612ff6e8e2a61 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk e1a03e9206f6a042a9147035915cb944e9242d570779bc3ccd7ed6a39df10cae -F make.bat a136fd0b1c93e89854a86d5f4edcf0386d211e5d5ec2434480f6eea436c7420c +F main.mk 00dd631c66c1f7922b2d691631163899eff1c3d1da780ff37a62f8d997b368e1 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -662,38 +669,38 @@ F mptest/multiwrite01.test dab5c5f8f9534971efce679152c5146da265222d F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 1b9c24374a85dfc7eb8fa7c4266ee0db4f9609cceecfc5481cd8307e5af04366 F sqlite3.pc.in e6dee284fba59ef500092fdc1843df3be8433323a3733c91da96690a50a5b398 -F src/alter.c fc36b19273ffe364aeb4d00ba04bda8798ad7a67fec7a035ee8ee56272e1bdbe +F src/alter.c fc7bbbeb9e89c7124bf5772ce474b333b7bdc18d6e080763211a40fde69fb1da F src/analyze.c 03bcfc083fc0cccaa9ded93604e1d4244ea245c17285d463ef6a60425fcb247d -F src/attach.c 7cf07d4fa42b9fc8662237c60c40b730326c30aa90ae5fffc0b18b2d726ebf61 -F src/auth.c ebec42df26b34a62b6750d30d9c2c03554a1c522020182476f7729a439fef04f +F src/attach.c 9af61b63b10ee702b1594ecd24fb8cea0839cfdb6addee52fba26fa879f5db9d +F src/auth.c 54ab9c6c5803b47c0d45b76ce27eff22a03b4b1f767c5945a3a4eb13aa4c78dc F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 F src/bitvec.c e242d4496774dfc88fa278177dd23b607dce369ccafb3f61b41638eea2c9b399 F src/btmutex.c 30dada73a819a1ef5b7583786370dce1842e12e1ad941e4d05ac29695528daea -F src/btree.c b744bf69d520534751c742cababe7ad28c3892f1e3a75242e75a20bca15a834a +F src/btree.c 240fa5b4ced4733ac5882b43448f433a9742a76d9d7e28aa9ce48d13a38ceb5d F src/btree.h e823c46d87f63d904d735a24b76146d19f51f04445ea561f71cc3382fd1307f0 F src/btreeInt.h 9c0f9ea5c9b5f4dcaea18111d43efe95f2ac276cd86d770dce10fd99ccc93886 -F src/build.c b993e4adef4c4cdfd7abf62e2676c467bb1923f25f40c3c7ab2a7bfbace3de7f +F src/build.c 611e07299d72ff04bbcb9e7109183467e30925d203c3e121ef9bb3cf6876289b F src/callback.c 3605bbf02bd7ed46c79cd48346db4a32fc51d67624400539c0532f4eead804ad -F src/carray.c 3efe3982d5fb323334c29328a4e189ccaef6b95612a6084ad5fa124fd5db1179 +F src/carray.c ff6081a31878fc34df8fa1052a9cbf17ddc22652544dcb3e2326886ed1053b55 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e -F src/date.c 61e92f1f7e2e88e1cd91e91dc69eb2b2854e7877254470f9fabd776bfac922b8 +F src/date.c e19e0cfff9a41bfdd884c655755f6f00bca4c1a22272b56e0dd6667b7ea893a2 F src/dbpage.c c9ea81c11727f27e02874611e92773e68e2a90a875ef2404b084564c235fd91f F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c -F src/delete.c 901499bed747c3b4b2be45be1abe912ba50a3f6a40ba88cc006ccf279f2d0e97 -F src/expr.c 8c3b23cb35f43c2d0570c1058b9a269e561e769e09c81ba192992c95022c1939 +F src/delete.c 03a77ba20e54f0f42ebd8eddf15411ed6bdb06a2c472ac4b6b336521bf7cea42 +F src/expr.c 28b1cc3d2f147cc888703d5482f9581f17656d02abfa331c34370cb3350776be F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 -F src/fkey.c fb0f74c57d19a2d3f113f3476826919d68feda7ff334abfdb479a9a6353b9fcd -F src/func.c 6e7de3551ae0f8205006e5109f025223246edd20186d54d90746dee7c1c5c093 +F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f +F src/func.c 0b802107498048d3dcac0b757720bcb8506507ce02159e213ab8161458eb293b F src/global.c a19e4b1ca1335f560e9560e590fc13081e21f670643367f99cb9e8f9dc7d615b F src/hash.c 03c8c0f4be9e8bcb6de65aa26d34a61d48a9430747084a69f9469fbb00ea52ca F src/hash.h 46b92795a95bfefb210f52f0c316e9d7cdbcdd7e7fcfb0d8be796d3a5767cddf -F src/hwtime.h 21c2cf1f736e7b97502c3674d0c386db3f06870d6f10d0cf8174e2a4b8cb726e +F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c dfd311b0ac2d4f6359e62013db67799757f4d2cc56cca5c10f4888acfbbfa3fd -F src/json.c 8b6341a419150b28530cc21e3951b2238c35cdc312f11b2ca29017fe4b1dedc0 +F src/json.c fb031340edee159c07ad37dbe668ffe945ed86f525b0eb3822e4a67cbc498a72 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa -F src/loadext.c 56a542244fbefc739a2ef57fac007c16b2aefdb4377f584e9547db2ce3e071f9 -F src/main.c 31a13302193fbd51279c7e69cdfa0320d0de7629f9151e0964c1d320e8bdd7a4 +F src/loadext.c a3bc9a2522dc3b960e38b7582d1818f6245a49289387c2c7b19f27bfeabf1e81 +F src/main.c 65d11c17890966d271c925c6cc55e3ba50fa08374633cb99c0dee4719a20915a F src/malloc.c 422f7e0498e1c9ef967f06283b6f2c0b16db6b905d8e06f6dbc8baaa3e4e6c5a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2 @@ -703,44 +710,44 @@ F src/mem5.c b7da5c10a726aacacc9ad7cdcb0667deec643e117591cc69cf9b4b9e7f3e96ff F src/memdb.c a3feb427cdd4036ea2db0ba56d152f14c8212ca760ccb05fb7aa49ff6b897df3 F src/memjournal.c c283c6c95d940eb9dc70f1863eef3ee40382dbd35e5a1108026e7817c206e8a0 F src/msvc.h 80b35f95d93bf996ccb3e498535255f2ef1118c78764719a7cd15ab4106ccac9 -F src/mutex.c 00b8cee206a67fd764d001f3a148494331d8d0b3b9c3974ecd69ff29bb444462 +F src/mutex.c 06bcd9c3dbf2d9b21fcd182606c00fafb9bfe0287983c8e17acd13d2c81a2fa9 F src/mutex.h a7b2293c48db5f27007c3bdb21d438873637d12658f5a0bf8ad025bb96803c4a F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 F src/mutex_unix.c f7ee5a2061a4c11815a2bf4fc0e2bfa6fb8d9dc89390eb613ca0cec32fc9a3d1 -F src/mutex_w32.c e1d317d29cb623667d43de94714264d1e1871cc4bb39fa67dd17048e8138c739 +F src/mutex_w32.c 28f8d480387db5b2ef5248705dd4e19db0cfc12c3ba426695a7d2c45c48e6885 F src/notify.c 57c2d1a2805d6dee32acd5d250d928ab94e02d76369ae057dee7d445fd64e878 F src/os.c 509452169d5ea739723e213b8e2481cf0e587f0e88579a912d200db5269f5f6d F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06 -F src/os_kv.c e7d96727db5b67e39d590a68cc61c86daf4c093c36c011a09ebfb521182ec28d +F src/os_kv.c fb7ba8d6204197357f1eb7e1c7450d09c10043bf7e99aba602f4aa46b8fb11a3 F src/os_setup.h 8efc64eda6a6c2f221387eefc2e7e45fd5a3d5c8337a7a83519ba4fbd2957ae2 -F src/os_unix.c fa5e09b4df35ad845440cad67b86908cfe1fd4c28c51915f82e23633d1992bf4 -F src/os_win.c 0d553b6e8b92c8eb85e7f1b4a8036fe8638c8b32c9ad8d9d72a861c10f81b4c5 -F src/os_win.h 5e168adf482484327195d10f9c3bce3520f598e04e07ffe62c9c5a8067c1037b -F src/pager.c fe34fd22ec251436985d7b6ebdd05bf238a17901c2cb23d3d28974dd2361a912 +F src/os_unix.c dcf7988ddbdd68619b821c9a722f9377abb46f1d26c9279eb5a50402fd43d749 +F src/os_win.c a89b501fc195085c7d6c9eec7f5bd782625e94bb2a96b000f4d009703df1083f +F src/os_win.h 4c247cdb6d407c75186c94a1e84d5a22cbae4adcec93fcae8d2bc1f956fd1f19 +F src/pager.c cd562b878ea1b44d021ba199abc9d3b54f6b3347500a9fed03f66d6000620945 F src/pager.h 6137149346e6c8a3ddc1eeb40aee46381e9bc8b0fcc6dda8a1efde993c2275b8 -F src/parse.y 3b784d6083380a950e3b1b32ce5ddd303e8c7c209d8ab788df2c62aaf9ee8eb3 +F src/parse.y 619c3e92a54686c5e47923688c4b9bf7ec534a4690db5677acc28b299c403250 F src/pcache.c 588cc3c5ccaaadde689ed35ce5c5c891a1f7b1f4d1f56f6cf0143b74d8ee6484 -F src/pcache.h 092b758d2c5e4dabb30eae46d8dfad77c0f70b16bf3ff1943f7a232b0fe0d4ba +F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c 131ca0daf4e66b4608d2945ae76d6ed90de3f60539afbd5ef9ec65667a5f2fcd F src/pragma.c ecec75795c1821520266e4f93fa8840cce48979af532db06f085e36a7813860f -F src/prepare.c f6a6e28a281bd1d1da12f47d370a81af46159b40f73bf7fa0b276b664f9c8b7d -F src/printf.c 9cff219dba73b1aa9a8113e83e962f03f7bea8b6eb51cefb25bc468d5a69fb2d +F src/prepare.c 2af0b5c1ec787c8eebd21baa9d79caf4a4dc3a18e76ce2edbf2027d706bca37a +F src/printf.c 7297c2aeed4d90d80c5ba82920d9e57b7bfad04b3466be1d7e042db382fe296e F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c -F src/resolve.c 928ff887f2a7c64275182060d94d06fdddbe32226c569781cf7e7edc6f58d7fd +F src/resolve.c 243888da9f31c48d0890f8c9df30ed4be6fd098d295b0fe1c297a7f3c453ca53 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c ffe199f025a0dd74670d2a77232bdea364a4d7b36f32c64a6572d39ba6a11576 -F src/shell.c.in d4e9ce266ca8f7364da6e86df011f8655beeb5f0d074d624215a2d8ce220a0ad -F src/sqlite.h.in 1f853f1d836af3e5a0b451521041d05658988a45f6978aaae08286e483fee5ac +F src/select.c 5abbd22cae7469dae0a600d78636be77a5682cc372cf8da744d68bbe23298df2 +F src/shell.c.in 2c7e751795f38bb1855c35b556419cab5b8ba22e0f6758f5a629338065d6b79f +F src/sqlite.h.in c0979f9ac1f5be887397dd2a0bb485636893a81b34d64df85123aae9650c42f2 F src/sqlite3.rc 015537e6ac1eec6c7050e17b616c2ffe6f70fca241835a84a4f0d5937383c479 -F src/sqlite3ext.h 1b7a0ee438bb5c2896d0609c537e917d8057b3340f6ad004d2de44f03e3d3cca -F src/sqliteInt.h 1c7f23ab9d6efdf3dc434880b6320f158937284f6e2cebd2a024def0c749cb04 -F src/sqliteLimit.h 904a3f520362c7065c18165aaabd504fb13cc1b76cb411f38bd41ac219e4af1e +F src/sqlite3ext.h 7f236ca1b175ffe03316d974ef57df79b3938466c28d2f95caef5e08c57f3a52 +F src/sqliteInt.h 601512a60e48f88e5148fc0e15f1efd03372df2a18bdee312d1590d14da6b5c1 +F src/sqliteLimit.h 4bc72c1519a27c538b7575c240a4472c829d78d27d69a00ddd5a046a0dbfd73a F src/status.c 7565d63a79aa2f326339a24a0461a60096d0bd2bce711fefb50b5c89335f3592 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 -F src/tclsqlite.c 85b5a20df96016e5d1d8fdc68c8a4c279c5b93e2049b77cd806c2cc50b9d8c56 +F src/tclsqlite.c 3c604c49e6cf4211960a9ddb9505280fd22cde32175f40884c641c0f5a286036 F src/tclsqlite.h 614b3780a62522bc9f8f2b9fb22689e8009958e7aa77e572d0f3149050af348a -F src/test1.c 3e3b013f59ffcb57dce00c90d55907072d71d4e970cb0a590cb261efe11bae9c +F src/test1.c 5d061afe479c7364842e0170be7220dea13389575fa6030d30b3e20bec4e1f75 F src/test2.c 62f0830958f9075692c29c6de51b495ae8969e1bef85f239ffcd9ba5fb44a5ff F src/test3.c 432646f581d8af1bb495e58fc98234380250954f5d5535e507fc785eccc3987a F src/test4.c 0ac87fc13cdb334ab3a71823f99b6c32a6bebe5d603cd6a71d84c823d43a25a0 @@ -750,10 +757,10 @@ F src/test8.c 206d8f3cc73950d252906656e2646b5de0d580b07187b635fcb3edd8c2c5fbc0 F src/test9.c df9ddc7db6ef1b8cf745866ee229090779728bcbe660c7f297d3127ab21d92af F src/test_autoext.c 14d4bbd3d0bd1eec0f6d16b29e28cf1e2d0b020d454835f0721a5f68121ac10f F src/test_backup.c a2bfd90d2ff2511b8635507bdb30fa9b605ade19c16b533066cae3077f5bdb72 -F src/test_bestindex.c d75fad21369d80910238032bcf8d9ca1f2bffda13c1ceec63bfbb7f704448b15 +F src/test_bestindex.c a9428931bec06de830b2630f57a7b1f2711761269f04df62b7aa1affcbce15bb F src/test_blob.c 77b994e17f2c87055f44fd96c9a206c5a7155bae2cda2769af60c2f3582f962c F src/test_btree.c 28283787d32b8fa953eb77412ad0de2c9895260e4e5bd5a94b3c7411664f90d5 -F src/test_config.c e02566c2c4ee2916324ce17123a798b47663cead2de546cfbd71d8cddb46bb26 +F src/test_config.c 18aa596d37de1d5968c439fd58ebf38bc4d9c9d1db63621504e241fde375cecd F src/test_delete.c d0e8f6dc55cfc98a7c27c057fb88d512260564bf0b611482656c68b8f7f401ed F src/test_demovfs.c 3efa2adf4f21e10d95521721687d5ca047aea91fa62dd8cc22ac9e5a9c942383 F src/test_devsym.c 649434ed34d0b03fbd5a6b42df80f0f9a7e53f94dd1710aad5dd8831e91c4e86 @@ -773,7 +780,7 @@ F src/test_mutex.c dacae6790956c0d4e705aaed2090227792e291b0496cccd688e9994c1e21f F src/test_onefile.c f31e52e891c5fef6709b9fcef54ce660648a34172423a9cbdf4cbce3ba0049f4 F src/test_osinst.c 269039d9c0820a02ee928014c30860d57ee757ecda54df42e463d0ca1377b835 F src/test_pcache.c 496da3f7e2ca66aefbc36bbf22138b1eff43ba0dff175c228b760fa020a37bd0 -F src/test_quota.c 5bb44452b9c6c248bb3c82d2466a20915aa6d12801f6c1784b6499aaa04d9811 +F src/test_quota.c 180e87437250bed7e17e4e61c106730939e39fec9be73d28961f27f579a92078 F src/test_quota.h 2a8ad1952d1d2ca9af0ce0465e56e6c023b5e15d F src/test_rtree.c d844d746a3cc027247318b970025a927f14772339c991f40e7911583ea5ed0d9 F src/test_schema.c b06d3ddc3edc173c143878f3edb869dd200d57d918ae2f38820534f9a5e3d7d9 @@ -788,21 +795,21 @@ F src/test_vfs.c b4135c1308516adf0dfd494e6d6c33114e03732be899eace0502919b674586b F src/test_window.c 6d80e11fba89a1796525e6f0048ff0c7789aa2c6b0b11c80827dc1437bd8ea72 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c f297bbf02037639e7a93b37d9c6e4415b3de1273395ee8fa8183e741e1e7fb72 -F src/treeview.c feaa59f14db4f7b5aacca9c5ad5aeb562c1f98262c1ffd74371f4186ade91fc5 -F src/trigger.c 4bf3bfb3851d165e4404a9f9e69357345f3f7103378c07e07139fdd8aeb7bd20 +F src/tokenize.c cb3294cf23c11106b50d9af6998a6c1bf389b52e15b17698c9fab97bbaa9b37f +F src/treeview.c 3ce7ac9835d2d70cc1c868b01b747ae8a062322e155701e58e3d62ca79aada7a +F src/trigger.c d5cf2541ff048f30b6a0507eb3d1ec4e695c53584e3b2298a5bf248714fe185e F src/update.c 3e5e7ff66fa19ebe4d1b113d480639a24cc1175adbefabbd1a948a07f28e37cf F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1 F src/utf.c 7267c3fb9e2467020507601af3354c2446c61f444387e094c779dccd5ca62165 -F src/util.c eccfa8b3b414bb64c6543421c9fd10e5f07e103baae36427a273a9131527694c -F src/vacuum.c d3d35d8ae893d419ade5fa196d761a83bddcbb62137a1a157ae751ef38b26e82 -F src/vdbe.c 5328c99dd256ee8132383565a86e253543a85daccfd7477c52f20bac6b385a7f -F src/vdbe.h 966d0677a540b7ea6549b7c4e1312fc0d830fce3a235a58c801f2cc31cf5ecf9 -F src/vdbeInt.h 42488247a80cd9d300627833c6c85ace067ae5011a99e7614e2358130d62feea -F src/vdbeapi.c 6cdcbe5c7afa754c998e73d2d5d2805556268362914b952811bdfb9c78a37cf1 -F src/vdbeaux.c 396d38a62a357b807eabae0cae441fc89d2767a57ab08026b7072bf7aa2dd00c +F src/util.c 36fb1150062957280777655976f3f9a75db236cb8207a0770ceae8d5ec17fcd3 +F src/vacuum.c 1bacdd0a81d2b5dc1c508fbf0d938c89fa78dd8d5b46ec92686d44030d4f4789 +F src/vdbe.c b44c366e83412d3b8c190feb1f029b7d02e1bd69252a57b32f195107f0d03964 +F src/vdbe.h be33bd7b17f2ec92939642416030491508c51071f6c14e27cd195983fec56b63 +F src/vdbeInt.h 2aaeb6df2938b181b4700a9328688a3986f2bba71e8b96f6a80671316618fa49 +F src/vdbeapi.c 790f199ec0d9c423f5efef58d7538ac9c6b34248d84180eb0dca3d635f1c9c9b +F src/vdbeaux.c 908d8a191aed444b2e4c920159249127f3ff67b94c56a16fad1dfdf9c7488f20 F src/vdbeblob.c b3f0640db9642fbdc88bd6ebcc83d6009514cafc98f062f675f2c8d505d82692 -F src/vdbemem.c 317ec5e870ddb16951b606c9fe8be22baef22ecbe46f58fdefc259662238afb7 +F src/vdbemem.c fdd023e357ad3129e1dcae46df47fccceeb8bd1ffa6c5d43a1e3f04460bb59b7 F src/vdbesort.c b69220f4ea9ffea5fdef34d968c60305444eea909252a81933b54c296d9cca70 F src/vdbetrace.c 49e689f751505839742f4a243a1a566e57d5c9eaf0d33bbaa26e2de3febf7b41 F src/vdbevtab.c fc46b9cbd759dc013f0b3724549cc0d71379183c667df3a5988f7e2f1bd485f3 @@ -811,11 +818,11 @@ F src/vxworks.h 9d18819c5235b49c2340a8a4d48195ec5d5afb637b152406de95a9436beeaeab F src/wal.c 88d94fd15a75f6eda831fa32d1148a267ea37bf0a4b69829a73dfde06244b08f F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 -F src/where.c 9f09ee7b260010138d5f9fb5f195b98051119eae3096a99d72ff16c83230f4af +F src/where.c 287324fe73a0ae8e55b3be89bb2fe4148e3a8394e1e2f10ed2113713a037d8a3 F src/whereInt.h 8d94cb116c9e06205c3d5ac87af065fc044f8cf08bfdccd94b6ea1c1308e65da -F src/wherecode.c 783ecd30061c875c919a5163e4b55f9a0eccdaf7c9b17ad2908a1668a8766bc4 -F src/whereexpr.c e9f7185fba366d9365aa7a97329609e4cf00b3dd0400d069fbaa5187350c17c6 -F src/window.c c0a38cd32473e8e8e7bc435039f914a36ca42465506dc491c65870c01ddac9fb +F src/wherecode.c fe08356c7f20f4e2290204b9147bded3bbfe5453e2c590be0f9b0b5f1c959e76 +F src/whereexpr.c 403a44eeec1a0f0914fccc6a59376b6924bc00ef6728fe6ffce4cf3051b320fc +F src/window.c 538195bbc75bb924e18e368fbd4ed731a3fe3f901351b44f6466ec486f53affe F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test 4d7a34d328e58ca2a2d78fd76c27614a41ca7ddf4312ded9c68c04f430b3b47d F test/affinity3.test 9b7d1133e11d5edd7805573c4ab6f3ba73b0b74a1f280d5b130d4bf3506a93ff @@ -830,14 +837,12 @@ F test/alter2.test 7e3d26ab409df52df887b366a63902c3429b935c41cb962fd58ffc25784f2 F test/alter3.test dcdd5f850f30656a45a0f05e41abfb52b74bbf6ccba165d0f7adf6b0116e4fd6 F test/alter4.test 37cafe164067a6590a0ee4cec780bddbbaa33dc50b11542dcfbe0e65626494fd F test/alterauth.test 63442ba61ceb0c1eeb63aac1f4f5cebfa509d352276059d27106ae256bafc959 -F test/alterauth2.test 4b74fa8f184f4736497317feb587b65759eb87d87acfe3a8ef433d4d18bb002b -F test/altercol.test 3661c432aacb42bc2198dd4611bbb9c3b09fc73251b59edda046109103b8ac00 -F test/altercons.test ea18def4a0f26b9066da56095c9c480df705df4d02e4ae151708fae76f7e3884 -F test/altercons2.test 3c1f58312817df43aeada3b1827fdc3ce3fc50c6f49a95ef62cf4cbbae8583a0 +F test/alterauth2.test 48967abae0494d9a300d1c92473d99fcb66edfcc23579c89322f033f49410adc +F test/altercol.test b43fb5725332f4cf8cff0280605202c1672e808281accea60a066d2ccc5129e5 F test/altercorrupt.test 2e1d705342cf9d7de884518ddbb053fd52d7e60d2b8869b7b63b2fda68435c12 F test/alterdropcol.test a653a3945f964d26845ec0cd0a8e74189f46de3119a984c5bc45457da392612e F test/alterdropcol2.test 527fce683b200d620f560f666c44ae33e22728e990a10a48a543280dfd4b4d41 -F test/alterfault.test 2bb3103954ea60f2e2777b1ae12e79ec3e1fd278f2b1398ad316c68835a62898 +F test/alterfault.test 289067108947bedca27534edd4ff251bcd298cf84402d7b24eaa3749305418c6 F test/alterlegacy.test f38c6d06cda39e1f7b955bbce57f2e3ef5b7cb566d3d1234502093e228c15811 F test/altermalloc.test 167a47de41b5c638f5f5c6efb59784002b196fff70f98d9b4ed3cd74a3fb80c9 F test/altermalloc2.test 17fb3724c4b004c469c27dc4ef181608aa644555fbd3f3236767584f73747c81 @@ -845,8 +850,8 @@ F test/altermalloc3.test 8040e486368403f2fdd6fc3998258b499bd4cc2f3ddbb5f8f874cd4 F test/alterqf.test 8ec03d776de9c391daa0078ea8f838903bdcfb11dfae4ba3576b48436834ccba F test/altertab.test 8a2712f9076da5012a002d0b5cc0a421398a5bf61c25bab41b77c427586a7a27 F test/altertab2.test 0889ba0700cc1cdb7bc7d25975aa61fece34f621de963d0886e2395716b38576 -F test/altertab3.test 575e771e2f02b13eb98798dc92eabacd187d6dbcf596e70f11d699b0b6b5d0b2 -F test/altertrig.test b1590647076add5a47aea0f2236c609ca0bc8a7a2462463edd3e5882c7894802 +F test/altertab3.test 471b8898d10bbc6488db9c23dc76811f405de6707d2d342b1b8b6fd1f13cd3c8 +F test/altertrig.test aacc980b657354fe2d3d4d3a004f07d04ccc1a93e5ef82d68a79088c274ddc6b F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 2fb21d7d64748636384e6cb8998dbf83968caf644c07fcb4f76c18f2e7ede94b F test/analyze3.test c5156cef33f04b90a6b9e9d5d0bbc273a0fb44147d4508407bf1080811e2c6c8 @@ -863,7 +868,6 @@ F test/analyzeF.test 40b5cc3ad7b10e81020d7ca86f1417647ecfae7477cfd88acc5aa7ae106 F test/analyzeG.test 623be33038c49648872746c8dd8b23b5792c08fef173c55e82f1b12fca259852 F test/analyzer1.test b6a624ec0af92eec209e1328465b66937c8fdf2fb442a3fa45321ddb3700f4aa F test/atof1.test bd21c4a0e718ab1470de07a2a79f2544d7903be34feebcc80de04beee4807b00 -F test/atof2.test 12912add57230495450e2fc94cb8ad1c9f3277f8843a3bc27079cae45c9782a1 F test/atomic.test 065a453dde33c77ff586d91ccaa6ed419829d492dbb1a5694b8a09f3f9d7d061 F test/atomic2.test b6863b4aa552543874f80b42fb3063f1c8c2e3d8e56b6562f00a3cc347b5c1da F test/atrc.c c388fac43dbba05c804432a7135ae688b32e8f25818e9994ffba4b64cf60c27c @@ -877,7 +881,7 @@ F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1 F test/auth3.test 76d20a7fa136d63bcfcf8bcb65c0b1455ed71078d81f22bcd0550d3eb18594ab F test/autoanalyze1.test b9cc3f32a990fa56669b668d237c6d53e983554ae80c0604992e18869a0b2dec F test/autoinc.test 9df9930966dbe92c55ef37a4d89112cfd537be0d0596d397177c12db9e581be0 -F test/autoindex1.test 2523a76f30734742c3f4d948d0cbf3b6627f775e7833814f425a2e289ba58b22 +F test/autoindex1.test 65931519206bbec71948b11e125af0656435a0937973fe5fed70d776a712911f F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df F test/autoindex3.test ca502c8050166ac6107a7b4fe4e951f4d3270a23a958af02b14f1b962b83c4b6 F test/autoindex4.test 3c2105e9172920e26f950ba3c5823e4972190e022c1e6f260ba476b0af24c593 @@ -885,7 +889,7 @@ F test/autoindex5.test 3fb938cbf4e7f3896563ce04e2a24b0bc653fc6245b4bf3268cd7b20f F test/autovacuum.test 00671369bbf96c6a49989a9425f5b78b94075d6a4b031e5e00000c2c32f365df F test/autovacuum2.test 76f7eb4fe6a6bf6d33a196a7141dba98886d2fb53a268d7feca285d5da4759d7 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 -F test/avfs.test 95bb8d04f8edad6dc9e600221d103f7e2cc3da398af84df215a3a819e560c45c +F test/avfs.test 76f59743dc1f5fa533840d1818b420fe1ee45e21c0fd6bbac7942ba677903128 F test/avtrans.test 7a6eae44763293024b137b53ff824d8500d754dbae060a8d940afbacfc1d4a15 F test/backcompat.test f2431465ed668f09fc3f6998e56e893a1506ccea6e8b6f409f085f759f431b48 F test/backup.test 3b08fd4af69f0fa786931103a31f4542b184aba16e239e5f22b18c3c2476697f @@ -905,14 +909,13 @@ F test/bestindex4.test 3039894f2dad50f3a68443dffad1b44c9b067ac03870102df1ce3d9a4 F test/bestindex5.test a0c90b2dad7836e80a01379e200e5f8ec9476d49b349af02c0dbff2fb75dc98d F test/bestindex6.test 16942535b551273f3ad9df8d7cc4b7f22b1fcd8882714358859eb049a6f99dd4 F test/bestindex7.test f094c669a6400777f4d2ddc3ed28e39169f1adb5be3d59b55f22ccf8c414b71e -F test/bestindex8.test 4d8b1e8f30a7f405988ce4dbcc2b95c0775f0bed9ec08e0291a07e2f35f7e653 +F test/bestindex8.test b63a4f171a2c83d481bb14c431a8b72e85d27b2ffdaa0435a95d58ca941678f9 F test/bestindex9.test 1a4b93db117fd8abe74ae9be982f86aa72f01e60cd4ac541e6ede39673a451a0 F test/bestindexA.test e1b5def6b190797cacf008e6815ffb78fb30261999030d60a728d572eef44c7f -F test/bestindexB.test 14db2f66ec9cc5064a74996033b74e5eec0fd2f3a327fbe34ff18de67e9d2671 +F test/bestindexB.test 328b97b69cd1a20928d5997f9ecb04d2e00f1d18e19ab27f9e9adb44d7bc51ce F test/bestindexC.test 95b4a527b1a5d07951d731604a6d4cf7e5a806b39cea0e7819d4c9667e11c3fc F test/bestindexD.test 6a8f6f84990bcf17dfa59652a1f935beddb7afd96f8302830fbc86b0a13df3c3 F test/bestindexE.test 297f3ea8500a8f3c17d6f78e55bdfee089064c6144ee84a110bd005a03338f49 -F test/bestindexF.test 4e53d606cbde40a2254aa016d500c5b71766a4065b8541202d195a3d9fe11b1c F test/between.test e7587149796101cbe8d5f8abae8d2a7b87f04d8226610aa1091615005dcf4d54 F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc @@ -947,7 +950,7 @@ F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 F test/capi3c.test 31d3a6778f2d06f2d9222bd7660c41a516d1518a059b069e96ebbeadb5a490f7 F test/capi3d.test 8b778794af891b0dca3d900bd345fbc8ebd2aa2aae425a9dccdd10d5233dfbde F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe -F test/carray01.test 17c1cf8287862b15dda949dba626fd5fee5c58471dcc1cae0341471c2ae7da01 +F test/carray01.test 49e2aedfdf2c715bc002d2773cdc1217166679639542c79c8aa4115f06421407 F test/carray02.test 9d070b54f24a34d1f3b3c552ba34db0375a9d1c4219067416fb07d1595987c9d F test/carrayfault.test 108a7d83904fc267c448e27c13b2a857c700bd6ddaa2f1e2518be718b159cb6b F test/cast.test a2a3b32df86e3c0601ffa2e9f028a18796305d251801efea807092dbf374a040 @@ -965,7 +968,7 @@ F test/collate1.test 0890fa372753b59eba53832d37328af815f6b8e4b16761823180eeb62c8 F test/collate2.test 471c6f74573382b89b0f8b88a05256faa52f7964f9e4799e76708a3b1ece6ba4 F test/collate3.test 89defc49983ddfbf0a0555aca8c0521a676f56a5 F test/collate4.test c953715fb498b87163e3e73dd94356bff1f317bd -F test/collate5.test 42daaf7799b04221206a219fb3c0f9efeede03e760f9562b8c0114b8df183fe3 +F test/collate5.test b1dfeff239ea69ee9225832553f423d37a6184eb730cee06f6846ab4e3c6dbef F test/collate6.test 8be65a182abaac8011a622131486dafb8076e907 F test/collate7.test 8ec29d98f3ee4ccebce6e16ce3863fb6b8c7b868 F test/collate8.test cd9b3d3f999b8520ffaa7cc1647061fc5bab1334 @@ -1002,7 +1005,7 @@ F test/corruptK.test ac13504593d89d69690d45479547616ed12644d42b5cb7eeb2e759a76fc F test/corruptL.test f15de2b4729c0851ea89916a26766b094d74bac79f9f9f2b0191935aa3b344c9 F test/corruptM.test 7d574320e08c1b36caa3e47262061f186367d593a7e305d35f15289cc2c3e067 F test/corruptN.test a034bb217bebd8d007625dfb078e76ec3d53515052dbceb68bd47b2c27674d5c -F test/cost.test 3786cd1cc6d1ab416004a1e39387fb0db0c8e259f46b0bce62cbdc328f2c55a0 +F test/cost.test cc434a026b1e9d0d98137a147e24e5daf1b1ad09e9ff7da63b34c83ddd136d92 F test/count.test cd4bd531066e8d77ef8fe1e3fc8253d042072e117ccab214b290cf83f1602249 F test/countofview.test 4088e461a10ee33e69803c177a69aa1d7bba81a9ffc2df66d76465a22ca7fdfc F test/coveridxscan.test f35c7208dedc4f98e471c569df64c0f95a49f6e072d8dc7c8f99bdee2697de1b @@ -1033,7 +1036,7 @@ F test/dbfuzz.c fc566102f72c8af84ae8077b4faf7f056c571e6fa7a32e98b66e42b7505f47b6 F test/dbfuzz001.test 6c9a4622029d69dc38926f115864b055cb2f39badd25ec22cbfb130c8ba8e9c3 F test/dbfuzz2-seed1.db e6225c6f3d7b63f9c5b6867146a5f329d997ab105bee64644dc2b3a2f2aebaee F test/dbfuzz2.c 4b3c12de4d98b1b2d908ab03d217d4619e47c8b23d5e67f8a6f2b1bdee7cae23 -F test/dblwidth-a.sql 59dd59aa78ce8fd8ab631a3816516831f4e947b143039257e6fe132c3cea4171 +F test/dblwidth-a.sql eb4141518610e52f931a55a984310075e98dc31eee5a28ae806b1e35377be85a F test/dbpage.test 63fab1eb026bada121107e53436fa749bbf83281dc9dea17af422f7a5c0f289f F test/dbpagefault.test ea39de2ca86041a9c6df1135645180a76d0a8da93ac159e2fafe38e39636530b F test/dbstatus.test 4a4221a883025ffd39696b3d1b3910b928fb097d77e671351acb35f3aed42759 @@ -1050,9 +1053,8 @@ F test/descidx2.test a0ba347037ff3b811f4c6ceca5fd0f9d5d72e74e59f2d9de346a9d2f6ad F test/descidx3.test 953c831df7ea219c73826dfbf2f6ee02d95040725aa88ccb4fa43d1a1999b926 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 691c9e850b0d0b56b66e7e235453198cb4cf0760e324b7403d3c5abbeab0a014 -F test/distinct2.test a6af6a90b2c1eea64c3cc87ea7f8feb832053f7276fe3c212abacf646de4762a +F test/distinct2.test 4d6316b6487a0aa5a90bee111575c957e2a5ba5a9be9156febe9533ce78876e8 F test/distinctagg.test 40d7169ae5846caaf62c6e307d2ca3c333daf9b6f7cde888956a339a97afe85f -F test/dotcmd01.sql 0388a778912ed08436ae5c80e03389d8bd347fa724611193257a18c69692019d F test/e_blobbytes.test 4c01dfe4f12087b92b20705a3fdfded45dc4ed16d5a211fed4e1d2786ba68a52 F test/e_blobclose.test 692fc02a058476c2222a63d97e3f3b2b809c1842e5525ded7f854d540ac2e075 F test/e_blobopen.test 29f6055ee453b8e679fe9570c4d3acfedbef821622c5dad16875148c5952ef50 @@ -1062,7 +1064,7 @@ F test/e_createtable.test 31b9bcb6ac8876bc7ec342d86d9c231a84c62b442093a6651dfd0f F test/e_delete.test ab39084f26ae1f033c940b70ebdbbd523dc4962e F test/e_droptrigger.test 235c610f8bf8ec44513e222b9085c7e49fad65ad0c1975ac2577109dd06fd8fa F test/e_dropview.test 74e405df7fa0f762e0c9445b166fe03955856532e2bb234c372f7c51228d75e7 -F test/e_expr.test 9bdb347b78b9f4eff9153ea97797facc179a821898588471a70808b4471a69b0 +F test/e_expr.test 0a1e175caddc78b27306647cb4ce2362c55790190f8cdd178b75fd6262eb8f76 F test/e_fkey.test feeba6238aeff9d809fb6236b351da8df4ae9bda89e088e54526b31a0cbfeec5 F test/e_fts3.test 17ba7c373aba4d4f5696ba147ee23fd1a1ef70782af050e03e262ca187c5ee07 F test/e_insert.test f02f7f17852b2163732c6611d193f84fc67bc641fb4882c77a464076e5eba80e @@ -1071,19 +1073,19 @@ F test/e_resolve.test a61751c368b109db73df0f20fc75fb47e166b1d8 F test/e_select.test 327a15f14068bbd6f647cedc67210f8680fcb2f05e481a0a855fccd2abfa1292 F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f F test/e_totalchanges.test c927f7499dc3aa28b9b556b7d6d115a2f0fe41f012b128d16bf1f3b30e9b41e4 -F test/e_update.test 9f8bb82b8760ac66f2c9c2aadb78a418bd639d22f041d712570c9db56f20afda +F test/e_update.test f46c2554d915c9197548681e8d8c33a267e84528 F test/e_uri.test 86564382132d9c453845eeb5293c7e375487b625900ab56c181a0464908417d8 F test/e_vacuum.test 89fc48e8beee2f9dfd6de1fbb2edea6542dae9121dc0fbe6313764169e742104 F test/e_wal.test db7c33642711cf3c7959714b5f012aca08cacfa78da0382f95e849eb3ba66aa4 F test/e_walauto.test 248af31e73c98df23476a22bdb815524c9dc3ba8 -F test/e_walckpt.test 16e7d006e8687654ee59e7ad5a6d285ba23f0fe0eeb87f790afd6bc9cf1d1924 +F test/e_walckpt.test 28c371a6bb5e5fe7f31679c1df1763a19d19e8a0 F test/e_walhook.test 01b494287ba9e60b70f6ebf3c6c62e0ffe01788e344a4846b08e5de0b344cb66 F test/emptytable.test a38110becbdfa6325cd65cb588dca658cd885f62 F test/enc.test b5503a87b31cea8a5084c6e447383f9ca08933bd2f29d97b6b6201081b2343eb F test/enc2.test 872afe58db772e7dfa1ad8e0759f8cc820e9efc8172d460fae83023101c2e435 F test/enc3.test 55ef64416d72975c66167310a51dc9fc544ba3ae4858b8d5ab22f4cb6500b087 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 -F test/eqp.test 1d653fe8d2612cd6764e5ea2f16dcf5f13a9f50448b9233bb1573804bccd7579 +F test/eqp.test 746db9fe11629a0d00328e1721cc2a2e4726d574b677ab14de35fd914f54cc82 F test/eqp2.test 6e8996148de88f0e7670491e92e712a2920a369b4406f21a27c3c9b6a46b68dd F test/errmsg.test eae9f091eb39ce7e20305de45d8e5d115b68fa856fba4ea6757b6ca3705ff7f9 F test/errofst1.test 6da78363739ba8991f498396ab331b5d64e7ab5c4172c12b5884683ef523ac53 @@ -1103,7 +1105,7 @@ F test/extension01.test 5de412c66276105901c370770175003381fdcb0c4da7054fa43cf4a3 F test/external_reader.test 6fdec43eeca23eb32faad1e95a4d1abc402bc8b3db70df12d6fc08a637f4a2b5 F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79 F test/fallocate.test 37a62e396a68eeede8f8d2ecf23573a80faceb630788d314d0a073d862616717 -F test/filectrl.test 3344987f143f3cb9914895dc63d9e9518a535ef1b1438d22caf3ba2fa76cc6da +F test/filectrl.test 4b720117388cf6766d0b798e2dddd785953f8f371633b0c0084d2f34cf72336a F test/filefmt.test f393e80c4b8d493b7a7f8f3809a8425bbf4292af1f5140f01cb1427798a2bbd4 F test/filter1.test 590f8ba9a0cd0823b80d89ac75c5ce72276189cef9225d2436adaf1ee87f3727 F test/filter2.tcl 44e525497ce07382915f01bd29ffd0fa49dab3adb87253b5e5103ba8f93393e8 @@ -1122,8 +1124,7 @@ F test/fordelete.test ba98f14446b310f9c9d935b97ec748753d0144a28b356ba30d1f4f6958 F test/fork-test.c 9ac2e6423a1d38df3d6be0e8ac15608b545de21e2b19d9d876254c5931b63edb F test/format4.test eeae341953db8b6bda7f549044797c3278a6cc345d11ada81471671b654f8ef4 F test/fp-speed-1.c b37de94eba034e1703668816225f54510ec60fb0685406608cc707afe6b8234d -F test/fpconv1.test 63f352682fa65601a326563ad633086df6ab194e6ed5e7366786f38a525a7fd7 -F test/fptest01.sql 210562ad8d5a7895f26273dd3be56561a41bcb51d78a28a337af0f1ceaa3bb8d +F test/fpconv1.test d5d8aa0c427533006c112fb1957cdd1ea68c1d0709470dabb9ca02c2e4c06ad8 F test/fts-9fd058691.test 78b887e30ae6816df0e1fed6259de4b5a64ad33c F test/fts3.test 672a040ea57036fb4b6fdc09027c18d7d24ab654 F test/fts3_common.tcl dffad248f9ce090800e272017d2898005c28ee6314fc1dd5550643a02666907a @@ -1149,7 +1150,7 @@ F test/fts3aux1.test 1880eaa75c586cd10f53080479a2b819b3915ae7ce55c4e0ba8f1fe05ac F test/fts3aux2.test 2459e7fa3e22734aed237d1e2ae192f5541c4d8b218956ad2d90754977bf907f F test/fts3b.test c15c4a9d04e210d0be67e54ce6a87b927168fbf9c1e3faec8c1a732c366fd491 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 -F test/fts3comp1.test 73a53ada3d25bf242c4b2a24cfe9d39e658be56cfa74754279b9e6db776ed7ce +F test/fts3comp1.test a0f5b16a2df44dd0b15751787130af2183167c0c F test/fts3conf.test c9cd45433b6787d48a43e84949aa2eb8b3b3d242bac7276731c1476290d31f29 F test/fts3corrupt.test 6732477c5ace050c5758a40a8b5706c8c0cccd416b9c558e0e15224805a40e57 F test/fts3corrupt2.test e318f0676e5e78d5a4b702637e2bb25265954c08a1b1e4aaf93c7880bb0c67d0 @@ -1199,7 +1200,7 @@ F test/fts3tok_err.test 52273cd193b9036282f7bacb43da78c6be87418d F test/fts3varint.test 0b84a3fd4eba8a39f3687523804d18f3b322e6d4539a55bf342079c3614f2ada F test/fts4aa.test 0e6bfd6a81695a39b23e448dda25d864e63dda75bde6949c45ddc95426c6c3f5 F test/fts4check.test f0ea5e5581951d8ef7a341eea14486daf6c5f516a2f3273b0d5e8cb8a6cd3bd2 -F test/fts4content.test 7f441866207f5b1e76e0f18bde5d9925d1ee8f60388054613dd14a29a79f0bc4 +F test/fts4content.test 73bbb123420d2c46ef2fb3b24761e9acdb78b0877179d3a5d7d57aada08066f6 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 F test/fts4growth.test 289833c34ad45a5e6e6133b53b6a71647231fb89d36ddcb8d9c87211b6721d7f F test/fts4growth2.test 13ad4e76451af6e6906c95cdc725d01b00044269 @@ -1211,7 +1212,7 @@ F test/fts4merge.test 57d093660a5093ae6e9fbd2d17592a88b45bbd66db2703c4b640b28828 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test 8d9ccb4a3d41c4c617a149d6c4b13ad02de797d0 F test/fts4merge4.test 66fce89934cd9508cbdc67de486558c34912ffb2e8ffe5c9a1bbb9b8a4408ba7 -F test/fts4merge5.test 987af90c930e8555f74ab994f597431caec7f8defc52de7718655c32da07af9e +F test/fts4merge5.test 69932d85cda8a1c4dcfb742865900ed8fbda51724b8cf9a45bbe226dfd06c596 F test/fts4min.test 1c11e4bde16674a0c795953509cbc3731a7d9cbd1ddc7f35467bf39d632d749f F test/fts4noti.test d5d933705b1b1516b67a5e3f8e514ecb19c6522fb3357bb744776d48427c2292 F test/fts4onepass.test d69ddc4ee3415e40b0c5d1d0408488a87614d4f63ba9c44f3e52db541d6b7cc7 @@ -1238,7 +1239,7 @@ F test/fuzz3.test 70ba57260364b83e964707b9d4b5625284239768ab907dd387c740c0370ce3 F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c 9096506277f33cc242eb59743c409c81306492b6ebb84571198f864e536ebe22 +F test/fuzzcheck.c 34a025386f84d818cd3343e69e9d9083091af83153e226d71d4e1c126b5f1dd0 F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -1250,7 +1251,7 @@ F test/fuzzdata8.db 8f34ae00d8d5d4747dd80983cf46161065e4f78324dcff3c893506ff8db3 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test f64c4aef4c9e9edf1d6dc0d3f1e65dcc81e67c996403c88d14f09b74807a42bc -F test/fuzzinvariants.c 6768bcd03290776cd982624729d2abee2e89e6aba62b4a2b839a98332725a167 +F test/fuzzinvariants.c 3ddfec7f5b970b018f1a982532de905cf180e0c1e48cd653be9365d3e6177625 F test/gcfault.test 4ea410ac161e685f17b19e1f606f58514a2850e806c65b846d05f60d436c5b0d F test/gencol1.test ceb3163b59cb77f4ad57ae4f01a143ce36b06fdd6a8dab1149235db89979ffd8 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 @@ -1261,8 +1262,6 @@ F test/hook.test 2d89bf9480646feb8093be3a58ea502d6521906779ed960de31dd9c4502c054 F test/hook2.test b9ff3b8c6519fb67f33192f1afe86e7782ee4ac8 F test/icu.test 8da7d52cd9722c82f33b0466ed915460cb03c23a38f18a9a2d3ff97da9a4a8c0 F test/ieee754.test 0d3ab84ab2069c9994c833a7cd820ee6037f0cf888e206a4a7fc05f735d5790a -F test/import01.sql 11a5f8325b8b04c66fe489d30558d462411432adf750cef1c0f7b06564691a8c -F test/imposter1.sql fc5ad0945bb19622688c7a1cd7dfd1cefa4b013bac9e2628c22b03c7309f021f F test/imposter1.test 5a20b2cdeb53e65fc57cdb10a33750bd4ef6259909eaf1972253b9e79f7a3fb2 F test/in.test edf979bff3244b9e47849e2b43886631354c8213791f42da92216f08012141af F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 @@ -1302,12 +1301,11 @@ F test/insert.test 97cfb30b83ca1622b9422a1e4c4831b4cb767cf5d654660945036d1e72067 F test/insert2.test 4d14b8f1b810a41995f6286b64a6943215d52208 F test/insert3.test 1b7db95a03ad9c5013fdf7d6722b6cd66ee55e30 F test/insert4.test 2bf81535a990c969665d66db51fcf76c23499b39893b5109f413d1de4ad34cd3 -F test/insert5.test 79f6b6efd0d3db5f4e3ff442300b7d9e7185adb345b29aacc3ea5a9c58ab9beb +F test/insert5.test 394f96728d1258f406fe5f5aeb0aaf29487c39a6 F test/insertfault.test ac63d14ea3b49c573673a572f4014b9117383a03e497c58f308b5c776e4a7f74 F test/instr.test 67ba309e9697c24a304e98a7c8f372456177dd4e32237d2a305e1e05f7bb79c2 F test/instrfault.test 95e28efade652e6d51ae11b377088fe523a581a07ec428009e152a4dd0e0f44c F test/intarray.test bb976b0b3df0ebb6a2eddfb61768280440e672beba5460ed49679ea984ccf440 -F test/intck01.sql f2d88bf41cdd64f2ed8c3d4f357cf520f017aa2986999ab9a62eb6506ef18106 F test/interrupt.test ac1ef50ec9ab8e4f0e17c47629f82539d4b22558904e321ed5abea2e6187da7a F test/interrupt2.test e4408ca770a6feafbadb0801e54a0dcd1a8d108d F test/intpkey.test 7d54711acf553cdd641a40e9c6cfc2bf1a76070074940c1b126442517054320f @@ -1320,7 +1318,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 5984da7bf74b6540aa356f2ab0c6ae68a6d12039a3d798a9ac6a100abc17d520 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/istrue.test e7f285bb70282625c258e866ce6337d4c762922f5a300e1b50f958aef6e7d9c9 -F test/join.test c706b382ed09ddc89eee7ad0ffd08d862655b0abc292a690d41d995c18c17b3f +F test/join.test 2fcfd84640cfd9ff48f31b4b0d370c4d5498c355ae4384544668ca54d37ae186 F test/join2.test f59d63264fb24784ae9c3bc9d867eb569cd6d442da5660f8852effe5c1938c27 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 @@ -1336,7 +1334,7 @@ F test/joinD.test 2ce62e7353a0702ca5e70008faf319c1d4686aa19fba34275c6d1da0e960be F test/joinE.test d5d182f3812771e2c0d97c9dcf5dbe4c41c8e21c82560e59358731c4a3981d6b F test/joinF.test 53dd66158806823ea680dd7543b5406af151b5aafa5cd06a7f3231cd94938127 F test/joinH.test 1d2fc3190be68525fd9ce749b9468c40ba2930181e52fb5ee6f836051b38effb -F test/joinI.test 249802b168ce96d8d57943ef9abafc1e36e28d91829f68bc2b6e87f2b4d33241 +F test/joinI.test a4d37143fcc39e915d9feb08e614a13f88dfe332d77152a3c526a2370ddb9a70 F test/journal1.test bc61a4228db11bffca118bd358ba4b868524bf080f3532749de6c539656e20fa F test/journal2.test 9dac6b4ba0ca79c3b21446bbae993a462c2397c4 F test/journal3.test e5aeff93a7776cf644dbc48dec277655cff80a1cd24689036abc87869b120ea6 @@ -1349,14 +1347,13 @@ F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb9 F test/json/json-speed-check.sh 7d5898808ce7542762318306ae6075a30f5e7ee115c4a409f487e123afe91d88 x F test/json/jsonb-q1.txt 1e180fe6491efab307e318b22879e3a736ac9a96539bbde7911a13ee5b33abc7 F test/json101.test cf53254f0f0c1399a01b21fc58fee0e63a12a556be91b9ee9faccdb8b82c083c -F test/json102.test ea5c9811e408e115c8fc539548deef431fda4924c23cacd79dd4b783f4449f07 -F test/json103.test e626d109cd0bdb8282ec9bf755af3befa50e3e03a255362fc53433d31e1d66d4 +F test/json102.test 9b2e5ada10845ff84853b3feaae2ce51ce7145ae458f74c6a6cecc6ef6ee3ae1 +F test/json103.test 355746a6b66aa438f214b4fae454b13068fad2444b5f693e0d538ad1c059b264 F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 -F test/json105.test 9900caa21888289873bc6c14f5ee41213d28ac9b7ca3395f8afb73d540e80f66 +F test/json105.test 043838b56e68f3252a0dcf5be1689016f6f3f05056f8dcfcdc9d074f4d932988 F test/json106.test 4aed3afd16549045d198a8d9cea00deea96e1f2ecf55864dce96cac558b8abef F test/json107.test 59054e815c8f6b67d634d44ace421cf975828fb5651c4460aa66015c8e19d562 F test/json108.test 0a5f1e2d4b35a1bc33052563d2a5ede03052e2099e58cb424547656c898e0f49 -F test/json109.test 441cea5d73c24a1a34d284101740dfae5a082237c048c8a66b03aeebe5e3643e F test/json501.test b95e2d14988b682a5cadf079dd6162f0f85fb74cd59c6b1f1624110104a974eb F test/json502.test 4ef68e4f272dfb083d4cbceb4e9e51d67ec1186a185e0c13637c50a4dc2f9796 F test/jsonb01.test f4cdfb4cf5a0c940091b17675ed9583f45add0c938f07d65b0de0e19d3a9a101 @@ -1429,7 +1426,7 @@ F test/misc1.test e3e36262aff1bd9b8b9bf1eeb3af04adb3fc1e23f0a92dbff708bba9e939ac F test/misc2.test a1a3573cc02662becd967766021d6f16c54684d56df5f227481c7ef0d9df0bd0 F test/misc3.test 651b88bca19b8ff6a7b6af73dae00c3fd5b3ea5bee0c0d1d91abd4c4b4748718 F test/misc4.test 10cd6addb2fa9093df4751a1b92b50440175dd5468a6ec84d0386e78f087db0e -F test/misc5.test 0a5d7604e197f10ee471280bfcaaf8229f9d8e2eebfef2c8853222cbc1ea9cd5 +F test/misc5.test 02fcaf4d42405be02ec975e946270a50b0282dac98c78303ade0d1392839d2b8 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test d595599972ec0b436985f0f02f243b68500ffc977b9b3194ec66c0866cfddcab F test/misc8.test 08d2380bc435486b12161521f225043ac2be26f02471c2c1ea4cac0b1548edbd @@ -1442,12 +1439,11 @@ F test/mmap4.test 2e2b4e32555b58da15176e6fe750f17c9dcf7f93 F test/mmapcorrupt.test 470fb44fe92e99c1d23701d156f8c17865f5b027063c9119dcfdb842791f4465 F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3 F test/mmapwarm.test 2272005969cd17a910077bd5082f70bc1fefad9a875afec7fc9af483898ecaf3 -F test/modeA.sql 3f2b5a7ce7074a52b2b7ec07b07dc1a08edba19e40bce9b4d65d3965413bbea3 F test/multiplex.test d74c034e52805f6de8cc5432cef8c9eb774bb64ec29b83a22effc8ca4dac1f08 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test fac575e0b1b852025575a6a8357701d80933e98b5d2fe6d35ddaa68f92f6a1f7 F test/multiplex4.test e8ae4c4bd70606a5727743241f13b5701990abe4 -F test/mutex1.test 2cdc320a3320521d73b8090a04a2245c1e625e5f90672882517bf5fedcec8f13 +F test/mutex1.test 42cb5e244c3a77bb0ef2b967e06fa5e7ba7d32d90a9b20bed98f6f5ede185a25 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test 73ea63ab43668313e2f8cc9ef9e9a966672c7934f3ce76926fbe991235d07d91 F test/nockpt.test 3db354270fc63b6871eebd40285d4c55324fb27be629c958adbff6d7fcaa8e14 @@ -1457,13 +1453,13 @@ F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161 F test/notify3.test 796c7b7157f55c93b4e672b724e9c923a6fc6aa72ac419379a623e2350472e22 F test/notnull.test a37b663d5bb728d66fc182016613fb8e4a0a4bbf3d75b8876a7527f7d4ed3f18 -F test/notnull2.test c2c7b670fb8fa6ffe5f9cc08af88864fbb8237e28b56ad528e8dee921019c5fe +F test/notnull2.test 5b7dd6e82c409b2d011ad6acf19ae4bf0816a9c69ccf600b529d7405d7c49874 F test/notnullfault.test fc4bb7845582a2b3db376001ef49118393b1b11abe0d24adb03db057ee2b73d5 F test/null.test b7ff206a1c60fe01aa2abd33ef9ea83c93727d993ca8a613de86e925c9f2bc6f F test/nulls1.test 7a5e4346ee4285034100b4cd20e6784f16a9d6c927e44ecdf10034086bbee9c9 F test/numcast.test 5d126f7f581432e86a90d1e35cac625164aec4a1 F test/numindex1.test 20a5450d4b056e48cd5db30e659f13347a099823 -F test/offset1.test c21e67d2d5ae8ed310243fbe84fc2f0dca49e9ffed3ea89110c0d5914c0de620 +F test/offset1.test 72cca52482cbd5bc687cfa67aa2566c859081b5a353fd2f9da9bbd3914dea1ef F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 F test/optfuzz-db01.c 9f2fa80b8f84ebbf1f2e8b13421a4e0477fe300f6686fbd76cac1d2db66e0fdc F test/optfuzz-db01.txt 21f6bdeadc701cf11528276e2a55c70bfcb846ba42df327f979bd9e7b6ce7041 @@ -1512,12 +1508,6 @@ F test/printf2.test 3f55c1871a5a65507416076f6eb97e738d5210aeda7595a74ee895f2224c F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc F test/pushdown.test 46a626ef1c0ca79b85296ff2e078b9da20a50e9b804b38f441590c3987580ddd -F test/qrf01.test abc3e558a75ae2678a3172051b39960dc6fd4b298b6d594afa50939759f4037f -F test/qrf02.test 39b4afdc000bedccdafc0aecf17638df67a67aaa2d2942865ae6abcc48ba0e92 -F test/qrf03.test e7efe46d204671726b4707585126cd78d107368de4a7d0c7b8d5157cdd8624ed -F test/qrf04.test 0894692c998d2401dcc33449c02051b503ecce0c94217be54fb007c82d2d1379 -F test/qrf05.test 8ade5bfa7ef0b448e531687203fa8ae9ef41f1d7e4c11d5ba0c4846af75b13d5 -F test/qrf06.test cd7d0f0e2904904ab88141630a8fff5718ef7e3cc23e5a9c519cf29bb0919d89 F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 F test/quickcheck.test a4b7e878cd97e46108291c409b0bf8214f29e18fddd68a42bc5c1375ad1fb80a @@ -1530,9 +1520,8 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 F test/readonly.test 0d307c335b3421898cfe64a783a376138aa003849b6bff61ee2d21e805bc0051 -F test/recover.test 643139b911ac880a1e881d7621f02cfb546b608b8f2494d7d26fd5ed103b1ceb -F test/regexp1.sql de2b5b33b16b664d655b41e780f2efca38de3e5559fc254b4c9783ff0bea96b0 -F test/regexp1.test 0023eae4073265641b826a70d81ba34d4dd66ad71871a5b4a1b7cf500d5c0c51 +F test/recover.test c76d05f33f0271fba0f0752170e03b0ab5952dc61dcea7ab3ba40df03c4c42de +F test/regexp1.test 8f2a8bc1569666e29a4cee6c1a666cd224eb6d50e2470d1dc1df995170f3e0f1 F test/regexp2.test 64f9726b2ddc71aea06725fcad53231833d038d58b936d49083ace658b370a13 F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2118d F test/reservebytes.test 6163640b5a5120c0dee6591481e673a0fa0bf0d12d4da7513bad692c1a49a162 @@ -1550,13 +1539,13 @@ F test/rowid.test d27191b5ce794c05bf61081e8b2c546a1844c1641321dcaf7fb785234256cc F test/rowvalue.test 93474d8e1c496e970bdcc3a7f54ac835adda667d2fd971957b4bce0c0b11707b F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b F test/rowvalue3.test 103e9a224ca0548dd0d67e439f39c5dd16de4200221a333927372408c025324c -F test/rowvalue4.test 6e160977d44ee715e142f63ec0e339586c61f12bbbffacee369b1cdc0b7390f0 +F test/rowvalue4.test bac9326d1e886656650f67c0ec484eb5f452244a8209c6af508e9a862ace08ed F test/rowvalue5.test 00740304ea6a53a8704640c7405690f0045d5d2a6b4b04dde7bccc14c3068ea7 F test/rowvalue6.test d19b54feb604d5601f8614b15e214e0774c01087 F test/rowvalue7.test 06ec0aca725bf683313d03793aa2943bc7f45a901848c7056a9665b769c8fc38 F test/rowvalue8.test 5900eddad9e2c3c2e26f1a95f74aafc1232ee5e0 F test/rowvalue9.test 7499a8fd7ca3a3f0e19d94e135355439aa2b596f86b775ca8de79672da2ca378 -F test/rowvalueA.test 1c5ed13f3b0641452ae35e6488d6ecc16cefce99f2adf7c07c513530e2aac6b7 +F test/rowvalueA.test be8d6ad8b476eb24c151bb20bfd487e0d50c5e99618b7b0e656035069d2fc2cf F test/rowvaluefault.test 963ae9cdaed30a85a29668dd514e639f3556cae903ee9f172ea972d511c54fff F test/rowvaluevtab.test cd9747bb3f308086944c07968f547ad6b05022e698d80b9ffbdfe09ce0b8da6f F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 @@ -1570,7 +1559,7 @@ F test/savepoint7.test 24c69af86d750c80d51cf6500fde9270717f2b6e5658f055b5e75af75 F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 F test/scanstatus.test b249328caf4d317e71058006872b8012598a5fa045b30bf24a81eeff650ab49e F test/scanstatus2.test d85d17f2b0b4c013dde95232f7beab749f11f0ef847f5ecffb9486d2f5ecf9f9 -F test/schema.test e615575f2d756df4629596523f11d9322384ecf9f980e58c774cff80ff041c33 +F test/schema.test 5dd11c96ba64744de955315d2e4f8992e447533690153b93377dffb2a5ef5431 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 8ed4ae66e082cdd8b1b1f22d8549e1e7a0db4527a8e6ee8b6193053ee1e5c9ce F test/schema4.test 3b26c9fa916abb6dadf894137adcf41b7796f7b9 @@ -1588,7 +1577,7 @@ F test/select5.test 8afc5e5dcdebc2be54472e73ebd9cd1adef1225fd15d37a1c62f969159f3 F test/select6.test da91e61d26b8dea4b61e4a862088dd6ab19998f7be22a16a5b0cfe806e597639 F test/select7.test b825420da8a0b5722fdb77f3369f6396a3d198c46e8787eb26ff9425d4ac9d27 F test/select8.test 8c8f5ae43894c891efc5755ed905467d1d67ad5d -F test/select9.test 108ceff733f31698fef41eb9a0c332f150c54e98be534ee38019a19943f3f5ae +F test/select9.test f7586b207ce2304ab80dc93d3146469a28fd4403621dd3a82d06644563d3c812 F test/selectA.test 1da8ce3884c326e11d2855baffb76436b0d7e044404af8a2a70d1399a4ff7e29 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 F test/selectC.test 38c530b0cc5728b793c3c11f52b52c70290d39822224acd39011c89c1853bd31 @@ -1612,17 +1601,16 @@ F test/sharedA.test 64bdd21216dda2c6a3bd3475348ccdc108160f34682c97f2f51c19fc0e21 F test/sharedB.test 1a84863d7a2204e0d42f2e1606577c5e92e4473fa37ea0f5bdf829e4bf8ee707 F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 -F test/shell1.test 2d658ceee13d9e4361d04d0ea16340ad17784ddf378fb6e9ca6d49c682cb4bae -F test/shell2.test dc541d2681503e55466a24d35a4cbf8ca5b90b8fcdef37fc4db07373a67d31d3 +F test/shell1.test ebe953d64c937ad42a0f33170ac0d2d2568faae26813fc7a95203756446d54aa +F test/shell2.test ab23f01ea2347e4b72bb2399af7ee82aa00f9c059141749f7c4064abca5ad728 F test/shell3.test 603b448e917537cf77be0f265c05c6f63bc677c63a533c8e96aae923b56f4a0e -F test/shell4.test e25580a792b7b54560c3a76b6968bd8189261f38979fe28e6bc6312c5db280db -F test/shell5.test a9cd2c8b62e125049ef500937674f47dd6787f0157ac0515aa554044a4dc3ea9 +F test/shell4.test 03593fa7908a55f255916ffeda707cdf55680c777736e3da62b1d78cde0d684d +F test/shell5.test d17e7927ab8b7f720efbdd9b5d05fceb6c3c56c25917901b315400214bf24ef4 F test/shell6.test e3b883b61d4916b6906678a35f9d19054861123ad91b856461e0a456273bdbb8 F test/shell7.test 43fd8e511c533bab5232e95c7b4be93b243451709e89582600d4b6e67693d5c3 -F test/shell8.test 38c9e4d7e85d2a3ecfacaa9f6cda4f7a81bf4fffb5f3f37f9cd76827c6883192 +F test/shell8.test 641cf21a99c59404c24e3062923734951c4099a6b6b6520de00cf7a1249ee871 F test/shell9.test 8742a5b390cdcef6369f5aa223e415aa4255a4129ef249b177887dc635a87209 -F test/shellA.test 05cdaafa1f79913654487ce3aefa038d4106245d58f52e02faf506140a76d480 -F test/shellB.test 7123d231158588401f332bf278754687b83ba5fc5b352ec8679fb19edfb4cc0a +F test/shellA.test 4ecff8b7b2c0122ba8174abfbcc4b0f59e44d80f2a911068f8cd4cfc6661032d F test/shmlock.test 9f1f729a7fe2c46c88b156af819ac9b72c0714ac6f7246638a73c5752b5fd13c F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 @@ -1656,8 +1644,8 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 377a0c48e5a92e0b11c1c5ebb1bc9d83a7312c922bc0cb05970ef5d6a96d1f0c -F test/speedtest.md ea0c85ebe0ecff8b45ba6cdb26e694871f469009a5a29dcfe634b055f05ab241 -F test/speedtest.tcl b06f6321ef90bb68f18f7b0e430e25203d9da79b80f8926986a0d5f21ac485fb x +F test/speedtest.md ee958457ae1b729d9715ae33c0320600000bf1d9ddea1a88dcf79f56729d6fad +F test/speedtest.tcl 6b66974d833d35a63d0e9ec344e0ffa92fbbfac83e173556f700a61cb3be96fc x F test/speedtest1.c 6c01252e66f46de0b6b8d5316e03521e2151782104f3608c10262aa5dce85721 F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 @@ -1692,24 +1680,23 @@ F test/sync.test a619e407ede58a7b6e3e44375328628559fc9695a9c24c47cb5690a866b0031 F test/sync2.test 06152269ed73128782c450c355988fe8dd794d305833af75e1a5e79edd4dae47 F test/syscall.test a067468b43b8cb2305e9f9fe414e5f40c875bb5d2cba5f00b8154396e95fcf37 F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 -F test/tabfunc01.test cfa96a9a235c39fb0cae69928b989b28bfec108f62d2533486f76e32dcedfdfb +F test/tabfunc01.test 56eeae736217204bb1d9f9ef38340d48058f809b64249217cf77ff4ba600cc21 F test/table.test e87294bf1c80bfd7792142b84ab32ea5beb4f3f71e535d7fb263a6b2068377bf F test/tableapi.test e37c33e6be2276e3a96bb54b00eea7f321277115d10e5b30fdb52a112b432750 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 -F test/tclsqlite.test 5d6c73bfe7006c85e2f7fb7db8638b521eb2043d5451aaacdac4851eab895443 +F test/tclsqlite.test 3f697424cfc1cdc9c076ec0cadb0e700f059400a3e3ce134b7d856fc9f880e1c F test/tempdb.test 4cdaa23ddd8acb4d79cbb1b68ccdfd09b0537aaba909ca69a876157c2a2cbd08 F test/tempdb2.test 353864e96fd3ae2f70773d0ffbf8b1fe48589b02c2ec05013b540879410c3440 F test/tempfault.test 0c0d349c9a99bf5f374655742577f8712c647900 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d1631311a16 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 -F test/temptrigfault.tes fc5918e64f3867156fefe7cfca9d8e1f495134a5229b2b511b0dc11c07f2eab4 -F test/temptrigger.test a00f258ed8d21a0e8fd4f322f15e8cfb5cef2e43655670e07a753e3fb4769d61 -F test/tester.tcl 2d943f60200e0a36bcd3f1f0baf181a751cd3604ef6b6bd4c8dc39b4e8a53116 +F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc +F test/tester.tcl 463ae33b8bf75ac77451df19bd65e7c415c2e9891227c7c9e657d0a2d8e1074a F test/testloadext.c 862b848783eaed9985fbce46c65cd214664376b549fae252b364d5d1ef350a27 -F test/testrunner.tcl 78d67079fc39caf2af3fd9d4c30bdac78dae7ec50b9fc802835e7a5189581e07 x -F test/testrunner_data.tcl 078e251983c8fc573567125147655f68132210f226c92922daf21fb913779717 -F test/testrunner_estwork.tcl 81e2ae10238f50540f42fbf2d94913052a99bfb494b69e546506323f195dcff9 +F test/testrunner.tcl 60d7efa1816c5dfc37df3e3454b94b9042c0c8c50b27ae296d4a797cd309ace6 x +F test/testrunner_data.tcl c507a9afa911c03446ed90442ffd4a98aca02882c3d51bd1177c24795674def8 +F test/testrunner_estwork.tcl 7927a84327259a32854926f68a75292e33a61e7e052fdbfcb01f18696c99c724 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1758,7 +1745,7 @@ F test/tkt-8454a207b9.test ead80b7a01438ca1436cee029694a96c821346cf1e24f06de12f8 F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356 F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 -F test/tkt-99378177930f87bd.test 1ee631d155f0d51a4547e9405ef35a3a9a32977352a37a10bcbbacc5e38356ad +F test/tkt-99378177930f87bd.test 9d6cff39b50d062c813ae1cb0ebbd1b7acf81ecc23ae5d5215e5bb05667dc137 F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667 F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6 F test/tkt-9f2eb3abac.test cb6123ac695a08b4454c3792fbe85108f67fabf8 @@ -1806,7 +1793,7 @@ F test/tkt2213.test a9702175601a57b61aba095a233b001d6f362474 F test/tkt2251.test 5aab8c7898cd2df2a68fe19289cc29e8f5cf8c82 F test/tkt2285.test cca17be61cf600b397188e77e7143844d2b977e9 F test/tkt2332.test fc955609b958ca86dfa102832243370a0cc84070 -F test/tkt2339.test bad48bd064594aa7b4de23f6d59a72b0b0c4175fd917f4b66907584732d41652 +F test/tkt2339.test 73bd17818924cd2ac442e5fd9916b58565739450 F test/tkt2391.test ab7a11be7402da8b51a5be603425367aa0684567 F test/tkt2409.test be0d60e7d283f639dccea4b0b5e1cd3a4851fb5b F test/tkt2450.test 77ed94863f2049c1420288ddfea2d41e5e0971d6 @@ -1940,15 +1927,15 @@ F test/vacuum4.test 7ea76b769fffeb41f925303b04cbcf5a5bbeabe55e4c60ae754ff24eeeb7 F test/vacuum5.test 263b144d537e92ad8e9ca8a73cc6e1583f41cfd0dda9432b87f7806174a2f48c F test/vacuum6.test b137b04bf3392d3f5c3b8fda0ce85a6775a70ca112f6559f74ff52dc9ce042fd F test/vacuummem.test 4b30f5b95a9ff86e9d5c20741e50a898b2dc10b0962a3211571eb165357003fb -F test/values.test 0e037c50789ac2a308746567d07b53b2f6026c1bb3a435d1b099424600e64caf +F test/values.test 0eda08a6ce6545f1ab012dff4cc72a7dd0fee2510f42444136bb2b2b5ed84bc0 F test/valuesfault.test 2ef23ed965e3bd08e268cdc38a0d11653390ddbbe1e8e2e98d16f55edd30f6e8 F test/varint.test bbce22cda8fc4d135bcc2b589574be8410614e62 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 F test/view.test 3c23d7a068e9e4a0c4e6907498042772adea725f0630c3d9638ffd4e5a08b92b F test/view2.test db32c8138b5b556f610b35dfddd38c5a58a292f07fda5281eedb0851b2672679 F test/view3.test ad8a8290ee2b55ff6ce66c9ef1ce3f1e47926273a3814e1c425293e128a95456 -F test/vt02.c c2faf56d74470d569cd00741acb3f1719ee95d668f84ef58acc3872635789680 -F test/vt100-a.sql a3e188a118ca78c08b41681a4db6d0f353e554ceb33f1573b1872d16e2d30596 +F test/vt02.c 5b44ac67b1a283fedecf2d6e2ceda61e7a157f01d44dcb4490dcb1e87d057060 +F test/vt100-a.sql 631eeab18c5adb531bab79aecf64eee3934b42c75a309ee395c814717a6a7651 F test/vtab1.test 09a72330d0f31eda2ffaa828b06a6b917fb86250ee72de0301570af725774c07 F test/vtab2.test 14d4ab26cee13ba6cf5c5601b158e4f57552d3b055cdd9406cf7f711e9c84082 F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e @@ -1989,7 +1976,7 @@ F test/wal_common.tcl 204d1721ac13c5e0c7fae6380315b5ab7f4e8423f580d826c5e9df1995 F test/walbak.test 018d4e5a3d45c6298d11b99f09a8ef6876527946 F test/walbig.test f437473a16cfb314867c6b5d1dbcd519e73e3434 F test/walblock.test 6bb472e82730e7e4e81395e907a01d8cfc2bd9e1f01f8a9184ca572e2955a4bf -F test/walckptnoop.test 5f6123750f40cb86633a7e014f9fb805d0eb494b811840086dc72e554e68c7c1 +F test/walckptnoop.test b13a2c3140f2c913cfd422d9a224544757d04b8b14ab4c267ab9910467c0b9be F test/walcksum.test 50e204500eed9c691b6045e467bb2923f49aa93d8adf315e2be135fdb202c1c2 F test/walcrash.test 21038858cc552077b0522f50b0fa87e38139306a F test/walcrash2.test a0edab4e5390f03b99a790de89aad15d6ec70b36 @@ -2004,7 +1991,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8d78a1ec91299163451417b451a2bac3481f8eb9f455b1ca507a6625c927ca6e F test/walprotocol.test 1b3f922125e341703f6e946d77fdc564d38fb3e07a9385cfdc6c99cac1ecf878 F test/walprotocol2.test 7d3b6b4bf0b12f8007121b1e6ef714bc99101fb3b48e46371df1db868eebc131 -F test/walrestart.test 5168c0c2414d1971d8dec949c1070a0144cf15402361ba0d0e6a8054f5598a64 +F test/walrestart.test 3b0a9198ad2eb9f716d8f3846b133ba9f4619fb56decb1e67bba27743c766289 F test/walro.test 78a84bc0fdae1385c06b017215c426b6845734d6a5a3ac75c918dd9b801b1b9d F test/walro2.test 33955a6fd874dd9724005e17f77fef89d334b3171454a1256fe4941a96766cdc F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68 @@ -2019,7 +2006,7 @@ F test/walslow.test 0c51843836c9dcf40a5ac05aa781bfb977b396ee2c872d92bd48b79d5dd9 F test/walthread.test d562f51a61191ccfab64940df7aa1cef87c902fa5ab742590ef7f859dfe6a44b F test/walvfs.test e1a6ad0f3c78e98b55c3d5f0889cf366cc0d0a1cb2bccb44ac9ec67384adc4a1 F test/where.test 5087c72d26fd075a1644c8512be9fe18de9bf2d2b0754f7fd9b74a1c6540c4fc -F test/where2.test 1dbff4ab847068a52b927a0c1a7cf7faed4c8cae081fbcdac9b052a3a209aefa +F test/where2.test 52237a8cb27ebbf6583469429bb5733d1b94ac37d09c3dbd0f487952ed0ab3f8 F test/where3.test 4ccb156ae33de86414a52775a6f590a9d60ba2cbc7a93a24fa331b7bcf5b6030 F test/where4.test 4a371bfcc607f41d233701bdec33ac2972908ba8 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 @@ -2037,7 +2024,7 @@ F test/whereG.test 875d020ac0a47828b31e36c54f1bf0cf81c9ea43b257bc21286eca1fe9a48 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/whereI.test c4bb7e2ca56d49bd8ab5c7bd085b8b83e353922b46904d68aefb3c7468643581 F test/whereJ.test fc05e374cc9f2dc204148d6c06822c380ad388895fe97a6d335b94a26a08aecf -F test/whereK.test 4fb96b078f2ecedc467fa53177787378ff659539e415a4256cae7ae4e2a804b2 +F test/whereK.test 0270ab7f04ba5436fb9156d31d642a1c82727f4c4bfe5ba90d435c78cf44684a F test/whereL.test cb115604cc9bd61acbc99a1f1df0eb1ea7a7875a77fef25ba9282f01d10283e1 F test/whereM.test 0dbc9998783458ddcf3cc078ca7c2951d8b2677d472ecf0028f449ed327c0250 F test/whereN.test 63a3584b71acfb6963416de82f26c6b1644abc5ca6080c76546b9246734c8803 @@ -2049,7 +2036,7 @@ F test/wherelimit3.test 22d73e046870cf8bbe15573eda6b432b07ebe64a88711f9f849c6b36 F test/widetab1.test c296a98e123762de79917350e45fa33fdf88577a2571eb3a64c8bf7e44ef74d1 F test/win32heap.test 1ec2ce646aee491ec23bfcdfd005b33c79f13bf91467966f374a76ffe7c7e85f F test/win32lock.test e56d7a9b6cf9d5f3867c2dd19ff36c5326881e4038c6867610ecb3a9868ea4eb -F test/win32longpath.test 2641e3a5dbb59f49456f6caf78c0acd6ec7dbba27cb56363bab8fbfe93995caa +F test/win32longpath.test 0f9837039b306735c13521c5f25b6ed42937b600dace58e28a3d2f8baf429b6a F test/win32nolock.test 95854dc0206b8a95e4aee15a76acc082767b38f079b2e24676aed6cbb0f32798 F test/window1.test b46d28b9698559e66aa4adafd8074b940faee498bf0c4fbdb62548bfcccc67e7 F test/window2.tcl 492c125fa550cda1dd3555768a2303b3effbeceee215293adf8871efc25f1476 @@ -2074,7 +2061,7 @@ F test/windowerr.tcl f5acd6fbc210d7b5546c0e879d157888455cd4a17a1d3f28f07c1c8a387 F test/windowerr.test a8b752402109c15aa1c5efe1b93ccb0ce1ef84fa964ae1cd6684dd0b3cc1819b F test/windowfault.test 15094c1529424e62f798bc679e3fe9dfab6e8ba2f7dfe8c923b6248c31660a7c F test/windowpushd.test c420e2265f0e09a0e798d0513a660d71b51602088d81b3dbd038918ee1339dcc -F test/with1.test 31db84788e0429885b63995149fab57d32e26196b752a3a926249ae74c0adddd +F test/with1.test 1ee171d7c306ab8b0771f3511d870f56c735607729836585bbceb1fc2f47e0b1 F test/with2.test 181674a6cc86a601ca2ac052741cdfad5b529e07e870435d2f6cdb92d589ff17 F test/with3.test e30369ea27aa27eb1bda4c5e510c8a9f782c8afd2ab99d1a02b8a7f25a5d3e65 F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205 @@ -2094,12 +2081,12 @@ F test/zeroblob.test 7b74cefc7b281dfa2b07cd237987fbe94b4a2037a7771e9e83f2d5f608b F test/zeroblobfault.test 861d8191a0d944dfebb3cb4d2c5b4e46a5a119eaec5a63dd996c2389f8063441 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc F test/zipfile.test a3fcfc43115e4226fdddadd43bdf31c8ca805ad08dad435634f1633d8f5840d9 -F test/zipfile2.test 21afaffcf4f7769df38bf16e4a9c4dfa6ba1b0f5b695f844ec61fafb92db0db7 +F test/zipfile2.test a577e0775e32ef8972e7d5e9a45bc071a5ae061b5b965a08c9c4b709ad036a25 F test/zipfilefault.test 44d4d7a7f7cca7521d569d7f71026b241d65a6b1757aa409c1a168827edbbc2c F tool/GetFile.cs 47852aa0d806fe47ed1ac5138bdce7f000fe87aaa7f28107d0cb1e26682aeb44 F tool/GetTclKit.bat d84033c6a93dfe735d247f48ba00292a1cc284dcf69963e5e672444e04534bbf F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 -F tool/build-all-msvc.bat 1ee9dbadcc07fc23268025854e97b392bcbad72376b47aee7b22f3797a4f2c87 x +F tool/build-all-msvc.bat 1960a7a3e5d8176c4329e31476f6e3dfa9543675355fa9020a569f4452628458 x F tool/build-shell.sh 369c4b171cc877ad974fef691e4da782b4c1e99fe8f4361316c735f64d49280f F tool/buildtclext.tcl d09b753d7858314104eeaf5f4def85d35784470279809e47a633f142226f2b3f F tool/cg_anno.tcl c1f875f5a4c9caca3d59937b16aff716f8b1883935f1b4c9ae23124705bc8099 x @@ -2108,7 +2095,7 @@ F tool/cktclsh.sh 6075eef9c6b9ba4b38fef2ca2a66d25f2311bd3c610498d18a9b01f861629c F tool/cp.tcl 9a0d663ad45828de13763ee7ca0200f31f56c6d742cf104a56ae80e027c242d8 F tool/custom.txt 24ed55e71c5edae0067ba159bbf09240d58b160331f7716e95816cd3aa0ba5c4 F tool/dbhash.c 5da0c61032d23d74f2ab84ffc5740f0e8abec94f2c45c0b4306be7eb3ae96df0 -F tool/dbtotxt.c cfeb957571735af345f253ba8417256031fa0dddf79468eefad184262d17211e +F tool/dbtotxt.c ca48d34eaca6d6b6e4bd6a7be2b72caf34475869054240244c60fa7e69a518d6 F tool/dbtotxt.md c9a57af8739957ef36d2cfad5c4b1443ff3688ed33e4901ee200c8b651f43f3c F tool/emcc.sh.in 41a049468c8155433e37e656ba5bae063a000768b1d627025f277732c4e7c4a4 F tool/enlargedb.c 3e8b2612b985cfa7e3e8800031ee191b43ae80de96abb5abbd5eada62651ee21 @@ -2121,25 +2108,24 @@ F tool/genfkey.README e550911fa984c8255ebed2ef97824125d83806eb5232582700de949edf F tool/genfkey.test b6afd7b825d797a1e1274f519ab5695373552ecad5cd373530c63533638a5a4f F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/index_usage.c f62a0c701b2c7ff2f3e21d206f093c123f222dbf07136a10ffd1ca15a5c706c5 -F tool/lemon.c 3fdc16b23f1ea0c91c049b518fc3f75c71843dbfe2b447fcb3cd92d9e4f219f8 -F tool/lempar.c b57e1780bf8098dd4a9a5bba537f994276ea825a420f6165153e5894dc2dfb07 +F tool/lemon.c 8f6c122e5727cb0e5f302b8efc91489b1947a8d98206d7a1b1cfc0ed685b6e7c +F tool/lempar.c bdffd3b233a4e4e78056c9c01fadd2bb3fe902435abde3bce3d769fdf0d5cca2 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c 63412f9790e5e8538fbde0b4f6db154aaaf80f7a10a01e3c94d14b773a8dd5a6 F tool/logest.c c34e5944318415de513d29a6098df247a9618c96d83c38d4abd88641fe46e669 F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439 F tool/merge-test.tcl de76b62f2de2a92d4c1ca4f976bce0aea6899e0229e250479b229b2a1914b176 F tool/mkamalzip.tcl 8aa5ebe7973c8b8774062d34e15fea9815c4cc2ceea3a9b184695f005910876a -F tool/mkautoconfamal.sh 06fbe090b81c24e592c1f22b404334f805ba74d482a9260f2ac81e6f3d3386d8 +F tool/mkautoconfamal.sh 647dada5e34c466bef62a4408e1c99a7e5e1922805479dd57944f33f9803f2f8 F tool/mkccode.tcl c42a8f8cf78f92e83795d5447460dbce7aaf78a3bbf9082f1507dc71a3665f3c x -F tool/mkcombo.tcl 2a5189b219c4a495e1ff7fc980bd568d3cfb82ae9d50c84e77f7a161e96fc132 F tool/mkctimec.tcl 3fb5cad05922f5da61262cb6bcd5868a34e94a49ca8833ae2d7796e7df075576 x -F tool/mkkeywordhash.c 82d5af1d0e677900739fba59155cddac172d8c712c2d91ab73d6e6bcb30060f0 +F tool/mkkeywordhash.c 6b0be901c47f9ad42215fc995eb2f4384ac49213b1fba395102ec3e999acf559 F tool/mkmsvcmin.tcl d76c45efda1cce2d4005bcea7b8a22bb752e3256009f331120fb4fecb14ebb7a F tool/mkopcodec.tcl 33d20791e191df43209b77d37f0ff0904620b28465cca6990cf8d60da61a07ef F tool/mkopcodeh.tcl 2b4e6967a670ef21bf53a164964c35c6163277d002a4c6f56fa231d68c88d023 F tool/mkopts.tcl 680f785fdb09729fd9ac50632413da4eadbdf9071535e3f26d03795828ab07fa F tool/mkpragmatab.tcl 3801ce32f8c55fe63a3b279f231fb26c2c1a2ea9a09d2dd599239d87a609acec -F tool/mkshellc.tcl da6918b128e928a8f0d663519e14829153e59465bd5eb596442e99fa10a411b7 +F tool/mkshellc.tcl bab0a72a68384181a5706712dfdf6815f6526446d4e8aacace2de5e80cda91b2 F tool/mksourceid.c 36aa8020014aed0836fd13c51d6dc9219b0df1761d6b5f58ff5b616211b079b9 F tool/mksqlite3c-noext.tcl 351c55256213154cabb051a3c870ef9f4487de905015141ae50dc7578a901b84 F tool/mksqlite3c.tcl 7a268139158e5deef27a370bc2f8db6ccf100c1ad7ac5e5b23743c0fd354f609 @@ -2151,27 +2137,26 @@ F tool/mkvsix.tcl 67b40996a50f985a573278eea32fc5a5eb6110bdf14d33f1d8086e48c69e54 F tool/mkwinarm64ec.tcl 171f79234fa53552a129b360356df5599fdab15239caffb3d29c571292728033 F tool/offsets.c 8ed2b344d33f06e71366a9b93ccedaa38c096cc1dbd4c3c26ad08c6115285845 F tool/omittest-msvc.tcl d6b8f501ac1d7798c4126065030f89812379012cad98a1735d6d7221492abc08 -F tool/omittest.tcl 436b7072e00e25e9b77145a9f67aa8e0eeabd186168827435fd03f8f981aac32 +F tool/omittest.tcl bec70ef0e16255c8d9eb06ecd7edf823c07a60a836186cdbce3528fb34b67995 F tool/opcodesum.tcl 740ed206ba8c5040018988129abbf3089a0ccf4a F tool/pagesig.c f98909b4168d9cac11a2de7f031adea0e2f3131faa7515a72807c03ec58eafeb F tool/replace.tcl 511c61acfe563dfb58675efb4628bb158a13d48ff8322123ac447e9d25a82d9a F tool/restore_jrnl.tcl 1079ecba47cc82fa82115b81c1f68097ab1f956f357ee8da5fc4b2589af6bd98 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 -F tool/showdb.c 1faa3661d2d634f206c76794cb21d89d3ea9082d07d5e983be0f025e40f21320 +F tool/showdb.c 3956d71e5193162609a60e8c9edfcf09274c00cfea2b1d221261427adb2b5cca F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818 F tool/showlocks.c 9cc5e66d4ebbf2d194f39db2527ece92077e86ae627ddd233ee48e16e8142564 F tool/showshm.c a0ab6ec32dd1f11218ca2a4018f8fb875b59414801ab8ceed8b2e69b7b45a809 F tool/showstat4.c b706fcbc4cd1a6e4a73ac32549afc4b460479d650402d64b23e8d813516e8de4 -F tool/showtmlog.c 2e9da6c4b4767113a0ad5ddabd4337ea100d38ff9c7fee260f9ccdefb2ffdc23 F tool/showwal.c 11eca547980a066b081f512636151233350ac679f29ecf4ebfce7f4530230b3d F tool/soak1.tcl a3892082ed1079671565c044e93b55c3c7f38829aedf53cc597c65d23ffdaddf F tool/spaceanal.tcl 1f83962090a6b60e1d7bf92495d643e622bef9fe82ea3f2d22350dcbce9a12d0 F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 4969fd642dad0ea483e4e104163021d92baf98f6a8eac981fe48525f9b873430 -F tool/sqldiff.c 847edc1e0d1e1feb652d3d6128e504456deaf254ab9ad3e7cebd4317d2037182 +F tool/sqldiff.c 134be7866be19f8beb32043d5aea5657f01aaeae2df8d33d758ff722c78666b9 F tool/sqlite3_analyzer.c.in 14f02cb5ec3c264cd6107d1f1dad77092b1cf440fc196c30b69ae87b56a1a43b -F tool/sqlite3_rsync.c f510a8b230e1c5b0f62842acd0e94ff15d2f77a00ae782f7d20f9e39919fa19b -F tool/sqltclsh.c.in c103c6fc7d42bce611f9d4596774d60b7ef3d0b291a1f58c9e6184e458b89296 +F tool/sqlite3_rsync.c d0e58a1e49fe2192c3ee0b697aed182d502bebfe5b4b406ba6b2baa52a04ecbe +F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 F tool/src-verify.c 6c655d9a8d6b30f3648fc78a79bf3838ed68f8543869d380c43ea9f17b3b8501 F tool/srcck1.c 559e703c6cca1d70398bdba1d7f91036c1a71adf718a1aaa6401a562ccaed154 @@ -2188,12 +2173,10 @@ F tool/version-info.c 33d0390ef484b3b1cb685d59362be891ea162123cea181cb8e6d2cf6dd F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 5fa49c4d592778fb82c4e25c77cf0442d3dc23cc7f8d91d25952c722af866930 -R b4a00b0f2cf0b157145ad751c36ae742 -T +sym-major-release * +P 6ca269b677c6cdc03628fc4f3634ee1b9b956170f9bc41ba8d37b15cc6571a88 +R 6723709c10ad0e4cbb0f48842d99491e T +sym-release * -T +sym-version-3.52.0 * +T +sym-version-3.51.3 * U drh -Z ee8b8689818d3dfbd1a3f7d6f78a36ce +Z 8a6b03453203488391a91516890c1c7d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.tags b/manifest.tags index e0b8607c1..cb78ee568 100644 --- a/manifest.tags +++ b/manifest.tags @@ -1,5 +1,4 @@ -branch trunk -tag trunk +branch branch-3.51 tag release -tag major-release -tag version-3.52.0 +tag branch-3.51 +tag version-3.51.3 diff --git a/manifest.uuid b/manifest.uuid index 55fe12eaa..bae34afb9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -557aeb43869d3585137b17690cb3b64f7de6921774daae9e56403c3717dceab6 +737ae4a34738ffa0c3ff7f9bb18df914dd1cad163f28fd6b6e114a344fe6d618 diff --git a/src/alter.c b/src/alter.c index fb5a37935..a7255e75e 100644 --- a/src/alter.c +++ b/src/alter.c @@ -491,7 +491,7 @@ void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ /* Look up the table being altered. */ assert( pParse->pNewTable==0 ); assert( sqlite3BtreeHoldsAllMutexes(db) ); - if( NEVER(db->mallocFailed) ) goto exit_begin_add_column; + if( db->mallocFailed ) goto exit_begin_add_column; pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); if( !pTab ) goto exit_begin_add_column; @@ -563,7 +563,7 @@ void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ ** Or, if pTab is not a view or virtual table, zero is returned. */ #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) -static int isRealTable(Parse *pParse, Table *pTab, int iOp){ +static int isRealTable(Parse *pParse, Table *pTab, int bDrop){ const char *zType = 0; #ifndef SQLITE_OMIT_VIEW if( IsView(pTab) ){ @@ -576,12 +576,9 @@ static int isRealTable(Parse *pParse, Table *pTab, int iOp){ } #endif if( zType ){ - const char *azMsg[] = { - "rename columns of", "drop column from", "edit constraints of" - }; - assert( iOp>=0 && iOp<ArraySize(azMsg) ); sqlite3ErrorMsg(pParse, "cannot %s %s \"%s\"", - azMsg[iOp], zType, pTab->zName + (bDrop ? "drop column from" : "rename columns of"), + zType, pTab->zName ); return 1; } @@ -1052,25 +1049,6 @@ static RenameToken *renameColumnTokenNext(RenameCtx *pCtx){ return pBest; } -/* -** Set the error message of the context passed as the first argument to -** the result of formatting zFmt using printf() style formatting. -*/ -static void errorMPrintf(sqlite3_context *pCtx, const char *zFmt, ...){ - sqlite3 *db = sqlite3_context_db_handle(pCtx); - char *zErr = 0; - va_list ap; - va_start(ap, zFmt); - zErr = sqlite3VMPrintf(db, zFmt, ap); - va_end(ap); - if( zErr ){ - sqlite3_result_error(pCtx, zErr, -1); - sqlite3DbFree(db, zErr); - }else{ - sqlite3_result_error_nomem(pCtx); - } -} - /* ** An error occurred while parsing or otherwise processing a database ** object (either pParse->pNewTable, pNewIndex or pNewTrigger) as part of an @@ -1368,8 +1346,8 @@ static int renameResolveTrigger(Parse *pParse){ sqlite3SelectPrep(pParse, pStep->pSelect, &sNC); if( pParse->nErr ) rc = pParse->rc; } - if( rc==SQLITE_OK && pStep->pSrc ){ - SrcList *pSrc = sqlite3SrcListDup(db, pStep->pSrc, 0); + if( rc==SQLITE_OK && pStep->zTarget ){ + SrcList *pSrc = sqlite3TriggerStepSrc(pParse, pStep); if( pSrc ){ Select *pSel = sqlite3SelectNew( pParse, pStep->pExprList, pSrc, 0, 0, 0, 0, 0, 0 @@ -1397,10 +1375,10 @@ static int renameResolveTrigger(Parse *pParse){ pSel->pSrc = 0; sqlite3SelectDelete(db, pSel); } - if( ALWAYS(pStep->pSrc) ){ + if( pStep->pFrom ){ int i; - for(i=0; i<pStep->pSrc->nSrc && rc==SQLITE_OK; i++){ - SrcItem *p = &pStep->pSrc->a[i]; + for(i=0; i<pStep->pFrom->nSrc && rc==SQLITE_OK; i++){ + SrcItem *p = &pStep->pFrom->a[i]; if( p->fg.isSubquery ){ assert( p->u4.pSubq!=0 ); sqlite3SelectPrep(pParse, p->u4.pSubq->pSelect, 0); @@ -1469,13 +1447,13 @@ static void renameWalkTrigger(Walker *pWalker, Trigger *pTrigger){ sqlite3WalkExpr(pWalker, pUpsert->pUpsertWhere); sqlite3WalkExpr(pWalker, pUpsert->pUpsertTargetWhere); } - if( pStep->pSrc ){ + if( pStep->pFrom ){ int i; - SrcList *pSrc = pStep->pSrc; - for(i=0; i<pSrc->nSrc; i++){ - if( pSrc->a[i].fg.isSubquery ){ - assert( pSrc->a[i].u4.pSubq!=0 ); - sqlite3WalkSelect(pWalker, pSrc->a[i].u4.pSubq->pSelect); + SrcList *pFrom = pStep->pFrom; + for(i=0; i<pFrom->nSrc; i++){ + if( pFrom->a[i].fg.isSubquery ){ + assert( pFrom->a[i].u4.pSubq!=0 ); + sqlite3WalkSelect(pWalker, pFrom->a[i].u4.pSubq->pSelect); } } } @@ -1646,8 +1624,8 @@ static void renameColumnFunc( if( rc!=SQLITE_OK ) goto renameColumnFunc_done; for(pStep=sParse.pNewTrigger->step_list; pStep; pStep=pStep->pNext){ - if( pStep->pSrc ){ - Table *pTarget = sqlite3LocateTableItem(&sParse, 0, &pStep->pSrc->a[0]); + if( pStep->zTarget ){ + Table *pTarget = sqlite3LocateTable(&sParse, 0, pStep->zTarget, zDb); if( pTarget==pTab ){ if( pStep->pUpsert ){ ExprList *pUpsertSet = pStep->pUpsert->pUpsertSet; @@ -1659,6 +1637,7 @@ static void renameColumnFunc( } } + /* Find tokens to edit in UPDATE OF clause */ if( sParse.pTriggerTab==pTab ){ renameColumnIdlistNames(&sParse, &sCtx,sParse.pNewTrigger->pColumns,zOld); @@ -1860,10 +1839,13 @@ static void renameTableFunc( if( rc==SQLITE_OK ){ renameWalkTrigger(&sWalker, pTrigger); for(pStep=pTrigger->step_list; pStep; pStep=pStep->pNext){ - if( pStep->pSrc ){ + if( pStep->zTarget && 0==sqlite3_stricmp(pStep->zTarget, zOld) ){ + renameTokenFind(&sParse, &sCtx, pStep->zTarget); + } + if( pStep->pFrom ){ int i; - for(i=0; i<pStep->pSrc->nSrc; i++){ - SrcItem *pItem = &pStep->pSrc->a[i]; + for(i=0; i<pStep->pFrom->nSrc; i++){ + SrcItem *pItem = &pStep->pFrom->a[i]; if( 0==sqlite3_stricmp(pItem->zName, zOld) ){ renameTokenFind(&sParse, &sCtx, pItem->zName); } @@ -2110,57 +2092,6 @@ static void renameTableTest( #endif } - -/* -** Return the number of bytes until the end of the next non-whitespace and -** non-comment token. For the purpose of this function, a "(" token includes -** all of the bytes through and including the matching ")", or until the -** first illegal token, whichever comes first. -** -** Write the token type into *piToken. -** -** The value returned is the number of bytes in the token itself plus -** the number of bytes of leading whitespace and comments skipped plus -** all bytes through the next matching ")" if the token is TK_LP. -** -** Example: (Note: '.' used in place of '*' in the example z[] text) -** -** ,--------- *piToken := TK_RP -** v -** z[] = " /.comment./ --comment\n (two three four) five" -** | | -** |<-------------------------------------->| -** | -** `--- return value -*/ -static int getConstraintToken(const u8 *z, int *piToken){ - int iOff = 0; - int t = 0; - do { - iOff += sqlite3GetToken(&z[iOff], &t); - }while( t==TK_SPACE || t==TK_COMMENT ); - - *piToken = t; - - if( t==TK_LP ){ - int nNest = 1; - while( nNest>0 ){ - iOff += sqlite3GetToken(&z[iOff], &t); - if( t==TK_LP ){ - nNest++; - }else if( t==TK_RP ){ - t = TK_LP; - nNest--; - }else if( t==TK_ILLEGAL ){ - break; - } - } - } - - *piToken = t; - return iOff; -} - /* ** The implementation of internal UDF sqlite_drop_column(). ** @@ -2205,24 +2136,15 @@ static void dropColumnFunc( goto drop_column_done; } + pCol = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol].zCnName); if( iCol<pTab->nCol-1 ){ RenameToken *pEnd; - pCol = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol].zCnName); pEnd = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol+1].zCnName); zEnd = (const char*)pEnd->t.z; }else{ - int eTok; assert( IsOrdinaryTable(pTab) ); - assert( iCol!=0 ); - /* Point pCol->t.z at the "," immediately preceding the definition of - ** the column being dropped. To do this, start at the name of the - ** previous column, and tokenize until the next ",". */ - pCol = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol-1].zCnName); - do { - pCol->t.z += getConstraintToken((const u8*)pCol->t.z, &eTok); - }while( eTok!=TK_COMMA ); - pCol->t.z--; zEnd = (const char*)&zSql[pTab->u.tab.addColOffset]; + while( ALWAYS(pCol->t.z[0]!=0) && pCol->t.z[0]!=',' ) pCol->t.z--; } zNew = sqlite3MPrintf(db, "%.*s%s", pCol->t.z-zSql, zSql, zEnd); @@ -2391,651 +2313,6 @@ void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const Token *pName){ sqlite3SrcListDelete(db, pSrc); } -/* -** Return the number of bytes of leading whitespace/comments in string z[]. -*/ -static int getWhitespace(const u8 *z){ - int nRet = 0; - while( 1 ){ - int t = 0; - int n = sqlite3GetToken(&z[nRet], &t); - if( t!=TK_SPACE && t!=TK_COMMENT ) break; - nRet += n; - } - return nRet; -} - - -/* -** Argument z points into the body of a constraint - specifically the -** second token of the constraint definition. For a named constraint, -** z points to the first token past the CONSTRAINT keyword. For an -** unnamed NOT NULL constraint, z points to the first byte past the NOT -** keyword. -** -** Return the number of bytes until the end of the constraint. -*/ -static int getConstraint(const u8 *z){ - int iOff = 0; - int t = 0; - - /* Now, the current constraint proceeds until the next occurence of one - ** of the following tokens: - ** - ** CONSTRAINT, PRIMARY, NOT, UNIQUE, CHECK, DEFAULT, - ** COLLATE, REFERENCES, FOREIGN, GENERATED, AS, RP, or COMMA - ** - ** Also exit the loop if ILLEGAL turns up. - */ - while( 1 ){ - int n = getConstraintToken(&z[iOff], &t); - if( t==TK_CONSTRAINT || t==TK_PRIMARY || t==TK_NOT || t==TK_UNIQUE - || t==TK_CHECK || t==TK_DEFAULT || t==TK_COLLATE || t==TK_REFERENCES - || t==TK_FOREIGN || t==TK_RP || t==TK_COMMA || t==TK_ILLEGAL - || t==TK_AS || t==TK_GENERATED - ){ - break; - } - iOff += n; - } - - return iOff; -} - -/* -** Compare two constraint names. -** -** Summary: *pRes := zQuote != zCmp -** -** Details: -** Compare the (possibly quoted) constraint name zQuote[0..nQuote-1] -** against zCmp[]. Write zero into *pRes if they are the same and -** non-zero if they differ. Normally return SQLITE_OK, except if there -** is an OOM, set the OOM error condition on ctx and return SQLITE_NOMEM. -*/ -static int quotedCompare( - sqlite3_context *ctx, /* Function context on which to report errors */ - int t, /* Token type */ - const u8 *zQuote, /* Possibly quoted text. Not zero-terminated. */ - int nQuote, /* Length of zQuote in bytes */ - const u8 *zCmp, /* Zero-terminated, unquoted name to compare against */ - int *pRes /* OUT: Set to 0 if equal, non-zero if unequal */ -){ - char *zCopy = 0; /* De-quoted, zero-terminated copy of zQuote[] */ - - if( t==TK_ILLEGAL ){ - *pRes = 1; - return SQLITE_OK; - } - zCopy = sqlite3MallocZero(nQuote+1); - if( zCopy==0 ){ - sqlite3_result_error_nomem(ctx); - return SQLITE_NOMEM_BKPT; - } - memcpy(zCopy, zQuote, nQuote); - sqlite3Dequote(zCopy); - *pRes = sqlite3_stricmp((const char*)zCopy, (const char*)zCmp); - sqlite3_free(zCopy); - return SQLITE_OK; -} - -/* -** zSql[] is a CREATE TABLE statement, supposedly. Find the offset -** into zSql[] of the first character past the first "(" and write -** that offset into *piOff and return SQLITE_OK. Or, if not found, -** set the SQLITE_CORRUPT error code and return SQLITE_ERROR. -*/ -static int skipCreateTable(sqlite3_context *ctx, const u8 *zSql, int *piOff){ - int iOff = 0; - - if( zSql==0 ) return SQLITE_ERROR; - - /* Jump past the "CREATE TABLE" bit. */ - while( 1 ){ - int t = 0; - iOff += sqlite3GetToken(&zSql[iOff], &t); - if( t==TK_LP ) break; - if( t==TK_ILLEGAL ){ - sqlite3_result_error_code(ctx, SQLITE_CORRUPT_BKPT); - return SQLITE_ERROR; - } - } - - *piOff = iOff; - return SQLITE_OK; -} - -/* -** Internal SQL function sqlite3_drop_constraint(): Given an input -** CREATE TABLE statement, return a revised CREATE TABLE statement -** with a constraint removed. Two forms, depending on the datatype -** of argv[2]: -** -** sqlite_drop_constraint(SQL, INT) -- Omit NOT NULL from the INT-th column -** sqlite_drop_constraint(SQL, TEXT) -- OMIT constraint with name TEXT -** -** In the first case, the left-most column is 0. -*/ -static void dropConstraintFunc( - sqlite3_context *ctx, - int NotUsed, - sqlite3_value **argv -){ - const u8 *zSql = sqlite3_value_text(argv[0]); - const u8 *zCons = 0; - int iNotNull = -1; - int ii; - int iOff = 0; - int iStart = 0; - int iEnd = 0; - char *zNew = 0; - int t = 0; - sqlite3 *db; - UNUSED_PARAMETER(NotUsed); - - if( zSql==0 ) return; - - /* Jump past the "CREATE TABLE" bit. */ - if( skipCreateTable(ctx, zSql, &iOff) ) return; - - if( sqlite3_value_type(argv[1])==SQLITE_INTEGER ){ - iNotNull = sqlite3_value_int(argv[1]); - }else{ - zCons = sqlite3_value_text(argv[1]); - } - - /* Search for the named constraint within column definitions. */ - for(ii=0; iEnd==0; ii++){ - - /* Now parse the column or table constraint definition. Search - ** for the token CONSTRAINT if this is a DROP CONSTRAINT command, or - ** NOT in the right column if this is a DROP NOT NULL. */ - while( 1 ){ - iStart = iOff; - iOff += getConstraintToken(&zSql[iOff], &t); - if( t==TK_CONSTRAINT && (zCons || iNotNull==ii) ){ - /* Check if this is the constraint we are searching for. */ - int nTok = 0; - int cmp = 1; - - /* Skip past any whitespace. */ - iOff += getWhitespace(&zSql[iOff]); - - /* Compare the next token - which may be quoted - with the name of - ** the constraint being dropped. */ - nTok = getConstraintToken(&zSql[iOff], &t); - if( zCons ){ - if( quotedCompare(ctx, t, &zSql[iOff], nTok, zCons, &cmp) ) return; - } - iOff += nTok; - - /* The next token is usually the first token of the constraint - ** definition. This is enough to tell the type of the constraint - - ** TK_NOT means it is a NOT NULL, TK_CHECK a CHECK constraint etc. - ** - ** There is also the chance that the next token is TK_CONSTRAINT - ** (or TK_DEFAULT or TK_COLLATE), for example if a table has been - ** created as follows: - ** - ** CREATE TABLE t1(cols, CONSTRAINT one CONSTRAINT two NOT NULL); - ** - ** In this case, allow the "CONSTRAINT one" bit to be dropped by - ** this command if that is what is requested, or to advance to - ** the next iteration of the loop with &zSql[iOff] still pointing - ** to the CONSTRAINT keyword. */ - nTok = getConstraintToken(&zSql[iOff], &t); - if( t==TK_CONSTRAINT || t==TK_DEFAULT || t==TK_COLLATE - || t==TK_COMMA || t==TK_RP || t==TK_GENERATED || t==TK_AS - ){ - t = TK_CHECK; - }else{ - iOff += nTok; - iOff += getConstraint(&zSql[iOff]); - } - - if( cmp==0 || (iNotNull>=0 && t==TK_NOT) ){ - if( t!=TK_NOT && t!=TK_CHECK ){ - errorMPrintf(ctx, "constraint may not be dropped: %s", zCons); - return; - } - iEnd = iOff; - break; - } - - }else if( t==TK_NOT && iNotNull==ii ){ - iEnd = iOff + getConstraint(&zSql[iOff]); - break; - }else if( t==TK_RP || t==TK_ILLEGAL ){ - iEnd = -1; - break; - }else if( t==TK_COMMA ){ - break; - } - } - } - - /* If the constraint has not been found it is an error. */ - if( iEnd<=0 ){ - if( zCons ){ - errorMPrintf(ctx, "no such constraint: %s", zCons); - }else{ - /* SQLite follows postgres in that a DROP NOT NULL on a column that is - ** not NOT NULL is not an error. So just return the original SQL here. */ - sqlite3_result_text(ctx, (const char*)zSql, -1, SQLITE_TRANSIENT); - } - }else{ - - /* Figure out if an extra space should be inserted after the constraint - ** is removed. And if an additional comma preceding the constraint - ** should be removed. */ - const char *zSpace = " "; - iEnd += getWhitespace(&zSql[iEnd]); - sqlite3GetToken(&zSql[iEnd], &t); - if( t==TK_RP || t==TK_COMMA ){ - zSpace = ""; - if( zSql[iStart-1]==',' ) iStart--; - } - - db = sqlite3_context_db_handle(ctx); - zNew = sqlite3MPrintf(db, "%.*s%s%s", iStart, zSql, zSpace, &zSql[iEnd]); - sqlite3_result_text(ctx, zNew, -1, SQLITE_DYNAMIC); - } -} - -/* -** Internal SQL function: -** -** sqlite_add_constraint(SQL, CONSTRAINT-TEXT, ICOL) -** -** SQL is a CREATE TABLE statement. Return a modified version of -** SQL that adds CONSTRAINT-TEXT at the end of the ICOL-th column -** definition. (The left-most column defintion is 0.) -*/ -static void addConstraintFunc( - sqlite3_context *ctx, - int NotUsed, - sqlite3_value **argv -){ - const u8 *zSql = sqlite3_value_text(argv[0]); - const char *zCons = (const char*)sqlite3_value_text(argv[1]); - int iCol = sqlite3_value_int(argv[2]); - int iOff = 0; - int ii; - char *zNew = 0; - int t = 0; - sqlite3 *db; - UNUSED_PARAMETER(NotUsed); - - if( skipCreateTable(ctx, zSql, &iOff) ) return; - - for(ii=0; ii<=iCol || (iCol<0 && t!=TK_RP); ii++){ - iOff += getConstraintToken(&zSql[iOff], &t); - while( 1 ){ - int nTok = getConstraintToken(&zSql[iOff], &t); - if( t==TK_COMMA || t==TK_RP ) break; - if( t==TK_ILLEGAL ){ - sqlite3_result_error_code(ctx, SQLITE_CORRUPT_BKPT); - return; - } - iOff += nTok; - } - } - - iOff += getWhitespace(&zSql[iOff]); - - db = sqlite3_context_db_handle(ctx); - if( iCol<0 ){ - zNew = sqlite3MPrintf(db, "%.*s, %s%s", iOff, zSql, zCons, &zSql[iOff]); - }else{ - zNew = sqlite3MPrintf(db, "%.*s %s%s", iOff, zSql, zCons, &zSql[iOff]); - } - sqlite3_result_text(ctx, zNew, -1, SQLITE_DYNAMIC); -} - -/* -** Find a column named pCol in table pTab. If successful, set output -** parameter *piCol to the index of the column in the table and return -** SQLITE_OK. Otherwise, set *piCol to -1 and return an SQLite error -** code. -*/ -static int alterFindCol(Parse *pParse, Table *pTab, Token *pCol, int *piCol){ - sqlite3 *db = pParse->db; - char *zName = sqlite3NameFromToken(db, pCol); - int rc = SQLITE_NOMEM; - int iCol = -1; - - if( zName ){ - iCol = sqlite3ColumnIndex(pTab, zName); - if( iCol<0 ){ - sqlite3ErrorMsg(pParse, "no such column: %s", zName); - rc = SQLITE_ERROR; - }else{ - rc = SQLITE_OK; - } - } - -#ifndef SQLITE_OMIT_AUTHORIZATION - if( rc==SQLITE_OK ){ - const char *zDb = db->aDb[sqlite3SchemaToIndex(db, pTab->pSchema)].zDbSName; - const char *zCol = pTab->aCol[iCol].zCnName; - if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, zCol) ){ - pTab = 0; - } - } -#endif - - sqlite3DbFree(db, zName); - *piCol = iCol; - return rc; -} - - -/* -** Find the table named by the first entry in source list pSrc. If successful, -** return a pointer to the Table structure and set output variable (*pzDb) -** to point to the name of the database containin the table (i.e. "main", -** "temp" or the name of an attached database). -** -** If the table cannot be located, return NULL. The value of the two output -** parameters is undefined in this case. -*/ -static Table *alterFindTable( - Parse *pParse, /* Parsing context */ - SrcList *pSrc, /* Name of the table to look for */ - int *piDb, /* OUT: write the iDb here */ - const char **pzDb, /* OUT: write name of schema here */ - int bAuth /* Do ALTER TABLE authorization checks if true */ -){ - sqlite3 *db = pParse->db; - Table *pTab = 0; - assert( sqlite3BtreeHoldsAllMutexes(db) ); - pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); - if( pTab ){ - int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - *pzDb = db->aDb[iDb].zDbSName; - *piDb = iDb; - - if( SQLITE_OK!=isRealTable(pParse, pTab, 2) - || SQLITE_OK!=isAlterableTable(pParse, pTab) - ){ - pTab = 0; - } - } -#ifndef SQLITE_OMIT_AUTHORIZATION - if( pTab && bAuth ){ - if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, *pzDb, pTab->zName, 0) ){ - pTab = 0; - } - } -#endif - sqlite3SrcListDelete(db, pSrc); - return pTab; -} - -/* -** Generate bytecode for one of: -** -** (1) ALTER TABLE pSrc DROP CONSTRAINT pCons -** (2) ALTER TABLE pSrc ALTER pCol DROP NOT NULL -** -** One of pCons and pCol must be NULL and the other non-null. -*/ -void sqlite3AlterDropConstraint( - Parse *pParse, /* Parsing context */ - SrcList *pSrc, /* The table being altered */ - Token *pCons, /* Name of the constraint to drop */ - Token *pCol /* Name of the column from which to remove the NOT NULL */ -){ - sqlite3 *db = pParse->db; - Table *pTab = 0; - int iDb = 0; - const char *zDb = 0; - char *zArg = 0; - - assert( (pCol==0)!=(pCons==0) ); - assert( pSrc->nSrc==1 ); - pTab = alterFindTable(pParse, pSrc, &iDb, &zDb, pCons!=0); - if( !pTab ) return; - - if( pCons ){ - zArg = sqlite3MPrintf(db, "%.*Q", pCons->n, pCons->z); - }else{ - int iCol; - if( alterFindCol(pParse, pTab, pCol, &iCol) ) return; - zArg = sqlite3MPrintf(db, "%d", iCol); - } - - /* Edit the SQL for the named table. */ - sqlite3NestedParse(pParse, - "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE " SET " - "sql = sqlite_drop_constraint(sql, %s) " - "WHERE type='table' AND tbl_name=%Q COLLATE nocase" - , zDb, zArg, pTab->zName - ); - sqlite3DbFree(db, zArg); - - /* Finally, reload the database schema. */ - renameReloadSchema(pParse, iDb, INITFLAG_AlterDropCons); -} - -/* -** The implementation of SQL function sqlite_fail(MSG). This takes a single -** argument, and returns it as an error message with the error code set to -** SQLITE_CONSTRAINT. -*/ -static void failConstraintFunc( - sqlite3_context *ctx, - int NotUsed, - sqlite3_value **argv -){ - const char *zText = (const char*)sqlite3_value_text(argv[0]); - int err = sqlite3_value_int(argv[1]); - (void)NotUsed; - sqlite3_result_error(ctx, zText, -1); - sqlite3_result_error_code(ctx, err); -} - -/* -** Buffer pCons, which is nCons bytes in size, contains the text of a -** NOT NULL or CHECK constraint that will be inserted into a CREATE TABLE -** statement. If successful, this function returns the size of the buffer in -** bytes not including any trailing whitespace or "--" style comments. Or, -** if an OOM occurs, it returns 0 and sets db->mallocFailed to true. -** -** C-style comments at the end are preserved. "--" style comments are -** removed because the comment terminator might be \000, and we are about -** to insert the pCons[] text into the middle of a larger string, and that -** will have the effect of removing the comment terminator and messing up -** the syntax. -*/ -static int alterRtrimConstraint( - sqlite3 *db, /* used to record OOM error */ - const char *pCons, /* Buffer containing constraint */ - int nCons /* Size of pCons in bytes */ -){ - u8 *zTmp = (u8*)sqlite3MPrintf(db, "%.*s", nCons, pCons); - int iOff = 0; - int iEnd = 0; - - if( zTmp==0 ) return 0; - - while( 1 ){ - int t = 0; - int nToken = sqlite3GetToken(&zTmp[iOff], &t); - if( t==TK_ILLEGAL ) break; - if( t!=TK_SPACE && (t!=TK_COMMENT || zTmp[iOff]!='-') ){ - iEnd = iOff+nToken; - } - iOff += nToken; - } - - sqlite3DbFree(db, zTmp); - return iEnd; -} - -/* -** Prepare a statement of the form: -** -** ALTER TABLE pSrc ALTER pCol SET NOT NULL -*/ -void sqlite3AlterSetNotNull( - Parse *pParse, /* Parsing context */ - SrcList *pSrc, /* Name of the table being altered */ - Token *pCol, /* Name of the column to add a NOT NULL constraint to */ - Token *pFirst /* The NOT token of the NOT NULL constraint text */ -){ - Table *pTab = 0; - int iCol = 0; - int iDb = 0; - const char *zDb = 0; - const char *pCons = 0; - int nCons = 0; - - /* Look up the table being altered. */ - assert( pSrc->nSrc==1 ); - pTab = alterFindTable(pParse, pSrc, &iDb, &zDb, 0); - if( !pTab ) return; - - /* Find the column being altered. */ - if( alterFindCol(pParse, pTab, pCol, &iCol) ){ - return; - } - - /* Find the length in bytes of the constraint definition */ - pCons = pFirst->z; - nCons = alterRtrimConstraint(pParse->db, pCons, pParse->sLastToken.z - pCons); - - /* Search for a constraint violation. Throw an exception if one is found. */ - sqlite3NestedParse(pParse, - "SELECT sqlite_fail('constraint failed', %d) " - "FROM %Q.%Q AS x WHERE x.%.*s IS NULL", - SQLITE_CONSTRAINT, zDb, pTab->zName, (int)pCol->n, pCol->z - ); - - /* Edit the SQL for the named table. */ - sqlite3NestedParse(pParse, - "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE " SET " - "sql = sqlite_add_constraint(sqlite_drop_constraint(sql, %d), %.*Q, %d) " - "WHERE type='table' AND tbl_name=%Q COLLATE nocase" - , zDb, iCol, nCons, pCons, iCol, pTab->zName - ); - - /* Finally, reload the database schema. */ - renameReloadSchema(pParse, iDb, INITFLAG_AlterDropCons); -} - -/* -** Implementation of internal SQL function: -** -** sqlite_find_constraint(SQL, CONSTRAINT-NAME) -** -** This function returns true if the SQL passed as the first argument is a -** CREATE TABLE that contains a constraint with the name CONSTRAINT-NAME, -** or false otherwise. -*/ -static void findConstraintFunc( - sqlite3_context *ctx, - int NotUsed, - sqlite3_value **argv -){ - const u8 *zSql = 0; - const u8 *zCons = 0; - int iOff = 0; - int t = 0; - - (void)NotUsed; - zSql = sqlite3_value_text(argv[0]); - zCons = sqlite3_value_text(argv[1]); - - if( zSql==0 || zCons==0 ) return; - while( t!=TK_LP && t!=TK_ILLEGAL ){ - iOff += sqlite3GetToken(&zSql[iOff], &t); - } - - while( 1 ){ - iOff += getConstraintToken(&zSql[iOff], &t); - if( t==TK_CONSTRAINT ){ - int nTok = 0; - int cmp = 0; - iOff += getWhitespace(&zSql[iOff]); - nTok = getConstraintToken(&zSql[iOff], &t); - if( quotedCompare(ctx, t, &zSql[iOff], nTok, zCons, &cmp) ) return; - if( cmp==0 ){ - sqlite3_result_int(ctx, 1); - return; - } - }else if( t==TK_ILLEGAL ){ - break; - } - } - - sqlite3_result_int(ctx, 0); -} - -/* -** Generate bytecode to implement: -** -** ALTER TABLE pSrc ADD [CONSTRAINT pName] CHECK(pExpr) -** -** Any "ON CONFLICT" text that occurs after the "CHECK(...)", up -** until pParse->sLastToken, is included as part of the new constraint. -*/ -void sqlite3AlterAddConstraint( - Parse *pParse, /* Parse context */ - SrcList *pSrc, /* Table to add constraint to */ - Token *pFirst, /* First token of new constraint */ - Token *pName, /* Name of new constraint. NULL if name omitted. */ - const char *pExpr, /* Text of CHECK expression */ - int nExpr /* Size of pExpr in bytes */ -){ - Table *pTab = 0; /* Table identified by pSrc */ - int iDb = 0; /* Which schema does pTab live in */ - const char *zDb = 0; /* Name of the schema in which pTab lives */ - const char *pCons = 0; /* Text of the constraint */ - int nCons; /* Bytes of text to use from pCons[] */ - - /* Look up the table being altered. */ - assert( pSrc->nSrc==1 ); - pTab = alterFindTable(pParse, pSrc, &iDb, &zDb, 1); - if( !pTab ) return; - - /* If this new constraint has a name, check that it is not a duplicate of - ** an existing constraint. It is an error if it is. */ - if( pName ){ - char *zName = sqlite3NameFromToken(pParse->db, pName); - - sqlite3NestedParse(pParse, - "SELECT sqlite_fail('constraint %q already exists', %d) " - "FROM \"%w\"." LEGACY_SCHEMA_TABLE " " - "WHERE type='table' AND tbl_name=%Q COLLATE nocase " - "AND sqlite_find_constraint(sql, %Q)", - zName, SQLITE_ERROR, zDb, pTab->zName, zName - ); - sqlite3DbFree(pParse->db, zName); - } - - /* Search for a constraint violation. Throw an exception if one is found. */ - sqlite3NestedParse(pParse, - "SELECT sqlite_fail('constraint failed', %d) " - "FROM %Q.%Q WHERE (%.*s) IS NOT TRUE", - SQLITE_CONSTRAINT, zDb, pTab->zName, nExpr, pExpr - ); - - /* Edit the SQL for the named table. */ - pCons = pFirst->z; - nCons = alterRtrimConstraint(pParse->db, pCons, pParse->sLastToken.z - pCons); - - sqlite3NestedParse(pParse, - "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE " SET " - "sql = sqlite_add_constraint(sql, %.*Q, -1) " - "WHERE type='table' AND tbl_name=%Q COLLATE nocase" - , zDb, nCons, pCons, pTab->zName - ); - - /* Finally, reload the database schema. */ - renameReloadSchema(pParse, iDb, INITFLAG_AlterDropCons); -} - /* ** Register built-in functions used to help implement ALTER TABLE */ @@ -3046,10 +2323,6 @@ void sqlite3AlterFunctions(void){ INTERNAL_FUNCTION(sqlite_rename_test, 7, renameTableTest), INTERNAL_FUNCTION(sqlite_drop_column, 3, dropColumnFunc), INTERNAL_FUNCTION(sqlite_rename_quotefix,2, renameQuotefixFunc), - INTERNAL_FUNCTION(sqlite_drop_constraint,2, dropConstraintFunc), - INTERNAL_FUNCTION(sqlite_fail, 2, failConstraintFunc), - INTERNAL_FUNCTION(sqlite_add_constraint, 3, addConstraintFunc), - INTERNAL_FUNCTION(sqlite_find_constraint,2, findConstraintFunc), }; sqlite3InsertBuiltinFuncs(aAlterTableFuncs, ArraySize(aAlterTableFuncs)); } diff --git a/src/attach.c b/src/attach.c index f27c1e6be..085e1b0ec 100644 --- a/src/attach.c +++ b/src/attach.c @@ -596,7 +596,7 @@ int sqlite3FixTriggerStep( if( sqlite3WalkSelect(&pFix->w, pStep->pSelect) || sqlite3WalkExpr(&pFix->w, pStep->pWhere) || sqlite3WalkExprList(&pFix->w, pStep->pExprList) - || sqlite3FixSrcList(pFix, pStep->pSrc) + || sqlite3FixSrcList(pFix, pStep->pFrom) ){ return 1; } diff --git a/src/auth.c b/src/auth.c index 1088f844a..9ec2e7d04 100644 --- a/src/auth.c +++ b/src/auth.c @@ -78,7 +78,7 @@ int sqlite3_set_authorizer( sqlite3_mutex_enter(db->mutex); db->xAuth = (sqlite3_xauth)xAuth; db->pAuthArg = pArg; - sqlite3ExpirePreparedStatements(db, 1); + if( db->xAuth ) sqlite3ExpirePreparedStatements(db, 1); sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } diff --git a/src/btree.c b/src/btree.c index 7e73c7fd7..8d0f92222 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3673,30 +3673,6 @@ static SQLITE_NOINLINE int btreeBeginTrans( } #endif -#ifdef SQLITE_EXPERIMENTAL_PRAGMA_20251114 - /* If both a read and write transaction will be opened by this call, - ** then issue a file-control as if the following pragma command had - ** been evaluated: - ** - ** PRAGMA experimental_pragma_20251114 = 1|2 - ** - ** where the RHS is "1" if wrflag is 1 (RESERVED lock), or "2" if wrflag - ** is 2 (EXCLUSIVE lock). Ignore any result or error returned by the VFS. - ** - ** WARNING: This code will likely remain part of SQLite only temporarily - - ** it exists to allow users to experiment with certain types of blocking - ** locks in custom VFS implementations. It MAY BE REMOVED AT ANY TIME. */ - if( pBt->pPage1==0 && wrflag ){ - sqlite3_file *fd = sqlite3PagerFile(pPager); - char *aFcntl[3] = {0,0,0}; - aFcntl[1] = "experimental_pragma_20251114"; - assert( wrflag==1 || wrflag==2 ); - aFcntl[2] = (wrflag==1 ? "1" : "2"); - sqlite3OsFileControlHint(fd, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); - sqlite3_free(aFcntl[0]); - } -#endif - /* Call lockBtree() until either pBt->pPage1 is populated or ** lockBtree() returns something other than SQLITE_OK. lockBtree() ** may return SQLITE_OK but leave pBt->pPage1 set to 0 if after @@ -5701,7 +5677,7 @@ int sqlite3BtreeIsEmpty(BtCursor *pCur, int *pRes){ assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - if( NEVER(pCur->eState==CURSOR_VALID) ){ + if( pCur->eState==CURSOR_VALID ){ *pRes = 0; return SQLITE_OK; } @@ -9782,7 +9758,7 @@ int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ }while( rc==SQLITE_OK && nOut>0 ); if( rc==SQLITE_OK && nRem>0 && ALWAYS(pPgnoOut) ){ - Pgno pgnoNew = 0; /* Prevent harmless static-analyzer warning */ + Pgno pgnoNew; MemPage *pNew = 0; rc = allocateBtreePage(pBt, &pNew, &pgnoNew, 0, 0); put4byte(pPgnoOut, pgnoNew); diff --git a/src/build.c b/src/build.c index c45194145..de890c2e9 100644 --- a/src/build.c +++ b/src/build.c @@ -486,7 +486,6 @@ Table *sqlite3LocateTableItem( const char *zDb; if( p->fg.fixedSchema ){ int iDb = sqlite3SchemaToIndex(pParse->db, p->u4.pSchema); - assert( iDb>=0 && iDb<pParse->db->nDb ); zDb = pParse->db->aDb[iDb].zDbSName; }else{ assert( !p->fg.isSubquery ); @@ -2060,8 +2059,8 @@ void sqlite3ChangeCookie(Parse *pParse, int iDb){ ** The estimate is conservative. It might be larger that what is ** really needed. */ -static i64 identLength(const char *z){ - i64 n; +static int identLength(const char *z){ + int n; for(n=0; *z; n++, z++){ if( *z=='"' ){ n++; } } @@ -2571,14 +2570,13 @@ void sqlite3MarkAllShadowTablesOf(sqlite3 *db, Table *pTab){ ** restored to its original value prior to this routine returning. */ int sqlite3ShadowTableName(sqlite3 *db, const char *zName){ - const char *zTail; /* Pointer to the last "_" in zName */ + char *zTail; /* Pointer to the last "_" in zName */ Table *pTab; /* Table that zName is a shadow of */ - char *zCopy; zTail = strrchr(zName, '_'); if( zTail==0 ) return 0; - zCopy = sqlite3DbStrNDup(db, zName, (int)(zTail-zName)); - pTab = zCopy ? sqlite3FindTable(db, zCopy, 0) : 0; - sqlite3DbFree(db, zCopy); + *zTail = 0; + pTab = sqlite3FindTable(db, zName, 0); + *zTail = '_'; if( pTab==0 ) return 0; if( !IsVirtual(pTab) ) return 0; return sqlite3IsShadowTableOf(db, pTab, zName); @@ -2731,7 +2729,6 @@ void sqlite3EndTable( convertToWithoutRowidTable(pParse, p); } iDb = sqlite3SchemaToIndex(db, p->pSchema); - assert( iDb>=0 && iDb<=db->nDb ); #ifndef SQLITE_OMIT_CHECK /* Resolve names in all CHECK constraint expressions. @@ -3027,7 +3024,6 @@ void sqlite3CreateView( sqlite3TwoPartName(pParse, pName1, pName2, &pName); iDb = sqlite3SchemaToIndex(db, p->pSchema); - assert( iDb>=0 && iDb<db->nDb ); sqlite3FixInit(&sFix, pParse, iDb, "view", pName); if( sqlite3FixSelect(&sFix, pSelect) ) goto create_view_fail; @@ -4624,7 +4620,6 @@ void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists){ goto exit_drop_index; } iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); - assert( iDb>=0 && iDb<db->nDb ); #ifndef SQLITE_OMIT_AUTHORIZATION { int code = SQLITE_DROP_INDEX; diff --git a/src/carray.c b/src/carray.c index ff0691a85..154d107dd 100644 --- a/src/carray.c +++ b/src/carray.c @@ -79,7 +79,6 @@ struct carray_bind { int nData; /* Number of elements */ int mFlags; /* Control flags */ void (*xDel)(void*); /* Destructor for aData */ - void *pDel; /* Alternative argument to xDel() */ }; @@ -412,7 +411,7 @@ static sqlite3_module carrayModule = { static void carrayBindDel(void *pPtr){ carray_bind *p = (carray_bind*)pPtr; if( p->xDel!=SQLITE_STATIC ){ - p->xDel(p->pDel); + p->xDel(p->aData); } sqlite3_free(p); } @@ -420,26 +419,14 @@ static void carrayBindDel(void *pPtr){ /* ** Invoke this interface in order to bind to the single-argument ** version of CARRAY(). -** -** pStmt The prepared statement to which to bind -** idx The index of the parameter of pStmt to which to bind -** aData The data to be bound -** nData The number of elements in aData -** mFlags One of SQLITE_CARRAY_xxxx indicating datatype of aData -** xDestroy Destructor for pDestroy or aData if pDestroy==NULL. -** pDestroy Invoke xDestroy on this pointer if not NULL -** -** The destructor is called pDestroy if pDestroy!=NULL, or against -** aData if pDestroy==NULL. */ -SQLITE_API int sqlite3_carray_bind_v2( +SQLITE_API int sqlite3_carray_bind( sqlite3_stmt *pStmt, int idx, void *aData, int nData, int mFlags, - void (*xDestroy)(void*), - void *pDestroy + void (*xDestroy)(void*) ){ carray_bind *pNew = 0; int i; @@ -516,38 +503,20 @@ SQLITE_API int sqlite3_carray_bind_v2( memcpy(pNew->aData, aData, sz); } pNew->xDel = sqlite3_free; - pNew->pDel = pNew->aData; }else{ pNew->aData = aData; pNew->xDel = xDestroy; - pNew->pDel = pDestroy; } return sqlite3_bind_pointer(pStmt, idx, pNew, "carray-bind", carrayBindDel); carray_bind_error: if( xDestroy!=SQLITE_STATIC && xDestroy!=SQLITE_TRANSIENT ){ - xDestroy(pDestroy); + xDestroy(aData); } sqlite3_free(pNew); return rc; } -/* -** Invoke this interface in order to bind to the single-argument -** version of CARRAY(). Same as sqlite3_carray_bind_v2() with the -** pDestroy parameter set to NULL. -*/ -SQLITE_API int sqlite3_carray_bind( - sqlite3_stmt *pStmt, - int idx, - void *aData, - int nData, - int mFlags, - void (*xDestroy)(void*) -){ - return sqlite3_carray_bind_v2(pStmt,idx,aData,nData,mFlags,xDestroy,aData); -} - /* ** Invoke this routine to register the carray() function. */ diff --git a/src/date.c b/src/date.c index 58a7ce544..5e7ae6f1f 100644 --- a/src/date.c +++ b/src/date.c @@ -429,7 +429,7 @@ static int parseDateOrTime( return 0; }else if( sqlite3StrICmp(zDate,"now")==0 && sqlite3NotPureFunc(context) ){ return setDateTimeToCurrent(context, p); - }else if( sqlite3AtoF(zDate, &r)>0 ){ + }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8)>0 ){ setRawDateNumber(p, r); return 0; }else if( (sqlite3StrICmp(zDate,"subsec")==0 @@ -875,7 +875,7 @@ static int parseModifier( ** date is already on the appropriate weekday, this is a no-op. */ if( sqlite3_strnicmp(z, "weekday ", 8)==0 - && sqlite3AtoF(&z[8], &r)>0 + && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8)>0 && r>=0.0 && r<7.0 && (n=(int)r)==r ){ sqlite3_int64 Z; computeYMD_HMS(p); @@ -946,11 +946,9 @@ static int parseModifier( case '8': case '9': { double rRounder; - int i, rx; + int i; int Y,M,D,h,m,x; const char *z2 = z; - char *zCopy; - sqlite3 *db = sqlite3_context_db_handle(pCtx); char z0 = z[0]; for(n=1; z[n]; n++){ if( z[n]==':' ) break; @@ -960,11 +958,7 @@ static int parseModifier( if( n==6 && getDigits(&z[1], "50f", &Y)==1 ) break; } } - zCopy = sqlite3DbStrNDup(db, z, n); - if( zCopy==0 ) break; - rx = sqlite3AtoF(zCopy, &r)<=0; - sqlite3DbFree(db, zCopy); - if( rx ){ + if( sqlite3AtoF(z, &r, n, SQLITE_UTF8)<=0 ){ assert( rc==1 ); break; } @@ -1784,7 +1778,7 @@ static void datedebugFunc( char *zJson; zJson = sqlite3_mprintf( "{iJD:%lld,Y:%d,M:%d,D:%d,h:%d,m:%d,tz:%d," - "s:%.3f,validJD:%d,validYMD:%d,validHMS:%d," + "s:%.3f,validJD:%d,validYMS:%d,validHMS:%d," "nFloor:%d,rawS:%d,isError:%d,useSubsec:%d," "isUtc:%d,isLocal:%d}", x.iJD, x.Y, x.M, x.D, x.h, x.m, x.tz, diff --git a/src/delete.c b/src/delete.c index d4d2337c7..8fac7c2f3 100644 --- a/src/delete.c +++ b/src/delete.c @@ -86,7 +86,7 @@ static int vtabIsReadOnly(Parse *pParse, Table *pTab){ ** * Only allow DELETE, INSERT, or UPDATE of non-SQLITE_VTAB_INNOCUOUS ** virtual tables if PRAGMA trusted_schema=ON. */ - if( (pParse->pToplevel!=0 || (pParse->prepFlags & SQLITE_PREPARE_FROM_DDL)) + if( pParse->pToplevel!=0 && pTab->u.vtab.p->eVtabRisk > ((pParse->db->flags & SQLITE_TrustedSchema)!=0) ){ @@ -924,6 +924,7 @@ void sqlite3GenerateRowIndexDelete( &iPartIdxLabel, pPrior, r1); sqlite3VdbeAddOp3(v, OP_IdxDelete, iIdxCur+i, r1, pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); + sqlite3VdbeChangeP5(v, 1); /* Cause IdxDelete to error if no entry found */ sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); pPrior = pIdx; } diff --git a/src/expr.c b/src/expr.c index d486e48e3..fdc05366c 100644 --- a/src/expr.c +++ b/src/expr.c @@ -935,22 +935,34 @@ Expr *sqlite3ExprAlloc( int dequote /* True to dequote */ ){ Expr *pNew; - int nExtra = pToken ? pToken->n+1 : 0; + int nExtra = 0; + int iValue = 0; assert( db!=0 ); + if( pToken ){ + if( op!=TK_INTEGER || pToken->z==0 + || sqlite3GetInt32(pToken->z, &iValue)==0 ){ + nExtra = pToken->n+1; /* tag-20240227-a */ + assert( iValue>=0 ); + } + } pNew = sqlite3DbMallocRawNN(db, sizeof(Expr)+nExtra); if( pNew ){ memset(pNew, 0, sizeof(Expr)); pNew->op = (u8)op; pNew->iAgg = -1; - if( nExtra ){ - assert( pToken!=0 ); - pNew->u.zToken = (char*)&pNew[1]; - assert( pToken->z!=0 || pToken->n==0 ); - if( pToken->n ) memcpy(pNew->u.zToken, pToken->z, pToken->n); - pNew->u.zToken[pToken->n] = 0; - if( dequote && sqlite3Isquote(pNew->u.zToken[0]) ){ - sqlite3DequoteExpr(pNew); + if( pToken ){ + if( nExtra==0 ){ + pNew->flags |= EP_IntValue|EP_Leaf|(iValue?EP_IsTrue:EP_IsFalse); + pNew->u.iValue = iValue; + }else{ + pNew->u.zToken = (char*)&pNew[1]; + assert( pToken->z!=0 || pToken->n==0 ); + if( pToken->n ) memcpy(pNew->u.zToken, pToken->z, pToken->n); + pNew->u.zToken[pToken->n] = 0; + if( dequote && sqlite3Isquote(pNew->u.zToken[0]) ){ + sqlite3DequoteExpr(pNew); + } } } #if SQLITE_MAX_EXPR_DEPTH>0 @@ -975,24 +987,6 @@ Expr *sqlite3Expr( return sqlite3ExprAlloc(db, op, &x, 0); } -/* -** Allocate an expression for a 32-bit signed integer literal. -*/ -Expr *sqlite3ExprInt32(sqlite3 *db, int iVal){ - Expr *pNew = sqlite3DbMallocRawNN(db, sizeof(Expr)); - if( pNew ){ - memset(pNew, 0, sizeof(Expr)); - pNew->op = TK_INTEGER; - pNew->iAgg = -1; - pNew->flags = EP_IntValue|EP_Leaf|(iVal?EP_IsTrue:EP_IsFalse); - pNew->u.iValue = iVal; -#if SQLITE_MAX_EXPR_DEPTH>0 - pNew->nHeight = 1; -#endif - } - return pNew; -} - /* ** Attach subtrees pLeft and pRight to the Expr node pRoot. ** @@ -1155,7 +1149,7 @@ Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ ){ sqlite3ExprDeferredDelete(pParse, pLeft); sqlite3ExprDeferredDelete(pParse, pRight); - return sqlite3ExprInt32(db, 0); + return sqlite3Expr(db, TK_INTEGER, "0"); }else{ return sqlite3PExpr(pParse, TK_AND, pLeft, pRight); } @@ -1280,9 +1274,7 @@ void sqlite3ExprFunctionUsable( ){ assert( !IN_RENAME_OBJECT ); assert( (pDef->funcFlags & (SQLITE_FUNC_DIRECT|SQLITE_FUNC_UNSAFE))!=0 ); - if( ExprHasProperty(pExpr, EP_FromDDL) - || pParse->prepFlags & SQLITE_PREPARE_FROM_DDL - ){ + if( ExprHasProperty(pExpr, EP_FromDDL) ){ if( (pDef->funcFlags & SQLITE_FUNC_DIRECT)!=0 || (pParse->db->flags & SQLITE_TrustedSchema)==0 ){ @@ -1978,7 +1970,9 @@ Select *sqlite3SelectDup(sqlite3 *db, const Select *pDup, int flags){ pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); pNew->iLimit = 0; pNew->iOffset = 0; - pNew->selFlags = p->selFlags; + pNew->selFlags = p->selFlags & ~(u32)SF_UsesEphemeral; + pNew->addrOpenEphm[0] = -1; + pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = p->nSelectRow; pNew->pWith = sqlite3WithDup(db, p->pWith); #ifndef SQLITE_OMIT_WINDOWFUNC @@ -2630,7 +2624,7 @@ static int exprIsConst(Parse *pParse, Expr *p, int initFlag){ /* ** Walk an expression tree. Return non-zero if the expression is constant -** or return zero if the expression involves variables or function calls. +** and 0 if it involves variables or function calls. ** ** For the purposes of this function, a double-quoted string (ex: "abc") ** is considered a variable but a single-quoted string (ex: 'abc') is @@ -3420,7 +3414,6 @@ int sqlite3FindInIndex( */ u32 savedNQueryLoop = pParse->nQueryLoop; int rMayHaveNull = 0; - int bloomOk = (inFlags & IN_INDEX_MEMBERSHIP)!=0; eType = IN_INDEX_EPH; if( inFlags & IN_INDEX_LOOP ){ pParse->nQueryLoop = 0; @@ -3428,13 +3421,7 @@ int sqlite3FindInIndex( *prRhsHasNull = rMayHaveNull = ++pParse->nMem; } assert( pX->op==TK_IN ); - if( !bloomOk - && ExprUseXSelect(pX) - && (pX->x.pSelect->selFlags & SF_ClonedRhsIn)!=0 - ){ - bloomOk = 1; - } - sqlite3CodeRhsOfIN(pParse, pX, iTab, bloomOk); + sqlite3CodeRhsOfIN(pParse, pX, iTab); if( rMayHaveNull ){ sqlite3SetHasNullFlag(v, iTab, rMayHaveNull); } @@ -3592,8 +3579,7 @@ static int findCompatibleInRhsSubrtn( void sqlite3CodeRhsOfIN( Parse *pParse, /* Parsing context */ Expr *pExpr, /* The IN operator */ - int iTab, /* Use this cursor number */ - int allowBloom /* True to allow the use of a Bloom filter */ + int iTab /* Use this cursor number */ ){ int addrOnce = 0; /* Address of the OP_Once instruction at top */ int addr; /* Address of OP_OpenEphemeral instruction */ @@ -3715,10 +3701,7 @@ void sqlite3CodeRhsOfIN( sqlite3SelectDestInit(&dest, SRT_Set, iTab); dest.zAffSdst = exprINAffinity(pParse, pExpr); pSelect->iLimit = 0; - if( addrOnce - && allowBloom - && OptimizationEnabled(pParse->db, SQLITE_BloomFilter) - ){ + if( addrOnce && OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){ int regBloom = ++pParse->nMem; addrBloom = sqlite3VdbeAddOp2(v, OP_Blob, 10000, regBloom); VdbeComment((v, "Bloom filter")); @@ -3939,7 +3922,7 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ || (pLeft->u.iValue!=1 && pLeft->u.iValue!=0) ){ sqlite3 *db = pParse->db; - pLimit = sqlite3ExprInt32(db, 0); + pLimit = sqlite3Expr(db, TK_INTEGER, "0"); if( pLimit ){ pLimit->affExpr = SQLITE_AFF_NUMERIC; pLimit = sqlite3PExpr(pParse, TK_NE, @@ -3950,7 +3933,7 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ } }else{ /* If there is no pre-existing limit add a limit of 1 */ - pLimit = sqlite3ExprInt32(pParse->db, 1); + pLimit = sqlite3Expr(pParse->db, TK_INTEGER, "1"); pSel->pLimit = sqlite3PExpr(pParse, TK_LIMIT, pLimit, 0); } pSel->iLimit = 0; @@ -4211,9 +4194,8 @@ static void sqlite3ExprCodeIN( if( ExprHasProperty(pExpr, EP_Subrtn) ){ const VdbeOp *pOp = sqlite3VdbeGetOp(v, pExpr->y.sub.iAddr); assert( pOp->opcode==OP_Once || pParse->nErr ); - if( pOp->p3>0 ){ /* tag-202407032019 */ - assert( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) - || pParse->nErr ); + if( pOp->opcode==OP_Once && pOp->p3>0 ){ /* tag-202407032019 */ + assert( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ); sqlite3VdbeAddOp4Int(v, OP_Filter, pOp->p3, destIfFalse, rLhs, nVector); VdbeCoverage(v); } @@ -4303,7 +4285,7 @@ static void sqlite3ExprCodeIN( static void codeReal(Vdbe *v, const char *z, int negateFlag, int iMem){ if( ALWAYS(z!=0) ){ double value; - sqlite3AtoF(z, &value); + sqlite3AtoF(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); assert( !sqlite3IsNaN(value) ); /* The new AtoF never returns NaN */ if( negateFlag ) value = -value; sqlite3VdbeAddOp4Dup8(v, OP_Real, 0, iMem, 0, (u8*)&value, P4_REAL); @@ -7404,10 +7386,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ if( pIEpr==0 ) break; if( NEVER(!ExprUseYTab(pExpr)) ) break; for(i=0; i<pSrcList->nSrc; i++){ - if( pSrcList->a[i].iCursor==pIEpr->iDataCur ){ - testcase( i>0 ); - break; - } + if( pSrcList->a[0].iCursor==pIEpr->iDataCur ) break; } if( i>=pSrcList->nSrc ) break; if( NEVER(pExpr->pAggInfo!=0) ) break; /* Resolved by outer context */ diff --git a/src/fkey.c b/src/fkey.c index 59edd8810..f1117a884 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -688,7 +688,6 @@ FKey *sqlite3FkReferences(Table *pTab){ static void fkTriggerDelete(sqlite3 *dbMem, Trigger *p){ if( p ){ TriggerStep *pStep = p->step_list; - sqlite3SrcListDelete(dbMem, pStep->pSrc); sqlite3ExprDelete(dbMem, pStep->pWhere); sqlite3ExprListDelete(dbMem, pStep->pExprList); sqlite3SelectDelete(dbMem, pStep->pSelect); @@ -908,7 +907,6 @@ void sqlite3FkCheck( if( !IsOrdinaryTable(pTab) ) return; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - assert( iDb>=00 && iDb<db->nDb ); zDb = db->aDb[iDb].zDbSName; /* Loop through all the foreign key constraints for which pTab is the @@ -1356,14 +1354,14 @@ static Trigger *fkActionTrigger( pTrigger = (Trigger *)sqlite3DbMallocZero(db, sizeof(Trigger) + /* struct Trigger */ - sizeof(TriggerStep) /* Single step in trigger program */ + sizeof(TriggerStep) + /* Single step in trigger program */ + nFrom + 1 /* Space for pStep->zTarget */ ); if( pTrigger ){ pStep = pTrigger->step_list = (TriggerStep *)&pTrigger[1]; - pStep->pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); - if( pStep->pSrc ){ - pStep->pSrc->a[0].zName = sqlite3DbStrNDup(db, zFrom, nFrom); - } + pStep->zTarget = (char *)&pStep[1]; + memcpy((char *)pStep->zTarget, zFrom, nFrom); + pStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); pStep->pExprList = sqlite3ExprListDup(db, pList, EXPRDUP_REDUCE); pStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); diff --git a/src/func.c b/src/func.c index d9d8f59ad..6dac7195a 100644 --- a/src/func.c +++ b/src/func.c @@ -466,7 +466,7 @@ static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ sqlite3_result_error_nomem(context); return; } - sqlite3AtoF(zBuf, &r); + sqlite3AtoF(zBuf, &r, sqlite3Strlen30(zBuf), SQLITE_UTF8); sqlite3_free(zBuf); } sqlite3_result_double(context, r); @@ -1104,7 +1104,7 @@ void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue, int bEscape){ sqlite3_str_appendf(pStr, "%!0.15g", r1); zVal = sqlite3_str_value(pStr); if( zVal ){ - sqlite3AtoF(zVal, &r2); + sqlite3AtoF(zVal, &r2, pStr->nChar, SQLITE_UTF8); if( r1!=r2 ){ sqlite3_str_reset(pStr); sqlite3_str_appendf(pStr, "%!0.20e", r1); @@ -1201,7 +1201,7 @@ static void unistrFunc( } i = j = 0; while( i<nIn ){ - const char *z = strchr(&zIn[i],'\\'); + char *z = strchr(&zIn[i],'\\'); if( z==0 ){ n = nIn - i; memmove(&zOut[j], &zIn[i], n); @@ -1238,7 +1238,7 @@ static void unistrFunc( } } zOut[j] = 0; - sqlite3_result_text64(context, zOut, j, sqlite3_free, SQLITE_UTF8_ZT); + sqlite3_result_text64(context, zOut, j, sqlite3_free, SQLITE_UTF8); return; unistr_error: @@ -1331,7 +1331,7 @@ static void charFunc( } \ } *zOut = 0; - sqlite3_result_text64(context, (char*)z, zOut-z,sqlite3_free,SQLITE_UTF8_ZT); + sqlite3_result_text64(context, (char*)z, zOut-z, sqlite3_free, SQLITE_UTF8); } /* @@ -1360,7 +1360,7 @@ static void hexFunc( } *z = 0; sqlite3_result_text64(context, zHex, (u64)(z-zHex), - sqlite3_free, SQLITE_UTF8_ZT); + sqlite3_free, SQLITE_UTF8); } } @@ -1698,7 +1698,7 @@ static void concatFuncCore( } z[j] = 0; assert( j<=n ); - sqlite3_result_text64(context, z, j, sqlite3_free, SQLITE_UTF8_ZT); + sqlite3_result_text64(context, z, j, sqlite3_free, SQLITE_UTF8); } /* @@ -2364,8 +2364,6 @@ void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive){ sqlite3CreateFunc(db, "like", nArg, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0, 0, 0); pDef = sqlite3FindFunction(db, "like", nArg, SQLITE_UTF8, 0); - assert( pDef!=0 ); /* The sqlite3CreateFunc() call above cannot fail - ** because the "like" SQL-function already exists */ pDef->funcFlags |= flags; pDef->funcFlags &= ~SQLITE_FUNC_UNSAFE; } diff --git a/src/hwtime.h b/src/hwtime.h index cf637edaf..f808fa40e 100644 --- a/src/hwtime.h +++ b/src/hwtime.h @@ -16,26 +16,35 @@ #ifndef SQLITE_HWTIME_H #define SQLITE_HWTIME_H -#if defined(_MSC_VER) && defined(_WIN32) - - #include "windows.h" - #include <profileapi.h> - - __inline sqlite3_uint64 sqlite3Hwtime(void){ - LARGE_INTEGER tm; - QueryPerformanceCounter(&tm); - return (sqlite3_uint64)tm.QuadPart; - } - -#elif !defined(__STRICT_ANSI__) && defined(__GNUC__) && \ +/* +** The following routine only works on Pentium-class (or newer) processors. +** It uses the RDTSC opcode to read the cycle count value out of the +** processor and returns that value. This can be used for high-res +** profiling. +*/ +#if !defined(__STRICT_ANSI__) && \ + (defined(__GNUC__) || defined(_MSC_VER)) && \ (defined(i386) || defined(__i386__) || defined(_M_IX86)) + #if defined(__GNUC__) + __inline__ sqlite_uint64 sqlite3Hwtime(void){ unsigned int lo, hi; __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); return (sqlite_uint64)hi << 32 | lo; } + #elif defined(_MSC_VER) + + __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ + __asm { + rdtsc + ret ; return value at EDX:EAX + } + } + + #endif + #elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) __inline__ sqlite_uint64 sqlite3Hwtime(void){ @@ -43,14 +52,6 @@ __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); return (sqlite_uint64)hi << 32 | lo; } - -#elif !defined(__STRICT_ANSI__) && defined(__GNUC__) && defined(__aarch64__) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - sqlite3_uint64 cnt; - __asm__ __volatile__ ("mrs %0, cntvct_el0" : "=r" (cnt)); - return cnt; - } #elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) diff --git a/src/json.c b/src/json.c index 795d3ed73..d0d3c53a2 100644 --- a/src/json.c +++ b/src/json.c @@ -328,10 +328,7 @@ struct JsonString { #define JSON_SQL 0x02 /* Result is always SQL */ #define JSON_ABPATH 0x03 /* Allow abbreviated JSON path specs */ #define JSON_ISSET 0x04 /* json_set(), not json_insert() */ -#define JSON_AINS 0x08 /* json_array_insert(), not json_insert() */ -#define JSON_BLOB 0x10 /* Use the BLOB output format */ - -#define JSON_INSERT_TYPE(X) (((X)&0xC)>>2) +#define JSON_BLOB 0x08 /* Use the BLOB output format */ /* A parsed JSON value. Lifecycle: @@ -377,7 +374,6 @@ struct JsonParse { #define JEDIT_REPL 2 /* Overwrite if exists */ #define JEDIT_INS 3 /* Insert if not exists */ #define JEDIT_SET 4 /* Insert or overwrite */ -#define JEDIT_AINS 5 /* array_insert() */ /* ** Maximum nesting depth of JSON for this implementation. @@ -2874,8 +2870,7 @@ static int jsonLabelCompare( */ #define JSON_LOOKUP_ERROR 0xffffffff #define JSON_LOOKUP_NOTFOUND 0xfffffffe -#define JSON_LOOKUP_NOTARRAY 0xfffffffd -#define JSON_LOOKUP_PATHERROR 0xfffffffc +#define JSON_LOOKUP_PATHERROR 0xfffffffd #define JSON_LOOKUP_ISERROR(x) ((x)>=JSON_LOOKUP_PATHERROR) /* Forward declaration */ @@ -2904,7 +2899,7 @@ static u32 jsonLookupStep(JsonParse*,u32,const char*,u32); static u32 jsonCreateEditSubstructure( JsonParse *pParse, /* The original JSONB that is being edited */ JsonParse *pIns, /* Populate this with the blob data to insert */ - const char *zTail /* Tail of the path that determines substructure */ + const char *zTail /* Tail of the path that determins substructure */ ){ static const u8 emptyObject[] = { JSONB_ARRAY, JSONB_OBJECT }; int rc; @@ -2939,9 +2934,9 @@ static u32 jsonCreateEditSubstructure( ** Return one of the JSON_LOOKUP error codes if problems are seen. ** ** This routine will also modify the blob. If pParse->eEdit is one of -** JEDIT_DEL, JEDIT_REPL, JEDIT_INS, JEDIT_SET, or JEDIT_AINS, then changes -** might be made to the selected value. If an edit is performed, then the -** return value does not necessarily point to the select element. If an edit +** JEDIT_DEL, JEDIT_REPL, JEDIT_INS, or JEDIT_SET, then changes might be +** made to the selected value. If an edit is performed, then the return +** value does not necessarily point to the select element. If an edit ** is performed, the return value is only useful for detecting error ** conditions. */ @@ -2967,13 +2962,6 @@ static u32 jsonLookupStep( jsonBlobEdit(pParse, iRoot, sz, 0, 0); }else if( pParse->eEdit==JEDIT_INS ){ /* Already exists, so json_insert() is a no-op */ - }else if( pParse->eEdit==JEDIT_AINS ){ - /* json_array_insert() */ - if( zPath[-1]!=']' ){ - return JSON_LOOKUP_NOTARRAY; - }else{ - jsonBlobEdit(pParse, iRoot, 0, pParse->aIns, pParse->nIns); - } }else{ /* json_set() or json_replace() */ jsonBlobEdit(pParse, iRoot, sz, pParse->aIns, pParse->nIns); @@ -3045,10 +3033,6 @@ static u32 jsonLookupStep( JsonParse ix; /* Header of the label to be inserted */ testcase( pParse->eEdit==JEDIT_INS ); testcase( pParse->eEdit==JEDIT_SET ); - testcase( pParse->eEdit==JEDIT_AINS ); - if( pParse->eEdit==JEDIT_AINS && sqlite3_strglob("*]",&zPath[i])!=0 ){ - return JSON_LOOKUP_NOTARRAY; - } memset(&ix, 0, sizeof(ix)); ix.db = pParse->db; jsonBlobAppendNode(&ix, rawKey?JSONB_TEXTRAW:JSONB_TEXT5, nKey, 0); @@ -3076,32 +3060,28 @@ static u32 jsonLookupStep( return rc; } }else if( zPath[0]=='[' ){ - u64 kk = 0; x = pParse->aBlob[iRoot] & 0x0f; if( x!=JSONB_ARRAY ) return JSON_LOOKUP_NOTFOUND; n = jsonbPayloadSize(pParse, iRoot, &sz); + k = 0; i = 1; while( sqlite3Isdigit(zPath[i]) ){ - if( kk<0xffffffff ) kk = kk*10 + zPath[i] - '0'; - /* ^^^^^^^^^^--- Allow kk to be bigger than any JSON array so that - ** we get NOTFOUND instead of PATHERROR, without overflowing kk. */ + k = k*10 + zPath[i] - '0'; i++; } if( i<2 || zPath[i]!=']' ){ if( zPath[1]=='#' ){ - kk = jsonbArrayCount(pParse, iRoot); + k = jsonbArrayCount(pParse, iRoot); i = 2; if( zPath[2]=='-' && sqlite3Isdigit(zPath[3]) ){ - u64 nn = 0; + unsigned int nn = 0; i = 3; do{ - if( nn<0xffffffff ) nn = nn*10 + zPath[i] - '0'; - /* ^^^^^^^^^^--- Allow nn to be bigger than any JSON array to - ** get NOTFOUND instead of PATHERROR, without overflowing nn. */ + nn = nn*10 + zPath[i] - '0'; i++; }while( sqlite3Isdigit(zPath[i]) ); - if( nn>kk ) return JSON_LOOKUP_NOTFOUND; - kk -= nn; + if( nn>k ) return JSON_LOOKUP_NOTFOUND; + k -= nn; } if( zPath[i]!=']' ){ return JSON_LOOKUP_PATHERROR; @@ -3113,22 +3093,21 @@ static u32 jsonLookupStep( j = iRoot+n; iEnd = j+sz; while( j<iEnd ){ - if( kk==0 ){ + if( k==0 ){ rc = jsonLookupStep(pParse, j, &zPath[i+1], 0); if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); return rc; } - kk--; + k--; n = jsonbPayloadSize(pParse, j, &sz); if( n==0 ) return JSON_LOOKUP_ERROR; j += n+sz; } if( j>iEnd ) return JSON_LOOKUP_ERROR; - if( kk>0 ) return JSON_LOOKUP_NOTFOUND; + if( k>0 ) return JSON_LOOKUP_NOTFOUND; if( pParse->eEdit>=JEDIT_INS ){ JsonParse v; testcase( pParse->eEdit==JEDIT_INS ); - testcase( pParse->eEdit==JEDIT_AINS ); testcase( pParse->eEdit==JEDIT_SET ); rc = jsonCreateEditSubstructure(pParse, &v, &zPath[i+1]); if( !JSON_LOOKUP_ISERROR(rc) @@ -3266,7 +3245,7 @@ static void jsonReturnFromBlob( to_double: z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); if( z==0 ) goto returnfromblob_oom; - rc = sqlite3AtoF(z, &r); + rc = sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); sqlite3DbFree(db, z); if( rc<=0 ) goto returnfromblob_malformed; sqlite3_result_double(pCtx, r); @@ -3453,15 +3432,9 @@ static int jsonFunctionArgToBlob( */ static char *jsonBadPathError( sqlite3_context *ctx, /* The function call containing the error */ - const char *zPath, /* The path with the problem */ - int rc /* Maybe JSON_LOOKUP_NOTARRAY */ + const char *zPath /* The path with the problem */ ){ - char *zMsg; - if( rc==(int)JSON_LOOKUP_NOTARRAY ){ - zMsg = sqlite3_mprintf("not an array element: %Q", zPath); - }else{ - zMsg = sqlite3_mprintf("bad JSON path: %Q", zPath); - } + char *zMsg = sqlite3_mprintf("bad JSON path: %Q", zPath); if( ctx==0 ) return zMsg; if( zMsg ){ sqlite3_result_error(ctx, zMsg, -1); @@ -3478,13 +3451,13 @@ static char *jsonBadPathError( ** and return the result. ** ** The specific operation is determined by eEdit, which can be one -** of JEDIT_INS, JEDIT_REPL, JEDIT_SET, or JEDIT_AINS. +** of JEDIT_INS, JEDIT_REPL, or JEDIT_SET. */ static void jsonInsertIntoBlob( sqlite3_context *ctx, int argc, sqlite3_value **argv, - int eEdit /* JEDIT_INS, JEDIT_REPL, JEDIT_SET, JEDIT_AINS */ + int eEdit /* JEDIT_INS, JEDIT_REPL, or JEDIT_SET */ ){ int i; u32 rc = 0; @@ -3536,7 +3509,7 @@ static void jsonInsertIntoBlob( if( rc==JSON_LOOKUP_ERROR ){ sqlite3_result_error(ctx, "malformed JSON", -1); }else{ - jsonBadPathError(ctx, zPath, rc); + jsonBadPathError(ctx, zPath); } return; } @@ -3978,7 +3951,7 @@ static void jsonArrayLengthFunc( if( i==JSON_LOOKUP_NOTFOUND ){ /* no-op */ }else if( i==JSON_LOOKUP_PATHERROR ){ - jsonBadPathError(ctx, zPath, 0); + jsonBadPathError(ctx, zPath); }else{ sqlite3_result_error(ctx, "malformed JSON", -1); } @@ -4083,7 +4056,7 @@ static void jsonExtractFunc( j = jsonLookupStep(p, 0, jx.zBuf, 0); jsonStringReset(&jx); }else{ - jsonBadPathError(ctx, zPath, 0); + jsonBadPathError(ctx, zPath); goto json_extract_error; } if( j<p->nBlob ){ @@ -4118,7 +4091,7 @@ static void jsonExtractFunc( sqlite3_result_error(ctx, "malformed JSON", -1); goto json_extract_error; }else{ - jsonBadPathError(ctx, zPath, 0); + jsonBadPathError(ctx, zPath); goto json_extract_error; } } @@ -4447,7 +4420,7 @@ static void jsonRemoveFunc( if( rc==JSON_LOOKUP_NOTFOUND ){ continue; /* No-op */ }else if( rc==JSON_LOOKUP_PATHERROR ){ - jsonBadPathError(ctx, zPath, rc); + jsonBadPathError(ctx, zPath); }else{ sqlite3_result_error(ctx, "malformed JSON", -1); } @@ -4459,7 +4432,7 @@ static void jsonRemoveFunc( return; json_remove_patherror: - jsonBadPathError(ctx, zPath, 0); + jsonBadPathError(ctx, zPath); json_remove_done: jsonParseFree(p); @@ -4503,18 +4476,16 @@ static void jsonSetFunc( int argc, sqlite3_value **argv ){ + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); - int eInsType = JSON_INSERT_TYPE(flags); - static const char *azInsType[] = { "insert", "set", "array_insert" }; - static const u8 aEditType[] = { JEDIT_INS, JEDIT_SET, JEDIT_AINS }; + int bIsSet = (flags&JSON_ISSET)!=0; if( argc<1 ) return; - assert( eInsType>=0 && eInsType<=2 ); if( (argc&1)==0 ) { - jsonWrongNumArgs(ctx, azInsType[eInsType]); + jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); return; } - jsonInsertIntoBlob(ctx, argc, argv, aEditType[eInsType]); + jsonInsertIntoBlob(ctx, argc, argv, bIsSet ? JEDIT_SET : JEDIT_INS); } /* @@ -4539,7 +4510,7 @@ static void jsonTypeFunc( zPath = (const char*)sqlite3_value_text(argv[1]); if( zPath==0 ) goto json_type_done; if( zPath[0]!='$' ){ - jsonBadPathError(ctx, zPath, 0); + jsonBadPathError(ctx, zPath); goto json_type_done; } i = jsonLookupStep(p, 0, zPath+1, 0); @@ -4547,7 +4518,7 @@ static void jsonTypeFunc( if( i==JSON_LOOKUP_NOTFOUND ){ /* no-op */ }else if( i==JSON_LOOKUP_PATHERROR ){ - jsonBadPathError(ctx, zPath, 0); + jsonBadPathError(ctx, zPath); }else{ sqlite3_result_error(ctx, "malformed JSON", -1); } @@ -4803,11 +4774,12 @@ static void jsonArrayStep( } static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ JsonString *pStr; - int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); if( pStr ){ + int flags; pStr->pCtx = ctx; jsonAppendChar(pStr, ']'); + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); if( pStr->eErr ){ jsonReturnString(pStr, 0, 0); return; @@ -4828,9 +4800,6 @@ static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); jsonStringTrimOneChar(pStr); } - }else if( flags & JSON_BLOB ){ - static const u8 emptyArray = 0x0b; - sqlite3_result_blob(ctx, &emptyArray, 1, SQLITE_STATIC); }else{ sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC); } @@ -4927,11 +4896,12 @@ static void jsonObjectStep( } static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ JsonString *pStr; - int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); if( pStr ){ + int flags; jsonAppendChar(pStr, '}'); pStr->pCtx = ctx; + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); if( pStr->eErr ){ jsonReturnString(pStr, 0, 0); return; @@ -4952,9 +4922,6 @@ static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); jsonStringTrimOneChar(pStr); } - }else if( flags & JSON_BLOB ){ - static const unsigned char emptyObject = 0x0c; - sqlite3_result_blob(ctx, &emptyObject, 1, SQLITE_STATIC); }else{ sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC); } @@ -5455,7 +5422,7 @@ static int jsonEachFilter( if( zRoot==0 ) return SQLITE_OK; if( zRoot[0]!='$' ){ sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot, 0); + cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot); jsonEachCursorReset(p); return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; } @@ -5473,7 +5440,7 @@ static int jsonEachFilter( return SQLITE_OK; } sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot, 0); + cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot); jsonEachCursorReset(p); return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; } @@ -5563,8 +5530,6 @@ void sqlite3RegisterJsonFunctions(void){ JFUNCTION(jsonb, 1,1,0, 0,1,0, jsonRemoveFunc), JFUNCTION(json_array, -1,0,1, 1,0,0, jsonArrayFunc), JFUNCTION(jsonb_array, -1,0,1, 1,1,0, jsonArrayFunc), - JFUNCTION(json_array_insert, -1,1,1, 1,0,JSON_AINS, jsonSetFunc), - JFUNCTION(jsonb_array_insert,-1,1,0, 1,1,JSON_AINS, jsonSetFunc), JFUNCTION(json_array_length, 1,1,0, 0,0,0, jsonArrayLengthFunc), JFUNCTION(json_array_length, 2,1,0, 0,0,0, jsonArrayLengthFunc), JFUNCTION(json_error_position,1,1,0, 0,0,0, jsonErrorFunc), diff --git a/src/loadext.c b/src/loadext.c index 55325edd7..c5177715e 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -522,17 +522,7 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_setlk_timeout, /* Version 3.51.0 and later */ sqlite3_set_errmsg, - sqlite3_db_status64, - /* Version 3.52.0 and later */ - sqlite3_str_truncate, - sqlite3_str_free, -#ifdef SQLITE_ENABLE_CARRAY - sqlite3_carray_bind, - sqlite3_carray_bind_v2 -#else - 0, - 0 -#endif + sqlite3_db_status64 }; /* True if x is the directory separator character @@ -634,42 +624,33 @@ static int sqlite3LoadExtension( ** entry point name "sqlite3_extension_init" was not found, then ** construct an entry point name "sqlite3_X_init" where the X is ** replaced by the lowercase value of every ASCII alphabetic - ** character in the filename after the last "/" up to the first ".", - ** and skipping the first three characters if they are "lib". + ** character in the filename after the last "/" upto the first ".", + ** and eliding the first three characters if they are "lib". ** Examples: ** ** /usr/local/lib/libExample5.4.3.so ==> sqlite3_example_init ** C:/lib/mathfuncs.dll ==> sqlite3_mathfuncs_init - ** - ** If that still finds no entry point, repeat a second time but this - ** time include both alphabetic and numeric characters up to the first - ** ".". Example: - ** - ** /usr/local/lib/libExample5.4.3.so ==> sqlite3_example5_init */ if( xInit==0 && zProc==0 ){ int iFile, iEntry, c; int ncFile = sqlite3Strlen30(zFile); - int cnt = 0; zAltEntry = sqlite3_malloc64(ncFile+30); if( zAltEntry==0 ){ sqlite3OsDlClose(pVfs, handle); return SQLITE_NOMEM_BKPT; } - do{ - memcpy(zAltEntry, "sqlite3_", 8); - for(iFile=ncFile-1; iFile>=0 && !DirSep(zFile[iFile]); iFile--){} - iFile++; - if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; - for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ - if( sqlite3Isalpha(c) || (cnt && sqlite3Isdigit(c)) ){ - zAltEntry[iEntry++] = (char)sqlite3UpperToLower[(unsigned)c]; - } + memcpy(zAltEntry, "sqlite3_", 8); + for(iFile=ncFile-1; iFile>=0 && !DirSep(zFile[iFile]); iFile--){} + iFile++; + if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; + for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ + if( sqlite3Isalpha(c) ){ + zAltEntry[iEntry++] = (char)sqlite3UpperToLower[(unsigned)c]; } - memcpy(zAltEntry+iEntry, "_init", 6); - zEntry = zAltEntry; - xInit = (sqlite3_loadext_entry)sqlite3OsDlSym(pVfs, handle, zEntry); - }while( xInit==0 && (++cnt)<2 ); + } + memcpy(zAltEntry+iEntry, "_init", 6); + zEntry = zAltEntry; + xInit = (sqlite3_loadext_entry)sqlite3OsDlSym(pVfs, handle, zEntry); } if( xInit==0 ){ if( pzErrMsg ){ diff --git a/src/main.c b/src/main.c index b44ac8dca..6efe538d4 100644 --- a/src/main.c +++ b/src/main.c @@ -971,14 +971,6 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){ rc = setupLookaside(db, pBuf, sz, cnt); break; } - case SQLITE_DBCONFIG_FP_DIGITS: { - int nIn = va_arg(ap, int); - int *pOut = va_arg(ap, int*); - if( nIn>3 && nIn<24 ) db->nFpDigit = (u8)nIn; - if( pOut ) *pOut = db->nFpDigit; - rc = SQLITE_OK; - break; - } default: { static const struct { int op; /* The opcode */ @@ -2534,9 +2526,6 @@ void *sqlite3_wal_hook( sqlite3_mutex_leave(db->mutex); return pRet; #else - UNUSED_PARAMETER(db); - UNUSED_PARAMETER(xCallback); - UNUSED_PARAMETER(pArg); return 0; #endif } @@ -2552,11 +2541,6 @@ int sqlite3_wal_checkpoint_v2( int *pnCkpt /* OUT: Total number of frames checkpointed */ ){ #ifdef SQLITE_OMIT_WAL - UNUSED_PARAMETER(db); - UNUSED_PARAMETER(zDb); - UNUSED_PARAMETER(eMode); - UNUSED_PARAMETER(pnLog); - UNUSED_PARAMETER(pnCkpt); return SQLITE_OK; #else int rc; /* Return code */ @@ -2570,12 +2554,11 @@ int sqlite3_wal_checkpoint_v2( if( pnLog ) *pnLog = -1; if( pnCkpt ) *pnCkpt = -1; - assert( SQLITE_CHECKPOINT_NOOP==-1 ); assert( SQLITE_CHECKPOINT_PASSIVE==0 ); assert( SQLITE_CHECKPOINT_FULL==1 ); assert( SQLITE_CHECKPOINT_RESTART==2 ); assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); - if( eMode<SQLITE_CHECKPOINT_NOOP || eMode>SQLITE_CHECKPOINT_TRUNCATE ){ + if( eMode<SQLITE_CHECKPOINT_PASSIVE || eMode>SQLITE_CHECKPOINT_TRUNCATE ){ /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint ** mode: */ return SQLITE_MISUSE_BKPT; @@ -2939,7 +2922,6 @@ static const int aHardLimit[] = { SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */ SQLITE_MAX_TRIGGER_DEPTH, SQLITE_MAX_WORKER_THREADS, - SQLITE_MAX_PARSER_DEPTH, }; /* @@ -2954,9 +2936,6 @@ static const int aHardLimit[] = { #if SQLITE_MAX_SQL_LENGTH>SQLITE_MAX_LENGTH # error SQLITE_MAX_SQL_LENGTH must not be greater than SQLITE_MAX_LENGTH #endif -#if SQLITE_MAX_SQL_LENGTH>2147482624 /* 1024 less than 2^31 */ -# error SQLITE_MAX_SQL_LENGTH must not be greater than 2147482624 -#endif #if SQLITE_MAX_COMPOUND_SELECT<2 # error SQLITE_MAX_COMPOUND_SELECT must be at least 2 #endif @@ -3012,7 +2991,6 @@ int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ assert( aHardLimit[SQLITE_LIMIT_SQL_LENGTH]==SQLITE_MAX_SQL_LENGTH ); assert( aHardLimit[SQLITE_LIMIT_COLUMN]==SQLITE_MAX_COLUMN ); assert( aHardLimit[SQLITE_LIMIT_EXPR_DEPTH]==SQLITE_MAX_EXPR_DEPTH ); - assert( aHardLimit[SQLITE_LIMIT_PARSER_DEPTH]==SQLITE_MAX_PARSER_DEPTH ); assert( aHardLimit[SQLITE_LIMIT_COMPOUND_SELECT]==SQLITE_MAX_COMPOUND_SELECT); assert( aHardLimit[SQLITE_LIMIT_VDBE_OP]==SQLITE_MAX_VDBE_OP ); assert( aHardLimit[SQLITE_LIMIT_FUNCTION_ARG]==SQLITE_MAX_FUNCTION_ARG ); @@ -3022,7 +3000,7 @@ int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS ); - assert( SQLITE_LIMIT_PARSER_DEPTH==(SQLITE_N_LIMIT-1) ); + assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-1) ); if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ @@ -3386,7 +3364,7 @@ static int openDatabase( db = sqlite3MallocZero( sizeof(sqlite3) ); if( db==0 ) goto opendb_out; if( isThreadsafe -#if defined(SQLITE_THREAD_MISUSE_WARNINGS) +#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS || sqlite3GlobalConfig.bCoreMutex #endif ){ @@ -3407,7 +3385,6 @@ static int openDatabase( db->aDb = db->aDbStatic; db->lookaside.bDisable = 1; db->lookaside.sz = 0; - db->nFpDigit = 17; assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); @@ -3853,12 +3830,6 @@ int sqlite3_collation_needed16( */ void *sqlite3_get_clientdata(sqlite3 *db, const char *zName){ DbClientData *p; -#ifdef SQLITE_ENABLE_API_ARMOR - if( !zName || !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif sqlite3_mutex_enter(db->mutex); for(p=db->pDbData; p; p=p->pNext){ if( strcmp(p->zName, zName)==0 ){ @@ -4932,7 +4903,6 @@ const char *sqlite3_filename_journal(const char *zFilename){ } const char *sqlite3_filename_wal(const char *zFilename){ #ifdef SQLITE_OMIT_WAL - UNUSED_PARAMETER(zFilename); return 0; #else zFilename = sqlite3_filename_journal(zFilename); diff --git a/src/mutex.c b/src/mutex.c index 21b47e511..62e09cb4f 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -27,28 +27,23 @@ static SQLITE_WSD int mutexIsInit = 0; #ifndef SQLITE_MUTEX_OMIT -#ifdef SQLITE_THREAD_MISUSE_WARNINGS +#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS /* -** This block (enclosed by SQLITE_THREAD_MISUSE_WARNINGS) contains +** This block (enclosed by SQLITE_ENABLE_MULTITHREADED_CHECKS) contains ** the implementation of a wrapper around the system default mutex ** implementation (sqlite3DefaultMutex()). ** ** Most calls are passed directly through to the underlying default ** mutex implementation. Except, if a mutex is configured by calling ** sqlite3MutexWarnOnContention() on it, then if contention is ever -** encountered within xMutexEnter() then a warning is emitted via -** sqlite3_log(). Furthermore, if SQLITE_THREAD_MISUSE_ABORT is -** defined then abort() is called after the sqlite3_log() warning. +** encountered within xMutexEnter() a warning is emitted via sqlite3_log(). ** -** This type of mutex is used on the database handle mutex when testing -** apps that usually use SQLITE_CONFIG_MULTITHREAD mode. A failure -** indicates that the app ought to be using SQLITE_OPEN_FULLMUTEX or -** similar because it is trying to use the same database handle from -** two different connections at the same time. +** This type of mutex is used as the database handle mutex when testing +** apps that usually use SQLITE_CONFIG_MULTITHREAD mode. */ /* -** Type for all mutexes used when SQLITE_THREAD_MISUSE_WARNINGS +** Type for all mutexes used when SQLITE_ENABLE_MULTITHREADED_CHECKS ** is defined. Variable CheckMutex.mutex is a pointer to the real mutex ** allocated by the system mutex implementation. Variable iType is usually set ** to the type of mutex requested - SQLITE_MUTEX_RECURSIVE, SQLITE_MUTEX_FAST @@ -84,12 +79,11 @@ static int checkMutexNotheld(sqlite3_mutex *p){ */ static int checkMutexInit(void){ pGlobalMutexMethods = sqlite3DefaultMutex(); - return pGlobalMutexMethods->xMutexInit(); + return SQLITE_OK; } static int checkMutexEnd(void){ - int rc = pGlobalMutexMethods->xMutexEnd(); pGlobalMutexMethods = 0; - return rc; + return SQLITE_OK; } /* @@ -166,9 +160,6 @@ static void checkMutexEnter(sqlite3_mutex *p){ sqlite3_log(SQLITE_MISUSE, "illegal multi-threaded access to database connection" ); -#if SQLITE_THREAD_MISUSE_ABORT - abort(); -#endif } pGlobalMutexMethods->xMutexEnter(pCheck->mutex); } @@ -220,7 +211,7 @@ void sqlite3MutexWarnOnContention(sqlite3_mutex *p){ pCheck->iType = SQLITE_MUTEX_WARNONCONTENTION; } } -#endif /* ifdef SQLITE_THREAD_MISUSE_WARNINGS */ +#endif /* ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS */ /* ** Initialize the mutex system. @@ -237,7 +228,7 @@ int sqlite3MutexInit(void){ sqlite3_mutex_methods *pTo = &sqlite3GlobalConfig.mutex; if( sqlite3GlobalConfig.bCoreMutex ){ -#ifdef SQLITE_THREAD_MISUSE_WARNINGS +#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS pFrom = multiThreadedCheckMutex(); #else pFrom = sqlite3DefaultMutex(); diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 20e41b161..7eb5b50be 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -129,7 +129,11 @@ static int winMutexInit(void){ if( InterlockedCompareExchange(&winMutex_lock, 1, 0)==0 ){ int i; for(i=0; i<ArraySize(winMutex_staticMutexes); i++){ +#if SQLITE_OS_WINRT + InitializeCriticalSectionEx(&winMutex_staticMutexes[i].mutex, 0, 0); +#else InitializeCriticalSection(&winMutex_staticMutexes[i].mutex); +#endif } winMutex_isInit = 1; }else{ @@ -219,7 +223,11 @@ static sqlite3_mutex *winMutexAlloc(int iType){ p->trace = 1; #endif #endif +#if SQLITE_OS_WINRT + InitializeCriticalSectionEx(&p->mutex, 0, 0); +#else InitializeCriticalSection(&p->mutex); +#endif } break; } diff --git a/src/os_kv.c b/src/os_kv.c index 1fd1c8e8c..c2d1f9b7a 100644 --- a/src/os_kv.c +++ b/src/os_kv.c @@ -21,7 +21,7 @@ ** Debugging logic */ -/* SQLITE_KV_TRACE() is used for tracing calls to kvrecord routines. */ +/* SQLITE_KV_TRACE() is used for tracing calls to kvstorage routines. */ #if 0 #define SQLITE_KV_TRACE(X) printf X #else @@ -35,6 +35,7 @@ #define SQLITE_KV_LOG(X) #endif + /* ** Forward declaration of objects used by this VFS implementation */ @@ -42,11 +43,6 @@ typedef struct KVVfsFile KVVfsFile; /* A single open file. There are only two files represented by this ** VFS - the database and the rollback journal. -** -** Maintenance reminder: if this struct changes in any way, the JSON -** rendering of its structure must be updated in -** sqlite3-wasm.c:sqlite3__wasm_enum_json(). There are no binary -** compatibility concerns, so it does not need an iVersion member. */ struct KVVfsFile { sqlite3_file base; /* IO methods */ @@ -96,7 +92,7 @@ static int kvvfsCurrentTime(sqlite3_vfs*, double*); static int kvvfsCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); static sqlite3_vfs sqlite3OsKvvfsObject = { - 2, /* iVersion */ + 1, /* iVersion */ sizeof(KVVfsFile), /* szOsFile */ 1024, /* mxPathname */ 0, /* pNext */ @@ -172,37 +168,23 @@ static sqlite3_io_methods kvvfs_jrnl_io_methods = { /* Forward declarations for the low-level storage engine */ -#ifndef SQLITE_WASM -/* In WASM builds these are implemented in JS. */ -static int kvrecordWrite(const char*, const char *zKey, const char *zData); -static int kvrecordDelete(const char*, const char *zKey); -static int kvrecordRead(const char*, const char *zKey, char *zBuf, int nBuf); -#endif -#ifndef KVRECORD_KEY_SZ -#define KVRECORD_KEY_SZ 32 -#endif +static int kvstorageWrite(const char*, const char *zKey, const char *zData); +static int kvstorageDelete(const char*, const char *zKey); +static int kvstorageRead(const char*, const char *zKey, char *zBuf, int nBuf); +#define KVSTORAGE_KEY_SZ 32 /* Expand the key name with an appropriate prefix and put the result ** in zKeyOut[]. The zKeyOut[] buffer is assumed to hold at least -** KVRECORD_KEY_SZ bytes. +** KVSTORAGE_KEY_SZ bytes. */ -static void kvrecordMakeKey( +static void kvstorageMakeKey( const char *zClass, const char *zKeyIn, char *zKeyOut ){ - assert( zKeyIn ); - assert( zKeyOut ); - assert( zClass ); - sqlite3_snprintf(KVRECORD_KEY_SZ, zKeyOut, "kvvfs-%s-%s", - zClass, zKeyIn); + sqlite3_snprintf(KVSTORAGE_KEY_SZ, zKeyOut, "kvvfs-%s-%s", zClass, zKeyIn); } -#ifndef SQLITE_WASM -/* In WASM builds do not define APIs which use fopen(), fwrite(), -** and the like because those APIs are a portability issue for -** WASM. -*/ /* Write content into a key. zClass is the particular namespace of the ** underlying key/value store to use - either "local" or "session". ** @@ -210,14 +192,14 @@ static void kvrecordMakeKey( ** ** Return the number of errors. */ -static int kvrecordWrite( +static int kvstorageWrite( const char *zClass, const char *zKey, const char *zData ){ FILE *fd; - char zXKey[KVRECORD_KEY_SZ]; - kvrecordMakeKey(zClass, zKey, zXKey); + char zXKey[KVSTORAGE_KEY_SZ]; + kvstorageMakeKey(zClass, zKey, zXKey); fd = fopen(zXKey, "wb"); if( fd ){ SQLITE_KV_TRACE(("KVVFS-WRITE %-15s (%d) %.50s%s\n", zXKey, @@ -235,9 +217,9 @@ static int kvrecordWrite( ** namespace given by zClass. If the key does not previously exist, ** this routine is a no-op. */ -static int kvrecordDelete(const char *zClass, const char *zKey){ - char zXKey[KVRECORD_KEY_SZ]; - kvrecordMakeKey(zClass, zKey, zXKey); +static int kvstorageDelete(const char *zClass, const char *zKey){ + char zXKey[KVSTORAGE_KEY_SZ]; + kvstorageMakeKey(zClass, zKey, zXKey); unlink(zXKey); SQLITE_KV_TRACE(("KVVFS-DELETE %-15s\n", zXKey)); return 0; @@ -258,7 +240,7 @@ static int kvrecordDelete(const char *zClass, const char *zKey){ ** zero-terminates zBuf at zBuf[0] and returns the size of the data ** without reading it. */ -static int kvrecordRead( +static int kvstorageRead( const char *zClass, const char *zKey, char *zBuf, @@ -266,8 +248,8 @@ static int kvrecordRead( ){ FILE *fd; struct stat buf; - char zXKey[KVRECORD_KEY_SZ]; - kvrecordMakeKey(zClass, zKey, zXKey); + char zXKey[KVSTORAGE_KEY_SZ]; + kvstorageMakeKey(zClass, zKey, zXKey); if( access(zXKey, R_OK)!=0 || stat(zXKey, &buf)!=0 || !S_ISREG(buf.st_mode) @@ -299,8 +281,6 @@ static int kvrecordRead( return (int)n; } } -#endif /* #ifndef SQLITE_WASM */ - /* ** An internal level of indirection which enables us to replace the @@ -308,27 +288,17 @@ static int kvrecordRead( ** Maintenance reminder: if this struct changes in any way, the JSON ** rendering of its structure must be updated in ** sqlite3-wasm.c:sqlite3__wasm_enum_json(). There are no binary -** compatibility concerns, so it does not need an iVersion member. +** compatibility concerns, so it does not need an iVersion +** member. */ typedef struct sqlite3_kvvfs_methods sqlite3_kvvfs_methods; struct sqlite3_kvvfs_methods { - int (*xRcrdRead)(const char*, const char *zKey, char *zBuf, int nBuf); - int (*xRcrdWrite)(const char*, const char *zKey, const char *zData); - int (*xRcrdDelete)(const char*, const char *zKey); + int (*xRead)(const char *zClass, const char *zKey, char *zBuf, int nBuf); + int (*xWrite)(const char *zClass, const char *zKey, const char *zData); + int (*xDelete)(const char *zClass, const char *zKey); const int nKeySize; - const int nBufferSize; -#ifndef SQLITE_WASM -# define MAYBE_CONST const -#else -# define MAYBE_CONST -#endif - MAYBE_CONST sqlite3_vfs * pVfs; - MAYBE_CONST sqlite3_io_methods *pIoDb; - MAYBE_CONST sqlite3_io_methods *pIoJrnl; -#undef MAYBE_CONST }; - /* ** This object holds the kvvfs I/O methods which may be swapped out ** for JavaScript-side implementations in WASM builds. In such builds @@ -343,20 +313,10 @@ struct sqlite3_kvvfs_methods { const #endif sqlite3_kvvfs_methods sqlite3KvvfsMethods = { -#ifndef SQLITE_WASM - .xRcrdRead = kvrecordRead, - .xRcrdWrite = kvrecordWrite, - .xRcrdDelete = kvrecordDelete, -#else - .xRcrdRead = 0, - .xRcrdWrite = 0, - .xRcrdDelete = 0, -#endif - .nKeySize = KVRECORD_KEY_SZ, - .nBufferSize = SQLITE_KVOS_SZ, - .pVfs = &sqlite3OsKvvfsObject, - .pIoDb = &kvvfs_db_io_methods, - .pIoJrnl = &kvvfs_jrnl_io_methods +kvstorageRead, +kvstorageWrite, +kvstorageDelete, +KVSTORAGE_KEY_SZ }; /****** Utility subroutines ************************************************/ @@ -383,10 +343,7 @@ sqlite3_kvvfs_methods sqlite3KvvfsMethods = { ** of hexadecimal and base-26 numbers, it is always clear where ** one stops and the next begins. */ -#ifndef SQLITE_WASM -static -#endif -int kvvfsEncode(const char *aData, int nData, char *aOut){ +static int kvvfsEncode(const char *aData, int nData, char *aOut){ int i, j; const unsigned char *a = (const unsigned char*)aData; for(i=j=0; i<nData; i++){ @@ -437,13 +394,9 @@ static const signed char kvvfsHexValue[256] = { ** Decode the text encoding back to binary. The binary content is ** written into pOut, which must be at least nOut bytes in length. ** -** The return value is the number of bytes actually written into aOut[], or -** -1 for malformed inputs. +** The return value is the number of bytes actually written into aOut[]. */ -#ifndef SQLITE_WASM -static -#endif -int kvvfsDecode(const char *a, char *aOut, int nOut){ +static int kvvfsDecode(const char *a, char *aOut, int nOut){ int i, j; int c; const unsigned char *aIn = (const unsigned char*)a; @@ -468,7 +421,7 @@ int kvvfsDecode(const char *a, char *aOut, int nOut){ }else{ aOut[j] = c<<4; c = kvvfsHexValue[aIn[++i]]; - if( c<0 ) return -1 /* hex bytes are always in pairs */; + if( c<0 ) break; aOut[j++] += c; i++; } @@ -521,14 +474,13 @@ static void kvvfsDecodeJournal( static sqlite3_int64 kvvfsReadFileSize(KVVfsFile *pFile){ char zData[50]; zData[0] = 0; - sqlite3KvvfsMethods.xRcrdRead(pFile->zClass, "sz", zData, - sizeof(zData)-1); + sqlite3KvvfsMethods.xRead(pFile->zClass, "sz", zData, sizeof(zData)-1); return strtoll(zData, 0, 0); } static int kvvfsWriteFileSize(KVVfsFile *pFile, sqlite3_int64 sz){ char zData[50]; sqlite3_snprintf(sizeof(zData), zData, "%lld", sz); - return sqlite3KvvfsMethods.xRcrdWrite(pFile->zClass, "sz", zData); + return sqlite3KvvfsMethods.xWrite(pFile->zClass, "sz", zData); } /****** sqlite3_io_methods methods ******************************************/ @@ -539,13 +491,10 @@ static int kvvfsWriteFileSize(KVVfsFile *pFile, sqlite3_int64 sz){ static int kvvfsClose(sqlite3_file *pProtoFile){ KVVfsFile *pFile = (KVVfsFile *)pProtoFile; - SQLITE_KV_LOG(("xClose %s %s\n", pFile->zClass, + SQLITE_KV_LOG(("xClose %s %s\n", pFile->zClass, pFile->isJournal ? "journal" : "db")); sqlite3_free(pFile->aJrnl); sqlite3_free(pFile->aData); -#ifdef SQLITE_WASM - memset(pFile, 0, sizeof(*pFile)); -#endif return SQLITE_OK; } @@ -554,30 +503,24 @@ static int kvvfsClose(sqlite3_file *pProtoFile){ */ static int kvvfsReadJrnl( sqlite3_file *pProtoFile, - void *zBuf, - int iAmt, + void *zBuf, + int iAmt, sqlite_int64 iOfst ){ KVVfsFile *pFile = (KVVfsFile*)pProtoFile; assert( pFile->isJournal ); SQLITE_KV_LOG(("xRead('%s-journal',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); if( pFile->aJrnl==0 ){ - int rc; - int szTxt = sqlite3KvvfsMethods.xRcrdRead(pFile->zClass, "jrnl", - 0, 0); + int szTxt = kvstorageRead(pFile->zClass, "jrnl", 0, 0); char *aTxt; if( szTxt<=4 ){ return SQLITE_IOERR; } aTxt = sqlite3_malloc64( szTxt+1 ); if( aTxt==0 ) return SQLITE_NOMEM; - rc = sqlite3KvvfsMethods.xRcrdRead(pFile->zClass, "jrnl", - aTxt, szTxt+1); - if( rc>=0 ){ - kvvfsDecodeJournal(pFile, aTxt, szTxt); - } + kvstorageRead(pFile->zClass, "jrnl", aTxt, szTxt+1); + kvvfsDecodeJournal(pFile, aTxt, szTxt); sqlite3_free(aTxt); - if( rc ) return rc; if( pFile->aJrnl==0 ) return SQLITE_IOERR; } if( iOfst+iAmt>pFile->nJrnl ){ @@ -592,8 +535,8 @@ static int kvvfsReadJrnl( */ static int kvvfsReadDb( sqlite3_file *pProtoFile, - void *zBuf, - int iAmt, + void *zBuf, + int iAmt, sqlite_int64 iOfst ){ KVVfsFile *pFile = (KVVfsFile*)pProtoFile; @@ -617,8 +560,8 @@ static int kvvfsReadDb( pgno = 1; } sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); - got = sqlite3KvvfsMethods.xRcrdRead(pFile->zClass, zKey, - aData, SQLITE_KVOS_SZ-1); + got = sqlite3KvvfsMethods.xRead(pFile->zClass, zKey, + aData, SQLITE_KVOS_SZ-1); if( got<0 ){ n = 0; }else{ @@ -650,8 +593,8 @@ static int kvvfsReadDb( */ static int kvvfsWriteJrnl( sqlite3_file *pProtoFile, - const void *zBuf, - int iAmt, + const void *zBuf, + int iAmt, sqlite_int64 iOfst ){ KVVfsFile *pFile = (KVVfsFile*)pProtoFile; @@ -678,15 +621,14 @@ static int kvvfsWriteJrnl( */ static int kvvfsWriteDb( sqlite3_file *pProtoFile, - const void *zBuf, - int iAmt, + const void *zBuf, + int iAmt, sqlite_int64 iOfst ){ KVVfsFile *pFile = (KVVfsFile*)pProtoFile; unsigned int pgno; char zKey[30]; char *aData = pFile->aData; - int rc; SQLITE_KV_LOG(("xWrite('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); assert( iAmt>=512 && iAmt<=65536 ); assert( (iAmt & (iAmt-1))==0 ); @@ -695,13 +637,13 @@ static int kvvfsWriteDb( pgno = 1 + iOfst/iAmt; sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); kvvfsEncode(zBuf, iAmt, aData); - rc = sqlite3KvvfsMethods.xRcrdWrite(pFile->zClass, zKey, aData); - if( 0==rc ){ - if( iOfst+iAmt > pFile->szDb ){ - pFile->szDb = iOfst + iAmt; - } + if( sqlite3KvvfsMethods.xWrite(pFile->zClass, zKey, aData) ){ + return SQLITE_IOERR; } - return rc; + if( iOfst+iAmt > pFile->szDb ){ + pFile->szDb = iOfst + iAmt; + } + return SQLITE_OK; } /* @@ -711,7 +653,7 @@ static int kvvfsTruncateJrnl(sqlite3_file *pProtoFile, sqlite_int64 size){ KVVfsFile *pFile = (KVVfsFile *)pProtoFile; SQLITE_KV_LOG(("xTruncate('%s-journal',%lld)\n", pFile->zClass, size)); assert( size==0 ); - sqlite3KvvfsMethods.xRcrdDelete(pFile->zClass, "jrnl"); + sqlite3KvvfsMethods.xDelete(pFile->zClass, "jrnl"); sqlite3_free(pFile->aJrnl); pFile->aJrnl = 0; pFile->nJrnl = 0; @@ -720,7 +662,7 @@ static int kvvfsTruncateJrnl(sqlite3_file *pProtoFile, sqlite_int64 size){ static int kvvfsTruncateDb(sqlite3_file *pProtoFile, sqlite_int64 size){ KVVfsFile *pFile = (KVVfsFile *)pProtoFile; if( pFile->szDb>size - && pFile->szPage>0 + && pFile->szPage>0 && (size % pFile->szPage)==0 ){ char zKey[50]; @@ -730,7 +672,7 @@ static int kvvfsTruncateDb(sqlite3_file *pProtoFile, sqlite_int64 size){ pgnoMax = 2 + pFile->szDb/pFile->szPage; while( pgno<=pgnoMax ){ sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); - sqlite3KvvfsMethods.xRcrdDelete(pFile->zClass, zKey); + sqlite3KvvfsMethods.xDelete(pFile->zClass, zKey); pgno++; } pFile->szDb = size; @@ -762,7 +704,7 @@ static int kvvfsSyncJrnl(sqlite3_file *pProtoFile, int flags){ }while( n>0 ); zOut[i++] = ' '; kvvfsEncode(pFile->aJrnl, pFile->nJrnl, &zOut[i]); - i = sqlite3KvvfsMethods.xRcrdWrite(pFile->zClass, "jrnl", zOut); + i = sqlite3KvvfsMethods.xWrite(pFile->zClass, "jrnl", zOut); sqlite3_free(zOut); return i ? SQLITE_IOERR : SQLITE_OK; } @@ -876,32 +818,33 @@ static int kvvfsOpen( KVVfsFile *pFile = (KVVfsFile*)pProtoFile; if( zName==0 ) zName = ""; SQLITE_KV_LOG(("xOpen(\"%s\")\n", zName)); - assert(!pFile->zClass); - assert(!pFile->aData); - assert(!pFile->aJrnl); - assert(!pFile->nJrnl); - assert(!pFile->base.pMethods); - pFile->szPage = -1; - pFile->szDb = -1; - if( 0==sqlite3_strglob("*-journal", zName) ){ + if( strcmp(zName, "local")==0 + || strcmp(zName, "session")==0 + ){ + pFile->isJournal = 0; + pFile->base.pMethods = &kvvfs_db_io_methods; + }else + if( strcmp(zName, "local-journal")==0 + || strcmp(zName, "session-journal")==0 + ){ pFile->isJournal = 1; pFile->base.pMethods = &kvvfs_jrnl_io_methods; - if( 0==strcmp("session-journal",zName) ){ - pFile->zClass = "session"; - }else if( 0==strcmp("local-journal",zName) ){ - pFile->zClass = "local"; - } }else{ - pFile->isJournal = 0; - pFile->base.pMethods = &kvvfs_db_io_methods; + return SQLITE_CANTOPEN; } - if( !pFile->zClass ){ - pFile->zClass = zName; + if( zName[0]=='s' ){ + pFile->zClass = "session"; + }else{ + pFile->zClass = "local"; } pFile->aData = sqlite3_malloc64(SQLITE_KVOS_SZ); if( pFile->aData==0 ){ return SQLITE_NOMEM; } + pFile->aJrnl = 0; + pFile->nJrnl = 0; + pFile->szPage = -1; + pFile->szDb = -1; return SQLITE_OK; } @@ -911,17 +854,13 @@ static int kvvfsOpen( ** returning. */ static int kvvfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ - int rc /* The JS impl can fail with OOM in argument conversion */; if( strcmp(zPath, "local-journal")==0 ){ - rc = sqlite3KvvfsMethods.xRcrdDelete("local", "jrnl"); + sqlite3KvvfsMethods.xDelete("local", "jrnl"); }else if( strcmp(zPath, "session-journal")==0 ){ - rc = sqlite3KvvfsMethods.xRcrdDelete("session", "jrnl"); - } - else{ - rc = 0; + sqlite3KvvfsMethods.xDelete("session", "jrnl"); } - return rc; + return SQLITE_OK; } /* @@ -929,48 +868,27 @@ static int kvvfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ ** is available, or false otherwise. */ static int kvvfsAccess( - sqlite3_vfs *pProtoVfs, - const char *zPath, - int flags, + sqlite3_vfs *pProtoVfs, + const char *zPath, + int flags, int *pResOut ){ SQLITE_KV_LOG(("xAccess(\"%s\")\n", zPath)); -#if 0 && defined(SQLITE_WASM) - /* - ** This is not having the desired effect in the JS bindings. - ** It's ostensibly the same logic as the #else block, but - ** it's not behaving that way. - ** - ** In JS we map all zPaths to Storage objects, and -journal files - ** are mapped to the storage for the main db (which is is exactly - ** what the mapping of "local-journal" -> "local" is doing). - */ - const char *zKey = (0==sqlite3_strglob("*-journal", zPath)) - ? "jrnl" : "sz"; - *pResOut = - sqlite3KvvfsMethods.xRcrdRead(zPath, zKey, 0, 0)>0; -#else if( strcmp(zPath, "local-journal")==0 ){ - *pResOut = - sqlite3KvvfsMethods.xRcrdRead("local", "jrnl", 0, 0)>0; + *pResOut = sqlite3KvvfsMethods.xRead("local", "jrnl", 0, 0)>0; }else if( strcmp(zPath, "session-journal")==0 ){ - *pResOut = - sqlite3KvvfsMethods.xRcrdRead("session", "jrnl", 0, 0)>0; + *pResOut = sqlite3KvvfsMethods.xRead("session", "jrnl", 0, 0)>0; }else if( strcmp(zPath, "local")==0 ){ - *pResOut = - sqlite3KvvfsMethods.xRcrdRead("local", "sz", 0, 0)>0; + *pResOut = sqlite3KvvfsMethods.xRead("local", "sz", 0, 0)>0; }else if( strcmp(zPath, "session")==0 ){ - *pResOut = - sqlite3KvvfsMethods.xRcrdRead("session", "sz", 0, 0)>0; + *pResOut = sqlite3KvvfsMethods.xRead("session", "sz", 0, 0)>0; }else { *pResOut = 0; } - /*all current JS tests avoid triggering: assert( *pResOut == 0 ); */ -#endif SQLITE_KV_LOG(("xAccess returns %d\n",*pResOut)); return SQLITE_OK; } @@ -981,9 +899,9 @@ static int kvvfsAccess( ** of at least (INST_MAX_PATHNAME+1) bytes. */ static int kvvfsFullPathname( - sqlite3_vfs *pVfs, - const char *zPath, - int nOut, + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, char *zOut ){ size_t nPath; @@ -1006,7 +924,7 @@ static void *kvvfsDlOpen(sqlite3_vfs *pVfs, const char *zPath){ } /* -** Populate the buffer pointed to by zBufOut with nByte bytes of +** Populate the buffer pointed to by zBufOut with nByte bytes of ** random data. */ static int kvvfsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ @@ -1015,7 +933,7 @@ static int kvvfsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ } /* -** Sleep for nMicro microseconds. Return the number of microseconds +** Sleep for nMicro microseconds. Return the number of microseconds ** actually slept. */ static int kvvfsSleep(sqlite3_vfs *pVfs, int nMicro){ @@ -1043,7 +961,7 @@ static int kvvfsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ #endif /* SQLITE_OS_KV || SQLITE_OS_UNIX */ #if SQLITE_OS_KV -/* +/* ** This routine is called initialize the KV-vfs as the default VFS. */ int sqlite3_os_init(void){ diff --git a/src/os_unix.c b/src/os_unix.c index 2f75829c8..d73d89924 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5190,7 +5190,7 @@ static int unixShmMap( } /* Map the requested memory region into this processes address space. */ - apNew = (char **)sqlite3_realloc64( + apNew = (char **)sqlite3_realloc( pShmNode->apRegion, nReqRegion*sizeof(char *) ); if( !apNew ){ diff --git a/src/os_win.c b/src/os_win.c index 7583ecc1f..a6b25f2e8 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -43,7 +43,7 @@ ** Are most of the Win32 ANSI APIs available (i.e. with certain exceptions ** based on the sub-platform)? */ -#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_ANSI) +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(SQLITE_WIN32_NO_ANSI) # define SQLITE_WIN32_HAS_ANSI #endif @@ -51,7 +51,7 @@ ** Are most of the Win32 Unicode APIs available (i.e. with certain exceptions ** based on the sub-platform)? */ -#if (SQLITE_OS_WINCE || SQLITE_OS_WINNT) && \ +#if (SQLITE_OS_WINCE || SQLITE_OS_WINNT || SQLITE_OS_WINRT) && \ !defined(SQLITE_WIN32_NO_WIDE) # define SQLITE_WIN32_HAS_WIDE #endif @@ -190,7 +190,16 @@ */ #if SQLITE_WIN32_FILEMAPPING_API && \ (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) +/* +** Two of the file mapping APIs are different under WinRT. Figure out which +** set we need. +*/ +#if SQLITE_OS_WINRT +WINBASEAPI HANDLE WINAPI CreateFileMappingFromApp(HANDLE, \ + LPSECURITY_ATTRIBUTES, ULONG, ULONG64, LPCWSTR); +WINBASEAPI LPVOID WINAPI MapViewOfFileFromApp(HANDLE, ULONG, ULONG64, SIZE_T); +#else #if defined(SQLITE_WIN32_HAS_ANSI) WINBASEAPI HANDLE WINAPI CreateFileMappingA(HANDLE, LPSECURITY_ATTRIBUTES, \ DWORD, DWORD, DWORD, LPCSTR); @@ -202,6 +211,7 @@ WINBASEAPI HANDLE WINAPI CreateFileMappingW(HANDLE, LPSECURITY_ATTRIBUTES, \ #endif /* defined(SQLITE_WIN32_HAS_WIDE) */ WINBASEAPI LPVOID WINAPI MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); +#endif /* SQLITE_OS_WINRT */ /* ** These file mapping APIs are common to both Win32 and WinRT. @@ -492,7 +502,7 @@ static LONG SQLITE_WIN32_VOLATILE sqlite3_os_type = 0; ** This function is not available on Windows CE or WinRT. */ -#if SQLITE_OS_WINCE +#if SQLITE_OS_WINCE || SQLITE_OS_WINRT # define osAreFileApisANSI() 1 #endif @@ -507,7 +517,7 @@ static struct win_syscall { sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ sqlite3_syscall_ptr pDefault; /* Default value */ } aSyscall[] = { -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT { "AreFileApisANSI", (SYSCALL)AreFileApisANSI, 0 }, #else { "AreFileApisANSI", (SYSCALL)0, 0 }, @@ -546,7 +556,7 @@ static struct win_syscall { #define osCreateFileA ((HANDLE(WINAPI*)(LPCSTR,DWORD,DWORD, \ LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[4].pCurrent) -#if defined(SQLITE_WIN32_HAS_WIDE) +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) { "CreateFileW", (SYSCALL)CreateFileW, 0 }, #else { "CreateFileW", (SYSCALL)0, 0 }, @@ -555,7 +565,7 @@ static struct win_syscall { #define osCreateFileW ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD, \ LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[5].pCurrent) -#if defined(SQLITE_WIN32_HAS_ANSI) && \ +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_ANSI) && \ (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) && \ SQLITE_WIN32_CREATEFILEMAPPINGA { "CreateFileMappingA", (SYSCALL)CreateFileMappingA, 0 }, @@ -566,8 +576,8 @@ static struct win_syscall { #define osCreateFileMappingA ((HANDLE(WINAPI*)(HANDLE,LPSECURITY_ATTRIBUTES, \ DWORD,DWORD,DWORD,LPCSTR))aSyscall[6].pCurrent) -#if (SQLITE_OS_WINCE || defined(SQLITE_WIN32_HAS_WIDE)) && \ - (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) +#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) { "CreateFileMappingW", (SYSCALL)CreateFileMappingW, 0 }, #else { "CreateFileMappingW", (SYSCALL)0, 0 }, @@ -576,7 +586,7 @@ static struct win_syscall { #define osCreateFileMappingW ((HANDLE(WINAPI*)(HANDLE,LPSECURITY_ATTRIBUTES, \ DWORD,DWORD,DWORD,LPCWSTR))aSyscall[7].pCurrent) -#if defined(SQLITE_WIN32_HAS_WIDE) +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) { "CreateMutexW", (SYSCALL)CreateMutexW, 0 }, #else { "CreateMutexW", (SYSCALL)0, 0 }, @@ -662,7 +672,7 @@ static struct win_syscall { #define osGetDiskFreeSpaceA ((BOOL(WINAPI*)(LPCSTR,LPDWORD,LPDWORD,LPDWORD, \ LPDWORD))aSyscall[18].pCurrent) -#if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_WIDE) +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) { "GetDiskFreeSpaceW", (SYSCALL)GetDiskFreeSpaceW, 0 }, #else { "GetDiskFreeSpaceW", (SYSCALL)0, 0 }, @@ -679,7 +689,7 @@ static struct win_syscall { #define osGetFileAttributesA ((DWORD(WINAPI*)(LPCSTR))aSyscall[20].pCurrent) -#if defined(SQLITE_WIN32_HAS_WIDE) +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) { "GetFileAttributesW", (SYSCALL)GetFileAttributesW, 0 }, #else { "GetFileAttributesW", (SYSCALL)0, 0 }, @@ -696,7 +706,11 @@ static struct win_syscall { #define osGetFileAttributesExW ((BOOL(WINAPI*)(LPCWSTR,GET_FILEEX_INFO_LEVELS, \ LPVOID))aSyscall[22].pCurrent) +#if !SQLITE_OS_WINRT { "GetFileSize", (SYSCALL)GetFileSize, 0 }, +#else + { "GetFileSize", (SYSCALL)0, 0 }, +#endif #define osGetFileSize ((DWORD(WINAPI*)(HANDLE,LPDWORD))aSyscall[23].pCurrent) @@ -709,7 +723,7 @@ static struct win_syscall { #define osGetFullPathNameA ((DWORD(WINAPI*)(LPCSTR,DWORD,LPSTR, \ LPSTR*))aSyscall[24].pCurrent) -#if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_WIDE) +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) { "GetFullPathNameW", (SYSCALL)GetFullPathNameW, 0 }, #else { "GetFullPathNameW", (SYSCALL)0, 0 }, @@ -744,10 +758,16 @@ static struct win_syscall { #define osGetProcAddressA ((FARPROC(WINAPI*)(HMODULE, \ LPCSTR))aSyscall[27].pCurrent) +#if !SQLITE_OS_WINRT { "GetSystemInfo", (SYSCALL)GetSystemInfo, 0 }, +#else + { "GetSystemInfo", (SYSCALL)0, 0 }, +#endif + #define osGetSystemInfo ((VOID(WINAPI*)(LPSYSTEM_INFO))aSyscall[28].pCurrent) { "GetSystemTime", (SYSCALL)GetSystemTime, 0 }, + #define osGetSystemTime ((VOID(WINAPI*)(LPSYSTEMTIME))aSyscall[29].pCurrent) #if !SQLITE_OS_WINCE @@ -767,7 +787,7 @@ static struct win_syscall { #define osGetTempPathA ((DWORD(WINAPI*)(DWORD,LPSTR))aSyscall[31].pCurrent) -#if defined(SQLITE_WIN32_HAS_WIDE) +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) { "GetTempPathW", (SYSCALL)GetTempPathW, 0 }, #else { "GetTempPathW", (SYSCALL)0, 0 }, @@ -775,7 +795,11 @@ static struct win_syscall { #define osGetTempPathW ((DWORD(WINAPI*)(DWORD,LPWSTR))aSyscall[32].pCurrent) +#if !SQLITE_OS_WINRT { "GetTickCount", (SYSCALL)GetTickCount, 0 }, +#else + { "GetTickCount", (SYSCALL)0, 0 }, +#endif #define osGetTickCount ((DWORD(WINAPI*)(VOID))aSyscall[33].pCurrent) @@ -788,7 +812,7 @@ static struct win_syscall { #define osGetVersionExA ((BOOL(WINAPI*)( \ LPOSVERSIONINFOA))aSyscall[34].pCurrent) -#if defined(SQLITE_WIN32_HAS_WIDE) && \ +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ SQLITE_WIN32_GETVERSIONEX { "GetVersionExW", (SYSCALL)GetVersionExW, 0 }, #else @@ -803,12 +827,20 @@ static struct win_syscall { #define osHeapAlloc ((LPVOID(WINAPI*)(HANDLE,DWORD, \ SIZE_T))aSyscall[36].pCurrent) +#if !SQLITE_OS_WINRT { "HeapCreate", (SYSCALL)HeapCreate, 0 }, +#else + { "HeapCreate", (SYSCALL)0, 0 }, +#endif #define osHeapCreate ((HANDLE(WINAPI*)(DWORD,SIZE_T, \ SIZE_T))aSyscall[37].pCurrent) +#if !SQLITE_OS_WINRT { "HeapDestroy", (SYSCALL)HeapDestroy, 0 }, +#else + { "HeapDestroy", (SYSCALL)0, 0 }, +#endif #define osHeapDestroy ((BOOL(WINAPI*)(HANDLE))aSyscall[38].pCurrent) @@ -826,12 +858,16 @@ static struct win_syscall { #define osHeapSize ((SIZE_T(WINAPI*)(HANDLE,DWORD, \ LPCVOID))aSyscall[41].pCurrent) +#if !SQLITE_OS_WINRT { "HeapValidate", (SYSCALL)HeapValidate, 0 }, +#else + { "HeapValidate", (SYSCALL)0, 0 }, +#endif #define osHeapValidate ((BOOL(WINAPI*)(HANDLE,DWORD, \ LPCVOID))aSyscall[42].pCurrent) -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT { "HeapCompact", (SYSCALL)HeapCompact, 0 }, #else { "HeapCompact", (SYSCALL)0, 0 }, @@ -847,7 +883,7 @@ static struct win_syscall { #define osLoadLibraryA ((HMODULE(WINAPI*)(LPCSTR))aSyscall[44].pCurrent) -#if defined(SQLITE_WIN32_HAS_WIDE) && \ +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ !defined(SQLITE_OMIT_LOAD_EXTENSION) { "LoadLibraryW", (SYSCALL)LoadLibraryW, 0 }, #else @@ -856,11 +892,15 @@ static struct win_syscall { #define osLoadLibraryW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[45].pCurrent) +#if !SQLITE_OS_WINRT { "LocalFree", (SYSCALL)LocalFree, 0 }, +#else + { "LocalFree", (SYSCALL)0, 0 }, +#endif #define osLocalFree ((HLOCAL(WINAPI*)(HLOCAL))aSyscall[46].pCurrent) -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT { "LockFile", (SYSCALL)LockFile, 0 }, #else { "LockFile", (SYSCALL)0, 0 }, @@ -882,7 +922,8 @@ static struct win_syscall { LPOVERLAPPED))aSyscall[48].pCurrent) #endif -#if SQLITE_OS_WINCE || !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 +#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) { "MapViewOfFile", (SYSCALL)MapViewOfFile, 0 }, #else { "MapViewOfFile", (SYSCALL)0, 0 }, @@ -910,12 +951,20 @@ static struct win_syscall { #define osSetEndOfFile ((BOOL(WINAPI*)(HANDLE))aSyscall[53].pCurrent) +#if !SQLITE_OS_WINRT { "SetFilePointer", (SYSCALL)SetFilePointer, 0 }, +#else + { "SetFilePointer", (SYSCALL)0, 0 }, +#endif #define osSetFilePointer ((DWORD(WINAPI*)(HANDLE,LONG,PLONG, \ DWORD))aSyscall[54].pCurrent) +#if !SQLITE_OS_WINRT { "Sleep", (SYSCALL)Sleep, 0 }, +#else + { "Sleep", (SYSCALL)0, 0 }, +#endif #define osSleep ((VOID(WINAPI*)(DWORD))aSyscall[55].pCurrent) @@ -924,7 +973,7 @@ static struct win_syscall { #define osSystemTimeToFileTime ((BOOL(WINAPI*)(const SYSTEMTIME*, \ LPFILETIME))aSyscall[56].pCurrent) -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT { "UnlockFile", (SYSCALL)UnlockFile, 0 }, #else { "UnlockFile", (SYSCALL)0, 0 }, @@ -962,6 +1011,15 @@ static struct win_syscall { #define osWriteFile ((BOOL(WINAPI*)(HANDLE,LPCVOID,DWORD,LPDWORD, \ LPOVERLAPPED))aSyscall[61].pCurrent) +#if SQLITE_OS_WINRT + { "CreateEventExW", (SYSCALL)CreateEventExW, 0 }, +#else + { "CreateEventExW", (SYSCALL)0, 0 }, +#endif + +#define osCreateEventExW ((HANDLE(WINAPI*)(LPSECURITY_ATTRIBUTES,LPCWSTR, \ + DWORD,DWORD))aSyscall[62].pCurrent) + /* ** For WaitForSingleObject(), MSDN says: ** @@ -971,7 +1029,7 @@ static struct win_syscall { { "WaitForSingleObject", (SYSCALL)WaitForSingleObject, 0 }, #define osWaitForSingleObject ((DWORD(WINAPI*)(HANDLE, \ - DWORD))aSyscall[62].pCurrent) + DWORD))aSyscall[63].pCurrent) #if !SQLITE_OS_WINCE { "WaitForSingleObjectEx", (SYSCALL)WaitForSingleObjectEx, 0 }, @@ -980,12 +1038,69 @@ static struct win_syscall { #endif #define osWaitForSingleObjectEx ((DWORD(WINAPI*)(HANDLE,DWORD, \ - BOOL))aSyscall[63].pCurrent) + BOOL))aSyscall[64].pCurrent) + +#if SQLITE_OS_WINRT + { "SetFilePointerEx", (SYSCALL)SetFilePointerEx, 0 }, +#else + { "SetFilePointerEx", (SYSCALL)0, 0 }, +#endif + +#define osSetFilePointerEx ((BOOL(WINAPI*)(HANDLE,LARGE_INTEGER, \ + PLARGE_INTEGER,DWORD))aSyscall[65].pCurrent) +#if SQLITE_OS_WINRT + { "GetFileInformationByHandleEx", (SYSCALL)GetFileInformationByHandleEx, 0 }, +#else + { "GetFileInformationByHandleEx", (SYSCALL)0, 0 }, +#endif + +#define osGetFileInformationByHandleEx ((BOOL(WINAPI*)(HANDLE, \ + FILE_INFO_BY_HANDLE_CLASS,LPVOID,DWORD))aSyscall[66].pCurrent) + +#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) + { "MapViewOfFileFromApp", (SYSCALL)MapViewOfFileFromApp, 0 }, +#else + { "MapViewOfFileFromApp", (SYSCALL)0, 0 }, +#endif + +#define osMapViewOfFileFromApp ((LPVOID(WINAPI*)(HANDLE,ULONG,ULONG64, \ + SIZE_T))aSyscall[67].pCurrent) + +#if SQLITE_OS_WINRT + { "CreateFile2", (SYSCALL)CreateFile2, 0 }, +#else + { "CreateFile2", (SYSCALL)0, 0 }, +#endif + +#define osCreateFile2 ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD,DWORD, \ + LPCREATEFILE2_EXTENDED_PARAMETERS))aSyscall[68].pCurrent) + +#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_LOAD_EXTENSION) + { "LoadPackagedLibrary", (SYSCALL)LoadPackagedLibrary, 0 }, +#else + { "LoadPackagedLibrary", (SYSCALL)0, 0 }, +#endif + +#define osLoadPackagedLibrary ((HMODULE(WINAPI*)(LPCWSTR, \ + DWORD))aSyscall[69].pCurrent) + +#if SQLITE_OS_WINRT + { "GetTickCount64", (SYSCALL)GetTickCount64, 0 }, +#else + { "GetTickCount64", (SYSCALL)0, 0 }, +#endif + +#define osGetTickCount64 ((ULONGLONG(WINAPI*)(VOID))aSyscall[70].pCurrent) + +#if SQLITE_OS_WINRT { "GetNativeSystemInfo", (SYSCALL)GetNativeSystemInfo, 0 }, +#else + { "GetNativeSystemInfo", (SYSCALL)0, 0 }, +#endif #define osGetNativeSystemInfo ((VOID(WINAPI*)( \ - LPSYSTEM_INFO))aSyscall[64].pCurrent) + LPSYSTEM_INFO))aSyscall[71].pCurrent) #if defined(SQLITE_WIN32_HAS_ANSI) { "OutputDebugStringA", (SYSCALL)OutputDebugStringA, 0 }, @@ -993,7 +1108,7 @@ static struct win_syscall { { "OutputDebugStringA", (SYSCALL)0, 0 }, #endif -#define osOutputDebugStringA ((VOID(WINAPI*)(LPCSTR))aSyscall[65].pCurrent) +#define osOutputDebugStringA ((VOID(WINAPI*)(LPCSTR))aSyscall[72].pCurrent) #if defined(SQLITE_WIN32_HAS_WIDE) { "OutputDebugStringW", (SYSCALL)OutputDebugStringW, 0 }, @@ -1001,11 +1116,20 @@ static struct win_syscall { { "OutputDebugStringW", (SYSCALL)0, 0 }, #endif -#define osOutputDebugStringW ((VOID(WINAPI*)(LPCWSTR))aSyscall[66].pCurrent) +#define osOutputDebugStringW ((VOID(WINAPI*)(LPCWSTR))aSyscall[73].pCurrent) { "GetProcessHeap", (SYSCALL)GetProcessHeap, 0 }, -#define osGetProcessHeap ((HANDLE(WINAPI*)(VOID))aSyscall[67].pCurrent) +#define osGetProcessHeap ((HANDLE(WINAPI*)(VOID))aSyscall[74].pCurrent) + +#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) + { "CreateFileMappingFromApp", (SYSCALL)CreateFileMappingFromApp, 0 }, +#else + { "CreateFileMappingFromApp", (SYSCALL)0, 0 }, +#endif + +#define osCreateFileMappingFromApp ((HANDLE(WINAPI*)(HANDLE, \ + LPSECURITY_ATTRIBUTES,ULONG,ULONG64,LPCWSTR))aSyscall[75].pCurrent) /* ** NOTE: On some sub-platforms, the InterlockedCompareExchange "function" @@ -1020,25 +1144,25 @@ static struct win_syscall { { "InterlockedCompareExchange", (SYSCALL)InterlockedCompareExchange, 0 }, #define osInterlockedCompareExchange ((LONG(WINAPI*)(LONG \ - SQLITE_WIN32_VOLATILE*, LONG,LONG))aSyscall[68].pCurrent) + SQLITE_WIN32_VOLATILE*, LONG,LONG))aSyscall[76].pCurrent) #endif /* defined(InterlockedCompareExchange) */ -#if !SQLITE_OS_WINCE && SQLITE_WIN32_USE_UUID +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID { "UuidCreate", (SYSCALL)UuidCreate, 0 }, #else { "UuidCreate", (SYSCALL)0, 0 }, #endif -#define osUuidCreate ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[69].pCurrent) +#define osUuidCreate ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[77].pCurrent) -#if !SQLITE_OS_WINCE && SQLITE_WIN32_USE_UUID +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID { "UuidCreateSequential", (SYSCALL)UuidCreateSequential, 0 }, #else { "UuidCreateSequential", (SYSCALL)0, 0 }, #endif #define osUuidCreateSequential \ - ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[70].pCurrent) + ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[78].pCurrent) #if !defined(SQLITE_NO_SYNC) && SQLITE_MAX_MMAP_SIZE>0 { "FlushViewOfFile", (SYSCALL)FlushViewOfFile, 0 }, @@ -1047,7 +1171,7 @@ static struct win_syscall { #endif #define osFlushViewOfFile \ - ((BOOL(WINAPI*)(LPCVOID,SIZE_T))aSyscall[71].pCurrent) + ((BOOL(WINAPI*)(LPCVOID,SIZE_T))aSyscall[79].pCurrent) /* ** If SQLITE_ENABLE_SETLK_TIMEOUT is defined, we require CreateEvent() @@ -1064,7 +1188,7 @@ static struct win_syscall { #define osCreateEvent ( \ (HANDLE(WINAPI*) (LPSECURITY_ATTRIBUTES,BOOL,BOOL,LPCSTR)) \ - aSyscall[72].pCurrent \ + aSyscall[80].pCurrent \ ) /* @@ -1081,7 +1205,7 @@ static struct win_syscall { { "CancelIo", (SYSCALL)0, 0 }, #endif -#define osCancelIo ((BOOL(WINAPI*)(HANDLE))aSyscall[73].pCurrent) +#define osCancelIo ((BOOL(WINAPI*)(HANDLE))aSyscall[81].pCurrent) #if defined(SQLITE_WIN32_HAS_WIDE) && defined(_WIN32) { "GetModuleHandleW", (SYSCALL)GetModuleHandleW, 0 }, @@ -1089,7 +1213,7 @@ static struct win_syscall { { "GetModuleHandleW", (SYSCALL)0, 0 }, #endif -#define osGetModuleHandleW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[74].pCurrent) +#define osGetModuleHandleW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[82].pCurrent) #ifndef _WIN32 { "getenv", (SYSCALL)getenv, 0 }, @@ -1097,7 +1221,7 @@ static struct win_syscall { { "getenv", (SYSCALL)0, 0 }, #endif -#define osGetenv ((const char *(*)(const char *))aSyscall[75].pCurrent) +#define osGetenv ((const char *(*)(const char *))aSyscall[83].pCurrent) #ifndef _WIN32 { "getcwd", (SYSCALL)getcwd, 0 }, @@ -1105,7 +1229,7 @@ static struct win_syscall { { "getcwd", (SYSCALL)0, 0 }, #endif -#define osGetcwd ((char*(*)(char*,size_t))aSyscall[76].pCurrent) +#define osGetcwd ((char*(*)(char*,size_t))aSyscall[84].pCurrent) #ifndef _WIN32 { "readlink", (SYSCALL)readlink, 0 }, @@ -1113,7 +1237,7 @@ static struct win_syscall { { "readlink", (SYSCALL)0, 0 }, #endif -#define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[77].pCurrent) +#define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[85].pCurrent) #ifndef _WIN32 { "lstat", (SYSCALL)lstat, 0 }, @@ -1121,7 +1245,7 @@ static struct win_syscall { { "lstat", (SYSCALL)0, 0 }, #endif -#define osLstat ((int(*)(const char*,struct stat*))aSyscall[78].pCurrent) +#define osLstat ((int(*)(const char*,struct stat*))aSyscall[86].pCurrent) #ifndef _WIN32 { "__errno", (SYSCALL)__errno, 0 }, @@ -1129,7 +1253,7 @@ static struct win_syscall { { "__errno", (SYSCALL)0, 0 }, #endif -#define osErrno (*((int*(*)(void))aSyscall[79].pCurrent)()) +#define osErrno (*((int*(*)(void))aSyscall[87].pCurrent)()) #ifndef _WIN32 { "cygwin_conv_path", (SYSCALL)cygwin_conv_path, 0 }, @@ -1138,7 +1262,7 @@ static struct win_syscall { #endif #define osCygwin_conv_path ((size_t(*)(unsigned int, \ - const void *, void *, size_t))aSyscall[80].pCurrent) + const void *, void *, size_t))aSyscall[88].pCurrent) }; /* End of the overrideable system calls */ @@ -1242,10 +1366,10 @@ int sqlite3_win32_compact_heap(LPUINT pnLargest){ hHeap = winMemGetHeap(); assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); -#if defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); #endif -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT if( (nLargest=osHeapCompact(hHeap, SQLITE_WIN32_HEAP_FLAGS))==0 ){ DWORD lastErrno = osGetLastError(); if( lastErrno==NO_ERROR ){ @@ -1358,11 +1482,28 @@ void sqlite3_win32_write_debug(const char *zBuf, int nBuf){ } #endif /* _WIN32 */ +/* +** The following routine suspends the current thread for at least ms +** milliseconds. This is equivalent to the Win32 Sleep() interface. +*/ +#if SQLITE_OS_WINRT +static HANDLE sleepObj = NULL; +#endif + void sqlite3_win32_sleep(DWORD milliseconds){ +#if SQLITE_OS_WINRT + if ( sleepObj==NULL ){ + sleepObj = osCreateEventExW(NULL, NULL, CREATE_EVENT_MANUAL_RESET, + SYNCHRONIZE); + } + assert( sleepObj!=NULL ); + osWaitForSingleObjectEx(sleepObj, milliseconds, FALSE); +#else osSleep(milliseconds); +#endif } -#if SQLITE_MAX_WORKER_THREADS>0 && !SQLITE_OS_WINCE && \ +#if SQLITE_MAX_WORKER_THREADS>0 && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && \ SQLITE_THREADSAFE>0 DWORD sqlite3Win32Wait(HANDLE hObject){ DWORD rc; @@ -1386,7 +1527,7 @@ DWORD sqlite3Win32Wait(HANDLE hObject){ #if !SQLITE_WIN32_GETVERSIONEX # define osIsNT() (1) -#elif SQLITE_OS_WINCE || !defined(SQLITE_WIN32_HAS_ANSI) +#elif SQLITE_OS_WINCE || SQLITE_OS_WINRT || !defined(SQLITE_WIN32_HAS_ANSI) # define osIsNT() (1) #elif !defined(SQLITE_WIN32_HAS_WIDE) # define osIsNT() (0) @@ -1399,7 +1540,13 @@ DWORD sqlite3Win32Wait(HANDLE hObject){ ** based on the NT kernel. */ int sqlite3_win32_is_nt(void){ -#if SQLITE_WIN32_GETVERSIONEX +#if SQLITE_OS_WINRT + /* + ** NOTE: The WinRT sub-platform is always assumed to be based on the NT + ** kernel. + */ + return 1; +#elif SQLITE_WIN32_GETVERSIONEX if( osInterlockedCompareExchange(&sqlite3_os_type, 0, 0)==0 ){ #if defined(SQLITE_WIN32_HAS_ANSI) OSVERSIONINFOA sInfo; @@ -1441,7 +1588,7 @@ static void *winMemMalloc(int nBytes){ hHeap = winMemGetHeap(); assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); -#if defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); #endif assert( nBytes>=0 ); @@ -1463,7 +1610,7 @@ static void winMemFree(void *pPrior){ hHeap = winMemGetHeap(); assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); -#if defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ); #endif if( !pPrior ) return; /* Passing NULL to HeapFree is undefined. */ @@ -1484,7 +1631,7 @@ static void *winMemRealloc(void *pPrior, int nBytes){ hHeap = winMemGetHeap(); assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); -#if defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ); #endif assert( nBytes>=0 ); @@ -1512,7 +1659,7 @@ static int winMemSize(void *p){ hHeap = winMemGetHeap(); assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); -#if defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, p) ); #endif if( !p ) return 0; @@ -1542,7 +1689,7 @@ static int winMemInit(void *pAppData){ assert( pWinMemData->magic1==WINMEM_MAGIC1 ); assert( pWinMemData->magic2==WINMEM_MAGIC2 ); -#if SQLITE_WIN32_HEAP_CREATE +#if !SQLITE_OS_WINRT && SQLITE_WIN32_HEAP_CREATE if( !pWinMemData->hHeap ){ DWORD dwInitialSize = SQLITE_WIN32_HEAP_INIT_SIZE; DWORD dwMaximumSize = (DWORD)sqlite3GlobalConfig.nHeap; @@ -1575,7 +1722,7 @@ static int winMemInit(void *pAppData){ #endif assert( pWinMemData->hHeap!=0 ); assert( pWinMemData->hHeap!=INVALID_HANDLE_VALUE ); -#if defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(pWinMemData->hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); #endif return SQLITE_OK; @@ -1593,7 +1740,7 @@ static void winMemShutdown(void *pAppData){ if( pWinMemData->hHeap ){ assert( pWinMemData->hHeap!=INVALID_HANDLE_VALUE ); -#if defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(pWinMemData->hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); #endif if( pWinMemData->bOwned ){ @@ -1974,6 +2121,17 @@ static int winGetLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ char *zOut = 0; if( osIsNT() ){ +#if SQLITE_OS_WINRT + WCHAR zTempWide[SQLITE_WIN32_MAX_ERRMSG_CHARS+1]; + dwLen = osFormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + lastErrno, + 0, + zTempWide, + SQLITE_WIN32_MAX_ERRMSG_CHARS, + 0); +#else LPWSTR zTempWide = NULL; dwLen = osFormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | @@ -1984,13 +2142,16 @@ static int winGetLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ (LPWSTR) &zTempWide, 0, 0); +#endif if( dwLen > 0 ){ /* allocate a buffer and convert to UTF8 */ sqlite3BeginBenignMalloc(); zOut = winUnicodeToUtf8(zTempWide); sqlite3EndBenignMalloc(); +#if !SQLITE_OS_WINRT /* free the system buffer allocated by FormatMessage */ osLocalFree(zTempWide); +#endif } } #ifdef SQLITE_WIN32_HAS_ANSI @@ -2651,6 +2812,7 @@ static int winHandleUnlock(HANDLE h, int iOff, int nByte){ static int winHandleSeek(HANDLE h, sqlite3_int64 iOffset){ int rc = SQLITE_OK; /* Return value */ +#if !SQLITE_OS_WINRT LONG upperBits; /* Most sig. 32 bits of new offset */ LONG lowerBits; /* Least sig. 32 bits of new offset */ DWORD dwRet; /* Value returned by SetFilePointer() */ @@ -2672,7 +2834,20 @@ static int winHandleSeek(HANDLE h, sqlite3_int64 iOffset){ rc = SQLITE_IOERR_SEEK; } } - OSTRACE(("SEEK file=%p, offset=%lld rc=%s\n", h, iOffset,sqlite3ErrName(rc))); +#else + /* This implementation works for WinRT. */ + LARGE_INTEGER x; /* The new offset */ + BOOL bRet; /* Value returned by SetFilePointerEx() */ + + x.QuadPart = iOffset; + bRet = osSetFilePointerEx(h, x, 0, FILE_BEGIN); + + if(!bRet){ + rc = SQLITE_IOERR_SEEK; + } +#endif + + OSTRACE(("SEEK file=%p, offset=%lld rc=%s\n", h, iOffset, sqlite3ErrName(rc))); return rc; } @@ -2973,6 +3148,17 @@ static int winHandleTruncate(HANDLE h, sqlite3_int64 nByte){ */ static int winHandleSize(HANDLE h, sqlite3_int64 *pnByte){ int rc = SQLITE_OK; + +#if SQLITE_OS_WINRT + FILE_STANDARD_INFO info; + BOOL b; + b = osGetFileInformationByHandleEx(h, FileStandardInfo, &info, sizeof(info)); + if( b ){ + *pnByte = info.EndOfFile.QuadPart; + }else{ + rc = SQLITE_IOERR_FSTAT; + } +#else DWORD upperBits = 0; DWORD lowerBits = 0; @@ -2982,6 +3168,8 @@ static int winHandleSize(HANDLE h, sqlite3_int64 *pnByte){ if( lowerBits==INVALID_FILE_SIZE && osGetLastError()!=NO_ERROR ){ rc = SQLITE_IOERR_FSTAT; } +#endif + return rc; } @@ -3180,6 +3368,20 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ assert( pSize!=0 ); SimulateIOError(return SQLITE_IOERR_FSTAT); OSTRACE(("SIZE file=%p, pSize=%p\n", pFile->h, pSize)); + +#if SQLITE_OS_WINRT + { + FILE_STANDARD_INFO info; + if( osGetFileInformationByHandleEx(pFile->h, FileStandardInfo, + &info, sizeof(info)) ){ + *pSize = info.EndOfFile.QuadPart; + }else{ + pFile->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_FSTAT, pFile->lastErrno, + "winFileSize", pFile->zPath); + } + } +#else { DWORD upperBits; DWORD lowerBits; @@ -3194,6 +3396,7 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ "winFileSize", pFile->zPath); } } +#endif OSTRACE(("SIZE file=%p, pSize=%p, *pSize=%lld, rc=%s\n", pFile->h, pSize, *pSize, sqlite3ErrName(rc))); return rc; @@ -4155,6 +4358,20 @@ static int winHandleOpen( ** TODO: retry-on-ioerr. */ if( osIsNT() ){ +#if SQLITE_OS_WINRT + CREATEFILE2_EXTENDED_PARAMETERS extendedParameters; + memset(&extendedParameters, 0, sizeof(extendedParameters)); + extendedParameters.dwSize = sizeof(extendedParameters); + extendedParameters.dwFileAttributes = FILE_ATTRIBUTE_NORMAL; + extendedParameters.dwFileFlags = flag_overlapped; + extendedParameters.dwSecurityQosFlags = SECURITY_ANONYMOUS; + h = osCreateFile2((LPCWSTR)zConverted, + (GENERIC_READ | (bReadonly ? 0 : GENERIC_WRITE)),/* dwDesiredAccess */ + FILE_SHARE_READ | FILE_SHARE_WRITE, /* dwShareMode */ + OPEN_ALWAYS, /* dwCreationDisposition */ + &extendedParameters + ); +#else h = osCreateFileW((LPCWSTR)zConverted, /* lpFileName */ (GENERIC_READ | (bReadonly ? 0 : GENERIC_WRITE)), /* dwDesiredAccess */ FILE_SHARE_READ | FILE_SHARE_WRITE, /* dwShareMode */ @@ -4163,6 +4380,7 @@ static int winHandleOpen( FILE_ATTRIBUTE_NORMAL|flag_overlapped, NULL ); +#endif }else{ /* Due to pre-processor directives earlier in this file, ** SQLITE_WIN32_HAS_ANSI is always defined if osIsNT() is false. */ @@ -4630,7 +4848,9 @@ static int winShmMap( HANDLE hMap = NULL; /* file-mapping handle */ void *pMap = 0; /* Mapped memory region */ -#if defined(SQLITE_WIN32_HAS_WIDE) +#if SQLITE_OS_WINRT + hMap = osCreateFileMappingFromApp(hShared, NULL, protect, nByte, NULL); +#elif defined(SQLITE_WIN32_HAS_WIDE) hMap = osCreateFileMappingW(hShared, NULL, protect, 0, nByte, NULL); #elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA hMap = osCreateFileMappingA(hShared, NULL, protect, 0, nByte, NULL); @@ -4642,9 +4862,15 @@ static int winShmMap( if( hMap ){ int iOffset = pShmNode->nRegion*szRegion; int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; +#if SQLITE_OS_WINRT + pMap = osMapViewOfFileFromApp(hMap, flags, + iOffset - iOffsetShift, szRegion + iOffsetShift + ); +#else pMap = osMapViewOfFile(hMap, flags, 0, iOffset - iOffsetShift, szRegion + iOffsetShift ); +#endif OSTRACE(("SHM-MAP-MAP pid=%lu, region=%d, offset=%d, size=%d, rc=%s\n", osGetCurrentProcessId(), pShmNode->nRegion, iOffset, szRegion, pMap ? "ok" : "failed")); @@ -4777,7 +5003,9 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ flags |= FILE_MAP_WRITE; } #endif -#if defined(SQLITE_WIN32_HAS_WIDE) +#if SQLITE_OS_WINRT + pFd->hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); +#elif defined(SQLITE_WIN32_HAS_WIDE) pFd->hMap = osCreateFileMappingW(pFd->h, NULL, protect, (DWORD)((nMap>>32) & 0xffffffff), (DWORD)(nMap & 0xffffffff), NULL); @@ -4797,7 +5025,11 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ } assert( (nMap % winSysInfo.dwPageSize)==0 ); assert( sizeof(SIZE_T)==sizeof(sqlite3_int64) || nMap<=0xffffffff ); +#if SQLITE_OS_WINRT + pNew = osMapViewOfFileFromApp(pFd->hMap, flags, 0, (SIZE_T)nMap); +#else pNew = osMapViewOfFile(pFd->hMap, flags, 0, 0, (SIZE_T)nMap); +#endif if( pNew==NULL ){ osCloseHandle(pFd->hMap); pFd->hMap = NULL; @@ -5132,6 +5364,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ } #endif +#if !SQLITE_OS_WINRT && defined(_WIN32) else if( osIsNT() ){ char *zMulti; LPWSTR zWidePath = sqlite3MallocZero( nMax*sizeof(WCHAR) ); @@ -5185,6 +5418,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ } } #endif /* SQLITE_WIN32_HAS_ANSI */ +#endif /* !SQLITE_OS_WINRT */ /* ** Check to make sure the temporary directory ends with an appropriate @@ -5359,6 +5593,13 @@ static int winOpen( memset(pFile, 0, sizeof(winFile)); pFile->h = INVALID_HANDLE_VALUE; +#if SQLITE_OS_WINRT + if( !zUtf8Name && !sqlite3_temp_directory ){ + sqlite3_log(SQLITE_ERROR, + "sqlite3_temp_directory variable should be set for WinRT"); + } +#endif + /* If the second argument to this function is NULL, generate a ** temporary file name to use */ @@ -5441,6 +5682,31 @@ static int winOpen( #endif if( osIsNT() ){ +#if SQLITE_OS_WINRT + CREATEFILE2_EXTENDED_PARAMETERS extendedParameters; + extendedParameters.dwSize = sizeof(CREATEFILE2_EXTENDED_PARAMETERS); + extendedParameters.dwFileAttributes = + dwFlagsAndAttributes & FILE_ATTRIBUTE_MASK; + extendedParameters.dwFileFlags = dwFlagsAndAttributes & FILE_FLAG_MASK; + extendedParameters.dwSecurityQosFlags = SECURITY_ANONYMOUS; + extendedParameters.lpSecurityAttributes = NULL; + extendedParameters.hTemplateFile = NULL; + do{ + h = osCreateFile2((LPCWSTR)zConverted, + dwDesiredAccess, + dwShareMode, + dwCreationDisposition, + &extendedParameters); + if( h!=INVALID_HANDLE_VALUE ) break; + if( isReadWrite ){ + int rc2; + sqlite3BeginBenignMalloc(); + rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ|NORETRY, &isRO); + sqlite3EndBenignMalloc(); + if( rc2==SQLITE_OK && isRO ) break; + } + }while( winRetryIoerr(&cnt, &lastErrno) ); +#else do{ h = osCreateFileW((LPCWSTR)zConverted, dwDesiredAccess, @@ -5457,6 +5723,7 @@ static int winOpen( if( rc2==SQLITE_OK && isRO ) break; } }while( winRetryIoerr(&cnt, &lastErrno) ); +#endif } #ifdef SQLITE_WIN32_HAS_ANSI else{ @@ -5593,7 +5860,25 @@ static int winDelete( } if( osIsNT() ){ do { +#if SQLITE_OS_WINRT + WIN32_FILE_ATTRIBUTE_DATA sAttrData; + memset(&sAttrData, 0, sizeof(sAttrData)); + if ( osGetFileAttributesExW(zConverted, GetFileExInfoStandard, + &sAttrData) ){ + attr = sAttrData.dwFileAttributes; + }else{ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND + || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } + break; + } +#else attr = osGetFileAttributesW(zConverted); +#endif if ( attr==INVALID_FILE_ATTRIBUTES ){ lastErrno = osGetLastError(); if( lastErrno==ERROR_FILE_NOT_FOUND @@ -5716,7 +6001,6 @@ static int winAccess( attr = sAttrData.dwFileAttributes; } }else{ - if( noRetry ) lastErrno = osGetLastError(); winLogIoerr(cnt, __LINE__); if( lastErrno!=ERROR_FILE_NOT_FOUND && lastErrno!=ERROR_PATH_NOT_FOUND ){ sqlite3_free(zConverted); @@ -5885,7 +6169,7 @@ static int winFullPathnameNoMutex( int nFull, /* Size of output buffer in bytes */ char *zFull /* Output buffer */ ){ -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT int nByte; void *zConverted; char *zOut; @@ -5974,7 +6258,7 @@ static int winFullPathnameNoMutex( } #endif /* __CYGWIN__ */ -#if SQLITE_OS_WINCE && defined(_WIN32) +#if (SQLITE_OS_WINCE || SQLITE_OS_WINRT) && defined(_WIN32) SimulateIOError( return SQLITE_ERROR ); /* WinCE has no concept of a relative pathname, or so I am told. */ /* WinRT has no way to convert a relative path to an absolute one. */ @@ -5993,7 +6277,7 @@ static int winFullPathnameNoMutex( return SQLITE_OK; #endif -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT #if defined(_WIN32) /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this @@ -6125,7 +6409,11 @@ static void *winDlOpen(sqlite3_vfs *pVfs, const char *zFilename){ return 0; } if( osIsNT() ){ +#if SQLITE_OS_WINRT + h = osLoadPackagedLibrary((LPCWSTR)zConverted, 0); +#else h = osLoadLibraryW((LPCWSTR)zConverted); +#endif } #ifdef SQLITE_WIN32_HAS_ANSI else{ @@ -6207,16 +6495,23 @@ static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ DWORD pid = osGetCurrentProcessId(); xorMemory(&e, (unsigned char*)&pid, sizeof(DWORD)); } +#if SQLITE_OS_WINRT + { + ULONGLONG cnt = osGetTickCount64(); + xorMemory(&e, (unsigned char*)&cnt, sizeof(ULONGLONG)); + } +#else { DWORD cnt = osGetTickCount(); xorMemory(&e, (unsigned char*)&cnt, sizeof(DWORD)); } +#endif /* SQLITE_OS_WINRT */ { LARGE_INTEGER i; osQueryPerformanceCounter(&i); xorMemory(&e, (unsigned char*)&i, sizeof(LARGE_INTEGER)); } -#if !SQLITE_OS_WINCE && SQLITE_WIN32_USE_UUID +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID { UUID id; memset(&id, 0, sizeof(UUID)); @@ -6226,7 +6521,7 @@ static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ osUuidCreateSequential(&id); xorMemory(&e, (unsigned char*)&id, sizeof(UUID)); } -#endif /* !SQLITE_OS_WINCE && SQLITE_WIN32_USE_UUID */ +#endif /* !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID */ return e.nXor>nBuf ? nBuf : e.nXor; #endif /* defined(SQLITE_TEST) || defined(SQLITE_OMIT_RANDOMNESS) */ } @@ -6457,16 +6752,15 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==81 ); - assert( strcmp(aSyscall[0].zName,"AreFileApisANSI")==0 ); - assert( strcmp(aSyscall[20].zName,"GetFileAttributesA")==0 ); - assert( strcmp(aSyscall[40].zName,"HeapReAlloc")==0 ); - assert( strcmp(aSyscall[60].zName,"WideCharToMultiByte")==0 ); - assert( strcmp(aSyscall[80].zName,"cygwin_conv_path")==0 ); + assert( ArraySize(aSyscall)==89 ); /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); +#if SQLITE_OS_WINRT + osGetNativeSystemInfo(&winSysInfo); +#else osGetSystemInfo(&winSysInfo); +#endif assert( winSysInfo.dwAllocationGranularity>0 ); assert( winSysInfo.dwPageSize>0 ); @@ -6490,9 +6784,17 @@ int sqlite3_os_init(void){ } int sqlite3_os_end(void){ +#if SQLITE_OS_WINRT + if( sleepObj!=NULL ){ + osCloseHandle(sleepObj); + sleepObj = NULL; + } +#endif + #ifndef SQLITE_OMIT_WAL winBigLock = 0; #endif + return SQLITE_OK; } diff --git a/src/os_win.h b/src/os_win.h index 696486c19..a0845f003 100644 --- a/src/os_win.h +++ b/src/os_win.h @@ -58,6 +58,14 @@ # define SQLITE_OS_WINCE 0 #endif +/* +** Determine if we are dealing with WinRT, which provides only a subset of +** the full Win32 API. +*/ +#if !defined(SQLITE_OS_WINRT) +# define SQLITE_OS_WINRT 0 +#endif + /* ** For WinCE, some API function parameters do not appear to be declared as ** volatile. @@ -72,7 +80,7 @@ ** For some Windows sub-platforms, the _beginthreadex() / _endthreadex() ** functions are not available (e.g. those not using MSVC, Cygwin, etc). */ -#if SQLITE_OS_WIN && !SQLITE_OS_WINCE && \ +#if SQLITE_OS_WIN && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && \ SQLITE_THREADSAFE>0 && !defined(__CYGWIN__) # define SQLITE_OS_WIN_THREADS 1 #else diff --git a/src/pager.c b/src/pager.c index 61b391d6b..cbaef186a 100644 --- a/src/pager.c +++ b/src/pager.c @@ -813,8 +813,6 @@ int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){ (void)sqlite3WalFindFrame(pPager->pWal, pgno, &iRead); if( iRead ) return 0; /* Case (4) */ } -#else - UNUSED_PARAMETER(pgno); #endif assert( pPager->fd->pMethods->xDeviceCharacteristics!=0 ); if( (pPager->fd->pMethods->xDeviceCharacteristics(pPager->fd) @@ -1235,17 +1233,17 @@ static int jrnlBufferSize(Pager *pPager){ */ #ifdef SQLITE_CHECK_PAGES /* -** Return a 64-bit hash of the page data for pPage. +** Return a 32-bit hash of the page data for pPage. */ -static u64 pager_datahash(int nByte, unsigned char *pData){ - u64 hash = 0; +static u32 pager_datahash(int nByte, unsigned char *pData){ + u32 hash = 0; int i; for(i=0; i<nByte; i++){ hash = (hash*1039) + pData[i]; } return hash; } -static u64 pager_pagehash(PgHdr *pPage){ +static u32 pager_pagehash(PgHdr *pPage){ return pager_datahash(pPage->pPager->pageSize, (unsigned char *)pPage->pData); } static void pager_set_pagehash(PgHdr *pPage){ @@ -4194,8 +4192,6 @@ int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize,a); pPager->pWal = 0; } -#else - UNUSED_PARAMETER(db); #endif pager_reset(pPager); if( MEMDB ){ diff --git a/src/parse.y b/src/parse.y index f5a6bed14..617eb7303 100644 --- a/src/parse.y +++ b/src/parse.y @@ -21,11 +21,9 @@ */ } -// Setup for the parser stack -%stack_size 50 // Initial stack size -%stack_size_limit parserStackSizeLimit // Function returning max stack size -%realloc parserStackRealloc // realloc() for the stack -%free parserStackFree // free() for the stack +// Function used to enlarge the parser stack, if needed +%realloc parserStackRealloc +%free sqlite3_free // All token codes are small integers with #defines that begin with "TK_" %token_prefix TK_ @@ -51,7 +49,7 @@ } } %stack_overflow { - if( pParse->nErr==0 ) sqlite3ErrorMsg(pParse, "Recursion limit"); + sqlite3OomFault(pParse->db); } // The name of the generated procedure that implements the parser @@ -585,23 +583,8 @@ cmd ::= select(X). { ** sqlite3_realloc() that includes a call to sqlite3FaultSim() to facilitate ** testing. */ - static void *parserStackRealloc( - void *pOld, /* Prior allocation */ - sqlite3_uint64 newSize, /* Requested new alloation size */ - Parse *pParse /* Parsing context */ - ){ - void *p = sqlite3FaultSim(700) ? 0 : sqlite3_realloc(pOld, newSize); - if( p==0 ) sqlite3OomFault(pParse->db); - return p; - } - static void parserStackFree(void *pOld, Parse *pParse){ - (void)pParse; - sqlite3_free(pOld); - } - - /* Return an integer that is the maximum allowed stack size */ - static int parserStackSizeLimit(Parse *pParse){ - return pParse->db->aLimit[SQLITE_LIMIT_PARSER_DEPTH]; + static void *parserStackRealloc(void *pOld, sqlite3_uint64 newSize){ + return sqlite3FaultSim(700) ? 0 : sqlite3_realloc(pOld, newSize); } } @@ -833,36 +816,19 @@ fullname(A) ::= nm(X) DOT nm(Y). { %type xfullname {SrcList*} %destructor xfullname {sqlite3SrcListDelete(pParse->db, $$);} -xfullname(A) ::= nm(X). { - A = sqlite3SrcListAppend(pParse,0,&X,0); - if( IN_RENAME_OBJECT && A ) sqlite3RenameTokenMap(pParse, A->a[0].zName, &X); +xfullname(A) ::= nm(X). + {A = sqlite3SrcListAppend(pParse,0,&X,0); /*A-overwrites-X*/} +xfullname(A) ::= nm(X) DOT nm(Y). + {A = sqlite3SrcListAppend(pParse,0,&X,&Y); /*A-overwrites-X*/} +xfullname(A) ::= nm(X) DOT nm(Y) AS nm(Z). { + A = sqlite3SrcListAppend(pParse,0,&X,&Y); /*A-overwrites-X*/ + if( A ) A->a[0].zAlias = sqlite3NameFromToken(pParse->db, &Z); } -xfullname(A) ::= nm(X) DOT nm(Y). { - A = sqlite3SrcListAppend(pParse,0,&X,&Y); - if( IN_RENAME_OBJECT && A ) sqlite3RenameTokenMap(pParse, A->a[0].zName, &Y); -} -xfullname(A) ::= nm(X) AS nm(Z). { - A = sqlite3SrcListAppend(pParse,0,&X,0); - if( A ){ - if( IN_RENAME_OBJECT ){ - sqlite3RenameTokenMap(pParse, A->a[0].zName, &X); - }else{ - A->a[0].zAlias = sqlite3NameFromToken(pParse->db, &Z); - } - } -} -xfullname(A) ::= nm(X) DOT nm(Y) AS nm(Z). { - A = sqlite3SrcListAppend(pParse,0,&X,&Y); - if( A ){ - if( IN_RENAME_OBJECT ){ - sqlite3RenameTokenMap(pParse, A->a[0].zName, &Y); - }else{ - A->a[0].zAlias = sqlite3NameFromToken(pParse->db, &Z); - } - } +xfullname(A) ::= nm(X) AS nm(Z). { + A = sqlite3SrcListAppend(pParse,0,&X,0); /*A-overwrites-X*/ + if( A ) A->a[0].zAlias = sqlite3NameFromToken(pParse->db, &Z); } - %type joinop {int} joinop(X) ::= COMMA|JOIN. { X = JT_INNER; } joinop(X) ::= JOIN_KW(A) JOIN. @@ -1193,12 +1159,7 @@ expr(A) ::= nm(X) DOT nm(Y) DOT nm(Z). { term(A) ::= NULL|FLOAT|BLOB(X). {A=tokenExpr(pParse,@X,X); /*A-overwrites-X*/} term(A) ::= STRING(X). {A=tokenExpr(pParse,@X,X); /*A-overwrites-X*/} term(A) ::= INTEGER(X). { - int iValue; - if( sqlite3GetInt32(X.z, &iValue)==0 ){ - A = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &X, 0); - }else{ - A = sqlite3ExprInt32(pParse->db, iValue); - } + A = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &X, 1); if( A ) A->w.iOfst = (int)(X.z - pParse->zTail); } expr(A) ::= VARIABLE(X). { @@ -1380,67 +1341,43 @@ expr(A) ::= expr(A) likeop(OP) expr(Y) ESCAPE expr(E). [LIKE_KW] { if( A ) A->flags |= EP_InfixFunc; } -%include { - /* Create a TK_ISNULL or TK_NOTNULL expression, perhaps optimized to - ** to TK_TRUEFALSE, if possible */ - static Expr *sqlite3PExprIsNull( - Parse *pParse, /* Parsing context */ - int op, /* TK_ISNULL or TK_NOTNULL */ - Expr *pLeft /* Operand */ - ){ - Expr *p = pLeft; - assert( op==TK_ISNULL || op==TK_NOTNULL ); - assert( pLeft!=0 ); - while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ - p = p->pLeft; - assert( p!=0 ); - } - switch( p->op ){ - case TK_INTEGER: - case TK_STRING: - case TK_FLOAT: - case TK_BLOB: - sqlite3ExprDeferredDelete(pParse, pLeft); - return sqlite3ExprInt32(pParse->db, op==TK_NOTNULL); - default: - break; - } - return sqlite3PExpr(pParse, op, pLeft, 0); - } +expr(A) ::= expr(A) ISNULL|NOTNULL(E). {A = sqlite3PExpr(pParse,@E,A,0);} +expr(A) ::= expr(A) NOT NULL. {A = sqlite3PExpr(pParse,TK_NOTNULL,A,0);} - /* Create a TK_IS or TK_ISNOT operator, perhaps optimized to - ** TK_ISNULL or TK_NOTNULL or TK_TRUEFALSE. */ - static Expr *sqlite3PExprIs( - Parse *pParse, /* Parsing context */ - int op, /* TK_IS or TK_ISNOT */ - Expr *pLeft, /* Left operand */ - Expr *pRight /* Right operand */ - ){ - if( pRight && pRight->op==TK_NULL ){ - sqlite3ExprDeferredDelete(pParse, pRight); - return sqlite3PExprIsNull(pParse, op==TK_IS ? TK_ISNULL : TK_NOTNULL, pLeft); +%include { + /* A routine to convert a binary TK_IS or TK_ISNOT expression into a + ** unary TK_ISNULL or TK_NOTNULL expression. */ + static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){ + sqlite3 *db = pParse->db; + if( pA && pY && pY->op==TK_NULL && !IN_RENAME_OBJECT ){ + pA->op = (u8)op; + sqlite3ExprDelete(db, pA->pRight); + pA->pRight = 0; } - return sqlite3PExpr(pParse, op, pLeft, pRight); } } -expr(A) ::= expr(A) ISNULL|NOTNULL(E). {A = sqlite3PExprIsNull(pParse,@E,A);} -expr(A) ::= expr(A) NOT NULL. {A = sqlite3PExprIsNull(pParse,TK_NOTNULL,A);} - -// expr1 IS expr2 same as expr1 IS NOT DISTINCT FROM expr2 -// expr1 IS NOT expr2 same as expr1 IS DISTINCT FROM expr2 +// expr1 IS expr2 +// expr1 IS NOT expr2 // +// If expr2 is NULL then code as TK_ISNULL or TK_NOTNULL. If expr2 +// is any other expression, code as TK_IS or TK_ISNOT. +// expr(A) ::= expr(A) IS expr(Y). { - A = sqlite3PExprIs(pParse, TK_IS, A, Y); + A = sqlite3PExpr(pParse,TK_IS,A,Y); + binaryToUnaryIfNull(pParse, Y, A, TK_ISNULL); } expr(A) ::= expr(A) IS NOT expr(Y). { - A = sqlite3PExprIs(pParse, TK_ISNOT, A, Y); + A = sqlite3PExpr(pParse,TK_ISNOT,A,Y); + binaryToUnaryIfNull(pParse, Y, A, TK_NOTNULL); } expr(A) ::= expr(A) IS NOT DISTINCT FROM expr(Y). { - A = sqlite3PExprIs(pParse, TK_IS, A, Y); + A = sqlite3PExpr(pParse,TK_IS,A,Y); + binaryToUnaryIfNull(pParse, Y, A, TK_ISNULL); } expr(A) ::= expr(A) IS DISTINCT FROM expr(Y). { - A = sqlite3PExprIs(pParse, TK_ISNOT, A, Y); + A = sqlite3PExpr(pParse,TK_ISNOT,A,Y); + binaryToUnaryIfNull(pParse, Y, A, TK_NOTNULL); } expr(A) ::= NOT(B) expr(X). @@ -1770,13 +1707,28 @@ when_clause(A) ::= WHEN expr(X). { A = X; } %type trigger_cmd_list {TriggerStep*} %destructor trigger_cmd_list {sqlite3DeleteTriggerStep(pParse->db, $$);} trigger_cmd_list(A) ::= trigger_cmd_list(A) trigger_cmd(X) SEMI. { + assert( A!=0 ); A->pLast->pNext = X; A->pLast = X; } trigger_cmd_list(A) ::= trigger_cmd(A) SEMI. { + assert( A!=0 ); A->pLast = A; } +// Disallow qualified table names on INSERT, UPDATE, and DELETE statements +// within a trigger. The table to INSERT, UPDATE, or DELETE is always in +// the same database as the table that the trigger fires on. +// +%type trnm {Token} +trnm(A) ::= nm(A). +trnm(A) ::= nm DOT nm(X). { + A = X; + sqlite3ErrorMsg(pParse, + "qualified table names are not allowed on INSERT, UPDATE, and DELETE " + "statements within triggers"); +} + // Disallow the INDEX BY and NOT INDEXED clauses on UPDATE and DELETE // statements within triggers. We make a specific error message for this // since it is an exception to the default grammar rules. @@ -1799,17 +1751,17 @@ tridxby ::= NOT INDEXED. { %destructor trigger_cmd {sqlite3DeleteTriggerStep(pParse->db, $$);} // UPDATE trigger_cmd(A) ::= - UPDATE(B) orconf(R) xfullname(X) tridxby SET setlist(Y) from(F) where_opt(Z) scanpt(E). - {A = sqlite3TriggerUpdateStep(pParse, X, F, Y, Z, R, B.z, E);} + UPDATE(B) orconf(R) trnm(X) tridxby SET setlist(Y) from(F) where_opt(Z) scanpt(E). + {A = sqlite3TriggerUpdateStep(pParse, &X, F, Y, Z, R, B.z, E);} // INSERT trigger_cmd(A) ::= scanpt(B) insert_cmd(R) INTO - xfullname(X) idlist_opt(F) select(S) upsert(U) scanpt(Z). { - A = sqlite3TriggerInsertStep(pParse,X,F,S,R,U,B,Z);/*A-overwrites-R*/ + trnm(X) idlist_opt(F) select(S) upsert(U) scanpt(Z). { + A = sqlite3TriggerInsertStep(pParse,&X,F,S,R,U,B,Z);/*A-overwrites-R*/ } // DELETE -trigger_cmd(A) ::= DELETE(B) FROM xfullname(X) tridxby where_opt(Y) scanpt(E). - {A = sqlite3TriggerDeleteStep(pParse, X, Y, B.z, E);} +trigger_cmd(A) ::= DELETE(B) FROM trnm(X) tridxby where_opt(Y) scanpt(E). + {A = sqlite3TriggerDeleteStep(pParse, &X, Y, B.z, E);} // SELECT trigger_cmd(A) ::= scanpt(B) select(X) scanpt(E). @@ -1879,42 +1831,22 @@ cmd ::= ANALYZE nm(X) dbnm(Y). {sqlite3Analyze(pParse, &X, &Y);} cmd ::= ALTER TABLE fullname(X) RENAME TO nm(Z). { sqlite3AlterRenameTable(pParse,X,&Z); } - -// The ALTER TABLE ADD COLUMN command. This is broken into two sections so -// that sqlite3AlterBeginAddColumn() is called before parsing the various -// constraints and so on (carglist) attached to the new column definition. -cmd ::= alter_add(Y) carglist. { +cmd ::= ALTER TABLE add_column_fullname + ADD kwcolumn_opt columnname(Y) carglist. { Y.n = (int)(pParse->sLastToken.z-Y.z) + pParse->sLastToken.n; sqlite3AlterFinishAddColumn(pParse, &Y); } -alter_add(A) ::= ALTER TABLE fullname(X) ADD kwcolumn_opt nm(Y) typetoken(Z). { - disableLookaside(pParse); - sqlite3AlterBeginAddColumn(pParse, X); - sqlite3AddColumn(pParse, Y, Z); - A = Y; -} - cmd ::= ALTER TABLE fullname(X) DROP kwcolumn_opt nm(Y). { sqlite3AlterDropColumn(pParse, X, &Y); } + +add_column_fullname ::= fullname(X). { + disableLookaside(pParse); + sqlite3AlterBeginAddColumn(pParse, X); +} cmd ::= ALTER TABLE fullname(X) RENAME kwcolumn_opt nm(Y) TO nm(Z). { sqlite3AlterRenameColumn(pParse, X, &Y, &Z); } -cmd ::= ALTER TABLE fullname(X) DROP CONSTRAINT nm(Y). { - sqlite3AlterDropConstraint(pParse, X, &Y, 0); -} -cmd ::= ALTER TABLE fullname(X) ALTER kwcolumn_opt nm(Y) DROP NOT NULL. { - sqlite3AlterDropConstraint(pParse, X, 0, &Y); -} -cmd ::= ALTER TABLE fullname(X) ALTER kwcolumn_opt nm(Y) SET NOT(Z) NULL onconf. { - sqlite3AlterSetNotNull(pParse, X, &Y, &Z); -} -cmd ::= ALTER TABLE fullname(X) ADD CONSTRAINT(Y) nm(Z) CHECK LP(A) expr RP(B) onconf. { - sqlite3AlterAddConstraint(pParse, X, &Y, &Z, A.z+1, (B.z-A.z-1)); -} -cmd ::= ALTER TABLE fullname(X) ADD CHECK(Y) LP(A) expr RP(B) onconf. { - sqlite3AlterAddConstraint(pParse, X, &Y, 0, A.z+1, (B.z-A.z-1)); -} kwcolumn_opt ::= . kwcolumn_opt ::= COLUMNKW. diff --git a/src/pcache.h b/src/pcache.h index dafb59390..f945dab1a 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -29,10 +29,10 @@ struct PgHdr { PCache *pCache; /* PRIVATE: Cache that owns this page */ PgHdr *pDirty; /* Transient list of dirty sorted by pgno */ Pager *pPager; /* The pager this page is part of */ + Pgno pgno; /* Page number for this page */ #ifdef SQLITE_CHECK_PAGES - u64 pageHash; /* Hash of page content */ + u32 pageHash; /* Hash of page content */ #endif - Pgno pgno; /* Page number for this page */ u16 flags; /* PGHDR flags defined below */ /********************************************************************** diff --git a/src/prepare.c b/src/prepare.c index be9e496f1..539360b74 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -33,8 +33,7 @@ static void corruptSchema( static const char *azAlterType[] = { "rename", "drop column", - "add column", - "drop constraint" + "add column" }; *pData->pzErrMsg = sqlite3MPrintf(db, "error in %s %s after %s: %s", azObj[0], azObj[1], diff --git a/src/printf.c b/src/printf.c index d9f3c229d..f75ed3b8a 100644 --- a/src/printf.c +++ b/src/printf.c @@ -549,7 +549,7 @@ void sqlite3_str_vappendf( }else{ iRound = precision+1; } - sqlite3FpDecode(&s, realvalue, iRound, flag_altform2 ? 20 : 16); + sqlite3FpDecode(&s, realvalue, iRound, flag_altform2 ? 26 : 16); if( s.isSpecial ){ if( s.isSpecial==2 ){ bufpt = flag_zeropad ? "null" : "NaN"; @@ -1228,14 +1228,6 @@ int sqlite3_str_length(sqlite3_str *p){ return p ? p->nChar : 0; } -/* Truncate the text of the string to be no more than N bytes. */ -void sqlite3_str_truncate(sqlite3_str *p, int N){ - if( p!=0 && N>=0 && (u32)N<p->nChar ){ - p->nChar = N; - p->zText[p->nChar] = 0; - } -} - /* Return the current value for p */ char *sqlite3_str_value(sqlite3_str *p){ if( p==0 || p->nChar==0 ) return 0; @@ -1256,17 +1248,6 @@ void sqlite3_str_reset(StrAccum *p){ p->zText = 0; } -/* -** Destroy a dynamically allocate sqlite3_str object and all -** of its content, all in one call. -*/ -void sqlite3_str_free(sqlite3_str *p){ - if( p ){ - sqlite3_str_reset(p); - sqlite3_free(p); - } -} - /* ** Initialize a string accumulator. ** diff --git a/src/resolve.c b/src/resolve.c index 5b6d6c9c5..c3ec56136 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -936,7 +936,7 @@ static int exprProbability(Expr *p){ double r = -1.0; if( p->op!=TK_FLOAT ) return -1; assert( !ExprHasProperty(p, EP_IntValue) ); - sqlite3AtoF(p->u.zToken, &r); + sqlite3AtoF(p->u.zToken, &r, sqlite3Strlen30(p->u.zToken), SQLITE_UTF8); assert( r>=0.0 ); if( r>1.0 ) return -1; return (int)(r*134217728.0); @@ -1656,8 +1656,10 @@ static int resolveCompoundOrderBy( /* Convert the ORDER BY term into an integer column number iCol, ** taking care to preserve the COLLATE clause if it exists. */ if( !IN_RENAME_OBJECT ){ - Expr *pNew = sqlite3ExprInt32(db, iCol); + Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); if( pNew==0 ) return 1; + pNew->flags |= EP_IntValue; + pNew->u.iValue = iCol; if( pItem->pExpr==pE ){ pItem->pExpr = pNew; }else{ @@ -2011,6 +2013,10 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ } #endif + /* The ORDER BY and GROUP BY clauses may not refer to terms in + ** outer queries + */ + sNC.pNext = 0; sNC.ncFlags |= NC_AllowAgg|NC_AllowWin; /* If this is a converted compound query, move the ORDER BY clause from diff --git a/src/select.c b/src/select.c index e8e9f36a8..3ef7cc0a3 100644 --- a/src/select.c +++ b/src/select.c @@ -21,7 +21,7 @@ */ typedef struct DistinctCtx DistinctCtx; struct DistinctCtx { - u8 isTnct; /* 0: Not distinct. 1: DISTINCT 2: DISTINCT and ORDER BY */ + u8 isTnct; /* 0: Not distinct. 1: DISTICT 2: DISTINCT and ORDER BY */ u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */ int tabTnct; /* Ephemeral table used for DISTINCT processing */ int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */ @@ -151,6 +151,8 @@ Select *sqlite3SelectNew( pNew->iLimit = 0; pNew->iOffset = 0; pNew->selId = ++pParse->nSelect; + pNew->addrOpenEphm[0] = -1; + pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = 0; if( pSrc==0 ) pSrc = sqlite3DbMallocZero(pParse->db, SZ_SRCLIST_1); pNew->pSrc = pSrc; @@ -659,10 +661,6 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ pRight->fg.isOn = 1; p->selFlags |= SF_OnToWhere; } - - if( IsVirtual(pRightTab) && joinType==EP_OuterON && pRight->u1.pFuncArg ){ - p->selFlags |= SF_OnToWhere; - } } return 0; } @@ -1302,6 +1300,29 @@ static void selectInnerLoop( } switch( eDest ){ + /* In this mode, write each query result to the key of the temporary + ** table iParm. + */ +#ifndef SQLITE_OMIT_COMPOUND_SELECT + case SRT_Union: { + int r1; + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol); + sqlite3ReleaseTempReg(pParse, r1); + break; + } + + /* Construct a record from the query result, but instead of + ** saving that record, use it as a key to delete elements from + ** the temporary table iParm. + */ + case SRT_Except: { + sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nResultCol); + break; + } +#endif /* SQLITE_OMIT_COMPOUND_SELECT */ + /* Store the result as data using a unique key. */ case SRT_Fifo: @@ -2414,8 +2435,8 @@ void sqlite3SubqueryColumnTypes( } } if( zType ){ - const i64 k = strlen(zType); - n = strlen(pCol->zCnName); + const i64 k = sqlite3Strlen30(zType); + n = sqlite3Strlen30(pCol->zCnName); pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+k+2); pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL); if( pCol->zCnName ){ @@ -2588,9 +2609,9 @@ static CollSeq *multiSelectCollSeq(Parse *pParse, Select *p, int iCol){ ** function is responsible for ensuring that this structure is eventually ** freed. */ -static KeyInfo *multiSelectByMergeKeyInfo(Parse *pParse, Select *p, int nExtra){ +static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){ ExprList *pOrderBy = p->pOrderBy; - int nOrderBy = (pOrderBy!=0) ? pOrderBy->nExpr : 0; + int nOrderBy = ALWAYS(pOrderBy!=0) ? pOrderBy->nExpr : 0; sqlite3 *db = pParse->db; KeyInfo *pRet = sqlite3KeyInfoAlloc(db, nOrderBy+nExtra, 1); if( pRet ){ @@ -2723,7 +2744,7 @@ static void generateWithRecursiveQuery( regCurrent = ++pParse->nMem; sqlite3VdbeAddOp3(v, OP_OpenPseudo, iCurrent, regCurrent, nCol); if( pOrderBy ){ - KeyInfo *pKeyInfo = multiSelectByMergeKeyInfo(pParse, p, 1); + KeyInfo *pKeyInfo = multiSelectOrderByKeyInfo(pParse, p, 1); sqlite3VdbeAddOp4(v, OP_OpenEphemeral, iQueue, pOrderBy->nExpr+2, 0, (char*)pKeyInfo, P4_KEYINFO); destQueue.pOrderBy = pOrderBy; @@ -2732,28 +2753,8 @@ static void generateWithRecursiveQuery( } VdbeComment((v, "Queue table")); if( iDistinct ){ - /* Generate an ephemeral table used to enforce distinctness on the - ** output of the recursive part of the CTE. - */ - KeyInfo *pKeyInfo; /* Collating sequence for the result set */ - CollSeq **apColl; /* For looping through pKeyInfo->aColl[] */ - - assert( p->pNext==0 ); - assert( p->pEList!=0 ); - nCol = p->pEList->nExpr; - pKeyInfo = sqlite3KeyInfoAlloc(pParse->db, nCol, 1); - if( pKeyInfo ){ - for(i=0, apColl=pKeyInfo->aColl; i<nCol; i++, apColl++){ - *apColl = multiSelectCollSeq(pParse, p, i); - if( 0==*apColl ){ - *apColl = pParse->db->pDfltColl; - } - } - sqlite3VdbeAddOp4(v, OP_OpenEphemeral, iDistinct, nCol, 0, - (void*)pKeyInfo, P4_KEYINFO); - }else{ - assert( pParse->nErr>0 ); - } + p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iDistinct, 0); + p->selFlags |= SF_UsesEphemeral; } /* Detach the ORDER BY clause from the compound SELECT */ @@ -2828,7 +2829,7 @@ static void generateWithRecursiveQuery( #endif /* SQLITE_OMIT_CTE */ /* Forward references */ -static int multiSelectByMerge( +static int multiSelectOrderBy( Parse *pParse, /* Parsing context */ Select *p, /* The right-most of SELECTs to be coded */ SelectDest *pDest /* What to do with query results */ @@ -2977,26 +2978,12 @@ static int multiSelect( generateWithRecursiveQuery(pParse, p, &dest); }else #endif + + /* Compound SELECTs that have an ORDER BY clause are handled separately. + */ if( p->pOrderBy ){ - /* If the compound has an ORDER BY clause, then always use the merge - ** algorithm. */ - return multiSelectByMerge(pParse, p, pDest); - }else if( p->op!=TK_ALL ){ - /* If the compound is EXCEPT, INTERSECT, or UNION (anything other than - ** UNION ALL) then also always use the merge algorithm. However, the - ** multiSelectByMerge() routine requires that the compound have an - ** ORDER BY clause, and it doesn't right now. So invent one first. */ - Expr *pOne = sqlite3ExprInt32(db, 1); - p->pOrderBy = sqlite3ExprListAppend(pParse, 0, pOne); - if( pParse->nErr ) goto multi_select_end; - assert( p->pOrderBy!=0 ); - p->pOrderBy->a[0].u.x.iOrderByCol = 1; - return multiSelectByMerge(pParse, p, pDest); + return multiSelectOrderBy(pParse, p, pDest); }else{ - /* For a UNION ALL compound without ORDER BY, simply run the left - ** query, then run the right query */ - int addr = 0; - int nLimit = 0; /* Initialize to suppress harmless compiler warning */ #ifndef SQLITE_OMIT_EXPLAIN if( pPrior->pPrior==0 ){ @@ -3004,49 +2991,300 @@ static int multiSelect( ExplainQueryPlan((pParse, 1, "LEFT-MOST SUBQUERY")); } #endif - assert( !pPrior->pLimit ); - pPrior->iLimit = p->iLimit; - pPrior->iOffset = p->iOffset; - pPrior->pLimit = sqlite3ExprDup(db, p->pLimit, 0); - TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL left...\n")); - rc = sqlite3Select(pParse, pPrior, &dest); - sqlite3ExprDelete(db, pPrior->pLimit); - pPrior->pLimit = 0; - if( rc ){ - goto multi_select_end; - } - p->pPrior = 0; - p->iLimit = pPrior->iLimit; - p->iOffset = pPrior->iOffset; - if( p->iLimit ){ - addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v); - VdbeComment((v, "Jump ahead if LIMIT reached")); - if( p->iOffset ){ - sqlite3VdbeAddOp3(v, OP_OffsetLimit, - p->iLimit, p->iOffset+1, p->iOffset); - } - } - ExplainQueryPlan((pParse, 1, "UNION ALL")); - TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL right...\n")); - rc = sqlite3Select(pParse, p, &dest); - testcase( rc!=SQLITE_OK ); - pDelete = p->pPrior; - p->pPrior = pPrior; - p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); - if( p->pLimit - && sqlite3ExprIsInteger(p->pLimit->pLeft, &nLimit, pParse) - && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) - ){ - p->nSelectRow = sqlite3LogEst((u64)nLimit); - } - if( addr ){ - sqlite3VdbeJumpHere(v, addr); + + /* Generate code for the left and right SELECT statements. + */ + switch( p->op ){ + case TK_ALL: { + int addr = 0; + int nLimit = 0; /* Initialize to suppress harmless compiler warning */ + assert( !pPrior->pLimit ); + pPrior->iLimit = p->iLimit; + pPrior->iOffset = p->iOffset; + pPrior->pLimit = p->pLimit; + TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL left...\n")); + rc = sqlite3Select(pParse, pPrior, &dest); + pPrior->pLimit = 0; + if( rc ){ + goto multi_select_end; + } + p->pPrior = 0; + p->iLimit = pPrior->iLimit; + p->iOffset = pPrior->iOffset; + if( p->iLimit ){ + addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v); + VdbeComment((v, "Jump ahead if LIMIT reached")); + if( p->iOffset ){ + sqlite3VdbeAddOp3(v, OP_OffsetLimit, + p->iLimit, p->iOffset+1, p->iOffset); + } + } + ExplainQueryPlan((pParse, 1, "UNION ALL")); + TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL right...\n")); + rc = sqlite3Select(pParse, p, &dest); + testcase( rc!=SQLITE_OK ); + pDelete = p->pPrior; + p->pPrior = pPrior; + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); + if( p->pLimit + && sqlite3ExprIsInteger(p->pLimit->pLeft, &nLimit, pParse) + && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) + ){ + p->nSelectRow = sqlite3LogEst((u64)nLimit); + } + if( addr ){ + sqlite3VdbeJumpHere(v, addr); + } + break; + } + case TK_EXCEPT: + case TK_UNION: { + int unionTab; /* Cursor number of the temp table holding result */ + u8 op = 0; /* One of the SRT_ operations to apply to self */ + int priorOp; /* The SRT_ operation to apply to prior selects */ + Expr *pLimit; /* Saved values of p->nLimit */ + int addr; + int emptyBypass = 0; /* IfEmpty opcode to bypass RHS */ + SelectDest uniondest; + + + testcase( p->op==TK_EXCEPT ); + testcase( p->op==TK_UNION ); + priorOp = SRT_Union; + if( dest.eDest==priorOp ){ + /* We can reuse a temporary table generated by a SELECT to our + ** right. + */ + assert( p->pLimit==0 ); /* Not allowed on leftward elements */ + unionTab = dest.iSDParm; + }else{ + /* We will need to create our own temporary table to hold the + ** intermediate results. + */ + unionTab = pParse->nTab++; + assert( p->pOrderBy==0 ); + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, unionTab, 0); + assert( p->addrOpenEphm[0] == -1 ); + p->addrOpenEphm[0] = addr; + findRightmost(p)->selFlags |= SF_UsesEphemeral; + assert( p->pEList ); + } + + + /* Code the SELECT statements to our left + */ + assert( !pPrior->pOrderBy ); + sqlite3SelectDestInit(&uniondest, priorOp, unionTab); + TREETRACE(0x200, pParse, p, ("multiSelect EXCEPT/UNION left...\n")); + rc = sqlite3Select(pParse, pPrior, &uniondest); + if( rc ){ + goto multi_select_end; + } + + /* Code the current SELECT statement + */ + if( p->op==TK_EXCEPT ){ + op = SRT_Except; + emptyBypass = sqlite3VdbeAddOp1(v, OP_IfEmpty, unionTab); + VdbeCoverage(v); + }else{ + assert( p->op==TK_UNION ); + op = SRT_Union; + } + p->pPrior = 0; + pLimit = p->pLimit; + p->pLimit = 0; + uniondest.eDest = op; + ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", + sqlite3SelectOpName(p->op))); + TREETRACE(0x200, pParse, p, ("multiSelect EXCEPT/UNION right...\n")); + rc = sqlite3Select(pParse, p, &uniondest); + testcase( rc!=SQLITE_OK ); + assert( p->pOrderBy==0 ); + pDelete = p->pPrior; + p->pPrior = pPrior; + p->pOrderBy = 0; + if( p->op==TK_UNION ){ + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); + } + if( emptyBypass ) sqlite3VdbeJumpHere(v, emptyBypass); + sqlite3ExprDelete(db, p->pLimit); + p->pLimit = pLimit; + p->iLimit = 0; + p->iOffset = 0; + + /* Convert the data in the temporary table into whatever form + ** it is that we currently need. + */ + assert( unionTab==dest.iSDParm || dest.eDest!=priorOp ); + assert( p->pEList || db->mallocFailed ); + if( dest.eDest!=priorOp && db->mallocFailed==0 ){ + int iCont, iBreak, iStart; + iBreak = sqlite3VdbeMakeLabel(pParse); + iCont = sqlite3VdbeMakeLabel(pParse); + computeLimitRegisters(pParse, p, iBreak); + sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v); + iStart = sqlite3VdbeCurrentAddr(v); + selectInnerLoop(pParse, p, unionTab, + 0, 0, &dest, iCont, iBreak); + sqlite3VdbeResolveLabel(v, iCont); + sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); VdbeCoverage(v); + sqlite3VdbeResolveLabel(v, iBreak); + sqlite3VdbeAddOp2(v, OP_Close, unionTab, 0); + } + break; + } + default: assert( p->op==TK_INTERSECT ); { + int tab1, tab2; + int iCont, iBreak, iStart; + Expr *pLimit; + int addr, iLimit, iOffset; + SelectDest intersectdest; + int r1; + int emptyBypass; + + /* INTERSECT is different from the others since it requires + ** two temporary tables. Hence it has its own case. Begin + ** by allocating the tables we will need. + */ + tab1 = pParse->nTab++; + tab2 = pParse->nTab++; + assert( p->pOrderBy==0 ); + + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab1, 0); + assert( p->addrOpenEphm[0] == -1 ); + p->addrOpenEphm[0] = addr; + findRightmost(p)->selFlags |= SF_UsesEphemeral; + assert( p->pEList ); + + /* Code the SELECTs to our left into temporary table "tab1". + */ + sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1); + TREETRACE(0x400, pParse, p, ("multiSelect INTERSECT left...\n")); + rc = sqlite3Select(pParse, pPrior, &intersectdest); + if( rc ){ + goto multi_select_end; + } + + /* Initialize LIMIT counters before checking to see if the LHS + ** is empty, in case the jump is taken */ + iBreak = sqlite3VdbeMakeLabel(pParse); + computeLimitRegisters(pParse, p, iBreak); + emptyBypass = sqlite3VdbeAddOp1(v, OP_IfEmpty, tab1); VdbeCoverage(v); + + /* Code the current SELECT into temporary table "tab2" + */ + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab2, 0); + assert( p->addrOpenEphm[1] == -1 ); + p->addrOpenEphm[1] = addr; + + /* Disable prior SELECTs and the LIMIT counters during the computation + ** of the RHS select */ + pLimit = p->pLimit; + iLimit = p->iLimit; + iOffset = p->iOffset; + p->pPrior = 0; + p->pLimit = 0; + p->iLimit = 0; + p->iOffset = 0; + + intersectdest.iSDParm = tab2; + ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", + sqlite3SelectOpName(p->op))); + TREETRACE(0x400, pParse, p, ("multiSelect INTERSECT right...\n")); + rc = sqlite3Select(pParse, p, &intersectdest); + testcase( rc!=SQLITE_OK ); + pDelete = p->pPrior; + p->pPrior = pPrior; + if( p->nSelectRow>pPrior->nSelectRow ){ + p->nSelectRow = pPrior->nSelectRow; + } + sqlite3ExprDelete(db, p->pLimit); + + /* Reinstate the LIMIT counters prior to running the final intersect */ + p->pLimit = pLimit; + p->iLimit = iLimit; + p->iOffset = iOffset; + + /* Generate code to take the intersection of the two temporary + ** tables. + */ + if( rc ) break; + assert( p->pEList ); + sqlite3VdbeAddOp1(v, OP_Rewind, tab1); + r1 = sqlite3GetTempReg(pParse); + iStart = sqlite3VdbeAddOp2(v, OP_RowData, tab1, r1); + iCont = sqlite3VdbeMakeLabel(pParse); + sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); + VdbeCoverage(v); + sqlite3ReleaseTempReg(pParse, r1); + selectInnerLoop(pParse, p, tab1, + 0, 0, &dest, iCont, iBreak); + sqlite3VdbeResolveLabel(v, iCont); + sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v); + sqlite3VdbeResolveLabel(v, iBreak); + sqlite3VdbeAddOp2(v, OP_Close, tab2, 0); + sqlite3VdbeJumpHere(v, emptyBypass); + sqlite3VdbeAddOp2(v, OP_Close, tab1, 0); + break; + } } -#ifndef SQLITE_OMIT_EXPLAIN + + #ifndef SQLITE_OMIT_EXPLAIN if( p->pNext==0 ){ ExplainQueryPlanPop(pParse); } -#endif + #endif + } + if( pParse->nErr ) goto multi_select_end; + + /* Compute collating sequences used by + ** temporary tables needed to implement the compound select. + ** Attach the KeyInfo structure to all temporary tables. + ** + ** This section is run by the right-most SELECT statement only. + ** SELECT statements to the left always skip this part. The right-most + ** SELECT might also skip this part if it has no ORDER BY clause and + ** no temp tables are required. + */ + if( p->selFlags & SF_UsesEphemeral ){ + int i; /* Loop counter */ + KeyInfo *pKeyInfo; /* Collating sequence for the result set */ + Select *pLoop; /* For looping through SELECT statements */ + CollSeq **apColl; /* For looping through pKeyInfo->aColl[] */ + int nCol; /* Number of columns in result set */ + + assert( p->pNext==0 ); + assert( p->pEList!=0 ); + nCol = p->pEList->nExpr; + pKeyInfo = sqlite3KeyInfoAlloc(db, nCol, 1); + if( !pKeyInfo ){ + rc = SQLITE_NOMEM_BKPT; + goto multi_select_end; + } + for(i=0, apColl=pKeyInfo->aColl; i<nCol; i++, apColl++){ + *apColl = multiSelectCollSeq(pParse, p, i); + if( 0==*apColl ){ + *apColl = db->pDfltColl; + } + } + + for(pLoop=p; pLoop; pLoop=pLoop->pPrior){ + for(i=0; i<2; i++){ + int addr = pLoop->addrOpenEphm[i]; + if( addr<0 ){ + /* If [0] is unused then [1] is also unused. So we can + ** always safely abort as soon as the first unused slot is found */ + assert( pLoop->addrOpenEphm[1]<0 ); + break; + } + sqlite3VdbeChangeP2(v, addr, nCol); + sqlite3VdbeChangeP4(v, addr, (char*)sqlite3KeyInfoRef(pKeyInfo), + P4_KEYINFO); + pLoop->addrOpenEphm[i] = -1; + } + } + sqlite3KeyInfoUnref(pKeyInfo); } multi_select_end: @@ -3078,8 +3316,8 @@ void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p){ ** Code an output subroutine for a coroutine implementation of a ** SELECT statement. ** -** The data to be output is contained in an array of pIn->nSdst registers -** starting at register pIn->iSdst. pDest is where the output should +** The data to be output is contained in pIn->iSdst. There are +** pIn->nSdst columns to be output. pDest is where the output should ** be sent. ** ** regReturn is the number of the register holding the subroutine @@ -3108,8 +3346,6 @@ static int generateOutputSubroutine( int iContinue; int addr; - assert( pIn->eDest==SRT_Coroutine ); - addr = sqlite3VdbeCurrentAddr(v); iContinue = sqlite3VdbeMakeLabel(pParse); @@ -3131,60 +3367,23 @@ static int generateOutputSubroutine( */ codeOffset(v, p->iOffset, iContinue); + assert( pDest->eDest!=SRT_Exists ); + assert( pDest->eDest!=SRT_Table ); switch( pDest->eDest ){ /* Store the result as data using a unique key. */ - case SRT_Fifo: - case SRT_DistFifo: - case SRT_Table: case SRT_EphemTab: { int r1 = sqlite3GetTempReg(pParse); int r2 = sqlite3GetTempReg(pParse); - int iParm = pDest->iSDParm; - testcase( pDest->eDest==SRT_Table ); - testcase( pDest->eDest==SRT_EphemTab ); - testcase( pDest->eDest==SRT_Fifo ); - testcase( pDest->eDest==SRT_DistFifo ); sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r1); -#if !defined(SQLITE_ENABLE_NULL_TRIM) && defined(SQLITE_DEBUG) - /* A destination of SRT_Table and a non-zero iSDParm2 parameter means - ** that this is an "UPDATE ... FROM" on a virtual table or view. In this - ** case set the p5 parameter of the OP_MakeRecord to OPFLAG_NOCHNG_MAGIC. - ** This does not affect operation in any way - it just allows MakeRecord - ** to process OPFLAG_NOCHANGE values without an assert() failing. */ - if( pDest->eDest==SRT_Table && pDest->iSDParm2 ){ - sqlite3VdbeChangeP5(v, OPFLAG_NOCHNG_MAGIC); - } -#endif -#ifndef SQLITE_OMIT_CTE - if( pDest->eDest==SRT_DistFifo ){ - /* If the destination is DistFifo, then cursor (iParm+1) is open - ** on an ephemeral index that is used to enforce uniqueness on the - ** total result. At this point, we are processing the setup portion - ** of the recursive CTE using the merge algorithm, so the results are - ** guaranteed to be unique anyhow. But we still need to populate the - ** (iParm+1) cursor for use by the subsequent recursive phase. - */ - sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm+1, r1, - pIn->iSdst, pIn->nSdst); - } -#endif - sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); - sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, r2); + sqlite3VdbeAddOp2(v, OP_NewRowid, pDest->iSDParm, r2); + sqlite3VdbeAddOp3(v, OP_Insert, pDest->iSDParm, r1, r2); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3ReleaseTempReg(pParse, r2); sqlite3ReleaseTempReg(pParse, r1); break; } - /* If any row exist in the result set, record that fact and abort. - */ - case SRT_Exists: { - sqlite3VdbeAddOp2(v, OP_Integer, 1, pDest->iSDParm); - /* The LIMIT clause will terminate the loop for us */ - break; - } - #ifndef SQLITE_OMIT_SUBQUERY /* If we are creating a set for an "expr IN (SELECT ...)". */ @@ -3231,51 +3430,9 @@ static int generateOutputSubroutine( break; } -#ifndef SQLITE_OMIT_CTE - /* Write the results into a priority queue that is order according to - ** pDest->pOrderBy (in pSO). pDest->iSDParm (in iParm) is the cursor for an - ** index with pSO->nExpr+2 columns. Build a key using pSO for the first - ** pSO->nExpr columns, then make sure all keys are unique by adding a - ** final OP_Sequence column. The last column is the record as a blob. - */ - case SRT_DistQueue: - case SRT_Queue: { - int nKey; - int r1, r2, r3, ii; - ExprList *pSO; - int iParm = pDest->iSDParm; - pSO = pDest->pOrderBy; - assert( pSO ); - nKey = pSO->nExpr; - r1 = sqlite3GetTempReg(pParse); - r2 = sqlite3GetTempRange(pParse, nKey+2); - r3 = r2+nKey+1; - - sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r3); - if( pDest->eDest==SRT_DistQueue ){ - sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r3); - } - for(ii=0; ii<nKey; ii++){ - sqlite3VdbeAddOp2(v, OP_SCopy, - pIn->iSdst + pSO->a[ii].u.x.iOrderByCol - 1, - r2+ii); - } - sqlite3VdbeAddOp2(v, OP_Sequence, iParm, r2+nKey); - sqlite3VdbeAddOp3(v, OP_MakeRecord, r2, nKey+2, r1); - sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, r2, nKey+2); - sqlite3ReleaseTempReg(pParse, r1); - sqlite3ReleaseTempRange(pParse, r2, nKey+2); - break; - } -#endif /* SQLITE_OMIT_CTE */ - - /* Ignore the output */ - case SRT_Discard: { - break; - } - /* If none of the above, then the result destination must be - ** SRT_Output. + ** SRT_Output. This routine is never called with any other + ** destination other than the ones handled above or SRT_Output. ** ** For SRT_Output, results are stored in a sequence of registers. ** Then the OP_ResultRow opcode is used to cause sqlite3_step() to @@ -3303,9 +3460,8 @@ static int generateOutputSubroutine( } /* -** Generate code for a compound SELECT statement using a merge -** algorithm. The compound must have an ORDER BY clause for this -** to work. +** Alternative compound select code generator for cases when there +** is an ORDER BY clause. ** ** We assume a query of the following form: ** @@ -3322,7 +3478,7 @@ static int generateOutputSubroutine( ** ** outB: Move the output of the selectB coroutine into the output ** of the compound query. (Only generated for UNION and -** UNION ALL. EXCEPT and INTERSECT never output a row that +** UNION ALL. EXCEPT and INSERTSECT never output a row that ** appears only in B.) ** ** AltB: Called when there is data from both coroutines and A<B. @@ -3375,8 +3531,10 @@ static int generateOutputSubroutine( ** AeqB: ... ** AgtB: ... ** Init: initialize coroutine registers -** yield coA, on eof goto EofA -** yield coB, on eof goto EofB +** yield coA +** if eof(A) goto EofA +** yield coB +** if eof(B) goto EofB ** Cmpr: Compare A, B ** Jump AltB, AeqB, AgtB ** End: ... @@ -3384,10 +3542,10 @@ static int generateOutputSubroutine( ** We call AltB, AeqB, AgtB, EofA, and EofB "subroutines" but they are not ** actually called using Gosub and they do not Return. EofA and EofB loop ** until all data is exhausted then jump to the "end" label. AltB, AeqB, -** and AgtB jump to either Cmpr or to one of EofA or EofB. +** and AgtB jump to either L2 or to one of EofA or EofB. */ #ifndef SQLITE_OMIT_COMPOUND_SELECT -static int multiSelectByMerge( +static int multiSelectOrderBy( Parse *pParse, /* Parsing context */ Select *p, /* The right-most of SELECTs to be coded */ SelectDest *pDest /* What to do with query results */ @@ -3459,8 +3617,10 @@ static int multiSelectByMerge( if( pItem->u.x.iOrderByCol==i ) break; } if( j==nOrderBy ){ - Expr *pNew = sqlite3ExprInt32(db, i); + Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); if( pNew==0 ) return SQLITE_NOMEM_BKPT; + pNew->flags |= EP_IntValue; + pNew->u.iValue = i; p->pOrderBy = pOrderBy = sqlite3ExprListAppend(pParse, pOrderBy, pNew); if( pOrderBy ) pOrderBy->a[nOrderBy++].u.x.iOrderByCol = (u16)i; } @@ -3468,29 +3628,26 @@ static int multiSelectByMerge( } /* Compute the comparison permutation and keyinfo that is used with - ** the permutation to determine if the next row of results comes - ** from selectA or selectB. Also add literal collations to the - ** ORDER BY clause terms so that when selectA and selectB are - ** evaluated, they use the correct collation. + ** the permutation used to determine if the next + ** row of results comes from selectA or selectB. Also add explicit + ** collations to the ORDER BY clause terms so that when the subqueries + ** to the right and the left are evaluated, they use the correct + ** collation. */ aPermute = sqlite3DbMallocRawNN(db, sizeof(u32)*(nOrderBy + 1)); if( aPermute ){ struct ExprList_item *pItem; - int bKeep = 0; aPermute[0] = nOrderBy; for(i=1, pItem=pOrderBy->a; i<=nOrderBy; i++, pItem++){ assert( pItem!=0 ); assert( pItem->u.x.iOrderByCol>0 ); assert( pItem->u.x.iOrderByCol<=p->pEList->nExpr ); aPermute[i] = pItem->u.x.iOrderByCol - 1; - if( aPermute[i]!=(u32)i-1 ) bKeep = 1; - } - if( bKeep==0 ){ - sqlite3DbFreeNN(db, aPermute); - aPermute = 0; } + pKeyMerge = multiSelectOrderByKeyInfo(pParse, p, 1); + }else{ + pKeyMerge = 0; } - pKeyMerge = multiSelectByMergeKeyInfo(pParse, p, 1); /* Allocate a range of temporary registers and the KeyInfo needed ** for the logic that removes duplicate result rows when the @@ -3569,7 +3726,7 @@ static int multiSelectByMerge( */ addrSelectA = sqlite3VdbeCurrentAddr(v) + 1; addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); - VdbeComment((v, "SUBR: next-A")); + VdbeComment((v, "left SELECT")); pPrior->iLimit = regLimitA; ExplainQueryPlan((pParse, 1, "LEFT")); sqlite3Select(pParse, pPrior, &destA); @@ -3581,7 +3738,7 @@ static int multiSelectByMerge( */ addrSelectB = sqlite3VdbeCurrentAddr(v) + 1; addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB); - VdbeComment((v, "SUBR: next-B")); + VdbeComment((v, "right SELECT")); savedLimit = p->iLimit; savedOffset = p->iOffset; p->iLimit = regLimitB; @@ -3595,7 +3752,7 @@ static int multiSelectByMerge( /* Generate a subroutine that outputs the current row of the A ** select as the next output row of the compound select. */ - VdbeNoopComment((v, "SUBR: out-A")); + VdbeNoopComment((v, "Output routine for A")); addrOutA = generateOutputSubroutine(pParse, p, &destA, pDest, regOutA, regPrev, pKeyDup, labelEnd); @@ -3604,7 +3761,7 @@ static int multiSelectByMerge( ** select as the next output row of the compound select. */ if( op==TK_ALL || op==TK_UNION ){ - VdbeNoopComment((v, "SUBR: out-B")); + VdbeNoopComment((v, "Output routine for B")); addrOutB = generateOutputSubroutine(pParse, p, &destB, pDest, regOutB, regPrev, pKeyDup, labelEnd); @@ -3617,12 +3774,10 @@ static int multiSelectByMerge( if( op==TK_EXCEPT || op==TK_INTERSECT ){ addrEofA_noB = addrEofA = labelEnd; }else{ - VdbeNoopComment((v, "SUBR: eof-A")); + VdbeNoopComment((v, "eof-A subroutine")); addrEofA = sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); - VdbeComment((v, "out-B")); addrEofA_noB = sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, labelEnd); VdbeCoverage(v); - VdbeComment((v, "next-B")); sqlite3VdbeGoto(v, addrEofA); p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); } @@ -3634,20 +3789,17 @@ static int multiSelectByMerge( addrEofB = addrEofA; if( p->nSelectRow > pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; }else{ - VdbeNoopComment((v, "SUBR: eof-B")); + VdbeNoopComment((v, "eof-B subroutine")); addrEofB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); - VdbeComment((v, "out-A")); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, labelEnd); VdbeCoverage(v); - VdbeComment((v, "next-A")); sqlite3VdbeGoto(v, addrEofB); } /* Generate code to handle the case of A<B */ + VdbeNoopComment((v, "A-lt-B subroutine")); addrAltB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); - VdbeComment((v, "out-A")); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA); VdbeCoverage(v); - VdbeComment((v, "next-A")); sqlite3VdbeGoto(v, labelCmpr); /* Generate code to handle the case of A==B @@ -3658,48 +3810,36 @@ static int multiSelectByMerge( addrAeqB = addrAltB; addrAltB++; }else{ - addrAeqB = addrAltB + 1; + VdbeNoopComment((v, "A-eq-B subroutine")); + addrAeqB = + sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA); VdbeCoverage(v); + sqlite3VdbeGoto(v, labelCmpr); } /* Generate code to handle the case of A>B */ + VdbeNoopComment((v, "A-gt-B subroutine")); addrAgtB = sqlite3VdbeCurrentAddr(v); if( op==TK_ALL || op==TK_UNION ){ sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); - VdbeComment((v, "out-B")); - sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); - VdbeComment((v, "next-B")); - sqlite3VdbeGoto(v, labelCmpr); - }else{ - addrAgtB++; /* Just do next-B. Might as well use the next-B call - ** in the next code block */ } + sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); + sqlite3VdbeGoto(v, labelCmpr); /* This code runs once to initialize everything. */ sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA_noB); VdbeCoverage(v); - VdbeComment((v, "next-A")); - /* v--- Also the A>B case for EXCEPT and INTERSECT */ sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); - VdbeComment((v, "next-B")); /* Implement the main merge loop */ - if( aPermute!=0 ){ - sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY); - } sqlite3VdbeResolveLabel(v, labelCmpr); + sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY); sqlite3VdbeAddOp4(v, OP_Compare, destA.iSdst, destB.iSdst, nOrderBy, (char*)pKeyMerge, P4_KEYINFO); - if( aPermute!=0 ){ - sqlite3VdbeChangeP5(v, OPFLAG_PERMUTE); - } - sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); - VdbeCoverageIf(v, op==TK_ALL); - VdbeCoverageIf(v, op==TK_UNION); - VdbeCoverageIf(v, op==TK_EXCEPT); - VdbeCoverageIf(v, op==TK_INTERSECT); + sqlite3VdbeChangeP5(v, OPFLAG_PERMUTE); + sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); VdbeCoverage(v); /* Jump to the this point in order to terminate the query. */ @@ -4616,7 +4756,7 @@ static int flattenSubquery( } pSubitem->fg.jointype |= jointype; - /* Begin substituting subquery result set expressions for + /* Now begin substituting subquery result set expressions for ** references to the iParent in the outer query. ** ** Example: @@ -4628,7 +4768,7 @@ static int flattenSubquery( ** We look at every expression in the outer query and every place we see ** "a" we substitute "x*3" and every place we see "b" we substitute "y+10". */ - if( pSub->pOrderBy ){ + if( pSub->pOrderBy && (pParent->selFlags & SF_NoopOrderBy)==0 ){ /* At this point, any non-zero iOrderByCol values indicate that the ** ORDER BY column expression is identical to the iOrderByCol'th ** expression returned by SELECT statement pSub. Since these values @@ -4636,9 +4776,9 @@ static int flattenSubquery( ** zero them before transferring the ORDER BY clause. ** ** Not doing this may cause an error if a subsequent call to this - ** function attempts to flatten a compound sub-query into pParent. - ** See ticket [d11a6e908f]. - */ + ** function attempts to flatten a compound sub-query into pParent + ** (the only way this can happen is if the compound sub-query is + ** currently part of pSub->pSrc). See ticket [d11a6e908f]. */ ExprList *pOrderBy = pSub->pOrderBy; for(i=0; i<pOrderBy->nExpr; i++){ pOrderBy->a[i].u.x.iOrderByCol = 0; @@ -5256,16 +5396,6 @@ static int pushDownWhereTerms( x.pEList = pSubq->pEList; x.pCList = findLeftmostExprlist(pSubq); pNew = substExpr(&x, pNew); - assert( pNew!=0 || pParse->nErr!=0 ); - if( pParse->nErr==0 && pNew->op==TK_IN && ExprUseXSelect(pNew) ){ - assert( pNew->x.pSelect!=0 ); - pNew->x.pSelect->selFlags |= SF_ClonedRhsIn; - assert( pWhere!=0 ); - assert( pWhere->op==TK_IN ); - assert( ExprUseXSelect(pWhere) ); - assert( pWhere->x.pSelect!=0 ); - pWhere->x.pSelect->selFlags |= SF_ClonedRhsIn; - } #ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pWin && 0==pushDownWindowCheck(pParse, pSubq, pNew) ){ /* Restriction 6c has prevented push-down in this case */ @@ -5500,14 +5630,14 @@ int sqlite3IndexedByLookup(Parse *pParse, SrcItem *pFrom){ ** SELECT * FROM (SELECT ... FROM t1 EXCEPT SELECT ... FROM t2) ** ORDER BY ... COLLATE ... ** -** This transformation is necessary because the multiSelectByMerge() routine +** This transformation is necessary because the multiSelectOrderBy() routine ** above that generates the code for a compound SELECT with an ORDER BY clause ** uses a merge algorithm that requires the same collating sequence on the ** result columns as on the ORDER BY clause. See ticket ** http://sqlite.org/src/info/6709574d2a ** ** This transformation is only needed for EXCEPT, INTERSECT, and UNION. -** The UNION ALL operator works fine with multiSelectByMerge() even when +** The UNION ALL operator works fine with multiSelectOrderBy() even when ** there are COLLATE terms in the ORDER BY. */ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ @@ -6053,7 +6183,7 @@ static int selectExpander(Walker *pWalker, Select *p){ } #ifndef SQLITE_OMIT_VIRTUALTABLE else if( ALWAYS(IsVirtual(pTab)) - && (pFrom->fg.fromDDL || (pParse->prepFlags & SQLITE_PREPARE_FROM_DDL)) + && pFrom->fg.fromDDL && ALWAYS(pTab->u.vtab.p!=0) && pTab->u.vtab.p->eVtabRisk > ((db->flags & SQLITE_TrustedSchema)!=0) ){ @@ -7006,7 +7136,7 @@ static int havingToWhereExprCb(Walker *pWalker, Expr *pExpr){ && pExpr->pAggInfo==0 ){ sqlite3 *db = pWalker->pParse->db; - Expr *pNew = sqlite3ExprInt32(db, 1); + Expr *pNew = sqlite3Expr(db, TK_INTEGER, "1"); if( pNew ){ Expr *pWhere = pS->pWhere; SWAP(Expr, *pNew, *pExpr); @@ -7357,6 +7487,7 @@ static SQLITE_NOINLINE void existsToJoin( ExprSetProperty(pWhere, EP_IntValue); assert( p->pWhere!=0 ); pSub->pSrc->a[0].fg.fromExists = 1; + pSub->pSrc->a[0].fg.jointype |= JT_CROSS; p->pSrc = sqlite3SrcListAppendList(pParse, p->pSrc, pSub->pSrc); if( pSubWhere ){ p->pWhere = sqlite3PExpr(pParse, TK_AND, p->pWhere, pSubWhere); @@ -7384,7 +7515,6 @@ typedef struct CheckOnCtx CheckOnCtx; struct CheckOnCtx { SrcList *pSrc; /* SrcList for this context */ int iJoin; /* Cursor numbers must be =< than this */ - int bFuncArg; /* True for table-function arg */ CheckOnCtx *pParent; /* Parent context */ }; @@ -7436,9 +7566,7 @@ static int selectCheckOnClausesExpr(Walker *pWalker, Expr *pExpr){ if( iTab>=pSrc->a[0].iCursor && iTab<=pSrc->a[pSrc->nSrc-1].iCursor ){ if( pCtx->iJoin && iTab>pCtx->iJoin ){ sqlite3ErrorMsg(pWalker->pParse, - "%s references tables to its right", - (pCtx->bFuncArg ? "table-function argument" : "ON clause") - ); + "ON clause references tables to its right"); return WRC_Abort; } break; @@ -7476,7 +7604,6 @@ static int selectCheckOnClausesSelect(Walker *pWalker, Select *pSelect){ void sqlite3SelectCheckOnClauses(Parse *pParse, Select *pSelect){ Walker w; CheckOnCtx sCtx; - int ii; assert( pSelect->selFlags & SF_OnToWhere ); assert( pSelect->pSrc!=0 && pSelect->pSrc->nSrc>=2 ); memset(&w, 0, sizeof(w)); @@ -7486,46 +7613,8 @@ void sqlite3SelectCheckOnClauses(Parse *pParse, Select *pSelect){ w.u.pCheckOnCtx = &sCtx; memset(&sCtx, 0, sizeof(sCtx)); sCtx.pSrc = pSelect->pSrc; - sqlite3WalkExpr(&w, pSelect->pWhere); + sqlite3WalkExprNN(&w, pSelect->pWhere); pSelect->selFlags &= ~SF_OnToWhere; - - /* Check for any table-function args that are attached to virtual tables - ** on the RHS of an outer join. They are subject to the same constraints - ** as ON clauses. */ - sCtx.bFuncArg = 1; - for(ii=0; ii<pSelect->pSrc->nSrc; ii++){ - SrcItem *pItem = &pSelect->pSrc->a[ii]; - if( pItem->fg.isTabFunc - && (pItem->fg.jointype & JT_OUTER) - ){ - sCtx.iJoin = pItem->iCursor; - sqlite3WalkExprList(&w, pItem->u1.pFuncArg); - } - } -} - -/* -** If p2 exists and p1 and p2 have the same number of terms, then change -** every term of p1 to have the same sort order as p2 and return true. -** -** If p2 is NULL or p1 and p2 are different lengths, then make no changes -** and return false. -** -** p1 must be non-NULL. -*/ -static int sqlite3CopySortOrder(ExprList *p1, ExprList *p2){ - assert( p1 ); - if( p2 && p1->nExpr==p2->nExpr ){ - int ii; - for(ii=0; ii<p1->nExpr; ii++){ - u8 sortFlags; - sortFlags = p2->a[ii].fg.sortFlags & KEYINFO_ORDER_DESC; - p1->a[ii].fg.sortFlags = sortFlags; - } - return 1; - }else{ - return 0; - } } /* @@ -7623,7 +7712,8 @@ int sqlite3Select( assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_Queue ); if( IgnorableDistinct(pDest) ){ - assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Discard || + assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union || + pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard || pDest->eDest==SRT_DistQueue || pDest->eDest==SRT_DistFifo ); /* All of these destinations are also able to ignore the ORDER BY clause */ if( p->pOrderBy ){ @@ -7639,6 +7729,7 @@ int sqlite3Select( p->pOrderBy = 0; } p->selFlags &= ~(u32)SF_Distinct; + p->selFlags |= SF_NoopOrderBy; } sqlite3SelectPrep(pParse, p, 0); if( pParse->nErr ){ @@ -8166,8 +8257,7 @@ int sqlite3Select( ** BY and DISTINCT, and an index or separate temp-table for the other. */ if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct - && sqlite3CopySortOrder(pEList, sSort.pOrderBy) - && sqlite3ExprListCompare(pEList, sSort.pOrderBy, -1)==0 + && sqlite3ExprListCompare(sSort.pOrderBy, pEList, -1)==0 && OptimizationEnabled(db, SQLITE_GroupByOrder) #ifndef SQLITE_OMIT_WINDOWFUNC && p->pWin==0 @@ -8381,10 +8471,21 @@ int sqlite3Select( ** but not actually sorted. Either way, record the fact that the ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp ** variable. */ - if( sqlite3CopySortOrder(pGroupBy, sSort.pOrderBy) - && sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 - ){ - orderByGrp = 1; + if( sSort.pOrderBy && pGroupBy->nExpr==sSort.pOrderBy->nExpr ){ + int ii; + /* The GROUP BY processing doesn't care whether rows are delivered in + ** ASC or DESC order - only that each group is returned contiguously. + ** So set the ASC/DESC flags in the GROUP BY to match those in the + ** ORDER BY to maximize the chances of rows being delivered in an + ** order that makes the ORDER BY redundant. */ + for(ii=0; ii<pGroupBy->nExpr; ii++){ + u8 sortFlags; + sortFlags = sSort.pOrderBy->a[ii].fg.sortFlags & KEYINFO_ORDER_DESC; + pGroupBy->a[ii].fg.sortFlags = sortFlags; + } + if( sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 ){ + orderByGrp = 1; + } } }else{ assert( 0==sqlite3LogEst(1) ); diff --git a/src/shell.c.in b/src/shell.c.in index ef30194fa..bd4483ff7 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file contains code to implement the "sqlite3" command line +** This file contains code to implement the "sqlite" command line ** utility for accessing SQLite databases. */ #if (defined(_WIN32) || defined(WIN32)) && !defined(_CRT_SECURE_NO_WARNINGS) @@ -31,6 +31,14 @@ typedef unsigned short int u16; # include SHELL_STRINGIFY(SQLITE_CUSTOM_INCLUDE) #endif +/* +** Determine if we are dealing with WinRT, which provides only a subset of +** the full Win32 API. +*/ +#if !defined(SQLITE_OS_WINRT) +# define SQLITE_OS_WINRT 0 +#endif + /* ** If SQLITE_SHELL_FIDDLE is defined then the shell is modified ** somewhat for use as a WASM module in a web browser. This flag @@ -39,10 +47,6 @@ typedef unsigned short int u16; ** and this build mode rewires the user input subsystem to account for ** that. */ -#if defined(SQLITE_SHELL_FIDDLE) -# undef SQLITE_OMIT_LOAD_EXTENSION -# define SQLITE_OMIT_LOAD_EXTENSION 1 -#endif /* ** Warning pragmas copied from msvc.h in the core. @@ -104,7 +108,6 @@ typedef unsigned char u8; #include <stdarg.h> #ifndef _WIN32 # include <sys/time.h> -# include <sys/ioctl.h> #endif #if !defined(_WIN32) && !defined(WIN32) @@ -174,6 +177,9 @@ typedef unsigned char u8; #endif #if defined(_WIN32) || defined(WIN32) +# if SQLITE_OS_WINRT +# define SQLITE_OMIT_POPEN 1 +# else # include <io.h> # include <fcntl.h> # define isatty(h) _isatty(h) @@ -188,6 +194,7 @@ typedef unsigned char u8; # endif # undef pclose # define pclose _pclose +# endif #else /* Make sure isatty() has a prototype. */ extern int isatty(int); @@ -218,6 +225,9 @@ typedef unsigned char u8; #define IsAlpha(X) isalpha((unsigned char)X) #if defined(_WIN32) || defined(WIN32) +#if SQLITE_OS_WINRT +#include <intrin.h> +#endif #undef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #include <windows.h> @@ -229,8 +239,6 @@ extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); INCLUDE ../ext/misc/sqlite3_stdio.h INCLUDE ../ext/misc/sqlite3_stdio.c -INCLUDE ../ext/qrf/qrf.h -INCLUDE ../ext/qrf/qrf.c /* Use console I/O package as a direct INCLUDE. */ #define SQLITE_INTERNAL_LINKAGE static @@ -244,67 +252,11 @@ INCLUDE ../ext/qrf/qrf.c # define SQLITE_CIO_NO_FLUSH #endif -/* -** Output routines that are able to redirect to memory rather than -** doing actually I/O. -** Works like. -** -------------- -** cli_printf(FILE*, const char*, ...); fprintf() -** cli_puts(const char*, FILE*); fputs() -** cli_vprintf(FILE*, const char*, va_list); vfprintf() -** -** These are just thin wrappers with the following added semantics: -** If the file-scope variable cli_output_capture is not NULL, and -** if the FILE* argument is stdout or stderr, then rather than -** writing to stdout/stdout, append the text to the cli_output_capture -** variable. -** -** The cli_exit(int) routine works like exit() except that it -** first dumps any capture output to stdout. -*/ -static sqlite3_str *cli_output_capture = 0; -static int cli_printf(FILE *out, const char *zFormat, ...){ - va_list ap; - int rc; - va_start(ap,zFormat); - if( cli_output_capture && (out==stdout || out==stderr) ){ - sqlite3_str_vappendf(cli_output_capture, zFormat, ap); - rc = 1; - }else{ - rc = sqlite3_vfprintf(out, zFormat, ap); - } - va_end(ap); - return rc; -} -static int cli_puts(const char *zText, FILE *out){ - if( cli_output_capture && (out==stdout || out==stderr) ){ - sqlite3_str_appendall(cli_output_capture, zText); - return 1; - } - return sqlite3_fputs(zText, out); -} -#if 0 /* Not currently used - available if we need it later */ -static int cli_vprintf(FILE *out, const char *zFormat, va_list ap){ - if( cli_output_capture && (out==stdout || out==stderr) ){ - sqlite3_str_vappendf(cli_output_capture, zFormat, ap); - return 1; - }else{ - return sqlite3_vfprintf(out, zFormat, ap); - } -} -#endif -static void cli_exit(int rc){ - if( cli_output_capture ){ - char *z = sqlite3_str_finish(cli_output_capture); - sqlite3_fputs(z, stdout); - fflush(stdout); - } - exit(rc); -} - +#define eputz(z) sqlite3_fputs(z,stderr) +#define sputz(fp,z) sqlite3_fputs(z,fp) -#define eputz(z) cli_puts(z,stderr) -#define sputz(fp,z) cli_puts(z,fp) +/* True if the timer is enabled */ +static int enableTimer = 0; /* A version of strcmp() that works with NULL values */ static int cli_strcmp(const char *a, const char *b){ @@ -322,7 +274,7 @@ static int cli_strncmp(const char *a, const char *b, size_t n){ ** Unix epoch (1970-01-01T00:00:00Z) */ static sqlite3_int64 timeOfDay(void){ -#if defined(_WIN64) && _WIN32_WINNT >= _WIN32_WINNT_WIN8 +#if defined(_WIN64) sqlite3_uint64 t; FILETIME tm; GetSystemTimePreciseAsFileTime(&tm); @@ -350,6 +302,152 @@ static sqlite3_int64 timeOfDay(void){ #endif } +#if !defined(_WIN32) && !defined(WIN32) && !defined(__minux) +#include <sys/time.h> +#include <sys/resource.h> + +/* VxWorks does not support getrusage() as far as we can determine */ +#if defined(_WRS_KERNEL) || defined(__RTP__) +struct rusage { + struct timeval ru_utime; /* user CPU time used */ + struct timeval ru_stime; /* system CPU time used */ +}; +#define getrusage(A,B) memset(B,0,sizeof(*B)) +#endif + + +/* Saved resource information for the beginning of an operation */ +static struct rusage sBegin; /* CPU time at start */ +static sqlite3_int64 iBegin; /* Wall-clock time at start */ + +/* +** Begin timing an operation +*/ +static void beginTimer(void){ + if( enableTimer ){ + getrusage(RUSAGE_SELF, &sBegin); + iBegin = timeOfDay(); + } +} + +/* Return the difference of two time_structs in seconds */ +static double timeDiff(struct timeval *pStart, struct timeval *pEnd){ + return (pEnd->tv_usec - pStart->tv_usec)*0.000001 + + (double)(pEnd->tv_sec - pStart->tv_sec); +} + +/* +** Print the timing results. +*/ +static void endTimer(FILE *out){ + if( enableTimer ){ + sqlite3_int64 iEnd = timeOfDay(); + struct rusage sEnd; + getrusage(RUSAGE_SELF, &sEnd); + sqlite3_fprintf(out, "Run Time: real %.6f user %.6f sys %.6f\n", + (iEnd - iBegin)*0.000001, + timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), + timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); + } +} + +#define BEGIN_TIMER beginTimer() +#define END_TIMER(X) endTimer(X) +#define HAS_TIMER 1 + +#elif (defined(_WIN32) || defined(WIN32)) + +/* Saved resource information for the beginning of an operation */ +static HANDLE hProcess; +static FILETIME ftKernelBegin; +static FILETIME ftUserBegin; +static sqlite3_int64 ftWallBegin; +typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME, + LPFILETIME, LPFILETIME); +static GETPROCTIMES getProcessTimesAddr = NULL; + +/* +** Check to see if we have timer support. Return 1 if necessary +** support found (or found previously). +*/ +static int hasTimer(void){ + if( getProcessTimesAddr ){ + return 1; + } else { +#if !SQLITE_OS_WINRT + /* GetProcessTimes() isn't supported in WIN95 and some other Windows + ** versions. See if the version we are running on has it, and if it + ** does, save off a pointer to it and the current process handle. + */ + hProcess = GetCurrentProcess(); + if( hProcess ){ + HINSTANCE hinstLib = LoadLibrary(TEXT("Kernel32.dll")); + if( NULL != hinstLib ){ + getProcessTimesAddr = + (GETPROCTIMES) GetProcAddress(hinstLib, "GetProcessTimes"); + if( NULL != getProcessTimesAddr ){ + return 1; + } + FreeLibrary(hinstLib); + } + } +#endif + } + return 0; +} + +/* +** Begin timing an operation +*/ +static void beginTimer(void){ + if( enableTimer && getProcessTimesAddr ){ + FILETIME ftCreation, ftExit; + getProcessTimesAddr(hProcess,&ftCreation,&ftExit, + &ftKernelBegin,&ftUserBegin); + ftWallBegin = timeOfDay(); + } +} + +/* Return the difference of two FILETIME structs in seconds */ +static double timeDiff(FILETIME *pStart, FILETIME *pEnd){ + sqlite_int64 i64Start = *((sqlite_int64 *) pStart); + sqlite_int64 i64End = *((sqlite_int64 *) pEnd); + return (double) ((i64End - i64Start) / 10000000.0); +} + +/* +** Print the timing results. +*/ +static void endTimer(FILE *out){ + if( enableTimer && getProcessTimesAddr){ + FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; + sqlite3_int64 ftWallEnd = timeOfDay(); + getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); +#ifdef _WIN64 + /* microsecond precision on 64-bit windows */ + sqlite3_fprintf(out, "Run Time: real %.6f user %f sys %f\n", + (ftWallEnd - ftWallBegin)*0.000001, + timeDiff(&ftUserBegin, &ftUserEnd), + timeDiff(&ftKernelBegin, &ftKernelEnd)); +#else + /* millisecond precisino on 32-bit windows */ + sqlite3_fprintf(out, "Run Time: real %.3f user %.3f sys %.3f\n", + (ftWallEnd - ftWallBegin)*0.000001, + timeDiff(&ftUserBegin, &ftUserEnd), + timeDiff(&ftKernelBegin, &ftKernelEnd)); +#endif + } +} + +#define BEGIN_TIMER beginTimer() +#define END_TIMER(X) endTimer(X) +#define HAS_TIMER hasTimer() + +#else +#define BEGIN_TIMER +#define END_TIMER(X) /*no-op*/ +#define HAS_TIMER 0 +#endif /* ** Used to prevent warnings about unused parameters @@ -374,17 +472,12 @@ static int bail_on_error = 0; static int stdin_is_interactive = 1; /* -** Treat stdout like a TTY if true. +** On Windows systems we need to know if standard output is a console +** in order to show that UTF-16 translation is done in the sign-on +** banner. The following variable is true if it is the console. */ static int stdout_is_console = 1; -/* -** Use this value as the width of the output device. Or, figure it -** out at runtime if the value is negative. Or use a default width -** if this value is zero. -*/ -static int stdout_tty_width = -1; - /* ** The following is the open SQLite database. We make a pointer ** to this database a static variable so that it can be accessed @@ -413,14 +506,6 @@ static char mainPrompt[PROMPT_LEN_MAX]; /* Continuation prompt. default: " ...> " */ static char continuePrompt[PROMPT_LEN_MAX]; -/* -** Write I/O traces to the following stream. -*/ -#ifdef SQLITE_ENABLE_IOTRACE -static FILE *iotrace = 0; -#endif - - /* This is variant of the standard-library strncpy() routine with the ** one change that the destination string is always zero-terminated, even ** if there is no zero-terminator in the first n-1 characters of the source @@ -530,7 +615,7 @@ static char *dynamicContinuePrompt(void){ /* Indicate out-of-memory and exit. */ static void shell_out_of_memory(void){ eputz("Error: out of memory\n"); - cli_exit(1); + exit(1); } /* Check a pointer to see if it is NULL. If it is NULL, exit with an @@ -540,6 +625,13 @@ static void shell_check_oom(const void *p){ if( p==0 ) shell_out_of_memory(); } +/* +** Write I/O traces to the following stream. +*/ +#ifdef SQLITE_ENABLE_IOTRACE +static FILE *iotrace = 0; +#endif + /* ** This routine works like printf in that its first argument is a ** format string and subsequent arguments are values to be substituted @@ -554,91 +646,373 @@ static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){ va_start(ap, zFormat); z = sqlite3_vmprintf(zFormat, ap); va_end(ap); - cli_printf(iotrace, "%s", z); + sqlite3_fprintf(iotrace, "%s", z); sqlite3_free(z); } #endif +/* Lookup table to estimate the number of columns consumed by a Unicode +** character. +*/ +static const struct { + unsigned char w; /* Width of the character in columns */ + int iFirst; /* First character in a span having this width */ +} aUWidth[] = { + /* {1, 0x00000}, */ + {0, 0x00300}, {1, 0x00370}, {0, 0x00483}, {1, 0x00487}, {0, 0x00488}, + {1, 0x0048a}, {0, 0x00591}, {1, 0x005be}, {0, 0x005bf}, {1, 0x005c0}, + {0, 0x005c1}, {1, 0x005c3}, {0, 0x005c4}, {1, 0x005c6}, {0, 0x005c7}, + {1, 0x005c8}, {0, 0x00600}, {1, 0x00604}, {0, 0x00610}, {1, 0x00616}, + {0, 0x0064b}, {1, 0x0065f}, {0, 0x00670}, {1, 0x00671}, {0, 0x006d6}, + {1, 0x006e5}, {0, 0x006e7}, {1, 0x006e9}, {0, 0x006ea}, {1, 0x006ee}, + {0, 0x0070f}, {1, 0x00710}, {0, 0x00711}, {1, 0x00712}, {0, 0x00730}, + {1, 0x0074b}, {0, 0x007a6}, {1, 0x007b1}, {0, 0x007eb}, {1, 0x007f4}, + {0, 0x00901}, {1, 0x00903}, {0, 0x0093c}, {1, 0x0093d}, {0, 0x00941}, + {1, 0x00949}, {0, 0x0094d}, {1, 0x0094e}, {0, 0x00951}, {1, 0x00955}, + {0, 0x00962}, {1, 0x00964}, {0, 0x00981}, {1, 0x00982}, {0, 0x009bc}, + {1, 0x009bd}, {0, 0x009c1}, {1, 0x009c5}, {0, 0x009cd}, {1, 0x009ce}, + {0, 0x009e2}, {1, 0x009e4}, {0, 0x00a01}, {1, 0x00a03}, {0, 0x00a3c}, + {1, 0x00a3d}, {0, 0x00a41}, {1, 0x00a43}, {0, 0x00a47}, {1, 0x00a49}, + {0, 0x00a4b}, {1, 0x00a4e}, {0, 0x00a70}, {1, 0x00a72}, {0, 0x00a81}, + {1, 0x00a83}, {0, 0x00abc}, {1, 0x00abd}, {0, 0x00ac1}, {1, 0x00ac6}, + {0, 0x00ac7}, {1, 0x00ac9}, {0, 0x00acd}, {1, 0x00ace}, {0, 0x00ae2}, + {1, 0x00ae4}, {0, 0x00b01}, {1, 0x00b02}, {0, 0x00b3c}, {1, 0x00b3d}, + {0, 0x00b3f}, {1, 0x00b40}, {0, 0x00b41}, {1, 0x00b44}, {0, 0x00b4d}, + {1, 0x00b4e}, {0, 0x00b56}, {1, 0x00b57}, {0, 0x00b82}, {1, 0x00b83}, + {0, 0x00bc0}, {1, 0x00bc1}, {0, 0x00bcd}, {1, 0x00bce}, {0, 0x00c3e}, + {1, 0x00c41}, {0, 0x00c46}, {1, 0x00c49}, {0, 0x00c4a}, {1, 0x00c4e}, + {0, 0x00c55}, {1, 0x00c57}, {0, 0x00cbc}, {1, 0x00cbd}, {0, 0x00cbf}, + {1, 0x00cc0}, {0, 0x00cc6}, {1, 0x00cc7}, {0, 0x00ccc}, {1, 0x00cce}, + {0, 0x00ce2}, {1, 0x00ce4}, {0, 0x00d41}, {1, 0x00d44}, {0, 0x00d4d}, + {1, 0x00d4e}, {0, 0x00dca}, {1, 0x00dcb}, {0, 0x00dd2}, {1, 0x00dd5}, + {0, 0x00dd6}, {1, 0x00dd7}, {0, 0x00e31}, {1, 0x00e32}, {0, 0x00e34}, + {1, 0x00e3b}, {0, 0x00e47}, {1, 0x00e4f}, {0, 0x00eb1}, {1, 0x00eb2}, + {0, 0x00eb4}, {1, 0x00eba}, {0, 0x00ebb}, {1, 0x00ebd}, {0, 0x00ec8}, + {1, 0x00ece}, {0, 0x00f18}, {1, 0x00f1a}, {0, 0x00f35}, {1, 0x00f36}, + {0, 0x00f37}, {1, 0x00f38}, {0, 0x00f39}, {1, 0x00f3a}, {0, 0x00f71}, + {1, 0x00f7f}, {0, 0x00f80}, {1, 0x00f85}, {0, 0x00f86}, {1, 0x00f88}, + {0, 0x00f90}, {1, 0x00f98}, {0, 0x00f99}, {1, 0x00fbd}, {0, 0x00fc6}, + {1, 0x00fc7}, {0, 0x0102d}, {1, 0x01031}, {0, 0x01032}, {1, 0x01033}, + {0, 0x01036}, {1, 0x01038}, {0, 0x01039}, {1, 0x0103a}, {0, 0x01058}, + {1, 0x0105a}, {2, 0x01100}, {0, 0x01160}, {1, 0x01200}, {0, 0x0135f}, + {1, 0x01360}, {0, 0x01712}, {1, 0x01715}, {0, 0x01732}, {1, 0x01735}, + {0, 0x01752}, {1, 0x01754}, {0, 0x01772}, {1, 0x01774}, {0, 0x017b4}, + {1, 0x017b6}, {0, 0x017b7}, {1, 0x017be}, {0, 0x017c6}, {1, 0x017c7}, + {0, 0x017c9}, {1, 0x017d4}, {0, 0x017dd}, {1, 0x017de}, {0, 0x0180b}, + {1, 0x0180e}, {0, 0x018a9}, {1, 0x018aa}, {0, 0x01920}, {1, 0x01923}, + {0, 0x01927}, {1, 0x01929}, {0, 0x01932}, {1, 0x01933}, {0, 0x01939}, + {1, 0x0193c}, {0, 0x01a17}, {1, 0x01a19}, {0, 0x01b00}, {1, 0x01b04}, + {0, 0x01b34}, {1, 0x01b35}, {0, 0x01b36}, {1, 0x01b3b}, {0, 0x01b3c}, + {1, 0x01b3d}, {0, 0x01b42}, {1, 0x01b43}, {0, 0x01b6b}, {1, 0x01b74}, + {0, 0x01dc0}, {1, 0x01dcb}, {0, 0x01dfe}, {1, 0x01e00}, {0, 0x0200b}, + {1, 0x02010}, {0, 0x0202a}, {1, 0x0202f}, {0, 0x02060}, {1, 0x02064}, + {0, 0x0206a}, {1, 0x02070}, {0, 0x020d0}, {1, 0x020f0}, {2, 0x02329}, + {1, 0x0232b}, {2, 0x02e80}, {0, 0x0302a}, {2, 0x03030}, {1, 0x0303f}, + {2, 0x03040}, {0, 0x03099}, {2, 0x0309b}, {1, 0x0a4d0}, {0, 0x0a806}, + {1, 0x0a807}, {0, 0x0a80b}, {1, 0x0a80c}, {0, 0x0a825}, {1, 0x0a827}, + {2, 0x0ac00}, {1, 0x0d7a4}, {2, 0x0f900}, {1, 0x0fb00}, {0, 0x0fb1e}, + {1, 0x0fb1f}, {0, 0x0fe00}, {2, 0x0fe10}, {1, 0x0fe1a}, {0, 0x0fe20}, + {1, 0x0fe24}, {2, 0x0fe30}, {1, 0x0fe70}, {0, 0x0feff}, {2, 0x0ff00}, + {1, 0x0ff61}, {2, 0x0ffe0}, {1, 0x0ffe7}, {0, 0x0fff9}, {1, 0x0fffc}, + {0, 0x10a01}, {1, 0x10a04}, {0, 0x10a05}, {1, 0x10a07}, {0, 0x10a0c}, + {1, 0x10a10}, {0, 0x10a38}, {1, 0x10a3b}, {0, 0x10a3f}, {1, 0x10a40}, + {0, 0x1d167}, {1, 0x1d16a}, {0, 0x1d173}, {1, 0x1d183}, {0, 0x1d185}, + {1, 0x1d18c}, {0, 0x1d1aa}, {1, 0x1d1ae}, {0, 0x1d242}, {1, 0x1d245}, + {2, 0x20000}, {1, 0x2fffe}, {2, 0x30000}, {1, 0x3fffe}, {0, 0xe0001}, + {1, 0xe0002}, {0, 0xe0020}, {1, 0xe0080}, {0, 0xe0100}, {1, 0xe01f0} +}; + /* -** Compute a string length that is limited to what can be stored in -** lower 30 bits of a 32-bit signed integer. -*/ -static int strlen30(const char *z){ - size_t n; - if( z==0 ) return 0; - n = strlen(z); - return n>0x3fffffff ? 0x3fffffff : (int)n; +** Return an estimate of the width, in columns, for the single Unicode +** character c. For normal characters, the answer is always 1. But the +** estimate might be 0 or 2 for zero-width and double-width characters. +** +** Different display devices display unicode using different widths. So +** it is impossible to know that true display width with 100% accuracy. +** Inaccuracies in the width estimates might cause columns to be misaligned. +** Unfortunately, there is nothing we can do about that. +*/ +int cli_wcwidth(int c){ + int iFirst, iLast; + + /* Fast path for common characters */ + if( c<=0x300 ) return 1; + + /* The general case */ + iFirst = 0; + iLast = sizeof(aUWidth)/sizeof(aUWidth[0]) - 1; + while( iFirst<iLast-1 ){ + int iMid = (iFirst+iLast)/2; + int cMid = aUWidth[iMid].iFirst; + if( cMid < c ){ + iFirst = iMid; + }else if( cMid > c ){ + iLast = iMid - 1; + }else{ + return aUWidth[iMid].w; + } + } + if( aUWidth[iLast].iFirst > c ) return aUWidth[iFirst].w; + return aUWidth[iLast].w; } /* -** Return open FILE * if zFile exists, can be opened for read -** and is an ordinary file or a character stream source. -** Otherwise return 0. +** Compute the value and length of a multi-byte UTF-8 character that +** begins at z[0]. Return the length. Write the Unicode value into *pU. +** +** This routine only works for *multi-byte* UTF-8 characters. */ -static FILE * openChrSource(const char *zFile){ -#if defined(_WIN32) || defined(WIN32) - struct __stat64 x = {0}; -# define STAT_CHR_SRC(mode) ((mode & (_S_IFCHR|_S_IFIFO|_S_IFREG))!=0) - /* On Windows, open first, then check the stream nature. This order - ** is necessary because _stat() and sibs, when checking a named pipe, - ** effectively break the pipe as its supplier sees it. */ - FILE *rv = sqlite3_fopen(zFile, "rb"); - if( rv==0 ) return 0; - if( _fstat64(_fileno(rv), &x) != 0 - || !STAT_CHR_SRC(x.st_mode)){ - fclose(rv); - rv = 0; +static int decodeUtf8(const unsigned char *z, int *pU){ + if( (z[0] & 0xe0)==0xc0 && (z[1] & 0xc0)==0x80 ){ + *pU = ((z[0] & 0x1f)<<6) | (z[1] & 0x3f); + return 2; } - return rv; -#else - struct stat x = {0}; - int rc = stat(zFile, &x); -# define STAT_CHR_SRC(mode) (S_ISREG(mode)||S_ISFIFO(mode)||S_ISCHR(mode)) - if( rc!=0 ) return 0; - if( STAT_CHR_SRC(x.st_mode) ){ - return sqlite3_fopen(zFile, "rb"); - }else{ - return 0; + if( (z[0] & 0xf0)==0xe0 && (z[1] & 0xc0)==0x80 && (z[2] & 0xc0)==0x80 ){ + *pU = ((z[0] & 0x0f)<<12) | ((z[1] & 0x3f)<<6) | (z[2] & 0x3f); + return 3; } -#endif -#undef STAT_CHR_SRC + if( (z[0] & 0xf8)==0xf0 && (z[1] & 0xc0)==0x80 && (z[2] & 0xc0)==0x80 + && (z[3] & 0xc0)==0x80 + ){ + *pU = ((z[0] & 0x0f)<<18) | ((z[1] & 0x3f)<<12) | ((z[2] & 0x3f))<<6 + | (z[3] & 0x3f); + return 4; + } + *pU = 0; + return 1; } + +#if 0 /* NOT USED */ /* -** This routine reads a line of text from FILE in, stores -** the text in memory obtained from malloc() and returns a pointer -** to the text. NULL is returned at end of file, or if malloc() -** fails, or if the length of the line is longer than about a gigabyte. +** Return the width, in display columns, of a UTF-8 string. ** -** If zLine is not NULL then it is a malloced buffer returned from -** a previous call to this routine that may be reused. +** Each normal character counts as 1. Zero-width characters count +** as zero, and double-width characters count as 2. */ -static char *local_getline(char *zLine, FILE *in){ - int nLine = zLine==0 ? 0 : 100; +int cli_wcswidth(const char *z){ + const unsigned char *a = (const unsigned char*)z; int n = 0; - - while( 1 ){ - if( n+100>nLine ){ - if( nLine>=1073741773 ){ - free(zLine); - return 0; - } - nLine = nLine*2 + 100; - zLine = realloc(zLine, nLine); - shell_check_oom(zLine); - } - if( sqlite3_fgets(&zLine[n], nLine - n, in)==0 ){ - if( n==0 ){ - free(zLine); - return 0; - } - zLine[n] = 0; - break; + int i = 0; + unsigned char c; + while( (c = a[i])!=0 ){ + if( c>=0xc0 ){ + int u; + int len = decodeUtf8(&a[i], &u); + i += len; + n += cli_wcwidth(u); + }else if( c>=' ' ){ + n++; + i++; + }else{ + i++; } - while( zLine[n] ) n++; - if( n>0 && zLine[n-1]=='\n' ){ - n--; - if( n>0 && zLine[n-1]=='\r' ) n--; + } + return n; +} +#endif + +/* +** Check to see if z[] is a valid VT100 escape. If it is, then +** return the number of bytes in the escape sequence. Return 0 if +** z[] is not a VT100 escape. +** +** This routine assumes that z[0] is \033 (ESC). +*/ +static int isVt100(const unsigned char *z){ + int i; + if( z[1]!='[' ) return 0; + i = 2; + while( z[i]>=0x30 && z[i]<=0x3f ){ i++; } + while( z[i]>=0x20 && z[i]<=0x2f ){ i++; } + if( z[i]<0x40 || z[i]>0x7e ) return 0; + return i+1; +} + +/* +** Output string zUtf to stdout as w characters. If w is negative, +** then right-justify the text. W is the width in UTF-8 characters, not +** in bytes. This is different from the %*.*s specification in printf +** since with %*.*s the width is measured in bytes, not characters. +** +** Take into account zero-width and double-width Unicode characters. +** In other words, a zero-width character does not count toward the +** the w limit. A double-width character counts as two. +** +** w should normally be a small number. A couple hundred at most. This +** routine caps w at 100 million to avoid integer overflow issues. +*/ +static void utf8_width_print(FILE *out, int w, const char *zUtf){ + const unsigned char *a = (const unsigned char*)zUtf; + static const int mxW = 10000000; + unsigned char c; + int i = 0; + int n = 0; + int k; + int aw; + if( w<-mxW ){ + w = -mxW; + }else if( w>mxW ){ + w= mxW; + } + aw = w<0 ? -w : w; + if( zUtf==0 ) zUtf = ""; + while( (c = a[i])!=0 ){ + if( (c&0xc0)==0xc0 ){ + int u; + int len = decodeUtf8(a+i, &u); + int x = cli_wcwidth(u); + if( x+n>aw ){ + break; + } + i += len; + n += x; + }else if( c==0x1b && (k = isVt100(&a[i]))>0 ){ + i += k; + }else if( n>=aw ){ + break; + }else{ + n++; + i++; + } + } + if( n>=aw ){ + sqlite3_fprintf(out, "%.*s", i, zUtf); + }else if( w<0 ){ + sqlite3_fprintf(out, "%*s%s", aw-n, "", zUtf); + }else{ + sqlite3_fprintf(out, "%s%*s", zUtf, aw-n, ""); + } +} + + +/* +** Determines if a string is a number of not. +*/ +static int isNumber(const char *z, int *realnum){ + if( *z=='-' || *z=='+' ) z++; + if( !IsDigit(*z) ){ + return 0; + } + z++; + if( realnum ) *realnum = 0; + while( IsDigit(*z) ){ z++; } + if( *z=='.' ){ + z++; + if( !IsDigit(*z) ) return 0; + while( IsDigit(*z) ){ z++; } + if( realnum ) *realnum = 1; + } + if( *z=='e' || *z=='E' ){ + z++; + if( *z=='+' || *z=='-' ) z++; + if( !IsDigit(*z) ) return 0; + while( IsDigit(*z) ){ z++; } + if( realnum ) *realnum = 1; + } + return *z==0; +} + +/* +** Compute a string length that is limited to what can be stored in +** lower 30 bits of a 32-bit signed integer. +*/ +static int strlen30(const char *z){ + const char *z2 = z; + while( *z2 ){ z2++; } + return 0x3fffffff & (int)(z2 - z); +} + +/* +** Return the length of a string in characters. Multibyte UTF8 characters +** count as a single character for single-width characters, or as two +** characters for double-width characters. +*/ +static int strlenChar(const char *z){ + int n = 0; + while( *z ){ + if( (0x80&z[0])==0 ){ + n++; + z++; + }else{ + int u = 0; + int len = decodeUtf8((const u8*)z, &u); + z += len; + n += cli_wcwidth(u); + } + } + return n; +} + +/* +** Return open FILE * if zFile exists, can be opened for read +** and is an ordinary file or a character stream source. +** Otherwise return 0. +*/ +static FILE * openChrSource(const char *zFile){ +#if defined(_WIN32) || defined(WIN32) + struct __stat64 x = {0}; +# define STAT_CHR_SRC(mode) ((mode & (_S_IFCHR|_S_IFIFO|_S_IFREG))!=0) + /* On Windows, open first, then check the stream nature. This order + ** is necessary because _stat() and sibs, when checking a named pipe, + ** effectively break the pipe as its supplier sees it. */ + FILE *rv = sqlite3_fopen(zFile, "rb"); + if( rv==0 ) return 0; + if( _fstat64(_fileno(rv), &x) != 0 + || !STAT_CHR_SRC(x.st_mode)){ + fclose(rv); + rv = 0; + } + return rv; +#else + struct stat x = {0}; + int rc = stat(zFile, &x); +# define STAT_CHR_SRC(mode) (S_ISREG(mode)||S_ISFIFO(mode)||S_ISCHR(mode)) + if( rc!=0 ) return 0; + if( STAT_CHR_SRC(x.st_mode) ){ + return sqlite3_fopen(zFile, "rb"); + }else{ + return 0; + } +#endif +#undef STAT_CHR_SRC +} + +/* +** This routine reads a line of text from FILE in, stores +** the text in memory obtained from malloc() and returns a pointer +** to the text. NULL is returned at end of file, or if malloc() +** fails, or if the length of the line is longer than about a gigabyte. +** +** If zLine is not NULL then it is a malloced buffer returned from +** a previous call to this routine that may be reused. +*/ +static char *local_getline(char *zLine, FILE *in){ + int nLine = zLine==0 ? 0 : 100; + int n = 0; + + while( 1 ){ + if( n+100>nLine ){ + if( nLine>=1073741773 ){ + free(zLine); + return 0; + } + nLine = nLine*2 + 100; + zLine = realloc(zLine, nLine); + shell_check_oom(zLine); + } + if( sqlite3_fgets(&zLine[n], nLine - n, in)==0 ){ + if( n==0 ){ + free(zLine); + return 0; + } + zLine[n] = 0; + break; + } + while( zLine[n] ) n++; + if( n>0 && zLine[n-1]=='\n' ){ + n--; + if( n>0 && zLine[n-1]=='\r' ) n--; zLine[n] = 0; break; } @@ -931,7 +1305,7 @@ static void shellDtostr( char z[400]; if( n<1 ) n = 1; if( n>350 ) n = 350; - sprintf(z, "%#+.*e", n, r); + sqlite3_snprintf(sizeof(z), z, "%#+.*e", n, r); sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT); } @@ -1089,24 +1463,31 @@ struct ExpertInfo { }; #endif -/* All the parameters that determine how to render query results. -*/ -typedef struct Mode { - u8 autoExplain; /* Automatically turn on .explain mode */ - u8 autoEQP; /* Run EXPLAIN QUERY PLAN prior to each SQL stmt */ - u8 autoEQPtrace; /* autoEQP is in trace mode */ - u8 scanstatsOn; /* True to display scan stats before each finalize */ - u8 bAutoScreenWidth; /* Using the TTY to determine screen width */ - u8 mFlags; /* MFLG_ECHO, MFLG_CRLF, etc. */ - u8 eMode; /* One of the MODE_ values */ - sqlite3_qrf_spec spec; /* Spec to be passed into QRF */ -} Mode; +/* A single line in the EQP output */ +typedef struct EQPGraphRow EQPGraphRow; +struct EQPGraphRow { + int iEqpId; /* ID for this row */ + int iParentId; /* ID of the parent row */ + EQPGraphRow *pNext; /* Next row in sequence */ + char zText[1]; /* Text to display for this row */ +}; -/* Flags for Mode.mFlags */ -#define MFLG_ECHO 0x01 /* Echo inputs to output */ -#define MFLG_CRLF 0x02 /* Use CR/LF output line endings */ -#define MFLG_HDR 0x04 /* .header used to change headers on/off */ +/* All EQP output is collected into an instance of the following */ +typedef struct EQPGraph EQPGraph; +struct EQPGraph { + EQPGraphRow *pRow; /* Linked list of all rows of the EQP output */ + EQPGraphRow *pLast; /* Last element of the pRow list */ + char zPrefix[100]; /* Graph prefix */ +}; +/* Parameters affecting columnar mode result display (defaulting together) */ +typedef struct ColModeOpts { + int iWrap; /* In columnar modes, wrap lines reaching this limit */ + u8 bQuote; /* Quote results for .mode box and table */ + u8 bWordWrap; /* In columnar modes, wrap at word boundaries */ +} ColModeOpts; +#define ColModeOpts_default { 60, 0, 0 } +#define ColModeOpts_default_qbox { 60, 1, 0 } /* ** State information about the database connection is contained in an @@ -1115,6 +1496,11 @@ typedef struct Mode { typedef struct ShellState ShellState; struct ShellState { sqlite3 *db; /* The database */ + u8 autoExplain; /* Automatically turn on .explain mode */ + u8 autoEQP; /* Run EXPLAIN QUERY PLAN prior to each SQL stmt */ + u8 autoEQPtest; /* autoEQP is in test mode */ + u8 autoEQPtrace; /* autoEQP is in trace mode */ + u8 scanstatsOn; /* True to display scan stats before each finalize */ u8 openMode; /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */ u8 doXdgOpen; /* Invoke start/open/xdg-open in output_reset() */ u8 nEqpLevel; /* Depth of the EQP output graph */ @@ -1122,44 +1508,48 @@ struct ShellState { u8 bSafeMode; /* True to prohibit unsafe operations */ u8 bSafeModePersist; /* The long-term value of bSafeMode */ u8 eRestoreState; /* See comments above doAutoDetectRestore() */ + u8 crlfMode; /* Do NL-to-CRLF translations when enabled (maybe) */ + u8 eEscMode; /* Escape mode for text output */ + ColModeOpts cmOpts; /* Option values affecting columnar mode output */ unsigned statsOn; /* True to display memory stats before each finalize */ unsigned mEqpLines; /* Mask of vertical lines in the EQP output graph */ - u8 nPopOutput; /* Revert .output settings when reaching zero */ - u8 nPopMode; /* Revert .mode settings when reaching zero */ - u8 enableTimer; /* Enable the timer. 2: permanently 1: only once */ int inputNesting; /* Track nesting level of .read and other redirects */ - double prevTimer; /* Last reported timer value */ - double tmProgress; /* --timeout option for .progress */ + int outCount; /* Revert to stdout when reaching zero */ + int cnt; /* Number of records displayed so far */ i64 lineno; /* Line number of last line read from in */ - const char *zInFile; /* Name of the input file */ int openFlags; /* Additional flags to open. (SQLITE_OPEN_NOFOLLOW) */ FILE *in; /* Read commands from this stream */ FILE *out; /* Write results here */ FILE *traceOut; /* Output for sqlite3_trace() */ int nErr; /* Number of errors seen */ + int mode; /* An output mode setting */ + int modePrior; /* Saved mode */ + int cMode; /* temporary output mode for the current query */ + int normalMode; /* Output mode before ".explain on" */ int writableSchema; /* True if PRAGMA writable_schema=ON */ + int showHeader; /* True to show column names in List or Column mode */ int nCheck; /* Number of ".check" commands run */ unsigned nProgress; /* Number of progress callbacks encountered */ unsigned mxProgress; /* Maximum progress callbacks before failing */ unsigned flgProgress; /* Flags for the progress callback */ unsigned shellFlgs; /* Various flags */ - unsigned nTestRun; /* Number of test cases run */ - unsigned nTestErr; /* Number of test cases that failed */ + unsigned priorShFlgs; /* Saved copy of flags */ sqlite3_int64 szMax; /* --maxsize argument to .open */ char *zDestTable; /* Name of destination table when MODE_Insert */ char *zTempFile; /* Temporary file that might need deleting */ - char *zErrPrefix; /* Alternative error message prefix */ char zTestcase[30]; /* Name of current test case */ + char colSeparator[20]; /* Column separator character for several modes */ + char rowSeparator[20]; /* Row separator character for MODE_Ascii */ + char colSepPrior[20]; /* Saved column separator */ + char rowSepPrior[20]; /* Saved row separator */ + int *colWidth; /* Requested width of each column in columnar modes */ + int *actualWidth; /* Actual width of each column */ + int nWidth; /* Number of slots in colWidth[] and actualWidth[] */ + char nullValue[20]; /* The text to print when a NULL comes back from + ** the database */ char outfile[FILENAME_MAX]; /* Filename for *out */ sqlite3_stmt *pStmt; /* Current statement if any. */ FILE *pLog; /* Write log output here */ - Mode mode; /* Current display mode */ - Mode modePrior; /* Backup */ - struct SavedMode { /* Ability to define custom mode configurations */ - char *zTag; /* Name of this saved mode */ - Mode mode; /* The saved mode */ - } *aSavedModes; /* Array of saved .mode settings. system malloc() */ - int nSavedModes; /* Number of saved .mode settings */ struct AuxDb { /* Storage space for auxiliary database connections */ sqlite3 *db; /* Connection pointer */ const char *zDbFilename; /* Filename used to open the connection */ @@ -1170,19 +1560,14 @@ struct ShellState { #endif } aAuxDb[5], /* Array of all database connections */ *pAuxDb; /* Currently active database connection */ + int *aiIndent; /* Array of indents used in MODE_Explain */ + int nIndent; /* Size of array aiIndent[] */ + int iIndent; /* Index of current op in aiIndent[] */ char *zNonce; /* Nonce for temporary safe-mode escapes */ + EQPGraph sGraph; /* Information for the graphical EXPLAIN QUERY PLAN */ #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) ExpertInfo expert; /* Valid if previous command was ".expert OPT..." */ #endif - struct DotCmdLine { /* Info about arguments to a dot-command */ - const char *zOrig; /* Original text of the dot-command */ - char *zCopy; /* Copy of zOrig, from malloc() */ - int nAlloc; /* Size of allocates for arrays below */ - int nArg; /* Number of argument slots actually used */ - char **azArg; /* Pointer to each argument, dequoted */ - int *aiOfst; /* Offset into zOrig[] for start of each arg */ - char *abQuot; /* True if the argment was originally quoted */ - } dot; #ifdef SQLITE_SHELL_FIDDLE struct { const char * zInput; /* Input string from wasm/JS proxy */ @@ -1197,7 +1582,7 @@ static ShellState shellState; #endif -/* Allowed values for ShellState.mode.autoEQP +/* Allowed values for ShellState.autoEQP */ #define AUTOEQP_off 0 /* Automatic EXPLAIN QUERY PLAN is off */ #define AUTOEQP_on 1 /* Automatic EQP is on */ @@ -1225,13 +1610,15 @@ static ShellState shellState; ** callback limit is reached, and for each ** top-level SQL statement */ #define SHELL_PROGRESS_ONCE 0x04 /* Cancel the --limit after firing once */ -#define SHELL_PROGRESS_TMOUT 0x08 /* Stop after tmProgress seconds */ -/* Names of values for Mode.spec.eEsc and Mode.spec.eText +/* Allowed values for ShellState.eEscMode. The default value should +** be 0, so to change the default, reorder the names. */ -static const char *qrfEscNames[] = { "auto", "off", "ascii", "symbol" }; -static const char *qrfQuoteNames[] = - { "off","off","sql","hex","csv","tcl","json","relaxed"}; +#define SHELL_ESC_ASCII 0 /* Substitute ^Y for X where Y=X+0x40 */ +#define SHELL_ESC_SYMBOL 1 /* Substitute U+2400 graphics */ +#define SHELL_ESC_OFF 2 /* Send characters verbatim */ + +static const char *shell_EscModeNames[] = { "ascii", "symbol", "off" }; /* ** These are the allowed shellFlgs values @@ -1240,8 +1627,10 @@ static const char *qrfQuoteNames[] = #define SHFLG_Lookaside 0x00000002 /* Lookaside memory is used */ #define SHFLG_Backslash 0x00000004 /* The --backslash option is used */ #define SHFLG_PreserveRowid 0x00000008 /* .dump preserves rowid values */ -#define SHFLG_NoErrLineno 0x00000010 /* Omit line numbers from error msgs */ +#define SHFLG_Newlines 0x00000010 /* .dump --newline flag */ #define SHFLG_CountChanges 0x00000020 /* .changes setting */ +#define SHFLG_Echo 0x00000040 /* .echo on/off, or --echo setting */ +#define SHFLG_HeaderSet 0x00000080 /* showHeader has been specified */ #define SHFLG_DumpDataOnly 0x00000100 /* .dump show data only */ #define SHFLG_DumpNoSys 0x00000200 /* .dump omits system tables */ #define SHFLG_TestingMode 0x00000400 /* allow unsafe testing features */ @@ -1254,107 +1643,54 @@ static const char *qrfQuoteNames[] = #define ShellClearFlag(P,X) ((P)->shellFlgs&=(~(X))) /* -** These are the allowed values for Mode.eMode. There is a lot of overlap -** between these values and the Mode.spec.eStyle values, but they are not -** one-to-one, and thus need to be tracked separately. -*/ -#define MODE_Ascii 0 /* Use ASCII unit and record separators (0x1F/0x1E) */ -#define MODE_Box 1 /* Unicode box-drawing characters */ -#define MODE_C 2 /* Comma-separated list of C-strings */ -#define MODE_Column 3 /* One record per line in neat columns */ -#define MODE_Count 4 /* Output only a count of the rows of output */ -#define MODE_Csv 5 /* Quote strings, numbers are plain */ -#define MODE_Html 6 /* Generate an XHTML table */ -#define MODE_Insert 7 /* Generate SQL "insert" statements */ -#define MODE_JAtom 8 /* Comma-separated list of JSON atoms */ -#define MODE_JObject 9 /* One JSON object per row */ -#define MODE_Json 10 /* Output JSON */ -#define MODE_Line 11 /* One column per line. Blank line between records */ -#define MODE_List 12 /* One record per line with a separator */ -#define MODE_Markdown 13 /* Markdown formatting */ -#define MODE_Off 14 /* No query output shown */ -#define MODE_Psql 15 /* Similar to psql */ -#define MODE_QBox 16 /* BOX with SQL-quoted content */ -#define MODE_Quote 17 /* Quote values as for SQL */ -#define MODE_Split 18 /* Split-column mode */ -#define MODE_Table 19 /* MySQL-style table formatting */ -#define MODE_Tabs 20 /* Tab-separated values */ -#define MODE_Tcl 21 /* Space-separated list of TCL strings */ -#define MODE_Www 22 /* Full web-page output */ - -#define MODE_BUILTIN 22 /* Maximum built-in mode */ -#define MODE_BATCH 50 /* Default mode for batch processing */ -#define MODE_TTY 51 /* Default mode for interactive processing */ -#define MODE_USER 75 /* First user-defined mode */ -#define MODE_N_USER 25 /* Maximum number of user-defined modes */ - -/* -** Information about built-in display modes -*/ -typedef struct ModeInfo ModeInfo; -struct ModeInfo { - char zName[9]; /* Symbolic name of the mode */ - unsigned char eCSep; /* Column separator */ - unsigned char eRSep; /* Row separator */ - unsigned char eNull; /* Null representation */ - unsigned char eText; /* Default text encoding */ - unsigned char eHdr; /* Default header encoding. */ - unsigned char eBlob; /* Default blob encoding. */ - unsigned char bHdr; /* Show headers by default. 0: n/a, 1: no 2: yes */ - unsigned char eStyle; /* Underlying QRF style */ - unsigned char eCx; /* 0: other, 1: line, 2: columnar */ - unsigned char mFlg; /* Flags. 1=border-off 2=split-column */ +** These are the allowed modes. +*/ +#define MODE_Line 0 /* One column per line. Blank line between records */ +#define MODE_Column 1 /* One record per line in neat columns */ +#define MODE_List 2 /* One record per line with a separator */ +#define MODE_Semi 3 /* Same as MODE_List but append ";" to each line */ +#define MODE_Html 4 /* Generate an XHTML table */ +#define MODE_Insert 5 /* Generate SQL "insert" statements */ +#define MODE_Quote 6 /* Quote values as for SQL */ +#define MODE_Tcl 7 /* Generate ANSI-C or TCL quoted elements */ +#define MODE_Csv 8 /* Quote strings, numbers are plain */ +#define MODE_Explain 9 /* Like MODE_Column, but do not truncate data */ +#define MODE_Ascii 10 /* Use ASCII unit and record separators (0x1F/0x1E) */ +#define MODE_Pretty 11 /* Pretty-print schemas */ +#define MODE_EQP 12 /* Converts EXPLAIN QUERY PLAN output into a graph */ +#define MODE_Json 13 /* Output JSON */ +#define MODE_Markdown 14 /* Markdown formatting */ +#define MODE_Table 15 /* MySQL-style table formatting */ +#define MODE_Box 16 /* Unicode box-drawing characters */ +#define MODE_Count 17 /* Output only a count of the rows of output */ +#define MODE_Off 18 /* No query output shown */ +#define MODE_ScanExp 19 /* Like MODE_Explain, but for ".scanstats vm" */ +#define MODE_Www 20 /* Full web-page output */ + +static const char *modeDescr[] = { + "line", + "column", + "list", + "semi", + "html", + "insert", + "quote", + "tcl", + "csv", + "explain", + "ascii", + "prettyprint", + "eqp", + "json", + "markdown", + "table", + "box", + "count", + "off", + "scanexp", + "www", }; -/* String constants used by built-in modes */ -static const char *aModeStr[] = - /* 0 1 2 3 4 5 6 7 8 */ - { 0, "\n", "|", " ", ",", "\r\n", "\036", "\037", "\t", - "", "NULL", "null", "\"\"", ": ", }; - /* 9 10 11 12 13 */ - -static const ModeInfo aModeInfo[] = { -/* zName eCSep eRSep eNull eText eHdr eBlob bHdr eStyle eCx mFlg */ - { "ascii", 7, 6, 9, 1, 1, 0, 1, 12, 0, 0 }, - { "box", 0, 0, 9, 1, 1, 0, 2, 1, 2, 0 }, - { "c", 4, 1, 10, 5, 5, 4, 1, 12, 0, 0 }, - { "column", 0, 0, 9, 1, 1, 0, 2, 2, 2, 0 }, - { "count", 0, 0, 0, 0, 0, 0, 0, 3, 0, 0 }, - { "csv", 4, 5, 9, 3, 3, 0, 1, 12, 0, 0 }, - { "html", 0, 0, 9, 4, 4, 0, 2, 7, 0, 0 }, - { "insert", 0, 0, 10, 2, 2, 0, 1, 8, 0, 0 }, - { "jatom", 4, 1, 11, 6, 6, 0, 1, 12, 0, 0 }, - { "jobject", 0, 1, 11, 6, 6, 0, 0, 10, 0, 0 }, - { "json", 0, 0, 11, 6, 6, 0, 0, 9, 0, 0 }, - { "line", 13, 1, 9, 1, 1, 0, 0, 11, 1, 0 }, - { "list", 2, 1, 9, 1, 1, 0, 1, 12, 0, 0 }, - { "markdown", 0, 0, 9, 1, 1, 0, 2, 13, 2, 0 }, - { "off", 0, 0, 0, 0, 0, 0, 0, 14, 0, 0 }, - { "psql", 0, 0, 9, 1, 1, 0, 2, 19, 2, 1 }, - { "qbox", 0, 0, 10, 2, 1, 0, 2, 1, 2, 0 }, - { "quote", 4, 1, 10, 2, 2, 0, 1, 12, 0, 0 }, - { "split", 0, 0, 9, 1, 1, 0, 1, 2, 2, 2 }, - { "table", 0, 0, 9, 1, 1, 0, 2, 19, 2, 0 }, - { "tabs", 8, 1, 9, 3, 3, 0, 1, 12, 0, 0 }, - { "tcl", 3, 1, 12, 5, 5, 4, 1, 12, 0, 0 }, - { "www", 0, 0, 9, 4, 4, 0, 2, 7, 0, 0 } -}; /* | / / | / / | | \ - ** | / / | / / | | \_ 2: columnar - ** Index into aModeStr[] | / / | | 1: line - ** | / / | | 0: other - ** | / / | \ - ** text encoding |/ | show | \ - ** v-------------------' | hdrs? | The QRF style - ** 0: n/a blob | v-----' - ** 1: plain v_---------' 0: n/a - ** 2: sql 0: auto 1: no - ** 3: csv 1: as-text 2: yes - ** 4: html 2: sql - ** 5: c 3: hex - ** 6: json 4: c - ** 5: json - ** 6: size - ******************************************************************/ /* ** These are the column/row/line separators used by the various ** import/export modes. @@ -1368,480 +1704,59 @@ static const ModeInfo aModeInfo[] = { #define SEP_Unit "\x1F" #define SEP_Record "\x1E" -/* -** Default values for the various QRF limits -*/ -#ifndef DFLT_CHAR_LIMIT -# define DFLT_CHAR_LIMIT 300 -#endif -#ifndef DFLT_LINE_LIMIT -# define DFLT_LINE_LIMIT 5 -#endif -#ifndef DFLT_TITLE_LIMIT -# define DFLT_TITLE_LIMIT 20 -#endif - /* ** Limit input nesting via .read or any other input redirect. ** It's not too expensive, so a generous allowance can be made. */ #define MAX_INPUT_NESTING 25 -/************************* BEGIN PERFORMANCE TIMER *****************************/ -#if !defined(_WIN32) && !defined(WIN32) && !defined(__minux) -#include <sys/time.h> -#include <sys/resource.h> -/* VxWorks does not support getrusage() as far as we can determine */ -#if defined(_WRS_KERNEL) || defined(__RTP__) -struct rusage { - struct timeval ru_utime; /* user CPU time used */ - struct timeval ru_stime; /* system CPU time used */ -}; -#define getrusage(A,B) memset(B,0,sizeof(*B)) -#endif - -/* Saved resource information for the beginning of an operation */ -static struct rusage sBegin; /* CPU time at start */ -static sqlite3_int64 iBegin; /* Wall-clock time at start */ - /* -** Begin timing an operation +** A callback for the sqlite3_log() interface. */ -static void beginTimer(ShellState *p){ - if( p->enableTimer || (p->flgProgress & SHELL_PROGRESS_TMOUT)!=0 ){ - getrusage(RUSAGE_SELF, &sBegin); - iBegin = timeOfDay(); - } -} - -/* Return the difference of two time_structs in seconds */ -static double timeDiff(struct timeval *pStart, struct timeval *pEnd){ - return (pEnd->tv_usec - pStart->tv_usec)*0.000001 + - (double)(pEnd->tv_sec - pStart->tv_sec); -} - -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK -/* Return the time since the start of the timer in -** seconds. */ -static double elapseTime(ShellState *NotUsed){ - (void)NotUsed; - if( iBegin==0 ) return 0.0; - return (timeOfDay() - iBegin)*0.000001; +static void shellLog(void *pArg, int iErrCode, const char *zMsg){ + ShellState *p = (ShellState*)pArg; + if( p->pLog==0 ) return; + sqlite3_fprintf(p->pLog, "(%d) %s\n", iErrCode, zMsg); + fflush(p->pLog); } -#endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ /* -** Print the timing results. +** SQL function: shell_putsnl(X) +** +** Write the text X to the screen (or whatever output is being directed) +** adding a newline at the end, and then return X. */ -static void endTimer(ShellState *p){ - if( p->enableTimer ){ - sqlite3_int64 iEnd = timeOfDay(); - struct rusage sEnd; - getrusage(RUSAGE_SELF, &sEnd); - p->prevTimer = (iEnd - iBegin)*0.000001; - cli_printf(p->out, "Run Time: real %.6f user %.6f sys %.6f\n", - p->prevTimer, - timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), - timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); - if( p->enableTimer==1 ) p->enableTimer = 0; - iBegin = 0; - } +static void shellPutsFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + ShellState *p = (ShellState*)sqlite3_user_data(pCtx); + (void)nVal; + sqlite3_fprintf(p->out, "%s\n", sqlite3_value_text(apVal[0])); + sqlite3_result_value(pCtx, apVal[0]); } -#define BEGIN_TIMER(X) beginTimer(X) -#define END_TIMER(X) endTimer(X) -#define ELAPSE_TIME(X) elapseTime(X) -#define HAS_TIMER 1 - -#elif (defined(_WIN32) || defined(WIN32)) - -/* Saved resource information for the beginning of an operation */ -static HANDLE hProcess; -static FILETIME ftKernelBegin; -static FILETIME ftUserBegin; -static sqlite3_int64 ftWallBegin; -typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME, - LPFILETIME, LPFILETIME); -static GETPROCTIMES getProcessTimesAddr = NULL; - /* -** Check to see if we have timer support. Return 1 if necessary -** support found (or found previously). +** If in safe mode, print an error message described by the arguments +** and exit immediately. */ -static int hasTimer(void){ - if( getProcessTimesAddr ){ - return 1; - } else { - /* GetProcessTimes() isn't supported in WIN95 and some other Windows - ** versions. See if the version we are running on has it, and if it - ** does, save off a pointer to it and the current process handle. - */ - hProcess = GetCurrentProcess(); - if( hProcess ){ - HINSTANCE hinstLib = LoadLibrary(TEXT("Kernel32.dll")); - if( NULL != hinstLib ){ - getProcessTimesAddr = - (GETPROCTIMES) GetProcAddress(hinstLib, "GetProcessTimes"); - if( NULL != getProcessTimesAddr ){ - return 1; - } - FreeLibrary(hinstLib); - } - } +static void failIfSafeMode( + ShellState *p, + const char *zErrMsg, + ... +){ + if( p->bSafeMode ){ + va_list ap; + char *zMsg; + va_start(ap, zErrMsg); + zMsg = sqlite3_vmprintf(zErrMsg, ap); + va_end(ap); + sqlite3_fprintf(stderr, "line %lld: %s\n", p->lineno, zMsg); + exit(1); } - return 0; } -/* -** Begin timing an operation -*/ -static void beginTimer(ShellState *p){ - if( (p->enableTimer || (p->flgProgress & SHELL_PROGRESS_TMOUT)!=0) - && getProcessTimesAddr - ){ - FILETIME ftCreation, ftExit; - getProcessTimesAddr(hProcess,&ftCreation,&ftExit, - &ftKernelBegin,&ftUserBegin); - ftWallBegin = timeOfDay(); - } -} - -/* Return the difference of two FILETIME structs in seconds */ -static double timeDiff(FILETIME *pStart, FILETIME *pEnd){ - sqlite_int64 i64Start = *((sqlite_int64 *) pStart); - sqlite_int64 i64End = *((sqlite_int64 *) pEnd); - return (double) ((i64End - i64Start) / 10000000.0); -} - -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK -/* Return the time since the start of the timer in -** seconds. */ -static double elapseTime(ShellState *NotUsed){ - (void)NotUsed; - if( ftWallBegin==0 ) return 0.0; - return (timeOfDay() - ftWallBegin)*0.000001; -} -#endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ - -/* -** Print the timing results. -*/ -static void endTimer(ShellState *p){ - if( p->enableTimer && getProcessTimesAddr){ - FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; - sqlite3_int64 ftWallEnd = timeOfDay(); - getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); - p->prevTimer = (ftWallEnd - ftWallBegin)*0.000001; -#ifdef _WIN64 - /* microsecond precision on 64-bit windows */ - cli_printf(p->out, "Run Time: real %.6f user %f sys %f\n", - p->prevTimer, - timeDiff(&ftUserBegin, &ftUserEnd), - timeDiff(&ftKernelBegin, &ftKernelEnd)); -#else - /* millisecond precisino on 32-bit windows */ - cli_printf(p->out, "Run Time: real %.3f user %.3f sys %.3f\n", - p->prevTimer, - timeDiff(&ftUserBegin, &ftUserEnd), - timeDiff(&ftKernelBegin, &ftKernelEnd)); -#endif - if( p->enableTimer==1 ) p->enableTimer = 0; - ftWallBegin = 0; - } -} - -#define BEGIN_TIMER(X) beginTimer(X) -#define ELAPSE_TIME(X) elapseTime(X) -#define END_TIMER(X) endTimer(X) -#define HAS_TIMER hasTimer() - -#else -#define BEGIN_TIMER(X) /* no-op */ -#define ELAPSE_TIME(X) 0.0 -#define END_TIMER(X) /*no-op*/ -#define HAS_TIMER 0 -#endif -/************************* END PERFORMANCE TIMER ******************************/ - -/* -** Clear a display mode, freeing any allocated memory that it -** contains. -*/ -static void modeFree(Mode *p){ - u8 autoExplain = p->autoExplain; - free(p->spec.aWidth); - free(p->spec.aAlign); - free(p->spec.zColumnSep); - free(p->spec.zRowSep); - free(p->spec.zTableName); - free(p->spec.zNull); - memset(p, 0, sizeof(*p)); - p->spec.iVersion = 1; - p->autoExplain = autoExplain; -} - -/* -** Duplicate Mode pSrc into pDest. pDest is assumed to be -** uninitialized prior to invoking this routine. -*/ -static void modeDup(Mode *pDest, Mode *pSrc){ - memcpy(pDest, pSrc, sizeof(*pDest)); - if( pDest->spec.aWidth ){ - size_t sz = sizeof(pSrc->spec.aWidth[0]) * pSrc->spec.nWidth; - pDest->spec.aWidth = malloc( sz ); - if( pDest->spec.aWidth ){ - memcpy(pDest->spec.aWidth, pSrc->spec.aWidth, sz); - }else{ - pDest->spec.nWidth = 0; - } - } - if( pDest->spec.aAlign ){ - size_t sz = sizeof(pSrc->spec.aAlign[0]) * pSrc->spec.nAlign; - pDest->spec.aAlign = malloc( sz ); - if( pDest->spec.aAlign ){ - memcpy(pDest->spec.aAlign, pSrc->spec.aAlign, sz); - }else{ - pDest->spec.nAlign = 0; - } - } - if( pDest->spec.zColumnSep ){ - pDest->spec.zColumnSep = strdup(pSrc->spec.zColumnSep); - } - if( pDest->spec.zRowSep ){ - pDest->spec.zRowSep = strdup(pSrc->spec.zRowSep); - } - if( pDest->spec.zTableName ){ - pDest->spec.zTableName = strdup(pSrc->spec.zTableName); - } - if( pDest->spec.zNull ){ - pDest->spec.zNull = strdup(pSrc->spec.zNull); - } -} - -/* -** Set a string value to a copy of the zNew string in memory -** obtained from system malloc(). -*/ -static void modeSetStr(char **az, const char *zNew){ - free(*az); - if( zNew==0 ){ - *az = 0; - }else{ - size_t n = strlen(zNew); - *az = malloc( n+1 ); - if( *az ){ - memcpy(*az, zNew, n+1 ); - } - } -} - -/* -** Change the mode to eMode -*/ -static void modeChange(ShellState *p, unsigned char eMode){ - const ModeInfo *pI; - if( eMode<ArraySize(aModeInfo) ){ - Mode *pM = &p->mode; - pI = &aModeInfo[eMode]; - pM->eMode = eMode; - if( pI->eCSep ) modeSetStr(&pM->spec.zColumnSep, aModeStr[pI->eCSep]); - if( pI->eRSep ) modeSetStr(&pM->spec.zRowSep, aModeStr[pI->eRSep]); - if( pI->eNull ) modeSetStr(&pM->spec.zNull, aModeStr[pI->eNull]); - pM->spec.eText = pI->eText; - pM->spec.eBlob = pI->eBlob; - if( (pM->mFlags & MFLG_HDR)==0 ){ - pM->spec.bTitles = pI->bHdr; - } - pM->spec.eTitle = pI->eHdr; - if( pI->mFlg & 0x01 ){ - pM->spec.bBorder = QRF_No; - }else{ - pM->spec.bBorder = QRF_Auto; - } - if( pI->mFlg & 0x02 ){ - pM->spec.bSplitColumn = QRF_Yes; - pM->bAutoScreenWidth = 1; - }else{ - pM->spec.bSplitColumn = QRF_No; - } - }else if( eMode>=MODE_USER && eMode-MODE_USER<p->nSavedModes ){ - modeFree(&p->mode); - modeDup(&p->mode, &p->aSavedModes[eMode-MODE_USER].mode); - }else if( eMode==MODE_BATCH ){ - u8 mFlags = p->mode.mFlags; - modeFree(&p->mode); - modeChange(p, MODE_List); - p->mode.mFlags = mFlags; - }else if( eMode==MODE_TTY ){ - u8 mFlags = p->mode.mFlags; - modeFree(&p->mode); - modeChange(p, MODE_QBox); - p->mode.bAutoScreenWidth = 1; - p->mode.spec.eText = QRF_TEXT_Relaxed; - p->mode.spec.nCharLimit = DFLT_CHAR_LIMIT; - p->mode.spec.nLineLimit = DFLT_LINE_LIMIT; - p->mode.spec.bTextJsonb = QRF_Yes; - p->mode.spec.nTitleLimit = DFLT_TITLE_LIMIT; - p->mode.mFlags = mFlags; - } -} - -/* -** Set the mode to the default. It assumed that the mode has -** already been freed and zeroed prior to calling this routine. -*/ -static void modeDefault(ShellState *p){ - p->mode.spec.iVersion = 1; - p->mode.autoExplain = 1; - if( stdin_is_interactive || stdout_is_console ){ - modeChange(p, MODE_TTY); - }else{ - modeChange(p, MODE_BATCH); - } -} - -/* -** Find the number of a display mode given its name. Return -1 if -** the name does not match any mode. -** -** Saved modes are also searched if p!=NULL. The number returned -** for a saved mode is the index into the p->aSavedModes[] array -** plus MODE_USER. -** -** Two special mode names are also available: "batch" and "tty". -** evaluate to the default mode for batch operation and interactive -** operation on a TTY, respectively. -*/ -static int modeFind(ShellState *p, const char *zName){ - int i; - for(i=0; i<ArraySize(aModeInfo); i++){ - if( cli_strcmp(aModeInfo[i].zName,zName)==0 ) return i; - } - for(i=0; i<p->nSavedModes; i++){ - if( cli_strcmp(p->aSavedModes[i].zTag,zName)==0 ) return i+MODE_USER; - } - if( strcmp(zName,"batch")==0 ) return MODE_BATCH; - if( strcmp(zName,"tty")==0 ) return MODE_TTY; - return -1; -} - -/* -** Save or restore the current output mode -*/ -static void modePush(ShellState *p){ - if( p->nPopMode==0 ){ - modeFree(&p->modePrior); - modeDup(&p->modePrior,&p->mode); - } -} -static void modePop(ShellState *p){ - if( p->modePrior.spec.iVersion>0 ){ - modeFree(&p->mode); - p->mode = p->modePrior; - memset(&p->modePrior, 0, sizeof(p->modePrior)); - } -} - - -/* -** A callback for the sqlite3_log() interface. -*/ -static void shellLog(void *pArg, int iErrCode, const char *zMsg){ - ShellState *p = (ShellState*)pArg; - if( p->pLog==0 ) return; - cli_printf(p->pLog, "(%d) %s\n", iErrCode, zMsg); - fflush(p->pLog); -} - -/* -** SQL function: shell_putsnl(X) -** -** Write the text X to the screen (or whatever output is being directed) -** adding a newline at the end, and then return X. -*/ -static void shellPutsFunc( - sqlite3_context *pCtx, - int nVal, - sqlite3_value **apVal -){ - ShellState *p = (ShellState*)sqlite3_user_data(pCtx); - (void)nVal; - cli_printf(p->out, "%s\n", sqlite3_value_text(apVal[0])); - sqlite3_result_value(pCtx, apVal[0]); -} - -/* -** Compute the name of the location of an input error in memory -** obtained from sqlite3_malloc(). -*/ -static char *shellErrorLocation(ShellState *p){ - char *zLoc; - if( p->zErrPrefix ){ - zLoc = sqlite3_mprintf("%s", p->zErrPrefix); - }else if( p->zInFile==0 || strcmp(p->zInFile,"<stdin>")==0){ - zLoc = sqlite3_mprintf("line %lld:", p->lineno); - }else{ - zLoc = sqlite3_mprintf("%s:%lld:", p->zInFile, p->lineno); - } - return zLoc; -} - -/* -** If in safe mode, print an error message described by the arguments -** and exit immediately. -*/ -static void failIfSafeMode( - ShellState *p, - const char *zErrMsg, - ... -){ - if( p->bSafeMode ){ - va_list ap; - char *zMsg; - char *zLoc = shellErrorLocation(p); - va_start(ap, zErrMsg); - zMsg = sqlite3_vmprintf(zErrMsg, ap); - va_end(ap); - cli_printf(stderr, "%s %s\n", zLoc, zMsg); - cli_exit(1); - } -} - -/* -** Issue an error message from a dot-command. -*/ -static void dotCmdError( - ShellState *p, /* Shell state */ - int iArg, /* Index of argument on which error occurred */ - const char *zBrief, /* Brief (<20 character) error description */ - const char *zDetail, /* Error details */ - ... -){ - FILE *out = stderr; - char *zLoc = shellErrorLocation(p); - if( zBrief!=0 && iArg>=0 && iArg<p->dot.nArg ){ - int i = p->dot.aiOfst[iArg]; - int nPrompt = strlen30(zBrief) + 5; - cli_printf(out, "%s %s\n", zLoc, p->dot.zOrig); - if( i > nPrompt ){ - cli_printf(out, "%s %*s%s ---^\n", zLoc, 1+i-nPrompt, "", zBrief); - }else{ - cli_printf(out, "%s %*s^--- %s\n", zLoc, i, "", zBrief); - } - } - if( zDetail ){ - char *zMsg; - va_list ap; - va_start(ap, zDetail); - zMsg = sqlite3_vmprintf(zDetail,ap); - va_end(ap); - cli_printf(out,"%s %s\n", zLoc, zMsg); - sqlite3_free(zMsg); - } - sqlite3_free(zLoc); -} - - /* ** SQL function: edit(VALUE) ** edit(VALUE,EDITOR) @@ -1986,12 +1901,28 @@ edit_func_end: } #endif /* SQLITE_NOHAVE_SYSTEM */ +/* +** Save or restore the current output mode +*/ +static void outputModePush(ShellState *p){ + p->modePrior = p->mode; + p->priorShFlgs = p->shellFlgs; + memcpy(p->colSepPrior, p->colSeparator, sizeof(p->colSeparator)); + memcpy(p->rowSepPrior, p->rowSeparator, sizeof(p->rowSeparator)); +} +static void outputModePop(ShellState *p){ + p->mode = p->modePrior; + p->shellFlgs = p->priorShFlgs; + memcpy(p->colSeparator, p->colSepPrior, sizeof(p->colSeparator)); + memcpy(p->rowSeparator, p->rowSepPrior, sizeof(p->rowSeparator)); +} + /* ** Set output mode to text or binary for Windows. */ static void setCrlfMode(ShellState *p){ #ifdef _WIN32 - if( p->mode.mFlags & MFLG_CRLF ){ + if( p->crlfMode ){ sqlite3_fsetmode(p->out, _O_TEXT); }else{ sqlite3_fsetmode(p->out, _O_BINARY); @@ -2001,6 +1932,126 @@ static void setCrlfMode(ShellState *p){ #endif } +/* +** Output the given string as a hex-encoded blob (eg. X'1234' ) +*/ +static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){ + int i; + unsigned char *aBlob = (unsigned char*)pBlob; + + char *zStr = sqlite3_malloc64((i64)nBlob*2 + 1); + shell_check_oom(zStr); + + for(i=0; i<nBlob; i++){ + static const char aHex[] = { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' + }; + zStr[i*2] = aHex[ (aBlob[i] >> 4) ]; + zStr[i*2+1] = aHex[ (aBlob[i] & 0x0F) ]; + } + zStr[i*2] = '\0'; + + sqlite3_fprintf(out, "X'%s'", zStr); + sqlite3_free(zStr); +} + +/* +** Output the given string as a quoted string using SQL quoting conventions: +** +** (1) Single quotes (') within the string are doubled +** (2) The while string is enclosed in '...' +** (3) Control characters other than \n, \t, and \r\n are escaped +** using \u00XX notation and if such substitutions occur, +** the whole string is enclosed in unistr('...') instead of '...'. +** +** Step (3) is omitted if the control-character escape mode is OFF. +** +** See also: output_quoted_escaped_string() which does the same except +** that it does not make exceptions for \n, \t, and \r\n in step (3). +*/ +static void output_quoted_string(ShellState *p, const char *zInX){ + int i; + int needUnistr = 0; + int needDblQuote = 0; + const unsigned char *z = (const unsigned char*)zInX; + unsigned char c; + FILE *out = p->out; + sqlite3_fsetmode(out, _O_BINARY); + if( z==0 ) return; + for(i=0; (c = z[i])!=0; i++){ + if( c=='\'' ){ needDblQuote = 1; } + if( c>0x1f ) continue; + if( c=='\t' || c=='\n' ) continue; + if( c=='\r' && z[i+1]=='\n' ) continue; + needUnistr = 1; + break; + } + if( (needDblQuote==0 && needUnistr==0) + || (needDblQuote==0 && p->eEscMode==SHELL_ESC_OFF) + ){ + sqlite3_fprintf(out, "'%s'",z); + }else if( p->eEscMode==SHELL_ESC_OFF ){ + char *zEncoded = sqlite3_mprintf("%Q", z); + sqlite3_fputs(zEncoded, out); + sqlite3_free(zEncoded); + }else{ + if( needUnistr ){ + sqlite3_fputs("unistr('", out); + }else{ + sqlite3_fputs("'", out); + } + while( *z ){ + for(i=0; (c = z[i])!=0; i++){ + if( c=='\'' ) break; + if( c>0x1f ) continue; + if( c=='\t' || c=='\n' ) continue; + if( c=='\r' && z[i+1]=='\n' ) continue; + break; + } + if( i ){ + sqlite3_fprintf(out, "%.*s", i, z); + z += i; + } + if( c==0 ) break; + if( c=='\'' ){ + sqlite3_fputs("''", out); + }else{ + sqlite3_fprintf(out, "\\u%04x", c); + } + z++; + } + if( needUnistr ){ + sqlite3_fputs("')", out); + }else{ + sqlite3_fputs("'", out); + } + } + setCrlfMode(p); +} + +/* +** Output the given string as a quoted string using SQL quoting conventions. +** Additionallly , escape the "\n" and "\r" characters so that they do not +** get corrupted by end-of-line translation facilities in some operating +** systems. +** +** This is like output_quoted_string() but with the addition of the \r\n +** escape mechanism. +*/ +static void output_quoted_escaped_string(ShellState *p, const char *z){ + char *zEscaped; + sqlite3_fsetmode(p->out, _O_BINARY); + if( p->eEscMode==SHELL_ESC_OFF ){ + zEscaped = sqlite3_mprintf("%Q", z); + }else{ + zEscaped = sqlite3_mprintf("%#Q", z); + } + sqlite3_fputs(zEscaped, p->out); + sqlite3_free(zEscaped); + setCrlfMode(p); +} + /* ** Find earliest of chars within s specified in zAny. ** With ns == ~0, is like strpbrk(s,zAny) and s must be 0-terminated. @@ -2064,14 +2115,13 @@ static void output_c_string(FILE *out, const char *z){ static const char *zDQBSRO = "\"\\\x7f"; /* double-quote, backslash, rubout */ char ace[3] = "\\?"; char cbsSay; - cli_puts(zq, out); - if( z==0 ) z = ""; + sqlite3_fputs(zq, out); while( *z!=0 ){ const char *pcDQBSRO = anyOfInStr(z, zDQBSRO, ~(size_t)0); const char *pcPast = zSkipValidUtf8(z, INT_MAX, ctrlMask); const char *pcEnd = (pcDQBSRO && pcDQBSRO < pcPast)? pcDQBSRO : pcPast; if( pcEnd > z ){ - cli_printf(out, "%.*s", (int)(pcEnd-z), z); + sqlite3_fprintf(out, "%.*s", (int)(pcEnd-z), z); } if( (c = *pcEnd)==0 ) break; ++pcEnd; @@ -2087,122 +2137,266 @@ static void output_c_string(FILE *out, const char *z){ } if( cbsSay ){ ace[1] = cbsSay; - cli_puts(ace, out); + sqlite3_fputs(ace, out); }else if( !isprint(c&0xff) ){ - cli_printf(out, "\\%03o", c&0xff); + sqlite3_fprintf(out, "\\%03o", c&0xff); }else{ ace[1] = (char)c; - cli_puts(ace+1, out); + sqlite3_fputs(ace+1, out); } z = pcEnd; } - cli_puts(zq, out); + sqlite3_fputs(zq, out); } -/* Encode input string z[] as a C-language string literal and -** append it to the sqlite3_str. If z is NULL render and empty string. +/* +** Output the given string as quoted according to JSON quoting rules. */ -static void append_c_string(sqlite3_str *out, const char *z){ - char c; +static void output_json_string(FILE *out, const char *z, i64 n){ + unsigned char c; static const char *zq = "\""; static long ctrlMask = ~0L; - static const char *zDQBSRO = "\"\\\x7f"; /* double-quote, backslash, rubout */ + static const char *zDQBS = "\"\\"; + const char *pcLimit; char ace[3] = "\\?"; char cbsSay; + if( z==0 ) z = ""; - sqlite3_str_appendall(out,zq); - while( *z!=0 ){ - const char *pcDQBSRO = anyOfInStr(z, zDQBSRO, ~(size_t)0); - const char *pcPast = zSkipValidUtf8(z, INT_MAX, ctrlMask); - const char *pcEnd = (pcDQBSRO && pcDQBSRO < pcPast)? pcDQBSRO : pcPast; + pcLimit = z + ((n<0)? strlen(z) : (size_t)n); + sqlite3_fputs(zq, out); + while( z < pcLimit ){ + const char *pcDQBS = anyOfInStr(z, zDQBS, pcLimit-z); + const char *pcPast = zSkipValidUtf8(z, (int)(pcLimit-z), ctrlMask); + const char *pcEnd = (pcDQBS && pcDQBS < pcPast)? pcDQBS : pcPast; if( pcEnd > z ){ - sqlite3_str_appendf(out, "%.*s", (int)(pcEnd-z), z); + sqlite3_fprintf(out, "%.*s", (int)(pcEnd-z), z); + z = pcEnd; } - if( (c = *pcEnd)==0 ) break; - ++pcEnd; + if( z >= pcLimit ) break; + c = (unsigned char)*(z++); switch( c ){ - case '\\': case '"': + case '"': case '\\': cbsSay = (char)c; break; - case '\t': cbsSay = 't'; break; + case '\b': cbsSay = 'b'; break; + case '\f': cbsSay = 'f'; break; case '\n': cbsSay = 'n'; break; case '\r': cbsSay = 'r'; break; - case '\f': cbsSay = 'f'; break; + case '\t': cbsSay = 't'; break; default: cbsSay = 0; break; } if( cbsSay ){ ace[1] = cbsSay; - sqlite3_str_appendall(out,ace); - }else if( !isprint(c&0xff) ){ - sqlite3_str_appendf(out, "\\%03o", c&0xff); + sqlite3_fputs(ace, out); + }else if( c<=0x1f || c>=0x7f ){ + sqlite3_fprintf(out, "\\u%04x", c); }else{ ace[1] = (char)c; - sqlite3_str_appendall(out, ace+1); + sqlite3_fputs(ace+1, out); } - z = pcEnd; } - sqlite3_str_appendall(out, zq); + sqlite3_fputs(zq, out); } /* -** This routine runs when the user presses Ctrl-C -*/ -static void interrupt_handler(int NotUsed){ - UNUSED_PARAMETER(NotUsed); - if( ++seenInterrupt>1 ) cli_exit(1); - if( globalDb ) sqlite3_interrupt(globalDb); -} - -/* Try to determine the screen width. Use the default if unable. +** Escape the input string if it is needed and in accordance with +** eEscMode. +** +** Escaping is needed if the string contains any control characters +** other than \t, \n, and \r\n +** +** If no escaping is needed (the common case) then set *ppFree to NULL +** and return the original string. If escaping is needed, write the +** escaped string into memory obtained from sqlite3_malloc64() or the +** equivalent, and return the new string and set *ppFree to the new string +** as well. +** +** The caller is responsible for freeing *ppFree if it is non-NULL in order +** to reclaim memory. */ -int shellScreenWidth(void){ - if( stdout_tty_width>0 ){ - return stdout_tty_width; - }else{ -#if defined(TIOCGSIZE) - struct ttysize ts; - if( ioctl(STDIN_FILENO, TIOCGSIZE, &ts)>=0 - || ioctl(STDOUT_FILENO, TIOCGSIZE, &ts)>=0 - || ioctl(STDERR_FILENO, TIOCGSIZE, &ts)>=0 +static const char *escapeOutput( + ShellState *p, + const char *zInX, + char **ppFree +){ + i64 i, j; + i64 nCtrl = 0; + unsigned char *zIn; + unsigned char c; + unsigned char *zOut; + + + /* No escaping if disabled */ + if( p->eEscMode==SHELL_ESC_OFF ){ + *ppFree = 0; + return zInX; + } + + /* Count the number of control characters in the string. */ + zIn = (unsigned char*)zInX; + for(i=0; (c = zIn[i])!=0; i++){ + if( c<=0x1f + && c!='\t' + && c!='\n' + && (c!='\r' || zIn[i+1]!='\n') ){ - return ts.ts_cols; + nCtrl++; } -#elif defined(TIOCGWINSZ) - struct winsize ws; - if( ioctl(STDIN_FILENO, TIOCGWINSZ, &ws)>=0 - || ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws)>=0 - || ioctl(STDERR_FILENO, TIOCGWINSZ, &ws)>=0 + } + if( nCtrl==0 ){ + *ppFree = 0; + return zInX; + } + if( p->eEscMode==SHELL_ESC_SYMBOL ) nCtrl *= 2; + zOut = sqlite3_malloc64( i + nCtrl + 1 ); + shell_check_oom(zOut); + for(i=j=0; (c = zIn[i])!=0; i++){ + if( c>0x1f + || c=='\t' + || c=='\n' + || (c=='\r' && zIn[i+1]=='\n') ){ - return ws.ws_col; + continue; } -#elif defined(_WIN32) - CONSOLE_SCREEN_BUFFER_INFO csbi; - if( GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi) - || GetConsoleScreenBufferInfo(GetStdHandle(STD_ERROR_HANDLE), &csbi) - || GetConsoleScreenBufferInfo(GetStdHandle(STD_INPUT_HANDLE), &csbi) - ){ - return csbi.srWindow.Right - csbi.srWindow.Left + 1; + if( i>0 ){ + memcpy(&zOut[j], zIn, i); + j += i; + } + zIn += i+1; + i = -1; + switch( p->eEscMode ){ + case SHELL_ESC_SYMBOL: + zOut[j++] = 0xe2; + zOut[j++] = 0x90; + zOut[j++] = 0x80+c; + break; + case SHELL_ESC_ASCII: + zOut[j++] = '^'; + zOut[j++] = 0x40+c; + break; } -#endif -#define DEFAULT_SCREEN_WIDTH 80 - return DEFAULT_SCREEN_WIDTH; } + if( i>0 ){ + memcpy(&zOut[j], zIn, i); + j += i; + } + zOut[j] = 0; + *ppFree = (char*)zOut; + return (char*)zOut; } -#if (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE) /* -** This routine runs for console events (e.g. Ctrl-C) on Win32 +** Output the given string with characters that are special to +** HTML escaped. */ -static BOOL WINAPI ConsoleCtrlHandler( - DWORD dwCtrlType /* One of the CTRL_*_EVENT constants */ -){ - if( dwCtrlType==CTRL_C_EVENT ){ - interrupt_handler(0); - return TRUE; +static void output_html_string(FILE *out, const char *z){ + int i; + if( z==0 ) z = ""; + while( *z ){ + for(i=0; z[i] + && z[i]!='<' + && z[i]!='&' + && z[i]!='>' + && z[i]!='\"' + && z[i]!='\''; + i++){} + if( i>0 ){ + sqlite3_fprintf(out, "%.*s",i,z); + } + if( z[i]=='<' ){ + sqlite3_fputs("&lt;", out); + }else if( z[i]=='&' ){ + sqlite3_fputs("&amp;", out); + }else if( z[i]=='>' ){ + sqlite3_fputs("&gt;", out); + }else if( z[i]=='\"' ){ + sqlite3_fputs("&quot;", out); + }else if( z[i]=='\'' ){ + sqlite3_fputs("&#39;", out); + }else{ + break; + } + z += i + 1; } - return FALSE; } -#endif + +/* +** If a field contains any character identified by a 1 in the following +** array, then the string must be quoted for CSV. +*/ +static const char needCsvQuote[] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +}; + +/* +** Output a single term of CSV. Actually, p->colSeparator is used for +** the separator, which may or may not be a comma. p->nullValue is +** the null value. Strings are quoted if necessary. The separator +** is only issued if bSep is true. +*/ +static void output_csv(ShellState *p, const char *z, int bSep){ + if( z==0 ){ + sqlite3_fprintf(p->out, "%s",p->nullValue); + }else{ + unsigned i; + for(i=0; z[i]; i++){ + if( needCsvQuote[((unsigned char*)z)[i]] ){ + i = 0; + break; + } + } + if( i==0 || strstr(z, p->colSeparator)!=0 ){ + char *zQuoted = sqlite3_mprintf("\"%w\"", z); + shell_check_oom(zQuoted); + sqlite3_fputs(zQuoted, p->out); + sqlite3_free(zQuoted); + }else{ + sqlite3_fputs(z, p->out); + } + } + if( bSep ){ + sqlite3_fputs(p->colSeparator, p->out); + } +} + +/* +** This routine runs when the user presses Ctrl-C +*/ +static void interrupt_handler(int NotUsed){ + UNUSED_PARAMETER(NotUsed); + if( ++seenInterrupt>1 ) exit(1); + if( globalDb ) sqlite3_interrupt(globalDb); +} + +#if (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE) +/* +** This routine runs for console events (e.g. Ctrl-C) on Win32 +*/ +static BOOL WINAPI ConsoleCtrlHandler( + DWORD dwCtrlType /* One of the CTRL_*_EVENT constants */ +){ + if( dwCtrlType==CTRL_C_EVENT ){ + interrupt_handler(0); + return TRUE; + } + return FALSE; +} +#endif #ifndef SQLITE_OMIT_AUTHORIZATION /* @@ -2222,7 +2416,6 @@ static int safeModeAuth( "fts3_tokenizer", "load_extension", "readfile", - "realpath", "writefile", "zipfile", "zipfile_cds", @@ -2285,23 +2478,23 @@ static int shellAuth( az[1] = zA2; az[2] = zA3; az[3] = zA4; - cli_printf(p->out, "authorizer: %s", azAction[op]); + sqlite3_fprintf(p->out, "authorizer: %s", azAction[op]); for(i=0; i<4; i++){ - cli_puts(" ", p->out); + sqlite3_fputs(" ", p->out); if( az[i] ){ output_c_string(p->out, az[i]); }else{ - cli_puts("NULL", p->out); + sqlite3_fputs("NULL", p->out); } } - cli_puts("\n", p->out); + sqlite3_fputs("\n", p->out); if( p->bSafeMode ) (void)safeModeAuth(pClientData, op, zA1, zA2, zA3, zA4); return SQLITE_OK; } #endif /* -** Print a schema statement. This is helper routine to dump_callbac(). +** Print a schema statement. Part of MODE_Semi and MODE_Pretty output. ** ** This routine converts some CREATE TABLE statements for shadow tables ** in FTS3/4/5 into CREATE TABLE IF NOT EXISTS statements. @@ -2332,12 +2525,18 @@ static void printSchemaLine(FILE *out, const char *z, const char *zTail){ } } if( sqlite3_strglob("CREATE TABLE ['\"]*", z)==0 ){ - cli_printf(out, "CREATE TABLE IF NOT EXISTS %s%s", z+13, zTail); + sqlite3_fprintf(out, "CREATE TABLE IF NOT EXISTS %s%s", z+13, zTail); }else{ - cli_printf(out, "%s%s", z, zTail); + sqlite3_fprintf(out, "%s%s", z, zTail); } sqlite3_free(zToFree); } +static void printSchemaLineN(FILE *out, char *z, int n, const char *zTail){ + char c = z[n]; + z[n] = 0; + printSchemaLine(out, z, zTail); + z[n] = c; +} /* ** Return true if string z[] has nothing but whitespace and comments to the @@ -2355,130 +2554,95 @@ static int wsToEol(const char *z){ } /* -** SQL Function: shell_format_schema(SQL,FLAGS) -** -** This function is internally by the CLI to assist with the -** ".schema", ".fullschema", and ".dump" commands. The first -** argument is the value from sqlite_schema.sql. The value returned -** is a modification of the input that can actually be run as SQL -** to recreate the schema object. -** -** When FLAGS is zero, the only changes is to append ";". If the -** 0x01 bit of FLAGS is set, then transformations are made to implement -** ".schema --indent". +** Add a new entry to the EXPLAIN QUERY PLAN data */ -static void shellFormatSchema( - sqlite3_context *pCtx, - int nVal, - sqlite3_value **apVal -){ - int flags; /* Value of 2nd parameter */ - const char *zSql; /* Value of 1st parameter */ - int nSql; /* Bytes of text in zSql[] */ - sqlite3_str *pOut; /* Output buffer */ - char *z; /* Writable copy of zSql */ - int i, j; /* Loop counters */ - int nParen = 0; - char cEnd = 0; - char c; - int nLine = 0; - int isIndex; - int isWhere = 0; - - assert( nVal==2 ); - pOut = sqlite3_str_new(sqlite3_context_db_handle(pCtx)); - nSql = sqlite3_value_bytes(apVal[0]); - zSql = (const char*)sqlite3_value_text(apVal[0]); - if( zSql==0 || zSql[0]==0 ) goto shellFormatSchema_finish; - flags = sqlite3_value_int(apVal[1]); - if( (flags & 0x01)==0 ){ - sqlite3_str_append(pOut, zSql, nSql); - sqlite3_str_append(pOut, ";", 1); - goto shellFormatSchema_finish; - } - if( sqlite3_strlike("CREATE VIEW%", zSql, 0)==0 - || sqlite3_strlike("CREATE TRIG%", zSql, 0)==0 - ){ - sqlite3_str_append(pOut, zSql, nSql); - sqlite3_str_append(pOut, ";", 1); - goto shellFormatSchema_finish; +static void eqp_append(ShellState *p, int iEqpId, int p2, const char *zText){ + EQPGraphRow *pNew; + i64 nText; + if( zText==0 ) return; + nText = strlen(zText); + if( p->autoEQPtest ){ + sqlite3_fprintf(p->out, "%d,%d,%s\n", iEqpId, p2, zText); } - isIndex = sqlite3_strlike("CREATE INDEX%", zSql, 0)==0 - || sqlite3_strlike("CREATE UNIQUE INDEX%", zSql, 0)==0; - z = sqlite3_mprintf("%s", zSql); - if( z==0 ){ - sqlite3_str_free(pOut); - sqlite3_result_error_nomem(pCtx); - return; + pNew = sqlite3_malloc64( sizeof(*pNew) + nText ); + shell_check_oom(pNew); + pNew->iEqpId = iEqpId; + pNew->iParentId = p2; + memcpy(pNew->zText, zText, nText+1); + pNew->pNext = 0; + if( p->sGraph.pLast ){ + p->sGraph.pLast->pNext = pNew; + }else{ + p->sGraph.pRow = pNew; } - j = 0; - for(i=0; IsSpace(z[i]); i++){} - for(; (c = z[i])!=0; i++){ - if( IsSpace(c) ){ - if( z[j-1]=='\r' ) z[j-1] = '\n'; - if( IsSpace(z[j-1]) || z[j-1]=='(' ) continue; - }else if( (c=='(' || c==')') && j>0 && IsSpace(z[j-1]) ){ - j--; - } - z[j++] = c; - } - while( j>0 && IsSpace(z[j-1]) ){ j--; } - z[j] = 0; - if( strlen30(z)>=79 ){ - for(i=j=0; (c = z[i])!=0; i++){ /* Copy from z[i] back to z[j] */ - if( c==cEnd ){ - cEnd = 0; - }else if( cEnd!=0){ - /* No-op */ - }else if( c=='"' || c=='\'' || c=='`' ){ - cEnd = c; - }else if( c=='[' ){ - cEnd = ']'; - }else if( c=='-' && z[i+1]=='-' ){ - cEnd = '\n'; - }else if( c=='(' ){ - nParen++; - }else if( c==')' ){ - nParen--; - if( nLine>0 && nParen==0 && j>0 && !isWhere ){ - sqlite3_str_append(pOut, z, j); - sqlite3_str_append(pOut, "\n", 1); - j = 0; - } - }else if( (c=='w' || c=='W') - && nParen==0 && isIndex - && sqlite3_strnicmp("WHERE",&z[i],5)==0 - && !IsAlnum(z[i+5]) && z[i+5]!='_' ){ - isWhere = 1; - }else if( isWhere && (c=='A' || c=='a') - && nParen==0 - && sqlite3_strnicmp("AND",&z[i],3)==0 - && !IsAlnum(z[i+3]) && z[i+3]!='_' ){ - sqlite3_str_append(pOut, z, j); - sqlite3_str_append(pOut, "\n ", 5); - j = 0; - } - z[j++] = c; - if( nParen==1 && cEnd==0 - && (c=='(' || c=='\n' || (c==',' && !wsToEol(z+i+1))) - && !isWhere - ){ - if( c=='\n' ) j--; - sqlite3_str_append(pOut, z, j); - sqlite3_str_append(pOut, "\n ", 3); - j = 0; - nLine++; - while( IsSpace(z[i+1]) ){ i++; } - } + p->sGraph.pLast = pNew; +} + +/* +** Free and reset the EXPLAIN QUERY PLAN data that has been collected +** in p->sGraph. +*/ +static void eqp_reset(ShellState *p){ + EQPGraphRow *pRow, *pNext; + for(pRow = p->sGraph.pRow; pRow; pRow = pNext){ + pNext = pRow->pNext; + sqlite3_free(pRow); + } + memset(&p->sGraph, 0, sizeof(p->sGraph)); +} + +/* Return the next EXPLAIN QUERY PLAN line with iEqpId that occurs after +** pOld, or return the first such line if pOld is NULL +*/ +static EQPGraphRow *eqp_next_row(ShellState *p, int iEqpId, EQPGraphRow *pOld){ + EQPGraphRow *pRow = pOld ? pOld->pNext : p->sGraph.pRow; + while( pRow && pRow->iParentId!=iEqpId ) pRow = pRow->pNext; + return pRow; +} + +/* Render a single level of the graph that has iEqpId as its parent. Called +** recursively to render sublevels. +*/ +static void eqp_render_level(ShellState *p, int iEqpId){ + EQPGraphRow *pRow, *pNext; + i64 n = strlen(p->sGraph.zPrefix); + char *z; + for(pRow = eqp_next_row(p, iEqpId, 0); pRow; pRow = pNext){ + pNext = eqp_next_row(p, iEqpId, pRow); + z = pRow->zText; + sqlite3_fprintf(p->out, "%s%s%s\n", p->sGraph.zPrefix, + pNext ? "|--" : "`--", z); + if( n<(i64)sizeof(p->sGraph.zPrefix)-7 ){ + memcpy(&p->sGraph.zPrefix[n], pNext ? "| " : " ", 4); + eqp_render_level(p, pRow->iEqpId); + p->sGraph.zPrefix[n] = 0; } - z[j] = 0; } - sqlite3_str_appendall(pOut, z); - sqlite3_str_append(pOut, ";", 1); - sqlite3_free(z); +} -shellFormatSchema_finish: - sqlite3_result_text(pCtx, sqlite3_str_finish(pOut), -1, sqlite3_free); +/* +** Display and reset the EXPLAIN QUERY PLAN data +*/ +static void eqp_render(ShellState *p, i64 nCycle){ + EQPGraphRow *pRow = p->sGraph.pRow; + if( pRow ){ + if( pRow->zText[0]=='-' ){ + if( pRow->pNext==0 ){ + eqp_reset(p); + return; + } + sqlite3_fprintf(p->out, "%s\n", pRow->zText+3); + p->sGraph.pRow = pRow->pNext; + sqlite3_free(pRow); + }else if( nCycle>0 ){ + sqlite3_fprintf(p->out, "QUERY PLAN (cycles=%lld [100%%])\n", nCycle); + }else{ + sqlite3_fputs("QUERY PLAN\n", p->out); + } + p->sGraph.zPrefix[0] = 0; + eqp_render_level(p, 0); + eqp_reset(p); + } } #ifndef SQLITE_OMIT_PROGRESS_CALLBACK @@ -2488,26 +2652,493 @@ shellFormatSchema_finish: static int progress_handler(void *pClientData) { ShellState *p = (ShellState*)pClientData; p->nProgress++; - if( (p->flgProgress & SHELL_PROGRESS_TMOUT)!=0 - && ELAPSE_TIME(p)>=p->tmProgress - ){ - cli_printf(p->out, "Progress timeout after %.6f seconds\n", - ELAPSE_TIME(p)); - return 1; - } if( p->nProgress>=p->mxProgress && p->mxProgress>0 ){ - cli_printf(p->out, "Progress limit reached (%u)\n", p->nProgress); + sqlite3_fprintf(p->out, "Progress limit reached (%u)\n", p->nProgress); if( p->flgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0; if( p->flgProgress & SHELL_PROGRESS_ONCE ) p->mxProgress = 0; return 1; } if( (p->flgProgress & SHELL_PROGRESS_QUIET)==0 ){ - cli_printf(p->out, "Progress %u\n", p->nProgress); + sqlite3_fprintf(p->out, "Progress %u\n", p->nProgress); } return 0; } #endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ +/* +** Print N dashes +*/ +static void print_dashes(FILE *out, int N){ + const char zDash[] = "--------------------------------------------------"; + const int nDash = sizeof(zDash) - 1; + while( N>nDash ){ + sqlite3_fputs(zDash, out); + N -= nDash; + } + sqlite3_fprintf(out, "%.*s", N, zDash); +} + +/* +** Print a markdown or table-style row separator using ascii-art +*/ +static void print_row_separator( + ShellState *p, + int nArg, + const char *zSep +){ + int i; + if( nArg>0 ){ + sqlite3_fputs(zSep, p->out); + print_dashes(p->out, p->actualWidth[0]+2); + for(i=1; i<nArg; i++){ + sqlite3_fputs(zSep, p->out); + print_dashes(p->out, p->actualWidth[i]+2); + } + sqlite3_fputs(zSep, p->out); + } + sqlite3_fputs("\n", p->out); +} + +/* +** This is the callback routine that the shell +** invokes for each row of a query result. +*/ +static int shell_callback( + void *pArg, + int nArg, /* Number of result columns */ + char **azArg, /* Text of each result column */ + char **azCol, /* Column names */ + int *aiType /* Column types. Might be NULL */ +){ + int i; + ShellState *p = (ShellState*)pArg; + + if( azArg==0 ) return 0; + switch( p->cMode ){ + case MODE_Count: + case MODE_Off: { + break; + } + case MODE_Line: { + int w = 5; + if( azArg==0 ) break; + for(i=0; i<nArg; i++){ + int len = strlen30(azCol[i] ? azCol[i] : ""); + if( len>w ) w = len; + } + if( p->cnt++>0 ) sqlite3_fputs(p->rowSeparator, p->out); + for(i=0; i<nArg; i++){ + char *pFree = 0; + const char *pDisplay; + pDisplay = escapeOutput(p, azArg[i] ? azArg[i] : p->nullValue, &pFree); + sqlite3_fprintf(p->out, "%*s = %s%s", w, azCol[i], + pDisplay, p->rowSeparator); + if( pFree ) sqlite3_free(pFree); + } + break; + } + case MODE_ScanExp: + case MODE_Explain: { + static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13}; + static const int aExplainMap[] = {0, 1, 2, 3, 4, 5, 6, 7 }; + static const int aScanExpWidth[] = {4, 15, 6, 13, 4, 4, 4, 13, 2, 13}; + static const int aScanExpMap[] = {0, 9, 8, 1, 2, 3, 4, 5, 6, 7 }; + + const int *aWidth = aExplainWidth; + const int *aMap = aExplainMap; + int nWidth = ArraySize(aExplainWidth); + int iIndent = 1; + + if( p->cMode==MODE_ScanExp ){ + aWidth = aScanExpWidth; + aMap = aScanExpMap; + nWidth = ArraySize(aScanExpWidth); + iIndent = 3; + } + if( nArg>nWidth ) nArg = nWidth; + + /* If this is the first row seen, print out the headers */ + if( p->cnt++==0 ){ + for(i=0; i<nArg; i++){ + utf8_width_print(p->out, aWidth[i], azCol[ aMap[i] ]); + sqlite3_fputs(i==nArg-1 ? "\n" : " ", p->out); + } + for(i=0; i<nArg; i++){ + print_dashes(p->out, aWidth[i]); + sqlite3_fputs(i==nArg-1 ? "\n" : " ", p->out); + } + } + + /* If there is no data, exit early. */ + if( azArg==0 ) break; + + for(i=0; i<nArg; i++){ + const char *zSep = " "; + int w = aWidth[i]; + const char *zVal = azArg[ aMap[i] ]; + if( i==nArg-1 ) w = 0; + if( zVal && strlenChar(zVal)>w ){ + w = strlenChar(zVal); + zSep = " "; + } + if( i==iIndent && p->aiIndent && p->pStmt ){ + if( p->iIndent<p->nIndent ){ + sqlite3_fprintf(p->out, "%*.s", p->aiIndent[p->iIndent], ""); + } + p->iIndent++; + } + utf8_width_print(p->out, w, zVal ? zVal : p->nullValue); + sqlite3_fputs(i==nArg-1 ? "\n" : zSep, p->out); + } + break; + } + case MODE_Semi: { /* .schema and .fullschema output */ + printSchemaLine(p->out, azArg[0], ";\n"); + break; + } + case MODE_Pretty: { /* .schema and .fullschema with --indent */ + char *z; + int j; + int nParen = 0; + char cEnd = 0; + char c; + int nLine = 0; + int isIndex; + int isWhere = 0; + assert( nArg==1 ); + if( azArg[0]==0 ) break; + if( sqlite3_strlike("CREATE VIEW%", azArg[0], 0)==0 + || sqlite3_strlike("CREATE TRIG%", azArg[0], 0)==0 + ){ + sqlite3_fprintf(p->out, "%s;\n", azArg[0]); + break; + } + isIndex = sqlite3_strlike("CREATE INDEX%", azArg[0], 0)==0 + || sqlite3_strlike("CREATE UNIQUE INDEX%", azArg[0], 0)==0; + z = sqlite3_mprintf("%s", azArg[0]); + shell_check_oom(z); + j = 0; + for(i=0; IsSpace(z[i]); i++){} + for(; (c = z[i])!=0; i++){ + if( IsSpace(c) ){ + if( z[j-1]=='\r' ) z[j-1] = '\n'; + if( IsSpace(z[j-1]) || z[j-1]=='(' ) continue; + }else if( (c=='(' || c==')') && j>0 && IsSpace(z[j-1]) ){ + j--; + } + z[j++] = c; + } + while( j>0 && IsSpace(z[j-1]) ){ j--; } + z[j] = 0; + if( strlen30(z)>=79 ){ + for(i=j=0; (c = z[i])!=0; i++){ /* Copy from z[i] back to z[j] */ + if( c==cEnd ){ + cEnd = 0; + }else if( c=='"' || c=='\'' || c=='`' ){ + cEnd = c; + }else if( c=='[' ){ + cEnd = ']'; + }else if( c=='-' && z[i+1]=='-' ){ + cEnd = '\n'; + }else if( c=='(' ){ + nParen++; + }else if( c==')' ){ + nParen--; + if( nLine>0 && nParen==0 && j>0 && !isWhere ){ + printSchemaLineN(p->out, z, j, "\n"); + j = 0; + } + }else if( (c=='w' || c=='W') + && nParen==0 && isIndex + && sqlite3_strnicmp("WHERE",&z[i],5)==0 + && !IsAlnum(z[i+5]) && z[i+5]!='_' ){ + isWhere = 1; + }else if( isWhere && (c=='A' || c=='a') + && nParen==0 + && sqlite3_strnicmp("AND",&z[i],3)==0 + && !IsAlnum(z[i+3]) && z[i+3]!='_' ){ + printSchemaLineN(p->out, z, j, "\n "); + j = 0; + } + z[j++] = c; + if( nParen==1 && cEnd==0 + && (c=='(' || c=='\n' || (c==',' && !wsToEol(z+i+1))) + && !isWhere + ){ + if( c=='\n' ) j--; + printSchemaLineN(p->out, z, j, "\n "); + j = 0; + nLine++; + while( IsSpace(z[i+1]) ){ i++; } + } + } + z[j] = 0; + } + printSchemaLine(p->out, z, ";\n"); + sqlite3_free(z); + break; + } + case MODE_List: { + if( p->cnt++==0 && p->showHeader ){ + for(i=0; i<nArg; i++){ + char *z = azCol[i]; + char *pFree; + const char *zOut = escapeOutput(p, z, &pFree); + sqlite3_fprintf(p->out, "%s%s", zOut, + i==nArg-1 ? p->rowSeparator : p->colSeparator); + if( pFree ) sqlite3_free(pFree); + } + } + if( azArg==0 ) break; + for(i=0; i<nArg; i++){ + char *z = azArg[i]; + char *pFree; + const char *zOut; + if( z==0 ) z = p->nullValue; + zOut = escapeOutput(p, z, &pFree); + sqlite3_fputs(zOut, p->out); + if( pFree ) sqlite3_free(pFree); + sqlite3_fputs((i<nArg-1)? p->colSeparator : p->rowSeparator, p->out); + } + break; + } + case MODE_Www: + case MODE_Html: { + if( p->cnt==0 && p->cMode==MODE_Www ){ + sqlite3_fputs( + "</PRE>\n" + "<TABLE border='1' cellspacing='0' cellpadding='2'>\n" + ,p->out + ); + } + if( p->cnt==0 && (p->showHeader || p->cMode==MODE_Www) ){ + sqlite3_fputs("<TR>", p->out); + for(i=0; i<nArg; i++){ + sqlite3_fputs("<TH>", p->out); + output_html_string(p->out, azCol[i]); + sqlite3_fputs("</TH>\n", p->out); + } + sqlite3_fputs("</TR>\n", p->out); + } + p->cnt++; + if( azArg==0 ) break; + sqlite3_fputs("<TR>", p->out); + for(i=0; i<nArg; i++){ + sqlite3_fputs("<TD>", p->out); + output_html_string(p->out, azArg[i] ? azArg[i] : p->nullValue); + sqlite3_fputs("</TD>\n", p->out); + } + sqlite3_fputs("</TR>\n", p->out); + break; + } + case MODE_Tcl: { + if( p->cnt++==0 && p->showHeader ){ + for(i=0; i<nArg; i++){ + output_c_string(p->out, azCol[i] ? azCol[i] : ""); + if(i<nArg-1) sqlite3_fputs(p->colSeparator, p->out); + } + sqlite3_fputs(p->rowSeparator, p->out); + } + if( azArg==0 ) break; + for(i=0; i<nArg; i++){ + output_c_string(p->out, azArg[i] ? azArg[i] : p->nullValue); + if(i<nArg-1) sqlite3_fputs(p->colSeparator, p->out); + } + sqlite3_fputs(p->rowSeparator, p->out); + break; + } + case MODE_Csv: { + sqlite3_fsetmode(p->out, _O_BINARY); + if( p->cnt++==0 && p->showHeader ){ + for(i=0; i<nArg; i++){ + output_csv(p, azCol[i] ? azCol[i] : "", i<nArg-1); + } + sqlite3_fputs(p->rowSeparator, p->out); + } + if( nArg>0 ){ + for(i=0; i<nArg; i++){ + output_csv(p, azArg[i], i<nArg-1); + } + sqlite3_fputs(p->rowSeparator, p->out); + } + setCrlfMode(p); + break; + } + case MODE_Insert: { + if( azArg==0 ) break; + sqlite3_fprintf(p->out, "INSERT INTO %s",p->zDestTable); + if( p->showHeader ){ + sqlite3_fputs("(", p->out); + for(i=0; i<nArg; i++){ + if( i>0 ) sqlite3_fputs(",", p->out); + if( quoteChar(azCol[i]) ){ + char *z = sqlite3_mprintf("\"%w\"", azCol[i]); + shell_check_oom(z); + sqlite3_fputs(z, p->out); + sqlite3_free(z); + }else{ + sqlite3_fprintf(p->out, "%s", azCol[i]); + } + } + sqlite3_fputs(")", p->out); + } + p->cnt++; + for(i=0; i<nArg; i++){ + sqlite3_fputs(i>0 ? "," : " VALUES(", p->out); + if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ + sqlite3_fputs("NULL", p->out); + }else if( aiType && aiType[i]==SQLITE_TEXT ){ + if( ShellHasFlag(p, SHFLG_Newlines) ){ + output_quoted_string(p, azArg[i]); + }else{ + output_quoted_escaped_string(p, azArg[i]); + } + }else if( aiType && aiType[i]==SQLITE_INTEGER ){ + sqlite3_fputs(azArg[i], p->out); + }else if( aiType && aiType[i]==SQLITE_FLOAT ){ + char z[50]; + double r = sqlite3_column_double(p->pStmt, i); + sqlite3_uint64 ur; + memcpy(&ur,&r,sizeof(r)); + if( ur==0x7ff0000000000000LL ){ + sqlite3_fputs("9.0e+999", p->out); + }else if( ur==0xfff0000000000000LL ){ + sqlite3_fputs("-9.0e+999", p->out); + }else{ + sqlite3_int64 ir = (sqlite3_int64)r; + if( r==(double)ir ){ + sqlite3_snprintf(50,z,"%lld.0", ir); + }else{ + sqlite3_snprintf(50,z,"%!.20g", r); + } + sqlite3_fputs(z, p->out); + } + }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ + const void *pBlob = sqlite3_column_blob(p->pStmt, i); + int nBlob = sqlite3_column_bytes(p->pStmt, i); + output_hex_blob(p->out, pBlob, nBlob); + }else if( isNumber(azArg[i], 0) ){ + sqlite3_fputs(azArg[i], p->out); + }else if( ShellHasFlag(p, SHFLG_Newlines) ){ + output_quoted_string(p, azArg[i]); + }else{ + output_quoted_escaped_string(p, azArg[i]); + } + } + sqlite3_fputs(");\n", p->out); + break; + } + case MODE_Json: { + if( azArg==0 ) break; + if( p->cnt==0 ){ + sqlite3_fputs("[{", p->out); + }else{ + sqlite3_fputs(",\n{", p->out); + } + p->cnt++; + for(i=0; i<nArg; i++){ + output_json_string(p->out, azCol[i], -1); + sqlite3_fputs(":", p->out); + if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ + sqlite3_fputs("null", p->out); + }else if( aiType && aiType[i]==SQLITE_FLOAT ){ + char z[50]; + double r = sqlite3_column_double(p->pStmt, i); + sqlite3_uint64 ur; + memcpy(&ur,&r,sizeof(r)); + if( ur==0x7ff0000000000000LL ){ + sqlite3_fputs("9.0e+999", p->out); + }else if( ur==0xfff0000000000000LL ){ + sqlite3_fputs("-9.0e+999", p->out); + }else{ + sqlite3_snprintf(50,z,"%!.20g", r); + sqlite3_fputs(z, p->out); + } + }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ + const void *pBlob = sqlite3_column_blob(p->pStmt, i); + int nBlob = sqlite3_column_bytes(p->pStmt, i); + output_json_string(p->out, pBlob, nBlob); + }else if( aiType && aiType[i]==SQLITE_TEXT ){ + output_json_string(p->out, azArg[i], -1); + }else{ + sqlite3_fputs(azArg[i], p->out); + } + if( i<nArg-1 ){ + sqlite3_fputs(",", p->out); + } + } + sqlite3_fputs("}", p->out); + break; + } + case MODE_Quote: { + if( azArg==0 ) break; + if( p->cnt==0 && p->showHeader ){ + for(i=0; i<nArg; i++){ + if( i>0 ) sqlite3_fputs(p->colSeparator, p->out); + output_quoted_string(p, azCol[i]); + } + sqlite3_fputs(p->rowSeparator, p->out); + } + p->cnt++; + for(i=0; i<nArg; i++){ + if( i>0 ) sqlite3_fputs(p->colSeparator, p->out); + if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ + sqlite3_fputs("NULL", p->out); + }else if( aiType && aiType[i]==SQLITE_TEXT ){ + output_quoted_string(p, azArg[i]); + }else if( aiType && aiType[i]==SQLITE_INTEGER ){ + sqlite3_fputs(azArg[i], p->out); + }else if( aiType && aiType[i]==SQLITE_FLOAT ){ + char z[50]; + double r = sqlite3_column_double(p->pStmt, i); + sqlite3_snprintf(50,z,"%!.20g", r); + sqlite3_fputs(z, p->out); + }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ + const void *pBlob = sqlite3_column_blob(p->pStmt, i); + int nBlob = sqlite3_column_bytes(p->pStmt, i); + output_hex_blob(p->out, pBlob, nBlob); + }else if( isNumber(azArg[i], 0) ){ + sqlite3_fputs(azArg[i], p->out); + }else{ + output_quoted_string(p, azArg[i]); + } + } + sqlite3_fputs(p->rowSeparator, p->out); + break; + } + case MODE_Ascii: { + if( p->cnt++==0 && p->showHeader ){ + for(i=0; i<nArg; i++){ + if( i>0 ) sqlite3_fputs(p->colSeparator, p->out); + sqlite3_fputs(azCol[i] ? azCol[i] : "", p->out); + } + sqlite3_fputs(p->rowSeparator, p->out); + } + if( azArg==0 ) break; + for(i=0; i<nArg; i++){ + if( i>0 ) sqlite3_fputs(p->colSeparator, p->out); + sqlite3_fputs(azArg[i] ? azArg[i] : p->nullValue, p->out); + } + sqlite3_fputs(p->rowSeparator, p->out); + break; + } + case MODE_EQP: { + eqp_append(p, atoi(azArg[0]), atoi(azArg[1]), azArg[3]); + break; + } + } + return 0; +} + +/* +** This is the callback routine that the SQLite library +** invokes for each row of a query result. +*/ +static int callback(void *pArg, int nArg, char **azArg, char **azCol){ + /* since we don't have type info, call the shell_callback with a NULL value */ + return shell_callback(pArg, nArg, azArg, azCol, NULL); +} + /* ** This is the callback routine from sqlite3_exec() that appends all ** output onto the end of a ShellText object. @@ -2567,7 +3198,7 @@ static void createSelftestTable(ShellState *p){ "DROP TABLE [_shell$self];" ,0,0,&zErrMsg); if( zErrMsg ){ - cli_printf(stderr, "SELFTEST initialization failure: %s\n", zErrMsg); + sqlite3_fprintf(stderr, "SELFTEST initialization failure: %s\n", zErrMsg); sqlite3_free(zErrMsg); } sqlite3_exec(p->db, "RELEASE selftest_init",0,0,0); @@ -2585,7 +3216,11 @@ static void set_table_name(ShellState *p, const char *zName){ p->zDestTable = 0; } if( zName==0 ) return; - p->zDestTable = sqlite3_mprintf("%s", zName); + if( quoteChar(zName) ){ + p->zDestTable = sqlite3_mprintf("\"%w\"", zName); + }else{ + p->zDestTable = sqlite3_mprintf("%s", zName); + } shell_check_oom(p->zDestTable); } @@ -2655,7 +3290,7 @@ static int run_table_dump_query( rc = sqlite3_prepare_v2(p->db, zSelect, -1, &pSelect, 0); if( rc!=SQLITE_OK || !pSelect ){ char *zContext = shell_error_context(zSelect, p->db); - cli_printf(p->out, "/**** ERROR: (%d) %s *****/\n%s", + sqlite3_fprintf(p->out, "/**** ERROR: (%d) %s *****/\n%s", rc, sqlite3_errmsg(p->db), zContext); sqlite3_free(zContext); if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; @@ -2665,22 +3300,22 @@ static int run_table_dump_query( nResult = sqlite3_column_count(pSelect); while( rc==SQLITE_ROW ){ z = (const char*)sqlite3_column_text(pSelect, 0); - cli_printf(p->out, "%s", z); + sqlite3_fprintf(p->out, "%s", z); for(i=1; i<nResult; i++){ - cli_printf(p->out, ",%s", sqlite3_column_text(pSelect, i)); + sqlite3_fprintf(p->out, ",%s", sqlite3_column_text(pSelect, i)); } if( z==0 ) z = ""; while( z[0] && (z[0]!='-' || z[1]!='-') ) z++; if( z[0] ){ - cli_puts("\n;\n", p->out); + sqlite3_fputs("\n;\n", p->out); }else{ - cli_puts(";\n", p->out); + sqlite3_fputs(";\n", p->out); } rc = sqlite3_step(pSelect); } rc = sqlite3_finalize(pSelect); if( rc!=SQLITE_OK ){ - cli_printf(p->out, "/**** ERROR: (%d) %s *****/\n", + sqlite3_fprintf(p->out, "/**** ERROR: (%d) %s *****/\n", rc, sqlite3_errmsg(p->db)); if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; } @@ -2740,7 +3375,7 @@ static void displayLinuxIoStats(FILE *out){ for(i=0; i<ArraySize(aTrans); i++){ int n = strlen30(aTrans[i].zPattern); if( cli_strncmp(aTrans[i].zPattern, z, n)==0 ){ - cli_printf(out, "%-36s %s", aTrans[i].zDesc, &z[n]); + sqlite3_fprintf(out, "%-36s %s", aTrans[i].zDesc, &z[n]); break; } } @@ -2772,7 +3407,7 @@ static void displayStatLine( }else{ sqlite3_snprintf(sizeof(zLine), zLine, zFormat, iHiwtr); } - cli_printf(out, "%-36s %s\n", zLabel, zLine); + sqlite3_fprintf(out, "%-36s %s\n", zLabel, zLine); } /* @@ -2794,22 +3429,22 @@ static int display_stats( sqlite3_stmt *pStmt = pArg->pStmt; char z[100]; nCol = sqlite3_column_count(pStmt); - cli_printf(out, "%-36s %d\n", "Number of output columns:", nCol); + sqlite3_fprintf(out, "%-36s %d\n", "Number of output columns:", nCol); for(i=0; i<nCol; i++){ sqlite3_snprintf(sizeof(z),z,"Column %d %nname:", i, &x); - cli_printf(out, "%-36s %s\n", z, sqlite3_column_name(pStmt,i)); + sqlite3_fprintf(out, "%-36s %s\n", z, sqlite3_column_name(pStmt,i)); #ifndef SQLITE_OMIT_DECLTYPE sqlite3_snprintf(30, z+x, "declared type:"); - cli_printf(out, "%-36s %s\n", z, sqlite3_column_decltype(pStmt, i)); + sqlite3_fprintf(out, "%-36s %s\n", z, sqlite3_column_decltype(pStmt, i)); #endif #ifdef SQLITE_ENABLE_COLUMN_METADATA sqlite3_snprintf(30, z+x, "database name:"); - cli_printf(out, "%-36s %s\n", z, + sqlite3_fprintf(out, "%-36s %s\n", z, sqlite3_column_database_name(pStmt,i)); sqlite3_snprintf(30, z+x, "table name:"); - cli_printf(out, "%-36s %s\n", z, sqlite3_column_table_name(pStmt,i)); + sqlite3_fprintf(out, "%-36s %s\n", z, sqlite3_column_table_name(pStmt,i)); sqlite3_snprintf(30, z+x, "origin name:"); - cli_printf(out, "%-36s %s\n", z,sqlite3_column_origin_name(pStmt,i)); + sqlite3_fprintf(out, "%-36s %s\n", z,sqlite3_column_origin_name(pStmt,i)); #endif } } @@ -2817,7 +3452,7 @@ static int display_stats( if( pArg->statsOn==3 ){ if( pArg->pStmt ){ iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP,bReset); - cli_printf(out, "VM-steps: %d\n", iCur); + sqlite3_fprintf(out, "VM-steps: %d\n", iCur); } return 0; } @@ -2846,55 +3481,55 @@ static int display_stats( iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, &iCur, &iHiwtr, bReset); - cli_printf(out, + sqlite3_fprintf(out, "Lookaside Slots Used: %d (max %d)\n", iCur, iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT, &iCur, &iHiwtr, bReset); - cli_printf(out, + sqlite3_fprintf(out, "Successful lookaside attempts: %d\n", iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, &iCur, &iHiwtr, bReset); - cli_printf(out, + sqlite3_fprintf(out, "Lookaside failures due to size: %d\n", iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, &iCur, &iHiwtr, bReset); - cli_printf(out, + sqlite3_fprintf(out, "Lookaside failures due to OOM: %d\n", iHiwtr); } iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset); - cli_printf(out, + sqlite3_fprintf(out, "Pager Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1); - cli_printf(out, + sqlite3_fprintf(out, "Page cache hits: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1); - cli_printf(out, + sqlite3_fprintf(out, "Page cache misses: %d\n", iCur); iHiwtr64 = iCur64 = -1; sqlite3_db_status64(db, SQLITE_DBSTATUS_TEMPBUF_SPILL, &iCur64, &iHiwtr64, 0); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1); - cli_printf(out, + sqlite3_fprintf(out, "Page cache writes: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_SPILL, &iCur, &iHiwtr, 1); - cli_printf(out, + sqlite3_fprintf(out, "Page cache spills: %d\n", iCur); - cli_printf(out, + sqlite3_fprintf(out, "Temporary data spilled to disk: %lld\n", iCur64); sqlite3_db_status64(db, SQLITE_DBSTATUS_TEMPBUF_SPILL, &iCur64, &iHiwtr64, 1); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); - cli_printf(out, + sqlite3_fprintf(out, "Schema Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_STMT_USED, &iCur, &iHiwtr, bReset); - cli_printf(out, + sqlite3_fprintf(out, "Statement Heap/Lookaside Usage: %d bytes\n", iCur); } @@ -2902,33 +3537,33 @@ static int display_stats( int iHit, iMiss; iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, bReset); - cli_printf(out, + sqlite3_fprintf(out, "Fullscan Steps: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_SORT, bReset); - cli_printf(out, + sqlite3_fprintf(out, "Sort Operations: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX,bReset); - cli_printf(out, + sqlite3_fprintf(out, "Autoindex Inserts: %d\n", iCur); iHit = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_HIT, bReset); iMiss = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_MISS, bReset); if( iHit || iMiss ){ - cli_printf(out, + sqlite3_fprintf(out, "Bloom filter bypass taken: %d/%d\n", iHit, iHit+iMiss); } iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); - cli_printf(out, + sqlite3_fprintf(out, "Virtual Machine Steps: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_REPREPARE,bReset); - cli_printf(out, + sqlite3_fprintf(out, "Reprepare operations: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_RUN, bReset); - cli_printf(out, + sqlite3_fprintf(out, "Number of times run: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_MEMUSED, bReset); - cli_printf(out, + sqlite3_fprintf(out, "Memory used by prepared stmt: %d\n", iCur); } @@ -2941,24 +3576,285 @@ static int display_stats( return 0; } + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +static int scanStatsHeight(sqlite3_stmt *p, int iEntry){ + int iPid = 0; + int ret = 1; + sqlite3_stmt_scanstatus_v2(p, iEntry, + SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid + ); + while( iPid!=0 ){ + int ii; + for(ii=0; 1; ii++){ + int iId; + int res; + res = sqlite3_stmt_scanstatus_v2(p, ii, + SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iId + ); + if( res ) break; + if( iId==iPid ){ + sqlite3_stmt_scanstatus_v2(p, ii, + SQLITE_SCANSTAT_PARENTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid + ); + } + } + ret++; + } + return ret; +} +#endif + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +static void display_explain_scanstats( + sqlite3 *db, /* Database to query */ + ShellState *pArg /* Pointer to ShellState */ +){ + static const int f = SQLITE_SCANSTAT_COMPLEX; + sqlite3_stmt *p = pArg->pStmt; + int ii = 0; + i64 nTotal = 0; + int nWidth = 0; + eqp_reset(pArg); + + for(ii=0; 1; ii++){ + const char *z = 0; + int n = 0; + if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&z) ){ + break; + } + n = (int)strlen(z) + scanStatsHeight(p, ii)*3; + if( n>nWidth ) nWidth = n; + } + nWidth += 4; + + sqlite3_stmt_scanstatus_v2(p, -1, SQLITE_SCANSTAT_NCYCLE, f, (void*)&nTotal); + for(ii=0; 1; ii++){ + i64 nLoop = 0; + i64 nRow = 0; + i64 nCycle = 0; + int iId = 0; + int iPid = 0; + const char *zo = 0; + const char *zName = 0; + char *zText = 0; + double rEst = 0.0; + + if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&zo) ){ + break; + } + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_EST,f,(void*)&rEst); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NLOOP,f,(void*)&nLoop); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NVISIT,f,(void*)&nRow); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NCYCLE,f,(void*)&nCycle); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_SELECTID,f,(void*)&iId); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_PARENTID,f,(void*)&iPid); + sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NAME,f,(void*)&zName); + + zText = sqlite3_mprintf("%s", zo); + if( nCycle>=0 || nLoop>=0 || nRow>=0 ){ + char *z = 0; + if( nCycle>=0 && nTotal>0 ){ + z = sqlite3_mprintf("%zcycles=%lld [%d%%]", z, + nCycle, ((nCycle*100)+nTotal/2) / nTotal + ); + } + if( nLoop>=0 ){ + z = sqlite3_mprintf("%z%sloops=%lld", z, z ? " " : "", nLoop); + } + if( nRow>=0 ){ + z = sqlite3_mprintf("%z%srows=%lld", z, z ? " " : "", nRow); + } + + if( zName && pArg->scanstatsOn>1 ){ + double rpl = (double)nRow / (double)nLoop; + z = sqlite3_mprintf("%z rpl=%.1f est=%.1f", z, rpl, rEst); + } + + zText = sqlite3_mprintf( + "% *z (%z)", -1*(nWidth-scanStatsHeight(p, ii)*3), zText, z + ); + } + + eqp_append(pArg, iId, iPid, zText); + sqlite3_free(zText); + } + + eqp_render(pArg, nTotal); +} +#endif + + /* -** Disable and restore .wheretrace and .treetrace/.selecttrace settings. +** Parameter azArray points to a zero-terminated array of strings. zStr +** points to a single nul-terminated string. Return non-zero if zStr +** is equal, according to strcmp(), to any of the strings in the array. +** Otherwise, return zero. */ -static unsigned int savedSelectTrace; -static unsigned int savedWhereTrace; -static void disable_debug_trace_modes(void){ - unsigned int zero = 0; - sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 0, &savedSelectTrace); - sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &zero); - sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 2, &savedWhereTrace); - sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &zero); -} -static void restore_debug_trace_modes(void){ - sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &savedSelectTrace); - sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &savedWhereTrace); +static int str_in_array(const char *zStr, const char **azArray){ + int i; + for(i=0; azArray[i]; i++){ + if( 0==cli_strcmp(zStr, azArray[i]) ) return 1; + } + return 0; } -/* Create the TEMP table used to store parameter bindings */ +/* +** If compiled statement pSql appears to be an EXPLAIN statement, allocate +** and populate the ShellState.aiIndent[] array with the number of +** spaces each opcode should be indented before it is output. +** +** The indenting rules are: +** +** * For each "Next", "Prev", "VNext" or "VPrev" instruction, indent +** all opcodes that occur between the p2 jump destination and the opcode +** itself by 2 spaces. +** +** * Do the previous for "Return" instructions for when P2 is positive. +** See tag-20220407a in wherecode.c and vdbe.c. +** +** * For each "Goto", if the jump destination is earlier in the program +** and ends on one of: +** Yield SeekGt SeekLt RowSetRead Rewind +** or if the P1 parameter is one instead of zero, +** then indent all opcodes between the earlier instruction +** and "Goto" by 2 spaces. +*/ +static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ + int *abYield = 0; /* True if op is an OP_Yield */ + int nAlloc = 0; /* Allocated size of p->aiIndent[], abYield */ + int iOp; /* Index of operation in p->aiIndent[] */ + + const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", + "Return", 0 }; + const char *azYield[] = { "Yield", "SeekLT", "SeekGT", "RowSetRead", + "Rewind", 0 }; + const char *azGoto[] = { "Goto", 0 }; + + /* The caller guarantees that the leftmost 4 columns of the statement + ** passed to this function are equivalent to the leftmost 4 columns + ** of EXPLAIN statement output. In practice the statement may be + ** an EXPLAIN, or it may be a query on the bytecode() virtual table. */ + assert( sqlite3_column_count(pSql)>=4 ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 0), "addr" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 1), "opcode" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 2), "p1" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 3), "p2" ) ); + + for(iOp=0; SQLITE_ROW==sqlite3_step(pSql); iOp++){ + int i; + int iAddr = sqlite3_column_int(pSql, 0); + const char *zOp = (const char*)sqlite3_column_text(pSql, 1); + int p1 = sqlite3_column_int(pSql, 2); + int p2 = sqlite3_column_int(pSql, 3); + + /* Assuming that p2 is an instruction address, set variable p2op to the + ** index of that instruction in the aiIndent[] array. p2 and p2op may be + ** different if the current instruction is part of a sub-program generated + ** by an SQL trigger or foreign key. */ + int p2op = (p2 + (iOp-iAddr)); + + /* Grow the p->aiIndent array as required */ + if( iOp>=nAlloc ){ + nAlloc += 100; + p->aiIndent = (int*)sqlite3_realloc64(p->aiIndent, nAlloc*sizeof(int)); + shell_check_oom(p->aiIndent); + abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int)); + shell_check_oom(abYield); + } + + abYield[iOp] = str_in_array(zOp, azYield); + p->aiIndent[iOp] = 0; + p->nIndent = iOp+1; + if( str_in_array(zOp, azNext) && p2op>0 ){ + for(i=p2op; i<iOp; i++) p->aiIndent[i] += 2; + } + if( str_in_array(zOp, azGoto) && p2op<iOp && (abYield[p2op] || p1) ){ + for(i=p2op; i<iOp; i++) p->aiIndent[i] += 2; + } + } + + p->iIndent = 0; + sqlite3_free(abYield); + sqlite3_reset(pSql); +} + +/* +** Free the array allocated by explain_data_prepare(). +*/ +static void explain_data_delete(ShellState *p){ + sqlite3_free(p->aiIndent); + p->aiIndent = 0; + p->nIndent = 0; + p->iIndent = 0; +} + +static void exec_prepared_stmt(ShellState*, sqlite3_stmt*); + +/* +** Display scan stats. +*/ +static void display_scanstats( + sqlite3 *db, /* Database to query */ + ShellState *pArg /* Pointer to ShellState */ +){ +#ifndef SQLITE_ENABLE_STMT_SCANSTATUS + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(pArg); +#else + if( pArg->scanstatsOn==3 ){ + const char *zSql = + " SELECT addr, opcode, p1, p2, p3, p4, p5, comment, nexec," + " format('% 6s (%.2f%%)'," + " CASE WHEN ncycle<100_000 THEN ncycle || ' '" + " WHEN ncycle<100_000_000 THEN (ncycle/1_000) || 'K'" + " WHEN ncycle<100_000_000_000 THEN (ncycle/1_000_000) || 'M'" + " ELSE (ncycle/1000_000_000) || 'G' END," + " ncycle*100.0/(sum(ncycle) OVER ())" + " ) AS cycles" + " FROM bytecode(?)"; + + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_stmt *pSave = pArg->pStmt; + pArg->pStmt = pStmt; + sqlite3_bind_pointer(pStmt, 1, pSave, "stmt-pointer", 0); + + pArg->cnt = 0; + pArg->cMode = MODE_ScanExp; + explain_data_prepare(pArg, pStmt); + exec_prepared_stmt(pArg, pStmt); + explain_data_delete(pArg); + + sqlite3_finalize(pStmt); + pArg->pStmt = pSave; + } + }else{ + display_explain_scanstats(db, pArg); + } +#endif +} + +/* +** Disable and restore .wheretrace and .treetrace/.selecttrace settings. +*/ +static unsigned int savedSelectTrace; +static unsigned int savedWhereTrace; +static void disable_debug_trace_modes(void){ + unsigned int zero = 0; + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 0, &savedSelectTrace); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &zero); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 2, &savedWhereTrace); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &zero); +} +static void restore_debug_trace_modes(void){ + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &savedSelectTrace); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &savedWhereTrace); +} + +/* Create the TEMP table used to store parameter bindings */ static void bind_table_init(ShellState *p){ int wrSchema = 0; int defensiveMode = 0; @@ -3032,8 +3928,6 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ memcpy(zBuf, &zVar[6], szVar-5); sqlite3_bind_text64(pStmt, i, zBuf, szVar-6, sqlite3_free, SQLITE_UTF8); } - }else if( strcmp(zVar, "$TIMER")==0 ){ - sqlite3_bind_double(pStmt, i, pArg->prevTimer); #ifdef SQLITE_ENABLE_CARRAY }else if( strncmp(zVar, "$carray_", 8)==0 ){ static char *azColorNames[] = { @@ -3072,6 +3966,580 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ sqlite3_finalize(pQ); } +/* +** UTF8 box-drawing characters. Imagine box lines like this: +** +** 1 +** | +** 4 --+-- 2 +** | +** 3 +** +** Each box characters has between 2 and 4 of the lines leading from +** the center. The characters are here identified by the numbers of +** their corresponding lines. +*/ +#define BOX_24 "\342\224\200" /* U+2500 --- */ +#define BOX_13 "\342\224\202" /* U+2502 | */ +#define BOX_23 "\342\224\214" /* U+250c ,- */ +#define BOX_34 "\342\224\220" /* U+2510 -, */ +#define BOX_12 "\342\224\224" /* U+2514 '- */ +#define BOX_14 "\342\224\230" /* U+2518 -' */ +#define BOX_123 "\342\224\234" /* U+251c |- */ +#define BOX_134 "\342\224\244" /* U+2524 -| */ +#define BOX_234 "\342\224\254" /* U+252c -,- */ +#define BOX_124 "\342\224\264" /* U+2534 -'- */ +#define BOX_1234 "\342\224\274" /* U+253c -|- */ + +/* Draw horizontal line N characters long using unicode box +** characters +*/ +static void print_box_line(FILE *out, int N){ + const char zDash[] = + BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 + BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24; + const int nDash = sizeof(zDash) - 1; + N *= 3; + while( N>nDash ){ + sqlite3_fputs(zDash, out); + N -= nDash; + } + sqlite3_fprintf(out, "%.*s", N, zDash); +} + +/* +** Draw a horizontal separator for a MODE_Box table. +*/ +static void print_box_row_separator( + ShellState *p, + int nArg, + const char *zSep1, + const char *zSep2, + const char *zSep3 +){ + int i; + if( nArg>0 ){ + sqlite3_fputs(zSep1, p->out); + print_box_line(p->out, p->actualWidth[0]+2); + for(i=1; i<nArg; i++){ + sqlite3_fputs(zSep2, p->out); + print_box_line(p->out, p->actualWidth[i]+2); + } + sqlite3_fputs(zSep3, p->out); + } + sqlite3_fputs("\n", p->out); +} + +/* +** z[] is a line of text that is to be displayed the .mode box or table or +** similar tabular formats. z[] might contain control characters such +** as \n, \t, \f, or \r. +** +** Compute characters to display on the first line of z[]. Stop at the +** first \r, \n, or \f. Expand \t into spaces. Return a copy (obtained +** from malloc()) of that first line, which caller should free sometime. +** Write anything to display on the next line into *pzTail. If this is +** the last line, write a NULL into *pzTail. (*pzTail is not allocated.) +*/ +static char *translateForDisplayAndDup( + ShellState *p, /* To access current settings */ + const unsigned char *z, /* Input text to be transformed */ + const unsigned char **pzTail, /* OUT: Tail of the input for next line */ + int mxWidth, /* Max width. 0 means no limit */ + u8 bWordWrap /* If true, avoid breaking mid-word */ +){ + int i; /* Input bytes consumed */ + int j; /* Output bytes generated */ + int k; /* Input bytes to be displayed */ + int n; /* Output column number */ + unsigned char *zOut; /* Output text */ + + if( z==0 ){ + *pzTail = 0; + return 0; + } + if( mxWidth<0 ) mxWidth = -mxWidth; + if( mxWidth==0 ) mxWidth = 1000000; + i = j = n = 0; + while( n<mxWidth ){ + unsigned char c = z[i]; + if( c>=0xc0 ){ + int u; + int len = decodeUtf8(&z[i], &u); + i += len; + j += len; + n += cli_wcwidth(u); + continue; + } + if( c>=' ' ){ + n++; + i++; + j++; + continue; + } + if( c==0 || c=='\n' || (c=='\r' && z[i+1]=='\n') ) break; + if( c=='\t' ){ + do{ + n++; + j++; + }while( (n&7)!=0 && n<mxWidth ); + i++; + continue; + } + if( c==0x1b && p->eEscMode==SHELL_ESC_OFF && (k = isVt100(&z[i]))>0 ){ + i += k; + j += k; + }else{ + n++; + j += 3; + i++; + } + } + if( n>=mxWidth && bWordWrap ){ + /* Perhaps try to back up to a better place to break the line */ + for(k=i; k>i/2; k--){ + if( IsSpace(z[k-1]) ) break; + } + if( k<=i/2 ){ + for(k=i; k>i/2; k--){ + if( IsAlnum(z[k-1])!=IsAlnum(z[k]) && (z[k]&0xc0)!=0x80 ) break; + } + } + if( k<=i/2 ){ + k = i; + }else{ + i = k; + while( z[i]==' ' ) i++; + } + }else{ + k = i; + } + if( n>=mxWidth && z[i]>=' ' ){ + *pzTail = &z[i]; + }else if( z[i]=='\r' && z[i+1]=='\n' ){ + *pzTail = z[i+2] ? &z[i+2] : 0; + }else if( z[i]==0 || z[i+1]==0 ){ + *pzTail = 0; + }else{ + *pzTail = &z[i+1]; + } + zOut = malloc( j+1 ); + shell_check_oom(zOut); + i = j = n = 0; + while( i<k ){ + unsigned char c = z[i]; + if( c>=0xc0 ){ + int u; + int len = decodeUtf8(&z[i], &u); + do{ zOut[j++] = z[i++]; }while( (--len)>0 ); + n += cli_wcwidth(u); + continue; + } + if( c>=' ' ){ + n++; + zOut[j++] = z[i++]; + continue; + } + if( c==0 ) break; + if( z[i]=='\t' ){ + do{ + n++; + zOut[j++] = ' '; + }while( (n&7)!=0 && n<mxWidth ); + i++; + continue; + } + switch( p->eEscMode ){ + case SHELL_ESC_SYMBOL: + zOut[j++] = 0xe2; + zOut[j++] = 0x90; + zOut[j++] = 0x80 + c; + break; + case SHELL_ESC_ASCII: + zOut[j++] = '^'; + zOut[j++] = 0x40 + c; + break; + case SHELL_ESC_OFF: { + int nn; + if( c==0x1b && (nn = isVt100(&z[i]))>0 ){ + memcpy(&zOut[j], &z[i], nn); + j += nn; + i += nn - 1; + }else{ + zOut[j++] = c; + } + break; + } + } + i++; + } + zOut[j] = 0; + return (char*)zOut; +} + +/* Return true if the text string z[] contains characters that need +** unistr() escaping. +*/ +static int needUnistr(const unsigned char *z){ + unsigned char c; + if( z==0 ) return 0; + while( (c = *z)>0x1f || c=='\t' || c=='\n' || (c=='\r' && z[1]=='\n') ){ z++; } + return c!=0; +} + +/* Extract the value of the i-th current column for pStmt as an SQL literal +** value. Memory is obtained from sqlite3_malloc64() and must be freed by +** the caller. +*/ +static char *quoted_column(sqlite3_stmt *pStmt, int i){ + switch( sqlite3_column_type(pStmt, i) ){ + case SQLITE_NULL: { + return sqlite3_mprintf("NULL"); + } + case SQLITE_INTEGER: + case SQLITE_FLOAT: { + return sqlite3_mprintf("%s",sqlite3_column_text(pStmt,i)); + } + case SQLITE_TEXT: { + const unsigned char *zText = sqlite3_column_text(pStmt,i); + return sqlite3_mprintf(needUnistr(zText)?"%#Q":"%Q",zText); + } + case SQLITE_BLOB: { + int j; + sqlite3_str *pStr = sqlite3_str_new(0); + const unsigned char *a = sqlite3_column_blob(pStmt,i); + int n = sqlite3_column_bytes(pStmt,i); + sqlite3_str_append(pStr, "x'", 2); + for(j=0; j<n; j++){ + sqlite3_str_appendf(pStr, "%02x", a[j]); + } + sqlite3_str_append(pStr, "'", 1); + return sqlite3_str_finish(pStr); + } + } + return 0; /* Not reached */ +} + +/* +** Run a prepared statement and output the result in one of the +** table-oriented formats: MODE_Column, MODE_Markdown, MODE_Table, +** or MODE_Box. +** +** This is different from ordinary exec_prepared_stmt() in that +** it has to run the entire query and gather the results into memory +** first, in order to determine column widths, before providing +** any output. +*/ +static void exec_prepared_stmt_columnar( + ShellState *p, /* Pointer to ShellState */ + sqlite3_stmt *pStmt /* Statement to run */ +){ + sqlite3_int64 nRow = 0; + int nColumn = 0; + char **azData = 0; + sqlite3_int64 nAlloc = 0; + char *abRowDiv = 0; + const unsigned char *uz; + const char *z; + char **azQuoted = 0; + int rc; + sqlite3_int64 i, nData; + int j, nTotal, w, n; + const char *colSep = 0; + const char *rowSep = 0; + const unsigned char **azNextLine = 0; + int bNextLine = 0; + int bMultiLineRowExists = 0; + int bw = p->cmOpts.bWordWrap; + const char *zEmpty = ""; + const char *zShowNull = p->nullValue; + + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_ROW ) return; + nColumn = sqlite3_column_count(pStmt); + if( nColumn==0 ) goto columnar_end; + nAlloc = nColumn*4; + if( nAlloc<=0 ) nAlloc = 1; + azData = sqlite3_malloc64( nAlloc*sizeof(char*) ); + shell_check_oom(azData); + azNextLine = sqlite3_malloc64( nColumn*sizeof(char*) ); + shell_check_oom(azNextLine); + memset((void*)azNextLine, 0, nColumn*sizeof(char*) ); + if( p->cmOpts.bQuote ){ + azQuoted = sqlite3_malloc64( nColumn*sizeof(char*) ); + shell_check_oom(azQuoted); + memset(azQuoted, 0, nColumn*sizeof(char*) ); + } + abRowDiv = sqlite3_malloc64( nAlloc/nColumn ); + shell_check_oom(abRowDiv); + if( nColumn>p->nWidth ){ + p->colWidth = realloc(p->colWidth, (nColumn+1)*2*sizeof(int)); + shell_check_oom(p->colWidth); + for(i=p->nWidth; i<nColumn; i++) p->colWidth[i] = 0; + p->nWidth = nColumn; + p->actualWidth = &p->colWidth[nColumn]; + } + memset(p->actualWidth, 0, nColumn*sizeof(int)); + for(i=0; i<nColumn; i++){ + w = p->colWidth[i]; + if( w<0 ) w = -w; + p->actualWidth[i] = w; + } + for(i=0; i<nColumn; i++){ + const unsigned char *zNotUsed; + int wx = p->colWidth[i]; + if( wx==0 ){ + wx = p->cmOpts.iWrap; + } + if( wx<0 ) wx = -wx; + uz = (const unsigned char*)sqlite3_column_name(pStmt,i); + if( uz==0 ) uz = (u8*)""; + azData[i] = translateForDisplayAndDup(p, uz, &zNotUsed, wx, bw); + } + do{ + int useNextLine = bNextLine; + bNextLine = 0; + if( (nRow+2)*nColumn >= nAlloc ){ + nAlloc *= 2; + azData = sqlite3_realloc64(azData, nAlloc*sizeof(char*)); + shell_check_oom(azData); + abRowDiv = sqlite3_realloc64(abRowDiv, nAlloc/nColumn); + shell_check_oom(abRowDiv); + } + abRowDiv[nRow] = 1; + nRow++; + for(i=0; i<nColumn; i++){ + int wx = p->colWidth[i]; + if( wx==0 ){ + wx = p->cmOpts.iWrap; + } + if( wx<0 ) wx = -wx; + if( useNextLine ){ + uz = azNextLine[i]; + if( uz==0 ) uz = (u8*)zEmpty; + }else if( p->cmOpts.bQuote ){ + assert( azQuoted!=0 ); + sqlite3_free(azQuoted[i]); + azQuoted[i] = quoted_column(pStmt,i); + uz = (const unsigned char*)azQuoted[i]; + }else{ + uz = (const unsigned char*)sqlite3_column_text(pStmt,i); + if( uz==0 ) uz = (u8*)zShowNull; + } + azData[nRow*nColumn + i] + = translateForDisplayAndDup(p, uz, &azNextLine[i], wx, bw); + if( azNextLine[i] ){ + bNextLine = 1; + abRowDiv[nRow-1] = 0; + bMultiLineRowExists = 1; + } + } + }while( bNextLine || sqlite3_step(pStmt)==SQLITE_ROW ); + nTotal = nColumn*(nRow+1); + for(i=0; i<nTotal; i++){ + z = azData[i]; + if( z==0 ) z = (char*)zEmpty; + n = strlenChar(z); + j = i%nColumn; + if( n>p->actualWidth[j] ) p->actualWidth[j] = n; + } + if( seenInterrupt ) goto columnar_end; + switch( p->cMode ){ + case MODE_Column: { + colSep = " "; + rowSep = "\n"; + if( p->showHeader ){ + for(i=0; i<nColumn; i++){ + w = p->actualWidth[i]; + if( p->colWidth[i]<0 ) w = -w; + utf8_width_print(p->out, w, azData[i]); + sqlite3_fputs(i==nColumn-1?"\n":" ", p->out); + } + for(i=0; i<nColumn; i++){ + print_dashes(p->out, p->actualWidth[i]); + sqlite3_fputs(i==nColumn-1?"\n":" ", p->out); + } + } + break; + } + case MODE_Table: { + colSep = " | "; + rowSep = " |\n"; + print_row_separator(p, nColumn, "+"); + sqlite3_fputs("| ", p->out); + for(i=0; i<nColumn; i++){ + w = p->actualWidth[i]; + n = strlenChar(azData[i]); + sqlite3_fprintf(p->out, "%*s%s%*s", (w-n)/2, "", + azData[i], (w-n+1)/2, ""); + sqlite3_fputs(i==nColumn-1?" |\n":" | ", p->out); + } + print_row_separator(p, nColumn, "+"); + break; + } + case MODE_Markdown: { + colSep = " | "; + rowSep = " |\n"; + sqlite3_fputs("| ", p->out); + for(i=0; i<nColumn; i++){ + w = p->actualWidth[i]; + n = strlenChar(azData[i]); + sqlite3_fprintf(p->out, "%*s%s%*s", (w-n)/2, "", + azData[i], (w-n+1)/2, ""); + sqlite3_fputs(i==nColumn-1?" |\n":" | ", p->out); + } + print_row_separator(p, nColumn, "|"); + break; + } + case MODE_Box: { + colSep = " " BOX_13 " "; + rowSep = " " BOX_13 "\n"; + print_box_row_separator(p, nColumn, BOX_23, BOX_234, BOX_34); + sqlite3_fputs(BOX_13 " ", p->out); + for(i=0; i<nColumn; i++){ + w = p->actualWidth[i]; + n = strlenChar(azData[i]); + sqlite3_fprintf(p->out, "%*s%s%*s%s", + (w-n)/2, "", azData[i], (w-n+1)/2, "", + i==nColumn-1?" "BOX_13"\n":" "BOX_13" "); + } + print_box_row_separator(p, nColumn, BOX_123, BOX_1234, BOX_134); + break; + } + } + for(i=nColumn, j=0; i<nTotal; i++, j++){ + if( j==0 && p->cMode!=MODE_Column ){ + sqlite3_fputs(p->cMode==MODE_Box?BOX_13" ":"| ", p->out); + } + z = azData[i]; + if( z==0 ) z = p->nullValue; + w = p->actualWidth[j]; + if( p->colWidth[j]<0 ) w = -w; + utf8_width_print(p->out, w, z); + if( j==nColumn-1 ){ + sqlite3_fputs(rowSep, p->out); + if( bMultiLineRowExists && abRowDiv[i/nColumn-1] && i+1<nTotal ){ + if( p->cMode==MODE_Table ){ + print_row_separator(p, nColumn, "+"); + }else if( p->cMode==MODE_Box ){ + print_box_row_separator(p, nColumn, BOX_123, BOX_1234, BOX_134); + }else if( p->cMode==MODE_Column ){ + sqlite3_fputs("\n", p->out); + } + } + j = -1; + if( seenInterrupt ) goto columnar_end; + }else{ + sqlite3_fputs(colSep, p->out); + } + } + if( p->cMode==MODE_Table ){ + print_row_separator(p, nColumn, "+"); + }else if( p->cMode==MODE_Box ){ + print_box_row_separator(p, nColumn, BOX_12, BOX_124, BOX_14); + } +columnar_end: + if( seenInterrupt ){ + sqlite3_fputs("Interrupt\n", p->out); + } + nData = (nRow+1)*nColumn; + for(i=0; i<nData; i++){ + z = azData[i]; + if( z!=zEmpty && z!=zShowNull ) free(azData[i]); + } + sqlite3_free(azData); + sqlite3_free((void*)azNextLine); + sqlite3_free(abRowDiv); + if( azQuoted ){ + for(i=0; i<nColumn; i++) sqlite3_free(azQuoted[i]); + sqlite3_free(azQuoted); + } +} + +/* +** Run a prepared statement +*/ +static void exec_prepared_stmt( + ShellState *pArg, /* Pointer to ShellState */ + sqlite3_stmt *pStmt /* Statement to run */ +){ + int rc; + sqlite3_uint64 nRow = 0; + + if( pArg->cMode==MODE_Column + || pArg->cMode==MODE_Table + || pArg->cMode==MODE_Box + || pArg->cMode==MODE_Markdown + ){ + exec_prepared_stmt_columnar(pArg, pStmt); + return; + } + + /* perform the first step. this will tell us if we + ** have a result set or not and how wide it is. + */ + rc = sqlite3_step(pStmt); + /* if we have a result set... */ + if( SQLITE_ROW == rc ){ + /* allocate space for col name ptr, value ptr, and type */ + int nCol = sqlite3_column_count(pStmt); + void *pData = sqlite3_malloc64(3*nCol*sizeof(const char*) + 1); + if( !pData ){ + shell_out_of_memory(); + }else{ + char **azCols = (char **)pData; /* Names of result columns */ + char **azVals = &azCols[nCol]; /* Results */ + int *aiTypes = (int *)&azVals[nCol]; /* Result types */ + int i, x; + assert(sizeof(int) <= sizeof(char *)); + /* save off ptrs to column names */ + for(i=0; i<nCol; i++){ + azCols[i] = (char *)sqlite3_column_name(pStmt, i); + } + do{ + nRow++; + /* extract the data and data types */ + for(i=0; i<nCol; i++){ + aiTypes[i] = x = sqlite3_column_type(pStmt, i); + if( x==SQLITE_BLOB + && pArg + && (pArg->cMode==MODE_Insert || pArg->cMode==MODE_Quote) + ){ + azVals[i] = ""; + }else{ + azVals[i] = (char*)sqlite3_column_text(pStmt, i); + } + if( !azVals[i] && (aiTypes[i]!=SQLITE_NULL) ){ + rc = SQLITE_NOMEM; + break; /* from for */ + } + } /* end for */ + + /* if data and types extracted successfully... */ + if( SQLITE_ROW == rc ){ + /* call the supplied callback with the result row data */ + if( shell_callback(pArg, nCol, azVals, azCols, aiTypes) ){ + rc = SQLITE_ABORT; + }else{ + rc = sqlite3_step(pStmt); + } + } + } while( SQLITE_ROW == rc ); + sqlite3_free(pData); + if( pArg->cMode==MODE_Json ){ + sqlite3_fputs("]\n", pArg->out); + }else if( pArg->cMode==MODE_Www ){ + sqlite3_fputs("</TABLE>\n<PRE>\n", pArg->out); + }else if( pArg->cMode==MODE_Count ){ + char zBuf[200]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "%llu row%s\n", + nRow, nRow!=1 ? "s" : ""); + printf("%s", zBuf); + } + } + } +} + #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) /* ** This function is called to process SQL if the previous shell command @@ -3123,8 +4591,8 @@ static int expertFinish( if( bVerbose ){ const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES); - cli_puts("-- Candidates -----------------------------\n", out); - cli_printf(out, "%s\n", zCand); + sqlite3_fputs("-- Candidates -----------------------------\n", out); + sqlite3_fprintf(out, "%s\n", zCand); } for(i=0; i<nQuery; i++){ const char *zSql = sqlite3_expert_report(p, i, EXPERT_REPORT_SQL); @@ -3132,12 +4600,12 @@ static int expertFinish( const char *zEQP = sqlite3_expert_report(p, i, EXPERT_REPORT_PLAN); if( zIdx==0 ) zIdx = "(no new indexes)\n"; if( bVerbose ){ - cli_printf(out, + sqlite3_fprintf(out, "-- Query %d --------------------------------\n" "%s\n\n" ,i+1, zSql); } - cli_printf(out, "%s\n%s\n", zIdx, zEQP); + sqlite3_fprintf(out, "%s\n%s\n", zIdx, zEQP); } } } @@ -3172,18 +4640,18 @@ static int expertDotCommand( } else if( n>=2 && 0==cli_strncmp(z, "-sample", n) ){ if( i==(nArg-1) ){ - cli_printf(stderr, "option requires an argument: %s\n", z); + sqlite3_fprintf(stderr, "option requires an argument: %s\n", z); rc = SQLITE_ERROR; }else{ iSample = (int)integerValue(azArg[++i]); if( iSample<0 || iSample>100 ){ - cli_printf(stderr,"value out of range: %s\n", azArg[i]); + sqlite3_fprintf(stderr,"value out of range: %s\n", azArg[i]); rc = SQLITE_ERROR; } } } else{ - cli_printf(stderr,"unknown option: %s\n", z); + sqlite3_fprintf(stderr,"unknown option: %s\n", z); rc = SQLITE_ERROR; } } @@ -3191,7 +4659,7 @@ static int expertDotCommand( if( rc==SQLITE_OK ){ pState->expert.pExpert = sqlite3_expert_new(pState->db, &zErr); if( pState->expert.pExpert==0 ){ - cli_printf(stderr, + sqlite3_fprintf(stderr, "sqlite3_expert_new: %s\n", zErr ? zErr : "out of memory"); rc = SQLITE_ERROR; }else{ @@ -3206,15 +4674,6 @@ static int expertDotCommand( } #endif /* !SQLITE_OMIT_VIRTUALTABLE && !SQLITE_OMIT_AUTHORIZATION */ -/* -** QRF write callback -*/ -static int shellWriteQR(void *pX, const char *z, sqlite3_int64 n){ - ShellState *pArg = (ShellState*)pX; - cli_printf(pArg->out, "%.*s", (int)n, z); - return SQLITE_OK; -} - /* ** Execute a statement or set of statements. Print ** any result rows/columns depending on the current mode @@ -3234,31 +4693,10 @@ static int shell_exec( int rc2; const char *zLeftover; /* Tail of unprocessed SQL */ sqlite3 *db = pArg->db; - unsigned char eStyle; - sqlite3_qrf_spec spec; if( pzErrMsg ){ *pzErrMsg = NULL; } - memcpy(&spec, &pArg->mode.spec, sizeof(spec)); - spec.xWrite = shellWriteQR; - spec.pWriteArg = (void*)pArg; - if( pArg->mode.eMode==MODE_Insert && ShellHasFlag(pArg, SHFLG_PreserveRowid) ){ - spec.bTitles = QRF_SW_On; - } - assert( pArg->mode.eMode>=0 && pArg->mode.eMode<ArraySize(aModeInfo) ); - eStyle = aModeInfo[pArg->mode.eMode].eStyle; - if( pArg->mode.bAutoScreenWidth ){ - spec.nScreenWidth = shellScreenWidth(); - } - if( spec.eBlob==QRF_BLOB_Auto ){ - switch( spec.eText ){ - case QRF_TEXT_Relaxed: /* fall through */ - case QRF_TEXT_Sql: spec.eBlob = QRF_BLOB_Sql; break; - case QRF_TEXT_Json: spec.eBlob = QRF_BLOB_Json; break; - default: spec.eBlob = QRF_BLOB_Text; break; - } - } #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) if( pArg->expert.pExpert ){ @@ -3267,7 +4705,7 @@ static int shell_exec( } #endif - while( zSql && zSql[0] && (SQLITE_OK == rc) ){ + while( zSql[0] && (SQLITE_OK == rc) ){ static const char *zStmtSql; rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover); if( SQLITE_OK != rc ){ @@ -3275,7 +4713,6 @@ static int shell_exec( *pzErrMsg = save_err_msg(db, "in prepare", rc, zSql); } }else{ - int isExplain; if( !pStmt ){ /* this happens for a comment or white-space */ zSql = zLeftover; @@ -3286,78 +4723,89 @@ static int shell_exec( if( zStmtSql==0 ) zStmtSql = ""; while( IsSpace(zStmtSql[0]) ) zStmtSql++; - /* save off the prepared statement handle */ + /* save off the prepared statement handle and reset row count */ if( pArg ){ pArg->pStmt = pStmt; + pArg->cnt = 0; } - + /* Show the EXPLAIN QUERY PLAN if .eqp is on */ - isExplain = sqlite3_stmt_isexplain(pStmt); - if( pArg && pArg->mode.autoEQP && isExplain==0 && pArg->dot.nArg==0 ){ + if( pArg && pArg->autoEQP && sqlite3_stmt_isexplain(pStmt)==0 ){ + sqlite3_stmt *pExplain; int triggerEQP = 0; - u8 savedEnableTimer = pArg->enableTimer; - pArg->enableTimer = 0; disable_debug_trace_modes(); sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, -1, &triggerEQP); - if( pArg->mode.autoEQP>=AUTOEQP_trigger ){ + if( pArg->autoEQP>=AUTOEQP_trigger ){ sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 1, 0); } - sqlite3_reset(pStmt); - spec.eStyle = QRF_STYLE_Auto; - sqlite3_stmt_explain(pStmt, 2); - sqlite3_format_query_result(pStmt, &spec, 0); - if( pArg->mode.autoEQP>=AUTOEQP_full ){ - sqlite3_reset(pStmt); - sqlite3_stmt_explain(pStmt, 1); - sqlite3_format_query_result(pStmt, &spec, 0); + pExplain = pStmt; + sqlite3_reset(pExplain); + rc = sqlite3_stmt_explain(pExplain, 2); + if( rc==SQLITE_OK ){ + bind_prepared_stmt(pArg, pExplain); + while( sqlite3_step(pExplain)==SQLITE_ROW ){ + const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3); + int iEqpId = sqlite3_column_int(pExplain, 0); + int iParentId = sqlite3_column_int(pExplain, 1); + if( zEQPLine==0 ) zEQPLine = ""; + if( zEQPLine[0]=='-' ) eqp_render(pArg, 0); + eqp_append(pArg, iEqpId, iParentId, zEQPLine); + } + eqp_render(pArg, 0); } - - if( pArg->mode.autoEQP>=AUTOEQP_trigger && triggerEQP==0 ){ + if( pArg->autoEQP>=AUTOEQP_full ){ + /* Also do an EXPLAIN for ".eqp full" mode */ + sqlite3_reset(pExplain); + rc = sqlite3_stmt_explain(pExplain, 1); + if( rc==SQLITE_OK ){ + pArg->cMode = MODE_Explain; + assert( sqlite3_stmt_isexplain(pExplain)==1 ); + bind_prepared_stmt(pArg, pExplain); + explain_data_prepare(pArg, pExplain); + exec_prepared_stmt(pArg, pExplain); + explain_data_delete(pArg); + } + } + if( pArg->autoEQP>=AUTOEQP_trigger && triggerEQP==0 ){ sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 0, 0); } sqlite3_reset(pStmt); sqlite3_stmt_explain(pStmt, 0); restore_debug_trace_modes(); - pArg->enableTimer = savedEnableTimer; } - bind_prepared_stmt(pArg, pStmt); - if( isExplain && pArg->mode.autoExplain ){ - spec.eStyle = isExplain==1 ? QRF_STYLE_Explain : QRF_STYLE_Eqp; - sqlite3_format_query_result(pStmt, &spec, pzErrMsg); - }else if( pArg->mode.eMode==MODE_Www ){ - cli_printf(pArg->out, - "</PRE>\n" - "<TABLE border='1' cellspacing='0' cellpadding='2'>\n"); - spec.eStyle = QRF_STYLE_Html; - sqlite3_format_query_result(pStmt, &spec, pzErrMsg); - cli_printf(pArg->out, - "</TABLE>\n" - "<PRE>"); - }else{ - spec.eStyle = eStyle; - sqlite3_format_query_result(pStmt, &spec, pzErrMsg); + if( pArg ){ + int bIsExplain = (sqlite3_stmt_isexplain(pStmt)==1); + pArg->cMode = pArg->mode; + if( pArg->autoExplain ){ + if( bIsExplain ){ + pArg->cMode = MODE_Explain; + } + if( sqlite3_stmt_isexplain(pStmt)==2 ){ + pArg->cMode = MODE_EQP; + } + } + + /* If the shell is currently in ".explain" mode, gather the extra + ** data required to add indents to the output.*/ + if( pArg->cMode==MODE_Explain && bIsExplain ){ + explain_data_prepare(pArg, pStmt); + } } + bind_prepared_stmt(pArg, pStmt); + exec_prepared_stmt(pArg, pStmt); + explain_data_delete(pArg); + eqp_render(pArg, 0); + /* print usage stats if stats on */ if( pArg && pArg->statsOn ){ display_stats(db, pArg, 0); } /* print loop-counters if required */ - if( pArg && pArg->mode.scanstatsOn ){ - char *zErr = 0; - switch( pArg->mode.scanstatsOn ){ - case 1: spec.eStyle = QRF_STYLE_Stats; break; - case 2: spec.eStyle = QRF_STYLE_StatsEst; break; - default: spec.eStyle = QRF_STYLE_StatsVm; break; - } - sqlite3_reset(pStmt); - rc = sqlite3_format_query_result(pStmt, &spec, &zErr); - if( rc ){ - cli_printf(stderr, "Stats query failed: %s\n", zErr); - sqlite3_free(zErr); - } + if( pArg && pArg->scanstatsOn ){ + display_scanstats(db, pArg); } /* Finalize the statement just executed. If this fails, save a @@ -3553,14 +5001,14 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ */ if( db_int(p->db, "SELECT count(*) FROM sqlite_sequence")>0 ){ if( !p->writableSchema ){ - cli_puts("PRAGMA writable_schema=ON;\n", p->out); + sqlite3_fputs("PRAGMA writable_schema=ON;\n", p->out); p->writableSchema = 1; } - cli_puts("CREATE TABLE IF NOT EXISTS sqlite_sequence(name,seq);\n" + sqlite3_fputs("CREATE TABLE IF NOT EXISTS sqlite_sequence(name,seq);\n" "DELETE FROM sqlite_sequence;\n", p->out); } }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 && !noSys ){ - if( !dataOnly ) cli_puts("ANALYZE sqlite_schema;\n", p->out); + if( !dataOnly ) sqlite3_fputs("ANALYZE sqlite_schema;\n", p->out); }else if( cli_strncmp(zTable, "sqlite_", 7)==0 ){ return 0; }else if( dataOnly ){ @@ -3568,7 +5016,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ }else if( cli_strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){ char *zIns; if( !p->writableSchema ){ - cli_puts("PRAGMA writable_schema=ON;\n", p->out); + sqlite3_fputs("PRAGMA writable_schema=ON;\n", p->out); p->writableSchema = 1; } zIns = sqlite3_mprintf( @@ -3576,7 +5024,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ "VALUES('table','%q','%q',0,'%q');", zTable, zTable, zSql); shell_check_oom(zIns); - cli_printf(p->out, "%s\n", zIns); + sqlite3_fprintf(p->out, "%s\n", zIns); sqlite3_free(zIns); return 0; }else{ @@ -3588,7 +5036,8 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ ShellText sTable; char **azCol; int i; - Mode savedMode; + char *savedDestTable; + int savedMode; azCol = tableColumnList(p, zTable); if( azCol==0 ){ @@ -3631,20 +5080,18 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ appendText(&sSelect, " FROM ", 0); appendText(&sSelect, zTable, quoteChar(zTable)); - + savedDestTable = p->zDestTable; savedMode = p->mode; - p->mode.spec.zTableName = (char*)zTable; - p->mode.eMode = MODE_Insert; - p->mode.spec.eText = QRF_TEXT_Sql; - p->mode.spec.eBlob = QRF_BLOB_Sql; - p->mode.spec.bTitles = QRF_No; + p->zDestTable = sTable.zTxt; + p->mode = p->cMode = MODE_Insert; rc = shell_exec(p, sSelect.zTxt, 0); if( (rc&0xff)==SQLITE_CORRUPT ){ - cli_puts("/****** CORRUPTION ERROR *******/\n", p->out); + sqlite3_fputs("/****** CORRUPTION ERROR *******/\n", p->out); toggleSelectOrder(p->db); shell_exec(p, sSelect.zTxt, 0); toggleSelectOrder(p->db); } + p->zDestTable = savedDestTable; p->mode = savedMode; freeText(&sTable); freeText(&sSelect); @@ -3670,9 +5117,9 @@ static int run_schema_dump_query( if( rc==SQLITE_CORRUPT ){ char *zQ2; int len = strlen30(zQuery); - cli_puts("/****** CORRUPTION ERROR *******/\n", p->out); + sqlite3_fputs("/****** CORRUPTION ERROR *******/\n", p->out); if( zErr ){ - cli_printf(p->out, "/****** %s ******/\n", zErr); + sqlite3_fprintf(p->out, "/****** %s ******/\n", zErr); sqlite3_free(zErr); zErr = 0; } @@ -3681,7 +5128,7 @@ static int run_schema_dump_query( sqlite3_snprintf(len+100, zQ2, "%s ORDER BY rowid DESC", zQuery); rc = sqlite3_exec(p->db, zQ2, dump_callback, p, &zErr); if( rc ){ - cli_printf(p->out, "/****** ERROR: %s ******/\n", zErr); + sqlite3_fprintf(p->out, "/****** ERROR: %s ******/\n", zErr); }else{ rc = SQLITE_CORRUPT; } @@ -3739,8 +5186,8 @@ static const char *(azHelp[]) = { ".cd DIRECTORY Change the working directory to DIRECTORY", #endif ".changes on|off Show number of rows changed by SQL", - ".check OPTIONS ... Verify the results of a .testcase", #ifndef SQLITE_SHELL_FIDDLE + ".check GLOB Fail if output since .testcase does not match", ".clone NEWDB Clone data into NEWDB from the existing database", #endif ".connection [close] [#] Open or close an auxiliary database connection", @@ -3782,10 +5229,23 @@ static const char *(azHelp[]) = { " --schema SCHEMA Use SCHEMA instead of \"main\"", " --help Show CMD details", ".fullschema ?--indent? Show schema and the content of sqlite_stat tables", - ",headers on|off Turn display of headers on or off", + ".headers on|off Turn display of headers on or off", ".help ?-all? ?PATTERN? Show help text for PATTERN", #ifndef SQLITE_SHELL_FIDDLE ".import FILE TABLE Import data from FILE into TABLE", + " Options:", + " --ascii Use \\037 and \\036 as column and row separators", + " --csv Use , and \\n as column and row separators", + " --skip N Skip the first N rows of input", + " --schema S Target table to be S.TABLE", + " -v \"Verbose\" - increase auxiliary output", + " Notes:", + " * If TABLE does not exist, it is created. The first row of input", + " determines the column names.", + " * If neither --csv or --ascii are used, the input mode is derived", + " from the \".mode\" output mode", + " * If FILE begins with \"|\" then it is a command that generates the", + " input text.", #endif #ifndef SQLITE_OMIT_TEST_CONTROL ".imposter INDEX TABLE Create imposter table TABLE on index INDEX", @@ -3810,12 +5270,42 @@ static const char *(azHelp[]) = { ".log on|off Turn logging on or off.", #endif ".mode ?MODE? ?OPTIONS? Set output mode", + " MODE is one of:", + " ascii Columns/rows delimited by 0x1F and 0x1E", + " box Tables using unicode box-drawing characters", + " csv Comma-separated values", + " column Output in columns. (See .width)", + " html HTML <table> code", + " insert SQL insert statements for TABLE", + " json Results in a JSON array", + " line One value per line", + " list Values delimited by \"|\"", + " markdown Markdown table format", + " qbox Shorthand for \"box --wrap 60 --quote\"", + " quote Escape answers as for SQL", + " table ASCII-art table", + " tabs Tab-separated values", + " tcl TCL list elements", + " OPTIONS: (for columnar modes or insert mode):", + " --escape T ctrl-char escape; T is one of: symbol, ascii, off", + " --wrap N Wrap output lines to no longer than N characters", + " --wordwrap B Wrap or not at word boundaries per B (on/off)", + " --ww Shorthand for \"--wordwrap 1\"", + " --quote Quote output text as SQL literals", + " --noquote Do not quote output text", + " TABLE The name of SQL table used for \"insert\" mode", #ifndef SQLITE_SHELL_FIDDLE ".nonce STRING Suspend safe mode for one command if nonce matches", #endif ".nullvalue STRING Use STRING in place of NULL values", #ifndef SQLITE_SHELL_FIDDLE ".once ?OPTIONS? ?FILE? Output for the next SQL command only to FILE", + " If FILE begins with '|' then open as a pipe", + " --bom Put a UTF8 byte-order mark at the beginning", + " -e Send output to the system text editor", + " --plain Use text/plain output instead of HTML for -w option", + " -w Send output as HTML to a web browser (same as \".www\")", + " -x Send output as CSV to a spreadsheet (same as \".excel\")", /* Note that .open is (partially) available in WASM builds but is ** currently only intended to be used by the fiddle tool, not ** end users, so is "undocumented." */ @@ -3841,6 +5331,14 @@ static const char *(azHelp[]) = { " --zip FILE is a ZIP archive", #ifndef SQLITE_SHELL_FIDDLE ".output ?FILE? Send output to FILE or stdout if FILE is omitted", + " If FILE begins with '|' then open it as a pipe.", + " If FILE is 'off' then output is disabled.", + " Options:", + " --bom Prefix output with a UTF8 byte-order mark", + " -e Send output to the system text editor", + " --plain Use text/plain for -w option", + " -w Send output to a web browser", + " -x Send output as CSV to a spreadsheet", #endif ".parameter CMD ... Manage SQL parameter bindings", " clear Erase all bindings", @@ -3856,7 +5354,6 @@ static const char *(azHelp[]) = { " --once Do no more than one progress interrupt", " --quiet|-q No output except at interrupts", " --reset Reset the count for each input and interrupt", - " --timeout S Halt after running for S seconds", #endif ".prompt MAIN CONTINUE Replace the standard prompts", #ifndef SQLITE_SHELL_FIDDLE @@ -3884,7 +5381,7 @@ static const char *(azHelp[]) = { " Options:", " --init Create a new SELFTEST table", " -v Verbose output", - ",separator COL ?ROW? Change the column and row separators", + ".separator COL ?ROW? Change the column and row separators", #if defined(SQLITE_ENABLE_SESSION) ".session ?NAME? CMD ... Create or control sessions", " Subcommands:", @@ -3911,7 +5408,7 @@ static const char *(azHelp[]) = { #if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) ".shell CMD ARGS... Run CMD ARGS... in a system shell", #endif - ",show Show the current values for various settings", + ".show Show the current values for various settings", ".stats ?ARG? Show stats or turn stats on or off", " off Turn off automatic stat display", " on Turn on automatic stat display", @@ -3921,11 +5418,13 @@ static const char *(azHelp[]) = { ".system CMD ARGS... Run CMD ARGS... in a system shell", #endif ".tables ?TABLE? List names of tables matching LIKE pattern TABLE", - ".testcase NAME Begin a test case.", +#ifndef SQLITE_SHELL_FIDDLE + ",testcase NAME Begin redirecting output to 'testcase-out.txt'", +#endif ",testctrl CMD ... Run various sqlite3_test_control() operations", " Run \".testctrl\" with no arguments for details", ".timeout MS Try opening locked tables for MS milliseconds", - ".timer on|off|once Turn SQL timer on or off.", + ".timer on|off Turn SQL timer on or off", #ifndef SQLITE_OMIT_TRACE ".trace ?OPTIONS? Output each SQL statement as it is run", " FILE Send output to FILE", @@ -3950,7 +5449,7 @@ static const char *(azHelp[]) = { ".vfsinfo ?AUX? Information about the top-level VFS", ".vfslist List all available VFSes", ".vfsname ?AUX? Print the name of the VFS stack", - ",width NUM1 NUM2 ... Set minimum column widths for columnar output", + ".width NUM1 NUM2 ... Set minimum column widths for columnar output", " Negative values right-justify", #ifndef SQLITE_SHELL_FIDDLE ".www Display output of the next command in web browser", @@ -3958,19 +5457,6 @@ static const char *(azHelp[]) = { #endif }; -INSERT-USAGE-TEXT-HERE - -/* -** Return a pointer to usage text for zCmd, or NULL if none exists. -*/ -static const char *findUsage(const char *zCmd){ - int i; - for(i=0; i<ArraySize(aUsage); i++){ - if( sqlite3_strglob(zCmd, aUsage[i].zCmd)==0 ) return aUsage[i].zUsage; - } - return 0; -} - /* ** Output help text for commands that match zPattern. ** @@ -4000,7 +5486,6 @@ static int showHelp(FILE *out, const char *zPattern){ int j = 0; int n = 0; char *zPat; - const char *zHit = 0; if( zPattern==0 ){ /* Show just the first line for all help topics */ zPattern = "[a-z]"; @@ -4018,46 +5503,37 @@ static int showHelp(FILE *out, const char *zPattern){ show = 0; }else if( azHelp[i][0]==',' ){ show = 1; - cli_printf(out, ".%s\n", &azHelp[i][1]); + sqlite3_fprintf(out, ".%s\n", &azHelp[i][1]); n++; }else if( show ){ - cli_printf(out, "%s\n", azHelp[i]); + sqlite3_fprintf(out, "%s\n", azHelp[i]); } } return n; } /* Seek documented commands for which zPattern is an exact prefix */ - zPat = sqlite3_mprintf(".%s*", zPattern[0]=='.' ? &zPattern[1] : zPattern); + zPat = sqlite3_mprintf(".%s*", zPattern); shell_check_oom(zPat); for(i=0; i<ArraySize(azHelp); i++){ if( sqlite3_strglob(zPat, azHelp[i])==0 ){ - if( zHit ) cli_printf(out, "%s\n", zHit); - zHit = azHelp[i]; + sqlite3_fprintf(out, "%s\n", azHelp[i]); j = i+1; n++; } } + sqlite3_free(zPat); if( n ){ if( n==1 ){ - const char *zUsage = findUsage(zPat); - if( zUsage ){ - cli_puts(zUsage, out); - }else{ - /* when zPattern is a prefix of exactly one command, then include - ** the details of that command, which should begin at offset j */ - cli_printf(out, "%s\n", zHit); - while( j<ArraySize(azHelp)-1 && azHelp[j][0]==' ' ){ - cli_printf(out, "%s\n", azHelp[j]); - j++; - } + /* when zPattern is a prefix of exactly one command, then include + ** the details of that command, which should begin at offset j */ + while( j<ArraySize(azHelp)-1 && azHelp[j][0]==' ' ){ + sqlite3_fprintf(out, "%s\n", azHelp[j]); + j++; } - }else{ - cli_printf(out, "%s\n", zHit); } + return n; } - sqlite3_free(zPat); - if( n ) return n; /* Look for documented commands that contain zPattern anywhere. ** Show complete text of all documented commands that match. */ @@ -4070,10 +5546,10 @@ static int showHelp(FILE *out, const char *zPattern){ } if( azHelp[i][0]=='.' ) j = i; if( sqlite3_strlike(zPat, azHelp[i], 0)==0 ){ - cli_printf(out, "%s\n", azHelp[j]); + sqlite3_fprintf(out, "%s\n", azHelp[j]); while( j<ArraySize(azHelp)-1 && azHelp[j+1][0]==' ' ){ j++; - cli_printf(out, "%s\n", azHelp[j]); + sqlite3_fprintf(out, "%s\n", azHelp[j]); } i = j; n++; @@ -4110,7 +5586,7 @@ static char *readFile(const char *zName, int *pnByte){ if( in==0 ) return 0; rc = fseek(in, 0, SEEK_END); if( rc!=0 ){ - cli_printf(stderr,"Error: '%s' not seekable\n", zName); + sqlite3_fprintf(stderr,"Error: '%s' not seekable\n", zName); fclose(in); return 0; } @@ -4118,7 +5594,7 @@ static char *readFile(const char *zName, int *pnByte){ rewind(in); pBuf = sqlite3_malloc64( nIn+1 ); if( pBuf==0 ){ - cli_puts("Error: out of memory\n", stderr); + sqlite3_fputs("Error: out of memory\n", stderr); fclose(in); return 0; } @@ -4126,7 +5602,7 @@ static char *readFile(const char *zName, int *pnByte){ fclose(in); if( nRead!=1 ){ sqlite3_free(pBuf); - cli_printf(stderr,"Error: cannot read '%s'\n", zName); + sqlite3_fprintf(stderr,"Error: cannot read '%s'\n", zName); return 0; } pBuf[nIn] = 0; @@ -4183,54 +5659,8 @@ static int session_filter(void *pCtx, const char *zTab){ #endif /* -** Return the size of the named file in bytes. Or return a negative -** number if the file does not exist. -*/ -static sqlite3_int64 fileSize(const char *zFile){ -#if defined(_WIN32) || defined(WIN32) - struct _stat64 x; - if( _stat64(zFile, &x)!=0 ) return -1; - return (sqlite3_int64)x.st_size; -#else - struct stat x; - if( stat(zFile, &x)!=0 ) return -1; - return (sqlite3_int64)x.st_size; -#endif -} - -/* -** Return true if zFile is an SQLite database. -** -** Algorithm: -** * If the file does not exist -> return false -** * If the size of the file is not a multiple of 512 -> return false -** * If sqlite3_open() fails -> return false -** * if sqlite3_prepare() or sqlite3_step() fails -> return false -** * Otherwise -> return true -*/ -static int isDatabaseFile(const char *zFile, int openFlags){ - sqlite3 *db = 0; - sqlite3_stmt *pStmt = 0; - int rc; - sqlite3_int64 sz = fileSize(zFile); - if( sz<512 || (sz%512)!=0 ) return 0; - if( sqlite3_open_v2(zFile, &db, openFlags, 0)==SQLITE_OK - && sqlite3_prepare_v2(db,"SELECT count(*) FROM sqlite_schema",-1,&pStmt,0) - ==SQLITE_OK - && sqlite3_step(pStmt)==SQLITE_ROW - ){ - rc = 1; - }else{ - rc = 0; - } - sqlite3_finalize(pStmt); - sqlite3_close(db); - return rc; -} - -/* -** Try to deduce the type of file for zName based on its content. Return -** one of the SHELL_OPEN_* constants. +** Try to deduce the type of file for zName based on its content. Return +** one of the SHELL_OPEN_* constants. ** ** If the file does not exist or is empty but its name looks like a ZIP ** archive and the dfltZip flag is true, then assume it is a ZIP archive. @@ -4240,12 +5670,20 @@ static int isDatabaseFile(const char *zFile, int openFlags){ int deduceDatabaseType(const char *zName, int dfltZip, int openFlags){ FILE *f; size_t n; + sqlite3 *db = 0; + sqlite3_stmt *pStmt = 0; int rc = SHELL_OPEN_UNSPEC; char zBuf[100]; if( access(zName,0)!=0 ) goto database_type_by_name; - if( isDatabaseFile(zName, openFlags) ){ + if( sqlite3_open_v2(zName, &db, openFlags, 0)==SQLITE_OK + && sqlite3_prepare_v2(db,"SELECT count(*) FROM sqlite_schema",-1,&pStmt,0) + ==SQLITE_OK + && sqlite3_step(pStmt)==SQLITE_ROW + ){ rc = SHELL_OPEN_NORMAL; } + sqlite3_finalize(pStmt); + sqlite3_close(db); if( rc==SHELL_OPEN_NORMAL ) return SHELL_OPEN_NORMAL; f = sqlite3_fopen(zName, "rb"); if( f==0 ) goto database_type_by_name; @@ -4280,35 +5718,6 @@ database_type_by_name: return rc; } -/* -** If the text in z[] is the name of a readable file and that file appears -** to contain SQL text and/or dot-commands, then return true. If z[] is -** not a file, or if the file is unreadable, or if the file is a database -** or anything else that is not SQL text and dot-commands, then return false. -** -** If the bLeaveUninit flag is set, then be sure to leave SQLite in an -** uninitialized state. This means invoking sqlite3_shutdown() after any -** SQLite API is used. -** -** Some amount of guesswork is involved in this decision. -*/ -static int isScriptFile(const char *z, int bLeaveUninit){ - sqlite3_int64 sz = fileSize(z); - if( sz<=0 ) return 0; - if( (sz%512)==0 ){ - int rc; - sqlite3_initialize(); - rc = isDatabaseFile(z, SQLITE_OPEN_READONLY); - if( bLeaveUninit ){ - sqlite3_shutdown(); - } - if( rc ) return 0; /* Is a database */ - } - if( sqlite3_strlike("%.sql",z,0)==0 ) return 1; - if( sqlite3_strlike("%.txt",z,0)==0 ) return 1; - return 0; -} - #ifndef SQLITE_OMIT_DESERIALIZE /* ** Reconstruct an in-memory database using the output from the "dbtotxt" @@ -4330,7 +5739,7 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){ if( zDbFilename ){ in = sqlite3_fopen(zDbFilename, "r"); if( in==0 ){ - cli_printf(stderr,"cannot open \"%s\" for reading\n", zDbFilename); + sqlite3_fprintf(stderr,"cannot open \"%s\" for reading\n", zDbFilename); return 0; } nLine = 0; @@ -4346,7 +5755,7 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){ if( rc!=2 ) goto readHexDb_error; if( n<0 ) goto readHexDb_error; if( pgsz<512 || pgsz>65536 || (pgsz & (pgsz-1))!=0 ){ - cli_puts("invalid pagesize\n", stderr); + sqlite3_fputs("invalid pagesize\n", stderr); goto readHexDb_error; } sz = ((i64)n+pgsz-1)&~(pgsz-1); /* Round up to nearest multiple of pgsz */ @@ -4357,7 +5766,7 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){ int j = 0; /* Page number from "| page" line */ int k = 0; /* Offset from "| page" line */ if( nLine>=2000000000 ){ - cli_printf(stderr, "input too big\n"); + sqlite3_fprintf(stderr, "input too big\n"); goto readHexDb_error; } rc = sscanf(zLine, "| page %d offset %d", &j, &k); @@ -4398,7 +5807,7 @@ readHexDb_error: p->lineno = nLine; } sqlite3_free(a); - cli_printf(stderr,"Error on line %lld of --hexdb input\n", nLine); + sqlite3_fprintf(stderr,"Error on line %lld of --hexdb input\n", nLine); return 0; } #endif /* SQLITE_OMIT_DESERIALIZE */ @@ -4503,19 +5912,19 @@ static void open_db(ShellState *p, int openFlags){ } } if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ - cli_printf(stderr,"Error: unable to open database \"%s\": %s\n", + sqlite3_fprintf(stderr,"Error: unable to open database \"%s\": %s\n", zDbFilename, sqlite3_errmsg(p->db)); if( (openFlags & OPEN_DB_KEEPALIVE)==0 ){ - cli_exit(1); + exit(1); } sqlite3_close(p->db); sqlite3_open(":memory:", &p->db); if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ - cli_puts("Also: unable to open substitute in-memory database.\n", + sqlite3_fputs("Also: unable to open substitute in-memory database.\n", stderr); - cli_exit(1); + exit(1); }else{ - cli_printf(stderr, + sqlite3_fprintf(stderr, "Notice: using substitute in-memory database instead of \"%s\"\n", zDbFilename); } @@ -4593,8 +6002,6 @@ static void open_db(ShellState *p, int openFlags){ shellModuleSchema, 0, 0); sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p, shellPutsFunc, 0, 0); - sqlite3_create_function(p->db, "shell_format_schema", 2, SQLITE_UTF8, p, - shellFormatSchema, 0, 0); sqlite3_create_function(p->db, "usleep",1,SQLITE_UTF8,0, shellUSleepFunc, 0, 0); #ifndef SQLITE_NOHAVE_SYSTEM @@ -4629,7 +6036,7 @@ static void open_db(ShellState *p, int openFlags){ SQLITE_DESERIALIZE_RESIZEABLE | SQLITE_DESERIALIZE_FREEONCLOSE); if( rc ){ - cli_printf(stderr,"Error: sqlite3_deserialize() returns %d\n", rc); + sqlite3_fprintf(stderr,"Error: sqlite3_deserialize() returns %d\n", rc); } if( p->szMax>0 ){ sqlite3_file_control(p->db, "main", SQLITE_FCNTL_SIZE_LIMIT, &p->szMax); @@ -4644,7 +6051,7 @@ static void open_db(ShellState *p, int openFlags){ } #endif sqlite3_db_config( - p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->mode.scanstatsOn, (int*)0 + p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->scanstatsOn, (int*)0 ); } } @@ -4655,7 +6062,7 @@ static void open_db(ShellState *p, int openFlags){ void close_db(sqlite3 *db){ int rc = sqlite3_close(db); if( rc ){ - cli_printf(stderr, + sqlite3_fprintf(stderr, "Error: sqlite3_close() returns %d: %s\n", rc, sqlite3_errmsg(db)); } } @@ -4828,7 +6235,7 @@ static int booleanValue(const char *zArg){ if( sqlite3_stricmp(zArg, "off")==0 || sqlite3_stricmp(zArg,"no")==0 ){ return 0; } - cli_printf(stderr, + sqlite3_fprintf(stderr, "ERROR: Not a boolean value: \"%s\". Assuming \"no\".\n", zArg); return 0; } @@ -4856,18 +6263,18 @@ static void output_file_close(FILE *f){ ** recognized and do the right thing. NULL is returned if the output ** filename is "off". */ -static FILE *output_file_open(ShellState *p, const char *zFile){ +static FILE *output_file_open(const char *zFile){ FILE *f; if( cli_strcmp(zFile,"stdout")==0 ){ f = stdout; }else if( cli_strcmp(zFile, "stderr")==0 ){ f = stderr; - }else if( cli_strcmp(zFile, "off")==0 || p->bSafeMode ){ + }else if( cli_strcmp(zFile, "off")==0 ){ f = 0; }else{ f = sqlite3_fopen(zFile, "w"); if( f==0 ){ - cli_printf(stderr,"Error: cannot open \"%s\"\n", zFile); + sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zFile); } } return f; @@ -4920,12 +6327,12 @@ static int sql_trace_callback( switch( mType ){ case SQLITE_TRACE_ROW: case SQLITE_TRACE_STMT: { - cli_printf(p->traceOut, "%.*s;\n", (int)nSql, zSql); + sqlite3_fprintf(p->traceOut, "%.*s;\n", (int)nSql, zSql); break; } case SQLITE_TRACE_PROFILE: { sqlite3_int64 nNanosec = pX ? *(sqlite3_int64*)pX : 0; - cli_printf(p->traceOut, + sqlite3_fprintf(p->traceOut, "%.*s; -- %lld ns\n", (int)nSql, zSql, nNanosec); break; } @@ -4954,9 +6361,7 @@ struct ImportCtx { const char *zFile; /* Name of the input file */ FILE *in; /* Read the CSV text from this input stream */ int (SQLITE_CDECL *xCloser)(FILE*); /* Func to close in */ - char *zIn; /* Input text */ char *z; /* Accumulated text for a field */ - i64 nUsed; /* Bytes of zIn[] used so far */ i64 n; /* Number of bytes in z */ i64 nAlloc; /* Space allocated for z[] */ int nLine; /* Current line number */ @@ -4966,8 +6371,6 @@ struct ImportCtx { int cTerm; /* Character that terminated the most recent field */ int cColSep; /* The column separator character. (Usually ",") */ int cRowSep; /* The row separator character. (Usually "\n") */ - int cQEscape; /* Escape character with "...". 0 for none */ - int cUQEscape; /* Escape character not with "...". 0 for none */ }; /* Clean up resourced used by an ImportCtx */ @@ -4978,28 +6381,9 @@ static void import_cleanup(ImportCtx *p){ } sqlite3_free(p->z); p->z = 0; - if( p->zIn ){ - sqlite3_free(p->zIn); - p->zIn = 0; - } -} - -/* Read a single character of the .import input text. Return EOF -** at end-of-file. -*/ -static int import_getc(ImportCtx *p){ - if( p->in ){ - return fgetc(p->in); - }else if( p->zIn && p->zIn[p->nUsed]!=0 ){ - return p->zIn[p->nUsed++]; - }else{ - return EOF; - } } -/* Append a single byte to the field value begin constructed -** in the p->z[] buffer -*/ +/* Append a single byte to z[] */ static void import_append_char(ImportCtx *p, int c){ if( p->n+1>=p->nAlloc ){ p->nAlloc += p->nAlloc + 100; @@ -5015,8 +6399,8 @@ static void import_append_char(ImportCtx *p, int c){ ** + Input comes from p->in. ** + Store results in p->z of length p->n. Space to hold p->z comes ** from sqlite3_malloc64(). -** + Use p->cColSep as the column separator. The default is ",". -** + Use p->cRowSep as the row separator. The default is "\n". +** + Use p->cSep as the column separator. The default is ",". +** + Use p->rSep as the row separator. The default is "\n". ** + Keep track of the line number in p->nLine. ** + Store the character that terminates the field in p->cTerm. Store ** EOF on end-of-file. @@ -5027,7 +6411,7 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ int cSep = (u8)p->cColSep; int rSep = (u8)p->cRowSep; p->n = 0; - c = import_getc(p); + c = fgetc(p->in); if( c==EOF || seenInterrupt ){ p->cTerm = EOF; return 0; @@ -5036,17 +6420,10 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ int pc, ppc; int startLine = p->nLine; int cQuote = c; - int cEsc = (u8)p->cQEscape; pc = ppc = 0; while( 1 ){ - c = import_getc(p); + c = fgetc(p->in); if( c==rSep ) p->nLine++; - if( c==cEsc && cEsc!=0 ){ - c = import_getc(p); - import_append_char(p, c); - ppc = pc = 0; - continue; - } if( c==cQuote ){ if( pc==cQuote ){ pc = 0; @@ -5063,11 +6440,11 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ break; } if( pc==cQuote && c!='\r' ){ - cli_printf(stderr,"%s:%d: unescaped %c character\n", - p->zFile, p->nLine, cQuote); + sqlite3_fprintf(stderr,"%s:%d: unescaped %c character\n", + p->zFile, p->nLine, cQuote); } if( c==EOF ){ - cli_printf(stderr,"%s:%d: unterminated %c-quoted field\n", + sqlite3_fprintf(stderr,"%s:%d: unterminated %c-quoted field\n", p->zFile, startLine, cQuote); p->cTerm = c; break; @@ -5079,13 +6456,12 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ }else{ /* If this is the first field being parsed and it begins with the ** UTF-8 BOM (0xEF BB BF) then skip the BOM */ - int cEsc = p->cUQEscape; if( (c&0xff)==0xef && p->bNotFirst==0 ){ import_append_char(p, c); - c = import_getc(p); + c = fgetc(p->in); if( (c&0xff)==0xbb ){ import_append_char(p, c); - c = import_getc(p); + c = fgetc(p->in); if( (c&0xff)==0xbf ){ p->bNotFirst = 1; p->n = 0; @@ -5094,9 +6470,8 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ } } while( c!=EOF && c!=cSep && c!=rSep ){ - if( c==cEsc && cEsc!=0 ) c = import_getc(p); import_append_char(p, c); - c = import_getc(p); + c = fgetc(p->in); } if( c==rSep ){ p->nLine++; @@ -5114,8 +6489,8 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ ** + Input comes from p->in. ** + Store results in p->z of length p->n. Space to hold p->z comes ** from sqlite3_malloc64(). -** + Use p->cColSep as the column separator. The default is "\x1F". -** + Use p->cRowSep as the row separator. The default is "\x1E". +** + Use p->cSep as the column separator. The default is "\x1F". +** + Use p->rSep as the row separator. The default is "\x1E". ** + Keep track of the row number in p->nLine. ** + Store the character that terminates the field in p->cTerm. Store ** EOF on end-of-file. @@ -5126,14 +6501,14 @@ static char *SQLITE_CDECL ascii_read_one_field(ImportCtx *p){ int cSep = (u8)p->cColSep; int rSep = (u8)p->cRowSep; p->n = 0; - c = import_getc(p); + c = fgetc(p->in); if( c==EOF || seenInterrupt ){ p->cTerm = EOF; return 0; } while( c!=EOF && c!=cSep && c!=rSep ){ import_append_char(p, c); - c = import_getc(p); + c = fgetc(p->in); } if( c==rSep ){ p->nLine++; @@ -5168,7 +6543,7 @@ static void tryToCloneData( shell_check_oom(zQuery); rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); if( rc ){ - cli_printf(stderr,"Error %d: %s on [%s]\n", + sqlite3_fprintf(stderr,"Error %d: %s on [%s]\n", sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery); goto end_data_xfer; } @@ -5185,7 +6560,7 @@ static void tryToCloneData( memcpy(zInsert+i, ");", 3); rc = sqlite3_prepare_v2(newDb, zInsert, -1, &pInsert, 0); if( rc ){ - cli_printf(stderr,"Error %d: %s on [%s]\n", + sqlite3_fprintf(stderr,"Error %d: %s on [%s]\n", sqlite3_extended_errcode(newDb), sqlite3_errmsg(newDb), zInsert); goto end_data_xfer; } @@ -5221,7 +6596,7 @@ static void tryToCloneData( } /* End for */ rc = sqlite3_step(pInsert); if( rc!=SQLITE_OK && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){ - cli_printf(stderr,"Error %d: %s\n", + sqlite3_fprintf(stderr,"Error %d: %s\n", sqlite3_extended_errcode(newDb), sqlite3_errmsg(newDb)); } sqlite3_reset(pInsert); @@ -5239,7 +6614,7 @@ static void tryToCloneData( shell_check_oom(zQuery); rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); if( rc ){ - cli_printf(stderr,"Warning: cannot step \"%s\" backwards", zTable); + sqlite3_fprintf(stderr,"Warning: cannot step \"%s\" backwards", zTable); break; } } /* End for(k=0...) */ @@ -5276,7 +6651,7 @@ static void tryToCloneSchema( shell_check_oom(zQuery); rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); if( rc ){ - cli_printf(stderr, + sqlite3_fprintf(stderr, "Error: (%d) %s on [%s]\n", sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery); goto end_schema_xfer; @@ -5286,10 +6661,10 @@ static void tryToCloneSchema( zSql = sqlite3_column_text(pQuery, 1); if( zName==0 || zSql==0 ) continue; if( sqlite3_stricmp((char*)zName, "sqlite_sequence")!=0 ){ - cli_printf(stdout, "%s... ", zName); fflush(stdout); + sqlite3_fprintf(stdout, "%s... ", zName); fflush(stdout); sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg); if( zErrMsg ){ - cli_printf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql); + sqlite3_fprintf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql); sqlite3_free(zErrMsg); zErrMsg = 0; } @@ -5307,7 +6682,7 @@ static void tryToCloneSchema( shell_check_oom(zQuery); rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); if( rc ){ - cli_printf(stderr,"Error: (%d) %s on [%s]\n", + sqlite3_fprintf(stderr,"Error: (%d) %s on [%s]\n", sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery); goto end_schema_xfer; } @@ -5316,10 +6691,10 @@ static void tryToCloneSchema( zSql = sqlite3_column_text(pQuery, 1); if( zName==0 || zSql==0 ) continue; if( sqlite3_stricmp((char*)zName, "sqlite_sequence")==0 ) continue; - cli_printf(stdout, "%s... ", zName); fflush(stdout); + sqlite3_fprintf(stdout, "%s... ", zName); fflush(stdout); sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg); if( zErrMsg ){ - cli_printf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql); + sqlite3_fprintf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql); sqlite3_free(zErrMsg); zErrMsg = 0; } @@ -5343,12 +6718,12 @@ static void tryToClone(ShellState *p, const char *zNewDb){ int rc; sqlite3 *newDb = 0; if( access(zNewDb,0)==0 ){ - cli_printf(stderr,"File \"%s\" already exists.\n", zNewDb); + sqlite3_fprintf(stderr,"File \"%s\" already exists.\n", zNewDb); return; } rc = sqlite3_open(zNewDb, &newDb); if( rc ){ - cli_printf(stderr, + sqlite3_fprintf(stderr, "Cannot create output database: %s\n", sqlite3_errmsg(newDb)); }else{ sqlite3_exec(p->db, "PRAGMA writable_schema=ON;", 0, 0, 0); @@ -5367,12 +6742,12 @@ static void tryToClone(ShellState *p, const char *zNewDb){ */ static void output_redir(ShellState *p, FILE *pfNew){ if( p->out != stdout ){ - cli_puts("Output already redirected.\n", stderr); + sqlite3_fputs("Output already redirected.\n", stderr); }else{ p->out = pfNew; setCrlfMode(p); - if( p->mode.eMode==MODE_Www ){ - cli_puts( + if( p->mode==MODE_Www ){ + sqlite3_fputs( "<!DOCTYPE html>\n" "<HTML><BODY><PRE>\n", p->out @@ -5394,8 +6769,8 @@ static void output_reset(ShellState *p){ pclose(p->out); #endif }else{ - if( p->mode.eMode==MODE_Www ){ - cli_puts("</PRE></BODY></HTML>\n", p->out); + if( p->mode==MODE_Www ){ + sqlite3_fputs("</PRE></BODY></HTML>\n", p->out); } output_file_close(p->out); #ifndef SQLITE_NOHAVE_SYSTEM @@ -5411,7 +6786,7 @@ static void output_reset(ShellState *p){ char *zCmd; zCmd = sqlite3_mprintf("%s %s", zXdgOpenCmd, p->zTempFile); if( system(zCmd) ){ - cli_printf(stderr,"Failed: [%s]\n", zCmd); + sqlite3_fprintf(stderr,"Failed: [%s]\n", zCmd); }else{ /* Give the start/open/xdg-open command some time to get ** going before we continue, and potential delete the @@ -5419,7 +6794,7 @@ static void output_reset(ShellState *p){ sqlite3_sleep(2000); } sqlite3_free(zCmd); - modePop(p); + outputModePop(p); p->doXdgOpen = 0; } #endif /* !defined(SQLITE_NOHAVE_SYSTEM) */ @@ -5427,10 +6802,6 @@ static void output_reset(ShellState *p){ p->outfile[0] = 0; p->out = stdout; setCrlfMode(p); - if( cli_output_capture ){ - sqlite3_str_free(cli_output_capture); - cli_output_capture = 0; - } } #else # define output_redir(SS,pfO) @@ -5515,7 +6886,7 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ "SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1", -1, &pStmt, 0); if( rc ){ - cli_printf(stderr,"error: %s\n", sqlite3_errmsg(p->db)); + sqlite3_fprintf(stderr,"error: %s\n", sqlite3_errmsg(p->db)); sqlite3_finalize(pStmt); return 1; } @@ -5528,28 +6899,28 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ memcpy(aHdr, pb, 100); sqlite3_finalize(pStmt); }else{ - cli_puts("unable to read database header\n", stderr); + sqlite3_fputs("unable to read database header\n", stderr); sqlite3_finalize(pStmt); return 1; } i = get2byteInt(aHdr+16); if( i==1 ) i = 65536; - cli_printf(p->out, "%-20s %d\n", "database page size:", i); - cli_printf(p->out, "%-20s %d\n", "write format:", aHdr[18]); - cli_printf(p->out, "%-20s %d\n", "read format:", aHdr[19]); - cli_printf(p->out, "%-20s %d\n", "reserved bytes:", aHdr[20]); + sqlite3_fprintf(p->out, "%-20s %d\n", "database page size:", i); + sqlite3_fprintf(p->out, "%-20s %d\n", "write format:", aHdr[18]); + sqlite3_fprintf(p->out, "%-20s %d\n", "read format:", aHdr[19]); + sqlite3_fprintf(p->out, "%-20s %d\n", "reserved bytes:", aHdr[20]); for(i=0; i<ArraySize(aField); i++){ int ofst = aField[i].ofst; unsigned int val = get4byteInt(aHdr + ofst); - cli_printf(p->out, "%-20s %u", aField[i].zName, val); + sqlite3_fprintf(p->out, "%-20s %u", aField[i].zName, val); switch( ofst ){ case 56: { - if( val==1 ) cli_puts(" (utf8)", p->out); - if( val==2 ) cli_puts(" (utf16le)", p->out); - if( val==3 ) cli_puts(" (utf16be)", p->out); + if( val==1 ) sqlite3_fputs(" (utf8)", p->out); + if( val==2 ) sqlite3_fputs(" (utf16le)", p->out); + if( val==3 ) sqlite3_fputs(" (utf16be)", p->out); } } - cli_puts("\n", p->out); + sqlite3_fputs("\n", p->out); } if( zDb==0 ){ zSchemaTab = sqlite3_mprintf("main.sqlite_schema"); @@ -5560,11 +6931,11 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ } for(i=0; i<ArraySize(aQuery); i++){ int val = db_int(p->db, aQuery[i].zSql, zSchemaTab); - cli_printf(p->out, "%-20s %d\n", aQuery[i].zName, val); + sqlite3_fprintf(p->out, "%-20s %d\n", aQuery[i].zName, val); } sqlite3_free(zSchemaTab); sqlite3_file_control(p->db, zDb, SQLITE_FCNTL_DATA_VERSION, &iDataVersion); - cli_printf(p->out, "%-20s %u\n", "data version", iDataVersion); + sqlite3_fprintf(p->out, "%-20s %u\n", "data version", iDataVersion); return 0; } #endif /* SQLITE_SHELL_HAVE_RECOVER */ @@ -5624,7 +6995,7 @@ static int shell_dbtotxt_command(ShellState *p, int nArg, char **azArg){ } zName = strdup(zTail); shell_check_oom(zName); - cli_printf(p->out, "| size %lld pagesize %d filename %s\n", + sqlite3_fprintf(p->out, "| size %lld pagesize %d filename %s\n", nPage*pgSz, pgSz, zName); sqlite3_finalize(pStmt); pStmt = 0; @@ -5640,27 +7011,27 @@ static int shell_dbtotxt_command(ShellState *p, int nArg, char **azArg){ for(j=0; j<16 && aLine[j]==0; j++){} if( j==16 ) continue; if( !seenPageLabel ){ - cli_printf(p->out, "| page %lld offset %lld\n",pgno,(pgno-1)*pgSz); + sqlite3_fprintf(p->out, "| page %lld offset %lld\n",pgno,(pgno-1)*pgSz); seenPageLabel = 1; } - cli_printf(p->out, "| %5d:", i); - for(j=0; j<16; j++) cli_printf(p->out, " %02x", aLine[j]); - cli_printf(p->out, " "); + sqlite3_fprintf(p->out, "| %5d:", i); + for(j=0; j<16; j++) sqlite3_fprintf(p->out, " %02x", aLine[j]); + sqlite3_fprintf(p->out, " "); for(j=0; j<16; j++){ unsigned char c = (unsigned char)aLine[j]; - cli_printf(p->out, "%c", bShow[c]); + sqlite3_fprintf(p->out, "%c", bShow[c]); } - cli_printf(p->out, "\n"); + sqlite3_fprintf(p->out, "\n"); } } sqlite3_finalize(pStmt); - cli_printf(p->out, "| end %s\n", zName); + sqlite3_fprintf(p->out, "| end %s\n", zName); free(zName); return 0; dbtotxt_error: if( rc ){ - cli_printf(stderr, "ERROR: %s\n", sqlite3_errmsg(p->db)); + sqlite3_fprintf(stderr, "ERROR: %s\n", sqlite3_errmsg(p->db)); } sqlite3_finalize(pStmt); free(zName); @@ -5671,7 +7042,7 @@ dbtotxt_error: ** Print the given string as an error message. */ static void shellEmitError(const char *zErr){ - cli_printf(stderr,"Error: %s\n", zErr); + sqlite3_fprintf(stderr,"Error: %s\n", zErr); } /* ** Print the current sqlite3_errmsg() value to stderr and return 1. @@ -5854,42 +7225,39 @@ static void clearTempFile(ShellState *p){ p->zTempFile = 0; } -/* Forward reference */ -static char *find_home_dir(int clearFlag); - /* ** Create a new temp file name with the given suffix. -** -** Because the classic temp folders like /tmp are no longer -** accessible to web browsers, for security reasons, create the -** temp file in the user's home directory. */ static void newTempFile(ShellState *p, const char *zSuffix){ - char *zHome; /* Home directory */ - int i; /* Loop counter */ - sqlite3_uint64 r = 0; /* Integer with 64 bits of randomness */ - char zRand[32]; /* Text string with 160 bits of randomness */ + clearTempFile(p); + sqlite3_free(p->zTempFile); + p->zTempFile = 0; + if( p->db ){ + sqlite3_file_control(p->db, 0, SQLITE_FCNTL_TEMPFILENAME, &p->zTempFile); + } + if( p->zTempFile==0 ){ + /* If p->db is an in-memory database then the TEMPFILENAME file-control + ** will not work and we will need to fallback to guessing */ + char *zTemp; + sqlite3_uint64 r; + sqlite3_randomness(sizeof(r), &r); + zTemp = getenv("TEMP"); + if( zTemp==0 ) zTemp = getenv("TMP"); + if( zTemp==0 ){ #ifdef _WIN32 - const char cDirSep = '\\'; + zTemp = "\\tmp"; #else - const char cDirSep = '/'; + zTemp = "/tmp"; #endif - - for(i=0; i<31; i++){ - if( (i%12)==0 ) sqlite3_randomness(sizeof(r),&r); - zRand[i] = "0123456789abcdefghijklmnopqrstuvwxyz"[r%36]; - r /= 36; + } + p->zTempFile = sqlite3_mprintf("%s/temp%llx.%s", zTemp, r, zSuffix); + }else{ + p->zTempFile = sqlite3_mprintf("%z.%s", p->zTempFile, zSuffix); } - zRand[i] = 0; - clearTempFile(p); - sqlite3_free(p->zTempFile); - p->zTempFile = 0; - zHome = find_home_dir(0); - p->zTempFile = sqlite3_mprintf("%s%ctemp-%s.%s", - zHome,cDirSep,zRand,zSuffix); shell_check_oom(p->zTempFile); } + /* ** The implementation of SQL scalar function fkey_collate_clause(), used ** by the ".lint fkey-indexes" command. This scalar function is always @@ -6034,7 +7402,7 @@ static int lintFkeyIndexes( zIndent = " "; } else{ - cli_printf(stderr, + sqlite3_fprintf(stderr, "Usage: %s %s ?-verbose? ?-groupbyparent?\n", azArg[0], azArg[1]); return SQLITE_ERROR; } @@ -6079,22 +7447,22 @@ static int lintFkeyIndexes( if( rc!=SQLITE_OK ) break; if( res<0 ){ - cli_puts("Error: internal error", stderr); + sqlite3_fputs("Error: internal error", stderr); break; }else{ if( bGroupByParent && (bVerbose || res==0) && (zPrev==0 || sqlite3_stricmp(zParent, zPrev)) ){ - cli_printf(out, "-- Parent table %s\n", zParent); + sqlite3_fprintf(out, "-- Parent table %s\n", zParent); sqlite3_free(zPrev); zPrev = sqlite3_mprintf("%s", zParent); } if( res==0 ){ - cli_printf(out, "%s%s --> %s\n", zIndent, zCI, zTarget); + sqlite3_fprintf(out, "%s%s --> %s\n", zIndent, zCI, zTarget); }else if( bVerbose ){ - cli_printf(out, + sqlite3_fprintf(out, "%s/* no extra indexes required for %s -> %s */\n", zIndent, zFrom, zTarget ); @@ -6104,16 +7472,16 @@ static int lintFkeyIndexes( sqlite3_free(zPrev); if( rc!=SQLITE_OK ){ - cli_printf(stderr,"%s\n", sqlite3_errmsg(db)); + sqlite3_fprintf(stderr,"%s\n", sqlite3_errmsg(db)); } rc2 = sqlite3_finalize(pSql); if( rc==SQLITE_OK && rc2!=SQLITE_OK ){ rc = rc2; - cli_printf(stderr,"%s\n", sqlite3_errmsg(db)); + sqlite3_fprintf(stderr,"%s\n", sqlite3_errmsg(db)); } }else{ - cli_printf(stderr,"%s\n", sqlite3_errmsg(db)); + sqlite3_fprintf(stderr,"%s\n", sqlite3_errmsg(db)); } return rc; @@ -6133,9 +7501,9 @@ static int lintDotCommand( return lintFkeyIndexes(pState, azArg, nArg); usage: - cli_printf(stderr,"Usage %s sub-command ?switches...?\n", azArg[0]); - cli_printf(stderr, "Where sub-commands are:\n"); - cli_printf(stderr, " fkey-indexes\n"); + sqlite3_fprintf(stderr,"Usage %s sub-command ?switches...?\n", azArg[0]); + sqlite3_fprintf(stderr, "Where sub-commands are:\n"); + sqlite3_fprintf(stderr, " fkey-indexes\n"); return SQLITE_ERROR; } @@ -6149,7 +7517,7 @@ static void shellPrepare( if( *pRc==SQLITE_OK ){ int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); if( rc!=SQLITE_OK ){ - cli_printf(stderr, + sqlite3_fprintf(stderr, "sql error: %s (%d)\n", sqlite3_errmsg(db), sqlite3_errcode(db)); *pRc = rc; } @@ -6194,7 +7562,7 @@ static void shellFinalize( int rc = sqlite3_finalize(pStmt); if( *pRc==SQLITE_OK ){ if( rc!=SQLITE_OK ){ - cli_printf(stderr,"SQL error: %s\n", sqlite3_errmsg(db)); + sqlite3_fprintf(stderr,"SQL error: %s\n", sqlite3_errmsg(db)); } *pRc = rc; } @@ -6216,7 +7584,7 @@ void shellReset( if( *pRc==SQLITE_OK ){ if( rc!=SQLITE_OK ){ sqlite3 *db = sqlite3_db_handle(pStmt); - cli_printf(stderr,"SQL error: %s\n", sqlite3_errmsg(db)); + sqlite3_fprintf(stderr,"SQL error: %s\n", sqlite3_errmsg(db)); } *pRc = rc; } @@ -6269,9 +7637,9 @@ static int arErrorMsg(ArCommand *pAr, const char *zFmt, ...){ va_end(ap); shellEmitError(z); if( pAr->fromCmdLine ){ - cli_puts("Use \"-A\" for more help\n", stderr); + sqlite3_fputs("Use \"-A\" for more help\n", stderr); }else{ - cli_puts("Use \".archive --help\" for more help\n", stderr); + sqlite3_fputs("Use \".archive --help\" for more help\n", stderr); } sqlite3_free(z); return SQLITE_ERROR; @@ -6371,7 +7739,7 @@ static int arParseCommand( struct ArSwitch *pEnd = &aSwitch[nSwitch]; if( nArg<=1 ){ - cli_printf(stderr, "Wrong number of arguments. Usage:\n"); + sqlite3_fprintf(stderr, "Wrong number of arguments. Usage:\n"); return arUsage(stderr); }else{ char *z = azArg[1]; @@ -6477,7 +7845,7 @@ static int arParseCommand( } } if( pAr->eCmd==0 ){ - cli_printf(stderr, "Required argument missing. Usage:\n"); + sqlite3_fprintf(stderr, "Required argument missing. Usage:\n"); return arUsage(stderr); } return SQLITE_OK; @@ -6520,7 +7888,7 @@ static int arCheckEntries(ArCommand *pAr){ } shellReset(&rc, pTest); if( rc==SQLITE_OK && bOk==0 ){ - cli_printf(stderr,"not found in archive: %s\n", z); + sqlite3_fprintf(stderr,"not found in archive: %s\n", z); rc = SQLITE_ERROR; } } @@ -6603,15 +7971,15 @@ static int arListCommand(ArCommand *pAr){ shellPreparePrintf(pAr->db, &rc, &pSql, zSql, azCols[pAr->bVerbose], pAr->zSrcTable, zWhere); if( pAr->bDryRun ){ - cli_printf(pAr->out, "%s\n", sqlite3_sql(pSql)); + sqlite3_fprintf(pAr->out, "%s\n", sqlite3_sql(pSql)); }else{ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ if( pAr->bVerbose ){ - cli_printf(pAr->out, "%s % 10d %s %s\n", + sqlite3_fprintf(pAr->out, "%s % 10d %s %s\n", sqlite3_column_text(pSql, 0), sqlite3_column_int(pSql, 1), sqlite3_column_text(pSql, 2),sqlite3_column_text(pSql, 3)); }else{ - cli_printf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0)); + sqlite3_fprintf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0)); } } } @@ -6638,7 +8006,7 @@ static int arRemoveCommand(ArCommand *pAr){ zSql = sqlite3_mprintf("DELETE FROM %s WHERE %s;", pAr->zSrcTable, zWhere); if( pAr->bDryRun ){ - cli_printf(pAr->out, "%s\n", zSql); + sqlite3_fprintf(pAr->out, "%s\n", zSql); }else{ char *zErr = 0; rc = sqlite3_exec(pAr->db, "SAVEPOINT ar;", 0, 0, 0); @@ -6651,7 +8019,7 @@ static int arRemoveCommand(ArCommand *pAr){ } } if( zErr ){ - cli_printf(stdout, "ERROR: %s\n", zErr); /* stdout? */ + sqlite3_fprintf(stdout, "ERROR: %s\n", zErr); /* stdout? */ sqlite3_free(zErr); } } @@ -6666,15 +8034,11 @@ static int arRemoveCommand(ArCommand *pAr){ */ static int arExtractCommand(ArCommand *pAr){ const char *zSql1 = - "WITH dest(dpath,dlen) AS (SELECT realpath($dir),length(realpath($dir)))\n" - "SELECT ($dir || name),\n" - " CASE WHEN $dryrun THEN 0\n" - " ELSE writefile($dir||name, %s, mode, mtime) END\n" - " FROM dest CROSS JOIN %s\n" - " WHERE (%s)\n" - " AND (data IS NULL OR $pass==0)\n" /* Dirs both passes */ - " AND dpath=substr(realpath($dir||name),1,dlen)\n" /* No escapes */ - " AND name NOT GLOB '*..[/\\]*'\n"; /* No /../ in paths */ + "SELECT " + " ($dir || name)," + " writefile(($dir || name), %s, mode, mtime) " + "FROM %s WHERE (%s) AND (data IS NULL OR $dirOnly = 0)" + " AND name NOT GLOB '*..[/\\]*'"; const char *azExtraArg[] = { "sqlar_uncompress(data, sz)", @@ -6709,28 +8073,24 @@ static int arExtractCommand(ArCommand *pAr){ if( rc==SQLITE_OK ){ j = sqlite3_bind_parameter_index(pSql, "$dir"); sqlite3_bind_text(pSql, j, zDir, -1, SQLITE_STATIC); - j = sqlite3_bind_parameter_index(pSql, "$dryrun"); - sqlite3_bind_int(pSql, j, pAr->bDryRun); - - /* Run the SELECT statement twice - ** (0) writefile() all files and directories - ** (1) writefile() for directory again - ** The second pass is so that the timestamps for extracted directories - ** will be reset to the value in the archive, since populating them - ** in the first pass will have changed the timestamp. */ + + /* Run the SELECT statement twice. The first time, writefile() is called + ** for all archive members that should be extracted. The second time, + ** only for the directories. This is because the timestamps for + ** extracted directories must be reset after they are populated (as + ** populating them changes the timestamp). */ for(i=0; i<2; i++){ - j = sqlite3_bind_parameter_index(pSql, "$pass"); + j = sqlite3_bind_parameter_index(pSql, "$dirOnly"); sqlite3_bind_int(pSql, j, i); if( pAr->bDryRun ){ - cli_printf(pAr->out, "%s\n", sqlite3_sql(pSql)); - if( pAr->bVerbose==0 ) break; - } - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ - if( i==0 && pAr->bVerbose ){ - cli_printf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0)); + sqlite3_fprintf(pAr->out, "%s\n", sqlite3_sql(pSql)); + }else{ + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ + if( i==0 && pAr->bVerbose ){ + sqlite3_fprintf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0)); + } } } - if( pAr->bDryRun ) break; shellReset(&rc, pSql); } shellFinalize(&rc, pSql); @@ -6747,13 +8107,13 @@ static int arExtractCommand(ArCommand *pAr){ static int arExecSql(ArCommand *pAr, const char *zSql){ int rc; if( pAr->bDryRun ){ - cli_printf(pAr->out, "%s\n", zSql); + sqlite3_fprintf(pAr->out, "%s\n", zSql); rc = SQLITE_OK; }else{ char *zErr = 0; rc = sqlite3_exec(pAr->db, zSql, 0, 0, &zErr); if( zErr ){ - cli_printf(stdout, "ERROR: %s\n", zErr); + sqlite3_fprintf(stdout, "ERROR: %s\n", zErr); sqlite3_free(zErr); } } @@ -6929,13 +8289,13 @@ static int arDotCommand( } cmd.db = 0; if( cmd.bDryRun ){ - cli_printf(cmd.out, "-- open database '%s'%s\n", cmd.zFile, + sqlite3_fprintf(cmd.out, "-- open database '%s'%s\n", cmd.zFile, eDbType==SHELL_OPEN_APPENDVFS ? " using 'apndvfs'" : ""); } rc = sqlite3_open_v2(cmd.zFile, &cmd.db, flags, eDbType==SHELL_OPEN_APPENDVFS ? "apndvfs" : 0); if( rc!=SQLITE_OK ){ - cli_printf(stderr, "cannot open file: %s (%s)\n", + sqlite3_fprintf(stderr, "cannot open file: %s (%s)\n", cmd.zFile, sqlite3_errmsg(cmd.db)); goto end_ar_command; } @@ -6949,7 +8309,7 @@ static int arDotCommand( if( cmd.eCmd!=AR_CMD_CREATE && sqlite3_table_column_metadata(cmd.db,0,"sqlar","name",0,0,0,0,0) ){ - cli_printf(stderr, "database does not contain an 'sqlar' table\n"); + sqlite3_fprintf(stderr, "database does not contain an 'sqlar' table\n"); rc = SQLITE_ERROR; goto end_ar_command; } @@ -7007,7 +8367,7 @@ end_ar_command: */ static int recoverSqlCb(void *pCtx, const char *zSql){ ShellState *pState = (ShellState*)pCtx; - cli_printf(pState->out, "%s;\n", zSql); + sqlite3_fprintf(pState->out, "%s;\n", zSql); return SQLITE_OK; } @@ -7050,7 +8410,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ bRowids = 0; } else{ - cli_printf(stderr,"unexpected option: %s\n", azArg[i]); + sqlite3_fprintf(stderr,"unexpected option: %s\n", azArg[i]); showHelp(pState->out, azArg[0]); return 1; } @@ -7060,19 +8420,17 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ pState->db, "main", recoverSqlCb, (void*)pState ); - if( !pState->bSafeMode ){ - sqlite3_recover_config(p, 789, (void*)zRecoveryDb); /* Debug use only */ - } + sqlite3_recover_config(p, 789, (void*)zRecoveryDb); /* Debug use only */ sqlite3_recover_config(p, SQLITE_RECOVER_LOST_AND_FOUND, (void*)zLAF); sqlite3_recover_config(p, SQLITE_RECOVER_ROWIDS, (void*)&bRowids); sqlite3_recover_config(p, SQLITE_RECOVER_FREELIST_CORRUPT,(void*)&bFreelist); - cli_printf(pState->out, ".dbconfig defensive off\n"); + sqlite3_fprintf(pState->out, ".dbconfig defensive off\n"); sqlite3_recover_run(p); if( sqlite3_recover_errcode(p)!=SQLITE_OK ){ const char *zErr = sqlite3_recover_errmsg(p); int errCode = sqlite3_recover_errcode(p); - cli_printf(stderr,"sql error: %s (%d)\n", zErr, errCode); + sqlite3_fprintf(stderr,"sql error: %s (%d)\n", zErr, errCode); } rc = sqlite3_recover_finish(p); return rc; @@ -7094,7 +8452,7 @@ static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){ while( SQLITE_OK==sqlite3_intck_step(p) ){ const char *zMsg = sqlite3_intck_message(p); if( zMsg ){ - cli_printf(pState->out, "%s\n", zMsg); + sqlite3_fprintf(pState->out, "%s\n", zMsg); nError++; } nStep++; @@ -7104,11 +8462,11 @@ static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){ } rc = sqlite3_intck_error(p, &zErr); if( zErr ){ - cli_printf(stderr,"%s\n", zErr); + sqlite3_fprintf(stderr,"%s\n", zErr); } sqlite3_intck_close(p); - cli_printf(pState->out, "%lld steps, %lld errors\n", nStep, nError); + sqlite3_fprintf(pState->out, "%lld steps, %lld errors\n", nStep, nError); } return rc; @@ -7131,7 +8489,7 @@ static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){ #define rc_err_oom_die(rc) \ if( rc==SQLITE_NOMEM ) shell_check_oom(0); \ else if(!(rc==SQLITE_OK||rc==SQLITE_DONE)) \ - cli_printf(stderr,"E:%d\n",rc), assert(0) + sqlite3_fprintf(stderr,"E:%d\n",rc), assert(0) #else static void rc_err_oom_die(int rc){ if( rc==SQLITE_NOMEM ) shell_check_oom(0); @@ -7246,7 +8604,7 @@ SELECT CASE WHEN (nc < 10) THEN 1 WHEN (nc < 100) THEN 2 \ SELECT\ '('||x'0a'\ || group_concat(\ - cname||' ANY',\ + cname||' TEXT',\ ','||iif((cpos-1)%4>0, ' ', x'0a'||' '))\ ||')' AS ColsSpec \ FROM (\ @@ -7282,1678 +8640,120 @@ FROM (\ rc_err_oom_die(rc); rc = sqlite3_step(pStmt); rc_err_oom_die(rc); - sqlite3_finalize(pStmt); - return 0; - }else if( *pDb==0 ){ - return 0; - }else{ - /* Formulate the columns spec, close the DB, zero *pDb. */ - char *zColsSpec = 0; - int hasDupes = db_int(*pDb, "%s", zHasDupes); - int nDigits = (hasDupes)? db_int(*pDb, "%s", zColDigits) : 0; - if( hasDupes ){ -#ifdef SHELL_COLUMN_RENAME_CLEAN - rc = sqlite3_exec(*pDb, zDedoctor, 0, 0, 0); - rc_err_oom_die(rc); -#endif - rc = sqlite3_exec(*pDb, zSetReps, 0, 0, 0); - rc_err_oom_die(rc); - rc = sqlite3_prepare_v2(*pDb, zRenameRank, -1, &pStmt, 0); - rc_err_oom_die(rc); - sqlite3_bind_int(pStmt, 1, nDigits); - rc = sqlite3_step(pStmt); - sqlite3_finalize(pStmt); - if( rc!=SQLITE_DONE ) rc_err_oom_die(SQLITE_NOMEM); - } - assert(db_int(*pDb, "%s", zHasDupes)==0); /* Consider: remove this */ - rc = sqlite3_prepare_v2(*pDb, zCollectVar, -1, &pStmt, 0); - rc_err_oom_die(rc); - rc = sqlite3_step(pStmt); - if( rc==SQLITE_ROW ){ - zColsSpec = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); - }else{ - zColsSpec = 0; - } - if( pzRenamed!=0 ){ - if( !hasDupes ) *pzRenamed = 0; - else{ - sqlite3_finalize(pStmt); - if( SQLITE_OK==sqlite3_prepare_v2(*pDb, zRenamesDone, -1, &pStmt, 0) - && SQLITE_ROW==sqlite3_step(pStmt) ){ - *pzRenamed = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); - }else - *pzRenamed = 0; - } - } - sqlite3_finalize(pStmt); - sqlite3_close(*pDb); - *pDb = 0; - return zColsSpec; - } -} - -/* -** Check if the sqlite_schema table contains one or more virtual tables. If -** parameter zLike is not NULL, then it is an SQL expression that the -** sqlite_schema row must also match. If one or more such rows are found, -** print the following warning to the output: -** -** WARNING: Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled -*/ -static int outputDumpWarning(ShellState *p, const char *zLike){ - int rc = SQLITE_OK; - sqlite3_stmt *pStmt = 0; - shellPreparePrintf(p->db, &rc, &pStmt, - "SELECT 1 FROM sqlite_schema o WHERE " - "sql LIKE 'CREATE VIRTUAL TABLE%%' AND %s", zLike ? zLike : "true" - ); - if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ - cli_puts("/* WARNING: " - "Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled */\n", - p->out - ); - } - shellFinalize(&rc, pStmt); - return rc; -} - -/* -** Fault-Simulator state and logic. -*/ -static struct { - int iId; /* ID that triggers a simulated fault. -1 means "any" */ - int iErr; /* The error code to return on a fault */ - int iCnt; /* Trigger the fault only if iCnt is already zero */ - int iInterval; /* Reset iCnt to this value after each fault */ - int eVerbose; /* When to print output */ - int nHit; /* Number of hits seen so far */ - int nRepeat; /* Turn off after this many hits. 0 for never */ - int nSkip; /* Skip this many before first fault */ -} faultsim_state = {-1, 0, 0, 0, 0, 0, 0, 0}; - -/* -** This is the fault-sim callback -*/ -static int faultsim_callback(int iArg){ - if( faultsim_state.iId>0 && faultsim_state.iId!=iArg ){ - return SQLITE_OK; - } - if( faultsim_state.iCnt ){ - if( faultsim_state.iCnt>0 ) faultsim_state.iCnt--; - if( faultsim_state.eVerbose>=2 ){ - cli_printf(stdout, - "FAULT-SIM id=%d no-fault (cnt=%d)\n", iArg, faultsim_state.iCnt); - } - return SQLITE_OK; - } - if( faultsim_state.eVerbose>=1 ){ - cli_printf(stdout, - "FAULT-SIM id=%d returns %d\n", iArg, faultsim_state.iErr); - } - faultsim_state.iCnt = faultsim_state.iInterval; - faultsim_state.nHit++; - if( faultsim_state.nRepeat>0 && faultsim_state.nRepeat<=faultsim_state.nHit ){ - faultsim_state.iCnt = -1; - } - return faultsim_state.iErr; -} - -/* -** pickStr(zArg, &zErr, zS1, zS2, ..., ""); -** -** Try to match zArg against zS1, zS2, and so forth until the first -** emptry string. Return the index of the match or -1 if none is found. -** If no match is found, and &zErr is not NULL, then write into -** zErr a message describing the valid choices. -*/ -static int pickStr(const char *zArg, char **pzErr, ...){ - int i, n; - const char *z; - sqlite3_str *pMsg; - va_list ap; - va_start(ap, pzErr); - i = 0; - while( (z = va_arg(ap,const char*))!=0 && z[0]!=0 ){ - if( cli_strcmp(zArg, z)==0 ) return i; - i++; - } - va_end(ap); - if( pzErr==0 ) return -1; - n = i; - pMsg = sqlite3_str_new(0); - va_start(ap, pzErr); - sqlite3_str_appendall(pMsg, "should be"); - i = 0; - while( (z = va_arg(ap, const char*))!=0 && z[0]!=0 ){ - if( i==n-1 ){ - sqlite3_str_append(pMsg,", or",4); - }else if( i>0 ){ - sqlite3_str_append(pMsg, ",", 1); - } - sqlite3_str_appendf(pMsg, " %s", z); - i++; - } - va_end(ap); - *pzErr = sqlite3_str_finish(pMsg); - return -1; -} - -/* -** DOT-COMMAND: .import -** -** USAGE: .import [OPTIONS] FILE TABLE -** -** Import CSV or similar text from FILE into TABLE. If TABLE does -** not exist, it is created using the first row of FILE as the column -** names. If FILE begins with "|" then it is a command that is run -** and the output from the command is used as the input data. If -** FILE begins with "<<" followed by a label, then content is read from -** the script until the first line that matches the label. -** -** The content of FILE is interpreted using RFC-4180 ("CSV") quoting -** rules unless the current mode is "ascii" or "tabs" or unless one -** the --ascii option is used. -** -** The column and row separators must be single ASCII characters. If -** multiple characters or a Unicode character are specified for the -** separators, then only the first byte of the separator is used. Except, -** if the row separator is \n and the mode is not --ascii, then \r\n is -** understood as a row separator too. -** -** Options: -** --ascii Do not use RFC-4180 quoting. Use \037 and \036 -** as column and row separators on input, unless other -** delimiters are specified using --colsep and/or --rowsep -** --colsep CHAR Use CHAR as the column separator. -** --csv Input is standard RFC-4180 CSV. -** --esc CHAR Use CHAR as an escape character in unquoted CSV inputs. -** --qesc CHAR Use CHAR as an escape character in quoted CSV inputs. -** --rowsep CHAR Use CHAR as the row separator. -** --schema S When creating TABLE, put it in schema S -** --skip N Ignore the first N rows of input -** -v Verbose mode -*/ -static int dotCmdImport(ShellState *p){ - int nArg = p->dot.nArg; /* Number of arguments */ - char **azArg = p->dot.azArg;/* Argument list */ - char *zTable = 0; /* Insert data into this table */ - char *zSchema = 0; /* Schema of zTable */ - char *zFile = 0; /* Name of file to extra content from */ - sqlite3_stmt *pStmt = NULL; /* A statement */ - int nCol; /* Number of columns in the table */ - i64 nByte; /* Number of bytes in an SQL string */ - int i, j; /* Loop counters */ - int needCommit; /* True to COMMIT or ROLLBACK at end */ - char *zSql = 0; /* An SQL statement */ - ImportCtx sCtx; /* Reader context */ - char *(SQLITE_CDECL *xRead)(ImportCtx*); /* Func to read one value */ - int eVerbose = 0; /* Larger for more console output */ - i64 nSkip = 0; /* Initial lines to skip */ - i64 iLineOffset = 0; /* Offset to the first line of input */ - char *zCreate = 0; /* CREATE TABLE statement text */ - int rc; /* Result code */ - - failIfSafeMode(p, "cannot run .import in safe mode"); - memset(&sCtx, 0, sizeof(sCtx)); - if( p->mode.eMode==MODE_Ascii ){ - xRead = ascii_read_one_field; - }else{ - xRead = csv_read_one_field; - } - for(i=1; i<nArg; i++){ - char *z = azArg[i]; - if( z[0]=='-' && z[1]=='-' ) z++; - if( z[0]!='-' ){ - if( zFile==0 ){ - zFile = z; - }else if( zTable==0 ){ - zTable = z; - }else{ - dotCmdError(p, i, "unknown argument", 0); - return 1; - } - }else if( cli_strcmp(z,"-v")==0 ){ - eVerbose++; - }else if( cli_strcmp(z,"-schema")==0 && i<nArg-1 ){ - zSchema = azArg[++i]; - }else if( cli_strcmp(z,"-skip")==0 && i<nArg-1 ){ - nSkip = integerValue(azArg[++i]); - }else if( cli_strcmp(z,"-ascii")==0 ){ - if( sCtx.cColSep==0 ) sCtx.cColSep = SEP_Unit[0]; - if( sCtx.cRowSep==0 ) sCtx.cRowSep = SEP_Record[0]; - xRead = ascii_read_one_field; - }else if( cli_strcmp(z,"-csv")==0 ){ - if( sCtx.cColSep==0 ) sCtx.cColSep = ','; - if( sCtx.cRowSep==0 ) sCtx.cRowSep = '\n'; - xRead = csv_read_one_field; - }else if( cli_strcmp(z,"-esc")==0 ){ - sCtx.cUQEscape = azArg[++i][0]; - }else if( cli_strcmp(z,"-qesc")==0 ){ - sCtx.cQEscape = azArg[++i][0]; - }else if( cli_strcmp(z,"-colsep")==0 ){ - if( i==nArg-1 ){ - dotCmdError(p, i, "missing argument", 0); - return 1; - } - i++; - sCtx.cColSep = azArg[i][0]; - }else if( cli_strcmp(z,"-rowsep")==0 ){ - if( i==nArg-1 ){ - dotCmdError(p, i, "missing argument", 0); - return 1; - } - i++; - sCtx.cRowSep = azArg[i][0]; - }else{ - dotCmdError(p, i, "unknown option", 0); - return 1; - } - } - if( zTable==0 ){ - dotCmdError(p, nArg, 0, "Missing %s argument\n", - zFile==0 ? "FILE" : "TABLE"); - return 1; - } - seenInterrupt = 0; - open_db(p, 0); - if( sCtx.cColSep==0 ){ - if( p->mode.spec.zColumnSep && p->mode.spec.zColumnSep[0]!=0 ){ - sCtx.cColSep = p->mode.spec.zColumnSep[0]; - }else{ - sCtx.cColSep = ','; - } - } - if( (sCtx.cColSep & 0x80)!=0 ){ - eputz("Error: .import column separator must be ASCII\n"); - return 1; - } - if( sCtx.cRowSep==0 ){ - if( p->mode.spec.zRowSep && p->mode.spec.zRowSep[0]!=0 ){ - sCtx.cRowSep = p->mode.spec.zRowSep[0]; - }else{ - sCtx.cRowSep = '\n'; - } - } - if( sCtx.cRowSep=='\r' && xRead!=ascii_read_one_field ){ - sCtx.cRowSep = '\n'; - } - if( (sCtx.cRowSep & 0x80)!=0 ){ - eputz("Error: .import row separator must be ASCII\n"); - return 1; - } - sCtx.zFile = zFile; - sCtx.nLine = 1; - if( sCtx.zFile[0]=='|' ){ -#ifdef SQLITE_OMIT_POPEN - eputz("Error: pipes are not supported in this OS\n"); - return 1; -#else - sCtx.in = sqlite3_popen(sCtx.zFile+1, "r"); - sCtx.zFile = "<pipe>"; - sCtx.xCloser = pclose; -#endif - }else if( sCtx.zFile[0]=='<' && sCtx.zFile[1]=='<' && sCtx.zFile[2]!=0 ){ - /* Input text comes from subsequent lines of script until the zFile - ** delimiter */ - int nEndMark = strlen30(zFile)-2; - char *zEndMark = &zFile[2]; - sqlite3_str *pContent = sqlite3_str_new(p->db); - int ckEnd = 1; - i64 iStart = p->lineno; - char zLine[2000]; - sCtx.zFile = p->zInFile; - sCtx.nLine = p->lineno+1; - iLineOffset = p->lineno; - while( sqlite3_fgets(zLine,sizeof(zLine),p->in) ){ - if( ckEnd && cli_strncmp(zLine,zEndMark,nEndMark)==0 ){ - ckEnd = 2; - if( strchr(zLine,'\n') ) p->lineno++; - break; - } - if( strchr(zLine,'\n') ){ - p->lineno++; - ckEnd = 1; - }else{ - ckEnd = 0; - } - sqlite3_str_appendall(pContent, zLine); - } - sCtx.zIn = sqlite3_str_finish(pContent); - if( sCtx.zIn==0 ){ - sCtx.zIn = sqlite3_mprintf(""); - } - if( ckEnd<2 ){ - i64 savedLn = p->lineno; - p->lineno = iStart; - dotCmdError(p, 0, 0,"Content terminator \"%s\" not found.",zEndMark); - p->lineno = savedLn; - import_cleanup(&sCtx); - return 1; - } - }else{ - sCtx.in = sqlite3_fopen(sCtx.zFile, "rb"); - sCtx.xCloser = fclose; - } - if( sCtx.in==0 && sCtx.zIn==0 ){ - dotCmdError(p, 0, 0, "cannot open \"%s\"", zFile); - import_cleanup(&sCtx); - return 1; - } - if( eVerbose>=1 ){ - char zSep[2]; - zSep[1] = 0; - zSep[0] = sCtx.cColSep; - cli_puts("Column separator ", p->out); - output_c_string(p->out, zSep); - cli_puts(", row separator ", p->out); - zSep[0] = sCtx.cRowSep; - output_c_string(p->out, zSep); - cli_puts("\n", p->out); - } - sCtx.z = sqlite3_malloc64(120); - if( sCtx.z==0 ){ - import_cleanup(&sCtx); - shell_out_of_memory(); - } - /* Below, resources must be freed before exit. */ - while( nSkip>0 ){ - nSkip--; - while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){} - } - import_append_char(&sCtx, 0); /* To ensure sCtx.z is allocated */ - if( sqlite3_table_column_metadata(p->db, zSchema, zTable,0,0,0,0,0,0) - && 0==db_int(p->db, "SELECT count(*) FROM \"%w\".sqlite_schema" - " WHERE name=%Q AND type='view'", - zSchema ? zSchema : "main", zTable) - ){ - /* Table does not exist. Create it. */ - sqlite3 *dbCols = 0; - char *zRenames = 0; - char *zColDefs; - zCreate = sqlite3_mprintf("CREATE TABLE \"%w\".\"%w\"", - zSchema ? zSchema : "main", zTable); - while( xRead(&sCtx) ){ - zAutoColumn(sCtx.z, &dbCols, 0); - if( sCtx.cTerm!=sCtx.cColSep ) break; - } - zColDefs = zAutoColumn(0, &dbCols, &zRenames); - if( zRenames!=0 ){ - cli_printf((stdin_is_interactive && p->in==stdin)? p->out : stderr, - "Columns renamed during .import %s due to duplicates:\n" - "%s\n", sCtx.zFile, zRenames); - sqlite3_free(zRenames); - } - assert(dbCols==0); - if( zColDefs==0 ){ - cli_printf(stderr,"%s: empty file\n", sCtx.zFile); - import_cleanup(&sCtx); - sqlite3_free(zCreate); - return 1; - } - zCreate = sqlite3_mprintf("%z%z\n", zCreate, zColDefs); - if( zCreate==0 ){ - import_cleanup(&sCtx); - shell_out_of_memory(); - } - if( eVerbose>=1 ){ - cli_printf(p->out, "%s\n", zCreate); - } - rc = sqlite3_exec(p->db, zCreate, 0, 0, 0); - if( rc ){ - cli_printf(stderr, - "%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db)); - } - sqlite3_free(zCreate); - zCreate = 0; - if( rc ){ - import_cleanup(&sCtx); - return 1; - } - } - zSql = sqlite3_mprintf("SELECT count(*) FROM pragma_table_info(%Q,%Q);", - zTable, zSchema); - if( zSql==0 ){ - import_cleanup(&sCtx); - shell_out_of_memory(); - } - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - zSql = 0; - if( rc ){ - if (pStmt) sqlite3_finalize(pStmt); - shellDatabaseError(p->db); - import_cleanup(&sCtx); - return 1; - } - if( sqlite3_step(pStmt)==SQLITE_ROW ){ - nCol = sqlite3_column_int(pStmt, 0); - }else{ - nCol = 0; - } - sqlite3_finalize(pStmt); - pStmt = 0; - if( nCol==0 ) return 0; /* no columns, no error */ - - nByte = 64 /* space for "INSERT INTO", "VALUES(", ")\0" */ - + (zSchema ? strlen(zSchema)*2 + 2: 0) /* Quoted schema name */ - + strlen(zTable)*2 + 2 /* Quoted table name */ - + nCol*2; /* Space for ",?" for each column */ - zSql = sqlite3_malloc64( nByte ); - if( zSql==0 ){ - import_cleanup(&sCtx); - shell_out_of_memory(); - } - if( zSchema ){ - sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\".\"%w\" VALUES(?", - zSchema, zTable); - }else{ - sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); - } - j = strlen30(zSql); - for(i=1; i<nCol; i++){ - zSql[j++] = ','; - zSql[j++] = '?'; - } - zSql[j++] = ')'; - zSql[j] = 0; - assert( j<nByte ); - if( eVerbose>=2 ){ - cli_printf(p->out, "Insert using: %s\n", zSql); - } - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - zSql = 0; - if( rc ){ - shellDatabaseError(p->db); - if (pStmt) sqlite3_finalize(pStmt); - import_cleanup(&sCtx); - return 1; - } - needCommit = sqlite3_get_autocommit(p->db); - if( needCommit ) sqlite3_exec(p->db, "BEGIN", 0, 0, 0); - do{ - int startLine = sCtx.nLine; - for(i=0; i<nCol; i++){ - char *z = xRead(&sCtx); - /* - ** Did we reach end-of-file before finding any columns? - ** If so, stop instead of NULL filling the remaining columns. - */ - if( z==0 && i==0 ) break; - /* - ** Did we reach end-of-file OR end-of-line before finding any - ** columns in ASCII mode? If so, stop instead of NULL filling - ** the remaining columns. - */ - if( p->mode.eMode==MODE_Ascii && (z==0 || z[0]==0) && i==0 ) break; - /* - ** For CSV mode, per RFC 4180, accept EOF in lieu of final - ** record terminator but only for last field of multi-field row. - ** (If there are too few fields, it's not valid CSV anyway.) - */ - if( z==0 && (xRead==csv_read_one_field) && i==nCol-1 && i>0 ){ - z = ""; - } - sqlite3_bind_text(pStmt, i+1, z, -1, SQLITE_TRANSIENT); - if( i<nCol-1 && sCtx.cTerm!=sCtx.cColSep ){ - if( i==0 && (strcmp(z,"\n")==0 || strcmp(z,"\r\n")==0) ){ - /* Ignore trailing \n or \r\n when some other row separator */ - break; - } - cli_printf(stderr,"%s:%d: expected %d columns but found %d" - " - filling the rest with NULL\n", - sCtx.zFile, startLine, nCol, i+1); - i += 2; - while( i<=nCol ){ sqlite3_bind_null(pStmt, i); i++; } - } - } - if( sCtx.cTerm==sCtx.cColSep ){ - do{ - xRead(&sCtx); - i++; - }while( sCtx.cTerm==sCtx.cColSep ); - cli_printf(stderr, - "%s:%d: expected %d columns but found %d - extras ignored\n", - sCtx.zFile, startLine, nCol, i); - } - if( i>=nCol ){ - sqlite3_step(pStmt); - rc = sqlite3_reset(pStmt); - if( rc!=SQLITE_OK ){ - cli_printf(stderr,"%s:%d: INSERT failed: %s\n", - sCtx.zFile, startLine, sqlite3_errmsg(p->db)); - sCtx.nErr++; - if( bail_on_error ) break; - }else{ - sCtx.nRow++; - } - } - }while( sCtx.cTerm!=EOF ); - - import_cleanup(&sCtx); - sqlite3_finalize(pStmt); - if( needCommit ) sqlite3_exec(p->db, "COMMIT", 0, 0, 0); - if( eVerbose>0 ){ - cli_printf(p->out, - "Added %d rows with %d errors using %d lines of input\n", - sCtx.nRow, sCtx.nErr, sCtx.nLine-1-iLineOffset); - } - return sCtx.nErr ? 1 : 0; -} - - -/* -** This function computes what to show the user about the configured -** titles (or column-names). Output is an integer between 0 and 3: -** -** 0: The titles do not matter. Never show anything. -** 1: Show "--titles off" -** 2: Show "--titles on" -** 3: Show "--title VALUE" where VALUE is an encoding method -** to use, one of: plain sql csv html tcl json -** -** Inputs are: -** -** spec.bTitles (bT) Whether or not to show the titles -** spec.eTitle (eT) The actual encoding to be used for titles -** ModeInfo.bHdr (bH) Default value for spec.bTitles -** ModeInfo.eHdr (eH) Default value for spec.eTitle -** bAll Whether the -v option is used -*/ -static int modeTitleDsply(ShellState *p, int bAll){ - int eMode = p->mode.eMode; - const ModeInfo *pI = &aModeInfo[eMode]; - int bT = p->mode.spec.bTitles; - int eT = p->mode.spec.eTitle; - int bH = pI->bHdr; - int eH = pI->eHdr; - - /* Variable "v" is the truth table that will determine the answer - ** - ** Actual encoding is different from default - ** vvvvvvvv */ - sqlite3_uint64 v = 0x0133013311220102; - /* ^^^^ ^^^^ - ** Upper 2-byte groups for when ON/OFF disagrees with - ** the default. */ - - if( bH==0 ) return 0; /* Header not appliable. Ex: off, count */ - - if( eT==0 ) eT = eH; /* Fill in missing spec.eTitle */ - if( bT==0 ) bT = bH; /* Fill in missing spec.bTitles */ - - if( eT!=eH ) v >>= 32; /* Encoding disagree in upper 4-bytes */ - if( bT!=bH ) v >>= 16; /* ON/OFF disagree in upper 2-byte pairs */ - if( bT<2 ) v >>= 8; /* ON in even bytes, OFF in odd bytes (1st byte 0) */ - if( !bAll ) v >>= 4; /* bAll values are in the lower half-byte */ - - return v & 3; /* Return the selected truth-table entry */ -} - -/* -** DOT-COMMAND: .mode -** -** USAGE: .mode [MODE] [OPTIONS] -** -** Change the output mode to MODE and/or apply OPTIONS to the output mode. -** Arguments are processed from left to right. If no arguments, show the -** current output mode and relevant options. -** -** Options: -** --align STRING Set the alignment of text in columnar modes -** String consists of characters 'L', 'C', 'R' -** meaning "left", "centered", and "right", with -** one letter per column starting from the left. -** Unspecified alignment defaults to 'L'. -** --blob-quote ARG ARG can be "auto", "text", "sql", "hex", "tcl", -** "json", or "size". Default is "auto". -** --border on|off Show outer border on "box" and "table" modes. -** --charlimit N Set the maximum number of output characters to -** show for any single SQL value to N. Longer values -** truncated. Zero means "no limit". -** --colsep STRING Use STRING as the column separator -** --escape ESC Enable/disable escaping of control characters -** found in the output. ESC can be "off", "ascii", -** or "symbol". -** --linelimit N Set the maximum number of output lines to show for -** any single SQL value to N. Longer values are -** truncated. Zero means "no limit". Only works -** in "line" mode and in columnar modes. -** --limits L,C,T Shorthand for "--linelimit L --charlimit C -** --titlelimit T". The ",T" can be omitted in which -** case the --titlelimit is unchanged. The argument -** can also be "off" to mean "0,0,0" or "on" to -** mean "5,300,20". -** --list List available modes -** --null STRING Render SQL NULL values as the given string -** --once Setting changes to the right are reverted after -** the next SQL command. -** --quote ARG Enable/disable quoting of text. ARG can be -** "off", "on", "sql", "relaxed", "csv", "html", -** "tcl", or "json". "off" means show the text as-is. -** "on" is an alias for "sql". -** --reset Changes all mode settings back to their default. -** --rowsep STRING Use STRING as the row separator -** --sw|--screenwidth N Declare the screen width of the output device -** to be N characters. An attempt may be made to -** wrap output text to fit within this limit. Zero -** means "no limit". Or N can be "auto" to set the -** width automatically. -** --tablename NAME Set the name of the table for "insert" mode. -** --tag NAME Save mode to the left as NAME. -** --textjsonb BOOLEAN If enabled, JSONB text is displayed as text JSON. -** --title ARG Whether or not to show column headers, and if so -** how to encode them. ARG can be "off", "on", -** "sql", "csv", "html", "tcl", or "json". -** --titlelimit N Limit the length of column titles to N characters. -** -v|--verbose Verbose output -** --widths LIST Set the columns widths for columnar modes. The -** argument is a list of integers, one for each -** column. A "0" width means use a dynamic width -** based on the actual width of data. If there are -** fewer entries in LIST than columns, "0" is used -** for the unspecified widths. -** --wordwrap BOOLEAN Enable/disable word wrapping -** --wrap N Wrap columns wider than N characters -** --ww Shorthand for "--wordwrap on" -*/ -static int dotCmdMode(ShellState *p){ - int nArg = p->dot.nArg; /* Number of arguments */ - char **azArg = p->dot.azArg;/* Argument list */ - int eMode = -1; /* New mode value, or -1 for none */ - int iMode = -1; /* Index of the argument that is the mode name */ - int i; /* Loop counter */ - int k; /* Misc index variable */ - int chng = 0; /* True if anything has changed */ - int bAll = 0; /* Show all details of the mode */ - - for(i=1; i<nArg; i++){ - const char *z = azArg[i]; - if( z[0]=='-' && z[1]=='-' ) z++; - if( z[0]!='-' - && iMode<0 - && (eMode = modeFind(p, azArg[i]))>=0 - && eMode!=MODE_Www - ){ - iMode = i; - modeChange(p, eMode); - /* (Legacy) If the mode is MODE_Insert and the next argument - ** is not an option, then the next argument must be the table - ** name. - */ - if( i+1<nArg && azArg[i+1][0]!='-' ){ - i++; - modeSetStr(&p->mode.spec.zTableName, azArg[i]); - } - chng = 1; - }else if( optionMatch(z,"align") ){ - char *zAlign; - int nAlign; - int nErr = 0; - if( i+1>=nArg ){ - dotCmdError(p, i, "missing argument", 0); - return 1; - } - i++; - zAlign = azArg[i]; - nAlign = 0x3fff & strlen(zAlign); - free(p->mode.spec.aAlign); - p->mode.spec.aAlign = malloc(nAlign); - shell_check_oom(p->mode.spec.aAlign); - for(k=0; k<nAlign; k++){ - unsigned char c = 0; - switch( zAlign[k] ){ - case 'l': case 'L': c = QRF_ALIGN_Left; break; - case 'c': case 'C': c = QRF_ALIGN_Center; break; - case 'r': case 'R': c = QRF_ALIGN_Right; break; - default: nErr++; break; - } - p->mode.spec.aAlign[k] = c; - } - p->mode.spec.nAlign = nAlign; - chng = 1; - if( nErr ){ - dotCmdError(p, i, "bad alignment string", - "Should contain only characters L, C, and R."); - return 1; - } - }else if( pickStr(z,0,"-blob","-blob-quote","")>=0 ){ - if( (++i)>=nArg ){ - dotCmdError(p, i-1, "missing argument", 0); - return 1; - } - k = pickStr(azArg[i], 0, - "auto", "text", "sql", "hex", "tcl", "json", "size", ""); - /* 0 1 2 3 4 5 6 - ** Must match QRF_BLOB_xxxx values. See also tag-20251124a */ - if( k>=0 ){ - p->mode.spec.eBlob = k & 0xff; - } - chng = 1; - }else if( optionMatch(z,"border") ){ - if( (++i)>=nArg ){ - dotCmdError(p, i-1, "missing argument", 0); - return 1; - } - k = pickStr(azArg[i], 0, "auto", "off", "on", ""); - if( k>=0 ){ - p->mode.spec.bBorder = k & 0x3; - } - chng = 1; - }else if( 0<=(k=pickStr(z,0,"-charlimit","-linelimit","-titlelimit","")) ){ - int w; /* 0 1 */ - if( i+1>=nArg ){ - dotCmdError(p, i, "missing argument", 0); - return 1; - } - w = integerValue(azArg[++i]); - switch( k ){ - case 0: p->mode.spec.nCharLimit = w; break; - case 1: p->mode.spec.nLineLimit = w; break; - default: p->mode.spec.nTitleLimit = w; break; - } - chng = 1; - }else if( 0<=(k=pickStr(z,0,"-tablename","-rowsep","-colsep","-null","")) ){ - /* 0 1 2 3 */ - if( i+1>=nArg ){ - dotCmdError(p, i, "missing argument", 0); - return 1; - } - i++; - switch( k ){ - case 0: modeSetStr(&p->mode.spec.zTableName, azArg[i]); break; - case 1: modeSetStr(&p->mode.spec.zRowSep, azArg[i]); break; - case 2: modeSetStr(&p->mode.spec.zColumnSep, azArg[i]); break; - case 3: modeSetStr(&p->mode.spec.zNull, azArg[i]); break; - } - chng = 1; - }else if( optionMatch(z,"escape") ){ - /* See similar code at tag-20250224-1 */ - char *zErr = 0; - if( (++i)>=nArg ){ - dotCmdError(p, i-1, "missing argument", 0); - return 1; - } /* 0 1 2 <-- One less than QRF_ESC_ */ - k = pickStr(azArg[i],&zErr,"off","ascii","symbol",""); - if( k<0 ){ - dotCmdError(p, i, "unknown escape type", "%s", zErr); - sqlite3_free(zErr); - return 1; - } - p->mode.spec.eEsc = k+1; - chng = 1; - }else if( optionMatch(z,"limits") ){ - if( (++i)>=nArg ){ - dotCmdError(p, i-1, "missing argument", 0); - return 1; - } - k = pickStr(azArg[i],0,"on","off",""); - if( k==0 ){ - p->mode.spec.nLineLimit = DFLT_LINE_LIMIT; - p->mode.spec.nCharLimit = DFLT_CHAR_LIMIT; - p->mode.spec.nTitleLimit = DFLT_TITLE_LIMIT; - }else if( k==1 ){ - p->mode.spec.nLineLimit = 0; - p->mode.spec.nCharLimit = 0; - p->mode.spec.nTitleLimit = 0; - }else{ - int L, C, T = 0; - int nNum = sscanf(azArg[i], "%d,%d,%d", &L, &C, &T); - if( nNum<2 || L<0 || C<0 || T<0){ - dotCmdError(p, i, "bad argument", "Should be \"L,C,T\" where L, C" - " and T are unsigned integers"); - return 1; - } - p->mode.spec.nLineLimit = L; - p->mode.spec.nCharLimit = C; - if( nNum==3 ) p->mode.spec.nTitleLimit = T; - } - chng = 1; - }else if( optionMatch(z,"list") ){ - int ii; - cli_puts("available modes:", p->out); - for(ii=0; ii<ArraySize(aModeInfo); ii++){ - if( ii==MODE_Www ) continue; - cli_printf(p->out, " %s", aModeInfo[ii].zName); - } - for(ii=0; ii<p->nSavedModes; ii++){ - cli_printf(p->out, " %s", p->aSavedModes[ii].zTag); - } - cli_puts(" batch tty\n", p->out); - chng = 1; /* Not really a change, but we still want to suppress the - ** "current mode" output */ - }else if( optionMatch(z,"once") ){ - p->nPopMode = 0; - modePush(p); - p->nPopMode = 1; - }else if( optionMatch(z,"noquote") ){ - /* (undocumented legacy) --noquote always turns quoting off */ - p->mode.spec.eText = QRF_TEXT_Plain; - p->mode.spec.eBlob = QRF_BLOB_Auto; - chng = 1; - }else if( optionMatch(z,"quote") ){ - if( i+1<nArg - && azArg[i+1][0]!='-' - && (iMode>0 || strcmp(azArg[i+1],"off")==0 || modeFind(p, azArg[i+1])<0) - ){ - /* --quote is followed by an argument other that is not an option - ** or a mode name. See it must be a boolean or a keyword to describe - ** how to set quoting. */ - i++; - if( (k = pickStr(azArg[i],0,"no","yes","0","1",""))>=0 ){ - k &= 1; /* 0 for "off". 1 for "on". */ - }else{ - char *zErr = 0; - k = pickStr(azArg[i],&zErr, - "off","on","sql","csv","html","tcl","json","relaxed",""); - /* 0 1 2 3 4 5 6 7 */ - if( k<0 ){ - dotCmdError(p, i, "unknown", "%z", zErr); - return 1; - } - } - }else{ - /* (Legacy) no following boolean argument. Turn quoting on */ - k = 1; - } - switch( k ){ - case 1: /* on */ - modeSetStr(&p->mode.spec.zNull, "NULL"); - /* Fall through */ - case 2: /* sql */ - p->mode.spec.eText = QRF_TEXT_Sql; - break; - case 3: /* csv */ - p->mode.spec.eText = QRF_TEXT_Csv; - break; - case 4: /* html */ - p->mode.spec.eText = QRF_TEXT_Html; - break; - case 5: /* tcl */ - p->mode.spec.eText = QRF_TEXT_Tcl; - break; - case 6: /* json */ - p->mode.spec.eText = QRF_TEXT_Json; - break; - case 7: /* relaxed */ - p->mode.spec.eText = QRF_TEXT_Relaxed; - break; - default: /* off */ - p->mode.spec.eText = QRF_TEXT_Plain; - break; - } - chng = 1; - }else if( optionMatch(z,"reset") ){ - int saved_eMode = p->mode.eMode; - modeFree(&p->mode); - modeChange(p, saved_eMode); - }else if( optionMatch(z,"screenwidth") || optionMatch(z,"sw") ){ - if( (++i)>=nArg ){ - dotCmdError(p, i-1, "missing argument", 0); - return 1; - } - k = pickStr(azArg[i],0,"off","auto",""); - if( k==0 ){ - p->mode.bAutoScreenWidth = 0; - p->mode.spec.nScreenWidth = 0; - }else if( k==1 ){ - p->mode.bAutoScreenWidth = 1; - }else{ - i64 w = integerValue(azArg[i]); - p->mode.bAutoScreenWidth = 0; - if( w<0 ) w = 0; - if( w>QRF_MAX_WIDTH ) w = QRF_MAX_WIDTH; - p->mode.spec.nScreenWidth = w; - } - chng = 1; - }else if( optionMatch(z,"tag") ){ - size_t nByte; - int n; - const char *zTag; - if( i+1>=nArg ){ - dotCmdError(p, i, "missing argument", 0); - return 1; - } - zTag = azArg[++i]; - if( modeFind(p, zTag)>=0 ){ - dotCmdError(p, i, "mode already exists", 0); - return 1; - } - if( p->nSavedModes > MODE_N_USER ){ - dotCmdError(p, i-1, "cannot add more modes", 0); - return 1; - } - n = p->nSavedModes++; - nByte = sizeof(p->aSavedModes[0]); - nByte *= n+1; - p->aSavedModes = realloc( p->aSavedModes, nByte ); - shell_check_oom(p->aSavedModes); - p->aSavedModes[n].zTag = strdup(zTag); - shell_check_oom(p->aSavedModes[n].zTag); - modeDup(&p->aSavedModes[n].mode, &p->mode); - chng = 1; - }else if( optionMatch(z,"textjsonb") ){ - if( i+1>=nArg ){ - dotCmdError(p, i, "missing argument", 0); - return 1; - } - p->mode.spec.bTextJsonb = booleanValue(azArg[++i]) ? QRF_Yes : QRF_No; - chng = 1; - }else if( optionMatch(z,"titles") || optionMatch(z,"title") ){ - char *zErr = 0; - if( i+1>=nArg ){ - dotCmdError(p, i, "missing argument", 0); - return 1; - } - k = pickStr(azArg[++i],&zErr, - "off","on","plain","sql","csv","html","tcl","json",""); - /* 0 1 2 3 4 5 6 7 */ - if( k<0 ){ - dotCmdError(p, i, "bad --titles value","%z", zErr); - return 1; - } - p->mode.spec.bTitles = k>=1 ? QRF_Yes : QRF_No; - p->mode.mFlags &= ~MFLG_HDR; - p->mode.spec.eTitle = k>1 ? k-1 : aModeInfo[p->mode.eMode].eHdr; - chng = 1; - }else if( optionMatch(z,"widths") || optionMatch(z,"width") ){ - int nWidth = 0; - short int *aWidth; - const char *zW; - if( i+1>=nArg ){ - dotCmdError(p, i, "missing argument", 0); - return 1; - } - zW = azArg[++i]; - /* Every width value takes at least 2 bytes in the input string to - ** specify, so strlen(zW) bytes should be plenty of space to hold the - ** result. */ - aWidth = malloc( strlen(zW) ); - while( IsSpace(zW[0]) ) zW++; - while( zW[0] ){ - int w = 0; - int nDigit = 0; - k = zW[0]=='-' && IsDigit(zW[1]); - while( IsDigit(zW[k]) ){ - w = w*10 + zW[k] - '0'; - if( w>QRF_MAX_WIDTH ){ - dotCmdError(p,i+1,"width too big", - "Maximum column width is %d", QRF_MAX_WIDTH); - free(aWidth); - return 1; - } - nDigit++; - k++; - } - if( nDigit==0 ){ - dotCmdError(p,i+1,"syntax error", - "should be a comma-separated list if integers"); - free(aWidth); - return 1; - } - if( zW[0]=='-' ) w = -w; - aWidth[nWidth++] = w; - zW += k; - if( zW[0]==',' ) zW++; - while( IsSpace(zW[0]) ) zW++; - } - free(p->mode.spec.aWidth); - p->mode.spec.aWidth = aWidth; - p->mode.spec.nWidth = nWidth; - chng = 1; - }else if( optionMatch(z,"wrap") ){ - int w; - if( i+1>=nArg ){ - dotCmdError(p, i, "missing argument", 0); - return 1; - } - w = integerValue(azArg[++i]); - if( w<(-QRF_MAX_WIDTH) ) w = -QRF_MAX_WIDTH; - if( w>QRF_MAX_WIDTH ) w = QRF_MAX_WIDTH; - p->mode.spec.nWrap = w; - chng = 1; - }else if( optionMatch(z,"ww") ){ - p->mode.spec.bWordWrap = QRF_Yes; - chng = 1; - }else if( optionMatch(z,"wordwrap") ){ - if( i+1>=nArg ){ - dotCmdError(p, i, "missing argument", 0); - return 1; - } - p->mode.spec.bWordWrap = (u8)booleanValue(azArg[++i]) ? QRF_Yes : QRF_No; - chng = 1; - }else if( optionMatch(z,"v") || optionMatch(z,"verbose") ){ - bAll = 1; - }else if( z[0]=='-' ){ - dotCmdError(p, i, "bad option", "Use \".help .mode\" for more info"); - return 1; - }else{ - dotCmdError(p, i, iMode>0?"bad argument":"unknown mode", - "Use \".help .mode\" for more info"); - return 1; - } - } - if( !chng || bAll ){ - const ModeInfo *pI = aModeInfo + p->mode.eMode; - sqlite3_str *pDesc = sqlite3_str_new(p->db); - char *zDesc; - const char *zSetting; - - if( p->nPopMode ) sqlite3_str_appendall(pDesc, "--once "); - sqlite3_str_appendall(pDesc,pI->zName); - if( bAll || (p->mode.spec.nAlign && pI->eCx==2) ){ - int ii; - sqlite3_str_appendall(pDesc, " --align \""); - for(ii=0; ii<p->mode.spec.nAlign; ii++){ - unsigned char a = p->mode.spec.aAlign[ii]; - sqlite3_str_appendchar(pDesc, 1, "LLCR"[a&3]); - } - sqlite3_str_append(pDesc, "\"", 1); - } - if( bAll - || (p->mode.spec.bBorder==QRF_No) != ((pI->mFlg&1)!=0) - ){ - sqlite3_str_appendf(pDesc," --border %s", - p->mode.spec.bBorder==QRF_No ? "off" : "on"); - } - if( bAll || p->mode.spec.eBlob!=QRF_BLOB_Auto ){ - const char *azBQuote[] = - { "auto", "text", "sql", "hex", "tcl", "json", "size" }; - /* 0 1 2 3 4 5 6 - ** Must match QRF_BLOB_xxxx values. See all instances of tag-20251124a */ - u8 e = p->mode.spec.eBlob; - sqlite3_str_appendf(pDesc, " --blob-quote %s", azBQuote[e]); - } - zSetting = aModeStr[pI->eCSep]; - if( bAll || (zSetting && cli_strcmp(zSetting,p->mode.spec.zColumnSep)!=0) ){ - sqlite3_str_appendf(pDesc, " --colsep "); - append_c_string(pDesc, p->mode.spec.zColumnSep); - } - if( bAll || p->mode.spec.eEsc!=QRF_Auto ){ - sqlite3_str_appendf(pDesc, " --escape %s",qrfEscNames[p->mode.spec.eEsc]); - } - if( bAll - || (p->mode.spec.nLineLimit>0 && pI->eCx>0) - || p->mode.spec.nCharLimit>0 - || (p->mode.spec.nTitleLimit>0 && pI->eCx>0) - ){ - if( p->mode.spec.nLineLimit==0 - && p->mode.spec.nCharLimit==0 - && p->mode.spec.nTitleLimit==0 - ){ - sqlite3_str_appendf(pDesc, " --limits off"); - }else if( p->mode.spec.nLineLimit==DFLT_LINE_LIMIT - && p->mode.spec.nCharLimit==DFLT_CHAR_LIMIT - && p->mode.spec.nTitleLimit==DFLT_TITLE_LIMIT - ){ - sqlite3_str_appendf(pDesc, " --limits on"); - }else{ - sqlite3_str_appendf(pDesc, " --limits %d,%d,%d", - p->mode.spec.nLineLimit, p->mode.spec.nCharLimit, - p->mode.spec.nTitleLimit); - } - } - zSetting = aModeStr[pI->eNull]; - if( bAll || (zSetting && cli_strcmp(zSetting,p->mode.spec.zNull)!=0) ){ - sqlite3_str_appendf(pDesc, " --null "); - append_c_string(pDesc, p->mode.spec.zNull); - } - if( bAll - || (pI->eText!=p->mode.spec.eText && (pI->eText>1 || p->mode.spec.eText>1)) - ){ - sqlite3_str_appendf(pDesc," --quote %s",qrfQuoteNames[p->mode.spec.eText]); - } - zSetting = aModeStr[pI->eRSep]; - if( bAll || (zSetting && cli_strcmp(zSetting,p->mode.spec.zRowSep)!=0) ){ - sqlite3_str_appendf(pDesc, " --rowsep "); - append_c_string(pDesc, p->mode.spec.zRowSep); - } - if( bAll - || (pI->eCx && (p->mode.spec.nScreenWidth>0 || p->mode.bAutoScreenWidth)) - ){ - if( p->mode.bAutoScreenWidth ){ - sqlite3_str_appendall(pDesc, " --sw auto"); - }else{ - sqlite3_str_appendf(pDesc," --sw %d", - p->mode.spec.nScreenWidth); - } - } - if( bAll || p->mode.eMode==MODE_Insert ){ - sqlite3_str_appendf(pDesc," --tablename "); - append_c_string(pDesc, p->mode.spec.zTableName); - } - if( bAll || p->mode.spec.bTextJsonb ){ - sqlite3_str_appendf(pDesc," --textjsonb %s", - p->mode.spec.bTextJsonb==QRF_Yes ? "on" : "off"); - } - k = modeTitleDsply(p, bAll); - if( k==1 ){ - sqlite3_str_appendall(pDesc, " --titles off"); - }else if( k==2 ){ - sqlite3_str_appendall(pDesc, " --titles on"); - }else if( k==3 ){ - static const char *azTitle[] = - { "plain", "sql", "csv", "html", "tcl", "json"}; - sqlite3_str_appendf(pDesc, " --titles %s", - azTitle[p->mode.spec.eTitle-1]); - } - if( p->mode.spec.nWidth>0 && (bAll || pI->eCx==2) ){ - int ii; - const char *zSep = " --widths "; - for(ii=0; ii<p->mode.spec.nWidth; ii++){ - sqlite3_str_appendf(pDesc, "%s%d", zSep, (int)p->mode.spec.aWidth[ii]); - zSep = ","; - } - }else if( bAll ){ - sqlite3_str_appendall(pDesc, " --widths \"\""); - } - if( bAll || (pI->eCx>0 && p->mode.spec.bWordWrap) ){ - if( bAll ){ - sqlite3_str_appendf(pDesc, " --wordwrap %s", - p->mode.spec.bWordWrap==QRF_Yes ? "on" : "off"); - } - if( p->mode.spec.nWrap ){ - sqlite3_str_appendf(pDesc, " --wrap %d", p->mode.spec.nWrap); - } - if( !bAll ) sqlite3_str_append(pDesc, " --ww", 5); - } - zDesc = sqlite3_str_finish(pDesc); - cli_printf(p->out, ".mode %s\n", zDesc); - fflush(p->out); - sqlite3_free(zDesc); - } - return 0; -} - -/* -** DOT-COMMAND: .output -** USAGE: .output [OPTIONS] [FILE] -** -** Begin redirecting output to FILE. Or if FILE is omitted, revert -** to sending output to the console. If FILE begins with "|" then -** the remainder of file is taken as a pipe and output is directed -** into that pipe. If FILE is "memory" then output is captured in an -** internal memory buffer. If FILE is "off" then output is redirected -** into /dev/null or the equivalent. -** -** Options: -** --bom Prepend a byte-order mark to the output -** -e Accumulate output in a temporary text file then -** launch a text editor when the redirection ends. -** --error-prefix X Use X as the left-margin prefix for error messages. -** Set to an empty string to restore the default. -** --keep Keep redirecting output to its current destination. -** Use this option in combination with --show or -** with --error-prefix when you do not want to stop -** a current redirection. -** --plain Use plain text rather than HTML tables with -w -** --show Show output text captured by .testcase or by -** redirecting to "memory". -** -w Show the output in a web browser. Output is -** written into a temporary HTML file until the -** redirect ends, then the web browser is launched. -** Query results are shown as HTML tables, unless -** the --plain is used too. -** -x Show the output in a spreadsheet. Output is -** written to a temp file as CSV then the spreadsheet -** is launched when -** -** DOT-COMMAND: .once -** USAGE: .once [OPTIONS] FILE ... -** -** Write the output for the next line of SQL or the next dot-command into -** FILE. If FILE begins with "|" then it is a program into which output -** is written. The FILE argument should be omitted if one of the -e, -w, -** or -x options is used. -** -** Options: -** -e Capture output into a temporary file then bring up -** a text editor on that temporary file. -** --plain Use plain text rather than HTML tables with -w -** -w Capture output into an HTML file then bring up that -** file in a web browser -** -x Show the output in a spreadsheet. Output is -** written to a temp file as CSV then the spreadsheet -** is launched when -** -** DOT-COMMAND: .excel -** Shorthand for ".once -x" -** -** DOT-COMMAND: .www [--plain] -** Shorthand for ".once -w" or ".once --plain -w" -*/ -static int dotCmdOutput(ShellState *p){ - int nArg = p->dot.nArg; /* Number of arguments */ - char **azArg = p->dot.azArg; /* Text of the arguments */ - char *zFile = 0; /* The FILE argument */ - int i; /* Loop counter */ - int eMode = 0; /* 0: .outout/.once, 'x'=.excel, 'w'=.www */ - int bOnce = 0; /* 0: .output, 1: .once, 2: .excel/.www */ - int bPlain = 0; /* --plain option */ - int bKeep = 0; /* Keep redirecting */ - static const char *zBomUtf8 = "\357\273\277"; - const char *zBom = 0; - char c = azArg[0][0]; - int n = strlen30(azArg[0]); - - failIfSafeMode(p, "cannot run .%s in safe mode", azArg[0]); - if( c=='e' ){ - eMode = 'x'; - bOnce = 2; - }else if( c=='w' ){ - eMode = 'w'; - bOnce = 2; - }else if( n>=2 && cli_strncmp(azArg[0],"once",n)==0 ){ - bOnce = 1; - } - for(i=1; i<nArg; i++){ - char *z = azArg[i]; - if( z[0]=='-' ){ - if( z[1]=='-' ) z++; - if( cli_strcmp(z,"-bom")==0 ){ - zBom = zBomUtf8; - }else if( cli_strcmp(z,"-plain")==0 ){ - bPlain = 1; - }else if( c=='o' && z[0]=='1' && z[1]!=0 && z[2]==0 - && (z[1]=='x' || z[1]=='e' || z[1]=='w') ){ - if( bKeep || eMode ){ - dotCmdError(p, i, "incompatible with prior options",0); - goto dotCmdOutput_error; - } - eMode = z[1]; - }else if( cli_strcmp(z,"-show")==0 ){ - if( cli_output_capture ){ - sqlite3_fprintf(stdout, "%s", sqlite3_str_value(cli_output_capture)); - } - }else if( cli_strcmp(z,"-keep")==0 ){ - bKeep = 1; - }else if( optionMatch(z,"error-prefix") ){ - if( i+1>=nArg ){ - dotCmdError(p, i, "missing argument", 0); - return 1; - } - free(p->zErrPrefix); - i++; - p->zErrPrefix = azArg[i][0]==0 ? 0 : strdup(azArg[i]); - }else{ - dotCmdError(p, i, "unknown option", 0); - sqlite3_free(zFile); - return 1; - } - }else if( zFile==0 && eMode==0 ){ - if( bKeep ){ - dotCmdError(p, i, "incompatible with prior options",0); - goto dotCmdOutput_error; - } - if( cli_strcmp(z, "memory")==0 && bOnce ){ - dotCmdError(p, 0, "cannot redirect to \"memory\"", 0); - goto dotCmdOutput_error; - } - if( cli_strcmp(z, "off")==0 ){ -#ifdef _WIN32 - zFile = sqlite3_mprintf("nul"); -#else - zFile = sqlite3_mprintf("/dev/null"); -#endif - }else{ - zFile = sqlite3_mprintf("%s", z); - } - if( zFile && zFile[0]=='|' ){ - while( i+1<nArg ) zFile = sqlite3_mprintf("%z %s", zFile, azArg[++i]); - break; - } - }else{ - dotCmdError(p, i, "surplus argument", 0); - sqlite3_free(zFile); - return 1; - } - } - if( zFile==0 && !bKeep ){ - zFile = sqlite3_mprintf("stdout"); - shell_check_oom(zFile); - } - if( bOnce ){ - p->nPopOutput = 2; - }else{ - p->nPopOutput = 0; - } - if( !bKeep ) output_reset(p); -#ifndef SQLITE_NOHAVE_SYSTEM - if( eMode=='e' || eMode=='x' || eMode=='w' ){ - p->doXdgOpen = 1; - modePush(p); - if( eMode=='x' ){ - /* spreadsheet mode. Output as CSV. */ - newTempFile(p, "csv"); - p->mode.mFlags &= ~MFLG_ECHO; - p->mode.eMode = MODE_Csv; - modeSetStr(&p->mode.spec.zColumnSep, SEP_Comma); - modeSetStr(&p->mode.spec.zRowSep, SEP_CrLf); -#ifdef _WIN32 - zBom = zBomUtf8; /* Always include the BOM on Windows, as Excel does - ** not work without it. */ -#endif - }else if( eMode=='w' ){ - /* web-browser mode. */ - newTempFile(p, "html"); - if( !bPlain ) p->mode.eMode = MODE_Www; - }else{ - /* text editor mode */ - newTempFile(p, "txt"); - } - sqlite3_free(zFile); - zFile = sqlite3_mprintf("%s", p->zTempFile); - } -#endif /* SQLITE_NOHAVE_SYSTEM */ - if( !bKeep ) shell_check_oom(zFile); - if( bKeep ){ - /* no-op */ - }else if( cli_strcmp(zFile,"memory")==0 ){ - if( cli_output_capture ){ - sqlite3_str_free(cli_output_capture); - } - cli_output_capture = sqlite3_str_new(0); - }else if( zFile[0]=='|' ){ -#ifdef SQLITE_OMIT_POPEN - eputz("Error: pipes are not supported in this OS\n"); - output_redir(p, stdout); - goto dotCmdOutput_error; -#else - FILE *pfPipe = sqlite3_popen(zFile + 1, "w"); - if( pfPipe==0 ){ - assert( stderr!=NULL ); - cli_printf(stderr,"Error: cannot open pipe \"%s\"\n", zFile + 1); - goto dotCmdOutput_error; - }else{ - output_redir(p, pfPipe); - if( zBom ) cli_puts(zBom, pfPipe); - sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); - } -#endif - }else{ - FILE *pfFile = output_file_open(p, zFile); - if( pfFile==0 ){ - if( cli_strcmp(zFile,"off")!=0 ){ - assert( stderr!=NULL ); - cli_printf(stderr,"Error: cannot write to \"%s\"\n", zFile); - } - goto dotCmdOutput_error; - } else { - output_redir(p, pfFile); - if( zBom ) cli_puts(zBom, pfFile); - if( bPlain && eMode=='w' ){ - cli_puts( - "<!DOCTYPE html>\n<BODY>\n<PLAINTEXT>\n", - pfFile - ); - } - sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); - } - } - sqlite3_free(zFile); - return 0; - -dotCmdOutput_error: - sqlite3_free(zFile); - return 1; -} - -/* -** DOT-COMMAND: .check -** USAGE: .check [OPTIONS] PATTERN -** -** Verify results of commands since the most recent .testcase command. -** Restore output to the console, unless --keep is used. -** -** If PATTERN starts with "<<ENDMARK" then the actual pattern is taken from -** subsequent lines of text up to the first line that begins with ENDMARK. -** All pattern lines and the ENDMARK are discarded. -** -** Options: -** --exact Do an exact comparison including leading and -** trailing whitespace. -** --glob Treat PATTERN as a GLOB -** --keep Do not reset the testcase. More .check commands -** will follow. -** --notglob Output should not match PATTERN -** --show Write testcase output to the screen, for debugging. -*/ -static int dotCmdCheck(ShellState *p){ - int nArg = p->dot.nArg; /* Number of arguments */ - char **azArg = p->dot.azArg; /* Text of the arguments */ - int i; /* Loop counter */ - int k; /* Result of pickStr() */ - char *zTest; /* Textcase result */ - int bKeep = 0; /* --keep option */ - char *zCheck = 0; /* PATTERN argument */ - char *zPattern = 0; /* Actual test pattern */ - int eCheck = 0; /* 1: --glob, 2: --notglob, 3: --exact */ - int isOk; /* True if results are OK */ - sqlite3_int64 iStart = p->lineno; /* Line number of .check statement */ - - if( p->zTestcase[0]==0 ){ - dotCmdError(p, 0, "no .testcase is active", 0); - return 1; - } - for(i=1; i<nArg; i++){ - char *z = azArg[i]; - if( z[0]=='-' && z[1]=='-' && z[2]!=0 ) z++; - if( cli_strcmp(z,"-keep")==0 ){ - bKeep = 1; - }else if( cli_strcmp(z,"-show")==0 ){ - if( cli_output_capture ){ - sqlite3_fprintf(stdout, "%s", sqlite3_str_value(cli_output_capture)); - } - bKeep = 1; - }else if( z[0]=='-' - && (k = pickStr(&z[1],0,"glob","notglob","exact",""))>=0 - ){ - if( eCheck && eCheck!=k+1 ){ - dotCmdError(p, i, "incompatible with prior options",0); - return 1; - } - eCheck = k+1; - }else if( zCheck ){ - dotCmdError(p, i, "unknown option", 0); - return 1; - }else{ - zCheck = azArg[i]; - } - } - if( zCheck==0 ){ - dotCmdError(p, 0, "no PATTERN specified", 0); - return 1; - } - if( cli_output_capture && sqlite3_str_length(cli_output_capture) ){ - zTest = sqlite3_str_value(cli_output_capture); - shell_check_oom(zTest); - }else{ - zTest = ""; - } - p->nTestRun++; - if( zCheck[0]=='<' && zCheck[1]=='<' && zCheck[2]!=0 ){ - int nCheck = strlen30(zCheck); - sqlite3_str *pPattern = sqlite3_str_new(p->db); - char zLine[2000]; - while( sqlite3_fgets(zLine,sizeof(zLine),p->in) ){ - if( strchr(zLine,'\n') ) p->lineno++; - if( cli_strncmp(&zCheck[2],zLine,nCheck-2)==0 ) break; - sqlite3_str_appendall(pPattern, zLine); - } - zPattern = sqlite3_str_finish(pPattern); - if( zPattern==0 ){ - zPattern = sqlite3_mprintf(""); - } + sqlite3_finalize(pStmt); + return 0; + }else if( *pDb==0 ){ + return 0; }else{ - zPattern = zCheck; - } - shell_check_oom(zPattern); - switch( eCheck ){ - case 1: { - char *zGlob = sqlite3_mprintf("*%s*", zPattern); - isOk = testcase_glob(zGlob, zTest)!=0; - sqlite3_free(zGlob); - break; - } - case 2: { - char *zGlob = sqlite3_mprintf("*%s*", zPattern); - isOk = testcase_glob(zGlob, zTest)==0; - sqlite3_free(zGlob); - break; + /* Formulate the columns spec, close the DB, zero *pDb. */ + char *zColsSpec = 0; + int hasDupes = db_int(*pDb, "%s", zHasDupes); + int nDigits = (hasDupes)? db_int(*pDb, "%s", zColDigits) : 0; + if( hasDupes ){ +#ifdef SHELL_COLUMN_RENAME_CLEAN + rc = sqlite3_exec(*pDb, zDedoctor, 0, 0, 0); + rc_err_oom_die(rc); +#endif + rc = sqlite3_exec(*pDb, zSetReps, 0, 0, 0); + rc_err_oom_die(rc); + rc = sqlite3_prepare_v2(*pDb, zRenameRank, -1, &pStmt, 0); + rc_err_oom_die(rc); + sqlite3_bind_int(pStmt, 1, nDigits); + rc = sqlite3_step(pStmt); + sqlite3_finalize(pStmt); + if( rc!=SQLITE_DONE ) rc_err_oom_die(SQLITE_NOMEM); } - case 3: { - isOk = cli_strcmp(zTest,zPattern)==0; - break; + assert(db_int(*pDb, "%s", zHasDupes)==0); /* Consider: remove this */ + rc = sqlite3_prepare_v2(*pDb, zCollectVar, -1, &pStmt, 0); + rc_err_oom_die(rc); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + zColsSpec = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); + }else{ + zColsSpec = 0; } - default: { - /* Skip leading and trailing \n and \r on both pattern and test output */ - const char *z1 = zPattern; - const char *z2 = zTest; - size_t n1, n2; - while( z1[0]=='\n' || z1[0]=='\r' ) z1++; - n1 = strlen(z1); - while( n1>0 && (z1[n1-1]=='\n' || z1[n1-1]=='\r') ) n1--; - while( z2[0]=='\n' || z2[0]=='\r' ) z2++; - n2 = strlen(z2); - while( n2>0 && (z2[n2-1]=='\n' || z2[n2-1]=='\r') ) n2--; - isOk = n1==n2 && memcmp(z1,z2,n1)==0; - break; + if( pzRenamed!=0 ){ + if( !hasDupes ) *pzRenamed = 0; + else{ + sqlite3_finalize(pStmt); + if( SQLITE_OK==sqlite3_prepare_v2(*pDb, zRenamesDone, -1, &pStmt, 0) + && SQLITE_ROW==sqlite3_step(pStmt) ){ + *pzRenamed = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); + }else + *pzRenamed = 0; + } } + sqlite3_finalize(pStmt); + sqlite3_close(*pDb); + *pDb = 0; + return zColsSpec; } - if( !isOk ){ - sqlite3_fprintf(stderr, - "%s:%lld: .check failed for testcase %s\n", - p->zInFile, iStart, p->zTestcase); - p->nTestErr++; - sqlite3_fprintf(stderr, "Expected: [%s]\n", zPattern); - sqlite3_fprintf(stderr, "Got: [%s]\n", zTest); - } - if( zPattern!=zCheck ){ - sqlite3_free(zPattern); - } - if( !bKeep ){ - output_reset(p); - p->zTestcase[0] = 0; - } - return 0; } /* -** DOT-COMMAND: .testcase -** USAGE: .testcase [OPTIONS] NAME -** -** Start a new test case identified by NAME. All output -** through the next ".check" command is captured for comparison. See the -** ".check" commandn for additional informatioon. +** Check if the sqlite_schema table contains one or more virtual tables. If +** parameter zLike is not NULL, then it is an SQL expression that the +** sqlite_schema row must also match. If one or more such rows are found, +** print the following warning to the output: ** -** Options: -** --error-prefix TEXT Change error message prefix text to TEXT +** WARNING: Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled */ -static int dotCmdTestcase(ShellState *p){ - int nArg = p->dot.nArg; /* Number of arguments */ - char **azArg = p->dot.azArg; /* Text of the arguments */ - int i; /* Loop counter */ - const char *zName = 0; /* Testcase name */ - - for(i=1; i<nArg; i++){ - char *z = azArg[i]; - if( z[0]=='-' && z[1]=='-' && z[2]!=0 ) z++; - if( optionMatch(z,"error-prefix") ){ - if( i+1>=nArg ){ - dotCmdError(p, i, "missing argument", 0); - return 1; - } - free(p->zErrPrefix); - i++; - p->zErrPrefix = azArg[i][0]==0 ? 0 : strdup(azArg[i]); - }else if( zName ){ - dotCmdError(p, i, "unknown option", 0); - return 1; - }else{ - zName = azArg[i]; - } - } - output_reset(p); - if( cli_output_capture ){ - sqlite3_str_free(cli_output_capture); - } - cli_output_capture = sqlite3_str_new(0); - if( zName ){ - sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "%s", zName); - }else{ - sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "%s:%lld", - p->zInFile, p->lineno); +static int outputDumpWarning(ShellState *p, const char *zLike){ + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + shellPreparePrintf(p->db, &rc, &pStmt, + "SELECT 1 FROM sqlite_schema o WHERE " + "sql LIKE 'CREATE VIRTUAL TABLE%%' AND %s", zLike ? zLike : "true" + ); + if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + sqlite3_fputs("/* WARNING: " + "Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled */\n", + p->out + ); } - return 0; + shellFinalize(&rc, pStmt); + return rc; } /* -** Enlarge the space allocated in p->dot so that it can hold more -** than nArg parsed command-line arguments. +** Fault-Simulator state and logic. */ -static void parseDotRealloc(ShellState *p, int nArg){ - p->dot.nAlloc = nArg+22; - p->dot.azArg = realloc(p->dot.azArg,p->dot.nAlloc*sizeof(char*)); - shell_check_oom(p->dot.azArg); - p->dot.aiOfst = realloc(p->dot.aiOfst,p->dot.nAlloc*sizeof(int)); - shell_check_oom(p->dot.aiOfst); - p->dot.abQuot = realloc(p->dot.abQuot,p->dot.nAlloc); - shell_check_oom(p->dot.abQuot); -} - +static struct { + int iId; /* ID that triggers a simulated fault. -1 means "any" */ + int iErr; /* The error code to return on a fault */ + int iCnt; /* Trigger the fault only if iCnt is already zero */ + int iInterval; /* Reset iCnt to this value after each fault */ + int eVerbose; /* When to print output */ + int nHit; /* Number of hits seen so far */ + int nRepeat; /* Turn off after this many hits. 0 for never */ + int nSkip; /* Skip this many before first fault */ +} faultsim_state = {-1, 0, 0, 0, 0, 0, 0, 0}; /* -** Parse input line zLine up into individual arguments. Retain the -** parse in the p->dot substructure. +** This is the fault-sim callback */ -static void parseDotCmdArgs(const char *zLine, ShellState *p){ - char *z; - int h = 1; - int nArg = 0; - size_t szLine; - - p->dot.zOrig = zLine; - free(p->dot.zCopy); - z = p->dot.zCopy = strdup(zLine); - shell_check_oom(z); - szLine = strlen(z); - while( szLine>0 && IsSpace(z[szLine-1]) ) szLine--; - if( szLine>0 && z[szLine-1]==';' ){ - szLine--; - while( szLine>0 && IsSpace(z[szLine-1]) ) szLine--; - } - z[szLine] = 0; - parseDotRealloc(p, 2); - while( z[h] ){ - while( IsSpace(z[h]) ){ h++; } - if( z[h]==0 ) break; - if( nArg+2>p->dot.nAlloc ){ - parseDotRealloc(p, nArg); - } - if( z[h]=='\'' || z[h]=='"' ){ - int delim = z[h++]; - p->dot.abQuot[nArg] = 1; - p->dot.azArg[nArg] = &z[h]; - p->dot.aiOfst[nArg] = h; - while( z[h] && z[h]!=delim ){ - if( z[h]=='\\' && delim=='"' && z[h+1]!=0 ) h++; - h++; - } - if( z[h]==delim ){ - z[h++] = 0; - } - if( delim=='"' ) resolve_backslashes(p->dot.azArg[nArg]); - }else{ - p->dot.abQuot[nArg] = 0; - p->dot.azArg[nArg] = &z[h]; - p->dot.aiOfst[nArg] = h; - while( z[h] && !IsSpace(z[h]) ){ h++; } - if( z[h] ) z[h++] = 0; +static int faultsim_callback(int iArg){ + if( faultsim_state.iId>0 && faultsim_state.iId!=iArg ){ + return SQLITE_OK; + } + if( faultsim_state.iCnt ){ + if( faultsim_state.iCnt>0 ) faultsim_state.iCnt--; + if( faultsim_state.eVerbose>=2 ){ + sqlite3_fprintf(stdout, + "FAULT-SIM id=%d no-fault (cnt=%d)\n", iArg, faultsim_state.iCnt); } - nArg++; + return SQLITE_OK; + } + if( faultsim_state.eVerbose>=1 ){ + sqlite3_fprintf(stdout, + "FAULT-SIM id=%d returns %d\n", iArg, faultsim_state.iErr); + } + faultsim_state.iCnt = faultsim_state.iInterval; + faultsim_state.nHit++; + if( faultsim_state.nRepeat>0 && faultsim_state.nRepeat<=faultsim_state.nHit ){ + faultsim_state.iCnt = -1; } - p->dot.nArg = nArg; - p->dot.azArg[nArg] = 0; + return faultsim_state.iErr; } /* @@ -8962,11 +8762,12 @@ static void parseDotCmdArgs(const char *zLine, ShellState *p){ ** ** Return 1 on error, 2 to exit, and 0 otherwise. */ -static int do_meta_command(const char *zLine, ShellState *p){ - int nArg; +static int do_meta_command(char *zLine, ShellState *p){ + int h = 1; + int nArg = 0; int n, c; int rc = 0; - char **azArg; + char *azArg[52]; #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) if( p->expert.pExpert ){ @@ -8974,11 +8775,29 @@ static int do_meta_command(const char *zLine, ShellState *p){ } #endif - /* Parse the input line into tokens stored in p->dot. + /* Parse the input line into tokens. */ - parseDotCmdArgs(zLine, p); - nArg = p->dot.nArg; - azArg = p->dot.azArg; + while( zLine[h] && nArg<ArraySize(azArg)-1 ){ + while( IsSpace(zLine[h]) ){ h++; } + if( zLine[h]==0 ) break; + if( zLine[h]=='\'' || zLine[h]=='"' ){ + int delim = zLine[h++]; + azArg[nArg++] = &zLine[h]; + while( zLine[h] && zLine[h]!=delim ){ + if( zLine[h]=='\\' && delim=='"' && zLine[h+1]!=0 ) h++; + h++; + } + if( zLine[h]==delim ){ + zLine[h++] = 0; + } + if( delim=='"' ) resolve_backslashes(azArg[nArg-1]); + }else{ + azArg[nArg++] = &zLine[h]; + while( zLine[h] && !IsSpace(zLine[h]) ){ h++; } + if( zLine[h] ) zLine[h++] = 0; + } + } + azArg[nArg] = 0; /* Process the input line. */ @@ -8990,7 +8809,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ #ifndef SQLITE_OMIT_AUTHORIZATION if( c=='a' && cli_strncmp(azArg[0], "auth", n)==0 ){ if( nArg!=2 ){ - cli_printf(stderr, "Usage: .auth ON|OFF\n"); + sqlite3_fprintf(stderr, "Usage: .auth ON|OFF\n"); rc = 1; goto meta_command_exit; } @@ -9037,7 +8856,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ bAsync = 1; }else { - dotCmdError(p, j, "unknown option", "should be -append or -async"); + sqlite3_fprintf(stderr,"unknown option: %s\n", azArg[j]); return 1; } }else if( zDestFile==0 ){ @@ -9046,19 +8865,19 @@ static int do_meta_command(const char *zLine, ShellState *p){ zDb = zDestFile; zDestFile = azArg[j]; }else{ - cli_printf(stderr, "Usage: .backup ?DB? ?OPTIONS? FILENAME\n"); + sqlite3_fprintf(stderr, "Usage: .backup ?DB? ?OPTIONS? FILENAME\n"); return 1; } } if( zDestFile==0 ){ - cli_printf(stderr, "missing FILENAME argument on .backup\n"); + sqlite3_fprintf(stderr, "missing FILENAME argument on .backup\n"); return 1; } if( zDb==0 ) zDb = "main"; rc = sqlite3_open_v2(zDestFile, &pDest, SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, zVfs); if( rc!=SQLITE_OK ){ - cli_printf(stderr,"Error: cannot open \"%s\"\n", zDestFile); + sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zDestFile); close_db(pDest); return 1; } @@ -9119,7 +8938,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ rc = chdir(azArg[1]); #endif if( rc ){ - cli_printf(stderr,"Cannot change to directory \"%s\"\n", azArg[1]); + sqlite3_fprintf(stderr,"Cannot change to directory \"%s\"\n", azArg[1]); rc = 1; } }else{ @@ -9138,13 +8957,31 @@ static int do_meta_command(const char *zLine, ShellState *p){ } }else +#ifndef SQLITE_SHELL_FIDDLE /* Cancel output redirection, if it is currently set (by .testcase) ** Then read the content of the testcase-out.txt file and compare against ** azArg[1]. If there are differences, report an error and exit. */ if( c=='c' && n>=3 && cli_strncmp(azArg[0], "check", n)==0 ){ - rc = dotCmdCheck(p); + char *zRes = 0; + output_reset(p); + if( nArg!=2 ){ + eputz("Usage: .check GLOB-PATTERN\n"); + rc = 2; + }else if( (zRes = readFile("testcase-out.txt", 0))==0 ){ + rc = 2; + }else if( testcase_glob(azArg[1],zRes)==0 ){ + sqlite3_fprintf(stderr, + "testcase-%s FAILED\n Expected: [%s]\n Got: [%s]\n", + p->zTestcase, azArg[1], zRes); + rc = 1; + }else{ + sqlite3_fprintf(p->out, "testcase-%s ok\n", p->zTestcase); + p->nCheck++; + } + sqlite3_free(zRes); }else +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ #ifndef SQLITE_SHELL_FIDDLE if( c=='c' && cli_strncmp(azArg[0], "clone", n)==0 ){ @@ -9172,9 +9009,9 @@ static int do_meta_command(const char *zLine, ShellState *p){ zFile = "(temporary-file)"; } if( p->pAuxDb == &p->aAuxDb[i] ){ - cli_printf(stdout, "ACTIVE %d: %s\n", i, zFile); + sqlite3_fprintf(stdout, "ACTIVE %d: %s\n", i, zFile); }else if( p->aAuxDb[i].db!=0 ){ - cli_printf(stdout, " %d: %s\n", i, zFile); + sqlite3_fprintf(stdout, " %d: %s\n", i, zFile); } } }else if( nArg==2 && IsDigit(azArg[1][0]) && azArg[1][1]==0 ){ @@ -9210,17 +9047,12 @@ static int do_meta_command(const char *zLine, ShellState *p){ ){ if( nArg==2 ){ #ifdef _WIN32 - if( booleanValue(azArg[1]) ){ - p->mode.mFlags |= MFLG_CRLF; - }else{ - p->mode.mFlags &= ~MFLG_CRLF; - } + p->crlfMode = booleanValue(azArg[1]); #else - p->mode.mFlags &= ~MFLG_CRLF; + p->crlfMode = 0; #endif } - cli_printf(stderr, "crlf is %s\n", - (p->mode.mFlags & MFLG_CRLF)!=0 ? "ON" : "OFF"); + sqlite3_fprintf(stderr, "crlf is %s\n", p->crlfMode ? "ON" : "OFF"); }else if( c=='d' && n>1 && cli_strncmp(azArg[0], "databases", n)==0 ){ @@ -9250,7 +9082,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ int eTxn = sqlite3_txn_state(p->db, azName[i*2]); int bRdonly = sqlite3_db_readonly(p->db, azName[i*2]); const char *z = azName[i*2+1]; - cli_printf(p->out, "%s: %s %s%s\n", + sqlite3_fprintf(p->out, "%s: %s %s%s\n", azName[i*2], z && z[0] ? z : "\"\"", bRdonly ? "r/o" : "r/w", eTxn==SQLITE_TXN_NONE ? "" : eTxn==SQLITE_TXN_READ ? " read-txn" : " write-txn"); @@ -9276,7 +9108,6 @@ static int do_meta_command(const char *zLine, ShellState *p){ { "enable_trigger", SQLITE_DBCONFIG_ENABLE_TRIGGER }, { "enable_view", SQLITE_DBCONFIG_ENABLE_VIEW }, { "fts3_tokenizer", SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER }, - { "fp_digits", SQLITE_DBCONFIG_FP_DIGITS }, { "legacy_alter_table", SQLITE_DBCONFIG_LEGACY_ALTER_TABLE }, { "legacy_file_format", SQLITE_DBCONFIG_LEGACY_FILE_FORMAT }, { "load_extension", SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION }, @@ -9293,24 +9124,16 @@ static int do_meta_command(const char *zLine, ShellState *p){ for(ii=0; ii<ArraySize(aDbConfig); ii++){ if( nArg>1 && cli_strcmp(azArg[1], aDbConfig[ii].zName)!=0 ) continue; if( nArg>=3 ){ - if( aDbConfig[ii].op==SQLITE_DBCONFIG_FP_DIGITS ){ - sqlite3_db_config(p->db, aDbConfig[ii].op, atoi(azArg[2]), 0); - }else{ - sqlite3_db_config(p->db, aDbConfig[ii].op, booleanValue(azArg[2]), 0); - } + sqlite3_db_config(p->db, aDbConfig[ii].op, booleanValue(azArg[2]), 0); } sqlite3_db_config(p->db, aDbConfig[ii].op, -1, &v); - if( aDbConfig[ii].op==SQLITE_DBCONFIG_FP_DIGITS ){ - cli_printf(p->out, "%19s %d\n", aDbConfig[ii].zName, v); - }else{ - cli_printf(p->out, "%19s %s\n", - aDbConfig[ii].zName, v ? "on" : "off"); - } + sqlite3_fprintf(p->out, "%19s %s\n", + aDbConfig[ii].zName, v ? "on" : "off"); if( nArg>1 ) break; } if( nArg>1 && ii==ArraySize(aDbConfig) ){ - dotCmdError(p, 1, "unknown dbconfig", - "Enter \".dbconfig\" with no arguments for a list"); + sqlite3_fprintf(stderr,"Error: unknown dbconfig \"%s\"\n", azArg[1]); + eputz("Enter \".dbconfig\" with no arguments for a list\n"); } }else @@ -9329,19 +9152,19 @@ static int do_meta_command(const char *zLine, ShellState *p){ char *zLike = 0; char *zSql; int i; + int savedShowHeader = p->showHeader; int savedShellFlags = p->shellFlgs; - Mode saved_mode; ShellClearFlag(p, - SHFLG_PreserveRowid|SHFLG_DumpDataOnly|SHFLG_DumpNoSys); + SHFLG_PreserveRowid|SHFLG_Newlines|SHFLG_Echo + |SHFLG_DumpDataOnly|SHFLG_DumpNoSys); for(i=1; i<nArg; i++){ if( azArg[i][0]=='-' ){ const char *z = azArg[i]+1; if( z[0]=='-' ) z++; if( cli_strcmp(z,"preserve-rowids")==0 ){ #ifdef SQLITE_OMIT_VIRTUALTABLE - dotCmdError(p, i, "unable", - "The --preserve-rowids option is not compatible" - " with SQLITE_OMIT_VIRTUALTABLE"); + eputz("The --preserve-rowids option is not compatible" + " with SQLITE_OMIT_VIRTUALTABLE\n"); rc = 1; sqlite3_free(zLike); goto meta_command_exit; @@ -9350,7 +9173,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ #endif }else if( cli_strcmp(z,"newlines")==0 ){ - /*ShellSetFlag(p, SHFLG_Newlines);*/ + ShellSetFlag(p, SHFLG_Newlines); }else if( cli_strcmp(z,"data-only")==0 ){ ShellSetFlag(p, SHFLG_DumpDataOnly); @@ -9359,7 +9182,8 @@ static int do_meta_command(const char *zLine, ShellState *p){ ShellSetFlag(p, SHFLG_DumpNoSys); }else { - dotCmdError(p, i, "unknown option", 0); + sqlite3_fprintf(stderr, + "Unknown option \"%s\" on \".dump\"\n", azArg[i]); rc = 1; sqlite3_free(zLike); goto meta_command_exit; @@ -9389,17 +9213,16 @@ static int do_meta_command(const char *zLine, ShellState *p){ open_db(p, 0); - modeDup(&saved_mode, &p->mode); outputDumpWarning(p, zLike); if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){ /* When playing back a "dump", the content might appear in an order ** which causes immediate foreign key constraints to be violated. ** So disable foreign-key constraint enforcement to prevent problems. */ - cli_puts("PRAGMA foreign_keys=OFF;\n", p->out); - cli_puts("BEGIN TRANSACTION;\n", p->out); + sqlite3_fputs("PRAGMA foreign_keys=OFF;\n", p->out); + sqlite3_fputs("BEGIN TRANSACTION;\n", p->out); } p->writableSchema = 0; - p->mode.spec.bTitles = QRF_No; + p->showHeader = 0; /* Set writable_schema=ON since doing so forces SQLite to initialize ** as much of the schema as it can even if the sqlite_schema table is ** corrupt. */ @@ -9428,27 +9251,22 @@ static int do_meta_command(const char *zLine, ShellState *p){ } sqlite3_free(zLike); if( p->writableSchema ){ - cli_puts("PRAGMA writable_schema=OFF;\n", p->out); + sqlite3_fputs("PRAGMA writable_schema=OFF;\n", p->out); p->writableSchema = 0; } sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0); sqlite3_exec(p->db, "RELEASE dump;", 0, 0, 0); if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){ - cli_puts(p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n", p->out); + sqlite3_fputs(p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n", p->out); } + p->showHeader = savedShowHeader; p->shellFlgs = savedShellFlags; - modeFree(&p->mode); - p->mode = saved_mode; rc = p->nErr>0; }else if( c=='e' && cli_strncmp(azArg[0], "echo", n)==0 ){ if( nArg==2 ){ - if( booleanValue(azArg[1]) ){ - p->mode.mFlags |= MFLG_ECHO; - }else{ - p->mode.mFlags &= ~MFLG_ECHO; - } + setOrClearFlag(p, SHFLG_Echo, azArg[1]); }else{ eputz("Usage: .echo on|off\n"); rc = 1; @@ -9462,24 +9280,28 @@ static int do_meta_command(const char *zLine, ShellState *p){ if( c=='e' && cli_strncmp(azArg[0], "eqp", n)==0 ){ if( nArg==2 ){ - if( p->mode.autoEQPtrace ){ + p->autoEQPtest = 0; + if( p->autoEQPtrace ){ if( p->db ) sqlite3_exec(p->db, "PRAGMA vdbe_trace=OFF;", 0, 0, 0); - p->mode.autoEQPtrace = 0; + p->autoEQPtrace = 0; } if( cli_strcmp(azArg[1],"full")==0 ){ - p->mode.autoEQP = AUTOEQP_full; + p->autoEQP = AUTOEQP_full; }else if( cli_strcmp(azArg[1],"trigger")==0 ){ - p->mode.autoEQP = AUTOEQP_trigger; + p->autoEQP = AUTOEQP_trigger; #ifdef SQLITE_DEBUG + }else if( cli_strcmp(azArg[1],"test")==0 ){ + p->autoEQP = AUTOEQP_on; + p->autoEQPtest = 1; }else if( cli_strcmp(azArg[1],"trace")==0 ){ - p->mode.autoEQP = AUTOEQP_full; - p->mode.autoEQPtrace = 1; + p->autoEQP = AUTOEQP_full; + p->autoEQPtrace = 1; open_db(p, 0); sqlite3_exec(p->db, "SELECT name FROM sqlite_schema LIMIT 1", 0, 0, 0); sqlite3_exec(p->db, "PRAGMA vdbe_trace=ON;", 0, 0, 0); #endif }else{ - p->mode.autoEQP = (u8)booleanValue(azArg[1]); + p->autoEQP = (u8)booleanValue(azArg[1]); } }else{ eputz("Usage: .eqp off|on|trace|trigger|full\n"); @@ -9489,7 +9311,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ #ifndef SQLITE_SHELL_FIDDLE if( c=='e' && cli_strncmp(azArg[0], "exit", n)==0 ){ - if( nArg>1 && (rc = (int)integerValue(azArg[1]))!=0 ) cli_exit(rc); + if( nArg>1 && (rc = (int)integerValue(azArg[1]))!=0 ) exit(rc); rc = 2; }else #endif @@ -9497,19 +9319,31 @@ static int do_meta_command(const char *zLine, ShellState *p){ /* The ".explain" command is automatic now. It is largely pointless. It ** retained purely for backwards compatibility */ if( c=='e' && cli_strncmp(azArg[0], "explain", n)==0 ){ + int val = 1; if( nArg>=2 ){ if( cli_strcmp(azArg[1],"auto")==0 ){ - p->mode.autoExplain = 1; + val = 99; }else{ - p->mode.autoExplain = booleanValue(azArg[1]); + val = booleanValue(azArg[1]); } } + if( val==1 && p->mode!=MODE_Explain ){ + p->normalMode = p->mode; + p->mode = MODE_Explain; + p->autoExplain = 0; + }else if( val==0 ){ + if( p->mode==MODE_Explain ) p->mode = p->normalMode; + p->autoExplain = 0; + }else if( val==99 ){ + if( p->mode==MODE_Explain ) p->mode = p->normalMode; + p->autoExplain = 1; + } }else #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) if( c=='e' && cli_strncmp(azArg[0], "expert", n)==0 ){ if( p->bSafeMode ){ - cli_printf(stderr, + sqlite3_fprintf(stderr, "Cannot run experimental commands such as \"%s\" in safe mode\n", azArg[0]); rc = 1; @@ -9567,9 +9401,9 @@ static int do_meta_command(const char *zLine, ShellState *p){ /* --help lists all file-controls */ if( cli_strcmp(zCmd,"help")==0 ){ - cli_puts("Available file-controls:\n", p->out); + sqlite3_fputs("Available file-controls:\n", p->out); for(i=0; i<ArraySize(aCtrl); i++){ - cli_printf(p->out, + sqlite3_fprintf(p->out, " .filectrl %s %s\n", aCtrl[i].zCtrlName, aCtrl[i].zUsage); } rc = 1; @@ -9585,7 +9419,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ filectrl = aCtrl[i].ctrlCode; iCtrl = i; }else{ - cli_printf(stderr,"Error: ambiguous file-control: \"%s\"\n" + sqlite3_fprintf(stderr,"Error: ambiguous file-control: \"%s\"\n" "Use \".filectrl --help\" for help\n", zCmd); rc = 1; goto meta_command_exit; @@ -9593,7 +9427,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ } } if( filectrl<0 ){ - cli_printf(stderr,"Error: unknown file-control: %s\n" + sqlite3_fprintf(stderr,"Error: unknown file-control: %s\n" "Use \".filectrl --help\" for help\n", zCmd); }else{ switch(filectrl){ @@ -9637,7 +9471,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ if( nArg!=2 ) break; sqlite3_file_control(p->db, zSchema, filectrl, &z); if( z ){ - cli_printf(p->out, "%s\n", z); + sqlite3_fprintf(p->out, "%s\n", z); sqlite3_free(z); } isOk = 2; @@ -9651,30 +9485,31 @@ static int do_meta_command(const char *zLine, ShellState *p){ } x = -1; sqlite3_file_control(p->db, zSchema, filectrl, &x); - cli_printf(p->out, "%d\n", x); + sqlite3_fprintf(p->out, "%d\n", x); isOk = 2; break; } } } if( isOk==0 && iCtrl>=0 ){ - cli_printf(p->out, "Usage: .filectrl %s %s\n", + sqlite3_fprintf(p->out, "Usage: .filectrl %s %s\n", zCmd, aCtrl[iCtrl].zUsage); rc = 1; }else if( isOk==1 ){ char zBuf[100]; sqlite3_snprintf(sizeof(zBuf), zBuf, "%lld", iRes); - cli_printf(p->out, "%s\n", zBuf); + sqlite3_fprintf(p->out, "%s\n", zBuf); } }else if( c=='f' && cli_strncmp(azArg[0], "fullschema", n)==0 ){ ShellState data; int doStats = 0; - int hasStat[5]; - int flgs = 0; - char *zSql; + memcpy(&data, p, sizeof(data)); + data.showHeader = 0; + data.cMode = data.mode = MODE_Semi; if( nArg==2 && optionMatch(azArg[1], "indent") ){ + data.cMode = data.mode = MODE_Pretty; nArg = 1; } if( nArg!=1 ){ @@ -9683,81 +9518,384 @@ static int do_meta_command(const char *zLine, ShellState *p){ goto meta_command_exit; } open_db(p, 0); - zSql = sqlite3_mprintf( - "SELECT shell_format_schema(sql,%d) FROM" + rc = sqlite3_exec(p->db, + "SELECT sql FROM" " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" " FROM sqlite_schema UNION ALL" " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_schema) " "WHERE type!='meta' AND sql NOTNULL" - " AND name NOT LIKE 'sqlite__%%' ESCAPE '_' " - "ORDER BY x", flgs); - memcpy(&data, p, sizeof(data)); - data.mode.spec.bTitles = QRF_No; - data.mode.eMode = MODE_List; - data.mode.spec.eText = QRF_TEXT_Plain; - data.mode.spec.nCharLimit = 0; - data.mode.spec.zRowSep = "\n"; - rc = shell_exec(&data,zSql,0); - sqlite3_free(zSql); + " AND name NOT LIKE 'sqlite__%' ESCAPE '_' " + "ORDER BY x", + callback, &data, 0 + ); if( rc==SQLITE_OK ){ sqlite3_stmt *pStmt; - memset(hasStat, 0, sizeof(hasStat)); rc = sqlite3_prepare_v2(p->db, - "SELECT substr(name,12,1) FROM sqlite_schema" + "SELECT rowid FROM sqlite_schema" " WHERE name GLOB 'sqlite_stat[134]'", -1, &pStmt, 0); if( rc==SQLITE_OK ){ - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - int k = sqlite3_column_int(pStmt,0); - assert( k==1 || k==3 || k==4 ); - hasStat[k] = 1; - doStats = 1; - } + doStats = sqlite3_step(pStmt)==SQLITE_ROW; + sqlite3_finalize(pStmt); } - sqlite3_finalize(pStmt); } if( doStats==0 ){ - cli_puts("/* No STAT tables available */\n", p->out); + sqlite3_fputs("/* No STAT tables available */\n", p->out); }else{ - cli_puts("ANALYZE sqlite_schema;\n", p->out); - data.mode.eMode = MODE_Insert; - if( hasStat[1] ){ - data.mode.spec.zTableName = "sqlite_stat1"; - shell_exec(&data, "SELECT * FROM sqlite_stat1", 0); - } - if( hasStat[4] ){ - data.mode.spec.zTableName = "sqlite_stat4"; - shell_exec(&data, "SELECT * FROM sqlite_stat4", 0); - } - cli_puts("ANALYZE sqlite_schema;\n", p->out); + sqlite3_fputs("ANALYZE sqlite_schema;\n", p->out); + data.cMode = data.mode = MODE_Insert; + data.zDestTable = "sqlite_stat1"; + shell_exec(&data, "SELECT * FROM sqlite_stat1", 0); + data.zDestTable = "sqlite_stat4"; + shell_exec(&data, "SELECT * FROM sqlite_stat4", 0); + sqlite3_fputs("ANALYZE sqlite_schema;\n", p->out); } }else if( c=='h' && cli_strncmp(azArg[0], "headers", n)==0 ){ if( nArg==2 ){ - p->mode.spec.bTitles = booleanValue(azArg[1]) ? QRF_Yes : QRF_No; - p->mode.mFlags |= MFLG_HDR; - p->mode.spec.eTitle = aModeInfo[p->mode.eMode].eHdr; + p->showHeader = booleanValue(azArg[1]); + p->shellFlgs |= SHFLG_HeaderSet; }else{ eputz("Usage: .headers on|off\n"); rc = 1; } - }else - - if( c=='h' && cli_strncmp(azArg[0], "help", n)==0 ){ - if( nArg>=2 ){ - n = showHelp(p->out, azArg[1]); - if( n==0 ){ - cli_printf(p->out, "Nothing matches '%s'\n", azArg[1]); + }else + + if( c=='h' && cli_strncmp(azArg[0], "help", n)==0 ){ + if( nArg>=2 ){ + n = showHelp(p->out, azArg[1]); + if( n==0 ){ + sqlite3_fprintf(p->out, "Nothing matches '%s'\n", azArg[1]); + } + }else{ + showHelp(p->out, 0); + } + }else + +#ifndef SQLITE_SHELL_FIDDLE + if( c=='i' && cli_strncmp(azArg[0], "import", n)==0 ){ + char *zTable = 0; /* Insert data into this table */ + char *zSchema = 0; /* Schema of zTable */ + char *zFile = 0; /* Name of file to extra content from */ + sqlite3_stmt *pStmt = NULL; /* A statement */ + int nCol; /* Number of columns in the table */ + i64 nByte; /* Number of bytes in an SQL string */ + int i, j; /* Loop counters */ + int needCommit; /* True to COMMIT or ROLLBACK at end */ + int nSep; /* Number of bytes in p->colSeparator[] */ + char *zSql = 0; /* An SQL statement */ + ImportCtx sCtx; /* Reader context */ + char *(SQLITE_CDECL *xRead)(ImportCtx*); /* Func to read one value */ + int eVerbose = 0; /* Larger for more console output */ + i64 nSkip = 0; /* Initial lines to skip */ + int useOutputMode = 1; /* Use output mode to determine separators */ + char *zCreate = 0; /* CREATE TABLE statement text */ + + failIfSafeMode(p, "cannot run .import in safe mode"); + memset(&sCtx, 0, sizeof(sCtx)); + if( p->mode==MODE_Ascii ){ + xRead = ascii_read_one_field; + }else{ + xRead = csv_read_one_field; + } + rc = 1; + for(i=1; i<nArg; i++){ + char *z = azArg[i]; + if( z[0]=='-' && z[1]=='-' ) z++; + if( z[0]!='-' ){ + if( zFile==0 ){ + zFile = z; + }else if( zTable==0 ){ + zTable = z; + }else{ + sqlite3_fprintf(p->out, "ERROR: extra argument: \"%s\". Usage:\n",z); + showHelp(p->out, "import"); + goto meta_command_exit; + } + }else if( cli_strcmp(z,"-v")==0 ){ + eVerbose++; + }else if( cli_strcmp(z,"-schema")==0 && i<nArg-1 ){ + zSchema = azArg[++i]; + }else if( cli_strcmp(z,"-skip")==0 && i<nArg-1 ){ + nSkip = integerValue(azArg[++i]); + }else if( cli_strcmp(z,"-ascii")==0 ){ + sCtx.cColSep = SEP_Unit[0]; + sCtx.cRowSep = SEP_Record[0]; + xRead = ascii_read_one_field; + useOutputMode = 0; + }else if( cli_strcmp(z,"-csv")==0 ){ + sCtx.cColSep = ','; + sCtx.cRowSep = '\n'; + xRead = csv_read_one_field; + useOutputMode = 0; + }else{ + sqlite3_fprintf(p->out, "ERROR: unknown option: \"%s\". Usage:\n", z); + showHelp(p->out, "import"); + goto meta_command_exit; + } + } + if( zTable==0 ){ + sqlite3_fprintf(p->out, "ERROR: missing %s argument. Usage:\n", + zFile==0 ? "FILE" : "TABLE"); + showHelp(p->out, "import"); + goto meta_command_exit; + } + seenInterrupt = 0; + open_db(p, 0); + if( useOutputMode ){ + /* If neither the --csv or --ascii options are specified, then set + ** the column and row separator characters from the output mode. */ + nSep = strlen30(p->colSeparator); + if( nSep==0 ){ + eputz("Error: non-null column separator required for import\n"); + goto meta_command_exit; + } + if( nSep>1 ){ + eputz("Error: multi-character column separators not allowed" + " for import\n"); + goto meta_command_exit; + } + nSep = strlen30(p->rowSeparator); + if( nSep==0 ){ + eputz("Error: non-null row separator required for import\n"); + goto meta_command_exit; + } + if( nSep==2 && p->mode==MODE_Csv + && cli_strcmp(p->rowSeparator,SEP_CrLf)==0 + ){ + /* When importing CSV (only), if the row separator is set to the + ** default output row separator, change it to the default input + ** row separator. This avoids having to maintain different input + ** and output row separators. */ + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); + nSep = strlen30(p->rowSeparator); + } + if( nSep>1 ){ + eputz("Error: multi-character row separators not allowed" + " for import\n"); + goto meta_command_exit; + } + sCtx.cColSep = (u8)p->colSeparator[0]; + sCtx.cRowSep = (u8)p->rowSeparator[0]; + } + sCtx.zFile = zFile; + sCtx.nLine = 1; + if( sCtx.zFile[0]=='|' ){ +#ifdef SQLITE_OMIT_POPEN + eputz("Error: pipes are not supported in this OS\n"); + goto meta_command_exit; +#else + sCtx.in = sqlite3_popen(sCtx.zFile+1, "r"); + sCtx.zFile = "<pipe>"; + sCtx.xCloser = pclose; +#endif + }else{ + sCtx.in = sqlite3_fopen(sCtx.zFile, "rb"); + sCtx.xCloser = fclose; + } + if( sCtx.in==0 ){ + sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zFile); + goto meta_command_exit; + } + if( eVerbose>=2 || (eVerbose>=1 && useOutputMode) ){ + char zSep[2]; + zSep[1] = 0; + zSep[0] = sCtx.cColSep; + sqlite3_fputs("Column separator ", p->out); + output_c_string(p->out, zSep); + sqlite3_fputs(", row separator ", p->out); + zSep[0] = sCtx.cRowSep; + output_c_string(p->out, zSep); + sqlite3_fputs("\n", p->out); + } + sCtx.z = sqlite3_malloc64(120); + if( sCtx.z==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + /* Below, resources must be freed before exit. */ + while( nSkip>0 ){ + nSkip--; + while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){} + } + import_append_char(&sCtx, 0); /* To ensure sCtx.z is allocated */ + if( sqlite3_table_column_metadata(p->db, zSchema, zTable,0,0,0,0,0,0) + && 0==db_int(p->db, "SELECT count(*) FROM \"%w\".sqlite_schema" + " WHERE name=%Q AND type='view'", + zSchema ? zSchema : "main", zTable) + ){ + /* Table does not exist. Create it. */ + sqlite3 *dbCols = 0; + char *zRenames = 0; + char *zColDefs; + zCreate = sqlite3_mprintf("CREATE TABLE \"%w\".\"%w\"", + zSchema ? zSchema : "main", zTable); + while( xRead(&sCtx) ){ + zAutoColumn(sCtx.z, &dbCols, 0); + if( sCtx.cTerm!=sCtx.cColSep ) break; + } + zColDefs = zAutoColumn(0, &dbCols, &zRenames); + if( zRenames!=0 ){ + sqlite3_fprintf((stdin_is_interactive && p->in==stdin)? p->out : stderr, + "Columns renamed during .import %s due to duplicates:\n" + "%s\n", sCtx.zFile, zRenames); + sqlite3_free(zRenames); + } + assert(dbCols==0); + if( zColDefs==0 ){ + sqlite3_fprintf(stderr,"%s: empty file\n", sCtx.zFile); + import_cleanup(&sCtx); + rc = 1; + sqlite3_free(zCreate); + goto meta_command_exit; + } + zCreate = sqlite3_mprintf("%z%z\n", zCreate, zColDefs); + if( zCreate==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + if( eVerbose>=1 ){ + sqlite3_fprintf(p->out, "%s\n", zCreate); + } + rc = sqlite3_exec(p->db, zCreate, 0, 0, 0); + if( rc ){ + sqlite3_fprintf(stderr, + "%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db)); + } + sqlite3_free(zCreate); + zCreate = 0; + if( rc ){ + import_cleanup(&sCtx); + rc = 1; + goto meta_command_exit; + } + } + zSql = sqlite3_mprintf("SELECT count(*) FROM pragma_table_info(%Q,%Q);", + zTable, zSchema); + if( zSql==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + zSql = 0; + if( rc ){ + if (pStmt) sqlite3_finalize(pStmt); + shellDatabaseError(p->db); + import_cleanup(&sCtx); + rc = 1; + goto meta_command_exit; + } + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + nCol = sqlite3_column_int(pStmt, 0); + }else{ + nCol = 0; + } + sqlite3_finalize(pStmt); + pStmt = 0; + if( nCol==0 ) return 0; /* no columns, no error */ + + nByte = 64 /* space for "INSERT INTO", "VALUES(", ")\0" */ + + (zSchema ? strlen(zSchema)*2 + 2: 0) /* Quoted schema name */ + + strlen(zTable)*2 + 2 /* Quoted table name */ + + nCol*2; /* Space for ",?" for each column */ + zSql = sqlite3_malloc64( nByte ); + if( zSql==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + if( zSchema ){ + sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\".\"%w\" VALUES(?", + zSchema, zTable); + }else{ + sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); + } + j = strlen30(zSql); + for(i=1; i<nCol; i++){ + zSql[j++] = ','; + zSql[j++] = '?'; + } + zSql[j++] = ')'; + zSql[j] = 0; + assert( j<nByte ); + if( eVerbose>=2 ){ + sqlite3_fprintf(p->out, "Insert using: %s\n", zSql); + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + zSql = 0; + if( rc ){ + shellDatabaseError(p->db); + if (pStmt) sqlite3_finalize(pStmt); + import_cleanup(&sCtx); + rc = 1; + goto meta_command_exit; + } + needCommit = sqlite3_get_autocommit(p->db); + if( needCommit ) sqlite3_exec(p->db, "BEGIN", 0, 0, 0); + do{ + int startLine = sCtx.nLine; + for(i=0; i<nCol; i++){ + char *z = xRead(&sCtx); + /* + ** Did we reach end-of-file before finding any columns? + ** If so, stop instead of NULL filling the remaining columns. + */ + if( z==0 && i==0 ) break; + /* + ** Did we reach end-of-file OR end-of-line before finding any + ** columns in ASCII mode? If so, stop instead of NULL filling + ** the remaining columns. + */ + if( p->mode==MODE_Ascii && (z==0 || z[0]==0) && i==0 ) break; + /* + ** For CSV mode, per RFC 4180, accept EOF in lieu of final + ** record terminator but only for last field of multi-field row. + ** (If there are too few fields, it's not valid CSV anyway.) + */ + if( z==0 && (xRead==csv_read_one_field) && i==nCol-1 && i>0 ){ + z = ""; + } + sqlite3_bind_text(pStmt, i+1, z, -1, SQLITE_TRANSIENT); + if( i<nCol-1 && sCtx.cTerm!=sCtx.cColSep ){ + sqlite3_fprintf(stderr,"%s:%d: expected %d columns but found %d" + " - filling the rest with NULL\n", + sCtx.zFile, startLine, nCol, i+1); + i += 2; + while( i<=nCol ){ sqlite3_bind_null(pStmt, i); i++; } + } } - }else{ - showHelp(p->out, 0); - } - }else + if( sCtx.cTerm==sCtx.cColSep ){ + do{ + xRead(&sCtx); + i++; + }while( sCtx.cTerm==sCtx.cColSep ); + sqlite3_fprintf(stderr, + "%s:%d: expected %d columns but found %d - extras ignored\n", + sCtx.zFile, startLine, nCol, i); + } + if( i>=nCol ){ + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + sqlite3_fprintf(stderr,"%s:%d: INSERT failed: %s\n", + sCtx.zFile, startLine, sqlite3_errmsg(p->db)); + sCtx.nErr++; + }else{ + sCtx.nRow++; + } + } + }while( sCtx.cTerm!=EOF ); -#ifndef SQLITE_SHELL_FIDDLE - if( c=='i' && cli_strncmp(azArg[0], "import", n)==0 ){ - rc = dotCmdImport(p); + import_cleanup(&sCtx); + sqlite3_finalize(pStmt); + if( needCommit ) sqlite3_exec(p->db, "COMMIT", 0, 0, 0); + if( eVerbose>0 ){ + sqlite3_fprintf(p->out, + "Added %d rows with %d errors using %d lines of input\n", + sCtx.nRow, sCtx.nErr, sCtx.nLine-1); + } }else #endif /* !defined(SQLITE_SHELL_FIDDLE) */ @@ -9790,10 +9928,10 @@ static int do_meta_command(const char *zLine, ShellState *p){ } zSql = sqlite3_mprintf( "SELECT rootpage, 0 FROM sqlite_schema" - " WHERE type='index' AND lower(name)=lower('%q')" + " WHERE name='%q' AND type='index'" "UNION ALL " "SELECT rootpage, 1 FROM sqlite_schema" - " WHERE type='table' AND lower(name)=lower('%q')" + " WHERE name='%q' AND type='table'" " AND sql LIKE '%%without%%rowid%%'", azArg[1], azArg[1] ); @@ -9831,7 +9969,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ } sqlite3_finalize(pStmt); if( i==0 || tnum==0 ){ - cli_printf(stderr,"no such index: \"%s\"\n", azArg[1]); + sqlite3_fprintf(stderr,"no such index: \"%s\"\n", azArg[1]); rc = 1; sqlite3_free(zCollist); goto meta_command_exit; @@ -9846,13 +9984,13 @@ static int do_meta_command(const char *zLine, ShellState *p){ rc = sqlite3_exec(p->db, zSql, 0, 0, 0); sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 0); if( rc ){ - cli_printf(stderr, + sqlite3_fprintf(stderr, "Error in [%s]: %s\n", zSql, sqlite3_errmsg(p->db)); }else{ - cli_printf(stdout, "%s;\n", zSql); + sqlite3_fprintf(stdout, "%s;\n", zSql); } }else{ - cli_printf(stderr,"SQLITE_TESTCTRL_IMPOSTER returns %d\n", rc); + sqlite3_fprintf(stderr,"SQLITE_TESTCTRL_IMPOSTER returns %d\n", rc); rc = 1; } sqlite3_free(zSql); @@ -9866,7 +10004,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ if( iArg==0 ) iArg = -1; } if( (nArg!=1 && nArg!=2) || iArg<0 ){ - cli_printf(stderr,"%s","Usage: .intck STEPS_PER_UNLOCK\n"); + sqlite3_fprintf(stderr,"%s","Usage: .intck STEPS_PER_UNLOCK\n"); rc = 1; goto meta_command_exit; } @@ -9887,7 +10025,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ }else{ iotrace = sqlite3_fopen(azArg[1], "w"); if( iotrace==0 ){ - cli_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); + sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); sqlite3IoTrace = 0; rc = 1; }else{ @@ -9906,7 +10044,6 @@ static int do_meta_command(const char *zLine, ShellState *p){ { "sql_length", SQLITE_LIMIT_SQL_LENGTH }, { "column", SQLITE_LIMIT_COLUMN }, { "expr_depth", SQLITE_LIMIT_EXPR_DEPTH }, - { "parser_depth", SQLITE_LIMIT_PARSER_DEPTH }, { "compound_select", SQLITE_LIMIT_COMPOUND_SELECT }, { "vdbe_op", SQLITE_LIMIT_VDBE_OP }, { "function_arg", SQLITE_LIMIT_FUNCTION_ARG }, @@ -9920,7 +10057,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ open_db(p, 0); if( nArg==1 ){ for(i=0; i<ArraySize(aLimit); i++){ - cli_printf(stdout, "%20s %d\n", aLimit[i].zLimitName, + sqlite3_fprintf(stdout, "%20s %d\n", aLimit[i].zLimitName, sqlite3_limit(p->db, aLimit[i].limitCode, -1)); } }else if( nArg>3 ){ @@ -9935,14 +10072,14 @@ static int do_meta_command(const char *zLine, ShellState *p){ if( iLimit<0 ){ iLimit = i; }else{ - cli_printf(stderr,"ambiguous limit: \"%s\"\n", azArg[1]); + sqlite3_fprintf(stderr,"ambiguous limit: \"%s\"\n", azArg[1]); rc = 1; goto meta_command_exit; } } } if( iLimit<0 ){ - cli_printf(stderr,"unknown limit: \"%s\"\n" + sqlite3_fprintf(stderr,"unknown limit: \"%s\"\n" "enter \".limits\" with no arguments for a list.\n", azArg[1]); rc = 1; @@ -9951,10 +10088,9 @@ static int do_meta_command(const char *zLine, ShellState *p){ if( nArg==3 ){ sqlite3_limit(p->db, aLimit[iLimit].limitCode, (int)integerValue(azArg[2])); - }else{ - cli_printf(stdout, "%20s %d\n", aLimit[iLimit].zLimitName, - sqlite3_limit(p->db, aLimit[iLimit].limitCode, -1)); } + sqlite3_fprintf(stdout, "%20s %d\n", aLimit[iLimit].zLimitName, + sqlite3_limit(p->db, aLimit[iLimit].limitCode, -1)); } }else @@ -10002,12 +10138,174 @@ static int do_meta_command(const char *zLine, ShellState *p){ } output_file_close(p->pLog); if( cli_strcmp(zFile,"on")==0 ) zFile = "stdout"; - p->pLog = output_file_open(p, zFile); + p->pLog = output_file_open(zFile); } }else if( c=='m' && cli_strncmp(azArg[0], "mode", n)==0 ){ - rc = dotCmdMode(p); + const char *zMode = 0; + const char *zTabname = 0; + int i, n2; + int chng = 0; /* 0x01: change to cmopts. 0x02: Any other change */ + ColModeOpts cmOpts = ColModeOpts_default; + for(i=1; i<nArg; i++){ + const char *z = azArg[i]; + if( optionMatch(z,"wrap") && i+1<nArg ){ + cmOpts.iWrap = integerValue(azArg[++i]); + chng |= 1; + }else if( optionMatch(z,"ww") ){ + cmOpts.bWordWrap = 1; + chng |= 1; + }else if( optionMatch(z,"wordwrap") && i+1<nArg ){ + cmOpts.bWordWrap = (u8)booleanValue(azArg[++i]); + chng |= 1; + }else if( optionMatch(z,"quote") ){ + cmOpts.bQuote = 1; + chng |= 1; + }else if( optionMatch(z,"noquote") ){ + cmOpts.bQuote = 0; + chng |= 1; + }else if( optionMatch(z,"escape") && i+1<nArg ){ + /* See similar code at tag-20250224-1 */ + const char *zEsc = azArg[++i]; + int k; + for(k=0; k<ArraySize(shell_EscModeNames); k++){ + if( sqlite3_stricmp(zEsc,shell_EscModeNames[k])==0 ){ + p->eEscMode = k; + chng |= 2; + break; + } + } + if( k>=ArraySize(shell_EscModeNames) ){ + sqlite3_fprintf(stderr, "unknown control character escape mode \"%s\"" + " - choices:", zEsc); + for(k=0; k<ArraySize(shell_EscModeNames); k++){ + sqlite3_fprintf(stderr, " %s", shell_EscModeNames[k]); + } + sqlite3_fprintf(stderr, "\n"); + rc = 1; + goto meta_command_exit; + } + }else if( zMode==0 ){ + zMode = z; + /* Apply defaults for qbox pseudo-mode. If that + * overwrites already-set values, user was informed of this. + */ + chng |= 1; + if( cli_strcmp(z, "qbox")==0 ){ + ColModeOpts cmo = ColModeOpts_default_qbox; + zMode = "box"; + cmOpts = cmo; + } + }else if( zTabname==0 ){ + zTabname = z; + }else if( z[0]=='-' ){ + sqlite3_fprintf(stderr,"unknown option: %s\n", z); + eputz("options:\n" + " --escape MODE\n" + " --noquote\n" + " --quote\n" + " --wordwrap on/off\n" + " --wrap N\n" + " --ww\n"); + rc = 1; + goto meta_command_exit; + }else{ + sqlite3_fprintf(stderr,"extra argument: \"%s\"\n", z); + rc = 1; + goto meta_command_exit; + } + } + if( !chng ){ + if( p->mode==MODE_Column + || (p->mode>=MODE_Markdown && p->mode<=MODE_Box) + ){ + sqlite3_fprintf(p->out, + "current output mode: %s --wrap %d --wordwrap %s " + "--%squote --escape %s\n", + modeDescr[p->mode], p->cmOpts.iWrap, + p->cmOpts.bWordWrap ? "on" : "off", + p->cmOpts.bQuote ? "" : "no", + shell_EscModeNames[p->eEscMode] + ); + }else{ + sqlite3_fprintf(p->out, + "current output mode: %s --escape %s\n", + modeDescr[p->mode], + shell_EscModeNames[p->eEscMode] + ); + } + } + if( zMode==0 ){ + zMode = modeDescr[p->mode]; + if( (chng&1)==0 ) cmOpts = p->cmOpts; + } + n2 = strlen30(zMode); + if( cli_strncmp(zMode,"lines",n2)==0 ){ + p->mode = MODE_Line; + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); + }else if( cli_strncmp(zMode,"columns",n2)==0 ){ + p->mode = MODE_Column; + if( (p->shellFlgs & SHFLG_HeaderSet)==0 ){ + p->showHeader = 1; + } + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); + p->cmOpts = cmOpts; + }else if( cli_strncmp(zMode,"list",n2)==0 ){ + p->mode = MODE_List; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Column); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); + }else if( cli_strncmp(zMode,"html",n2)==0 ){ + p->mode = MODE_Html; + }else if( cli_strncmp(zMode,"tcl",n2)==0 ){ + p->mode = MODE_Tcl; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Space); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); + }else if( cli_strncmp(zMode,"csv",n2)==0 ){ + p->mode = MODE_Csv; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf); + }else if( cli_strncmp(zMode,"tabs",n2)==0 ){ + p->mode = MODE_List; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Tab); + }else if( cli_strncmp(zMode,"insert",n2)==0 ){ + p->mode = MODE_Insert; + set_table_name(p, zTabname ? zTabname : "table"); + if( p->eEscMode==SHELL_ESC_OFF ){ + ShellSetFlag(p, SHFLG_Newlines); + }else{ + ShellClearFlag(p, SHFLG_Newlines); + } + }else if( cli_strncmp(zMode,"quote",n2)==0 ){ + p->mode = MODE_Quote; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); + }else if( cli_strncmp(zMode,"ascii",n2)==0 ){ + p->mode = MODE_Ascii; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Unit); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Record); + }else if( cli_strncmp(zMode,"markdown",n2)==0 ){ + p->mode = MODE_Markdown; + p->cmOpts = cmOpts; + }else if( cli_strncmp(zMode,"table",n2)==0 ){ + p->mode = MODE_Table; + p->cmOpts = cmOpts; + }else if( cli_strncmp(zMode,"box",n2)==0 ){ + p->mode = MODE_Box; + p->cmOpts = cmOpts; + }else if( cli_strncmp(zMode,"count",n2)==0 ){ + p->mode = MODE_Count; + }else if( cli_strncmp(zMode,"off",n2)==0 ){ + p->mode = MODE_Off; + }else if( cli_strncmp(zMode,"json",n2)==0 ){ + p->mode = MODE_Json; + }else{ + eputz("Error: mode should be one of: " + "ascii box column csv html insert json line list markdown " + "qbox quote table tabs tcl\n"); + rc = 1; + } + p->cMode = p->mode; }else #ifndef SQLITE_SHELL_FIDDLE @@ -10016,9 +10314,9 @@ static int do_meta_command(const char *zLine, ShellState *p){ eputz("Usage: .nonce NONCE\n"); rc = 1; }else if( p->zNonce==0 || cli_strcmp(azArg[1],p->zNonce)!=0 ){ - cli_printf(stderr,"line %lld: incorrect nonce: \"%s\"\n", + sqlite3_fprintf(stderr,"line %lld: incorrect nonce: \"%s\"\n", p->lineno, azArg[1]); - cli_exit(1); + exit(1); }else{ p->bSafeMode = 0; return 0; /* Return immediately to bypass the safe mode reset @@ -10029,7 +10327,8 @@ static int do_meta_command(const char *zLine, ShellState *p){ if( c=='n' && cli_strncmp(azArg[0], "nullvalue", n)==0 ){ if( nArg==2 ){ - modeSetStr(&p->mode.spec.zNull, azArg[1]); + sqlite3_snprintf(sizeof(p->nullValue), p->nullValue, + "%.*s", (int)ArraySize(p->nullValue)-1, azArg[1]); }else{ eputz("Usage: .nullvalue STRING\n"); rc = 1; @@ -10044,8 +10343,6 @@ static int do_meta_command(const char *zLine, ShellState *p){ int openMode = SHELL_OPEN_UNSPEC; int openFlags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; - if( p->bSafeMode ) openFlags = SQLITE_OPEN_READONLY; - /* Check for command-line arguments */ for(iName=1; iName<nArg; iName++){ const char *z = azArg[iName]; @@ -10053,10 +10350,10 @@ static int do_meta_command(const char *zLine, ShellState *p){ if( optionMatch(z,"new") ){ newFlag = 1; #ifdef SQLITE_HAVE_ZLIB - }else if( optionMatch(z, "zip") && !p->bSafeMode ){ + }else if( optionMatch(z, "zip") ){ openMode = SHELL_OPEN_ZIPFILE; #endif - }else if( optionMatch(z, "append") && !p->bSafeMode ){ + }else if( optionMatch(z, "append") ){ openMode = SHELL_OPEN_APPENDVFS; }else if( optionMatch(z, "readonly") ){ openFlags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); @@ -10080,11 +10377,11 @@ static int do_meta_command(const char *zLine, ShellState *p){ }else #endif /* !SQLITE_SHELL_FIDDLE */ if( z[0]=='-' ){ - cli_printf(stderr,"unknown option: %s\n", z); + sqlite3_fprintf(stderr,"unknown option: %s\n", z); rc = 1; goto meta_command_exit; }else if( zFN ){ - cli_printf(stderr,"extra argument: \"%s\"\n", z); + sqlite3_fprintf(stderr,"extra argument: \"%s\"\n", z); rc = 1; goto meta_command_exit; }else{ @@ -10135,7 +10432,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ p->pAuxDb->zDbFilename = zNewFilename; open_db(p, OPEN_DB_KEEPALIVE); if( p->db==0 ){ - cli_printf(stderr,"Error: cannot open '%s'\n", zNewFilename); + sqlite3_fprintf(stderr,"Error: cannot open '%s'\n", zNewFilename); sqlite3_free(zNewFilename); }else{ p->pAuxDb->zFreeOnClose = zNewFilename; @@ -10155,7 +10452,145 @@ static int do_meta_command(const char *zLine, ShellState *p){ || (c=='e' && n==5 && cli_strcmp(azArg[0],"excel")==0) || (c=='w' && n==3 && cli_strcmp(azArg[0],"www")==0) ){ - rc = dotCmdOutput(p); + char *zFile = 0; + int i; + int eMode = 0; /* 0: .outout/.once, 'x'=.excel, 'w'=.www */ + int bOnce = 0; /* 0: .output, 1: .once, 2: .excel/.www */ + int bPlain = 0; /* --plain option */ + static const char *zBomUtf8 = "\357\273\277"; + const char *zBom = 0; + + failIfSafeMode(p, "cannot run .%s in safe mode", azArg[0]); + if( c=='e' ){ + eMode = 'x'; + bOnce = 2; + }else if( c=='w' ){ + eMode = 'w'; + bOnce = 2; + }else if( cli_strncmp(azArg[0],"once",n)==0 ){ + bOnce = 1; + } + for(i=1; i<nArg; i++){ + char *z = azArg[i]; + if( z[0]=='-' ){ + if( z[1]=='-' ) z++; + if( cli_strcmp(z,"-bom")==0 ){ + zBom = zBomUtf8; + }else if( cli_strcmp(z,"-plain")==0 ){ + bPlain = 1; + }else if( c=='o' && cli_strcmp(z,"-x")==0 ){ + eMode = 'x'; /* spreadsheet */ + }else if( c=='o' && cli_strcmp(z,"-e")==0 ){ + eMode = 'e'; /* text editor */ + }else if( c=='o' && cli_strcmp(z,"-w")==0 ){ + eMode = 'w'; /* Web browser */ + }else{ + sqlite3_fprintf(p->out, + "ERROR: unknown option: \"%s\". Usage:\n", azArg[i]); + showHelp(p->out, azArg[0]); + rc = 1; + sqlite3_free(zFile); + goto meta_command_exit; + } + }else if( zFile==0 && eMode==0 ){ + if( cli_strcmp(z, "off")==0 ){ +#ifdef _WIN32 + zFile = sqlite3_mprintf("nul"); +#else + zFile = sqlite3_mprintf("/dev/null"); +#endif + }else{ + zFile = sqlite3_mprintf("%s", z); + } + if( zFile && zFile[0]=='|' ){ + while( i+1<nArg ) zFile = sqlite3_mprintf("%z %s", zFile, azArg[++i]); + break; + } + }else{ + sqlite3_fprintf(p->out, + "ERROR: extra parameter: \"%s\". Usage:\n", azArg[i]); + showHelp(p->out, azArg[0]); + rc = 1; + sqlite3_free(zFile); + goto meta_command_exit; + } + } + if( zFile==0 ){ + zFile = sqlite3_mprintf("stdout"); + } + shell_check_oom(zFile); + if( bOnce ){ + p->outCount = 2; + }else{ + p->outCount = 0; + } + output_reset(p); +#ifndef SQLITE_NOHAVE_SYSTEM + if( eMode=='e' || eMode=='x' || eMode=='w' ){ + p->doXdgOpen = 1; + outputModePush(p); + if( eMode=='x' ){ + /* spreadsheet mode. Output as CSV. */ + newTempFile(p, "csv"); + ShellClearFlag(p, SHFLG_Echo); + p->mode = MODE_Csv; + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf); +#ifdef _WIN32 + zBom = zBomUtf8; /* Always include the BOM on Windows, as Excel does + ** not work without it. */ +#endif + }else if( eMode=='w' ){ + /* web-browser mode. */ + newTempFile(p, "html"); + if( !bPlain ) p->mode = MODE_Www; + }else{ + /* text editor mode */ + newTempFile(p, "txt"); + } + sqlite3_free(zFile); + zFile = sqlite3_mprintf("%s", p->zTempFile); + } +#endif /* SQLITE_NOHAVE_SYSTEM */ + shell_check_oom(zFile); + if( zFile[0]=='|' ){ +#ifdef SQLITE_OMIT_POPEN + eputz("Error: pipes are not supported in this OS\n"); + rc = 1; + output_redir(p, stdout); +#else + FILE *pfPipe = sqlite3_popen(zFile + 1, "w"); + if( pfPipe==0 ){ + assert( stderr!=NULL ); + sqlite3_fprintf(stderr,"Error: cannot open pipe \"%s\"\n", zFile + 1); + rc = 1; + }else{ + output_redir(p, pfPipe); + if( zBom ) sqlite3_fputs(zBom, pfPipe); + sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); + } +#endif + }else{ + FILE *pfFile = output_file_open(zFile); + if( pfFile==0 ){ + if( cli_strcmp(zFile,"off")!=0 ){ + assert( stderr!=NULL ); + sqlite3_fprintf(stderr,"Error: cannot write to \"%s\"\n", zFile); + } + rc = 1; + } else { + output_redir(p, pfFile); + if( zBom ) sqlite3_fputs(zBom, pfFile); + if( bPlain && eMode=='w' ){ + sqlite3_fputs( + "<!DOCTYPE html>\n<BODY>\n<PLAINTEXT>\n", + pfFile + ); + } + sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); + } + } + sqlite3_free(zFile); }else #endif /* !defined(SQLITE_SHELL_FIDDLE) */ @@ -10192,7 +10627,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ "SELECT key, quote(value) " "FROM temp.sqlite_parameters;", -1, &pStmt, 0); while( rx==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ - cli_printf(p->out, + sqlite3_fprintf(p->out, "%-*s %s\n", len, sqlite3_column_text(pStmt,0), sqlite3_column_text(pStmt,1)); } @@ -10238,7 +10673,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ rx = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); if( rx!=SQLITE_OK ){ - cli_printf(p->out, "Error: %s\n", sqlite3_errmsg(p->db)); + sqlite3_fprintf(p->out, "Error: %s\n", sqlite3_errmsg(p->db)); sqlite3_finalize(pStmt); pStmt = 0; rc = 1; @@ -10268,10 +10703,10 @@ static int do_meta_command(const char *zLine, ShellState *p){ if( c=='p' && n>=3 && cli_strncmp(azArg[0], "print", n)==0 ){ int i; for(i=1; i<nArg; i++){ - if( i>1 ) cli_puts(" ", p->out); - cli_puts(azArg[i], p->out); + if( i>1 ) sqlite3_fputs(" ", p->out); + sqlite3_fputs(azArg[i], p->out); } - cli_puts("\n", p->out); + sqlite3_fputs("\n", p->out); }else #ifndef SQLITE_OMIT_PROGRESS_CALLBACK @@ -10298,19 +10733,6 @@ static int do_meta_command(const char *zLine, ShellState *p){ p->flgProgress |= SHELL_PROGRESS_ONCE; continue; } - if( cli_strcmp(z,"timeout")==0 ){ - if( i==nArg-1 ){ - dotCmdError(p, i, "missing argument", 0); - return 1; - } - i++; - p->tmProgress = atof(azArg[i]); - if( p->tmProgress>0.0 ){ - p->flgProgress = SHELL_PROGRESS_QUIET|SHELL_PROGRESS_TMOUT; - if( nn==0 ) nn = 100; - } - continue; - } if( cli_strcmp(z,"limit")==0 ){ if( i+1>=nArg ){ eputz("Error: missing argument on --limit\n"); @@ -10321,7 +10743,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ } continue; } - cli_printf(stderr,"Error: unknown option: \"%s\"\n", azArg[i]); + sqlite3_fprintf(stderr,"Error: unknown option: \"%s\"\n", azArg[i]); rc = 1; goto meta_command_exit; }else{ @@ -10365,7 +10787,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ #else p->in = sqlite3_popen(azArg[1]+1, "r"); if( p->in==0 ){ - cli_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); + sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); rc = 1; }else{ rc = process_input(p, "<pipe>"); @@ -10373,12 +10795,10 @@ static int do_meta_command(const char *zLine, ShellState *p){ } #endif }else if( (p->in = openChrSource(azArg[1]))==0 ){ - cli_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); + sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); rc = 1; }else{ - char *zFilename = strdup(azArg[1]); - rc = process_input(p, zFilename); - free(zFilename); + rc = process_input(p, azArg[1]); fclose(p->in); } p->in = inSaved; @@ -10408,7 +10828,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ } rc = sqlite3_open(zSrcFile, &pSrc); if( rc!=SQLITE_OK ){ - cli_printf(stderr,"Error: cannot open \"%s\"\n", zSrcFile); + sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zSrcFile); close_db(pSrc); return 1; } @@ -10446,21 +10866,21 @@ static int do_meta_command(const char *zLine, ShellState *p){ ){ if( nArg==2 ){ if( cli_strcmp(azArg[1], "vm")==0 ){ - p->mode.scanstatsOn = 3; + p->scanstatsOn = 3; }else if( cli_strcmp(azArg[1], "est")==0 ){ - p->mode.scanstatsOn = 2; + p->scanstatsOn = 2; }else{ - p->mode.scanstatsOn = (u8)booleanValue(azArg[1]); + p->scanstatsOn = (u8)booleanValue(azArg[1]); } open_db(p, 0); sqlite3_db_config( - p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->mode.scanstatsOn, (int*)0 + p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->scanstatsOn, (int*)0 ); #if !defined(SQLITE_ENABLE_STMT_SCANSTATUS) eputz("Warning: .scanstats not available in this build.\n"); #elif !defined(SQLITE_ENABLE_BYTECODE_VTAB) - if( p->mode.scanstatsOn==3 ){ + if( p->scanstatsOn==3 ){ eputz("Warning: \".scanstats vm\" not available in this build.\n"); } #endif @@ -10471,6 +10891,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ }else if( c=='s' && cli_strncmp(azArg[0], "schema", n)==0 ){ + ShellText sSelect; ShellState data; char *zErrMsg = 0; const char *zDiv = "("; @@ -10478,27 +10899,22 @@ static int do_meta_command(const char *zLine, ShellState *p){ int iSchema = 0; int bDebug = 0; int bNoSystemTabs = 0; - int bIndent = 0; int ii; - sqlite3_str *pSql; - sqlite3_stmt *pStmt = 0; - + open_db(p, 0); memcpy(&data, p, sizeof(data)); - data.mode.spec.bTitles = QRF_No; - data.mode.eMode = MODE_List; - data.mode.spec.eText = QRF_TEXT_Plain; - data.mode.spec.nCharLimit = 0; - data.mode.spec.zRowSep = "\n"; + data.showHeader = 0; + data.cMode = data.mode = MODE_Semi; + initText(&sSelect); for(ii=1; ii<nArg; ii++){ if( optionMatch(azArg[ii],"indent") ){ - bIndent = 1; + data.cMode = data.mode = MODE_Pretty; }else if( optionMatch(azArg[ii],"debug") ){ bDebug = 1; }else if( optionMatch(azArg[ii],"nosys") ){ bNoSystemTabs = 1; }else if( azArg[ii][0]=='-' ){ - cli_printf(stderr,"Unknown option: \"%s\"\n", azArg[ii]); + sqlite3_fprintf(stderr,"Unknown option: \"%s\"\n", azArg[ii]); rc = 1; goto meta_command_exit; }else if( zName==0 ){ @@ -10515,85 +10931,96 @@ static int do_meta_command(const char *zLine, ShellState *p){ || sqlite3_strlike(zName,"sqlite_temp_master", '\\')==0 || sqlite3_strlike(zName,"sqlite_temp_schema", '\\')==0; if( isSchema ){ - cli_printf(p->out, - "CREATE TABLE %ssqlite_schema (\n" + char *new_argv[2], *new_colv[2]; + new_argv[0] = sqlite3_mprintf( + "CREATE TABLE %s (\n" " type text,\n" " name text,\n" " tbl_name text,\n" " rootpage integer,\n" " sql text\n" - ");\n", - sqlite3_strlike("sqlite_t%",zName,0)==0 ? "temp." : "" - ); + ")", zName); + shell_check_oom(new_argv[0]); + new_argv[1] = 0; + new_colv[0] = "sql"; + new_colv[1] = 0; + callback(&data, 1, new_argv, new_colv); + sqlite3_free(new_argv[0]); } } - rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); - if( rc ){ - shellDatabaseError(p->db); + if( zDiv ){ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(p->db, "SELECT name FROM pragma_database_list", + -1, &pStmt, 0); + if( rc ){ + shellDatabaseError(p->db); + sqlite3_finalize(pStmt); + rc = 1; + goto meta_command_exit; + } + appendText(&sSelect, "SELECT sql FROM", 0); + iSchema = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zDb = (const char*)sqlite3_column_text(pStmt, 0); + char zScNum[30]; + sqlite3_snprintf(sizeof(zScNum), zScNum, "%d", ++iSchema); + appendText(&sSelect, zDiv, 0); + zDiv = " UNION ALL "; + appendText(&sSelect, "SELECT shell_add_schema(sql,", 0); + if( sqlite3_stricmp(zDb, "main")!=0 ){ + appendText(&sSelect, zDb, '\''); + }else{ + appendText(&sSelect, "NULL", 0); + } + appendText(&sSelect, ",name) AS sql, type, tbl_name, name, rowid,", 0); + appendText(&sSelect, zScNum, 0); + appendText(&sSelect, " AS snum, ", 0); + appendText(&sSelect, zDb, '\''); + appendText(&sSelect, " AS sname FROM ", 0); + appendText(&sSelect, zDb, quoteChar(zDb)); + appendText(&sSelect, ".sqlite_schema", 0); + } sqlite3_finalize(pStmt); - - rc = 1; - goto meta_command_exit; - } - pSql = sqlite3_str_new(p->db); - sqlite3_str_appendf(pSql, "SELECT sql FROM", 0); - iSchema = 0; - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - const char *zDb = (const char*)sqlite3_column_text(pStmt, 1); - char zScNum[30]; - sqlite3_snprintf(sizeof(zScNum), zScNum, "%d", ++iSchema); - sqlite3_str_appendall(pSql, zDiv); - zDiv = " UNION ALL "; - if( sqlite3_stricmp(zDb, "main")==0 ){ - sqlite3_str_appendf(pSql, - "SELECT shell_format_schema(shell_add_schema(sql,NULL,name),%d)", - bIndent); - }else{ - sqlite3_str_appendf(pSql, - "SELECT shell_format_schema(shell_add_schema(sql,%Q,name),%d)", - zDb, bIndent); +#ifndef SQLITE_OMIT_INTROSPECTION_PRAGMAS + if( zName ){ + appendText(&sSelect, + " UNION ALL SELECT shell_module_schema(name)," + " 'table', name, name, name, 9e+99, 'main' FROM pragma_module_list", + 0); + } +#endif + appendText(&sSelect, ") WHERE ", 0); + if( zName ){ + char *zQarg = sqlite3_mprintf("%Q", zName); + int bGlob; + shell_check_oom(zQarg); + bGlob = strchr(zName, '*') != 0 || strchr(zName, '?') != 0 || + strchr(zName, '[') != 0; + if( strchr(zName, '.') ){ + appendText(&sSelect, "lower(printf('%s.%s',sname,tbl_name))", 0); + }else{ + appendText(&sSelect, "lower(tbl_name)", 0); + } + appendText(&sSelect, bGlob ? " GLOB " : " LIKE ", 0); + appendText(&sSelect, zQarg, 0); + if( !bGlob ){ + appendText(&sSelect, " ESCAPE '\\' ", 0); + } + appendText(&sSelect, " AND ", 0); + sqlite3_free(zQarg); } - sqlite3_str_appendf(pSql, - " AS sql, type, tbl_name, name, rowid, %d AS snum, %Q as sname", - ++iSchema, zDb); - sqlite3_str_appendf(pSql," FROM \"%w\".sqlite_schema", zDb); - } - sqlite3_finalize(pStmt); -#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) \ - && !defined(SQLITE_OMIT_VIRTUALTABLE) - if( zName ){ - sqlite3_str_appendall(pSql, - " UNION ALL SELECT shell_module_schema(name)," - " 'table', name, name, name, 9e+99, 'main' FROM pragma_module_list"); - } -#endif - sqlite3_str_appendf(pSql, ") WHERE ", 0); - if( zName ){ - int bGlob; - bGlob = strchr(zName, '*') != 0 || strchr(zName, '?') != 0 || - strchr(zName, '[') != 0; - if( strchr(zName, '.') ){ - sqlite3_str_appendall(pSql, "lower(format('%%s.%%s',sname,tbl_name))"); - }else{ - sqlite3_str_appendall(pSql, "lower(tbl_name)"); + if( bNoSystemTabs ){ + appendText(&sSelect, "name NOT LIKE 'sqlite__%%' ESCAPE '_' AND ", 0); } - if( bGlob ){ - sqlite3_str_appendf(pSql, " GLOB %Q AND ", zName); + appendText(&sSelect, "sql IS NOT NULL" + " ORDER BY snum, rowid", 0); + if( bDebug ){ + sqlite3_fprintf(p->out, "SQL: %s;\n", sSelect.zTxt); }else{ - sqlite3_str_appendf(pSql, " LIKE %Q ESCAPE '\\' AND ", zName); + rc = sqlite3_exec(p->db, sSelect.zTxt, callback, &data, &zErrMsg); } + freeText(&sSelect); } - if( bNoSystemTabs ){ - sqlite3_str_appendf(pSql, " name NOT LIKE 'sqlite__%%' ESCAPE '_' AND "); - } - sqlite3_str_appendf(pSql, "sql IS NOT NULL ORDER BY snum, rowid"); - if( bDebug ){ - cli_printf(p->out, "SQL: %s;\n", sqlite3_str_value(pSql)); - }else{ - rc = shell_exec(&data, sqlite3_str_value(pSql), &zErrMsg); - } - sqlite3_str_free(pSql); - if( zErrMsg ){ shellEmitError(zErrMsg); sqlite3_free(zErrMsg); @@ -10649,7 +11076,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ }else{ rc = sqlite3session_attach(pSession->p, azCmd[1]); if( rc ){ - cli_printf(stderr, + sqlite3_fprintf(stderr, "ERROR: sqlite3session_attach() returns %d\n",rc); rc = 0; } @@ -10669,7 +11096,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ if( pSession->p==0 ) goto session_not_open; out = sqlite3_fopen(azCmd[1], "wb"); if( out==0 ){ - cli_printf(stderr,"ERROR: cannot open \"%s\" for writing\n", + sqlite3_fprintf(stderr,"ERROR: cannot open \"%s\" for writing\n", azCmd[1]); }else{ int szChng; @@ -10680,12 +11107,12 @@ static int do_meta_command(const char *zLine, ShellState *p){ rc = sqlite3session_patchset(pSession->p, &szChng, &pChng); } if( rc ){ - cli_printf(stdout, "Error: error code %d\n", rc); + sqlite3_fprintf(stdout, "Error: error code %d\n", rc); rc = 0; } if( pChng && fwrite(pChng, szChng, 1, out)!=1 ){ - cli_printf(stderr, + sqlite3_fprintf(stderr, "ERROR: Failed to write entire %d-byte output\n", szChng); } sqlite3_free(pChng); @@ -10713,7 +11140,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); if( pAuxDb->nSession ){ ii = sqlite3session_enable(pSession->p, ii); - cli_printf(p->out, + sqlite3_fprintf(p->out, "session %s enable flag = %d\n", pSession->zName, ii); } }else @@ -10750,7 +11177,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); if( pAuxDb->nSession ){ ii = sqlite3session_indirect(pSession->p, ii); - cli_printf(p->out, + sqlite3_fprintf(p->out, "session %s indirect flag = %d\n", pSession->zName, ii); } }else @@ -10763,7 +11190,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ if( nCmd!=1 ) goto session_syntax_error; if( pAuxDb->nSession ){ ii = sqlite3session_isempty(pSession->p); - cli_printf(p->out, + sqlite3_fprintf(p->out, "session %s isempty flag = %d\n", pSession->zName, ii); } }else @@ -10773,7 +11200,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ */ if( cli_strcmp(azCmd[0],"list")==0 ){ for(i=0; i<pAuxDb->nSession; i++){ - cli_printf(p->out, "%d %s\n", i, pAuxDb->aSession[i].zName); + sqlite3_fprintf(p->out, "%d %s\n", i, pAuxDb->aSession[i].zName); } }else @@ -10788,19 +11215,19 @@ static int do_meta_command(const char *zLine, ShellState *p){ if( zName[0]==0 ) goto session_syntax_error; for(i=0; i<pAuxDb->nSession; i++){ if( cli_strcmp(pAuxDb->aSession[i].zName,zName)==0 ){ - cli_printf(stderr,"Session \"%s\" already exists\n", zName); + sqlite3_fprintf(stderr,"Session \"%s\" already exists\n", zName); goto meta_command_exit; } } if( pAuxDb->nSession>=ArraySize(pAuxDb->aSession) ){ - cli_printf(stderr, + sqlite3_fprintf(stderr, "Maximum of %d sessions\n", ArraySize(pAuxDb->aSession)); goto meta_command_exit; } pSession = &pAuxDb->aSession[pAuxDb->nSession]; rc = sqlite3session_create(p->db, azCmd[1], &pSession->p); if( rc ){ - cli_printf(stderr,"Cannot open session: error code=%d\n", rc); + sqlite3_fprintf(stderr,"Cannot open session: error code=%d\n", rc); rc = 0; goto meta_command_exit; } @@ -10824,7 +11251,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ int i, v; for(i=1; i<nArg; i++){ v = booleanValue(azArg[i]); - cli_printf(p->out, "%s: %d 0x%x\n", azArg[i], v, v); + sqlite3_fprintf(p->out, "%s: %d 0x%x\n", azArg[i], v, v); } } if( cli_strncmp(azArg[0]+9, "integer", n-9)==0 ){ @@ -10833,7 +11260,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ char zBuf[200]; v = integerValue(azArg[i]); sqlite3_snprintf(sizeof(zBuf),zBuf,"%s: %lld 0x%llx\n", azArg[i],v,v); - cli_puts(zBuf, p->out); + sqlite3_fputs(zBuf, p->out); } } }else @@ -10860,9 +11287,9 @@ static int do_meta_command(const char *zLine, ShellState *p){ bVerbose++; }else { - cli_printf(stderr, + sqlite3_fprintf(stderr, "Unknown option \"%s\" on \"%s\"\n", azArg[i], azArg[0]); - cli_puts("Should be one of: --init -v\n", stderr); + sqlite3_fputs("Should be one of: --init -v\n", stderr); rc = 1; goto meta_command_exit; } @@ -10907,10 +11334,10 @@ static int do_meta_command(const char *zLine, ShellState *p){ if( zAns==0 ) continue; k = 0; if( bVerbose>0 ){ - cli_printf(stdout, "%d: %s %s\n", tno, zOp, zSql); + sqlite3_fprintf(stdout, "%d: %s %s\n", tno, zOp, zSql); } if( cli_strcmp(zOp,"memo")==0 ){ - cli_printf(p->out, "%s\n", zSql); + sqlite3_fprintf(p->out, "%s\n", zSql); }else if( cli_strcmp(zOp,"run")==0 ){ char *zErrMsg = 0; @@ -10919,22 +11346,22 @@ static int do_meta_command(const char *zLine, ShellState *p){ rc = sqlite3_exec(p->db, zSql, captureOutputCallback, &str, &zErrMsg); nTest++; if( bVerbose ){ - cli_printf(p->out, "Result: %s\n", str.zTxt); + sqlite3_fprintf(p->out, "Result: %s\n", str.zTxt); } if( rc || zErrMsg ){ nErr++; rc = 1; - cli_printf(p->out, "%d: error-code-%d: %s\n", tno, rc,zErrMsg); + sqlite3_fprintf(p->out, "%d: error-code-%d: %s\n", tno, rc,zErrMsg); sqlite3_free(zErrMsg); }else if( cli_strcmp(zAns,str.zTxt)!=0 ){ nErr++; rc = 1; - cli_printf(p->out, "%d: Expected: [%s]\n", tno, zAns); - cli_printf(p->out, "%d: Got: [%s]\n", tno, str.zTxt); + sqlite3_fprintf(p->out, "%d: Expected: [%s]\n", tno, zAns); + sqlite3_fprintf(p->out, "%d: Got: [%s]\n", tno, str.zTxt); } } else{ - cli_printf(stderr, + sqlite3_fprintf(stderr, "Unknown operation \"%s\" on selftest line %d\n", zOp, tno); rc = 1; break; @@ -10943,7 +11370,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ sqlite3_finalize(pStmt); } /* End loop over k */ freeText(&str); - cli_printf(p->out, "%d errors out of %d tests\n", nErr, nTest); + sqlite3_fprintf(p->out, "%d errors out of %d tests\n", nErr, nTest); }else if( c=='s' && cli_strncmp(azArg[0], "separator", n)==0 ){ @@ -10952,10 +11379,12 @@ static int do_meta_command(const char *zLine, ShellState *p){ rc = 1; } if( nArg>=2 ){ - modeSetStr(&p->mode.spec.zColumnSep, azArg[1]); + sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, + "%.*s", (int)ArraySize(p->colSeparator)-1, azArg[1]); } if( nArg>=3 ){ - modeSetStr(&p->mode.spec.zRowSep,azArg[2]); + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, + "%.*s", (int)ArraySize(p->rowSeparator)-1, azArg[2]); } }else @@ -10989,7 +11418,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ bDebug = 1; }else { - cli_printf(stderr, + sqlite3_fprintf(stderr, "Unknown option \"%s\" on \"%s\"\n", azArg[i], azArg[0]); showHelp(p->out, azArg[0]); rc = 1; @@ -11068,7 +11497,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ freeText(&sQuery); freeText(&sSql); if( bDebug ){ - cli_printf(p->out, "%s\n", zSql); + sqlite3_fprintf(p->out, "%s\n", zSql); }else{ shell_exec(p, zSql, 0); } @@ -11098,7 +11527,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ "' OR ') as query, tname from tabcols group by tname)" , zRevText); shell_check_oom(zRevText); - if( bDebug ) cli_printf(p->out, "%s\n", zRevText); + if( bDebug ) sqlite3_fprintf(p->out, "%s\n", zRevText); lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0); if( lrc!=SQLITE_OK ){ /* assert(lrc==SQLITE_NOMEM); // might also be SQLITE_ERROR if the @@ -11111,7 +11540,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ const char *zGenQuery = (char*)sqlite3_column_text(pStmt,0); sqlite3_stmt *pCheckStmt; lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0); - if( bDebug ) cli_printf(p->out, "%s\n", zGenQuery); + if( bDebug ) sqlite3_fprintf(p->out, "%s\n", zGenQuery); if( lrc!=SQLITE_OK ){ rc = 1; }else{ @@ -11119,7 +11548,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ double countIrreversible = sqlite3_column_double(pCheckStmt, 0); if( countIrreversible>0 ){ int sz = (int)(countIrreversible + 0.5); - cli_printf(stderr, + sqlite3_fprintf(stderr, "Digest includes %d invalidly encoded text field%s.\n", sz, (sz>1)? "s": ""); } @@ -11158,7 +11587,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ x = zCmd!=0 ? system(zCmd) : 1; /*consoleRenewSetup();*/ sqlite3_free(zCmd); - if( x ) cli_printf(stderr,"System command returns %d\n", x); + if( x ) sqlite3_fprintf(stderr,"System command returns %d\n", x); }else #endif /* !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) */ @@ -11171,51 +11600,48 @@ static int do_meta_command(const char *zLine, ShellState *p){ rc = 1; goto meta_command_exit; } - cli_printf(p->out, "%12.12s: %s\n","echo", - azBool[(p->mode.mFlags & MFLG_ECHO)!=0]); - cli_printf(p->out, "%12.12s: %s\n","eqp", azBool[p->mode.autoEQP&3]); - cli_printf(p->out, "%12.12s: %s\n","explain", - p->mode.autoExplain ? "auto" : "off"); - cli_printf(p->out, "%12.12s: %s\n","headers", - azBool[p->mode.spec.bTitles==QRF_Yes]); - if( p->mode.spec.eStyle==QRF_STYLE_Column - || p->mode.spec.eStyle==QRF_STYLE_Box - || p->mode.spec.eStyle==QRF_STYLE_Table - || p->mode.spec.eStyle==QRF_STYLE_Markdown + sqlite3_fprintf(p->out, "%12.12s: %s\n","echo", + azBool[ShellHasFlag(p, SHFLG_Echo)]); + sqlite3_fprintf(p->out, "%12.12s: %s\n","eqp", azBool[p->autoEQP&3]); + sqlite3_fprintf(p->out, "%12.12s: %s\n","explain", + p->mode==MODE_Explain ? "on" : p->autoExplain ? "auto" : "off"); + sqlite3_fprintf(p->out, "%12.12s: %s\n","headers", + azBool[p->showHeader!=0]); + if( p->mode==MODE_Column + || (p->mode>=MODE_Markdown && p->mode<=MODE_Box) ){ - cli_printf(p->out, + sqlite3_fprintf(p->out, "%12.12s: %s --wrap %d --wordwrap %s --%squote\n", "mode", - aModeInfo[p->mode.eMode].zName, p->mode.spec.nWrap, - p->mode.spec.bWordWrap==QRF_Yes ? "on" : "off", - p->mode.spec.eText==QRF_TEXT_Sql ? "" : "no"); + modeDescr[p->mode], p->cmOpts.iWrap, + p->cmOpts.bWordWrap ? "on" : "off", + p->cmOpts.bQuote ? "" : "no"); }else{ - cli_printf(p->out, "%12.12s: %s\n","mode", - aModeInfo[p->mode.eMode].zName); + sqlite3_fprintf(p->out, "%12.12s: %s\n","mode", modeDescr[p->mode]); } - cli_printf(p->out, "%12.12s: ", "nullvalue"); - output_c_string(p->out, p->mode.spec.zNull); - cli_puts("\n", p->out); - cli_printf(p->out, "%12.12s: %s\n","output", + sqlite3_fprintf(p->out, "%12.12s: ", "nullvalue"); + output_c_string(p->out, p->nullValue); + sqlite3_fputs("\n", p->out); + sqlite3_fprintf(p->out, "%12.12s: %s\n","output", strlen30(p->outfile) ? p->outfile : "stdout"); - cli_printf(p->out, "%12.12s: ", "colseparator"); - output_c_string(p->out, p->mode.spec.zColumnSep); - cli_puts("\n", p->out); - cli_printf(p->out, "%12.12s: ", "rowseparator"); - output_c_string(p->out, p->mode.spec.zRowSep); - cli_puts("\n", p->out); + sqlite3_fprintf(p->out, "%12.12s: ", "colseparator"); + output_c_string(p->out, p->colSeparator); + sqlite3_fputs("\n", p->out); + sqlite3_fprintf(p->out, "%12.12s: ", "rowseparator"); + output_c_string(p->out, p->rowSeparator); + sqlite3_fputs("\n", p->out); switch( p->statsOn ){ case 0: zOut = "off"; break; default: zOut = "on"; break; case 2: zOut = "stmt"; break; case 3: zOut = "vmstep"; break; } - cli_printf(p->out, "%12.12s: %s\n","stats", zOut); - cli_printf(p->out, "%12.12s: ", "width"); - for(i=0; i<p->mode.spec.nWidth; i++){ - cli_printf(p->out, "%d ", (int)p->mode.spec.aWidth[i]); + sqlite3_fprintf(p->out, "%12.12s: %s\n","stats", zOut); + sqlite3_fprintf(p->out, "%12.12s: ", "width"); + for (i=0;i<p->nWidth;i++) { + sqlite3_fprintf(p->out, "%d ", p->colWidth[i]); } - cli_puts("\n", p->out); - cli_printf(p->out, "%12.12s: %s\n", "filename", + sqlite3_fputs("\n", p->out); + sqlite3_fprintf(p->out, "%12.12s: %s\n", "filename", p->pAuxDb->zDbFilename ? p->pAuxDb->zDbFilename : ""); }else @@ -11241,9 +11667,11 @@ static int do_meta_command(const char *zLine, ShellState *p){ || cli_strncmp(azArg[0], "indexes", n)==0) ) ){ sqlite3_stmt *pStmt; - sqlite3_str *pSql; - const char *zPattern = nArg>1 ? azArg[1] : 0; - + char **azResult; + int nRow, nAlloc; + int ii; + ShellText s; + initText(&s); open_db(p, 0); rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); if( rc ){ @@ -11260,53 +11688,103 @@ static int do_meta_command(const char *zLine, ShellState *p){ sqlite3_finalize(pStmt); goto meta_command_exit; } - pSql = sqlite3_str_new(p->db); - while( sqlite3_step(pStmt)==SQLITE_ROW ){ + for(ii=0; sqlite3_step(pStmt)==SQLITE_ROW; ii++){ const char *zDbName = (const char*)sqlite3_column_text(pStmt, 1); if( zDbName==0 ) continue; - if( sqlite3_str_length(pSql) ){ - sqlite3_str_appendall(pSql, " UNION ALL "); - } + if( s.zTxt && s.zTxt[0] ) appendText(&s, " UNION ALL ", 0); if( sqlite3_stricmp(zDbName, "main")==0 ){ - sqlite3_str_appendall(pSql, "SELECT name FROM "); + appendText(&s, "SELECT name FROM ", 0); }else{ - sqlite3_str_appendf(pSql, "SELECT %Q||'.'||name FROM ", zDbName); + appendText(&s, "SELECT ", 0); + appendText(&s, zDbName, '\''); + appendText(&s, "||'.'||name FROM ", 0); } - sqlite3_str_appendf(pSql, "\"%w\".sqlite_schema", zDbName); + appendText(&s, zDbName, '"'); + appendText(&s, ".sqlite_schema ", 0); if( c=='t' ){ - sqlite3_str_appendf(pSql, - " WHERE type IN ('table','view')" - " AND name NOT LIKE 'sqlite__%%' ESCAPE '_'" - ); - if( zPattern ){ - sqlite3_str_appendf(pSql," AND name LIKE %Q", zPattern); - } + appendText(&s," WHERE type IN ('table','view')" + " AND name NOT LIKE 'sqlite__%' ESCAPE '_'" + " AND name LIKE ?1", 0); }else{ - sqlite3_str_appendf(pSql, " WHERE type='index'"); - if( zPattern ){ - sqlite3_str_appendf(pSql," AND tbl_name LIKE %Q", zPattern); - } + appendText(&s," WHERE type='index'" + " AND tbl_name LIKE ?1", 0); } } rc = sqlite3_finalize(pStmt); if( rc==SQLITE_OK ){ - sqlite3_str_appendall(pSql, " ORDER BY 1"); + appendText(&s, " ORDER BY 1", 0); + rc = sqlite3_prepare_v2(p->db, s.zTxt, -1, &pStmt, 0); } - - /* Run the SQL statement in "split" mode. */ - modePush(p); - modeChange(p, MODE_Split); - shell_exec(p, sqlite3_str_value(pSql), 0); - sqlite3_str_free(pSql); - modePop(p); + freeText(&s); if( rc ) return shellDatabaseError(p->db); + + /* Run the SQL statement prepared by the above block. Store the results + ** as an array of nul-terminated strings in azResult[]. */ + nRow = nAlloc = 0; + azResult = 0; + if( nArg>1 ){ + sqlite3_bind_text(pStmt, 1, azArg[1], -1, SQLITE_TRANSIENT); + }else{ + sqlite3_bind_text(pStmt, 1, "%", -1, SQLITE_STATIC); + } + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + if( nRow>=nAlloc ){ + char **azNew; + sqlite3_int64 n2 = 2*(sqlite3_int64)nAlloc + 10; + azNew = sqlite3_realloc64(azResult, sizeof(azResult[0])*n2); + shell_check_oom(azNew); + nAlloc = (int)n2; + azResult = azNew; + } + azResult[nRow] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); + shell_check_oom(azResult[nRow]); + nRow++; + } + if( sqlite3_finalize(pStmt)!=SQLITE_OK ){ + rc = shellDatabaseError(p->db); + } + + /* Pretty-print the contents of array azResult[] to the output */ + if( rc==0 && nRow>0 ){ + int len, maxlen = 0; + int i, j; + int nPrintCol, nPrintRow; + for(i=0; i<nRow; i++){ + len = strlen30(azResult[i]); + if( len>maxlen ) maxlen = len; + } + nPrintCol = 80/(maxlen+2); + if( nPrintCol<1 ) nPrintCol = 1; + nPrintRow = (nRow + nPrintCol - 1)/nPrintCol; + for(i=0; i<nPrintRow; i++){ + for(j=i; j<nRow; j+=nPrintRow){ + char *zSp = j<nPrintRow ? "" : " "; + sqlite3_fprintf(p->out, + "%s%-*s", zSp, maxlen, azResult[j] ? azResult[j]:""); + } + sqlite3_fputs("\n", p->out); + } + } + + for(ii=0; ii<nRow; ii++) sqlite3_free(azResult[ii]); + sqlite3_free(azResult); }else - /* Set the p->zTestcase name and begin redirecting output into - ** the cli_output_capture sqlite3_str */ +#ifndef SQLITE_SHELL_FIDDLE + /* Begin redirecting output to the file "testcase-out.txt" */ if( c=='t' && cli_strcmp(azArg[0],"testcase")==0 ){ - rc = dotCmdTestcase(p); + output_reset(p); + p->out = output_file_open("testcase-out.txt"); + if( p->out==0 ){ + eputz("Error: cannot open 'testcase-out.txt'\n"); + } + if( nArg>=2 ){ + sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "%s", azArg[1]); + }else{ + sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "?"); + } }else +#endif /* !defined(SQLITE_SHELL_FIDDLE) */ #ifndef SQLITE_UNTESTABLE if( c=='t' && n>=8 && cli_strncmp(azArg[0], "testctrl", n)==0 ){ @@ -11359,10 +11837,10 @@ static int do_meta_command(const char *zLine, ShellState *p){ /* --help lists all test-controls */ if( cli_strcmp(zCmd,"help")==0 ){ - cli_puts("Available test-controls:\n", p->out); + sqlite3_fputs("Available test-controls:\n", p->out); for(i=0; i<ArraySize(aCtrl); i++){ if( aCtrl[i].unSafe && !ShellHasFlag(p,SHFLG_TestingMode) ) continue; - cli_printf(p->out, " .testctrl %s %s\n", + sqlite3_fprintf(p->out, " .testctrl %s %s\n", aCtrl[i].zCtrlName, aCtrl[i].zUsage); } rc = 1; @@ -11379,7 +11857,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ testctrl = aCtrl[i].ctrlCode; iCtrl = i; }else{ - cli_printf(stderr,"Error: ambiguous test-control: \"%s\"\n" + sqlite3_fprintf(stderr,"Error: ambiguous test-control: \"%s\"\n" "Use \".testctrl --help\" for help\n", zCmd); rc = 1; goto meta_command_exit; @@ -11387,7 +11865,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ } } if( testctrl<0 ){ - cli_printf(stderr,"Error: unknown test-control: %s\n" + sqlite3_fprintf(stderr,"Error: unknown test-control: %s\n" "Use \".testctrl --help\" for help\n", zCmd); }else{ switch(testctrl){ @@ -11467,13 +11945,13 @@ static int do_meta_command(const char *zLine, ShellState *p){ if( sqlite3_stricmp(zLabel, aLabel[jj].zLabel)==0 ) break; } if( jj>=ArraySize(aLabel) ){ - cli_printf(stderr, + sqlite3_fprintf(stderr, "Error: no such optimization: \"%s\"\n", zLabel); - cli_puts("Should be one of:", stderr); + sqlite3_fputs("Should be one of:", stderr); for(jj=0; jj<ArraySize(aLabel); jj++){ - cli_printf(stderr," %s", aLabel[jj].zLabel); + sqlite3_fprintf(stderr," %s", aLabel[jj].zLabel); } - cli_puts("\n", stderr); + sqlite3_fputs("\n", stderr); rc = 1; goto meta_command_exit; } @@ -11491,23 +11969,23 @@ static int do_meta_command(const char *zLine, ShellState *p){ if( m & newOpt ) nOff++; } if( nOff<12 ){ - cli_puts("+All", p->out); + sqlite3_fputs("+All", p->out); for(ii=0; ii<ArraySize(aLabel); ii++){ if( !aLabel[ii].bDsply ) continue; if( (newOpt & aLabel[ii].mask)!=0 ){ - cli_printf(p->out, " -%s", aLabel[ii].zLabel); + sqlite3_fprintf(p->out, " -%s", aLabel[ii].zLabel); } } }else{ - cli_puts("-All", p->out); + sqlite3_fputs("-All", p->out); for(ii=0; ii<ArraySize(aLabel); ii++){ if( !aLabel[ii].bDsply ) continue; if( (newOpt & aLabel[ii].mask)==0 ){ - cli_printf(p->out, " +%s", aLabel[ii].zLabel); + sqlite3_fprintf(p->out, " +%s", aLabel[ii].zLabel); } } } - cli_puts("\n", p->out); + sqlite3_fputs("\n", p->out); rc2 = isOk = 3; break; } @@ -11547,7 +12025,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ sqlite3 *db; if( ii==0 && cli_strcmp(azArg[2],"random")==0 ){ sqlite3_randomness(sizeof(ii),&ii); - cli_printf(stdout, "-- random seed: %d\n", ii); + sqlite3_fprintf(stdout, "-- random seed: %d\n", ii); } if( nArg==3 ){ db = 0; @@ -11600,7 +12078,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ case SQLITE_TESTCTRL_SEEK_COUNT: { u64 x = 0; rc2 = sqlite3_test_control(testctrl, p->db, &x); - cli_printf(p->out, "%llu\n", x); + sqlite3_fprintf(p->out, "%llu\n", x); isOk = 3; break; } @@ -11631,11 +12109,11 @@ static int do_meta_command(const char *zLine, ShellState *p){ int val = 0; rc2 = sqlite3_test_control(testctrl, -id, &val); if( rc2!=SQLITE_OK ) break; - if( id>1 ) cli_puts(" ", p->out); - cli_printf(p->out, "%d: %d", id, val); + if( id>1 ) sqlite3_fputs(" ", p->out); + sqlite3_fprintf(p->out, "%d: %d", id, val); id++; } - if( id>1 ) cli_puts("\n", p->out); + if( id>1 ) sqlite3_fputs("\n", p->out); isOk = 3; } break; @@ -11674,7 +12152,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ int ii, jj, x; int *aOp; if( nArg!=4 ){ - cli_printf(stderr, + sqlite3_fprintf(stderr, "ERROR - should be: \".testctrl bitvec_test SIZE INT-ARRAY\"\n" ); rc = 1; @@ -11697,7 +12175,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ } aOp[jj] = x; x = sqlite3_test_control(testctrl, iSize, aOp); - cli_printf(p->out, "result: %d\n", x); + sqlite3_fprintf(p->out, "result: %d\n", x); free(aOp); break; } @@ -11720,21 +12198,21 @@ static int do_meta_command(const char *zLine, ShellState *p){ faultsim_state.nHit = 0; sqlite3_test_control(testctrl, faultsim_callback); }else if( cli_strcmp(z,"status")==0 ){ - cli_printf(p->out, "faultsim.iId: %d\n", + sqlite3_fprintf(p->out, "faultsim.iId: %d\n", faultsim_state.iId); - cli_printf(p->out, "faultsim.iErr: %d\n", + sqlite3_fprintf(p->out, "faultsim.iErr: %d\n", faultsim_state.iErr); - cli_printf(p->out, "faultsim.iCnt: %d\n", + sqlite3_fprintf(p->out, "faultsim.iCnt: %d\n", faultsim_state.iCnt); - cli_printf(p->out, "faultsim.nHit: %d\n", + sqlite3_fprintf(p->out, "faultsim.nHit: %d\n", faultsim_state.nHit); - cli_printf(p->out, "faultsim.iInterval: %d\n", + sqlite3_fprintf(p->out, "faultsim.iInterval: %d\n", faultsim_state.iInterval); - cli_printf(p->out, "faultsim.eVerbose: %d\n", + sqlite3_fprintf(p->out, "faultsim.eVerbose: %d\n", faultsim_state.eVerbose); - cli_printf(p->out, "faultsim.nRepeat: %d\n", + sqlite3_fprintf(p->out, "faultsim.nRepeat: %d\n", faultsim_state.nRepeat); - cli_printf(p->out, "faultsim.nSkip: %d\n", + sqlite3_fprintf(p->out, "faultsim.nSkip: %d\n", faultsim_state.nSkip); }else if( cli_strcmp(z,"-v")==0 ){ if( faultsim_state.eVerbose<2 ) faultsim_state.eVerbose++; @@ -11753,7 +12231,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ }else if( cli_strcmp(z,"-?")==0 || sqlite3_strglob("*help*",z)==0){ bShowHelp = 1; }else{ - cli_printf(stderr, + sqlite3_fprintf(stderr, "Unrecognized fault_install argument: \"%s\"\n", azArg[kk]); rc = 1; @@ -11762,7 +12240,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ } } if( bShowHelp ){ - cli_puts( + sqlite3_fputs( "Usage: .testctrl fault_install ARGS\n" "Possible arguments:\n" " off Disable faultsim\n" @@ -11784,13 +12262,13 @@ static int do_meta_command(const char *zLine, ShellState *p){ } } if( isOk==0 && iCtrl>=0 ){ - cli_printf(p->out, + sqlite3_fprintf(p->out, "Usage: .testctrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage); rc = 1; }else if( isOk==1 ){ - cli_printf(p->out, "%d\n", rc2); + sqlite3_fprintf(p->out, "%d\n", rc2); }else if( isOk==2 ){ - cli_printf(p->out, "0x%08x\n", rc2); + sqlite3_fprintf(p->out, "0x%08x\n", rc2); } }else #endif /* !defined(SQLITE_UNTESTABLE) */ @@ -11802,17 +12280,13 @@ static int do_meta_command(const char *zLine, ShellState *p){ if( c=='t' && n>=5 && cli_strncmp(azArg[0], "timer", n)==0 ){ if( nArg==2 ){ - if( cli_strcmp(azArg[1],"once")==0 ){ - p->enableTimer = 1; - }else{ - p->enableTimer = 2*booleanValue(azArg[1]); - } - if( p->enableTimer && !HAS_TIMER ){ + enableTimer = booleanValue(azArg[1]); + if( enableTimer && !HAS_TIMER ){ eputz("Error: timer not available on this system.\n"); - p->enableTimer = 0; + enableTimer = 0; } }else{ - eputz("Usage: .timer on|off|once\n"); + eputz("Usage: .timer on|off\n"); rc = 1; } }else @@ -11849,13 +12323,13 @@ static int do_meta_command(const char *zLine, ShellState *p){ mType |= SQLITE_TRACE_CLOSE; } else { - cli_printf(stderr,"Unknown option \"%s\" on \".trace\"\n", z); + sqlite3_fprintf(stderr,"Unknown option \"%s\" on \".trace\"\n", z); rc = 1; goto meta_command_exit; } }else{ output_file_close(p->traceOut); - p->traceOut = output_file_open(p, z); + p->traceOut = output_file_open(z); } } if( p->traceOut==0 ){ @@ -11894,21 +12368,21 @@ static int do_meta_command(const char *zLine, ShellState *p){ if( c=='v' && cli_strncmp(azArg[0], "version", n)==0 ){ char *zPtrSz = sizeof(void*)==8 ? "64-bit" : "32-bit"; - cli_printf(p->out, "SQLite %s %s\n" /*extra-version-info*/, + sqlite3_fprintf(p->out, "SQLite %s %s\n" /*extra-version-info*/, sqlite3_libversion(), sqlite3_sourceid()); #if SQLITE_HAVE_ZLIB - cli_printf(p->out, "zlib version %s\n", zlibVersion()); + sqlite3_fprintf(p->out, "zlib version %s\n", zlibVersion()); #endif #define CTIMEOPT_VAL_(opt) #opt #define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt) #if defined(__clang__) && defined(__clang_major__) - cli_printf(p->out, "clang-" CTIMEOPT_VAL(__clang_major__) "." + sqlite3_fprintf(p->out, "clang-" CTIMEOPT_VAL(__clang_major__) "." CTIMEOPT_VAL(__clang_minor__) "." CTIMEOPT_VAL(__clang_patchlevel__) " (%s)\n", zPtrSz); #elif defined(_MSC_VER) - cli_printf(p->out, "msvc-" CTIMEOPT_VAL(_MSC_VER) " (%s)\n", zPtrSz); + sqlite3_fprintf(p->out, "msvc-" CTIMEOPT_VAL(_MSC_VER) " (%s)\n", zPtrSz); #elif defined(__GNUC__) && defined(__VERSION__) - cli_printf(p->out, "gcc-" __VERSION__ " (%s)\n", zPtrSz); + sqlite3_fprintf(p->out, "gcc-" __VERSION__ " (%s)\n", zPtrSz); #endif }else @@ -11918,10 +12392,10 @@ static int do_meta_command(const char *zLine, ShellState *p){ if( p->db ){ sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFS_POINTER, &pVfs); if( pVfs ){ - cli_printf(p->out, "vfs.zName = \"%s\"\n", pVfs->zName); - cli_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); - cli_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); - cli_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); + sqlite3_fprintf(p->out, "vfs.zName = \"%s\"\n", pVfs->zName); + sqlite3_fprintf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); + sqlite3_fprintf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); + sqlite3_fprintf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); } } }else @@ -11933,13 +12407,13 @@ static int do_meta_command(const char *zLine, ShellState *p){ sqlite3_file_control(p->db, "main", SQLITE_FCNTL_VFS_POINTER, &pCurrent); } for(pVfs=sqlite3_vfs_find(0); pVfs; pVfs=pVfs->pNext){ - cli_printf(p->out, "vfs.zName = \"%s\"%s\n", pVfs->zName, + sqlite3_fprintf(p->out, "vfs.zName = \"%s\"%s\n", pVfs->zName, pVfs==pCurrent ? " <--- CURRENT" : ""); - cli_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); - cli_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); - cli_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); + sqlite3_fprintf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); + sqlite3_fprintf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); + sqlite3_fprintf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); if( pVfs->pNext ){ - cli_puts("-----------------------------------\n", p->out); + sqlite3_fputs("-----------------------------------\n", p->out); } } }else @@ -11950,7 +12424,7 @@ static int do_meta_command(const char *zLine, ShellState *p){ if( p->db ){ sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFSNAME, &zVfsName); if( zVfsName ){ - cli_printf(p->out, "%s\n", zVfsName); + sqlite3_fprintf(p->out, "%s\n", zVfsName); sqlite3_free(zVfsName); } } @@ -11963,31 +12437,31 @@ static int do_meta_command(const char *zLine, ShellState *p){ if( c=='w' && cli_strncmp(azArg[0], "width", n)==0 ){ int j; - p->mode.spec.nWidth = nArg-1; - p->mode.spec.aWidth = realloc(p->mode.spec.aWidth, - (p->mode.spec.nWidth+1)*sizeof(short int)); - shell_check_oom(p->mode.spec.aWidth); + assert( nArg<=ArraySize(azArg) ); + p->nWidth = nArg-1; + p->colWidth = realloc(p->colWidth, (p->nWidth+1)*sizeof(int)*2); + if( p->colWidth==0 && p->nWidth>0 ) shell_out_of_memory(); + if( p->nWidth ) p->actualWidth = &p->colWidth[p->nWidth]; for(j=1; j<nArg; j++){ i64 w = integerValue(azArg[j]); - if( w < -QRF_MAX_WIDTH ) w = -QRF_MAX_WIDTH; - if( w > QRF_MAX_WIDTH ) w = QRF_MAX_WIDTH; - p->mode.spec.aWidth[j-1] = (short int)w; + if( w < -30000 ) w = -30000; + if( w > +30000 ) w = +30000; + p->colWidth[j-1] = (int)w; } }else { - cli_printf(stderr,"Error: unknown command or invalid arguments: " + sqlite3_fprintf(stderr,"Error: unknown command or invalid arguments: " " \"%s\". Enter \".help\" for help\n", azArg[0]); rc = 1; } meta_command_exit: - if( p->nPopOutput ){ - p->nPopOutput--; - if( p->nPopOutput==0 ) output_reset(p); + if( p->outCount ){ + p->outCount--; + if( p->outCount==0 ) output_reset(p); } p->bSafeMode = p->bSafeModePersist; - p->dot.nArg = 0; return rc; } @@ -12224,9 +12698,9 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ open_db(p, 0); if( ShellHasFlag(p,SHFLG_Backslash) ) resolve_backslashes(zSql); if( p->flgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0; - BEGIN_TIMER(p); + BEGIN_TIMER; rc = shell_exec(p, zSql, &zErrMsg); - END_TIMER(p); + END_TIMER(p->out); if( rc || zErrMsg ){ char zPrefix[100]; const char *zErrorTail; @@ -12250,7 +12724,7 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ }else{ sqlite3_snprintf(sizeof(zPrefix), zPrefix, "%s:", zErrorType); } - cli_printf(stderr,"%s %s\n", zPrefix, zErrorTail); + sqlite3_fprintf(stderr,"%s %s\n", zPrefix, zErrorTail); sqlite3_free(zErrMsg); zErrMsg = 0; return 1; @@ -12259,7 +12733,7 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ sqlite3_snprintf(sizeof(zLineBuf), zLineBuf, "changes: %lld total_changes: %lld", sqlite3_changes64(p->db), sqlite3_total_changes64(p->db)); - cli_printf(p->out, "%s\n", zLineBuf); + sqlite3_fprintf(p->out, "%s\n", zLineBuf); } if( doAutoDetectRestore(p, zSql) ) return 1; @@ -12267,8 +12741,8 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ } static void echo_group_input(ShellState *p, const char *zDo){ - if( p->mode.mFlags & MFLG_ECHO ){ - cli_printf(p->out, "%s\n", zDo); + if( ShellHasFlag(p, SHFLG_Echo) ){ + sqlite3_fprintf(p->out, "%s\n", zDo); fflush(p->out); } } @@ -12325,19 +12799,14 @@ static int process_input(ShellState *p, const char *zSrc){ int errCnt = 0; /* Number of errors seen */ i64 startline = 0; /* Line number for start of current input */ QuickScanState qss = QSS_Start; /* Accumulated line status (so far) */ - const char *saved_zInFile; /* Prior value of p->zInFile */ - i64 saved_lineno; /* Prior value of p->lineno */ if( p->inputNesting==MAX_INPUT_NESTING ){ /* This will be more informative in a later version. */ - cli_printf(stderr,"%s: Input nesting limit (%d) reached at line %lld." + sqlite3_fprintf(stderr,"%s: Input nesting limit (%d) reached at line %lld." " Check recursion.\n", zSrc, MAX_INPUT_NESTING, p->lineno); return 1; } ++p->inputNesting; - saved_zInFile = p->zInFile; - p->zInFile = zSrc; - saved_lineno = p->lineno; p->lineno = 0; CONTINUE_PROMPT_RESET; while( errCnt==0 || !bail_on_error || (p->in==0 && stdin_is_interactive) ){ @@ -12345,7 +12814,7 @@ static int process_input(ShellState *p, const char *zSrc){ zLine = one_input_line(p->in, zLine, nSql>0); if( zLine==0 ){ /* End of input */ - if( p->in==0 && stdin_is_interactive ) cli_puts("\n", p->out); + if( p->in==0 && stdin_is_interactive ) sqlite3_fputs("\n", p->out); break; } if( seenInterrupt ){ @@ -12402,7 +12871,7 @@ static int process_input(ShellState *p, const char *zSrc){ if( nSql>0x7fff0000 ){ char zSize[100]; sqlite3_snprintf(sizeof(zSize),zSize,"%,lld",nSql); - cli_printf(stderr, "%s:%lld: Input SQL is too big: %s bytes\n", + sqlite3_fprintf(stderr, "%s:%lld: Input SQL is too big: %s bytes\n", zSrc, startline, zSize); nSql = 0; errCnt++; @@ -12412,16 +12881,12 @@ static int process_input(ShellState *p, const char *zSrc){ errCnt += runOneSqlLine(p, zSql, p->in, startline); CONTINUE_PROMPT_RESET; nSql = 0; - if( p->nPopOutput ){ + if( p->outCount ){ output_reset(p); - p->nPopOutput = 0; + p->outCount = 0; }else{ clearTempFile(p); } - if( p->nPopMode ){ - modePop(p); - p->nPopMode = 0; - } p->bSafeMode = p->bSafeModePersist; qss = QSS_Start; }else if( nSql && QSS_PLAINWHITE(qss) ){ @@ -12439,8 +12904,6 @@ static int process_input(ShellState *p, const char *zSrc){ free(zSql); free(zLine); --p->inputNesting; - p->zInFile = saved_zInFile; - p->lineno = saved_lineno; return errCnt>0; } @@ -12601,13 +13064,13 @@ static void process_sqliterc( p->in = sqliterc ? sqlite3_fopen(sqliterc,"rb") : 0; if( p->in ){ if( stdin_is_interactive ){ - cli_printf(stderr,"-- Loading resources from %s\n", sqliterc); + sqlite3_fprintf(stderr,"-- Loading resources from %s\n", sqliterc); } - if( process_input(p, sqliterc) && bail_on_error ) cli_exit(1); + if( process_input(p, sqliterc) && bail_on_error ) exit(1); fclose(p->in); }else if( sqliterc_override!=0 ){ - cli_printf(stderr,"cannot open: \"%s\"\n", sqliterc); - if( bail_on_error ) cli_exit(1); + sqlite3_fprintf(stderr,"cannot open: \"%s\"\n", sqliterc); + if( bail_on_error ) exit(1); } p->in = inSaved; p->lineno = savedLineno; @@ -12629,8 +13092,8 @@ static const char zOptions[] = " -bail stop after hitting an error\n" " -batch force batch I/O\n" " -box set output mode to 'box'\n" - " -cmd COMMAND run \"COMMAND\" before reading stdin\n" " -column set output mode to 'column'\n" + " -cmd COMMAND run \"COMMAND\" before reading stdin\n" " -csv set output mode to 'csv'\n" #if !defined(SQLITE_OMIT_DESERIALIZE) " -deserialize open the database using sqlite3_deserialize()\n" @@ -12661,7 +13124,6 @@ static const char zOptions[] = #endif " -newline SEP set output row separator. Default: '\\n'\n" " -nofollow refuse to open symbolic links to database files\n" - " -noinit Do not read the ~/.sqliterc file at startup\n" " -nonce STRING set the safe-mode escape nonce\n" " -no-rowid-in-view Disable rowid-in-view using sqlite3_config()\n" " -nullvalue TEXT set text string for NULL values. Default ''\n" @@ -12670,7 +13132,6 @@ static const char zOptions[] = " -quote set output mode to 'quote'\n" " -readonly open the database read-only\n" " -safe enable safe-mode\n" - " -screenwidth N use N as the default screenwidth \n" " -separator SEP set output column separator. Default: '|'\n" #ifdef SQLITE_ENABLE_SORTER_REFERENCES " -sorterref SIZE sorter references threshold size\n" @@ -12687,11 +13148,11 @@ static const char zOptions[] = #endif ; static void usage(int showDetail){ - cli_printf(stderr,"Usage: %s [OPTIONS] [FILENAME [SQL...]]\n" + sqlite3_fprintf(stderr,"Usage: %s [OPTIONS] [FILENAME [SQL...]]\n" "FILENAME is the name of an SQLite database. A new database is created\n" "if the file does not previously exist. Defaults to :memory:.\n", Argv0); if( showDetail ){ - cli_printf(stderr,"OPTIONS include:\n%s", zOptions); + sqlite3_fprintf(stderr,"OPTIONS include:\n%s", zOptions); }else{ eputz("Use the -help option for additional information\n"); } @@ -12712,11 +13173,19 @@ static void verify_uninitialized(void){ /* ** Initialize the state information in data */ -static void main_init(ShellState *p) { - memset(p, 0, sizeof(*p)); - p->pAuxDb = &p->aAuxDb[0]; - p->shellFlgs = SHFLG_Lookaside; - sqlite3_config(SQLITE_CONFIG_LOG, shellLog, p); +static void main_init(ShellState *data) { + memset(data, 0, sizeof(*data)); + data->normalMode = data->cMode = data->mode = MODE_List; + data->autoExplain = 1; +#ifdef _WIN32 + data->crlfMode = 1; +#endif + data->pAuxDb = &data->aAuxDb[0]; + memcpy(data->colSeparator,SEP_Column, 2); + memcpy(data->rowSeparator,SEP_Row, 2); + data->showHeader = 0; + data->shellFlgs = SHFLG_Lookaside; + sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); #if !defined(SQLITE_SHELL_FIDDLE) verify_uninitialized(); #endif @@ -12731,18 +13200,22 @@ static void main_init(ShellState *p) { */ #if defined(_WIN32) || defined(WIN32) static void printBold(const char *zText){ +#if !SQLITE_OS_WINRT HANDLE out = GetStdHandle(STD_OUTPUT_HANDLE); CONSOLE_SCREEN_BUFFER_INFO defaultScreenInfo; GetConsoleScreenBufferInfo(out, &defaultScreenInfo); SetConsoleTextAttribute(out, FOREGROUND_RED|FOREGROUND_INTENSITY ); +#endif sputz(stdout, zText); +#if !SQLITE_OS_WINRT SetConsoleTextAttribute(out, defaultScreenInfo.wAttributes); +#endif } #else static void printBold(const char *zText){ - cli_printf(stdout, "\033[1m%s\033[0m", zText); + sqlite3_fprintf(stdout, "\033[1m%s\033[0m", zText); } #endif @@ -12752,9 +13225,9 @@ static void printBold(const char *zText){ */ static char *cmdline_option_value(int argc, char **argv, int i){ if( i==argc ){ - cli_printf(stderr, + sqlite3_fprintf(stderr, "%s: Error: missing argument to %s\n", argv[0], argv[argc-1]); - cli_exit(1); + exit(1); } return argv[i]; } @@ -12767,66 +13240,30 @@ static void sayAbnormalExit(void){ */ static int vfstraceOut(const char *z, void *pArg){ ShellState *p = (ShellState*)pArg; - cli_puts(z, p->out); + sqlite3_fputs(z, p->out); fflush(p->out); return 1; } -/* Alternative name to the entry point for Fiddle */ +#ifndef SQLITE_SHELL_IS_UTF8 +# if (defined(_WIN32) || defined(WIN32)) \ + && (defined(_MSC_VER) || (defined(UNICODE) && defined(__GNUC__))) +# define SQLITE_SHELL_IS_UTF8 (0) +# else +# define SQLITE_SHELL_IS_UTF8 (1) +# endif +#endif + #ifdef SQLITE_SHELL_FIDDLE # define main fiddle_main #endif -/* Use the wmain() entry point on Windows. Translate arguments to -** UTF8, then invoke the traditional main() entry point which is -** renamed using a #define to utf8_main() . -*/ -#if defined(_WIN32) && !defined(main) -# define main utf8_main /* Rename entry point to utf_main() */ -int SQLITE_CDECL utf8_main(int,char**); /* Forward declaration */ -int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ - int rc, i; - char **argv = malloc( sizeof(char*) * (argc+1) ); - char **orig = argv; - if( argv==0 ){ - fprintf(stderr, "malloc failed\n"); - exit(1); - } - for(i=0; i<argc; i++){ - int nByte = WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, 0, 0, 0, 0); - if( nByte==0 ){ - argv[i] = 0; - }else{ - argv[i] = malloc( nByte ); - if( argv[i]==0 ){ - fprintf(stderr, "malloc failed\n"); - exit(1); - } - nByte = WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, argv[i],nByte,0,0); - if( nByte==0 ){ - free(argv[i]); - argv[i] = 0; - } - } - } - argv[argc] = 0; - rc = utf8_main(argc, argv); - for(i=0; i<argc; i++) free(orig[i]); - free(argv); - return rc; -} -#endif /* WIN32 */ - -/* -** This is the main entry point for the process. Everything starts here. -** -** The "main" identifier may have been #defined to something else: -** -** utf8_main On Windows -** fiddle_main In Fiddle -** sqlite3_shell Other projects that use shell.c as a subroutine -*/ +#if SQLITE_SHELL_IS_UTF8 int SQLITE_CDECL main(int argc, char **argv){ +#else +int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ + char **argv; +#endif #ifdef SQLITE_DEBUG sqlite3_int64 mem_main_enter = 0; #endif @@ -12841,13 +13278,15 @@ int SQLITE_CDECL main(int argc, char **argv){ int rc = 0; int warnInmemoryDb = 0; int readStdin = 1; - int noInit = 0; /* Do not read ~/.sqliterc if true */ int nCmd = 0; int nOptsEnd = argc; int bEnableVfstrace = 0; char **azCmd = 0; - int *aiCmd = 0; const char *zVfs = 0; /* Value of -vfs command-line option */ +#if !SQLITE_SHELL_IS_UTF8 + char **argvToFree = 0; + int argcToFree = 0; +#endif setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */ #ifdef SQLITE_SHELL_FIDDLE @@ -12866,7 +13305,7 @@ int SQLITE_CDECL main(int argc, char **argv){ if( getenv("SQLITE_DEBUG_BREAK") ){ if( isatty(0) && isatty(2) ){ char zLine[100]; - cli_printf(stderr, + sqlite3_fprintf(stderr, "attach debugger to process %d and press ENTER to continue...", GETPID()); if( sqlite3_fgets(zLine, sizeof(zLine), stdin)!=0 @@ -12876,7 +13315,11 @@ int SQLITE_CDECL main(int argc, char **argv){ } }else{ #if defined(_WIN32) || defined(WIN32) +#if SQLITE_OS_WINRT + __debugbreak(); +#else DebugBreak(); +#endif #elif defined(SIGTRAP) raise(SIGTRAP); #endif @@ -12894,7 +13337,7 @@ int SQLITE_CDECL main(int argc, char **argv){ #if USE_SYSTEM_SQLITE+0!=1 if( cli_strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){ - cli_printf(stderr, + sqlite3_fprintf(stderr, "SQLite header and source version mismatch\n%s\n%s\n", sqlite3_sourceid(), SQLITE_SOURCE_ID); exit(1); @@ -12902,6 +13345,32 @@ int SQLITE_CDECL main(int argc, char **argv){ #endif main_init(&data); + /* On Windows, we must translate command-line arguments into UTF-8. + ** The SQLite memory allocator subsystem has to be enabled in order to + ** do this. But we want to run an sqlite3_shutdown() afterwards so that + ** subsequent sqlite3_config() calls will work. So copy all results into + ** memory that does not come from the SQLite memory allocator. + */ +#if !SQLITE_SHELL_IS_UTF8 + sqlite3_initialize(); + argvToFree = malloc(sizeof(argv[0])*argc*2); + shell_check_oom(argvToFree); + argcToFree = argc; + argv = argvToFree + argc; + for(i=0; i<argc; i++){ + char *z = sqlite3_win32_unicode_to_utf8(wargv[i]); + i64 n; + shell_check_oom(z); + n = strlen(z); + argv[i] = malloc( n+1 ); + shell_check_oom(argv[i]); + memcpy(argv[i], z, n+1); + argvToFree[i] = argv[i]; + sqlite3_free(z); + } + sqlite3_shutdown(); +#endif + assert( argc>=1 && argv && argv[0] ); Argv0 = argv[0]; @@ -12917,7 +13386,7 @@ int SQLITE_CDECL main(int argc, char **argv){ } #endif - /* Do an initial pass through the command-line arguments to locate + /* Do an initial pass through the command-line argument to locate ** the name of the database file, the name of the initialization file, ** the size of the alternative malloc heap, options affecting commands ** or SQL run from the command line, and the first command to execute. @@ -12929,20 +13398,16 @@ int SQLITE_CDECL main(int argc, char **argv){ char *z; z = argv[i]; if( z[0]!='-' || i>nOptsEnd ){ - if( data.aAuxDb->zDbFilename==0 && !isScriptFile(z,1) ){ + if( data.aAuxDb->zDbFilename==0 ){ data.aAuxDb->zDbFilename = z; }else{ /* Excess arguments are interpreted as SQL (or dot-commands) and ** mean that nothing is read from stdin */ readStdin = 0; - stdin_is_interactive = 0; nCmd++; azCmd = realloc(azCmd, sizeof(azCmd[0])*nCmd); shell_check_oom(azCmd); - aiCmd = realloc(aiCmd, sizeof(aiCmd[0])*nCmd); - shell_check_oom(azCmd); azCmd[nCmd-1] = z; - aiCmd[nCmd-1] = i; } continue; } @@ -12965,15 +13430,6 @@ int SQLITE_CDECL main(int argc, char **argv){ ** we do the actual processing of arguments later in a second pass. */ stdin_is_interactive = 0; - stdout_is_console = 0; - modeChange(&data, MODE_BATCH); - }else if( cli_strcmp(z,"-screenwidth")==0 ){ - int n = atoi(cmdline_option_value(argc, argv, ++i)); - if( n<2 ){ - sqlite3_fprintf(stderr,"minimum --screenwidth is 2\n"); - exit(1); - } - stdout_tty_width = n; }else if( cli_strcmp(z,"-utf8")==0 ){ }else if( cli_strcmp(z,"-no-utf8")==0 ){ }else if( cli_strcmp(z,"-no-rowid-in-view")==0 ){ @@ -13007,7 +13463,7 @@ int SQLITE_CDECL main(int argc, char **argv){ int szHdr = 0; sqlite3_config(SQLITE_CONFIG_PCACHE_HDRSZ, &szHdr); sz += szHdr; - cli_printf(stdout, "Page cache size increased to %d to accommodate" + sqlite3_fprintf(stdout, "Page cache size increased to %d to accommodate" " the %d-byte headers\n", (int)sz, szHdr); } verify_uninitialized(); @@ -13068,8 +13524,6 @@ int SQLITE_CDECL main(int argc, char **argv){ data.openFlags |= SQLITE_OPEN_READONLY; }else if( cli_strcmp(z,"-nofollow")==0 ){ data.openFlags |= SQLITE_OPEN_NOFOLLOW; - }else if( cli_strcmp(z,"-noinit")==0 ){ - noInit = 1; }else if( cli_strcmp(z,"-exclusive")==0 ){ /* UNDOCUMENTED */ data.openFlags |= SQLITE_OPEN_EXCLUSIVE; }else if( cli_strcmp(z,"-ifexists")==0 ){ @@ -13097,16 +13551,6 @@ int SQLITE_CDECL main(int argc, char **argv){ }else if( cli_strcmp(z,"-escape")==0 && i+1<argc ){ /* skip over the argument */ i++; - }else if( cli_strcmp(z,"-test-argv")==0 ){ - /* Undocumented test option. Print the values in argv[] and exit. - ** Use this to verify that any translation of the argv[], for example - ** on Windows that receives wargv[] from the OS and must convert - ** to UTF8 prior to calling this routine. */ - int kk; - for(kk=0; kk<argc; kk++){ - sqlite3_fprintf(stdout,"argv[%d] = \"%s\"\n", kk, argv[kk]); - } - return 0; } } #ifndef SQLITE_SHELL_FIDDLE @@ -13133,27 +13577,8 @@ int SQLITE_CDECL main(int argc, char **argv){ sqlite3_vfs *pVfs = sqlite3_vfs_find(zVfs); if( pVfs ){ sqlite3_vfs_register(pVfs, 1); - } -#if !defined(SQLITE_OMIT_LOAD_EXTENSION) - else if( access(zVfs,0)==0 ){ - /* If the VFS name is not the name of an existing VFS, but it is - ** the name of a file, then try to load that file as an extension. - ** Presumably the extension implements the desired VFS. */ - sqlite3 *db = 0; - char *zErr = 0; - sqlite3_open(":memory:", &db); - sqlite3_enable_load_extension(db, 1); - rc = sqlite3_load_extension(db, zVfs, 0, &zErr); - sqlite3_close(db); - if( (rc&0xff)!=SQLITE_OK ){ - cli_printf(stderr, "could not load extension VFS \"%s\": %s\n", - zVfs, zErr); - exit(1); - } - } -#endif - else{ - cli_printf(stderr,"no such VFS: \"%s\"\n", zVfs); + }else{ + sqlite3_fprintf(stderr,"no such VFS: \"%s\"\n", zVfs); exit(1); } } @@ -13163,10 +13588,9 @@ int SQLITE_CDECL main(int argc, char **argv){ data.pAuxDb->zDbFilename = ":memory:"; warnInmemoryDb = argc==1; #else - cli_printf(stderr, + sqlite3_fprintf(stderr, "%s: Error: no database filename specified\n", Argv0); - rc = 1; - goto shell_main_exit; + return 1; #endif } data.out = stdout; @@ -13176,7 +13600,6 @@ int SQLITE_CDECL main(int argc, char **argv){ #ifndef SQLITE_SHELL_FIDDLE sqlite3_appendvfs_init(0,0,0); #endif - modeDefault(&data); /* Go ahead and open the database file if it already exists. If the ** file does not exist, delay opening it. This prevents empty database @@ -13191,9 +13614,9 @@ int SQLITE_CDECL main(int argc, char **argv){ ** is given on the command line, look for a file named ~/.sqliterc and ** try to process it. */ - if( !noInit ) process_sqliterc(&data,zInitFile); + process_sqliterc(&data,zInitFile); - /* Make a second pass through the command-line arguments and set + /* Make a second pass through the command-line argument and set ** options. This second pass is delayed until after the initialization ** file is processed so that the command-line arguments will override ** settings in the initialization file. @@ -13205,44 +13628,45 @@ int SQLITE_CDECL main(int argc, char **argv){ if( cli_strcmp(z,"-init")==0 ){ i++; }else if( cli_strcmp(z,"-html")==0 ){ - modeChange(&data, MODE_Html); + data.mode = MODE_Html; }else if( cli_strcmp(z,"-list")==0 ){ - modeChange(&data, MODE_List); + data.mode = MODE_List; }else if( cli_strcmp(z,"-quote")==0 ){ - modeChange(&data, MODE_Quote); + data.mode = MODE_Quote; + sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator, SEP_Comma); + sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator, SEP_Row); }else if( cli_strcmp(z,"-line")==0 ){ - modeChange(&data, MODE_Line); + data.mode = MODE_Line; }else if( cli_strcmp(z,"-column")==0 ){ - modeChange(&data, MODE_Column); + data.mode = MODE_Column; }else if( cli_strcmp(z,"-json")==0 ){ - modeChange(&data, MODE_Json); + data.mode = MODE_Json; }else if( cli_strcmp(z,"-markdown")==0 ){ - modeChange(&data, MODE_Markdown); + data.mode = MODE_Markdown; }else if( cli_strcmp(z,"-table")==0 ){ - modeChange(&data, MODE_Table); - }else if( cli_strcmp(z,"-psql")==0 ){ - modeChange(&data, MODE_Psql); + data.mode = MODE_Table; }else if( cli_strcmp(z,"-box")==0 ){ - modeChange(&data, MODE_Box); + data.mode = MODE_Box; }else if( cli_strcmp(z,"-csv")==0 ){ - modeChange(&data, MODE_Csv); + data.mode = MODE_Csv; + memcpy(data.colSeparator,",",2); }else if( cli_strcmp(z,"-escape")==0 && i+1<argc ){ /* See similar code at tag-20250224-1 */ const char *zEsc = argv[++i]; int k; - for(k=0; k<ArraySize(qrfEscNames); k++){ - if( sqlite3_stricmp(zEsc,qrfEscNames[k])==0 ){ - data.mode.spec.eEsc = k; + for(k=0; k<ArraySize(shell_EscModeNames); k++){ + if( sqlite3_stricmp(zEsc,shell_EscModeNames[k])==0 ){ + data.eEscMode = k; break; } } - if( k>=ArraySize(qrfEscNames) ){ - cli_printf(stderr, "unknown control character escape mode \"%s\"" + if( k>=ArraySize(shell_EscModeNames) ){ + sqlite3_fprintf(stderr, "unknown control character escape mode \"%s\"" " - choices:", zEsc); - for(k=0; k<ArraySize(qrfEscNames); k++){ - cli_printf(stderr, " %s", qrfEscNames[k]); + for(k=0; k<ArraySize(shell_EscModeNames); k++){ + sqlite3_fprintf(stderr, " %s", shell_EscModeNames[k]); } - cli_printf(stderr, "\n"); + sqlite3_fprintf(stderr, "\n"); exit(1); } #ifdef SQLITE_HAVE_ZLIB @@ -13262,40 +13686,44 @@ int SQLITE_CDECL main(int argc, char **argv){ data.openFlags |= SQLITE_OPEN_READONLY; }else if( cli_strcmp(z,"-nofollow")==0 ){ data.openFlags |= SQLITE_OPEN_NOFOLLOW; - }else if( cli_strcmp(z,"-noinit")==0 ){ - /* No-op */ }else if( cli_strcmp(z,"-exclusive")==0 ){ /* UNDOCUMENTED */ data.openFlags |= SQLITE_OPEN_EXCLUSIVE; }else if( cli_strcmp(z,"-ifexists")==0 ){ data.openFlags &= ~(SQLITE_OPEN_CREATE); if( data.openFlags==0 ) data.openFlags = SQLITE_OPEN_READWRITE; }else if( cli_strcmp(z,"-ascii")==0 ){ - modeChange(&data, MODE_Ascii); + data.mode = MODE_Ascii; + sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,SEP_Unit); + sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator,SEP_Record); }else if( cli_strcmp(z,"-tabs")==0 ){ - modeChange(&data, MODE_Tabs); + data.mode = MODE_List; + sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,SEP_Tab); + sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator,SEP_Row); }else if( cli_strcmp(z,"-separator")==0 ){ - modeSetStr(&data.mode.spec.zColumnSep, - cmdline_option_value(argc,argv,++i)); + sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator, + "%s",cmdline_option_value(argc,argv,++i)); }else if( cli_strcmp(z,"-newline")==0 ){ - modeSetStr(&data.mode.spec.zRowSep, - cmdline_option_value(argc,argv,++i)); + sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator, + "%s",cmdline_option_value(argc,argv,++i)); }else if( cli_strcmp(z,"-nullvalue")==0 ){ - modeSetStr(&data.mode.spec.zNull, - cmdline_option_value(argc,argv,++i)); + sqlite3_snprintf(sizeof(data.nullValue), data.nullValue, + "%s",cmdline_option_value(argc,argv,++i)); }else if( cli_strcmp(z,"-header")==0 ){ - data.mode.spec.bTitles = QRF_Yes; + data.showHeader = 1; + ShellSetFlag(&data, SHFLG_HeaderSet); }else if( cli_strcmp(z,"-noheader")==0 ){ - data.mode.spec.bTitles = QRF_No; + data.showHeader = 0; + ShellSetFlag(&data, SHFLG_HeaderSet); }else if( cli_strcmp(z,"-echo")==0 ){ - data.mode.mFlags |= MFLG_ECHO; + ShellSetFlag(&data, SHFLG_Echo); }else if( cli_strcmp(z,"-eqp")==0 ){ - data.mode.autoEQP = AUTOEQP_on; + data.autoEQP = AUTOEQP_on; }else if( cli_strcmp(z,"-eqpfull")==0 ){ - data.mode.autoEQP = AUTOEQP_full; + data.autoEQP = AUTOEQP_full; }else if( cli_strcmp(z,"-stats")==0 ){ data.statsOn = 1; }else if( cli_strcmp(z,"-scanstats")==0 ){ - data.mode.scanstatsOn = 1; + data.scanstatsOn = 1; }else if( cli_strcmp(z,"-backslash")==0 ){ /* Undocumented command-line option: -backslash ** Causes C-style backslash escapes to be evaluated in SQL statements @@ -13306,10 +13734,9 @@ int SQLITE_CDECL main(int argc, char **argv){ }else if( cli_strcmp(z,"-bail")==0 ){ /* No-op. The bail_on_error flag should already be set. */ }else if( cli_strcmp(z,"-version")==0 ){ - cli_printf(stdout, "%s %s (%d-bit)\n", + sqlite3_fprintf(stdout, "%s %s (%d-bit)\n", sqlite3_libversion(), sqlite3_sourceid(), 8*(int)sizeof(char*)); - rc = 0; - goto shell_main_exit; + return 0; }else if( cli_strcmp(z,"-interactive")==0 ){ /* Need to check for interactive override here to so that it can ** affect console setup (for Windows only) and testing thereof. @@ -13317,8 +13744,6 @@ int SQLITE_CDECL main(int argc, char **argv){ stdin_is_interactive = 1; }else if( cli_strcmp(z,"-batch")==0 ){ /* already handled */ - }else if( cli_strcmp(z,"-screenwidth")==0 ){ - i++; }else if( cli_strcmp(z,"-utf8")==0 ){ /* already handled */ }else if( cli_strcmp(z,"-no-utf8")==0 ){ @@ -13364,26 +13789,23 @@ int SQLITE_CDECL main(int argc, char **argv){ z = cmdline_option_value(argc,argv,++i); if( z[0]=='.' ){ rc = do_meta_command(z, &data); - if( rc && (bail_on_error || rc==2) ){ - if( rc==2 ) rc = 0; - goto shell_main_exit; - } + if( rc && bail_on_error ) return rc==2 ? 0 : rc; }else{ open_db(&data, 0); rc = shell_exec(&data, z, &zErrMsg); if( zErrMsg!=0 ){ shellEmitError(zErrMsg); sqlite3_free(zErrMsg); - if( !rc ) rc = 1; + if( bail_on_error ) return rc!=0 ? rc : 1; }else if( rc!=0 ){ - cli_printf(stderr,"Error: unable to process SQL \"%s\"\n", z); + sqlite3_fprintf(stderr,"Error: unable to process SQL \"%s\"\n", z); + if( bail_on_error ) return rc; } - if( bail_on_error ) goto shell_main_exit; } #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) }else if( cli_strncmp(z, "-A", 2)==0 ){ if( nCmd>0 ){ - cli_printf(stderr,"Error: cannot mix regular SQL or dot-commands" + sqlite3_fprintf(stderr,"Error: cannot mix regular SQL or dot-commands" " with \"%s\"\n", z); rc = 1; goto shell_main_exit; @@ -13396,7 +13818,6 @@ int SQLITE_CDECL main(int argc, char **argv){ arDotCommand(&data, 1, argv+i, argc-i); } readStdin = 0; - stdin_is_interactive = 0; break; #endif }else if( cli_strcmp(z,"-safe")==0 ){ @@ -13404,11 +13825,11 @@ int SQLITE_CDECL main(int argc, char **argv){ }else if( cli_strcmp(z,"-unsafe-testing")==0 ){ /* Acted upon in first pass. */ }else{ - cli_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); + sqlite3_fprintf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); eputz("Use -help for a list of options.\n"); - rc = 1; - goto shell_main_exit; + return 1; } + data.cMode = data.mode; } if( !readStdin ){ @@ -13418,26 +13839,8 @@ int SQLITE_CDECL main(int argc, char **argv){ */ for(i=0; i<nCmd; i++){ echo_group_input(&data, azCmd[i]); - if( isScriptFile(azCmd[i],0) ){ - FILE *inSaved = data.in; - i64 savedLineno = data.lineno; - int res = 1; - if( (data.in = openChrSource(azCmd[i]))!=0 ){ - res = process_input(&data, azCmd[i]); - fclose(data.in); - } - data.in = inSaved; - data.lineno = savedLineno; - if( res ) i = nCmd; - }else if( azCmd[i][0]=='.' ){ - char *zErrCtx = malloc( 64 ); - shell_check_oom(zErrCtx); - sqlite3_snprintf(64,zErrCtx,"argv[%i]:",aiCmd[i]); - data.zInFile = "<cmdline>"; - data.zErrPrefix = zErrCtx; + if( azCmd[i][0]=='.' ){ rc = do_meta_command(azCmd[i], &data); - free(data.zErrPrefix); - data.zErrPrefix = 0; if( rc ){ if( rc==2 ) rc = 0; goto shell_main_exit; @@ -13449,23 +13852,13 @@ int SQLITE_CDECL main(int argc, char **argv){ if( zErrMsg!=0 ){ shellEmitError(zErrMsg); }else{ - cli_printf(stderr, + sqlite3_fprintf(stderr, "Error: unable to process SQL: %s\n", azCmd[i]); } sqlite3_free(zErrMsg); if( rc==0 ) rc = 1; goto shell_main_exit; } - if( data.nPopMode ){ - modePop(&data); - data.nPopMode = 0; - } - } - if( data.nPopOutput ){ - output_reset(&data); - data.nPopOutput = 0; - }else{ - clearTempFile(&data); } } }else{ @@ -13474,7 +13867,7 @@ int SQLITE_CDECL main(int argc, char **argv){ if( stdin_is_interactive ){ char *zHome; char *zHistory; - cli_printf(stdout, + sqlite3_fprintf(stdout, "SQLite version %s %.19s\n" /*extra-version-info*/ "Enter \".help\" for usage hints.\n", sqlite3_libversion(), sqlite3_sourceid()); @@ -13527,7 +13920,6 @@ int SQLITE_CDECL main(int argc, char **argv){ #endif shell_main_exit: free(azCmd); - free(aiCmd); set_table_name(&data, 0); if( data.db ){ session_close_all(&data, -1); @@ -13544,28 +13936,12 @@ int SQLITE_CDECL main(int argc, char **argv){ output_reset(&data); data.doXdgOpen = 0; clearTempFile(&data); - modeFree(&data.mode); - if( data.nSavedModes ){ - int ii; - for(ii=0; ii<data.nSavedModes; ii++){ - modeFree(&data.aSavedModes[ii].mode); - free(data.aSavedModes[ii].zTag); - } - free(data.aSavedModes); - } - free(data.zErrPrefix); +#if !SQLITE_SHELL_IS_UTF8 + for(i=0; i<argcToFree; i++) free(argvToFree[i]); + free(argvToFree); +#endif + free(data.colWidth); free(data.zNonce); - free(data.dot.zCopy); - free(data.dot.azArg); - free(data.dot.aiOfst); - free(data.dot.abQuot); - if( data.nTestRun ){ - sqlite3_fprintf(stdout, "%d test%s run with %d error%s\n", - data.nTestRun, data.nTestRun==1 ? "" : "s", - data.nTestErr, data.nTestErr==1 ? "" : "s"); - fflush(stdout); - rc = data.nTestErr>0; - } /* Clear the global data structure so that valgrind will detect memory ** leaks */ memset(&data, 0, sizeof(data)); @@ -13574,7 +13950,7 @@ int SQLITE_CDECL main(int argc, char **argv){ } #ifdef SQLITE_DEBUG if( sqlite3_memory_used()>mem_main_enter ){ - cli_printf(stderr,"Memory leaked: %u bytes\n", + sqlite3_fprintf(stderr,"Memory leaked: %u bytes\n", (unsigned int)(sqlite3_memory_used()-mem_main_enter)); } #endif @@ -13614,7 +13990,7 @@ sqlite3_vfs * fiddle_db_vfs(const char *zDbName){ /* Only for emcc experimentation purposes. */ sqlite3 * fiddle_db_arg(sqlite3 *arg){ - cli_printf(stdout, "fiddle_db_arg(%p)\n", (const void*)arg); + sqlite3_fprintf(stdout, "fiddle_db_arg(%p)\n", (const void*)arg); return arg; } @@ -13651,7 +14027,7 @@ void fiddle_reset_db(void){ ** Resolve problem reported in ** https://sqlite.org/forum/forumpost/0b41a25d65 */ - cli_puts("Rolling back in-progress transaction.\n", stdout); + sqlite3_fputs("Rolling back in-progress transaction.\n", stdout); sqlite3_exec(globalDb,"ROLLBACK", 0, 0, 0); } rc = sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index b6773e1d9..f6ed48c20 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1490,7 +1490,7 @@ typedef const char *sqlite3_filename; ** greater and the function pointer is not NULL) and will fall back ** to xCurrentTime() if xCurrentTimeInt64() is unavailable. ** -** ^The xSetSystemCall(), xGetSystemCall(), and xNextSystemCall() interfaces +** ^The xSetSystemCall(), xGetSystemCall(), and xNestSystemCall() interfaces ** are not used by the SQLite core. These optional interfaces are provided ** by some VFSes to facilitate testing of the VFS code. By overriding ** system calls with functions under its control, a test program can @@ -2567,15 +2567,12 @@ struct sqlite3_mem_methods { ** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]] ** <dt>SQLITE_DBCONFIG_STMT_SCANSTATUS</dt> ** <dd>The SQLITE_DBCONFIG_STMT_SCANSTATUS option is only useful in -** [SQLITE_ENABLE_STMT_SCANSTATUS] builds. In this case, it sets or clears -** a flag that enables collection of run-time performance statistics -** used by [sqlite3_stmt_scanstatus_v2()] and the [nexec and ncycle] -** columns of the [bytecode virtual table]. -** For statistics to be collected, the flag must be set on -** the database handle both when the SQL statement is -** [sqlite3_prepare|prepared] and when it is [sqlite3_step|stepped]. -** The flag is set (collection of statistics is enabled) by default. -** <p>This option takes two arguments: an integer and a pointer to +** SQLITE_ENABLE_STMT_SCANSTATUS builds. In this case, it sets or clears +** a flag that enables collection of the sqlite3_stmt_scanstatus_v2() +** statistics. For statistics to be collected, the flag must be set on +** the database handle both when the SQL statement is prepared and when it +** is stepped. The flag is set (collection of statistics is enabled) +** by default. <p>This option takes two arguments: an integer and a pointer to ** an integer. The first argument is 1, 0, or -1 to enable, disable, or ** leave unchanged the statement scanstatus option. If the second argument ** is not NULL, then the value of the statement scanstatus setting after @@ -2648,22 +2645,6 @@ struct sqlite3_mem_methods { ** comments are allowed in SQL text after processing the first argument. ** </dd> ** -** [[SQLITE_DBCONFIG_FP_DIGITS]] -** <dt>SQLITE_DBCONFIG_FP_DIGITS</dt> -** <dd>The SQLITE_DBCONFIG_FP_DIGITS setting is a small integer that determines -** the number of significant digits that SQLite will attempt to preserve when -** converting floating point numbers (IEEE 754 "doubles") into text. The -** default value 17, as of SQLite version 3.52.0. The value was 15 in all -** prior versions.<p> -** This option takes two arguments which are an integer and a pointer -** to an integer. The first argument is a small integer, between 3 and 23, or -** zero. The FP_DIGITS setting is changed to that small integer, or left -** altered if the first argument is zero or out of range. The second argument -** is a pointer to an integer. If the pointer is not NULL, then the value of -** the FP_DIGITS setting, after possibly being modified by the first -** arguments, is written into the integer to which the second argument points. -** </dd> -** ** </dl> ** ** [[DBCONFIG arguments]] <h3>Arguments To SQLITE_DBCONFIG Options</h3> @@ -2681,10 +2662,9 @@ struct sqlite3_mem_methods { ** the first argument. ** ** <p>While most SQLITE_DBCONFIG options use the argument format -** described in the previous paragraph, the [SQLITE_DBCONFIG_MAINDBNAME], -** [SQLITE_DBCONFIG_LOOKASIDE], and [SQLITE_DBCONFIG_FP_DIGITS] options -** are different. See the documentation of those exceptional options for -** details. +** described in the previous paragraph, the [SQLITE_DBCONFIG_MAINDBNAME] +** and [SQLITE_DBCONFIG_LOOKASIDE] options are different. See the +** documentation of those exceptional options for details. */ #define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ #define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ @@ -2709,8 +2689,7 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE 1020 /* int int* */ #define SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE 1021 /* int int* */ #define SQLITE_DBCONFIG_ENABLE_COMMENTS 1022 /* int int* */ -#define SQLITE_DBCONFIG_FP_DIGITS 1023 /* int int* */ -#define SQLITE_DBCONFIG_MAX 1023 /* Largest DBCONFIG */ +#define SQLITE_DBCONFIG_MAX 1022 /* Largest DBCONFIG */ /* ** CAPI3REF: Enable Or Disable Extended Result Codes @@ -4192,7 +4171,6 @@ void sqlite3_free_filename(sqlite3_filename); ** <li> sqlite3_errmsg() ** <li> sqlite3_errmsg16() ** <li> sqlite3_error_offset() -** <li> sqlite3_db_handle() ** </ul> ** ** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language @@ -4239,7 +4217,7 @@ const char *sqlite3_errstr(int); int sqlite3_error_offset(sqlite3 *db); /* -** CAPI3REF: Set Error Code And Message +** CAPI3REF: Set Error Codes And Message ** METHOD: sqlite3 ** ** Set the error code of the database handle passed as the first argument @@ -4358,10 +4336,6 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** [[SQLITE_LIMIT_EXPR_DEPTH]] ^(<dt>SQLITE_LIMIT_EXPR_DEPTH</dt> ** <dd>The maximum depth of the parse tree on any expression.</dd>)^ ** -** [[SQLITE_LIMIT_PARSER_DEPTH]] ^(<dt>SQLITE_LIMIT_PARSER_DEPTH</dt> -** <dd>The maximum depth of the LALR(1) parser stack used to analyze -** input SQL statements.</dd>)^ -** ** [[SQLITE_LIMIT_COMPOUND_SELECT]] ^(<dt>SQLITE_LIMIT_COMPOUND_SELECT</dt> ** <dd>The maximum number of terms in a compound SELECT statement.</dd>)^ ** @@ -4406,7 +4380,6 @@ int sqlite3_limit(sqlite3*, int id, int newVal); #define SQLITE_LIMIT_VARIABLE_NUMBER 9 #define SQLITE_LIMIT_TRIGGER_DEPTH 10 #define SQLITE_LIMIT_WORKER_THREADS 11 -#define SQLITE_LIMIT_PARSER_DEPTH 12 /* ** CAPI3REF: Prepare Flags @@ -4451,29 +4424,12 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** fails, the sqlite3_prepare_v3() call returns the same error indications ** with or without this flag; it just omits the call to [sqlite3_log()] that ** logs the error. -** -** [[SQLITE_PREPARE_FROM_DDL]] <dt>SQLITE_PREPARE_FROM_DDL</dt> -** <dd>The SQLITE_PREPARE_FROM_DDL flag causes the SQL compiler to enforce -** security constraints that would otherwise only be enforced when parsing -** the database schema. In other words, the SQLITE_PREPARE_FROM_DDL flag -** causes the SQL compiler to treat the SQL statement being prepared as if -** it had come from an attacker. When SQLITE_PREPARE_FROM_DDL is used and -** [SQLITE_DBCONFIG_TRUSTED_SCHEMA] is off, SQL functions may only be called -** if they are tagged with [SQLITE_INNOCUOUS] and virtual tables may only -** be used if they are tagged with [SQLITE_VTAB_INNOCUOUS]. Best practice -** is to use the SQLITE_PREPARE_FROM_DDL option when preparing any SQL that -** is derived from parts of the database schema. In particular, virtual -** table implementations that run SQL statements that are derived from -** arguments to their CREATE VIRTUAL TABLE statement should always use -** [sqlite3_prepare_v3()] and set the SQLITE_PREPARE_FROM_DDL flag to -** prevent bypass of the [SQLITE_DBCONFIG_TRUSTED_SCHEMA] security checks. ** </dl> */ #define SQLITE_PREPARE_PERSISTENT 0x01 #define SQLITE_PREPARE_NORMALIZE 0x02 #define SQLITE_PREPARE_NO_VTAB 0x04 #define SQLITE_PREPARE_DONT_LOG 0x10 -#define SQLITE_PREPARE_FROM_DDL 0x20 /* ** CAPI3REF: Compiling An SQL Statement @@ -4487,9 +4443,8 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** ** The preferred routine to use is [sqlite3_prepare_v2()]. The ** [sqlite3_prepare()] interface is legacy and should be avoided. -** [sqlite3_prepare_v3()] has an extra -** [SQLITE_PREPARE_FROM_DDL|"prepFlags" option] that is some times -** needed for special purpose or to pass along security restrictions. +** [sqlite3_prepare_v3()] has an extra "prepFlags" option that is used +** for special purposes. ** ** The use of the UTF-8 interfaces is preferred, as SQLite currently ** does all parsing using UTF-8. The UTF-16 interfaces are provided @@ -4894,8 +4849,8 @@ typedef struct sqlite3_context sqlite3_context; ** it should be a pointer to well-formed UTF16 text. ** ^If the third parameter to sqlite3_bind_text64() is not NULL, then ** it should be a pointer to a well-formed unicode string that is -** either UTF8 if the sixth parameter is SQLITE_UTF8 or SQLITE_UTF8_ZT, -** or UTF16 otherwise. +** either UTF8 if the sixth parameter is SQLITE_UTF8, or UTF16 +** otherwise. ** ** [[byte-order determination rules]] ^The byte-order of ** UTF16 input text is determined by the byte-order mark (BOM, U+FEFF) @@ -4941,15 +4896,10 @@ typedef struct sqlite3_context sqlite3_context; ** object and pointer to it must remain valid until then. ^SQLite will then ** manage the lifetime of its private copy. ** -** ^The sixth argument (the E argument) -** to sqlite3_bind_text64(S,K,Z,N,D,E) must be one of -** [SQLITE_UTF8], [SQLITE_UTF8_ZT], [SQLITE_UTF16], [SQLITE_UTF16BE], -** or [SQLITE_UTF16LE] to specify the encoding of the text in the -** third parameter, Z. The special value [SQLITE_UTF8_ZT] means that the -** string argument is both UTF-8 encoded and is zero-terminated. In other -** words, SQLITE_UTF8_ZT means that the Z array is allocated to hold at -** least N+1 bytes and that the Z&#91;N&#93; byte is zero. If -** the E argument to sqlite3_bind_text64(S,K,Z,N,D,E) is not one of the +** ^The sixth argument to sqlite3_bind_text64() must be one of +** [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE] +** to specify the encoding of the text in the third parameter. If +** the sixth argument to sqlite3_bind_text64() is not one of the ** allowed values shown above, or if the text encoding is different ** from the encoding specified by the sixth parameter, then the behavior ** is undefined. @@ -5816,51 +5766,6 @@ int sqlite3_create_window_function( ** ** These constants define integer codes that represent the various ** text encodings supported by SQLite. -** -** <dl> -** [[SQLITE_UTF8]] <dt>SQLITE_UTF8</dt><dd>Text is encoding as UTF-8</dd> -** -** [[SQLITE_UTF16LE]] <dt>SQLITE_UTF16LE</dt><dd>Text is encoding as UTF-16 -** with each code point being expressed "little endian" - the least significant -** byte first. This is the usual encoding, for example on Windows.</dd> -** -** [[SQLITE_UTF16BE]] <dt>SQLITE_UTF16BE</dt><dd>Text is encoding as UTF-16 -** with each code point being expressed "big endian" - the most significant -** byte first. This encoding is less common, but is still sometimes seen, -** specially on older systems. -** -** [[SQLITE_UTF16]] <dt>SQLITE_UTF16</dt><dd>Text is encoding as UTF-16 -** with each code point being expressed either little endian or as big -** endian, according to the native endianness of the host computer. -** -** [[SQLITE_ANY]] <dt>SQLITE_ANY</dt><dd>This encoding value may only be used -** to declare the preferred text for [application-defined SQL functions] -** created using [sqlite3_create_function()] and similar. If the preferred -** encoding (the 4th parameter to sqlite3_create_function() - the eTextRep -** parameter) is SQLITE_ANY, that indicates that the function does not have -** a preference regarding the text encoding of its parameters and can take -** any text encoding that the SQLite core find convenient to supply. This -** option is deprecated. Please do not use it in new applications. -** -** [[SQLITE_UTF16_ALIGNED]] <dt>SQLITE_UTF16_ALIGNED</dt><dd>This encoding -** value may be used as the 3rd parameter (the eTextRep parameter) to -** [sqlite3_create_collation()] and similar. This encoding value means -** that the application-defined collating sequence created expects its -** input strings to be in UTF16 in native byte order, and that the start -** of the strings must be aligned to a 2-byte boundary. -** -** [[SQLITE_UTF8_ZT]] <dt>SQLITE_UTF8_ZT</dt><dd>This option can only be -** used to specify the text encoding to strings input to [sqlite3_result_text64()] -** and [sqlite3_bind_text64()]. It means that the input string (call it "z") -** is UTF-8 encoded and that it is zero-terminated. If the length parameter -** (call it "n") is non-negative, this encoding option means that the caller -** guarantees that z array contains at least n+1 bytes and that the z&#91;n&#93; -** byte has a value of zero. -** This option gives the same output as SQLITE_UTF8, but can be more efficient -** by avoiding the need to make a copy of the input string, in some cases. -** However, if z is allocated to hold fewer than n+1 bytes or if the -** z&#91;n&#93; byte is not zero, undefined behavior may result. -** </dl> */ #define SQLITE_UTF8 1 /* IMP: R-37514-35566 */ #define SQLITE_UTF16LE 2 /* IMP: R-03371-37637 */ @@ -5868,7 +5773,6 @@ int sqlite3_create_window_function( #define SQLITE_UTF16 4 /* Use native byte order */ #define SQLITE_ANY 5 /* Deprecated */ #define SQLITE_UTF16_ALIGNED 8 /* sqlite3_create_collation only */ -#define SQLITE_UTF8_ZT 16 /* Zero-terminated UTF8 */ /* ** CAPI3REF: Function Flags @@ -6374,14 +6278,10 @@ void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); ** ** There is no limit (other than available memory) on the number of different ** client data pointers (with different names) that can be attached to a -** single database connection. However, the current implementation stores -** the content on a linked list. Insert and retrieval performance will -** be proportional to the number of entries. The design use case, and -** the use case for which the implementation is optimized, is -** that an application will store only small number of client data names, -** typically just one or two. This interface is not intended to be a -** generalized key/value store for thousands or millions of keys. It -** will work for that, but performance might be disappointing. +** single database connection. However, the implementation is optimized +** for the case of having only one or two different client data names. +** Applications and wrapper libraries are discouraged from using more than +** one client data name each. ** ** There is no way to enumerate the client data pointers ** associated with a database connection. The N parameter can be thought @@ -6489,14 +6389,10 @@ typedef void (*sqlite3_destructor_type)(void*); ** set the return value of the application-defined function to be ** a text string which is represented as UTF-8, UTF-16 native byte order, ** UTF-16 little endian, or UTF-16 big endian, respectively. -** ^The sqlite3_result_text64(C,Z,N,D,E) interface sets the return value of an +** ^The sqlite3_result_text64() interface sets the return value of an ** application-defined function to be a text string in an encoding -** specified the E parameter, which must be one -** of [SQLITE_UTF8], [SQLITE_UTF8_ZT], [SQLITE_UTF16], [SQLITE_UTF16BE], -** or [SQLITE_UTF16LE]. ^The special value [SQLITE_UTF8_ZT] means that -** the result text is both UTF-8 and zero-terminated. In other words, -** SQLITE_UTF8_ZT means that the Z array holds at least N+1 byes and that -** the Z&#91;N&#93; is zero. +** specified by the fifth (and last) parameter, which must be one +** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]. ** ^SQLite takes the text result from the application from ** the 2nd parameter of the sqlite3_result_text* interfaces. ** ^If the 3rd parameter to any of the sqlite3_result_text* interfaces @@ -6583,7 +6479,7 @@ void sqlite3_result_int(sqlite3_context*, int); void sqlite3_result_int64(sqlite3_context*, sqlite3_int64); void sqlite3_result_null(sqlite3_context*); void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); -void sqlite3_result_text64(sqlite3_context*, const char *z, sqlite3_uint64 n, +void sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64, void(*)(void*), unsigned char encoding); void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); @@ -7522,7 +7418,7 @@ int sqlite3_table_column_metadata( ** ^The sqlite3_load_extension() interface attempts to load an ** [SQLite extension] library contained in the file zFile. If ** the file cannot be loaded directly, attempts are made to load -** with various operating-system specific filename extensions added. +** with various operating-system specific extensions added. ** So for example, if "samplelib" cannot be loaded, then names like ** "samplelib.so" or "samplelib.dylib" or "samplelib.dll" might ** be tried also. @@ -7530,10 +7426,10 @@ int sqlite3_table_column_metadata( ** ^The entry point is zProc. ** ^(zProc may be 0, in which case SQLite will try to come up with an ** entry point name on its own. It first tries "sqlite3_extension_init". -** If that does not work, it tries names of the form "sqlite3_X_init" -** where X consists of the lower-case equivalent of all ASCII alphabetic -** characters or all ASCII alphanumeric characters in the filename from -** the last "/" to the first following "." and omitting any initial "lib".)^ +** If that does not work, it constructs a name "sqlite3_X_init" where +** X consists of the lower-case equivalent of all ASCII alphabetic +** characters in the filename from the last "/" to the first following +** "." and omitting any initial "lib".)^ ** ^The sqlite3_load_extension() interface returns ** [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. ** ^If an error occurs and pzErrMsg is not 0, then the @@ -8826,22 +8722,17 @@ sqlite3_str *sqlite3_str_new(sqlite3*); ** pass the returned value to [sqlite3_free()] to avoid a memory leak. ** ^The [sqlite3_str_finish(X)] interface may return a NULL pointer if any ** errors were encountered during construction of the string. ^The -** [sqlite3_str_finish(X)] interface might also return a NULL pointer if the +** [sqlite3_str_finish(X)] interface will also return a NULL pointer if the ** string in [sqlite3_str] object X is zero bytes long. -** -** ^The [sqlite3_str_free(X)] interface destroys both the sqlite3_str object -** X and the string content it contains. Calling sqlite3_str_free(X) is -** the equivalent of calling [sqlite3_free](sqlite3_str_finish(X)). */ char *sqlite3_str_finish(sqlite3_str*); -void sqlite3_str_free(sqlite3_str*); /* ** CAPI3REF: Add Content To A Dynamic String ** METHOD: sqlite3_str ** -** These interfaces add or remove content to an sqlite3_str object -** previously obtained from [sqlite3_str_new()]. +** These interfaces add content to an sqlite3_str object previously obtained +** from [sqlite3_str_new()]. ** ** ^The [sqlite3_str_appendf(X,F,...)] and ** [sqlite3_str_vappendf(X,F,V)] interfaces uses the [built-in printf] @@ -8864,10 +8755,6 @@ void sqlite3_str_free(sqlite3_str*); ** ^The [sqlite3_str_reset(X)] method resets the string under construction ** inside [sqlite3_str] object X back to zero bytes in length. ** -** ^The [sqlite3_str_truncate(X,N)] method changes the length of the string -** under construction to be N bytes are less. This routine is a no-op if -** N is negative or if the string is already N bytes or smaller in size. -** ** These methods do not return a result code. ^If an error occurs, that fact ** is recorded in the [sqlite3_str] object and can be recovered by a ** subsequent call to [sqlite3_str_errcode(X)]. @@ -8878,7 +8765,6 @@ void sqlite3_str_append(sqlite3_str*, const char *zIn, int N); void sqlite3_str_appendall(sqlite3_str*, const char *zIn); void sqlite3_str_appendchar(sqlite3_str*, int N, char C); void sqlite3_str_reset(sqlite3_str*); -void sqlite3_str_truncate(sqlite3_str*,int N); /* ** CAPI3REF: Status Of A Dynamic String @@ -10712,9 +10598,9 @@ int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); ** a variable pointed to by the "pOut" parameter. ** ** The "flags" parameter must be passed a mask of flags. At present only -** one flag is defined - [SQLITE_SCANSTAT_COMPLEX]. If SQLITE_SCANSTAT_COMPLEX +** one flag is defined - SQLITE_SCANSTAT_COMPLEX. If SQLITE_SCANSTAT_COMPLEX ** is specified, then status information is available for all elements -** of a query plan that are reported by "[EXPLAIN QUERY PLAN]" output. If +** of a query plan that are reported by "EXPLAIN QUERY PLAN" output. If ** SQLITE_SCANSTAT_COMPLEX is not specified, then only query plan elements ** that correspond to query loops (the "SCAN..." and "SEARCH..." elements of ** the EXPLAIN QUERY PLAN output) are available. Invoking API @@ -10728,8 +10614,7 @@ int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); ** elements used to implement the statement - a non-zero value is returned and ** the variable that pOut points to is unchanged. ** -** See also: [sqlite3_stmt_scanstatus_reset()] and the -** [nexec and ncycle] columnes of the [bytecode virtual table]. +** See also: [sqlite3_stmt_scanstatus_reset()] */ int sqlite3_stmt_scanstatus( sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ @@ -11271,41 +11156,19 @@ int sqlite3_deserialize( /* ** CAPI3REF: Bind array values to the CARRAY table-valued function ** -** The sqlite3_carray_bind_v2(S,I,P,N,F,X,D) interface binds an array value to -** parameter that is the first argument of the [carray() table-valued function]. -** The S parameter is a pointer to the [prepared statement] that uses the carray() -** functions. I is the parameter index to be bound. I must be the index of the -** parameter that is the first argument to the carray() table-valued function. -** P is a pointer to the array to be bound, and N is the number of elements in -** the array. The F argument is one of constants [SQLITE_CARRAY_INT32], -** [SQLITE_CARRAY_INT64], [SQLITE_CARRAY_DOUBLE], [SQLITE_CARRAY_TEXT], -** or [SQLITE_CARRAY_BLOB] to indicate the datatype of the array P. -** -** If the X argument is not a NULL pointer or one of the special -** values [SQLITE_STATIC] or [SQLITE_TRANSIENT], then SQLite will invoke -** the function X with argument D when it is finished using the data in P. -** The call to X(D) is a destructor for the array P. The destructor X(D) -** is invoked even if the call to sqlite3_carray_bind() fails. If the X -** parameter is the special-case value [SQLITE_STATIC], then SQLite assumes -** that the data static and the destructor is never invoked. If the X -** parameter is the special-case value [SQLITE_TRANSIENT], then -** sqlite3_carray_bind_v2() makes its own private copy of the data prior -** to returning and never invokes the destructor X. -** -** The sqlite3_carray_bind() function works the same as sqlite_carray_bind_v2() -** with a D parameter set to P. In other words, -** sqlite3_carray_bind(S,I,P,N,F,X) is same as -** sqlite3_carray_bind(S,I,P,N,F,X,P). -*/ -int sqlite3_carray_bind_v2( - sqlite3_stmt *pStmt, /* Statement to be bound */ - int i, /* Parameter index */ - void *aData, /* Pointer to array data */ - int nData, /* Number of data elements */ - int mFlags, /* CARRAY flags */ - void (*xDel)(void*), /* Destructor for aData */ - void *pDel /* Optional argument to xDel() */ -); +** The sqlite3_carray_bind(S,I,P,N,F,X) interface binds an array value to +** one of the first argument of the [carray() table-valued function]. The +** S parameter is a pointer to the [prepared statement] that uses the carray() +** functions. I is the parameter index to be bound. P is a pointer to the +** array to be bound, and N is the number of eements in the array. The +** F argument is one of constants [SQLITE_CARRAY_INT32], [SQLITE_CARRAY_INT64], +** [SQLITE_CARRAY_DOUBLE], [SQLITE_CARRAY_TEXT], or [SQLITE_CARRAY_BLOB] to +** indicate the datatype of the array being bound. The X argument is not a +** NULL pointer, then SQLite will invoke the function X on the P parameter +** after it has finished using P, even if the call to +** sqlite3_carray_bind() fails. The special-case finalizer +** SQLITE_TRANSIENT has no effect here. +*/ int sqlite3_carray_bind( sqlite3_stmt *pStmt, /* Statement to be bound */ int i, /* Parameter index */ diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index cad1a2a00..5258faaed 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -371,11 +371,7 @@ struct sqlite3_api_routines { /* Version 3.51.0 and later */ int (*set_errmsg)(sqlite3*,int,const char*); int (*db_status64)(sqlite3*,int,sqlite3_int64*,sqlite3_int64*,int); - /* Version 3.52.0 and later */ - void (*str_truncate)(sqlite3_str*,int); - void (*str_free)(sqlite3_str*); - int (*carray_bind)(sqlite3_stmt*,int,void*,int,int,void(*)(void*)); - int (*carray_bind_v2)(sqlite3_stmt*,int,void*,int,int,void(*)(void*),void*); + }; /* @@ -714,11 +710,6 @@ typedef int (*sqlite3_loadext_entry)( /* Version 3.51.0 and later */ #define sqlite3_set_errmsg sqlite3_api->set_errmsg #define sqlite3_db_status64 sqlite3_api->db_status64 -/* Version 3.52.0 and later */ -#define sqlite3_str_truncate sqlite3_api->str_truncate -#define sqlite3_str_free sqlite3_api->str_free -#define sqlite3_carray_bind sqlite3_api->carray_bind -#define sqlite3_carray_bind_v2 sqlite3_api->carray_bind_v2 #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) diff --git a/src/sqliteInt.h b/src/sqliteInt.h index c4a876861..fa53fe694 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -661,7 +661,6 @@ # define float sqlite_int64 # define fabs(X) ((X)<0?-(X):(X)) # define sqlite3IsOverflow(X) 0 -# define INFINITY (9223372036854775807LL) # ifndef SQLITE_BIG_DBL # define SQLITE_BIG_DBL (((sqlite3_int64)1)<<50) # endif @@ -1071,7 +1070,6 @@ typedef INT16_TYPE LogEst; #else # define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&7)==0) #endif -#define TWO_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&1)==0) /* ** Disable MMAP on platforms where it is known to not work @@ -1539,7 +1537,7 @@ struct Schema { ** The number of different kinds of things that can be limited ** using the sqlite3_limit() interface. */ -#define SQLITE_N_LIMIT (SQLITE_LIMIT_PARSER_DEPTH+1) +#define SQLITE_N_LIMIT (SQLITE_LIMIT_WORKER_THREADS+1) /* ** Lookaside malloc is a set of fixed-size buffers that can be used @@ -1693,7 +1691,6 @@ struct sqlite3 { u8 noSharedCache; /* True if no shared-cache backends */ u8 nSqlExec; /* Number of pending OP_SqlExec opcodes */ u8 eOpenState; /* Current condition of the connection */ - u8 nFpDigit; /* Significant digits to keep on double->text */ int nextPagesize; /* Pagesize after VACUUM if >0 */ i64 nChange; /* Value returned by sqlite3_changes() */ i64 nTotalChange; /* Value returned by sqlite3_total_changes() */ @@ -3588,6 +3585,19 @@ struct Upsert { /* ** An instance of the following structure contains all information ** needed to generate code for a single SELECT statement. +** +** See the header comment on the computeLimitRegisters() routine for a +** detailed description of the meaning of the iLimit and iOffset fields. +** +** addrOpenEphm[] entries contain the address of OP_OpenEphemeral opcodes. +** These addresses must be stored so that we can go back and fill in +** the P4_KEYINFO and P2 parameters later. Neither the KeyInfo nor +** the number of columns in P2 can be computed at the same time +** as the OP_OpenEphm instruction is coded because not +** enough information about the compound query is known at that point. +** The KeyInfo for addrOpenTran[0] and [1] contains collating sequences +** for the result set. The KeyInfo for addrOpenEphm[2] contains collating +** sequences for the ORDER BY clause. */ struct Select { u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ @@ -3595,6 +3605,7 @@ struct Select { u32 selFlags; /* Various SF_* values */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ u32 selId; /* Unique identifier number for this SELECT */ + int addrOpenEphm[2]; /* OP_OpenEphem opcodes related to this select */ ExprList *pEList; /* The fields of the result */ SrcList *pSrc; /* The FROM clause */ Expr *pWhere; /* The WHERE clause */ @@ -3626,7 +3637,7 @@ struct Select { #define SF_Resolved 0x0000004 /* Identifiers have been resolved */ #define SF_Aggregate 0x0000008 /* Contains agg functions or a GROUP BY */ #define SF_HasAgg 0x0000010 /* Contains aggregate functions */ -#define SF_ClonedRhsIn 0x0000020 /* Cloned RHS of an IN operator */ +#define SF_UsesEphemeral 0x0000020 /* Uses the OpenEphemeral opcode */ #define SF_Expanded 0x0000040 /* sqlite3SelectExpand() called on this */ #define SF_HasTypeInfo 0x0000080 /* FROM subqueries have Table metadata */ #define SF_Compound 0x0000100 /* Part of a compound query */ @@ -3636,14 +3647,14 @@ struct Select { #define SF_MinMaxAgg 0x0001000 /* Aggregate containing min() or max() */ #define SF_Recursive 0x0002000 /* The recursive part of a recursive CTE */ #define SF_FixedLimit 0x0004000 /* nSelectRow set by a constant LIMIT */ -/* 0x0008000 // available for reuse */ +#define SF_MaybeConvert 0x0008000 /* Need convertCompoundSelectToSubquery() */ #define SF_Converted 0x0010000 /* By convertCompoundSelectToSubquery() */ #define SF_IncludeHidden 0x0020000 /* Include hidden columns in output */ #define SF_ComplexResult 0x0040000 /* Result contains subquery or function */ #define SF_WhereBegin 0x0080000 /* Really a WhereBegin() call. Debug Only */ #define SF_WinRewrite 0x0100000 /* Window function rewrite accomplished */ #define SF_View 0x0200000 /* SELECT statement is a view */ -/* 0x0400000 // available for reuse */ +#define SF_NoopOrderBy 0x0400000 /* ORDER BY is ignored for this query */ #define SF_UFSrcCheck 0x0800000 /* Check pSrc as required by UPDATE...FROM */ #define SF_PushDown 0x1000000 /* Modified by WHERE-clause push-down opt */ #define SF_MultiPart 0x2000000 /* Has multiple incompatible PARTITIONs */ @@ -3663,6 +3674,11 @@ struct Select { ** by one of the following macros. The "SRT" prefix means "SELECT Result ** Type". ** +** SRT_Union Store results as a key in a temporary index +** identified by pDest->iSDParm. +** +** SRT_Except Remove results from the temporary index pDest->iSDParm. +** ** SRT_Exists Store a 1 in memory cell pDest->iSDParm if the result ** set is not empty. ** @@ -3726,28 +3742,30 @@ struct Select { ** table. (pDest->iSDParm) is the number of key columns in ** each index record in this case. */ -#define SRT_Exists 1 /* Store 1 if the result is not empty */ -#define SRT_Discard 2 /* Do not save the results anywhere */ -#define SRT_DistFifo 3 /* Like SRT_Fifo, but unique results only */ -#define SRT_DistQueue 4 /* Like SRT_Queue, but unique results only */ +#define SRT_Union 1 /* Store result as keys in an index */ +#define SRT_Except 2 /* Remove result from a UNION index */ +#define SRT_Exists 3 /* Store 1 if the result is not empty */ +#define SRT_Discard 4 /* Do not save the results anywhere */ +#define SRT_DistFifo 5 /* Like SRT_Fifo, but unique results only */ +#define SRT_DistQueue 6 /* Like SRT_Queue, but unique results only */ /* The DISTINCT clause is ignored for all of the above. Not that ** IgnorableDistinct() implies IgnorableOrderby() */ #define IgnorableDistinct(X) ((X->eDest)<=SRT_DistQueue) -#define SRT_Queue 5 /* Store result in an queue */ -#define SRT_Fifo 6 /* Store result as data with an automatic rowid */ +#define SRT_Queue 7 /* Store result in an queue */ +#define SRT_Fifo 8 /* Store result as data with an automatic rowid */ /* The ORDER BY clause is ignored for all of the above */ #define IgnorableOrderby(X) ((X->eDest)<=SRT_Fifo) -#define SRT_Output 7 /* Output each row of result */ -#define SRT_Mem 8 /* Store result in a memory cell */ -#define SRT_Set 9 /* Store results as keys in an index */ -#define SRT_EphemTab 10 /* Create transient tab and store like SRT_Table */ -#define SRT_Coroutine 11 /* Generate a single row of result */ -#define SRT_Table 12 /* Store result as data with an automatic rowid */ -#define SRT_Upfrom 13 /* Store result as data with rowid */ +#define SRT_Output 9 /* Output each row of result */ +#define SRT_Mem 10 /* Store result in a memory cell */ +#define SRT_Set 11 /* Store results as keys in an index */ +#define SRT_EphemTab 12 /* Create transient tab and store like SRT_Table */ +#define SRT_Coroutine 13 /* Generate a single row of result */ +#define SRT_Table 14 /* Store result as data with an automatic rowid */ +#define SRT_Upfrom 15 /* Store result as data with rowid */ /* ** An instance of this object describes where to put of the results of @@ -3883,12 +3901,17 @@ struct Parse { u8 nested; /* Number of nested calls to the parser/code generator */ u8 nTempReg; /* Number of temporary registers in aTempReg[] */ u8 isMultiWrite; /* True if statement may modify/insert multiple rows */ + u8 mayAbort; /* True if statement may throw an ABORT exception */ + u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ u8 disableLookaside; /* Number of times lookaside has been disabled */ u8 prepFlags; /* SQLITE_PREPARE_* flags */ u8 withinRJSubrtn; /* Nesting level for RIGHT JOIN body subroutines */ + u8 bHasExists; /* Has a correlated "EXISTS (SELECT ....)" expression */ u8 mSubrtnSig; /* mini Bloom filter on available SubrtnSig.selId */ u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ + u8 bReturning; /* Coding a RETURNING trigger */ u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ + u8 disableTriggers; /* True to disable triggers */ #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) u8 earlyCleanup; /* OOM inside sqlite3ParserAddCleanup() */ #endif @@ -3897,15 +3920,10 @@ struct Parse { u8 isCreate; /* CREATE TABLE, INDEX, or VIEW (but not TRIGGER) ** and ALTER TABLE ADD COLUMN. */ #endif - bft disableTriggers:1; /* True to disable triggers */ - bft mayAbort :1; /* True if statement may throw an ABORT exception */ - bft hasCompound :1; /* Need to invoke convertCompoundSelectToSubquery() */ - bft bReturning :1; /* Coding a RETURNING trigger */ - bft bHasExists :1; /* Has a correlated "EXISTS (SELECT ....)" expression */ - bft colNamesSet :1; /* TRUE after OP_ColumnName has been issued to pVdbe */ - bft bHasWith :1; /* True if statement contains WITH */ - bft okConstFactor:1; /* OK to factor out constants */ - bft checkSchema :1; /* Causes schema cookie check after an error */ + bft colNamesSet :1; /* TRUE after OP_ColumnName has been issued to pVdbe */ + bft bHasWith :1; /* True if statement contains WITH */ + bft okConstFactor :1; /* OK to factor out constants */ + bft checkSchema :1; /* Causes schema cookie check after an error */ int nRangeReg; /* Size of the temporary register block */ int iRangeReg; /* First register in temporary register block */ int nErr; /* Number of errors seen */ @@ -4134,19 +4152,19 @@ struct Trigger { ** orconf -> stores the ON CONFLICT algorithm ** pSelect -> The content to be inserted - either a SELECT statement or ** a VALUES clause. -** pSrc -> Table to insert into. +** zTarget -> Dequoted name of the table to insert into. ** pIdList -> If this is an INSERT INTO ... (<column-names>) VALUES ... ** statement, then this stores the column-names to be ** inserted into. ** pUpsert -> The ON CONFLICT clauses for an Upsert ** ** (op == TK_DELETE) -** pSrc -> Table to delete from +** zTarget -> Dequoted name of the table to delete from. ** pWhere -> The WHERE clause of the DELETE statement if one is specified. ** Otherwise NULL. ** ** (op == TK_UPDATE) -** pSrc -> Table to update, followed by any FROM clause tables. +** zTarget -> Dequoted name of the table to update. ** pWhere -> The WHERE clause of the UPDATE statement if one is specified. ** Otherwise NULL. ** pExprList -> A list of the columns to update and the expressions to update @@ -4166,7 +4184,8 @@ struct TriggerStep { u8 orconf; /* OE_Rollback etc. */ Trigger *pTrig; /* The trigger that this step is a part of */ Select *pSelect; /* SELECT statement or RHS of INSERT INTO SELECT ... */ - SrcList *pSrc; /* Table to insert/update/delete */ + char *zTarget; /* Target table for DELETE, UPDATE, INSERT */ + SrcList *pFrom; /* FROM clause for UPDATE statement (if any) */ Expr *pWhere; /* The WHERE clause for DELETE or UPDATE steps */ ExprList *pExprList; /* SET clause for UPDATE, or RETURNING clause */ IdList *pIdList; /* Column names for INSERT */ @@ -4249,11 +4268,10 @@ typedef struct { /* ** Allowed values for mInitFlags */ -#define INITFLAG_AlterMask 0x0007 /* Types of ALTER */ +#define INITFLAG_AlterMask 0x0003 /* Types of ALTER */ #define INITFLAG_AlterRename 0x0001 /* Reparse after a RENAME */ #define INITFLAG_AlterDrop 0x0002 /* Reparse after a DROP COLUMN */ #define INITFLAG_AlterAdd 0x0003 /* Reparse after an ADD COLUMN */ -#define INITFLAG_AlterDropCons 0x0004 /* Reparse after an ADD COLUMN */ /* Tuning parameters are set using SQLITE_TESTCTRL_TUNE and are controlled ** on debug-builds of the CLI using ".testctrl tune ID VALUE". Tuning @@ -4383,7 +4401,6 @@ struct Walker { NameContext *pNC; /* Naming context */ int n; /* A counter */ int iCur; /* A cursor number */ - int sz; /* String literal length */ SrcList *pSrcList; /* FROM clause */ struct CCurHint *pCCurHint; /* Used by codeCursorHint() */ struct RefSrcList *pRefSrcList; /* sqlite3ReferencesSrcList() */ @@ -4788,20 +4805,7 @@ int sqlite3LookasideUsed(sqlite3*,int*); sqlite3_mutex *sqlite3Pcache1Mutex(void); sqlite3_mutex *sqlite3MallocMutex(void); - -/* The SQLITE_THREAD_MISUSE_WARNINGS compile-time option used to be called -** SQLITE_ENABLE_MULTITHREADED_CHECKS. Keep that older macro for backwards -** compatibility, at least for a while... */ -#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS -# define SQLITE_THREAD_MISUSE_WARNINGS 1 -#endif - -/* SQLITE_THREAD_MISUSE_ABORT implies SQLITE_THREAD_MISUSE_WARNINGS */ -#ifdef SQLITE_THREAD_MISUSE_ABORT -# define SQLITE_THREAD_MISUSE_WARNINGS 1 -#endif - -#if defined(SQLITE_THREAD_MISUSE_WARNINGS) && !defined(SQLITE_MUTEX_OMIT) +#if defined(SQLITE_ENABLE_MULTITHREADED_CHECKS) && !defined(SQLITE_MUTEX_OMIT) void sqlite3MutexWarnOnContention(sqlite3_mutex*); #else # define sqlite3MutexWarnOnContention(x) @@ -4835,12 +4839,12 @@ struct PrintfArguments { ** value into an approximate decimal representation. */ struct FpDecode { + char sign; /* '+' or '-' */ + char isSpecial; /* 1: Infinity 2: NaN */ int n; /* Significant digits in the decode */ int iDP; /* Location of the decimal point */ char *z; /* Start of significant digits */ - char zBuf[20]; /* Storage for significant digits */ - char sign; /* '+' or '-' */ - char isSpecial; /* 1: Infinity 2: NaN */ + char zBuf[24]; /* Storage for significant digits */ }; void sqlite3FpDecode(FpDecode*,double,int,int); @@ -4929,7 +4933,6 @@ int sqlite3NoTempsInRange(Parse*,int,int); #endif Expr *sqlite3ExprAlloc(sqlite3*,int,const Token*,int); Expr *sqlite3Expr(sqlite3*,int,const char*); -Expr *sqlite3ExprInt32(sqlite3*,int); void sqlite3ExprAttachSubtrees(sqlite3*,Expr*,Expr*,Expr*); Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*); void sqlite3PExprAddSelect(Parse*, Expr*, Select*); @@ -5181,7 +5184,6 @@ int sqlite3ExprContainsSubquery(Expr*); int sqlite3ExprIsInteger(const Expr*, int*, Parse*); int sqlite3ExprCanBeNull(const Expr*); int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); -int sqlite3ExprIsLikeOperator(const Expr*); int sqlite3IsRowid(const char*); const char *sqlite3RowidAlias(Table *pTab); void sqlite3GenerateRowDelete( @@ -5250,16 +5252,17 @@ void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,int); void sqlite3DeleteTriggerStep(sqlite3*, TriggerStep*); TriggerStep *sqlite3TriggerSelectStep(sqlite3*,Select*, const char*,const char*); - TriggerStep *sqlite3TriggerInsertStep(Parse*,SrcList*, IdList*, + TriggerStep *sqlite3TriggerInsertStep(Parse*,Token*, IdList*, Select*,u8,Upsert*, const char*,const char*); - TriggerStep *sqlite3TriggerUpdateStep(Parse*,SrcList*,SrcList*,ExprList*, + TriggerStep *sqlite3TriggerUpdateStep(Parse*,Token*,SrcList*,ExprList*, Expr*, u8, const char*,const char*); - TriggerStep *sqlite3TriggerDeleteStep(Parse*,SrcList*, Expr*, + TriggerStep *sqlite3TriggerDeleteStep(Parse*,Token*, Expr*, const char*,const char*); void sqlite3DeleteTrigger(sqlite3*, Trigger*); void sqlite3UnlinkAndDeleteTrigger(sqlite3*,int,const char*); u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Table*,int); + SrcList *sqlite3TriggerStepSrc(Parse*, TriggerStep*); # define sqlite3ParseToplevel(p) ((p)->pToplevel ? (p)->pToplevel : (p)) # define sqlite3IsToplevel(p) ((p)->pToplevel==0) #else @@ -5273,6 +5276,7 @@ void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,int); # define sqlite3ParseToplevel(p) p # define sqlite3IsToplevel(p) 1 # define sqlite3TriggerColmask(A,B,C,D,E,F,G) 0 +# define sqlite3TriggerStepSrc(A,B) 0 #endif int sqlite3JoinType(Parse*, Token*, Token*, Token*); @@ -5305,7 +5309,7 @@ int sqlite3FixTriggerStep(DbFixer*, TriggerStep*); int sqlite3RealSameAsInt(double,sqlite3_int64); i64 sqlite3RealToI64(double); int sqlite3Int64ToText(i64,char*); -int sqlite3AtoF(const char *z, double*); +int sqlite3AtoF(const char *z, double*, int, u8); int sqlite3GetInt32(const char *, int*); int sqlite3GetUInt32(const char*, u32*); int sqlite3Atoi(const char*); @@ -5449,13 +5453,10 @@ void sqlite3Reindex(Parse*, Token*, Token*); void sqlite3AlterFunctions(void); void sqlite3AlterRenameTable(Parse*, SrcList*, Token*); void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*); -void sqlite3AlterDropConstraint(Parse*,SrcList*,Token*,Token*); -void sqlite3AlterAddConstraint(Parse*,SrcList*,Token*,Token*,const char*,int); -void sqlite3AlterSetNotNull(Parse*, SrcList*, Token*, Token*); i64 sqlite3GetToken(const unsigned char *, int *); void sqlite3NestedParse(Parse*, const char*, ...); void sqlite3ExpirePreparedStatements(sqlite3*, int); -void sqlite3CodeRhsOfIN(Parse*, Expr*, int, int); +void sqlite3CodeRhsOfIN(Parse*, Expr*, int); int sqlite3CodeSubselect(Parse*, Expr*); void sqlite3SelectPrep(Parse*, Select*, NameContext*); int sqlite3ExpandSubquery(Parse*, SrcItem*); diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index ecbd0858e..ee467836a 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -81,42 +81,21 @@ ** It used to be the case that setting this value to zero would ** turn the limit off. That is no longer true. It is not possible ** to turn this limit off. -** -** The hard limit is the largest possible 32-bit signed integer less -** 1024, or 2147482624. */ #ifndef SQLITE_MAX_SQL_LENGTH # define SQLITE_MAX_SQL_LENGTH 1000000000 #endif /* -** The maximum depth of an expression tree. The expression tree depth -** is also limited indirectly by SQLITE_MAX_SQL_LENGTH and by -** SQLITE_MAX_PARSER_DEPTH. Reducing the maximum complexity of -** expressions can help prevent excess memory usage by hostile SQL. -** -** A value of 0 for this compile-time option causes all expression -** depth limiting code to be omitted. +** The maximum depth of an expression tree. This is limited to +** some extent by SQLITE_MAX_SQL_LENGTH. But sometime you might +** want to place more severe limits on the complexity of an +** expression. A value of 0 means that there is no limit. */ #ifndef SQLITE_MAX_EXPR_DEPTH # define SQLITE_MAX_EXPR_DEPTH 1000 #endif -/* -** The maximum depth of the LALR(1) stack used in the parser that -** interprets SQL inputs. The parser stack depth can also be limited -** indirectly by SQLITE_MAX_SQL_LENGTH. Limiting the parser stack -** depth can help prevent excess memory usage and excess CPU stack -** usage when processing hostile SQL. -** -** Prior to version 3.45.0 (2024-01-15), the parser stack was -** hard-coded to 100 entries, and that worked fine for almost all -** applications. So the upper bound on this limit need not be large. -*/ -#ifndef SQLITE_MAX_PARSER_DEPTH -# define SQLITE_MAX_PARSER_DEPTH 2500 -#endif - /* ** The maximum number of terms in a compound SELECT statement. ** The code generator for compound SELECT statements does one diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 2c7918926..02a4d84e4 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -124,15 +124,6 @@ /* Forward declaration */ typedef struct SqliteDb SqliteDb; -/* Add -DSQLITE_ENABLE_QRF_IN_TCL to add the Query Result Formatter (QRF) -** into the build of the TCL extension, when building using separate -** source files. The QRF is included automatically when building from -** the tclsqlite3.c amalgamation. -*/ -#if defined(SQLITE_ENABLE_QRF_IN_TCL) -#include "qrf.h" -#endif - /* ** New SQL functions can be created as TCL scripts. Each such function ** is described by an instance of the following structure. @@ -2044,367 +2035,6 @@ static void DbHookCmd( sqlite3_wal_hook(db, (pDb->pWalHook?DbWalHandler:0), pDb); } -/* -** Implementation of the "db format" command. -** -** Based on provided options, format the results of the SQL statement(s) -** provided into human-readable form using the Query Result Formatter (QRF) -** and return the resuling text. -** -** Syntax: db format OPTIONS SQL -** -** OPTIONS may be: -** -** -style ("auto"|"box"|"column"|...) Output style -** -esc ("auto"|"off"|"ascii"|"symbol") How to deal with ctrl chars -** -text ("auto"|"off"|"sql"|"csv"|...) How to escape TEXT values -** -title ("auto"|"off"|"sql"|...|"off") How to escape column names -** -blob ("auto"|"text"|"sql"|...) How to escape BLOB values -** -wordwrap ("auto"|"off"|"on") Try to wrap at word boundry? -** -textjsonb ("auto"|"off"|"on") Auto-convert JSONB to text? -** -splitcolumn ("auto"|"off"|"on") Enable split-column mode -** -defaultalign ("auto"|"left"|...) Default alignment -** -titalalign ("auto"|"left"|"right"|...) Default column name alignment -** -border ("auto"|"off"|"on") Border for box and table styles -** -wrap NUMBER Max width of any single column -** -screenwidth NUMBER Width of the display TTY -** -linelimit NUMBER Max lines for any cell -** -charlimit NUMBER Content truncated to this size -** -titlelimit NUMBER Max width of column titles -** -align LIST-OF-ALIGNMENT Alignment of columns -** -widths LIST-OF-NUMBERS Widths for individual columns -** -columnsep TEXT Column separator text -** -rowsep TEXT Row separator text -** -tablename TEXT Table name for style "insert" -** -null TEXT Text for NULL values -** -** A mapping from TCL "format" command options to sqlite3_qrf_spec fields -** is below. Use this to reference the QRF documentation: -** -** TCL Option spec field -** ---------- ---------- -** -style eStyle -** -esc eEsc -** -text eText -** -title eTitle, bTitle -** -blob eBlob -** -wordwrap bWordWrap -** -textjsonb bTextJsonb -** -splitcolumn bSplitColumn -** -defaultalign eDfltAlign -** -titlealign eTitleAlign -** -border bBorder -** -wrap nWrap -** -screenwidth nScreenWidth -** -linelimit nLineLimit -** -charlimit nCharLimit -** -titlelimit nTitleLimit -** -align nAlign, aAlign -** -widths nWidth, aWidth -** -columnsep zColumnSep -** -rowsep zRowSep -** -tablename zTableName -** -null zNull -*/ -static int dbQrf(SqliteDb *pDb, int objc, Tcl_Obj *const*objv){ -#ifndef SQLITE_QRF_H - Tcl_SetResult(pDb->interp, "QRF not available in this build", TCL_VOLATILE); - return TCL_ERROR; -#else - char *zResult = 0; /* Result to be returned */ - const char *zSql = 0; /* SQL to run */ - int i; /* Loop counter */ - int rc; /* Result code */ - sqlite3_qrf_spec qrf; /* Formatting spec */ - static const char *azAlign[] = { - "auto", "bottom", "c", - "center", "e", "left", - "middle", "n", "ne", - "nw", "right", "s", - "se", "sw", "top", - "w", 0 - }; - static const unsigned char aAlignMap[] = { - QRF_ALIGN_Auto, QRF_ALIGN_Bottom, QRF_ALIGN_C, - QRF_ALIGN_Center, QRF_ALIGN_E, QRF_ALIGN_Left, - QRF_ALIGN_Middle, QRF_ALIGN_N, QRF_ALIGN_NE, - QRF_ALIGN_NW, QRF_ALIGN_Right, QRF_ALIGN_S, - QRF_ALIGN_SE, QRF_ALIGN_SW, QRF_ALIGN_Top, - QRF_ALIGN_W - }; - - memset(&qrf, 0, sizeof(qrf)); - qrf.iVersion = 1; - qrf.pzOutput = &zResult; - for(i=2; i<objc; i++){ - const char *zArg = Tcl_GetString(objv[i]); - const char *azBool[] = { "auto", "yes", "no", "on", "off", 0 }; - const unsigned char aBoolMap[] = { 0, 2, 1, 2, 1 }; - if( zArg[0]!='-' ){ - if( zSql ){ - Tcl_AppendResult(pDb->interp, "unknown argument: ", zArg, (char*)0); - rc = TCL_ERROR; - goto format_failed; - } - zSql = zArg; - }else if( i==objc-1 ){ - Tcl_AppendResult(pDb->interp, "option has no argument: ", zArg, (char*)0); - rc = TCL_ERROR; - goto format_failed; - }else if( strcmp(zArg,"-style")==0 ){ - static const char *azStyles[] = { - "auto", "box", "column", - "count", "csv", "eqp", - "explain", "html", "insert", - "jobject", "json", "line", - "list", "markdown", "quote", - "stats", "stats-est", "stats-vm", - "table", 0 - }; - static unsigned char aStyleMap[] = { - QRF_STYLE_Auto, QRF_STYLE_Box, QRF_STYLE_Column, - QRF_STYLE_Count, QRF_STYLE_Csv, QRF_STYLE_Eqp, - QRF_STYLE_Explain, QRF_STYLE_Html, QRF_STYLE_Insert, - QRF_STYLE_JObject, QRF_STYLE_Json, QRF_STYLE_Line, - QRF_STYLE_List, QRF_STYLE_Markdown, QRF_STYLE_Quote, - QRF_STYLE_Stats, QRF_STYLE_StatsEst, QRF_STYLE_StatsVm, - QRF_STYLE_Table - }; - int style; - rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azStyles, - "format style (-style)", 0, &style); - if( rc ) goto format_failed; - qrf.eStyle = aStyleMap[style]; - i++; - }else if( strcmp(zArg,"-esc")==0 ){ - static const char *azEsc[] = { - "ascii", "auto", "off", "symbol", 0 - }; - static unsigned char aEscMap[] = { - QRF_ESC_Ascii, QRF_ESC_Auto, QRF_ESC_Off, QRF_ESC_Symbol - }; - int esc; - rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azEsc, - "control character escape (-esc)", 0, &esc); - if( rc ) goto format_failed; - qrf.eEsc = aEscMap[esc]; - i++; - }else if( strcmp(zArg,"-text")==0 || strcmp(zArg, "-title")==0 ){ - /* NB: --title can be "off" or "on but --text may not be. Thus we put - ** the "off" and "on" choices first and start the search on the - ** thrid element of the array when processing --text */ - static const char *azText[] = { "off", "on", - "auto", "csv", "html", - "json", "plain", "relaxed", - "sql", "tcl", 0 - }; - static unsigned char aTextMap[] = { - QRF_TEXT_Auto, QRF_TEXT_Csv, QRF_TEXT_Html, - QRF_TEXT_Json, QRF_TEXT_Plain, QRF_TEXT_Relaxed, - QRF_TEXT_Sql, QRF_TEXT_Tcl - }; - int txt; - int k = zArg[2]=='e'; - rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], &azText[k*2], zArg, - 0, &txt); - if( rc ) goto format_failed; - if( k ){ - qrf.eText = aTextMap[txt]; - }else if( txt<=1 ){ - qrf.bTitles = txt ? QRF_Yes : QRF_No; - qrf.eTitle = QRF_TEXT_Auto; - }else{ - qrf.bTitles = QRF_Yes; - qrf.eTitle = aTextMap[txt-2]; - } - i++; - }else if( strcmp(zArg,"-blob")==0 ){ - static const char *azBlob[] = { - "auto", "hex", "json", - "tcl", "text", "sql", - "size", 0 - }; - static unsigned char aBlobMap[] = { - QRF_BLOB_Auto, QRF_BLOB_Hex, QRF_BLOB_Json, - QRF_BLOB_Tcl, QRF_BLOB_Text, QRF_BLOB_Sql, - QRF_BLOB_Size - }; - int blob; - rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azBlob, - "BLOB encoding (-blob)", 0, &blob); - if( rc ) goto format_failed; - qrf.eBlob = aBlobMap[blob]; - i++; - }else if( strcmp(zArg,"-wordwrap")==0 ){ - int v = 0; - rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azBool, - "-wordwrap", 0, &v); - if( rc ) goto format_failed; - qrf.bWordWrap = aBoolMap[v]; - i++; - }else if( strcmp(zArg,"-textjsonb")==0 - || strcmp(zArg,"-splitcolumn")==0 - || strcmp(zArg,"-border")==0 - ){ - int v = 0; - rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azBool, - zArg, 0, &v); - if( rc ) goto format_failed; - if( zArg[1]=='t' ){ - qrf.bTextJsonb = aBoolMap[v]; - }else if( zArg[1]=='b' ){ - qrf.bBorder = aBoolMap[v]; - }else{ - qrf.bSplitColumn = aBoolMap[v]; - } - i++; - }else if( strcmp(zArg,"-defaultalign")==0 || strcmp(zArg,"-titlealign")==0){ - int ax = 0; - rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azAlign, - zArg[1]=='d' ? "default alignment (-defaultalign)" : - "title alignment (-titlealign)", - 0, &ax); - if( rc ) goto format_failed; - if( zArg[1]=='d' ){ - qrf.eDfltAlign = aAlignMap[ax]; - }else{ - qrf.eTitleAlign = aAlignMap[ax]; - } - i++; - }else if( strcmp(zArg,"-wrap")==0 - || strcmp(zArg,"-screenwidth")==0 - || strcmp(zArg,"-linelimit")==0 - || strcmp(zArg,"-titlelimit")==0 - ){ - int v = 0; - rc = Tcl_GetIntFromObj(pDb->interp, objv[i+1], &v); - if( rc ) goto format_failed; - if( v<QRF_MIN_WIDTH ){ - v = QRF_MIN_WIDTH; - }else if( v>QRF_MAX_WIDTH ){ - v = QRF_MAX_WIDTH; - } - if( zArg[1]=='w' ){ - qrf.nWrap = v; - }else if( zArg[1]=='s' ){ - qrf.nScreenWidth = v; - }else if( zArg[1]=='t' ){ - qrf.nTitleLimit = v; - }else{ - qrf.nLineLimit = v; - } - i++; - }else if( strcmp(zArg,"-charlimit")==0 ){ - int v = 0; - rc = Tcl_GetIntFromObj(pDb->interp, objv[i+1], &v); - if( rc ) goto format_failed; - if( v<0 ) v = 0; - qrf.nCharLimit = v; - i++; - }else if( strcmp(zArg,"-align")==0 ){ - Tcl_Size n = 0; - int jj; - rc = Tcl_ListObjLength(pDb->interp, objv[i+1], &n); - if( rc ) goto format_failed; - sqlite3_free(qrf.aAlign); - qrf.aAlign = sqlite3_malloc64( (n+1)*sizeof(qrf.aAlign[0]) ); - if( qrf.aAlign==0 ){ - Tcl_AppendResult(pDb->interp, "out of memory", (char*)0); - rc = TCL_ERROR; - goto format_failed; - } - memset(qrf.aAlign, 0, (n+1)*sizeof(qrf.aAlign[0])); - qrf.nAlign = n; - for(jj=0; jj<n; jj++){ - int x; - Tcl_Obj *pTerm; - rc = Tcl_ListObjIndex(pDb->interp, objv[i+1], jj, &pTerm); - if( rc ) goto format_failed; - rc = Tcl_GetIndexFromObj(pDb->interp, pTerm, azAlign, - "column alignment (-align)", 0, &x); - if( rc ) goto format_failed; - qrf.aAlign[jj] = aAlignMap[x]; - } - i++; - }else if( strcmp(zArg,"-widths")==0 ){ - Tcl_Size n = 0; - int jj; - rc = Tcl_ListObjLength(pDb->interp, objv[i+1], &n); - if( rc ) goto format_failed; - sqlite3_free(qrf.aWidth); - qrf.aWidth = sqlite3_malloc64( (n+1)*sizeof(qrf.aWidth[0]) ); - if( qrf.aWidth==0 ){ - Tcl_AppendResult(pDb->interp, "out of memory", (char*)0); - rc = TCL_ERROR; - goto format_failed; - } - memset(qrf.aWidth, 0, (n+1)*sizeof(qrf.aWidth[0])); - qrf.nWidth = n; - for(jj=0; jj<n; jj++){ - Tcl_Obj *pTerm; - int v; - rc = Tcl_ListObjIndex(pDb->interp, objv[i+1], jj, &pTerm); - if( rc ) goto format_failed; - rc = Tcl_GetIntFromObj(pDb->interp, pTerm, &v); - if( v<(-QRF_MAX_WIDTH) ){ - v = -QRF_MAX_WIDTH; - }else if( v>QRF_MAX_WIDTH ){ - v = QRF_MAX_WIDTH; - } - qrf.aWidth[jj] = (short int)v; - } - i++; - }else if( strcmp(zArg,"-columnsep")==0 ){ - qrf.zColumnSep = Tcl_GetString(objv[i+1]); - i++; - }else if( strcmp(zArg,"-rowsep")==0 ){ - qrf.zRowSep = Tcl_GetString(objv[i+1]); - i++; - }else if( strcmp(zArg,"-tablename")==0 ){ - qrf.zTableName = Tcl_GetString(objv[i+1]); - i++; - }else if( strcmp(zArg,"-null")==0 ){ - qrf.zNull = Tcl_GetString(objv[i+1]); - i++; - }else if( strcmp(zArg,"-version")==0 ){ - /* Undocumented. Testing use only */ - qrf.iVersion = atoi(Tcl_GetString(objv[i+1])); - i++; - }else{ - Tcl_AppendResult(pDb->interp, "unknown option: ", zArg, (char*)0); - rc = TCL_ERROR; - goto format_failed; - } - } - while( zSql && zSql[0] ){ - SqlPreparedStmt *pStmt = 0; /* Next statement to run */ - char *zErr = 0; /* Error message from QRF */ - - rc = dbPrepareAndBind(pDb, zSql, &zSql, &pStmt); - if( rc ) goto format_failed; - if( pStmt==0 ) continue; - rc = sqlite3_format_query_result(pStmt->pStmt, &qrf, &zErr); - dbReleaseStmt(pDb, pStmt, 0); - if( rc ){ - Tcl_SetResult(pDb->interp, zErr, TCL_VOLATILE); - sqlite3_free(zErr); - rc = TCL_ERROR; - goto format_failed; - } - } - Tcl_SetResult(pDb->interp, zResult, TCL_VOLATILE); - rc = TCL_OK; - /* Fall through...*/ - -format_failed: - sqlite3_free(qrf.aWidth); - sqlite3_free(qrf.aAlign); - sqlite3_free(zResult); - return rc; - -#endif -} - /* ** The "sqlite" command below creates a new Tcl command for each ** connection it opens to an SQLite database. This routine is invoked @@ -2434,15 +2064,15 @@ static int SQLITE_TCLAPI DbObjCmd( "commit_hook", "complete", "config", "copy", "deserialize", "enable_load_extension", "errorcode", "erroroffset", "eval", - "exists", "format", "function", - "incrblob", "interrupt", "last_insert_rowid", - "nullvalue", "onecolumn", "preupdate", - "profile", "progress", "rekey", - "restore", "rollback_hook", "serialize", - "status", "timeout", "total_changes", - "trace", "trace_v2", "transaction", - "unlock_notify", "update_hook", "version", - "wal_hook", 0 + "exists", "function", "incrblob", + "interrupt", "last_insert_rowid", "nullvalue", + "onecolumn", "preupdate", "profile", + "progress", "rekey", "restore", + "rollback_hook", "serialize", "status", + "timeout", "total_changes", "trace", + "trace_v2", "transaction", "unlock_notify", + "update_hook", "version", "wal_hook", + 0 }; enum DB_enum { DB_AUTHORIZER, DB_BACKUP, DB_BIND_FALLBACK, @@ -2451,15 +2081,14 @@ static int SQLITE_TCLAPI DbObjCmd( DB_COMMIT_HOOK, DB_COMPLETE, DB_CONFIG, DB_COPY, DB_DESERIALIZE, DB_ENABLE_LOAD_EXTENSION, DB_ERRORCODE, DB_ERROROFFSET, DB_EVAL, - DB_EXISTS, DB_FORMAT, DB_FUNCTION, - DB_INCRBLOB, DB_INTERRUPT, DB_LAST_INSERT_ROWID, - DB_NULLVALUE, DB_ONECOLUMN, DB_PREUPDATE, - DB_PROFILE, DB_PROGRESS, DB_REKEY, - DB_RESTORE, DB_ROLLBACK_HOOK, DB_SERIALIZE, - DB_STATUS, DB_TIMEOUT, DB_TOTAL_CHANGES, - DB_TRACE, DB_TRACE_V2, DB_TRANSACTION, - DB_UNLOCK_NOTIFY, DB_UPDATE_HOOK, DB_VERSION, - DB_WAL_HOOK + DB_EXISTS, DB_FUNCTION, DB_INCRBLOB, + DB_INTERRUPT, DB_LAST_INSERT_ROWID, DB_NULLVALUE, + DB_ONECOLUMN, DB_PREUPDATE, DB_PROFILE, + DB_PROGRESS, DB_REKEY, DB_RESTORE, + DB_ROLLBACK_HOOK, DB_SERIALIZE, DB_STATUS, + DB_TIMEOUT, DB_TOTAL_CHANGES, DB_TRACE, + DB_TRACE_V2, DB_TRANSACTION, DB_UNLOCK_NOTIFY, + DB_UPDATE_HOOK, DB_VERSION, DB_WAL_HOOK, }; /* don't leave trailing commas on DB_enum, it confuses the AIX xlc compiler */ @@ -3349,18 +2978,6 @@ static int SQLITE_TCLAPI DbObjCmd( break; } - /* - ** $db format [OPTIONS] SQL - ** - ** Run the SQL statement(s) given as the final argument. Use the - ** Query Result Formatter extension of SQLite to format the output as - ** text and return that text. - */ - case DB_FORMAT: { - rc = dbQrf(pDb, objc, objv); - break; - } - /* ** $db function NAME [OPTIONS] SCRIPT ** diff --git a/src/test1.c b/src/test1.c index 3ca5c837a..f8e83dc42 100644 --- a/src/test1.c +++ b/src/test1.c @@ -4411,7 +4411,7 @@ static void delIntptr(void *p){ } /* -** bind_carray_intptr STMT IPARAM INT-0 INT-1 INT-2... +** bind_carray_intptr STMT IPARAM INT0 INT1 INT2... */ static int SQLITE_TCLAPI bind_carray_intptr( void * clientData, @@ -4455,7 +4455,6 @@ static int SQLITE_TCLAPI bind_carray_intptr( ** -malloc ** -transient ** -static -** -v2 ** -int32 ** -int64 ** -double @@ -4478,7 +4477,6 @@ static int SQLITE_TCLAPI test_carray_bind( void *aData = 0; int isTransient = 0; int isStatic = 0; - int isV2 = 0; int isMalloc = 0; /* True to use custom xDel function */ int idx; int i, j; @@ -4511,22 +4509,16 @@ static int SQLITE_TCLAPI test_carray_bind( const char *z = Tcl_GetString(objv[i]); if( strcmp(z, "-transient")==0 ){ isTransient = 1; - isStatic = isMalloc = 0; xDel = SQLITE_TRANSIENT; }else if( strcmp(z, "-static")==0 ){ isStatic = 1; - isMalloc = isTransient = 0; xDel = SQLITE_STATIC; }else if( strcmp(z, "-malloc")==0 ){ isMalloc = 1; - isStatic = isTransient = 0; xDel = testCarrayFree; }else - if( strcmp(z, "-v2")==0 ){ - isV2 = 1; - }else if( strcmp(z, "-int32")==0 ){ eType = 0; /* CARRAY_INT32 */ }else @@ -4695,20 +4687,7 @@ static int SQLITE_TCLAPI test_carray_bind( if( rc==SQLITE_OK ){ if( mFlagsOverride==0 ) mFlagsOverride = eType; - if( isV2 ){ - void *pDel; - if( xDel==testCarrayFree ){ - u8 *p2 = (u8*)aData; - pDel = (void*)&p2[-16]; - xDel = sqlite3_free; - }else{ - pDel = aData; - } - rc = sqlite3_carray_bind_v2(pStmt, idx, aData, nData, mFlagsOverride, - xDel, pDel); - }else{ - rc = sqlite3_carray_bind(pStmt, idx, aData, nData, mFlagsOverride, xDel); - } + rc = sqlite3_carray_bind(pStmt, idx, aData, nData, mFlagsOverride, xDel); } if( isTransient ){ if( eType==3 && aData ){ @@ -7391,7 +7370,6 @@ static int SQLITE_TCLAPI test_limit( { "SQLITE_LIMIT_SQL_LENGTH", SQLITE_LIMIT_SQL_LENGTH }, { "SQLITE_LIMIT_COLUMN", SQLITE_LIMIT_COLUMN }, { "SQLITE_LIMIT_EXPR_DEPTH", SQLITE_LIMIT_EXPR_DEPTH }, - { "SQLITE_LIMIT_PARSER_DEPTH", SQLITE_LIMIT_PARSER_DEPTH }, { "SQLITE_LIMIT_COMPOUND_SELECT", SQLITE_LIMIT_COMPOUND_SELECT }, { "SQLITE_LIMIT_VDBE_OP", SQLITE_LIMIT_VDBE_OP }, { "SQLITE_LIMIT_FUNCTION_ARG", SQLITE_LIMIT_FUNCTION_ARG }, @@ -7403,7 +7381,7 @@ static int SQLITE_TCLAPI test_limit( /* Out of range test cases */ { "SQLITE_LIMIT_TOOSMALL", -1, }, - { "SQLITE_LIMIT_TOOBIG", SQLITE_LIMIT_PARSER_DEPTH+1 }, + { "SQLITE_LIMIT_TOOBIG", SQLITE_LIMIT_WORKER_THREADS+1 }, }; int i, id = 0; int val; @@ -7697,8 +7675,7 @@ static int SQLITE_TCLAPI test_wal_checkpoint_v2( int nCkpt = -555; Tcl_Obj *pRet; - const char * aMode[] = {"noop", "passive", "full", "restart", "truncate", 0}; - assert( SQLITE_CHECKPOINT_NOOP==-1 ); + const char * aMode[] = { "passive", "full", "restart", "truncate", 0 }; assert( SQLITE_CHECKPOINT_PASSIVE==0 ); assert( SQLITE_CHECKPOINT_FULL==1 ); assert( SQLITE_CHECKPOINT_RESTART==2 ); @@ -7712,15 +7689,12 @@ static int SQLITE_TCLAPI test_wal_checkpoint_v2( if( objc==4 ){ zDb = Tcl_GetString(objv[3]); } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) || ( + TCL_OK!=Tcl_GetIntFromObj(0, objv[2], &eMode) + && TCL_OK!=Tcl_GetIndexFromObj(interp, objv[2], aMode, "mode", 0, &eMode) + )){ return TCL_ERROR; } - if( TCL_OK!=Tcl_GetIntFromObj(0, objv[2], &eMode) ){ - if( TCL_OK!=Tcl_GetIndexFromObj(interp, objv[2], aMode, "mode", 0,&eMode) ){ - return TCL_ERROR; - } - eMode = eMode - 1; - } rc = sqlite3_wal_checkpoint_v2(db, zDb, eMode, &nLog, &nCkpt); if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ @@ -8644,7 +8618,6 @@ static int SQLITE_TCLAPI test_sqlite3_db_config( { "ATTACH_CREATE", SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE }, { "ATTACH_WRITE", SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE }, { "COMMENTS", SQLITE_DBCONFIG_ENABLE_COMMENTS }, - { "FP_DIGITS", SQLITE_DBCONFIG_FP_DIGITS }, }; int i; int v = 0; diff --git a/src/test_bestindex.c b/src/test_bestindex.c index 963abfec0..f6b5db0fb 100644 --- a/src/test_bestindex.c +++ b/src/test_bestindex.c @@ -828,6 +828,7 @@ static int tclUpdate( tcl_vtab *pTab = (tcl_vtab*)tab; Tcl_Interp *interp = pTab->interp; Tcl_Obj *pEval = Tcl_DuplicateObj(pTab->pCmd); + Tcl_Obj *pRes = 0; int rc = TCL_OK; Tcl_IncrRefCount(pEval); diff --git a/src/test_config.c b/src/test_config.c index bebf8625a..3dbef3c9a 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -82,7 +82,7 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options","configslower","1.0",TCL_GLOBAL_ONLY); #endif -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT Tcl_SetVar2(interp, "sqlite_options", "curdir", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "curdir", "0", TCL_GLOBAL_ONLY); @@ -94,6 +94,12 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "win32malloc", "0", TCL_GLOBAL_ONLY); #endif +#if defined(SQLITE_OS_WINRT) && SQLITE_OS_WINRT + Tcl_SetVar2(interp, "sqlite_options", "winrt", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "winrt", "0", TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_DEBUG Tcl_SetVar2(interp, "sqlite_options", "debug", "1", TCL_GLOBAL_ONLY); #else @@ -679,14 +685,6 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); Tcl_SetVar2(interp, "sqlite_options", "trace", "1", TCL_GLOBAL_ONLY); #endif -#ifdef SQLITE_THREAD_MISUSE_WARNINGS - Tcl_SetVar2(interp, "sqlite_options", "thread_misuse_warnings", - "1", TCL_GLOBAL_ONLY); -#else - Tcl_SetVar2(interp, "sqlite_options", "thread_misuse_warnings", - "0", TCL_GLOBAL_ONLY); -#endif - #ifdef SQLITE_OMIT_TRIGGER Tcl_SetVar2(interp, "sqlite_options", "trigger", "0", TCL_GLOBAL_ONLY); #else diff --git a/src/test_quota.c b/src/test_quota.c index 3eeacc7e7..d2f9cddd1 100644 --- a/src/test_quota.c +++ b/src/test_quota.c @@ -387,7 +387,11 @@ static char *quota_utf8_to_mbcs(const char *zUtf8){ zTmpWide = (LPWSTR)sqlite3_malloc( (nWide+1)*sizeof(zTmpWide[0]) ); if( zTmpWide==0 ) return 0; MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, zTmpWide, nWide); +#ifdef SQLITE_OS_WINRT + codepage = CP_ACP; +#else codepage = AreFileApisANSI() ? CP_ACP : CP_OEMCP; +#endif nMbcs = WideCharToMultiByte(codepage, 0, zTmpWide, nWide, 0, 0, 0, 0); zMbcs = nMbcs ? (char*)sqlite3_malloc( nMbcs+1 ) : 0; if( zMbcs ){ diff --git a/src/tokenize.c b/src/tokenize.c index 884d1acb8..152ada64f 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -508,7 +508,7 @@ i64 sqlite3GetToken(const unsigned char *z, int *tokenType){ } case CC_DOLLAR: case CC_VARALPHA: { - i64 n = 0; + int n = 0; testcase( z[0]=='$' ); testcase( z[0]=='@' ); testcase( z[0]==':' ); testcase( z[0]=='#' ); *tokenType = TK_VARIABLE; @@ -604,7 +604,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql){ int tokenType; /* type of the next token */ int lastTokenParsed = -1; /* type of the previous token */ sqlite3 *db = pParse->db; /* The database connection */ - i64 mxSqlLen; /* Max length of an SQL string */ + int mxSqlLen; /* Max length of an SQL string */ Parse *pParentParse = 0; /* Outer parse context, if any */ #ifdef sqlite3Parser_ENGINEALWAYSONSTACK yyParser sEngine; /* Space to hold the Lemon-generated Parser object */ diff --git a/src/treeview.c b/src/treeview.c index b482e1581..153fec88d 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -1300,13 +1300,7 @@ void sqlite3TreeViewTrigger( void sqlite3ShowExpr(const Expr *p){ sqlite3TreeViewExpr(0,p,0); } void sqlite3ShowExprList(const ExprList *p){ sqlite3TreeViewExprList(0,p,0,0);} void sqlite3ShowIdList(const IdList *p){ sqlite3TreeViewIdList(0,p,0,0); } -void sqlite3ShowSrcList(const SrcList *p){ - TreeView *pView = 0; - sqlite3TreeViewPush(&pView, 0); - sqlite3TreeViewLine(pView, "SRCLIST"); - sqlite3TreeViewSrcList(pView,p); - sqlite3TreeViewPop(&pView); -} +void sqlite3ShowSrcList(const SrcList *p){ sqlite3TreeViewSrcList(0,p); } void sqlite3ShowSelect(const Select *p){ sqlite3TreeViewSelect(0,p,0); } void sqlite3ShowWith(const With *p){ sqlite3TreeViewWith(0,p,0); } void sqlite3ShowUpsert(const Upsert *p){ sqlite3TreeViewUpsert(0,p,0); } diff --git a/src/trigger.c b/src/trigger.c index 4f9068ad8..799fbe57f 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -26,7 +26,7 @@ void sqlite3DeleteTriggerStep(sqlite3 *db, TriggerStep *pTriggerStep){ sqlite3SelectDelete(db, pTmp->pSelect); sqlite3IdListDelete(db, pTmp->pIdList); sqlite3UpsertDelete(db, pTmp->pUpsert); - sqlite3SrcListDelete(db, pTmp->pSrc); + sqlite3SrcListDelete(db, pTmp->pFrom); sqlite3DbFree(db, pTmp->zSpan); sqlite3DbFree(db, pTmp); @@ -215,16 +215,11 @@ void sqlite3BeginTrigger( } } - /* NB: The SQLITE_ALLOW_TRIGGERS_ON_SYSTEM_TABLES compile-time option is - ** experimental and unsupported. Do not use it unless understand the - ** implications and you cannot get by without this capability. */ -#if !defined(SQLITE_ALLOW_TRIGGERS_ON_SYSTEM_TABLES) /* Experimental */ /* Do not create a trigger on a system table */ if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ sqlite3ErrorMsg(pParse, "cannot create trigger on system table"); goto trigger_cleanup; } -#endif /* INSTEAD of triggers are only for views and views only support INSTEAD ** of triggers. @@ -336,7 +331,6 @@ void sqlite3FinishTrigger( if( NEVER(pParse->nErr) || !pTrig ) goto triggerfinish_cleanup; zName = pTrig->zName; iDb = sqlite3SchemaToIndex(pParse->db, pTrig->pSchema); - assert( iDb>=00 && iDb<db->nDb ); pTrig->step_list = pStepList; while( pStepList ){ pStepList->pTrig = pTrig; @@ -371,12 +365,12 @@ void sqlite3FinishTrigger( if( sqlite3ReadOnlyShadowTables(db) ){ TriggerStep *pStep; for(pStep=pTrig->step_list; pStep; pStep=pStep->pNext){ - if( pStep->pSrc!=0 - && sqlite3ShadowTableName(db, pStep->pSrc->a[0].zName) + if( pStep->zTarget!=0 + && sqlite3ShadowTableName(db, pStep->zTarget) ){ sqlite3ErrorMsg(pParse, "trigger \"%s\" may not write to shadow table \"%s\"", - pTrig->zName, pStep->pSrc->a[0].zName); + pTrig->zName, pStep->zTarget); goto triggerfinish_cleanup; } } @@ -467,39 +461,26 @@ TriggerStep *sqlite3TriggerSelectStep( static TriggerStep *triggerStepAllocate( Parse *pParse, /* Parser context */ u8 op, /* Trigger opcode */ - SrcList *pTabList, /* Target table */ + Token *pName, /* The target name */ const char *zStart, /* Start of SQL text */ const char *zEnd /* End of SQL text */ ){ - Trigger *pNew = pParse->pNewTrigger; sqlite3 *db = pParse->db; - TriggerStep *pTriggerStep = 0; + TriggerStep *pTriggerStep; - if( pParse->nErr==0 ){ - if( pNew - && pNew->pSchema!=db->aDb[1].pSchema - && pTabList->a[0].u4.zDatabase - ){ - sqlite3ErrorMsg(pParse, - "qualified table names are not allowed on INSERT, UPDATE, and DELETE " - "statements within triggers"); - }else{ - pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep)); - if( pTriggerStep ){ - pTriggerStep->pSrc = sqlite3SrcListDup(db, pTabList, EXPRDUP_REDUCE); - pTriggerStep->op = op; - pTriggerStep->zSpan = triggerSpanDup(db, zStart, zEnd); - if( pTriggerStep->pSrc && IN_RENAME_OBJECT ){ - sqlite3RenameTokenRemap(pParse, - pTriggerStep->pSrc->a[0].zName, - pTabList->a[0].zName - ); - } - } + if( pParse->nErr ) return 0; + pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep) + pName->n + 1); + if( pTriggerStep ){ + char *z = (char*)&pTriggerStep[1]; + memcpy(z, pName->z, pName->n); + sqlite3Dequote(z); + pTriggerStep->zTarget = z; + pTriggerStep->op = op; + pTriggerStep->zSpan = triggerSpanDup(db, zStart, zEnd); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, pTriggerStep->zTarget, pName); } } - - sqlite3SrcListDelete(db, pTabList); return pTriggerStep; } @@ -512,7 +493,7 @@ static TriggerStep *triggerStepAllocate( */ TriggerStep *sqlite3TriggerInsertStep( Parse *pParse, /* Parser */ - SrcList *pTabList, /* Table to INSERT into */ + Token *pTableName, /* Name of the table into which we insert */ IdList *pColumn, /* List of columns in pTableName to insert into */ Select *pSelect, /* A SELECT statement that supplies values */ u8 orconf, /* The conflict algorithm (OE_Abort, OE_Replace, etc.) */ @@ -525,7 +506,7 @@ TriggerStep *sqlite3TriggerInsertStep( assert(pSelect != 0 || db->mallocFailed); - pTriggerStep = triggerStepAllocate(pParse, TK_INSERT, pTabList, zStart, zEnd); + pTriggerStep = triggerStepAllocate(pParse, TK_INSERT, pTableName,zStart,zEnd); if( pTriggerStep ){ if( IN_RENAME_OBJECT ){ pTriggerStep->pSelect = pSelect; @@ -557,7 +538,7 @@ TriggerStep *sqlite3TriggerInsertStep( */ TriggerStep *sqlite3TriggerUpdateStep( Parse *pParse, /* Parser */ - SrcList *pTabList, /* Name of the table to be updated */ + Token *pTableName, /* Name of the table to be updated */ SrcList *pFrom, /* FROM clause for an UPDATE-FROM, or NULL */ ExprList *pEList, /* The SET clause: list of column and new values */ Expr *pWhere, /* The WHERE clause */ @@ -568,36 +549,21 @@ TriggerStep *sqlite3TriggerUpdateStep( sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; - pTriggerStep = triggerStepAllocate(pParse, TK_UPDATE, pTabList, zStart, zEnd); + pTriggerStep = triggerStepAllocate(pParse, TK_UPDATE, pTableName,zStart,zEnd); if( pTriggerStep ){ - SrcList *pFromDup = 0; if( IN_RENAME_OBJECT ){ pTriggerStep->pExprList = pEList; pTriggerStep->pWhere = pWhere; - pFromDup = pFrom; + pTriggerStep->pFrom = pFrom; pEList = 0; pWhere = 0; pFrom = 0; }else{ pTriggerStep->pExprList = sqlite3ExprListDup(db, pEList, EXPRDUP_REDUCE); pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); - pFromDup = sqlite3SrcListDup(db, pFrom, EXPRDUP_REDUCE); + pTriggerStep->pFrom = sqlite3SrcListDup(db, pFrom, EXPRDUP_REDUCE); } pTriggerStep->orconf = orconf; - - if( pFromDup && !IN_RENAME_OBJECT){ - Select *pSub; - Token as = {0, 0}; - pSub = sqlite3SelectNew(pParse, 0, pFromDup, 0,0,0,0, SF_NestedFrom, 0); - pFromDup = sqlite3SrcListAppendFromTerm(pParse, 0, 0, 0, &as, pSub ,0); - } - if( pFromDup && pTriggerStep->pSrc ){ - pTriggerStep->pSrc = sqlite3SrcListAppendList( - pParse, pTriggerStep->pSrc, pFromDup - ); - }else{ - sqlite3SrcListDelete(db, pFromDup); - } } sqlite3ExprListDelete(db, pEList); sqlite3ExprDelete(db, pWhere); @@ -612,7 +578,7 @@ TriggerStep *sqlite3TriggerUpdateStep( */ TriggerStep *sqlite3TriggerDeleteStep( Parse *pParse, /* Parser */ - SrcList *pTabList, /* The table from which rows are deleted */ + Token *pTableName, /* The table from which rows are deleted */ Expr *pWhere, /* The WHERE clause */ const char *zStart, /* Start of SQL text */ const char *zEnd /* End of SQL text */ @@ -620,7 +586,7 @@ TriggerStep *sqlite3TriggerDeleteStep( sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; - pTriggerStep = triggerStepAllocate(pParse, TK_DELETE, pTabList, zStart, zEnd); + pTriggerStep = triggerStepAllocate(pParse, TK_DELETE, pTableName,zStart,zEnd); if( pTriggerStep ){ if( IN_RENAME_OBJECT ){ pTriggerStep->pWhere = pWhere; @@ -820,7 +786,6 @@ static SQLITE_NOINLINE Trigger *triggersReallyExist( p = pList; if( (pParse->db->flags & SQLITE_EnableTrigger)==0 && pTab->pTrigger!=0 - && sqlite3SchemaToIndex(pParse->db, pTab->pTrigger->pSchema)!=1 ){ /* The SQLITE_DBCONFIG_ENABLE_TRIGGER setting is off. That means that ** only TEMP triggers are allowed. Truncate the pList so that it @@ -883,6 +848,52 @@ Trigger *sqlite3TriggersExist( return triggersReallyExist(pParse,pTab,op,pChanges,pMask); } +/* +** Convert the pStep->zTarget string into a SrcList and return a pointer +** to that SrcList. +** +** This routine adds a specific database name, if needed, to the target when +** forming the SrcList. This prevents a trigger in one database from +** referring to a target in another database. An exception is when the +** trigger is in TEMP in which case it can refer to any other database it +** wants. +*/ +SrcList *sqlite3TriggerStepSrc( + Parse *pParse, /* The parsing context */ + TriggerStep *pStep /* The trigger containing the target token */ +){ + sqlite3 *db = pParse->db; + SrcList *pSrc; /* SrcList to be returned */ + char *zName = sqlite3DbStrDup(db, pStep->zTarget); + pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); + assert( pSrc==0 || pSrc->nSrc==1 ); + assert( zName || pSrc==0 ); + if( pSrc ){ + Schema *pSchema = pStep->pTrig->pSchema; + pSrc->a[0].zName = zName; + if( pSchema!=db->aDb[1].pSchema ){ + assert( pSrc->a[0].fg.fixedSchema || pSrc->a[0].u4.zDatabase==0 ); + pSrc->a[0].u4.pSchema = pSchema; + pSrc->a[0].fg.fixedSchema = 1; + } + if( pStep->pFrom ){ + SrcList *pDup = sqlite3SrcListDup(db, pStep->pFrom, 0); + if( pDup && pDup->nSrc>1 && !IN_RENAME_OBJECT ){ + Select *pSubquery; + Token as; + pSubquery = sqlite3SelectNew(pParse,0,pDup,0,0,0,0,SF_NestedFrom,0); + as.n = 0; + as.z = 0; + pDup = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&as,pSubquery,0); + } + pSrc = sqlite3SrcListAppendList(pParse, pSrc, pDup); + } + }else{ + sqlite3DbFree(db, zName); + } + return pSrc; +} + /* ** Return true if the pExpr term from the RETURNING clause argument ** list is of the form "*". Raise an error if the terms if of the @@ -1148,7 +1159,7 @@ static int codeTriggerProgram( switch( pStep->op ){ case TK_UPDATE: { sqlite3Update(pParse, - sqlite3SrcListDup(db, pStep->pSrc, 0), + sqlite3TriggerStepSrc(pParse, pStep), sqlite3ExprListDup(db, pStep->pExprList, 0), sqlite3ExprDup(db, pStep->pWhere, 0), pParse->eOrconf, 0, 0, 0 @@ -1158,7 +1169,7 @@ static int codeTriggerProgram( } case TK_INSERT: { sqlite3Insert(pParse, - sqlite3SrcListDup(db, pStep->pSrc, 0), + sqlite3TriggerStepSrc(pParse, pStep), sqlite3SelectDup(db, pStep->pSelect, 0), sqlite3IdListDup(db, pStep->pIdList), pParse->eOrconf, @@ -1169,7 +1180,7 @@ static int codeTriggerProgram( } case TK_DELETE: { sqlite3DeleteFrom(pParse, - sqlite3SrcListDup(db, pStep->pSrc, 0), + sqlite3TriggerStepSrc(pParse, pStep), sqlite3ExprDup(db, pStep->pWhere, 0), 0, 0 ); sqlite3VdbeAddOp0(v, OP_ResetCount); diff --git a/src/util.c b/src/util.c index 071029173..8e4fd516e 100644 --- a/src/util.c +++ b/src/util.c @@ -458,262 +458,48 @@ u8 sqlite3StrIHash(const char *z){ return h; } -/* -** Two inputs are multiplied to get a 128-bit result. Return -** the high-order 64 bits of that result. -*/ -static u64 sqlite3Multiply128(u64 a, u64 b){ -#if (defined(__GNUC__) || defined(__clang__)) \ - && (defined(__x86_64__) || defined(__aarch64__) || defined(__riscv)) - return ((__uint128_t)a * b) >> 64; -#elif defined(_MSC_VER) && defined(_M_X64) - return __umulh(a, b); -#else - u64 a1 = (u32)a; - u64 a2 = a >> 32; - u64 b1 = (u32)b; - u64 b2 = b >> 32; - u64 p0 = a1 * b1; - u64 p1 = a1 * b2; - u64 p2 = a2 * b1; - u64 p3 = a2 * b2; - u64 carry = ((p0 >> 32) + (u32)p1 + (u32)p2) >> 32; - return p3 + (p1 >> 32) + (p2 >> 32) + carry; -#endif -} - -/* -** Return a u64 with the N-th bit set. -*/ -#define U64_BIT(N) (((u64)1)<<(N)) - -/* -** Range of powers of 10 that we need to deal with when converting -** IEEE754 doubles to and from decimal. -*/ -#define POWERSOF10_FIRST (-348) -#define POWERSOF10_LAST (+347) - -/* -** For any p between -348 and +347, return the integer part of -** -** pow(10,p) * pow(2,63-pow10to2(p)) -** -** Or, in other words, for any p in range, return the most significant -** 64 bits of pow(10,p). The pow(10,p) value is shifted left or right, -** as appropriate so the most significant 64 bits fit exactly into a -** 64-bit unsigned integer. +/* Double-Double multiplication. (x[0],x[1]) *= (y,yy) ** -** Algorithm: -** -** (1) For p between 0 and 26, return the value directly from the aBase[] -** lookup table. -** -** (2) For p outside the range 0 to 26, use aScale[] for the initial value -** then refine that result (if necessary) by a single multiplication -** against aBase[]. +** Reference: +** T. J. Dekker, "A Floating-Point Technique for Extending the +** Available Precision". 1971-07-26. */ -static u64 powerOfTen(int p){ - static const u64 aBase[] = { - 0x8000000000000000LLU, /* 0: 1.0e+0 << 63 */ - 0xa000000000000000LLU, /* 1: 1.0e+1 << 60 */ - 0xc800000000000000LLU, /* 2: 1.0e+2 << 57 */ - 0xfa00000000000000LLU, /* 3: 1.0e+3 << 54 */ - 0x9c40000000000000LLU, /* 4: 1.0e+4 << 50 */ - 0xc350000000000000LLU, /* 5: 1.0e+5 << 47 */ - 0xf424000000000000LLU, /* 6: 1.0e+6 << 44 */ - 0x9896800000000000LLU, /* 7: 1.0e+7 << 40 */ - 0xbebc200000000000LLU, /* 8: 1.0e+8 << 37 */ - 0xee6b280000000000LLU, /* 9: 1.0e+9 << 34 */ - 0x9502f90000000000LLU, /* 10: 1.0e+10 << 30 */ - 0xba43b74000000000LLU, /* 11: 1.0e+11 << 27 */ - 0xe8d4a51000000000LLU, /* 12: 1.0e+12 << 24 */ - 0x9184e72a00000000LLU, /* 13: 1.0e+13 << 20 */ - 0xb5e620f480000000LLU, /* 14: 1.0e+14 << 17 */ - 0xe35fa931a0000000LLU, /* 15: 1.0e+15 << 14 */ - 0x8e1bc9bf04000000LLU, /* 16: 1.0e+16 << 10 */ - 0xb1a2bc2ec5000000LLU, /* 17: 1.0e+17 << 7 */ - 0xde0b6b3a76400000LLU, /* 18: 1.0e+18 << 4 */ - 0x8ac7230489e80000LLU, /* 19: 1.0e+19 >> 0 */ - 0xad78ebc5ac620000LLU, /* 20: 1.0e+20 >> 3 */ - 0xd8d726b7177a8000LLU, /* 21: 1.0e+21 >> 6 */ - 0x878678326eac9000LLU, /* 22: 1.0e+22 >> 10 */ - 0xa968163f0a57b400LLU, /* 23: 1.0e+23 >> 13 */ - 0xd3c21bcecceda100LLU, /* 24: 1.0e+24 >> 16 */ - 0x84595161401484a0LLU, /* 25: 1.0e+25 >> 20 */ - 0xa56fa5b99019a5c8LLU, /* 26: 1.0e+26 >> 23 */ - }; - static const u64 aScale[] = { - 0x8049a4ac0c5811aeLLU, /* 0: 1.0e-351 << 1229 */ - 0xcf42894a5dce35eaLLU, /* 1: 1.0e-324 << 1140 */ - 0xa76c582338ed2622LLU, /* 2: 1.0e-297 << 1050 */ - 0x873e4f75e2224e68LLU, /* 3: 1.0e-270 << 960 */ - 0xda7f5bf590966849LLU, /* 4: 1.0e-243 << 871 */ - 0xb080392cc4349dedLLU, /* 5: 1.0e-216 << 781 */ - 0x8e938662882af53eLLU, /* 6: 1.0e-189 << 691 */ - 0xe65829b3046b0afaLLU, /* 7: 1.0e-162 << 602 */ - 0xba121a4650e4ddecLLU, /* 8: 1.0e-135 << 512 */ - 0x964e858c91ba2655LLU, /* 9: 1.0e-108 << 422 */ - 0xf2d56790ab41c2a3LLU, /* 10: 1.0e-81 << 333 */ - 0xc428d05aa4751e4dLLU, /* 11: 1.0e-54 << 243 */ - 0x9e74d1b791e07e48LLU, /* 12: 1.0e-27 << 153 */ - 0x8000000000000000LLU, /* 13: 1.0e+0 << 63 */ - 0xcecb8f27f4200f3aLLU, /* 14: 1.0e+27 >> 26 */ - 0xa70c3c40a64e6c52LLU, /* 15: 1.0e+54 >> 116 */ - 0x86f0ac99b4e8dafdLLU, /* 16: 1.0e+81 >> 206 */ - 0xda01ee641a708deaLLU, /* 17: 1.0e+108 >> 295 */ - 0xb01ae745b101e9e4LLU, /* 18: 1.0e+135 >> 385 */ - 0x8e41ade9fbebc27dLLU, /* 19: 1.0e+162 >> 475 */ - 0xe5d3ef282a242e82LLU, /* 20: 1.0e+189 >> 564 */ - 0xb9a74a0637ce2ee1LLU, /* 21: 1.0e+216 >> 654 */ - 0x95f83d0a1fb69cd9LLU, /* 22: 1.0e+243 >> 744 */ - 0xf24a01a73cf2dcd0LLU, /* 23: 1.0e+270 >> 833 */ - 0xc3b8358109e84f07LLU, /* 24: 1.0e+297 >> 923 */ - 0x9e19db92b4e31ba9LLU, /* 25: 1.0e+324 >> 1013 */ - }; - int g, n; - u64 x, y; - - assert( p>=POWERSOF10_FIRST && p<=POWERSOF10_LAST ); - if( p<0 ){ - g = p/27; - n = p%27; - if( n ){ - g--; - n += 27; - } - }else if( p<27 ){ - return aBase[p]; - }else{ - g = p/27; - n = p%27; - } - y = aScale[g+13]; - if( n==0 ){ - return y; - } - x = sqlite3Multiply128(aBase[n],y); - if( (U64_BIT(63) & x)==0 ){ - x = (x<<1)|1; - } - return x; -} - -/* -** pow10to2(x) computes floor(log2(pow(10,x))). -** pow2to10(y) computes floor(log10(pow(2,y))). -** -** Conceptually, pow10to2(p) converts a base-10 exponent p into -** a corresponding base-2 exponent, and pow2to10(e) converts a base-2 -** exponent into a base-10 exponent. -** -** The conversions are based on the observation that: -** -** ln(10.0)/ln(2.0) == 108853/32768 (approximately) -** ln(2.0)/ln(10.0) == 78913/262144 (approximately) -** -** These ratios are approximate, but they are accurate to 5 digits, -** which is close enough for the usage here. Right-shift is used -** for division so that rounding of negative numbers happens in the -** right direction. -*/ -static int pwr10to2(int p){ return (p*108853) >> 15; } -static int pwr2to10(int p){ return (p*78913) >> 18; } - -/* -** Count leading zeros for a 64-bit unsigned integer. -*/ -static int countLeadingZeros(u64 m){ -#if defined(__GNUC__) || defined(__clang__) - return __builtin_clzll(m); -#else - int n = 0; - if( m <= 0x00000000ffffffffULL) { n += 32; m <<= 32; } - if( m <= 0x0000ffffffffffffULL) { n += 16; m <<= 16; } - if( m <= 0x00ffffffffffffffULL) { n += 8; m <<= 8; } - if( m <= 0x0fffffffffffffffULL) { n += 4; m <<= 4; } - if( m <= 0x3fffffffffffffffULL) { n += 2; m <<= 2; } - if( m <= 0x7fffffffffffffffULL) { n += 1; } - return n; -#endif -} - -/* -** Given m and e, which represent a quantity r == m*pow(2,e), -** return values *pD and *pP such that r == (*pD)*pow(10,*pP), -** approximately. *pD should contain at least n significant digits. -** -** The input m is required to have its highest bit set. In other words, -** m should be left-shifted, and e decremented, to maximize the value of m. -*/ -static void sqlite3Fp2Convert10(u64 m, int e, int n, u64 *pD, int *pP){ - int p; - u64 h; - assert( n>=1 && n<=18 ); - p = n - 1 - pwr2to10(e+63); - h = sqlite3Multiply128(m, powerOfTen(p)); - assert( -(e + pwr10to2(p) + 2) >= 0 ); - assert( -(e + pwr10to2(p) + 1) <= 63 ); - if( n==18 ){ - h >>= -(e + pwr10to2(p) + 2); - *pD = (h + ((h<<1)&2))>>1; - }else{ - *pD = h >> -(e + pwr10to2(p) + 1); - } - *pP = -p; -} - -/* -** Return an IEEE754 floating point value that approximates d*pow(10,p). -*/ -static double sqlite3Fp10Convert2(u64 d, int p){ - u64 out; - int e1; - int lz; - int lp; - int x; - u64 h; - double r; - assert( (d & U64_BIT(63))==0 ); - assert( d!=0 ); - if( p<POWERSOF10_FIRST ){ - return 0.0; - } - if( p>POWERSOF10_LAST ){ - return INFINITY; - } - lz = countLeadingZeros(d); - lp = pwr10to2(p); - e1 = lz - (lp + 11); - if( e1>1074 ){ - if( e1>=1130 ) return 0.0; - e1 = 1074; - } - h = sqlite3Multiply128(d<<lz, powerOfTen(p)); - x = lz - (e1 + lp + 3); - assert( x >= 0 ); - assert( x <= 63 ); - out = h >> x; - if( out >= U64_BIT(55)-2 ){ - out >>= 1; - e1--; - } - if( e1<=(-972) ){ - return INFINITY; - } - out = (out + 2) >> 2; - if( (out & U64_BIT(52))!=0 ){ - out = (out & ~U64_BIT(52)) | ((u64)(1075-e1)<<52); - } - memcpy(&r, &out, 8); - return r; +static void dekkerMul2(volatile double *x, double y, double yy){ + /* + ** The "volatile" keywords on parameter x[] and on local variables + ** below are needed force intermediate results to be truncated to + ** binary64 rather than be carried around in an extended-precision + ** format. The truncation is necessary for the Dekker algorithm to + ** work. Intel x86 floating point might omit the truncation without + ** the use of volatile. + */ + volatile double tx, ty, p, q, c, cc; + double hx, hy; + u64 m; + memcpy(&m, (void*)&x[0], 8); + m &= 0xfffffffffc000000LL; + memcpy(&hx, &m, 8); + tx = x[0] - hx; + memcpy(&m, &y, 8); + m &= 0xfffffffffc000000LL; + memcpy(&hy, &m, 8); + ty = y - hy; + p = hx*hy; + q = hx*ty + tx*hy; + c = p+q; + cc = p - c + q + tx*ty; + cc = x[0]*yy + x[1]*y + cc; + x[0] = c + cc; + x[1] = c - x[0]; + x[1] += cc; } /* ** The string z[] is an text representation of a real number. ** Convert this string to a double and write it into *pResult. ** -** z[] must be UTF-8 and zero-terminated. +** The string z[] is length bytes in length (bytes, not characters) and +** uses the encoding enc. The string is not necessarily zero-terminated. ** ** Return TRUE if the result is a valid real number (or integer) and FALSE ** if the string is empty or contains extraneous text. More specifically @@ -740,131 +526,198 @@ static double sqlite3Fp10Convert2(u64 d, int p){ #if defined(_MSC_VER) #pragma warning(disable : 4756) #endif -int sqlite3AtoF(const char *z, double *pResult){ +int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ #ifndef SQLITE_OMIT_FLOATING_POINT + int incr; + const char *zEnd; /* sign * significand * (10 ^ (esign * exponent)) */ - int neg = 0; /* True for a negative value */ - u64 s = 0; /* mantissa */ - int d = 0; /* Value is s * pow(10,d) */ + int sign = 1; /* sign of significand */ + u64 s = 0; /* significand */ + int d = 0; /* adjust exponent for shifting decimal point */ + int esign = 1; /* sign of exponent */ + int e = 0; /* exponent */ + int eValid = 1; /* True exponent is either not used or is well-formed */ int nDigit = 0; /* Number of digits processed */ - int eType = 1; /* 1: pure integer, 2+: fractional */ + int eType = 1; /* 1: pure integer, 2+: fractional -1 or less: bad UTF16 */ + u64 s2; /* round-tripped significand */ + double rr[2]; + assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); *pResult = 0.0; /* Default return value, in case of an error */ + if( length==0 ) return 0; + + if( enc==SQLITE_UTF8 ){ + incr = 1; + zEnd = z + length; + }else{ + int i; + incr = 2; + length &= ~1; + assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + testcase( enc==SQLITE_UTF16LE ); + testcase( enc==SQLITE_UTF16BE ); + for(i=3-enc; i<length && z[i]==0; i+=2){} + if( i<length ) eType = -100; + zEnd = &z[i^1]; + z += (enc&1); + } /* skip leading spaces */ - while( sqlite3Isspace(*z) ) z++; + while( z<zEnd && sqlite3Isspace(*z) ) z+=incr; + if( z>=zEnd ) return 0; /* get sign of significand */ if( *z=='-' ){ - neg = 1; - z++; + sign = -1; + z+=incr; }else if( *z=='+' ){ - z++; + z+=incr; } /* copy max significant digits to significand */ - while( sqlite3Isdigit(*z) ){ + while( z<zEnd && sqlite3Isdigit(*z) ){ s = s*10 + (*z - '0'); - z++; nDigit++; - if( s>=((LARGEST_INT64-9)/10) ){ + z+=incr; nDigit++; + if( s>=((LARGEST_UINT64-9)/10) ){ /* skip non-significant significand digits ** (increase exponent by d to shift decimal left) */ - while( sqlite3Isdigit(*z) ){ z++; d++; } + while( z<zEnd && sqlite3Isdigit(*z) ){ z+=incr; d++; } } } + if( z>=zEnd ) goto do_atof_calc; /* if decimal point is present */ if( *z=='.' ){ - z++; + z+=incr; eType++; /* copy digits from after decimal to significand ** (decrease exponent by d to shift decimal right) */ - while( sqlite3Isdigit(*z) ){ - if( s<((LARGEST_INT64-9)/10) ){ + while( z<zEnd && sqlite3Isdigit(*z) ){ + if( s<((LARGEST_UINT64-9)/10) ){ s = s*10 + (*z - '0'); d--; nDigit++; } - z++; + z+=incr; } } + if( z>=zEnd ) goto do_atof_calc; /* if exponent is present */ if( *z=='e' || *z=='E' ){ - int esign = 1; /* sign of exponent */ - z++; + z+=incr; + eValid = 0; eType++; + /* This branch is needed to avoid a (harmless) buffer overread. The + ** special comment alerts the mutation tester that the correct answer + ** is obtained even if the branch is omitted */ + if( z>=zEnd ) goto do_atof_calc; /*PREVENTS-HARMLESS-OVERREAD*/ + /* get sign of exponent */ if( *z=='-' ){ esign = -1; - z++; + z+=incr; }else if( *z=='+' ){ - z++; + z+=incr; } /* copy digits to exponent */ - if( sqlite3Isdigit(*z) ){ - int exp = *z - '0'; - z++; - while( sqlite3Isdigit(*z) ){ - exp = exp<10000 ? (exp*10 + (*z - '0')) : 10000; - z++; - } - d += esign*exp; - }else{ - eType = -1; + while( z<zEnd && sqlite3Isdigit(*z) ){ + e = e<10000 ? (e*10 + (*z - '0')) : 10000; + z+=incr; + eValid = 1; } } /* skip trailing spaces */ - while( sqlite3Isspace(*z) ) z++; + while( z<zEnd && sqlite3Isspace(*z) ) z+=incr; +do_atof_calc: /* Zero is a special case */ if( s==0 ){ - *pResult = neg ? -0.0 : +0.0; + *pResult = sign<0 ? -0.0 : +0.0; + goto atof_return; + } + + /* adjust exponent by d, and update sign */ + e = (e*esign) + d; + + /* Try to adjust the exponent to make it smaller */ + while( e>0 && s<((LARGEST_UINT64-0x7ff)/10) ){ + s *= 10; + e--; + } + while( e<0 && (s%10)==0 ){ + s /= 10; + e++; + } + + rr[0] = (double)s; + assert( sizeof(s2)==sizeof(rr[0]) ); +#ifdef SQLITE_DEBUG + rr[1] = 18446744073709549568.0; + memcpy(&s2, &rr[1], sizeof(s2)); + assert( s2==0x43efffffffffffffLL ); +#endif + /* Largest double that can be safely converted to u64 + ** vvvvvvvvvvvvvvvvvvvvvv */ + if( rr[0]<=18446744073709549568.0 ){ + s2 = (u64)rr[0]; + rr[1] = s>=s2 ? (double)(s - s2) : -(double)(s2 - s); + }else{ + rr[1] = 0.0; + } + assert( rr[1]<=1.0e-10*rr[0] ); /* Equal only when rr[0]==0.0 */ + + if( e>0 ){ + while( e>=100 ){ + e -= 100; + dekkerMul2(rr, 1.0e+100, -1.5902891109759918046e+83); + } + while( e>=10 ){ + e -= 10; + dekkerMul2(rr, 1.0e+10, 0.0); + } + while( e>=1 ){ + e -= 1; + dekkerMul2(rr, 1.0e+01, 0.0); + } }else{ - *pResult = sqlite3Fp10Convert2(s,d); - if( neg ) *pResult = -*pResult; - assert( !sqlite3IsNaN(*pResult) ); + while( e<=-100 ){ + e += 100; + dekkerMul2(rr, 1.0e-100, -1.99918998026028836196e-117); + } + while( e<=-10 ){ + e += 10; + dekkerMul2(rr, 1.0e-10, -3.6432197315497741579e-27); + } + while( e<=-1 ){ + e += 1; + dekkerMul2(rr, 1.0e-01, -5.5511151231257827021e-18); + } } + *pResult = rr[0]+rr[1]; + if( sqlite3IsNaN(*pResult) ) *pResult = 1e300*1e300; + if( sign<0 ) *pResult = -*pResult; + assert( !sqlite3IsNaN(*pResult) ); +atof_return: /* return true if number and no extra non-whitespace characters after */ - if( z[0]==0 && nDigit>0 ){ + if( z==zEnd && nDigit>0 && eValid && eType>0 ){ return eType; - }else if( eType>=2 && nDigit>0 ){ + }else if( eType>=2 && (eType==3 || eValid) && nDigit>0 ){ return -1; }else{ return 0; } #else - return !sqlite3Atoi64(z, pResult, strlen(z), SQLITE_UTF8); + return !sqlite3Atoi64(z, pResult, length, enc); #endif /* SQLITE_OMIT_FLOATING_POINT */ } #if defined(_MSC_VER) #pragma warning(default : 4756) #endif -/* -** Digit pairs used to convert a U64 or I64 into text, two digits -** at a time. -*/ -static const union { - char a[201]; - short int forceAlignment; -} sqlite3DigitPairs = { - "00010203040506070809" - "10111213141516171819" - "20212223242526272829" - "30313233343536373839" - "40414243444546474849" - "50515253545556575859" - "60616263646566676869" - "70717273747576777879" - "80818283848586878889" - "90919293949596979899" -}; - - /* ** Render an signed 64-bit integer as text. Store the result in zOut[] and ** return the length of the string that was stored, in bytes. The value @@ -876,35 +729,23 @@ static const union { int sqlite3Int64ToText(i64 v, char *zOut){ int i; u64 x; - union { - char a[23]; - u16 forceAlignment; - } u; - if( v>0 ){ - x = v; - }else if( v==0 ){ - zOut[0] = '0'; - zOut[1] = 0; - return 1; - }else{ + char zTemp[22]; + if( v<0 ){ x = (v==SMALLEST_INT64) ? ((u64)1)<<63 : (u64)-v; + }else{ + x = v; } - i = sizeof(u.a)-1; - u.a[i] = 0; - while( x>=10 ){ - int kk = (x%100)*2; - assert( TWO_BYTE_ALIGNMENT(&sqlite3DigitPairs.a[kk]) ); - assert( TWO_BYTE_ALIGNMENT(&u.a[i-2]) ); - *(u16*)(&u.a[i-2]) = *(u16*)&sqlite3DigitPairs.a[kk]; - i -= 2; - x /= 100; - } - if( x ){ - u.a[--i] = x + '0'; - } - if( v<0 ) u.a[--i] = '-'; - memcpy(zOut, &u.a[i], sizeof(u.a)-i); - return sizeof(u.a)-1-i; + i = sizeof(zTemp)-2; + zTemp[sizeof(zTemp)-1] = 0; + while( 1 /*exit-by-break*/ ){ + zTemp[i] = (x%10) + '0'; + x = x/10; + if( x==0 ) break; + i--; + }; + if( v<0 ) zTemp[--i] = '-'; + memcpy(zOut, &zTemp[i], sizeof(zTemp)-i); + return sizeof(zTemp)-1-i; } /* @@ -1161,7 +1002,7 @@ int sqlite3Atoi(const char *z){ ** representation. ** ** If iRound<=0 then round to -iRound significant digits to the -** the right of the decimal point, or to a maximum of mxRound total +** the left of the decimal point, or to a maximum of mxRound total ** significant digits. ** ** If iRound>0 round to min(iRound,mxRound) significant digits total. @@ -1174,14 +1015,13 @@ int sqlite3Atoi(const char *z){ ** The p->z[] array is *not* zero-terminated. */ void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRound){ - int i; /* Index into zBuf[] where to put next character */ - int n; /* Number of digits */ - u64 v; /* mantissa */ - int e, exp = 0; /* Base-2 and base-10 exponent */ - char *zBuf; /* Local alias for p->zBuf */ - char *z; /* Local alias for p->z */ + int i; + u64 v; + int e, exp = 0; + double rr[2]; p->isSpecial = 0; + p->z = p->zBuf; assert( mxRound>0 ); /* Convert negative numbers to positive. Deal with Infinity, 0.0, and @@ -1199,94 +1039,78 @@ void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRound){ p->sign = '+'; } memcpy(&v,&r,8); - e = (v>>52)&0x7ff; - if( e==0x7ff ){ + e = v>>52; + if( (e&0x7ff)==0x7ff ){ p->isSpecial = 1 + (v!=0x7ff0000000000000LL); p->n = 0; p->iDP = 0; - p->z = p->zBuf; return; } - v &= 0x000fffffffffffffULL; - if( e==0 ){ - int nn = countLeadingZeros(v); - v <<= nn; - e = -1074 - nn; + + /* Multiply r by powers of ten until it lands somewhere in between + ** 1.0e+19 and 1.0e+17. + ** + ** Use Dekker-style double-double computation to increase the + ** precision. + ** + ** The error terms on constants like 1.0e+100 computed using the + ** decimal extension, for example as follows: + ** + ** SELECT decimal_exp(decimal_sub('1.0e+100',decimal(1.0e+100))); + */ + rr[0] = r; + rr[1] = 0.0; + if( rr[0]>9.223372036854774784e+18 ){ + while( rr[0]>9.223372036854774784e+118 ){ + exp += 100; + dekkerMul2(rr, 1.0e-100, -1.99918998026028836196e-117); + } + while( rr[0]>9.223372036854774784e+28 ){ + exp += 10; + dekkerMul2(rr, 1.0e-10, -3.6432197315497741579e-27); + } + while( rr[0]>9.223372036854774784e+18 ){ + exp += 1; + dekkerMul2(rr, 1.0e-01, -5.5511151231257827021e-18); + } }else{ - v = (v<<11) | U64_BIT(63); - e -= 1086; + while( rr[0]<9.223372036854774784e-83 ){ + exp -= 100; + dekkerMul2(rr, 1.0e+100, -1.5902891109759918046e+83); + } + while( rr[0]<9.223372036854774784e+07 ){ + exp -= 10; + dekkerMul2(rr, 1.0e+10, 0.0); + } + while( rr[0]<9.22337203685477478e+17 ){ + exp -= 1; + dekkerMul2(rr, 1.0e+01, 0.0); + } } - sqlite3Fp2Convert10(v, e, (iRound<=0||iRound>=18)?18:iRound+1, &v, &exp); + v = rr[1]<0.0 ? (u64)rr[0]-(u64)(-rr[1]) : (u64)rr[0]+(u64)rr[1]; - /* Extract significant digits, start at the right-most slot in p->zBuf - ** and working back to the right. "i" keeps track of the next slot in - ** which to store a digit. */ + /* Extract significant digits. */ i = sizeof(p->zBuf)-1; - zBuf = p->zBuf; assert( v>0 ); - while( v>=10 ){ - int kk = (v%100)*2; - assert( TWO_BYTE_ALIGNMENT(&sqlite3DigitPairs.a[kk]) ); - assert( TWO_BYTE_ALIGNMENT(&zBuf[i-1]) ); - *(u16*)(&zBuf[i-1]) = *(u16*)&sqlite3DigitPairs.a[kk]; - i -= 2; - v /= 100; - } - if( v ){ - assert( v<10 ); - zBuf[i--] = v + '0'; - } + while( v ){ p->zBuf[i--] = (v%10) + '0'; v /= 10; } assert( i>=0 && i<sizeof(p->zBuf)-1 ); - n = sizeof(p->zBuf) - 1 - i; /* Total number of digits extracted */ - assert( n>0 ); - assert( n<sizeof(p->zBuf) ); - testcase( n==sizeof(p->zBuf)-1 ); - p->iDP = n + exp; + p->n = sizeof(p->zBuf) - 1 - i; + assert( p->n>0 ); + assert( p->n<sizeof(p->zBuf) ); + p->iDP = p->n + exp; if( iRound<=0 ){ iRound = p->iDP - iRound; - if( iRound==0 && zBuf[i+1]>='5' ){ + if( iRound==0 && p->zBuf[i+1]>='5' ){ iRound = 1; - zBuf[i--] = '0'; - n++; + p->zBuf[i--] = '0'; + p->n++; p->iDP++; } } - z = &zBuf[i+1]; /* z points to the first digit */ - if( iRound>0 && (iRound<n || n>mxRound) ){ + if( iRound>0 && (iRound<p->n || p->n>mxRound) ){ + char *z = &p->zBuf[i+1]; if( iRound>mxRound ) iRound = mxRound; - if( iRound==17 ){ - /* If the precision is exactly 17, which only happens with the "!" - ** flag (ex: "%!.17g") then try to reduce the precision if that - ** yields text that will round-trip to the original floating-point. - ** value. Thus, for exaple, 49.47 will render as 49.47, rather than - ** as 49.469999999999999. */ - if( z[15]=='9' && z[14]=='9' ){ - int jj, kk; - u64 v2; - for(jj=14; jj>0 && z[jj-1]=='9'; jj--){} - if( jj==0 ){ - v2 = 1; - }else{ - v2 = z[0] - '0'; - for(kk=1; kk<jj; kk++) v2 = (v2*10) + z[kk] - '0'; - v2++; - } - if( r==sqlite3Fp10Convert2(v2, exp + n - jj) ){ - iRound = jj+1; - } - }else if( p->iDP>=n || (z[15]=='0' && z[14]=='0' && z[13]=='0') ){ - int jj, kk; - u64 v2; - assert( z[0]!='0' ); - for(jj=14; z[jj-1]=='0'; jj--){} - v2 = z[0] - '0'; - for(kk=1; kk<jj; kk++) v2 = (v2*10) + z[kk] - '0'; - if( r==sqlite3Fp10Convert2(v2, exp + n - jj) ){ - iRound = jj+1; - } - } - } - n = iRound; + p->n = iRound; if( z[iRound]>='5' ){ int j = iRound-1; while( 1 /*exit-by-break*/ ){ @@ -1294,9 +1118,8 @@ void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRound){ if( z[j]<='9' ) break; z[j] = '0'; if( j==0 ){ - z--; - z[0] = '1'; - n++; + p->z[i--] = '1'; + p->n++; p->iDP++; break; }else{ @@ -1305,13 +1128,13 @@ void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRound){ } } } - assert( n>0 ); - while( z[n-1]=='0' ){ - n--; - assert( n>0 ); + p->z = &p->zBuf[i+1]; + assert( i+p->n < sizeof(p->zBuf) ); + assert( p->n>0 ); + while( p->z[p->n-1]=='0' ){ + p->n--; + assert( p->n>0 ); } - p->n = n; - p->z = z; } /* diff --git a/src/vacuum.c b/src/vacuum.c index 70e62e1ef..1b4838040 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -230,11 +230,9 @@ SQLITE_NOINLINE int sqlite3RunVacuum( pDb = &db->aDb[nDb]; assert( strcmp(pDb->zDbSName,zDbVacuum)==0 ); pTemp = pDb->pBt; - nRes = sqlite3BtreeGetRequestedReserve(pMain); if( pOut ){ sqlite3_file *id = sqlite3PagerFile(sqlite3BtreePager(pTemp)); i64 sz = 0; - const char *zFilename; if( id->pMethods!=0 && (sqlite3OsFileSize(id, &sz)!=SQLITE_OK || sz>0) ){ rc = SQLITE_ERROR; sqlite3SetString(pzErrMsg, db, "output file already exists"); @@ -246,16 +244,8 @@ SQLITE_NOINLINE int sqlite3RunVacuum( ** they are for the database being vacuumed, except that PAGER_CACHESPILL ** is always set. */ pgflags = db->aDb[iDb].safety_level | (db->flags & PAGER_FLAGS_MASK); - - /* If the VACUUM INTO target file is a URI filename and if the - ** "reserve=N" query parameter is present, reset the reserve to the - ** amount specified, if the amount is within range */ - zFilename = sqlite3BtreeGetFilename(pTemp); - if( ALWAYS(zFilename) ){ - int nNew = (int)sqlite3_uri_int64(zFilename, "reserve", nRes); - if( nNew>=0 && nNew<=255 ) nRes = nNew; - } } + nRes = sqlite3BtreeGetRequestedReserve(pMain); sqlite3BtreeSetCacheSize(pTemp, db->aDb[iDb].pSchema->cache_size); sqlite3BtreeSetSpillSize(pTemp, sqlite3BtreeSetSpillSize(pMain,0)); diff --git a/src/vdbe.c b/src/vdbe.c index e2e98eb5f..b5a262e63 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -353,9 +353,10 @@ static int alsoAnInt(Mem *pRec, double rValue, i64 *piValue){ */ static void applyNumericAffinity(Mem *pRec, int bTryForInt){ double rValue; + u8 enc = pRec->enc; int rc; assert( (pRec->flags & (MEM_Str|MEM_Int|MEM_Real|MEM_IntReal))==MEM_Str ); - rValue = sqlite3MemRealValueRC(pRec, &rc); + rc = sqlite3AtoF(pRec->z, &rValue, pRec->n, enc); if( rc<=0 ) return; if( rc==1 && alsoAnInt(pRec, rValue, &pRec->u.i) ){ pRec->flags |= MEM_Int; @@ -437,10 +438,7 @@ int sqlite3_value_numeric_type(sqlite3_value *pVal){ int eType = sqlite3_value_type(pVal); if( eType==SQLITE_TEXT ){ Mem *pMem = (Mem*)pVal; - assert( pMem->db!=0 ); - sqlite3_mutex_enter(pMem->db->mutex); applyNumericAffinity(pMem, 0); - sqlite3_mutex_leave(pMem->db->mutex); eType = sqlite3_value_type(pVal); } return eType; @@ -473,7 +471,7 @@ static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ pMem->u.i = 0; return MEM_Int; } - pMem->u.r = sqlite3MemRealValueRC(pMem, &rc); + rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); if( rc<=0 ){ if( rc==0 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1 ){ pMem->u.i = ix; @@ -6622,15 +6620,20 @@ case OP_SorterInsert: { /* in2 */ break; } -/* Opcode: IdxDelete P1 P2 P3 * * +/* Opcode: IdxDelete P1 P2 P3 * P5 ** Synopsis: key=r[P2@P3] ** ** The content of P3 registers starting at register P2 form ** an unpacked index key. This opcode removes that entry from the ** index opened by cursor P1. ** -** Raise an SQLITE_CORRUPT_INDEX error if no matching index entry is found -** and not in writable_schema mode. +** If P5 is not zero, then raise an SQLITE_CORRUPT_INDEX error +** if no matching index entry is found. This happens when running +** an UPDATE or DELETE statement and the index entry to be updated +** or deleted is not found. For some uses of IdxDelete +** (example: the EXCEPT operator) it does not matter that no matching +** entry is found. For those cases, P5 is zero. Also, do not raise +** this (self-correcting and non-critical) error if in writable_schema mode. */ case OP_IdxDelete: { VdbeCursor *pC; @@ -6656,7 +6659,7 @@ case OP_IdxDelete: { if( res==0 ){ rc = sqlite3BtreeDelete(pCrsr, BTREE_AUXDELETE); if( rc ) goto abort_due_to_error; - }else if( !sqlite3WritableSchema(db) ){ + }else if( pOp->p5 && !sqlite3WritableSchema(db) ){ rc = sqlite3ReportError(SQLITE_CORRUPT_INDEX, __LINE__, "index corruption"); goto abort_due_to_error; } diff --git a/src/vdbe.h b/src/vdbe.h index a2905eae4..28df764bc 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -186,7 +186,7 @@ typedef struct VdbeOpList VdbeOpList; ** Additional non-public SQLITE_PREPARE_* flags */ #define SQLITE_PREPARE_SAVESQL 0x80 /* Preserve SQL text */ -#define SQLITE_PREPARE_MASK 0x3f /* Mask of public flags */ +#define SQLITE_PREPARE_MASK 0x1f /* Mask of public flags */ /* ** Prototypes for the VDBE interface. See comments on the implementation diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 320721d06..8b68c339a 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -630,7 +630,6 @@ void sqlite3VdbeMemShallowCopy(Mem*, const Mem*, int); void sqlite3VdbeMemMove(Mem*, Mem*); int sqlite3VdbeMemNulTerminate(Mem*); int sqlite3VdbeMemSetStr(Mem*, const char*, i64, u8, void(*)(void*)); -int sqlite3VdbeMemSetText(Mem*, const char*, i64, void(*)(void*)); void sqlite3VdbeMemSetInt64(Mem*, i64); #ifdef SQLITE_OMIT_FLOATING_POINT # define sqlite3VdbeMemSetDouble sqlite3VdbeMemSetInt64 @@ -649,14 +648,13 @@ int sqlite3VdbeMemSetZeroBlob(Mem*,int); int sqlite3VdbeMemIsRowSet(const Mem*); #endif int sqlite3VdbeMemSetRowSet(Mem*); -int sqlite3VdbeMemZeroTerminateIfAble(Mem*); +void sqlite3VdbeMemZeroTerminateIfAble(Mem*); int sqlite3VdbeMemMakeWriteable(Mem*); int sqlite3VdbeMemStringify(Mem*, u8, u8); int sqlite3IntFloatCompare(i64,double); i64 sqlite3VdbeIntValue(const Mem*); int sqlite3VdbeMemIntegerify(Mem*); double sqlite3VdbeRealValue(Mem*); -SQLITE_NOINLINE double sqlite3MemRealValueRC(Mem*, int*); int sqlite3VdbeBooleanValue(Mem*, int ifNull); void sqlite3VdbeIntegerAffinity(Mem*); int sqlite3VdbeMemRealify(Mem*); diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 9fd4715ce..ec849cc4f 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -392,23 +392,7 @@ static void setResultStrOrError( void (*xDel)(void*) /* Destructor function */ ){ Mem *pOut = pCtx->pOut; - int rc; - if( enc==SQLITE_UTF8 ){ - rc = sqlite3VdbeMemSetText(pOut, z, n, xDel); - }else if( enc==SQLITE_UTF8_ZT ){ - /* It is usually considered improper to assert() on an input. However, - ** the following assert() is checking for inputs that are documented - ** to result in undefined behavior. */ - assert( z==0 - || n<0 - || n>pOut->db->aLimit[SQLITE_LIMIT_LENGTH] - || z[n]==0 - ); - rc = sqlite3VdbeMemSetText(pOut, z, n, xDel); - pOut->flags |= MEM_Term; - }else{ - rc = sqlite3VdbeMemSetStr(pOut, z, n, enc, xDel); - } + int rc = sqlite3VdbeMemSetStr(pOut, z, n, enc, xDel); if( rc ){ if( rc==SQLITE_TOOBIG ){ sqlite3_result_error_toobig(pCtx); @@ -601,7 +585,7 @@ void sqlite3_result_text64( #endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); assert( xDel!=SQLITE_DYNAMIC ); - if( enc!=SQLITE_UTF8 && enc!=SQLITE_UTF8_ZT ){ + if( enc!=SQLITE_UTF8 ){ if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; n &= ~(u64)1; } @@ -752,8 +736,6 @@ static int doWalCallbacks(sqlite3 *db){ } } } -#else - UNUSED_PARAMETER(db); #endif return rc; } @@ -1710,25 +1692,13 @@ static int bindText( assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */ if( zData!=0 ){ pVar = &p->aVar[i-1]; - if( encoding==SQLITE_UTF8 ){ - rc = sqlite3VdbeMemSetText(pVar, zData, nData, xDel); - }else if( encoding==SQLITE_UTF8_ZT ){ - /* It is usually consider improper to assert() on an input. - ** However, the following assert() is checking for inputs - ** that are documented to result in undefined behavior. */ - assert( zData==0 - || nData<0 - || nData>pVar->db->aLimit[SQLITE_LIMIT_LENGTH] - || ((u8*)zData)[nData]==0 - ); - rc = sqlite3VdbeMemSetText(pVar, zData, nData, xDel); - pVar->flags |= MEM_Term; - }else{ - rc = sqlite3VdbeMemSetStr(pVar, zData, nData, encoding, xDel); - if( encoding==0 ) pVar->enc = ENC(p->db); - } - if( rc==SQLITE_OK && encoding!=0 ){ - rc = sqlite3VdbeChangeEncoding(pVar, ENC(p->db)); + rc = sqlite3VdbeMemSetStr(pVar, zData, nData, encoding, xDel); + if( rc==SQLITE_OK ){ + if( encoding==0 ){ + pVar->enc = ENC(p->db); + }else{ + rc = sqlite3VdbeChangeEncoding(pVar, ENC(p->db)); + } } if( rc ){ sqlite3Error(p->db, rc); @@ -1840,7 +1810,7 @@ int sqlite3_bind_text64( unsigned char enc ){ assert( xDel!=SQLITE_DYNAMIC ); - if( enc!=SQLITE_UTF8 && enc!=SQLITE_UTF8_ZT ){ + if( enc!=SQLITE_UTF8 ){ if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; nData &= ~(u64)1; } diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 603e85ddf..5368c0c42 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2901,7 +2901,7 @@ int sqlite3VdbeSetColName( } assert( p->aColName!=0 ); pColName = &(p->aColName[idx+var*p->nResAlloc]); - rc = sqlite3VdbeMemSetText(pColName, zName, -1, xDel); + rc = sqlite3VdbeMemSetStr(pColName, zName, -1, SQLITE_UTF8, xDel); assert( rc!=0 || !zName || (pColName->flags&MEM_Term)!=0 ); return rc; } diff --git a/src/vdbemem.c b/src/vdbemem.c index 5689cb755..144f88936 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -107,27 +107,21 @@ static void vdbeMemRenderNum(int sz, char *zBuf, Mem *p){ StrAccum acc; assert( p->flags & (MEM_Int|MEM_Real|MEM_IntReal) ); assert( sz>22 ); - if( p->flags & (MEM_Int|MEM_IntReal) ){ -#if GCC_VERSION>=7000000 && GCC_VERSION<15000000 && defined(__i386__) - /* Work-around for GCC bug or bugs: - ** https://gcc.gnu.org/bugzilla/show_bug.cgi?id=96270 - ** https://gcc.gnu.org/bugzilla/show_bug.cgi?id=114659 - ** The problem appears to be fixed in GCC 15 */ + if( p->flags & MEM_Int ){ +#if GCC_VERSION>=7000000 + /* Work-around for GCC bug + ** https://gcc.gnu.org/bugzilla/show_bug.cgi?id=96270 */ i64 x; - assert( (MEM_Str&~p->flags)*4==sizeof(x) ); - memcpy(&x, (char*)&p->u, (MEM_Str&~p->flags)*4); + assert( (p->flags&MEM_Int)*2==sizeof(x) ); + memcpy(&x, (char*)&p->u, (p->flags&MEM_Int)*2); p->n = sqlite3Int64ToText(x, zBuf); #else p->n = sqlite3Int64ToText(p->u.i, zBuf); #endif - if( p->flags & MEM_IntReal ){ - memcpy(zBuf+p->n,".0", 3); - p->n += 2; - } }else{ sqlite3StrAccumInit(&acc, 0, zBuf, sz, 0); - sqlite3_str_appendf(&acc, "%!.*g", - (p->db ? p->db->nFpDigit : 17), p->u.r); + sqlite3_str_appendf(&acc, "%!.15g", + (p->flags & MEM_IntReal)!=0 ? (double)p->u.i : p->u.r); assert( acc.zText==zBuf && acc.mxAlloc<=0 ); zBuf[acc.nChar] = 0; /* Fast version of sqlite3StrAccumFinish(&acc) */ p->n = acc.nChar; @@ -176,9 +170,6 @@ int sqlite3VdbeMemValidStrRep(Mem *p){ assert( p->enc==SQLITE_UTF8 || p->z[((p->n+1)&~1)+1]==0 ); } if( (p->flags & (MEM_Int|MEM_Real|MEM_IntReal))==0 ) return 1; - if( p->db==0 ){ - return 1; /* db->nFpDigit required to validate p->z[] */ - } memcpy(&tmp, p, sizeof(tmp)); vdbeMemRenderNum(sizeof(zBuf), zBuf, &tmp); z = p->z; @@ -329,16 +320,13 @@ int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ ** ** This is an optimization. Correct operation continues even if ** this routine is a no-op. -** -** Return true if the strig is zero-terminated after this routine is -** called and false if it is not. */ -int sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ +void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ if( (pMem->flags & (MEM_Str|MEM_Term|MEM_Ephem|MEM_Static))!=MEM_Str ){ /* pMem must be a string, and it cannot be an ephemeral or static string */ - return 0; + return; } - if( pMem->enc!=SQLITE_UTF8 ) return 0; + if( pMem->enc!=SQLITE_UTF8 ) return; assert( pMem->z!=0 ); if( pMem->flags & MEM_Dyn ){ if( pMem->xDel==sqlite3_free @@ -346,19 +334,18 @@ int sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ ){ pMem->z[pMem->n] = 0; pMem->flags |= MEM_Term; - return 1; + return; } if( pMem->xDel==sqlite3RCStrUnref ){ /* Blindly assume that all RCStr objects are zero-terminated */ pMem->flags |= MEM_Term; - return 1; + return; } }else if( pMem->szMalloc >= pMem->n+1 ){ pMem->z[pMem->n] = 0; pMem->flags |= MEM_Term; - return 1; + return; } - return 0; } /* @@ -656,70 +643,18 @@ i64 sqlite3VdbeIntValue(const Mem *pMem){ } } -/* -** Invoke sqlite3AtoF() on the text value of pMem and return the -** double result. If sqlite3AtoF() returns an error code, write -** that code into *pRC if (*pRC)!=NULL. -** -** The caller must ensure that pMem->db!=0 and that pMem is in -** mode MEM_Str or MEM_Blob. -*/ -SQLITE_NOINLINE double sqlite3MemRealValueRC(Mem *pMem, int *pRC){ - double val = (double)0; - int rc = 0; - assert( pMem->db!=0 ); - assert( pMem->flags & (MEM_Str|MEM_Blob) ); - if( pMem->z==0 ){ - /* no-op */ - }else if( pMem->enc==SQLITE_UTF8 - && ((pMem->flags & MEM_Term)!=0 || sqlite3VdbeMemZeroTerminateIfAble(pMem)) - ){ - rc = sqlite3AtoF(pMem->z, &val); - }else if( pMem->n==0 ){ - /* no-op */ - }else if( pMem->enc==SQLITE_UTF8 ){ - char *zCopy = sqlite3DbStrNDup(pMem->db, pMem->z, pMem->n); - if( zCopy ){ - rc = sqlite3AtoF(zCopy, &val); - sqlite3DbFree(pMem->db, zCopy); - } - }else{ - int n, i, j; - char *zCopy; - const char *z; - - n = pMem->n & ~1; - zCopy = sqlite3DbMallocRaw(pMem->db, n/2 + 2); - if( zCopy ){ - z = pMem->z; - if( pMem->enc==SQLITE_UTF16LE ){ - for(i=j=0; i<n-1; i+=2, j++){ - zCopy[j] = z[i]; - if( z[i+1]!=0 ) break; - } - }else{ - for(i=j=0; i<n-1; i+=2, j++){ - if( z[i]!=0 ) break; - zCopy[j] = z[i+1]; - } - } - assert( j<=n/2 ); - zCopy[j] = 0; - rc = sqlite3AtoF(zCopy, &val); - if( i<n ) rc = -100; - sqlite3DbFree(pMem->db, zCopy); - } - } - if( pRC ) *pRC = rc; - return val; -} - /* ** Return the best representation of pMem that we can get into a ** double. If pMem is already a double or an integer, return its ** value. If it is a string or blob, try to convert it to a double. ** If it is a NULL, return 0.0. */ +static SQLITE_NOINLINE double memRealValue(Mem *pMem){ + /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ + double val = (double)0; + sqlite3AtoF(pMem->z, &val, pMem->n, pMem->enc); + return val; +} double sqlite3VdbeRealValue(Mem *pMem){ assert( pMem!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); @@ -730,7 +665,7 @@ double sqlite3VdbeRealValue(Mem *pMem){ testcase( pMem->flags & MEM_IntReal ); return (double)pMem->u.i; }else if( pMem->flags & (MEM_Str|MEM_Blob) ){ - return sqlite3MemRealValueRC(pMem, 0); + return memRealValue(pMem); }else{ /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ return (double)0; @@ -854,7 +789,7 @@ int sqlite3VdbeMemNumerify(Mem *pMem){ sqlite3_int64 ix; assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - pMem->u.r = sqlite3MemRealValueRC(pMem, &rc); + rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); if( ((rc==0 || rc==1) && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1) || sqlite3RealSameAsInt(pMem->u.r, (ix = sqlite3RealToI64(pMem->u.r))) ){ @@ -1319,84 +1254,6 @@ int sqlite3VdbeMemSetStr( return SQLITE_OK; } -/* Like sqlite3VdbeMemSetStr() except: -** -** enc is always SQLITE_UTF8 -** pMem->db is always non-NULL -*/ -int sqlite3VdbeMemSetText( - Mem *pMem, /* Memory cell to set to string value */ - const char *z, /* String pointer */ - i64 n, /* Bytes in string, or negative */ - void (*xDel)(void*) /* Destructor function */ -){ - i64 nByte = n; /* New value for pMem->n */ - u16 flags; - - assert( pMem!=0 ); - assert( pMem->db!=0 ); - assert( sqlite3_mutex_held(pMem->db->mutex) ); - assert( !sqlite3VdbeMemIsRowSet(pMem) ); - - /* If z is a NULL pointer, set pMem to contain an SQL NULL. */ - if( !z ){ - sqlite3VdbeMemSetNull(pMem); - return SQLITE_OK; - } - - if( nByte<0 ){ - nByte = strlen(z); - flags = MEM_Str|MEM_Term; - }else{ - flags = MEM_Str; - } - if( nByte>(i64)pMem->db->aLimit[SQLITE_LIMIT_LENGTH] ){ - if( xDel && xDel!=SQLITE_TRANSIENT ){ - if( xDel==SQLITE_DYNAMIC ){ - sqlite3DbFree(pMem->db, (void*)z); - }else{ - xDel((void*)z); - } - } - sqlite3VdbeMemSetNull(pMem); - return sqlite3ErrorToParser(pMem->db, SQLITE_TOOBIG); - } - - /* The following block sets the new values of Mem.z and Mem.xDel. It - ** also sets a flag in local variable "flags" to indicate the memory - ** management (one of MEM_Dyn or MEM_Static). - */ - if( xDel==SQLITE_TRANSIENT ){ - i64 nAlloc = nByte + 1; - testcase( nAlloc==31 ); - testcase( nAlloc==32 ); - if( sqlite3VdbeMemClearAndResize(pMem, (int)MAX(nAlloc,32)) ){ - return SQLITE_NOMEM_BKPT; - } - assert( pMem->z!=0 ); - memcpy(pMem->z, z, nByte); - pMem->z[nByte] = 0; - }else{ - sqlite3VdbeMemRelease(pMem); - pMem->z = (char *)z; - if( xDel==SQLITE_DYNAMIC ){ - pMem->zMalloc = pMem->z; - pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); - pMem->xDel = 0; - }else if( xDel==SQLITE_STATIC ){ - pMem->xDel = xDel; - flags |= MEM_Static; - }else{ - pMem->xDel = xDel; - flags |= MEM_Dyn; - } - } - pMem->flags = flags; - pMem->n = (int)(nByte & 0x7fffffff); - pMem->enc = SQLITE_UTF8; - return SQLITE_OK; -} - /* ** Move data out of a btree key or data field and into a Mem structure. ** The data is payload from the entry that pCur is currently pointing @@ -1825,7 +1682,7 @@ static int valueFromExpr( if( affinity==SQLITE_AFF_BLOB ){ if( op==TK_FLOAT ){ assert( pVal && pVal->z && pVal->flags==(MEM_Str|MEM_Term) ); - sqlite3AtoF(pVal->z, &pVal->u.r); + sqlite3AtoF(pVal->z, &pVal->u.r, pVal->n, SQLITE_UTF8); pVal->flags = MEM_Real; }else if( op==TK_INTEGER ){ /* This case is required by -9223372036854775808 and other strings @@ -2093,11 +1950,6 @@ int sqlite3Stat4ValueFromExpr( ** ** If *ppVal is initially NULL then the caller is responsible for ** ensuring that the value written into *ppVal is eventually freed. -** -** If the buffer does not contain a well-formed record, this routine may -** read several bytes past the end of the buffer. Callers must therefore -** ensure that any buffer which may contain a corrupt record is padded -** with at least 8 bytes of addressable memory. */ int sqlite3Stat4Column( sqlite3 *db, /* Database handle */ diff --git a/src/where.c b/src/where.c index 2ef2ce0be..c4f2c5543 100644 --- a/src/where.c +++ b/src/where.c @@ -1514,14 +1514,11 @@ static sqlite3_index_info *allocateIndexInfo( break; } if( i==n ){ - int bSortByGroup = (pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0; nOrderBy = n; if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) && !pSrc->fg.rowidUsed ){ - eDistinct = 2 + bSortByGroup; + eDistinct = 2 + ((pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0); }else if( pWInfo->wctrlFlags & WHERE_GROUPBY ){ - eDistinct = 1 - bSortByGroup; - }else if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ - eDistinct = 3; + eDistinct = 1; } } } @@ -2932,67 +2929,6 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ return rc; } -/* -** Callback for estLikePatternLength(). -** -** If this node is a string literal that is longer pWalker->sz, then set -** pWalker->sz to the byte length of that string literal. -** -** pWalker->eCode indicates how to count characters: -** -** eCode==0 Count as a GLOB pattern -** eCode==1 Count as a LIKE pattern -*/ -static int exprNodePatternLengthEst(Walker *pWalker, Expr *pExpr){ - if( pExpr->op==TK_STRING ){ - int sz = 0; /* Pattern size in bytes */ - u8 *z = (u8*)pExpr->u.zToken; /* The pattern */ - u8 c; /* Next character of the pattern */ - u8 c1, c2, c3; /* Wildcards */ - if( pWalker->eCode ){ - c1 = '%'; - c2 = '_'; - c3 = 0; - }else{ - c1 = '*'; - c2 = '?'; - c3 = '['; - } - while( (c = *(z++))!=0 ){ - if( c==c3 ){ - if( *z ) z++; - while( *z && *z!=']' ) z++; - }else if( c!=c1 && c!=c2 ){ - sz++; - } - } - if( sz>pWalker->u.sz ) pWalker->u.sz = sz; - } - return WRC_Continue; -} - -/* -** Return the length of the longest string literal in the given -** expression. -** -** eCode indicates how to count characters: -** -** eCode==0 Count as a GLOB pattern -** eCode==1 Count as a LIKE pattern -*/ -static int estLikePatternLength(Expr *p, u16 eCode){ - Walker w; - w.u.sz = 0; - w.eCode = eCode; - w.xExprCallback = exprNodePatternLengthEst; - w.xSelectCallback = sqlite3SelectWalkFail; -#ifdef SQLITE_DEBUG - w.xSelectCallback2 = sqlite3SelectWalkAssert2; -#endif - sqlite3WalkExpr(&w, p); - return w.u.sz; -} - /* ** Adjust the WhereLoop.nOut value downward to account for terms of the ** WHERE clause that reference the loop but which are not used by an @@ -3021,13 +2957,6 @@ static int estLikePatternLength(Expr *p, u16 eCode){ ** "x" column is boolean or else -1 or 0 or 1 is a common default value ** on the "x" column and so in that case only cap the output row estimate ** at 1/2 instead of 1/4. -** -** Heuristic 3: If there is a LIKE or GLOB (or REGEXP or MATCH) operator -** with a large constant pattern, then reduce the size of the search -** space according to the length of the pattern, under the theory that -** longer patterns are less likely to match. This heuristic was added -** to give better output-row count estimates when preparing queries for -** the Join-Order Benchmarks. See forum thread 2026-01-30T09:57:54z */ static void whereLoopOutputAdjust( WhereClause *pWC, /* The WHERE clause */ @@ -3077,14 +3006,13 @@ static void whereLoopOutputAdjust( }else{ /* In the absence of explicit truth probabilities, use heuristics to ** guess a reasonable truth probability. */ - Expr *pOpExpr = pTerm->pExpr; pLoop->nOut--; if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && (pTerm->wtFlags & TERM_HIGHTRUTH)==0 /* tag-20200224-1 */ ){ - Expr *pRight = pOpExpr->pRight; + Expr *pRight = pTerm->pExpr->pRight; int k = 0; - testcase( pOpExpr->op==TK_IS ); + testcase( pTerm->pExpr->op==TK_IS ); if( sqlite3ExprIsInteger(pRight, &k, 0) && k>=(-1) && k<=1 ){ k = 10; }else{ @@ -3094,23 +3022,6 @@ static void whereLoopOutputAdjust( pTerm->wtFlags |= TERM_HEURTRUTH; iReduce = k; } - }else - if( ExprHasProperty(pOpExpr, EP_InfixFunc) - && pOpExpr->op==TK_FUNCTION - ){ - int eOp; - assert( ExprUseXList(pOpExpr) ); - assert( pOpExpr->x.pList->nExpr>=2 ); - eOp = sqlite3ExprIsLikeOperator(pOpExpr); - if( ALWAYS(eOp>0) ){ - int szPattern; - Expr *pRHS = pOpExpr->x.pList->a[0].pExpr; - eOp = eOp==SQLITE_INDEX_CONSTRAINT_LIKE; - szPattern = estLikePatternLength(pRHS, eOp); - if( szPattern>0 ){ - pLoop->nOut -= szPattern*2; - } - } } } } @@ -3182,7 +3093,7 @@ static int whereRangeVectorLen( idxaff = sqlite3TableColumnAffinity(pIdx->pTable, pLhs->iColumn); if( aff!=idxaff ) break; - pColl = sqlite3ExprCompareCollSeq(pParse, pTerm->pExpr); + pColl = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); if( pColl==0 ) break; if( sqlite3StrICmp(pColl->zName, pIdx->azColl[i+nEq]) ) break; } @@ -3571,7 +3482,6 @@ static int whereLoopAddBtreeIndex( pNew->rRun += nInMul + nIn; pNew->nOut += nInMul + nIn; whereLoopOutputAdjust(pBuilder->pWC, pNew, rSize); - if( pSrc->fg.fromExists ) pNew->nOut = 0; rc = whereLoopInsert(pBuilder, pNew); if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ @@ -4168,8 +4078,6 @@ static int whereLoopAddBtree( if( pSrc->fg.isSubquery ){ if( pSrc->fg.viaCoroutine ) pNew->wsFlags |= WHERE_COROUTINE; pNew->u.btree.pOrderBy = pSrc->u4.pSubq->pSelect->pOrderBy; - }else if( pSrc->fg.fromExists ){ - pNew->nOut = 0; } rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; @@ -4272,7 +4180,6 @@ static int whereLoopAddBtree( ** positioned to the correct row during the right-join no-match ** loop. */ }else{ - if( pSrc->fg.fromExists ) pNew->nOut = 0; rc = whereLoopInsert(pBuilder, pNew); } pNew->nOut = rSize; @@ -4935,7 +4842,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ sqlite3 *db = pWInfo->pParse->db; int rc = SQLITE_OK; int bFirstPastRJ = 0; - int hasRightCrossJoin = 0; + int hasRightJoin = 0; WhereLoop *pNew; @@ -4962,34 +4869,15 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ ** prevents the right operand of a RIGHT JOIN from being swapped with ** other elements even further to the right. ** - ** The hasRightCrossJoin flag prevent FROM-clause terms from moving - ** from the right side of a LEFT JOIN or CROSS JOIN over to the - ** left side of that same join. This is a required restriction in - ** the case of LEFT JOIN - an incorrect answer may results if it is - ** not enforced. This restriction is not required for CROSS JOIN. - ** It is provided merely as a means of controlling join order, under - ** the theory that no real-world queries that care about performance - ** actually use the CROSS JOIN syntax. + ** The JT_LTORJ case and the hasRightJoin flag work together to + ** prevent FROM-clause terms from moving from the right side of + ** a LEFT JOIN over to the left side of that join if the LEFT JOIN + ** is itself on the left side of a RIGHT JOIN. */ - if( pItem->fg.jointype & (JT_LTORJ|JT_CROSS) ){ - testcase( pItem->fg.jointype & JT_LTORJ ); - testcase( pItem->fg.jointype & JT_CROSS ); - hasRightCrossJoin = 1; - } + if( pItem->fg.jointype & JT_LTORJ ) hasRightJoin = 1; mPrereq |= mPrior; bFirstPastRJ = (pItem->fg.jointype & JT_RIGHT)!=0; - }else if( pItem->fg.fromExists ){ - /* joins that result from the EXISTS-to-JOIN optimization should not - ** be moved to the left of any of their dependencies */ - WhereClause *pWC = &pWInfo->sWC; - WhereTerm *pTerm; - int i; - for(i=pWC->nBase, pTerm=pWC->a; i>0; i--, pTerm++){ - if( (pNew->maskSelf & pTerm->prereqAll)!=0 ){ - mPrereq |= (pTerm->prereqAll & (pNew->maskSelf-1)); - } - } - }else if( !hasRightCrossJoin ){ + }else if( !hasRightJoin ){ mPrereq = 0; } #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -5212,7 +5100,9 @@ static i8 wherePathSatisfiesOrderBy( pLoop = pLast; } if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ - if( pLoop->u.vtab.isOrdered && pWInfo->pOrderBy==pOrderBy ){ + if( pLoop->u.vtab.isOrdered + && ((wctrlFlags&(WHERE_DISTINCTBY|WHERE_SORTBYGROUP))!=WHERE_DISTINCTBY) + ){ obSat = obDone; }else{ /* No further ORDER BY terms may be matched. So this call should @@ -5588,21 +5478,12 @@ static LogEst whereSortingCost( ** 12 otherwise ** ** For the purposes of this heuristic, a star-query is defined as a query -** with a central "fact" table that is joined against multiple -** "dimension" tables, subject to the following constraints: -** -** (aa) Only a five-way or larger join is considered for this -** optimization. If there are fewer than four terms in the FROM -** clause, this heuristic does not apply. -** -** (bb) The join between the fact table and the dimension tables must -** be an INNER join. CROSS and OUTER JOINs do not qualify. -** -** (cc) A table must have 3 or more dimension tables in order to be -** considered a fact table. (Was 4 prior to 2026-02-10.) -** -** (dd) A table that is a self-join cannot be a dimension table. -** Dimension tables are joined against fact tables. +** with a large central table that is joined using an INNER JOIN, +** not CROSS or OUTER JOINs, against four or more smaller tables. +** The central table is called the "fact" table. The smaller tables +** that get joined are "dimension tables". Also, any table that is +** self-joined cannot be a dimension table; we assume that dimension +** tables may only be joined against fact tables. ** ** SIDE EFFECT: (and really the whole point of this subroutine) ** @@ -5655,7 +5536,7 @@ static int computeMxChoice(WhereInfo *pWInfo){ } #endif /* SQLITE_DEBUG */ - if( nLoop>=4 /* Constraint (aa) */ + if( nLoop>=5 && !pWInfo->bStarDone && OptimizationEnabled(pWInfo->pParse->db, SQLITE_StarQuery) ){ @@ -5667,7 +5548,7 @@ static int computeMxChoice(WhereInfo *pWInfo){ pWInfo->bStarDone = 1; /* Only do this computation once */ - /* Look for fact tables with three or more dimensions where the + /* Look for fact tables with four or more dimensions where the ** dimension tables are not separately from the fact tables by an outer ** or cross join. Adjust cost weights if found. */ @@ -5684,17 +5565,18 @@ static int computeMxChoice(WhereInfo *pWInfo){ if( (pFactTab->fg.jointype & (JT_OUTER|JT_CROSS))!=0 ){ /* If the candidate fact-table is the right table of an outer join ** restrict the search for dimension-tables to be tables to the right - ** of the fact-table. Constraint (bb) */ - if( iFromIdx+3 > nLoop ){ - break; /* ^-- Impossible to reach nDep>=2 - Constraint (cc) */ - } + ** of the fact-table. */ + if( iFromIdx+4 > nLoop ) break; /* Impossible to reach nDep>=4 */ while( pStart && pStart->iTab<=iFromIdx ){ pStart = pStart->pNextLoop; } } for(pWLoop=pStart; pWLoop; pWLoop=pWLoop->pNextLoop){ if( (aFromTabs[pWLoop->iTab].fg.jointype & (JT_OUTER|JT_CROSS))!=0 ){ - break; /* Constraint (bb) */ + /* Fact-tables and dimension-tables cannot be separated by an + ** outer join (at least for the definition of fact- and dimension- + ** used by this heuristic). */ + break; } if( (pWLoop->prereq & m)!=0 /* pWInfo depends on iFromIdx */ && (pWLoop->maskSelf & mSeen)==0 /* pWInfo not already a dependency */ @@ -5708,9 +5590,7 @@ static int computeMxChoice(WhereInfo *pWInfo){ } } } - if( nDep<=2 ){ - continue; /* Constraint (cc) */ - } + if( nDep<=3 ) continue; /* If we reach this point, it means that pFactTab is a fact table ** with four or more dimensions connected by inner joins. Proceed @@ -5723,23 +5603,6 @@ static int computeMxChoice(WhereInfo *pWInfo){ pWLoop->rStarDelta = 0; } } -#endif -#ifdef WHERETRACE_ENABLED /* 0x80000 */ - if( sqlite3WhereTrace & 0x80000 ){ - Bitmask mShow = mSeen; - sqlite3DebugPrintf("Fact table %s(%d), dimensions:", - pFactTab->zAlias ? pFactTab->zAlias : pFactTab->pSTab->zName, - iFromIdx); - for(pWLoop=pStart; pWLoop; pWLoop=pWLoop->pNextLoop){ - if( mShow & pWLoop->maskSelf ){ - SrcItem *pDim = aFromTabs + pWLoop->iTab; - mShow &= ~pWLoop->maskSelf; - sqlite3DebugPrintf(" %s(%d)", - pDim->zAlias ? pDim->zAlias: pDim->pSTab->zName, pWLoop->iTab); - } - } - sqlite3DebugPrintf("\n"); - } #endif pWInfo->bStarUsed = 1; @@ -5763,8 +5626,10 @@ static int computeMxChoice(WhereInfo *pWInfo){ if( sqlite3WhereTrace & 0x80000 ){ SrcItem *pDim = aFromTabs + pWLoop->iTab; sqlite3DebugPrintf( - "Increase SCAN cost of %s to %d\n", - pDim->zAlias ? pDim->zAlias: pDim->pSTab->zName, mxRun + "Increase SCAN cost of dimension %s(%d) of fact %s(%d) to %d\n", + pDim->zAlias ? pDim->zAlias: pDim->pSTab->zName, pWLoop->iTab, + pFactTab->zAlias ? pFactTab->zAlias : pFactTab->pSTab->zName, + iFromIdx, mxRun ); } pWLoop->rStarDelta = mxRun - pWLoop->rRun; @@ -6578,7 +6443,6 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin( for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){ if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){ pTerm->wtFlags |= TERM_CODED; - pTerm->prereqAll = 0; } } if( i!=pWInfo->nLevel-1 ){ @@ -7566,15 +7430,14 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ } #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */ } - if( pTabList->a[pLevel->iFrom].fg.fromExists - && (i==pWInfo->nLevel-1 - || pTabList->a[pWInfo->a[i+1].iFrom].fg.fromExists==0) - ){ - /* This is an EXISTS-to-JOIN optimization which is either the - ** inner-most loop, or the inner-most of a group of nested - ** EXISTS-to-JOIN optimization loops. If this loop sees a successful - ** row, it should break out of itself as well as other EXISTS-to-JOIN - ** loops in which is is directly nested. */ + if( pTabList->a[pLevel->iFrom].fg.fromExists && i==pWInfo->nLevel-1 ){ + /* If the EXISTS-to-JOIN optimization was applied, then the EXISTS + ** loop(s) will be the inner-most loops of the join. There might be + ** multiple EXISTS loops, but they will all be nested, and the join + ** order will not have been changed by the query planner. If the + ** inner-most EXISTS loop sees a single successful row, it should + ** break out of *all* EXISTS loops. But only the inner-most of the + ** nested EXISTS loops should do this breakout. */ int nOuter = 0; /* Nr of outer EXISTS that this one is nested within */ while( nOuter<i ){ if( !pTabList->a[pLevel[-nOuter-1].iFrom].fg.fromExists ) break; @@ -7582,11 +7445,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ } testcase( nOuter>0 ); sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel[-nOuter].addrBrk); - if( nOuter ){ - VdbeComment((v, "EXISTS break %d..%d", i-nOuter, i)); - }else{ - VdbeComment((v, "EXISTS break %d", i)); - } + VdbeComment((v, "EXISTS break")); } sqlite3VdbeResolveLabel(v, pLevel->addrCont); if( pLevel->op!=OP_Noop ){ diff --git a/src/wherecode.c b/src/wherecode.c index 31d7990ad..65ed980b8 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -1573,7 +1573,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( if( SMASKBIT32(j) & pLoop->u.vtab.mHandleIn ){ int iTab = pParse->nTab++; int iCache = ++pParse->nMem; - sqlite3CodeRhsOfIN(pParse, pTerm->pExpr, iTab, 0); + sqlite3CodeRhsOfIN(pParse, pTerm->pExpr, iTab); sqlite3VdbeAddOp3(v, OP_VInitIn, iTab, iTarget, iCache); }else{ codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); @@ -2892,6 +2892,15 @@ SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( sqlite3ExprDup(pParse->db, pTerm->pExpr, 0)); } } + if( pLevel->iIdxCur ){ + /* pSubWhere may contain expressions that read from an index on the + ** table on the RHS of the right join. All such expressions first test + ** if the index is pointing at a NULL row, and if so, read from the + ** table cursor instead. So ensure that the index cursor really is + ** pointing at a NULL row here, so that no values are read from it during + ** the scan of the RHS of the RIGHT join below. */ + sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur); + } pFrom = &uSrc.sSrc; pFrom->nSrc = 1; pFrom->nAlloc = 1; diff --git a/src/whereexpr.c b/src/whereexpr.c index 74bf624c8..0d99ca85e 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -293,14 +293,13 @@ static int isLikeOrGlob( ){ int isNum; double rDummy; - assert( zNew[iTo]==0 ); - isNum = sqlite3AtoF(zNew, &rDummy); + isNum = sqlite3AtoF(zNew, &rDummy, iTo, SQLITE_UTF8); if( isNum<=0 ){ if( iTo==1 && zNew[0]=='-' ){ isNum = +1; }else{ zNew[iTo-1]++; - isNum = sqlite3AtoF(zNew, &rDummy); + isNum = sqlite3AtoF(zNew, &rDummy, iTo, SQLITE_UTF8); zNew[iTo-1]--; } } @@ -343,34 +342,6 @@ static int isLikeOrGlob( } #endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ -/* -** If pExpr is one of "like", "glob", "match", or "regexp", then -** return the corresponding SQLITE_INDEX_CONSTRAINT_xxxx value. -** If not, return 0. -** -** pExpr is guaranteed to be a TK_FUNCTION. -*/ -int sqlite3ExprIsLikeOperator(const Expr *pExpr){ - static const struct { - const char *zOp; - unsigned char eOp; - } aOp[] = { - { "match", SQLITE_INDEX_CONSTRAINT_MATCH }, - { "glob", SQLITE_INDEX_CONSTRAINT_GLOB }, - { "like", SQLITE_INDEX_CONSTRAINT_LIKE }, - { "regexp", SQLITE_INDEX_CONSTRAINT_REGEXP } - }; - int i; - assert( pExpr->op==TK_FUNCTION ); - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - for(i=0; i<ArraySize(aOp); i++){ - if( sqlite3StrICmp(pExpr->u.zToken, aOp[i].zOp)==0 ){ - return aOp[i].eOp; - } - } - return 0; -} - #ifndef SQLITE_OMIT_VIRTUALTABLE /* @@ -407,6 +378,15 @@ static int isAuxiliaryVtabOperator( Expr **ppRight /* Expression to left of MATCH/op2 */ ){ if( pExpr->op==TK_FUNCTION ){ + static const struct Op2 { + const char *zOp; + unsigned char eOp2; + } aOp[] = { + { "match", SQLITE_INDEX_CONSTRAINT_MATCH }, + { "glob", SQLITE_INDEX_CONSTRAINT_GLOB }, + { "like", SQLITE_INDEX_CONSTRAINT_LIKE }, + { "regexp", SQLITE_INDEX_CONSTRAINT_REGEXP } + }; ExprList *pList; Expr *pCol; /* Column reference */ int i; @@ -426,11 +406,16 @@ static int isAuxiliaryVtabOperator( */ pCol = pList->a[1].pExpr; assert( pCol->op!=TK_COLUMN || (ExprUseYTab(pCol) && pCol->y.pTab!=0) ); - if( ExprIsVtab(pCol) && (i = sqlite3ExprIsLikeOperator(pExpr))!=0 ){ - *peOp2 = i; - *ppRight = pList->a[0].pExpr; - *ppLeft = pCol; - return 1; + if( ExprIsVtab(pCol) ){ + for(i=0; i<ArraySize(aOp); i++){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + if( sqlite3StrICmp(pExpr->u.zToken, aOp[i].zOp)==0 ){ + *peOp2 = aOp[i].eOp2; + *ppRight = pList->a[0].pExpr; + *ppLeft = pCol; + return 1; + } + } } /* We can also match against the first column of overloaded @@ -564,22 +549,16 @@ static void whereCombineDisjuncts( Expr *pNew; /* New virtual expression */ int op; /* Operator for the combined expression */ int idxNew; /* Index in pWC of the next virtual term */ - Expr *pA, *pB; /* Expressions associated with pOne and pTwo */ if( (pOne->wtFlags | pTwo->wtFlags) & TERM_VNULL ) return; if( (pOne->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; if( (pTwo->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; if( (eOp & (WO_EQ|WO_LT|WO_LE))!=eOp && (eOp & (WO_EQ|WO_GT|WO_GE))!=eOp ) return; - pA = pOne->pExpr; - pB = pTwo->pExpr; - assert( pA->pLeft!=0 && pA->pRight!=0 ); - assert( pB->pLeft!=0 && pB->pRight!=0 ); - if( sqlite3ExprCompare(0,pA->pLeft, pB->pLeft, -1) ) return; - if( sqlite3ExprCompare(0,pA->pRight, pB->pRight,-1) ) return; - if( ExprHasProperty(pA,EP_Commuted)!=ExprHasProperty(pB,EP_Commuted) ){ - return; - } + assert( pOne->pExpr->pLeft!=0 && pOne->pExpr->pRight!=0 ); + assert( pTwo->pExpr->pLeft!=0 && pTwo->pExpr->pRight!=0 ); + if( sqlite3ExprCompare(0,pOne->pExpr->pLeft, pTwo->pExpr->pLeft, -1) ) return; + if( sqlite3ExprCompare(0,pOne->pExpr->pRight, pTwo->pExpr->pRight,-1) )return; /* If we reach this point, it means the two subterms can be combined */ if( (eOp & (eOp-1))!=0 ){ if( eOp & (WO_LT|WO_LE) ){ @@ -590,7 +569,7 @@ static void whereCombineDisjuncts( } } db = pWC->pWInfo->pParse->db; - pNew = sqlite3ExprDup(db, pA, 0); + pNew = sqlite3ExprDup(db, pOne->pExpr, 0); if( pNew==0 ) return; for(op=TK_EQ; eOp!=(WO_EQ<<(op-TK_EQ)); op++){ assert( op<TK_GE ); } pNew->op = op; @@ -1630,11 +1609,13 @@ static void whereAddLimitExpr( int iVal = 0; if( sqlite3ExprIsInteger(pExpr, &iVal, pParse) && iVal>=0 ){ - Expr *pVal = sqlite3ExprInt32(db, iVal); + Expr *pVal = sqlite3Expr(db, TK_INTEGER, 0); if( pVal==0 ) return; + ExprSetProperty(pVal, EP_IntValue); + pVal->u.iValue = iVal; pNew = sqlite3PExpr(pParse, TK_MATCH, 0, pVal); }else{ - Expr *pVal = sqlite3ExprAlloc(db, TK_REGISTER, 0, 0); + Expr *pVal = sqlite3Expr(db, TK_REGISTER, 0); if( pVal==0 ) return; pVal->iTable = iReg; pNew = sqlite3PExpr(pParse, TK_MATCH, 0, pVal); diff --git a/src/window.c b/src/window.c index ea2781864..1f22ab194 100644 --- a/src/window.c +++ b/src/window.c @@ -717,7 +717,7 @@ void sqlite3WindowUpdate( pWin->eEnd = aUp[i].eEnd; pWin->eExclude = 0; if( pWin->eStart==TK_FOLLOWING ){ - pWin->pStart = sqlite3ExprInt32(db, 1); + pWin->pStart = sqlite3Expr(db, TK_INTEGER, "1"); } break; } @@ -1062,7 +1062,9 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){ ** keep everything legal in this case. */ if( pSublist==0 ){ - pSublist = sqlite3ExprListAppend(pParse, 0, sqlite3ExprInt32(db, 0)); + pSublist = sqlite3ExprListAppend(pParse, 0, + sqlite3Expr(db, TK_INTEGER, "0") + ); } pSub = sqlite3SelectNew( diff --git a/test/alterauth2.test b/test/alterauth2.test index 260b635c3..6f9242d36 100644 --- a/test/alterauth2.test +++ b/test/alterauth2.test @@ -116,55 +116,4 @@ do_auth_test 1.3 { {SQLITE_UPDATE sqlite_temp_master sql temp {}} } -do_auth_test 1.4 { - ALTER TABLE t2 ALTER COLUMN b SET NOT NULL; -} { - {SQLITE_ALTER_TABLE main t2 b {}} - {SQLITE_FUNCTION {} sqlite_add_constraint {} {}} - {SQLITE_FUNCTION {} sqlite_drop_constraint {} {}} - {SQLITE_FUNCTION {} sqlite_fail {} {}} - {SQLITE_READ sqlite_master sql main {}} - {SQLITE_READ sqlite_master tbl_name main {}} - {SQLITE_READ sqlite_master type main {}} - {SQLITE_READ t2 b main {}} - {SQLITE_SELECT {} {} {} {}} - {SQLITE_UPDATE sqlite_master sql main {}} -} -do_auth_test 1.5 { - ALTER TABLE t2 ALTER COLUMN 'b' DROP NOT NULL; -} { - {SQLITE_ALTER_TABLE main t2 b {}} - {SQLITE_FUNCTION {} sqlite_drop_constraint {} {}} - {SQLITE_READ sqlite_master sql main {}} - {SQLITE_READ sqlite_master tbl_name main {}} - {SQLITE_READ sqlite_master type main {}} - {SQLITE_UPDATE sqlite_master sql main {}} -} - -do_auth_test 1.6 { - ALTER TABLE t2 ADD CONSTRAINT abc CHECK (aaa>b) -} { - {SQLITE_ALTER_TABLE main t2 {} {}} - {SQLITE_FUNCTION {} sqlite_add_constraint {} {}} - {SQLITE_FUNCTION {} sqlite_fail {} {}} - {SQLITE_FUNCTION {} sqlite_find_constraint {} {}} - {SQLITE_READ sqlite_master sql main {}} - {SQLITE_READ sqlite_master tbl_name main {}} - {SQLITE_READ sqlite_master type main {}} - {SQLITE_READ t2 aaa main {}} - {SQLITE_READ t2 b main {}} - {SQLITE_SELECT {} {} {} {}} - {SQLITE_UPDATE sqlite_master sql main {}} -} -do_auth_test 1.7 { - ALTER TABLE t2 DROP CONSTRAINT abc; -} { - {SQLITE_ALTER_TABLE main t2 {} {}} - {SQLITE_FUNCTION {} sqlite_drop_constraint {} {}} - {SQLITE_READ sqlite_master sql main {}} - {SQLITE_READ sqlite_master tbl_name main {}} - {SQLITE_READ sqlite_master type main {}} - {SQLITE_UPDATE sqlite_master sql main {}} -} - finish_test diff --git a/test/altercol.test b/test/altercol.test index d94e0529c..5f7de57a4 100644 --- a/test/altercol.test +++ b/test/altercol.test @@ -946,17 +946,4 @@ do_execsql_test 23.20 { ROLLBACK; } {t4new} -#------------------------------------------------------------------------- -reset_db -do_execsql_test 24.0 { - CREATE TABLE t1(a PRIMARY KEY, b); - CREATE TABLE t2(a, b, c); - INSERT INTO t2 VALUES(1, 1, 1); -} - -do_catchsql_test 24.1 { - PRAGMA foreign_keys = 1; - ALTER TABLE t2 ADD COLUMN d REFERENCES t1 DEFAULT 123; -} {1 {Cannot add a REFERENCES column with non-NULL default value}} - finish_test diff --git a/test/altercons.test b/test/altercons.test deleted file mode 100644 index fecdf858b..000000000 --- a/test/altercons.test +++ /dev/null @@ -1,442 +0,0 @@ -# 2025 September 18 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#************************************************************************* -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl -set testprefix altercons - -# If SQLITE_OMIT_ALTERTABLE is defined, omit this file. -ifcapable !altertable { - finish_test - return -} - -foreach {tn before after} { - 1 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b)) } - { CREATE TABLE t1(a, b) } - - 2 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b) NOT NULL) } - { CREATE TABLE t1(a, b NOT NULL) } - - 3 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b)NOT NULL) } - { CREATE TABLE t1(a, b NOT NULL) } - - 3 { CREATE TABLE t1(a, b NOT NULL CONSTRAINT abc CHECK(t1.a != t1.b)); } - { CREATE TABLE t1(a, b NOT NULL) } - - 4 { CREATE TABLE t1(a, b, CONSTRAINT abc CHECK(t1.a != t1.b)) } - { CREATE TABLE t1(a, b) } - - 5 { CREATE TABLE t1(a, b, CONSTRAINT abc CHECK(t1.a != t1.b), PRIMARY KEY(a))} - { CREATE TABLE t1(a, b, PRIMARY KEY(a)) } - - 6 { CREATE TABLE t1(a, b,CONSTRAINT abc CHECK(t1.a != t1.b),PRIMARY KEY(a))} - { CREATE TABLE t1(a, b,PRIMARY KEY(a)) } - - 7 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b) CONSTRAINT def UNIQUE) } - { CREATE TABLE t1(a, b CONSTRAINT def UNIQUE) } - - 8 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b) CHECK (123)) } - { CREATE TABLE t1(a, b CHECK (123)) } - - 9 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b) DEFAULT NULL) } - { CREATE TABLE t1(a, b DEFAULT NULL) } - - 10 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b) COLLATE nocase) } - { CREATE TABLE t1(a, b COLLATE nocase) } - - 11 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b) REFERENCES t2) } - { CREATE TABLE t1(a, b REFERENCES t2) } - - 12 { CREATE TABLE t1(a, b, c, CONSTRAINT one CONSTRAINT abc CHECK(a!=b) CONSTRAINT three) } - { CREATE TABLE t1(a, b, c, CONSTRAINT one CONSTRAINT three) } - - 13 { CREATE TABLE t1(a, b, c, CONSTRAINT abc CONSTRAINT one CHECK(a!=b) CONSTRAINT three) } - { CREATE TABLE t1(a, b, c, CONSTRAINT one CHECK(a!=b) CONSTRAINT three) } - - 14 { CREATE TABLE t1(a, b, c, CONSTRAINT abc) } - { CREATE TABLE t1(a, b, c) } - - 15 { CREATE TABLE t1(a, b, c, - CONSTRAINT abc, CHECK( a!=b )) } - { CREATE TABLE t1(a, b, c, CHECK( a!=b )) } - - 16 { CREATE TABLE t1(a, b, c, CONSTRAINT abc /* hello */ CHECK( a!=b )) } - { CREATE TABLE t1(a, b, c) } - - 17 { CREATE TABLE t1(a, b, c, /* world */ CONSTRAINT abc CHECK( a!=b )) } - { CREATE TABLE t1(a, b, c) } - - 18 { CREATE TABLE t1(a, b, c -- comment - CONSTRAINT abc NOT NULL - ) } - { CREATE TABLE t1(a, b, c) } - - 19 { CREATE TABLE t1(a, b, c, -- comment - CONSTRAINT abc CHECK (a>b) CONSTRAINT two - ) } - { CREATE TABLE t1(a, b, c, CONSTRAINT two - ) } - - 20 { CREATE TABLE t1(a, b, c, CONSTRAINT one CONSTRAINT abc CHECK (a>b)CONSTRAINT two) } - { CREATE TABLE t1(a, b, c, CONSTRAINT one CONSTRAINT two) } - - 21 { CREATE TABLE t1(a, b, c CONSTRAINT abc AS (b+1)) } - { CREATE TABLE t1(a, b, c AS (b+1)) } - - 22 { CREATE TABLE t1(a, b, c CONSTRAINT abc GENERATED ALWAYS AS (b+1) STORED) } - { CREATE TABLE t1(a, b, c GENERATED ALWAYS AS (b+1) STORED) } -} { - reset_db - - do_execsql_test 1.$tn.0 $before - - do_execsql_test 1.$tn.1 { - ALTER TABLE t1 DROP CONSTRAINT abc; - } {} - - do_execsql_test 1.$tn.2 { - SELECT sql FROM sqlite_schema WHERE name='t1' - } [list [string trim $after]] -} - -#------------------------------------------------------------------------- - -do_execsql_test 2.0 { - CREATE TABLE t2(x, y CONSTRAINT ccc UNIQUE); -} -do_catchsql_test 2.1 { - ALTER TABLE t2 DROP CONSTRAINT ccc -} {1 {constraint may not be dropped: ccc}} -do_catchsql_test 2.2 { - ALTER TABLE t2 DROP CONSTRAINT ddd -} {1 {no such constraint: ddd}} - -#------------------------------------------------------------------------- -reset_db -foreach {tn col before after} { - 1 a { CREATE TABLE t1(a NOT NULL, b) } - { CREATE TABLE t1(a, b) } - - 2 a { CREATE TABLE t1(a NOT NULL ON CONFLICT FAIL, b) } - { CREATE TABLE t1(a, b) } - - 3 a { CREATE TABLE t1(a NOT NULL ON CONFLICT FAIL UNIQUE, b) } - { CREATE TABLE t1(a UNIQUE, b) } - - 4 b { CREATE TABLE t1(a NOT NULL ON CONFLICT FAIL UNIQUE, b) } - { CREATE TABLE t1(a NOT NULL ON CONFLICT FAIL UNIQUE, b) } - - 5 a { CREATE TABLE t1(a CHECK(a<b) NOT NULL, b) } - { CREATE TABLE t1(a CHECK(a<b), b) } - - 6 a { CREATE TABLE t1(a CHECK(a<b) CONSTRAINT nn NOT NULL, b) } - { CREATE TABLE t1(a CHECK(a<b), b) } - - 7 b { CREATE TABLE t1(a, b NOT NULL PRIMARY KEY) } - { CREATE TABLE t1(a, b PRIMARY KEY) } - - 8 b { CREATE TABLE t1(a, b CHECK ((b+a) IS NOT NULL) NOT NULL PRIMARY KEY) } - { CREATE TABLE t1(a, b CHECK ((b+a) IS NOT NULL) PRIMARY KEY) } - - 9 b { CREATE TABLE t1(a, b CONSTRAINT nn CHECK (b IS NOT NULL) NOT NULL) } - { CREATE TABLE t1(a, b CONSTRAINT nn CHECK (b IS NOT NULL)) } - - 10 b { CREATE TABLE t1(a, b NOT NULL AS (a+1)) } - { CREATE TABLE t1(a, b AS (a+1)) } - - 11 b { CREATE TABLE t1(a, b NOT NULL GENERATED ALWAYS AS (a+1)) } - { CREATE TABLE t1(a, b GENERATED ALWAYS AS (a+1)) } -} { - reset_db - - do_execsql_test 3.$tn.0 $before - - do_execsql_test 3.$tn.1 " - ALTER TABLE t1 ALTER COLUMN $col DROP NOT NULL - " - - do_execsql_test 3.$tn.2 { - SELECT sql FROM sqlite_schema WHERE name='t1' - } [list [string trim $after]] -} - -#------------------------------------------------------------------------- -# -reset_db -do_execsql_test 4.0 { - CREATE TABLE t2(x, y CONSTRAINT ccc UNIQUE); -} -do_execsql_test 4.1 { - ALTER TABLE t2 ALTER x DROP NOT NULL; - ALTER TABLE t2 ALTER x DROP NOT NULL; - ALTER TABLE t2 ALTER x DROP NOT NULL; -} {} - -#------------------------------------------------------------------------- -# -reset_db - -do_execsql_test 5.1 { - CREATE TABLE t3(a INTEGER PRIMARY KEY, b); - INSERT INTO t3 VALUES(1000, NULL); -} - -do_catchsql_test 5.2.1 { - ALTER TABLE t3 ALTER b SET NOT NULL -} {1 {constraint failed}} - -do_test 5.2.2 { - sqlite3_errcode db -} {SQLITE_CONSTRAINT} - -foreach {tn before alter after} { - 1 { CREATE TABLE t1(a, b) } - { ALTER TABLE t1 ALTER a SET NOT NULL } - { CREATE TABLE t1(a NOT NULL, b) } - - 2 { CREATE TABLE t1(a, b) } - { ALTER TABLE t1 ALTER a SET NOT NULL ON CONFLICT FAIL } - { CREATE TABLE t1(a NOT NULL ON CONFLICT FAIL, b) } - - 3 { CREATE TABLE t1(a, b) } - { ALTER TABLE t1 ALTER a SET NOT NULL ON CONFLICT fail; } - { CREATE TABLE t1(a NOT NULL ON CONFLICT fail, b) } - - 4 { CREATE TABLE t1(a, b) } - { ALTER TABLE t1 ALTER b SET NOT NULL ON CONFLICT IGNORE ; } - { CREATE TABLE t1(a, b NOT NULL ON CONFLICT IGNORE) } - - 5 { CREATE TABLE t1(a, 'a b c' VARCHAR(10), UNIQUE(a)) } - { ALTER TABLE t1 ALTER 'a b c' SET NOT NULL } - { CREATE TABLE t1(a, 'a b c' VARCHAR(10) NOT NULL, UNIQUE(a)) } -} { - reset_db - do_execsql_test 5.3.$tn.1 $before - do_execsql_test 5.3.$tn.2 $alter - do_execsql_test 5.3.$tn.3 { - SELECT sql FROM sqlite_schema WHERE name='t1'; - } [list [string trim $after]] -} - -do_execsql_test 5.4.1 { - CREATE TABLE x1(a, b, c); -} -do_catchsql_test 5.4.2 { - ALTER TABLE x1 ALTER d SET NOT NULL; -} {1 {no such column: d}} -do_catchsql_test 5.4.3 { - ALTER TABLE x2 ALTER c SET NOT NULL; -} {1 {no such table: x2}} -do_catchsql_test 5.4.4 { - ALTER TABLE temp.x1 ALTER c SET NOT NULL; -} {1 {no such table: temp.x1}} - -#------------------------------------------------------------------------- -# -reset_db - -do_execsql_test 6.1 { - CREATE TABLE t1(a, b, c); - INSERT INTO t1 VALUES(1, 2, 3); - INSERT INTO t1 VALUES(4, 5, 6); -} - -do_catchsql_test 6.2.1 { - ALTER TABLE t1 ADD CONSTRAINT nn CHECK (c!=6); -} {1 {constraint failed}} -do_execsql_test 6.2.2 { - DELETE FROM t1 WHERE c=6; - ALTER TABLE t1 ADD CONSTRAINT nn CHECK (c!=6); -} {} -do_catchsql_test 6.2.3 { - INSERT INTO t1 VALUES(4, 5, 6); -} {1 {CHECK constraint failed: nn}} - -foreach {tn before alter after} { - 1 { CREATE TABLE t1(a, b) } - { ALTER TABLE t1 ADD CONSTRAINT nn CHECK (a>=0) } - { CREATE TABLE t1(a, b, CONSTRAINT nn CHECK (a>=0)) } - - 2 { CREATE TABLE t1(a, b ) } - { ALTER TABLE t1 ADD CONSTRAINT nn CHECK (a>=0) } - { CREATE TABLE t1(a, b , CONSTRAINT nn CHECK (a>=0)) } - - 3 { CREATE TABLE t1(a, b ) } - { ALTER TABLE t1 ADD CHECK (a>=0) } - { CREATE TABLE t1(a, b , CHECK (a>=0)) } -} { - reset_db - do_execsql_test 6.3.$tn.1 $before - do_execsql_test 6.3.$tn.2 $alter - do_execsql_test 6.3.$tn.3 { - SELECT sql FROM sqlite_schema WHERE type='table'; - } [list [string trim $after]] -} - -do_execsql_test 6.4.1 { - CREATE TABLE b1(a, b, CONSTRAINT abc CHECK (a!=2)); -} -do_catchsql_test 6.4.2 { - ALTER TABLE b1 ADD CONSTRAINT abc CHECK (a!=3); -} {1 {constraint abc already exists}} -do_execsql_test 6.4.1 { - SELECT sql FROM sqlite_schema WHERE tbl_name='b1' -} {{CREATE TABLE b1(a, b, CONSTRAINT abc CHECK (a!=2))}} - -do_execsql_test 6.5 { - CREATE TABLE abc(x,y); -} - -do_catchsql_test 6.6 { - ALTER TABLE abc ADD CHECK (z>=0); -} {1 {no such column: z}} - -#------------------------------------------------------------------------- -# Try attaching a NOT NULL to a generated column. -# -reset_db -do_execsql_test 7.0 { - CREATE TABLE x1(a, b AS (a+1)); - INSERT INTO x1 VALUES(1), (2), (3), (NULL); -} - -do_catchsql_test 7.1 { - ALTER TABLE x1 ALTER b SET NOT NULL; -} {1 {constraint failed}} - -do_catchsql_test 7.2 { - DELETE FROM x1 WHERE b IS NULL; - ALTER TABLE x1 ALTER b SET NOT NULL; -} {0 {}} - -do_execsql_test 7.3 { - SELECT b FROM x1 -} {2 3 4} - -do_catchsql_test 7.4 { - ALTER TABLE x1 ALTER rowid SET NOT NULL; -} {1 {no such column: rowid}} - -do_execsql_test 7.5 { - CREATE VIEW v1 AS SELECT a, b FROM x1; -} -do_catchsql_test 7.6 { - ALTER TABLE v1 RENAME a TO c; -} {1 {cannot rename columns of view "v1"}} -do_catchsql_test 7.7 { - ALTER TABLE v1 ALTER a SET NOT NULL; -} {1 {cannot edit constraints of view "v1"}} -do_catchsql_test 7.8 { - ALTER TABLE sqlite_schema ALTER sql SET NOT NULL; -} {1 {table sqlite_master may not be altered}} -do_catchsql_test 7.9 { - ALTER TABLE v1 ALTER a DROP NOT NULL -} {1 {cannot edit constraints of view "v1"}} - -#------------------------------------------------------------------------- -reset_db -do_execsql_test 8.0 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b NOT NULL, c CHECK (c!=555), d); - INSERT INTO t1 VALUES(1, 1, 1, 1); - INSERT INTO t1 VALUES(2, 2, 2, 2); - INSERT INTO t1 VALUES(3, 3, 3, 3); -} - -do_execsql_test 8.1.1 { - ALTER TABLE t1 ALTER a SET NOT NULL; - ALTER TABLE t1 ALTER b SET NOT NULL; - ALTER TABLE t1 ALTER c SET NOT NULL; - ALTER TABLE t1 ALTER d SET NOT NULL; -} - -do_execsql_test 8.1.2 { - SELECT sql FROM sqlite_schema WHERE tbl_name = 't1' -} {{CREATE TABLE t1(a INTEGER PRIMARY KEY NOT NULL, b NOT NULL, c CHECK (c!=555) NOT NULL, d NOT NULL)}} - -do_execsql_test 8.1.3 { - SELECT * FROM t1 WHERE a=2; -} {2 2 2 2} - -do_execsql_test 8.2.1 { - ALTER TABLE t1 ALTER a DROP NOT NULL; - ALTER TABLE t1 ALTER b DROP NOT NULL; - ALTER TABLE t1 ALTER c DROP NOT NULL; - ALTER TABLE t1 ALTER d DROP NOT NULL; -} - -do_execsql_test 8.2.2 { - SELECT sql FROM sqlite_schema WHERE tbl_name = 't1' -} {{CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c CHECK (c!=555), d)}} - -do_execsql_test 8.2.3 { - SELECT * FROM t1 WHERE a=3; -} {3 3 3 3} - -#------------------------------------------------------------------------- -reset_db -forcedelete test.db2 -do_execsql_test 9.0 { - CREATE TABLE t1(x, y, z); - ATTACH 'test.db2' AS aux; - CREATE TABLE aux.t1(x, y, z); - INSERT INTO aux.t1 VALUES(1, 1, 1); - INSERT INTO aux.t1 VALUES(2, 2, 2); - INSERT INTO aux.t1 VALUES(3, 3, NULL); - - CREATE TABLE aux.t2(x, y, z); -} - -do_catchsql_test 9.1.1 { - ALTER TABLE aux.t1 ALTER COLUMN z SET NOT NULL -} {1 {constraint failed}} -do_execsql_test 9.1.2 { - UPDATE aux.t1 SET z=x; - ALTER TABLE aux.t1 ALTER COLUMN z SET NOT NULL; - SELECT sql FROM aux.sqlite_schema WHERE name='t1'; -} {{CREATE TABLE t1(x, y, z NOT NULL)}} -do_execsql_test 9.1.3 { - ALTER TABLE aux.t1 ALTER z DROP NOT NULL; - SELECT sql FROM aux.sqlite_schema WHERE name='t1'; -} {{CREATE TABLE t1(x, y, z)}} -do_execsql_test 9.1.4 { - ALTER TABLE t2 ALTER x SET NOT NULL; - SELECT sql FROM aux.sqlite_schema WHERE name='t2'; -} {{CREATE TABLE t2(x NOT NULL, y, z)}} -do_execsql_test 9.1.5 { - ALTER TABLE t2 ALTER x DROP NOT NULL; - SELECT sql FROM aux.sqlite_schema WHERE name='t2'; -} {{CREATE TABLE t2(x, y, z)}} - -do_catchsql_test 9.2.1 { - ALTER TABLE aux.t1 ADD CONSTRAINT bill CHECK (y!=2); -} {1 {constraint failed}} -do_execsql_test 9.2.2 { - UPDATE aux.t1 SET y=4 WHERE y=2; - ALTER TABLE aux.t1 ADD CONSTRAINT bill CHECK (y!=2); - SELECT sql FROM aux.sqlite_schema WHERE name='t1'; -} {{CREATE TABLE t1(x, y, z, CONSTRAINT bill CHECK (y!=2))}} -do_execsql_test 9.2.3 { - ALTER TABLE aux.t1 DROP CONSTRAINT bill; - SELECT sql FROM aux.sqlite_schema WHERE name='t1'; -} {{CREATE TABLE t1(x, y, z)}} -do_execsql_test 9.2.4 { - ALTER TABLE t2 ADD CONSTRAINT william CHECK (z!=''); - SELECT sql FROM aux.sqlite_schema WHERE name='t2'; -} {{CREATE TABLE t2(x, y, z, CONSTRAINT william CHECK (z!=''))}} -do_execsql_test 9.2.5 { - ALTER TABLE t2 DROP CONSTRAINT william; - SELECT sql FROM aux.sqlite_schema WHERE name='t2'; -} {{CREATE TABLE t2(x, y, z)}} - -finish_test - diff --git a/test/altercons2.test b/test/altercons2.test deleted file mode 100644 index a5bbaf6fc..000000000 --- a/test/altercons2.test +++ /dev/null @@ -1,247 +0,0 @@ -# 2025 September 18 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#************************************************************************* -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl -set testprefix altercons2 - -# If SQLITE_OMIT_ALTERTABLE is defined, omit this file. -ifcapable !altertable { - finish_test - return -} - -foreach {tn newsql alter res final} { - 1 "CREATE TABLE t1(a, b" - "ALTER TABLE t1 ALTER c SET NOT NULL" - {1 {database disk image is malformed}} - "CREATE TABLE t1(a, b" - - 2 "CREATE TABLE t1(a, b, " - "ALTER TABLE t1 ALTER c DROP NOT NULL" - {0 {}} - "CREATE TABLE t1(a, b, " - - 3 "CREATE TABLE t1(a, b, CHECK( ..." - "ALTER TABLE t1 ALTER c DROP NOT NULL" - {0 {}} - "CREATE TABLE t1(a, b, CHECK( ..." - - 4 "CREATE TABLE t1(a, b, c NOT NULL" - "ALTER TABLE t1 ALTER c DROP NOT NULL" - {0 {}} - "CREATE TABLE t1(a, b, c " - - 5 "CREATE TABLE" - "ALTER TABLE t1 ALTER c DROP NOT NULL" - {1 {database disk image is malformed}} - "CREATE TABLE" - - 6 "CREATE TABLE" - "ALTER TABLE t1 ADD CONSTRAINT nn CHECK (a!=0)" - {1 {database disk image is malformed}} - "CREATE TABLE" - -} { - reset_db - do_execsql_test 1.$tn.0 { - CREATE TABLE t1(a, b, c NOT NULL, CONSTRAINT xyz CHECK( a!=0 )); - } - do_execsql_test 1.$tn.1 { - PRAGMA writable_schema = 1; - UPDATE sqlite_schema SET sql = $::newsql - } - do_catchsql_test 1.$tn.2 $alter $res - - do_execsql_test 1.$tn.3 { - SELECT sql FROM sqlite_schema WHERE name='t1' - } [list $final] -} - -#------------------------------------------------------------------------- - -reset_db -proc xAuth {t args} { - if {$t=="SQLITE_ALTER_TABLE"} { - return "SQLITE_DENY" - } - return "SQLITE_OK" -} -sqlite3 db test.db -db auth xAuth - -do_execsql_test 2.0 { - CREATE TABLE x1(a PRIMARY KEY, b CHECK(a!=b) NOT NULL, c); -} - -do_catchsql_test 2.1.1 { - ALTER TABLE x1 ADD CONSTRAINT ccc CHECK (a!='a') -} {1 {not authorized}} -do_execsql_test 2.1.2 { - SELECT sql FROM sqlite_schema WHERE name='x1' -} {{CREATE TABLE x1(a PRIMARY KEY, b CHECK(a!=b) NOT NULL, c)}} - -do_catchsql_test 2.2.1 { - ALTER TABLE x1 ALTER c SET NOT NULL -} {1 {not authorized}} -do_execsql_test 2.2.2 { - SELECT sql FROM sqlite_schema WHERE name='x1' -} {{CREATE TABLE x1(a PRIMARY KEY, b CHECK(a!=b) NOT NULL, c)}} - -#------------------------------------------------------------------------- -reset_db -do_execsql_test 3.0 { - CREATE TABLE t1(x); -} -do_execsql_test 3.1 { - ALTER TABLE t1 ALTER x SET NOT NULL; -} - -#------------------------------------------------------------------------- -reset_db -do_execsql_test 4.0 { - CREATE TABLE abc(a, b, c, CONSTRAINT one CONSTRAINT two CHECK (b!=c)); -} -do_execsql_test 4.1 { - ALTER TABLE abc DROP CONSTRAINT one -} -do_execsql_test 4.2 { - SELECT sql FROM sqlite_schema -} { - {CREATE TABLE abc(a, b, c, CONSTRAINT two CHECK (b!=c))} -} - -#------------------------------------------------------------------------- -reset_db - -# The columns must come before the table constraints in a CREATE TABLE -# statement. This is useful, as it means the DROP CONSTRAINT code does -# not have to handle the constraint immediately following the '(' at -# the start of the column-list. -do_catchsql_test 5.0 { - CREATE TABLE abc(a, b, c, CONSTRAINT two CHECK (b!=c), d) -} {1 {near "d": syntax error}} -do_catchsql_test 5.1 { - CREATE TABLE def(CONSTRAINT abc CHECK( b!=c ), a, b, c); -} {1 {near "CONSTRAINT": syntax error}} - -#------------------------------------------------------------------------- -reset_db -do_execsql_test 6.0 { - CREATE TABLE abc(a, b CONSTRAINT two COLLATE nocase CHECK (a!=b), c CONSTRAINT one DEFAULT 'abc'); -} - -do_execsql_test 6.1 { - ALTER TABLE abc DROP CONSTRAINT one; - ALTER TABLE abc DROP CONSTRAINT two; -} - -do_execsql_test 6.2 { - SELECT sql FROM sqlite_schema -} { - {CREATE TABLE abc(a, b COLLATE nocase CHECK (a!=b), c DEFAULT 'abc')} -} - -#------------------------------------------------------------------------- -reset_db -do_execsql_test 7.0 { - CREATE TABLE abc(a, b, c, CONSTRAINT one CHECK (a>b) FOREIGN KEY(a) REFERENCES abc); -} -do_execsql_test 7.1 { - ALTER TABLE abc DROP CONSTRAINT one -} -do_execsql_test 7.2 { - SELECT sql FROM sqlite_schema -} { - {CREATE TABLE abc(a, b, c, FOREIGN KEY(a) REFERENCES abc)} -} - -#------------------------------------------------------------------------- -reset_db -do_execsql_test 8.0 { - CREATE TABLE abc(a, b, c, CONSTRAINT one FOREIGN KEY(a) REFERENCES abc); -} -do_catchsql_test 8.1 { - ALTER TABLE abc DROP CONSTRAINT one -} {1 {constraint may not be dropped: one}} - -#------------------------------------------------------------------------- -reset_db -do_execsql_test 9.0 { - CREATE TABLE abc(a, b NOT NULL AS (a+1)) -} -do_execsql_test 9.1 { - ALTER TABLE abc ALTER b DROP NOT NULL; - SELECT sql FROM sqlite_schema; -} {{CREATE TABLE abc(a, b AS (a+1))}} - -#------------------------------------------------------------------------- -reset_db -do_execsql_test 10.0 { - CREATE TABLE abc(a, b GENERATED ALWAYS AS (a+1)); - INSERT INTO abc VALUES(1), (2); - SELECT * FROM abc; -} {1 2 2 3} - -do_execsql_test 10.1 { - ALTER TABLE abc ALTER b SET NOT NULL; -} -do_catchsql_test 10.2 { - INSERT INTO abc VALUES(NULL); -} {1 {NOT NULL constraint failed: abc.b}} -do_execsql_test 10.3 { - INSERT INTO abc VALUES(3); - ALTER TABLE abc ALTER COLUMN b DROP NOT NULL; -} -do_execsql_test 10.4 { - INSERT INTO abc VALUES(NULL); -} - -#------------------------------------------------------------------------- -reset_db -do_execsql_test 11.0 { - CREATE TABLE t1(a, b, c); -} - -do_execsql_test 11.1.1 { - ALTER TABLE t1 ADD CONSTRAINT c1 CHECK(a=b) --comment - ; -} - -do_execsql_test 11.1.2 {ALTER TABLE t1 ADD CONSTRAINT c2 CHECK(a=b) --comment} - -do_execsql_test 11.1.3 { - SELECT sql FROM sqlite_schema; -} { - {CREATE TABLE t1(a, b, c, CONSTRAINT c1 CHECK(a=b), CONSTRAINT c2 CHECK(a=b))} -} - -do_execsql_test 11.2.1 { - CREATE TABLE t2(a, b); -} -do_execsql_test 11.2.2 {ALTER TABLE t2 ALTER b SET NOT NULL --new cons} -do_execsql_test 11.2.3 { - SELECT sql FROM sqlite_schema WHERE name='t2'; -} { - {CREATE TABLE t2(a, b NOT NULL)} -} -do_execsql_test 11.2.3 { - ALTER TABLE t2 ALTER b DROP NOT NULL; - SELECT sql FROM sqlite_schema WHERE name='t2'; -} { - {CREATE TABLE t2(a, b)} -} -do_execsql_test 11.2.2 {ALTER TABLE t2 ALTER b SET NOT NULL --new cons -; -} -finish_test - diff --git a/test/alterfault.test b/test/alterfault.test index c9c0d77ba..b6b42973e 100644 --- a/test/alterfault.test +++ b/test/alterfault.test @@ -23,9 +23,6 @@ ifcapable !altertable { do_execsql_test 1.0 { CREATE TABLE t1(a); - CREATE TEMP TRIGGER tr1 AFTER INSERT ON t1 BEGIN - SELECT 123; - END; } faultsim_save_and_close @@ -39,40 +36,6 @@ do_faultsim_test 1.1 -faults oom* -prep { faultsim_test_result {0 {}} } -reset_db -do_execsql_test 2.0 { - CREATE TABLE x1(d, e CONSTRAINT abc NOT NULL, f); -} -faultsim_save_and_close - -foreach {tn sql} { - 1 { ALTER TABLE x1 ADD CHECK (d!=1) } - 2 { ALTER TABLE x1 ADD CONSTRAINT xyz CHECK (f>d+e); } - 3 { ALTER TABLE x1 DROP CONSTRAINT abc } - 4 { ALTER TABLE x1 ALTER f SET NOT NULL } - 5 { ALTER TABLE x1 ALTER e DROP NOT NULL } -} { - do_faultsim_test 2.$tn -faults oom* -prep { - faultsim_restore_and_reopen - } -body { - execsql $::sql - } -test { - faultsim_test_result {0 {}} - } -} -# Test an OOM when returning an error. -# -do_faultsim_test 2.e -faults oom* -prep { - faultsim_restore_and_reopen -} -body { - execsql { - ALTER TABLE x1 DROP CONSTRAINT nosuchconstraint - } -} -test { - faultsim_test_result \ - {1 {no such constraint: nosuchconstraint}} \ - {1 {SQL logic error}} -} finish_test diff --git a/test/altertab3.test b/test/altertab3.test index 36e08c769..92060fb41 100644 --- a/test/altertab3.test +++ b/test/altertab3.test @@ -778,52 +778,4 @@ do_execsql_test 31.2 { SELECT rr FROM t1 LIMIT 1 } {5.0} -#------------------------------------------------------------------------- -reset_db -do_execsql_test 32.1.0 { - CREATE TABLE t1( - a INT, - b INT, - -- comment with comma - c INT - ); -} -do_execsql_test 32.1.1 { - ALTER TABLE t1 DROP COLUMN c; -} -do_execsql_test 32.1.2 { - SELECT sql FROM sqlite_schema -} {{CREATE TABLE t1( - a INT, - b INT)}} - -reset_db -do_execsql_test 32.2.0 { - CREATE TABLE t1( - a INT, - b INT, - -- comment with, comma - c INT - ); -} -do_execsql_test 32.2.1 { - ALTER TABLE t1 DROP COLUMN c; -} -do_execsql_test 32.2.2 { - SELECT sql FROM sqlite_schema -} {{CREATE TABLE t1( - a INT, - b INT)}} - -#------------------------------------------------------------------------- -reset_db -do_execsql_test 33.1 { - CREATE TABLE x1(a TEXT, b INTEGER, c CHECK(c!=0)); -} - -do_execsql_test 33.2 { - ALTER TABLE x1 DROP COLUMN b; - SELECT sql FROM sqlite_schema; -} {{CREATE TABLE x1(a TEXT, c CHECK(c!=0))}} - finish_test diff --git a/test/altertrig.test b/test/altertrig.test index 223feaf8f..556dc3fea 100644 --- a/test/altertrig.test +++ b/test/altertrig.test @@ -41,7 +41,7 @@ do_execsql_test 1.0 { CREATE TABLE t4(a); CREATE TRIGGER r1 INSERT ON t1 BEGIN - UPDATE t1 SET d='xyz' FROM t2, t3; + UPDATE t1 SET d='xyz' FROM t2, t3; END; } diff --git a/test/atof2.test b/test/atof2.test deleted file mode 100644 index 5a68d1352..000000000 --- a/test/atof2.test +++ /dev/null @@ -1,35 +0,0 @@ -# 2026-02-20 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# Tests of the sqlite3AtoF() function. -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl - -# Rounding cases: -# -do_execsql_test atof2-1.0 { - SELECT format('%g',192.496475); -} 192.496 -do_execsql_test atof2-1.1 { - SELECT format('%g',192.496501); -} 192.497 - -load_static_extension db ieee754 -do_execsql_test atof2-2.1 { - SELECT format('%!.30f',ieee754_inc(100.0,-1)); -} 99.9999999999999858 -do_execsql_test atof2-2.2 { - SELECT format('%!.30f',ieee754_inc(100.0,-2)); -} 99.9999999999999716 - -finish_test diff --git a/test/autoindex1.test b/test/autoindex1.test index be41d702c..1c8ce007f 100644 --- a/test/autoindex1.test +++ b/test/autoindex1.test @@ -185,7 +185,8 @@ do_eqp_test autoindex1-500.1 { QUERY PLAN |--SEARCH t501 USING INTEGER PRIMARY KEY (rowid=?) `--LIST SUBQUERY xxxxxx - `--SCAN t502 + |--SCAN t502 + `--CREATE BLOOM FILTER } do_eqp_test autoindex1-501 { SELECT b FROM t501 diff --git a/test/avfs.test b/test/avfs.test index 1503e8613..ffd6b309f 100644 --- a/test/avfs.test +++ b/test/avfs.test @@ -332,11 +332,9 @@ do_test 4.3 { set ofd [open $shdo w] if {$::cliDoesAr} { puts $ofd ".ar -u $shdo" - puts $ofd ".mode list" puts $ofd "select count(*) from sqlar where name = '$shdo';" } else { puts $ofd "insert into sqlar values (1);" - puts $ofd ".mode list" puts $ofd "select count(*) from sqlar;" } puts $ofd ".q" diff --git a/test/bestindex8.test b/test/bestindex8.test index 43d9cf6af..3ed7f6703 100644 --- a/test/bestindex8.test +++ b/test/bestindex8.test @@ -80,17 +80,17 @@ do_execsql_test 1.0 { foreach {tn sql bDistinct idxinsert bConsumed res} { 1 "SELECT a, b FROM vt1" 0 0 0 {a b c d a b c d} - 2 "SELECT DISTINCT a, b FROM vt1" 2 0 1 {a b c d} - 3 "SELECT DISTINCT a FROM vt1" 2 0 1 {a c} + 2 "SELECT DISTINCT a, b FROM vt1" 2 1 1 {a b c d} + 3 "SELECT DISTINCT a FROM vt1" 2 1 1 {a c} 4 "SELECT DISTINCT b FROM vt1" 2 1 0 {b d} - 5 "SELECT DISTINCT b FROM vt1 ORDER BY a" 3 1 1 {b d} - 6 "SELECT DISTINCT t0.c0 FROM vt1, t0 ORDER BY vt1.a" 3 1 1 {1 0} + 5 "SELECT DISTINCT b FROM vt1 ORDER BY a" 0 1 1 {b d} + 6 "SELECT DISTINCT t0.c0 FROM vt1, t0 ORDER BY vt1.a" 0 1 1 {1 0} 7 "SELECT DISTINCT a, b FROM vt1 ORDER BY a, b" 3 0 1 {a b c d} - 8 "SELECT DISTINCT a, b FROM vt1 ORDER BY a" 3 1 1 {a b c d} - 9 "SELECT DISTINCT a FROM vt1 ORDER BY a, b" 3 1 1 {a c} + 8 "SELECT DISTINCT a, b FROM vt1 ORDER BY a" 0 1 1 {a b c d} + 9 "SELECT DISTINCT a FROM vt1 ORDER BY a, b" 0 1 1 {a c} - 10 "SELECT DISTINCT a, b FROM vt1 WHERE b='b'" 2 0 1 {a b} - 11 "SELECT DISTINCT a, b FROM vt1 WHERE +b='b'" 2 0 1 {a b} + 10 "SELECT DISTINCT a, b FROM vt1 WHERE b='b'" 2 1 1 {a b} + 11 "SELECT DISTINCT a, b FROM vt1 WHERE +b='b'" 2 1 1 {a b} } { set ::lBestIndexDistinct "" set ::lOrderByConsumed 0 diff --git a/test/bestindexB.test b/test/bestindexB.test index 5850e35bd..b50e74fee 100644 --- a/test/bestindexB.test +++ b/test/bestindexB.test @@ -34,7 +34,7 @@ proc vtab_command {method args} { set orderby [$hdl orderby] if {[info exists ::xbestindex_sql]} { - # explain_i $::xbestindex_sql + explain_i $::xbestindex_sql set ::xbestindex_res [ execsql $::xbestindex_sql ] } diff --git a/test/bestindexF.test b/test/bestindexF.test deleted file mode 100644 index 4f49f610d..000000000 --- a/test/bestindexF.test +++ /dev/null @@ -1,294 +0,0 @@ -# 2025-12-15 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl -set testprefix bestindexF - -ifcapable !vtab { - finish_test - return -} - - -proc vtab_command {method args} { - switch -- $method { - xConnect { - return "CREATE TABLE t1(a, b, c)" - } - - xBestIndex { - set hdl [lindex $args 0] - set ::vtab_orderby [$hdl orderby] - set ::vtab_distinct [$hdl distinct] - - if {$::vtab_orderby == "{column 0 desc 0} {column 1 desc 0}" - || $::vtab_orderby == "{column 0 desc 0}" - } { - return [list orderby 1] - } - - return "" - } - - xFilter { - set sql { - SELECT 1, 1, 'a', 555 - UNION ALL - SELECT 2, 1, 'a', NULL - UNION ALL - SELECT 3, 1, 'b', 'text' - UNION ALL - SELECT 4, 2, 'a', 3.14 - UNION ALL - SELECT 5, 2, 'b', 0 - } - return [list sql $sql] - } - } - - return {} -} - -register_tcl_module db - -do_execsql_test 1.0 { - CREATE VIRTUAL TABLE t1 USING tcl(vtab_command) -} - -proc uses_idxinsert {sql} { - return [expr [lsearch [db eval "explain $sql"] IdxInsert]>=0] -} -proc do_idxinsert_test {tn sql res} { - set uses [uses_idxinsert $sql] - uplevel [list do_execsql_test $tn "SELECT $uses ; $sql" $res] -} - -do_idxinsert_test 1.1.1 { - SELECT DISTINCT a, b FROM t1 -} {0 1 a 1 b 2 a 2 b} - -do_test 1.1.2 { - list $::vtab_distinct $::vtab_orderby -} {2 {{column 0 desc 0} {column 1 desc 0}}} - -do_execsql_test 1.3 { - CREATE TABLE t0(c0); - INSERT INTO t0 VALUES(0); - INSERT INTO t0 VALUES(1); -} - -do_idxinsert_test 1.4.1 { - SELECT DISTINCT t0.c0 FROM t1, t0 ORDER BY t1.a; -} {1 0 1} - -do_test 1.4.2 { - list $::vtab_distinct $::vtab_orderby -} {3 {{column 0 desc 0}}} - -#------------------------------------------------------------------------- -# -reset_db -proc vtab_command {method args} { - switch -- $method { - xConnect { - return "CREATE TABLE t1(a, b, c)" - } - - xBestIndex { - set hdl [lindex $args 0] - set ::vtab_orderby [$hdl orderby] - set ::vtab_distinct [$hdl distinct] - - # Set idxNum to 1 if DISTINCT is to be used in xFilter. - # - set idxStr [list ""] - if {$::vtab_distinct==2 || $::vtab_distinct==3} { - set idxStr [list DISTINCT] - } - - set orderby 0 - if {$::vtab_orderby == "{column 0 desc 1}" - || $::vtab_orderby == "{column 0 desc 0}" - } { - set orderby 1 - if {$::vtab_distinct==1 || $::vtab_distinct==2} { - lappend idxStr "ORDER BY ((a+2)%5)" - } else { - set sort "ORDER BY a" - if {$::vtab_orderby == "{column 0 desc 1}"} { - append sort " DESC" - } - lappend idxStr $sort - } - } else { - lappend idxStr "" - } - - return [list orderby $orderby idxstr $idxStr] - return "" - } - - xFilter { - set idxstr [lindex $args 1] - - set distinct [lindex $idxstr 0] - set orderby [lindex $idxstr 1] - set sql " - SELECT $distinct 0, a, b FROM real_t1 $orderby - " - return [list sql $sql] - } - } - - return {} -} - -do_execsql_test 2.0 { - CREATE TABLE real_t1(a, b); - - INSERT INTO real_t1 VALUES (1, 'a'); - INSERT INTO real_t1 VALUES (2, 'a'); - INSERT INTO real_t1 VALUES (1, 'a'); - - INSERT INTO real_t1 VALUES (2, 'b'); - INSERT INTO real_t1 VALUES (1, 'b'); - INSERT INTO real_t1 VALUES (2, 'b'); - - INSERT INTO real_t1 VALUES (3, 'a'); - INSERT INTO real_t1 VALUES (4, 'b'); - INSERT INTO real_t1 VALUES (3, 'a'); - - INSERT INTO real_t1 VALUES (4, 'b'); - INSERT INTO real_t1 VALUES (3, 'a'); - INSERT INTO real_t1 VALUES (4, 'b'); -} - -register_tcl_module db -do_execsql_test 2.0 { - CREATE VIRTUAL TABLE t1 USING tcl(vtab_command) -} - -do_execsql_test 2.1 { - SELECT a, b FROM t1 -} { - 1 a 2 a 1 a - 2 b 1 b 2 b - 3 a 4 b 3 a - 4 b 3 a 4 b -} - -# This is like do_execsql_test, except one value is prepended to the -# expected result - the P4 (idxStr) of the VFilter opcode. It is an error -# if $sql generates two or more VFilter instructions. -# -proc do_vtabsorter_test {tn sql expect} { - set vm [db eval "EXPLAIN $sql"] - - set ii [lsearch $vm VFilter] - set ::res [lindex $vm [expr $ii+4]] - - set ::idx [expr [lsearch $vm IdxInsert]>=0] - - set iSort [lsearch $vm SorterSort] - if {$iSort>=0} { - error "query is using sorter" - } - - uplevel [list do_test $tn.0 { set ::idx } [lindex $expect 0]] - uplevel [list do_test $tn.1 { set ::res } [lindex $expect 1]] - uplevel [list do_execsql_test $tn.2 $sql [lrange $expect 2 end]] -} - -do_vtabsorter_test 2.2 { - SELECT a, b FROM t1 -} { 0 "{} {}" - 1 a 2 a 1 a - 2 b 1 b 2 b - 3 a 4 b 3 a - 4 b 3 a 4 b -} - -do_vtabsorter_test 2.3 { - SELECT DISTINCT a FROM t1 -} { 0 "DISTINCT {ORDER BY ((a+2)%5)}" - 3 4 1 2 -} - -do_vtabsorter_test 2.4 { - SELECT DISTINCT a FROM t1 ORDER BY a -} { 0 "DISTINCT {ORDER BY a}" - 1 2 3 4 -} - -do_vtabsorter_test 2.5 { - SELECT DISTINCT a FROM t1 ORDER BY a DESC -} { 0 "DISTINCT {ORDER BY a DESC}" - 4 3 2 1 -} - -do_vtabsorter_test 2.6 { - SELECT a FROM t1 ORDER BY a -} { 0 "{} {ORDER BY a}" - 1 1 1 - 2 2 2 - 3 3 3 - 4 4 4 -} - -do_vtabsorter_test 2.7 { - SELECT a FROM t1 ORDER BY a DESC -} { 0 "{} {ORDER BY a DESC}" - 4 4 4 - 3 3 3 - 2 2 2 - 1 1 1 -} - -do_vtabsorter_test 2.8 { - SELECT a, count(*) FROM t1 GROUP BY a ORDER BY a -} { 0 "{} {ORDER BY a}" - 1 3 - 2 3 - 3 3 - 4 3 -} - -do_vtabsorter_test 2.9 { - SELECT a, count(*) FROM t1 GROUP BY a ORDER BY a DESC -} { 0 "{} {ORDER BY a DESC}" - 4 3 - 3 3 - 2 3 - 1 3 -} - -do_vtabsorter_test 2.10 { - SELECT a, count(*) FROM t1 GROUP BY a -} { 0 "{} {ORDER BY ((a+2)%5)}" - 3 3 - 4 3 - 1 3 - 2 3 -} - -do_vtabsorter_test 2.11 { - SELECT DISTINCT a, count(*) FROM t1 GROUP BY a -} { 1 "{} {ORDER BY ((a+2)%5)}" - 3 3 - 4 3 - 1 3 - 2 3 -} - -finish_test - diff --git a/test/carray01.test b/test/carray01.test index b17a481e1..86ea06996 100644 --- a/test/carray01.test +++ b/test/carray01.test @@ -51,10 +51,6 @@ do_test 101 { run_stmt $STMT } {1} do_test 102 { - sqlite3_carray_bind -v2 -malloc $STMT 3 1 2 3 4 5 6 7 - run_stmt $STMT -} {1} -do_test 103 { set STMT2 [sqlite3_prepare_v2 db { SELECT DISTINCT typeof(value) FROM carray(?3)} -1] sqlite3_carray_bind $STMT2 3 1 2 3 4 5 6 7 @@ -128,10 +124,6 @@ do_test 160 { sqlite3_carray_bind -double $STMT 3 1 2 3 4 5 6 7 run_stmt $STMT } {1} -do_test 161 { - sqlite3_carray_bind -double -v2 $STMT 3 1 2 3 4 5 6 7 - run_stmt $STMT -} {1} do_test 170 { sqlite3_carray_bind -text -static $STMT 3 1 2 3 4 6 7 run_stmt $STMT diff --git a/test/collate5.test b/test/collate5.test index b474c39d5..71d4efe25 100644 --- a/test/collate5.test +++ b/test/collate5.test @@ -122,9 +122,9 @@ do_test collate5-2.0 { } {} do_test collate5-2.1.1 { - string toupper [execsql { + execsql { SELECT a FROM collate5t1 UNION select a FROM collate5t2; - }] + } } {A B N} do_test collate5-2.1.2 { execsql { @@ -132,10 +132,10 @@ do_test collate5-2.1.2 { } } {A B N a b n} do_test collate5-2.1.3 { - string tolower [execsql { + execsql { SELECT a, b FROM collate5t1 UNION select a, b FROM collate5t2; - }] -} {a apple a apple b banana b banana n {}} + } +} {A Apple A apple B Banana b banana N {}} do_test collate5-2.1.4 { execsql { SELECT a, b FROM collate5t2 UNION select a, b FROM collate5t1; @@ -143,9 +143,9 @@ do_test collate5-2.1.4 { } {A Apple B banana N {} a apple b banana n {}} do_test collate5-2.2.1 { - string toupper [execsql { + execsql { SELECT a FROM collate5t1 EXCEPT select a FROM collate5t2; - }] + } } {N} do_test collate5-2.2.2 { execsql { @@ -153,10 +153,10 @@ do_test collate5-2.2.2 { } } {A a} do_test collate5-2.2.3 { - string tolower [execsql { + execsql { SELECT a, b FROM collate5t1 EXCEPT select a, b FROM collate5t2; - }] -} {a apple n {}} + } +} {A Apple N {}} do_test collate5-2.2.4 { execsql { SELECT a, b FROM collate5t2 EXCEPT select a, b FROM collate5t1 @@ -165,9 +165,9 @@ do_test collate5-2.2.4 { } {A apple a apple} do_test collate5-2.3.1 { - string toupper [execsql { + execsql { SELECT a FROM collate5t1 INTERSECT select a FROM collate5t2; - }] + } } {A B} do_test collate5-2.3.2 { execsql { @@ -175,10 +175,10 @@ do_test collate5-2.3.2 { } } {B b} do_test collate5-2.3.3 { - string tolower [execsql { + execsql { SELECT a, b FROM collate5t1 INTERSECT select a, b FROM collate5t2; - }] -} {a apple b banana} + } +} {a apple B banana} do_test collate5-2.3.4 { execsql { SELECT a, b FROM collate5t2 INTERSECT select a, b FROM collate5t1; diff --git a/test/cost.test b/test/cost.test index 5ef84c2b2..6106caba8 100644 --- a/test/cost.test +++ b/test/cost.test @@ -203,8 +203,8 @@ do_eqp_test 8.2 { } { QUERY PLAN |--SCAN track - |--SEARCH composer USING INTEGER PRIMARY KEY (rowid=?) |--SEARCH album USING INTEGER PRIMARY KEY (rowid=?) + |--SEARCH composer USING INTEGER PRIMARY KEY (rowid=?) `--USE TEMP B-TREE FOR DISTINCT } diff --git a/test/dblwidth-a.sql b/test/dblwidth-a.sql index bcd60359f..38c219698 100644 --- a/test/dblwidth-a.sql +++ b/test/dblwidth-a.sql @@ -1,50 +1,20 @@ -#!sqlite3 /* ** Run this script using "sqlite3" to confirm that the command-line ** shell properly handles the output of double-width characters. ** ** https://sqlite.org/forum/forumpost/008ac80276 */ -.testcase 100 .mode box CREATE TABLE data(word TEXT, description TEXT); INSERT INTO data VALUES('〈οὐκέτι〉','Greek without dblwidth <...>'); +.print .mode box SELECT * FROM data; -.check <<END -╭────────────┬──────────────────────────────╮ -│ word │ description │ -╞════════════╪══════════════════════════════╡ -│ 〈οὐκέτι〉 │ Greek without dblwidth <...> │ -╰────────────┴──────────────────────────────╯ -END - -.testcase 200 .mode table +.print .mode table SELECT * FROM data; -.check <<END -+------------+------------------------------+ -| word | description | -+------------+------------------------------+ -| 〈οὐκέτι〉 | Greek without dblwidth <...> | -+------------+------------------------------+ -END - -.testcase 300 .mode qbox +.print .mode qbox SELECT * FROM data; -.check <<END -╭──────────────┬────────────────────────────────╮ -│ word │ description │ -╞══════════════╪════════════════════════════════╡ -│ '〈οὐκέτι〉' │ 'Greek without dblwidth <...>' │ -╰──────────────┴────────────────────────────────╯ -END - -.testcase 400 .mode column +.print .mode column SELECT * FROM data; -.check <<END - word description ----------- ---------------------------- -〈οὐκέτι〉 Greek without dblwidth <...> -END diff --git a/test/distinct2.test b/test/distinct2.test index 5de4d3094..980b0b1e3 100644 --- a/test/distinct2.test +++ b/test/distinct2.test @@ -311,7 +311,7 @@ do_execsql_test 4010 { } do_execsql_test 4020 { SELECT b FROM t1 UNION SELECT 1; -} {1 {}} +} {1 { }} #------------------------------------------------------------------------- # diff --git a/test/dotcmd01.sql b/test/dotcmd01.sql deleted file mode 100644 index 751d32659..000000000 --- a/test/dotcmd01.sql +++ /dev/null @@ -1,63 +0,0 @@ -#!sqlite3 -# -# 2026-01-30 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# Miscellaneous tests for dot-commands -# -# ./sqlite3 test/dotcmd01.sql -# - -.testcase setup -.open :memory: -.mode tty -.check '' - -# The ".eqp on" setting does not affect the output from .fullschema -# and similar. -# -.testctrl opt -Stat4 -.testcase 100 -CREATE TABLE t1(a,b,c); -WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<300) - INSERT INTO t1(a,b,c) - SELECT n%10, n%30, n%100 FROM c; -CREATE INDEX t1a ON t1(a); -CREATE INDEX t1b ON t1(b); -ANALYZE; -.eqp on -.fullschema -.check <<END -CREATE TABLE t1(a,b,c); -CREATE INDEX t1a ON t1(a); -CREATE INDEX t1b ON t1(b); -ANALYZE sqlite_schema; -INSERT INTO sqlite_stat1 VALUES('t1','t1b','300 10'); -INSERT INTO sqlite_stat1 VALUES('t1','t1a','300 30'); -ANALYZE sqlite_schema; -END - -.testcase 110 -.schema -.check <<END -CREATE TABLE t1(a,b,c); -CREATE INDEX t1a ON t1(a); -CREATE INDEX t1b ON t1(b); -CREATE TABLE sqlite_stat1(tbl,idx,stat); -END - -.testcase 120 -.tables -.check --glob "t1" - -.testcase 130 -.indexes -.check --glob t1a*t1b diff --git a/test/e_expr.test b/test/e_expr.test index 5c0bfb0c1..81d2fd172 100644 --- a/test/e_expr.test +++ b/test/e_expr.test @@ -1744,10 +1744,10 @@ do_execsql_test e_expr-32.2.8 { integer 9223372036854775807 \ integer 9223372036854775807 \ integer 9223372036854775807 \ - real 9.2233720368547758e+18 \ - real 9.2233720368547758e+18 \ - real 9.2233720368547758e+18 \ - real 9.2233720368547758e+18 \ + real 9.22337203685478e+18 \ + real 9.22337203685478e+18 \ + real 9.22337203685478e+18 \ + real 9.22337203685478e+18 \ integer -5 \ integer -5 \ ] diff --git a/test/e_update.test b/test/e_update.test index 2b3341dc7..a13b059b3 100644 --- a/test/e_update.test +++ b/test/e_update.test @@ -325,8 +325,6 @@ foreach {tn sql error ac data } { # EVIDENCE-OF: R-43190-62442 In other words, the schema-name. prefix on # the table name of the UPDATE is not allowed within triggers. # -# Update: Unless the trigger is in the temp schema. -# do_update_tests e_update-2.1 -error { qualified table names are not allowed on INSERT, UPDATE, and DELETE statements within triggers } { @@ -341,14 +339,12 @@ do_update_tests e_update-2.1 -error { UPDATE aux.t1 SET a=1, b=2; END; } {} -} -# Qualified table name is allowed as t4 is a temp table. -do_execsql_test e_update-2.1.3 { - CREATE TRIGGER tr1 AFTER DELETE ON t4 BEGIN - UPDATE main.t1 SET a=1, b=2; - END; - DROP TRIGGER tr1; + 3 { + CREATE TRIGGER tr1 AFTER DELETE ON t4 BEGIN + UPDATE main.t1 SET a=1, b=2; + END; + } {} } # EVIDENCE-OF: R-06085-13761 Unless the table to which the trigger is diff --git a/test/e_walckpt.test b/test/e_walckpt.test index 4aae52a78..3b1f3b015 100644 --- a/test/e_walckpt.test +++ b/test/e_walckpt.test @@ -605,15 +605,14 @@ foreach {tn script} { sqlite3 db test.db foreach {tn mode res} { 0 -1001 {1 {SQLITE_MISUSE - not an error}} - 1 -2 {1 {SQLITE_MISUSE - not an error}} - 2 -1 {0 {0 -1 -1}} - 3 0 {0 {0 -1 -1}} - 4 1 {0 {0 -1 -1}} - 5 2 {0 {0 -1 -1}} - 6 3 {0 {0 -1 -1}} - 7 4 {1 {SQLITE_MISUSE - not an error}} - 8 114 {1 {SQLITE_MISUSE - not an error}} - 9 1000000 {1 {SQLITE_MISUSE - not an error}} + 1 -1 {1 {SQLITE_MISUSE - not an error}} + 2 0 {0 {0 -1 -1}} + 3 1 {0 {0 -1 -1}} + 4 2 {0 {0 -1 -1}} + 5 3 {0 {0 -1 -1}} + 6 4 {1 {SQLITE_MISUSE - not an error}} + 7 114 {1 {SQLITE_MISUSE - not an error}} + 8 1000000 {1 {SQLITE_MISUSE - not an error}} } { do_test 4.$tn { list [catch "wal_checkpoint_v2 db $mode" msg] $msg diff --git a/test/eqp.test b/test/eqp.test index d2bdc49e3..147b5ceaf 100644 --- a/test/eqp.test +++ b/test/eqp.test @@ -124,10 +124,10 @@ do_eqp_test 1.8 { } { QUERY PLAN |--CO-ROUTINE (subquery-xxxxxx) - | `--MERGE (UNION) - | |--LEFT + | `--COMPOUND QUERY + | |--LEFT-MOST SUBQUERY | | `--SCAN CONSTANT ROW - | `--RIGHT + | `--UNION USING TEMP B-TREE | `--SCAN CONSTANT ROW |--SCAN (subquery-xxxxxx) `--SCAN t3 @@ -137,26 +137,24 @@ do_eqp_test 1.9 { } { QUERY PLAN |--CO-ROUTINE abc - | `--MERGE (EXCEPT) - | |--LEFT + | `--COMPOUND QUERY + | |--LEFT-MOST SUBQUERY | | `--SCAN CONSTANT ROW - | `--RIGHT - | |--SCAN t3 - | `--USE TEMP B-TREE FOR ORDER BY - |--SCAN t3 - `--SCAN abc + | `--EXCEPT USING TEMP B-TREE + | `--SCAN t3 + |--SCAN abc + `--SCAN t3 } do_eqp_test 1.10 { SELECT * FROM t3 JOIN (SELECT 1 INTERSECT SELECT a FROM t3 LIMIT 17) AS abc } { QUERY PLAN |--CO-ROUTINE abc - | `--MERGE (INTERSECT) - | |--LEFT + | `--COMPOUND QUERY + | |--LEFT-MOST SUBQUERY | | `--SCAN CONSTANT ROW - | `--RIGHT - | |--SCAN t3 - | `--USE TEMP B-TREE FOR ORDER BY + | `--INTERSECT USING TEMP B-TREE + | `--SCAN t3 |--SCAN abc `--SCAN t3 } @@ -457,11 +455,10 @@ do_eqp_test 4.3.1 { SELECT x FROM t1 UNION SELECT x FROM t2 } { QUERY PLAN - `--MERGE (UNION) - |--LEFT - | |--SCAN t1 - | `--USE TEMP B-TREE FOR ORDER BY - `--RIGHT + `--COMPOUND QUERY + |--LEFT-MOST SUBQUERY + | `--SCAN t1 + `--UNION USING TEMP B-TREE `--SCAN t2 USING COVERING INDEX t2i1 } @@ -469,17 +466,13 @@ do_eqp_test 4.3.2 { SELECT x FROM t1 UNION SELECT x FROM t2 UNION SELECT x FROM t1 } { QUERY PLAN - `--MERGE (UNION) - |--LEFT - | `--MERGE (UNION) - | |--LEFT - | | |--SCAN t1 - | | `--USE TEMP B-TREE FOR ORDER BY - | `--RIGHT - | `--SCAN t2 USING COVERING INDEX t2i1 - `--RIGHT - |--SCAN t1 - `--USE TEMP B-TREE FOR ORDER BY + `--COMPOUND QUERY + |--LEFT-MOST SUBQUERY + | `--SCAN t1 + |--UNION USING TEMP B-TREE + | `--SCAN t2 USING COVERING INDEX t2i1 + `--UNION USING TEMP B-TREE + `--SCAN t1 } do_eqp_test 4.3.3 { SELECT x FROM t1 UNION SELECT x FROM t2 UNION SELECT x FROM t1 ORDER BY 1 diff --git a/test/filectrl.test b/test/filectrl.test index eea9a4773..9b1a1c758 100644 --- a/test/filectrl.test +++ b/test/filectrl.test @@ -36,11 +36,13 @@ do_test filectrl-1.5 { sqlite3 db test_control_lockproxy.db file_control_lockproxy_test db [get_pwd] } {} -do_test filectrl-1.6 { - sqlite3 db test.db - set fn [file_control_tempfilename db] - set fn -} {/etilqs_/} +ifcapable !winrt { + do_test filectrl-1.6 { + sqlite3 db test.db + set fn [file_control_tempfilename db] + set fn + } {/etilqs_/} +} db close forcedelete .test_control_lockproxy.db-conch test.proxy forcedelete test.db test2.db diff --git a/test/fpconv1.test b/test/fpconv1.test index a93489907..195fdf990 100644 --- a/test/fpconv1.test +++ b/test/fpconv1.test @@ -17,82 +17,28 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl - -# Unusual rendering cases: -# -do_execsql_test fpconv1-1.0 { - SELECT 1.23 - 2.34; -} {-1.1099999999999999} -# ^--- Not -1.11 as you would expect. -1.11 has a different bit pattern - -do_execsql_test fpconv1-1.1 { - SELECT 1.23 * 2.34; -} {2.8781999999999996} -# ^--- Not 2.8782 as you would expect. 2.8782 has a different bit pattern - -# Change significant digits to 15 and get a different result. -sqlite3_db_config db FP_DIGITS 15 -do_execsql_test fpconv1-1.2 { - SELECT 1.23 - 2.34; -} {-1.11} -do_execsql_test fpconv1-1.3 { - SELECT 1.23 * 2.34; -} {2.8782} -sqlite3_db_config db FP_DIGITS 17 - - if {[catch {load_static_extension db decimal} error]} { puts "Skipping decimal tests, hit load error: $error" finish_test; return } sqlite3_create_function db -do_execsql_test fpconv1-2.0 { +do_execsql_test fpconv1-1.0 { WITH RECURSIVE /* Number of random floating-point values to try. - ** On a circa 2021 Ryzen 5950X running Mint Linux, and - ** compiled with -O0 -DSQLITE_DEBUG, this test runs at - ** about 150000 cases per second ------------------vvvvvvv */ - c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<500_000), + ** On a circa 2016 x64 linux box, this test runs at + ** about 80000 cases per second -------------------vvvvvv */ + c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100000), fp(y) AS MATERIALIZED ( SELECT CAST( format('%+d.%019d0e%+03d', random()%10,abs(random()),random()%200) AS real) FROM c ) SELECT y FROM fp - WHERE -log10(abs(decimal_sub(dtostr(y,24),format('%!.24e',y))/y))<17.0; + WHERE -log10(abs(decimal_sub(dtostr(y,24),format('%!.24e',y))/y))<15.0; /* Number of digits of accuracy required -------^^^^ */ } {} # ^---- Expect a empty set as the result. The output is all tested numbers -# that fail to preserve at least 16 significant digits of accuracy. - -######################################################################## -# Random test to ensure that double -> text -> double conversions -# round-trip exactly. -# - -load_static_extension db ieee754 - -do_execsql_test fpconv1-3.0 { - WITH RECURSIVE - c(x,s) AS MATERIALIZED (VALUES(1,random()&0xffefffffffffffff) - UNION ALL - SELECT x+1,random()&0xffefffffffffffff - FROM c WHERE x<1_000_000), - fp(y,s) AS ( - SELECT ieee754_from_int(s),s FROM c - ), - fp2(yt,y,s) AS ( - SELECT y||'', y, s FROM fp - ) - SELECT format('%#016x',s) aS 'orig-hex', - format('%#016x',ieee754_to_int(CAST(yt AS real))) AS 'full-roundtrip', - yt AS 'rendered-as', - decimal_exp(yt,30) AS 'text-decimal', - decimal_exp(ieee754_from_int(s),30) AS 'float-decimal' - FROM fp2 - WHERE ieee754_to_int(CAST(yt AS real))<>s; -} {} -# ^---- Values that fail to round-trip will be reported +# that fail to preserve at least 15 significant digits of accuracy. finish_test diff --git a/test/fptest01.sql b/test/fptest01.sql deleted file mode 100644 index 6221760cf..000000000 --- a/test/fptest01.sql +++ /dev/null @@ -1,76 +0,0 @@ -#!sqlite3 -# -# 2026-03-01 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# Floating-point to text conversions -# - -# Verify that binary64 -> text -> binary64 conversions round-trip -# successfully for 98,256 different edge-case binary64 values. The -# query result is all cases that do not round-trip without change, -# and so the query result should be an empty set. -# -.testcase 100 -.mode list -WITH - i1(i) AS (VALUES(0) UNION ALL SELECT i+1 FROM i1 WHERE i<15), - i2(j) AS (VALUES(0) UNION ALL SELECT j+1 FROM i2 WHERE j<0x7fe), - i3(k) AS (VALUES(0x0000000000000000), - (0x000ffffffffffff0), - (0x0008080808080800)), - fpint(n) AS (SELECT (j<<52)+i+k FROM i2, i1, i3), - fp(n,r) AS (SELECT n, ieee754_from_int(n) FROM fpint) -SELECT n, r FROM fp WHERE r<>(0.0 + (r||'')); -.check '' - -# Another batch of 106,444 edge cases: All postiive floating point -# values that have only a single bit set in the mantissa part of the -# number. -# -.testcase 110 -WITH - i1(i) AS (VALUES(0) UNION ALL SELECT i+1 FROM i1 WHERE i<51), - i2(j) AS (VALUES(0) UNION ALL SELECT j+1 FROM i2 WHERE j<0x7fe), - fpint(n) AS (SELECT (j<<52)+(1<<i) FROM i2, i1), - fp(n,r) AS (SELECT n, ieee754_from_int(n) FROM fpint) -SELECT n, r FROM fp WHERE r<>(0.0 + (r||'')); -.check '' - -# Verify that text -> binary64 conversions agree with system strtod(). -# for 98,256 different edge-cases. -# -.testcase 200 -.mode list -WITH - i1(i) AS (VALUES(0) UNION ALL SELECT i+1 FROM i1 WHERE i<15), - i2(j) AS (VALUES(0) UNION ALL SELECT j+1 FROM i2 WHERE j<0x7fe), - i3(k) AS (VALUES(0x0000000000000000), - (0x000ffffffffffff0), - (0x0008080808080800)), - fpint(n) AS (SELECT (j<<52)+i+k FROM i2, i1, i3), - fp(r) AS (SELECT ieee754_from_int(n) FROM fpint) -SELECT r FROM fp WHERE r<>strtod(r||''); -.check '' - - -# Another batch of 106,444 edge cases: All postiive floating point -# values that have only a single bit set in the mantissa part of the -# number. -# -.testcase 210 -WITH - i1(i) AS (VALUES(0) UNION ALL SELECT i+1 FROM i1 WHERE i<51), - i2(j) AS (VALUES(0) UNION ALL SELECT j+1 FROM i2 WHERE j<0x7fe), - fpint(n) AS (SELECT (j<<52)+(1<<i) FROM i2, i1), - fp(r) AS (SELECT ieee754_from_int(n) FROM fpint) -SELECT r FROM fp WHERE r<>strtod(r||''); -.check '' diff --git a/test/fts3comp1.test b/test/fts3comp1.test index b5077a355..9f13aaa64 100644 --- a/test/fts3comp1.test +++ b/test/fts3comp1.test @@ -112,81 +112,4 @@ do_catchsql_test 2.2 { CREATE VIRTUAL TABLE t2 USING fts4(x, uncompress=unzip) } {1 {missing compress parameter in fts4 constructor}} -#-------------------------------------------------------------------------- -reset_db -do_execsql_test 3.0 { - PRAGMA trusted_schema = OFF; -} - -set ::myfunc_invoked 0 -proc myfunc {data} { - incr ::myfunc_invoked - return $data -} -db func myfunc myfunc - -do_execsql_test 3.1 { - CREATE VIEW v1 AS SELECT myfunc('xyz'); -} - -do_catchsql_test 3.2 { - SELECT * FROM v1 -} {1 {unsafe use of myfunc()}} - -do_execsql_test 3.3 { - CREATE VIRTUAL TABLE f1 USING fts4(x, compress=myfunc, uncompress=myfunc); -} - -do_catchsql_test 3.4 { - INSERT INTO f1(rowid, x) VALUES(123, 'one two three'); -} {1 {SQL logic error}} - -do_test 3.5 { - set ::myfunc_invoked -} {0} - -do_execsql_test 3.6.1 { - CREATE TABLE t1(x); - CREATE TABLE t2(y); - - CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN - INSERT INTO t2 VALUES( myfunc(new.x) ); - END; -} - -do_catchsql_test 3.6.2 { - INSERT INTO t1 VALUES('hello world'); -} {1 {unsafe use of myfunc()}} - -#------------------------------------------------------------------------- -reset_db -do_execsql_test 4.0 { - CREATE VIRTUAL TABLE v1 USING fts4(x, compress=comp, uncompress=uncomp); -} - -proc comp {data} { return $data } -proc uncomp {data} { return $data } - -db func comp comp -db func uncomp uncomp - -do_catchsql_test 4.1 { - INSERT INTO v1 VALUES('one two three'); -} {0 {}} - -db close -sqlite3 db test.db -db func comp -directonly comp - -do_catchsql_test 4.2 { - INSERT INTO v1 VALUES('one two three'); -} {1 {SQL logic error}} - -db func uncomp -directonly uncomp - -do_catchsql_test 4.3 { - SELECT * FROM v1 -} {1 {SQL logic error}} - - finish_test diff --git a/test/fts4content.test b/test/fts4content.test index 8268e734a..980586ea3 100644 --- a/test/fts4content.test +++ b/test/fts4content.test @@ -638,6 +638,7 @@ do_catchsql_test 11.1 { # Check that an fts4 table cannot be its own content table. # reset_db +breakpoint do_execsql_test 12.1.1 { CREATE VIRTUAL TABLE t1 USING fts4(a, content=t1 ); INSERT INTO t1(rowid, a) VALUES(1, 'abc'); @@ -668,64 +669,6 @@ do_catchsql_test 12.2.4 { SELECT count(*) FROM t1; } {1 {SQL logic error}} -#--------------------------------------------------------------------------- -# Check that an fts4 table cannot read from an unsafe vtab in a non-trusted -# schema. -reset_db -do_execsql_test 13.0 { - PRAGMA trusted_schema = off; - CREATE VIRTUAL TABLE t1 USING fts4(data, content=sqlite_dbpage); -} - -do_catchsql_test 13.1 { - INSERT INTO t1(t1) VALUES('rebuild'); -} {1 {SQL logic error}} - -proc vtab_command {method args} { - switch -- $method { - xConnect { - return "CREATE TABLE t1(a)" - } - - xBestIndex { - return "" - } - - xFilter { - return [list sql {SELECT 1, 123}] - } - - xUpdate { - return 123 - } - } - - return {} -} - -register_tcl_module db xyz - -do_execsql_test 13.2.0 { - CREATE VIRTUAL TABLE aa USING tcl(vtab_command); -} - -do_execsql_test 13.2.1 { - INSERT INTO aa VALUES('one two three'); -} - -do_test 13.2.2 { - set ::stmt [sqlite3_prepare_v3 db \ - "INSERT INTO aa VALUES('one two three');" -1 0x00 - ] - sqlite3_finalize $::stmt -} {SQLITE_OK} -do_test 13.2.2 { - list [catch { - set ::stmt [sqlite3_prepare_v3 db \ - "INSERT INTO aa VALUES('one two three');" -1 0x20 - ] - } msg] $msg -} {1 {(1) unsafe use of virtual table "aa"}} finish_test diff --git a/test/fts4merge5.test b/test/fts4merge5.test index 90870ca34..1fad778b9 100644 --- a/test/fts4merge5.test +++ b/test/fts4merge5.test @@ -53,9 +53,6 @@ for {set tn 1} {1} {incr tn} { } } -do_catchsql_test 1.5 { - INSERT INTO x1(x1) VALUES('maxpendinAB64'); -} {1 {SQL logic error}} finish_test diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index f056d2d93..a3377770a 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -1954,7 +1954,6 @@ int main(int argc, char **argv){ char **azSrcDb = 0; /* Array of source database names */ int iSrcDb; /* Loop over all source databases */ int nTest = 0; /* Total number of tests performed */ - int nSliceSkip = 0; /* Skipped due to --slice */ char *zDbName = ""; /* Appreviated name of a source database */ const char *zFailCode = 0; /* Value of the TEST_FAILURE env variable */ int cellSzCkFlag = 0; /* --cell-size-check */ @@ -2521,7 +2520,6 @@ int main(int argc, char **argv){ if( isDbSql(pSql->a, pSql->sz) ){ if( iSliceSz>0 && (nTest%iSliceSz)!=iSliceIdx ){ nTest++; - nSliceSkip++; continue; } sqlite3_snprintf(sizeof(g.zTestName), g.zTestName, "sqlid=%d",pSql->id); @@ -2584,7 +2582,6 @@ int main(int argc, char **argv){ const char *zVfs = "inmem"; if( iSliceSz>0 && (nTest%iSliceSz)!=iSliceIdx ){ nTest++; - nSliceSkip++; continue; } sqlite3_snprintf(sizeof(g.zTestName), g.zTestName, "sqlid=%d,dbid=%d", @@ -2696,13 +2693,12 @@ int main(int argc, char **argv){ if( briefFlag ){ sqlite3_int64 iElapse = timeOfDay() - iBegin; if( iSliceSz>0 ){ - printf( - "%s %s: 0 errors out of %d tests, slice %d/%d, %d.%03d seconds\n", - pathTail(argv[0]), pathTail(g.zDbFile), - nTest - nSliceSkip, iSliceIdx, iSliceSz, - (int)(iElapse/1000), (int)(iElapse%1000)); + printf("%s %s: slice %d/%d of %d tests, %d.%03d seconds\n", + pathTail(argv[0]), pathTail(g.zDbFile), + iSliceIdx, iSliceSz, nTest, + (int)(iElapse/1000), (int)(iElapse%1000)); }else{ - printf("%s %s: 0 errors out of %d tests, %d.%03d seconds\n", + printf("%s %s: 0 errors, %d tests, %d.%03d seconds\n", pathTail(argv[0]), pathTail(g.zDbFile), nTest, (int)(iElapse/1000), (int)(iElapse%1000)); } diff --git a/test/fuzzinvariants.c b/test/fuzzinvariants.c index 9341b2056..6a5cfda68 100644 --- a/test/fuzzinvariants.c +++ b/test/fuzzinvariants.c @@ -261,12 +261,9 @@ int fuzz_invariant( sqlite3_finalize(pCk); /* Invariants do not necessarily work if there are virtual tables - ** or scalar subqueries involved in the query */ - rc = sqlite3_prepare_v2(db, - "SELECT 1 FROM bytecode(?1)" - " WHERE opcode='VOpen' OR" - " (opcode='Explain' AND p4 GLOB 'SCALAR SUBQUERY*')", - -1, &pCk, 0); + ** involved in the query */ + rc = sqlite3_prepare_v2(db, + "SELECT 1 FROM bytecode(?1) WHERE opcode='VOpen'", -1, &pCk, 0); if( rc==SQLITE_OK ){ if( eVerbosity>=2 ){ char *zSql = sqlite3_expanded_sql(pCk); diff --git a/test/import01.sql b/test/import01.sql deleted file mode 100644 index 6e029b8b2..000000000 --- a/test/import01.sql +++ /dev/null @@ -1,217 +0,0 @@ -#!sqlite3 -# -# 2025-12-28 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# Test cases for the ".import" command of the CLI. -# To run these tests: -# -# ./sqlite3 test/import01.sql -# - -.testcase setup -.open :memory: -.mode tty -.check '' - -.testcase 100 -CREATE TABLE t1(a,b,c); -.import -csv <<END t1 -111,222,333 -abc,def,ghi -END -SELECT * FROM t1; -.check <<END -╭───────┬───────┬───────╮ -│ a │ b │ c │ -╞═══════╪═══════╪═══════╡ -│ '111' │ '222' │ '333' │ -│ abc │ def │ ghi │ -╰───────┴───────┴───────╯ -END - -.testcase 110 -DELETE FROM t1; -.import -colsep ";" <<END t1 -this;is a;test -one;two;three -END -SELECT * FROM t1; -.check <<END -╭──────┬──────┬───────╮ -│ a │ b │ c │ -╞══════╪══════╪═══════╡ -│ this │ is a │ test │ -│ one │ two │ three │ -╰──────┴──────┴───────╯ -END - -.testcase 120 -DELETE FROM t1; -.import -colsep "," -rowsep ';' <<END t1 -this,is a,test;one,two,three; -END -SELECT * FROM t1; -.check <<END -╭──────┬──────┬───────╮ -│ a │ b │ c │ -╞══════╪══════╪═══════╡ -│ this │ is a │ test │ -│ one │ two │ three │ -╰──────┴──────┴───────╯ -END - -.testcase 130 -DELETE FROM t1; -.import -csv -colsep "," -rowsep "\n" <<END t1 -this,"is a","test ""with quotes""" -"second",,"line" -END -SELECT * FROM t1; -.check <<END -╭────────┬──────┬────────────────────╮ -│ a │ b │ c │ -╞════════╪══════╪════════════════════╡ -│ this │ is a │ test "with quotes" │ -│ second │ │ line │ -╰────────┴──────┴────────────────────╯ -END -.testcase 131 -DELETE FROM t1; -.import -ascii -colsep "," -rowsep "\n" <<END t1 -this,"is a","test ""with quotes""" -"second",,"line" -END -SELECT * FROM t1; -.check <<END -╭──────────┬────────┬────────────────────────╮ -│ a │ b │ c │ -╞══════════╪════════╪════════════════════════╡ -│ this │ "is a" │ "test ""with quotes""" │ -│ "second" │ │ "line" │ -╰──────────┴────────┴────────────────────────╯ -END - -.testcase 140 -DROP TABLE t1; -.import -csv <<END t1 -"abc","def","xy z" -"This","is","a" -"test","...", -END -SELECT * FROM t1; -.check <<END -╭──────┬─────┬──────╮ -│ abc │ def │ xy z │ -╞══════╪═════╪══════╡ -│ This │ is │ a │ -│ test │ ... │ │ -╰──────┴─────┴──────╯ -END -.testcase 141 -SELECT sql FROM sqlite_schema WHERE name='t1'; -.check <<END -╭───────────────────────────────────╮ -│ sql │ -╞═══════════════════════════════════╡ -│ CREATE TABLE "t1"( │ -│ "abc" ANY, "def" ANY, "xy z" ANY) │ -╰───────────────────────────────────╯ -END - -.testcase 150 -DROP TABLE t1; -.import -csv -v <<END t1 -"abc","def","xy z" -"This","is","a" -"test","...", -END -SELECT * FROM t1; -.check <<END -Column separator ",", row separator "\n" -CREATE TABLE "main"."t1"( -"abc" ANY, "def" ANY, "xy z" ANY) - -Added 2 rows with 0 errors using 3 lines of input -╭──────┬─────┬──────╮ -│ abc │ def │ xy z │ -╞══════╪═════╪══════╡ -│ This │ is │ a │ -│ test │ ... │ │ -╰──────┴─────┴──────╯ -END - -.testcase 160 -DROP TABLE t1; -.import -csv -schema TEMP <<END t2 -"x" -"abcdef" -END -SELECT * FROM t2; -.tables -.check <<END -╭────────╮ -│ x │ -╞════════╡ -│ abcdef │ -╰────────╯ -temp.t2 -END - -.testcase 170 -.import -csv -skip 2 <<END t3 -a,b,c -d,e,f -g,h,i -j,k,l -m,n,o -END -SELECT * FROM t3; -.check <<END -╭───┬───┬───╮ -│ g │ h │ i │ -╞═══╪═══╪═══╡ -│ j │ k │ l │ -│ m │ n │ o │ -╰───┴───┴───╯ -END - -.testcase 180 -DELETE FROM t3; -.import -csv -skip 7 <<END t3 -a,b,c -d,e,f -g,h,i -j,k,l -m,n,o -END -SELECT * FROM t3; -.check <<END -END - -.testcase 200 --error-prefix ERROR: -.import -csv -.check 'ERROR: Missing FILE argument' -.testcase 201 -.import -csv file1.csv -.check 'ERROR: Missing TABLE argument' -.testcase --error-prefix test-error: 202 -.import -csvxyzzy file1.csv -.check <<END -test-error: .import -csvxyzzy file1.csv -test-error: ^--- unknown option -END -.testcase 203 -.import -csv file1.csv t4 -colsep -.check <<END -test-error: .import -csv file1.csv t4 -colsep -test-error: missing argument ---^ -END diff --git a/test/imposter1.sql b/test/imposter1.sql deleted file mode 100644 index 9604b43f4..000000000 --- a/test/imposter1.sql +++ /dev/null @@ -1,32 +0,0 @@ -#!sqlite3 -# -# 2025-12-16 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# Test cases for the .imposter command. -# -.mode box -reset -.testcase 100 -CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c INT); -INSERT INTO t1 VALUES(1,'two',3),(4,'five',6); -CREATE INDEX t1bc ON t1(b,c); -.imposter T1BC x1 -----------^^^^--- Different case that the original -SELECT * FROM x1; -.check <<END -CREATE TABLE "x1"("b","c","_ROWID_",PRIMARY KEY("b","c","_ROWID_"))WITHOUT ROWID; -╭──────┬───┬─────────╮ -│ b │ c │ _ROWID_ │ -╞══════╪═══╪═════════╡ -│ five │ 6 │ 4 │ -│ two │ 3 │ 1 │ -╰──────┴───┴─────────╯ -END diff --git a/test/insert5.test b/test/insert5.test index 3ae99a5d2..1e58902e0 100644 --- a/test/insert5.test +++ b/test/insert5.test @@ -96,19 +96,22 @@ do_test insert5-2.8 { } } {1} +# UPDATE: Using a column from the outer query (main.id) in the GROUP BY +# or ORDER BY of a sub-query is no longer supported. +# +# do_test insert5-2.9 { +# uses_temp_table { +# INSERT INTO b +# SELECT * FROM main +# WHERE id > 10 AND (SELECT count(*) FROM v2 GROUP BY main.id) +# } +# } {} do_test insert5-2.9 { catchsql { INSERT INTO b SELECT * FROM main WHERE id > 10 AND (SELECT count(*) FROM v2 GROUP BY main.id) } -} {0 {}} -do_execsql_test insert5-2.10 { - CREATE TABLE t1(a INT); - INSERT INTO t1 VALUES(2); - CREATE TABLE t2(c INT, d INT); - INSERT INTO t2 VALUES(3,4),(10,NULL); - SELECT (SELECT c FROM t2 ORDER BY coalesce(d,a) LIMIT 1) FROM t1; -} {10} +} {1 {no such column: main.id}} finish_test diff --git a/test/intck01.sql b/test/intck01.sql deleted file mode 100644 index b1996aeeb..000000000 --- a/test/intck01.sql +++ /dev/null @@ -1,23 +0,0 @@ -#!sqlite3 -# -# 2026-03-01 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# Bug report sqlite.org/forum/forumpost/efc9bc9cb3 -# -.testcase 100 -.mode quote -.intck 1 -SELECT parse_create_index('CREATE IDEX i ON t("x',0); -.check <<END -1 steps, 0 errors -NULL -END diff --git a/test/join.test b/test/join.test index a1ce7da0c..b33a7560a 100644 --- a/test/join.test +++ b/test/join.test @@ -1369,45 +1369,4 @@ do_execsql_test join-32.3 { INNER JOIN t2 ON +y IS z; } {NULL NULL 123 NULL} -# 2025-12-24 https://sqlite.org/forum/forumpost/11a53f2bad -# -# Chained omit-noop-join optimization -# -reset_db -db null NULL -do_execsql_test join-33.1 { - CREATE TABLE t1(a1 INTEGER PRIMARY KEY, b1); - CREATE TABLE t2(a2 INTEGER PRIMARY KEY, b2); - CREATE TABLE t3(a3 INTEGER PRIMARY KEY, b3); - CREATE TABLE t4(a4 INTEGER PRIMARY KEY, b4); - INSERT INTO t1 VALUES(1,11),(2,12),(3,13), (5,15); - INSERT INTO t2 VALUES(1,21), (3,23),(4,24),(5,25); - INSERT INTO t3 VALUES (2,32),(3,33), (5,35); - INSERT INTO t4 VALUES(1,41),(2,42), (4,44),(5,45); - CREATE VIEW vchain AS - SELECT a1, b1, b2, b3, b4 - FROM t1 LEFT JOIN t2 ON a1=a2 - LEFT JOIN t3 ON a2=a3 - LEFT JOIN t4 ON a3=a4; -} -do_execsql_test join-33.2 { - SELECT a1 FROM vchain ORDER BY a1; -} {1 2 3 5} -do_eqp_test join-33.2-eqp { - SELECT a1 FROM vchain ORDER BY a1; -} { - QUERY PLAN - `--SCAN t1 -} -do_execsql_test join-33.3 { - SELECT a1, b2 FROM vchain ORDER BY a1; -} {1 21 2 NULL 3 23 5 25} -do_eqp_test join-33.3-eqp { - SELECT a1, b2 FROM vchain ORDER BY a1; -} { - QUERY PLAN - |--SCAN t1 - `--SEARCH t2 USING INTEGER PRIMARY KEY (rowid=?) LEFT-JOIN -} - finish_test diff --git a/test/joinI.test b/test/joinI.test index 3390afc6e..d0194579d 100644 --- a/test/joinI.test +++ b/test/joinI.test @@ -168,5 +168,48 @@ do_catchsql_test 6.8 { ) FROM t5; } {1 {ON clause references tables to its right}} +#------------------------------------------------------------------------- +# Forum post: https://sqlite.org/forum/forumpost/e3dba5426a +# +reset_db +do_execsql_test 7.0 { + CREATE TABLE t1(a); + INSERT INTO t1 VALUES(1); + CREATE TABLE t2(d, e); + CREATE INDEX t2def ON t2(d, (e+1)); + INSERT INTO t2 VALUES(1, 3); + INSERT INTO t2 VALUES(2, 555); + INSERT INTO t2 VALUES(3, 3); +} +do_execsql_test 7.1 { + SELECT * FROM t1 RIGHT JOIN t2 ON ( a=d ) WHERE (t2.e+1)!=4; +} {{} 2 555} + +reset_db +do_execsql_test 7.2 { + CREATE TABLE rt0(c0 INTEGER PRIMARY KEY, c1, c2); + CREATE TABLE t1 (c1 INTEGER, c2 BLOB, c4 INTEGER); + CREATE UNIQUE INDEX i81 ON t1(c1, c4, +c2); + + INSERT INTO t1(c4) VALUES ('a'); + INSERT INTO rt0(c1, c2) VALUES (0.0, 0.0); + INSERT INTO t1(c2, c1) VALUES (1, 'b'); +} +do_execsql_test 7.2 { + SELECT count(*) FROM ( + SELECT DISTINCT * FROM rt0 FULL JOIN t1 ON rt0.c0=t1.c1 + WHERE ((rt0.c1 OR t1.c4)) IS NOT (+ t1.c2) + ); +} {1} +do_execsql_test 7.3 { + SELECT count(*) FROM ( + SELECT DISTINCT * FROM rt0 LEFT JOIN t1 ON rt0.c0=t1.c1 + WHERE ((rt0.c1 OR t1.c4)) IS NOT (+ t1.c2) + UNION + SELECT DISTINCT * FROM rt0 RIGHT JOIN t1 ON rt0.c0=t1.c1 + WHERE ((rt0.c1 OR t1.c4)) IS NOT (+ t1.c2) + ); +} {1} + finish_test diff --git a/test/json102.test b/test/json102.test index 54f66a83f..54a0e1e0e 100644 --- a/test/json102.test +++ b/test/json102.test @@ -375,27 +375,6 @@ do_execsql_test json102-440-3 { do_execsql_test json102-440-4 { SELECT json(jsonb_remove(jsonb('[0,1,2,3,4]'),'$[2]')); } {{[0,1,3,4]}} -do_execsql_test json102-445-1 { - SELECT json_remove('[0,1,2,3,4]','$[5]'); -} {{[0,1,2,3,4]}} -do_execsql_test json102-445-2 { - SELECT json_remove('[0,1,2,3,4]','$[6]'); -} {{[0,1,2,3,4]}} -do_execsql_test json102-445-3 { - SELECT json_remove('[0,1,2,3,4]','$[4294967295]'); -} {{[0,1,2,3,4]}} -do_execsql_test json102-445-4 { - SELECT json_remove('[0,1,2,3,4]','$[4294967296]'); -} {{[0,1,2,3,4]}} -do_execsql_test json102-445-5 { - SELECT json_remove('[0,1,2,3,4]','$[4294967297]'); -} {{[0,1,2,3,4]}} -do_execsql_test json102-445-6 { - SELECT json_remove('[0,1,2,3,4]','$[42949672950]'); -} {{[0,1,2,3,4]}} -do_execsql_test json102-445-7 { - SELECT json_remove('[0,1,2,3,4]','$[42949672960]'); -} {{[0,1,2,3,4]}} do_execsql_test json102-450 { SELECT json_remove('[0,1,2,3,4]','$[2]','$[0]'); } {{[1,3,4]}} diff --git a/test/json103.test b/test/json103.test index 9eadf29f8..f94217ac1 100644 --- a/test/json103.test +++ b/test/json103.test @@ -27,9 +27,6 @@ do_execsql_test json103-100 { do_catchsql_test json103-101 { SELECT json_group_array(a) FROM t1; } {1 {JSON cannot hold BLOB values}} -do_execsql_test json103-102 { - SELECT quote(jsonb_group_array(a)) FROM t1 WHERE a<0 AND typeof(a)!='blob'; -} {X'0B'} do_execsql_test json103-110 { SELECT json_group_array(a) FROM t1 WHERE rowid BETWEEN 31 AND 39; @@ -48,10 +45,6 @@ do_execsql_test json103-200 { do_catchsql_test json103-201 { SELECT json_group_object(c,a) FROM t1; } {1 {JSON cannot hold BLOB values}} -do_execsql_test json103-202 { - SELECT quote(jsonb_group_object(c,a)) FROM t1 WHERE a<0 AND typeof(a)!='blob'; -} {X'0C'} - do_execsql_test json103-210 { SELECT json_group_object(c,a) FROM t1 diff --git a/test/json105.test b/test/json105.test index 4a5572cf0..509db94e1 100644 --- a/test/json105.test +++ b/test/json105.test @@ -29,11 +29,6 @@ json_extract_test 30 {'$.b[#-2]'} {'[2,3]'} json_extract_test 31 {'$.b[#-02]'} {'[2,3]'} json_extract_test 40 {'$.b[#-3]'} 1 json_extract_test 50 {'$.b[#-4]'} NULL -json_extract_test 51 {'$.b[#-4296967295]'} NULL -json_extract_test 52 {'$.b[#-4296967296]'} NULL -json_extract_test 53 {'$.b[#-4296967297]'} NULL -json_extract_test 54 {'$.b[#-42969672950]'} NULL -json_extract_test 55 {'$.b[#-42969672960]'} NULL json_extract_test 60 {'$.b[#-2][#-1]'} 3 json_extract_test 70 {'$.b[0]','$.b[#-1]'} {'[1,4]'} diff --git a/test/json109.test b/test/json109.test deleted file mode 100644 index 1631a1f0f..000000000 --- a/test/json109.test +++ /dev/null @@ -1,72 +0,0 @@ -# 2026-01-17 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# Test cases for json_array_insert(). -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl -set testprefix json109 - -do_execsql_test 1.1 { - SELECT json_array_insert('[1,2,3]','$[0]',999,'$[0]',888); -} {{[888,999,1,2,3]}} -do_execsql_test 1.2 { - SELECT json_array_insert('[1,2,3]','$[0]',999,'$[#]',888); -} {{[999,1,2,3,888]}} -do_execsql_test 1.3 { - SELECT json_array_insert('[1,2,3]','$[1]',888); -} {{[1,888,2,3]}} -do_execsql_test 1.4 { - SELECT json_array_insert('[1,2,3]','$[2]',888); -} {{[1,2,888,3]}} -do_execsql_test 1.5 { - SELECT json_array_insert('[1,2,3]','$[3]',888); -} {{[1,2,3,888]}} -do_execsql_test 1.6 { - SELECT json_array_insert('[1,2,3]','$[#-1]',888); -} {{[1,2,888,3]}} -do_execsql_test 1.7 { - SELECT json_array_insert('[1,2,3]','$[#-2]',888); -} {{[1,888,2,3]}} -do_execsql_test 1.8 { - SELECT json_array_insert('[1,2,3]','$[#-3]',888); -} {{[888,1,2,3]}} -do_execsql_test 1.9 { - SELECT json_array_insert('[1,2,3]','$[#-4]',888); -} {{[1,2,3]}} - -do_catchsql_test 2.1 { - SELECT json_array_insert('{a:[1,2,3]}','$.a',888); -} {1 {not an array element: '$.a'}} -do_catchsql_test 2.2 { - SELECT json_array_insert('{a:[1,2,3]}','$.b',888); -} {1 {not an array element: '$.b'}} -do_catchsql_test 2.3 { - SELECT json_array_insert('{a:[1,2,3]}','$.b[0]',888); -} {0 {{{"a":[1,2,3],"b":[888]}}}} -do_catchsql_test 2.4 { - SELECT json_array_insert('{a:[1,2,3]}','$.b.c.d[0]',888); -} {0 {{{"a":[1,2,3],"b":{"c":{"d":[888]}}}}}} -do_catchsql_test 2.5 { - SELECT json_array_insert('{a:[1,2,3]}','$.b.c.d[0',888); -} {1 {not an array element: '$.b.c.d[0'}} -do_catchsql_test 2.6 { - SELECT json_array_insert('{a:[1,2,3]}','$.b.c.d',888); -} {1 {not an array element: '$.b.c.d'}} -do_catchsql_test 2.7 { - SELECT json_array_insert('{a:[1,2,3]}','$[0]',888); -} {0 {{{"a":[1,2,3]}}}} -do_catchsql_test 2.8 { - SELECT json_array_insert('{a:[1,2,3]}','$.b[0]',888,'$.a[1]','999','$.c',0); -} {1 {not an array element: '$.c'}} - -finish_test diff --git a/test/misc5.test b/test/misc5.test index 80b8d3c67..43ee2781a 100644 --- a/test/misc5.test +++ b/test/misc5.test @@ -595,7 +595,7 @@ do_test misc5-7.1.2 { } append sql "0$tail); SELECT * FROM t1;" catchsql $sql -} {1 {Recursion limit}} +} {0 900} # Parser stack overflow is silently ignored when it occurs while parsing the diff --git a/test/modeA.sql b/test/modeA.sql deleted file mode 100644 index 4e62093b2..000000000 --- a/test/modeA.sql +++ /dev/null @@ -1,303 +0,0 @@ -#!sqlite3 -# -# 2025-11-12 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# Test cases for the ".mode" command of the CLI. -# To run these tests: -# -# ./sqlite3 test/modeA.sql -# -# -.open :memory: -CREATE TABLE t1(a,b,c,d,e); -INSERT INTO t1 VALUES(1,2.5,'three',x'4444',NULL); -INSERT INTO t1 SELECT b,c,d,e,a FROM t1; -INSERT INTO t1 SELECT d,e,a,b,c FROM t1; -.mode box - -.testcase 100 -SELECT * FROM t1; -.check <<END -╭─────┬───────┬───────┬───────┬───────╮ -│ a │ b │ c │ d │ e │ -╞═════╪═══════╪═══════╪═══════╪═══════╡ -│ 1 │ 2.5 │ three │ DD │ │ -│ 2.5 │ three │ DD │ │ 1 │ -│ DD │ │ 1 │ 2.5 │ three │ -│ │ 1 │ 2.5 │ three │ DD │ -╰─────┴───────┴───────┴───────┴───────╯ -END - -.testcase 110 -.mode --null xyz -SELECT * FROM t1; -.check <<END -╭─────┬───────┬───────┬───────┬───────╮ -│ a │ b │ c │ d │ e │ -╞═════╪═══════╪═══════╪═══════╪═══════╡ -│ 1 │ 2.5 │ three │ DD │ xyz │ -│ 2.5 │ three │ DD │ xyz │ 1 │ -│ DD │ xyz │ 1 │ 2.5 │ three │ -│ xyz │ 1 │ 2.5 │ three │ DD │ -╰─────┴───────┴───────┴───────┴───────╯ -END - -# Default output mode is qbox --quote relaxed -# -.mode tty --wrap 10 -CREATE TABLE t2(a,b,c,d); -INSERT INTO t2 VALUES(1,2.5,'three',x'4444'); -INSERT INTO t2 VALUES('The quick fox jumps over the lazy brown dog',2,3,4); -INSERT INTO t2 VALUES('10','', -1.25,NULL); -INSERT INTO t2 VALUES('a,b,c','"Double-Quoted"','-1.25','NULL'); -.testcase 120 -SELECT * FROM t2; -.check <<END -╭────────────┬────────────┬─────────┬─────────╮ -│ a │ b │ c │ d │ -╞════════════╪════════════╪═════════╪═════════╡ -│ 1 │ 2.5 │ three │ x'4444' │ -├────────────┼────────────┼─────────┼─────────┤ -│ The quick │ 2 │ 3 │ 4 │ -│ fox jumps │ │ │ │ -│ over the │ │ │ │ -│ lazy brown │ │ │ │ -│ dog │ │ │ │ -├────────────┼────────────┼─────────┼─────────┤ -│ '10' │ │ -1.25 │ NULL │ -├────────────┼────────────┼─────────┼─────────┤ -│ a,b,c │ "Double- │ '-1.25' │ 'NULL' │ -│ │ Quoted" │ │ │ -╰────────────┴────────────┴─────────┴─────────╯ -END -.testcase 130 -.mode -.check <<END -.mode qbox --limits on --quote relaxed --sw auto --textjsonb on -END -.testcase 140 -.mode -v -.check <<END -.mode qbox --align "" --border on --blob-quote auto --colsep "" --escape auto --limits on --null "NULL" --quote relaxed --rowsep "" --sw auto --tablename "" --textjsonb on --titles on --widths "" --wordwrap off --wrap 10 -END -.testcase 150 --error-prefix "Error:" -.mode foo -.check <<END -Error: .mode foo -Error: ^--- unknown mode -Error: Use ".help .mode" for more info -END - -.testcase 160 -.mode --null xyzzy -v -.output -glob ' --null "xyzzy"' -.testcase 170 -.mode -null abcde -v -.output -glob ' --null "abcde"' - -# Test cases for the ".explain off" command -.mode box -reset -.testcase 180 -EXPLAIN SELECT * FROM t1; -.output --notglob *────* --keep -.output --notglob "* id │ parent │ notused │ detail *" --keep -.output --glob "* Init *" -.testcase 190 -EXPLAIN QUERY PLAN SELECT * FROM t1; -.output --glob "*`--SCAN *" -.explain off -.testcase 200 -EXPLAIN SELECT * FROM t1; -.output --glob *────* -.testcase 210 -EXPLAIN QUERY PLAN SELECT * FROM t1; -.output --glob "* id │ parent │ notused │ detail *" -.explain auto - -# Test cases for limit settings in the .mode command. -.testcase 300 -.mode box --reset -.mode -.check <<END -.mode box -END -.testcase 310 -.mode --limits 5,300,20 -.mode -.check <<END -.mode box --limits on -END -.testcase 320 -.mode --limits 5,300,19 -.mode -.check <<END -.mode box --limits 5,300,19 -END -.testcase 330 -.mode --limits 0,0,0 -.mode -v -.check <<END -.mode box --align "" --border on --blob-quote auto --colsep "" --escape auto --limits off --null "" --quote off --rowsep "" --sw 0 --tablename "" --textjsonb off --titles on --widths "" --wordwrap off -END - -.testcase 400 -.mode --linelimit 123 -.mode -.check <<END -.mode box --limits 123,0,0 -END - -.testcase 410 -.mode --linelimit 0 -charlimit 123 -.mode -.check <<END -.mode box --limits 0,123,0 -END - -.testcase 420 -.mode --charlimit 0 -titlelimit 123 -.mode -.check <<END -.mode box --limits 0,0,123 -END - -.testcase 430 -.mode list -.mode -.check <<END -.mode list -END - -.testcase 440 -.mode -limits 0,123,0 -.mode -.check <<END -.mode list --limits 0,123,0 -END - -.testcase 450 -.mode -limits 123,0,0 -.mode -.check <<END -.mode list -END - -# --titlelimit functionality -# -.testcase 500 -.mode line --limits off --titlelimit 20 -SELECT a AS 'abcdefghijklmnopqrstuvwxyz', b FROM t2 WHERE c=3; -.check <<END -abcdefghijklmnopq...: The quick fox jumps over the lazy brown dog - b: 2 -END -.testcase 510 -.mode line --titlelimit 10 -SELECT a AS 'abcdefghijklmnopqrstuvwxyz', b FROM t2 WHERE c=3; -.check <<END -abcdefg...: The quick fox jumps over the lazy brown dog - b: 2 -END -.testcase 520 -.mode line --titlelimit 2 -SELECT a AS 'abcdefghijklmnopqrstuvwxyz', b FROM t2 WHERE c=3; -.check <<END -ab: The quick fox jumps over the lazy brown dog - b: 2 -END -.testcase 530 -.mode line --titlelimit 4 -SELECT a AS 'abcd', b FROM t2 WHERE c=3; -.check <<END -abcd: The quick fox jumps over the lazy brown dog - b: 2 -END -.testcase 540 -.mode line --titlelimit 3 -SELECT a AS 'abcd', b FROM t2 WHERE c=3; -.check <<END -...: The quick fox jumps over the lazy brown dog - b: 2 -END - -# https://sqlite.org/forum/forumpost/2025-12-31T19:14:24z -# -# For legacy compatibility, ".header" settings are not changed -# by ".mode" unless the --title or --reset option is used on .mode. -# -.testcase 600 -DROP TABLE IF EXISTS t1; -CREATE TABLE t1(a,b,c); -INSERT INTO t1 VALUES(1,2,3); -.header on -.mode csv -SELECT * FROM t1; -.check --glob a,b,c* - -.testcase 610 -.mode csv -reset -SELECT * FROM t1; -.check 1,2,3 - -.testcase 620 -.mode tty -.mode csv -.header on -SELECT * FROM t1; -.check --glob a,b,c* - -.testcase 630 -.mode tty -.mode csv --title on -SELECT * FROM t1; -.check --glob a,b,c* -.testcase 631 -.mode tty -.mode csv --title off -SELECT * FROM t1; -.check 1,2,3 - -# Verification of claims about .insert mode in the climode.html -# documentation. -.testcase 700 -CREATE TABLE tbl1(one,two); -INSERT INTO tbl1 VALUES('hello!',10),('goodbye',20); -.mode insert new_table -SELECT * FROM tbl1; -.check <<END -INSERT INTO new_table VALUES('hello!',10); -INSERT INTO new_table VALUES('goodbye',20); -END -.testcase 710 -.mode insert new_table --titles on -SELECT * FROM tbl1; -.check <<END -INSERT INTO new_table(one,two) VALUES('hello!',10); -INSERT INTO new_table(one,two) VALUES('goodbye',20); -END -.testcase 720 -.mode insert new_table --titles off -SELECT * FROM tbl1; -.check <<END -INSERT INTO new_table VALUES('hello!',10); -INSERT INTO new_table VALUES('goodbye',20); -END - -# QRF reports an error if the string is too big. -# -.testcase 800 -.mode box -.limit length 1000 -WITH c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<100) -SELECT hex(randomblob(100)) c; -.check -glob "*: string or blob too big" -.limit length 10000000 diff --git a/test/mutex1.test b/test/mutex1.test index de291f4c9..cb189a7a8 100644 --- a/test/mutex1.test +++ b/test/mutex1.test @@ -115,10 +115,6 @@ ifcapable threadsafe1&&shared_cache { } } { - ifcapable thread_misuse_warnings { - if {$mode ne "serialized"} continue - } - # For journal_mode=memory, the static_prng mutex is not required. This # is because the header of an in-memory journal does not contain # any random bytes, and so no call to sqlite3_randomness() is made. @@ -181,18 +177,16 @@ ifcapable threadsafe1&&shared_cache { # Open and use a connection in "nomutex" mode. Test that no recursive # mutexes are obtained. - ifcapable !thread_misuse_warnings { - do_test mutex1.3.1 { - catch {db close} - clear_mutex_counters - sqlite3 db test.db -nomutex 1 - execsql { SELECT * FROM abc } - } {1 2 3 1 2 3 1 2 3} - do_test mutex1.3.2 { - mutex_counters counters - set counters(recursive) - } {0} - } + do_test mutex1.3.1 { + catch {db close} + clear_mutex_counters + sqlite3 db test.db -nomutex 1 + execsql { SELECT * FROM abc } + } {1 2 3 1 2 3 1 2 3} + do_test mutex1.3.2 { + mutex_counters counters + set counters(recursive) + } {0} } # Test the sqlite3_db_mutex() function. diff --git a/test/notnull2.test b/test/notnull2.test index f49a13b56..67d7c26a8 100644 --- a/test/notnull2.test +++ b/test/notnull2.test @@ -66,7 +66,7 @@ do_vmstep_test 1.5.2 { SELECT count(*) FROM t2 WHERE EXISTS( SELECT 1 FROM t1 WHERE t1.a=450 AND t2.c IS NULL ) -} 5000 {0} +} 4000 {0} #------------------------------------------------------------------------- reset_db diff --git a/test/offset1.test b/test/offset1.test index fb68f0d02..5b04bd836 100644 --- a/test/offset1.test +++ b/test/offset1.test @@ -190,20 +190,6 @@ do_execsql_test offset1-2.0 { ORDER BY salary asc); } {} do_execsql_test offset1-2.1 { - SELECT * FROM v ORDER BY +id; -} { - 11 Diane London hr 70 - 12 Bob London hr 78 - 21 Emma London it 84 - 22 Grace Berlin it 90 - 23 Henry London it 104 - 24 Irene Berlin it 104 - 25 Frank Berlin it 120 - 31 Cindy Berlin sales 96 - 32 Dave London sales 96 - 33 Alice Berlin sales 100 -} -do_execsql_test offset1-2.2 { SELECT * FROM v LIMIT 5 OFFSET 2; } { 22 Grace Berlin it 90 @@ -212,19 +198,5 @@ do_execsql_test offset1-2.2 { 11 Diane London hr 70 33 Alice Berlin sales 100 } -do_execsql_test offset1-2.3 { - SELECT * FROM v LIMIT 3 OFFSET 6; -} { - 33 Alice Berlin sales 100 - 23 Henry London it 104 - 24 Irene Berlin it 104 -} -do_execsql_test offset1-2.4 { - SELECT * FROM v LIMIT 3 OFFSET 1; -} { - 32 Dave London sales 96 - 22 Grace Berlin it 90 - 21 Emma London it 84 -} finish_test diff --git a/test/qrf01.test b/test/qrf01.test deleted file mode 100644 index 3ae027957..000000000 --- a/test/qrf01.test +++ /dev/null @@ -1,1167 +0,0 @@ -# 2025-11-05 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# Test cases for the Query Result Formatter (QRF) -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl -set testprefix qrf01 - -do_execsql_test 1.0 { - CREATE TABLE t1(a, b, c); - INSERT INTO t1 VALUES(1,2.5,'three'),(x'424c4f42',NULL,'Ἀμήν'); -} - -do_test 1.10 { - set result "\n[db format {SELECT * FROM t1}]" -} { -╭──────┬─────┬───────╮ -│ a │ b │ c │ -╞══════╪═════╪═══════╡ -│ 1 │ 2.5 │ three │ -│ BLOB │ │ Ἀμήν │ -╰──────┴─────┴───────╯ -} -do_test 1.11a { - set result "\n[db format -title off {SELECT * FROM t1}]" -} { -╭──────┬─────┬───────╮ -│ 1 │ 2.5 │ three │ -│ BLOB │ │ Ἀμήν │ -╰──────┴─────┴───────╯ -} -do_test 1.11b { - set result "\n[db format -text sql {SELECT * FROM t1}]" -} { -╭─────────────┬─────┬─────────╮ -│ a │ b │ c │ -╞═════════════╪═════╪═════════╡ -│ 1 │ 2.5 │ 'three' │ -│ x'424c4f42' │ │ 'Ἀμήν' │ -╰─────────────┴─────┴─────────╯ -} -do_test 1.11c { - set result "\n[db format -text sql -border off {SELECT * FROM t1}]" -} { - a │ b │ c -═════════════╪═════╪═════════ - 1 │ 2.5 │ 'three' - x'424c4f42' │ │ 'Ἀμήν' -} -do_test 1.11d { - set result "\n[db format -text relaxed -blob sql -border off \ - {SELECT * FROM t1}]" -} { - a │ b │ c -═════════════╪═════╪═══════ - 1 │ 2.5 │ three - x'424c4f42' │ │ Ἀμήν -} -do_test 1.12 { - set result "\n[db format -text csv {SELECT * FROM t1}]" -} { -╭────────────────────┬─────┬────────╮ -│ a │ b │ c │ -╞════════════════════╪═════╪════════╡ -│ 1 │ 2.5 │ three │ -│ "\102\114\117\102" │ │ "Ἀμήν" │ -╰────────────────────┴─────┴────────╯ -} -do_test 1.13 { - set result "\n[db format -text csv -blob hex {SELECT * FROM t1}]" -} { -╭──────────┬─────┬────────╮ -│ a │ b │ c │ -╞══════════╪═════╪════════╡ -│ 1 │ 2.5 │ three │ -│ 424c4f42 │ │ "Ἀμήν" │ -╰──────────┴─────┴────────╯ -} -do_test 1.14 { - catch {db format -text unk -blob hex {SELECT * FROM t1}} res - set res -} {bad -text "unk": must be auto, csv, html, json, plain, relaxed, sql, or tcl} -do_test 1.15 { - catch {db format -text sql -blob unk {SELECT * FROM t1}} res - set res -} {bad BLOB encoding (-blob) "unk": must be auto, hex, json, tcl, text, sql, or size} -do_test 1.16 { - catch {db format -text sql -style unk {SELECT * FROM t1}} res - set res -} {bad format style (-style) "unk": must be auto, box, column, count, csv, eqp, explain, html, insert, jobject, json, line, list, markdown, quote, stats, stats-est, stats-vm, or table} - - -do_test 1.20 { - set result "\n[db format -style box {SELECT * FROM t1}]" -} { -╭──────┬─────┬───────╮ -│ a │ b │ c │ -╞══════╪═════╪═══════╡ -│ 1 │ 2.5 │ three │ -│ BLOB │ │ Ἀμήν │ -╰──────┴─────┴───────╯ -} - -do_test 1.30 { - set result "\n[db format -style table {SELECT * FROM t1}]" -} { -+------+-----+-------+ -| a | b | c | -+------+-----+-------+ -| 1 | 2.5 | three | -| BLOB | | Ἀμήν | -+------+-----+-------+ -} -do_test 1.31 { - set result "\n[db format -style table -title off {SELECT * FROM t1}]" -} { -+------+-----+-------+ -| 1 | 2.5 | three | -| BLOB | | Ἀμήν | -+------+-----+-------+ -} -do_test 1.32 { - set result "\n[db format -style table -border off {SELECT * FROM t1}]" -} { - a | b | c -------+-----+------- - 1 | 2.5 | three - BLOB | | Ἀμήν -} -do_test 1.33 { - set result "\n[db format -style table -border off \ - -screenwidth 15 \ - {SELECT * FROM t1}]" -} { - a | b | c -----+---+----- - 1|2.5|three -BLOB| |Ἀμήν -} -do_test 1.34 { - set result "\n[db format -style box -border off \ - -screenwidth 30 \ - {SELECT * FROM t1}]" -} { - a │ b │ c -══════╪═════╪═══════ - 1 │ 2.5 │ three - BLOB │ │ Ἀμήν -} -do_test 1.35 { - set result "\n[db format -style box -border off \ - -screenwidth 15 \ - {SELECT * FROM t1}]" -} { - a │ b │ c -════╪═══╪═════ - 1│2.5│three -BLOB│ │Ἀμήν -} - -do_test 1.40 { - set result "\n[db format -style column {SELECT * FROM t1}]" -} { -a b c ----- --- ----- -1 2.5 three -BLOB Ἀμήν -} -do_test 1.41 { - set result "\n[db format -style column -title off {SELECT * FROM t1}]" -} { -1 2.5 three -BLOB Ἀμήν -} - -do_test 1.50 { - db format -style count {SELECT * FROM t1} -} 2 - -do_test 1.60a { - db format -style list -columnsep , -rowsep \r\n -text csv -blob tcl {SELECT * FROM t1} -} "1,2.5,three\r\n\"\\102\\114\\117\\102\",,\"Ἀμήν\"\r\n" -do_test 1.60b { - db format -style csv -columnsep xyz -rowsep pqr -text sql -blob sql {SELECT * FROM t1} -} "1,2.5,three\r\nx'424c4f42',,\"Ἀμήν\"\r\n" -do_test 1.61a { - db format -style list -columnsep , -rowsep \r\n -text csv -title auto -blob tcl {SELECT * FROM t1} -} "a,b,c\r\n1,2.5,three\r\n\"\\102\\114\\117\\102\",,\"Ἀμήν\"\r\n" -do_test 1.61b { - db format -style csv -title auto -blob tcl {SELECT * FROM t1} -} "a,b,c\r\n1,2.5,three\r\n\"\\102\\114\\117\\102\",,\"Ἀμήν\"\r\n" -do_test 1.62a { - db format -style list -columnsep , -rowsep \r\n -text csv -title csv -blob tcl {SELECT a AS 'a x y', b, c FROM t1} -} "\"a x y\",b,c\r\n1,2.5,three\r\n\"\\102\\114\\117\\102\",,\"Ἀμήν\"\r\n" -do_test 1.62b { - db format -style csv -title csv -blob tcl {SELECT a AS 'a x y', b, c FROM t1} -} "\"a x y\",b,c\r\n1,2.5,three\r\n\"\\102\\114\\117\\102\",,\"Ἀμήν\"\r\n" - -do_test 1.70 { - set result "\n[db format -style html {SELECT * FROM t1}]" -} { -<TR> -<TD>1 -<TD>2.5 -<TD>three -</TR> -<TR> -<TD>BLOB -<TD>null -<TD>Ἀμήν -</TR> -} -do_test 1.71 { - set result "\n[db format -style html -title auto {SELECT * FROM t1}]" -} { -<TR> -<TH>a -<TH>b -<TH>c -</TR> -<TR> -<TD>1 -<TD>2.5 -<TD>three -</TR> -<TR> -<TD>BLOB -<TD>null -<TD>Ἀμήν -</TR> -} - -do_test 1.80 { - set result "\n[db format -style insert {SELECT * FROM t1}]" -} { -INSERT INTO tab VALUES(1,2.5,'three'); -INSERT INTO tab VALUES(x'424c4f42',NULL,'Ἀμήν'); -} -do_test 1.81 { - set result "\n[db format -style insert -tablename t1 {SELECT * FROM t1}]" -} { -INSERT INTO t1 VALUES(1,2.5,'three'); -INSERT INTO t1 VALUES(x'424c4f42',NULL,'Ἀμήν'); -} -do_test 1.82 { - set result "\n[db format -style insert -tablename t1 -title auto \ - {SELECT * FROM t1}]" -} { -INSERT INTO t1(a,b,c) VALUES(1,2.5,'three'); -INSERT INTO t1(a,b,c) VALUES(x'424c4f42',NULL,'Ἀμήν'); -} -do_test 1.83 { - set result "\n[db format -style insert -tablename drop -title on \ - {SELECT a AS "a-b", b, c AS "123" FROM t1}]" -} { -INSERT INTO "drop"("a-b",b,"123") VALUES(1,2.5,'three'); -INSERT INTO "drop"("a-b",b,"123") VALUES(x'424c4f42',NULL,'Ἀμήν'); -} - -do_test 1.90 { - set result "\n[db format -style json {SELECT * FROM t1}]" -} { -[{"a":1,"b":2.5,"c":"three"}, -{"a":"\u0042\u004c\u004f\u0042","b":null,"c":"Ἀμήν"}] -} -do_test 1.91 { - set result "\n[db format -style jobject {SELECT * FROM t1}]" -} { -{"a":1,"b":2.5,"c":"three"} -{"a":"\u0042\u004c\u004f\u0042","b":null,"c":"Ἀμήν"} -} -do_test 1.92 { - set result "\n[db format -style jobject {SELECT *, unistr('abc\u000a123\u000d\u000axyz') AS xyz FROM t1}]" -} { -{"a":1,"b":2.5,"c":"three","xyz":"abc\n123\r\nxyz"} -{"a":"\u0042\u004c\u004f\u0042","b":null,"c":"Ἀμήν","xyz":"abc\n123\r\nxyz"} -} - -do_test 1.100 { - set result "\n[db format -style line {SELECT * FROM t1}]" -} { -a: 1 -b: 2.5 -c: three - -a: BLOB -b: -c: Ἀμήν -} -do_test 1.101 { - set result "\n[db format -style line -null (NULL) {SELECT * FROM t1}]" -} { -a: 1 -b: 2.5 -c: three - -a: BLOB -b: (NULL) -c: Ἀμήν -} -do_test 1.102 { - set result "\n[db format -style line -null (NULL) -columnsep { = } \ - -text sql {SELECT * FROM t1}]" -} { -a = 1 -b = 2.5 -c = 'three' - -a = x'424c4f42' -b = (NULL) -c = 'Ἀμήν' -} - -do_test 1.110 { - set result "\n[db format -style list {SELECT * FROM t1}]" -} { -1|2.5|three -BLOB||Ἀμήν -} -do_test 1.111 { - set result "\n[db format -style list -title on {SELECT * FROM t1}]" -} { -a|b|c -1|2.5|three -BLOB||Ἀμήν -} -do_test 1.112 { - set result "\n[db format -style list -title on -text sql -null NULL \ - -title plain {SELECT * FROM t1}]" -} { -a|b|c -1|2.5|'three' -x'424c4f42'|NULL|'Ἀμήν' -} -do_test 1.118 { - set rc [catch {db format -style list -title unk {SELECT * FROM t1}} res] - lappend rc $res -} {1 {bad -title "unk": must be off, on, auto, csv, html, json, plain, relaxed, sql, or tcl}} - - -do_test 1.120 { - set result "\n[db format -style markdown {SELECT * FROM t1}]" -} { -| a | b | c | -|------|-----|-------| -| 1 | 2.5 | three | -| BLOB | | Ἀμήν | -} -do_test 1.121 { - set result "\n[db format -style markdown -title off {SELECT * FROM t1}]" -} { -| 1 | 2.5 | three | -| BLOB | | Ἀμήν | -} - -do_test 1.130 { - set result "\n[db format -style quote {SELECT * FROM t1}]" -} { -1,2.5,'three' -x'424c4f42',NULL,'Ἀμήν' -} -do_test 1.131 { - set result "\n[db format -style quote -title on {SELECT * FROM t1}]" -} { -'a','b','c' -1,2.5,'three' -x'424c4f42',NULL,'Ἀμήν' -} - - -do_execsql_test 2.0 { - DELETE FROM t1; - INSERT INTO t1 VALUES(1,2,'The quick fox jumps over the lazy brown dog.'); -} -do_test 2.1 { - set result "\n[db format -widths {5 -5 19} -wordwrap on \ - {SELECT * FROM t1}]" -} { -╭───────┬───────┬─────────────────────╮ -│ a │ b │ c │ -╞═══════╪═══════╪═════════════════════╡ -│ 1 │ 2 │ The quick fox jumps │ -│ │ │ over the lazy brown │ -│ │ │ dog. │ -╰───────┴───────┴─────────────────────╯ -} -do_test 2.2 { - set result "\n[db format -widths {5 -5 19} -wordwrap off \ - {SELECT * FROM t1}]" -} { -╭───────┬───────┬─────────────────────╮ -│ a │ b │ c │ -╞═══════╪═══════╪═════════════════════╡ -│ 1 │ 2 │ The quick fox jumps │ -│ │ │ over the lazy brown │ -│ │ │ dog. │ -╰───────┴───────┴─────────────────────╯ -} -do_test 2.3 { - set result "\n[db format -widths {5 -5 18} -wordwrap on \ - {SELECT * FROM t1}]" -} { -╭───────┬───────┬────────────────────╮ -│ a │ b │ c │ -╞═══════╪═══════╪════════════════════╡ -│ 1 │ 2 │ The quick fox │ -│ │ │ jumps over the │ -│ │ │ lazy brown dog. │ -╰───────┴───────┴────────────────────╯ -} -do_test 2.4 { - set result "\n[db format -widths {5 -5 -18} -wordwrap on \ - {SELECT * FROM t1}]" -} { -╭───────┬───────┬────────────────────╮ -│ a │ b │ c │ -╞═══════╪═══════╪════════════════════╡ -│ 1 │ 2 │ The quick fox │ -│ │ │ jumps over the │ -│ │ │ lazy brown dog. │ -╰───────┴───────┴────────────────────╯ -} -do_test 2.5 { - set result "\n[db format -widths {5 -5 19} -wordwrap off \ - {SELECT * FROM t1}]" -} { -╭───────┬───────┬─────────────────────╮ -│ a │ b │ c │ -╞═══════╪═══════╪═════════════════════╡ -│ 1 │ 2 │ The quick fox jumps │ -│ │ │ over the lazy brown │ -│ │ │ dog. │ -╰───────┴───────┴─────────────────────╯ -} -do_test 2.6 { - set result "\n[db format -widths {5 -5 18} -wordwrap off \ - {SELECT * FROM t1}]" -} { -╭───────┬───────┬────────────────────╮ -│ a │ b │ c │ -╞═══════╪═══════╪════════════════════╡ -│ 1 │ 2 │ The quick fox jump │ -│ │ │ s over the lazy br │ -│ │ │ own dog. │ -╰───────┴───────┴────────────────────╯ -} -do_test 2.7 { - set result "\n[db format -widths {5 5 18} -wordwrap yes \ - -align {left center right} -titlealign right \ - {SELECT * FROM t1}]" -} { -╭───────┬───────┬────────────────────╮ -│ a │ b │ c │ -╞═══════╪═══════╪════════════════════╡ -│ 1 │ 2 │ The quick fox │ -│ │ │ jumps over the │ -│ │ │ lazy brown dog. │ -╰───────┴───────┴────────────────────╯ -} -do_test 2.8 { - set result "\n[db format -widths {5 8 11} -wordwrap yes \ - -align {auto auto center} -titlealign left \ - -defaultalign right \ - {SELECT * FROM t1}]" -} { -╭───────┬──────────┬─────────────╮ -│ a │ b │ c │ -╞═══════╪══════════╪═════════════╡ -│ 1 │ 2 │ The quick │ -│ │ │ fox jumps │ -│ │ │ over the │ -│ │ │ lazy brown │ -│ │ │ dog. │ -╰───────┴──────────┴─────────────╯ -} -do_test 2.9 { - catch {db format -align {auto xyz 123} {SELECT * FROM t1}} res - set res -} {bad column alignment (-align) "xyz": must be auto, bottom, c, center, e, left, middle, n, ne, nw, right, s, se, sw, top, or w} -do_test 2.10 { - catch {db format -defaultalign xyz {SELECT * FROM t1}} res - set res -} {bad default alignment (-defaultalign) "xyz": must be auto, bottom, c, center, e, left, middle, n, ne, nw, right, s, se, sw, top, or w} -do_test 2.11 { - catch {db format -titlealign xyz {SELECT * FROM t1}} res - set res -} {bad title alignment (-titlealign) "xyz": must be auto, bottom, c, center, e, left, middle, n, ne, nw, right, s, se, sw, top, or w} - - -do_execsql_test 2.30 { - UPDATE t1 SET c='Η γρήγορη αλεπού πηδάει πάνω από το τεμπέλικο καφέ σκυλί'; - SELECT hex(c) FROM t1; -} {CE9720CEB3CF81CEAECEB3CEBFCF81CEB720CEB1CEBBCEB5CF80CEBFCF8D20CF80CEB7CEB4CEACCEB5CEB920CF80CEACCEBDCF8920CEB1CF80CF8C20CF84CEBF20CF84CEB5CEBCCF80CEADCEBBCEB9CEBACEBF20CEBACEB1CF86CEAD20CF83CEBACF85CEBBCEAF} -do_test 2.31 { - set result "\n[db format -widths {5 -5 18} -wordwrap on \ - {SELECT * FROM t1}]" -} { -╭───────┬───────┬────────────────────╮ -│ a │ b │ c │ -╞═══════╪═══════╪════════════════════╡ -│ 1 │ 2 │ Η γρήγορη αλεπού │ -│ │ │ πηδάει πάνω από το │ -│ │ │ τεμπέλικο καφέ │ -│ │ │ σκυλί │ -╰───────┴───────┴────────────────────╯ -} -do_test 2.32 { - set result "\n[db format -widths {5 5 18} -align {left center center} -wordwrap on \ - {SELECT * FROM t1}]" -} { -╭───────┬───────┬────────────────────╮ -│ a │ b │ c │ -╞═══════╪═══════╪════════════════════╡ -│ 1 │ 2 │ Η γρήγορη αλεπού │ -│ │ │ πηδάει πάνω από το │ -│ │ │ τεμπέλικο καφέ │ -│ │ │ σκυλί │ -╰───────┴───────┴────────────────────╯ -} - - -do_execsql_test 3.0 { - DELETE FROM t1; - INSERT INTO t1 VALUES(1,2,unistr('abc\u001b[1;31m123\u001b[0mxyz')); -} -do_test 3.1 { - set result "\n[db format {SELECT * FROM t1}]" -} { -╭───┬───┬────────────────────────╮ -│ a │ b │ c │ -╞═══╪═══╪════════════════════════╡ -│ 1 │ 2 │ abc^[[1;31m123^[[0mxyz │ -╰───┴───┴────────────────────────╯ -} -do_test 3.2 { - set result "\n[db format -esc off {SELECT * FROM t1}]" - string map [list \033 X] $result -} { -╭───┬───┬───────────╮ -│ a │ b │ c │ -╞═══╪═══╪═══════════╡ -│ 1 │ 2 │ abcX[1;31m123X[0mxyz │ -╰───┴───┴───────────╯ -} -do_test 3.3 { - set result "\n[db format -esc symbol {SELECT * FROM t1}]" -} { -╭───┬───┬──────────────────────╮ -│ a │ b │ c │ -╞═══╪═══╪══════════════════════╡ -│ 1 │ 2 │ abc␛[1;31m123␛[0mxyz │ -╰───┴───┴──────────────────────╯ -} -do_test 3.4 { - set result "\n[db format -esc ascii {SELECT * FROM t1}]" -} { -╭───┬───┬────────────────────────╮ -│ a │ b │ c │ -╞═══╪═══╪════════════════════════╡ -│ 1 │ 2 │ abc^[[1;31m123^[[0mxyz │ -╰───┴───┴────────────────────────╯ -} -do_test 3.5 { - catch {db format -esc unk {SELECT * FROM t1}} res - set res -} {bad control character escape (-esc) "unk": must be ascii, auto, off, or symbol} - -do_execsql_test 4.0 { - DELETE FROM t1; - INSERT INTO t1 VALUES(json('{a:5,b:6}'), jsonb('{c:1,d:2}'), 99); -} -do_test 4.1 { - set result "\n[db format -text sql {SELECT * FROM t1}]" -} { -╭─────────────────┬───────────────────────┬────╮ -│ a │ b │ c │ -╞═════════════════╪═══════════════════════╪════╡ -│ '{"a":5,"b":6}' │ x'8c1763133117641332' │ 99 │ -╰─────────────────┴───────────────────────┴────╯ -} -do_test 4.2 { - set result "\n[db format -text sql -textjsonb on {SELECT * FROM t1}]" -} { -╭─────────────────┬────────────────────────┬────╮ -│ a │ b │ c │ -╞═════════════════╪════════════════════════╪════╡ -│ '{"a":5,"b":6}' │ jsonb('{"c":1,"d":2}') │ 99 │ -╰─────────────────┴────────────────────────┴────╯ -} -do_test 4.3 { - set result "\n[db format -text plain -textjsonb on -wrap 11 \ - {SELECT a AS json, b AS jsonb, c AS num FROM t1}]" -} { -╭─────────────┬─────────────┬─────╮ -│ json │ jsonb │ num │ -╞═════════════╪═════════════╪═════╡ -│ {"a":5,"b": │ {"c":1,"d": │ 99 │ -│ 6} │ 2} │ │ -╰─────────────┴─────────────┴─────╯ -} - -do_execsql_test 5.0 { - DROP TABLE t1; - CREATE TABLE t1(name, mtime, value); - INSERT INTO t1 VALUES - ('entry-one',1708791504,zeroblob(300)), - (unistr('one\u000atwo\u000athree'),1333206973,NULL), - ('sample-jsonb',1333101221,jsonb('{ - "alpha":53.11688723, - "beta":"qrfWidthPrint(p, p->pOut, -p->u.sLine.mxColWth);", - "zeta":[15,null,1333206973,"fd8ffe000104a46494600010101"]}')); -} -do_test 5.1 { - set sql {SELECT name, mtime, datetime(mtime,'unixepoch') AS time, - value FROM t1 ORDER BY mtime} - set result "\n[db format -style line -screenwidth 60 -blob sql \ - -text sql -wordwrap off -linelimit 77 \ - -columnsep { = } $sql]" -} { - name = 'sample-jsonb' -mtime = 1333101221 - time = '2012-03-30 09:53:41' -value = x'cc7c57616c706861b535332e31313638383732334762657461 - c73071726657696474685072696e7428702c20702d3e704f7574 - 2c202d702d3e752e734c696e652e6d78436f6c577468293b477a - 657461cb2c23313500a331333333323036393733c71b66643866 - 6665303030313034613436343934363030303130313031' - - name = unistr('one\u000atwo\u000athree') -mtime = 1333206973 - time = '2012-03-31 15:16:13' -value = - - name = 'entry-one' -mtime = 1708791504 - time = '2024-02-24 16:18:24' -value = x'00000000000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000000000 - 000000000000000000000000000000' -} -do_test 5.2a { - set sql {SELECT name, mtime, datetime(mtime,'unixepoch') AS time, - value FROM t1 ORDER BY mtime} - set result "\n[db format -style line -screenwidth 60 -blob sql \ - -text plain -esc off -textjsonb yes -columnsep { = }\ - -wordwrap yes -linelimit 3 $sql]" -} { - name = sample-jsonb -mtime = 1333101221 - time = 2012-03-30 09:53:41 -value = {"alpha":53.11688723,"beta":"qrfWidthPrint(p, - p->pOut, -p->u.sLine.mxColWth);","zeta":[15,null, - 1333206973,"fd8ffe000104a46494600010101"]} - - name = one - two - three -mtime = 1333206973 - time = 2012-03-31 15:16:13 -value = - - name = entry-one -mtime = 1708791504 - time = 2024-02-24 16:18:24 -value = x'00000000000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000000000 - ... -} -set sqlnolabel "SELECT name, mtime, datetime(mtime,'unixepoch'),\ - value FROM t1 ORDER BY mtime" -do_test 5.2b { - set result "\n[db format -style line -screenwidth 60 -blob sql \ - -text plain -esc off -textjsonb no -titlelimit 12 \ - -wordwrap yes -linelimit 3 $sqlnolabel ]" -} { - name: sample-jsonb - mtime: 1333101221 -datetime(...: 2012-03-30 09:53:41 - value: x'cc7c57616c706861b535332e31313638383732334762 - 657461c73071726657696474685072696e7428702c2070 - 2d3e704f75742c202d702d3e752e734c696e652e6d7843 - ... - - name: one - two - three - mtime: 1333206973 -datetime(...: 2012-03-31 15:16:13 - value: - - name: entry-one - mtime: 1708791504 -datetime(...: 2024-02-24 16:18:24 - value: x'00000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000 - ... -} -set sql "SELECT name, mtime, datetime(mtime,'unixepoch') AS time,\ - value FROM t1 ORDER BY mtime" -do_test 5.3a { - set result "\n[db format -style box -widths {0 10 10 14}\ - -align {left right right center} \ - -blob sql \ - -text plain -esc off -textjsonb no \ - -wordwrap yes -linelimit 2 $sql]" -} { -╭──────────────┬────────────┬────────────┬────────────────╮ -│ name │ mtime │ time │ value │ -╞══════════════╪════════════╪════════════╪════════════════╡ -│ sample-jsonb │ 1333101221 │ 2012-03-30 │ x'cc7c57616c70 │ -│ │ │ 09:53:41 │ 6861b535332e31 │ -│ │ │ │ ... │ -├──────────────┼────────────┼────────────┼────────────────┤ -│ one │ 1333206973 │ 2012-03-31 │ │ -│ two │ │ 15:16:13 │ │ -│ ... │ │ │ │ -├──────────────┼────────────┼────────────┼────────────────┤ -│ entry-one │ 1708791504 │ 2024-02-24 │ x'000000000000 │ -│ │ │ 16:18:24 │ 00000000000000 │ -│ │ │ │ ... │ -╰──────────────┴────────────┴────────────┴────────────────╯ -} -do_test 5.3b { - set result "\n[db format -style box -widths {0 10 0 14} \ - -align {left right right center} \ - -blob sql -titlelimit 12 \ - -text plain -esc off -textjsonb no \ - -wordwrap yes -linelimit 2 $sqlnolabel ]" -} { -╭──────────────┬────────────┬─────────────────────┬────────────────╮ -│ name │ mtime │ datetime(... │ value │ -╞══════════════╪════════════╪═════════════════════╪════════════════╡ -│ sample-jsonb │ 1333101221 │ 2012-03-30 09:53:41 │ x'cc7c57616c70 │ -│ │ │ │ 6861b535332e31 │ -│ │ │ │ ... │ -├──────────────┼────────────┼─────────────────────┼────────────────┤ -│ one │ 1333206973 │ 2012-03-31 15:16:13 │ │ -│ two │ │ │ │ -│ ... │ │ │ │ -├──────────────┼────────────┼─────────────────────┼────────────────┤ -│ entry-one │ 1708791504 │ 2024-02-24 16:18:24 │ x'000000000000 │ -│ │ │ │ 00000000000000 │ -│ │ │ │ ... │ -╰──────────────┴────────────┴─────────────────────┴────────────────╯ -} -do_test 5.3c { - set result "\n[db format -style table -widths {0 10 10 14}\ - -align {center right right right} \ - -blob sql \ - -text plain -esc off -textjsonb no \ - -wordwrap yes -linelimit 2 $sql]" -} { -+--------------+------------+------------+----------------+ -| name | mtime | time | value | -+--------------+------------+------------+----------------+ -| sample-jsonb | 1333101221 | 2012-03-30 | x'cc7c57616c70 | -| | | 09:53:41 | 6861b535332e31 | -| | | | ... | -+--------------+------------+------------+----------------+ -| one | 1333206973 | 2012-03-31 | | -| two | | 15:16:13 | | -| ... | | | | -+--------------+------------+------------+----------------+ -| entry-one | 1708791504 | 2024-02-24 | x'000000000000 | -| | | 16:18:24 | 00000000000000 | -| | | | ... | -+--------------+------------+------------+----------------+ -} -do_test 5.3c { - set result "\n[db format -style column -widths {0 10 10 14}\ - -align {center right right right} \ - -blob sql \ - -text plain -esc off -textjsonb no \ - -wordwrap yes -linelimit 2 $sql]" -} { - name mtime time value ------------- ---------- ---------- -------------- -sample-jsonb 1333101221 2012-03-30 x'cc7c57616c70 - 09:53:41 6861b535332e31 - ... - - one 1333206973 2012-03-31 - two 15:16:13 - ... - - entry-one 1708791504 2024-02-24 x'000000000000 - 16:18:24 00000000000000 - ... -} -do_test 5.4 { - db eval { - CREATE TABLE t2(a,b,c,d,e); - WITH v(x) AS (SELECT 'abcdefghijklmnopqrstuvwxyz') - INSERT INTO t2 SELECT x,x,x,x,x FROM v; - } - set sql {SELECT char(0x61,0xa,0x62,0xa,0x63,0xa,0x64) a, - mtime b, mtime c, mtime d, mtime e FROM t1} - set result "\n[db format -style box -widths {1 2 3 4 5}\ - -linelimit 3 -wordwrap off {SELECT *, 'x' AS x FROM t2}]" -} { -╭────┬────┬─────┬──────┬───────┬───╮ -│ a │ b │ c │ d │ e │ x │ -╞════╪════╪═════╪══════╪═══════╪═══╡ -│ ab │ ab │ abc │ abcd │ abcde │ x │ -│ cd │ cd │ def │ efgh │ fghij │ │ -│ ef │ ef │ ghi │ ijkl │ klmno │ │ -│ .. │ .. │ ... │ ... │ ... │ │ -╰────┴────┴─────┴──────┴───────┴───╯ -} - -do_execsql_test 6.0 { - DELETE FROM t2; - INSERT INTO t2 VALUES - (1, 2.5, 'three', x'342028666f757229', null); -} -do_test 6.1a { - set result "\n[db format -style list -null NULL \ - -text tcl -columnsep , \ - {SELECT * FROM t2}]" -} { -1,2.5,"three","\064\040\050\146\157\165\162\051",NULL -} - -do_execsql_test 7.0 { - CREATE TABLE t7(a,b); - INSERT INTO t7 VALUES('abcdefghijklmnop', - 'abcぁdefかghiのjklはmnop'); -} -do_test 7.1 { - set result "\n[db format -style list -charlimit 13 \ - {SELECT * FROM t7}]" -} { -abcdefghijklm...|abcぁdefかghi... -} -do_test 7.2 { - set result "\n[db format -style list -charlimit 14 \ - {SELECT * FROM t7}]" -} { -abcdefghijklmn...|abcぁdefかghi... -} -do_test 7.3 { - set result "\n[db format -style list -charlimit 15 \ - {SELECT * FROM t7}]" -} { -abcdefghijklmno...|abcぁdefかghiの... -} -do_test 7.4 { - set result "\n[db format -style list -charlimit 16 \ - {SELECT * FROM t7}]" -} { -abcdefghijklmnop|abcぁdefかghiのj... -} - -do_test 8.0 { - set result "\n[db format -style table { - WITH c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<10) - SELECT 'aaa' AS 'a', format('%.*c',n,'b')||char(9)||'xx' AS x FROM c}]" -} { -+-----+--------------------+ -| a | x | -+-----+--------------------+ -| aaa | b xx | -| aaa | bb xx | -| aaa | bbb xx | -| aaa | bbbb xx | -| aaa | bbbbb xx | -| aaa | bbbbbb xx | -| aaa | bbbbbbb xx | -| aaa | bbbbbbbb xx | -| aaa | bbbbbbbbb xx | -| aaa | bbbbbbbbbb xx | -+-----+--------------------+ -} -do_test 8.1 { - set result "\n[db format -style table { - WITH c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<10) - SELECT 'aaaa' AS 'a', format('%.*c',n,'b')||char(9)||'xx' AS x FROM c}]" -} { -+------+--------------------+ -| a | x | -+------+--------------------+ -| aaaa | b xx | -| aaaa | bb xx | -| aaaa | bbb xx | -| aaaa | bbbb xx | -| aaaa | bbbbb xx | -| aaaa | bbbbbb xx | -| aaaa | bbbbbbb xx | -| aaaa | bbbbbbbb xx | -| aaaa | bbbbbbbbb xx | -| aaaa | bbbbbbbbbb xx | -+------+--------------------+ -} -do_test 8.3 { - set result "\n[db format -style table -esc off { - WITH c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<15) - SELECT 'aaa' AS 'a',n, 'xx'||char(n)||'yy' AS xy FROM c - WHERE n NOT IN (8,10,13,14)}]" -} { -+-----+----+------------+ -| a | n | xy | -+-----+----+------------+ -| aaa | 1 | xx␁yy | -| aaa | 2 | xx␂yy | -| aaa | 3 | xx␃yy | -| aaa | 4 | xx␄yy | -| aaa | 5 | xx␅yy | -| aaa | 6 | xx␆yy | -| aaa | 7 | xx␇yy | -| aaa | 9 | xx yy | -| aaa | 11 | xx␋yy | -| aaa | 12 | xx␌yy | -| aaa | 15 | xx␏yy | -+-----+----+------------+ -} -do_test 8.4 { - set result "\n[db format -style table -esc off { - WITH c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<15) - SELECT 'aaa' AS 'a',n, 'xx'||char(n)||'yy'||char(9)||'zz' AS xyz FROM c - WHERE n NOT IN (8,10,13,14)}]" -} { -+-----+----+--------------------+ -| a | n | xyz | -+-----+----+--------------------+ -| aaa | 1 | xx␁yy zz | -| aaa | 2 | xx␂yy zz | -| aaa | 3 | xx␃yy zz | -| aaa | 4 | xx␄yy zz | -| aaa | 5 | xx␅yy zz | -| aaa | 6 | xx␆yy zz | -| aaa | 7 | xx␇yy zz | -| aaa | 9 | xx yy zz | -| aaa | 11 | xx␋yy zz | -| aaa | 12 | xx␌yy zz | -| aaa | 15 | xx␏yy zz | -+-----+----+--------------------+ -} - -do_test 9.1 { - db eval { - CREATE TABLE t9(x); - INSERT INTO t9 VALUES - (x'4331323334'), - (x'c30431323334'), - (x'd3000431323334'), - (x'e30000000431323334'), - (x'f3000000000000000431323334'); - } - db format -style list -text plain -rowsep , -textjsonb on \ - {SELECT * FROM t9} -} {1234,1234,1234,1234,1234,} -do_test 9.2 { - db format -style list -text sql -rowsep , -textjsonb on \ - {SELECT * FROM t9} -} {jsonb('1234'),jsonb('1234'),jsonb('1234'),jsonb('1234'),jsonb('1234'),} -do_test 9.3 { - db format -style json {SELECT * FROM t9 WHERE rowid<0} -} {} -do_test 9.4 { - db format -style jobject {SELECT * FROM t9 WHERE rowid<0} -} {} - -do_test 10.1 { - db eval { - DROP TABLE IF EXISTS t1; - CREATE TABLE t1(x); - INSERT INTO t1(x) VALUES - ('alice'), - ('bob'), - ('cinderella-cinderella'), - ('daniel'), - ('emma'), - ('fred'), - ('gertrude'), - ('harold'), - ('ingrid'), - ('jake'), - ('lisa'), - ('mike'), - ('nina'), - ('octavian'), - ('paula'), - ('quintus'), - ('rita'), - ('sam'), - ('tammy'), - ('ulysses'), - ('violet'), - ('william'), - ('xanthippe'), - ('yates'), - ('zoe'); - } - set result "\n[db format -style column -title off -screenwidth 41 -splitcolumn on \ - {SELECT x FROM t1}]" -} { -alice octavian -bob paula -cinderella-cinderella quintus -daniel rita -emma sam -fred tammy -gertrude ulysses -harold violet -ingrid william -jake xanthippe -lisa yates -mike zoe -nina -} -do_test 10.2 { - set result "\n[db format -style column -title off -screenwidth 42 -splitcolumn on \ - {SELECT x FROM t1}]" -} { -alice jake tammy -bob lisa ulysses -cinderella-cinderella mike violet -daniel nina william -emma octavian xanthippe -fred paula yates -gertrude quintus zoe -harold rita -ingrid sam -} -do_test 10.3 { - set result "\n[db format -style column -title off -screenwidth 51 -splitcolumn on \ - {SELECT x FROM t1}]" -} { -alice harold paula william -bob ingrid quintus xanthippe -cinderella-cinderella jake rita yates -daniel lisa sam zoe -emma mike tammy -fred nina ulysses -gertrude octavian violet -} -do_test 10.4 { - set result "\n[db format -style column -title off -screenwidth 61 -splitcolumn on \ - {SELECT x FROM t1}]" -} { -alice fred lisa quintus violet -bob gertrude mike rita william -cinderella-cinderella harold nina sam xanthippe -daniel ingrid octavian tammy yates -emma jake paula ulysses zoe -} -do_test 10.5 { - set result "\n[db format -style column -title off -screenwidth 74 -splitcolumn on \ - {SELECT x FROM t1}]" -} { -alice emma ingrid nina rita violet zoe -bob fred jake octavian sam william -cinderella-cinderella gertrude lisa paula tammy xanthippe -daniel harold mike quintus ulysses yates -} - -do_test 11.1 { - set result "\n[db format -style table -blob size {SELECT randomblob(1234)}]" -} { -+------------------+ -| randomblob(1234) | -+------------------+ -| (1234-byte blob) | -+------------------+ -} - -do_test 12.1 { - set result "\n[db format -style box -text html \ - {SELECT 'abc','','xyz'}]" -} { -╭───────┬────┬───────╮ -│ 'abc' │ '' │ 'xyz' │ -╞═══════╪════╪═══════╡ -│ abc │ │ xyz │ -╰───────┴────┴───────╯ -} - -# Tests for "relaxed" quoting -# -do_test 13.2 { - db eval { - CREATE TABLE t13(a,b); - INSERT INTO t13(a,b) VALUES - (1,'NULL'), - (0,'-NULL-'), - (0,''), - (1,'''abcde'), - (1,'abcde'''), - (0,'abcde'), - (1,' abcde'), - (1,'abcde '), - (1,'+0'), - (1,'-0'), - (1,'012345'), - (0,'012xyz345'), - (1,'0123.45'), - (0,'12.34.56'), - (0,'12.3e'), - (1,'12.3e+123'), - (1,'12.3e-34'), - (1,'12.3E56'), - (1,'12E56'), - (0,'12.5E5.6'), - (0,'12.5e+'), - (0,'12.5e-'), - (1,'+Inf'),(1,'-Inf'),(1,'Inf'); - } - set result \n[db format -style box -text relaxed -null NULL \ - -align {center left} \ - {SELECT if(a,'yes','') AS 'quoted?', b AS string - FROM t13 ORDER BY rowid}] -} { -╭─────────┬─────────────╮ -│ quoted? │ string │ -╞═════════╪═════════════╡ -│ yes │ 'NULL' │ -│ │ -NULL- │ -│ │ │ -│ yes │ '''abcde' │ -│ yes │ 'abcde''' │ -│ │ abcde │ -│ yes │ ' abcde' │ -│ yes │ 'abcde ' │ -│ yes │ '+0' │ -│ yes │ '-0' │ -│ yes │ '012345' │ -│ │ 012xyz345 │ -│ yes │ '0123.45' │ -│ │ 12.34.56 │ -│ │ 12.3e │ -│ yes │ '12.3e+123' │ -│ yes │ '12.3e-34' │ -│ yes │ '12.3E56' │ -│ yes │ '12E56' │ -│ │ 12.5E5.6 │ -│ │ 12.5e+ │ -│ │ 12.5e- │ -│ yes │ '+Inf' │ -│ yes │ '-Inf' │ -│ yes │ 'Inf' │ -╰─────────┴─────────────╯ -} - -db close - -finish_test diff --git a/test/qrf02.test b/test/qrf02.test deleted file mode 100644 index 07e1568f7..000000000 --- a/test/qrf02.test +++ /dev/null @@ -1,47 +0,0 @@ -# 2025-11-05 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# Test cases for the Query Result Formatter (QRF) -# -# These tests are for EXPLAIN and EXPLAIN QUERY PLAN formatting, the -# output of which can change when enhancments are made to the query -# planner. So expect to have to modify the expected results of these -# test cases in the future. -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl -set testprefix qrf02 - -do_execsql_test 1.0 { - CREATE TABLE t1(a); - INSERT INTO t1 VALUES(1); -} - -set result [db format {EXPLAIN SELECT * FROM t1}] -do_test 1.10 { - set result -} {/*addr opcode p1 p2 p3 p4 p5 comment ----- ------------- ---- ---- ---- ------------- -- ------------- -0 Init */} -regsub -all {\d+} $result {N} result2 -do_test 1.11 { - set result2 -} "/.*\nN Rewind .*\nN Column .*/" - -do_test 1.20 { - set result "\n[db format {EXPLAIN QUERY PLAN SELECT * FROM t1}]" -} { -QUERY PLAN -`--SCAN t1 -} - -finish_test diff --git a/test/qrf03.test b/test/qrf03.test deleted file mode 100644 index c0457df7f..000000000 --- a/test/qrf03.test +++ /dev/null @@ -1,176 +0,0 @@ -# 2025-11-15 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# Test cases for the Query Result Formatter (QRF) -# -# Format narrowing due to nScreenWidth -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl -set testprefix qrf03 - -do_execsql_test 1.0 { - CREATE TABLE mlink( - mid INTEGER, - fid INTEGER, - pmid INTEGER, - pid INTEGER, - fnid INTEGER REFERENCES filename, - pfnid INTEGER, - mperm INTEGER, - isaux BOOLEAN DEFAULT 0 - ); - INSERT INTO mlink VALUES(28775,28774,28773,28706,1,0,0,0); - INSERT INTO mlink VALUES(28773,28706,28770,28685,1,0,0,0); - INSERT INTO mlink VALUES(28770,28736,28769,28695,2,0,0,0); - INSERT INTO mlink VALUES(28770,28697,28769,28698,3,0,0,0); - INSERT INTO mlink VALUES(28767,28768,28759,28746,4,0,0,0); - CREATE TABLE event( - type TEXT, - mtime DATETIME, - objid INTEGER PRIMARY KEY, - tagid INTEGER, - uid INTEGER REFERENCES user, - bgcolor TEXT, - euser TEXT, - user TEXT, - ecomment TEXT, - comment TEXT, - brief TEXT, - omtime DATETIME - ); - INSERT INTO event VALUES('ci',2460994.978048461023,126223,NULL,NULL,NULL,NULL,'drh',NULL,unistr('Data structure improvements on columnar layout. Prep work for getting\u000acolumnar layouts to respond to nScreenWidth.'),NULL,2460994.978048461023); - INSERT INTO event VALUES('ci',2460994.836955601816,126218,NULL,NULL,NULL,NULL,'stephan',NULL,'API doc typo fix.',NULL,2460994.836955601816); - INSERT INTO event VALUES('ci',2460994.88823369192,126212,NULL,NULL,NULL,NULL,'stephan',NULL,'Move sqlite3-api-cleanup.js into post-js-footer.js to remove the final direct Emscripten dependency from the intermediary build product sqlite3-api.js (the whole library, waiting to be bootstrapped). This is partly in response to [forum:4b7d45433731d2e0|forum post 4b7d45433731d2e0], which demonstrates a potential use case for a standalone sqlite3-api.js. This is a build/doc change, not a functional one.',NULL,2460994.88823369192); - INSERT INTO event VALUES('ci',2460994.516081551089,126211,NULL,NULL,NULL,NULL,'drh',NULL,unistr('Improve columnar layout in QRF so that it correctly deals with control\u000acharacters, and especially tabs.'),NULL,2460994.516081551089); - INSERT INTO event VALUES('ci',2460994.409343171865,126208,NULL,NULL,NULL,NULL,'drh',NULL,'Make use of the new sqlite3_str_free() interface in the CLI.',NULL,2460994.409343171865); -} - -do_test 1.10 { - set x "\n[db format -style box -screenwidth 68 \ - {SELECT * FROM mlink ORDER BY rowid}]" - -} { -╭───────┬───────┬───────┬───────┬──────┬───────┬───────┬───────╮ -│ mid │ fid │ pmid │ pid │ fnid │ pfnid │ mperm │ isaux │ -╞═══════╪═══════╪═══════╪═══════╪══════╪═══════╪═══════╪═══════╡ -│ 28775 │ 28774 │ 28773 │ 28706 │ 1 │ 0 │ 0 │ 0 │ -│ 28773 │ 28706 │ 28770 │ 28685 │ 1 │ 0 │ 0 │ 0 │ -│ 28770 │ 28736 │ 28769 │ 28695 │ 2 │ 0 │ 0 │ 0 │ -│ 28770 │ 28697 │ 28769 │ 28698 │ 3 │ 0 │ 0 │ 0 │ -│ 28767 │ 28768 │ 28759 │ 28746 │ 4 │ 0 │ 0 │ 0 │ -╰───────┴───────┴───────┴───────┴──────┴───────┴───────┴───────╯ -} -do_test 1.11 { - set x "\n[db format -style box -screenwidth 52 \ - {SELECT * FROM mlink ORDER BY rowid}]" - -} { -╭─────┬─────┬─────┬─────┬────┬─────┬─────┬─────╮ -│ mid │ fid │pmid │ pid │fnid│pfnid│mperm│isaux│ -╞═════╪═════╪═════╪═════╪════╪═════╪═════╪═════╡ -│28775│28774│28773│28706│ 1│ 0│ 0│ 0│ -│28773│28706│28770│28685│ 1│ 0│ 0│ 0│ -│28770│28736│28769│28695│ 2│ 0│ 0│ 0│ -│28770│28697│28769│28698│ 3│ 0│ 0│ 0│ -│28767│28768│28759│28746│ 4│ 0│ 0│ 0│ -╰─────┴─────┴─────┴─────┴────┴─────┴─────┴─────╯ -} - -do_test 1.20 { - set x "\n[db format -style table -screenwidth 68 \ - {SELECT * FROM mlink ORDER BY rowid}]" - -} { -+-------+-------+-------+-------+------+-------+-------+-------+ -| mid | fid | pmid | pid | fnid | pfnid | mperm | isaux | -+-------+-------+-------+-------+------+-------+-------+-------+ -| 28775 | 28774 | 28773 | 28706 | 1 | 0 | 0 | 0 | -| 28773 | 28706 | 28770 | 28685 | 1 | 0 | 0 | 0 | -| 28770 | 28736 | 28769 | 28695 | 2 | 0 | 0 | 0 | -| 28770 | 28697 | 28769 | 28698 | 3 | 0 | 0 | 0 | -| 28767 | 28768 | 28759 | 28746 | 4 | 0 | 0 | 0 | -+-------+-------+-------+-------+------+-------+-------+-------+ -} -do_test 1.21 { - set x "\n[db format -style table -screenwidth 52 \ - {SELECT * FROM mlink ORDER BY rowid}]" - -} { -+-----+-----+-----+-----+----+-----+-----+-----+ -| mid | fid |pmid | pid |fnid|pfnid|mperm|isaux| -+-----+-----+-----+-----+----+-----+-----+-----+ -|28775|28774|28773|28706| 1| 0| 0| 0| -|28773|28706|28770|28685| 1| 0| 0| 0| -|28770|28736|28769|28695| 2| 0| 0| 0| -|28770|28697|28769|28698| 3| 0| 0| 0| -|28767|28768|28759|28746| 4| 0| 0| 0| -+-----+-----+-----+-----+----+-----+-----+-----+ -} - -do_test 1.30 { - set x "\n[db format -style markdown -screenwidth 68 \ - {SELECT * FROM mlink ORDER BY rowid}]" - -} { -| mid | fid | pmid | pid | fnid | pfnid | mperm | isaux | -|-------|-------|-------|-------|------|-------|-------|-------| -| 28775 | 28774 | 28773 | 28706 | 1 | 0 | 0 | 0 | -| 28773 | 28706 | 28770 | 28685 | 1 | 0 | 0 | 0 | -| 28770 | 28736 | 28769 | 28695 | 2 | 0 | 0 | 0 | -| 28770 | 28697 | 28769 | 28698 | 3 | 0 | 0 | 0 | -| 28767 | 28768 | 28759 | 28746 | 4 | 0 | 0 | 0 | -} -do_test 1.31 { - set x "\n[db format -style markdown -screenwidth 52 \ - {SELECT * FROM mlink ORDER BY rowid}]" - -} { -| mid | fid |pmid | pid |fnid|pfnid|mperm|isaux| -|-----|-----|-----|-----|----|-----|-----|-----| -|28775|28774|28773|28706| 1| 0| 0| 0| -|28773|28706|28770|28685| 1| 0| 0| 0| -|28770|28736|28769|28695| 2| 0| 0| 0| -|28770|28697|28769|28698| 3| 0| 0| 0| -|28767|28768|28759|28746| 4| 0| 0| 0| -} - -do_test 1.40 { - set x "\n[db format -style column -screenwidth 68 \ - {SELECT * FROM mlink ORDER BY rowid}]" - -} { - mid fid pmid pid fnid pfnid mperm isaux ------ ----- ----- ----- ---- ----- ----- ----- -28775 28774 28773 28706 1 0 0 0 -28773 28706 28770 28685 1 0 0 0 -28770 28736 28769 28695 2 0 0 0 -28770 28697 28769 28698 3 0 0 0 -28767 28768 28759 28746 4 0 0 0 -} -do_test 1.41 { - set x "\n[db format -style column -screenwidth 52 \ - {SELECT * FROM mlink ORDER BY rowid}]" - -} { - mid fid pmid pid fnid pfnid mperm isaux ------ ----- ----- ----- ---- ----- ----- ----- -28775 28774 28773 28706 1 0 0 0 -28773 28706 28770 28685 1 0 0 0 -28770 28736 28769 28695 2 0 0 0 -28770 28697 28769 28698 3 0 0 0 -28767 28768 28759 28746 4 0 0 0 -} - - - -finish_test diff --git a/test/qrf04.test b/test/qrf04.test deleted file mode 100644 index 0b231d921..000000000 --- a/test/qrf04.test +++ /dev/null @@ -1,750 +0,0 @@ -# 2025-11-23 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# Test cases for the Query Result Formatter (QRF), and especially -# the bSplitColumn feature. -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl -set testprefix qrf01 - -# The expected output from test 1.1. The "do_test" procedure normally -# ignores differences in whitespace, but whitespace is important for -# this test, so we have to do the comparison ourselves. -# -set expected { -<---- 22 ----> -alice -bob -cinderella-cinderella -daniel -emma -fred -gertrude -harold -ingrid -jake -lisa -mike -nina -octavian -paula -quintus -rita -sam -tammy -ulysses -violet -william -xanthippe -yates -zoe -<---- 23 ----> -alice -bob -cinderella-cinderella -daniel -emma -fred -gertrude -harold -ingrid -jake -lisa -mike -nina -octavian -paula -quintus -rita -sam -tammy -ulysses -violet -william -xanthippe -yates -zoe -<---- 24 ----> -alice -bob -cinderella-cinderella -daniel -emma -fred -gertrude -harold -ingrid -jake -lisa -mike -nina -octavian -paula -quintus -rita -sam -tammy -ulysses -violet -william -xanthippe -yates -zoe -<---- 25 ----> -alice -bob -cinderella-cinderella -daniel -emma -fred -gertrude -harold -ingrid -jake -lisa -mike -nina -octavian -paula -quintus -rita -sam -tammy -ulysses -violet -william -xanthippe -yates -zoe -<---- 26 ----> -alice -bob -cinderella-cinderella -daniel -emma -fred -gertrude -harold -ingrid -jake -lisa -mike -nina -octavian -paula -quintus -rita -sam -tammy -ulysses -violet -william -xanthippe -yates -zoe -<---- 27 ----> -alice -bob -cinderella-cinderella -daniel -emma -fred -gertrude -harold -ingrid -jake -lisa -mike -nina -octavian -paula -quintus -rita -sam -tammy -ulysses -violet -william -xanthippe -yates -zoe -<---- 28 ----> -alice -bob -cinderella-cinderella -daniel -emma -fred -gertrude -harold -ingrid -jake -lisa -mike -nina -octavian -paula -quintus -rita -sam -tammy -ulysses -violet -william -xanthippe -yates -zoe -<---- 29 ----> -alice -bob -cinderella-cinderella -daniel -emma -fred -gertrude -harold -ingrid -jake -lisa -mike -nina -octavian -paula -quintus -rita -sam -tammy -ulysses -violet -william -xanthippe -yates -zoe -<---- 30 ----> -alice -bob -cinderella-cinderella -daniel -emma -fred -gertrude -harold -ingrid -jake -lisa -mike -nina -octavian -paula -quintus -rita -sam -tammy -ulysses -violet -william -xanthippe -yates -zoe -<---- 31 ----> -alice -bob -cinderella-cinderella -daniel -emma -fred -gertrude -harold -ingrid -jake -lisa -mike -nina -octavian -paula -quintus -rita -sam -tammy -ulysses -violet -william -xanthippe -yates -zoe -<---- 32 ----> -alice octavian -bob paula -cinderella-cinderella quintus -daniel rita -emma sam -fred tammy -gertrude ulysses -harold violet -ingrid william -jake xanthippe -lisa yates -mike zoe -nina -<---- 33 ----> -alice octavian -bob paula -cinderella-cinderella quintus -daniel rita -emma sam -fred tammy -gertrude ulysses -harold violet -ingrid william -jake xanthippe -lisa yates -mike zoe -nina -<---- 34 ----> -alice octavian -bob paula -cinderella-cinderella quintus -daniel rita -emma sam -fred tammy -gertrude ulysses -harold violet -ingrid william -jake xanthippe -lisa yates -mike zoe -nina -<---- 35 ----> -alice octavian -bob paula -cinderella-cinderella quintus -daniel rita -emma sam -fred tammy -gertrude ulysses -harold violet -ingrid william -jake xanthippe -lisa yates -mike zoe -nina -<---- 36 ----> -alice octavian -bob paula -cinderella-cinderella quintus -daniel rita -emma sam -fred tammy -gertrude ulysses -harold violet -ingrid william -jake xanthippe -lisa yates -mike zoe -nina -<---- 37 ----> -alice octavian -bob paula -cinderella-cinderella quintus -daniel rita -emma sam -fred tammy -gertrude ulysses -harold violet -ingrid william -jake xanthippe -lisa yates -mike zoe -nina -<---- 38 ----> -alice octavian -bob paula -cinderella-cinderella quintus -daniel rita -emma sam -fred tammy -gertrude ulysses -harold violet -ingrid william -jake xanthippe -lisa yates -mike zoe -nina -<---- 39 ----> -alice octavian -bob paula -cinderella-cinderella quintus -daniel rita -emma sam -fred tammy -gertrude ulysses -harold violet -ingrid william -jake xanthippe -lisa yates -mike zoe -nina -<---- 40 ----> -alice octavian -bob paula -cinderella-cinderella quintus -daniel rita -emma sam -fred tammy -gertrude ulysses -harold violet -ingrid william -jake xanthippe -lisa yates -mike zoe -nina -<---- 41 ----> -alice octavian -bob paula -cinderella-cinderella quintus -daniel rita -emma sam -fred tammy -gertrude ulysses -harold violet -ingrid william -jake xanthippe -lisa yates -mike zoe -nina -<---- 42 ----> -alice jake tammy -bob lisa ulysses -cinderella-cinderella mike violet -daniel nina william -emma octavian xanthippe -fred paula yates -gertrude quintus zoe -harold rita -ingrid sam -<---- 43 ----> -alice jake tammy -bob lisa ulysses -cinderella-cinderella mike violet -daniel nina william -emma octavian xanthippe -fred paula yates -gertrude quintus zoe -harold rita -ingrid sam -<---- 44 ----> -alice jake tammy -bob lisa ulysses -cinderella-cinderella mike violet -daniel nina william -emma octavian xanthippe -fred paula yates -gertrude quintus zoe -harold rita -ingrid sam -<---- 45 ----> -alice jake tammy -bob lisa ulysses -cinderella-cinderella mike violet -daniel nina william -emma octavian xanthippe -fred paula yates -gertrude quintus zoe -harold rita -ingrid sam -<---- 46 ----> -alice jake tammy -bob lisa ulysses -cinderella-cinderella mike violet -daniel nina william -emma octavian xanthippe -fred paula yates -gertrude quintus zoe -harold rita -ingrid sam -<---- 47 ----> -alice jake tammy -bob lisa ulysses -cinderella-cinderella mike violet -daniel nina william -emma octavian xanthippe -fred paula yates -gertrude quintus zoe -harold rita -ingrid sam -<---- 48 ----> -alice jake tammy -bob lisa ulysses -cinderella-cinderella mike violet -daniel nina william -emma octavian xanthippe -fred paula yates -gertrude quintus zoe -harold rita -ingrid sam -<---- 49 ----> -alice jake tammy -bob lisa ulysses -cinderella-cinderella mike violet -daniel nina william -emma octavian xanthippe -fred paula yates -gertrude quintus zoe -harold rita -ingrid sam -<---- 50 ----> -alice jake tammy -bob lisa ulysses -cinderella-cinderella mike violet -daniel nina william -emma octavian xanthippe -fred paula yates -gertrude quintus zoe -harold rita -ingrid sam -<---- 51 ----> -alice harold paula william -bob ingrid quintus xanthippe -cinderella-cinderella jake rita yates -daniel lisa sam zoe -emma mike tammy -fred nina ulysses -gertrude octavian violet -<---- 52 ----> -alice harold paula william -bob ingrid quintus xanthippe -cinderella-cinderella jake rita yates -daniel lisa sam zoe -emma mike tammy -fred nina ulysses -gertrude octavian violet -<---- 53 ----> -alice harold paula william -bob ingrid quintus xanthippe -cinderella-cinderella jake rita yates -daniel lisa sam zoe -emma mike tammy -fred nina ulysses -gertrude octavian violet -<---- 54 ----> -alice harold paula william -bob ingrid quintus xanthippe -cinderella-cinderella jake rita yates -daniel lisa sam zoe -emma mike tammy -fred nina ulysses -gertrude octavian violet -<---- 55 ----> -alice harold paula william -bob ingrid quintus xanthippe -cinderella-cinderella jake rita yates -daniel lisa sam zoe -emma mike tammy -fred nina ulysses -gertrude octavian violet -<---- 56 ----> -alice harold paula william -bob ingrid quintus xanthippe -cinderella-cinderella jake rita yates -daniel lisa sam zoe -emma mike tammy -fred nina ulysses -gertrude octavian violet -<---- 57 ----> -alice harold paula william -bob ingrid quintus xanthippe -cinderella-cinderella jake rita yates -daniel lisa sam zoe -emma mike tammy -fred nina ulysses -gertrude octavian violet -<---- 58 ----> -alice harold paula william -bob ingrid quintus xanthippe -cinderella-cinderella jake rita yates -daniel lisa sam zoe -emma mike tammy -fred nina ulysses -gertrude octavian violet -<---- 59 ----> -alice harold paula william -bob ingrid quintus xanthippe -cinderella-cinderella jake rita yates -daniel lisa sam zoe -emma mike tammy -fred nina ulysses -gertrude octavian violet -<---- 60 ----> -alice harold paula william -bob ingrid quintus xanthippe -cinderella-cinderella jake rita yates -daniel lisa sam zoe -emma mike tammy -fred nina ulysses -gertrude octavian violet -<---- 61 ----> -alice fred lisa quintus violet -bob gertrude mike rita william -cinderella-cinderella harold nina sam xanthippe -daniel ingrid octavian tammy yates -emma jake paula ulysses zoe -<---- 62 ----> -alice fred lisa quintus violet -bob gertrude mike rita william -cinderella-cinderella harold nina sam xanthippe -daniel ingrid octavian tammy yates -emma jake paula ulysses zoe -<---- 63 ----> -alice fred lisa quintus violet -bob gertrude mike rita william -cinderella-cinderella harold nina sam xanthippe -daniel ingrid octavian tammy yates -emma jake paula ulysses zoe -<---- 64 ----> -alice fred lisa quintus violet -bob gertrude mike rita william -cinderella-cinderella harold nina sam xanthippe -daniel ingrid octavian tammy yates -emma jake paula ulysses zoe -<---- 65 ----> -alice fred lisa quintus violet -bob gertrude mike rita william -cinderella-cinderella harold nina sam xanthippe -daniel ingrid octavian tammy yates -emma jake paula ulysses zoe -<---- 66 ----> -alice fred lisa quintus violet -bob gertrude mike rita william -cinderella-cinderella harold nina sam xanthippe -daniel ingrid octavian tammy yates -emma jake paula ulysses zoe -<---- 67 ----> -alice fred lisa quintus violet -bob gertrude mike rita william -cinderella-cinderella harold nina sam xanthippe -daniel ingrid octavian tammy yates -emma jake paula ulysses zoe -<---- 68 ----> -alice fred lisa quintus violet -bob gertrude mike rita william -cinderella-cinderella harold nina sam xanthippe -daniel ingrid octavian tammy yates -emma jake paula ulysses zoe -<---- 69 ----> -alice fred lisa quintus violet -bob gertrude mike rita william -cinderella-cinderella harold nina sam xanthippe -daniel ingrid octavian tammy yates -emma jake paula ulysses zoe -<---- 70 ----> -alice fred lisa quintus violet -bob gertrude mike rita william -cinderella-cinderella harold nina sam xanthippe -daniel ingrid octavian tammy yates -emma jake paula ulysses zoe -<---- 71 ----> -alice fred lisa quintus violet -bob gertrude mike rita william -cinderella-cinderella harold nina sam xanthippe -daniel ingrid octavian tammy yates -emma jake paula ulysses zoe -<---- 72 ----> -alice fred lisa quintus violet -bob gertrude mike rita william -cinderella-cinderella harold nina sam xanthippe -daniel ingrid octavian tammy yates -emma jake paula ulysses zoe -<---- 73 ----> -alice fred lisa quintus violet -bob gertrude mike rita william -cinderella-cinderella harold nina sam xanthippe -daniel ingrid octavian tammy yates -emma jake paula ulysses zoe -<---- 74 ----> -alice emma ingrid nina rita violet zoe -bob fred jake octavian sam william -cinderella-cinderella gertrude lisa paula tammy xanthippe -daniel harold mike quintus ulysses yates -<---- 75 ----> -alice emma ingrid nina rita violet zoe -bob fred jake octavian sam william -cinderella-cinderella gertrude lisa paula tammy xanthippe -daniel harold mike quintus ulysses yates -<---- 76 ----> -alice emma ingrid nina rita violet zoe -bob fred jake octavian sam william -cinderella-cinderella gertrude lisa paula tammy xanthippe -daniel harold mike quintus ulysses yates -<---- 77 ----> -alice emma ingrid nina rita violet zoe -bob fred jake octavian sam william -cinderella-cinderella gertrude lisa paula tammy xanthippe -daniel harold mike quintus ulysses yates -<---- 78 ----> -alice emma ingrid nina rita violet zoe -bob fred jake octavian sam william -cinderella-cinderella gertrude lisa paula tammy xanthippe -daniel harold mike quintus ulysses yates -<---- 79 ----> -alice emma ingrid nina rita violet zoe -bob fred jake octavian sam william -cinderella-cinderella gertrude lisa paula tammy xanthippe -daniel harold mike quintus ulysses yates -<---- 80 ----> -alice emma ingrid nina rita violet zoe -bob fred jake octavian sam william -cinderella-cinderella gertrude lisa paula tammy xanthippe -daniel harold mike quintus ulysses yates -} - -do_test 1.0 { - db eval { - CREATE TABLE t1(x); - INSERT INTO t1(x) VALUES - ('alice'), - ('bob'), - ('cinderella-cinderella'), - ('daniel'), - ('emma'), - ('fred'), - ('gertrude'), - ('harold'), - ('ingrid'), - ('jake'), - ('lisa'), - ('mike'), - ('nina'), - ('octavian'), - ('paula'), - ('quintus'), - ('rita'), - ('sam'), - ('tammy'), - ('ulysses'), - ('violet'), - ('william'), - ('xanthippe'), - ('yates'), - ('zoe'); - } - set res \n - for {set i 22} {$i<=80} {incr i} { - set sp [expr {$i-13}] - append res [format "<----%*s%3d%*s---->\n" \ - [expr {$sp/2}] {} $i [expr {$sp-$sp/2}] {}] - append res [db format -style column -title off \ - -screenwidth $i -splitcolumn on \ - {SELECT x FROM t1 ORDER BY x ASC}] - } - expr {$res eq $::expected} -} {1} diff --git a/test/qrf05.test b/test/qrf05.test deleted file mode 100644 index 0d5a4d7f9..000000000 --- a/test/qrf05.test +++ /dev/null @@ -1,37 +0,0 @@ -# 2025-12-02 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# Test cases for the Query Result Formatter (QRF) -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl -set testprefix qrf05 - -do_execsql_test 1.0 { - CREATE TABLE t1(a INT NOT NULL); -} -do_test 1.1 { - set rc [catch {db format -style list \ - {INSERT INTO t1 VALUES(123) RETURNING *}} msg] - list $rc [string trim $msg] -} {0 123} -do_test 1.2 { - set rc [catch {db format -style list \ - {INSERT INTO t1 VALUES(NULL) RETURNING *}} msg] - list $rc [string trim $msg] -} {1 {NOT NULL constraint failed: t1.a}} -do_test 1.3 { - set rc [catch {db format -version 99 {SELECT * FROM t1}} msg] - list $rc [string trim $msg] -} {1 {unusable sqlite3_qrf_spec.iVersion (99)}} - -finish_test diff --git a/test/qrf06.test b/test/qrf06.test deleted file mode 100644 index 5fa62c26f..000000000 --- a/test/qrf06.test +++ /dev/null @@ -1,576 +0,0 @@ -# 2025-12-02 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# Test cases for the Query Result Formatter (QRF), and especially -# the sqlite3_qrf_wcwidth() function and its utilization. -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl -set testprefix qrf06 - -# Data -db eval { - BEGIN TRANSACTION; - CREATE TABLE language(name TEXT); - INSERT INTO language(name) VALUES - ('العربية'), - ('Deutsch'), - ('English'), - ('Español'), - ('فارسی'), - ('Français'), - ('Italiano'), - ('مصرى'), - ('Nederlands'), - ('日本語'), - ('Polski'), - ('Português'), - ('Sinugboanong Binisaya'), - ('Svenska'), - ('Українська'), - ('Tiếng Việt'), - ('Winaray'), - ('中文'), - ('Русский'), - ('Afrikaans'), - ('Shqip'), - ('Asturianu'), - ('Azərbaycanca'), - ('Български'), - ('閩南語 / Bân-lâm-gú'), - ('বাংলা'), - ('Беларуская'), - ('Català'), - ('Čeština'), - ('Cymraeg'), - ('Dansk'), - ('Eesti'), - ('Ελληνικά'), - ('Esperanto'), - ('Euskara'), - ('Galego'), - ('한국어'), - ('Հայերեն'), - ('हिन्दी'), - ('Hrvatski'), - ('Bahasa Indonesia'), - ('עברית'), - ('ქართული'), - ('Ladin'), - ('Latina'), - ('Latviešu'), - ('Lietuvių'), - ('Magyar'), - ('Македонски'), - ('Malagasy'), - ('मराठी'), - ('Bahasa Melayu'), - ('Bahaso Minangkabau'), - ('မြန်မာဘာသာ'), - ('Norskbokmålnynorsk'), - ('Нохчийн'), - ('Oʻzbekcha / Ўзбекча'), - ('Қазақша / Qazaqşa / قازاقشا'), - ('Română'), - ('Simple English'), - ('Slovenčina'), - ('Slovenščina'), - ('Српски / Srpski'), - ('Srpskohrvatski / Српскохрватски'), - ('Suomi'), - ('Kiswahili'), - ('தமிழ்'), - ('Татарча / Tatarça'), - ('తెలుగు'), - ('ภาษาไทย'), - ('Тоҷикӣ'), - ('تۆرکجه'), - ('Türkçe'), - ('اردو'), - ('粵語'), - ('Bahsa Acèh'), - ('Alemannisch'), - ('አማርኛ'), - ('Aragonés'), - ('Արեւմտահայերէն'), - ('Bahasa Hulontalo'), - ('Basa Bali'), - ('Bahasa Banjar'), - ('Basa Banyumasan'), - ('Башҡортса'), - ('Беларуская (тарашкевіца)'), - ('Bikol Central'), - ('বিষ্ণুপ্রিয়া মণিপুরী'), - ('Boarisch'), - ('Bosanski'), - ('Brezhoneg'), - ('Чӑвашла'), - ('Dagbanli'), - ('الدارجة'), - ('Diné Bizaad'), - ('Emigliàn–Rumagnòl'), - ('Fiji Hindi'), - ('Føroyskt'), - ('Frysk'), - ('Fulfulde'), - ('Gaeilge'), - ('Gàidhlig'), - ('گیلکی'), - ('ગુજરાતી'), - ('Hak-kâ-ngî / 客家語'), - ('Hausa'), - ('Hornjoserbsce'), - ('Ido'), - ('Igbo'), - ('Ilokano'), - ('Interlingua'), - ('Interlingue'), - ('Ирон'), - ('Íslenska'), - ('Jawa'), - ('ಕನ್ನಡ'), - ('Kapampangan'), - ('ភាសាខ្មែរ'), - ('Kotava'), - ('Kreyòl Ayisyen'), - ('Kurdî / كوردی'), - ('کوردیی ناوەندی'), - ('Кыргызча'), - ('Кырык мары'), - ('Lëtzebuergesch'), - ('Lìgure'), - ('Limburgs'), - ('Lombard'), - ('मैथिली'), - ('മലയാളം'), - ('მარგალური'), - ('مازِرونی'), - ('Mìng-dĕ̤ng-ngṳ̄ / 閩東語'), - ('Монгол'), - ('Napulitano'), - ('नेपाल भाषा'), - ('Nordfriisk'), - ('Occitan'), - ('Олык марий'), - ('ଓଡି଼ଆ'), - ('অসমীযা়'), - ('ਪੰਜਾਬੀ'), - ('پنجابی (شاہ مکھی)'), - ('پښتو'), - ('Piemontèis'), - ('Plattdüütsch'), - ('Qaraqalpaqsha'), - ('Qırımtatarca'), - ('Runa Simi'), - ('Русиньскый'), - ('संस्कृतम्'), - ('ᱥᱟᱱᱛᱟᱲᱤ'), - ('سرائیکی'), - ('Саха Тыла'), - ('Scots'), - ('ChiShona'), - ('Sicilianu'), - ('සිංහල'), - ('سنڌي'), - ('Ślůnski'), - ('Basa Sunda'), - ('Taclḥit'), - ('Tagalog'), - ('ၽႃႇသႃႇတႆး'), - ('ⵜⴰⵎⴰⵣⵉⵖⵜ ⵜⴰⵏⴰⵡⴰⵢⵜ'), - ('tolışi'), - ('chiTumbuka'), - ('Basa Ugi'), - ('Vèneto'), - ('Volapük'), - ('Walon'), - ('文言'), - ('吴语'), - ('ייִדיש'), - ('Yorùbá'), - ('Zazaki'), - ('žemaitėška'), - ('isiZulu'), - ('नेपाली'), - ('ꯃꯤꯇꯩ ꯂꯣꯟ'), - ('Dzhudezmo / לאדינו'), - ('Адыгэбзэ'), - ('Ænglisc'), - ('Anarâškielâ'), - ('अंगिका'), - ('Аԥсшәа'), - ('armãneashti'), - ('Arpitan'), - ('atikamekw'), - ('ܐܬܘܪܝܐ'), - ('Avañe’ẽ'), - ('Авар'), - ('Aymar'), - ('Batak Toba'), - ('Betawi'), - ('भोजपुरी'), - ('Bislama'), - ('བོད་ཡིག'), - ('Буряад'), - ('Chavacano de Zamboanga'), - ('Chichewa'), - ('Corsu'), - ('Vahcuengh / 話僮'), - ('Dagaare'), - ('Davvisámegiella'), - ('Deitsch'), - ('ދިވެހިބަސް'), - ('Dolnoserbski'), - ('Dusun Bundu-liwan'), - ('Эрзянь'), - ('Estremeñu'), - ('Eʋegbe'), - ('Farefare'), - ('Fɔ̀ngbè'), - ('Furlan'), - ('Gaelg'), - ('Gagauz'), - ('ГӀалгӀай'), - ('Ghanaian Pidgin'), - ('Gĩkũyũ'), - ('赣语 / 贛語'), - ('Gungbe'), - ('Хальмг'), - ('ʻŌlelo Hawaiʻi'), - ('Ikinyarwanda'), - ('Jaku Iban'), - ('Kabɩyɛ'), - ('Yerwa Kanuri'), - ('Kaszëbsczi'), - ('Kernewek'), - ('Коми'), - ('Перем коми'), - ('Kongo'), - ('कोंकणी / Konknni'), - ('كٲشُر'), - ('Kriyòl Gwiyannen'), - ('Kumoring'), - ('Kʋsaal'), - ('ພາສາລາວ'), - ('Лакку'), - ('Latgaļu'), - ('Лезги'), - ('Li Niha'), - ('Lingála'), - ('Lingua Franca Nova'), - ('livvinkarjala'), - ('lojban'), - ('Luganda'), - ('Madhurâ'), - ('Malti'), - ('Mandailing'), - ('Māori'), - ('Mfantse'), - ('Mirandés'), - ('Мокшень'), - ('ဘာသာ မန်'), - ('Moore'), - ('ߒߞߏ'), - ('Na Vosa Vaka-Viti'), - ('Nāhuatlahtōlli'), - ('Naijá'), - ('Nedersaksisch'), - ('Nouormand / Normaund'), - ('Novial'), - ('Afaan Oromoo'), - ('ပအိုဝ်ႏဘာႏသာႏ'), - ('Pangasinán'), - ('Pangcah'), - ('Papiamentu'), - ('Patois'), - ('Pfälzisch'), - ('Picard'), - ('Къарачай–малкъар'), - ('Ripoarisch'), - ('Rumantsch'), - ('Sakizaya'), - ('Gagana Sāmoa'), - ('Sardu'), - ('Seediq'), - ('Seeltersk'), - ('Sesotho'), - ('Sesotho sa Leboa'), - ('Setswana'), - ('ꠍꠤꠟꠐꠤ'), - ('Словѣ́ньскъ / ⰔⰎⰑⰂⰡⰐⰠⰔⰍⰟ'), - ('Soomaaliga'), - ('Sranantongo'), - ('SiSwati'), - ('Reo tahiti'), - ('Taqbaylit'), - ('Tarandíne'), - ('Tayal'), - ('Tetun'), - ('Tok Pisin'), - ('faka Tonga'), - ('Türkmençe'), - ('Twi'), - ('Tyap'), - ('Тыва дыл'), - ('Удмурт'), - ('ئۇيغۇرچه'), - ('Vepsän'), - ('võro'), - ('West-Vlams'), - ('Wolof'), - ('isiXhosa'), - ('Zeêuws'), - ('алтай тил'), - ('अवधी'), - ('डोटेली'), - ('ತುಳು'), - ('ရခိုင်'), - ('Bajau Sama'), - ('Bamanankan'), - ('Chamoru'), - ('རྫོང་ཁ'), - ('𐌲𐌿𐍄𐌹𐍃𐌺x'), - ('Igala'), - ('ᐃᓄᒃᑎᑐᑦ / Inuktitut'), - ('Iñupiak'), - ('isiNdebele seSewula'), - ('Kalaallisut'), - ('Nupe'), - ('Obolo'), - ('पालि'), - ('pinayuanan'), - ('Ποντιακά'), - ('romani čhib'), - ('Ikirundi'), - ('руски'), - ('Sängö'), - ('ᥖᥭᥰᥖᥬᥳᥑᥨᥒᥰ'), - ('ትግርኛ'), - ('Thuɔŋjäŋ'), - ('ᏣᎳᎩ'), - ('Tsėhesenėstsestotse'), - ('Xitsonga'), - ('Tshivenḓa'), - ('Wayuunaiki'), - ('адыгабзэ'); - COMMIT; -} - -do_test 1.2 { - set res \n[db format -style box { - SELECT name, rowid AS id FROM language - WHERE length(name)=2 - ORDER BY name - }] - set exp { -╭──────┬─────╮ -│ name │ id │ -╞══════╪═════╡ -│ 中文 │ 18 │ -│ 吴语 │ 173 │ -│ 文言 │ 172 │ -│ 粵語 │ 75 │ -╰──────┴─────╯ -} - if {$res ne $exp} { - puts [list $res] - puts [list $exp] - } - string compare $res $exp -} {0} - -do_test 1.3 { - set res \n[db format -style box { - SELECT name, rowid AS id FROM language - WHERE length(name)=3 - ORDER BY name - }] - set exp { -╭────────┬─────╮ -│ name │ id │ -╞════════╪═════╡ -│ Ido │ 108 │ -│ Twi │ 297 │ -│ ߒߞߏ │ 258 │ -│ ᏣᎳᎩ │ 335 │ -│ 日本語 │ 10 │ -│ 한국어 │ 37 │ -╰────────┴─────╯ -} - if {$res ne $exp} { - puts [list $res] - puts [list $exp] - } - string compare $res $exp -} {0} - -do_test 1.4 { - set res \n[db format -style box { - SELECT name, rowid AS id FROM language - WHERE length(name)=4 - ORDER BY name - }] - set exp { -╭──────┬─────╮ -│ name │ id │ -╞══════╪═════╡ -│ Igbo │ 109 │ -│ Jawa │ 115 │ -│ Nupe │ 323 │ -│ Tyap │ 298 │ -│ võro │ 303 │ -│ Авар │ 192 │ -│ Ирон │ 113 │ -│ Коми │ 231 │ -│ اردو │ 74 │ -│ سنڌي │ 159 │ -│ مصرى │ 8 │ -│ پښتو │ 144 │ -│ अवधी │ 309 │ -│ पालि │ 325 │ -│ ತುಳು │ 311 │ -│ ትግርኛ │ 333 │ -│ አማርኛ │ 78 │ -╰──────┴─────╯ -} - if {$res ne $exp} { - puts [list $res] - puts [list $exp] - } - string compare $res $exp -} {0} - -do_test 1.5 { - set res \n[db format -style box { - SELECT name, rowid AS id FROM language - WHERE length(name)=5 - ORDER BY name - }] - set exp { -╭───────┬─────╮ -│ name │ id │ -╞═══════╪═════╡ -│ Aymar │ 193 │ -│ Corsu │ 202 │ -│ Dansk │ 31 │ -│ Eesti │ 32 │ -│ Frysk │ 99 │ -│ Gaelg │ 216 │ -│ Hausa │ 106 │ -│ Igala │ 318 │ -│ Kongo │ 233 │ -│ Ladin │ 44 │ -│ Malti │ 250 │ -│ Moore │ 257 │ -│ Māori │ 252 │ -│ Naijá │ 261 │ -│ Obolo │ 324 │ -│ Sardu │ 278 │ -│ Scots │ 155 │ -│ Shqip │ 21 │ -│ Suomi │ 65 │ -│ Sängö │ 331 │ -│ Tayal │ 292 │ -│ Tetun │ 293 │ -│ Walon │ 171 │ -│ Wolof │ 305 │ -│ Лакку │ 240 │ -│ Лезги │ 242 │ -│ руски │ 330 │ -│ עברית │ 42 │ -│ فارسی │ 5 │ -│ كٲشُر │ 235 │ -│ گیلکی │ 103 │ -│ मराठी │ 51 │ -│ বাংলা │ 26 │ -│ ଓଡି଼ଆ │ 140 │ -│ தமிழ் │ 67 │ -│ ಕನ್ನಡ │ 116 │ -│ සිංහල │ 158 │ -│ ꠍꠤꠟꠐꠤ │ 284 │ -╰───────┴─────╯ -} - if {$res ne $exp} { - puts [list $res] - puts [list $exp] - } - string compare $res $exp -} {0} - -do_test 1.6 { - set res \n[db format -style box { - SELECT name, rowid AS id FROM language - WHERE length(name)=6 - ORDER BY name - }] - set exp { -╭────────┬─────╮ -│ name │ id │ -╞════════╪═════╡ -│ Betawi │ 195 │ -│ Català │ 28 │ -│ Eʋegbe │ 212 │ -│ Furlan │ 215 │ -│ Gagauz │ 217 │ -│ Galego │ 36 │ -│ Gungbe │ 222 │ -│ Gĩkũyũ │ 220 │ -│ Kabɩyɛ │ 227 │ -│ Kotava │ 119 │ -│ Kʋsaal │ 238 │ -│ Latina │ 45 │ -│ Lìgure │ 126 │ -│ Magyar │ 48 │ -│ Novial │ 264 │ -│ Patois │ 270 │ -│ Picard │ 272 │ -│ Polski │ 11 │ -│ Română │ 59 │ -│ Seediq │ 279 │ -│ Türkçe │ 73 │ -│ Vepsän │ 302 │ -│ Vèneto │ 169 │ -│ Yorùbá │ 175 │ -│ Zazaki │ 176 │ -│ Zeêuws │ 307 │ -│ lojban │ 247 │ -│ tolışi │ 166 │ -│ Аԥсшәа │ 186 │ -│ Буряад │ 199 │ -│ Монгол │ 134 │ -│ Тоҷикӣ │ 71 │ -│ Удмурт │ 300 │ -│ Хальмг │ 223 │ -│ Эрзянь │ 210 │ -│ ייִדיש │ 174 │ -│ تۆرکجه │ 72 │ -│ ܐܬܘܪܝܐ │ 190 │ -│ अंगिका │ 185 │ -│ डोटेली │ 310 │ -│ नेपाली │ 179 │ -│ मैथिली │ 129 │ -│ हिन्दी │ 39 │ -│ ਪੰਜਾਬੀ │ 142 │ -│ తెలుగు │ 69 │ -│ മലയാളം │ 130 │ -│ རྫོང་ཁ │ 316 │ -│ ရခိုင် │ 312 │ -╰────────┴─────╯ -} - if {$res ne $exp} { - puts [list $res] - puts [list $exp] - } - string compare $res $exp -} {0} - -finish_test diff --git a/test/recover.test b/test/recover.test index 7035e407c..ad6b7298d 100644 --- a/test/recover.test +++ b/test/recover.test @@ -42,7 +42,7 @@ proc compare_dbs {db1 db2} { proc recover_with_opts {opts} { set cmd ".recover $opts" - set fd [open [list |$::CLI -noinit test.db $cmd]] + set fd [open [list |$::CLI test.db $cmd]] fconfigure $fd -translation binary set sql [read $fd] close $fd diff --git a/test/regexp1.sql b/test/regexp1.sql deleted file mode 100644 index c1938885a..000000000 --- a/test/regexp1.sql +++ /dev/null @@ -1,32 +0,0 @@ -#!sqlite3 -# -# 2025-12-16 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# Test cases for the oversized patterns in the REGEXP extension found -# at ext/misc/regexp.c. -# -.mode list -.testcase 100 --- 0 1 2 3 4 --- 123456789 123456789 123456789 123456789 123 -SELECT 'abcdefg' REGEXP '((((((((((((((((((abcdefg))))))))))))))))))'; -.check 1 - -.testcase 110 -.limit like_pattern_length 42 -SELECT 'abcdefg' REGEXP '((((((((((((((((((abcdefg))))))))))))))))))'; -.check -glob "Error near line #: REGEXP pattern too big*" - -.testcase 120 -.limit like_pattern_length 43 -SELECT 'abcdefg' REGEXP '((((((((((((((((((abcdefg))))))))))))))))))'; -.check 1 diff --git a/test/regexp1.test b/test/regexp1.test index fb123284b..0401b13d7 100644 --- a/test/regexp1.test +++ b/test/regexp1.test @@ -331,29 +331,5 @@ do_execsql_test regexp1-7.12 { SELECT char(0x61,0x10ffff,0x62) REGEXP char(0x10ffff); } 1 -do_execsql_test regexp1-8.0 { - CREATE TABLE t2(a); - INSERT INTO t2 VALUES('abc-def'); - SELECT length(a) FROM t2; -} {7} - -do_execsql_test regexp1-8.1 { - SELECT rowid FROM t2 WHERE a REGEXP '[1-5]'; -} {} -do_execsql_test regexp1-8.2 { - SELECT rowid FROM t2 WHERE a REGEXP '[1\-5]'; -} {1} -do_execsql_test regexp1-8.3 { - SELECT rowid FROM t2 WHERE a REGEXP '[x\-]'; -} {1} -do_catchsql_test regexp1-8.4 { - SELECT rowid FROM t2 WHERE a REGEXP '[x-]'; -} {1 {unclosed '['}} -do_execsql_test regexp1-8.5 { - SELECT rowid FROM t2 WHERE a REGEXP '-'; -} {1} -do_execsql_test regexp1-8.6 { - SELECT rowid FROM t2 WHERE a REGEXP '\-'; -} {1} finish_test diff --git a/test/rowvalue4.test b/test/rowvalue4.test index 5e02f0fc2..1ef5fc292 100644 --- a/test/rowvalue4.test +++ b/test/rowvalue4.test @@ -236,7 +236,8 @@ do_eqp_test 5.1 { QUERY PLAN |--SEARCH d2 USING INDEX d2ab (a=? AND b=?) |--LIST SUBQUERY xxxxxx - | `--SCAN d1 + | |--SCAN d1 + | `--CREATE BLOOM FILTER `--LIST SUBQUERY xxxxxx |--SCAN d1 `--CREATE BLOOM FILTER diff --git a/test/rowvalueA.test b/test/rowvalueA.test index 16429f985..8760c2c39 100644 --- a/test/rowvalueA.test +++ b/test/rowvalueA.test @@ -73,29 +73,4 @@ do_catchsql_test 2.3 { SELECT 2 IN ( (1, 2), (3, 4), (5, 6) ) } {1 {row value misused}} -#------------------------------------------------------------------------- -# Test the fix for forum post https://sqlite.org/forum/forumpost/6ceca07fc3 -# -do_execsql_test 3.0 { - CREATE TABLE x2 (x, y); - INSERT INTO x2 VALUES (1234, 'abc'); - - CREATE TABLE x1 (a, b PRIMARY KEY COLLATE NOCASE) WITHOUT ROWID; - INSERT INTO x1 VALUES (1234, 'ABCD'); -} - -do_execsql_test 3.1 { - SELECT * FROM x2 CROSS JOIN x1 WHERE (1234, x2.y) > (x1.a, x1.b); -} {1234 abc 1234 ABCD} - -do_execsql_test 3.2 { - CREATE INDEX x1a ON x1(a); -} - -do_execsql_test 3.3 { - SELECT * FROM x2 CROSS JOIN x1 WHERE (1234, x2.y) > (x1.a, x1.b); -} {1234 abc 1234 ABCD} - - - finish_test diff --git a/test/schema.test b/test/schema.test index a6564293b..c7daef20b 100644 --- a/test/schema.test +++ b/test/schema.test @@ -227,10 +227,10 @@ ifcapable auth { set ::STMT [sqlite3_prepare $::DB {SELECT * FROM sqlite_master} -1 TAIL] db auth {} sqlite3_step $::STMT - } {SQLITE_ERROR} + } {SQLITE_ROW} do_test schema-8.12 { sqlite3_finalize $::STMT - } {SQLITE_SCHEMA} + } {SQLITE_OK} } diff --git a/test/select9.test b/test/select9.test index bef56d83f..bbed8e18f 100644 --- a/test/select9.test +++ b/test/select9.test @@ -406,7 +406,7 @@ do_test select9-4.4 { do_test select9-4.5 { execsql { CREATE VIEW v1 AS SELECT a FROM t1 UNION SELECT d FROM t2 } cksort { SELECT a FROM v1 ORDER BY 1 LIMIT 5 } -} {1 2 3 4 5 nosort} +} {1 2 3 4 5 sort} do_test select9-4.X { execsql { DROP INDEX i1; diff --git a/test/shell1.test b/test/shell1.test index 1d111d616..abf214a90 100644 --- a/test/shell1.test +++ b/test/shell1.test @@ -216,14 +216,10 @@ do_test shell1-2.2.4 { } {0 {}} do_test shell1-2.2.5 { catchcmd "test.db" ".mode \"insert FOO" -} {1 {line 1: .mode "insert FOO -line 1: ^--- unknown mode -line 1: Use ".help .mode" for more info}} +} {1 {Error: mode should be one of: ascii box column csv html insert json line list markdown qbox quote table tabs tcl}} do_test shell1-2.2.6 { catchcmd "test.db" ".mode \'insert FOO" -} {1 {line 1: .mode 'insert FOO -line 1: ^--- unknown mode -line 1: Use ".help .mode" for more info}} +} {1 {Error: mode should be one of: ascii box column csv html insert json line list markdown qbox quote table tabs tcl}} # check multiple tokens, and quoted tokens do_test shell1-2.3.1 { @@ -251,9 +247,7 @@ do_test shell1-2.3.7 { # check quoted args are unquoted do_test shell1-2.4.1 { catchcmd "test.db" ".mode FOO" -} {1 {line 1: .mode FOO -line 1: ^--- unknown mode -line 1: Use ".help .mode" for more info}} +} {1 {Error: mode should be one of: ascii box column csv html insert json line list markdown qbox quote table tabs tcl}} do_test shell1-2.4.2 { catchcmd "test.db" ".mode csv" } {0 {}} @@ -302,7 +296,9 @@ do_test shell1-3.2.4 { catchcmd "test.db" ".bail OFF BAD" } {1 {Usage: .bail on|off}} -ifcapable vtab { +# This test will not work on winrt, as winrt has no concept of the absolute +# paths that the test expects in the result. It uses relative paths only. +ifcapable vtab&&!winrt { # .databases List names and files of attached databases do_test shell1-3.3.1 { catchcmd "-csv test.db" ".databases" @@ -368,6 +364,7 @@ do_test shell1-3.7.4 { catchcmd "test.db" ".explain OFF BAD" } {0 {}} + # .header(s) ON|OFF Turn display of headers on or off do_test shell1-3.9.1 { catchcmd "test.db" ".header" @@ -403,7 +400,7 @@ do_test shell1-3.10.1 { # look for a few of the possible help commands list [regexp {.help} $res] \ [regexp {.quit} $res] \ - [regexp {.mode} $res] + [regexp {.show} $res] } {1 1 1} do_test shell1-3.10.2 { # we allow .help to take extra args (it is help after all) @@ -411,24 +408,20 @@ do_test shell1-3.10.2 { # look for a few of the possible help commands list [regexp {.help} $res] \ [regexp {.quit} $res] \ - [regexp {.mode} $res] + [regexp {.show} $res] } {1 1 1} # .import FILE TABLE Import data from FILE into TABLE do_test shell1-3.11.1 { catchcmd "test.db" ".import" -} {/1 .line 1: Missing FILE argument.*/} +} {/1 .ERROR: missing FILE argument.*/} do_test shell1-3.11.2 { catchcmd "test.db" ".import FOO" -} {/1 .line 1: Missing TABLE argument.*/} +} {/1 .ERROR: missing TABLE argument.*/} do_test shell1-3.11.3 { # too many arguments catchcmd "test.db" ".import FOO BAR BAD" -} {1 {line 1: .import FOO BAR BAD -line 1: ^--- unknown argument}} -do_test shell1-3.11.4 { - catchcmd "test.db" ".import <<END t1\na,b,c\n1,2,3" -} {1 {line 1: Content terminator "END" not found.}} +} {/1 .ERROR: extra argument: "BAD".*./} # .indexes ?TABLE? Show names of all indexes # If TABLE specified, only show indexes for tables @@ -458,13 +451,11 @@ do_test shell1-3.12.3 { # tabs Tab-separated values # tcl TCL list elements do_test shell1-3.13.1 { - catchcmd "test.db" ".mode batch\n.mode" -} {0 {.mode list}} + catchcmd "test.db" ".mode" +} {0 {current output mode: list --escape ascii}} do_test shell1-3.13.2 { catchcmd "test.db" ".mode FOO" -} {1 {line 1: .mode FOO -line 1: ^--- unknown mode -line 1: Use ".help .mode" for more info}} +} {1 {Error: mode should be one of: ascii box column csv html insert json line list markdown qbox quote table tabs tcl}} do_test shell1-3.13.3 { catchcmd "test.db" ".mode csv" } {0 {}} @@ -512,7 +503,7 @@ do_test shell1-3.15.1 { .print x" } {0 x} do_test shell1-3.15.2 { - catchcmd "test.db" ".mode batch\n.output FOO + catchcmd "test.db" ".output FOO .print x .output SELECT readfile('FOO');" @@ -521,8 +512,17 @@ SELECT readfile('FOO');" do_test shell1-3.15.3 { # too many arguments catchcmd "test.db" ".output FOO BAD" -} {1 {line 1: .output FOO BAD -line 1: ^--- surplus argument}} +} {1 {ERROR: extra parameter: "BAD". Usage: +.output ?FILE? Send output to FILE or stdout if FILE is omitted + If FILE begins with '|' then open it as a pipe. + If FILE is 'off' then output is disabled. + Options: + --bom Prefix output with a UTF8 byte-order mark + -e Send output to the system text editor + --plain Use text/plain for -w option + -w Send output to a web browser + -x Send output as CSV to a spreadsheet +child process exited abnormally}} # .output stdout Send output to the screen do_test shell1-3.16.1 { @@ -531,8 +531,17 @@ do_test shell1-3.16.1 { do_test shell1-3.16.2 { # too many arguments catchcmd "test.db" ".output stdout BAD" -} {1 {line 1: .output stdout BAD -line 1: ^--- surplus argument}} +} {1 {ERROR: extra parameter: "BAD". Usage: +.output ?FILE? Send output to FILE or stdout if FILE is omitted + If FILE begins with '|' then open it as a pipe. + If FILE is 'off' then output is disabled. + Options: + --bom Prefix output with a UTF8 byte-order mark + -e Send output to the system text editor + --plain Use text/plain for -w option + -w Send output to a web browser + -x Send output as CSV to a spreadsheet +child process exited abnormally}} # .prompt MAIN CONTINUE Replace the standard prompts do_test shell1-3.17.1 { @@ -616,20 +625,6 @@ CREATE VIEW v1 AS SELECT y+1 FROM v2 catch {db eval {DROP VIEW v1; DROP VIEW v2; DROP TABLE t1;}} } -do_test shell1-3.21.5 { - exec {*}$CLI -noinit test.db \ - {CREATE TABLE t2(a INTEGER PRIMARY KEY, b BLOB DEFAULT(jsonb('[]')),c TEXT NOT NULL)STRICT;} \ - {.schema -indent t2} -} {CREATE TABLE t2( - a INTEGER PRIMARY KEY, - b BLOB DEFAULT(jsonb('[]')), - c TEXT NOT NULL -)STRICT;} -do_test shell1-3.21.6 { - exec {*}$CLI -noinit test.db \ - {DROP TABLE t2;} \ - {.schema -indent t2} -} {} # .separator STRING Change column separator used by output and .import do_test shell1-3.22.1 { @@ -648,7 +643,7 @@ do_test shell1-3.22.4 { # .show Show the current values for various settings do_test shell1-3.23.1 { - set res [catchcmd "test.db" ".mode batch\n.show"] + set res [catchcmd "test.db" ".show"] list [regexp {echo:} $res] \ [regexp {explain:} $res] \ [regexp {headers:} $res] \ @@ -684,7 +679,7 @@ do_test shell1-3.23b.4 { # Adverse interaction between .stats and .eqp # do_test shell1-3.23b.5 { - catchcmd "test.db" [string map {"\n " "\n"} {.mode batch + catchcmd "test.db" [string map {"\n " "\n"} { CREATE TEMP TABLE t1(x); INSERT INTO t1 VALUES(1),(2); .stats on @@ -746,27 +741,30 @@ do_test shell1-3.26.5 { do_test shell1-3.26.6 { catchcmd "test.db" ".mode column\n.header off\n.width -10 10\nSELECT 'abcdefg', 123456;" # this should be treated the same as a '1' width for col 1 and 2 -} {0 { abcdefg 123456}} +} {0 { abcdefg 123456 }} # .timer ON|OFF Turn the CPU timer measurement on or off do_test shell1-3.27.1 { catchcmd "test.db" ".timer" -} {1 {Usage: .timer on|off|once}} -do_test shell1-3.27.2 { - catchcmd "test.db" ".timer ON" -} {0 {}} +} {1 {Usage: .timer on|off}} +ifcapable !winrt { + # No timer support on winrt. + do_test shell1-3.27.2 { + catchcmd "test.db" ".timer ON" + } {0 {}} +} do_test shell1-3.27.3 { catchcmd "test.db" ".timer OFF" } {0 {}} do_test shell1-3.27.4 { # too many arguments catchcmd "test.db" ".timer OFF BAD" -} {1 {Usage: .timer on|off|once}} +} {1 {Usage: .timer on|off}} -do_test shell1-3.28.1 { +do_test shell1-3-28.1 { catchcmd test.db \ - ".mode batch\n.log stdout\nSELECT coalesce(sqlite_log(123,'hello'),'456');" + ".log stdout\nSELECT coalesce(sqlite_log(123,'hello'),'456');" } "0 {(123) hello\n456}" do_test shell1-3-29.1 { @@ -805,14 +803,14 @@ INSERT INTO t1 VALUES(''); INSERT INTO t1 VALUES(1); INSERT INTO t1 VALUES(2.25); INSERT INTO t1 VALUES('hello'); -INSERT INTO t1 VALUES(x'807f'); +INSERT INTO t1 VALUES(X'807f'); CREATE TABLE t3(x,y); INSERT INTO t3 VALUES(1,NULL); INSERT INTO t3 VALUES(2,''); INSERT INTO t3 VALUES(3,1); INSERT INTO t3 VALUES(4,2.25); INSERT INTO t3 VALUES(5,'hello'); -INSERT INTO t3 VALUES(6,x'807f'); +INSERT INTO t3 VALUES(6,X'807f'); COMMIT;}} @@ -830,14 +828,14 @@ INSERT INTO t1(rowid,x) VALUES(2,''); INSERT INTO t1(rowid,x) VALUES(3,1); INSERT INTO t1(rowid,x) VALUES(4,2.25); INSERT INTO t1(rowid,x) VALUES(5,'hello'); -INSERT INTO t1(rowid,x) VALUES(6,x'807f'); +INSERT INTO t1(rowid,x) VALUES(6,X'807f'); CREATE TABLE t3(x,y); INSERT INTO t3(rowid,x,y) VALUES(1,1,NULL); INSERT INTO t3(rowid,x,y) VALUES(2,2,''); INSERT INTO t3(rowid,x,y) VALUES(3,3,1); INSERT INTO t3(rowid,x,y) VALUES(4,4,2.25); INSERT INTO t3(rowid,x,y) VALUES(5,5,'hello'); -INSERT INTO t3(rowid,x,y) VALUES(6,6,x'807f'); +INSERT INTO t3(rowid,x,y) VALUES(6,6,X'807f'); COMMIT;}} # If the table contains an INTEGER PRIMARY KEY, do not record a separate @@ -856,12 +854,12 @@ do_test shell1-4.1.2 { } {0 {PRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE t1(x INTEGER PRIMARY KEY, y); -INSERT INTO t1(x,y) VALUES(1,NULL); -INSERT INTO t1(x,y) VALUES(2,''); -INSERT INTO t1(x,y) VALUES(3,1); -INSERT INTO t1(x,y) VALUES(4,2.25); -INSERT INTO t1(x,y) VALUES(5,'hello'); -INSERT INTO t1(x,y) VALUES(6,x'807f'); +INSERT INTO t1 VALUES(1,NULL); +INSERT INTO t1 VALUES(2,''); +INSERT INTO t1 VALUES(3,1); +INSERT INTO t1 VALUES(4,2.25); +INSERT INTO t1 VALUES(5,'hello'); +INSERT INTO t1 VALUES(6,X'807f'); COMMIT;}} # Verify that the table named [table] is correctly quoted and that @@ -885,7 +883,7 @@ INSERT INTO "table"(rowid,x,y) VALUES(2,12,''); INSERT INTO "table"(rowid,x,y) VALUES(3,23,1); INSERT INTO "table"(rowid,x,y) VALUES(4,34,2.25); INSERT INTO "table"(rowid,x,y) VALUES(5,45,'hello'); -INSERT INTO "table"(rowid,x,y) VALUES(6,56,x'807f'); +INSERT INTO "table"(rowid,x,y) VALUES(6,56,X'807f'); COMMIT;}} # Do not record rowids for a WITHOUT ROWID table. Also check correct quoting @@ -904,12 +902,12 @@ do_test shell1-4.1.4 { } {0 {PRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE [ta<>ble](x INTEGER PRIMARY KEY, y) WITHOUT ROWID; -INSERT INTO "ta<>ble"(x,y) VALUES(1,NULL); -INSERT INTO "ta<>ble"(x,y) VALUES(12,''); -INSERT INTO "ta<>ble"(x,y) VALUES(23,1); -INSERT INTO "ta<>ble"(x,y) VALUES(34,2.25); -INSERT INTO "ta<>ble"(x,y) VALUES(45,'hello'); -INSERT INTO "ta<>ble"(x,y) VALUES(56,x'807f'); +INSERT INTO "ta<>ble" VALUES(1,NULL); +INSERT INTO "ta<>ble" VALUES(12,''); +INSERT INTO "ta<>ble" VALUES(23,1); +INSERT INTO "ta<>ble" VALUES(34,2.25); +INSERT INTO "ta<>ble" VALUES(45,'hello'); +INSERT INTO "ta<>ble" VALUES(56,X'807f'); COMMIT;}} # Do not record rowids if the rowid is inaccessible @@ -926,9 +924,9 @@ do_test shell1-4.1.5 { } {0 {PRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE t1(_ROWID_,rowid,oid); -INSERT INTO t1(_ROWID_,rowid,oid) VALUES(1,NULL,'alpha'); -INSERT INTO t1(_ROWID_,rowid,oid) VALUES(12,'',99); -INSERT INTO t1(_ROWID_,rowid,oid) VALUES(23,1,x'b0b1b2'); +INSERT INTO t1 VALUES(1,NULL,'alpha'); +INSERT INTO t1 VALUES(12,'',99); +INSERT INTO t1 VALUES(23,1,X'b0b1b2'); COMMIT;}} } else { @@ -943,7 +941,7 @@ do_test shell1-4.1.6 { (4,2.25), (5,'hello'), (6,x'807f'); } catchcmd test2.db {.dump --preserve-rowids} -} {/.* --preserve-rowids option is not compatible with SQLITE_OMIT_VIRTUALTABLE/} +} {1 {The --preserve-rowids option is not compatible with SQLITE_OMIT_VIRTUALTABLE}} } @@ -1022,7 +1020,7 @@ INSERT INTO t1 VALUES(''); INSERT INTO t1 VALUES(1); INSERT INTO t1 VALUES(2.25); INSERT INTO t1 VALUES('hello'); -INSERT INTO t1 VALUES(x'807f');}} +INSERT INTO t1 VALUES(X'807f');}} # Test the output of ".mode insert" with headers # @@ -1033,7 +1031,7 @@ INSERT INTO t1(x) VALUES(''); INSERT INTO t1(x) VALUES(1); INSERT INTO t1(x) VALUES(2.25); INSERT INTO t1(x) VALUES('hello'); -INSERT INTO t1(x) VALUES(x'807f');}} +INSERT INTO t1(x) VALUES(X'807f');}} # Test the output of ".mode insert" # @@ -1044,7 +1042,7 @@ INSERT INTO t3 VALUES(2,''); INSERT INTO t3 VALUES(3,1); INSERT INTO t3 VALUES(4,2.25); INSERT INTO t3 VALUES(5,'hello'); -INSERT INTO t3 VALUES(6,x'807f');}} +INSERT INTO t3 VALUES(6,X'807f');}} # Test the output of ".mode insert" with headers # @@ -1055,7 +1053,7 @@ INSERT INTO t3(x,y) VALUES(2,''); INSERT INTO t3(x,y) VALUES(3,1); INSERT INTO t3(x,y) VALUES(4,2.25); INSERT INTO t3(x,y) VALUES(5,'hello'); -INSERT INTO t3(x,y) VALUES(6,x'807f');}} +INSERT INTO t3(x,y) VALUES(6,X'807f');}} # Test the output of ".mode tcl" # @@ -1071,8 +1069,8 @@ do_test shell1-4.3 { catchcmd test.db ".mode tcl\nselect * from t1;" } {0 {"" "" -1 -2.25 +"1" +"2.25" "hello" "\200\177"}} @@ -1085,15 +1083,15 @@ do_test shell1-4.4 { } catchcmd test.db ".mode tcl\nselect * from t2;" } {0 {"" "" -1 2.25 +"1" "2.25" "hello" "\200\177"}} # Test the output of ".mode tcl" with ".nullvalue" # do_test shell1-4.5 { catchcmd test.db ".mode tcl\n.nullvalue NULL\nselect * from t2;" -} {0 {NULL "" -1 2.25 +} {0 {"NULL" "" +"1" "2.25" "hello" "\200\177"}} # Test the output of ".mode tcl" with Tcl reserved characters @@ -1117,11 +1115,10 @@ do_test shell1-4.6 { # do_test shell1-4.7 { catchcmd test.db ".mode quote\nselect x'0123456789ABCDEF';" -} {0 x'0123456789abcdef'} +} {0 X'0123456789abcdef'} # Test using arbitrary byte data with the shell via standard input/output. # -if 0 { # Causes a valgrind error in TCL. Seems to be a TCL problem. do_test shell1-5.0 { # # NOTE: Skip NUL byte because it appears to be incompatible with command @@ -1188,7 +1185,6 @@ do_test shell1-5.0 { } } } {} -} # These test cases do not work on MinGW if 0 { @@ -1278,7 +1274,7 @@ do_test shell1-7.1.7 { # information. # do_test shell1-8.1 { - catchcmd ":memory:" {.mode batch + catchcmd ":memory:" { -- The pow2 table will hold all the necessary powers of two. CREATE TABLE pow2(x INTEGER PRIMARY KEY, v TEXT); WITH RECURSIVE c(x,v) AS ( @@ -1304,20 +1300,20 @@ do_test_with_ansi_output shell1-8.2 { .mode box SELECT ieee754(47.49) AS x; } -} {0 {╭───────────────────────────────╮ +} {0 {┌───────────────────────────────┐ │ x │ -╞═══════════════════════════════╡ +├───────────────────────────────┤ │ ieee754(6683623321994527,-47) │ -╰───────────────────────────────╯}} +└───────────────────────────────┘}} do_test_with_ansi_output shell1-8.3 { catchcmd ":memory: --box" { select ieee754(6683623321994527,-47) as x; } -} {0 {╭───────╮ +} {0 {┌───────┐ │ x │ -╞═══════╡ +├───────┤ │ 47.49 │ -╰───────╯}} +└───────┘}} do_test shell1-8.4 { catchcmd ":memory: --table" {SELECT ieee754_mantissa(47.49) AS M, ieee754_exponent(47.49) AS E;} } {0 {+------------------+-----+ @@ -1325,42 +1321,35 @@ do_test shell1-8.4 { +------------------+-----+ | 6683623321994527 | -47 | +------------------+-----+}} -do_test shell1-8.4b { - catchcmd ":memory: --psql" \ - {SELECT ieee754_mantissa(47.49) AS M, ieee754_exponent(47.49) AS E;} -} {0 { M | E -------------------+----- - 6683623321994527 | -47}} do_test_with_ansi_output shell1-8.5 { catchcmd ":memory: --box" { create table t(a text, b int); insert into t values ('too long for one line', 1), ('shorter', NULL); .header on -.mode box --wordwrap off .width 10 10 .nullvalue NADA select * from t;} -} {0 {╭────────────┬────────────╮ +} {0 {┌────────────┬────────────┐ │ a │ b │ -╞════════════╪════════════╡ -│ too long f │ 1 │ +├────────────┼────────────┤ +│ too long f │ 1 │ │ or one lin │ │ │ e │ │ ├────────────┼────────────┤ │ shorter │ NADA │ -╰────────────┴────────────╯}} +└────────────┴────────────┘}} #---------------------------------------------------------------------------- # Test cases shell1-9.*: Basic test that "dot" commands and SQL intermix ok. # do_test shell1-9.1 { catchcmd :memory: { -.mode csv --rowsep "\n" +.mode csv /* x */ select 1,2; --x -- .nada ; -.mode csv --rowsep "\n" +.mode csv --x select 2,1; select 3,4; } @@ -1398,30 +1387,4 @@ select base85(zeroblob(2000000000)); } } {/1.*too big.*/} -#---------------------------------------------------------------------------- -# As of 2025-11-17, the default mode is: -# -# qbox --screenwidth auto --linelimit 5 --charlimit 300 --textjsonb on -# -do_test shell1-12.1 { - catchcmd :memory: {.mode tty -quote sql -.print -SELECT jsonb(1234) AS x;} -} {0 { -╭───────────────╮ -│ x │ -╞═══════════════╡ -│ jsonb('1234') │ -╰───────────────╯}} -do_test shell1-12.2 { - catchcmd :memory: {.mode box --textjsonb on -.print -SELECT jsonb(1234) AS x;} -} {0 { -╭──────╮ -│ x │ -╞══════╡ -│ 1234 │ -╰──────╯}} - finish_test diff --git a/test/shell2.test b/test/shell2.test index 7141c4d49..5f700a9a1 100644 --- a/test/shell2.test +++ b/test/shell2.test @@ -44,7 +44,7 @@ do_test shell2-1.1.1 { # Shell silently ignores extra parameters. # Ticket [f5cb008a65]. do_test shell2-1.2.1 { - catchcmdex {:memory: -list "select+3" "select+4"} + catchcmdex {:memory: "select+3" "select+4"} } {0 {3 4 }} @@ -64,7 +64,7 @@ do_test shell2-1.3 { UPDATE OR REPLACE t5 SET a = 4 WHERE a = 1; } -} {1 {Error near line 9: too many levels of trigger recursion}} +} {1 {Runtime error near line 9: too many levels of trigger recursion}} @@ -75,8 +75,7 @@ do_test shell2-1.3 { # NB. whitespace is important do_test shell2-1.4.1 { forcedelete foo.db - catchcmd "foo.db" {.mode batch -CREATE TABLE foo(a); + catchcmd "foo.db" {CREATE TABLE foo(a); INSERT INTO foo(a) VALUES(1); SELECT * FROM foo;} } {0 1} @@ -97,9 +96,7 @@ SELECT * FROM foo; # NB. whitespace is important do_test shell2-1.4.3 { forcedelete foo.db - catchcmd "foo.db" { -.mode batch -.echo ON + catchcmd "foo.db" {.echo ON CREATE TABLE foo(a); INSERT INTO foo(a) VALUES(1); SELECT * FROM foo;} @@ -113,9 +110,7 @@ SELECT * FROM foo; # NB. whitespace is important do_test shell2-1.4.4 { forcedelete foo.db - catchcmd "foo.db" { -.mode batch -.echo ON + catchcmd "foo.db" {.echo ON CREATE TABLE foo(a); .echo OFF INSERT INTO foo(a) VALUES(1); @@ -129,9 +124,7 @@ SELECT * FROM foo;} # NB. whitespace is important do_test shell2-1.4.5 { forcedelete foo.db - catchcmdex "foo.db" { -.mode batch -.echo ON + catchcmdex "foo.db" {.echo ON CREATE TABLE foo1(a); INSERT INTO foo1(a) VALUES(1); CREATE TABLE foo2(b); @@ -160,9 +153,7 @@ SELECT * FROM foo1; SELECT * FROM foo2; # NB. whitespace is important do_test shell2-1.4.6 { forcedelete foo.db - catchcmdex "foo.db" { -.mode batch -.echo ON + catchcmdex "foo.db" {.echo ON .headers ON CREATE TABLE foo1(a); INSERT INTO foo1(a) VALUES(1); @@ -217,7 +208,6 @@ do_test shell2-1.4.9 { do_test shell2-1.4.9 { forcedelete clone.db set res [catchcmd :memory: [string trim { -.mode batch CREATE TABLE t(id INTEGER PRIMARY KEY AUTOINCREMENT); INSERT INTO t VALUES (1),(2); .clone clone.db @@ -232,7 +222,6 @@ ifcapable vtab { # See overflow report at https://sqlite.org/forum/forumpost/5d34ce5280 do_test shell2-1.4.10 { set res [catchcmd :memory: [string trim { - .mode batch SELECT * FROM generate_series(9223372036854775807,9223372036854775807,1); SELECT * FROM generate_series(9223372036854775807,9223372036854775807,-1); SELECT avg(value),min(value),max(value) FROM generate_series( @@ -259,65 +248,6 @@ do_test shell2-1.4.10 { 0 1 2}} -do_test shell2-1.4.10b { - set res [catchcmd :memory: [string trim { - .mode tty -.print - SELECT * FROM generate_series(9223372036854775807,9223372036854775807,1); - SELECT * FROM generate_series(9223372036854775807,9223372036854775807,-1); - SELECT avg(value),min(value),max(value) FROM generate_series( - -9223372036854775808,9223372036854775807,1085102592571150095); - SELECT * FROM generate_series(-9223372036854775808,9223372036854775807, - 9223372036854775807); - SELECT value FROM generate_series(-4611686018427387904, - 4611686018427387904, 4611686018427387904) ORDER BY value DESC; - SELECT * FROM generate_series(0,-2,-1); - SELECT * FROM generate_series(0,-2); - SELECT * FROM generate_series(0,2) LIMIT 3;}]] -} {0 { -╭─────────────────────╮ -│ value │ -╞═════════════════════╡ -│ 9223372036854775807 │ -╰─────────────────────╯ -╭─────────────────────╮ -│ value │ -╞═════════════════════╡ -│ 9223372036854775807 │ -╰─────────────────────╯ -╭────────────┬──────────────────────┬─────────────────────╮ -│ avg(value) │ min(value) │ max(value) │ -╞════════════╪══════════════════════╪═════════════════════╡ -│ -0.5 │ -9223372036854775808 │ 9223372036854775807 │ -╰────────────┴──────────────────────┴─────────────────────╯ -╭──────────────────────╮ -│ value │ -╞══════════════════════╡ -│ -9223372036854775808 │ -│ -1 │ -│ 9223372036854775806 │ -╰──────────────────────╯ -╭──────────────────────╮ -│ value │ -╞══════════════════════╡ -│ 4611686018427387904 │ -│ 0 │ -│ -4611686018427387904 │ -╰──────────────────────╯ -╭───────╮ -│ value │ -╞═══════╡ -│ 0 │ -│ -1 │ -│ -2 │ -╰───────╯ -╭───────╮ -│ value │ -╞═══════╡ -│ 0 │ -│ 1 │ -│ 2 │ -╰───────╯}} } ;# ifcapable vtab ifcapable vtab { @@ -330,17 +260,17 @@ do_test shell2-1.4.11 { close $df set res [catchcmd :memory: [string trim { CREATE TABLE t(line text); -.mode ascii -colsep "\377" -rowsep "\n" +.mode ascii +.separator "\377" "\n" .import dummy.csv t SELECT count(*) FROM t;}]] -} {1 {0 -Error: .import column separator must be ASCII}} +} {0 1} } ;# ifcapable vtab # Bug from forum post 7cbe081746dd3803 # Keywords as column names were producing an error message. do_test shell2-1.4.12 { - set res [catchcmd :memory: [string trim {.mode batch + set res [catchcmd :memory: [string trim { CREATE TABLE "group"("order" text); INSERT INTO "group" VALUES ('ABC'); .sha3sum}]] diff --git a/test/shell4.test b/test/shell4.test index 3614909c7..3ced0702e 100644 --- a/test/shell4.test +++ b/test/shell4.test @@ -136,15 +136,15 @@ SELECT * FROM t1;} do_test shell4-3.1 { set fd [open t1.txt wb] - puts $fd ".mode list\nSELECT 'squirrel';" + puts $fd "SELECT 'squirrel';" close $fd - exec $::CLI_ONLY --noinit :memory: --interactive ".read t1.txt" + exec $::CLI_ONLY :memory: --interactive ".read t1.txt" } {squirrel} do_test_with_ansi_output shell4-3.2 { set fd [open t1.txt wb] - puts $fd ".mode list\nSELECT 'pound: \302\243';" + puts $fd "SELECT 'pound: \302\243';" close $fd - exec $::CLI_ONLY --noinit :memory: --interactive ".read t1.txt" + exec $::CLI_ONLY :memory: --interactive ".read t1.txt" } {pound: £} do_test shell4-4.1 { diff --git a/test/shell5.test b/test/shell5.test index 559dc3ce7..70a2298bc 100644 --- a/test/shell5.test +++ b/test/shell5.test @@ -33,15 +33,14 @@ forcedelete test.db test.db-journal test.db-wal # .import FILE TABLE Import data from FILE into TABLE do_test shell5-1.1.1 { catchcmd "test.db" ".import" -} {/1 .line 1: Missing FILE argument.*/} +} {/1 .ERROR: missing FILE argument.*/} do_test shell5-1.1.2 { catchcmd "test.db" ".import FOO" -} {/1 .line 1: Missing TABLE argument.*/} +} {/1 .ERROR: missing TABLE argument.*/} do_test shell5-1.1.3 { # too many arguments catchcmd "test.db" ".import FOO BAR BAD" -} {1 {line 1: .import FOO BAR BAD -line 1: ^--- unknown argument}} +} {/1 .ERROR: extra argument.*/} # .separator STRING Change separator used by output mode and .import do_test shell5-1.2.1 { @@ -60,13 +59,13 @@ do_test shell5-1.2.4 { # column separator should default to "|" do_test shell5-1.3.1.1 { - set res [catchcmd "test.db" ".mode list\n.show"] + set res [catchcmd "test.db" ".show"] list [regexp {colseparator: \"\|\"} $res] } {1} # row separator should default to "\n" do_test shell5-1.3.1.2 { - set res [catchcmd "test.db" ".mode list\n.show"] + set res [catchcmd "test.db" ".show"] list [regexp {rowseparator: \"\\n\"} $res] } {1} @@ -83,7 +82,7 @@ do_test shell5-1.4.1 { forcedelete FOO set res [catchcmd "test.db" {CREATE TABLE t1(a, b); .import FOO t1}] -} {1 {line 2: cannot open "FOO"}} +} {1 {Error: cannot open "FOO"}} # the remainder of these test cases require virtual tables. # @@ -98,9 +97,7 @@ do_test shell5-1.4.2 { forcedelete shell5.csv set in [open shell5.csv w] close $in - set res [catchcmd ":memory:" { -.mode list -ATTACH 'test.db' AS test; + set res [catchcmd ":memory:" {ATTACH 'test.db' AS test; .import -schema test shell5.csv t1 SELECT COUNT(*) FROM test.t1;}] } {0 0} @@ -119,7 +116,7 @@ do_test shell5-1.4.4 { set in [open shell5.csv w] puts $in "1|2|3" close $in - set res [catchcmd ":memory: --list" {ATTACH 'test.db' AS test; + set res [catchcmd ":memory:" {ATTACH 'test.db' AS test; .import --schema test shell5.csv t1}] } {1 {shell5.csv:1: expected 2 columns but found 3 - extras ignored}} @@ -128,7 +125,7 @@ do_test shell5-1.4.5 { set in [open shell5.csv w] puts $in "1|2" close $in - set res [catchcmd "test.db -list" {DELETE FROM t1; + set res [catchcmd "test.db" {DELETE FROM t1; .import shell5.csv t1 SELECT COUNT(*) FROM t1;}] } {0 1} @@ -141,9 +138,7 @@ do_test shell5-1.4.6 { puts $in "2|3" puts $in "3|4" close $in - set res [catchcmd ":memory:" { -.mode list -ATTACH 'test.db' AS test; + set res [catchcmd ":memory:" {ATTACH 'test.db' AS test; .import -schema test shell5.csv t1 SELECT COUNT(*) FROM test.t1;}] } {0 3} @@ -153,9 +148,7 @@ do_test shell5-1.4.7 { set in [open shell5.csv w] puts $in "4,5" close $in - set res [catchcmd ":memory:" { -.mode list -ATTACH 'test.db' AS test; + set res [catchcmd ":memory:" {ATTACH 'test.db' AS test; .separator , .import --schema test shell5.csv t1 SELECT COUNT(*) FROM test.t1;}] @@ -166,13 +159,12 @@ do_test shell5-1.4.8.1 { set in [open shell5.csv w] puts $in "5|Now is the time for all good men to come to the aid of their country." close $in - set res [catchcmd "test.db" {.mode list -.import shell5.csv t1 + set res [catchcmd "test.db" {.import shell5.csv t1 SELECT COUNT(*) FROM t1;}] } {0 5} do_test shell5-1.4.8.2 { - catchcmd "test.db -list" {SELECT b FROM t1 WHERE a='5';} + catchcmd "test.db" {SELECT b FROM t1 WHERE a='5';} } {0 {Now is the time for all good men to come to the aid of their country.}} # import file with 1 row, 2 columns, quoted text data @@ -182,12 +174,12 @@ do_test shell5-1.4.9.1 { set in [open shell5.csv w] puts $in "6|'Now is the time for all good men to come to the aid of their country.'" close $in - set res [catchcmd "test.db -list" {.import shell5.csv t1 + set res [catchcmd "test.db" {.import shell5.csv t1 SELECT COUNT(*) FROM t1;}] } {0 6} do_test shell5-1.4.9.2 { - catchcmd "test.db -list" {SELECT b FROM t1 WHERE a='6';} + catchcmd "test.db" {SELECT b FROM t1 WHERE a='6';} } {0 {'Now is the time for all good men to come to the aid of their country.'}} # import file with 1 row, 2 columns, quoted text data @@ -195,12 +187,12 @@ do_test shell5-1.4.10.1 { set in [open shell5.csv w] puts $in "7|\"Now is the time for all good men to come to the aid of their country.\"" close $in - set res [catchcmd "test.db -list" {.import shell5.csv t1 + set res [catchcmd "test.db" {.import shell5.csv t1 SELECT COUNT(*) FROM t1;}] } {0 7} do_test shell5-1.4.10.2 { - catchcmd "test.db -list" {SELECT b FROM t1 WHERE a='7';} + catchcmd "test.db" {SELECT b FROM t1 WHERE a='7';} } {0 {Now is the time for all good men to come to the aid of their country.}} # import file with 2 rows, 2 columns and an initial BOM @@ -211,7 +203,7 @@ do_test shell5-1.4.11 { puts $in "2|3" puts $in "4|5" close $in - set res [catchcmd "test.db -list" {CREATE TABLE t2(x INT, y INT); + set res [catchcmd "test.db" {CREATE TABLE t2(x INT, y INT); .import shell5.csv t2 .mode quote .header on @@ -226,7 +218,7 @@ do_test shell5-1.4.12 { puts $in "\xef\xbb\xbf\"two\"|3" puts $in "4|5" close $in - set res [catchcmd "test.db -list" {DELETE FROM t2; + set res [catchcmd "test.db" {DELETE FROM t2; .import shell5.csv t2 .mode quote .header on @@ -240,7 +232,7 @@ do_test shell5-1.5.1 { set in [open shell5.csv w] puts $in "8|$str" close $in - set res [catchcmd "test.db -list" {.import shell5.csv t1 + set res [catchcmd "test.db" {.import shell5.csv t1 SELECT length(b) FROM t1 WHERE a='8';}] } {0 999} @@ -260,7 +252,7 @@ do_test shell5-1.6.1 { set in [open shell5.csv w] puts $in $data close $in - set res [catchcmd "test.db -list" {DROP TABLE IF EXISTS t2; + set res [catchcmd "test.db" {DROP TABLE IF EXISTS t2; .import shell5.csv t2 SELECT COUNT(*) FROM t2;}] } {0 1} @@ -276,7 +268,6 @@ do_test shell5-1.7.1 { close $in set res [catchcmd "test.db" {.mode csv .import shell5.csv t3 -.mode quote SELECT COUNT(*) FROM t3;}] } [list 0 $rows] @@ -338,24 +329,6 @@ do_test shell5-1.10 { db eval {SELECT hex(c) FROM t1 ORDER BY rowid} } {636F6C756D6E33 783320220D0A64617461222033 783320220A64617461222033} -# The --escape option -# -do_test shell5-1.10.1 { - set out [open shell5.csv w] - fconfigure $out -translation lf - puts $out {column1,column2,column3,column4} - puts $out "x1,x2%\"x3,\"x3\\\"data\\\"3\",x4" - close $out - db close - forcedelete test.db - catchcmd test.db { - CREATE TABLE t1(a,b,c,d); -.import --csv --qesc \\ --esc % shell5.csv t1 - } - sqlite3 db test.db - db eval {SELECT b, c FROM t1 ORDER BY rowid} -} {column2 column3 x2\"x3 x3\"data\"3} - # Blank last column with \r\n line endings. do_test shell5-1.11 { set out [open shell5.csv w] @@ -522,10 +495,9 @@ do_test shell5-4.4 { CREATE TEMP TABLE t8(a, b, c); .import shell5.csv t8 .nullvalue # -.mode quote SELECT * FROM temp.t8 }] -} {0 '1','2','3'} +} {0 1,2,3} #---------------------------------------------------------------------------- # Tests for the shell automatic column rename. @@ -541,7 +513,7 @@ do_test shell5-5.1 { close $out forcedelete test.db catchcmd test.db {.import -csv shell5.csv t1 -.mode line --colsep ' = ' +.mode line SELECT * FROM t1;} } {1 { ? = 0 x_02 = x2 @@ -557,7 +529,7 @@ Columns renamed during .import shell5.csv due to duplicates: "z" to "z_05", "z" to "z_08"}} -do_test shell5-5.1b { +do_test shell5-5.1 { set out [open shell5.csv w] fconfigure $out -translation lf puts $out {"COW","cow","CoW","cOw"} @@ -567,10 +539,10 @@ do_test shell5-5.1b { catchcmd test.db {.import -csv shell5.csv t1 .mode line SELECT * FROM t1;} -} {1 {COW_1: uuu -cow_2: lll -CoW_3: ulu -cOw_4: lul +} {1 {COW_1 = uuu +cow_2 = lll +CoW_3 = ulu +cOw_4 = lul Columns renamed during .import shell5.csv due to duplicates: "COW" to "COW_1", "cow" to "cow_2", @@ -589,7 +561,7 @@ do_test_with_ansi_output shell5-6.1 { close $out forcedelete test.db catchcmd test.db {.import -csv shell5.csv t1 -.mode line --colsep " = " +.mode line SELECT * FROM t1;} } {0 { あい = 1 うえお = 2}} @@ -604,8 +576,8 @@ do_test_with_ansi_output shell5-6.2 { catchcmd test.db {.import -csv shell5.csv t1 .mode line SELECT * FROM t1;} -} {0 {1: あい -2: うえお}} +} {0 { 1 = あい + 2 = うえお}} # 2024-03-11 https://sqlite.org/forum/forumpost/ca014d7358 # Import into a table that contains computed columns. @@ -616,7 +588,7 @@ do_test shell5-7.1 { puts $out {aaa|bbb} close $out forcedelete test.db - catchcmd ":memory: -list" {CREATE TABLE t1(a TEXT, b TEXT, c AS (a||b)); + catchcmd :memory: {CREATE TABLE t1(a TEXT, b TEXT, c AS (a||b)); .import shell5.csv t1 SELECT * FROM t1;} } {0 aaa|bbb|aaabbb} @@ -633,36 +605,4 @@ do_test shell5-8.1 { catchcmd :memory: {.import --csv shell5.csv '""""""""""""""""""""""""""""""""""""""""""""""'} } {0 {}} - -# 2025-12-29 https://sqlite.org/forum/forumpost/6c1c0e213d -# .import honor .bail -# -do_test shell5-9.1 { - catchcmd ":memory:" { - CREATE TABLE t1(a,b,c INT CHECK(c<>5)); -.bail on -.import -csv <<END t1 -1,2,3 -"a","b","c" -3,4,5 -"q","r","s" -END -SELECT * FROM t1;} -} {1 {<stdin>:7: INSERT failed: CHECK constraint failed: c<>5}} -do_test shell5-9.2 { - catchcmd ":memory:" { - CREATE TABLE t1(a,b,c INT CHECK(c<>5)); -.bail off -.import -csv <<END t1 -1,2,3 -"a","b","c" -3,4,5 -"q","r","s" -END -SELECT * FROM t1;} -} {1 {1|2|3 -a|b|c -q|r|s -<stdin>:7: INSERT failed: CHECK constraint failed: c<>5}} - finish_test diff --git a/test/shell8.test b/test/shell8.test index 40579a599..e55539636 100644 --- a/test/shell8.test +++ b/test/shell8.test @@ -217,46 +217,6 @@ if {$tcl_platform(platform)=="unix"} { do_test 3.3 { catchcmd shell8.db {.ar -x} } {0 {}} - - # Test defenses against using symlinks to write outside - # of the destination directory. See forum thread at - # sqlite.org/forum/forumpost/2026-02-21T11:04:36z - # - forcedelete shell8.db - forcedelete ar1 - forcedelete ar2 - forcedelete ar3 - file mkdir ar2 - file mkdir ar3 - set pwd [pwd] - sqlite3 db shell8.db - db eval { - CREATE TABLE sqlar( - name TEXT PRIMARY KEY, -- name of the file - mode INT, -- access permissions - mtime INT, -- last modification time - sz INT, -- original file size - data BLOB -- compressed content - ); - INSERT INTO sqlar VALUES - ('abc',33188,0,-1,'content for abc'), - ('escape',40960,0,-1,$pwd||'/ar3'), - ('escape/def',33188,0,-1,'content for escape/def'), - ('ghi',33188,0,-1,'content for ghi'); - } - do_test 3.4.1 { - catchcmd shell8.db {.ar -x --directory ar2} - lsort [glob -tails -directory ar2 -nocomplain *] - } {abc escape ghi} - do_test 3.4.2 { - lsort [glob -tails -directory ar3 -nocomplain *] - } {} - # ^^--- An extraction into ar2 should not leak any files into ar3 - - forcedelete shell8.db - forcedelete ar2 - forcedelete ar3 - } finish_test diff --git a/test/shellA.test b/test/shellA.test index 3b28c921c..f3959d428 100644 --- a/test/shellA.test +++ b/test/shellA.test @@ -36,11 +36,11 @@ do_execsql_test shellA-1.0 { # and that our calls to the CLI are working. # do_test_with_ansi_output shellA-1.2 { - exec {*}$CLI -noinit test.db {.mode box -quote off --escape symbol} {SELECT * FROM t1;} + exec {*}$CLI test.db {.mode box --escape symbol} {SELECT * FROM t1;} } { -╭───┬──────────────────────────╮ +┌───┬──────────────────────────┐ │ a │ x │ -╞═══╪══════════════════════════╡ +├───┼──────────────────────────┤ │ 1 │ line with ' single quote │ ├───┼──────────────────────────┤ │ 2 │ ␛[31mVT-100 codes␛[0m │ @@ -57,39 +57,39 @@ do_test_with_ansi_output shellA-1.2 { │ 7 │ carriage␍return │ ├───┼──────────────────────────┤ │ 8 │ last line │ -╰───┴──────────────────────────╯ +└───┴──────────────────────────┘ } # ".mode list" # do_test shellA-1.3 { - exec {*}$CLI -noinit -list test.db {SELECT x FROM t1 WHERE a=2;} + exec {*}$CLI test.db {SELECT x FROM t1 WHERE a=2;} } { ^[[31mVT-100 codes^[[0m } do_test_with_ansi_output shellA-1.4 { - exec {*}$CLI -noinit -list test.db --escape symbol {SELECT x FROM t1 WHERE a=2;} + exec {*}$CLI test.db --escape symbol {SELECT x FROM t1 WHERE a=2;} } { ␛[31mVT-100 codes␛[0m } do_test shellA-1.5 { - exec {*}$CLI -noinit -list test.db --escape ascii {SELECT x FROM t1 WHERE a=2;} + exec {*}$CLI test.db --escape ascii {SELECT x FROM t1 WHERE a=2;} } { ^[[31mVT-100 codes^[[0m } do_test_with_ansi_output shellA-1.6 { - exec {*}$CLI -noinit test.db {.mode list --escape symbol} {SELECT x FROM t1 WHERE a=2;} + exec {*}$CLI test.db {.mode list --escape symbol} {SELECT x FROM t1 WHERE a=2;} } { ␛[31mVT-100 codes␛[0m } do_test shellA-1.7 { - exec {*}$CLI -noinit test.db {.mode list --escape ascii} {SELECT x FROM t1 WHERE a=2;} + exec {*}$CLI test.db {.mode list --escape ascii} {SELECT x FROM t1 WHERE a=2;} } { ^[[31mVT-100 codes^[[0m } do_test shellA-1.8 { file delete -force out.txt - exec {*}$CLI -noinit test.db {.mode list --escape off} {SELECT x FROM t1 WHERE a=7;} \ + exec {*}$CLI test.db {.mode list --escape off} {SELECT x FROM t1 WHERE a=7;} \ >out.txt set fd [open out.txt rb] set res [read $fd] @@ -98,93 +98,92 @@ do_test shellA-1.8 { } "carriage\rreturn" do_test shellA-1.9 { set rc [catch { - exec {*}$CLI -noinit test.db {.mode test --escape xyz} + exec {*}$CLI test.db {.mode test --escape xyz} } msg] lappend rc $msg -} {1 {argv[3]: .mode test --escape xyz -argv[3]: ^--- unknown mode -argv[3]: Use ".help .mode" for more info}} +} {1 {unknown control character escape mode "xyz" - choices: ascii symbol off}} do_test shellA-1.10 { set rc [catch { - exec {*}$CLI --noinit --escape abc test.db .q + exec {*}$CLI --escape abc test.db .q } msg] lappend rc $msg -} {1 {unknown control character escape mode "abc" - choices: auto off ascii symbol}} +} {1 {unknown control character escape mode "abc" - choices: ascii symbol off}} # ".mode quote" # do_test shellA-2.1 { - exec {*}$CLI -noinit test.db --quote {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} + exec {*}$CLI test.db --quote {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} } { 1,'line with '' single quote' 2,unistr('\u001b[31mVT-100 codes\u001b[0m') -6,unistr('new\u000aline') +6,'new +line' 7,unistr('carriage\u000dreturn') 8,'last line' } do_test shellA-2.2 { - exec {*}$CLI -noinit test.db --quote {.mode -v} -} {/*.mode quote* --escape auto*/} + exec {*}$CLI test.db --quote {.mode} +} {current output mode: quote --escape ascii} do_test shellA-2.3 { - exec {*}$CLI -noinit test.db --quote --escape SYMBOL {.mode} -} {.mode quote --escape symbol} + exec {*}$CLI test.db --quote --escape SYMBOL {.mode} +} {current output mode: quote --escape symbol} do_test shellA-2.4 { - exec {*}$CLI -noinit test.db --quote --escape OFF {.mode} -} {.mode quote --escape off} + exec {*}$CLI test.db --quote --escape OFF {.mode} +} {current output mode: quote --escape off} # ".mode line" # do_test_with_ansi_output shellA-3.1 { - exec {*}$CLI -noinit test.db --line --escape symbol \ + exec {*}$CLI test.db --line --escape symbol \ {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} } { - a: 1 - x: line with ' single quote + a = 1 + x = line with ' single quote - a: 2 - x: ␛[31mVT-100 codes␛[0m + a = 2 + x = ␛[31mVT-100 codes␛[0m - a: 6 - x: new - line + a = 6 + x = new +line - a: 7 - x: carriage␍return + a = 7 + x = carriage␍return - a: 8 - x: last line + a = 8 + x = last line } do_test shellA-3.2 { - exec {*}$CLI -noinit test.db --line --escape ascii \ + exec {*}$CLI test.db --line --escape ascii \ {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} } { - a: 1 - x: line with ' single quote + a = 1 + x = line with ' single quote - a: 2 - x: ^[[31mVT-100 codes^[[0m + a = 2 + x = ^[[31mVT-100 codes^[[0m - a: 6 - x: new - line + a = 6 + x = new +line - a: 7 - x: carriage^Mreturn + a = 7 + x = carriage^Mreturn - a: 8 - x: last line + a = 8 + x = last line } # ".mode box" # do_test_with_ansi_output shellA-4.1 { - exec {*}$CLI -noinit test.db --box --escape ascii \ + exec {*}$CLI test.db --box --escape ascii \ {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} } { -╭───┬──────────────────────────╮ +┌───┬──────────────────────────┐ │ a │ x │ -╞═══╪══════════════════════════╡ +├───┼──────────────────────────┤ │ 1 │ line with ' single quote │ ├───┼──────────────────────────┤ │ 2 │ ^[[31mVT-100 codes^[[0m │ @@ -195,56 +194,31 @@ do_test_with_ansi_output shellA-4.1 { │ 7 │ carriage^Mreturn │ ├───┼──────────────────────────┤ │ 8 │ last line │ -╰───┴──────────────────────────╯ -} -do_test_with_ansi_output shellA-4.1b { - exec {*}$CLI -noinit test.db --box --escape ascii \ - {.mode -border off} \ - {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} -} { - a │ x -═══╪══════════════════════════ - 1 │ line with ' single quote -───┼────────────────────────── - 2 │ ^[[31mVT-100 codes^[[0m -───┼────────────────────────── - 6 │ new - │ line -───┼────────────────────────── - 7 │ carriage^Mreturn -───┼────────────────────────── - 8 │ last line +└───┴──────────────────────────┘ } do_test_with_ansi_output shellA-4.2 { - exec {*}$CLI -noinit test.db {.mode qbox} {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} + exec {*}$CLI test.db {.mode qbox} {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} } { -╭───┬───────────────────────────────────────────╮ +┌───┬───────────────────────────────────────────┐ │ a │ x │ -╞═══╪═══════════════════════════════════════════╡ +├───┼───────────────────────────────────────────┤ │ 1 │ 'line with '' single quote' │ +├───┼───────────────────────────────────────────┤ │ 2 │ unistr('\u001b[31mVT-100 codes\u001b[0m') │ -│ 6 │ unistr('new\u000aline') │ +├───┼───────────────────────────────────────────┤ +│ 6 │ 'new │ +│ │ line' │ +├───┼───────────────────────────────────────────┤ │ 7 │ unistr('carriage\u000dreturn') │ +├───┼───────────────────────────────────────────┤ │ 8 │ 'last line' │ -╰───┴───────────────────────────────────────────╯ -} -do_test_with_ansi_output shellA-4.2b { - exec {*}$CLI -noinit test.db {.mode qbox -border off} \ - {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} -} { - a │ x -═══╪═══════════════════════════════════════════ - 1 │ 'line with '' single quote' - 2 │ unistr('\u001b[31mVT-100 codes\u001b[0m') - 6 │ unistr('new\u000aline') - 7 │ unistr('carriage\u000dreturn') - 8 │ 'last line' +└───┴───────────────────────────────────────────┘ } # ".mode insert" # do_test shellA-5.1 { - exec {*}$CLI -noinit test.db {.mode insert t1 --escape ascii} \ + exec {*}$CLI test.db {.mode insert t1 --escape ascii} \ {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} } { INSERT INTO t1 VALUES(1,'line with '' single quote'); @@ -254,7 +228,7 @@ INSERT INTO t1 VALUES(7,unistr('carriage\u000dreturn')); INSERT INTO t1 VALUES(8,'last line'); } do_test shellA-5.2 { - exec {*}$CLI -noinit test.db {.mode insert t1 --escape symbol} \ + exec {*}$CLI test.db {.mode insert t1 --escape symbol} \ {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} } { INSERT INTO t1 VALUES(1,'line with '' single quote'); @@ -265,7 +239,7 @@ INSERT INTO t1 VALUES(8,'last line'); } do_test shellA-5.3 { file delete -force out.txt - exec {*}$CLI -noinit test.db {.mode insert t1 --escape off} \ + exec {*}$CLI test.db {.mode insert t1 --escape off} \ {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} >out.txt set fd [open out.txt rb] set res [read $fd] @@ -280,83 +254,4 @@ INSERT INTO t1 VALUES(7,'carriage\rreturn'); INSERT INTO t1 VALUES(8,'last line'); " -# ".mode split" -# -do_test shellA-6.1 { - db eval { - CREATE TABLE t2(x); - INSERT INTO t2(x) VALUES - ('one'), ('two'), ('three'), ('four'), ('five'), - ('six'), ('seven'), ('eight'), ('nine'), ('ten'), - ('eleven'), ('twelve'), ('thirteen'), ('fourteen'); - } - exec {*}$CLI -noinit test.db \ - {.print} \ - {.mode split -screenwidth 30} \ - {SELECT x FROM t2} -} { -one five nine thirteen -two six ten fourteen -three seven eleven -four eight twelve} -# 3456789 123456789 123456789 - -do_test shellA-6.2 { - exec {*}$CLI -noinit test.db \ - {.print} \ - {.mode split -screenwidth 30} \ - {SELECT x FROM t2} \ - {.mode column -titles off} \ - {SELECT x FROM t2} -} { -one five nine thirteen -two six ten fourteen -three seven eleven -four eight twelve -one -two -three -four -five -six -seven -eight -nine -ten -eleven -twelve -thirteen -fourteen} - -do_test shellA-6.3 { - exec {*}$CLI -noinit test.db \ - {.print} \ - {.mode table} \ - {.mode --once split -screenwidth 30} \ - {SELECT x FROM t2} \ - {SELECT x FROM t2} -} { -one five nine thirteen -two six ten fourteen -three seven eleven -four eight twelve -+----------+ -| x | -+----------+ -| one | -| two | -| three | -| four | -| five | -| six | -| seven | -| eight | -| nine | -| ten | -| eleven | -| twelve | -| thirteen | -| fourteen | -+----------+} - finish_test diff --git a/test/shellB.test b/test/shellB.test deleted file mode 100644 index d98a77cb5..000000000 --- a/test/shellB.test +++ /dev/null @@ -1,53 +0,0 @@ -# 2025-11-12 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# TESTRUNNER: shell -# -# Test cases for the command-line shell using the newly renovated -# ".testcase" and ".check" commands. -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl -set CLI [test_cli_invocation] - -# Run an instance of the CLI on the file $name. -# Capture the number of test cases and the number of -# errors and increment the counts. -# -proc do_clitest {name} { - set mapping [list <NAME> $::testdir/$name <CLI> $::CLI] - set script [string map $mapping { - catch {exec <CLI> :memory: ".read <NAME>" 2>@stdout} res - set ntest 0 - set nerr 999 - regexp {(\d+) tests? run with (\d+) errors?} $res all ntest nerr - set_test_counter count [expr {[set_test_counter count]+$ntest-1}] - set_test_counter errors [expr {[set_test_counter errors]+$nerr}] - if {$nerr==0} {set res "error count: 0"} - set res - }] - # puts $script - do_test shellB-$name $script {error count: 0} -} - -do_clitest modeA.sql -do_clitest dblwidth-a.sql -do_clitest vt100-a.sql -do_clitest regexp1.sql -do_clitest imposter1.sql -do_clitest dotcmd01.sql -ifcapable vtab { - do_clitest import01.sql - do_clitest intck01.sql -} -do_clitest fptest01.sql - -finish_test diff --git a/test/speedtest.md b/test/speedtest.md index 98cba93ff..135e562ae 100644 --- a/test/speedtest.md +++ b/test/speedtest.md @@ -8,9 +8,8 @@ You will need: * valgrind * tclsh * A script or program named "open" that brings up *.txt files in an - editor for viewing. (Macs provide this by default. On Linux it's - called xdg-open and some distributions symlink it to "open". You'll - need to come up with your own on Windows.) + editor for viewing. (Macs provide this by default. You'll need to + come up with your own on Linux and Windows.) * An SQLite source tree The procedure described in this document is not the only way to make diff --git a/test/speedtest.tcl b/test/speedtest.tcl index 26bc27530..9cb81c0fc 100755 --- a/test/speedtest.tcl +++ b/test/speedtest.tcl @@ -27,7 +27,7 @@ Other options include: --lookaside N SZ Lookahead uses N slots of SZ bytes each. --osmalloc Use the OS native malloc() instead of MEMSYS5 --pagesize N Use N as the page size. - --quiet | -q "Quiet". Put results in file but don't pop up editor + --quiet | -q "Quite". Put results in file but don't pop up editor --size N Change the test size. 100 means 100%. Default: 5. --testset TEST Specify the specific testset to use. The default is "mix1". Other options include: "main", "json", diff --git a/test/tabfunc01.test b/test/tabfunc01.test index 60f546ce4..9a2017c46 100644 --- a/test/tabfunc01.test +++ b/test/tabfunc01.test @@ -668,20 +668,6 @@ do_execsql_test 1370 { SELECT * FROM generate_series(0,0,0); } {} -reset_db -load_static_extension db series -do_execsql_test 1400 { - CREATE TABLE t1(x); - CREATE TABLE t2(y); -} -do_catchsql_test 1410 { - SELECT x, y, value - FROM (t1 RIGHT JOIN generate_series(t2.y,5) AS value) JOIN t2; -} {1 {table-function argument references tables to its right}} -do_catchsql_test 1420 { - SELECT x, y, value - FROM t2 JOIN (t1 RIGHT JOIN generate_series(t2.y,5) AS value) -} {1 {no such column: t2.y}} # Free up memory allocations diff --git a/test/tclsqlite.test b/test/tclsqlite.test index 6cababad3..5f373ea18 100644 --- a/test/tclsqlite.test +++ b/test/tclsqlite.test @@ -42,7 +42,7 @@ do_test tcl-1.1.1 { do_test tcl-1.2 { set v [catch {db bogus} msg] lappend v $msg -} {1 {bad option "bogus": must be authorizer, backup, bind_fallback, busy, cache, changes, close, collate, collation_needed, commit_hook, complete, config, copy, deserialize, enable_load_extension, errorcode, erroroffset, eval, exists, format, function, incrblob, interrupt, last_insert_rowid, nullvalue, onecolumn, preupdate, profile, progress, rekey, restore, rollback_hook, serialize, status, timeout, total_changes, trace, trace_v2, transaction, unlock_notify, update_hook, version, or wal_hook}} +} {1 {bad option "bogus": must be authorizer, backup, bind_fallback, busy, cache, changes, close, collate, collation_needed, commit_hook, complete, config, copy, deserialize, enable_load_extension, errorcode, erroroffset, eval, exists, function, incrblob, interrupt, last_insert_rowid, nullvalue, onecolumn, preupdate, profile, progress, rekey, restore, rollback_hook, serialize, status, timeout, total_changes, trace, trace_v2, transaction, unlock_notify, update_hook, version, or wal_hook}} do_test tcl-1.2.1 { set v [catch {db cache bogus} msg] lappend v $msg diff --git a/test/temptrigfault.tes b/test/temptrigfault.tes deleted file mode 100644 index 4b124e1f7..000000000 --- a/test/temptrigfault.tes +++ /dev/null @@ -1,120 +0,0 @@ -# 2025 November 11 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl -set testprefix temptrigfault - -forcedelete test.db2 -do_execsql_test 1.0 { - CREATE TABLE t1(x, y); - ATTACH 'test.db2' AS aux; - CREATE TABLE aux.t1(x, y); -} - -do_faultsim_test 1.1 -faults oom* -prep { -} -body { - execsql { - CREATE TEMP TRIGGER tmptrig AFTER INSERT ON t1 BEGIN - INSERT INTO aux.t1 VALUES(new.x, new.y); - END; - } -} -test { - faultsim_test_result {0 {}} - catchsql { DROP TRIGGER tmptrig } -} - -do_execsql_test 2.0 { - CREATE TEMP TRIGGER tmptrig AFTER INSERT ON t1 BEGIN - INSERT INTO aux.t1 VALUES(new.x, new.y); - END; -} - -do_faultsim_test 2 -faults oom* -prep { -} -body { - execsql { - INSERT INTO t1 VALUES('x', 'y'); - } -} -test { - faultsim_test_result {0 {}} -} - -do_execsql_test 3.0.1 { - SELECT * FROM t1; - DELETE FROM t1; - DELETE FROM aux.t1; -} [db eval {SELECT * FROM aux.t1}] - -do_execsql_test 3.0.2 { - CREATE TEMP TRIGGER tmptrig2 AFTER INSERT ON aux.t1 BEGIN - INSERT INTO t1 VALUES(new.x||'2', new.y||'2'); - END; -} - -do_faultsim_test 3 -faults oom* -prep { -} -body { - execsql { - INSERT INTO aux.t1 VALUES('aaa', 'bbb'); - } -} -test { - faultsim_test_result {0 {}} -} - -proc repeatlist {list n} { - set ret [list] - for {set i 0} {$i < $n} {incr i} { - set ret [concat $ret $list] - } - set ret -} - -do_execsql_test 3.x.1 { - SELECT * FROM main.t1; -} [repeatlist {aaa2 bbb2} 5] - -do_execsql_test 3.x.2 { - SELECT * FROM aux.t1; -} [repeatlist {aaa bbb aaa2 bbb2} 5] - -faultsim_save_and_close -do_faultsim_test 4 -faults oom* -prep { - faultsim_restore_and_reopen - execsql { ATTACH 'test.db2' AS aux; } -} -body { - execsql { - CREATE TEMP TRIGGER xyz AFTER DELETE ON main.t1 BEGIN - DELETE FROM aux.t1 WHERE rowid=old.rowid; - END; - - DELETE FROM t1 WHERE rowid=2; - } -} -test { - faultsim_test_result {0 {}} {1 {unable to open a temporary database file for storing temporary tables}} -} - -faultsim_save_and_close -do_faultsim_test 5 -faults oom* -prep { - faultsim_restore_and_reopen - execsql { ATTACH 'test.db2' AS aux; } -} -body { - execsql { - CREATE TEMP TRIGGER xyz AFTER UPDATE ON aux.t1 BEGIN - UPDATE main.t1 SET x=new.x, y=new.y WHERE rowid=new.rowid; - END; - UPDATE aux.t1 SET x=x||x WHERE rowid=1+abs(random() % 5); - } -} -test { - faultsim_test_result {0 {}} {1 {unable to open a temporary database file for storing temporary tables}} -} - - -finish_test diff --git a/test/temptrigger.test b/test/temptrigger.test index 74390f3b8..e4277adf6 100644 --- a/test/temptrigger.test +++ b/test/temptrigger.test @@ -276,191 +276,4 @@ do_catchsql_test 6.3 { } {1 error} db2 close -#------------------------------------------------------------------------- -reset_db -forcedelete test.db2 - -do_execsql_test 7.0 { - CREATE TABLE m1(a, b); - ATTACH 'test.db2' AS aux; - CREATE TABLE aux.a1(c, d); -} - -do_execsql_test 7.1 { - CREATE TEMP TRIGGER tr1 AFTER INSERT ON m1 BEGIN - INSERT INTO a1 VALUES(new.a, new.b); - END; - - INSERT INTO m1 VALUES(5, 6); - SELECT * FROM aux.a1; -} {5 6} - -do_execsql_test 7.2 { - CREATE TABLE a1(e, f); - INSERT INTO m1 VALUES(7, 8); -} - -do_execsql_test 7.3.1 { SELECT * FROM main.a1 } {7 8} -do_execsql_test 7.3.2 { SELECT * FROM aux.a1 } {5 6} - -do_execsql_test 7.4 { - DROP TRIGGER tr1; - CREATE TEMP TRIGGER tr1 AFTER INSERT ON m1 BEGIN - INSERT INTO a1 SELECT d, c FROM aux.a1; - END; - - DELETE FROM aux.a1; - DELETE FROM main.a1; - INSERT INTO aux.a1 VALUES('hello', 'world'); -} - -do_execsql_test 7.5 { - INSERT INTO m1 VALUES(9, 10); - SELECT * FROM main.a1; -} {world hello} - -do_catchsql_test 7.6 { - DROP TRIGGER tr1; - CREATE TRIGGER tr1 AFTER INSERT ON m1 BEGIN - INSERT INTO a1 SELECT d, c FROM aux.a1; - END; -} {1 {trigger tr1 cannot reference objects in database aux}} - -#------------------------------------------------------------------------- -# Check that temp triggers may INSERT/UPDATE/DELETE to fully qualified -# table names. -reset_db -forcedelete {*}[glob -nocomplain *mj*] -forcedelete test.db2 -do_execsql_test 8.0 { - ATTACH 'test.db2' AS aux; - CREATE TABLE t1(a, b); - CREATE TABLE t2(c, d); - CREATE TABLE aux.t1(e, f); - CREATE TABLE aux.t2(g, h); -} - -do_catchsql_test 8.1.1 { - CREATE TRIGGER tr1 AFTER INSERT ON t2 BEGIN - INSERT INTO aux.t1 VALUES(new.c, new.d); - END; -} {1 {qualified table names are not allowed on INSERT, UPDATE, and DELETE statements within triggers}} - -do_execsql_test 8.1.2 { - CREATE TEMP TRIGGER tr1 AFTER INSERT ON t2 BEGIN - INSERT INTO aux.t1 VALUES(new.c, new.d); - END; - - INSERT INTO main.t2 VALUES('x', 'y'); - SELECT * FROM aux.t1; -} {x y} - -do_execsql_test 8.1.3 { SELECT * FROM t1 } {} - -do_catchsql_test 8.2.1 { - CREATE TRIGGER aux.tr2 AFTER UPDATE ON aux.t1 BEGIN - UPDATE main.t2 SET c=new.e, d=new.f; - END; -} {1 {qualified table names are not allowed on INSERT, UPDATE, and DELETE statements within triggers}} - -do_execsql_test 8.2.2 { - CREATE TEMP TRIGGER tr2 AFTER UPDATE ON aux.t1 BEGIN - UPDATE main.t2 SET c=new.e, d=new.f; - END; - - UPDATE aux.t1 SET e=1, f=2; - SELECT * FROM t2; -} {1 2} - -do_execsql_test 8.2.3 { SELECT * FROM aux.t2 } {} - -do_catchsql_test 8.3.1 { - CREATE TRIGGER tr3 AFTER DELETE ON t2 BEGIN - DELETE FROM aux.t1; - END; -} {1 {qualified table names are not allowed on INSERT, UPDATE, and DELETE statements within triggers}} - -do_execsql_test 8.3.2 { - INSERT INTO main.t1 VALUES('a', 'b'); - CREATE TEMP TRIGGER tr3 AFTER DELETE ON t2 BEGIN - DELETE FROM aux.t1; - END; - - DELETE FROM main.t2; - SELECT * FROM aux.t1; -} {} - -do_execsql_test 8.3.3 { SELECT * FROM t1 } {a b} - -#------------------------------------------------------------------------- -reset_db -set nDb 8 -do_test 9.0 { - for {set ii 0} {$ii < $nDb} {incr ii} { - db eval "ATTACH ':memory:' AS db$ii" - db eval "CREATE TABLE db$ii.tbl(a, b, c)" - } - - for {set ii 0} {$ii < ($nDb-1)} {incr ii} { - set jj [expr $ii+1] - db eval " - CREATE TEMP TRIGGER tr$ii AFTER INSERT ON db$ii.tbl BEGIN - INSERT INTO db$jj.tbl VALUES(new.b, new.c, new.a); - END; - " - } -} {} - -do_execsql_test 9.1 { INSERT INTO db0.tbl VALUES('a', 'b', 'c'); } -do_execsql_test 9.1.1 { SELECT * FROM db0.tbl } {a b c} -do_execsql_test 9.1.2 { SELECT * FROM db1.tbl } {b c a} -do_execsql_test 9.1.3 { SELECT * FROM db2.tbl } {c a b} -do_execsql_test 9.1.1 { SELECT * FROM db3.tbl } {a b c} -do_execsql_test 9.1.2 { SELECT * FROM db4.tbl } {b c a} -do_execsql_test 9.1.3 { SELECT * FROM db5.tbl } {c a b} -do_execsql_test 9.1.1 { SELECT * FROM db6.tbl } {a b c} -do_execsql_test 9.1.2 { SELECT * FROM db7.tbl } {b c a} - -do_test 9.2 { - for {set ii 0} {$ii < ($nDb-1)} {incr ii} { - set jj [expr $ii+1] - db eval " - CREATE TEMP TRIGGER tru$ii AFTER UPDATE ON db$ii.tbl BEGIN - UPDATE db$jj.tbl SET a=new.b, b=new.c, c=new.a; - END; - " - } -} {} - -do_execsql_test 9.3 { UPDATE db0.tbl SET a=1, b=2, c=3 } -do_execsql_test 9.3.1 { SELECT * FROM db0.tbl } {1 2 3} -do_execsql_test 9.3.2 { SELECT * FROM db1.tbl } {2 3 1} -do_execsql_test 9.3.3 { SELECT * FROM db2.tbl } {3 1 2} -do_execsql_test 9.3.1 { SELECT * FROM db3.tbl } {1 2 3} -do_execsql_test 9.3.2 { SELECT * FROM db4.tbl } {2 3 1} -do_execsql_test 9.3.3 { SELECT * FROM db5.tbl } {3 1 2} -do_execsql_test 9.3.1 { SELECT * FROM db6.tbl } {1 2 3} -do_execsql_test 9.3.2 { SELECT * FROM db7.tbl } {2 3 1} - -do_test 9.4 { - for {set ii 0} {$ii < ($nDb-1)} {incr ii} { - set jj [expr $ii+1] - db eval " - CREATE TEMP TRIGGER trd$ii BEFORE DELETE ON db$ii.tbl BEGIN - DELETE FROM db$jj.tbl; - END; - " - } -} {} - -do_execsql_test 9.5 { DELETE FROM db0.tbl } -do_execsql_test 9.5.1 { SELECT * FROM db0.tbl } {} -do_execsql_test 9.5.2 { SELECT * FROM db1.tbl } {} -do_execsql_test 9.5.3 { SELECT * FROM db2.tbl } {} -do_execsql_test 9.5.1 { SELECT * FROM db3.tbl } {} -do_execsql_test 9.5.2 { SELECT * FROM db4.tbl } {} -do_execsql_test 9.5.3 { SELECT * FROM db5.tbl } {} -do_execsql_test 9.5.1 { SELECT * FROM db6.tbl } {} -do_execsql_test 9.5.2 { SELECT * FROM db7.tbl } {} - finish_test diff --git a/test/tester.tcl b/test/tester.tcl index 856df5421..3fad39668 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -1785,6 +1785,11 @@ proc crashsql {args} { # cfSync(), which can be different then what TCL uses by # default, so here we force it to the "nativename" format. set cfile [string map {\\ \\\\} [file nativename [file join [get_pwd] $crashfile]]] + ifcapable winrt { + # Except on winrt. Winrt has no way to transform a relative path into + # an absolute one, so it just uses the relative paths. + set cfile $crashfile + } set f [open crash.tcl w] puts $f "sqlite3_initialize ; sqlite3_shutdown" diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 1f81690f2..515036368 100755 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -101,7 +101,6 @@ Usage: $a0 help $a0 joblist ?PATTERN? $a0 njob ?NJOB? - $a0 retest $a0 script ?-msvc? CONFIG $a0 status ?-d SECS? ?--cls? $a0 halt @@ -121,21 +120,20 @@ Usage: --stop-on-error Stop running after any reported error --zipvfs ZIPVFSDIR ZIPVFS source directory -Special values for PERMUTATION include: +Special values for PERMUTATION that work with plain tclsh: - list - show allowed PERMUTATION arguments. + list - show all allowed PERMUTATION arguments. mdevtest - tests recommended prior to normal development check-ins. - devtest - alias for "mdevtest" release - full release test with various builds. sdevtest - like mdevtest but using ASAN and UBSAN. + +Other PERMUTATION arguments must be run using testfixture, not tclsh: + all - all tcl test scripts, plus a subset of test scripts rerun with various permutations. full - all tcl test scripts. veryquick - a fast subset of the tcl test scripts. This is the default. -The interpreter that runs this script can be an ordinary "tclsh" as long -as "package require sqlite3" works, or it can be "testfixture". - If no PATTERN arguments are present, all tests specified by the PERMUTATION are run. Otherwise, each pattern is interpreted as a glob pattern. Only those tcl tests for which the final component of the filename matches at @@ -169,9 +167,6 @@ only the parts that contain the error messages. The --summary option just shows the jobs that failed. If PATTERN are provided, the error information is only provided for jobs that match PATTERN. -The "retest" command reruns tests that failed or were never completed -by a prior invocation of testrunner.tcl. - Full documentation here: https://sqlite.org/src/doc/trunk/doc/testrunner.md }]] @@ -217,8 +212,7 @@ proc default_njob {} { if {$nCore<=2} { set nHelper 1 } else { - set nHelper [expr int($nCore*0.8)] - if {$nHelper>20} {set nHelper 20} + set nHelper [expr int($nCore*0.5)] } return $nHelper } @@ -303,8 +297,6 @@ switch -nocase -glob -- $tcl_platform(os) { error "cannot determine platform!" } } -set TRG(testfixture-fullpath) [file join $dir $TRG(testfixture)] -set TRG(interp) [info nameofexec] #------------------------------------------------------------------------- #------------------------------------------------------------------------- @@ -714,18 +706,9 @@ if {[llength $argv]>=1 } } - set once 1 - while {![file readable $TRG(dbname)]} { - if {$delay==0} { - puts "Database missing: $TRG(dbname)" - exit - } - if {$once} { - set once 0 - puts "Waiting for testing to start...." - flush stdout - } - after [expr {$delay*1000}] + if {![file readable $TRG(dbname)]} { + puts "Database missing: $TRG(dbname)" + exit } sqlite3 mydb $TRG(dbname) mydb timeout 2000 @@ -1165,7 +1148,7 @@ proc add_tcl_jobs {build config patternlist {shelldepid ""}} { set testrunner_tcl [file normalize [info script]] if {$build==""} { - set testfixture $TRG(interp) + set testfixture [info nameofexec] } else { set testfixture [file join [lindex $build 1] $TRG(testfixture)] } @@ -1286,26 +1269,6 @@ proc add_fuzztest_jobs {buildname patternlist} { set subcmd [lrange $interpreter 1 end] set interpreter [lindex $interpreter 0] - # For fuzzcheck-asan and fuzzcheck-ubsan, break up some - # fuzzdata files into multiple slices, for improved - # concurrency. - # - if {[string match *fuzzcheck-*san $interpreter]} { - set newscripts {} - foreach s $scripts { - if {[string match {*fuzzdata[12].db} $s] - && ![string match slice $s]} { - set N 6 - for {set i 0} {$i<$N} {incr i} { - lappend newscripts [list --slice $i $N $s] - } - } else { - lappend newscripts $s - } - } - set scripts $newscripts - } - if {[string match fuzzcheck* $interpreter] && [info exists env(FUZZDB)] && [file readable $env(FUZZDB)] @@ -1396,30 +1359,14 @@ proc add_devtest_jobs {lBld patternlist} { } } -# Check to ensure that TRG(interp) is a full-blown "testfixture" and -# not just a "tclsh". -# -# The value of TRG(interp) defaults to whatever interpreter is running -# this script, which might be either tclsh or testfixture. If tclsh is -# running this script, change $TRG(interp) to be an instance of testfixture. -# If no testfixture exists in the directory from which this script is run, -# attempt to build one. -# -# Do not return unless $TRG(interp) is a valid testfixture. If unable -# to find and/or construct one, abort with an error message. +# Check to ensure that the interpreter is a full-blown "testfixture" +# build and not just a "tclsh". If this is not the case, issue an +# error message and exit. # proc must_be_testfixture {} { - global TRG if {[lsearch [info commands] sqlite3_soft_heap_limit]<0} { - if {![file exec $TRG(testfixture-fullpath)]} { - puts "make testfixture" - catch {exec make testfixture >@stdout 2>@stderr} - } - if {![file exec $TRG(testfixture-fullpath)]} { - puts "Requires testfixture, and I was unable to build it." - exit 1 - } - set TRG(interp) $TRG(testfixture-fullpath) + puts "Use testfixture, not tclsh, for these arguments." + exit 1 } } @@ -1494,15 +1441,11 @@ proc add_jobs_from_cmdline {patternlist} { list { set allperm [array names ::testspec] - lappend allperm all devtest mdevtest sdevtest release list + lappend allperm all mdevtest sdevtest release list puts "Allowed values for the PERMUTATION argument: [lsort $allperm]" exit 0 } - retest { - # no-op - } - default { must_be_testfixture if {[info exists ::testspec($first)]} { @@ -1539,8 +1482,6 @@ proc add_jobs_from_cmdline {patternlist} { } } -# Initializer, or reinitialize, the testrunner.db database file. -# proc make_new_testset {} { global TRG @@ -1584,8 +1525,7 @@ proc mark_job_as_finished {jobid output state endtm} { SET output=$output, state=$state, endtime=$endtm, span=$endtm-starttime, ntest=$ntest, nerr=$nerr, svers=$svers, pltfm=$pltfm WHERE jobid=$jobid; - UPDATE jobs SET state=$childstate - WHERE depid=$jobid AND state!='halt' AND state!='done'; + UPDATE jobs SET state=$childstate WHERE depid=$jobid AND state!='halt'; UPDATE config SET value=value+$nerr WHERE name='nfail'; UPDATE config SET value=value+$ntest WHERE name='ntest'; } @@ -1657,7 +1597,7 @@ proc launch_another_job {iJob} { global O global T - set testfixture $TRG(interp) + set testfixture [info nameofexec] set script $TRG(info_script) set O($iJob) "" @@ -1858,27 +1798,6 @@ proc run_testset {} { } -# If the argument is "retest", simply rerun all tests from the previous -# run that are marked as one of "ready", "running", "failed", or "omit" -# plus redo any build of dependencies those tests. -# -proc handle_retest {} { - set cnt 0 - if {[catch {trdb exists {SELECT jobid FROM jobs}} cnt] || $cnt==0} { - puts "No test available to rerun" - exit 1 - } - trdb eval {UPDATE jobs SET state='ready' - WHERE state IN ('running','failed','omit')} - for {set kk 0} {$kk<2} {incr kk} { - trdb eval { - UPDATE jobs SET state='ready' - WHERE jobid IN (SELECT depid FROM jobs WHERE state='ready'); - UPDATE jobs SET state='' WHERE state='ready' AND depid<>''; - } - } -} - # Handle the --buildonly option, if it was specified. # proc handle_buildonly {} { @@ -1917,21 +1836,14 @@ proc explain_tests {} { sqlite3 trdb $TRG(dbname) trdb timeout $TRG(timeout) -if {[llength $TRG(patternlist)]==1 && $TRG(patternlist) eq "retest"} { - set tm 0 - handle_retest -} else { - set tm [lindex [time { make_new_testset }] 0] -} +set tm [lindex [time { make_new_testset }] 0] if {$TRG(explain)} { explain_tests } else { if {$TRG(nJob)>1} { puts "splitting work across $TRG(nJob) cores" } - if {$tm>0} { - puts "built testset in [expr $tm/1000]ms.." - } + puts "built testset in [expr $tm/1000]ms.." handle_buildonly run_testset } diff --git a/test/testrunner_data.tcl b/test/testrunner_data.tcl index 4daee0274..e74caee1d 100644 --- a/test/testrunner_data.tcl +++ b/test/testrunner_data.tcl @@ -37,6 +37,7 @@ namespace eval trd { set tcltest(win.Windows-Memdebug) veryquick set tcltest(win.Windows-Win32Heap) veryquick set tcltest(win.Windows-Sanitize) veryquick + set tcltest(win.Windows-WinRT) veryquick set tcltest(win.Default) {full win_unc_locking} # Extra [make xyz] tests that should be run for various builds. @@ -192,7 +193,6 @@ namespace eval trd { -DSQLITE_ENABLE_HIDDEN_COLUMNS -DSQLITE_MAX_ATTACHED=125 -DSQLITE_MUTATION_TEST - -DSQLITE_THREAD_MISUSE_ABORT --enable-fts5 } set build(Debug-Two) { @@ -364,6 +364,12 @@ namespace eval trd { set build(Windows-Sanitize) { ASAN=1 } + + set build(Windows-WinRT) { + FOR_WINRT=1 + ENABLE_SETLK=1 + -DSQLITE_TEMP_STORE=3 + } } diff --git a/test/testrunner_estwork.tcl b/test/testrunner_estwork.tcl index e02eb22dc..c139394a5 100644 --- a/test/testrunner_estwork.tcl +++ b/test/testrunner_estwork.tcl @@ -364,7 +364,6 @@ set estwork(shell6.test) 3 set estwork(shell8.test) 104 set estwork(shell9.test) 3 set estwork(shellA.test) 2 -set estwork(shellB.test) 2 set estwork(shmlock.test) 27 set estwork(sidedelete.test) 10 set estwork(skipscan1.test) 7 diff --git a/test/tkt-99378177930f87bd.test b/test/tkt-99378177930f87bd.test index 495867280..ba9fdc702 100644 --- a/test/tkt-99378177930f87bd.test +++ b/test/tkt-99378177930f87bd.test @@ -33,8 +33,6 @@ do_execsql_test tkt-99378-100 { (2, '{"x":2}', 4, 5), (3, '{"x":1}', 6, 7); CREATE INDEX t1x ON t1(d, a, b->>'x', c); - CREATE TABLE t2(y); - INSERT INTO t2(y) VALUES(9); } {} do_execsql_test tkt-99378-110 { SELECT a, @@ -50,20 +48,6 @@ do_execsql_test tkt-99378-110 { 2 2 1 26 22 3 1 1 6 6 } -do_execsql_test tkt-99378-111 { - SELECT if(a,a,y), - SUM(1) AS t1, - SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, - SUM(c) AS t3, - SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 - FROM t2 CROSS JOIN t1 - WHERE d BETWEEN 0 and 10 - GROUP BY a; -} { - 1 2 1 16 12 - 2 2 1 26 22 - 3 1 1 6 6 -} # The proof that the index on the expression is being used is in the # fact that the byte code contains no "Function" opcodes. In other words, @@ -81,17 +65,6 @@ do_execsql_test tkt-99378-120 { WHERE d BETWEEN 0 and 10 GROUP BY a; } {~/Function/} -do_execsql_test tkt-99378-121 { - EXPLAIN - SELECT if(a,a,y), - SUM(1) AS t1, - SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, - SUM(c) AS t3, - SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 - FROM t2 CROSS JOIN t1 - WHERE d BETWEEN 0 and 10 - GROUP BY a; -} {~/Function/} do_execsql_test tkt-99378-130 { @@ -209,7 +182,6 @@ do_execsql_test tkt-99378-310 { # do_execsql_test tkt-99378-400 { DROP TABLE t1; - DROP TABLE t2; CREATE TABLE t0(w); INSERT INTO t0(w) VALUES(1); CREATE TABLE t1(x); diff --git a/test/tkt2339.test b/test/tkt2339.test index 4bbedb828..41acd377c 100644 --- a/test/tkt2339.test +++ b/test/tkt2339.test @@ -47,12 +47,12 @@ do_test tkt2339.2 { } } {4 3 14 13} do_test tkt2339.3 { - lsort -integer [execsql { + execsql { SELECT * FROM (SELECT * FROM t1 ORDER BY num DESC) UNION ALL SELECT * FROM (SELECT * FROM t2 ORDER BY num DESC LIMIT 2) - }] -} {1 2 3 4 13 14} + } +} {4 3 2 1 14 13} do_test tkt2339.4 { execsql { SELECT * FROM (SELECT * FROM t1 ORDER BY num DESC LIMIT 2) diff --git a/test/values.test b/test/values.test index 58e764ce6..c3c52ceb1 100644 --- a/test/values.test +++ b/test/values.test @@ -21,9 +21,9 @@ do_execsql_test 1.0 { } -#explain_i { -# INSERT INTO x1(a, b, c) VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4); -#} +explain_i { + INSERT INTO x1(a, b, c) VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4); +} do_execsql_test 1.1.1 { INSERT INTO x1 VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4); } diff --git a/test/vt02.c b/test/vt02.c index e08d649c1..06fade096 100644 --- a/test/vt02.c +++ b/test/vt02.c @@ -45,29 +45,8 @@ ** a=... AND b=... AND c=... ** a=... AND b=... AND c=... AND d=... ** -** The table will also recognize IN constraints on column D using the -** sqlite3_vtab_in_first() and sqlite3_vtab_in_next() interfaces: -** -** a=... AND d IN (...) -** a=... AND b=... AND d IN (...) -** a=... AND b=... AND c=... AND d IN (...) -** -** Various ORDER BY constraints are also recognized and consumed. -** -** ORDER BY x -** ORDER BY a -** ORDER BY a, b -** ORDER BY a, b, c -** ORDER BY a, b, c, d -** -** The above also work if every term is DESC rather than ASC. However, -** the orderByConsumed is not set if there are a mixture of ASC and DESC -** terms in the ORDER BY clause. -** -** The sqlite3_vtab_distinct() interface is used to recognize DISTINCT -** and GROUP BY constraints and consume the corresponding sorting requirements. -** -** The OFFSET constraint is recognized and consumed. +** Various ORDER BY constraints are also recognized and consumed. The +** OFFSET constraint is recognized and consumed. ** ** ## TABLE-VALUED FUNCTION ** @@ -111,13 +90,6 @@ ** vector of results sent to xFilter. Only the first ** few are used, as required by idxNum. ** -** 0x80 If sqlite3_vtab_distinct() says that duplicate rows -** may be omitted (values 2 or 3), then maybe omit -** some but not all of the duplicate rows. -** -** 0x100 Do not omit any duplicate rows even if -** sqlite3_vtab_distinct() says that is ok to do. -** ** Because these flags take effect during xBestIndex, the RHS of the ** flag= constraint must be accessible. In other words, the RHS of flag= ** needs to be an integer literal, not another column of a join or a @@ -164,12 +136,14 @@ ** ## COMPILING AND RUNNING ** ** This file can also be compiled separately as a loadable extension -** for SQLite (as long as the -DTH3_VERSION is not defined). To compile as a -** loadable extension do something like this: +** for SQLite (as long as the -DTH3_VERSION is not defined). To compile as a +** loadable extension do his: +** +** gcc -Wall -g -shared -fPIC -I. -DSQLITE_DEBUG vt02.c -o vt02.so ** -** (linux) gcc -shared -fPIC -I. vt02.c -o vt02.so -** (mac) clang -dynamiclib -fPIC -I. vt02.c -o vt02.dylib -** (windows) cl vt02.c -link -dll -out:vt02.dll +** Or on Windows: +** +** cl vt02.c -link -dll -out:vt02.dll ** ** Then load into the CLI using: ** @@ -193,7 +167,6 @@ ** 2x increment by 100 ** 3x increment by 1000 ** 1xx Use offset provided by argv[N] -** 1xxx Reverse output order (to implement ORDER BY ... DESC) */ #ifndef TH3_VERSION /* These bits for separate compilation as a loadable extension, only */ @@ -222,8 +195,6 @@ struct vt02_vtab { #define VT02_NO_OFFSET 0x0004 /* Omit the offset optimization */ #define VT02_ALLOC_IDXSTR 0x0008 /* Alloate an idxStr */ #define VT02_BAD_IDXNUM 0x0010 /* Generate an invalid idxNum */ -#define VT02_PARTIAL_DEDUP 0x0080 /* Omit some but not all duplicate rows */ -#define VT02_NO_DEDUP 0x0100 /* Include all duplicate rows */ /* ** A cursor @@ -232,7 +203,6 @@ struct vt02_cur { sqlite3_vtab_cursor parent; /* Base class. Must be first */ sqlite3_int64 i; /* Current entry */ sqlite3_int64 iEof; /* Indicate EOF when reaching this value */ - sqlite3_int64 iMin; /* EOF if dropping below this value */ int iIncr; /* Amount by which to increment */ unsigned int mD; /* Mask of allowed D-column values */ }; @@ -322,7 +292,7 @@ static int vt02Close(sqlite3_vtab_cursor *pCursor){ */ static int vt02Eof(sqlite3_vtab_cursor *pCursor){ vt02_cur *pCur = (vt02_cur*)pCursor; - return pCur->i<pCur->iMin || pCur->i>=pCur->iEof; + return pCur->i<0 || pCur->i>=pCur->iEof; } /* Advance the cursor to the next row in the table @@ -331,8 +301,8 @@ static int vt02Next(sqlite3_vtab_cursor *pCursor){ vt02_cur *pCur = (vt02_cur*)pCursor; do{ pCur->i += pCur->iIncr; - if( pCur->i<pCur->iMin || pCur->i>=pCur->iEof ) break; - }while( (pCur->mD & (1<<(pCur->i%10)))==0 ); + if( pCur->i<0 ) pCur->i = pCur->iEof; + }while( (pCur->mD & (1<<(pCur->i%10)))==0 && pCur->i<pCur->iEof ); return SQLITE_OK; } @@ -354,7 +324,6 @@ static int vt02Next(sqlite3_vtab_cursor *pCursor){ ** 2x increment by 100 ** 3x increment by 1000 ** 1xx Use offset provided by argv[N] -** 1xxx Output rows in reverse order */ static int vt02Filter( sqlite3_vtab_cursor *pCursor, /* The cursor to rewind */ @@ -365,17 +334,11 @@ static int vt02Filter( ){ vt02_cur *pCur = (vt02_cur*)pCursor; /* The vt02 cursor */ int bUseOffset = 0; /* True to use OFFSET value */ - int bReverse = 0; /* Output rows in reverse order */ int iArg = 0; /* argv[] values used so far */ int iOrigIdxNum = idxNum; /* Original value for idxNum */ pCur->iIncr = 1; - pCur->iMin = 0; pCur->mD = 0x3ff; - if( idxNum>=1000 ){ - bReverse = 1; - idxNum -= 1000; - } if( idxNum>=100 ){ bUseOffset = 1; idxNum -= 100; @@ -453,17 +416,9 @@ static int vt02Filter( }else{ goto vt02_bad_idxnum; } - if( bReverse ){ - sqlite3_int64 x; - x = pCur->i + ((pCur->iEof - pCur->i)/pCur->iIncr)*pCur->iIncr; - if( x>=pCur->iEof ) x -= pCur->iIncr; - pCur->iIncr = -pCur->iIncr; - pCur->iMin = pCur->i; - pCur->i = x; - } if( bUseOffset ){ int nSkip = sqlite3_value_int(argv[iArg]); - while( nSkip-- > 0 && !vt02Eof(pCursor) ) vt02Next(pCursor); + while( nSkip-- > 0 && pCur->i<pCur->iEof ) vt02Next(pCursor); } return SQLITE_OK; @@ -883,52 +838,35 @@ static int vt02BestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ ** the same answer. */ if( pInfo->nOrderBy>0 && (flags & VT02_NO_SORT_OPT)==0 ){ - int eDistinct = sqlite3_vtab_distinct(pInfo); if( pInfo->idxNum==1 ){ /* There will only be one row of output. So it is always sorted. */ pInfo->orderByConsumed = 1; }else - if( pInfo->aOrderBy[0].iColumn<=0 ){ - /* First column of order by is X */ - if( pInfo->aOrderBy[0].desc ){ - pInfo->idxNum += 1000; /* Reverse output order */ - } + if( pInfo->aOrderBy[0].iColumn<=0 + && pInfo->aOrderBy[0].desc==0 + ){ + /* First column of order by is X ascending */ pInfo->orderByConsumed = 1; }else - if( eDistinct>=1 ){ + if( sqlite3_vtab_distinct(pInfo)>=1 ){ unsigned int x = 0; - int nDesc = 0; - int nAsc = 0; for(i=0; i<pInfo->nOrderBy; i++){ int iCol = pInfo->aOrderBy[i].iColumn; if( iCol<0 ) iCol = 0; - if( pInfo->aOrderBy[i].desc ){ - nDesc++; - }else{ - nAsc++; - } x |= 1<<iCol; } - if( nDesc>0 && nAsc>0 ){ - if( eDistinct!=1 ) eDistinct = -999; /* Never set orderByConsumed */ - }else if( nAsc==0 ){ - pInfo->idxNum += 1000; /* Reverse output order */ - } - if( eDistinct>=2 && (flags & VT02_NO_DEDUP)!=0 ){ - eDistinct = 1; - } - if( eDistinct>=2 ){ /* DISTINCT or (DISTINCT and ORDER BY) */ + if( sqlite3_vtab_distinct(pInfo)==2 ){ if( x==0x02 ){ /* DISTINCT A */ - pInfo->idxNum += (flags & VT02_PARTIAL_DEDUP)!=0 ? 20 : 30; + pInfo->idxNum += 30; pInfo->orderByConsumed = 1; }else if( x==0x06 ){ /* DISTINCT A,B */ - pInfo->idxNum += (flags & VT02_PARTIAL_DEDUP)!=0 ? 10 : 20; + pInfo->idxNum += 20; pInfo->orderByConsumed = 1; }else if( x==0x0e ){ /* DISTINCT A,B,C */ - pInfo->idxNum += (flags & VT02_PARTIAL_DEDUP)!=0 ? 0 : 10; + pInfo->idxNum += 10; pInfo->orderByConsumed = 1; }else if( x & 0x01 ){ /* DISTINCT X */ @@ -937,7 +875,7 @@ static int vt02BestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ /* DISTINCT A,B,C,D */ pInfo->orderByConsumed = 1; } - }else if( eDistinct==1 ){ /* GROUP BY */ + }else{ if( x==0x02 ){ /* GROUP BY A */ pInfo->orderByConsumed = 1; @@ -955,21 +893,6 @@ static int vt02BestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ pInfo->orderByConsumed = 1; } } - }else{ - int nDesc = 0; - int nAsc = 0; - for(i=0; i<pInfo->nOrderBy; i++){ - if( pInfo->aOrderBy[i].iColumn!=i+1 ) break; - if( pInfo->aOrderBy[i].desc ){ - nDesc++; - }else{ - nAsc++; - } - } - if( i==pInfo->nOrderBy && (nDesc==0 || nAsc==0) ){ - pInfo->orderByConsumed = 1; - if( nDesc ) pInfo->idxNum += 1000; - } } } @@ -978,7 +901,7 @@ static int vt02BestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ pInfo->needToFreeIdxStr = 1; } if( flags & VT02_BAD_IDXNUM ){ - pInfo->idxNum += 10000; + pInfo->idxNum += 1000; } if( iOffset>=0 ){ diff --git a/test/vt100-a.sql b/test/vt100-a.sql index f141f637f..a0d3f46be 100644 --- a/test/vt100-a.sql +++ b/test/vt100-a.sql @@ -9,40 +9,11 @@ INSERT INTO t1 VALUES ('one','twotwotwo','thirty-three'), (unistr('\u001b[91mRED\u001b[0m'),'fourfour','fifty-five'), ('six','seven','eighty-eight'); -.testcase 100 +.print With -escape off SELECT * FROM t1; -.check <<END -╭─────┬───────────┬──────────────╮ -│ a │ b │ c │ -╞═════╪═══════════╪══════════════╡ -│ one │ twotwotwo │ thirty-three │ -│ RED │ fourfour │ fifty-five │ -│ six │ seven │ eighty-eight │ -╰─────┴───────────┴──────────────╯ -END - .mode box -escape ascii -.testcase 200 +.print With -escape ascii SELECT * FROM t1; -.check <<END -╭────────────────┬───────────┬──────────────╮ -│ a │ b │ c │ -╞════════════════╪═══════════╪══════════════╡ -│ one │ twotwotwo │ thirty-three │ -│ ^[[91mRED^[[0m │ fourfour │ fifty-five │ -│ six │ seven │ eighty-eight │ -╰────────────────┴───────────┴──────────────╯ -END - -.testcase 300 .mode box -escape symbol +.print With -escape symbol SELECT * FROM t1; -.check <<END -╭──────────────┬───────────┬──────────────╮ -│ a │ b │ c │ -╞══════════════╪═══════════╪══════════════╡ -│ one │ twotwotwo │ thirty-three │ -│ ␛[91mRED␛[0m │ fourfour │ fifty-five │ -│ six │ seven │ eighty-eight │ -╰──────────────┴───────────┴──────────────╯ -END diff --git a/test/walckptnoop.test b/test/walckptnoop.test index 89055316f..7ff8e90b8 100644 --- a/test/walckptnoop.test +++ b/test/walckptnoop.test @@ -102,11 +102,7 @@ do_catchsql_test 1.8 { PRAGMA wal_checkpoint = noop; } {0 {0 5 0}} -do_test 1.9 { - sqlite3_wal_checkpoint_v2 db noop -} {0 5 0} - -do_execsql_test 1.10 { +do_execsql_test 1.9 { PRAGMA journal_mode = delete; PRAGMA wal_checkpoint = noop; } {delete 0 -1 -1} diff --git a/test/walrestart.test b/test/walrestart.test index cf27a4098..4274b2e33 100644 --- a/test/walrestart.test +++ b/test/walrestart.test @@ -78,12 +78,10 @@ do_execsql_test -db db2 1.3 { proc faultsim {n} { return 0 } do_execsql_test 1.4 { PRAGMA wal_checkpoint; -} {/0 5. 5./} +} {0 58 58} do_catchsql_test 1.5 { PRAGMA integrity_check } {0 ok} -sqlite3_test_control_fault_install - finish_test diff --git a/test/where2.test b/test/where2.test index 6045cb117..83740bffd 100644 --- a/test/where2.test +++ b/test/where2.test @@ -794,41 +794,4 @@ do_execsql_test where2-15.1 { ORDER BY a,EXISTS(SELECT 1 FROM t1 LEFT JOIN (SELECT x AS y FROM t2) AS s2 ON t1.b=s2.y),x; } {12 34 NULL | 56 78 78 | 90 12 12 |} -# Demonstrate that CROSS JOIN is a join reordering barrier. -# -reset_db -do_execsql_test where2-16.1 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b INT); - CREATE TABLE t2(c INTEGER PRIMARY KEY, d INT); - CREATE TABLE t3(e INTEGER PRIMARY KEY, f INT); - CREATE TABLE t4(g INTEGER PRIMARY KEY, h INT); -} - -# Here the query planner wants to move t4 forward so that it is in front of -# t1. Ensure that does not happen. -do_execsql_test where2-16.2 { - EXPLAIN QUERY PLAN - SELECT * - FROM t1, t2 CROSS JOIN t3, t4 - WHERE t4.g=1 - AND t1.a=t4.h - AND t2.c=t1.b - AND t3.e=t2.d; -} {~/.* t4 .* t[12] .*/} - -# In this case the planner wants to move t1 to come after t4. Ensure that -# does not happen. -do_execsql_test where2-16.2 { - EXPLAIN QUERY PLAN - SELECT * - FROM t1, t2 CROSS JOIN t3, t4 - WHERE t2.c=1 - AND t3.e=t2.d - AND t4.g=t3.f - AND t1.a=t4.h; -} {~/.* t[34] .* t1 .*/} - - - - finish_test diff --git a/test/whereK.test b/test/whereK.test index 995c08371..060d470ff 100644 --- a/test/whereK.test +++ b/test/whereK.test @@ -69,17 +69,4 @@ do_execsql_test 1.5eqp { ORDER BY +a; } {/SEARCH t1 USING INDEX t1bc/} -# https://sqlite.org/forum/forumpost/2026-01-16T11:35:28Z -do_execsql_test 2.1 { - DROP TABLE t1; - CREATE TABLE t0(x COLLATE NOCASE); - CREATE INDEX t0x ON t0(x); - CREATE TABLE t1(y); - INSERT INTO t0 VALUES('a'); - INSERT INTO t1 VALUES('AB'); - SELECT count(*) FROM t0, t1 WHERE (y BETWEEN 1 AND x) OR (x>=y AND x); - SELECT count(*) FROM t0, t1 WHERE (x>=y AND x) OR (y BETWEEN 1 AND x); -} {1 1} - - finish_test diff --git a/test/win32longpath.test b/test/win32longpath.test index 43905f26e..2545c55a1 100644 --- a/test/win32longpath.test +++ b/test/win32longpath.test @@ -115,7 +115,11 @@ do_test 1.6 { db3 close +# winrt platforms do not handle paths with unix-style '/' directory separators. +# set lUri [list 1a 1b 1c 1d 1e 1f] +ifcapable winrt { set lUri [list 1a 1c 1e] } + foreach tn $lUri { sqlite3 db3 $uri($tn) -vfs win32-longpath -uri 1 -translatefilename 0 diff --git a/test/with1.test b/test/with1.test index c87082583..5ddf9dce0 100644 --- a/test/with1.test +++ b/test/with1.test @@ -1237,23 +1237,4 @@ do_execsql_test 27.1 { SELECT k, cte_map, main_map, '|' FROM log ORDER BY k; } {1 cte1 main1 | 2 cte2 main2 |} -# forum post https://sqlite.org/forum/forumpost/2026-03-04T05:06:26Z -# -db null NULL -do_execsql_test 28.1 { - DROP TABLE t1; - CREATE TABLE t1(x INTEGER PRIMARY KEY); - INSERT INTO t1 VALUES(1),(4),(999); - SELECT ( - WITH RECURSIVE t2(y) AS ( - SELECT 4 - UNION - SELECT NULL - UNION - SELECT y+1 FROM t2 WHERE y=4 ORDER BY 1 - ) - SELECT 1 FROM t2 WHERE y=x - ) FROM t1; -} {NULL 1 NULL} - finish_test diff --git a/test/zipfile2.test b/test/zipfile2.test index c1498872a..8ee90d310 100644 --- a/test/zipfile2.test +++ b/test/zipfile2.test @@ -302,19 +302,4 @@ do_execsql_test 7.1 { SELECT length(name) FROM t1; } {60000} - -# https://sqlite.org/forum/forumpost/721a05d2c5 -# -if {[catch { load_static_extension db fileio }]==0} { - forcedelete test.zip - set fd [open test.zip wb] - fconfigure $fd -translation binary - puts -nonewline $fd [db one {SELECT X'504b0506000000000100010030000000160000000000504b01021400140000000000000000000000000000000000000000000100010000000000000000000000000000006100'}] - close $fd - - do_catchsql_test 8.0 { - SELECT name,sz FROM zipfile(readfile('test.zip')); - } {1 {failed to read LFH at offset 0}} -} - finish_test diff --git a/tool/build-all-msvc.bat b/tool/build-all-msvc.bat index ae0d0e5b6..83d660deb 100755 --- a/tool/build-all-msvc.bat +++ b/tool/build-all-msvc.bat @@ -105,6 +105,7 @@ REM When set, these values are expanded and passed to the NMAKE command line, REM after its other arguments. These may be used to specify additional NMAKE REM options, for example: REM +REM SET NMAKE_ARGS=FOR_WINRT=1 REM SET NMAKE_ARGS_DEBUG=MEMDEBUG=1 REM SET NMAKE_ARGS_RETAIL=WIN32HEAP=1 REM @@ -860,4 +861,4 @@ GOTO no_errors GOTO end_of_file :end_of_file -%__ECHO% EXIT /B %ERRORLEVEL% +%__ECHO% EXIT /B %ERRORLEVEL% diff --git a/tool/dbtotxt.c b/tool/dbtotxt.c index ed347840a..fbd6e3d51 100644 --- a/tool/dbtotxt.c +++ b/tool/dbtotxt.c @@ -1,12 +1,6 @@ /* -** 2018-12-13 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** Copyright 2008 D. Richard Hipp and Hipp, Wyrick & Company, Inc. +** All Rights Reserved ** ****************************************************************************** ** diff --git a/tool/lemon.c b/tool/lemon.c index 9071e7719..324dda0c5 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -494,7 +494,6 @@ struct lemon { char *filename; /* Name of the input file */ char *outname; /* Name of the current output file */ char *tokenprefix; /* A prefix added to token names in the .h file */ - char *stackSizeLimit; /* Function to return the stack size limit */ char *reallocFunc; /* Function to use to allocate stack space */ char *freeFunc; /* Function to use to free stack space */ int nconflict; /* Number of parsing conflicts */ @@ -2639,9 +2638,6 @@ static void parseonetoken(struct pstate *psp) }else if( strcmp(x,"default_type")==0 ){ psp->declargslot = &(psp->gp->vartype); psp->insertLineMacro = 0; - }else if( strcmp(x,"stack_size_limit")==0 ){ - psp->declargslot = &(psp->gp->stackSizeLimit); - psp->insertLineMacro = 0; }else if( strcmp(x,"realloc")==0 ){ psp->declargslot = &(psp->gp->reallocFunc); psp->insertLineMacro = 0; @@ -3719,7 +3715,7 @@ PRIVATE int compute_action(struct lemon *lemp, struct action *ap) return act; } -#define LINESIZE 10000 +#define LINESIZE 1000 /* The next cluster of routines are for reading the template file ** and writing the results to the generated parser */ /* The first function transfers data from "in" to "out" until @@ -3754,9 +3750,12 @@ PRIVATE void tplt_xfer(char *name, FILE *in, FILE *out, int *lineno) /* Skip forward past the header of the template file to the first "%%" */ -PRIVATE void tplt_skip_header(FILE *in){ +PRIVATE void tplt_skip_header(FILE *in, int *lineno) +{ char line[LINESIZE]; - while( fgets(line,LINESIZE,in) && (line[0]!='%' || line[1]!='%') ){} + while( fgets(line,LINESIZE,in) && (line[0]!='%' || line[1]!='%') ){ + (*lineno)++; + } } /* The next function finds the template file and opens it, returning @@ -3826,14 +3825,12 @@ PRIVATE void tplt_linedir(FILE *out, int lineno, char *filename) filename++; } fprintf(out,"\"\n"); - fflush(out); } /* Print a string to the file and keep the linenumber up to date */ PRIVATE void tplt_print(FILE *out, struct lemon *lemp, char *str, int *lineno) { if( str==0 ) return; - fflush(out); while( *str ){ putc(*str,out); if( *str=='\n' ) (*lineno)++; @@ -3846,7 +3843,6 @@ PRIVATE void tplt_print(FILE *out, struct lemon *lemp, char *str, int *lineno) if (!lemp->nolinenosflag) { (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); } - fflush(out); return; } @@ -4411,13 +4407,6 @@ static void writeRuleText(FILE *out, struct rule *rp){ } } -/* -** Return true if the string is not NULL and not empty. -*/ -static int notnull(const char *z){ - return z && z[0]; -} - /* Generate C source code for the parser */ void ReportTable( @@ -4425,7 +4414,7 @@ void ReportTable( int mhflag, /* Output in makeheaders format if true */ int sqlFlag /* Generate the *.sql file too */ ){ - FILE *out, *in; + FILE *out, *in, *sql; int lineno; struct state *stp; struct action *ap; @@ -4450,10 +4439,18 @@ void ReportTable( in = tplt_open(lemp); if( in==0 ) return; - if( sqlFlag ){ - FILE *sql = file_open(lemp, ".sql", "wb"); + out = file_open(lemp,".c","wb"); + if( out==0 ){ + fclose(in); + return; + } + if( sqlFlag==0 ){ + sql = 0; + }else{ + sql = file_open(lemp, ".sql", "wb"); if( sql==0 ){ fclose(in); + fclose(out); return; } fprintf(sql, @@ -4518,12 +4515,6 @@ void ReportTable( } } fprintf(sql, "COMMIT;\n"); - fclose(sql); - } - out = file_open(lemp,".c","wb"); - if( out==0 ){ - fclose(in); - return; } lineno = 1; @@ -4552,7 +4543,7 @@ void ReportTable( } } if( lemp->include[0]=='/' ){ - tplt_skip_header(in); + tplt_skip_header(in,&lineno); }else{ tplt_xfer(lemp->name,in,out,&lineno); } @@ -4572,7 +4563,7 @@ void ReportTable( if( mhflag ){ fprintf(out,"#if INTERFACE\n"); lineno++; }else{ - fprintf(out,"#ifndef %s%s\n", prefix, lemp->symbols[1]->name); lineno++; + fprintf(out,"#ifndef %s%s\n", prefix, lemp->symbols[1]->name); } for(i=1; i<lemp->nterminal; i++){ fprintf(out,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i); @@ -4621,33 +4612,25 @@ void ReportTable( fprintf(out,"#define %sARG_FETCH\n",name); lineno++; fprintf(out,"#define %sARG_STORE\n",name); lineno++; } - fprintf(out, "#undef YYREALLOC\n"); lineno++; if( lemp->reallocFunc ){ fprintf(out,"#define YYREALLOC %s\n", lemp->reallocFunc); lineno++; }else{ fprintf(out,"#define YYREALLOC realloc\n"); lineno++; } - fprintf(out, "#undef YYFREE\n"); lineno++; if( lemp->freeFunc ){ fprintf(out,"#define YYFREE %s\n", lemp->freeFunc); lineno++; }else{ fprintf(out,"#define YYFREE free\n"); lineno++; } - fprintf(out, "#undef YYDYNSTACK\n"); lineno++; if( lemp->reallocFunc && lemp->freeFunc ){ fprintf(out,"#define YYDYNSTACK 1\n"); lineno++; }else{ fprintf(out,"#define YYDYNSTACK 0\n"); lineno++; } - fprintf(out, "#undef YYSIZELIMIT\n"); lineno++; - if( notnull(lemp->ctx) ){ + if( lemp->ctx && lemp->ctx[0] ){ i = lemonStrlen(lemp->ctx); while( i>=1 && ISSPACE(lemp->ctx[i-1]) ) i--; while( i>=1 && (ISALNUM(lemp->ctx[i-1]) || lemp->ctx[i-1]=='_') ) i--; - if( notnull(lemp->stackSizeLimit) ){ - fprintf(out,"#define YYSIZELIMIT %s\n", lemp->stackSizeLimit); lineno++; - } - fprintf(out,"#define %sCTX(P) ((P)->%s)\n",name,&lemp->ctx[i]); lineno++; fprintf(out,"#define %sCTX_SDECL %s;\n",name,lemp->ctx); lineno++; fprintf(out,"#define %sCTX_PDECL ,%s\n",name,lemp->ctx); lineno++; fprintf(out,"#define %sCTX_PARAM ,%s\n",name,&lemp->ctx[i]); lineno++; @@ -4656,7 +4639,6 @@ void ReportTable( fprintf(out,"#define %sCTX_STORE yypParser->%s=%s;\n", name,&lemp->ctx[i],&lemp->ctx[i]); lineno++; }else{ - fprintf(out,"#define %sCTX(P) 0\n",name); lineno++; fprintf(out,"#define %sCTX_SDECL\n",name); lineno++; fprintf(out,"#define %sCTX_PDECL\n",name); lineno++; fprintf(out,"#define %sCTX_PARAM\n",name); lineno++; @@ -4666,13 +4648,10 @@ void ReportTable( if( mhflag ){ fprintf(out,"#endif\n"); lineno++; } - fprintf(out, "#undef YYERRORSYMBOL\n"); lineno++; - fprintf(out, "#undef YYERRSYMDT\n"); lineno++; if( lemp->errsym && lemp->errsym->useCnt ){ fprintf(out,"#define YYERRORSYMBOL %d\n",lemp->errsym->index); lineno++; fprintf(out,"#define YYERRSYMDT yy%d\n",lemp->errsym->dtnum); lineno++; } - fprintf(out,"#undef YYFALLBACK\n"); lineno++; if( lemp->has_fallback ){ fprintf(out,"#define YYFALLBACK 1\n"); lineno++; } @@ -5024,6 +5003,7 @@ void ReportTable( sp2->destLineno = -1; /* Avoid emitting this destructor again */ } } + emit_destructor_code(out,lemp->symbols[i],lemp,&lineno); fprintf(out," break;\n"); lineno++; } @@ -5123,6 +5103,7 @@ void ReportTable( acttab_free(pActtab); fclose(in); fclose(out); + if( sql ) fclose(sql); return; } diff --git a/tool/lempar.c b/tool/lempar.c index 7b654e650..74314efea 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -299,24 +299,15 @@ static int yyGrowStack(yyParser *p){ int newSize; int idx; yyStackEntry *pNew; -#ifdef YYSIZELIMIT - int nLimit = YYSIZELIMIT(ParseCTX(p)); -#endif newSize = oldSize*2 + 100; -#ifdef YYSIZELIMIT - if( newSize>nLimit ){ - newSize = nLimit; - if( newSize<=oldSize ) return 1; - } -#endif idx = (int)(p->yytos - p->yystack); if( p->yystack==p->yystk0 ){ - pNew = YYREALLOC(0, newSize*sizeof(pNew[0]), ParseCTX(p)); + pNew = YYREALLOC(0, newSize*sizeof(pNew[0])); if( pNew==0 ) return 1; memcpy(pNew, p->yystack, oldSize*sizeof(pNew[0])); }else{ - pNew = YYREALLOC(p->yystack, newSize*sizeof(pNew[0]), ParseCTX(p)); + pNew = YYREALLOC(p->yystack, newSize*sizeof(pNew[0])); if( pNew==0 ) return 1; } p->yystack = pNew; @@ -468,9 +459,7 @@ void ParseFinalize(void *p){ } #if YYGROWABLESTACK - if( pParser->yystack!=pParser->yystk0 ){ - YYFREE(pParser->yystack, ParseCTX(pParser)); - } + if( pParser->yystack!=pParser->yystk0 ) YYFREE(pParser->yystack); #endif } diff --git a/tool/mkautoconfamal.sh b/tool/mkautoconfamal.sh index 002a3b8ee..3835799a6 100644 --- a/tool/mkautoconfamal.sh +++ b/tool/mkautoconfamal.sh @@ -59,8 +59,7 @@ cp $TOP/src/sqlite3.rc $TMPSPACE cp $TOP/tool/Replace.cs $TMPSPACE cp $TOP/VERSION $TMPSPACE cp $TOP/main.mk $TMPSPACE -cp $TOP/make.bat $TMPSPACE -tree $TMPSPACE + cd $TMPSPACE #if true; then diff --git a/tool/mkcombo.tcl b/tool/mkcombo.tcl deleted file mode 100644 index 71368ec41..000000000 --- a/tool/mkcombo.tcl +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/tclsh -# -# Use this script to combine multiple source code files into a single -# file. Example: -# -# tclsh mkcombo.tcl file1.c file2.c file3.c -o file123.c -# - -set help {Usage: tclsh mkcombo.tcl [OPTIONS] [FILELIST] - where OPTIONS is zero or more of the following with these effects: - --linemacros=? => Emit #line directives into output or not. (? = 1 or 0) - --o FILE => write to alternative output file named FILE - --help => See this. -} - -set linemacros 0 -set fname {} -set src [list] - - -for {set i 0} {$i<[llength $argv]} {incr i} { - set x [lindex $argv $i] - if {[regexp {^-?-linemacros(?:=([01]))?$} $x ma ulm]} { - if {$ulm == ""} {set ulm 1} - set linemacros $ulm - } elseif {[regexp {^-o$} $x]} { - incr i - if {$i==[llength $argv]} { - error "No argument following $x" - } - set fname [lindex $argv $i] - } elseif {[regexp {^-?-((help)|\?)$} $x]} { - puts $help - exit 0 - } elseif {[regexp {^-?-} $x]} { - error "unknown command-line option: $x" - } else { - lappend src $x - } -} - -# Open the output file and write a header comment at the beginning -# of the file. -# -if {![info exists fname]} { - set fname sqlite3.c - if {$enable_recover} { set fname sqlite3r.c } -} -set out [open $fname wb] - -# Return a string consisting of N "*" characters. -# -proc star N { - set r {} - for {set i 0} {$i<$N} {incr i} {append r *} - return $r -} - -# Force the output to use unix line endings, even on Windows. -fconfigure $out -translation binary -puts $out "/[star 78]" -puts $out {** The following is an amalgamation of these source code files:} -puts $out {**} -foreach s $src { - regsub {^.*/(src|ext)/} $s {\1/} s2 - puts $out "** $s2" -} -puts $out {**} -puts $out "[star 78]/" - -# Insert a comment into the code -# -proc section_comment {text} { - global out s78 - set n [string length $text] - set nstar [expr {60 - $n}] - puts $out "/************** $text [star $nstar]/" -} - -# Read the source file named $filename and write it into the -# sqlite3.c output file. The only transformation is the trimming -# of EOL whitespace. -# -proc copy_file_verbatim {filename} { - global out - set in [open $filename rb] - set tail [file tail $filename] - section_comment "Begin file $tail" - while {![eof $in]} { - set line [string trimright [gets $in]] - puts $out $line - } - section_comment "End of $tail" -} -set taillist "" -foreach file $src { - copy_file_verbatim $file - append taillist ", [file tail $file]" -} - -set taillist "End of the amalgamation of [string range $taillist 2 end]" -set n [string length $taillist] -set ns [expr {(75-$n)/2}] -if {$ns<3} {set ns 3} -puts $out "/[star $ns] $taillist [star $ns]/" -close $out diff --git a/tool/mkkeywordhash.c b/tool/mkkeywordhash.c index d6d54a5a4..188c0a29a 100644 --- a/tool/mkkeywordhash.c +++ b/tool/mkkeywordhash.c @@ -668,8 +668,8 @@ int main(int argc, char **argv){ printf("/* Check to see if z[0..n-1] is a keyword. If it is, write the\n"); printf("** parser symbol code for that keyword into *pType. Always\n"); printf("** return the integer n (the length of the token). */\n"); - printf("static i64 keywordCode(const char *z, i64 n, int *pType){\n"); - printf(" i64 i, j;\n"); + printf("static int keywordCode(const char *z, int n, int *pType){\n"); + printf(" int i, j;\n"); printf(" const char *zKW;\n"); printf(" assert( n>=2 );\n"); printf(" i = ((charMap(z[0])*%d) %c", HASH_C0, HASH_CC); diff --git a/tool/mkshellc.tcl b/tool/mkshellc.tcl index 45452621b..2f7a6ea25 100644 --- a/tool/mkshellc.tcl +++ b/tool/mkshellc.tcl @@ -13,90 +13,28 @@ set topdir [file dir [file dir [file normal $argv0]]] set out stdout fconfigure stdout -translation binary if {[lindex $argv 0]!=""} { - set output_file [lindex $argv 0] - file delete -force $output_file - set out [open $output_file wb] -} else { - set output_file {} + set out [open [lindex $argv 0] wb] } - -############################## FIRST PASS ################################ -# Read through the shell.c.in source file to gather information. Do not -# yet generate any code -# -set in [open $topdir/src/shell.c.in] -fconfigure $in -translation binary -set allSource(src/shell.c.in) 1 -set inUsage 0 -set dotcmd {} -while {1} { - set lx [gets $in] - if {[eof $in]} break; - if {[regexp {^INCLUDE } $lx]} { - set cfile [lindex $lx 1] - if {[string match ../* $cfile]} { - set xfile [string range $cfile 3 end] - } else { - set xfile "src/$cfile" - } - set allSource($xfile) 1 - } elseif {[regexp {^\*\* USAGE:\s+([^\s]+)} $lx all dotcmd]} { - set inUsage 1 - set details [string trim [string range $lx 2 end]] - } elseif {$inUsage} { - if {![regexp {^\*\*} $lx] || [regexp { DOT-COMMAND: } $lx]} { - set inUsage 0 - set Usage($dotcmd) [string trim $details] - } else { - append details \n - append details [string range [string trimright $lx] 3 end] - } - } -} - -# Generate dot-command usage text based on the data accumulated in -# the Usage() array. -# -proc generate_usage {out} { - global Usage - puts $out "/**************************************************************" - puts $out "** \"Usage\" help text automatically generated from comments */" - puts $out "static const struct \173" - puts $out " const char *zCmd; /* Name of the dot-command */" - puts $out " const char *zUsage; /* Documentation */" - puts $out "\175 aUsage\[\] = \173" - foreach dotcmd [array names Usage] { - puts $out " \173 \"$dotcmd\"," - foreach line [split $Usage($dotcmd) \n] { - set x [string map [list \\ \\\\ \" \\\"] $line] - puts $out "\"$x\\n\"" - } - puts $out " \175," - } - puts $out "\175;" -} -# generate_usage stderr - -###### SECOND PASS ####### -# Make a second pass through shell.c.in to generate the the final -# output, based on data gathered during the first pass. -# - -puts $out {/* -** This is the amalgamated source code to the "sqlite3" or "sqlite3.exe" -** command-line shell (CLI) for SQLite. This file is automatically -** generated by the tool/mkshellc.tcl script from the following sources: -**} -foreach fn [lsort [array names allSource]] { - puts $out "** $fn" -} -puts $out {** +puts $out {/* DO NOT EDIT! +** This file is automatically generated by the script in the canonical +** SQLite source tree at tool/mkshellc.tcl. That script combines source +** code from various constituent source files of SQLite into this single +** "shell.c" file used to implement the SQLite command-line shell. +** +** Most of the code found below comes from the "src/shell.c.in" file in +** the canonical SQLite source tree. That main file contains "INCLUDE" +** lines that specify other files in the canonical source tree that are +** inserted to getnerate this complete program source file. +** +** The code from multiple files is combined into this single "shell.c" +** source file to help make the command-line program easier to compile. +** ** To modify this program, get a copy of the canonical SQLite source tree, -** edit the src/shell.c.in file and/or some of the other files that are -** listed above, then rerun the command "make shell.c". +** edit the src/shell.c.in" and/or some of the other files that are included +** by "src/shell.c.in", then rerun the tool/mkshellc.tcl script. */} -seek $in 0 start -puts $out "/************************* Begin src/shell.c.in ******************/" +set in [open $topdir/src/shell.c.in] +fconfigure $in -translation binary proc omit_redundant_typedefs {line} { global typedef_seen if {[regexp {^typedef .* ([a-zA-Z0-9_]+);} $line all typename]} { @@ -115,14 +53,9 @@ while {1} { incr iLine if {[regexp {^INCLUDE } $lx]} { set cfile [lindex $lx 1] - if {[string match ../* $cfile]} { - set xfile [string range $cfile 3 end] - } else { - set xfile "src/$cfile" - } - puts $out "/************************* Begin $xfile ******************/" -# puts $out "#line 1 \"$xfile\"" - set in2 [open $topdir/$xfile] + puts $out "/************************* Begin $cfile ******************/" +# puts $out "#line 1 \"$cfile\"" + set in2 [open $topdir/src/$cfile] fconfigure $in2 -translation binary while {![eof $in2]} { set lx [omit_redundant_typedefs [gets $in2]] @@ -136,14 +69,11 @@ while {1} { puts $out $lx } close $in2 - puts $out "/************************* End $xfile ********************/" + puts $out "/************************* End $cfile ********************/" # puts $out "#line [expr $iLine+1] \"shell.c.in\"" - } elseif {[regexp {^INSERT-USAGE-TEXT-HERE} $lx]} { - generate_usage $out - } else { - puts $out $lx + continue } + puts $out $lx } -puts $out "/************************* End src/shell.c.in ******************/" close $in close $out diff --git a/tool/omittest.tcl b/tool/omittest.tcl index 03c9220cd..0452a4c6f 100644 --- a/tool/omittest.tcl +++ b/tool/omittest.tcl @@ -123,6 +123,7 @@ set CompileOptionsToTest { SQLITE_ENABLE_MEMSYS SQLITE_ENABLE_MODULE_COMMENTS SQLITE_ENABLE_MULTIPLEX + SQLITE_ENABLE_MULTITHREADED_CHECKS SQLITE_ENABLE_NORMALIZE SQLITE_ENABLE_NULL_TRIM SQLITE_ENABLE_OFFSET_SQL_FUNC @@ -151,7 +152,6 @@ set CompileOptionsToTest { SQLITE_ENABLE_VFSTRACE SQLITE_ENABLE_WHERETRACE SQLITE_ENABLE_ZIPVFS - SQLITE_THREAD_MISUSE_WARNINGS } # Parse command-line options. diff --git a/tool/showdb.c b/tool/showdb.c index 84813b839..f0bd9737c 100644 --- a/tool/showdb.c +++ b/tool/showdb.c @@ -8,7 +8,6 @@ #include <sys/types.h> #include <sys/stat.h> #include <fcntl.h> -#include <stdint.h> #if !defined(_MSC_VER) #include <unistd.h> @@ -29,21 +28,13 @@ typedef sqlite3_uint64 u64; /* unsigned 64-bit */ static struct GlobalData { i64 pagesize; /* Size of a database page */ - i64 usablesize; /* pagesize-nRes */ int dbfd; /* File descriptor for reading the DB */ u32 mxPage; /* Last page number */ - u32 nRes; /* Amount of reserve space */ int perLine; /* HEX elements to print per line */ int bRaw; /* True to access db file via OS APIs */ - int bCSV; /* CSV output for "pgidx" */ - int bTmstmp; /* Interpret tmstmpvfs tags on "pgidx" */ sqlite3_file *pFd; /* File descriptor for non-raw mode */ sqlite3 *pDb; /* Database handle that owns pFd */ - char **zPageUse; /* Use for each page */ - struct TmstmpTag { - unsigned char a[16]; /* tmstmpvfs tag for each page */ - } *aPageTag; -} g = {4096, 4096, -1, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0}; +} g = {1024, -1, 0, 16, 0, 0, 0}; /* ** Convert the var-int format into i64. Return the number of bytes @@ -151,12 +142,11 @@ static void fileClose(){ static unsigned char *fileRead(sqlite3_int64 ofst, int nByte){ unsigned char *aData; int got; - int rc; aData = sqlite3_malloc64(32+(i64)nByte); if( aData==0 ) out_of_memory(); memset(aData, 0, nByte+32); if( g.bRaw==0 ){ - rc = g.pFd->pMethods->xRead(g.pFd, (void*)aData, nByte, ofst); + int rc = g.pFd->pMethods->xRead(g.pFd, (void*)aData, nByte, ofst); if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ fprintf(stderr, "error in xRead() - %d\n", rc); exit(1); @@ -164,21 +154,7 @@ static unsigned char *fileRead(sqlite3_int64 ofst, int nByte){ }else{ lseek(g.dbfd, (long)ofst, SEEK_SET); got = read(g.dbfd, aData, nByte); - if( got==nByte ){ - rc = SQLITE_OK; - }else if( got>0 && got<nByte ){ - memset(aData+got, 0, nByte-got); - rc = SQLITE_IOERR_SHORT_READ; - }else{ - memset(aData,0,nByte); - rc = SQLITE_IOERR; - } - } - if( g.aPageTag && nByte==(int)g.pagesize ){ - unsigned int pgno = (unsigned int)(ofst/g.pagesize) + 1; - if( pgno>0 && pgno<=g.mxPage ){ - memcpy(g.aPageTag[pgno].a, &aData[nByte-16], 16); - } + if( got>0 && got<nByte ) memset(aData+got, 0, nByte-got); } return aData; } @@ -406,14 +382,14 @@ static i64 localPayload(i64 nPayload, char cType){ i64 nLocal; if( cType==13 ){ /* Table leaf */ - maxLocal = g.usablesize-35; - minLocal = (g.usablesize-12)*32/255-23; + maxLocal = g.pagesize-35; + minLocal = (g.pagesize-12)*32/255-23; }else{ - maxLocal = (g.usablesize-12)*64/255-23; - minLocal = (g.usablesize-12)*32/255-23; + maxLocal = (g.pagesize-12)*64/255-23; + minLocal = (g.pagesize-12)*32/255-23; } if( nPayload>maxLocal ){ - surplus = minLocal + (nPayload-minLocal)%(g.usablesize-4); + surplus = minLocal + (nPayload-minLocal)%(g.pagesize-4); if( surplus<=maxLocal ){ nLocal = surplus; }else{ @@ -775,7 +751,7 @@ static void decode_trunk_page( print_decode_line(a, 4, 4, "Number of entries on this page"); if( detail ){ n = decodeInt32(&a[4]); - for(i=0; i<n && i<g.usablesize/4; i++){ + for(i=0; i<n && i<g.pagesize/4; i++){ u32 x = decodeInt32(&a[8+4*i]); char zIdx[13]; sprintf(zIdx, "[%d]", i); @@ -793,6 +769,11 @@ static void decode_trunk_page( } } +/* +** A short text comment on the use of each page. +*/ +static char **zPageUse; + /* ** Add a comment on the use of a page. */ @@ -809,13 +790,13 @@ static void page_usage_msg(u32 pgno, const char *zFormat, ...){ sqlite3_free(zMsg); return; } - if( g.zPageUse[pgno]!=0 ){ + if( zPageUse[pgno]!=0 ){ printf("ERROR: page %d used multiple times:\n", pgno); - printf("ERROR: previous: %s\n", g.zPageUse[pgno]); + printf("ERROR: previous: %s\n", zPageUse[pgno]); printf("ERROR: current: %s\n", zMsg); - sqlite3_free(g.zPageUse[pgno]); + sqlite3_free(zPageUse[pgno]); } - g.zPageUse[pgno] = zMsg; + zPageUse[pgno] = zMsg; } /* @@ -856,7 +837,7 @@ static void page_usage_cell( while( ovfl && (cnt++)<g.mxPage ){ page_usage_msg(ovfl, "overflow %d from cell %d of page %u", cnt, cellno, pgno); - a = fileRead((ovfl-1)*(sqlite3_int64)g.pagesize, g.pagesize); + a = fileRead((ovfl-1)*(sqlite3_int64)g.pagesize, 4); ovfl = decodeInt32(a); sqlite3_free(a); } @@ -935,12 +916,12 @@ static void page_usage_btree( u32 ofst; cellidx = cellstart + i*2; - if( cellidx+1 >= g.usablesize ){ + if( cellidx+1 >= g.pagesize ){ printf("ERROR: page %d too many cells (%d)\n", pgno, nCell); break; } ofst = a[cellidx]*256 + a[cellidx+1]; - if( ofst<cellidx+2 || ofst+4>=g.usablesize ){ + if( ofst<cellidx+2 || ofst+4>=g.pagesize ){ printf("ERROR: page %d cell %d out of bounds\n", pgno, i); continue; } @@ -978,9 +959,9 @@ static void page_usage_freelist(u32 pgno){ a = fileRead((pgno-1)*g.pagesize, g.pagesize); iNext = decodeInt32(a); n = decodeInt32(a+4); - if( n>(g.usablesize - 8)/4 ){ + if( n>(g.pagesize - 8)/4 ){ printf("ERROR: page %d too many freelist entries (%d)\n", pgno, n); - n = (g.usablesize - 8)/4; + n = (g.pagesize - 8)/4; } for(i=0; i<n; i++){ int child = decodeInt32(a + (i*4+8)); @@ -1009,63 +990,6 @@ static void page_usage_ptrmap(u8 *a){ } } -/* -** The six bytes at a[] are a big-endian unsigned integer which is the -** number of milliseconds since 1970. Decode that value into an ISO 8601 -** date/time string stored in static space and return a pointer to that -** string. -*/ -static const char *decodeTimestamp(const unsigned char *a){ - uint64_t ms; /* Milliseconds since 1970 */ - uint64_t days; /* Days since 1970-01-01 */ - uint64_t sod; /* Start of date specified by ms */ - uint64_t z; /* Days since 0000-03-01 */ - uint64_t era; /* 400-year era */ - int i; /* Loop counter */ - int h; /* hour */ - int m; /* minute */ - int s; /* second */ - int f; /* millisecond */ - int Y; /* year */ - int M; /* month */ - int D; /* day */ - int y; /* year assuming March is first month */ - unsigned int doe; /* day of 400-year era */ - unsigned int yoe; /* year of 400-year era */ - unsigned int doy; /* day of year */ - unsigned int mp; /* month with March==0 */ - static char zOut[50]; /* Return results here */ - - for(ms=0, i=0; i<=5; i++) ms = (ms<<8) + a[i]; - if( ms==0 ){ - return " "; - }else if( ms>4102444800000LL ){ /* 2100-01-01 */ - /* YYYY-MM-DD HH:MM:SS.SSS */ - return " (bad date) "; - } - days = ms/86400000; - sod = (ms%86400000)/1000; - f = (int)(ms%1000); - - h = sod/3600; - m = (sod%3600)/60; - s = sod%60; - z = days + 719468; - era = z/146097; - doe = (unsigned)(z - era*146097); - yoe = (doe - doe/1460 + doe/36524 - doe/146096)/365; - y = (int)yoe + era*400; - doy = doe - (365*yoe + yoe/4 - yoe/100); - mp = (5*doy + 2)/153; - D = doy - (153*mp + 2)/5 + 1; - M = mp + (mp<10 ? 3 : -9); - Y = y + (M <=2); - snprintf(zOut, sizeof(zOut), - "%04d-%02d-%02d %02d:%02d:%02d.%03d", - Y, M, D, h, m, s, f); - return zOut; -} - /* ** Try to figure out how every page in the database file is being used. */ @@ -1086,22 +1010,14 @@ static void page_usage_report(const char *zPrg, const char *zDbName){ /* Open the database file */ db = openDatabase(zPrg, zDbName); - /* Set up global variables g.zPageUse[] and g.mxPage to record page + /* Set up global variables zPageUse[] and g.mxPage to record page ** usages */ - g.zPageUse = sqlite3_malloc64( sizeof(g.zPageUse[0])*(g.mxPage+1) ); - if( g.zPageUse==0 ) out_of_memory(); - memset(g.zPageUse, 0, sizeof(g.zPageUse[0])*(g.mxPage+1)); + zPageUse = sqlite3_malloc64( sizeof(zPageUse[0])*(g.mxPage+1) ); + if( zPageUse==0 ) out_of_memory(); + memset(zPageUse, 0, sizeof(zPageUse[0])*(g.mxPage+1)); /* Discover the usage of each page */ a = fileRead(0, 100); - if( g.bTmstmp && a[20]==16 ){ - g.aPageTag = sqlite3_malloc64( sizeof(struct TmstmpTag)*(g.mxPage+1) ); - if( g.aPageTag==0 ) out_of_memory(); - memset(g.aPageTag, 0, sizeof(struct TmstmpTag)*(g.mxPage+1) ); - }else{ - g.bTmstmp = 0; - g.aPageTag = 0; - } page_usage_freelist(decodeInt32(a+32)); page_usage_ptrmap(a); sqlite3_free(a); @@ -1126,68 +1042,15 @@ static void page_usage_report(const char *zPrg, const char *zDbName){ sqlite3_close(db); /* Print the report and free memory used */ - if( g.bCSV ){ - if( g.bTmstmp ){ - printf("pgno,tm,frame,flg,salt,parent,child,ovfl,txt\r\n"); - }else{ - printf("pgno,parent,child,ovfl,txt\r\n"); - } - } for(i=1; i<=g.mxPage; i++){ - if( g.zPageUse[i]==0 ){ - g.zPageUse[i] = sqlite3_mprintf("???"); - if( g.zPageUse[i]==0 ) continue; - } - if( g.bCSV ){ - const char *z = g.zPageUse[i]; - const char *s; - printf("%u,", i); - if( g.bTmstmp ){ - const unsigned char *a = g.aPageTag[i].a; - sqlite3_uint64 tm = 0; - unsigned int x; - int k; - for(k=2; k<=7; k++) tm = (tm<<8)+a[k]; - printf("%llu.%03u,", tm/1000, (unsigned int)(tm%1000)); - for(x=0, k=8; k<=11; k++) x = (x<<8)+a[k]; - printf("%u,", x); - printf("%u,", a[12]); - for(x=0, k=13; k<=15; k++) x = (x<<8)+a[k]; - printf("%u,", x); - } - if( (s = strstr(z, " of page "))!=0 ){ - printf("%d,", atoi(s+9)); - }else if( (s = strstr(z, " of trunk page "))!=0 ){ - printf("%d,", atoi(s+15)); - }else{ - printf("0,"); - } - if( (s = strstr(z, "], child "))!=0 ){ - printf("%d,", atoi(s+9)); - }else if( (s = strstr(z, " from cell "))!=0 ){ - printf("%d,", atoi(s+12)); - }else{ - printf("-1,"); - } - if( strncmp(z,"overflow ", 9)==0 ){ - printf("%d,", atoi(z+9)); - }else{ - printf("-1,"); - } - printf("\"%s\"\r\n", z); - }else if( g.bTmstmp ){ - printf("%5u: %s %s\n", i, - decodeTimestamp(&g.aPageTag[i].a[2]), - g.zPageUse[i]); - }else{ - printf("%5u: %s\n", i, g.zPageUse[i]); - } + if( zPageUse[i]==0 ) page_usage_btree(i, -1, 0, 0); + printf("%5u: %s\n", i, zPageUse[i] ? zPageUse[i] : "???"); } for(i=1; i<=g.mxPage; i++){ - sqlite3_free(g.zPageUse[i]); + sqlite3_free(zPageUse[i]); } - sqlite3_free(g.zPageUse); - g.zPageUse = 0; + sqlite3_free(zPageUse); + zPageUse = 0; } /* @@ -1220,7 +1083,7 @@ static void ptrmap_coverage_report(const char *zDbName){ for(pgno=2; pgno<=g.mxPage; pgno += perPage+1){ printf("%5llu: PTRMAP page covering %llu..%llu\n", pgno, pgno+1, pgno+perPage); - a = fileRead((pgno-1)*g.pagesize, g.pagesize); + a = fileRead((pgno-1)*g.pagesize, usable); for(i=0; i+5<=usable; i+=5){ const char *zType; u32 iFrom = decodeInt32(&a[i+1]); @@ -1252,7 +1115,7 @@ static void ptrmap_coverage_report(const char *zDbName){ ** Check the range validity for a page number. Print an error and ** exit if the page is out of range. */ -static void checkPageValidity(unsigned int iPage){ +static void checkPageValidity(int iPage){ if( iPage<1 || iPage>g.mxPage ){ fprintf(stderr, "Invalid page number %d: valid range is 1..%d\n", iPage, g.mxPage); @@ -1267,9 +1130,7 @@ static void usage(const char *argv0){ fprintf(stderr, "Usage %s ?--uri? FILENAME ?args...?\n\n", argv0); fprintf(stderr, "switches:\n" - " --csv CSV output for \"pgidx\"\n" " --raw Read db file directly, bypassing SQLite VFS\n" - " --tmstmp Interpret tmstmpvfs tags\n" "args:\n" " dbheader Show database header\n" " pgidx Index of how each page is used\n" @@ -1293,28 +1154,14 @@ int main(int argc, char **argv){ char **azArg = argv; int nArg = argc; - /* Check for the switches. */ - while( nArg>1 && azArg[1][0]=='-' ){ - const char *z = azArg[1]; - if( z[1]=='-' && z[2]!=0 ) z++; - if( sqlite3_stricmp("-raw", z)==0 ){ + /* Check for the "--uri" or "-uri" switch. */ + if( nArg>1 ){ + if( sqlite3_stricmp("-raw", azArg[1])==0 + || sqlite3_stricmp("--raw", azArg[1])==0 + ){ g.bRaw = 1; azArg++; nArg--; - }else - if( strcmp("-csv", z)==0 ){ - g.bCSV = 1; - azArg++; - nArg--; - }else - if( strcmp("-tmstmp", z)==0 ){ - g.bTmstmp = 1; - azArg++; - nArg--; - }else - { - usage(zPrg); - exit(1); } } @@ -1326,19 +1173,15 @@ int main(int argc, char **argv){ fileOpen(zPrg, azArg[1]); szFile = fileGetsize(); - zPgSz = fileRead(0, 24); - g.pagesize = zPgSz[16]*256 + zPgSz[17]*65536; - if( g.pagesize==0 ) g.pagesize = 4096; - g.nRes = zPgSz[20]; - g.usablesize = g.pagesize - g.nRes; + zPgSz = fileRead(16, 2); + g.pagesize = zPgSz[0]*256 + zPgSz[1]*65536; + if( g.pagesize==0 ) g.pagesize = 1024; sqlite3_free(zPgSz); + + printf("Pagesize: %d\n", (int)g.pagesize); g.mxPage = (u32)((szFile+g.pagesize-1)/g.pagesize); - if( !g.bCSV ){ - printf("Pagesize: %d\n", (int)g.pagesize); - if( g.nRes ) printf("Useable-size: %d\n", (int)g.usablesize); - printf("Available pages: 1..%u\n", g.mxPage); - } + printf("Available pages: 1..%u\n", g.mxPage); if( nArg==2 ){ u32 i; for(i=1; i<=g.mxPage; i++) print_page(i); diff --git a/tool/showtmlog.c b/tool/showtmlog.c deleted file mode 100644 index 4c35b777b..000000000 --- a/tool/showtmlog.c +++ /dev/null @@ -1,254 +0,0 @@ -/* -** A utility program to decode tmstmpvfs log files. -*/ -#include <stdio.h> -#include <assert.h> -#include <string.h> -#include <stdint.h> -#include <stdlib.h> - -/* -** The six bytes at a[] are a big-endian unsigned integer which is the -** number of milliseconds since 1970. Decode that value into an ISO 8601 -** date/time string stored in static space and return a pointer to that -** string. -*/ -static const char *decodeTimestamp(const unsigned char *a){ - uint64_t ms; /* Milliseconds since 1970 */ - uint64_t days; /* Days since 1970-01-01 */ - uint64_t sod; /* Start of date specified by ms */ - uint64_t z; /* Days since 0000-03-01 */ - uint64_t era; /* 400-year era */ - int i; /* Loop counter */ - int h; /* hour */ - int m; /* minute */ - int s; /* second */ - int f; /* millisecond */ - int Y; /* year */ - int M; /* month */ - int D; /* day */ - int y; /* year assuming March is first month */ - unsigned int doe; /* day of 400-year era */ - unsigned int yoe; /* year of 400-year era */ - unsigned int doy; /* day of year */ - unsigned int mp; /* month with March==0 */ - static char zOut[50]; /* Return results here */ - - for(ms=0, i=0; i<=5; i++) ms = (ms<<8) + a[i]; - if( ms==0 ){ - return " "; - }else if( ms>4102444800000LL ){ /* 2100-01-01 */ - /* YYYY-MM-DD HH:MM:SS.SSS */ - return " (bad date) "; - } - days = ms/86400000; - sod = (ms%86400000)/1000; - f = (int)(ms%1000); - - h = sod/3600; - m = (sod%3600)/60; - s = sod%60; - z = days + 719468; - era = z/146097; - doe = (unsigned)(z - era*146097); - yoe = (doe - doe/1460 + doe/36524 - doe/146096)/365; - y = (int)yoe + era*400; - doy = doe - (365*yoe + yoe/4 - yoe/100); - mp = (5*doy + 2)/153; - D = doy - (153*mp + 2)/5 + 1; - M = mp + (mp<10 ? 3 : -9); - Y = y + (M <=2); - snprintf(zOut, sizeof(zOut), - "%04d-%02d-%02d %02d:%02d:%02d.%03d", - Y, M, D, h, m, s, f); - return zOut; -} - -/* -** Render a single 16-byte tmstmpvfs log record as a line to a CSV file. -** -** Columns: tmstmp,fileno,op,pid,pgno,frame,salt,txn -*/ -static void renderCSV(int iFile, unsigned char *a){ - unsigned int a2, a3; - int j; - uint64_t ms; - - for(ms=0, j=2; j<=7; j++) ms = (ms<<8) + a[j]; - printf("%u.%03u,%d,", (unsigned int)(ms/1000), (unsigned)(ms%1000), iFile); - for(a2=0, j=8; j<=11; j++) a2 = (a2<<8)+a[j]; - for(a3=0, j=12; j<=15; j++) a3 = (a3<<8)+a[j]; - switch( a[0] ){ - case 0x01: { - printf("\"open-db\",%u,,,,\r\n",a2); - break; - } - case 0x02: { - printf("\"open-wal\",%u,,,,\r\n", a2); - break; - } - case 0x03: { - printf("\"wal-page\",,%u,%u,,%d\r\n", a2, a3, a[1]); - break; - } - case 0x04: { - printf("\"db-page\",,%u,,,\r\n", a2); - break; - } - case 0x05: { - printf("\"ckpt-start\",,,,,\r\n"); - break; - } - case 0x06: { - printf("\"ckpt-page\",,%u,%u,,\r\n", a2, a3); - break; - } - case 0x07: { - printf("\"ckpt-end\",,,,,\r\n"); - break; - } - case 0x08: { - printf("\"wal-reset\",,,,%u,\r\n", a3); - break; - } - case 0x0e: { - printf("\"close-wal\",,,,,\r\n"); - break; - } - case 0x0f: { - printf("\"close-db\",,,,,\r\n"); - break; - } - default: { - printf("\"invalid-record\",,,,,\r\n"); - break; - } - } -} - -/* -** Render a single 16-byte tmstmpvfs log record as human-readable text -** on stdout. -*/ -static void renderText(unsigned char *a){ - unsigned int a2, a3; - int j; - - printf("%s ", decodeTimestamp(a+2)); - for(a2=0, j=8; j<=11; j++) a2 = (a2<<8)+a[j]; - for(a3=0, j=12; j<=15; j++) a3 = (a3<<8)+a[j]; - switch( a[0] ){ - case 0x01: { - printf("open-db pid %u\n", a2); - break; - } - case 0x02: { - printf("open-wal pid %u\n", a2); - break; - } - case 0x03: { - printf("wal-page pgno %-8u frame %-8u%s\n", a2, a3, - a[1]==1 ? " txn" : ""); - break; - } - case 0x04: { - printf("db-page pgno %-8u\n", a2); - break; - } - case 0x05: { - printf("ckpt-start\n"); - break; - } - case 0x06: { - printf("ckpt-page pgno %-8u frame %-8u\n", a2, a3); - break; - } - case 0x07: { - printf("ckpt-end\n"); - break; - } - case 0x08: { - printf("wal-reset salt1 0x%08x\n", a3); - break; - } - case 0x0e: { - printf("close-wal\n"); - break; - } - case 0x0f: { - printf("close-db\n"); - break; - } - default: { - printf("invalid-record\n"); - break; - } - } -} - -static void usage(const char *argv0){ - printf("Usage: %s [--csv] LOGFILE ...\n", argv0); - printf("Decode one or more tmstmpvfs log files and display the results\n" - "on stdout. Render as CSV if the --csv option is used.\n"); -} - -int main(int argc, char **argv){ - int i; - FILE *in; - unsigned char a[16]; - int bCSV = 0; - const char *z; - int nFile = 0; - int iFile; - for(i=1; i<argc; i++){ - z = argv[i]; - if( z[0]=='-' ){ - if( z[1]=='-' ) z++; - if( strcmp(z,"-csv")==0 ){ - bCSV = 1; - }else - if( strcmp(z,"-help")==0 || strcmp(z,"-?")==0 ){ - usage(argv[0]); - return 0; - }else - { - printf("unknown command-line option: \"%s\"\n", - argv[i]); - usage(argv[0]); - return 1; - } - }else{ - nFile++; - } - } - if( nFile==0 ){ - usage(argv[0]); - return 1; - } - iFile = 0; - if( bCSV ){ - printf("tmstmp,fileno,op,pid,pgno,frame,salt,txn\r\n"); - } - for(i=1; i<argc; i++){ - z = argv[i]; - if( z[0]=='-' ) continue; - in = fopen(z, "rb"); - if( in==0 ){ - printf("%s: can't open\n", z); - continue; - } - iFile++; - if( nFile>1 && !bCSV ){ - printf("*** %s ***\n", z); - } - while( 16==fread(a, 1, 16, in) ){ - if( bCSV ){ - renderCSV(iFile, a); - }else{ - renderText(a); - } - } - fclose(in); - } - return 0; -} diff --git a/tool/sqldiff.c b/tool/sqldiff.c index d27a62e14..44bf488f8 100644 --- a/tool/sqldiff.c +++ b/tool/sqldiff.c @@ -28,9 +28,6 @@ #include "sqlite3.h" #include "sqlite3_stdio.h" -typedef sqlite3_int64 i64; -typedef sqlite3_uint64 u64; - /* ** All global variables are gathered into the "g" singleton. */ @@ -205,12 +202,12 @@ static char **columnNames( int *pbRowid /* OUT: True if PK is an implicit rowid */ ){ char **az = 0; /* List of column names to be returned */ - i64 naz = 0; /* Number of entries in az[] */ + int naz = 0; /* Number of entries in az[] */ sqlite3_stmt *pStmt; /* SQL statement being run */ char *zPkIdxName = 0; /* Name of the PRIMARY KEY index */ int truePk = 0; /* PRAGMA table_info identifies the PK to use */ - i64 nPK = 0; /* Number of PRIMARY KEY columns */ - i64 i, j; /* Loop counters */ + int nPK = 0; /* Number of PRIMARY KEY columns */ + int i, j; /* Loop counters */ if( g.bSchemaPK==0 ){ /* Normal case: Figure out what the true primary key is for the table. @@ -274,7 +271,7 @@ static char **columnNames( } *pnPKey = nPK; naz = nPK; - az = sqlite3_malloc64( sizeof(char*)*(nPK+1) ); + az = sqlite3_malloc( sizeof(char*)*(nPK+1) ); if( az==0 ) runtimeError("out of memory"); memset(az, 0, sizeof(char*)*(nPK+1)); if( g.bSchemaCompare ){ @@ -291,7 +288,7 @@ static char **columnNames( || !(strcmp(sid,"rootpage")==0 ||strcmp(sid,"name")==0 ||strcmp(sid,"type")==0)){ - az = sqlite3_realloc64(az, sizeof(char*)*(naz+2) ); + az = sqlite3_realloc(az, sizeof(char*)*(naz+2) ); if( az==0 ) runtimeError("out of memory"); az[naz++] = sid; } @@ -957,7 +954,7 @@ static int rbuDeltaCreate( unsigned int i, base; char *zOrigDelta = zDelta; hash h; - i64 nHash; /* Number of hash table entries */ + int nHash; /* Number of hash table entries */ int *landmark; /* Primary hash table */ int *collide; /* Collision chain */ int lastRead = -1; /* Last byte of zSrc read by a COPY command */ @@ -985,7 +982,7 @@ static int rbuDeltaCreate( ** source file. */ nHash = lenSrc/NHASH; - collide = sqlite3_malloc64( nHash*2*sizeof(int) ); + collide = sqlite3_malloc( nHash*2*sizeof(int) ); landmark = &collide[nHash]; memset(landmark, -1, nHash*sizeof(int)); memset(collide, -1, nHash*sizeof(int)); @@ -1289,9 +1286,9 @@ static void rbudiff_one_table(const char *zTab, FILE *out){ } }else{ char *zOtaControl; - i64 nOtaControl = sqlite3_column_bytes(pStmt, nCol); + int nOtaControl = sqlite3_column_bytes(pStmt, nCol); - zOtaControl = (char*)sqlite3_malloc64(nOtaControl+1); + zOtaControl = (char*)sqlite3_malloc(nOtaControl+1); memcpy(zOtaControl, sqlite3_column_text(pStmt, nCol), nOtaControl+1); for(i=0; i<nCol; i++){ @@ -1303,11 +1300,11 @@ static void rbudiff_one_table(const char *zTab, FILE *out){ const char *aSrc = sqlite3_column_blob(pStmt, nCol+1+i); int nSrc = sqlite3_column_bytes(pStmt, nCol+1+i); const char *aFinal = sqlite3_column_blob(pStmt, i); - i64 nFinal = sqlite3_column_bytes(pStmt, i); + int nFinal = sqlite3_column_bytes(pStmt, i); char *aDelta; int nDelta; - aDelta = sqlite3_malloc64(nFinal + 60); + aDelta = sqlite3_malloc(nFinal + 60); nDelta = rbuDeltaCreate(aSrc, nSrc, aFinal, nFinal, aDelta); if( nDelta<nFinal ){ int j; @@ -1552,10 +1549,10 @@ static void changeset_one_table(const char *zTab, FILE *out){ sqlite3_stmt *pStmt; /* SQL statment */ char *zId = safeId(zTab); /* Escaped name of the table */ char **azCol = 0; /* List of escaped column names */ - i64 nCol = 0; /* Number of columns */ + int nCol = 0; /* Number of columns */ int *aiFlg = 0; /* 0 if column is not part of PK */ int *aiPk = 0; /* Column numbers for each PK column */ - i64 nPk = 0; /* Number of PRIMARY KEY columns */ + int nPk = 0; /* Number of PRIMARY KEY columns */ sqlite3_str *pSql; /* SQL for the diff query */ int i, k; /* Loop counters */ const char *zSep; /* List separator */ @@ -1567,16 +1564,16 @@ static void changeset_one_table(const char *zTab, FILE *out){ pStmt = db_prepare("PRAGMA main.table_info=%Q", zTab); while( SQLITE_ROW==sqlite3_step(pStmt) ){ nCol++; - azCol = sqlite3_realloc64(azCol, sizeof(char*)*nCol); + azCol = sqlite3_realloc(azCol, sizeof(char*)*nCol); if( azCol==0 ) runtimeError("out of memory"); - aiFlg = sqlite3_realloc64(aiFlg, sizeof(int)*nCol); + aiFlg = sqlite3_realloc(aiFlg, sizeof(int)*nCol); if( aiFlg==0 ) runtimeError("out of memory"); azCol[nCol-1] = safeId((const char*)sqlite3_column_text(pStmt,1)); aiFlg[nCol-1] = i = sqlite3_column_int(pStmt,5); if( i>0 ){ if( i>nPk ){ nPk = i; - aiPk = sqlite3_realloc64(aiPk, sizeof(int)*nPk); + aiPk = sqlite3_realloc(aiPk, sizeof(int)*nPk); if( aiPk==0 ) runtimeError("out of memory"); } aiPk[i-1] = nCol-1; @@ -1899,11 +1896,6 @@ static void showHelp(void){ ); } -/* work-around the Microsoft "WorstFit" bug */ -#ifdef _WIN32 -#define main utf8_main -#endif - int main(int argc, char **argv){ const char *zDb1 = 0; const char *zDb2 = 0; @@ -1916,7 +1908,7 @@ int main(int argc, char **argv){ FILE *out = stdout; void (*xDiff)(const char*,FILE*) = diff_one_table; #ifndef SQLITE_OMIT_LOAD_EXTENSION - i64 nExt = 0; + int nExt = 0; char **azExt = 0; #endif int useTransaction = 0; diff --git a/tool/sqlite3_rsync.c b/tool/sqlite3_rsync.c index b10224b2f..ad9f132bb 100644 --- a/tool/sqlite3_rsync.c +++ b/tool/sqlite3_rsync.c @@ -32,7 +32,6 @@ static const char zUsage[] = "\n" " --exe PATH Name of the sqlite3_rsync program on the remote side\n" " --help Show this help screen\n" - " -p|--port PORT Run SSH over TCP port PORT instead of the default 22\n" " --protocol N Use sync protocol version N.\n" " --ssh PATH Name of the SSH program used to reach the remote side\n" " -v Verbose. Multiple v's for increasing output\n" @@ -325,29 +324,15 @@ static int popen2( ** Close the connection to a child process previously created using ** popen2(). */ -static int pclose2(FILE *pIn, FILE *pOut, int childPid){ +static void pclose2(FILE *pIn, FILE *pOut, int childPid){ #ifdef _WIN32 /* Not implemented, yet */ fclose(pIn); fclose(pOut); - return 0; #else - int wp, rc = 0; fclose(pIn); fclose(pOut); - do{ - wp = waitpid(0, &rc, WNOHANG); - if( wp>0 ){ - if( WIFEXITED(rc) ){ - rc = WEXITSTATUS(rc); - }else if( WIFSIGNALED(rc) ){ - rc = WTERMSIG(rc); - }else{ - rc = 0/*???*/; - } - } - } while( wp>0 ); - return rc; + while( waitpid(0, 0, WNOHANG)>0 ) {} #endif } /***************************************************************************** @@ -2069,7 +2054,6 @@ int main(int argc, char const * const *argv){ FILE *pOut = 0; int childPid = 0; const char *zSsh = "ssh"; - int iPort = 0; const char *zExe = "sqlite3_rsync"; char *zCmd = 0; sqlite3_int64 tmStart; @@ -2110,15 +2094,6 @@ int main(int argc, char const * const *argv){ zSsh = cli_opt_val; continue; } - if( strcmp(z, "-port")==0 || strcmp(z, "-p")==0 ){ - const char *zPort = cli_opt_val; - iPort = atoi(zPort); - if( iPort<1 || iPort>65535 ){ - fprintf(stderr, "invalid TCP port number: \"%s\"\n", zPort); - return 1; - } - continue; - } if( strcmp(z, "-exe")==0 ){ zExe = cli_opt_val; continue; @@ -2260,7 +2235,6 @@ int main(int argc, char const * const *argv){ for(iRetry=0; 1 /*exit-via-break*/; iRetry++){ sqlite3_str *pStr = sqlite3_str_new(0); append_escaped_arg(pStr, zSsh, 1); - if( iPort>0 ) sqlite3_str_appendf(pStr, " -p %d", iPort); sqlite3_str_appendf(pStr, " -e none"); append_escaped_arg(pStr, ctx.zOrigin, 0); if( iRetry ) add_path_argument(pStr); @@ -2309,7 +2283,6 @@ int main(int argc, char const * const *argv){ for(iRetry=0; 1 /*exit-by-break*/; iRetry++){ sqlite3_str *pStr = sqlite3_str_new(0); append_escaped_arg(pStr, zSsh, 1); - if( iPort>0 ) sqlite3_str_appendf(pStr, " -p %d", iPort); sqlite3_str_appendf(pStr, " -e none"); append_escaped_arg(pStr, ctx.zReplica, 0); if( iRetry==1 ) add_path_argument(pStr); @@ -2376,7 +2349,7 @@ int main(int argc, char const * const *argv){ } originSide(&ctx); } - ctx.nErr += !!pclose2(ctx.pIn, ctx.pOut, childPid); + pclose2(ctx.pIn, ctx.pOut, childPid); if( ctx.pLog ) fclose(ctx.pLog); tmEnd = currentTime(); tmElapse = tmEnd - tmStart; /* Elapse time in milliseconds */ diff --git a/tool/sqltclsh.c.in b/tool/sqltclsh.c.in index 3e4d38b2d..da354ee93 100644 --- a/tool/sqltclsh.c.in +++ b/tool/sqltclsh.c.in @@ -33,7 +33,7 @@ INCLUDE $ROOT/ext/misc/appendvfs.c INCLUDE $ROOT/ext/misc/zipfile.c INCLUDE $ROOT/ext/misc/sqlar.c #endif -INCLUDE tclsqlite-ex.c +INCLUDE $ROOT/src/tclsqlite.c const char *sqlite3_tclapp_init_proc(Tcl_Interp *interp){ (void)interp; diff --git a/tool/winmain.c b/tool/winmain.c deleted file mode 100644 index 72ae00d5c..000000000 --- a/tool/winmain.c +++ /dev/null @@ -1,79 +0,0 @@ -/* -** 2025-12-26 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This file implements a substitute process entry point for windows -** that correctly interprets command-line arguments as UTF-8. This is -** a work-around for the (unfixed) Windows bug known as "WorstFit" and -** described at: -** -** * https://news.ycombinator.com/item?id=42647101 -** -** USAGE: -** -** If you have a command-line program coded to C-language standard in which -** the entry point is: -** -** int main(int argc, char **argv){...} -** -** That does not work on Windows if there are non-ASCII characters on the -** command line. Programs are expected to use an alternative entry point: -** -** int wmain(int wargc, wchar_t **wargv){...} -** -** This file implements _wmain() but then converts all of the wargv elements -** to UTF-8 and passes them off to utf8_main(). -** -** So, a command-line tool that implements the standard entry point can be -** modified by adding the following lines just prior to main(): -** -** #ifdef _WIN32 -** #define main utf8_main -** #endif -** -** Then, include this file in the list of files that implement the program, -** and the program will work correctly, even on Windows. -*/ -#include <windows.h> -#include <stdio.h> - -extern int utf8_main(int,char**); -int wmain(int argc, wchar_t **wargv){ - int rc, i; - char **argv = malloc( sizeof(char*) * (argc+1) ); - char **orig = argv; - if( argv==0 ){ - fprintf(stderr, "malloc failed\n"); - exit(1); - } - for(i=0; i<argc; i++){ - int nByte = WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, 0, 0, 0, 0); - if( nByte==0 ){ - argv[i] = 0; - }else{ - argv[i] = malloc( nByte ); - if( argv[i]==0 ){ - fprintf(stderr, "malloc failed\n"); - exit(1); - } - nByte = WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, argv[i],nByte,0,0); - if( nByte==0 ){ - free(argv[i]); - argv[i] = 0; - } - } - } - argv[argc] = 0; - rc = utf8_main(argc, argv); - for(i=0; i<argc; i++) free(orig[i]); - free(argv); - return rc; -} From 778ab890cfc30c3631212dcceb0295498abdcd3e Mon Sep 17 00:00:00 2001 From: Stephen Lombardo <sjlombardo@zetetic.net> Date: Fri, 13 Mar 2026 16:50:55 -0400 Subject: [PATCH 67/75] Update changelog to reflect upstream SQLite 3.51.3 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51fcfd558..cab05a7d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ Notable changes to this project are documented in this file. ## [4.14.0] - (? 2026 - [4.14.0 changes]) -- Updates baseline to SQLite 3.52.0 +- Updates baseline to SQLite 3.51.3 - Restores and improves upon LibTomCrypto provder - Minor test improvements From c24cab20994c38dec124f2cc4d9d047ae138df56 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo <sjlombardo@zetetic.net> Date: Wed, 18 Mar 2026 17:59:57 -0400 Subject: [PATCH 68/75] Updates version number to 4.15.0 --- CHANGELOG.md | 6 +++++- src/sqlcipher.c | 2 +- test/sqlcipher-pragmas.test | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cab05a7d2..2f4bab97c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,9 @@ # SQLCipher Change Log Notable changes to this project are documented in this file. -## [4.14.0] - (? 2026 - [4.14.0 changes]) +## [4.15.0] - (? 2026 - [4.15.0 changes]) + +## [4.14.0] - (March 2026 - [4.14.0 changes]) - Updates baseline to SQLite 3.51.3 - Restores and improves upon LibTomCrypto provder - Minor test improvements @@ -327,6 +329,8 @@ Notable changes to this project are documented in this file. ### Security - Change KDF iteration length from 4,000 to 64,000 +[4.15.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.15.0 +[4.15.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.14.0...v4.15.0 [4.14.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.14.0 [4.14.0 changes]: https://github.com/sqlcipher/sqlcipher/compare/v4.13.0...v4.14.0 [4.13.0]: https://github.com/sqlcipher/sqlcipher/tree/v4.13.0 diff --git a/src/sqlcipher.c b/src/sqlcipher.c index c93dcf26f..89aa00eb7 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -96,7 +96,7 @@ void sqlite3pager_reset(Pager *pPager); #define CIPHER_STR(s) #s #ifndef CIPHER_VERSION_NUMBER -#define CIPHER_VERSION_NUMBER 4.14.0 +#define CIPHER_VERSION_NUMBER 4.15.0 #endif #ifndef CIPHER_VERSION_BUILD diff --git a/test/sqlcipher-pragmas.test b/test/sqlcipher-pragmas.test index 74c758baf..94916790e 100644 --- a/test/sqlcipher-pragmas.test +++ b/test/sqlcipher-pragmas.test @@ -46,7 +46,7 @@ do_test verify-pragma-cipher-version { execsql { PRAGMA cipher_version; } -} {{4.14.0 community}} +} {{4.15.0 community}} db close file delete -force test.db From 752d0c867a06d3490b91b8c9ceda1767fd4fcc56 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo <sjlombardo@zetetic.net> Date: Thu, 19 Mar 2026 13:17:53 -0400 Subject: [PATCH 69/75] Removes const from pzErrMesg in sqlcipher_export_init Fixes issue #590 --- src/sqlcipher.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 89aa00eb7..75483aa28 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -412,7 +412,7 @@ static void (*const sqlcipher_fini_func)(void) __attribute__((used, section(".fi static void sqlcipher_exportFunc(sqlite3_context*, int, sqlite3_value**); -static int sqlcipher_export_init(sqlite3* db, const char** errmsg, const struct sqlite3_api_routines* api) { +static int sqlcipher_export_init(sqlite3* db, char** errmsg, const struct sqlite3_api_routines* api) { sqlite3_create_function_v2(db, "sqlcipher_export", -1, SQLITE_UTF8, 0, sqlcipher_exportFunc, 0, 0, 0); return SQLITE_OK; } From 26c7d9eb93f3ce64c6a1267fe7f8def8f34ed0f4 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo <sjlombardo@zetetic.net> Date: Wed, 8 Apr 2026 13:24:40 -0400 Subject: [PATCH 70/75] Improve error handling in sqlcipher_extra_init --- src/sqlcipher.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 75483aa28..34246c976 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -424,12 +424,15 @@ static int sqlcipher_export_init(sqlite3* db, char** errmsg, const struct sqlite int sqlcipher_extra_init(const char* arg) { int rc = SQLITE_OK, i=0; void* provider_ctx = NULL; + int mutex_held = 0; sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + mutex_held = 1; if(sqlcipher_init) { /* if this init routine already completed successfully return immediately */ sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + mutex_held = 0; return SQLITE_OK; } @@ -549,12 +552,14 @@ int sqlcipher_extra_init(const char* arg) { } default_provider->ctx_free(&provider_ctx); + provider_ctx = NULL; sqlcipher_init = 1; sqlcipher_shutdown = 0; /* leave the master mutex so we can proceed with auto extension registration */ sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + mutex_held = 0; /* finally, extension registration occurs outside of the mutex because it is * uses SQLITE_MUTEX_STATIC_MASTER itself */ @@ -578,9 +583,10 @@ int sqlcipher_extra_init(const char* arg) { sqlcipher_static_mutex[i] = NULL; } } + if(provider_ctx) default_provider->ctx_free(&provider_ctx); /* post cleanup return the error code back up to sqlite3_init() */ - sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); + if(mutex_held) sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); sqlcipher_init_error = rc; return rc; } From 48aeef303e5ed395ded1cfbdd23d72ca58ad18fd Mon Sep 17 00:00:00 2001 From: Stephen Lombardo <sjlombardo@zetetic.net> Date: Wed, 8 Apr 2026 13:44:02 -0400 Subject: [PATCH 71/75] Minor code cleanups --- src/sqlcipher.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 34246c976..56c5223d3 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -124,7 +124,7 @@ void sqlite3pager_reset(Pager *pPager); #ifndef DEFAULT_CIPHER_FLAGS -#define DEFAULT_CIPHER_FLAGS CIPHER_FLAG_HMAC | CIPHER_FLAG_LE_PGNO +#define DEFAULT_CIPHER_FLAGS (CIPHER_FLAG_HMAC | CIPHER_FLAG_LE_PGNO) #endif @@ -349,7 +349,7 @@ static void cipher_hex2bin(const unsigned char *hex, int sz, unsigned char *out) static void cipher_bin2hex(const unsigned char* in, int sz, char *out) { int i; for(i=0; i < sz; i++) { - sqlite3_snprintf(3, out + (i*2), "%02x ", in[i]); + sqlite3_snprintf(3, out + (i*2), "%02x", in[i]); } } @@ -2070,7 +2070,7 @@ static int sqlcipher_codec_ctx_migrate(codec_ctx *ctx) { memcpy(migrated_db_filename, temp, sqlite3Strlen30(temp)); sqlcipher_free(temp, sqlite3Strlen30(temp)); - attach_command = sqlite3_mprintf("ATTACH DATABASE '%s' as migrate;", migrated_db_filename, pass); + attach_command = sqlite3_mprintf("ATTACH DATABASE '%s' as migrate;", migrated_db_filename); set_user_version = sqlite3_mprintf("PRAGMA migrate.user_version = %d;", user_version); rc = sqlite3_exec(db, pragma_compat, NULL, NULL, NULL); From b936bb7df98a6d6f1628bfb2dad35e11b2dad2cc Mon Sep 17 00:00:00 2001 From: Stephen Lombardo <sjlombardo@zetetic.net> Date: Thu, 23 Apr 2026 13:09:58 -0400 Subject: [PATCH 72/75] Snapshot of upstream SQLite 3.53.0 --- Makefile.in | 16 +- Makefile.msc | 269 +- README.md | 101 +- VERSION | 2 +- autoconf/Makefile.in | 15 +- autoconf/Makefile.msc | 100 +- autosetup/README.md | 12 +- autosetup/jimsh0.c | 10 +- autosetup/proj.tcl | 4 +- autosetup/sqlite-config.tcl | 54 +- doc/compile-for-unix.md | 40 +- doc/compile-for-windows.md | 36 + doc/lemon.html | 32 +- doc/testrunner.md | 159 +- ext/expert/expert1.test | 2 +- ext/fts3/fts3.c | 9 +- ext/fts3/fts3Int.h | 20 +- ext/fts3/fts3_aux.c | 2 +- ext/fts3/fts3_write.c | 42 +- ext/fts5/fts5_aux.c | 2 +- ext/fts5/fts5_buffer.c | 2 +- ext/fts5/fts5_config.c | 4 +- ext/fts5/fts5_expr.c | 4 +- ext/fts5/fts5_hash.c | 2 +- ext/fts5/fts5_index.c | 70 +- ext/fts5/fts5_main.c | 70 +- ext/fts5/fts5_tcl.c | 8 +- ext/fts5/fts5_test_tok.c | 10 +- ext/fts5/fts5_tokenize.c | 8 +- ext/fts5/fts5_vocab.c | 2 +- ext/fts5/test/fts5corrupt9.test | 69 + ext/fts5/test/fts5integrity.test | 3 - ext/fts5/test/fts5interrupt.test | 17 + ext/fts5/test/fts5join.test | 9 + ext/fts5/tool/fts5cost.tcl | 153 + ext/intck/sqlite3intck.c | 2 +- ext/jni/README.md | 35 +- ext/jni/src/c/sqlite3-jni.c | 94 +- ext/jni/src/c/sqlite3-jni.h | 4 +- ext/jni/src/org/sqlite/jni/capi/CApi.java | 8 +- ext/jni/src/org/sqlite/jni/capi/Tester1.java | 4 +- .../src/org/sqlite/jni/wrapper1/Sqlite.java | 1 + ext/misc/amatch.c | 4 +- ext/misc/base64.c | 6 +- ext/misc/base85.c | 18 +- ext/misc/btreeinfo.c | 6 +- ext/misc/closure.c | 6 +- ext/misc/completion.c | 42 +- ext/misc/csv.c | 21 +- ext/misc/decimal.c | 80 +- ext/misc/explain.c | 4 +- ext/misc/fileio.c | 249 +- ext/misc/fossildelta.c | 2 +- ext/misc/fuzzer.c | 6 +- ext/misc/ieee754.c | 36 +- ext/misc/memstat.c | 4 +- ext/misc/prefixes.c | 4 +- ext/misc/qpvtab.c | 4 +- ext/misc/regexp.c | 32 +- ext/misc/scrub.c | 2 +- ext/misc/series.c | 4 +- ext/misc/sha1.c | 23 +- ext/misc/sqlar.c | 6 +- ext/misc/sqlite3_stdio.c | 32 +- ext/misc/sqlite3_stdio.h | 3 + ext/misc/stmtrand.c | 2 +- ext/misc/templatevtab.c | 4 +- ext/misc/tmstmpvfs.c | 1042 + ext/misc/vfsstat.c | 4 +- ext/misc/vfstrace.c | 2 +- ext/misc/vtablog.c | 58 +- ext/misc/vtshim.c | 10 +- ext/misc/wholenumber.c | 4 +- ext/misc/zipfile.c | 13 +- ext/qrf/README.md | 775 + ext/qrf/dev-notes.md | 14 + ext/qrf/qrf.c | 3007 +++ ext/qrf/qrf.h | 201 + ext/rbu/rbu11.test | 28 + ext/rbu/sqlite3rbu.c | 4 +- ext/repair/README.md | 16 - ext/repair/checkfreelist.c | 310 - ext/repair/checkindex.c | 929 - ext/repair/sqlite3_checker.c.in | 85 - ext/repair/sqlite3_checker.tcl | 264 - ext/repair/test/README.md | 13 - ext/repair/test/checkfreelist01.test | 92 - ext/repair/test/checkindex01.test | 349 - ext/repair/test/test.tcl | 67 - ext/rtree/geopoly.c | 2 +- ext/rtree/rtree.c | 25 +- ext/session/session4.test | 1 + ext/session/sessionC.test | 67 + ext/session/sessionchange2.test | 400 + ext/session/sessionfault3.test | 16 +- ext/session/sqlite3session.c | 604 +- ext/session/sqlite3session.h | 226 + ext/session/test_session.c | 310 +- ext/wasm/EXPORTED_FUNCTIONS.fiddle.in | 10 - ext/wasm/GNUmakefile | 505 +- ...S.sqlite3-core => EXPORTED_FUNCTIONS.c-pp} | 91 + .../api/EXPORTED_FUNCTIONS.sqlite3-extras | 68 - ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see | 5 - .../api/EXPORTED_RUNTIME_METHODS.sqlite3-api | 3 - ext/wasm/api/README.md | 84 +- ext/wasm/api/extern-post-js.c-pp.js | 12 +- ext/wasm/api/opfs-common-inline.c-pp.js | 191 + ext/wasm/api/opfs-common-shared.c-pp.js | 1301 ++ ext/wasm/api/post-js-footer.js | 69 + ext/wasm/api/post-js-header.js | 11 +- ext/wasm/api/pre-js.c-pp.js | 53 +- ext/wasm/api/sqlite3-api-cleanup.js | 83 - ext/wasm/api/sqlite3-api-glue.c-pp.js | 159 +- ext/wasm/api/sqlite3-api-oo1.c-pp.js | 301 +- ext/wasm/api/sqlite3-api-prologue.js | 478 +- ext/wasm/api/sqlite3-api-worker1.c-pp.js | 2 +- .../api/sqlite3-license-version-header.js | 3 +- ...xy.js => sqlite3-opfs-async-proxy.c-pp.js} | 527 +- ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js | 2102 ++ ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js | 24 +- ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js | 128 + ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 1462 +- ext/wasm/api/sqlite3-vtab-helper.c-pp.js | 5 +- ext/wasm/api/sqlite3-wasm.c | 299 +- ext/wasm/api/sqlite3-worker1-promiser.c-pp.js | 34 +- ext/wasm/api/sqlite3-worker1.c-pp.js | 8 +- ext/wasm/c-pp-lite.c | 2767 --- ext/wasm/common/SqliteTestUtil.js | 3 +- ext/wasm/common/whwasmutil.js | 67 +- ext/wasm/demo-jsstorage.js | 8 +- ext/wasm/demo-worker1-promiser.c-pp.html | 12 +- ext/wasm/demo-worker1-promiser.c-pp.js | 4 +- ext/wasm/demo-worker1.js | 6 +- ext/wasm/fiddle/fiddle-worker.js | 4 - .../fiddle/{index.html => index.c-pp.html} | 17 +- ext/wasm/index.html | 10 + ext/wasm/jaccwabyt/jaccwabyt.js | 851 +- ext/wasm/jaccwabyt/jaccwabyt.md | 698 +- ext/wasm/libcmpp.c | 16877 ++++++++++++++++ ext/wasm/mkdist.sh | 2 +- ext/wasm/mkwasmbuilds.c | 124 +- ext/wasm/speedtest1-worker.js | 13 +- ext/wasm/speedtest1.html | 203 +- ext/wasm/tester1-worker.c-pp.html | 18 +- ext/wasm/tester1.c-pp.html | 22 +- ext/wasm/tester1.c-pp.js | 1326 +- ext/wasm/tests/opfs/concurrency/index.html | 15 +- ext/wasm/tests/opfs/concurrency/test.js | 124 +- ext/wasm/tests/opfs/concurrency/worker.js | 135 +- main.mk | 212 +- make.bat | 2 + manifest | 605 +- manifest.tags | 7 +- manifest.uuid | 2 +- src/alter.c | 779 +- src/attach.c | 2 +- src/auth.c | 2 +- src/btree.c | 34 +- src/build.c | 186 +- src/carray.c | 39 +- src/date.c | 16 +- src/delete.c | 7 +- src/expr.c | 95 +- src/fkey.c | 24 +- src/func.c | 31 +- src/hwtime.h | 41 +- src/json.c | 229 +- src/loadext.c | 47 +- src/main.c | 47 +- src/mutex.c | 29 +- src/mutex_w32.c | 8 - src/os_kv.c | 266 +- src/os_unix.c | 2 +- src/os_win.c | 436 +- src/os_win.h | 10 +- src/pager.c | 12 +- src/parse.y | 206 +- src/pcache.h | 4 +- src/pragma.c | 16 +- src/prepare.c | 3 +- src/printf.c | 136 +- src/resolve.c | 10 +- src/select.c | 707 +- src/shell.c.in | 8338 ++++---- src/sqlite.h.in | 339 +- src/sqlite3ext.h | 11 +- src/sqliteInt.h | 141 +- src/sqliteLimit.h | 33 +- src/tclsqlite.c | 426 +- src/test1.c | 43 +- src/test2.c | 1 + src/test_bestindex.c | 1 - src/test_config.c | 16 +- src/test_quota.c | 4 - src/threads.c | 1 + src/tokenize.c | 10 +- src/treeview.c | 8 +- src/trigger.c | 137 +- src/util.c | 997 +- src/vacuum.c | 12 +- src/vdbe.c | 100 +- src/vdbe.h | 30 +- src/vdbeInt.h | 5 +- src/vdbeapi.c | 50 +- src/vdbeaux.c | 223 +- src/vdbemem.c | 243 +- src/wal.c | 9 +- src/where.c | 254 +- src/wherecode.c | 2 +- src/whereexpr.c | 79 +- src/window.c | 6 +- test/alterauth2.test | 51 + test/altercol.test | 13 + test/altercons.test | 442 + test/altercons2.test | 247 + test/alterfault.test | 37 + test/altertab3.test | 48 + test/altertrig.test | 2 +- test/atof2.test | 35 + test/auth.test | 8 +- test/autoindex1.test | 3 +- test/avfs.test | 2 + test/bestindex8.test | 16 +- test/bestindexB.test | 2 +- test/bestindexF.test | 294 + test/carray01.test | 8 + test/collate5.test | 30 +- test/cost.test | 2 +- test/dblwidth-a.sql | 38 +- test/distinct2.test | 34 +- test/dotcmd01.sql | 63 + test/e_expr.test | 8 +- test/e_update.test | 14 +- test/e_walckpt.test | 17 +- test/eqp.test | 55 +- test/expridx1.test | 273 + test/filectrl.test | 12 +- test/fkey8.test | 40 + test/fp-speed-1.c | 103 +- test/fp-speed-2.c | 263 + test/fpconv1.test | 66 +- test/fptest01.sql | 186 + test/fts3comp1.test | 77 + test/fts4content.test | 59 +- test/fts4merge5.test | 3 + test/fuzzcheck.c | 38 +- test/fuzzinvariants.c | 9 +- test/import01.sql | 217 + test/imposter1.sql | 32 + test/insert5.test | 19 +- test/intck01.sql | 23 + test/join.test | 41 + test/json/json-speed-check.sh | 2 +- test/json102.test | 21 + test/json103.test | 7 + test/json105.test | 5 + test/json109.test | 72 + test/misc5.test | 2 +- test/modeA.sql | 323 + test/mutex1.test | 26 +- test/notnull2.test | 2 +- test/offset1.test | 28 + test/printf.test | 5 + test/qrf01.test | 1186 ++ test/qrf02.test | 47 + test/qrf03.test | 176 + test/qrf04.test | 750 + test/qrf05.test | 37 + test/qrf06.test | 576 + test/recover.test | 2 +- test/regexp1.sql | 32 + test/regexp1.test | 24 + test/rowvalue4.test | 3 +- test/rowvalueA.test | 49 + test/schema.test | 4 +- test/select9.test | 2 +- test/shell1.test | 234 +- test/shell2.test | 92 +- test/shell3.test | 2 +- test/shell4.test | 8 +- test/shell5.test | 124 +- test/shell8.test | 40 + test/shell9.test | 2 +- test/shellA.test | 233 +- test/shellB.test | 53 + test/speedtest.md | 5 +- test/speedtest.tcl | 2 +- test/tabfunc01.test | 14 + test/tclsqlite.test | 2 +- test/temptrigfault.tes | 120 + test/temptrigger.test | 187 + test/tester.tcl | 5 - test/testrunner.tcl | 158 +- test/testrunner_data.tcl | 10 +- test/testrunner_estwork.tcl | 1 + test/tkt-99378177930f87bd.test | 28 + test/tkt2339.test | 6 +- test/values.test | 6 +- test/vt02.c | 125 +- test/vt100-a.sql | 35 +- test/walckptnoop.test | 6 +- test/walrestart.test | 4 +- test/where2.test | 37 + test/whereK.test | 13 + test/win32longpath.test | 4 - test/with1.test | 19 + test/with5.test | 21 + test/zipfile2.test | 15 + tool/build-all-msvc.bat | 3 +- tool/dbtotxt.c | 10 +- tool/lemon.c | 63 +- tool/lempar.c | 17 +- tool/mkautoconfamal.sh | 3 +- tool/mkcombo.tcl | 106 + tool/mkctimec.tcl | 1 + tool/mkfptab.c | 238 + tool/mkkeywordhash.c | 4 +- tool/mkshellc.tcl | 120 +- tool/omittest.tcl | 2 +- tool/showdb.c | 247 +- tool/showtmlog.c | 254 + tool/sqldiff.c | 42 +- tool/sqlite3_rsync.c | 33 +- tool/sqltclsh.c.in | 2 +- tool/warnings.sh | 1 + tool/winmain.c | 79 + 326 files changed, 50402 insertions(+), 16884 deletions(-) create mode 100644 ext/fts5/tool/fts5cost.tcl create mode 100644 ext/misc/tmstmpvfs.c create mode 100644 ext/qrf/README.md create mode 100644 ext/qrf/dev-notes.md create mode 100644 ext/qrf/qrf.c create mode 100644 ext/qrf/qrf.h delete mode 100644 ext/repair/README.md delete mode 100644 ext/repair/checkfreelist.c delete mode 100644 ext/repair/checkindex.c delete mode 100644 ext/repair/sqlite3_checker.c.in delete mode 100644 ext/repair/sqlite3_checker.tcl delete mode 100644 ext/repair/test/README.md delete mode 100644 ext/repair/test/checkfreelist01.test delete mode 100644 ext/repair/test/checkindex01.test delete mode 100644 ext/repair/test/test.tcl create mode 100644 ext/session/sessionchange2.test delete mode 100644 ext/wasm/EXPORTED_FUNCTIONS.fiddle.in rename ext/wasm/api/{EXPORTED_FUNCTIONS.sqlite3-core => EXPORTED_FUNCTIONS.c-pp} (60%) delete mode 100644 ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras delete mode 100644 ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see delete mode 100644 ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api create mode 100644 ext/wasm/api/opfs-common-inline.c-pp.js create mode 100644 ext/wasm/api/opfs-common-shared.c-pp.js delete mode 100644 ext/wasm/api/sqlite3-api-cleanup.js rename ext/wasm/api/{sqlite3-opfs-async-proxy.js => sqlite3-opfs-async-proxy.c-pp.js} (62%) create mode 100644 ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js create mode 100644 ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js delete mode 100644 ext/wasm/c-pp-lite.c rename ext/wasm/fiddle/{index.html => index.c-pp.html} (95%) create mode 100644 ext/wasm/libcmpp.c create mode 100644 make.bat create mode 100644 test/altercons.test create mode 100644 test/altercons2.test create mode 100644 test/atof2.test create mode 100644 test/bestindexF.test create mode 100644 test/dotcmd01.sql create mode 100644 test/expridx1.test create mode 100644 test/fp-speed-2.c create mode 100644 test/fptest01.sql create mode 100644 test/import01.sql create mode 100644 test/imposter1.sql create mode 100644 test/intck01.sql create mode 100644 test/json109.test create mode 100644 test/modeA.sql create mode 100644 test/qrf01.test create mode 100644 test/qrf02.test create mode 100644 test/qrf03.test create mode 100644 test/qrf04.test create mode 100644 test/qrf05.test create mode 100644 test/qrf06.test create mode 100644 test/regexp1.sql create mode 100644 test/shellB.test create mode 100644 test/temptrigfault.tes create mode 100644 tool/mkcombo.tcl create mode 100644 tool/mkfptab.c create mode 100644 tool/showtmlog.c create mode 100644 tool/winmain.c diff --git a/Makefile.in b/Makefile.in index a597a86a3..bec5f9d3e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -131,9 +131,15 @@ libsqlite3.DLL.install-rules = @SQLITE_DLL_INSTALL_RULES@ CFLAGS.fuzzcheck-asan.fsanitize = @CFLAGS_ASAN_FSANITIZE@ # -# Intended to either be empty or be set to -g -DSQLITE_DEBUG=1. -# -T.cc.TARGET_DEBUG = @TARGET_DEBUG@ +# TARGET_DEBUG is intended to either be empty or be set to -g +# -DSQLITE_DEBUG=1, plus any other flags relevant for debug-only +# builds. These get added near the front of $(T.cc) so that they can +# be overridden by CFLAGS passed directly to make. Historically (prior +# to 2026-03-30) these were at/near the end of the flags but that made +# it impossible to pass custom -O* flags to specific binaries without +# reconfiguring. +# +T.cc += @TARGET_DEBUG@ # # $(JIMSH) and $(CFLAGS.jimsh) are documented in main.mk. $(JIMSH) @@ -164,7 +170,9 @@ OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@ $(OPTIONS) PACKAGE_VERSION = @PACKAGE_VERSION@ # -# Filename extensions for binaries and libraries +# Filename extensions for binaries and libraries. B.* are for the +# being-built-on platform and T.* are for the build's target platform. +# i.e. B.* is for build-time tooling and T.* is for deliverables. # B.exe = @BUILD_EXEEXT@ T.exe = @TARGET_EXEEXT@ diff --git a/Makefile.msc b/Makefile.msc index 52807ff7f..8e06c11cb 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -11,6 +11,8 @@ TOP = . # <<mark>> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# # Set this non-0 to create and use the SQLite amalgamation file. # !IFNDEF USE_AMALGAMATION @@ -101,13 +103,6 @@ NO_WARN = $(NO_WARN) -wd4210 -wd4232 -wd4244 -wd4305 -wd4306 -wd4702 -wd4706 !ENDIF !ENDIF -# Set this non-0 to use the library paths and other options necessary for -# Windows Phone 8.1. -# -!IFNDEF USE_WP81_OPTS -USE_WP81_OPTS = 0 -!ENDIF - # Set this non-0 to split the SQLite amalgamation file into chunks to # be used for debugging with Visual Studio. # @@ -116,14 +111,8 @@ SPLIT_AMALGAMATION = 0 !ENDIF # <<mark>> -# Set this non-0 to have this makefile assume the Tcl shell executable -# (tclsh*.exe) is available in the PATH. By default, this is disabled -# for compatibility with older build environments. This setting only -# applies if TCLSH_CMD is not set manually. +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc # -!IFNDEF USE_TCLSH_IN_PATH -USE_TCLSH_IN_PATH = 0 -!ENDIF # Set this non-0 to use zlib, possibly compiling it from source code. # @@ -186,14 +175,6 @@ USE_NATIVE_LIBPATHS = 0 USE_RC = 1 !ENDIF -# Set this non-0 to compile binaries suitable for the WinRT environment. -# This setting does not apply to any binaries that require Tcl to operate -# properly (i.e. the text fixture, etc). -# -!IFNDEF FOR_WINRT -FOR_WINRT = 0 -!ENDIF - # Set this non-0 to compile binaries suitable for the UWP environment. # This setting does not apply to any binaries that require Tcl to operate # properly (i.e. the text fixture, etc). @@ -209,6 +190,8 @@ FOR_WIN10 = 0 !ENDIF # <<mark>> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# # Set this non-0 to skip attempting to look for and/or link with the Tcl # runtime library. # @@ -265,6 +248,8 @@ DEBUG = 0 !ENDIF # <<mark>> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# # By default, use --linemacros=1 argument to the mksqlite3c.tcl tool, which # is used to build the amalgamation. This can be turned off to ease debug # of the amalgamation away from the source tree. @@ -360,6 +345,8 @@ SQLITE3EXEPDB = /pdb:sqlite3sh.pdb !ENDIF # <<mark>> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# # These are the names of the customized Tcl header files used by various parts # of this makefile when the stdcall calling convention is in use. It is not # used for any other purpose. @@ -641,16 +628,24 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) -I$(TOP)\src $(RCOPTS) $(RCCOPTS) !IF "$(PLATFORM)"=="x86" CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall + # <<mark>> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# TEST_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl # <</mark>> + !ELSE !IFNDEF PLATFORM CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall + # <<mark>> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# TEST_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall -DINCLUDE_SQLITE_TCL_H=1 -DSQLITE_TCLAPI=__cdecl # <</mark>> + !ELSE CORE_CCONV_OPTS = SHELL_CCONV_OPTS = @@ -777,18 +772,6 @@ SHELL_LINK_OPTS = $(SHELL_CORE_LIB) TCC = $(TCC) -FAcs !ENDIF -# When compiling the library for use in the WinRT environment, -# the following compile-time options must be used as well to -# disable use of Win32 APIs that are not available and to enable -# use of Win32 APIs that are specific to Windows 8 and/or WinRT. -# -!IF $(FOR_WINRT)!=0 -TCC = $(TCC) -DSQLITE_OS_WINRT=1 -RCC = $(RCC) -DSQLITE_OS_WINRT=1 -TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP -RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP -!ENDIF - # C compiler options for the Windows 10 platform (needs MSVC 2015). # !IF $(FOR_WIN10)!=0 @@ -801,7 +784,7 @@ BCC = $(BCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE # USE_CRT_DLL option is set to force dynamically linking to the # MSVC runtime library. # -!IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0 +!IF $(USE_CRT_DLL)!=0 !IF $(DEBUG)>1 TCC = $(TCC) -MDd BCC = $(BCC) -MDd @@ -824,6 +807,8 @@ ZLIBCFLAGS = -nologo -MT -W3 -O2 -Oy- -Zi !ENDIF # <<mark>> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# # The mksqlite3c.tcl and mksqlite3h.tcl scripts will pull in # any extension header files by default. For non-amalgamation # builds, we need to make sure the compiler can find these. @@ -874,16 +859,6 @@ MKSQLITE3H_ARGS = !ENDIF # <</mark>> -# Define -DNDEBUG to compile without debugging (i.e., for production usage) -# Omitting the define will cause extra debugging code to be inserted and -# includes extra comments when "EXPLAIN stmt" is used. -# -!IF $(DEBUG)==0 -TCC = $(TCC) -DNDEBUG -BCC = $(BCC) -DNDEBUG -RCC = $(RCC) -DNDEBUG -!ENDIF - !IF $(DEBUG)>0 || $(API_ARMOR)!=0 || $(FOR_WIN10)!=0 TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1 RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 @@ -949,6 +924,8 @@ TCC = $(TCC) /fsanitize=address !ENDIF # <<mark>> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# # The locations of the Tcl header and library files. Also, the library that # non-stubs enabled programs using Tcl must link against. These variables # (TCLINCDIR, TCLLIBDIR, and LIBTCL) may be overridden via the environment @@ -1188,6 +1165,8 @@ BCC = $(BCC) -Zi !ENDIF # <<mark>> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# # If zlib support is enabled, add the compiler options for it. # !IF $(USE_ZLIB)!=0 @@ -1240,56 +1219,6 @@ LTLINKOPTS = /NOLOGO LTLIBOPTS = /NOLOGO !ENDIF -# When compiling for use in the WinRT environment, the following -# linker option must be used to mark the executable as runnable -# only in the context of an application container. -# -!IF $(FOR_WINRT)!=0 -LTLINKOPTS = $(LTLINKOPTS) /APPCONTAINER -!IF "$(VISUALSTUDIOVERSION)"=="12.0" || "$(VISUALSTUDIOVERSION)"=="14.0" -!IFNDEF STORELIBPATH -!IF "$(PLATFORM)"=="x86" -STORELIBPATH = $(CRTLIBPATH)\store -!ELSEIF "$(PLATFORM)"=="x64" -STORELIBPATH = $(CRTLIBPATH)\store\amd64 -!ELSEIF "$(PLATFORM)"=="ARM" -STORELIBPATH = $(CRTLIBPATH)\store\arm -!ELSE -STORELIBPATH = $(CRTLIBPATH)\store -!ENDIF -!ENDIF -STORELIBPATH = $(STORELIBPATH:\\=\) -LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(STORELIBPATH)" -!ENDIF -!ENDIF - -# When compiling for Windows Phone 8.1, an extra library path is -# required. -# -!IF $(USE_WP81_OPTS)!=0 -!IFNDEF WP81LIBPATH -!IF "$(PLATFORM)"=="x86" -WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 -!ELSEIF "$(PLATFORM)"=="ARM" -WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\ARM -!ELSE -WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 -!ENDIF -!ENDIF -!ENDIF - -# When compiling for Windows Phone 8.1, some extra linker options -# are also required. -# -!IF $(USE_WP81_OPTS)!=0 -!IFDEF WP81LIBPATH -LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(WP81LIBPATH)" -!ENDIF -LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE -LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib -LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib -!ENDIF - # When compiling for UWP or the Windows 10 platform, some extra linker # options are also required. # @@ -1313,12 +1242,14 @@ LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib # If either debugging or symbols are enabled, enable PDBs. # !IF $(DEBUG)>1 || $(SYMBOLS)!=0 -LDFLAGS = /DEBUG $(LDOPTS) +LDFLAGS = /NODEFAULTLIB:msvcrt /DEBUG $(LDOPTS) !ELSE -LDFLAGS = $(LDOPTS) +LDFLAGS = /NODEFAULTLIB:msvcrt $(LDOPTS) !ENDIF # <<mark>> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# # Start with the Tcl related linker options. # !IF $(NO_TCL)==0 @@ -1345,6 +1276,8 @@ LTLIBS = $(LTLIBS) $(LIBICU) ############################################################################### # <<mark>> +# ^^^^^^^^-- content between mark and /mark omitted from autoconf/Makefile.msc +# # Object files for the SQLite library (non-amalgamation). # LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ @@ -1463,7 +1396,7 @@ SRC01 = \ $(TOP)\src\status.c \ $(TOP)\src\table.c \ $(TOP)\src\threads.c \ - $(TOP)\src\tclsqlite.c \ + tclsqlite-ex.c \ $(TOP)\src\tokenize.c \ $(TOP)\src\treeview.c \ $(TOP)\src\trigger.c \ @@ -1774,6 +1707,7 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_PERCENTILE=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_STRICT_SUBTYPE=1 !ENDIF @@ -1785,6 +1719,7 @@ FUZZERSHELL_COMPILE_OPTS = FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -I$(TOP)\test -I$(TOP)\ext\recover FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MEMSYS5 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OSS_FUZZ +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_DECIMAL_MAX_DIGIT=1000 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_CARRAY FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB @@ -1831,7 +1766,17 @@ FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\fuzzinvariants.c FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\vt02.c FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\dbdata.c FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\sqlite3recover.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\base64.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\base85.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\completion.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\decimal.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\ieee754.c FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\randomjson.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\regexp.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\series.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\shathree.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\sha1.c +FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\stmtrand.c OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION @@ -1932,8 +1877,8 @@ $(SQLITE3EXE): shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLIT /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) # <<mark>> -sqldiff.exe: $(TOP)\tool\sqldiff.c $(TOP)\ext\misc\sqlite3_stdio.h $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) - $(LTLINK) $(NO_WARN) -I$(TOP)\ext\misc $(TOP)\tool\sqldiff.c $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS) +sqldiff.exe: $(TOP)\tool\sqldiff.c $(TOP)\ext\misc\sqlite3_stdio.h $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) $(TOP)\tool\winmain.c + $(LTLINK) $(NO_WARN) -I$(TOP)\ext\misc $(TOP)\tool\sqldiff.c $(TOP)\ext\misc\sqlite3_stdio.c $(TOP)\tool\winmain.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS) dbhash.exe: $(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) @@ -2023,8 +1968,18 @@ sqlite3.c: .target_source sqlite3ext.h sqlite3session.h $(MKSQLITE3C_TOOL) src-v sqlite3-all.c: sqlite3.c $(TOP)\tool\split-sqlite3c.tcl $(JIM_TCLSH) $(JIM_TCLSH) $(TOP)\tool\split-sqlite3c.tcl + +TCLSQLITEEX = \ + $(TOP)\ext\qrf\qrf.h \ + $(TOP)\ext\qrf\qrf.c \ + $(TOP)\src\tclsqlite.c + +tclsqlite-ex.c: $(TCLSQLITEEX) $(TOP)\tool\mkcombo.tcl $(JIM_TCLSH) + $(JIM_TCLSH) $(TOP)\tool\mkcombo.tcl $(TCLSQLITEEX) -o $@ # <</mark>> +tclsqlite-ex.c: + # Rule to build the amalgamation # sqlite3.lo: $(SQLITE3C) @@ -2324,11 +2279,11 @@ whereexpr.lo: $(TOP)\src\whereexpr.c $(HDR) window.lo: $(TOP)\src\window.c $(HDR) $(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\window.c -tclsqlite.lo: $(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP) - $(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c +tclsqlite.lo: tclsqlite-ex.c $(HDR) $(SQLITE_TCL_DEP) + $(LTCOMPILE) $(NO_WARN) -DUSE_TCL_STUBS=1 -DBUILD_sqlite -I$(TCLINCDIR) -c tclsqlite-ex.c /OUT:tclsqlite.lo -tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR) $(SQLITE_TCL_DEP) - $(LTCOMPILE) $(NO_WARN) -DTCLSH -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c +tclsqlite-shell.lo: tclsqlite-ex.c $(HDR) $(SQLITE_TCL_DEP) + $(LTCOMPILE) $(NO_WARN) -DTCLSH -DBUILD_sqlite -I$(TCLINCDIR) -c tclsqlite-ex.c tclsqlite3.exe: tclsqlite-shell.lo $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) $(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS) @@ -2381,6 +2336,8 @@ keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe # Source and header files that shell.c depends on SHELL_DEP = \ $(TOP)\src\shell.c.in \ + $(TOP)\ext\qrf\qrf.c \ + $(TOP)\ext\qrf\qrf.h \ $(TOP)\ext\expert\sqlite3expert.c \ $(TOP)\ext\expert\sqlite3expert.h \ $(TOP)\ext\intck\sqlite3intck.c \ @@ -2544,9 +2501,9 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_STRICT_SUBTYPE=1 TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2) TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C) !IF $(USE_AMALGAMATION)==0 -TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0) +TESTFIXTURE_SRC = $(TESTSRC) tclsqlite-ex.c $(TESTFIXTURE_SRC0) !ELSE -TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC1) +TESTFIXTURE_SRC = $(TESTSRC) tclsqlite-ex.c $(TESTFIXTURE_SRC1) !ENDIF !IF $(USE_STDCALL)!=0 || $(FOR_WIN10)!=0 @@ -2594,43 +2551,19 @@ coretestprogs: testfixture.exe sqlite3.exe testprogs: $(TESTPROGS) srcck1.exe fuzzcheck.exe sessionfuzz.exe -fulltest: alltest fuzztest - -alltest: $(TESTPROGS) - @set PATH=$(LIBTCLPATH);$(PATH) - .\testfixture.exe $(TOP)\test\all.test $(TESTOPTS) - -soaktest: $(TESTPROGS) - @set PATH=$(LIBTCLPATH);$(PATH) - .\testfixture.exe $(TOP)\test\all.test -soak=1 $(TESTOPTS) - -fulltestonly: $(TESTPROGS) fuzztest - @set PATH=$(LIBTCLPATH);$(PATH) - .\testfixture.exe $(TOP)\test\full.test - -queryplantest: testfixture.exe shell - @set PATH=$(LIBTCLPATH);$(PATH) - .\testfixture.exe $(TOP)\test\permutations.test queryplanner $(TESTOPTS) - fuzztest: fuzzcheck.exe .\fuzzcheck.exe $(FUZZDATA) -# Legacy testing target for third-party integrators. The SQLite -# developers seldom use this target themselves. Instead -# they use "nmake /f Makefile.msc devtest" which runs tests on -# a standard set of options -# -test: $(TESTPROGS) sourcetest fuzztest tcltest - -# Minimal testing that runs in less than 3 minutes (on a fast machine) # -quicktest: testfixture.exe sourcetest - @set PATH=$(LIBTCLPATH);$(PATH) - .\testfixture.exe $(TOP)\test\extraquick.test $(TESTOPTS) - -# This is the common case. Run many tests that do not take too long, -# including fuzzcheck, sqlite3_analyzer, and sqldiff tests. +# Legacy testing targets, no longer used by the developers and +# now aliased to one of the commonly used testing targets. # +quicktest: devtest +test: devtest +fulltest: releasetest +alltest: releasetest +soaktest: releasetest +fulltestonly: releasetest # The veryquick.test TCL tests. # @@ -2655,6 +2588,22 @@ devtest: srctree-check sourcetest mdevtest: $(TCLSH_CMD) $(TOP)\test\testrunner.tcl mdevtest +# Rerun test cases that failed on the previous testrunner invocation +# +retest: + $(TCLSH_CMD) $(TOP)\test\testrunner.tcl retest + +# Show all errors from the most reason testrunner invocation +# +errors: + $(TCLSH_CMD) $(TOP)\test\testrunner.tcl errors + +# Show the status of the current testrunner invocation, +# updated every couple of seconds +# +status: + $(TCLSH_CMD) $(TOP)\test\testrunner.tcl status -d 2 + # Validate that various generated files in the source tree # are up-to-date. # @@ -2678,17 +2627,23 @@ smoketest: $(TESTPROGS) @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\main.test $(TESTOPTS) -shelltest: $(TESTPROGS) - .\testfixture.exe $(TOP)\test\permutations.test shell +# Measure the performance of floating-point conversions. +# +fp-speed-test: fp-speed-1.exe fp-speed-2.exe + fp-speed-1 1000000 + fp-speed-2 1000000 + +shelltest: + $(TCLSH_CMD) $(TOP)\test\testrunner.tcl release shell -sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(TOP)\ext\misc\sqlite3_stdio.h $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE_TCL_DEP) +sqlite3_analyzer.c: $(SQLITE3C) $(SQLITE3H) tclsqlite-ex.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(TOP)\ext\misc\sqlite3_stdio.h $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE_TCL_DEP) $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl -DINCLUDE_SQLITE3_C $(TOP)\tool\sqlite3_analyzer.c.in > $@ sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS) $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \ /link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS) -sqltclsh.c: sqlite3.c $(TOP)\src\tclsqlite.c $(TOP)\tool\sqltclsh.tcl $(TOP)\ext\misc\appendvfs.c $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in +sqltclsh.c: sqlite3.c tclsqlite-ex.c $(TOP)\tool\sqltclsh.tcl $(TOP)\ext\misc\appendvfs.c $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqltclsh.c.in >sqltclsh.c sqltclsh.exe: sqltclsh.c $(SHELL_CORE_DEP) $(LIBRESOBJS) @@ -2698,23 +2653,6 @@ sqltclsh.exe: sqltclsh.c $(SHELL_CORE_DEP) $(LIBRESOBJS) sqlite3_expert.exe: $(SQLITE3C) $(TOP)\ext\expert\sqlite3expert.h $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(LTLINK) $(NO_WARN) $(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(SQLITE3C) $(TLIBS) -CHECKER_DEPS =\ - $(TOP)\tool\mkccode.tcl \ - sqlite3.c \ - $(TOP)\src\tclsqlite.c \ - $(TOP)\ext\repair\sqlite3_checker.tcl \ - $(TOP)\ext\repair\checkindex.c \ - $(TOP)\ext\repair\checkfreelist.c \ - $(TOP)\ext\misc\btreeinfo.c \ - $(TOP)\ext\repair\sqlite3_checker.c.in - -sqlite3_checker.c: $(CHECKER_DEPS) - $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\ext\repair\sqlite3_checker.c.in > $@ - -sqlite3_checker.exe: sqlite3_checker.c $(LIBRESOBJS) - $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_checker.c \ - /link $(LDFLAGS) $(LTLINKOPTS) $(TCLLIBPATHS) $(LTLIBPATHS) $(LIBRESOBJS) $(TCLLIBS) $(LTLIBS) $(TLIBS) - dbdump.exe: $(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) $(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \ /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) @@ -2747,6 +2685,9 @@ showwal.exe: $(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H) showshm.exe: $(TOP)\tool\showshm.c $(LTLINK) $(NO_WARN) $(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS) +showtmlog.exe: $(TOP)\tool\showtmlog.c + $(LTLINK) $(NO_WARN) $(TOP)\tool\showtmlog.c /link $(LDFLAGS) $(LTLINKOPTS) + index_usage.exe: $(TOP)\tool\index_usage.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\tool\index_usage.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) @@ -2784,6 +2725,14 @@ speedtest1.exe: $(TOP)\test\speedtest1.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(ST_COMPILE_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +fp-speed-1.exe: $(TOP)\test\fp-speed-1.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(ST_COMPILE_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION \ + $(TOP)\test\fp-speed-1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + +fp-speed-2.exe: $(TOP)\test\fp-speed-2.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(ST_COMPILE_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION \ + $(TOP)\test\fp-speed-2.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + kvtest.exe: $(TOP)\test\kvtest.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(KV_COMPILE_OPTS) \ $(TOP)\test\kvtest.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) @@ -2938,6 +2887,7 @@ env: @echo USE_STDCALL = $(USE_STDCALL) @echo USE_ZLIB = $(USE_ZLIB) @echo XCOMPILE = $(XCOMPILE) + @echo ZLIBCFLAGS = $(ZLIBCFLAGS) @echo ZLIBDIR = $(ZLIBDIR) @echo ZLIBINCDIR = $(ZLIBINCDIR) @echo ZLIBLIBDIR = $(ZLIBLIBDIR) @@ -2972,6 +2922,7 @@ clean: del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe dbdump.exe 2>NUL del /Q changeset.exe 2>NUL del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe 2>NUL + del /Q fp-speed-1.exe fp-speed-2.exe 2>NUL del /Q mptester.exe wordcount.exe rbu.exe srcck1.exe 2>NUL del /Q sqlite3.c sqlite3-*.c sqlite3.h 2>NUL del /Q sqlite3rc.h 2>NUL diff --git a/README.md b/README.md index d44bf5cb7..a1a1a7b87 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,9 @@ <h1 align="center">SQLite Source Repository</h1> This repository contains the complete source code for the -[SQLite database engine](https://sqlite.org/), including -many tests. Additional tests and most documentation +[SQLite database engine](https://sqlite.org/) going back +to 2000-05-29. The tree includes many tests and some +documentation, though additional tests and most documentation are managed separately. See the [on-line documentation](https://sqlite.org/) for more information @@ -96,19 +97,21 @@ the build products. It is recommended, but not required, that the build directory be separate from the source directory. Cd into the build directory and then from the build directory run the configure script found at the root of the source tree. Then run "make". +See the [compile-for-unix.md](doc/compile-for-unix.md) document for +more detail. For example: - apt install gcc make tcl-dev ;# Make sure you have all the necessary build tools + apt install gcc make tcl-dev ;# Install the necessary build tools tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite" - mkdir bld ;# Build will occur in a sibling directory + mkdir bld ;# Build happens in a sibling directory cd bld ;# Change to the build directory ../sqlite/configure ;# Run the configure script - make sqlite3 ;# Builds the "sqlite3" command-line tool - make sqlite3.c ;# Build the "amalgamation" source file - make sqldiff ;# Builds the "sqldiff" command-line tool - # Makefile targets below this point require tcl-dev - make tclextension-install ;# Build and install the SQLite TCL extension + make sqlite3 ;# The "sqlite3" command-line tool + make sqlite3.c ;# The "amalgamation" source file + make sqldiff ;# The "sqldiff" command-line tool + #### Targets below require tcl-dev #### + make tclextension-install ;# Install the SQLite TCL extension make devtest ;# Run development tests make releasetest ;# Run full release tests make sqlite3_analyzer ;# Builds the "sqlite3_analyzer" tool @@ -116,14 +119,15 @@ For example: See the makefile for additional targets. For debugging builds, the core developers typically run "configure" with options like this: - ../sqlite/configure --enable-all --enable-debug CFLAGS='-O0 -g' + ../sqlite/configure --all --debug CFLAGS='-O0 -g' For release builds, the core developers usually do: - ../sqlite/configure --enable-all + ../sqlite/configure --all -Almost all makefile targets require a "tclsh" TCL interpreter version 8.6 or -later. The "tclextension-install" target and the test targets that follow +Core deliverables (sqlite3.c, sqlite3) can be built without a TCL, but +many makefile targets require a "tclsh" TCL interpreter version 8.6 +or later. The "tclextension-install" target and the test targets that follow all require TCL development libraries too. ("apt install tcl-dev"). It is helpful, but is not required, to install the SQLite TCL extension (the "tclextension-install" target) prior to running tests. The "releasetest" @@ -133,20 +137,20 @@ On "make" command-lines, one can add "OPTIONS=..." to specify additional compile-time options over and above those set by ./configure. For example, to compile with the SQLITE_OMIT_DEPRECATED compile-time option, one could say: - ./configure --enable-all + ./configure --all make OPTIONS=-DSQLITE_OMIT_DEPRECATED sqlite3 -The configure script uses autoconf 2.61 and libtool. If the configure -script does not work out for you, there is a generic makefile named -"Makefile.linux-gcc" in the top directory of the source tree that you -can copy and edit to suit your needs. Comments on the generic makefile -show what changes are needed. +The configure script uses [autosetup](https://msteveb.github.io/autosetup/). +If the configure script does not work out for you, there is a generic +makefile named "Makefile.linux-gcc" in the top directory of the source tree +that you can copy and edit to suit your needs. Comments on the generic +makefile show what changes are needed. ## Compiling for Windows Using MSVC On Windows, everything can be compiled with MSVC. -You will also need a working installation of TCL if you want to run tests. -TCL is not required if you just want to build SQLite itself. +You will also need a working installation of TCL if you want to run tests, +though TCL is not required if you just want to build SQLite itself. See the [compile-for-windows.md](doc/compile-for-windows.md) document for additional information about how to install MSVC and TCL and configure your build environment. @@ -156,24 +160,25 @@ TCL library, using a command like this: set TCLDIR=c:\Tcl -SQLite uses "tclsh.exe" as part of the build process, and so that -program will need to be somewhere on your %PATH%. SQLite itself -does not contain any TCL code, but it does use TCL to run tests. -You may need to install TCL development -libraries in order to successfully complete some makefile targets. -It is helpful, but is not required, to install the SQLite TCL extension -(the "tclextension-install" target) prior to running tests. - -Build using Makefile.msc. Example: - - nmake /f Makefile.msc sqlite3.exe - nmake /f Makefile.msc sqlite3.c - nmake /f Makefile.msc sqldiff.exe - # Makefile targets below this point require TCL development libraries - nmake /f Makefile.msc tclextension-install - nmake /f Makefile.msc devtest - nmake /f Makefile.msc releasetest - nmake /f Makefile.msc sqlite3_analyzer.exe +SQLite itself does not contain any TCL code, but it does use TCL to run +tests. You may need to install TCL development libraries in order to +successfully complete some makefile targets. It is helpful, but is not +required, to install the SQLite TCL extension (the "tclextension-install" +target) prior to running tests. + +The source tree contains a "make.bat" file that allows the same "make" +commands of Unix to work on Windows. In the following, you can substitute +"nmake /f Makefile.msc" in place of "make", if you prefer to avoid this BAT +file: + + make sqlite3.exe + make sqlite3.c + make sqldiff.exe + #### Targets below require TCL development libraries #### + make tclextension-install + make devtest + make releasetest + make sqlite3_analyzer.exe There are many other makefile targets. See comments in Makefile.msc for details. @@ -181,7 +186,7 @@ details. As with the unix Makefile, the OPTIONS=... argument can be passed on the nmake command-line to enable new compile-time options. For example: - nmake /f Makefile.msc OPTIONS=-DSQLITE_OMIT_DEPRECATED sqlite3.exe + make OPTIONS=-DSQLITE_OMIT_DEPRECATED sqlite3.exe ## Source Tree Map @@ -194,8 +199,7 @@ command-line to enable new compile-time options. For example: * **test/** - This directory and its subdirectories contains code used for testing. Files that end in "`.test`" are TCL scripts that run tests using an augmented TCL interpreter named "testfixture". Use - a command like "`make testfixture`" (unix) or - "`nmake /f Makefile.msc testfixture.exe`" (windows) to build that + a command like "`make testfixture`" to build that augmented TCL interpreter, then run individual tests using commands like "`testfixture test/main.test`". This test/ subdirectory also contains additional C code modules and scripts for other kinds of testing. @@ -377,10 +381,11 @@ implementation. It will not be the easiest library in the world to hack. (and some other test programs too) is built and run when you type "make test". - * **VERSION**, **manifest**, and **manifest.uuid** - These files define - the current SQLite version number. The "VERSION" file is human generated, - but the "manifest" and "manifest.uuid" files are automatically generated - by the [Fossil version control system](https://fossil-scm.org/). + * **VERSION**, **manifest**, **manifest.tags**, and **manifest.uuid** - + These files define the current SQLite version number. The "VERSION" file + is human generated, but the "manifest", "manifest.tags", and + "manifest.uuid" files are automatically generated by the + [Fossil version control system](https://fossil-scm.org/). There are many other source files. Each has a succinct header comment that describes its purpose and role within the larger system. @@ -406,10 +411,6 @@ makefile: > make verify-source -Or on windows: - -> nmake /f Makefile.msc verify-source - Using the makefile to verify source integrity is good for detecting accidental changes to the source tree, but malicious changes could be hidden by also modifying the makefiles. diff --git a/VERSION b/VERSION index f17126c72..c5c343b57 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.51.3 +3.53.0 diff --git a/autoconf/Makefile.in b/autoconf/Makefile.in index c938ffe1b..8e6b358be 100644 --- a/autoconf/Makefile.in +++ b/autoconf/Makefile.in @@ -117,19 +117,6 @@ sqlite_cfg.h: $(AS_AUTO_DEF) # CFLAGS for sqlite3$(T.exe) # SHELL_OPT ?= @OPT_SHELL@ -SHELL_OPT += -DSQLITE_DQS=0 -SHELL_OPT += -DSQLITE_ENABLE_FTS4 -#SHELL_OPT += -DSQLITE_ENABLE_FTS5 -SHELL_OPT += -DSQLITE_ENABLE_RTREE -SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS -SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION -SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB -SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB -SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB -SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB -SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC -SHELL_OPT += -DSQLITE_ENABLE_PERCENTILE -SHELL_OPT += -DSQLITE_STRICT_SUBTYPE=1 # # Library-level feature flags @@ -295,7 +282,7 @@ DIST_FILES := \ README.txt VERSION \ auto.def autosetup configure tea \ sqlite3.h sqlite3.c shell.c sqlite3ext.h \ - Makefile.in Makefile.msc Makefile.fallback \ + Makefile.in Makefile.msc Makefile.fallback make.bat \ sqlite3.rc sqlite3rc.h Replace.cs \ sqlite3.pc.in sqlite3.1 diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index 365e1f0fb..34e41d8aa 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -101,13 +101,6 @@ NO_WARN = $(NO_WARN) -wd4210 -wd4232 -wd4244 -wd4305 -wd4306 -wd4702 -wd4706 !ENDIF !ENDIF -# Set this non-0 to use the library paths and other options necessary for -# Windows Phone 8.1. -# -!IFNDEF USE_WP81_OPTS -USE_WP81_OPTS = 0 -!ENDIF - # Set this non-0 to split the SQLite amalgamation file into chunks to # be used for debugging with Visual Studio. # @@ -156,14 +149,6 @@ USE_NATIVE_LIBPATHS = 0 USE_RC = 1 !ENDIF -# Set this non-0 to compile binaries suitable for the WinRT environment. -# This setting does not apply to any binaries that require Tcl to operate -# properly (i.e. the text fixture, etc). -# -!IFNDEF FOR_WINRT -FOR_WINRT = 0 -!ENDIF - # Set this non-0 to compile binaries suitable for the UWP environment. # This setting does not apply to any binaries that require Tcl to operate # properly (i.e. the text fixture, etc). @@ -563,10 +548,14 @@ RCC = $(RC) -DSQLITE_OS_WIN=1 -I. -I$(TOP) $(RCOPTS) $(RCCOPTS) !IF "$(PLATFORM)"=="x86" CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall + + !ELSE !IFNDEF PLATFORM CORE_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall SHELL_CCONV_OPTS = -Gz -guard:cf -DSQLITE_CDECL=__cdecl -DSQLITE_APICALL=__stdcall -DSQLITE_CALLBACK=__stdcall -DSQLITE_SYSAPI=__stdcall + + !ELSE CORE_CCONV_OPTS = SHELL_CCONV_OPTS = @@ -667,18 +656,6 @@ SHELL_LINK_OPTS = $(SHELL_CORE_LIB) TCC = $(TCC) -FAcs !ENDIF -# When compiling the library for use in the WinRT environment, -# the following compile-time options must be used as well to -# disable use of Win32 APIs that are not available and to enable -# use of Win32 APIs that are specific to Windows 8 and/or WinRT. -# -!IF $(FOR_WINRT)!=0 -TCC = $(TCC) -DSQLITE_OS_WINRT=1 -RCC = $(RCC) -DSQLITE_OS_WINRT=1 -TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP -RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP -!ENDIF - # C compiler options for the Windows 10 platform (needs MSVC 2015). # !IF $(FOR_WIN10)!=0 @@ -691,7 +668,7 @@ BCC = $(BCC) /d2guard4 -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE # USE_CRT_DLL option is set to force dynamically linking to the # MSVC runtime library. # -!IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0 +!IF $(USE_CRT_DLL)!=0 !IF $(DEBUG)>1 TCC = $(TCC) -MDd BCC = $(BCC) -MDd @@ -714,16 +691,6 @@ ZLIBCFLAGS = -nologo -MT -W3 -O2 -Oy- -Zi !ENDIF -# Define -DNDEBUG to compile without debugging (i.e., for production usage) -# Omitting the define will cause extra debugging code to be inserted and -# includes extra comments when "EXPLAIN stmt" is used. -# -!IF $(DEBUG)==0 -TCC = $(TCC) -DNDEBUG -BCC = $(BCC) -DNDEBUG -RCC = $(RCC) -DNDEBUG -!ENDIF - !IF $(DEBUG)>0 || $(API_ARMOR)!=0 || $(FOR_WIN10)!=0 TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR=1 RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR=1 @@ -913,56 +880,6 @@ LTLINKOPTS = /NOLOGO LTLIBOPTS = /NOLOGO !ENDIF -# When compiling for use in the WinRT environment, the following -# linker option must be used to mark the executable as runnable -# only in the context of an application container. -# -!IF $(FOR_WINRT)!=0 -LTLINKOPTS = $(LTLINKOPTS) /APPCONTAINER -!IF "$(VISUALSTUDIOVERSION)"=="12.0" || "$(VISUALSTUDIOVERSION)"=="14.0" -!IFNDEF STORELIBPATH -!IF "$(PLATFORM)"=="x86" -STORELIBPATH = $(CRTLIBPATH)\store -!ELSEIF "$(PLATFORM)"=="x64" -STORELIBPATH = $(CRTLIBPATH)\store\amd64 -!ELSEIF "$(PLATFORM)"=="ARM" -STORELIBPATH = $(CRTLIBPATH)\store\arm -!ELSE -STORELIBPATH = $(CRTLIBPATH)\store -!ENDIF -!ENDIF -STORELIBPATH = $(STORELIBPATH:\\=\) -LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(STORELIBPATH)" -!ENDIF -!ENDIF - -# When compiling for Windows Phone 8.1, an extra library path is -# required. -# -!IF $(USE_WP81_OPTS)!=0 -!IFNDEF WP81LIBPATH -!IF "$(PLATFORM)"=="x86" -WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 -!ELSEIF "$(PLATFORM)"=="ARM" -WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\ARM -!ELSE -WP81LIBPATH = $(PROGRAMFILES_X86)\Windows Phone Kits\8.1\lib\x86 -!ENDIF -!ENDIF -!ENDIF - -# When compiling for Windows Phone 8.1, some extra linker options -# are also required. -# -!IF $(USE_WP81_OPTS)!=0 -!IFDEF WP81LIBPATH -LTLINKOPTS = $(LTLINKOPTS) "/LIBPATH:$(WP81LIBPATH)" -!ENDIF -LTLINKOPTS = $(LTLINKOPTS) /DYNAMICBASE -LTLINKOPTS = $(LTLINKOPTS) WindowsPhoneCore.lib RuntimeObject.lib PhoneAppModelHost.lib -LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib -!ENDIF - # When compiling for UWP or the Windows 10 platform, some extra linker # options are also required. # @@ -986,9 +903,9 @@ LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib # If either debugging or symbols are enabled, enable PDBs. # !IF $(DEBUG)>1 || $(SYMBOLS)!=0 -LDFLAGS = /DEBUG $(LDOPTS) +LDFLAGS = /NODEFAULTLIB:msvcrt /DEBUG $(LDOPTS) !ELSE -LDFLAGS = $(LDOPTS) +LDFLAGS = /NODEFAULTLIB:msvcrt $(LDOPTS) !ENDIF @@ -1024,6 +941,7 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_PERCENTILE=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_STRICT_SUBTYPE=1 !ENDIF @@ -1072,6 +990,8 @@ $(SQLITE3EXE): shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLIT /link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) +tclsqlite-ex.c: + # Rule to build the amalgamation # sqlite3.lo: $(SQLITE3C) diff --git a/autosetup/README.md b/autosetup/README.md index c8da5c643..ac013080a 100644 --- a/autosetup/README.md +++ b/autosetup/README.md @@ -375,6 +375,11 @@ configure process, and check it in. Patching Autosetup for Project-local Changes ------------------------------------------------------------------------ +The autosetup files require the following patches after updating +from their upstream sources: + +### `--debug` flag + Autosetup reserves the flag name **`--debug`** for its own purposes, and its own special handling of `--enable-...` flags makes `--debug` an alias for `--enable-debug`. As this project has a long history of @@ -388,6 +393,11 @@ If autosetup is upgraded and this patch is _not_ applied the invoking `./configure` will fail loudly because of the declaration of the `debug` flag in `auto.def` - duplicated flags are not permitted. +### Fail on `malloc()` error + +See [check-in 72c8a5b94cdf5d](/info/72c8a5b94cdf5d). + + <a name="branch-customization"></a> Branch-specific Customization ======================================================================== @@ -451,4 +461,4 @@ all other significant processing. [sqlite-config.tcl]: /file/autosetup/sqlite-config.tcl [Makefile.in]: /file/Makefile.in [main.mk]: /file/main.mk -[JimTCL]: https://jim.tcl.tk +[JimTCL]: https://msteveb.github.io/jimtcl/ diff --git a/autosetup/jimsh0.c b/autosetup/jimsh0.c index 1a6453d0c..0f0a89088 100644 --- a/autosetup/jimsh0.c +++ b/autosetup/jimsh0.c @@ -7409,11 +7409,13 @@ void *JimDefaultAllocator(void *ptr, size_t size) free(ptr); return NULL; } - else if (ptr) { - return realloc(ptr, size); - } else { - return malloc(size); + void *p = realloc(ptr, size); + if( p==0 ){ + fprintf(stderr,"Out of memory\n"); + exit(1); + } + return p; } } diff --git a/autosetup/proj.tcl b/autosetup/proj.tcl index 86f4df44e..caa679ad6 100644 --- a/autosetup/proj.tcl +++ b/autosetup/proj.tcl @@ -1842,7 +1842,7 @@ proc proj-setup-autoreconfig {defName} { } # -# @prop-append-to defineName args... +# @prop-define-append defineName args... # # A proxy for Autosetup's [define-append]. Appends all non-empty $args # to [define-append $defineName]. @@ -1873,7 +1873,7 @@ proc proj-define-append {defineName args} { # but it is technically correct and still relevant on some # environments. # -# See: proj-append-to +# See: proj-define-append # proc proj-define-amend {args} { set defName "" diff --git a/autosetup/sqlite-config.tcl b/autosetup/sqlite-config.tcl index 7c798b31a..59ecb7192 100644 --- a/autosetup/sqlite-config.tcl +++ b/autosetup/sqlite-config.tcl @@ -557,7 +557,25 @@ proc proc-debug {msg} { } define OPT_FEATURE_FLAGS {} ; # -DSQLITE_OMIT/ENABLE flags. -define OPT_SHELL {} ; # Feature-related CFLAGS for the sqlite3 CLI app +# +# OPT_SHELL = feature-related CFLAGS for the sqlite3 CLI app. The +# list's initial values are defaults which are always applied and not +# affected by --feature-flags. The list is appended to by various +# --feature-flags. +define OPT_SHELL { + -DSQLITE_DQS=0 + -DSQLITE_ENABLE_FTS4 + -DSQLITE_ENABLE_RTREE + -DSQLITE_ENABLE_EXPLAIN_COMMENTS + -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + -DSQLITE_ENABLE_STMTVTAB + -DSQLITE_ENABLE_DBPAGE_VTAB + -DSQLITE_ENABLE_DBSTAT_VTAB + -DSQLITE_ENABLE_BYTECODE_VTAB + -DSQLITE_ENABLE_OFFSET_SQL_FUNC + -DSQLITE_ENABLE_PERCENTILE + -DSQLITE_STRICT_SUBTYPE=1 +} ######################################################################## # Adds $args, if not empty, to OPT_FEATURE_FLAGS. If the first arg is # -shell then it strips that arg and passes the remaining args the @@ -661,6 +679,7 @@ proc sqlite-check-common-system-deps {} { define HAVE_ZLIB 1 define LDFLAGS_ZLIB -lz sqlite-add-shell-opt -DSQLITE_HAVE_ZLIB=1 + sqlite-add-feature-flag -DSQLITE_HAVE_ZLIB=1 } else { define HAVE_ZLIB 0 define LDFLAGS_ZLIB "" @@ -724,12 +743,11 @@ proc sqlite-setup-default-cflags {} { # compiling binaries for the target system (CC a.k.a. $(T.cc)). # Normally they're the same, but they will differ when # cross-compiling. - # - # When cross-compiling we default to not using the -g flag, based on a - # /chat discussion prompted by - # https://sqlite.org/forum/forumpost/9a67df63eda9925c set defaultCFlags {-O2} if {!$::sqliteConfig(is-cross-compiling)} { + # When cross-compiling we default to not using the -g flag, based + # on a /chat discussion prompted by + # https://sqlite.org/forum/forumpost/9a67df63eda9925c lappend defaultCFlags -g } define CFLAGS [proj-get-env CFLAGS $defaultCFlags] @@ -787,7 +805,8 @@ proc sqlite-handle-common-feature-flags {} { sqlite-add-feature-flag -DSQLITE_ENABLE_MEMSYS3 } } - scanstatus -DSQLITE_ENABLE_STMT_SCANSTATUS {} + bytecode-vtab -DSQLITE_ENABLE_BYTECODE_VTAB {} + scanstatus {-DSQLITE_ENABLE_STMT_SCANSTATUS -DSQLITE_ENABLE_BYTECODE_VTAB} {} column-metadata -DSQLITE_ENABLE_COLUMN_METADATA {} dbpage -DSQLITE_ENABLE_DBPAGE_VTAB {} dbstat -DSQLITE_ENABLE_DBSTAT_VTAB {} @@ -1118,6 +1137,27 @@ proc sqlite-check-line-editing {} { set editLibName "readline" ; # "readline" or "editline" set editLibDef "HAVE_READLINE" ; # "HAVE_READLINE" or "HAVE_EDITLINE" set dirLn [opt-val with-linenoise] + + # If none of --with-linenoise, --enable-readline, or --enable-editline + # are provided, but there exists a directory "linenoise" at $HOME or + # a sibling of the build or source directory, then try to use that linenoise + # direcctory. + # + if {"" eq $dirLn + && ![proj-opt-was-provided readline] + && ![proj-opt-was-provided editline] + } { + set dirlist ../linenoise + catch {lappend dirlist [file-normalize $::autosetup(srcdir)/../linenoise]} + catch {lappend dirlist $::env(HOME)/linenoise} + foreach d $dirlist { + if {[file exists $d/linenoise.c] && [file exists $d/linenoise.h]} { + set dirLn $d + break + } + } + } + if {"" ne $dirLn} { # Use linenoise from a copy of its sources (not a library)... if {![file isdir $dirLn]} { @@ -1153,6 +1193,8 @@ proc sqlite-check-line-editing {} { if {$::sqliteConfig(use-jim-for-codegen) && 2 == $lnVal} { define-append CFLAGS_JIMSH -DUSE_LINENOISE [get-define CFLAGS_READLINE] user-notice "Adding linenoise support to jimsh." + } else { + msg-result "Using linenoise at [file-normalize $dirLn]" } return "linenoise ($flavor)" } elseif {[opt-bool editline]} { diff --git a/doc/compile-for-unix.md b/doc/compile-for-unix.md index ce76b97ba..e3777618c 100644 --- a/doc/compile-for-unix.md +++ b/doc/compile-for-unix.md @@ -22,9 +22,10 @@ guidance on building for Windows. or <https://sqlite.org/tmp/tcl9.0.0.tar.gz>. <li>Untar the source archive. CD into the "unix/" subfolder of the source tree. - <li>Run: `mkdir $HOME/local` - <li>Run: `./configure --prefix=$HOME/local` - <li>Run: `make install` + <li>Run: &ensp; `mkdir $HOME/local` + <li>Run: &ensp; `./configure --prefix=$HOME/local`<br> + SQLite deliverable builds add: &emsp; `--static CFLAGS=-Os` + <li>Run: &ensp; `make install` </ol> <p> As of 2024-10-25, TCL is not longer required for many @@ -36,12 +37,35 @@ guidance on building for Windows. 4. Download the SQLite source tree and unpack it. CD into the toplevel directory of the source tree. - 5. Run: `./configure --enable-all --with-tclsh=$HOME/local/bin/tclsh9.0` + 5. *(Optional):* Download Antirez's "linenoise" command-line editing + library to provide command-line editing in the CLI. You can find + a suitable copy of the linenoise sources at + <https://sqlite.org/linenoise.tar.gz> or at various other locations + on the internet. If you put the linenoise source tree in + a directory named $HOME/linenoise or a directory "linenoise" which + is a sibling of the SQLite source tree, then the SQLite ./configure + script will automatically find and use that source code to provide + command-line editing in the CLI. If you would rather use the readline + or editline libraries or a precompiled linenoise library, there are + ./configure options to accommodate that choice. The SQLite developers + typically use $HOME/linenoise since linenoise is small, has no + external dependencies, "just works", and the ./configure script + will pick it up and use it automatically. But you do what works + best for you. + <p> + You are not required to have any command-line editing support in + order to use SQLite. But command-line editing does make the + interactive experience more enjoyable. + + 6. Run: `./configure --enable-all --with-tclsh=$HOME/local/bin/tclsh9.0` You do not need to use --with-tclsh if the tclsh you want to use is the first one on your PATH or if you are building without TCL. - 6. Run the "`Makefile`" makefile with an appropriate target. + Lots of other options to ./configure are available. + Run `./configure --help` for further guidance. + + 7. Run the "`Makefile`" makefile with an appropriate target. Examples: <ul> <li> `make sqlite3.c` @@ -66,5 +90,7 @@ guidance on building for Windows. of SQLite. - 7. For a debugging build of the CLI, where the ".treetrace" and ".wheretrace" - commands work, add the the --with-debug argument to configure. + 8. For a debugging build of the CLI, use `./configure --dev`. A debugging + build contains lots of extra debugging code, so it is slow and a lot + bigger. You probably do not want to deploy a debugging build. But if + you are working on the code, a debugging build works much better. diff --git a/doc/compile-for-windows.md b/doc/compile-for-windows.md index 0e59c83fe..30536d5fd 100644 --- a/doc/compile-for-windows.md +++ b/doc/compile-for-windows.md @@ -180,3 +180,39 @@ statically linked so that it does not depend on separate DLL: 6. After your executable is built, you can verify that it does not depend on the TCL DLL by running: <blockquote><pre>dumpbin /dependents sqlite3_analyzer.exe</pre></blockquote> + +## Linking Against ZLIB + +Some feature (such as zip-file support in the CLI) require the ZLIB +compression library. That library is more or less universally available +on unix platforms, but is seldom provided on Windows. You will probably +need to provide it yourself. Here the the steps needed: + + 1. Download the zlib-1.3.1.tar.gz tarball (or a similar version). + Unpack the tarball sources. You can put them wherever you like. + For the purposes of this document, let's assume you put the source + tree in c:\\zlib-64. Note: If you are building for both x64 and + x86, you will need separate builds of ZLIB for each, thus separate + build directories. + + 2. Before building SQLite (as described above) first make these + environment changes. The lead-programmer for SQLite (who writes + these words) has BAT files named "env-x64.bat" and "env-x32.bat" + and "env-arm64.bat" that make these changes, and he runs those + BAT file whenever he starts a new shell. These are the settings + needed: + <blockquote> + set USE_ZLIB=1<br> + set BUILD_ZLIB=0<br> + set ZLIBDIR=c:\\zlib-64 + </blockquote> + + 3. Because the settings in step 2 specify "BUILD_ZLIB=0", you will need + to build the library at least once. I recommand: + <blockquote> + make clean sqlite3.exe BUILD_ZLIB=1 + </blockquote> + + 4. After making the environment changes specified in steps 1 through 3 + above, you then build and test SQLite as you normally would. The + environment variable changes will cause ZLIB to be linked automatically. diff --git a/doc/lemon.html b/doc/lemon.html index 965f305c0..a994b396b 100644 --- a/doc/lemon.html +++ b/doc/lemon.html @@ -696,6 +696,7 @@ <h3>4.4 Special Directives</h3> <li><tt><a href='#pright'>%right</a></tt> <li><tt><a href='#reallc'>%realloc</a></tt> <li><tt><a href='#stack_overflow'>%stack_overflow</a></tt> +<li><tt><a href='#reallc'>%stack_size_limit</a></tt> <li><tt><a href='#stack_size'>%stack_size</a></tt> <li><tt><a href='#start_symbol'>%start_symbol</a></tt> <li><tt><a href='#syntax_error'>%syntax_error</a></tt> @@ -1203,20 +1204,33 @@ <h4>4.4.25 The <tt>%wildcard</tt> directive</h4> The wildcard token is only matched if there are no alternatives.</p> <a id='reallc'></a> -<h4>4.4.26 The <tt>%realloc</tt> and <tt>%free</tt> directives</h4> +<h4>4.4.26 The <tt>%realloc</tt>, <tt>%free</tt>, and +<tt>%stack_size_limit</tt> directives</h4> <p>The <tt>%realloc</tt> and <tt>%free</tt> directives defines function -that allocate and free heap memory. The signatures of these functions -should be the same as the realloc() and free() functions from the standard -C library. - -<p>If both of these functions are defined -then these functions are used to allocate and free -memory for supplemental parser stack space, if the initial -parse stack space is exceeded. The initial parser stack size +that allocate and free heap memory. The signatures and semantics of +these functions are similar to the realloc() and free() functions from +the standard C library, except that these functions take an extra +parameter at the end that is determined by %extra_context. If +%extra_context is not defined, then the extra argument is 0. The +extra parameter provides the capability to do better error reporting +in the event of a memory allocation error, and/or to use an alternative +private application heap. + +<p>If both of these functions are defined then they are used to +allocate and free memory for supplemental parser stack space, if +the initial parse stack space is exceeded. The initial parser stack size is specified by either <tt>%stack_size</tt> or the -DYYSTACKDEPTH compile-time flag. +<p>The <tt>%stack_size_limit</tt> directive defines a function that returns +the maximum allowed parser stack size. If this diretive does not exist, +no size limit is enforced. This function takes a single argument which +is the %extra_context value or "0" if %extra_context is not defined. +The function should return an integer that is the maximum +number of parser stack entries. If more stack space +than this is needed, the %stack_overflow code is invoked. + <a id='errors'></a> <h2>5.0 Error Processing</h2> diff --git a/doc/testrunner.md b/doc/testrunner.md index d1696e9d1..90ef4b71f 100644 --- a/doc/testrunner.md +++ b/doc/testrunner.md @@ -4,6 +4,12 @@ <ul type=none> <li> 1. <a href=#overview>Overview</a> +<ul type=none> + <li> 1.1. <a href=#runtr>Running testrunner.tcl</a> + <li> 1.2. <a href=#runviamake>Run using "make"</a> + <li> 1.3. <a href=#outputs>Outputs from testrunner.tcl</a> + <li> 1.4. <a href=#help>Built-in help</a> +</ul> <li> 2. <a href=#binary_tests>Binary Tests</a> <ul type=none> <li> 2.1. <a href=#organization_tests>Organization of Tcl Tests</a> @@ -26,17 +32,40 @@ The testrunner.tcl program is a Tcl script used to run multiple SQLite tests in parallel, thus reducing testing time on multi-core machines. -It supports the following types of tests: +The testrunner.tcl supports running tests that based on `testfixture`, +`sqlite3`, and `fuzzcheck`. + +<a name="runtr"></a> +## 1.1 Running testrunner.tcl + +The testrunner.tcl script is located in the "test" subdirectory of the +SQLite source tree. So if your shell is current positioned at the top +of the source tree, you would normally run the script using the command: +"<tt>test/testrunner.tcl</tt>". On Windows, you have to specify the +`tclsh` interpreter command first, like this: +"<tt>tclsh&nbsp;test/testrunner.tcl</tt>". + +In this document, we will assume that you are on a unix-like OS +(not on Windows) and that your current directory is the root +of the SQLite source tree, and so all invocations of the testrunner.tcl +script will be of the form "<tt>test/testrunner.tcl</tt>". If you +are in a different directory, then make appropriate adjustments to +the path. On Windows, add the "<tt>tclsh</tt>" interpreter command +up front. - * Tcl test scripts. +<a name="runviamake"></a> +## 1.2 Run using make - * Fuzzcheck tests, including using an external fuzzcheck database. +The standard Makefiles for SQLite include targets that invoke +testrunner.tcl. So the following commands also run testrunner.tcl: - * Tests run with `make` commands. Examples: - - `make devtest` - - `make releasetest` - - `make sdevtest` - - `make testrunner` + * `make devtest` + * `make releasetest` + * `make sdevtest` + * `make testrunner` + +<a name="outputs"></a> +## 1.3 Outputs from testrunner.tcl The testrunner.tcl program stores output of all tests and builds run in log file **testrunner.log**, created in the current working directory. @@ -54,17 +83,19 @@ A useful query might be: ``` You can get a summary of errors in a prior run by invoking commands like -these: +those shown below. Note that the testrunner.tcl script can be run directly +on unix systems (including Macs) but you will need to add <tt>tclsh</tt> +to the front on Windows. ``` - tclsh $(TESTDIR)/testrunner.tcl errors - tclsh $(TESTDIR)/testrunner.tcl errors -v + test/testrunner.tcl errors + test/testrunner.tcl errors -v ``` Running the command: ``` - tclsh $(TESTDIR)/testrunner.tcl status + test/testrunner.tcl status ``` in the directory containing the testrunner.db database runs various queries @@ -73,32 +104,43 @@ A good way to keep and eye on test progress is to run either of the two following commands: ``` - watch tclsh $(TESTDIR)/testrunner.tcl status - tclsh $(TESTDIR)/testrunner.tcl status -d 2 + watch test/testrunner.tcl status + test/testrunner.tcl status -d 2 ``` Both of the commands above accomplish about the same thing, but the second one has the advantage of not requiring "watch" to be installed on your system. -Sometimes testrunner.tcl uses the `testfixture` binary that it is run with -to run tests (see "Binary Tests" below). Sometimes it builds testfixture and +Sometimes testrunner.tcl uses the `testfixture` and `sqlite3` binaries that +are in the directory from which testrunner.tcl is run. +(see "Binary Tests" below). Sometimes it builds testfixture and other binaries in specific configurations to test (see "Source Tests"). +<a name=help></a> +## 1.4 Built-in help + +Run this command: + +``` + test/testrunner.tcl help +``` + +To get a summary of all of the various command-line options available +with testrunner.tcl + + <a name=binary_tests></a> # 2. Binary Tests The commands described in this section all run various combinations of the Tcl -test scripts using the `testfixture` binary used to run the testrunner.tcl -script (i.e. they do not invoke the compiler to build new binaries, or the -`make` command to run tests that are not Tcl scripts). The procedure to run -these tests is therefore: - - 1. Build the "testfixture" (or "testfixture.exe" for windows) binary using - whatever method seems convenient. - - 2. Test the binary built in step 1 by running testrunner.tcl with it, - perhaps with various options. +test scripts using whatever `testfixture` binary (and maybe also the `sqlite3` +binary, depending on the test) that is found in the directory from which +testrunner.tcl is launched. So typically, one must first run something +like "`make testfixture sqlite3`" before launching binary tests. In other +words, testrunner.tcl does not automatically build the binaries under test +for binary tests. The testrunner.tcl expects the binaries to be available +already. The following sub-sections describe the various options that can be passed to testrunner.tcl to test binary testfixture builds. @@ -140,22 +182,22 @@ are defined in file *testrunner_data.tcl*. To run the "veryquick" test set, use either of the following: ``` - ./testfixture $TESTDIR/testrunner.tcl - ./testfixture $TESTDIR/testrunner.tcl veryquick + test/testrunner.tcl + test/testrunner.tcl veryquick ``` To run the "full" test suite: ``` - ./testfixture $TESTDIR/testrunner.tcl full + test/testrunner.tcl full ``` To run the subset of the "full" test suite for which the test file name matches a specified pattern (e.g. all tests that start with "fts5"), either of: ``` - ./testfixture $TESTDIR/testrunner.tcl fts5% - ./testfixture $TESTDIR/testrunner.tcl 'fts5*' + test/testrunner.tcl fts5% + test/testrunner.tcl 'fts5*' ``` Strictly speaking, for a test to be run the pattern must match the script @@ -167,7 +209,7 @@ characters specified as part of the pattern are transformed to "\*". To run "all" tests (full + permutations): ``` - ./testfixture $TESTDIR/testrunner.tcl all + test/testrunner.tcl all ``` <a name=binary_test_failures></a> @@ -186,10 +228,15 @@ If there is no permutation, the individual test script may be run with: Or, if the failure occured as part of a permutation: ``` - ./testfixture $TESTDIR/testrunner.tcl $PERMUTATION $PATH_TO_SCRIPT + test/testrunner.tcl $PERMUTATION $PATH_TO_SCRIPT ``` -TODO: An example instead of "$PERMUTATION" and $PATH\_TO\_SCRIPT? +One can also rerun all tests that failed or did not complete +in the previous invocation by typing: + +``` + test/testrunner.tcl retest +``` <a name=source_code_tests></a> # 3. Source Code Tests @@ -204,11 +251,9 @@ other tests. The advantages of this are that: * it ensures that tests are always run using binaries created with the same set of compiler options. -The testrunner.tcl commands described in this section may be run using -either a *testfixture* (or testfixture.exe) build, or with any other Tcl -shell that supports SQLite 3.31.1 or newer via "package require sqlite3". - -TODO: ./configure + Makefile.msc build systems. +The testrunner.tcl commands described in this section do not require that +the testfixture and/or sqlite3 binaries be built ahead of time. Those +binaries will be constructed automatically. <a name=commands_to_run_tests></a> ## 3.1. Commands to Run SQLite Tests @@ -218,7 +263,7 @@ the `make fuzztest` target once for each of two --enable-all builds - one with debugging enabled and one without: ``` - tclsh $TESTDIR/testrunner.tcl mdevtest + test/testrunner.tcl mdevtest ``` In other words, it is equivalent to running: @@ -227,13 +272,13 @@ In other words, it is equivalent to running: $TOP/configure --enable-all --enable-debug make fuzztest make testfixture - ./testfixture $TOP/test/testrunner.tcl veryquick + $TOP/test/testrunner.tcl veryquick # Then, after removing files created by the tests above: $TOP/configure --enable-all OPTS="-O0" make fuzztest make testfixture - ./testfixture $TOP/test/testrunner.tcl veryquick + $TOP/test/testrunner.tcl veryquick ``` The **sdevtest** command is identical to the mdevtest command, except that the @@ -241,7 +286,7 @@ second of the two builds is a sanitizer build. Specifically, this means that OPTS="-fsanitize=address,undefined" is specified instead of OPTS="-O0": ``` - tclsh $TESTDIR/testrunner.tcl sdevtest + test/testrunner.tcl sdevtest ``` The **release** command runs lots of tests under lots of builds. It runs @@ -250,7 +295,7 @@ on Linux, Windows or OSX. Refer to *testrunner\_data.tcl* for the details of the specific tests run. ``` - tclsh $TESTDIR/testrunner.tcl release + test/testrunner.tcl release ``` As with <a href=#source code tests>source code tests</a>, one or more patterns @@ -258,7 +303,7 @@ may be appended to any of the above commands (mdevtest, sdevtest or release). Pattern matching is used for both Tcl tests and fuzz tests. ``` - tclsh $TESTDIR/testrunner.tcl release rtree% + test/testrunner.tcl release rtree% ``` <a name=zipvfs_tests></a> @@ -268,14 +313,14 @@ testrunner.tcl can build a zipvfs-enabled testfixture and use it to run tests from the Zipvfs project with the following command: ``` - tclsh $TESTDIR/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS + test/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS ``` This can be combined with any of "mdevtest", "sdevtest" or "release" to test both SQLite and Zipvfs with a single command: ``` - tclsh $TESTDIR/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS mdevtest + test/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS mdevtest ``` <a name=source_code_test_failures></a> @@ -295,10 +340,10 @@ a dos \*.bat file on windows. For example: ``` # Create a script that recreates build configuration "Device-One" on # Linux or OSX: - tclsh $TESTDIR/testrunner.tcl script Device-One > make.sh + test/testrunner.tcl script Device-One > make.sh # Create a script that recreates build configuration "Have-Not" on Windows: - tclsh $TESTDIR/testrunner.tcl script Have-Not > make.bat + test/testrunner.tcl script Have-Not > make.bat ``` The generated bash or \*.bat file script accepts a single argument - a makefile @@ -321,7 +366,7 @@ Thus, for example, to run a full releasetest including an external dbsqlfuzz database, run a command like one of these: ``` - tclsh test/testrunner.tcl releasetest --fuzzdb ../fuzz/20250415.db + test/testrunner.tcl releasetest --fuzzdb ../fuzz/20250415.db FUZZDB=../fuzz/20250415.db make releasetest nmake /f Makefile.msc FUZZDB=../fuzz/20250415.db releasetest ``` @@ -331,7 +376,7 @@ databases. So if you want to run *only* tests involving the external database, you can use a command something like this: ``` - tclsh test/testrunner.tcl releasetest 20250415 --fuzzdb ../fuzz/20250415.db + test/testrunner.tcl releasetest 20250415 --fuzzdb ../fuzz/20250415.db ``` <a name=testrunner_options></a> @@ -345,7 +390,7 @@ required by a test, not to run any actual tests. For example: ``` # Build binaries required by release test. - tclsh $TESTDIR/testrunner.tcl --buildonly release" + test/testrunner.tcl --buildonly release" ``` The **--dryrun** option prevents testrunner.tcl from building any binaries @@ -354,7 +399,7 @@ would normally execute into the testrunner.log file. Example: ``` # Log the shell commmands that make up the mdevtest test. - tclsh $TESTDIR/testrunner.tcl --dryrun mdevtest" + test/testrunner.tcl --dryrun mdevtest" ``` The **--explain** option is similar to --dryrun in that it prevents @@ -364,7 +409,7 @@ summary of all the builds and tests that would have been run. ``` # Show what builds and tests would have been run - tclsh $TESTDIR/testrunner.tcl --explain mdevtest + test/testrunner.tcl --explain mdevtest ``` The **--status** option uses VT100 escape sequences to display the test @@ -380,7 +425,7 @@ When running either binary or source code tests, testrunner.tcl reports the number of jobs it intends to use to stdout. e.g. ``` - $ ./testfixture $TESTDIR/testrunner.tcl + $ test/testrunner.tcl splitting work across 16 jobs ... more output ... ``` @@ -390,7 +435,7 @@ of real cores on the machine. This can be overridden using the "--jobs" (or -j) switch: ``` - $ ./testfixture $TESTDIR/testrunner.tcl --jobs 8 + $ test/testrunner.tcl --jobs 8 splitting work across 8 jobs ... more output ... ``` @@ -400,5 +445,5 @@ running by exucuting the following command from the directory containing the testrunner.log and testrunner.db files: ``` - $ ./testfixture $TESTDIR/testrunner.tcl njob $NEW_NUMBER_OF_JOBS + $ test/testrunner.tcl njob $NEW_NUMBER_OF_JOBS ``` diff --git a/ext/expert/expert1.test b/ext/expert/expert1.test index 0c3b512af..aaea03711 100644 --- a/ext/expert/expert1.test +++ b/ext/expert/expert1.test @@ -90,7 +90,7 @@ foreach {tn setup} { proc do_rec_test {tn sql res} { set res [squish [string trim $res]] set tst [subst -nocommands { - squish [string trim [exec $::CLI test.db ".expert" {$sql;}]] + squish [string trim [exec $::CLI -noinit test.db ".expert" {$sql;}]] }] uplevel [list do_test $tn $tst $res] } diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index f178abafe..368e9b189 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1816,9 +1816,7 @@ static int fts3CursorSeekStmt(Fts3Cursor *pCsr){ zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); if( !zSql ) return SQLITE_NOMEM; p->bLock++; - rc = sqlite3_prepare_v3( - p->db, zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0 - ); + rc = sqlite3Fts3PrepareStmt(p, zSql, 1, 1, &pCsr->pStmt); p->bLock--; sqlite3_free(zSql); } @@ -3393,9 +3391,7 @@ static int fts3FilterMethod( } if( zSql ){ p->bLock++; - rc = sqlite3_prepare_v3( - p->db,zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0 - ); + rc = sqlite3Fts3PrepareStmt(p, zSql, 1, 1, &pCsr->pStmt); p->bLock--; sqlite3_free(zSql); }else{ @@ -4018,6 +4014,7 @@ static int fts3IntegrityMethod( UNUSED_PARAMETER(isQuick); rc = sqlite3Fts3IntegrityCheck(p, &bOk); + assert( pVtab->zErrMsg==0 || rc!=SQLITE_OK ); assert( rc!=SQLITE_CORRUPT_VTAB ); if( rc==SQLITE_ERROR || (rc&0xFF)==SQLITE_CORRUPT ){ *pzErr = sqlite3_mprintf("unable to validate the inverted index for" diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index e98b90a75..fea31aae8 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -203,7 +203,16 @@ typedef sqlite3_int64 i64; /* 8-byte signed integer */ #define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) #define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) -#define deliberate_fall_through +#if !defined(deliberate_fall_through) +# if defined(__has_attribute) +# if __has_attribute(fallthrough) +# define deliberate_fall_through __attribute__((fallthrough)); +# endif +# endif +#endif +#if !defined(deliberate_fall_through) +# define deliberate_fall_through +#endif /* ** Macros needed to provide flexible arrays in a portable way @@ -601,6 +610,15 @@ int sqlite3Fts3Incrmerge(Fts3Table*,int,int); (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ ) +int sqlite3Fts3PrepareStmt( + Fts3Table *p, /* Prepare for this connection */ + const char *zSql, /* SQL to prepare */ + int bPersist, /* True to set SQLITE_PREPARE_PERSISTENT */ + int bAllowVtab, /* True to omit SQLITE_PREPARE_NO_VTAB */ + sqlite3_stmt **pp /* OUT: Prepared statement */ +); + + /* fts3.c */ void sqlite3Fts3ErrMsg(char**,const char*,...); int sqlite3Fts3PutVarint(char *, sqlite3_int64); diff --git a/ext/fts3/fts3_aux.c b/ext/fts3/fts3_aux.c index 439d57936..042fe5394 100644 --- a/ext/fts3/fts3_aux.c +++ b/ext/fts3/fts3_aux.c @@ -325,7 +325,7 @@ static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){ pCsr->aStat[1].nDoc++; } eState = 2; - /* fall through */ + /* no break */ deliberate_fall_through case 2: if( v==0 ){ /* 0x00. Next integer will be a docid. */ diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 19dff31f0..1b8bca70f 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -98,9 +98,9 @@ typedef struct SegmentWriter SegmentWriter; ** incrementally. See function fts3PendingListAppend() for details. */ struct PendingList { - int nData; + sqlite3_int64 nData; char *aData; - int nSpace; + sqlite3_int64 nSpace; sqlite3_int64 iLastDocid; sqlite3_int64 iLastCol; sqlite3_int64 iLastPos; @@ -273,6 +273,24 @@ struct SegmentNode { #define SQL_UPDATE_LEVEL_IDX 38 #define SQL_UPDATE_LEVEL 39 +/* +** Wrapper around sqlite3_prepare_v3() to ensure that SQLITE_PREPARE_FROM_DDL +** is always set. +*/ +int sqlite3Fts3PrepareStmt( + Fts3Table *p, /* Prepare for this connection */ + const char *zSql, /* SQL to prepare */ + int bPersist, /* True to set SQLITE_PREPARE_PERSISTENT */ + int bAllowVtab, /* True to omit SQLITE_PREPARE_NO_VTAB */ + sqlite3_stmt **pp /* OUT: Prepared statement */ +){ + int f = SQLITE_PREPARE_FROM_DDL + |((bAllowVtab==0) ? SQLITE_PREPARE_NO_VTAB : 0) + |(bPersist ? SQLITE_PREPARE_PERSISTENT : 0); + + return sqlite3_prepare_v3(p->db, zSql, -1, f, pp, NULL); +} + /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, @@ -398,12 +416,12 @@ static int fts3SqlStmt( pStmt = p->aStmt[eStmt]; if( !pStmt ){ - int f = SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_NO_VTAB; + int bAllowVtab = 0; char *zSql; if( eStmt==SQL_CONTENT_INSERT ){ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ - f &= ~SQLITE_PREPARE_NO_VTAB; + bAllowVtab = 1; zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist); }else{ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); @@ -411,7 +429,7 @@ static int fts3SqlStmt( if( !zSql ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3_prepare_v3(p->db, zSql, -1, f, &pStmt, NULL); + rc = sqlite3Fts3PrepareStmt(p, zSql, 1, bAllowVtab, &pStmt); sqlite3_free(zSql); assert( rc==SQLITE_OK || pStmt==0 ); p->aStmt[eStmt] = pStmt; @@ -760,7 +778,9 @@ static int fts3PendingTermsAddOne( pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); if( pList ){ - p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); + assert( (i64)pList->nData+(i64)nToken+(i64)sizeof(Fts3HashElem) + <= (i64)p->nPendingData ); + p->nPendingData -= (int)(pList->nData + nToken + sizeof(Fts3HashElem)); } if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ @@ -773,7 +793,9 @@ static int fts3PendingTermsAddOne( } } if( rc==SQLITE_OK ){ - p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + assert( (i64)p->nPendingData + pList->nData + nToken + + sizeof(Fts3HashElem) <= 0x3fffffff ); + p->nPendingData += (int)(pList->nData + nToken + sizeof(Fts3HashElem)); } return rc; } @@ -3574,7 +3596,7 @@ static int fts3DoRebuild(Fts3Table *p){ if( !zSql ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + rc = sqlite3Fts3PrepareStmt(p, zSql, 0, 1, &pStmt); sqlite3_free(zSql); } @@ -5327,7 +5349,7 @@ int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){ if( !zSql ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + rc = sqlite3Fts3PrepareStmt(p, zSql, 0, 1, &pStmt); sqlite3_free(zSql); } @@ -5457,7 +5479,7 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ v = atoi(&zVal[9]); if( v>=24 && v<=p->nPgsz-35 ) p->nNodeSize = v; rc = SQLITE_OK; - }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ + }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 11) ){ v = atoi(&zVal[11]); if( v>=64 && v<=FTS3_MAX_PENDING_DATA ) p->nMaxPendingData = v; rc = SQLITE_OK; diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index 95b33ea31..ee43ca6cc 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -455,7 +455,7 @@ static void fts5SnippetFunction( iBestCol = (iCol>=0 ? iCol : 0); nPhrase = pApi->xPhraseCount(pFts); - aSeen = sqlite3_malloc(nPhrase); + aSeen = sqlite3_malloc64(nPhrase); if( aSeen==0 ){ rc = SQLITE_NOMEM; } diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c index afcd83b6b..d799e34cb 100644 --- a/ext/fts5/fts5_buffer.c +++ b/ext/fts5/fts5_buffer.c @@ -288,7 +288,7 @@ char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){ if( nIn<0 ){ nIn = (int)strlen(pIn); } - zRet = (char*)sqlite3_malloc(nIn+1); + zRet = (char*)sqlite3_malloc64((i64)nIn+1); if( zRet ){ memcpy(zRet, pIn, nIn); zRet[nIn] = '\0'; diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index eea82b046..cea14b500 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -576,7 +576,7 @@ int sqlite3Fts5ConfigParse( sqlite3_int64 nByte; int bUnindexed = 0; /* True if there are one or more UNINDEXED */ - *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config)); + *ppOut = pRet = (Fts5Config*)sqlite3_malloc64(sizeof(Fts5Config)); if( pRet==0 ) return SQLITE_NOMEM; memset(pRet, 0, sizeof(Fts5Config)); pRet->pGlobal = pGlobal; @@ -1123,5 +1123,3 @@ void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){ va_end(ap); } - - diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 352df81f4..8ecaca34f 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -314,7 +314,7 @@ int sqlite3Fts5ExprNew( assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 ); if( sParse.rc==SQLITE_OK ){ - *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr)); + *ppNew = pNew = sqlite3_malloc64(sizeof(Fts5Expr)); if( pNew==0 ){ sParse.rc = SQLITE_NOMEM; sqlite3Fts5ParseNodeFree(sParse.pExpr); @@ -466,7 +466,7 @@ int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){ p2->pRoot = 0; if( sParse.rc==SQLITE_OK ){ - Fts5ExprPhrase **ap = (Fts5ExprPhrase**)sqlite3_realloc( + Fts5ExprPhrase **ap = (Fts5ExprPhrase**)sqlite3_realloc64( p1->apExprPhrase, nPhrase * sizeof(Fts5ExprPhrase*) ); if( ap==0 ){ diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c index a33dec9a9..ba4a030b7 100644 --- a/ext/fts5/fts5_hash.c +++ b/ext/fts5/fts5_hash.c @@ -91,7 +91,7 @@ int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){ int rc = SQLITE_OK; Fts5Hash *pNew; - *ppNew = pNew = (Fts5Hash*)sqlite3_malloc(sizeof(Fts5Hash)); + *ppNew = pNew = (Fts5Hash*)sqlite3_malloc64(sizeof(Fts5Hash)); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index b10df893f..164d61388 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -2093,7 +2093,7 @@ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ /* If necessary, grow the pIter->aRowidOffset[] array. */ if( iRowidOffset>=pIter->nRowidOffset ){ - int nNew = pIter->nRowidOffset + 8; + i64 nNew = pIter->nRowidOffset + 8; int *aNew = (int*)sqlite3_realloc64(pIter->aRowidOffset,nNew*sizeof(int)); if( aNew==0 ){ p->rc = SQLITE_NOMEM; @@ -5399,31 +5399,31 @@ static void fts5DoSecureDelete( ** is another term following it on this page. So the subsequent term ** needs to be moved to replace the term associated with the entry ** being removed. */ - int nPrefix = 0; - int nSuffix = 0; - int nPrefix2 = 0; - int nSuffix2 = 0; + u64 nPrefix = 0; + u64 nSuffix = 0; + u64 nPrefix2 = 0; + u64 nSuffix2 = 0; iDelKeyOff = iNextOff; - iNextOff += fts5GetVarint32(&aPg[iNextOff], nPrefix2); - iNextOff += fts5GetVarint32(&aPg[iNextOff], nSuffix2); + iNextOff += fts5GetVarint(&aPg[iNextOff], &nPrefix2); + iNextOff += fts5GetVarint(&aPg[iNextOff], &nSuffix2); if( iKey!=1 ){ - iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nPrefix); + iKeyOff += fts5GetVarint(&aPg[iKeyOff], &nPrefix); } - iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nSuffix); + iKeyOff += fts5GetVarint(&aPg[iKeyOff], &nSuffix); nPrefix = MIN(nPrefix, nPrefix2); nSuffix = (nPrefix2 + nSuffix2) - nPrefix; - if( (iKeyOff+nSuffix)>iPgIdx || (iNextOff+nSuffix2)>iPgIdx ){ + if( (iKeyOff+nSuffix)>(u64)iPgIdx || (iNextOff+nSuffix2)>(u64)iPgIdx ){ FTS5_CORRUPT_IDX(p); }else{ if( iKey!=1 ){ iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix); } iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix); - if( nPrefix2>pSeg->term.n ){ + if( nPrefix2>(u64)pSeg->term.n ){ FTS5_CORRUPT_IDX(p); }else if( nPrefix2>nPrefix ){ memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix); @@ -5454,7 +5454,7 @@ static void fts5DoSecureDelete( u8 *aTermIdx = &pTerm->p[pTerm->szLeaf]; int nTermIdx = pTerm->nn - pTerm->szLeaf; int iTermIdx = 0; - int iTermOff = 0; + i64 iTermOff = 0; while( 1 ){ u32 iVal = 0; @@ -5465,12 +5465,15 @@ static void fts5DoSecureDelete( } nTermIdx = iTermIdx; - memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); - fts5PutU16(&pTerm->p[2], iTermOff); - - fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); - if( nTermIdx==0 ){ - fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); + if( iTermOff>pTerm->szLeaf ){ + FTS5_CORRUPT_IDX(p); + }else{ + memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); + fts5PutU16(&pTerm->p[2], iTermOff); + fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); + if( nTermIdx==0 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); + } } } fts5DataRelease(pTerm); @@ -5493,7 +5496,9 @@ static void fts5DoSecureDelete( int iPrevKeyOut = 0; int iKeyIn = 0; - memmove(&aPg[iOff], &aPg[iNextOff], nMove); + if( nMove>0 ){ + memmove(&aPg[iOff], &aPg[iNextOff], nMove); + } iPgIdx -= nShift; nPg = iPgIdx; fts5PutU16(&aPg[2], iPgIdx); @@ -6413,16 +6418,16 @@ struct Fts5TokenDataMap { ** aMap[] variables. */ struct Fts5TokenDataIter { - int nMapAlloc; /* Allocated size of aMap[] in entries */ - int nMap; /* Number of valid entries in aMap[] */ + i64 nMapAlloc; /* Allocated size of aMap[] in entries */ + i64 nMap; /* Number of valid entries in aMap[] */ Fts5TokenDataMap *aMap; /* Array of (rowid+pos -> token) mappings */ /* The following are used for prefix-queries only. */ Fts5Buffer terms; /* The following are used for other full-token tokendata queries only. */ - int nIter; - int nIterAlloc; + i64 nIter; + i64 nIterAlloc; Fts5PoslistReader *aPoslistReader; int *aPoslistToIter; Fts5Iter *apIter[FLEXARRAY]; @@ -6478,11 +6483,11 @@ static void fts5TokendataIterAppendMap( ){ if( p->rc==SQLITE_OK ){ if( pT->nMap==pT->nMapAlloc ){ - int nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64; - int nAlloc = nNew * sizeof(Fts5TokenDataMap); + i64 nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64; + i64 nAlloc = nNew * sizeof(Fts5TokenDataMap); Fts5TokenDataMap *aNew; - aNew = (Fts5TokenDataMap*)sqlite3_realloc(pT->aMap, nAlloc); + aNew = (Fts5TokenDataMap*)sqlite3_realloc64(pT->aMap, nAlloc); if( aNew==0 ){ p->rc = SQLITE_NOMEM; return; @@ -6508,7 +6513,7 @@ static void fts5TokendataIterAppendMap( */ static void fts5TokendataIterSortMap(Fts5Index *p, Fts5TokenDataIter *pT){ Fts5TokenDataMap *aTmp = 0; - int nByte = pT->nMap * sizeof(Fts5TokenDataMap); + i64 nByte = pT->nMap * sizeof(Fts5TokenDataMap); aTmp = (Fts5TokenDataMap*)sqlite3Fts5MallocZero(&p->rc, nByte); if( aTmp ){ @@ -7042,9 +7047,10 @@ static Fts5TokenDataIter *fts5AppendTokendataIter( if( p->rc==SQLITE_OK ){ if( pIn==0 || pIn->nIter==pIn->nIterAlloc ){ - int nAlloc = pIn ? pIn->nIterAlloc*2 : 16; - int nByte = SZ_FTS5TOKENDATAITER(nAlloc+1); - Fts5TokenDataIter *pNew = (Fts5TokenDataIter*)sqlite3_realloc(pIn, nByte); + i64 nAlloc = pIn ? pIn->nIterAlloc*2 : 16; + i64 nByte = SZ_FTS5TOKENDATAITER(nAlloc+1); + Fts5TokenDataIter *pNew; + pNew = (Fts5TokenDataIter*)sqlite3_realloc64(pIn, nByte); if( pNew==0 ){ p->rc = SQLITE_NOMEM; @@ -7141,8 +7147,8 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){ /* Ensure the token-mapping is large enough */ if( eDetail==FTS5_DETAIL_FULL && pT->nMapAlloc<(pT->nMap + nByte) ){ - int nNew = (pT->nMapAlloc + nByte) * 2; - Fts5TokenDataMap *aNew = (Fts5TokenDataMap*)sqlite3_realloc( + i64 nNew = (pT->nMapAlloc + nByte) * 2; + Fts5TokenDataMap *aNew = (Fts5TokenDataMap*)sqlite3_realloc64( pT->aMap, nNew*sizeof(Fts5TokenDataMap) ); if( aNew==0 ){ diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index f45b9ef90..2e3b5b3af 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -517,7 +517,7 @@ static void fts5SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ if( sqlite3_libversion_number()>=3008002 ) #endif { - pIdxInfo->estimatedRows = nRow; + pIdxInfo->estimatedRows = MAX(1, nRow); } #endif } @@ -586,19 +586,30 @@ static int fts5UsePatternMatch( ** a) If a MATCH operator is present, the cost depends on the other ** constraints also present. As follows: ** -** * No other constraints: cost=1000.0 -** * One rowid range constraint: cost=750.0 -** * Both rowid range constraints: cost=500.0 -** * An == rowid constraint: cost=100.0 +** * No other constraints: cost=50000.0 +** * One rowid range constraint: cost=37500.0 +** * Both rowid range constraints: cost=30000.0 +** * An == rowid constraint: cost=25000.0 ** ** b) Otherwise, if there is no MATCH: ** -** * No other constraints: cost=1000000.0 -** * One rowid range constraint: cost=750000.0 -** * Both rowid range constraints: cost=250000.0 -** * An == rowid constraint: cost=10.0 +** * No other constraints: cost=3000000.0 +** * One rowid range constraints: cost=2250000.0 +** * Both rowid range constraint: cost=750000.0 +** * An == rowid constraint: cost=25.0 ** ** Costs are not modified by the ORDER BY clause. +** +** The ratios used in case (a) are based on informal results obtained from +** the tool/fts5cost.tcl script. The "MATCH and ==" combination has the +** cost set quite high because the query may be a prefix query. Unless +** there is a prefix index, prefix queries with rowid constraints are much +** more expensive than non-prefix queries with rowid constraints. +** +** The estimated rows returned is set to the cost/40. For simple queries, +** experimental results show that cost/4 might be about right. But for +** more complex queries that use multiple terms the number of rows might +** be far fewer than this. So we compromise and use cost/40. */ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ Fts5Table *pTab = (Fts5Table*)pVTab; @@ -631,7 +642,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ return SQLITE_ERROR; } - idxStr = (char*)sqlite3_malloc(pInfo->nConstraint * 8 + 1); + idxStr = (char*)sqlite3_malloc64((i64)pInfo->nConstraint * 8 + 1); if( idxStr==0 ) return SQLITE_NOMEM; pInfo->idxStr = idxStr; pInfo->needToFreeIdxStr = 1; @@ -724,21 +735,35 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ /* Calculate the estimated cost based on the flags set in idxFlags. */ if( bSeenEq ){ - pInfo->estimatedCost = nSeenMatch ? 1000.0 : 25.0; - fts5SetUniqueFlag(pInfo); + pInfo->estimatedCost = nSeenMatch ? 25000.0 : 25.0; fts5SetEstimatedRows(pInfo, 1); + fts5SetUniqueFlag(pInfo); }else{ - if( bSeenLt && bSeenGt ){ - pInfo->estimatedCost = nSeenMatch ? 5000.0 : 750000.0; - }else if( bSeenLt || bSeenGt ){ - pInfo->estimatedCost = nSeenMatch ? 7500.0 : 2250000.0; + i64 nEstRows; + if( nSeenMatch ){ + if( bSeenLt && bSeenGt ){ + pInfo->estimatedCost = 50000.0; + }else if( bSeenLt || bSeenGt ){ + pInfo->estimatedCost = 37500.0; + }else{ + pInfo->estimatedCost = 50000.0; + } + nEstRows = (i64)(pInfo->estimatedCost / 40.0); + for(i=1; i<nSeenMatch; i++){ + pInfo->estimatedCost *= 2.5; + nEstRows = nEstRows / 2; + } }else{ - pInfo->estimatedCost = nSeenMatch ? 10000.0 : 3000000.0; - } - for(i=1; i<nSeenMatch; i++){ - pInfo->estimatedCost *= 0.4; + if( bSeenLt && bSeenGt ){ + pInfo->estimatedCost = 750000.0; + }else if( bSeenLt || bSeenGt ){ + pInfo->estimatedCost = 2250000.0; + }else{ + pInfo->estimatedCost = 3000000.0; + } + nEstRows = (i64)(pInfo->estimatedCost / 4.0); } - fts5SetEstimatedRows(pInfo, (i64)(pInfo->estimatedCost / 4.0)); + fts5SetEstimatedRows(pInfo, nEstRows); } pInfo->idxNum = idxFlags; @@ -2081,6 +2106,7 @@ static int fts5UpdateMethod( } update_out: + sqlite3Fts5IndexCloseReader(pTab->p.pIndex); pTab->p.pConfig->pzErrmsg = 0; return rc; } @@ -3762,7 +3788,7 @@ static int fts5Init(sqlite3 *db){ int rc; Fts5Global *pGlobal = 0; - pGlobal = (Fts5Global*)sqlite3_malloc(sizeof(Fts5Global)); + pGlobal = (Fts5Global*)sqlite3_malloc64(sizeof(Fts5Global)); if( pGlobal==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index 25cd5c063..f5d8705ff 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -391,7 +391,7 @@ static int SQLITE_TCLAPI xF5tApi( break; } CASE(12, "xSetAuxdata") { - F5tAuxData *pData = (F5tAuxData*)sqlite3_malloc(sizeof(F5tAuxData)); + F5tAuxData *pData = (F5tAuxData*)sqlite3_malloc64(sizeof(F5tAuxData)); if( pData==0 ){ Tcl_AppendResult(interp, "out of memory", (char*)0); return TCL_ERROR; @@ -780,7 +780,7 @@ static int SQLITE_TCLAPI f5tTokenize( } if( nText>0 ){ - pCopy = sqlite3_malloc(nText); + pCopy = sqlite3_malloc64(nText); if( pCopy==0 ){ tokenizer.xDelete(pTok); Tcl_AppendResult(interp, "error in sqlite3_malloc()", (char*)0); @@ -1420,7 +1420,7 @@ static int f5tOrigintextCreate( void *pTokCtx = 0; int rc = SQLITE_OK; - pTok = (OriginTextTokenizer*)sqlite3_malloc(sizeof(OriginTextTokenizer)); + pTok = (OriginTextTokenizer*)sqlite3_malloc64(sizeof(OriginTextTokenizer)); if( pTok==0 ){ rc = SQLITE_NOMEM; }else if( nArg<1 ){ @@ -1480,7 +1480,7 @@ static int xOriginToken( int nReq = nToken + 1 + (iEnd-iStart); if( nReq>p->nBuf ){ sqlite3_free(p->aBuf); - p->aBuf = sqlite3_malloc(nReq*2); + p->aBuf = sqlite3_malloc64(nReq*2); if( p->aBuf==0 ) return SQLITE_NOMEM; p->nBuf = nReq*2; } diff --git a/ext/fts5/fts5_test_tok.c b/ext/fts5/fts5_test_tok.c index 994d304dc..c77c49de7 100644 --- a/ext/fts5/fts5_test_tok.c +++ b/ext/fts5/fts5_test_tok.c @@ -194,7 +194,7 @@ static int fts5tokConnectMethod( } if( rc==SQLITE_OK ){ - pTab = (Fts5tokTable*)sqlite3_malloc(sizeof(Fts5tokTable)); + pTab = (Fts5tokTable*)sqlite3_malloc64(sizeof(Fts5tokTable)); if( pTab==0 ){ rc = SQLITE_NOMEM; }else{ @@ -275,7 +275,7 @@ static int fts5tokBestIndexMethod( static int fts5tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ Fts5tokCursor *pCsr; - pCsr = (Fts5tokCursor *)sqlite3_malloc(sizeof(Fts5tokCursor)); + pCsr = (Fts5tokCursor *)sqlite3_malloc64(sizeof(Fts5tokCursor)); if( pCsr==0 ){ return SQLITE_NOMEM; } @@ -347,7 +347,7 @@ static int fts5tokCb( if( pCsr->nRow ){ pRow->iPos = pRow[-1].iPos + ((tflags & FTS5_TOKEN_COLOCATED) ? 0 : 1); } - pRow->zToken = sqlite3_malloc(nToken+1); + pRow->zToken = sqlite3_malloc64((sqlite3_int64)nToken+1); if( pRow->zToken==0 ) return SQLITE_NOMEM; memcpy(pRow->zToken, pToken, nToken); pRow->zToken[nToken] = 0; @@ -373,8 +373,8 @@ static int fts5tokFilterMethod( fts5tokResetCursor(pCsr); if( idxNum==1 ){ const char *zByte = (const char *)sqlite3_value_text(apVal[0]); - int nByte = sqlite3_value_bytes(apVal[0]); - pCsr->zInput = sqlite3_malloc(nByte+1); + sqlite3_int64 nByte = sqlite3_value_bytes(apVal[0]); + pCsr->zInput = sqlite3_malloc64(nByte+1); if( pCsr->zInput==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c index b8a113646..990810239 100644 --- a/ext/fts5/fts5_tokenize.c +++ b/ext/fts5/fts5_tokenize.c @@ -72,7 +72,7 @@ static int fts5AsciiCreate( if( nArg%2 ){ rc = SQLITE_ERROR; }else{ - p = sqlite3_malloc(sizeof(AsciiTokenizer)); + p = sqlite3_malloc64(sizeof(AsciiTokenizer)); if( p==0 ){ rc = SQLITE_NOMEM; }else{ @@ -367,7 +367,7 @@ static int fts5UnicodeCreate( if( nArg%2 ){ rc = SQLITE_ERROR; }else{ - p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer)); + p = (Unicode61Tokenizer*)sqlite3_malloc64(sizeof(Unicode61Tokenizer)); if( p ){ const char *zCat = "L* N* Co"; int i; @@ -590,7 +590,7 @@ static int fts5PorterCreate( zBase = azArg[0]; } - pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer)); + pRet = (PorterTokenizer*)sqlite3_malloc64(sizeof(PorterTokenizer)); if( pRet ){ memset(pRet, 0, sizeof(PorterTokenizer)); rc = pApi->xFindTokenizer_v2(pApi, zBase, &pUserdata, &pV2); @@ -1297,7 +1297,7 @@ static int fts5TriCreate( rc = SQLITE_ERROR; }else{ int i; - pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew)); + pNew = (TrigramTokenizer*)sqlite3_malloc64(sizeof(*pNew)); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c index 3a6a968f7..295ace6ba 100644 --- a/ext/fts5/fts5_vocab.c +++ b/ext/fts5/fts5_vocab.c @@ -666,7 +666,7 @@ static int fts5VocabFilterMethod( const char *zCopy = (const char *)sqlite3_value_text(pLe); if( zCopy==0 ) zCopy = ""; pCsr->nLeTerm = sqlite3_value_bytes(pLe); - pCsr->zLeTerm = sqlite3_malloc(pCsr->nLeTerm+1); + pCsr->zLeTerm = sqlite3_malloc64((i64)pCsr->nLeTerm+1); if( pCsr->zLeTerm==0 ){ rc = SQLITE_NOMEM; }else{ diff --git a/ext/fts5/test/fts5corrupt9.test b/ext/fts5/test/fts5corrupt9.test index 102a4135b..6cf06f836 100644 --- a/ext/fts5/test/fts5corrupt9.test +++ b/ext/fts5/test/fts5corrupt9.test @@ -54,6 +54,75 @@ do_catchsql_test 1.3 { DELETE FROM t WHERE rowid=3 } {0 {}} +#------------------------------------------------------------------------- +reset_db + +set nRow 8000 +set zText aaa + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t USING fts5(content, detail=none); + INSERT INTO t(t, rank) VALUES('secure-delete', 1); + BEGIN; +} +do_test 2.1 { + for {set ii 0} {$ii<$nRow} {incr ii} { + execsql { INSERT INTO t(content) VALUES($zText); } + } + execsql { + COMMIT; + INSERT INTO t(t) VALUES('optimize'); + } +} {} + +set hex "00040f7d9f49[string repeat {01} 3958]80" + +do_execsql_test 2.1 " + UPDATE t_data SET block = X'$hex' WHERE rowid=137438953474; +" + +do_execsql_test 2.3 { + DELETE FROM t WHERE rowid=7999 +} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t USING fts5(content, detail=none); + INSERT INTO t(t, rank) VALUES('secure-delete', 1); + INSERT INTO t(content) VALUES('aaa'); + INSERT INTO t(content) VALUES('bbb'); + INSERT INTO t(t) VALUES('optimize'); +} + +do_execsql_test 3.1 { + UPDATE t_data SET block = X'000000100430616161010187ffffff7f0406' WHERE rowid=412316860417; +} + +do_catchsql_test 3.2 { + DELETE FROM t WHERE rowid=1; +} {1 {fts5: corruption in table "t"}} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t USING fts5(content); + INSERT INTO t(t, rank) VALUES('secure-delete', 1); + INSERT INTO t(content) VALUES('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'); + INSERT INTO t(t) VALUES('optimize'); +} + +do_execsql_test 4.1 { + UPDATE t_data SET block = X'00000fce9f4830616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161610487ffffff7f' WHERE rowid=137438953473; + UPDATE t_data SET block = X'0004000801040203' WHERE rowid=137438953474; +} + +do_catchsql_test 4.2 { + DELETE FROM t WHERE rowid = 1; +} {1 {fts5: corruption in table "t"}} + sqlite3_fts5_may_be_corrupt 0 finish_test diff --git a/ext/fts5/test/fts5integrity.test b/ext/fts5/test/fts5integrity.test index 4bf120c44..9b2720faf 100644 --- a/ext/fts5/test/fts5integrity.test +++ b/ext/fts5/test/fts5integrity.test @@ -379,9 +379,6 @@ do_execsql_test 12.2 { db close sqlite3 db test.db -readonly 1 -explain_i { - PRAGMA integrity_check - } do_execsql_test 12.3 { PRAGMA integrity_check } {ok} diff --git a/ext/fts5/test/fts5interrupt.test b/ext/fts5/test/fts5interrupt.test index 67ef5f7e9..87b232b05 100644 --- a/ext/fts5/test/fts5interrupt.test +++ b/ext/fts5/test/fts5interrupt.test @@ -64,4 +64,21 @@ foreach {tn sql} { } } +#------------------------------------------------------------------------- +# Verify that https://sqlite.org/forum/forumpost/95413eb410 has been +# fixed. +# +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE f1 USING fts5(x); + BEGIN TRANSACTION; + INSERT INTO f1(x) VALUES('abc def ghi'); +} +do_test 2.1 { + sqlite3_interrupt db +} {} +do_execsql_test 2.2 { + ROLLBACK +} + finish_test diff --git a/ext/fts5/test/fts5join.test b/ext/fts5/test/fts5join.test index e4d3b69b7..2b9945a6f 100644 --- a/ext/fts5/test/fts5join.test +++ b/ext/fts5/test/fts5join.test @@ -65,5 +65,14 @@ do_eqp_test 1.4 { `--SCAN vt VIRTUAL TABLE INDEX 0:= } +do_eqp_test 1.5 { + SELECT * FROM vt, t1 + WHERE vt.rowid = t1.rowid AND vt MATCH ? AND b = ? +} { + QUERY PLAN + |--SCAN vt VIRTUAL TABLE INDEX 0:M1 + `--SEARCH t1 USING INTEGER PRIMARY KEY (rowid=?) +} + finish_test diff --git a/ext/fts5/tool/fts5cost.tcl b/ext/fts5/tool/fts5cost.tcl new file mode 100644 index 000000000..4f53d29eb --- /dev/null +++ b/ext/fts5/tool/fts5cost.tcl @@ -0,0 +1,153 @@ +# +# 2026 March 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#-------------------------------------------------------------------------- +# +# This script extracts the documentation for the API used by fts5 auxiliary +# functions from header file fts5.h. It outputs html text on stdout that +# is included in the documentation on the web. +# + + +sqlite3 db fts5cost.db + +# Create an IPK table with 1,000,000 entries. Short records. +# +set res [list [catch { db eval {SELECT count(*) FROM t1} } msg] $msg] +if {$res!="0 1000000"} { + db eval { + PRAGMA mmap_size = 1000000000; -- 1GB + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1_000_000 + ) + INSERT INTO t1 SELECT i, hex(randomblob(8)) FROM s; + } +} + +# Create an FTS5 table with 1,000,000 entries. Each row contains a single +# column containing a document of 100 terms chosen pseudo-randomly from +# a vocabularly of 2000. +set res [list [catch { db eval {SELECT count(*) FROM f1} } msg] $msg] +if {$res!="0 1000000"} { + set nVocab 2000 + set nTerm 100 + db eval { + BEGIN; + DROP TABLE IF EXISTS vocab1; + CREATE TABLE vocab1(w); + } + for {set ii 0} {$ii<$nVocab} {incr ii} { + set word [format %06x [expr {int(abs(rand()) * 0xFFFFFF)}]] + db eval { INSERT INTO vocab1 VALUES($word) } + lappend lVocab $word + } + db func doc doc + proc doc {} { + for {set ii 0} {$ii<$::nTerm} {incr ii} { + lappend ret [lindex $::lVocab [expr int(abs(rand())*$::nVocab)]] + } + set ret + } + db eval { + DROP TABLE IF EXISTS f1; + CREATE VIRTUAL TABLE f1 USING fts5(x); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1_000_000 + ) + INSERT INTO f1(rowid, x) SELECT i, doc() FROM s; + COMMIT; + } +} else { + set lVocab [db eval { SELECT * FROM vocab1 }] + set nVocab [llength $lVocab] +} + +proc rowid_query {n} { + set rowid 654 + for {set ii 0} {$ii<$n} {incr ii} { + db eval { SELECT b FROM t1 WHERE a = $rowid } + set rowid [expr {($rowid + 7717) % 1000000}] + } +} + +proc rowid_query_fts {n} { + set rowid 654 + for {set ii 0} {$ii<$n} {incr ii} { + db eval { SELECT * FROM f1 WHERE rowid = $rowid } + set rowid [expr {($rowid + 7717) % 1000000}] + } +} + +proc match_query_fts {n} { + set idx 654 + for {set ii 0} {$ii<$n} {incr ii} { + set match [lrange $::lVocab $idx $idx+1] + db eval { SELECT * FROM f1($match) } + set idx [expr {($idx + 7717) % $::nVocab}] + } +} + +proc prefix_query_fts {n} { + set idx 654 + for {set ii 0} {$ii<$n} {incr ii} { + set match "[lindex $::lVocab $idx]*" + db eval { SELECT * FROM f1($match) } + set idx [expr {($idx + 7717) % $::nVocab}] + } +} + +proc match_rowid_query_fts {n} { + set idx 654 + for {set ii 0} {$ii<$n} {incr ii} { + set match "[lindex $::lVocab $idx]" + db eval { SELECT * FROM f1($match) WHERE rowid=500000 } + set idx [expr {($idx + 7717) % $::nVocab}] + } +} + +proc prefix_rowid_query_fts {n} { + set idx 654 + for {set ii 0} {$ii<$n} {incr ii} { + set match "[lindex $::lVocab $idx]*" + db eval { SELECT * FROM f1($match) WHERE rowid=500000 } + set idx [expr {($idx + 7717) % $::nVocab}] + } +} + + +proc mytime {cmd div} { + set tm [time $cmd] + expr {[lindex $tm 0] / $div} +} + +#set us [mytime { match_rowid_query_fts 1000 } 1000] +#puts "1000 match/rowid queries on fts5 table: ${us} per query" + +set us [mytime { prefix_rowid_query_fts 1000 } 1000] +puts "1000 prefix/rowid queries on fts5 table: ${us} per query" + +set us [mytime { match_query_fts 10 } 10] +puts "10 match queries on fts5 table: ${us} per query" + +set us [mytime { prefix_query_fts 10 } 10] +puts "10 prefix queries on fts5 table: ${us} per query" + +set us [mytime { prefix_rowid_query_fts 1000 } 1000] +puts "1000 prefix/rowid queries on fts5 table: ${us} per query" + +set us [mytime { rowid_query 10000 } 10000] +puts "10000 by-rowid queries on normal table: ${us} per query" + +set us [mytime { rowid_query_fts 10000 } 10000] +puts "10000 by-rowid queries on fts5 table: ${us} per query" + + diff --git a/ext/intck/sqlite3intck.c b/ext/intck/sqlite3intck.c index 5f645fae6..e3fef7763 100644 --- a/ext/intck/sqlite3intck.c +++ b/ext/intck/sqlite3intck.c @@ -319,7 +319,7 @@ static int intckGetToken(const char *z){ char c = z[0]; int iRet = 1; if( c=='\'' || c=='"' || c=='`' ){ - while( 1 ){ + while( z[iRet] ){ if( z[iRet]==c ){ iRet++; if( z[iRet]!=c ) break; diff --git a/ext/jni/README.md b/ext/jni/README.md index 5ad79fce9..0bdbde91e 100644 --- a/ext/jni/README.md +++ b/ext/jni/README.md @@ -13,11 +13,14 @@ Technical support is available in the forum: <https://sqlite.org/forum> -> **FOREWARNING:** this subproject is very much in development and - subject to any number of changes. Please do not rely on any - information about its API until this disclaimer is removed. The JNI - bindings released with version 3.43 are a "tech preview." Once - finalized, strong backward compatibility guarantees will apply. +> **FOREWARNING:** the JNI subproject is experimental and subject to + any number of changes. This API is "feature-complete", with only a + few difficult-to-reach corners of the C API not represented here, + but it is not a supported deliverable of the project so does not + have same backward compatibility guarantees which the C APIs + do. That said: the [C-style API](#1to1ish) is especially resistent + to compatibility breakage because it's designed to be as close to + the C API as feasible. Project goals/requirements: @@ -162,11 +165,13 @@ or propagate exceptions and must return error information (if any) via result codes or `null`. The only cases where the C-style APIs may throw is through client-side misuse, e.g. passing in a null where it may cause a `NullPointerException`. The APIs clearly mark function -parameters which should not be null, but does not generally actively -defend itself against such misuse. Some C-style APIs explicitly accept -`null` as a no-op for usability's sake, and some of the JNI APIs -deliberately return an error code, instead of segfaulting, when passed -a `null`. +parameters which should not be null, and it internally uses the +`SQLITE_API_ARMOR` mechanism to help product against such misuse. Some +C-style APIs explicitly accept `null` as a no-op for usability's sake, +and some of the JNI APIs deliberately return an error code, instead of +segfaulting, when passed a `null`. There are no known cases where it +will misuse memory if passed a `null` or out-of-range value from +client code. Client-defined callbacks _must never throw exceptions_ unless _very explicitly documented_ as being throw-safe. Exceptions are generally @@ -194,7 +199,8 @@ Some constructs, when modelled 1-to-1 from C to Java, are unduly clumsy to work with in Java because they try to shoehorn C's way of doing certain things into Java's wildly different ways. The following subsections cover those, starting with a verbose explanation and -demonstration of where such changes are "really necessary"... +demonstration of where such changes are "really necessary" for +usability's sake... ### Custom Collations @@ -286,12 +292,9 @@ binding. The Java API has only one core function-registration function: ```java int sqlite3_create_function(sqlite3 db, String funcName, int nArgs, - int encoding, SQLFunction func); + int flags, SQLFunction func); ``` -> Design question: does the encoding argument serve any purpose in - Java? That's as-yet undetermined. If not, it will be removed. - `SQLFunction` is not used directly, but is instead instantiated via one of its three subclasses: @@ -313,4 +316,4 @@ in-flux nature of this API. Various APIs which accept callbacks, e.g. `sqlite3_trace_v2()` and `sqlite3_update_hook()`, use interfaces similar to those shown above. Despite the changes in signature, the JNI layer makes every effort to -provide the same semantics as the C API documentation suggests. +provide the same semantics as the C API documentation describes. diff --git a/ext/jni/src/c/sqlite3-jni.c b/ext/jni/src/c/sqlite3-jni.c index f130eff04..8cdba9bcf 100644 --- a/ext/jni/src/c/sqlite3-jni.c +++ b/ext/jni/src/c/sqlite3-jni.c @@ -133,11 +133,9 @@ ** Which sqlite3.c we're using needs to be configurable to enable ** building against a custom copy, e.g. the SEE variant. We have to ** include sqlite3.c, as opposed to sqlite3.h, in order to get access -** to some internal details like SQLITE_MAX_... and friends. This -** increases the rebuild time considerably but we need this in order -** to access some internal functionality and keep the to-Java-exported -** values of SQLITE_MAX_... and SQLITE_LIMIT_... in sync with the C -** build. +** to some internal details like SQLITE_MAX_... and friends, and keep +** those consistent with this build. This increases the rebuild time +** considerably, however. */ #ifndef SQLITE_C # define SQLITE_C sqlite3.c @@ -335,7 +333,7 @@ struct S3JniNphOp { const char * const zMember /* Name of member property */; const char * const zTypeSig /* JNI type signature of zMember */; /* - ** klazz is a global ref to the class represented by pRef. + ** klazz is a global ref to the class represented by zName. ** ** According to: ** @@ -999,19 +997,20 @@ static S3JniEnv * S3JniEnv__get(JNIEnv * const env){ ** JNI bindings such as sqlite3_prepare_v2/v3(), and definitely not ** from client code. ** -** Returns err_code. +** Returns err_code _unless_ err_code is 0 and sqlite3_set_errmsg() +** fails with OOM, in which case it may return SQLITE_OOM or fail +** fatally. +** +** This function predates sqlite3_set_errmsg(), which is why it has a +** slightly different interface. Before that function was introduced, +** this code used the SQLite-internal APIs to do this. */ -static int s3jni_db_error(sqlite3* const db, int err_code, - const char * const zMsg){ +static int s3jni_db_error(JNIEnv * env, sqlite3* const db, + int err_code, const char * const zMsg){ if( db!=0 ){ - if( 0==zMsg ){ - sqlite3Error(db, err_code); - }else{ - const int nMsg = sqlite3Strlen30(zMsg); - sqlite3_mutex_enter(sqlite3_db_mutex(db)); - sqlite3ErrorWithMsg(db, err_code, "%.*s", nMsg, zMsg); - sqlite3_mutex_leave(sqlite3_db_mutex(db)); - } + int const rc = sqlite3_set_errmsg(db, err_code, zMsg); + s3jni_oom_fatal(0==rc); + if( rc && !err_code ) err_code=rc; } return err_code; } @@ -1235,11 +1234,11 @@ static int s3jni__db_exception(JNIEnv * const env, sqlite3 * const pDb, char * zMsg; S3JniExceptionClear; zMsg = s3jni_exception_error_msg(env, ex); - s3jni_db_error(pDb, errCode, zMsg ? zMsg : zDfltMsg); + s3jni_db_error(env, pDb, errCode, zMsg ? zMsg : zDfltMsg); sqlite3_free(zMsg); S3JniUnrefLocal(ex); }else if( zDfltMsg ){ - s3jni_db_error(pDb, errCode, zDfltMsg); + s3jni_db_error(env, pDb, errCode, zDfltMsg); } return errCode; } @@ -1956,15 +1955,6 @@ static void S3JniUdf_finalizer(void * s){ S3JniUdf_free(s3jni_env(), (S3JniUdf*)s, 1); } -/* -** Helper for processing args to UDF handlers with signature -** (sqlite3_context*,int,sqlite3_value**). -*/ -typedef struct { - jobject jcx /* sqlite3_context */; - jobjectArray jargv /* sqlite3_value[] */; -} udf_jargs; - /* ** Converts the given (cx, argc, argv) into arguments for the given ** UDF, writing the result (Java wrappers for cx and argv) in the @@ -2006,7 +1996,7 @@ static int udf_args(JNIEnv *env, /* ** Requires that jCx and jArgv are sqlite3_context ** resp. array-of-sqlite3_value values initialized by udf_args(). The -** latter will be 0-and-NULL for UDF types with no arguments. This +** (argc,argv) are (0,NULL) for UDF types with no arguments. This ** function zeroes out the nativePointer member of jCx and each entry ** in jArgv. This is a safety-net precaution to avoid undefined ** behavior if a Java-side UDF holds a reference to its context or one @@ -2099,19 +2089,19 @@ static int udf_xFSI(sqlite3_context* const pCx, int argc, sqlite3_value** const argv, S3JniUdf * const s, jmethodID xMethodID, const char * const zFuncType){ S3JniDeclLocal_env; - udf_jargs args = {0,0}; - int rc = udf_args(env, pCx, argc, argv, &args.jcx, &args.jargv); - + jobject jcx = 0 /* sqlite3_context */; + jobjectArray jargv = 0 /* sqlite3_value[] */; + int rc = udf_args(env, pCx, argc, argv, &jcx, &jargv); if( 0 == rc ){ - (*env)->CallVoidMethod(env, s->jObj, xMethodID, args.jcx, args.jargv); + (*env)->CallVoidMethod(env, s->jObj, xMethodID, jcx, jargv); S3JniIfThrew{ rc = udf_report_exception(env, 'F'==zFuncType[1]/*xFunc*/, pCx, s->zFuncName, zFuncType); } - udf_unargs(env, args.jcx, argc, args.jargv); + udf_unargs(env, jcx, argc, jargv); } - S3JniUnrefLocal(args.jcx); - S3JniUnrefLocal(args.jargv); + S3JniUnrefLocal(jcx); + S3JniUnrefLocal(jargv); return rc; } @@ -3300,7 +3290,7 @@ static jobject s3jni_commit_rollback_hook(int isCommit, JNIEnv * const env, S3JniDb_mutex_enter; ps = S3JniDb_from_jlong(jpDb); if( !ps ){ - s3jni_db_error(ps->pDb, SQLITE_MISUSE, 0); + s3jni_db_error(env, ps->pDb, SQLITE_MISUSE, 0); S3JniDb_mutex_leave; return 0; } @@ -3320,13 +3310,14 @@ static jobject s3jni_commit_rollback_hook(int isCommit, JNIEnv * const env, else sqlite3_rollback_hook(ps->pDb, 0, 0); }else{ jclass const klazz = (*env)->GetObjectClass(env, jHook); - jmethodID const xCallback = (*env)->GetMethodID(env, klazz, "call", - isCommit ? "()I" : "()V"); + jmethodID const xCallback = + (*env)->GetMethodID(env, klazz, "call", + isCommit ? "()I" : "()V"); S3JniUnrefLocal(klazz); S3JniIfThrew { S3JniExceptionReport; S3JniExceptionClear; - s3jni_db_error(ps->pDb, SQLITE_ERROR, + s3jni_db_error(env, ps->pDb, SQLITE_ERROR, "Cannot not find matching call() method in" "hook object."); }else{ @@ -3606,7 +3597,7 @@ S3JniApi(sqlite3_create_collation() sqlite3_create_collation_v2(), (*env)->GetMethodID(env, klazz, "call", "([B[B)I"); S3JniUnrefLocal(klazz); S3JniIfThrew{ - rc = s3jni_db_error(ps->pDb, SQLITE_ERROR, + rc = s3jni_db_error(env, ps->pDb, SQLITE_ERROR, "Could not get call() method from " "CollationCallback object."); }else{ @@ -3645,15 +3636,15 @@ S3JniApi(sqlite3_create_function() sqlite3_create_function_v2() if( !pDb || !jFuncName ){ return SQLITE_MISUSE; - }else if( !encodingTypeIsValid(eTextRep) ){ - return s3jni_db_error(pDb, SQLITE_FORMAT, + }else if( !encodingTypeIsValid(eTextRep & 0x0f) ){ + return s3jni_db_error(env, pDb, SQLITE_FORMAT, "Invalid function encoding option."); } s = S3JniUdf_alloc(env, jFunctor); if( !s ) return SQLITE_NOMEM; if( UDF_UNKNOWN_TYPE==s->type ){ - rc = s3jni_db_error(pDb, SQLITE_MISUSE, + rc = s3jni_db_error(env, pDb, SQLITE_MISUSE, "Cannot unambiguously determine function type."); S3JniUdf_free(env, s, 1); goto error_cleanup; @@ -4012,7 +4003,7 @@ S3JniApi(sqlite3_jni_db_error(), jint, 1jni_1db_1error)( zStr = jStr ? s3jni_jstring_to_utf8( jStr, 0) : NULL; - rc = s3jni_db_error( ps->pDb, (int)jRc, zStr ); + rc = s3jni_db_error(env, ps->pDb, (int)jRc, zStr ); sqlite3_free(zStr); } return rc; @@ -4321,7 +4312,7 @@ static void s3jni_updatepre_hook_impl(void * pState, sqlite3 *pDb, int opId, jTable = jDbName ? s3jni_utf8_to_jstring( zTable, -1) : 0; S3JniIfThrew { S3JniExceptionClear; - s3jni_db_error(ps->pDb, SQLITE_NOMEM, 0); + s3jni_db_error(env, ps->pDb, SQLITE_NOMEM, 0); }else{ assert( hook.jObj ); assert( hook.midCallback ); @@ -4423,7 +4414,7 @@ static jobject s3jni_updatepre_hook(JNIEnv * env, int isPre, jlong jpDb, jobject S3JniUnrefLocal(klazz); S3JniIfThrew { S3JniExceptionClear; - s3jni_db_error(ps->pDb, SQLITE_ERROR, + s3jni_db_error(env, ps->pDb, SQLITE_ERROR, "Cannot not find matching callback on " "(pre)update hook object."); }else{ @@ -4532,7 +4523,7 @@ S3JniApi(sqlite3_progress_handler(),void,1progress_1handler)( S3JniUnrefLocal(klazz); S3JniIfThrew { S3JniExceptionClear; - s3jni_db_error(ps->pDb, SQLITE_ERROR, + s3jni_db_error(env, ps->pDb, SQLITE_ERROR, "Cannot not find matching xCallback() on " "ProgressHandler object."); }else{ @@ -4906,8 +4897,9 @@ S3JniApi(sqlite3_set_authorizer(),jint,1set_1authorizer)( ")I"); S3JniUnrefLocal(klazz); S3JniIfThrew { - rc = s3jni_db_error(ps->pDb, SQLITE_ERROR, - "Error setting up Java parts of authorizer hook."); + rc = s3jni_db_error(env, ps->pDb, SQLITE_ERROR, + "Error setting up Java parts of " + "authorizer hook."); }else{ rc = sqlite3_set_authorizer(ps->pDb, s3jni_xAuth, ps); } @@ -5191,7 +5183,7 @@ S3JniApi(sqlite3_trace_v2(),jint,1trace_1v2)( S3JniUnrefLocal(klazz); S3JniIfThrew { S3JniExceptionClear; - rc = s3jni_db_error(ps->pDb, SQLITE_ERROR, + rc = s3jni_db_error(env, ps->pDb, SQLITE_ERROR, "Cannot not find matching call() on " "TracerCallback object."); }else{ diff --git a/ext/jni/src/c/sqlite3-jni.h b/ext/jni/src/c/sqlite3-jni.h index 81af5cbde..c326fa8ea 100644 --- a/ext/jni/src/c/sqlite3-jni.h +++ b/ext/jni/src/c/sqlite3-jni.h @@ -245,8 +245,10 @@ extern "C" { #define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_USED_SHARED 11L #undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_SPILL #define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_CACHE_SPILL 12L +#undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_TEMPBUF_SPILL +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_TEMPBUF_SPILL 13L #undef org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_MAX -#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_MAX 12L +#define org_sqlite_jni_capi_CApi_SQLITE_DBSTATUS_MAX 13L #undef org_sqlite_jni_capi_CApi_SQLITE_UTF8 #define org_sqlite_jni_capi_CApi_SQLITE_UTF8 1L #undef org_sqlite_jni_capi_CApi_SQLITE_UTF16LE diff --git a/ext/jni/src/org/sqlite/jni/capi/CApi.java b/ext/jni/src/org/sqlite/jni/capi/CApi.java index 0b840c362..1bdc5300d 100644 --- a/ext/jni/src/org/sqlite/jni/capi/CApi.java +++ b/ext/jni/src/org/sqlite/jni/capi/CApi.java @@ -115,8 +115,9 @@ private static byte[] nulTerminateUtf8(String s){ JNIEnv is not cached, else returns true, but this information is primarily for testing of the JNI bindings and is not information which client-level code can use to make any informed - decisions. Its return type and semantics are not considered - stable and may change at any time. + decisions. The semantics of its return type and value are not + considered stable and may change at any time. i.e. act as if it + returns null. */ public static native boolean sqlite3_java_uncache_thread(); @@ -2585,7 +2586,8 @@ public static int sqlite3_value_type(@NotNull sqlite3_value v){ public static final int SQLITE_DBSTATUS_DEFERRED_FKS = 10; public static final int SQLITE_DBSTATUS_CACHE_USED_SHARED = 11; public static final int SQLITE_DBSTATUS_CACHE_SPILL = 12; - public static final int SQLITE_DBSTATUS_MAX = 12; + public static final int SQLITE_DBSTATUS_TEMPBUF_SPILL = 13; + public static final int SQLITE_DBSTATUS_MAX = 13; // encodings public static final int SQLITE_UTF8 = 1; diff --git a/ext/jni/src/org/sqlite/jni/capi/Tester1.java b/ext/jni/src/org/sqlite/jni/capi/Tester1.java index 9d14c954b..891bdea54 100644 --- a/ext/jni/src/org/sqlite/jni/capi/Tester1.java +++ b/ext/jni/src/org/sqlite/jni/capi/Tester1.java @@ -815,7 +815,9 @@ public void xDestroy(){ }; // Register and use the function... - int rc = sqlite3_create_function(db, "myfunc", -1, SQLITE_UTF8, func); + int rc = sqlite3_create_function(db, "myfunc", -1, + SQLITE_UTF8 | SQLITE_INNOCUOUS, + func); affirm(0 == rc); affirm(0 == xFuncAccum.value); final sqlite3_stmt stmt = prepare(db, "SELECT myfunc(1,2,3)"); diff --git a/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java b/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java index d259e0ce6..ba2ffd119 100644 --- a/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java +++ b/ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java @@ -171,6 +171,7 @@ public final class Sqlite implements AutoCloseable { public static final int DBSTATUS_DEFERRED_FKS = CApi.SQLITE_DBSTATUS_DEFERRED_FKS; public static final int DBSTATUS_CACHE_USED_SHARED = CApi.SQLITE_DBSTATUS_CACHE_USED_SHARED; public static final int DBSTATUS_CACHE_SPILL = CApi.SQLITE_DBSTATUS_CACHE_SPILL; + public static final int DBSTATUS_TEMPBUF_SPILL = CApi.SQLITE_DBSTATUS_TEMPBUF_SPILL; // Limits public static final int LIMIT_LENGTH = CApi.SQLITE_LIMIT_LENGTH; diff --git a/ext/misc/amatch.c b/ext/misc/amatch.c index 587c610b9..21504777f 100644 --- a/ext/misc/amatch.c +++ b/ext/misc/amatch.c @@ -847,7 +847,7 @@ static int amatchConnect( (void)pAux; *ppVtab = 0; - pNew = sqlite3_malloc( sizeof(*pNew) ); + pNew = sqlite3_malloc64( sizeof(*pNew) ); if( pNew==0 ) return SQLITE_NOMEM; rc = SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); @@ -928,7 +928,7 @@ static int amatchConnect( static int amatchOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ amatch_vtab *p = (amatch_vtab*)pVTab; amatch_cursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); pCur->pVtab = p; diff --git a/ext/misc/base64.c b/ext/misc/base64.c index 2da767bb0..3334222f7 100644 --- a/ext/misc/base64.c +++ b/ext/misc/base64.c @@ -232,7 +232,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_text(context,"",-1,SQLITE_STATIC); break; } - cBuf = sqlite3_malloc(nc); + cBuf = sqlite3_malloc64(nc); if( !cBuf ) goto memFail; nc = (int)(toBase64(bBuf, nb, cBuf) - cBuf); sqlite3_result_text(context, cBuf, nc, sqlite3_free); @@ -254,7 +254,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_zeroblob(context, 0); break; } - bBuf = sqlite3_malloc(nb); + bBuf = sqlite3_malloc64(nb); if( !bBuf ) goto memFail; nb = (int)(fromBase64(cBuf, nc, bBuf) - bBuf); sqlite3_result_blob(context, bBuf, nb, sqlite3_free); @@ -275,7 +275,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ #ifdef _WIN32 __declspec(dllexport) #endif -int sqlite3_base_init +int sqlite3_base64_init #else static int sqlite3_base64_init #endif diff --git a/ext/misc/base85.c b/ext/misc/base85.c index 63245e2e4..a2e6c3ab4 100644 --- a/ext/misc/base85.c +++ b/ext/misc/base85.c @@ -262,7 +262,7 @@ static int allBase85( char *p, int len ){ #ifndef BASE85_STANDALONE -# ifndef OMIT_BASE85_CHECKER +#ifndef OMIT_BASE85_CHECKER /* This function does the work for the SQLite is_base85(t) UDF. */ static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ assert(na==1); @@ -282,7 +282,7 @@ static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ return; } } -# endif +#endif /* This function does the work for the SQLite base85(x) UDF. */ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ @@ -309,7 +309,7 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_text(context,"",-1,SQLITE_STATIC); break; } - cBuf = sqlite3_malloc(nc); + cBuf = sqlite3_malloc64(nc); if( !cBuf ) goto memFail; nc = (int)(toBase85(bBuf, nb, cBuf, "\n") - cBuf); sqlite3_result_text(context, cBuf, nc, sqlite3_free); @@ -331,7 +331,7 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_zeroblob(context, 0); break; } - bBuf = sqlite3_malloc(nb); + bBuf = sqlite3_malloc64(nb); if( !bBuf ) goto memFail; nb = (int)(fromBase85(cBuf, nc, bBuf) - bBuf); sqlite3_result_blob(context, bBuf, nb, sqlite3_free); @@ -352,14 +352,14 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ #ifdef _WIN32 __declspec(dllexport) #endif -int sqlite3_base_init +int sqlite3_base85_init #else static int sqlite3_base85_init #endif (sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ SQLITE_EXTENSION_INIT2(pApi); (void)pzErr; -# ifndef OMIT_BASE85_CHECKER +#ifndef OMIT_BASE85_CHECKER { int rc = sqlite3_create_function (db, "is_base85", 1, @@ -367,7 +367,7 @@ static int sqlite3_base85_init 0, is_base85, 0, 0); if( rc!=SQLITE_OK ) return rc; } -# endif +#endif return sqlite3_create_function (db, "base85", 1, SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, @@ -432,9 +432,9 @@ int main(int na, char *av[]){ int nc = strlen(cBuf); size_t nbo = fromBase85( cBuf, nc, bBuf ) - bBuf; if( 1 != fwrite(bBuf, nbo, 1, fb) ) rc = 1; -# ifndef OMIT_BASE85_CHECKER +#ifndef OMIT_BASE85_CHECKER b85Clean &= allBase85( cBuf, nc ); -# endif +#endif } break; default: diff --git a/ext/misc/btreeinfo.c b/ext/misc/btreeinfo.c index 9c726f5f1..24645f226 100644 --- a/ext/misc/btreeinfo.c +++ b/ext/misc/btreeinfo.c @@ -306,6 +306,10 @@ static int binfoCompute(sqlite3 *db, int pgno, BinfoCursor *pCsr){ nEntry *= (nCell+1); if( aData[0]==10 || aData[0]==13 ) break; nPage *= (nCell+1); + if( 14+2*(nCell/2)>=pgsz ){ + rc = SQLITE_CORRUPT; + break; + } if( nCell<=1 ){ pgno = get_uint32(aData+8); }else{ @@ -339,7 +343,7 @@ static int binfoColumn( sqlite3 *db = sqlite3_context_db_handle(ctx); int rc = binfoCompute(db, pgno, pCsr); if( rc ){ - pCursor->pVtab->zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + pCursor->pVtab->zErrMsg = sqlite3_mprintf("%s", sqlite3_errstr(rc)); return SQLITE_ERROR; } } diff --git a/ext/misc/closure.c b/ext/misc/closure.c index 14caf271f..22bfd888f 100644 --- a/ext/misc/closure.c +++ b/ext/misc/closure.c @@ -516,7 +516,7 @@ static int closureConnect( (void)pAux; *ppVtab = 0; - pNew = sqlite3_malloc( sizeof(*pNew) ); + pNew = sqlite3_malloc64( sizeof(*pNew) ); if( pNew==0 ) return SQLITE_NOMEM; rc = SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); @@ -579,7 +579,7 @@ static int closureConnect( static int closureOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ closure_vtab *p = (closure_vtab*)pVTab; closure_cursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); pCur->pVtab = p; @@ -638,7 +638,7 @@ static int closureInsertNode( sqlite3_int64 id, /* The node ID */ int iGeneration /* The generation number for this node */ ){ - closure_avl *pNew = sqlite3_malloc( sizeof(*pNew) ); + closure_avl *pNew = sqlite3_malloc64( sizeof(*pNew) ); if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); pNew->id = id; diff --git a/ext/misc/completion.c b/ext/misc/completion.c index 67b40d84d..37237d9c9 100644 --- a/ext/misc/completion.c +++ b/ext/misc/completion.c @@ -132,7 +132,7 @@ static int completionConnect( " phase INT HIDDEN" /* Used for debugging only */ ")"); if( rc==SQLITE_OK ){ - pNew = sqlite3_malloc( sizeof(*pNew) ); + pNew = sqlite3_malloc64( sizeof(*pNew) ); *ppVtab = (sqlite3_vtab*)pNew; if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); @@ -154,7 +154,7 @@ static int completionDisconnect(sqlite3_vtab *pVtab){ */ static int completionOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ completion_cursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); pCur->db = ((completion_vtab*)p)->db; @@ -199,6 +199,7 @@ static int completionNext(sqlite3_vtab_cursor *cur){ completion_cursor *pCur = (completion_cursor*)cur; int eNextPhase = 0; /* Next phase to try if current phase reaches end */ int iCol = -1; /* If >=0, step pCur->pStmt and use the i-th column */ + int rc; pCur->iRowid++; while( pCur->ePhase!=COMPLETION_EOF ){ switch( pCur->ePhase ){ @@ -224,22 +225,27 @@ static int completionNext(sqlite3_vtab_cursor *cur){ case COMPLETION_TABLES: { if( pCur->pStmt==0 ){ sqlite3_stmt *pS2; + sqlite3_str* pStr = sqlite3_str_new(pCur->db); char *zSql = 0; const char *zSep = ""; sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pS2, 0); while( sqlite3_step(pS2)==SQLITE_ROW ){ const char *zDb = (const char*)sqlite3_column_text(pS2, 1); - zSql = sqlite3_mprintf( - "%z%s" + sqlite3_str_appendf(pStr, + "%s" "SELECT name FROM \"%w\".sqlite_schema", - zSql, zSep, zDb + zSep, zDb ); - if( zSql==0 ) return SQLITE_NOMEM; zSep = " UNION "; } - sqlite3_finalize(pS2); - sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + rc = sqlite3_finalize(pS2); + zSql = sqlite3_str_finish(pStr); + if( zSql==0 ) return SQLITE_NOMEM; + if( rc==SQLITE_OK ){ + sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + } sqlite3_free(zSql); + if( rc ) return rc; } iCol = 0; eNextPhase = COMPLETION_COLUMNS; @@ -248,24 +254,29 @@ static int completionNext(sqlite3_vtab_cursor *cur){ case COMPLETION_COLUMNS: { if( pCur->pStmt==0 ){ sqlite3_stmt *pS2; + sqlite3_str *pStr = sqlite3_str_new(pCur->db); char *zSql = 0; const char *zSep = ""; sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pS2, 0); while( sqlite3_step(pS2)==SQLITE_ROW ){ const char *zDb = (const char*)sqlite3_column_text(pS2, 1); - zSql = sqlite3_mprintf( - "%z%s" + sqlite3_str_appendf(pStr, + "%s" "SELECT pti.name FROM \"%w\".sqlite_schema AS sm" " JOIN pragma_table_xinfo(sm.name,%Q) AS pti" " WHERE sm.type='table'", - zSql, zSep, zDb, zDb + zSep, zDb, zDb ); - if( zSql==0 ) return SQLITE_NOMEM; zSep = " UNION "; } - sqlite3_finalize(pS2); - sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + rc = sqlite3_finalize(pS2); + zSql = sqlite3_str_finish(pStr); + if( zSql==0 ) return SQLITE_NOMEM; + if( rc==SQLITE_OK ){ + sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + } sqlite3_free(zSql); + if( rc ) return rc; } iCol = 0; eNextPhase = COMPLETION_EOF; @@ -282,9 +293,10 @@ static int completionNext(sqlite3_vtab_cursor *cur){ pCur->szRow = sqlite3_column_bytes(pCur->pStmt, iCol); }else{ /* When all rows are finished, advance to the next phase */ - sqlite3_finalize(pCur->pStmt); + rc = sqlite3_finalize(pCur->pStmt); pCur->pStmt = 0; pCur->ePhase = eNextPhase; + if( rc ) return rc; continue; } } diff --git a/ext/misc/csv.c b/ext/misc/csv.c index 1caaaec87..eaf9cbba7 100644 --- a/ext/misc/csv.c +++ b/ext/misc/csv.c @@ -24,8 +24,8 @@ ** schema= parameter, like this: ** ** CREATE VIRTUAL TABLE temp.csv2 USING csv( -** filename = "../http.log", -** schema = "CREATE TABLE x(date,ipaddr,url,referrer,userAgent)" +** filename = '../http.log', +** schema = 'CREATE TABLE x(date,ipaddr,url,referrer,userAgent)' ** ); ** ** Instead of specifying a file, the text of the CSV can be loaded using @@ -64,6 +64,7 @@ SQLITE_EXTENSION_INIT1 #ifndef SQLITEINT_H typedef sqlite3_int64 i64; +typedef sqlite3_uint64 u64; #endif /* Max size of the error message in a CsvReader */ @@ -128,7 +129,7 @@ static int csv_reader_open( const char *zData /* ... or use this data */ ){ if( zFilename ){ - p->zIn = sqlite3_malloc( CSV_INBUFSZ ); + p->zIn = sqlite3_malloc64( CSV_INBUFSZ ); if( p->zIn==0 ){ csv_errmsg(p, "out of memory"); return 1; @@ -221,7 +222,7 @@ static char *csv_read_one_field(CsvReader *p){ } if( c=='"' ){ int pc, ppc; - int startLine = p->nLine; + i64 startLine = p->nLine; pc = ppc = 0; while( 1 ){ c = csv_getc(p); @@ -325,7 +326,7 @@ typedef struct CsvCursor { sqlite3_vtab_cursor base; /* Base class. Must be first */ CsvReader rdr; /* The CsvReader object */ char **azVal; /* Value of the current row */ - int *aLen; /* Length of each entry */ + i64 *aLen; /* Length of each entry */ sqlite3_int64 iRowid; /* The current rowid. Negative for EOF */ } CsvCursor; @@ -497,7 +498,7 @@ static int csvtabConnect( CsvTable *pNew = 0; /* The CsvTable object to construct */ int bHeader = -1; /* header= flags. -1 means not seen yet */ int rc = SQLITE_OK; /* Result code from this routine */ - int i, j; /* Loop counters */ + u64 i, j; /* Loop counters */ #ifdef SQLITE_TEST int tstFlags = 0; /* Value for testflags=N parameter */ #endif @@ -516,7 +517,7 @@ static int csvtabConnect( assert( sizeof(azPValue)==sizeof(azParam) ); memset(&sRdr, 0, sizeof(sRdr)); memset(azPValue, 0, sizeof(azPValue)); - for(i=3; i<argc; i++){ + for(i=3; i<(u64)argc; i++){ const char *z = argv[i]; const char *zValue; for(j=0; j<sizeof(azParam)/sizeof(azParam[0]); j++){ @@ -563,7 +564,7 @@ static int csvtabConnect( ){ goto csvtab_connect_error; } - pNew = sqlite3_malloc( sizeof(*pNew) ); + pNew = sqlite3_malloc64( sizeof(*pNew) ); *ppVtab = (sqlite3_vtab*)pNew; if( pNew==0 ) goto csvtab_connect_oom; memset(pNew, 0, sizeof(*pNew)); @@ -709,12 +710,12 @@ static int csvtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ CsvTable *pTab = (CsvTable*)p; CsvCursor *pCur; size_t nByte; - nByte = sizeof(*pCur) + (sizeof(char*)+sizeof(int))*pTab->nCol; + nByte = sizeof(*pCur) + (sizeof(char*)+sizeof(i64))*pTab->nCol; pCur = sqlite3_malloc64( nByte ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, nByte); pCur->azVal = (char**)&pCur[1]; - pCur->aLen = (int*)&pCur->azVal[pTab->nCol]; + pCur->aLen = (i64*)&pCur->azVal[pTab->nCol]; *ppCursor = &pCur->base; if( csv_reader_open(&pCur->rdr, pTab->zFilename, pTab->zData) ){ csv_xfer_error(pTab, &pCur->rdr); diff --git a/ext/misc/decimal.c b/ext/misc/decimal.c index f87699f96..66d4e3042 100644 --- a/ext/misc/decimal.c +++ b/ext/misc/decimal.c @@ -31,6 +31,10 @@ SQLITE_EXTENSION_INIT1 #define IsSpace(X) isspace((unsigned char)X) #endif +#ifndef SQLITE_DECIMAL_MAX_DIGIT +# define SQLITE_DECIMAL_MAX_DIGIT 10000000 +#endif + /* A decimal object */ typedef struct Decimal Decimal; struct Decimal { @@ -69,7 +73,8 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ int i; int iExp = 0; - p = sqlite3_malloc( sizeof(*p) ); + if( zIn==0 ) goto new_from_text_failed; + p = sqlite3_malloc64( sizeof(*p) ); if( p==0 ) goto new_from_text_failed; p->sign = 0; p->oom = 0; @@ -128,9 +133,10 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ } } if( iExp>0 ){ - p->a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + signed char *a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + (sqlite3_int64)iExp + 1 ); - if( p->a==0 ) goto new_from_text_failed; + if( a==0 ) goto new_from_text_failed; + p->a = a; memset(p->a+p->nDigit, 0, iExp); p->nDigit += iExp; } @@ -148,9 +154,10 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ } } if( iExp>0 ){ - p->a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + signed char *a = sqlite3_realloc64(p->a, (sqlite3_int64)p->nDigit + (sqlite3_int64)iExp + 1 ); - if( p->a==0 ) goto new_from_text_failed; + if( a==0 ) goto new_from_text_failed; + p->a = a; memmove(p->a+iExp, p->a, p->nDigit); memset(p->a, 0, iExp); p->nDigit += iExp; @@ -161,6 +168,7 @@ static Decimal *decimalNewFromText(const char *zIn, int n){ for(i=0; i<p->nDigit && p->a[i]==0; i++){} if( i>=p->nDigit ) p->sign = 0; } + if( p->nDigit>SQLITE_DECIMAL_MAX_DIGIT ) goto new_from_text_failed; return p; new_from_text_failed: @@ -291,12 +299,38 @@ static void decimal_result(sqlite3_context *pCtx, Decimal *p){ sqlite3_result_text(pCtx, z, i, sqlite3_free); } +/* +** Round a decimal value to N significant digits. N must be positive. +*/ +static void decimal_round(Decimal *p, int N){ + int i; + int nZero; + if( N<1 ) return; + if( p==0 ) return; + if( p->nDigit<=N ) return; + for(nZero=0; nZero<p->nDigit && p->a[nZero]==0; nZero++){} + N += nZero; + if( p->nDigit<=N ) return; + if( p->a[N]>4 ){ + p->a[N-1]++; + for(i=N-1; i>0 && p->a[i]>9; i--){ + p->a[i] = 0; + p->a[i-1]++; + } + if( p->a[0]>9 ){ + p->a[0] = 1; + p->nFrac--; + } + } + memset(&p->a[N], 0, p->nDigit - N); +} + /* ** Make the given Decimal the result in an format similar to '%+#e'. ** In other words, show exponential notation with leading and trailing ** zeros omitted. */ -static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ +static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p, int N){ char *z; /* The output buffer */ int i; /* Loop counter */ int nZero; /* Number of leading zeros */ @@ -314,7 +348,8 @@ static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ sqlite3_result_null(pCtx); return; } - for(nDigit=p->nDigit; nDigit>0 && p->a[nDigit-1]==0; nDigit--){} + if( N<1 ) N = 0; + for(nDigit=p->nDigit; nDigit>N && p->a[nDigit-1]==0; nDigit--){} for(nZero=0; nZero<nDigit && p->a[nZero]==0; nZero++){} nFrac = p->nFrac + (nDigit - p->nDigit); nDigit -= nZero; @@ -430,15 +465,18 @@ static void decimalCmpFunc( static void decimal_expand(Decimal *p, int nDigit, int nFrac){ int nAddSig; int nAddFrac; + signed char *a; if( p==0 ) return; nAddFrac = nFrac - p->nFrac; nAddSig = (nDigit - p->nDigit) - nAddFrac; if( nAddFrac==0 && nAddSig==0 ) return; - p->a = sqlite3_realloc64(p->a, nDigit+1); - if( p->a==0 ){ + if( nDigit+1>SQLITE_DECIMAL_MAX_DIGIT ){ p->oom = 1; return; } + a = sqlite3_realloc64(p->a, nDigit+1); + if( a==0 ){ p->oom = 1; return; } + p->a = a; if( nAddSig ){ memmove(p->a+nAddSig, p->a, p->nDigit); memset(p->a, 0, nAddSig); @@ -533,14 +571,18 @@ static void decimalMul(Decimal *pA, Decimal *pB){ signed char *acc = 0; int i, j, k; int minFrac; + sqlite3_int64 sumDigit; if( pA==0 || pA->oom || pA->isNull || pB==0 || pB->oom || pB->isNull ){ goto mul_end; } - acc = sqlite3_malloc64( (sqlite3_int64)pA->nDigit + - (sqlite3_int64)pB->nDigit + 2 ); + sumDigit = pA->nDigit; + sumDigit += pB->nDigit; + sumDigit += 2; + if( sumDigit>SQLITE_DECIMAL_MAX_DIGIT ){ pA->oom = 1; return; } + acc = sqlite3_malloc64( sumDigit ); if( acc==0 ){ pA->oom = 1; goto mul_end; @@ -677,10 +719,16 @@ static void decimalFunc( sqlite3_value **argv ){ Decimal *p = decimal_new(context, argv[0], 0); - UNUSED_PARAMETER(argc); + int N; + if( argc==2 ){ + N = sqlite3_value_int(argv[1]); + if( N>0 ) decimal_round(p, N); + }else{ + N = 0; + } if( p ){ if( sqlite3_user_data(context)!=0 ){ - decimal_result_sci(context, p); + decimal_result_sci(context, p, N); }else{ decimal_result(context, p); } @@ -766,7 +814,7 @@ static void decimalSumStep( if( p==0 ) return; if( !p->isInit ){ p->isInit = 1; - p->a = sqlite3_malloc(2); + p->a = sqlite3_malloc64(2); if( p->a==0 ){ p->oom = 1; }else{ @@ -850,7 +898,7 @@ static void decimalPow2Func( UNUSED_PARAMETER(argc); if( sqlite3_value_type(argv[0])==SQLITE_INTEGER ){ Decimal *pA = decimalPow2(sqlite3_value_int(argv[0])); - decimal_result_sci(context, pA); + decimal_result_sci(context, pA, 0); decimal_free(pA); } } @@ -871,7 +919,9 @@ int sqlite3_decimal_init( void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } aFunc[] = { { "decimal", 1, 0, decimalFunc }, + { "decimal", 2, 0, decimalFunc }, { "decimal_exp", 1, 1, decimalFunc }, + { "decimal_exp", 2, 1, decimalFunc }, { "decimal_cmp", 2, 0, decimalCmpFunc }, { "decimal_add", 2, 0, decimalAddFunc }, { "decimal_sub", 2, 0, decimalSubFunc }, diff --git a/ext/misc/explain.c b/ext/misc/explain.c index 726af76b9..132041882 100644 --- a/ext/misc/explain.c +++ b/ext/misc/explain.c @@ -92,7 +92,7 @@ static int explainConnect( rc = sqlite3_declare_vtab(db, "CREATE TABLE x(addr,opcode,p1,p2,p3,p4,p5,comment,sql HIDDEN)"); if( rc==SQLITE_OK ){ - pNew = sqlite3_malloc( sizeof(*pNew) ); + pNew = sqlite3_malloc64( sizeof(*pNew) ); *ppVtab = (sqlite3_vtab*)pNew; if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); @@ -114,7 +114,7 @@ static int explainDisconnect(sqlite3_vtab *pVtab){ */ static int explainOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ explain_cursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); pCur->db = ((explain_vtab*)p)->db; diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c index 6cc2ae008..91da383e7 100644 --- a/ext/misc/fileio.c +++ b/ext/misc/fileio.c @@ -73,11 +73,6 @@ ** $path is a relative path, then $path is interpreted relative to $dir. ** And the paths returned in the "name" column of the table are also ** relative to directory $dir. -** -** Notes on building this extension for Windows: -** Unless linked statically with the SQLite library, a preprocessor -** symbol, FILEIO_WIN32_DLL, must be #define'd to create a stand-alone -** DLL form of this extension for WIN32. See its use below for details. */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -94,12 +89,16 @@ SQLITE_EXTENSION_INIT1 # include <utime.h> # include <sys/time.h> # define STRUCT_STAT struct stat +# include <limits.h> +# include <stdlib.h> #else # include "windirent.h" # include <direct.h> # define STRUCT_STAT struct _stat # define chmod(path,mode) fileio_chmod(path,mode) # define mkdir(path,mode) fileio_mkdir(path) + extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*); + extern char *sqlite3_win32_unicode_to_utf8(LPCWSTR); #endif #include <time.h> #include <errno.h> @@ -131,12 +130,9 @@ SQLITE_EXTENSION_INIT1 */ #if defined(_WIN32) || defined(WIN32) static int fileio_chmod(const char *zPath, int pmode){ - sqlite3_int64 sz = strlen(zPath); - wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(b1[0]) ); int rc; + wchar_t *b1 = sqlite3_win32_utf8_to_unicode(zPath); if( b1==0 ) return -1; - sz = MultiByteToWideChar(CP_UTF8, 0, zPath, sz, b1, sz); - b1[sz] = 0; rc = _wchmod(b1, pmode); sqlite3_free(b1); return rc; @@ -148,12 +144,9 @@ static int fileio_chmod(const char *zPath, int pmode){ */ #if defined(_WIN32) || defined(WIN32) static int fileio_mkdir(const char *zPath){ - sqlite3_int64 sz = strlen(zPath); - wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(b1[0]) ); int rc; + wchar_t *b1 = sqlite3_win32_utf8_to_unicode(zPath); if( b1==0 ) return -1; - sz = MultiByteToWideChar(CP_UTF8, 0, zPath, sz, b1, sz); - b1[sz] = 0; rc = _wmkdir(b1); sqlite3_free(b1); return rc; @@ -266,50 +259,7 @@ static sqlite3_uint64 fileTimeToUnixTime( return (fileIntervals.QuadPart - epochIntervals.QuadPart) / 10000000; } - - -#if defined(FILEIO_WIN32_DLL) && (defined(_WIN32) || defined(WIN32)) -# /* To allow a standalone DLL, use this next replacement function: */ -# undef sqlite3_win32_utf8_to_unicode -# define sqlite3_win32_utf8_to_unicode utf8_to_utf16 -# -LPWSTR utf8_to_utf16(const char *z){ - int nAllot = MultiByteToWideChar(CP_UTF8, 0, z, -1, NULL, 0); - LPWSTR rv = sqlite3_malloc(nAllot * sizeof(WCHAR)); - if( rv!=0 && 0 < MultiByteToWideChar(CP_UTF8, 0, z, -1, rv, nAllot) ) - return rv; - sqlite3_free(rv); - return 0; -} -#endif - -/* -** This function attempts to normalize the time values found in the stat() -** buffer to UTC. This is necessary on Win32, where the runtime library -** appears to return these values as local times. -*/ -static void statTimesToUtc( - const char *zPath, - STRUCT_STAT *pStatBuf -){ - HANDLE hFindFile; - WIN32_FIND_DATAW fd; - LPWSTR zUnicodeName; - extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*); - zUnicodeName = sqlite3_win32_utf8_to_unicode(zPath); - if( zUnicodeName ){ - memset(&fd, 0, sizeof(WIN32_FIND_DATAW)); - hFindFile = FindFirstFileW(zUnicodeName, &fd); - if( hFindFile!=NULL ){ - pStatBuf->st_ctime = (time_t)fileTimeToUnixTime(&fd.ftCreationTime); - pStatBuf->st_atime = (time_t)fileTimeToUnixTime(&fd.ftLastAccessTime); - pStatBuf->st_mtime = (time_t)fileTimeToUnixTime(&fd.ftLastWriteTime); - FindClose(hFindFile); - } - sqlite3_free(zUnicodeName); - } -} -#endif +#endif /* _WIN32 */ /* ** This function is used in place of stat(). On Windows, special handling @@ -321,14 +271,22 @@ static int fileStat( STRUCT_STAT *pStatBuf ){ #if defined(_WIN32) - sqlite3_int64 sz = strlen(zPath); - wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(b1[0]) ); int rc; + wchar_t *b1 = sqlite3_win32_utf8_to_unicode(zPath); if( b1==0 ) return 1; - sz = MultiByteToWideChar(CP_UTF8, 0, zPath, sz, b1, sz); - b1[sz] = 0; rc = _wstat(b1, pStatBuf); - if( rc==0 ) statTimesToUtc(zPath, pStatBuf); + if( rc==0 ){ + HANDLE hFindFile; + WIN32_FIND_DATAW fd; + memset(&fd, 0, sizeof(WIN32_FIND_DATAW)); + hFindFile = FindFirstFileW(b1, &fd); + if( hFindFile!=NULL ){ + pStatBuf->st_ctime = (time_t)fileTimeToUnixTime(&fd.ftCreationTime); + pStatBuf->st_atime = (time_t)fileTimeToUnixTime(&fd.ftLastAccessTime); + pStatBuf->st_mtime = (time_t)fileTimeToUnixTime(&fd.ftLastWriteTime); + FindClose(hFindFile); + } + } sqlite3_free(b1); return rc; #else @@ -460,7 +418,6 @@ static int writeFile( if( mtime>=0 ){ #if defined(_WIN32) -#if !SQLITE_OS_WINRT /* Windows */ FILETIME lastAccess; FILETIME lastWrite; @@ -491,7 +448,6 @@ static int writeFile( }else{ return 1; } -#endif #elif defined(AT_FDCWD) && 0 /* utimensat() is not universally available */ /* Recent unix */ struct timespec times[2]; @@ -657,7 +613,7 @@ static int fsdirConnect( (void)pzErr; rc = sqlite3_declare_vtab(db, "CREATE TABLE x" FSDIR_SCHEMA); if( rc==SQLITE_OK ){ - pNew = (fsdir_tab*)sqlite3_malloc( sizeof(*pNew) ); + pNew = (fsdir_tab*)sqlite3_malloc64( sizeof(*pNew) ); if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); @@ -680,7 +636,7 @@ static int fsdirDisconnect(sqlite3_vtab *pVtab){ static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ fsdir_cursor *pCur; (void)p; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); pCur->iLvl = -1; @@ -1095,6 +1051,154 @@ static int fsdirRegister(sqlite3 *db){ # define fsdirRegister(x) SQLITE_OK #endif +/* +** This version of realpath() works on any system. The string +** returned is held in memory allocated using sqlite3_malloc64(). +** The caller is responsible for calling sqlite3_free(). +*/ +static char *portable_realpath(const char *zPath){ +#if !defined(_WIN32) /* BEGIN unix */ + + char *zOut = 0; /* Result */ + char *z; /* Temporary buffer */ +#if defined(PATH_MAX) + char zBuf[PATH_MAX+1]; /* Space for the temporary buffer */ +#endif + + if( zPath==0 ) return 0; +#if defined(PATH_MAX) + z = realpath(zPath, zBuf); + if( z ){ + zOut = sqlite3_mprintf("%s", zBuf); + } +#endif /* defined(PATH_MAX) */ + if( zOut==0 ){ + /* Try POSIX.1-2008 malloc behavior */ + z = realpath(zPath, NULL); + if( z ){ + zOut = sqlite3_mprintf("%s", z); + free(z); + } + } + return zOut; + +#else /* End UNIX, Begin WINDOWS */ + + wchar_t *zPath16; /* UTF16 translation of zPath */ + char *zOut = 0; /* Result */ + wchar_t *z = 0; /* Temporary buffer */ + + if( zPath==0 ) return 0; + + zPath16 = sqlite3_win32_utf8_to_unicode(zPath); + if( zPath16==0 ) return 0; + z = _wfullpath(NULL, zPath16, 0); + sqlite3_free(zPath16); + if( z ){ + zOut = sqlite3_win32_unicode_to_utf8(z); + free(z); + } + return zOut; + +#endif /* End WINDOWS, Begin common code */ +} + +/* +** SQL function: realpath(X) +** +** Try to convert file or pathname X into its real, absolute pathname. +** Return NULL if unable. +** +** The file or directory X is not required to exist. The answer is formed +** by calling system realpath() on the prefix of X that does exist and +** appending the tail of X that does not (yet) exist. +*/ +static void realpathFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zPath; /* Original input path */ + char *zCopy; /* An editable copy of zPath */ + char *zOut; /* The result */ + char cSep = 0; /* Separator turned into \000 */ + size_t len; /* Prefix length before cSep */ +#ifdef _WIN32 + const int isWin = 1; +#else + const int isWin = 0; +#endif + + (void)argc; + zPath = (const char*)sqlite3_value_text(argv[0]); + if( zPath==0 ) return; + if( zPath[0]==0 ) zPath = "."; + zCopy = sqlite3_mprintf("%s",zPath); + len = strlen(zCopy); + while( len>1 && (zCopy[len-1]=='/' || (isWin && zCopy[len-1]=='\\')) ){ + len--; + } + zCopy[len] = 0; + while( 1 /*exit-by-break*/ ){ + zOut = portable_realpath(zCopy); + zCopy[len] = cSep; + if( zOut ){ + if( cSep ){ + zOut = sqlite3_mprintf("%z%s",zOut,&zCopy[len]); + } + break; + }else{ + size_t i = len-1; + while( i>0 ){ + if( zCopy[i]=='/' || (isWin && zCopy[i]=='\\') ) break; + i--; + } + if( i<=0 ){ + if( zCopy[0]=='/' ){ + zOut = zCopy; + zCopy = 0; + }else if( (zOut = portable_realpath("."))!=0 ){ + zOut = sqlite3_mprintf("%z/%s", zOut, zCopy); + } + break; + } + cSep = zCopy[i]; + zCopy[i] = 0; + len = i; + } + } + sqlite3_free(zCopy); + if( zOut ){ + /* Simplify any "/./" or "/../" that might have snuck into the + ** pathname due to appending of zCopy. We only have to consider + ** unix "/" separators, because the _wfilepath() system call on + ** Windows will have already done this simplification for us. */ + size_t i, j, n; + n = strlen(zOut); + for(i=j=0; i<n; i++){ + if( zOut[i]=='/' ){ + if( zOut[i+1]=='/' ) continue; + if( zOut[i+1]=='.' && i+2<n && zOut[i+2]=='/' ){ + i += 1; + continue; + } + if( zOut[i+1]=='.' && i+3<n && zOut[i+2]=='.' && zOut[i+3]=='/' ){ + while( j>0 && zOut[j-1]!='/' ){ j--; } + if( j>0 ){ j--; } + i += 2; + continue; + } + } + zOut[j++] = zOut[i]; + } + zOut[j] = 0; + + /* Return the result */ + sqlite3_result_text(context, zOut, -1, sqlite3_free); + } +} + + #ifdef _WIN32 __declspec(dllexport) #endif @@ -1121,13 +1225,10 @@ int sqlite3_fileio_init( if( rc==SQLITE_OK ){ rc = fsdirRegister(db); } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "realpath", 1, + SQLITE_UTF8, 0, + realpathFunc, 0, 0); + } return rc; } - -#if defined(FILEIO_WIN32_DLL) && (defined(_WIN32) || defined(WIN32)) -/* To allow a standalone DLL, make test_windirent.c use the same - * redefined SQLite API calls as the above extension code does. - * Just pull in this .c to accomplish this. As a beneficial side - * effect, this extension becomes a single translation unit. */ -# include "test_windirent.c" -#endif diff --git a/ext/misc/fossildelta.c b/ext/misc/fossildelta.c index 2dc29b3c3..e2de0ec40 100644 --- a/ext/misc/fossildelta.c +++ b/ext/misc/fossildelta.c @@ -843,7 +843,7 @@ static int deltaparsevtabDisconnect(sqlite3_vtab *pVtab){ */ static int deltaparsevtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ deltaparsevtab_cursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); *ppCursor = &pCur->base; diff --git a/ext/misc/fuzzer.c b/ext/misc/fuzzer.c index e16d005d9..12785e3a4 100644 --- a/ext/misc/fuzzer.c +++ b/ext/misc/fuzzer.c @@ -556,7 +556,7 @@ static int fuzzerConnect( static int fuzzerOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ fuzzer_vtab *p = (fuzzer_vtab*)pVTab; fuzzer_cursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); pCur->pVtab = p; @@ -617,12 +617,12 @@ static int fuzzerRender( int *pnBuf /* Size of the buffer */ ){ const fuzzer_rule *pRule = pStem->pRule; - int n; /* Size of output term without nul-term */ + sqlite3_int64 n; /* Size of output term without nul-term */ char *z; /* Buffer to assemble output term in */ n = pStem->nBasis + pRule->nTo - pRule->nFrom; if( (*pnBuf)<n+1 ){ - (*pzBuf) = sqlite3_realloc((*pzBuf), n+100); + (*pzBuf) = sqlite3_realloc64((*pzBuf), n+100); if( (*pzBuf)==0 ) return SQLITE_NOMEM; (*pnBuf) = n+100; } diff --git a/ext/misc/ieee754.c b/ext/misc/ieee754.c index 7f1c6d9e1..f551b2265 100644 --- a/ext/misc/ieee754.c +++ b/ext/misc/ieee754.c @@ -179,9 +179,9 @@ static void ieee754func( } if( m<0 ){ + if( m<(-9223372036854775807LL) ) return; isNeg = 1; m = -m; - if( m<0 ) return; }else if( m==0 && e>-1000 && e<1000 ){ sqlite3_result_double(context, 0.0); return; @@ -259,6 +259,38 @@ static void ieee754func_to_blob( } } +/* +** Functions to convert between 64-bit integers and floats. +** +** The bit patterns are copied. The numeric values are different. +*/ +static void ieee754func_from_int( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_INTEGER ){ + double r; + sqlite3_int64 v = sqlite3_value_int64(argv[0]); + memcpy(&r, &v, sizeof(r)); + sqlite3_result_double(context, r); + } +} +static void ieee754func_to_int( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_FLOAT ){ + double r = sqlite3_value_double(argv[0]); + sqlite3_uint64 v; + memcpy(&v, &r, sizeof(v)); + sqlite3_result_int64(context, v); + } +} + /* ** SQL Function: ieee754_inc(r,N) ** @@ -311,6 +343,8 @@ int sqlite3_ieee_init( { "ieee754_exponent", 1, 2, ieee754func }, { "ieee754_to_blob", 1, 0, ieee754func_to_blob }, { "ieee754_from_blob", 1, 0, ieee754func_from_blob }, + { "ieee754_to_int", 1, 0, ieee754func_to_int }, + { "ieee754_from_int", 1, 0, ieee754func_from_int }, { "ieee754_inc", 2, 0, ieee754inc }, }; unsigned int i; diff --git a/ext/misc/memstat.c b/ext/misc/memstat.c index 8e69b4695..5002a1359 100644 --- a/ext/misc/memstat.c +++ b/ext/misc/memstat.c @@ -85,7 +85,7 @@ static int memstatConnect( rc = sqlite3_declare_vtab(db,"CREATE TABLE x(name,schema,value,hiwtr)"); if( rc==SQLITE_OK ){ - pNew = sqlite3_malloc( sizeof(*pNew) ); + pNew = sqlite3_malloc64( sizeof(*pNew) ); *ppVtab = (sqlite3_vtab*)pNew; if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); @@ -107,7 +107,7 @@ static int memstatDisconnect(sqlite3_vtab *pVtab){ */ static int memstatOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ memstat_cursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); pCur->db = ((memstat_vtab*)p)->db; diff --git a/ext/misc/prefixes.c b/ext/misc/prefixes.c index e6517e719..3c47933c0 100644 --- a/ext/misc/prefixes.c +++ b/ext/misc/prefixes.c @@ -75,7 +75,7 @@ static int prefixesConnect( "CREATE TABLE prefixes(prefix TEXT, original_string TEXT HIDDEN)" ); if( rc==SQLITE_OK ){ - pNew = sqlite3_malloc( sizeof(*pNew) ); + pNew = sqlite3_malloc64( sizeof(*pNew) ); *ppVtab = (sqlite3_vtab*)pNew; if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); @@ -98,7 +98,7 @@ static int prefixesDisconnect(sqlite3_vtab *pVtab){ */ static int prefixesOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ prefixes_cursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); *ppCursor = &pCur->base; diff --git a/ext/misc/qpvtab.c b/ext/misc/qpvtab.c index b7c2a0512..15071883c 100644 --- a/ext/misc/qpvtab.c +++ b/ext/misc/qpvtab.c @@ -152,7 +152,7 @@ static int qpvtabConnect( #define QPVTAB_FLAGS 11 #define QPVTAB_NONE 12 if( rc==SQLITE_OK ){ - pNew = sqlite3_malloc( sizeof(*pNew) ); + pNew = sqlite3_malloc64( sizeof(*pNew) ); *ppVtab = (sqlite3_vtab*)pNew; if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); @@ -174,7 +174,7 @@ static int qpvtabDisconnect(sqlite3_vtab *pVtab){ */ static int qpvtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ qpvtab_cursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); *ppCursor = &pCur->base; diff --git a/ext/misc/regexp.c b/ext/misc/regexp.c index f1babf4ab..4f40e3f1c 100644 --- a/ext/misc/regexp.c +++ b/ext/misc/regexp.c @@ -32,7 +32,7 @@ ** ^X X occurring at the beginning of the string ** X$ X occurring at the end of the string ** . Match any single character -** \c Character c where c is one of \{}()[]|*+?. +** \c Character c where c is one of \{}()[]|*+?-. ** \c C-language escapes for c in afnrtv. ex: \t or \n ** \uXXXX Where XXXX is exactly 4 hex digits, unicode value XXXX ** \xXX Where XX is exactly 2 hex digits, unicode value XX @@ -417,7 +417,7 @@ static int re_hex(int c, int *pV){ ** return its interpretation. */ static unsigned re_esc_char(ReCompiled *p){ - static const char zEsc[] = "afnrtv\\()*.+?[$^{|}]"; + static const char zEsc[] = "afnrtv\\()*.+?[$^{|}]-"; static const char zTrans[] = "\a\f\n\r\t\v"; int i, v = 0; char c; @@ -676,7 +676,7 @@ static const char *re_compile( int i, j; *ppRe = 0; - pRe = sqlite3_malloc( sizeof(*pRe) ); + pRe = sqlite3_malloc64( sizeof(*pRe) ); if( pRe==0 ){ return "out of memory"; } @@ -740,11 +740,18 @@ static const char *re_compile( } /* -** Compute a reasonable limit on the length of the REGEXP NFA. +** The value of LIMIT_MAX_PATTERN_LENGTH. */ static int re_maxlen(sqlite3_context *context){ sqlite3 *db = sqlite3_context_db_handle(context); - return 75 + sqlite3_limit(db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH,-1)/2; + return sqlite3_limit(db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH,-1); +} + +/* +** Maximum NFA size given a maximum pattern length. +*/ +static int re_maxnfa(int mxlen){ + return 75+mxlen/2; } /* @@ -770,10 +777,17 @@ static void re_sql_func( (void)argc; /* Unused */ pRe = sqlite3_get_auxdata(context, 0); if( pRe==0 ){ + int mxLen = re_maxlen(context); + int nPattern; zPattern = (const char*)sqlite3_value_text(argv[0]); if( zPattern==0 ) return; - zErr = re_compile(&pRe, zPattern, re_maxlen(context), - sqlite3_user_data(context)!=0); + nPattern = sqlite3_value_bytes(argv[0]); + if( nPattern>mxLen ){ + zErr = "REGEXP pattern too big"; + }else{ + zErr = re_compile(&pRe, zPattern, re_maxnfa(mxLen), + sqlite3_user_data(context)!=0); + } if( zErr ){ re_free(pRe); sqlite3_result_error(context, zErr, -1); @@ -814,7 +828,6 @@ static void re_bytecode_func( int i; int n; char *z; - (void)argc; static const char *ReOpName[] = { "EOF", "MATCH", @@ -837,9 +850,10 @@ static void re_bytecode_func( "ATSTART", }; + (void)argc; zPattern = (const char*)sqlite3_value_text(argv[0]); if( zPattern==0 ) return; - zErr = re_compile(&pRe, zPattern, re_maxlen(context), + zErr = re_compile(&pRe, zPattern, re_maxnfa(re_maxlen(context)), sqlite3_user_data(context)!=0); if( zErr ){ re_free(pRe); diff --git a/ext/misc/scrub.c b/ext/misc/scrub.c index 9fbf2aed4..2406d39f2 100644 --- a/ext/misc/scrub.c +++ b/ext/misc/scrub.c @@ -92,7 +92,7 @@ static void scrubBackupErr(ScrubState *p, const char *zFormat, ...){ static u8 *scrubBackupAllocPage(ScrubState *p){ u8 *pPage; if( p->rcErr ) return 0; - pPage = sqlite3_malloc( p->szPage ); + pPage = sqlite3_malloc64( p->szPage ); if( pPage==0 ) p->rcErr = SQLITE_NOMEM; return pPage; } diff --git a/ext/misc/series.c b/ext/misc/series.c index ffdb23c1a..ac8f4597f 100644 --- a/ext/misc/series.c +++ b/ext/misc/series.c @@ -239,7 +239,7 @@ static int seriesConnect( rc = sqlite3_declare_vtab(db, "CREATE TABLE x(value,start hidden,stop hidden,step hidden)"); if( rc==SQLITE_OK ){ - pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + pNew = *ppVtab = sqlite3_malloc64( sizeof(*pNew) ); if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); @@ -261,7 +261,7 @@ static int seriesDisconnect(sqlite3_vtab *pVtab){ static int seriesOpen(sqlite3_vtab *pUnused, sqlite3_vtab_cursor **ppCursor){ series_cursor *pCur; (void)pUnused; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); *ppCursor = &pCur->base; diff --git a/ext/misc/sha1.c b/ext/misc/sha1.c index 07d797060..fb8f625f5 100644 --- a/ext/misc/sha1.c +++ b/ext/misc/sha1.c @@ -230,13 +230,16 @@ static void hash_finish( *****************************************************************************/ /* -** Implementation of the sha1(X) function. +** Two SQL functions: sha1(X) and sha1b(X). ** -** Return a lower-case hexadecimal rendering of the SHA1 hash of the -** argument X. If X is a BLOB, it is hashed as is. For all other +** sha1(X) returns a lower-case hexadecimal rendering of the SHA1 hash +** of the argument X. If X is a BLOB, it is hashed as is. For all other ** types of input, X is converted into a UTF-8 string and the string -** is hash without the trailing 0x00 terminator. The hash of a NULL +** is hashed without the trailing 0x00 terminator. The hash of a NULL ** value is NULL. +** +** sha1b(X) is the same except that it returns a 20-byte BLOB containing +** the binary hash instead of a hexadecimal string. */ static void sha1Func( sqlite3_context *context, @@ -246,22 +249,27 @@ static void sha1Func( SHA1Context cx; int eType = sqlite3_value_type(argv[0]); int nByte = sqlite3_value_bytes(argv[0]); + const unsigned char *pData; char zOut[44]; assert( argc==1 ); if( eType==SQLITE_NULL ) return; hash_init(&cx); if( eType==SQLITE_BLOB ){ - hash_step(&cx, sqlite3_value_blob(argv[0]), nByte); + pData = (const unsigned char*)sqlite3_value_blob(argv[0]); }else{ - hash_step(&cx, sqlite3_value_text(argv[0]), nByte); + pData = (const unsigned char*)sqlite3_value_text(argv[0]); } + if( pData==0 ) return; + hash_step(&cx, pData, nByte); if( sqlite3_user_data(context)!=0 ){ + /* sha1b() - binary result */ hash_finish(&cx, zOut, 1); sqlite3_result_blob(context, zOut, 20, SQLITE_TRANSIENT); }else{ + /* sha1() - hexadecimal text result */ hash_finish(&cx, zOut, 0); - sqlite3_result_blob(context, zOut, 40, SQLITE_TRANSIENT); + sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); } } @@ -315,6 +323,7 @@ static void sha1QueryFunc( } nCol = sqlite3_column_count(pStmt); z = sqlite3_sql(pStmt); + if( z==0 ) z = ""; n = (int)strlen(z); hash_step_vformat(&cx,"S%d:",n); hash_step(&cx,(unsigned char*)z,n); diff --git a/ext/misc/sqlar.c b/ext/misc/sqlar.c index 9f726f0b8..30ccc4f55 100644 --- a/ext/misc/sqlar.c +++ b/ext/misc/sqlar.c @@ -46,7 +46,7 @@ static void sqlarCompressFunc( uLongf nOut = compressBound(nData); Bytef *pOut; - pOut = (Bytef*)sqlite3_malloc(nOut); + pOut = (Bytef*)sqlite3_malloc64(nOut); if( pOut==0 ){ sqlite3_result_error_nomem(context); return; @@ -84,14 +84,14 @@ static void sqlarUncompressFunc( sqlite3_int64 sz; assert( argc==2 ); - sz = sqlite3_value_int(argv[1]); + sz = sqlite3_value_int64(argv[1]); if( sz<=0 || sz==(nData = sqlite3_value_bytes(argv[0])) ){ sqlite3_result_value(context, argv[0]); }else{ uLongf szf = sz; const Bytef *pData= sqlite3_value_blob(argv[0]); - Bytef *pOut = sqlite3_malloc(sz); + Bytef *pOut = sqlite3_malloc64(sz); if( pOut==0 ){ sqlite3_result_error_nomem(context); }else if( Z_OK!=uncompress(pOut, &szf, pData, nData) ){ diff --git a/ext/misc/sqlite3_stdio.c b/ext/misc/sqlite3_stdio.c index c9bceb194..049dd5174 100644 --- a/ext/misc/sqlite3_stdio.c +++ b/ext/misc/sqlite3_stdio.c @@ -101,8 +101,8 @@ FILE *sqlite3_fopen(const char *zFilename, const char *zMode){ sz1 = (int)strlen(zFilename); sz2 = (int)strlen(zMode); - b1 = sqlite3_malloc( (sz1+1)*sizeof(b1[0]) ); - b2 = sqlite3_malloc( (sz2+1)*sizeof(b1[0]) ); + b1 = sqlite3_malloc64( (sz1+1)*sizeof(b1[0]) ); + b2 = sqlite3_malloc64( (sz2+1)*sizeof(b1[0]) ); if( b1 && b2 ){ sz1 = MultiByteToWideChar(CP_UTF8, 0, zFilename, sz1, b1, sz1); b1[sz1] = 0; @@ -127,8 +127,8 @@ FILE *sqlite3_popen(const char *zCommand, const char *zMode){ sz1 = (int)strlen(zCommand); sz2 = (int)strlen(zMode); - b1 = sqlite3_malloc( (sz1+1)*sizeof(b1[0]) ); - b2 = sqlite3_malloc( (sz2+1)*sizeof(b1[0]) ); + b1 = sqlite3_malloc64( (sz1+1)*sizeof(b1[0]) ); + b2 = sqlite3_malloc64( (sz2+1)*sizeof(b1[0]) ); if( b1 && b2 ){ sz1 = MultiByteToWideChar(CP_UTF8, 0, zCommand, sz1, b1, sz1); b1[sz1] = 0; @@ -151,7 +151,7 @@ char *sqlite3_fgets(char *buf, int sz, FILE *in){ ** that into UTF-8. Otherwise, non-ASCII characters all get translated ** into '?'. */ - wchar_t *b1 = sqlite3_malloc( sz*sizeof(wchar_t) ); + wchar_t *b1 = sqlite3_malloc64( sz*sizeof(wchar_t) ); if( b1==0 ) return 0; #ifdef SQLITE_USE_W32_FOR_CONSOLE_IO DWORD nRead = 0; @@ -226,7 +226,7 @@ int sqlite3_fputs(const char *z, FILE *out){ ** to the console on Windows. */ int sz = (int)strlen(z); - wchar_t *b1 = sqlite3_malloc( (sz+1)*sizeof(wchar_t) ); + wchar_t *b1 = sqlite3_malloc64( (sz+1)*sizeof(wchar_t) ); if( b1==0 ) return 0; sz = MultiByteToWideChar(CP_UTF8, 0, z, sz, b1, sz); b1[sz] = 0; @@ -258,7 +258,7 @@ int sqlite3_fputs(const char *z, FILE *out){ /* -** Work-alike for fprintf() from the standard C library. +** Work-alikes for fprintf() and vfprintf() from the standard C library. */ int sqlite3_fprintf(FILE *out, const char *zFormat, ...){ int rc; @@ -285,6 +285,24 @@ int sqlite3_fprintf(FILE *out, const char *zFormat, ...){ } return rc; } +int sqlite3_vfprintf(FILE *out, const char *zFormat, va_list ap){ + int rc; + if( UseWtextForOutput(out) ){ + /* When writing to the command-prompt in Windows, it is necessary + ** to use _O_WTEXT input mode and write UTF-16 characters. + */ + char *z; + z = sqlite3_vmprintf(zFormat, ap); + sqlite3_fputs(z, out); + rc = (int)strlen(z); + sqlite3_free(z); + }else{ + /* Writing to a file or other destination, just write bytes without + ** any translation. */ + rc = vfprintf(out, zFormat, ap); + } + return rc; +} /* ** Set the mode for an output stream. mode argument is typically _O_BINARY or diff --git a/ext/misc/sqlite3_stdio.h b/ext/misc/sqlite3_stdio.h index dd0eefad0..75368df9f 100644 --- a/ext/misc/sqlite3_stdio.h +++ b/ext/misc/sqlite3_stdio.h @@ -31,6 +31,7 @@ #ifdef _WIN32 /**** Definitions For Windows ****/ #include <stdio.h> +#include <stdarg.h> #include <windows.h> FILE *sqlite3_fopen(const char *zFilename, const char *zMode); @@ -38,6 +39,7 @@ FILE *sqlite3_popen(const char *zCommand, const char *type); char *sqlite3_fgets(char *s, int size, FILE *stream); int sqlite3_fputs(const char *s, FILE *stream); int sqlite3_fprintf(FILE *stream, const char *format, ...); +int sqlite3_vfprintf(FILE *stream, const char *format, va_list); void sqlite3_fsetmode(FILE *stream, int mode); @@ -49,6 +51,7 @@ void sqlite3_fsetmode(FILE *stream, int mode); #define sqlite3_fgets fgets #define sqlite3_fputs fputs #define sqlite3_fprintf fprintf +#define sqlite3_vfprintf vfprintf #define sqlite3_fsetmode(F,X) /*no-op*/ #endif diff --git a/ext/misc/stmtrand.c b/ext/misc/stmtrand.c index b5e3b89a3..7e52ef25b 100644 --- a/ext/misc/stmtrand.c +++ b/ext/misc/stmtrand.c @@ -52,7 +52,7 @@ static void stmtrandFunc( p = (Stmtrand*)sqlite3_get_auxdata(context, STMTRAND_KEY); if( p==0 ){ unsigned int seed; - p = sqlite3_malloc( sizeof(*p) ); + p = sqlite3_malloc64( sizeof(*p) ); if( p==0 ){ sqlite3_result_error_nomem(context); return; diff --git a/ext/misc/templatevtab.c b/ext/misc/templatevtab.c index 5865f5214..ba7937343 100644 --- a/ext/misc/templatevtab.c +++ b/ext/misc/templatevtab.c @@ -101,7 +101,7 @@ static int templatevtabConnect( #define TEMPLATEVTAB_A 0 #define TEMPLATEVTAB_B 1 if( rc==SQLITE_OK ){ - pNew = sqlite3_malloc( sizeof(*pNew) ); + pNew = sqlite3_malloc64( sizeof(*pNew) ); *ppVtab = (sqlite3_vtab*)pNew; if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); @@ -123,7 +123,7 @@ static int templatevtabDisconnect(sqlite3_vtab *pVtab){ */ static int templatevtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ templatevtab_cursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); *ppCursor = &pCur->base; diff --git a/ext/misc/tmstmpvfs.c b/ext/misc/tmstmpvfs.c new file mode 100644 index 000000000..6f1af36f7 --- /dev/null +++ b/ext/misc/tmstmpvfs.c @@ -0,0 +1,1042 @@ +/* +** 2026-01-05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements a VFS shim that writes a timestamp and other tracing +** information into 16 byts of reserved space at the end of each page of the +** database file. +** +** The VFS also tries to generate log-files with names of the form: +** +** $(DATABASE)-tmstmp/$(TIME)-$(PID)-$(ID) +** +** Log files are only generated if directory $(DATABASE)-tmstmp exists. +** The name of each log file is the current ISO8601 time in milliseconds, +** the process ID, and a random 32-bit value (to disambiguate multiple +** connections from the same process) separated by dashes. The log file +** contains 16-bytes records for various events, such as opening or close +** of the database or WAL file, writes to the WAL file, checkpoints, and +** similar. The logfile is only generated if the connection attempts to +** modify the database. There is a separate log file for each open database +** connection. +** +** COMPILING +** +** To build this extension as a separately loaded shared library or +** DLL, use compiler command-lines similar to the following: +** +** (linux) gcc -fPIC -shared tmstmpvfs.c -o tmstmpvfs.so +** (mac) clang -fPIC -dynamiclib tmstmpvfs.c -o tmstmpvfs.dylib +** (windows) cl tmstmpvfs.c -link -dll -out:tmstmpvfs.dll +** +** You may want to add additional compiler options, of course, +** according to the needs of your project. +** +** Another option is to statically link both SQLite and this extension +** into your application. If both this file and "sqlite3.c" are statically +** linked, and if "sqlite3.c" is compiled with an option like: +** +** -DSQLITE_EXTRA_INIT=sqlite3_register_tmstmpvfs +** +** Then SQLite will use the tmstmp VFS by default throughout your +** application. +** +** LOADING +** +** To load this extension as a shared library, you first have to +** bring up a dummy SQLite database connection to use as the argument +** to the sqlite3_load_extension() API call. Then you invoke the +** sqlite3_load_extension() API and shutdown the dummy database +** connection. All subsequent database connections that are opened +** will include this extension. For example: +** +** sqlite3 *db; +** sqlite3_open(":memory:", &db); +** sqlite3_load_extension(db, "./tmstmpvfs"); +** sqlite3_close(db); +** +** Tmstmpvfs is a VFS Shim. When loaded, "tmstmpvfs" becomes the new +** default VFS and it uses the prior default VFS as the next VFS +** down in the stack. This is normally what you want. However, in +** complex situations where multiple VFS shims are being loaded, +** it might be important to ensure that tmstmpvfs is loaded in the +** correct order so that it sequences itself into the default VFS +** Shim stack in the right order. +** +** When running the CLI, you can load this extension at invocation by +** adding a command-line option like this: "--vfs ./tmstmpvfs.so". +** The --vfs option usually specifies the symbolic name of a built-in VFS. +** But if the argument to --vfs is not a built-in VFS but is instead the +** name of a file, the CLI tries to load that file as an extension. Note +** that the full name of the extension file must be provided, including +** the ".so" or ".dylib" or ".dll" suffix. +** +** An application can see if the tmstmpvfs is being used by examining +** the results from SQLITE_FCNTL_VFSNAME (or the .vfsname command in +** the CLI). If the answer include "tmstmp", then this VFS is being +** used. +** +** USING +** +** Open database connections using the sqlite3_open() or +** sqlite3_open_v2() interfaces, as normal. Ordinary database files +** (without a timestamp) will operate normally. +** +** Timestamping only works on databases that have a reserve-bytes +** value of exactly 16. The default value for reserve-bytes is 0. +** Hence, newly created database files will omit the timestamp by +** default. To create a database that includes a timestamp, change +** the reserve-bytes value to 16 by running: +** +** int n = 16; +** sqlite3_file_control(db, 0, SQLITE_FCNTL_RESERVE_BYTES, &n); +** +** If you do this immediately after creating a new database file, +** before anything else has been written into the file, then that +** might be all that you need to do. Otherwise, the API call +** above should be followed by: +** +** sqlite3_exec(db, "VACUUM", 0, 0, 0); +** +** It never hurts to run the VACUUM, even if you don't need it. +** +** From the CLI, use the ".filectrl reserve_bytes 16" command, +** followed by "VACUUM;". +** +** SQLite allows the number of reserve-bytes to be increased, but +** not decreased. If you want to restore the reserve-bytes to 0 +** (to disable tmstmpvfs), the easiest approach is to use VACUUM INTO +** with a URI filename as the argument and include "reserve=0" query +** parameter on the URI. Example: +** +** VACUUM INTO 'file:notimestamps.db?reserve=0'; +** +** Then switch over to using the new database file. The reserve=0 query +** parameter only works on SQLite 3.52.0 and later. +** +** IMPLEMENTATION NOTES +** +** The timestamp information is stored in the last 16 bytes of each page. +** This module only operates if the "bytes of reserved space on each page" +** value at offset 20 the SQLite database header is exactly 16. If +** the reserved-space value is not 16, no timestamp information is added +** to database pages. Some, but not all, logfile entries will be made +** still, but the size of the logs will be greatly reduced. +** +** The timestamp layout is as follows: +** +** bytes 0,1 Zero. Reserved for future expansion +** bytes 2-7 Milliseconds since the Unix Epoch +** bytes 8-11 WAL frame number +** bytes 12 0: WAL write 2: rollback write +** bytes 13-15 Lower 24 bits of Salt-1 +** +** For transactions that occur in rollback mode, only the timestamp +** in bytes 2-7 and byte 12 are non-zero. Byte 12 is set to 2 for +** rollback writes. +** +** The 16-byte tag is added to each database page when the content +** is written into the database file itself. This shim does not make +** any changes to the page as it is written to the WAL file, since +** that would mess up the WAL checksum. +** +** LOGGING +** +** An open database connection that attempts to write to the database +** will create a log file if a directory name $(DATABASE)-tmstmp exists. +** The name of the log file is: +** +** $(TIME)-$(PID)-$(RANDOM) +** +** Where TIME is an ISO 8601 date in milliseconds with no punctuation, +** PID is the process ID, and RANDOM is a 32-bit random number expressed +** as hexadecimal. +** +** The log consists of 16-byte records. Each record consists of five +** unsigned integers: +** +** 1 1 6 4 4 <--- bytes +** op a1 ts a2 a3 +** +** The meanings of the a1-a3 values depend on op. ts is the timestamp +** in milliseconds since the unix epoch (1970-01-01 00:00:00). +** Opcodes are defined by the ELOG_* #defines below. +** +** ELOG_OPEN_DB "Open a connection to the database file" +** op = 0x01 +** a2 = process-ID +** +** ELOG_OPEN_WAL "Open a connection to the -wal file" +** op = 0x02 +** a2 = process-ID +** +** ELOG_WAL_PAGE "New page added to the WAL file" +** op = 0x03 +** a1 = 1 if last page of a txn. 0 otherwise. +** a2 = page number in the DB file +** a3 = frame number in the WAL file +** +** ELOG_DB_PAGE "Database page updated using rollback mode" +** op = 0x04 +** a2 = page number in the DB file +** +** ELOG_CKPT_START "Start of a checkpoint operation" +** op = 0x05 +** +** ELOG_CKPT_PAGE "Page xfer from WAL to database" +** op = 0x06 +** a2 = database page number +** a3 = frame number in the WAL file +** +** ELOG_CKPT_END "Start of a checkpoint operation" +** op = 0x07 +** +** ELOG_WAL_RESET "WAL file header overwritten" +** op = 0x08 +** a3 = Salt1 value +** +** ELOG_CLOSE_WAL "Close the WAL file connection" +** op = 0x0e +** +** ELOG_CLOSE_DB "Close the DB connection" +** op = 0x0f +** +** VIEWING TIMESTAMPS AND LOGS +** +** The command-line utility at tool/showtmlog.c will read and display +** the content of one or more tmstmpvfs.c log files. If all of the +** log files are stored in directory $(DATABASE)-tmstmp, then you can +** view them all using a command like shown below (with an extra "?" +** inserted on the wildcard to avoid closing the C-language comment +** that contains this text): +** +** showtmlog $(DATABASE)-tmstmp/?* +** +** The command-line utility at tools/showdb.c can be used to show the +** timestamps on pages of a database file, using a command like this: +** +** showdb --tmstmp $(DATABASE) pgidx +* +** The command above shows the timestamp and the intended use of every +** pages in the database, in human-readable form. If you also add +** the --csv option to the command above, then the command generates +** a Comma-Separated-Value (CSV) file as output, which contains a +** decoding of the complete timestamp tag on each page of the database. +** This CVS file can be easily imported into another SQLite database +** using a CLI command like the following: +** +** .import --csv '|showdb --tmstmp -csv orig.db pgidx' ts_table +** +** In the command above, the database containing the timestamps is +** "orig.db" and the content is imported into a new table named "ts_table". +** The "ts_table" is created automatically, using the column names found +** in the first line of the CSV file. All columns of the automatically +** created ts_table are of type TEXT. It might make more sense to +** create the table yourself, using more sensible datatypes, like this: +** +** CREATE TABLE ts_table ( +** pgno INT, -- page number +** tm REAL, -- seconds since 1970-01-01 +** frame INT, -- WAL frame number +** flg INT, -- flag (tag byte 12) +** salt INT, -- WAL salt (tag bytes 13-15) +** parent INT, -- Parent page number +** child INT, -- Index of this page in its parent +** ovfl INT, -- Index of this page on the overflow chain +** txt TEXT -- Description of this page +** ); +** +** Then import using: +** +** .import --csv --skip 1 '|showdb --tmstmp --csv orig.db pgidx' ts_table +** +** Note the addition of the "--skip 1" option on ".import" to bypass the +** first line of the CSV file that contains the column names. +** +** Both programs "showdb" and "showtmlog" can be built by running +** "make showtmlog showdb" from the top-level of a recent SQLite +** source tree. +*/ +#if defined(SQLITE_AMALGAMATION) && !defined(SQLITE_TMSTMPVFS_STATIC) +# define SQLITE_TMSTMPVFS_STATIC +#endif +#ifdef SQLITE_TMSTMPVFS_STATIC +# include "sqlite3.h" +#else +# include "sqlite3ext.h" + SQLITE_EXTENSION_INIT1 +#endif +#include <string.h> +#include <assert.h> +#include <stdio.h> + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct sqlite3_vfs TmstmpVfs; +typedef struct TmstmpFile TmstmpFile; +typedef struct TmstmpLog TmstmpLog; + +/* +** Bytes of reserved space used by this extension +*/ +#define TMSTMP_RESERVE 16 + +/* +** The magic number used to identify TmstmpFile objects +*/ +#define TMSTMP_MAGIC 0x2a87b72d + +/* +** Useful datatype abbreviations +*/ +#if !defined(SQLITE_AMALGAMATION) + typedef unsigned char u8; + typedef unsigned int u32; +#endif + +/* +** Current process id +*/ +#if defined(_WIN32) +# include <windows.h> +# define GETPID (u32)GetCurrentProcessId() +#else +# include <unistd.h> +# define GETPID (u32)getpid() +#endif + +/* Access to a lower-level VFS that (might) implement dynamic loading, +** access to randomness, etc. +*/ +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) +#define ORIGFILE(p) ((sqlite3_file*)(((TmstmpFile*)(p))+1)) + +/* Information for the tmstmp log file. */ +struct TmstmpLog { + char *zLogname; /* Log filename */ + FILE *log; /* Open log file */ + int n; /* Bytes of a[] used */ + unsigned char a[16*6]; /* Buffered header for the log */ +}; + +/* An open WAL or DB file */ +struct TmstmpFile { + sqlite3_file base; /* IO methods */ + u32 uMagic; /* Magic number for sanity checking */ + u32 salt1; /* Last WAL salt-1 value */ + u32 iFrame; /* Last WAL frame number */ + u32 pgno; /* Current page number */ + u32 pgsz; /* Size of each page, in bytes */ + u8 isWal; /* True if this is a WAL file */ + u8 isDb; /* True if this is a DB file */ + u8 isCommit; /* Last WAL frame header was a transaction commit */ + u8 hasCorrectReserve; /* File has the correct reserve size */ + u8 inCkpt; /* True if in a checkpoint */ + TmstmpLog *pLog; /* Log file */ + TmstmpFile *pPartner; /* DB->WAL or WAL->DB mapping */ + sqlite3_int64 iOfst; /* Offset of last WAL frame header */ + sqlite3_vfs *pSubVfs; /* Underlying VFS */ +}; + +/* +** Event log opcodes +*/ +#define ELOG_OPEN_DB 0x01 +#define ELOG_OPEN_WAL 0x02 +#define ELOG_WAL_PAGE 0x03 +#define ELOG_DB_PAGE 0x04 +#define ELOG_CKPT_START 0x05 +#define ELOG_CKPT_PAGE 0x06 +#define ELOG_CKPT_DONE 0x07 +#define ELOG_WAL_RESET 0x08 +#define ELOG_CLOSE_WAL 0x0e +#define ELOG_CLOSE_DB 0x0f + +/* +** Methods for TmstmpFile +*/ +static int tmstmpClose(sqlite3_file*); +static int tmstmpRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int tmstmpWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int tmstmpTruncate(sqlite3_file*, sqlite3_int64 size); +static int tmstmpSync(sqlite3_file*, int flags); +static int tmstmpFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int tmstmpLock(sqlite3_file*, int); +static int tmstmpUnlock(sqlite3_file*, int); +static int tmstmpCheckReservedLock(sqlite3_file*, int *pResOut); +static int tmstmpFileControl(sqlite3_file*, int op, void *pArg); +static int tmstmpSectorSize(sqlite3_file*); +static int tmstmpDeviceCharacteristics(sqlite3_file*); +static int tmstmpShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); +static int tmstmpShmLock(sqlite3_file*, int offset, int n, int flags); +static void tmstmpShmBarrier(sqlite3_file*); +static int tmstmpShmUnmap(sqlite3_file*, int deleteFlag); +static int tmstmpFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int tmstmpUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for TmstmpVfs +*/ +static int tmstmpOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int tmstmpDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int tmstmpAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int tmstmpFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *tmstmpDlOpen(sqlite3_vfs*, const char *zFilename); +static void tmstmpDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*tmstmpDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void tmstmpDlClose(sqlite3_vfs*, void*); +static int tmstmpRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int tmstmpSleep(sqlite3_vfs*, int microseconds); +static int tmstmpCurrentTime(sqlite3_vfs*, double*); +static int tmstmpGetLastError(sqlite3_vfs*, int, char *); +static int tmstmpCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); +static int tmstmpSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr); +static sqlite3_syscall_ptr tmstmpGetSystemCall(sqlite3_vfs*, const char *z); +static const char *tmstmpNextSystemCall(sqlite3_vfs*, const char *zName); + +static sqlite3_vfs tmstmp_vfs = { + 3, /* iVersion (set when registered) */ + 0, /* szOsFile (set when registered) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "tmstmpvfs", /* zName */ + 0, /* pAppData (set when registered) */ + tmstmpOpen, /* xOpen */ + tmstmpDelete, /* xDelete */ + tmstmpAccess, /* xAccess */ + tmstmpFullPathname, /* xFullPathname */ + tmstmpDlOpen, /* xDlOpen */ + tmstmpDlError, /* xDlError */ + tmstmpDlSym, /* xDlSym */ + tmstmpDlClose, /* xDlClose */ + tmstmpRandomness, /* xRandomness */ + tmstmpSleep, /* xSleep */ + tmstmpCurrentTime, /* xCurrentTime */ + tmstmpGetLastError, /* xGetLastError */ + tmstmpCurrentTimeInt64, /* xCurrentTimeInt64 */ + tmstmpSetSystemCall, /* xSetSystemCall */ + tmstmpGetSystemCall, /* xGetSystemCall */ + tmstmpNextSystemCall /* xNextSystemCall */ +}; + +static const sqlite3_io_methods tmstmp_io_methods = { + 3, /* iVersion */ + tmstmpClose, /* xClose */ + tmstmpRead, /* xRead */ + tmstmpWrite, /* xWrite */ + tmstmpTruncate, /* xTruncate */ + tmstmpSync, /* xSync */ + tmstmpFileSize, /* xFileSize */ + tmstmpLock, /* xLock */ + tmstmpUnlock, /* xUnlock */ + tmstmpCheckReservedLock, /* xCheckReservedLock */ + tmstmpFileControl, /* xFileControl */ + tmstmpSectorSize, /* xSectorSize */ + tmstmpDeviceCharacteristics, /* xDeviceCharacteristics */ + tmstmpShmMap, /* xShmMap */ + tmstmpShmLock, /* xShmLock */ + tmstmpShmBarrier, /* xShmBarrier */ + tmstmpShmUnmap, /* xShmUnmap */ + tmstmpFetch, /* xFetch */ + tmstmpUnfetch /* xUnfetch */ +}; + +/* +** Write a 6-byte millisecond timestamp into aOut[] +*/ +static void tmstmpPutTS(TmstmpFile *p, unsigned char *aOut){ + sqlite3_uint64 tm = 0; + p->pSubVfs->xCurrentTimeInt64(p->pSubVfs, (sqlite3_int64*)&tm); + tm -= 210866760000000LL; + aOut[0] = (tm>>40)&0xff; + aOut[1] = (tm>>32)&0xff; + aOut[2] = (tm>>24)&0xff; + aOut[3] = (tm>>16)&0xff; + aOut[4] = (tm>>8)&0xff; + aOut[5] = tm&0xff; +} + +/* +** Read a 32-bit big-endian unsigned integer and return it. +*/ +static u32 tmstmpGetU32(const unsigned char *a){ + return (a[0]<<24) + (a[1]<<16) + (a[2]<<8) + a[3]; +} + +/* Write a 32-bit integer as big-ending into a[] +*/ +static void tmstmpPutU32(u32 v, unsigned char *a){ + a[0] = (v>>24) & 0xff; + a[1] = (v>>16) & 0xff; + a[2] = (v>>8) & 0xff; + a[3] = v & 0xff; +} + +/* Free a TmstmpLog object */ +static void tmstmpLogFree(TmstmpLog *pLog){ + if( pLog==0 ) return; + if( pLog->log ) fclose(pLog->log); + sqlite3_free(pLog->zLogname); + sqlite3_free(pLog); +} + +/* Flush log content. Open the file if necessary. Return the +** number of errors. */ +static int tmstmpLogFlush(TmstmpFile *p){ + TmstmpLog *pLog = p->pLog; + assert( pLog!=0 ); + if( pLog->log==0 ){ + pLog->log = fopen(pLog->zLogname, "wb"); + if( pLog->log==0 ){ + tmstmpLogFree(pLog); + p->pLog = 0; + return 1; + } + } + (void)fwrite(pLog->a, pLog->n, 1, pLog->log); + fflush(pLog->log); + pLog->n = 0; + return 0; +} + +/* +** Write a record onto the event log +*/ +static void tmstmpEvent( + TmstmpFile *p, + u8 op, + u8 a1, + u32 a2, + u32 a3, + u8 *pTS +){ + unsigned char *a; + TmstmpLog *pLog; + if( p->isWal ){ + p = p->pPartner; + assert( p!=0 ); + assert( p->isDb ); + } + pLog = p->pLog; + if( pLog==0 ) return; + if( pLog->n >= (int)sizeof(pLog->a) ){ + if( tmstmpLogFlush(p) ) return; + } + a = pLog->a + pLog->n; + a[0] = op; + a[1] = a1; + if( pTS ){ + memcpy(a+2, pTS, 6); + }else{ + tmstmpPutTS(p, a+2); + } + tmstmpPutU32(a2, a+8); + tmstmpPutU32(a3, a+12); + pLog->n += 16; + if( pLog->log || (op>=ELOG_WAL_PAGE && op<=ELOG_WAL_RESET) ){ + (void)tmstmpLogFlush(p); + } +} + +/* +** Close a connection +*/ +static int tmstmpClose(sqlite3_file *pFile){ + TmstmpFile *p = (TmstmpFile *)pFile; + if( p->hasCorrectReserve ){ + tmstmpEvent(p, p->isDb ? ELOG_CLOSE_DB : ELOG_CLOSE_WAL, 0, 0, 0, 0); + } + tmstmpLogFree(p->pLog); + if( p->pPartner ){ + assert( p->pPartner->pPartner==p ); + p->pPartner->pPartner = 0; + p->pPartner = 0; + } + pFile = ORIGFILE(pFile); + return pFile->pMethods->xClose(pFile); +} + +/* +** Read bytes from a file +*/ +static int tmstmpRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + int rc; + TmstmpFile *p = (TmstmpFile*)pFile; + pFile = ORIGFILE(pFile); + rc = pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst); + if( rc!=SQLITE_OK ) return rc; + if( p->isDb + && iOfst==0 + && iAmt>=100 + ){ + const unsigned char *a = (unsigned char*)zBuf; + p->hasCorrectReserve = (a[20]==TMSTMP_RESERVE); + p->pgsz = (a[16]<<8) + a[17]; + if( p->pgsz==1 ) p->pgsz = 65536; + if( p->pPartner ){ + p->pPartner->hasCorrectReserve = p->hasCorrectReserve; + p->pPartner->pgsz = p->pgsz; + } + } + if( p->isWal + && p->inCkpt + && iAmt>=512 && iAmt<=65535 && (iAmt&(iAmt-1))==0 + ){ + p->pPartner->iFrame = (iOfst-56)/(p->pgsz+24) + 1; + } + return rc; +} + +/* +** Write data to a tmstmp-file. +*/ +static int tmstmpWrite( + sqlite3_file *pFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + TmstmpFile *p = (TmstmpFile*)pFile; + sqlite3_file *pSub = ORIGFILE(pFile); + if( !p->hasCorrectReserve ){ + /* The database does not have the correct reserve size. No-op */ + }else if( p->isWal ){ + /* Writing into a WAL file */ + if( iAmt==24 ){ + /* A frame header */ + u32 x = 0; + p->iFrame = (iOfst - 32)/(p->pgsz+24)+1; + p->pgno = tmstmpGetU32((const u8*)zBuf); + p->salt1 = tmstmpGetU32(((const u8*)zBuf)+8); + memcpy(&x, ((const u8*)zBuf)+4, 4); + p->isCommit = (x!=0); + p->iOfst = iOfst; + }else if( iAmt>=512 && iOfst==p->iOfst+24 ){ + unsigned char s[TMSTMP_RESERVE]; + memset(s, 0, TMSTMP_RESERVE); + tmstmpPutTS(p, s+2); + tmstmpEvent(p, ELOG_WAL_PAGE, p->isCommit, p->pgno, p->iFrame, s+2); + }else if( iAmt==32 && iOfst==0 ){ + p->salt1 = tmstmpGetU32(((const u8*)zBuf)+16); + tmstmpEvent(p, ELOG_WAL_RESET, 0, 0, p->salt1, 0); + } + }else if( p->inCkpt ){ + unsigned char *s = (unsigned char*)zBuf+iAmt-TMSTMP_RESERVE; + memset(s, 0, TMSTMP_RESERVE); + tmstmpPutTS(p, s+2); + tmstmpPutU32(p->iFrame, s+8); + tmstmpPutU32(p->pPartner->salt1 & 0xffffff, s+12); + assert( p->pgsz>0 ); + tmstmpEvent(p, ELOG_CKPT_PAGE, 0, (iOfst/p->pgsz)+1, p->iFrame, 0); + }else if( p->pPartner==0 ){ + /* Writing into a database in rollback mode */ + unsigned char *s = (unsigned char*)zBuf+iAmt-TMSTMP_RESERVE; + memset(s, 0, TMSTMP_RESERVE); + tmstmpPutTS(p, s+2); + s[12] = 2; + assert( p->pgsz>0 ); + tmstmpEvent(p, ELOG_DB_PAGE, 0, (u32)(iOfst/p->pgsz)+1, 0, s+2); + } + return pSub->pMethods->xWrite(pSub,zBuf,iAmt,iOfst); +} + +/* +** Truncate a tmstmp-file. +*/ +static int tmstmpTruncate(sqlite3_file *pFile, sqlite_int64 size){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xTruncate(pFile, size); +} + +/* +** Sync a tmstmp-file. +*/ +static int tmstmpSync(sqlite3_file *pFile, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSync(pFile, flags); +} + +/* +** Return the current file-size of a tmstmp-file. +*/ +static int tmstmpFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + TmstmpFile *p = (TmstmpFile *)pFile; + pFile = ORIGFILE(p); + return pFile->pMethods->xFileSize(pFile, pSize); +} + +/* +** Lock a tmstmp-file. +*/ +static int tmstmpLock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xLock(pFile, eLock); +} + +/* +** Unlock a tmstmp-file. +*/ +static int tmstmpUnlock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xUnlock(pFile, eLock); +} + +/* +** Check if another file-handle holds a RESERVED lock on a tmstmp-file. +*/ +static int tmstmpCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xCheckReservedLock(pFile, pResOut); +} + +/* +** File control method. For custom operations on a tmstmp-file. +*/ +static int tmstmpFileControl(sqlite3_file *pFile, int op, void *pArg){ + int rc; + TmstmpFile *p = (TmstmpFile*)pFile; + pFile = ORIGFILE(pFile); + rc = pFile->pMethods->xFileControl(pFile, op, pArg); + switch( op ){ + case SQLITE_FCNTL_VFSNAME: { + if( p->hasCorrectReserve && rc==SQLITE_OK ){ + *(char**)pArg = sqlite3_mprintf("tmstmp/%z", *(char**)pArg); + } + break; + } + case SQLITE_FCNTL_CKPT_START: { + p->inCkpt = 1; + assert( p->isDb ); + assert( p->pPartner!=0 ); + p->pPartner->inCkpt = 1; + if( p->hasCorrectReserve ){ + tmstmpEvent(p, ELOG_CKPT_START, 0, 0, 0, 0); + } + rc = SQLITE_OK; + break; + } + case SQLITE_FCNTL_CKPT_DONE: { + p->inCkpt = 0; + assert( p->isDb ); + assert( p->pPartner!=0 ); + p->pPartner->inCkpt = 0; + if( p->hasCorrectReserve ){ + tmstmpEvent(p, ELOG_CKPT_DONE, 0, 0, 0, 0); + } + rc = SQLITE_OK; + break; + } + } + return rc; +} + +/* +** Return the sector-size in bytes for a tmstmp-file. +*/ +static int tmstmpSectorSize(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSectorSize(pFile); +} + +/* +** Return the device characteristic flags supported by a tmstmp-file. +*/ +static int tmstmpDeviceCharacteristics(sqlite3_file *pFile){ + int devchar = 0; + pFile = ORIGFILE(pFile); + devchar = pFile->pMethods->xDeviceCharacteristics(pFile); + return (devchar & ~SQLITE_IOCAP_SUBPAGE_READ); +} + +/* Create a shared memory file mapping */ +static int tmstmpShmMap( + sqlite3_file *pFile, + int iPg, + int pgsz, + int bExtend, + void volatile **pp +){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp); +} + +/* Perform locking on a shared-memory segment */ +static int tmstmpShmLock(sqlite3_file *pFile, int offset, int n, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmLock(pFile,offset,n,flags); +} + +/* Memory barrier operation on shared memory */ +static void tmstmpShmBarrier(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + pFile->pMethods->xShmBarrier(pFile); +} + +/* Unmap a shared memory segment */ +static int tmstmpShmUnmap(sqlite3_file *pFile, int deleteFlag){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmUnmap(pFile,deleteFlag); +} + +/* Fetch a page of a memory-mapped file */ +static int tmstmpFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xFetch(pFile, iOfst, iAmt, pp); +} + +/* Release a memory-mapped page */ +static int tmstmpUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xUnfetch(pFile, iOfst, pPage); +} + + +/* +** Open a tmstmp file handle. +*/ +static int tmstmpOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + TmstmpFile *p, *pDb; + sqlite3_file *pSubFile; + sqlite3_vfs *pSubVfs; + int rc; + + pSubVfs = ORIGVFS(pVfs); + if( (flags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_WAL))==0 ){ + /* If the file is not a persistent database or a WAL file, then + ** bypass the timestamp logic all together */ + return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags); + } + if( (flags & SQLITE_OPEN_WAL)!=0 ){ + pDb = (TmstmpFile*)sqlite3_database_file_object(zName); + if( pDb==0 + || pDb->uMagic!=TMSTMP_MAGIC + || !pDb->isDb + || pDb->pPartner!=0 + ){ + return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags); + } + }else{ + pDb = 0; + } + p = (TmstmpFile*)pFile; + memset(p, 0, sizeof(*p)); + pSubFile = ORIGFILE(pFile); + pFile->pMethods = &tmstmp_io_methods; + p->pSubVfs = pSubVfs; + p->uMagic = TMSTMP_MAGIC; + rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags); + if( rc ) goto tmstmp_open_done; + if( pDb!=0 ){ + p->isWal = 1; + p->pPartner = pDb; + pDb->pPartner = p; + }else{ + u32 r2; + u32 pid; + TmstmpLog *pLog; + sqlite3_uint64 r1; /* Milliseconds since 1970-01-01 */ + sqlite3_uint64 days; /* Days since 1970-01-01 */ + sqlite3_uint64 sod; /* Start of date specified by r1 */ + sqlite3_uint64 z; /* Days since 0000-03-01 */ + sqlite3_uint64 era; /* 400-year era */ + int h; /* hour */ + int m; /* minute */ + int s; /* second */ + int f; /* millisecond */ + int Y; /* year */ + int M; /* month */ + int D; /* day */ + int y; /* year assuming March is first month */ + unsigned int doe; /* day of 400-year era */ + unsigned int yoe; /* year of 400-year era */ + unsigned int doy; /* day of year */ + unsigned int mp; /* month with March==0 */ + + p->isDb = 1; + r1 = 0; + pLog = sqlite3_malloc64( sizeof(TmstmpLog) ); + if( pLog==0 ){ + pSubFile->pMethods->xClose(pSubFile); + rc = SQLITE_NOMEM; + goto tmstmp_open_done; + } + memset(pLog, 0, sizeof(pLog[0])); + p->pLog = pLog; + p->pSubVfs->xCurrentTimeInt64(p->pSubVfs, (sqlite3_int64*)&r1); + r1 -= 210866760000000LL; + days = r1/86400000; + sod = (r1%86400000)/1000; + f = (int)(r1%1000); + + h = sod/3600; + m = (sod%3600)/60; + s = sod%60; + z = days + 719468; + era = z/146097; + doe = (unsigned)(z - era*146097); + yoe = (doe - doe/1460 + doe/36524 - doe/146096)/365; + y = (int)yoe + era*400; + doy = doe - (365*yoe + yoe/4 - yoe/100); + mp = (5*doy + 2)/153; + D = doy - (153*mp + 2)/5 + 1; + M = mp + (mp<10 ? 3 : -9); + Y = y + (M <=2); + sqlite3_randomness(sizeof(r2), &r2); + pid = GETPID; + pLog->zLogname = sqlite3_mprintf( + "%s-tmstmp/%04d%02d%02dT%02d%02d%02d%03d-%08d-%08x", + zName, Y, M, D, h, m, s, f, pid, r2); + } + tmstmpEvent(p, p->isWal ? ELOG_OPEN_WAL : ELOG_OPEN_DB, 0, GETPID, 0, 0); + +tmstmp_open_done: + if( rc ) pFile->pMethods = 0; + return rc; +} + +/* +** All VFS interfaces other than xOpen are passed down into the Sub-VFS. +*/ +static int tmstmpDelete(sqlite3_vfs *p, const char *zName, int syncDir){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xDelete(pSub,zName,syncDir); +} +static int tmstmpAccess(sqlite3_vfs *p, const char *zName, int flags, int *pR){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xAccess(pSub,zName,flags,pR); +} +static int tmstmpFullPathname(sqlite3_vfs*p,const char *zName,int n,char *zOut){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xFullPathname(pSub,zName,n,zOut); +} +static void *tmstmpDlOpen(sqlite3_vfs *p, const char *zFilename){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xDlOpen(pSub,zFilename); +} +static void tmstmpDlError(sqlite3_vfs *p, int nByte, char *zErrMsg){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xDlError(pSub,nByte,zErrMsg); +} +static void(*tmstmpDlSym(sqlite3_vfs *p, void *pDl, const char *zSym))(void){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xDlSym(pSub,pDl,zSym); +} +static void tmstmpDlClose(sqlite3_vfs *p, void *pDl){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xDlClose(pSub,pDl); +} +static int tmstmpRandomness(sqlite3_vfs *p, int nByte, char *zOut){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xRandomness(pSub,nByte,zOut); +} +static int tmstmpSleep(sqlite3_vfs *p, int microseconds){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xSleep(pSub,microseconds); +} +static int tmstmpCurrentTime(sqlite3_vfs *p, double *prNow){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xCurrentTime(pSub,prNow); +} +static int tmstmpGetLastError(sqlite3_vfs *p, int a, char *b){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xGetLastError(pSub,a,b); +} +static int tmstmpCurrentTimeInt64(sqlite3_vfs *p, sqlite3_int64 *piNow){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xCurrentTimeInt64(pSub,piNow); +} +static int tmstmpSetSystemCall(sqlite3_vfs *p, const char *zName, + sqlite3_syscall_ptr x){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xSetSystemCall(pSub,zName,x); +} +static sqlite3_syscall_ptr tmstmpGetSystemCall(sqlite3_vfs *p, const char *z){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xGetSystemCall(pSub,z); +} +static const char *tmstmpNextSystemCall(sqlite3_vfs *p, const char *zName){ + sqlite3_vfs *pSub = ORIGVFS(p); + return pSub->xNextSystemCall(pSub,zName); +} + +/* +** Register the tmstmp VFS as the default VFS for the system. +*/ +static int tmstmpRegisterVfs(void){ + int rc = SQLITE_OK; + sqlite3_vfs *pOrig = sqlite3_vfs_find(0); + if( pOrig==0 ) return SQLITE_ERROR; + if( pOrig==&tmstmp_vfs ) return SQLITE_OK; + tmstmp_vfs.iVersion = pOrig->iVersion; + tmstmp_vfs.pAppData = pOrig; + tmstmp_vfs.szOsFile = pOrig->szOsFile + sizeof(TmstmpFile); + rc = sqlite3_vfs_register(&tmstmp_vfs, 1); + return rc; +} + +#if defined(SQLITE_TMSTMPVFS_STATIC) +/* This variant of the initializer runs when the extension is +** statically linked. +*/ +int sqlite3_register_tmstmpvfs(const char *NotUsed){ + (void)NotUsed; + return tmstmpRegisterVfs(); +} +int sqlite3_unregister_tmstmpvfs(void){ + if( sqlite3_vfs_find("tmstmpvfs") ){ + sqlite3_vfs_unregister(&tmstmp_vfs); + } + return SQLITE_OK; +} +#endif /* defined(SQLITE_TMSTMPVFS_STATIC */ + +#if !defined(SQLITE_TMSTMPVFS_STATIC) +/* This variant of the initializer function is used when the +** extension is shared library to be loaded at run-time. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +/* +** This routine is called by sqlite3_load_extension() when the +** extension is first loaded. +***/ +int sqlite3_tmstmpvfs_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* not used */ + (void)db; /* not used */ + rc = tmstmpRegisterVfs(); + if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; + return rc; +} +#endif /* !defined(SQLITE_TMSTMPVFS_STATIC) */ diff --git a/ext/misc/vfsstat.c b/ext/misc/vfsstat.c index 504c0b31d..a7a17fffd 100644 --- a/ext/misc/vfsstat.c +++ b/ext/misc/vfsstat.c @@ -613,7 +613,7 @@ static int vstattabConnect( rc = sqlite3_declare_vtab(db,"CREATE TABLE x(file,stat,count)"); if( rc==SQLITE_OK ){ - pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + pNew = *ppVtab = sqlite3_malloc64( sizeof(*pNew) ); if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); } @@ -633,7 +633,7 @@ static int vstattabDisconnect(sqlite3_vtab *pVtab){ */ static int vstattabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ VfsStatCursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); *ppCursor = &pCur->base; diff --git a/ext/misc/vfstrace.c b/ext/misc/vfstrace.c index 9d75a8b64..24e75e117 100644 --- a/ext/misc/vfstrace.c +++ b/ext/misc/vfstrace.c @@ -892,7 +892,7 @@ static int vfstraceOpen( vfstrace_printf(pInfo, "%s.xOpen(%s,flags=0x%x)", pInfo->zVfsName, p->zFName, flags); if( p->pReal->pMethods ){ - sqlite3_io_methods *pNew = sqlite3_malloc( sizeof(*pNew) ); + sqlite3_io_methods *pNew = sqlite3_malloc64( sizeof(*pNew) ); const sqlite3_io_methods *pSub = p->pReal->pMethods; memset(pNew, 0, sizeof(*pNew)); pNew->iVersion = pSub->iVersion; diff --git a/ext/misc/vtablog.c b/ext/misc/vtablog.c index 2b3e30355..a48f3a632 100644 --- a/ext/misc/vtablog.c +++ b/ext/misc/vtablog.c @@ -14,6 +14,8 @@ ** on stdout when its key interfaces are called. This is intended for ** interactive analysis and debugging of virtual table interfaces. ** +** HOW TO COMPILE: +** ** To build this extension as a separately loaded shared library or ** DLL, use compiler command-lines similar to the following: ** @@ -21,7 +23,7 @@ ** (mac) clang -fPIC -dynamiclib vtablog.c -o vtablog.dylib ** (windows) cl vtablog.c -link -dll -out:vtablog.dll ** -** Usage example: +** USAGE EXAMPLE: ** ** .load ./vtablog ** CREATE VIRTUAL TABLE temp.log USING vtablog( @@ -29,6 +31,23 @@ ** rows=25 ** ); ** SELECT * FROM log; +** +** ARGUMENTS TO CREATE VIRTUAL TABLE: +** +** In "CREATE VIRTUAL TABLE temp.log AS vtablog(ARGS....)" statement, the +** ARGS argument is a list of key-value pairs that can be any of the +** following. +** +** schema=TEXT Text is a CREATE TABLE statement that defines +** the schema of the new virtual table. +** +** rows=N The table as N rows. +** +** consume_order_by=N If the left-most ORDER BY terms is ASC and +** against column N (where the leftmost column +** is #1) then set the orderByConsumed=1 flag in +** xBestIndex. Or if the left-most ORDER BY is +** DESC and against column -N, do likewise. */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -49,6 +68,8 @@ struct vtablog_vtab { char *zName; /* Table name. argv[2] of xConnect/xCreate */ int nRow; /* Number of rows in the table */ int nCursor; /* Number of cursors created */ + int iConsumeOB; /* Consume the ORDER BY clause if on column N-th + ** and consumeOB=N or consumeOB=(-N) and DESC */ }; /* vtablog_cursor is a subclass of sqlite3_vtab_cursor which will @@ -180,6 +201,7 @@ static int vtablogConnectCreate( int rc; char *zSchema = 0; char *zNRow = 0; + char *zConsumeOB = 0; printf("%s.%s.%s():\n", argv[1], argv[2], isCreate ? "xCreate" : "xConnect"); @@ -203,6 +225,10 @@ static int vtablogConnectCreate( rc = SQLITE_ERROR; goto vtablog_end_connect; } + if( vtablog_string_parameter(pzErr, "consume_order_by", z, &zConsumeOB) ){ + rc = SQLITE_ERROR; + goto vtablog_end_connect; + } } if( zSchema==0 ){ zSchema = sqlite3_mprintf("%s","CREATE TABLE x(a,b);"); @@ -214,13 +240,17 @@ static int vtablogConnectCreate( printf(" schema = '%s'\n", zSchema); rc = sqlite3_declare_vtab(db, zSchema); if( rc==SQLITE_OK ){ - pNew = sqlite3_malloc( sizeof(*pNew) ); + pNew = sqlite3_malloc64( sizeof(*pNew) ); *ppVtab = (sqlite3_vtab*)pNew; if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); pNew->nRow = 10; if( zNRow ) pNew->nRow = atoi(zNRow); printf(" nrow = %d\n", pNew->nRow); + if( zConsumeOB ) pNew->iConsumeOB = atoi(zConsumeOB); + if( pNew->iConsumeOB ){ + printf(" consume_order_by = %d\n", pNew->iConsumeOB); + } pNew->zDb = sqlite3_mprintf("%s", argv[1]); pNew->zName = sqlite3_mprintf("%s", argv[2]); } @@ -228,6 +258,7 @@ static int vtablogConnectCreate( vtablog_end_connect: sqlite3_free(zSchema); sqlite3_free(zNRow); + sqlite3_free(zConsumeOB); return rc; } static int vtablogCreate( @@ -282,7 +313,7 @@ static int vtablogOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ vtablog_cursor *pCur; printf("%s.%s.xOpen(cursor=%d)\n", pTab->zDb, pTab->zName, ++pTab->nCursor); - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); pCur->iCursor = pTab->nCursor; @@ -514,16 +545,27 @@ static int vtablogBestIndex( } } printf(" nOrderBy: %d\n", p->nOrderBy); - for(i=0; i<p->nOrderBy; i++){ - printf(" orderby[%d]: col=%d desc=%d\n", - i, - p->aOrderBy[i].iColumn, - p->aOrderBy[i].desc); + if( p->nOrderBy ){ + for(i=0; i<p->nOrderBy; i++){ + printf(" orderby[%d]: col=%d desc=%d\n", + i, + p->aOrderBy[i].iColumn, + p->aOrderBy[i].desc); + } + if( pTab->iConsumeOB ){ + int N = p->aOrderBy[0].iColumn+1; + if( (p->aOrderBy[0].desc && N==-pTab->iConsumeOB) + || (!p->aOrderBy[0].desc && N==pTab->iConsumeOB) + ){ + p->orderByConsumed = 1; + } + } } p->estimatedCost = (double)500; p->estimatedRows = 500; printf(" idxNum=%d\n", p->idxNum); printf(" idxStr=NULL\n"); + printf(" sqlite3_vtab_distinct()=%d\n", sqlite3_vtab_distinct(p)); printf(" orderByConsumed=%d\n", p->orderByConsumed); printf(" estimatedCost=%g\n", p->estimatedCost); printf(" estimatedRows=%lld\n", p->estimatedRows); diff --git a/ext/misc/vtshim.c b/ext/misc/vtshim.c index 3f7945724..ed6c568f6 100644 --- a/ext/misc/vtshim.c +++ b/ext/misc/vtshim.c @@ -86,7 +86,7 @@ static int vtshimCreate( } return SQLITE_ERROR; } - pNew = sqlite3_malloc( sizeof(*pNew) ); + pNew = sqlite3_malloc64( sizeof(*pNew) ); *ppVtab = (sqlite3_vtab*)pNew; if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); @@ -125,7 +125,7 @@ static int vtshimConnect( } return SQLITE_ERROR; } - pNew = sqlite3_malloc( sizeof(*pNew) ); + pNew = sqlite3_malloc64( sizeof(*pNew) ); *ppVtab = (sqlite3_vtab*)pNew; if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(*pNew)); @@ -192,7 +192,7 @@ static int vtshimOpen(sqlite3_vtab *pBase, sqlite3_vtab_cursor **ppCursor){ int rc; *ppCursor = 0; if( pAux->bDisposed ) return SQLITE_ERROR; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); rc = pAux->pMod->xOpen(pVtab->pChild, &pCur->pChild); @@ -444,7 +444,7 @@ static int vtshimCopyModule( ){ sqlite3_module *p; if( !pMod || !ppMod ) return SQLITE_ERROR; - p = sqlite3_malloc( sizeof(*p) ); + p = sqlite3_malloc64( sizeof(*p) ); if( p==0 ) return SQLITE_NOMEM; memcpy(p, pMod, sizeof(*p)); *ppMod = p; @@ -464,7 +464,7 @@ void *sqlite3_create_disposable_module( vtshim_aux *pAux; sqlite3_module *pMod; int rc; - pAux = sqlite3_malloc( sizeof(*pAux) ); + pAux = sqlite3_malloc64( sizeof(*pAux) ); if( pAux==0 ){ if( xDestroy ) xDestroy(pClientData); return 0; diff --git a/ext/misc/wholenumber.c b/ext/misc/wholenumber.c index 4c955925d..fe5fc83ab 100644 --- a/ext/misc/wholenumber.c +++ b/ext/misc/wholenumber.c @@ -47,7 +47,7 @@ static int wholenumberConnect( char **pzErr ){ sqlite3_vtab *pNew; - pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + pNew = *ppVtab = sqlite3_malloc64( sizeof(*pNew) ); if( pNew==0 ) return SQLITE_NOMEM; sqlite3_declare_vtab(db, "CREATE TABLE x(value)"); sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); @@ -69,7 +69,7 @@ static int wholenumberDisconnect(sqlite3_vtab *pVtab){ */ static int wholenumberOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ wholenumber_cursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); *ppCursor = &pCur->base; diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index 086b058cc..9b127cc5a 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -456,7 +456,7 @@ static int zipfileDisconnect(sqlite3_vtab *pVtab){ static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){ ZipfileTab *pTab = (ZipfileTab*)p; ZipfileCsr *pCsr; - pCsr = sqlite3_malloc(sizeof(*pCsr)); + pCsr = sqlite3_malloc64(sizeof(*pCsr)); *ppCsr = (sqlite3_vtab_cursor*)pCsr; if( pCsr==0 ){ return SQLITE_NOMEM; @@ -705,7 +705,12 @@ static int zipfileScanExtra(u8 *aExtra, int nExtra, u32 *pmTime){ u8 *p = aExtra; u8 *pEnd = &aExtra[nExtra]; - while( p<pEnd ){ + /* Stop when there are less than 9 bytes left to scan in the buffer. This + ** is because the timestamp field requires exactly 9 bytes - 4 bytes of + ** header fields and 5 bytes of data. If there are less than 9 bytes + ** remaining, either it is some other field or else the extra data + ** is corrupt. Either way, do not process it. */ + while( p+(2*sizeof(u16) + 1 + sizeof(u32))<=pEnd ){ u16 id = zipfileRead16(p); u16 nByte = zipfileRead16(p); @@ -990,7 +995,7 @@ static void zipfileInflate( int nIn, /* Size of buffer aIn[] in bytes */ int nOut /* Expected output size */ ){ - u8 *aRes = sqlite3_malloc(nOut); + u8 *aRes = sqlite3_malloc64(nOut); if( aRes==0 ){ sqlite3_result_error_nomem(pCtx); }else{ @@ -1387,7 +1392,7 @@ static int zipfileBestIndex( static ZipfileEntry *zipfileNewEntry(const char *zPath){ ZipfileEntry *pNew; - pNew = sqlite3_malloc(sizeof(ZipfileEntry)); + pNew = sqlite3_malloc64(sizeof(ZipfileEntry)); if( pNew ){ memset(pNew, 0, sizeof(ZipfileEntry)); pNew->cds.zFile = sqlite3_mprintf("%s", zPath); diff --git a/ext/qrf/README.md b/ext/qrf/README.md new file mode 100644 index 000000000..8555cb078 --- /dev/null +++ b/ext/qrf/README.md @@ -0,0 +1,775 @@ +# SQLite Query Result Formatting Subsystem + +The "Query Result Formatter" or "QRF" subsystem is a C-language +subroutine that formats the output from an SQLite query for display using +a fix-width font, for example on a terminal window over an SSH connection. +The output format is configurable. The application can request various +table formats, with flexible column widths and alignments, row-oriented +formats, such as CSV and similar, as well as various special purpose formats +like JSON. + +For the first 25 years of SQLite's existance, the +[command-line interface](https://sqlite.org/cli.html) (CLI) +formatted query results using a hodge-podge of routines +that had grown slowly by accretion. The QRF was created +in fall of 2025 to refactor and reorganize this code into +a more usable form. The idea behind QRF is to implement all the +query result formatting capabilities of the CLI in a subroutine +that can be incorporated and reused by other applications. + +## 1.0 Overview Of Operation + +Suppose variable `sqlite3_stmt *pStmt` is a pointer to an SQLite +prepared statement that has been reset and bound and is ready to run. +Then to format the output from this prepared statement, use code +similar to the following: + +> ~~~ +sqlite3_qrf_spec spec; /* Format specification */ +char *zErrMsg; /* Text error message (optional) */ +char *zResult = 0; /* Formatted output written here */ +int rc; /* Result code */ + +memset(&spec, 0, sizeof(spec)); /* Initialize the spec */ +spec.iVersion = 1; /* Version number must be 1 */ +spec.pzOutput = &zResult; /* Write results in variable zResult */ +/* Optionally fill in other settings in spec here, as needed */ +zErrMsg = 0; /* Not required; just being pedantic */ +rc = sqlite3_format_query_result(pStmt, &spec, &zErrMsg); /* Format results */ +if( rc ){ + printf("Error (%d): %s\n", rc, zErrMsg); /* Report an error */ + sqlite3_free(zErrMsg); /* Free the error message text */ +}else{ + printf("%s", zResult); /* Report the results */ +} +sqlite3_free(zResult); /* Free memory used to hold results */ +~~~ + +The `sqlite3_qrf_spec` object describes the desired output format +and where to send the generated output. Most of the work in using +the QRF involves filling out the sqlite3_qrf_spec. + +### 1.1 Using QRF with SQL text + +If you start with SQL text instead of an sqlite3_stmt pointer, and +especially if the SQL text might comprise two or more statements, then +the SQL text needs to be converted into sqlite3_stmt objects separately. +If the original SQL text is in a variable `const char *zSql` and the +database connection is in variable `sqlite3 *db`, then code +similar to the following should work: + +> ~~~ +sqlite3_qrf_spec spec; /* Format specification */ +char *zErrMsg; /* Text error message (optional) */ +char *zResult = 0; /* Formatted output written here */ +sqlite3_stmt *pStmt; /* Next prepared statement */ +int rc; /* Result code */ + +memset(&spec, 0, sizeof(spec)); /* Initialize the spec */ +spec.iVersion = 1; /* Version number must be 1 */ +spec.pzOutput = &zResult; /* Write results in variable zResult */ +/* Optionally fill in other settings in spec here, as needed */ +zErrMsg = 0; /* Not required; just being pedantic */ +while( zSql && zSql[0] ){ + pStmt = 0; /* Not required; just being pedantic */ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zSql); + if( rc!=SQLITE_OK ){ + printf("Error: %s\n", sqlite3_errmsg(db)); + }else{ + rc = sqlite3_format_query_result(pStmt, &spec, &zErrMsg); /* Get results */ + if( rc ){ + printf("Error (%d): %s\n", rc, zErrMsg); /* Report an error */ + sqlite3_free(zErrMsg); /* Free the error message text */ + }else{ + printf("%s", zResult); /* Report the results */ + sqlite3_free(zResult); /* Free memory used to hold results */ + zResult = 0; + } + } + sqlite3_finalize(pStmt); +} +~~~ + +<a id="spec"></a> +## 2.0 The `sqlite3_qrf_spec` object + +The `sqlite3_qrf_spec` looks like this: + +> ~~~ +typedef struct sqlite3_qrf_spec sqlite3_qrf_spec; +struct sqlite3_qrf_spec { + unsigned char iVersion; /* Version number of this structure */ + unsigned char eStyle; /* Formatting style. "box", "csv", etc... */ + unsigned char eEsc; /* How to escape control characters in text */ + unsigned char eText; /* Quoting style for text */ + unsigned char eTitle; /* Quating style for the text of column names */ + unsigned char eBlob; /* Quoting style for BLOBs */ + unsigned char bTitles; /* True to show column names */ + unsigned char bWordWrap; /* Try to wrap on word boundaries */ + unsigned char bTextJsonb; /* Render JSONB blobs as JSON text */ + unsigned char eDfltAlign; /* Default alignment, no covered by aAlignment */ + unsigned char eTitleAlign; /* Alignment for column headers */ + unsigned char bSplitColumn; /* Wrap single-column output into many columns */ + unsigned char bBorder; /* Show outer border in Box and Table styles */ + short int nWrap; /* Wrap columns wider than this */ + short int nScreenWidth; /* Maximum overall table width */ + short int nLineLimit; /* Maximum number of lines for any row */ + short int nTitleLimit; /* Maximum number of characters in a title */ + unsigned int nMultiInsert; /* Add rows to one INSERT until size exceeds */ + int nCharLimit; /* Maximum number of characters in a cell */ + int nWidth; /* Number of entries in aWidth[] */ + int nAlign; /* Number of entries in aAlignment[] */ + short int *aWidth; /* Column widths */ + unsigned char *aAlign; /* Column alignments */ + char *zColumnSep; /* Alternative column separator */ + char *zRowSep; /* Alternative row separator */ + char *zTableName; /* Output table name */ + char *zNull; /* Rendering of NULL */ + char *(*xRender)(void*,sqlite3_value*); /* Render a value */ + int (*xWrite)(void*,const char*,sqlite3_int64); /* Write output */ + void *pRenderArg; /* First argument to the xRender callback */ + void *pWriteArg; /* First argument to the xWrite callback */ + char **pzOutput; /* Storage location for output string */ + /* Additional fields may be added in the future */ +}; +~~~ + +Do not be alarmed by the complexity of this structure. Everything can +be zeroed except for: + + * `.iVersion` + * One of `.pzOutput` or `.xWrite`. + +You do not need to understand and configure every field of this object +in order to use QRF effectively. Start by zeroing out the whole structure, +then initializing iVersion and one of pzOutput or xWrite. Then maybe +tweak one or two other settings to get the output you want. + +Further detail on the meanings of each of the fields in the +`sqlite3_qrf_spec` object is in the subsequent sections. + +### 2.1 Structure Version Number + +The sqlite3_qrf_spec.iVersion field must be 1. Future enhancements to +the QRF might add new fields to the bottom of the sqlite3_qrf_spec +object. Those new fields will only be accessible if the iVersion is greater +than 1. Thus the iVersion field is used to support upgradability. + +### 2.2 Output Deposition (xWrite and pzOutput) + +The formatted output can either be sent to a callback function +or accumulated into an output buffer in memory obtained +from sqlite3_malloc(). If the sqlite3_qrf_spec.xWrite column is not NULL, +then that function is invoked (using sqlite3_qrf_spec.xWriteArg as its +first argument) to transmit the formatted output. Or, if +sqlite3_qrf_spec.pzOutput points to a pointer to a character, then that +pointer is made to point to memory obtained from sqlite3_malloc() that +contains the complete text of the formatted output. If spec.pzOutput\[0\] +is initially non-NULL, then it is assumed to already point to memory obtained +from sqlite3_malloc(). In that case, the buffer is resized using +sqlite3_realloc() and the new text is appended. + +One of either sqlite3_qrf_spec.xWrite and sqlite3_qrf_spec.pzOutput must be +non-NULL and the other must be NULL. + +The return value from xWrite is an SQLITE result code. The usual return +should be SQLITE_OK. But if for some reason the write fails, a different +value might be returned. + +### 2.3 Output Format + +The sqlite3_qrf_spec.eStyle field is an integer code that defines the +specific output format that will be generated. See [section 4.0](#style) +below for details on the meaning of the various style options. + +Other fields in sqlite3_qrf_spec might be used or might be +ignored, depending on the value of eStyle. + +### 2.4 Show Column Names (bTitles) + +The sqlite3_qrf_spec.bTitles field can be either QRF_SW_Auto, +QRF_SW_On, or QRF_SW_Off. Those three constants also have shorter +alternative spellings: QRF_Auto, QRF_No, and +QRF_Yes. + +> ~~~ +#define QRF_SW_Auto 0 /* Let QRF choose the best value */ +#define QRF_SW_Off 1 /* This setting is forced off */ +#define QRF_SW_On 2 /* This setting is forced on */ +#define QRF_Auto 0 /* Alternate spelling for QRF_SW_Auto and others */ +#define QRF_No 1 /* Alternate spelling for QRF_SW_Off */ +#define QRF_Yes 2 /* Alternate spelling for QRF_SW_On */ +~~~ + +If the value is QRF_Yes, then column names appear in the output. +If the value is QRF_No, column names are omitted. If the +value is QRF_Auto, then an appropriate default is chosen. + +### 2.5 Control Character Escapes (eEsc) + +The sqlite3_qrf_spec.eEsc determines how ASCII control characters are +formatted when displaying TEXT values in the result. These are the allowed +values: + +> ~~~ +#define QRF_ESC_Auto 0 /* Choose the ctrl-char escape automatically */ +#define QRF_ESC_Off 1 /* Do not escape control characters */ +#define QRF_ESC_Ascii 2 /* Unix-style escapes. Ex: U+0007 shows ^G */ +#define QRF_ESC_Symbol 3 /* Unicode escapes. Ex: U+0007 shows U+2407 */ +~~~ + +If the value of eEsc is QRF_ESC_Ascii, then the control character +with value X is displayed as ^Y where Y is X+0x40. Hence, a +backspace character (U+0008) is shown as "^H". + +If eEsc is QRF_ESC_Symbol, then control characters in the range of U+0001 +through U+001f are mapped into U+2401 through U+241f, respectively. + +If the value of eEsc is QRF_ESC_Off, then no translation occurs +and control characters that appear in TEXT strings are transmitted +to the formatted output as-is. This can be dangerous in applications, +since an adversary who can control TEXT values might be able to +inject ANSI cursor movement sequences to hide nefarious values. + +The QRF_ESC_Auto value for eEsc means that the query result formatter +gets to pick whichever control-character encoding it thinks is best for +the situation. This will usually be QRF_ESC_Ascii. + +The TAB (U+0009), LF (U+000a) and CR-LF (U+000d,U+000a) character +sequence are always output literally and are not mapped to alternative +display values, regardless of this setting. + +### 2.6 Display of TEXT values (eText, eTitle) + +The sqlite3_qrf_spec.eText controls how text values are rendered in the +display. sqlite3_qrf_spec.eTitle controls how column names are rendered. +Both fields can have one of the following values: + +> ~~~ +#define QRF_TEXT_Auto 0 /* Choose text encoding automatically */ +#define QRF_TEXT_Plain 1 /* Literal text */ +#define QRF_TEXT_Sql 2 /* Quote as an SQL literal */ +#define QRF_TEXT_Csv 3 /* CSV-style quoting */ +#define QRF_TEXT_Html 4 /* HTML-style quoting */ +#define QRF_TEXT_Tcl 5 /* C/Tcl quoting */ +#define QRF_TEXT_Json 6 /* JSON quoting */ +#define QRF_TEXT_Relaxed 7 /* Relaxed SQL quoting */ +~~~ + +A value of QRF_TEXT_Auto means that the query result formatter will choose +what it thinks will be the best text encoding. + +A value of QRF_TEXT_Plain means that text values appear in the output exactly +as they are found in the database file, with no translation. + +A value of QRF_TEXT_Sql means that text values are escaped so that they +look like SQL literals. That means the value will be surrounded by +single-quotes (U+0027) and any single-quotes contained within the text +will be doubled. + +QRF_TEXT_Relaxed is similar to QRF_TEXT_Sql, except that it automatically +reverts to QRF_TEXT_Plain if the value to be displayed does not contain +special characters and is not easily confused with a NULL or a numeric +value. QRF_TEXT_Relaxed strives to minimize the amount of quoting syntax +while keeping the result unambiguous and easy for humans to read. The +precise rules for when quoting is omitted in QRF_TEXT_Relaxed, and when +it is applied, might be adjusted in future releases. + +A value of QRF_TEXT_Csv means that text values are escaped in accordance +with RFC&nbsp;4180, which defines Comma-Separated-Value or CSV files. +Text strings that contain no special values appears as-is. Text strings +that contain special values are contained in double-quotes (U+0022) and +any double-quotes within the value are doubled. + +A value of QRF_TEXT_Html means that text values are escaped for use in +HTML. Special characters "&lt;", "&amp;", "&gt;", "&quot;", and "&#39;" +are displayed as "&amp;lt;", "&amp;amp;", "&amp;gt;", "&amp;quot;", +and "&amp;#39;", respectively. + +A value of QRF_TEXT_Tcl means that text values are displayed inside of +double-quotes and special characters within the string are escaped using +backslash escape, as in ANSI-C or TCL or Perl or other popular programming +languages. + +A value of QRF_TEXT_Json gives similar results as QRF_TEXT_Tcl except that the +rules are adjusted so that the displayed string is strictly conforming +the JSON specification. + +### 2.7 How to display BLOB values (eBlob and bTextJsonb) + +If the sqlite3_qrf_spec.bTextJsonb flag is QRF_SW_On and if the value to be +displayed is JSONB, then the JSONB is translated into text JSON and the +text is shown according to the sqlite3_qrf_spec.eText setting as +described in the previous section. + +If the bTextJsonb flag is QRF_SW_Off (the usual case) or if the BLOB value to +be displayed is not JSONB, then the sqlite3_qrf_spec.eBlob field determines +how the BLOB value is formatted. The following options are available; + +> ~~~ +#define QRF_BLOB_Auto 0 /* Determine BLOB quoting using eText */ +#define QRF_BLOB_Text 1 /* Display content exactly as it is */ +#define QRF_BLOB_Sql 2 /* Quote as an SQL literal */ +#define QRF_BLOB_Hex 3 /* Hexadecimal representation */ +#define QRF_BLOB_Tcl 4 /* "\000" notation */ +#define QRF_BLOB_Json 5 /* A JSON string */ +#define QRF_BLOB_Size 6 /* Display the blob size only */ +~~~ + +A value of QRF_BLOB_Auto means that display format is selected automatically +by sqlite3_format_query_result() based on eStyle and eText. + +A value of QRF_BLOB_Text means that BLOB values are interpreted as UTF8 +text and are displayed using formatting results set by eEsc and +eText. + +A value of QRF_BLOB_Sql means that BLOB values are shown as SQL BLOB +literals: a prefix "`x'`" following by hexadecimal and ending with a +final "`'`". + +A value of QRF_BLOB_Hex means that BLOB values are shown as +hexadecimal text with no delimiters. + +A value of QRF_BLOB_Tcl means that BLOB values are shown as a +C/Tcl/Perl string literal where every byte is an octal backslash +escape. So a BLOB of `x'052881f3'` would be displayed as +`"\005\050\201\363"`. + +A value of QRF_BLOB_Json is similar to QRF_BLOB_Tcl except that is +uses unicode backslash escapes, since JSON does not understand +the C/Tcl/Perl octal backslash escapes. So the string from the +previous paragraph would be shown as +`"\u0005\u0028\u0081\u00f3"`. + +A value of QRF_BLOB_Size does not show any BLOB content at all. +Instead, it substitutes a text string that says how many bytes +the BLOB contains. + +### 2.8 Maximum size of displayed content (nLineLimit, nCharLimit, nTitleLimit) + +If the sqlite3_qrf_spec.nCharLimit setting is non-zero, then the formatter +will display only the first nCharLimit characters of each value. +Only characters that take up space are counted when enforcing this +limit. Zero-width characters and VT100 escape sequences do not count +toward this limit. The count is in characters, not bytes. When +imposing this limit, the formatter adds the three characters "..." +to the end of the value. Those added characters are not counted +as part of the limit. Very small limits still result in truncation, +but might render a few more characters than the limit. + +If the sqlite3_qrf_spec.nLineLimit setting is non-zero, then the +formatter will only display the first nLineLimit lines of each value. +It does not matter if the value is split because it contains a newline +character, or if it split by wrapping. This setting merely limits +the number of displayed lines. The nLineLimit setting currently only +works for **Box**, **Column**, **Line**, **Markdown**, and **Table** +styles, though that limitation might change in future releases. + +The idea behind both of these settings is to prevent large renderings +when doing a query that (unexpectedly) contains very large text or +blob values: perhaps megabyes of text. + +If the sqlite3_qrf_spec.nTitleLimit is non-zero, then the formatter +attempts to limits the size of column titles to at most nTitleLimit +display characters in width and a single line of text. The nTitleLimit +is useful for queries that have result columns that are scalar +subqueries or complex expressions. If those columns lack an AS +clause, then the name of the column will be a copy of the expression +that defines the column, which in some queries can be hundreds of +characters and multiple lines in length, which can reduce the readability +of tabular displays. An nTitleLimit somewhere in the range of 10 to 20. +can improve readability. The nTitleLimit setting currently only +works for **Box**, **Column**, **Line**, **Markdown**, and **Table** +styles, though that limitation might change in future releases. + +### 2.9 Multiple Tuples Per INSERT In QRF_STYLE_Insert (nMultiInsert) + +If the sqlite3_qrf_spec.nMultiInsert value is positive, then the +QRF_STYLE_Insert output mode will generate multiple tuples in +each INSERT statement until the total number of bytes in the +statement exceeds nMultiInsert. A value of a few thousand is +recommended here, in order to generate SQL output that is parsed +and inserted at maximum speed by SQLite. + +### 2.10 Word Wrapping In Columnar Styles (nWrap, bWordWrap) + +When using columnar formatting modes (QRF_STYLE_Box, QRF_STYLE_Column, +QRF_STYLE_Markdown, or QRF_STYLE_Table), the formatter attempts to limit +the width of any individual column to sqlite3_qrf_spec.nWrap characters +if nWrap is non-zero. A zero value for nWrap means "unlimited". +The nWrap limit might be exceeded if the limit is very small. + +In order to keep individual columns within requested width limits, +it is sometimes necessary to wrap the content for a single row of +a single column across multiple lines. When this +becomes necessary and if the bWordWrap setting is QRF_Yes, then the +formatter attempts to split the content on whitespace or at a word boundary. +If bWordWrap is QRF_No, then the formatter is free to split content +anywhere, including in the middle of a word. + +For narrow columns and wide words, it might sometimes be necessary to split +a column in the middle of a word, even when bWordWrap is QRF_Yes. + +### 2.11 Helping The Output To Fit On The Terminal (nScreenWidth) + +The sqlite3_qrf_spec.nScreenWidth field can be set the number of +characters that will fit on one line on the viewer output device. +This is typically a number like 80 or 132. The formatter will attempt +to reduce the length of output lines, depending on the style, so +that all output fits on that screen. + +A value of zero for nScreenWidth means "unknown" or "no width limit". +When the value is zero, the formatter makes no attempt to keep the +lines of output short. + +The nScreenWidth is a hint to the formatter, not a requirement. +The formatter trieds to keep lines below the nScreenWidth limit, +but it does not guarantee that it will. + +The nScreenWidth field currently only makes a difference in +columnar styles (**Box**, **Column**, **Markdown**, and **Table**) +and in the **Line** style. + +### 2.12 Individual Column Width (nWidth and aWidth) + +The sqlite3_qrf_spec.aWidth field is a pointer to an array of +signed 16-bit integers that control the width of individual columns +in columnar output modes (QRF_STYLE_Box, QRF_STYLE_Column, +QRF_STYLE_Markdown, or QRF_STYLE_Table). The sqlite3_qrf_spec.nWidth +field is the number of integers in the aWidth array. + +If aWidth is a NULL pointer or if nWidth is zero, then the array is +assumed to be all zeros. If nWidth is less then the number of +columns in the output, then zero is used for the width +for all columns past then end of the aWidth array. + +The aWidth array is deliberately an array of 16-bit signed integers. +Only 16 bits are used because no good comes for having very large +column widths. The range if further restricted as follows: + +> ~~~ +#define QRF_MAX_WIDTH 10000 /* Maximum column width */ +#define QRF_MIN_WIDTH 0 /* Minimum column width */ +~~~ + +A width greater than then QRF_MAX_WIDTH is interpreted as QRF_MAX_WIDTH. + +Any aWidth\[\] value of zero means the formatter should use a flexible +width column (limited only by sqlite_qrf_spec.mxWidth) that is just +big enough to hold the largest row. + +For historical compatibility, aWidth\[\] can contain negative values, +down to -QRF_MAX_WIDTH. The column width used is the absolute value +of the number in aWidth\[\]. The only difference is that negative +values cause the default horizontal alignment to be QRF_ALIGN_Right. +The sign of the aWidth\[\] values only affects alignment if the +alignment is not otherwise specified by aAlign\[\] or eDfltAlign. +Again, negative values for aWidth\[\] entries are supported for +backwards compatibility only, and are not recommended for new +applications. + +### 2.13 Alignment (nAlignment, aAlignment, eDfltAlign, eTitleAlign) + +Some cells in a display table might contain a lot of text and thus +be wide, or they might contain newline characters or be wrapped by +width constraints so that they span many rows of text. Other cells +might be narrower and shorter. In columnar formats, the display width +of a cell is the maximum of the widest value in the same column, and the +display height is the height of the tallest value in the same row. +So some cells might be much taller and wider than necessary to hold +their values. + +Alignment determines where smaller values are placed within larger cells. + +The sqlite3_qrf_spec.aAlign field points to an array of unsigned characters +that specifies alignment (both vertical and horizontal) of individual +columns within the table. The sqlite3_qrf_spec.nAlign fields holds +the number of entries in the aAlign\[\] array. + +If sqlite3_qrf_spec.aAlign is a NULL pointer or if sqlite3_qrf_spec.nAlign +is zero, or for columns to the right of what are specified by +sqlite3_qrf_spec.nAlign, the sqlite3_qrf_spec.eDfltAlign value is used +for the alignment. Column names can be (and often are) aligned +differently, as specified by sqlite3_qrf_spec.eTitleAlign. + +Each alignment value specifies both vertical and horizontal alignment. +Horizontal alignment can be left, center, right, or no preference. +Vertical alignment can be top, middle, bottom, or no preference. +Thus there are 16 possible alignment values, as follows: + +> ~~~ +/* +** Horizontal Vertial +** ---------- -------- */ +#define QRF_ALIGN_Auto 0 /* auto auto */ +#define QRF_ALIGN_Left 1 /* left auto */ +#define QRF_ALIGN_Center 2 /* center auto */ +#define QRF_ALIGN_Right 3 /* right auto */ +#define QRF_ALIGN_Top 4 /* auto top */ +#define QRF_ALIGN_NW 5 /* left top */ +#define QRF_ALIGN_N 6 /* center top */ +#define QRF_ALIGN_NE 7 /* right top */ +#define QRF_ALIGN_Middle 8 /* auto middle */ +#define QRF_ALIGN_W 9 /* left middle */ +#define QRF_ALIGN_C 10 /* center middle */ +#define QRF_ALIGN_E 11 /* right middle */ +#define QRF_ALIGN_Bottom 12 /* auto bottom */ +#define QRF_ALIGN_SW 13 /* left bottom */ +#define QRF_ALIGN_S 14 /* center bottom */ +#define QRF_ALIGN_SE 15 /* right bottom */ +~~~ + +Notice how alignment values with an unspecified horizontal +or vertical component can be added to another alignment value +for which that component is specified, to get a fully +specified alignment. For eample: + +> QRF_ALIGN_Center + QRF_ALIGN_Bottom == QRF_ALIGN_S. + +The alignment for column names is always determined by the +eTitleAlign setting. If eTitleAlign is QRF_Auto, then column +names use center-bottom alignment, QRF_ALIGN_W, value 14. +The aAlign\[\] and eDfltAlign settings have no affect on +column names. + +For data in the first nAlign columns, the aAlign\[\] array +entry for that column takes precedence. If either the horizontal +or vertical alignment has an "auto" value for that column or if +a column is beyond the first nAlign entries, then eDfltAlign +is used as a backup. If neither aAlign\[\] nor eDfltAlign +specify a horizontal alignment, then values are right-aligned +(QRF_ALIGN_Right) if they are numeric and left-aligned +(QRF_ALIGN_Left) otherwise. If neither aAlign\[\] nor eDfltAlign +specify a vertical alignment, then values are top-aligned +(QRF_ALIGN_Top). + +*As of 2025-11-08, only horizontal alignment is implemented. +The vertical alignment settings are currently ignored and +the vertical alignment is always QRF_ALIGN_Top.* + +### 2.14 Row and Column Separator Strings + +The sqlite3_qrf_spec.zColumnSep and sqlite3_qrf_spec.zRowSep strings +are alternative column and row separator character sequences. If not +specified (if these pointers are left as NULL) then appropriate defaults +are used. Some output styles have hard-coded column and row separators +and these settings are ignored for those styles. + +### 2.15 The Output Table Name + +The sqlite3_qrf_spec.zTableName value is the name of the output table +when eStyle is QRF_STYLE_Insert. + +### 2.16 The Rendering Of NULL (zNull) + +If a value is NULL then show the NULL using the string +found in sqlite3_qrf_spec.zNull. If zNull is itself a NULL pointer +then NULL values are rendered as an empty string. + +### 2.17 Optional Value Rendering Callback + +If the sqlite3_qrf_spec.xRender field is not NULL, then each +sqlite3_value coming out of the query is first passed to the +xRender function, giving that function an opportunity to render +the results itself, using whatever custom format is desired. +If xRender chooses to render, it should write the rendering +into memory obtained from sqlite3_malloc() and return a pointer +to that memory. The xRender function can decline +to render (for example, based on the sqlite3_value_type() or other +characteristics of the value) in which case it can simply return a +NULL pointer and the usual default rendering will be used instead. + +The sqlite3_format_query_result() function (which calls xRender) +will take responsibility for freeing the string returned by xRender +after it has finished using it. + +The eText, eBlob, and eEsc settings above become no-ops if the xRender +routine returns non-NULL. In other words, the application-supplied +xRender routine is expected to do all of its own quoting and formatting. + +The xRender routine is expected to do character length limiting itself. +So the nCharLimit setting becomes a no-op if xRender is used. However +the nLineLimit setting is still applied. The nTitleLimit setting is +not applicable to xRender because title values come from the +sqlite3_column_name() interface not from sqlite3_column_value(), +and so that names of columns are never processed by xRender. + +## 3.0 The `sqlite3_format_query_result()` Interface + +Invoke the `sqlite3_format_query_result(P,S,E)` interface to run +the prepared statement P and format its results according to the +specification found in S. The sqlite3_format_query_result() function +will return an SQLite result code, usually SQLITE_OK, but perhaps +SQLITE_NOMEM or SQLITE_ERROR or similar. If an error occurs and if +the E parameter is not NULL, then error message text might be written +into *E. Any error message text will be stored in memory obtained +from sqlite3_malloc() and it is the responsibility of the caller to +free that memory by a subsequent call to sqlite3_free(). + +<a id="style"></a> +## 4.0 Output Styles + +The result formatter supports a variety of output styles. The +output style (sometimes called "output mode") is determined by +the eStyle field of the sqlite3_qrf_spec object. The set of +supported output modes might increase in future versions. +The following output modes are currently defined: + +> ~~~ +#define QRF_STYLE_Auto 0 /* Choose a style automatically */ +#define QRF_STYLE_Box 1 /* Unicode box-drawing characters */ +#define QRF_STYLE_Column 2 /* One record per line in neat columns */ +#define QRF_STYLE_Count 3 /* Output only a count of the rows of output */ +#define QRF_STYLE_Csv 4 /* Comma-separated-value */ +#define QRF_STYLE_Eqp 5 /* Format EXPLAIN QUERY PLAN output */ +#define QRF_STYLE_Explain 6 /* EXPLAIN output */ +#define QRF_STYLE_Html 7 /* Generate an XHTML table */ +#define QRF_STYLE_Insert 8 /* Generate SQL "insert" statements */ +#define QRF_STYLE_Json 9 /* Output is a list of JSON objects */ +#define QRF_STYLE_JObject 10 /* Independent JSON objects for each row */ +#define QRF_STYLE_Line 11 /* One column per line. */ +#define QRF_STYLE_List 12 /* One record per line with a separator */ +#define QRF_STYLE_Markdown 13 /* Markdown formatting */ +#define QRF_STYLE_Off 14 /* No query output shown */ +#define QRF_STYLE_Quote 15 /* SQL-quoted, comma-separated */ +#define QRF_STYLE_Stats 16 /* EQP-like output but with performance stats */ +#define QRF_STYLE_StatsEst 17 /* EQP-like output with planner estimates */ +#define QRF_STYLE_StatsVm 18 /* EXPLAIN-like output with performance stats */ +#define QRF_STYLE_Table 19 /* MySQL-style table formatting */ +~~~ + +In the following subsections, these styles will often be referred +to without the "QRF_STYLE_" prefix. + +### 4.1 Default Style (Auto) + +The **Auto** style means QRF gets to choose an appropriate output +style. It will usually choose **Box**, but might also pick one of +**Explain** or **Eqp** if the `sqlite3_stmt_explain()` function +returns 1 or 2, respectively. + +### 4.2 Columnar Styles (Box, Column, Markdown, Table) + +The **Box**, **Column**, **Markdown**, and **Table** +modes are columnar. This means the output is arranged into neat, +uniform-width columns. These styles can use more memory, especially when +the query result has many rows, because they need to load the entire output +into memory first in order to determine how wide to make each column. + +The nWidth, aWidth, and mxWidth fields of the `sqlite3_qrf_spec` object +are used by these styles only, and are ignored by all other styles. +The zRowSep and zColumnSep settings are ignored by these styles. The +bTitles setting is honored by these styles; it defaults to QRF_SW_On. + +The **Box** style uses Unicode box-drawing character to draw a grid +of columns and rows to show the result. The **Table** is the same, +except that it uses ASCII-art rather than Unicode box-drawing characters +to draw the grid. The **Column** arranges the results in neat columns +but does not draw in column or row separator, except that it does draw +lines horizontal lines using "`-`" characters to separate the column names +from the data below. This is very similar to default output styling in +psql. The **Markdown** renders its result in the Markdown table format. + +The **Box** and **Table** styles normally have a border that surrounds +the entire result. However, if sqlite3_qrf_spec.bBorder is QRF_No, then +that border is omitted, saving a little space both horizontally and +vertically. + +#### 4.2.1 Split Column Mode + +If the bSplitColumn field is QRF_Yes, and eStyle is QRF_STYLE_Column, +and bTitles is QRF_No, and nScreenWidth is greater than zero, and if +the query only returns a single column, then a special rendering known +as "Split Column Mode" will be used. In split column mode, instead +of showing all results in one tall column, the content wraps vertically +so that it appears on the screen as multiple columns, as many as will +fit in the available screen width. + +### 4.3 Line-oriented Styles + +The line-oriented styles output each row of result as it is received from +the prepared statement. + +The **List** style is the most familiar line-oriented output format. +The **List** style shows output columns for each row on the +same line, each separated by a single "`|`" character and with lines +terminated by a single newline (\\u000a or \\n). These column +and row separator choices can be overridden using the zColumnSep +and zRowSep fields of the `sqlite3_qrf_spec` structure. The text +formatting is QRF_TEXT_Plain, and BLOB encoding is QRF_BLOB_Text. So +characters appear in the output exactly as they appear in the database. +Except the eEsp mode defaults to `QRF_ESC_On`, so that control +characters are escaped, for safety. + +The **Csv** and **Quote** styles are simply variations on **List** +with hard-coded values for some of the sqlite3_qrf_spec settings: + +<table border=1 cellpadding=2 cellspacing=0> +<tr><th>&nbsp;<th>Quote<th>Csv +<tr><td>zColumnSep<td>","<td>"," +<tr><td>zRowSep<td>"\\n"<td>"\\r\\n" +<tr><td>zNull<td>"NULL"<td>"" +<tr><td>eText<td>QRF_TEXT_Sql<td>QRF_TEXT_Csv +<tr><td>eBlob<td>QRF_BLOB_Sql<td>QRF_BLOB_Text +</table> + +The **Html** style generates HTML table content, just without +the `<TABLE>..</TABLE>` around the outside. + +The **Insert** style generates a series of SQL "INSERT" statements +that will inserts the data that is output into a table whose name is defined +by the zTableName field of `sqlite3_qrf_spec`. If zTableName is NULL, +then a substitute name is used. If nMultiInsert is positive, then the +output will add multiple rows to each INSERT statement until the size +of the INSERT statement exceeds nMultiInsert bytes before starting +a new INSERT statement. + +The **Json** and **JObject** styles generates JSON text for the query result. +The **Json** style produces a JSON array of structures with one +structure per row. **JObject** outputs independent JSON objects, one per +row, with each structure on a separate line all by itself, and not +part of a larger array. In both cases, the labels on the elements of the +JSON objects are taken from the column names of the SQL query. So if +you have an SQL query that has two or more output columns with the same +name, you will end up with JSON structures that have duplicate elements. + +Finally, the **Line** style paints each column of a row on a +separate line with the column name on the left and a "`=`" separating the +column name from its value. A single blank line appears between rows. + +### 4.4 EXPLAIN Styles (Eqp, Explain) + +The **Eqp** and **Explain** styles format output for +EXPLAIN QUERY PLAN and EXPLAIN statements, respectively. If the input +statement is not already an EXPLAIN QUERY PLAN or EXPLAIN statement is +is temporarily converted for the duration of the rendering, but +is converted back before `sqlite3_format_query_result()` returns. + +### 4.5 ScanStatus Styles (Stats, StatsEst, StatsVm) + +The **Stats**, **StatsEst**, and **StatsVm** styles are similar to **Eqp** +and **Explain** except that they include profiling information +from prior executions of the input prepared statement. +These modes only work if SQLite has been compiled with +-DSQLITE_ENABLE_STMT_SCANSTATUS and if the SQLITE_DBCONFIG_STMT_SCANSTATUS +is enabled for the database connection. The **StatsVm** style +also requires the bytecode() virtual table which is enabled using +the -DSQLITE_ENABLE_BYTECODE_VTAB compile-time option. + +### 4.6 Other Styles (Count, Off) + +The **Count** style discards all query results and returns +a count of the number of rows of output at the end. The **Off** +style is completely silent; it generates no output. These corner-case +modes are sometimes useful for debugging. + +### 5.0 Source Code Files + +The SQLite Query Result Formatter is implemented in three source code files: + + * `qrf.c` &rarr; The implementation, written in portable C99 + * `qrf.h` &rarr; A header file defining interfaces + * `README.md` &rarr; This documentation + +To use the SQLite result formatter, include the "`qrf.h`" header file +and link the application against the "`qrf.c`" source file. diff --git a/ext/qrf/dev-notes.md b/ext/qrf/dev-notes.md new file mode 100644 index 000000000..a46aada83 --- /dev/null +++ b/ext/qrf/dev-notes.md @@ -0,0 +1,14 @@ +# Developer Notes + +## Measuring Test Coverage On Linux + +On Mint Linux, as of 2025-12-02: + +> ~~~ +./configure --dev CFLAGS='-O0 -g -fprofile-arcs -ftest-coverage' +make clean testfixture +./testfixture test/qrf*.test +gcov -b -c testfixture-tclsqlite-ex.c +~~~ + +View results in tclsqlite-ex.c.gcov diff --git a/ext/qrf/qrf.c b/ext/qrf/qrf.c new file mode 100644 index 000000000..4d559581c --- /dev/null +++ b/ext/qrf/qrf.c @@ -0,0 +1,3007 @@ +/* +** 2025-10-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Implementation of the Query Result-Format or "qrf" utility library for +** SQLite. See the README.md documentation for additional information. +*/ +#ifndef SQLITE_QRF_H +#include "qrf.h" +#endif +#include <string.h> +#include <assert.h> +#include <stdint.h> + +#ifndef SQLITE_AMALGAMATION +typedef sqlite3_int64 i64; +#endif + +/* A single line in the EQP output */ +typedef struct qrfEQPGraphRow qrfEQPGraphRow; +struct qrfEQPGraphRow { + int iEqpId; /* ID for this row */ + int iParentId; /* ID of the parent row */ + qrfEQPGraphRow *pNext; /* Next row in sequence */ + char zText[1]; /* Text to display for this row */ +}; + +/* All EQP output is collected into an instance of the following */ +typedef struct qrfEQPGraph qrfEQPGraph; +struct qrfEQPGraph { + qrfEQPGraphRow *pRow; /* Linked list of all rows of the EQP output */ + qrfEQPGraphRow *pLast; /* Last element of the pRow list */ + int nWidth; /* Width of the graph */ + char zPrefix[400]; /* Graph prefix */ +}; + +/* +** Private state information. Subject to change from one release to the +** next. +*/ +typedef struct Qrf Qrf; +struct Qrf { + sqlite3_stmt *pStmt; /* The statement whose output is to be rendered */ + sqlite3 *db; /* The corresponding database connection */ + sqlite3_stmt *pJTrans; /* JSONB to JSON translator statement */ + char **pzErr; /* Write error message here, if not NULL */ + sqlite3_str *pOut; /* Accumulated output */ + int iErr; /* Error code */ + int nCol; /* Number of output columns */ + int expMode; /* Original sqlite3_stmt_isexplain() plus 1 */ + int mxWidth; /* Screen width */ + int mxHeight; /* nLineLimit */ + union { + struct { /* Content for QRF_STYLE_Line */ + int mxColWth; /* Maximum display width of any column */ + char **azCol; /* Names of output columns (MODE_Line) */ + } sLine; + qrfEQPGraph *pGraph; /* EQP graph (Eqp, Stats, and StatsEst) */ + struct { /* Content for QRF_STYLE_Explain */ + int nIndent; /* Slots allocated for aiIndent */ + int iIndent; /* Current slot */ + int *aiIndent; /* Indentation for each opcode */ + } sExpln; + unsigned int nIns; /* Bytes used for current INSERT stmt */ + } u; + sqlite3_int64 nRow; /* Number of rows handled so far */ + int *actualWidth; /* Actual width of each column */ + sqlite3_qrf_spec spec; /* Copy of the original spec */ +}; + +/* +** Data for substitute ctype.h functions. Used for x-platform +** consistency and so that '_' is counted as an alphabetic +** character. +** +** 0x01 - space +** 0x02 - digit +** 0x04 - alphabetic, including '_' +*/ +static const char qrfCType[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 4, + 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; +#define qrfSpace(x) ((qrfCType[(unsigned char)x]&1)!=0) +#define qrfDigit(x) ((qrfCType[(unsigned char)x]&2)!=0) +#define qrfAlpha(x) ((qrfCType[(unsigned char)x]&4)!=0) +#define qrfAlnum(x) ((qrfCType[(unsigned char)x]&6)!=0) + +#ifndef deliberate_fall_through +/* Quiet some compilers about some of our intentional code. */ +# if defined(GCC_VERSION) && GCC_VERSION>=7000000 +# define deliberate_fall_through __attribute__((fallthrough)); +# else +# define deliberate_fall_through +# endif +#endif + +/* +** Set an error code and error message. +*/ +static void qrfError( + Qrf *p, /* Query result state */ + int iCode, /* Error code */ + const char *zFormat, /* Message format (or NULL) */ + ... +){ + p->iErr = iCode; + if( p->pzErr!=0 ){ + sqlite3_free(*p->pzErr); + *p->pzErr = 0; + if( zFormat ){ + va_list ap; + va_start(ap, zFormat); + *p->pzErr = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + } + } +} + +/* +** Out-of-memory error. +*/ +static void qrfOom(Qrf *p){ + qrfError(p, SQLITE_NOMEM, "out of memory"); +} + +/* +** Transfer any error in pStr over into p. +*/ +static void qrfStrErr(Qrf *p, sqlite3_str *pStr){ + int rc = pStr ? sqlite3_str_errcode(pStr) : 0; + if( rc ){ + qrfError(p, rc, sqlite3_errstr(rc)); + } +} + + +/* +** Add a new entry to the EXPLAIN QUERY PLAN data +*/ +static void qrfEqpAppend(Qrf *p, int iEqpId, int p2, const char *zText){ + qrfEQPGraphRow *pNew; + sqlite3_int64 nText; + if( zText==0 ) return; + if( p->u.pGraph==0 ){ + p->u.pGraph = sqlite3_malloc64( sizeof(qrfEQPGraph) ); + if( p->u.pGraph==0 ){ + qrfOom(p); + return; + } + memset(p->u.pGraph, 0, sizeof(qrfEQPGraph) ); + } + nText = strlen(zText); + pNew = sqlite3_malloc64( sizeof(*pNew) + nText ); + if( pNew==0 ){ + qrfOom(p); + return; + } + pNew->iEqpId = iEqpId; + pNew->iParentId = p2; + memcpy(pNew->zText, zText, nText+1); + pNew->pNext = 0; + if( p->u.pGraph->pLast ){ + p->u.pGraph->pLast->pNext = pNew; + }else{ + p->u.pGraph->pRow = pNew; + } + p->u.pGraph->pLast = pNew; +} + +/* +** Free and reset the EXPLAIN QUERY PLAN data that has been collected +** in p->u.pGraph. +*/ +static void qrfEqpReset(Qrf *p){ + qrfEQPGraphRow *pRow, *pNext; + if( p->u.pGraph ){ + for(pRow = p->u.pGraph->pRow; pRow; pRow = pNext){ + pNext = pRow->pNext; + sqlite3_free(pRow); + } + sqlite3_free(p->u.pGraph); + p->u.pGraph = 0; + } +} + +/* Return the next EXPLAIN QUERY PLAN line with iEqpId that occurs after +** pOld, or return the first such line if pOld is NULL +*/ +static qrfEQPGraphRow *qrfEqpNextRow(Qrf *p, int iEqpId, qrfEQPGraphRow *pOld){ + qrfEQPGraphRow *pRow = pOld ? pOld->pNext : p->u.pGraph->pRow; + while( pRow && pRow->iParentId!=iEqpId ) pRow = pRow->pNext; + return pRow; +} + +/* Render a single level of the graph that has iEqpId as its parent. Called +** recursively to render sublevels. +*/ +static void qrfEqpRenderLevel(Qrf *p, int iEqpId){ + qrfEQPGraphRow *pRow, *pNext; + i64 n = strlen(p->u.pGraph->zPrefix); + char *z; + for(pRow = qrfEqpNextRow(p, iEqpId, 0); pRow; pRow = pNext){ + pNext = qrfEqpNextRow(p, iEqpId, pRow); + z = pRow->zText; + sqlite3_str_appendf(p->pOut, "%s%s%s\n", p->u.pGraph->zPrefix, + pNext ? "|--" : "`--", z); + if( n<(i64)sizeof(p->u.pGraph->zPrefix)-7 ){ + memcpy(&p->u.pGraph->zPrefix[n], pNext ? "| " : " ", 4); + qrfEqpRenderLevel(p, pRow->iEqpId); + p->u.pGraph->zPrefix[n] = 0; + } + } +} + +/* +** Render the 64-bit value N in a more human-readable format into +** pOut. +** +** + Only show the first three significant digits. +** + Append suffixes K, M, G, T, P, and E for 1e3, 1e6, ... 1e18 +*/ +static void qrfApproxInt64(sqlite3_str *pOut, i64 N){ + static const char aSuffix[] = { 'K', 'M', 'G', 'T', 'P', 'E' }; + int i; + if( N<0 ){ + N = N==INT64_MIN ? INT64_MAX : -N; + sqlite3_str_append(pOut, "-", 1); + } + if( N<10000 ){ + sqlite3_str_appendf(pOut, "%4lld ", N); + return; + } + for(i=1; i<=18; i++){ + N = (N+5)/10; + if( N<10000 ){ + int n = (int)N; + switch( i%3 ){ + case 0: + sqlite3_str_appendf(pOut, "%d.%02d", n/1000, (n%1000)/10); + break; + case 1: + sqlite3_str_appendf(pOut, "%2d.%d", n/100, (n%100)/10); + break; + case 2: + sqlite3_str_appendf(pOut, "%4d", n/10); + break; + } + sqlite3_str_append(pOut, &aSuffix[i/3], 1); + break; + } + } +} + +/* +** Display and reset the EXPLAIN QUERY PLAN data +*/ +static void qrfEqpRender(Qrf *p, i64 nCycle){ + qrfEQPGraphRow *pRow; + if( p->u.pGraph!=0 && (pRow = p->u.pGraph->pRow)!=0 ){ + if( pRow->zText[0]=='-' ){ + if( pRow->pNext==0 ){ + qrfEqpReset(p); + return; + } + sqlite3_str_appendf(p->pOut, "%s\n", pRow->zText+3); + p->u.pGraph->pRow = pRow->pNext; + sqlite3_free(pRow); + }else if( nCycle>0 ){ + int nSp = p->u.pGraph->nWidth - 2; + if( p->spec.eStyle==QRF_STYLE_StatsEst ){ + sqlite3_str_appendchar(p->pOut, nSp, ' '); + sqlite3_str_appendall(p->pOut, + "Cycles Loops (est) Rows (est)\n"); + sqlite3_str_appendchar(p->pOut, nSp, ' '); + sqlite3_str_appendall(p->pOut, + "---------- ------------ ------------\n"); + }else{ + sqlite3_str_appendchar(p->pOut, nSp, ' '); + sqlite3_str_appendall(p->pOut, + "Cycles Loops Rows \n"); + sqlite3_str_appendchar(p->pOut, nSp, ' '); + sqlite3_str_appendall(p->pOut, + "---------- ----- -----\n"); + } + sqlite3_str_appendall(p->pOut, "QUERY PLAN"); + sqlite3_str_appendchar(p->pOut, nSp - 10, ' '); + qrfApproxInt64(p->pOut, nCycle); + sqlite3_str_appendall(p->pOut, " 100%\n"); + }else{ + sqlite3_str_appendall(p->pOut, "QUERY PLAN\n"); + } + p->u.pGraph->zPrefix[0] = 0; + qrfEqpRenderLevel(p, 0); + qrfEqpReset(p); + } +} + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +/* +** Helper function for qrfExpStats(). +** +*/ +static int qrfStatsHeight(sqlite3_stmt *p, int iEntry){ + int iPid = 0; + int ret = 1; + sqlite3_stmt_scanstatus_v2(p, iEntry, + SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid + ); + while( iPid!=0 ){ + int ii; + for(ii=0; 1; ii++){ + int iId; + int res; + res = sqlite3_stmt_scanstatus_v2(p, ii, + SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iId + ); + if( res ) break; + if( iId==iPid ){ + sqlite3_stmt_scanstatus_v2(p, ii, + SQLITE_SCANSTAT_PARENTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid + ); + } + } + ret++; + } + return ret; +} +#endif /* SQLITE_ENABLE_STMT_SCANSTATUS */ + + +/* +** Generate ".scanstatus est" style of EQP output. +*/ +static void qrfEqpStats(Qrf *p){ +#ifndef SQLITE_ENABLE_STMT_SCANSTATUS + qrfError(p, SQLITE_ERROR, "not available in this build"); +#else + static const int f = SQLITE_SCANSTAT_COMPLEX; + sqlite3_stmt *pS = p->pStmt; + int i = 0; + i64 nTotal = 0; + int nWidth = 0; + int prevPid = -1; /* Previous iPid */ + double rEstCum = 1.0; /* Cumulative row estimate */ + sqlite3_str *pLine = sqlite3_str_new(p->db); + sqlite3_str *pStats = sqlite3_str_new(p->db); + qrfEqpReset(p); + + for(i=0; 1; i++){ + const char *z = 0; + int n = 0; + if( sqlite3_stmt_scanstatus_v2(pS,i,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&z) ){ + break; + } + n = (int)strlen(z) + qrfStatsHeight(pS,i)*3; + if( n>nWidth ) nWidth = n; + } + nWidth += 2; + + sqlite3_stmt_scanstatus_v2(pS,-1, SQLITE_SCANSTAT_NCYCLE, f, (void*)&nTotal); + for(i=0; 1; i++){ + i64 nLoop = 0; + i64 nRow = 0; + i64 nCycle = 0; + int iId = 0; + int iPid = 0; + const char *zo = 0; + const char *zName = 0; + double rEst = 0.0; + + if( sqlite3_stmt_scanstatus_v2(pS,i,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&zo) ){ + break; + } + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_PARENTID,f,(void*)&iPid); + if( iPid!=prevPid ){ + prevPid = iPid; + rEstCum = 1.0; + } + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_EST,f,(void*)&rEst); + rEstCum *= rEst; + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_NLOOP,f,(void*)&nLoop); + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_NVISIT,f,(void*)&nRow); + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_NCYCLE,f,(void*)&nCycle); + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_SELECTID,f,(void*)&iId); + sqlite3_stmt_scanstatus_v2(pS,i, SQLITE_SCANSTAT_NAME,f,(void*)&zName); + + if( nCycle>=0 || nLoop>=0 || nRow>=0 ){ + int nSp = 0; + sqlite3_str_reset(pStats); + if( nCycle>=0 && nTotal>0 ){ + qrfApproxInt64(pStats, nCycle); + sqlite3_str_appendf(pStats, " %3d%%", + ((nCycle*100)+nTotal/2) / nTotal + ); + nSp = 2; + } + if( nLoop>=0 ){ + if( nSp ) sqlite3_str_appendchar(pStats, nSp, ' '); + qrfApproxInt64(pStats, nLoop); + nSp = 2; + if( p->spec.eStyle==QRF_STYLE_StatsEst ){ + sqlite3_str_appendf(pStats, " "); + qrfApproxInt64(pStats, (i64)(rEstCum/rEst)); + } + } + if( nRow>=0 ){ + if( nSp ) sqlite3_str_appendchar(pStats, nSp, ' '); + qrfApproxInt64(pStats, nRow); + nSp = 2; + if( p->spec.eStyle==QRF_STYLE_StatsEst ){ + sqlite3_str_appendf(pStats, " "); + qrfApproxInt64(pStats, (i64)rEstCum); + } + } + sqlite3_str_appendf(pLine, + "% *s %s", -1*(nWidth-qrfStatsHeight(pS,i)*3), zo, + sqlite3_str_value(pStats) + ); + sqlite3_str_reset(pStats); + qrfEqpAppend(p, iId, iPid, sqlite3_str_value(pLine)); + sqlite3_str_reset(pLine); + }else{ + qrfEqpAppend(p, iId, iPid, zo); + } + } + if( p->u.pGraph ) p->u.pGraph->nWidth = nWidth; + qrfStrErr(p, pLine); + sqlite3_free(sqlite3_str_finish(pLine)); + qrfStrErr(p, pStats); + sqlite3_free(sqlite3_str_finish(pStats)); +#endif +} + + +/* +** Reset the prepared statement. +*/ +static void qrfResetStmt(Qrf *p){ + int rc = sqlite3_reset(p->pStmt); + if( rc!=SQLITE_OK && p->iErr==SQLITE_OK ){ + qrfError(p, rc, "%s", sqlite3_errmsg(p->db)); + } +} + +/* +** If xWrite is defined, send all content of pOut to xWrite and +** reset pOut. +*/ +static void qrfWrite(Qrf *p){ + int n; + if( p->spec.xWrite && (n = sqlite3_str_length(p->pOut))>0 ){ + int rc = p->spec.xWrite(p->spec.pWriteArg, + sqlite3_str_value(p->pOut), + (sqlite3_int64)n); + sqlite3_str_reset(p->pOut); + if( rc ){ + qrfError(p, rc, "Failed to write %d bytes of output", n); + } + } +} + +/* Lookup table to estimate the number of columns consumed by a Unicode +** character. +*/ +static const struct { + unsigned char w; /* Width of the character in columns */ + int iFirst; /* First character in a span having this width */ +} aQrfUWidth[] = { + /* {1, 0x00000}, */ + {0, 0x00300}, {1, 0x00370}, {0, 0x00483}, {1, 0x00487}, {0, 0x00488}, + {1, 0x0048a}, {0, 0x00591}, {1, 0x005be}, {0, 0x005bf}, {1, 0x005c0}, + {0, 0x005c1}, {1, 0x005c3}, {0, 0x005c4}, {1, 0x005c6}, {0, 0x005c7}, + {1, 0x005c8}, {0, 0x00600}, {1, 0x00604}, {0, 0x00610}, {1, 0x00616}, + {0, 0x0064b}, {1, 0x0065f}, {0, 0x00670}, {1, 0x00671}, {0, 0x006d6}, + {1, 0x006e5}, {0, 0x006e7}, {1, 0x006e9}, {0, 0x006ea}, {1, 0x006ee}, + {0, 0x0070f}, {1, 0x00710}, {0, 0x00711}, {1, 0x00712}, {0, 0x00730}, + {1, 0x0074b}, {0, 0x007a6}, {1, 0x007b1}, {0, 0x007eb}, {1, 0x007f4}, + {0, 0x00901}, {1, 0x00903}, {0, 0x0093c}, {1, 0x0093d}, {0, 0x00941}, + {1, 0x00949}, {0, 0x0094d}, {1, 0x0094e}, {0, 0x00951}, {1, 0x00955}, + {0, 0x00962}, {1, 0x00964}, {0, 0x00981}, {1, 0x00982}, {0, 0x009bc}, + {1, 0x009bd}, {0, 0x009c1}, {1, 0x009c5}, {0, 0x009cd}, {1, 0x009ce}, + {0, 0x009e2}, {1, 0x009e4}, {0, 0x00a01}, {1, 0x00a03}, {0, 0x00a3c}, + {1, 0x00a3d}, {0, 0x00a41}, {1, 0x00a43}, {0, 0x00a47}, {1, 0x00a49}, + {0, 0x00a4b}, {1, 0x00a4e}, {0, 0x00a70}, {1, 0x00a72}, {0, 0x00a81}, + {1, 0x00a83}, {0, 0x00abc}, {1, 0x00abd}, {0, 0x00ac1}, {1, 0x00ac6}, + {0, 0x00ac7}, {1, 0x00ac9}, {0, 0x00acd}, {1, 0x00ace}, {0, 0x00ae2}, + {1, 0x00ae4}, {0, 0x00b01}, {1, 0x00b02}, {0, 0x00b3c}, {1, 0x00b3d}, + {0, 0x00b3f}, {1, 0x00b40}, {0, 0x00b41}, {1, 0x00b44}, {0, 0x00b4d}, + {1, 0x00b4e}, {0, 0x00b56}, {1, 0x00b57}, {0, 0x00b82}, {1, 0x00b83}, + {0, 0x00bc0}, {1, 0x00bc1}, {0, 0x00bcd}, {1, 0x00bce}, {0, 0x00c3e}, + {1, 0x00c41}, {0, 0x00c46}, {1, 0x00c49}, {0, 0x00c4a}, {1, 0x00c4e}, + {0, 0x00c55}, {1, 0x00c57}, {0, 0x00cbc}, {1, 0x00cbd}, {0, 0x00cbf}, + {1, 0x00cc0}, {0, 0x00cc6}, {1, 0x00cc7}, {0, 0x00ccc}, {1, 0x00cce}, + {0, 0x00ce2}, {1, 0x00ce4}, {0, 0x00d41}, {1, 0x00d44}, {0, 0x00d4d}, + {1, 0x00d4e}, {0, 0x00dca}, {1, 0x00dcb}, {0, 0x00dd2}, {1, 0x00dd5}, + {0, 0x00dd6}, {1, 0x00dd7}, {0, 0x00e31}, {1, 0x00e32}, {0, 0x00e34}, + {1, 0x00e3b}, {0, 0x00e47}, {1, 0x00e4f}, {0, 0x00eb1}, {1, 0x00eb2}, + {0, 0x00eb4}, {1, 0x00eba}, {0, 0x00ebb}, {1, 0x00ebd}, {0, 0x00ec8}, + {1, 0x00ece}, {0, 0x00f18}, {1, 0x00f1a}, {0, 0x00f35}, {1, 0x00f36}, + {0, 0x00f37}, {1, 0x00f38}, {0, 0x00f39}, {1, 0x00f3a}, {0, 0x00f71}, + {1, 0x00f7f}, {0, 0x00f80}, {1, 0x00f85}, {0, 0x00f86}, {1, 0x00f88}, + {0, 0x00f90}, {1, 0x00f98}, {0, 0x00f99}, {1, 0x00fbd}, {0, 0x00fc6}, + {1, 0x00fc7}, {0, 0x0102d}, {1, 0x01031}, {0, 0x01032}, {1, 0x01033}, + {0, 0x01036}, {1, 0x0103b}, {0, 0x01058}, + {1, 0x0105a}, {2, 0x01100}, {0, 0x01160}, {1, 0x01200}, {0, 0x0135f}, + {1, 0x01360}, {0, 0x01712}, {1, 0x01715}, {0, 0x01732}, {1, 0x01735}, + {0, 0x01752}, {1, 0x01754}, {0, 0x01772}, {1, 0x01774}, {0, 0x017b4}, + {1, 0x017b6}, {0, 0x017b7}, {1, 0x017be}, {0, 0x017c6}, {1, 0x017c7}, + {0, 0x017c9}, {1, 0x017d4}, {0, 0x017dd}, {1, 0x017de}, {0, 0x0180b}, + {1, 0x0180e}, {0, 0x018a9}, {1, 0x018aa}, {0, 0x01920}, {1, 0x01923}, + {0, 0x01927}, {1, 0x01929}, {0, 0x01932}, {1, 0x01933}, {0, 0x01939}, + {1, 0x0193c}, {0, 0x01a17}, {1, 0x01a19}, {0, 0x01b00}, {1, 0x01b04}, + {0, 0x01b34}, {1, 0x01b35}, {0, 0x01b36}, {1, 0x01b3b}, {0, 0x01b3c}, + {1, 0x01b3d}, {0, 0x01b42}, {1, 0x01b43}, {0, 0x01b6b}, {1, 0x01b74}, + {0, 0x01dc0}, {1, 0x01dcb}, {0, 0x01dfe}, {1, 0x01e00}, {0, 0x0200b}, + {1, 0x02010}, {0, 0x0202a}, {1, 0x0202f}, {0, 0x02060}, {1, 0x02064}, + {0, 0x0206a}, {1, 0x02070}, {0, 0x020d0}, {1, 0x020f0}, {2, 0x02329}, + {1, 0x0232b}, {2, 0x02e80}, {0, 0x0302a}, {2, 0x03030}, {1, 0x0303f}, + {2, 0x03040}, {0, 0x03099}, {2, 0x0309b}, {1, 0x0a4d0}, {0, 0x0a806}, + {1, 0x0a807}, {0, 0x0a80b}, {1, 0x0a80c}, {0, 0x0a825}, {1, 0x0a827}, + {2, 0x0ac00}, {1, 0x0d7a4}, {2, 0x0f900}, {1, 0x0fb00}, {0, 0x0fb1e}, + {1, 0x0fb1f}, {0, 0x0fe00}, {2, 0x0fe10}, {1, 0x0fe1a}, {0, 0x0fe20}, + {1, 0x0fe24}, {2, 0x0fe30}, {1, 0x0fe70}, {0, 0x0feff}, {2, 0x0ff00}, + {1, 0x0ff61}, {2, 0x0ffe0}, {1, 0x0ffe7}, {0, 0x0fff9}, {1, 0x0fffc}, + {0, 0x10a01}, {1, 0x10a04}, {0, 0x10a05}, {1, 0x10a07}, {0, 0x10a0c}, + {1, 0x10a10}, {0, 0x10a38}, {1, 0x10a3b}, {0, 0x10a3f}, {1, 0x10a40}, + {0, 0x1d167}, {1, 0x1d16a}, {0, 0x1d173}, {1, 0x1d183}, {0, 0x1d185}, + {1, 0x1d18c}, {0, 0x1d1aa}, {1, 0x1d1ae}, {0, 0x1d242}, {1, 0x1d245}, + {2, 0x20000}, {1, 0x2fffe}, {2, 0x30000}, {1, 0x3fffe}, {0, 0xe0001}, + {1, 0xe0002}, {0, 0xe0020}, {1, 0xe0080}, {0, 0xe0100}, {1, 0xe01f0} +}; + +/* +** Return an estimate of the width, in columns, for the single Unicode +** character c. For normal characters, the answer is always 1. But the +** estimate might be 0 or 2 for zero-width and double-width characters. +** +** Different display devices display unicode using different widths. So +** it is impossible to know that true display width with 100% accuracy. +** Inaccuracies in the width estimates might cause columns to be misaligned. +** Unfortunately, there is nothing we can do about that. +*/ +int sqlite3_qrf_wcwidth(int c){ + int iFirst, iLast; + + /* Fast path for common characters */ + if( c<0x300 ) return 1; + + /* The general case */ + iFirst = 0; + iLast = sizeof(aQrfUWidth)/sizeof(aQrfUWidth[0]) - 1; + while( iFirst<iLast-1 ){ + int iMid = (iFirst+iLast)/2; + int cMid = aQrfUWidth[iMid].iFirst; + if( cMid < c ){ + iFirst = iMid; + }else if( cMid > c ){ + iLast = iMid - 1; + }else{ + return aQrfUWidth[iMid].w; + } + } + if( aQrfUWidth[iLast].iFirst > c ) return aQrfUWidth[iFirst].w; + return aQrfUWidth[iLast].w; +} + +/* +** Compute the value and length of a multi-byte UTF-8 character that +** begins at z[0]. Return the length. Write the Unicode value into *pU. +** +** This routine only works for *multi-byte* UTF-8 characters. It does +** not attempt to detect illegal characters. +*/ +int sqlite3_qrf_decode_utf8(const unsigned char *z, int *pU){ + if( (z[0] & 0xe0)==0xc0 && (z[1] & 0xc0)==0x80 ){ + *pU = ((z[0] & 0x1f)<<6) | (z[1] & 0x3f); + return 2; + } + if( (z[0] & 0xf0)==0xe0 && (z[1] & 0xc0)==0x80 && (z[2] & 0xc0)==0x80 ){ + *pU = ((z[0] & 0x0f)<<12) | ((z[1] & 0x3f)<<6) | (z[2] & 0x3f); + return 3; + } + if( (z[0] & 0xf8)==0xf0 && (z[1] & 0xc0)==0x80 && (z[2] & 0xc0)==0x80 + && (z[3] & 0xc0)==0x80 + ){ + *pU = ((z[0] & 0x0f)<<18) | ((z[1] & 0x3f)<<12) | ((z[2] & 0x3f))<<6 + | (z[3] & 0x3f); + return 4; + } + *pU = 0; + return 1; +} + +/* +** Check to see if z[] is a valid VT100 escape. If it is, then +** return the number of bytes in the escape sequence. Return 0 if +** z[] is not a VT100 escape. +** +** This routine assumes that z[0] is \033 (ESC). +*/ +static int qrfIsVt100(const unsigned char *z){ + int i; + if( z[1]!='[' ) return 0; + i = 2; + while( z[i]>=0x30 && z[i]<=0x3f ){ i++; } + while( z[i]>=0x20 && z[i]<=0x2f ){ i++; } + if( z[i]<0x40 || z[i]>0x7e ) return 0; + return i+1; +} + +/* +** Return the length of a string in display characters. +** +** Most characters of the input string count as 1, including +** multi-byte UTF8 characters. However, zero-width unicode +** characters and VT100 escape sequences count as zero, and +** double-width characters count as two. +** +** The definition of "zero-width" and "double-width" characters +** is not precise. It depends on the output device, to some extent, +** and it varies according to the Unicode version. This routine +** makes the best guess that it can. +*/ +size_t sqlite3_qrf_wcswidth(const char *zIn){ + const unsigned char *z = (const unsigned char*)zIn; + size_t n = 0; + while( *z ){ + if( z[0]<' ' ){ + int k; + if( z[0]=='\033' && (k = qrfIsVt100(z))>0 ){ + z += k; + }else{ + z++; + } + }else if( (0x80&z[0])==0 ){ + n++; + z++; + }else{ + int u = 0; + int len = sqlite3_qrf_decode_utf8(z, &u); + z += len; + n += sqlite3_qrf_wcwidth(u); + } + } + return n; +} + +/* +** Return the display width of the longest line of text +** in the (possibly) multi-line input string zIn[0..nByte]. +** zIn[] is not necessarily zero-terminated. Take +** into account tab characters, zero- and double-width +** characters, CR and NL, and VT100 escape codes. +** +** Write the number of newlines into *pnNL. So, *pnNL will +** return 0 if everything fits on one line, or positive it +** it will need to be split. +*/ +static int qrfDisplayWidth(const char *zIn, sqlite3_int64 nByte, int *pnNL){ + const unsigned char *z; + const unsigned char *zEnd; + int mx = 0; + int n = 0; + int nNL = 0; + if( zIn==0 ) zIn = ""; + z = (const unsigned char*)zIn; + zEnd = &z[nByte]; + while( z<zEnd ){ + if( z[0]<' ' ){ + int k; + if( z[0]=='\033' && (k = qrfIsVt100(z))>0 ){ + z += k; + }else{ + if( z[0]=='\t' ){ + n = (n+8)&~7; + }else if( z[0]=='\n' || z[0]=='\r' ){ + nNL++; + if( n>mx ) mx = n; + n = 0; + } + z++; + } + }else if( (0x80&z[0])==0 ){ + n++; + z++; + }else{ + int u = 0; + int len = sqlite3_qrf_decode_utf8(z, &u); + z += len; + n += sqlite3_qrf_wcwidth(u); + } + } + if( mx>n ) n = mx; + if( pnNL ) *pnNL = nNL; + return n; +} + +/* +** Escape the input string if it is needed and in accordance with +** eEsc, which is either QRF_ESC_Ascii or QRF_ESC_Symbol. +** +** Escaping is needed if the string contains any control characters +** other than \t, \n, and \r\n +** +** If no escaping is needed (the common case) then set *ppOut to NULL +** and return 0. If escaping is needed, write the escaped string into +** memory obtained from sqlite3_malloc64() and make *ppOut point to that +** memory and return 0. If an error occurs, return non-zero. +** +** The caller is responsible for freeing *ppFree if it is non-NULL in order +** to reclaim memory. +*/ +static void qrfEscape( + int eEsc, /* QRF_ESC_Ascii or QRF_ESC_Symbol */ + sqlite3_str *pStr, /* String to be escaped */ + int iStart /* Begin escapding on this byte of pStr */ +){ + sqlite3_int64 i, j; /* Loop counters */ + sqlite3_int64 sz; /* Size of the string prior to escaping */ + sqlite3_int64 nCtrl = 0;/* Number of control characters to escape */ + unsigned char *zIn; /* Text to be escaped */ + unsigned char c; /* A single character of the text */ + unsigned char *zOut; /* Where to write the results */ + + /* Find the text to be escaped */ + zIn = (unsigned char*)sqlite3_str_value(pStr); + if( zIn==0 ) return; + zIn += iStart; + + /* Count the control characters */ + for(i=0; (c = zIn[i])!=0; i++){ + if( c<=0x1f + && c!='\t' + && c!='\n' + && (c!='\r' || zIn[i+1]!='\n') + ){ + nCtrl++; + } + } + if( nCtrl==0 ) return; /* Early out if no control characters */ + + /* Make space to hold the escapes. Copy the original text to the end + ** of the available space. */ + sz = sqlite3_str_length(pStr) - iStart; + if( eEsc==QRF_ESC_Symbol ) nCtrl *= 2; + sqlite3_str_appendchar(pStr, nCtrl, ' '); + zOut = (unsigned char*)sqlite3_str_value(pStr); + if( zOut==0 ) return; + zOut += iStart; + zIn = zOut + nCtrl; + memmove(zIn,zOut,sz); + + /* Convert the control characters */ + for(i=j=0; (c = zIn[i])!=0; i++){ + if( c>0x1f + || c=='\t' + || c=='\n' + || (c=='\r' && zIn[i+1]=='\n') + ){ + continue; + } + if( i>0 ){ + memmove(&zOut[j], zIn, i); + j += i; + } + zIn += i+1; + i = -1; + if( eEsc==QRF_ESC_Symbol ){ + zOut[j++] = 0xe2; + zOut[j++] = 0x90; + zOut[j++] = 0x80+c; + }else{ + zOut[j++] = '^'; + zOut[j++] = 0x40+c; + } + } +} + +/* +** Determine if the string z[] can be shown as plain text. Return true +** if z[] is unambiguously text. Return false if z[] needs to be +** quoted. +** +** All of the following must be true in order for z[] to be relaxable: +** +** (1) z[] does not begin or end with ' or whitespace +** (2) z[] is not the same as the NULL rendering +** (3) z[] does not looks like a numeric literal +*/ +static int qrfRelaxable(Qrf *p, const char *z){ + size_t i, n; + if( z[0]=='\'' || qrfSpace(z[0]) ) return 0; + if( z[0]==0 ){ + return (p->spec.zNull!=0 && p->spec.zNull[0]!=0); + } + n = strlen(z); + if( n==0 || z[n-1]=='\'' || qrfSpace(z[n-1]) ) return 0; + if( p->spec.zNull && strcmp(p->spec.zNull,z)==0 ) return 0; + i = (z[0]=='-' || z[0]=='+'); + if( strcmp(z+i,"Inf")==0 ) return 0; + if( !qrfDigit(z[i]) ) return 1; + i++; + while( qrfDigit(z[i]) ){ i++; } + if( z[i]==0 ) return 0; + if( z[i]=='.' ){ + i++; + while( qrfDigit(z[i]) ){ i++; } + if( z[i]==0 ) return 0; + } + if( z[i]=='e' || z[i]=='E' ){ + i++; + if( z[i]=='+' || z[i]=='-' ){ i++; } + if( !qrfDigit(z[i]) ) return 1; + i++; + while( qrfDigit(z[i]) ){ i++; } + } + return z[i]!=0; +} + +/* +** If a field contains any character identified by a 1 in the following +** array, then the string must be quoted for CSV. +*/ +static const char qrfCsvQuote[] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +}; + +/* +** Encode text appropriately and append it to pOut. +*/ +static void qrfEncodeText(Qrf *p, sqlite3_str *pOut, const char *zTxt){ + int iStart = sqlite3_str_length(pOut); + switch( p->spec.eText ){ + case QRF_TEXT_Relaxed: + if( qrfRelaxable(p, zTxt) ){ + sqlite3_str_appendall(pOut, zTxt); + break; + } + deliberate_fall_through; /* FALLTHRU */ + case QRF_TEXT_Sql: { + if( p->spec.eEsc==QRF_ESC_Off ){ + sqlite3_str_appendf(pOut, "%Q", zTxt); + }else{ + sqlite3_str_appendf(pOut, "%#Q", zTxt); + } + break; + } + case QRF_TEXT_Csv: { + unsigned int i; + for(i=0; zTxt[i]; i++){ + if( qrfCsvQuote[((const unsigned char*)zTxt)[i]] ){ + i = 0; + break; + } + } + if( i==0 || strstr(zTxt, p->spec.zColumnSep)!=0 ){ + sqlite3_str_appendf(pOut, "\"%w\"", zTxt); + }else{ + sqlite3_str_appendall(pOut, zTxt); + } + break; + } + case QRF_TEXT_Html: { + const unsigned char *z = (const unsigned char*)zTxt; + while( *z ){ + unsigned int i = 0; + unsigned char c; + while( (c=z[i])>'>' + || (c && c!='<' && c!='>' && c!='&' && c!='\"' && c!='\'') + ){ + i++; + } + if( i>0 ){ + sqlite3_str_append(pOut, (const char*)z, i); + } + switch( z[i] ){ + case '>': sqlite3_str_append(pOut, "&lt;", 4); break; + case '&': sqlite3_str_append(pOut, "&amp;", 5); break; + case '<': sqlite3_str_append(pOut, "&lt;", 4); break; + case '"': sqlite3_str_append(pOut, "&quot;", 6); break; + case '\'': sqlite3_str_append(pOut, "&#39;", 5); break; + default: i--; + } + z += i + 1; + } + break; + } + case QRF_TEXT_Tcl: + case QRF_TEXT_Json: { + const unsigned char *z = (const unsigned char*)zTxt; + sqlite3_str_append(pOut, "\"", 1); + while( *z ){ + unsigned int i; + for(i=0; z[i]>=0x20 && z[i]!='\\' && z[i]!='"'; i++){} + if( i>0 ){ + sqlite3_str_append(pOut, (const char*)z, i); + } + if( z[i]==0 ) break; + switch( z[i] ){ + case '"': sqlite3_str_append(pOut, "\\\"", 2); break; + case '\\': sqlite3_str_append(pOut, "\\\\", 2); break; + case '\b': sqlite3_str_append(pOut, "\\b", 2); break; + case '\f': sqlite3_str_append(pOut, "\\f", 2); break; + case '\n': sqlite3_str_append(pOut, "\\n", 2); break; + case '\r': sqlite3_str_append(pOut, "\\r", 2); break; + case '\t': sqlite3_str_append(pOut, "\\t", 2); break; + default: { + if( p->spec.eText==QRF_TEXT_Json ){ + sqlite3_str_appendf(pOut, "\\u%04x", z[i]); + }else{ + sqlite3_str_appendf(pOut, "\\%03o", z[i]); + } + break; + } + } + z += i + 1; + } + sqlite3_str_append(pOut, "\"", 1); + break; + } + default: { + sqlite3_str_appendall(pOut, zTxt); + break; + } + } + if( p->spec.eEsc!=QRF_ESC_Off ){ + qrfEscape(p->spec.eEsc, pOut, iStart); + } +} + +/* +** Do a quick sanity check to see aBlob[0..nBlob-1] is valid JSONB +** return true if it is and false if it is not. +** +** False positives are possible, but not false negatives. +*/ +static int qrfJsonbQuickCheck(unsigned char *aBlob, int nBlob){ + unsigned char x; /* Payload size half-byte */ + int i; /* Loop counter */ + int n; /* Bytes in the payload size integer */ + sqlite3_uint64 sz; /* value of the payload size integer */ + + if( nBlob==0 ) return 0; + x = aBlob[0]>>4; + if( x<=11 ) return nBlob==(1+x); + n = x<14 ? x-11 : 4*(x-13); + if( nBlob<1+n ) return 0; + sz = aBlob[1]; + for(i=1; i<n; i++) sz = (sz<<8) + aBlob[i+1]; + return sz+n+1==(sqlite3_uint64)nBlob; +} + +/* +** The current iCol-th column of p->pStmt is known to be a BLOB. Check +** to see if that BLOB is really a JSONB blob. If it is, then translate +** it into a text JSON representation and return a pointer to that text JSON. +** If the BLOB is not JSONB, then return a NULL pointer. +** +** The memory used to hold the JSON text is managed internally by the +** "p" object and is overwritten and/or deallocated upon the next call +** to this routine (with the same p argument) or when the p object is +** finailized. +*/ +static const char *qrfJsonbToJson(Qrf *p, int iCol){ + int nByte; + const void *pBlob; + int rc; + nByte = sqlite3_column_bytes(p->pStmt, iCol); + pBlob = sqlite3_column_blob(p->pStmt, iCol); + if( qrfJsonbQuickCheck((unsigned char*)pBlob, nByte)==0 ){ + return 0; + } + if( p->pJTrans==0 ){ + sqlite3 *db; + rc = sqlite3_open(":memory:",&db); + if( rc ){ + sqlite3_close(db); + return 0; + } + rc = sqlite3_prepare_v2(db, "SELECT json(?1)", -1, &p->pJTrans, 0); + if( rc ){ + sqlite3_finalize(p->pJTrans); + p->pJTrans = 0; + sqlite3_close(db); + return 0; + } + }else{ + sqlite3_reset(p->pJTrans); + } + sqlite3_bind_blob(p->pJTrans, 1, (void*)pBlob, nByte, SQLITE_STATIC); + rc = sqlite3_step(p->pJTrans); + if( rc==SQLITE_ROW ){ + return (const char*)sqlite3_column_text(p->pJTrans, 0); + }else{ + return 0; + } +} + +/* +** Adjust the input string zIn[] such that it is no more than N display +** characters wide. If it is wider than that, then truncate and add +** ellipsis. Or if zIn[] contains a \r or \n, truncate at that point, +** adding ellipsis. Embedded tabs in zIn[] are converted into ordinary +** spaces. +** +** Return this display width of the modified title string. +*/ +static int qrfTitleLimit(char *zIn, int N){ + unsigned char *z = (unsigned char*)zIn; + int n = 0; + unsigned char *zEllipsis = 0; + while( 1 /*exit-by-break*/ ){ + if( z[0]<' ' ){ + int k; + if( z[0]==0 ){ + zEllipsis = 0; + break; + }else if( z[0]=='\033' && (k = qrfIsVt100(z))>0 ){ + z += k; + }else if( z[0]=='\t' ){ + z[0] = ' '; + }else if( z[0]=='\n' || z[0]=='\r' ){ + z[0] = ' '; + }else{ + z++; + } + }else if( (0x80&z[0])==0 ){ + if( n>=(N-3) && zEllipsis==0 ) zEllipsis = z; + if( n==N ){ z[0] = 0; break; } + n++; + z++; + }else{ + int u = 0; + int len = sqlite3_qrf_decode_utf8(z, &u); + if( n+len>(N-3) && zEllipsis==0 ) zEllipsis = z; + if( n+len>N ){ z[0] = 0; break; } + z += len; + n += sqlite3_qrf_wcwidth(u); + } + } + if( zEllipsis && N>=3 ) memcpy(zEllipsis,"...",4); + return n; +} + + +/* +** Render value pVal into pOut +*/ +static void qrfRenderValue(Qrf *p, sqlite3_str *pOut, int iCol){ +#if SQLITE_VERSION_NUMBER>=3052000 + int iStartLen = sqlite3_str_length(pOut); +#endif + if( p->spec.xRender ){ + sqlite3_value *pVal; + char *z; + pVal = sqlite3_value_dup(sqlite3_column_value(p->pStmt,iCol)); + z = p->spec.xRender(p->spec.pRenderArg, pVal); + sqlite3_value_free(pVal); + if( z ){ + sqlite3_str_appendall(pOut, z); + sqlite3_free(z); + return; + } + } + switch( sqlite3_column_type(p->pStmt,iCol) ){ + case SQLITE_INTEGER: { + sqlite3_str_appendf(pOut, "%lld", sqlite3_column_int64(p->pStmt,iCol)); + break; + } + case SQLITE_FLOAT: { + const char *zTxt = (const char*)sqlite3_column_text(p->pStmt,iCol); + sqlite3_str_appendall(pOut, zTxt); + break; + } + case SQLITE_BLOB: { + if( p->spec.bTextJsonb==QRF_Yes ){ + const char *zJson = qrfJsonbToJson(p, iCol); + if( zJson ){ + if( p->spec.eText==QRF_TEXT_Sql ){ + sqlite3_str_append(pOut,"jsonb(",6); + qrfEncodeText(p, pOut, zJson); + sqlite3_str_append(pOut,")",1); + }else{ + qrfEncodeText(p, pOut, zJson); + } + break; + } + } + switch( p->spec.eBlob ){ + case QRF_BLOB_Hex: + case QRF_BLOB_Sql: { + int iStart; + int nBlob = sqlite3_column_bytes(p->pStmt,iCol); + int i, j; + char *zVal; + const unsigned char *a = sqlite3_column_blob(p->pStmt,iCol); + if( p->spec.eBlob==QRF_BLOB_Sql ){ + sqlite3_str_append(pOut, "x'", 2); + } + iStart = sqlite3_str_length(pOut); + sqlite3_str_appendchar(pOut, nBlob, ' '); + sqlite3_str_appendchar(pOut, nBlob, ' '); + if( p->spec.eBlob==QRF_BLOB_Sql ){ + sqlite3_str_appendchar(pOut, 1, '\''); + } + if( sqlite3_str_errcode(pOut) ) return; + zVal = sqlite3_str_value(pOut); + for(i=0, j=iStart; i<nBlob; i++, j+=2){ + unsigned char c = a[i]; + zVal[j] = "0123456789abcdef"[(c>>4)&0xf]; + zVal[j+1] = "0123456789abcdef"[(c)&0xf]; + } + break; + } + case QRF_BLOB_Tcl: + case QRF_BLOB_Json: { + int iStart; + int nBlob = sqlite3_column_bytes(p->pStmt,iCol); + int i, j; + char *zVal; + const unsigned char *a = sqlite3_column_blob(p->pStmt,iCol); + int szC = p->spec.eBlob==QRF_BLOB_Json ? 6 : 4; + sqlite3_str_append(pOut, "\"", 1); + iStart = sqlite3_str_length(pOut); + for(i=szC; i>0; i--){ + sqlite3_str_appendchar(pOut, nBlob, ' '); + } + sqlite3_str_appendchar(pOut, 1, '"'); + if( sqlite3_str_errcode(pOut) ) return; + zVal = sqlite3_str_value(pOut); + for(i=0, j=iStart; i<nBlob; i++, j+=szC){ + unsigned char c = a[i]; + zVal[j] = '\\'; + if( szC==4 ){ + zVal[j+1] = '0' + ((c>>6)&3); + zVal[j+2] = '0' + ((c>>3)&7); + zVal[j+3] = '0' + (c&7); + }else{ + zVal[j+1] = 'u'; + zVal[j+2] = '0'; + zVal[j+3] = '0'; + zVal[j+4] = "0123456789abcdef"[(c>>4)&0xf]; + zVal[j+5] = "0123456789abcdef"[(c)&0xf]; + } + } + break; + } + case QRF_BLOB_Size: { + int nBlob = sqlite3_column_bytes(p->pStmt,iCol); + sqlite3_str_appendf(pOut, "(%d-byte blob)", nBlob); + break; + } + default: { + const char *zTxt = (const char*)sqlite3_column_text(p->pStmt,iCol); + qrfEncodeText(p, pOut, zTxt); + } + } + break; + } + case SQLITE_NULL: { + sqlite3_str_appendall(pOut, p->spec.zNull); + break; + } + case SQLITE_TEXT: { + const char *zTxt = (const char*)sqlite3_column_text(p->pStmt,iCol); + qrfEncodeText(p, pOut, zTxt); + break; + } + } +#if SQLITE_VERSION_NUMBER>=3052000 + if( p->spec.nCharLimit>0 + && (sqlite3_str_length(pOut) - iStartLen) > p->spec.nCharLimit + ){ + const unsigned char *z; + int ii = 0, w = 0, limit = p->spec.nCharLimit; + z = (const unsigned char*)sqlite3_str_value(pOut) + iStartLen; + if( limit<4 ) limit = 4; + while( 1 ){ + if( z[ii]<' ' ){ + int k; + if( z[ii]=='\033' && (k = qrfIsVt100(z+ii))>0 ){ + ii += k; + }else if( z[ii]==0 ){ + break; + }else{ + ii++; + } + }else if( (0x80&z[ii])==0 ){ + w++; + if( w>limit ) break; + ii++; + }else{ + int u = 0; + int len = sqlite3_qrf_decode_utf8(&z[ii], &u); + w += sqlite3_qrf_wcwidth(u); + if( w>limit ) break; + ii += len; + } + } + if( w>limit ){ + sqlite3_str_truncate(pOut, iStartLen+ii); + sqlite3_str_append(pOut, "...", 3); + } + } +#endif +} + +/* Trim spaces of the end if pOut +*/ +static void qrfRTrim(sqlite3_str *pOut){ +#if SQLITE_VERSION_NUMBER>=3052000 + int nByte = sqlite3_str_length(pOut); + const char *zOut = sqlite3_str_value(pOut); + while( nByte>0 && zOut[nByte-1]==' ' ){ nByte--; } + sqlite3_str_truncate(pOut, nByte); +#endif +} + +/* +** Store string zUtf to pOut as w characters. If w is negative, +** then right-justify the text. W is the width in display characters, not +** in bytes. Double-width unicode characters count as two characters. +** VT100 escape sequences count as zero. And so forth. +*/ +static void qrfWidthPrint(Qrf *p, sqlite3_str *pOut, int w, const char *zUtf){ + const unsigned char *a = (const unsigned char*)zUtf; + static const int mxW = 10000000; + unsigned char c; + int i = 0; + int n = 0; + int k; + int aw; + (void)p; + if( w<-mxW ){ + w = -mxW; + }else if( w>mxW ){ + w= mxW; + } + aw = w<0 ? -w : w; + if( a==0 ) a = (const unsigned char*)""; + while( (c = a[i])!=0 ){ + if( (c&0xc0)==0xc0 ){ + int u; + int len = sqlite3_qrf_decode_utf8(a+i, &u); + int x = sqlite3_qrf_wcwidth(u); + if( x+n>aw ){ + break; + } + i += len; + n += x; + }else if( c==0x1b && (k = qrfIsVt100(&a[i]))>0 ){ + i += k; + }else if( n>=aw ){ + break; + }else{ + n++; + i++; + } + } + if( n>=aw ){ + sqlite3_str_append(pOut, zUtf, i); + }else if( w<0 ){ + if( aw>n ) sqlite3_str_appendchar(pOut, aw-n, ' '); + sqlite3_str_append(pOut, zUtf, i); + }else{ + sqlite3_str_append(pOut, zUtf, i); + if( aw>n ) sqlite3_str_appendchar(pOut, aw-n, ' '); + } +} + +/* +** (*pz)[] is a line of text that is to be displayed the box or table or +** similar tabular formats. z[] contain newlines or might be too wide +** to fit in the columns so will need to be split into multiple line. +** +** This routine determines: +** +** * How many bytes of z[] should be shown on the current line. +** * How many character positions those bytes will cover. +** * The byte offset to the start of the next line. +*/ +static void qrfWrapLine( + const char *zIn, /* Input text to be displayed */ + int w, /* Column width in characters (not bytes) */ + int bWrap, /* True if we should do word-wrapping */ + int *pnThis, /* OUT: How many bytes of z[] for the current line */ + int *pnWide, /* OUT: How wide is the text of this line */ + int *piNext /* OUT: Offset into z[] to start of the next line */ +){ + int i; /* Input bytes consumed */ + int k; /* Bytes in a VT100 code */ + int n; /* Output column number */ + const unsigned char *z = (const unsigned char*)zIn; + unsigned char c = 0; + + if( z[0]==0 ){ + *pnThis = 0; + *pnWide = 0; + *piNext = 0; + return; + } + n = 0; + for(i=0; n<=w; i++){ + c = z[i]; + if( c>=0xc0 ){ + int u; + int len = sqlite3_qrf_decode_utf8(&z[i], &u); + int wcw = sqlite3_qrf_wcwidth(u); + if( wcw+n>w ) break; + i += len-1; + n += wcw; + continue; + } + if( c>=' ' ){ + if( n==w ) break; + n++; + continue; + } + if( c==0 || c=='\n' ) break; + if( c=='\r' && z[i+1]=='\n' ){ c = z[++i]; break; } + if( c=='\t' ){ + int wcw = 8 - (n&7); + if( n+wcw>w ) break; + n += wcw; + continue; + } + if( c==0x1b && (k = qrfIsVt100(&z[i]))>0 ){ + i += k-1; + }else if( n==w ){ + break; + }else{ + n++; + } + } + if( c==0 ){ + *pnThis = i; + *pnWide = n; + *piNext = i; + return; + } + if( c=='\n' ){ + *pnThis = i; + *pnWide = n; + *piNext = i+1; + return; + } + + /* If we get this far, that means the current line will end at some + ** point that is neither a "\n" or a 0x00. Figure out where that + ** split should occur + */ + if( bWrap && z[i]!=0 && !qrfSpace(z[i]) && qrfAlnum(c)==qrfAlnum(z[i]) ){ + /* Perhaps try to back up to a better place to break the line */ + for(k=i-1; k>=i/2; k--){ + if( qrfSpace(z[k]) ) break; + } + if( k<i/2 ){ + for(k=i; k>=i/2; k--){ + if( qrfAlnum(z[k-1])!=qrfAlnum(z[k]) && (z[k]&0xc0)!=0x80 ) break; + } + } + if( k>=i/2 ){ + i = k; + n = qrfDisplayWidth((const char*)z, k, 0); + } + } + *pnThis = i; + *pnWide = n; + while( zIn[i]==' ' || zIn[i]=='\t' || zIn[i]=='\r' ){ i++; } + *piNext = i; +} + +/* +** Append nVal bytes of text from zVal onto the end of pOut. +** Convert tab characters in zVal to the appropriate number of +** spaces. +*/ +static void qrfAppendWithTabs( + sqlite3_str *pOut, /* Append text here */ + const char *zVal, /* Text to append */ + int nVal /* Use only the first nVal bytes of zVal[] */ +){ + int i = 0; + unsigned int col = 0; + unsigned char *z = (unsigned char *)zVal; + while( i<nVal ){ + unsigned char c = z[i]; + if( c<' ' ){ + int k; + sqlite3_str_append(pOut, (const char*)z, i); + nVal -= i; + z += i; + i = 0; + if( c=='\033' && (k = qrfIsVt100(z))>0 ){ + sqlite3_str_append(pOut, (const char*)z, k); + z += k; + nVal -= k; + }else if( c=='\t' ){ + k = 8 - (col&7); + sqlite3_str_appendchar(pOut, k, ' '); + col += k; + z++; + nVal--; + }else if( c=='\r' && nVal==1 ){ + z++; + nVal--; + }else{ + char zCtrlPik[4]; + col++; + zCtrlPik[0] = 0xe2; + zCtrlPik[1] = 0x90; + zCtrlPik[2] = 0x80+c; + sqlite3_str_append(pOut, zCtrlPik, 3); + z++; + nVal--; + } + }else if( (0x80&c)==0 ){ + i++; + col++; + }else{ + int u = 0; + int len = sqlite3_qrf_decode_utf8(&z[i], &u); + i += len; + col += sqlite3_qrf_wcwidth(u); + } + } + sqlite3_str_append(pOut, (const char*)z, i); +} + +/* +** GCC does not define the offsetof() macro so we'll have to do it +** ourselves. +*/ +#ifndef offsetof +# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) +#endif + +/* +** Data for columnar layout, collected into a single object so +** that it can be more easily passed into subroutines. +*/ +typedef struct qrfColData qrfColData; +struct qrfColData { + Qrf *p; /* The QRF instance */ + int nCol; /* Number of columns in the table */ + unsigned char bMultiRow; /* One or more cells will span multiple lines */ + unsigned char nMargin; /* Width of column margins */ + sqlite3_int64 nRow; /* Number of rows */ + sqlite3_int64 nAlloc; /* Number of cells allocated */ + sqlite3_int64 n; /* Number of cells. nCol*nRow */ + char **az; /* Content of all cells */ + int *aiWth; /* Width of each cell */ + unsigned char *abNum; /* True for each numeric cell */ + struct qrfPerCol { /* Per-column data */ + char *z; /* Cache of text for current row */ + int w; /* Computed width of this column */ + int mxW; /* Maximum natural (unwrapped) width */ + unsigned char e; /* Alignment */ + unsigned char fx; /* Width is fixed */ + unsigned char bNum; /* True if is numeric */ + } *a; /* One per column */ +}; + +/* +** Output horizontally justified text into pOut. The text is the +** first nVal bytes of zVal. Include nWS bytes of whitespace, either +** split between both sides, or on the left, or on the right, depending +** on eAlign. +*/ +static void qrfPrintAligned( + sqlite3_str *pOut, /* Append text here */ + struct qrfPerCol *pCol, /* Information about the text to print */ + int nVal, /* Use only the first nVal bytes of zVal[] */ + int nWS /* Whitespace for horizonal alignment */ +){ + unsigned char eAlign = pCol->e & QRF_ALIGN_HMASK; + if( eAlign==QRF_Auto && pCol->bNum ) eAlign = QRF_ALIGN_Right; + if( eAlign==QRF_ALIGN_Center ){ + /* Center the text */ + sqlite3_str_appendchar(pOut, nWS/2, ' '); + qrfAppendWithTabs(pOut, pCol->z, nVal); + sqlite3_str_appendchar(pOut, nWS - nWS/2, ' '); + }else if( eAlign==QRF_ALIGN_Right ){ + /* Right justify the text */ + sqlite3_str_appendchar(pOut, nWS, ' '); + qrfAppendWithTabs(pOut, pCol->z, nVal); + }else{ + /* Left justify the text */ + qrfAppendWithTabs(pOut, pCol->z, nVal); + sqlite3_str_appendchar(pOut, nWS, ' '); + } +} + +/* +** Free all the memory allocates in the qrfColData object +*/ +static void qrfColDataFree(qrfColData *p){ + sqlite3_int64 i; + for(i=0; i<p->n; i++) sqlite3_free(p->az[i]); + sqlite3_free(p->az); + sqlite3_free(p->aiWth); + sqlite3_free(p->abNum); + sqlite3_free(p->a); + memset(p, 0, sizeof(*p)); +} + +/* +** Allocate space for more cells in the qrfColData object. +** Return non-zero if a memory allocation fails. +*/ +static int qrfColDataEnlarge(qrfColData *p){ + char **azData; + int *aiWth; + unsigned char *abNum; + p->nAlloc = 2*p->nAlloc + 10*p->nCol; + azData = sqlite3_realloc64(p->az, p->nAlloc*sizeof(char*)); + if( azData==0 ){ + qrfOom(p->p); + qrfColDataFree(p); + return 1; + } + p->az = azData; + aiWth = sqlite3_realloc64(p->aiWth, p->nAlloc*sizeof(int)); + if( aiWth==0 ){ + qrfOom(p->p); + qrfColDataFree(p); + return 1; + } + p->aiWth = aiWth; + abNum = sqlite3_realloc64(p->abNum, p->nAlloc); + if( abNum==0 ){ + qrfOom(p->p); + qrfColDataFree(p); + return 1; + } + p->abNum = abNum; + return 0; +} + +/* +** Print a markdown or table-style row separator using ascii-art +*/ +static void qrfRowSeparator(sqlite3_str *pOut, qrfColData *p, char cSep){ + int i; + if( p->nCol>0 ){ + int useBorder = p->p->spec.bBorder!=QRF_No; + if( useBorder ){ + sqlite3_str_append(pOut, &cSep, 1); + } + sqlite3_str_appendchar(pOut, p->a[0].w+p->nMargin, '-'); + for(i=1; i<p->nCol; i++){ + sqlite3_str_append(pOut, &cSep, 1); + sqlite3_str_appendchar(pOut, p->a[i].w+p->nMargin, '-'); + } + if( useBorder ){ + sqlite3_str_append(pOut, &cSep, 1); + } + } + sqlite3_str_append(pOut, "\n", 1); +} + +/* +** UTF8 box-drawing characters. Imagine box lines like this: +** +** 1 +** | +** 4 --+-- 2 +** | +** 3 +** +** Each box characters has between 2 and 4 of the lines leading from +** the center. The characters are here identified by the numbers of +** their corresponding lines. +*/ +#define BOX_24 "\342\224\200" /* U+2500 --- */ +#define BOX_13 "\342\224\202" /* U+2502 | */ +#define BOX_23 "\342\224\214" /* U+250c ,- */ +#define BOX_34 "\342\224\220" /* U+2510 -, */ +#define BOX_12 "\342\224\224" /* U+2514 '- */ +#define BOX_14 "\342\224\230" /* U+2518 -' */ +#define BOX_123 "\342\224\234" /* U+251c |- */ +#define BOX_134 "\342\224\244" /* U+2524 -| */ +#define BOX_234 "\342\224\254" /* U+252c -,- */ +#define BOX_124 "\342\224\264" /* U+2534 -'- */ +#define BOX_1234 "\342\224\274" /* U+253c -|- */ + +/* Rounded corners: */ +#define BOX_R12 "\342\225\260" /* U+2570 '- */ +#define BOX_R23 "\342\225\255" /* U+256d ,- */ +#define BOX_R34 "\342\225\256" /* U+256e -, */ +#define BOX_R14 "\342\225\257" /* U+256f -' */ + +/* Doubled horizontal lines: */ +#define DBL_24 "\342\225\220" /* U+2550 === */ +#define DBL_123 "\342\225\236" /* U+255e |= */ +#define DBL_134 "\342\225\241" /* U+2561 =| */ +#define DBL_1234 "\342\225\252" /* U+256a =|= */ + +/* Draw horizontal line N characters long using unicode box +** characters +*/ +static void qrfBoxLine(sqlite3_str *pOut, int N, int bDbl){ + const char *azDash[2] = { + BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24, + DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 DBL_24 + };/* 0 1 2 3 4 5 6 7 8 9 */ + const int nDash = 30; + N *= 3; + while( N>nDash ){ + sqlite3_str_append(pOut, azDash[bDbl], nDash); + N -= nDash; + } + sqlite3_str_append(pOut, azDash[bDbl], N); +} + +/* +** Draw a horizontal separator for a QRF_STYLE_Box table. +*/ +static void qrfBoxSeparator( + sqlite3_str *pOut, + qrfColData *p, + const char *zSep1, + const char *zSep2, + const char *zSep3, + int bDbl +){ + int i; + if( p->nCol>0 ){ + int useBorder = p->p->spec.bBorder!=QRF_No; + if( useBorder ){ + sqlite3_str_appendall(pOut, zSep1); + } + qrfBoxLine(pOut, p->a[0].w+p->nMargin, bDbl); + for(i=1; i<p->nCol; i++){ + sqlite3_str_appendall(pOut, zSep2); + qrfBoxLine(pOut, p->a[i].w+p->nMargin, bDbl); + } + if( useBorder ){ + sqlite3_str_appendall(pOut, zSep3); + } + } + sqlite3_str_append(pOut, "\n", 1); +} + +/* +** Load into pData the default alignment for the body of a table. +*/ +static void qrfLoadAlignment(qrfColData *pData, Qrf *p){ + sqlite3_int64 i; + for(i=0; i<pData->nCol; i++){ + pData->a[i].e = p->spec.eDfltAlign; + if( i<p->spec.nAlign ){ + unsigned char ax = p->spec.aAlign[i]; + if( (ax & QRF_ALIGN_HMASK)!=0 ){ + pData->a[i].e = (ax & QRF_ALIGN_HMASK) | + (pData->a[i].e & QRF_ALIGN_VMASK); + } + }else if( i<p->spec.nWidth ){ + if( p->spec.aWidth[i]<0 ){ + pData->a[i].e = QRF_ALIGN_Right | + (pData->a[i].e & QRF_ALIGN_VMASK); + } + } + } +} + +/* +** If the single column in pData->a[] with pData->n entries can be +** laid out as nCol columns with a 2-space gap between each such +** that all columns fit within nSW, then return a pointer to an array +** of integers which is the width of each column from left to right. +** +** If the layout is not possible, return a NULL pointer. +** +** Space to hold the returned array is from sqlite_malloc64(). +*/ +static int *qrfValidLayout( + qrfColData *pData, /* Collected query results */ + Qrf *p, /* On which to report an OOM */ + int nCol, /* Attempt this many columns */ + int nSW /* Screen width */ +){ + int i; /* Loop counter */ + int nr; /* Number of rows */ + int w = 0; /* Width of the current column */ + int t; /* Total width of all columns */ + int *aw; /* Array of individual column widths */ + + aw = sqlite3_malloc64( sizeof(int)*nCol ); + if( aw==0 ){ + qrfOom(p); + return 0; + } + nr = (pData->n + nCol - 1)/nCol; + for(i=0; i<pData->n; i++){ + if( (i%nr)==0 ){ + if( i>0 ) aw[i/nr-1] = w; + w = pData->aiWth[i]; + }else if( pData->aiWth[i]>w ){ + w = pData->aiWth[i]; + } + } + aw[nCol-1] = w; + for(t=i=0; i<nCol; i++) t += aw[i]; + t += 2*(nCol-1); + if( t>nSW ){ + sqlite3_free(aw); + return 0; + } + return aw; +} + +/* +** The output is single-column and the bSplitColumn flag is set. +** Check to see if the single-column output can be split into multiple +** columns that appear side-by-side. Adjust pData appropriately. +*/ +static void qrfSplitColumn(qrfColData *pData, Qrf *p){ + int nCol = 1; + int *aw = 0; + char **az = 0; + int *aiWth = 0; + unsigned char *abNum = 0; + int nColNext = 2; + int w; + struct qrfPerCol *a = 0; + sqlite3_int64 nRow = 1; + sqlite3_int64 i; + while( 1/*exit-by-break*/ ){ + int *awNew = qrfValidLayout(pData, p, nColNext, p->spec.nScreenWidth); + if( awNew==0 ) break; + sqlite3_free(aw); + aw = awNew; + nCol = nColNext; + nRow = (pData->n + nCol - 1)/nCol; + if( nRow==1 ) break; + nColNext++; + while( (pData->n + nColNext - 1)/nColNext == nRow ) nColNext++; + } + if( nCol==1 ){ + sqlite3_free(aw); + return; /* Cannot do better than 1 column */ + } + az = sqlite3_malloc64( nRow*nCol*sizeof(char*) ); + if( az==0 ){ + qrfOom(p); + return; + } + aiWth = sqlite3_malloc64( nRow*nCol*sizeof(int) ); + if( aiWth==0 ){ + sqlite3_free(az); + qrfOom(p); + return; + } + a = sqlite3_malloc64( nCol*sizeof(struct qrfPerCol) ); + if( a==0 ){ + sqlite3_free(az); + sqlite3_free(aiWth); + qrfOom(p); + return; + } + abNum = sqlite3_malloc64( nRow*nCol ); + if( abNum==0 ){ + sqlite3_free(az); + sqlite3_free(aiWth); + sqlite3_free(a); + qrfOom(p); + return; + } + for(i=0; i<pData->n; i++){ + sqlite3_int64 j = (i%nRow)*nCol + (i/nRow); + az[j] = pData->az[i]; + abNum[j]= pData->abNum[i]; + pData->az[i] = 0; + aiWth[j] = pData->aiWth[i]; + } + while( i<nRow*nCol ){ + sqlite3_int64 j = (i%nRow)*nCol + (i/nRow); + az[j] = sqlite3_mprintf(""); + if( az[j]==0 ) qrfOom(p); + aiWth[j] = 0; + abNum[j] = 0; + i++; + } + for(i=0; i<nCol; i++){ + a[i].fx = a[i].mxW = a[i].w = aw[i]; + a[i].e = pData->a[0].e; + } + sqlite3_free(pData->az); + sqlite3_free(pData->aiWth); + sqlite3_free(pData->a); + sqlite3_free(pData->abNum); + sqlite3_free(aw); + pData->az = az; + pData->aiWth = aiWth; + pData->a = a; + pData->abNum = abNum; + pData->nCol = nCol; + pData->n = pData->nAlloc = nRow*nCol; + for(i=w=0; i<nCol; i++) w += a[i].w; + pData->nMargin = (p->spec.nScreenWidth - w)/(nCol - 1); + if( pData->nMargin>5 ) pData->nMargin = 5; +} + +/* +** Adjust the layout for the screen width restriction +*/ +static void qrfRestrictScreenWidth(qrfColData *pData, Qrf *p){ + int sepW; /* Width of all box separators and margins */ + int sumW; /* Total width of data area over all columns */ + int targetW; /* Desired total data area */ + int i; /* Loop counters */ + int nCol; /* Number of columns */ + + pData->nMargin = 2; /* Default to normal margins */ + if( p->spec.nScreenWidth==0 ) return; + if( p->spec.eStyle==QRF_STYLE_Column ){ + sepW = pData->nCol*2 - 2; + }else{ + sepW = pData->nCol*3 + 1; + if( p->spec.bBorder==QRF_No ) sepW -= 2; + } + nCol = pData->nCol; + for(i=sumW=0; i<nCol; i++) sumW += pData->a[i].w; + if( p->spec.nScreenWidth >= sumW+sepW ) return; + + /* First thing to do is reduce the separation between columns */ + pData->nMargin = 0; + if( p->spec.eStyle==QRF_STYLE_Column ){ + sepW = pData->nCol - 1; + }else{ + sepW = pData->nCol + 1; + if( p->spec.bBorder==QRF_No ) sepW -= 2; + } + targetW = p->spec.nScreenWidth - sepW; + +#define MIN_SQUOZE 8 +#define MIN_EX_SQUOZE 16 + /* Reduce the width of the widest eligible column. A column is + ** eligible for narrowing if: + ** + ** * It is not a fixed-width column (a[0].fx is false) + ** * The current width is more than MIN_SQUOZE + ** * Either: + ** + The current width is more then MIN_EX_SQUOZE, or + ** + The current width is more than half the max width (a[].mxW) + ** + ** Keep making reductions until either no more reductions are + ** possible or until the size target is reached. + */ + while( sumW > targetW ){ + int gain, w; + int ix = -1; + int mx = 0; + for(i=0; i<nCol; i++){ + if( pData->a[i].fx==0 + && (w = pData->a[i].w)>mx + && w>MIN_SQUOZE + && (w>MIN_EX_SQUOZE || w*2>pData->a[i].mxW) + ){ + ix = i; + mx = w; + } + } + if( ix<0 ) break; + if( mx>=MIN_SQUOZE*2 ){ + gain = mx/2; + }else{ + gain = mx - MIN_SQUOZE; + } + if( sumW - gain < targetW ){ + gain = sumW - targetW; + } + sumW -= gain; + pData->a[ix].w -= gain; + pData->bMultiRow = 1; + } +} + +/* +** Columnar modes require that the entire query be evaluated first, with +** results written into memory, so that we can compute appropriate column +** widths. +*/ +static void qrfColumnar(Qrf *p){ + sqlite3_int64 i, j; /* Loop counters */ + const char *colSep = 0; /* Column separator text */ + const char *rowSep = 0; /* Row terminator text */ + const char *rowStart = 0; /* Row start text */ + int szColSep, szRowSep, szRowStart; /* Size in bytes of previous 3 */ + int rc; /* Result code */ + int nColumn = p->nCol; /* Number of columns */ + int bWW; /* True to do word-wrap */ + sqlite3_str *pStr; /* Temporary rendering */ + qrfColData data; /* Columnar layout data */ + int bRTrim; /* Trim trailing space */ + + rc = sqlite3_step(p->pStmt); + if( rc!=SQLITE_ROW || nColumn==0 ){ + return; /* No output */ + } + + /* Initialize the data container */ + memset(&data, 0, sizeof(data)); + data.nCol = p->nCol; + data.p = p; + data.a = sqlite3_malloc64( nColumn*sizeof(struct qrfPerCol) ); + if( data.a==0 ){ + qrfOom(p); + return; + } + memset(data.a, 0, nColumn*sizeof(struct qrfPerCol) ); + if( qrfColDataEnlarge(&data) ) return; + assert( data.az!=0 ); + + /* Load the column header names and all cell content into data */ + if( p->spec.bTitles==QRF_Yes ){ + unsigned char saved_eText = p->spec.eText; + p->spec.eText = p->spec.eTitle; + memset(data.abNum, 0, nColumn); + for(i=0; i<nColumn; i++){ + const char *z = (const char*)sqlite3_column_name(p->pStmt,i); + int nNL = 0; + int n, w; + pStr = sqlite3_str_new(p->db); + qrfEncodeText(p, pStr, z ? z : ""); + n = sqlite3_str_length(pStr); + qrfStrErr(p, pStr); + z = data.az[data.n] = sqlite3_str_finish(pStr); + if( p->spec.nTitleLimit ){ + nNL = 0; + data.aiWth[data.n] = w = qrfTitleLimit(data.az[data.n], + p->spec.nTitleLimit ); + }else{ + data.aiWth[data.n] = w = qrfDisplayWidth(z, n, &nNL); + } + data.n++; + if( w>data.a[i].mxW ) data.a[i].mxW = w; + if( nNL ) data.bMultiRow = 1; + } + p->spec.eText = saved_eText; + p->nRow++; + } + do{ + if( data.n+nColumn > data.nAlloc ){ + if( qrfColDataEnlarge(&data) ) return; + } + for(i=0; i<nColumn; i++){ + char *z; + int nNL = 0; + int n, w; + int eType = sqlite3_column_type(p->pStmt,i); + pStr = sqlite3_str_new(p->db); + qrfRenderValue(p, pStr, i); + n = sqlite3_str_length(pStr); + qrfStrErr(p, pStr); + z = data.az[data.n] = sqlite3_str_finish(pStr); + data.abNum[data.n] = eType==SQLITE_INTEGER || eType==SQLITE_FLOAT; + data.aiWth[data.n] = w = qrfDisplayWidth(z, n, &nNL); + data.n++; + if( w>data.a[i].mxW ) data.a[i].mxW = w; + if( nNL ) data.bMultiRow = 1; + } + p->nRow++; + }while( sqlite3_step(p->pStmt)==SQLITE_ROW && p->iErr==SQLITE_OK ); + if( p->iErr ){ + qrfColDataFree(&data); + return; + } + + /* Compute the width and alignment of every column */ + if( p->spec.bTitles==QRF_No ){ + qrfLoadAlignment(&data, p); + }else{ + unsigned char e; + if( p->spec.eTitleAlign==QRF_Auto ){ + e = QRF_ALIGN_Center; + }else{ + e = p->spec.eTitleAlign; + } + for(i=0; i<nColumn; i++) data.a[i].e = e; + } + + for(i=0; i<nColumn; i++){ + int w = 0; + if( i<p->spec.nWidth ){ + w = p->spec.aWidth[i]; + if( w==(-32768) ){ + w = 0; + if( p->spec.nAlign>i && (p->spec.aAlign[i] & QRF_ALIGN_HMASK)==0 ){ + data.a[i].e |= QRF_ALIGN_Right; + } + }else if( w<0 ){ + w = -w; + if( p->spec.nAlign>i && (p->spec.aAlign[i] & QRF_ALIGN_HMASK)==0 ){ + data.a[i].e |= QRF_ALIGN_Right; + } + } + if( w ) data.a[i].fx = 1; + } + if( w==0 ){ + w = data.a[i].mxW; + if( p->spec.nWrap>0 && w>p->spec.nWrap ){ + w = p->spec.nWrap; + data.bMultiRow = 1; + } + }else if( (data.bMultiRow==0 || w==1) && data.a[i].mxW>w ){ + data.bMultiRow = 1; + if( w==1 ){ + /* If aiWth[j] is 2 or more, then there might be a double-wide + ** character somewhere. So make the column width at least 2. */ + w = 2; + } + } + data.a[i].w = w; + } + + if( nColumn==1 + && data.n>1 + && p->spec.bSplitColumn==QRF_Yes + && p->spec.eStyle==QRF_STYLE_Column + && p->spec.bTitles==QRF_No + && p->spec.nScreenWidth>data.a[0].w+3 + ){ + /* Attempt to convert single-column tables into multi-column by + ** verticle wrapping, if the screen is wide enough and if the + ** bSplitColumn flag is set. */ + qrfSplitColumn(&data, p); + nColumn = data.nCol; + }else{ + /* Adjust the column widths due to screen width restrictions */ + qrfRestrictScreenWidth(&data, p); + } + + /* Draw the line across the top of the table. Also initialize + ** the row boundary and column separator texts. */ + switch( p->spec.eStyle ){ + case QRF_STYLE_Box: + if( data.nMargin ){ + rowStart = BOX_13 " "; + colSep = " " BOX_13 " "; + rowSep = " " BOX_13 "\n"; + }else{ + rowStart = BOX_13; + colSep = BOX_13; + rowSep = BOX_13 "\n"; + } + if( p->spec.bBorder==QRF_No){ + rowStart += 3; + rowSep = "\n"; + }else{ + qrfBoxSeparator(p->pOut, &data, BOX_R23, BOX_234, BOX_R34, 0); + } + break; + case QRF_STYLE_Table: + if( data.nMargin ){ + rowStart = "| "; + colSep = " | "; + rowSep = " |\n"; + }else{ + rowStart = "|"; + colSep = "|"; + rowSep = "|\n"; + } + if( p->spec.bBorder==QRF_No ){ + rowStart += 1; + rowSep = "\n"; + }else{ + qrfRowSeparator(p->pOut, &data, '+'); + } + break; + case QRF_STYLE_Column: { + static const char zSpace[] = " "; + rowStart = ""; + if( data.nMargin<2 ){ + colSep = " "; + }else if( data.nMargin<=5 ){ + colSep = &zSpace[5-data.nMargin]; + }else{ + colSep = zSpace; + } + rowSep = "\n"; + break; + } + default: /*case QRF_STYLE_Markdown:*/ + if( data.nMargin ){ + rowStart = "| "; + colSep = " | "; + rowSep = " |\n"; + }else{ + rowStart = "|"; + colSep = "|"; + rowSep = "|\n"; + } + break; + } + szRowStart = (int)strlen(rowStart); + szRowSep = (int)strlen(rowSep); + szColSep = (int)strlen(colSep); + + bWW = (p->spec.bWordWrap==QRF_Yes && data.bMultiRow); + if( p->spec.eStyle==QRF_STYLE_Column + || (p->spec.bBorder==QRF_No + && (p->spec.eStyle==QRF_STYLE_Box || p->spec.eStyle==QRF_STYLE_Table) + ) + ){ + bRTrim = 1; + }else{ + bRTrim = 0; + } + for(i=0; i<data.n && sqlite3_str_errcode(p->pOut)==SQLITE_OK; i+=nColumn){ + int bMore; + int nRow = 0; + + /* Draw a single row of the table. This might be the title line + ** (if there is a title line) or a row in the body of the table. + ** The column number will be j. The row number is i/nColumn. + */ + for(j=0; j<nColumn; j++){ + data.a[j].z = data.az[i+j]; + if( data.a[j].z==0 ) data.a[j].z = ""; + data.a[j].bNum = data.abNum[i+j]; + } + do{ + sqlite3_str_append(p->pOut, rowStart, szRowStart); + bMore = 0; + for(j=0; j<nColumn; j++){ + int nThis = 0; + int nWide = 0; + int iNext = 0; + int nWS; + qrfWrapLine(data.a[j].z, data.a[j].w, bWW, &nThis, &nWide, &iNext); + nWS = data.a[j].w - nWide; + qrfPrintAligned(p->pOut, &data.a[j], nThis, nWS); + data.a[j].z += iNext; + if( data.a[j].z[0]!=0 ){ + bMore = 1; + } + if( j<nColumn-1 ){ + sqlite3_str_append(p->pOut, colSep, szColSep); + }else{ + if( bRTrim ) qrfRTrim(p->pOut); + sqlite3_str_append(p->pOut, rowSep, szRowSep); + } + } + }while( bMore && ++nRow < p->mxHeight ); + if( bMore ){ + /* This row was terminated by nLineLimit. Show ellipsis. */ + sqlite3_str_append(p->pOut, rowStart, szRowStart); + for(j=0; j<nColumn; j++){ + if( data.a[j].z[0]==0 ){ + sqlite3_str_appendchar(p->pOut, data.a[j].w, ' '); + }else{ + int nE = 3; + if( nE>data.a[j].w ) nE = data.a[j].w; + data.a[j].z = "..."; + qrfPrintAligned(p->pOut, &data.a[j], nE, data.a[j].w-nE); + } + if( j<nColumn-1 ){ + sqlite3_str_append(p->pOut, colSep, szColSep); + }else{ + if( bRTrim ) qrfRTrim(p->pOut); + sqlite3_str_append(p->pOut, rowSep, szRowSep); + } + } + } + + /* Draw either (1) the separator between the title line and the body + ** of the table, or (2) separators between individual rows of the table + ** body. isTitleDataSeparator will be true if we are doing (1). + */ + if( (i==0 || data.bMultiRow) && i+nColumn<data.n ){ + int isTitleDataSeparator = (i==0 && p->spec.bTitles==QRF_Yes); + if( isTitleDataSeparator ){ + qrfLoadAlignment(&data, p); + } + switch( p->spec.eStyle ){ + case QRF_STYLE_Table: { + if( isTitleDataSeparator || data.bMultiRow ){ + qrfRowSeparator(p->pOut, &data, '+'); + } + break; + } + case QRF_STYLE_Box: { + if( isTitleDataSeparator ){ + qrfBoxSeparator(p->pOut, &data, DBL_123, DBL_1234, DBL_134, 1); + }else if( data.bMultiRow ){ + qrfBoxSeparator(p->pOut, &data, BOX_123, BOX_1234, BOX_134, 0); + } + break; + } + case QRF_STYLE_Markdown: { + if( isTitleDataSeparator ){ + qrfRowSeparator(p->pOut, &data, '|'); + } + break; + } + case QRF_STYLE_Column: { + if( isTitleDataSeparator ){ + for(j=0; j<nColumn; j++){ + sqlite3_str_appendchar(p->pOut, data.a[j].w, '-'); + if( j<nColumn-1 ){ + sqlite3_str_append(p->pOut, colSep, szColSep); + }else{ + qrfRTrim(p->pOut); + sqlite3_str_append(p->pOut, rowSep, szRowSep); + } + } + }else if( data.bMultiRow ){ + qrfRTrim(p->pOut); + sqlite3_str_append(p->pOut, "\n", 1); + } + break; + } + } + } + } + + /* Draw the line across the bottom of the table */ + if( p->spec.bBorder!=QRF_No ){ + switch( p->spec.eStyle ){ + case QRF_STYLE_Box: + qrfBoxSeparator(p->pOut, &data, BOX_R12, BOX_124, BOX_R14, 0); + break; + case QRF_STYLE_Table: + qrfRowSeparator(p->pOut, &data, '+'); + break; + } + } + qrfWrite(p); + + qrfColDataFree(&data); + return; +} + +/* +** Parameter azArray points to a zero-terminated array of strings. zStr +** points to a single nul-terminated string. Return non-zero if zStr +** is equal, according to strcmp(), to any of the strings in the array. +** Otherwise, return zero. +*/ +static int qrfStringInArray(const char *zStr, const char **azArray){ + int i; + if( zStr==0 ) return 0; + for(i=0; azArray[i]; i++){ + if( 0==strcmp(zStr, azArray[i]) ) return 1; + } + return 0; +} + +/* +** Print out an EXPLAIN with indentation. This is a two-pass algorithm. +** +** On the first pass, we compute aiIndent[iOp] which is the amount of +** indentation to apply to the iOp-th opcode. The output actually occurs +** on the second pass. +** +** The indenting rules are: +** +** * For each "Next", "Prev", "VNext" or "VPrev" instruction, indent +** all opcodes that occur between the p2 jump destination and the opcode +** itself by 2 spaces. +** +** * Do the previous for "Return" instructions for when P2 is positive. +** See tag-20220407a in wherecode.c and vdbe.c. +** +** * For each "Goto", if the jump destination is earlier in the program +** and ends on one of: +** Yield SeekGt SeekLt RowSetRead Rewind +** or if the P1 parameter is one instead of zero, +** then indent all opcodes between the earlier instruction +** and "Goto" by 2 spaces. +*/ +static void qrfExplain(Qrf *p){ + int *abYield = 0; /* abYield[iOp] is rue if opcode iOp is an OP_Yield */ + int *aiIndent = 0; /* Indent the iOp-th opcode by aiIndent[iOp] */ + i64 nAlloc = 0; /* Allocated size of aiIndent[], abYield */ + int nIndent = 0; /* Number of entries in aiIndent[] */ + int iOp; /* Opcode number */ + int i; /* Column loop counter */ + + const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", + "Return", 0 }; + const char *azYield[] = { "Yield", "SeekLT", "SeekGT", "RowSetRead", + "Rewind", 0 }; + const char *azGoto[] = { "Goto", 0 }; + + /* The caller guarantees that the leftmost 4 columns of the statement + ** passed to this function are equivalent to the leftmost 4 columns + ** of EXPLAIN statement output. In practice the statement may be + ** an EXPLAIN, or it may be a query on the bytecode() virtual table. */ + assert( sqlite3_column_count(p->pStmt)>=4 ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(p->pStmt, 0), "addr" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(p->pStmt, 1), "opcode" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(p->pStmt, 2), "p1" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(p->pStmt, 3), "p2" ) ); + + for(iOp=0; SQLITE_ROW==sqlite3_step(p->pStmt) && !p->iErr; iOp++){ + int iAddr = sqlite3_column_int(p->pStmt, 0); + const char *zOp = (const char*)sqlite3_column_text(p->pStmt, 1); + int p1 = sqlite3_column_int(p->pStmt, 2); + int p2 = sqlite3_column_int(p->pStmt, 3); + + /* Assuming that p2 is an instruction address, set variable p2op to the + ** index of that instruction in the aiIndent[] array. p2 and p2op may be + ** different if the current instruction is part of a sub-program generated + ** by an SQL trigger or foreign key. */ + int p2op = (p2 + (iOp-iAddr)); + + /* Grow the aiIndent array as required */ + if( iOp>=nAlloc ){ + nAlloc += 100; + aiIndent = (int*)sqlite3_realloc64(aiIndent, nAlloc*sizeof(int)); + abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int)); + if( aiIndent==0 || abYield==0 ){ + qrfOom(p); + sqlite3_free(aiIndent); + sqlite3_free(abYield); + return; + } + } + + abYield[iOp] = qrfStringInArray(zOp, azYield); + aiIndent[iOp] = 0; + nIndent = iOp+1; + if( qrfStringInArray(zOp, azNext) && p2op>0 ){ + for(i=p2op; i<iOp; i++) aiIndent[i] += 2; + } + if( qrfStringInArray(zOp, azGoto) && p2op<iOp && (abYield[p2op] || p1) ){ + for(i=p2op; i<iOp; i++) aiIndent[i] += 2; + } + } + sqlite3_free(abYield); + + /* Second pass. Actually generate output */ + sqlite3_reset(p->pStmt); + if( p->iErr==SQLITE_OK ){ + static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13}; + static const int aExplainMap[] = {0, 1, 2, 3, 4, 5, 6, 7 }; + static const int aScanExpWidth[] = {4,15, 6, 13, 4, 4, 4, 13, 2, 13}; + static const int aScanExpMap[] = {0, 9, 8, 1, 2, 3, 4, 5, 6, 7 }; + const int *aWidth = aExplainWidth; + const int *aMap = aExplainMap; + int nWidth = sizeof(aExplainWidth)/sizeof(int); + int iIndent = 1; + int nArg = p->nCol; + if( p->spec.eStyle==QRF_STYLE_StatsVm ){ + aWidth = aScanExpWidth; + aMap = aScanExpMap; + nWidth = sizeof(aScanExpWidth)/sizeof(int); + iIndent = 3; + } + if( nArg>nWidth ) nArg = nWidth; + + for(iOp=0; sqlite3_step(p->pStmt)==SQLITE_ROW && !p->iErr; iOp++){ + /* If this is the first row seen, print out the headers */ + if( iOp==0 ){ + for(i=0; i<nArg; i++){ + const char *zCol = sqlite3_column_name(p->pStmt, aMap[i]); + qrfWidthPrint(p,p->pOut, aWidth[i], zCol); + if( i==nArg-1 ){ + sqlite3_str_append(p->pOut, "\n", 1); + }else{ + sqlite3_str_append(p->pOut, " ", 2); + } + } + for(i=0; i<nArg; i++){ + sqlite3_str_appendf(p->pOut, "%.*c", aWidth[i], '-'); + if( i==nArg-1 ){ + sqlite3_str_append(p->pOut, "\n", 1); + }else{ + sqlite3_str_append(p->pOut, " ", 2); + } + } + } + + for(i=0; i<nArg; i++){ + const char *zSep = " "; + int w = aWidth[i]; + const char *zVal = (const char*)sqlite3_column_text(p->pStmt, aMap[i]); + int len; + if( i==nArg-1 ) w = 0; + if( zVal==0 ) zVal = ""; + len = (int)sqlite3_qrf_wcswidth(zVal); + if( len>w ){ + w = len; + zSep = " "; + } + if( i==iIndent && aiIndent && iOp<nIndent ){ + sqlite3_str_appendchar(p->pOut, aiIndent[iOp], ' '); + } + qrfWidthPrint(p, p->pOut, w, zVal); + if( i==nArg-1 ){ + sqlite3_str_append(p->pOut, "\n", 1); + }else{ + sqlite3_str_appendall(p->pOut, zSep); + } + } + p->nRow++; + } + qrfWrite(p); + } + sqlite3_free(aiIndent); +} + +/* +** Do a "scanstatus vm" style EXPLAIN listing on p->pStmt. +** +** p->pStmt is probably not an EXPLAIN query. Instead, construct a +** new query that is a bytecode() rendering of p->pStmt with extra +** columns for the "scanstatus vm" outputs, and run the results of +** that new query through the normal EXPLAIN formatting. +*/ +static void qrfScanStatusVm(Qrf *p){ + sqlite3_stmt *pOrigStmt = p->pStmt; + sqlite3_stmt *pExplain; + int rc; + static const char *zSql = + " SELECT addr, opcode, p1, p2, p3, p4, p5, comment, nexec," + " format('% 6s (%.2f%%)'," + " CASE WHEN ncycle<100_000 THEN ncycle || ' '" + " WHEN ncycle<100_000_000 THEN (ncycle/1_000) || 'K'" + " WHEN ncycle<100_000_000_000 THEN (ncycle/1_000_000) || 'M'" + " ELSE (ncycle/1000_000_000) || 'G' END," + " ncycle*100.0/(sum(ncycle) OVER ())" + " ) AS cycles" + " FROM bytecode(?1)"; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pExplain, 0); + if( rc ){ + qrfError(p, rc, "%s", sqlite3_errmsg(p->db)); + sqlite3_finalize(pExplain); + return; + } + sqlite3_bind_pointer(pExplain, 1, pOrigStmt, "stmt-pointer", 0); + p->pStmt = pExplain; + p->nCol = 10; + qrfExplain(p); + sqlite3_finalize(pExplain); + p->pStmt = pOrigStmt; +} + +/* +** Attempt to determine if identifier zName needs to be quoted, either +** because it contains non-alphanumeric characters, or because it is an +** SQLite keyword. Be conservative in this estimate: When in doubt assume +** that quoting is required. +** +** Return 1 if quoting is required. Return 0 if no quoting is required. +*/ + +static int qrf_need_quote(const char *zName){ + int i; + const unsigned char *z = (const unsigned char*)zName; + if( z==0 ) return 1; + if( !qrfAlpha(z[0]) ) return 1; + for(i=0; z[i]; i++){ + if( !qrfAlnum(z[i]) ) return 1; + } + return sqlite3_keyword_check(zName, i)!=0; +} + +/* +** Helper function for QRF_STYLE_Json and QRF_STYLE_JObject. +** The initial "{" for a JSON object that will contain row content +** has been output. Now output all the content. +*/ +static void qrfOneJsonRow(Qrf *p){ + int i, nItem; + for(nItem=i=0; i<p->nCol; i++){ + const char *zCName; + zCName = sqlite3_column_name(p->pStmt, i); + if( nItem>0 ) sqlite3_str_append(p->pOut, ",", 1); + nItem++; + qrfEncodeText(p, p->pOut, zCName); + sqlite3_str_append(p->pOut, ":", 1); + qrfRenderValue(p, p->pOut, i); + } + qrfWrite(p); +} + +/* +** Render a single row of output for non-columnar styles - any +** style that lets us render row by row as the content is received +** from the query. +*/ +static void qrfOneSimpleRow(Qrf *p){ + int i; + switch( p->spec.eStyle ){ + case QRF_STYLE_Off: + case QRF_STYLE_Count: { + /* No-op */ + break; + } + case QRF_STYLE_Json: { + if( p->nRow==0 ){ + sqlite3_str_append(p->pOut, "[{", 2); + }else{ + sqlite3_str_append(p->pOut, "},\n{", 4); + } + qrfOneJsonRow(p); + break; + } + case QRF_STYLE_JObject: { + if( p->nRow==0 ){ + sqlite3_str_append(p->pOut, "{", 1); + }else{ + sqlite3_str_append(p->pOut, "}\n{", 3); + } + qrfOneJsonRow(p); + break; + } + case QRF_STYLE_Html: { + if( p->nRow==0 && p->spec.bTitles==QRF_Yes ){ + sqlite3_str_append(p->pOut, "<TR>", 4); + for(i=0; i<p->nCol; i++){ + const char *zCName = sqlite3_column_name(p->pStmt, i); + sqlite3_str_append(p->pOut, "\n<TH>", 5); + qrfEncodeText(p, p->pOut, zCName); + } + sqlite3_str_append(p->pOut, "\n</TR>\n", 7); + } + sqlite3_str_append(p->pOut, "<TR>", 4); + for(i=0; i<p->nCol; i++){ + sqlite3_str_append(p->pOut, "\n<TD>", 5); + qrfRenderValue(p, p->pOut, i); + } + sqlite3_str_append(p->pOut, "\n</TR>\n", 7); + qrfWrite(p); + break; + } + case QRF_STYLE_Insert: { + unsigned int mxIns = p->spec.nMultiInsert; + int szStart = sqlite3_str_length(p->pOut); + if( p->u.nIns==0 || p->u.nIns>=mxIns ){ + if( p->u.nIns ){ + sqlite3_str_append(p->pOut, ";\n", 2); + p->u.nIns = 0; + } + if( qrf_need_quote(p->spec.zTableName) ){ + sqlite3_str_appendf(p->pOut,"INSERT INTO \"%w\"",p->spec.zTableName); + }else{ + sqlite3_str_appendf(p->pOut,"INSERT INTO %s",p->spec.zTableName); + } + if( p->spec.bTitles==QRF_Yes ){ + for(i=0; i<p->nCol; i++){ + const char *zCName = sqlite3_column_name(p->pStmt, i); + if( qrf_need_quote(zCName) ){ + sqlite3_str_appendf(p->pOut, "%c\"%w\"", + i==0 ? '(' : ',', zCName); + }else{ + sqlite3_str_appendf(p->pOut, "%c%s", + i==0 ? '(' : ',', zCName); + } + } + sqlite3_str_append(p->pOut, ")", 1); + } + sqlite3_str_append(p->pOut," VALUES(", 8); + }else{ + sqlite3_str_append(p->pOut,",\n (", 5); + } + for(i=0; i<p->nCol; i++){ + if( i>0 ) sqlite3_str_append(p->pOut, ",", 1); + qrfRenderValue(p, p->pOut, i); + } + p->u.nIns += sqlite3_str_length(p->pOut) + 2 - szStart; + if( p->u.nIns>=mxIns ){ + sqlite3_str_append(p->pOut, ");\n", 3); + p->u.nIns = 0; + }else{ + sqlite3_str_append(p->pOut, ")", 1); + } + qrfWrite(p); + break; + } + case QRF_STYLE_Line: { + sqlite3_str *pVal; + int mxW; + int bWW; + int nSep; + if( p->u.sLine.azCol==0 ){ + p->u.sLine.azCol = sqlite3_malloc64( p->nCol*sizeof(char*) ); + if( p->u.sLine.azCol==0 ){ + qrfOom(p); + break; + } + p->u.sLine.mxColWth = 0; + for(i=0; i<p->nCol; i++){ + int sz; + const char *zCName = sqlite3_column_name(p->pStmt, i); + if( zCName==0 ) zCName = "unknown"; + p->u.sLine.azCol[i] = sqlite3_mprintf("%s", zCName); + if( p->spec.nTitleLimit>0 ){ + (void)qrfTitleLimit(p->u.sLine.azCol[i], p->spec.nTitleLimit); + } + sz = (int)sqlite3_qrf_wcswidth(p->u.sLine.azCol[i]); + if( sz > p->u.sLine.mxColWth ) p->u.sLine.mxColWth = sz; + } + } + if( p->nRow ) sqlite3_str_append(p->pOut, "\n", 1); + pVal = sqlite3_str_new(p->db); + nSep = (int)strlen(p->spec.zColumnSep); + mxW = p->mxWidth - (nSep + p->u.sLine.mxColWth); + bWW = p->spec.bWordWrap==QRF_Yes; + for(i=0; i<p->nCol; i++){ + const char *zVal; + int cnt = 0; + qrfWidthPrint(p, p->pOut, -p->u.sLine.mxColWth, p->u.sLine.azCol[i]); + sqlite3_str_append(p->pOut, p->spec.zColumnSep, nSep); + qrfRenderValue(p, pVal, i); + zVal = sqlite3_str_value(pVal); + if( zVal==0 ) zVal = ""; + do{ + int nThis, nWide, iNext; + qrfWrapLine(zVal, mxW, bWW, &nThis, &nWide, &iNext); + if( cnt ){ + sqlite3_str_appendchar(p->pOut,p->u.sLine.mxColWth+nSep,' '); + } + cnt++; + if( cnt>p->mxHeight ){ + zVal = "..."; + nThis = iNext = 3; + } + sqlite3_str_append(p->pOut, zVal, nThis); + sqlite3_str_append(p->pOut, "\n", 1); + zVal += iNext; + }while( zVal[0] ); + sqlite3_str_reset(pVal); + } + qrfStrErr(p, pVal); + sqlite3_free(sqlite3_str_finish(pVal)); + qrfWrite(p); + break; + } + case QRF_STYLE_Eqp: { + const char *zEqpLine = (const char*)sqlite3_column_text(p->pStmt,3); + int iEqpId = sqlite3_column_int(p->pStmt, 0); + int iParentId = sqlite3_column_int(p->pStmt, 1); + if( zEqpLine==0 ) zEqpLine = ""; + if( zEqpLine[0]=='-' ) qrfEqpRender(p, 0); + qrfEqpAppend(p, iEqpId, iParentId, zEqpLine); + break; + } + default: { /* QRF_STYLE_List */ + if( p->nRow==0 && p->spec.bTitles==QRF_Yes ){ + int saved_eText = p->spec.eText; + p->spec.eText = p->spec.eTitle; + for(i=0; i<p->nCol; i++){ + const char *zCName = sqlite3_column_name(p->pStmt, i); + if( i>0 ) sqlite3_str_appendall(p->pOut, p->spec.zColumnSep); + qrfEncodeText(p, p->pOut, zCName); + } + sqlite3_str_appendall(p->pOut, p->spec.zRowSep); + qrfWrite(p); + p->spec.eText = saved_eText; + } + for(i=0; i<p->nCol; i++){ + if( i>0 ) sqlite3_str_appendall(p->pOut, p->spec.zColumnSep); + qrfRenderValue(p, p->pOut, i); + } + sqlite3_str_appendall(p->pOut, p->spec.zRowSep); + qrfWrite(p); + break; + } + } + p->nRow++; +} + +/* +** Initialize the internal Qrf object. +*/ +static void qrfInitialize( + Qrf *p, /* State object to be initialized */ + sqlite3_stmt *pStmt, /* Query whose output to be formatted */ + const sqlite3_qrf_spec *pSpec, /* Format specification */ + char **pzErr /* Write errors here */ +){ + size_t sz; /* Size of pSpec[], based on pSpec->iVersion */ + memset(p, 0, sizeof(*p)); + p->pzErr = pzErr; + if( pSpec->iVersion>1 ){ + qrfError(p, SQLITE_ERROR, + "unusable sqlite3_qrf_spec.iVersion (%d)", + pSpec->iVersion); + return; + } + p->pStmt = pStmt; + p->db = sqlite3_db_handle(pStmt); + p->pOut = sqlite3_str_new(p->db); + if( p->pOut==0 ){ + qrfOom(p); + return; + } + p->iErr = SQLITE_OK; + p->nCol = sqlite3_column_count(p->pStmt); + p->nRow = 0; + sz = sizeof(sqlite3_qrf_spec); + memcpy(&p->spec, pSpec, sz); + if( p->spec.zNull==0 ) p->spec.zNull = ""; + p->mxWidth = p->spec.nScreenWidth; + if( p->mxWidth<=0 ) p->mxWidth = QRF_MAX_WIDTH; + p->mxHeight = p->spec.nLineLimit; + if( p->mxHeight<=0 ) p->mxHeight = 2147483647; + if( p->spec.eStyle>QRF_STYLE_Table ) p->spec.eStyle = QRF_Auto; + if( p->spec.eEsc>QRF_ESC_Symbol ) p->spec.eEsc = QRF_Auto; + if( p->spec.eText>QRF_TEXT_Relaxed ) p->spec.eText = QRF_Auto; + if( p->spec.eTitle>QRF_TEXT_Relaxed ) p->spec.eTitle = QRF_Auto; + if( p->spec.eBlob>QRF_BLOB_Size ) p->spec.eBlob = QRF_Auto; +qrf_reinit: + switch( p->spec.eStyle ){ + case QRF_Auto: { + switch( sqlite3_stmt_isexplain(pStmt) ){ + case 0: p->spec.eStyle = QRF_STYLE_Box; break; + case 1: p->spec.eStyle = QRF_STYLE_Explain; break; + default: p->spec.eStyle = QRF_STYLE_Eqp; break; + } + goto qrf_reinit; + } + case QRF_STYLE_List: { + if( p->spec.zColumnSep==0 ) p->spec.zColumnSep = "|"; + if( p->spec.zRowSep==0 ) p->spec.zRowSep = "\n"; + break; + } + case QRF_STYLE_JObject: + case QRF_STYLE_Json: { + p->spec.eText = QRF_TEXT_Json; + p->spec.zNull = "null"; + break; + } + case QRF_STYLE_Html: { + p->spec.eText = QRF_TEXT_Html; + p->spec.zNull = "null"; + break; + } + case QRF_STYLE_Insert: { + p->spec.eText = QRF_TEXT_Sql; + p->spec.zNull = "NULL"; + if( p->spec.zTableName==0 || p->spec.zTableName[0]==0 ){ + p->spec.zTableName = "tab"; + } + p->u.nIns = 0; + break; + } + case QRF_STYLE_Line: { + if( p->spec.zColumnSep==0 ){ + p->spec.zColumnSep = ": "; + } + break; + } + case QRF_STYLE_Csv: { + p->spec.eStyle = QRF_STYLE_List; + p->spec.eText = QRF_TEXT_Csv; + p->spec.zColumnSep = ","; + p->spec.zRowSep = "\r\n"; + p->spec.zNull = ""; + break; + } + case QRF_STYLE_Quote: { + p->spec.eText = QRF_TEXT_Sql; + p->spec.zNull = "NULL"; + p->spec.zColumnSep = ","; + p->spec.zRowSep = "\n"; + break; + } + case QRF_STYLE_Eqp: { + int expMode = sqlite3_stmt_isexplain(p->pStmt); + if( expMode!=2 ){ + sqlite3_stmt_explain(p->pStmt, 2); + p->expMode = expMode+1; + } + break; + } + case QRF_STYLE_Explain: { + int expMode = sqlite3_stmt_isexplain(p->pStmt); + if( expMode!=1 ){ + sqlite3_stmt_explain(p->pStmt, 1); + p->expMode = expMode+1; + } + break; + } + } + if( p->spec.eEsc==QRF_Auto ){ + p->spec.eEsc = QRF_ESC_Ascii; + } + if( p->spec.eText==QRF_Auto ){ + p->spec.eText = QRF_TEXT_Plain; + } + if( p->spec.eTitle==QRF_Auto ){ + switch( p->spec.eStyle ){ + case QRF_STYLE_Box: + case QRF_STYLE_Column: + case QRF_STYLE_Table: + p->spec.eTitle = QRF_TEXT_Plain; + break; + default: + p->spec.eTitle = p->spec.eText; + break; + } + } + if( p->spec.eBlob==QRF_Auto ){ + switch( p->spec.eText ){ + case QRF_TEXT_Sql: p->spec.eBlob = QRF_BLOB_Sql; break; + case QRF_TEXT_Csv: p->spec.eBlob = QRF_BLOB_Tcl; break; + case QRF_TEXT_Tcl: p->spec.eBlob = QRF_BLOB_Tcl; break; + case QRF_TEXT_Json: p->spec.eBlob = QRF_BLOB_Json; break; + default: p->spec.eBlob = QRF_BLOB_Text; break; + } + } + if( p->spec.bTitles==QRF_Auto ){ + switch( p->spec.eStyle ){ + case QRF_STYLE_Box: + case QRF_STYLE_Csv: + case QRF_STYLE_Column: + case QRF_STYLE_Table: + case QRF_STYLE_Markdown: + p->spec.bTitles = QRF_Yes; + break; + default: + p->spec.bTitles = QRF_No; + break; + } + } + if( p->spec.bWordWrap==QRF_Auto ){ + p->spec.bWordWrap = QRF_Yes; + } + if( p->spec.bTextJsonb==QRF_Auto ){ + p->spec.bTextJsonb = QRF_No; + } + if( p->spec.zColumnSep==0 ) p->spec.zColumnSep = ","; + if( p->spec.zRowSep==0 ) p->spec.zRowSep = "\n"; +} + +/* +** Finish rendering the results +*/ +static void qrfFinalize(Qrf *p){ + switch( p->spec.eStyle ){ + case QRF_STYLE_Count: { + sqlite3_str_appendf(p->pOut, "%lld\n", p->nRow); + break; + } + case QRF_STYLE_Json: { + if( p->nRow>0 ){ + sqlite3_str_append(p->pOut, "}]\n", 3); + } + break; + } + case QRF_STYLE_JObject: { + if( p->nRow>0 ){ + sqlite3_str_append(p->pOut, "}\n", 2); + } + break; + } + case QRF_STYLE_Insert: { + if( p->u.nIns ){ + sqlite3_str_append(p->pOut, ";\n", 2); + } + break; + } + case QRF_STYLE_Line: { + if( p->u.sLine.azCol ){ + int i; + for(i=0; i<p->nCol; i++) sqlite3_free(p->u.sLine.azCol[i]); + sqlite3_free(p->u.sLine.azCol); + } + break; + } + case QRF_STYLE_Stats: + case QRF_STYLE_StatsEst: { + i64 nCycle = 0; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + sqlite3_stmt_scanstatus_v2(p->pStmt, -1, SQLITE_SCANSTAT_NCYCLE, + SQLITE_SCANSTAT_COMPLEX, (void*)&nCycle); +#endif + qrfEqpRender(p, nCycle); + break; + } + case QRF_STYLE_Eqp: { + qrfEqpRender(p, 0); + break; + } + } + qrfWrite(p); + qrfStrErr(p, p->pOut); + if( p->spec.pzOutput ){ + if( p->spec.pzOutput[0] ){ + sqlite3_int64 n, sz; + char *zCombined; + sz = strlen(p->spec.pzOutput[0]); + n = sqlite3_str_length(p->pOut); + zCombined = sqlite3_realloc64(p->spec.pzOutput[0], sz+n+1); + if( zCombined==0 ){ + sqlite3_free(p->spec.pzOutput[0]); + p->spec.pzOutput[0] = 0; + qrfOom(p); + }else{ + p->spec.pzOutput[0] = zCombined; + memcpy(zCombined+sz, sqlite3_str_value(p->pOut), n+1); + } + sqlite3_free(sqlite3_str_finish(p->pOut)); + }else{ + p->spec.pzOutput[0] = sqlite3_str_finish(p->pOut); + } + }else if( p->pOut ){ + sqlite3_free(sqlite3_str_finish(p->pOut)); + } + if( p->expMode>0 ){ + sqlite3_stmt_explain(p->pStmt, p->expMode-1); + } + if( p->actualWidth ){ + sqlite3_free(p->actualWidth); + } + if( p->pJTrans ){ + sqlite3 *db = sqlite3_db_handle(p->pJTrans); + sqlite3_finalize(p->pJTrans); + sqlite3_close(db); + } +} + +/* +** Run the prepared statement pStmt and format the results according +** to the specification provided in pSpec. Return an error code. +** If pzErr is not NULL and if an error occurs, write an error message +** into *pzErr. +*/ +int sqlite3_format_query_result( + sqlite3_stmt *pStmt, /* Statement to evaluate */ + const sqlite3_qrf_spec *pSpec, /* Format specification */ + char **pzErr /* Write error message here */ +){ + Qrf qrf; /* The new Qrf being created */ + + if( pStmt==0 ) return SQLITE_OK; /* No-op */ + if( pSpec==0 ) return SQLITE_MISUSE; + qrfInitialize(&qrf, pStmt, pSpec, pzErr); + switch( qrf.spec.eStyle ){ + case QRF_STYLE_Box: + case QRF_STYLE_Column: + case QRF_STYLE_Markdown: + case QRF_STYLE_Table: { + /* Columnar modes require that the entire query be evaluated and the + ** results stored in memory, so that we can compute column widths */ + qrfColumnar(&qrf); + break; + } + case QRF_STYLE_Explain: { + qrfExplain(&qrf); + break; + } + case QRF_STYLE_StatsVm: { + qrfScanStatusVm(&qrf); + break; + } + case QRF_STYLE_Stats: + case QRF_STYLE_StatsEst: { + qrfEqpStats(&qrf); + break; + } + default: { + /* Non-columnar modes where the output can occur after each row + ** of result is received */ + while( qrf.iErr==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + qrfOneSimpleRow(&qrf); + } + break; + } + } + qrfResetStmt(&qrf); + qrfFinalize(&qrf); + return qrf.iErr; +} diff --git a/ext/qrf/qrf.h b/ext/qrf/qrf.h new file mode 100644 index 000000000..e5171b01a --- /dev/null +++ b/ext/qrf/qrf.h @@ -0,0 +1,201 @@ +/* +** 2025-10-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Header file for the Query Result-Format or "qrf" utility library for +** SQLite. See the README.md documentation for additional information. +*/ +#ifndef SQLITE_QRF_H +#define SQLITE_QRF_H +#ifdef __cplusplus +extern "C" { +#endif +#include <stdlib.h> +#include "sqlite3.h" + +/* +** Specification used by clients to define the output format they want +*/ +typedef struct sqlite3_qrf_spec sqlite3_qrf_spec; +struct sqlite3_qrf_spec { + unsigned char iVersion; /* Version number of this structure */ + unsigned char eStyle; /* Formatting style. "box", "csv", etc... */ + unsigned char eEsc; /* How to escape control characters in text */ + unsigned char eText; /* Quoting style for text */ + unsigned char eTitle; /* Quating style for the text of column names */ + unsigned char eBlob; /* Quoting style for BLOBs */ + unsigned char bTitles; /* True to show column names */ + unsigned char bWordWrap; /* Try to wrap on word boundaries */ + unsigned char bTextJsonb; /* Render JSONB blobs as JSON text */ + unsigned char eDfltAlign; /* Default alignment, no covered by aAlignment */ + unsigned char eTitleAlign; /* Alignment for column headers */ + unsigned char bSplitColumn; /* Wrap single-column output into many columns */ + unsigned char bBorder; /* Show outer border in Box and Table styles */ + short int nWrap; /* Wrap columns wider than this */ + short int nScreenWidth; /* Maximum overall table width */ + short int nLineLimit; /* Maximum number of lines for any row */ + short int nTitleLimit; /* Maximum number of characters in a title */ + unsigned int nMultiInsert; /* Add rows to one INSERT until size exceeds */ + int nCharLimit; /* Maximum number of characters in a cell */ + int nWidth; /* Number of entries in aWidth[] */ + int nAlign; /* Number of entries in aAlignment[] */ + short int *aWidth; /* Column widths */ + unsigned char *aAlign; /* Column alignments */ + char *zColumnSep; /* Alternative column separator */ + char *zRowSep; /* Alternative row separator */ + char *zTableName; /* Output table name */ + char *zNull; /* Rendering of NULL */ + char *(*xRender)(void*,sqlite3_value*); /* Render a value */ + int (*xWrite)(void*,const char*,sqlite3_int64); /* Write output */ + void *pRenderArg; /* First argument to the xRender callback */ + void *pWriteArg; /* First argument to the xWrite callback */ + char **pzOutput; /* Storage location for output string */ + /* Additional fields may be added in the future */ +}; + +/* +** Interfaces +*/ +int sqlite3_format_query_result( + sqlite3_stmt *pStmt, /* SQL statement to run */ + const sqlite3_qrf_spec *pSpec, /* Result format specification */ + char **pzErr /* OUT: Write error message here */ +); + +/* +** Range of values for sqlite3_qrf_spec.aWidth[] entries and for +** sqlite3_qrf_spec.mxColWidth and .nScreenWidth +*/ +#define QRF_MAX_WIDTH 10000 +#define QRF_MIN_WIDTH 0 + +/* +** Output styles: +*/ +#define QRF_STYLE_Auto 0 /* Choose a style automatically */ +#define QRF_STYLE_Box 1 /* Unicode box-drawing characters */ +#define QRF_STYLE_Column 2 /* One record per line in neat columns */ +#define QRF_STYLE_Count 3 /* Output only a count of the rows of output */ +#define QRF_STYLE_Csv 4 /* Comma-separated-value */ +#define QRF_STYLE_Eqp 5 /* Format EXPLAIN QUERY PLAN output */ +#define QRF_STYLE_Explain 6 /* EXPLAIN output */ +#define QRF_STYLE_Html 7 /* Generate an XHTML table */ +#define QRF_STYLE_Insert 8 /* Generate SQL "insert" statements */ +#define QRF_STYLE_Json 9 /* Output is a list of JSON objects */ +#define QRF_STYLE_JObject 10 /* Independent JSON objects for each row */ +#define QRF_STYLE_Line 11 /* One column per line. */ +#define QRF_STYLE_List 12 /* One record per line with a separator */ +#define QRF_STYLE_Markdown 13 /* Markdown formatting */ +#define QRF_STYLE_Off 14 /* No query output shown */ +#define QRF_STYLE_Quote 15 /* SQL-quoted, comma-separated */ +#define QRF_STYLE_Stats 16 /* EQP-like output but with performance stats */ +#define QRF_STYLE_StatsEst 17 /* EQP-like output with planner estimates */ +#define QRF_STYLE_StatsVm 18 /* EXPLAIN-like output with performance stats */ +#define QRF_STYLE_Table 19 /* MySQL-style table formatting */ + +/* +** Quoting styles for text. +** Allowed values for sqlite3_qrf_spec.eText +*/ +#define QRF_TEXT_Auto 0 /* Choose text encoding automatically */ +#define QRF_TEXT_Plain 1 /* Literal text */ +#define QRF_TEXT_Sql 2 /* Quote as an SQL literal */ +#define QRF_TEXT_Csv 3 /* CSV-style quoting */ +#define QRF_TEXT_Html 4 /* HTML-style quoting */ +#define QRF_TEXT_Tcl 5 /* C/Tcl quoting */ +#define QRF_TEXT_Json 6 /* JSON quoting */ +#define QRF_TEXT_Relaxed 7 /* Relaxed SQL quoting */ + +/* +** Quoting styles for BLOBs +** Allowed values for sqlite3_qrf_spec.eBlob +*/ +#define QRF_BLOB_Auto 0 /* Determine BLOB quoting using eText */ +#define QRF_BLOB_Text 1 /* Display content exactly as it is */ +#define QRF_BLOB_Sql 2 /* Quote as an SQL literal */ +#define QRF_BLOB_Hex 3 /* Hexadecimal representation */ +#define QRF_BLOB_Tcl 4 /* "\000" notation */ +#define QRF_BLOB_Json 5 /* A JSON string */ +#define QRF_BLOB_Size 6 /* Display the blob size only */ + +/* +** Control-character escape modes. +** Allowed values for sqlite3_qrf_spec.eEsc +*/ +#define QRF_ESC_Auto 0 /* Choose the ctrl-char escape automatically */ +#define QRF_ESC_Off 1 /* Do not escape control characters */ +#define QRF_ESC_Ascii 2 /* Unix-style escapes. Ex: U+0007 shows ^G */ +#define QRF_ESC_Symbol 3 /* Unicode escapes. Ex: U+0007 shows U+2407 */ + +/* +** Allowed values for "boolean" fields, such as "bColumnNames", "bWordWrap", +** and "bTextJsonb". There is an extra "auto" variants so these are actually +** tri-state settings, not booleans. +*/ +#define QRF_SW_Auto 0 /* Let QRF choose the best value */ +#define QRF_SW_Off 1 /* This setting is forced off */ +#define QRF_SW_On 2 /* This setting is forced on */ +#define QRF_Auto 0 /* Alternate spelling for QRF_*_Auto */ +#define QRF_No 1 /* Alternate spelling for QRF_SW_Off */ +#define QRF_Yes 2 /* Alternate spelling for QRF_SW_On */ + +/* +** Possible alignment values alignment settings +** +** Horizontal Vertial +** ---------- -------- */ +#define QRF_ALIGN_Auto 0 /* auto auto */ +#define QRF_ALIGN_Left 1 /* left auto */ +#define QRF_ALIGN_Center 2 /* center auto */ +#define QRF_ALIGN_Right 3 /* right auto */ +#define QRF_ALIGN_Top 4 /* auto top */ +#define QRF_ALIGN_NW 5 /* left top */ +#define QRF_ALIGN_N 6 /* center top */ +#define QRF_ALIGN_NE 7 /* right top */ +#define QRF_ALIGN_Middle 8 /* auto middle */ +#define QRF_ALIGN_W 9 /* left middle */ +#define QRF_ALIGN_C 10 /* center middle */ +#define QRF_ALIGN_E 11 /* right middle */ +#define QRF_ALIGN_Bottom 12 /* auto bottom */ +#define QRF_ALIGN_SW 13 /* left bottom */ +#define QRF_ALIGN_S 14 /* center bottom */ +#define QRF_ALIGN_SE 15 /* right bottom */ +#define QRF_ALIGN_HMASK 3 /* Horizontal alignment mask */ +#define QRF_ALIGN_VMASK 12 /* Vertical alignment mask */ + +/* +** Auxiliary routines contined within this module that might be useful +** in other contexts, and which are therefore exported. +*/ +/* +** Return an estimate of the width, in columns, for the single Unicode +** character c. For normal characters, the answer is always 1. But the +** estimate might be 0 or 2 for zero-width and double-width characters. +** +** Different devices display unicode using different widths. So +** it is impossible to know that true display width with 100% accuracy. +** Inaccuracies in the width estimates might cause columns to be misaligned. +** Unfortunately, there is nothing we can do about that. +*/ +int sqlite3_qrf_wcwidth(int c); + +/* +** Return an estimate of the number of display columns used by the +** string in the argument. The width of individual characters is +** determined as for sqlite3_qrf_wcwidth(). VT100 escape code sequences +** are assigned a width of zero. +*/ +size_t sqlite3_qrf_wcswidth(const char*); + + +#ifdef __cplusplus +} +#endif +#endif /* !defined(SQLITE_QRF_H) */ diff --git a/ext/rbu/rbu11.test b/ext/rbu/rbu11.test index a42163cce..513ab29f6 100644 --- a/ext/rbu/rbu11.test +++ b/ext/rbu/rbu11.test @@ -192,4 +192,32 @@ do_test 4.7.2 { list [catch {rbu close} msg] $msg } {1 {SQLITE_ERROR - rbu_state mismatch error}} +#------------------------------------------------------------------------- +# https://sqlite.org/forum/info/6d0a31e22a435877 +# +reset_db +forcedelete rbu.db + +do_execsql_test 5.0 { + CREATE TABLE t1(a BLOB); + INSERT INTO t1 VALUES(x'41'); +} + +sqlite3 dbRbu rbu.db +dbRbu eval { + CREATE TABLE data_t1(a, rbu_control, rbu_rowid); + INSERT INTO data_t1 VALUES(X'310a313a5a337e7e7e7e7e40302c303b','f',1); +} +dbRbu close + +do_test 5.1 { + sqlite3rbu rbu test.db rbu.db + rbu step +} {SQLITE_ERROR} + +do_test 5.2 { + list [catch {rbu close} msg] $msg +} {1 {SQLITE_ERROR - corrupt fossil delta}} + + finish_test diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index e3bcd5fc7..f377d5c30 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -2269,8 +2269,8 @@ static char *rbuObjIterGetIndexWhere(sqlite3rbu *p, RbuObjIter *pIter){ /* If necessary, grow the pIter->aIdxCol[] array */ if( iIdxCol==nIdxAlloc ){ - RbuSpan *aIdxCol = (RbuSpan*)sqlite3_realloc( - pIter->aIdxCol, (nIdxAlloc+16)*sizeof(RbuSpan) + RbuSpan *aIdxCol = (RbuSpan*)sqlite3_realloc64( + pIter->aIdxCol, nIdxAlloc*sizeof(RbuSpan) + 16*sizeof(RbuSpan) ); if( aIdxCol==0 ){ rc = SQLITE_NOMEM; diff --git a/ext/repair/README.md b/ext/repair/README.md deleted file mode 100644 index 927ceb7c4..000000000 --- a/ext/repair/README.md +++ /dev/null @@ -1,16 +0,0 @@ -This folder contains extensions and utility programs intended to analyze -live database files, detect problems, and possibly fix them. - -As SQLite is being used on larger and larger databases, database sizes -are growing into the terabyte range. At that size, hardware malfunctions -and/or cosmic rays will occasionally corrupt a database file. Detecting -problems and fixing errors a terabyte-sized databases can take hours or days, -and it is undesirable to take applications that depend on the databases -off-line for such a long time. -The utilities in the folder are intended to provide mechanisms for -detecting and fixing problems in large databases while those databases -are in active use. - -The utilities and extensions in this folder are experimental and under -active development at the time of this writing (2017-10-12). If and when -they stabilize, this README will be updated to reflect that fact. diff --git a/ext/repair/checkfreelist.c b/ext/repair/checkfreelist.c deleted file mode 100644 index d1d3d5407..000000000 --- a/ext/repair/checkfreelist.c +++ /dev/null @@ -1,310 +0,0 @@ -/* -** 2017 October 11 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** -** This module exports a single C function: -** -** int sqlite3_check_freelist(sqlite3 *db, const char *zDb); -** -** This function checks the free-list in database zDb (one of "main", -** "temp", etc.) and reports any errors by invoking the sqlite3_log() -** function. It returns SQLITE_OK if successful, or an SQLite error -** code otherwise. It is not an error if the free-list is corrupted but -** no IO or OOM errors occur. -** -** If this file is compiled and loaded as an SQLite loadable extension, -** it adds an SQL function "checkfreelist" to the database handle, to -** be invoked as follows: -** -** SELECT checkfreelist(<database-name>); -** -** This function performs the same checks as sqlite3_check_freelist(), -** except that it returns all error messages as a single text value, -** separated by newline characters. If the freelist is not corrupted -** in any way, an empty string is returned. -** -** To compile this module for use as an SQLite loadable extension: -** -** gcc -Os -fPIC -shared checkfreelist.c -o checkfreelist.so -*/ - -#include "sqlite3ext.h" -SQLITE_EXTENSION_INIT1 - -#ifndef SQLITE_AMALGAMATION -# include <string.h> -# include <stdio.h> -# include <stdlib.h> -# include <assert.h> -# if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) -# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1 -# endif -# if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS) -# define ALWAYS(X) (1) -# define NEVER(X) (0) -# elif !defined(NDEBUG) -# define ALWAYS(X) ((X)?1:(assert(0),0)) -# define NEVER(X) ((X)?(assert(0),1):0) -# else -# define ALWAYS(X) (X) -# define NEVER(X) (X) -# endif - typedef unsigned char u8; - typedef unsigned short u16; - typedef unsigned int u32; -#define get4byte(x) ( \ - ((u32)((x)[0])<<24) + \ - ((u32)((x)[1])<<16) + \ - ((u32)((x)[2])<<8) + \ - ((u32)((x)[3])) \ -) -#endif - -/* -** Execute a single PRAGMA statement and return the integer value returned -** via output parameter (*pnOut). -** -** The SQL statement passed as the third argument should be a printf-style -** format string containing a single "%s" which will be replace by the -** value passed as the second argument. e.g. -** -** sqlGetInteger(db, "main", "PRAGMA %s.page_count", pnOut) -** -** executes "PRAGMA main.page_count" and stores the results in (*pnOut). -*/ -static int sqlGetInteger( - sqlite3 *db, /* Database handle */ - const char *zDb, /* Database name ("main", "temp" etc.) */ - const char *zFmt, /* SQL statement format */ - u32 *pnOut /* OUT: Integer value */ -){ - int rc, rc2; - char *zSql; - sqlite3_stmt *pStmt = 0; - int bOk = 0; - - zSql = sqlite3_mprintf(zFmt, zDb); - if( zSql==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - } - - if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - *pnOut = (u32)sqlite3_column_int(pStmt, 0); - bOk = 1; - } - - rc2 = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) rc = rc2; - if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_ERROR; - return rc; -} - -/* -** Argument zFmt must be a printf-style format string and must be -** followed by its required arguments. If argument pzOut is NULL, -** then the results of printf()ing the format string are passed to -** sqlite3_log(). Otherwise, they are appended to the string -** at (*pzOut). -*/ -static int checkFreelistError(char **pzOut, const char *zFmt, ...){ - int rc = SQLITE_OK; - char *zErr = 0; - va_list ap; - - va_start(ap, zFmt); - zErr = sqlite3_vmprintf(zFmt, ap); - if( zErr==0 ){ - rc = SQLITE_NOMEM; - }else{ - if( pzOut ){ - *pzOut = sqlite3_mprintf("%s%z%s", *pzOut?"\n":"", *pzOut, zErr); - if( *pzOut==0 ) rc = SQLITE_NOMEM; - }else{ - sqlite3_log(SQLITE_ERROR, "checkfreelist: %s", zErr); - } - sqlite3_free(zErr); - } - va_end(ap); - return rc; -} - -static int checkFreelist( - sqlite3 *db, - const char *zDb, - char **pzOut -){ - /* This query returns one row for each page on the free list. Each row has - ** two columns - the page number and page content. */ - const char *zTrunk = - "WITH freelist_trunk(i, d, n) AS (" - "SELECT 1, NULL, sqlite_readint32(data, 32) " - "FROM sqlite_dbpage(:1) WHERE pgno=1 " - "UNION ALL " - "SELECT n, data, sqlite_readint32(data) " - "FROM freelist_trunk, sqlite_dbpage(:1) WHERE pgno=n " - ")" - "SELECT i, d FROM freelist_trunk WHERE i!=1;"; - - int rc, rc2; /* Return code */ - sqlite3_stmt *pTrunk = 0; /* Compilation of zTrunk */ - u32 nPage = 0; /* Number of pages in db */ - u32 nExpected = 0; /* Expected number of free pages */ - u32 nFree = 0; /* Number of pages on free list */ - - if( zDb==0 ) zDb = "main"; - - if( (rc = sqlGetInteger(db, zDb, "PRAGMA %s.page_count", &nPage)) - || (rc = sqlGetInteger(db, zDb, "PRAGMA %s.freelist_count", &nExpected)) - ){ - return rc; - } - - rc = sqlite3_prepare_v2(db, zTrunk, -1, &pTrunk, 0); - if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_text(pTrunk, 1, zDb, -1, SQLITE_STATIC); - while( rc==SQLITE_OK && sqlite3_step(pTrunk)==SQLITE_ROW ){ - u32 i; - u32 iTrunk = (u32)sqlite3_column_int(pTrunk, 0); - const u8 *aData = (const u8*)sqlite3_column_blob(pTrunk, 1); - u32 nData = (u32)sqlite3_column_bytes(pTrunk, 1); - u32 iNext = get4byte(&aData[0]); - u32 nLeaf = get4byte(&aData[4]); - - if( nLeaf>((nData/4)-2-6) ){ - rc = checkFreelistError(pzOut, - "leaf count out of range (%d) on trunk page %d", - (int)nLeaf, (int)iTrunk - ); - nLeaf = (nData/4) - 2 - 6; - } - - nFree += 1+nLeaf; - if( iNext>nPage ){ - rc = checkFreelistError(pzOut, - "trunk page %d is out of range", (int)iNext - ); - } - - for(i=0; rc==SQLITE_OK && i<nLeaf; i++){ - u32 iLeaf = get4byte(&aData[8 + 4*i]); - if( iLeaf==0 || iLeaf>nPage ){ - rc = checkFreelistError(pzOut, - "leaf page %d is out of range (child %d of trunk page %d)", - (int)iLeaf, (int)i, (int)iTrunk - ); - } - } - } - - if( rc==SQLITE_OK && nFree!=nExpected ){ - rc = checkFreelistError(pzOut, - "free-list count mismatch: actual=%d header=%d", - (int)nFree, (int)nExpected - ); - } - - rc2 = sqlite3_finalize(pTrunk); - if( rc==SQLITE_OK ) rc = rc2; - return rc; -} - -int sqlite3_check_freelist(sqlite3 *db, const char *zDb){ - return checkFreelist(db, zDb, 0); -} - -static void checkfreelist_function( - sqlite3_context *pCtx, - int nArg, - sqlite3_value **apArg -){ - const char *zDb; - int rc; - char *zOut = 0; - sqlite3 *db = sqlite3_context_db_handle(pCtx); - - assert( nArg==1 ); - zDb = (const char*)sqlite3_value_text(apArg[0]); - rc = checkFreelist(db, zDb, &zOut); - if( rc==SQLITE_OK ){ - sqlite3_result_text(pCtx, zOut?zOut:"ok", -1, SQLITE_TRANSIENT); - }else{ - sqlite3_result_error_code(pCtx, rc); - } - - sqlite3_free(zOut); -} - -/* -** An SQL function invoked as follows: -** -** sqlite_readint32(BLOB) -- Decode 32-bit integer from start of blob -*/ -static void readint_function( - sqlite3_context *pCtx, - int nArg, - sqlite3_value **apArg -){ - const u8 *zBlob; - int nBlob; - int iOff = 0; - u32 iRet = 0; - - if( nArg!=1 && nArg!=2 ){ - sqlite3_result_error( - pCtx, "wrong number of arguments to function sqlite_readint32()", -1 - ); - return; - } - if( nArg==2 ){ - iOff = sqlite3_value_int(apArg[1]); - } - - zBlob = sqlite3_value_blob(apArg[0]); - nBlob = sqlite3_value_bytes(apArg[0]); - - if( nBlob>=(iOff+4) ){ - iRet = get4byte(&zBlob[iOff]); - } - - sqlite3_result_int64(pCtx, (sqlite3_int64)iRet); -} - -/* -** Register the SQL functions. -*/ -static int cflRegister(sqlite3 *db){ - int rc = sqlite3_create_function( - db, "sqlite_readint32", -1, SQLITE_UTF8, 0, readint_function, 0, 0 - ); - if( rc!=SQLITE_OK ) return rc; - rc = sqlite3_create_function( - db, "checkfreelist", 1, SQLITE_UTF8, 0, checkfreelist_function, 0, 0 - ); - return rc; -} - -/* -** Extension load function. -*/ -#ifdef _WIN32 -__declspec(dllexport) -#endif -int sqlite3_checkfreelist_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - SQLITE_EXTENSION_INIT2(pApi); - return cflRegister(db); -} diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c deleted file mode 100644 index ed30357e5..000000000 --- a/ext/repair/checkindex.c +++ /dev/null @@ -1,929 +0,0 @@ -/* -** 2017 October 27 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -*/ - -#include "sqlite3ext.h" -SQLITE_EXTENSION_INIT1 - -/* -** Stuff that is available inside the amalgamation, but which we need to -** declare ourselves if this module is compiled separately. -*/ -#ifndef SQLITE_AMALGAMATION -# include <string.h> -# include <stdio.h> -# include <stdlib.h> -# include <assert.h> -typedef unsigned char u8; -typedef unsigned short u16; -typedef unsigned int u32; -#define get4byte(x) ( \ - ((u32)((x)[0])<<24) + \ - ((u32)((x)[1])<<16) + \ - ((u32)((x)[2])<<8) + \ - ((u32)((x)[3])) \ -) -#endif - -typedef struct CidxTable CidxTable; -typedef struct CidxCursor CidxCursor; - -struct CidxTable { - sqlite3_vtab base; /* Base class. Must be first */ - sqlite3 *db; -}; - -struct CidxCursor { - sqlite3_vtab_cursor base; /* Base class. Must be first */ - sqlite3_int64 iRowid; /* Row number of the output */ - char *zIdxName; /* Copy of the index_name parameter */ - char *zAfterKey; /* Copy of the after_key parameter */ - sqlite3_stmt *pStmt; /* SQL statement that generates the output */ -}; - -typedef struct CidxColumn CidxColumn; -struct CidxColumn { - char *zExpr; /* Text for indexed expression */ - int bDesc; /* True for DESC columns, otherwise false */ - int bKey; /* Part of index, not PK */ -}; - -typedef struct CidxIndex CidxIndex; -struct CidxIndex { - char *zWhere; /* WHERE clause, if any */ - int nCol; /* Elements in aCol[] array */ - CidxColumn aCol[1]; /* Array of indexed columns */ -}; - -static void *cidxMalloc(int *pRc, int n){ - void *pRet = 0; - assert( n!=0 ); - if( *pRc==SQLITE_OK ){ - pRet = sqlite3_malloc(n); - if( pRet ){ - memset(pRet, 0, n); - }else{ - *pRc = SQLITE_NOMEM; - } - } - return pRet; -} - -static void cidxCursorError(CidxCursor *pCsr, const char *zFmt, ...){ - va_list ap; - va_start(ap, zFmt); - assert( pCsr->base.pVtab->zErrMsg==0 ); - pCsr->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap); - va_end(ap); -} - -/* -** Connect to the incremental_index_check virtual table. -*/ -static int cidxConnect( - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVtab, - char **pzErr -){ - int rc = SQLITE_OK; - CidxTable *pRet; - -#define IIC_ERRMSG 0 -#define IIC_CURRENT_KEY 1 -#define IIC_INDEX_NAME 2 -#define IIC_AFTER_KEY 3 -#define IIC_SCANNER_SQL 4 - rc = sqlite3_declare_vtab(db, - "CREATE TABLE xyz(" - " errmsg TEXT," /* Error message or NULL if everything is ok */ - " current_key TEXT," /* SQLite quote() text of key values */ - " index_name HIDDEN," /* IN: name of the index being scanned */ - " after_key HIDDEN," /* IN: Start scanning after this key */ - " scanner_sql HIDDEN" /* debugging info: SQL used for scanner */ - ")" - ); - pRet = cidxMalloc(&rc, sizeof(CidxTable)); - if( pRet ){ - pRet->db = db; - } - - *ppVtab = (sqlite3_vtab*)pRet; - return rc; -} - -/* -** Disconnect from or destroy an incremental_index_check virtual table. -*/ -static int cidxDisconnect(sqlite3_vtab *pVtab){ - CidxTable *pTab = (CidxTable*)pVtab; - sqlite3_free(pTab); - return SQLITE_OK; -} - -/* -** idxNum and idxStr are not used. There are only three possible plans, -** which are all distinguished by the number of parameters. -** -** No parameters: A degenerate plan. The result is zero rows. -** 1 Parameter: Scan all of the index starting with first entry -** 2 parameters: Scan the index starting after the "after_key". -** -** Provide successively smaller costs for each of these plans to encourage -** the query planner to select the one with the most parameters. -*/ -static int cidxBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pInfo){ - int iIdxName = -1; - int iAfterKey = -1; - int i; - - for(i=0; i<pInfo->nConstraint; i++){ - struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; - if( p->usable==0 ) continue; - if( p->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; - - if( p->iColumn==IIC_INDEX_NAME ){ - iIdxName = i; - } - if( p->iColumn==IIC_AFTER_KEY ){ - iAfterKey = i; - } - } - - if( iIdxName<0 ){ - pInfo->estimatedCost = 1000000000.0; - }else{ - pInfo->aConstraintUsage[iIdxName].argvIndex = 1; - pInfo->aConstraintUsage[iIdxName].omit = 1; - if( iAfterKey<0 ){ - pInfo->estimatedCost = 1000000.0; - }else{ - pInfo->aConstraintUsage[iAfterKey].argvIndex = 2; - pInfo->aConstraintUsage[iAfterKey].omit = 1; - pInfo->estimatedCost = 1000.0; - } - } - - return SQLITE_OK; -} - -/* -** Open a new btreeinfo cursor. -*/ -static int cidxOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ - CidxCursor *pRet; - int rc = SQLITE_OK; - - pRet = cidxMalloc(&rc, sizeof(CidxCursor)); - - *ppCursor = (sqlite3_vtab_cursor*)pRet; - return rc; -} - -/* -** Close a btreeinfo cursor. -*/ -static int cidxClose(sqlite3_vtab_cursor *pCursor){ - CidxCursor *pCsr = (CidxCursor*)pCursor; - sqlite3_finalize(pCsr->pStmt); - sqlite3_free(pCsr->zIdxName); - sqlite3_free(pCsr->zAfterKey); - sqlite3_free(pCsr); - return SQLITE_OK; -} - -/* -** Move a btreeinfo cursor to the next entry in the file. -*/ -static int cidxNext(sqlite3_vtab_cursor *pCursor){ - CidxCursor *pCsr = (CidxCursor*)pCursor; - int rc = sqlite3_step(pCsr->pStmt); - if( rc!=SQLITE_ROW ){ - rc = sqlite3_finalize(pCsr->pStmt); - pCsr->pStmt = 0; - if( rc!=SQLITE_OK ){ - sqlite3 *db = ((CidxTable*)pCsr->base.pVtab)->db; - cidxCursorError(pCsr, "Cursor error: %s", sqlite3_errmsg(db)); - } - }else{ - pCsr->iRowid++; - rc = SQLITE_OK; - } - return rc; -} - -/* We have reached EOF if previous sqlite3_step() returned -** anything other than SQLITE_ROW; -*/ -static int cidxEof(sqlite3_vtab_cursor *pCursor){ - CidxCursor *pCsr = (CidxCursor*)pCursor; - return pCsr->pStmt==0; -} - -static char *cidxMprintf(int *pRc, const char *zFmt, ...){ - char *zRet = 0; - va_list ap; - va_start(ap, zFmt); - zRet = sqlite3_vmprintf(zFmt, ap); - if( *pRc==SQLITE_OK ){ - if( zRet==0 ){ - *pRc = SQLITE_NOMEM; - } - }else{ - sqlite3_free(zRet); - zRet = 0; - } - va_end(ap); - return zRet; -} - -static sqlite3_stmt *cidxPrepare( - int *pRc, CidxCursor *pCsr, const char *zFmt, ... -){ - sqlite3_stmt *pRet = 0; - char *zSql; - va_list ap; /* ... printf arguments */ - va_start(ap, zFmt); - - zSql = sqlite3_vmprintf(zFmt, ap); - if( *pRc==SQLITE_OK ){ - if( zSql==0 ){ - *pRc = SQLITE_NOMEM; - }else{ - sqlite3 *db = ((CidxTable*)pCsr->base.pVtab)->db; - *pRc = sqlite3_prepare_v2(db, zSql, -1, &pRet, 0); - if( *pRc!=SQLITE_OK ){ - cidxCursorError(pCsr, "SQL error: %s", sqlite3_errmsg(db)); - } - } - } - sqlite3_free(zSql); - va_end(ap); - - return pRet; -} - -static void cidxFinalize(int *pRc, sqlite3_stmt *pStmt){ - int rc = sqlite3_finalize(pStmt); - if( *pRc==SQLITE_OK ) *pRc = rc; -} - -char *cidxStrdup(int *pRc, const char *zStr){ - char *zRet = 0; - if( *pRc==SQLITE_OK ){ - int n = (int)strlen(zStr); - zRet = cidxMalloc(pRc, n+1); - if( zRet ) memcpy(zRet, zStr, n+1); - } - return zRet; -} - -static void cidxFreeIndex(CidxIndex *pIdx){ - if( pIdx ){ - int i; - for(i=0; i<pIdx->nCol; i++){ - sqlite3_free(pIdx->aCol[i].zExpr); - } - sqlite3_free(pIdx->zWhere); - sqlite3_free(pIdx); - } -} - -static int cidx_isspace(char c){ - return c==' ' || c=='\t' || c=='\r' || c=='\n'; -} - -static int cidx_isident(char c){ - return c<0 - || (c>='0' && c<='9') || (c>='a' && c<='z') - || (c>='A' && c<='Z') || c=='_'; -} - -#define CIDX_PARSE_EOF 0 -#define CIDX_PARSE_COMMA 1 /* "," */ -#define CIDX_PARSE_OPEN 2 /* "(" */ -#define CIDX_PARSE_CLOSE 3 /* ")" */ - -/* -** Argument zIn points into the start, middle or end of a CREATE INDEX -** statement. If argument pbDoNotTrim is non-NULL, then this function -** scans the input until it finds EOF, a comma (",") or an open or -** close parenthesis character. It then sets (*pzOut) to point to said -** character and returns a CIDX_PARSE_XXX constant as appropriate. The -** parser is smart enough that special characters inside SQL strings -** or comments are not returned for. -** -** Or, if argument pbDoNotTrim is NULL, then this function sets *pzOut -** to point to the first character of the string that is not whitespace -** or part of an SQL comment and returns CIDX_PARSE_EOF. -** -** Additionally, if pbDoNotTrim is not NULL and the element immediately -** before (*pzOut) is an SQL comment of the form "-- comment", then -** (*pbDoNotTrim) is set before returning. In all other cases it is -** cleared. -*/ -static int cidxFindNext( - const char *zIn, - const char **pzOut, - int *pbDoNotTrim /* OUT: True if prev is -- comment */ -){ - const char *z = zIn; - - while( 1 ){ - while( cidx_isspace(*z) ) z++; - if( z[0]=='-' && z[1]=='-' ){ - z += 2; - while( z[0]!='\n' ){ - if( z[0]=='\0' ) return CIDX_PARSE_EOF; - z++; - } - while( cidx_isspace(*z) ) z++; - if( pbDoNotTrim ) *pbDoNotTrim = 1; - }else - if( z[0]=='/' && z[1]=='*' ){ - z += 2; - while( z[0]!='*' || z[1]!='/' ){ - if( z[1]=='\0' ) return CIDX_PARSE_EOF; - z++; - } - z += 2; - }else{ - *pzOut = z; - if( pbDoNotTrim==0 ) return CIDX_PARSE_EOF; - switch( *z ){ - case '\0': - return CIDX_PARSE_EOF; - case '(': - return CIDX_PARSE_OPEN; - case ')': - return CIDX_PARSE_CLOSE; - case ',': - return CIDX_PARSE_COMMA; - - case '"': - case '\'': - case '`': { - char q = *z; - z++; - while( *z ){ - if( *z==q ){ - z++; - if( *z!=q ) break; - } - z++; - } - break; - } - - case '[': - while( *z++!=']' ); - break; - - default: - z++; - break; - } - *pbDoNotTrim = 0; - } - } - - assert( 0 ); - return -1; -} - -static int cidxParseSQL(CidxCursor *pCsr, CidxIndex *pIdx, const char *zSql){ - const char *z = zSql; - const char *z1; - int e; - int rc = SQLITE_OK; - int nParen = 1; - int bDoNotTrim = 0; - CidxColumn *pCol = pIdx->aCol; - - e = cidxFindNext(z, &z, &bDoNotTrim); - if( e!=CIDX_PARSE_OPEN ) goto parse_error; - z1 = z+1; - z++; - while( nParen>0 ){ - e = cidxFindNext(z, &z, &bDoNotTrim); - if( e==CIDX_PARSE_EOF ) goto parse_error; - if( (e==CIDX_PARSE_COMMA || e==CIDX_PARSE_CLOSE) && nParen==1 ){ - const char *z2 = z; - if( pCol->zExpr ) goto parse_error; - - if( bDoNotTrim==0 ){ - while( cidx_isspace(z[-1]) ) z--; - if( !sqlite3_strnicmp(&z[-3], "asc", 3) && 0==cidx_isident(z[-4]) ){ - z -= 3; - while( cidx_isspace(z[-1]) ) z--; - }else - if( !sqlite3_strnicmp(&z[-4], "desc", 4) && 0==cidx_isident(z[-5]) ){ - z -= 4; - while( cidx_isspace(z[-1]) ) z--; - } - while( cidx_isspace(z1[0]) ) z1++; - } - - pCol->zExpr = cidxMprintf(&rc, "%.*s", z-z1, z1); - pCol++; - z = z1 = z2+1; - } - if( e==CIDX_PARSE_OPEN ) nParen++; - if( e==CIDX_PARSE_CLOSE ) nParen--; - z++; - } - - /* Search for a WHERE clause */ - cidxFindNext(z, &z, 0); - if( 0==sqlite3_strnicmp(z, "where", 5) ){ - pIdx->zWhere = cidxMprintf(&rc, "%s\n", &z[5]); - }else if( z[0]!='\0' ){ - goto parse_error; - } - - return rc; - - parse_error: - cidxCursorError(pCsr, "Parse error in: %s", zSql); - return SQLITE_ERROR; -} - -static int cidxLookupIndex( - CidxCursor *pCsr, /* Cursor object */ - const char *zIdx, /* Name of index to look up */ - CidxIndex **ppIdx, /* OUT: Description of columns */ - char **pzTab /* OUT: Table name */ -){ - int rc = SQLITE_OK; - char *zTab = 0; - CidxIndex *pIdx = 0; - - sqlite3_stmt *pFindTab = 0; - sqlite3_stmt *pInfo = 0; - - /* Find the table for this index. */ - pFindTab = cidxPrepare(&rc, pCsr, - "SELECT tbl_name, sql FROM sqlite_schema WHERE name=%Q AND type='index'", - zIdx - ); - if( rc==SQLITE_OK && sqlite3_step(pFindTab)==SQLITE_ROW ){ - const char *zSql = (const char*)sqlite3_column_text(pFindTab, 1); - zTab = cidxStrdup(&rc, (const char*)sqlite3_column_text(pFindTab, 0)); - - pInfo = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx); - if( rc==SQLITE_OK ){ - int nAlloc = 0; - int iCol = 0; - - while( sqlite3_step(pInfo)==SQLITE_ROW ){ - const char *zName = (const char*)sqlite3_column_text(pInfo, 2); - const char *zColl = (const char*)sqlite3_column_text(pInfo, 4); - CidxColumn *p; - if( zName==0 ) zName = "rowid"; - if( iCol==nAlloc ){ - int nByte = sizeof(CidxIndex) + sizeof(CidxColumn)*(nAlloc+8); - pIdx = (CidxIndex*)sqlite3_realloc(pIdx, nByte); - nAlloc += 8; - } - p = &pIdx->aCol[iCol++]; - p->bDesc = sqlite3_column_int(pInfo, 3); - p->bKey = sqlite3_column_int(pInfo, 5); - if( zSql==0 || p->bKey==0 ){ - p->zExpr = cidxMprintf(&rc, "\"%w\" COLLATE %s",zName,zColl); - }else{ - p->zExpr = 0; - } - pIdx->nCol = iCol; - pIdx->zWhere = 0; - } - cidxFinalize(&rc, pInfo); - } - - if( rc==SQLITE_OK && zSql ){ - rc = cidxParseSQL(pCsr, pIdx, zSql); - } - } - - cidxFinalize(&rc, pFindTab); - if( rc==SQLITE_OK && zTab==0 ){ - rc = SQLITE_ERROR; - } - - if( rc!=SQLITE_OK ){ - sqlite3_free(zTab); - cidxFreeIndex(pIdx); - }else{ - *pzTab = zTab; - *ppIdx = pIdx; - } - - return rc; -} - -static int cidxDecodeAfter( - CidxCursor *pCsr, - int nCol, - const char *zAfterKey, - char ***pazAfter -){ - char **azAfter; - int rc = SQLITE_OK; - int nAfterKey = (int)strlen(zAfterKey); - - azAfter = cidxMalloc(&rc, sizeof(char*)*nCol + nAfterKey+1); - if( rc==SQLITE_OK ){ - int i; - char *zCopy = (char*)&azAfter[nCol]; - char *p = zCopy; - memcpy(zCopy, zAfterKey, nAfterKey+1); - for(i=0; i<nCol; i++){ - while( *p==' ' ) p++; - - /* Check NULL values */ - if( *p=='N' ){ - if( memcmp(p, "NULL", 4) ) goto parse_error; - p += 4; - } - - /* Check strings and blob literals */ - else if( *p=='X' || *p=='\'' ){ - azAfter[i] = p; - if( *p=='X' ) p++; - if( *p!='\'' ) goto parse_error; - p++; - while( 1 ){ - if( *p=='\0' ) goto parse_error; - if( *p=='\'' ){ - p++; - if( *p!='\'' ) break; - } - p++; - } - } - - /* Check numbers */ - else{ - azAfter[i] = p; - while( (*p>='0' && *p<='9') - || *p=='.' || *p=='+' || *p=='-' || *p=='e' || *p=='E' - ){ - p++; - } - } - - while( *p==' ' ) p++; - if( *p!=(i==(nCol-1) ? '\0' : ',') ){ - goto parse_error; - } - *p++ = '\0'; - } - } - - *pazAfter = azAfter; - return rc; - - parse_error: - sqlite3_free(azAfter); - *pazAfter = 0; - cidxCursorError(pCsr, "%s", "error parsing after value"); - return SQLITE_ERROR; -} - -static char *cidxWhere( - int *pRc, CidxColumn *aCol, char **azAfter, int iGt, int bLastIsNull -){ - char *zRet = 0; - const char *zSep = ""; - int i; - - for(i=0; i<iGt; i++){ - zRet = cidxMprintf(pRc, "%z%s(%s) IS %s", zRet, - zSep, aCol[i].zExpr, (azAfter[i] ? azAfter[i] : "NULL") - ); - zSep = " AND "; - } - - if( bLastIsNull ){ - zRet = cidxMprintf(pRc, "%z%s(%s) IS NULL", zRet, zSep, aCol[iGt].zExpr); - } - else if( azAfter[iGt] ){ - zRet = cidxMprintf(pRc, "%z%s(%s) %s %s", zRet, - zSep, aCol[iGt].zExpr, (aCol[iGt].bDesc ? "<" : ">"), - azAfter[iGt] - ); - }else{ - zRet = cidxMprintf(pRc, "%z%s(%s) IS NOT NULL", zRet, zSep,aCol[iGt].zExpr); - } - - return zRet; -} - -#define CIDX_CLIST_ALL 0 -#define CIDX_CLIST_ORDERBY 1 -#define CIDX_CLIST_CURRENT_KEY 2 -#define CIDX_CLIST_SUBWHERE 3 -#define CIDX_CLIST_SUBEXPR 4 - -/* -** This function returns various strings based on the contents of the -** CidxIndex structure and the eType parameter. -*/ -static char *cidxColumnList( - int *pRc, /* IN/OUT: Error code */ - const char *zIdx, - CidxIndex *pIdx, /* Indexed columns */ - int eType /* True to include ASC/DESC */ -){ - char *zRet = 0; - if( *pRc==SQLITE_OK ){ - const char *aDir[2] = {"", " DESC"}; - int i; - const char *zSep = ""; - - for(i=0; i<pIdx->nCol; i++){ - CidxColumn *p = &pIdx->aCol[i]; - assert( pIdx->aCol[i].bDesc==0 || pIdx->aCol[i].bDesc==1 ); - switch( eType ){ - - case CIDX_CLIST_ORDERBY: - zRet = cidxMprintf(pRc, "%z%s%d%s", zRet, zSep, i+1, aDir[p->bDesc]); - zSep = ","; - break; - - case CIDX_CLIST_CURRENT_KEY: - zRet = cidxMprintf(pRc, "%z%squote(i%d)", zRet, zSep, i); - zSep = "||','||"; - break; - - case CIDX_CLIST_SUBWHERE: - if( p->bKey==0 ){ - zRet = cidxMprintf(pRc, "%z%s%s IS i.i%d", zRet, - zSep, p->zExpr, i - ); - zSep = " AND "; - } - break; - - case CIDX_CLIST_SUBEXPR: - if( p->bKey==1 ){ - zRet = cidxMprintf(pRc, "%z%s%s IS i.i%d", zRet, - zSep, p->zExpr, i - ); - zSep = " AND "; - } - break; - - default: - assert( eType==CIDX_CLIST_ALL ); - zRet = cidxMprintf(pRc, "%z%s(%s) AS i%d", zRet, zSep, p->zExpr, i); - zSep = ", "; - break; - } - } - } - - return zRet; -} - -/* -** Generate SQL (in memory obtained from sqlite3_malloc()) that will -** continue the index scan for zIdxName starting after zAfterKey. -*/ -int cidxGenerateScanSql( - CidxCursor *pCsr, /* The cursor which needs the new statement */ - const char *zIdxName, /* index to be scanned */ - const char *zAfterKey, /* start after this key, if not NULL */ - char **pzSqlOut /* OUT: Write the generated SQL here */ -){ - int rc; - char *zTab = 0; - char *zCurrentKey = 0; - char *zOrderBy = 0; - char *zSubWhere = 0; - char *zSubExpr = 0; - char *zSrcList = 0; - char **azAfter = 0; - CidxIndex *pIdx = 0; - - *pzSqlOut = 0; - rc = cidxLookupIndex(pCsr, zIdxName, &pIdx, &zTab); - - zOrderBy = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ORDERBY); - zCurrentKey = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_CURRENT_KEY); - zSubWhere = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBWHERE); - zSubExpr = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBEXPR); - zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL); - - if( rc==SQLITE_OK && zAfterKey ){ - rc = cidxDecodeAfter(pCsr, pIdx->nCol, zAfterKey, &azAfter); - } - - if( rc==SQLITE_OK ){ - if( zAfterKey==0 ){ - *pzSqlOut = cidxMprintf(&rc, - "SELECT (SELECT %s FROM %Q AS t WHERE %s), %s " - "FROM (SELECT %s FROM %Q INDEXED BY %Q %s%sORDER BY %s) AS i", - zSubExpr, zTab, zSubWhere, zCurrentKey, - zSrcList, zTab, zIdxName, - (pIdx->zWhere ? "WHERE " : ""), (pIdx->zWhere ? pIdx->zWhere : ""), - zOrderBy - ); - }else{ - const char *zSep = ""; - char *zSql; - int i; - - zSql = cidxMprintf(&rc, - "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (", - zSubExpr, zTab, zSubWhere, zCurrentKey - ); - for(i=pIdx->nCol-1; i>=0; i--){ - int j; - if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue; - for(j=0; j<2; j++){ - char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j); - zSql = cidxMprintf(&rc, "%z" - "%sSELECT * FROM (" - "SELECT %s FROM %Q INDEXED BY %Q WHERE %s%s%z ORDER BY %s" - ")", - zSql, zSep, zSrcList, zTab, zIdxName, - pIdx->zWhere ? pIdx->zWhere : "", - pIdx->zWhere ? " AND " : "", - zWhere, zOrderBy - ); - zSep = " UNION ALL "; - if( pIdx->aCol[i].bDesc==0 ) break; - } - } - *pzSqlOut = cidxMprintf(&rc, "%z) AS i", zSql); - } - } - - sqlite3_free(zTab); - sqlite3_free(zCurrentKey); - sqlite3_free(zOrderBy); - sqlite3_free(zSubWhere); - sqlite3_free(zSubExpr); - sqlite3_free(zSrcList); - cidxFreeIndex(pIdx); - sqlite3_free(azAfter); - return rc; -} - - -/* -** Position a cursor back to the beginning. -*/ -static int cidxFilter( - sqlite3_vtab_cursor *pCursor, - int idxNum, const char *idxStr, - int argc, sqlite3_value **argv -){ - int rc = SQLITE_OK; - CidxCursor *pCsr = (CidxCursor*)pCursor; - const char *zIdxName = 0; - const char *zAfterKey = 0; - - sqlite3_free(pCsr->zIdxName); - pCsr->zIdxName = 0; - sqlite3_free(pCsr->zAfterKey); - pCsr->zAfterKey = 0; - sqlite3_finalize(pCsr->pStmt); - pCsr->pStmt = 0; - - if( argc>0 ){ - zIdxName = (const char*)sqlite3_value_text(argv[0]); - if( argc>1 ){ - zAfterKey = (const char*)sqlite3_value_text(argv[1]); - } - } - - if( zIdxName ){ - char *zSql = 0; - pCsr->zIdxName = sqlite3_mprintf("%s", zIdxName); - pCsr->zAfterKey = zAfterKey ? sqlite3_mprintf("%s", zAfterKey) : 0; - rc = cidxGenerateScanSql(pCsr, zIdxName, zAfterKey, &zSql); - if( zSql ){ - pCsr->pStmt = cidxPrepare(&rc, pCsr, "%z", zSql); - } - } - - if( pCsr->pStmt ){ - assert( rc==SQLITE_OK ); - rc = cidxNext(pCursor); - } - pCsr->iRowid = 1; - return rc; -} - -/* -** Return a column value. -*/ -static int cidxColumn( - sqlite3_vtab_cursor *pCursor, - sqlite3_context *ctx, - int iCol -){ - CidxCursor *pCsr = (CidxCursor*)pCursor; - assert( iCol>=IIC_ERRMSG && iCol<=IIC_SCANNER_SQL ); - switch( iCol ){ - case IIC_ERRMSG: { - const char *zVal = 0; - if( sqlite3_column_type(pCsr->pStmt, 0)==SQLITE_INTEGER ){ - if( sqlite3_column_int(pCsr->pStmt, 0)==0 ){ - zVal = "row data mismatch"; - } - }else{ - zVal = "row missing"; - } - sqlite3_result_text(ctx, zVal, -1, SQLITE_STATIC); - break; - } - case IIC_CURRENT_KEY: { - sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, 1)); - break; - } - case IIC_INDEX_NAME: { - sqlite3_result_text(ctx, pCsr->zIdxName, -1, SQLITE_TRANSIENT); - break; - } - case IIC_AFTER_KEY: { - sqlite3_result_text(ctx, pCsr->zAfterKey, -1, SQLITE_TRANSIENT); - break; - } - case IIC_SCANNER_SQL: { - char *zSql = 0; - cidxGenerateScanSql(pCsr, pCsr->zIdxName, pCsr->zAfterKey, &zSql); - sqlite3_result_text(ctx, zSql, -1, sqlite3_free); - break; - } - } - return SQLITE_OK; -} - -/* Return the ROWID for the sqlite_btreeinfo table */ -static int cidxRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ - CidxCursor *pCsr = (CidxCursor*)pCursor; - *pRowid = pCsr->iRowid; - return SQLITE_OK; -} - -/* -** Register the virtual table modules with the database handle passed -** as the only argument. -*/ -static int ciInit(sqlite3 *db){ - static sqlite3_module cidx_module = { - 0, /* iVersion */ - 0, /* xCreate */ - cidxConnect, /* xConnect */ - cidxBestIndex, /* xBestIndex */ - cidxDisconnect, /* xDisconnect */ - 0, /* xDestroy */ - cidxOpen, /* xOpen - open a cursor */ - cidxClose, /* xClose - close a cursor */ - cidxFilter, /* xFilter - configure scan constraints */ - cidxNext, /* xNext - advance a cursor */ - cidxEof, /* xEof - check for end of scan */ - cidxColumn, /* xColumn - read data */ - cidxRowid, /* xRowid - read data */ - 0, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindMethod */ - 0, /* xRename */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0, /* xRollbackTo */ - 0, /* xShadowName */ - 0 /* xIntegrity */ - }; - return sqlite3_create_module(db, "incremental_index_check", &cidx_module, 0); -} - -/* -** Extension load function. -*/ -#ifdef _WIN32 -__declspec(dllexport) -#endif -int sqlite3_checkindex_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - SQLITE_EXTENSION_INIT2(pApi); - return ciInit(db); -} diff --git a/ext/repair/sqlite3_checker.c.in b/ext/repair/sqlite3_checker.c.in deleted file mode 100644 index 96b15f271..000000000 --- a/ext/repair/sqlite3_checker.c.in +++ /dev/null @@ -1,85 +0,0 @@ -/* -** Read an SQLite database file and analyze its space utilization. Generate -** text on standard output. -*/ -#define TCLSH_INIT_PROC sqlite3_checker_init_proc -#define SQLITE_ENABLE_DBPAGE_VTAB 1 -#undef SQLITE_THREADSAFE -#define SQLITE_THREADSAFE 0 -#undef SQLITE_ENABLE_COLUMN_METADATA -#define SQLITE_OMIT_DECLTYPE 1 -#define SQLITE_OMIT_DEPRECATED 1 -#define SQLITE_OMIT_PROGRESS_CALLBACK 1 -#define SQLITE_OMIT_SHARED_CACHE 1 -#define SQLITE_DEFAULT_MEMSTATUS 0 -#define SQLITE_MAX_EXPR_DEPTH 0 -INCLUDE sqlite3.c -INCLUDE $ROOT/src/tclsqlite.c -INCLUDE $ROOT/ext/misc/btreeinfo.c -INCLUDE $ROOT/ext/repair/checkindex.c -INCLUDE $ROOT/ext/repair/checkfreelist.c - -/* -** Decode a pointer to an sqlite3 object. -*/ -int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb){ - struct SqliteDb *p; - Tcl_CmdInfo cmdInfo; - if( Tcl_GetCommandInfo(interp, zA, &cmdInfo) ){ - p = (struct SqliteDb*)cmdInfo.objClientData; - *ppDb = p->db; - return TCL_OK; - }else{ - *ppDb = 0; - return TCL_ERROR; - } - return TCL_OK; -} - -/* -** sqlite3_imposter db main rootpage {CREATE TABLE...} ;# setup an imposter -** sqlite3_imposter db main ;# rm all imposters -*/ -static int sqlite3_imposter( - void *clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - sqlite3 *db; - const char *zSchema; - int iRoot; - const char *zSql; - - if( objc!=3 && objc!=5 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB SCHEMA [ROOTPAGE SQL]"); - return TCL_ERROR; - } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; - zSchema = Tcl_GetString(objv[2]); - if( objc==3 ){ - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 0, 1); - }else{ - if( Tcl_GetIntFromObj(interp, objv[3], &iRoot) ) return TCL_ERROR; - zSql = Tcl_GetString(objv[4]); - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 1, iRoot); - sqlite3_exec(db, zSql, 0, 0, 0); - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 0, 0); - } - return TCL_OK; -} - -#include <stdio.h> - -const char *sqlite3_checker_init_proc(Tcl_Interp *interp){ - Tcl_CreateObjCommand(interp, "sqlite3_imposter", - (Tcl_ObjCmdProc*)sqlite3_imposter, 0, 0); - sqlite3_auto_extension((void(*)(void))sqlite3_btreeinfo_init); - sqlite3_auto_extension((void(*)(void))sqlite3_checkindex_init); - sqlite3_auto_extension((void(*)(void))sqlite3_checkfreelist_init); - return -BEGIN_STRING -INCLUDE $ROOT/ext/repair/sqlite3_checker.tcl -END_STRING -; -} diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl deleted file mode 100644 index 2ae6e15b1..000000000 --- a/ext/repair/sqlite3_checker.tcl +++ /dev/null @@ -1,264 +0,0 @@ -# This TCL script is the main driver script for the sqlite3_checker utility -# program. -# - -# Special case: -# -# sqlite3_checker --test FILENAME ARGS -# -# uses FILENAME in place of this script. -# -if {[lindex $argv 0]=="--test" && [llength $argv]>1} { - set ::argv0 [lindex $argv 1] - set argv [lrange $argv 2 end] - source $argv0 - exit 0 -} - -# Emulate a TCL shell -# -proc tclsh {} { - set line {} - while {![eof stdin]} { - if {$line!=""} { - puts -nonewline "> " - } else { - puts -nonewline "% " - } - flush stdout - append line [gets stdin] - if {[info complete $line]} { - if {[catch {uplevel #0 $line} result]} { - puts stderr "Error: $result" - } elseif {$result!=""} { - puts $result - } - set line {} - } else { - append line \n - } - } -} - -# Do an incremental integrity check of a single index -# -proc check_index {idxname batchsize bTrace} { - set i 0 - set more 1 - set nerr 0 - set pct 00.0 - set max [db one {SELECT nEntry FROM sqlite_btreeinfo('main') - WHERE name=$idxname}] - puts -nonewline "$idxname: $i of $max rows ($pct%)\r" - flush stdout - if {$bTrace} { - set sql {SELECT errmsg, current_key AS key, - CASE WHEN rowid=1 THEN scanner_sql END AS traceOut - FROM incremental_index_check($idxname) - WHERE after_key=$key - LIMIT $batchsize} - } else { - set sql {SELECT errmsg, current_key AS key, NULL AS traceOut - FROM incremental_index_check($idxname) - WHERE after_key=$key - LIMIT $batchsize} - } - while {$more} { - set more 0 - db eval $sql { - set more 1 - if {$errmsg!=""} { - incr nerr - puts "$idxname: key($key): $errmsg" - } elseif {$traceOut!=""} { - puts "$idxname: $traceOut" - } - incr i - - } - set x [format {%.1f} [expr {($i*100.0)/$max}]] - if {$x!=$pct} { - puts -nonewline "$idxname: $i of $max rows ($pct%)\r" - flush stdout - set pct $x - } - } - puts "$idxname: $nerr errors out of $i entries" -} - -# Print a usage message on standard error, then quit. -# -proc usage {} { - set argv0 [file rootname [file tail [info nameofexecutable]]] - puts stderr "Usage: $argv0 OPTIONS database-filename" - puts stderr { -Do sanity checking on a live SQLite3 database file specified by the -"database-filename" argument. - -Options: - - --batchsize N Number of rows to check per transaction - - --freelist Perform a freelist check - - --index NAME Run a check of the index NAME - - --summary Print summary information about the database - - --table NAME Run a check of all indexes for table NAME - - --tclsh Run the built-in TCL interpreter (for debugging) - - --trace (Debugging only:) Output trace information on the scan - - --version Show the version number of SQLite -} - exit 1 -} - -set file_to_analyze {} -append argv {} -set bFreelistCheck 0 -set bSummary 0 -set zIndex {} -set zTable {} -set batchsize 1000 -set bAll 1 -set bTrace 0 -set argc [llength $argv] -for {set i 0} {$i<$argc} {incr i} { - set arg [lindex $argv $i] - if {[regexp {^-+tclsh$} $arg]} { - tclsh - exit 0 - } - if {[regexp {^-+version$} $arg]} { - sqlite3 mem :memory: - puts [mem one {SELECT sqlite_version()||' '||sqlite_source_id()}] - mem close - exit 0 - } - if {[regexp {^-+freelist$} $arg]} { - set bFreelistCheck 1 - set bAll 0 - continue - } - if {[regexp {^-+summary$} $arg]} { - set bSummary 1 - set bAll 0 - continue - } - if {[regexp {^-+trace$} $arg]} { - set bTrace 1 - continue - } - if {[regexp {^-+batchsize$} $arg]} { - incr i - if {$i>=$argc} { - puts stderr "missing argument on $arg" - exit 1 - } - set batchsize [lindex $argv $i] - continue - } - if {[regexp {^-+index$} $arg]} { - incr i - if {$i>=$argc} { - puts stderr "missing argument on $arg" - exit 1 - } - set zIndex [lindex $argv $i] - set bAll 0 - continue - } - if {[regexp {^-+table$} $arg]} { - incr i - if {$i>=$argc} { - puts stderr "missing argument on $arg" - exit 1 - } - set zTable [lindex $argv $i] - set bAll 0 - continue - } - if {[regexp {^-} $arg]} { - puts stderr "Unknown option: $arg" - usage - } - if {$file_to_analyze!=""} { - usage - } else { - set file_to_analyze $arg - } -} -if {$file_to_analyze==""} usage - -# If a TCL script is specified on the command-line, then run that -# script. -# -if {[file extension $file_to_analyze]==".tcl"} { - source $file_to_analyze - exit 0 -} - -set root_filename $file_to_analyze -regexp {^file:(//)?([^?]*)} $file_to_analyze all x1 root_filename -if {![file exists $root_filename]} { - puts stderr "No such file: $root_filename" - exit 1 -} -if {![file readable $root_filename]} { - puts stderr "File is not readable: $root_filename" - exit 1 -} - -if {[catch {sqlite3 db $file_to_analyze} res]} { - puts stderr "Cannot open datababase $root_filename: $res" - exit 1 -} - -if {$bFreelistCheck || $bAll} { - puts -nonewline "freelist-check: " - flush stdout - db eval BEGIN - puts [db one {SELECT checkfreelist('main')}] - db eval END -} -if {$bSummary} { - set scale 0 - set pgsz [db one {PRAGMA page_size}] - db eval {SELECT nPage*$pgsz AS sz, name, tbl_name - FROM sqlite_btreeinfo - WHERE type='index' - ORDER BY 1 DESC, name} { - if {$scale==0} { - if {$sz>10000000} { - set scale 1000000.0 - set unit MB - } else { - set scale 1000.0 - set unit KB - } - } - puts [format {%7.1f %s index %s of table %s} \ - [expr {$sz/$scale}] $unit $name $tbl_name] - } -} -if {$zIndex!=""} { - check_index $zIndex $batchsize $bTrace -} -if {$zTable!=""} { - foreach idx [db eval {SELECT name FROM sqlite_master - WHERE type='index' AND rootpage>0 - AND tbl_name=$zTable}] { - check_index $idx $batchsize $bTrace - } -} -if {$bAll} { - set allidx [db eval {SELECT name FROM sqlite_btreeinfo('main') - WHERE type='index' AND rootpage>0 - ORDER BY nEntry}] - foreach idx $allidx { - check_index $idx $batchsize $bTrace - } -} diff --git a/ext/repair/test/README.md b/ext/repair/test/README.md deleted file mode 100644 index 8cc954adf..000000000 --- a/ext/repair/test/README.md +++ /dev/null @@ -1,13 +0,0 @@ -To run these tests, first build sqlite3_checker: - - -> make sqlite3_checker - - -Then run the "test.tcl" script using: - - -> ./sqlite3_checker --test $path/test.tcl - - -Optionally add the full pathnames of individual *.test modules diff --git a/ext/repair/test/checkfreelist01.test b/ext/repair/test/checkfreelist01.test deleted file mode 100644 index 7e2dd51c3..000000000 --- a/ext/repair/test/checkfreelist01.test +++ /dev/null @@ -1,92 +0,0 @@ -# 2017-10-11 - -set testprefix checkfreelist - -do_execsql_test 1.0 { - PRAGMA page_size=1024; - CREATE TABLE t1(a, b); -} - -do_execsql_test 1.2 { SELECT checkfreelist('main') } {ok} -do_execsql_test 1.3 { - WITH s(i) AS ( - SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10000 - ) - INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s; - DELETE FROM t1 WHERE rowid%3; - PRAGMA freelist_count; -} {6726} - -do_execsql_test 1.4 { SELECT checkfreelist('main') } {ok} -do_execsql_test 1.5 { - WITH freelist_trunk(i, d, n) AS ( - SELECT 1, NULL, sqlite_readint32(data, 32) FROM sqlite_dbpage WHERE pgno=1 - UNION ALL - SELECT n, data, sqlite_readint32(data) - FROM freelist_trunk, sqlite_dbpage WHERE pgno=n - ) - SELECT i FROM freelist_trunk WHERE i!=1; -} { - 10009 9715 9343 8969 8595 8222 7847 7474 7102 6727 6354 5982 5608 5234 - 4860 4487 4112 3740 3367 2992 2619 2247 1872 1499 1125 752 377 5 -} - -do_execsql_test 1.6 { SELECT checkfreelist('main') } {ok} - -proc set_int {blob idx newval} { - binary scan $blob I* ints - lset ints $idx $newval - binary format I* $ints -} -db func set_int set_int - -proc get_int {blob idx} { - binary scan $blob I* ints - lindex $ints $idx -} -db func get_int get_int - -do_execsql_test 1.7 { - BEGIN; - UPDATE sqlite_dbpage - SET data = set_int(data, 1, get_int(data, 1)-1) - WHERE pgno=4860; - SELECT checkfreelist('main'); - ROLLBACK; -} {{free-list count mismatch: actual=6725 header=6726}} - -do_execsql_test 1.8 { - BEGIN; - UPDATE sqlite_dbpage - SET data = set_int(data, 5, (SELECT * FROM pragma_page_count)+1) - WHERE pgno=4860; - SELECT checkfreelist('main'); - ROLLBACK; -} {{leaf page 10092 is out of range (child 3 of trunk page 4860)}} - -do_execsql_test 1.9 { - BEGIN; - UPDATE sqlite_dbpage - SET data = set_int(data, 5, 0) - WHERE pgno=4860; - SELECT checkfreelist('main'); - ROLLBACK; -} {{leaf page 0 is out of range (child 3 of trunk page 4860)}} - -do_execsql_test 1.10 { - BEGIN; - UPDATE sqlite_dbpage - SET data = set_int(data, get_int(data, 1)+1, 0) - WHERE pgno=5; - SELECT checkfreelist('main'); - ROLLBACK; -} {{leaf page 0 is out of range (child 247 of trunk page 5)}} - -do_execsql_test 1.11 { - BEGIN; - UPDATE sqlite_dbpage - SET data = set_int(data, 1, 249) - WHERE pgno=5; - SELECT checkfreelist('main'); - ROLLBACK; -} {{leaf count out of range (249) on trunk page 5}} diff --git a/ext/repair/test/checkindex01.test b/ext/repair/test/checkindex01.test deleted file mode 100644 index 97973aee7..000000000 --- a/ext/repair/test/checkindex01.test +++ /dev/null @@ -1,349 +0,0 @@ -# 2017-10-11 -# -set testprefix checkindex - -do_execsql_test 1.0 { - CREATE TABLE t1(a, b); - CREATE INDEX i1 ON t1(a); - INSERT INTO t1 VALUES('one', 2); - INSERT INTO t1 VALUES('two', 4); - INSERT INTO t1 VALUES('three', 6); - INSERT INTO t1 VALUES('four', 8); - INSERT INTO t1 VALUES('five', 10); - - CREATE INDEX i2 ON t1(a DESC); -} {} - -proc incr_index_check {idx nStep} { - set Q { - SELECT errmsg, current_key FROM incremental_index_check($idx, $after) - LIMIT $nStep - } - - set res [list] - while {1} { - unset -nocomplain current_key - set res1 [db eval $Q] - if {[llength $res1]==0} break - set res [concat $res $res1] - set after [lindex $res end] - } - - return $res -} - -proc do_index_check_test {tn idx res} { - uplevel [list do_execsql_test $tn.1 " - SELECT errmsg, current_key FROM incremental_index_check('$idx'); - " $res] - - uplevel [list do_test $tn.2 "incr_index_check $idx 1" [list {*}$res]] - uplevel [list do_test $tn.3 "incr_index_check $idx 2" [list {*}$res]] - uplevel [list do_test $tn.4 "incr_index_check $idx 5" [list {*}$res]] -} - - -do_execsql_test 1.2.1 { - SELECT rowid, errmsg IS NULL, current_key FROM incremental_index_check('i1'); -} { - 1 1 'five',5 - 2 1 'four',4 - 3 1 'one',1 - 4 1 'three',3 - 5 1 'two',2 -} -do_execsql_test 1.2.2 { - SELECT errmsg IS NULL, current_key, index_name, after_key, scanner_sql - FROM incremental_index_check('i1') LIMIT 1; -} { - 1 - 'five',5 - i1 - {} - {SELECT (SELECT a IS i.i0 FROM 't1' AS t WHERE "rowid" COLLATE BINARY IS i.i1), quote(i0)||','||quote(i1) FROM (SELECT (a) AS i0, ("rowid" COLLATE BINARY) AS i1 FROM 't1' INDEXED BY 'i1' ORDER BY 1,2) AS i} -} - -do_index_check_test 1.3 i1 { - {} 'five',5 - {} 'four',4 - {} 'one',1 - {} 'three',3 - {} 'two',2 -} - -do_index_check_test 1.4 i2 { - {} 'two',2 - {} 'three',3 - {} 'one',1 - {} 'four',4 - {} 'five',5 -} - -do_test 1.5 { - set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t1' }] - sqlite3_imposter db main $tblroot {CREATE TABLE xt1(a,b)} - db eval { - UPDATE xt1 SET a='six' WHERE rowid=3; - DELETE FROM xt1 WHERE rowid = 5; - } - sqlite3_imposter db main -} {} - -do_index_check_test 1.6 i1 { - {row missing} 'five',5 - {} 'four',4 - {} 'one',1 - {row data mismatch} 'three',3 - {} 'two',2 -} - -do_index_check_test 1.7 i2 { - {} 'two',2 - {row data mismatch} 'three',3 - {} 'one',1 - {} 'four',4 - {row missing} 'five',5 -} - -#-------------------------------------------------------------------------- -do_execsql_test 2.0 { - - CREATE TABLE t2(a INTEGER PRIMARY KEY, b, c, d); - - INSERT INTO t2 VALUES(1, NULL, 1, 1); - INSERT INTO t2 VALUES(2, 1, NULL, 1); - INSERT INTO t2 VALUES(3, 1, 1, NULL); - - INSERT INTO t2 VALUES(4, 2, 2, 1); - INSERT INTO t2 VALUES(5, 2, 2, 2); - INSERT INTO t2 VALUES(6, 2, 2, 3); - - INSERT INTO t2 VALUES(7, 2, 2, 1); - INSERT INTO t2 VALUES(8, 2, 2, 2); - INSERT INTO t2 VALUES(9, 2, 2, 3); - - CREATE INDEX i3 ON t2(b, c, d); - CREATE INDEX i4 ON t2(b DESC, c DESC, d DESC); - CREATE INDEX i5 ON t2(d, c DESC, b); -} {} - -do_index_check_test 2.1 i3 { - {} NULL,1,1,1 - {} 1,NULL,1,2 - {} 1,1,NULL,3 - {} 2,2,1,4 - {} 2,2,1,7 - {} 2,2,2,5 - {} 2,2,2,8 - {} 2,2,3,6 - {} 2,2,3,9 -} - -do_index_check_test 2.2 i4 { - {} 2,2,3,6 - {} 2,2,3,9 - {} 2,2,2,5 - {} 2,2,2,8 - {} 2,2,1,4 - {} 2,2,1,7 - {} 1,1,NULL,3 - {} 1,NULL,1,2 - {} NULL,1,1,1 -} - -do_index_check_test 2.3 i5 { - {} NULL,1,1,3 - {} 1,2,2,4 - {} 1,2,2,7 - {} 1,1,NULL,1 - {} 1,NULL,1,2 - {} 2,2,2,5 - {} 2,2,2,8 - {} 3,2,2,6 - {} 3,2,2,9 -} - -#-------------------------------------------------------------------------- -do_execsql_test 3.0 { - - CREATE TABLE t3(w, x, y, z PRIMARY KEY) WITHOUT ROWID; - CREATE INDEX t3wxy ON t3(w, x, y); - CREATE INDEX t3wxy2 ON t3(w DESC, x DESC, y DESC); - - INSERT INTO t3 VALUES(NULL, NULL, NULL, 1); - INSERT INTO t3 VALUES(NULL, NULL, NULL, 2); - INSERT INTO t3 VALUES(NULL, NULL, NULL, 3); - - INSERT INTO t3 VALUES('a', NULL, NULL, 4); - INSERT INTO t3 VALUES('a', NULL, NULL, 5); - INSERT INTO t3 VALUES('a', NULL, NULL, 6); - - INSERT INTO t3 VALUES('a', 'b', NULL, 7); - INSERT INTO t3 VALUES('a', 'b', NULL, 8); - INSERT INTO t3 VALUES('a', 'b', NULL, 9); - -} {} - -do_index_check_test 3.1 t3wxy { - {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3 - {} 'a',NULL,NULL,4 {} 'a',NULL,NULL,5 {} 'a',NULL,NULL,6 - {} 'a','b',NULL,7 {} 'a','b',NULL,8 {} 'a','b',NULL,9 -} -do_index_check_test 3.2 t3wxy2 { - {} 'a','b',NULL,7 {} 'a','b',NULL,8 {} 'a','b',NULL,9 - {} 'a',NULL,NULL,4 {} 'a',NULL,NULL,5 {} 'a',NULL,NULL,6 - {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3 -} - -#-------------------------------------------------------------------------- -# Test with an index that uses non-default collation sequences. -# -do_execsql_test 4.0 { - CREATE TABLE t4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT); - INSERT INTO t4 VALUES(1, 'aaa', 'bbb'); - INSERT INTO t4 VALUES(2, 'AAA', 'CCC'); - INSERT INTO t4 VALUES(3, 'aab', 'ddd'); - INSERT INTO t4 VALUES(4, 'AAB', 'EEE'); - - CREATE INDEX t4cc ON t4(c1 COLLATE nocase, c2 COLLATE nocase); -} - -do_index_check_test 4.1 t4cc { - {} 'aaa','bbb',1 - {} 'AAA','CCC',2 - {} 'aab','ddd',3 - {} 'AAB','EEE',4 -} - -do_test 4.2 { - set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t4' }] - sqlite3_imposter db main $tblroot \ - {CREATE TABLE xt4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT)} - - db eval { - UPDATE xt4 SET c1='hello' WHERE rowid=2; - DELETE FROM xt4 WHERE rowid = 3; - } - sqlite3_imposter db main -} {} - -do_index_check_test 4.3 t4cc { - {} 'aaa','bbb',1 - {row data mismatch} 'AAA','CCC',2 - {row missing} 'aab','ddd',3 - {} 'AAB','EEE',4 -} - -#-------------------------------------------------------------------------- -# Test an index on an expression. -# -do_execsql_test 5.0 { - CREATE TABLE t5(x INTEGER PRIMARY KEY, y TEXT, UNIQUE(y)); - INSERT INTO t5 VALUES(1, '{"x":1, "y":1}'); - INSERT INTO t5 VALUES(2, '{"x":2, "y":2}'); - INSERT INTO t5 VALUES(3, '{"x":3, "y":3}'); - INSERT INTO t5 VALUES(4, '{"w":4, "z":4}'); - INSERT INTO t5 VALUES(5, '{"x":5, "y":5}'); - - CREATE INDEX t5x ON t5( json_extract(y, '$.x') ); - CREATE INDEX t5y ON t5( json_extract(y, '$.y') DESC ); -} - -do_index_check_test 5.1.1 t5x { - {} NULL,4 {} 1,1 {} 2,2 {} 3,3 {} 5,5 -} - -do_index_check_test 5.1.2 t5y { - {} 5,5 {} 3,3 {} 2,2 {} 1,1 {} NULL,4 -} - -do_index_check_test 5.1.3 sqlite_autoindex_t5_1 { - {} {'{"w":4, "z":4}',4} - {} {'{"x":1, "y":1}',1} - {} {'{"x":2, "y":2}',2} - {} {'{"x":3, "y":3}',3} - {} {'{"x":5, "y":5}',5} -} - -do_test 5.2 { - set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t5' }] - sqlite3_imposter db main $tblroot \ - {CREATE TABLE xt5(a INTEGER PRIMARY KEY, c1 TEXT);} - db eval { - UPDATE xt5 SET c1='{"x":22, "y":11}' WHERE rowid=1; - DELETE FROM xt5 WHERE rowid = 4; - } - sqlite3_imposter db main -} {} - -do_index_check_test 5.3.1 t5x { - {row missing} NULL,4 - {row data mismatch} 1,1 - {} 2,2 - {} 3,3 - {} 5,5 -} - -do_index_check_test 5.3.2 sqlite_autoindex_t5_1 { - {row missing} {'{"w":4, "z":4}',4} - {row data mismatch} {'{"x":1, "y":1}',1} - {} {'{"x":2, "y":2}',2} - {} {'{"x":3, "y":3}',3} - {} {'{"x":5, "y":5}',5} -} - -#------------------------------------------------------------------------- -# -do_execsql_test 6.0 { - CREATE TABLE t6(x INTEGER PRIMARY KEY, y, z); - CREATE INDEX t6x1 ON t6(y, /* one,two,three */ z); - CREATE INDEX t6x2 ON t6(z, -- hello,world, - y); - - CREATE INDEX t6x3 ON t6(z -- hello,world - , y); - - INSERT INTO t6 VALUES(1, 2, 3); - INSERT INTO t6 VALUES(4, 5, 6); -} - -do_index_check_test 6.1 t6x1 { - {} 2,3,1 - {} 5,6,4 -} -do_index_check_test 6.2 t6x2 { - {} 3,2,1 - {} 6,5,4 -} -do_index_check_test 6.2 t6x3 { - {} 3,2,1 - {} 6,5,4 -} - -#------------------------------------------------------------------------- -# -do_execsql_test 7.0 { - CREATE TABLE t7(x INTEGER PRIMARY KEY, y, z); - INSERT INTO t7 VALUES(1, 1, 1); - INSERT INTO t7 VALUES(2, 2, 0); - INSERT INTO t7 VALUES(3, 3, 1); - INSERT INTO t7 VALUES(4, 4, 0); - - CREATE INDEX t7i1 ON t7(y) WHERE z=1; - CREATE INDEX t7i2 ON t7(y) /* hello,world */ WHERE z=1; - CREATE INDEX t7i3 ON t7(y) WHERE -- yep - z=1; - CREATE INDEX t7i4 ON t7(y) WHERE z=1 -- yep; -} -do_index_check_test 7.1 t7i1 { - {} 1,1 {} 3,3 -} -do_index_check_test 7.2 t7i2 { - {} 1,1 {} 3,3 -} -do_index_check_test 7.3 t7i3 { - {} 1,1 {} 3,3 -} -do_index_check_test 7.4 t7i4 { - {} 1,1 {} 3,3 -} diff --git a/ext/repair/test/test.tcl b/ext/repair/test/test.tcl deleted file mode 100644 index c073bb73c..000000000 --- a/ext/repair/test/test.tcl +++ /dev/null @@ -1,67 +0,0 @@ -# Run this script using -# -# sqlite3_checker --test $thisscript $testscripts -# -# The $testscripts argument is optional. If omitted, all *.test files -# in the same directory as $thisscript are run. -# -set NTEST 0 -set NERR 0 - - -# Invoke the do_test procedure to run a single test -# -# The $expected parameter is the expected result. The result is the return -# value from the last TCL command in $cmd. -# -# Normally, $expected must match exactly. But if $expected is of the form -# "/regexp/" then regular expression matching is used. If $expected is -# "~/regexp/" then the regular expression must NOT match. If $expected is -# of the form "#/value-list/" then each term in value-list must be numeric -# and must approximately match the corresponding numeric term in $result. -# Values must match within 10%. Or if the $expected term is A..B then the -# $result term must be in between A and B. -# -proc do_test {name cmd expected} { - if {[info exists ::testprefix]} { - set name "$::testprefix$name" - } - - incr ::NTEST - puts -nonewline $name... - flush stdout - - if {[catch {uplevel #0 "$cmd;\n"} result]} { - puts -nonewline $name... - puts "\nError: $result" - incr ::NERR - } else { - set ok [expr {[string compare $result $expected]==0}] - if {!$ok} { - puts "\n! $name expected: \[$expected\]\n! $name got: \[$result\]" - incr ::NERR - } else { - puts " Ok" - } - } - flush stdout -} - -# -# do_execsql_test TESTNAME SQL RES -# -proc do_execsql_test {testname sql {result {}}} { - uplevel [list do_test $testname [list db eval $sql] [list {*}$result]] -} - -if {[llength $argv]==0} { - set dir [file dirname $argv0] - set argv [glob -nocomplain $dir/*.test] -} -foreach testfile $argv { - file delete -force test.db - sqlite3 db test.db - source $testfile - catch {db close} -} -puts "$NERR errors out of $NTEST tests" diff --git a/ext/rtree/geopoly.c b/ext/rtree/geopoly.c index 0ae42e7b7..22166a6f9 100644 --- a/ext/rtree/geopoly.c +++ b/ext/rtree/geopoly.c @@ -200,7 +200,7 @@ static int geopolyParseNumber(GeoParse *p, GeoCoord *pVal){ /* The sqlite3AtoF() routine is much much faster than atof(), if it ** is available */ double r; - (void)sqlite3AtoF((const char*)p->z, &r, j, SQLITE_UTF8); + (void)sqlite3AtoF((const char*)p->z, &r); *pVal = r; #else *pVal = (GeoCoord)atof((const char*)p->z); diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index b3d29283e..faebdce78 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -1037,7 +1037,17 @@ static void rtreeRelease(Rtree *pRtree){ pRtree->inWrTrans = 0; assert( pRtree->nCursor==0 ); nodeBlobReset(pRtree); - assert( pRtree->nNodeRef==0 || pRtree->bCorrupt ); + if( pRtree->nNodeRef ){ + int i; + assert( pRtree->bCorrupt ); + for(i=0; i<HASHSIZE; i++){ + while( pRtree->aHash[i] ){ + RtreeNode *pNext = pRtree->aHash[i]->pNext; + sqlite3_free(pRtree->aHash[i]); + pRtree->aHash[i] = pNext; + } + } + } sqlite3_finalize(pRtree->pWriteNode); sqlite3_finalize(pRtree->pDeleteNode); sqlite3_finalize(pRtree->pReadRowid); @@ -2329,7 +2339,7 @@ static int AdjustTree( int iCell; cnt++; - if( NEVER(cnt>100) ){ + if( cnt>100 ){ RTREE_IS_CORRUPT(pRtree); return SQLITE_CORRUPT_VTAB; } @@ -2687,15 +2697,6 @@ static int SplitNode( rc = updateMapping(pRtree, pCell->iRowid, pLeft, iHeight); } - if( rc==SQLITE_OK ){ - rc = nodeRelease(pRtree, pRight); - pRight = 0; - } - if( rc==SQLITE_OK ){ - rc = nodeRelease(pRtree, pLeft); - pLeft = 0; - } - splitnode_out: nodeRelease(pRtree, pRight); nodeRelease(pRtree, pLeft); @@ -2880,7 +2881,7 @@ static int rtreeInsertCell( rc = SplitNode(pRtree, pNode, pCell, iHeight); }else{ rc = AdjustTree(pRtree, pNode, pCell); - if( ALWAYS(rc==SQLITE_OK) ){ + if( rc==SQLITE_OK ){ if( iHeight==0 ){ rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode); }else{ diff --git a/ext/session/session4.test b/ext/session/session4.test index 55cb76f15..5e44a8eb6 100644 --- a/ext/session/session4.test +++ b/ext/session/session4.test @@ -135,6 +135,7 @@ foreach {tn blob} { 54 540101743400120003001200010000000000000002120002400C000000000002120002400C00000000000050040100000074310017FF0050040100000074310017FF7F00000000000000050100000000000000030100000003001700010000666F7572 55 540101743400120003001200010000000000000002120002400C00000000000050040100000074310017000100010080000001000000020003010100000300170100000003001700010000666F7572 56 5487ffffff7f + 57 54015052494e5446110017120004 } { do_test 2.$tn { set changeset [binary decode hex $blob] diff --git a/ext/session/sessionC.test b/ext/session/sessionC.test index 74370cb79..afe927608 100644 --- a/ext/session/sessionC.test +++ b/ext/session/sessionC.test @@ -192,6 +192,73 @@ do_test 3.3 { } } {1 1 3 3} +#------------------------------------------------------------------------- +# +reset_db +set C [binary format c* {0x54 0x01 0x01 0x00 0x12 0x00 0x05}] + +do_test 4.0 { + sqlite3changegroup grp + list [catch { grp add $C } msg] $msg +} {0 {}} +grp delete + +#------------------------------------------------------------------------- +# +reset_db +set C [binary format c* {0x54 0xda 0xda 0xda 0xda 0xda}] + +do_execsql_test 5.0 { + CREATE TABLE t1(a PRIMARY KEY, b, c, d); +} + +do_test 5.1 { + list [catch { sqlite3changeset_apply db $C noop xFilter } msg] $msg +} {1 SQLITE_CORRUPT} + +#------------------------------------------------------------------------- +# + +foreach {tn type C2hex C3hex} { + 1 changeset 54020100743100170001000000000000000A030374656E000306656C6576656E + 54020100743100170001000000000000000A030374656E0003FFFF01656C6576656E + + 2 patchset 50020100743100170001000000000000000A0306656C6576656E + 50020100743100170001000000000000000A03FFFF06656C6576656E + + 3 changeset 54020100743100170001000000000000000A030374656E000306656C6576656E + 54020100743100170001000000000000000A030374656E0003FFFF +} { + reset_db + + do_execsql_test 6.$tn.0 { + CREATE TABLE t1(x INTEGER PRIMARY KEY, y TEXT); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); + INSERT INTO t1 VALUES(3, 'three'); + } {} + + do_test 6.$tn.1 { + set C1 [${type}_from_sql { + INSERT INTO t1 VALUES(10, 'ten'); + }] + set C2 [${type}_from_sql { + UPDATE t1 SET y='eleven' WHERE x=10; + }] + db one { SELECT quote($C2) } + } "X'$C2hex'" + + do_test 6.$tn.2 { + changeset_to_list [sqlite3changeset_concat $C1 $C2] + } {{INSERT t1 0 X. {} {i 10 t eleven}}} + + set C3 [db one {SELECT unhex($C3hex)}] + + do_test 6.$tn.3 { + list [catch { sqlite3changeset_concat $C1 $C3 } msg] $msg + } {1 SQLITE_CORRUPT} +} finish_test + diff --git a/ext/session/sessionchange2.test b/ext/session/sessionchange2.test new file mode 100644 index 000000000..64208d223 --- /dev/null +++ b/ext/session/sessionchange2.test @@ -0,0 +1,400 @@ +# 2026 January 09 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix sessionchange2 + +do_test 1.1.1 { + sqlite3changegroup grp + list [catch { grp change_begin INSERT "nosuchtable" 0 } msg] $msg +} {1 {SQLITE_ERROR - no such table: nosuchtable}} + +do_test 1.1.2 { + grp schema db main + list [catch { grp change_begin INSERT "nosuchtable" 0 } msg] $msg +} {1 {SQLITE_ERROR - no such table: nosuchtable}} + +do_test 1.1.3 { + list [catch { grp change_begin_ne INSERT "nosuchtable" 0 } msg] $msg +} {1 SQLITE_ERROR} + +do_execsql_test 1.2.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); +} +do_test 1.2.2 { + list [catch { grp change_begin 435 "t1" 0 } msg] $msg +} {1 SQLITE_ERROR} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a, b); +} + +do_test 2.1.1 { + sqlite3changegroup grp + grp schema db main + grp change_begin INSERT "t1" 0 + grp change_int64 new 0 55 + grp change_int64 new 1 101 + grp change_finish false +} {} + +do_test 2.1.2 { + set C [grp output] + grp delete + changeset_to_list $C +} { + {INSERT t1 0 X. {} {i 55 i 101}} +} + +do_test 2.2.1 { + sqlite3changegroup grp + grp schema db main + grp change_begin INSERT "t2" 0 + grp change_int64 new 0 -5 + grp change_int64 new 1 -10 + grp change_int64 new 2 -20 + grp change_finish false +} {} + +do_test 2.2.2 { + set C [grp output] + grp delete + changeset_to_list $C +} { + {INSERT t2 0 X.. {} {i -5 i -10 i -20}} +} + +do_test 2.2.3 { + sqlite3changegroup grp + grp schema db main + grp change_begin INSERT "t1" 0 + grp change_int64 new 0 223344 + grp change_null new 1 + grp change_finish false +} {} + +do_test 2.2.4 { + set C [grp output] + grp delete + changeset_to_list $C +} { + {INSERT t1 0 X. {} {i 223344 n {}}} +} + +do_test 2.2.5 { + sqlite3changegroup grp + grp schema db main + grp change_begin DELETE "t1" 0 + + grp change_int64 old 0 1 + grp change_int64 old 1 123 + + grp change_finish false +} {} + +do_test 2.2.6 { + set C [grp output] + grp delete + changeset_to_list $C +} { + {DELETE t1 0 X. {i 1 i 123} {}} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + CREATE TABLE t2(x, y); +} + +foreach {tn script error} { + 1 { + grp change_begin UPDATE t1 0 + } {SQLITE_ERROR - invalid change: undefined value in PK of old.* record} + + 2 { + grp change_begin UPDATE t1 0 + grp change_int64 old 0 1234 + grp change_int64 new 0 5678 + } {SQLITE_ERROR - invalid change: defined value in PK of new.* record} + + 3 { + grp change_begin UPDATE t1 0 + grp change_null old 0 + } {SQLITE_ERROR - invalid change: null value in PK of old.* record} + + 4 { + grp change_begin UPDATE t1 0 + grp change_int64 old 0 1234 + grp change_int64 old 1 20 + } {SQLITE_ERROR - invalid change: column 1 - old.* value is defined but new.* is undefined} + + 5 { + grp change_begin UPDATE t1 0 + grp change_int64 old 0 1234 + grp change_int64 new 1 20 + } {SQLITE_ERROR - invalid change: column 1 - old.* value is undefined but new.* is defined} + + 6 { + grp change_begin INSERT t1 0 + grp change_null new 0 + grp change_int64 new 1 20 + } {SQLITE_ERROR - invalid change: null value in PK} + + 7 { + grp change_begin INSERT t1 0 + grp change_int64 new 1 20 + } {SQLITE_ERROR - invalid change: column 0 is undefined} + + 8 { + grp change_begin INSERT t1 0 + grp change_int64 new 0 20 + } {SQLITE_ERROR - invalid change: column 1 is undefined} + + 9 { + grp change_begin DELETE t1 0 + grp change_int64 old 0 20 + } {SQLITE_ERROR - invalid change: column 1 is undefined} + + 10 { + grp change_begin DELETE t1 0 + grp change_int64 old 1 20 + } {SQLITE_ERROR - invalid change: column 0 is undefined} + +} { + sqlite3changegroup grp + grp schema db main + eval $script + do_test 3.1.$tn { + list [catch { grp change_finish false } msg] $msg + } [list 1 $error] + grp delete +} + +do_test 3.2.1 { + sqlite3changegroup grp + grp schema db main + grp change_begin DELETE t1 0 + list [catch {grp change_int64 new 0 20} msg] $msg +} {1 SQLITE_ERROR} +do_test 3.2.2 { + grp change_finish true + grp change_begin INSERT t1 0 + list [catch {grp change_int64 old 0 20} msg] $msg +} {1 SQLITE_ERROR} + +grp delete + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a, b); +} + +proc do_changegroup_script_test {tn script changeset patchset} { + +breakpoint + sqlite3changegroup grp + grp schema db main + eval $script + do_test $tn.1 { + changeset_to_list [grp output] + } [list {*}$changeset] + grp delete + + sqlite3changegroup grp + grp config patchset 1 + grp schema db main + eval $script + do_test $tn.2 { + changeset_to_list [grp output] + } [list {*}$patchset] + grp delete + +} + +proc do_changegroup_patchset_test {tn script patchset} { + + sqlite3changegroup grp + grp config patchset 1 + grp schema db main + eval $script + do_test $tn { + changeset_to_list [grp output] + } [list {*}$patchset] + grp delete + +} + +do_changegroup_script_test 3.1 { + grp change_begin UPDATE t1 0 + grp change_int64 old 0 123 + grp change_int64 old 1 456 + grp change_int64 new 1 789 + grp change_finish false + + grp change_begin INSERT t1 0 + grp change_int64 new 0 124 + grp change_text new 1 hello_world + grp change_finish false + + grp change_begin DELETE t1 0 + grp change_int64 old 0 125 + grp change_double old 1 45.67 + grp change_finish false +} { + {DELETE t1 0 X. {i 125 f 45.67} {}} + {INSERT t1 0 X. {} {i 124 t hello_world}} + {UPDATE t1 0 X. {i 123 i 456} {{} {} i 789}} +} { + {DELETE t1 0 X. {i 125 {} {}} {}} + {INSERT t1 0 X. {} {i 124 t hello_world}} + {UPDATE t1 0 X. {i 123 {} {}} {{} {} i 789}} +} + +do_changegroup_script_test 3.1.2 { + grp change_begin INSERT t1 0 + grp change_int64 new 0 124 + grp change_text new 1 hello_world + grp change_finish false +} { + {INSERT t1 0 X. {} {i 124 t hello_world}} +} { + {INSERT t1 0 X. {} {i 124 t hello_world}} +} + +do_changegroup_patchset_test 3.2 { + grp change_begin UPDATE t1 0 + grp change_int64 old 0 123 + grp change_int64 new 1 789 + grp change_finish false + + grp change_begin DELETE t1 1 + grp change_int64 old 0 122 + grp change_finish false +} { + {DELETE t1 1 X. {i 122 {} {}} {}} + {UPDATE t1 0 X. {i 123 {} {}} {{} {} i 789}} +} + +do_changegroup_patchset_test 3.3 { + grp change_begin DELETE t1 0 + grp change_int64 old 0 123 + grp change_finish false +} { + {DELETE t1 0 X. {i 123 {} {}} {}} +} + +do_changegroup_script_test 3.4 { + grp change_begin INSERT t1 0 + grp change_int64 new 0 124 + grp change_text new 1 hello_world + grp change_finish false + + grp change_begin DELETE t1 0 + grp change_int64 old 0 124 + grp change_text old 1 hello_world + grp change_finish false + + grp change_begin INSERT t1 0 + grp change_int64 new 0 1000 + grp change_blob new 1 abcd + grp change_finish false + + grp change_begin UPDATE t1 0 + grp change_int64 old 0 1000 + grp change_blob old 1 abcd + grp change_blob new 1 bcda + grp change_finish false +} { + {INSERT t1 0 X. {} {i 1000 b bcda}} +} { + {INSERT t1 0 X. {} {i 1000 b bcda}} +} + +do_changegroup_script_test 3.5 { + grp change_begin DELETE t1 0 + grp change_int64 old 0 1000 + grp change_text-1 old 1 "hello world" + grp change_finish false +} { + {DELETE t1 0 X. {i 1000 t {hello world}} {}} +} { + {DELETE t1 0 X. {i 1000 {} {}} {}} +} + +#------------------------------------------------------------------------- + +# Calling sqlite3changegroup_change_finish() if a change has not been +# started is a no-op. +# +do_test 4.0 { + sqlite3changegroup grp + grp change_finish false +} {} + +# But calling any other sqlite3changegroup_change_xxx() function if a +# change has not been started is a misuse. +# +foreach {tn cmd} { + 1 { grp change_blob old 0 abcd } + 2 { grp change_text new 1 helloworld } + 3 { grp change_int64 old 2 12345678 } + 4 { grp change_double new 3 12.345 } + 5 { grp change_null old 4 } +} { + do_test 4.1.$tn { + list [catch $::cmd msg] $msg + } {1 SQLITE_MISUSE} +} +grp delete + +do_execsql_test 4.2.1 { + CREATE TABLE t3(a, b PRIMARY KEY); +} +do_test 4.2 { + sqlite3changegroup grp + grp schema db main + grp change_begin INSERT t3 0 + + list [catch { grp change_begin INSERT t3 0 } msg] $msg +} {1 SQLITE_MISUSE} +grp delete + +do_test 4.3 { + sqlite3changegroup grp + grp schema db main + grp change_begin UPDATE t3 0 +} {} +foreach {tn cmd} { + 1 { grp change_blob old -1 abcd } + 2 { grp change_text new 2 helloworld } +} { + do_test 4.4.$tn { + list [catch $::cmd msg] $msg + } {1 SQLITE_RANGE} +} +grp delete + +finish_test + diff --git a/ext/session/sessionfault3.test b/ext/session/sessionfault3.test index 5af9c9ed7..f2bcb8941 100644 --- a/ext/session/sessionfault3.test +++ b/ext/session/sessionfault3.test @@ -43,7 +43,7 @@ set C2 [changeset_from_sql { INSERT INTO t1 VALUES(0, 0, 0, 0); }] -do_faultsim_test 1 -faults oom* -prep { +do_faultsim_test 1.2 -faults oom* -prep { sqlite3changegroup G } -body { G schema db main @@ -56,6 +56,20 @@ do_faultsim_test 1 -faults oom* -prep { faultsim_test_result {0 {}} {1 SQLITE_NOMEM} } +do_faultsim_test 1.3 -faults oom* -prep { + sqlite3changegroup G +} -body { + G schema db main + G change_begin INSERT t1 0 + G change_int64 new 0 12345 + G change_int64 new 1 67890 + G output + set {} {} +} -test { + catch { G delete } + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} +} + #------------------------------------------------------------------------- reset_db do_execsql_test 2.0 { diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index 90fedc6db..cb5f4c1cc 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -349,6 +349,21 @@ static int sessionVarintGet(const u8 *aBuf, int *piVal){ return getVarint32(aBuf, *piVal); } +/* +** Read a varint value from buffer aBuf[], size nBuf bytes, into *piVal. +** Return the number of bytes read. +*/ +static int sessionVarintGetSafe(const u8 *aBuf, int nBuf, int *piVal){ + u8 aCopy[9]; + const u8 *aRead = aBuf; + memset(aCopy, 0, sizeof(aCopy)); + if( nBuf<sizeof(aCopy) ){ + memcpy(aCopy, aBuf, nBuf); + aRead = aCopy; + } + return getVarint32(aRead, *piVal); +} + /* Load an unaligned and unsigned 32-bit integer */ #define SESSION_UINT32(x) (((u32)(x)[0]<<24)|((x)[1]<<16)|((x)[2]<<8)|(x)[3]) @@ -377,6 +392,19 @@ static void sessionPutI64(u8 *aBuf, sqlite3_int64 i){ aBuf[7] = (i>> 0) & 0xFF; } +/* +** Write a double value to the buffer aBuf[]. +*/ +static void sessionPutDouble(u8 *aBuf, double r){ + /* TODO: SQLite does something special to deal with mixed-endian + ** floating point values (e.g. ARM7). This code probably should + ** too. */ + u64 i; + assert( sizeof(double)==8 && sizeof(u64)==8 ); + memcpy(&i, &r, 8); + sessionPutI64(aBuf, i); +} + /* ** This function is used to serialize the contents of value pValue (see ** comment titled "RECORD FORMAT" above). @@ -414,16 +442,13 @@ static int sessionSerializeValue( /* TODO: SQLite does something special to deal with mixed-endian ** floating point values (e.g. ARM7). This code probably should ** too. */ - u64 i; if( eType==SQLITE_INTEGER ){ - i = (u64)sqlite3_value_int64(pValue); + u64 i = (u64)sqlite3_value_int64(pValue); + sessionPutI64(&aBuf[1], i); }else{ - double r; - assert( sizeof(double)==8 && sizeof(u64)==8 ); - r = sqlite3_value_double(pValue); - memcpy(&i, &r, 8); + double r = sqlite3_value_double(pValue); + sessionPutDouble(&aBuf[1], r); } - sessionPutI64(&aBuf[1], i); } nByte = 9; break; @@ -643,14 +668,10 @@ static unsigned int sessionChangeHash( int isPK = pTab->abPK[i]; if( bPkOnly && isPK==0 ) continue; - /* It is not possible for eType to be SQLITE_NULL here. The session - ** module does not record changes for rows with NULL values stored in - ** primary key columns. */ assert( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT || eType==SQLITE_TEXT || eType==SQLITE_BLOB || eType==SQLITE_NULL || eType==0 ); - assert( !isPK || (eType!=0 && eType!=SQLITE_NULL) ); if( isPK ){ a++; @@ -658,12 +679,16 @@ static unsigned int sessionChangeHash( if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ h = sessionHashAppendI64(h, sessionGetI64(a)); a += 8; - }else{ + }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ int n; a += sessionVarintGet(a, &n); h = sessionHashAppendBlob(h, n, a); a += n; } + /* It should not be possible for eType to be SQLITE_NULL or 0x00 here, + ** as the session module does not record changes for rows with NULL + ** values stored in primary key columns. But a corrupt changesets + ** may contain such a value. */ }else{ a += sessionSerialLen(a); } @@ -1355,9 +1380,7 @@ static void sessionUpdateOneChange( case SQLITE_FLOAT: { double rVal = sqlite3_column_double(pDflt, iField); - i64 iVal = 0; - memcpy(&iVal, &rVal, sizeof(rVal)); - sessionPutI64(&pNew->aRecord[pNew->nRecord], iVal); + sessionPutDouble(&pNew->aRecord[pNew->nRecord], rVal); pNew->nRecord += 8; break; } @@ -2614,15 +2637,14 @@ static void sessionAppendCol( int eType = sqlite3_column_type(pStmt, iCol); sessionAppendByte(p, (u8)eType, pRc); if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ - sqlite3_int64 i; u8 aBuf[8]; if( eType==SQLITE_INTEGER ){ - i = sqlite3_column_int64(pStmt, iCol); + sqlite3_int64 i = sqlite3_column_int64(pStmt, iCol); + sessionPutI64(aBuf, i); }else{ double r = sqlite3_column_double(pStmt, iCol); - memcpy(&i, &r, 8); + sessionPutDouble(aBuf, r); } - sessionPutI64(aBuf, i); sessionAppendBlob(p, aBuf, 8, pRc); } if( eType==SQLITE_BLOB || eType==SQLITE_TEXT ){ @@ -3072,10 +3094,13 @@ static int sessionGenerateChangeset( } if( pSession->rc ) return pSession->rc; - rc = sqlite3_exec(pSession->db, "SAVEPOINT changeset", 0, 0, 0); - if( rc!=SQLITE_OK ) return rc; sqlite3_mutex_enter(sqlite3_db_mutex(db)); + rc = sqlite3_exec(pSession->db, "SAVEPOINT changeset", 0, 0, 0); + if( rc!=SQLITE_OK ){ + sqlite3_mutex_leave(sqlite3_db_mutex(db)); + return rc; + } for(pTab=pSession->pTable; rc==SQLITE_OK && pTab; pTab=pTab->pNext){ if( pTab->nEntry ){ @@ -3558,7 +3583,8 @@ static int sessionReadRecord( u8 *aVal = &pIn->aData[pIn->iNext]; if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ int nByte; - pIn->iNext += sessionVarintGet(aVal, &nByte); + int nRem = pIn->nData - pIn->iNext; + pIn->iNext += sessionVarintGetSafe(aVal, nRem, &nByte); rc = sessionInputBuffer(pIn, nByte); if( rc==SQLITE_OK ){ if( nByte<0 || nByte>pIn->nData-pIn->iNext ){ @@ -3611,7 +3637,8 @@ static int sessionChangesetBufferTblhdr(SessionInput *pIn, int *pnByte){ rc = sessionInputBuffer(pIn, 9); if( rc==SQLITE_OK ){ - nRead += sessionVarintGet(&pIn->aData[pIn->iNext + nRead], &nCol); + int nBuf = pIn->nData - pIn->iNext; + nRead += sessionVarintGetSafe(&pIn->aData[pIn->iNext], nBuf, &nCol); /* The hard upper limit for the number of columns in an SQLite ** database table is, according to sqliteLimit.h, 32676. So ** consider any table-header that purports to have more than 65536 @@ -3631,8 +3658,15 @@ static int sessionChangesetBufferTblhdr(SessionInput *pIn, int *pnByte){ while( (pIn->iNext + nRead)<pIn->nData && pIn->aData[pIn->iNext + nRead] ){ nRead++; } + + /* Break out of the loop if if the nul-terminator byte has been found. + ** Otherwise, read some more input data and keep seeking. If there is + ** no more input data, consider the changeset corrupt. */ if( (pIn->iNext + nRead)<pIn->nData ) break; rc = sessionInputBuffer(pIn, nRead + 100); + if( rc==SQLITE_OK && (pIn->iNext + nRead)>=pIn->nData ){ + rc = SQLITE_CORRUPT_BKPT; + } } *pnByte = nRead+1; return rc; @@ -3653,7 +3687,7 @@ static int sessionChangesetBufferRecord( int *pnByte /* OUT: Size of record in bytes */ ){ int rc = SQLITE_OK; - int nByte = 0; + i64 nByte = 0; int i; for(i=0; rc==SQLITE_OK && i<nCol; i++){ int eType; @@ -3662,13 +3696,17 @@ static int sessionChangesetBufferRecord( eType = pIn->aData[pIn->iNext + nByte++]; if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ int n; - nByte += sessionVarintGet(&pIn->aData[pIn->iNext+nByte], &n); + int nRem = pIn->nData - (pIn->iNext + nByte); + nByte += sessionVarintGetSafe(&pIn->aData[pIn->iNext+nByte], nRem, &n); nByte += n; rc = sessionInputBuffer(pIn, nByte); }else if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ nByte += 8; } } + if( (pIn->iNext+nByte)>pIn->nData ){ + rc = SQLITE_CORRUPT_BKPT; + } } *pnByte = nByte; return rc; @@ -3764,10 +3802,10 @@ static int sessionChangesetNextOne( memset(p->apValue, 0, sizeof(sqlite3_value*)*p->nCol*2); } - /* Make sure the buffer contains at least 10 bytes of input data, or all - ** remaining data if there are less than 10 bytes available. This is - ** sufficient either for the 'T' or 'P' byte and the varint that follows - ** it, or for the two single byte values otherwise. */ + /* Make sure the buffer contains at least 2 bytes of input data, or all + ** remaining data if there are less than 2 bytes available. This is + ** sufficient either for the 'T' or 'P' byte that begins a new table, + ** or for the "op" and "bIndirect" single bytes otherwise. */ p->rc = sessionInputBuffer(&p->in, 2); if( p->rc!=SQLITE_OK ) return p->rc; @@ -3797,11 +3835,13 @@ static int sessionChangesetNextOne( return (p->rc = SQLITE_CORRUPT_BKPT); } - p->op = op; - p->bIndirect = p->in.aData[p->in.iNext++]; - if( p->op!=SQLITE_UPDATE && p->op!=SQLITE_DELETE && p->op!=SQLITE_INSERT ){ + if( (op!=SQLITE_UPDATE && op!=SQLITE_DELETE && op!=SQLITE_INSERT) + || (p->in.iNext>=p->in.nData) + ){ return (p->rc = SQLITE_CORRUPT_BKPT); } + p->op = op; + p->bIndirect = p->in.aData[p->in.iNext++]; if( paRec ){ int nVal; /* Number of values to buffer */ @@ -5649,6 +5689,21 @@ int sqlite3changeset_apply_strm( ); } +/* +** The parts of the sqlite3_changegroup structure used by the +** sqlite3changegroup_change_xxx() APIs. +*/ +typedef struct ChangeData ChangeData; +struct ChangeData { + SessionTable *pTab; + int bIndirect; + int eOp; + + int nBufAlloc; + SessionBuffer *aBuf; + SessionBuffer record; +}; + /* ** sqlite3_changegroup handle. */ @@ -5660,12 +5715,17 @@ struct sqlite3_changegroup { sqlite3 *db; /* Configured by changegroup_schema() */ char *zDb; /* Configured by changegroup_schema() */ + ChangeData cd; /* Used by changegroup_change_xxx() APIs. */ }; /* ** This function is called to merge two changes to the same row together as ** part of an sqlite3changeset_concat() operation. A new change object is ** allocated and a pointer to it stored in *ppNew. +** +** Because they have been vetted by sqlite3changegroup_add() or similar, +** both the aRec[] change and the pExist change are safe to use without +** checking for buffer overflows. */ static int sessionChangeMerge( SessionTable *pTab, /* Table structure */ @@ -5806,7 +5866,7 @@ static int sessionChangeMerge( memcpy(aCsr, aRec, nRec); aCsr += nRec; }else{ - if( 0==sessionMergeUpdate(&aCsr, pTab, bPatchset, aExist, 0,aRec,0) ){ + if( 0==sessionMergeUpdate(&aCsr, pTab, bPatchset, aExist,0,aRec,0) ){ sqlite3_free(pNew); pNew = 0; } @@ -5899,15 +5959,14 @@ static int sessionChangesetExtendRecord( switch( eType ){ case SQLITE_FLOAT: case SQLITE_INTEGER: { - i64 iVal; - if( eType==SQLITE_INTEGER ){ - iVal = sqlite3_column_int64(pTab->pDfltStmt, ii); - }else{ - double rVal = sqlite3_column_int64(pTab->pDfltStmt, ii); - memcpy(&iVal, &rVal, sizeof(i64)); - } if( SQLITE_OK==sessionBufferGrow(pOut, 8, &rc) ){ - sessionPutI64(&pOut->aBuf[pOut->nBuf], iVal); + if( eType==SQLITE_INTEGER ){ + sqlite3_int64 iVal = sqlite3_column_int64(pTab->pDfltStmt, ii); + sessionPutI64(&pOut->aBuf[pOut->nBuf], iVal); + }else{ + double rVal = sqlite3_column_double(pTab->pDfltStmt, ii); + sessionPutDouble(&pOut->aBuf[pOut->nBuf], rVal); + } pOut->nBuf += 8; } break; @@ -5978,13 +6037,19 @@ static int sessionChangesetFindTable( int nCol = 0; *ppTab = 0; - sqlite3changeset_pk(pIter, &abPK, &nCol); /* Search the list for an existing table */ for(pTab = pGrp->pList; pTab; pTab=pTab->pNext){ if( 0==sqlite3_strnicmp(pTab->zName, zTab, nTab+1) ) break; } + + if( pIter ){ + sqlite3changeset_pk(pIter, &abPK, &nCol); + }else if( !pTab && !pGrp->db ){ + return SQLITE_OK; + } + /* If one was not found above, create a new table now */ if( !pTab ){ SessionTable **ppNew; @@ -5996,15 +6061,17 @@ static int sessionChangesetFindTable( memset(pTab, 0, sizeof(SessionTable)); pTab->nCol = nCol; pTab->abPK = (u8*)&pTab[1]; - memcpy(pTab->abPK, abPK, nCol); + if( nCol>0 ){ + memcpy(pTab->abPK, abPK, nCol); + } pTab->zName = (char*)&pTab->abPK[nCol]; memcpy(pTab->zName, zTab, nTab+1); if( pGrp->db ){ pTab->nCol = 0; rc = sessionInitTable(0, pTab, pGrp->db, pGrp->zDb); - if( rc ){ - assert( pTab->azCol==0 ); + if( rc || pTab->nCol==0 ){ + sqlite3_free(pTab->azCol); sqlite3_free(pTab); return rc; } @@ -6019,7 +6086,7 @@ static int sessionChangesetFindTable( } /* Check that the table is compatible. */ - if( !sessionChangesetCheckCompat(pTab, nCol, abPK) ){ + if( pIter && !sessionChangesetCheckCompat(pTab, nCol, abPK) ){ rc = SQLITE_SCHEMA; } @@ -6028,44 +6095,27 @@ static int sessionChangesetFindTable( } /* -** Add the change currently indicated by iterator pIter to the hash table -** belonging to changegroup pGrp. +** Add a single change to the changegroup pGrp. */ static int sessionOneChangeToHash( - sqlite3_changegroup *pGrp, - sqlite3_changeset_iter *pIter, - int bRebase + sqlite3_changegroup *pGrp, /* Changegroup to update */ + SessionTable *pTab, /* Table change pertains to */ + int op, /* One of SQLITE_INSERT, UPDATE, DELETE */ + int bIndirect, /* True to flag change as "indirect" */ + int nCol, /* Number of columns in record(s) */ + u8 *aRec, /* Serialized change record(s) */ + int nRec, /* Size of aRec[] in bytes */ + int bRebase /* True if this is a rebase blob */ ){ int rc = SQLITE_OK; - int nCol = 0; - int op = 0; int iHash = 0; - int bIndirect = 0; SessionChange *pChange = 0; SessionChange *pExist = 0; SessionChange **pp = 0; - SessionTable *pTab = 0; - u8 *aRec = &pIter->in.aData[pIter->in.iCurrent + 2]; - int nRec = (pIter->in.iNext - pIter->in.iCurrent) - 2; assert( nRec>0 ); - /* Ensure that only changesets, or only patchsets, but not a mixture - ** of both, are being combined. It is an error to try to combine a - ** changeset and a patchset. */ - if( pGrp->pList==0 ){ - pGrp->bPatch = pIter->bPatchset; - }else if( pIter->bPatchset!=pGrp->bPatch ){ - rc = SQLITE_ERROR; - } - - if( rc==SQLITE_OK ){ - const char *zTab = 0; - sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); - rc = sessionChangesetFindTable(pGrp, zTab, pIter, &pTab); - } - - if( rc==SQLITE_OK && nCol<pTab->nCol ){ + if( nCol<pTab->nCol ){ SessionBuffer *pBuf = &pGrp->rec; rc = sessionChangesetExtendRecord(pGrp, pTab, nCol, op, aRec, nRec, pBuf); aRec = pBuf->aBuf; @@ -6073,7 +6123,7 @@ static int sessionOneChangeToHash( assert( pGrp->db ); } - if( rc==SQLITE_OK && sessionGrowHash(0, pIter->bPatchset, pTab) ){ + if( rc==SQLITE_OK && sessionGrowHash(0, pGrp->bPatch, pTab) ){ rc = SQLITE_NOMEM; } @@ -6081,12 +6131,12 @@ static int sessionOneChangeToHash( /* Search for existing entry. If found, remove it from the hash table. ** Code below may link it back in. */ iHash = sessionChangeHash( - pTab, (pIter->bPatchset && op==SQLITE_DELETE), aRec, pTab->nChange + pTab, (pGrp->bPatch && op==SQLITE_DELETE), aRec, pTab->nChange ); for(pp=&pTab->apChange[iHash]; *pp; pp=&(*pp)->pNext){ int bPkOnly1 = 0; int bPkOnly2 = 0; - if( pIter->bPatchset ){ + if( pGrp->bPatch ){ bPkOnly1 = (*pp)->op==SQLITE_DELETE; bPkOnly2 = op==SQLITE_DELETE; } @@ -6101,7 +6151,7 @@ static int sessionOneChangeToHash( if( rc==SQLITE_OK ){ rc = sessionChangeMerge(pTab, bRebase, - pIter->bPatchset, pExist, op, bIndirect, aRec, nRec, &pChange + pGrp->bPatch, pExist, op, bIndirect, aRec, nRec, &pChange ); } if( rc==SQLITE_OK && pChange ){ @@ -6110,6 +6160,47 @@ static int sessionOneChangeToHash( pTab->nEntry++; } + return rc; +} + +/* +** Add the change currently indicated by iterator pIter to the hash table +** belonging to changegroup pGrp. +*/ +static int sessionOneChangeIterToHash( + sqlite3_changegroup *pGrp, + sqlite3_changeset_iter *pIter, + int bRebase +){ + u8 *aRec = &pIter->in.aData[pIter->in.iCurrent + 2]; + int nRec = (pIter->in.iNext - pIter->in.iCurrent) - 2; + const char *zTab = 0; + int nCol = 0; + int op = 0; + int bIndirect = 0; + int rc = SQLITE_OK; + SessionTable *pTab = 0; + + /* Ensure that only changesets, or only patchsets, but not a mixture + ** of both, are being combined. It is an error to try to combine a + ** changeset and a patchset. */ + if( pGrp->pList==0 ){ + pGrp->bPatch = pIter->bPatchset; + }else if( pIter->bPatchset!=pGrp->bPatch ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK ){ + sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); + rc = sessionChangesetFindTable(pGrp, zTab, pIter, &pTab); + } + + if( rc==SQLITE_OK ){ + rc = sessionOneChangeToHash( + pGrp, pTab, op, bIndirect, nCol, aRec, nRec, bRebase + ); + } + if( rc==SQLITE_OK ) rc = pIter->rc; return rc; } @@ -6129,7 +6220,7 @@ static int sessionChangesetToHash( pIter->in.bNoDiscard = 1; while( SQLITE_ROW==(sessionChangesetNext(pIter, &aRec, &nRec, 0)) ){ - rc = sessionOneChangeToHash(pGrp, pIter, bRebase); + rc = sessionOneChangeIterToHash(pGrp, pIter, bRebase); if( rc!=SQLITE_OK ) break; } @@ -6219,6 +6310,33 @@ int sqlite3changegroup_new(sqlite3_changegroup **pp){ return rc; } +/* +** Configure a changegroup object. +*/ +int sqlite3changegroup_config( + sqlite3_changegroup *pGrp, + int op, + void *pArg +){ + int rc = SQLITE_OK; + + switch( op ){ + case SQLITE_CHANGEGROUP_CONFIG_PATCHSET: { + int arg = *(int*)pArg; + if( pGrp->pList==0 && arg>=0 ){ + pGrp->bPatch = (arg>0); + } + *(int*)pArg = pGrp->bPatch; + break; + } + default: + rc = SQLITE_MISUSE; + break; + } + + return rc; +} + /* ** Provide a database schema to the changegroup object. */ @@ -6277,7 +6395,7 @@ int sqlite3changegroup_add_change( rc = SQLITE_ERROR; }else{ pIter->in.bNoDiscard = 1; - rc = sessionOneChangeToHash(pGrp, pIter, 0); + rc = sessionOneChangeIterToHash(pGrp, pIter, 0); } return rc; } @@ -6329,6 +6447,12 @@ int sqlite3changegroup_output_strm( */ void sqlite3changegroup_delete(sqlite3_changegroup *pGrp){ if( pGrp ){ + int ii; + for(ii=0; ii<pGrp->cd.nBufAlloc; ii++){ + sqlite3_free(pGrp->cd.aBuf[ii].aBuf); + } + sqlite3_free(pGrp->cd.record.aBuf); + sqlite3_free(pGrp->cd.aBuf); sqlite3_free(pGrp->zDb); sessionDeleteTable(0, pGrp->pList); sqlite3_free(pGrp->rec.aBuf); @@ -6759,4 +6883,326 @@ int sqlite3session_config(int op, void *pArg){ return rc; } +/* +** Begin adding a change to a changegroup object. +*/ +int sqlite3changegroup_change_begin( + sqlite3_changegroup *pGrp, + int eOp, + const char *zTab, + int bIndirect, + char **pzErr +){ + SessionTable *pTab = 0; + int rc = SQLITE_OK; + + if( pGrp->cd.pTab ){ + rc = SQLITE_MISUSE; + }else if( eOp!=SQLITE_INSERT && eOp!=SQLITE_UPDATE && eOp!=SQLITE_DELETE ){ + rc = SQLITE_ERROR; + }else{ + rc = sessionChangesetFindTable(pGrp, zTab, 0, &pTab); + } + if( rc==SQLITE_OK ){ + if( pTab==0 ){ + if( pzErr ){ + *pzErr = sqlite3_mprintf("no such table: %s", zTab); + } + rc = SQLITE_ERROR; + }else{ + int nReq = pTab->nCol * (eOp==SQLITE_UPDATE ? 2 : 1); + pGrp->cd.pTab = pTab; + pGrp->cd.eOp = eOp; + pGrp->cd.bIndirect = bIndirect; + + if( pGrp->cd.nBufAlloc<nReq ){ + SessionBuffer *aBuf = (SessionBuffer*)sqlite3_realloc( + pGrp->cd.aBuf, nReq * sizeof(SessionBuffer) + ); + if( aBuf==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(&aBuf[pGrp->cd.nBufAlloc], 0, + sizeof(SessionBuffer) * (nReq - pGrp->cd.nBufAlloc) + ); + pGrp->cd.aBuf = aBuf; + pGrp->cd.nBufAlloc = nReq; + } + } + +#ifdef SQLITE_DEBUG + { + /* Assert that all column values are currently undefined */ + int ii; + for(ii=0; ii<pGrp->cd.nBufAlloc; ii++){ + assert( pGrp->cd.aBuf[ii].nBuf==0 ); + } + } +#endif + } + } + + return rc; +} + +/* +** This function does processing common to the _change_int64(), _change_text() +** and other similar APIs. +*/ +static int checkChangeParams( + sqlite3_changegroup *pGrp, + int bNew, + int iCol, + sqlite3_int64 nReq, + SessionBuffer **ppBuf +){ + int rc = SQLITE_OK; + if( pGrp->cd.pTab==0 ){ + rc = SQLITE_MISUSE; + }else if( iCol<0 || iCol>=pGrp->cd.pTab->nCol ){ + rc = SQLITE_RANGE; + }else if( + (bNew && pGrp->cd.eOp==SQLITE_DELETE) + || (!bNew && pGrp->cd.eOp==SQLITE_INSERT) + ){ + rc = SQLITE_ERROR; + }else{ + SessionBuffer *pBuf = &pGrp->cd.aBuf[iCol]; + if( pGrp->cd.eOp==SQLITE_UPDATE && bNew ){ + pBuf += pGrp->cd.pTab->nCol; + } + pBuf->nBuf = 0; + sessionBufferGrow(pBuf, nReq, &rc); + pBuf->nBuf = nReq; + *ppBuf = pBuf; + } + return rc; +} + +/* +** Configure the change currently under construction with an integer value. +*/ +int sqlite3changegroup_change_int64( + sqlite3_changegroup *pGrp, + int bNew, + int iCol, + sqlite3_int64 iVal +){ + int rc = SQLITE_OK; + SessionBuffer *pBuf = 0; + + if( SQLITE_OK!=(rc = checkChangeParams(pGrp, bNew, iCol, 9, &pBuf)) ){ + return rc; + } + + pBuf->aBuf[0] = SQLITE_INTEGER; + sessionPutI64(&pBuf->aBuf[1], iVal); + return SQLITE_OK; +} + +/* +** Configure the change currently under construction with a null value. +*/ +int sqlite3changegroup_change_null( + sqlite3_changegroup *pGrp, + int bNew, + int iCol +){ + int rc = SQLITE_OK; + SessionBuffer *pBuf = 0; + + if( SQLITE_OK!=(rc = checkChangeParams(pGrp, bNew, iCol, 1, &pBuf)) ){ + return rc; + } + + pBuf->aBuf[0] = SQLITE_NULL; + return SQLITE_OK; +} + +/* +** Configure the change currently under construction with a real value. +*/ +int sqlite3changegroup_change_double( + sqlite3_changegroup *pGrp, + int bNew, + int iCol, + double fVal +){ + int rc = SQLITE_OK; + SessionBuffer *pBuf = 0; + + if( SQLITE_OK!=(rc = checkChangeParams(pGrp, bNew, iCol, 9, &pBuf)) ){ + return rc; + } + + pBuf->aBuf[0] = SQLITE_FLOAT; + sessionPutDouble(&pBuf->aBuf[1], fVal); + return SQLITE_OK; +} + +/* +** Configure the change currently under construction with a text value. +*/ +int sqlite3changegroup_change_text( + sqlite3_changegroup *pGrp, + int bNew, + int iCol, + const char *pVal, + int nVal +){ + int nText = nVal>=0 ? nVal : strlen(pVal); + sqlite3_int64 nByte = 1 + sessionVarintLen(nText) + nText; + int rc = SQLITE_OK; + SessionBuffer *pBuf = 0; + + if( SQLITE_OK!=(rc = checkChangeParams(pGrp, bNew, iCol, nByte, &pBuf)) ){ + return rc; + } + + pBuf->aBuf[0] = SQLITE_TEXT; + pBuf->nBuf = (1 + sessionVarintPut(&pBuf->aBuf[1], nText)); + memcpy(&pBuf->aBuf[pBuf->nBuf], pVal, nText); + pBuf->nBuf += nText; + + return SQLITE_OK; +} + +/* +** Configure the change currently under construction with a blob value. +*/ +int sqlite3changegroup_change_blob( + sqlite3_changegroup *pGrp, + int bNew, + int iCol, + const void *pVal, + int nVal +){ + sqlite3_int64 nByte = 1 + sessionVarintLen(nVal) + nVal; + int rc = SQLITE_OK; + SessionBuffer *pBuf = 0; + + if( SQLITE_OK!=(rc = checkChangeParams(pGrp, bNew, iCol, nByte, &pBuf)) ){ + return rc; + } + + pBuf->aBuf[0] = SQLITE_BLOB; + pBuf->nBuf = (1 + sessionVarintPut(&pBuf->aBuf[1], nVal)); + memcpy(&pBuf->aBuf[pBuf->nBuf], pVal, nVal); + pBuf->nBuf += nVal; + + return SQLITE_OK; +} + +/* +** Finish any change currently being constructed by the changegroup object. +*/ +int sqlite3changegroup_change_finish( + sqlite3_changegroup *pGrp, + int bDiscard, + char **pzErr +){ + int rc = SQLITE_OK; + if( pGrp->cd.pTab ){ + SessionBuffer *aBuf = pGrp->cd.aBuf; + int ii; + + if( bDiscard==0 ){ + int nBuf = pGrp->cd.pTab->nCol; + u8 eUndef = SQLITE_NULL; + if( pGrp->cd.eOp==SQLITE_UPDATE ){ + for(ii=0; ii<nBuf; ii++){ + if( pGrp->cd.pTab->abPK[ii] ){ + if( aBuf[ii].nBuf<=1 ){ + *pzErr = sqlite3_mprintf( + "invalid change: %s value in PK of old.* record", + aBuf[ii].nBuf==1 ? "null" : "undefined" + ); + rc = SQLITE_ERROR; + break; + }else if( aBuf[ii + nBuf].nBuf>0 ){ + *pzErr = sqlite3_mprintf( + "invalid change: defined value in PK of new.* record" + ); + rc = SQLITE_ERROR; + break; + } + }else + if( pGrp->bPatch==0 && (aBuf[ii].nBuf>0)!=(aBuf[ii+nBuf].nBuf>0) ){ + *pzErr = sqlite3_mprintf( + "invalid change: column %d " + "- old.* value is %sdefined but new.* is %sdefined", + ii, aBuf[ii].nBuf ? "" : "un", aBuf[ii+nBuf].nBuf ? "" : "un" + ); + rc = SQLITE_ERROR; + break; + } + } + eUndef = 0x00; + if( pGrp->bPatch==0 ) nBuf = nBuf * 2; + }else{ + for(ii=0; ii<nBuf; ii++){ + int isPK = pGrp->cd.pTab->abPK[ii]; + if( (pGrp->cd.eOp==SQLITE_INSERT || pGrp->bPatch==0 || isPK) + && aBuf[ii].nBuf==0 + ){ + *pzErr = sqlite3_mprintf( + "invalid change: column %d is undefined", ii + ); + rc = SQLITE_ERROR; + break; + } + if( aBuf[ii].nBuf==1 && isPK ){ + *pzErr = sqlite3_mprintf( + "invalid change: null value in PK" + ); + rc = SQLITE_ERROR; + break; + } + } + } + + pGrp->cd.record.nBuf = 0; + for(ii=0; ii<nBuf; ii++){ + SessionBuffer *p = &pGrp->cd.aBuf[ii]; + if( pGrp->bPatch ){ + if( pGrp->cd.pTab->abPK[ii]==0 ){ + if( pGrp->cd.eOp==SQLITE_UPDATE ){ + p += pGrp->cd.pTab->nCol; + }else if( pGrp->cd.eOp==SQLITE_DELETE ){ + continue; + } + } + } + if( 0==sessionBufferGrow(&pGrp->cd.record, p->nBuf?p->nBuf:1, &rc) ){ + if( p->nBuf ){ + memcpy(&pGrp->cd.record.aBuf[pGrp->cd.record.nBuf],p->aBuf,p->nBuf); + pGrp->cd.record.nBuf += p->nBuf; + }else{ + pGrp->cd.record.aBuf[pGrp->cd.record.nBuf++] = eUndef; + } + } + } + if( rc==SQLITE_OK ){ + rc = sessionOneChangeToHash( + pGrp, pGrp->cd.pTab, + pGrp->cd.eOp, pGrp->cd.bIndirect, pGrp->cd.pTab->nCol, + pGrp->cd.record.aBuf, pGrp->cd.record.nBuf, 0 + ); + } + } + + /* Reset all aBuf[] entries to "undefined". */ + { + int nZero = pGrp->cd.pTab->nCol; + if( pGrp->cd.eOp==SQLITE_UPDATE ) nZero += nZero; + for(ii=0; ii<nZero; ii++){ + pGrp->cd.aBuf[ii].nBuf = 0; + } + } + pGrp->cd.pTab = 0; + } + + return rc; +} + #endif /* SQLITE_ENABLE_SESSION && SQLITE_ENABLE_PREUPDATE_HOOK */ diff --git a/ext/session/sqlite3session.h b/ext/session/sqlite3session.h index 28b90eb6b..fb2336d32 100644 --- a/ext/session/sqlite3session.h +++ b/ext/session/sqlite3session.h @@ -1853,6 +1853,232 @@ int sqlite3session_config(int op, void *pArg); */ #define SQLITE_SESSION_CONFIG_STRMSIZE 1 +/* +** CAPI3REF: Configure a changegroup object +** +** Configure the changegroup object passed as the first argument. +** At present the only valid value for the second parameter is +** [SQLITE_CHANGEGROUP_CONFIG_PATCHSET]. +*/ +int sqlite3changegroup_config(sqlite3_changegroup*, int, void *pArg); + +/* +** CAPI3REF: Options for sqlite3changegroup_config(). +** +** The following values may be passed as the 2nd parameter to +** sqlite3changegroup_config(). +** +** <dt>SQLITE_CHANGEGROUP_CONFIG_PATCHSET <dd> +** A changegroup object generates either a changeset or patchset. Usually, +** this is determined by whether the first call to sqlite3changegroup_add() +** is passed a changeset or a patchset. Or, if the first changes are added +** to the changegroup object using the sqlite3changegroup_change_xxx() +** APIs, then this option may be used to configure whether the changegroup +** object generates a changeset or patchset. +** +** When this option is invoked, parameter pArg must point to a value of +** type int. If the changegroup currently contains zero changes, and the +** value of the int variable is zero or greater than zero, then the +** changegroup is configured to generate a changeset or patchset, +** respectively. It is a no-op, not an error, if the changegroup is not +** configured because it has already started accumulating changes. +** +** Before returning, the int variable is set to 0 if the changegroup is +** configured to generate a changeset, or 1 if it is configured to generate +** a patchset. +*/ +#define SQLITE_CHANGEGROUP_CONFIG_PATCHSET 1 + + +/* +** CAPI3REF: Begin adding a change to a changegroup +** +** This API is used, in concert with other sqlite3changegroup_change_xxx() +** APIs, to add changes to a changegroup object one at a time. To add a +** single change, the caller must: +** +** 1. Invoke sqlite3changegroup_change_begin() to indicate the type of +** change (INSERT, UPDATE or DELETE), the affected table and whether +** or not the change should be marked as indirect. +** +** 2. Invoke sqlite3changegroup_change_int64() or one of the other four +** value functions - _null(), _double(), _text() or _blob() - one or +** more times to specify old.* and new.* values for the change being +** constructed. +** +** 3. Invoke sqlite3changegroup_change_finish() to either finish adding +** the change to the group, or to discard the change altogether. +** +** The first argument to this function must be a pointer to the existing +** changegroup object that the change will be added to. The second argument +** must be SQLITE_INSERT, SQLITE_UPDATE or SQLITE_DELETE. The third is the +** name of the table that the change affects, and the fourth is a boolean +** flag specifying whether the change should be marked as "indirect" (if +** bIndirect is non-zero) or not indirect (if bIndirect is zero). +** +** Following a successful call to this function, this function may not be +** called again on the same changegroup object until after +** sqlite3changegroup_change_finish() has been called. Doing so is an +** SQLITE_MISUSE error. +** +** The changegroup object passed as the first argument must be already +** configured with schema data for the specified table. It may be configured +** either by calling sqlite3changegroup_schema() with a database that contains +** the table, or sqlite3changegroup_add() with a changeset that contains the +** table. If the changegroup object has not been configured with a schema for +** the specified table when this function is called, SQLITE_ERROR is returned. +** +** If successful, SQLITE_OK is returned. Otherwise, if an error occurs, an +** SQLite error code is returned. In this case, if argument pzErr is non-NULL, +** then (*pzErr) may be set to point to a buffer containing a utf-8 formated, +** nul-terminated, English language error message. It is the responsibility +** of the caller to eventually free this buffer using sqlite3_free(). +*/ +int sqlite3changegroup_change_begin( + sqlite3_changegroup*, + int eOp, + const char *zTab, + int bIndirect, + char **pzErr +); + +/* +** CAPI3REF: Add a 64-bit integer to a changegroup +** +** This function may only be called between a successful call to +** sqlite3changegroup_change_begin() and its matching +** sqlite3changegroup_change_finish() call. If it is called at any +** other time, it is an SQLITE_MISUSE error. Calling this function +** specifies a 64-bit integer value to be used in the change currently being +** added to the changegroup object passed as the first argument. +** +** The second parameter, bNew, specifies whether the value is to be part of +** the new.* (if bNew is non-zero) or old.* (if bNew is zero) record of +** the change under construction. If this does not match the type of change +** specified by the preceding call to sqlite3changegroup_change_begin() (i.e. +** an old.* value for an SQLITE_INSERT change, or a new.* value for an +** SQLITE_DELETE), then SQLITE_ERROR is returned. +** +** The third parameter specifies the column of the old.* or new.* record that +** the value will be a part of. If the specified table has an explicit primary +** key, then this is the index of the table column, numbered from 0 in the order +** specified within the CREATE TABLE statement. Or, if the table uses an +** implicit rowid key, then the column 0 is the rowid and the explicit columns +** are numbered starting from 1. If the iCol parameter is less than 0 or greater +** than the index of the last column in the table, SQLITE_RANGE is returned. +** +** The fourth parameter is the integer value to use as part of the old.* or +** new.* record. +** +** If this call is successful, SQLITE_OK is returned. Otherwise, if an +** error occurs, an SQLite error code is returned. +*/ +int sqlite3changegroup_change_int64( + sqlite3_changegroup*, + int bNew, + int iCol, + sqlite3_int64 iVal +); + +/* +** CAPI3REF: Add a NULL to a changegroup +** +** This function is similar to sqlite3changegroup_change_int64(). Except that +** it configures the change currently under construction with a NULL value +** instead of a 64-bit integer. +*/ +int sqlite3changegroup_change_null(sqlite3_changegroup*, int, int); + +/* +** CAPI3REF: Add an double to a changegroup +** +** This function is similar to sqlite3changegroup_change_int64(). Except that +** it configures the change currently being constructed with a real value +** instead of a 64-bit integer. +*/ +int sqlite3changegroup_change_double(sqlite3_changegroup*, int, int, double); + +/* +** CAPI3REF: Add a text value to a changegroup +** +** This function is similar to sqlite3changegroup_change_int64(). It configures +** the currently accumulated change with a text value instead of a 64-bit +** integer. Parameter pVal points to a buffer containing the text encoded using +** utf-8. Parameter nVal may either be the size of the text value in bytes, or +** else a negative value, in which case the buffer pVal points to is assumed to +** be nul-terminated. +*/ +int sqlite3changegroup_change_text( + sqlite3_changegroup*, int, int, const char *pVal, int nVal +); + +/* +** CAPI3REF: Add a blob to a changegroup +** +** This function is similar to sqlite3changegroup_change_int64(). It configures +** the currently accumulated change with a blob value instead of a 64-bit +** integer. Parameter pVal points to a buffer containing the blob. Parameter +** nVal is the size of the blob in bytes. +*/ +int sqlite3changegroup_change_blob( + sqlite3_changegroup*, int, int, const void *pVal, int nVal +); + +/* +** CAPI3REF: Finish adding one-at-at-time changes to a changegroup +** +** This function may only be called following a successful call to +** sqlite3changegroup_change_begin(). Otherwise, it is an SQLITE_MISUSE error. +** +** If parameter bDiscard is non-zero, then the current change is simply +** discarded. In this case this function is always successful and SQLITE_OK +** returned. +** +** If parameter bDiscard is zero, then an attempt is made to add the current +** change to the changegroup. Assuming the changegroup is configured to +** produce a changeset (not a patchset), this requires that: +** +** * If the change is an INSERT or DELETE, then a value must be specified +** for all columns of the new.* or old.* record, respectively. +** +** * If the change is an UPDATE record, then values must be provided for +** the PRIMARY KEY columns of the old.* record, but must not be provided +** for PRIMARY KEY columns of the new.* record. +** +** * If the change is an UPDATE record, then for each non-PRIMARY KEY +** column in the old.* record for which a value has been provided, a +** value must also be provided for the same column in the new.* record. +** Similarly, for each non-PK column in the old.* record for which +** a value is not provided, a value must not be provided for the same +** column in the new.* record. +** +** * All values specified for PRIMARY KEY columns must be non-NULL. +** +** Otherwise, it is an error. +** +** If the changegroup already contains a change for the same row (identified +** by PRIMARY KEY columns), then the current change is combined with the +** existing change in the same way as for sqlite3changegroup_add(). +** +** For a patchset, all of the above rules apply except that it doesn't matter +** whether or not values are provided for the non-PK old.* record columns +** for an UPDATE or DELETE change. This means that code used to produce +** a changeset using the sqlite3changegroup_change_xxx() APIs may also +** be used to produce patchsets. +** +** If the call is successful, SQLITE_OK is returned. Otherwise, if an error +** occurs, an SQLite error code is returned. If an error is returned and +** parameter pzErr is not NULL, then (*pzErr) may be set to point to a buffer +** containing a nul-terminated, utf-8 encoded, English language error message. +** It is the responsibility of the caller to eventually free any such error +** message buffer using sqlite3_free(). +*/ +int sqlite3changegroup_change_finish( + sqlite3_changegroup*, + int bDiscard, + char **pzErr +); + /* ** Make sure we can call this stuff from C++. */ diff --git a/ext/session/test_session.c b/ext/session/test_session.c index 6ad5b3774..1b0971422 100644 --- a/ext/session/test_session.c +++ b/ext/session/test_session.c @@ -7,10 +7,14 @@ #include <string.h> #include "tclsqlite.h" +#include <stdlib.h> + #ifndef SQLITE_AMALGAMATION typedef unsigned char u8; #endif +extern const char *sqlite3ErrName(int); + typedef struct TestSession TestSession; struct TestSession { sqlite3_session *pSession; @@ -395,7 +399,6 @@ static int SQLITE_TCLAPI test_session_cmd( } rc = sqlite3session_object_config(pSession, aOpt[iOpt].opt, &iArg); if( rc!=SQLITE_OK ){ - extern const char *sqlite3ErrName(int); Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); }else{ Tcl_SetObjResult(interp, Tcl_NewIntObj(iArg)); @@ -856,6 +859,21 @@ static int testStreamInput( return SQLITE_OK; } +/* +** This works like Tcl_GetByteArrayFromObj(), except that it returns a buffer +** allocated using malloc() that must be freed by the caller. This is done +** because Tcl's buffers are often padded by a few bytes, which prevents +** small overreads from being detected when tests are run under asan. +*/ +static void *testGetByteArrayFromObj(Tcl_Obj *p, Tcl_Size *pnByte){ + Tcl_Size nByte = 0; + void *aByte = Tcl_GetByteArrayFromObj(p, &nByte); + void *aCopy = malloc(nByte ? (size_t)nByte : 1); + memcpy(aCopy, aByte, (size_t)nByte); + *pnByte = nByte; + return aCopy; +} + static int SQLITE_TCLAPI testSqlite3changesetApply( int iVersion, @@ -920,7 +938,7 @@ static int SQLITE_TCLAPI testSqlite3changesetApply( return TCL_ERROR; } db = *(sqlite3 **)info.objClientData; - pChangeset = (void *)Tcl_GetByteArrayFromObj(objv[2], &nChangeset); + pChangeset = (void *)testGetByteArrayFromObj(objv[2], &nChangeset); ctx.pConflictScript = objv[3]; ctx.pFilterScript = objc==5 ? objv[4] : 0; ctx.interp = interp; @@ -972,6 +990,7 @@ static int SQLITE_TCLAPI testSqlite3changesetApply( } } + free(pChangeset); if( rc!=SQLITE_OK ){ return test_session_error(interp, rc, 0); }else{ @@ -1096,6 +1115,21 @@ static int SQLITE_TCLAPI test_sqlite3changeset_invert( return rc; } +/* +** Copy buffer aIn[] to a new nIn byte buffer obtained from malloc(). Use +** plain malloc() instead of any Tcl function because valgrind and asan are +** better at detecting small overflows in that case. Avoid sqlite3_malloc() +** here because that means dealing with injected OOM errors. +** +** The caller is responsible for eventually calling free() on the returned +** value. +*/ +static u8 *copyToMalloc(const u8 *aIn, int nIn){ + u8 *pRet = malloc(nIn); + memcpy(pRet, aIn, nIn); + return pRet; +} + /* ** sqlite3changeset_concat LEFT RIGHT */ @@ -1126,6 +1160,9 @@ static int SQLITE_TCLAPI test_sqlite3changeset_concat( sLeft.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR); sRight.nStream = sLeft.nStream; + sLeft.aData = copyToMalloc(sLeft.aData, sLeft.nData); + sRight.aData = copyToMalloc(sRight.aData, sRight.nData); + if( sLeft.nStream>0 ){ rc = sqlite3changeset_concat_strm( testStreamInput, (void*)&sLeft, @@ -1138,6 +1175,9 @@ static int SQLITE_TCLAPI test_sqlite3changeset_concat( ); } + free(sLeft.aData); + free(sRight.aData); + if( rc!=SQLITE_OK ){ rc = test_session_error(interp, rc, 0); }else{ @@ -1195,7 +1235,12 @@ static int SQLITE_TCLAPI test_sqlite3session_foreach( pCS = objv[2]; pScript = objv[3]; - pChangeset = (void *)Tcl_GetByteArrayFromObj(pCS, &nChangeset); + /* Take a copy of the changeset into an exact sized buffer allocated + ** using malloc(). The Tcl buffer will be padded by a few bytes, which + ** prevents small overreads from being detected by ASAN when the tests + ** are run. */ + pChangeset = (void*)testGetByteArrayFromObj(pCS, &nChangeset); + sStr.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR); if( isInvert ){ int f = SQLITE_CHANGESETSTART_INVERT; @@ -1216,32 +1261,33 @@ static int SQLITE_TCLAPI test_sqlite3session_foreach( rc = sqlite3changeset_start_strm(&pIter, testStreamInput, (void*)&sStr); } } - if( rc!=SQLITE_OK ){ - return test_session_error(interp, rc, 0); - } - while( SQLITE_ROW==sqlite3changeset_next(pIter) ){ - Tcl_Obj *pVar = 0; /* Tcl value to set $VARNAME to */ - pVar = testIterData(pIter); - Tcl_ObjSetVar2(interp, pVarname, 0, pVar, 0); - rc = Tcl_EvalObjEx(interp, pScript, 0); - if( rc!=TCL_OK && rc!=TCL_CONTINUE ){ - sqlite3changeset_finalize(pIter); - return rc==TCL_BREAK ? TCL_OK : rc; + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==sqlite3changeset_next(pIter) ){ + Tcl_Obj *pVar = 0; /* Tcl value to set $VARNAME to */ + pVar = testIterData(pIter); + Tcl_ObjSetVar2(interp, pVarname, 0, pVar, 0); + rc = Tcl_EvalObjEx(interp, pScript, 0); + if( rc!=TCL_OK && rc!=TCL_CONTINUE ){ + sqlite3changeset_finalize(pIter); + free(pChangeset); + return rc==TCL_BREAK ? TCL_OK : rc; + } } - } - if( isCheckNext ){ - int rc2 = sqlite3changeset_next(pIter); - rc = sqlite3changeset_finalize(pIter); - assert( (rc2==SQLITE_DONE && rc==SQLITE_OK) || rc2==rc ); - }else{ - rc = sqlite3changeset_finalize(pIter); + if( isCheckNext ){ + int rc2 = sqlite3changeset_next(pIter); + rc = sqlite3changeset_finalize(pIter); + assert( (rc2==SQLITE_DONE && rc==SQLITE_OK) || rc2==rc ); + }else{ + rc = sqlite3changeset_finalize(pIter); + } } + + free(pChangeset); if( rc!=SQLITE_OK ){ return test_session_error(interp, rc, 0); } - return TCL_OK; } @@ -1530,6 +1576,14 @@ static void test_changegroup_del(void *clientData){ ckfree(pGrp); } +static int testGetNewOrOld(Tcl_Interp *interp, Tcl_Obj *pObj, int *pbNew){ + const char *azVal[] = { "old", "new", 0 }; + int iIdx = 0; + int rc = Tcl_GetIndexFromObj(interp, pObj, azVal, "record", 0, &iIdx); + *pbNew = iIdx; + return rc; +} + /* ** Tclcmd: $changegroup schema DB DBNAME ** Tclcmd: $changegroup add CHANGESET @@ -1547,14 +1601,25 @@ static int SQLITE_TCLAPI test_changegroup_cmd( const char *zSub; int nArg; const char *zMsg; - int iSub; } aSub[] = { - { "schema", 2, "DB DBNAME", }, /* 0 */ - { "add", 1, "CHANGESET", }, /* 1 */ - { "output", 0, "", }, /* 2 */ - { "delete", 0, "", }, /* 3 */ - { "add_change", 1, "ITERATOR", }, /* 4 */ - { 0 } + { "schema", 2, "DB DBNAME" }, /* 0 */ + { "add", 1, "CHANGESET" }, /* 1 */ + { "output", 0, "" }, /* 2 */ + { "delete", 0, "" }, /* 3 */ + { "add_change", 1, "ITERATOR" }, /* 4 */ + + { "change_begin", 3, "TYPE TABLE INDIRECT" }, /* 5 */ + { "change_int64", 3, "[new|old] ICOL VALUE" }, /* 6 */ + { "change_null", 2, "[new|old] ICOL" }, /* 7 */ + { "change_double", 3, "[new|old] ICOL VALUE" }, /* 8 */ + { "change_text", 3, "[new|old] ICOL VALUE" }, /* 9 */ + { "change_blob", 3, "[new|old] ICOL VALUE" }, /* 10 */ + { "change_finish", 1, "BDISCARD" }, /* 11 */ + + { "config", 2, "OPTION INTVAL" }, /* 12 */ + { "change_text-1", 3, "[new|old] ICOL VALUE" }, /* 13 */ + { "change_begin_ne", 3, "TYPE TABLE INDIRECT" }, /* 14 */ + { 0, 0, 0 } }; int rc = TCL_OK; int iSub = 0; @@ -1623,6 +1688,193 @@ static int SQLITE_TCLAPI test_changegroup_cmd( break; }; + case 14: /* change_beginne */ + case 5: { /* change_begin */ + struct ChangeType { + const char *zType; + int eType; + } aType[] = { + { "INSERT", SQLITE_INSERT }, + { "UPDATE", SQLITE_UPDATE }, + { "DELETE", SQLITE_DELETE }, + { 0, 0 } + }; + int eType = 0; + const char *zTab = 0; + int bIndirect; + int iIdx = 0; + char *zErr = 0; + char **pz = ((iSub==5) ? &zErr : 0); + + if( TCL_OK!=Tcl_GetIntFromObj(0, objv[2], &eType) ){ + rc = Tcl_GetIndexFromObjStruct( + interp, objv[2], aType, sizeof(aType[0]), "TYPE", 0, &iIdx + ); + if( rc!=TCL_OK ) return rc; + eType = aType[iIdx].eType; + } + zTab = Tcl_GetString(objv[3]); + if( Tcl_GetBooleanFromObj(interp, objv[4], &bIndirect) ){ + return TCL_ERROR; + } + + rc = sqlite3changegroup_change_begin(p->pGrp, eType, zTab, bIndirect, pz); + assert( zErr==0 || rc!=SQLITE_OK ); + if( rc!=SQLITE_OK ){ + rc = test_session_error(interp, rc, zErr); + } + + break; + } + + case 6: { /* change_int64 */ + int bNew = 0; + int iCol = 0; + sqlite3_int64 iVal = 0; + if( TCL_OK!=testGetNewOrOld(interp, objv[2], &bNew) + || TCL_OK!=Tcl_GetIntFromObj(interp, objv[3], &iCol) + || TCL_OK!=Tcl_GetWideIntFromObj(interp, objv[4], &iVal) + ){ + rc = TCL_ERROR; + }else{ + rc = sqlite3changegroup_change_int64(p->pGrp, bNew, iCol, iVal); + if( rc!=SQLITE_OK ){ + rc = test_session_error(interp, rc, 0); + } + } + break; + } + + case 7: { /* change_null */ + int bNew = 0; + int iCol = 0; + if( TCL_OK!=testGetNewOrOld(interp, objv[2], &bNew) + || TCL_OK!=Tcl_GetIntFromObj(interp, objv[3], &iCol) + ){ + rc = TCL_ERROR; + }else{ + rc = sqlite3changegroup_change_null(p->pGrp, bNew, iCol); + if( rc!=SQLITE_OK ){ + rc = test_session_error(interp, rc, 0); + } + } + break; + } + + case 8: { /* change_double */ + int bNew = 0; + int iCol = 0; + double rVal = 0; + if( TCL_OK!=testGetNewOrOld(interp, objv[2], &bNew) + || TCL_OK!=Tcl_GetIntFromObj(interp, objv[3], &iCol) + || TCL_OK!=Tcl_GetDoubleFromObj(interp, objv[4], &rVal) + ){ + rc = TCL_ERROR; + }else{ + rc = sqlite3changegroup_change_double(p->pGrp, bNew, iCol, rVal); + if( rc!=SQLITE_OK ){ + rc = test_session_error(interp, rc, 0); + } + } + break; + } + + case 9: { /* change_text */ + int bNew = 0; + int iCol = 0; + if( TCL_OK!=testGetNewOrOld(interp, objv[2], &bNew) + || TCL_OK!=Tcl_GetIntFromObj(interp, objv[3], &iCol) + ){ + rc = TCL_ERROR; + }else{ + Tcl_Size nVal = 0; + const char *pVal = Tcl_GetStringFromObj(objv[4], &nVal); + rc = sqlite3changegroup_change_text(p->pGrp, bNew, iCol, pVal, nVal); + if( rc!=SQLITE_OK ){ + rc = test_session_error(interp, rc, 0); + } + } + break; + } + + case 10: { /* change_blob */ + int bNew = 0; + int iCol = 0; + if( TCL_OK!=testGetNewOrOld(interp, objv[2], &bNew) + || TCL_OK!=Tcl_GetIntFromObj(interp, objv[3], &iCol) + ){ + rc = TCL_ERROR; + }else{ + Tcl_Size nVal = 0; + const u8 *pVal = Tcl_GetByteArrayFromObj(objv[4], &nVal); + rc = sqlite3changegroup_change_blob(p->pGrp, bNew, iCol, pVal, nVal); + if( rc!=SQLITE_OK ){ + rc = test_session_error(interp, rc, 0); + } + } + break; + } + + case 11: { /* change_finish */ + int bDiscard = 0; + if( TCL_OK!=Tcl_GetBooleanFromObj(interp, objv[2], &bDiscard) ){ + rc = TCL_ERROR; + }else{ + char *zErr = 0; + rc = sqlite3changegroup_change_finish(p->pGrp, bDiscard, &zErr); + if( rc!=SQLITE_OK ){ + rc = test_session_error(interp, rc, zErr); + } + } + break; + } + + case 12: { /* config */ + struct OptionName { + const char *zOpt; + int op; + } aOp[] = { + { "patchset", SQLITE_CHANGEGROUP_CONFIG_PATCHSET }, + { 0, 0 } + }; + int iIdx = 0; + int iArg = 0; + rc = Tcl_GetIndexFromObjStruct( + interp, objv[2], aOp, sizeof(aOp[0]), "option", 0, &iIdx + ); + if( rc==TCL_OK + && (rc = Tcl_GetIntFromObj(interp, objv[3], &iArg))==TCL_OK + ){ + int op = aOp[iIdx].op; + void *pArg = (void*)&iArg; + + rc = sqlite3changegroup_config(p->pGrp, op, pArg); + if( rc!=SQLITE_OK ){ + rc = test_session_error(interp, rc, 0); + }else{ + Tcl_SetObjResult(interp, Tcl_NewIntObj(iArg)); + } + } + break; + } + + case 13: { /* change_text-1 */ + int bNew = 0; + int iCol = 0; + if( TCL_OK!=testGetNewOrOld(interp, objv[2], &bNew) + || TCL_OK!=Tcl_GetIntFromObj(interp, objv[3], &iCol) + ){ + rc = TCL_ERROR; + }else{ + const char *pVal = Tcl_GetString(objv[4]); + rc = sqlite3changegroup_change_text(p->pGrp, bNew, iCol, pVal, -1); + if( rc!=SQLITE_OK ){ + rc = test_session_error(interp, rc, 0); + } + } + break; + } + default: { /* delete */ assert( iSub==3 ); Tcl_DeleteCommand(interp, Tcl_GetString(objv[0])); diff --git a/ext/wasm/EXPORTED_FUNCTIONS.fiddle.in b/ext/wasm/EXPORTED_FUNCTIONS.fiddle.in deleted file mode 100644 index 103704df1..000000000 --- a/ext/wasm/EXPORTED_FUNCTIONS.fiddle.in +++ /dev/null @@ -1,10 +0,0 @@ -_fiddle_db_arg -_fiddle_db_filename -_fiddle_exec -_fiddle_experiment -_fiddle_interrupt -_fiddle_main -_fiddle_reset_db -_fiddle_db_handle -_fiddle_db_vfs -_fiddle_export_db diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 937e16d6e..05375d42b 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -1,5 +1,6 @@ -####################################################################### -# This GNU makefile creates the canonical sqlite3 WASM builds. +# +# This GNU makefile creates the canonical sqlite3 WASM builds. Plus some +# others. # # This build assumes a Linux platform and is not intended for # general-purpose client-level use, except for creating builds with @@ -10,28 +11,33 @@ # # default, all = build in dev mode # -# o0, o1, o2, o3, os, oz = full clean/rebuild with the -Ox level indicated -# by the target name. Rebuild is necessary for all components to get -# the desired optimization level. +# o0, o1, o2, o3, os, oz = full clean/rebuild with the -Ox level +# indicated by the target name. A clean rebuild is necessary for +# all components to get the desired optimization level. # # dist = create end user deliverables. Add dist.build=oX to build -# with a specific optimization level, where oX is one of the -# above-listed o? or qo? target names. +# with a specific optimization level, where oX is one of the +# above-listed o? target names. # # snapshot = like dist, but uses a zip file name which clearly -# marks it as a prerelease/snapshot build. +# marks it as a prerelease/snapshot build. # # clean = clean up # # Required tools beyond those needed for the canonical builds: # # - Emscripten SDK: https://emscripten.org/docs/getting_started/downloads.html +# # - The bash shell +# # - GNU make, GNU sed, GNU awk, GNU grep (all in the $PATH and without # a "g" prefix like they have on some non-GNU systems) -# - wasm-strip for release builds: https://github.com/WebAssembly/wabt +# +# - wasm-strip for release builds: https://github.com/WebAssembly/wabt. +# It will build without this but the .wasm files will be huge. +# # - InfoZip for 'dist' zip file -######################################################################## +# default: all MAKEFILE = $(lastword $(MAKEFILE_LIST)) CLEAN_FILES = @@ -80,6 +86,7 @@ emo.strip = 💈 emo.test = 🧪 emo.tool = 🔨 emo.wasm-opt = 🧼 +emo.cleanup = 🧼 # 👷🪄🧮🧫🧽🍿⛽🚧🎱🪚🏆🧼 # @@ -199,7 +206,7 @@ b.mkdir@ = if [ ! -d $(dir $@) ]; then \ # $1 = logtag, $2 = src file(s). $3 = dest dir b.cp = $(call b.mkdir@); \ echo '$(logtag.$(1)) $(emo.disk) $(2) ==> $3'; \ - cp -p $(2) $(3) || exit + cp -f -p $(2) $(3) || exit # # $(call b.c-pp.shcmd,LOGTAG,src,dest,-Dx=y...) @@ -213,9 +220,8 @@ b.cp = $(call b.mkdir@); \ # $4 = optional $(bin.c-pp) flags define b.c-pp.shcmd $(call b.mkdir@); \ -$(call b.echo,$(1),$(emo.disk)$(emo.lock) $(bin.c-pp) $(4) $(if $(loud.if),$(2))); \ -rm -f $(3); \ -$(bin.c-pp) -o $(3) $(4) $(2) || exit; \ +$(call b.echo,$(1),$(emo.disk)$(emo.lock)[$(3)] $(4)); \ +rm -f $(3); $(bin.c-pp) -o $(3) $(4) $(2) || exit; \ chmod -w $(3) endef @@ -227,10 +233,38 @@ endef # Args: as for $(b.c-pp.shcmd). define b.c-pp.target $(3): $$(MAKEFILE_LIST) $$(bin.c-pp) $(2) - @$$(call b.c-pp.shcmd,$(1),$(2),$(3),$(4) $(b.c-pp.target.flags)) + @$$(call b.mkdir@) + @$$(call b.c-pp.shcmd,$(1),$(2),$(3),$(4) $$(b.c-pp.target.flags)) CLEAN_FILES += $(3) endef + +# +# The various -D... values used by *.c-pp.js include: +# +# -Dtarget:es6-module: for all ESM module builds +# +# -Dtarget:node: for node.js builds +# +# -Dtarget:es6-module -Dtarget:es6-bundler-friendly: intended for +# "bundler-friendly" ESM module build. These have some restrictions +# on how URL() objects are constructed in some contexts: URLs which +# refer to files which are part of this project must be referenced +# as string literals so that bundlers' static-analysis tools can +# find those files and include them in their bundles. +# +# -Dtarget:es6-module -Dtarget:node: is intended for use by node.js +# for node.js, as opposed to by node.js on behalf of a +# browser. Mixing -sENVIRONMENT=web and -sENVIRONMENT=node leads to +# ambiguity and confusion on node's part, as it's unable to +# reliably determine whether the target is a browser or node. +# +# To repeat: all node.js builds are 100% untested and unsupported. +# +# Most c-pp.D.X are set via $(bin.mkwb) and X is a build name. +# Those make rules reference c-pp.D.64bit, so it should be defined in +# advance. +# c-pp.D.64bit = -Dbits64 # @@ -248,11 +282,21 @@ c-pp.D.64bit = -Dbits64 # # This is intended to be used in makefile targets which generate an # Emscripten module and where $@ is the module's .js/.mjs file. +# +# This is inherently fragile and has been broken by Emscripten updates +# before. +# +ifeq (1,1) +define b.strip-js-emcc-bindings +echo "$(1) $(emo.garbage) Stripping export wrappers."; \ +sed -i -e '/var _sqlite3.*makeInvalidEarly.*;/d' \ +-e '/assert.*sqlite3.*missing.*;/d' \ +-e '/_sqlite.*createExportWrapper.*;/d' $@ +endef +else b.strip-js-emcc-bindings = \ - sed -i -e '/^.*= \(_sqlite3\|_fiddle\)[^=]*=.*createExportWrapper/d' \ - -e '/^var \(_sqlite3\|_fiddle\)[^=]*=.*makeInvalidEarlyAccess/d' $@ || exit; \ - echo '$(1) $(emo.garbage) (Probably) /createExportWrapper()/d and /makeInvalidEarlyAccess()/d' - + echo '$(1) $(emo.bug) (disabled because it breaks emsdk 4.0.16+)' +endif # # Set up sqlite3.c and sqlite3.h... @@ -271,33 +315,36 @@ b.strip-js-emcc-bindings = \ # $(sqlite3.canonical.c) must point to the sqlite3.c in # the sqlite3 canonical source tree, as that source file # is required for certain utility and test code. +# sqlite3.canonical.c = $(dir.top)/sqlite3.c sqlite3.c ?= $(firstword $(wildcard $(dir.top)/sqlite3-see.c) $(sqlite3.canonical.c)) -sqlite3.h = $(dir $(sqlite3.c))/sqlite3.h +sqlite3.h = $(dir $(sqlite3.c))sqlite3.h # # bin.version-info = binary to output various sqlite3 version info for # embedding in the JS files and in building the distribution zip file. # It must NOT be in $(dir.tmp) because we need it to survive the # cleanup process for the dist build to work properly. +# bin.version-info = ./version-info $(bin.version-info): $(dir.tool)/version-info.c $(sqlite3.h) $(dir.top)/Makefile $(CC) -o $@ -I$(dir $(sqlite3.h)) $(dir.tool)/version-info.c t-version-info: $(bin.version-info) DISTCLEAN_FILES += $(bin.version-info) + # # bin.stripcomments is used for stripping C/C++-style comments from JS # files. The JS files contain large chunks of documentation which we # don't need for all builds. That app's -k flag is of particular # importance here, as it allows us to retain the opening comment # block(s), which contain the license header and version info. +# bin.stripccomments = $(dir.tool)/stripccomments $(bin.stripccomments): $(bin.stripccomments).c $(MAKEFILE) $(CC) -o $@ $< t-stripccomments: $(bin.stripccomments) DISTCLEAN_FILES += $(bin.stripccomments) - ifeq (1,$(MAKING_CLEAN)) SQLITE_C_IS_SEE = 0 else @@ -325,7 +372,7 @@ endif # undefine barebones # relatively new gmake feature, not ubiquitous # -# It's important that sqlite3.h be built to completion before any +# It's important that sqlite3.[ch] be built to completion before any # other parts of the build run, thus we use .NOTPARALLEL to disable # parallel build of that file and its dependants. However, that makes # the whole build non-parallelizable because everything has a dep on @@ -342,8 +389,10 @@ $(sqlite3.h): # $(MAKE) -C $(dir.top) sqlite3.c $(sqlite3.c): $(sqlite3.h) +# # Common options for building sqlite3-wasm.c and speedtest1.c. # Explicit ENABLEs... +# SQLITE_OPT.common = \ -DSQLITE_THREADSAFE=0 \ -DSQLITE_TEMP_STORE=2 \ @@ -360,11 +409,15 @@ SQLITE_OPT.common = \ # removing them from this list will serve only to break the speedtest1 # builds. +# # Currently always needed but TODO is paring tester1.c-pp.js down # to be able to run without this: +# SQLITE_OPT.common += -DSQLITE_WASM_ENABLE_C_TESTS +# # Extra flags for full-featured builds... +# SQLITE_OPT.full-featured = \ -DSQLITE_ENABLE_BYTECODE_VTAB \ -DSQLITE_ENABLE_DBPAGE_VTAB \ @@ -421,13 +474,14 @@ else # -DSQLITE_OMIT_WINDOWFUNC endif +# #SQLITE_OPT += -DSQLITE_DEBUG # Enabling SQLITE_DEBUG will break sqlite3_wasm_vfs_create_file() # (and thus sqlite3_js_vfs_create_file()). Those functions are # deprecated and alternatives are in place, but this crash behavior # can be used to find errant uses of sqlite3_js_vfs_create_file() # in client code. -######################################################################## +# # The following flags are hard-coded into sqlite3-wasm.c and cannot be # modified via the build process: # @@ -436,10 +490,9 @@ endif # SQLITE_OMIT_DEPRECATED # SQLITE_OMIT_UTF16 # SQLITE_OMIT_SHARED_CACHE -######################################################################## - +# -######################################################################## +# # Adding custom C code via sqlite3_wasm_extra_init.c: # # If the canonical build process finds the file @@ -460,7 +513,7 @@ endif # make sqlite3_wasm_extra_init.c=my_custom_stuff.c # # See example_extra_init.c for an example implementation. -######################################################################## +# sqlite3_wasm_extra_init.c ?= $(wildcard sqlite3_wasm_extra_init.c) cflags.wasm_extra_init = ifneq (,$(sqlite3_wasm_extra_init.c)) @@ -481,7 +534,7 @@ endif # WASM_CUSTOM_INSTANTIATE = 1 -######################################################################## +# # $(bin.c-pp): a minimal text file preprocessor. Like C's but much # less so. # @@ -512,18 +565,26 @@ WASM_CUSTOM_INSTANTIATE = 1 # # -D... flags which should be included in all invocations should be # appended to $(b.c-pp.target.flags). -bin.c-pp = ./c-pp-lite -$(bin.c-pp): c-pp-lite.c $(sqlite3.c) $(MAKEFILE) - $(CC) -O0 -o $@ c-pp-lite.c $(sqlite3.c) '-DCMPP_DEFAULT_DELIM="//#"' -I$(dir.top) \ +# +bin.c-pp = ./c-pp +$(bin.c-pp): libcmpp.c $(sqlite3.c) $(MAKEFILE) + $(CC) -O0 -o $@ libcmpp.c $(dir.top)/sqlite3.c -I$(dir.top) \ -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_UTF16 \ -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_OMIT_WAL -DSQLITE_THREADSAFE=0 \ - -DSQLITE_TEMP_STORE=3 + -DSQLITE_TEMP_STORE=3 \ + '-DCMPP_DEFAULT_DELIM="//#"' -DCMPP_MAIN -DCMPP_OMIT_D_MODULE \ + -DCMPP_OMIT_D_PIPE DISTCLEAN_FILES += $(bin.c-pp) b.c-pp.target.flags ?= ifeq (1,$(SQLITE_C_IS_SEE)) b.c-pp.target.flags += -Denable-see endif +api.oo1 ?= 1 +ifeq (0,$(api.oo1)) + b.c-pp.target.flags += -Domit-oo1 +endif +# # cflags.common = C compiler flags for all builds cflags.common = -I. -I$(dir $(sqlite3.c)) -std=c99 -fPIC # emcc.WASM_BIGINT = 1 for BigInt (C int64) support, else 0. The API @@ -531,13 +592,14 @@ cflags.common = -I. -I$(dir $(sqlite3.c)) -std=c99 -fPIC # _are not tested_ on any regular basis. emcc.WASM_BIGINT ?= 1 emcc.MEMORY64 ?= 0 -######################################################################## +# # https://emscripten.org/docs/tools_reference/settings_reference.html#memory64 # # 64-bit build requires wasm-strip 1.0.36 (maybe 1.0.35, but not # 1.0.34) or will fail to strip with "tables may not be 64-bit". -######################################################################## +# +# # emcc_opt = optimization-related flags. These are primarily used by # the various oX targets. build times for -O levels higher than 0 are # painful at dev-time. @@ -549,14 +611,17 @@ emcc.MEMORY64 ?= 0 # -O2 (which consistently creates the fastest-running deliverables). # Build time suffers greatly compared to -O0, which is why -O0 is the # default. +# ifeq (,$(filter $(OPTIMIZED_TARGETS),$(MAKECMDGOALS))) emcc_opt ?= -O0 else emcc_opt ?= -Oz endif +# # When passing emcc_opt from the CLI, += and re-assignment have no # effect, so emcc_opt+=-g3 doesn't work. So... +# emcc_opt_full = $(emcc_opt) -g3 # ^^^ ALWAYS use -g3. See below for why. # @@ -583,26 +648,26 @@ emcc_opt_full = $(emcc_opt) -g3 # Much practice has demonstrated that -O2 consistently gives the best # runtime speeds, but not by a large enough factor to rule out use of # -Oz when smaller deliverable size is a priority. -######################################################################## +# -######################################################################## +# # EXPORTED_FUNCTIONS.* = files for use with Emscripten's # -sEXPORTED_FUNCTION flag. -EXPORTED_FUNCTIONS.api.core = $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-core -EXPORTED_FUNCTIONS.api.in = $(EXPORTED_FUNCTIONS.api.core) -ifeq (1,$(SQLITE_C_IS_SEE)) - EXPORTED_FUNCTIONS.api.in += $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-see -endif -ifeq (0,$(wasm-bare-bones)) - EXPORTED_FUNCTIONS.api.in += $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-extras -endif +# +EXPORTED_FUNCTIONS.api.in = $(dir.api)/EXPORTED_FUNCTIONS.c-pp EXPORTED_FUNCTIONS.api = $(dir.tmp)/EXPORTED_FUNCTIONS.api -$(EXPORTED_FUNCTIONS.api): $(EXPORTED_FUNCTIONS.api.in) $(sqlite3.c) $(MAKEFILE) - @$(call b.mkdir@) - cat $(EXPORTED_FUNCTIONS.api.in) > $@ +EXPORTED_FUNCTIONS.c-pp.flags = +ifeq (1,$(wasm-bare-bones)) + EXPORTED_FUNCTIONS.c-pp.flags += -Dbare-bones +endif +$(eval $(call b.c-pp.target,filter,\ + $(EXPORTED_FUNCTIONS.api.in),\ + $(EXPORTED_FUNCTIONS.api),\ + $(EXPORTED_FUNCTIONS.c-pp.flags))) -######################################################################## +# # emcc flags for .c/.o/.wasm/.js. +# emcc.flags = ifeq (1,$(emcc.verbose)) emcc.flags += -v @@ -690,9 +755,9 @@ ifeq (,$(emcc.INITIAL_MEMORY.$(emcc.INITIAL_MEMORY))) $(error emcc.INITIAL_MEMORY must be one of: 4, 8, 16, 32, 64, 96, 128 (megabytes)) endif emcc.jsflags += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.$(emcc.INITIAL_MEMORY)) +# # /INITIAL_MEMORY -######################################################################## -#emcc.jsflags += -sMEMORY64=$(emcc.MEMORY64) +# emcc.jsflags += $(emcc.environment) emcc.jsflags += -sSTACK_SIZE=512KB @@ -701,7 +766,8 @@ emcc.jsflags += -sSTACK_SIZE=512KB # VFS, which requires twice that for its xRead() and xWrite() methods. # 2023-03: those methods have since been adapted to use a malloc()'d # buffer. -######################################################################## + +# # $(sqlite3.js.init-func) is the name Emscripten assigns our exported # module init/load function. This symbol name is hard-coded in # $(extern-post-js.js) as well as in numerous docs. @@ -741,7 +807,7 @@ emcc.jsflags += -sLLD_REPORT_UNDEFINED #emcc.jsflags += --experimental-pic --unresolved-symbols=ingore-all --import-undefined #emcc.jsflags += --unresolved-symbols=ignore-all -######################################################################## +# # -sSINGLE_FILE: # https://github.com/emscripten-core/emscripten/blob/main/src/settings.js # @@ -750,12 +816,7 @@ emcc.jsflags += -sLLD_REPORT_UNDEFINED # cannot wasm-strip the binary before it gets encoded into the JS # file. The result is that the generated JS file is, because of the # -g3 debugging info, _huge_. -######################################################################## - - -sqlite3.wasm = $(dir.dout)/sqlite3.wasm -sqlite3-wasm.c = $(dir.api)/sqlite3-wasm.c -sqlite3-wasm.c.in = $(sqlite3-wasm.c) $(sqlite3_wasm_extra_init.c) +# # # b.call.patch-export-default is used by mkwasmbuilds.c and the @@ -798,51 +859,6 @@ if [ x1 = x$(1) ]; then \ fi endef -# -# The various -D... values used by *.c-pp.js include: -# -# -Dtarget:es6-module: for all ESM module builds -# -# -Dtarget:node: for node.js builds -# -# -Dtarget:es6-module -Dtarget:es6-bundler-friendly: intended for -# "bundler-friendly" ESM module build. These have some restrictions -# on how URL() objects are constructed in some contexts: URLs which -# refer to files which are part of this project must be referenced -# as string literals so that bundlers' static-analysis tools can -# find those files and include them in their bundles. -# -# -Dtarget:es6-module -Dtarget:node: is intended for use by node.js -# for node.js, as opposed to by node.js on behalf of a -# browser. Mixing -sENVIRONMENT=web and -sENVIRONMENT=node leads to -# ambiguity and confusion on node's part, as it's unable to -# reliably determine whether the target is a browser or node. -# -# To repeat: all node.js builds are 100% untested and unsupported. -# -######################################################################## - -# -# Inputs/outputs for the sqlite3-api.js family. -# -# sqlite3-api.jses = the list of JS files which make up -# sqlite3-api.js, in the order they need to be assembled. -sqlite3-api.jses = $(sqlite3-license-version.js) -sqlite3-api.jses += $(dir.api)/sqlite3-api-prologue.js -sqlite3-api.jses += $(dir.common)/whwasmutil.js -sqlite3-api.jses += $(dir.jacc)/jaccwabyt.js -sqlite3-api.jses += $(dir.api)/sqlite3-api-glue.c-pp.js -sqlite3-api.jses += $(sqlite3-api-build-version.js) -sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.c-pp.js -sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.c-pp.js -sqlite3-api.jses += $(dir.api)/sqlite3-vfs-helper.c-pp.js -ifeq (0,$(wasm-bare-bones)) - sqlite3-api.jses += $(dir.api)/sqlite3-vtab-helper.c-pp.js -endif -sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js -sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.c-pp.js -sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js - # # $(sqlite3-license-version.js) contains the license header and # in-comment build version info. @@ -852,17 +868,15 @@ sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js # sqlite3-license-version.js = $(dir.tmp)/sqlite3-license-version.js $(sqlite3-license-version.js): $(bin.version-info) \ - $(dir.api)/sqlite3-license-version-header.js - @echo '$(logtag.@) $(emo.disk)'; { \ - $(call b.mkdir@); \ + $(dir.api)/sqlite3-license-version-header.js $(MAKEFILE) + @$(call b.mkdir@); echo '$(logtag.@) $(emo.disk)'; { \ cat $(dir.api)/sqlite3-license-version-header.js || exit $$?; \ - echo '/*'; \ + echo '/* @preserve'; \ echo '** This code was built from sqlite3 version...'; \ echo '**'; \ awk '/define SQLITE_VERSION/{$$1=""; print "**" $$0}' $(sqlite3.h); \ awk '/define SQLITE_SOURCE_ID/{$$1=""; print "**" $$0}' $(sqlite3.h); \ echo '**'; echo '** Emscripten SDK: $(emcc.version)'; \ - echo '**'; \ echo '*/'; \ } > $@ @@ -881,6 +895,36 @@ $(sqlite3-api-build-version.js): $(bin.version-info) $(MAKEFILE) echo '});'; \ } > $@ +# +# Inputs/outputs for the sqlite3-api.js family. +# +# sqlite3-api.jses = the list of JS files which make up +# sqlite3-api.js, in the order they need to be assembled. +sqlite3-api.jses = $(sqlite3-license-version.js) +sqlite3-api.jses += $(dir.api)/sqlite3-api-prologue.js +sqlite3-api.jses += $(sqlite3-api-build-version.js) +sqlite3-api.jses += $(dir.common)/whwasmutil.js +sqlite3-api.jses += $(dir.jacc)/jaccwabyt.js +sqlite3-api.jses += $(dir.api)/sqlite3-api-glue.c-pp.js +sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.c-pp.js +sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.c-pp.js +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-helper.c-pp.js +ifeq (0,$(wasm-bare-bones)) + sqlite3-api.jses += $(dir.api)/sqlite3-vtab-helper.c-pp.js +endif +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-kvvfs.c-pp.js +sqlite3-api.jses += $(dir.api)/opfs-common-shared.c-pp.js +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.c-pp.js +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-wl.c-pp.js + +# Parallel builds can fail if $(sqlite3-license-version.js) is not +# created early enough, so make all files in $(sqlite-api.jses) except +# for $(sqlite3-license-version.js) depend on +# $(sqlite3-license-version.js). +deps.jses = $(filter-out $(sqlite3-license-version.js),$(sqlite3-api.jses)) +$(deps.jses): $(sqlite3-license-version.js) + # # extern-post-js* and extern-pre-js* are files for use with # Emscripten's --extern-pre-js and --extern-post-js flags. @@ -918,7 +962,7 @@ $(post-js.in.js): $(MKDIR.bld) $(post-jses.js) $(MAKEFILE) done > $@ # -# speedtest1 decls needed before the $(bin.mkws)-generated makefile +# speedtest1 decls needed before the $(bin.mkwb)-generated makefile # is included. # bin.speedtest1 = ../../speedtest1 @@ -957,12 +1001,21 @@ endif # /shell.c ######################################################################## +# +# Fiddle-related decls we need before .wasmbuilds is included +# + +fiddle.c.in = $(dir.top)/shell.c $(sqlite3-wasm.c) + EXPORTED_FUNCTIONS.fiddle = $(dir.tmp)/EXPORTED_FUNCTIONS.fiddle -$(EXPORTED_FUNCTIONS.fiddle): $(fiddle.EXPORTED_FUNCTIONS.in) $(MAKEFILE_LIST) - @$(b.mkdir@) - @sort -u $(fiddle.EXPORTED_FUNCTIONS.in) > $@ +$(EXPORTED_FUNCTIONS.fiddle): $(EXPORTED_FUNCTIONS.api.in) \ + $(MAKEFILE_LIST) $(bin.c-pp) + @$(call b.mkdir@) + @$(call b.c-pp.shcmd,fiddle,$(EXPORTED_FUNCTIONS.api.in),\ + $@,$(EXPORTED_FUNCTIONS.c-pp.flags) -Dfiddle) @echo $(logtag.@) $(emo.disk) + emcc.flags.fiddle = \ $(emcc.cflags) $(emcc_opt_full) \ --minify 0 \ @@ -987,26 +1040,23 @@ emcc.flags.fiddle = \ -USQLITE_WASM_BARE_BONES \ -DSQLITE_SHELL_FIDDLE -clean: clean-fiddle -clean-fiddle: - rm -f $(dir.fiddle)/fiddle-module.js \ - $(dir.fiddle)/*.wasm \ - $(dir.fiddle)/sqlite3-opfs-*.js \ - $(dir.fiddle)/*.gz \ - EXPORTED_FUNCTIONS.fiddle - rm -fr $(dir.fiddle-debug) - emcc.flags.fiddle.debug = $(emcc.flags.fiddle) \ -DSQLITE_DEBUG \ -DSQLITE_ENABLE_SELECTTRACE \ -DSQLITE_ENABLE_WHERETRACE -fiddle.EXPORTED_FUNCTIONS.in = \ - EXPORTED_FUNCTIONS.fiddle.in \ - $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-core \ - $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-extras - -fiddle.c.in = $(dir.top)/shell.c $(sqlite3-wasm.c) +clean: clean-fiddle +clean-fiddle: + rm -f $(dir.fiddle)/fiddle-module.js \ + $(dir.fiddle)/*.wasm \ + $(dir.fiddle)/sqlite3-opfs-*.js \ + $(dir.fiddle)/*.gz \ + $(dir.fiddle)/index.html \ + $(EXPORTED_FUNCTIONS.fiddle) + rm -fr $(dir.fiddle-debug) +distclean: distclean-fiddle +distclean-fiddle: + rm -fr $(dir.fiddle)/jqterm # # WASMFS build - unsupported and untested. We used WASMFS @@ -1028,6 +1078,14 @@ cflags.wasmfs = -DSQLITE_ENABLE_WASMFS # end wasmfs (the rest is in mkwasmbuilds.c) # +# +# +# +sqlite3-wasm.c = $(dir.api)/sqlite3-wasm.c +# List of input files for compiling $(sqlite3-wasm.c). That file +# #include's sqlite3.c directly, so it's implicitly includes here. +sqlite3-wasm.c.in = $(sqlite3-wasm.c) $(sqlite3_wasm_extra_init.c) + # # $(bin.mkwb) is used for generating much of the makefile code for the # various wasm builds. It used to be generated in this makefile via a @@ -1083,10 +1141,10 @@ sqlite3.ext.js = define gen-worker1 # $1 = X.ext part of sqlite3-worker1X.ext # $2 = $(c-pp.D.NAME) -$(call b.c-pp.target,filter,$(dir.api)/sqlite3-worker1.c-pp.js,\ - $(dir.dout)/sqlite3-worker1$(1),$(2)) -sqlite3.ext.js += $(dir.dout)/sqlite3-worker1$(1) -all: $(dir.dout)/sqlite3-worker1$(1) +$(call b.c-pp.target,filter,$$(dir.api)/sqlite3-worker1.c-pp.js,\ + $$(dir.dout)/sqlite3-worker1$(1),$(2)) +sqlite3.ext.js += $$(dir.dout)/sqlite3-worker1$(1) +all: $$(dir.dout)/sqlite3-worker1$(1) endef $(eval $(call gen-worker1,.js,$(c-pp.D.vanilla))) @@ -1100,10 +1158,10 @@ $(eval $(call gen-worker1,-bundler-friendly.mjs,$(c-pp.D.bundler))) define gen-promiser # $1 = X.ext part of sqlite3-worker1-promiserX.ext # $2 = $(c-pp.D.NAME) -$(call b.c-pp.target,filter,$(dir.api)/sqlite3-worker1-promiser.c-pp.js,\ - $(dir.dout)/sqlite3-worker1-promiser$(1),$(2)) -sqlite3.ext.js += $(dir.dout)/sqlite3-worker1-promiser$(1) -all: $(dir.dout)/sqlite3-worker1-promiser$(1) +$(call b.c-pp.target,filter,$$(dir.api)/sqlite3-worker1-promiser.c-pp.js,\ + $$(dir.dout)/sqlite3-worker1-promiser$(1),$(2)) +sqlite3.ext.js += $$(dir.dout)/sqlite3-worker1-promiser$(1) +all: $$(dir.dout)/sqlite3-worker1-promiser$(1) endef $(eval $(call gen-promiser,.js,$(c-pp.D.vanilla))) @@ -1128,13 +1186,13 @@ all: demos ####################################################################### # -# "SOAP" is a static file which is not part of the amalgamation but -# gets copied into the build output folder and into each of the fiddle -# builds. +# "SOAP" is not part of the amalgamation but gets copied into the +# build output folder and into each of the fiddle builds. # sqlite3.ext.js += $(dir.dout)/sqlite3-opfs-async-proxy.js -$(dir.dout)/sqlite3-opfs-async-proxy.js: $(dir.api)/sqlite3-opfs-async-proxy.js - @$(call b.cp,@,$<,$@) +$(eval $(call b.c-pp.target,soap,\ + $(dir.api)/sqlite3-opfs-async-proxy.c-pp.js,\ + $(dir.dout)/sqlite3-opfs-async-proxy.js)) # # Add a dep of $(sqlite3.ext.js) on every individual build's JS file. @@ -1145,12 +1203,19 @@ $(dir.dout)/sqlite3-opfs-async-proxy.js: $(dir.api)/sqlite3-opfs-async-proxy.js # we don't otherwise have a great place to attach them such that # they're always copied when we need them. # +# The var $(out.$(B).js) comes from $(bin.mkwb) and $(B) is the name +# of a build set up by that tool, e.g. b-vanilla or b-esm64. +# $(foreach B,$(b.names),$(eval $(out.$(B).js): $(sqlite3.ext.js))) + # # b-all: builds all available js/wasm builds. # $(foreach B,$(b.names),$(eval b-all: $(out.$(B).js))) +#$(foreach B,$(b.names),$(eval pre: $(pre-js.$(B).js))) +$(foreach B,$(b.names),$(eval post: $(post-js.$(B).js))) + # # speedtest1 is our primary benchmarking tool. # @@ -1160,7 +1225,7 @@ $(foreach B,$(b.names),$(eval b-all: $(out.$(B).js))) # These flags get applied via $(bin.mkwb). emcc.speedtest1.common = $(emcc_opt_full) emcc.speedtest1 = -I. -I$(dir $(sqlite3.canonical.c)) -emcc.speedtest1 += -sENVIRONMENT=web +emcc.speedtest1 += -sENVIRONMENT=web,worker emcc.speedtest1 += -sALLOW_MEMORY_GROWTH emcc.speedtest1 += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.32) emcc.speedtest1.common += -sINVOKE_RUN=0 @@ -1199,28 +1264,37 @@ speedtest1.exit-runtime1 = -sEXIT_RUNTIME=1 # -sEXIT_RUNTIME=1 but we need EXIT_RUNTIME=0 for the worker-based app # which runs speedtest1 multiple times. -$(EXPORTED_FUNCTIONS.speedtest1): $(EXPORTED_FUNCTIONS.api.core) - @$(call b.echo,@,$(emo.disk)); \ - $(call b.mkdir@); \ - { echo _wasm_main; cat $(EXPORTED_FUNCTIONS.api.core); } > $@ || exit +$(EXPORTED_FUNCTIONS.speedtest1): $(EXPORTED_FUNCTIONS.api) + @$(call b.mkdir@); $(call b.echo,@,$(emo.disk)); \ + { echo _wasm_main; cat $(EXPORTED_FUNCTIONS.api); } > $@ || exit speedtest1: b-speedtest1 +st: speedtest1 # # Generate 64-bit variants of speedtest1*.{js,html} # +# $1 = input file +# $2 = output file +# +# TODO: preprocess these like we do the rest. +# define gen-st64 $(2): $(1) @$$(call b.echo,speedtest164,$$(emo.disk)$(emo.lock) Creating from $$<) - @rm -f $$@; \ - sed -e 's/$(3)\.js/$(3)-64bit\.js/' < $$< > $$@; \ + rm -f $$@; \ + sed -e 's/speedtest1\.js/speedtest1-64bit\.js/' \ + -e 's/speedtest1-worker\.js/speedtest1-worker-64bit\.js/' \ + < $$< > $$@; \ chmod -w $$@ -b-speedtest164: $(2) +$(2): b-speedtest164 +speedtest1: $(1) $(2) CLEAN_FILES += $(2) endef +speedtest1: b-speedtest164 -$(eval $(call gen-st64,speedtest1.html,speedtest1-64bit.html,speedtest1)) -$(eval $(call gen-st64,speedtest1-worker.html,speedtest1-worker-64bit.html,speedtest1-worker)) -$(eval $(call gen-st64,speedtest1-worker.js,speedtest1-worker-64bit.js,speedtest1-worker)) +$(eval $(call gen-st64,speedtest1.html,speedtest1-64bit.html)) +$(eval $(call gen-st64,speedtest1-worker.html,speedtest1-worker-64bit.html)) +$(eval $(call gen-st64,speedtest1-worker.js,speedtest1-worker-64bit.js)) # end speedtest1.js ######################################################################## @@ -1244,9 +1318,11 @@ $(eval $(call gen-st64,speedtest1-worker.js,speedtest1-worker-64bit.js,speedtest # # To create those, we filter tester1.c-pp.js/html with $(bin.c-pp)... +# # tester1.js variants: +# define gen-tester1.js -# $1 = build name to have dep on +# $1 = build name to have a dep on # $2 = suffix for tester1SUFFIX JS # $3 = $(bin.c-pp) flags $(call b.c-pp.target,test,tester1.c-pp.js,tester1$(2),$(3)) @@ -1255,20 +1331,18 @@ tester1-$(1): tester1$(2) tester1: tester1$(2) endef -$(eval $(call gen-tester1.js,vanilla,.js, \ - $(c-pp.D.vanilla) \ - -Dsqlite3.js=$(dir.dout)/sqlite3.js)) -$(eval $(call gen-tester1.js,vanilla64,-64bit.js, \ - $(c-pp.D.vanilla64) \ - -Dsqlite3.js=$(dir.dout)/sqlite3-64bit.js)) -$(eval $(call gen-tester1.js,esm,.mjs, \ - $(c-pp.D.esm) \ - -Dsqlite3.js=$(dir.dout)/sqlite3.mjs)) -$(eval $(call gen-tester1.js,esm64,-64bit.mjs, \ - $(c-pp.D.esm64) \ - -Dsqlite3.js=$(dir.dout)/sqlite3-64bit.mjs)) +$(eval $(call gen-tester1.js,vanilla,.js,\ + $(c-pp.D.vanilla) -Dsqlite3.js=$(dir.dout)/sqlite3.js)) +$(eval $(call gen-tester1.js,vanilla64,-64bit.js,\ + $(c-pp.D.vanilla64) -Dsqlite3.js=$(dir.dout)/sqlite3-64bit.js)) +$(eval $(call gen-tester1.js,esm,.mjs,\ + $(c-pp.D.esm) -Dsqlite3.js=$(dir.dout)/sqlite3.mjs)) +$(eval $(call gen-tester1.js,esm64,-64bit.mjs,\ + $(c-pp.D.esm64) -Dsqlite3.js=$(dir.dout)/sqlite3-64bit.mjs)) +# # tester1.html variants: +# define gen-tester1.html # $1 = build name to have a dep on # $2 = filename suffix: empty, -64bit, -esm, esm-64bit @@ -1303,9 +1377,11 @@ $(eval $(call gen-tester1.html,esm64,-esm-64bit,\ -Dtester1.js=tester1-64bit.mjs \ -Dsqlite3.js=$(dir.dout)/sqlite3-64bit.mjs)) -# tester1-worker.html variants: -# There is no ESM variant of this file. Instead, that page accepts a -# ?esm URL flag to switch to ESM mode. +# +# tester1-worker.html variants: There is no ESM variant of this +# file. Instead, that page accepts the ?esm URL flag to switch to ESM +# mode. +# $(eval $(call b.c-pp.target,test,tester1-worker.c-pp.html,\ tester1-worker.html,-Dbitness=32)) $(eval $(call b.c-pp.target,test,tester1-worker.c-pp.html,\ @@ -1315,8 +1391,57 @@ tester1-worker.html: tester1.mjs tester1-worker-64bit.html: tester1-64bit.mjs all: tester1 +# # end tester1 -######################################################################## +# + +# +# jquery.terminal support for fiddle: +# +# If a clone of https://github.com/jcubic/jquery.terminal +# is found in $(JQTERM), defaulting to $(HOME)/src/jquery.terminal +# then add jquery.terminal support to fiddle. +# +# To build that package, from its checkout dir: +# +# npm install +# make +# +c-pp.D.fiddle ?= +JQTERM ?= $(HOME)/src/jquery.terminal +dir.jqtermExt = $(firstword $(wildcard $(JQTERM))) +#$(info dir.jqtermExt=$(dir.jqtermExt)) +ifeq (0,$(MAKING_CLEAN)) +ifeq (,$(wildcard $(dir.jqtermExt)/js/jquery.terminal.min.js)) +$(info $(emo.magic) To add jquery.terminal support to fiddle, set JQTERM=/path/to/its/built/checkout) +else +$(info $(emo.magic) jquery.terminal found in $(dir.jqtermExt) - adding it to fiddle. Make sure it is built!) + +dir.jqterm = $(dir.fiddle)/jqterm +$(dir.fiddle)/jqterm/jquery.terminal.bundle.min.js: + @$(call b.mkdir@) + cat $(dir.jqtermExt)/js/jquery-1*.min.js \ + $(dir.jqtermExt)/js/jquery.terminal.min.js > $@ + +$(dir.fiddle)/jqterm/jquery.terminal.min.css: $(dir.jqtermExt)/css/jquery.terminal.min.css + @$(call b.mkdir@) + @$(call b.cp,fiddle,$<,$(dir $@)) + +$(dir.fiddle)/index.html: $(dir.fiddle)/jqterm/jquery.terminal.bundle.min.js \ + $(dir.fiddle)/jqterm/jquery.terminal.min.css +c-pp.D.fiddle += -Djqterm +endif +endif +# ^^^ JQTERM/MAKING_CLEAN + +# +# Generate fiddle/index.html. Must come after JQTERM is handled. +# +$(dir.fiddle)/index.html: $(dir.fiddle)/index.c-pp.html +$(eval $(call b.c-pp.target,fiddle,\ + $(dir.fiddle)/index.c-pp.html,$(dir.fiddle)/index.html,$(c-pp.D.fiddle))) +$(out.fiddle.wasm): $(dir.fiddle)/index.html + # # Convenience rules to rebuild with various -Ox levels. Much @@ -1327,6 +1452,7 @@ all: tester1 # # Achtung: build times with anything higher than -O0 are somewhat # painful, which is why -O0 is the default. +# .PHONY: o0 o1 o2 o3 os oz emcc-opt-extra = #ifeq (1,$(wasm-bare-bones)) @@ -1377,7 +1503,9 @@ push-testing: ssh wasm-testing 'cd $(wasm-testing.dir) && bash .gzip' || \ echo "SSH failed: it's likely that stale content will be served via old gzip files." +# # build everything needed by push-testing with -Oz +# .PHONY: for-testing for-testing: emcc_opt=-Oz for-testing: loud=1 @@ -1404,9 +1532,9 @@ update-docs: exit 127 else wasm.docs.jswasm = $(wasm.docs.home)/jswasm -update-docs: $(bin.stripccomments) $(out.sqlite3.js) $(out.sqlite3.wasm) +update-docs: $(bin.stripccomments) $(out.vanilla.js) $(out.vanilla.wasm) @echo "Copying files to the /wasm docs. Be sure to use an -Oz build for this!"; - cp -p $(sqlite3.wasm) $(wasm.docs.jswasm)/. + cp -p $(out.vanilla.wasm) $(wasm.docs.jswasm)/. $(bin.stripccomments) -k -k < $(out.vanilla.js) \ | sed -e '/^[ \t]*$$/d' > $(wasm.docs.jswasm)/sqlite3.js cp -p demo-123.js demo-123.html demo-123-worker.html $(wasm.docs.home)/. @@ -1462,13 +1590,46 @@ endif dist-name-prefix = sqlite-wasm$(dist-name-extra) .PHONY: dist dist: - ./mkdist.sh $(dist-name-prefix) + $(bin.bash) ./mkdist.sh $(dist-name-prefix) snapshot: - ./mkdist.sh $(dist-name-prefix) --snapshot + $(bin.bash) ./mkdist.sh $(dist-name-prefix) --snapshot endif # ^^^ making dist/snapshot CLEAN_FILES += $(wildcard sqlite-wasm-*.zip) +######################################################################## +# The npm target is specifically for preparing files for the downstream +# https://github.com/sqlite/sqlite-wasm (npm) distribution. +# +# Per agreement with that project's maintainers, these filenames need +# to remain stable. To avoid breakage in their deployment process, any +# changes (like renaming files) which potentially break their deployment +# needs to be communicated to that project via opening a new ticket or +# direct coordination with its maintainers. +# +# This target does a full clean/rebuild so that we can ensure that the +# optimization level is set consistently across all files. +# +npm.bundle.zip = npm-bundle.zip +CLEAN_FILES += $(npm.bundle.zip) +# Distributables which need to be built for npm: +npm_files = $(addprefix $(dir.dout)/, \ +sqlite3-bundler-friendly.mjs \ +sqlite3-opfs-async-proxy.js \ +sqlite3-worker1-bundler-friendly.mjs \ +sqlite3-worker1-promiser.mjs \ +sqlite3-worker1.mjs \ +sqlite3.mjs \ +sqlite3.wasm \ +sqlite3-node.mjs \ +) +npm: $(sqlite3.canonical.c) + @echo "$(emo.cleanup) Forcing a clean rebuild to ensure consistent optimization flags." + $(MAKE) clean + $(MAKE) -e "emcc_opt=-Oz $(emcc-opt-extra)" $(npm_files) + rm -f $(npm.bundle.zip); zip -r $(npm.bundle.zip) $(npm_files) + unzip -l $(npm.bundle.zip) + ######################################################################## # Explanation of, and some commentary on, various emcc build flags # follows. Full docs for these can be found at: diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core b/ext/wasm/api/EXPORTED_FUNCTIONS.c-pp similarity index 60% rename from ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core rename to ext/wasm/api/EXPORTED_FUNCTIONS.c-pp index 506054510..2b8397a6d 100644 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.c-pp @@ -13,6 +13,7 @@ _sqlite3_bind_parameter_index _sqlite3_bind_parameter_name _sqlite3_bind_pointer _sqlite3_bind_text +_sqlite3_bind_zeroblob _sqlite3_busy_handler _sqlite3_busy_timeout _sqlite3_cancel_auto_extension @@ -75,6 +76,7 @@ _sqlite3_limit _sqlite3_malloc _sqlite3_malloc64 _sqlite3_msize +_sqlite3_next_stmt _sqlite3_open _sqlite3_open_v2 _sqlite3_overload_function @@ -155,3 +157,92 @@ _sqlite3_vtab_in_next _sqlite3_vtab_nochange _sqlite3_vtab_on_conflict _sqlite3_vtab_rhs_value +//#if not bare-bones +_sqlite3_column_database_name +_sqlite3_column_origin_name +_sqlite3_column_table_name +_sqlite3_create_module +_sqlite3_create_module_v2 +_sqlite3_create_window_function +_sqlite3_declare_vtab +_sqlite3_drop_modules +_sqlite3_preupdate_blobwrite +_sqlite3_preupdate_count +_sqlite3_preupdate_depth +_sqlite3_preupdate_hook +_sqlite3_preupdate_new +_sqlite3_preupdate_old +_sqlite3_progress_handler +_sqlite3_set_authorizer +_sqlite3_vtab_collation +_sqlite3_vtab_distinct +_sqlite3_vtab_in +_sqlite3_vtab_in_first +_sqlite3_vtab_in_next +_sqlite3_vtab_nochange +_sqlite3_vtab_on_conflict +_sqlite3_vtab_rhs_value +_sqlite3changegroup_add +_sqlite3changegroup_add_strm +_sqlite3changegroup_delete +_sqlite3changegroup_new +_sqlite3changegroup_output +_sqlite3changegroup_output_strm +_sqlite3changeset_apply +_sqlite3changeset_apply_strm +_sqlite3changeset_apply_v2 +_sqlite3changeset_apply_v2_strm +_sqlite3changeset_apply_v3 +_sqlite3changeset_apply_v3_strm +_sqlite3changeset_concat +_sqlite3changeset_concat_strm +_sqlite3changeset_conflict +_sqlite3changeset_finalize +_sqlite3changeset_fk_conflicts +_sqlite3changeset_invert +_sqlite3changeset_invert_strm +_sqlite3changeset_new +_sqlite3changeset_next +_sqlite3changeset_old +_sqlite3changeset_op +_sqlite3changeset_pk +_sqlite3changeset_start +_sqlite3changeset_start_strm +_sqlite3changeset_start_v2 +_sqlite3changeset_start_v2_strm +_sqlite3session_attach +_sqlite3session_changeset +_sqlite3session_changeset_size +_sqlite3session_changeset_strm +_sqlite3session_config +_sqlite3session_create +_sqlite3session_delete +_sqlite3session_diff +_sqlite3session_enable +_sqlite3session_indirect +_sqlite3session_isempty +_sqlite3session_memory_used +_sqlite3session_object_config +_sqlite3session_patchset +_sqlite3session_patchset_strm +_sqlite3session_table_filter +//#/if not bare-bones +//#if enable-see +_sqlite3_key +_sqlite3_key_v2 +_sqlite3_rekey +_sqlite3_rekey_v2 +_sqlite3_activate_see +//#/if enable-see +//#if fiddle +_fiddle_db_arg +_fiddle_db_filename +_fiddle_exec +_fiddle_experiment +_fiddle_interrupt +_fiddle_main +_fiddle_reset_db +_fiddle_db_handle +_fiddle_db_vfs +_fiddle_export_db +//#/if fiddle diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras deleted file mode 100644 index e8304b5f2..000000000 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras +++ /dev/null @@ -1,68 +0,0 @@ -_sqlite3_column_database_name -_sqlite3_column_origin_name -_sqlite3_column_table_name -_sqlite3_create_module -_sqlite3_create_module_v2 -_sqlite3_create_window_function -_sqlite3_declare_vtab -_sqlite3_drop_modules -_sqlite3_preupdate_blobwrite -_sqlite3_preupdate_count -_sqlite3_preupdate_depth -_sqlite3_preupdate_hook -_sqlite3_preupdate_new -_sqlite3_preupdate_old -_sqlite3_progress_handler -_sqlite3_set_authorizer -_sqlite3_vtab_collation -_sqlite3_vtab_distinct -_sqlite3_vtab_in -_sqlite3_vtab_in_first -_sqlite3_vtab_in_next -_sqlite3_vtab_nochange -_sqlite3_vtab_on_conflict -_sqlite3_vtab_rhs_value -_sqlite3changegroup_add -_sqlite3changegroup_add_strm -_sqlite3changegroup_delete -_sqlite3changegroup_new -_sqlite3changegroup_output -_sqlite3changegroup_output_strm -_sqlite3changeset_apply -_sqlite3changeset_apply_strm -_sqlite3changeset_apply_v2 -_sqlite3changeset_apply_v2_strm -_sqlite3changeset_apply_v3 -_sqlite3changeset_apply_v3_strm -_sqlite3changeset_concat -_sqlite3changeset_concat_strm -_sqlite3changeset_conflict -_sqlite3changeset_finalize -_sqlite3changeset_fk_conflicts -_sqlite3changeset_invert -_sqlite3changeset_invert_strm -_sqlite3changeset_new -_sqlite3changeset_next -_sqlite3changeset_old -_sqlite3changeset_op -_sqlite3changeset_pk -_sqlite3changeset_start -_sqlite3changeset_start_strm -_sqlite3changeset_start_v2 -_sqlite3changeset_start_v2_strm -_sqlite3session_attach -_sqlite3session_changeset -_sqlite3session_changeset_size -_sqlite3session_changeset_strm -_sqlite3session_config -_sqlite3session_create -_sqlite3session_delete -_sqlite3session_diff -_sqlite3session_enable -_sqlite3session_indirect -_sqlite3session_isempty -_sqlite3session_memory_used -_sqlite3session_object_config -_sqlite3session_patchset -_sqlite3session_patchset_strm -_sqlite3session_table_filter diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see deleted file mode 100644 index 83f3a97db..000000000 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see +++ /dev/null @@ -1,5 +0,0 @@ -_sqlite3_key -_sqlite3_key_v2 -_sqlite3_rekey -_sqlite3_rekey_v2 -_sqlite3_activate_see diff --git a/ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api b/ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api deleted file mode 100644 index aab1d8bd3..000000000 --- a/ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api +++ /dev/null @@ -1,3 +0,0 @@ -FS -wasmMemory - diff --git a/ext/wasm/api/README.md b/ext/wasm/api/README.md index 3c9669e6b..279d216bf 100644 --- a/ext/wasm/api/README.md +++ b/ext/wasm/api/README.md @@ -23,7 +23,7 @@ this writing, but is not set in stone forever and may change at any time. This doc targets maintainers of this code and those wanting to dive in to the details, not end user. -First off, a pikchr of the proverbial onion: +First off, a [pikchr][] of the proverbial onion: ```pikchr toggle center scale = 0.85 @@ -60,14 +60,14 @@ maintenance point of view. At the center of the onion is `sqlite3-api.js`, which gets generated by concatenating the following files together in their listed order: -- **`sqlite3-api-prologue.js`**\ +- **`sqlite3-api-prologue.js`** Contains the initial bootstrap setup of the sqlite3 API - objects. This is exposed as a function, rather than objects, so that + objects. This is exposed as a bootstrapping function so that the next step can pass in a config object which abstracts away parts of the WASM environment, to facilitate plugging it in to arbitrary WASM toolchains. The bootstrapping function gets removed from the global scope in a later stage of the bootstrapping process. -- **`../common/whwasmutil.js`**\ +- **`../common/whwasmutil.js`** A semi-third-party collection of JS/WASM utility code intended to replace much of the Emscripten glue. The sqlite3 APIs internally use these APIs instead of their Emscripten counterparts, in order to be @@ -77,79 +77,78 @@ by concatenating the following files together in their listed order: toolchains. It is "semi-third-party" in that it was created in order to support this tree but is standalone and maintained together with... -- **`../jaccwabyt/jaccwabyt.js`**\ +- **`../jaccwabyt/jaccwabyt.js`** Another semi-third-party API which creates bindings between JS and C structs, such that changes to the struct state from either JS or C are visible to the other end of the connection. This is also an independent spinoff project, conceived for the sqlite3 project but maintained separately. -- **`sqlite3-api-glue.js`**\ +- **`sqlite3-api-glue.js`** Invokes functionality exposed by the previous two files to flesh out low-level parts of `sqlite3-api-prologue.js`. Most of these pieces involve populating the `sqlite3.capi.wasm` object and creating `sqlite3.capi.sqlite3_...()` bindings. This file also deletes most global-scope symbols the above files create, effectively moving them into the scope being used for initializing the API. -- **`<build>/sqlite3-api-build-version.js`**\ +- **`<build>/sqlite3-api-build-version.js`** Gets created by the build process and populates the `sqlite3.version` object. This part is not critical, but records the version of the library against which this module was built. -- **`sqlite3-api-oo1.js`**\ +- **`sqlite3-api-oo1.js`** Provides a high-level object-oriented wrapper to the lower-level C API, colloquially known as OO API #1. Its API is similar to other high-level sqlite3 JS wrappers and should feel relatively familiar to anyone familiar with such APIs. It is not a "required component" and can be elided from builds which do not want it. -- **`sqlite3-api-worker1.js`**\ +- **`sqlite3-api-worker1.js`** A Worker-thread-based API which uses OO API #1 to provide an interface to a database which can be driven from the main Window thread via the Worker message-passing interface. Like OO API #1, this is an optional component, offering one of any number of potential implementations for such an API. - - **`sqlite3-worker1.js`**\ + - **`sqlite3-worker1.js`** Is not part of the amalgamated sources and is intended to be loaded by a client Worker thread. It loads the sqlite3 module and runs the Worker #1 API which is implemented in `sqlite3-api-worker1.js`. - - **`sqlite3-worker1-promiser.js`**\ + - **`sqlite3-worker1-promiser.js`** Is likewise not part of the amalgamated sources and provides a Promise-based interface into the Worker #1 API. This is a far user-friendlier way to interface with databases running in a Worker thread. -- **`sqlite3-vfs-helper.js`**\ +- **`sqlite3-vfs-helper.c-pp.js`** Installs the `sqlite3.vfs` namespace, which contain helpers for use by downstream code which creates `sqlite3_vfs` implementations. -- **`sqlite3-vtab-helper.js`**\ +- **`sqlite3-vtab-helper.c-pp.js`** Installs the `sqlite3.vtab` namespace, which contain helpers for use by downstream code which creates `sqlite3_module` implementations. -- **`sqlite3-vfs-opfs.c-pp.js`**\ +- **`sqlite3-vfs-opfs.c-pp.js`** is an sqlite3 VFS implementation which supports the [Origin-Private FileSystem (OPFS)][OPFS] as a storage layer to provide persistent storage for database files in a browser. It requires... - - **`sqlite3-opfs-async-proxy.js`**\ + - **`sqlite3-opfs-async-proxy.js`** is the asynchronous backend part of the [OPFS][] proxy. It speaks directly to the (async) OPFS API and channels those results back to its synchronous counterpart. This file, because it must be started in its own Worker, is not part of the amalgamation. -- **`sqlite3-vfs-opfs-sahpool.c-pp.js`**\ +- **`sqlite3-vfs-opfs-sahpool.c-pp.js`** is another sqlite3 VFS supporting the [OPFS][], but uses a completely different approach than the above-listed one. -- **`sqlite3-api-cleanup.js`**\ - The previous files do not immediately extend the library. Instead - they add callback functions to be called during its - bootstrapping. Some also temporarily create global objects in order - to communicate their state to the files which follow them. This file - cleans up any dangling globals and runs the API bootstrapping - process, which is what finally executes the initialization code - installed by the previous files. As of this writing, this code - ensures that the previous files leave no more than a single global - symbol installed - `sqlite3InitModule()`. When adapting the API for - non-Emscripten toolchains, this "should" be the only file, of those - in this list, where changes are needed. The Emscripten-specific - pieces described below may also require counterparts in any as-yet - hypothetical alternative build. +The previous files do not immediately extend the library. Instead they +install a global function `sqlite3ApiBootstrap()`, which downstream +code must call to configure the library for the current JS/WASM +environment. Each file listed above pushes a callback into the +bootstrapping queue, to be called as part of `sqlite3ApiBootstrap()`. +Some files also temporarily create global objects in order to +communicate their state to the files which follow them. Those +get cleaned up vi `post-js-footer.js`, described below. + +Adapting the build for non-Emscripten toolchains essentially requires packaging +the above files, concatated together, into that toolchain's "JS glue" +and, in the final stage of that glue, call `sqlite3ApiBootstrap()` and +return its result to the end user. **Files with the extension `.c-pp.js`** are intended [to be processed with `c-pp`](#c-pp), noting that such preprocessing may be applied @@ -171,23 +170,27 @@ from this file rather than `sqlite3.c`. The following Emscripten-specific files are injected into the build-generated `sqlite3.js` along with `sqlite3-api.js`. -- **`extern-pre-js.js`**\ +- **`extern-pre-js.js`** Emscripten-specific header for Emscripten's `--extern-pre-js` flag. As of this writing, that file is only used for experimentation purposes and holds no code relevant to the production deliverables. -- **`pre-js.c-pp.js`**\ +- **`pre-js.c-pp.js`** Emscripten-specific header for Emscripten's `--pre-js` flag. This file overrides certain Emscripten behavior before Emscripten does most of its work. -- **`post-js-header.js`**\ +- **`post-js-header.js`** Emscripten-specific header for the `--post-js` input. It opens up, but does not close, a function used for initializing the library. -- (**`sqlite3-api.js`** gets sandwiched between these &uarr; two - &darr; files.) -- **`post-js-footer.js`**\ +- **`sqlite3-api.js`** gets sandwiched between these &uarr; two + &darr; files. +- **`post-js-footer.js`** Emscripten-specific footer for the `--post-js` input. This closes - off the function opened by `post-js-header.js`. -- **`extern-post-js.c-pp.js`**\ + off the function opened by `post-js-header.js`. This file cleans up + any dangling globals and runs `sqlite3ApiBootstrap()`. As of this + writing, this code ensures that the previous files leave no more + than a single global symbol installed - `sqlite3InitModule()`. + +- **`extern-post-js.c-pp.js`** Emscripten-specific header for Emscripten's `--extern-post-js` flag. This file is run in the global scope. It overwrites the Emscripten-installed `sqlite3InitModule()` function with one which @@ -205,9 +208,10 @@ Preprocessing of Source Files Certain files in the build require preprocessing to filter in/out parts which differ between vanilla JS, ES6 Modules, and node.js builds. The preprocessor application itself is in -[`c-pp.c`](/file/ext/wasm/c-pp.c) and the complete technical details -of such preprocessing are maintained in +[`c-pp-lite.c`](/file/ext/wasm/c-pp-lite.c) and the complete technical +details of such preprocessing are maintained in [`GNUMakefile`](/file/ext/wasm/GNUmakefile). [OPFS]: https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system +[pikchr]: https://pikchr.org diff --git a/ext/wasm/api/extern-post-js.c-pp.js b/ext/wasm/api/extern-post-js.c-pp.js index 606e02ae2..b2e760d6a 100644 --- a/ext/wasm/api/extern-post-js.c-pp.js +++ b/ext/wasm/api/extern-post-js.c-pp.js @@ -14,7 +14,7 @@ */ //#if target:es6-module const toExportForESM = -//#endif +//#/if (function(){ //console.warn("this is extern-post-js"); /** @@ -26,7 +26,7 @@ const toExportForESM = */ const originalInit = sqlite3InitModule; if(!originalInit){ - throw new Error("Expecting globalThis.sqlite3InitModule to be defined by the Emscripten build."); + throw new Error("Expecting sqlite3InitModule to be defined by the Emscripten build."); } /** We need to add some state which our custom Module.locateFile() @@ -73,6 +73,8 @@ const toExportForESM = const sIM = globalThis.sqlite3InitModule = function ff(...args){ //console.warn("Using replaced sqlite3InitModule()",globalThis.location); + sIMS.emscriptenLocateFile = args[0]?.locateFile /* see pre-js.c-pp.js [tag:locateFile] */; + sIMS.emscriptenInstantiateWasm = args[0]?.instantiateWasm /* see pre-js.c-pp.js [tag:locateFile] */; return originalInit(...args).then((EmscriptenModule)=>{ sIMS.debugModule("sqlite3InitModule() sIMS =",sIMS); sIMS.debugModule("sqlite3InitModule() EmscriptenModule =",EmscriptenModule); @@ -95,7 +97,7 @@ const toExportForESM = //console.warn("sqlite3InitModule() returning E-module.",EmscriptenModule); return EmscriptenModule; } -//#endif +//#/if return s; }).catch((e)=>{ console.error("Exception loading sqlite3 module:",e); @@ -126,10 +128,10 @@ const toExportForESM = } /* AMD modules get injected in a way we cannot override, so we can't handle those here. */ -//#endif // !target:es6-module +//#/if // !target:es6-module return sIM; })(); //#if target:es6-module sqlite3InitModule = toExportForESM; export default sqlite3InitModule; -//#endif +//#/if diff --git a/ext/wasm/api/opfs-common-inline.c-pp.js b/ext/wasm/api/opfs-common-inline.c-pp.js new file mode 100644 index 000000000..74e911e56 --- /dev/null +++ b/ext/wasm/api/opfs-common-inline.c-pp.js @@ -0,0 +1,191 @@ +//#if 0 +/** + This file is for preprocessor #include into the "opfs" and + "opfs-wl" impls, as well as their async-proxy part. It must be + inlined in those files, as opposed to being a shared copy in the + library, because (A) the async proxy does not load the library and + (B) it references an object which is local to each of those files + but which has a 99% identical structure for each. +*/ +//#/if +//#// vfs.metrics.enable is a refactoring crutch. +//#define vfs.metrics.enable 0 +const initS11n = function(){ + /** + This proxy de/serializes cross-thread function arguments and + output-pointer values via the state.sabIO SharedArrayBuffer, + using the region defined by (state.sabS11nOffset, + state.sabS11nOffset + state.sabS11nSize]. Only one dataset is + recorded at a time. + + This is not a general-purpose format. It only supports the + range of operations, and data sizes, needed by the + sqlite3_vfs and sqlite3_io_methods operations. Serialized + data are transient and this serialization algorithm may + change at any time. + + The data format can be succinctly summarized as: + + Nt...Td...D + + Where: + + - N = number of entries (1 byte) + + - t = type ID of first argument (1 byte) + + - ...T = type IDs of the 2nd and subsequent arguments (1 byte + each). + + - d = raw bytes of first argument (per-type size). + + - ...D = raw bytes of the 2nd and subsequent arguments (per-type + size). + + All types except strings have fixed sizes. Strings are stored + using their TextEncoder/TextDecoder representations. It would + arguably make more sense to store them as Int16Arrays of + their JS character values, but how best/fastest to get that + in and out of string form is an open point. Initial + experimentation with that approach did not gain us any speed. + + Historical note: this impl was initially about 1% this size by + using using JSON.stringify/parse(), but using fit-to-purpose + serialization saves considerable runtime. + */ + if(state.s11n) return state.s11n; + const textDecoder = new TextDecoder(), + textEncoder = new TextEncoder('utf-8'), + viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), + viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + state.s11n = Object.create(null); + /* Only arguments and return values of these types may be + serialized. This covers the whole range of types needed by the + sqlite3_vfs API. */ + const TypeIds = Object.create(null); + TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; + TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; + TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; + TypeIds.string = { id: 4 }; + + const getTypeId = (v)=>( + TypeIds[typeof v] + || toss("Maintenance required: this value type cannot be serialized.",v) + ); + const getTypeIdById = (tid)=>{ + switch(tid){ + case TypeIds.number.id: return TypeIds.number; + case TypeIds.bigint.id: return TypeIds.bigint; + case TypeIds.boolean.id: return TypeIds.boolean; + case TypeIds.string.id: return TypeIds.string; + default: toss("Invalid type ID:",tid); + } + }; + + /** + Returns an array of the deserialized state stored by the most + recent serialize() operation (from this thread or the + counterpart thread), or null if the serialization buffer is + empty. If passed a truthy argument, the serialization buffer + is cleared after deserialization. + */ + state.s11n.deserialize = function(clear=false){ +//#if vfs.metrics.enable + ++metrics.s11n.deserialize.count; +//#/if + const t = performance.now(); + const argc = viewU8[0]; + const rc = argc ? [] : null; + if(argc){ + const typeIds = []; + let offset = 1, i, n, v; + for(i = 0; i < argc; ++i, ++offset){ + typeIds.push(getTypeIdById(viewU8[offset])); + } + for(i = 0; i < argc; ++i){ + const t = typeIds[i]; + if(t.getter){ + v = viewDV[t.getter](offset, state.littleEndian); + offset += t.size; + }else{/*String*/ + n = viewDV.getInt32(offset, state.littleEndian); + offset += 4; + v = textDecoder.decode(viewU8.slice(offset, offset+n)); + offset += n; + } + rc.push(v); + } + } + if(clear) viewU8[0] = 0; + //log("deserialize:",argc, rc); +//#if vfs.metrics.enable + metrics.s11n.deserialize.time += performance.now() - t; +//#/if + return rc; + }; + + /** + Serializes all arguments to the shared buffer for consumption + by the counterpart thread. + + This routine is only intended for serializing OPFS VFS + arguments and (in at least one special case) result values, + and the buffer is sized to be able to comfortably handle + those. + + If passed no arguments then it zeroes out the serialization + state. + */ + state.s11n.serialize = function(...args){ + const t = performance.now(); +//#if vfs.metrics.enable + ++metrics.s11n.serialize.count; +//#/if + if(args.length){ + //log("serialize():",args); + const typeIds = []; + let i = 0, offset = 1; + viewU8[0] = args.length & 0xff /* header = # of args */; + for(; i < args.length; ++i, ++offset){ + /* Write the TypeIds.id value into the next args.length + bytes. */ + typeIds.push(getTypeId(args[i])); + viewU8[offset] = typeIds[i].id; + } + for(i = 0; i < args.length; ++i) { + /* Deserialize the following bytes based on their + corresponding TypeIds.id from the header. */ + const t = typeIds[i]; + if(t.setter){ + viewDV[t.setter](offset, args[i], state.littleEndian); + offset += t.size; + }else{/*String*/ + const s = textEncoder.encode(args[i]); + viewDV.setInt32(offset, s.byteLength, state.littleEndian); + offset += 4; + viewU8.set(s, offset); + offset += s.byteLength; + } + } + //log("serialize() result:",viewU8.slice(0,offset)); + }else{ + viewU8[0] = 0; + } +//#if vfs.metrics.enable + metrics.s11n.serialize.time += performance.now() - t; +//#/if + }; + +//#if defined opfs-async-proxy + state.s11n.storeException = state.asyncS11nExceptions + ? ((priority,e)=>{ + if(priority<=state.asyncS11nExceptions){ + state.s11n.serialize([e.name,': ',e.message].join("")); + } + }) + : ()=>{}; +//#/if + + return state.s11n; +//#undef vfs.metrics.enable +}/*initS11n()*/; diff --git a/ext/wasm/api/opfs-common-shared.c-pp.js b/ext/wasm/api/opfs-common-shared.c-pp.js new file mode 100644 index 000000000..24ae2632f --- /dev/null +++ b/ext/wasm/api/opfs-common-shared.c-pp.js @@ -0,0 +1,1301 @@ +//#if not target:node +/* + 2026-03-04 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file holds code shared by sqlite3-vfs-opfs{,-wl}.c-pp.js. It + creates a private/internal sqlite3.opfs namespace common to the two + and used (only) by them and the test framework. It is not part of + the public API. The library deletes sqlite3.opfs in its final + bootstrapping steps unless it's specifically told to keep them (for + testing purposes only) using an undocumented and unsupported + mechanism. +*/ +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + 'use strict'; + if( sqlite3.config.disable?.vfs?.opfs && + sqlite3.config.disable.vfs['opfs-vfs'] ){ + return; + } + const toss = sqlite3.util.toss, + capi = sqlite3.capi, + util = sqlite3.util, + wasm = sqlite3.wasm; + + /** + Generic utilities for working with OPFS. This will get filled out + by the Promise setup and, on success, installed as sqlite3.opfs. + + This is an internal/private namespace intended for use solely by + the OPFS VFSes and test code for them. The library bootstrapping + process removes this object in non-testing contexts. + */ + const opfsUtil = sqlite3.opfs = Object.create(null); + + /** + Returns true if _this_ thread has access to the OPFS APIs. + */ + opfsUtil.thisThreadHasOPFS = ()=>{ + return globalThis.FileSystemHandle && + globalThis.FileSystemDirectoryHandle && + globalThis.FileSystemFileHandle && + globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle && + navigator?.storage?.getDirectory; + }; + + /** + Must be called by the OPFS VFSes immediately after they determine + whether OPFS is available by calling + thisThreadHasOPFS(). Resolves to the OPFS storage root directory + and sets opfsUtil.rootDirectory to that value. + */ + opfsUtil.getRootDir = async function f(){ + return f.promise ??= navigator.storage.getDirectory().then(d=>{ + opfsUtil.rootDirectory = d; + return d; + }).catch(e=>{ + delete f.promise; + throw e; + }); + }; + + /** + Expects an OPFS file path. It gets resolved, such that ".." + components are properly expanded, and returned. If the 2nd arg + is true, the result is returned as an array of path elements, + else an absolute path string is returned. + */ + opfsUtil.getResolvedPath = function(filename,splitIt){ + const p = new URL(filename, "file://irrelevant").pathname; + return splitIt ? p.split('/').filter((v)=>!!v) : p; + }; + + /** + Takes the absolute path to a filesystem element. Returns an + array of [handleOfContainingDir, filename]. If the 2nd argument + is truthy then each directory element leading to the file is + created along the way. Throws if any creation or resolution + fails. + */ + opfsUtil.getDirForFilename = async function f(absFilename, createDirs = false){ + const path = opfsUtil.getResolvedPath(absFilename, true); + const filename = path.pop(); + let dh = await opfsUtil.getRootDir(); + for(const dirName of path){ + if(dirName){ + dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); + } + } + return [dh, filename]; + }; + + /** + Creates the given directory name, recursively, in + the OPFS filesystem. Returns true if it succeeds or the + directory already exists, else false. + */ + opfsUtil.mkdir = async function(absDirName){ + try { + await opfsUtil.getDirForFilename(absDirName+"/filepart", true); + return true; + }catch(e){ + //sqlite3.config.warn("mkdir(",absDirName,") failed:",e); + return false; + } + }; + + /** + Checks whether the given OPFS filesystem entry exists, + returning true if it does, false if it doesn't or if an + exception is intercepted while trying to make the + determination. + */ + opfsUtil.entryExists = async function(fsEntryName){ + try { + const [dh, fn] = await opfsUtil.getDirForFilename(fsEntryName); + await dh.getFileHandle(fn); + return true; + }catch(e){ + return false; + } + }; + + /** + Generates a random ASCII string len characters long, intended for + use as a temporary file name. + */ + opfsUtil.randomFilename = function f(len=16){ + if(!f._chars){ + f._chars = "abcdefghijklmnopqrstuvwxyz"+ + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ + "012346789"; + f._n = f._chars.length; + } + const a = []; + let i = 0; + for( ; i < len; ++i){ + const ndx = Math.random() * (f._n * 64) % f._n | 0; + a[i] = f._chars[ndx]; + } + return a.join(""); + /* + An alternative impl. with an unpredictable length + but much simpler: + + Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36) + */ + }; + + /** + Returns a promise which resolves to an object which represents + all files and directories in the OPFS tree. The top-most object + has two properties: `dirs` is an array of directory entries + (described below) and `files` is a list of file names for all + files in that directory. + + Traversal starts at sqlite3.opfs.rootDirectory. + + Each `dirs` entry is an object in this form: + + ``` + { name: directoryName, + dirs: [...subdirs], + files: [...file names] + } + ``` + + The `files` and `subdirs` entries are always set but may be + empty arrays. + + The returned object has the same structure but its `name` is + an empty string. All returned objects are created with + Object.create(null), so have no prototype. + + Design note: the entries do not contain more information, + e.g. file sizes, because getting such info is not only + expensive but is subject to locking-related errors. + */ + opfsUtil.treeList = async function(){ + const doDir = async function callee(dirHandle,tgt){ + tgt.name = dirHandle.name; + tgt.dirs = []; + tgt.files = []; + for await (const handle of dirHandle.values()){ + if('directory' === handle.kind){ + const subDir = Object.create(null); + tgt.dirs.push(subDir); + await callee(handle, subDir); + }else{ + tgt.files.push(handle.name); + } + } + }; + const root = Object.create(null); + const dir = await opfsUtil.getRootDir(); + await doDir(dir, root); + return root; + }; + + /** + Irrevocably deletes _all_ files in the current origin's OPFS. + Obviously, this must be used with great caution. It may throw + an exception if removal of anything fails (e.g. a file is + locked), but the precise conditions under which the underlying + APIs will throw are not documented (so we cannot tell you what + they are). + */ + opfsUtil.rmfr = async function(){ + const rd = await opfsUtil.getRootDir(); + const dir = rd, opt = {recurse: true}; + for await (const handle of dir.values()){ + dir.removeEntry(handle.name, opt); + } + }; + + /** + Deletes the given OPFS filesystem entry. As this environment + has no notion of "current directory", the given name must be an + absolute path. If the 2nd argument is truthy, deletion is + recursive (use with caution!). + + The returned Promise resolves to true if the deletion was + successful, else false (but...). The OPFS API reports the + reason for the failure only in human-readable form, not + exceptions which can be type-checked to determine the + failure. Because of that... + + If the final argument is truthy then this function will + propagate any exception on error, rather than returning false. + */ + opfsUtil.unlink = async function(fsEntryName, recursive = false, + throwOnError = false){ + try { + const [hDir, filenamePart] = + await opfsUtil.getDirForFilename(fsEntryName, false); + await hDir.removeEntry(filenamePart, {recursive}); + return true; + }catch(e){ + if(throwOnError){ + throw new Error("unlink(",arguments[0],") failed: "+e.message,{ + cause: e + }); + } + return false; + } + }; + + /** + Traverses the OPFS filesystem, calling a callback for each + entry. The argument may be either a callback function or an + options object with any of the following properties: + + - `callback`: function which gets called for each filesystem + entry. It gets passed 3 arguments: 1) the + FileSystemFileHandle or FileSystemDirectoryHandle of each + entry (noting that both are instanceof FileSystemHandle). 2) + the FileSystemDirectoryHandle of the parent directory. 3) the + current depth level, with 0 being at the top of the tree + relative to the starting directory. If the callback returns a + literal false, as opposed to any other falsy value, traversal + stops without an error. Any exceptions it throws are + propagated. Results are undefined if the callback manipulate + the filesystem (e.g. removing or adding entries) because the + how OPFS iterators behave in the face of such changes is + undocumented. + + - `recursive` [bool=true]: specifies whether to recurse into + subdirectories or not. Whether recursion is depth-first or + breadth-first is unspecified! + + - `directory` [FileSystemDirectoryEntry=sqlite3.opfs.rootDirectory] + specifies the starting directory. + + If this function is passed a function, it is assumed to be the + callback. + + Returns a promise because it has to (by virtue of being async) + but that promise has no specific meaning: the traversal it + performs is synchronous. The promise must be used to catch any + exceptions propagated by the callback, however. + */ + opfsUtil.traverse = async function(opt){ + const defaultOpt = { + recursive: true, + directory: await opfsUtil.getRootDir() + }; + if('function'===typeof opt){ + opt = {callback:opt}; + } + opt = Object.assign(defaultOpt, opt||{}); + const doDir = async function callee(dirHandle, depth){ + for await (const handle of dirHandle.values()){ + if(false === opt.callback(handle, dirHandle, depth)) return false; + else if(opt.recursive && 'directory' === handle.kind){ + if(false === await callee(handle, depth + 1)) break; + } + } + }; + doDir(opt.directory, 0); + }; + + /** + Impl of opfsUtil.importDb() when it's given a function as its + second argument. + */ + const importDbChunked = async function(filename, callback){ + const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); + const hFile = await hDir.getFileHandle(fnamePart, {create:true}); + let sah = await hFile.createSyncAccessHandle(); + let nWrote = 0, chunk, checkedHeader = false, err = false; + try{ + sah.truncate(0); + while( undefined !== (chunk = await callback()) ){ + if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk); + if( !checkedHeader && 0===nWrote && chunk.byteLength>=15 ){ + util.affirmDbHeader(chunk); + checkedHeader = true; + } + sah.write(chunk, {at: nWrote}); + nWrote += chunk.byteLength; + } + if( nWrote < 512 || 0!==nWrote % 512 ){ + toss("Input size",nWrote,"is not correct for an SQLite database."); + } + if( !checkedHeader ){ + const header = new Uint8Array(20); + sah.read( header, {at: 0} ); + util.affirmDbHeader( header ); + } + sah.write(new Uint8Array([1,1]), {at: 18}/*force db out of WAL mode*/); + return nWrote; + }catch(e){ + await sah.close(); + sah = undefined; + await hDir.removeEntry( fnamePart ).catch(()=>{}); + throw e; + }finally { + if( sah ) await sah.close(); + } + }; + + /** + Asynchronously imports the given bytes (a byte array or + ArrayBuffer) into the given database file. + + Results are undefined if the given db name refers to an opened + db. + + If passed a function for its second argument, its behaviour + changes: imports its data in chunks fed to it by the given + callback function. It calls the callback (which may be async) + repeatedly, expecting either a Uint8Array or ArrayBuffer (to + denote new input) or undefined (to denote EOF). For so long as + the callback continues to return non-undefined, it will append + incoming data to the given VFS-hosted database file. When + called this way, the resolved value of the returned Promise is + the number of bytes written to the target file. + + It very specifically requires the input to be an SQLite3 + database and throws if that's not the case. It does so in + order to prevent this function from taking on a larger scope + than it is specifically intended to. i.e. we do not want it to + become a convenience for importing arbitrary files into OPFS. + + This routine rewrites the database header bytes in the output + file (not the input array) to force disabling of WAL mode. + + On error this throws and the state of the input file is + undefined (it depends on where the exception was triggered). + + On success, resolves to the number of bytes written. + */ + opfsUtil.importDb = async function(filename, bytes){ + if( bytes instanceof Function ){ + return importDbChunked(filename, bytes); + } + if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes); + util.affirmIsDb(bytes); + const n = bytes.byteLength; + const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); + let sah, err, nWrote = 0; + try { + const hFile = await hDir.getFileHandle(fnamePart, {create:true}); + sah = await hFile.createSyncAccessHandle(); + sah.truncate(0); + nWrote = sah.write(bytes, {at: 0}); + if(nWrote != n){ + toss("Expected to write "+n+" bytes but wrote "+nWrote+"."); + } + sah.write(new Uint8Array([1,1]), {at: 18}) /* force db out of WAL mode */; + return nWrote; + }catch(e){ + if( sah ){ await sah.close(); sah = undefined; } + await hDir.removeEntry( fnamePart ).catch(()=>{}); + throw e; + }finally{ + if( sah ) await sah.close(); + } + }; + + /** + Checks for features required for OPFS VFSes and throws with a + descriptive error message if they're not found. This is intended + to be run as part of async VFS installation steps. + */ + opfsUtil.vfsInstallationFeatureCheck = function(vfsName){ + if( !globalThis.SharedArrayBuffer || !globalThis.Atomics ){ + toss("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics.", + "The server must emit the COOP/COEP response headers to enable those.", + "See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep"); + }else if( 'undefined'===typeof WorkerGlobalScope ){ + toss("The OPFS sqlite3_vfs cannot run in the main thread", + "because it requires Atomics.wait()."); + }else if( !globalThis.FileSystemHandle || + !globalThis.FileSystemDirectoryHandle || + !globalThis.FileSystemFileHandle?.prototype?.createSyncAccessHandle || + !navigator?.storage?.getDirectory ){ + toss("Missing required OPFS APIs."); + }else if( 'opfs-wl'===vfsName && !globalThis.Atomics.waitAsync ){ + toss('The',vfsName,'VFS requires Atomics.waitAsync(), which is not available.'); + } + }; + + /** + Must be called by the VFS's main installation routine and passed + the options object that function receives and a reference to that + function itself (we don't need this anymore). + + It throws if OPFS is not available. + + If it returns falsy, it detected that OPFS should be disabled, in + which case the callee should immediately return/resolve to the + sqlite3 object. + + Else it returns a new copy of the options object, fleshed out + with any missing defaults. The caller must: + + - Set up any local state they need. + + - Call opfsUtil.createVfsState(vfsName,opt), where opt is the + object returned by this function. + + - Set up any references they may need to state returned + by the previous step. + + - Call opfvs.bindVfs() + */ + opfsUtil.initOptions = function callee(vfsName, options){ + const urlParams = new URL(globalThis.location.href).searchParams; + if( urlParams.has(vfsName+'-disable') ){ + //sqlite3.config.warn('Explicitly not installing "opfs" VFS due to opfs-disable flag.'); + return; + } + try{ + opfsUtil.vfsInstallationFeatureCheck(vfsName); + }catch(e){ + return; + } + options = util.nu(options); + options.vfsName = vfsName; + options.verbose ??= urlParams.has('opfs-verbose') + ? +urlParams.get('opfs-verbose') : 1; + options.sanityChecks ??= urlParams.has('opfs-sanity-check'); + + if( !opfsUtil.proxyUri ){ + opfsUtil.proxyUri = "sqlite3-opfs-async-proxy.js"; + if( sqlite3.scriptInfo?.sqlite3Dir ){ + /* Doing this from one scope up, outside of this function, does + not work. */ + opfsUtil.proxyUri = ( + sqlite3.scriptInfo.sqlite3Dir + opfsUtil.proxyUri + ); + } + } + options.proxyUri ??= opfsUtil.proxyUri; + if('function' === typeof options.proxyUri){ + options.proxyUri = options.proxyUri(); + } + //sqlite3.config.warn("opfsUtil options =",JSON.stringify(options), 'urlParams =', urlParams); + return opfsUtil.options = options; + }; + + /** + Creates, populates, and returns the main state object used by the + "opfs" and "opfs-wl" VFSes, and transfered from those to their + async counterparts. + + The returned object's vfs property holds the fully-populated + capi.sqlite3_vfs instance, tagged with lots of extra state which + the current VFSes need to have exposed to them. + + After setting up any local state needed, the caller must call + theVfs.bindVfs(X,Y), where X is an object containing the + sqlite3_io_methods to override and Y is a callback which gets + triggered if init succeeds, before the final Promise decides + whether or not to reject. + + This object must, when it's passed to the async part, contain + only cloneable or sharable objects. After the worker's "inited" + message arrives, other types of data may be added to it. + */ + opfsUtil.createVfsState = function(){ + const state = util.nu(); + const options = opfsUtil.options; + state.verbose = options.verbose; + + const loggers = [ + sqlite3.config.error, + sqlite3.config.warn, + sqlite3.config.log + ]; + const vfsName = options.vfsName + || toss("Maintenance required: missing VFS name"); + const logImpl = (level,...args)=>{ + if(state.verbose>level) loggers[level](vfsName+":",...args); + }; + const log = (...args)=>logImpl(2, ...args), + warn = (...args)=>logImpl(1, ...args), + error = (...args)=>logImpl(0, ...args), + capi = sqlite3.capi, + wasm = sqlite3.wasm; + + const opfsVfs = state.vfs = new capi.sqlite3_vfs(); + const opfsIoMethods = opfsVfs.ioMethods = new capi.sqlite3_io_methods(); + + opfsIoMethods.$iVersion = 1; + opfsVfs.$iVersion = 2/*yes, two*/; + opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; + opfsVfs.$mxPathname = 1024/* sure, why not? The OPFS name length limit + is undocumented/unspecified. */; + opfsVfs.$zName = wasm.allocCString(vfsName); + opfsVfs.addOnDispose( + '$zName', opfsVfs.$zName, opfsIoMethods + /** + Pedantic sidebar: the entries in this array are items to + clean up when opfsVfs.dispose() is called, but in this + environment it will never be called. The VFS instance simply + hangs around until the WASM module instance is cleaned up. We + "could" _hypothetically_ clean it up by "importing" an + sqlite3_os_end() impl into the wasm build, but the shutdown + order of the wasm engine and the JS one are undefined so + there is no guaranty that the opfsVfs instance would be + available in one environment or the other when + sqlite3_os_end() is called (_if_ it gets called at all in a + wasm build, which is undefined). i.e. addOnDispose() here is + a matter of "correctness", not necessity. It just wouldn't do + to leave the impression that we're blindly leaking memory. + */ + ); + + opfsVfs.metrics = util.nu({ + counters: util.nu(), + dump: function(){ + let k, n = 0, t = 0, w = 0; + for(k in state.opIds){ + const m = metrics[k]; + n += m.count; + t += m.time; + w += m.wait; + m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; + m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0; + } + sqlite3.config.log(globalThis.location.href, + "metrics for",globalThis.location.href,":",metrics, + "\nTotal of",n,"op(s) for",t, + "ms (incl. "+w+" ms of waiting on the async side)"); + sqlite3.config.log("Serialization metrics:",opfsVfs.metrics.counters.s11n); + opfsVfs.worker?.postMessage?.({type:'opfs-async-metrics'}); + }, + reset: function(){ + let k; + const r = (m)=>(m.count = m.time = m.wait = 0); + const m = opfsVfs.metrics.counters; + for(k in state.opIds){ + r(m[k] = Object.create(null)); + } + let s = m.s11n = Object.create(null); + s = s.serialize = Object.create(null); + s.count = s.time = 0; + s = m.s11n.deserialize = Object.create(null); + s.count = s.time = 0; + } + })/*opfsVfs.metrics*/; + + /** + asyncIdleWaitTime is how long (ms) to wait, in the async proxy, + for each Atomics.wait() when waiting on inbound VFS API calls. + We need to wake up periodically to give the thread a chance to + do other things. If this is too high (e.g. 500ms) then even two + workers/tabs can easily run into locking errors. Some multiple + of this value is also used for determining how long to wait on + lock contention to free up. + */ + state.asyncIdleWaitTime = 150; + + /** + Whether the async counterpart should log exceptions to + the serialization channel. That produces a great deal of + noise for seemingly innocuous things like xAccess() checks + for missing files, so this option may have one of 3 values: + + 0 = no exception logging. + + 1 = only log exceptions for "significant" ops like xOpen(), + xRead(), and xWrite(). Exceptions related to, e.g., wait/retry + loops in acquiring SyncAccessHandles are not logged. + + 2 = log all exceptions. + */ + state.asyncS11nExceptions = 1; + /* Size of file I/O buffer block. 64k = max sqlite3 page size, and + xRead/xWrite() will never deal in blocks larger than that. */ + state.fileBufferSize = 1024 * 64; + state.sabS11nOffset = state.fileBufferSize; + /** + The size of the block in our SAB for serializing arguments and + result values. Needs to be large enough to hold serialized + values of any of the proxied APIs. Filenames are the largest + part but are limited to opfsVfs.$mxPathname bytes. We also + store exceptions there, so it needs to be long enough to hold + a reasonably long exception string. + */ + state.sabS11nSize = opfsVfs.$mxPathname * 2; + /** + The SAB used for all data I/O between the synchronous and + async halves (file i/o and arg/result s11n). + */ + state.sabIO = new SharedArrayBuffer( + state.fileBufferSize/* file i/o block */ + + state.sabS11nSize/* argument/result serialization block */ + ); + + /** + For purposes of Atomics.wait() and Atomics.notify(), we use a + SharedArrayBuffer with one slot reserved for each of the API + proxy's methods. The sync side of the API uses Atomics.wait() + on the corresponding slot and the async side uses + Atomics.notify() on that slot. state.opIds holds the SAB slot + IDs of each of those. + */ + state.opIds = Object.create(null); + { + /* Indexes for use in our SharedArrayBuffer... */ + let i = 0; + /* SAB slot used to communicate which operation is desired + between both workers. This worker writes to it and the other + listens for changes and clears it. The values written to it + are state.opIds.x[A-Z][a-z]+, defined below.*/ + state.opIds.whichOp = i++; + /* Slot for storing return values. This side listens to that + slot and the async proxy writes to it. */ + state.opIds.rc = i++; + /* Each function gets an ID which this worker writes to the + state.opIds.whichOp slot. The async-api worker uses + Atomic.wait() on the whichOp slot to figure out which + operation to run next. */ + state.opIds.xAccess = i++; + state.opIds.xClose = i++; + state.opIds.xDelete = i++; + state.opIds.xDeleteNoWait = i++; + state.opIds.xFileSize = i++; + state.opIds.xLock = i++; + state.opIds.xOpen = i++; + state.opIds.xRead = i++; + state.opIds.xSleep = i++; + state.opIds.xSync = i++; + state.opIds.xTruncate = i++; + state.opIds.xUnlock = i++; + state.opIds.xWrite = i++; + state.opIds.mkdir = i++ /*currently unused*/; + /** Internal signals which are used only during development and + testing via the dev console. */ + state.opIds['opfs-async-metrics'] = i++; + state.opIds['opfs-async-shutdown'] = i++; + /* The retry slot is used by the async part for wait-and-retry + semantics. It is never written to, only used as a convenient + place to wait-with-timeout for a value which will never be + written, i.e. sleep()ing, before retrying a failed attempt to + acquire a SharedAccessHandle. */ + state.opIds.retry = i++; + state.sabOP = new SharedArrayBuffer( + i * 4/* 4==sizeof int32, noting that Atomics.wait() and + friends can only function on Int32Array views of an + SAB. */); + } + /** + SQLITE_xxx constants to export to the async worker + counterpart... + */ + state.sq3Codes = Object.create(null); + for(const k of [ + 'SQLITE_ACCESS_EXISTS', + 'SQLITE_ACCESS_READWRITE', + 'SQLITE_BUSY', + 'SQLITE_CANTOPEN', + 'SQLITE_ERROR', + 'SQLITE_IOERR', + 'SQLITE_IOERR_ACCESS', + 'SQLITE_IOERR_CLOSE', + 'SQLITE_IOERR_DELETE', + 'SQLITE_IOERR_FSYNC', + 'SQLITE_IOERR_LOCK', + 'SQLITE_IOERR_READ', + 'SQLITE_IOERR_SHORT_READ', + 'SQLITE_IOERR_TRUNCATE', + 'SQLITE_IOERR_UNLOCK', + 'SQLITE_IOERR_WRITE', + 'SQLITE_LOCK_EXCLUSIVE', + 'SQLITE_LOCK_NONE', + 'SQLITE_LOCK_PENDING', + 'SQLITE_LOCK_RESERVED', + 'SQLITE_LOCK_SHARED', + 'SQLITE_LOCKED', + 'SQLITE_MISUSE', + 'SQLITE_NOTFOUND', + 'SQLITE_OPEN_CREATE', + 'SQLITE_OPEN_DELETEONCLOSE', + 'SQLITE_OPEN_MAIN_DB', + 'SQLITE_OPEN_READONLY', + 'SQLITE_LOCK_NONE', + 'SQLITE_LOCK_SHARED', + 'SQLITE_LOCK_RESERVED', + 'SQLITE_LOCK_PENDING', + 'SQLITE_LOCK_EXCLUSIVE' + ]){ + state.sq3Codes[k] = + capi[k] ?? toss("Maintenance required: not found:",k); + } + + state.opfsFlags = Object.assign(Object.create(null),{ + /** + Flag for use with xOpen(). URI flag "opfs-unlock-asap=1" + enables this. See defaultUnlockAsap, below. + */ + OPFS_UNLOCK_ASAP: 0x01, + /** + Flag for use with xOpen(). URI flag "delete-before-open=1" + tells the VFS to delete the db file before attempting to open + it. This can be used, e.g., to replace a db which has been + corrupted (without forcing us to expose a delete/unlink() + function in the public API). + + Failure to unlink the file is ignored but may lead to + downstream errors. An unlink can fail if, e.g., another tab + has the handle open. + + It goes without saying that deleting a file out from under + another instance results in Undefined Behavior. + */ + OPFS_UNLINK_BEFORE_OPEN: 0x02, + /** + If true, any async routine which must implicitly acquire a + sync access handle (i.e. an OPFS lock), without an active + xLock(), will release that lock at the end of the call which + acquires it. If false, such implicit locks are not released + until the VFS is idle for some brief amount of time, as + defined by state.asyncIdleWaitTime. + + The benefit of enabling this is higher concurrency. The + down-side is much-reduced performance (as much as a 4x + decrease in speedtest1). + */ + defaultUnlockAsap: false + }); + + opfsVfs.metrics.reset()/*must not be called until state.opIds is set up*/; + const metrics = opfsVfs.metrics.counters; + + /** + Runs the given operation (by name) in the async worker + counterpart, waits for its response, and returns the result + which the async worker writes to SAB[state.opIds.rc]. The 2nd + and subsequent arguments must be the arguments for the async op + (see sqlite3-opfs-async-proxy.c-pp.js). + */ + const opRun = opfsVfs.opRun = (op,...args)=>{ + const opNdx = state.opIds[op] || toss(opfsVfs.vfsName+": Invalid op ID:",op); + state.s11n.serialize(...args); + Atomics.store(state.sabOPView, state.opIds.rc, -1); + Atomics.store(state.sabOPView, state.opIds.whichOp, opNdx); + Atomics.notify(state.sabOPView, state.opIds.whichOp) + /* async thread will take over here */; + const t = performance.now(); + while('not-equal'!==Atomics.wait(state.sabOPView, state.opIds.rc, -1)){ + /* + The reason for this loop is buried in the details of a long + discussion at: + + https://github.com/sqlite/sqlite-wasm/issues/12 + + Summary: in at least one browser flavor, under high loads, + the wait()/notify() pairings can get out of sync and/or + spuriously wake up. Calling wait() here until it returns + 'not-equal' gets them back in sync. + */ + } + /* When the above wait() call returns 'not-equal', the async + half will have completed the operation and reported its + results in the state.opIds.rc slot of the SAB. It may have + also serialized an exception for us. */ + const rc = Atomics.load(state.sabOPView, state.opIds.rc); + metrics[op].wait += performance.now() - t; + if(rc && state.asyncS11nExceptions){ + const err = state.s11n.deserialize(); + if(err) error(op+"() async error:",...err); + } + return rc; + }; + + const opTimer = Object.create(null); + opTimer.op = undefined; + opTimer.start = undefined; + const mTimeStart = opfsVfs.mTimeStart = (op)=>{ + opTimer.start = performance.now(); + opTimer.op = op; + ++metrics[op].count; + }; + const mTimeEnd = opfsVfs.mTimeEnd = ()=>( + metrics[opTimer.op].time += performance.now() - opTimer.start + ); + + /** + Map of sqlite3_file pointers to objects constructed by xOpen(). + */ + const __openFiles = opfsVfs.__openFiles = Object.create(null); + + /** + Impls for the sqlite3_io_methods methods. Maintenance reminder: + members are in alphabetical order to simplify finding them. + */ + const ioSyncWrappers = opfsVfs.ioSyncWrappers = util.nu({ + xCheckReservedLock: function(pFile,pOut){ + /** + After consultation with a topic expert: "opfs-wl" will + continue to use the same no-op impl which "opfs" does + because: + + - xCheckReservedLock() is just a hint. If SQLite needs to + lock, it's still going to try to lock. + + - We cannot do this check synchronously in "opfs-wl", + so would need to pass it to the async proxy. That would + make it inordinately expensive considering that it's + just a hint. + */ + wasm.poke(pOut, 0, 'i32'); + return 0; + }, + xClose: function(pFile){ + mTimeStart('xClose'); + let rc = 0; + const f = __openFiles[pFile]; + if(f){ + delete __openFiles[pFile]; + rc = opRun('xClose', pFile); + if(f.sq3File) f.sq3File.dispose(); + } + mTimeEnd(); + return rc; + }, + xDeviceCharacteristics: function(pFile){ + return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; + }, + xFileControl: function(pFile, opId, pArg){ + /*mTimeStart('xFileControl'); + mTimeEnd();*/ + return capi.SQLITE_NOTFOUND; + }, + xFileSize: function(pFile,pSz64){ + mTimeStart('xFileSize'); + let rc = opRun('xFileSize', pFile); + if(0==rc){ + try { + const sz = state.s11n.deserialize()[0]; + wasm.poke(pSz64, sz, 'i64'); + }catch(e){ + error("Unexpected error reading xFileSize() result:",e); + rc = state.sq3Codes.SQLITE_IOERR; + } + } + mTimeEnd(); + return rc; + }, + xRead: function(pFile,pDest,n,offset64){ + mTimeStart('xRead'); + const f = __openFiles[pFile]; + let rc; + try { + rc = opRun('xRead',pFile, n, Number(offset64)); + if(0===rc || capi.SQLITE_IOERR_SHORT_READ===rc){ + /** + Results get written to the SharedArrayBuffer f.sabView. + Because the heap is _not_ a SharedArrayBuffer, we have + to copy the results. TypedArray.set() seems to be the + fastest way to copy this. */ + wasm.heap8u().set(f.sabView.subarray(0, n), Number(pDest)); + } + }catch(e){ + error("xRead(",arguments,") failed:",e,f); + rc = capi.SQLITE_IOERR_READ; + } + mTimeEnd(); + return rc; + }, + xSync: function(pFile,flags){ + mTimeStart('xSync'); + const rc = opRun('xSync', pFile, flags); + mTimeEnd(); + return rc; + }, + xTruncate: function(pFile,sz64){ + mTimeStart('xTruncate'); + const rc = opRun('xTruncate', pFile, Number(sz64)); + mTimeEnd(); + return rc; + }, + xWrite: function(pFile,pSrc,n,offset64){ + mTimeStart('xWrite'); + const f = __openFiles[pFile]; + let rc; + try { + f.sabView.set(wasm.heap8u().subarray( + Number(pSrc), Number(pSrc) + n + )); + rc = opRun('xWrite', pFile, n, Number(offset64)); + }catch(e){ + error("xWrite(",arguments,") failed:",e,f); + rc = capi.SQLITE_IOERR_WRITE; + } + mTimeEnd(); + return rc; + } + })/*ioSyncWrappers*/; + + /** + Impls for the sqlite3_vfs methods. Maintenance reminder: members + are in alphabetical order to simplify finding them. + */ + const vfsSyncWrappers = opfsVfs.vfsSyncWrappers = { + xAccess: function(pVfs,zName,flags,pOut){ + mTimeStart('xAccess'); + const rc = opRun('xAccess', wasm.cstrToJs(zName)); + wasm.poke( pOut, (rc ? 0 : 1), 'i32' ); + mTimeEnd(); + return 0; + }, + xCurrentTime: function(pVfs,pOut){ + wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), + 'double'); + return 0; + }, + xCurrentTimeInt64: function(pVfs,pOut){ + wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(), + 'i64'); + return 0; + }, + xDelete: function(pVfs, zName, doSyncDir){ + mTimeStart('xDelete'); + const rc = opRun('xDelete', wasm.cstrToJs(zName), doSyncDir, false); + mTimeEnd(); + return rc; + }, + xFullPathname: function(pVfs,zName,nOut,pOut){ + /* Until/unless we have some notion of "current dir" + in OPFS, simply copy zName to pOut... */ + const i = wasm.cstrncpy(pOut, zName, nOut); + return i<nOut ? 0 : capi.SQLITE_CANTOPEN + /*CANTOPEN is required by the docs but SQLITE_RANGE would be a closer match*/; + }, + xGetLastError: function(pVfs,nOut,pOut){ + /* Mutex use in the overlying APIs cause xGetLastError() to + not be terribly useful for us. e.g. it can't be used to + convey error messages from xOpen() because there would be a + race condition between sqlite3_open()'s call to xOpen() and + this function. */ + sqlite3.config.warn("OPFS xGetLastError() has nothing sensible to return."); + return 0; + }, + //xSleep is optionally defined below + xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){ + mTimeStart('xOpen'); + let opfsFlags = 0; + let jzName, zToFree; + if( !zName ){ + jzName = opfsUtil.randomFilename(); + zName = zToFree = wasm.allocCString(jzName); + }else if(wasm.isPtr(zName)){ + if(capi.sqlite3_uri_boolean(zName, "opfs-unlock-asap", 0)){ + /* -----------------------^^^^^ MUST pass the untranslated + C-string here. */ + opfsFlags |= state.opfsFlags.OPFS_UNLOCK_ASAP; + } + if(capi.sqlite3_uri_boolean(zName, "delete-before-open", 0)){ + opfsFlags |= state.opfsFlags.OPFS_UNLINK_BEFORE_OPEN; + } + jzName = wasm.cstrToJs(zName); + //sqlite3.config.warn("xOpen zName =",zName, "opfsFlags =",opfsFlags); + }else{ + sqlite3.config.error("Impossible zName value in xOpen?", zName); + return capi.SQLITE_CANTOPEN; + } + const fh = util.nu({ + fid: pFile, + filename: jzName, + sab: new SharedArrayBuffer(state.fileBufferSize), + flags: flags, + readOnly: !(capi.SQLITE_OPEN_CREATE & flags) + && !!(flags & capi.SQLITE_OPEN_READONLY) + }); + const rc = opRun('xOpen', pFile, jzName, flags, opfsFlags); + if(rc){ + if( zToFree ) wasm.dealloc(zToFree); + }else{ + /* Recall that sqlite3_vfs::xClose() will be called, even on + error, unless pFile->pMethods is NULL. */ + if(fh.readOnly){ + wasm.poke(pOutFlags, capi.SQLITE_OPEN_READONLY, 'i32'); + } + __openFiles[pFile] = fh; + fh.sabView = state.sabFileBufView; + fh.sq3File = new capi.sqlite3_file(pFile); + if( zToFree ) fh.sq3File.addOnDispose(zToFree); + fh.sq3File.$pMethods = opfsIoMethods.pointer; + fh.lockType = capi.SQLITE_LOCK_NONE; + } + mTimeEnd(); + return rc; + }/*xOpen()*/ + }/*vfsSyncWrappers*/; + + const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; + if(pDVfs){ + const dVfs = new capi.sqlite3_vfs(pDVfs); + opfsVfs.$xRandomness = dVfs.$xRandomness; + opfsVfs.$xSleep = dVfs.$xSleep; + dVfs.dispose(); + } + if(!opfsVfs.$xRandomness){ + /* If the default VFS has no xRandomness(), add a basic JS impl... */ + opfsVfs.vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){ + const heap = wasm.heap8u(); + let i = 0; + const npOut = Number(pOut); + for(; i < nOut; ++i) heap[npOut + i] = (Math.random()*255000) & 0xFF; + return i; + }; + } + if(!opfsVfs.$xSleep){ + /* If we can inherit an xSleep() impl from the default VFS then + assume it's sane and use it, otherwise install a JS-based + one. */ + opfsVfs.vfsSyncWrappers.xSleep = function(pVfs,ms){ + mTimeStart('xSleep'); + Atomics.wait(state.sabOPView, state.opIds.xSleep, 0, ms); + mTimeEnd(); + return 0; + }; + } + +//#define vfs.metrics.enable +//#// import initS11n() +//#include api/opfs-common-inline.c-pp.js +//#undef vfs.metrics.enable + opfsVfs.initS11n = initS11n; + + /** + To be called by the VFS's main installation routine after it has + wired up enough state to provide its overridden io-method impls + (which must be properties of the ioMethods argument). Returns a + Promise which the installation routine must return. callback must + be a function which performs any post-bootstrap touchups, namely + plugging in a sqlite3.oo1 wrapper. It is passed (sqlite3, opfsVfs), + where opfsVfs is the sqlite3_vfs object which was set up by + opfsUtil.createVfsState(). + */ + opfsVfs.bindVfs = function(ioMethods, callback){ + Object.assign(opfsVfs.ioSyncWrappers, ioMethods); + const thePromise = new Promise(function(promiseResolve_, promiseReject_){ + let promiseWasRejected = undefined; + const promiseReject = (err)=>{ + promiseWasRejected = true; + opfsVfs.dispose(); + return promiseReject_(err); + }; + const promiseResolve = ()=>{ + try{ + callback(sqlite3, opfsVfs); + }catch(e){ + return promiseReject(e); + } + promiseWasRejected = false; + return promiseResolve_(sqlite3); + }; + const options = opfsUtil.options; + let proxyUri = options.proxyUri +( + (options.proxyUri.indexOf('?')<0) ? '?' : '&' + )+'vfs='+vfsName; + //sqlite3.config.error("proxyUri",options.proxyUri, (new Error())); + const W = opfsVfs.worker = +//#if target:es6-bundler-friendly + (()=>{ + /* _Sigh_... */ + switch(vfsName){ + case 'opfs': + return new Worker(new URL("sqlite3-opfs-async-proxy.js?vfs=opfs", import.meta.url)); + case 'opfs-wl': + return new Worker(new URL("sqlite3-opfs-async-proxy.js?vfs=opfs-wl", import.meta.url)); + } + })(); +//#elif target:es6-module + new Worker(new URL(proxyUri, import.meta.url)); +//#else + new Worker(proxyUri); +//#/if + let zombieTimer = setTimeout(()=>{ + /* At attempt to work around a browser-specific quirk in which + the Worker load is failing in such a way that we neither + resolve nor reject it. This workaround gives that resolve/reject + a time limit and rejects if that timer expires. Discussion: + https://sqlite.org/forum/forumpost/a708c98dcb3ef */ + if(undefined===promiseWasRejected){ + promiseReject( + new Error("Timeout while waiting for OPFS async proxy worker.") + ); + } + }, 4000); + W._originalOnError = W.onerror /* will be restored later */; + W.onerror = function(err){ + // The error object doesn't contain any useful info when the + // failure is, e.g., that the remote script is 404. + error("Error initializing OPFS asyncer:",err); + promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); + }; + + const opRun = opfsVfs.opRun; +//#if 0 + /** + Not part of the public API. Only for test/development use. + */ + opfsVfs.debug = { + asyncShutdown: ()=>{ + warn("Shutting down OPFS async listener. The OPFS VFS will no longer work."); + opRun('opfs-async-shutdown'); + }, + asyncRestart: ()=>{ + warn("Attempting to restart OPFS VFS async listener. Might work, might not."); + W.postMessage({type: 'opfs-async-restart'}); + } + }; +//#/if + + const sanityCheck = function(){ + const scope = wasm.scopedAllocPush(); + const sq3File = new capi.sqlite3_file(); + try{ + const fid = sq3File.pointer; + const openFlags = capi.SQLITE_OPEN_CREATE + | capi.SQLITE_OPEN_READWRITE + //| capi.SQLITE_OPEN_DELETEONCLOSE + | capi.SQLITE_OPEN_MAIN_DB; + const pOut = wasm.scopedAlloc(8); + const dbFile = "/sanity/check/file"+randomFilename(8); + const zDbFile = wasm.scopedAllocCString(dbFile); + let rc; + state.s11n.serialize("This is ä string."); + rc = state.s11n.deserialize(); + log("deserialize() says:",rc); + if("This is ä string."!==rc[0]) toss("String d13n error."); + opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.peek(pOut,'i32'); + log("xAccess(",dbFile,") exists ?=",rc); + rc = opfsVfs.vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile, + fid, openFlags, pOut); + log("open rc =",rc,"state.sabOPView[xOpen] =", + state.sabOPView[state.opIds.xOpen]); + if(0!==rc){ + error("open failed with code",rc); + return; + } + opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.peek(pOut,'i32'); + if(!rc) toss("xAccess() failed to detect file."); + rc = opfsVfs.ioSyncWrappers.xSync(sq3File.pointer, 0); + if(rc) toss('sync failed w/ rc',rc); + rc = opfsVfs.ioSyncWrappers.xTruncate(sq3File.pointer, 1024); + if(rc) toss('truncate failed w/ rc',rc); + wasm.poke(pOut,0,'i64'); + rc = opfsVfs.ioSyncWrappers.xFileSize(sq3File.pointer, pOut); + if(rc) toss('xFileSize failed w/ rc',rc); + log("xFileSize says:",wasm.peek(pOut, 'i64')); + rc = opfsVfs.ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1); + if(rc) toss("xWrite() failed!"); + const readBuf = wasm.scopedAlloc(16); + rc = opfsVfs.ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2); + wasm.poke(readBuf+6,0); + let jRead = wasm.cstrToJs(readBuf); + log("xRead() got:",jRead); + if("sanity"!==jRead) toss("Unexpected xRead() value."); + if(opfsVfs.vfsSyncWrappers.xSleep){ + log("xSleep()ing before close()ing..."); + opfsVfs.vfsSyncWrappers.xSleep(opfsVfs.pointer,2000); + log("waking up from xSleep()"); + } + rc = opfsVfs.ioSyncWrappers.xClose(fid); + log("xClose rc =",rc,"sabOPView =",state.sabOPView); + log("Deleting file:",dbFile); + opfsVfs.vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234); + opfsVfs.vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.peek(pOut,'i32'); + if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete()."); + warn("End of OPFS sanity checks."); + }finally{ + sq3File.dispose(); + wasm.scopedAllocPop(scope); + } + }/*sanityCheck()*/; + + W.onmessage = function({data}){ + //sqlite3.config.warn(vfsName,"Worker.onmessage:",data); + switch(data.type){ + case 'opfs-unavailable': + /* Async proxy has determined that OPFS is unavailable. There's + nothing more for us to do here. */ + promiseReject(new Error(data.payload.join(' '))); + break; + case 'opfs-async-loaded': + /* Arrives as soon as the asyc proxy finishes loading. + Pass our config and shared state on to the async + worker. */ + delete state.vfs; + W.postMessage({type: 'opfs-async-init', args: util.nu(state)}); + break; + case 'opfs-async-inited': { + /* Indicates that the async partner has received the 'init' + and has finished initializing, so the real work can + begin... */ + if(true===promiseWasRejected){ + break /* promise was already rejected via timer */; + } + clearTimeout(zombieTimer); + zombieTimer = null; + try { + sqlite3.vfs.installVfs({ + io: {struct: opfsVfs.ioMethods, methods: opfsVfs.ioSyncWrappers}, + vfs: {struct: opfsVfs, methods: opfsVfs.vfsSyncWrappers} + }); + state.sabOPView = new Int32Array(state.sabOP); + state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); + state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + opfsVfs.initS11n(); + delete opfsVfs.initS11n; + if(options.sanityChecks){ + warn("Running sanity checks because of opfs-sanity-check URL arg..."); + sanityCheck(); + } + if(opfsUtil.thisThreadHasOPFS()){ + opfsUtil.getRootDir().then((d)=>{ + W.onerror = W._originalOnError; + delete W._originalOnError; + log("End of OPFS sqlite3_vfs setup.", opfsVfs); + promiseResolve(); + }).catch(promiseReject); + }else{ + promiseResolve(); + } + }catch(e){ + error(e); + promiseReject(e); + } + break; + } + case 'debug': + warn("debug message from worker:",data); + break; + default: { + const errMsg = ( + "Unexpected message from the OPFS async worker: " + + JSON.stringify(data) + ); + error(errMsg); + promiseReject(new Error(errMsg)); + break; + } + }/*switch(data.type)*/ + }/*W.onmessage()*/; + })/*thePromise*/; + return thePromise; + }/*bindVfs()*/; + + return state; + }/*createVfsState()*/; + +}/*sqlite3ApiBootstrap.initializers*/); +//#/if target:node diff --git a/ext/wasm/api/post-js-footer.js b/ext/wasm/api/post-js-footer.js index c6a2e1517..f8050ddd3 100644 --- a/ext/wasm/api/post-js-footer.js +++ b/ext/wasm/api/post-js-footer.js @@ -1,3 +1,72 @@ +/* + 2022-07-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file is the tail end of the sqlite3-api.js constellation, + closing the function scope opened by post-js-header.js. + + In terms of amalgamation code placement, this file is appended + immediately after the final sqlite3-api-*.js piece. Those files + cooperate to prepare sqlite3ApiBootstrap() and this file calls it. + It is run within a context which gives it access to Emscripten's + Module object, after sqlite3.wasm is loaded but before + sqlite3ApiBootstrap() has been called. + + Because this code resides (after building) inside the function + installed by post-js-header.js, it has access to state set up by + pre-js.c-pp.js and friends. +*/ +try{ + /* We are in the closing block of Module.runSQLite3PostLoadInit(), so + its arguments are visible here. */ + + /* Config options for sqlite3ApiBootstrap(). */ + const bootstrapConfig = Object.assign( + Object.create(null), + /** The WASM-environment-dependent configuration for sqlite3ApiBootstrap() */ + { + memory: ('undefined'!==typeof wasmMemory) + ? wasmMemory + : EmscriptenModule['wasmMemory'], + exports: ('undefined'!==typeof wasmExports) + ? wasmExports /* emscripten >=3.1.44 */ + : (Object.prototype.hasOwnProperty.call(EmscriptenModule,'wasmExports') + ? EmscriptenModule['wasmExports'] + : EmscriptenModule['asm']/* emscripten <=3.1.43 */) + }, + globalThis.sqlite3ApiBootstrap.defaultConfig, // default options + globalThis.sqlite3ApiConfig || {} // optional client-provided options + ); + + sqlite3InitScriptInfo.debugModule("Bootstrapping lib config", bootstrapConfig); + + /** + For purposes of the Emscripten build, call sqlite3ApiBootstrap(). + Ideally clients should be able to inject their own config here, + but that's not practical in this particular build constellation + because of the order everything happens in. Clients may either + define globalThis.sqlite3ApiConfig or modify + globalThis.sqlite3ApiBootstrap.defaultConfig to tweak the default + configuration used by a no-args call to sqlite3ApiBootstrap(), + but must have first loaded their WASM module in order to be able + to provide the necessary configuration state. + */ + const p = globalThis.sqlite3ApiBootstrap(bootstrapConfig); + delete globalThis.sqlite3ApiBootstrap; + return p /* the eventual result of globalThis.sqlite3InitModule() */; +}catch(e){ + console.error("sqlite3ApiBootstrap() error:",e); + throw e; +} + //console.warn("This is the end of the Module.runSQLite3PostLoadInit handler."); }/*Module.runSQLite3PostLoadInit(...)*/; //console.warn("This is the end of the setup of the (pending) Module.runSQLite3PostLoadInit"); diff --git a/ext/wasm/api/post-js-header.js b/ext/wasm/api/post-js-header.js index cdc6b3a38..348f80ea0 100644 --- a/ext/wasm/api/post-js-header.js +++ b/ext/wasm/api/post-js-header.js @@ -3,14 +3,13 @@ post-js.js for use with Emscripten's --post-js flag, so it gets injected in the earliest stages of sqlite3InitModule(). - This function wraps the whole SQLite3 library but does not - bootstrap it. - Running this function will bootstrap the library and return a Promise to the sqlite3 namespace object. + + In the canonical builds, this gets called by extern-post-js.c-pp.js */ -Module.runSQLite3PostLoadInit = function( - sqlite3InitScriptInfo /* populated by extern-post-js.c-pp.js */, +Module.runSQLite3PostLoadInit = async function( + sqlite3InitScriptInfo, EmscriptenModule/*the Emscripten-style module object*/, sqlite3IsUnderTest ){ @@ -35,7 +34,7 @@ Module.runSQLite3PostLoadInit = function( - sqlite3-vtab-helper.c-pp.js => Utilities for virtual table impls - sqlite3-vfs-opfs.c-pp.js => OPFS VFS - sqlite3-vfs-opfs-sahpool.c-pp.js => OPFS SAHPool VFS - - sqlite3-api-cleanup.js => final bootstrapping phase + - sqlite3-vfs-opfs-wl.c-pp.js => WebLock-using OPFS VFS - post-js-footer.js => this file's epilogue And all of that gets sandwiched between extern-pre-js.js and diff --git a/ext/wasm/api/pre-js.c-pp.js b/ext/wasm/api/pre-js.c-pp.js index 8a4a0f9fd..fbb48f9ea 100644 --- a/ext/wasm/api/pre-js.c-pp.js +++ b/ext/wasm/api/pre-js.c-pp.js @@ -14,12 +14,37 @@ itself. i.e. try to keep file-local symbol names obnoxiously collision-resistant. */ +/** + This file was preprocessed using: + +//#@ policy error + @c-pp::argv@ +//#@ policy off +*/ +//#if unsupported-build +/** + UNSUPPORTED BUILD: + + This SQLite JS build configuration is entirely unsupported! It has + not been tested beyond the ability to compile it. It may not + load. It may not work properly. Only builds _directly_ targeting + browser environments ("vanilla" JS and ESM modules) are supported + and tested. Builds which _indirectly_ target browsers (namely + bundler-friendly builds and any node builds) are not supported + deliverables. +*/ +//#/if +//#if not target:es6-bundler-friendly (function(Module){ const sIMS = globalThis.sqlite3InitModuleState/*from extern-post-js.c-pp.js*/ || Object.assign(Object.create(null),{ - debugModule: ()=>{ - console.warn("globalThis.sqlite3InitModuleState is missing"); + /* In WASMFS builds this file gets loaded once per thread, + but sqlite3InitModuleState is not getting set for the + worker threads? That those workers seem to function fine + despite that is curious. */ + debugModule: function(){ + console.warn("globalThis.sqlite3InitModuleState is missing",arguments); } }); delete globalThis.sqlite3InitModuleState; @@ -47,6 +72,14 @@ approach. */ Module['locateFile'] = function(path, prefix) { + if( this.emscriptenLocateFile instanceof Function ){ + /* [tag:locateFile] Client-overridden impl. We do not support + this but offer it as a back-door which will go away the + moment either Emscripten changes that interface or we manage + to get non-Emscripten builds working. + https://sqlite.org/forum/forumpost/1eec339854c935bd */ + return this.emscriptenLocateFile(path, prefix); + } //#if target:es6-module return new URL(path, import.meta.url).href; //#else @@ -69,11 +102,10 @@ "result =", theFile ); return theFile; -//#endif target:es6-module +//#/if target:es6-module }.bind(sIMS); -//#if Module.instantiateWasm -//#if not wasmfs +//#if Module.instantiateWasm and not wasmfs and not target:node /** Override Module.instantiateWasm(). @@ -82,8 +114,15 @@ https://github.com/emscripten-core/emscripten/issues/17951 In such builds we must disable this. + + It's disabled in the (unsupported/untested) node builds because + node does not do fetch(). */ Module['instantiateWasm'] = function callee(imports,onSuccess){ + if( this.emscriptenInstantiateWasm instanceof Function ){ + /* See [tag:locateFile]. Same story here */ + return this.emscriptenInstantiateWasm(imports, onSuccess); + } const sims = this; const uri = Module.locateFile( sims.wasmFilename, ( @@ -109,7 +148,7 @@ .then(finalThen) return loadWasm(); }.bind(sIMS); -//#endif not wasmfs -//#endif Module.instantiateWasm +//#/if Module.instantiateWasm and not wasmfs })(Module); +//#/if not target:es6-bundler-friendly /* END FILE: api/pre-js.js. */ diff --git a/ext/wasm/api/sqlite3-api-cleanup.js b/ext/wasm/api/sqlite3-api-cleanup.js deleted file mode 100644 index 223566326..000000000 --- a/ext/wasm/api/sqlite3-api-cleanup.js +++ /dev/null @@ -1,83 +0,0 @@ -/* - 2022-07-22 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - This file is the tail end of the sqlite3-api.js constellation, - intended to be appended after all other sqlite3-api-*.js files so - that it can finalize any setup and clean up any global symbols - temporarily used for setting up the API's various subsystems. - - In Emscripten builds it's run in the context of what amounts to a - Module.postRun handler, though it's no longer actually a postRun - handler because Emscripten 4.0 changed postRun semantics in an - incompatible way. - - In terms of amalgamation code placement, this file is appended - immediately after the final sqlite3-api-*.js piece. Those files - cooperate to prepare sqlite3ApiBootstrap() and this file calls it. - It is run within a context which gives it access to Emscripten's - Module object, after sqlite3.wasm is loaded but before - sqlite3ApiBootstrap() has been called. - - Because this code resides (after building) inside the function - installed by post-js-header.js, it has access to the -*/ -'use strict'; -if( 'undefined' === typeof EmscriptenModule/*from post-js-header.js*/ ){ - console.warn("This is not running in the context of Module.runSQLite3PostLoadInit()"); - throw new Error("sqlite3-api-cleanup.js expects to be running in the "+ - "context of its Emscripten module loader."); -} -try{ - /* Config options for sqlite3ApiBootstrap(). */ - const bootstrapConfig = Object.assign( - Object.create(null), - globalThis.sqlite3ApiBootstrap.defaultConfig, // default options - globalThis.sqlite3ApiConfig || {}, // optional client-provided options - /** The WASM-environment-dependent configuration for sqlite3ApiBootstrap() */ - { - memory: ('undefined'!==typeof wasmMemory) - ? wasmMemory - : EmscriptenModule['wasmMemory'], - exports: ('undefined'!==typeof wasmExports) - ? wasmExports /* emscripten >=3.1.44 */ - : (Object.prototype.hasOwnProperty.call(EmscriptenModule,'wasmExports') - ? EmscriptenModule['wasmExports'] - : EmscriptenModule['asm']/* emscripten <=3.1.43 */) - } - ); - - /** Figure out if this is a 32- or 64-bit WASM build. */ - bootstrapConfig.wasmPtrIR = - 'number'===(typeof bootstrapConfig.exports.sqlite3_libversion()) - ? 'i32' :'i64'; - const sIMS = sqlite3InitScriptInfo; - sIMS.debugModule("Bootstrapping lib config", sIMS); - - /** - For purposes of the Emscripten build, call sqlite3ApiBootstrap(). - Ideally clients should be able to inject their own config here, - but that's not practical in this particular build constellation - because of the order everything happens in. Clients may either - define globalThis.sqlite3ApiConfig or modify - globalThis.sqlite3ApiBootstrap.defaultConfig to tweak the default - configuration used by a no-args call to sqlite3ApiBootstrap(), - but must have first loaded their WASM module in order to be able - to provide the necessary configuration state. - */ - const p = globalThis.sqlite3ApiBootstrap(bootstrapConfig); - delete globalThis.sqlite3ApiBootstrap; - return p /* the eventual result of globalThis.sqlite3InitModule() */; -}catch(e){ - console.error("sqlite3ApiBootstrap() error:",e); - throw e; -} -throw new Error("Maintenance required: this line should never be reached"); diff --git a/ext/wasm/api/sqlite3-api-glue.c-pp.js b/ext/wasm/api/sqlite3-api-glue.c-pp.js index 1c42b0150..9c525bd4c 100644 --- a/ext/wasm/api/sqlite3-api-glue.c-pp.js +++ b/ext/wasm/api/sqlite3-api-glue.c-pp.js @@ -98,6 +98,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_bind_parameter_name", "string", "sqlite3_stmt*", "int"], ["sqlite3_bind_pointer", "int", "sqlite3_stmt*", "int", "*", "string:static", "*"], + /* sqlite_bind_text() is hand-written */ + ["sqlite3_bind_zeroblob", "int", "sqlite3_stmt*", "int", "int"], ["sqlite3_busy_handler","int", [ "sqlite3*", new wasm.xWrap.FuncPtrAdapter({ @@ -120,7 +122,11 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_column_double","f64", "sqlite3_stmt*", "int"], ["sqlite3_column_int","int", "sqlite3_stmt*", "int"], ["sqlite3_column_name","string", "sqlite3_stmt*", "int"], +//#define proxy-text-apis 1 +//#if not proxy-text-apis +/* Search this file for tag:proxy-text-apis to see what this is about. */ ["sqlite3_column_text","string", "sqlite3_stmt*", "int"], +//#/if ["sqlite3_column_type","int", "sqlite3_stmt*", "int"], ["sqlite3_column_value","sqlite3_value*", "sqlite3_stmt*", "int"], ["sqlite3_commit_hook", "void*", [ @@ -196,6 +202,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_libversion_number", "int"], ["sqlite3_limit", "int", ["sqlite3*", "int", "int"]], ["sqlite3_malloc", "*","int"], + ["sqlite3_next_stmt", "sqlite3_stmt*", ["sqlite3*","sqlite3_stmt*"]], ["sqlite3_open", "int", "string", "*"], ["sqlite3_open_v2", "int", "string", "*", "int", "string"], /* sqlite3_prepare_v2() and sqlite3_prepare_v3() are handled @@ -317,7 +324,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_value_numeric_type", "int", "sqlite3_value*"], ["sqlite3_value_pointer", "*", "sqlite3_value*", "string:static"], ["sqlite3_value_subtype", "int", "sqlite3_value*"], +//#if not proxy-text-apis ["sqlite3_value_text", "string", "sqlite3_value*"], +//#/if ["sqlite3_value_type", "int", "sqlite3_value*"], ["sqlite3_vfs_find", "*", "string"], ["sqlite3_vfs_register", "int", "sqlite3_vfs*", "int"], @@ -493,7 +502,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ["sqlite3_activate_see", undefined, "string"] ); } -//#endif enable-see +//#/if enable-see if( wasm.bigIntEnabled && !!wasm.exports.sqlite3_declare_vtab ){ bindingSignatures.int64.push( @@ -969,10 +978,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ "entry SQLITE_WASM_DEALLOC (=="+capi.SQLITE_WASM_DEALLOC+")."); } const __rcMap = Object.create(null); - for(const t of ['resultCodes']){ - for(const e of Object.entries(wasm.ctype[t])){ - __rcMap[e[1]] = e[0]; - } + for(const e of Object.entries(wasm.ctype['resultCodes'])){ + __rcMap[e[1]] = e[0]; } /** For the given integer, returns the SQLITE_xxx result code as a @@ -984,8 +991,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const notThese = Object.assign(Object.create(null),{ // For each struct to NOT register, map its name to true: WasmTestStruct: true, - /* We unregister the kvvfs VFS from Worker threads below. */ - sqlite3_kvvfs_methods: !util.isUIThread(), /* sqlite3_index_info and friends require int64: */ sqlite3_index_info: !wasm.bigIntEnabled, sqlite3_index_constraint: !wasm.bigIntEnabled, @@ -1654,6 +1659,45 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }/*sqlite3_bind_text/blob()*/ +//#if proxy-text-apis + if(!capi.sqlite3_column_text){ + /*[tag:proxy-text-apis] + As discussed at: + + https://sqlite.org/forum/forumpost/d77281aec2df9ada + + Summary: there are opinions that sqlite3_column_text() and + sqlite3_value_text() should handle strings such that embedded + NULs are retained. This block does that. This block does _not_ + apply that special-case behavior to any number of _other_ + APIs which return C-strings. That discrepancy makes this + block highly arguable, but one can also argue that these two + specific functions can get away with such acrobatics without + it being called voodoo in a pejorative sense. + */ + const argStmt = wasm.xWrap.argAdapter('sqlite3_stmt*'), + argInt = wasm.xWrap.argAdapter('int'), + argValue = wasm.xWrap.argAdapter('sqlite3_value*'), + newStr = + (cstr,n)=>wasm.typedArrayToString(wasm.heap8u(), + Number(cstr), Number(cstr)+n) + capi.sqlite3_column_text = function(stmt, colIndex){ + const a0 = argStmt(stmt), a1 = argInt(colIndex); + const cstr = wasm.exports.sqlite3_column_text(a0, a1); + return cstr + ? newStr(cstr,wasm.exports.sqlite3_column_bytes(a0, a1)) + : null; + }; + capi.sqlite3_value_text = function(val){ + const a0 = argValue(val); + const cstr = wasm.exports.sqlite3_value_text(a0); + return cstr + ? newStr(cstr,wasm.exports.sqlite3_value_bytes(a0)) + : null; + }; + }/*text-return-related bindings*/ +//#/if proxy-text-apis + {/* sqlite3_config() */ /** Wraps a small subset of the C API's sqlite3_config() options. @@ -1740,105 +1784,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }; }/* auto-extension */ - const pKvvfs = capi.sqlite3_vfs_find("kvvfs"); - if( pKvvfs ){/* kvvfs-specific glue */ - if(util.isUIThread()){ - const kvvfsMethods = new capi.sqlite3_kvvfs_methods( - wasm.exports.sqlite3__wasm_kvvfs_methods() - ); - delete capi.sqlite3_kvvfs_methods; - - const kvvfsMakeKey = wasm.exports.sqlite3__wasm_kvvfsMakeKeyOnPstack, - pstack = wasm.pstack; - - const kvvfsStorage = (zClass)=> - ((115/*=='s'*/===wasm.peek(zClass)) - ? sessionStorage : localStorage); - - /** - Implementations for members of the object referred to by - sqlite3__wasm_kvvfs_methods(). We swap out the native - implementations with these, which use localStorage or - sessionStorage for their backing store. - */ - const kvvfsImpls = { - xRead: (zClass, zKey, zBuf, nBuf)=>{ - const stack = pstack.pointer, - astack = wasm.scopedAllocPush(); - try { - const zXKey = kvvfsMakeKey(zClass,zKey); - if(!zXKey) return -3/*OOM*/; - const jKey = wasm.cstrToJs(zXKey); - const jV = kvvfsStorage(zClass).getItem(jKey); - if(!jV) return -1; - const nV = jV.length /* We are relying 100% on v being - ASCII so that jV.length is equal - to the C-string's byte length. */; - if(nBuf<=0) return nV; - else if(1===nBuf){ - wasm.poke(zBuf, 0); - return nV; - } - const zV = wasm.scopedAllocCString(jV); - if(nBuf > nV + 1) nBuf = nV + 1; - wasm.heap8u().copyWithin( - Number(zBuf), Number(zV), wasm.ptr.addn(zV, nBuf,- 1) - ); - wasm.poke(wasm.ptr.add(zBuf, nBuf, -1), 0); - return nBuf - 1; - }catch(e){ - sqlite3.config.error("kvstorageRead()",e); - return -2; - }finally{ - pstack.restore(stack); - wasm.scopedAllocPop(astack); - } - }, - xWrite: (zClass, zKey, zData)=>{ - const stack = pstack.pointer; - try { - const zXKey = kvvfsMakeKey(zClass,zKey); - if(!zXKey) return 1/*OOM*/; - const jKey = wasm.cstrToJs(zXKey); - kvvfsStorage(zClass).setItem(jKey, wasm.cstrToJs(zData)); - return 0; - }catch(e){ - sqlite3.config.error("kvstorageWrite()",e); - return capi.SQLITE_IOERR; - }finally{ - pstack.restore(stack); - } - }, - xDelete: (zClass, zKey)=>{ - const stack = pstack.pointer; - try { - const zXKey = kvvfsMakeKey(zClass,zKey); - if(!zXKey) return 1/*OOM*/; - kvvfsStorage(zClass).removeItem(wasm.cstrToJs(zXKey)); - return 0; - }catch(e){ - sqlite3.config.error("kvstorageDelete()",e); - return capi.SQLITE_IOERR; - }finally{ - pstack.restore(stack); - } - } - }/*kvvfsImpls*/; - for(const k of Object.keys(kvvfsImpls)){ - kvvfsMethods[kvvfsMethods.memberKey(k)] = - wasm.installFunction( - kvvfsMethods.memberSignature(k), - kvvfsImpls[k] - ); - } - }else{ - /* Worker thread: unregister kvvfs to avoid it being used - for anything other than local/sessionStorage. It "can" - be used that way but it's not really intended to be. */ - capi.sqlite3_vfs_unregister(pKvvfs); - } - }/*pKvvfs*/ - /* Warn if client-level code makes use of FuncPtrAdapter. */ wasm.xWrap.FuncPtrAdapter.warnOnUse = true; @@ -1944,7 +1889,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } tgt[memKey] = fProxy; }else{ - const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name)); + const pFunc = wasm.installFunction(fProxy, sigN); tgt[memKey] = pFunc; if(!tgt.ondispose || !tgt.ondispose.__removeFuncList){ tgt.addOnDispose('ondispose.__removeFuncList handler', diff --git a/ext/wasm/api/sqlite3-api-oo1.c-pp.js b/ext/wasm/api/sqlite3-api-oo1.c-pp.js index f7a4e9ebd..13aa42719 100644 --- a/ext/wasm/api/sqlite3-api-oo1.c-pp.js +++ b/ext/wasm/api/sqlite3-api-oo1.c-pp.js @@ -26,6 +26,20 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ the sqlite3 binding if, e.g., the wrapper is in the main thread and the sqlite3 API is in a worker. */ + const outWrapper = function(f){ + return (...args)=>f("sqlite3.oo1:",...args); + }; + + const debug = sqlite3.__isUnderTest + ? outWrapper(console.debug.bind(console)) + : outWrapper(sqlite3.config.debug); + const warn = sqlite3.__isUnderTest + ? outWrapper(console.warn.bind(console)) + : outWrapper(sqlite3.config.warn); + const error = sqlite3.__isUnderTest + ? outWrapper(console.error.bind(console)) + : outWrapper(sqlite3.config.error); + /** In order to keep clients from manipulating, perhaps inadvertently, the underlying pointer values of DB and Stmt @@ -88,7 +102,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ wasm.installFunction('i(ippp)', function(t,c,p,x){ if(capi.SQLITE_TRACE_STMT===t){ // x == SQL, p == sqlite3_stmt* - console.log("SQL TRACE #"+(++this.counter)+' via sqlite3@'+c+':', + console.log("SQL TRACE #"+(++this.counter), + 'via sqlite3@'+c+'['+capi.sqlite3_db_filename(c,null)+']', wasm.cstrToJs(x)); } }.bind({counter: 0})); @@ -200,7 +215,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ if(stmt) stmt.finalize(); } }; -//#endif enable-see +//#/if enable-see /** A proxy for DB class constructors. It must be called with the @@ -213,41 +228,18 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ properties: - `.filename`: the db filename. It may be a special name like ":memory:" - or "". + or "". It may also be a URI-style name. - `.flags`: as documented in the DB constructor. - `.vfs`: as documented in the DB constructor. It also accepts those as the first 3 arguments. + + In non-default builds it may accept additional configuration + options. */ const dbCtorHelper = function ctor(...args){ - if(!ctor._name2vfs){ - /** - Map special filenames which we handle here (instead of in C) - to some helpful metadata... - - As of 2022-09-20, the C API supports the names :localStorage: - and :sessionStorage: for kvvfs. However, C code cannot - determine (without embedded JS code, e.g. via Emscripten's - EM_JS()) whether the kvvfs is legal in the current browser - context (namely the main UI thread). In order to help client - code fail early on, instead of it being delayed until they - try to read or write a kvvfs-backed db, we'll check for those - names here and throw if they're not legal in the current - context. - */ - ctor._name2vfs = Object.create(null); - const isWorkerThread = ('function'===typeof importScripts/*===running in worker thread*/) - ? (n)=>toss3("The VFS for",n,"is only available in the main window thread.") - : false; - ctor._name2vfs[':localStorage:'] = { - vfs: 'kvvfs', filename: isWorkerThread || (()=>'local') - }; - ctor._name2vfs[':sessionStorage:'] = { - vfs: 'kvvfs', filename: isWorkerThread || (()=>'session') - }; - } const opt = ctor.normalizeArgs(...args); //sqlite3.config.debug("DB ctor",opt); let pDb; @@ -269,12 +261,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ sqlite3.config.error("Invalid DB ctor args",opt,arguments); toss3("Invalid arguments for DB constructor:", arguments, "opts:", opt); } - let fnJs = wasm.isPtr(fn) ? wasm.cstrToJs(fn) : fn; - const vfsCheck = ctor._name2vfs[fnJs]; - if(vfsCheck){ - vfsName = vfsCheck.vfs; - fn = fnJs = vfsCheck.filename(fnJs); - } let oflags = 0; if( flagsStr.indexOf('c')>=0 ){ oflags |= capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE; @@ -299,7 +285,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }finally{ wasm.pstack.restore(stack); } - this.filename = fnJs; + this.filename = + /* A poor design choice we have to keep: this.filename may be + in the form "file:....?....". It really should have been + sqlite3_db_filename(pDb) but that discrepancy went too long + unnoticed to be able to change without risk of + breakage. DB.dbFilename() can be used to fetch _just_ the + name part. + */ wasm.isPtr(fn) ? wasm.cstrToJs(fn) : fn; + } __ptrMap.set(this, pDb); __stmtMap.set(this, Object.create(null)); @@ -307,7 +301,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ try{ //#if enable-see dbCtorApplySEEKey(this,opt); -//#endif +//#/if // Check for per-VFS post-open SQL/callback... const pVfs = capi.sqlite3_js_db_vfs(pDb) || toss3("Internal error: cannot get VFS for new db handle."); @@ -390,12 +384,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ The given db filename must be resolvable using whatever filesystem layer (virtual or otherwise) is set up for the default - sqlite3 VFS. + sqlite3 VFS or a VFS which can resolve it must be specified. - Note that the special sqlite3 db names ":memory:" and "" - (temporary db) have their normal special meanings here and need - not resolve to real filenames, but "" uses an on-storage - temporary database and requires that the VFS support that. + The special sqlite3 db names ":memory:" and "" (temporary db) + have their normal special meanings here and need not resolve to + real filenames, but "" uses an on-storage temporary database and + requires that the VFS support that. The second argument specifies the open/create mode for the database. It must be string containing a sequence of letters (in @@ -459,7 +453,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ is supplied and the database is encrypted, execution of the post-initialization SQL will fail, causing the constructor to throw. -//#endif enable-see +//#/if enable-see The `filename` and `vfs` arguments may be either JS strings or C-strings allocated via WASM. `flags` is required to be a JS @@ -808,6 +802,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ sqlite3_db_filename() value for the given database name, defaulting to "main". The argument may be either a JS string or a pointer to a WASM-allocated C-string. + + this.filename may be in the form of a URI-style string, whereas + the returned string contains only the filename part. */ dbFilename: function(dbName='main'){ return capi.sqlite3_db_filename(affirmDbOpen(this).pointer, dbName); @@ -929,15 +926,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ result set, but only if that statement has any result rows. The callback's "this" is the options object, noting that this function synthesizes one if the caller does not pass one to - exec(). The second argument passed to the callback is always - the current Stmt object, as it's needed if the caller wants to - fetch the column names or some such (noting that they could - also be fetched via `this.columnNames`, if the client provides - the `columnNames` option). If the callback returns a literal - `false` (as opposed to any other falsy value, e.g. an implicit - `undefined` return), any ongoing statement-`step()` iteration - stops without an error. The return value of the callback is - otherwise ignored. + exec(). The first argument passed to the callback is described + below. The second argument is always the current Stmt object, + as it's needed if the caller wants to fetch the column names or + some such (noting that they could also be fetched via + `this.columnNames`, if the client provides the `columnNames` + option). If the callback returns a literal `false` (as opposed + to any other falsy value, e.g. an implicit `undefined` return), + any ongoing statement-`step()` iteration stops without an + error. The return value of the callback is otherwise ignored. ACHTUNG: The callback MUST NOT modify the Stmt object. Calling any of the Stmt.get() variants, Stmt.getColumnName(), or @@ -970,20 +967,23 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ A.3) `'stmt'` causes the current Stmt to be passed to the callback, but this mode will trigger an exception if `resultRows` is an array because appending the transient - statement to the array would be downright unhelpful. + statement to the array would be downright unhelpful. This + option is a legacy feature, retained for backwards + compatibility. The statement object is passed as the second + argument to the callback, as described above. B) An integer, indicating a zero-based column in the result - row. Only that one single value will be passed on. + row. Only that one single value, in JS form, will be passed on. C) A string with a minimum length of 2 and leading character of '$' will fetch the row as an object, extract that one field, - and pass that field's value to the callback. Note that these - keys are case-sensitive so must match the case used in the + and pass that field's value to the callback. These keys are + case-sensitive so must match the case used in the SQL. e.g. `"select a A from t"` with a `rowMode` of `'$A'` would work but `'$a'` would not. A reference to a column not in the result set will trigger an exception on the first row (as - the check is not performed until rows are fetched). Note also - that `$` is a legal identifier character in JS so need not be + the check is not performed until rows are fetched). Note that + `$` is a legal identifier character in JS so need not be quoted. Any other `rowMode` value triggers an exception. @@ -1023,6 +1023,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - `callback` and `resultRows`: permit an array entries with semantics similar to those described for `bind` above. + OTOH, this function already does too much. */ exec: function(/*(sql [,obj]) || (obj)*/){ affirmDbOpen(this); @@ -1046,7 +1047,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /* Optimization: if the SQL is a TypedArray we can save some string conversion costs. */; /* Allocate the two output pointers (ppStmt, pzTail) and heap - space for the SQL (pSql). When prepare_v2() returns, pzTail + space for the SQL (pSql). When prepare_v3() returns, pzTail will point to somewhere in pSql. */ let sqlByteLen = isTA ? arg.sql.byteLength : wasm.jstrlen(arg.sql); const ppStmt = wasm.scopedAlloc( @@ -1058,8 +1059,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const pSqlEnd = wasm.ptr.add(pSql, sqlByteLen); if(isTA) wasm.heap8().set(arg.sql, pSql); else wasm.jstrcpy(arg.sql, wasm.heap8(), pSql, sqlByteLen, false); - wasm.poke(wasm.ptr.add(pSql, sqlByteLen), 0/*NUL terminator*/); - while(pSql && wasm.peek(pSql, 'i8') + wasm.poke8(wasm.ptr.add(pSql, sqlByteLen), 0/*NUL terminator*/); + while(pSql && wasm.peek8(pSql) /* Maintenance reminder:^^^ _must_ be 'i8' or else we will very likely cause an endless loop. What that's doing is checking for a terminating NUL byte. If we @@ -1123,6 +1124,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /* In order to trigger an exception in the INSERT...RETURNING locking scenario: https://sqlite.org/forum/forumpost/36f7a2e7494897df + [tag:insert-returning-reset] */).finalize(); stmt = null; }/*prepare() loop*/ @@ -1139,6 +1141,132 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return arg.returnVal(); }/*exec()*/, +//#if 0 + /** + Experimental and untested - do not use. + + Prepares one or more SQL statements, passing each to a callback + for processing. + + It requires an options object with the following properties: + + - "sql": SQL in any format accepted by exec(). + + - "callback" (function): gets passed each prepared statement, + as described below. + + - "asPointer" (bool=false): if true, the callback is passed the + WASM (sqlite3*) pointer instead of a Stmt object. + + - "saveSql" (array): if set, the SQL of each prepared statement + is appended to this array. This can be used without a callback + to split SQL into its component statements. Purely empty + statements (for for which sqlite3_prepare() returns a NULL + sqlite3_stmt, i.e. spaces and comments) are not added to this + list unless... + + - "saveEmpty" (bool=false): If true, empty statements are + retained in opt.saveSql, but their leading/trailing whitespace + is trimmed (as for queries) so they may be empty. + + For each statement in the input SQL: + + 1) If opt.saveSql is set, the SQL is appended to it. + + 2) If callback is set, callback(S) is called, where S is either + a Stmt object (by default) or an (sqlite3*) WASM pointer (if + opt.asPointer is true). If the callback returns a literal true + (as opposed to any other truthy value), ownership of S is + transferred to the callback, otherwise S is reset and finalized + as soon as the callback returns. If the callback throws, S is + unconditionally finalized. + + If neither of opt.saveSql nor opt.callback are set, this + function does nothing more than prepare and finalize each + statement, which will trigger an exception if any of them + contain invalid SQL. + */ + forEachStmt: function(opt){ + affirmDbOpen(this); + opt ??= Object.create(null); + if(!opt.sql){ + return toss3("exec() requires an SQL string."); + } + const sql = util.flexibleString(opt.sql); + const callback = opt.callback; + let stmt, pStmt; + const stack = wasm.scopedAllocPush(); + const saveSql = Array.isArray(opt.saveSql) ? opt.saveSql : undefined; + try{ + const isTA = util.isSQLableTypedArray(opt.sql) + /* Optimization: if the SQL is a TypedArray we can save some string + conversion costs. */; + /* Allocate the two output pointers (ppStmt, pzTail) and heap + space for the SQL (pSql). When prepare_v3() returns, pzTail + will point to somewhere in pSql. */ + let sqlByteLen = isTA ? opt.sql.byteLength : wasm.jstrlen(sql); + const ppStmt = wasm.scopedAlloc( + /* output (sqlite3_stmt**) arg and pzTail */ + (2 * wasm.ptr.size) + (sqlByteLen + 1/* SQL + NUL */) + ); + const pzTail = wasm.ptr.add(ppStmt, wasm.ptr.size) /* final arg to sqlite3_prepare_v2() */; + let pSql = wasm.ptr.add(pzTail, wasm.ptr.size) /* start of the SQL string */; + const pSqlEnd = wasm.ptr.add(pSql, sqlByteLen); + if(isTA) wasm.heap8().set(sql, pSql); + else wasm.jstrcpy(sql, wasm.heap8(), pSql, sqlByteLen, false); + wasm.poke8(wasm.ptr.add(pSql, sqlByteLen), 0/*NUL terminator*/); + while( pSql && wasm.peek8(pSql) ){ + pStmt = stmt = null; + wasm.pokePtr([ppStmt, pzTail], 0); + const zHead = pSql; + DB.checkRc(this, capi.sqlite3_prepare_v3( + this.pointer, pSql, sqlByteLen, 0, ppStmt, pzTail + )); + [pStmt, pSql] = wasm.peekPtr([ppStmt, pzTail]); + sqlByteLen = wasm.ptr.addn(pSqlEnd,-pSql); + if(opt.saveSql){ + if( pStmt ) opt.saveSql.push(capi.sqlite3_sql(pStmt).trim()); + else if( opt.saveEmpty ){ + saveSql.push(wasm.typedArrayToString( + wasm.heap8u(), Number(zHead), + wasm.ptr.addn(zHead, sqlByteLen) + ).trim(/*arguable*/)); + } + } + if(!pStmt) continue; + //sqlite3.config.debug("forEachStmt() pSql =",capi.sqlite3_sql(pStmt)); + if( !opt.callback ){ + capi.sqlite3_finalize(pStmt); + pStmt = null; + continue; + } + stmt = opt.asPointer ? null : new Stmt(this, pStmt, BindTypes); + if( true===callaback(stmt || pStmt) ){ + stmt = pStmt = null /*callback took ownership */; + }else if(stmt){ + pStmt = null; + stmt.reset( + /* See [tag:insert-returning-reset]. The thinking here is + that if the callback didn't throw for this, it + probably should have. + */).finalize(); + stmt = null; + }else{ + const rx = capi.sqlite3_reset(pStmt/*[tag:insert-returning-reset]*/); + capi.sqlite3_finalize(pStmt); + pStmt = null; + DB.checkRc(this, rx); + } + }/*prepare() loop*/ + }finally{ + if(stmt) stmt.finalize(); + else if(pStmt) capi.sqlite3_finalize(pStmt); + wasm.scopedAllocPop(stack); + } + return this; + }/*forEachStmt()*/, +//#/if nope + /** Creates a new UDF (User-Defined Function) which is accessible via SQL code. This function may be called in any of the @@ -2295,56 +2423,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Stmt }/*oo1 object*/; - if(util.isUIThread()){ - /** - Functionally equivalent to DB(storageName,'c','kvvfs') except - that it throws if the given storage name is not one of 'local' - or 'session'. - - As of version 3.46, the argument may optionally be an options - object in the form: - - { - filename: 'session'|'local', - ... etc. (all options supported by the DB ctor) - } - - noting that the 'vfs' option supported by main DB - constructor is ignored here: the vfs is always 'kvvfs'. - */ - sqlite3.oo1.JsStorageDb = function(storageName='session'){ - const opt = dbCtorHelper.normalizeArgs(...arguments); - storageName = opt.filename; - if('session'!==storageName && 'local'!==storageName){ - toss3("JsStorageDb db name must be one of 'session' or 'local'."); - } - opt.vfs = 'kvvfs'; - dbCtorHelper.call(this, opt); - }; - const jdb = sqlite3.oo1.JsStorageDb; - jdb.prototype = Object.create(DB.prototype); - /** Equivalent to sqlite3_js_kvvfs_clear(). */ - jdb.clearStorage = capi.sqlite3_js_kvvfs_clear; - /** - Clears this database instance's storage or throws if this - instance has been closed. Returns the number of - database blocks which were cleaned up. - */ - jdb.prototype.clearStorage = function(){ - return jdb.clearStorage(affirmDbOpen(this).filename); - }; - /** Equivalent to sqlite3_js_kvvfs_size(). */ - jdb.storageSize = capi.sqlite3_js_kvvfs_size; - /** - Returns the _approximate_ number of bytes this database takes - up in its storage or throws if this instance has been closed. - */ - jdb.prototype.storageSize = function(){ - return jdb.storageSize(affirmDbOpen(this).filename); - }; - }/*main-window-only bits*/ - }); //#else /* Built with the omit-oo1 flag. */ -//#endif if not omit-oo1 +//#/if if not omit-oo1 diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 069f3fdb5..e7b775fe5 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -27,13 +27,14 @@ /** sqlite3ApiBootstrap() is the only global symbol persistently exposed by this API. It is intended to be called one time at the - end of the API amalgamation process, passed configuration details - for the current environment, and then optionally be removed from - the global object using `delete globalThis.sqlite3ApiBootstrap`. + end of the API amalgamation process and passed configuration details + for the current environment. This function is not intended for client-level use. It is intended for use in creating bundles configured for specific WASM - environments. + environments. That said, the "sqlite3-api.js" intermediary build + file aims to be suitable for dropping in to custom builds, and it + exposes only this function. This function expects a configuration object, intended to abstract away details specific to any given WASM environment, primarily so @@ -93,9 +94,17 @@ can be replaced with (e.g.) empty functions to squelch all such output. - - `wasmfsOpfsDir`[^1]: Specifies the "mount point" of the OPFS-backed - filesystem in WASMFS-capable builds. + - `wasmfsOpfsDir`[^1]: Specifies the "mount point" of the + OPFS-backed filesystem in WASMFS-capable builds. This is only + used in WASMFS-capable builds of the library (which the canonical + builds do not include). + - `disable` (as of 3.53.0) may be an object with the following + properties: + - `vfs`, an object, may contain a map of VFS names to booleans. + Any mapping to falsy are disabled. The supported names + are: "kvvfs", "opfs", "opfs-sahpool", "opfs-wl". + - Other disabling options may be added in the future. [^1] = This property may optionally be a function, in which case this function calls that function to fetch the value, @@ -125,7 +134,8 @@ Both sqlite3ApiBootstrap.defaultConfig and globalThis.sqlite3ApiConfig get deleted by sqlite3ApiBootstrap() because any changes to them made after that point would have no - useful effect. + useful effect. This function also deletes itself from globalThis + when it's called. This function returns a Promise to the sqlite3 namespace object, which resolves after the async pieces of the library init are @@ -142,7 +152,8 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( ); return sqlite3ApiBootstrap.sqlite3; } - const config = Object.assign(Object.create(null),{ + const nu = (...obj)=>Object.assign(Object.create(null),...obj); + const config = nu({ exports: undefined, memory: undefined, bigIntEnabled: !!globalThis.BigInt64Array, @@ -159,7 +170,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( certain wasm.xWrap.resultAdapter()s. */ useStdAlloc: false - }, apiConfig || {}); + }, apiConfig); Object.assign(config, { allocExportName: config.useStdAlloc ? 'malloc' : 'sqlite3_malloc', @@ -177,14 +188,6 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( } }); - /** - Eliminate any confusion about whether these config objects may - be used after library initialization by eliminating the outward-facing - objects... - */ - delete globalThis.sqlite3ApiConfig; - delete sqlite3ApiBootstrap.defaultConfig; - /** The main sqlite3 binding API gets installed into this object, mimicking the C API as closely as we can. The numerous members @@ -200,7 +203,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( not documented are installed as 1-to-1 proxies for their C-side counterparts. */ - const capi = Object.create(null); + const capi = nu(); /** Holds state which are specific to the WASM-related infrastructure and glue code. @@ -209,7 +212,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( dynamically after the api object is fully constructed, so not all are documented in this file. */ - const wasm = Object.create(null); + const wasm = nu(); /** Internal helper for SQLite3Error ctor. */ const __rcStr = (rc)=>{ @@ -757,6 +760,12 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( toss: function(...args){throw new Error(args.join(' '))}, toss3, typedArrayPart: wasm.typedArrayPart, + nu, + assert: function(arg,msg){ + if( !arg ){ + util.toss("Assertion failed:",msg); + } + }, /** Given a byte array or ArrayBuffer, this function throws if the lead bytes of that buffer do not hold a SQLite3 database header, @@ -796,25 +805,10 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( /** wasm.X properties which are used for configuring the wasm - environment via whwashutil.js. + environment via whwashutil.js. This object gets fleshed out with + a number of WASM-specific utilities, in sqlite3-api-glue.c-pp.js. */ Object.assign(wasm, { - /** - The WASM IR (Intermediate Representation) value for - pointer-type values. If set then it MUST be one of 'i32' or - 'i64' (else an exception will be thrown). If it's not set, it - will default to 'i32'. - */ - pointerIR: config.wasmPtrIR, - - /** - True if BigInt support was enabled via (e.g.) the - Emscripten -sWASM_BIGINT flag, else false. When - enabled, certain 64-bit sqlite3 APIs are enabled which - are not otherwise enabled due to JS/WASM int64 - impedance mismatches. - */ - bigIntEnabled: !!config.bigIntEnabled, /** The symbols exported by the WASM environment. @@ -825,8 +819,9 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( /** When Emscripten compiles with `-sIMPORTED_MEMORY`, it initializes the heap and imports it into wasm, as opposed to - the other way around. In this case, the memory is not - available via this.exports.memory. + the other way around. In this case, the memory is not available + via this.exports.memory so the client must pass it in via + config.memory. */ memory: config.memory || config.exports['memory'] @@ -834,6 +829,29 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( "in either config.exports.memory (exported)", "or config.memory (imported)."), + /** + The WASM pointer size. If set then it MUST be one of 4 or 8 and + it MUST correspond to the WASM environment's pointer size. We + figure out the size by calling some un-JS-wrapped WASM function + which returns a pointer-type value. If that value is a BigInt, + it's 64-bit, else it's 32-bit. The pieces which populate + sqlite3.wasm (whwasmutil.js) can figure this out _if_ they can + allocate, but we have a chicken/egg situation there which makes + it illegal for that code to invoke wasm.dealloc() at the time + it would be needed. So we need to configure it ahead of time + (here) instead. + */ + pointerSize: ('number'===typeof config.exports.sqlite3_libversion()) ? 4 : 8, + + /** + True if BigInt support was enabled via (e.g.) the + Emscripten -sWASM_BIGINT flag, else false. When + enabled, certain 64-bit sqlite3 APIs are enabled which + are not otherwise enabled due to JS/WASM int64 + impedance mismatches. + */ + bigIntEnabled: !!config.bigIntEnabled, + /** WebAssembly.Table object holding the indirect function call table. Defaults to exports.__indirect_function_table. @@ -883,7 +901,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( Like this.alloc.impl(), this.realloc.impl() is a direct binding to the underlying realloc() implementation which does not throw - exceptions, instead returning 0 on allocation error. + exceptions, instead returning 0 (or 0n) on allocation error. */ realloc: undefined/*installed later*/, @@ -949,7 +967,11 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( }; wasm.realloc.impl = wasm.exports[keyRealloc]; wasm.dealloc = function f(m){ - f.impl(wasm.ptr.coerce(m)/*tag:64bit*/); + f.impl(wasm.ptr.coerce(m)/*tag:64bit*/) + /* This coerce() is the reason we have to set wasm.pointerSize before + calling WhWasmUtilInstaller(). If we don't, that code will call + into this very early in its init, before wasm.ptr has been set up, + resulting in a null deref here. */; }; wasm.dealloc.impl = wasm.exports[keyDealloc]; } @@ -995,7 +1017,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( rv[1] = m ? (f._rxInt.test(m[2]) ? +m[2] : m[2]) : true; }; } - const rc = Object.create(null), ov = [0,0]; + const rc = nu(), ov = [0,0]; let i = 0, k; while((k = capi.sqlite3_compileoption_get(i++))){ f._opt(k,ov); @@ -1003,7 +1025,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( } return f._result = rc; }else if(Array.isArray(optName)){ - const rc = Object.create(null); + const rc = nu(); optName.forEach((v)=>{ rc[v] = capi.sqlite3_compileoption_used(v); }); @@ -1020,18 +1042,18 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( }/*compileOptionUsed()*/; /** - sqlite3.wasm.pstack (pseudo-stack) holds a special-case intended - solely for short-lived, small data. In practice, it's primarily - used to allocate output pointers. It mus not be used for any - memory which needs to outlive the scope in which it's obtained - from pstack. + sqlite3.wasm.pstack (pseudo-stack) holds a special-case allocator + intended solely for short-lived, small data. In practice, it's + primarily used to allocate output pointers. It must not be used + for any memory which needs to outlive the scope in which it's + obtained from pstack. The library guarantees only that a minimum of 2kb are available in this allocator, and it may provide more (it's a build-time value). pstack.quota and pstack.remaining can be used to get the total resp. remaining amount of memory. - It has only a single intended usage: + It has only a single intended usage pattern: ``` const stackPos = pstack.pointer; @@ -1048,15 +1070,13 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( ``` This allocator is much faster than a general-purpose one but is - limited to usage patterns like the one shown above. + limited to usage patterns like the one shown above (which are + pretty common when using sqlite3.capi). - It operates from a static range of memory which lives outside of - space managed by Emscripten's stack-management, so does not - collide with Emscripten-provided stack allocation APIs. The - memory lives in the WASM heap and can be used with routines such - as wasm.poke() and wasm.heap8u().slice(). + The memory lives in the WASM heap and can be used with routines + such as wasm.poke() and wasm.heap8u().slice(). */ - wasm.pstack = Object.assign(Object.create(null),{ + wasm.pstack = nu({ /** Sets the current pstack position to the given pointer. Results are undefined if the passed-in value did not come from @@ -1128,7 +1148,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( argument: if it's 1, it returns a single pointer value. If it's more than 1, it returns the same as allocChunks(). - When a returned pointers will refer to a 64-bit value, e.g. a + When a returned pointer will refer to a 64-bit value, e.g. a double or int64, and that value must be written or fetched, e.g. using wasm.poke() or wasm.peek(), it is important that the pointer in question be aligned to an 8-byte @@ -1196,6 +1216,9 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( } })/*wasm.pstack properties*/; + /** + Docs: https://sqlite.org/wasm/doc/trunk/api-c-style.md#sqlite3_randomness + */ capi.sqlite3_randomness = (...args)=>{ if(1===args.length && util.isTypedArray(args[0]) @@ -1230,8 +1253,6 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( wasm.exports.sqlite3_randomness(...args); }; - /** State for sqlite3_wasmfs_opfs_dir(). */ - let __wasmfsOpfsDir = undefined; /** If the wasm environment has a WASMFS/OPFS-backed persistent storage directory, its path is returned by this function. If it @@ -1255,7 +1276,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( WASMFS capability requires a custom build. */ capi.sqlite3_wasmfs_opfs_dir = function(){ - if(undefined !== __wasmfsOpfsDir) return __wasmfsOpfsDir; + if(undefined !== this.dir) return this.dir; // If we have no OPFS, there is no persistent dir const pdir = config.wasmfsOpfsDir; if(!pdir @@ -1263,21 +1284,21 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( || !globalThis.FileSystemDirectoryHandle || !globalThis.FileSystemFileHandle || !wasm.exports.sqlite3__wasm_init_wasmfs){ - return __wasmfsOpfsDir = ""; + return this.dir = ""; } try{ if(pdir && 0===wasm.xCallWrapped( 'sqlite3__wasm_init_wasmfs', 'i32', ['string'], pdir )){ - return __wasmfsOpfsDir = pdir; + return this.dir = pdir; }else{ - return __wasmfsOpfsDir = ""; + return this.dir = ""; } }catch(e){ // sqlite3__wasm_init_wasmfs() is not available - return __wasmfsOpfsDir = ""; + return this.dir = ""; } - }; + }.bind(nu()); /** Returns true if sqlite3.capi.sqlite3_wasmfs_opfs_dir() is a @@ -1333,7 +1354,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( */ capi.sqlite3_js_vfs_list = function(){ const rc = []; - let pVfs = capi.sqlite3_vfs_find(wasm.ptr.coerce(0)); + let pVfs = capi.sqlite3_vfs_find(wasm.ptr.null); while(pVfs){ const oVfs = new capi.sqlite3_vfs(pVfs); rc.push(wasm.cstrToJs(oVfs.$zName)); @@ -1401,7 +1422,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( or not provided, then "main" is assumed. */ capi.sqlite3_js_db_vfs = - (dbPointer, dbName=0)=>util.sqlite3__wasm_db_vfs(dbPointer, dbName); + (dbPointer, dbName=wasm.ptr.null)=>util.sqlite3__wasm_db_vfs(dbPointer, dbName); /** A thin wrapper around capi.sqlite3_aggregate_context() which @@ -1597,86 +1618,6 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( return x===v ? undefined : x; } - if( util.isUIThread() ){ - /* Features specific to the main window thread... */ - - /** - Internal helper for sqlite3_js_kvvfs_clear() and friends. - Its argument should be one of ('local','session',""). - */ - const __kvvfsInfo = function(which){ - const rc = Object.create(null); - rc.prefix = 'kvvfs-'+which; - rc.stores = []; - if('session'===which || ""===which) rc.stores.push(globalThis.sessionStorage); - if('local'===which || ""===which) rc.stores.push(globalThis.localStorage); - return rc; - }; - - /** - Clears all storage used by the kvvfs DB backend, deleting any - DB(s) stored there. Its argument must be either 'session', - 'local', or "". In the first two cases, only sessionStorage - resp. localStorage is cleared. If it's an empty string (the - default) then both are cleared. Only storage keys which match - the pattern used by kvvfs are cleared: any other client-side - data are retained. - - This function is only available in the main window thread. - - Returns the number of entries cleared. - */ - capi.sqlite3_js_kvvfs_clear = function(which=""){ - let rc = 0; - const kvinfo = __kvvfsInfo(which); - kvinfo.stores.forEach((s)=>{ - const toRm = [] /* keys to remove */; - let i; - for( i = 0; i < s.length; ++i ){ - const k = s.key(i); - if(k.startsWith(kvinfo.prefix)) toRm.push(k); - } - toRm.forEach((kk)=>s.removeItem(kk)); - rc += toRm.length; - }); - return rc; - }; - - /** - This routine guesses the approximate amount of - window.localStorage and/or window.sessionStorage in use by the - kvvfs database backend. Its argument must be one of - ('session', 'local', ""). In the first two cases, only - sessionStorage resp. localStorage is counted. If it's an empty - string (the default) then both are counted. Only storage keys - which match the pattern used by kvvfs are counted. The returned - value is the "length" value of every matching key and value, - noting that JavaScript stores each character in 2 bytes. - - Note that the returned size is not authoritative from the - perspective of how much data can fit into localStorage and - sessionStorage, as the precise algorithms for determining - those limits are unspecified and may include per-entry - overhead invisible to clients. - */ - capi.sqlite3_js_kvvfs_size = function(which=""){ - let sz = 0; - const kvinfo = __kvvfsInfo(which); - kvinfo.stores.forEach((s)=>{ - let i; - for(i = 0; i < s.length; ++i){ - const k = s.key(i); - if(k.startsWith(kvinfo.prefix)){ - sz += k.length; - sz += s.getItem(k).length; - } - } - }); - return sz * 2 /* because JS uses 2-byte char encoding */; - }; - - }/* main-window-only bits */ - /** Wraps all known variants of the C-side variadic sqlite3_db_config(). @@ -1711,6 +1652,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( case capi.SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE: case capi.SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE: case capi.SQLITE_DBCONFIG_ENABLE_COMMENTS: + case capi.SQLITE_DBCONFIG_FP_DIGITS: if( !this.ip ){ this.ip = wasm.xWrap('sqlite3__wasm_db_config_ip','int', ['sqlite3*', 'int', 'int', '*']); @@ -1732,7 +1674,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( default: return capi.SQLITE_MISUSE; } - }.bind(Object.create(null)); + }.bind(nu()); /** Given a (sqlite3_value*), this function attempts to convert it @@ -1953,55 +1895,114 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( return (0===v) ? undefined : capi.sqlite3_value_to_js(v, throwIfCannotConvert); }; - /** - Internal impl of sqlite3_preupdate_new/old_js() and - sqlite3changeset_new/old_js(). - */ - const __newOldValue = function(pObj, iCol, impl){ - impl = capi[impl]; - if(!this.ptr) this.ptr = wasm.allocPtr(); - else wasm.pokePtr(this.ptr, 0); - const rc = impl(pObj, iCol, this.ptr); - if(rc) return SQLite3Error.toss(rc,arguments[2]+"() failed with code "+rc); - const pv = wasm.peekPtr(this.ptr); - return pv ? capi.sqlite3_value_to_js( pv, true ) : undefined; - }.bind(Object.create(null)); + if( true ){ /* changeset/preupdate additions... */ + /** + Internal impl of sqlite3_preupdate_new/old_js() and + sqlite3changeset_new/old_js(). + */ + const __newOldValue = function(pObj, iCol, impl){ + impl = capi[impl]; + if(!this.ptr) this.ptr = wasm.allocPtr(); + else wasm.pokePtr(this.ptr, 0); + const rc = impl(pObj, iCol, this.ptr); + if(rc) return SQLite3Error.toss(rc,arguments[2]+"() failed with code "+rc); + const pv = wasm.peekPtr(this.ptr); + return pv ? capi.sqlite3_value_to_js( pv, true ) : undefined; + }.bind(nu()); - /** - A wrapper around sqlite3_preupdate_new() which fetches the - sqlite3_value at the given index and returns the result of - passing it to sqlite3_value_to_js(). Throws on error. - */ - capi.sqlite3_preupdate_new_js = - (pDb, iCol)=>__newOldValue(pDb, iCol, 'sqlite3_preupdate_new'); + /** + A wrapper around sqlite3_preupdate_new() which fetches the + sqlite3_value at the given index and returns the result of + passing it to sqlite3_value_to_js(). Throws on error. + */ + capi.sqlite3_preupdate_new_js = + (pDb, iCol)=>__newOldValue(pDb, iCol, 'sqlite3_preupdate_new'); - /** - The sqlite3_preupdate_old() counterpart of - sqlite3_preupdate_new_js(), with an identical interface. - */ - capi.sqlite3_preupdate_old_js = - (pDb, iCol)=>__newOldValue(pDb, iCol, 'sqlite3_preupdate_old'); + /** + The sqlite3_preupdate_old() counterpart of + sqlite3_preupdate_new_js(), with an identical interface. + */ + capi.sqlite3_preupdate_old_js = + (pDb, iCol)=>__newOldValue(pDb, iCol, 'sqlite3_preupdate_old'); - /** - A wrapper around sqlite3changeset_new() which fetches the - sqlite3_value at the given index and returns the result of - passing it to sqlite3_value_to_js(). Throws on error. + /** + A wrapper around sqlite3changeset_new() which fetches the + sqlite3_value at the given index and returns the result of + passing it to sqlite3_value_to_js(). Throws on error. + + If sqlite3changeset_new() succeeds but has no value to report, + this function returns the undefined value, noting that + undefined is not a valid conversion from an `sqlite3_value`, so + is unambiguous. + */ + capi.sqlite3changeset_new_js = + (pChangesetIter, iCol) => __newOldValue(pChangesetIter, iCol, + 'sqlite3changeset_new'); - If sqlite3changeset_new() succeeds but has no value to report, - this function returns the undefined value, noting that undefined - is a valid conversion from an `sqlite3_value`, so is unambiguous. - */ - capi.sqlite3changeset_new_js = - (pChangesetIter, iCol) => __newOldValue(pChangesetIter, iCol, - 'sqlite3changeset_new'); + /** + The sqlite3changeset_old() counterpart of + sqlite3changeset_new_js(), with an identical interface. + */ + capi.sqlite3changeset_old_js = + (pChangesetIter, iCol)=>__newOldValue(pChangesetIter, iCol, + 'sqlite3changeset_old'); + }/*changeset/preupdate additions*/ /** - The sqlite3changeset_old() counterpart of - sqlite3changeset_new_js(), with an identical interface. + EXPERIMENTAL. For tentative addition in 3.53.0. + + sqlite3_js_retry_busy(maxTimes,callback[,beforeRetry]) + + Calls the given _synchronous_ callback function. If that function + returns sqlite3.capi.SQLITE_BUSY _or_ throws an SQLite3Error + with a resultCode property of that value then it will suppress + that error and try again, up to the given maximum number of + times. If the callback returns any other value than that, + it is returned. If the maximum number of retries has been + reached, an SQLite3Error with a resultCode value of + sqlite3.capi.SQLITE_BUSY is thrown. If the callback throws any + exception other than the aforementioned BUSY exception, it is + propagated. If it throws a BUSY exception on its final attempt, + that is propagated as well. + + If the beforeRetry argument is given, it must be a _synchronous_ + function. It is called immediately before each retry of the + callback (not for the initial call), passed the attempt number + (so it starts with 2, not 1). If it throws, the exception is + handled as described above. Its result value is ignored. + + To effectively retry "forever", pass a negative maxTimes value, + with the caveat that there is no recovery from that unless the + beforeRetry() can figure out when to throw. + + TODO: an async variant of this. */ - capi.sqlite3changeset_old_js = - (pChangesetIter, iCol)=>__newOldValue(pChangesetIter, iCol, - 'sqlite3changeset_old'); + capi.sqlite3_js_retry_busy = function(maxTimes, callback, beforeRetry){ + for(let n = 1; n <= maxTimes; ++n){ + try{ + if( beforeRetry && n>1 ) beforeRetry(n); + const rc = callback(); + if( capi.SQLITE_BUSY===rc ){ + if( n===maxTimes ){ + throw new SQLite3Error(rc, [ + "sqlite3_js_retry_busy() max retry attempts (", + maxTimes, + ") reached." + ].join('')); + } + continue; + } + return rc; + }catch(e){ + if( n<maxTimes + && (e instanceof SQLite3Error) + && e.resultCode===capi.SQLITE_BUSY ){ + continue; + } + throw e; + } + } + }; /* The remainder of the API will be set up in later steps. */ const sqlite3 = { @@ -2013,15 +2014,15 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( config, /** Holds the version info of the sqlite3 source tree from which - the generated sqlite3-api.js gets built. Note that its version - may well differ from that reported by sqlite3_libversion(), but - that should be considered a source file mismatch, as the JS and - WASM files are intended to be built and distributed together. + the generated sqlite3-api.js gets built. Its version may well + differ from that reported by sqlite3_libversion(), but that + should be considered a source file mismatch, as the JS and WASM + files are intended to be built and distributed together. This object is initially a placeholder which gets replaced by a build-generated object. */ - version: Object.create(null), + version: nu(), /** The library reserves the 'client' property for client-side use @@ -2041,9 +2042,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( async init will be fatal to the init as a whole, but init routines are themselves welcome to install dummy catch() handlers which are not fatal if their failure should be - considered non-fatal. If called more than once, the second and - subsequent calls are no-ops which return a pre-resolved - Promise. + considered non-fatal. Ideally this function is called as part of the Promise chain which handles the loading and bootstrapping of the API. If not @@ -2060,19 +2059,16 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( */ asyncPostInit: async function ff(){ if(ff.isReady instanceof Promise) return ff.isReady; - let lia = sqlite3ApiBootstrap.initializersAsync; - delete sqlite3ApiBootstrap.initializersAsync; + let lia = this.initializersAsync; + delete this.initializersAsync; const postInit = async ()=>{ if(!sqlite3.__isUnderTest){ /* Delete references to internal-only APIs which are used by some initializers. Retain them when running in test mode so that we can add tests for them. */ delete sqlite3.util; - /* It's conceivable that we might want to expose - StructBinder to client-side code, but it's only useful if - clients build their own sqlite3.wasm which contains their - own C struct types. */ delete sqlite3.StructBinder; + delete sqlite3.opfs; } return sqlite3; }; @@ -2090,21 +2086,26 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( let p = Promise.resolve(sqlite3); while(lia.length) p = p.then(lia.shift()); return ff.isReady = p.catch(catcher); - }, + }.bind(sqlite3ApiBootstrap), /** - scriptInfo ideally gets injected into this object by the - infrastructure which assembles the JS/WASM module. It contains - state which must be collected before sqlite3ApiBootstrap() can - be declared. It is not necessarily available to any - sqlite3ApiBootstrap.initializers but "should" be in place (if - it's added at all) by the time that - sqlite3ApiBootstrap.initializersAsync is processed. + scriptInfo holds information about the currenty-loading script + so that we can locate the WASM file if it's somewhere other + than the build-time-defined directory. It ideally gets injected + into this object by the infrastructure which assembles the + JS/WASM module. It contains state which must be collected + before sqlite3ApiBootstrap() can be declared. It is not + necessarily available to any sqlite3ApiBootstrap.initializers + but "should" be in place (if it's added at all) by the time + that sqlite3ApiBootstrap.initializersAsync is processed. This state is not part of the public API, only intended for use with the sqlite3 API bootstrapping and wasm-loading process. */ scriptInfo: undefined }; + if( 'undefined'!==typeof sqlite3IsUnderTest/* from post-js-header.js */ ){ + sqlite3.__isUnderTest = !!sqlite3IsUnderTest; + } try{ sqlite3ApiBootstrap.initializers.forEach((f)=>{ f(sqlite3); @@ -2117,16 +2118,34 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( } delete sqlite3ApiBootstrap.initializers; sqlite3ApiBootstrap.sqlite3 = sqlite3; - delete globalThis.sqlite3ApiBootstrap; - delete globalThis.sqlite3ApiConfig; - sqlite3InitScriptInfo.debugModule( - "sqlite3ApiBootstrap() complete", sqlite3 - ); - sqlite3.scriptInfo /* used by some async init code */ = - sqlite3InitScriptInfo /* from post-js-header.js */; - if( (sqlite3.__isUnderTest = sqlite3IsUnderTest /* from post-js-header.js */) ){ - sqlite3.config.emscripten = EmscriptenModule; - const iw = sqlite3InitScriptInfo.instantiateWasm; + if( 'undefined'!==typeof sqlite3InitScriptInfo/* from post-js-header.js */ ){ + sqlite3InitScriptInfo.debugModule( + "sqlite3ApiBootstrap() complete", sqlite3 + ); + sqlite3.scriptInfo + /* Used by some async init code. As of 2025-11-15 this is still + in use by the OPFS VFS for locating its worker. In non-Emscripten + builds, this would need to be injected in somewhere to get + that VFS loading. */ = sqlite3InitScriptInfo; + } + if( sqlite3.__isUnderTest ){ + if( 'undefined'!==typeof EmscriptenModule ){ + sqlite3.config.emscripten = EmscriptenModule; + } + /* + The problem with exposing these pieces (in non-testing runs) via + sqlite3.wasm is that it exposes non-SQLite pieces to the + clients, who may come to expect it to remain. _We_ only have + these data because we've overridden Emscripten's wasm file + loader, and if we lose that capability for some reason then + we'll lose access to this metadata. + + These data are interesting for exploring how the wasm/JS pieces + connect, e.g. for exploring exactly what Emscripten imports into + WASM from its JS glue, but it's not SQLite-related and is not + required for the library to work. + */ + const iw = sqlite3.scriptInfo?.instantiateWasm; if( iw ){ /* Metadata injected by the custom Module.instantiateWasm() in pre-js.c-pp.js. */ @@ -2135,10 +2154,21 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( sqlite3.wasm.imports = iw.imports; } } + + /** + Eliminate any confusion about whether these config objects may + be used after library initialization by eliminating the outward-facing + objects... + */ + delete globalThis.sqlite3ApiConfig; + delete globalThis.sqlite3ApiBootstrap; + delete sqlite3ApiBootstrap.defaultConfig; return sqlite3.asyncPostInit().then((s)=>{ - sqlite3InitScriptInfo.debugModule( - "sqlite3.asyncPostInit() complete", sqlite3 - ); + if( 'undefined'!==typeof sqlite3InitScriptInfo/* from post-js-header.js */ ){ + sqlite3InitScriptInfo.debugModule( + "sqlite3.asyncPostInit() complete", s + ); + } delete s.asyncPostInit; delete s.scriptInfo; delete s.emscripten; diff --git a/ext/wasm/api/sqlite3-api-worker1.c-pp.js b/ext/wasm/api/sqlite3-api-worker1.c-pp.js index 25262abf8..d2103ca85 100644 --- a/ext/wasm/api/sqlite3-api-worker1.c-pp.js +++ b/ext/wasm/api/sqlite3-api-worker1.c-pp.js @@ -677,4 +677,4 @@ sqlite3.initWorker1API = function(){ }); //#else /* Built with the omit-oo1 flag. */ -//#endif if not omit-oo1 +//#/if if not omit-oo1 diff --git a/ext/wasm/api/sqlite3-license-version-header.js b/ext/wasm/api/sqlite3-license-version-header.js index 482989463..dd32f4666 100644 --- a/ext/wasm/api/sqlite3-license-version-header.js +++ b/ext/wasm/api/sqlite3-license-version-header.js @@ -1,4 +1,5 @@ -/* +/* @preserve +** ** LICENSE for the sqlite3 WebAssembly/JavaScript APIs. ** ** This bundle (typically released as sqlite3.js or sqlite3.mjs) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js similarity index 62% rename from ext/wasm/api/sqlite3-opfs-async-proxy.js rename to ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js index e10d0dd50..d5df0b9f7 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js @@ -1,4 +1,4 @@ -/* +/* @preserve 2022-09-16 The author disclaims copyright to this source code. In place of a @@ -46,10 +46,52 @@ theFunc().then(...) is not compatible with the change to synchronous, but we do do not use those APIs that way. i.e. we don't _need_ to change anything for this, but at some point (after Chrome - versions (approximately) 104-107 are extinct) should change our + versions (approximately) 104-107 are extinct) we should change our usage of those methods to remove the "await". */ +//#if 0 +/** + 2026-04-04: this file gets included by both the "opfs" and "opfs-wl" + VFSes. It would, in hindsight, hypothetically be possible to restructure + it very slightly to support both VFSes via a single Worker instance. + + Some of the changes we would need for that: + + - The xLock/xUnlock "op codes" would need to differ for each impl. + i.e. we'd need state.opIds.xLock{,WL} and state.opIds.xUnlock{,WL} + to distinguish between the two, rather than doing so when this Worker + is loaded. + + - We would need to centralize loading of this Worker, outside of + the VFS-specific pieces, and change the handshake in order to be + able to distinguish between clients which support + Atomics.waitAsync() and those which do not ("opfs-wl" requires + waitAsync()). + + One down-side would be for clients which, for whatever reason, want + to use both "opfs" and "opfs-wl" within the same session: because + both would go through the same Worker, any operations for one VFS + would, while they're being processed on this side of the proxy, + effectively block the other VFS from doing anything, potentially + deadlocking. This use case seems unlikely enough that it can + possibly be ruled out (or even reasonably flat-out prohibited by + the library). +*/ +//#/if + "use strict"; +const urlParams = new URL(globalThis.location.href).searchParams; +const vfsName = urlParams.get('vfs'); +if( !vfsName ){ + throw new Error("Expecting vfs=opfs|opfs-wl URL argument for this worker"); +} +/** + We use this to allow us to differentiate debug output from + multiple instances, e.g. multiple Workers to the "opfs" + VFS or both the "opfs" and "opfs-wl" VFSes. +*/ +const workerId = (Math.random() * 10000000) | 0; +const isWebLocker = 'opfs-wl'===urlParams.get('vfs'); const wPost = (type,...args)=>postMessage({type, payload:args}); const installAsyncProxy = function(){ const toss = function(...args){throw new Error(args.join(' '))}; @@ -66,6 +108,13 @@ const installAsyncProxy = function(){ */ const state = Object.create(null); + /* initS11n() is preprocessor-injected so that we have identical + copies in the synchronous and async halves. This side does not + load the SQLite library, so does not have access to that copy. */ +//#define opfs-async-proxy +//#include api/opfs-common-inline.c-pp.js +//#undef opfs-async-proxy + /** verbose: @@ -82,7 +131,7 @@ const installAsyncProxy = function(){ 2:console.log.bind(console) }; const logImpl = (level,...args)=>{ - if(state.verbose>level) loggers[level]("OPFS asyncer:",...args); + if(state.verbose>level) loggers[level](vfsName+' async-proxy',workerId+":",...args); }; const log = (...args)=>logImpl(2, ...args); const warn = (...args)=>logImpl(1, ...args); @@ -97,12 +146,13 @@ const installAsyncProxy = function(){ */ const __openFiles = Object.create(null); /** - __implicitLocks is a Set of sqlite3_file pointers (integers) which were - "auto-locked". i.e. those for which we obtained a sync access - handle without an explicit xLock() call. Such locks will be - released during db connection idle time, whereas a sync access - handle obtained via xLock(), or subsequently xLock()'d after - auto-acquisition, will not be released until xUnlock() is called. + __implicitLocks is a Set of sqlite3_file pointers (integers) + which were "auto-locked". i.e. those for which we necessarily + obtain a sync access handle without an explicit xLock() call + guarding access. Such locks will be released during + `waitLoop()`'s idle time, whereas a sync access handle obtained + via xLock(), or subsequently xLock()'d after auto-acquisition, + will not be released until xUnlock() is called. Maintenance reminder: if we relinquish auto-locks at the end of the operation which acquires them, we pay a massive performance @@ -271,10 +321,11 @@ const installAsyncProxy = function(){ In order to help alleviate cross-tab contention for a dabase, if an exception is thrown while acquiring the handle, this routine - will wait briefly and try again, up to some fixed number of - times. If acquisition still fails at that point it will give up - and propagate the exception. Client-level code will see that as - an I/O error. + will wait briefly and try again, up to `maxTries` of times. If + acquisition still fails at that point it will give up and + propagate the exception. Client-level code will see that either + as an I/O error or SQLITE_BUSY, depending on the exception and + the context. 2024-06-12: there is a rare race condition here which has been reported a single time: @@ -289,13 +340,31 @@ const installAsyncProxy = function(){ there's another race condition there). That's easy to say but creating a viable test for that condition has proven challenging so far. + + Interface quirk: if fh.xLock is falsy and the handle is acquired + then fh.fid is added to __implicitLocks(). If fh.xLock is truthy, + it is not added as an implicit lock. i.e. xLock() impls must set + fh.xLock immediately _before_ calling this and must arrange to + restore it to its previous value if this function throws. + + 2026-03-06: + + - baseWaitTime is the number of milliseconds to wait for the + first retry, increasing by one factor for each retry. It defaults + to (state.asyncIdleWaitTime*2). + + - maxTries is the number of attempt to make, each one spaced out + by one additional factor of the baseWaitTime (e.g. 300, then 600, + then 900, the 1200...). This MUST be an integer >0. + + Only the Web Locks impl should use the 3rd and 4th parameters. */ - const getSyncHandle = async (fh,opName)=>{ + const getSyncHandle = async (fh, opName, baseWaitTime, maxTries = 6)=>{ if(!fh.syncHandle){ const t = performance.now(); log("Acquiring sync handle for",fh.filenameAbs); - const maxTries = 6, - msBase = state.asyncIdleWaitTime * 2; + const msBase = baseWaitTime ?? (state.asyncIdleWaitTime * 2); + maxTries ??= 6; let i = 1, ms = msBase; for(; true; ms = msBase * ++i){ try { @@ -329,6 +398,9 @@ const installAsyncProxy = function(){ /** Stores the given value at state.sabOPView[state.opIds.rc] and then Atomics.notify()'s it. + + The opName is only used for logging and debugging - all result + codes are expected on the same state.sabOPView slot. */ const storeAndNotify = (opName, value)=>{ log(opName+"() => notify(",value,")"); @@ -458,24 +530,12 @@ const installAsyncProxy = function(){ await releaseImplicitLock(fh); storeAndNotify('xFileSize', rc); }, - xLock: async function(fid/*sqlite3_file pointer*/, - lockType/*SQLITE_LOCK_...*/){ - const fh = __openFiles[fid]; - let rc = 0; - const oldLockType = fh.xLock; - fh.xLock = lockType; - if( !fh.syncHandle ){ - try { - await getSyncHandle(fh,'xLock'); - __implicitLocks.delete(fid); - }catch(e){ - state.s11n.storeException(1,e); - rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK); - fh.xLock = oldLockType; - } - } - storeAndNotify('xLock',rc); - }, + /** + The first argument is semantically invalid here - it's an + address in the synchronous side's heap. We can do nothing with + it here except use it as a unique-per-file identifier. + i.e. a lookup key. + */ xOpen: async function(fid/*sqlite3_file pointer*/, filename, flags/*SQLITE_OPEN_...*/, opfsFlags/*OPFS_...*/){ @@ -533,7 +593,7 @@ const installAsyncProxy = function(){ rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ; } }catch(e){ - error("xRead() failed",e,fh); + //error("xRead() failed",e,fh); state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_READ); } @@ -560,29 +620,13 @@ const installAsyncProxy = function(){ affirmNotRO('xTruncate', fh); await (await getSyncHandle(fh,'xTruncate')).truncate(size); }catch(e){ - error("xTruncate():",e,fh); + //error("xTruncate():",e,fh); state.s11n.storeException(2,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_TRUNCATE); } await releaseImplicitLock(fh); storeAndNotify('xTruncate',rc); }, - xUnlock: async function(fid/*sqlite3_file pointer*/, - lockType/*SQLITE_LOCK_...*/){ - let rc = 0; - const fh = __openFiles[fid]; - if( fh.syncHandle - && state.sq3Codes.SQLITE_LOCK_NONE===lockType - /* Note that we do not differentiate between lock types in - this VFS. We're either locked or unlocked. */ ){ - try { await closeSyncHandle(fh) } - catch(e){ - state.s11n.storeException(1,e); - rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; - } - } - storeAndNotify('xUnlock',rc); - }, xWrite: async function(fid/*sqlite3_file pointer*/,n,offset64){ let rc; const fh = __openFiles[fid]; @@ -594,7 +638,7 @@ const installAsyncProxy = function(){ {at: Number(offset64)}) ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; }catch(e){ - error("xWrite():",e,fh); + //error("xWrite():",e,fh); state.s11n.storeException(1,e); rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_WRITE); } @@ -603,152 +647,274 @@ const installAsyncProxy = function(){ } }/*vfsAsyncImpls*/; - const initS11n = ()=>{ - /** - ACHTUNG: this code is 100% duplicated in the other half of this - proxy! The documentation is maintained in the "synchronous half". + if( isWebLocker ){ + /* We require separate xLock() and xUnlock() implementations for the + original and Web Lock implementations. The ones in this block + are for the WebLock impl. + + The Golden Rule for this impl is: if we have a web lock, we + must also hold the SAH. When "upgrading" an implicit lock to a + requested (explicit) lock, we must remove the SAH from the + __implicitLocks set. When we unlock, we release both the web + lock and the SAH. That invariant must be kept intact or race + conditions on SAHs will ensue. */ - if(state.s11n) return state.s11n; - const textDecoder = new TextDecoder(), - textEncoder = new TextEncoder('utf-8'), - viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), - viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - state.s11n = Object.create(null); - const TypeIds = Object.create(null); - TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; - TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; - TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; - TypeIds.string = { id: 4 }; - const getTypeId = (v)=>( - TypeIds[typeof v] - || toss("Maintenance required: this value type cannot be serialized.",v) - ); - const getTypeIdById = (tid)=>{ - switch(tid){ - case TypeIds.number.id: return TypeIds.number; - case TypeIds.bigint.id: return TypeIds.bigint; - case TypeIds.boolean.id: return TypeIds.boolean; - case TypeIds.string.id: return TypeIds.string; - default: toss("Invalid type ID:",tid); - } - }; - state.s11n.deserialize = function(clear=false){ - const argc = viewU8[0]; - const rc = argc ? [] : null; - if(argc){ - const typeIds = []; - let offset = 1, i, n, v; - for(i = 0; i < argc; ++i, ++offset){ - typeIds.push(getTypeIdById(viewU8[offset])); + /** Registry of active Web Locks: fid -> { mode, resolveRelease } */ + const __activeWebLocks = Object.create(null); + + vfsAsyncImpls.xLock = async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/, + isFromUnlock/*only if called from this.xUnlock()*/){ + const whichOp = isFromUnlock ? 'xUnlock' : 'xLock'; + const fh = __openFiles[fid]; + //error("xLock()",fid, lockType, isFromUnlock, fh); + const requestedMode = (lockType >= state.sq3Codes.SQLITE_LOCK_RESERVED) + ? 'exclusive' : 'shared'; + const existing = __activeWebLocks[fid]; + if( existing ){ + if( existing.mode === requestedMode + || (existing.mode === 'exclusive' + && requestedMode === 'shared') ) { + fh.xLock = lockType; + storeAndNotify(whichOp, 0); + /* Don't do this: existing.mode = requestedMode; + + Paraphrased from advice given by a consulting developer: + + If you hold an exclusive lock and SQLite requests shared, + you should keep exiting.mode as exclusive in because the + underlying Web Lock is still exclusive. Changing it to + shared would trick xLock into thinking it needs to + perform a release/re-acquire dance if an exclusive is + later requested. + */ + return 0 /* Already held at required or higher level */; } - for(i = 0; i < argc; ++i){ - const t = typeIds[i]; - if(t.getter){ - v = viewDV[t.getter](offset, state.littleEndian); - offset += t.size; - }else{/*String*/ - n = viewDV.getInt32(offset, state.littleEndian); - offset += 4; - v = textDecoder.decode(viewU8.slice(offset, offset+n)); - offset += n; + /* + Upgrade path: we must release shared and acquire exclusive. + This transition is NOT atomic in Web Locks API. + + It _effectively_ is atomic if we don't call + closeSyncHandle(fh), as no other worker can lock that until + we let it go. But we can't do that without eventually + leading to deadly embrace situations, so we don't do that. + (That's not a hypothetical, it has happened.) + */ + await closeSyncHandle(fh); + existing.resolveRelease(); + delete __activeWebLocks[fid]; + } + + const lockName = "sqlite3-vfs-opfs:" + fh.filenameAbs; + const oldLockType = fh.xLock; + return new Promise((resolveWaitLoop) => { + //log("xLock() initial promise entered..."); + navigator.locks.request(lockName, { mode: requestedMode }, async (lock) => { + //log("xLock() Web Lock entered.", fh); + __implicitLocks.delete(fid); + let rc = 0; + try{ + fh.xLock = lockType/*must be set before getSyncHandle() is called!*/; + await getSyncHandle(fh, 'xLock', state.asyncIdleWaitTime, 5); + }catch(e){ + fh.xLock = oldLockType; + state.s11n.storeException(1, e); + rc = GetSyncHandleError.convertRc(e, state.sq3Codes.SQLITE_BUSY); } - rc.push(v); - } + const releasePromise = rc + ? undefined + : new Promise((resolveRelease) => { + __activeWebLocks[fid] = { mode: requestedMode, resolveRelease }; + }); + storeAndNotify(whichOp, rc) /* unblock the C side */; + resolveWaitLoop(0) /* unblock waitLoop() */; + await releasePromise /* hold the lock until xUnlock */; + }); + }); + }; + + /** Internal helper for the opfs-wl xUnlock() */ + const wlCloseHandle = async(fh)=>{ + let rc = 0; + try{ + /* For the record, we've never once seen closeSyncHandle() + throw, nor should it because destructors do not throw. */ + await closeSyncHandle(fh); + }catch(e){ + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; } - if(clear) viewU8[0] = 0; - //log("deserialize:",argc, rc); return rc; }; - state.s11n.serialize = function(...args){ - if(args.length){ - //log("serialize():",args); - const typeIds = []; - let i = 0, offset = 1; - viewU8[0] = args.length & 0xff /* header = # of args */; - for(; i < args.length; ++i, ++offset){ - /* Write the TypeIds.id value into the next args.length - bytes. */ - typeIds.push(getTypeId(args[i])); - viewU8[offset] = typeIds[i].id; - } - for(i = 0; i < args.length; ++i) { - /* Deserialize the following bytes based on their - corresponding TypeIds.id from the header. */ - const t = typeIds[i]; - if(t.setter){ - viewDV[t.setter](offset, args[i], state.littleEndian); - offset += t.size; - }else{/*String*/ - const s = textEncoder.encode(args[i]); - viewDV.setInt32(offset, s.byteLength, state.littleEndian); - offset += 4; - viewU8.set(s, offset); - offset += s.byteLength; - } + + vfsAsyncImpls.xUnlock = async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/){ + const fh = __openFiles[fid]; + const existing = __activeWebLocks[fid]; + if( !existing ){ + const rc = await wlCloseHandle(fh); + storeAndNotify('xUnlock', rc); + return rc; + } + //log("xUnlock()",fid, lockType, fh); + let rc = 0; + if( lockType === state.sq3Codes.SQLITE_LOCK_NONE ){ + /* SQLite usually unlocks all the way to NONE */ + rc = await wlCloseHandle(fh); + existing.resolveRelease(); + delete __activeWebLocks[fid]; + fh.xLock = lockType; + }else if( lockType === state.sq3Codes.SQLITE_LOCK_SHARED + && existing.mode === 'exclusive' ){ + /* downgrade EXCLUSIVE -> SHARED */ + rc = await wlCloseHandle(fh); + if( 0===rc ){ + fh.xLock = lockType; + existing.resolveRelease(); + delete __activeWebLocks[fid]; + return vfsAsyncImpls.xLock(fid, lockType, true); } - //log("serialize() result:",viewU8.slice(0,offset)); }else{ - viewU8[0] = 0; + /* ??? */ + error("xUnlock() unhandled condition", fh); + } + storeAndNotify('xUnlock', rc); + return 0; + } + + }else{ + /* Original/"legacy" xLock() and xUnlock() */ + + vfsAsyncImpls.xLock = async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/){ + const fh = __openFiles[fid]; + let rc = 0; + const oldLockType = fh.xLock; + fh.xLock = lockType; + if( !fh.syncHandle ){ + try { + await getSyncHandle(fh,'xLock'); + __implicitLocks.delete(fid); + }catch(e){ + state.s11n.storeException(1,e); + rc = GetSyncHandleError.convertRc(e,state.sq3Codes.SQLITE_IOERR_LOCK); + fh.xLock = oldLockType; + } } + storeAndNotify('xLock',rc); }; - state.s11n.storeException = state.asyncS11nExceptions - ? ((priority,e)=>{ - if(priority<=state.asyncS11nExceptions){ - state.s11n.serialize([e.name,': ',e.message].join("")); + vfsAsyncImpls.xUnlock = async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/){ + let rc = 0; + const fh = __openFiles[fid]; + if( fh.syncHandle + && state.sq3Codes.SQLITE_LOCK_NONE===lockType + /* Note that we do not differentiate between lock types in + this VFS. We're either locked or unlocked. */ ){ + try { await closeSyncHandle(fh) } + catch(e){ + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; } - }) - : ()=>{}; + } + storeAndNotify('xUnlock',rc); + } - return state.s11n; - }/*initS11n()*/; + }/*xLock() and xUnlock() impls*/ const waitLoop = async function f(){ - const opHandlers = Object.create(null); - for(let k of Object.keys(state.opIds)){ - const vi = vfsAsyncImpls[k]; - if(!vi) continue; - const o = Object.create(null); - opHandlers[state.opIds[k]] = o; - o.key = k; - o.f = vi; + if( !f.inited ){ + f.inited = true; + f.opHandlers = Object.create(null); + for(let k of Object.keys(state.opIds)){ + const vi = vfsAsyncImpls[k]; + if(!vi) continue; + const o = Object.create(null); + f.opHandlers[state.opIds[k]] = o; + o.key = k; + o.f = vi; + } } + const opIds = state.opIds; + const opView = state.sabOPView; + const slotWhichOp = opIds.whichOp; + const idleWaitTime = state.asyncIdleWaitTime; + const hasWaitAsync = !!Atomics.waitAsync; +//#if 0 + error("waitLoop init: isWebLocker",isWebLocker, + "idleWaitTime",idleWaitTime, + "hasWaitAsync",hasWaitAsync); +//#/if while(!flagAsyncShutdown){ try { - if('not-equal'!==Atomics.wait( - state.sabOPView, state.opIds.whichOp, 0, state.asyncIdleWaitTime - )){ - /* Maintenance note: we compare against 'not-equal' because - - https://github.com/tomayac/sqlite-wasm/issues/12 - - is reporting that this occasionally, under high loads, - returns 'ok', which leads to the whichOp being 0 (which - isn't a valid operation ID and leads to an exception, - along with a corresponding ugly console log - message). Unfortunately, the conditions for that cannot - be reliably reproduced. The only place in our code which - writes a 0 to the state.opIds.whichOp SharedArrayBuffer - index is a few lines down from here, and that instance - is required in order for clear communication between - the sync half of this proxy and this half. + let opId; + if( hasWaitAsync ){ + opId = Atomics.load(opView, slotWhichOp); + if( 0===opId ){ + const rv = Atomics.waitAsync(opView, slotWhichOp, 0, + idleWaitTime); + if( rv.async ) await rv.value; + await releaseImplicitLocks(); + continue; + } + }else{ + /** + For browsers without Atomics.waitAsync(), we require + the legacy implementation. Browser versions where + waitAsync() arrived: + + Chrome: 90 (2021-04-13) + Firefox: 145 (2025-11-11) + Safari: 16.4 (2023-03-27) + + The "opfs" VFS was not born until Chrome was somewhere in + the v104-108 range (Summer/Autumn 2022) and did not work + with Safari < v17 (2023-09-18) due to a WebKit bug which + restricted OPFS access from sub-Workers. + + The waitAsync() counterpart of this block can be used by + both "opfs" and "opfs-wl", whereas this block can only be + used by "opfs". Performance comparisons between the two + in high-contention tests have been indecisive. */ - await releaseImplicitLocks(); - continue; + if('not-equal'!==Atomics.wait( + state.sabOPView, slotWhichOp, 0, state.asyncIdleWaitTime + )){ + /* Maintenance note: we compare against 'not-equal' because + + https://github.com/tomayac/sqlite-wasm/issues/12 + + is reporting that this occasionally, under high loads, + returns 'ok', which leads to the whichOp being 0 (which + isn't a valid operation ID and leads to an exception, + along with a corresponding ugly console log + message). Unfortunately, the conditions for that cannot + be reliably reproduced. The only place in our code which + writes a 0 to the state.opIds.whichOp SharedArrayBuffer + index is a few lines down from here, and that instance + is required in order for clear communication between + the sync half of this proxy and this half. + + Much later (2026-03-07): that phenomenon is apparently + called a spurious wakeup. + */ + await releaseImplicitLocks(); + continue; + } + opId = Atomics.load(state.sabOPView, slotWhichOp); } - const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); - Atomics.store(state.sabOPView, state.opIds.whichOp, 0); - const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); + Atomics.store(opView, slotWhichOp, 0); + const hnd = f.opHandlers[opId]?.f ?? toss("No waitLoop handler for whichOp #",opId); const args = state.s11n.deserialize( true /* clear s11n to keep the caller from confusing this with an exception string written by the upcoming operation */ ) || []; - //warn("waitLoop() whichOp =",opId, hnd, args); - if(hnd.f) await hnd.f(...args); - else error("Missing callback for opId",opId); + //error("waitLoop() whichOp =",opId, f.opHandlers[opId].key, args); + await hnd(...args); }catch(e){ - error('in waitLoop():',e); + error('in waitLoop():', e); } } }; @@ -756,6 +922,7 @@ const installAsyncProxy = function(){ navigator.storage.getDirectory().then(function(d){ state.rootDir = d; globalThis.onmessage = function({data}){ + //log(globalThis.location.href,"onmessage()",data); switch(data.type){ case 'opfs-async-init':{ /* Receive shared state from synchronous partner */ @@ -771,6 +938,7 @@ const installAsyncProxy = function(){ } }); initS11n(); + //warn("verbosity =",opt.verbose, state.verbose); log("init state",state); wPost('opfs-async-inited'); waitLoop(); @@ -782,22 +950,27 @@ const installAsyncProxy = function(){ flagAsyncShutdown = false; waitLoop(); } - break; + break; } }; wPost('opfs-async-loaded'); }).catch((e)=>error("error initializing OPFS asyncer:",e)); }/*installAsyncProxy()*/; -if(!globalThis.SharedArrayBuffer){ +if(globalThis.window === globalThis){ + wPost('opfs-unavailable', + "This code cannot run from the main thread.", + "Load it as a Worker from a separate Worker."); +}else if(!globalThis.SharedArrayBuffer){ wPost('opfs-unavailable', "Missing SharedArrayBuffer API.", "The server must emit the COOP/COEP response headers to enable that."); }else if(!globalThis.Atomics){ wPost('opfs-unavailable', "Missing Atomics API.", "The server must emit the COOP/COEP response headers to enable that."); +}else if(isWebLocker && !globalThis.Atomics.waitAsync){ + wPost('opfs-unavailable',"Missing required Atomics.waitSync() for "+vfsName); }else if(!globalThis.FileSystemHandle || !globalThis.FileSystemDirectoryHandle || - !globalThis.FileSystemFileHandle || - !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || + !globalThis.FileSystemFileHandle?.prototype?.createSyncAccessHandle || !navigator?.storage?.getDirectory){ wPost('opfs-unavailable',"Missing required OPFS APIs."); }else{ diff --git a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js new file mode 100644 index 000000000..0db303bc4 --- /dev/null +++ b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js @@ -0,0 +1,2102 @@ +/* + 2025-11-21 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file houses the "kvvfs" pieces of the SQLite3 JS API. Most of + kvvfs is implemented in src/os_kv.c and exposed/extended for use + here via sqlite3-wasm.c. + + Main project home page: https://sqlite.org + + Documentation home page: https://sqlite.org/wasm +*/ +//#if omit-kvvfs +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + /* These are JS plumbing, not part of the public API */ + delete sqlite3.capi.sqlite3_kvvfs_methods; + delete sqlite3.capi.KVVfsFile; +} +//#else +//#@ policy error +//#savepoint begin +//#define kvvfs-v2-added-in "3.52.0" + +/** + kvvfs - the Key/Value VFS - is an SQLite3 VFS which delegates + storage of its pages and metadata to a key-value store. + + It was conceived in order to support JS's localStorage and + sessionStorage objects. Its native implementation uses files as + key/value storage (one file per record) but the JS implementation + replaces a few methods so that it can use the aforementioned + objects as storage. + + It uses a bespoke ASCII encoding to store each db page as a + separate record and stores some metadata, like the db's unencoded + size and its journal, as individual records. + + kvvfs is significantly less efficient than a plain in-memory db but + it also, as a side effect of its design, offers a JSON-friendly + interchange format for exporting and importing databases. + + kvvfs is _not_ designed for heavy db loads. It is relatively + malloc()-heavy, having to de/allocate frequently, and it + spends much of its time converting the raw db pages into and out of + an ASCII encoding. + + But it _does_ work and is "performant enough" for db work of the + scale of a db which will fit within sessionStorage or localStorage + (just 2-3mb). + + "Version 2" extends it to support using Storage-like objects as + backing storage, Storage being the JS class which localStorage and + sessionStorage both derive from. This essentially moves the backing + store from whatever localStorage and sessionStorage use to an + in-memory object. + + This effort is primarily a stepping stone towards eliminating, if + it proves possible, the POSIX I/O API dependencies in SQLite's WASM + builds. That is: if this VFS works properly, it can be set as the + default VFS and we can eliminate the "unix" VFS from the JS/WASM + builds (as opposed to server-wise/WASI builds). That still, as of + 2025-11-23, a ways away, but it's the main driver for version 2 of + kvvfs. + + Version 2 remains compatible with version 1 databases and always + writes localStorage/sessionStorage metadata in the v1 format, so + such dbs can be manipulated freely by either version. For transient + storage objects (new in version 2), the format of its record keys + is simpified, requiring less space than v1 keys by eliding + redundant (in this context) info from the keys. + + Another benefit of v2 is its ability to export dbs into a + JSON-friendly (but not human-friendly) format. + + A potential, as-yet-unproven, benefit, would be the ability to plug + arbitrary Storage-compatible objects in so that clients could, + e.g. asynchronously post updates to db pages to some back-end for + backups. +*/ +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + if( sqlite3.config.disable?.vfs?.kvvfs ){ + return; + } + 'use strict'; + const capi = sqlite3.capi, + sqlite3_kvvfs_methods = capi.sqlite3_kvvfs_methods, + KVVfsFile = capi.KVVfsFile, + pKvvfs = sqlite3.capi.sqlite3_vfs_find("kvvfs") + + /* These are JS plumbing, not part of the public API */ + delete capi.sqlite3_kvvfs_methods; + delete capi.KVVfsFile; + + if( !pKvvfs ) return /* nothing to do */; + if( 0 ){ + /* This working would be our proverbial holy grail, in that it + would allow us to eliminate the current default VFS, which + relies on POSIX I/O APIs. Eliminating that dependency would get + us one giant step closer to creating wasi-sdk builds. */ + capi.sqlite3_vfs_register(pKvvfs, 1); + } + + const util = sqlite3.util, + wasm = sqlite3.wasm, + toss3 = util.toss3, + hop = (o,k)=>Object.prototype.hasOwnProperty.call(o,k); + + const kvvfsMethods = new sqlite3_kvvfs_methods( + /* Wraps the static sqlite3_api_methods singleton */ + wasm.exports.sqlite3__wasm_kvvfs_methods() + ); + util.assert( 32<=kvvfsMethods.$nKeySize, "unexpected kvvfsMethods.$nKeySize: "+kvvfsMethods.$nKeySize); + + /** + Most of the VFS-internal state. + */ + const cache = Object.assign(Object.create(null),{ + /** Regex matching journal file names. */ + rxJournalSuffix: /-journal$/, + /** Frequently-used C-string. */ + zKeyJrnl: wasm.allocCString("jrnl"), + /** Frequently-used C-string. */ + zKeySz: wasm.allocCString("sz"), + /** + The maximum size of a kvvfs record key. It is historically only + 32, a limitation currently retained only because it's convenient to + do so (the underlying code has outgrown the need for the artifically + low limit). + + We cache this value here because the end of this init code will + dispose of kvvfsMethods, invalidating it. + */ + keySize: kvvfsMethods.$nKeySize, + /** + WASM heap memory buffers to optimize out some frequent + allocations. + */ + buffer: Object.assign(Object.create(null),{ + /** + The size of each buffer in this.pool. + + kvvfsMethods.$nBufferSize is slightly larger than the output + space needed for a kvvfs-encoded 64kb db page in a worse-cast + encoding (128kb). It is not suitable for arbitrary buffer + use, only page de/encoding. + */ + n: kvvfsMethods.$nBufferSize, + /** + Map of buffer ids to wasm.alloc()'d pointers of size + this.n. (Re)used by various internals. + + Buffer ids 0 and 1 are used in the API internals. Other + names are used in higher-level APIs. + + See memBuffer() and memBufferFree(). + */ + pool: Object.create(null) + }) + }); + + /** + Returns a (cached) wasm.alloc()'d buffer of cache.buffer.n size, + throwing on OOM. + + We leak this one-time alloc because we've no better option. + sqlite3_vfs does not have a finalizer, so we've no place to hook + in the cleanup. We "could" extend sqlite3_shutdown() to have a + cleanup list for stuff like this but that function is never + used in JS, so it's hardly worth it. + */ + cache.memBuffer = (id=0)=>cache.buffer.pool[id] ??= wasm.alloc(cache.buffer.n); + + /** Frees the buffer with the given id. */ + cache.memBufferFree = (id)=>{ + const b = cache.buffer.pool[id]; + if( b ){ + wasm.dealloc(b); + delete cache.buffer.pool[id]; + } + }; + + const noop = ()=>{}; + const debug = sqlite3.__isUnderTest + ? (...args)=>sqlite3.config.debug?.("kvvfs:", ...args) + : noop; + const warn = (...args)=>sqlite3.config.warn?.("kvvfs:", ...args); + const error = (...args)=>sqlite3.config.error?.("kvvfs:", ...args); + + /** + Implementation of JS's Storage interface for use as backing store + of the kvvfs. Storage is a native class and its constructor + cannot be legally called from JS, making it impossible to + directly subclass Storage. This class implements (only) the + Storage interface, to make it a drop-in replacement for + localStorage/sessionStorage. (Any behavioral discrepancies are to + be considered bugs.) + + This impl simply proxies a plain, prototype-less Object, suitable + for JSON-ing. + + Design note: Storage has a bit of an odd iteration-related + interface as does not (AFAIK) specify specific behavior regarding + modification during traversal. Because of that, this class does + some seemingly unnecessary things with its #keys member, deleting + and recreating it whenever a property index might be invalidated. + */ + class KVVfsStorage { + #map = Object.create(null); + #keys = null; + #size = 0; + + constructor(){ + this.clear(); + } + + #getKeys(){ + return this.#keys ??= Object.keys(this.#map); + } + + key(n){ + if(n < 0 || n >= this.#size) return null; + return this.#getKeys()[n]; + } + + getItem(k){ + return this.#map[k] ?? null; + } + + setItem(k,v){ + if( !(k in this.#map) ){ + ++this.#size; + this.#keys = null; + } + this.#map[k] = ''+v; + } + + removeItem(k){ + if( k in this.#map ){ + delete this.#map[k]; + --this.#size; + this.#keys = null; + } + } + + clear(){ + this.#map = Object.create(null); + this.#keys = null; + this.#size = 0; + } + + get length() { + return this.#size; + } + }/*KVVfsStorage*/; + + /** True if v is the name of one of the special persistant Storage + objects. */ + const kvvfsIsPersistentName = (v)=>'local'===v || 'session'===v; + + /** + Keys in kvvfs have a prefix of "kvvfs-NAME-", where NAME is the + db name. This key is redundant in JS but it's how kvvfs works (it + saves each key to a separate file, so needs a distinct namespace + per data source name). We retain this prefix in 'local' and + 'session' storage for backwards compatibility and so that they + can co-exist with client data in their storage, but we elide them + from "v2" storage, where they're superfluous. + */ + const kvvfsKeyPrefix = (v)=>kvvfsIsPersistentName(v) ? 'kvvfs-'+v+'-' : ''; + + /** + Throws if storage name n (JS string) is not valid for use as a + storage name. Much of this goes back to kvvfs having a fixed + buffer size for its keys, and the storage name needing to be + encoded in the keys for local/session storage. + + The second argument must only be true when called from xOpen() - + it makes names with a "-journal" suffix legal. + */ + const validateStorageName = function(n,mayBeJournal=false){ + if( kvvfsIsPersistentName(n) ) return; + const len = (new Blob([n])).size/*byte length*/; + if( !len ) toss3(capi.SQLITE_MISUSE, "Empty name is not permitted."); + let maxLen = cache.keySize - 1; + if( cache.rxJournalSuffix.test(n) ){ + if( !mayBeJournal ){ + toss3(capi.SQLITE_MISUSE, + "Storage names may not have a '-journal' suffix."); + } + }else if( ['-wal','-shm'].filter(v=>n.endsWith(v)).length ){ + toss3(capi.SQLITE_MISUSE, + "Storage names may not have a -wal or -shm suffix."); + }else{ + maxLen -= 8 /* so we have space for a matching "-journal" suffix */; + } + if( len > maxLen ){ + toss3(capi.SQLITE_RANGE, "Storage name is too long. Limit =", maxLen); + } + let i; + for( i = 0; i < len; ++i ){ + const ch = n.codePointAt(i); + if( ch<32 ){ + toss3(capi.SQLITE_RANGE, + "Illegal character ("+ch+"d) in storage name:",n); + } + } + }; + + /** + Create a new instance of the objects which go into + cache.storagePool, with a refcount of 1. If passed a Storage-like + object as its second argument, it is used for the storage, + otherwise it creates a new KVVfsStorage object. + */ + const newStorageObj = (name,storage=undefined)=>Object.assign(Object.create(null),{ + /** + JS string value of this KVVfsFile::$zClass. i.e. the storage's + name. + */ + jzClass: name, + /** + Refcount. This keeps dbs and journals pointing to the same + storage for the life of both and enables kvvfs to behave more + like a conventional filesystem (a stepping stone towards + downstream API goals). Managed by xOpen() and xClose(). + */ + refc: 1, + /** + If true, this storage will be removed by xClose() or + sqlite3_js_kvvfs_unlink() when refc reaches 0. The others will + persist when refc==0, to give the illusion of real back-end + storage. Managed by xOpen() and sqlite3_js_kvvfs_reserve(). By + default this is false but the delete-on-close=1 flag can be + used to set this to true. + */ + deleteAtRefc0: false, + /** + The backing store. Must implement the Storage interface. + */ + storage: storage || new KVVfsStorage, + /** + The storage prefix used for kvvfs keys. It is + "kvvfs-STORAGENAME-" for local/session storage and an empty + string for other storage. local/session storage must use the + long form (A) for backwards compatibility and (B) so that kvvfs + can coexist with non-db client data in those backends. Neither + (A) nor (B) are concerns for KVVfsStorage objects. + + This prefix mirrors the one generated by os_kv.c's + kvrecordMakeKey() and must stay in sync with that one. + */ + keyPrefix: kvvfsKeyPrefix(name), + /** + KVVfsFile instances currently using this storage. Managed by + xOpen() and xClose(). + */ + files: [], + /** + If set, it's an array of objects with various event + callbacks. See sqlite3_js_kvvfs_listen(). When there are no + listeners, this member is set to undefined (instead of an empty + array) to allow us to more easily optimize out calls to + notifyListeners() for the common case of no listeners. + */ + listeners: undefined + }); + + /** + Public interface for kvvfs v2. The capi.sqlite3_js_kvvfs_...() + routines remain in place for v1. Some members of this class proxy + to those functions but use different default argument values in + some cases. + */ + const kvvfs = sqlite3.kvvfs = Object.create(null); + if( sqlite3.__isUnderTest ){ + /* For inspection via the dev tools console. */ + kvvfs.log = Object.assign(Object.create(null),{ + xOpen: false, + xClose: false, + xWrite: false, + xRead: false, + xSync: false, + xAccess: false, + xFileControl: false, + xRcrdRead: false, + xRcrdWrite: false, + xRcrdDelete: false, + }); + } + + /** + Deletes the cache.storagePool entries for store (a + cache.storagePool entry) and its db/journal counterpart. + */ + const deleteStorage = function(store){ + const other = cache.rxJournalSuffix.test(store.jzClass) + ? store.jzClass.replace(cache.rxJournalSuffix,'') + : store.jzClass+'-journal'; + kvvfs?.log?.xClose + && debug("cleaning up storage handles [", store.jzClass, other,"]",store); + delete cache.storagePool[store.jzClass]; + delete cache.storagePool[other]; + if( !sqlite3.__isUnderTest ){ + /* In test runs, leave these for inspection. If we delete them here, + any prior dumps of them emitted via the console get cleared out + because the console shows live objects instead of call-time + static dumps. */ + delete store.storage; + delete store.refc; + } + }; + + /** + Add both store.jzClass and store.jzClass+"-journal" + to cache,storagePool. + */ + const installStorageAndJournal = (store)=> + cache.storagePool[store.jzClass] = + cache.storagePool[store.jzClass+'-journal'] = store; + + /** + The public name of the current thread's transient storage + object. A storage object with this name gets preinstalled. + */ + const nameOfThisThreadStorage = '.'; + + /** + Map of JS-stringified KVVfsFile::zClass names to + reference-counted Storage objects. These objects are created in + xOpen(). Their refcount is decremented in xClose(), and the + record is destroyed if the refcount reaches 0. We refcount so + that concurrent active xOpen()s on a given name, and within a + given thread, use the same storage object. + */ + cache.storagePool = Object.assign(Object.create(null),{ + /* Start off with mappings for well-known names. */ + [nameOfThisThreadStorage]: newStorageObj(nameOfThisThreadStorage) + }); + + if( globalThis.Storage ){ + /* If available, install local/session storage. */ + if( globalThis.localStorage instanceof globalThis.Storage ){ + cache.storagePool.local = newStorageObj('local', globalThis.localStorage); + } + if( globalThis.sessionStorage instanceof globalThis.Storage ){ + cache.storagePool.session = newStorageObj('session', globalThis.sessionStorage); + } + } + + cache.builtinStorageNames = Object.keys(cache.storagePool); + + const isBuiltinName = (n)=>cache.builtinStorageNames.indexOf(n)>-1; + + /* Add "-journal" twins for each cache.storagePool entry... */ + for(const k of Object.keys(cache.storagePool)){ + /* Journals in kvvfs are are stored as individual records within + their Storage-ish object, named "{storage.keyPrefix}jrnl". We + always map the db and its journal to the same Storage + object. */ + const orig = cache.storagePool[k]; + cache.storagePool[k+'-journal'] = orig; + } + + cache.setError = (e=undefined, dfltErrCode=capi.SQLITE_ERROR)=>{ + if( e ){ + cache.lastError = e; + return (e.resultCode | 0) || dfltErrCode; + } + delete cache.lastError; + return 0; + }; + + cache.popError = ()=>{ + const e = cache.lastError; + delete cache.lastError; + return e; + }; + + /** Exception handler for notifyListeners(). */ + const catchForNotify = (e)=>{ + warn("kvvfs.listener handler threw:",e); + }; + + const kvvfsDecode = wasm.exports.sqlite3__wasm_kvvfs_decode; + const kvvfsEncode = wasm.exports.sqlite3__wasm_kvvfs_encode; + + /** + Listener events and their argument(s) (via the callback(ev) + ev.data member): + + 'open': number of opened handles on this storage. + + 'close': number of opened handles on this storage. + + 'write': key, value + + 'delete': key + + 'sync': true if it's from xSync(), false if it's from + xFileControl(). + + For efficiency's sake, all calls to this function should + be in the form: + + store.listeners && notifyListeners(...); + + Failing to do so will trigger an exceptin in this function (which + will be ignored but may produce a console warning). + */ + const notifyListeners = async function(eventName,store,...args){ + try{ + //cache.rxPageNoSuffix ??= /(\d+)$/; + if( store.keyPrefix && args[0] ){ + args[0] = args[0].replace(store.keyPrefix,''); + } + let u8enc, z0, z1, wcache; + for(const ear of store.listeners){ + const ev = Object.create(null); + ev.storageName = store.jzClass; + ev.type = eventName; + const decodePages = ear.decodePages; + const f = ear.events[eventName]; + if( f ){ + if( !ear.includeJournal && args[0]==='jrnl' ){ + continue; + } + if( 'write'===eventName && ear.decodePages && +args[0]>0 ){ + /* Decode pages to Uint8Array, caching the result in + wcache in case we have more listeners. */ + ev.data = [args[0]]; + if( wcache?.[args[0]] ){ + ev.data[1] = wcache[args[0]]; + continue; + } + u8enc ??= new TextEncoder('utf-8'); + z0 ??= cache.memBuffer(10); + z1 ??= cache.memBuffer(11); + const u = u8enc.encode(args[1]); + const heap = wasm.heap8u(); + heap.set(u, Number(z0)); + heap[wasm.ptr.addn(z0, u.length)] = 0; + const rc = kvvfsDecode(z0, z1, cache.buffer.n); + if( rc>0 ){ + wcache ??= Object.create(null); + wcache[args[0]] + = ev.data[1] + = heap.slice(Number(z1), wasm.ptr.addn(z1,rc)); + }else{ + continue; + } + }else{ + ev.data = args.length + ? ((args.length===1) ? args[0] : args) + : undefined; + } + try{f(ev)?.catch?.(catchForNotify)} + catch(e){ + warn("notifyListeners [",store.jzClass,"]",eventName,e); + } + } + } + }catch(e){ + catchForNotify(e); + } + }/*notifyListeners()*/; + + /** + Returns the storage object mapped to the given string zClass + (C-string pointer or JS string). + */ + const storageForZClass = (zClass)=> + 'string'===typeof zClass + ? cache.storagePool[zClass] + : cache.storagePool[wasm.cstrToJs(zClass)]; + +//#if 0 + // fileForDb() works but we don't have a current need for it. + /** + Expects an (sqlite3*). Uses sqlite3_file_control() to extract its + (sqlite3_file*). On success it returns a new KVVfsFile instance + wrapping that pointer, which the caller must eventual call + dispose() on (which won't free the underlying pointer, just the + wrapper). Returns null if no handle is found (which would + indicate either that pDb is not using kvvfs or a severe bug in + its management). + */ + const fileForDb = function(pDb){ + const stack = wasm.pstack.pointer; + try{ + const pOut = wasm.pstack.allocPtr(); + return wasm.exports.sqlite3_file_control( + pDb, wasm.ptr.null, capi.SQLITE_FCNTL_FILE_POINTER, pOut + ) + ? null + : new KVVfsFile(wasm.peekPtr(pOut)); + }finally{ + wasm.pstack.restore(stack); + } + }; + + /** + Expects an object from the storagePool map. The $szPage and + $szDb members of each store.files entry is set to -1 in an attempt + to trigger those values to reload. + */ + const alertFilesToReload = (store)=>{ + try{ + for( const f of store.files ){ + // FIXME: we need to use one of the C APIs for this, maybe an + // fcntl. + f.$szPage = -1; + f.$szDb = -1n + } + }catch(e){ + error("alertFilesToReload()",store,e); + throw e; + } + }; +//#/if + + const kvvfsMakeKey = wasm.exports.sqlite3__wasm_kvvfsMakeKey; + /** + Returns a C string from kvvfsMakeKey() OR returns zKey. In the + former case the memory is static, so must be copied before a + second call. zKey MUST be a pointer passed to a VFS/file method, + to allow us to avoid an alloc and/or an snprintf(). It requires + C-string arguments for zClass and zKey. zClass may be NULL but + zKey may not. + */ + const zKeyForStorage = (store, zClass, zKey)=>{ + //debug("zKeyForStorage(",store, wasm.cstrToJs(zClass), wasm.cstrToJs(zKey)); + return (zClass && store.keyPrefix) ? kvvfsMakeKey(zClass, zKey) : zKey; + }; + + const jsKeyForStorage = (store,zClass,zKey)=> + wasm.cstrToJs(zKeyForStorage(store, zClass, zKey)); + + const storageGetDbSize = (store)=>+store.storage.getItem(store.keyPrefix + "sz"); + + /** + sqlite3_file pointers => objects, each of which has: + + .file = KVVfsFile instance + + .jzClass = JS-string form of f.$zClass + + .storage = Storage object. It is shared between a db and its + journal. + */ + const pFileHandles = new Map(); + + /** + Original WASM functions for methods we partially override. + */ + const originalMethods = { + vfs: Object.create(null), + ioDb: Object.create(null), + ioJrnl: Object.create(null) + }; + + /** Returns the appropriate originalMethods[X] instance for the + given a KVVfsFile instance. */ + const originalIoMethods = (kvvfsFile)=> + originalMethods[kvvfsFile.$isJournal ? 'ioJrnl' : 'ioDb']; + + const pVfs = new capi.sqlite3_vfs(kvvfsMethods.$pVfs); + const pIoDb = new capi.sqlite3_io_methods(kvvfsMethods.$pIoDb); + const pIoJrnl = new capi.sqlite3_io_methods(kvvfsMethods.$pIoJrnl); + const recordHandler = + Object.create(null)/** helper for some vfs + routines. Populated later. */; + const kvvfsInternal = Object.assign(Object.create(null),{ + pFileHandles, + cache, + storageForZClass, + KVVfsStorage, + /** + BUG: changing to a page size other than the default, + then vacuuming, corrupts the db. As a workaround, + until this is resolved, we forcibly disable + (pragma page_size=...) changes. + */ + disablePageSizeChange: true + }); + if( kvvfs.log ){ + // this is a test build + kvvfs.internal = kvvfsInternal; + } + + /** + Implementations for members of the object referred to by + sqlite3__wasm_kvvfs_methods(). We swap out some native + implementations with these so that we can use JS Storage for + their backing store. + */ + const methodOverrides = { + + /** + sqlite3_kvvfs_methods's member methods. These perform the + fetching, setting, and removal of storage keys on behalf of + kvvfs. In the native impl these write each db page to a + separate file. This impl stores each db page as a single + record in a Storage object which is mapped to zClass. + + A db's size is stored in a record named kvvfs[-storagename]-sz + and the journal is stored in kvvfs[-storagename]-jrnl. The + [-storagename] part is a remnant of the native impl (so that + it has unique filenames per db) and is only used for + localStorage and sessionStorage. We elide that part (to save + space) from other storage objects but retain it on those two + to avoid invalidating pre-version-2 session/localStorage dbs. + + The interface docs for these methods are in src/os_kv.c's + kvrecordRead(), kvrecordWrite(), and kvrecordDelete(). + */ + recordHandler: { + xRcrdRead: (zClass, zKey, zBuf, nBuf)=>{ + try{ + const jzClass = wasm.cstrToJs(zClass); + const store = storageForZClass(jzClass); + if( !store ) return -1; + const jXKey = jsKeyForStorage(store, zClass, zKey); + kvvfs?.log?.xRcrdRead && warn("xRcrdRead", jzClass, jXKey, nBuf, store ); + const jV = store.storage.getItem(jXKey); + if(null===jV) return -1; + const nV = jV.length /* We are relying 100% on v being + ** ASCII so that jV.length is equal + ** to the C-string's byte length. */; + if( 0 ){ + debug("xRcrdRead", jXKey, store, jV); + } + if(nBuf<=0) return nV; + else if(1===nBuf){ + wasm.poke(zBuf, 0); + return nV; + } + if( nBuf+1<nV ){ + toss3(capi.SQLITE_RANGE, + "xRcrdRead()",jzClass,jXKey, + "input buffer is too small: need", + nV,"but have",nBuf); + } + if( 0 ){ + debug("xRcrdRead", nBuf, zClass, wasm.cstrToJs(zClass), + wasm.cstrToJs(zKey), nV, jV, store); + } + const zV = cache.memBuffer(0); + //if( !zV ) return -3 /*OOM*/; + const heap = wasm.heap8(); + let i; + for(i = 0; i < nV; ++i){ + heap[wasm.ptr.add(zV,i)] = jV.codePointAt(i) & 0xFF; + } + heap.copyWithin( + Number(zBuf), Number(zV), wasm.ptr.addn(zV, i) + ); + heap[wasm.ptr.add(zBuf, nV)] = 0; + return nBuf; + }catch(e){ + error("kvrecordRead()",e); + cache.setError(e); + return -2; + } + }, + + xRcrdWrite: (zClass, zKey, zData)=>{ + try { + const store = storageForZClass(zClass); + const jxKey = jsKeyForStorage(store, zClass, zKey); + const jData = wasm.cstrToJs(zData); + kvvfs?.log?.xRcrdWrite && warn("xRcrdWrite",jxKey, store); + store.storage.setItem(jxKey, jData); + store.listeners && notifyListeners('write', store, jxKey, jData); + return 0; + }catch(e){ + error("kvrecordWrite()",e); + return cache.setError(e, capi.SQLITE_IOERR); + } + }, + + xRcrdDelete: (zClass, zKey)=>{ + try { + const store = storageForZClass(zClass); + const jxKey = jsKeyForStorage(store, zClass, zKey); + kvvfs?.log?.xRcrdDelete && warn("xRcrdDelete",jxKey, store); + store.storage.removeItem(jxKey); + store.listeners && notifyListeners('delete', store, jxKey); + return 0; + }catch(e){ + error("kvrecordDelete()",e); + return cache.setError(e, capi.SQLITE_IOERR); + } + } + }/*recordHandler*/, + + /** + Override certain operations of the underlying sqlite3_vfs and + the two sqlite3_io_methods instances so that we can tie + Storage objects to db names. + */ + vfs:{ + /* sqlite3_kvvfs_methods::pVfs's methods */ + xOpen: function(pProtoVfs,zName,pProtoFile,flags,pOutFlags){ + cache.popError(); + let zToFree /* alloc()'d memory for temp db name */; + if( 0 ){ + /* tester1.js makes it a lot further if we do this. */ + flags |= capi.SQLITE_OPEN_CREATE; + } + try{ + if( !zName ){ + zToFree = wasm.allocCString(""+pProtoFile+"." + +(Math.random() * 100000 | 0)); + zName = zToFree; + } + const jzClass = wasm.cstrToJs(zName); + kvvfs?.log?.xOpen && debug("xOpen",jzClass,"flags =",flags); + validateStorageName(jzClass, true); + if( (flags & (capi.SQLITE_OPEN_MAIN_DB + | capi.SQLITE_OPEN_TEMP_DB + | capi.SQLITE_OPEN_TRANSIENT_DB)) + && cache.rxJournalSuffix.test(jzClass) ){ + toss3(capi.SQLITE_ERROR, + "DB files may not have a '-journal' suffix."); + } + let s = storageForZClass(jzClass); + if( !s && !(flags & capi.SQLITE_OPEN_CREATE) ){ + toss3(capi.SQLITE_ERROR, "Storage not found:", jzClass); + } + const rc = originalMethods.vfs.xOpen(pProtoVfs, zName, pProtoFile, + flags, pOutFlags); + if( rc ) return rc; + let deleteAt0 = !!(capi.SQLITE_OPEN_DELETEONCLOSE & flags); + if(wasm.isPtr(arguments[1]/*original zName*/)){ + if(capi.sqlite3_uri_boolean(zName, "delete-on-close", 0)){ + deleteAt0 = true; + } + } + const f = new KVVfsFile(pProtoFile); + util.assert(f.$zClass, "Missing f.$zClass"); + f.addOnDispose(zToFree); + zToFree = undefined; + //debug("xOpen", jzClass, s); + if( s ){ + ++s.refc; + //no if( true===deleteAt0 ) s.deleteAtRefc0 = true; + s.files.push(f); + wasm.poke32(pOutFlags, flags); + }else{ + wasm.poke32(pOutFlags, flags | capi.SQLITE_OPEN_CREATE); + util.assert( !f.$isJournal, "Opening a journal before its db? "+jzClass ); + /* Map both zName and zName-journal to the same storage. */ + const nm = jzClass.replace(cache.rxJournalSuffix,''); + s = newStorageObj(nm); + installStorageAndJournal(s); + s.files.push(f); + s.deleteAtRefc0 = deleteAt0; + kvvfs?.log?.xOpen + && debug("xOpen installed storage handle [",nm, nm+"-journal","]", s); + } + pFileHandles.set(pProtoFile, {store: s, file: f, jzClass}); + s.listeners && notifyListeners('open', s, s.files.length); + return 0; + }catch(e){ + warn("xOpen:",e); + return cache.setError(e); + }finally{ + zToFree && wasm.dealloc(zToFree); + } + }/*xOpen()*/, + + xDelete: function(pVfs, zName, iSyncFlag){ + cache.popError(); + try{ + const jzName = wasm.cstrToJs(zName); + if( cache.rxJournalSuffix.test(jzName) ){ + recordHandler.xRcrdDelete(zName, cache.zKeyJrnl); + }/* + else: historically not done, but maybe otherwise delete + all db pages from storageForZClass(zName)? + */ + return 0; + }catch(e){ + warn("xDelete",e); + return cache.setError(e); + } + }, + + xAccess: function(pProtoVfs, zPath, flags, pResOut){ + cache.popError(); + try{ + const s = storageForZClass(zPath); + const jzPath = s?.jzClass || wasm.cstrToJs(zPath); + if( kvvfs?.log?.xAccess ){ + debug("xAccess",jzPath,"flags =", + flags,"*pResOut =",wasm.peek32(pResOut), + "store =",s); + } + if( !s ){ + // From the API docs: + /** The xAccess method returns [SQLITE_OK] on success or some + ** non-zero error code if there is an I/O error or if the name of + ** the file given in the second argument is illegal. + */ + // However, returning non-0 from here is fatal, so we don't do that. + try{validateStorageName(jzPath)} + catch(e){ + //warn("xAccess is ignoring name validation failure:",e); + wasm.poke32(pResOut, 0); + return 0; + } + } + if( s ){ + const key = s.keyPrefix+ + (cache.rxJournalSuffix.test(jzPath) ? "jrnl" : "1"); + const res = s.storage.getItem(key) ? 0 : 1; + /* This res value looks completely backwards to me, and + is the opposite of the native kvvfs's impl, but it's + working, whereas reimplementing the native one + faithfully does not. Read the lib-level code of where + this is invoked, my expectation is that we set res to 0 + for not-exists. */ + //warn("access res",jzPath,res); + wasm.poke32(pResOut, res); + }else{ + wasm.poke32(pResOut, 0); + } + return 0; + }catch(e){ + error('xAccess',e); + return cache.setError(e); + } + }, + + xRandomness: function(pVfs, nOut, pOut){ + const heap = wasm.heap8u(); + let i = 0; + const npOut = Number(pOut); + for(; i < nOut; ++i) heap[npOut + i] = (Math.random()*255000) & 0xFF; + return nOut; + }, + + xGetLastError: function(pVfs,nOut,pOut){ + const e = cache.popError(); + debug('xGetLastError',e); + if(e){ + const scope = wasm.scopedAllocPush(); + try{ + const [cMsg, n] = wasm.scopedAllocCString(e.message, true); + wasm.cstrncpy(pOut, cMsg, nOut); + if(n > nOut) wasm.poke8(wasm.ptr.add(pOut,nOut,-1), 0); + debug("set xGetLastError",e.message); + return (e.resultCode | 0) || capi.SQLITE_IOERR; + }catch(e){ + return capi.SQLITE_NOMEM; + }finally{ + wasm.scopedAllocPop(scope); + } + } + return 0; + } + +//#if 0 + // these impls work but there's currently no pressing need _not_ use + // the native impls. + xCurrentTime: function(pVfs,pOut){ + wasm.poke64f(pOut, 2440587.5 + (Date.now()/86400000)); + return 0; + }, + + xCurrentTimeInt64: function(pVfs,pOut){ + wasm.poke64(pOut, (2440587.5 * 86400000) + Date.now()); + return 0; + } +//#/if + }/*.vfs*/, + + /** + kvvfs has separate sqlite3_api_methods impls for some of the + methods depending on whether it's a db or journal file. Some + of the methods use shared impls but others are specific to + either db or journal files. + */ + ioDb:{ + /* sqlite3_kvvfs_methods::pIoDb's methods */ + xClose: function(pFile){ + cache.popError(); + try{ + const h = pFileHandles.get(pFile); + kvvfs?.log?.xClose && debug("xClose", pFile, h); + if( h ){ + pFileHandles.delete(pFile); + const s = h.store;//storageForZClass(h.jzClass); + s.files = s.files.filter((v)=>v!==h.file); + if( --s.refc<=0 && s.deleteAtRefc0 ){ + deleteStorage(s); + } + originalMethods.ioDb/*same for journals*/.xClose(pFile); + h.file.dispose(); + s.listeners && notifyListeners('close', s, s.files.length); + }else{ + /* Can happen if xOpen fails */ + } + return 0; + }catch(e){ + error("xClose",e); + return cache.setError(e); + } + }, + + xFileControl: function(pFile, opId, pArg){ + cache.popError(); + try{ + const h = pFileHandles.get(pFile); + util.assert(h, "Missing KVVfsFile handle"); + kvvfs?.log?.xFileControl && debug("xFileControl",h,'op =',opId); + if( opId===capi.SQLITE_FCNTL_PRAGMA + && kvvfsInternal.disablePageSizeChange ){ + /* pArg== length-3 (char**) */ + //const argv = wasm.cArgvToJs(3, pArg); // the easy way + const zName = wasm.peekPtr(wasm.ptr.add(pArg, wasm.ptr.size)); + if( "page_size"===wasm.cstrToJs(zName) ){ + kvvfs?.log?.xFileControl + && debug("xFileControl pragma",wasm.cstrToJs(zName)); + const zVal = wasm.peekPtr(wasm.ptr.add(pArg, 2*wasm.ptr.size)); + if( zVal ){ + /* Without this, pragma page_size=N; followed by a + vacuum breaks the db. With this, it continues + working but does not actually change the page + size. */ + kvvfs?.log?.xFileControl + && warn("xFileControl pragma", h, + "NOT setting page size to", wasm.cstrToJs(zVal)); + h.file.$szPage = -1; + return 0/*corrupts: capi.SQLITE_NOTFOUND*/; + }else if( h.file.$szPage>0 ){ + kvvfs?.log?.xFileControl && + warn("xFileControl", h, "getting page size",h.file.$szPage); + wasm.pokePtr(pArg, wasm.allocCString(""+h.file.$szPage) + /* memory now owned by the library */); + return 0;//capi.SQLITE_NOTFOUND; + } + } + } + const rc = originalMethods.ioDb.xFileControl(pFile, opId, pArg); + if( 0==rc && capi.SQLITE_FCNTL_SYNC===opId ){ + h.store.listeners && notifyListeners('sync', h.store, false); + } + return rc; + }catch(e){ + error("xFileControl",e); + return cache.setError(e); + } + }, + + xSync: function(pFile,flags){ + cache.popError(); + try{ + const h = pFileHandles.get(pFile); + kvvfs?.log?.xSync && debug("xSync", h); + util.assert(h, "Missing KVVfsFile handle"); + const rc = originalMethods.ioDb.xSync(pFile, flags); + if( 0==rc && h.store.listeners ) notifyListeners('sync', h.store, true); + return rc; + }catch(e){ + error("xSync",e); + return cache.setError(e); + } + }, + +//#if 0 + // We override xRead/xWrite only for logging/debugging. They + // should otherwise be disabled (it's faster that way). + xRead: function(pFile,pTgt,n,iOff64){ + cache.popError(); + try{ + if( kvvfs?.log?.xRead ){ + const h = pFileHandles.get(pFile); + util.assert(h, "Missing KVVfsFile handle"); + debug("xRead", n, iOff64, h); + } + return originalMethods.ioDb.xRead(pFile, pTgt, n, iOff64); + }catch(e){ + error("xRead",e); + return cache.setError(e); + } + }, + xWrite: function(pFile,pSrc,n,iOff64){ + cache.popError(); + try{ + if( kvvfs?.log?.xWrite ){ + const h = pFileHandles.get(pFile); + util.assert(h, "Missing KVVfsFile handle"); + debug("xWrite", n, iOff64, h); + } + return originalMethods.ioDb.xWrite(pFile, pSrc, n, iOff64); + }catch(e){ + error("xWrite",e); + return cache.setError(e); + } + }, +//#/if + +//#if 0 + xTruncate: function(pFile,i64){}, + xFileSize: function(pFile,pi64Out){}, + xLock: function(pFile,iLock){}, + xUnlock: function(pFile,iLock){}, + xCheckReservedLock: function(pFile,piOut){}, + xSectorSize: function(pFile){}, + xDeviceCharacteristics: function(pFile){} +//#/if + }/*.ioDb*/, + + ioJrnl:{ + /* sqlite3_kvvfs_methods::pIoJrnl's methods. Those set to true + are copied as-is from the ioDb objects. Others are specific + to journal files. */ + xClose: true, +//#if 0 + xRead: function(pFile,pTgt,n,iOff64){}, + xWrite: function(pFile,pSrc,n,iOff64){}, + xTruncate: function(pFile,i64){}, + xSync: function(pFile,flags){}, + xFileControl: function(pFile, opId, pArg){}, + xFileSize: function(pFile,pi64Out){}, + xLock: true, + xUnlock: true, + xCheckReservedLock: true, + xSectorSize: true, + xDeviceCharacteristics: true +//#/if + }/*.ioJrnl*/ + }/*methodOverrides*/; + +//#if 0 + debug("pVfs and friends", pVfs, pIoDb, pIoJrnl, + kvvfsMethods, capi.sqlite3_file.structInfo, + KVVfsFile.structInfo); +//#/if + + try { + util.assert( cache.buffer.n>1024*129, "Heap buffer is not large enough" + /* Native is SQLITE_KVOS_SZ is 133073 as of this writing */ ); + for(const e of Object.entries(methodOverrides.recordHandler)){ + // Overwrite kvvfsMethods's callbacks + const k = e[0], f = e[1]; + recordHandler[k] = f; + if( 0 ){ + // bug: this should work + kvvfsMethods.installMethod(k, f); + }else{ + kvvfsMethods[kvvfsMethods.memberKey(k)] = + wasm.installFunction(kvvfsMethods.memberSignature(k), f); + } + } + for(const e of Object.entries(methodOverrides.vfs)){ + // Overwrite some pVfs entries and stash the original impls + const k = e[0], f = e[1], km = pVfs.memberKey(k), + member = pVfs.structInfo.members[k] + || util.toss("Missing pVfs.structInfo[",k,"]"); + originalMethods.vfs[k] = wasm.functionEntry(pVfs[km]); + pVfs[km] = wasm.installFunction(member.signature, f); + } + for(const e of Object.entries(methodOverrides.ioDb)){ + // Similar treatment for pVfs.$pIoDb a.k.a. pIoDb... + const k = e[0], f = e[1], km = pIoDb.memberKey(k); + originalMethods.ioDb[k] = wasm.functionEntry(pIoDb[km]) + || util.toss("Missing native pIoDb[",km,"]"); + pIoDb[km] = wasm.installFunction(pIoDb.memberSignature(k), f); + } + for(const e of Object.entries(methodOverrides.ioJrnl)){ + // Similar treatment for pVfs.$pIoJrnl a.k.a. pIoJrnl... + const k = e[0], f = e[1], km = pIoJrnl.memberKey(k); + originalMethods.ioJrnl[k] = wasm.functionEntry(pIoJrnl[km]) + || util.toss("Missing native pIoJrnl[",km,"]"); + if( true===f ){ + /* use pIoDb's copy */ + pIoJrnl[km] = pIoDb[km] || util.toss("Missing copied pIoDb[",km,"]"); + }else{ + pIoJrnl[km] = wasm.installFunction(pIoJrnl.memberSignature(k), f); + } + } + }finally{ + kvvfsMethods.dispose(); + pVfs.dispose(); + pIoDb.dispose(); + pIoJrnl.dispose(); + } + + /* + That gets all of the low-level bits out of the way. What follows + are the public API additions. + */ + + /** + Clears all storage used by the kvvfs DB backend, deleting any + DB(s) stored there. + + Its argument must be the name of a kvvfs storage object: + + - 'session' + - 'local' + - '' - see below. + - A transient kvvfs storage object name. + + In the first two cases, only sessionStorage resp. localStorage is + cleared. An empty string resolves to both 'local' and 'session' + storage. + + Returns the number of entries cleared. + + As of kvvfs version 2: + + This API is available in Worker threads but does not have access + to localStorage or sessionStorage in them. Prior versions did not + include this API in Worker threads. + + Differences in this function in version 2: + + - It accepts an arbitrary storage name. In v1 this was a silent + no-op for any names other than ('local','session',''). + + - It throws if a db currently has the storage opened UNLESS the + storage object is localStorage or sessionStorage. That version 1 + did not throw for this case was due to an architectural + limitation which has since been overcome, but removal of + JsStorageDb.prototype.clearStorage() would be a backwards compatibility + break, so this function permits wiping the storage for those two + cases even if they are opened. Use with care. + */ + const sqlite3_js_kvvfs_clear = function callee(which){ + if( ''===which ){ + return callee('local') + callee('session'); + } + const store = storageForZClass(which); + if( !store ) return 0; + if( store.files.length ){ + if( globalThis.localStorage===store.storage + || globalThis.sessionStorage===store.storage ){ + /* backwards compatibility: allow these to be cleared + while opened. */ + }else{ + /* Interestingly, kvvfs recovers just fine when the storage is + wiped, so long as the db is not in use and its schema is + recreated before it's used, but client apps should not have + to be faced with that eventuality mid-query (where it + _will_ cause failures). Therefore we disallow it when + storage handles are opened. Kvvfs version 1 could not + detect this case - see the if() block above. + */ + toss3(capi.SQLITE_ACCESS, + "Cannot clear in-use database storage."); + } + } + const s = store.storage; + const toRm = [] /* keys to remove */; + let i, n = s.length; + //debug("kvvfs_clear",store,s); + for( i = 0; i < n; ++i ){ + const k = s.key(i); + //debug("kvvfs_clear ?",k); + if(!store.keyPrefix || k.startsWith(store.keyPrefix)) toRm.push(k); + } + toRm.forEach((kk)=>s.removeItem(kk)); + //alertFilesToReload(store); + return toRm.length; + }; + + /** + This routine estimates the approximate amount of + storage used by the given kvvfs back-end. + + Its arguments are as documented for sqlite3_js_kvvfs_clear(), + only the operation this performs is different. + + The returned value is twice the "length" value of every matching + key and value, noting that JavaScript stores each character in 2 + bytes. + + The returned size is not authoritative from the perspective of + how much data can fit into localStorage and sessionStorage, as + the precise algorithms for determining those limits are + unspecified and may include per-entry overhead invisible to + clients. + */ + const sqlite3_js_kvvfs_size = function callee(which){ + if( ''===which ){ + return callee('local') + callee('session'); + } + const store = storageForZClass(which); + if( !store ) return 0; + const s = store.storage; + let i, sz = 0; + for(i = 0; i < s.length; ++i){ + const k = s.key(i); + if(!store.keyPrefix || k.startsWith(store.keyPrefix)){ + sz += k.length; + sz += s.getItem(k).length; + } + } + return sz * 2 /* because JS uses 2-byte char encoding */; + }; + + /** + Exports a kvvfs storage object to an object, optionally + JSON-friendly. + + Usages: + + thisfunc(storageName); + thisfunc(options); + + In the latter case, the options object must be an object with + the following properties: + + - "name" (string) required. The storage to export. + + - "decodePages" (bool=false). If true, the .pages result property + holdes Uint8Array objects holding the raw binary-format db + pages. The default is to use kvvfs-encoded string pages + (JSON-friendly). + + - "includeJournal" (bool=false). If true and the db has a current + journal, it is exported as well. (Kvvfs journals are stored as a + single record within the db's storage object.) + + The returned object is structured as follows... + + - "name": the name of the storage. This is 'local' or 'session' + for localStorage resp. sessionStorage, and an arbitrary name for + transient storage. This propery may be changed before passing + this object to sqlite3_js_kvvfs_import() in order to + import into a different storage object. + + - "timestamp": the time this function was called, in Unix + epoch milliseconds. + + - "size": the unencoded db size. + + - "journal": if options.includeJournal is true and this db has a + journal, it is stored as a string here, otherwise this property + is not set. + + - "pages": An array holding the raw encoded db pages in their + proper order. + + Throws if this db is not opened. + + The encoding of the underlying database is not part of this + interface - it is simply passed on as-is. Interested parties are + directed to src/os_kv.c in the SQLite source tree, with the + caveat that that code also does not offer a public interface. + i.e. the encoding is a private implementation detail of kvvfs. + The format may be changed in the future but kvvfs will continue + to support the current form. + + Added in version @kvvfs-v2-added-in@. + */ + const sqlite3_js_kvvfs_export = function callee(...args){ + let opt; + if( 1===args.length && 'object'===typeof args[0] ){ + opt = args[0]; + }else if(args.length){ + opt = Object.assign(Object.create(null),{ + name: args[0], + //decodePages: true + }); + } + const store = opt ? storageForZClass(opt.name) : null; + if( !store ){ + toss3(capi.SQLITE_NOTFOUND, + "There is no kvvfs storage named",opt?.name); + } + //debug("store to export=",store); + const s = store.storage; + const rc = Object.assign(Object.create(null),{ + name: store.jzClass, + timestamp: Date.now(), + pages: [] + }); + const pages = Object.create(null); + let xpages; + const keyPrefix = store.keyPrefix; + const rxTail = keyPrefix + ? /^kvvfs-[^-]+-(\w+)/ /* X... part of kvvfs-NAME-X... */ + : undefined; + let i = 0, n = s.length; + for( ; i < n; ++i ){ + const k = s.key(i); + if( !keyPrefix || k.startsWith(keyPrefix) ){ + let kk = (keyPrefix ? rxTail.exec(k) : undefined)?.[1] ?? k; + switch( kk ){ + case 'jrnl': + if( opt.includeJournal ) rc.journal = s.getItem(k); + break; + case 'sz': + rc.size = +s.getItem(k); + break; + default: + kk = +kk /* coerce to number */; + if( !util.isInt32(kk) || kk<=0 ){ + toss3(capi.SQLITE_RANGE, "Malformed kvvfs key: "+k); + } + if( opt.decodePages ){ + const spg = s.getItem(k), + n = spg.length, + z = cache.memBuffer(0), + zDec = cache.memBuffer(1), + heap = wasm.heap8u()/* MUST be inited last*/; + let i = 0; + for( ; i < n; ++i ){ + heap[wasm.ptr.add(z, i)] = spg.codePointAt(i) & 0xff; + } + heap[wasm.ptr.add(z, i)] = 0; + //debug("Decoding",i,"page bytes"); + const nDec = kvvfsDecode( + z, zDec, cache.buffer.n + ); + //debug("Decoded",nDec,"page bytes"); + pages[kk] = heap.slice(Number(zDec), wasm.ptr.addn(zDec, nDec)); + }else{ + pages[kk] = s.getItem(k); + } + break; + } + } + } + if( opt.decodePages ) cache.memBufferFree(1); + /* Now sort the page numbers and move them into an array. In JS + property keys are always strings, so we have to coerce them to + numbers so we can get them sorted properly for the array. */ + Object.keys(pages).map((v)=>+v).sort().forEach( + (v)=>rc.pages.push(pages[v]) + ); + return rc; + }/* sqlite3_js_kvvfs_export */; + + /** + The counterpart of sqlite3_js_kvvfs_export(). Its + argument must be the result of that function() or + a compatible one. + + This either replaces the contents of an existing transient + storage object or installs one named exp.name, setting + the storage's db contents to that of the exp object. + + Throws on error. Error conditions include: + + - The given storage object is currently opened by any db. + Performing this page-by-page import would invoke undefined + behavior on them. + + - Malformed input object. + + If it throws after starting the import then it clears the storage + before returning, to avoid leaving the db in an undefined + state. It may throw for any of the above-listed conditions before + reaching that step, in which case the db is not modified. If + exp.name refers to a new storage name then if it throws, the name + does not get installed. + + Added in version @kvvfs-v2-added-in@. + */ + const sqlite3_js_kvvfs_import = function(exp, overwrite=false){ + if( !exp?.timestamp + || !exp.name + || undefined===exp.size + || !Array.isArray(exp.pages) ){ + toss3(capi.SQLITE_MISUSE, "Malformed export object."); + }else if( !exp.size + || (exp.size !== (exp.size | 0)) + //|| (exp.size % cache.fixedPageSize) + || exp.size>=0x7fffffff ){ + toss3(capi.SQLITE_RANGE, "Invalid db size: "+exp.size); + } + + validateStorageName(exp.name); + let store = storageForZClass(exp.name); + const isNew = !store; + if( store ){ + if( !overwrite ){ + //warn("Storage exists:",arguments,store); + toss3(capi.SQLITE_ACCESS, + "Storage '"+exp.name+"' already exists and", + "overwrite was not specified."); + }else if( !store.files || !store.jzClass ){ + toss3(capi.SQLITE_ERROR, + "Internal storage object", exp.name,"seems to be malformed."); + }else if( store.files.length ){ + toss3(capi.SQLITE_IOERR_ACCESS, + "Cannot import db storage while it is in use."); + } + sqlite3_js_kvvfs_clear(exp.name); + }else{ + store = newStorageObj(exp.name); + //warn("Installing new storage:",store); + } + //debug("Importing store",store.poolEntry.files.length, store); + //debug("object to import:",exp); + const keyPrefix = kvvfsKeyPrefix(exp.name); + let zEnc; + try{ + /* Force the native KVVfsFile instances to re-read the db + and page size. */; + const s = store.storage; + s.setItem(keyPrefix+'sz', exp.size); + if( exp.journal ) s.setItem(keyPrefix+'jrnl', exp.journal); + if( exp.pages[0] instanceof Uint8Array ){ + /* raw binary pages */ + //debug("pages",exp.pages); + exp.pages.forEach((u,ndx)=>{ + const n = u.length; + if( 0 && cache.fixedPageSize !== n ){ + util.toss3(capi.SQLITE_RANGE,"Unexpected page size:", n); + } + zEnc ??= cache.memBuffer(1); + const zBin = cache.memBuffer(0), + heap = wasm.heap8u()/*MUST be inited last*/; + /* Copy u to the heap and encode the heap copy via C. This + is _presumably_ faster than porting the encoding algo to + JS. */ + heap.set(u, Number(zBin)); + heap[wasm.ptr.addn(zBin,n)] = 0; + const rc = kvvfsEncode(zBin, n, zEnc); + util.assert( rc < cache.buffer.n, + "Impossibly long output - possibly smashed the heap" ); + util.assert( 0===wasm.peek8(wasm.ptr.add(zEnc,rc)), + "Expecting NUL-terminated encoded output" ); + const jenc = wasm.cstrToJs(zEnc); + //debug("(un)encoded page:",u,jenc); + s.setItem(keyPrefix+(ndx+1), jenc); + }); + }else if( exp.pages[0] ){ + /* kvvfs-encoded pages */ + exp.pages.forEach((v,ndx)=>s.setItem(keyPrefix+(ndx+1), v)); + } + if( isNew ) installStorageAndJournal(store); + }catch{ + if( !isNew ){ + try{sqlite3_js_kvvfs_clear(exp.name);}catch(ee){/*ignored*/} + } + }finally{ + if( zEnc ) cache.memBufferFree(1); + } + return this; + }; + + /** + If no kvvfs storage exists with the given name, one is + installed. If one exists, its reference count is increased so + that it won't be freed by the closing of a database or journal + file. + + Throws if the name is not valid for a new storage object. + + Added in version @kvvfs-v2-added-in@. + */ + const sqlite3_js_kvvfs_reserve = function(name){ + let store = storageForZClass(name); + if( store ){ + ++store.refc; + return; + } + validateStorageName(name); + installStorageAndJournal(newStorageObj(name)); + }; + + /** + Conditionally "unlinks" a kvvfs storage object, reducing its + reference count by 1. + + This is a no-op if name ends in "-journal" or refers to a + built-in storage object. + + It will not lower the refcount below the number of + currently-opened db/journal files for the storage (so that it + cannot delete it out from under them). + + If the refcount reaches 0 then the storage object is + removed. + + Returns true if it reduces the refcount, else false. A result of + true does not necessarily mean that the storage unit was removed, + just that its refcount was lowered. Similarly, a result of false + does not mean that the storage is removed - it may still have + opened handles. + + Added in version @kvvfs-v2-added-in@. + */ + const sqlite3_js_kvvfs_unlink = function(name){ + const store = storageForZClass(name); + if( !store + || kvvfsIsPersistentName(store.jzClass) + || isBuiltinName(store.jzClass) + || cache.rxJournalSuffix.test(name) ) return false; + if( store.refc > store.files.length || 0===store.files.length ){ + if( --store.refc<=0 ){ + /* Ignoring deleteAtRefc0 for an explicit unlink */ + deleteStorage(store); + } + return true; + } + return false; + }; + + /** + Adds an event listener to a kvvfs storage object. The idea is + that this can be used to asynchronously back up one kvvfs storage + object to another or another channel entirely. (The caveat in the + latter case is that kvvfs's format is not readily consumable by + downstream code.) + + Its argument must be an object with the following properties: + + - storage: the name of the kvvfs storage object. + + - reserve [=false]: if true, sqlite3_js_kvvfs_reserve() is used + to ensure that the storage exists if it does not already. + If this is false and the storage does not exist then an + exception is thrown. + + - events: an object which may have any of the following + callback function properties: open, close, write, delete. + + - decodePages [=false]: if true, write events will receive each + db page write in the form of a Uint8Array holding the raw binary + db page. The default is to emit the kvvfs-format page because it + requires no extra work, we already have it in hand, and it's + often smaller. It's not great for interchange, though. + + - includeJournal [=false]: if true, writes and deletes of + "jrnl" records are included. If false, no events are sent + for journal updates. + + Passing the same object to sqlite3_js_kvvfs_unlisten() will + remove the listener. + + Each one of the events callbacks will be called asynchronously + when the given storage performs those operations. They may be + asynchronous functions but are not required to be (the events are + fired async either way, but making the event callbacks async may + be advantageous when multiple listeners are involved). All + exceptions, including those via Promises, are ignored but may (or + may not) trigger warning output on the console. + + Each callback gets passed a single object with the following + properties: + + .type = the same as the name of the callback + + .storageName = the name of the storage object + + .data = callback-dependent: + + - 'open' and 'close' get an integer, the number of + currently-opened handles on the storage. + + - 'write' gets a length-two array holding the key and value which + were written. The key is always a string, even if it's a db page + number. For db-page records, the value's type depends on + opt.decodePages. All others, including the journal, are strings. + (The journal, being a kvvfs-specific format, is delivered in + that same JSON-friendly format.) More details below. + + - 'delete' gets the string-type key of the deleted record. + + - 'sync' gets a boolean value: true if it was triggered by db + file's xSync(), false if it was triggered by xFileControl(). The + latter triggers before the xSync() and also triggers if the DB + has PRAGMA SYNCHRONOUS=OFF (in which case xSync() is not + triggered). + + The key/value arguments to 'write', and key argument to 'delete', + are in one of the following forms: + + - 'sz' = the unencoded db size as a string. This specific key is + key is never deleted, so is only ever passed to 'write' events. + + - 'jrnl' = the current db journal as a kvvfs-encoded string. This + journal format is not useful anywhere except in the kvvfs + internals. These events are not fired if opt.includeJournal is + false. + + - '[1-9][0-9]*' (a db page number) = Its type depends on + opt.decodePages. These may be written and deleted in arbitrary + order. + + Design note: JS has StorageEvents but only in the main thread, + which is why the listeners are not based on that. + + Added in version @kvvfs-v2-added-in@. + */ + const sqlite3_js_kvvfs_listen = function(opt){ + if( !opt || 'object'!==typeof opt ){ + toss3(capi.SQLITE_MISUSE, "Expecting a listener object."); + } + let store = storageForZClass(opt.storage); + if( !store ){ + if( opt.storage && opt.reserve ){ + sqlite3_js_kvvfs_reserve(opt.storage); + store = storageForZClass(opt.storage); + util.assert(store, + "Unexpectedly cannot fetch reserved storage " + +opt.storage); + }else{ + toss3(capi.SQLITE_NOTFOUND,"No such storage:",opt.storage); + } + } + if( opt.events ){ + (store.listeners ??= []).push(opt); + } + }; + + /** + Removes the kvvfs event listeners for the given options + object. It must be passed the same object instance which was + passed to sqlite3_js_kvvfs_listen(). + + This has no side effects if opt is invalid or is not a match for + any listeners. + + Return true if it unregisters its argument, else false. + + Added in version @kvvfs-v2-added-in@. + */ + const sqlite3_js_kvvfs_unlisten = function(opt){ + const store = storageForZClass(opt?.storage); + if( store?.listeners && opt.events ){ + const n = store.listeners.length; + store.listeners = store.listeners.filter((v)=>v!==opt); + const rc = n>store.listeners.length; + if( !store.listeners.length ){ + // to speed up downstream checks for listeners + store.listeners = undefined; + } + return rc; + } + return false; + }; + + sqlite3.kvvfs.reserve = sqlite3_js_kvvfs_reserve; + sqlite3.kvvfs.import = sqlite3_js_kvvfs_import; + sqlite3.kvvfs.export = sqlite3_js_kvvfs_export; + sqlite3.kvvfs.unlink = sqlite3_js_kvvfs_unlink; + sqlite3.kvvfs.listen = sqlite3_js_kvvfs_listen; + sqlite3.kvvfs.unlisten = sqlite3_js_kvvfs_unlisten; + sqlite3.kvvfs.exists = (name)=>!!storageForZClass(name); + sqlite3.kvvfs.estimateSize = sqlite3_js_kvvfs_size; + sqlite3.kvvfs.clear = sqlite3_js_kvvfs_clear; + + + if( globalThis.Storage ){ + /** + Prior to version 2, kvvfs was only available in the main + thread. We retain that for the v1 APIs, exposing them only in + the main UI thread. As of version 2, kvvfs is available in all + threads but only via its v2 interface (sqlite3.kvvfs). + + These versions have a default argument value of "" which the v2 + versions lack. + */ + capi.sqlite3_js_kvvfs_size = (which="")=>sqlite3_js_kvvfs_size(which); + capi.sqlite3_js_kvvfs_clear = (which="")=>sqlite3_js_kvvfs_clear(which); + } + +//#if not omit-oo1 + if(sqlite3.oo1?.DB){ + /** + Functionally equivalent to DB(storageName,'c','kvvfs') except + that it throws if the given storage name is not one of 'local' + or 'session'. + + As of version 3.46, the argument may optionally be an options + object in the form: + + { + filename: 'session'|'local', + ... etc. (all options supported by the DB ctor) + } + + noting that the 'vfs' option supported by main DB + constructor is ignored here: the vfs is always 'kvvfs'. + */ + const DB = sqlite3.oo1.DB; + sqlite3.oo1.JsStorageDb = function( + storageName = sqlite3.oo1.JsStorageDb.defaultStorageName + ){ + const opt = DB.dbCtorHelper.normalizeArgs(...arguments); + opt.vfs = 'kvvfs'; + if( 0 ){ + // Current tests rely on these, but that's arguably a bug + if( opt.flags ) opt.flags = 'cw'+opt.flags; + else opt.flags = 'cw'; + } + switch( opt.filename ){ + /* sqlite3_open(), in these builds, recognizes the names + below and performs some magic which we want to bypass + here for sanity's sake. */ + case ":sessionStorage:": opt.filename = 'session'; break; + case ":localStorage:": opt.filename = 'local'; break; + } + const m = /(file:(\/\/)?)([^?]+)/.exec(opt.filename); + validateStorageName( m ? m[3] : opt.filename); + DB.dbCtorHelper.call(this, opt); + }; + sqlite3.oo1.JsStorageDb.defaultStorageName + = cache.storagePool.session ? 'session' : nameOfThisThreadStorage; + const jdb = sqlite3.oo1.JsStorageDb; + jdb.prototype = Object.create(DB.prototype); + jdb.clearStorage = sqlite3_js_kvvfs_clear; + /** + DEPRECATED: the inherited method of this name (as opposed to + the "static" class method) is deprecated with version 2 of + kvvfs. This function will, for backwards comaptibility, + continue to work with localStorage and sessionStorage, but will + throw for all other storage because they are opened. Version 1 + was not capable of recognizing that the storage was opened so + permitted wiping it out at any time, but that was arguably a + bug. + + Clears this database instance's storage or throws if this + instance has been closed. Returns the number of + database pages which were cleaned up. + */ + jdb.prototype.clearStorage = function(){ + return jdb.clearStorage(this.affirmOpen().dbFilename(), true); + }; + /** Equivalent to sqlite3_js_kvvfs_size(). */ + jdb.storageSize = sqlite3_js_kvvfs_size; + /** + Returns the _approximate_ number of bytes this database takes + up in its storage or throws if this instance has been closed. + */ + jdb.prototype.storageSize = function(){ + return jdb.storageSize(this.affirmOpen().dbFilename(), true); + }; + }/*sqlite3.oo1.JsStorageDb*/ +//#/if not omit-oo1 + + if( sqlite3.__isUnderTest && sqlite3.vtab ){ + /** + An eponymous vtab for inspecting the kvvfs state. This is only + intended for use in testing and development, not part of the + public API. + */ + const cols = Object.assign(Object.create(null),{ + rowid: {type: 'INTEGER'}, + name: {type: 'TEXT'}, + nRef: {type: 'INTEGER'}, + nOpen: {type: 'INTEGER'}, + isTransient: {type: 'INTEGER'}, + dbSize: {type: 'INTEGER'} + }); + Object.keys(cols).forEach((v,i)=>cols[v].colId = i); + + const VT = sqlite3.vtab; + const ProtoCursor = Object.assign(Object.create(null),{ + row: function(){ + return cache.storagePool[this.names[this.rowid]]; + } + }); + Object.assign(Object.create(ProtoCursor),{ + rowid: 0, + names: Object.keys(cache.storagePool) + .filter(v=>!cache.rxJournalSuffix.test(v)) + }); + const cursorState = function(cursor, reset){ + const o = (cursor instanceof capi.sqlite3_vtab_cursor) + ? cursor + : VT.xCursor.get(cursor); + if( reset || !o.vTabState ){ + o.vTabState = Object.assign(Object.create(ProtoCursor),{ + rowid: 0, + names: Object.keys(cache.storagePool) + .filter(v=>!cache.rxJournalSuffix.test(v)) + }); + } + return o.vTabState; + }; + + const dbg = 1 ? ()=>{} : (...args)=>debug("vtab",...args); + + const theModule = function f(){ + return f.mod ??= new sqlite3.capi.sqlite3_module().setupModule({ + catchExceptions: true, + methods: { + xConnect: function(pDb, pAux, argc, argv, ppVtab, pzErr){ + dbg("xConnect"); + try{ + const xcol = []; + Object.keys(cols).forEach((k)=>{ + xcol.push(k+" "+cols[k].type); + }); + const rc = capi.sqlite3_declare_vtab( + pDb, "CREATE TABLE ignored("+xcol.join(',')+")" + ); + if(0===rc){ + const t = VT.xVtab.create(ppVtab); + util.assert( + (t === VT.xVtab.get(wasm.peekPtr(ppVtab))), + "output pointer check failed" + ); + } + return rc; + }catch(e){ + return VT.xError('xConnect', e, capi.SQLITE_ERROR); + } + }, + xCreate: wasm.ptr.null, // eponymous only + //xCreate: true, // copy xConnect, i.e. also eponymous only + xDisconnect: function(pVtab){ + dbg("xDisconnect",...arguments); + VT.xVtab.dispose(pVtab); + return 0; + }, + xOpen: function(pVtab, ppCursor){ + dbg("xOpen",...arguments); + VT.xCursor.create(ppCursor); + return 0; + }, + xClose: function(pCursor){ + dbg("xClose",...arguments); + const c = VT.xCursor.unget(pCursor); + delete c.vTabState; + c.dispose(); + return 0; + }, + xNext: function(pCursor){ + dbg("xNext",...arguments); + const c = VT.xCursor.get(pCursor); + ++cursorState(c).rowid; + return 0; + }, + xColumn: function(pCursor, pCtx, iCol){ + dbg("xColumn",...arguments); + //const c = VT.xCursor.get(pCursor); + const st = cursorState(pCursor); + const store = st.row(); + util.assert(store, "Unexpected xColumn call"); + switch(iCol){ + case cols.rowid.colId: + capi.sqlite3_result_int(pCtx, st.rowid); + break; + case cols.name.colId: + capi.sqlite3_result_text(pCtx, store.jzClass, -1, capi.SQLITE_TRANSIENT); + break; + case cols.nRef.colId: + capi.sqlite3_result_int(pCtx, store.refc); + break; + case cols.nOpen.colId: + capi.sqlite3_result_int(pCtx, store.files.length); + break; + case cols.isTransient.colId: + capi.sqlite3_result_int(pCtx, !!store.deleteAtRefc0); + break; + case cols.dbSize.colId: + capi.sqlite3_result_int(pCtx, storageGetDbSize(store)); + break; + default: + capi.sqlite3_result_error(pCtx, "Invalid column id: "+iCol); + return capi.SQLITE_RANGE; + } + return 0; + }, + xRowid: function(pCursor, ppRowid64){ + dbg("xRowid",...arguments); + const st = cursorState(pCursor); + VT.xRowid(ppRowid64, st.rowid); + return 0; + }, + xEof: function(pCursor){ + const st = cursorState(pCursor); + dbg("xEof?="+(!st.row()),...arguments); + return !st.row(); + }, + xFilter: function(pCursor, idxNum, idxCStr, + argc, argv/* [sqlite3_value* ...] */){ + dbg("xFilter",...arguments); + const st = cursorState(pCursor, true); + return 0; + }, + xBestIndex: function(pVtab, pIdxInfo){ + dbg("xBestIndex",...arguments); + //const t = VT.xVtab.get(pVtab); + const pii = new capi.sqlite3_index_info(pIdxInfo); + pii.$estimatedRows = cache.storagePool.size; + pii.$estimatedCost = 1.0; + pii.dispose(); + return 0; + } + } + })/*setupModule*/; + }/*theModule()*/; + + sqlite3.kvvfs.create_module = function(pDb, name="sqlite_kvvfs"){ + return capi.sqlite3_create_module(pDb, name, theModule(), + wasm.ptr.null); + }; + + }/* virtual table */ + +//#if 0 + /** + The idea here is a simpler wrapper for listening to kvvfs + changes. Clients would override its onXyz() event methods + instead of providing callbacks for sqlite3.kvvfs.listen(), the + main (only?) benefit of which is that this class would do the + sorting-out and validation of event state before calling the + overloaded callbacks. + */ + kvvfs.Listener = class KvvfsListener { + #store; + #listener; + + constructor(opt){ + this.#listenTo(opt); + } + + #event(ev){ + switch(ev.type){ + case 'open': this.onOpen(ev.data); break; + case 'close': this.onClose(ev.data); break; + case 'sync': this.onSync(ev.data); break; + case 'delete': + switch(ev.data){ + case 'jrnl': break; + default:{ + const n = +ev.data; + util.assert( n>0, "Expecting positive db page number" ); + this.onPageChange(n, null); + break; + } + } + break; + case 'write':{ + const key = ev.data[0], val = ev.data[1]; + switch( key ){ + case 'jrnl': break; + case 'sz':{ + const sz = +val; + util.assert( sz>0, "Expecting a db page number" ); + this.onSizeChange(sz); + break; + } + default: + T.assert( +key>0, "Expecting a positive db page number" ); + this.onPageChange(+key, val); + break; + } + break; + } + } + } + + #listenTo(opt){ + if(this.#listener){ + sqlite3_js_kvvfs_unlisten(this.#listener); + this.#listener = undefined; + } + const eventHandler = async function(ev){this.event(ev)}.bind(this); + const li = Object.assign( + { /* Defaults */ + reserve: false, + includeJournal: false, + decodePages: false, + storage: null + }, + (/*client options*/opt||{}), + {/*hard-coded options*/ + events: Object.assign(Object.create(null),{ + 'open': eventHandler, + 'close': eventHandler, + 'write': eventHandler, + 'delete': eventHandler, + 'sync': eventHandler + }) + } + ); + sqlite3_js_kvvfs_listen(li); + this.#listener = li; + } + + async onSizeChange(sz){} + async onPageChange(pgNo,content/*null for delete*/){} + async onSync(mode/*true=xSync, false=xFileControl*/){} + async onOpen(count){} + async onClose(count){} + }/*KvvfsListener*/; +//#/if nope + +})/*globalThis.sqlite3ApiBootstrap.initializers*/; +//#savepoint rollback +//#/if not omit-kvvfs diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js index 69be338b0..2990fb147 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js @@ -55,6 +55,10 @@ */ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'use strict'; + if( sqlite3.config.disable?.vfs?.['opfs-sahpool'] ){ + return; + } + const toss = sqlite3.util.toss; const toss3 = sqlite3.util.toss3; const initPromises = Object.create(null) /* cache of (name:result) of VFS init results */; @@ -358,7 +362,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ try{ const [cMsg, n] = wasm.scopedAllocCString(e.message, true); wasm.cstrncpy(pOut, cMsg, nOut); - if(n > nOut) wasm.poke8(pOut + nOut - 1, 0); + if(n > nOut) wasm.poke8(wasm.ptr.add(pOut,nOut,-1), 0); }catch(e){ return capi.SQLITE_NOMEM; }finally{ @@ -410,7 +414,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }/*vfsMethods*/; /** - Creates and initializes an sqlite3_vfs instance for an + Creates, initializes, and returns an sqlite3_vfs instance for an OpfsSAHPool. The argument is the VFS's name (JS string). Throws if the VFS name is already registered or if something @@ -1157,8 +1161,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ described at the end of these docs. This function accepts an options object to configure certain - parts but it is only acknowledged for the very first call and - ignored for all subsequent calls. + parts but it is only acknowledged for the very first call for + each distinct name and ignored for all subsequent calls with that + same name. The options, in alphabetical order: @@ -1224,7 +1229,14 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - Paths given to it _must_ be absolute. Relative paths will not be properly recognized. This is arguably a bug but correcting it requires some hoop-jumping in routines which have no business - doing such tricks. + doing such tricks. (2026-01-19 (2.5 years later): the specifics + are lost to history, but this was a side effect of xOpen() + receiving an immutable C-string filename, to which no implicit + "/" can be prefixed without causing a discrepancy between what + the user provided and what the VFS stores. Its conceivable that + that quirk could be glossed over in xFullPathname(), but + regressions when doing so cannot be ruled out, so there are no + current plans to change this behavior.) - It is possible to install multiple instances under different names, each sandboxed from one another inside their own private @@ -1459,4 +1471,4 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ The OPFS SAH Pool VFS parts are elided from builds targeting node.js. */ -//#endif target:node +//#/if target:node diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js new file mode 100644 index 000000000..a3baee426 --- /dev/null +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -0,0 +1,128 @@ +//#if not target:node +/* + 2026-02-20 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file is a reimplementation of the "opfs" VFS (as distinct from + "opfs-sahpool") which uses WebLocks for locking instead of a bespoke + Atomics.wait()/notify() protocol. This file holds the "synchronous + half" of the VFS, whereas it shares the "asynchronous half" with the + "opfs" VFS. + + Testing has failed to show any genuine functional difference between + these VFSes other than "opfs-wl" being able to dole out xLock() + requests in a strictly FIFO manner by virtue of WebLocks being + globally managed by the browser. This tends to lead to, but does not + guaranty, fairer distribution of locks. Differences are unlikely to + be noticed except, perhaps, under very high contention. + + This file is intended to be appended to the main sqlite3 JS + deliverable somewhere after opfs-common-shared.c-pp.js. +*/ +'use strict'; +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + if( !sqlite3.opfs || sqlite3.config.disable?.vfs?.['opfs-wl'] ){ + return; + } + const util = sqlite3.util, + toss = sqlite3.util.toss; + const opfsUtil = sqlite3.opfs; + const vfsName = 'opfs-wl'; +/** + installOpfsWlVfs() returns a Promise which, on success, installs an + sqlite3_vfs named "opfs-wl", suitable for use with all sqlite3 APIs + which accept a VFS. It is intended to be called via + sqlite3ApiBootstrap.initializers or an equivalent mechanism. + + This VFS is essentially identical to the "opfs" VFS but uses + WebLocks for its xLock() and xUnlock() implementations. + + Quirks specific to this VFS: + + - The (officially undocumented) 'opfs-wl-disable' URL + argument will disable OPFS, making this function a no-op. + + Aside from locking differences in the VFSes, this function + otherwise behaves the same as + sqlite3-vfs-opfs.c-pp.js:installOpfsVfs(). +*/ +const installOpfsWlVfs = async function(options){ + options = opfsUtil.initOptions(vfsName,options); + if( !options ) return sqlite3; + const capi = sqlite3.capi, + state = opfsUtil.createVfsState(), + opfsVfs = state.vfs, + metrics = opfsVfs.metrics.counters, + mTimeStart = opfsVfs.mTimeStart, + mTimeEnd = opfsVfs.mTimeEnd, + opRun = opfsVfs.opRun, + debug = (...args)=>sqlite3.config.debug(vfsName+":",...args), + warn = (...args)=>sqlite3.config.warn(vfsName+":",...args), + __openFiles = opfsVfs.__openFiles; + + //debug("state",JSON.stringify(options)); + /* + At this point, createVfsState() has populated: + + - state: the configuration object we share with the async proxy. + + - opfsVfs: an sqlite3_vfs instance with lots of JS state attached + to it. + + with any code common to both the "opfs" and "opfs-wl" VFSes. Now + comes the VFS-dependent work... + */ + return opfsVfs.bindVfs(util.nu({ + xLock: function(pFile,lockType){ + mTimeStart('xLock'); + //debug("xLock()..."); + const f = __openFiles[pFile]; + const rc = opRun('xLock', pFile, lockType); + if( !rc ) f.lockType = lockType; + mTimeEnd(); + return rc; + }, + xUnlock: function(pFile,lockType){ + mTimeStart('xUnlock'); + const f = __openFiles[pFile]; + const rc = opRun('xUnlock', pFile, lockType); + if( !rc ) f.lockType = lockType; + mTimeEnd(); + return rc; + } + }), function(sqlite3, vfs){ + /* Post-VFS-registration initialization... */ + if(sqlite3.oo1){ + const OpfsWlDb = function(...args){ + const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); + opt.vfs = vfs.$zName; + sqlite3.oo1.DB.dbCtorHelper.call(this, opt); + }; + OpfsWlDb.prototype = Object.create(sqlite3.oo1.DB.prototype); + sqlite3.oo1.OpfsWlDb = OpfsWlDb; + OpfsWlDb.importDb = opfsUtil.importDb; + /* The "opfs" VFS variant adds a + oo1.DB.dbCtorHelper.setVfsPostOpenCallback() callback to set + a high busy_timeout. That was a design mis-decision and is + inconsistent with sqlite3_open() and friends, but is retained + against the risk of introducing regressions if it's removed. + This variant does not repeat that mistake. + */ + } + })/*bindVfs()*/; +}/*installOpfsWlVfs()*/; +globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ + return installOpfsWlVfs().catch((e)=>{ + sqlite3.config.warn("Ignoring inability to install the",vfsName,"sqlite3_vfs:",e); + }); +}); +}/*sqlite3ApiBootstrap.initializers.push()*/); +//#/if target:node diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 2b636460d..8edfab4da 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -16,11 +16,16 @@ asynchronous Origin-Private FileSystem (OPFS) APIs using a second Worker, implemented in sqlite3-opfs-async-proxy.js. This file is intended to be appended to the main sqlite3 JS deliverable somewhere - after sqlite3-api-oo1.js and before sqlite3-api-cleanup.js. + after sqlite3-api-oo1.js. */ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ -/** + if( !sqlite3.opfs || sqlite3.config.disable?.vfs?.opfs ){ + return; + } + const util = sqlite3.util, + opfsUtil = sqlite3.opfs || sqlite3.util.toss("Missing sqlite3.opfs"); + /** installOpfsVfs() returns a Promise which, on success, installs an sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs which accept a VFS. It is intended to be called via @@ -58,7 +63,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ The argument may optionally be a plain object with the following configuration options: - - proxyUri: name of the async proxy JS file. + - proxyUri: name of the async proxy JS file or a synchronous function + which, when called, returns such a name. - verbose (=2): an integer 0-3. 0 disables all logging, 1 enables logging of errors. 2 enables logging of warnings and errors. 3 @@ -70,1391 +76,105 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Promise resolves. This is only intended for testing and development of the VFS, not client-side use. + Additionaly, the (officially undocumented) 'opfs-disable' URL + argument will disable OPFS, making this function a no-op. + On success, the Promise resolves to the top-most sqlite3 namespace - object and that object gets a new object installed in its - `opfs` property, containing several OPFS-specific utilities. + object. Success does not necessarily mean that it installs the VFS, + as there are legitimate non-error reasons for OPFS not to be + available. */ -const installOpfsVfs = function callee(options){ - if(!globalThis.SharedArrayBuffer - || !globalThis.Atomics){ - return Promise.reject( - new Error("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics. "+ - "The server must emit the COOP/COEP response headers to enable those. "+ - "See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep") - ); - }else if('undefined'===typeof WorkerGlobalScope){ - return Promise.reject( - new Error("The OPFS sqlite3_vfs cannot run in the main thread "+ - "because it requires Atomics.wait().") - ); - }else if(!globalThis.FileSystemHandle || - !globalThis.FileSystemDirectoryHandle || - !globalThis.FileSystemFileHandle || - !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || - !navigator?.storage?.getDirectory){ - return Promise.reject( - new Error("Missing required OPFS APIs.") - ); - } - if(!options || 'object'!==typeof options){ - options = Object.create(null); - } - const urlParams = new URL(globalThis.location.href).searchParams; - if(urlParams.has('opfs-disable')){ - //sqlite3.config.warn('Explicitly not installing "opfs" VFS due to opfs-disable flag.'); - return Promise.resolve(sqlite3); - } - if(undefined===options.verbose){ - options.verbose = urlParams.has('opfs-verbose') - ? (+urlParams.get('opfs-verbose') || 2) : 1; - } - if(undefined===options.sanityChecks){ - options.sanityChecks = urlParams.has('opfs-sanity-check'); - } - if(undefined===options.proxyUri){ - options.proxyUri = callee.defaultProxyUri; - } - - //sqlite3.config.warn("OPFS options =",options,globalThis.location); - - if('function' === typeof options.proxyUri){ - options.proxyUri = options.proxyUri(); - } - const thePromise = new Promise(function(promiseResolve_, promiseReject_){ - const loggers = [ - sqlite3.config.error, - sqlite3.config.warn, - sqlite3.config.log - ]; - const logImpl = (level,...args)=>{ - if(options.verbose>level) loggers[level]("OPFS syncer:",...args); - }; - const log = (...args)=>logImpl(2, ...args); - const warn = (...args)=>logImpl(1, ...args); - const error = (...args)=>logImpl(0, ...args); - const toss = sqlite3.util.toss; - const capi = sqlite3.capi; - const util = sqlite3.util; - const wasm = sqlite3.wasm; - const sqlite3_vfs = capi.sqlite3_vfs; - const sqlite3_file = capi.sqlite3_file; - const sqlite3_io_methods = capi.sqlite3_io_methods; - /** - Generic utilities for working with OPFS. This will get filled out - by the Promise setup and, on success, installed as sqlite3.opfs. - - ACHTUNG: do not rely on these APIs in client code. They are - experimental and subject to change or removal as the - OPFS-specific sqlite3_vfs evolves. - */ - const opfsUtil = Object.create(null); - - /** - Returns true if _this_ thread has access to the OPFS APIs. - */ - const thisThreadHasOPFS = ()=>{ - return globalThis.FileSystemHandle && - globalThis.FileSystemDirectoryHandle && - globalThis.FileSystemFileHandle && - globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle && - navigator?.storage?.getDirectory; - }; - - /** - Not part of the public API. Solely for internal/development - use. - */ - opfsUtil.metrics = { - dump: function(){ - let k, n = 0, t = 0, w = 0; - for(k in state.opIds){ - const m = metrics[k]; - n += m.count; - t += m.time; - w += m.wait; - m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; - m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0; - } - sqlite3.config.log(globalThis.location.href, - "metrics for",globalThis.location.href,":",metrics, - "\nTotal of",n,"op(s) for",t, - "ms (incl. "+w+" ms of waiting on the async side)"); - sqlite3.config.log("Serialization metrics:",metrics.s11n); - W.postMessage({type:'opfs-async-metrics'}); - }, - reset: function(){ - let k; - const r = (m)=>(m.count = m.time = m.wait = 0); - for(k in state.opIds){ - r(metrics[k] = Object.create(null)); - } - let s = metrics.s11n = Object.create(null); - s = s.serialize = Object.create(null); - s.count = s.time = 0; - s = metrics.s11n.deserialize = Object.create(null); - s.count = s.time = 0; - } - }/*metrics*/; - const opfsIoMethods = new sqlite3_io_methods(); - const opfsVfs = new sqlite3_vfs() - .addOnDispose( ()=>opfsIoMethods.dispose()); - let promiseWasRejected = undefined; - const promiseReject = (err)=>{ - promiseWasRejected = true; - opfsVfs.dispose(); - return promiseReject_(err); - }; - const promiseResolve = ()=>{ - promiseWasRejected = false; - return promiseResolve_(sqlite3); - }; - const W = -//#if target:es6-bundler-friendly - new Worker(new URL("sqlite3-opfs-async-proxy.js", import.meta.url)); -//#elif target:es6-module - new Worker(new URL(options.proxyUri, import.meta.url)); -//#else - new Worker(options.proxyUri); -//#endif - setTimeout(()=>{ - /* At attempt to work around a browser-specific quirk in which - the Worker load is failing in such a way that we neither - resolve nor reject it. This workaround gives that resolve/reject - a time limit and rejects if that timer expires. Discussion: - https://sqlite.org/forum/forumpost/a708c98dcb3ef */ - if(undefined===promiseWasRejected){ - promiseReject( - new Error("Timeout while waiting for OPFS async proxy worker.") - ); - } - }, 4000); - W._originalOnError = W.onerror /* will be restored later */; - W.onerror = function(err){ - // The error object doesn't contain any useful info when the - // failure is, e.g., that the remote script is 404. - error("Error initializing OPFS asyncer:",err); - promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); - }; - const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; - const dVfs = pDVfs - ? new sqlite3_vfs(pDVfs) - : null /* dVfs will be null when sqlite3 is built with - SQLITE_OS_OTHER. */; - opfsIoMethods.$iVersion = 1; - opfsVfs.$iVersion = 2/*yes, two*/; - opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; - opfsVfs.$mxPathname = 1024/* sure, why not? The OPFS name length limit - is undocumented/unspecified. */; - opfsVfs.$zName = wasm.allocCString("opfs"); - // All C-side memory of opfsVfs is zeroed out, but just to be explicit: - opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null; - opfsVfs.addOnDispose( - '$zName', opfsVfs.$zName, - 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null) - ); - /** - Pedantic sidebar about opfsVfs.ondispose: the entries in that array - are items to clean up when opfsVfs.dispose() is called, but in this - environment it will never be called. The VFS instance simply - hangs around until the WASM module instance is cleaned up. We - "could" _hypothetically_ clean it up by "importing" an - sqlite3_os_end() impl into the wasm build, but the shutdown order - of the wasm engine and the JS one are undefined so there is no - guaranty that the opfsVfs instance would be available in one - environment or the other when sqlite3_os_end() is called (_if_ it - gets called at all in a wasm build, which is undefined). - */ - /** - State which we send to the async-api Worker or share with it. - This object must initially contain only cloneable or sharable - objects. After the worker's "inited" message arrives, other types - of data may be added to it. - - For purposes of Atomics.wait() and Atomics.notify(), we use a - SharedArrayBuffer with one slot reserved for each of the API - proxy's methods. The sync side of the API uses Atomics.wait() - on the corresponding slot and the async side uses - Atomics.notify() on that slot. - - The approach of using a single SAB to serialize comms for all - instances might(?) lead to deadlock situations in multi-db - cases. We should probably have one SAB here with a single slot - for locking a per-file initialization step and then allocate a - separate SAB like the above one for each file. That will - require a bit of acrobatics but should be feasible. The most - problematic part is that xOpen() would have to use - postMessage() to communicate its SharedArrayBuffer, and mixing - that approach with Atomics.wait/notify() gets a bit messy. - */ - const state = Object.create(null); - state.verbose = options.verbose; - state.littleEndian = (()=>{ - const buffer = new ArrayBuffer(2); - new DataView(buffer).setInt16(0, 256, true /* ==>littleEndian */); - // Int16Array uses the platform's endianness. - return new Int16Array(buffer)[0] === 256; - })(); - /** - asyncIdleWaitTime is how long (ms) to wait, in the async proxy, - for each Atomics.wait() when waiting on inbound VFS API calls. - We need to wake up periodically to give the thread a chance to - do other things. If this is too high (e.g. 500ms) then even two - workers/tabs can easily run into locking errors. Some multiple - of this value is also used for determining how long to wait on - lock contention to free up. - */ - state.asyncIdleWaitTime = 150; - - /** - Whether the async counterpart should log exceptions to - the serialization channel. That produces a great deal of - noise for seemingly innocuous things like xAccess() checks - for missing files, so this option may have one of 3 values: - - 0 = no exception logging. - - 1 = only log exceptions for "significant" ops like xOpen(), - xRead(), and xWrite(). - - 2 = log all exceptions. - */ - state.asyncS11nExceptions = 1; - /* Size of file I/O buffer block. 64k = max sqlite3 page size, and - xRead/xWrite() will never deal in blocks larger than that. */ - state.fileBufferSize = 1024 * 64; - state.sabS11nOffset = state.fileBufferSize; - /** - The size of the block in our SAB for serializing arguments and - result values. Needs to be large enough to hold serialized - values of any of the proxied APIs. Filenames are the largest - part but are limited to opfsVfs.$mxPathname bytes. We also - store exceptions there, so it needs to be long enough to hold - a reasonably long exception string. - */ - state.sabS11nSize = opfsVfs.$mxPathname * 2; - /** - The SAB used for all data I/O between the synchronous and - async halves (file i/o and arg/result s11n). - */ - state.sabIO = new SharedArrayBuffer( - state.fileBufferSize/* file i/o block */ - + state.sabS11nSize/* argument/result serialization block */ - ); - state.opIds = Object.create(null); - const metrics = Object.create(null); - { - /* Indexes for use in our SharedArrayBuffer... */ - let i = 0; - /* SAB slot used to communicate which operation is desired - between both workers. This worker writes to it and the other - listens for changes. */ - state.opIds.whichOp = i++; - /* Slot for storing return values. This worker listens to that - slot and the other worker writes to it. */ - state.opIds.rc = i++; - /* Each function gets an ID which this worker writes to - the whichOp slot. The async-api worker uses Atomic.wait() - on the whichOp slot to figure out which operation to run - next. */ - state.opIds.xAccess = i++; - state.opIds.xClose = i++; - state.opIds.xDelete = i++; - state.opIds.xDeleteNoWait = i++; - state.opIds.xFileSize = i++; - state.opIds.xLock = i++; - state.opIds.xOpen = i++; - state.opIds.xRead = i++; - state.opIds.xSleep = i++; - state.opIds.xSync = i++; - state.opIds.xTruncate = i++; - state.opIds.xUnlock = i++; - state.opIds.xWrite = i++; - state.opIds.mkdir = i++; - state.opIds['opfs-async-metrics'] = i++; - state.opIds['opfs-async-shutdown'] = i++; - /* The retry slot is used by the async part for wait-and-retry - semantics. Though we could hypothetically use the xSleep slot - for that, doing so might lead to undesired side effects. */ - state.opIds.retry = i++; - state.sabOP = new SharedArrayBuffer( - i * 4/* ==sizeof int32, noting that Atomics.wait() and friends - can only function on Int32Array views of an SAB. */); - opfsUtil.metrics.reset(); - } - /** - SQLITE_xxx constants to export to the async worker - counterpart... - */ - state.sq3Codes = Object.create(null); - [ - 'SQLITE_ACCESS_EXISTS', - 'SQLITE_ACCESS_READWRITE', - 'SQLITE_BUSY', - 'SQLITE_CANTOPEN', - 'SQLITE_ERROR', - 'SQLITE_IOERR', - 'SQLITE_IOERR_ACCESS', - 'SQLITE_IOERR_CLOSE', - 'SQLITE_IOERR_DELETE', - 'SQLITE_IOERR_FSYNC', - 'SQLITE_IOERR_LOCK', - 'SQLITE_IOERR_READ', - 'SQLITE_IOERR_SHORT_READ', - 'SQLITE_IOERR_TRUNCATE', - 'SQLITE_IOERR_UNLOCK', - 'SQLITE_IOERR_WRITE', - 'SQLITE_LOCK_EXCLUSIVE', - 'SQLITE_LOCK_NONE', - 'SQLITE_LOCK_PENDING', - 'SQLITE_LOCK_RESERVED', - 'SQLITE_LOCK_SHARED', - 'SQLITE_LOCKED', - 'SQLITE_MISUSE', - 'SQLITE_NOTFOUND', - 'SQLITE_OPEN_CREATE', - 'SQLITE_OPEN_DELETEONCLOSE', - 'SQLITE_OPEN_MAIN_DB', - 'SQLITE_OPEN_READONLY' - ].forEach((k)=>{ - if(undefined === (state.sq3Codes[k] = capi[k])){ - toss("Maintenance required: not found:",k); - } - }); - state.opfsFlags = Object.assign(Object.create(null),{ - /** - Flag for use with xOpen(). URI flag "opfs-unlock-asap=1" - enables this. See defaultUnlockAsap, below. - */ - OPFS_UNLOCK_ASAP: 0x01, - /** - Flag for use with xOpen(). URI flag "delete-before-open=1" - tells the VFS to delete the db file before attempting to open - it. This can be used, e.g., to replace a db which has been - corrupted (without forcing us to expose a delete/unlink() - function in the public API). - - Failure to unlink the file is ignored but may lead to - downstream errors. An unlink can fail if, e.g., another tab - has the handle open. - - It goes without saying that deleting a file out from under another - instance results in Undefined Behavior. - */ - OPFS_UNLINK_BEFORE_OPEN: 0x02, - /** - If true, any async routine which implicitly acquires a sync - access handle (i.e. an OPFS lock) will release that lock at - the end of the call which acquires it. If false, such - "autolocks" are not released until the VFS is idle for some - brief amount of time. - - The benefit of enabling this is much higher concurrency. The - down-side is much-reduced performance (as much as a 4x decrease - in speedtest1). - */ - defaultUnlockAsap: false - }); - - /** - Runs the given operation (by name) in the async worker - counterpart, waits for its response, and returns the result - which the async worker writes to SAB[state.opIds.rc]. The - 2nd and subsequent arguments must be the arguments for the - async op. - */ - const opRun = (op,...args)=>{ - const opNdx = state.opIds[op] || toss("Invalid op ID:",op); - state.s11n.serialize(...args); - Atomics.store(state.sabOPView, state.opIds.rc, -1); - Atomics.store(state.sabOPView, state.opIds.whichOp, opNdx); - Atomics.notify(state.sabOPView, state.opIds.whichOp) - /* async thread will take over here */; - const t = performance.now(); - while('not-equal'!==Atomics.wait(state.sabOPView, state.opIds.rc, -1)){ - /* - The reason for this loop is buried in the details of a long - discussion at: - - https://github.com/sqlite/sqlite-wasm/issues/12 - - Summary: in at least one browser flavor, under high loads, - the wait()/notify() pairings can get out of sync. Calling - wait() here until it returns 'not-equal' gets them back in - sync. - */ - } - /* When the above wait() call returns 'not-equal', the async - half will have completed the operation and reported its results - in the state.opIds.rc slot of the SAB. */ - const rc = Atomics.load(state.sabOPView, state.opIds.rc); - metrics[op].wait += performance.now() - t; - if(rc && state.asyncS11nExceptions){ - const err = state.s11n.deserialize(); - if(err) error(op+"() async error:",...err); +const installOpfsVfs = async function(options){ + options = opfsUtil.initOptions('opfs',options); + if( !options ) return sqlite3; + const capi = sqlite3.capi, + state = opfsUtil.createVfsState(), + opfsVfs = state.vfs, + metrics = opfsVfs.metrics.counters, + mTimeStart = opfsVfs.mTimeStart, + mTimeEnd = opfsVfs.mTimeEnd, + opRun = opfsVfs.opRun, + debug = (...args)=>sqlite3.config.debug("opfs:",...args), + warn = (...args)=>sqlite3.config.warn("opfs:",...args), + __openFiles = opfsVfs.__openFiles; + + //debug("options:",JSON.stringify(options)); + /* + At this point, createVfsState() has populated: + + - state: the configuration object we share with the async proxy. + + - opfsVfs: an sqlite3_vfs instance with lots of JS state attached + to it. + + with any code common to both the "opfs" and "opfs-wl" VFSes. Now + comes the VFS-dependent work... + */ + return opfsVfs.bindVfs(util.nu({ + xLock: function(pFile,lockType){ + mTimeStart('xLock'); + ++metrics.xLock.count; + const f = __openFiles[pFile]; + let rc = 0; + /* All OPFS locks are exclusive locks. If xLock() has + previously succeeded, do nothing except record the lock + type. If no lock is active, have the async counterpart + lock the file. */ + if( f.lockType ) { + f.lockType = lockType; + }else{ + rc = opRun('xLock', pFile, lockType); + if( 0===rc ) f.lockType = lockType; } + mTimeEnd(); return rc; - }; - - /** - Not part of the public API. Only for test/development use. - */ - opfsUtil.debug = { - asyncShutdown: ()=>{ - warn("Shutting down OPFS async listener. The OPFS VFS will no longer work."); - opRun('opfs-async-shutdown'); - }, - asyncRestart: ()=>{ - warn("Attempting to restart OPFS VFS async listener. Might work, might not."); - W.postMessage({type: 'opfs-async-restart'}); - } - }; - - const initS11n = ()=>{ - /** - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ACHTUNG: this code is 100% duplicated in the other half of - this proxy! The documentation is maintained in the - "synchronous half". - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - This proxy de/serializes cross-thread function arguments and - output-pointer values via the state.sabIO SharedArrayBuffer, - using the region defined by (state.sabS11nOffset, - state.sabS11nOffset + state.sabS11nSize]. Only one dataset is - recorded at a time. - - This is not a general-purpose format. It only supports the - range of operations, and data sizes, needed by the - sqlite3_vfs and sqlite3_io_methods operations. Serialized - data are transient and this serialization algorithm may - change at any time. - - The data format can be succinctly summarized as: - - Nt...Td...D - - Where: - - - N = number of entries (1 byte) - - - t = type ID of first argument (1 byte) - - - ...T = type IDs of the 2nd and subsequent arguments (1 byte - each). - - - d = raw bytes of first argument (per-type size). - - - ...D = raw bytes of the 2nd and subsequent arguments (per-type - size). - - All types except strings have fixed sizes. Strings are stored - using their TextEncoder/TextDecoder representations. It would - arguably make more sense to store them as Int16Arrays of - their JS character values, but how best/fastest to get that - in and out of string form is an open point. Initial - experimentation with that approach did not gain us any speed. - - Historical note: this impl was initially about 1% this size by - using using JSON.stringify/parse(), but using fit-to-purpose - serialization saves considerable runtime. - */ - if(state.s11n) return state.s11n; - const textDecoder = new TextDecoder(), - textEncoder = new TextEncoder('utf-8'), - viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), - viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - state.s11n = Object.create(null); - /* Only arguments and return values of these types may be - serialized. This covers the whole range of types needed by the - sqlite3_vfs API. */ - const TypeIds = Object.create(null); - TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; - TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; - TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; - TypeIds.string = { id: 4 }; - - const getTypeId = (v)=>( - TypeIds[typeof v] - || toss("Maintenance required: this value type cannot be serialized.",v) - ); - const getTypeIdById = (tid)=>{ - switch(tid){ - case TypeIds.number.id: return TypeIds.number; - case TypeIds.bigint.id: return TypeIds.bigint; - case TypeIds.boolean.id: return TypeIds.boolean; - case TypeIds.string.id: return TypeIds.string; - default: toss("Invalid type ID:",tid); - } - }; - - /** - Returns an array of the deserialized state stored by the most - recent serialize() operation (from from this thread or the - counterpart thread), or null if the serialization buffer is - empty. If passed a truthy argument, the serialization buffer - is cleared after deserialization. - */ - state.s11n.deserialize = function(clear=false){ - ++metrics.s11n.deserialize.count; - const t = performance.now(); - const argc = viewU8[0]; - const rc = argc ? [] : null; - if(argc){ - const typeIds = []; - let offset = 1, i, n, v; - for(i = 0; i < argc; ++i, ++offset){ - typeIds.push(getTypeIdById(viewU8[offset])); - } - for(i = 0; i < argc; ++i){ - const t = typeIds[i]; - if(t.getter){ - v = viewDV[t.getter](offset, state.littleEndian); - offset += t.size; - }else{/*String*/ - n = viewDV.getInt32(offset, state.littleEndian); - offset += 4; - v = textDecoder.decode(viewU8.slice(offset, offset+n)); - offset += n; - } - rc.push(v); - } - } - if(clear) viewU8[0] = 0; - //log("deserialize:",argc, rc); - metrics.s11n.deserialize.time += performance.now() - t; - return rc; - }; - - /** - Serializes all arguments to the shared buffer for consumption - by the counterpart thread. - - This routine is only intended for serializing OPFS VFS - arguments and (in at least one special case) result values, - and the buffer is sized to be able to comfortably handle - those. - - If passed no arguments then it zeroes out the serialization - state. - */ - state.s11n.serialize = function(...args){ - const t = performance.now(); - ++metrics.s11n.serialize.count; - if(args.length){ - //log("serialize():",args); - const typeIds = []; - let i = 0, offset = 1; - viewU8[0] = args.length & 0xff /* header = # of args */; - for(; i < args.length; ++i, ++offset){ - /* Write the TypeIds.id value into the next args.length - bytes. */ - typeIds.push(getTypeId(args[i])); - viewU8[offset] = typeIds[i].id; - } - for(i = 0; i < args.length; ++i) { - /* Deserialize the following bytes based on their - corresponding TypeIds.id from the header. */ - const t = typeIds[i]; - if(t.setter){ - viewDV[t.setter](offset, args[i], state.littleEndian); - offset += t.size; - }else{/*String*/ - const s = textEncoder.encode(args[i]); - viewDV.setInt32(offset, s.byteLength, state.littleEndian); - offset += 4; - viewU8.set(s, offset); - offset += s.byteLength; - } - } - //log("serialize() result:",viewU8.slice(0,offset)); - }else{ - viewU8[0] = 0; - } - metrics.s11n.serialize.time += performance.now() - t; - }; - return state.s11n; - }/*initS11n()*/; - - /** - Generates a random ASCII string len characters long, intended for - use as a temporary file name. - */ - const randomFilename = function f(len=16){ - if(!f._chars){ - f._chars = "abcdefghijklmnopqrstuvwxyz"+ - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ - "012346789"; - f._n = f._chars.length; - } - const a = []; - let i = 0; - for( ; i < len; ++i){ - const ndx = Math.random() * (f._n * 64) % f._n | 0; - a[i] = f._chars[ndx]; - } - return a.join(""); - /* - An alternative impl. with an unpredictable length - but much simpler: - - Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(36) - */ - }; - - /** - Map of sqlite3_file pointers to objects constructed by xOpen(). - */ - const __openFiles = Object.create(null); - - const opTimer = Object.create(null); - opTimer.op = undefined; - opTimer.start = undefined; - const mTimeStart = (op)=>{ - opTimer.start = performance.now(); - opTimer.op = op; - ++metrics[op].count; - }; - const mTimeEnd = ()=>( - metrics[opTimer.op].time += performance.now() - opTimer.start - ); - - /** - Impls for the sqlite3_io_methods methods. Maintenance reminder: - members are in alphabetical order to simplify finding them. - */ - const ioSyncWrappers = { - xCheckReservedLock: function(pFile,pOut){ - /** - As of late 2022, only a single lock can be held on an OPFS - file. We have no way of checking whether any _other_ db - connection has a lock except by trying to obtain and (on - success) release a sync-handle for it, but doing so would - involve an inherent race condition. For the time being, - pending a better solution, we simply report whether the - given pFile is open. - - Update 2024-06-12: based on forum discussions, this - function now always sets pOut to 0 (false): - - https://sqlite.org/forum/forumpost/a2f573b00cda1372 - */ - wasm.poke(pOut, 0, 'i32'); - return 0; - }, - xClose: function(pFile){ - mTimeStart('xClose'); - let rc = 0; - const f = __openFiles[pFile]; - if(f){ - delete __openFiles[pFile]; - rc = opRun('xClose', pFile); - if(f.sq3File) f.sq3File.dispose(); - } - mTimeEnd(); - return rc; - }, - xDeviceCharacteristics: function(pFile){ - return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; - }, - xFileControl: function(pFile, opId, pArg){ - /*mTimeStart('xFileControl'); - mTimeEnd();*/ - return capi.SQLITE_NOTFOUND; - }, - xFileSize: function(pFile,pSz64){ - mTimeStart('xFileSize'); - let rc = opRun('xFileSize', pFile); - if(0==rc){ - try { - const sz = state.s11n.deserialize()[0]; - wasm.poke(pSz64, sz, 'i64'); - }catch(e){ - error("Unexpected error reading xFileSize() result:",e); - rc = state.sq3Codes.SQLITE_IOERR; - } - } - mTimeEnd(); - return rc; - }, - xLock: function(pFile,lockType){ - mTimeStart('xLock'); - const f = __openFiles[pFile]; - let rc = 0; - /* All OPFS locks are exclusive locks. If xLock() has - previously succeeded, do nothing except record the lock - type. If no lock is active, have the async counterpart - lock the file. */ - if( !f.lockType ) { - rc = opRun('xLock', pFile, lockType); - if( 0===rc ) f.lockType = lockType; - }else{ - f.lockType = lockType; - } - mTimeEnd(); - return rc; - }, - xRead: function(pFile,pDest,n,offset64){ - mTimeStart('xRead'); - const f = __openFiles[pFile]; - let rc; - try { - rc = opRun('xRead',pFile, n, Number(offset64)); - if(0===rc || capi.SQLITE_IOERR_SHORT_READ===rc){ - /** - Results get written to the SharedArrayBuffer f.sabView. - Because the heap is _not_ a SharedArrayBuffer, we have - to copy the results. TypedArray.set() seems to be the - fastest way to copy this. */ - wasm.heap8u().set(f.sabView.subarray(0, n), Number(pDest)); - } - }catch(e){ - error("xRead(",arguments,") failed:",e,f); - rc = capi.SQLITE_IOERR_READ; - } - mTimeEnd(); - return rc; - }, - xSync: function(pFile,flags){ - mTimeStart('xSync'); - ++metrics.xSync.count; - const rc = opRun('xSync', pFile, flags); - mTimeEnd(); - return rc; - }, - xTruncate: function(pFile,sz64){ - mTimeStart('xTruncate'); - const rc = opRun('xTruncate', pFile, Number(sz64)); - mTimeEnd(); - return rc; - }, - xUnlock: function(pFile,lockType){ - mTimeStart('xUnlock'); - const f = __openFiles[pFile]; - let rc = 0; - if( capi.SQLITE_LOCK_NONE === lockType + }, + xUnlock: function(pFile,lockType){ + mTimeStart('xUnlock'); + ++metrics.xUnlock.count; + const f = __openFiles[pFile]; + let rc = 0; + if( capi.SQLITE_LOCK_NONE === lockType && f.lockType ){ - rc = opRun('xUnlock', pFile, lockType); - } - if( 0===rc ) f.lockType = lockType; - mTimeEnd(); - return rc; - }, - xWrite: function(pFile,pSrc,n,offset64){ - mTimeStart('xWrite'); - const f = __openFiles[pFile]; - let rc; - try { - f.sabView.set(wasm.heap8u().subarray( - Number(pSrc), Number(pSrc) + n - )); - rc = opRun('xWrite', pFile, n, Number(offset64)); - }catch(e){ - error("xWrite(",arguments,") failed:",e,f); - rc = capi.SQLITE_IOERR_WRITE; - } - mTimeEnd(); - return rc; + rc = opRun('xUnlock', pFile, lockType); } - }/*ioSyncWrappers*/; - - /** - Impls for the sqlite3_vfs methods. Maintenance reminder: members - are in alphabetical order to simplify finding them. - */ - const vfsSyncWrappers = { - xAccess: function(pVfs,zName,flags,pOut){ - mTimeStart('xAccess'); - const rc = opRun('xAccess', wasm.cstrToJs(zName)); - wasm.poke( pOut, (rc ? 0 : 1), 'i32' ); - mTimeEnd(); - return 0; - }, - xCurrentTime: function(pVfs,pOut){ - /* If it turns out that we need to adjust for timezone, see: - https://stackoverflow.com/a/11760121/1458521 */ - wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), - 'double'); - return 0; - }, - xCurrentTimeInt64: function(pVfs,pOut){ - wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(), - 'i64'); - return 0; - }, - xDelete: function(pVfs, zName, doSyncDir){ - mTimeStart('xDelete'); - const rc = opRun('xDelete', wasm.cstrToJs(zName), doSyncDir, false); - mTimeEnd(); - return rc; - }, - xFullPathname: function(pVfs,zName,nOut,pOut){ - /* Until/unless we have some notion of "current dir" - in OPFS, simply copy zName to pOut... */ - const i = wasm.cstrncpy(pOut, zName, nOut); - return i<nOut ? 0 : capi.SQLITE_CANTOPEN - /*CANTOPEN is required by the docs but SQLITE_RANGE would be a closer match*/; - }, - xGetLastError: function(pVfs,nOut,pOut){ - /* TODO: store exception.message values from the async - partner in a dedicated SharedArrayBuffer, noting that we'd have - to encode them... TextEncoder can do that for us. */ - warn("OPFS xGetLastError() has nothing sensible to return."); - return 0; - }, - //xSleep is optionally defined below - xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){ - mTimeStart('xOpen'); - let opfsFlags = 0; - if(0===zName){ - zName = randomFilename(); - }else if(wasm.isPtr(zName)){ - if(capi.sqlite3_uri_boolean(zName, "opfs-unlock-asap", 0)){ - /* -----------------------^^^^^ MUST pass the untranslated - C-string here. */ - opfsFlags |= state.opfsFlags.OPFS_UNLOCK_ASAP; - } - if(capi.sqlite3_uri_boolean(zName, "delete-before-open", 0)){ - opfsFlags |= state.opfsFlags.OPFS_UNLINK_BEFORE_OPEN; - } - zName = wasm.cstrToJs(zName); - //warn("xOpen zName =",zName, "opfsFlags =",opfsFlags); - } - const fh = Object.create(null); - fh.fid = pFile; - fh.filename = zName; - fh.sab = new SharedArrayBuffer(state.fileBufferSize); - fh.flags = flags; - fh.readOnly = !(sqlite3.SQLITE_OPEN_CREATE & flags) - && !!(flags & capi.SQLITE_OPEN_READONLY); - const rc = opRun('xOpen', pFile, zName, flags, opfsFlags); - if(!rc){ - /* Recall that sqlite3_vfs::xClose() will be called, even on - error, unless pFile->pMethods is NULL. */ - if(fh.readOnly){ - wasm.poke(pOutFlags, capi.SQLITE_OPEN_READONLY, 'i32'); - } - __openFiles[pFile] = fh; - fh.sabView = state.sabFileBufView; - fh.sq3File = new sqlite3_file(pFile); - fh.sq3File.$pMethods = opfsIoMethods.pointer; - fh.lockType = capi.SQLITE_LOCK_NONE; - } - mTimeEnd(); - return rc; - }/*xOpen()*/ - }/*vfsSyncWrappers*/; - - if(dVfs){ - opfsVfs.$xRandomness = dVfs.$xRandomness; - opfsVfs.$xSleep = dVfs.$xSleep; - } - if(!opfsVfs.$xRandomness){ - /* If the default VFS has no xRandomness(), add a basic JS impl... */ - vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){ - const heap = wasm.heap8u(); - let i = 0; - const npOut = Number(pOut); - for(; i < nOut; ++i) heap[npOut + i] = (Math.random()*255000) & 0xFF; - return i; - }; - } - if(!opfsVfs.$xSleep){ - /* If we can inherit an xSleep() impl from the default VFS then - assume it's sane and use it, otherwise install a JS-based - one. */ - vfsSyncWrappers.xSleep = function(pVfs,ms){ - Atomics.wait(state.sabOPView, state.opIds.xSleep, 0, ms); - return 0; - }; + if( 0===rc ) f.lockType = lockType; + mTimeEnd(); + return rc; } - - /** - Expects an OPFS file path. It gets resolved, such that ".." - components are properly expanded, and returned. If the 2nd arg - is true, the result is returned as an array of path elements, - else an absolute path string is returned. - */ - opfsUtil.getResolvedPath = function(filename,splitIt){ - const p = new URL(filename, "file://irrelevant").pathname; - return splitIt ? p.split('/').filter((v)=>!!v) : p; - }; - - /** - Takes the absolute path to a filesystem element. Returns an - array of [handleOfContainingDir, filename]. If the 2nd argument - is truthy then each directory element leading to the file is - created along the way. Throws if any creation or resolution - fails. - */ - opfsUtil.getDirForFilename = async function f(absFilename, createDirs = false){ - const path = opfsUtil.getResolvedPath(absFilename, true); - const filename = path.pop(); - let dh = opfsUtil.rootDirectory; - for(const dirName of path){ - if(dirName){ - dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); - } - } - return [dh, filename]; - }; - - /** - Creates the given directory name, recursively, in - the OPFS filesystem. Returns true if it succeeds or the - directory already exists, else false. - */ - opfsUtil.mkdir = async function(absDirName){ - try { - await opfsUtil.getDirForFilename(absDirName+"/filepart", true); - return true; - }catch(e){ - //sqlite3.config.warn("mkdir(",absDirName,") failed:",e); - return false; - } - }; - /** - Checks whether the given OPFS filesystem entry exists, - returning true if it does, false if it doesn't or if an - exception is intercepted while trying to make the - determination. - */ - opfsUtil.entryExists = async function(fsEntryName){ - try { - const [dh, fn] = await opfsUtil.getDirForFilename(fsEntryName); - await dh.getFileHandle(fn); - return true; - }catch(e){ - return false; - } - }; - - /** - Generates a random ASCII string, intended for use as a - temporary file name. Its argument is the length of the string, - defaulting to 16. - */ - opfsUtil.randomFilename = randomFilename; - - /** - Returns a promise which resolves to an object which represents - all files and directories in the OPFS tree. The top-most object - has two properties: `dirs` is an array of directory entries - (described below) and `files` is a list of file names for all - files in that directory. - - Traversal starts at sqlite3.opfs.rootDirectory. - - Each `dirs` entry is an object in this form: - - ``` - { name: directoryName, - dirs: [...subdirs], - files: [...file names] - } - ``` - - The `files` and `subdirs` entries are always set but may be - empty arrays. - - The returned object has the same structure but its `name` is - an empty string. All returned objects are created with - Object.create(null), so have no prototype. - - Design note: the entries do not contain more information, - e.g. file sizes, because getting such info is not only - expensive but is subject to locking-related errors. - */ - opfsUtil.treeList = async function(){ - const doDir = async function callee(dirHandle,tgt){ - tgt.name = dirHandle.name; - tgt.dirs = []; - tgt.files = []; - for await (const handle of dirHandle.values()){ - if('directory' === handle.kind){ - const subDir = Object.create(null); - tgt.dirs.push(subDir); - await callee(handle, subDir); - }else{ - tgt.files.push(handle.name); - } - } - }; - const root = Object.create(null); - await doDir(opfsUtil.rootDirectory, root); - return root; - }; - - /** - Irrevocably deletes _all_ files in the current origin's OPFS. - Obviously, this must be used with great caution. It may throw - an exception if removal of anything fails (e.g. a file is - locked), but the precise conditions under which the underlying - APIs will throw are not documented (so we cannot tell you what - they are). - */ - opfsUtil.rmfr = async function(){ - const dir = opfsUtil.rootDirectory, opt = {recurse: true}; - for await (const handle of dir.values()){ - dir.removeEntry(handle.name, opt); - } - }; - - /** - Deletes the given OPFS filesystem entry. As this environment - has no notion of "current directory", the given name must be an - absolute path. If the 2nd argument is truthy, deletion is - recursive (use with caution!). - - The returned Promise resolves to true if the deletion was - successful, else false (but...). The OPFS API reports the - reason for the failure only in human-readable form, not - exceptions which can be type-checked to determine the - failure. Because of that... - - If the final argument is truthy then this function will - propagate any exception on error, rather than returning false. - */ - opfsUtil.unlink = async function(fsEntryName, recursive = false, - throwOnError = false){ - try { - const [hDir, filenamePart] = - await opfsUtil.getDirForFilename(fsEntryName, false); - await hDir.removeEntry(filenamePart, {recursive}); - return true; - }catch(e){ - if(throwOnError){ - throw new Error("unlink(",arguments[0],") failed: "+e.message,{ - cause: e - }); - } - return false; - } - }; - - /** - Traverses the OPFS filesystem, calling a callback for each - entry. The argument may be either a callback function or an - options object with any of the following properties: - - - `callback`: function which gets called for each filesystem - entry. It gets passed 3 arguments: 1) the - FileSystemFileHandle or FileSystemDirectoryHandle of each - entry (noting that both are instanceof FileSystemHandle). 2) - the FileSystemDirectoryHandle of the parent directory. 3) the - current depth level, with 0 being at the top of the tree - relative to the starting directory. If the callback returns a - literal false, as opposed to any other falsy value, traversal - stops without an error. Any exceptions it throws are - propagated. Results are undefined if the callback manipulate - the filesystem (e.g. removing or adding entries) because the - how OPFS iterators behave in the face of such changes is - undocumented. - - - `recursive` [bool=true]: specifies whether to recurse into - subdirectories or not. Whether recursion is depth-first or - breadth-first is unspecified! - - - `directory` [FileSystemDirectoryEntry=sqlite3.opfs.rootDirectory] - specifies the starting directory. - - If this function is passed a function, it is assumed to be the - callback. - - Returns a promise because it has to (by virtue of being async) - but that promise has no specific meaning: the traversal it - performs is synchronous. The promise must be used to catch any - exceptions propagated by the callback, however. - */ - opfsUtil.traverse = async function(opt){ - const defaultOpt = { - recursive: true, - directory: opfsUtil.rootDirectory - }; - if('function'===typeof opt){ - opt = {callback:opt}; - } - opt = Object.assign(defaultOpt, opt||{}); - const doDir = async function callee(dirHandle, depth){ - for await (const handle of dirHandle.values()){ - if(false === opt.callback(handle, dirHandle, depth)) return false; - else if(opt.recursive && 'directory' === handle.kind){ - if(false === await callee(handle, depth + 1)) break; - } - } - }; - doDir(opt.directory, 0); - }; - - /** - impl of importDb() when it's given a function as its second - argument. - */ - const importDbChunked = async function(filename, callback){ - const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); - const hFile = await hDir.getFileHandle(fnamePart, {create:true}); - let sah = await hFile.createSyncAccessHandle(); - let nWrote = 0, chunk, checkedHeader = false, err = false; - try{ - sah.truncate(0); - while( undefined !== (chunk = await callback()) ){ - if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk); - if( !checkedHeader && 0===nWrote && chunk.byteLength>=15 ){ - util.affirmDbHeader(chunk); - checkedHeader = true; - } - sah.write(chunk, {at: nWrote}); - nWrote += chunk.byteLength; - } - if( nWrote < 512 || 0!==nWrote % 512 ){ - toss("Input size",nWrote,"is not correct for an SQLite database."); - } - if( !checkedHeader ){ - const header = new Uint8Array(20); - sah.read( header, {at: 0} ); - util.affirmDbHeader( header ); - } - sah.write(new Uint8Array([1,1]), {at: 18}/*force db out of WAL mode*/); - return nWrote; - }catch(e){ - await sah.close(); - sah = undefined; - await hDir.removeEntry( fnamePart ).catch(()=>{}); - throw e; - }finally { - if( sah ) await sah.close(); - } - }; - - /** - Asynchronously imports the given bytes (a byte array or - ArrayBuffer) into the given database file. - - Results are undefined if the given db name refers to an opened - db. - - If passed a function for its second argument, its behaviour - changes: imports its data in chunks fed to it by the given - callback function. It calls the callback (which may be async) - repeatedly, expecting either a Uint8Array or ArrayBuffer (to - denote new input) or undefined (to denote EOF). For so long as - the callback continues to return non-undefined, it will append - incoming data to the given VFS-hosted database file. When - called this way, the resolved value of the returned Promise is - the number of bytes written to the target file. - - It very specifically requires the input to be an SQLite3 - database and throws if that's not the case. It does so in - order to prevent this function from taking on a larger scope - than it is specifically intended to. i.e. we do not want it to - become a convenience for importing arbitrary files into OPFS. - - This routine rewrites the database header bytes in the output - file (not the input array) to force disabling of WAL mode. - - On error this throws and the state of the input file is - undefined (it depends on where the exception was triggered). - - On success, resolves to the number of bytes written. - */ - opfsUtil.importDb = async function(filename, bytes){ - if( bytes instanceof Function ){ - return importDbChunked(filename, bytes); - } - if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes); - util.affirmIsDb(bytes); - const n = bytes.byteLength; - const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true); - let sah, err, nWrote = 0; - try { - const hFile = await hDir.getFileHandle(fnamePart, {create:true}); - sah = await hFile.createSyncAccessHandle(); - sah.truncate(0); - nWrote = sah.write(bytes, {at: 0}); - if(nWrote != n){ - toss("Expected to write "+n+" bytes but wrote "+nWrote+"."); - } - sah.write(new Uint8Array([1,1]), {at: 18}) /* force db out of WAL mode */; - return nWrote; - }catch(e){ - if( sah ){ await sah.close(); sah = undefined; } - await hDir.removeEntry( fnamePart ).catch(()=>{}); - throw e; - }finally{ - if( sah ) await sah.close(); - } - }; - + }), function(sqlite3, vfs){ + /* Post-VFS-registration initialization... */ if(sqlite3.oo1){ const OpfsDb = function(...args){ const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); - opt.vfs = opfsVfs.$zName; + opt.vfs = vfs.$zName; sqlite3.oo1.DB.dbCtorHelper.call(this, opt); }; OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); sqlite3.oo1.OpfsDb = OpfsDb; OpfsDb.importDb = opfsUtil.importDb; - sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenCallback( - opfsVfs.pointer, - function(oo1Db, sqlite3){ - /* Set a relatively high default busy-timeout handler to - help OPFS dbs deal with multi-tab/multi-worker - contention. */ - sqlite3.capi.sqlite3_busy_timeout(oo1Db, 10000); - } - ); - }/*extend sqlite3.oo1*/ - - const sanityCheck = function(){ - const scope = wasm.scopedAllocPush(); - const sq3File = new sqlite3_file(); - try{ - const fid = sq3File.pointer; - const openFlags = capi.SQLITE_OPEN_CREATE - | capi.SQLITE_OPEN_READWRITE - //| capi.SQLITE_OPEN_DELETEONCLOSE - | capi.SQLITE_OPEN_MAIN_DB; - const pOut = wasm.scopedAlloc(8); - const dbFile = "/sanity/check/file"+randomFilename(8); - const zDbFile = wasm.scopedAllocCString(dbFile); - let rc; - state.s11n.serialize("This is ä string."); - rc = state.s11n.deserialize(); - log("deserialize() says:",rc); - if("This is ä string."!==rc[0]) toss("String d13n error."); - vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.peek(pOut,'i32'); - log("xAccess(",dbFile,") exists ?=",rc); - rc = vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile, - fid, openFlags, pOut); - log("open rc =",rc,"state.sabOPView[xOpen] =", - state.sabOPView[state.opIds.xOpen]); - if(0!==rc){ - error("open failed with code",rc); - return; - } - vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.peek(pOut,'i32'); - if(!rc) toss("xAccess() failed to detect file."); - rc = ioSyncWrappers.xSync(sq3File.pointer, 0); - if(rc) toss('sync failed w/ rc',rc); - rc = ioSyncWrappers.xTruncate(sq3File.pointer, 1024); - if(rc) toss('truncate failed w/ rc',rc); - wasm.poke(pOut,0,'i64'); - rc = ioSyncWrappers.xFileSize(sq3File.pointer, pOut); - if(rc) toss('xFileSize failed w/ rc',rc); - log("xFileSize says:",wasm.peek(pOut, 'i64')); - rc = ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1); - if(rc) toss("xWrite() failed!"); - const readBuf = wasm.scopedAlloc(16); - rc = ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2); - wasm.poke(readBuf+6,0); - let jRead = wasm.cstrToJs(readBuf); - log("xRead() got:",jRead); - if("sanity"!==jRead) toss("Unexpected xRead() value."); - if(vfsSyncWrappers.xSleep){ - log("xSleep()ing before close()ing..."); - vfsSyncWrappers.xSleep(opfsVfs.pointer,2000); - log("waking up from xSleep()"); - } - rc = ioSyncWrappers.xClose(fid); - log("xClose rc =",rc,"sabOPView =",state.sabOPView); - log("Deleting file:",dbFile); - vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234); - vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.peek(pOut,'i32'); - if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete()."); - warn("End of OPFS sanity checks."); - }finally{ - sq3File.dispose(); - wasm.scopedAllocPop(scope); - } - }/*sanityCheck()*/; - - W.onmessage = function({data}){ - //log("Worker.onmessage:",data); - switch(data.type){ - case 'opfs-unavailable': - /* Async proxy has determined that OPFS is unavailable. There's - nothing more for us to do here. */ - promiseReject(new Error(data.payload.join(' '))); - break; - case 'opfs-async-loaded': - /* Arrives as soon as the asyc proxy finishes loading. - Pass our config and shared state on to the async - worker. */ - W.postMessage({type: 'opfs-async-init',args: state}); - break; - case 'opfs-async-inited': { - /* Indicates that the async partner has received the 'init' - and has finished initializing, so the real work can - begin... */ - if(true===promiseWasRejected){ - break /* promise was already rejected via timer */; - } - try { - sqlite3.vfs.installVfs({ - io: {struct: opfsIoMethods, methods: ioSyncWrappers}, - vfs: {struct: opfsVfs, methods: vfsSyncWrappers} - }); - state.sabOPView = new Int32Array(state.sabOP); - state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); - state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - initS11n(); - if(options.sanityChecks){ - warn("Running sanity checks because of opfs-sanity-check URL arg..."); - sanityCheck(); - } - if(thisThreadHasOPFS()){ - navigator.storage.getDirectory().then((d)=>{ - W.onerror = W._originalOnError; - delete W._originalOnError; - sqlite3.opfs = opfsUtil; - opfsUtil.rootDirectory = d; - log("End of OPFS sqlite3_vfs setup.", opfsVfs); - promiseResolve(); - }).catch(promiseReject); - }else{ - promiseResolve(); - } - }catch(e){ - error(e); - promiseReject(e); - } - break; + if( true ){ + /* 2026-03-06: this was a design mis-decision and is + inconsistent with sqlite3_open() and friends, but is + retained against the risk of introducing regressions if + it's removed. */ + sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenCallback( + opfsVfs.pointer, + function(oo1Db, sqlite3){ + /* Set a relatively high default busy-timeout handler to + help OPFS dbs deal with multi-tab/multi-worker + contention. */ + sqlite3.capi.sqlite3_busy_timeout(oo1Db, 10000); } - default: { - const errMsg = ( - "Unexpected message from the OPFS async worker: " + - JSON.stringify(data) - ); - error(errMsg); - promiseReject(new Error(errMsg)); - break; - } - }/*switch(data.type)*/ - }/*W.onmessage()*/; - })/*thePromise*/; - return thePromise; + ); + } + }/*extend sqlite3.oo1*/ + })/*bindVfs()*/; }/*installOpfsVfs()*/; -installOpfsVfs.defaultProxyUri = - "sqlite3-opfs-async-proxy.js"; globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ - try{ - let proxyJs = installOpfsVfs.defaultProxyUri; - if(sqlite3.scriptInfo.sqlite3Dir){ - installOpfsVfs.defaultProxyUri = - sqlite3.scriptInfo.sqlite3Dir + proxyJs; - //sqlite3.config.warn("installOpfsVfs.defaultProxyUri =",installOpfsVfs.defaultProxyUri); - } - return installOpfsVfs().catch((e)=>{ - sqlite3.config.warn("Ignoring inability to install OPFS sqlite3_vfs:",e.message); - }); - }catch(e){ - sqlite3.config.error("installOpfsVfs() exception:",e); - return Promise.reject(e); - } + return installOpfsVfs().catch((e)=>{ + sqlite3.config.warn("Ignoring inability to install 'opfs' sqlite3_vfs:",e); + }) }); }/*sqlite3ApiBootstrap.initializers.push()*/); -//#else -/* The OPFS VFS parts are elided from builds targeting node.js. */ -//#endif target:node +//#/if target:node diff --git a/ext/wasm/api/sqlite3-vtab-helper.c-pp.js b/ext/wasm/api/sqlite3-vtab-helper.c-pp.js index 4c2338fc5..80f4bfac2 100644 --- a/ext/wasm/api/sqlite3-vtab-helper.c-pp.js +++ b/ext/wasm/api/sqlite3-vtab-helper.c-pp.js @@ -172,10 +172,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Works like unget() plus it calls dispose() on the StructType object. */ - dispose: (pCObj)=>{ - const o = __xWrap(pCObj,true); - if(o) o.dispose(); - } + dispose: (pCObj)=>__xWrap(pCObj,true)?.dispose?.() }); }; diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 4d5e9b296..0c5f4f8ea 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -93,6 +93,18 @@ #undef SQLITE_ENABLE_API_ARMOR #define SQLITE_ENABLE_API_ARMOR 1 +/**********************************************************************/ +/* SQLITE_EXPERIMENTAL_PRAGMA_20251114 */ +/* +** See: +** https://sqlite.org/src/info/e2b3f1a9480a9be3 +** https://github.com/rhashimoto/wa-sqlite/discussions/301 +** +** It is enabled here for the sake of VFS experimentors. +*/ +#undef SQLITE_EXPERIMENTAL_PRAGMA_20251114 +#define SQLITE_EXPERIMENTAL_PRAGMA_20251114 + /**********************************************************************/ /* SQLITE_O... */ #undef SQLITE_OMIT_DEPRECATED @@ -136,8 +148,8 @@ /* ** If SQLITE_WASM_BARE_BONES is defined, undefine most of the ENABLE ** macros. This will, when using the canonical makefile, also elide -** any C functions from the WASM exports which are listed in -** ./EXPORT_FUNCTIONS.sqlite3-extras. +** any C functions from the WASM exports: see +** ./EXPORTED_FUNCTIONS.c-pp. */ #ifdef SQLITE_WASM_BARE_BONES # undef SQLITE_ENABLE_COLUMN_METADATA @@ -217,15 +229,18 @@ ** not by client code, so an argument can be made for reducing their ** visibility by not including them in any build-time export lists. ** -** 2022-09-11: it's not yet _proven_ that this approach works in -** non-Emscripten builds. If not, such builds will need to export -** those using the --export=... wasm-ld flag (or equivalent). As of -** this writing we are tied to Emscripten for various reasons -** and cannot test the library with other build environments. +** 2025-12-01: for use in non-Emscripten builds, we need a more +** invasive macro which explicitly names the export: +** SQLITE_WASM_EXPORT2. */ #define SQLITE_WASM_EXPORT __attribute__((used,visibility("default"))) -// See also: -//__attribute__((export_name("theExportedName"), used, visibility("default"))) +#define SQLITE_WASM_EXPORT_NAMED(X) __attribute__((export_name(#X),used,visibility("default"))) +#define SQLITE_WASM_EXPORT2(RETTYPE,NAME,SIG) SQLITE_WASM_EXPORT_NAMED(NAME) RETTYPE NAME SIG + +#if 1 +/** Increase the kvvfs key size limit from 32. */ +#define KVRECORD_KEY_SZ 128 +#endif /* ** Which sqlite3.c we're using needs to be configurable to enable @@ -249,45 +264,6 @@ #undef INC__STRINGIFY #undef SQLITE_C -#if 0 -/* -** An EXPERIMENT in implementing a stack-based allocator analog to -** Emscripten's stackSave(), stackAlloc(), stackRestore(). -** Unfortunately, this cannot work together with Emscripten because -** Emscripten defines its own native one and we'd stomp on each -** other's memory. Other than that complication, basic tests show it -** to work just fine. -** -** Another option is to malloc() a chunk of our own and call that our -** "stack". -*/ -SQLITE_WASM_EXPORT void * sqlite3__wasm_stack_end(void){ - extern void __heap_base - /* see https://stackoverflow.com/questions/10038964 */; - return &__heap_base; -} -SQLITE_WASM_EXPORT void * sqlite3__wasm_stack_begin(void){ - extern void __data_end; - return &__data_end; -} -static void * pWasmStackPtr = 0; -SQLITE_WASM_EXPORT void * sqlite3__wasm_stack_ptr(void){ - if(!pWasmStackPtr) pWasmStackPtr = sqlite3__wasm_stack_end(); - return pWasmStackPtr; -} -SQLITE_WASM_EXPORT void sqlite3__wasm_stack_restore(void * p){ - pWasmStackPtr = p; -} -SQLITE_WASM_EXPORT void * sqlite3__wasm_stack_alloc(int n){ - if(n<=0) return 0; - n = (n + 7) & ~7 /* align to 8-byte boundary */; - unsigned char * const p = (unsigned char *)sqlite3__wasm_stack_ptr(); - unsigned const char * const b = (unsigned const char *)sqlite3__wasm_stack_begin(); - if(b + n >= p || b + n < b/*overflow*/) return 0; - return pWasmStackPtr = p - n; -} -#endif /* stack allocator experiment */ - /* ** State for the "pseudo-stack" allocator implemented in ** sqlite3__wasm_pstack_xyz(). In order to avoid colliding with @@ -326,7 +302,7 @@ SQLITE_WASM_EXPORT void * sqlite3__wasm_pstack_ptr(void){ */ SQLITE_WASM_EXPORT void sqlite3__wasm_pstack_restore(unsigned char * p){ assert(p>=PStack.pBegin && p<=PStack.pEnd && p>=PStack.pPos); - assert(0==((unsigned long long)p & 0x7)); + assert(0==((unsigned long long)p & 0x7) /* 8-byte aligned */); if(p>=PStack.pBegin && p<=PStack.pEnd /*&& p>=PStack.pPos*/){ PStack.pPos = p; } @@ -336,10 +312,10 @@ SQLITE_WASM_EXPORT void sqlite3__wasm_pstack_restore(unsigned char * p){ ** the memory on success, 0 on error (including a negative n value). n ** is always adjusted to be a multiple of 8 and returned memory is ** always zeroed out before returning (because this keeps the client -** JS code from having to do so, and most uses of the pstack will -** call for doing so). +** JS code from having to do so, and most uses of the pstack call for +** doing so). */ -SQLITE_WASM_EXPORT void * sqlite3__wasm_pstack_alloc(int n){ +SQLITE_WASM_EXPORT2(void *,sqlite3__wasm_pstack_alloc,(int n)){ if( n<=0 ) return 0; n = (n + 7) & ~7 /* align to 8-byte boundary */; if( PStack.pBegin + n > PStack.pPos /*not enough space left*/ @@ -351,7 +327,7 @@ SQLITE_WASM_EXPORT void * sqlite3__wasm_pstack_alloc(int n){ ** Return the number of bytes left which can be ** sqlite3__wasm_pstack_alloc()'d. */ -SQLITE_WASM_EXPORT int sqlite3__wasm_pstack_remaining(void){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_pstack_remaining,(void)){ assert(PStack.pPos >= PStack.pBegin); assert(PStack.pPos <= PStack.pEnd); return (int)(PStack.pPos - PStack.pBegin); @@ -362,7 +338,7 @@ SQLITE_WASM_EXPORT int sqlite3__wasm_pstack_remaining(void){ ** any space which is currently allocated. This value is a ** compile-time constant. */ -SQLITE_WASM_EXPORT int sqlite3__wasm_pstack_quota(void){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_pstack_quota,(void)){ return (int)(PStack.pEnd - PStack.pBegin); } @@ -375,8 +351,7 @@ struct WasmTestStruct { void (*xFunc)(void*); }; typedef struct WasmTestStruct WasmTestStruct; -SQLITE_WASM_EXPORT -void sqlite3__wasm_test_struct(WasmTestStruct * s){ +SQLITE_WASM_EXPORT2(void,sqlite3__wasm_test_struct,(WasmTestStruct * s)){ if(s){ if( 0 ){ /* Do not be alarmed by the small (and odd) pointer values. @@ -416,8 +391,7 @@ void sqlite3__wasm_test_struct(WasmTestStruct * s){ ** fails to compile with "tables may not be 64-bit" but does not tell ** us where it's happening. */ -SQLITE_WASM_EXPORT -const char * sqlite3__wasm_enum_json(void){ +SQLITE_WASM_EXPORT2(const char *,sqlite3__wasm_enum_json,(void)){ static char aBuffer[1024 * 20] = {0} /* where the JSON goes. 2025-09-19: output size=19295, but that can vary slightly from build to build, so a little @@ -597,6 +571,7 @@ const char * sqlite3__wasm_enum_json(void){ DefInt(SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE); DefInt(SQLITE_DBCONFIG_ENABLE_COMMENTS); DefInt(SQLITE_DBCONFIG_MAX); + DefInt(SQLITE_DBCONFIG_FP_DIGITS); } _DefGroup; DefGroup(dbStatus){ @@ -620,6 +595,7 @@ const char * sqlite3__wasm_enum_json(void){ DefGroup(encodings) { /* Noting that the wasm binding only aims to support UTF-8. */ DefInt(SQLITE_UTF8); + DefInt(SQLITE_UTF8_ZT); DefInt(SQLITE_UTF16LE); DefInt(SQLITE_UTF16BE); DefInt(SQLITE_UTF16); @@ -723,6 +699,8 @@ const char * sqlite3__wasm_enum_json(void){ DefInt(SQLITE_MAX_TRIGGER_DEPTH); DefInt(SQLITE_LIMIT_WORKER_THREADS); DefInt(SQLITE_MAX_WORKER_THREADS); + DefInt(SQLITE_LIMIT_PARSER_DEPTH); + DefInt(SQLITE_MAX_PARSER_DEPTH); } _DefGroup; DefGroup(openFlags) { @@ -756,6 +734,7 @@ const char * sqlite3__wasm_enum_json(void){ DefInt(SQLITE_PREPARE_PERSISTENT); DefInt(SQLITE_PREPARE_NORMALIZE); DefInt(SQLITE_PREPARE_NO_VTAB); + DefInt(SQLITE_PREPARE_FROM_DDL); } _DefGroup; DefGroup(resultCodes) { @@ -1007,13 +986,15 @@ const char * sqlite3__wasm_enum_json(void){ /** ^^^ indirection needed to expand CurrentStruct */ #define StructBinder StructBinder_(CurrentStruct) #define _StructBinder CloseBrace(2) -#define M(MEMBER,SIG) \ - outf("%s\"%s\": " \ - "{\"offset\":%d,\"sizeof\": %d,\"signature\":\"%s\"}", \ - (n++ ? ", " : ""), #MEMBER, \ - (int)offsetof(CurrentStruct,MEMBER), \ - (int)sizeof(((CurrentStruct*)0)->MEMBER), \ - SIG) +#define M3(MEMBER,SIG,READONLY) \ + outf("%s\"%s\": " \ + "{\"offset\":%d,\"sizeof\":%d,\"signature\":\"%s\"%s}", \ + (n++ ? ", " : ""), #MEMBER, \ + (int)offsetof(CurrentStruct,MEMBER), \ + (int)sizeof(((CurrentStruct*)0)->MEMBER), \ + SIG, (READONLY ? ",\"readOnly\":true" : "")) +#define M(MEMBER,SIG) M3(MEMBER,SIG,0) +#define MRO(MEMBER,SIG) M3(MEMBER,SIG,1) nStruct = 0; out(", \"structs\": ["); { @@ -1076,11 +1057,30 @@ const char * sqlite3__wasm_enum_json(void){ #undef CurrentStruct #define CurrentStruct sqlite3_kvvfs_methods + /* From os_kv.c */ + StructBinder { + M(xRcrdRead, "i(sspi)"); + M(xRcrdWrite, "i(sss)"); + M(xRcrdDelete, "i(ss)"); + MRO(nKeySize, "i"); + MRO(nBufferSize, "i"); + M(pVfs, "p"); + M(pIoDb, "p"); + M(pIoJrnl, "p"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct KVVfsFile + /* From os_kv.c */ StructBinder { - M(xRead, "i(sspi)"); - M(xWrite, "i(sss)"); - M(xDelete, "i(ss)"); - M(nKeySize, "i"); + M(base, "p")/*sqlite3_file base*/; + M(zClass, "s"); + M(isJournal, "i"); + M(nJrnl, "i")/*actually unsigned!*/; + M(aJrnl, "p"); + M(szPage, "i"); + M(szDb, "j"); + M(aData, "p"); } _StructBinder; #undef CurrentStruct @@ -1137,7 +1137,13 @@ const char * sqlite3__wasm_enum_json(void){ ** sqlite3_index_info, we have to uplift those into constructs we ** can access by type name. These structs _must_ match their ** in-sqlite3_index_info counterparts byte for byte. - */ + ** + ** 2025-11-21: this uplifing is no longer necessary, as Jaccwabyt + ** can now handle nested structs, but "it ain't broke" so there's + ** no pressing need to rewire this. Also, it's conceivable that + ** rewiring it might break downstream vtab impls, so it shouldn't + ** be rewired. + */ typedef struct { int iColumn; unsigned char op; @@ -1232,6 +1238,8 @@ const char * sqlite3__wasm_enum_json(void){ #undef StructBinder_ #undef StructBinder__ #undef M +#undef MRO +#undef M3 #undef _StructBinder #undef CloseBrace #undef out @@ -1249,8 +1257,7 @@ const char * sqlite3__wasm_enum_json(void){ ** method, SQLITE_MISUSE is returned, else the result of the xDelete() ** call is returned. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_vfs_unlink(sqlite3_vfs *pVfs, const char *zName){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_vfs_unlink,(sqlite3_vfs *pVfs, const char *zName)){ int rc = SQLITE_MISUSE /* ??? */; if( 0==pVfs && 0!=zName ) pVfs = sqlite3_vfs_find(0); if( zName && pVfs && pVfs->xDelete ){ @@ -1267,8 +1274,7 @@ int sqlite3__wasm_vfs_unlink(sqlite3_vfs *pVfs, const char *zName){ ** defaulting to "main" if zDbName is 0. Returns 0 if no db with the ** given name is open. */ -SQLITE_WASM_EXPORT -sqlite3_vfs * sqlite3__wasm_db_vfs(sqlite3 *pDb, const char *zDbName){ +SQLITE_WASM_EXPORT2(sqlite3_vfs *,sqlite3__wasm_db_vfs,(sqlite3 *pDb, const char *zDbName)){ sqlite3_vfs * pVfs = 0; sqlite3_file_control(pDb, zDbName ? zDbName : "main", SQLITE_FCNTL_VFS_POINTER, &pVfs); @@ -1290,8 +1296,7 @@ sqlite3_vfs * sqlite3__wasm_db_vfs(sqlite3 *pDb, const char *zDbName){ ** Returns 0 on success, an SQLITE_xxx code on error. Returns ** SQLITE_MISUSE if pDb is NULL. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_db_reset(sqlite3 *pDb){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_db_reset,(sqlite3 *pDb)){ int rc = SQLITE_MISUSE; if( pDb ){ sqlite3_table_column_metadata(pDb, "main", 0, 0, 0, 0, 0, 0, 0); @@ -1372,10 +1377,10 @@ int sqlite3__wasm_db_export_chunked( sqlite3* pDb, ** If `*pOut` is not NULL, the caller is responsible for passing it to ** sqlite3_free() to free it. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_db_serialize( sqlite3 *pDb, const char *zSchema, - unsigned char **pOut, - sqlite3_int64 *nOut, unsigned int mFlags ){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_db_serialize, + (sqlite3 *pDb, const char *zSchema, + unsigned char **pOut, + sqlite3_int64 *nOut, unsigned int mFlags)){ unsigned char * z; if( !pDb || !pOut ) return SQLITE_MISUSE; if( nOut ) *nOut = 0; @@ -1436,11 +1441,9 @@ int sqlite3__wasm_db_serialize( sqlite3 *pDb, const char *zSchema, ** portability, so that the API can still work in builds where BigInt ** support is disabled or unavailable. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_vfs_create_file( sqlite3_vfs *pVfs, - const char *zFilename, - const unsigned char * pData, - int nData ){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_vfs_create_file, + (sqlite3_vfs *pVfs, const char *zFilename, + const unsigned char * pData, int nData)){ int rc; sqlite3_file *pFile = 0; sqlite3_io_methods const *pIo; @@ -1526,10 +1529,9 @@ int sqlite3__wasm_vfs_create_file( sqlite3_vfs *pVfs, ** zFilename, appends pData bytes to it, and returns 0 on success or ** SQLITE_IOERR on error. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_posix_create_file( const char *zFilename, - const unsigned char * pData, - int nData ){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_posix_create_file, + (const char *zFilename, const unsigned char * pData, + int nData)){ int rc; FILE * pFile = 0; int fileExisted = 0; @@ -1549,22 +1551,30 @@ int sqlite3__wasm_posix_create_file( const char *zFilename, ** This function is NOT part of the sqlite3 public API. It is strictly ** for use by the sqlite project's own JS/WASM bindings. ** -** Allocates sqlite3KvvfsMethods.nKeySize bytes from -** sqlite3__wasm_pstack_alloc() and returns 0 if that allocation fails, -** else it passes that string to kvstorageMakeKey() and returns a -** NUL-terminated pointer to that string. It is up to the caller to -** use sqlite3__wasm_pstack_restore() to free the returned pointer. +** This returns either a pointer to a static buffer or zKeyIn directly +** (if zClass is NULL or empty). */ -SQLITE_WASM_EXPORT -char * sqlite3__wasm_kvvfsMakeKeyOnPstack(const char *zClass, - const char *zKeyIn){ +SQLITE_WASM_EXPORT2(const char *,sqlite3__wasm_kvvfsMakeKey, + (const char *zClass, const char *zKeyIn)){ + static char buf[SQLITE_KVOS_SZ+1] = {0}; assert(sqlite3KvvfsMethods.nKeySize>24); - char *zKeyOut = - (char *)sqlite3__wasm_pstack_alloc(sqlite3KvvfsMethods.nKeySize); - if(zKeyOut){ - kvstorageMakeKey(zClass, zKeyIn, zKeyOut); + if( zClass && *zClass ){ + kvrecordMakeKey(zClass, zKeyIn, buf); + return buf; + }else{ +#if 1 + /* We can return zKeyIn here only because the JS API takes special + ** care with its lifetime.*/ + return zKeyIn; +#else + /* It would be nice to be able to return zKeyIn directly here, but + ** it may have been allocated as part of the automated JS-to-WASM + ** conversions, in which case it will be freed before reaching the + ** caller. */ + sqlite3_snprintf(KVRECORD_KEY_SZ, buf, "%s", zKeyIn); + return buf; +#endif } - return zKeyOut; } /* @@ -1574,8 +1584,7 @@ char * sqlite3__wasm_kvvfsMakeKeyOnPstack(const char *zClass, ** Returns the pointer to the singleton object which holds the kvvfs ** I/O methods and associated state. */ -SQLITE_WASM_EXPORT -sqlite3_kvvfs_methods * sqlite3__wasm_kvvfs_methods(void){ +SQLITE_WASM_EXPORT2(sqlite3_kvvfs_methods *,sqlite3__wasm_kvvfs_methods,(void)){ return &sqlite3KvvfsMethods; } @@ -1590,8 +1599,8 @@ sqlite3_kvvfs_methods * sqlite3__wasm_kvvfs_methods(void){ ** sqlite3_vtab_config(), or SQLITE_MISUSE if the 2nd arg is not a ** valid value. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_vtab_config(sqlite3 *pDb, int op, int arg){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_vtab_config, + (sqlite3 *pDb, int op, int arg)){ switch(op){ case SQLITE_VTAB_DIRECTONLY: case SQLITE_VTAB_INNOCUOUS: @@ -1611,8 +1620,8 @@ int sqlite3__wasm_vtab_config(sqlite3 *pDb, int op, int arg){ ** Wrapper for the variants of sqlite3_db_config() which take ** (int,int*) variadic args. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_db_config_ip(sqlite3 *pDb, int op, int arg1, int* pArg2){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_db_config_ip, + (sqlite3 *pDb, int op, int arg1, int* pArg2)){ switch(op){ case SQLITE_DBCONFIG_ENABLE_FKEY: case SQLITE_DBCONFIG_ENABLE_TRIGGER: @@ -1635,6 +1644,7 @@ int sqlite3__wasm_db_config_ip(sqlite3 *pDb, int op, int arg1, int* pArg2){ case SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE: case SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE: case SQLITE_DBCONFIG_ENABLE_COMMENTS: + case SQLITE_DBCONFIG_FP_DIGITS: return sqlite3_db_config(pDb, op, arg1, pArg2); default: return SQLITE_MISUSE; } @@ -1647,8 +1657,9 @@ int sqlite3__wasm_db_config_ip(sqlite3 *pDb, int op, int arg1, int* pArg2){ ** Wrapper for the variants of sqlite3_db_config() which take ** (void*,int,int) variadic args. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_db_config_pii(sqlite3 *pDb, int op, void * pArg1, int arg2, int arg3){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_db_config_pii, + (sqlite3 *pDb, int op, void * pArg1, int arg2, + int arg3)){ switch(op){ case SQLITE_DBCONFIG_LOOKASIDE: return sqlite3_db_config(pDb, op, pArg1, arg2, arg3); @@ -1663,8 +1674,8 @@ int sqlite3__wasm_db_config_pii(sqlite3 *pDb, int op, void * pArg1, int arg2, in ** Wrapper for the variants of sqlite3_db_config() which take ** (const char *) variadic args. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_db_config_s(sqlite3 *pDb, int op, const char *zArg){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_db_config_s,(sqlite3 *pDb, int op, + const char *zArg)){ switch(op){ case SQLITE_DBCONFIG_MAINDBNAME: return sqlite3_db_config(pDb, op, zArg); @@ -1680,8 +1691,7 @@ int sqlite3__wasm_db_config_s(sqlite3 *pDb, int op, const char *zArg){ ** Binding for combinations of sqlite3_config() arguments which take ** a single integer argument. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_config_i(int op, int arg){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_config_i,(int op, int arg)){ return sqlite3_config(op, arg); } @@ -1692,8 +1702,7 @@ int sqlite3__wasm_config_i(int op, int arg){ ** Binding for combinations of sqlite3_config() arguments which take ** two int arguments. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_config_ii(int op, int arg1, int arg2){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_config_ii,(int op, int arg1, int arg2)){ return sqlite3_config(op, arg1, arg2); } @@ -1704,8 +1713,7 @@ int sqlite3__wasm_config_ii(int op, int arg1, int arg2){ ** Binding for combinations of sqlite3_config() arguments which take ** a single i64 argument. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_config_j(int op, sqlite3_int64 arg){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_config_j,(int op, sqlite3_int64 arg)){ return sqlite3_config(op, arg); } @@ -1717,8 +1725,7 @@ int sqlite3__wasm_config_j(int op, sqlite3_int64 arg){ ** sqlite3_mprintf()'s %Q modifier (if addQuotes is true) or %q (if ** addQuotes is 0). Returns NULL if z is NULL or on OOM. */ -SQLITE_WASM_EXPORT -char * sqlite3__wasm_qfmt_token(char *z, int addQuotes){ +SQLITE_WASM_EXPORT2(char *,sqlite3__wasm_qfmt_token,(char *z, int addQuotes)){ char * rc = 0; if( z ){ rc = addQuotes @@ -1728,6 +1735,21 @@ char * sqlite3__wasm_qfmt_token(char *z, int addQuotes){ return rc; } +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** A WASM wrapper for the interal os_kv.c:kvvfsDecode() for internal +** use by the kvvfs v2 API. +*/ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_kvvfs_decode,(const char *a, char *aOut, int nOut)){ + return kvvfsDecode(a, aOut, nOut); +} +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_kvvfs_encode,(const char *a, int nA, char *aOut)){ + return kvvfsEncode(a, nA, aOut); +} + + #if defined(__EMSCRIPTEN__) && defined(SQLITE_ENABLE_WASMFS) #include <emscripten/console.h> #include <emscripten/wasmfs.h> @@ -1753,8 +1775,7 @@ char * sqlite3__wasm_qfmt_token(char *z, int addQuotes){ ** the virtual FS fails. In builds compiled without SQLITE_ENABLE_WASMFS ** defined, SQLITE_NOTFOUND is returned without side effects. */ -SQLITE_WASM_EXPORT -int sqlite3__wasm_init_wasmfs(const char *zMountPoint){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_init_wasmfs,(const char *zMountPoint)){ static backend_t pOpfs = 0; if( !zMountPoint || !*zMountPoint ) zMountPoint = "/opfs"; if( !pOpfs ){ @@ -1773,8 +1794,7 @@ int sqlite3__wasm_init_wasmfs(const char *zMountPoint){ return pOpfs ? 0 : SQLITE_NOMEM; } #else -SQLITE_WASM_EXPORT -int sqlite3__wasm_init_wasmfs(const char *zUnused){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_init_wasmfs,(const char *zUnused)){ //emscripten_console_warn("WASMFS OPFS is not compiled in."); (void)zUnused; return SQLITE_NOTFOUND; @@ -1783,52 +1803,43 @@ int sqlite3__wasm_init_wasmfs(const char *zUnused){ #if SQLITE_WASM_ENABLE_C_TESTS -SQLITE_WASM_EXPORT -int sqlite3__wasm_test_intptr(int * p){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_test_intptr,(int * p)){ return *p = *p * 2; } -SQLITE_WASM_EXPORT -void * sqlite3__wasm_test_voidptr(void * p){ +SQLITE_WASM_EXPORT2(void *,sqlite3__wasm_test_voidptr,(void * p)){ return p; } -SQLITE_WASM_EXPORT -int64_t sqlite3__wasm_test_int64_max(void){ +SQLITE_WASM_EXPORT2(int64_t,sqlite3__wasm_test_int64_max,(void)){ return (int64_t)0x7fffffffffffffff; } -SQLITE_WASM_EXPORT -int64_t sqlite3__wasm_test_int64_min(void){ +SQLITE_WASM_EXPORT2(int64_t,sqlite3__wasm_test_int64_min,(void)){ return ~sqlite3__wasm_test_int64_max(); } -SQLITE_WASM_EXPORT -int64_t sqlite3__wasm_test_int64_times2(int64_t x){ +SQLITE_WASM_EXPORT2(int64_t,sqlite3__wasm_test_int64_times2,(int64_t x)){ return x * 2; } -SQLITE_WASM_EXPORT -void sqlite3__wasm_test_int64_minmax(int64_t * min, int64_t *max){ +SQLITE_WASM_EXPORT2(void,sqlite3__wasm_test_int64_minmax,(int64_t * min, int64_t *max)){ *max = sqlite3__wasm_test_int64_max(); *min = sqlite3__wasm_test_int64_min(); /*printf("minmax: min=%lld, max=%lld\n", *min, *max);*/ } -SQLITE_WASM_EXPORT -int64_t sqlite3__wasm_test_int64ptr(int64_t * p){ +SQLITE_WASM_EXPORT2(int64_t,sqlite3__wasm_test_int64ptr,(int64_t * p)){ /*printf("sqlite3__wasm_test_int64ptr( @%lld = 0x%llx )\n", (int64_t)p, *p);*/ return *p = *p * 2; } -SQLITE_WASM_EXPORT -void sqlite3__wasm_test_stack_overflow(int recurse){ +SQLITE_WASM_EXPORT2(void,sqlite3__wasm_test_stack_overflow,(int recurse)){ if(recurse) sqlite3__wasm_test_stack_overflow(recurse); } /* For testing the 'string:dealloc' whwasmutil.xWrap() conversion. */ -SQLITE_WASM_EXPORT -char * sqlite3__wasm_test_str_hello(int fail){ +SQLITE_WASM_EXPORT2(char *,sqlite3__wasm_test_str_hello,(int fail)){ char * s = fail ? 0 : (char *)sqlite3_malloc(6); if(s){ memcpy(s, "hello", 5); @@ -1942,8 +1953,8 @@ static int sqlite3__wasm_SQLTester_strnotglob(const char *zGlob, const char *z){ return *z==0; } -SQLITE_WASM_EXPORT -int sqlite3__wasm_SQLTester_strglob(const char *zGlob, const char *z){ +SQLITE_WASM_EXPORT2(int,sqlite3__wasm_SQLTester_strglob, + (const char *zGlob, const char *z)){ return !sqlite3__wasm_SQLTester_strnotglob(zGlob, z); } diff --git a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js index 1a09bf9a6..bcbf3fa9f 100644 --- a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js +++ b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js @@ -19,10 +19,12 @@ slightly simpler client-side interface than the slightly-lower-level Worker API does. - This script necessarily exposes one global symbol, but clients may - freely `delete` that symbol after calling it. + In non-ESM builds this file necessarily exposes one global symbol, + but clients may freely `delete` that symbol after calling it. */ +//#if not defined target:es6-module 'use strict'; +//#/if /** Configures an sqlite3 Worker API #1 Worker such that it can be manipulated via a Promise-based interface and returns a factory @@ -109,10 +111,12 @@ the callback is called one time for each row of the result set, passed the same worker message format as the worker API emits: - {type:typeString, + { + type:typeString, row:VALUE, rowNumber:1-based-#, - columnNames: array} + columnNames: array + } Where `typeString` is an internally-synthesized message type string used temporarily for worker message dispatching. It can be ignored @@ -123,10 +127,9 @@ callback. At the end of the result set, the same event is fired with - (row=undefined, rowNumber=null) to indicate that - the end of the result set has been reached. Note that the rows - arrive via worker-posted messages, with all the implications - of that. + (row=undefined, rowNumber=null) to indicate that the end of the + result set has been reached. The rows arrive via worker-posted + messages, with all the implications of that. Notable shortcomings: @@ -257,7 +260,9 @@ globalThis.sqlite3Worker1Promiser.defaultConfig = { type: 'module' }); //#elif target:es6-module - return new Worker(new URL("sqlite3-worker1.js", import.meta.url)); + return new Worker(new URL("sqlite3-worker1.mjs", import.meta.url),{ + type: 'module' + }); //#else let theJs = "sqlite3-worker1.js"; if(this.currentScript){ @@ -273,15 +278,15 @@ globalThis.sqlite3Worker1Promiser.defaultConfig = { } } return new Worker(theJs + globalThis.location.search); -//#endif +//#/if } //#if not target:es6-module .bind({ currentScript: globalThis?.document?.currentScript }) -//#endif +//#/if , - onerror: (...args)=>console.error('worker1 promiser error',...args) + onerror: (...args)=>console.error('sqlite3Worker1Promiser():',...args) }/*defaultConfig*/; /** @@ -343,7 +348,8 @@ globalThis.sqlite3Worker1Promiser.v2.defaultConfig = incompatibility. */ export default sqlite3Worker1Promiser.v2; -//#endif /* target:es6-module */ +delete globalThis.sqlite3Worker1Promiser; +//#/if /* target:es6-module */ //#else /* Built with the omit-oo1 flag. */ -//#endif if not omit-oo1 +//#/if if not omit-oo1 diff --git a/ext/wasm/api/sqlite3-worker1.c-pp.js b/ext/wasm/api/sqlite3-worker1.c-pp.js index 036c4c6ea..046243baa 100644 --- a/ext/wasm/api/sqlite3-worker1.c-pp.js +++ b/ext/wasm/api/sqlite3-worker1.c-pp.js @@ -33,9 +33,9 @@ directory from which `sqlite3.js` will be loaded. */ //#if target:es6-bundler-friendly -import {default as sqlite3InitModule} from './sqlite3-bundler-friendly.mjs'; +import sqlite3InitModule from './sqlite3-bundler-friendly.mjs'; //#elif target:es6-module - return new Worker(new URL("sqlite3.js", import.meta.url)); +import sqlite3InitModule from './sqlite3.mjs'; //#else "use strict"; { @@ -49,8 +49,8 @@ import {default as sqlite3InitModule} from './sqlite3-bundler-friendly.mjs'; //console.warn("worker1 theJs =",theJs); importScripts(theJs); } -//#endif +//#/if sqlite3InitModule().then(sqlite3 => sqlite3.initWorker1API()); //#else /* Built with the omit-oo1 flag. */ -//#endif if not omit-oo1 +//#/if if not omit-oo1 diff --git a/ext/wasm/c-pp-lite.c b/ext/wasm/c-pp-lite.c deleted file mode 100644 index 2120c457d..000000000 --- a/ext/wasm/c-pp-lite.c +++ /dev/null @@ -1,2767 +0,0 @@ -/* -** 2022-11-12: -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** * May you do good and not evil. -** * May you find forgiveness for yourself and forgive others. -** * May you share freely, never taking more than you give. -** -************************************************************************ -** -** The C-minus Preprocessor: a truly minimal C-like preprocessor. -** Why? Because C preprocessors _can_ process non-C code but generally make -** quite a mess of it. The purpose of this application is an extremely -** minimal preprocessor with only the most basic functionality of a C -** preprocessor, namely. -** -** The supported preprocessor directives are documented in the -** README.md hosted with this file. -** -** Any mention of "#" in the docs, e.g. "#if", is symbolic. The -** directive delimiter is configurable and defaults to "##". Define -** CMPP_DEFAULT_DELIM to a string when compiling to define the default -** at build-time. -** -** This preprocessor has only minimal support for replacement of tokens -** which live in the "content" blocks of inputs (that is, the pieces -** which are not prepocessor lines). -** -** See this file's README.md for details. -** -** Design note: this code makes use of sqlite3. Though not _strictly_ -** needed in order to implement it, this tool was specifically created -** for use with the sqlite3 project's own JavaScript code, so there's -** no reason not to make use of it to do some of the heavy lifting. It -** does not require any cutting-edge sqlite3 features and should be -** usable with any version which supports `WITHOUT ROWID`. -** -** Author(s): -** -** - Stephan Beal <https://wanderinghorse.net/home/stephan/> -** -** Canonical homes: -** -** - https://fossil.wanderinghorse.net/r/c-pp -** - https://sqlite.org/src/file/ext/wasm/c-pp.c -** -** With the former hosting this app's SCM and the latter being the -** single known deployment of c-pp.c, where much of its development -** happens. -*/ - -#include <stdlib.h> -#include <stdio.h> -#include <errno.h> -#include <string.h> -#include <stdarg.h> -#include <assert.h> -#include <ctype.h> - -#include "sqlite3.h" - -#if defined(_WIN32) || defined(WIN32) -# include <io.h> -# include <fcntl.h> -# ifndef access -# define access(f,m) _access((f),(m)) -# endif -#else -# include <unistd.h> -#endif - -#ifndef CMPP_DEFAULT_DELIM -#define CMPP_DEFAULT_DELIM "##" -#endif - -#ifndef CMPP_ATSIGN -#define CMPP_ATSIGN (unsigned char)'@' -#endif - -#if 1 -# define CMPP_NORETURN __attribute__((noreturn)) -#else -# define CMPP_NORETURN -#endif - -/* Fatally exits the app with the given printf-style message. */ -static CMPP_NORETURN void fatalv__base(char const *zFile, int line, - char const *zFmt, va_list); -static CMPP_NORETURN void fatal__base(char const *zFile, int line, - char const *zFmt, ...); -#define fatalv(...) fatalv__base(__FILE__,__LINE__,__VA_ARGS__) -#define fatal(...) fatal__base(__FILE__,__LINE__,__VA_ARGS__) - -/** Proxy for free(), for symmetry with cmpp_realloc(). */ -static void cmpp_free(void *p); -/** A realloc() proxy which dies fatally on allocation error. */ -static void * cmpp_realloc(void * p, unsigned n); -#if 0 -/** A malloc() proxy which dies fatally on allocation error. */ -static void * cmpp_malloc(unsigned n); -#endif - -static void check__oom2(void const *p, char const *zFile, int line){ - if(!p) fatal("Alloc failed at %s:%d", zFile, line); -} -#define check__oom(P) check__oom2((P), __FILE__, __LINE__) - -/* -** If p is stdin or stderr then this is a no-op, else it is a -** proxy for fclose(). This is a no-op if p is NULL. -*/ -static void FILE_close(FILE *p); -/* -** Works like fopen() but accepts the special name "-" to mean either -** stdin (if zMode indicates a real-only mode) or stdout. Fails -** fatally on error. -*/ -static FILE * FILE_open(char const *zName, const char * zMode); -/* -** Reads the entire contents of the given file, allocating it in a -** buffer which gets assigned to `*pOut`. `*nOut` gets assigned the -** length of the output buffer. Fails fatally on error. -*/ -static void FILE_slurp(FILE *pFile, unsigned char **pOut, - unsigned * nOut); - -/* -** Intended to be passed an sqlite3 result code. If it's a non-0 value -** other than SQLITE_ROW or SQLITE_DONE then it emits a fatal error -** message which contains both the given string and the -** sqlite3_errmsg() from the application's database instance. -*/ -static void db_affirm_rc(int rc, const char * zMsg); - -/* -** Proxy for sqlite3_str_finish() which fails fatally if that -** routine returns NULL. -*/ -static char * db_str_finish(sqlite3_str *s, int * n); -/* -** Proxy for sqlite3_str_new() which fails fatally if that -** routine returns NULL. -*/ -static sqlite3_str * db_str_new(void); - -/* -** Proxy for sqlite3_step() which fails fatally if the result -** is anything other than SQLITE_ROW or SQLITE_DONE. -*/ -static int db_step(sqlite3_stmt *pStmt); -/* -** Proxy for sqlite3_bind_int() which fails fatally on error. -*/ -static void db_bind_int(sqlite3_stmt *pStmt, int col, int val); -/* -** Proxy for sqlite3_bind_null() which fails fatally on error. -*/ -static void db_bind_null(sqlite3_stmt *pStmt, int col); -/* -** Proxy for sqlite3_bind_text() which fails fatally on error. -*/ -static void db_bind_text(sqlite3_stmt *pStmt, int col, const char * zStr); -/* -** Proxy for sqlite3_bind_text() which fails fatally on error. -*/ -static void db_bind_textn(sqlite3_stmt *pStmt, int col, const char * zStr, int len); -#if 0 -/* -** Proxy for sqlite3_bind_text() which fails fatally on error. It uses -** sqlite3_str_vappendf() so supports all of its formatting options. -*/ -static void db_bind_textv(sqlite3_stmt *pStmt, int col, const char * zFmt, ...); -#endif -/* -** Proxy for sqlite3_free(), to be passed any memory which is allocated -** by sqlite3_malloc(). -*/ -static void db_free(void *m); - -/* -** Returns true if the first nKey bytes of zKey are a legal string. If -** it returns false and zErrPos is not null, *zErrPos is set to the -** position of the illegal character. If nKey is negative, strlen() is -** used to calculate it. -*/ -static int cmpp_is_legal_key(char const *zKey, int nKey, char const **zErrPos); - -/* -** Fails fatally if !cmpp_is_legal_key(zKey). -*/ -static void cmpp_affirm_legal_key(char const *zKey, int nKey); - -/* -** Adds the given `#define` macro name to the list of macros, ignoring -** any duplicates. Fails fatally on error. -** -** If zVal is NULL then zKey may contain an '=', from which the value -** will be extracted. If zVal is not NULL then zKey may _not_ contain -** an '='. -*/ -static void db_define_add(const char * zKey, char const *zVal); - -/* -** Returns true if the given key is already in the `#define` list, -** else false. Fails fatally on db error. -** -** nName is the length of the key part of zName (which might have -** a following =y part. If it's negative, strlen() is used to -** calculate it. -*/ -static int db_define_has(const char * zName, int nName); - -/* -** Returns true if the given key is already in the `#define` list, and -** it has a truthy value (is not empty and not equal to '0'), else -** false. Fails fatally on db error. -** -** nName is the length of zName, or <0 to use strlen() to figure -** it out. -*/ -static int db_define_get_bool(const char * zName, int nName); - -/* -** Searches for a define where (k GLOB zName). If one is found, a copy -** of it is assigned to *zVal (the caller must eventually db_free() -** it)), *nVal (if nVal is not NULL) is assigned its strlen, and -** returns non-0. If no match is found, 0 is returned and neither -** *zVal nor *nVal are modified. If more than one result matches, a -** fatal error is triggered. -** -** It is legal for *zVal to be NULL (and *nVal to be 0) if it returns -** non-0. That just means that the key was defined with no value part. -*/ -static int db_define_get(const char * zName, int nName, char **zVal, unsigned int *nVal); - -/* -** Removes the given `#define` macro name from the list of -** macros. Fails fatally on error. -*/ -static void db_define_rm(const char * zKey); -/* -** Adds the given filename to the list of being-`#include`d files, -** using the given source file name and line number of error reporting -** purposes. If recursion is later detected. -*/ -static void db_including_add(const char * zKey, const char * zSrc, int srcLine); -/* -** Adds the given dir to the list of includes. They are checked in the -** order they are added. -*/ -static void db_include_dir_add(const char * zKey); -/* -** Returns a resolved path of PREFIX+'/'+zKey, where PREFIX is one of -** the `#include` dirs (db_include_dir_add()). If no file match is -** found, NULL is returned. Memory must eventually be passed to -** db_free() to free it. -*/ -static char * db_include_search(const char * zKey); -/* -** Removes the given key from the `#include` list. -*/ -static void db_include_rm(const char * zKey); -/* -** A proxy for sqlite3_prepare() which fails fatally on error. -*/ -static void db_prepare(sqlite3_stmt **pStmt, const char * zSql, ...); - -/* -** Opens the given file and processes its contents as c-pp, sending -** all output to the global c-pp output channel. Fails fatally on -** error. -*/ -static void cmpp_process_file(const char * zName); - -/* -** Operator policy for cmpp_kvp_parse(). -*/ -enum cmpp_key_op_e { - /* Fail if the key contains an operator. */ - cmpp_key_op_none, - /* Accept only '='. */ - cmpp_key_op_eq1 -}; -typedef enum cmpp_key_op_e cmpp_key_op_e; - -/* -** Operators and operator policies for use with X=Y-format keys. -*/ -#define cmpp_kvp_op_map(E) \ - E(none,"") \ - E(eq1,"=") \ - E(eq2,"==") \ - E(lt,"<") \ - E(le,"<=") \ - E(gt,">") \ - E(ge,">=") - -enum cmpp_kvp_op_e { -#define E(N,S) cmpp_kvp_op_ ## N, - cmpp_kvp_op_map(E) -#undef E -}; -typedef enum cmpp_kvp_op_e cmpp_kvp_op_e; - -/* -** A snippet from a string. -*/ -struct cmpp_snippet { - char const *z; - unsigned int n; -}; -typedef struct cmpp_snippet cmpp_snippet; -#define cmpp_snippet_empty_m {0,0} - -/* -** Result type for cmpp_kvp_parse(). -*/ -struct cmpp_kvp { - cmpp_snippet k; - cmpp_snippet v; - cmpp_kvp_op_e op; -}; - -typedef struct cmpp_kvp cmpp_kvp; -#define cmpp_kvp_empty_m \ - {cmpp_snippet_empty_m,cmpp_snippet_empty_m,cmpp_kvp_op_none} -static const cmpp_kvp cmpp_kvp_empty = cmpp_kvp_empty_m; - -/* -** Parses X or X=Y into p. Fails fatally on error. -** -** If nKey is negative then strlen() is used to calculate it. -** -** The third argument specifies whether/how to permit/treat the '=' -** part of X=Y. -*/ -static void cmpp_kvp_parse(cmpp_kvp * p, - char const *zKey, int nKey, - cmpp_kvp_op_e opPolicy); - -/* -** Wrapper around a FILE handle. -*/ -typedef struct FileWrapper FileWrapper; -struct FileWrapper { - /* File's name. */ - char const *zName; - /* FILE handle. */ - FILE * pFile; - /* Where FileWrapper_slurp() stores the file's contents. */ - unsigned char * zContent; - /* Size of this->zContent, as set by FileWrapper_slurp(). */ - unsigned nContent; - /* See Global::pFiles. */ - FileWrapper * pTail; -}; -#define FileWrapper_empty_m {0,0,0,0,0} -static const FileWrapper FileWrapper_empty = FileWrapper_empty_m; - -/* -** Proxy for FILE_close() and frees all memory owned by p. A no-op if -** p is already closed. -*/ -static void FileWrapper_close(FileWrapper * p); -/* Proxy for FILE_open(). Closes p first if it's currently opened. */ -static void FileWrapper_open(FileWrapper * p, const char * zName, const char *zMode); -/* Proxy for FILE_slurp(). */ -static void FileWrapper_slurp(FileWrapper * p); -/* -** If p->zContent ends in \n or \r\n, that part is replaced with 0 and -** p->nContent is adjusted. Returns true if it chomps, else false. -*/ -int FileWrapper_chomp(FileWrapper * p); - -/* -** Outputs a printf()-formatted message to stderr. -*/ -static void g_stderr(char const *zFmt, ...); -/* -** Outputs a printf()-formatted message to stderr. -*/ -static void g_stderrv(char const *zFmt, va_list); -#define g_debug(lvl,pfexpr) \ - if(lvl<=g.flags.doDebug) g_stderr("%s @ %s():%d: ",g.zArgv0,__func__,__LINE__); \ - if(lvl<=g.flags.doDebug) g_stderr pfexpr - -#define g_warn(zFmt,...) g_stderr("%s:%d %s() " zFmt "\n", __FILE__, __LINE__, __func__, __VA_ARGS__) -#define g_warn0(zMsg) g_stderr("%s:%d %s() %s\n", __FILE__, __LINE__, __func__, zMsg) - -void cmpp_free(void *p){ - sqlite3_free(p); -} - -void * cmpp_realloc(void * p, unsigned n){ - void * const rc = sqlite3_realloc(p, n); - if(!rc) fatal("realloc(P,%u) failed", n); - return rc; -} - -#if 0 -void * cmpp_malloc(unsigned n){ - void * const rc = sqlite3_alloc(n); - if(!rc) fatal("malloc(%u) failed", n); - return rc; -} -#endif - -FILE * FILE_open(char const *zName, const char * zMode){ - FILE * p; - if('-'==zName[0] && 0==zName[1]){ - p = strstr(zMode,"w") ? stdout : stdin; - }else{ - p = fopen(zName, zMode); - if(!p) fatal("Cannot open file [%s] with mode [%s]", zName, zMode); - } - return p; -} - -void FILE_close(FILE *p){ - if(p && p!=stdout && p!=stderr){ - fclose(p); - } -} - -void FILE_slurp(FILE *pFile, unsigned char **pOut, - unsigned * nOut){ - unsigned char zBuf[1024 * 8]; - unsigned char * pDest = 0; - unsigned nAlloc = 0; - unsigned nOff = 0; - /* Note that this needs to be able to work on non-seekable streams, - ** thus we read in chunks instead of doing a single alloc and - ** filling it in one go. */ - while( !feof(pFile) ){ - size_t const n = fread(zBuf, 1, sizeof(zBuf), pFile); - if(n>0){ - if(nAlloc < nOff + n + 1){ - nAlloc = nOff + n + 1; - pDest = cmpp_realloc(pDest, nAlloc); - } - memcpy(pDest + nOff, zBuf, n); - nOff += n; - } - } - if(pDest) pDest[nOff] = 0; - *pOut = pDest; - *nOut = nOff; -} - -void FileWrapper_close(FileWrapper * p){ - if(p->pFile) FILE_close(p->pFile); - if(p->zContent) cmpp_free(p->zContent); - *p = FileWrapper_empty; -} - -void FileWrapper_open(FileWrapper * p, const char * zName, - const char * zMode){ - FileWrapper_close(p); - p->pFile = FILE_open(zName, zMode); - p->zName = zName; -} - -void FileWrapper_slurp(FileWrapper * p){ - assert(!p->zContent); - assert(p->pFile); - FILE_slurp(p->pFile, &p->zContent, &p->nContent); -} - -int FileWrapper_chomp(FileWrapper * p){ - if( p->nContent && '\n'==p->zContent[p->nContent-1] ){ - p->zContent[--p->nContent] = 0; - if( p->nContent && '\r'==p->zContent[p->nContent-1] ){ - p->zContent[--p->nContent] = 0; - } - return 1; - } - return 0; -} - -enum CmppParseState { -TS_Start = 1, -TS_If, -TS_IfPassed, -TS_Else, -TS_Error -}; -typedef enum CmppParseState CmppParseState; -enum CmppTokenType { - -#define CmppToken_map(E) \ - E(Invalid,0) \ - E(Assert,"assert") \ - E(AtPolicy,"@policy") \ - E(Comment,"//") \ - E(Define,"define") \ - E(Elif,"elif") \ - E(Else,"else") \ - E(Endif,"endif") \ - E(Error,"error") \ - E(If,"if") \ - E(Include,"include") \ - E(Line,0) \ - E(Opaque,0) \ - E(Pragma,"pragma") \ - E(Savepoint,"savepoint") \ - E(Stderr,"stderr") \ - E(Undef,"undef") - -#define E(N,TOK) TT_ ## N, - CmppToken_map(E) -#undef E -}; -typedef enum CmppTokenType CmppTokenType; - -/* -** Map of directive (formerly keyword) names and their token types. -*/ -static const struct { -#define E(N,TOK) struct cmpp_snippet N; - CmppToken_map(E) -#undef E -} DStrings = { -#define E(N,TOK) .N = {TOK,sizeof(TOK)-1}, - CmppToken_map(E) -#undef E -}; - -//static -char const * TT_cstr(int tt){ - switch(tt){ -#define E(N,TOK) case TT_ ## N: return DStrings.N.z; - CmppToken_map(E) -#undef E - } - return NULL; -} - -struct CmppToken { - CmppTokenType ttype; - /* Line number of this token in the source file. */ - unsigned lineNo; - /* Start of the token. */ - unsigned char const * zBegin; - /* One-past-the-end byte of the token. */ - unsigned char const * zEnd; -}; -typedef struct CmppToken CmppToken; -#define CmppToken_empty_m {TT_Invalid,0,0,0} -static const CmppToken CmppToken_empty = CmppToken_empty_m; - -/* -** CmppLevel represents one "level" of tokenization, starting at the -** top of the main input, incrementing once for each level of `#if`, -** and decrementing for each `#endif`. -** pushes a level. -*/ -typedef struct CmppLevel CmppLevel; -struct CmppLevel { - unsigned short flags; - /* - ** Used for controlling which parts of an if/elif/...endif chain - ** should get output. - */ - unsigned short skipLevel; - /* The token which started this level (an 'if' or 'include'). */ - CmppToken token; - CmppParseState pstate; -}; -#define CmppLevel_empty_m {0U,0U,CmppToken_empty_m,TS_Start} -static const CmppLevel CmppLevel_empty = CmppLevel_empty_m; -enum CmppLevel_Flags { -/* Max depth of nested `#if` constructs in a single tokenizer. */ -CmppLevel_Max = 10, -/* Max number of keyword arguments. */ -CmppArgs_Max = 15, -/* Directive line buffer size */ -CmppArgs_BufSize = 1024, -/* Flag indicating that output for a CmpLevel should be elided. */ -CmppLevel_F_ELIDE = 0x01, -/* -** Mask of CmppLevel::flags which are inherited when CmppLevel_push() -** is used. -*/ -CmppLevel_F_INHERIT_MASK = CmppLevel_F_ELIDE -}; - -typedef struct CmppTokenizer CmppTokenizer; -typedef struct CmppKeyword CmppKeyword; -typedef void (*cmpp_keyword_f)(CmppKeyword const * pKw, CmppTokenizer * t); -struct CmppKeyword { - const char *zName; - unsigned nName; - int bTokenize; - CmppTokenType ttype; - cmpp_keyword_f xCall; -}; - -static CmppKeyword const * CmppKeyword_search(const char *zName); -static void cmpp_process_keyword(CmppTokenizer * const t); - -/* -** Tokenizer for c-pp input files. -*/ -struct CmppTokenizer { - const char * zName; /* Input (file) name for error reporting */ - unsigned const char * zBegin; /* start of input */ - unsigned const char * zEnd; /* one-after-the-end of input */ - unsigned const char * zPos; /* current position */ - unsigned int lineNo; /* line # of current pos */ - unsigned nSavepoint; - CmppParseState pstate; - CmppToken token; /* current token result */ - struct { - unsigned ndx; - CmppLevel stack[CmppLevel_Max]; - } level; - /* Args for use in cmpp_keyword_f() impls. */ - struct { - CmppKeyword const * pKw; - int argc; - const unsigned char * argv[CmppArgs_Max]; - unsigned char lineBuf[CmppArgs_BufSize]; - } args; -}; -#define CT_level(t) (t)->level.stack[(t)->level.ndx] -#define CT_pstate(t) CT_level(t).pstate -#define CT_skipLevel(t) CT_level(t).skipLevel -#define CLvl_skip(lvl) ((lvl)->skipLevel || ((lvl)->flags & CmppLevel_F_ELIDE)) -#define CT_skip(t) CLvl_skip(&CT_level(t)) -#define CmppTokenizer_empty_m { \ - .zName=0, .zBegin=0, .zEnd=0, \ - .zPos=0, \ - .lineNo=1U, \ - .pstate = TS_Start, \ - .token = CmppToken_empty_m, \ - .level = {0U,{CmppLevel_empty_m}}, \ - .args = {0,0,{0},{0}} \ - } -static const CmppTokenizer CmppTokenizer_empty = CmppTokenizer_empty_m; - -static void CmppTokenizer_cleanup(CmppTokenizer * const t); - -static void cmpp_t_out(CmppTokenizer * t, void const *z, unsigned int n); -/*static void cmpp_t_outf(CmppTokenizer * t, char const *zFmt, ...);*/ - -/* -** Pushes a new level into the given tokenizer. Fails fatally if -** it's too deep. -*/ -static void CmppLevel_push(CmppTokenizer * const t); -/* -** Pops a level from the tokenizer. Fails fatally if the top -** level is popped. -*/ -static void CmppLevel_pop(CmppTokenizer * const t); -/* -** Returns the current level object. -*/ -static CmppLevel * CmppLevel_get(CmppTokenizer * const t); - -/* -** Policies for how to handle undefined @tokens@ when performing -** content filtering. -*/ -enum AtPolicy { - AT_invalid = -1, - /** Turn off @foo@ parsing. */ - AT_OFF = 0, - /** Retain undefined @foo@ - emit it as-is. */ - AT_RETAIN, - /** Elide undefined @foo@. */ - AT_ELIDE, - /** Error for undefined @foo@. */ - AT_ERROR, - AT_DEFAULT = AT_ERROR -}; -typedef enum AtPolicy AtPolicy; - -static AtPolicy AtPolicy_fromStr(char const *z, int bEnforce){ - if( 0==strcmp(z, "retain") ) return AT_RETAIN; - if( 0==strcmp(z, "elide") ) return AT_ELIDE; - if( 0==strcmp(z, "error") ) return AT_ERROR; - if( 0==strcmp(z, "off") ) return AT_OFF; - if( bEnforce ){ - fatal("Invalid @ policy value: %s. " - "Try one of retain|elide|error|off.", z); - } - return AT_invalid; -} - -/* -** Global app state singleton. -*/ -static struct Global { - /* main()'s argv[0]. */ - const char * zArgv0; - /* App's db instance. */ - sqlite3 * db; - /* Current tokenizer (for error reporting purposes). */ - CmppTokenizer const * tok; - /* - ** We use a linked-list of these to keep track of our opened - ** files so that we can clean then up via atexit() in the case of - ** fatal error (to please valgrind). - */ - FileWrapper * pFiles; - /* Output channel. */ - FileWrapper out; - struct { - /* - ** Bytes of the keyword delimiter/prefix. Owned - ** elsewhere. - */ - const char * z; - /* Byte length of this->zDelim. */ - unsigned short n; - /* - ** The @token@ delimiter. - ** - ** Potential TODO is replace this with a pair of opener/closer - ** strings, e.g. "{{" and "}}". - */ - const unsigned char chAt; - } delim; - struct { -#define CMPP_SAVEPOINT_NAME "_cmpp_" -#define GStmt_map(E) \ - E(defIns,"INSERT OR REPLACE INTO def(k,v) VALUES(?,?)") \ - E(defDel,"DELETE FROM def WHERE k GLOB ?") \ - E(defHas,"SELECT 1 FROM def WHERE k GLOB ?") \ - E(defGet,"SELECT k,v FROM def WHERE k GLOB ?") \ - E(defGetBool, \ - "SELECT 1 FROM def WHERE k = ?1" \ - " AND v IS NOT NULL" \ - " AND '0'!=v AND ''!=v") \ - E(defSelAll,"SELECT k,v FROM def ORDER BY k") \ - E(inclIns,"INSERT OR FAIL INTO incl(file,srcFile," \ - "srcLine) VALUES(?,?,?)") \ - E(inclDel,"DELETE FROM incl WHERE file=?") \ - E(inclHas,"SELECT 1 FROM incl WHERE file=?") \ - E(inclPathAdd,"INSERT OR FAIL INTO " \ - "inclpath(seq,dir) VALUES(?,?)") \ - E(inclSearch, \ - "SELECT ?1 fn WHERE fileExists(fn) " \ - "UNION ALL SELECT * FROM (" \ - "SELECT replace(dir||'/'||?1, '//','/') AS fn " \ - "FROM inclpath WHERE fileExists(fn) ORDER BY seq"\ - ")") \ - E(spBegin,"SAVEPOINT " CMPP_SAVEPOINT_NAME) \ - E(spRollback,"ROLLBACK TO SAVEPOINT " \ - CMPP_SAVEPOINT_NAME) \ - E(spRelease,"RELEASE SAVEPOINT " CMPP_SAVEPOINT_NAME) - -#define E(N,S) sqlite3_stmt * N; - GStmt_map(E) -#undef E - } stmt; - struct { - FILE * pFile; - int expandSql; - } sqlTrace; - struct { - AtPolicy atPolicy; - /* If true, enables certain debugging output. */ - char doDebug; - /* If true, chomp() files read via -Fx=file. */ - char chompF; - } flags; -} g = { - .zArgv0 = "?", - .db = 0, - .tok = 0, - .pFiles = 0, - .out = FileWrapper_empty_m, - .delim = { - .z = CMPP_DEFAULT_DELIM, - .n = (unsigned short) sizeof(CMPP_DEFAULT_DELIM)-1, - .chAt = '@' - }, - .stmt = { - .defIns = 0, - .defDel = 0, - .defHas = 0, - .defGet = 0, - .defGetBool = 0, - .inclIns = 0, - .inclDel = 0, - .inclHas = 0, - .inclPathAdd = 0, - .inclSearch = 0 - }, - .sqlTrace = { - .pFile = 0, - .expandSql = 0 - }, - .flags = { - .atPolicy = AT_OFF, - .doDebug = 0, - .chompF = 0 - } -}; - -/** Distinct IDs for each g.stmt member. */ -enum GStmt_e { - GStmt_none = 0, -#define E(N,S) GStmt_ ## N, - GStmt_map(E) -#undef E -}; - -/* -** Returns the g.stmt.X corresponding to `which`, initializing it if -** needed. It does not return NULL - it fails fatally on error. -*/ -static sqlite3_stmt * g_stmt(enum GStmt_e which){ - sqlite3_stmt ** q = 0; - char const * zSql = 0; - switch(which){ - case GStmt_none: - fatal("GStmt_none is not a valid statement handle"); - return NULL; -#define E(N,S) case GStmt_ ## N: zSql = S; q = &g.stmt.N; break; - GStmt_map(E) -#undef E - } - assert( q ); - assert( zSql && *zSql ); - if( !*q ){ - db_prepare(q, "%s", zSql); - assert( *q ); - } - return *q; -} -static void g_stmt_reset(sqlite3_stmt * const q){ - sqlite3_clear_bindings(q); - sqlite3_reset(q); -} - -#if 0 -/* -** Outputs a printf()-formatted message to c-pp's global output -** channel. -*/ -static void g_outf(char const *zFmt, ...); -void g_outf(char const *zFmt, ...){ - va_list va; - va_start(va, zFmt); - vfprintf(g.out.pFile, zFmt, va); - va_end(va); -} -#endif - -/* Outputs n bytes from z to c-pp's global output channel. */ -static void g_out(void const *z, unsigned int n); -void g_out(void const *z, unsigned int n){ - if(g.out.pFile && 1!=fwrite(z, n, 1, g.out.pFile)){ - int const err = errno; - fatal("fwrite() output failed with errno #%d", err); - } -} - -void g_stderrv(char const *zFmt, va_list va){ - if( g.out.pFile==stdout ){ - fflush(g.out.pFile); - } - vfprintf(stderr, zFmt, va); -} - -void g_stderr(char const *zFmt, ...){ - va_list va; - va_start(va, zFmt); - g_stderrv(zFmt, va); - va_end(va); -} - -/* -** Emits n bytes of z if CT_skip(t) is false. -*/ -void cmpp_t_out(CmppTokenizer * t, void const *z, unsigned int n){ - g_debug(3,("CT_skipLevel() ?= %d\n",CT_skipLevel(t))); - g_debug(3,("CT_skip() ?= %d\n",CT_skip(t))); - if(!CT_skip(t)) g_out(z, n); -} - -void CmppLevel_push(CmppTokenizer * const t){ - CmppLevel * pPrev; - CmppLevel * p; - if(t->level.ndx+1 == (unsigned)CmppLevel_Max){ - fatal("%sif nesting level is too deep. Max=%d\n", - g.delim.z, CmppLevel_Max); - } - pPrev = &CT_level(t); - g_debug(3,("push from tokenizer level=%u flags=%04x\n", - t->level.ndx, pPrev->flags)); - p = &t->level.stack[++t->level.ndx]; - *p = CmppLevel_empty; - p->token = t->token; - p->flags = (CmppLevel_F_INHERIT_MASK & pPrev->flags); - if(CLvl_skip(pPrev)) p->flags |= CmppLevel_F_ELIDE; - g_debug(3,("push to tokenizer level=%u flags=%04x\n", - t->level.ndx, p->flags)); -} - -void CmppLevel_pop(CmppTokenizer * const t){ - if(!t->level.ndx){ - fatal("Internal error: CmppLevel_pop() at the top of the stack"); - } - g_debug(3,("pop from tokenizer level=%u, flags=%04x skipLevel?=%d\n", - t->level.ndx, - t->level.stack[t->level.ndx].flags, CT_skipLevel(t))); - g_debug(3,("CT_skipLevel() ?= %d\n",CT_skipLevel(t))); - g_debug(3,("CT_skip() ?= %d\n",CT_skip(t))); - t->level.stack[t->level.ndx--] = CmppLevel_empty; - g_debug(3,("pop to tokenizer level=%u, flags=%04x\n", t->level.ndx, - t->level.stack[t->level.ndx].flags)); - g_debug(3,("CT_skipLevel() ?= %d\n",CT_skipLevel(t))); - g_debug(3,("CT_skip() ?= %d\n",CT_skip(t))); -} - -CmppLevel * CmppLevel_get(CmppTokenizer * const t){ - return &t->level.stack[t->level.ndx]; -} - - -void db_affirm_rc(int rc, const char * zMsg){ - switch(rc){ - case 0: - case SQLITE_DONE: - case SQLITE_ROW: - break; - default: - assert( g.db ); - fatal("Db error #%d %s: %s", rc, zMsg, - sqlite3_errmsg(g.db)); - } -} - -int db_step(sqlite3_stmt *pStmt){ - int const rc = sqlite3_step(pStmt); - switch( rc ){ - case SQLITE_ROW: - case SQLITE_DONE: - break; - default: - db_affirm_rc(rc, "from db_step()"); - } - return rc; -} - -static sqlite3_str * db_str_new(void){ - sqlite3_str * rc = sqlite3_str_new(g.db); - if(!rc) fatal("Alloc failed for sqlite3_str_new()"); - return rc; -} - -static char * db_str_finish(sqlite3_str *s, int * n){ - int const rc = sqlite3_str_errcode(s); - if(rc) fatal("Error #%d from sqlite3_str_errcode()", rc); - if(n) *n = sqlite3_str_length(s); - char * z = sqlite3_str_finish(s); - if(!z) fatal("Alloc failed for sqlite3_str_new()"); - return z; -} - -void db_prepare(sqlite3_stmt **pStmt, const char * zSql, ...){ - int rc; - sqlite3_str * str = db_str_new(); - char * z = 0; - int n = 0; - va_list va; - if(!str) fatal("sqlite3_str_new() failed"); - va_start(va, zSql); - sqlite3_str_vappendf(str, zSql, va); - va_end(va); - rc = sqlite3_str_errcode(str); - if(rc) fatal("sqlite3_str_errcode() = %d", rc); - z = db_str_finish(str, &n); - rc = sqlite3_prepare_v2(g.db, z, n, pStmt, 0); - if(rc) fatal("Error #%d (%s) preparing: %s", - rc, sqlite3_errmsg(g.db), z); - sqlite3_free(z); -} - -void db_bind_int(sqlite3_stmt *pStmt, int col, int val){ - db_affirm_rc(sqlite3_bind_int(pStmt, col, val), - "from db_bind_int()"); -} - -void db_bind_null(sqlite3_stmt *pStmt, int col){ - db_affirm_rc(sqlite3_bind_null(pStmt, col), - "from db_bind_null()"); -} - -void db_bind_textn(sqlite3_stmt *pStmt, int col, - const char * zStr, int n){ - db_affirm_rc( - (zStr && n) - ? sqlite3_bind_text(pStmt, col, zStr, n, SQLITE_TRANSIENT) - : sqlite3_bind_null(pStmt, col), - "from db_bind_textn()" - ); -} - -void db_bind_text(sqlite3_stmt *pStmt, int col, - const char * zStr){ - db_bind_textn(pStmt, col, zStr, -1); -} - -#if 0 -void db_bind_textv(sqlite3_stmt *pStmt, int col, - const char * zFmt, ...){ - int rc; - sqlite3_str * str = db_str_new(); - int n = 0; - char * z; - va_list va; - va_start(va,zFmt); - sqlite3_str_vappendf(str, zFmt, va); - va_end(va); - z = db_str_finish(str, &n); - rc = sqlite3_bind_text(pStmt, col, z, n, sqlite3_free); - db_affirm_rc(rc,"from db_bind_textv()"); -} -#endif - -void db_free(void *m){ - sqlite3_free(m); -} - -void db_define_add(const char * zKey, char const *zVal){ - cmpp_kvp kvp = cmpp_kvp_empty; - cmpp_kvp_parse(&kvp, zKey, -1, - zVal - ? cmpp_key_op_none - : cmpp_key_op_eq1 - ); - if( kvp.v.z ){ - if( zVal ){ - assert(!"cannot happen - cmpp_key_op_none will prevent it"); - fatal("Cannot assign two values to [%.*s] [%.*s] [%s]", - kvp.k.n, kvp.k.z, kvp.v.n, kvp.v.z, zVal); - } - }else{ - kvp.v.z = zVal; - kvp.v.n = zVal ? (int)strlen(zVal) : 0; - } - sqlite3_stmt * const q = g_stmt(GStmt_defIns); - //g_stderr("zKey=%s\nzVal=%s\nzEq=%s\n", zKey, zVal, zEq); - db_bind_textn(q, 1, kvp.k.z, kvp.k.n); - if( kvp.v.z ){ - if( kvp.v.n ){ - db_bind_textn(q, 2, kvp.v.z, (int)kvp.v.n); - }else{ - db_bind_null(q, 2); - } - }else{ - db_bind_int(q, 2, 1); - } - db_step(q); - g_debug(2,("define: %s%s%s\n", - zKey, - zVal ? " with value " : "", - zVal ? zVal : "")); - sqlite3_reset(q); -} - -static void db_define_add_file(const char * zKey){ - cmpp_kvp kvp = cmpp_kvp_empty; - cmpp_kvp_parse(&kvp, zKey, -1, cmpp_kvp_op_eq1); - if( !kvp.v.z || !kvp.v.n ){ - fatal("Invalid filename: %s", zKey); - } - sqlite3_stmt * q = 0; - FileWrapper fw = FileWrapper_empty; - FileWrapper_open(&fw, kvp.v.z, "r"); - FileWrapper_slurp(&fw); - q = g_stmt(GStmt_defIns); - //g_stderr("zKey=%s\nzVal=%s\nzEq=%s\n", zKey, zVal, zEq); - db_bind_textn(q, 1, kvp.k.z, (int)kvp.k.n); - if( g.flags.chompF ){ - FileWrapper_chomp(&fw); - } - if( fw.nContent ){ - db_affirm_rc( - sqlite3_bind_text(q, 2, - (char const *)fw.zContent, - (int)fw.nContent, sqlite3_free), - "binding file content"); - fw.zContent = 0 /* transfered ownership */; - fw.nContent = 0; - }else{ - db_affirm_rc( sqlite3_bind_null(q, 2), - "binding empty file content"); - } - FileWrapper_close(&fw); - db_step(q); - g_stmt_reset(q); - g_debug(2,("define: %s%s%s\n", - kvp.k.z, - kvp.v.z ? " with value " : "", - kvp.v.z ? kvp.v.z : "")); -} - -#define ustr_c(X) ((unsigned char const *)X) - -static inline unsigned int cmpp_strlen(char const *z, int n){ - return n<0 ? (int)strlen(z) : (unsigned)n; -} - - -int db_define_has(const char * zName, int nName){ - int rc; - sqlite3_stmt * const q = g_stmt(GStmt_defHas); - nName = cmpp_strlen(zName, nName); - db_bind_textn(q, 1, zName, nName); - rc = db_step(q); - if(SQLITE_ROW == rc){ - rc = 1; - }else{ - assert(SQLITE_DONE==rc); - rc = 0; - } - g_debug(1,("defined [%s] ?= %d\n",zName, rc)); - g_stmt_reset(q); - return rc; -} - -int db_define_get_bool(const char * zName, int nName){ - sqlite3_stmt * const q = g_stmt(GStmt_defGetBool); - int rc = 0; - nName = cmpp_strlen(zName, nName); - db_bind_textn(q, 1, zName, nName); - rc = db_step(q); - if(SQLITE_ROW == rc){ - if( SQLITE_ROW==sqlite3_step(q) ){ - fatal("Key is ambiguous: %s", zName); - } - rc = 1; - }else{ - assert(SQLITE_DONE==rc); - rc = 0; - } - g_stmt_reset(q); - return rc; -} - -int db_define_get(const char * zName, int nName, - char **zVal, unsigned int *nVal){ - sqlite3_stmt * q = g_stmt(GStmt_defGet); - nName = cmpp_strlen(zName, nName); - db_bind_textn(q, 1, zName, nName); - int n = 0; - int rc = db_step(q); - if(SQLITE_ROW == rc){ - const unsigned char * z = sqlite3_column_text(q, 1); - n = sqlite3_column_bytes(q,1); - if( nVal ) *nVal = (unsigned)n; - *zVal = sqlite3_mprintf("%.*s", n, z); - if( n && z ) check__oom(*zVal); - if( SQLITE_ROW==sqlite3_step(q) ){ - db_free(*zVal); - *zVal = 0; - fatal("Key is ambiguous: %.*s\n", - nName, zName); - } - rc = 1; - }else{ - assert(SQLITE_DONE==rc); - rc = 0; - } - g_debug(1,("define [%.*s] ?= %d %.*s\n", - nName, zName, rc, - *zVal ? n : 0, - *zVal ? *zVal : "<NULL>")); - g_stmt_reset(q); - return rc; -} - -void db_define_rm(const char * zKey){ - int rc; - int n = 0; - sqlite3_stmt * const q = g_stmt(GStmt_defDel); - db_bind_text(q, 1, zKey); - rc = db_step(q); - if(SQLITE_DONE != rc){ - db_affirm_rc(rc, "Stepping DELETE on def"); - } - g_debug(2,("undefine: %.*s\n",n, zKey)); - g_stmt_reset(q); -} - -void db_including_add(const char * zKey, const char * zSrc, int srcLine){ - int rc; - sqlite3_stmt * const q = g_stmt(GStmt_inclIns); - db_bind_text(q, 1, zKey); - db_bind_text(q, 2, zSrc); - db_bind_int(q, 3, srcLine); - rc = db_step(q); - if(SQLITE_DONE != rc){ - db_affirm_rc(rc, "Stepping INSERT on incl"); - } - g_debug(2,("is-including-file add [%s] from [%s]:%d\n", zKey, zSrc, srcLine)); - g_stmt_reset(q); -} - -void db_include_rm(const char * zKey){ - int rc; - sqlite3_stmt * const q = g_stmt(GStmt_inclDel); - db_bind_text(q, 1, zKey); - rc = db_step(q); - if(SQLITE_DONE != rc){ - db_affirm_rc(rc, "Stepping DELETE on incl"); - } - g_debug(2,("inclpath rm [%s]\n", zKey)); - g_stmt_reset(q); -} - -char * db_include_search(const char * zKey){ - char * zName = 0; - sqlite3_stmt * const q = g_stmt(GStmt_inclSearch); - db_bind_text(q, 1, zKey); - if(SQLITE_ROW==db_step(q)){ - const unsigned char * z = sqlite3_column_text(q, 0); - zName = z ? sqlite3_mprintf("%s", z) : 0; - if(!zName) fatal("Alloc failed"); - } - g_stmt_reset(q); - return zName; -} - -static int db_including_has(const char * zName){ - int rc; - sqlite3_stmt * const q = g_stmt(GStmt_inclHas); - db_bind_text(q, 1, zName); - rc = db_step(q); - if(SQLITE_ROW == rc){ - rc = 1; - }else{ - assert(SQLITE_DONE==rc); - rc = 0; - } - g_debug(2,("inclpath has [%s] = %d\n",zName, rc)); - g_stmt_reset(q); - return rc; -} - -#if 0 -/* -** Fails fatally if the `#include` list contains the given key. -*/ -static void db_including_check(const char * zKey); -void db_including_check(const char * zName){ - if(db_including_has(zName)){ - fatal("Recursive include detected: %s\n", zName); - } -} -#endif - -void db_include_dir_add(const char * zDir){ - static int seq = 0; - int rc; - sqlite3_stmt * const q = g_stmt(GStmt_inclPathAdd); - db_bind_int(q, 1, ++seq); - db_bind_text(q, 2, zDir); - rc = db_step(q); - if(SQLITE_DONE != rc){ - db_affirm_rc(rc, "Stepping INSERT on inclpath"); - } - g_debug(2,("inclpath add #%d: %s\n",seq, zDir)); - g_stmt_reset(q); -} - -void g_FileWrapper_link(FileWrapper *fp){ - assert(!fp->pTail); - fp->pTail = g.pFiles; - g.pFiles = fp; -} - -void g_FileWrapper_close(FileWrapper *fp){ - assert(fp); - assert(fp->pTail || g.pFiles==fp); - g.pFiles = fp->pTail; - fp->pTail = 0; - FileWrapper_close(fp); -} - -static void g_cleanup(int bCloseFileChain){ - if( g.db ){ -#define E(N,S) sqlite3_finalize(g.stmt.N); g.stmt.N = 0; - GStmt_map(E) -#undef E - } - if( bCloseFileChain ){ - FileWrapper * fpNext = 0; - for( FileWrapper * fp=g.pFiles; fp; fp=fpNext ){ - fpNext = fp->pTail; - fp->pTail = 0; - FileWrapper_close(fp); - } - } - FileWrapper_close(&g.out); - if(g.db){ - sqlite3_close(g.db); - g.db = 0; - } -} - -static void cmpp_atexit(void){ - g_cleanup(1); -} - -int cmpp_is_legal_key(char const *zKey, int nKey, char const **zAt){ - char const * z = zKey; - nKey = cmpp_strlen(zKey, nKey); - if( !nKey ){ - if( zAt ) *zAt = z; - return 0; - } - char const * const zEnd = z ? z + nKey : NULL; - for( ; z < zEnd; ++z ){ - switch( (0x80 & *z) ? 0 : *z ){ - case 0: - case '_': - continue; - case '-': - case '.': - case '/': - case ':': - case '=': - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - if( z==zKey ) break; - continue; - default: - if( isalpha((int)*z) ) continue; - } - if( zAt ) *zAt = z; - return 0; - } - assert( z==zEnd ); - return 1; -} - -void cmpp_affirm_legal_key(char const *zKey, int nKey){ - char const *zAt = 0; - nKey = cmpp_strlen(zKey, nKey); - if( !cmpp_is_legal_key(zKey, nKey, &zAt) ){ - assert( zAt ); - fatal("Illegal character 0x%02x in key [%.*s]\n", - (int)*zAt, nKey, zKey); - } -} - -/* -** sqlite3 UDF which returns true if its argument refers to an -** accessible file, else false. -*/ -static void udf_file_exists( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const char *zName; - (void)(argc); /* Unused parameter */ - zName = (const char*)sqlite3_value_text(argv[0]); - if( zName==0 ) return; - sqlite3_result_int(context, 0==access(zName, 0)); -} - -/** - ** This sqlite3_trace_v2() callback outputs tracing info using - ** g.sqlTrace.pFile. -*/ -static int cmpp__db_sq3TraceV2(unsigned t,void*c,void*p,void*x){ - static unsigned int counter = 0; - switch(t){ - case SQLITE_TRACE_STMT:{ - FILE * const fp = g.sqlTrace.pFile; - if( fp ){ - char const * const zSql = (char const *)x; - char * const zExp = g.sqlTrace.expandSql - ? sqlite3_expanded_sql((sqlite3_stmt*)p) - : 0; - fprintf(fp, "SQL TRACE #%u: %s\n", - ++counter, zExp ? zExp : zSql); - sqlite3_free(zExp); - } - break; - } - } - return 0; -} - -/* Initialize g.db, failing fatally on error. */ -static void cmpp_initdb(void){ - int rc; - char * zErr = 0; - const char * zSchema = - "CREATE TABLE def(" - /* ^^^ defines */ - "k TEXT PRIMARY KEY NOT NULL," - "v TEXT DEFAULT NULL" - ") WITHOUT ROWID;" - "CREATE TABLE incl(" - /* ^^^ files currently being included */ - "file TEXT PRIMARY KEY NOT NULL," - "srcFile TEXT DEFAULT NULL," - "srcLine INTEGER DEFAULT 0" - ") WITHOUT ROWID;" - "CREATE TABLE inclpath(" - /* ^^^ include path */ - "seq INTEGER UNIQUE ON CONFLICT IGNORE, " - "dir TEXT PRIMARY KEY NOT NULL ON CONFLICT IGNORE" - ");" - "BEGIN;" - ; - assert(0==g.db); - if(g.db) return; - rc = sqlite3_open_v2(":memory:", &g.db, SQLITE_OPEN_READWRITE, 0); - if(rc) fatal("Error opening :memory: db."); - sqlite3_trace_v2(g.db, SQLITE_TRACE_STMT, cmpp__db_sq3TraceV2, 0); - rc = sqlite3_exec(g.db, zSchema, 0, 0, &zErr); - if(rc) fatal("Error initializing database: %s", zErr); - rc = sqlite3_create_function(g.db, "fileExists", 1, - SQLITE_UTF8|SQLITE_DIRECTONLY, 0, - udf_file_exists, 0, 0); - db_affirm_rc(rc, "UDF registration failed."); -} - -/* -** For position zPos, which must be in the half-open range -** [zBegin,zEnd), returns g.delim.n if it is at the start of a line and -** starts with g.delim.z, else returns 0. -*/ -//static -unsigned short cmpp_is_delim(unsigned char const *zBegin, - unsigned char const *zEnd, - unsigned char const *zPos){ - assert(zEnd>zBegin); - assert(zPos<zEnd); - assert(zPos>=zBegin); - if(zPos>zBegin && - ('\n'!=*(zPos - 1) - || ((unsigned)(zEnd - zPos) <= g.delim.n))){ - return 0; - }else if(0==memcmp(zPos, g.delim.z, g.delim.n)){ - return g.delim.n; - }else{ - return 0; - } -} - -static void cmpp_t_out_expand(CmppTokenizer * const t, - unsigned char const * zFrom, - unsigned int n); - -static inline int cmpp__isspace(int ch){ - return ' '==ch || '\t'==ch; -} - -static inline unsigned cmpp__strlenu(unsigned char const *z, int n){ - return n<0 ? (unsigned)strlen((char const *)z) : (unsigned)n; -} - -static inline void cmpp__skip_space_c( unsigned char const **p, - unsigned char const *zEnd ){ - unsigned char const * z = *p; - while( z<zEnd && cmpp__isspace(*z) ) ++z; - *p = z; -} - -#define ustr_c(X) ((unsigned char const *)X) -//#define ustr_nc(X) ((unsigned char *)X) - -/** - Scan [t->zPos,t->zEnd) for a derective delimiter. Emits any - non-delimiter output found along the way. - - This updtes t->zPos and t->lineNo as it goes. - - If a delimiter is found, it updates t->token and returns 0. - On no match returns 0. -*/ -static -int CmppTokenizer__delim_search(CmppTokenizer * const t){ - if(!t->zPos) t->zPos = t->zBegin; - if( t->zPos>=t->zEnd ){ - return 0; - } - assert( (t->zPos==t->zBegin || t->zPos[-1]=='\n') - && "Else we've mismanaged something."); - char const * const zD = g.delim.z; - unsigned short const nD = g.delim.n; - unsigned char const * const zEnd = t->zEnd; - unsigned char const * zLeft = t->zPos; - unsigned char const * z = zLeft; - - assert( 0==*zEnd && "Else we'll misinteract with strcspn()" ); - if( *zEnd ){ - fatal("Input must be NUL-terminated."); - return 0; - } -#define tflush \ - if(z>zEnd) z=zEnd; \ - if( z>zLeft ) { \ - cmpp_t_out_expand(t, zLeft, (unsigned)(z-zLeft)); \ - } zLeft = z - while(z < zEnd){ - size_t nNlTotal = 0; - unsigned char const * zNl; - size_t nNl2 = strcspn((char const *)z, "\n"); - zNl = (z + nNl2 >= zEnd ? zEnd : z + nNl2); - if( nNl2 >= CmppArgs_BufSize /* too long */ - //|| '\n'!=(char)*zNl /* end of input */ - /* ^^^ we have to accept a missing trailing EOL for the - sake of -e scripts. */ - ){ - /* we'd like to error out here, but only if we know we're - reading reading a directive line. */ - ++t->lineNo; - z = zNl + 1; - tflush; - continue; - } - nNlTotal += nNl2; - assert( '\n'==*zNl || !*zNl ); - assert( '\n'==*zNl || zNl==zEnd ); - //g_stderr("input: zNl=%d z=<<<%.*s>>>", (int)*zNl, (zNl-z), z); - unsigned char const * const zBOL = z; - cmpp__skip_space_c(&z, zNl); - if( z+nD < zNl && 0==memcmp(z, zD, nD) ){ - /* Found a directive delimiter. */ - if( zBOL!=z ){ - /* Do not emit space from the same line which preceeds a - delimiter */ - zLeft = z; - } - while( zNl>z && zNl<zEnd - && '\n'==*zNl && '\\'==zNl[-1] ){ - /* Backslash-escaped newline: extend the token - to consume it all. */ - ++t->lineNo; - ++zNl; - nNl2 = strcspn((char const *)zNl, "\n"); - if( !nNl2 ) break; - nNlTotal += nNl2; - zNl += nNl2; - } - assert( zNl<=zEnd && "Else our input was not NUL-terminated"); - if( nNlTotal >= CmppArgs_BufSize ){ - fatal("Directive line is too long (%u)", - (unsigned)(zNl-z)); - break; - } - tflush; - t->token.zBegin = z + nD; - t->token.zEnd = zNl; - cmpp__skip_space_c(&t->token.zBegin, t->token.zEnd); - t->token.ttype = TT_Line; - t->token.lineNo = t->lineNo++; - t->zPos = t->token.zEnd + 1; - if( 0 ){ - g_stderr("token=<<%.*s>>", (t->token.zEnd - t->token.zBegin), - t->token.zBegin); - } - return 1; - } - z = zNl+1; - ++t->lineNo; - tflush; - //g_stderr("line #%d no match\n",(int)t->lineNo); - } - tflush; - t->zPos = z; - return 0; -#undef tflush -} - -void cmpp_kvp_parse(cmpp_kvp * p, char const *zKey, int nKey, - cmpp_kvp_op_e opPolicy){ - char chEq = 0; - char opLen = 0; - *p = cmpp_kvp_empty; - p->k.z = zKey; - p->k.n = cmpp_strlen(zKey, nKey); - switch( opPolicy ){ - case cmpp_kvp_op_none: break; - case cmpp_kvp_op_eq1: - chEq = '='; - opLen = 1; - break; - default: - assert(!"don't use these yet"); - /* todo: ==, !=, <=, <, >, >= */ - chEq = '='; - opLen = 1; - break; - } - assert( chEq ); - p->op = cmpp_kvp_op_none; - const char * const zEnd = p->k.z + p->k.n; - for(const char * zPos = p->k.z ; *zPos && zPos<zEnd ; ++zPos) { - if( chEq==*zPos ){ - if( cmpp_kvp_op_none==opPolicy ){ - fatal("Illegal operator in key: %s", zKey); - } - p->op = cmpp_kvp_op_eq1; - p->k.n = (unsigned)(zPos - zKey); - zPos += opLen; - assert( zPos <= zEnd ); - p->v.z = zPos; - p->v.n = (unsigned)(zEnd - zPos); - break; - } - } - cmpp_affirm_legal_key(p->k.z, p->k.n); -} - -static void cmpp_t_out_expand(CmppTokenizer * const t, - unsigned char const * zFrom, - unsigned int n){ - unsigned char const *zLeft = zFrom; - unsigned char const * const zEnd = zFrom + n; - unsigned char const *z = AT_OFF==g.flags.atPolicy ? zEnd : zLeft; - unsigned char const chEol = (unsigned char)'\n'; - int state = 0 /* 0==looking for opening @ - ** 1==looking for closing @ */; - if( 0 ){ - g_warn("zLeft=%d %c", (int)*zLeft, *zLeft); - } -#define tflush \ - if(z>zEnd) z=zEnd; \ - if(zLeft<z){ cmpp_t_out(t, zLeft, (z-zLeft)); } zLeft = z - for( ; z<zEnd; ++z ){ - zLeft = z; - for( ;z<zEnd; ++z ){ - if( chEol==*z ){ - state = 0; - continue; - } - if( g.delim.chAt==*z ){ - if( 0==state ){ - tflush; - state = 1; - }else{ /* process chAt...chAt */ - char *zVal = 0; - unsigned int nVal = 0; - assert( 1==state ); - assert( g.delim.chAt==*zLeft ); - assert( zLeft<z ); - if( z==zLeft+1 ){ - tflush; - }else{ - char const *zKey = (char const*)zLeft+1; - int const nKey = (z-zLeft-1); - if( db_define_get(zKey, nKey, &zVal, &nVal) ){ - if( nVal ){ - cmpp_t_out(t, (unsigned char const*)zVal, nVal); - }else{ - /* Elide it */ - } - zLeft = z+1/*skip closing g.delim.chAt*/; - db_free(zVal); - }else{ - assert( !zVal ); - /* No match. Emit it as-is. */ - switch( g.flags.atPolicy ){ - case AT_RETAIN: - tflush; - break; - case AT_ERROR: - fatal("Undefined key: %c%.*s%c", - g.delim.chAt, nKey, zKey, g.delim.chAt ); - break; - case AT_ELIDE: - zLeft = z+1; - break; - case AT_invalid: - case AT_OFF: - fatal("Unhandled g.flags.atPolicy #%d!", - g.flags.atPolicy); - break; - } - } - state = 0; - } - }/* process chAt...chAt */ - }/*g.delim.chAt*/ - }/*per-line loop*/ - }/*outer loop*/ - tflush; -#undef tflush - return; -} - -/* -** Scans t to the next keyword line, emitting all input before that -** which is _not_ a keyword line unless it's elided due to being -** inside a block which elides its content. Returns 0 if no keyword -** line was found, in which case the end of the input has been -** reached, else returns a truthy value and sets up t's state for use -** with cmpp_process_keyword(), which should then be called. -*/ -static int cmpp_next_keyword_line(CmppTokenizer * const t){ - CmppToken * const tok = &t->token; - - assert(t->zBegin); - assert(t->zEnd > t->zBegin); - if(!t->zPos) t->zPos = t->zBegin; - t->args.pKw = 0; - t->args.argc = 0; - *tok = CmppToken_empty; - if( !CmppTokenizer__delim_search(t) ){ - return 0; - } - /* Split t->token into arguments for the line's keyword */ - int i, argc = 0, prevChar = 0; - const unsigned tokLen = (unsigned)(tok->zEnd - tok->zBegin); - unsigned char * zKwd; - unsigned char * zEsc; - unsigned char * zz; - - assert(TT_Line==tok->ttype); - g_debug(2,("token @ line %u len=%u [[[%.*s]]]\n", - tok->lineNo, tokLen, tokLen, tok->zBegin)); - zKwd = &t->args.lineBuf[0]; - memcpy(zKwd, tok->zBegin, tokLen); - memset(zKwd + tokLen, 0, sizeof(t->args.lineBuf) - tokLen); - for( zEsc = 0, zz = zKwd; *zz; ++zz ){ - /* Convert backslash-escaped newlines to whitespace */ - switch((int)*zz){ - case (int)'\\': - if(zEsc) zEsc = 0; - else zEsc = zz; - break; - case (int)'\n': - assert(zEsc && "Should not have an unescaped newline?"); - if(zEsc==zz-1){ - *zEsc = (unsigned char)' '; - /* FIXME?: memmove() lnBuf content one byte to the left here - ** to collapse backslash and newline into a single - ** byte. Also consider collapsing all leading space on the - ** next line. (Much later: or just collapse the output as we go, - ** effectively shrinking the line.) */ - } - zEsc = 0; - *zz = (unsigned char)' '; - break; - default: - zEsc = 0; - break; - } - } - t->args.argv[argc++] = zKwd; - for( zz = zKwd; *zz; ++zz ){ - if(isspace(*zz)){ - *zz = 0; - break; - } - } - t->args.pKw = CmppKeyword_search((char const *)zKwd); - if(!t->args.pKw){ - fatal("Unknown keyword '%s' at line %u\n", (char const *)zKwd, - tok->lineNo); - } - for( ++zz ; *zz && isspace(*zz); ++zz ){} - if(t->args.pKw->bTokenize){ - for( ; *zz; prevChar = *zz, ++zz ){ - /* Split string into word-shaped tokens. - ** TODO ?= quoted strings, for the sake of the - ** #error keyword. */ - if(isspace(*zz)){ - assert(zz!=zKwd && "Leading space was stripped earlier."); - *zz = 0; - }else{ - if(argc == (int)CmppArgs_Max){ - fatal("Too many arguments @ line %u: %.*s", - tok->lineNo, tokLen, tok->zBegin); - }else if(zz>zKwd && !prevChar){ - t->args.argv[argc++] = zz; - } - } - } - }else{ - /* Treat rest of line as one token */ - if(*zz) t->args.argv[argc++] = zz; - } - tok->ttype = t->args.pKw->ttype; - if(g.flags.doDebug>1){ - for(i = 0; i < argc; ++i){ - g_debug(0,("line %u arg #%d=%s\n", - tok->lineNo, i, - (char const *)t->args.argv[i])); - } - } - t->args.argc = argc; - return 1; -} - -/* Internal error reporting helper for cmpp_keyword_f() impls. */ -static CMPP_NORETURN void cmpp_kwd__err_(char const *zFile, int line, - CmppKeyword const * pKw, - CmppTokenizer const *t, - char const *zFmt, ...){ - va_list va; - g_stderr("%s @ %s line %u:", - pKw->zName, t->zName, t->token.lineNo); - va_start(va, zFmt); - g.tok = 0 /* stop fatalv__base() from duplicating the file info */; - fatalv__base(zFile, line, zFmt, va); - /* not reached */ - va_end(va); -} -#define cmpp_kwd__err(...) cmpp_kwd__err_(__FILE__,__LINE__, __VA_ARGS__) -#define cmpp_t__err(T,...) cmpp_kwd__err_(__FILE__,__LINE__, (T)->args.pKw, (T), __VA_ARGS__) - -/* No-op cmpp_keyword_f() impl. */ -static void cmpp_kwd_noop(CmppKeyword const * pKw, CmppTokenizer *t){ - (void)pKw; - (void)t; -} - -/* #error impl. */ -static void cmpp_kwd_error(CmppKeyword const * pKw, CmppTokenizer *t){ - if(CT_skip(t)) return; - else{ - assert(t->args.argc < 3); - const char *zBegin = t->args.argc>1 - ? (const char *)t->args.argv[1] : 0; - cmpp_t__err(t, "%s", zBegin ? zBegin : "(no additional info)"); - } -} - -/* Impl. for #define, #undef */ -static void cmpp_kwd_define(CmppKeyword const * pKw, CmppTokenizer *t){ - if(CT_skip(t)) return; - if(t->args.argc<2){ - cmpp_kwd__err(pKw, t, "Expecting one or more arguments"); - }else{ - int i = 1; - for( ; i < t->args.argc; ++i){ - char const * const zArg = (char const *)t->args.argv[i]; - cmpp_affirm_legal_key(zArg, -1); - if( TT_Define==pKw->ttype ){ - db_define_add( zArg, NULL ); - }else{ - db_define_rm( zArg ); - } - } - } -} - -static int cmpp_val_matches(char const *zGlob, char const *zRhs){ - return 0==sqlite3_strglob(zGlob, zRhs); -} - -typedef int (*cmpp_vcmp_f)(char const *zLhs, char const *zRhs); - -/* -** Accepts a key in the form X or X=Y. In the former case, it uses -** db_define_get_bool(kvp->k) to determine its truthiness, else it -** compares the kvp->v part to kvp->k's defined value to determine -** truthiness. -** -** Unless... -** -** If bCheckDefined is true is true then (A) it returns true if the -** value is defined and (B) fails fatally if given an X=Y-format key. -** -** Returns true if zKey evals to true, else false. -*/ -//static -int cmpp_kvp_truth(CmppKeyword const * const pKw, - CmppTokenizer const * const t, - cmpp_kvp const * const kvp, - int bCheckDefined){ - int buul = 0; - if( kvp->v.z ){ - if( bCheckDefined ){ - cmpp_kwd__err(pKw, t, "Value part is not legal for " - "is-defined checks: %.s", - kvp->k.n, kvp->k.z); - } - char * zVal = 0; - unsigned int nVal = 0; - buul = db_define_get(kvp->k.z, (int)kvp->k.n, &zVal, &nVal); - //g_debug(0,("checking key[%.*s]=%.*s\n", (zEq-zKey), zKey, nVal, zVal)); - if( kvp->v.n && nVal ){ - /* FIXME? do this with a query */ - /*g_debug(0,("if get-define [%.*s]=[%.*s] zValPart=%s\n", - (zEq-zKey), zKey, - nVal, zVal, zValPart));*/ - buul = cmpp_val_matches(kvp->v.z, zVal); - //g_debug(0,("buul=%d\n", buul)); - }else{ - assert( 0==kvp->v.n || 0==nVal ); - buul = kvp->v.n == nVal; - } - db_free(zVal); - }else{ - if( bCheckDefined ){ - buul = db_define_has(kvp->k.z, kvp->k.n); - }else{ - buul = db_define_get_bool(kvp->k.z, kvp->k.n); - } - } - return buul; -} - -#if 0 -/* -** A thin proxy for cmpp_kvp_truth(). -*/ -static int cmpp_key_truth(CmppKeyword const * pKw, - CmppTokenizer const * t, - char const *zKey, int bCheckDefined){ - cmpp_kvp kvp = cmpp_kvp_empty; - cmpp_kvp_parse(&kvp, zKey, -1, cmpp_kvp_op_eq1); - return cmpp_kvp_truth(pKw, t, &kvp, bCheckDefined); -} -#endif - -//static -cmpp_kvp_op_e cmpp_t_is_op(CmppTokenizer const * t, int arg){ - if( t->args.argc > arg ){ - char const * const z = (char const *)t->args.argv[arg]; -#define E(N,S) if( strcmp(S,z) ) return cmpp_kvp_op_ ## N; else - cmpp_kvp_op_map(E) -#undef E - if(0) {} - } - return cmpp_kvp_op_none; -} - -/* -** A single part of an #if-type expression. They are parsed from -** CmppTokenizer::args in this form: -** -** not* defined{0,1} key[=[value]] -*/ -struct CmppExprDef { - /* The key part of the input. */ - cmpp_kvp kvp; - struct { - int ndx; - int next; - } arg; - CmppTokenizer const * tizer; - /* Set to 0 or 1 depending how many "not" are parsed. */ - unsigned char bNegated; - /* Set to 1 if "defined" is parsed. */ - unsigned char bCheckDefined; -}; -typedef struct CmppExprDef CmppExprDef; -#define CmppExprDef_empty_m {cmpp_kvp_empty_m,{0,0},0,0,0} -static const CmppExprDef CmppExprDef_empty = CmppExprDef_empty_m; - -/* -** Evaluate cep to true or false and return that value: -** -** If cep->bCheckDefined, return the result of db_define_has(). -** -** Else if cep->kvp.v.z is not NULL then fetch the define's value -** and return the result of cmpp_val_matches(cep->kvp.v.z,thatValue). -** -** Else return the result of db_define_get_bool(). -** -** The returned result accounts for cep->bNegated. -*/ -static int CmppExprDef_eval(CmppExprDef const * cep){ - int buul = 0; - - if( cep->bCheckDefined ){ - assert( !cep->kvp.v.n ); - buul = db_define_has(cep->kvp.k.z, (int)cep->kvp.k.n); - }else if( cep->kvp.v.z ){ - unsigned nVal = 0; - char * zVal = 0; - buul = db_define_get(cep->kvp.k.z, cep->kvp.k.n, &zVal, &nVal); - if( nVal ){ - buul = cmpp_val_matches(cep->kvp.v.z, zVal); - } - db_free(zVal); - }else{ - buul = db_define_get_bool(cep->kvp.k.z, cep->kvp.k.n); - } - return cep->bNegated ? !buul : buul; -} - -/* -** Expects t->args, starting at t->args.argv[startArg], to parse to -** one CmmpExprDef. It clears cep and repopulates it with info about -** the parse. Fails fatally on a parse error. -** -** Returns true if it reads one, false if it doesn't, and fails fatally -** if what it tries to parse is not empty but is not a CmppExprDef. -** -** Specifically, it parses: -** -** not+ defined? Word[=value] -** -*/ -static int CmppExprDef_read_one(CmppKeyword const * pKw, - CmppTokenizer const * t, - int startArg, CmppExprDef * cep){ - char const *zKey = 0; - *cep = CmppExprDef_empty; - cep->arg.ndx = startArg; - assert( t->args.pKw ); - assert( t->args.pKw==pKw ); - cep->tizer = t; - for(int i = startArg; !zKey && i<t->args.argc; ++i ){ - char const * z = (char const *)t->args.argv[i]; - if( 0==strcmp(z, "not") ){ - cep->bNegated = !cep->bNegated; - }else if( 0==strcmp(z,"defined") ){ - if( cep->bCheckDefined ){ - cmpp_kwd__err(pKw, t, - "Cannot use 'defined' more than once"); - } - cep->bCheckDefined = 1; - }else{ - assert( !zKey ); - cmpp_kvp_parse(&cep->kvp, z, -1, cmpp_kvp_op_eq1); - if( cep->bCheckDefined && cep->kvp.v.z ){ - cmpp_kwd__err(pKw, t, "Cannot use X=Y keys with 'defined'"); - cep->arg.next = ++i; - } - return 1; - } - } - return 0; -} - -/* -** Evals pStart and then proceeds to process any remaining arguments -** in t->args as RHS expressions. Returns the result of the expression -** as a bool. -** -** Specifically, it parses: -** -** and|or CmppExprDef -** -** Where CmppExprDef is the result of CmppExprDef_read_one(). -*/ -static int CmppExprDef_parse_cond(CmppKeyword const *pKw, - CmppTokenizer *t, - CmppExprDef const * pStart){ - enum { Op_none = 0, Op_And, Op_Or }; - int lhs = CmppExprDef_eval(pStart); - int op = Op_none; - int i = pStart->arg.next; - for( ; i < t->args.argc; ++i ){ - CmppExprDef eNext = CmppExprDef_empty; - char const *z = (char const *)t->args.argv[i]; - if( 0==strcmp("and",z) ){ - if( Op_none!=op ) goto multiple_ops; - op = Op_And; - continue; - }else if( 0==strcmp("or",z) ){ - if( Op_none!=op ) goto multiple_ops; - op = Op_Or; - continue; - }else if( !CmppExprDef_read_one(pKw, t, i, &eNext) ){ - if( Op_none!=op ){ - cmpp_t__err(t, "Stray operator: %s",z); - } - } - assert( eNext.kvp.k.z ); - int const rhs = CmppExprDef_eval(&eNext); - switch( op ){ - case Op_none: break; - case Op_And: lhs = lhs && rhs; break; - case Op_Or: lhs = lhs || rhs; break; - default: - assert(!"cannot happen"); - fatal("this cannot happen"); - } - op = Op_none; - } - if( Op_none!=op ){ - cmpp_t__err(t, "Extra operator at end of expression"); - }else if( i < t->args.argc ){ - assert(!"cannot happen"); - cmpp_kwd__err(t->args.pKw, t, "Unhandled extra arguments"); - }else{ - return lhs; - } - assert(!"not reached"); -multiple_ops: - cmpp_t__err(t,"Cannot have multiple operators"); - return 0 /* not reached */; -} - -/* Impl. for #if, #elif, #assert. */ -static void cmpp_kwd_if(CmppKeyword const * pKw, CmppTokenizer *t){ - CmppParseState tmpState = TS_Start; - CmppExprDef cep = CmppExprDef_empty; - //int buul = 0; - assert( TT_If==pKw->ttype - || TT_Elif==pKw->ttype - || TT_Assert==pKw->ttype); - if(t->args.argc<2){ - cmpp_kwd__err(pKw, t, "Expecting an argument"); - } - CmppExprDef_read_one(pKw, t, 1, &cep); - if( !cep.kvp.k.z ){ - cmpp_kwd__err(pKw, t, "Missing key argument"); - } - /*g_debug(0,("%s %s level %u pstate=%d bNot=%d bCheckDefined=%d\n", - pKw->zName, zKey, t->level.ndx, (int)CT_pstate(t), - bNot, bCheckDefined));*/ - switch(pKw->ttype){ - case TT_Assert: - break; - case TT_Elif: - switch(CT_pstate(t)){ - case TS_If: break; - case TS_IfPassed: CT_level(t).flags |= CmppLevel_F_ELIDE; return; - default: - cmpp_kwd__err(pKw, t, "'%s' used out of context", - pKw->zName); - } - break; - case TT_If: - CmppLevel_push(t); - break; - default: - assert(!"cannot happen"); - cmpp_kwd__err(pKw, t, "Unexpected keyword token type"); - break; - } - if( CmppExprDef_parse_cond( pKw, t, &cep ) ){ - CT_pstate(t) = tmpState = TS_IfPassed; - CT_skipLevel(t) = 0; - }else{ - if( TT_Assert==pKw->ttype ){ - cmpp_kwd__err(pKw, t, "Assertion failed: %s", - /* fixme: emit the whole line. We don't have it - handy in a readily-printable form. */ - cep.kvp.k.z); - } - CT_pstate(t) = TS_If /* also for TT_Elif */; - CT_skipLevel(t) = 1; - g_debug(3,("setting CT_skipLevel = 1 @ level %d\n", t->level.ndx)); - } - if( TT_If==pKw->ttype ){ - unsigned const lvlIf = t->level.ndx; - CmppToken const lvlToken = CT_level(t).token; - while(cmpp_next_keyword_line(t)){ - cmpp_process_keyword(t); - if(lvlIf > t->level.ndx){ - assert(TT_Endif == t->token.ttype); - break; - } -#if 0 - if(TS_IfPassed==tmpState){ - tmpState = TS_Start; - t->level.stack[lvlIf].flags |= CmppLevel_F_ELIDE; - g_debug(1,("Setting ELIDE for TS_IfPassed @ lv %d (lvlIf=%d)\n", t->level.ndx, lvlIf)); - } -#endif - } - if(lvlIf <= t->level.ndx){ - cmpp_kwd__err(pKw, t, - "Input ended inside an unterminated %sif " - "opened at [%s] line %u", - g.delim.z, t->zName, lvlToken.lineNo); - } - } -} - -/* Impl. for #else. */ -static void cmpp_kwd_else(CmppKeyword const * pKw, CmppTokenizer *t){ - if(t->args.argc>1){ - cmpp_kwd__err(pKw, t, "Expecting no arguments"); - } - switch(CT_pstate(t)){ - case TS_IfPassed: CT_skipLevel(t) = 1; break; - case TS_If: CT_skipLevel(t) = 0; break; - default: - cmpp_kwd__err(pKw, t, "'%s' with no matching 'if'", - pKw->zName); - } - /*g_debug(0,("else flags=0x%02x skipLevel=%u\n", - CT_level(t).flags, CT_level(t).skipLevel));*/ - CT_pstate(t) = TS_Else; -} - -/* Impl. for #endif. */ -static void cmpp_kwd_endif(CmppKeyword const * pKw, CmppTokenizer *t){ - /* Maintenance reminder: we ignore all arguments after the endif - ** to allow for constructs like: - ** - ** #endif // foo - ** - ** in a manner which does not require a specific comment style */ - switch(CT_pstate(t)){ - case TS_Else: - case TS_If: - case TS_IfPassed: - break; - default: - cmpp_kwd__err(pKw, t, "'%s' with no matching 'if'", - pKw->zName); - } - CmppLevel_pop(t); -} - -/* Impl. for #include. */ -static void cmpp_kwd_include(CmppKeyword const * pKw, CmppTokenizer *t){ - char const * zFile; - char * zResolved; - if(CT_skip(t)) return; - else if(t->args.argc!=2){ - cmpp_kwd__err(pKw, t, "Expecting exactly 1 filename argument"); - } - zFile = (const char *)t->args.argv[1]; - if(db_including_has(zFile)){ - /* Note that different spellings of the same filename - ** will elude this check, but that seems okay, as different - ** spellings means that we're not re-running the exact same - ** invocation. We might want some other form of multi-include - ** protection, rather than this, however. There may well be - ** sensible uses for recursion. */ - cmpp_t__err(t, "Recursive include of file: %s", zFile); - } - zResolved = db_include_search(zFile); - if(zResolved){ - db_including_add(zFile, t->zName, t->token.lineNo); - cmpp_process_file(zResolved); - db_include_rm(zFile); - db_free(zResolved); - }else{ - cmpp_t__err(t, "file not found: %s", zFile); - } -} - - -static void cmpp_dump_defines( FILE * fp, int bIndent ){ - sqlite3_stmt * const q = g_stmt(GStmt_defSelAll); - while( SQLITE_ROW==sqlite3_step(q) ){ - unsigned char const * zK = sqlite3_column_text(q, 0); - unsigned char const * zV = sqlite3_column_text(q, 1); - int const nK = sqlite3_column_bytes(q, 0); - int const nV = sqlite3_column_bytes(q, 1); - fprintf(fp, "%s%.*s = %.*s\n", - bIndent ? "\t" : "", nK, zK, nV, zV); - } - g_stmt_reset(q); -} - -/* Impl. for #pragma. */ -static void cmpp_kwd_pragma(CmppKeyword const * pKw, CmppTokenizer *t){ - const char * zArg; - if(CT_skip(t)) return; - else if(t->args.argc<2){ - cmpp_kwd__err(pKw, t, "Expecting an argument"); - } - zArg = (const char *)t->args.argv[1]; -#define M(X) 0==strcmp(zArg,X) - if(M("defines")){ - cmpp_dump_defines(stderr, 1); - } - else if(M("chomp-F")){ - g.flags.chompF = 1; - }else if(M("no-chomp-F")){ - g.flags.chompF = 0; - } -#if 0 - /* now done by cmpp_kwd_at_policy() */ - else if(M("@")){ - if(t->args.argc>2){ - g.flags.atPolicy = - AtPolicy_fromStr((char const *)t->args.argv[2], 1); - }else{ - g.flags.atPolicy = AT_DEFAULT; - } - }else if(M("no-@")){ - g.flags.atPolicy = AT_OFF; - } -#endif - else{ - cmpp_kwd__err(pKw, t, "Unknown pragma: %s", zArg); - } -#undef M -} - -static void db_step_reset(sqlite3_stmt * const q, char const * zErrTip){ - db_affirm_rc(sqlite3_step(q), zErrTip); - g_stmt_reset(q); -} - -static void cmpp_sp_begin(CmppTokenizer * const t){ - db_step_reset(g_stmt(GStmt_spBegin), "Starting savepoint"); - ++t->nSavepoint; -} - -static void cmpp_sp_rollback(CmppTokenizer * const t){ - if( !t->nSavepoint ){ - cmpp_t__err(t, "Cannot roll back: no active savepoint"); - } - db_step_reset(g_stmt(GStmt_spRollback), - "Rolling back savepoint"); - db_step_reset(g_stmt(GStmt_spRelease), - "Releasing rolled-back savepoint"); - --t->nSavepoint; -} - -static void cmpp_sp_commit(CmppTokenizer * const t){ - if( !t->nSavepoint ){ - cmpp_t__err(t, "Cannot commit: no active savepoint"); - } - db_step_reset(g_stmt(GStmt_spRelease), "Rolling back savepoint"); - --t->nSavepoint; -} - -void CmppTokenizer_cleanup(CmppTokenizer * const t){ - while( t->nSavepoint ){ - cmpp_sp_rollback(t); - } -} - -/* Impl. for #savepoint. */ -static void cmpp_kwd_savepoint(CmppKeyword const * pKw, CmppTokenizer *t){ - const char * zArg; - if(CT_skip(t)) return; - else if(t->args.argc!=2){ - cmpp_kwd__err(pKw, t, "Expecting one argument"); - } - zArg = (const char *)t->args.argv[1]; -#define M(X) 0==strcmp(zArg,X) - if(M("begin")){ - cmpp_sp_begin(t); - }else if(M("rollback")){ - cmpp_sp_rollback(t); - }else if(M("commit")){ - cmpp_sp_commit(t); - }else{ - cmpp_kwd__err(pKw, t, "Unknown savepoint option: %s", zArg); - } -#undef SP_NAME -#undef M -} - -/* #stder impl. */ -static void cmpp_kwd_stderr(CmppKeyword const * pKw, CmppTokenizer *t){ - if(CT_skip(t)) return; - else{ - const char *zBegin = t->args.argc>1 - ? (const char *)t->args.argv[1] : 0; - if(zBegin){ - g_stderr("%s:%u: %s\n", t->zName, t->token.lineNo, zBegin); - }else{ - g_stderr("%s:%u: (no %.*s%s argument)\n", - t->zName, t->token.lineNo, - g.delim.n, g.delim.z, pKw->zName); - } - } -} - -/* Impl. for the @ policy. */ -static void cmpp_kwd_at_policy(CmppKeyword const * pKw, CmppTokenizer *t){ - if(CT_skip(t)) return; - else if(t->args.argc<2){ - g.flags.atPolicy = AT_DEFAULT; - }else{ - g.flags.atPolicy = AtPolicy_fromStr((char const*)t->args.argv[1], 1); - } -} - - -#if 0 -/* Impl. for dummy placeholder. */ -static void cmpp_kwd_todo(CmppKeyword const * pKw, CmppTokenizer *t){ - (void)t; - g_debug(0,("TODO: keyword handler for %s\n", pKw->zName)); -} -#endif - -CmppKeyword aKeywords[] = { -/* Keep these sorted by zName */ -#define S(NAME) DStrings.NAME.z, DStrings.NAME.n - {S(Comment), 0, TT_Comment, cmpp_kwd_noop}, - {S(AtPolicy), 1, TT_AtPolicy, cmpp_kwd_at_policy}, - {S(Assert),1, TT_Assert, cmpp_kwd_if}, - {S(Define), 1, TT_Define, cmpp_kwd_define}, - {S(Elif), 1, TT_Elif, cmpp_kwd_if}, - {S(Else), 1, TT_Else, cmpp_kwd_else}, - {S(Endif), 0, TT_Endif, cmpp_kwd_endif}, - {S(Error), 0, TT_Error, cmpp_kwd_error}, - {S(If), 1, TT_If, cmpp_kwd_if}, - {S(Include), 0, TT_Include, cmpp_kwd_include}, - {S(Pragma), 1, TT_Pragma, cmpp_kwd_pragma}, - {S(Savepoint), 1, TT_Savepoint, cmpp_kwd_savepoint}, - {S(Stderr), 0, TT_Stderr, cmpp_kwd_stderr}, - {S(Undef), 1, TT_Undef, cmpp_kwd_define}, -#undef S - {0,0,TT_Invalid, 0} -}; - -static int cmpp_CmppKeyword(const void *p1, const void *p2){ - char const * zName = (const char *)p1; - CmppKeyword const * kw = (CmppKeyword const *)p2; - return strcmp(zName, kw->zName); -} - -CmppKeyword const * CmppKeyword_search(const char *zName){ - return (CmppKeyword const *)bsearch(zName, &aKeywords[0], - sizeof(aKeywords)/sizeof(aKeywords[0]) - 1, - sizeof(aKeywords[0]), - cmpp_CmppKeyword); -} - -void cmpp_process_keyword(CmppTokenizer * const t){ - assert(t->args.pKw); - assert(t->args.argc); - t->args.pKw->xCall(t->args.pKw, t); - t->args.pKw = 0; - t->args.argc = 0; -} - -void cmpp_process_string(const char * zName, - unsigned char const * zIn, - int nIn){ - nIn = cmpp__strlenu(zIn, nIn); - if( !nIn ) return; - CmppTokenizer const * const oldTok = g.tok; - CmppTokenizer ct = CmppTokenizer_empty; - ct.zName = zName; - ct.zBegin = zIn; - ct.zEnd = zIn + nIn; - while(cmpp_next_keyword_line(&ct)){ - cmpp_process_keyword(&ct); - } - if(0!=ct.level.ndx){ - CmppLevel const * const lv = CmppLevel_get(&ct); - fatal("Input ended inside an unterminated nested construct " - "opened at [%s] line %u", zName, lv->token.lineNo); - } - CmppTokenizer_cleanup(&ct); - g.tok = oldTok; -} - -void cmpp_process_file(const char * zName){ - FileWrapper fw = FileWrapper_empty; - FileWrapper_open(&fw, zName, "r"); - g_FileWrapper_link(&fw); - FileWrapper_slurp(&fw); - g_debug(1,("Read %u byte(s) from [%s]\n", fw.nContent, fw.zName)); - if( fw.zContent ){ - cmpp_process_string(zName, fw.zContent, fw.nContent); - } - g_FileWrapper_close(&fw); -} - - -void fatalv__base(char const *zFile, int line, - char const *zFmt, va_list va){ - FILE * const fp = stderr; - fflush(stdout); - fputc('\n', fp); - if( g.flags.doDebug ){ - fprintf(fp, "%s: ", g.zArgv0); - if( zFile ){ - fprintf(fp, "%s:%d ",zFile, line); - } - } - if( g.tok ){ - fprintf(fp,"@%s:%d: ", - (g.tok->zName && 0==strcmp("-",g.tok->zName)) - ? "<stdin>" - : g.tok->zName, - g.tok->lineNo); - } - if(zFmt && *zFmt){ - vfprintf(fp, zFmt, va); - } - fputc('\n', fp); - fflush(fp); - exit(1); -} - -void fatal__base(char const *zFile, int line, - char const *zFmt, ...){ - va_list va; - va_start(va, zFmt); - fatalv__base(zFile, line, zFmt, va); - va_end(va); -} - -#undef CT_level -#undef CT_pstate -#undef CT_skipLevel -#undef CT_skip -#undef CLvl_skip - -static void usage(int isErr){ - FILE * const fOut = isErr ? stderr : stdout; - fprintf(fOut, "Usage: %s [flags] [infile...]\n", g.zArgv0); - fprintf(fOut, - "Flags and filenames may be in any order and " - "they are processed in that order.\n" - "\nFlags:\n"); -#define GAP " " -#define arg(F,D) fprintf(fOut,"\n %s\n" GAP "%s\n",F, D) - arg("-o|--outfile FILE","Send output to FILE (default=- (stdout)).\n" - GAP "Because arguments are processed in order, this should\n" - GAP "normally be given before -f."); - arg("-f|--file FILE","Process FILE (default=- (stdin)).\n" - GAP "All non-flag arguments are assumed to be the input files."); - arg("-DXYZ[=value]","Define XYZ to the given value (default=1)."); - arg("-UXYZ","Undefine all defines matching glob XYZ."); - arg("-IXYZ","Add dir XYZ to the " CMPP_DEFAULT_DELIM "include path."); - arg("-FXYZ=filename", - "Define XYZ to the raw contents of the given file.\n" - GAP "The file is not processed as by " CMPP_DEFAULT_DELIM"include\n" - GAP "Maybe it should be. Or maybe we need a new flag for that."); - arg("-d|--delimiter VALUE", "Set keyword delimiter to VALUE " - "(default=" CMPP_DEFAULT_DELIM ")."); - arg("--@policy retain|elide|error|off", - "Specifies how to handle @tokens@ (default=off).\n" - GAP "off = do not look for @tokens@\n" - GAP "retain = parse @tokens@ and retain any undefined ones\n" - GAP "elide = parse @tokens@ and elide any undefined ones\n" - GAP "error = parse @tokens@ and error out for any undefined ones" - ); - arg("-@", "Equivalent to --@policy=error."); - arg("-no-@", "Equivalent to --@policy=off (the default)."); - arg("--sql-trace", "Send a trace of all SQL to stderr."); - arg("--sql-trace-x", - "Like --sql-trace but expand all bound values in the SQL."); - arg("--no-sql-trace", "Disable SQL tracing (default)."); - arg("--chomp-F", "One trailing newline is trimmed from files " - "read via -FXYZ=filename."); - arg("--no-chomp-F", "Disable --chomp-F (default)."); -#undef arg -#undef GAP - fputs("\nFlags which require a value accept either " - "--flag=value or --flag value.\n\n",fOut); -} - -/* -** Expects that *ndx points to the current argv entry and that it is a -** flag which expects a value. This function checks for --flag=val and -** (--flag val) forms. If a value is found then *ndx is adjusted (if -** needed) to point to the next argument after the value and *zVal is -** pointed to the value. If no value is found then it fails fatally. -*/ -static void get_flag_val(int argc, char const * const * argv, int * ndx, - char const **zVal){ - char const * zEq = strchr(argv[*ndx], '='); - if( zEq ){ - *zVal = zEq+1; - return; - } - if(*ndx+1>=argc){ - fatal("Missing value for flag '%s'", argv[*ndx]); - } - *zVal = argv[++*ndx]; -} - -static int arg_is_flag( char const *zFlag, char const *zArg, - char const **zValIfEqX ){ - *zValIfEqX = 0; - if( 0==strcmp(zFlag, zArg) ) return 1; - char const * z = strchr(zArg,'='); - if( z && z>zArg ){ - /* compare the part before the '=' */ - if( 0==strncmp(zFlag, zArg, z-zArg) ){ - if( !zFlag[z-zArg] ){ - *zValIfEqX = z+1; - return 1; - } - /* Else it was a prefix match. */ - } - } - return 0; -} - -int main(int argc, char const * const * argv){ - int rc = 0; - int inclCount = 0; - int nFile = 0; - int ndxTrace = 0; - int expandMode = 0; - char const * zVal = 0; -#define ARGVAL if( !zVal ) get_flag_val(argc, argv, &i, &zVal) -#define M(X) arg_is_flag(X, zArg, &zVal) -#define ISFLAG(X) else if(M(X)) -#define ISFLAG2(X,Y) else if(M(X) || M(Y)) -#define NOVAL if( zVal ) fatal("Unexpected value for %s", zArg) -#define g_out_open \ - if(!g.out.pFile) FileWrapper_open(&g.out, "-", "w"); \ - if(!inclCount){ db_include_dir_add("."); ++inclCount; } (void)0 - - g.zArgv0 = argv[0]; -#define DOIT if(doIt) - for(int doIt = 0; doIt<2; ++doIt){ - /** - Loop through the flags twice. The first time we just validate - and look for --help/-?. The second time we process the flags. - This approach allows us to easily chain multiple files and - flags: - - ./c-pp -Dfoo -o foo x.y -Ufoo -Dbar -o bar x.y - */ - DOIT{ - atexit(cmpp_atexit); - if( 1==ndxTrace ){ - /* Ensure that we start with tracing in the early stage if - --sql-trace is the first arg, in order to log schema - setup. */ - g.sqlTrace.pFile = stderr; - g.sqlTrace.expandSql = expandMode; - } - cmpp_initdb(); - } - for(int i = 1; i < argc; ++i){ - int negate = 0; - char const * zArg = argv[i]; - //g_stderr("i=%d zArg=%s\n", i, zArg); - zVal = 0; - while('-'==*zArg) ++zArg; - if(zArg==argv[i]/*not a flag*/){ - zVal = zArg; - goto do_infile; - } - if( 0==strncmp(zArg,"no-",3) ){ - zArg += 3; - negate = 1; - } - ISFLAG2("?","help"){ - NOVAL; - usage(0); - goto end; - }else if('D'==*zArg){ - ++zArg; - if(!*zArg) fatal("Missing key for -D"); - DOIT { - db_define_add(zArg, 0); - } - }else if('F'==*zArg){ - ++zArg; - if(!*zArg) fatal("Missing key for -F"); - DOIT { - db_define_add_file(zArg); - } - }else if('U'==*zArg){ - ++zArg; - if(!*zArg) fatal("Missing key for -U"); - DOIT { - db_define_rm(zArg); - } - }else if('I'==*zArg){ - ++zArg; - if(!*zArg) fatal("Missing directory for -I"); - DOIT { - db_include_dir_add(zArg); - ++inclCount; - } - } - ISFLAG2("o","outfile"){ - ARGVAL; - DOIT { - FileWrapper_open(&g.out, zVal, "w"); - } - } - ISFLAG2("f","file"){ - ARGVAL; - do_infile: - DOIT { - ++nFile; - g_out_open; - cmpp_process_file(zVal); - } - } - ISFLAG("e"){ - ARGVAL; - DOIT { - ++nFile; - g_out_open; - cmpp_process_string("-e script", ustr_c(zVal), -1); - } - } - ISFLAG("@"){ - NOVAL; - DOIT { - assert( AT_DEFAULT!=AT_OFF ); - g.flags.atPolicy = negate ? AT_OFF : AT_DEFAULT; - } - } - ISFLAG("@policy"){ - AtPolicy aup; - ARGVAL; - aup = AtPolicy_fromStr(zVal, 1); - DOIT { - g.flags.atPolicy = aup; - } - } - ISFLAG("debug"){ - NOVAL; - g.flags.doDebug += negate ? -1 : 1; - } - ISFLAG("sql-trace"){ - NOVAL; - /* Needs to be set before the start of the second pass, when - the db is inited. */ - g.sqlTrace.expandSql = 0; - DOIT { - g.sqlTrace.pFile = negate ? (FILE*)0 : stderr; - }else if( !ndxTrace && !negate ){ - ndxTrace = i; - expandMode = 0; - } - } - ISFLAG("sql-trace-x"){ - NOVAL; - g.sqlTrace.expandSql = 1; - DOIT { - g.sqlTrace.pFile = negate ? (FILE*)0 : stderr; - }else if( !ndxTrace && !negate ){ - ndxTrace = i; - expandMode = 1; - } - } - ISFLAG("chomp-F"){ - NOVAL; - DOIT g.flags.chompF = !negate; - } - ISFLAG2("d","delimiter"){ - ARGVAL; - if( !doIt ){ - g.delim.z = zVal; - g.delim.n = (unsigned short)strlen(zVal); - if(!g.delim.n) fatal("Keyword delimiter may not be empty."); - } - } - ISFLAG2("dd", "dump-defines"){ - DOIT { - FILE * const fp = stderr; - fprintf(fp, "All %sdefine entries:\n", g.delim.z); - cmpp_dump_defines(fp, 1); - } - } - else{ - fatal("Unhandled flag: %s", argv[i]); - } - } - DOIT { - if(!nFile){ - if(!g.out.zName) g.out.zName = "-"; - if(!inclCount){ - db_include_dir_add("."); - ++inclCount; - } - FileWrapper_open(&g.out, g.out.zName, "w"); - cmpp_process_file("-"); - } - } - } - end: - g_cleanup(1); - return rc ? EXIT_FAILURE : EXIT_SUCCESS; -} diff --git a/ext/wasm/common/SqliteTestUtil.js b/ext/wasm/common/SqliteTestUtil.js index 2c17824c5..a817b79f8 100644 --- a/ext/wasm/common/SqliteTestUtil.js +++ b/ext/wasm/common/SqliteTestUtil.js @@ -44,7 +44,8 @@ /** abort() if expr is false. If expr is a function, it is called and its result is evaluated. */ - assert: function f(expr, msg){ + assert: function f(expr, ...msg){ + msg = msg?.join?.(' '); if(!f._){ f._ = ('undefined'===typeof abort ? (msg)=>{throw new Error(msg)} diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js index 1c678f31f..bca05a1ee 100644 --- a/ext/wasm/common/whwasmutil.js +++ b/ext/wasm/common/whwasmutil.js @@ -16,27 +16,35 @@ More specifically: - https://fossil.wanderinghorse.net/r/jaccwabyt/file/common/whwasmutil.js + https://fossil.wanderinghorse.net/r/jaccwabyt/dir/wasmutil and SQLite: https://sqlite.org This file is kept in sync between both of those trees. + + This build was generated using: + + ./c-pp -o js/whwasmutil.js -@policy=error wasmutil/whwasmutil.c-pp.js + + by libcmpp 2.x 2fc4afc31f6505c27b9c34988973a2bd9b157d559247cdd26868ae75632c3a5e @ 2025-11-16 23:03:27.352 UTC */ /** - The primary goal of this function is to replace, where possible, - Emscripten-generated glue code with equivalent utility code which - can be used in arbitrary WASM environments built with toolchains - other than Emscripten. To that end, it populates the given object - with various WASM-specific APIs. These APIs work with both 32- and - 64-bit WASM builds. + The primary goal of this function is to provide JS/WASM utility + code similar to some of that provided by Emscripten-generated + builds, the difference being that this one can be used in arbitrary + WASM environments built with toolchains other than Emscripten. To + that end, it populates the given object with various WASM-specific + APIs. These APIs work with both 32- and 64-bit WASM builds. Forewarning: this API explicitly targets only browser environments. If a given non-browser environment has the capabilities needed for a given feature (e.g. TextEncoder), great, but it does not go out of its way to account for them and does not provide compatibility - crutches for them. + crutches for them. That said: no specific incompatibilities with, + e.g., node.js are known (whereas it is known that some folks + use this with node.js). Intended usage: @@ -217,7 +225,9 @@ newly-created (or config-provided) target. The current approach seemed better at the time. */ -globalThis.WhWasmUtilInstaller = function(target){ +'use strict'; +globalThis.WhWasmUtilInstaller = +function WhWasmUtilInstaller(target){ 'use strict'; if(undefined===target.bigIntEnabled){ target.bigIntEnabled = !!globalThis['BigInt64Array']; @@ -227,6 +237,14 @@ globalThis.WhWasmUtilInstaller = function(target){ all args with a space between each. */ const toss = (...args)=>{throw new Error(args.join(' '))}; + if( !target.pointerSize && !target.pointerIR + && target.alloc && target.dealloc ){ + /* Try to determine the pointer size by allocating. */ + const ptr = target.alloc(1); + target.pointerSize = ('bigint'===typeof ptr ? 8 : 4); + target.dealloc(ptr); + } + /** As of 2025-09-21, this library works with 64-bit WASM modules built with Emscripten's -sMEMORY64=1. @@ -659,12 +677,14 @@ globalThis.WhWasmUtilInstaller = function(target){ const ft = target.functionTable(); const oldLen = __asPtrType(ft.length); let ptr; - while(cache.freeFuncIndexes.length){ - ptr = cache.freeFuncIndexes.pop(); - if(ft.get(ptr)){ /* Table was modified via a different API */ + while( (ptr = cache.freeFuncIndexes.pop()) ){ + if(ft.get(ptr)){ + /* freeFuncIndexes's entry is stale. Table was modified via a + different API */ ptr = null; continue; }else{ + /* This index is free. We'll re-use it. */ break; } } @@ -755,10 +775,10 @@ globalThis.WhWasmUtilInstaller = function(target){ has no side effects and returns undefined. */ target.uninstallFunction = function(ptr){ - if(!ptr && 0!==ptr) return undefined; - const fi = cache.freeFuncIndexes; + if(!ptr && __NullPtr!==ptr) return undefined; + const ft = target.functionTable(); - fi.push(ptr); + cache.freeFuncIndexes.push(ptr); const rc = ft.get(ptr); ft.set(ptr, null); return rc; @@ -996,12 +1016,12 @@ globalThis.WhWasmUtilInstaller = function(target){ target.heap8u(). */ target.cstrlen = function(ptr){ - if(!ptr || !target.isPtr(ptr)) return null; + if(!ptr || !target.isPtr/*64*/(ptr)) return null; ptr = Number(ptr) /*tag:64bit*/; const h = heapWrappers().HEAP8U; let pos = ptr; for( ; h[pos] !== 0; ++pos ){} - return Number(pos - ptr); + return pos - ptr; }; /** Internal helper to use in operations which need to distinguish @@ -2452,7 +2472,8 @@ globalThis.WhWasmUtilInstaller = function(target){ - If `wasmUtilTarget.alloc` is not set and `instance.exports.malloc` is, it installs `wasmUtilTarget.alloc()` and `wasmUtilTarget.dealloc()` - wrappers for the exports `malloc` and `free` functions. + wrappers for the exports' `malloc` and `free` functions + if exports.malloc exists. It returns a function which, when called, initiates loading of the module and returns a Promise. When that Promise resolves, it calls @@ -2475,7 +2496,9 @@ globalThis.WhWasmUtilInstaller = function(target){ Error handling is up to the caller, who may attach a `catch()` call to the promise. */ -globalThis.WhWasmUtilInstaller.yawl = function(config){ +globalThis.WhWasmUtilInstaller +.yawl = function yawl(config){ + 'use strict'; const wfetch = ()=>fetch(config.uri, {credentials: 'same-origin'}); const wui = this; const finalThen = function(arg){ @@ -2500,7 +2523,7 @@ globalThis.WhWasmUtilInstaller.yawl = function(config){ tgt.alloc = function(n){ return exports.malloc(n) || toss("Allocation of",n,"bytes failed."); }; - tgt.dealloc = function(m){exports.free(m)}; + tgt.dealloc = function(m){m && exports.free(m)}; } wui(tgt); } @@ -2519,4 +2542,6 @@ globalThis.WhWasmUtilInstaller.yawl = function(config){ .then(finalThen) ; return loadWasm; -}.bind(globalThis.WhWasmUtilInstaller)/*yawl()*/; +}.bind( +globalThis.WhWasmUtilInstaller +)/*yawl()*/; diff --git a/ext/wasm/demo-jsstorage.js b/ext/wasm/demo-jsstorage.js index 587aa9cc5..e3ab5a9e5 100644 --- a/ext/wasm/demo-jsstorage.js +++ b/ext/wasm/demo-jsstorage.js @@ -16,7 +16,7 @@ */ 'use strict'; (function(){ - const T = self.SqliteTestUtil; + const T = globalThis.SqliteTestUtil; const toss = function(...args){throw new Error(args.join(' '))}; const debug = console.debug.bind(console); const eOutput = document.querySelector('#test-output'); @@ -40,7 +40,7 @@ const error = function(...args){ logHtml('error',...args); }; - + const runTests = function(sqlite3){ const capi = sqlite3.capi, oo = sqlite3.oo1, @@ -51,7 +51,7 @@ error("This build is not kvvfs-capable."); return; } - + const dbStorage = 0 ? 'session' : 'local'; const theStore = 's'===dbStorage[0] ? sessionStorage : localStorage; const db = new oo.JsStorageDb( dbStorage ); @@ -108,7 +108,7 @@ } }; - sqlite3InitModule(self.sqlite3TestModule).then((sqlite3)=>{ + sqlite3InitModule(globalThis.sqlite3TestModule).then((sqlite3)=>{ runTests(sqlite3); }); })(); diff --git a/ext/wasm/demo-worker1-promiser.c-pp.html b/ext/wasm/demo-worker1-promiser.c-pp.html index e0b487bdf..c54b46aad 100644 --- a/ext/wasm/demo-worker1-promiser.c-pp.html +++ b/ext/wasm/demo-worker1-promiser.c-pp.html @@ -6,11 +6,11 @@ <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon"> <link rel="stylesheet" href="common/emscripten.css"/> <link rel="stylesheet" href="common/testing.css"/> -//#if target=es6-module - <title>worker-promise (via ESM) tests</title> +//#if target:es6-module + <title>Worker1-promiser (ESM) tests</title> //#else - <title>worker-promise tests</title> -//#endif + <title>Worker1-promiser tests</title> +//#/if </head> <body> <header id='titlebar'><span>worker-promise tests</span></header> @@ -32,11 +32,11 @@ <hr> <div id='test-output'></div> <script src="common/SqliteTestUtil.js"></script> -//#if target=es6-module +//#if target:es6-module <script src="demo-worker1-promiser.mjs" type="module"></script> //#else <script src="jswasm/sqlite3-worker1-promiser.js"></script> <script src="demo-worker1-promiser.js"></script> -//#endif +//#/if </body> </html> diff --git a/ext/wasm/demo-worker1-promiser.c-pp.js b/ext/wasm/demo-worker1-promiser.c-pp.js index c129e2128..1521edfc1 100644 --- a/ext/wasm/demo-worker1-promiser.c-pp.js +++ b/ext/wasm/demo-worker1-promiser.c-pp.js @@ -19,7 +19,7 @@ import {default as promiserFactory} from "./jswasm/sqlite3-worker1-promiser.mjs" "use strict"; const promiserFactory = globalThis.sqlite3Worker1Promiser.v2; delete globalThis.sqlite3Worker1Promiser; -//#endif +//#/if (async function(){ const T = globalThis.SqliteTestUtil; const eOutput = document.querySelector('#test-output'); @@ -53,7 +53,7 @@ delete globalThis.sqlite3Worker1Promiser; before workerPromise is set. */ console.warn("This is the v2 interface - you don't need an onready() function."); }, -//#endif +//#/if debug: 1 ? undefined : (...args)=>console.debug('worker debug',...args), onunhandled: function(ev){ error("Unhandled worker message:",ev.data); diff --git a/ext/wasm/demo-worker1.js b/ext/wasm/demo-worker1.js index 1a05cc7ac..348741bf8 100644 --- a/ext/wasm/demo-worker1.js +++ b/ext/wasm/demo-worker1.js @@ -18,7 +18,7 @@ */ 'use strict'; (function(){ - const T = self.SqliteTestUtil; + const T = globalThis.SqliteTestUtil; const SW = new Worker("jswasm/sqlite3-worker1.js"); const DbState = { id: undefined @@ -323,7 +323,7 @@ switch(ev.result){ case 'worker1-ready': log("Message:",ev); - self.sqlite3TestModule.setStatus(null); + globalThis.sqlite3TestModule.setStatus(null); runTests(); return; default: @@ -344,5 +344,5 @@ }; log("Init complete, but async init bits may still be running."); log("Installing Worker into global scope SW for dev purposes."); - self.SW = SW; + globalThis.SW = SW; })(); diff --git a/ext/wasm/fiddle/fiddle-worker.js b/ext/wasm/fiddle/fiddle-worker.js index a5f3e25b7..4b1ea2c53 100644 --- a/ext/wasm/fiddle/fiddle-worker.js +++ b/ext/wasm/fiddle/fiddle-worker.js @@ -175,10 +175,6 @@ "features (e.g. upload) do not yet work with OPFS."); } stdout('\nEnter ".help" for usage hints.'); - this.exec([ // initialization commands... - '.nullvalue NULL', - '.headers on' - ].join('\n')); return true; }, /** diff --git a/ext/wasm/fiddle/index.html b/ext/wasm/fiddle/index.c-pp.html similarity index 95% rename from ext/wasm/fiddle/index.html rename to ext/wasm/fiddle/index.c-pp.html index 378cb3902..7fe9a12a7 100644 --- a/ext/wasm/fiddle/index.html +++ b/ext/wasm/fiddle/index.c-pp.html @@ -5,20 +5,29 @@ <meta http-equiv="Content-Type" content="text/html; charset=utf-8"> <title>SQLite3 Fiddle</title> <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon"> +//#if jqterm <!-- To add a terminal-style view using jquery.terminal[^1], uncomment the following two HTML lines and ensure that these files are on the web server. - jquery-bundle.min.js is a concatenation of jquery.min.js from + jquery.terminal.bundle.min.js is a concatenation of jquery.min.js from [^2] and jquery.terminal.min.js from [^1]. - jquery.terminal.min.css is from [^1]. + jquery.terminal.min.css is from [^1]. Alterntely, jquery-VERSION.min.js + can be found in jquery.terminal's source tree. + + Fiddle automatically enables support for this if it's installed. + + In the canonical build process, if the JQTERM env var is set + to a dir containing a clone of [^1] then this block gets + enabled and a copy of [^1] is integrated automatically. [^1]: https://github.com/jcubic/jquery.terminal [^2]: https://jquery.com --> - <!--script src="jqterm/jqterm-bundle.min.js"></script> - <link rel="stylesheet" href="jqterm/jquery.terminal.min.css"--> + <script src="jqterm/jquery.terminal.bundle.min.js"></script> + <link rel="stylesheet" href="jqterm/jquery.terminal.min.css"> +//#/if <style> /* The following styles are for app-level use. */ :root { diff --git a/ext/wasm/index.html b/ext/wasm/index.html index 55e4cdb75..6f22dc150 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -105,11 +105,21 @@ (<a href='speedtest1-worker.html?size=15'>32-bit</a>, <a href='speedtest1-worker-64bit.html?size=15'>64-bit</a>): an interactive Worker-thread variant of speedtest1.</li> + <li>speedtest1-worker?vfs=kvvfs + (<a href='speedtest1-worker.html?vfs=kvvfs&size=10'>32-bit</a>, + <a href='speedtest1-worker-64bit.html?vfs=kvvfs&size=10'>64-bit</a>): + speedtest1-worker with the + kvvfs VFS preselected and configured for a moderate workload.</li> <li>speedtest1-worker?vfs=opfs (<a href='speedtest1-worker.html?vfs=opfs&size=10'>32-bit</a>, <a href='speedtest1-worker-64bit.html?vfs=opfs&size=10'>64-bit</a>): speedtest1-worker with the OPFS VFS preselected and configured for a moderate workload.</li> + <li>speedtest1-worker?vfs=opfs-wl + (<a href='speedtest1-worker.html?vfs=opfs-wl&size=10'>32-bit</a>, + <a href='speedtest1-worker-64bit.html?vfs=opfs-wl&size=10'>64-bit</a>): + speedtest1-worker with the + OPFS Web Locks VFS preselected and configured for a moderate workload.</li> <li>speedtest1-worker?vfs=opfs-sahpool (<a href='speedtest1-worker.html?vfs=opfs-sahpool&size=10'>32-bit</a>, <a href='speedtest1-worker-64bit.html?vfs=opfs-sahpool&size=10'>64-bit</a>): diff --git a/ext/wasm/jaccwabyt/jaccwabyt.js b/ext/wasm/jaccwabyt/jaccwabyt.js index 8ea08e213..6fa86c341 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.js +++ b/ext/wasm/jaccwabyt/jaccwabyt.js @@ -16,9 +16,17 @@ Project homes: - https://fossil.wanderinghorse.net/r/jaccwabyt - https://sqlite.org/src/dir/ext/wasm/jaccwabyt + + This build was generated using: + + ./c-pp -o js/jaccwabyt.js -@policy=error jaccwabyt/jaccwabyt.c-pp.js + + by libcmpp 2.x 2fc4afc31f6505c27b9c34988973a2bd9b157d559247cdd26868ae75632c3a5e @ 2025-11-16 23:03:27.352 UTC */ 'use strict'; -globalThis.Jaccwabyt = function StructBinderFactory(config){ +globalThis.Jaccwabyt = +function StructBinderFactory(config){ + 'use strict'; /* ^^^^ it is recommended that clients move that object into wherever they'd like to have it and delete the globalThis-held copy. This API does not require the global reference - it is simply installed @@ -30,35 +38,55 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ all args with a space between each. */ const toss = (...args)=>{throw new Error(args.join(' '))}; - if(!(config.heap instanceof WebAssembly.Memory) - && !(config.heap instanceof Function)){ - toss("config.heap must be WebAssembly.Memory instance or a function."); + { + let h = config.heap; + if( h instanceof WebAssembly.Memory ){ + h = function(){return new Uint8Array(this.buffer)}.bind(h); + }else if( !(h instanceof Function) ){ + //console.warn("The bothersome StructBinderFactory config:",config); + toss("config.heap must be WebAssembly.Memory instance or", + "a function which returns one."); + } + config.heap = h; } ['alloc','dealloc'].forEach(function(k){ (config[k] instanceof Function) || toss("Config option '"+k+"' must be a function."); }); - const __heap = config.heap; const SBF = StructBinderFactory; - const heap = __heap ? __heap : ()=>new Uint8Array(__heap.buffer), + const heap = config.heap, alloc = config.alloc, dealloc = config.dealloc, + realloc = (config.realloc || function(){ + toss("This StructBinderFactory was configured without realloc()"); + /* We can't know the original memory's size from here unless + we internally proxy alloc()/dealloc() to track all + pointers (not going to happen), so we can't fall back to + doing alloc()/copy/dealloc(). */ + }), log = config.log || console.debug.bind(console), memberPrefix = (config.memberPrefix || ""), memberSuffix = (config.memberSuffix || ""), BigInt = globalThis['BigInt'], BigInt64Array = globalThis['BigInt64Array'], - bigIntEnabled = config.bigIntEnabled ?? !!BigInt64Array, - ptrIR = config.pointerIR + bigIntEnabled = config.bigIntEnabled ?? !!BigInt64Array; + + //console.warn("config",config); + let ptr; + const ptrSize = config.pointerSize + || config.ptrSize/*deprecated*/ + || ('bigint'===typeof (ptr = alloc(1)) ? 8 : 4); + const ptrIR = config.pointerIR/*deprecated*/ || config.ptrIR/*deprecated*/ - || 'i32', - /* Undocumented (on purpose) config options: */ - ptrSize = config.ptrSize/*deprecated*/ - || ('i32'===ptrIR ? 4 : 8) - ; + || (4===ptrSize ? 'i32' : 'i64'); + if( ptr ){ + dealloc(ptr); + ptr = undefined; + } + //console.warn("ptrIR =",ptrIR,"ptrSize =",ptrSize); - if(ptrIR!=='i32' && ptrIR!=='i64') toss("Invalid pointer representation:",ptrIR); if(ptrSize!==4 && ptrSize!==8) toss("Invalid pointer size:",ptrSize); + if(ptrIR!=='i32' && ptrIR!=='i64') toss("Invalid pointer representation:",ptrIR); /** Either BigInt or, if !bigIntEnabled, a function which throws complaining that BigInt is not enabled. */ @@ -86,6 +114,10 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ return rc; }; + const __ptrAddSelf = function(...args){ + return __ptrAdd(this.pointer,...args); + }; + if(!SBF.debugFlags){ SBF.__makeDebugFlags = function(deriveFrom=null){ /* This is disgustingly overengineered. :/ */ @@ -115,12 +147,12 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ SBF.debugFlags = SBF.__makeDebugFlags(); }/*static init*/ - const isLittleEndian = (function() { + const isLittleEndian = true || (function() { const buffer = new ArrayBuffer(2); new DataView(buffer).setInt16(0, 256, true /* littleEndian */); // Int16Array uses the platform's endianness. return new Int16Array(buffer)[0] === 256; - })(); + })() /* WASM is, by definition, Little Endian */; /** Some terms used in the internal docs: @@ -136,13 +168,14 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ /** True if SIG s looks like a function signature, else false. */ const isFuncSig = (s)=>'('===s[1]; - /** True if SIG s is-a pointer signature. */ - const isPtrSig = (s)=>'p'===s || 'P'===s; + /** True if SIG s is-a pointer-type signature. */ + const isPtrSig = (s)=>'p'===s || 'P'===s || 's'===s; const isAutoPtrSig = (s)=>'P'===s /*EXPERIMENTAL*/; /** Returns p if SIG s is a function SIG, else returns s[0]. */ - const sigLetter = (s)=>isFuncSig(s) ? 'p' : s[0]; - /** Returns the WASM IR form of the Emscripten-conventional letter - at SIG s[0]. Throws for an unknown SIG. */ + const sigLetter = (s)=>s ? (isFuncSig(s) ? 'p' : s[0]) : undefined; + + /** Returns the WASM IR form of the letter at SIG s[0]. Throws for + an unknown SIG. */ const sigIR = function(s){ switch(sigLetter(s)){ case 'c': case 'C': return 'i8'; @@ -155,8 +188,23 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ toss("Unhandled signature IR:",s); }; + /** Returns the WASM sizeof of the letter at SIG s[0]. Throws for an + unknown SIG. */ + const sigSize = function(s){ + switch(sigLetter(s)){ + case 'c': case 'C': return 1; + case 'i': return 4; + case 'p': case 'P': case 's': return ptrSize; + case 'j': return 8; + case 'f': return 4; + case 'd': return 8; + } + toss("Unhandled signature sizeof:",s); + }; + const affirmBigIntArray = BigInt64Array ? ()=>true : ()=>toss('BigInt64Array is not available.'); + /** Returns the name of a DataView getter method corresponding to the given SIG. */ const sigDVGetter = function(s){ @@ -177,6 +225,7 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ } toss("Unhandled DataView getter for signature:",s); }; + /** Returns the name of a DataView setter method corresponding to the given SIG. */ const sigDVSetter = function(s){ @@ -228,15 +277,44 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ /** In order to completely hide StructBinder-bound struct pointers from JS code, we store them in a scope-local WeakMap which maps - the struct-bound objects to their WASM pointers. The pointers are - accessible via boundObject.pointer, which is gated behind a - property interceptor, but are not exposed anywhere else in the - object. + the struct-bound objects to an object with their metadata: + + { + .p = the native pointer, + .o = self (for an eventual reverse-mapping), + .xb = extra bytes allocated for p, + .zod = zeroOnDispose, + .ownsPointer = true if this object owns p + } + + The .p data are accessible via obj.pointer, which is gated behind + a property interceptor, but are not exposed anywhere else in the + public API. */ - const __instancePointerMap = new WeakMap(); + const getInstanceHandle = function f(obj, create=true){ + let ii = f.map.get(obj); + if( !ii && create ){ + f.map.set(obj, (ii=f.create(obj))); + } + return ii; + }; + getInstanceHandle.map = new WeakMap; + getInstanceHandle.create = (forObj)=>{ + return Object.assign(Object.create(null),{ + o: forObj, + p: undefined/*native ptr*/, + ownsPointer: false, + zod: false/*zeroOnDispose*/, + xb: 0/*extraBytes*/ + }); + }; - /** Property name for the pointer-is-external marker. */ - const xPtrPropName = '(pointer-is-external)'; + /** + Remove the getInstanceHandle() mapping for obj. + */ + const rmInstanceHandle = (obj)=>getInstanceHandle.map.delete(obj) + /* If/when we have a reverse map of ptr-to-objects, we need to + clean that here. */; const __isPtr32 = (ptr)=>('number'===typeof ptr && (ptr===(ptr|0)) && ptr>=0); const __isPtr64 = (ptr)=>( @@ -249,83 +327,199 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ */ const __isPtr = (4===ptrSize) ? __isPtr32 : __isPtr64; - /** Frees the obj.pointer memory and clears the pointer - property. */ + const __isNonNullPtr = (v)=>__isPtr(v) && (v>0); + + /** Frees the obj.pointer memory (a.k.a. m), handles obj.ondispose, + and unmaps obj from its native resources. */ const __freeStruct = function(ctor, obj, m){ - if(!m) m = __instancePointerMap.get(obj); - if(m) { - __instancePointerMap.delete(obj); - if(Array.isArray(obj.ondispose)){ - let x; - while((x = obj.ondispose.shift())){ - try{ - if(x instanceof Function) x.call(obj); - else if(x instanceof StructType) x.dispose(); - else if(__isPtr(x)) dealloc(x); - // else ignore. Strings are permitted to annotate entries - // to assist in debugging. - }catch(e){ - console.warn("ondispose() for",ctor.structName,'@', - m,'threw. NOT propagating it.',e); - } - } - }else if(obj.ondispose instanceof Function){ - try{obj.ondispose()} - catch(e){ - /*do not rethrow: destructors must not throw*/ + const ii = getInstanceHandle(obj, false); + if( !ii ) return; + rmInstanceHandle(obj); + if( !m && !(m = ii.p) ){ + console.warn("Cannot(?) happen: __freeStruct() found no instanceInfo"); + return; + } + if(Array.isArray(obj.ondispose)){ + let x; + while((x = obj.ondispose.pop())){ + try{ + if(x instanceof Function) x.call(obj); + else if(x instanceof StructType) x.dispose(); + else if(__isPtr(x)) dealloc(x); + // else ignore. Strings are permitted to annotate entries + // to assist in debugging. + }catch(e){ console.warn("ondispose() for",ctor.structName,'@', m,'threw. NOT propagating it.',e); } } - delete obj.ondispose; - if(ctor.debugFlags.__flags.dealloc){ - log("debug.dealloc:",(obj[xPtrPropName]?"EXTERNAL":""), - ctor.structName,"instance:", - ctor.structInfo.sizeof,"bytes @"+m); + }else if(obj.ondispose instanceof Function){ + try{obj.ondispose()} + catch(e){ + /*do not rethrow: destructors must not throw*/ + console.warn("ondispose() for",ctor.structName,'@', + m,'threw. NOT propagating it.',e); + } + } + delete obj.ondispose; + if(ctor.debugFlags.__flags.dealloc){ + log("debug.dealloc:",(ii.ownsPointer?"":"EXTERNAL"), + ctor.structName,"instance:", + ctor.structInfo.sizeof,"bytes @"+m); + } + if(ii.ownsPointer){ + if( ii.zod || ctor.structInfo.zeroOnDispose ){ + heap().fill(0, Number(m), + Number(m) + ctor.structInfo.sizeof + ii.xb); } - if(!obj[xPtrPropName]) dealloc(m); + dealloc(m); } }; + /** Returns a skeleton for a read-only, non-iterable property + * descriptor. */ + const rop0 = ()=>{return {configurable: false, writable: false, + iterable: false}}; + /** Returns a skeleton for a read-only property accessor wrapping value v. */ - const rop = (v)=>{return {configurable: false, writable: false, - iterable: false, value: v}}; + const rop = (v)=>{return {...rop0(), value: v}}; /** Allocates obj's memory buffer based on the size defined in ctor.structInfo.sizeof. */ - const __allocStruct = function(ctor, obj, m){ - let fill = !m; - if(m) Object.defineProperty(obj, xPtrPropName, rop(m)); - else{ - m = alloc(ctor.structInfo.sizeof); - if(!m) toss("Allocation of",ctor.structName,"structure failed."); + const __allocStruct = function f(ctor, obj, xm){ + let opt; + const checkPtr = (ptr)=>{ + __isNonNullPtr(ptr) || + toss("Invalid pointer value",arguments[0],"for",ctor.structName,"constructor."); + }; + if( arguments.length>=3 ){ + if( xm && ('object'===typeof xm) ){ + opt = xm; + xm = opt?.wrap; + }else{ + checkPtr(xm); + opt = {wrap: xm}; + } + }else{ + opt = {} + } + + const fill = !xm /* true if we need to zero the memory */; + let nAlloc = 0; + let ownsPointer = false; + if(xm){ + /* Externally-allocated memory. */ + checkPtr(xm); + ownsPointer = !!opt?.takeOwnership; + }else{ + const nX = opt?.extraBytes ?? 0; + if( nX<0 || (nX!==(nX|0)) ){ + toss("Invalid extraBytes value:",opt?.extraBytes); + } + nAlloc = ctor.structInfo.sizeof + nX; + xm = alloc(nAlloc) + || toss("Allocation of",ctor.structName,"structure failed."); + ownsPointer = true; } try { - if(ctor.debugFlags.__flags.alloc){ + if( opt?.debugFlags ){ + /* specifically undocumented */ + obj.debugFlags(opt.debugFlags); + } + if(ctor./*prototype.???*/debugFlags.__flags.alloc){ log("debug.alloc:",(fill?"":"EXTERNAL"), ctor.structName,"instance:", - ctor.structInfo.sizeof,"bytes @"+m); + ctor.structInfo.sizeof,"bytes @"+xm); } if(fill){ - heap().fill(0, Number(m), Number(m) + ctor.structInfo.sizeof); + heap().fill(0, Number(xm), Number(xm) + nAlloc); + } + const ii = getInstanceHandle(obj); + ii.p = xm; + ii.ownsPointer = ownsPointer; + ii.xb = nAlloc ? (nAlloc-ctor.structInfo.sizeof) : 0; + ii.zod = !!opt?.zeroOnDispose; + if( opt?.ondispose && opt.ondispose!==xm ){ + obj.addOnDispose( opt.ondispose ); } - __instancePointerMap.set(obj, m); }catch(e){ - __freeStruct(ctor, obj, m); + __freeStruct(ctor, obj, xm); throw e; } }; - /** Gets installed as the memoryDump() method of all structs. */ - const __memoryDump = function(){ - const p = this.pointer; - return p - ? new Uint8Array(heap().slice(Number(p), Number(p) + this.structInfo.sizeof)) - : null; + + /** True if sig looks like an emscripten/jaccwabyt + type signature, else false. */ + const looksLikeASig = function f(sig){ + f.rxSig1 ??= /^[ipPsjfdcC]$/; + f.rxSig2 ??= /^[vipPsjfdcC]\([ipPsjfdcC]*\)$/; + return f.rxSig1.test(sig) || f.rxSig2.test(sig); + }; + + /** Returns a pair of adaptor maps (objects) in a length-3 + array specific to the given object. */ + const __adaptorsFor = function(who){ + let x = this.get(who); + if( !x ){ + x = [ Object.create(null), Object.create(null), Object.create(null) ]; + this.set(who, x); + } + return x; + }.bind(new WeakMap); + + /** Code de-duplifier for __adaptGet(), __adaptSet(), and + __adaptStruct(). */ + const __adaptor = function(who, which, key, proxy){ + const a = __adaptorsFor(who)[which]; + if(3===arguments.length) return a[key]; + if( proxy ) return a[key] = proxy; + return delete a[key]; + }; + + const noopAdapter = (x)=>x; + + // StructBinder::adaptGet() + const __adaptGet = function(key, ...args){ + return __adaptor(this, 0, key, ...args); + }; + + const __affirmNotASig = function(ctx,key){ + looksLikeASig(key) && + toss(ctx,"(",key,") collides with a data type signature."); + }; + + // StructBinder::adaptSet() + const __adaptSet = function(key, ...args){ + __affirmNotASig('Setter adaptor',key); + return __adaptor(this, 1, key, ...args); + }; + + // StructBinder::adaptStruct() + const __adaptStruct = function(key, ...args){ + __affirmNotASig('Struct adaptor',key); + return __adaptor(this, 2, key, ...args); + }; + + /** + An internal counterpart of __adaptStruct(). If key is-a string, + uses __adaptor(who) to fetch the struct-adaptor entry for key, + else key is assumed to be a struct description object. If it + resolves to an object, that's returned, else an exception is + thrown. + */ + const __adaptStruct2 = function(who,key){ + const si = ('string'===typeof key) + ? __adaptor(who, 2, key) : key; + if( 'object'!==typeof si ){ + toss("Invalid struct mapping object. Arg =",key,JSON.stringify(si)); + } + return si; }; const __memberKey = (k)=>memberPrefix + k + memberSuffix; const __memberKeyProp = rop(__memberKey); + //const __adaptGetProp = rop(__adaptGet); /** Looks up a struct member in structInfo.members. Throws if found @@ -342,7 +536,8 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ if(v.key===memberName){ m = v; break; } } if(!m && tossIfNotFound){ - toss(sPropName(structInfo.name,memberName),'is not a mapped struct member.'); + toss(sPropName(structInfo.name || structInfo.structName, memberName), + 'is not a mapped struct member.'); } } return m; @@ -359,15 +554,6 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ return emscriptenFormat ? f._(m.signature) : m.signature; }; - const __ptrPropDescriptor = { - configurable: false, enumerable: false, - get: function(){return __instancePointerMap.get(this)}, - set: ()=>toss("Cannot assign the 'pointer' property of a struct.") - // Reminder: leaving `set` undefined makes assignments - // to the property _silently_ do nothing. Current unit tests - // rely on it throwing, though. - }; - /** Impl of X.memberKeys() for StructType and struct ctors. */ const __structMemberKeys = rop(function(){ const a = []; @@ -501,13 +687,13 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ Prototype for all StructFactory instances (the constructors returned from StructBinder). */ - const StructType = function ctor(structName, structInfo){ - if(arguments[2]!==rop){ + const StructType = function StructType(structName, structInfo){ + if(arguments[2]!==rop/*internal sentinel value*/){ toss("Do not call the StructType constructor", "from client-level code."); } Object.defineProperties(this,{ - //isA: rop((v)=>v instanceof ctor), + //isA: rop((v)=>v instanceof StructType), structName: rop(structName), structInfo: rop(structInfo) }); @@ -533,8 +719,31 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ memberSignature: rop(function(memberName, emscriptenFormat=false){ return __memberSignature(this, memberName, emscriptenFormat); }), - memoryDump: rop(__memoryDump), - pointer: __ptrPropDescriptor, + memoryDump: rop(function(){ + const p = this.pointer; + return p + ? new Uint8Array(heap().slice(Number(p), Number(p) + this.structInfo.sizeof)) + : null; + }), + extraBytes: { + configurable: false, enumerable: false, + get: function(){return getInstanceHandle(this, false)?.xb ?? 0;} + }, + zeroOnDispose: { + configurable: false, enumerable: false, + get: function(){ + return getInstanceHandle(this, false)?.zod + ?? !!this.structInfo.zeroOnDispose; + } + }, + pointer: { + configurable: false, enumerable: false, + get: function(){return getInstanceHandle(this, false)?.p}, + set: ()=>toss("Cannot assign the 'pointer' property of a struct.") + // Reminder: leaving `set` undefined makes assignments + // to the property _silently_ do nothing. Current unit tests + // rely on it throwing, though. + }, setMemberCString: rop(function(memberName, str){ return __setMemberCString(this, memberName, str); }) @@ -553,183 +762,435 @@ globalThis.Jaccwabyt = function StructBinderFactory(config){ Object.defineProperties(StructType, { allocCString: rop(__allocCString), isA: rop((v)=>v instanceof StructType), - hasExternalPointer: rop((v)=>(v instanceof StructType) && !!v[xPtrPropName]), + hasExternalPointer: rop((v)=>{ + const ii = getInstanceHandle(v, false); + return !!(ii?.p && !ii?.ownsPointer); + }), memberKey: __memberKeyProp + //ptrAdd = rop(__ptrAdd) no b/c one might think that it adds based on this.pointer. }); /** - Pass this a StructBinder-generated prototype, and the struct - member description object. It will define property accessors for - proto[memberKey] which read from/write to memory in - this.pointer. It modifies descr to make certain downstream - operations much simpler. + If struct description object si has a getter proxy, return it (a + function), else return undefined. + */ + const memberGetterProxy = function(si){ + return si.get || (si.adaptGet + ? StructBinder.adaptGet(si.adaptGet) + : undefined); + }; + + /** + If struct description object si has a setter proxy, return it (a + function), else return undefined. + */ + const memberSetterProxy = function(si){ + return si.set || (si.adaptSet + ? StructBinder.adaptSet(si.adaptSet) + : undefined); + }; + + /** + To be called by makeMemberWrapper() when si has a 'members' + member, i.e. is an embedded struct. This function sets up that + struct like any other and also sets up property accessor for + ctor.memberKey(name) which returns an instance of that new + StructType when the member is accessed. That instance wraps the + memory of the member's part of the containing C struct instance. + + That is, if struct Foo has member bar which is an inner struct + then: + + const f = new Foo; + const b = f.bar; + assert( b is-a StructType object ); + assert( b.pointer === f.b.pointer ); + + b will be disposed of when f() is. Calling b.dispose() will not + do any permanent harm, as the wrapper object will be recreated + when accessing f.bar, pointing to the same memory in f. + + The si.zeroOnDispose flag has no effect on embedded structs because + they wrap "external" memory, so do not own it, and are thus never + freed, as such. */ - const makeMemberWrapper = function f(ctor,name, descr){ - if(!f._){ - /*cache all available getters/setters/set-wrappers for - direct reuse in each accessor function. */ - f._ = {getters: {}, setters: {}, sw:{}}; + const makeMemberStructWrapper = function callee(ctor, name, si){ + /** + Where we store inner-struct member proxies. Keys are a + combination of the parent object's pointer address and the + property's name. The values are StructType instances. + */ + const __innerStructs = (callee.innerStructs ??= new Map()); + const key = ctor.memberKey(name); + if( undefined!==si.signature ){ + toss("'signature' cannot be used on an embedded struct (", + ctor.structName,".",key,")."); + } + if( memberSetterProxy(si) ){ + toss("'set' and 'adaptSet' are not permitted for nested struct members."); + } + //console.warn("si =",ctor.structName, name, JSON.stringify(si,' ')); + si.structName ??= ctor.structName+'::'+name; + si.key = key; + si.name = name; + si.constructor = this.call(this, si.structName, si); + //console.warn("si.constructor =",si.constructor); + //console.warn("si =",si,"ctor=",ctor); + const getterProxy = memberGetterProxy(si); + const prop = Object.assign(Object.create(null),{ + configurable: false, + enumerable: false, + set: __propThrowOnSet(ctor/*not si.constructor*/.structName, key), + get: function(){ + const dbg = this.debugFlags.__flags; + const p = this.pointer; + const k = p+'.'+key; + let s = __innerStructs.get(k); + if(dbg.getter){ log("debug.getter: k =",k); } + if( !s ){ + s = new si.constructor(__ptrAdd(p, si.offset)); + __innerStructs.set(k, s); + this.addOnDispose(()=>s.dispose()); + s.addOnDispose(()=>__innerStructs.delete(k)); + //console.warn("Created",k,"proxy"); + } + if(getterProxy) s = getterProxy.apply(this,[s,key]); + if(dbg.getter) log("debug.getter: result =",s); + return s; + } + }); + Object.defineProperty(ctor.prototype, key, prop); + }/*makeMemberStructWrapper()*/; + + /** + This is where most of the magic happens. + + Pass this a StructBinderImpl-generated constructor, a member + property name, and the struct member description object. It will + define property accessors for proto[memberKey] which read + from/write to memory in this.pointer. It modifies si to make + certain downstream operations simpler. + */ + const makeMemberWrapper = function f(ctor, name, si){ + si = __adaptStruct2(this, si); + if( si.members ){ + return makeMemberStructWrapper.call(this, ctor, name, si); + } + + if(!f.cache){ + /* Cache all available getters/setters/set-wrappers for + direct reuse in each accessor function. */ + f.cache = {getters: {}, setters: {}, sw:{}}; const a = ['i','c','C','p','P','s','f','d','v()']; if(bigIntEnabled) a.push('j'); a.forEach(function(v){ - //const ir = sigIR(v); - f._.getters[v] = sigDVGetter(v) /* DataView[MethodName] values for GETTERS */; - f._.setters[v] = sigDVSetter(v) /* DataView[MethodName] values for SETTERS */; - f._.sw[v] = sigDVSetWrapper(v) /* BigInt or Number ctor to wrap around values + f.cache.getters[v] = sigDVGetter(v) /* DataView[MethodName] values for GETTERS */; + f.cache.setters[v] = sigDVSetter(v) /* DataView[MethodName] values for SETTERS */; + f.cache.sw[v] = sigDVSetWrapper(v) /* BigInt or Number ctor to wrap around values for conversion */; }); - const rxSig1 = /^[ipPsjfdcC]$/, - rxSig2 = /^[vipPsjfdcC]\([ipPsjfdcC]*\)$/; f.sigCheck = function(obj, name, key,sig){ if(Object.prototype.hasOwnProperty.call(obj, key)){ toss(obj.structName,'already has a property named',key+'.'); } - rxSig1.test(sig) || rxSig2.test(sig) + looksLikeASig(sig) || toss("Malformed signature for", sPropName(obj.structName,name)+":",sig); }; } const key = ctor.memberKey(name); - f.sigCheck(ctor.prototype, name, key, descr.signature); - descr.key = key; - descr.name = name; - const sigGlyph = sigLetter(descr.signature); - const xPropName = sPropName(ctor.prototype.structName,key); - const dbg = ctor.prototype.debugFlags.__flags; + f.sigCheck(ctor.prototype, name, key, si.signature); + si.key = key; + si.name = name; + const sigGlyph = sigLetter(si.signature); + const xPropName = sPropName(ctor.structName,key); + const dbg = ctor.debugFlags.__flags; /* - TODO?: set prototype of descr to an object which can set/fetch + TODO?: set prototype of si to an object which can set/fetch its preferred representation, e.g. conversion to string or mapped function. Advantage: we can avoid doing that via if/else if/else in the get/set methods. */ + const getterProxy = memberGetterProxy(si); const prop = Object.create(null); prop.configurable = false; prop.enumerable = false; prop.get = function(){ + /** + This getter proxy reads its value from the appropriate pointer + address in the heap. It knows where and how much to read based on + this.pointer, si.offset, and si.sizeof. + */ if(dbg.getter){ - log("debug.getter:",f._.getters[sigGlyph],"for", sigIR(sigGlyph), - xPropName,'@', this.pointer,'+',descr.offset,'sz',descr.sizeof); + log("debug.getter:",f.cache.getters[sigGlyph],"for", sigIR(sigGlyph), + xPropName,'@', this.pointer,'+',si.offset,'sz',si.sizeof); } let rc = ( - new DataView(heap().buffer, Number(this.pointer) + descr.offset, descr.sizeof) - )[f._.getters[sigGlyph]](0, isLittleEndian); + new DataView(heap().buffer, Number(this.pointer) + si.offset, si.sizeof) + )[f.cache.getters[sigGlyph]](0, isLittleEndian); + + if(getterProxy) rc = getterProxy.apply(this,[key,rc]); if(dbg.getter) log("debug.getter:",xPropName,"result =",rc); return rc; }; - if(descr.readOnly){ + if(si.readOnly){ prop.set = __propThrowOnSet(ctor.prototype.structName,key); }else{ + const setterProxy = memberSetterProxy(si); prop.set = function(v){ + /** + The converse of prop.get(), this encodes v into the appropriate + spot in the WASM heap. + */ if(dbg.setter){ - log("debug.setter:",f._.setters[sigGlyph],"for", sigIR(sigGlyph), - xPropName,'@', this.pointer,'+',descr.offset,'sz',descr.sizeof, v); + log("debug.setter:",f.cache.setters[sigGlyph],"for", sigIR(sigGlyph), + xPropName,'@', this.pointer,'+',si.offset,'sz',si.sizeof, v); } if(!this.pointer){ - toss("Cannot set struct property on disposed instance."); + toss("Cannot set native property on a disposed", + this.structName,"instance."); } - if(null===v) v = __NullPtr; - else while(!__isPtr(v)){ - if(isAutoPtrSig(descr.signature) && (v instanceof StructType)){ - // It's a struct instance: let's store its pointer value! + if( setterProxy ) v = setterProxy.apply(this,[key,v]); + if( null===v || undefined===v ) v = __NullPtr; + else if( isPtrSig(si.signature) && !__isPtr(v) ){ + if(isAutoPtrSig(si.signature) && (v instanceof StructType)){ + // It's a struct instance: store its pointer value v = v.pointer || __NullPtr; if(dbg.setter) log("debug.setter:",xPropName,"resolved to",v); - break; + }else{ + toss("Invalid value for pointer-type",xPropName+'.'); } - toss("Invalid value for pointer-type",xPropName+'.'); } ( - new DataView(heap().buffer, Number(this.pointer) + descr.offset, - descr.sizeof) - )[f._.setters[sigGlyph]](0, f._.sw[sigGlyph](v), isLittleEndian); + new DataView(heap().buffer, Number(this.pointer) + si.offset, + si.sizeof) + )[f.cache.setters[sigGlyph]](0, f.cache.sw[sigGlyph](v), isLittleEndian); }; } Object.defineProperty(ctor.prototype, key, prop); - }/*makeMemberWrapper*/; + }/*makeMemberWrapper()*/; /** The main factory function which will be returned to the - caller. + caller. The third argument is structly for internal use. + + This level of indirection is to avoid that clients can pass a + third argument to this, as that's only for internal use. + + internalOpt options: + + - None right now. This is for potential use in recursion. + + Usages: + + StructBinder(string, obj [,optObj]); + StructBinder(obj); */ - const StructBinder = function StructBinder(structName, structInfo){ - if(1===arguments.length){ - structInfo = structName; - structName = structInfo.name; - }else if(!structInfo.name){ - structInfo.name = structName; - } - if(!structName) toss("Struct name is required."); - let lastMember = false; - Object.keys(structInfo.members).forEach((k)=>{ - // Sanity checks of sizeof/offset info... - const m = structInfo.members[k]; - if(!m.sizeof) toss(structName,"member",k,"is missing sizeof."); - else if(m.sizeof===1){ - (m.signature === 'c' || m.signature === 'C') || - toss("Unexpected sizeof==1 member", - sPropName(structInfo.name,k), - "with signature",m.signature); - }else{ - // sizes and offsets of size-1 members may be odd values, but - // others may not. - if(0!==(m.sizeof%4)){ - console.warn("Invalid struct member description =",m,"from",structInfo); - toss(structName,"member",k,"sizeof is not aligned. sizeof="+m.sizeof); - } - if(0!==(m.offset%4)){ - console.warn("Invalid struct member description =",m,"from",structInfo); - toss(structName,"member",k,"offset is not aligned. offset="+m.offset); - } - } - if(!lastMember || lastMember.offset < m.offset) lastMember = m; - }); - if(!lastMember) toss("No member property descriptions found."); - else if(structInfo.sizeof < lastMember.offset+lastMember.sizeof){ - toss("Invalid struct config:",structName, - "max member offset ("+lastMember.offset+") ", - "extends past end of struct (sizeof="+structInfo.sizeof+")."); - } - const debugFlags = rop(SBF.__makeDebugFlags(StructBinder.debugFlags)); - /** Constructor for the StructCtor. */ - const zeroAsPtr = __asPtrType(0); - const StructCtor = function StructCtor(externalMemory){ - externalMemory = __asPtrType(externalMemory); - //console.warn("externalMemory",externalMemory,arguments[0]); + const StructBinderImpl = function StructBinderImpl( + structName, si, opt = Object.create(null) + ){ + /** + StructCtor is the eventual return value of this function. We + need to populate this early on so that we can do some trickery + in feeding it through recursion. + + Uses: + + // heap-allocated: + const x = new StructCtor; + // externally-managed memory: + const y = new StructCtor( aPtrToACompatibleCStruct ); + + or, more recently: + + const z = new StructCtor({ + extraBytes: [int=0] extra bytes to allocate after the struct + + wrap: [aPtrToACompatibleCStruct=undefined]. If provided, this + instance waps, but does not (by default) own the memory, else + a new instance is allocated from the WASM heap. + + ownsPointer: true if this object takes over ownership of + wrap. + + zeroOnDispose: [bool=StructCtor.structInfo.zeroOnDispose] + + autoCalcSizeOffset: [bool=false] Automatically calculate + sizeof an offset. This is fine for pure-JS structs (which + probably aren't useful beyond testing of Jaccwabyt) but it's + dangerous to use with actual WASM objects because we cannot + be guaranteed to have the same memory layout as an ostensibly + matching C struct. This applies recursively to all children + of the struct description. + + // TODO? Per-instance overrides of the struct-level flags? + + get: (k,v)=>v, + set: (k,v)=>v, + adaptGet: string, + adaptSet: string + + // That wouldn't fit really well right now, apparently. + }); + + */ + const StructCtor = function StructCtor(arg){ + //console.warn("opt",opt,arguments[0]); if(!(this instanceof StructCtor)){ toss("The",structName,"constructor may only be called via 'new'."); - }else if(arguments.length){ - if(Number.isNaN(externalMemory) || externalMemory<=zeroAsPtr){ - toss("Invalid pointer value",arguments[0],"for",structName,"constructor."); - } - __allocStruct(StructCtor, this, externalMemory); - }else{ - __allocStruct(StructCtor, this); } + __allocStruct(StructCtor, this, ...arguments); }; + const self = this; + /** + "Convert" struct description x to a struct description, if + needed. This expands adaptStruct() mappings and transforms + {memberName:signatureString} signature syntax to object form. + */ + const ads = (x)=>{ + //console.warn("looksLikeASig(",x,") =",looksLikeASig(x)); + return (('string'===typeof x) && looksLikeASig(x)) + ? {signature: x} : __adaptStruct2(self,x); + }; + if(1===arguments.length){ + si = ads(structName); + structName = si.structName || si.name; + }else if(2===arguments.length){ + si = ads(si); + si.name ??= structName; + }else{ + si = ads(si); + } + structName ??= si.structName; + //console.warn("arguments =",JSON.stringify(arguments)); + structName ??= opt.structName; + if( !structName ) toss("One of 'name' or 'structName' are required."); + if( si.adapt ){ + /* Install adaptGet(), adaptSet(), and adaptStruct() proxies. */ + Object.keys(si.adapt.struct||{}).forEach((k)=>{ + __adaptStruct.call(StructBinderImpl, k, si.adapt.struct[k]); + }); + Object.keys(si.adapt.set||{}).forEach((k)=>{ + __adaptSet.call(StructBinderImpl, k, si.adapt.set[k]); + }); + Object.keys(si.adapt.get||{}).forEach((k)=>{ + __adaptGet.call(StructBinderImpl, k, si.adapt.get[k]); + }); + } + if(!si.members && !si.sizeof){ + si.sizeof = sigSize(si.signature); + } + + const debugFlags = rop(SBF.__makeDebugFlags(StructBinder.debugFlags)); Object.defineProperties(StructCtor,{ debugFlags: debugFlags, isA: rop((v)=>v instanceof StructCtor), memberKey: __memberKeyProp, memberKeys: __structMemberKeys, - methodInfoForKey: rop(function(mKey){ - }), - structInfo: rop(structInfo), - structName: rop(structName) + //methodInfoForKey: rop(function(mKey){/*???*/}), + structInfo: rop(si), + structName: rop(structName), + ptrAdd: rop(__ptrAdd) }); - StructCtor.prototype = new StructType(structName, structInfo, rop); + StructCtor.prototype = new StructType(structName, si, rop); Object.defineProperties(StructCtor.prototype,{ debugFlags: debugFlags, constructor: rop(StructCtor) /*if we assign StructCtor.prototype and don't do - this then StructCtor!==instance.constructor!*/ + this then StructCtor!==instance.constructor*/, + ptrAdd: rop(__ptrAddSelf) }); - Object.keys(structInfo.members).forEach( - (name)=>makeMemberWrapper(StructCtor, name, structInfo.members[name]) - ); + let lastMember = false; + let offset = 0; + const autoCalc = !!si.autoCalcSizeOffset; + //console.warn(structName,"si =",si); + if( !autoCalc ){ + if( !si.sizeof ){ + toss(structName,"description is missing its sizeof property."); + } + /*if( undefined===si.offset ){ + toss(structName,"description is missing its offset property."); + }*/ + si.offset ??= 0; + }else{ + si.offset ??= 0; + } + Object.keys(si.members || {}).forEach((k)=>{ + // Sanity checks of sizeof/offset info... + let m = ads(si.members[k]); + if(!m.members && !m.sizeof){ + /* ^^^^ fixme: we need to recursively collect all sizeofs + before updating that. */ + m.sizeof = sigSize(m.signature); + if(!m.sizeof){ + toss(sPropName(structName,k), "is missing a sizeof property.",m); + } + } + if( undefined===m.offset ){ + if( autoCalc ) m.offset = offset; + else{ + toss(sPropName(structName,k),"is missing its offset.", + JSON.stringify(m)); + } + /* A missing offset on the initial child is okay (it's always + zero), but we don't know for sure that the members are + their natural order, so we don't know, at this point, which + one is "first". */ + } + si.members[k] = m /* in case ads() resolved it to something else */; + if(!lastMember || lastMember.offset < m.offset) lastMember = m; + const oldAutoCalc = !!m.autoCalc; + if( autoCalc ) m.autoCalcSizeOffset = true; + makeMemberWrapper.call(self, StructCtor, k, m); + if( oldAutoCalc ) m.autoCalcSizeOffset = true; + else delete m.autoCalcSizeOffset; + offset += m.sizeof; + //console.warn("offset",sPropName(structName,k),offset); + }); + + if( !lastMember ) toss("No member property descriptions found."); + if( !si.sizeof ) si.sizeof = offset; + if(si.sizeof===1){ + (si.signature === 'c' || si.signature === 'C') || + toss("Unexpected sizeof==1 member", + sPropName(structName,k), + "with signature",si.signature); + }else{ + // sizes and offsets of size-1 members may be odd values, but + // others may not. + if(0!==(si.sizeof%4)){ + console.warn("Invalid struct member description",si); + toss(structName,"sizeof is not aligned. sizeof="+si.sizeof); + } + if(0!==(si.offset%4)){ + console.warn("Invalid struct member description",si); + toss(structName,"offset is not aligned. offset="+si.offset); + } + } + if( si.sizeof < offset ){ + console.warn("Suspect struct description:",si,"offset =",offset); + toss("Mismatch in the calculated vs. the provided sizeof/offset info.", + "Expected sizeof",offset,"but got",si.sizeof,"for",si); + /* It is legal for the native struct to be larger, so long as + we're pointing to all the right offsets for the members + exposed here. */ + } + delete si.autoCalcSizeOffset; return StructCtor; + }/*StructBinderImpl*/; + + const StructBinder = function StructBinder(structName, structInfo){ + return (1==arguments.length) + ? StructBinderImpl.call(StructBinder, structName) + : StructBinderImpl.call(StructBinder, structName, structInfo); }; StructBinder.StructType = StructType; StructBinder.config = config; StructBinder.allocCString = __allocCString; + StructBinder.adaptGet = __adaptGet; + StructBinder.adaptSet = __adaptSet; + StructBinder.adaptStruct = __adaptStruct; + StructBinder.ptrAdd = __ptrAdd; if(!StructBinder.debugFlags){ StructBinder.debugFlags = SBF.__makeDebugFlags(SBF.debugFlags); } diff --git a/ext/wasm/jaccwabyt/jaccwabyt.md b/ext/wasm/jaccwabyt/jaccwabyt.md index 5ec3151d5..5c30268e8 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.md +++ b/ext/wasm/jaccwabyt/jaccwabyt.md @@ -12,6 +12,35 @@ friction. (If that means nothing to you, neither will the rest of this page!) +To the best of its creator's fallible knowledge, Jaccwabyt is the only +library of its kind (as of 2025-11). Aside from wrapping existing +structs, e.g. to integrate "legacy" C code into JS/WASM, it can also +model C structs without requiring a native C counterpart, a feature +which probably has only exceedingly obscure uses in JS-side +implementations for native callbacks. + +How it works: + +- The client provides a JSON-friendly description of a C struct, + describing the names, sizes, and offsets of each member. +- Pass that description to a factory function to create + a JS constructor for that C struct. +- That constructor allocates a block of heap memory of the C struct's + size and maps it to the new JS-side struct instance. Each instance + inherits property interceptors for each struct member, such that + fetching the C struct's members reads directly from the struct's + memory and setting them writes to that memory. Similarly, these + objects can be provided with memory constructed elsewhere, e.g. + a struct pointer returned from a WASM function, and can proxy + that memory via the struct's interface. +- Clients eventually call the `dispose()` method to free the + instance's heap memory and disassociate the JS instance with its + WASM-side resources. + +Easy peasy! + +**Build instructions**: [see Appendix B](#appendix-b). + **Browser compatibility**: this library requires a _recent_ browser and makes no attempt whatsoever to accommodate "older" or lesser-capable ones, where "recent," _very roughly_, means released in @@ -25,7 +54,10 @@ are based solely on feature compatibility tables provided at **Non-browser compatibility**: this code does not target non-browser JS engines and is completely untested on them. That said, it "might -work". +work". These JS APIs do not use the DOM API in any way, so are not +specifically tied to a browser, but they _are_ fully untested in such +environments. This code is known to work with both [Emscripten][] builds +and [WASI-SDK][] SDK builds (at of this writing, 2025-11-08). **64-bit WASM:** as of 2025-09-21 this API supports 64-bit WASM builds but it has to be configured for it (see [](#api-binderfactory) for @@ -57,7 +89,7 @@ project was spawned: ----- -<a name='overview'></a> +<a id='overview'></a> Table of Contents ============================================================ @@ -72,16 +104,18 @@ Table of Contents - APIs - [Struct Binder Factory](#api-binderfactory) - [Struct Binder](#api-structbinder) - - [Struct Type](#api-structtype) + - [Struct Description Objects](#struct-descr) +- [Struct Type](#api-structtype) - [Struct Constructors](#api-structctor) - [Struct Protypes](#api-structprototype) - [Struct Instances](#api-structinstance) - Appendices - [Appendix A: Limitations, TODOs, etc.](#appendix-a) + - [Appendix B: Build](#appendix-b) - [Appendix D: Debug Info](#appendix-d) - [Appendix G: Generating Struct Descriptions](#appendix-g) -<a name='overview'></a> +<a id='overview'></a> Overview ============================================================ @@ -114,15 +148,16 @@ Portability notes: because it is the most widespread WASM toolchain, but this code is specifically designed to be usable in arbitrary WASM environments. It abstracts away a few Emscripten-specific features into - configurable options. Similarly, the build tree requires Emscripten - but Jaccwabyt does not have any hard Emscripten dependencies. + configurable options. The build tree supports both [Emscripten][] + and [WASI-SDK][] to demonstrate that it has no dependencies on + either. - This code is encapsulated into a single JavaScript function. It should be trivial to copy/paste into arbitrary WASM/JS-using projects. - The source tree includes C code, but only for testing and - demonstration purposes. It is not part of the core distributable. + demonstration purposes. It is not a core distributable. -<a name='architecture'></a> +<a id='architecture'></a> Architecture ------------------------------------------------------------ @@ -156,9 +191,10 @@ Its major classes and functions are: an appropriate configuration, to generate a single... - **[StructBinder][]** is a factory function which converts an arbitrary number struct descriptions into... -- **[StructTypes][StructCtors]** are constructors, one per struct +- **[StructType][]** are [constructors][StructCtor], one per struct description, which inherit from - **[`StructBinder.StructType`][StructType]** and are used to instantiate... + **[`StructBinder.StructType`][StructType]** and are used to + instantiate... - **[Struct instances][StructInstance]** are objects representing individual instances of generated struct types. @@ -167,7 +203,7 @@ need only one. Each StructBinder is effectively a separate namespace for struct creation. -<a name='creating-binding'></a> +<a id='creating-binding'></a> Creating and Binding Structs ============================================================ @@ -190,7 +226,7 @@ essentially boils down to: Detailed instructions for each of those steps follows... -<a name='step-1'></a> +<a id='step-1'></a> Step 1: Configure Jaccwabyt for the Environment ------------------------------------------------------------ @@ -208,7 +244,7 @@ const MyBinder = StructBinderFactory({ a Uint8Array or Int8Array view of the WASM memory, alloc: function(howMuchMemory){...}, dealloc: function(pointerToFree){...}, - pointerIR: 'i32' or 'i64' // WASM pointer type - default = 'i32' + pointerSize: 4 or 8 // WASM pointer size }); ``` @@ -234,7 +270,7 @@ a conventional Emscripten setup, that config might simply look like: The StructBinder factory function returns a function which can then be used to create bindings for our structs. -<a name='step-2'></a> +<a id='step-2'></a> Step 2: Create a Struct Description ------------------------------------------------------------ @@ -267,100 +303,9 @@ Its JSON description looks like: } ``` -These data _must_ match up with the C-side definition of the struct -(if any). See [Appendix G][appendix-g] for one way to easily generate -these from C code. - -Each entry in the `members` object maps the member's name to -its low-level layout: - -- `offset`: the byte offset from the start of the struct, as reported - by C's `offsetof()` feature. -- `sizeof`: as reported by C's `sizeof()`. -- `signature`: described below. -- `readOnly`: optional. If set to true, the binding layer will - throw if JS code tries to set that property. - -The order of the `members` entries is not important: their memory -layout is determined by their `offset` and `sizeof` members. The -`name` property is technically optional, but one of the steps in the -binding process requires that either it be passed an explicit name or -there be one in the struct description. The names of the `members` -entries need not match their C counterparts. Project conventions may -call for giving them different names in the JS side and the -[StructBinderFactory][] can be configured to automatically add a -prefix and/or suffix to their names. - -Nested structs are as-yet unsupported by this tool. +This is described in more detail in [][StructBinder]. -Struct member "signatures" describe the data types of the members and -are an extended variant of the format used by Emscripten's -`addFunction()`. A signature for a non-function-pointer member, or -function pointer member which is to be modelled as an opaque pointer, -is a single letter. A signature for a function pointer may also be -modelled as a series of letters describing the call signature. The -supported letters are: - -- **`v`** = `void` (only used as return type for function pointer members) -- **`i`** = `int32` (4 bytes) -- **`j`** = `int64` (8 bytes) is only really usable if this code is built - with BigInt support (e.g. using the Emscripten `-sWASM_BIGINT` build - flag). Without that, this API may throw when encountering the `j` - signature entry. -- **`f`** = `float` (4 bytes) -- **`d`** = `double` (8 bytes) -- **`c`** = `int8` (1 byte) char - see notes below! -- **`C`** = `uint8` (1 byte) unsigned char - see notes below! -- **`p`** = `int32` (see notes below!) -- **`P`** = Like `p` but with extra handling. Described below. -- **`s`** = like `int32` but is a _hint_ that it's a pointer to a - string so that _some_ (very limited) contexts may treat it as such, - noting that such algorithms must, for lack of information to the - contrary, assume both that the encoding is UTF-8 and that the - pointer's member is NUL-terminated. If that is _not_ the case for a - given string member, do not use `s`: use `i` or `p` instead and do - any string handling yourself. - -Noting that: - -- **All of these types are numeric**. Attempting to set any - struct-bound property to a non-numeric value will trigger an - exception except in cases explicitly noted otherwise. -- **"Char" types**: WASM does not define an `int8` type, nor does its - JS representation distinguish between signed and unsigned. This API - treats `c` as `int8` and `C` as `uint8` for purposes of getting and - setting values when using the `DataView` class. It is _not_ - recommended that client code use these types in new WASM-capable - code, but they were added for the sake of binding some immutable - legacy code to WASM. - -> Sidebar: Emscripten's public docs do not mention `p`, but their -generated code includes `p` as an alias for `i`, presumably to mean -"pointer". Though `i` is legal for pointer types in the signature, `p` -is more descriptive, so this framework encourages the use of `p` for -pointer-type members. Using `p` for pointers also helps future-proof -the signatures against the eventuality that WASM eventually supports -64-bit pointers. Note that sometimes `p` _really_ means a -pointer-to-pointer. We simply have to be aware of when we need to deal -with pointers and pointers-to-pointers in JS code. - -> Trivia: this API treates `p` as distinctly different from `i` in -some contexts, so its use is encouraged for pointer types. - -Signatures in the form `x(...)` denote function-pointer members and -`x` denotes non-function members. Functions with no arguments use the -form `x()`. For function-type signatures, the strings are formulated -such that they can be passed to Emscripten's `addFunction()` after -stripping out the `(` and `)` characters. For good measure, to match -the public Emscripten docs, `p`, `c`, and `C`, should also be replaced -with `i`. In JavaScript that might look like: - -> -``` -signature.replace(/[^vipPsjfdcC]/g,'').replace(/[pPscC]/g,'i'); -``` - -<a name='step-2-pvsp'></a> +<a id='step-2-pvsp'></a> ### `P` vs `p` in Method Signatures *This support is experimental and subject to change.* @@ -379,7 +324,7 @@ stored in `myStruct.x`. If `y` is neither a pointer nor a or `P` is used). -<a name='step-3'></a> +<a id='step-3'></a> Step 3: Binding the Struct ------------------------------------------------------------ @@ -402,7 +347,7 @@ simplify certain later operations. If that is not desired, then feed it a copy of the original, e.g. by passing it `JSON.parse(JSON.stringify(structDefinition))`. -<a name='step-4'></a> +<a id='step-4'></a> Step 4: Creating, Using, and Destroying Struct Instances ------------------------------------------------------------ @@ -461,11 +406,11 @@ Now that we have struct instances, there are a number of things we can do with them, as covered in the rest of this document. -<a name='api'></a> +<a id='api'></a> API Reference ============================================================ -<a name='api-binderfactory'></a> +<a id='api-binderfactory'></a> API: Binder Factory ------------------------------------------------------------ @@ -480,14 +425,14 @@ Function StructBinderFactory(object configOptions); It returns a function which these docs refer to as a [StructBinder][] (covered in the next section). It throws on error. -The binder factory supports the following options in its -configuration object argument: +The binder factory supports the following options in its configuration +object argument: -- `pointerIR` (Added 2025-09-21) - Optionally specify the WASM pointer size with the string `'i32'` or - `'i64'`, defaulting to the former. When using with 64-bit WASM - builds, this must be set to `'i64'` by the client. Any other value - triggers an exception. +- `pointerSize` (Added 2025-11-15 to replace `pointerIR`) + Optionally specify the WASM pointer size of 4 (32-bit) or 8 + (64-bit). Any other truthy value triggers an exception. If + `pointerSize` is not set then it will guess the size by `alloc()`ing + one byte, checking the result type, and `dealloc()`ing it. - `heap` Must be either a `WebAssembly.Memory` instance representing the WASM @@ -498,21 +443,24 @@ configuration object argument: for the WASM heap to grow at runtime. - `alloc` - Must be a function semantically compatible with Emscripten's - `Module._malloc()`. That is, it is passed the number of bytes to - allocate and it returns a pointer. On allocation failure it may - either return 0 or throw an exception. This API will throw an - exception if allocation fails or will propagate whatever exception - the allocator throws. The allocator _must_ use the same heap as the - `heap` config option. + Must be a function semantically compatible with C's + `malloc(3)`. That is, it is passed the number of bytes to allocate + and it returns a pointer. On allocation failure it may either return + 0 or throw an exception. This API will throw an exception if + allocation fails or will propagate whatever exception the allocator + throws. The allocator _must_ use the same heap as the `heap` config + option. - `dealloc` - Must be a function semantically compatible with Emscripten's - `Module._free()`. That is, it takes a pointer returned from - `alloc()` and releases that memory. It must never throw and must - accept a value of 0/null to mean "do nothing" (noting that 0 is - _technically_ a legal memory address in WASM, but that seems like a - design flaw). + Must be a function semantically compatible with C's `free(3)`. That + is, it takes a pointer returned from `alloc()` and releases that + memory. It must never throw and must accept a value of 0/null to + mean "do nothing". + +- `realloc` + Optional but required for (eventual (and optional) realloc support + of structs. If set, it must be a function semantically compatible + with C's `realloc()`. See `alloc`, above, for other requirements. - `bigIntEnabled` (bool=true if BigInt64Array is available, else false) If true, the WASM bits this code is used with must have been @@ -542,13 +490,13 @@ configuration object argument: (like `console.debug` does). See [Appendix D](#appendix-d) for info about enabling debugging output. -<a name='api-structbinder'></a> +<a id='api-structbinder'></a> API: Struct Binder ------------------------------------------------------------ Struct Binders are factories which are created by the [StructBinderFactory][]. A given Struct Binder can process any number -of distinct structs. In a typical setup, an app will have ony one +of distinct structs. In a typical setup, an app will have only one shared Binder Factory and one Struct Binder. Struct Binders which are created via different [StructBinderFactory][] calls are unrelated to each other, sharing no state except, perhaps, indirectly via @@ -568,7 +516,7 @@ The returned object is a constructor for instances of the struct described by its argument(s), each of which derives from a separate [StructType][] instance. -The Struct Binder has the following members: +StructBinder has the following members: - `allocCString(str)` Allocates a new UTF-8-encoded, NUL-terminated copy of the given JS @@ -583,7 +531,221 @@ The Struct Binder has the following members: any of its "significant" configuration values may have undefined results. -<a name='api-structtype'></a> +- `adaptGet(key [,func])` + Gets or sets a "get adaptor" by name - an arbitrary client-defined + string, e.g. `"to-js-string"`. Struct description objects may have + their `adaptGet` property set to the name of a mapped getter to + behave exactly as if that struct description had set the given + function as its `get` property. This offers a JSON-friendly way of + storing adaptor mappings, with the caveat that the adaptors need to + be defined _somewhere_ outside of JSON (typically it should be done + immediately after creating the StructBinder). + +- `adaptSet(key [,func])` + The "set" counterpart of `adaptGet`. + +- `ptrAdd(...)` + Coerces all of its arguments to the WASM pointer type, adds them + together, and returns a result of that same type. This is a + workaround for mixed-BigInt/Number pointer math being illegal in JS. + +The `structDescription` argument is described in detail in the +following section. + +<a id='struct-descr'></a> +### Struct Description Object + +C structs are described in a JSON-friendly format: + +> +```json +{ + "name": "MyStruct", + "sizeof": 16, + "members": { + "member1": {"offset": 0,"sizeof": 4,"signature": "i"}, + "member2": {"offset": 4,"sizeof": 4,"signature": "p"}, + "member3": {"offset": 8,"sizeof": 8,"signature": "j"} + } +} +``` + +Forewarning: these data _must_ match up with the C-side definition of +the struct (if any). See [Appendix G][appendix-g] for one way to +easily generate these from C code. + +Every struct must have a `sizeof`. (Though we _could_ calculate it +based on the list of members, we don't. Actually, we do, then we throw +if the values don't match up.) The `name` is required as well but it +may optionally be passed as the first argument to +`StructBinder(structName,structDescription)`. the `name` property +represents the member's symbolic name, typically its native name. + +Abstractly speaking, a struct description in an object with the +properties `sizeof`, `offset`, and either `signature` _or_ +`member`. `offset` is optional only in the top-most object of a struct +description. Every sub-object (a.k.a. member description object) +requires the `offset` property. + +Member description objects are those in the `members` property: + +`"members": {"memberName": {...member description...}, ...}` + +A struct description which has its own `members` object represents a +nested struct, with an identical description syntax to that of a +top-level struct except that nested structs require an `offset` +property. + +Each entry in a struct/member description object maps the member's +name to its low-level layout and other metadata: + +- `offset` + The byte offset from the start of the struct, as reported by C's + `offsetof()` feature. For nested structs's members, this value is + relative to the nested struct, not the parent struct. +- `sizeof` + As reported by C's `sizeof()`. +- `signature` + A type-id signature for this member. Described below. +- `readOnly [=false]` + Optional boolean. If set to true, the binding layer will throw if JS + code tries to set that property. +- `zeroOnDispose [=false]` + If true, then the library will zero out the memory of instances of + this struct when their `dispose()` method is called. Because + `StructType.dispose()` does not free instances which wrap + externally-provided memory, those instances are not wiped when + disposed (doing so would likely interfere with other users of that + memory). (There is no need for a `zeroOnAlloc` counterpart because + newly-allocated instances are always zero-filled for sanity's + sake.) +- `members` + This object describes the individual struct members, mapping their + names to a member description object. `members` gets processed + recursively. Any member with its own `members` property is a + nested-struct, and the property accessor for such members will + return an instance of the distinct StructType which wraps that + member's native memory. + Nested-struct members cannot be assigned over. `signature` is + illegal if `members` is set. +- `get` + Optional function. When fetching this member, this getter is passed + `(K,V)`, where `K` is the struct member's key and `V` is the + _native_ value. `get()`'s return value becomes the value of the + property access operation. This enables custom "native-to-JS" + conversions. If the member is a nested struct, the value passed to + the getter is a StructType proxy object which provides access to its + own members, a subset of the parent object's memory. In the context + of the getter call, "this" is the object upon which the get is being + performed. +- `set` + Optional function. When setting this member, this setter is passed + `(K,V)`, where `K` is the struct member's key and `V` is the _JS_ + value the property is being assigned to. The `set()` return value is + assigned to the _native_ struct member. Thus `set()` _must_ return + an appropriate numeric value and can perform "JS-to-native" + conversions. `set` is not currently legal for nested struct values, + but it is on their own non-nested-struct members. In the context of + the setter call, "this" is the object upon which the set is being + performed. +- `adaptGet` and `adaptSet` + JSON-friendly variants of `get` and `set`. Each may be assigned a + string value, and each such string must be mapped with + `StructBinder.adaptGet(key,func)` + resp. `StructBinder.adaptSet(key,func)`. With that in place, these + behave like `get` resp. `set`. +- `structName` + Optional descriptive name, possibly distinct from the `name`, + primarily used for nested structs. The intent is that this be some + form of the struct type's name, optionally with leading parts of + this object is a nested struct. +- `name` + Is usually optional, and is always optional in `members` entries + because their name is conveniently derived from their containing + object. `name` must be provided only for the top-most struct. The + intent is that `name` maps to the member's property name and that + `structName` optionally be set for nested structs (it will be + derived from the name if it's not set). + +The order of the `members` entries is not important: their memory +layout is determined by their `offset` and `sizeof` members. The +`name` property is technically optional, but one of the steps in the +binding process requires that either it be passed an explicit name or +there be one in the struct description. The names of the `members` +entries need not match their C counterparts. Project conventions may +call for giving them different names in the JS side and the +[StructBinderFactory][] can be configured to automatically add a +prefix and/or suffix to their names. + +Struct member "signatures" describe the data types of the members and +are an extended variant of the format used by Emscripten's +`addFunction()`. A signature for a non-function-pointer member, or +function pointer member which is to be modelled as an opaque pointer, +is a single letter. A signature for a function pointer may also be +modelled as a series of letters describing the call signature. The +supported letters are: + +- **`v`** = `void` (only used as return type for function pointer members) +- **`i`** = `int32` (4 bytes) +- **`j`** = `int64` (8 bytes) is only really usable if this code is built + with BigInt support (e.g. using the Emscripten `-sWASM_BIGINT` build + flag). Without that, this API may throw when encountering the `j` + signature entry. +- **`f`** = `float` (4 bytes) +- **`d`** = `double` (8 bytes) +- **`c`** = `int8` (1 byte) char - see notes below! +- **`C`** = `uint8` (1 byte) unsigned char - see notes below! +- **`p`** = `int32` (see notes below!) +- **`P`** = Like `p` but with extra handling. Described below. +- **`s`** = like `int32` but is a _hint_ that it's a pointer to a + string so that _some_ (very limited) contexts may treat it as such, + noting that such algorithms must, for lack of information to the + contrary, assume both that the encoding is UTF-8 and that the + pointer's member is NUL-terminated. If that is _not_ the case for a + given string member, do not use `s`: use `i` or `p` instead and do + any string handling yourself. + +Noting that: + +- **All of these types are numeric**. Attempting to set any + struct-bound property to a non-numeric value will trigger an + exception except in cases explicitly noted otherwise. +- **"Char" types**: WASM does not define an `int8` type, nor does its + JS representation distinguish between signed and unsigned. This API + treats `c` as `int8` and `C` as `uint8` for purposes of getting and + setting values when using the `DataView` class. It is _not_ + recommended that client code use these types in new WASM-capable + code, but they were added for the sake of binding some immutable + legacy code to WASM. + +> Sidebar: Emscripten's public docs do not mention `p`, but their +generated code includes `p` as an alias for `i`, presumably to mean +"pointer". Though `i` is legal for pointer types in the signature, `p` +is more descriptive, so this framework encourages the use of `p` for +pointer-type members. Using `p` for pointers also helps future-proof +the signatures against the eventuality that WASM eventually supports +64-bit pointers. Note that sometimes `p` _really_ means a +pointer-to-pointer. We simply have to be aware of when we need to deal +with pointers and pointers-to-pointers in JS code. + +> Trivia: this API treates `p` as distinctly different from `i` in +some contexts, so its use is encouraged for pointer types. + +Signatures in the form `x(...)` denote function-pointer members and +`x` denotes non-function members. Functions with no arguments use the +form `x()`. For function-type signatures, the strings are formulated +such that they can be passed to Emscripten's `addFunction()` after +stripping out the `(` and `)` characters. For good measure, to match +the public Emscripten docs, `p`, `c`, and `C`, should also be replaced +with `i`. In JavaScript that might look like: + +> +``` +signature.replace(/[^vipPsjfdcC]/g,'').replace(/[pPscC]/g,'i'); +``` + + +<a id='api-structtype'></a> API: Struct Type ------------------------------------------------------------ @@ -598,7 +760,7 @@ config options. The StructType constructor cannot be called from client code. It is only called by the [StructBinder][]-generated -[constructors][StructCtors]. The `StructBinder.StructType` object +[constructors][StructCtor]. The `StructBinder.StructType` object has the following "static" properties (^Which are accessible from individual instances via `theInstance.constructor`.): @@ -608,7 +770,8 @@ individual instances via `theInstance.constructor`.): a function-typed `ondispose` property, this call replaces it with an array and moves that function into the array. In all other cases, `ondispose` is assumed to be an array and the argument(s) is/are - appended to it. Returns `this`. + appended to it. Returns `this`. See `dispose()`, below, for where + this applies. - `allocCString(str)` Identical to the [StructBinder][] method of the same name. @@ -616,7 +779,7 @@ individual instances via `theInstance.constructor`.): - `hasExternalPointer(object)` Returns true if the given object's `pointer` member refers to an "external" object. That is the case when a pointer is passed to a - [struct's constructor][StructCtors]. If true, the memory is owned by + [struct's constructor][StructCtor]. If true, the memory is owned by someone other than the object and must outlive the object. - `isA(value)` @@ -633,7 +796,7 @@ individual instances via `theInstance.constructor`.): The base StructType prototype has the following members, all of which are inherited by [struct instances](#api-structinstance) and may only -legally be called on concrete struct instances unless noted otherwise: +legally be used with concrete struct instances unless noted otherwise: - `dispose()` Frees, if appropriate, the WASM-allocated memory which is allocated @@ -641,11 +804,11 @@ legally be called on concrete struct instances unless noted otherwise: cleans up the object, a leak in the WASM heap memory pool will result. When `dispose()` is called, if the object has a property named `ondispose` then it is treated as follows: - - If it is a function, it is called with the struct object as its `this`. - That method must not throw - if it does, the exception will be - ignored. + - If it is a function, it is called with the struct object as its + `this`. That method must not throw - if it does, the exception + will be ignored. - If it is an array, it may contain functions, pointers, other - [StructType] instances, and/or JS strings. If an entry is a + [StructType][] instances, and/or JS strings. If an entry is a function, it is called as described above. If it's a number, it's assumed to be a pointer and is passed to the `dealloc()` function configured for the parent [StructBinder][]. If it's a @@ -655,7 +818,12 @@ legally be called on concrete struct instances unless noted otherwise: supported primarily for use as debugging information. - Some struct APIs will manipulate the `ondispose` member, creating it as an array or converting it from a function to array as - needed. + needed. Most simply, `addOnDispose()` is used to manipulate the + on-dispose data. + +- `extraBytes` (integer, read-only) + If this instance was allocated with the `extraBytes` option, this is + that value, else it is 0. - `lookupMember(memberName,throwIfNotFound=true)` Given the name of a mapped struct member, it returns the member @@ -708,6 +876,19 @@ legally be called on concrete struct instances unless noted otherwise: the struct will invalidate older serialized data and (B) serializing member pointers is useless. +- `pointer` (number, read-only) + A read-only numeric property which is the "pointer" returned by the + configured allocator when this object is constructed. After + `dispose()` (inherited from [StructType][]) is called, this property + has the `undefined` value. When calling C-side code which takes a + pointer to a struct of this type, simply pass it `myStruct.pointer`. + Whether this member is of type Number or BigInt depends on whether + the WASM environment is 32-bit (Number) or 64-bit (BigInt). + +- `ptrAdd(args...)` + Equivalent to [StructBinder][]`.ptrAdd(this.pointer, args...)` + or [StructCtor][]`.ptrAdd(this.pointer, args...)`. + - `setMemberCString(memberName,str)` Uses `StructType.allocCString()` to allocate a new C-style string, assign it to the given member, and add the new string to this @@ -725,9 +906,13 @@ legally be called on concrete struct instances unless noted otherwise: from JS be kept to a minimum or that the relationship be one-way: let C manage the strings and only fetch them from JS using, e.g., `memberToJsString()`. - -<a name='api-structctor'></a> +- `zeroOnDispose` (bool, read-only) + True if this instance or its prototype were configured with + the `zeroOnDispose` flag. + + +<a id='api-structctor'></a> API: Struct Constructors ------------------------------------------------------------ @@ -741,29 +926,90 @@ const x = new MyStruct; ``` Normally they should be passed no arguments, but they optionally -accept a single argument: a WASM heap pointer address of memory -which the object will use for storage. It does _not_ take over -ownership of that memory and that memory must be valid at -for least as long as this struct instance. This is used, for example, -to proxy static/shared C-side instances: +accept a single argument: a WASM heap pointer address of memory which +the object will use for storage. It does _not_ take over ownership of +that memory and that memory must remain valid for at least as long as +this struct instance. This is used, for example, to proxy +static/shared C-side instances or those which we simply want to +get access to via this higher-level API for a while: > ``` const x = new MyStruct( someCFuncWhichReturnsAMyStructPointer() ); ... -x.dispose(); // does NOT free the memory +x.dispose(); // does NOT free or zero the memory ``` The JS-side construct does not own the memory in that case and has no way of knowing when the C-side struct is destroyed. Results are specifically undefined if the JS-side struct is used after the C-side -struct's member is freed. +struct's member is freed. However, if the client has allocated that +memory themselves and wants to transfer ownership of it to the +instance, that can be done with: + +> +``` +x.addOnDispose( thePtr ); +``` + +Which will free `thePtr` when `x.dispose()` is called. + +As of 2025-11-10, a third option is available: -> Potential TODO: add a way of passing ownership of the C-side struct -to the JS-side object. e.g. maybe simply pass `true` as the second -argument to tell the constructor to take over ownership. Currently the -pointer can be taken over using something like -`myStruct.ondispose=[myStruct.pointer]` immediately after creation. +> +``` +const x = new MyStruct({ + wrap: ptr, // as per MyStruct(ptr) + takeOwnership: bool, // If true, take ownership of the wrap ptr. + zeroOnDispose: bool , // if true, overrides MyStruct.structInfo.zeroOnDispose + extraBytes: int // bytes to alloc after the end of the struct +}); +``` + +- If `wrap` is set then (A) it must be at least + `MyStruct.structInfo.sizeof` of memory owned by the caller, (B) it + _must_ have been allocated using the same allocator as Jaccwabyt is + configured for, and (C) ownership of it is transfered to the new + object. `zeroOnDispose` and `extraBytes` are disregarded if `wrap` + is set. A falsy `wrap` value is equivalent to not providing one and + a non-integer value, or a number less than 0, triggers an error. + +- If `takeOwnership` is truthy then this object takes over ownership + of the `wrap` pointer (if any). This flag is otherwise ignored. + +- If `zeroOnDispose` or `extraBytes` are are used without `wrap`, each + which is used is set as a read-only propperty on the resulting + `MyStruct` object, except that `zeroOnDispose` is only recorded if + it is truthy and `extraBytes` is only recorded if it is not 0 (a + negative value, or non-integer, triggers an exception). + +- `ondispose`: if set it is passed to the new object's + `addOnDispose()` before the constructor returns, so may have any + value legal for that method. Results are undefined if `ondispose` + and `wrap` have the same pointer value (because `wrap` will already + be cleaned up via `dispose()`, as if it had been passed to + `addOnDispose()`). + +In the case of `extraBytes`, a pointer to the tail of the memory can +be obtained by adding `theInstance.pointer` to +`theInstance.structInfo.sizeof`, with the caveat that `pointer` may be +a BigInt value (on 64-bit WASM), `sizeof` will be a Number and, +spoiler alert, `(BigInt(1) + Number(1))` is not legal. Thus this API +adds a small convenience method to work around that portability issue: + +> +``` +const ptrTail = MyStruct.ptrAdd(theInstance.pointer, theInstance.extraBytes); +``` + +`typeof ptrTail` is `'bigint'` in 64-bit builds and `'number'` in +32-bit. i.e. it's the same type as `theInstance.pointer`. +Equivalently, the inherited `MyStruct` instance method with the same +name adds an instance's own `pointer` value to its arguments: + +``` +const ptrTail = theInstance.ptrAdd(theInstance.extraBytes); +``` These constructors have the following "static" members: @@ -776,6 +1022,10 @@ These constructors have the following "static" members: - `memberKeys(string)` Works exactly as documented for [StructType][]. +- `ptrAdd(args...)` + Equivalent to [StructBinder][]`.ptrAdd(args...)`. The [_inherited_ + method with the same name][StructType] behaves differently. + - `structInfo` The structure description passed to [StructBinder][] when this constructor was generated. @@ -783,14 +1033,14 @@ These constructors have the following "static" members: - `structName` The structure name passed to [StructBinder][] when this constructor was generated. - -<a name='api-structprototype'></a> + +<a id='api-structprototype'></a> API: Struct Prototypes ------------------------------------------------------------ The prototypes of structs created via [the constructors described in -the previous section][StructCtors] are each a struct-type-specific +the previous section][StructCtor] are each a struct-type-specific instance of [StructType][] and add the following struct-type-specific properties to the mix: @@ -802,78 +1052,57 @@ properties to the mix: The name of the struct, as it was given to the [StructBinder][] which created this class. -<a name='api-structinstance'></a> +<a id='api-structinstance'></a> API: Struct Instances ------------------------------------------------------------------------ Instances of structs created via [the constructors described -above][StructCtors] each have the following instance-specific state in -common: - -- `pointer` - A read-only numeric property which is the "pointer" returned by the - configured allocator when this object is constructed. After - `dispose()` (inherited from [StructType][]) is called, this property - has the `undefined` value. When calling C-side code which takes a - pointer to a struct of this type, simply pass it `myStruct.pointer`. +above][StructCtor]. Each inherits all of the methods and properties +from their constructor's prototype. -<a name='appendices'></a> +<a id='appendices'></a> Appendices ============================================================ -<a name='appendix-a'></a> +<a id='appendix-a'></a> Appendix A: Limitations, TODOs, and Non-TODOs ------------------------------------------------------------ - This library only supports the basic set of member types supported - by WASM: numbers (which includes pointers). Nested structs are not - handled except that a member may be a _pointer_ to such a - struct. Whether or not it ever will depends entirely on whether its - developer ever needs that support. Conversion of strings between - JS and C requires infrastructure specific to each WASM environment - and is not directly supported by this library. - -- Binding functions to struct instances, such that C can see and call - JS-defined functions, is not as transparent as it really could be, - due to [shortcomings in the Emscripten - `addFunction()`/`removeFunction()` - interfaces](https://github.com/emscripten-core/emscripten/issues/17323). Until - a replacement for that API can be written, this support will be - quite limited. It _is_ possible to bind a JS-defined function to a - C-side function pointer and call that function from C. What's - missing is easier-to-use/more transparent support for doing so. - - In the meantime, a [standalone - subproject](/file/common/whwasmutil.js) of Jaccwabyt provides such a - binding mechanism, but integrating it directly with Jaccwabyt would - not only more than double its size but somehow feels inappropriate, so - experimentation is in order for how to offer that capability via - completely optional [StructBinderFactory][] config options. - -- It "might be interesting" to move access of the C-bound members into - a sub-object. e.g., from JS they might be accessed via - `myStructInstance.s.structMember`. The main advantage is that it would - eliminate any potential confusion about which members are part of - the C struct and which exist purely in JS. "The problem" with that - is that it requires internally mapping the `s` member back to the - object which contains it, which makes the whole thing more costly - and adds one more moving part which can break. Even so, it's - something to try out one rainy day. Maybe even make it optional and - make the `s` name configurable via the [StructBinderFactory][] - options. (Over-engineering is an arguably bad habit of mine.) - -- It "might be interesting" to offer (de)serialization support. It - would be very limited, e.g. we can't serialize arbitrary pointers in - any meaningful way, but "might" be useful for structs which contain - only numeric or C-string state. As it is, it's easy enough for - client code to write wrappers for that and handle the members in - ways appropriate to their apps. Any impl provided in this library - would have the shortcoming that it may inadvertently serialize - pointers (since they're just integers), resulting in potential chaos - after deserialization. Perhaps the struct description can be - extended to tag specific members as serializable and how to - serialize them. - -<a name='appendix-d'></a> + by WASM: numbers (which includes pointers). + +- Binding JS functions to struct instances, such that C can see and + call JS-defined functions, is not as transparent as it really could + be. [The WhWasmUtil API][whwasmutil.js], and + standalone subproject co-developed with Jaccwabyt, provides such a + binding mechanism. There is some overlap between the two APIs and + they arguably belong bundled together, but doing so would more than + triple Jaccwabyt's size. (That said, the only known Jaccwabyt + deployment uses both APIs, so maybe it's time to merge them (he says + on 2025-11-10). As of this writing, jaccwabyt.js is 38k, half of + which is comments/docs, whereas whwasmutil.js is 100kb and 75% + docs). + +<a id='appendix-b'></a> +Appendix B: Build +------------------------------------------------------------------------ + +In order to support both vanilla JS and ESM (ES6 module) builds from a +single source, this project uses [a preprocessor][c-pp], which requires +only a C compiler: + +> $ make + +The makefile requires GNU make, not BSD or POSIX make. + +The resulting files are in the `js/` subdirectory, in both "vanilla" +JS and ESM formats. + +This tree [includes all of the requisite sources](/dir/tool) and +requires no out-of-tree dependencies beyond the system's libc. + + +<a id='appendix-d'></a> Appendix D: Debug Info ------------------------------------------------------------ @@ -894,7 +1123,7 @@ client code: [StructType][]. -<a name='appendix-g'></a> +<a id='appendix-g'></a> Appendix G: Generating Struct Descriptions From C ------------------------------------------------------------ @@ -1064,18 +1293,23 @@ div.content h3::before { div.content h3 {border-left-width: 2.5em} </style> -[sqlite3]: https://sqlite.org -[emscripten]: https://emscripten.org -[sgb]: https://wanderinghorse.net/home/stephan/ [appendix-g]: #appendix-g -[StructBinderFactory]: #api-binderfactory -[StructCtors]: #api-structctor -[StructType]: #api-structtype +[BigInt64Array]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt64Array +[c-pp]: https://fossil.wanderinghorse.net/r/c-pp +[Emscripten]: https://emscripten.org +[jaccwabyt.js]: /file/jaccwabyt/jaccwabyt.c-pp.js +[MDN]: https://developer.mozilla.org/docs/Web/API +[sgb]: https://wanderinghorse.net/home/stephan/ +[sqlite3]: https://sqlite.org [StructBinder]: #api-structbinder +[StructBinderFactory]: #api-binderfactory +[StructCtor]: #api-structctor [StructInstance]: #api-structinstance -[^export-func]: In Emscripten, add its name, prefixed with `_`, to the - project's `EXPORT_FUNCTIONS` list. -[BigInt64Array]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt64Array +[StructType]: #api-structtype [TextDecoder]: https://developer.mozilla.org/en-US/docs/Web/API/TextDecoder [TextEncoder]: https://developer.mozilla.org/en-US/docs/Web/API/TextEncoder -[MDN]: https://developer.mozilla.org/docs/Web/API +[WASI-SDK]: https://github.com/WebAssembly/wasi-sdk +[whwasmutil.js]: /file/wasmutil/whwasmutil.c-pp.js + +[^export-func]: In Emscripten, add its name, prefixed with `_`, to the + project's `EXPORT_FUNCTIONS` list. diff --git a/ext/wasm/libcmpp.c b/ext/wasm/libcmpp.c new file mode 100644 index 000000000..717f56ea9 --- /dev/null +++ b/ext/wasm/libcmpp.c @@ -0,0 +1,16877 @@ +/** + This C file contains both the header and source file for c-pp, + a.k.a. libcmpp. +*/ +#if !defined(NET_WANDERINGHORSE_LIBCMPP_C_INCLUDED) +#define NET_WANDERINGHORSE_LIBCMPP_C_INCLUDED +#if !defined(_POSIX_C_SOURCE) +# define _POSIX_C_SOURCE 200809L /* for fdopen() in stdio.h */ +#endif +#define CMPP_AMALGAMATION +#if !defined(NET_WANDERINGHORSE_LIBCMPP_H_INCLUDED) +/** + This is the auto-generated "amalgamation build" of libcmpp. It was amalgamated + using: + + ./c-pp -I. -I./src -Dsrcdir=./src -Dsed=/usr/bin/sed -o libcmpp.h ./tool/libcmpp.c-pp.h -o libcmpp.c ./tool/libcmpp.c-pp.c + + with libcmpp 2.0.x c02f3e3e2d3f3573a9a33c1474c2e52fc48e52c70730404a90d0ae51517e7d37 @ 2026-03-08 14:50:35.123 UTC +*/ +#define CMPP_PACKAGE_NAME "libcmpp" +#define CMPP_LIB_VERSION "2.0.x" +#define CMPP_LIB_VERSION_HASH "c02f3e3e2d3f3573a9a33c1474c2e52fc48e52c70730404a90d0ae51517e7d37" +#define CMPP_LIB_VERSION_TIMESTAMP "2026-03-08 14:50:35.123 UTC" +#define CMPP_LIB_CONFIG_TIMESTAMP "2026-03-08 15:32 GMT" +#define CMPP_VERSION CMPP_LIB_VERSION " " CMPP_LIB_VERSION_HASH " @ " CMPP_LIB_VERSION_TIMESTAMP +#define CMPP_PLATFORM_EXT_DLL ".so" +#define CMPP_MODULE_PATH ".:/usr/local/lib/cmpp" + +#if !defined(NET_WANDERINGHORSE_CMPP_H_INCLUDED_) +#define NET_WANDERINGHORSE_CMPP_H_INCLUDED_ +/* +** 2022-11-12: +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +** +************************************************************************ +** +** The C-minus Preprocessor: C-like preprocessor. Why? Because C +** preprocessors _can_ process non-C code but generally make quite a +** mess of it. The purpose of this library is a customizable +** preprocessor suitable for use with arbitrary UTF-8-encoded text. +** +** The supported preprocessor directives are documented in the +** README.md hosted with this file (or see the link below). +** +** Any mention of "#" in the docs, e.g. "#if", is symbolic. The +** directive delimiter is configurable and defaults to "##". Define +** CMPP_DEFAULT_DELIM to a string when compiling to define the default +** at build-time. +** +** This API is presented as a library but was evolved from a +** monolithic app. Thus is library interface is likely still missing +** some pieces needed to make it more readily usable as a library. +** +** Author(s): +** +** - Stephan Beal <https://wanderinghorse.net/home/stephan/> +** +** Canonical homes: +** +** - https://fossil.wanderinghorse.net/r/c-pp +** - https://sqlite.org/src/file/ext/wasm/c-pp-lite.c +** +** With the former hosting this app's SCM and the latter being the +** original deployment of c-pp.c, from which this library +** evolved. SQLite uses a "lite" version of c-pp, whereas _this_ copy +** is its much-heavier-weight fork. +*/ + +#if defined(CMPP_HAVE_AUTOCONFIG_H) +#include "libcmpp-autoconfig.h" +#endif +#if defined(HAVE_AUTOCONFIG_H) +#include "autoconfig.h" +#endif +#if defined(HAVE_CONFIG_H) +#include "config.h" +#endif + +#ifdef _WIN32 +# if defined(BUILD_libcmpp_static) || defined(CMPP_AMALGAMATION_BUILD) +# define CMPP_EXPORT extern +# elif defined(BUILD_libcmpp) +# define CMPP_EXPORT extern __declspec(dllexport) +# else +# define CMPP_EXPORT extern __declspec(dllimport) +# endif +#else +# define CMPP_EXPORT extern +#endif + +/** + cmpp_FILE is a portability hack for WASM builds, where we want to + elide the (FILE*)-using pieces to avoid having a dependency on + Emscripten's POSIX I/O proxies. In all non-WASM builds it is + guaranteed to be an alias for FILE. On WASM builds it is guaranteed + to be an alias for void and the cmpp APIs which use it become + inoperative in WASM builds. + + That said: the code does not yet support completely compiling out + (FILE*) dependencies, and may not be able to because canonical + sqlite3 (upon which it is based) depends heavily on file + descriptors and slightly on FILE handles. +*/ +#if defined(__EMSCRIPTEN__) || defined(__wasm__) || defined(__wasi__) + typedef void cmpp_FILE; +# define CMPP_PLATFORM_IS_WASM 1 +#else + #include <stdio.h> + typedef FILE cmpp_FILE; +# define CMPP_PLATFORM_IS_WASM 0 +#endif + +#include <stdint.h> +#include <inttypes.h> /* PRIu32 and friends */ +#include <stdbool.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/** + 32-bit flag bitmask type. This typedef exists primarily to improve + legibility of function signatures and member structs by conveying + their intent for use as flags instead of result codes or lengths. +*/ +typedef uint32_t cmpp_flag32_t; +//typedef uint16_t cmpp_flag16_t; + +/** + An X-macro which invokes its argument (a macro name) to expand to + all possible values of cmpp_rc_e entries. The macro name passed to + it is invoked once for each entry and passed 3 arguments: the enum + entry's full name (CMPP_RC_...), its integer value, and a help-text + string. +*/ +#define cmpp_rc_e_map(E) \ + E(CMPP_RC_OK, 0, \ + "The quintessential not-an-error value.") \ + E(CMPP_RC_ERROR, 100, \ + "Generic/unknown error.") \ + E(CMPP_RC_NYI, 101, \ + "A placeholder return value for not yet implemented functions.") \ + E(CMPP_RC_OOM, 102, \ + "Out of memory. Indicates that a resource allocation " \ + "request failed.") \ + E(CMPP_RC_MISUSE, 103, \ + "API misuse (invalid args)") \ + E(CMPP_RC_RANGE, 104, \ + "A range was violated.") \ + E(CMPP_RC_ACCESS, 105, \ + "Access to or locking of a resource was denied " \ + "by some security mechanism or other.") \ + E(CMPP_RC_IO, 106, \ + "Indicates an I/O error. Whether it was reading or " \ + "writing is context-dependent.") \ + E(CMPP_RC_NOT_FOUND, 107, \ + "Requested resource not found.") \ + E(CMPP_RC_ALREADY_EXISTS, 108, \ + "Indicates that a to-be-created resource already exists.") \ + E(CMPP_RC_CORRUPT, 109, \ + "Data consistency problem.") \ + E(CMPP_RC_SYNTAX, 110, \ + "Some sort of syntax error.") \ + E(CMPP_RC_NOOP, 111, \ + "Special sentinel value for some APIs.") \ + E(CMPP_RC_UNSUPPORTED, 112, \ + "An unsupported operation was request.") \ + E(CMPP_RC_DB, 113, \ + "Indicates db-level error (e.g. statement prep failed). In such " \ + "cases, the error state of the related db handle (cmpp_db) " \ + "will be updated to contain more information directly from the " \ + "db driver.") \ + E(CMPP_RC_NOT_DEFINED, 114, \ + "Failed to expand an undefined value.") \ + E(CMPP_RC_ASSERT, 116, "An #assert failed.") \ + E(CMPP_RC_TYPE, 118, \ + "Indicates that some data type or logical type is incorrect.") \ + E(CMPP_RC_CANNOT_HAPPEN, 140, \ + "This is intended only for internal use, to " \ + "report conditions which \"cannot possibly happen\".") \ + E(CMPP_RC_HELP, 141, \ + "--help was used in the arguments to cmpp_process_argv()") \ + E(CMPP_RC_NO_DIRECTIVE, 142, \ + "A special case of CMPP_RC_NOT_FOUND needed to disambiguate.") \ + E(CMPP_RC_end,200, \ + "Must be the final entry in the enum. Used for creating client-side " \ + "result codes which are guaranteed to live outside of this one's " \ + "range.") + +/** + Most functions in this library which return an int return result + codes from the cmpp_rc_e enum. None of these entries are + guaranteed to have a specific value across library versions except + for CMPP_RC_OK, which is guaranteed to always be 0 (and the API + guarantees that no other code shall have a value of zero). + + The only reasons numbers are hard-coded to the values is to + simplify debugging during development. Clients may use + cmpp_rc_cstr() to get some human-readable (or programmer-readable) + form for any given value in this enum. +*/ +enum cmpp_rc_e { +#define E(N,V,H) N = V, + cmpp_rc_e_map(E) +#undef E +}; +typedef enum cmpp_rc_e cmpp_rc_e; + +/** + Returns the string form of the given cmpp_rc_e value or NULL if + it's not a member of that enum. +*/ +char const * cmpp_rc_cstr(int rc); + +/** + CMPP_BITNESS specifies whether the library should use 32- or 64-bit + integer types for its size/length measurements. It's difficult to + envision use cases for a preprocessor which would require counters + or rangers larger than 32 bits provide for, so the default is 32 + bits. Builds created with different CMPP_BITNESS values are + not binary-compatible. +*/ +#define CMPP_BITNESS 32 +#if 32==CMPP_BITNESS +/** + Unsigned integer type for string/stream lengths. 32 bits is + sufficient for all but the weirdest of inputs and outputs. +*/ +typedef uint32_t cmpp_size_t; + +/** + A signed integer type indicating the maximum length of strings or + byte ranges in a stream. It is most frequently used in API + signatures where a negative value means "if it's negative then use + strlen() to count it". +*/ +typedef int32_t cmpp_ssize_t; + +/** + The printf-format-compatible format letter (or group of letters) + appropriate for use with cmpp_size_t. Contrary to popular usage, + size_t cannot be portably used with printf(), without careful + casting, because it has neither a fixed size nor a standardized + printf/scanf format specifier (like the stdint.h types do). +*/ +#define CMPP_SIZE_T_PFMT PRIu32 +#elif 64==CMPP_BITNESS +typedef uint64_t cmpp_size_t; +typedef int64_t cmpp_ssize_t; +#define CMPP_SIZE_T_PFMT PRIu64 +#else +#error "Invalid CMPP_BITNESS value. Expecting 32 or 64." +#endif + +/** + Generic interface for streaming in data. Implementations must read + (at most) *n bytes from their input, copy it to dest, assign *n to + the number of bytes actually read, return 0 on success, and return + non-0 cmpp_rc_e value on error (e.g. CMPP_RC_IO). + + When called, *n is the max length to read. On return, *n must be + set to the amount actually read. Implementations may need to + internally distinguish a short read due to EOF from a short read + due to an I/O error, e.g. using feof() and/or ferror(). A short + read for EOF is not an error but a short read for input failure is. + This library invariably treats a short read as EOF. + + The state parameter is the implementation-specified input + file/buffer/whatever channel. +*/ +typedef int (*cmpp_input_f)(void * state, void * dest, cmpp_size_t * n); + +/** + Generic interface for streaming out data. Implementations must + write n bytes from src to their destination channel and return 0 on + success, or a value from the cmpp_rc_e enum on error + (e.g. CMPP_RC_IO). The state parameter is the + implementation-specified output channel. + + It is implementation-defined whether an n of 0 is legal. This + library refrains from passing 0 to these functions. + + In the context of cmpp, the library makes no guarantees that output + will always end at a character boundary. It may send any given + multibyte character as the end resp. start of two calls to this + function. If that is of a concern for implementors of these + functions (e.g. because they're appending the output to a UI + widget), they may need to buffer all of the output before applying + it (see cmpp_b), or otherwise account for partial characters. + + That said: the core library, by an accident of design, will always + emit data at character boundaries, assuming that its input is + well-formed UTF-8 text (which cmpp does not validate to be the + case). Custom cmpp_dx_f() implementations are not strictly + required to do so but, because of how cmpp is used, almost + certainly will. But relying on that is ill-advised. +*/ +typedef int (*cmpp_output_f)(void * state, void const * src, + cmpp_size_t n); + +/** + Generic interface for flushing arbitrary output streams. Must + return 0 on success, a non-0 cmpp_rc_e value on error. When in + doubt, return CMPP_RC_IO on error. The interpretation of the state + parameter is implementation-specific. +*/ +typedef int (*cmpp_flush_f)(void * state); + +typedef struct cmpp_pimpl cmpp_pimpl; +typedef struct cmpp_api_thunk cmpp_api_thunk; +typedef struct cmpp_outputer cmpp_outputer; + +/** + The library's primary class. Each one of these represents a + separate preprocessor instance. + + See also: cmpp_dx (the class which client-side extensions interact + with the most). +*/ +struct cmpp { + + /** + API thunk object to support use via loadable modules. Client code + does not normally need to access this member, but it's exposed + here to give loadable modules more flexibility in how they use + the thunk. + + This pointer is _always_ the same singleton object. The library + never exposes a cmpp object with a NULL api member. + */ + cmpp_api_thunk const * const api; + + /** + Private internal state. + */ + cmpp_pimpl * const pimpl; +}; +typedef struct cmpp cmpp; + +/** + Flags for use with cmpp_ctor_cfg::flags. +*/ +enum cmpp_ctor_e { + /* Sentinel value. */ + cmpp_ctor_F_none = 0, + /* Disables #include. */ + cmpp_ctor_F_NO_INCLUDE = 0x01, + /* Disables #pipe. */ + cmpp_ctor_F_NO_PIPE = 0x02, + /* Disables #attach, #detach, and #query. */ + cmpp_ctor_F_NO_DB = 0x04, + /* Disables #module. */ + cmpp_ctor_F_NO_MODULE = 0x08, + /** + Disable all built-in directives which may work with the filesystem + or invoke external processes. Client-defined directives with the + cmpp_d_F_NOT_IN_SAFEMODE flag are also disabled. Directives + disabled via the cmpp_ctor_F_NO_... flags (or equivalent library + built-time options) do not get registered, so will trigger + "unknown directive" errors rather than safe-mode violation errors. + */ + cmpp_ctor_F_SAFEMODE = 0x10, +}; + +/** + A configuration object for cmpp_ctor(). This type may be extended + as new construction-time customization opportunities are + discovered. +*/ +struct cmpp_ctor_cfg { + /** + Bitmask from the cmpp_ctor_e enum. + */ + cmpp_flag32_t flags; + /** + If not NULL then this must name either an existing SQLite3 db + file or the name of one which can be created on demand. If NULL + then an in-memory or temporary database is used (which one is + unspecified). The library copies these bytes, so they need not be + valid after a call to cmpp_ctor(). + */ + char const * dbFile; +}; +typedef struct cmpp_ctor_cfg cmpp_ctor_cfg; + +/** + Assigns *pp to a new cmpp or NULL on OOM. Any non-NULL return + value must eventually be passed to cmpp_dtor() to free it. + + The cfg argument, if not NULL, holds config info for the new + instance. If NULL, an instance with unspecified defaults is + used. These configuration pieces may not be modified after the + instance is created. + + It returns 0 if *pp is ready to use and non-0 if either allocation + fails (in which case *pp will be set to 0) or initialization of *pp + failed (in which case cmpp_err_get() can be used to determine why + it failed). In either case, the caller must eventually pass *pp to + cmpp_dtor() to free it. + + If the library is built with the symbol CMPP_CTOR_INSTANCE_INIT + defined, it must refer to a function with this signature: + + int CMPP_CTOR_INSTANCE_INIT(cmpp *); + + The library calls this before returning and arranges to call it + lazily if pp gets reset. The intent is that the init function + installs custom directives using cmpp_d_register(). That + initialization, on error, is expected to set its argument's error + state with cmpp_err_set(). +*/ +CMPP_EXPORT int cmpp_ctor(cmpp **pp, cmpp_ctor_cfg const * cfg); + +/** + If pp is not NULL, it is passed to cmpp_reset() and then freed. +*/ +CMPP_EXPORT void cmpp_dtor(cmpp *pp); + + +/** + realloc(3)-compatible allocator used by the library. + + This API very specifically uses sqlite3_realloc() as its basis. +*/ +CMPP_EXPORT void * cmpp_mrealloc(void * p, size_t n); + +/** + malloc(3)-compatible allocator used by the library. + + This API very specifically uses sqlite3_malloc() as its basis. +*/ +CMPP_EXPORT void * cmpp_malloc(size_t n); + +/** + free(3)-compatible deallocator. It can also be used as a destructor + for cmpp_d_register() _if_ the memory in question is allocated by + cmpp_malloc(), cmpp_realloc(), or the sqlite3_malloc() family of + APIs. + + This is not called cmpp_free() to try to avoid any confusion with + cmpp_dtor(). +*/ +CMPP_EXPORT void cmpp_mfree(void *); + +/** + If m is NULL then pp's persistent error code is set to CMPP_RC_OOM, + else this is a no-op. Returns pp's error code. + + To simplify certain uses, pp may be NULL, in which case this + function returns CMPP_RC_OOM if m is NULL and 0 if it's not. +*/ +CMPP_EXPORT int cmpp_check_oom(cmpp * const pp, void const * const m ); + +/** + Re-initializes all state of pp. This saves some memory for reuse + but resets it all to default states. This closes the database and + will also reset any autoloader, policies, or delimiter + configurations to their compile-time defaults. It retains only a + small amount of state, like any configuration which was passed to + cmpp_ctor(). + + After calling this, pp is in a cleanly-initialized state and may be + re-used with the cmpp API. Its database will not be initialized + until an API which needs it is called, so pp can be used with + functions which may otherwise be prohibited after the db is + opened. (Do we still have any?) + + As of this writing, this is the only way to reliably recover a cmpp + instance from any significant errors. Errors may do things like + leave savepoints out of balance, and this cleanup step resets all + of that state. However, it also loses state like the autoloader. + + TODO?: we need(?) a partial-clear operation which keeps some of the + instance's state, most notably custom directives, the db handle, + and any cached prepared statements. See cmpp_err_set() for the + distinction between recoverable and non-recoverable errors. +*/ +CMPP_EXPORT void cmpp_reset(cmpp *pp); + +#if 0 +Not yet; +/** + If called before pp has initialized its database, this sets the + file name used for that database. If called afterwards, pp's error + state is updated and CMPP_RC_MISUSE is returned. If called while + pp has error state set, that code is returned without side-effects. + + This does not open the database. It is opened on demand when + processing starts. + + On success it returns 0 and this function makes a copy of zName. + + As a special case, zName may be NULL to use the default name, but + there is little reason to do so unless one changes their mind after + setting it to non-NULL. + */ +CMPP_EXPORT int cmpp_db_name_set(cmpp *pp, const char * zName); +#endif + +/** + Returns true if the bytes in the range [zName, zName+n) comprise a + legal name for a directive or a define. + + It disallows any control characters, spaces, and most punctuation, + but allows alphanumeric (but must not start with a number) as well + as any of: -./:_ (but it may not start with '-'). Any characters + with a high bit set are assumed to be UTF-8 and are permitted as + well. + + The name's length is limited, rather arbitrarily, to 64 bytes. + + If the key is not legal then false is returned and if zErrPos is + not NULL then *zErrPos is set to the position in zName of the first + offending character. If validation fails because n is too long then + *zErrPos (if zErrPos is not NULL) will be set to 0. + + Design note: this takes unsigned characters because it most + commonly takes input from cmpp_args::z strings. +*/ +CMPP_EXPORT bool cmpp_is_legal_key(unsigned char const *zName, + cmpp_size_t n, + unsigned char const **zErrPos); + +/** + Adds the given `#define` macro name to the list of macros, overwriting + any previous value. + + zKey must be NUL-terminated and legal as a key. The rules are the + same as for cmpp_is_legal_key() except that a '=' is also permitted + if it's not at the start of the string because... + + If zVal is NULL then zKey may contain an '=', from which the value + will be extracted. If zVal is not NULL then zKey may _not_ contain + an '='. + + The ability for zKey to contain a key=val was initially to + facilitate input from the CLI (e.g. -Dfoo=bar) because cmpp was + initially a CLI app (as opposed to a library). It's considered a + "legacy" feature, not recommended for most purposes, but it _is_ + convenient for that particular purpose. + + Returns 0 on success and updates pp's error state on error. + + See: cmpp_define_v2() + See: cmpp_undef() +*/ +CMPP_EXPORT int cmpp_define_legacy(cmpp *pp, const char * zKey, + char const *zVal); + +/** + Works like cmpp_define_legacy() except that it does not examine zKey to + see if it contains an '='. +*/ +CMPP_EXPORT int cmpp_define_v2(cmpp *pp, const char * zKey, char const *zVal); + +/** + Removes the given `#define` macro name from the list of + macros. zKey is, in this case, treated as a GLOB pattern, and all + matching defines are deleted. + + If nRemoved is not NULL then, on success, it is set to the number + of entries removed by this call. + + Returns 0 on success and updates pp's error state on error. It is not + an error if no value was undefined. + + This does _not_ affect defines made using cmpp_define_shadow(). +*/ +CMPP_EXPORT int cmpp_undef(cmpp *pp, const char * zKey, + unsigned int *nRemoved); + +/** + This works similarly to cmpp_define_v2() except that: + + - It does not permit its zKey argument to contain the value + part like that function does. + + - The new define "shadows", rather than overwrites, an existing + define with the same name. + + All APIs which look up define keys will get the value of the shadow + define. The shadow can be uninstalled with cmpp_define_unshadow(), + effectively restoring its previous value (if any). That function + should be called one time for each call to this one, passing the + same key to each call. A given key may be shadowed any number of + times by this routine. Each one saves the internal ID of the shadow + into *pId (and pId must not be NULL). That value must be passed to + cmpp_define_unshadow() to ensure that the "shadow stack" stays + balanced in the face of certain error-handling paths. + + cmpp_undef() will _not_ undefine an entry added through this + interface. + + Returns pp's persistent error code (0 on success). + + Design note: this function was added to support adding a define + named __FILE__ to input scripts which works like it does in a C + preprocessor. Alas, supporting __LINE__ would be much more costly, + as it would have to be updated in the db from several places, so + its cost would outweigh its meager benefits. +*/ +CMPP_EXPORT int cmpp_define_shadow(cmpp *pp, char const *zKey, + char const *zVal, + int64_t * pId); + +/** + Removes the most shadow define matching the zKey and id values + which where previously passed to cmpp_define_shadow(). It is not + an error if no match is found, in which case this function has no + visible side-effects. + + Unlike cmpp_undef(), zKey is matched precisely, not against a glob. + + In order to keep the "shadow stack" properly balanced, this will + delete any shadow entries for the given key which have the same id + or a newer one (i.e. they were left over from a missed call to + cmpp_define_unshadow()). + + Returns pp's persistent error code (0 on success). +*/ +CMPP_EXPORT int cmpp_define_unshadow(cmpp *pp, char const *zKey, + int64_t id); + +/** + Adds the given dir to the list of includes. They are checked in the + order they are added. +*/ +CMPP_EXPORT int cmpp_include_dir_add(cmpp *pp, const char * zKey); + +/** + Sets pp's default output channel. If pp already has a channel, it + is closed[^1]. + + The second argument, if not NULL, is _bitwise copied_, which has + implications for the ownership of out->state (see below). If it is + is NULL, cmpp_outputer_empty is copied in its place, which makes + further output a no-op. + + The third argument is a symbolic name for the channel (perhaps its + file name). It is used in debugging and error messages. cmpp does + _not_ copy it, so its bytes must outlive the cmpp instance. (In + practice, the byte names come from main()'s argv or scope-local + strings in the same scope as the cmpp instance.) This argument + should only be NULL if the second argument is. + + cmpp_reset(), or opening another channel, will end up calling + out->cleanup() (if it's not NULL) and passing it a pointer to a + _different_ cmpp_outputer object, but with the _same_ + cmpp_outputer::state pointer, which may invalidate out->state. + + To keep cmpp from doing that, make a copy of the output object, set + the cleanup member of that copy to NULL, then pass that copy to this + function. It is then up to the client to call out->cleanup(out) when + the time is right. + + For example: + + ``` + cmpp_outputer my = cmpp_outputer_FILE; + my.state = cmpp_fopen("/some/file", "wb"); + cmpp_outputer tmp = my; + tmp.cleanup = NULL; + cmpp_outputer_set( pp, &tmp, "my file"); + ... + my.cleanup(&my); // will cmpp_fclose(my.state) + ``` + + Potential TODO: internally store the output channel as a pointer. + It's not clear whether that would resolve the above grief or + compound it. + + [^1]: depending on the output channel, it might not _actually_ be + closed, but pp is disassociated from it, in any case. +*/ +CMPP_EXPORT +void cmpp_outputer_set(cmpp *pp, cmpp_outputer const *out, char const *zName); + +/** + Treats the range (zIn,zIn+nIn] as a complete cmpp input and process + it appropriately. zName is the name of the input for purposes of + error messages. If nIn is negative, strlen() is used to calculate + it. + + This is a no-op if pp has any error state set. It returns pp's + persistent error code. +*/ +CMPP_EXPORT int cmpp_process_string(cmpp *pp, const char * zName, + unsigned char const * zIn, + cmpp_ssize_t nIn); + +/** + A thin proxy for cmpp_process_string() which reads its input from + the given file. Returns 0 on success, else returns pp's persistent + error code. +*/ +CMPP_EXPORT int cmpp_process_file(cmpp *pp, const char * zName); + +/** + A thin proxy for cmpp_process_string() which reads its input from + the given input source, consuming it all before passing it + on. Returns 0 on success, else returns pp's persistent error code. +*/ +CMPP_EXPORT int cmpp_process_stream(cmpp *pp, const char * zName, + cmpp_input_f src, void * srcState); + +/** + Process the given main()-style arguments list. When calling from + main(), be sure to pass it main()'s (argc+1, argv+1) to skip argv[0] + (the binary's name). + + Each argument is expected to be one of the following: + + 1) One of --help or -?: causes this function to return CMPP_RC_HELP + without emitting any output. + + 2) -DX or -DX=Y: sets define X to 1 (if no "=" is used and no Y given) or to + Y. + + 3) -UX: unsets all defines matching glob X. + + 4) -FX=Y: works like -DX=Y but treats Y as a filename and sets X to + the contents of that file. + + 5) -IX: adds X to the "include path". If _no_ include path is + provided then cmpp assumes a path of ".", but if _any_ paths are + provided then it does not assume that "." is in the path. + + 6) --chomp-F: specifies whether subsequent -F flags should "chomp" + one trailing newline from their input files. + + 7) --delimiter|-d=X sets the directive delimiter to X. Its default + is a compile-time constant. + + 8) --output|-o=filename: sets the output channel to the given file. + A value of "-" means stdout. If no output channel is opened when + this is called, and files are to be processed, stdout is + assumed. (That's a historical artifact from earlier evolutions.) + To override that behavior use cmpp_outputer_set(). + + 9) --file|-f=filename: sets the input channel to the given file. + A value of "-" means stdin. + + 10) -e=SCRIPT Treat SCRIPT as a complete c-pp input and process it. + Because it's difficult to pack multiple lines of text into this, + it's really of use for testing #expr and #assert. + + 11) --@policy=X sets the @token@ parsing policy. X must be + one of (retain, elide, error, off) and defaults to off. + + 12) -@: shorthand for --@policy=error. + + 13) --sql-trace: enables tracing of all SQL statements to stderr. + This is useful for seeing how a script interacts with the + database. Use --no-sql-trace to disable it. + + 14) --sql-trace-x: like --sql-trace but replaces bound parameter + placeholders with their SQL values. Use --no-sql-trace to disable + it. + + 15) --dump-defines: emit all defines to stdout. They should + arguably go to stderr but that interferes with automated testing. + + Any argument which does not match one of the above flags, and does + not start with a "-", is treated as if it were passed to the --file + flag. + + Flags may start with either 1 or 2 dashes - they are equivalent. + + Flags which take a value may either be in the form X=Y or X Y, i.e. + may be a single argv entry or a pair of them. + + It performs two passes on the arguments: the first is for validation + checking for --help/-?. No processing of the input(s) and output(s) + happens unless the first pass completes. Similarly, no validation of + whether any provided filename are actually readable is performed + until the second pass. + + Arguments are processed in the order they are given. Thus the following + have completely different meanings: + + 1) -f foo.in -Dfoo + 2) -Dfoo -f foo.in + + The former will process foo.in before defining foo. + + This behavior makes it possible to process multiple input files in + a single go: + + --output foo.out foo.in -Dfoo foo.in -Ufoo --output bar.out -Dbar foo.in + + It returns 0 on success. cmpp_err_get() can be used to fetch any + error message. +*/ +CMPP_EXPORT int cmpp_process_argv(cmpp *pp, int argc, char const * const * argv); + +/** + Intended to be called if cmpp_process_argv() returns CMPP_RC_HELP. + It emits --help-style text to the given output stream. + As the first argument pass it either argv[0] or NULL. The second + should normally be stdout or stderr. + + Reminder to self: this could take a (cmpp_output_f,void*) pair + instead, and should do so for the sake of WASI builds, but its impl + currently relies heavily on fprintf() formatting. +*/ +CMPP_EXPORT void cmpp_process_argv_usage(char const *zAppName, + cmpp_FILE *os); + +/** + Returns pp's current error number (from the cmpp_rc_e enum) and + sets *zMsg (if zMsg is not NULL) to the error string. The bytes are + owned by pp and may be invalidated by any functions which take pp + as an argument. + + See cmpp_err_get() for more information. + +*/ +CMPP_EXPORT int cmpp_err_get(cmpp *pp, char const **zMsg); + +/** + Sets or clears (if 0==rc) pp's persistent error state. zFmt may be + NULL or a format string compatible with sqlite3_mprintf(). + + To simplify certain uses, this is a no-op if pp is NULL, returning + rc without other side effects. + + Returns rc with one exception: if allocation of a copy of the error + string fails then CMPP_RC_OOM will be returned (and pp will be + updated appropriately). + + If pp is currently processing a script, the resulting error string + will be prefixed with the name of the current input script and the + line number of the directive which triggered the error. + + It is legal for zFmt to be NULL or an empty string, in which case a + default, vague error message is used (without requiring allocation + of a new string). + + Recoverable vs. unrecoverable errors: + + Most cmpp APIs become no-ops if their cmpp object has error state + set, treating any error as unrecoverable. That approach simplifies + writing code for it by allowing multiple calls to be chained + without concern for whether the previous one succeeded. + + ACHTUNG: simply clearing the error state by passing 0 as the 2nd + argument to this function is _not_ enough to recover from certain + errors. e.g. an error in the middle of a script may leave db + savepoints imbalanced. The only way to _fully_ recover from any + significant failures is to use cmpp_reset(), which resets all of + pp's state. + + APIs which may set the error state but are recoverable by simply + clearing that state will document that. Errors from APIs which do + not claim to be recoverable in error cases must be treated as + unrecoverable. + + See cmpp_err_get() for more information. + + FIXME: we need a different variant for WASM builds, where variadics + aren't a usable thing. + + Potential TODO: change the error-reporting interface to support + distinguishing from recoverable and non-recoverable errors. "The + problem" is that no current uses need that - they simply quit and + free up the cmpp instance on error. Maybe that's the way it + _should_ be. +*/ +CMPP_EXPORT int cmpp_err_set(cmpp *pp, int rc, char const *zFmt, ...); + +/** + A variant of cmpp_err_set() which is not variadic, as a consolation + for WASM builds. zMsg may be NULL. The given string, if not NULL, + is copied. +*/ +CMPP_EXPORT int cmpp_err_set1(cmpp *pp, int rc, char const *zMsg); + +#if 0 +/** + Clears any error state in pp. Most cmpp APIs become no-ops if their + cmpp instance has its error flag set. + + See cmpp_err_get() for important details about doing this. +*/ +//CMPP_EXPORT void cmpp_err_clear(cmpp *pp); + +/** + This works like a combination of cmpp_err_get() and + cmpp_err_clear(), in that it clears pp's error state by transferring + ownership of it to the caller. If pp has any error state, *zMsg is + set to the error string and the error code is returned, else 0 is + returned and *zMsg is set to 0. + + The string returned via *zMsg must eventually be passed to + cmpp_mfree() to free it. + + This function is provided simply as an optimization to avoid + having to copy the error string in some cases. + + ACHTUNG: see the ACHTUNG in cmpp_err_clear(). +*/ +CMPP_EXPORT int cmpp_err_take(cmpp *pp, char **zMsg); +#endif + +/** + Returns pp's current error code, which will be 0 if it currently + has no error state. + + To simplify certain uses, this is a no-op if pp is NULL, returning + 0. +*/ +CMPP_EXPORT int cmpp_err_has(cmpp const * pp); + +/** + Returns true if pp was initialized in "safe mode". That is: if the + cmpp_ctor_F_SAFEMODE flag was passed to cmpp_ctor(). + + To simplify certain uses, this is a no-op if pp is NULL, returning + false. +*/ +CMPP_EXPORT bool cmpp_is_safemode(cmpp const * pp); + +/** + Starts a new SAVEPOINT in the database. Returns non-0, and updates + pp's persistent error state, on failure. + + If this returns 0, the caller is obligated to later call either + cmpp_sp_commit() or cmpp_sp_rollback() later. +*/ +CMPP_EXPORT int cmpp_sp_begin(cmpp *pp); + +/** + Commits the most recently-opened savepoint UNLESS pp's error state + is set, in which case this behaves like cmpp_sp_rollback(). + Returns 0 on success. + + A call to cmpp_sp_begin() which returns 0 obligates the caller to + call either cmpp_sp_rollback() or cmpp_sp_commit(). It is illegal + for either to be called in any other context. +*/ +CMPP_EXPORT int cmpp_sp_commit(cmpp *pp); + +/** + Rolls back the most recently-opened savepoint. Returns 0 on + success. + + A call to cmpp_sp_begin() which returns 0 obligates the caller to + call either cmpp_sp_rollback() or cmpp_sp_commit(). It is illegal + for either to be called in any other context. +*/ +CMPP_EXPORT int cmpp_sp_rollback(cmpp *pp); + +/** + A cmpp_output_f() impl which requires state to be a (FILE*), which + this function passes the call on to fwrite(). Returns 0 on + success, CMPP_RC_IO on error. + + If state is NULL then stdout is used. +*/ +CMPP_EXPORT int cmpp_output_f_FILE(void * state, void const * src, cmpp_size_t n); + +/** + A cmpp_output_f() impl which requires state to be a ([const] int*) + referring to a writable file descriptor, which this function + dereferences and passes to write(2). +*/ +CMPP_EXPORT int cmpp_output_f_fd(void * state, void const * src, cmpp_size_t n); + +/** + A cmpp_input_f() implementation which requires that state be + a readable (FILE*) handle, which it passes to fread(3). +*/ +CMPP_EXPORT int cmpp_input_f_FILE(void * state, void * dest, cmpp_size_t * n); + +/** + A cmpp_input_f() implementation which requires that state be a + readable file descriptor, in the form of an ([const] int*), which + this function passes to write(2). +*/ +CMPP_EXPORT int cmpp_input_f_fd(void * state, void * dest, cmpp_size_t * n); + +/** + A cmpp_flush_f() impl which expects pFile to be-a (FILE*) opened + for writing, which this function passes the call on to + fflush(). If fflush() returns 0, so does this function, else it + returns non-0. +*/ +CMPP_EXPORT int cmpp_flush_f_FILE(void * pFile); + +/** + A generic streaming routine which copies data from an + cmpp_input_f() to an cmpp_outpuf_f(). + + Reads all data from inF(inState,...) in chunks of an unspecified + size and passes them on to outF(outState,...). It reads until inF() + returns fewer bytes than requested or returns non-0. Returns the + result of the last call to outF() or (if reading fails) inF(). + Results are undefined if either of inState or outState arguments + are NULL and their callbacks require non-NULL. (This function + cannot know whether a NULL state argument is legal for the given + callbacks.) + + Here is an example which basically does the same thing as the + cat(1) command on Unix systems: + + ``` + cmpp_stream(cmpp_input_f_FILE, stdin, cmpp_output_f_FILE, stdout); + ``` + + Or copy a FILE to a string buffer: + + ``` + cmpp_b os = cmpp_b_empty; + FILE * f = cmpp_fopen(...); + rc = cmpp_stream(cmpp_input_f_FILE, f, cmpp_output_f_b, &os); + // On error os might be partially populated. + // Eventually clean up the buffer: + cmpp_b_clear(&os); + ``` +*/ +CMPP_EXPORT int cmpp_stream(cmpp_input_f inF, void * inState, + cmpp_output_f outF, void * outState); + +/** + Reads the entire contents of the given input stream, allocating it + in a buffer. On success, returns 0, assigns *pOut to the buffer, + and *nOut to the number of bytes read (which will be fewer than are + allocated). It guarantees that on success it NUL-terminates the + buffer at one byte after the returned size, with one exception: if + the string has no input, both *pOut and *nOut will be set to 0. + + On error it returns whatever code xIn() returns. +*/ +CMPP_EXPORT int cmpp_slurp(cmpp_input_f xIn, void *stateIn, + unsigned char **pOut, cmpp_size_t * nOut); + +/** + _Almost_ equivalent to fopen(3) but: + + - If name=="-", it returns one of stdin or stdout, depending on the + mode string: stdout is returned if 'w' or '+' appear, otherwise + stdin. + + If it returns NULL, the global errno "should" contain a description + of the problem unless the problem was argument validation. + + If at all possible, use cmpp_fclose() (as opposed to fclose()) to + close these handles, as it has logic to skip closing the three + standard streams. +*/ +CMPP_EXPORT cmpp_FILE * cmpp_fopen(char const * name, char const *mode); + +/** + Passes f to fclose(3) unless f is NULL or one of the C-standard + handles (stdin, stdout, stderr), in which cases it does nothing at + all. +*/ +CMPP_EXPORT void cmpp_fclose(cmpp_FILE * f); + +/** + A cleanup callback interface for use with cmpp_outputer::cleanup(). + Implementations must handle self->state appropriately for its type, + and clear self->state if appropriate, but must not free the self + object. It is implementation-specified whether self->state and/or + self->name are set to NULL by this function. Whether they should be + often depends on how they're used. +*/ +typedef void (*cmpp_outputer_cleanup_f)(cmpp_outputer *self); + +/** + An interface which encapsulates data for managing a streaming + output destination, primarily intended for use with cmpp_stream() + but also used internally by cmpp for directing output to a buffer. +*/ +struct cmpp_outputer { + /** + An optional descriptive name for the channel. The bytes + are owned elsewhere and are typically static or similarly + long-lived. + */ + char const * name; + + /** + Output channel. + */ + cmpp_output_f out; + + /** + flush() implementation. This may be NULL for most uses of this + class. Cases which specifically require it must document that + requirement so. + */ + cmpp_flush_f flush; + + /** + Optional: if not NULL, it must behave appropriately for its state + type, cleaning up any memory it owns. + */ + cmpp_outputer_cleanup_f cleanup; + + /** + State to be used when calling this->out() and this->flush(), + namely: this->out(this->state, ... ) and + this->flush(this->state). + + Whether or not any given instance of this class owns the memory + pointed to by this member must be documented for their cleanup() + method. + + Because cmpp_outputer instances frequently need to be stashed and + unstashed via bitwise copying, it is illegal to replace this + pointer after its initial assignment. The object it points to may + be mutated freely, but this pointer must stay stable for the life + of this object. + */ + void * state; +}; + +/** + Empty-initialized cmpp_outputer instance, intended for + const-copy initialization. +*/ +#define cmpp_outputer_empty_m \ + {.name=NULL, .out = NULL,.flush = NULL, .cleanup = NULL, .state =NULL} + +/** + Empty-initialized cmpp_outputer instance, intended for + non-const-copy initialization. These copies can, for purposes of + cmpp's output API, be used as-is to have cmpp process its inputs + but generate no output. +*/ +CMPP_EXPORT const cmpp_outputer cmpp_outputer_empty; + +/** + If o->out is not NULL, the result of o->out(o->state,p,n) is + returned, else 0 is returned. +*/ +CMPP_EXPORT int cmpp_outputer_out(cmpp_outputer *o, void const *p, cmpp_size_t n); + +/** + If o->flush is not NULL, the result of o->flush(o->state) is + returned, else 0 is returned. +*/ +CMPP_EXPORT int cmpp_outputer_flush(cmpp_outputer *o); + +/** + If o->cleanup is not NULL, it is called, otherwise this is a no-op. +*/ +CMPP_EXPORT void cmpp_outputer_cleanup(cmpp_outputer *o); + +/** + A cmpp_outputer initializer which uses cmpp_flush_f_FILE(), + cmpp_output_f_FILE(), and cmpp_outputer_cleanup_f_FILE() for its + implementation. After copying this, the state member must be + pointed to an opened-for-writing (FILE*). +*/ +CMPP_EXPORT const cmpp_outputer cmpp_outputer_FILE; + +/** + The cmpp_outputer_cleanup_f() impl used by cmpp_outputer_FILE. If + self->state is not NULL then it is passed to fclose() (_unless_ it + is stdin, stdout, or stderr) and set to NULL. self->name is + also set to NULL. +*/ +CMPP_EXPORT void cmpp_outputer_cleanup_f_FILE(cmpp_outputer *self); + +/** + Sets pp's current directive delimiter to a copy of the + NUL-terminated zDelim. The delimiter is the sequence which starts + line and distinguishes cmpp directives from other input, in the + same way that C preprocessors use '#' as a delimiter. + + If zDelim is NULL then the default delimiter is used. The default + delimiter can be set when compiling the library by defining + CMPP_DEFAULT_DELIM to a quoted string value. + + zDelim is assumed to be in UTF-8 encoding. If any bytes in the + range (0,32) are found, CMPP_RC_MISUSE is returned and pp's + persistent error state is set. + + The delimiter must be short and syntactically unambiguous for the + intended inputs. It has a rather arbitrary maximum length of 12, + but it's difficult to envision it being remotely human-friendly + with a delimiter longer than 3 bytes. It's conceivable, but + seemingly far-fetched, that longer delimiters might be interesting + in some machine-generated cases, e.g. using a random sequence as + the delimiter. + + Returns 0 on success. Returns non-0 if called when the delimiter + stack is empty, if it cannot copy the string or zDelim is deemed + unsuitable for use as a delimiter. Calling this when the stack is + empty represents a serious API misuse (indicating that + cmpp_delimiter_pop() was used out of scope) and will trigger an + assert() in debug builds. Except for that last case, errors from + this function are recoverable (see cmpp_err_set()). +*/ +CMPP_EXPORT int cmpp_delimiter_set(cmpp *pp, char const *zDelim); + +/** + Fetches pp's current delimiter string, assigning it to *zDelim. + The string is owned by pp and will be invalidated by any call to + cmpp_delimiter_set() or the #delimiter script directive. + + If, by some odd usage constellation, this is called after an + allocation of the delimiter stack has failed, this will set *zDelim + to the compile-time-default delimiter. That "cannot happen" in normal use because such a failure + would have been reacted to and this would not be called. +*/ +CMPP_EXPORT void cmpp_delimiter_get(cmpp const *pp, char const **zDelim); + +/** + Pushes zDelim as the current directive delimiter. Returns 0 on + success and non-zero on error (invalid zDelim value or allocation + error). If this returns 0 then the caller is obligated to + eventually call cmpp_delimiter_pop() one time. If it returns non-0 + then they must _not_ call that function. + */ +CMPP_EXPORT int cmpp_delimiter_push(cmpp *pp, char const *zDelim); + +/** + Must be called one time for each successful call to + cmpp_delimiter_push(). It restores the directive delimimter to the + value it has when cmpp_delimiter_push() was last called. + + Returns pp's current error code, and will set it to non-0 if called + when no cmpp_delimiter_push() is active. Popping an empty stack + represents a serious API misuse and may fail an assert() in debug + builds. +*/ +CMPP_EXPORT int cmpp_delimiter_pop(cmpp *pp); + +/** + If z[*n] ends on a \n or \r\n pair, it/they are stripped, + *z is NUL-terminated there, and *n is adjusted downwards + by 1 or 2. Returns true if it chomped, else false. +*/ +CMPP_EXPORT bool cmpp_chomp(unsigned char * z, cmpp_size_t * n); + +/** + A basic memory buffer class. This is primarily used with + cmpp_outputer_b to capture arbitrary output for later use. + It's also used for incrementally creating dynamic strings. + + TODO: add the heuristic that an nAlloc of 0 with a non-NULL z + refers to externally-owned memory. This would change the + buffer-write APIs to automatically copy it before making any + changes. We have code for this in the trees this class derives + from, it just needs to be ported over. It would allow us to avoid + allocating in some cases where we need a buffer but it will always + (or commonly) be a copy of a static string, like a single space. +*/ +struct cmpp_b { + /** + This buffer's memory, owned by this object. This library exclusively + uses sqlite3_realloc() and friends for memory management. + + If this pointer is taken away from this object then it must + eventually be passed to cmpp_mfree(). + */ + unsigned char * z; + /** + Number of bytes of this->z which are in use, not counting any + automatic NUL terminator which this class's APIs may add. + */ + cmpp_size_t n; + /** + Number of bytes allocated in this->z. + + Potential TODO: use a value of zero here, with a non-zero + this->n, to mean that this->z is owned elsewhere. This would + cause cmpp_b_append() to copy its original source before + appending. Similarly, cmpp_b_clear() would necessarily _not_ + free this->z. We've used that heuristic in a predecessor of this + class in another tree to good effect for years, but it's not + certain that we'd get the same level of utility out of that + capability as we do in that other project. + */ + cmpp_size_t nAlloc; + + /** + cmpp_b APIs which may fail will set this. Similarly, most + of the cmpp_b APIs become no-ops if this is non-0. + */ + int errCode; +}; + +typedef struct cmpp_b cmpp_b; + +/** + An empty-initialized cmpp_b struct for use in const-copy + initialization. +*/ +#define cmpp_b_empty_m {.z=0,.n=0,.nAlloc=0,.errCode=0} + +/** + An empty-initialized cmpp_b struct for use in non-copy copy + initialization. +*/ +extern const cmpp_b cmpp_b_empty; + +/** + Frees s->z and zeroes out s but does not free s. +*/ +CMPP_EXPORT void cmpp_b_clear(cmpp_b *s); + +/** + If s has content, s->nUsed is set to 0 and s->z is NUL-terminated + at its first byte, else this is a no-op. s->errCode is + set to 0. Returns s. + */ +CMPP_EXPORT cmpp_b * cmpp_b_reuse(cmpp_b *s); + +/** + Swaps all contents of the given buffers, including their persistent + error code. +*/ +CMPP_EXPORT void cmpp_b_swap(cmpp_b * l, cmpp_b * r); + +/** + If s->errCode is 0 and s->nAlloc is less than n, s->z is + reallocated to have at least n bytes, else this is a no-op. Returns + 0 on success, CMPP_RC_OOM on error. +*/ +CMPP_EXPORT int cmpp_b_reserve(cmpp_b *s, cmpp_size_t n); + +/** + Works just like cmpp_b_reserve() but on allocation error it + updates pp's error state. +*/ +CMPP_EXPORT int cmpp_b_reserve3(cmpp * pp, cmpp_b * os, cmpp_size_t n); + +/** + Appends n bytes from src to os, reallocating os as necessary. + Returns 0 on succes, CMPP_RC_OOM on allocation error. + + Errors from this function, and the other cmpp_b_append...() + variants, are recoverable (see cmpp_err_set()). +*/ +CMPP_EXPORT int cmpp_b_append(cmpp_b * os, void const *src, + cmpp_size_t n); + +/** + Works just like cmpp_b_append() but on allocation error it + updates pp's error state. +*/ +CMPP_EXPORT int cmpp_b_append4(cmpp * pp, + cmpp_b * os, + void const * src, + cmpp_size_t n); + +/** + Appends ch to the end of os->z, expanding as necessary, and + NUL-terminates os. Returns os->errCode and is a no-op if that is + non-0 when this is called. This is slightly more efficient than + passing length-1 strings to cmpp_b_append() _if_ os's memory + is pre-allocated with cmpp_b_reserve(), otherwise it may be + less efficient because it may need to allocate frequently if used + repeatedly. +*/ +CMPP_EXPORT int cmpp_b_append_ch(cmpp_b * os, char ch); + +/** + Appends a decimal string representation of d to os. Returns + os->errCode and is a no-op if that is non-0 when this is called. +*/ +CMPP_EXPORT int cmpp_b_append_i32(cmpp_b * os, int32_t d); + +/** int64_t counterpart of cmpp_b_append_i32(). */ +CMPP_EXPORT int cmpp_b_append_i64(cmpp_b * os, int64_t d); + +/** + A thin wrapper around cmpp_chomp() which chomps b->z. +*/ +CMPP_EXPORT bool cmpp_b_chomp(cmpp_b * b); + +/** + A cmpp_output_f() impl which requires that its first argument be a + (cmpp_b*) or be NULL. If buffer is not NULL then it appends n bytes + of src to buffer, reallocating as needed. Returns CMPP_RC_OOM in + reallocation error. On success it always NUL-terminates buffer->z. + A NULL buffer is treated as success but has no side effects. + + Example usage: + + ``` + cmpp_b os = cmpp_b_empty; + int rc = cmpp_stream(cmpp_input_f_FILE, stdin, + cmpp_output_f_b, &os); + ... + cmpp_b_clear(&os); + ``` +*/ +CMPP_EXPORT int cmpp_output_f_b(void * buffer, void const * src, + cmpp_size_t n); + +/** + A cmpp_outputer_cleanup_f() implementation which requires that + self->state be either NULL or a cmpp_b pointer. This function + passes it to cmpp_b_clear(). It does _not_ set self->state or + self->name to NULL. +*/ +CMPP_EXPORT void cmpp_outputer_cleanup_f_b(cmpp_outputer *self); + +/** + A cmpp_outputer prototype which can be copied to use a dynamic + string buffer as an output source. Its state member must be set (by + the client) to a cmpp_b instance. Its out() method is + cmpp_output_f_b(). Its cleanup() method is + cmpp_outputer_cleanup_f_b(). It has no flush() method. +*/ +extern const cmpp_outputer cmpp_outputer_b; + +/** + Returns a string containing version information in an unspecified + format. +*/ +CMPP_EXPORT char const * cmpp_version(void); + +/** + Type IDs for directive lines and argument-parsing tokens. + + This is largely a historical artifact and work is underway + to factor this back out of the public API. +*/ +enum cmpp_tt { + +/** + X-macro which defines token types. It invokes E(X,Y) for each + entry, where X is the base name part of the token type and Y is the + token name as it appears in input scripts (if any, else it's 0). + + Maintenance reminder: their ordering in this map is insignificant + except that None must be first and must have the value 0. + + Some of the more significant ones are: + + - Word: an unquoted word-like token. + + - String: a quoted string. + + - StringAt: an @"..." string. + + - GroupParen, GroupBrace, GroupSquiggly: (), [], and {} + + - All which start with D_ are directives. D_Line is a transitional + state between "unparsed" and another D_... value. +*/ +#define cmpp_tt_map(E) \ + E(None, 0) \ + E(RawLine, 0) \ + E(Unknown, 0) \ + E(Word, 0) \ + E(Noop, 0) \ + E(Int, 0) \ + E(Null, 0) \ + E(String, 0) \ + E(StringAt, 0) \ + E(GroupParen, 0) \ + E(GroupBrace, 0) \ + E(GroupSquiggly,0) \ + E(OpEq, "=") \ + E(OpNeq, "!=") \ + E(OpLt, "<") \ + E(OpLe, "<=") \ + E(OpGt, ">") \ + E(OpGe, ">=") \ + E(ArrowR, "->") \ + E(ArrowL, "<-") \ + E(Plus, "+") \ + E(Minus, "-") \ + E(ShiftR, ">>") \ + E(ShiftL, "<<") \ + E(ShiftL3, "<<<") \ + E(OpNot, "not") \ + E(OpAnd, "and") \ + E(OpOr, "or") \ + E(OpDefined, "defined") \ + E(OpGlob, "glob") \ + E(OpNotGlob, "not glob") \ + E(AnyType, 0) \ + E(Eof, 0) + +#define E(N,TOK) cmpp_TT_ ## N, + cmpp_tt_map(E) +#undef E + /** Used by cmpp_d_register() to assign new IDs. */ + cmpp_TT__last +}; +typedef enum cmpp_tt cmpp_tt; + +/** + For all of the cmpp_tt enum entries, returns a string form of the + enum entry name, e.g. "cmpp_TT_D_If". Returns NULL for any other + values +*/ +CMPP_EXPORT char const * cmpp_tt_cstr(int tt); + +/** + Policies for how to handle undefined @tokens@ when performing + content filtering. +*/ +enum cmpp_atpol_e { + /** Sentinel value. */ + cmpp_atpol_invalid = -1, + /** Turn off @token@ parsing. */ + cmpp_atpol_OFF = 0, + /** Retain undefined @token@ - emit it as-is. */ + cmpp_atpol_RETAIN, + /** Elide undefined @token@. */ + cmpp_atpol_ELIDE, + /** Error for undefined @token@. */ + cmpp_atpol_ERROR, + /** A sentinel value for use with cmpp_dx_out_expand(). */ + cmpp_atpol_CURRENT, + /** + This isn't _really_ the default. It's the default for the + --@policy CLI flag and #@pragma when it's given no value. + */ + cmpp_atpol_DEFAULT_FOR_FLAG = cmpp_atpol_ERROR, + /** + The compile-time default for all cmpp instances. + */ + cmpp_atpol_DEFAULT = cmpp_atpol_OFF + +}; +typedef enum cmpp_atpol_e cmpp_atpol_e; + +/** + Policies describing how cmpp should react to attempts to use + undefined keys. +*/ +enum cmpp_unpol_e { + /* Sentinel. */ + cmpp_unpol_invalid, + /** Treat undefined keys as NULL/falsy. This is the default. */ + cmpp_unpol_NULL, + /** Trigger an error for undefined keys. This should probably be the + default. */ + cmpp_unpol_ERROR, + /** + The compile-time default for all cmpp instances. + */ + cmpp_unpol_DEFAULT = cmpp_unpol_NULL +}; +typedef enum cmpp_unpol_e cmpp_unpol_e; + +typedef struct cmpp_arg cmpp_arg; +/** + A single argument for a directive. When a cmpp_d::flags have + cmpp_d_F_ARGS_V2 set then the part of the input immediately + following the directive (and on the same line) is parsed into a + cmpp_args, a container for these. +*/ +struct cmpp_arg { + /** Token type. */ + cmpp_tt ttype; + /** + The arg's string value, shorn of any opening/closing quotes or () + or {} or []. The args-parsing process guarantees to NUL-terminate + this. The bytes are typically owned by a cmpp_args object, but + clients may direct them wherever the need to, so long as the + bytes are valid longer than this object is. + */ + unsigned char const * z; + /** + The arg's effective length, in bytes, after opening/closing chars + are stripped. That is, its string form is the range [z,z+n). + */ + unsigned n; + /** + The next argument in the list. It is owned by whatever code set + it up (typically cmpp_args_parse()). + */ + cmpp_arg const * next; +}; + +/** + Empty-initialized cmpp_arg instance, intended for + const-copy initialization. +*/ +#define cmpp_arg_empty_m {cmpp_TT_None,0,0,0} + +/** + Empty-initialized cmpp_outputer instance, intended for + non-const-copy initialization. +*/ +extern const cmpp_arg cmpp_arg_empty; + +typedef struct cmpp_dx cmpp_dx; +typedef struct cmpp_dx_pimpl cmpp_dx_pimpl; +typedef struct cmpp_d cmpp_d; + +/** + Flags for use with cmpp_d::flags. +*/ +enum cmpp_d_e { + /** Sentinel value. */ + cmpp_d_F_none = 0, + /** + cmpp_dx_next() will not parse the directive's arguments. Instead, + it makes cmpp_dx::arg0 encapsulate the whole line of the + directive (sans the directive's name) as a single argument. The + only transformation which is performed is the removal of + backslashes from backslash-escaped newlines. It is up to the + directive's callback to handle (or not) the arguments. + */ + cmpp_d_F_ARGS_RAW = 0x01, + + /** + cmpp_dx_next() will parse the directive's arguments. + cmpp_dx::arg0 will point to the first argument in the list, or + NULL if there are no arguments. + + If both cmpp_d_F_ARGS_LIST and cmpp_d_F_ARGS_RAW are specified, + cmpp_d_F_ARGS_LIST will win. + */ + cmpp_d_F_ARGS_LIST = 0x02, + + /** + Indicates that the direction should not be available if the cmpp + instance is configured with any of the cmpp_ctor_F_SAFEMODE flags. + All directives when do any of the following are obligated to + set this flag: + + - Filesystem or network access. + - Invoking external processes. + + Or anything else which might be deamed "security-relevant". + + When registering a directive which has both opener and closer + implementations, it is sufficient to set this only on the opener. + + The library imposes this flag in the following places: + + - Registration of a directive with this flag will fail if + cmpp_is_safemode() is true for that cmpp instance. + + - cmpp_dx_process() will refuse to invoke a directive with this + flag when cmpp_is_safemode() is true. + */ + cmpp_d_F_NOT_IN_SAFEMODE = 0x04, + + /** + Call-only directives are only usable in [directive ...] "call" + contexts. They are not permitted to have a closing directive. + */ + cmpp_d_F_CALL_ONLY = 0x08, + /** + Indicates that the directive is incapable of working in a [call] + context and an error should be trigger if it is. _Most_ + directives which have a closing directive should have this + flag. The exceptions are directives which only conditionally use + a closing directive, like #query. + */ + cmpp_d_F_NO_CALL = 0x10, + + /** + Mask of the client-usable range for this enum. Values outside of + this mask are reserved for internal use and will be stripped from + registrations made with cmpp_d_register(). + */ + cmpp_d_F_MASK = 0x0000ffff + +}; + +/** + Callback type for cmpp_d::impl::callback(). cmpp directives are all + implemented as functions with this signature. Implementations are + called only by cmpp_dx_process() (and only after cmpp_dx_next() has + found a preprocessor line), passed the current context object. + These callbacks are only ever passed directives which were + specifically registered with them (see cmpp_d_register()). + + The first rule of callback is: to report errors (all of which end + processing of the current input) call cmpp_dx_err_set(), passing it + the callback's only argument, then clean up any local resources, + then return. The library will recognize the error and propagate it. + + dx's memory is only valid for the duration of this call. It must + not be held on to longer than that. dx->args.arg0 has slightly different + lifetime: if this callback does _not_ call back in to + cmpp_dx_next() then dx->args.arg0 and its neighbors will survive until + this call is completed. Calling cmpp_dx_next(), or any API which + invokes it, invalidates dx->args.arg0's memory. Thus directives which + call into that must _copy_ any data they need from their own + arguments before doing so, as their arguments list will be + invalidated. +*/ +typedef void (*cmpp_dx_f)(cmpp_dx * dx); + +/** + A typedef for generic deallocation routines. +*/ +typedef void (*cmpp_finalizer_f)(void *); + +/** + State specific to concrete cmpp_d implementations. + + TODO: move this, except for the state pointer, out of cmpp_d + so that directives cannot invoke these callbacks directly. Getting + that to work requires moving the builtin directives into the + dynamic directives list. +*/ +struct cmpp_d_impl { + /** + Callback func. If any API other othan cmpp_dx_process() invokes + this, behavior is undefined. + */ + cmpp_dx_f callback; + + /** + For custom directives with a non-NULL this->state, this will be + called, and passed that object, when the directive is cleaned + up. For directives with both an opening and a closing tag, this + destructor is only attached to the opening tag. + + If any API other othan cmpp's internal cleanup routines invoke + this, behavior is undefined. + */ + cmpp_finalizer_f dtor; + + /** + State for the directive's callback. It is accessible in + cmpp_dx_f() impls via theDx->d->impl.state. For custom + directives with both an opening and closing directive, this + same state object gets assigned to both. + */ + void * state; +}; +typedef struct cmpp_d_impl cmpp_d_impl; +#define cmpp_d_impl_empty_m {0,0,0} + +/** + Each c-pp "directive" is modeled by one of these. +*/ +struct cmpp_d { + + struct { + /** + The directive's name, as it must appear after the directive + delimiter. Its bytes are assumed to be static or otherwise + outlive this object. + */ + const char *z; + /** Byte length of this->z. We record this to speed up searches. */ + unsigned n; + } name; + + /** + Bitmask of flags from cmpp_d_e plus possibly internal flags. + */ + cmpp_flag32_t flags; + + /** + The directive which acts as this directive's closing element + element, or 0 if it has none. + */ + cmpp_d const * closer; + + /** + State specific to concrete implementations. + */ + cmpp_d_impl impl; +}; + +/** + Each instance of the cmpp_dx class (a.k.a. "directive context") + manages a single input source. It's responsible for the + tokenization of all input, locating directives, and processing + ("running") directives. The directive-specific work happens in + cmpp_dx_f() implementations, and this class internally manages the + setup, input traversal, and teardown. + + These objects only exist while cmpp is actively processing + input. Client code interacts with them only through cmpp_dx_f() + implementations which the library invokes. + + The process of filtering input to look for directives is to call + cmpp_dx_next() until it indicates either an error or that a + directive was found. In the latter case, the cmpp_dx object is + populated with info about the current directive. cmpp_dx_process() + will run that directive, but cmpp_dx_f() implementations sometimes + need to make decisions based on the located directive before doing + so (and sometimes they need to skip running it). + + If cmpp_dx_next() finds no directive, the end of the input has been + reached and there is no further output to generate. + + Content encountered before a directive is found is passed on to the + output stream via cmpp_dx_out_raw() or cmpp_dx_out_expand(). +*/ +struct cmpp_dx { + /** + The cmpp object which owns this context. + */ + cmpp * const pp; + + /** + The directive on whose behalf this context is active. + */ + cmpp_d const *d; + + /** + Name of the input for error reporting. Typically an input script + file name, but it need not refer to a file. + */ + unsigned const char * const sourceName; + + /** + State related to arguments passed to the current directive. + + It is important to keep in mind that the memory for the members + of this sub-struct may be modified or reallocated + (i.e. invalidated) by any APIs which call in to cmpp_dx_next(). + cmpp_dx_f() implementations must take care not to use any of this + memory after calling into that function, cmpp_dx_consume(), or + similar. If needed, it must be copied (e.g. using + cmpp_args_clone() to create a local copy of the parsed + arguments). + */ + struct { + /** + Starting byte of unparsed arguments. This is for cmpp_d_f() + implementations which need custom argument parsing. + */ + unsigned const char * z; + + /** + The byte length of z. + */ + cmpp_size_t nz; + + /** + The parsed arg count for the this->arg0 list. + */ + unsigned argc; + + /** + The first parsed arg or NULL. How this is set up is affected by + cmpp_d::flags. + + This is specifically _NOT_ defined as a sequential array and + using pointer math to traverse it invokes undefined behavior. + + To traverse the list: + + for( cmpp_arg const *a = dx->args.arg0; a; a=a->next ){ + ... + } + */ + cmpp_arg const * arg0; + } args; + + /** + Private impl details. + */ + cmpp_dx_pimpl * const pimpl; +}; + +/** + Thin proxy for cmpp_err_set(), replacing only the first argument. +*/ +CMPP_EXPORT int cmpp_dx_err_set(cmpp_dx *dx, int rc, + char const *zFmt, ...); + + +/** + Returns true if dx's current call into the API is the result + of a function call, else false. Any APIs which recurse into + input processing will reset this to false, so it needs to be + evaluated before doing any such work. + + Design note: this flag is actually tied to dx's arguments, which + get reset by APIs which consume from the input stream. +*/ +CMPP_EXPORT bool cmpp_dx_is_call(cmpp_dx * const dx); + +/** + Returns true if dx->pp has error state, else false. If this + function returns true, cmpp_dx_f() implementations are required to + stop working, clean up any local resources, and return. Continuing + to use dx when it's in an error state may exacerbate the problem. +*/ +#define cmpp_dx_err_check(DX) (DX)->pp->api->err_has((DX)->pp) + +/** + Scans dx to the next directive line, emitting all input before that + which is _not_ a directive line to dx->pp's output channel unless + it's elided due to being inside a block which elides its content + (e.g. #if). + + Returns 0 if no errors were triggered, else a cmpp_rc_e code. This + is a no-op if dx->pp has persistent error state set, and that error + code is returned. + + If it returns 0 then it sets *pGotOne to true if a directive was + found and false if not (in which case the end of the input has + been reached and further calls to this function for the same input + source will be no-ops). If it sets *pGotOne to true then it also + sets up dx's state for use with cmpp_dx_process(), which should + (normally) then be called. + + ACHTUNG: calling this resets any argument-handling-related state of + dx. That is important for cmpp_dx_f() implemenations, which _must + not_ hold copies of any pointers from dx->args.arg0 or dx->args.z + beyond a call to this function. Any state they need must be + evaluated, potentially copied, before calling this function(). +*/ +CMPP_EXPORT int cmpp_dx_next(cmpp_dx * dx, bool * pGotOne); + +/** + This is only legal to call immediately after a successful call to + cmpp_dx_next(). It requires that cmpp_dx_next() has just located + the next directive. This function runs that directive. Returns 0 + on success and all that. + + Design note: directive-Search and directive-process are two + distinctly separate steps because directives which have both + open/closing tags frequently discard the closing directive without + running it (it exists to tell the directive how far to read). Those + closing directives exist independently, though, and will trigger + errors when encountered outside of the context of their opening + directive tag (e.g. an "#/if" without an "#if"). +*/ +CMPP_EXPORT int cmpp_dx_process(cmpp_dx * dx); + +/** + A bitmask of flags for use with cmpp_dx_consume() +*/ +enum cmpp_dx_consume_e { + /** + Tells cmpp_dx_consume() to process any directives it encounters + which are not in the specified set of closing directives. Its + default is to fail if another directive is seen. + */ + cmpp_dx_consume_F_PROCESS_OTHER_D = 0x01, + /** + Tells cmpp_dx_consume() that non-directive content encountered + before the designated closing directive(s) must use an at-policy + of cmpp_atpol_OFF. That is: the output target of that function will + get the raw, unfiltered content. This is for cases where the + consumer will later re-emit that content, delaying @token@ + parsing until a later step (e.g. #query does this). + + This may misinteract in unpredictable ways when used with + cmpp_dx_consume_F_PROCESS_OTHER_D. Please report them as bugs. + */ + cmpp_dx_consume_F_RAW = 0x02 +}; + +/** + A helper for cmpp_dx_f() implementations which read in their + blocked-off content instead of passing it through the normal output + channel. e.g. `#define x <<` stores that content in a define named + "x". + + This function runs a cmpp_dx_next() loop which does the following: + + If the given output channel is not NULL then it first replaces the + output channel with the given one, such that all output which would + normally be produced will be sent there until this function + returns, at which point the output channel is restored. If the + given channel is NULL then output is not captured - it instead goes + dx's current output channel. + + dClosers must be a list of legal closing tags nClosers entries + long. Typically this is the single closing directive/tag of the + current directive, available to the opening directive's cmpp_dx_f() + impl via dx->d->closer. Some special cases require multiple + candidates, however. + + The flags argument may be 0 or a bitmask of values from the + cmpp_dx_consume_e enum. + + If flags does not have the cmpp_dx_consume_F_PROCESS_OTHER_D bit set + then this function requires that the next directive in the input be + one specified by dClosers. If the next directive is not one of + those, it will fail with code CMPP_RC_SYNTAX. + + If flags has the cmpp_dx_consume_F_PROCESS_OTHER_D bit set then it + will continue to search for and process directives until the + dCloser directive is found. Calling into other directives will + invalidate certain state that a cmpp_dx_f() has access to - see + below for details. If dCloser is not found before EOF, a + CMPP_RC_SYNTAX error is triggered. + + Once one of dCloser is found, this function returns with dx->d + referring to the that directive. In practice, the caller should + _not_ call cmpp_dx_process() at that point - the closing directive + is typically a no-op placeholder which exists only to mark the end + of the block. If the closer has work to do, however, the caller of + this function should call cmpp_dx_process() at that point. + + On success it returns 0, the input stream will have been consumed + between the directive dx and its closing tag, and dx->d will point + to the new directive. If os is not NULL then os will have been + sent any content. + + On error, processing of the directive must end immediately, + returning from the cmpp_dx_f() impl after cleaning up any local + resources. + + ACHTUNG: since this invokes cmpp_dx_next(), it invalidates + dx->args.arg0. Its dx->d is also replaced but the previous value + remains valid until the cmpp instance is cleaned up. + + Example from the context of a cmpp_dx_f() implementation + + ``` + // "dx" is the cmpp_dx arg to this function + cmpp_outputer oss = cmpp_outputer_b; + cmpp_b os = cmpp_b_empty; + oss.state = &os; + if( 0==cmpp_dx_consume(dx, &oss, dx->d->closer, 0) ){ + cmpp_b_chomp( &os ); + ... maybe modify the buffer or decorate the output in some way... + cmpp_dx_out_raw(dx, os.z, os.n); + } + cmpp_b_clear(&os); + ``` + + Design issue: this API does not currently have a way to handle + directives which have multiple potential waypoints/endpoints, in + the way that an #if may optionally have an #elif or #else before + the #/if. Such processing has to be done in the directive's + impl. +*/ +CMPP_EXPORT int cmpp_dx_consume(cmpp_dx * dx, cmpp_outputer * os, + cmpp_d const *const * dClosers, + unsigned nClosers, + cmpp_flag32_t flags); + +/** + Equivalent to cmpp_dx_consume(), capturing to the given buffer + instead of a cmpp_outputer object. +*/ +CMPP_EXPORT int cmpp_dx_consume_b(cmpp_dx * dx, cmpp_b * b, + cmpp_d const * const * dClosers, + unsigned nClosers, + cmpp_flag32_t flags); + +/** + If arg is not NULL, cleans up any resources owned by + arg but does not free arg. + + As of this writing, they own none and some code still requires + that. That is Olde Thynking, though. +*/ +CMPP_EXPORT void cmpp_arg_cleanup(cmpp_arg *arg); + +/** + If arg is not NULL resets arg to be re-used. arg must have + initially been cleanly initialized by copying cmpp_arg_empty (or + equivalent, i.e. zeroing it out). +*/ +CMPP_EXPORT void cmpp_arg_reuse(cmpp_arg *arg); + +/** + This is the core argument-parsing function used by the library's + provided directives. Its is available in the public API as a + convenience for custom cmpp_dx_f() implementations, but custom + implementations are not required to make use of it. + + Populates a cmpp_arg object by parsing the next token from its + input source. + + Expects *pzIn to point to the start of input for parsing arguments + and zInEnd to be the logical EOF of that range. This function + populates pOut with the info of the parse. Returns 0 on success, + non-0 (and updates pp's error state) on error. + + Output (the parsed token) is written to *pzOut. zOutEnd must be the + logical EOF of *pzOut. *pzOut needs to be, at most, + (zInEnd-*pzIn)+1 bytes long. This function range checks the output + and will not write to or past zOutEnd, but that will trigger a + CMPP_RC_RANGE error. + + On success, *pzIn will be set to 1 byte after the last one parsed + for pOut and *pzOut will be set to one byte after the final output + (NUL-terminated). pOut->z will point to the start of *pzOut and + pOut->n will be set to the byte-length of pOut->z. + + When the end of the input is reached, this function returns 0 + and sets pOut->ttype to cmpp_TT_EOF. + + General tokenization rules: + + Tokens come in the following flavors: + + - Quoted strings: single- or double-quoted. cmpp_arg::ttype: + cmpp_TT_String. + + - "At-strings": @"..." and @'...'. cmpp_arg::ttype value: + cmpp_TT_StringAt. + + - Decimal integers with an optional sign. cmpp_arg::ttype value: + cmpp_TT_Int. + + - Groups: (...), {...}, and [...]. cmpp_arg::ttype values: + cmpp_TT_GroupParen, cmpp_TT_GroupSquiggly, and + cmpp_TT_GroupBrace. These types do not automatically get parsed + recursively. To recurse into one of these, pass cmpp_arg_parse() + the grouping argument's bytes as the input range. + + - Word: anything which doesn't look like one of these above. Token + type IDs: cmpp_TT_Word. These are most often interpreted as + #define keys but cmpp_dx_f() implementations sometimes treat + them as literal values. + + - A small subset of words and operator-like tokens, e.g. '=' and + '!=', get a very specific ttype, e.g. cmpp_TT_OpNeq, but these + can generally be treated as strings. + + - Outside of strings and groups, spaces, tabs, carriage-returns, + and newlines are skipped. + + These are explained in more detail in the user's manual + (a.k.a. README.md). + + There are many other token types, mostly used internally. + + This function supports _no_ backslash escape sequences in + tokens. All backslashes, with the obligatory exception of those + which make up backslash-escaped newlines in the input stream, are + retained as-is in all token types. That means, for example, that + strings may not contain their own quote character. + + As an example of where this function is useful: cmpp_dx_f() + implementations which need to selectively parse a subset of the + directive's arguments can use this. As input, their dx argument's + args.z and args.nz members delineate the current directive line's + arguments. See c-pp.c:cmpp_dx_f_pipe() for an example. +*/ +CMPP_EXPORT int cmpp_arg_parse(cmpp_dx * dx, + cmpp_arg *pOut, + unsigned char const **pzIn, + unsigned char const *zInEnd, + unsigned char ** pzOut, + unsigned char const * zOutEnd); + +/** + True if (cmpp_arg const *)ARG's contents match the string literal + STR, else false. +*/ +#define cmpp_arg_equals(ARG,STR) \ + (sizeof(STR)-1==(ARG)->n && 0==memcmp(STR,(ARG)->z,sizeof(STR)-1)) + +/** + True if (cmpp_arg const *)ARG's contents match the string literal + STR or ("-" STR), else false. The intent is that "-flag" be passed + here to tolerantly accept either "-flag" or "--flag". +*/ +#define cmpp_arg_isflag(ARG,STR) \ + cmpp_arg_equals(ARG,STR) || cmpp_arg_equals(ARG, "-" STR) + +/** + Creates a copy of arg->z. If allocation fails then pp's persistent + error code is set to CMPP_RC_OOM. If pp's error code is not 0 when + this is called then this is a no-op and returns NULL. In other + words, if this function returns NULL, pp's error state was either + already set when this was called or it was set because allocation + failed. + + Ownership of the returned memory is transferred to the caller, who + must eventually free it using cmpp_mfree(). +*/ +CMPP_EXPORT char * cmpp_arg_strdup(cmpp *pp, cmpp_arg const *arg); + +/** + Flag bitmasks for use with cmpp_arg_to_b(). With my apologies + for the long names (but consistency calls for them). +*/ +enum cmpp_arg_to_b_e { + /** + Specifies that the argument's string value should be used as-is, + rather than expanding it (if the arg's ttype would normally cause + it to be expanded). + */ + cmpp_arg_to_b_F_FORCE_STRING = 0x01, + + /** + Tells cmpp_arg_to_b() to not expand arguments with type + cmpp_TT_Word, which it normally treats as define keys. It instead + treats these as strings. + */ + cmpp_arg_to_b_F_NO_DEFINES = 0x02, + + /** + If set, arguments with a ttype of cmpp_TT_GroupBrace will be + "called" by passing them to cmpp_call_arg(). The space-trimmed + result of the call becomes the output of the cmpp_arg_to_b() + call. + + FIXME: make this opt-out instead of opt-in. We end up _almost_ + always wanting this. + */ + cmpp_arg_to_b_F_BRACE_CALL = 0x04, + + /** + Explicitly disable [call] expansion even if + cmpp_arg_to_b_F_BRACE_CALL is set in the flags. + */ + cmpp_arg_to_b_F_NO_BRACE_CALL = 0x08 + + /** + TODO? cmpp_arg_to_b_F_UNESCAPE + */ +}; + +/** + Appends some form of arg to the given buffer. + + arg->ttype values of cmpp_TT_Word (define keys) and + cmpp_TT_StringAt cause the value to be expanded appropriately (the + latter according to dx->pp's current at-policy). Others get emitted + as-is. + + The flags argument influences the expansion decisions, as documented + in the cmpp_arg_to_b_e enum. + + Returns 0 on success and all that. + + See: cmpp_atpol_get(), cmpp_atpol_set() + + Reminder to self: though this function may, via script-side + function call resolution, recurse into the library, any such + recursion gets its own cmpp_dx instance. In this context that's + significant because it means this call won't invalidate arg's + memory like cmpp_dx_consume() or cmpp_dx_next() can (depending on + where args came from - typically it's owned by dx but + cmpp_args_clone() exists solely to work around such potential + invalidation). +*/ +CMPP_EXPORT int cmpp_arg_to_b(cmpp_dx * dx, cmpp_arg const *arg, + cmpp_b * os, cmpp_flag32_t flags); + +/** + Flags for use with cmpp_call_str() and friends. +*/ +enum cmpp_call_e { + /** Do not trim a newline from the result. */ + cmpp_call_F_NO_TRIM = 0x01, + /** Trim all leading and trailing space and newlines + from the result. */ + cmpp_call_F_TRIM_ALL = 0x02 +}; + +/** + This assumes that arg->z holds a "callable" directive + string in the form: + + directiveName ...args + + This function composes a new cmpp input source from that line + (prefixed with dx's current directive prefix if it's not already + got one), processes it with cmpp_process_string(), redirecting the + output to dest (which gets appended to, so be sure to + cmpp_b_reuse() it if needed before calling this). + + To simplify common expected usage, by default the output is trimmed + of a single newline. The flags argument, 0 or a bitmask of values + from the cmpp_call_e enum, can be used to modify that behavior. + + This is the basis of "function calls" in cmpp. + + Returns 0 on success. +*/ +int cmpp_call_str(cmpp *dx, + unsigned char const * z, + cmpp_ssize_t n, + cmpp_b * dest, + cmpp_flag32_t flags); + +/** + Convert an errno value to a cmpp_rc_e approximation, defaulting to + dflt if no known match is found. This is intended for use by + cmpp_dx_f implementations which use errno-using APIs. +*/ +CMPP_EXPORT int cmpp_errno_rc(int errNo, int dflt); + +/** + Configuration object for use with cmpp_d_register(). +*/ +struct cmpp_d_reg { + /** + The name of the directive as it will be used in + input scripts, e.g. "mydirective". It will be copied by + cmpp_d_register(). + */ + char const *name; + /** + A combination of bits from the cmpp_d_e enum. + + These flags are currently applied only to this->opener. + this->closer, because of how it's typically used, assumes + */ + struct { + /** + Callback for the directive's opening tag. + */ + cmpp_dx_f f; + /** + Flags from cmpp_d_e. Typically one of cmpp_d_F_ARGS_LIST or + cmpp_d_F_ARGS_RAW. + */ + cmpp_flag32_t flags; + } opener; + struct { + /** + Callback for the directive's closing tag, if any. + + This is only relevant for directives which have both an open and + a closing tag (even if that closing tag is only needed in some + contexts, e.g. "#define X <<" (with a closer) vs "#define X Y" + (without)). See cmpp_dx_f_dangling_closer() for a default + implementation which triggers an error if it's seen in the input + and not consumed by its counterpart opening directive. That + implementation has proven useful for #define, #pipe, and friends. + + Design notes: it's as yet unclear how to model, in the public + interface, directives which have a whole family of cooperating + directives, namely #if/#elif/#else. + */ + cmpp_dx_f f; + /** + Flags from cmpp_d_e. For closers this can typically be + left at 0. + */ + cmpp_flag32_t flags; + } closer; + /** + If not NULL then it is assigned to the directive's opener part + and will be called by the library in either of the following + cases: + + - When the custom directive is cleaned up. + + - If cmpp_d_register() fails (returns non-0), regardless of how + it fails. + + It is passed this->state. + */ + cmpp_finalizer_f dtor; + /** + Implementation state for the callbacks. + */ + void * state; +}; +typedef struct cmpp_d_reg cmpp_d_reg; +/** + Empty-initialized cmpp_d_reg instance, intended for const-copy + initialization. +*/ +#define cmpp_d_reg_empty_m {0,{0,0},{0,0},0,0} +/** + Empty-initialized instance, intended for non-const-copy + initialization. +*/ +//extern const cmpp_d_reg cmpp_d_reg_empty; + +/** + Registers a new directive, or a pair of opening/closing directives, + with pp. + + The semantics of r's members are documented in the cmpp_d_reg + class. r->name and r->opener.f are required. The remainder may be + 0/NULL. Its members are copied - r need not live longer than this + call. + + When the new directive is seen in a script, r->opener.f() will be + called. If the closing directive (if any) is seen in a script, + r->closer.f() is called. In both cases, the callback + implementation can get access to the r->state object via + cmdd_dx::d::impl::state (a.k.a dx->d->impl.state). + + If r->closer.f is not NULL then the closing directive will be named + "/${zName}". (Design note: it is thought that forcing a common + end-directive syntax will lead to fewer issues than allowing + free-form closing tag names, e.g. fewer chances of a name collision + or not quite remembering the spelling of a given closing tag + (#endef vs #enddefine vs #/define).) + + Returns 0 on success and updates pp's error state on error. Similarly, + this is a no-op if pp has an error code when this is called, in which + case it returns that result code without other side-effects. + + On success, if pOut is not NULL then it is set to the directive + pointer, memory owned by pp until it cleans up its directives. This + is the only place in the API a non-const pointer to a directive can + be found, and it is provided only for very specific use-cases where + a directive needs to be manipulated (carefully) after + registration[^post-reg-manipulation]. If this function also + registered a closing directive, it is available as (*pOut)->closer. + pOut should normally be NULL. + + Failure modes include: + + - Returns CMPP_RC_RANGE if zName is not legal for use as a + directive name. See cmpp_is_legal_key(). + + - Returns CMPP_RC_OOM on an allocation error. + + Errors from this function are recoverable (see cmpp_err_set()). A + failed registration, even one which doesn't fail until the + registration of the closing element, will leave pp in a + well-defined state (with neither of r's directives being + registered). + + [^post-reg-manipulation]: The one known use case if the #if family + of directives, all of which use the same #/if closing + directive. The public registration API does not account for sharing + of closers that way, and whether it _should_ is still TBD. The + workaround, for this case, is to get the directives as they're + registered and point the cmpp_d::closer of each of #if, #elif, and + #else to #/if. +*/ +CMPP_EXPORT int cmpp_d_register(cmpp * pp, cmpp_d_reg const * r, + cmpp_d **pOut); + +/** + A cmpp_dx_f() impl which is intended to be used as a callback for + directive closing tags for directives in which the opening tag's + implementation consumes the input up to the closing tag. This impl + triggers an error if called, indicating that the directive closing + was seen in the input without its accompanying directive opening. +*/ +CMPP_EXPORT void cmpp_dx_f_dangling_closer(cmpp_dx *dx); + +/** + Writes the first n bytes of z to dx->pp's current output channel + without performing any @token@ parsing. + + Returns dx->pp's persistent error code (0 on success) and sets that + code to non-0 on error. This is a no-op if dx->pp has a non-0 error + state, returning that code. + + See: cmpp_dx_out_expand() +*/ +CMPP_EXPORT int cmpp_dx_out_raw(cmpp_dx * dx, void const *z, + cmpp_size_t n); + +/** + Sends [zFrom,zFrom+n) to pOut, performing @token@ expansion if the + given policy says to (else it passes the content through as-is, as + per cmpp_dx_out_raw()). A policy of cmpp_atpol_CURRENT uses dx->pp's + current policy. A policy of cmpp_atpol_OFF behaves exactly like + cmpp_dx_out_raw(). + + Returns dx->pp's persistent error code (0 on success) and sets that + code to non-0 on error. This is a no-op if dx->pp has a non-0 error + state, returning that code. + + If pOut is NULL then dx->pp's default channel is used, with the + caveat that atPolicy's only legal value in that case is + cmpp_atpol_CURRENT. (The internals do not allow the at-policy to be + overridden for that particular output channel, to avoid accidental + filtering when it's not enabled. They do not impose that + restriction for other output channels, which are frequently used + for filtering intermediary results.) + + See: cmpp_dx_out_raw() + + Notes regarding how this is used internally: + + - This function currently specifically does nothing when invoked in + skip-mode[^1]. Hypothetically it cannot ever be called in skip-mode + except when evaluating #elif expressions (previous #if/#elifs + having failed and put us in skip-mode), where it's expanding + expression operands. That part currently (as of 2025-10-21) uses + dx->pp's current policy, and it's not clear whether that is + sufficient or whether we need to force it to expand (and which + policy to use when doing so). We could possibly get away with + always using cmpp_atpol_ERROR for purposes of evaluating at-string + expression operands. + + [^1]: Skip-mode is the internal mechanism which keeps directives + from running, and content from being emitted, within a falsy branch + of an #if/#elif block. Only flow-control directives are ever run + when skip-mode is active, and client-provided directives cannot + easily provide flow-control support. Ergo, much of this paragraph + is not relevant for client-level code, but it is for this library's + own use of this function. +*/ +CMPP_EXPORT int cmpp_dx_out_expand(cmpp_dx const * dx, + cmpp_outputer * pOut, + unsigned char const * zFrom, + cmpp_size_t n, + cmpp_atpol_e policy); + +/** + This creates a formatted string using sqlite3_mprintf() and emits it + using cmpp_dx_out_raw(). Returns CMPP_RC_OOM if allocation of the + string fails, else it returns whatever cmpp_dx_out_raw() returns. + + This is a no-op if dx->pp is in an error state, returning + that code. +*/ +CMPP_EXPORT int cmpp_dx_outf(cmpp_dx *dx, char const *zFmt, ...); + +/** + Convenience form of cmpp_delimiter_get() which returns the + delimiter which was active at the time when the currently-running + cmpp_dx_f() was called. This memory may be invalidated by any calls + into cmpp_dx_process() or cmpp_delimiter_set(), so a copy of this + pointer must not be retained past such a point. + + This function is primarily intended for use in generating debug and + error messages. + + If the delimiter stack is empty, this function returns NULL. +*/ +CMPP_EXPORT char const * cmpp_dx_delim(cmpp_dx const *dx); + +/** + Borrows a buffer from pp's buffer recycling pool, allocating one if + needed. It returns NULL only on allocation error, in which case it + updates pp's error state. + + This transfers ownership of the buffer to the caller, who is + obligated to eventually do ONE of the following: + + - Pass it to cmpp_b_return() with the same dx argument. + + - Pass it to cmpp_b_clear() then cmpp_mfree(). + + The purpose of this function is a memory reuse optimization. Most + directives, and many internals, need to use buffers for something + or other and this gives them a way to reuse buffers. + + Potential TODO: How this pool optimizes (or not) buffer allotment + is an internal detail. Maybe add an argument which provides a hint + about the buffer usage. e.g. argument-conversion buffers are + normally small but block content buffers can be arbitrarily large. +*/ +CMPP_EXPORT cmpp_b * cmpp_b_borrow(cmpp *dx); + +/** + Returns a buffer borrowed from cmpp_b_borrow(), transferring + ownership back to pp. Passing a non-NULL b which was not returned + by cmpp_b_borrow() invoked undefined behavior (possibly delayed + until the list is cleaned up). To simplify usage, b may be NULL. + + After calling this, b must be considered "freed" - it must not be + used again. This function is free (as it were) to immediately free + the object's memory instead of recycling it. +*/ +CMPP_EXPORT void cmpp_b_return(cmpp *dx, cmpp_b *b); + +/** + If NUL-terminated z matches one of the strings listed below, its + corresponding cmpp_atpol_e entry is returned, else + cmpp_atpol_invalid is returned. + + If pp is not NULL then (A) this also sets its current at-policy and + (B) it recognizes an additional string (see below). In this case, + if z is not a valid string then pp's persistent error state is set. + + Its accepted values each correspond to a like-named policy value: + + - "off" (the default): no processing of `@` is performed. + + - "error": fail if an undefined `X` is referenced in @token@ + parsing. + + - "retain": emit any unresolved `@X@` tokens as-is to the output + stream. i.e. `@X@` renders as `@X@`. + + - "elide": omit unresolved `@X@` from the output, as if their values + were empty. i.e. `@X@` renders as an empty string, i.e. is not + emitted at all. + + - "current": if pp!=NULL then it returns the current policy, else + this string resolves to cmpp_atpol_invalid. +*/ +CMPP_EXPORT cmpp_atpol_e cmpp_atpol_from_str(cmpp * pp, char const *z); + +/** + Returns pp's current at-token policy. +*/ +CMPP_EXPORT cmpp_atpol_e cmpp_atpol_get(cmpp const * const pp); + +/** + Sets pp's current at-token policy. Returns 0 if pol is valid, else + it updates pp's error state and returns CMPP_RC_RANGE. This is a + no-op if pp has error state, returning that code instead. + + The policy cmpp_atpol_CURRENT is a no-op, permitted to simplify + certain client-side usage. +*/ +CMPP_EXPORT int cmpp_atpol_set(cmpp * const pp, cmpp_atpol_e pol); + +/** + Pushes pol as the current at-policy. Returns 0 on success and + non-zero on error (bad pol value or allocation error). If this + returns 0 then the caller is obligated to eventually call + cmpp_atpol_pop() one time. If it returns non-0 then they _must not_ + call that function. +*/ +CMPP_EXPORT int cmpp_atpol_push(cmpp *pp, cmpp_atpol_e pol); + +/** + Must be called one time for each successful call to + cmpp_atpol_push(). It restores the at-policy to the value it + has when cmpp_atpol_push() was last called. + + If called when no cmpp_delimiter_push() is active then debug builds + will fail an assert(), else pp's error state is updated if it has + none already. +*/ +CMPP_EXPORT void cmpp_atpol_pop(cmpp *pp); + +/** + The cmpp_unpol_e counterpart of cmpp_atpol_from_str(). It + behaves identically, just for a different policy group with + different names. + + Its accepted values are: "null" and "error". The value "current" is + only legal if pp!=NULL, else it resolves to cmpp_unpol_invalid. +*/ +CMPP_EXPORT cmpp_unpol_e cmpp_unpol_from_str(cmpp * pp, char const *z); + +/** + Returns pp's current policy regarding use of undefined define keys. +*/ +CMPP_EXPORT cmpp_unpol_e cmpp_unpol_get(cmpp const * const pp); + +/** + Sets pp's current policy regarding use of undefined define keys. + Returns 0 if pol is valid, else it updates pp's error state and + returns CMPP_RC_RANGE. +*/ +CMPP_EXPORT int cmpp_unpol_set(cmpp * const pp, cmpp_unpol_e pol); + +/** + The undefined-policy counterpart of cmpp_atpol_push(). +*/ +CMPP_EXPORT int cmpp_unpol_push(cmpp *pp, cmpp_unpol_e pol); + +/** + The undefined-policy counterpart of cmpp_atpol_pop(). +*/ +CMPP_EXPORT void cmpp_unpol_pop(cmpp *pp); + +/** + The at-token counterpart of cmpp_delimiter_get(). This sets *zOpen + (if zOpen is not NULL) to the opening delimiter and *zClose (if + zClose is not NULL) to the closing delimiter. The memory is owned + by pp and may be invalidated by any calls to cmpp_atdelim_set(), + cmpp_atdelim_push(), or any APIs which consume input. Each string + is NUL-terminated and must be copied by the caller if they need + these strings past a point where they might be invalidated. + + If called when the the delimiter stack is empty, debug builds with + fail an assert() and non-debug builds will behave as if the stack + contains the compile-time default delimiters. +*/ +CMPP_EXPORT void cmpp_atdelim_get(cmpp const * pp, + char const **zOpen, + char const **zClose); +/** + The `@token@`-delimiter counterpart of cmpp_delimeter_set(). + + This sets the delimiter for `@token@` content to the given opening + and closing strings (which the library makes a copy of). If zOpen + is NULL then the compile-time default is assumed. If zClose is NULL + then zOpen is assumed. + + Returns 0 on success. Returns non-0 if called when the delimiter + stack is empty, if it cannot copy the string or zDelim is deemed + unsuitable for use as a delimiter. + + In debug builds this will trigger an assert if no `@token@` + delimiter has been set, but pp starts with one level in place, so + it is safe to call without having made an explicit + cmpp_atdelim_push() unless cmpp_atdelim_pop() has been misused. +*/ +CMPP_EXPORT int cmpp_atdelim_set(cmpp * pp, + char const *zOpen, + char const *zClose); + +/** + The `@token@`-delimiter counterpart of cmpp_delimeter_push(). + + See cmpp_atdelim_set() for the semantics of the arguments. +*/ +CMPP_EXPORT int cmpp_atdelim_push(cmpp *pp, + char const *zOpen, + char const *zClose); + +/** + The @token@-delimiter counterpart of cmpp_delimiter_pop(). +*/ +CMPP_EXPORT int cmpp_atdelim_pop(cmpp *pp); + +/** + Searches the given path (zPath), split on the given path separator + (pathSep), for the given file (zBaseName), optionally with the + given file extension (zExt). + + If zBaseName or zBaseName+zExt are found as-is, without any + search path prefix, that will be the result, else the result + is either zBaseName or zBaseName+zExt prefixed by one of the + search directories. + + On success, returns a new string, transfering ownership to the + caller (who must eventually pass it to cmpp_mfree() to deallocate). + + If no match is found, or on error, returns NULL. On a genuine + error, pp's error state is updated and the error is unlikely to be + recoverable (see cmpp_err_set()). + + This function is a no-op if called when pp's error state is set, + returning NULL. + + Results are undefined (in the sense of whether it will work or not, + as opposed to whether it will crash or not) if pathSep is a control + character. + + Design note: this is implemented as a Common Table Expression + query. +*/ +CMPP_EXPORT char * cmpp_path_search(cmpp *pp, + char const *zPath, + char pathSep, + char const *zBaseName, + char const *zExt); + +/** + Scans [*zPos,zEnd) for the next chSep character. Sets *zPos to one + after the last consumed byte, so its result includes the separator + character unless EOF is hit before then. If pCounter is not NULL + then it does ++*pCounter when finding chSep. + + Returns true if any input is consumed, else false (EOF). When it + returns false, *zPos will have the same value it had when this was + called. If it returns true, *zPos will be greater than it was + before this call and <= zEnd. + + Usage: + + ``` + unsigned char const * zBegin = ...; + unsigned char const * const zEnd = zBegin + strlen(zBegin); + unsigned char const * zEol = zBegin; + cmpp_size_t nLn = 0; + while( cmpp_next_chunk(&zEol, zEnd, '\n', &nLn) ){ + ... + } + ``` +*/ +CMPP_EXPORT +bool cmpp_next_chunk(unsigned char const **zPos, + unsigned char const *zEnd, + unsigned char chSep, + cmpp_size_t *pCounter); + +/** + Flags and constants related to the cmpp_args type. +*/ +enum cmpp_args_e { + /** + cmpp_args_parse() flag which tells cmpp_args_parse() not to + dive into (...) group tokens. It insteads leaves them to be parsed + (or not) by downstream code. The only reason to parse them in + advance is to catch syntax errors sooner rather than later. + */ + cmpp_args_F_NO_PARENS = 0x01 +}; + +/** + An internal detail of cmpp_args. +*/ +typedef struct cmpp_args_pimpl cmpp_args_pimpl; + +/** + A container for parsing a line's worth of cmpp_arg + objects. + + Instances MUST be cleanly initialized by bitwise-copying either + cmpp_args_empty or (depending on the context) cmpp_args_empty_m. + + Instances MUST eventually be passed to cmpp_args_cleanup(). + + Design notes: this class is provided to the public API as a + convenience, not as a core/required component. It offers one of + many possible solutions for dealing with argument lists and is not + the End All/Be All of solutions. I didn't _really_ want to expose + this class in the public API at all but I also want client-side + directives to have the _option_ to to do some of the things + currently builtin directives can do which are (as of this writing) + unavailable in the public API, e.g. evaluate expressions (in that + limited form which this library supports). A stepping stone to + doing so is making this class public. +*/ +struct cmpp_args { + /** + Number of parsed args. In the context of a cmpp_dx_f(), argument + lists do not include their directive's name as an argument. + */ + unsigned argc; + + /** + The list of args. This is very specifically NOT an array (or at + least not one which client code can rely on to behave + sensibly). Some internal APIs adjust a cmpp_args's arg list, + re-linking the entries via cmpp_arg::next and making array-style + traversal a foot-gun. + + To loop over them: + + for( cmpp_arg const * arg = args->arg0; arg; arg = arg->next ){...} + + This really ought to be const but it currenty cannot be for + internal reasons. Client code really should not modify these + objects, though. Doing so invokes undefined behavior. + + For directives with the cmpp_d_F_ARGS_RAW flag, this member will, + after a successful call to cmpp_dx_next(), point to a single + argument which holds the directive's entire argument string, + stripped of leading spaces. + */ + cmpp_arg * arg0; + + /** + Internal implementation details. This is initialized via + cmpp_args_parse() and freed via cmpp_args_cleanup(). + */ + cmpp_args_pimpl * pimpl; +}; +typedef struct cmpp_args cmpp_args; + +/** + Empty-initialized cmpp_args instance, intended for const-copy + initialization. +*/ +#define cmpp_args_empty_m { \ + .argc = 0, \ + .arg0 = 0, \ + .pimpl = 0 \ +} + +/** + Empty-initialized instance, intended for non-const-copy + initialization. +*/ +extern const cmpp_args cmpp_args_empty; + +/** + Parses the range [zInBegin,zInBegin+nIn) into a list of cmpp_arg + objects by iteratively processing that range with cmpp_arg_parse(). + If nIn is negative, strlen() is used to calculate it. + + Requires that arg be a cleanly-initialized instance (via + bitwise-copying cmpp_args_empty) or that it have been successfully + used with this function before. Behavior is undefined if pArgs was + not properly initialized. + + The 3rd argument is an optional bitmask of flags from the + cmpp_args_e enum. + + On success it populates arg, noting that an empty list is valid. + The memory pointed to by the arguments made available via + arg->arg0 is all owned by arg and will be invalidated by either a + subsequent call to this function (the memory will be overwritten or + reallocated) or cmpp_args_cleanup() (the memory will be freed). + + On error, returns non-0 and updates pp's error state with info + about the problem. +*/ +CMPP_EXPORT int cmpp_args_parse(cmpp_dx * dx, + cmpp_args * pOut, + unsigned char const * zInBegin, + cmpp_ssize_t nIn, + cmpp_flag32_t flags); + +/** + Frees any resources owned by its argument but does not free the + argument (which is typically stack-allocated). After calling this, + the object may again be used with cmpp_args_parse() (in which case + it eventually needs to be passed to this again). + + This is a harmless no-op if `a` is already cleaned up but `a` must + not be NULL. +*/ +CMPP_EXPORT void cmpp_args_cleanup(cmpp_args *a); + +/** + A wrapper around cmpp_args_parse() which uses dx->args.z as an + input source. This is sometimes convenient in cmpp_dx_f() + implementations which use cmpp_dx_next(), or similar, to read and + process custom directives, as doing so invalidates dx->arg's + memory. + + On success, returns 0 and populates args. On error, returns non-0 + and sets dx->pp's error state. + + cmpp_dx_args_clone() does essentially the same thing, but is more + efficient when dx->args.arg0 is is already parsed. +*/ +CMPP_EXPORT int cmpp_dx_args_parse(cmpp_dx *dx, cmpp_args *args); + +/** + Populates pOut, replacing any current content, with a copy of each + arg in dx->args.arg0 (traversing arg0->next). + + *pOut MUST be cleanly initialized via copying cmpp_args_empty or it + must have previously been used with either cmpp_args_parse() (which + has the same initialization requirement) or this function has + undefined results. + + On success, pOut->argc and pOut->arg0 will refer to pOut's copy + of the arguments. + + Copying of arguments is necessary in cmpp_dx_f() implementations + which need to hold on to arguments for use _after_ calling + cmpp_dx_next() or any API which calls that (which most directives + don't do). See that function for why. +*/ +CMPP_EXPORT int cmpp_dx_args_clone(cmpp_dx * dx, cmpp_args *pOut); + +/** Flags for cmpp_popen(). */ +enum cmpp_popen_e { + /** + Use execl[p](CMD, CMD,0) instead of + execl[p]("/bin/sh","-c",CMD,0). + */ + cmpp_popen_F_DIRECT = 0x01, + /** Use execlp() or execvp() instead of execl() or execv(). */ + cmpp_popen_F_PATH = 0x02 +}; + +/** + Result state for cmpp_popen() and friends. +*/ +struct cmpp_popen_t { + /** + The child process ID. + */ + int childPid; + /** + The child process's stdout. + */ + int fdFromChild; + /** + If not NULL, cmpp_popen() will set *fpToChild to a FILE handle + mapped to the child process's stdin. If it is NULL, the child + process's stdin will be closed instead. + */ + cmpp_FILE **fpToChild; +}; +typedef struct cmpp_popen_t cmpp_popen_t; +/** + Empty-initialized cmpp_popen_t instance, intended for const-copy + initialization. +*/ +#define cmpp_popen_t_empty_m {-1,-1,0} +/** + Empty-initialized instance, intended for non-const-copy + initialization. +*/ +extern const cmpp_popen_t cmpp_popen_t_empty; + +/** + Uses fork()/exec() to run a command in a separate process and open + a two-way stream to it. It is provided in this API to facilitate + the creation of custom directives which shell out to external + processes. + + zCmd must contain the NUL-terminated command to run and any flags + for that command, e.g. "myapp --flag --other-flag". It is passed as + the 4th argument to: + + execl("/bin/sh", "/bin/sh", "-c", zCmd, NULL) + + The po object MUST be cleanly initialized before calling this by + bitwise copying cmpp_popen_t_empty or (depending on the context) + cmpp_popen_t_empty_m. + + Flags: + + - cmpp_popen_F_DIRECT: zCmd is passed to execl(zCmd, zCmd, NULL). + instead of exec(). That can only work if zCmd is a single command + without arguments. + + - cmpp_popen_F_PATH: tells it to use execlp() or execvp(), which + performs path lookup of its initial argument. Again, that can + only work if zCmd is a single command without arguments. + + On success: + + - po->childPid will be set to the PID of the child process. + + - po->fdFromChild is set to the child's stdout file + descriptor. read(2) from it to read from the child. + + - If po->fpToChild is not NULL then *po->fpToChild is set to a + buffered output handle to the child's stdin. fwrite(3) to it to + send the child stuff. Be sure to fflush(3) and/or fclose(3) it to + keep it from hanging forever. If po->fpToChild is NULL then the + stdin of the child is closed. (Why buffered instead of unbuffered? + My attempts at getting unbuffered child stdin to work have all + failed when write() is called on it.) + + On success, the caller is obligated to pass po to cmpp_pclose(). + The caller may pass pi to cmpp_pclose() on error, if that's easier + for them, provided that the po argument was cleanly initialized + before passing it to this function. + + If the caller fclose(3)s *po->fpToChild then they must set it to + NULL so that passing it to cmpp_pclose() knows not to close it. + + On error: you know the drill. This function is a no-op if pp has + error state when it's called, and the current error code is + returned instead. + + This function is only available on non-WASM Unix-like environments. + On others it will always trigger a CMPP_RC_UNSUPPORTED error. + + Bugs: because the command is run via /bin/sh -c ... we cannot tell + if it's actually found. All we can tell is that /bin/sh ran. + + Also: this doesn't capture stderr, so commands should redirect + stderr to stdout. Adding the child's stderr handle to cmpp_popen_t is + a potential TODO without a current use case. + + See: cmpp_pclose() + See: cmpp_popenv() +*/ +CMPP_EXPORT int cmpp_popen(cmpp *pp, unsigned char const *zCmd, + cmpp_flag32_t flags, cmpp_popen_t *po); + +/** + Works like cmpp_popen() except that: + + - It takes it arguments in the form of a main()-style array of + strings because it uses execv() instead of exec(). The + cmpp_popen_F_PATH flag causes it to use execvp(). + + - It does not honor the cmpp_popen_F_DIRECT flag because all + arguments have to be passed in via the arguments array. + + As per execv()'s requirements: azCmd _MUST_ end with a NULL entry. +*/ +CMPP_EXPORT int cmpp_popenv(cmpp *pp, char * const * azCmd, + cmpp_flag32_t flags, cmpp_popen_t *po); + +/** + Closes handles returned by cmpp_popen() and zeroes out po. If the + caller fclose()d *po->fpToChild then they need to set it to NULL so + that this function does not double-close it. + + Returns the result code of the child process. + + After calling this, po may again be used as an argument to + cmpp_popen(). +*/ +CMPP_EXPORT int cmpp_pclose(cmpp_popen_t *po); + +/** + A cmpp_popenv() proxy which builds up an execv()-style array of + arguments from the given args. It has a hard, and mostly arbitrary, + upper limit on the number of args it can take in order to avoid + extra allocation. +*/ +CMPP_EXPORT int cmpp_popen_args(cmpp_dx *dx, cmpp_args const * args, + cmpp_popen_t *p); + + +/** + Callback type for use with cmpp_kav_each(). + + cmpp_kav_each() calls this one time per key/value in such a list, + passing it the relevant key/value strings and lengths, plus the + opaque state pointer which is passed to cmpp_kav_each(). + + Must return 0 on success or update (or propagate) dx->pp's error + state on error. +*/ +typedef int cmpp_kav_each_f( + cmpp_dx *dx, + unsigned char const *zKey, cmpp_size_t nKey, + unsigned char const *zVal, cmpp_size_t nVal, + void* callbackState +); + +/** + Flag bitmask for use with cmpp_kav_each() and cmpp_str_each(). +*/ +enum cmpp_kav_each_e { + /** + The key argument should be expanded using cmpp_arg_to_b() + with a 0 flags value. This flag should normally not be used. + */ + cmpp_kav_each_F_EXPAND_KEY = 0x01, + /** + The key argument should be expanded using cmpp_arg_to_b() + with a 0 flags value. This flag should normally be used. + */ + cmpp_kav_each_F_EXPAND_VAL = 0x02, + /** + Treat (...) value tokens (ttype=cmpp_TT_GroupParen) as integer + expressions. Keys are never treated this way. Without this flag, + the token expands to the ... part of (...). + */ + cmpp_kav_each_F_PARENS_EXPR = 0x04, + /** + Indicates that an empty input list is an error. If this flag is + not set and the list is empty, the callback will not be called + and no error will be triggered. + */ + cmpp_kav_each_F_NOT_EMPTY = 0x08, + /** + Indicates that the list does not have the '->' part(s). That is, + the list needs to be in pairs of KEY VAL rather than triples of + KEY -> VALUE. + */ + cmpp_kav_each_F_NO_ARROW = 0x10, + + /** + If set, keys get the cmpp_arg_to_b_F_BRACE_CALL flag added + to them. This implies cmpp_kav_each_F_EXPAND_KEY. + */ + cmpp_kav_each_F_CALL_KEY = 0x20, + /** Value counterpart of cmpp_kav_each_F_CALL_KEY. */ + cmpp_kav_each_F_CALL_VAL = 0x40, + /** Both cmpp_kav_each_F_CALL_KEY and cmpp_kav_each_F_CALL_VAL. */ + cmpp_kav_each_F_CALL = 0x60, + + //TODO: append to defines which already exist + cmpp_kav_each_F_APPEND = 0, + cmpp_kav_each_F_APPEND_SPACE = 0, + cmpp_kav_each_F_APPEND_NL = 0 +}; + +/** + A helper for cmpp_dx_f() implementations in processing directive + arguments which are lists in this form: + + { a -> b c -> d ... } + + ("kav" is short for "key arrow value".) + + The range [zBegin,zBegin+nIn) contains the raw list (not including + any containing braces, parentheses, quotes, or the like). If nIn is + negative, strlen() is used to calculate it. + + The range is parsed using cmpp_args_parse(). + + For each key/arrow/value triplet in that list, callback() is passed + the stringified form of the key and the value, plus the + callbackState pointer. + + The flags argument controls whether the keys and values get + expanded or not. (Typically the keys should not be expanded but the + values should.) + + Returns 0 on success. If the callback returns non-0, it is expected + to have updated dx's error state. callback() will never be called + when dx's error state is non-0. + + Error results include: + + - CMPP_RC_RANGE: the list is empty does not contain the correct + number of entries (groups of 3, or 2 if flags has + cmpp_kav_each_F_NO_ARROW). + + - CMPP_RC_OOM: allocation error. + + - Any value returned by cmpp_args_parse(). + + - Any number of errors can be triggered during expansion of + keys and values. +*/ +CMPP_EXPORT int cmpp_kav_each( + cmpp_dx *dx, + unsigned char const *zBegin, + cmpp_ssize_t nIn, + cmpp_kav_each_f callback, void *callbackState, + cmpp_flag32_t flags +); + +/** + This works like cmpp_kav_each() except that it treats each token in + the list as a single entry. + + When the callback is triggered, the "key" part will be the raw + token and the "value" part will be the expanded form of that + value. Its flags may contain most of the cmpp_kav_each_F_... flags, + with the exception that cmpp_kav_each_F_EXPAND_KEY has no effect + here. If cmpp_kav_each_F_EXPAND_VAL is not in the flags then the + callback receives the same string for both the key and value. +*/ +CMPP_EXPORT int cmpp_str_each( + cmpp_dx *dx, + unsigned char const *zBegin, + cmpp_ssize_t nIn, + cmpp_kav_each_f callback, void *callbackState, + cmpp_flag32_t flags +); + +/** + An interface for clients to provide directives to the library + on-demand. + + This is called when pp has encountered a directive name is does not + know. It is passed the cmpp object, the name of the directive, and + the opaque state pointer which was passed to cmpp_d_autoloader_et(). + + Implementations should compare dname to any directives they know + about. If they find no match they must return CMPP_RC_NO_DIRECTIVE + _without_ using cmpp_err_set() to make the error persistent. + + If they find a match, they must use cmpp_d_register() to register + it and (on success) return 0. The library will then look again in + the registered directive list for the directive before giving up. + + If they find a match but registration fails then the result of that + failure must be returned. + + For implementation-specific errors, e.g. trying to load a directive + from a DLL but the loading of the DLL fails, implementations are + expected to use cmpp_err_set() to report the error and to return + that result code after performing any necessary cleanup. + + It is legal for an implementation to register multiple directives + in a single invocation (in particular a pair of opening/closing + directives), as well as to register directives other than the one + requested (if necessary). Regardless of which one(s) it registers, + it must return 0 only if it registers one named dname. +*/ +typedef int (*cmpp_d_autoloader_f)(cmpp *pp, char const *dname, void *state); + +/** + A c-pp directive "autoloader". See cmpp_d_autoloader_set() + and cmpp_d_autoloader_take(). +*/ +struct cmpp_d_autoloader { + /** The autoloader callback. */ + cmpp_d_autoloader_f f; + /** + Finalizer for this->state. After calling this, if there's any + chance that this object might be later used, then it is important + that this->state be set to 0 (which this finalizer cannot + do). "Best practice" is to bitwise copy cmpp_d_autoloader_empty + over any instances immediately after calling dtor(). + */ + cmpp_finalizer_f dtor; + /** + Implementation-specific state, to be passed as the final argument + to this->f and this->dtor. + */ + void * state; +}; +typedef struct cmpp_d_autoloader cmpp_d_autoloader; +/** + Empty-initialized cmpp_d_autoloader instance, intended for + const-copy initialization. +*/ +#define cmpp_d_autoloader_empty_m {.f=0,.dtor=0,.state=0} +/** + Empty-initialized cmpp_d_autoloader instance, intended for + non-const-copy initialization. +*/ +extern const cmpp_d_autoloader cmpp_d_autoloader_empty; + +/** + Sets pp's "directive autoloader". Each cmpp instance has but a + single autoloader but this API is provided so that several + instances may be chained from client-side code. + + This function will call the existing autoloader's destructor (if + any), invalidating any pointers to its state object. + + If pNew is not NULL then pp's autoloader is set to a bitwise copy + of *pNew, otherwise it is zeroed out. This transfers ownership of + pNew->state to pp. + + See cmpp_d_autoloader_f()'s docs for how pNew must behave. + + This function has no error conditions but downstream results are + undefined if if pNew and an existing autoloader refer to the same + dtor/state values (a gateway to double-frees). +*/ +CMPP_EXPORT void cmpp_d_autoloader_set(cmpp *pp, cmpp_d_autoloader const * pNew); + +/** + Moves pp's current autoloader state into pOld, transerring + ownership of it to the caller. + + This obligates the caller to eventually either pass that same + pointer to cmpp_d_autoloader_set() (to transfer ownership back to + pp) or to call pOld->dtor() (if it's not NULL), passing it it + pOld->state (even if pOld->state is NULL). In either case, all + contents of pOld are semantically invalidated and perhaps freed. + + This would normally be a prelude to cmpp_d_autoloader_set() to + install a custom, perhaps chained, autoloader. +*/ +CMPP_EXPORT void cmpp_d_autoloader_take(cmpp *pp, cmpp_d_autoloader * pOld); + +/** + True only for ' ' and '\t'. +*/ +CMPP_EXPORT bool cmpp_isspace(int ch); + +/** + Reassigns *p to the address of the first non-space character at or + after the initial *p value. It stops looking if it reaches zEnd. + + If `*p` does not point to memory before zEnd, or is not a part of + the same logical string, results are undefined. + + + Achtung: do not pass this the address of a cmpp_b::z, + or similar, as that will effectively corrupt the buffer's + memory. To trim a whole buffer, use something like: + + ``` + cmpp_b ob = cmpp_b_empty; + ... populate ob...; + // get the trimmed range: + unsigned char const *zB = ob.z; + unsigned char const *zE = zB + n; + cmpp_skip_snl(&zB, zE); + assert( zB<=zE ); + cmpp_skip_snl_trailing(zB, &zE); + assert( zE>=zB ); + printf("trimmed range: [%.*s]\n", (int)(zE-zB), zB); + ``` + + Those assert()s are not error handling - they're demonstrating + invariants of the calls made before them. +*/ +CMPP_EXPORT void cmpp_skip_space( unsigned char const **p, + unsigned char const *zEnd ); + +/** + Works just like cmpp_skip_space() but it also + skips newlines. + + FIXME (2026-02-21): it does not recognize CRNL pairs as + atomic newlines. +*/ +CMPP_EXPORT void cmpp_skip_snl( unsigned char const **p, + unsigned char const *zEnd ); + +/** + "Trims" trailing cmpp_isspace() characters from the range [zBegin, + *p). *p must initially point to one byte after the end of zBegin + (i.e. its NUL byte or virtual EOF). Upon return *p will be modified + leftwards (if at all) until a non-space is found or *p==zBegin. + */ +CMPP_EXPORT void cmpp_skip_space_trailing( unsigned char const *zBegin, + unsigned char const **p ); + +/** + Works just like cmpp_skip_space_trailing() but + skips cmpp_skip_snl() characters. + + FIXME (2026-02-21): it does not recognize CRNL pairs as + atomic newlines. +*/ +CMPP_EXPORT void cmpp_skip_snl_trailing( unsigned char const *zBegin, + unsigned char const **p ); + + +/** + Generic array-of-T list memory-reservation routine. + + *list is the input array-of-T. nDesired is the number of entries to + reserve (list entry count, not byte length). *nAlloc is the number + of entries allocated in the list. sizeOfEntry is the sizeof(T) for + each entry in *list. T may be either a value type or a pointer + type and sizeofEntry must match, i.e. it must be sizeof(T*) for a + list-of-pointers and sizeof(T) for a list-of-objects. + + If pp is not NULL then this function updates pp's error state on + error, else it simply returns CMPP_RC_OOM on error. If pp is not + NULL then this function is a no-op if called when pp's error state + is set, returning that code without other side-effects. + + If nDesired > *nAlloc then *list is reallocated to contain at least + nDesired entries, else this function returns without side effects. + + On success *list is re-assigned to the reallocated list memory, all + newly-(re)allocated memory is zeroed out, and *nAlloc is updated to + the new allocation size of *list (the number of list entries, not + the number of bytes). + + On failure neither *list nor *nAlloc are modified. + + Returns 0 on success or CMPP_RC_OOM on error. Errors generated by + this routine are, at least in principle, recoverable (see + cmpp_err_set()), though that simply means that the pp object is + left in a well-defined state, not that the app can necessarily + otherwise recover from an OOM. + + This seemingly-out-of-API-scope routine is in the public API as a + convenience for client-level cmpp_dx_f() implementations[^1]. This API + internally has an acute need for basic list management and non-core + extensions inherit that as well. + + [^1]: this project's own directives are written as if they were + client-side whenever feasible. Some require cmpp-internal state to + do their jobs, though. +*/ +CMPP_EXPORT int cmpp_array_reserve(cmpp *pp, void **list, cmpp_size_t nDesired, + cmpp_size_t * nAlloc, unsigned sizeOfEntry); + + +/** + The current cmpp_api_thunk::apiVersion value. + See cmpp_api_thunk_map. +*/ +#define cmpp_api_thunk_version 20260206 + +/** + A helper for use with cmpp_api_thunk. + + V() defines the API version number. It invokes + V(NAME,TYPE,VERSION) once. NAME is the member name for the + cmpp_api_thunk struct. TYPE is an integer type. VERSION is the + cmpp_api_thunk object version. This is initially 0 and will + eventually be given a number which increments which new members + appended. This is to enable DLLs to check whether their + cmpp_api_thunk object has the methods they're looking for. + + Then it invokes F(NAME,RETTYPE,PARAMS) and O(NAME,TYPE) + once for each cmpp_api_thunk member in an unspecified order, and + and A(VERSION) an arbitrary number of times. + + F() is for functions. O() is for objects, which are exposed here as + pointers to those objects so that we don't copy them. A() is + injected at each point where a new API version was introduced, and + that number (an integer) is its only argument. A()'s definition + can normally be empty. + + In all cases, NAME is the public API symbol name minus the "cmpp_" + prefix. RETTYPE is the function return type or object type. PARAMS + is the function parameters, wrapped in (...). For O(), TYPE is the + const-qualified type of the object referred to by + NAME. cmpp_api_thunk necessarily exposes those as pointers, but + that pointer is not part of the TYPE argument. + + See cmpp_api_thunk for details. + + In order to help DLLs to not inadvertently use invalid areas of the + API object by referencing members which they loading c-pp version + does not have, this list must only ever be modified by appending to + it. That enables DLLs to check their compile-time + cmpp_api_thunk_version against the dx->pp->api->apiVersion. If + the runtime version is older (less than) than their compile-time + version, the DLL must not access any methods added after + dx->pp->api->apiVersion. +*/ +#define cmpp_api_thunk_map(A,V,F,O) \ + A(0) \ + V(apiVersion,unsigned,cmpp_api_thunk_version) \ + F(mrealloc,void *,(void * p, size_t n)) \ + F(malloc,void *,(size_t n)) \ + F(mfree,void,(void *)) \ + F(ctor,int,(cmpp **pp, cmpp_ctor_cfg const *)) \ + F(dtor,void,(cmpp *pp)) \ + F(reset,void,(cmpp *pp)) \ + F(check_oom,int,(cmpp * const pp, void const * m)) \ + F(is_legal_key,bool,(unsigned char const *, cmpp_size_t n, \ + unsigned char const **)) \ + F(define_legacy,int,(cmpp *, const char *,char const *)) \ + F(define_v2,int,(cmpp *, const char *, char const *)) \ + F(undef,int,(cmpp *, const char *, unsigned int *)) \ + F(define_shadow,int,(cmpp *, char const *, char const *, \ + int64_t *)) \ + F(define_unshadow,int,(cmpp *, char const *, int64_t)) \ + F(process_string,int,(cmpp *, const char *, \ + unsigned char const *, cmpp_ssize_t)) \ + F(process_file,int,(cmpp *, const char *)) \ + F(process_stream,int,(cmpp *, const char *, \ + cmpp_input_f, void *)) \ + F(process_argv,int,(cmpp *, int, char const * const *)) \ + F(err_get,int,(cmpp *, char const **)) \ + F(err_set,int,(cmpp *, int, char const *, ...)) \ + F(err_set1,int,(cmpp *, int, char const *)) \ + F(err_has,int,(cmpp const *)) \ + F(is_safemode,bool,(cmpp const *)) \ + F(sp_begin,int,(cmpp *)) \ + F(sp_commit,int,(cmpp *)) \ + F(sp_rollback,int,(cmpp *)) \ + F(output_f_FILE,int,(void *, void const *, cmpp_size_t)) \ + F(output_f_fd,int,(void *, void const *, cmpp_size_t)) \ + F(input_f_FILE,int,(void *, void *, cmpp_size_t *)) \ + F(input_f_fd,int,(void *, void *, cmpp_size_t *)) \ + F(flush_f_FILE,int,(void *)) \ + F(stream,int,(cmpp_input_f, void *, \ + cmpp_output_f, void *)) \ + F(slurp,int,(cmpp_input_f, void *, \ + unsigned char **, cmpp_size_t *)) \ + F(fopen,cmpp_FILE *,(char const *, char const *)) \ + F(fclose,void,(cmpp_FILE * )) \ + F(outputer_out,int,(cmpp_outputer *, void const *, cmpp_size_t)) \ + F(outputer_flush,int,(cmpp_outputer *)) \ + F(outputer_cleanup,void,(cmpp_outputer *)) \ + F(outputer_cleanup_f_FILE,void,(cmpp_outputer *)) \ + F(delimiter_set,int,(cmpp *, char const *)) \ + F(delimiter_get,void,(cmpp const *, char const **)) \ + F(chomp,bool,(unsigned char *, cmpp_size_t *)) \ + F(b_clear,void,(cmpp_b *)) \ + F(b_reuse,cmpp_b *,(cmpp_b *)) \ + F(b_swap,void,(cmpp_b *, cmpp_b *)) \ + F(b_reserve,int,(cmpp_b *, cmpp_size_t)) \ + F(b_reserve3,int,(cmpp *, cmpp_b *,cmpp_size_t)) \ + F(b_append,int,(cmpp_b *, void const *,cmpp_size_t)) \ + F(b_append4,int,(cmpp *,cmpp_b *,void const *, \ + cmpp_size_t)) \ + F(b_append_ch, int,(cmpp_b *, char)) \ + F(b_append_i32,int,(cmpp_b *, int32_t)) \ + F(b_append_i64,int,(cmpp_b *, int64_t)) \ + F(b_chomp,bool,(cmpp_b *)) \ + F(output_f_b,int,(void *, void const *,cmpp_size_t)) \ + F(outputer_cleanup_f_b,void,(cmpp_outputer *self)) \ + F(version,char const *,(void)) \ + F(tt_cstr,char const *,(int tt)) \ + F(dx_err_set,int,(cmpp_dx *dx, int rc, char const *zFmt, ...)) \ + F(dx_next,int,(cmpp_dx * dx, bool * pGotOne)) \ + F(dx_process,int,(cmpp_dx * dx)) \ + F(dx_consume,int,(cmpp_dx *, cmpp_outputer *, \ + cmpp_d const *const *, unsigned, cmpp_flag32_t)) \ + F(dx_consume_b,int,(cmpp_dx *, cmpp_b *, cmpp_d const * const *, \ + unsigned, cmpp_flag32_t)) \ + F(arg_parse,int,(cmpp_dx * dx, cmpp_arg *, \ + unsigned char const **, unsigned char const *, \ + unsigned char ** , unsigned char const * )) \ + F(arg_strdup,char *,(cmpp *pp, cmpp_arg const *arg)) \ + F(arg_to_b,int,(cmpp_dx * dx, cmpp_arg const *arg, \ + cmpp_b * os, cmpp_flag32_t flags)) \ + F(errno_rc,int,(int errNo, int dflt)) \ + F(d_register,int,(cmpp * pp, cmpp_d_reg const * r, cmpp_d **pOut)) \ + F(dx_f_dangling_closer,void,(cmpp_dx *dx)) \ + F(dx_out_raw,int,(cmpp_dx * dx, void const *z, cmpp_size_t n)) \ + F(dx_out_expand,int,(cmpp_dx const * dx, cmpp_outputer * pOut, \ + unsigned char const * zFrom, cmpp_size_t n, \ + cmpp_atpol_e policy)) \ + F(dx_outf,int,(cmpp_dx *dx, char const *zFmt, ...)) \ + F(dx_delim,char const *,(cmpp_dx const *dx)) \ + F(atpol_from_str,cmpp_atpol_e,(cmpp * pp, char const *z)) \ + F(atpol_get,cmpp_atpol_e,(cmpp const * const pp)) \ + F(atpol_set,int,(cmpp * const pp, cmpp_atpol_e pol)) \ + F(atpol_push,int,(cmpp * pp, cmpp_atpol_e pol)) \ + F(atpol_pop,void,(cmpp * pp)) \ + F(unpol_from_str,cmpp_unpol_e,(cmpp * pp,char const *z)) \ + F(unpol_get,cmpp_unpol_e,(cmpp const * const pp)) \ + F(unpol_set,int,(cmpp * const pp, cmpp_unpol_e pol)) \ + F(unpol_push,int,(cmpp * pp, cmpp_unpol_e pol)) \ + F(unpol_pop,void,(cmpp * pp)) \ + F(path_search,char *,(cmpp *pp, char const *zPath, char pathSep, \ + char const *zBaseName, char const *zExt)) \ + F(args_parse,int,(cmpp_dx * dx, cmpp_args * pOut, \ + unsigned char const * zInBegin, \ + cmpp_ssize_t nIn, cmpp_flag32_t flags)) \ + F(args_cleanup,void,(cmpp_args *a)) \ + F(dx_args_clone,int,(cmpp_dx * dx, cmpp_args *pOut)) \ + F(popen,int,(cmpp *, unsigned char const *, cmpp_flag32_t, \ + cmpp_popen_t *)) \ + F(popenv,int,(cmpp *pp, char * const * azCmd, cmpp_flag32_t flags, \ + cmpp_popen_t *po)) \ + F(pclose,int,(cmpp_popen_t *po)) \ + F(popen_args,int,(cmpp_dx *, cmpp_args const *, cmpp_popen_t *)) \ + F(kav_each,int, (cmpp_dx *,unsigned char const *, cmpp_ssize_t, \ + cmpp_kav_each_f, void *, cmpp_flag32_t)) \ + F(d_autoloader_set,void,(cmpp *pp, cmpp_d_autoloader const * pNew)) \ + F(d_autoloader_take,void,(cmpp *pp, cmpp_d_autoloader * pOld)) \ + F(isspace,bool,(int ch)) \ + F(skip_space,void,(unsigned char const **, unsigned char const *)) \ + F(skip_snl,void,(unsigned char const **, unsigned char const *)) \ + F(skip_space_trailing,void,(unsigned char const *zBegin, \ + unsigned char const **p)) \ + F(skip_snl_trailing,void,(unsigned char const *zBegin, \ + unsigned char const **p)) \ + F(array_reserve,int,(cmpp *pp, void **list, cmpp_size_t nDesired, \ + cmpp_size_t * nAlloc, unsigned sizeOfEntry)) \ + F(module_load,int,(cmpp *, char const *,char const *)) \ + F(module_dir_add,int,(cmpp *, const char *)) \ + O(outputer_FILE,cmpp_outputer const) \ + O(outputer_b,cmpp_outputer const) \ + O(outputer_empty,cmpp_outputer const) \ + O(b_empty,cmpp_b const) \ + A(20251116) \ + F(next_chunk,bool,(unsigned char const **,unsigned char const *, \ + unsigned char,cmpp_size_t*)) \ + A(20251118) \ + F(atdelim_get,void,(cmpp const *,char const **,char const **)) \ + F(atdelim_set,int,(cmpp *,char const *,char const *)) \ + F(atdelim_push,int,(cmpp *,char const *,char const *)) \ + F(atdelim_pop,int,(cmpp *)) \ + A(20251224) \ + F(dx_pos_save,void,(cmpp_dx const *, cmpp_dx_pos *)) \ + F(dx_pos_restore,void,(cmpp_dx *, cmpp_dx_pos const *)) \ + A(20260130) \ + F(dx_is_call,bool,(cmpp_dx * const)) \ + A(20260206) \ + F(b_borrow,cmpp_b *,(cmpp *dx)) \ + F(b_return,void,(cmpp *dx, cmpp_b*)) \ + A(1+cmpp_api_thunk_version) + + +/** + Callback signature for cmpp module import routines. + + This is called by the library after having first encountering this + module (typically after looking for it in a DLL, but static + instances are supported). + + The primary intended purpose of this interface is for + implementations to call cmpp_d_register() (any number of times). It + is also legal to use APIs which set or query defines. This + interface is not intended to interact with pp's I/O in any way + (that's the job of the directives which these functions + register). Violating that will invoke undefined results, perhaps + stepping on the toes of any being-processed directive which + triggered the dynamic load of this directive. + + Errors in module initialization must be reported via cmpp_err_set() + and that code must be returned. + + Implementations must typically call cmpp_api_init(pp) as their + first operation. + + See the files named d-*.c in libcmpp's source tree for examples. +*/ +typedef int (*cmpp_module_init_f)(cmpp * pp); + +/** + Holds information for mapping a cmpp_module_init_f to a name. + Its purpose is to get installed by the CMPP_MODULE_xxx family of + macros and referenced later via a module-loading mechanism. +*/ +struct cmpp_module{ + /** + Symbolic name of the module. + */ + char const * name; + + /** + The initialization routine for the module. + */ + cmpp_module_init_f init; +}; + +/** Convenience typedef. */ +typedef struct cmpp_module cmpp_module; + +/** @def CMPP_MODULE_DECL + + Declares an extern (cmpp_module*) symbol called + cmpp_module__#\#CNAME. + + Use CMPP_MODULE_IMPL2() or CMPP_MODULE_IMPL3() to create the + matching implementation code. + + This macro should be used in the C or H file for a loadable module. + It may be compined in a file with a single CMPP_MODULE_IMPL_SOLO() + declaration with the same name, such that the module can be loaded + both with and without the explicit symbol name. +*/ +#define CMPP_MODULE_DECL(CNAME) \ + extern const cmpp_module * cmpp_module__##CNAME + +/** @def CMPP_MODULE_IMPL + + Intended to be used to implement module declarations. If a module + has both C and H files, CMPP_MODULE_DECL(CNAME) should be used in the + H file and CMPP_MODULE_IMPL2() should be used in the C file. If the + DLL has only a C file (or no public H file), CMPP_MODULE_DECL is + unnecessary. + + If the module's human-use name is a legal C identifier, + CMPP_MODULE_IMPL2() is slightly easier to use than this macro. + + Implements a static cmpp_module object named + cmpp_module__#\#CNAME#\#_impl and a non-static + (cmpp_module*) named cmpp_module__#\#CNAME which points to + cmpp_module__#\#CNAME#\#_impl. (The latter symbol may optionally be + declared in a header file via CMPP_MODULE_DECL.) NAME is used as + the cmpp_module::name value. + + INIT_F must be a cmpp_module_init_f() function pointer. That function + is called when cmpp_module_load() loads the module. + + This macro may be combined in a file with a single + CMPP_MODULE_IMPL_SOLO() declaration using the same CNAME value, + such that the module can be loaded both with and without the + explicit symbol name. + + Example usage, in a module's header file, if any: + + ``` + CMPP_MODULE_DECL(mymodule); + ``` + + (The declaration is not strictly necessary - it is more of a matter + of documentation.) + + And in the C file: + + ``` + CMPP_MODULE_IMPL3(mymodule,"mymodule",mymodule_install); + // OR: + CMPP_MODULE_IMPL2(mymodule,mymodule_install); + ``` + + If it will be the only module in the target DLL, one can also add + this: + + ``` + CMPP_MODULE_IMPL2(mymodule,mymodule_install); + // _OR_ (every so slightly different): + CMPP_MODULE_STANDALONE_IMPL2(mymodule,mymodule_install); + ``` + + Which simplifies client-side module loading by allowing them to + leave out the module name when loading, but that approach only + works if modules are compiled one per DLL (as opposed to being + packaged together in one DLL). + + @see CMPP_MODULE_DECL + @see CMPP_MODULE_IMPL_SOLO +*/ +#define CMPP_MODULE_IMPL3(CNAME,NAME,INIT_F) \ + static const cmpp_module \ + cmpp_module__##CNAME##_impl = { NAME, INIT_F }; \ + const cmpp_module * \ + cmpp_module__##CNAME = &cmpp_module__##CNAME##_impl + +/** @def CMPP_MODULE_IMPL3 + + A simplier form of CMPP_MODULE_IMPL3() for cases where a module name + is a legal C symbol name. +*/ +#define CMPP_MODULE_IMPL2(CNAME,INIT_F) \ + CMPP_MODULE_IMPL3(CNAME,#CNAME,INIT_F) + +/** @def CMPP_MODULE_IMPL_SOLO + + Implements a static cmpp_module symbol called + cmpp_module1_impl and a non-static (cmpp_module*) named + cmpp_module1 which points to cmpp_module1_impl + + INIT_F must be a cmpp_module_init_f. + + This macro must only be used in the C file for a loadable module + when that module is to be the only one in the resuling DLL. Do not + use it when packaging multiple modules into one DLL: use + CMPP_MODULE_IMPL for those cases (CMPP_MODULE_IMPL can also be used + together with this macro). + + @see CMPP_MODULE_IMPL + @see CMPP_MODULE_DECL + @see CMPP_MODULE_STANDALONE_IMPL +*/ +#define CMPP_MODULE_IMPL_SOLO(NAME,INIT_F) \ + static const cmpp_module \ + cmpp_module1_impl = { NAME, INIT_F }; \ + const cmpp_module * cmpp_module1 = &cmpp_module1_impl +/** @def CMPP_MODULE_STANDALONE_IMPL + + CMPP_MODULE_STANDALONE_IMPL2() works like CMPP_MODULE_IMPL_SOLO() + but is only fully expanded if the preprocessor variable + CMPP_MODULE_STANDALONE is defined (to any value). If + CMPP_MODULE_STANDALONE is not defined, this macro expands to a + dummy placeholder which does nothing (but has to expand to + something to avoid leaving a trailing semicolon in the C code, + which upsets the compiler (the other alternative would be to not + require a semicolon after the macro call, but that upsets emacs' + sense of indentation (and keeping emacs happy is more important + than keeping compilers happy (all of these parens are _not_ a + reference to emacs lisp, by the way)))). + + This macro may be used in the same source file as + CMPP_MODULE_IMPL. + + The intention is that DLLs prefer this option over + CMPP_MODULE_IMPL_SOLO, to allow that the DLLs can be built as + standalone DLLs, multi-plugin DLLs, and compiled directly into a + project (in which case the code linking it in needs to resolve and + call the cmpp_module entry for each built-in module). + + @see CMPP_MODULE_IMPL_SOLO + @see CMPP_MODULE_REGISTER +*/ +#if defined(CMPP_MODULE_STANDALONE) +# define CMPP_MODULE_STANDALONE_IMPL2(NAME,INIT_F) \ + CMPP_MODULE_IMPL_SOLO(NAME,INIT_F) +//arguably too much magic in one place: +//# if !defined(CMPP_API_THUNK) +//# define CMPP_API_THUNK +//# endif +#else +# define CMPP_MODULE_STANDALONE_IMPL2(NAME,INIT_F) \ + extern void cmpp_module__dummy_does_not_exist__(void) +#endif + +/** @def CMPP_MODULE_REGISTER3 + + Performs all the necessary setup for registering a loadable module, + including declaration and definition. NAME is the stringified name + of the module. This is normally called immediately after defining + the plugin's init func (which is passed as the 3rd argument to this + macro). + + See CMPP_MODULE_IMPL3() and CMPP_MODULE_STANDALONE_IMPL2() for + the fine details. +*/ +#define CMPP_MODULE_REGISTER3(CNAME,NAME,INIT_F) \ + CMPP_MODULE_IMPL3(CNAME,NAME,INIT_F); \ + CMPP_MODULE_STANDALONE_IMPL2(NAME,INIT_F) + +/** + Slight convenience form of CMPP_MODULE_REGISTER3() which assumes a + registration function name of cpp_ext_${CNAME}_register(). +*/ +#define CMPP_MODULE_REGISTER2(CNAME,NAME) \ + CMPP_MODULE_REGISTER3(CNAME,NAME,cmpp_module__ ## CNAME ## _register) + +/** + Slight convenience form of CMPP_MODULE_REGISTER2() for cases when + CNAME and NAME are the same. +*/ +#define CMPP_MODULE_REGISTER1(CNAME) \ + CMPP_MODULE_REGISTER3(CNAME,#CNAME,cmpp_module__ ## CNAME ## _register) + +/** + This looks for a DLL file named fname. If found, it is dlopen()ed + (or equivalent) and searched for a symbol named symName. If found, + it is assumed to be a cmpp_module instance and its init() method is + invoked. + + If fname is NULL then the module is looked up in the + currently-running program. + + If symName is NULL then the name "cmpp_module1" is assumed, which + is the name used by CMPP_MODULE_IMPL_SOLO() and friends (for use + when a module is the only one in its DLL). + + If no match is found, or there's a problem loading the DLL or + resolving the name, non-0 is returned. Similarly, if the init() + method fails, non-0 is returned. + + The file name is searched using the cmpp_module_dir_add() path, and + if fname is an exact match, or an exact when the system's + conventional DLL file extension is appended to it, that is used + rather than any potential match from the search path. + + On error, pp's error state will contain more information. It's + indeterminate which errors from this API are recoverable. + + This function is a no-op if called when pp's error state is set, + returning that code. + + If built without module-loading support then this will always + fail with CMPP_RC_UNSUPPORTED. +*/ +CMPP_EXPORT int cmpp_module_load(cmpp * pp, char const * fname, + char const * symName); + +/** + Adds the directory or directories listed in zDirs to the search + path used by cmpp_module_load(). The entries are expected to be + either colon- or semicolon-delimited, depending on the platform the + library was built for. + + If zDirs is NULL and pp's library path is empty then it looks for + the environment variable CMPP_MODULE_PATH. If that is set, it is + used in place of zDirs, otherwise the library's compile-time + default is used (as set by the CMPP_MODULE_PATH compile-time value, + which defaults to ".:$prefix/lib/cmpp" in the canonical builds). + This should only be done once per cmpp instance, as the path will + otherwise be extended each time. (The current list structure does + not make it easy to recognize duplicates.) + + Returns 0 on success or if zDirs is empty. Returns CMPP_RC_OOM on + allocation error (ostensibly recoverable - see cmpp_err_set()). + + This is a no-op if called when pp has error state, returning that + code without other side-effects. + + If modules are not enabled then this function is a no-op and always + returns CMPP_RC_UNSUPPORTED _without_ setting pp's error state (as + it's not an error, per se). That can typically be ignored as a + non-error. +*/ +CMPP_EXPORT int cmpp_module_dir_add(cmpp *pp, const char * zDirs); + + +/** + State for a cmpp_dx_pimpl which we need in order to snapshot the + parse position for purposes of restoring it later. This is + basically to support that #query can contain other #query + directives, but this same capability is required by any directives + which want to both process directives in their content block and + loop over the content block. +*/ +struct cmpp_dx_pos { + /** Current parse pos. */ + unsigned char const *z; + /** Current line number. */ + cmpp_size_t lineNo; +}; +typedef struct cmpp_dx_pos cmpp_dx_pos; +#define cmpp_dx_pos_empty_m {.z=0,.lineNo=0U}//,.dline=CmppDLine_empty_m} + +/** + Stores dx's current input position into pos. pos gets completely + initialized by this routine - it need not (in contrast to many + other functions in this library) be cleanly initialized by the + caller first. +*/ +CMPP_EXPORT void cmpp_dx_pos_save(cmpp_dx const * dx, cmpp_dx_pos *pos); + +/** + Restores dx's input position from pos. Results are undefined if pos + is not populated with the result of having passed the same dx/pos + pointer combination to cmpp_dx_pos_save(). +*/ +CMPP_EXPORT void cmpp_dx_pos_restore(cmpp_dx * dx, cmpp_dx_pos const * pos); + +/** + A "thunk" for use with loadable modules, encapsulating all of the + functions from the public cmpp API into an object. This allows + loadable modules to call into the cmpp API if the binary which + loads them not built in such a way that it exports libcmpp's + symbols to the DLL. (On Linux systems, that means if it's not + linked with -rdynamic.) + + For every public cmpp function, this struct has a member with the + same signature and name, minus the "cmpp_" name prefix. Thus + cmpp_foo(...) is accessible as api->foo(...). + + Object-type exports, e.g. cmpp_b_empty, are exposed here as + pointers instead of objects. The CMPP_API_THUNK-installed API + wrapper macros account for that. + + There is only one instance of this class and it gets passed into + cmpp_module_init_f() methods. It is also assigned to the + cmpp_dx::api member of cmpp_dx instances which get passed to + cmpp_dx_f() implementations. + + Loadable modules "should" use this interface to access the API, + rather than the global symbols. If they don't then the module may, + depending on how the loading application was linked, throw + unresolved symbols errors when loading. +*/ +struct cmpp_api_thunk { +#define A(VER) +#define V(N,T,VER) T N; +#define F(N,T,P) T (*N)P; +#define O(N,T) T * const N; + cmpp_api_thunk_map(A,V,F,O) +#undef F +#undef O +#undef V +#undef A +}; + +/** + For loadable modules to be able portably access the cmpp API, + without requiring that their loading binary be linked with + -rdynamic, we need a "thunk". The API exposes cmpp_api_thunk + for that purpose. The following macros set up the thunk for + a given compilation unit. They are intended to only be used + by loadable modules, not generic client code. + + Before including this header, define CMPP_API_THUNK with no value + and/or define CMPP_API_THUNK_NAME to a C symbol name. The latter + macro implies the former and defines the name of the static symbol + to be the local cmpp_api_thunk instance, defaulting to cmppApi. + + The first line of a module's registration function should then be: + + cmpp_api_init(pp); + + where pp is the name of the sole argument to the registration + callback. After that is done, the cmpp_...() APIs may be used via + the macros defined below, all of which route through the thunk + object. +*/ +#if defined(CMPP_API_THUNK) || defined(CMPP_API_THUNK_NAME) +# if !defined(CMPP_API_THUNK) +# define CMPP_API_THUNK +# endif +# if !defined(CMPP_API_THUNK_NAME) +# define CMPP_API_THUNK_NAME cmppApi +# endif +# if !defined(CMPP_API_THUNK__defined) +# define CMPP_API_THUNK__defined +static cmpp_api_thunk const * CMPP_API_THUNK_NAME = 0; +# endif +/** + cmpp_api_init() must be invoked from the module's registration + function, passed the only argument to that function. It sets the + global symbol CMPP_API_THUNK_NAME to its argument. From that point + on, the thunk's API is accessible via cmpp_foo macros which proxy + theThunk->foo. + + It is safe to call this from, e.g. a cmpp_dx_f() implementation, as + it will always have the same pointer, so long as it is not passed + NULL, which would make the next cmpp_...() call segfault. +*/ +# if !defined(CMPP_API_THUNK__assigned) +# define CMPP_API_THUNK__assigned +# define cmpp_api_init(PP) CMPP_API_THUNK_NAME = (PP)->api +# else +# define cmpp_api_init(PP) (void)(PP)/*CMPP_API_THUNK_NAME*/ +# endif +/* What follows is generated code from c-pp's (#pragma api-thunk). */ +/* Thunk APIs which follow are available as of version 0... */ +#define cmpp_mrealloc CMPP_API_THUNK_NAME->mrealloc +#define cmpp_malloc CMPP_API_THUNK_NAME->malloc +#define cmpp_mfree CMPP_API_THUNK_NAME->mfree +#define cmpp_ctor CMPP_API_THUNK_NAME->ctor +#define cmpp_dtor CMPP_API_THUNK_NAME->dtor +#define cmpp_reset CMPP_API_THUNK_NAME->reset +#define cmpp_check_oom CMPP_API_THUNK_NAME->check_oom +#define cmpp_is_legal_key CMPP_API_THUNK_NAME->is_legal_key +#define cmpp_define_legacy CMPP_API_THUNK_NAME->define_legacy +#define cmpp_define_v2 CMPP_API_THUNK_NAME->define_v2 +#define cmpp_undef CMPP_API_THUNK_NAME->undef +#define cmpp_define_shadow CMPP_API_THUNK_NAME->define_shadow +#define cmpp_define_unshadow CMPP_API_THUNK_NAME->define_unshadow +#define cmpp_process_string CMPP_API_THUNK_NAME->process_string +#define cmpp_process_file CMPP_API_THUNK_NAME->process_file +#define cmpp_process_stream CMPP_API_THUNK_NAME->process_stream +#define cmpp_process_argv CMPP_API_THUNK_NAME->process_argv +#define cmpp_err_get CMPP_API_THUNK_NAME->err_get +#define cmpp_err_set CMPP_API_THUNK_NAME->err_set +#define cmpp_err_set1 CMPP_API_THUNK_NAME->err_set1 +#define cmpp_err_has CMPP_API_THUNK_NAME->err_has +#define cmpp_is_safemode CMPP_API_THUNK_NAME->is_safemode +#define cmpp_sp_begin CMPP_API_THUNK_NAME->sp_begin +#define cmpp_sp_commit CMPP_API_THUNK_NAME->sp_commit +#define cmpp_sp_rollback CMPP_API_THUNK_NAME->sp_rollback +#define cmpp_output_f_FILE CMPP_API_THUNK_NAME->output_f_FILE +#define cmpp_output_f_fd CMPP_API_THUNK_NAME->output_f_fd +#define cmpp_input_f_FILE CMPP_API_THUNK_NAME->input_f_FILE +#define cmpp_input_f_fd CMPP_API_THUNK_NAME->input_f_fd +#define cmpp_flush_f_FILE CMPP_API_THUNK_NAME->flush_f_FILE +#define cmpp_stream CMPP_API_THUNK_NAME->stream +#define cmpp_slurp CMPP_API_THUNK_NAME->slurp +#define cmpp_fopen CMPP_API_THUNK_NAME->fopen +#define cmpp_fclose CMPP_API_THUNK_NAME->fclose +#define cmpp_outputer_out CMPP_API_THUNK_NAME->outputer_out +#define cmpp_outputer_flush CMPP_API_THUNK_NAME->outputer_flush +#define cmpp_outputer_cleanup CMPP_API_THUNK_NAME->outputer_cleanup +#define cmpp_outputer_cleanup_f_FILE CMPP_API_THUNK_NAME->outputer_cleanup_f_FILE +#define cmpp_delimiter_set CMPP_API_THUNK_NAME->delimiter_set +#define cmpp_delimiter_get CMPP_API_THUNK_NAME->delimiter_get +#define cmpp_chomp CMPP_API_THUNK_NAME->chomp +#define cmpp_b_clear CMPP_API_THUNK_NAME->b_clear +#define cmpp_b_reuse CMPP_API_THUNK_NAME->b_reuse +#define cmpp_b_swap CMPP_API_THUNK_NAME->b_swap +#define cmpp_b_reserve CMPP_API_THUNK_NAME->b_reserve +#define cmpp_b_reserve3 CMPP_API_THUNK_NAME->b_reserve3 +#define cmpp_b_append CMPP_API_THUNK_NAME->b_append +#define cmpp_b_append4 CMPP_API_THUNK_NAME->b_append4 +#define cmpp_b_append_ch CMPP_API_THUNK_NAME->b_append_ch +#define cmpp_b_append_i32 CMPP_API_THUNK_NAME->b_append_i32 +#define cmpp_b_append_i64 CMPP_API_THUNK_NAME->b_append_i64 +#define cmpp_b_chomp CMPP_API_THUNK_NAME->b_chomp +#define cmpp_output_f_b CMPP_API_THUNK_NAME->output_f_b +#define cmpp_outputer_cleanup_f_b CMPP_API_THUNK_NAME->outputer_cleanup_f_b +#define cmpp_version CMPP_API_THUNK_NAME->version +#define cmpp_tt_cstr CMPP_API_THUNK_NAME->tt_cstr +#define cmpp_dx_err_set CMPP_API_THUNK_NAME->dx_err_set +#define cmpp_dx_next CMPP_API_THUNK_NAME->dx_next +#define cmpp_dx_process CMPP_API_THUNK_NAME->dx_process +#define cmpp_dx_consume CMPP_API_THUNK_NAME->dx_consume +#define cmpp_dx_consume_b CMPP_API_THUNK_NAME->dx_consume_b +#define cmpp_arg_parse CMPP_API_THUNK_NAME->arg_parse +#define cmpp_arg_strdup CMPP_API_THUNK_NAME->arg_strdup +#define cmpp_arg_to_b CMPP_API_THUNK_NAME->arg_to_b +#define cmpp_errno_rc CMPP_API_THUNK_NAME->errno_rc +#define cmpp_d_register CMPP_API_THUNK_NAME->d_register +#define cmpp_dx_f_dangling_closer CMPP_API_THUNK_NAME->dx_f_dangling_closer +#define cmpp_dx_out_raw CMPP_API_THUNK_NAME->dx_out_raw +#define cmpp_dx_out_expand CMPP_API_THUNK_NAME->dx_out_expand +#define cmpp_dx_outf CMPP_API_THUNK_NAME->dx_outf +#define cmpp_dx_delim CMPP_API_THUNK_NAME->dx_delim +#define cmpp_atpol_from_str CMPP_API_THUNK_NAME->atpol_from_str +#define cmpp_atpol_get CMPP_API_THUNK_NAME->atpol_get +#define cmpp_atpol_set CMPP_API_THUNK_NAME->atpol_set +#define cmpp_atpol_push CMPP_API_THUNK_NAME->atpol_push +#define cmpp_atpol_pop CMPP_API_THUNK_NAME->atpol_pop +#define cmpp_unpol_from_str CMPP_API_THUNK_NAME->unpol_from_str +#define cmpp_unpol_get CMPP_API_THUNK_NAME->unpol_get +#define cmpp_unpol_set CMPP_API_THUNK_NAME->unpol_set +#define cmpp_unpol_push CMPP_API_THUNK_NAME->unpol_push +#define cmpp_unpol_pop CMPP_API_THUNK_NAME->unpol_pop +#define cmpp_path_search CMPP_API_THUNK_NAME->path_search +#define cmpp_args_parse CMPP_API_THUNK_NAME->args_parse +#define cmpp_args_cleanup CMPP_API_THUNK_NAME->args_cleanup +#define cmpp_dx_args_clone CMPP_API_THUNK_NAME->dx_args_clone +#define cmpp_popen CMPP_API_THUNK_NAME->popen +#define cmpp_popenv CMPP_API_THUNK_NAME->popenv +#define cmpp_pclose CMPP_API_THUNK_NAME->pclose +#define cmpp_popen_args CMPP_API_THUNK_NAME->popen_args +#define cmpp_kav_each CMPP_API_THUNK_NAME->kav_each +#define cmpp_d_autoloader_set CMPP_API_THUNK_NAME->d_autoloader_set +#define cmpp_d_autoloader_take CMPP_API_THUNK_NAME->d_autoloader_take +#define cmpp_isspace CMPP_API_THUNK_NAME->isspace +#define cmpp_isnl CMPP_API_THUNK_NAME->isnl +#define cmpp_issnl CMPP_API_THUNK_NAME->issnl +#define cmpp_skip_space CMPP_API_THUNK_NAME->skip_space +#define cmpp_skip_snl CMPP_API_THUNK_NAME->skip_snl +#define cmpp_skip_space_trailing CMPP_API_THUNK_NAME->skip_space_trailing +#define cmpp_skip_snl_trailing CMPP_API_THUNK_NAME->skip_snl_trailing +#define cmpp_array_reserve CMPP_API_THUNK_NAME->array_reserve +#define cmpp_module_load CMPP_API_THUNK_NAME->module_load +#define cmpp_module_dir_add CMPP_API_THUNK_NAME->module_dir_add +#define cmpp_outputer_FILE (*CMPP_API_THUNK_NAME->outputer_FILE) +#define cmpp_outputer_b (*CMPP_API_THUNK_NAME->outputer_b) +#define cmpp_outputer_empty (*CMPP_API_THUNK_NAME->outputer_empty) +#define cmpp_b_empty (*CMPP_API_THUNK_NAME->b_empty) +/* Thunk APIs which follow are available as of version 20251116... */ +#define cmpp_next_chunk CMPP_API_THUNK_NAME->next_chunk +/* Thunk APIs which follow are available as of version 20251118... */ +#define cmpp_atdelim_get CMPP_API_THUNK_NAME->atdelim_get +#define cmpp_atdelim_set CMPP_API_THUNK_NAME->atdelim_set +#define cmpp_atdelim_push CMPP_API_THUNK_NAME->atdelim_push +#define cmpp_atdelim_pop CMPP_API_THUNK_NAME->atdelim_pop +/* Thunk APIs which follow are available as of version 20251224... */ +#define cmpp_dx_pos_save CMPP_API_THUNK_NAME->dx_pos_save +#define cmpp_dx_pos_restore CMPP_API_THUNK_NAME->dx_pos_restore +/* Thunk APIs which follow are available as of version 20260130... */ +#define cmpp_dx_is_call CMPP_API_THUNK_NAME->dx_is_call +/* Thunk APIs which follow are available as of version 20260206... */ +#define cmpp_b_borrow CMPP_API_THUNK_NAME->b_borrow +#define cmpp_b_return CMPP_API_THUNK_NAME->b_return + + +#else /* not CMPP_API_THUNK */ +/** + cmpp_api_init() is a no-op when not including a file-local API + thunk. +*/ +# define cmpp_api_init(PP) (void)0 +#endif /* CMPP_API_THUNK */ + +#ifdef __cplusplus +} /* extern "C" */ +#endif +#endif /* include guard */ +#endif /* NET_WANDERINGHORSE_LIBCMPP_H_INCLUDED */ +#if !defined(NET_WANDERINGHORSE_CMPP_INTERNAL_H_INCLUDED) +#define NET_WANDERINGHORSE_CMPP_INTERNAL_H_INCLUDED +/** + This file houses declarations and macros for the private/internal + libcmpp APIs. +*/ +#include "sqlite3.h" + +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <stdarg.h> +#include <stdio.h> +#include <errno.h> + +/* write() and friends */ +#if defined(_WIN32) || defined(WIN32) +# include <io.h> +# include <fcntl.h> +# ifndef access +# define access(f,m) _access((f),(m)) +# endif +#else +# include <unistd.h> +# include <sys/wait.h> +#endif + +#ifndef CMPP_DEFAULT_DELIM +#define CMPP_DEFAULT_DELIM "##" +#endif + +#ifndef CMPP_ATSIGN +#define CMPP_ATSIGN (unsigned char)'@' +#endif + +#ifndef CMPP_MODULE_PATH +#define CMPP_MODULE_PATH "." +#endif + +#if defined(NDEBUG) +#define cmpp__staticAssert(NAME,COND) (void)1 +#else +#define cmpp__staticAssert(NAME,COND) \ + static const char staticAssert_ ## NAME[ (COND) ? 1 : -1 ] = {0}; \ + (void)staticAssert_ ## NAME +#endif + +#if defined(CMPP_OMIT_ALL_UNSAFE) +#undef CMPP_OMIT_D_PIPE +#define CMPP_OMIT_D_PIPE +#undef CMPP_OMIT_D_DB +#define CMPP_OMIT_D_DB +#undef CMPP_OMIT_D_INCLUDE +#define CMPP_OMIT_D_INCLUDE +#undef CMPP_OMIT_D_MODULE +#define CMPP_OMIT_D_MODULE +#endif + +#if !defined(CMPP_VERSION) +#error "exporting CMPP_VERSION to have been set up" +#endif + +#define CMPP__DB_MAIN_NAME "cmpp" + +#if defined(CMPP_AMALGAMATION) +#define CMPP_PRIVATE static +#else +#define CMPP_PRIVATE +#endif + +#if CMPP_PLATFORM_IS_WASM +# define CMPP_PLATFORM_IS_WINDOWS 0 +# define CMPP_PLATFORM_IS_UNIX 0 +# define CMPP_PLATFORM_PLATFORM "wasm" +# define CMPP_PATH_SEPARATOR ':' +# define CMPP__EXPORT_NAMED(X) __attribute__((export_name(#X),used,visibility("default"))) +// See also: +//__attribute__((export_name("theExportedName"), used, visibility("default"))) +# define CMPP_OMIT_FILE_IO /* potential todo but with a large footprint */ +# if !defined(CMPP_PLATFORM_EXT_DLL) +# define CMPP_PLATFORM_EXT_DLL "" +# endif +#else +//# define CMPP_WASM_EXPORT +# define CMPP__EXPORT_NAMED(X) +# if defined(_WIN32) || defined(WIN32) +# define CMPP_PLATFORM_IS_WINDOWS 1 +# define CMPP_PLATFORM_IS_UNIX 0 +# define CMPP_PLATFORM_PLATFORM "windows" +# define CMPP_PATH_SEPARATOR ';' +//# include <io.h> +# elif defined(__MINGW32__) || defined(__MINGW64__) +# define CMPP_PLATFORM_IS_WINDOWS 1 +# define CMPP_PLATFORM_IS_UNIX 0 +# define CMPP_PLATFORM_PLATFORM "windows" +# define CMPP_PATH_SEPARATOR ':' /*?*/ +# elif defined(__CYGWIN__) +# define CMPP_PLATFORM_IS_WINDOWS 0 +# define CMPP_PLATFORM_IS_UNIX 1 +# define CMPP_PLATFORM_PLATFORM "unix" +# define CMPP_PATH_SEPARATOR ':' +# else +# define CMPP_PLATFORM_IS_WINDOWS 0 +# define CMPP_PLATFORM_IS_UNIX 1 +# define CMPP_PLATFORM_PLATFORM "unix" +# define CMPP_PATH_SEPARATOR ':' +# endif +#endif + +#define CMPP__EXPORT(RETTYPE,NAME) CMPP__EXPORT_NAMED(NAME) RETTYPE NAME + +#if !defined(CMPP_PLATFORM_EXT_DLL) +# error "Expecting CMPP_PLATFORM_EXT_DLL to have been set by the auto-configured bits" +# define CMPP_PLATFORM_EXT_DLL "???" +#endif + +#if 1 +# define CMPP_NORETURN __attribute__((noreturn)) +#else +# define CMPP_NORETURN +#endif + +/** @def CMPP_HAVE_DLOPEN + + If set to true, use dlopen() and friends. Requires + linking to -ldl on some platforms. + + Only one of CMPP_HAVE_DLOPEN and CMPP_HAVE_LTDLOPEN may be + true. +*/ +/** @def CMPP_HAVE_LTDLOPEN + + If set to true, use lt_dlopen() and friends. Requires + linking to -lltdl on most platforms. + + Only one of CMPP_HAVE_DLOPEN and CMPP_HAVE_LTDLOPEN may be + true. +*/ +#if !defined(CMPP_HAVE_DLOPEN) +# if defined(HAVE_DLOPEN) +# define CMPP_HAVE_DLOPEN HAVE_DLOPEN +# else +# define CMPP_HAVE_DLOPEN 0 +# endif +#endif + +#if !defined(CMPP_HAVE_LTDLOPEN) +# if defined(HAVE_LTDLOPEN) +# define CMPP_HAVE_LTDLOPEN HAVE_LTDLOPEN +# else +# define CMPP_HAVE_LTDLOPEN 0 +# endif +#endif + +#if !defined(CMPP_ENABLE_DLLS) +# define CMPP_ENABLE_DLLS (CMPP_HAVE_LTDLOPEN || CMPP_HAVE_DLOPEN) +#endif +#if CMPP_ENABLE_DLLS && !defined(CMPP_OMIT_D_MODULE) +# define CMPP_D_MODULE 1 +#else +# define CMPP_D_MODULE 0 +#endif + +/** + Many years of practice have taught that it is literally impossible + to safely close DLLs because simply opening one may trigger + arbitrary code (at least for C++ DLLs) which "might" be used by the + application. e.g. some classloaders use DLL initialization to inject + new classes into the application without the app having to do + anything more than open the DLL. (That's precisely what the cmpp + port of this code is doing except that we don't call it classloading + here.) + + So cmpp does not close DLLs. Except (...sigh...) to please valgrind. + + When CMPP_CLOSE_DLLS is true then this API will keep track of DLL + handles so that they can be closed, and offers the ability for + higher-level clients to close them (all at once, not individually). +*/ +#if !defined(CMPP_CLOSE_DLLS) +#define CMPP_CLOSE_DLLS 1 +#endif + +/** Proxy for cmpp_malloc() which (A) is a no-op if ppCode + and (B) sets pp->err on OOM. +*/ +CMPP_PRIVATE void * cmpp__malloc(cmpp* pp, cmpp_size_t n); + +/** + Internal-use-only flags for use with cmpp_d::flags. + + These supplement the ones from the public API's cmpp_d_e. +*/ +enum cmpp_d_ext_e { + /** + Mask of flag bits from this enum and cmpp_d_e which are for + internal use only and are disallowed in client-side directives. + */ + cmpp_d_F_MASK_INTERNAL = ~cmpp_d_F_MASK, + + /** + If true, and if cmpp_d_F_ARGS_LIST is set, then cmpp_args_parse() + will pass its results to cmpp_args__not_simplify(). Only + directives which eval cmpp_arg expressions need this, and the + library does not expose the pieces for evaluating such + expressions. As such, this flag is for internal use only. This + only has an effect if cmpp_d_F_ARGS_LIST is also used. + */ + cmpp_d_F_NOT_SIMPLIFY = 0x10000, + + /** + Most directives are inert when they are seen in the "falsy" part + of an if/else. The callbacks for such directives are skipped, as + opposed to requiring each directive's callback to check whether + they should be skipping. This flag indicates that a directive + must always be run, even when skipping content (e.g. inside of an + #if 0 block). Only flow-control directives may have the + FLOW_CONTROL bit set. The library API does not expose enough of + its internals for client-defined directives to make flow-control + decisions. + + i really want to get rid of this flag but it seems to be a + necessary evil. + */ + cmpp_d_F_FLOW_CONTROL = 0x20000 +}; + +/** + A single directive line from an input stream. +*/ +struct CmppDLine { + /** Line number in the source input. */ + cmpp_size_t lineNo; + /** Start of the line within its source input. */ + unsigned char const * zBegin; + /** One-past-the-end byte of the line. A virtual EOF. It will only + actually be NUL-terminated if it is the last line of the input + and that input has no trailing newline. */ + unsigned char const * zEnd; +}; +typedef struct CmppDLine CmppDLine; +#define CmppDLine_empty_m {0U,0,0} + +/** + A snippet from a string. +*/ +struct CmppSnippet { + /* Start of the range. */ + unsigned char const *z; + /* Number of bytes. */ + unsigned int n; +}; +typedef struct CmppSnippet CmppSnippet; +#define CmppSnippet_empty_m {0,0} + +/** + CmppLvl represents one "level" of parsing, pushing one level + for each of `#if` and popping one for each `#/if`. + + These pieces are ONLY for use with flow-control directives. It's + not proven that they can be of any use to more than a single + flow-control directive. e.g. if we had a hypothetical #foreach, we + may need to extend this. +*/ +struct CmppLvl { +#if 0 + /** + The directive on whose behalf this level was opened. + */ + cmpp_d const * d; + /** + Opaque directive-specific immutable state. It's provided as a way + for a directive to see whether the top of the stack is correct + after it processes inner directives. + */ + void const * state; +#endif + /** + Bitmask of CmppLvl_F_... + */ + cmpp_flag32_t flags; + /** + The directive line number which started this level. This is used for + reporting the starting lines of erroneously unclosed block + constructs. + */ + cmpp_size_t lineNo; +}; +typedef struct CmppLvl CmppLvl; +#define CmppLvl_empty_m {/*.d=0, .state=0,*/ .flags=0U, .lineNo=0U} + +/** + Declares struct T as a container for a list-of-MT. MT may be + pointer-qualified. cmpp__ListType_impl() with the same arguments + implements T_reserve() for basic list allocation. Cleanup, alas, is + MT-dependent. +*/ +#define cmpp__ListType_decl(T,MT) \ + struct T { \ + MT * list; \ + cmpp_size_t n; \ + cmpp_size_t nAlloc; \ + }; \ + typedef struct T T; \ + int T ## _reserve(cmpp *pp, T *li, cmpp_size_t min) +#define CMPP__MAX(X,Y) ((X)<=(Y) ? (X) : (Y)) +#define cmpp__ListType_impl(T,MT) \ + int T ## _reserve(cmpp *pp,struct T *li, cmpp_size_t min) { \ + return cmpp_array_reserve(pp, (void**)&li->list, min, \ + &li->nAlloc, sizeof(MT)); \ + } + +#define cmpp__LIST_T_empty_m {.list=0,.n=0,.nAlloc=0} + +/** + A dynamically-allocated list of CmppLvl objects. +*/ +cmpp__ListType_decl(CmppLvlList,CmppLvl*); +#define CmppLvlList_empty_m cmpp__LIST_T_empty_m +CMPP_PRIVATE CmppLvl * CmppLvl_push(cmpp_dx *dx); +CMPP_PRIVATE CmppLvl * CmppLvl_get(cmpp_dx const *dx); +CMPP_PRIVATE void CmppLvl_pop(cmpp_dx *dx, CmppLvl *lvl); +CMPP_PRIVATE void CmppLvl_elide(CmppLvl *lvl, bool on); +CMPP_PRIVATE bool CmppLvl_is_eliding(CmppLvl const *lvl); +CMPP_PRIVATE bool cmpp_dx_is_eliding(cmpp_dx const *dx); + +/** + A dynamically-allocated list of cmpp_b objects. +*/ +cmpp__ListType_decl(cmpp_b_list,cmpp_b*); +#define cmpp_b_list_empty_m cmpp__LIST_T_empty_m +extern const cmpp_b_list cmpp_b_list_empty; +CMPP_PRIVATE void cmpp_b_list_cleanup(cmpp_b_list *li); +//CMPP_PRIVATE cmpp_b * cmpp_b_list_push(cmpp_b_list *li); +//CMPP_PRIVATE void cmpp_b_list_reuse(cmpp_b_list *li); +/** + cmpp_b_list sorting policies. NULL entries must + always sort last. +*/ +enum cmpp_b_list_e { + cmpp_b_list_UNSORTED, + /* Smallest first. */ + cmpp_b_list_ASC, + /* Largest first. */ + cmpp_b_list_DESC +}; + +/** + A dynamically-allocated list of cmpp_arg objects. Used by + cmmp_args. +*/ +cmpp__ListType_decl(CmppArgList,cmpp_arg); +#define CmppArgList_empty_m cmpp__LIST_T_empty_m + +/** Allocate a new arg, owned by li, and return it (cleanly zeroed + out). Returns NULL and updates pp->err on error. */ +CMPP_PRIVATE cmpp_arg * CmppArgList_append(cmpp *pp, CmppArgList *li); + +/** + The internal part of the cmpp_args interface. +*/ +struct cmpp_args_pimpl { + /** + We need(?) a (cmpp*) here for finalization/recycling purposes. + */ + cmpp *pp; + bool isCall; + /** + Next entry in the free-list. + */ + cmpp_args_pimpl * nextFree; + /** Version 3 of the real args memory. */ + CmppArgList argli; + /** + cmpp_args_parse() copies each argument's bytes into here, + each one NUL-terminated. + */ + cmpp_b argOut; +}; +#define cmpp_args_pimpl_empty_m { \ + .pp = 0, \ + .isCall = false, \ + .nextFree = 0, \ + .argli = CmppArgList_empty_m, \ + .argOut = cmpp_b_empty_m \ +} +extern const cmpp_args_pimpl cmpp_args_pimpl_empty; +void cmpp_args_pimpl_cleanup(cmpp_args_pimpl *p); + +/** + The internal part of the cmpp_dx interface. +*/ +struct cmpp_dx_pimpl { + /** Start of input. */ + unsigned const char * zBegin; + /** One-after-the-end of input. */ + unsigned const char * zEnd; + /** + Current input position. Generally speaking, only + cmpp_dx_delim_search() should update this, but it turns out that + the ability to rewind the input is necessary for looping + constructs, like #query, when they want to be able to include + other directives in their bodies. + */ + cmpp_dx_pos pos; + /** + Currently input line. + */ + CmppDLine dline; + /** Number of active #savepoints. */ + unsigned nSavepoint; + /** Current directive's args. */ + cmpp_args args; + /** + A stack of state used by #if and friends to inform the innards + that they must not generate output. This is largely historical + and could have been done differently had this code started as a + library instead of a monolithic app. + + TODO is to figure out how best to move this state completely into + the #if handler, rather than fiddle with this all throughout the + processing. We could maybe move this stack into CmppIfState? + */ + CmppLvlList dxLvl; + struct { + /** + A copy of this->d's input line which gets translated + slightly from its native form for futher processing. + */ + cmpp_b line; + /** + Holds the semi-raw input line, stripped only of backslash-escaped + newlines and leading spaces. This is primarily for debug output + but also for custom arg parsing for some directives. + */ + cmpp_b argsRaw; + } buf; + + /** + Record IDs for/from cmpp_[un]define_shadow(). + */ + struct { + /** ID for __FILE__. */ + int64_t sidFile; + /** Rowid for #include path entry. */ + int64_t ridInclPath; + } shadow; + + struct { + /** + Set when we're searching for directives so that we know whether + cmpp_out_expand() should count newlines. + */ + unsigned short countLines; + /** + True if the next directive is the start of a [call]. + */ + bool nextIsCall; + } flags; +}; +/** + Initializes or resets a. Returns non-0 on OOM. +*/ +CMPP_PRIVATE int cmpp_args__init(cmpp *pp, cmpp_args *a); + +/** + If a has state then it's recycled for reuse, else this zeroes out a + except for a->pimpl, which is retained (but may be NULL). +*/ +CMPP_PRIVATE void cmpp_args_reuse(cmpp_args *a); + +#define cmpp_dx_pimpl_empty_m { \ + .zBegin=0, .zEnd=0, \ + .pos=cmpp_dx_pos_empty_m, \ + .dline=CmppDLine_empty_m, \ + .nSavepoint=0, \ + .args = cmpp_args_empty_m, \ + .dxLvl = CmppLvlList_empty_m, \ + .buf = { \ + cmpp_b_empty_m, \ + cmpp_b_empty_m \ + }, \ + .shadow = { \ + .sidFile = 0, \ + .ridInclPath = 0 \ + }, \ + .flags = { \ + .countLines = 0, \ + .nextIsCall = false \ + } \ +} + +/** + A level of indirection for CmppDList in order to be able to + manage ownership of their name (string) lifetimes. +*/ +struct CmppDList_entry { + /** this->d.name.z points to this, which is owned by the CmppDList + which manages this object. */ + char * zName; + /* Potential TODO: move d->id into here. That doesn't eliminate our + dependency on it, though. */ + cmpp_d d; + //cmpp_d_reg reg; +}; +typedef struct CmppDList_entry CmppDList_entry; +#define CmppDList_entry_empty_m {0,cmpp_d_empty_m/*,cmpp_d_reg_empty_m*/} + +/** + A dynamically-allocated list of cmpp_arg objects. Used by CmmpArgs. +*/ +cmpp__ListType_decl(CmppDList,CmppDList_entry*); +#define CmppDList_empty_m cmpp__LIST_T_empty_m + +/** + State for keeping track of DLL handles, a.k.a. shared-object + handles, a.k.a. "soh". + + Instances of this must be either cleanly initialized by bitwise + copying CmppSohList_empty, memset() (or equivalent) them to 0, or + allocating them with CmppSohList_new(). +*/ +cmpp__ListType_decl(CmppSohList,void*); +#define CmppSohList_empty_m cmpp__LIST_T_empty_m + +/** + Closes all handles which have been CmppSohList_append()ed to soli + and frees any memory it owns, but does not free soli (which might + be stack-allocated or part of another struct). + + Special case: if built without DLL-closing support then this + is no-op. +*/ +CMPP_PRIVATE void CmppSohList_close(CmppSohList *soli); + +/** + Operators and operator policies for use with X=Y-format keys. This + is legacy stuff, actually, but some of the #define management still + needs it. +*/ +#define CmppKvp_op_map(E) \ + E(none,"") \ + E(eq1,"=") + +enum CmppKvp_op_e { +#define E(N,S) CmppKvp_op_ ## N, + CmppKvp_op_map(E) +#undef E +}; +typedef enum CmppKvp_op_e CmppKvp_op_e; + +/** + Result type for CmppKvp_parse(). +*/ +struct CmppKvp { + /* Key part of the kvp. */ + CmppSnippet k; + /* Key part of the kvp. Might be empty. */ + CmppSnippet v; + /* Operator part of kvp, if any. */ + CmppKvp_op_e op; +}; +typedef struct CmppKvp CmppKvp; +extern const CmppKvp CmppKvp_empty; + +/** + Parses X or X=Y into p. Sets pp's error state on error. + + If nKey is negative then strlen() is used to calculate it. + + The third argument specifies whether/how to permit/treat the '=' + part of X=Y. +*/ +CMPP_PRIVATE int CmppKvp_parse(cmpp *pp, CmppKvp * p, + unsigned char const *zKey, + cmpp_ssize_t nKey, + CmppKvp_op_e opPolicy); + + +/** + Stack of POD values. Intended for use with cmpp at-token and + undefined key policies. +*/ +#define cmpp__PodList_decl(ST,ET) \ + struct ST { \ + /* current stack index */ \ + cmpp_size_t n; \ + cmpp_size_t na; \ + ET * stack; \ + }; typedef struct ST ST; \ + void ST ## _wipe(ST * s, ET v); \ + int ST ## _push(cmpp *pp, ST * s, ET v); \ + void ST ## _set(ST * s, ET v); \ + void ST ## _finalize(ST * s); \ + void ST ## _pop(ST *s); \ + int ST ## _reserve(cmpp *, ST *, cmpp_size_t min) + +#define cmpp__PodList_impl(ST,ET) \ + void ST ## _wipe(ST * const s, ET v){ \ + if( s->na ) memset(s->stack, (int)v, sizeof(ET)*s->na); \ + s->n = 0; \ + } \ + int ST ## _reserve(cmpp * const pp, ST * const s, \ + cmpp_size_t min){ \ + return cmpp_array_reserve(pp, (void**)&s->stack, min>0 \ + ? min : (s->n \ + ? (s->n==s->na-1 \ + ? s->na*2 : s->n+1) \ + : 8), \ + &s->na, sizeof(ET)); \ + } \ + int ST ## _push(cmpp * const pp, ST * s, ET v){ \ + if( 0== ST ## _reserve(pp, s, 0) ) s->stack[++s->n] = v; \ + return ppCode; \ + } \ + void ST ## _set(ST * s, ET v){ \ + assert(s->n); \ + if( 0== ST ## _reserve(NULL, s, 0) ){ \ + s->stack[s->n] = v; \ + } \ + } \ + ET ST ## _get(ST const * const s){ \ + assert(s->na && s->na >=s->n); \ + return s->stack[s->n]; \ + } \ + void ST ## _pop(ST *s){ \ + assert(s->n); \ + if(s->n) --s->n; \ + } \ + void ST ## _finalize(ST *s){ \ + cmpp_mfree(s->stack); \ + s->stack = NULL; \ + s->n = s->na = 0; \ + } + +cmpp__PodList_decl(PodList__atpol,cmpp_atpol_e); +cmpp__PodList_decl(PodList__unpol,cmpp_unpol_e); + +#define cmpp__epol(PP,WHICH) (PP)->pimpl->policy.WHICH +#define cmpp__policy(PP,WHICH) \ + cmpp__epol(PP,WHICH).stack[cmpp__epol(PP,WHICH).n] + +/** + A "delimiter" object. That is, the "#" referred to in the libcmpp + docs. It's also outfitted for a second delimiter so that it can be + used for the opening/closing delimiters of @tokens@. +*/ +struct cmpp__delim { + /** + Bytes of the directive delimiter/prefix or the @token@ opening + delimiter. Owned elsewhere but often points at this->zOwns. + */ + CmppSnippet open; + /** + Closing @token@ delimiter. This has no meaning for the directive + delimiter. + */ + CmppSnippet close; + /** + Memory, owned by this object, for this->open and this->close. In + the latter case, it's one string with both delimiters encoded in + it. + */ + unsigned char * zOwns; +}; +typedef struct cmpp__delim cmpp__delim; +#define cmpp__delim_empty_m { \ + .open={ \ + .z=(unsigned char*)CMPP_DEFAULT_DELIM, \ + .n=sizeof(CMPP_DEFAULT_DELIM)-1 \ + }, \ + .close=CmppSnippet_empty_m, \ + .zOwns=0 \ +} + +extern const cmpp__delim cmpp__delim_empty; +void cmpp__delim_cleanup(cmpp__delim *d); + +/** + A dynamically-allocated list of cmpp__delim objects. +*/ +cmpp__ListType_decl(cmpp__delim_list,cmpp__delim); +#define cmpp__delim_list_empty_m {0,0,0} +extern const cmpp__delim_list cmpp__delim_list_empty; + +CMPP_PRIVATE cmpp__delim * cmpp__delim_list_push(cmpp *pp, cmpp__delim_list *li); +static inline cmpp__delim * cmpp__delim_list_get(cmpp__delim_list const *li){ + return li->n ? li->list+(li->n-1) : NULL; +} +static inline void cmpp__delim_list_pop(cmpp__delim_list *li){ + assert(li->n); + if( li->n ) cmpp__delim_cleanup(li->list + --li->n); +} +static inline void cmpp__delim_list_reuse(cmpp__delim_list *li){ + while( li->n ) cmpp__delim_cleanup(li->list + --li->n); +} + +/** + An untested experiment: an output buffer proxy. Integrating this + fully would require some surgery, but it might also inspire me to + do the same with input and stream it rather than slurp it all at + once. +*/ +#define CMPP__OBUF 0 + +typedef struct cmpp__obuf cmpp__obuf; +#if CMPP__OBUF +/** + An untested experiment. +*/ +struct cmpp__obuf { + /** Start of the output buffer. */ + unsigned char * begin; + /** One-after-the-end of this->begin. Virtual EOF. */ + unsigned char const * end; + /** Current write position. Must initially be + this->begin. */ + unsigned char * cursor; + /** + True if this object owns this->begin, which must have been + allocated using cmpp_malloc() or cmpp_realloc(). + */ + bool ownsMemory; + /** Propagating result code. */ + int rc; + /** + The output channel to buffer for. Flushing + */ + cmpp_outputer dest; +}; + +#define cmpp__obuf_empty_m { \ + .begin=0, .end=0, .cursor=0, .ownsMemory=false, \ + .rc=0, .dest=cmpp_outputer_empty_m \ +} +extern const cmpp__obuf cmpp__obuf_empty; +extern const cmpp_outputer cmpp_outputer_obuf; +#endif /* CMPP__OBUF */ +/** + The main public-API context type for this library. +*/ +struct cmpp_pimpl { + /* Internal workhorse. */ + struct { + sqlite3 * dbh; + /** + Optional filename. Memory is owned by this object. + */ + char * zName; + } db; + /** + Current directive context. It's const, primarily to help protect + cmpp_dx_f()'s from inadvertent side effects of changes which + lower-level APIs might make to it. Maybe it shouldn't be: if it + were not then we could update dx->zDelim from + cmpp__delimiter_set(). + */ + cmpp_dx const * dx; + /* Output channel. */ + cmpp_outputer out; + /** + Delimiters version 2. + */ + struct { + /** + Directive delimiter. + */ + cmpp__delim_list d; + /** + @token@ delimiters. + */ + cmpp__delim_list at; + } delim; + struct { + +#define CMPP__SEL_V_FROM(N) \ + "(SELECT v FROM " CMPP__DB_MAIN_NAME ".vdef WHERE k=?" #N \ + " ORDER BY source LIMIT 1)" + + /** + One entry for each distinct query used by cmpp: E(X,SQL), where + X is the member's name and SQL is its SQL. + */ +#define CMPP_SAVEPOINT_NAME "_cmpp_" +#define CmppStmt_map(E) \ + E(sdefIns, \ + "INSERT INTO " \ + CMPP__DB_MAIN_NAME ".sdef" \ + "(t,k,v) VALUES(?1,?2,?3) RETURNING id") \ + E(defIns, \ + "INSERT OR REPLACE INTO " \ + CMPP__DB_MAIN_NAME ".def" \ + "(t,k,v) VALUES(?1,?2,?3)") \ + E(defDel, \ + "DELETE FROM " \ + CMPP__DB_MAIN_NAME ".def" \ + " WHERE k GLOB ?1") \ + E(sdefDel, \ + "DELETE FROM " \ + CMPP__DB_MAIN_NAME ".sdef" \ + " WHERE k=?1 AND id>=?2") \ + E(defHas, \ + "SELECT 1 FROM " \ + CMPP__DB_MAIN_NAME ".vdef" \ + " WHERE k = ?1") \ + E(defGet, \ + "SELECT source,t,k,v FROM " \ + CMPP__DB_MAIN_NAME ".vdef" \ + " WHERE k = ?1 ORDER BY source LIMIT 1") \ + E(defGetBool, \ + "SELECT cmpp_truthy(v) FROM " \ + CMPP__DB_MAIN_NAME ".vdef" \ + " WHERE k = ?1" \ + " ORDER BY source LIMIT 1") \ + E(defGetInt, \ + "SELECT CAST(v AS INTEGER)" \ + " FROM " CMPP__DB_MAIN_NAME ".vdef" \ + " WHERE k = ?1" \ + " ORDER BY source LIMIT 1") \ + E(defSelAll, "SELECT t,k,v" \ + " FROM " CMPP__DB_MAIN_NAME ".vdef" \ + " ORDER BY source, k") \ + E(inclIns," INSERT OR FAIL INTO " \ + CMPP__DB_MAIN_NAME ".incl(" \ + " file,srcFile, srcLine" \ + ") VALUES(?,?,?)") \ + E(inclDel, "DELETE FROM " \ + CMPP__DB_MAIN_NAME ".incl WHERE file=?") \ + E(inclHas, "SELECT 1 FROM " \ + CMPP__DB_MAIN_NAME ".incl WHERE file=?") \ + E(inclPathAdd, "INSERT INTO " \ + CMPP__DB_MAIN_NAME ".inclpath(priority,dir) " \ + "VALUES(coalesce(?1,0),?2) " \ + "ON CONFLICT DO NOTHING " \ + "RETURNING rowid /*xlates to 0 on conflict*/") \ + E(inclPathRmId, "DELETE FROM " \ + CMPP__DB_MAIN_NAME ".inclpath WHERE rowid=?1 " \ + "RETURNING rowid") \ + E(inclSearch, \ + "SELECT ?1 fn WHERE cmpp_file_exists(fn) " \ + "UNION ALL SELECT fn FROM (" \ + " SELECT replace(dir||'/'||?1, '//','/') AS fn " \ + " FROM " CMPP__DB_MAIN_NAME ".inclpath" \ + " WHERE cmpp_file_exists(fn) " \ + " ORDER BY priority DESC, rowid LIMIT 1" \ + ")") \ + E(cmpVV, "SELECT cmpp_compare(?1,?2)") \ + E(cmpDV, \ + "SELECT cmpp_compare(" \ + CMPP__SEL_V_FROM(1) ", ?2" \ + ")") \ + E(cmpVD, \ + "SELECT cmpp_compare(" \ + "?1," CMPP__SEL_V_FROM(2) \ + ")") \ + E(cmpDD, \ + "SELECT cmpp_compare(" \ + CMPP__SEL_V_FROM(1) \ + "," \ + CMPP__SEL_V_FROM(2) \ + ")") \ + E(dbAttach, \ + "ATTACH ?1 AS ?2") \ + E(dbDetach, \ + "DETACH ?1") \ + E(spBegin, "SAVEPOINT " CMPP_SAVEPOINT_NAME) \ + E(spRollback, \ + "ROLLBACK TO SAVEPOINT " CMPP_SAVEPOINT_NAME) \ + E(spRelease, \ + "RELEASE SAVEPOINT " CMPP_SAVEPOINT_NAME) \ + E(insTtype, \ + "INSERT INTO " CMPP__DB_MAIN_NAME ".ttype" \ + "(t,n,s) VALUES(?1,?2,?3)") \ + E(selPathSearch, \ + /* sqlite.org/forum/forumpost/840c98a8e87c2207 */ \ + "WITH path(basename, sep, ext, path) AS (\n" \ + " select\n" \ + " ?1 basename,\n" \ + " ?2 sep,\n" \ + " ?3 ext,\n" \ + " ?4 path\n" \ + "),\n" \ + "pathsplit(i, l, c, r) AS (\n" \ + "-- i = sequential ID\n" \ + "-- l = Length remaining\n" \ + "-- c = text remaining\n" \ + "-- r = current unpacked value\n" \ + " SELECT 1,\n" \ + " length(p.path)+length(p.sep),\n" \ + " p.path||p.sep, ''\n" \ + " FROM path p\n" \ + " UNION ALL\n" \ + " SELECT i+1, instr( c, p.sep ) l,\n" \ + " substr( c, instr( c, p.sep ) + 1) c,\n" \ + " trim( substr( c, 1,\n" \ + " instr( c, p.sep) - 1) ) r\n" \ + " FROM pathsplit, path p\n" \ + " WHERE l > 0\n" \ + "),\n" \ + "thefile (f) AS (\n" \ + " select basename f FROM path\n" \ + " union all\n" \ + " select basename||ext\n" \ + " from path where ext is not null\n" \ + ")\n" \ + "select 0 i, replace(f,'//','/') AS fn\n" \ + "from thefile where cmpp_file_exists(fn)\n" \ + "union all\n" \ + "select i, replace(r||'/'||f,'//','/') fn\n" \ + "from pathsplit, thefile\n" \ + "where r<>'' and cmpp_file_exists(fn)\n" \ + "order by i\n" \ + "limit 1;") + + /* trivia: selPathSearch (^^^) was generated using + cmpp's #c-code directive. */ + +#define E(N,S) sqlite3_stmt * N; + CmppStmt_map(E) +#undef E + + } stmt; + + /** Error state. */ + struct { + /** Result code. */ + int code; + /** Error string owned by this object. */ + char * zMsg; + /** Either this->zMsg or an external error string. */ + char const * zMsgC; + } err; + + /** State for SQL tracing. */ + struct { + bool expandSql; + cmpp_size_t counter; + cmpp_outputer out; + } sqlTrace; + + struct { + /** If set properly, cmpp_dtor() will free this + object, else it will not. */ + void const * allocStamp; + /** + How many dirs we believe are in the #include search list. We + only do this for the sake of the historical "if no path was + added, assume '.'" behavior. This really ought to go away. + */ + unsigned nIncludeDir; + /** + The current depth of cmpp_process_string() calls. We do this so + the directory part of #include'd files can get added to the + #include path and be given a higher priority than previous + include path entries in the stack. + */ + int nDxDepth; + /* Number of active #savepoints. */ + unsigned nSavepoint; + /* If >0, enables certain debugging output. */ + char doDebug; + /* If true, chomp() files read via -Fx=file. */ + unsigned char chompF; + /* Flags passed to cmpp_ctor(). */ + cmpp_flag32_t newFlags; + + /** + An ugly hack for getting cmpp_d_register() to get + syntactically-illegal directive names, like "@policy", + to register. + */ + bool isInternalDirectiveReg; + /** + True if the next directive is the start of a [call]. This is + used for: + + 1) To set cmpp_dx::isCall, which is useful in certain + directives. + + 2) So that the cmpp_dx object created for the call can inherit + the line number from its parent context. That's significant + for error reporting. + + 3) So that #2's cmpp_dx object can communicate that flag to + cmpp_dx_next(). + */ + bool nextIsCall; + + /** + True until the cmpp's (sometimes) lazy init has been run. This + is essentially a kludge to work around a wrench cmpp_reset() + throws into cmpp state. Maybe we should just remove + cmpp_reset() from the interface, since error recovery in this + context is not really a thing. + */ + bool needsLazyInit; + } flags; + + /** Policies. */ + struct { + /** @token@-parsing policy. */ + PodList__atpol at; + /** Policy towards referencing undefined symbols. */ + PodList__unpol un; + } policy; + + /** + Directive state. + */ + struct { + /** + Runtime-installed directives. + */ + CmppDList list; + /** + Directive autoloader/auto-registerer. + */ + cmpp_d_autoloader autoload; + } d; + + struct { + /** + List of DLL handles opened by cmpp_module_extract(). + */ + CmppSohList sohList; + /** + Search path for DLLs, delimited by this->pathSep. + */ + cmpp_b path; + /** + File extension for DLLs. + */ + char const * soExt; + /** Separator char for this->path. */ + char pathSep; + } mod; + + struct { + /** + Buffer cache. Managed by cmpp_b_borrow() and cmpp_b_return(). + */ + cmpp_b_list buf; + /** How/whether this->list is sorted. */ + enum cmpp_b_list_e bufSort; + /** + Head of the free-list. + */ + cmpp_args_pimpl * argPimpl; + } recycler; +}; + +/** IDs Distinct for each cmpp::stmt member. */ +enum CmppStmt_e { + CmppStmt_none = 0, +#define E(N,S) CmppStmt_ ## N, + CmppStmt_map(E) +#undef E +}; + +static inline cmpp__delim * cmpp__pp_delim(cmpp const *pp){ + return cmpp__delim_list_get(&pp->pimpl->delim.d); +} +static inline char const * cmpp__pp_zdelim(cmpp const *pp){ + cmpp__delim const * const d = cmpp__pp_delim(pp); + return d ? (char const *)d->open.z : NULL; +} +#define cmpp__dx_delim(DX) cmpp__pp_delim(DX->pp) +#define cmpp__dx_zdelim(DX) cmpp__pp_zdelim(DX->pp) + +/** + Emit [z,(char*)z+n) to the given output channel if + (A) pOut->out is not NULL and (B) pp has no error state and (C) + n>0. On error, pp's error state is updated. Returns pp->err.code. + + Skip level is not honored. +*/ +CMPP_PRIVATE int cmpp__out2(cmpp *pp, cmpp_outputer *pOut, void const *z, cmpp_size_t n); + +CMPP_PRIVATE void cmpp__err_clear(cmpp *pp); + + +/** + Initialize pp->db.dbh. If it's already open or ppCode!=0 + then ppCode is returned. +*/ +int cmpp__db_init(cmpp *pp); + +/** + Returns the pp->pimpl->stmt.X corresponding to `which`, initializing it if + needed. If it returns NULL then either this was called when pp has + its error state set or this function will set the error state. + + If prepEvenIfErr is true then the ppCode check is bypassed, but it + will still fail if pp->pimpl->db is not opened or if the preparation itself + fails. +*/ +sqlite3_stmt * cmpp__stmt(cmpp * pp, enum CmppStmt_e which, + bool prepEvenIfErr); + +/** + Reminder to self: this must return an SQLITE_... code, not a + CMPP_RC_... code. + + On success it returns 0, SQLITE_ROW, or SQLITE_DONE. On error it + returns another non-0 SQLITE_... code and updates pp->pimpl->err. + + This is a no-op if called when pp has an error set, returning + SQLITE_ERROR. + + If resetIt is true, q is passed to cmpp__stmt_reset(), else the + caller must eventually reset it. +*/ +int cmpp__step(cmpp * const pp, sqlite3_stmt * const q, bool resetIt); + +/** Resets and clear bindings from q (if q is not NULL). */ +void cmpp__stmt_reset(sqlite3_stmt * const q); + +/** + Expects an SQLite result value. If it's SQLITE_OK, SQLITE_ROW, or + SQLITE_DONE, 0 is returned without side-effects, otherwise pp->err + is updated with pp->db's current error state. zMsgSuffix is an + optional prefix for the error message. +*/ +int cmpp__db_rc(cmpp *pp, int dbRc, char const *zMsgSuffix); + +/* Proxy for sqlite3_bind_int64(). */ +int cmpp__bind_int(cmpp *pp, sqlite3_stmt *pStmt, int col, int64_t val); + +/** + Proxy for cmpp__bind_text() which encodes val as a string. + + For queries which compare values, it's important that they all have + the same type, so some cases where we might want an int needs to be + bound as text instead. See #query for one such case. +*/ +int cmpp__bind_int_text(cmpp *pp, sqlite3_stmt *pStmt, int col, int64_t val); + +/* Proxy for sqlite3_bind_null(). */ +int cmpp__bind_null(cmpp *pp, sqlite3_stmt *pStmt, int col); + +/* Proxy for sqlite3_bind_text() which updates pp->err on error. */ +int cmpp__bind_text(cmpp *pp,sqlite3_stmt *pStmt, int col, + unsigned const char * zStr); + +/* Proxy for sqlite3_bind_text() which updates pp->err on error. */ +int cmpp__bind_textn(cmpp *pp,sqlite3_stmt *pStmt, int col, + unsigned const char *zStr, cmpp_ssize_t len); + +/** + Adds zDir to the include path, using the given priority value (use + 0 except for the implicit cwd path which #include should (but does + not yet) set). If pRowid is not NULL then *pRowid gets set to + either 0 (if zDir was already in the path) or the row id of the + newly-inserted record, which can later be used to delete just that + entry. + + If this returns a non-zero value via pRowid, the caller is + obligated to eventually pass *pRowid to cmpp__include_dir_rm_id(), + even if pp is in an error state. + + TODO: normalize zDir (at least remove trailing slashes) before + insertion to avoid that both a/b and a/b/ get be inserted. +*/ +int cmpp__include_dir_add(cmpp *pp, const char * zDir, int priority, int64_t * pRowid); + +/** + Deletes the include path entry with the given rowid. This will make + make the attempt even if pp is in an error state but also retains + any existing error rather than overwriting it if this operation + somehow fails. Returns pp's error code. + + It is not an error for the given entry to not exist. +*/ +int cmpp__include_dir_rm_id(cmpp *pp, int64_t pRowid); + + +#if 0 +/** + Proxy for sqlite3_bind_text(). It uses sqlite3_str_vappendf() so + supports all of its formatting options. +*/ +int cmpp__bind_textv(cmpp*pp, sqlite3_stmt *pStmt, int col, + const char * zFmt, ...); +#endif + +/** + Proxy for sqlite3_str_finish() which updates pp's error state if s + has error state. Returns s's string on success and NULL on + error. The returned string must eventualy be passed to + cmpp_mfree(). It also, it turns out, returns NULL if s is empty, so + callers must check pp->err to see if NULL is an error. + + If n is not NULL then on success it is set to the byte length of + the returned string. +*/ +char * cmpp_str_finish(cmpp *pp, sqlite3_str *s, int * n); + +/** + Searches pp's list of directives. If found, return it else return + NULL. See cmpp__d_search3(). +*/ +cmpp_d const * cmpp__d_search(cmpp *pp, const char *zName); + +/** + Flags for use with the final argument to + cmpp__d_search3(). +*/ +enum cmpp__d_search3_e { + /** Internal delayed-registered directives. */ + cmpp__d_search3_F_DELAYED = 0x01, + /** Directive autoloader. */ + cmpp__d_search3_F_AUTOLOADER = 0x02, + /** Search for a DLL. */ + cmpp__d_search3_F_DLL = 0x04, + /** Options which do not trigger DLL lookup. */ + cmpp__d_search3_F_NO_DLL = 0 + | cmpp__d_search3_F_DELAYED + | cmpp__d_search3_F_AUTOLOADER, + /** All options. */ + cmpp__d_search3_F_ALL = 0 + | cmpp__d_search3_F_DELAYED + | cmpp__d_search3_F_AUTOLOADER + | cmpp__d_search3_F_DLL +}; + +/** + Like cmpp__d_search() but if no match is found then it will search + through its other options and, if found, register it. + + The final argument specifies where to search. cmpp__d_search() + always checked first. After that, depending on "what", the search + order is: (1) internal delayed-load modules, (2) autoloader, (3) + DLL. + + This may update pp's error state, in which case it will return + NULL. +*/ +cmpp_d const * cmpp__d_search3(cmpp *pp, const char *zName, + cmpp_flag32_t what); + +/** + Sets pp's error state (A) if it's not set already and (B) if + !cmpp_is_legal_key(zKey). If permitEqualSign is true then '=' is + legal (to support legacy CLI pieces). Returns ppCode. +*/ +int cmpp__legal_key_check(cmpp *pp, unsigned char const *zKey, + cmpp_ssize_t nKey, + bool permitEqualSign); + +/** + Appends DLL handle soh to soli. Returns 0 on success, CMPP_RC_OOM + on error. If pp is not NULL then its error state is updated as + well. + + Results are undefined if soli was not cleanly initialized (by + copying CmppSohList_empty or using CmppSohList_new()). + + Special case: if built without DLL-closing support, this is a no-op + returning 0. +*/ +int CmppSohList_append(cmpp *pp, CmppSohList *soli, void *soh); + +/** True if arg is of type cmpp_TT_Word and it looks like it + _might_ be a filename or flag argument. Might. */ +bool cmpp__arg_wordIsPathOrFlag(cmpp_arg const * const arg); + +/** + Helper for #query and friends. Binds aVal's value to column bindNdx + of q. + + It expands cmpp_TT_StringAt and cmpp_TT_Word aVal. cmpp_TT_String + and cmpp_TT_Int are bound as strings. A cmpp_TT_GroupParen aVal is + eval'ed as an integer and that int gets bound as a string. + + This function strictly binds everything as strings, even if the + value being bound is of type cmpp_TT_Int or cmpp_TT_GroupParen, so that + comparison queries will work as expected. + + Returns ppCode. +*/ +int cmpp__bind_arg(cmpp_dx * const dx, sqlite3_stmt * q, + int bindNdx, cmpp_arg const * aVal); + +/** + Helper for #query's bind X part, where aGroup is that X. + + A wrapper around cmpp__bind_arg(). Requires aGroup->ttype to be + either cmpp_TT_GroupBrace or cmpp_TT_GroupSquiggly and to have + non-empty content. cmpp_TT_GroupBrace treats it as a list of values + to bind. cmpp_TT_GroupSquiggly expects sets of 3 tokens per stmt + column in one of these forms: + + :bindName -> value + $bindName -> value + + Each LHS refers to a same-named binding in q's SQL, including the + ':' or '$' prefix. (SQLite supports an '@' prefix but we don't + allow it here to avoid confusion with cmpp_TT_StringAt tokens.) + + Each bound value is passed to cmpp__bind_arg() for processing. + + On success, each aGroup entry is bound to q. On error q's state is + unspecified. Returns ppCode. + + See cmpp__bind_arg() for notes about the bind data type. +*/ +int cmpp__bind_group(cmpp_dx * const dx, sqlite3_stmt * const q, + cmpp_arg const * const aGroup); + +/** + Returns true if the given key is already in the `#define` list, + else false. Sets pp's error state on db error. + + nName is the length of the key part of zName (which might have + a following =y part. If it's negative, strlen() is used to + calculate it. +*/ +int cmpp_has(cmpp *pp, const char * zName, cmpp_ssize_t nName); + +/** + Returns true if the given key is already in the `#define` list, and + it has a truthy value (is not empty and not equal to '0'), else + false. If zName contains an '=' then only the part preceding that + is used as the key. + + nName is the length of zName, or <0 to use strlen() to figure + it out. + + Updates ppCode on error. +*/ +int cmpp__get_bool(cmpp *pp, unsigned const char * zName, + cmpp_ssize_t nName); + +/** + Fetches the given define. If found, sets *pOut to it, else pOut is + unmodified. Returns pp->err.code. If bRequire is true and no entry + is found p->err.code is updated. +*/ +int cmpp__get_int(cmpp *pp, unsigned const char * zName, + cmpp_ssize_t nName, int *pOut); + +/** + Searches for a define where (k GLOB zName). If one is found, a copy + of it is assigned to *zVal (the caller must eventually db_free() + it), *nVal (if nVal is not NULL) is assigned its strlen, and + returns non-0. If no match is found, 0 is returned and neither + *zVal nor *nVal are modified. If more than one result matches, a + fatal error is triggered. + + It is legal for *zVal to be NULL (and *nVal to be 0) if it returns + non-0. That just means that the key was defined with no value part. + + Fixme: return 0 on success and set output *gotOne=0|1. +*/ +int cmpp__get(cmpp *pp, unsigned const char * zName, + cmpp_ssize_t nName, + unsigned char **zVal, unsigned int *nVal); + +/** + Like cmp__get() but puts its output in os. +*/ +int cmpp__get_b(cmpp *pp, unsigned const char * zName, + cmpp_ssize_t nName, cmpp_b * os, + bool enforceUndefPolicy); + + +/** + Helper for #query and friends. + + It expects that q has just been stepped. For each column in the + row, it sets a define named after the column. If q has row data + then the values come from there. If q has no row then: if + defineIfNoRow is true then it defines each column name to an empty + value else it defines nothing. +*/ +int cmpp__define_from_row(cmpp * const pp, sqlite3_stmt * const q, + bool defineIfNoRow); + +/** Start a new savepoint for dx. */ +int cmpp__dx_sp_begin(cmpp_dx * const dx); +/** Commit and close dx's current savepoint. */ +int cmpp__dx_sp_commit(cmpp_dx * const dx); +/** Roll back and close dx's current savepoint. */ +int cmpp__dx_sp_rollback(cmpp_dx * const dx); + +/** + Append's dx's file/line information to sstr. It returns void + because that's how sqlite3_str_appendf() and friends work. +*/ +void cmpp__dx_append_script_info(cmpp_dx const * dx, + sqlite3_str * sstr); + +/** + If zName matches one of the delayed-load directives, that directive + is registered and 0 is returned. CMPP_RC_NO_DIRECTIVE is returned if + no match is found, but pp's error state is not updated in that + case. If a match is found and registration fails, that result code + will propagate via pp. +*/ +int cmpp__d_delayed_load(cmpp *pp, char const *zName); + +void cmpp__dump_defines(cmpp *pp, cmpp_FILE * fp, int bIndent); + +/** + Like cmpp_tt_cstr(), but if bSymbolName is false then it returns + the higher-level token name, which is NULL for most token types. +*/ +char const * cmpp__tt_cstr(int tt, bool bSymbolName); + +/** + Expects **zPos to be one of ('(' '{' '[' '"' '\'') and zEnd to be + the logical EOF for *zPos. + + This looks for a matching closing token, accounting for nesting. On + success, returns 0 and sets *zPos to the closing character. + + On error it update's pp's error state and returns that code. pp may + be NULL. + + If pNl is not NULL then *pNl is incremented for each '\n' character + seen while looking for the closing character. +*/ +int cmpp__find_closing2(cmpp *pp, + unsigned char const **zPos, + unsigned char const *zEnd, + cmpp_size_t *pNl); + +#define cmpp__find_closing(PP,Z0,Z1) \ + cmpp__find_closing2(PP, Z0, Z1, NULL) + +static inline cmpp_size_t cmpp__strlen(char const *z, cmpp_ssize_t n){ + return n<0 ? (cmpp_size_t)strlen(z) : (cmpp_size_t)n; +} +static inline cmpp_size_t cmpp__strlenu(unsigned char const *z, cmpp_ssize_t n){ + return n<0 ? (cmpp_size_t)strlen((char const *)z) : (cmpp_size_t)n; +} + +/** + If ppCode is not set and pol resolves to cmpp_atpol_OFF then this + updates ppCode with a message about the lack of support for + at-strings. If cmpp_atpol_CURRENT==pol then pp's current policy is + checked. Returns ppCode. +*/ +int cmpp__StringAtIsOk(cmpp * const pp, cmpp_atpol_e pol); + +/** + "define"s zKey to zVal, recording the value type as tType. +*/ +int cmpp__define2(cmpp *pp, + unsigned char const * zKey, + cmpp_ssize_t nKey, + unsigned char const *zVal, + cmpp_ssize_t nVal, + cmpp_tt tType); + +/** + Evals pArgs's arguments as an integer expression. On success, sets + *pResult to the value. + + Returns ppCode. +*/ +int cmpp__args_evalToInt(cmpp_dx * dx, cmpp_args const *pArgs, + int * pResult); + +/** Passes the contents of arg through to cmpp__args_evalToInt(). */ +int cmpp__arg_evalSubToInt(cmpp_dx *dx, cmpp_arg const *arg, + int * pResult); + +/** + Evaluated arg as an integer/bool, placing the result in *pResult + and setting *pNext to the first argument to arg's right which this + routine did not consume. Non-0 on error and all that. +*/ +int cmpp__arg_toBool(cmpp_dx * dx, cmpp_arg const *arg, + int * pResult, cmpp_arg const **pNext); + +/** + If thisTtype==cmpp_TT_AnyType or thisTtype==arg->ttype and arg->z + looks like it might contain an at-string then os is re-used to hold + the @token@-expanded version of arg's content. os is unconditionally + passed to cmpp_b_reuse() before it begines work. + + It uses the given atPolicy to determine whether or not the content + is expanded, as per cmpp_dx_out_expand(). + + Returns 0 on success. If it expands content then *pExp is set to + os->z, else *pExp is set to arg->z. If nExp is not NULL then *nExp + gets set to the length of *pExp (geither os->n or arg->n). + + Returns ppCode. + + Much later: what does this give us that cmpp_arg_to_b() + doesn't? Oh - that one calls into this one. i.e. this one is + lower-level. +*/ +int cmpp__arg_expand_ats(cmpp_dx const * const dx, + cmpp_b * os, + cmpp_atpol_e atPolicy, + cmpp_arg const * const arg, + cmpp_tt thisTtype, + unsigned char const **pExp, + cmpp_size_t * nExp); + +typedef struct cmpp_argOp cmpp_argOp; +typedef void (*cmpp_argOp_f)(cmpp_dx *dx, + cmpp_argOp const *op, + cmpp_arg const *vLhs, + cmpp_arg const **pvRhs, + int *pResult); +struct cmpp_argOp { + int ttype; + /* 1 or 2 */ + unsigned char arity; + /* 0=none/left, 1=right (unary ops only) */ + signed char assoc; + cmpp_argOp_f xCall; +}; + +cmpp_argOp const * cmpp_argOp_for_tt(cmpp_tt tt); + + +bool cmpp__is_int(unsigned char const *z, unsigned n, + int *pOut); +bool cmpp__is_int64(unsigned char const *z, unsigned n, int64_t *pOut); + +char const * cmpp__atpol_name(cmpp *pp, cmpp_atpol_e p); +char const * cmpp__unpol_name(cmpp *pp, cmpp_unpol_e p); + +/** + Uncerimoniously bitwise-replaces pp's output channel with oNew. It + does _not_ clean up the previous channel, on the assumption that + the caller is taking any necessary measures. + + Apropos necessary measures for cleanup: if oPrev is not NULL, + *oPrev is set to a bitwise copy of the previous channel. + + Intended usage: + + ``` + cmpp_outputer oMine = cmpp_outputer_b; + cmpp_b bMine = cmpp_b_empty; + cmpp_outputer oOld = {0}; + oMine.state = &bMine; + cmpp_outputer_swap(pp, &myOut, &oOld); + ...do some work then ALWAYS do... + cmpp_outputer_swap(pp, &oOld, &oMine); + ``` + + Because this involves bitwise copying, care must be taken with + stream state, e.g. bMine.z (above) can be reallocated, so we have + to be sure to swap it back before using bMine again. +*/ +void cmpp__outputer_swap(cmpp *pp, cmpp_outputer const *oNew, + cmpp_outputer *oPrev); + +/** + Init code which is usually run as part of the ctor but may have to + be run later, after cmpp_reset(). We can't run it from cmpp_reset() + because that could leave post-reset in an error state, which is + icky. This call is a no-op after the first. +*/ +int cmpp__lazy_init(cmpp *pp); + +CMPP_NORETURN void cmpp__fatalv_base(char const *zFile, int line, + char const *zFmt, va_list); +#define cmpp__fatalv(...) cmpp__fatalv_base(__FILE__,__LINE__,__VA_ARGS__) +CMPP_NORETURN void cmpp__fatal_base(char const *zFile, int line, + char const *zFmt, ...); +#define cmpp__fatal(...) cmpp__fatal_base(__FILE__,__LINE__,__VA_ARGS__) + +/** + Outputs a printf()-formatted message to stderr. +*/ +void g_stderr(char const *zFmt, ...); +#define g_warn(zFmt,...) g_stderr("%s:%d %s() " zFmt "\n", __FILE__, __LINE__, __func__, __VA_ARGS__) +#define g_warn0(zMsg) g_stderr("%s:%d %s() %s\n", __FILE__, __LINE__, __func__, zMsg) +#if 0 +#define g_debug(PP,lvl,pfexpr) (void)0 +#else +#define g_debug(PP,lvl,pfexpr) \ + if(lvl<=(PP)->pimpl->flags.doDebug) { \ + if( (PP)->pimpl->dx ){ \ + g_stderr("%s:%" CMPP_SIZE_T_PFMT ": ", \ + (PP)->pimpl->dx->sourceName, \ + (PP)->pimpl->dx->pimpl->dline.lineNo); \ + } \ + g_stderr("%s():%d: ", \ + __func__,__LINE__); \ + g_stderr pfexpr; \ + } (void)0 +#endif + +/** Returns true if zFile is readable, else false. */ +bool cmpp__file_is_readable(char const *zFile); + +#define ustr_c(X) ((unsigned char const *)X) +#define ustr_nc(X) ((unsigned char *)X) +#define ppCode pp->pimpl->err.code +#define dxppCode dx->ppCode +#define cmpp__pi(PP) cmpp_pimpl * const pi = PP->pimpl +#define cmpp__dx_pi(DX) cmpp_dx_pimpl * const dpi = DX->pimpl +#define serr(...) cmpp_err_set(pp, CMPP_RC_SYNTAX, __VA_ARGS__) +#define dxserr(...) cmpp_err_set(dx->pp, CMPP_RC_SYNTAX, __VA_ARGS__) +#define cmpp__li_reserve1_size(li,nInitial) \ + (li->n ? (li->n==li->nAlloc ? li->nAlloc * 2 : li->n+1) : nInitial) + +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + +#endif /* include guard */ +/* +** 2022-11-12: +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +** +************************************************************************ +** This file houses the core of libcmpp (it used to house all of it, +** but the library grew beyond the confines of a single file). +** +** See the accompanying c-pp.h and README.md and/or c-pp.h for more +** details. +*/ +#include "sqlite3.h" + +char const * cmpp_version(void){ return CMPP_VERSION; } + +const cmpp__delim cmpp__delim_empty = cmpp__delim_empty_m; +const cmpp__delim_list cmpp__delim_list_empty = cmpp__delim_list_empty_m; +const cmpp_outputer cmpp_outputer_empty = cmpp_outputer_empty_m; +const cmpp_outputer cmpp_outputer_FILE = { + .out = cmpp_output_f_FILE, + .flush = cmpp_flush_f_FILE, + .cleanup = cmpp_outputer_cleanup_f_FILE, + .state = NULL +}; +const cmpp_b_list cmpp_b_list_empty = + cmpp_b_list_empty_m; +const cmpp_outputer cmpp_outputer_b = { + .out = cmpp_output_f_b, + .flush = 0, + .cleanup = cmpp_outputer_cleanup_f_b, + .state = NULL +}; + +/** + Default delimiters for @tokens@. +*/ +static const cmpp__delim delimAtDefault = { + .open = { .z = ustr_c("@"), .n = 1 }, + .close = { .z = ustr_c("@"), .n = 1 }, + .zOwns = NULL +}; + +static const cmpp_api_thunk cmppApiMethods = { +#define A(V) +#define V(N,T,V) .N = V, +#define F(N,T,P) .N = cmpp_ ##N, +#define O(N,T) .N = &cmpp_ ##N, + cmpp_api_thunk_map(A,V,F,O) +#undef F +#undef O +#undef V +#undef A +}; + +/* Fatally exits the app with the given printf-style message. */ + +CMPP__EXPORT(bool, cmpp_isspace)(int ch){ + return ' '==ch || '\t'==ch; +} + +//CMPP__EXPORT(int, cmpp_isnl)(char const * z, char const *zEnd){} +static inline int cmpp_isnl(unsigned char const * z, unsigned char const *zEnd){ + //assert(z<zEnd && "Caller-level pointer mis-traversal"); + switch( z<zEnd ? *z : 0 ){ + case 0: return 0; + case '\r': return ((z+1<zEnd) && '\n'==z[1]) ? 2 : 0; + default: return '\n'==*z; + } +} + +static inline bool cmpp_issnl(int ch){ + /* TODO: replace this in line with cmpp_isnl(), but it needs + a new interface for that. It's only used in two places and they + have different traversal directions, so we can probably + get rid of this and do the direct CRNL check in each of those + places. */ + return ' '==ch || '\t'==ch || '\n'==ch; +} + +CMPP__EXPORT(void, cmpp_skip_space)( + unsigned char const **p, + unsigned char const *zEnd +){ + assert( *p <= zEnd ); + unsigned char const * z = *p; + while( z<zEnd && cmpp_isspace(*z) ) ++z; + *p = z; +} + +CMPP__EXPORT(void, cmpp_skip_snl)( unsigned char const **p, + unsigned char const *zEnd ){ + unsigned char const * z = *p; + /* FIXME: CRNL. */ + while( z<zEnd && cmpp_issnl(*z) ) ++z; + *p = z; +} + +CMPP__EXPORT(void, cmpp_skip_space_trailing)( unsigned char const *zBegin, + unsigned char const **p ){ + assert( *p >= zBegin ); + unsigned char const * z = *p; + while( z>zBegin && cmpp_isspace(z[-1]) ) --z; + *p = z; +} + +CMPP__EXPORT(void, cmpp_skip_snl_trailing)( unsigned char const *zBegin, + unsigned char const **p ){ + assert( *p >= zBegin ); + unsigned char const * z = *p; + /* FIXME: CRNL. */ + while( z>zBegin && cmpp_issnl(*z) ) --z; + *p = z; +} + +/* Set pp's error state. */ +static int cmpp__errv(cmpp *pp, int rc, char const *zFmt, va_list); +/** + Sets pp's error state. +*/ +CMPP__EXPORT(int, cmpp_err_set)(cmpp *pp, int rc, char const *zFmt, ...); +#define cmpp__err cmpp_err_set +#define cmpp_dx_err cmpp_dx_err_set + +/* Open/close pp's output channel. */ +static int cmpp__out_fopen(cmpp *pp, const char *zName); +static void cmpp__out_close(cmpp *pp); + +#define CmppKvp_empty_m \ + {CmppSnippet_empty_m,CmppSnippet_empty_m,CmppKvp_op_none} +const CmppKvp CmppKvp_empty = CmppKvp_empty_m; + +/* Wrapper around a cmpp_FILE handle. Legacy stuff from when we just + supported cmpp_FILE input. */ +typedef struct FileWrapper FileWrapper; +struct FileWrapper { + /* File's name. */ + char const *zName; + /* cmpp_FILE handle. */ + cmpp_FILE * pFile; + /* Where FileWrapper_slurp() stores the file's contents. */ + unsigned char * zContent; + /* Size of this->zContent, as set by FileWrapper_slurp(). */ + cmpp_size_t nContent; +}; +#define FileWrapper_empty_m {0,0,0,0} +static const FileWrapper FileWrapper_empty = FileWrapper_empty_m; + +/** + Proxy for cmpp_fclose() and frees all memory owned by p. It is not + an error if p is already closed. +*/ +static void FileWrapper_close(FileWrapper * p); + +/** Proxy for cmpp_fopen(). Closes p first if it's currently opened. */ +static int FileWrapper_open(FileWrapper * p, const char * zName, const char *zMode); + +/* Populates p->zContent and p->nContent from p->pFile. */ +//static int FileWrapper_slurp(FileWrapper * p, int bCloseFile ); + +/** + If p->zContent ends in \n or \r\n, that part is replaced with 0 and + p->nContent is adjusted. Returns true if it chomps, else false. +*/ +static bool FileWrapper_chomp(FileWrapper * p); + +/* +** Outputs a printf()-formatted message to stderr. +*/ +static void g_stderrv(char const *zFmt, va_list); + +CMPP__EXPORT(char const *, cmpp_rc_cstr)(int rc){ + switch((cmpp_rc_e)rc){ +#define E(N,V,H) case N: return # N; + cmpp_rc_e_map(E) +#undef E + } + return NULL; +} + +CMPP__EXPORT(void, cmpp_mfree)(void *p){ + /* This MUST be a proxy for sqlite3_free() because allocate memory + exclusively using sqlite3_malloc() and friends. */ + sqlite3_free(p); +} + +CMPP__EXPORT(void *, cmpp_mrealloc)(void * p, size_t n){ + return sqlite3_realloc64(p, n); +} + +CMPP__EXPORT(void *, cmpp_malloc)(size_t n){ +#if 1 + return sqlite3_malloc64(n); +#else + void * p = sqlite3_malloc64(n); + if( p ) memset(p, 0, n); + return p; +#endif +} + +cmpp_FILE * cmpp_fopen(const char *zName, const char *zMode){ + cmpp_FILE *f; + if(zName && ('-'==*zName && !zName[1])){ + f = (strchr(zMode, 'w') || strchr(zMode,'+')) + ? stdout + : stdin + ; + }else{ + f = fopen(zName, zMode); + } + return f; +} + +void cmpp_fclose( cmpp_FILE * f ){ + if(f && (stdin!=f) && (stdout!=f) && (stderr!=f)){ + fclose(f); + } +} + +int cmpp_slurp(cmpp_input_f fIn, void *sIn, + unsigned char **pOut, cmpp_size_t * nOut){ + unsigned char zBuf[1024 * 16]; + unsigned char * pDest = 0; + unsigned nAlloc = 0; + unsigned nOff = 0; + int rc = 0; + cmpp_size_t nr = 0; + while( 0==rc ){ + nr = sizeof(zBuf); + if( (rc = fIn(sIn, zBuf, &nr)) ){ + break; + } + if(nr>0){ + if(nAlloc < nOff + nr + 1){ + nAlloc = nOff + nr + 1; + pDest = cmpp_mrealloc(pDest, nAlloc); + } + memcpy(pDest + nOff, zBuf, nr); + nOff += nr; + }else{ + break; + } + } + if( 0==rc ){ + if(pDest) pDest[nOff] = 0; + *pOut = pDest; + *nOut = nOff; + }else{ + cmpp_mfree(pDest); + } + return rc; +} + +void FileWrapper_close(FileWrapper * p){ + if(p->pFile) cmpp_fclose(p->pFile); + if(p->zContent) cmpp_mfree(p->zContent); + *p = FileWrapper_empty; +} + +int FileWrapper_open(FileWrapper * p, const char * zName, + const char * zMode){ + FileWrapper_close(p); + if( (p->pFile = cmpp_fopen(zName, zMode)) ){ + p->zName = zName; + return 0; + }else{ + return cmpp_errno_rc(errno, CMPP_RC_IO); + } +} + +int FileWrapper_slurp(FileWrapper * p, int bCloseFile){ + assert(!p->zContent); + assert(p->pFile); + int const rc = cmpp_slurp(cmpp_input_f_FILE, p->pFile, + &p->zContent, &p->nContent); + if( bCloseFile ){ + cmpp_fclose(p->pFile); + p->pFile = 0; + } + return rc; +} + +CMPP__EXPORT(bool, cmpp_chomp)(unsigned char * z, cmpp_size_t * n){ + if( *n && '\n'==z[*n-1] ){ + z[--*n] = 0; + if( *n && '\r'==z[*n-1] ){ + z[--*n] = 0; + } + return true; + } + return false; +} + +bool FileWrapper_chomp(FileWrapper * p){ + return cmpp_chomp(p->zContent, &p->nContent); +} + + +#if 0 +/** + Returns the number newline characters between the given starting + point and inclusive ending point. Results are undefined if zFrom is + greater than zTo. +*/ +static unsigned cmpp__count_lines(unsigned char const * zFrom, + unsigned char const *zTo); + +unsigned cmpp__count_lines(unsigned char const * zFrom, + unsigned char const *zTo){ + unsigned ln = 0; + assert(zFrom && zTo); + assert(zFrom <= zTo); + for(; zFrom < zTo; ++zFrom){ + if((unsigned char)'\n' == *zFrom) ++ln; + } + return ln; +} +#endif + +char const * cmpp__tt_cstr(int tt, bool bSymbolName){ + switch(tt){ +#define E(N,TOK) case cmpp_TT_ ## N: \ + return bSymbolName ? "cmpp_TT_" #N : TOK; + cmpp_tt_map(E) +#undef E + } + return NULL; +} + +char const * cmpp_tt_cstr(int tt){ + return cmpp__tt_cstr(tt, true); +} + +/** Flags and constants related to CmppLvl. */ +enum CmppLvl_e { + /** + Flag indicating that all ostensible output for a CmpLevel should + be elided. This also suppresses non-flow-control directives from + being processed. + */ + CmppLvl_F_ELIDE = 0x01, + /** + Mask of CmppLvl::flags which are inherited when + CmppLvl_push() is used. + */ + CmppLvl_F_INHERIT_MASK = CmppLvl_F_ELIDE +}; + +//static const CmppDLine CmppDLine_empty = CmppDLine_empty_m; + +/** Free all memory owned by li but does not free li. */ +static void CmppLvlList_cleanup(CmppLvlList *li); + +/** + Allocate a list entry, owned by li, and return it (cleanly zeroed + out). Returns NULL and updates pp->err on error. It is expected + that the caller will populate the entry's zName using + sqlite3_mprintf() or equivalent. +*/ +static CmppLvl * CmppLvlList_push(cmpp *pp, CmppLvlList *li); + +/** Returns the most-recently-appended element of li back to li's + free-list. It expects to receive that value as a sanity-checking + measure and may fail fatally of that's not upheld. */ +static void CmppLvlList_pop(cmpp *pp, CmppLvlList *li, CmppLvl * lvl); + +static const cmpp_dx_pimpl cmpp_dx_pimpl_empty = + cmpp_dx_pimpl_empty_m; + +#define cmpp_dx_empty_m { \ + .pp=0, \ + .d=0, \ + .sourceName=0, \ + .args={ \ + .z=0, .nz=0, \ + .argc=0, .arg0=0 \ + }, \ + .pimpl = 0 \ +} + +const cmpp_dx cmpp_dx_empty = cmpp_dx_empty_m; +#define cmpp_d_empty_m {{0,0},0,0,cmpp_d_impl_empty_m} +//static const cmpp_d cmpp_d_empty = cmpp_d_empty_m; + +static const CmppDList_entry CmppDList_entry_empty = + CmppDList_entry_empty_m; + +/** Free all memory owned by li but does not free li. */ +static void CmppDList_cleanup(CmppDList *li); +/** + Allocate a list entry, owned by li, and return it (cleanly zeroed + out). Returns NULL and updates pp->err on error. It is expected + that the caller will populate the entry's zName using + sqlite3_mprintf() or equivalent. +*/ +static CmppDList_entry * CmppDList_append(cmpp *pp, CmppDList *li); +/** Returns the most-recently-appended element of li back to li's + free-list. */ +static void CmppDList_unappend(CmppDList *li); +/** Resets li's list for re-use but does not free it. Returns li. */ +//static CmppDList * CmppDList_reuse(CmppDList *li); +static CmppDList_entry * CmppDList_search(CmppDList const * li, + char const *zName); + +/** Reset dx and free any memory it may own. */ +static void cmpp_dx_cleanup(cmpp_dx * const dx); +/** + Reset some of dx's parsing-related state in prep for fetching the + next line. +*/ +static void cmpp_dx__reset(cmpp_dx * const dx); + +/* Returns dx's current directive. */ +static inline cmpp_d const * cmpp_dx_d(cmpp_dx const * const dx){ + return dx->d; +} + +static const cmpp_pimpl cmpp_pimpl_empty = { + .db = { + .dbh = 0, + .zName = 0 + }, + .dx = 0, + .out = cmpp_outputer_empty_m, + .delim = { + .d = cmpp__delim_list_empty_m, + .at = cmpp__delim_list_empty_m + }, + .stmt = { +#define E(N,S) .N = 0, + CmppStmt_map(E) +#undef E + }, + .err = { + .code = 0, + .zMsg = 0, + .zMsgC = 0 + }, + .sqlTrace = { + .expandSql = false, + .counter = 0, + .out = cmpp_outputer_empty_m + }, + .flags = { + .allocStamp = 0, + .nIncludeDir = 0, + .nDxDepth = 0, + .nSavepoint = 0, + .doDebug = 0, + .chompF = 0, + .newFlags = 0, + .isInternalDirectiveReg = false, + .nextIsCall = false, + .needsLazyInit = true + }, + .policy = { + .at = {0,0,0}, + .un = {0,0,0} + }, + .d = { + .list = CmppDList_empty_m, + .autoload = cmpp_d_autoloader_empty_m + }, + .mod = { + .sohList = CmppSohList_empty_m, + .path = cmpp_b_empty_m, + .soExt = CMPP_PLATFORM_EXT_DLL, + /* Yes, '*'. It makes sense in context. */ + .pathSep = '*' + // 0x1e /* "record separator" */ doesn't work. Must be non-ctrl. + }, + .recycler = { + .buf = cmpp_b_list_empty_m, + .bufSort = cmpp_b_list_UNSORTED, + .argPimpl = NULL + } +}; + +#if 0 +static inline int cmpp__out(cmpp *pp, void const *z, cmpp_size_t n){ + return cmpp__out2(pp, &pp->out, z, n); +} +#endif + +/** + Returns an approximate cmpp_tt for the given SQLITE_... value from + sqlite3_column_type() or sqlite3_value_type(). +*/ +static cmpp_tt cmpp__tt_for_sqlite(int sqType); + +/** + Init code which is usually run as part of the ctor but may have to + be run later, after cmpp_reset(). We can't run it from cmpp_reset() + because that could leave post-reset in an error state, which is + icky. +*/ +int cmpp__lazy_init(cmpp *pp){ + if( !ppCode && pp->pimpl->flags.needsLazyInit ){ + pp->pimpl->flags.needsLazyInit = false; + cmpp__delim_list * li = &pp->pimpl->delim.d; + if( !li->n ) cmpp_delimiter_push(pp, NULL); + li = &pp->pimpl->delim.at; + if( !li->n ) cmpp_atdelim_push(pp, NULL, NULL); +#if defined(CMPP_CTOR_INSTANCE_INIT) + if( !ppCode ){ + extern int CMPP_CTOR_INSTANCE_INIT(cmpp*); + int const rc = CMPP_CTOR_INSTANCE_INIT(pp); + if( rc && !ppCode ){ + cmpp__err(pp, rc, + "Initialization via CMPP_CTOR_INSTANCE_INIT() failed " + "with code %d/%s.", rc, cmpp_rc_cstr(rc) ); + } + } +#endif + } + return ppCode; +} + +static void cmpp__wipe_policies(cmpp *pp){ + if( 0==ppCode ){ + PodList__atpol_reserve(pp, &cmpp__epol(pp,at), 0); + PodList__unpol_reserve(pp, &cmpp__epol(pp,un), 0); + if( 0==ppCode ){ + PodList__atpol_wipe(&cmpp__epol(pp,at), cmpp_atpol_DEFAULT); + PodList__unpol_wipe(&cmpp__epol(pp,un), cmpp_unpol_DEFAULT); + } + } +} + +CMPP__EXPORT(int, cmpp_ctor)(cmpp **pOut, cmpp_ctor_cfg const * cfg){ + cmpp_pimpl * pi = 0; + cmpp * pp = 0; + void * const mv = cmpp_malloc(sizeof(cmpp) + sizeof(*pi)); + if( mv ){ + if( !cfg ){ + static const cmpp_ctor_cfg dfltCfg = {0}; + cfg = &dfltCfg; + } + cmpp const x = { + .api = &cmppApiMethods, + .pimpl = (cmpp_pimpl*)((unsigned char *)mv + sizeof(cmpp)) + /* ^^^ (T const * const) members */ + }; + memcpy(mv, &x, sizeof(x)) + /* FWIW, i'm convinced that this is a legal way to transfer + these const-pointers-to-const. If not, we'll need to change + those cmpp members from (T const * const) to (T const *). */; + pp = mv; + assert(pp->api == &cmppApiMethods); + assert(pp->pimpl); + pi = pp->pimpl; + *pOut = pp; + *pi = cmpp_pimpl_empty; + assert( pi->flags.needsLazyInit ); + pi->flags.newFlags = cfg->flags; + pi->flags.allocStamp = &cmpp_pimpl_empty; + if( cfg->dbFile ){ + pi->db.zName = sqlite3_mprintf("%s", cfg->dbFile); + cmpp_check_oom(pp, pi->db.zName); + } + if( 0==ppCode ){ + cmpp__wipe_policies(pp); + cmpp__lazy_init(pp); + } + } + return pp ? ppCode : CMPP_RC_OOM; +} + +CMPP__EXPORT(void, cmpp_reset)(cmpp *pp){ + cmpp__pi(pp); + cmpp_outputer_cleanup(&pi->sqlTrace.out); + pi->sqlTrace.out = cmpp_outputer_empty; + if( pi->d.autoload.dtor ){ + pi->d.autoload.dtor(pi->d.autoload.state); + } + pi->d.autoload = cmpp_pimpl_empty.d.autoload; + cmpp_b_clear(&pi->mod.path); + if( pi->stmt.spRelease && pi->stmt.spRollback ){ + /* Cleanly kill all savepoint levels. This is truly superfluous, + as they'll all be rolled back (if the db is persistent) or + nuked (if using a :memory: db) momentarily. However, we'll + eventually need this for a partial-clear operation which leaves + the db and custom directives intact. For now it lives here but + will eventually move to wherever that ends up being. + + 2025-11-16: or not. It's fine here, really. + */ + sqlite3_reset(pi->stmt.spRelease); + while( SQLITE_DONE==sqlite3_step(pi->stmt.spRelease) ){ + sqlite3_reset(pi->stmt.spRollback); + sqlite3_step(pi->stmt.spRollback); + sqlite3_reset(pi->stmt.spRelease); + } + } + cmpp__out_close(pp); + CmppDList_cleanup(&pi->d.list); +#define E(N,S) \ + if(pi->stmt.N) {sqlite3_finalize(pi->stmt.N); pi->stmt.N = 0;} + CmppStmt_map(E) (void)0; +#undef E + if( pi->db.dbh ){ + if( SQLITE_TXN_WRITE==sqlite3_txn_state(pi->db.dbh, NULL) ){ + sqlite3_exec(pi->db.dbh, "COMMIT;", 0, 0, NULL) + /* ignoring error */; + } + sqlite3_close(pi->db.dbh); + pi->db.dbh = 0; + } + cmpp__delim_list_reuse(&pi->delim.d); + cmpp__delim_list_reuse(&pi->delim.at); + //why? cmpp_b_list_reuse(&pi->cache.buf); + cmpp__err_clear(pp); + {/* Zero out pi but save some pieces for later, when pp is + cmpp_dtor()'d */ + cmpp_pimpl const tmp = *pi; + *pi = cmpp_pimpl_empty; + pi->db = tmp.db /* restore db.zName */; + pi->recycler = tmp.recycler; + pi->policy = tmp.policy; + pi->delim = tmp.delim; + pi->mod.sohList = tmp.mod.sohList; + cmpp__wipe_policies(pp); + pi->flags.allocStamp = tmp.flags.allocStamp; + pi->flags.newFlags = tmp.flags.newFlags; + pi->flags.needsLazyInit = true; + } +} + +static void cmpp__delim_list_cleanup(cmpp__delim_list *li); + +CMPP__EXPORT(void, cmpp_dtor)(cmpp *pp){ + if( pp ){ + cmpp__pi(pp); + cmpp_reset(pp); + cmpp_mfree(pi->db.zName); + PodList__atpol_finalize(&cmpp__epol(pp,at)); + assert(!cmpp__epol(pp,at).na); + PodList__unpol_finalize(&cmpp__epol(pp,un)); + assert(!cmpp__epol(pp,un).na); + cmpp_b_list_cleanup(&pi->recycler.buf); + cmpp__delim_list_cleanup(&pi->delim.d); + cmpp__delim_list_cleanup(&pi->delim.at); + for( cmpp_args_pimpl * apNext = 0, + * ap = pi->recycler.argPimpl; + ap; ap = apNext ){ + apNext = ap->nextFree; + ap->nextFree = 0; + cmpp_args_pimpl_cleanup(ap); + cmpp_mfree(ap); + } + CmppSohList_close(&pi->mod.sohList); + if( &cmpp_pimpl_empty==pi->flags.allocStamp ){ + pi->flags.allocStamp = 0; + cmpp_mfree(pp); + } + } +} + +CMPP__EXPORT(bool, cmpp_is_safemode)(cmpp const * pp){ + return pp ? 0!=(cmpp_ctor_F_SAFEMODE & pp->pimpl->flags.newFlags) : false; +} + +/** Sets ppCode if m is NULL. Returns ppCode. */ +CMPP__EXPORT(int, cmpp_check_oom)(cmpp * const pp, void const * const m ){ + int rc; + if( pp ){ + if( !m ){ + //assert(!"oom"); + cmpp__err(pp, CMPP_RC_OOM, 0); + } + rc = ppCode; + }else{ + rc = m ? 0 : CMPP_RC_OOM; + } + return rc; +} + +//CxMPP_WASM_EXPORT +void *cmpp__malloc(cmpp *pp, cmpp_size_t n){ + void *p = 0; + if( 0==ppCode ){ + p = cmpp_malloc(n); + cmpp_check_oom(pp, p); + } + return p; +} + +/** + If ppCode is not 0 then it flushes pp's output channel. If that + fails, it sets ppCode. Returns ppCode. +*/ +static int cmpp__flush(cmpp *pp){ + if( !ppCode && pp->pimpl->out.flush ){ + int const rc = pp->pimpl->out.flush(pp->pimpl->out.state); + if( rc && !ppCode ){ + cmpp_err_set(pp, rc, "Flush failed."); + } + } + return ppCode; +} + +void cmpp__out_close(cmpp *pp){ + cmpp__flush(pp)/*ignoring result*/; + cmpp_outputer_cleanup(&pp->pimpl->out); + pp->pimpl->out = cmpp_pimpl_empty.out; +} + +int cmpp__out_fopen(cmpp *pp, const char *zName){ + cmpp__out_close(pp); + if( !ppCode ){ + cmpp_FILE * const f = cmpp_fopen(zName, "wb"); + if( f ){ + ppCode = 0; + pp->pimpl->out = cmpp_outputer_FILE; + pp->pimpl->out.state = f; + pp->pimpl->out.name = zName; + }else{ + ppCode = cmpp__err( + pp, cmpp_errno_rc(errno, CMPP_RC_IO), + "Error opening file %s", zName + ); + } + } + return ppCode; +} + +static int cmpp__FileWrapper_open(cmpp *pp, FileWrapper * fw, + const char * zName, + const char * zMode){ + int const rc = FileWrapper_open(fw, zName, zMode); + if( rc ){ + cmpp__err(pp, rc, "Error %s opening file [%s] " + "with mode [%s]", + cmpp_rc_cstr(rc), zName, zMode); + } + return ppCode; +} + +static int cmpp__FileWrapper_slurp(cmpp* pp, FileWrapper * fw){ + assert( fw->pFile ); + int const rc = FileWrapper_slurp(fw, 1); + if( rc ){ + cmpp__err(pp, rc, "Error %s slurping file %s", + cmpp_rc_cstr(rc), fw->zName); + } + return ppCode; +} + +void g_stderrv(char const *zFmt, va_list va){ + vfprintf(0 ? stdout : stderr, zFmt, va); +} + +void g_stderr(char const *zFmt, ...){ + va_list va; + va_start(va, zFmt); + g_stderrv(zFmt, va); + va_end(va); +} + +CMPP__EXPORT(char const *, cmpp_dx_delim)(cmpp_dx const *dx){ + return (char const *)cmpp__dx_zdelim(dx); +} + +int cmpp__out2(cmpp *pp, cmpp_outputer *pOut, + void const *z, cmpp_size_t n){ + assert( pOut ); + if( !ppCode && pOut->out && n ){ + int const rc = pOut->out(pOut->state, z, n); + if( rc ){ + cmpp__err(pp, rc, + "Write of %" CMPP_SIZE_T_PFMT + " bytes to output stream failed.", n); + } + } + return ppCode; +} + +CMPP__EXPORT(int, cmpp_dx_out_raw)(cmpp_dx * dx, void const *z, cmpp_size_t n){ + if( dxppCode || cmpp_dx_is_eliding(dx) ) return dxppCode; + return cmpp__out2(dx->pp, &dx->pp->pimpl->out, z, n); +} + +CMPP__EXPORT(int, cmpp_outfv2)(cmpp *pp, cmpp_outputer *out, char const *zFmt, va_list va){ + assert( out ); + if( !ppCode && zFmt && *zFmt && out->out ){ + char * s = sqlite3_vmprintf(zFmt, va); + if( 0==cmpp_check_oom(pp, s) ){ + cmpp__out2(pp, out, s, cmpp__strlen(s, -1)); + } + cmpp_mfree(s); + } + return ppCode; +} + +CMPP__EXPORT(int, cmpp_outf2)(cmpp *pp, cmpp_outputer *out, char const *zFmt, ...){ + assert( out ); + if( !ppCode && zFmt && *zFmt && out->out ){ + va_list va; + va_start(va, zFmt); + cmpp_outfv2(pp, out, zFmt, va); + va_end(va); + } + return ppCode; +} + +CMPP__EXPORT(int, cmpp_outfv)(cmpp *pp, char const *zFmt, va_list va){ + return cmpp_outfv2(pp, &pp->pimpl->out, zFmt, va); +} + +CMPP__EXPORT(int, cmpp_outf)(cmpp *pp, char const *zFmt, ...){ + if( !ppCode ){ + va_list va; + va_start(va, zFmt); + cmpp_outfv2(pp, &pp->pimpl->out, zFmt, va); + va_end(va); + } + return ppCode; +} + +CMPP__EXPORT(int, cmpp_dx_outf)(cmpp_dx *dx, char const *zFmt, ...){ + if( !dxppCode && zFmt && *zFmt && dx->pp->pimpl->out.out ){ + va_list va; + va_start(va, zFmt); + cmpp_outfv(dx->pp, zFmt, va); + va_end(va); + } + return dxppCode; +} + +static int cmpp__affirm_undef_policy(cmpp *pp, + unsigned char const *zName, + cmpp_size_t nName){ + if( 0==ppCode + && cmpp_unpol_ERROR==cmpp__policy(pp,un) ){ + cmpp__err(pp, CMPP_RC_NOT_DEFINED, + "Key '%.*s' was not found and the undefined-value " + "policy is 'error'.", + (int)nName, zName); + } + return ppCode; +} + +static int cmpp__out_expand(cmpp * pp, cmpp_outputer * pOut, + unsigned char const * zFrom, + cmpp_size_t n, cmpp_atpol_e atPolicy){ + enum state_e { + /* looking for @token@ opening @ */ + state_opening, + /* looking for @token@ closing @ */ + state_closing + }; + cmpp__pi(pp); + if( ppCode ) return ppCode; + if( cmpp_atpol_CURRENT==atPolicy ) atPolicy = cmpp__policy(pp,at); + assert( cmpp_atpol_invalid!=atPolicy ); + unsigned char const *zLeft = zFrom; + unsigned char const * const zEnd = zFrom + n; + unsigned char const *z = + (cmpp_atpol_OFF==atPolicy || cmpp_atpol_invalid==atPolicy) + ? zEnd + : zLeft; + unsigned char const chEol = (unsigned char)'\n'; + cmpp__delim const * delim = + cmpp__delim_list_get(&pp->pimpl->delim.at); + if( !delim && z<zEnd ){ + return cmpp_err_set(pp, CMPP_RC_ERROR, + "@token@ delimiter stack is empty."); + } + enum state_e state = state_opening; + cmpp_b * const bCall = cmpp_b_borrow(pp); + cmpp_b * const bVal = cmpp_b_borrow(pp); + if( !bCall || !bVal ) return ppCode; + if(0) g_warn("looking to expand from %d bytes: [%.*s]", (int)n, + (int)n,zLeft); + if( !pOut ){ + if( 0 && atPolicy!=cmpp__policy(pp,at) ){ + /* This might be too strict. It was initially in place to ensure + that we did not _accidentally_ do @token@ parsing to the main + output channel. We frequently use it internally on other + output channels to buffer/build results. + + Advantage to removing this check: #query could impose + its own at-policy without having to use an intermediary + buffer. + + Disadvantage: less protection against accidentally + @-filtering the output when we shouldn't. + */ + return cmpp_err_set(pp, CMPP_RC_MISUSE, + "%s(): when sending to the default " + "output channel, the @policy must be " + "cmpp_atpol_CURRENT.", __func__); + } + pOut = &pi->out; + } + assert( pi->dx ? !cmpp_dx_is_eliding(pi->dx) : 1 ); + +#define tflush \ + if(z>zEnd) z=zEnd; \ + if(zLeft<z){ \ + if(0) g_warn("flush %d [%.*s]", (int)(z-zLeft), (int)(z-zLeft), zLeft); \ + cmpp__out2(pp, pOut, zLeft, (z-zLeft)); \ + } zLeft = z + cmpp_dx_pimpl * const dxp = pp->pimpl->dx ? pp->pimpl->dx->pimpl : NULL; + for( ; z<zEnd && 0==ppCode; ++z ){ + zLeft = z; + for( ;z<zEnd && 0==ppCode; ++z ){ + again: + if( chEol==*z ){ +#if 0 + broken; + if( dxp && dxp->flags.countLines ){ + ++dxp->lineNo; + } +#endif + state = state_opening; + continue; + } + if( state_opening==state ){ + if( z + delim->open.n < zEnd + && 0==memcmp(z, delim->open.z, delim->open.n) ){ + tflush; + z += delim->open.n; + if( 0 ) g_warn("zLeft..z=[%.*s]", (int)(z-zLeft), zLeft); + if( 0 ){ + g_warn("\nzLeft..=[%s]\nz=[%s]", zLeft, z); + } + state = state_closing; +#if 1 + /* Handle call of @[directive ...args]@ + + i'm not a huge fan of this syntax, but that may go away + if we replace the single-char separator with a pair of + opening/closing delimiters. + */ + if( z<zEnd && '['==*z ){ + unsigned char const * zb = z; + cmpp_size_t nl = 0; + //g_warn("Scanning: <<%.*s>>", (int)(zEnd-zb), zb); + if( cmpp__find_closing2(pp, &zb, zEnd, &nl) ){ + break; + } + //g_warn("Found: <<%.*s>>", (int)(zb+1-z), z); + if( zb + delim->close.n >= zEnd + || 0!=memcmp(zb+1, delim->close.z, delim->close.n) ){ + serr("Expecting '%s' after closing ']'.", delim->close.z); + break; + } + if( nl && dxp && dxp->flags.countLines ){ + dxp->pos.lineNo +=nl; + } + cmpp_call_str(pp, z+delim->open.n, + (zb - z - delim->open.n), + cmpp_b_reuse(bCall), 0); + if( 0==ppCode ){ + cmpp__out2(pp, pOut, bCall->z, bCall->n); + state = state_opening; + zLeft = z = zb + delim->close.n + 1; + //g_warn("post-@[]@ z=%.*s", (int)(zEnd-z), z); + } + } +#endif + if( z>=zEnd ) break; + goto again /* avoid adjusting z again */; + } + }else{/*we're looking for delim->closer*/ + assert( state_closing==state ); + if( z + delim->close.n <= zEnd + && 0==memcmp(z, delim->close.z, delim->close.n ) ){ + /* process the ... part of @...@ */ + assert( state_closing==state ); + assert( zLeft<z ); + assert( z<=zEnd ); + if( 0 ) g_warn("zLeft..z=[%.*s]", (int)(z-zLeft), zLeft); + if( 0 ) g_warn("zLeft..=%s", zLeft); + assert( 0==memcmp(zLeft, delim->open.z, delim->open.n) ); + unsigned char const *zKey = + zLeft + delim->open.n; + cmpp_ssize_t const nKey = z - zLeft - delim->open.n; + if( 0 ) g_warn("nKey=%d zKey=[%.*s]", nKey, nKey, zKey); + assert( nKey>= 0 ); + if( !nKey ){ + serr("Empty key is not permitted in %s...%s.", + delim->open.z, delim->close.z); + break; + } + if( cmpp__get_b(pp, zKey, nKey, cmpp_b_reuse(bVal), true) ){ + if(0){ + g_warn("nVal=%d zVal=[%.*s]", (int)bVal->n, + (int)bVal->n, bVal->z); + } + if( bVal->n ){ + cmpp__out2(pp, pOut, bVal->z, bVal->n); + }else{ + /* Elide it */ + } + zLeft = z + delim->close.n; + assert( zLeft<=zEnd ); + }else if( !ppCode ){ + assert( !bVal->n ); + /* No matching define . */ + switch( atPolicy ){ + case cmpp_atpol_ELIDE: zLeft = z + delim->close.n; break; + case cmpp_atpol_RETAIN: tflush; break; + case cmpp_atpol_ERROR: + cmpp__err(pp, CMPP_RC_NOT_DEFINED, + "Undefined %skey%s: %.*s", + delim->open.z, delim->close.z, nKey, zKey); + break; + case cmpp_atpol_invalid: + case cmpp_atpol_CURRENT: + case cmpp_atpol_OFF: + assert(!"this shouldn't be reachable" ); + cmpp__err(pp, CMPP_RC_ERROR, "Unhandled atPolicy #%d", + atPolicy); + break; + } + }/* process @...@ */ + state = state_opening; + assert( z<=zEnd ); + }/*matched a closer*/ + }/*state_closer==state*/ + assert( z<=zEnd ); + }/*per-line loop*/ + }/*outer loop*/ +#if 0 + if( 0==ppCode && state_closer==state ){ + serr("Opening '%s' found without a closing '%s'.", + delim->open.z, delim->close.z); + } +#endif + tflush; +#undef tflush + cmpp_b_return(pp, bCall); + cmpp_b_return(pp, bVal); + return ppCode; +} + +CMPP__EXPORT(int, cmpp_dx_out_expand)(cmpp_dx const * const dx, + cmpp_outputer * pOut, + unsigned char const * zFrom, + cmpp_size_t n, + cmpp_atpol_e atPolicy){ + if( dxppCode || cmpp_dx_is_eliding(dx) ) return dxppCode; + return cmpp__out_expand(dx->pp, pOut, zFrom, n, atPolicy); +} + +CmppLvl * CmppLvl_get(cmpp_dx const *dx){ + return dx->pimpl->dxLvl.n + ? dx->pimpl->dxLvl.list[dx->pimpl->dxLvl.n-1] + : 0; +} + +static const CmppLvl CmppLvl_empty = CmppLvl_empty_m; + +CmppLvl * CmppLvl_push(cmpp_dx *dx){ + CmppLvl * p = 0; + if( !dxppCode ){ + CmppLvl * const pPrev = CmppLvl_get(dx); + p = CmppLvlList_push(dx->pp, &dx->pimpl->dxLvl); + if( p ){ + *p = CmppLvl_empty; + p->lineNo = dx->pimpl->dline.lineNo; + //p->d = dx->d; + if( pPrev ){ + p->flags = (CmppLvl_F_INHERIT_MASK & pPrev->flags); + //if(CLvl_isSkip(pPrev)) p->flags |= CmppLvl_F_ELIDE; + } + } + } + return p; +} + +void CmppLvl_pop(cmpp_dx *dx, CmppLvl * lvl){ + CmppLvlList_pop(dx->pp, &dx->pimpl->dxLvl, lvl); +} + +void CmppLvl_elide(CmppLvl *lvl, bool on){ + if( on ) lvl->flags |= CmppLvl_F_ELIDE; + else lvl->flags &= ~CmppLvl_F_ELIDE; +} + +bool CmppLvl_is_eliding(CmppLvl const *lvl){ + return lvl && !!(lvl->flags & CmppLvl_F_ELIDE); +} + +#if 0 +void cmpp_dx_elide_mode(cmpp_dx *dx, bool on){ + CmppLvl_elide(CmppLvl_get(dx), on); +} +#endif + +bool cmpp_dx_is_eliding(cmpp_dx const *dx){ + return CmppLvl_is_eliding(CmppLvl_get(dx)); +} + + +char * cmpp_str_finish(cmpp *pp, sqlite3_str *s, int * n){ + char * z = 0; + int const rc = sqlite3_str_errcode(s); + cmpp__db_rc(pp, rc, "sqlite3_str_errcode()"); + if(0==rc){ + int const nStr = sqlite3_str_length(s); + if(n) *n = nStr; + z = sqlite3_str_finish(s); + if( !z ){ + assert( 0==nStr && "else rc!=0" ); + } + }else{ + cmpp_mfree( sqlite3_str_finish(s) ); + } + return z; +} + +int cmpp__bind_int(cmpp *pp, sqlite3_stmt *pStmt, int col, int64_t val){ + return ppCode + ? ppCode + : cmpp__db_rc(pp, sqlite3_bind_int64(pStmt, col, val), + "from cmpp__bind_int()"); +} + +int cmpp__bind_int_text(cmpp *pp, sqlite3_stmt *pStmt, int col, + int64_t val){ + unsigned char buf[32]; + snprintf((char *)buf, sizeof(buf), "%" PRIi64, val); + return cmpp__bind_textn(pp, pStmt, col, buf, -1); +} + +int cmpp__bind_null(cmpp *pp, sqlite3_stmt *pStmt, int col){ + return ppCode + ? ppCode + : cmpp__db_rc(pp, sqlite3_bind_null(pStmt, col), + "from cmpp__bind_null()"); +} + +static int cmpp__bind_textx(cmpp *pp, sqlite3_stmt *pStmt, int col, + unsigned const char * zStr, cmpp_ssize_t n, + void (*dtor)(void *)){ + if( 0==ppCode ){ + cmpp__db_rc( + pp, (zStr && n) + ? sqlite3_bind_text(pStmt, col, + (char const *)zStr, + (int)n, dtor) + : sqlite3_bind_null(pStmt, col), + sqlite3_sql(pStmt) + ); + } + return ppCode; +} + +int cmpp__bind_textn(cmpp *pp, sqlite3_stmt *pStmt, int col, + unsigned const char * zStr, cmpp_ssize_t n){ + return cmpp__bind_textx(pp, pStmt, col, zStr, (int)n, + SQLITE_TRANSIENT); +} + +int cmpp__bind_text(cmpp *pp, sqlite3_stmt *pStmt, int col, + unsigned const char * zStr){ + return cmpp__bind_textn(pp, pStmt, col, zStr, -1); +} + +#if 0 +int cmpp__bind_textv(cmpp*pp, sqlite3_stmt *pStmt, int col, + const char * zFmt, ...){ + if( 0==p->err.code ){ + int rc; + sqlite3_str * str = sqlite3_str_new(pp->pimpl->db.dbh); + int n = 0; + char * z; + va_list va; + if( !str ) return ppCode; + va_start(va,zFmt); + sqlite3_str_vappendf(str, zFmt, va); + va_end(va); + z = cmpp_str_finish(str, &n); + cmpp__db_rc( + pp, z + ? sqlite3_bind_text(pStmt, col, z, n, sqlite3_free) + : sqlite3_bind_null(pStmt, col), + sqlite3_sql(pStmt) + ); + cmpp_mfree(z); + } + return p->err.code; +} +#endif + +void cmpp_outputer_set(cmpp *pp, cmpp_outputer const *out, + char const *zName){ + cmpp__pi(pp); + cmpp_outputer_cleanup(&pi->out); + if( out ) pi->out = *out; + else pi->out = cmpp_outputer_empty; + pi->out.name = zName; +} + +void cmpp__outputer_swap(cmpp *pp, cmpp_outputer const *oNew, + cmpp_outputer *oPrev){ + if( oPrev ){ + *oPrev = pp->pimpl->out; + } + pp->pimpl->out = *oNew; +} + +#if 0 +static void delim__list_dump(cmpp const *pp){ + cmpp__delim_list const *li = &pp->pimpl->delim.d; + if( li->n ){ + g_warn0("delimiter stack:"); + for(cmpp_size_t i = 0; i < li->n; ++i ){ + g_warn("#%d: %s", (int)i, li->list[i].z); + } + } + +} +#endif + +static bool cmpp__valid_delim(cmpp * const pp, + char const *z, + char const *zEnd){ + char const * const zB = z; + for( ; z < zEnd; ++z ){ + if( *z<33 || 127==*z ){ + cmpp_err_set(pp, CMPP_RC_SYNTAX, + "Delimiters may not contain " + "control characters."); + return false; + } + } + if( zB==z ){ + cmpp_err_set(pp, CMPP_RC_SYNTAX, + "Delimiters may not be empty."); + } + return z>zB; +} + +CMPP__EXPORT(int, cmpp_delimiter_set)(cmpp *pp, char const *zDelim){ + if( ppCode ) return ppCode; + unsigned n; + if( zDelim ){ + n = cmpp__strlen(zDelim, -1); + if( !cmpp__valid_delim(pp, zDelim, zDelim+n) ){ + return ppCode; + }else if( n>12 /* arbitrary but seems sensible enough */ ){ + return cmpp__err(pp, CMPP_RC_MISUSE, + "Invalid delimiter (too long): %s", zDelim); + } + } + cmpp__pi(pp); + if( pi->delim.d.n ){ + cmpp__delim * const delim = cmpp__pp_delim(pp); + if( !cmpp_check_oom(pp, delim) ){ + cmpp__delim_cleanup(delim); + if( zDelim ){ + delim->open.n = n; + delim->open.z = delim->zOwns = + (unsigned char*)sqlite3_mprintf("%.*s", n, zDelim); + cmpp_check_oom(pp, delim->zOwns); + }else{ + assert( delim->open.z ); + assert( !delim->zOwns ); + assert( delim->open.n==sizeof(CMPP_DEFAULT_DELIM)-1 ); + } + } + }else{ + assert(!"Cannot set delimiter on an empty stack!"); + cmpp_err_set(pp, CMPP_RC_MISUSE, + "Directive delimter stack is empty."); + } + return ppCode; +} + +CMPP__EXPORT(void, cmpp_delimiter_get)(cmpp const *pp, char const **zDelim){ + cmpp__delim const * d = cmpp__pp_delim(pp); + if( !d ) d = &cmpp__delim_empty; + *zDelim = (char const *)d->open.z; +} + +CMPP__EXPORT(int, cmpp_delimiter_push)(cmpp *pp, char const *zDelim){ + cmpp__delim * const d = + cmpp__delim_list_push(pp, &pp->pimpl->delim.d); + if( d && cmpp_delimiter_set(pp, zDelim) ){ + cmpp__delim_list_pop(&pp->pimpl->delim.d); + } + return ppCode; +} + +CMPP__EXPORT(int, cmpp_delimiter_pop)(cmpp *pp){ + cmpp__delim_list * const li = &pp->pimpl->delim.d; + if( li->n ){ + //g_warn("Popping delimiter: %s", cmpp__pp_zdelim(pp)); + cmpp__delim_list_pop(li); + if( 0 && li->n ){ + g_warn("restored delimiter: %s", cmpp__pp_zdelim(pp)); + } + }else if( !ppCode ){ + assert(!"Attempt to pop an empty delimiter stack."); + cmpp_err_set(pp, CMPP_RC_MISUSE, + "Cannot pop an empty delimiter stack."); + } + return ppCode; +} + +CMPP__EXPORT(int, cmpp_atdelim_set)(cmpp * const pp, + char const *zOpen, + char const *zClose){ + if( 0==ppCode ){ + cmpp__pi(pp); + cmpp__delim * const d = pi->delim.at.n + ? &pi->delim.at.list[pi->delim.at.n-1] + : NULL; + assert( d ); + if( !d ){ + return cmpp__err(pp, CMPP_RC_MISUSE, + "@token@ delimiter stack is currently empty."); + } + if( 0==zOpen ){ + zOpen = (char const *)delimAtDefault.open.z; + zClose = (char const *)delimAtDefault.close.z; + }else if( 0==zClose ){ + zClose = zOpen; + } + cmpp_size_t const nO = cmpp__strlen(zOpen, -1); + cmpp_size_t const nC = cmpp__strlen(zClose, -1); + assert( zOpen && zClose ); + if( !cmpp__valid_delim(pp, zOpen, zOpen+nO) + || !cmpp__valid_delim(pp, zClose, zClose+nC) ){ + return ppCode; + } + cmpp_b b = cmpp_b_empty + /* Don't use cmpp_b_borrow() here because we'll unconditionally + transfer ownership of b.z to d. */; + if( 0==cmpp_b_reserve3(pp, &b, nO + nC + 2) ){ +#ifndef NDEBUG + unsigned char const * const zReallocCheck = b.z; +#endif + /* Copy the open/close tokens to a single string to simplify + management. */ + cmpp_b_append4(pp, &b, zOpen, nO); + cmpp_b_append_ch(&b, '\0'); + cmpp_b_append4(pp, &b, zClose, nC); + assert( zReallocCheck==b.z + && "Else buffer was not properly pre-sized" ); + cmpp__delim_cleanup(d); + d->open.z = b.z; + d->open.n = nO; + d->close.z = d->open.z + nO + 1/*NUL*/; + d->close.n = nC; + d->zOwns = b.z; + b = cmpp_b_empty /* transfer memory ownership */; + } + } + return ppCode; +} + +CMPP__EXPORT(int, cmpp_atdelim_push)(cmpp *pp, char const *zOpen, + char const *zClose){ + cmpp__delim * const d = + cmpp__delim_list_push(pp, &pp->pimpl->delim.at); + if( d && cmpp_atdelim_set(pp, zOpen, zClose) ){ + cmpp__delim_list_pop(&pp->pimpl->delim.at); + } + return ppCode; +} + +CMPP__EXPORT(int, cmpp_atdelim_pop)(cmpp *pp){ + cmpp__delim_list * const li = &pp->pimpl->delim.at; + if( li->n ){ + //g_warn("Popping delimiter: %s", cmpp__pp_zdelim(pp)); + cmpp__delim_list_pop(li); + }else if( !ppCode ){ + assert(!"Attempt to pop an empty @token@ delim stack."); + cmpp_err_set(pp, CMPP_RC_MISUSE, + "Cannot pop an empty @token@ delimiter stack."); + } + return ppCode; +} + +CMPP__EXPORT(void, cmpp_atdelim_get)(cmpp const * const pp, + char const **zOpen, + char const **zClose){ + cmpp__delim const * d + = cmpp__delim_list_get(&pp->pimpl->delim.at); + assert( d ); + if( !d ) d = &delimAtDefault; + if( zClose ) *zClose = (char const *)d->close.z; + if( zOpen ) *zOpen = (char const *)d->open.z; +} + +#define cmpp__scan_int2(SZ,PFMT,Z,N,TGT) \ + (N<SZ) && 1==sscanf((char const *)Z, "%" #SZ PFMT, TGT) + +bool cmpp__is_int(unsigned char const *z, unsigned n, + int *pOut){ + int d = 0; + return cmpp__scan_int2(16, PRIi32, z, n, pOut ? pOut : &d); +} + +bool cmpp__is_int64(unsigned char const *z, unsigned n, int64_t *pOut){ + int64_t d = 0; + return cmpp__scan_int2(24, PRIi64, z, n, pOut ? pOut : &d); +} +#undef cmpp__scan_int2 + +/** + Impl for the -Fx=filename flag. + + TODO?: refactor to take an optional zVal to make it suitable for + the public API. This impl requires that zKey contain + "key=filename". +*/ +static int cmpp__set_file(cmpp *pp, unsigned const char * zKey, + cmpp_ssize_t nKey){ + if(ppCode) return ppCode; + CmppKvp kvp = CmppKvp_empty; + nKey = cmpp__strlenu(zKey, nKey); + if( CmppKvp_parse(pp, &kvp, zKey, nKey, CmppKvp_op_eq1) ){ + return ppCode; + } + sqlite3_stmt * q = 0; + FileWrapper fw = FileWrapper_empty; + if( cmpp__FileWrapper_open(pp, &fw, (char const *)kvp.v.z, "rb") ){ + assert(ppCode); + return ppCode; + } + cmpp__FileWrapper_slurp(pp, &fw); + q = cmpp__stmt(pp, CmppStmt_defIns, false); + if( q && 0==cmpp__bind_textn(pp, q, 2, kvp.k.z, (int)kvp.k.n) ){ + //g_warn("zKey=%.*s", (int)kvp.k.n, kvp.k.z); + if( pp->pimpl->flags.chompF ){ + FileWrapper_chomp(&fw); + } + if( fw.nContent ){ + cmpp__bind_textx(pp, q, 3, fw.zContent, + (cmpp_ssize_t)fw.nContent, sqlite3_free); + fw.zContent = 0 /* transferred ownership */; + fw.nContent = 0; + }else{ + cmpp__bind_null(pp, q, 2); + } + cmpp__step(pp, q, true); + g_debug(pp,2,("define: %s%s%s\n", + kvp.k.z, + kvp.v.z ? " with value " : "", + kvp.v.z ? (char const *)kvp.v.z : "")); + } + FileWrapper_close(&fw); + return ppCode; +} + +CMPP__EXPORT(int, cmpp_has)(cmpp *pp, const char * zName, cmpp_ssize_t nName){ + int rc = 0; + sqlite3_stmt * const q = cmpp__stmt(pp, CmppStmt_defHas, false); + if( q ){ + nName = cmpp__strlen(zName, nName); + cmpp__bind_textn(pp, q, 1, ustr_c(zName), nName); + if(SQLITE_ROW == cmpp__step(pp, q, true)){ + rc = 1; + }else{ + rc = 0; + } + g_debug(pp,1,("has [%s] ?= %d\n",zName, rc)); + } + return rc; +} + +int cmpp__get_bool(cmpp *pp, unsigned const char *zName, cmpp_ssize_t nName){ + int rc = 0; + sqlite3_stmt * const q = cmpp__stmt(pp, CmppStmt_defGetBool, false); + if( q ){ + nName = cmpp__strlenu(zName, nName); + cmpp__bind_textn(pp, q, 1, zName, nName); + assert(0==ppCode); + if(SQLITE_ROW == cmpp__step(pp, q, false)){ + rc = sqlite3_column_int(q, 0); + }else{ + rc = 0; + cmpp__affirm_undef_policy(pp, zName, nName); + } + cmpp__stmt_reset(q); + } + return rc; +} + +int cmpp__get_int(cmpp *pp, unsigned const char * zName, + cmpp_ssize_t nName, int *pOut ){ + sqlite3_stmt * const q = cmpp__stmt(pp, CmppStmt_defGetInt, false); + if( q ){ + nName = cmpp__strlenu(zName, nName); + cmpp__bind_textn(pp, q, 1, zName, nName); + assert(0==ppCode); + if(SQLITE_ROW == cmpp__step(pp, q, false)){ + *pOut = sqlite3_column_int(q,0); + }else{ + cmpp__affirm_undef_policy(pp, zName, nName); + } + cmpp__stmt_reset(q); + } + return ppCode; +} + +int cmpp__get_b(cmpp *pp, unsigned const char * zName, + cmpp_ssize_t nName, cmpp_b * os, bool enforceUndefPolicy){ + int rc = 0; + sqlite3_stmt * const q = cmpp__stmt(pp, CmppStmt_defGet, false); + if( q ){ + nName = cmpp__strlenu(zName, nName); + cmpp__bind_textn(pp, q, 1, zName, nName); + int n = 0; + if(SQLITE_ROW == cmpp__step(pp, q, false)){ + const unsigned char * z = sqlite3_column_text(q, 3); + n = sqlite3_column_bytes(q, 3); + cmpp_b_append4(pp, os, z, (cmpp_size_t)n); + rc = 1; + }else{ + if( enforceUndefPolicy ){ + cmpp__affirm_undef_policy(pp, zName, nName); + } + rc = 0; + } + cmpp__stmt_reset(q); + g_debug(pp,1,("get-define [%.*s] ?= %d %.*s\n", + nName, zName, rc, os->n, os->z)); + } + return rc; +} + +int cmpp__get(cmpp *pp, unsigned const char * zName, + cmpp_ssize_t nName, unsigned char **zVal, + unsigned int *nVal){ + int rc = 0; + sqlite3_stmt * const q = cmpp__stmt(pp, CmppStmt_defGet, false); + if( q ){ + nName = cmpp__strlenu(zName, nName); + cmpp__bind_textn(pp, q, 1, zName, nName); + int n = 0; + if(SQLITE_ROW == cmpp__step(pp, q, false)){ + const unsigned char * z = sqlite3_column_text(q, 3); + n = sqlite3_column_bytes(q, 3); + if( nVal ) *nVal = (unsigned)n; + *zVal = ustr_nc(sqlite3_mprintf("%.*s", n, z)) + /* TODO? Return NULL for the n==0 case? */; + if( n && cmpp_check_oom(pp, *zVal) ){ + assert(!*zVal); + }else{ + rc = 1; + } + }else{ + cmpp__affirm_undef_policy(pp, zName, nName); + rc = 0; + } + cmpp__stmt_reset(q); + g_debug(pp,1,("get-define [%.*s] ?= %d %.*s\n", + nName, zName, rc, + *zVal ? n : 0, + *zVal ? (char const *)*zVal : "<NULL>")); + } + return rc; +} + +CMPP__EXPORT(int, cmpp_undef)(cmpp *pp, const char * zKey, + unsigned int *nRemoved){ + sqlite3_stmt * const q = cmpp__stmt(pp, CmppStmt_defDel, false); + if( q ){ + unsigned int const n = strlen(zKey); + cmpp__bind_textn(pp, q, 1, ustr_c(zKey), (cmpp_ssize_t)n); + cmpp__step(pp, q, true); + if( nRemoved ){ + *nRemoved = (unsigned)sqlite3_changes(pp->pimpl->db.dbh); + } + g_debug(pp,2,("undefine: %.*s\n",n, zKey)); + } + return ppCode; +} + +int cmpp__include_dir_add(cmpp *pp, const char * zDir, int priority, int64_t * pRowid){ + if( pRowid ) *pRowid = 0; + if( !ppCode && zDir && *zDir ){ + sqlite3_stmt * const q = cmpp__stmt(pp, CmppStmt_inclPathAdd, false); + if( q ){ + /* TODO: normalize zDir before insertion so that a/b and a/b/ + are equivalent. The relavent code is in another tree, + awaiting a decision on whether to import it or re-base cmpp + on top of that library (which would, e.g., replace cmpp_b + with that one, which is more mature). + */ + cmpp__bind_int(pp, q, 1, priority); + cmpp__bind_textn(pp, q, 2, ustr_c(zDir), -1); + int const rc = cmpp__step(pp, q, false); + if( SQLITE_ROW==rc ){ + ++pp->pimpl->flags.nIncludeDir; + if( pRowid ){ + *pRowid = sqlite3_column_int64(q, 0); + } + } + cmpp__stmt_reset(q); + /*g_warn("inclpath add: rc=%d rowid=%" PRIi64 " prio=%d %s", + rc, pRowid ? *pRowid : 0, priority, zDir);*/ + g_debug(pp,2,("inclpath add: prio=%d %s\n", priority, zDir)); + } + } + return ppCode; +} + +CMPP__EXPORT(int, cmpp_include_dir_add)(cmpp *pp, const char * zDir){ + return cmpp__include_dir_add(pp, zDir, 0, NULL); +} + +int cmpp__include_dir_rm_id(cmpp *pp, int64_t rowid){ + sqlite3_stmt * const q = cmpp__stmt(pp, CmppStmt_inclPathRmId, true); + if( q ){ + /* Hoop-jumping to allow this to work even if pp's in an error + state. */ + int rc = sqlite3_bind_int64(q, 1, rowid); + if( 0==rc ){ + rc = sqlite3_step(q); + if( SQLITE_ROW==rc ){ + --pp->pimpl->flags.nIncludeDir; + rc = 0; + }else if( SQLITE_DONE==rc ){ + rc = 0; + } + } + if( rc && !ppCode ){ + cmpp__db_rc(pp, rc, sqlite3_sql(q)); + } + cmpp__stmt_reset(q); + g_debug(pp,2,("inclpath rm #%"PRIi64 "\n", rowid)); + } + return ppCode; +} + +CMPP__EXPORT(int, cmpp_module_dir_add)(cmpp *pp, const char * zDirs){ +#if CMPP_ENABLE_DLLS + if( !ppCode ){ + cmpp_b * const ob = &pp->pimpl->mod.path; + if( !zDirs && !ob->n ){ + zDirs = getenv("CMPP_MODULE_PATH"); + if( !zDirs ){ + zDirs = CMPP_MODULE_PATH; + } + } + if( !zDirs || !*zDirs ) return 0; + char const * z = zDirs; + char const * const zEnd = zDirs + strlen(zDirs); + if( 0==cmpp_b_reserve3(pp, ob, ob->n + (zEnd - z) + 3) ){ + unsigned char * zo = ob->z + ob->n; + unsigned i = 0; + for( ; z < zEnd && !ppCode; ++z ){ + switch( *z ){ + case CMPP_PATH_SEPARATOR: + *zo++ = pp->pimpl->mod.pathSep; + break; + default: + if( 1==++i && ob->n ){ + cmpp_b_append_ch(ob, pp->pimpl->mod.pathSep); + } + *zo++ = *z; + break; + } + } + *zo = 0; + ob->n = (zo - ob->z); + } + } + return ppCode; +#else + return CMPP_RC_UNSUPPORTED; +#endif +} + +CMPP__EXPORT(int, cmpp_db_name_set)(cmpp *pp, const char * zName){ + if( 0==ppCode ){ + cmpp__pi(pp); + if( pi->db.dbh ){ + return cmpp__err(pp, CMPP_RC_MISUSE, + "DB name cannot be set after db initialization."); + } + if( zName ){ + char * const z = sqlite3_mprintf("%s", zName); + if( 0==cmpp_check_oom(pp, z) ){ + cmpp_mfree(pi->db.zName); + pi->db.zName = z; + } + }else{ + cmpp_mfree(pi->db.zName); + pi->db.zName = 0; + } + } + return ppCode; +} + +bool cmpp__is_legal_key(unsigned char const *zName, + cmpp_size_t n, + unsigned char const **zErrPos, + bool equalIsLegal){ + if( !n || n>64/*arbitrary*/ ){ + if( zErrPos ) *zErrPos = 0; + return false; + } + unsigned char const * z = zName; + unsigned char const * const zEnd = zName + n; + for( ; z<zEnd; ++z ){ + if( !((*z>='a' && *z<='z') + || (*z>='A' && *z<='Z') + || (z>zName && + ('-'==*z + /* This is gonna bite us if we extend the expresions to + support +/-. Expressions currently parse X=Y (no + spaces) as the three tokens X = Y, but we'd need to + require a space between X-Y in expressions because + '-' is a legal symbol character. i've looked at + making '-' illegal but it's just too convenient for + use in define keys. Once one is used to + tcl-style-naming of stuff, it's painful to have to go + back to snake_case. + */ + || (*z>='0' && *z<='9'))) + || (*z>='.' && *z<='/') + || (*z==':') + || (*z=='_') + || (equalIsLegal && z>zName && '='==*z) + || (*z & 0x80) + ) ){ + if( zErrPos ) *zErrPos = z; + return false; + } + } + return true; +} + +bool cmpp_is_legal_key(unsigned char const *zName, + cmpp_size_t n, + unsigned char const **zErrPos){ + return cmpp__is_legal_key(zName, n, zErrPos, false); +} + +int cmpp__legal_key_check(cmpp *pp, unsigned char const *zKey, + cmpp_ssize_t nKey, bool permitEqualSign){ + if( !ppCode ){ + unsigned char const *zAt = 0; + nKey = cmpp__strlenu(zKey, nKey); + if( !cmpp__is_legal_key(zKey, nKey, &zAt, permitEqualSign) ){ + cmpp__err(pp, CMPP_RC_SYNTAX, + "Illegal character 0x%02x in key [%.*s]", + (int)*zAt, nKey, zKey); + } + } + return ppCode; +} + +CMPP__EXPORT(bool, cmpp_next_chunk)(unsigned char const **zPos, + unsigned char const *zEnd, + unsigned char chSep, + cmpp_size_t *pCounter){ + assert( zPos ); + assert( *zPos ); + assert( zEnd ); + if( *zPos >= zEnd ) return false; + unsigned char const * z = *zPos; + while( z<zEnd ){ + if( chSep==*z ){ + ++z; + if( pCounter ) ++*pCounter; + break; + } + ++z; + } + if( *zPos==z ) return false; + *zPos = z; + return true; +} + +/** + Scans dx for the next newline. It updates ln to contain the result, + which includes the trailing newline unless EOF is hit before a + newline. + + Returns true if it has input, false at EOF. It has no error + conditions beyond invalid dx->pimpl state, which "doesn't happen". +*/ +//static +bool cmpp__dx_next_line(cmpp_dx * const dx, CmppDLine *ln){ + assert( !dxppCode ); + cmpp_dx_pimpl * const dxp = dx->pimpl; + if(!dxp->pos.z) dxp->pos.z = dxp->zBegin; + assert( dxp->zEnd ); + if( dxp->pos.z>=dxp->zEnd ){ + return false; + } + assert( (dxp->pos.z==dxp->zBegin || dxp->pos.z[-1]=='\n') + && "Else we've mismanaged something."); + cmpp__dx_pi(dx); + ln->lineNo = dpi->pos.lineNo; + ln->zBegin = dpi->pos.z; + ln->zEnd = ln->zBegin; + return cmpp_next_chunk(&ln->zEnd, dpi->zEnd, (unsigned char)'\n', + &dpi->pos.lineNo); +} + +/** + Scans [dx->pos.z,dx->zEnd) for a directive delimiter. Emits any + non-delimiter output found along the way to dx->pp's output + channel. + + This updates dx->pimpl->pos.z and dx->pimpl->pos.lineNo as it goes. + + If a delimiter is found, it sets *gotOne to true and updates + dx->pimpl->dline to point to the remainder of that line. On no match + *gotOne will be false and EOF will have been reached. + + Returns dxppCode. If it returns non-0 then the state of dx's + tokenization pieces are unspecified. i.e. it's illegal to call this + again without a reset. +*/ +static int cmpp_dx_delim_search(cmpp_dx * const dx, bool * gotOne){ + if( dxppCode ) return dxppCode; + cmpp_dx_pimpl * const dxp = dx->pimpl; + if(!dxp->pos.z) dxp->pos.z = dxp->zBegin; + if( dxp->pos.z>=dxp->zEnd ){ + *gotOne = false; + return 0; + } + assert( (dxp->pos.z==dxp->zBegin || dxp->pos.z[-1]=='\n') + && "Else we've mismanaged something."); + cmpp__pi(dx->pp); + cmpp__delim const * const delim = cmpp__dx_delim(dx); + if(!delim) { + return cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "The directive delimiter stack is empty."); + } + unsigned char const * const zD = delim->open.z; + unsigned short const nD = delim->open.n; + unsigned char const * const zEnd = dxp->zEnd; + unsigned char const * zLeft = dxp->pos.z; + unsigned char const * z = zLeft; + assert(zD); + assert(nD); +#if 0 + assert( 0==*zEnd && "Else we'll misinteract with strcspn()" ); + if( *zEnd ){ + return cmpp_dx_err(dx, CMPP_RC_RANGE, + "Input must be NUL-terminated."); + } +#endif + ++dxp->flags.countLines; + while( z<zEnd && '\n'==*z ){ + /* Skip leading newlines. We have to delay the handling of + leading whitepace until later so that: + + | #if + |^^ those two spaces do not get emitted. + */ + ++dxp->pos.lineNo; + ++z; + } +#define tflush \ + if( z>zEnd ) z=zEnd; \ + if( z>zLeft && cmpp_dx_out_expand(dx, &pi->out, zLeft, \ + (cmpp_size_t)(z-zLeft), \ + cmpp_atpol_CURRENT) ){ \ + --dxp->flags.countLines; \ + return dxppCode; \ + } zLeft = z + + CmppDLine * const dline = &dxp->dline; + bool atBOL = true /* At the start of a line? Successful calls to + this always end at either BOL or EOF. */; + if( 0 ){ + g_warn("scanning... <<%.*s...>>", + (zEnd-z)>20?20:(zEnd-z), z); + } + while( z<zEnd && !dxppCode ){ + if( !atBOL ){ + /* We're continuing the scan of a line on which the first bytes + didn't match a delimiter. */ + while( z<zEnd ){ + while((z<zEnd && '\n'==*z) + || (z+1<zEnd && '\r'==*z && '\n'==z[1]) ){ + ++dxp->pos.lineNo; + z += 1 + ('\r'==*z); + atBOL = true; + } + if( atBOL ){ + break; + } + ++z; + } + if( !atBOL ) break; + } + if( 0 ){ + g_warn("at BOL... <<%.*s...>>", + (zEnd-z) > 20 ? 20 : (zEnd-z), z); + } + + /* We're at BOL. Check for a delimiter with optional leading + spaces. */ + tflush; + cmpp_skip_space(&z, zEnd); + int const skip = cmpp_isnl(z, zEnd); + if( skip ){ + /* Special case: a line comprised solely of whitespace. If we + don't catch this here, we won't recognize a delimiter which + starts on the next line. */ + tflush; + z += skip; + ++dxp->pos.lineNo; + continue; + } + if( 0 ){ + g_warn("at BOL... <<%.*s...>>", + (zEnd-z) > 20 ? 20 : (zEnd-z), z); + } + if( z + nD>zEnd ){ + /* Too short for a delimiter. We'll catch the z+nD==zEnd corner + case in a moment. */ + z = zEnd; + break; + } + if( memcmp(z, zD, nD) ){ + /* Not a delimiter. Keep trying. */ + atBOL = false; + ++z; + continue; + } + + /* z now points to a delimiter which sits at the start of a line + (ignoring leading spaces). */ + z += nD /* skip the delimiter */; + cmpp_skip_space(&z, zEnd) /* skip spaces immediately following + the delimiter. */; + if( z>=zEnd || cmpp_isnl(z, zEnd) ){ + dxserr("No directive name found after %s.", zD); + /* We could arguably treat this as no match and pass this line + through as-is but that currently sounds like a pothole. */ + break; + } + /* Set up dx->pimpl->dline to encompass the whole directive line sans + delimiter and leading spaces. */ + dline->zBegin = z + /* dx->pimpl->dline starts at the directive name and extends until the + next EOL/EOF. We don't yet know if it's a legal directive + name - cmpp_dx_next() figures that part out. */; + dline->lineNo = dxp->pos.lineNo; + /* Now find the end of the line or EOF, accounting for + backslash-escaped newlines and _not_ requiring backslashes to + escape newlines inside of {...}, (...), or [...]. We could also + add the double-quotes to this, but let's start without that. */ + bool keepGoing = true; + zLeft = z; + while( keepGoing && z<zEnd ){ + switch( *z ){ + case '(': case '{': case '[':{ + zLeft = z; + if( cmpp__find_closing2(dx->pp, &z, zEnd, &dxp->pos.lineNo) ){ + --dxp->flags.countLines; + return dxppCode; + } + ++z /* group-closing character */; + /* + Sidebar: this only checks top-level groups. It is + possible that an inner group is malformed, e.g.: + + { ( } + + It's also possible that that's perfectly legal for a + specific use case. + + Such cases will, if they're indeed syntax errors, be + recognized as such in the arguments-parsing + steps. Catching them here would require that we + recursively validate all of [zLeft,z) for group + constructs, whereas that traversal happens as a matter of + course in argument parsing. It would also require the + assumption that such constructs are not legal, which is + invalid once we start dealing with free-form input like + #query SQL. + */ + break; + } + case '\n': + assert( z!=dline->zBegin && "Checked up above" ); + if( '\\'==z[-1] + || (z>zLeft+1 && '\r'==z[-1] && '\\'==z[-2]) ){ + /* Backslash-escaped newline. */ + ++z; + }else{ + /* EOL for this directive. */ + keepGoing = false; + } + ++dxp->pos.lineNo; + break; + default: + ++z; + } + } + assert( z==zEnd || '\n'==*z ); + dline->zEnd = z; + dxp->pos.z = dline->zEnd + 1 + /* For the next call to this function, skip the trailing newline + or EOF */; + assert( dline->zBegin < dline->zEnd && "Was checked above" ); + if( 0 ){ + g_warn("line= %u <<%.*s>>", (dline->zEnd-dline->zBegin), + (dline->zEnd-dline->zBegin), dline->zBegin); + } + *gotOne = true; + assert( !dxppCode ); + --dxp->flags.countLines; + return 0; + } + /* No directives found. We're now at EOL or EOF. Flush any pending + LHS content. */ + tflush; + dx->pimpl->pos.z = z; + *gotOne = false; + return dxppCode; +#undef tflush +} + +int CmppKvp_parse(cmpp *pp, CmppKvp * p, unsigned char const *zKey, + cmpp_ssize_t nKey, CmppKvp_op_e opPolicy){ + if(ppCode) return ppCode; + char chEq = 0; + char opLen = 0; + *p = CmppKvp_empty; + p->k.z = zKey; + p->k.n = cmpp__strlenu(zKey, nKey); + switch( opPolicy ){ + case CmppKvp_op_none:// break; + case CmppKvp_op_eq1: + chEq = '='; + opLen = 1; + break; + default: + assert(!"don't use these"); + /* no longer todo: ==, !=, <=, <, >, >= */ + chEq = '='; + opLen = 1; + break; + } + assert( chEq ); + p->op = CmppKvp_op_none; + unsigned const char * const zEnd = p->k.z + p->k.n; + for(unsigned const char * zPos = p->k.z ; *zPos && zPos<zEnd ; ++zPos) { + if( chEq==*zPos ){ + if( CmppKvp_op_none==opPolicy ){ + cmpp__err(pp, CMPP_RC_SYNTAX, + "Illegal operator in key: %s", zKey); + }else{ + p->op = CmppKvp_op_eq1; + p->k.n = (unsigned)(zPos - ustr_c(zKey)); + zPos += opLen; + assert( zPos <= zEnd ); + p->v.z = zPos; + p->v.n = (unsigned)(zEnd - zPos); + } + break; + } + } + cmpp__legal_key_check(pp, p->k.z, p->k.n, false); + return ppCode; +} + +int cmpp_array_reserve(cmpp *pp, void **list, cmpp_size_t nDesired, + cmpp_size_t * nAlloc, unsigned sizeOfEntry){ + int rc = pp ? ppCode : 0; + if( 0==rc && nDesired > *nAlloc ){ + cmpp_size_t const nA = nDesired < 10 ? 10 : nDesired; + void * const p = cmpp_mrealloc(*list, sizeOfEntry * nA); + rc = cmpp_check_oom(pp, p); + if( p ){ + memset((unsigned char *)p + + (sizeOfEntry * *nAlloc), 0, + sizeOfEntry * (nA - *nAlloc)); + *list = p; + *nAlloc = nA; + } + } + return rc; +} + +CmppLvl * CmppLvlList_push(cmpp *pp, CmppLvlList *li){ + CmppLvl * p = 0; + assert( li->list ? li->nAlloc : 0==li->nAlloc ); + if( 0==ppCode + && 0==CmppLvlList_reserve(pp, li, + cmpp__li_reserve1_size(li,5)) ){ + p = li->list[li->n]; + if( !p ){ + p = cmpp__malloc(pp, sizeof(*p)); + } + if( p ){ + li->list[li->n++] = p; + *p = CmppLvl_empty; + } + } + return p; +} + +void CmppLvlList_pop(cmpp * const pp, CmppLvlList * const li, + CmppLvl * const lvl){ + assert( li->n ); + if( li->n ){ + if( lvl==li->list[li->n-1] ){ + *lvl = CmppLvl_empty; + cmpp_mfree(lvl); + li->list[--li->n] = 0; + }else{ + if( pp ){ + cmpp_err_set(pp, CMPP_RC_ASSERT, + "Misuse of %s(): not passed the top of the stack. " + "The CmppLvl stack is now out of whack.", + __func__); + }else{ + cmpp__fatal("Misuse of %s(): not passed the top of the stack", + __func__); + } + /* do not free it - CmppLvlList_cleanup() will catch it. */ + } + } +} + +void CmppLvlList_cleanup(CmppLvlList *li){ + const CmppLvlList CmppLvlList_empty = CmppLvlList_empty_m; + while( li->nAlloc ){ + cmpp_mfree( li->list[--li->nAlloc] ); + } + cmpp_mfree(li->list); + *li = CmppLvlList_empty; +} + +static inline void CmppDList_entry_clean(CmppDList_entry * const e){ + if( e->d.impl.dtor ){ + e->d.impl.dtor( e->d.impl.state ); + } + cmpp_mfree(e->zName); + *e = CmppDList_entry_empty; +} + +#if 0 +CmppDList * CmppDList_reuse(CmppDList *li){ + while( li->n ){ + CmppDList_entry_clean( li->list[--li->n] ); + } + return li; +} +#endif + +void CmppDList_cleanup(CmppDList *li){ + static const CmppDList CmppDList_empty = CmppDList_empty_m; + while( li->n ){ + CmppDList_entry_clean( li->list[--li->n] ); + cmpp_mfree( li->list[li->n] ); + li->list[li->n] = 0; + } + cmpp_mfree(li->list); + *li = CmppDList_empty; +} + +void CmppDList_unappend(CmppDList *li){ + assert( li->n ); + if( li->n ){ + CmppDList_entry_clean(li->list[--li->n]); + } +} + + +/** bsearch()/qsort() comparison for (cmpp_d**), sorting by name. */ +static +int CmppDList_entry_cmp_pp(const void *p1, const void *p2){ + CmppDList_entry const * eL = *(CmppDList_entry const * const *)p1; + CmppDList_entry const * eR = *(CmppDList_entry const * const *)p2; + return eL->d.name.n==eR->d.name.n + ? memcmp(eL->d.name.z, eR->d.name.z, eL->d.name.n) + : strcmp((char const *)eL->d.name.z, + (char const *)eR->d.name.z); +} + +static void CmppDList_sort(CmppDList * const li){ + if( li->n>1 ){ + qsort(li->list, li->n, sizeof(CmppDList_entry*), + CmppDList_entry_cmp_pp); + } +} + +CmppDList_entry * CmppDList_append(cmpp *pp, CmppDList *li){ + CmppDList_entry * p = 0; + assert( li->list ? li->nAlloc : 0==li->nAlloc ); + if( 0==ppCode + && 0==cmpp_array_reserve(pp, (void **)&li->list, + cmpp__li_reserve1_size(li, 15), + &li->nAlloc, sizeof(p)) ){ + p = li->list[li->n]; + if( !p ){ + li->list[li->n] = p = cmpp__malloc(pp, sizeof(*p)); + } + if( p ){ + ++li->n; + *p = CmppDList_entry_empty; + } + } + return p; +} + +CmppDList_entry * CmppDList_search(CmppDList const * li, + char const *zName){ + if( li->n > 2 ){ + CmppDList_entry const key = { + .d = { + .name = { + .z = zName, + .n = strlen(zName) + } + } + }; + CmppDList_entry const * pKey = &key; + CmppDList_entry ** pRv + = bsearch(&pKey, li->list, li->n, sizeof(li->list[0]), + CmppDList_entry_cmp_pp); + //g_warn("search in=%s out=%s", zName, (pRv ? (*pRv)->d.name.z : "<null>")); + return pRv ? *pRv : 0; + }else{ + cmpp_size_t const nName = cmpp__strlen(zName, -1); + for( cmpp_size_t i = 0; i < li->n; ++i ){ + CmppDList_entry * const e = li->list[i]; + if( nName==e->d.name.n && 0==strcmp(zName, e->d.name.z) ){ + //g_warn("search in=%s out=%s", zName, e->d.name.z); + return e; + } + } + return 0; + } +} + +void cmpp__delim_cleanup(cmpp__delim *d){ + cmpp__delim const dd = cmpp__delim_empty_m; + cmpp_mfree(d->zOwns); + *d = dd; + assert(!d->zOwns); + assert(d->open.z); + assert(0==strcmp((char*)d->open.z, CMPP_DEFAULT_DELIM)); + assert(d->open.n == sizeof(CMPP_DEFAULT_DELIM)-1); +} + +cmpp__delim * cmpp__delim_list_push(cmpp *pp, cmpp__delim_list *li){ + cmpp__delim * p = 0; + assert( li->list ? li->nAlloc : 0==li->nAlloc ); + if( 0==ppCode + && 0==cmpp_array_reserve(pp, (void **)&li->list, + cmpp__li_reserve1_size(li,4), + &li->nAlloc, sizeof(cmpp__delim)) ){ + p = &li->list[li->n++]; + *p = cmpp__delim_empty; + } + return p; +} + +void cmpp__delim_list_cleanup(cmpp__delim_list *li){ + while( li->nAlloc ) cmpp__delim_cleanup(li->list + --li->nAlloc); + cmpp_mfree(li->list); + *li = cmpp__delim_list_empty; +} + +CMPP__EXPORT(int, cmpp_dx_next)(cmpp_dx * const dx, bool * pGotOne){ + if( dxppCode ) return dxppCode; + + CmppDLine * const tok = &dx->pimpl->dline; + if( !dx->pimpl->zBegin ){ + *pGotOne = false; + return 0; + } + assert(dx->pimpl->zEnd); + assert(dx->pimpl->zEnd > dx->pimpl->zBegin); + *pGotOne = false; + cmpp_dx__reset(dx); + bool foundDelim = false; + if( cmpp_dx_delim_search(dx, &foundDelim) || !foundDelim ){ + return dxppCode; + } + if( cmpp_args__init(dx->pp, &dx->pimpl->args) ){ + return dxppCode; + } + cmpp_skip_space( &tok->zBegin, tok->zEnd ); + g_debug(dx->pp,2,("Directive @ line %u: <<%.*s>>\n", + tok->lineNo, + (int)(tok->zEnd-tok->zBegin), tok->zBegin)); + /* Normalize the directive's line and parse arguments */ + const unsigned lineLen = (unsigned)(tok->zEnd - tok->zBegin); + if(!lineLen){ + return cmpp_dx_err(dx, CMPP_RC_SYNTAX, + "Line #%u has no directive after %s", + tok->lineNo, cmpp_dx_delim(dx)); + } + unsigned char const * zi = tok->zBegin /* Start of input */; + unsigned char const * ziEnd = tok->zEnd /* Input EOF */; + cmpp_b * const bufLine = + cmpp_b_reuse(&dx->pimpl->buf.line) + /* Slightly-transformed copy of the input. */; + if( cmpp_b_reserve3(dx->pp, bufLine, lineLen+1) ){ + return dxppCode; + } + unsigned char * zo = bufLine->z /* Start of output */; + unsigned char const * const zoEnd = + zo + bufLine->nAlloc /* Output EOF. */; + g_debug(dx->pp,2,("Directive @ line %u len=%u <<%.*s>>\n", + tok->lineNo, lineLen, lineLen, tok->zBegin)); + //memset(bufLine->z, 0, bufLine->nAlloc); +#define out(CH) if(zo==zoEnd) break; (*zo++)=CH + /* + bufLine is now populated with a copy of the whole input line. + Now normalize that buffer a bit before trying to parse it. + */ + unsigned char const * zEsc = 0; + cmpp_dx_pimpl * const pimpl = dx->pimpl; + for( ; zi<ziEnd && *zi && zo<zoEnd; + ++zi ){ + /* Write the line to bufLine for the upcoming args parsing to deal + with. Strip backslashes from backslash-escaped newlines. We + leave the newlines intact so that downstream error reporting + can get more precise location info. Backslashes which do not + precede a newline are retained. + */ + switch((int)*zi){ + case (int)'\\': + if( !zi[1] || zi==ziEnd-1 ){ + // special case: ending input with a backslash + out(*zi); + zEsc = 0; + }else if( zEsc ){ + assert( zEsc==zi-1 ); + /* Put them both back. */ + out(*zEsc); + out(*zi); + zEsc = 0; + }else{ + zEsc = zi; + } + break; + case (int)'\n': + out(*zi); + zEsc = 0; + break; + default: + if(zEsc){ + assert( zEsc==zi-1 ); + out(*zEsc); + zEsc = 0; + } + out(*zi); + break; + } + } + if( zo>=zoEnd ){ + return cmpp_dx_err(dx, CMPP_RC_RANGE, + "Ran out of argument-processing space."); + } + *zo = 0; +#undef out + bufLine->n = (cmpp_size_t)(zo - bufLine->z); + if( 0 ) g_warn("bufLine.n=%u line=<<%s>>", bufLine->n, bufLine->z); + /* Line has now been normalized into bufLine->z. */ + for( zo = bufLine->z; zo<zoEnd && *zo; ++zo ){ + /* NULL-terminate the directive so we can search for it. */ + if( cmpp_isspace(*zo) ){ + *zo = 0; + break; + } + } + unsigned char * const zDirective = bufLine->z; + dx->d = cmpp__d_search3(dx->pp, (char const *)zDirective, + cmpp__d_search3_F_ALL); + if( dxppCode ){ + return dxppCode; + }else if(!dx->d){ + return cmpp_dx_err(dx, CMPP_RC_NOT_FOUND, + "Unknown directive at line %" + CMPP_SIZE_T_PFMT ": %.*s\n", + (unsigned)tok->lineNo, + (int)bufLine->n, bufLine->z); + } + assert( zDirective == bufLine->z ); + const bool isCall + = dx->pimpl->args.pimpl->isCall + = dx->pimpl->flags.nextIsCall; + dx->pimpl->flags.nextIsCall = false; + if( isCall ){ + if( cmpp_d_F_NO_CALL & dx->d->flags ){ + return cmpp_dx_err(dx, CMPP_RC_SYNTAX, + "%s%s cannot be used in a [call] context.", + cmpp_dx_delim(dx), + dx->d->name.z); + } + }else if( cmpp_d_F_CALL_ONLY & dx->d->flags ){ + return cmpp_dx_err(dx, CMPP_RC_TYPE, + "'%s' is a call-only directive, " + "not legal here.", dx->d->name.z); + } + if( bufLine->n > dx->d->name.n ){ + dx->args.z = zDirective + dx->d->name.n + 1; + assert( dx->args.z > bufLine->z ); + assert( dx->args.z <= bufLine->z+bufLine->n ); + dx->args.nz = cmpp__strlenu(dx->args.z, -1); + assert( bufLine->nAlloc > dx->args.nz ); + }else{ + dx->args.z = ustr_c("\0"); + dx->args.nz = 0; + } + if( 0 ){ + g_warn("bufLine.n=%u zArgs offset=%u line=<<%s>>\nzArgs=<<%s>>", + bufLine->n, (dx->args.z - zDirective), bufLine->z, dx->args.z); + } + cmpp_skip_snl(&dx->args.z, dx->args.z + dx->args.nz); + if(0){ + g_warn("zArgs %u = <<%.*s>>", (int)dx->args.nz, + (int)dx->args.nz, dx->args.z); + } + assert( !pimpl->buf.argsRaw.n ); + if( dx->args.nz ){ + if( 0 ){ + g_warn("lineLen=%u zargs len=%u: [%.*s]\n", + (unsigned)lineLen, + (int)dx->args.nz, (int)dx->args.nz, + dx->args.z + ); + } + if( cmpp_b_append4(dx->pp, &pimpl->buf.argsRaw, + dx->args.z, dx->args.nz) ){ + return dxppCode; + } + } + assert( !pimpl->args.arg0 ); + assert( !pimpl->args.argc ); + assert( !pimpl->args.pimpl->argOut.n ); + assert( !pimpl->args.pimpl->argli.n ); + assert( dx->args.z ); + if( //1 || //pleases valgrind. Well, it did at one point. + !cmpp_dx_is_eliding(dx) || 0!=(cmpp_d_F_FLOW_CONTROL & dx->d->flags) ){ + if( cmpp_d_F_ARGS_LIST & dx->d->flags ){ + cmpp_dx_args_parse(dx, &pimpl->args); + }else if( cmpp_d_F_ARGS_RAW & dx->d->flags ){ + /* Treat rest of line as one token */ + cmpp_arg * const arg = + CmppArgList_append(dx->pp, &pimpl->args.pimpl->argli); + if( !arg ) return dxppCode; + pimpl->args.arg0 = arg; + pimpl->args.argc = 1; + arg->ttype = cmpp_TT_RawLine; + arg->z = pimpl->buf.argsRaw.z; + arg->n = pimpl->buf.argsRaw.n; + //g_warn("arg->n/z=%u %s", (unsigned)arg->n, arg->z); + } + } + if( 0==dxppCode ){ + dx->args.arg0 = pimpl->args.arg0; + dx->args.argc = pimpl->args.argc; + } + *pGotOne = true; + return dxppCode; +} + +CMPP_EXPORT bool cmpp_dx_is_call(cmpp_dx * const dx){ + return dx->pimpl->args.pimpl->isCall; +} + +CMPP__EXPORT(int, cmpp_d_register)(cmpp * pp, cmpp_d_reg const * r, + cmpp_d ** dOut){ + CmppDList_entry * e1 = 0, * e2 = 0; + bool const isCallOnly = + (cmpp_d_F_CALL_ONLY & r->opener.flags); + if( ppCode ){ + goto end; + } + if( (cmpp_d_F_NOT_IN_SAFEMODE & (r->opener.flags | r->closer.flags)) + && (cmpp_ctor_F_SAFEMODE & pp->pimpl->flags.newFlags) ){ + cmpp__err(pp, CMPP_RC_ACCESS, + "Directive %s%s flag cmpp_d_F_NOT_IN_SAFE_MODE is set " + "and the preprocessor is running in safe mode.", + cmpp__pp_zdelim(pp), r->name); + goto end; + } + if( isCallOnly && r->closer.f ){ + cmpp__err(pp, CMPP_RC_MISUSE, + "Call-only directives may not have a closing directive."); + goto end; + } +#if 0 + if( pp->pimpl->dx ){ + cmpp__err(pp, CMPP_RC_MISUSE, + "Directives may not be added while a " + "directive is running." + /* because that might reallocate being-run directives. + 2025-10-25: that's since been resolved but we need a + use case before enabling this. + */); + goto end; + } +#endif + if( !pp->pimpl->flags.isInternalDirectiveReg + && !cmpp_is_legal_key(ustr_c(r->name), + cmpp__strlen(r->name,-1), NULL) ){ + cmpp__err(pp, CMPP_RC_RANGE, + "\"%s\" is not a legal directive name.", r->name); + goto end; + } + if( cmpp__d_search(pp, r->name) ){ + cmpp__err(pp, CMPP_RC_ALREADY_EXISTS, + "Directive name '%s' is already in use.", + r->name); + goto end; + } + e1 = CmppDList_append(pp, &pp->pimpl->d.list); + if( !e1 ) goto end; + e1->d.impl.callback = r->opener.f; + e1->d.impl.state = r->state; + e1->d.impl.dtor = r->dtor; + if( pp->pimpl->flags.isInternalDirectiveReg ){ + e1->d.flags = r->opener.flags; + }else{ + e1->d.flags = r->opener.flags & cmpp_d_F_MASK; + } + e1->zName = sqlite3_mprintf("%s", r->name); + if( 0==cmpp_check_oom(pp, e1->zName) ){ + //e1->reg = *r; e1->reg.zName = e1->zName; + e1->d.name.z = e1->zName; + e1->d.name.n = strlen(e1->zName); + if( r->closer.f + && (e2 = CmppDList_append(pp, &pp->pimpl->d.list)) ){ + e2->d.impl.callback = r->closer.f; + e2->d.impl.state = r->state; + if( pp->pimpl->flags.isInternalDirectiveReg ){ + e2->d.flags = r->closer.flags; + }else{ + e2->d.flags = r->closer.flags & cmpp_d_F_MASK; + } + e1->d.closer = &e2->d; + e2->zName = sqlite3_mprintf("/%s", r->name); + if( 0==cmpp_check_oom(pp, e2->zName) ){ + e2->d.name.z = e2->zName; + e2->d.name.n = e1->d.name.n + 1; + } + } + } + +end: + if( ppCode ){ + if( e2 ) CmppDList_unappend(&pp->pimpl->d.list); + if( e1 ) CmppDList_unappend(&pp->pimpl->d.list); + else if( r->dtor ){ + r->dtor( r->state ); + } + }else{ + CmppDList_sort(&pp->pimpl->d.list); + if( dOut ){ + *dOut = &e1->d; + } + if( 0 ){ + g_warn("Registered: %s%s%s", e1->zName, + e2 ? " and " : "", + e2 ? e2->zName : ""); + } + } + return ppCode; +} + +CMPP__EXPORT(int, cmpp_dx_consume)(cmpp_dx * const dx, cmpp_outputer * const os, + cmpp_d const * const * const dClosers, + unsigned nClosers, + cmpp_flag32_t flags){ + assert( !dxppCode ); + bool gotOne = false; + cmpp_outputer const oldOut = dx->pp->pimpl->out; + bool const allowOtherDirectives = + (flags & cmpp_dx_consume_F_PROCESS_OTHER_D); + cmpp_d const * const d = cmpp_dx_d(dx); + cmpp_size_t const lineNo = dx->pimpl->dline.lineNo; + bool const pushAt = (cmpp_dx_consume_F_RAW & flags); + if( pushAt && cmpp_atpol_push(dx->pp, cmpp_atpol_OFF) ){ + return dxppCode; + } + if( os ){ + dx->pp->pimpl->out = *os; + } + while( 0==dxppCode + && 0==cmpp_dx_next(dx, &gotOne) + /* ^^^^^^^ resets dx->d, dx->pimpl->args and friends */ ){ + if( !gotOne ){ + dxserr("No closing directive found for " + "%s%s opened on line %" CMPP_SIZE_T_PFMT ".", + cmpp_dx_delim(dx), d->name.z, lineNo); + }else{ + cmpp_d const * const d2 = cmpp_dx_d(dx); + gotOne = false; + for( unsigned i = 0; !gotOne && i < nClosers; ++i ){ + gotOne = d2==dClosers[i]; + } + //g_warn("gotOne=%d d2=%s", gotOne, d2->name.z); + if( gotOne ) break; + else if( !allowOtherDirectives ){ + dxserr("%s%s at line %" CMPP_SIZE_T_PFMT + " may not contain %s%s.", + cmpp_dx_delim(dx), d->name.z, lineNo, + cmpp_dx_delim(dx), d2->name.z); + }else{ + cmpp_dx_process(dx); + } + } + } + if( pushAt ){ + cmpp_atpol_pop(dx->pp); + } + if( os ){ + dx->pp->pimpl->out = oldOut; + } + return dxppCode; +} + +CMPP__EXPORT(int, cmpp_dx_consume_b)(cmpp_dx * const dx, cmpp_b * const b, + cmpp_d const * const * dClosers, + unsigned nClosers, cmpp_flag32_t flags){ + cmpp_outputer oss = cmpp_outputer_b; + oss.state = b; + return cmpp_dx_consume(dx, &oss, dClosers, nClosers, flags); +} + +char const * cmpp__atpol_name(cmpp *pp, cmpp_atpol_e p){ +again: + switch(p){ + case cmpp_atpol_CURRENT:{ + if( pp ){ + assert( p!=cmpp__policy(pp, at) ); + p = cmpp__policy(pp, at); + pp = 0; + goto again; + } + return NULL; + } + case cmpp_atpol_invalid: return NULL; + case cmpp_atpol_OFF: return "off"; + case cmpp_atpol_RETAIN: return "retain"; + case cmpp_atpol_ELIDE: return "elide"; + case cmpp_atpol_ERROR: return "error"; + } + return NULL; +} + +cmpp_atpol_e cmpp_atpol_from_str(cmpp * const pp, char const *z){ + cmpp_atpol_e rv = cmpp_atpol_invalid; + if( 0==strcmp(z, "retain") ) rv = cmpp_atpol_RETAIN; + else if( 0==strcmp(z, "elide") ) rv = cmpp_atpol_ELIDE; + else if( 0==strcmp(z, "error") ) rv = cmpp_atpol_ERROR; + else if( 0==strcmp(z, "off") ) rv = cmpp_atpol_OFF; + if( pp ){ + if( cmpp_atpol_invalid==rv + && 0==strcmp(z, "current") ){ + rv = cmpp__policy(pp,at); + }else if( cmpp_atpol_invalid==rv ){ + cmpp__err(pp, CMPP_RC_RANGE, + "Invalid @ policy value: %s." + " Try one of retain|elide|error|off|current.", z); + }else{ + cmpp__policy(pp,at) = rv; + } + } + return rv; +} + +int cmpp__StringAtIsOk(cmpp * pp, cmpp_atpol_e pol){ + if( 0==ppCode ){ + if( pol==cmpp_atpol_CURRENT ) pol=cmpp__policy(pp,at); + if(cmpp_atpol_OFF==pol ){ + cmpp_err_set(pp, CMPP_RC_UNSUPPORTED, + "@policy is \"off\", so cannot use @\"strings\"."); + } + } + return ppCode; +} + +cmpp__PodList_impl(PodList__atpol,cmpp_atpol_e) +cmpp__PodList_impl(PodList__unpol,cmpp_unpol_e) + +int cmpp_atpol_push(cmpp * pp, cmpp_atpol_e pol){ + if( cmpp_atpol_CURRENT==pol ) pol = cmpp__policy(pp,at); + assert( cmpp_atpol_CURRENT!=pol && "Else internal mismanagement." ); + if( 0==PodList__atpol_push(pp, &cmpp__epol(pp,at), pol) + && 0!=cmpp_atpol_set(pp, pol)/*for validation*/ ){ + PodList__atpol_pop(&cmpp__epol(pp,at)); + } + return ppCode; +} + +void cmpp_atpol_pop(cmpp * pp){ + assert( cmpp__epol(pp,at).n ); + if( cmpp__epol(pp,at).n ){ + PodList__atpol_pop(&cmpp__epol(pp,at)); + }else if( !ppCode ){ + cmpp_err_set(pp, CMPP_RC_MISUSE, + "%s() called when no cmpp_atpol_push() is active.", + __func__); + } +} + +int cmpp_unpol_push(cmpp * pp, cmpp_unpol_e pol){ + if( 0==PodList__unpol_push(pp, &cmpp__epol(pp,un), pol) + && cmpp_unpol_set(pp, pol)/*for validation*/ ){ + PodList__unpol_pop(&cmpp__epol(pp,un)); + } + return ppCode; +} + +void cmpp_unpol_pop(cmpp * pp){ + assert( cmpp__epol(pp,un).n ); + if( cmpp__epol(pp,un).n ){ + PodList__unpol_pop(&cmpp__epol(pp,un)); + }else if( !ppCode ){ + cmpp_err_set(pp, CMPP_RC_MISUSE, + "%s() called when no cmpp_unpol_push() is active.", + __func__); + } +} + +CMPP__EXPORT(cmpp_atpol_e, cmpp_atpol_get)(cmpp const * const pp){ + return cmpp__epol(pp,at).na + ? cmpp__policy(pp,at) : cmpp_atpol_DEFAULT; +} + +CMPP__EXPORT(int, cmpp_atpol_set)(cmpp * const pp, cmpp_atpol_e pol){ + if( 0==ppCode ){ + switch(pol){ + case cmpp_atpol_OFF: + case cmpp_atpol_RETAIN: + case cmpp_atpol_ELIDE: + case cmpp_atpol_ERROR: + assert(cmpp__epol(pp,at).na); + cmpp__policy(pp,at) = pol; + break; + case cmpp_atpol_CURRENT: + break; + default: + cmpp__err(pp, CMPP_RC_RANGE, "Invalid policy value: %d", + (int)pol); + } + } + return ppCode; +} + + +char const * cmpp__unpol_name(cmpp *pp, cmpp_unpol_e p){ + (void)pp; + switch(p){ + case cmpp_unpol_NULL: return "null"; + case cmpp_unpol_ERROR: return "error"; + case cmpp_unpol_invalid: return NULL; + } + return NULL; +} + +cmpp_unpol_e cmpp_unpol_from_str(cmpp * const pp, + char const *z){ + cmpp_unpol_e rv = cmpp_unpol_invalid; + if( 0==strcmp(z, "null") ) rv = cmpp_unpol_NULL; + else if( 0==strcmp(z, "error") ) rv = cmpp_unpol_ERROR; + if( pp ){ + if( cmpp_unpol_invalid==rv + && 0==strcmp(z, "current") ){ + rv = cmpp__policy(pp,un); + }else if( cmpp_unpol_invalid==rv ){ + cmpp__err(pp, CMPP_RC_RANGE, + "Invalid undefined key policy value: %s." + " Try one of null|error.", z); + }else{ + cmpp_unpol_set(pp, rv); + } + } + return rv; +} + +CMPP__EXPORT(cmpp_unpol_e, cmpp_unpol_get)(cmpp const * const pp){ + return cmpp__epol(pp,un).na + ? cmpp__policy(pp,un) : cmpp_unpol_DEFAULT; +} + +CMPP__EXPORT(int, cmpp_unpol_set)(cmpp * const pp, cmpp_unpol_e pol){ + if( 0==ppCode ){ + switch(pol){ + case cmpp_unpol_NULL: + case cmpp_unpol_ERROR: + cmpp__policy(pp,un) = pol; + break; + default: + cmpp__err(pp, CMPP_RC_RANGE, "Invalid policy value: %d", + (int)pol); + } + } + return ppCode; +} + +/** + Reminders to self re. savepoint tracking: + + cmpp_dx tracks per-input-source savepoints. We always want + savepoints which are created via scripts to be limited to that + script. cmpp instances, on the other hand, don't care about that. + + Thus we have two different APIs for starting/ending savepoints. +*/ +CMPP__EXPORT(int, cmpp_sp_begin)(cmpp *pp){ + if( 0==ppCode ){ + sqlite3_stmt * const q = cmpp__stmt(pp, CmppStmt_spBegin, true); + assert( q || !"db init would have otherwise failed"); + if( q && SQLITE_DONE==cmpp__step(pp, q, true) ){ + ++pp->pimpl->flags.nSavepoint; + } + } + return ppCode; +} + +int cmpp__dx_sp_begin(cmpp_dx * const dx){ + if( 0==dxppCode && 0==cmpp_sp_begin(dx->pp) ){ + ++dx->pimpl->nSavepoint; + } + return dxppCode; +} + +CMPP__EXPORT(int, cmpp_sp_rollback)(cmpp *const pp){ + /* Remember that rollback must (mostly) ignore the + pending error state. */ + if( !pp->pimpl->flags.nSavepoint ){ + if( 0==ppCode ){ + cmpp__err(pp, CMPP_RC_MISUSE, + "Cannot roll back: no active savepoint"); + } + }else{ + sqlite3_stmt * q = cmpp__stmt(pp, CmppStmt_spRollback, true); + assert( q || !"db init would have otherwise failed"); + if( q && SQLITE_DONE==cmpp__step(pp, q, true) ){ + q = cmpp__stmt(pp, CmppStmt_spRelease, true); + if( q && SQLITE_DONE==cmpp__step(pp, q, true) ){ + --pp->pimpl->flags.nSavepoint; + } + } + } + return ppCode; +} + +int cmpp__dx_sp_rollback(cmpp_dx * const dx){ + /* Remember that rollback must (mostly) ignore the pending error state. */ + if( !dx->pimpl->nSavepoint ){ + if( 0==dxppCode ){ + cmpp_dx_err(dx, CMPP_RC_MISUSE, + "Cannot roll back: no active savepoint"); + } + }else{ + cmpp_sp_rollback(dx->pp); + --dx->pimpl->nSavepoint; + } + return dxppCode; +} + +CMPP__EXPORT(int, cmpp_sp_commit)(cmpp * const pp){ + if( 0==ppCode ){ + if( !pp->pimpl->flags.nSavepoint ){ + cmpp__err(pp, CMPP_RC_MISUSE, + "Cannot commit: no active savepoint"); + }else{ + sqlite3_stmt * const q = cmpp__stmt(pp, CmppStmt_spRelease, true); + assert( q || !"db init would have otherwise failed"); + if( q && SQLITE_DONE==cmpp__step(pp, q, true) ){ + --pp->pimpl->flags.nSavepoint; + } + } + }else{ + cmpp_sp_rollback(pp); + } + return ppCode; +} + +int cmpp__dx_sp_commit(cmpp_dx * const dx){ + if( 0==dxppCode ){ + if( !dx->pimpl->nSavepoint ){ + cmpp_dx_err(dx, CMPP_RC_MISUSE, + "Cannot commit: no active savepoint"); + }else if( 0==cmpp_sp_commit(dx->pp) ){ + --dx->pimpl->nSavepoint; + } + } + return dxppCode; +} + +static void cmpp_dx_pimpl_reuse(cmpp_dx_pimpl *p){ +#if 0 + /* no: we need most of the state to remain + intact. */ + cmpp_dx_pimpl const tmp = *p; + *p = cmpp_dx_pimpl_empty; + p->buf = tmp.buf; + p->args = tmp.args; +#endif + cmpp_b_reuse(&p->buf.line); + cmpp_b_reuse(&p->buf.argsRaw); + cmpp_args_reuse(&p->args); +} + +void cmpp_dx_pimpl_cleanup(cmpp_dx_pimpl *p){ + cmpp_b_clear(&p->buf.line); + cmpp_b_clear(&p->buf.argsRaw); + cmpp_args_cleanup(&p->args); + *p = cmpp_dx_pimpl_empty; +} + +void cmpp_dx__reset(cmpp_dx * const dx){ + dx->args = cmpp_dx_empty.args; + cmpp_dx_pimpl_reuse(dx->pimpl); + dx->d = 0; + //no: dx->sourceName = 0; +} + +void cmpp_dx_cleanup(cmpp_dx * const dx){ + unsigned prev = 0; + CmppLvlList_cleanup(&dx->pimpl->dxLvl); + while( dx->pimpl->nSavepoint && prev!=dx->pimpl->nSavepoint ){ + prev = dx->pimpl->nSavepoint; + cmpp__dx_sp_rollback(dx); + } + cmpp_dx_pimpl_cleanup(dx->pimpl); + memset(dx, 0, sizeof(*dx)); +} + +int cmpp__find_closing2(cmpp *pp, + unsigned char const **zPos, + unsigned char const *zEnd, + cmpp_size_t * pNl){ + unsigned char const * z = *zPos; + unsigned char const opener = *z; + unsigned char closer = 0; + switch(opener){ + case '(': closer = ')'; break; + case '[': closer = ']'; break; + case '{': closer = '}'; break; + case '"': case '\'': closer = opener; break; + default: + return cmpp__err(pp, CMPP_RC_MISUSE, + "Invalid starting char (0x%x) for %s()", + (int)opener, __func__); + } + int count = 1; + for( ++z; z < zEnd; ++z ){ + if( closer == *z && 0==--count ){ + /* Have to check this first for the case of "" and ''. */ + break; + }else if( opener == *z ){ + ++count; + }else if( pNl && '\n'==*z ){ + ++*pNl; + } + } + if( closer!=*z ){ + if( 0 ){ + g_warn("Closer=%dd Full range: <<%.*s>>", (int)*z, + (zEnd - *zPos), *zPos); + } + //assert(!"here"); + cmpp__err(pp, CMPP_RC_SYNTAX, + "Unbalanced %c%c: %.*s", + opener, closer, + (int)(z-*zPos), *zPos); + }else{ + if( 0 ){ + g_warn("group: n=%u <<%.*s>>", (z + 1 - *zPos), (z +1 - *zPos), *zPos); + } + *zPos = z; + } + return ppCode; +} + +cmpp_tt cmpp__tt_for_sqlite(int sqType){ + cmpp_tt rv; + switch( sqType ){ + case SQLITE_INTEGER: rv = cmpp_TT_Int; break; + case SQLITE_NULL: rv = cmpp_TT_Null; break; + default: rv = cmpp_TT_String; break; + } + return rv; +} + +int cmpp__define_from_row(cmpp * const pp, sqlite3_stmt * const q, + bool defineIfNoRow){ + if( 0==ppCode ){ + int const nCol = sqlite3_column_count(q); + assert( sqlite3_data_count(q)>0 || defineIfNoRow); + /* Create a #define for each column */ + bool const hasRow = sqlite3_data_count(q)>0; + for( int i = 0; !ppCode && i < nCol; ++i ){ + char const * const zCol = sqlite3_column_name(q, i); + if( hasRow ){ + unsigned char const * const zVal = sqlite3_column_text(q, i); + int const nVal = sqlite3_column_bytes(q, i); + cmpp_tt const ttype = + cmpp__tt_for_sqlite(sqlite3_column_type(q,i)); + cmpp__define2(pp, ustr_c(zCol), -1, zVal, nVal, ttype); + }else if(defineIfNoRow){ + cmpp__define2(pp, ustr_c(zCol), -1, ustr_c(""), 0, cmpp_TT_Null); + }else{ + break; + } + } + } + return ppCode; +} + +cmpp_d const * cmpp__d_search(cmpp *pp, const char *zName){ + cmpp_d const * d = 0;//cmpp__d_search(zName); + if( !d ){ + CmppDList_entry const * e = + CmppDList_search(&pp->pimpl->d.list, zName); + if( e ) d = &e->d; + } + return d; +} + +cmpp_d const * cmpp__d_search3(cmpp *pp, const char *zName, + cmpp_flag32_t what){ + cmpp_d const * d = cmpp__d_search(pp, zName); + if( !d ){ + CmppDList_entry const * e = 0; + if( cmpp__d_search3_F_DELAYED & what ){ + int rc = cmpp__d_delayed_load(pp, zName); + if( 0==rc ){ + e = CmppDList_search(&pp->pimpl->d.list, zName); + }else if( CMPP_RC_NO_DIRECTIVE!=rc ){ + assert( ppCode ); + return NULL; + } + } + if( !e + && (cmpp__d_search3_F_AUTOLOADER & what) + && pp->pimpl->d.autoload.f + && 0==pp->pimpl->d.autoload.f(pp, zName, pp->pimpl->d.autoload.state) ){ + e = CmppDList_search(&pp->pimpl->d.list, zName); + } +#if CMPP_D_MODULE + if( !e + && !ppCode + && (cmpp__d_search3_F_DLL & what) ){ + char * z = sqlite3_mprintf("libcmpp-d-%s", zName); + cmpp_check_oom(pp, z); + int rc = cmpp_module_load(pp, z, NULL); + sqlite3_free(z); + if( rc ){ + if( CMPP_RC_NOT_FOUND==rc ){ + cmpp__err_clear(pp); + } + return NULL; + } + e = CmppDList_search(&pp->pimpl->d.list, zName); + } +#endif + if( e ) d = &e->d; + } + return d; +} + +int cmpp_dx_process(cmpp_dx * const dx){ + if( 0==dxppCode ){ + cmpp_d const * const d = cmpp_dx_d(dx); + assert( d ); + if( !cmpp_dx_is_eliding(dx) || (d->flags & cmpp_d_F_FLOW_CONTROL) ){ + if( (cmpp_d_F_NOT_IN_SAFEMODE & d->flags) + && (cmpp_ctor_F_SAFEMODE & dx->pp->pimpl->flags.newFlags) ){ + cmpp_dx_err(dx, CMPP_RC_ACCESS, + "Directive %s%s is disabled by safe mode.", + cmpp_dx_delim(dx), dx->d->name.z); + }else{ + assert(d->impl.callback); + d->impl.callback(dx); + } + } + } + return dxppCode; +} + + +static void cmpp_dx__setup_include_path(cmpp_dx * dx){ + /* Add the leading dir part of dx->sourceName as the + highest-priority include path. It gets removed + in cmpp_dx__teardown(). */ + assert( dx->sourceName ); + enum { BufSize = 512 * 4 }; + unsigned char buf[BufSize] = {0}; + unsigned char *z = &buf[0]; + cmpp_size_t n = cmpp__strlenu(dx->sourceName, -1); + if( n > (unsigned)BufSize-1 ) return; + memcpy(z, dx->sourceName, n); + buf[n] = 0; + cmpp_ssize_t i = n - 1; + for( ; i > 0; --i ){ + if( '/'==z[i] || '\\'==z[i] ){ + z[i] = 0; + n = i; + break; + } + } + if( n>(cmpp_size_t)i ){ + /* No path separator found. Assuming '.'. This is intended to + replace the historical behavior of automatically adding '.' if + no -I flags are used. Potential TODO is getcwd() here instead + of using '.' */ + n = 1; + buf[0] = '.'; + buf[1] = 0; + } + int64_t rowid = 0; + cmpp__include_dir_add(dx->pp, (char const*)buf, + dx->pp->pimpl->flags.nDxDepth, + &rowid); + if( rowid ){ + //g_warn("Adding #include path #%" PRIi64 ": %s", rowid, z); + dx->pimpl->shadow.ridInclPath = rowid; + } +} + +static int cmpp_dx__setup(cmpp *pp, cmpp_dx *dx, + unsigned char const * zIn, + cmpp_ssize_t nIn){ + if( 0==ppCode ){ + assert( dx->sourceName ); + assert( dx->pimpl ); + assert( pp==dx->pp ); + nIn = cmpp__strlenu(zIn, nIn); + if( !nIn ) return 0; + pp->pimpl->dx = dx; + dx->pimpl->zBegin = zIn; + dx->pimpl->zEnd = zIn + nIn; + cmpp_define_shadow(pp, "__FILE__", (char const *)dx->sourceName, + &dx->pimpl->shadow.sidFile); + ++dx->pp->pimpl->flags.nDxDepth; + cmpp_dx__setup_include_path(dx); + } + return ppCode; +} + +static void cmpp_dx__teardown(cmpp_dx *dx){ + if( dx->pimpl->shadow.ridInclPath>0 ){ + cmpp__include_dir_rm_id(dx->pp, dx->pimpl->shadow.ridInclPath); + dx->pimpl->shadow.ridInclPath = 0; + } + if( dx->pimpl->shadow.sidFile ){ + cmpp_define_unshadow(dx->pp, "__FILE__", + dx->pimpl->shadow.sidFile); + } + --dx->pp->pimpl->flags.nDxDepth; + cmpp_dx_cleanup(dx); +} + +CMPP__EXPORT(int, cmpp_process_string)( + cmpp *pp, const char * zName, + unsigned char const * zIn, + cmpp_ssize_t nIn +){ + if( !zName ) zName = ""; + if( 0==cmpp__db_init(pp) ){ + cmpp_dx const * const oldDx = pp->pimpl->dx; + cmpp_dx_pimpl dxp = cmpp_dx_pimpl_empty; + cmpp_dx dx = { + .pp = pp, + .sourceName = ustr_c(zName), + .args = cmpp_dx_empty.args, + .pimpl = &dxp + }; + dxp.flags.nextIsCall = pp->pimpl->flags.nextIsCall; + pp->pimpl->flags.nextIsCall = false; + if( dxp.flags.nextIsCall ){ + assert( pp->pimpl->dx ); + dxp.pos.lineNo = pp->pimpl->dx->pimpl->pos.lineNo; + } + bool gotOne = false; + (void)cmpp__stmt(pp, CmppStmt_sdefIns, true); + (void)cmpp__stmt(pp, CmppStmt_inclPathAdd, true); + (void)cmpp__stmt(pp, CmppStmt_inclPathRmId, true); + (void)cmpp__stmt(pp, CmppStmt_sdefDel, true) + /* hack: ensure that those queries are allocated now, as an + error in processing may keep them from being created + later. We might want to rethink the + prepare-no-statements-on-error bits, but will have to go back + and fix routines which currently rely on that. */; + cmpp_dx__setup(pp, &dx, zIn, nIn); + while(0==ppCode + && 0==cmpp_dx_next(&dx, &gotOne) + && gotOne){ + cmpp_dx_process(&dx); + } + if(0==ppCode && 0!=dx.pimpl->dxLvl.n){ + CmppLvl const * const lv = CmppLvl_get(&dx); + cmpp_dx_err(&dx, CMPP_RC_SYNTAX, + "Input ended inside an unterminated nested construct " + "opened at [%s] line %" CMPP_SIZE_T_PFMT ".", zName, + lv ? lv->lineNo : (cmpp_size_t)0); + } + cmpp_dx__teardown(&dx); + pp->pimpl->dx = oldDx; + } + if( !ppCode ){ + cmpp_outputer_flush(&pp->pimpl->out) + /* We're going to ignore a result code just this once. */; + } + return ppCode; +} + +int cmpp_process_file(cmpp *pp, const char * zName){ + if( 0==ppCode ){ + FileWrapper fw = FileWrapper_empty; + if( 0==cmpp__FileWrapper_open(pp, &fw, zName, "rb") + && 0==cmpp__FileWrapper_slurp(pp, &fw) ){ + cmpp_process_string(pp, zName, fw.zContent, fw.nContent); + } + FileWrapper_close(&fw); + } + return ppCode; +} + +CMPP__EXPORT(int, cmpp_process_stream)(cmpp *pp, const char * zName, + cmpp_input_f src, void * srcState){ + if( 0==ppCode ){ + cmpp_b * const os = cmpp_b_borrow(pp); + int const rc = os + ? cmpp_stream(src, srcState, cmpp_output_f_b, os) + : ppCode; + if( 0==rc ){ + cmpp_process_string(pp, zName, os->z, os->n); + }else{ + cmpp__err(pp, rc, "Error reading from input stream '%s'.", zName); + } + cmpp_b_return(pp, os); + } + return ppCode; +} + +CMPP__EXPORT(int, cmpp_call_str)( + cmpp *pp, unsigned char const * z, cmpp_ssize_t n, + cmpp_b * dest, cmpp_flag32_t flags +){ + if( ppCode ) return ppCode; + cmpp_args args = cmpp_args_empty; + cmpp_b * const b = cmpp_b_borrow(pp); + cmpp_b * const bo = cmpp_b_borrow(pp); + cmpp_outputer oB = cmpp_outputer_b; + if( !b || !bo ) return ppCode; + cmpp__pi(pp); + oB.state = bo; + oB.name = pi->out.name;//"[call]"; + n = cmpp__strlenu(z, n); + //g_warn("calling: <<%.*s>>", (int)n, z); + unsigned char const * zEnd = z+n; + cmpp_skip_snl(&z, zEnd); + cmpp_skip_snl_trailing(z, &zEnd); + n = (zEnd-z); + if( !n ){ + cmpp_err_set(pp, CMPP_RC_SYNTAX, + "Empty [call] is not permitted."); + goto end; + } + //g_warn("calling: <<%.*s>>", (int)n, z); + cmpp__delim const * const delim = cmpp__pp_delim(pp); + assert(delim); + if( (cmpp_size_t)n<=delim->open.n + || 0!=memcmp(z, delim->open.z, delim->open.n) ){ + /* If it doesn't start with the current delimiter, + prepend one. */ + cmpp_b_reserve3(pp, b, delim->open.n + n + 2); + cmpp_b_append4(pp, b, delim->open.z, delim->open.n); + } + cmpp_b_append4(pp, b, z, n); + if( !ppCode ){ + cmpp_outputer oOld = cmpp_outputer_empty; + pi->flags.nextIsCall = true + /* Convey (indirectly) that the first cmpp_dx_next() call made + via cmpp_process_string() is a call context. */; + cmpp__outputer_swap(pp, &oB, &oOld); + cmpp_process_string(pp, (char*)b->z, b->z, b->n); + cmpp__outputer_swap(pp, &oOld, &oB); + assert( !pi->flags.nextIsCall || ppCode ); + pi->flags.nextIsCall = false; + } + if( !ppCode ){ + unsigned char const * zz = bo->z; + unsigned char const * zzEnd = bo->z + bo->n; + if( cmpp_call_F_TRIM_ALL & flags ){ + cmpp_skip_snl(&zz, zzEnd); + cmpp_skip_snl_trailing(zz, &zzEnd); + }else if( 0==(cmpp_call_F_NO_TRIM & flags) ){ + cmpp_b_chomp(bo); + zzEnd = bo->z + bo->n; + } + if( (zzEnd-zz) ){ + cmpp_b_append4(pp, dest, zz, (zzEnd-zz)); + } + } +end: + cmpp_b_return(pp, b); + cmpp_b_return(pp, bo); + cmpp_args_cleanup(&args); + return ppCode; +} + +CMPP__EXPORT(int, cmpp_errno_rc)(int errNo, int dflt){ + switch(errNo){ + /* Please expand on this as tests/use cases call for it... */ + case 0: + return 0; + case EINVAL: + return CMPP_RC_MISUSE; + case ENOMEM: + return CMPP_RC_OOM; + case EROFS: + case EACCES: + case EBUSY: + case EPERM: + case EDQUOT: + case EAGAIN: + case ETXTBSY: + return CMPP_RC_ACCESS; + case EISDIR: + case ENOTDIR: + return CMPP_RC_TYPE; + case ENAMETOOLONG: + case ELOOP: + case ERANGE: + return CMPP_RC_RANGE; + case ENOENT: + case ESRCH: + return CMPP_RC_NOT_FOUND; + case EEXIST: + case ENOTEMPTY: + return CMPP_RC_ALREADY_EXISTS; + case EIO: + return CMPP_RC_IO; + default: + return dflt; + } +} + +int cmpp_flush_f_FILE(void * _FILE){ + return fflush(_FILE) ? cmpp_errno_rc(errno, CMPP_RC_IO) : 0; +} + +int cmpp_output_f_FILE( void * state, + void const * src, cmpp_size_t n ){ + return (1 == fwrite(src, n, 1, state ? (cmpp_FILE*)state : stdout)) + ? 0 : CMPP_RC_IO; +} + +int cmpp_output_f_fd( void * state, void const * src, cmpp_size_t n ){ + int const fd = *((int*)state); + ssize_t const wn = write(fd, src, n); + return wn<0 ? cmpp_errno_rc(errno, CMPP_RC_IO) : 0; +} + +int cmpp_input_f_FILE( void * state, void * dest, cmpp_size_t * n ){ + cmpp_FILE * f = state; + cmpp_size_t const rn = *n; + *n = (cmpp_size_t)fread(dest, 1, rn, f); + return *n==rn ? 0 : (feof(f) ? 0 : CMPP_RC_IO); +} + +int cmpp_input_f_fd( void * state, void * dest, cmpp_size_t * n ){ + int const fd = *((int*)state); + ssize_t const rn = read(fd, dest, *n); + if( rn<0 ){ + return cmpp_errno_rc(errno, CMPP_RC_IO); + }else{ + *n = (cmpp_size_t)rn; + return 0; + } +} + +void cmpp_outputer_cleanup_f_FILE(cmpp_outputer *self){ + if( self->state ){ + cmpp_fclose( self->state ); + self->name = NULL; + self->state = NULL; + } +} + +CMPP__EXPORT(void, cmpp_outputer_cleanup_f_b)(cmpp_outputer *self){ + if( self->state ) cmpp_b_clear(self->state); +} + +CMPP__EXPORT(int, cmpp_outputer_out)(cmpp_outputer *o, void const *p, cmpp_size_t n){ + return o->out ? o->out(o->state, p, n) : 0; +} + +CMPP__EXPORT(int, cmpp_outputer_flush)(cmpp_outputer *o){ + return o->flush ? o->flush(o->state) : 0; +} + +CMPP__EXPORT(void, cmpp_outputer_cleanup)(cmpp_outputer *o){ + if( o->cleanup ){ + o->cleanup( o ); + } +} + +CMPP__EXPORT(int, cmpp_stream)( cmpp_input_f inF, void * inState, + cmpp_output_f outF, void * outState ){ + int rc = 0; + enum { BufSize = 1024 * 4 }; + unsigned char buf[BufSize]; + cmpp_size_t rn = BufSize; + while( 0==rc + && (rn==BufSize) + && (0==(rc=inF(inState, buf, &rn))) ){ + if(rn) rc = outF(outState, buf, rn); + } + return rc; +} + +void cmpp__fatalv_base(char const *zFile, int line, + char const *zFmt, va_list va){ + cmpp_FILE * const fp = stderr; + fflush(stdout); + fprintf(fp, "\n%s:%d: ", zFile, line); + if(zFmt && *zFmt){ + vfprintf(fp, zFmt, va); + fputc('\n', fp); + } + fflush(fp); + exit(1); +} + +void cmpp__fatal_base(char const *zFile, int line, + char const *zFmt, ...){ + va_list va; + va_start(va, zFmt); + cmpp__fatalv_base(zFile, line, zFmt, va); + va_end(va); +} + +CMPP__EXPORT(int, cmpp_err_get)(cmpp *pp, char const **zMsg){ + if( zMsg && ppCode ) *zMsg = pp->pimpl->err.zMsg; + return ppCode; +} + +CMPP__EXPORT(int, cmpp_err_take)(cmpp *pp, char **zMsg){ + int const rc = ppCode; + if( rc ){ + *zMsg = pp->pimpl->err.zMsg; + pp->pimpl->err = cmpp_pimpl_empty.err; + } + return rc; +} + +//CMPP_WASM_EXPORT +void cmpp__err_clear(cmpp *pp){ + cmpp_mfree(pp->pimpl->err.zMsg); + pp->pimpl->err = cmpp_pimpl_empty.err; +} + +CMPP__EXPORT(int, cmpp_err_has)(cmpp const * pp){ + return pp ? pp->pimpl->err.code : 0; +} + +CMPP__EXPORT(void, cmpp_dx_pos_save)(cmpp_dx const * dx, cmpp_dx_pos *pos){ + *pos = dx->pimpl->pos; +} + +CMPP__EXPORT(void, cmpp_dx_pos_restore)(cmpp_dx * dx, cmpp_dx_pos const * pos){ + dx->pimpl->pos = *pos; +} + + +//CMPP_WASM_EXPORT +void cmpp__dx_append_script_info(cmpp_dx const * dx, + sqlite3_str * const sstr){ + sqlite3_str_appendf( + sstr, + "%s%s@ %s line %" CMPP_SIZE_T_PFMT, + dx->d ? dx->d->name.z : "", + dx->d ? " " : "", + (dx->sourceName + && 0==strcmp("-", (char const *)dx->sourceName)) + ? "<stdin>" + : (char const *)dx->sourceName, + dx->pimpl->dline.lineNo + ); +} + +int cmpp__errv(cmpp *pp, int rc, char const *zFmt, va_list va){ + if( pp ){ + cmpp__err_clear(pp); + ppCode = rc; + if( 0==rc ) return rc; + if( CMPP_RC_OOM==rc ){ + oom: + pp->pimpl->err.zMsgC = "An allocation failed."; + return pp->pimpl->err.code = CMPP_RC_OOM; + } + assert( !pp->pimpl->err.zMsg ); + if( pp->pimpl->dx || (zFmt && *zFmt) ){ + sqlite3_str * sstr = 0; + sstr = sqlite3_str_new(pp->pimpl->db.dbh); + if( pp->pimpl->dx ){ + cmpp__dx_append_script_info(pp->pimpl->dx, sstr); + sqlite3_str_append(sstr, ": ", 2); + } + if( zFmt && *zFmt ){ + sqlite3_str_vappendf(sstr, zFmt, va); + }else{ + sqlite3_str_appendf(sstr, "No error info provided."); + } + pp->pimpl->err.zMsgC = + pp->pimpl->err.zMsg = sqlite3_str_finish(sstr); + if( !pp->pimpl->err.zMsg ){ + goto oom; + } + }else{ + pp->pimpl->err.zMsgC = "No error info provided."; + } + rc = ppCode; + } + return rc; +} + +//CMPP_WASM_EXPORT no - variadic +int cmpp_err_set(cmpp *pp, int rc, + char const *zFmt, ...){ + if( pp ){ + va_list va; + va_start(va, zFmt); + rc = cmpp__errv(pp, rc, zFmt, va); + va_end(va); + } + return rc; +} + +const cmpp_d_autoloader cmpp_d_autoloader_empty = + cmpp_d_autoloader_empty_m; + +CMPP__EXPORT(void, cmpp_d_autoloader_set)(cmpp *pp, cmpp_d_autoloader const * pNew){ + if( pp->pimpl->d.autoload.dtor ) pp->pimpl->d.autoload.dtor(pp->pimpl->d.autoload.state); + if( pNew ) pp->pimpl->d.autoload = *pNew; + else pp->pimpl->d.autoload = cmpp_d_autoloader_empty; +} + +CMPP__EXPORT(void, cmpp_d_autoloader_take)(cmpp *pp, cmpp_d_autoloader * pOld){ + *pOld = pp->pimpl->d.autoload; + pp->pimpl->d.autoload = cmpp_d_autoloader_empty; +} + +//CMPP_WASM_EXPORT no - variadic +int cmpp_dx_err_set(cmpp_dx *dx, int rc, + char const *zFmt, ...){ + va_list va; + va_start(va, zFmt); + rc = cmpp__errv(dx->pp, rc, zFmt, va); + va_end(va); + return rc; +} + +CMPP__EXPORT(int, cmpp_err_set1)(cmpp *pp, int rc, char const *zMsg){ + return cmpp_err_set(pp, rc, (zMsg && *zMsg) ? "%s" : 0, zMsg); +} + +//no: CMPP_WASM_EXPORT +char * cmpp_path_search(cmpp *pp, + char const *zPath, + char pathSep, + char const *zBaseName, + char const *zExt){ + char * zrc = 0; + if( !ppCode ){ + sqlite3_stmt * const q = + cmpp__stmt(pp, CmppStmt_selPathSearch, false); + if( q ){ + unsigned char sep[2] = {pathSep, 0}; + cmpp__bind_text(pp, q, 1, ustr_c(zBaseName)); + cmpp__bind_text(pp, q, 2, sep); + cmpp__bind_text(pp, q, 3, ustr_c((zExt ? zExt : ""))); + cmpp__bind_text(pp, q, 4, ustr_c((zPath ? zPath: ""))); + int const dbrc = cmpp__step(pp, q, false); + if( SQLITE_ROW==dbrc ){ + unsigned char const * s = sqlite3_column_text(q, 1); + zrc = sqlite3_mprintf("%s", s); + cmpp_check_oom(pp, zrc); + } + cmpp__stmt_reset(q); + } + } + return zrc; +} + +#if CMPP__OBUF +int cmpp__obuf_flush(cmpp__obuf * b){ + if( 0==b->rc && b->cursor > b->begin ){ + if( b->dest.out ){ + b->rc = b->dest.out(b->dest.state, b->begin, + b->cursor-b->begin); + } + b->cursor = b->begin; + } + if( 0==b->rc && b->dest.flush ){ + b->rc = b->dest.flush(b->dest.state); + } + return b->rc; +} + +void cmpp__obuf_cleanup(cmpp__obuf * b){ + if( b ){ + cmpp__obuf_flush(b);/*ignoring result*/; + if( b->ownsMemory ){ + cmpp_mfree(b->begin); + } + *b = cmpp__obuf_empty; + } +} + +int cmpp__obuf_write(cmpp__obuf * b, void const * src, cmpp_size_t n){ + assert( b ); + if( n && !b->rc && b->dest.out ){ + assert( b->end ); + assert( b->cursor ); + assert( b->cursor <= b->end ); + assert( b->end>b->begin ); + if( b->cursor + n >= b->end ){ + if( 0==cmpp_flush_f_obuf(b) ){ + if( b->cursor + n >= b->end ){ + /* Go ahead and write it all */ + b->rc = b->dest.out(b->dest.state, src, n); + }else{ + goto copy_it; + } + } + }else{ + copy_it: + memcpy(b->cursor, src, n); + b->cursor += n; + } + } + return b->rc; +} + +int cmpp_flush_f_obuf(void * b){ + return cmpp__obuf_flush(b); +} + +int cmpp_output_f_obuf(void * state, void const * src, cmpp_size_t n){ + return cmpp__obuf_write(state, src, n); +} + +void cmpp_outputer_cleanup_f_obuf(cmpp_outputer * o){ + cmpp__obuf_cleanup(o->state); +} +#endif /* CMPP__OBUF */ + +//cmpp__ListType_impl(cmpp__delim_list,cmpp__delim) +//cmpp__ListType_impl(CmppDList,CmppDList_entry*) +//cmpp__ListType_impl(CmppSohList,void*) +cmpp__ListType_impl(CmppArgList,cmpp_arg) +cmpp__ListType_impl(cmpp_b_list,cmpp_b*) +cmpp__ListType_impl(CmppLvlList,CmppLvl*) + +/** + Expects that *ndx points to the current argv entry and that it is a + flag which expects a value. This function checks for --flag=val and + (--flag val) forms. If a value is found then *ndx is adjusted (if + needed) to point to the next argument after the value and *zVal is + * pointed to the value. If no value is found then it returns false. +*/ +static bool get_flag_val(int argc, + char const * const * argv, int * ndx, + char const **zVal){ + char const * zEq = strchr(argv[*ndx], '='); + if( zEq ){ + *zVal = zEq+1; + return 1; + }else if(*ndx+1>=argc){ + return 0; + }else{ + *zVal = argv[++*ndx]; + return 1; + } +} + +static +bool cmpp__arg_is_flag( char const *zFlag, char const *zArg, + char const **zValIfEqX ); +bool cmpp__arg_is_flag( char const *zFlag, char const *zArg, + char const **zValIfEqX ){ + if( zValIfEqX ) *zValIfEqX = 0; + if( 0==strcmp(zFlag, zArg) ) return true; + char const * z = strchr(zArg,'='); + if( z && z>zArg ){ + /* compare the part before the '=' */ + if( 0==strncmp(zFlag, zArg, z-zArg) ){ + if( !zFlag[z-zArg] ){ + if( zValIfEqX ) *zValIfEqX = z+1; + return true; + } + /* Else it was a prefix match. */ + } + } + return false; +} + +void cmpp__dump_defines(cmpp *pp, cmpp_FILE * fp, int bIndent){ + sqlite3_stmt * const q = cmpp__stmt(pp, CmppStmt_defSelAll, false); + if( q ){ + while( SQLITE_ROW==sqlite3_step(q) ){ + int const tt = sqlite3_column_int(q, 0); + unsigned char const * zK = sqlite3_column_text(q, 1); + unsigned char const * zV = sqlite3_column_text(q, 2); + int const nK = sqlite3_column_bytes(q, 1); + int const nV = sqlite3_column_bytes(q, 2); + char const * zTt = cmpp__tt_cstr(tt, true); + if( tt && zTt ) zTt += 3; + else zTt = "String"; + fprintf(fp, "%s%.*s = [%s] %.*s\n", bIndent ? "\t" : "", + nK, zK, zTt, nV, zV); + } + cmpp__stmt_reset(q); + } +} + +/** + This is what was originally the main() of cmpp v1, back when it was + a monolithic app. It still serves as the driver for main() but is + otherwise unused. +*/ +CMPP__EXPORT(int, cmpp_process_argv)(cmpp *pp, int argc, + char const * const * argv){ + if( ppCode ) return ppCode; + int nFile = 0 /* number of files/-e scripts seen */; + +#define ARGVAL if( !zVal && !get_flag_val(argc, argv, &i, &zVal) ){ \ + cmpp__err(pp, CMPP_RC_MISUSE, "Missing value for flag '%s'", \ + argv[i]); \ + break; \ + } +#define M(X) cmpp__arg_is_flag(X, zArg, &zVal) +#define ISFLAG(X) else if(M(X)) +#define ISFLAG2(X,Y) else if(M(X) || M(Y)) +#define NOVAL if( zVal ){ \ + cmpp__err(pp,CMPP_RC_MISUSE,"Unexpected value for %s", zArg); \ + break; \ + } (void)0 + +#define open_output_if_needed \ + if( !pp->pimpl->out.out && cmpp__out_fopen(pp, "-") ) break + + cmpp__staticAssert(TT_None,0==(int)cmpp_TT_None); + cmpp__staticAssert(Mask1, cmpp_d_F_MASK_INTERNAL & cmpp_d_F_FLOW_CONTROL); + cmpp__staticAssert(Mask2, cmpp_d_F_MASK_INTERNAL & cmpp_d_F_NOT_SIMPLIFY); + cmpp__staticAssert(Mask3, 0==(cmpp_d_F_MASK_INTERNAL & cmpp_d_F_MASK)); + + for(int doIt = 0; doIt<2 && 0==ppCode; ++doIt){ + /** + Loop through the flags twice. The first time we just validate + and look for --help/-?. The second time we process the flags. + This approach allows us to easily chain multiple files and + flags: + + ./c-pp -Dfoo -o foo x.y -Ufoo -Dbar -o bar x.y + + Which, it turns out, is a surprisingly useful way to work. + */ +#define DOIT if(1==doIt) + for(int i = 0; i < argc && 0==ppCode; ++i){ + char const * zVal = 0; + int isNoFlag = 0; + char const * zArg = argv[i]; + //g_stderr("i=%d zArg=%s\n", i, zArg); + zVal = 0; + while('-'==*zArg) ++zArg; + if(zArg==argv[i]/*not a flag*/){ + zVal = zArg; + goto do_infile; + } + //g_warn("zArg=%s", zArg); + if( 0==strncmp(zArg,"no-",3) ){ + zArg += 3; + isNoFlag = 1; + } + if( M("?") || M("help") ){ + NOVAL; + cmpp__err(pp, CMPP_RC_HELP, "%s", argv[i]); + break; + }else if('D'==*zArg){ + ++zArg; + if(!*zArg){ + cmpp__err(pp,CMPP_RC_MISUSE,"Missing key for -D"); + }else DOIT { + cmpp_define_legacy(pp, zArg, 0); + } + }else if('F'==*zArg){ + ++zArg; + if(!*zArg){ + cmpp__err(pp,CMPP_RC_MISUSE,"Missing key for -F"); + }else DOIT { + cmpp__set_file(pp, ustr_c(zArg), -1); + } + } + ISFLAG("e"){ + ARGVAL; + DOIT { + ++nFile; + open_output_if_needed; + cmpp_process_string(pp, "-e script", + (unsigned char const *)zVal, -1); + } + }else if('U'==*zArg){ + ++zArg; + if(!*zArg){ + cmpp__err(pp,CMPP_RC_MISUSE,"Missing key for -U"); + }else DOIT { + cmpp_undef(pp, zArg, NULL); + } + }else if('I'==*zArg){ + ++zArg; + if(!*zArg){ + cmpp__err(pp,CMPP_RC_MISUSE,"Missing directory for -I"); + }else DOIT { + cmpp_include_dir_add(pp, zArg); + } + }else if('L'==*zArg){ + ++zArg; + if(!*zArg){ + cmpp__err(pp,CMPP_RC_MISUSE,"Missing directory for -L"); + }else DOIT { + cmpp_module_dir_add(pp, zArg); + } + } + ISFLAG2("o","outfile"){ + ARGVAL; + DOIT { + cmpp__out_fopen(pp, zVal); + } + } + ISFLAG2("f","file"){ + ARGVAL; + do_infile: + DOIT { + if( !pp->pimpl->mod.path.z ){ + cmpp_module_dir_add(pp, NULL); + } + ++nFile; + if( 0 + && !pp->pimpl->flags.nIncludeDir + && cmpp_include_dir_add(pp, ".") ){ + break; + } + open_output_if_needed; + cmpp_process_file(pp, zVal); + } + } + ISFLAG("@"){ + NOVAL; + DOIT { + assert( cmpp_atpol_DEFAULT_FOR_FLAG!=cmpp_atpol_OFF ); + cmpp_atpol_set(pp, isNoFlag + ? cmpp_atpol_OFF + : cmpp_atpol_DEFAULT_FOR_FLAG); + } + } + ISFLAG("@policy"){ + ARGVAL; + cmpp_atpol_from_str(pp, zVal); + } + ISFLAG("debug"){ + NOVAL; + DOIT { + pp->pimpl->flags.doDebug += isNoFlag ? -1 : 1; + } + } + ISFLAG2("u","undefined-policy"){ + ARGVAL; + cmpp_unpol_from_str(pp, zVal); + } + ISFLAG("sql-trace"){ + NOVAL; + /* Needs to be set before the start of the second pass, when + the db is inited. */ + DOIT { + pp->pimpl->sqlTrace.expandSql = false; + do_trace_flag: + cmpp_outputer_cleanup(&pp->pimpl->sqlTrace.out); + if( isNoFlag ){ + pp->pimpl->sqlTrace.out = cmpp_outputer_empty; + }else{ + pp->pimpl->sqlTrace.out = cmpp_outputer_FILE; + pp->pimpl->sqlTrace.out.state = stderr; + } + } + } + ISFLAG("sql-trace-x"){ + NOVAL; + DOIT { + pp->pimpl->sqlTrace.expandSql = true; + goto do_trace_flag; + } + } + ISFLAG("chomp-F"){ + NOVAL; + DOIT pp->pimpl->flags.chompF = !isNoFlag; + } + ISFLAG2("d","delimiter"){ + ARGVAL; + DOIT { + cmpp_delimiter_set(pp, zVal); + } + } + ISFLAG2("dd", "dump-defines"){ + DOIT { + cmpp_FILE * const fp = + /* tcl's exec treats output to stderr as failure. + If we use [exec -ignorestderr] then it instead replaces + stderr's output with its own message, invalidating + test expectations. */ + 1 ? stdout : stderr; + fprintf(fp, "All %sdefine entries:\n", + cmpp__pp_zdelim(pp)); + cmpp__dump_defines(pp, fp, 1); + } + } +#if !defined(CMPP_OMIT_D_DB) + ISFLAG2("db", "db-file"){ + /* Undocumented flag used for testing purposes. */ + ARGVAL; + DOIT { + cmpp_db_name_set(pp, zVal); + } + } +#endif + ISFLAG("version"){ + NOVAL; +#if !defined(CMPP_OMIT_FILE_IO) + fprintf(stdout, "c-pp version %s\nwith SQLite %s %s\n", + cmpp_version(), + sqlite3_libversion(), + sqlite3_sourceid()); +#endif + doIt = 100; + break; + } +#if defined(CMPP_MAIN) && !defined(CMPP_MAIN_SAFEMODE) + ISFLAG("safe-mode"){ + if( i>0 ){ + cmpp_err_set(pp, CMPP_RC_MISUSE, + "--%s, if used, must be the first argument.", + zArg); + break; + } + } +#endif + else{ + cmpp__err(pp,CMPP_RC_MISUSE, + "Unhandled flag: %s", argv[i]); + } + } + DOIT { + if(!nFile){ + /* We got no file arguments, so read from stdin. */ + if(0 + && !pp->pimpl->flags.nIncludeDir + && cmpp_include_dir_add(pp, ".") ){ + break; + } + open_output_if_needed; + cmpp_process_file(pp, "-"); + } + } +#undef DOIT + } + return ppCode; +#undef ARGVAL +#undef M +#undef ISFLAG +#undef ISFLAG2 +#undef NOVAL +#undef open_output_if_needed +} + +void cmpp_process_argv_usage(char const *zAppName, cmpp_FILE *fOut){ +#if defined(CMPP_OMIT_FILE_IO) + (void)zAppName; (void)fOut; +#else + fprintf(fOut, "%s version %s\nwith SQLite %s %s\n", + zAppName ? zAppName : "c-pp", + cmpp_version(), + sqlite3_libversion(), + sqlite3_sourceid()); + fprintf(fOut, "Usage: %s [flags] [infile...]\n", zAppName); + fprintf(fOut, + "Flags and filenames may be in any order and " + "they are processed in that order.\n" + "\nFlags:\n"); +#define GAP " " +#define arg(F,D) fprintf(fOut,"\n %s\n" GAP "%s\n",F, D) +#if defined(CMPP_MAIN) && !defined(CMPP_MAIN_SAFEMODE) + arg("--safe-mode", + "Disables preprocessing directives which use the filesystem " + "or invoke external processes. If used, it must be the first " + "argument."); +#endif + + arg("-o|--outfile FILE","Send output to FILE (default=- (stdout)).\n" + GAP "Because arguments are processed in order, this should\n" + GAP "normally be given before -f."); + arg("-f|--file FILE","Process FILE (default=- (stdin)).\n" + GAP "All non-flag arguments are assumed to be the input files."); + arg("-e SCRIPT", + "Treat SCRIPT as a complete c-pp input and process it.\n" + GAP "Doing anything marginally useful with this requires\n" + GAP "using it several times, once per directive. It will not\n" + GAP "work with " CMPP_DEFAULT_DELIM "if but is fine for " + CMPP_DEFAULT_DELIM "expr, " + CMPP_DEFAULT_DELIM "assert, and " + CMPP_DEFAULT_DELIM "define."); + arg("-DXYZ[=value]","Define XYZ to the given value (default=1)."); + arg("-UXYZ","Undefine all defines matching glob XYZ."); + arg("-IXYZ","Add dir XYZ to the " CMPP_DEFAULT_DELIM "include path."); + arg("-LXYZ","Add dir XYZ to the loadable module search path."); + arg("-FXYZ=filename", + "Define XYZ to the raw contents of the given file.\n" + GAP "The file is not processed as by " CMPP_DEFAULT_DELIM"include.\n" + GAP "Maybe it should be. Or maybe we need a new flag for that."); + arg("-d|--delimiter VALUE", "Set directive delimiter to VALUE " + "(default=" CMPP_DEFAULT_DELIM ")."); + arg("--@policy retain|elide|error|off", + "Specifies how to handle @tokens@ (default=off).\n" + GAP "off = do not look for @tokens@\n" + GAP "retain = parse @tokens@ and retain any undefined ones\n" + GAP "elide = parse @tokens@ and elide any undefined ones\n" + GAP "error = parse @tokens@ and error out for any undefined ones" + ); + arg("-u|--undefined-policy NAME", + "Sets the policy for how to handle references to undefined key:\n" + GAP "null = treat them as empty/falsy. This is the default.\n" + GAP "error = trigger an error. This should probably be " + "the default." + ); + arg("-@", "Equivalent to --@policy=error."); + arg("-no-@", "Equivalent to --@policy=off (the default)."); + arg("--sql-trace", "Send a trace of all SQL to stderr."); + arg("--sql-trace-x", + "Like --sql-trace but expand all bound values in the SQL."); + arg("--no-sql-trace", "Disable SQL tracing (default)."); + arg("--chomp-F", "One trailing newline is trimmed from files " + "read via -FXYZ=filename."); + arg("--no-chomp-F", "Disable --chomp-F (default)."); +#undef arg +#undef GAP + fputs("\nFlags which require a value accept either " + "--flag=value or --flag value. " + "The exceptions are that the -D... and -F... flags " + "require their '=' to be part of the flag (because they " + "are parsed elsewhere).\n\n",fOut); +#endif /*CMPP_OMIT_FILE_IO*/ +} + +#if defined(CMPP_MAIN) /* add main() */ +int main(int argc, char const * const * argv){ + int rc = 0; + cmpp * pp = 0; + cmpp_flag32_t newFlags = 0 +#if defined(CMPP_MAIN_SAFEMODE) + | cmpp_ctor_F_SAFEMODE +#endif + ; + cmpp_b bArgs = cmpp_b_empty; + sqlite3_config(SQLITE_CONFIG_URI,1); + { + /* Copy argv to a string so we can #define it. This has proven + helpful in testing, debugging, and output validation. */ + for( int i = 0; i < argc; ++i ){ + if( i ) cmpp_b_append_ch(&bArgs,' '); + cmpp_b_append(&bArgs, argv[i], strlen(argv[i])); + } + if( (rc = bArgs.errCode) ) goto end; + if( argc>1 && cmpp__arg_is_flag("--safe-mode", argv[1], NULL) ){ + newFlags |= cmpp_ctor_F_SAFEMODE; + --argc; + ++argv; + } + } + cmpp_ctor_cfg const cfg = { + .flags = newFlags + }; + rc = cmpp_ctor(&pp, &cfg); + if( rc ) goto end; + /** + Define CMPP_MAIN_INIT to the name of a function with the signature + + int (*)(cmpp*) + + to have it called here. The intent is that custom directives can + be installed this way without having to edit this code. + */ +#if defined(CMPP_MAIN_INIT) + extern int CMPP_MAIN_INIT(cmpp*); + if( 0!=(rc = CMPP_MAIN_INIT(pp)) ){ + g_warn0("Initialization via CMPP_MAIN_INIT() failed"); + goto end; + } +#endif +#if defined(CMPP_MAIN_AUTOLOADER) + { + extern int CMPP_MAIN_AUTOLOADER(cmpp*,char const *,void*); + cmpp_d_autoloader al = cmpp_d_autoloader_empty; + al.f = CMPP_MAIN_AUTOLOADER; + cmpp_d_autoloader_set(pp, &al); + } +#endif + if( cmpp_define_v2(pp, "c-pp::argv", (char*)bArgs.z) ) goto end; + cmpp_b_clear(&bArgs); + rc = cmpp_process_argv(pp, argc-1, argv+1); + switch( rc ){ + case 0: break; + case CMPP_RC_HELP: + rc = 0; + cmpp_process_argv_usage(argv[0], stdout); + break; + default: + break; + } +end: + cmpp_b_clear(&bArgs); + if( pp ){ + char const *zErr = 0; + rc = cmpp_err_get(pp, &zErr); + if( rc && CMPP_RC_HELP!=rc ){ + g_warn("error %s: %s", cmpp_rc_cstr(rc), zErr); + } + cmpp_dtor(pp); + }else if( rc && CMPP_RC_HELP!=rc ){ + g_warn("error #%d/%s", rc, cmpp_rc_cstr(rc)); + } + sqlite3_shutdown(); + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} +#endif /* CMPP_MAIN */ +/* +** 2022-11-12: +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +** +************************************************************************ +** This file houses the cmpp_b-related parts of libcmpp. +*/ + +const cmpp_b cmpp_b_empty = cmpp_b_empty_m; + +CMPP__EXPORT(int, cmpp_b_append4)(cmpp * const pp, + cmpp_b * const os, + void const * src, + cmpp_size_t n){ + if( !ppCode && cmpp_b_append(os, src, n) ){ + cmpp_check_oom(pp, 0); + } + return ppCode; +} + +CMPP__EXPORT(int, cmpp_b_reserve3)(cmpp * const pp, + cmpp_b * const os, + cmpp_size_t n){ + if( !ppCode && cmpp_b_reserve(os, n) ){ + cmpp_check_oom(pp, 0); + } + return ppCode; +} + + +CMPP__EXPORT(void, cmpp_b_clear)(cmpp_b *s){ + if( s->z ) cmpp_mfree(s->z); + *s = cmpp_b_empty; +} + +CMPP__EXPORT(cmpp_b *, cmpp_b_reuse)(cmpp_b * const s){ + if( s->z ){ +#if 1 + memset(s->z, 0, s->nAlloc) + /* valgrind pushes for this, which is curious because + cmpp_b_reserve[3]() memset()s new space to 0. + + Try the following without this block using one commit after + [5f9c31d1da1d] (that'll be the commit that this comment and #if + block were added): + + ##define foo + ##if not defined a + ##/if + ##query define {select ?1 a} bind [1] + + There's a misuse complaint about a jump depending on + uninitialized memory deep under cmpp__is_int(), in strlen(), on + the "define" argument of the ##query. It does not appear if + the lines above it are removed, which indicates that it's at + least semi-genuine. gcc v13.3.0, if it matters. + */; +#else + s->z[0] = 0; +#endif + s->n = 0; + } + s->errCode = 0; + return s; +} + +CMPP__EXPORT(void, cmpp_b_swap)(cmpp_b * const l, cmpp_b * const r){ + if( l!=r ){ + cmpp_b const x = *l; + *l = *r; + *r = x; + } +} + +CMPP__EXPORT(int, cmpp_b_reserve)(cmpp_b *s, cmpp_size_t n){ + if( 0==s->errCode && s->nAlloc < n ){ + void * const m = cmpp_mrealloc(s->z, s->nAlloc + n); + if( m ){ + memset((unsigned char *)m + s->nAlloc, 0, (n - s->nAlloc)) + /* valgrind convincingly recommends this. */; + s->z = m; + s->nAlloc += n; + }else{ + s->errCode = CMPP_RC_OOM; + } + } + return s->errCode; +} + +CMPP__EXPORT(int, cmpp_b_append)(cmpp_b * os, void const *src, + cmpp_size_t n){ + if(0==os->errCode){ + cmpp_size_t const nNeeded = os->n + n + 1; + if( nNeeded>=os->nAlloc && cmpp_b_reserve(os, nNeeded) ){ + assert( CMPP_RC_OOM==os->errCode ); + return os->errCode; + } + memcpy(os->z + os->n, src, n); + os->n += n; + os->z[os->n] = 0; + if( 0 ) { + g_warn("n=%u z=[%.*s] nUsed=%d", (unsigned)n, (int)n, + (char const*) src, (int)os->n); + } + } + return os->errCode; +} + +CMPP__EXPORT(int, cmpp_b_append_ch)(cmpp_b * os, char ch){ + if( 0==os->errCode + && (os->n+1<os->nAlloc + || 0==cmpp_b_reserve(os, os->n+2)) ){ + os->z[os->n++] = (unsigned char)ch; + os->z[os->n] = 0; + } + return os->errCode; +} + +CMPP__EXPORT(int, cmpp_b_append_i32)(cmpp_b * os, int32_t d){ + if( 0==os->errCode ){ + char buf[16] = {0}; + int const n = snprintf(buf, sizeof(buf), "%" PRIi32, d); + cmpp_b_append(os, buf, (unsigned)n); + } + return os->errCode; +} + +CMPP__EXPORT(int, cmpp_b_append_i64)(cmpp_b * os, int64_t d){ + if( 0==os->errCode ){ + char buf[32] = {0}; + int const n = snprintf(buf, sizeof(buf), "%" PRIi64, d); + cmpp_b_append(os, buf, (unsigned)n); + } + return os->errCode; +} + +CMPP__EXPORT(bool, cmpp_b_chomp)(cmpp_b * b){ + return cmpp_chomp(b->z, &b->n); +} + +CMPP__EXPORT(void, cmpp_b_list_cleanup)(cmpp_b_list *li){ + while( li->nAlloc ){ + cmpp_b * const b = li->list[--li->nAlloc]; + if(b){ + cmpp_b_clear(b); + cmpp_mfree(b); + } + } + cmpp_mfree(li->list); + *li = cmpp_b_list_empty; +} + +CMPP__EXPORT(void, cmpp_b_list_reuse)(cmpp_b_list *li){ + while( li->n ){ + cmpp_b * const b = li->list[li->n--]; + if(b) cmpp_b_reuse(b); + } +} + +static cmpp_b * cmpp_b_list_push(cmpp_b_list *li){ + cmpp_b * p = 0; + assert( li->list ? li->nAlloc : 0==li->nAlloc ); + if( !cmpp_b_list_reserve(NULL, li, + cmpp__li_reserve1_size(li, 20)) ){ + p = li->list[li->n]; + if( p ){ + cmpp_b_reuse(p); + }else{ + p = cmpp_malloc(sizeof(*p)); + if( p ){ + li->list[li->n++] = p; + *p = cmpp_b_empty; + } + } + } + return p; +} + +/** + bsearch()/qsort() comparison for (cmpp_b**), sorting by size, + largest first and empty slots last. +*/ +static int cmpp_b__cmp_desc(const void *p1, const void *p2){ + cmpp_b const * const eL = *(cmpp_b const **)p1; + cmpp_b const * const eR = *(cmpp_b const **)p2; + if( eL==eR ) return 0; + else if( !eL ) return 1; + else if (!eR ) return -1; + return (int)(/*largest first*/eL->nAlloc - eR->nAlloc); +} + +/** + bsearch()/qsort() comparison for (cmpp_b**), sorting by size, + smallest first and empty slots last. +*/ +static int cmpp_b__cmp_asc(const void *p1, const void *p2){ + cmpp_b const * const eL = *(cmpp_b const **)p1; + cmpp_b const * const eR = *(cmpp_b const **)p2; + if( eL==eR ) return 0; + else if( !eL ) return 1; + else if (!eR ) return -1; + return (int)(/*smallest first*/eR->nAlloc - eL->nAlloc); +} + +/** + Sort li's buffer list using the given policy. NULL entries always + sort last. This is a no-op of how == cmpp_b_list_UNSORTED or + li->n<2. +*/ +static void cmpp_b_list__sort(cmpp_b_list * const li, + enum cmpp_b_list_e how){ + switch( li->n<2 ? cmpp_b_list_UNSORTED : how ){ + case cmpp_b_list_UNSORTED: + break; + case cmpp_b_list_DESC: + qsort(li->list, li->n, sizeof(cmpp_b*), cmpp_b__cmp_desc); + break; + case cmpp_b_list_ASC: + qsort(li->list, li->n, sizeof(cmpp_b*), cmpp_b__cmp_asc); + break; + } +} + +CMPP__EXPORT(cmpp_b *, cmpp_b_borrow)(cmpp *pp){ + cmpp__pi(pp); + cmpp_b_list * const li = &pi->recycler.buf; + cmpp_b * b = 0; + if( cmpp_b_list_UNSORTED==pi->recycler.bufSort ){ + pi->recycler.bufSort = cmpp_b_list_DESC; + cmpp_b_list__sort(li, pi->recycler.bufSort); + assert( cmpp_b_list_UNSORTED!=pi->recycler.bufSort + || pi->recycler.buf.n<2 ); + } + for( cmpp_size_t i = 0; i < li->n; ++i ){ + b = li->list[i]; + if( b ){ + li->list[i] = 0; + assert( !b->n && + "Someone wrote to a buffer after giving it back" ); + if( i < li->n-1 ){ + pi->recycler.bufSort = cmpp_b_list_UNSORTED; + } + return cmpp_b_reuse(b); + } + } + /** + Allocate the list entry now and then remove the buffer from it to + "borrow" it. We allocate now, instead of in cmpp_b_return(), so + that that function has no OOM condition (handling it properly in + higher-level code would be a mess). + */ + b = cmpp_b_list_push(li); + if( 0==cmpp_check_oom(pp, b) ) { + assert( b==li->list[li->n-1] ); + li->list[li->n-1] = 0; + } + return b; +} + +CMPP__EXPORT(void, cmpp_b_return)(cmpp *pp, cmpp_b *b){ + if( !b ) return; + cmpp__pi(pp); + cmpp_b_list * const li = &pi->recycler.buf; + for( cmpp_size_t i = 0; i < li->n; ++i ){ + if( !li->list[i] ){ + li->list[i] = cmpp_b_reuse(b); + pi->recycler.bufSort = cmpp_b_list_UNSORTED; + return; + } + } + assert( !"This shouldn't be possible - no slot in recycler.buf" ); + cmpp_b_clear(b); + cmpp_mfree(b); +} + +CMPP__EXPORT(int, cmpp_output_f_b)( + void * state, void const * src, cmpp_size_t n +){ + if( state ){ + return cmpp_b_append(state, src, n); + } + return 0; +} + +#if CMPP__OBUF +int cmpp__obuf_flush(cmpp__obuf * b); +int cmpp__obuf_write(cmpp__obuf * b, void const * src, cmpp_size_t n); +void cmpp__obuf_cleanup(cmpp__obuf * b); +int cmpp_output_f_obuf(void * state, void const * src, cmpp_size_t n); +int cmpp_flush_f_obuf(void * state); +void cmpp_outputer_cleanup_f_obuf(cmpp_outputer * o); +const cmpp__obuf cmpp__obuf_empty = cmpp__obuf_empty_m; +const cmpp_outputer cmpp_outputer_obuf = { + .out = cmpp_output_f_obuf, + .flush = cmpp_flush_f_obuf, + .cleanup = cmpp_outputer_cleanup_f_obuf +}; +#endif +/* +** 2025-11-07: +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +** +************************************************************************ +** This file houses the db-related pieces of libcmpp. +*/ + +/** + A proxy for sqlite3_prepare() which updates pp->pimpl->err on error. +*/ +static int cmpp__prepare(cmpp *pp, sqlite3_stmt **pStmt, + const char * zSql, ...){ + /* We need for pp->pimpl->stmt.sp* to work regardless of pending errors so + that we can, when appropriate, create the rollback statements. */ + sqlite3_str * str = sqlite3_str_new(pp->pimpl->db.dbh); + char * z = 0; + int n = 0; + va_list va; + assert( pp->pimpl->db.dbh ); + va_start(va, zSql); + sqlite3_str_vappendf(str, zSql, va); + va_end(va); + z = cmpp_str_finish(pp, str, &n); + if( z ){ + int const rc = sqlite3_prepare_v2(pp->pimpl->db.dbh, z, n, pStmt, 0); + cmpp__db_rc(pp, rc, z); + sqlite3_free(z); + } + return ppCode; +} + +sqlite3_stmt * cmpp__stmt(cmpp * pp, enum CmppStmt_e which, + bool prepEvenIfErr){ + if( !pp->pimpl->db.dbh && cmpp__db_init(pp) ) return NULL; + sqlite3_stmt ** q = 0; + char const * zSql = 0; + switch(which){ + default: + cmpp__fatal("Maintenance required: not a valid CmppStmt ID: %d", which); + return NULL; +#define E(N,S) case CmppStmt_ ## N: zSql = S; q = &pp->pimpl->stmt.N; break; + CmppStmt_map(E) +#undef E + } + assert( q ); + assert( zSql && *zSql ); + if( !*q && (!ppCode || prepEvenIfErr) ){ + cmpp__prepare(pp, q, "%s", zSql); + } + return *q; +} + +void cmpp__stmt_reset(sqlite3_stmt * const q){ + if( q ){ + sqlite3_clear_bindings(q); + sqlite3_reset(q); + } +} + +static inline int cmpp__stmt_is_sp(cmpp const * const pp, + sqlite3_stmt const * const q){ + return q==pp->pimpl->stmt.spBegin + || q==pp->pimpl->stmt.spRelease + || q==pp->pimpl->stmt.spRollback; +} + +int cmpp__step(cmpp * const pp, sqlite3_stmt * const q, bool resetIt){ + int rc = SQLITE_ERROR; + assert( q ); + if( !ppCode || cmpp__stmt_is_sp(pp,q) ){ + rc = sqlite3_step(q); + cmpp__db_rc(pp, rc, sqlite3_sql(q)); + } + if( resetIt /* even if ppCode!=0 */ ) cmpp__stmt_reset(q); + assert( 0!=rc ); + return rc; +} + + +/** + Expects an SQLITE_... result code and returns an approximate match + from cmpp_rc_e. It specifically treats SQLITE_ROW and SQLITE_DONE + as non-errors, returning 0 for those. +*/ +static int cmpp__db_errcode(sqlite3 * const db, int sqliteCode); +int cmpp__db_errcode(sqlite3 * const db, int sqliteCode){ + (void)db; + int rc = 0; + switch(sqliteCode & 0xff){ + case SQLITE_ROW: + case SQLITE_DONE: + case SQLITE_OK: rc = 0; break; + case SQLITE_NOMEM: rc = CMPP_RC_OOM; break; + case SQLITE_CORRUPT: rc = CMPP_RC_CORRUPT; break; + case SQLITE_TOOBIG: + case SQLITE_FULL: + case SQLITE_RANGE: rc = CMPP_RC_RANGE; break; + case SQLITE_NOTFOUND: rc = CMPP_RC_NOT_FOUND; break; + case SQLITE_PERM: + case SQLITE_AUTH: + case SQLITE_BUSY: + case SQLITE_LOCKED: + case SQLITE_READONLY: rc = CMPP_RC_ACCESS; break; + case SQLITE_CANTOPEN: + case SQLITE_IOERR: rc = CMPP_RC_IO; break; + case SQLITE_NOLFS: rc = CMPP_RC_UNSUPPORTED; break; + default: + //MARKER(("sqlite3_errcode()=0x%04x\n", rc)); + rc = CMPP_RC_DB; break; + } + return rc; +} + +int cmpp__db_rc(cmpp *pp, int dbRc, char const *zMsg){ + switch(dbRc){ + case 0: + case SQLITE_DONE: + case SQLITE_ROW: + return 0; + default: + return cmpp_err_set( + pp, cmpp__db_errcode(pp->pimpl->db.dbh, dbRc), + "SQLite error #%d: %s%s%s", + dbRc, + pp->pimpl->db.dbh + ? sqlite3_errmsg(pp->pimpl->db.dbh) + : "<no db handle>", + zMsg ? ": " : "", + zMsg ? zMsg : "" + ); + } +} + +/** + The base "define" impl. Requires q to be an INSERT for one of the + define tables and have the (t,k,v) columns set up to bind to ?1, + ?2, and ?3. +*/ +static +int cmpp__define_impl(cmpp * const pp, + sqlite3_stmt * const q, + unsigned char const * zKey, + cmpp_ssize_t nKey, + unsigned char const *zVal, + cmpp_ssize_t nVal, + int tType, + bool resetStmt){ + if( 0==ppCode){ + assert( q ); + nKey = cmpp__strlenu(zKey, nKey); + nVal = cmpp__strlenu(zVal, nVal); + if( 0==cmpp__bind_textn(pp, q, 2, zKey, (int)nKey) + && 0==cmpp__bind_int(pp, q, 1, tType) ){ + //g_stderr("zKey=%s\nzVal=%s\nzEq=%s\n", zKey, zVal, zEq); + /* TODO? if tType==cmpp_TT_Blob, bind it as a blob */ + if( zVal ){ + if( nVal ){ + cmpp__bind_textn(pp, q, 3, zVal, (int)nVal); + }else{ + /* Arguable */ + cmpp__bind_null(pp, q, 3); + } + }else{ + cmpp__bind_int(pp, q, 3, 1); + } + cmpp__step(pp, q, resetStmt); + g_debug(pp,2,("define: %s [%s]=[%.*s]\n", + cmpp_tt_cstr(tType), zKey, (int)nVal, zVal)); + } + } + return ppCode; +} + +int cmpp__define2(cmpp *pp, + unsigned char const * zKey, + cmpp_ssize_t nKey, + unsigned char const *zVal, + cmpp_ssize_t nVal, + cmpp_tt tType){ + sqlite3_stmt * const q = cmpp__stmt(pp, CmppStmt_defIns, false); + if( q ){ + cmpp__define_impl(pp, q, zKey, nKey, zVal, nVal, tType, true); + } + return ppCode; +} + +/** + The legacy variant of define() which accepts X=Y in zKey. This + continues to exist because it's convenient for passing args from + main(). +*/ +static int cmpp__define_legacy(cmpp *pp, const char * zKey, char const *zVal, + cmpp_tt ttype ){ + + if(ppCode) return ppCode; + CmppKvp kvp = CmppKvp_empty; + if( CmppKvp_parse(pp, &kvp, ustr_c(zKey), -1, + zVal + ? CmppKvp_op_none + : CmppKvp_op_eq1) ) { + return ppCode; + } + if( kvp.v.z ){ + if( zVal ){ + assert(!"cannot happen - CmppKvp_op_none will prevent it"); + return cmpp_err_set(pp, CMPP_RC_MISUSE, + "Cannot assign two values to [%.*s] [%.*s] [%s]", + kvp.k.n, kvp.k.z, kvp.v.n, kvp.v.z, zVal); + } + }else{ + kvp.v.z = (unsigned char const *)zVal; + kvp.v.n = zVal ? (int)strlen(zVal) : 0; + } + sqlite3_stmt * const q = cmpp__stmt(pp, CmppStmt_defIns, false); + if( !q ) return ppCode; + int64_t intCheck = 0; + switch( ttype ){ + case cmpp_TT_Unknown: + if(kvp.v.n){ + if( cmpp__is_int64(kvp.v.z, kvp.v.n, &intCheck) ){ + ttype = cmpp_TT_Int; + if( '+'==*kvp.v.z ){ + ++kvp.v.z; + --kvp.v.n; + } + }else{ + ttype = cmpp_TT_String; + } + }else if( kvp.v.z ){ + ttype = cmpp_TT_String; + }else{ + ttype = cmpp_TT_Int; + intCheck = 1 /* No value ==> value of 1. */; + } + break; + case cmpp_TT_Int: + if( !cmpp__is_int64(kvp.v.z, kvp.v.n, &intCheck) ){ + ttype = cmpp_TT_String; + } + break; + default: + break; + } + if( 0==cmpp__bind_textn(pp, q, 2, kvp.k.z, kvp.k.n) + && 0==cmpp__bind_int(pp, q, 1, ttype) ){ + //g_stderr("zKey=%s\nzVal=%s\nzEq=%s\n", zKey, zVal, zEq); + switch( ttype ){ + case cmpp_TT_Int: + cmpp__bind_int(pp, q, 3, intCheck); + break; + case cmpp_TT_Null: + cmpp__bind_null(pp, q, 3); + break; + default: + cmpp__bind_textn(pp, q, 3, kvp.v.z, (int)kvp.v.n); + break; + } + cmpp__step(pp, q, true); + g_debug(pp,2,("define: [%.*s]=[%.*s]\n", + kvp.k.n, kvp.k.z, + kvp.v.n, kvp.v.z)); + } + return ppCode; +} + +CMPP__EXPORT(int, cmpp_define_legacy)(cmpp *pp, const char * zKey, char const *zVal){ + return cmpp__define_legacy(pp, zKey, zVal, cmpp_TT_Unknown); +} + +CMPP__EXPORT(int, cmpp_define_v2)(cmpp *pp, const char * zKey, char const *zVal){ + return cmpp__define2(pp, ustr_c(zKey), -1, ustr_c(zVal), -1, + cmpp_TT_String); +} + +static +int cmpp__define_shadow(cmpp *pp, unsigned char const *zKey, + cmpp_ssize_t nKey, + unsigned char const *zVal, + cmpp_ssize_t nVal, + int ttype, + int64_t * pId){ + sqlite3_stmt * const q = cmpp__stmt(pp, CmppStmt_sdefIns, false); + if( q ){ + if( 0==cmpp__define_impl(pp, q, zKey, nKey, zVal, nVal, ttype, false) + && pId ){ + *pId = sqlite3_column_int64(q, 0); + assert( *pId ); + } + cmpp__stmt_reset(q); + } + return ppCode; +} + +CMPP__EXPORT(int, cmpp_define_shadow)(cmpp *pp, char const *zKey, + char const *zVal, int64_t *pId){ + assert( pId ); + return cmpp__define_shadow(pp, ustr_c(zKey), -1, + ustr_c(zVal), -1, cmpp_TT_String, pId); +} + +static +int cmpp__define_unshadow(cmpp *pp, unsigned char const *zKey, + cmpp_ssize_t nKey, int64_t id){ + sqlite3_stmt * const q = cmpp__stmt(pp, CmppStmt_sdefDel, false); + if( q ){ + cmpp__bind_textn(pp, q, 1, zKey, (int)nKey); + cmpp__bind_int(pp, q, 2, id); + cmpp__step(pp, q, true); + } + return ppCode; +} + +CMPP__EXPORT(int, cmpp_define_unshadow)(cmpp *pp, char const *zKey, int64_t id){ + return cmpp__define_unshadow(pp, ustr_c(zKey), -1, id); +} + +/* +** This sqlite3_trace_v2() callback outputs tracing info using +** ((cmpp*)c)->sqlTrace.pFile. +*/ +static int cmpp__db_sq3TraceV2(unsigned dx,void*c,void*p,void*x){ + switch(dx){ + case SQLITE_TRACE_STMT:{ + char const * const zSql = x; + cmpp * const pp = c; + cmpp__pi(pp); + if(pi->sqlTrace.out.out){ + char * const zExp = pi->sqlTrace.expandSql + ? sqlite3_expanded_sql((sqlite3_stmt*)p) + : 0; + sqlite3_str * const s = sqlite3_str_new(pi->db.dbh); + if( pi->dx ){ + cmpp__dx_append_script_info(pi->dx, s); + sqlite3_str_appendchar(s, 1, ':'); + sqlite3_str_appendchar(s, 1, ' '); + } + sqlite3_str_appendall(s, zExp ? zExp : zSql); + sqlite3_str_appendchar(s, 1, '\n'); + int const n = sqlite3_str_length(s); + if( n ){ + char * const z = sqlite3_str_finish(s); + if( z ){ + cmpp__out2(pp, &pi->sqlTrace.out, z, (cmpp_size_t)n); + sqlite3_free(z); + } + } + sqlite3_free(zExp); + } + break; + } + } + return 0; +} + +#include <sys/stat.h> +/* +** sqlite3 UDF which returns true if its argument refers to an +** accessible file, else false. +*/ +static void cmpp__udf_file_exists( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zName; + (void)(argc); /* Unused parameter */ + zName = (const char*)sqlite3_value_text(argv[0]); + if( 0!=zName ){ + struct stat sb; + sqlite3_result_int(context, stat(zName, &sb) + ? 0 + : S_ISREG(sb.st_mode)); + } +} + +static void cmpp__udf_truthy( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + (void)(argc); /* Unused parameter */ + assert(1==argc); + int buul = 0; + sqlite3_value * const sv = argv[0]; + switch( sqlite3_value_type(sv) ){ + case SQLITE_NULL: + break; + case SQLITE_FLOAT: + buul = 0.0!=sqlite3_value_double(sv); + break; + case SQLITE_INTEGER: + buul = 0!=sqlite3_value_int(sv); + break; + case SQLITE_TEXT: + case SQLITE_BLOB:{ + int const n = sqlite3_value_bytes(sv); + if( n>1 ) buul = 1; + else if( 1==n ){ + const char *z = + (const char*)sqlite3_value_text(sv); + buul = z + ? 0!=strcmp(z,"0") + : 0; + } + } + } + sqlite3_result_int(context, buul); +} + +/** + SQLite3 UDF which compares its two arguments using memcmp() + semantics. NULL will compare equal to NULL, but less than anything + else. +*/ +static void cmpp__udf_compare( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + (void)(argc); /* Unused parameter */ + assert(2==argc); + sqlite3_value * const v1 = argv[0]; + sqlite3_value * const v2 = argv[1]; + unsigned char const * const z1 = sqlite3_value_text(v1); + unsigned char const * const z2 = sqlite3_value_text(v2); + int const n1 = sqlite3_value_bytes(v1); + int const n2 = sqlite3_value_bytes(v2); + int rv; + if( !z1 ){ + rv = z2 ? -1 : 0; + }else if( !z2 ){ + rv = 1; + }else{ + rv = strncmp((char const *)z1, (char const *)z2, n1>n2 ? n1 : n2); + } + if(0) g_stderr("udf_compare (%s,%s) = %d\n", z1, z2, rv); + sqlite3_result_int(context, rv); +} + +int cmpp__db_init(cmpp *pp){ + cmpp__pi(pp); + if( pi->db.dbh || ppCode ) return ppCode; + int rc; + char * zErr = 0; + const char * zDrops = + "BEGIN EXCLUSIVE;" + "DROP TABLE IF EXISTS " CMPP__DB_MAIN_NAME ".def;" + "DROP TABLE IF EXISTS " CMPP__DB_MAIN_NAME ".incl;" + "DROP TABLE IF EXISTS " CMPP__DB_MAIN_NAME ".inclpath;" + "DROP TABLE IF EXISTS " CMPP__DB_MAIN_NAME ".predef;" + "DROP TABLE IF EXISTS " CMPP__DB_MAIN_NAME ".ttype;" + "DROP VIEW IF EXISTS " CMPP__DB_MAIN_NAME ".vdef;" + "COMMIT;" + ; + const char * zSchema = + "BEGIN EXCLUSIVE;" + "CREATE TABLE " CMPP__DB_MAIN_NAME ".def(" + /* ^^^ defines */ + "t INTEGER DEFAULT NULL," + /*^^ type: cmpp_tt or NULL */ + "k TEXT PRIMARY KEY NOT NULL," + "v TEXT DEFAULT NULL" + ") WITHOUT ROWID;" + + "CREATE TABLE " CMPP__DB_MAIN_NAME ".incl(" + /* ^^^ files currently being included */ + "file TEXT PRIMARY KEY NOT NULL," + "srcFile TEXT DEFAULT NULL," + "srcLine INTEGER DEFAULT 0" + ") WITHOUT ROWID;" + + "CREATE TABLE " CMPP__DB_MAIN_NAME ".inclpath(" + /* ^^^ include path. We use (ORDER BY priority DESC, rowid) to + make their priority correct. priority should only be set by the + #include directive for its cwd entry. */ + "priority INTEGER DEFAULT 0," /* higher sorts first */ + "dir TEXT UNIQUE NOT NULL ON CONFLICT IGNORE" + ");" + + "CREATE TABLE " CMPP__DB_MAIN_NAME ".modpath(" + /* ^^^ module path. We use ORDER BY ROWID to make their + priority correct. */ + "dir TEXT PRIMARY KEY NOT NULL ON CONFLICT IGNORE" + ");" + + "CREATE TABLE " CMPP__DB_MAIN_NAME ".predef(" + /* ^^^ pre-defines */ + "t INTEGER DEFAULT NULL," /* a cmpp_tt or NULL */ + "k TEXT PRIMARY KEY NOT NULL," + "v TEXT DEFAULT NULL" + ") WITHOUT ROWID;" + "INSERT INTO " CMPP__DB_MAIN_NAME ".predef (t,k,v)" + " VALUES(NULL,'cmpp::version','" CMPP_VERSION "')" + ";" + + /** + sdefs - "scoped defines" or "shadow defines". The problem these + solve is the one of supporting a __FILE__ define in cmpp input + sources, such that it remains valid both before and after an + #include, but has a new name in the scope of an #include. We + can't use savepoints for that because they're a nuclear option + affecting _all_ #defines in the #include'd file, whereas we + normally want #defines to stick around across files. + + See cmpp_define_shadow() and cmpp_define_unshadow(). + */ + "CREATE TABLE " CMPP__DB_MAIN_NAME ".sdef(" + "id INTEGER PRIMARY KEY AUTOINCREMENT," + "t INTEGER DEFAULT NULL," /* a cmpp_tt or NULL */ + "k TEXT NOT NULL," + "v TEXT DEFAULT NULL" + ");" + + /** + vdef is a view consolidating the various #define stores. It's + intended to be used for all general-purpose fetching of defines + and it orders the results such that the library's defines + supercede all others, then scoped keys, then client-level + defines. + + To push a new sdef we simply insert into sdef. Then vdef will + order the newest sdef before any entry from the def table. + */ + "CREATE VIEW " CMPP__DB_MAIN_NAME ".vdef(source,t,k,v) AS" + " SELECT NULL,t,k,v FROM " CMPP__DB_MAIN_NAME ".predef" + /* ------^^^^ sorts before numbers */ + " UNION ALL" + " SELECT -rowid,t,k,v FROM " CMPP__DB_MAIN_NAME ".sdef" + /* ^^^^ sorts newest of matching keys first */ + " UNION ALL" + " SELECT 0,t,k,v FROM " CMPP__DB_MAIN_NAME ".def" + " ORDER BY 1, 3" + ";" + +#if 0 + "CREATE TABLE " CMPP__DB_MAIN_NAME ".ttype(" + /* ^^^ token types */ + "t INTEGER PRIMARY KEY NOT NULL," + /*^^ type: cmpp_tt */ + "n TEXT NOT NULL," + /*^^ cmpp_TT_... name. */ + "s TEXT DEFAULT NULL" + /* Symbolic or directive name, if any. */ + ");" +#endif + + "COMMIT;" + "BEGIN EXCLUSIVE;" + ; + cmpp__err_clear(pp); + int openFlags = SQLITE_OPEN_READWRITE; + if( pi->db.zName ){ + openFlags |= SQLITE_OPEN_CREATE; + } + rc = sqlite3_open_v2( + pi->db.zName ? pi->db.zName : ":memory:", + &pi->db.dbh, openFlags, 0); + if(rc){ + cmpp__db_rc(pp, rc, pi->db.zName + ? pi->db.zName + : ":memory:"); + sqlite3_close(pi->db.dbh); + pi->db.dbh = 0; + assert(ppCode); + return rc; + } + sqlite3_busy_timeout(pi->db.dbh, 5000); + sqlite3_db_config(pi->db.dbh, SQLITE_DBCONFIG_MAINDBNAME, + CMPP__DB_MAIN_NAME); + rc = sqlite3_trace_v2(pi->db.dbh, SQLITE_TRACE_STMT, + cmpp__db_sq3TraceV2, pp); + if( cmpp__db_rc(pp, rc, "Installing tracer failed") ){ + goto end; + } + //g_warn("Schema:\n%s\n",zSchema); + struct { + /* SQL UDFs */ + char const * const zName; + void (*xUdf)(sqlite3_context *,int,sqlite3_value **); + int arity; + int flags; + } aFunc[] = { + { + .zName = "cmpp_file_exists", + .xUdf = cmpp__udf_file_exists, + .arity = 1, + .flags = SQLITE_UTF8 | SQLITE_DIRECTONLY + }, + { + .zName = "cmpp_truthy", + .xUdf = cmpp__udf_truthy, + .arity = 1, + .flags = SQLITE_UTF8 | SQLITE_DIRECTONLY | SQLITE_DETERMINISTIC + }, + { + .zName = "cmpp_compare", + .xUdf = cmpp__udf_compare, + .arity = 2, + .flags = SQLITE_UTF8 | SQLITE_DIRECTONLY | SQLITE_DETERMINISTIC + } + }; + assert( 0==rc ); + for( unsigned int i = 0; 0==rc && i < sizeof(aFunc)/sizeof(aFunc[0]); ++i ){ + rc = sqlite3_create_function( + pi->db.dbh, aFunc[i].zName, aFunc[i].arity, + aFunc[i].flags, 0, aFunc[i].xUdf, 0, 0 + ); + } + if( cmpp__db_rc(pp, rc, "UDF registration failed.") ){ + return ppCode; + } + if( pi->db.zName ){ + /* Drop all cmpp tables when using a persistent db so that we are + not beholden to a given structure. TODO: a config flag to + toggle this. */ + rc = sqlite3_exec(pi->db.dbh, zDrops, 0, 0, &zErr); + } + if( !rc ){ + rc = sqlite3_exec(pi->db.dbh, zSchema, 0, 0, &zErr); + } + + if( !rc ){ + extern int sqlite3_series_init(sqlite3 *, char **, const sqlite3_api_routines *); + rc = sqlite3_series_init(pi->db.dbh, &zErr, NULL); + } + + if(rc){ + if( zErr ){ + cmpp_err_set(pp, cmpp__db_errcode(pi->db.dbh, rc), + "SQLite error #%d initializing DB: %s", rc, zErr); + sqlite3_free(zErr); + }else{ + cmpp_err_set(pp, cmpp__db_errcode(pi->db.dbh, rc), + "SQLite error #%d initializing DB", rc); + } + goto end; + } + + while(0){ + /* Insert the ttype mappings. We don't yet make use of this but + only for lack of a use case ;). */ + sqlite3_stmt * const q = cmpp__stmt(pp, CmppStmt_insTtype, false); + if( !q ) goto end; +#define E(N,STR) \ + cmpp__bind_int(pp, q, 1, cmpp_TT_ ## N); \ + cmpp__bind_textn(pp, q, 2, \ + ustr_c("cmpp_TT_ " # N), sizeof("cmpp_TT_" # N)-1); \ + if( STR ) cmpp__bind_textn(pp, q, 3, ustr_c(STR), sizeof(STR)-1); \ + else cmpp__bind_null(pp, q, 3); \ + if( SQLITE_DONE!=cmpp__step(pp, q, true) ) return ppCode; + cmpp_tt_map(E) +#undef E + sqlite3_finalize(q); + pi->stmt.insTtype = 0; + break; + } + +end: + if( !ppCode ){ + /* + ** Keep us from getting in the situation later that delayed + ** preparation if one of the savepoint statements fails (e.g. due + ** to OOM or memory corruption). + */ + cmpp__stmt(pp, CmppStmt_spBegin, false); + cmpp__stmt(pp, CmppStmt_spRelease, false); + cmpp__stmt(pp, CmppStmt_spRollback, false); + cmpp__lazy_init(pp); + } + return ppCode; +} +/* +** 2022-11-12: +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +** +************************************************************************ +** This file houses the core cmpp_dx_f() implementations of libcmpp. +*/ + +static int cmpp__dx_err_just_once(cmpp_dx *dx, cmpp_arg const *arg){ + return cmpp_dx_err_set(dx, CMPP_RC_MISUSE, "'%s' may only be used once.", + arg->z); +} + +/* No-op cmpp_dx_f() impl. */ +static void cmpp_dx_f_noop(cmpp_dx *dx){ + (void)dx; +} + +/** + cmpp_kav_each_f() impl for use by #define {k->v}. +*/ +static int cmpp_kav_each_f_define__group( + cmpp_dx *dx, + unsigned char const *zKey, cmpp_size_t nKey, + unsigned char const *zVal, cmpp_size_t nVal, + void* callbackState +){ + if( (callbackState==dx) + && cmpp_has(dx->pp, (char const*)zKey, nKey) ){ + return dxppCode; + } + return cmpp__define2(dx->pp, zKey, nKey, zVal, nVal, cmpp_TT_String); +} + +/* #error impl. */ +static void cmpp_dx_f_error(cmpp_dx *dx){ + const char *zBegin = (char const *)dx->args.z; + unsigned n = (unsigned)dx->args.nz; + if( n>2 && (('"' ==*zBegin || '\''==*zBegin) && zBegin[n-1]==*zBegin) ){ + ++zBegin; + n -= 2; + } + if( n ){ + cmpp_dx_err_set(dx, CMPP_RC_ERROR, "%.*s", n, zBegin); + }else{ + cmpp_dx_err_set(dx, CMPP_RC_ERROR, "(no additional info)"); + } +} + +/* Impl. for #define. */ +static void cmpp_dx_f_define(cmpp_dx *dx){ + cmpp_d const * const d = dx->d; + assert(d); + if( !dx->args.arg0 ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Expecting one or more arguments"); + return; + } + cmpp_arg const * aKey = 0; + int argNdx = 0; + int nChomp = 0; + unsigned nHeredoc = 0; + unsigned char acHeredoc[128] = {0} /* TODO: cmpp_args_clone() */; + bool ifNotDefined = false /* true if '?' arg */; + cmpp_arg const *aAppend = 0; +#define checkIsDefined(ARG) \ + if(ifNotDefined && (cmpp_has(dx->pp, (char const*)ARG->z, ARG->n) \ + || dxppCode)) break + + for( cmpp_arg const * arg = dx->args.arg0; + 0==dxppCode && arg; + arg = arg->next, ++argNdx ){ + //g_warn("arg=%s", arg->z); + if( 0==argNdx && cmpp_arg_equals(arg, "?") ){ + /* Only set the key if it's not already defined. */ + ifNotDefined = true; + continue; + } + switch( arg->ttype ){ + case cmpp_TT_ShiftL3: + ++nChomp; + /* fall through */ + case cmpp_TT_ShiftL: + if( arg->next || argNdx<1 ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Ill-placed '%s'.", arg->z); + }else if( arg->n >= sizeof(acHeredoc)-1 ){ + cmpp_dx_err_set(dx, CMPP_RC_RANGE, + "Heredoc name is too large."); + }else if( !aKey ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Missing key before %s.", + cmpp__tt_cstr(cmpp_TT_ShiftL, false)); + }else{ + assert( aKey ); + nHeredoc = aKey->n; + memcpy(acHeredoc, aKey->z, aKey->n+1/*NUL*/); + } + break; + case cmpp_TT_OpEq: + if( 1 /*seenEq || argNdx!=1 || !arg->next*/ ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Ill-placed '%s'.", arg->z); + break; + } + continue; + case cmpp_TT_StringAt: + if( cmpp__StringAtIsOk(dx->pp, cmpp_atpol_CURRENT) ){ + break; + } + /* fall through */ + case cmpp_TT_Int: + case cmpp_TT_String: + case cmpp_TT_Word: + if( cmpp_arg_isflag(arg,"-chomp") ){ + ++nChomp; + break; + } + if( cmpp_arg_isflag(arg,"-append") + || cmpp_arg_isflag(arg,"-a") ){ + aAppend = arg->next; + if( !aAppend ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Expecting argument for %s", + arg->z); + } + arg = aAppend; + break; + } + if( aKey ){ + /* This is the second arg - the value */ + checkIsDefined(aKey); + cmpp_b * const os = cmpp_b_borrow(dx->pp); + cmpp_b * const ba = aAppend ? cmpp_b_borrow(dx->pp) : 0; + while( os ){ + if( ba ){ + cmpp__get_b(dx->pp, aKey->z, aKey->n, ba, false); + if( dxppCode ) break; + if( 0 ){ + g_warn("key=%s\n", aKey->z); + g_warn("ba=%u %.*s\n", ba->n, ba->n, ba->z); + } + } + if( cmpp_arg_to_b(dx, arg, os, + cmpp_arg_to_b_F_BRACE_CALL) ) break; + cmpp_b * const which = (ba && ba->n) ? ba : os; + if( which==ba && os->n ){ + if( ba->n ) cmpp_b_append4(dx->pp, ba, aAppend->z, aAppend->n); + cmpp_b_append4(dx->pp, ba, os->z, os->n); + } + cmpp__define2(dx->pp, aKey->z, aKey->n, which->z, which->n, + arg->ttype); + if( 0 ){ + g_warn("aKey=%u z=[%.*s]\n", aKey->n, (int)aKey->n, aKey->z); + g_warn("nExp=%u z=[%.*s]\n", which->n, (int)which->n, which->z); + } + break; + } + cmpp_b_return(dx->pp, os); + cmpp_b_return(dx->pp, ba); + aKey = 0; + }else if( cmpp_TT_Word!=arg->ttype ){ + cmpp_dx_err_set(dx, CMPP_RC_TYPE, + "Expecting a define-name token here."); + }else if( arg->next ){ + aKey = arg; + }else{ + /* No value = a value of 1. */ + checkIsDefined(arg); + cmpp__define2(dx->pp, arg->z, arg->n, + ustr_c("1"), 1, cmpp_TT_Int); + } + break; + case cmpp_TT_GroupSquiggly: + assert( !acHeredoc[0] ); + if( (ifNotDefined ? argNdx>1 : argNdx>0) || arg->next ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "{...} must be the only argument.") + /* This is for simplicity's sake. */; + }else{ + cmpp_kav_each(dx, arg->z, arg->n, + cmpp_kav_each_f_define__group, + ifNotDefined ? dx : NULL, + cmpp_kav_each_F_NOT_EMPTY + | cmpp_kav_each_F_CALL_VAL + | cmpp_kav_each_F_PARENS_EXPR + //TODO cmpp_kav_each_F_IF_UNDEF + ); + } + aKey = 0; + break; + case cmpp_TT_GroupParen:{ + if( !aKey ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "(...) is not permitted as a key."); + break; + } + checkIsDefined(aKey); + int d = 0; + if( 0==cmpp__arg_evalSubToInt(dx, arg, &d) ){ + char exprBuf[32] = {0}; + cmpp_size_t nVal = + (cmpp_size_t)snprintf(&exprBuf[0], + sizeof(exprBuf), "%d", d); + assert(nVal>0); + cmpp__define2(dx->pp, aKey->z, aKey->n, + ustr_c(&exprBuf[0]), nVal, cmpp_TT_Int); + } + break; + } + case cmpp_TT_GroupBrace:{ + if( !aKey ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "[...] is not permitted as a key."); + break; + } + checkIsDefined(aKey); + cmpp_b * const b = cmpp_b_borrow(dx->pp); + if( b && 0==cmpp_call_str(dx->pp, arg->z, arg->n, b, 0) ){ + cmpp__define2(dx->pp, aKey->z, aKey->n, + b->z, b->n, cmpp_TT_AnyType); + } + cmpp_b_return(dx->pp, b); + break; + } + default: + // TODO: treat (...) as an expression + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Unhandled arg type %s: %s", + cmpp__tt_cstr(arg->ttype, true), arg->z); + break; + } + } + if( 0==nHeredoc && nChomp ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "-chomp can only be used with <<."); + } + if( 0==dxppCode && nHeredoc ){ + // Process (#define KEY <<) + cmpp_b * const os = cmpp_b_borrow(dx->pp); + assert( dx->d->closer ); + if( os && + 0==cmpp_dx_consume_b(dx, os, &dx->d->closer, 1, + cmpp_dx_consume_F_PROCESS_OTHER_D) ){ + while( nChomp-- && cmpp_b_chomp(os) ){} + g_debug(dx->pp,2,("define heredoc: [%s]=[%.*s]\n", + acHeredoc, (int)os->n, os->z)); + if( !ifNotDefined + || !cmpp_has(dx->pp, (char const*)acHeredoc, nHeredoc) ){ + cmpp__define2( + dx->pp, acHeredoc, nHeredoc, os->z, os->n, cmpp_TT_String + ); + } + } + cmpp_b_return(dx->pp, os); + } +#undef checkIsDefined + return; +} + +/* Impl. for #undef */ +static void cmpp_dx_f_undef(cmpp_dx *dx){ + if( !dx->args.arg0 ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Expecting one or more arguments"); + return; + } + cmpp_d const * const d = dx->d; + for( cmpp_arg const * arg = dx->args.arg0; + 0==dxppCode && arg; + arg = arg->next ){ + if( 0 ){ + g_stderr(" %s: %s %p n=%d %.*s\n", d->name.z, + cmpp__tt_cstr(arg->ttype, true), arg->z, + (int)arg->n, (int)arg->n, arg->z); + } + if( cmpp_TT_Word==arg->ttype ){ +#if 0 + /* Too strict? */ + if( 0==cmpp__legal_key_check(dx->pp, arg->z, + (cmpp_ssize_t)arg->n, false) ) { + cmpp_undef(dx->pp, (char const *)arg->z); + } +#else + cmpp_undef(dx->pp, (char const *)arg->z, NULL); +#endif + }else{ + cmpp_err_set(dx->pp, CMPP_RC_MISUSE, "Invalid arg for %s: %s", + d->name.z, arg->z); + } + } +} + +/* Impl. for #once. */ +static void cmpp_dx_f_once(cmpp_dx *dx){ + cmpp_d const * const d = dx->d; + assert(d); + assert(d->closer); + if( dx->args.arg0 ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Expecting no arguments"); + return; + } + cmpp_dx_pimpl * const dxp = dx->pimpl; + cmpp_b * const b = cmpp_b_borrow(dx->pp); + if( !b ) return; + cmpp_b_append_ch(b, '#'); + cmpp_b_append4(dx->pp, b, d->name.z, d->name.n); + cmpp_b_append_ch(b, ':'); + cmpp__get_b(dx->pp, ustr_c("__FILE__"), 8, b, true) + /* Wonky return semantics. */; + if( b->errCode + || dxppCode + || cmpp_b_append_ch(b, ':') + || cmpp_b_append_i32(b, (int)dxp->pos.lineNo) ){ + goto end; + } + //g_debug(dx->pp,1,("#once key: %s", b->z)); + int const had = cmpp_has(dx->pp, (char const *)b->z, b->n); + if( dxppCode ) goto end; + else if( had ){ + CmppLvl * const lvl = CmppLvl_push(dx); + if( lvl ){ + CmppLvl_elide(lvl, true); + cmpp_outputer devNull = cmpp_outputer_empty; + cmpp_dx_consume(dx, &devNull, &d->closer, 1, + cmpp_dx_consume_F_PROCESS_OTHER_D); + CmppLvl_pop(dx, lvl); + } + }else if( !cmpp_define_v2(dx->pp, (char const*)b->z, "1") ){ + cmpp_dx_consume(dx, NULL, &d->closer, 1, + cmpp_dx_consume_F_PROCESS_OTHER_D); + } +end: + cmpp_b_return(dx->pp, b); + return; +} + + +/* Impl. for #/define, /#query, /#pipe. */ +CMPP__EXPORT(void, cmpp_dx_f_dangling_closer)(cmpp_dx *dx){ + cmpp_d const * const d = dx->d; + char const * const zD = cmpp_dx_delim(dx); + dxserr("%s%s used without its opening directive.", + zD, d->name.z); +} + +#ifndef CMPP_OMIT_D_INCLUDE +static int cmpp__including_has(cmpp *pp, unsigned const char * zName){ + int rc = 0; + sqlite3_stmt * const q = cmpp__stmt(pp, CmppStmt_inclHas, false); + if( q && 0==cmpp__bind_text(pp, q, 1, zName) ){ + if(SQLITE_ROW == cmpp__step(pp, q, true)){ + rc = 1; + }else{ + rc = 0; + } + g_debug(pp,2,("inclpath has [%s] = %d\n",zName, rc)); + } + return rc; +} + +/** + Returns a resolved path of PREFIX+'/'+zKey, where PREFIX is one of + the `#include` dirs (cmpp_include_dir_add()). If no file match is + found, NULL is returned. Memory must eventually be passed to + cmpp_mfree() to free it. +*/ +static char * cmpp__include_search(cmpp *pp, unsigned const char * zKey, + int * nVal){ + char * zName = 0; + sqlite3_stmt * const q = cmpp__stmt(pp, CmppStmt_inclSearch, false); + if( nVal ) *nVal = 0; + if( q && 0==cmpp__bind_text(pp, q, 1, zKey) ){ + int const rc = cmpp__step(pp, q, false); + if(SQLITE_ROW==rc){ + const unsigned char * z = sqlite3_column_text(q, 0); + int const n = sqlite3_column_bytes(q,0); + zName = n ? sqlite3_mprintf("%.*s", n, z) : 0; + if( n ) cmpp_check_oom(pp, zName); + if( nVal ) *nVal = n; + } + cmpp__stmt_reset(q); + } + return zName; +} + +/** + Removes zKey from the currently-being-`#include`d list + list. +*/ +static int cmpp__include_rm(cmpp *pp, unsigned const char * zKey){ + sqlite3_stmt * const q = cmpp__stmt(pp, CmppStmt_inclDel, false); + if( q ){ + cmpp__bind_text(pp, q, 1, ustr_c(zKey)); + cmpp__step(pp, q, true); + g_debug(pp,2,("incl rm [%s]\n", zKey)); + } + return ppCode; +} + +#if 0 +/* +** Sets pp's error state if the `#include` list contains the given +** key. +*/ +static int cmpp__including_check(cmpp *pp, const char * zKey); +int cmpp__including_check(cmpp *pp, const char * zName){ + if( !ppCode ){ + if(cmpp__including_has(pp, zName)){ + cmpp__err(pp, CMPP_RC_MISUSE, + "Recursive include detected: %s\n", zName); + } + } + return ppCode; +} +#endif + + +/** + Adds the given filename to the list of being-`#include`d files, + using the given source file name and line number of error reporting + purposes. If recursion is later detected. +*/ +static int cmpp__including_add(cmpp *pp, unsigned const char * zKey, + unsigned const char * zSrc, cmpp_size_t srcLine){ + sqlite3_stmt * const q = cmpp__stmt(pp, CmppStmt_inclIns, false); + if( q ){ + cmpp__bind_text(pp, q, 1, zKey); + cmpp__bind_text(pp, q, 2, zSrc); + cmpp__bind_int(pp, q, 3, srcLine); + cmpp__step(pp, q, true); + g_debug(pp,2,("is-including-file add [%s] from [%s]:%" + CMPP_SIZE_T_PFMT "\n", zKey, zSrc, srcLine)); + } + return ppCode; +} + +/* Impl. for #include. */ +static void cmpp_dx_f_include(cmpp_dx *dx){ + char * zResolved = 0; + int nResolved = 0; + cmpp_b * const ob = cmpp_b_borrow(dx->pp); + bool raw = false; + cmpp_args args = cmpp_args_empty; + if( !ob || cmpp_dx_args_clone(dx, &args) ){ + goto end; + } + assert(args.pimpl && args.pimpl->pp==dx->pp); + cmpp_arg const * arg = args.arg0; + for( ; arg; arg = arg->next){ +#define FLAG(X)if( cmpp_arg_isflag(arg, X) ) + FLAG("-raw"){ + raw = true; + continue; + } + break; +#undef FLAG + } + if( !arg ){ + cmpp_dx_err_set(dx, CMPP_RC_SYNTAX, + "Expecting at least one filename argument."); + } + for( ; !dxppCode && arg; arg = arg->next ){ + cmpp_flag32_t a2bf = cmpp_arg_to_b_F_BRACE_CALL; + if( cmpp_TT_Word==arg->ttype && cmpp__arg_wordIsPathOrFlag(arg) ){ + a2bf |= cmpp_arg_to_b_F_NO_DEFINES; + } + if( cmpp_arg_to_b(dx, arg, cmpp_b_reuse(ob), a2bf) ){ + break; + } + //g_stderr("zFile=%s zResolved=%s\n", zFile, zResolved); + if(!raw && cmpp__including_has(dx->pp, ob->z)){ + /* Note that different spellings of the same filename + ** will elude this check, but that seems okay, as different + ** spellings means that we're not re-running the exact same + ** invocation. We might want some other form of multi-include + ** protection, rather than this, however. There may well be + ** sensible uses for recursion. */ + cmpp_dx_err_set(dx, CMPP_RC_RANGE, "Recursive include of file: %s", + ob->z); + break; + } + cmpp_mfree(zResolved); + nResolved = 0; + zResolved = cmpp__include_search(dx->pp, ob->z, &nResolved); + if(!zResolved){ + if( !dxppCode ){ + cmpp_dx_err_set(dx, CMPP_RC_NOT_FOUND, "file not found: %s", ob->z); + } + break; + } + if( raw ){ + if( !dx->pp->pimpl->out.out ) break; + FILE * const fp = cmpp_fopen(zResolved, "r"); + if( fp ){ + int const rc = cmpp_stream(cmpp_input_f_FILE, fp, + dx->pp->pimpl->out.out, + dx->pp->pimpl->out.state); + if( rc ){ + cmpp_dx_err_set(dx, rc, "Unknown error streaming file %s.", + arg->z); + } + cmpp_fclose(fp); + }else{ + cmpp_dx_err_set(dx, cmpp_errno_rc(errno, CMPP_RC_IO), + "Unknown error opening file %s.", arg->z); + } + }else{ + cmpp__including_add(dx->pp, ob->z, ustr_c(dx->sourceName), + dx->pimpl->dline.lineNo); + cmpp_process_file(dx->pp, zResolved); + cmpp__include_rm(dx->pp, ob->z); + } + } +end: + cmpp_mfree(zResolved); + cmpp_args_cleanup(&args); + cmpp_b_return(dx->pp, ob); +} +#endif /* #ifndef CMPP_OMIT_D_INCLUDE */ + +/** + cmpp_dx_f() callback state for cmpp_dx_f_if(): pointers to the + various directives of that family. +*/ +struct CmppIfState { + cmpp_d * dIf; + cmpp_d * dElif; + cmpp_d * dElse; + cmpp_d const * dEndif; +}; +typedef struct CmppIfState CmppIfState; + +/* Version 2 of #if. */ +static void cmpp_dx_f_if(cmpp_dx *dx){ + /* Reminder to self: + + We need to be able to recurse, even in skip mode, for #if nesting + to work. That's not great because it means we are evaluating + stuff we ideally should be skipping over, but it's keeping the + current tests working as-is. We can/do, however, avoid evaluating + expressions and such when recursing via skip mode. If we can + eliminate that here, by keeping track of the #if stack depth, + then we can possibly eliminate the whole CmppLvl_F_ELIDE + flag stuff. + + The more convoluted version 1 #if (which this replaced not hours + ago) kept track of the skip state across a separate directive + function for #if and #/if. That was more complex but did avoid + having to recurse into #if in order to straighten out #elif and + #else. Update: tried a non-recursive variant involving moving + this function's gotTruth into the CmppLvl object() and + managing the CmppLvl stack here, but it just didn't want to + work for me and i was too tired to figure out why. + */ + int gotTruth = 0 /*expr result*/; + CmppIfState const * const cis = dx->d->impl.state; + cmpp_d const * dClosers[] = { + cis->dElif, cis->dElse, cis->dEndif + }; + CmppLvl * lvl = 0; + CmppDLine const dline = dx->pimpl->dline; + cmpp_args args = cmpp_args_empty; + char delim[20] = {0}; +#define skipOn CmppLvl_elide((lvl), true) +#define skipOff CmppLvl_elide((lvl), false) + + assert( dx->d==cis->dIf ); + if( !dx->args.arg0 ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, "Expecting an expression."); + return; + } + snprintf(delim, sizeof(delim), "%s", cmpp_dx_delim(dx)); + delim[sizeof(delim)-1] = 0; + lvl = CmppLvl_push(dx); + if( !lvl ) goto end; + if( cmpp_dx_is_eliding(dx) ){ + gotTruth = 1; + }else if( cmpp__args_evalToInt(dx, &dx->pimpl->args, &gotTruth) ){ + goto end; + }else if( !gotTruth ){ + skipOn; + } + + cmpp_d const * dPrev = dx->d; + cmpp_outputer devNull = cmpp_outputer_empty; + while( !dxppCode ){ + dPrev = dx->d; + bool const isFinal = dPrev==cis->dElse + /* true if expecting an #/if. */; + if( cmpp_dx_consume(dx, + CmppLvl_is_eliding(lvl) ? &devNull : NULL, + isFinal ? &cis->dEndif : dClosers, + isFinal ? 1 : sizeof(dClosers)/sizeof(dClosers[0]), + cmpp_dx_consume_F_PROCESS_OTHER_D) ){ + break; + } + cmpp_d const * const d2 = dx->d; + if( !d2 ){ + dxserr("Reached end of input in an untermined %s%s opened " + "at line %" CMPP_SIZE_T_PFMT ".", + delim, cis->dIf->name.z, dline.lineNo); + } + if( d2==cis->dEndif ){ + break; + }else if( isFinal ){ + assert(!"cannot happen - caught by consume()"); + dxserr("Expecting %s%s to close %s%s.", + delim, cis->dEndif->name.z, + delim, dPrev->name.z); + break; + }else if( gotTruth ){ + skipOn; + continue; + }else if( d2==cis->dElif ){ + if( 0==cmpp_dx_args_parse(dx, &args) + && 0==cmpp__args_evalToInt(dx, &args, &gotTruth) ){ + if( gotTruth ) skipOff; + else skipOn; + } + continue; + }else{ + assert( d2==cis->dElse + && "Else (haha!) we cannot have gotten here" ); + skipOff; + continue; + } + assert(!"unreachable"); + } + +#undef skipOff +#undef skipOn +end: + cmpp_args_cleanup(&args); + if( lvl ){ + bool const lvlIsOk = CmppLvl_get(dx)==lvl; + CmppLvl_pop(dx, lvl); + if( !lvlIsOk && !dxppCode ){ + assert(!"i naively believe that this is not possible"); + cmpp_dx_err_set(dx, CMPP_RC_SYNTAX, + "Mis-terminated %s%s opened at line " + "%" CMPP_SIZE_T_PFMT ".", + delim, cis->dIf->name.z, dline.lineNo); + } + } + return; +} + +/* Version 2 of #elif, #else, and #/if. */ +static void cmpp_dx_f_if_dangler(cmpp_dx *dx){ + CmppIfState const * const cis = dx->d->impl.state; + char const *zDelim = cmpp_dx_delim(dx); + cmpp_dx_err_set(dx, CMPP_RC_SYNTAX, + "%s%s with no matching %s%s", + zDelim, dx->d->name.z, + zDelim, cis->dIf->name.z); +} + +static void cmpp__dump_sizeofs(cmpp_dx*dx){ + (void)dx; +#define SO(X) printf("sizeof(" # X ") = %u\n", (unsigned)sizeof(X)) + SO(cmpp); + SO(cmpp_api_thunk); + SO(cmpp_arg); + SO(cmpp_args); + SO(cmpp_args_pimpl); + SO(cmpp_b); + SO(cmpp_d); + SO(cmpp_d_reg); + SO(cmpp__delim); + SO(cmpp__delim_list); + SO(cmpp_dx); + SO(cmpp_dx_pimpl); + SO(cmpp_outputer); + SO(cmpp_pimpl); + SO(((cmpp_pimpl*)0)->stmt); + SO(((cmpp_pimpl*)0)->policy); + SO(CmppArgList); + SO(CmppDLine); + SO(CmppDList); + SO(CmppDList_entry); + SO(CmppLvl); + SO(CmppSnippet); + SO(PodList__atpol); + printf("cmpp_TT__last = %d\n", + cmpp_TT__last); +#undef SO +} + + +/* Impl. for #pragma. */ +static void cmpp_dx_f_pragma(cmpp_dx *dx){ + cmpp_arg const * arg = dx->args.arg0; + if(!arg){ + cmpp_dx_err_set(dx, CMPP_RC_SYNTAX, "Expecting an argument"); + return; + }else if(arg->next){ + cmpp_dx_err_set(dx, CMPP_RC_SYNTAX, "Too many arguments"); + return; + } + const char * const zArg = (char const *)arg->z; +#define M(X) 0==strcmp(zArg,X) + if(M("defines")){ + cmpp__dump_defines(dx->pp, stderr, 1); + }else if(M("sizeof")){ + cmpp__dump_sizeofs(dx); + }else if(M("chomp-F")){ + dx->pp->pimpl->flags.chompF = 1; + }else if(M("no-chomp-F")){ + dx->pp->pimpl->flags.chompF = 0; + }else if(M("api-thunk")){ + /* Generate macros for CMPP_API_THUNK and friends from + cmpp_api_thunk_map. */ + char const * zName = "CMPP_API_THUNK_NAME"; + char buf[256]; +#define out(FMT,...) snprintf(buf, sizeof(buf), FMT,__VA_ARGS__); \ + cmpp_dx_out_raw(dx, buf, strlen(buf)) + if( 0 ){ + out("/* libcmpp API thunk. */\n" + "static cmpp_api_thunk const * %s = 0;\n" + "#define cmpp_api_init(PP) %s = (PP)->api\n", zName, zName); + } +#define A(V) \ + if(V<=cmpp_api_thunk_version) { \ + out("/* Thunk APIs which follow are available as of " \ + "version %d... */\n",V); \ + } +#define V(N,T,V) +#define F(N,T,P) out("#define cmpp_%s %s->%s\n", # N, zName, # N); +#define O(N,T) out("#define cmpp_%s (*%s->%s)\n", # N, zName, # N); +cmpp_api_thunk_map(A,V,F,O) +#undef V +#undef F +#undef O +#undef A +#undef out + }else{ + cmpp_dx_err_set(dx, CMPP_RC_NOT_FOUND, "Unknown pragma: %s", zArg); + } +#undef M +} + +/* Impl. for #savepoint. */ +static void cmpp_dx_f_savepoint(cmpp_dx *dx){ + if(!dx->args.arg0 || dx->args.arg0->next){ + cmpp_dx_err_set(dx, CMPP_RC_SYNTAX, "Expecting one argument"); + }else{ + const char * const zArg = (const char *)dx->args.arg0->z; +#define M(X) else if( 0==strcmp(zArg,X) ) + if( 0 ){} + M("begin"){ + cmpp__dx_sp_begin(dx); + } + M("rollback"){ + cmpp__dx_sp_rollback(dx); + }M("commit"){ + cmpp__dx_sp_commit(dx); + }else{ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Unknown savepoint option: %s", zArg); + } + } +#undef M +} + +/* #stderr impl. */ +static void cmpp_dx_f_stderr(cmpp_dx *dx){ + if(dx->args.z){ + g_stderr("%s:%" CMPP_SIZE_T_PFMT ": %.*s\n", dx->sourceName, + dx->pimpl->dline.lineNo, + (int)dx->args.nz, dx->args.z); + }else{ + cmpp_d const * d = dx->d; + g_stderr("%s:%" CMPP_SIZE_T_PFMT ": (no %s%s argument)\n", + dx->sourceName, dx->pimpl->dline.lineNo, + cmpp_dx_delim(dx), d->name.z); + } +} + +/** + Manages both the @token@ policy and the delimiters. + + #@ ?push? policy NAME ?<<? + #@ ?push? delimiter OPEN CLOSE ?<<? + #@ ?push? policy NAME delimiter OPEN CLOSE ?<<? + #@ pop policy + #@ pop delimiter + #@ pop policy delimiter + #@ pop both + + Function call forms: + + [@ policy] + [@ delimiter] +*/ +static void cmpp_dx_f_at(cmpp_dx *dx){ + cmpp_arg const * arg = dx->args.arg0; + if( !arg ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Expecting arguments."); + return; + } + enum ops { op_none, op_set, op_push, op_pop, op_heredoc }; + enum popWhichE { pop_policy = 0x01, pop_delim = 0x02, + pop_both = pop_policy | pop_delim }; + enum ops op = op_none /* what to do */; + int popWhich = 0 /* what to pop */; + bool gotPolicy = false; + bool checkedCallForm = !cmpp_dx_is_call(dx); + cmpp_arg const * argDelimO = 0 /* @token@ opener */; + cmpp_arg const * argDelimC = 0 /* @token@ closer */; + cmpp__pi(dx->pp); + cmpp_atpol_e polNew = cmpp_atpol_get(dx->pp); + for( ; arg; arg = arg ? arg->next : NULL ){ + //g_warn("arg=%s", arg->z); + if( !checkedCallForm ){ + assert( cmpp_dx_is_call(dx) ); + checkedCallForm = true; + if( cmpp_arg_equals(arg, "policy") ){ + char const * z = + cmpp__atpol_name(dx->pp, cmpp__policy(dx->pp,at)); + if( z ){ + cmpp_dx_out_raw(dx, z, strlen(z)); + } + }else if( cmpp_arg_equals(arg, "delimiter") ){ + char const * zO = 0; + char const * zC = 0; + cmpp_atdelim_get(dx->pp, &zO, &zC); + if( zC ){ + cmpp_dx_out_raw(dx, zO, strlen(zO)); + cmpp_dx_out_raw(dx, " ", 1); + cmpp_dx_out_raw(dx, zC, strlen(zC)); + } + goto end; + }else{ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "In call form, '%s' expects one of " + "'policy' or delimiter'."); + } + goto end; + }/* checkedCallForm */ + if( !argDelimC && op_none==op ){ + /* Look for push|pop. */ + if( cmpp_arg_equals(arg, "pop") ){ + arg = arg->next; + if( !arg ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "'pop' expects arguments of 'policy' " + "and/or 'delimiter' and/or 'both'."); + goto end; + } + for( ; arg; arg = arg->next ){ + if( 0==(pop_policy & popWhich) + && cmpp_arg_equals(arg, "policy") ){ + popWhich |= pop_policy; + }else if( 0==(pop_delim & popWhich) + && cmpp_arg_equals(arg, "delimiter") ){ + popWhich |= pop_delim; + }else if( 0==(pop_both & popWhich) + && cmpp_arg_equals(arg, "both") ){ + popWhich |= pop_both; + }else{ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Invalid argument to 'pop': ", arg->z); + goto end; + } + } + assert( !arg ); + op = op_pop; + break; + }/* pop */ + if( cmpp_arg_equals(arg, "push") ){ + op = op_push; + continue; + } + if( cmpp_arg_equals(arg, "set") ){ + /* set is implied if neither of push/pop are and we get + a policy name. */ + op = op_set; + continue; + } + /* Fall through */ + }/* !argDelimC && op_none==op */ + if( !gotPolicy && cmpp_arg_equals(arg, "policy") ){ + arg = arg->next; + if( !arg ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "'policy' requires a policy name argument."); + goto end; + } + polNew = cmpp_atpol_from_str(NULL, (char const*)arg->z); + if( cmpp_atpol_invalid==polNew ){ + cmpp_atpol_from_str(dx->pp, (char const*)arg->z) + /* Will set the error state to something informative. */; + goto end; + } + if( op_none==op ) op = op_set; + gotPolicy = true; + continue; + } + if( !argDelimC && cmpp_arg_equals(arg, "delimiter") ){ + assert( !argDelimO && !argDelimC ); + argDelimO = arg->next; + argDelimC = argDelimO ? argDelimO->next : NULL; + if( !argDelimC ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "'delimiter' requires two arguments."); + goto end; + } + arg = argDelimC->next; + continue; + } + if( op_pop!=op ){ + if( cmpp_arg_equals(arg,"<<") ){ + if( arg->next ) { + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "'%s' must be the final argument.", + arg->z); + goto end; + } + op = op_heredoc; + break; + } + } + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, "Unhandled argument: %s", arg->z); + return; + }/*arg collection*/ + + assert( !dxppCode ); + assert( cmpp_atpol_invalid!=polNew ); + +#define popcheck(LIST) \ + if(dxppCode) goto end; \ + if(!LIST.n) goto bad_pop + + if( op_pop==op ){ + assert( popWhich>0 && popWhich<=3 ); + if( pop_policy & popWhich ){ + popcheck(pi->policy.at); + cmpp_atpol_pop(dx->pp); + } + if( pop_delim & popWhich ){ + popcheck(pi->delim.at); + cmpp_atdelim_pop(dx->pp); + } + goto end; + } + + assert( op_set==op || op_push==op || op_heredoc==op ); + if( argDelimC ){ + /* Push or set the @token@ delimiters */ + if( 0 ){ + g_warn("%s @delims@: %s %s", (op_set==op) ? "set" : "push", + argDelimO->z, argDelimC->z); + } + if( op_push==op || op_heredoc==op ){ + if( cmpp_atdelim_push(dx->pp, (char const*)argDelimO->z, + (char const*)argDelimC->z) ){ + goto end; + } + argDelimO = 0 /* Re-use argDelimC as a flag in case we need to + roll this back on an error below. */; + }else{ + assert( op_set==op ); + if( cmpp_atdelim_set(dx->pp, (char const*)argDelimO->z, + (char const*)argDelimC->z) ){ + goto end; + } + argDelimO = argDelimC = 0; + } + } + + assert( !dxppCode ); + assert( !argDelimO ); + if( op_heredoc==op ){ + if( cmpp_atpol_push(dx->pp, polNew) ){ + if( argDelimC ){ + popcheck(pi->delim.at); + cmpp_atdelim_pop(dx->pp); + } + }else{ + bool const pushedDelim = NULL!=argDelimC; + assert( dx->d->closer ); + cmpp_dx_consume(dx, NULL, &dx->d->closer, 1, + cmpp_dx_consume_F_PROCESS_OTHER_D) + /* !Invalidates argDelimO and argDelimC! */; + popcheck(pi->policy.at); + cmpp_atpol_pop(dx->pp); + if( pushedDelim ) cmpp_atdelim_pop(dx->pp); + } + }else if( op_push==op ){ + if( cmpp_atpol_push(dx->pp, polNew) && argDelimC ){ + /* Roll back delimiter push */ + cmpp_atdelim_pop(dx->pp); + } + }else{ + assert( op_set==op ); + if( cmpp__policy(dx->pp,at)!=polNew ){ + cmpp_atpol_set(dx->pp, polNew); + } + } +end: + return; +bad_pop: + cmpp_dx_err_set(dx, CMPP_RC_RANGE, + "Cannot pop an empty stack."); +#undef popcheck +} + + +static void cmpp_dx_f_expr(cmpp_dx *dx){ + int rv = 0; + assert( dx->args.z ); + if( 0 ){ + g_stderr("%s() argc=%d arg0 [%.*s]\n", __func__, dx->args.argc, + dx->args.arg0->n, dx->args.arg0->z); + g_stderr("%s() dx->args.z [%.*s]\n", __func__, + (int)dx->args.nz, dx->args.z); + } + if( !dx->args.argc ){ + dxserr("An empty expression is not permitted."); + return; + } +#if 0 + for( cmpp_arg const * a = dx->args.arg0; a; a = a->next ){ + g_stderr("got type=%s n=%u z=%.*s\n", + cmpp__tt_cstr(a->ttype, true), + (unsigned)a->n, (int)a->n, a->z); + } +#endif + if( 0==cmpp__args_evalToInt(dx, &dx->pimpl->args, &rv) ){ + if( 'a'==dx->d->name.z[0] ){ + if( !rv ){ + cmpp_dx_err_set(dx, CMPP_RC_ASSERT, "Assertion failed: %s", + dx->pimpl->buf.argsRaw.z); + } + }else{ + char buf[60]; + snprintf(buf, sizeof(buf), "%d\n", rv); + cmpp_dx_out_raw(dx, buf, strlen(buf)); + } + } +} + +static void cmpp_dx_f_undef_policy(cmpp_dx *dx){ + cmpp_unpol_e up = cmpp_unpol_invalid; + int nSeen = 0; + cmpp_arg const * arg = dx->args.arg0; + enum ops { op_set, op_push, op_pop }; + enum ops op = op_set; + if( !dx->args.argc ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Expecting one of: error, null"); + return; + } +again: + ++nSeen; + if( cmpp_arg_equals(arg,"error") ) up = cmpp_unpol_ERROR; + else if( cmpp_arg_equals(arg,"null") ) up = cmpp_unpol_NULL; + else if( 1==nSeen ){ + if( cmpp_arg_equals(arg, "push") ){ + if( !arg->next ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Expecting argument to 'push'."); + return; + } + op = op_push; + arg = arg->next; + goto again; + }else if( cmpp_arg_equals(arg, "pop") ){ + if( arg->next ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Extra argument after 'pop': %s", + arg->next->z); + return; + } + op = op_pop; + } + } + if( op_pop!=op && cmpp_unpol_invalid==up ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Unhandled undefined-policy '%s'." + " Try one of: error, null", + arg->z); + }else if( op_set==op ){ + cmpp_unpol_set(dx->pp, up); + }else if( op_push==op ){ + cmpp_unpol_push(dx->pp, up); + }else{ + assert( op_pop==op ); + if( !cmpp__epol(dx->pp,un).n ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "No %s%s push is active.", + cmpp_dx_delim(dx), dx->d->name.z); + }else{ + cmpp_unpol_pop(dx->pp); + } + } +} + +#ifndef CMPP_OMIT_D_DB +/* Impl. for #attach. */ +static void cmpp_dx_f_attach(cmpp_dx *dx){ + if( 3!=dx->args.argc ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "%s expects: STRING as NAME", dx->d->name.z); + return; + } + cmpp_arg const * pNext = 0; + cmpp_b osDbFile = cmpp_b_empty; + cmpp_b osSchema = cmpp_b_empty; + for( cmpp_arg const * arg = dx->args.arg0; + 0==dxppCode && arg; + arg = pNext ){ + pNext = arg->next; + if( !osDbFile.n ){ + if( 0==cmpp_arg_to_b(dx, arg, &osDbFile, + cmpp_arg_to_b_F_BRACE_CALL) + && !osDbFile.n ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Empty db file name is not permitted. " + "If '%s' is intended as a value, " + "it should be quoted.", arg->z); + break; + } + assert( pNext ); + if( !pNext || !cmpp_arg_equals(pNext, "as") ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Expecting 'as' after db file name."); + break; + } + pNext = pNext->next; + }else if( !osSchema.n ){ + if( 0==cmpp_arg_to_b(dx, arg, &osSchema, + cmpp_arg_to_b_F_BRACE_CALL) + && !osSchema.n ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Empty db schema name is not permitted." + "If '%s' is intended as a value, " + "it should be quoted.", + arg->z); + break; + } + } + } + if( dxppCode ) goto end; + if( !osSchema.n ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, "Missing schema name."); + goto end; + } + sqlite3_stmt * const q = + cmpp__stmt(dx->pp, CmppStmt_dbAttach, false); + if( q ){ + cmpp__bind_textn(dx->pp, q, 1, osDbFile.z, osDbFile.n); + cmpp__bind_textn(dx->pp, q, 2, osSchema.z, osSchema.n); + cmpp__step(dx->pp, q, true); + } +end: + cmpp_b_clear(&osDbFile); + cmpp_b_clear(&osSchema); +} + +/* Impl. for #detach. */ +static void cmpp_dx_f_detach(cmpp_dx *dx){ + cmpp_d const * d = dx->d; + if( 1!=dx->args.argc ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "%s expects: NAME", d->name.z); + return; + } + cmpp_arg const * const arg = dx->args.arg0; + cmpp_b os = cmpp_b_empty; + if( cmpp_arg_to_b(dx, arg, &os, cmpp_arg_to_b_F_BRACE_CALL) ){ + goto end; + } + if( !os.n ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Empty db schema name is not permitted."); + goto end; + } + sqlite3_stmt * const q = + cmpp__stmt(dx->pp, CmppStmt_dbDetach, false); + if( q ){ + cmpp__bind_textn(dx->pp, q, 1, os.z, os.n); + cmpp__step(dx->pp, q, true); + } +end: + cmpp_b_clear(&os); +} +#endif /* #ifndef CMPP_OMIT_D_DB */ + +static void cmpp_dx_f_delimiter(cmpp_dx *dx){ + cmpp_arg const * arg = dx->args.arg0; + enum ops { op_none, op_set, op_push, op_pop }; + enum ops op = op_none; + cmpp_arg const * argD = 0; + bool doHeredoc = false; + bool const isCall = cmpp_dx_is_call(dx); + for( ; arg; arg = arg->next ){ + if( op_none==op ){ + /* Look for push|pop. */ + if( cmpp_arg_equals(arg, "push") ){ + op = op_push; + continue; + }else if( cmpp_arg_equals(arg, "pop") ){ + if( arg->next ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "'pop' expects no arguments."); + return; + } + op = op_pop; + break; + } + /* Fall through */ + } + if( !argD ){ + if( op_none==op ) op = op_set; + argD = arg; + continue; + }else if( !doHeredoc && cmpp_arg_equals(arg,"<<") ){ + if( isCall ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "'%s' is not legal in [call] form.", arg->z); + return; + }else if( arg->next ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "'%s' must be the final argument.", arg->z); + return; + } + op = op_push; + doHeredoc = true; + continue; + } + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, "Unhandled arg: %s", arg->z); + return; + } + if( op_pop==op ){ + cmpp_delimiter_pop(dx->pp); + }else if( !argD ){ + if( isCall ){ + cmpp__delim const * const del = cmpp__dx_delim(dx); + if( del ) cmpp_dx_out_raw(dx, del->open.z, del->open.n); + }else{ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, "No delimiter specified."); + } + return; + }else{ + char const * const z = + (0==strcmp("default",(char*)argD->z)) + ? NULL + : (char const*)argD->z; + if( op_push==op ){ + cmpp_delimiter_push(dx->pp, (char const*)argD->z); + }else{ + assert( op_set==op ); + if( doHeredoc ) cmpp_delimiter_push(dx->pp, z); + else cmpp_delimiter_set(dx->pp, z); + } + } + if( !cmpp_dx_err_check(dx) ){ + if( isCall ){ + cmpp__delim const * const del = cmpp__dx_delim(dx); + if( del ) cmpp_dx_out_raw(dx, del->open.z, del->open.n); + }else if( doHeredoc ){ + assert( op_push==op ); + cmpp_dx_consume(dx, NULL, &dx->d->closer, 1, + cmpp_dx_consume_F_PROCESS_OTHER_D); + cmpp_delimiter_pop(dx->pp); + } + } +} + +#ifndef NDEBUG +/* Experimenting grounds. */ +static void cmpp_dx_f_experiment(cmpp_dx *dx){ + void * st = dx->d->impl.state; + (void)st; + g_warn("raw args: %s", dx->pimpl->buf.argsRaw.z); + g_warn("argc=%u", dx->args.argc); + g_warn("isCall=%d\n", cmpp_dx_is_call(dx)); + if( 1 ){ + for( cmpp_arg const * a = dx->args.arg0; a; a = a->next ){ + g_stderr("got type=%s n=%u z=%.*s\n", + cmpp__tt_cstr(a->ttype, true), + (unsigned)a->n, (int)a->n, a->z); + } + } + if( 0 ){ + int rv = 0; + if( 0==cmpp__args_evalToInt(dx, &dx->pimpl->args, &rv) ){ + g_stderr("expr result: %d\n", rv); + } + } + + if( 0 ){ + char const * zIn = "a strspn test @# and @"; + g_stderr("strlen : %u\n", (unsigned)strlen(zIn)); + g_stderr("strspn 1: %u, %u\n", + (unsigned)strspn(zIn, "#@"), + (unsigned)strspn(zIn, "@#")); + g_stderr("strcspn 2: %u, %u\n", + (unsigned)strcspn(zIn, "#@"), + (unsigned)strcspn(zIn, "@#")); + g_stderr("strcspn 3: %u, %u\n", + (unsigned)strcspn(zIn, "a strspn"), + (unsigned)strcspn(zIn, "nope")); + } + + if( 1 ){ + cmpp__dump_sizeofs(dx); + } +} +#endif /* #ifndef NDEBUG */ + +#ifndef CMPP_OMIT_D_DB + +/** + Helper for #query and friends. Expects arg to be an SQL value. If + arg->next is "bind" then this consumes the following two arguments( + "bind" BIND_ARG), where BIND_ARG must be one of either + cmpp_TT_GroupSquiggly or cmpp_TT_GroupBrace. + + If it returns 0 then: + + - If "bind" was found then *pBind is set to the BIND_ARG argument + and *pNext is set to the one after that. + + - Else *pBind is set to NULL and and *pNext is set to + arg->next. + + In either case, *pNext may be set to NULL. +*/ +static +int cmpp__consume_sql_args(cmpp *pp, cmpp_arg const *arg, + cmpp_arg const **pBind, + cmpp_arg const **pNext){ + if( 0==ppCode ){ + *pBind = 0; + cmpp_arg const *pN = arg->next; + if( pN && cmpp_arg_equals(pN, "bind") ){ + pN = pN->next; + if( !pN || ( + cmpp_TT_GroupSquiggly!=pN->ttype + && cmpp_TT_GroupBrace!=pN->ttype + ) ){ + return serr("Expecting {...} or [...] after 'bind'."); + } + *pBind = pN; + *pNext = pN->next; + } else { + *pBind = 0; + *pNext = pN; + } + } + return ppCode; +} + +/** + cmpp_kav_each_f() impl for used by #query's `bind {...}` argument. +*/ +static int cmpp_kav_each_f_query__bind( + cmpp_dx * const dx, + unsigned char const * const zKey, cmpp_size_t nKey, + unsigned char const * const zVal, cmpp_size_t nVal, + void * const callbackState +){ + /* Expecting: :bindName -> bindValue */ + if( ':'!=zKey[0] && '$'!=zKey[0] /*&& '@'!=zKey[0]*/ ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Bind keys must start with ':' or '$'."); + }else{ + sqlite3_stmt * const q = callbackState; + assert( q ); + int const bindNdx = + sqlite3_bind_parameter_index(q, (char const*)zKey); + if( bindNdx ){ + cmpp__bind_textn(dx->pp, q, bindNdx, zVal, nVal); + }else{ + cmpp_err_set(dx->pp, CMPP_RC_RANGE, "Invalid bind name: %.*s", + (int)nKey, zKey); + } + } + return dxppCode; +} + +int cmpp__bind_group(cmpp_dx * const dx, sqlite3_stmt * const q, + cmpp_arg const * const aGroup){ + if( dxppCode ) return dxppCode; + if( cmpp_TT_GroupSquiggly==aGroup->ttype ){ + return cmpp_kav_each( + dx, aGroup->z, aGroup->n, + cmpp_kav_each_f_query__bind, q, + cmpp_kav_each_F_NOT_EMPTY + | cmpp_kav_each_F_CALL_VAL + | cmpp_kav_each_F_PARENS_EXPR + ); + } + if( cmpp_TT_GroupBrace!=aGroup->ttype ){ + return cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Expecting {...} or [...] " + "for SQL binding list."); + } + int bindNdx = 0; + cmpp_args args = cmpp_args_empty; + cmpp_args_parse(dx, &args, aGroup->z, aGroup->n, 0); + if( !args.argc && !dxppCode ){ + cmpp_err_set(dx->pp, CMPP_RC_RANGE, + "Empty SQL bind list is not permitted."); + /* Keep going so we can clean up a partially-parsed args. */ + } + for( cmpp_arg const * aVal = args.arg0; + !dxppCode && aVal; + aVal = aVal->next ){ + ++bindNdx; + if( 0 ){ + g_warn("bind #%d %s <<%s>>", bindNdx, + cmpp__tt_cstr(aVal->ttype, true), aVal->z); + } + cmpp__bind_arg(dx, q, bindNdx, aVal); + } + cmpp_args_cleanup(&args); + return dxppCode; +} + +/** #query impl */ +static void cmpp_dx_f_query(cmpp_dx *dx){ + //cmpp_d const * d = cmpp_dx_d(dx); + if( !dx->args.arg0 ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Expecting one or more arguments"); + return; + } + cmpp * const pp = dx->pp; + sqlite3_stmt * q = 0; + cmpp_b * const obBody = cmpp_b_borrow(dx->pp); + cmpp_b * const sql = cmpp_b_borrow(dx->pp); + cmpp_outputer obNull = cmpp_outputer_empty; + //cmpp_b obBindArgs = cmpp_b_empty; + cmpp_args args = cmpp_args_empty + /* We need to copy the args or do some arg-type-specific work to + copy the memory for specific cases. */; + int nChomp = 0; + bool spStarted = false; + bool seenDefine = false; + bool batchMode = false; + cmpp_arg const * pNext = 0; + cmpp_arg const * aBind = 0; + cmpp_d const * const dNoRows = dx->d->impl.state; + cmpp_d const * const dClosers[2] = {dx->d->closer, dNoRows}; + + if( !obBody || !sql ) goto cleanup; + + assert( dNoRows ); + if( cmpp_dx_args_clone(dx, &args) ){ + goto cleanup; + } + //g_warn("args.argc=%d", args.argc); + for( cmpp_arg const * arg = args.arg0; + 0==dxppCode && arg; + arg = pNext ){ + //g_warn("arg=%s <<%s>>", cmpp_tt_cstr(arg->ttype), arg->z); + pNext = arg->next; + if( cmpp_arg_equals(arg, "define") ){ + if( seenDefine ){ + cmpp__dx_err_just_once(dx, arg); + goto cleanup; + } + seenDefine = true; + continue; + } + if( cmpp_arg_equals(arg, "-chomp") ){ + ++nChomp; + continue; + } + if( cmpp_arg_equals(arg, "-batch") ){ + if( batchMode ){ + cmpp__dx_err_just_once(dx, arg); + goto cleanup; + } + batchMode = true; + continue; + } + if( !sql->n ){ + if( cmpp__consume_sql_args(pp, arg, &aBind, &pNext) ){ + goto cleanup; + } + if( cmpp_arg_to_b(dx, arg, sql, cmpp_arg_to_b_F_BRACE_CALL) ){ + goto cleanup; + } + //g_warn("SQL: <<%s>>", sql->z); + continue; + } + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, "Unhandled arg: %s", arg->z); + goto cleanup; + } + if( ppCode ) goto cleanup; + if( seenDefine ){ + if( nChomp ){ + serr("-chomp and define may not be used together."); + goto cleanup; + }else if( batchMode ){ + serr("-batch and define may not be used together."); + goto cleanup; + } + } + if( !sql->n ){ + serr("Expecting an SQL-string argument."); + goto cleanup; + } + + if( batchMode ){ + if( aBind ){ + serr("Bindable values may not be used with -batch."); + goto cleanup; + } + char *zErr = 0; + cmpp__pi(dx->pp); + int rc = sqlite3_exec(pi->db.dbh, (char const *)sql->z, 0, 0, &zErr); + rc = cmpp__db_rc(dx->pp, rc, zErr); + sqlite3_free(zErr); + goto cleanup; + } + + if( cmpp__db_rc(pp, sqlite3_prepare_v2( + pp->pimpl->db.dbh, (char const *)sql->z, + (int)sql->n, &q, 0), 0) ){ + goto cleanup; + }else if( !q ){ + cmpp_err_set(pp, CMPP_RC_RANGE, + "Empty SQL is not permitted."); + goto cleanup; + } + //g_warn("SQL via stmt: <<%s>>", sqlite3_sql(q)); + int const nCol = sqlite3_column_count(q); + if( !nCol ){ + cmpp_err_set(pp, CMPP_RC_RANGE, + "SQL does not have any result columns."); + goto cleanup; + } + if( !seenDefine ){ + if( cmpp_sp_begin(pp) ) goto cleanup; + spStarted = true; + } + + if( aBind && cmpp__bind_group(dx, q, aBind) ){ + goto cleanup; + } + + bool gotARow = false; + cmpp_dx_pos dxPosStart; + cmpp_flag32_t const consumeFlags = cmpp_dx_consume_F_PROCESS_OTHER_D; + cmpp_dx_pos_save(dx, &dxPosStart); + int const nChompOrig = nChomp; + while( 0==ppCode ){ + int const dbrc = cmpp__step(pp, q, false); + if( SQLITE_ROW==dbrc ){ + nChomp = nChompOrig; + gotARow = true; + if( cmpp__define_from_row(pp, q, false) ) break; + if( seenDefine ) break; + cmpp_dx_pos_restore(dx, &dxPosStart); + cmpp_b_reuse(obBody); + /* If it weren't for -chomp, we wouldn't need to + buffer this. */ + if( cmpp_dx_consume_b(dx, obBody, dClosers, + sizeof(dClosers)/sizeof(dClosers[0]), + consumeFlags) ){ + goto cleanup; + } + assert( dx->d == dClosers[0] || dx->d == dClosers[1] ); + while( nChomp-- && cmpp_b_chomp(obBody) ){} + if( obBody->n && cmpp_dx_out_raw(dx, obBody->z, obBody->n) ) break; + if( dx->d == dNoRows ){ + if( cmpp_dx_consume(dx, &obNull, dClosers, 1/*one!*/, + consumeFlags) ){ + goto cleanup; + } + assert( dx->d == dClosers[0] ); + /* TODO? chomp? */ + } + continue; + } + if( 0==ppCode && seenDefine ){ + /* If we got here, there was no result row. */ + cmpp__define_from_row(pp, q, true); + } + break; + }/*result row loop*/ + cmpp__stmt_reset(q); + if( ppCode ) goto cleanup; + + while( !seenDefine && !gotARow ){ + /* No result rows. Skip past the body, emitting the #query:no-rows + content, if any. We disable @token processing for that first + step because (A) the output is not going anywhere, so no need + to expand it (noting that expanding may have side effects via + @[call...]@) and (B) the @tokens@ referring to this query's + results will not have been set because there was no row to set + them from, so @expanding@ them would fail. */ + cmpp_atpol_e const atpol = cmpp_atpol_get(dx->pp); + if( cmpp_atpol_set(dx->pp, cmpp_atpol_OFF) ) break; + cmpp_dx_consume(dx, &obNull, dClosers, + sizeof(dClosers)/sizeof(dClosers[0]), + consumeFlags); + cmpp_atpol_set(dx->pp, atpol); + if( dxppCode ) break; + assert( dx->d == dClosers[0] || dx->d == dClosers[1] ); + if( dx->d == dNoRows ){ + if( cmpp_dx_consume(dx, 0, dClosers, 1/*one!*/, + consumeFlags) ){ + break; + } + assert( dx->d == dClosers[0] ); + /* TODO? chomp? */ + } + break; + } + +cleanup: + cmpp_args_cleanup(&args); + cmpp_b_return(dx->pp, obBody); + cmpp_b_return(dx->pp, sql); + sqlite3_finalize(q); + if( spStarted ) cmpp_sp_rollback(pp); +} +#endif /* #ifndef CMPP_OMIT_D_DB */ + +#ifndef CMPP_OMIT_D_PIPE +/** #pipe impl. */ +static void cmpp_dx_f_pipe(cmpp_dx *dx){ + //cmpp_d const * d = cmpp_dx_d(dx); + unsigned char const * zArgs = dx->args.z; + assert( dx->args.arg0->n == dx->args.nz ); + unsigned char const * const zArgsEnd = zArgs + dx->args.nz; + if( zArgs==zArgsEnd ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Expecting a command and arguments to pipe."); + return; + } + cmpp_FILE * fpToChild = 0; + int nChompIn = 0, nChompOut = 0; + cmpp_b * const chout = cmpp_b_borrow(dx->pp); + cmpp_b * const cmd = cmpp_b_borrow(dx->pp); + cmpp_b * const body = cmpp_b_borrow(dx->pp); + cmpp_b * const bArg = cmpp_b_borrow(dx->pp) + /* arg parsing and the initial command name part of the + external command. */; + cmpp_args cmdArgs = cmpp_args_empty; + /* TODOs and FIXMEs: + + We need flags to optionally @token@-parse before and/or after + filtering. + */ + bool seenDD = false /* true if seen "--" or [...] */; + bool doCapture = true /* true if we need a closing /pipe */; + bool argsAsGroup = false /* true if args is [...] */; + bool dumpDebug = false; + cmpp_flag32_t popenFlags = 0; + cmpp_popen_t po = cmpp_popen_t_empty; + if( cmpp_b_reserve3(dx->pp, cmd, zArgsEnd-zArgs + 1) + || cmpp_b_reserve3(dx->pp, bArg, cmd->nAlloc) ){ + goto cleanup; + } + + unsigned char * zOut = bArg->z; + unsigned char const * const zOutEnd = bArg->z + bArg->nAlloc - 1; + while( 0==dxppCode ){ + cmpp_arg arg = cmpp_arg_empty; + zOut = bArg->z; + if( cmpp_arg_parse(dx, &arg, &zArgs, zArgsEnd, + &zOut, zOutEnd) ){ + goto cleanup; + } + if( cmpp_arg_equals(&arg, "--") ){ + zOut = bArg->z; + if( cmpp_arg_parse(dx, &arg, &zArgs, zArgsEnd, + &zOut, zOutEnd) ){ + goto cleanup; + } + if( !arg.n ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Expecting external command name " + "or [...] after --."); + goto cleanup; + } + do_arg_list: + seenDD = true; + cmpp_flag32_t a2bFlags = cmpp_arg_to_b_F_BRACE_CALL; + if( cmpp_TT_GroupBrace==arg.ttype ){ + argsAsGroup = true; + a2bFlags |= cmpp_arg_to_b_F_NO_BRACE_CALL; + }else if( cmpp__arg_wordIsPathOrFlag(&arg) ){ + /* If it looks like it is a path, do not + expand it as a word. */ + arg.ttype = cmpp_TT_String; + } + if( cmpp_arg_to_b(dx, &arg, cmd, a2bFlags) + || (!argsAsGroup && cmpp_b_append_ch(cmd, ' ')) ){ + goto cleanup; + } + //g_warn("command: [%s]=>%s", arg.z, cmd->z); + if( cmd->n<2 ){ + cmpp_dx_err_set(dx, CMPP_RC_RANGE, + "Command name '%s' resolves to empty. " + "This is most commonly caused by not " + "quoting it but it can also mean that it " + "is an unknown define key.", arg.z); + goto cleanup; + } + //g_warn("arg=%s", arg.z); + //g_warn("cmd=%s", cmd->z); + break; + } + if( cmpp_TT_GroupBrace==arg.ttype ){ + goto do_arg_list; + } +#define FLAG(X)if( cmpp_arg_isflag(&arg, X) ) + FLAG("-no-input"){ + doCapture = false; + continue; + } + FLAG("-chomp-output"){ + ++nChompOut; + continue; + } + FLAG("-chomp"){ + ++nChompIn; + continue; + } + FLAG("-exec-direct"){ + popenFlags |= cmpp_popen_F_DIRECT; + continue; + } + FLAG("-path"){ + popenFlags |= cmpp_popen_F_PATH; + continue; + } + FLAG("-debug"){ + dumpDebug = true; + continue; + } +#undef FLAG + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Unhandled argument: %s. %s%s requires -- " + "before its external command name.", + arg.z, cmpp_dx_delim(dx), + dx->d->name.z); + goto cleanup; + } + + if( !seenDD ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "%s%s requires a -- before the name of " + "its external app.", + cmpp_dx_delim(dx), dx->d->name.z); + goto cleanup; + } + + //g_warn("zArgs n=%u zArgs=%s", (unsigned)(zArgsEnd-zArgs), zArgs); + /* dx->pimpl->args gets overwritten by cmpp_dx_consume(), so we have to copy + the args. */ + if( argsAsGroup ){ + assert( cmd->z ); + if( cmpp_args_parse(dx, &cmdArgs, cmd->z, cmd->n, 0) ){ + goto cleanup; + } + }else{ + /* zArgs can have newlines in it. We need to strip those out + before passing it on. We elide them entirely, as opposed to + replacing them with a space. */ + cmpp_skip_snl(&zArgs, zArgsEnd); + if( cmpp_b_reserve3(dx->pp, cmd, cmd->n + (zArgsEnd-zArgs) + 1) ){ + goto cleanup; + } + unsigned char * zo = cmd->z + cmd->n; + unsigned char const *zi = zArgs; +#if !defined(NDEBUG) + unsigned char const * zoEnd = cmd->z + cmd->nAlloc; +#endif + for( ; zi<zArgsEnd; ++zi){ + if( '\n'!=*zi && '\r'!=*zi ) *zo++ = *zi; + } + assert( zoEnd > zo ); + *zo = 0; + cmd->n = zo - cmd->z; + } + assert( !dxppCode ); + + if( doCapture ){ + assert( dx->d->closer ); + if( cmpp_dx_consume_b(dx, body, &dx->d->closer, 1, + cmpp_dx_consume_F_PROCESS_OTHER_D) ){ + goto cleanup; + } + while( nChompIn-- && cmpp_b_chomp(body) ){} + po.fpToChild = &fpToChild; + } + + if( dumpDebug ){ + g_warn("%s%s -debug: cmd argsAsGroup=%d n=%u z=%s", + cmpp_dx_delim(dx), dx->d->name.z, + (int)argsAsGroup, + (unsigned)cmd->n, cmd->z); + } + if( argsAsGroup ){ + cmpp_popen_args(dx, &cmdArgs, &po); + }else{ + unsigned char const * z = cmd->z; + //cmpp_skip_snl(&z, cmd->z + cmd->n); + cmpp_popen(dx->pp, z, popenFlags, &po); + } + if( dxppCode ) goto cleanup; + int rc = 0; + if( doCapture ){ + /* Bug: if body is too bug (no idea how much that is), this will + block while waiting on input from the child. This can easily + happen with #include -raw. */ +#if 0 + /* Failed attempt to work around it. */ + assert( fpToChild ); + enum { BufSize = 128 }; + unsigned char buf[BufSize]; + cmpp_size_t nLeft = body->n; + unsigned char const * z = body->z; + while( nLeft>0 && !dxppCode ){ + cmpp_size_t nWrite = nLeft < BufSize ? nLeft : BufSize; + g_warn("writing %u to child...", (unsigned)nWrite); + rc = cmpp_output_f_FILE(fpToChild, z, nWrite); + if( rc ){ + cmpp_dx_err_set(dx, rc, "Error feeding stdin to piped process."); + break; + } + z += nWrite; + nLeft -= nWrite; + fflush(fpToChild); + cmpp_size_t nRead = BufSize; + rc = cmpp_input_f_fd(&po.fdFromChild, &buf[0], &nRead); + if( rc ) goto err_reading; + cmpp_b_append4(dx->pp, &chout, buf, nRead);\ + } + if( !dxppCode ){ + g_warn0("reading from child..."); + rc = cmpp_stream( cmpp_input_f_fd, &po.fdFromChild, + cmpp_output_f_b, chout ); + if( rc ) goto err_reading; + } + g_warn0("I/O done"); +#else + //g_warn("writing %u bytes to child...", (unsigned)body->n); + rc = cmpp_output_f_FILE(fpToChild, body->z, body->n); + if( rc ){ + cmpp_dx_err_set(dx, rc, "Error feeding stdin to piped process."); + goto cleanup; + } + //g_warn("wrote %u bytes to child.", (unsigned)body->n); + fclose(fpToChild); + fpToChild = 0; + if( dxppCode ) goto cleanup; + goto stream_chout; +#endif + }else{ + stream_chout: + //g_warn0("waiting on child..."); + rc = cmpp_stream(cmpp_input_f_fd, &po.fdFromChild, + cmpp_output_f_b, chout); + //g_warn0("I/O done"); + if( rc ){ + //err_reading: + cmpp_dx_err_set(dx, rc, "Error reading stdout from piped process."); + goto cleanup; + } + } + while( nChompOut-- && cmpp_b_chomp(chout) ){} + //g_warn("Read in:\n%.*s", (int)chout->n, chout->z); + cmpp_dx_out_raw(dx, chout->z, chout->n); + +cleanup: + cmpp_args_cleanup(&cmdArgs); + cmpp_b_return(dx->pp, chout); + cmpp_b_return(dx->pp, cmd); + cmpp_b_return(dx->pp, body); + cmpp_b_return(dx->pp, bArg); + cmpp_pclose(&po); +} +#endif /* #ifndef CMPP_OMIT_D_PIPE */ + +/** + #sum ...args + + Emits the sum of its arguments, treating each as an + integer. Non-integer arguments are silently skipped. +*/ +static void cmpp_dx_f_sum(cmpp_dx *dx){ + int64_t n = 0, i = 0; + cmpp_b b = cmpp_b_empty; + for( cmpp_arg const * arg = dx->args.arg0; + arg && !cmpp_dx_err_check(dx); arg = arg->next ){ + if( 0==cmpp_arg_to_b(dx, arg, cmpp_b_reuse(&b), + cmpp_arg_to_b_F_BRACE_CALL) + && cmpp__is_int64(b.z, b.n, &i) ){ + n += i; + } + } + cmpp_b_append_i64(cmpp_b_reuse(&b), n); + cmpp_dx_out_raw(dx, b.z, b.n); + cmpp_b_clear(&b); +} + +/** + #arg ?flags? the-arg + + -trim-left + -trim-right + -trim: trim both sides + + It sends its arg to cmpp_arg_to_b() to expand it, optionally + trims the result, and emits that value. + + This directive is not expected to be useful except, perhaps in + testing cmpp itself. Its trim flags, in particular, aren't commonly + useful because #arg is only useful in a function call context and + those unconditionally trim their output. +*/ +static void cmpp_dx_f_arg(cmpp_dx *dx){ + cmpp_flag32_t a2bFlags = cmpp_arg_to_b_F_BRACE_CALL; + bool trimL = false, trimR = false; + cmpp_arg const * arg = dx->args.arg0; + for( ; arg && !cmpp_dx_err_check(dx); arg = arg->next ){ +#define FLAG(X)if( cmpp_arg_isflag(arg, X) ) + FLAG("-raw") { + a2bFlags = cmpp_arg_to_b_F_FORCE_STRING; + continue; + } + FLAG("-trim-left") { trimL=true; continue; } + FLAG("-trim-right") { trimR=true; continue; } + FLAG("-trim") { trimL=trimR=true; continue; } +#undef FLAG + break; + } + if( arg ){ + cmpp_b * const b = cmpp_b_borrow(dx->pp); + if( b && 0==cmpp_arg_to_b(dx, arg, b, a2bFlags) ){ + unsigned char const * zz = b->z; + unsigned char const * zzEnd = b->z + b->n; + if( trimL ) cmpp_skip_snl(&zz, zzEnd); + if( trimR ) cmpp_skip_snl_trailing(zz, &zzEnd); + if( zzEnd-zz ){ + cmpp_dx_out_raw(dx, zz, zzEnd-zz); + } + } + cmpp_b_return(dx->pp, b); + }else if( !cmpp_dx_err_check(dx) ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, "Expecting an argument."); + } +} + +/** + #join ?flags? ...args + + -s SEPARATOR: sets the separator for its RHS arguments. Default=space. + + -nl: append a newline (will be stripped by [call]s!). This is the default + when !cmpp_dx_is_call(dx). + + -nonl: do not append a newline. Default when dx->isCall. +*/ +static void cmpp_dx_f_join(cmpp_dx *dx){ + cmpp_b * const b = cmpp_b_borrow(dx->pp); + cmpp_b * const bSep = cmpp_b_borrow(dx->pp); + cmpp_flag32_t a2bFlags = cmpp_arg_to_b_F_BRACE_CALL; + bool addNl = !cmpp_dx_is_call(dx); + int n = 0; + if( !b || !bSep ) goto end; + if( !dx->args.argc ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "%s%s expects ?flags? ...args", + cmpp_dx_delim(dx), dx->d->name.z); + goto end; + } + cmpp_b_append_ch(bSep, ' '); + cmpp_check_oom(dx->pp, bSep->z); + for( cmpp_arg const * arg = dx->args.arg0; arg + && !b->errCode + && !bSep->errCode + && !cmpp_dx_err_check(dx); + arg = arg->next ){ +#define FLAG(X)if( cmpp_arg_isflag(arg, X) ) + FLAG("-s"){ + if( !arg->next ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Missing SEPARATOR argument to -s."); + break; + } + cmpp_arg_to_b(dx, arg->next, + cmpp_b_reuse(bSep), + cmpp_arg_to_b_F_BRACE_CALL); + arg = arg->next; + continue; + } + //FLAG("-nl"){ addNl=true; continue; } + FLAG("-nonl"){ addNl=false; continue; } +#undef FLAG + if( n++ && cmpp_dx_out_raw(dx, bSep->z, bSep->n) ){ + break; + } + if( cmpp_arg_to_b(dx, arg, cmpp_b_reuse(b), a2bFlags) ){ + break; + } + cmpp_dx_out_raw(dx, b->z, b->n); + } + if( !cmpp_dx_err_check(dx) ){ + if( !n ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Expecting at least one argument."); + }else if( addNl ){ + cmpp_dx_out_raw(dx, "\n", 1); + } + } +end: + cmpp_b_return(dx->pp, b); + cmpp_b_return(dx->pp, bSep); +} + + +/* Impl. for #file */ +static void cmpp_dx_f_file(cmpp_dx *dx){ + if( !dx->args.arg0 ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Expecting one or more arguments"); + return; + } + cmpp_d const * const d = dx->d; + enum e_op { + op_none, op_exists, op_join + }; + cmpp_b * const b0 = cmpp_b_borrow(dx->pp); + if( !b0 ) goto end; + enum e_op op = op_none; + cmpp_arg const * opArg = 0; + cmpp_arg const * arg = 0; + for( arg = dx->args.arg0; + 0==dxppCode && arg; + arg = arg->next ){ + if( op_none==op ){ + if( cmpp_arg_equals(arg, "exists") ){ + op = op_exists; + opArg = arg->next; + arg = opArg->next; + if( arg ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "%s%s exists: too many arguments", + cmpp_dx_delim(dx), d->name.z); + goto end; + } + break; + }else if( cmpp_arg_equals(arg, "join") ){ + op = op_join; + if( !arg->next ) goto missing_arg; + arg = arg->next; + break; + }else{ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Unknown %s%s command: %s", + cmpp_dx_delim(dx), d->name.z, arg->z); + goto end; + } + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, "%s%s unhandled argument: %s", + cmpp_dx_delim(dx), d->name.z, arg->z); + goto end; + } + } + switch( op ){ + case op_none: goto missing_arg; + case op_join: { + int i = 0; + cmpp_flag32_t const bFlags = cmpp_arg_to_b_F_BRACE_CALL; + for( ; arg; arg = arg->next, ++i ){ + if( cmpp_arg_to_b(dx, arg, cmpp_b_reuse(b0), bFlags) + || (i && cmpp_dx_out_raw(dx, "/", 1)) + || (b0->n && cmpp_dx_out_raw(dx, b0->z, b0->n)) ){ + break; + } + } + cmpp_dx_out_raw(dx, "\n", 1); + break; + } + case op_exists: { + assert( opArg ); + bool const b = cmpp__file_is_readable((char const *)opArg->z); + cmpp_dx_out_raw(dx, b ? "1\n" : "0\n", 2); + break; + } + } +end: + cmpp_b_return(dx->pp, b0); + return; +missing_arg: + if( arg ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, "%s%s %s: missing argument", + cmpp_dx_delim(dx), d->name.z, arg->z ); + }else{ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, "%s%s: missing subcommand", + cmpp_dx_delim(dx), d->name.z); + } + goto end; +} + + +/** + #cmp LHS op RHS +*/ +static void cmpp_dx_f_cmp(cmpp_dx *dx){ + cmpp_b * const bL = cmpp_b_borrow(dx->pp); + cmpp_b * const bR = cmpp_b_borrow(dx->pp); + cmpp_flag32_t a2bFlags = cmpp_arg_to_b_F_BRACE_CALL; + if( !bL || !!bR ) goto end; + for( cmpp_arg const * arg = dx->args.arg0; arg + && !cmpp_dx_err_check(dx); + arg = arg->next ){ + if( !bL->z ){ + cmpp_arg_to_b(dx, arg, bL, a2bFlags); + continue; + } + if( !bR->z ){ + cmpp_arg_to_b(dx, arg, bR, a2bFlags); + continue; + } + goto usage; + } + + if( cmpp_dx_err_check(dx) ) goto end; + if( !bL->z || !bR->z ){ + usage: + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Usage: LHS RHS"); + goto end; + } + assert( bL->z ); + assert( bR->z ); + char cbuf[20]; + int const cmp = strcmp((char*)bL->z, (char*)bR->z); + int const n = snprintf(cbuf, sizeof(cbuf), "%d", cmp); + assert(n>0); + cmpp_dx_out_raw(dx, cbuf, (cmpp_size_t)n); + +end: + cmpp_b_return(dx->pp, bL); + cmpp_b_return(dx->pp, bR); +} + + +#if 0 +/* Impl. for dummy placeholder. */ +static void cmpp_dx_f_todo(cmpp_dx *dx){ + cmpp_d const * d = cmpp_dx_d(dx); + g_warn("TODO: directive handler for %s", d->name.z); +} +#endif + +/** + If zName matches one of the delayed-load directives, that directive + is registered and 0 is returned. CMPP_RC_NO_DIRECTIVE is returned if + no match is found, but pp's error state is not updated in that + case. If a match is found and registration fails, that result code + will propagate via pp. +*/ +int cmpp__d_delayed_load(cmpp *pp, char const *zName){ + if( ppCode ) return ppCode; + int rc = CMPP_RC_NO_DIRECTIVE; + unsigned const nName = strlen(zName); + + pp->pimpl->flags.isInternalDirectiveReg = true; + +#define M(NAME) (nName==sizeof(NAME)-1 && 0==strcmp(zName,NAME)) +#define M_OC(NAME) (M(NAME) || M("/" NAME)) +#define M_IF(NAME) if( M(NAME) ) +#define CF(X) cmpp_d_F_ ## X +#define F_A_RAW CF(ARGS_RAW) +#define F_A_LIST CF(ARGS_LIST) +#define F_EXPR CF(ARGS_LIST) | CF(NOT_SIMPLIFY) +#define F_UNSAFE cmpp_d_F_NOT_IN_SAFEMODE +#define F_NC cmpp_d_F_NO_CALL +#define F_CALL cmpp_d_F_CALL_ONLY +#define DREG0(SYMNAME, NAME, OPENER, OFLAGS, CLOSER, CFLAGS) \ + cmpp_d_reg SYMNAME = { \ + .name = NAME, \ + .opener = { \ + .f = OPENER, \ + .flags = OFLAGS \ + }, \ + .closer = { \ + .f = CLOSER, \ + .flags = CFLAGS \ + }, \ + .dtor = 0, \ + .state = 0 \ + } + +#define DREG(NAME, OPENER, OFLAGS, CLOSER, CFLAGS ) \ + DREG0(const rReg, NAME, OPENER, OFLAGS, CLOSER, CFLAGS ); \ + rc = cmpp_d_register(pp, &rReg, NULL); \ + goto end + + /* The #if family requires some hand-holding... */ + if( M_OC("if") || M("elif") || M("else") ) { + DREG0(rIf, "if", + cmpp_dx_f_if, F_EXPR | F_NC | CF(FLOW_CONTROL), + cmpp_dx_f_if_dangler, 0); + DREG0(rElif, "elif", + cmpp_dx_f_if_dangler, F_NC, + 0, 0); + DREG0(rElse, "else", + cmpp_dx_f_if_dangler, F_NC, + 0, 0); + CmppIfState * const cis = cmpp__malloc(pp, sizeof(*cis)); + if( !cis ) goto end; + memset(cis, 0, sizeof(*cis)); + rIf.state = cis; + rIf.dtor = cmpp_mfree; + if( cmpp_d_register(pp, &rIf, &cis->dIf) + /* rIf must be first to avoid leaking cis on error */ + || cmpp_d_register(pp, &rElif, &cis->dElif) + || cmpp_d_register(pp, &rElse, &cis->dElse) ){ + rc = ppCode; + }else{ + assert( cis->dIf && cis->dElif && cis->dElse ); + assert( !cis->dEndif ); + assert( cis == cis->dIf->impl.state ); + assert( cmpp_mfree==cis->dIf->impl.dtor ); + cis->dElif->impl.state + = cis->dElse->impl.state + = cis; + cis->dElif->closer + = cis->dElse->closer + = cis->dEndif + = cis->dIf->closer; + rc = 0; + } + goto end; + }/* #if and friends */ + + /* Basic core directives... */ +#define M_IF_CORE(N,OPENER,OFLAGS,CLOSER,CFLAGS) \ + if( M_OC(N) ){ \ + DREG(N, OPENER, OFLAGS, CLOSER, CFLAGS); \ + } (void)0 + + M_IF_CORE("@", cmpp_dx_f_at, F_A_LIST, + cmpp_dx_f_dangling_closer, 0); + M_IF_CORE("arg", cmpp_dx_f_arg, F_A_LIST, 0, 0); + M_IF_CORE("assert", cmpp_dx_f_expr, F_EXPR, 0, 0); + M_IF_CORE("cmp", cmpp_dx_f_cmp, F_A_LIST, 0, 0); + M_IF_CORE("define", cmpp_dx_f_define, F_A_LIST, + cmpp_dx_f_dangling_closer, 0); + M_IF_CORE("delimiter", cmpp_dx_f_delimiter, F_A_LIST, + cmpp_dx_f_dangling_closer, 0); + M_IF_CORE("error", cmpp_dx_f_error, F_A_RAW, 0, 0); + M_IF_CORE("expr", cmpp_dx_f_expr, F_EXPR, 0, 0); + M_IF_CORE("join", cmpp_dx_f_join, F_A_LIST, 0, 0); + M_IF_CORE("once", cmpp_dx_f_once, F_A_LIST | F_NC, + cmpp_dx_f_dangling_closer, 0); + M_IF_CORE("pragma", cmpp_dx_f_pragma, F_A_LIST, 0, 0); + M_IF_CORE("savepoint", cmpp_dx_f_savepoint, F_A_LIST, 0, 0); + M_IF_CORE("stderr", cmpp_dx_f_stderr, F_A_RAW, 0, 0); + M_IF_CORE("sum", cmpp_dx_f_sum, F_A_LIST, 0, 0); + M_IF_CORE("undef", cmpp_dx_f_undef, F_A_LIST, 0, 0); + M_IF_CORE("undefined-policy", cmpp_dx_f_undef_policy, F_A_LIST, 0, 0); + M_IF_CORE("//", cmpp_dx_f_noop, F_A_RAW, 0, 0); + M_IF_CORE("file", cmpp_dx_f_file, + F_A_LIST | F_UNSAFE, 0, 0); + +#undef M_IF_CORE + + + /* Directives which can be disabled via build flags or + flags to cmpp_ctor()... */ +#define M_IF_FLAGGED(NAME,FLAG,OPENER,OFLAGS,CLOSER,CFLAGS) \ + M_IF(NAME) { \ + if( 0==(FLAG & pp->pimpl->flags.newFlags) ) { \ + DREG(NAME,OPENER,OFLAGS,CLOSER,CFLAGS); \ + } \ + goto end; \ + } + +#ifndef CMPP_OMIT_D_INCLUDE + M_IF_FLAGGED("include", cmpp_ctor_F_NO_INCLUDE, + cmpp_dx_f_include, F_A_LIST | F_UNSAFE, + 0, 0); +#endif + +#ifndef CMPP_OMIT_D_PIPE + M_IF_FLAGGED("pipe", cmpp_ctor_F_NO_PIPE, + cmpp_dx_f_pipe, F_A_RAW | F_UNSAFE, + cmpp_dx_f_dangling_closer, 0); +#endif + +#ifndef CMPP_OMIT_D_DB + M_IF_FLAGGED("attach", cmpp_ctor_F_NO_DB, + cmpp_dx_f_attach, F_A_LIST | F_UNSAFE, + 0, 0); + M_IF_FLAGGED("detach", cmpp_ctor_F_NO_DB, + cmpp_dx_f_detach, F_A_LIST | F_UNSAFE, + 0, 0); + if( 0==(cmpp_ctor_F_NO_DB & pp->pimpl->flags.newFlags) + && (M_OC("query") || M("query:no-rows")) ){ + DREG0(rQ, "query", cmpp_dx_f_query, F_A_LIST | F_UNSAFE, + cmpp_dx_f_dangling_closer, 0); + cmpp_d * dQ = 0; + rc = cmpp_d_register(pp, &rQ, &dQ); + if( 0==rc ){ + /* + It would be preferable to delay registration of query:no-rows + until we need it, but doing so causes an error when: + + |#if 0 + |#query + |... + |#query:no-rows HERE + |... + |#/query + |#/if + + Because query:no-rows won't have been registered, and unknown + directives are an error even in skip mode. Maybe they + shouldn't be. Maybe we should just skip them in skip mode. + That's only been an issue since doing delayed registration of + directives, so it's not come up until recently (as of + 2025-10-27). i was so hoping to be able to get _rid_ of skip + mode at some point. + */ + cmpp_d * dNoRows = 0; + cmpp_d_reg const rNR = { + .name = "query:no-rows", + .opener = { + .f = cmpp_dx_f_dangling_closer, + .flags = F_NC + } + }; + rc = cmpp_d_register(pp, &rNR, &dNoRows); + if( 0==rc ){ + dNoRows->closer = dQ->closer; + assert( !dQ->impl.state ); + dQ->impl.state = dNoRows; + } + } + goto end; + } +#endif /*CMPP_OMIT_D_DB*/ + +#if CMPP_D_MODULE + extern void cmpp_dx_f_module(cmpp_dx *); + M_IF_FLAGGED("module", cmpp_ctor_F_NO_MODULE, + cmpp_dx_f_module, F_A_LIST | F_UNSAFE, + 0, 0); +#endif + +#undef M_IF_FLAGGED + +#ifndef NDEBUG + M_IF("experiment"){ + DREG("experiment", cmpp_dx_f_experiment, + F_A_LIST | F_UNSAFE, 0, 0); + } +#endif + +end: +#undef DREG +#undef DREG0 +#undef F_EXPR +#undef F_A_RAW +#undef F_A_LIST +#undef F_UNSAFE +#undef F_NC +#undef F_CALL +#undef CF +#undef M +#undef M_OC +#undef M_IF + pp->pimpl->flags.isInternalDirectiveReg = false; + return ppCode ? ppCode : rc; +} +/* +** 2026-02-07: +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +** +************************************************************************ +** This file houses filesystem-related APIs libcmpp. +*/ + +#include <unistd.h> + +/** + There are APIs i'd _like_ to have here, but the readily-available + code for them BSD license, so can't be pasted in here. Examples: + + - Filename canonicalization. + + - Cross-platform getcwd() (see below). + + - Windows support. This requires, in addition to the different + filesystem APIs, converting strings into something it can use. + + All of that adds up to infrastructure... which already exists + elsewhere but can't be copied here while retaining this project's + license. +*/ + +bool cmpp__file_is_readable(char const *zFile){ + return 0==access(zFile, R_OK); +} + +#if 0 +FILE *cmpp__fopen(const char *zName, const char *zMode){ + FILE *f; + if(zName && ('-'==*zName && !zName[1])){ + f = (strchr(zMode, 'w') || strchr(zMode,'+')) + ? stdout + : stdin + ; + }else{ + f = fopen(zName, zMode); + } + return f; +} + +void cmpp__fclose( FILE * f ){ + if(f && (stdin!=f) && (stdout!=f) && (stderr!=f)){ + fclose(f); + } +} +#endif +/* +** 2025-11-07: +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +** +************************************************************************ +** This file houses the arguments-handling-related pieces for libcmpp. +*/ + +const cmpp_args_pimpl cmpp_args_pimpl_empty = + cmpp_args_pimpl_empty_m; +const cmpp_args cmpp_args_empty = cmpp_args_empty_m; +const cmpp_arg cmpp_arg_empty = cmpp_arg_empty_m; + +//just in case these ever get dynamic state +void cmpp_arg_cleanup(cmpp_arg *arg){ + if( arg ) *arg = cmpp_arg_empty; +} + +//just in case these ever get dynamic state +void cmpp_arg_reuse(cmpp_arg *arg){ + if( arg ) *arg = cmpp_arg_empty; +} + +/** Resets li's list for re-use but does not free it. Returns li. */ +static CmppArgList * CmppArgList_reuse(CmppArgList *li){ + for(cmpp_size_t n = li->nAlloc; n; ){ + cmpp_arg_reuse( &li->list[--n] ); + assert( !li->list[n].next ); + } + li->n = 0; + return li; +} + +/** Free all memory owned by li but does not free li. */ +void CmppArgList_cleanup(CmppArgList *li){ + const CmppArgList CmppArgList_empty = CmppArgList_empty_m; + while( li->nAlloc ){ + cmpp_arg_cleanup( &li->list[--li->nAlloc] ); + } + cmpp_mfree(li->list); + *li = CmppArgList_empty; +} + +/** Returns the most-recently-appended arg of li back to li's + free-list. */ +static void CmppArgList_unappend(CmppArgList *li){ + assert( li->n ); + if( li->n ){ + cmpp_arg_reuse( &li->list[--li->n] ); + } +} + +cmpp_arg * CmppArgList_append(cmpp *pp, CmppArgList *li){ + cmpp_arg * p = 0; + assert( li->list ? li->nAlloc : 0==li->nAlloc ); + if( 0==ppCode + && 0==CmppArgList_reserve(pp, li, + cmpp__li_reserve1_size(li,10)) ){ + p = &li->list[li->n++]; + cmpp_arg_reuse( p ); + } + return p; +} + +void cmpp_args_pimpl_cleanup(cmpp_args_pimpl *p){ + assert( !p->nextFree ); + cmpp_b_clear(&p->argOut); + CmppArgList_cleanup(&p->argli); + *p = cmpp_args_pimpl_empty; +} + +static void cmpp_args_pimpl_reuse(cmpp_args_pimpl *p){ + assert( !p->nextFree ); + cmpp_b_reuse(&p->argOut); + CmppArgList_reuse(&p->argli); + assert( !p->argOut.n ); + assert( !p->argli.n ); +} + +static void cmpp_args_pimpl_return(cmpp *pp, cmpp_args_pimpl *p){ + if( p ){ + assert( p->pp ); + cmpp__pi(pp); + assert( !p->nextFree ); + cmpp_args_pimpl_reuse(p); + p->nextFree = pi->recycler.argPimpl; + pi->recycler.argPimpl = p; + } +} + +static cmpp_args_pimpl * cmpp_args_pimpl_borrow(cmpp *pp){ + cmpp__pi(pp); + cmpp_args_pimpl * p = 0; + if( pi->recycler.argPimpl ){ + p = pi->recycler.argPimpl; + pi->recycler.argPimpl = p->nextFree; + p->nextFree = 0; + p->pp = pp; + assert( !p->argOut.n && "Buffer was used when not borrowed" ); + }else{ + p = cmpp__malloc(pp, sizeof(*p)); + if( 0==cmpp_check_oom(pp, p) ) { + *p = cmpp_args_pimpl_empty; + p->pp = pp; + } + } + return p; +} + +CMPP__EXPORT(void, cmpp_args_cleanup)(cmpp_args *a){ + if( a ){ + if( a->pimpl ){ + cmpp * const pp = a->pimpl->pp; + assert( pp ); + if( pp ){ + cmpp_args_pimpl_return(pp, a->pimpl); + }else{ + cmpp_args_pimpl_cleanup(a->pimpl); + cmpp_mfree(a->pimpl); + } + } + *a = cmpp_args_empty; + } +} + +CMPP__EXPORT(void, cmpp_args_reuse)(cmpp_args *a){ + cmpp_args_pimpl * const p = a->pimpl; + if( p ) cmpp_args_pimpl_reuse(p); + *a = cmpp_args_empty; + a->pimpl = p; +} + +int cmpp_args__init(cmpp * pp, cmpp_args * a){ + if( 0==ppCode ){ + if( a->pimpl ){ + assert( a->pimpl->pp == pp ); + cmpp_args_reuse(a); + assert(! a->pimpl->argOut.n ); + assert( a->pimpl->pp == pp ); + }else{ + a->pimpl = cmpp_args_pimpl_borrow(pp); + assert( !a->pimpl || a->pimpl->pp==pp ); + } + } + return ppCode; +} + +/** + Declare cmpp_argOp_f_NAME(). +*/ +#define cmpp_argOp_decl(NAME) \ + static void cmpp_argOp_f_ ## NAME (cmpp_dx *dx, \ + cmpp_argOp const *op, \ + cmpp_arg const *vLhs, \ + cmpp_arg const **pvRhs, \ + int *pResult) +cmpp_argOp_decl(compare); + +#if 0 +cmpp_argOp_decl(logical1); +cmpp_argOp_decl(logical2); +cmpp_argOp_decl(defined); +#endif + +static const struct { + const cmpp_argOp opAnd; + const cmpp_argOp opOr; + const cmpp_argOp opGlob; + const cmpp_argOp opNotGlob; + const cmpp_argOp opNot; + const cmpp_argOp opDefined; +#define cmpp_argOps_cmp_map(E) E(Eq) E(Neq) E(Lt) E(Le) E(Gt) E(Ge) +#define E(NAME) const cmpp_argOp op ## NAME; + cmpp_argOps_cmp_map(E) +#undef E +} cmpp_argOps = { + .opAnd = { + .ttype = cmpp_TT_OpAnd, + .arity = 2, + .assoc = 0, + .xCall = 0//cmpp_argOp_f_logical2 + }, + .opOr = { + .ttype = cmpp_TT_OpOr, + .arity = 2, + .assoc = 0, + .xCall = 0//cmpp_argOp_f_logical2 + }, + .opGlob = { + .ttype = cmpp_TT_OpGlob, + .arity = 2, + .assoc = 0, + .xCall = 0//cmpp_argOp_f_glob + }, + .opNotGlob = { + .ttype = cmpp_TT_OpNotGlob, + .arity = 2, + .assoc = 0, + .xCall = 0//cmpp_argOp_f_glob + }, + .opNot = { + .ttype = cmpp_TT_OpNot, + .arity = 1, + .assoc = 1, + .xCall = 0//cmpp_argOp_f_logical1 + }, + .opDefined = { + .ttype = cmpp_TT_OpDefined, + .arity = 1, + .assoc = 1, + .xCall = 0//cmpp_argOp_f_defined + }, + /* Comparison ops... */ +#define E(NAME) .op ## NAME = { \ + .ttype = cmpp_TT_Op ## NAME, .arity = 2, .assoc = 0, \ + .xCall = cmpp_argOp_f_compare }, + cmpp_argOps_cmp_map(E) +#undef E +}; + +cmpp_argOp const * cmpp_argOp_for_tt(cmpp_tt tt){ + switch(tt){ + case cmpp_TT_OpAnd: return &cmpp_argOps.opAnd; + case cmpp_TT_OpOr: return &cmpp_argOps.opOr; + case cmpp_TT_OpGlob: return &cmpp_argOps.opGlob; + case cmpp_TT_OpNot: return &cmpp_argOps.opNot; + case cmpp_TT_OpDefined: return &cmpp_argOps.opDefined; +#define E(NAME) case cmpp_TT_Op ## NAME: return &cmpp_argOps.op ## NAME; + cmpp_argOps_cmp_map(E) +#undef E + default: return NULL; + } +} +#define argOp(ARG) cmpp_argOp_for_tt((ARG)->ttype) + +#if 0 +cmpp_argOp_decl(logical1){ + assert( cmpp_TT_OpNot==op->ttype ); + assert( !vRhs ); + assert( vLhs ); + if( 0==cmpp__arg_toBool(dx, vLhs, pResult) ){ + *pResult = !*pResult; + } +} + +cmpp_argOp_decl(logical2){ + assert( vRhs ); + assert( vLhs ); + int vL = 0; + int vR = 0; + if( 0==cmpp__arg_toBool(dx, vLhs, &vL) + && 0==cmpp__arg_toBool(dx, vRhs, &vR) ){ + switch( op->ttype ){ + case cmpp_TT_OpAnd: *pResult = vL && vR; break; + case cmpp_TT_OpOr: *pResult = vL || vR; break; + default: + cmpp__fatal("Cannot happen: illegal op mapping"); + } + } +} + +cmpp_argOp_decl(defined){ + assert( cmpp_TT_OpDefined==op->ttype ); + assert( !vRhs ); + assert( vLhs ); + if( cmpp_TT_Word==vLhs->ttype ){ + *pResult = cmpp_has(pp, (char const *)vLhs->z, vLhs->n); + if( !*pResult && vLhs->n>1 && '#'==vLhs->z[0] ){ + *pResult = !!cmpp__d_search3(pp, vLhs->z+1, + cmpp__d_search3_F_NO_DLL); + } + }else{ + cmpp__err(pp, CMPP_RC_TYPE, "Invalid token type %s for %s", + cmpp__tt_cstr(vLhs->ttype, true), + cmpp__tt_cstr(op->ttype, false)); + } +} +#endif + +#if 0 +static cmpp_argOp const * cmpp_argOp_isCompare(cmpp_tt tt){ + cmpp_argOp const * const p = cmpp_argOp_for_tt(tt); + switch( p ? p->ttype : cmpp_TT_None ){ +#define E(NAME) case cmpp_TT_Op ## NAME: return p; + cmpp_argOps_cmp_map(E) +#undef E + return p; + case cmpp_TT_None: + default: + return NULL; + } +} +#endif + +/** + An internal helper for cmpp_argOp_...(). It binds some value of + *paArg to column bindNdx of query q and sets *paArg to the next + argument to be consumed. This function expects that q is set up to + do the right thing when *paArg is a Word-type value (see + cmpp_argOp_f_compare()). +*/ +static void cmpp_argOp__cmp_bind(cmpp_dx * const dx, + sqlite3_stmt * const q, + int bindNdx, + cmpp_arg const ** paArg){ + cmpp_arg const * const arg = *paArg; + assert(arg); + switch( dxppCode ? 0 : arg->ttype ){ + case 0: break; + case cmpp_TT_Word: + /* In this case, q is supposed to be set up to use + CMPP__SEL_V_FROM(bindNdx), i.e. it expects the verbatim word + and performs the expansion to its value in the query. */ + cmpp__bind_textn(dx->pp, q, bindNdx, arg->z, arg->n); + *paArg = arg->next; + break; + case cmpp_TT_StringAt: + case cmpp_TT_String: + case cmpp_TT_Int:{ + cmpp__bind_arg(dx, q, bindNdx, arg); + *paArg = arg->next; + break; + } + case cmpp_TT_OpNot: + case cmpp_TT_OpDefined: + case cmpp_TT_GroupParen:{ + int rv = 0; + if( 0==cmpp__arg_toBool(dx, arg, &rv, paArg) ){ + cmpp__bind_int(dx->pp, q, bindNdx, rv); + } + *paArg = arg->next; + break; + } + /* TODO? cmpp_TT_GroupParen */ + default: + cmpp_dx_err_set(dx, CMPP_RC_TYPE, + "Invalid argument type (%s) for the comparison " + "queries: %s", + cmpp_tt_cstr(arg->ttype), arg->z); + } +} + +/** + Internal helper for cmp_argOp_...(). + + Expects q to be a query with an integer in result column 0. This + steps/resets the query and applies the given comparison operator's + logic to column 0's value, placing the result of the operator in + *pResult. + + If q has no result row, a default value of 0 is assumed. +*/ +static void cmpp_argOp__cmp_apply(cmpp * const pp, + cmpp_argOp const * const op, + sqlite3_stmt * const q, + int * const pResult){ + if( 0==ppCode ){ + int rc = cmpp__step(pp, q, false); + assert( SQLITE_ROW==rc || ppCode ); + if( SQLITE_ROW==rc ){ + rc = sqlite3_column_int(q, 0); + }else{ + rc = 0; + } + switch( op->ttype ){ + case 0: break; + case cmpp_TT_OpEq: *pResult = 0==rc; break; + case cmpp_TT_OpNeq: *pResult = 0!=rc; break; + case cmpp_TT_OpLt: *pResult = rc<0; break; + case cmpp_TT_OpLe: *pResult = rc<=0; break; + case cmpp_TT_OpGt: *pResult = rc>0; break; + case cmpp_TT_OpGe: *pResult = rc>=0; break; + default: + cmpp__fatal("Cannot happen: invalid arg mapping"); + } + } + cmpp__stmt_reset(q); +} + +/** + Applies *paRhs as the RHS of an integer binary operator, the LHS of + which is the lhs argument. The result is put in *pResult. On + success *paRhs is set to the next argument for the expression to + parse. +*/ +static void cmpp_argOp_applyTo(cmpp_dx *dx, + cmpp_argOp const * const op, + int lhs, + cmpp_arg const ** paRhs, + int * pResult){ + sqlite3_stmt * q = 0; + cmpp_arg const * aRhs = *paRhs; + assert(aRhs); + q = cmpp_TT_Word==aRhs->ttype + ? cmpp__stmt(dx->pp, CmppStmt_cmpVD, false) + : cmpp__stmt(dx->pp, CmppStmt_cmpVV, false); + if( q ){ + char numbuf[32]; + int const nNum = snprintf(numbuf, sizeof(numbuf), "%d", lhs); + cmpp__bind_textn(dx->pp, q, 1, ustr_c(numbuf), nNum); + cmpp_argOp__cmp_bind(dx, q, 2, paRhs); + cmpp_argOp__cmp_apply(dx->pp, op, q, pResult); + } +} + +cmpp_argOp_decl(compare){ + cmpp_arg const * const vRhs = *pvRhs; + sqlite3_stmt * q = 0; + /* Select which query to use, depending on whether each + of the LHS/RHS are Word tokens. For Word tokens + the corresponding query columns get bound to + a subquery which resolves the word. Non-word + tokens get bound as-is. */ + if( cmpp_TT_Word==vLhs->ttype ){ + q = cmpp_TT_Word==vRhs->ttype + ? cmpp__stmt(dx->pp, CmppStmt_cmpDD, false) + : cmpp__stmt(dx->pp, CmppStmt_cmpDV, false); + if(0){ + g_warn("\nvLhs=%s %s\nvRhs=%s %s\n", + cmpp_tt_cstr(vLhs->ttype), vLhs->z, + cmpp_tt_cstr(vRhs->ttype), vRhs->z); + } + }else if( cmpp_TT_Word==vRhs->ttype ){ + q = cmpp__stmt(dx->pp, CmppStmt_cmpVD, false); + }else{ + q = cmpp__stmt(dx->pp, CmppStmt_cmpVV, false); + } + if( q ){ + //cmpp__bind_textn(pp, q, 1, vLhs->z, vLhs->n); + cmpp_argOp__cmp_bind(dx, q, 1, &vLhs); + cmpp_argOp__cmp_bind(dx, q, 2, pvRhs); + cmpp_argOp__cmp_apply(dx->pp, op, q, pResult); + } +} + +#undef cmpp_argOp_decl + +#if 0 +static inline int cmpp_dxt_isBinOp(cmpp_tt tt){ + cmpp_argOp const * const a = cmpp_argOp_for_tt(tt); + return a ? 2==a->arity : 0; +} + +static inline int cmpp_dxt_isUnaryOp(cmpp_tt tt){ + return tt==cmpp_TT_OpNot || cmpp_TT_OpDefined; +} + +static inline int cmpp_dxt_isGroup(cmpp_tt tt){ + return tt==cmpp_TT_GroupParen || tt==cmpp_TT_GroupBrace || cmpp_TT_GroupSquiggly; +} +#endif + +int cmpp__arg_evalSubToInt(cmpp_dx *dx, + cmpp_arg const *arg, + int * pResult){ + cmpp_args sub = cmpp_args_empty; + if( 0==cmpp_args_parse(dx, &sub, arg->z, arg->n, 0) ){ + cmpp__args_evalToInt(dx, &sub, pResult); + } + cmpp_args_cleanup(&sub); + return dxppCode; +} + +int cmpp__args_evalToInt(cmpp_dx * const dx, + cmpp_args const *pArgs, + int * pResult){ + if( dxppCode ) return dxppCode; + + cmpp_arg const * pNext = 0; + cmpp_arg const * pPrev = 0; + int result = *pResult; + cmpp_b osL = cmpp_b_empty; + cmpp_b osR = cmpp_b_empty; + static int level = 0; + ++level; + +#define lout(fmt,...) if(0) g_stderr("%.*c" fmt, level*2, ' ', __VA_ARGS__) + + //lout("START %s(): %s\n", __func__, pArgs->pimpl->buf.argsRaw.z); + for( cmpp_arg const *arg = pArgs->arg0; + arg && 0==dxppCode; + pPrev = arg, arg = pNext ){ + pNext = arg->next; + if( cmpp_TT_Noop==arg->ttype ){ + arg = pPrev /* help the following arg to DTRT */; + continue; + } + cmpp_argOp const * const thisOp = argOp(arg); + cmpp_argOp const * const nextOp = pNext ? argOp(pNext) : 0; + if( 0 ){ + lout("arg: %s @%p %s\n", + cmpp__tt_cstr(arg->ttype, true), arg, arg->z); + if(1){ + if( pPrev ) lout(" prev arg: %s %s\n", + cmpp__tt_cstr(pPrev->ttype, true), pPrev->z); + if( pNext ) lout(" next arg: %s %s\n", + cmpp__tt_cstr(pNext->ttype, true), pNext->z); + } + } + if( thisOp ){ /* Basic validation */ + if( !pNext ){ + dxserr("Missing '%s' RHS.", + cmpp__tt_cstr(thisOp->ttype, false)); + break; + }else if( !pPrev && 2==thisOp->arity ){ + dxserr("Missing %s LHS.", + cmpp__tt_cstr(thisOp->ttype, false)); + break; + } + if( nextOp && nextOp->arity>1 ){ + dxserr("Invalid '%s' RHS: %s", arg->z, pNext->z); + break; + } + } + + switch( arg->ttype ){ + + case cmpp_TT_OpNot: + case cmpp_TT_OpDefined: + if( pPrev && !argOp(pPrev) ){ + cmpp_dx_err_set(dx, CMPP_RC_CANNOT_HAPPEN, + "We expected to have consumed '%s' by " + "this point.", + pPrev->z); + }else{ + cmpp__arg_toBool(dx, arg, &result, &pNext); + } + break; + + case cmpp_TT_OpAnd: + case cmpp_TT_OpOr:{ + assert( pNext ); + assert( pPrev ); + /* Reminder to self: we can't add short-circuiting of the RHS + right now because the handling of chained unary ops on the + RHS is handled via cmpp__arg_toBool(). */ + int rv = 0; + if( 0==cmpp__arg_toBool(dx, pNext, &rv, &pNext) ){ + if( cmpp_TT_OpAnd==arg->ttype ) result = result && rv; + else result = result || rv; + } + //g_warn("post-and/or pNext=%s\n", pNext ? pNext->z : 0); + break; + } + + case cmpp_TT_OpNotGlob: + case cmpp_TT_OpGlob:{ + assert( pNext ); + assert( pPrev ); + assert( pNext!=arg ); + assert( pPrev!=arg ); + if( cmpp_arg_to_b(dx, pNext, &osL, 0) ){ + break; + } + unsigned char const * const zGlob = osL.z; + if( 0==cmpp_arg_to_b(dx, pPrev, &osR, 0) ){ + if( 0 ){ + g_warn("zGlob=[%s] z=[%s]", zGlob, osR.z); + } + result = 0==sqlite3_strglob((char const *)zGlob, + (char const *)osR.z); + if( cmpp_TT_OpNotGlob==arg->ttype ){ + result = !result; + } + //g_warn("\nzGlob=%s\nz=%s\nresult=%d", zGlob, z, result); + } + pNext = pNext->next; + break; + } + +#define E(NAME) case cmpp_TT_Op ## NAME: + cmpp_argOps_cmp_map(E) { + cmpp_argOp const * const prevOp = pPrev ? argOp(pPrev) : 0; + if( prevOp ){ + /* Chained operators */ + cmpp_argOp_applyTo(dx, thisOp, result, &pNext, &result); + }else{ + assert( pNext ); + assert( pPrev ); + assert( thisOp ); + assert( thisOp->xCall ); + thisOp->xCall(dx, thisOp, pPrev, &pNext, &result); + } + break; + } +#undef E + +#define checkConsecutiveNonOps \ + if( pPrev && !argOp(pPrev) ){ \ + dxserr("Illegal consecutive non-operators: %s %s", \ + pPrev->z, arg->z); \ + break; \ + }(void)0 + + case cmpp_TT_Int: + case cmpp_TT_String: + checkConsecutiveNonOps; + if( !cmpp__is_int(arg->z, arg->n, &result) ){ + /* This is mostly for and/or ops. glob will reach back and + grab arg->z. */ + result = 0; + } + break; + case cmpp_TT_Word: + checkConsecutiveNonOps; + cmpp__get_int(dx->pp, arg->z, arg->n, &result); + break; + case cmpp_TT_GroupParen:{ + checkConsecutiveNonOps; + cmpp_args sub = cmpp_args_empty; + if( 0==cmpp_args_parse(dx, &sub, arg->z, arg->n, 0) ){ + cmpp__args_evalToInt(dx, &sub, &result); + } + cmpp_args_cleanup(&sub); + break; + } + case cmpp_TT_GroupBrace:{ + checkConsecutiveNonOps; + cmpp_b b = cmpp_b_empty; + if( 0==cmpp_call_str(dx->pp, arg->z, arg->n, &b, 0) ){ + cmpp__is_int(b.z, b.n, &result); + } + cmpp_b_clear(&b); + break; + } +#undef checkConsecutiveNonOps + default: + assert( arg->z ); + dxserr("Illegal expression token %s: %s", + cmpp__tt_cstr(arg->ttype, true), arg->z); + }/*switch(arg->ttype)*/ + }/* foreach arg */ + if( 0 ){ + lout("END %s() result=%d\n", __func__, result); + } + --level; + if( !dxppCode ){ + *pResult = result; + } + cmpp_b_clear(&osL); + cmpp_b_clear(&osR); + return dxppCode; +#undef lout +} + +#undef argOp +#undef cmpp_argOp_decl + +static inline cmpp_tt cmpp_dxt_is_group(cmpp_tt ttype){ + switch(ttype){ + case cmpp_TT_GroupParen: + case cmpp_TT_GroupBrace: + case cmpp_TT_GroupSquiggly: + return ttype; + default: + return cmpp_TT_None; + } +} + +int cmpp_args_parse(cmpp_dx * const dx, + cmpp_args * const pArgs, + unsigned char const * const zInBegin, + cmpp_ssize_t nIn, + cmpp_flag32_t flags){ + assert( zInBegin ); + unsigned char const * const zInEnd = + zInBegin + cmpp__strlenu(zInBegin, nIn); + + if( cmpp_args__init(dx->pp, pArgs) ) return dxppCode; + if( 0 ){ + g_warn("whole input = <<%.*s>>", (int)(zInEnd-zInBegin), + zInBegin); + } + unsigned char const * zPos = zInBegin; + cmpp_size_t const nBuffer = + /* Buffer size for our copy of the args. We need to know the + size before we start so that we can have each arg reliably + point back into this without it being reallocated during + parsing. */ + (cmpp_size_t)(zInEnd - zInBegin) + /* Plus we need one final NUL and one NUL byte per argument, but + we don't yet know how many arguments we will have, so let's + estimate... */ + + ((cmpp_size_t)(zInEnd - zInBegin))/3 + + 5/*fudge room*/; + cmpp_b * const buffer = &pArgs->pimpl->argOut; + assert( !buffer->n ); + if( cmpp_b_reserve3(dx->pp, buffer, nBuffer) ){ + return dxppCode; + } + unsigned char * zOut = buffer->z; + unsigned char const * const zOutEnd = zOut + buffer->nAlloc - 1; + cmpp_arg * prevArg = 0; +#if !defined(NDEBUG) + unsigned char const * const zReallocCheck = buffer->z; +#endif + + if(0) g_warn("pre-parsed line: %.*s", (zInEnd - zInBegin), + zInBegin); + pArgs->arg0 = NULL; + pArgs->argc = 0; + for( int i = 0; zPos<zInEnd; ++i){ + //g_stderr("i=%d prevArg=%p\n",i, prevArg); + cmpp_arg * const arg = + CmppArgList_append(dx->pp, &pArgs->pimpl->argli); + if( !arg ) return dxppCode; + assert( pArgs->pimpl->argli.n ); + if( 0 ) g_warn("zPos=<<%.*s>>", (int)(zInEnd-zPos), zPos); + if( cmpp_arg_parse(dx, arg, &zPos, zInEnd, &zOut, zOutEnd) ){ + if( 0 ) g_warn("zPos=<<%.*s>>", (int)(zInEnd-zPos), zPos); + break; + } + if( 0 ){ + g_warn("#%d zPos=<<%.*s>>", i, (int)(zInEnd-zPos), zPos); + g_warn("#%d arg n=%u z=<<%.*s>> %s", i, (int)arg->n, (int)arg->n, arg->z, arg->z); + } + assert( zPos<=zInEnd ); + if( 0 ){ + g_stderr("ttype=%d %s n=%u z=%.*s\n", arg->ttype, + cmpp__tt_cstr(arg->ttype, true), + (unsigned)arg->n, (int)arg->n, arg->z); + } + if( cmpp_TT_Eof==arg->ttype ){ + CmppArgList_unappend(&pArgs->pimpl->argli); + break; + } + switch( 0==(flags & cmpp_args_F_NO_PARENS) + ? cmpp_dxt_is_group( arg->ttype ) + : 0 ){ + case cmpp_TT_GroupParen:{ + /* Sub-expression. We tokenize it here just to ensure that we + can, so we can fail earlier rather than later. This is why + we need a recycler for the cmpp_args buffer memory. */ + cmpp_args sub = cmpp_args_empty; + cmpp_args_parse(dx, &sub, arg->z, arg->n, flags); + //g_stderr("Parsed sub-expr: %s\n", sub.buffer.z); + cmpp_args_cleanup(&sub); + break; + } + case cmpp_TT_GroupBrace: + case cmpp_TT_GroupSquiggly: + default: break; + } + if( dxppCode ) break; + if( prevArg ){ + assert( !prevArg->next ); + prevArg->next = arg; + } + prevArg = arg; + }/*foreach input char*/ + //g_stderr("rc=%s argc=%d\n", cmpp_rc_cstr(dxppCode), pArgs->args.n); + if( 0==dxppCode ){ + pArgs->argc = pArgs->pimpl->argli.n; + assert( !pArgs->arg0 ); + if( pArgs->argc ) pArgs->arg0 = pArgs->pimpl->argli.list; + if( zOut<zInEnd ) *zOut = 0; + if( 0 ){ + for( cmpp_arg const * a = pArgs->arg0; a; a = a->next ){ + g_stderr(" got: %s %.*s\n", cmpp__tt_cstr(a->ttype, true), + a->n, a->z); + } + } + } + assert(zReallocCheck==buffer->z + && "Else buffer was reallocated, invalidating argN->z"); + return dxppCode; +} + +CMPP__EXPORT(int, cmpp_args_clone)(cmpp *pp, cmpp_arg const * const a0, + cmpp_args * const dest){ + if( cmpp_args__init(pp, dest) || !a0 ) return ppCode; + cmpp_b * const ob = &dest->pimpl->argOut; + CmppArgList * const argli = &dest->pimpl->argli; + unsigned int i = 0; + cmpp_size_t nReserve = 0 /* arg buffer mem to preallocate */; + + assert( !ob->n ); + assert( !dest->arg0 ); + assert( !dest->argc ); + assert( !argli->n ); + + /* Preallocate ob->z to fit a copy of a0's args. */ + for( cmpp_arg const * a = a0; a; ++i, a = a->next ){ + nReserve += a->n + 1/*NUL byte*/; + } + if( cmpp_b_reserve3(pp, ob, nReserve+1) + || CmppArgList_reserve(pp, argli, i) ){ + goto end; + } + assert( argli->nAlloc>=i ); + i = 0; +#ifndef NDEBUG + unsigned char const * const zReallocCheck = ob->z; +#endif + for( cmpp_arg const * a = a0; a; ++i, a = a->next ){ + cmpp_arg * const aNew = &argli->list[i]; + aNew->n = a->n; + aNew->z = ob->z + ob->n; + aNew->ttype = a->ttype; + if( i ) argli->list[i-1].next = aNew; + assert( !a->z[a->n] && "Expecting a NUL byte there" ); + cmpp_b_append4(pp, ob, a->z, a->n+1/*NUL byte*/); + if( 0 ){ + g_warn("arg#%d=%s <<<%.*s>>> %s", i, cmpp_tt_cstr(a->ttype), + (int)a->n, a->z, a->z); + } + assert( zReallocCheck==ob->z + && "This cannot fail: ob->z was pre-allocated" ); + } + dest->argc = i; + dest->arg0 = i ? &argli->list[0] : 0; +end: + if( ppCode ){ + cmpp_args_reuse(dest); + } + return ppCode; +} + +CMPP__EXPORT(int, cmpp_dx_args_clone)(cmpp_dx * dx, cmpp_args *pOut){ + return cmpp_args_clone(dx->pp, dx->args.arg0, pOut); +} + +char * cmpp_arg_strdup(cmpp *pp, cmpp_arg const *arg){ + char * z = 0; + if( 0==ppCode ){ + z = sqlite3_mprintf("%s",arg->z); + cmpp_check_oom(pp, z); + } + return z; +} + +static cmpp_tt cmpp_tt_forWord(unsigned char const *z, unsigned n, + cmpp_tt dflt){ + static const struct { +#define E(NAME,STR) struct CmppSnippet NAME; + cmpp_tt_map(E) +#undef E + } ttStr = { +#define E(NAME,STR) \ + .NAME = {(unsigned char const *)STR,sizeof(STR)-1}, + cmpp_tt_map(E) +#undef E + }; +#define CASE(NAME) if( 0==memcmp(ttStr.NAME.z, z, n) ) return cmpp_TT_ ## NAME + switch( n ){ + case 1: + CASE(OpEq); + CASE(Plus); + CASE(Minus); + break; + case 2: + CASE(OpOr); + CASE(ShiftL); + //CASE(ShiftR); + //CASE(ArrowL); + CASE(ArrowR); + CASE(OpNeq); + CASE(OpLt); + CASE(OpLe); + CASE(OpGt); + CASE(OpGe); + break; + case 3: + CASE(OpAnd); + CASE(OpNot); + CASE(ShiftL3); + break; + case 4: + CASE(OpGlob); + break; + case 7: + CASE(OpDefined); + break; +#undef CASE + } +#if 0 + bool b = cmpp__is_int(z, n, NULL); + if( 1|| !b ){ + g_warn("is_int(%s)=%d", z, b); + } + return b ? cmpp_TT_Int : dflt; +#else + return cmpp__is_int(z, n, NULL) ? cmpp_TT_Int : dflt; +#endif +} + +int cmpp_arg_parse(cmpp_dx * const dx, cmpp_arg *pOut, + unsigned char const **pzIn, + unsigned char const *zInEnd, + unsigned char ** pzOut, + unsigned char const * zOutEnd){ + unsigned char const * zi = *pzIn; + unsigned char * zo = *pzOut; + cmpp_tt ttype = cmpp_TT_None; + +#if 0 + // trying to tickle valgrind + for(unsigned char const *x = zi; x < zInEnd; ++x ){ + assert(*x); + } +#endif + cmpp_arg_reuse( pOut ); + cmpp_skip_snl( &zi, zInEnd ); + if( zi>=zInEnd ){ + *pzIn = zi; + pOut->ttype = cmpp_TT_Eof; + return 0; + } +#define out(CH) if(zo>=zOutEnd) goto notEnoughOut; *zo++ = CH +#define eot_break if( cmpp_TT_None!=ttype ){ keepGoing = 0; break; } (void)0 + pOut->z = zo; + bool keepGoing = true; + for( ; keepGoing + && 0==dxppCode + && zi<zInEnd + && zo<zOutEnd; ){ + cmpp_tt ttOverride = cmpp_TT_None; + switch( (int)*zi ){ + case 0: keepGoing = false; break; + case ' ': case '\t': case '\n': case '\r': + eot_break; + cmpp_skip_snl( &zi, zInEnd ); + break; + case '-': + if ('>'==zi[1] ){ + ttype = cmpp_TT_ArrowR; + out(*zi++); + out(*zi++); + keepGoing = false; + }else{ + goto do_word; + } + break; + case '=': + eot_break; keepGoing = false; ttype = cmpp_TT_OpEq; out(*zi++); break; +#define opcmp(CH,TT,TTEQ,TTSHIFT,TTARROW) \ + case CH: eot_break; keepGoing = false; ttype = TT; out(*zi++); \ + if( zi<zInEnd && '='==*zi ){ out(*zi++); ttype = TTEQ; } \ + else if( zi<zInEnd && CH==*zi ){ out(*zi++); ttype = TTSHIFT; } \ + else if( (int)TTARROW && zi<zInEnd && '-'==*zi ){ out(*zi++); ttype = TTARROW; } + + opcmp('>',cmpp_TT_OpGt,cmpp_TT_OpGe,cmpp_TT_ShiftR,0) break; + opcmp('<',cmpp_TT_OpLt,cmpp_TT_OpLe,cmpp_TT_ShiftL,cmpp_TT_ArrowL) + if( cmpp_TT_ShiftL==ttype && zi<zInEnd && '<'==zi[0] ) { + out(*zi++); + ttype = cmpp_TT_ShiftL3; + } + break; +#undef opcmp + case '!': + eot_break; + keepGoing = false; + out(*zi++); + if( zi < zInEnd && '='==*zi ){ + ttype = cmpp_TT_OpNeq; + out('='); + ++zi; + }else{ + while( zi < zInEnd && '!'==*zi ){ + out(*zi++); + } + ttype = cmpp_TT_OpNot; + } + break; + case '@': + if( zi+2 >= zInEnd || ('"'!=zi[1] && '\''!=zi[1]) ){ + goto do_word; + } + //if( cmpp__StringAtIsOk(dx->pp) ) break; + ttOverride = cmpp_TT_StringAt; + ++zi /* consume opening '@' */; + //g_stderr("@-string override\n"); + /* fall through */ + case '"': + case '\'': { + /* Parse a string. We do not support backslash-escaping of any + sort here. Strings which themselves must contain quotes + should use the other quote type. */ + keepGoing = false; + if( cmpp_TT_None!=ttype ){ + cmpp_dx_err_set(dx, CMPP_RC_SYNTAX, + "Misplaced quote character near: %.*s", + (int)(zi+1 - *pzIn), *pzIn); + break; + } + unsigned char const * zQuoteAt = zi; + if( cmpp__find_closing(dx->pp, &zQuoteAt, zInEnd) ){ + break; + } + assert( zi+1 <= zQuoteAt ); + assert( *zi == *zQuoteAt ); + if( (zQuoteAt - zi - 2) >= (zOutEnd-zo) ){ + goto notEnoughOut; + } + memcpy(zo, zi+1, zQuoteAt - zi - 1); + //g_warn("string=<<%.*s>>", (zQuoteAt-zi-1), zo); + zo += zQuoteAt - zi - 1; + zi = zQuoteAt + 1/* closing quote */; + ttype = (cmpp_TT_None==ttOverride ? cmpp_TT_String : ttOverride); + break; + } + case '[': + case '{': + case '(': { + /* Slurp these as a single token for later sub-parsing */ + keepGoing = false; + unsigned char const * zAt = zi; + if( cmpp__find_closing(dx->pp, &zi, zInEnd) ) break; + /* Transform the output, eliding the open/close characters and + trimming spaces. We need to keep newlines intact, as the + content may be free-form, intended for other purposes, e.g. + the #pipe or #query directives. */ + ttype = ('('==*zAt + ? cmpp_TT_GroupParen + : ('['==*zAt + ? cmpp_TT_GroupBrace + : cmpp_TT_GroupSquiggly)); + ++zAt /* consume opening brace */; + /* Trim leading and trailing space, but retain tabs and all but + the first and last newline. */ + cmpp_skip_space(&zAt, zi); + if( zAt<zInEnd ){ + if( '\n'==*zAt ) ++zAt; + else if(zAt+1<zInEnd && '\r'==*zAt && '\n'==zAt[1]) zAt+=2; + } + for( ; zAt<zi; ++zAt ){ + out(*zAt); + } + if(0) g_warn("parse1: group n=%u [%.*s]\n", + (zi-zAt), (zi-zAt), zAt); + while( zo>*pzOut && ' '==zo[-1] ) *--zo = 0; + if( zo>*pzOut && '\n'==zo[-1] ){ + *--zo = 0; + if( zo>*pzOut && '\r'==zo[-1] ){ + *--zo = 0; + } + } + ++zi /* consume the closer */; + break; + } + default: + ; do_word: + out(*zi++); + ttype = cmpp_TT_Word; + break; + } + //g_stderr("kg=%d char=%d %c\n", keepGoing, (int)*zi, *zi); + } + if( dxppCode ){ + /* problem already reported */ + }else if( zo>=zOutEnd-1 ){ + notEnoughOut: + cmpp_dx_err_set(dx, CMPP_RC_RANGE, + "Ran out of output space (%u bytes) while " + "parsing an argument", (unsigned)(zOutEnd-*pzOut)); + }else{ + pOut->n = (zo - *pzOut); + if( cmpp_TT_None==ttype ){ + pOut->ttype = cmpp_TT_Eof; + }else if( cmpp_TT_Word==ttype && pOut->n ){ + pOut->ttype = cmpp_tt_forWord(pOut->z, pOut->n, ttype); + }else{ + pOut->ttype = ttype; + } + *zo++ = 0; + *pzIn = zi; + *pzOut = zo; + switch( pOut->ttype ){ + case cmpp_TT_Int: + if( '+'==*pOut->z ){ /* strip leading + */ + ++pOut->z; + --pOut->n; + } + break; + default: + break; + } + if(0){ + g_stderr("parse1: %s n=%u <<%.*s>>", + cmpp__tt_cstr(pOut->ttype, true), pOut->n, + pOut->n, pOut->z); + } + } +#undef out +#undef eot_break + return dxppCode; +} + +int cmpp__arg_toBool(cmpp_dx * const dx, cmpp_arg const *arg, + int * pResult, cmpp_arg const **pNext){ + switch( dxppCode ? 0 : arg->ttype ){ + case 0: break; + + case cmpp_TT_Word: + *pNext = arg->next; + *pResult = cmpp__get_bool(dx->pp, arg->z, arg->n); + break; + + case cmpp_TT_Int: + *pNext = arg->next; + cmpp__is_int(arg->z, arg->n, pResult)/*was already validated*/; + break; + + case cmpp_TT_String: + case cmpp_TT_StringAt:{ + unsigned char const * z = 0; + cmpp_size_t n = 0; + cmpp_b os = cmpp_b_empty; + if( 0==cmpp__arg_expand_ats(dx, &os, cmpp_atpol_CURRENT, + arg, cmpp_TT_StringAt, &z, &n) ){ + *pNext = arg->next; + *pResult = n>0 && 0!=memcmp("0\0", z, 2); + } + cmpp_b_clear(&os); + break; + } + + case cmpp_TT_GroupParen:{ + *pNext = arg->next; + cmpp_args sub = cmpp_args_empty; + if( 0==cmpp_args_parse(dx, &sub, arg->z, arg->n, 0) ){ + cmpp__args_evalToInt(dx, &sub, pResult); + } + cmpp_args_cleanup(&sub); + break; + } + + case cmpp_TT_OpDefined: + if( !arg->next ){ + dxserr("Missing '%s' RHS.", arg->z); + }else if( cmpp_TT_Word!=arg->next->ttype ){ + dxserr( "Invalid '%s' RHS: %s", arg->z, arg->next->z); + }else{ + cmpp_arg const * aOperand = arg->next; + *pNext = aOperand->next; + if( aOperand->n>1 + && '#'==aOperand->z[0] + && !!cmpp__d_search3(dx->pp, (char const*)aOperand->z+1, + cmpp__d_search3_F_NO_DLL) ){ + *pResult = 1; + }else{ + *pResult = cmpp_has(dx->pp, (char const *)aOperand->z, + aOperand->n); + } + } + break; + + case cmpp_TT_OpNot:{ + assert( arg->next && "See cmpp_args__not_simplify()"); + assert( cmpp_TT_OpNot!=arg->next->ttype && "See cmpp_args__not_simplify()"); + if( 0==cmpp__arg_toBool(dx, arg->next, pResult, pNext) ){ + *pResult = !*pResult; + } + break; + } + + default: + dxserr("Invalid token type %s for %s(): %s", + cmpp__tt_cstr(arg->ttype, true), __func__, arg->z); + break; + } + return dxppCode; +} + +CMPP__EXPORT(int, cmpp_arg_to_b)(cmpp_dx * const dx, cmpp_arg const *arg, + cmpp_b * ob, cmpp_flag32_t flags){ + /** + Reminder to self: this function specifically does not do any + expression evaluation of its arguments. Please avoid the + temptation to make it do so. Unless it proves necessary. Or + useful. Even then, though, consider the implications deeply + before doing so. + */ + switch( dxppCode + ? 0 + : ((cmpp_arg_to_b_F_FORCE_STRING & flags) + ? cmpp_TT_String : arg->ttype) ){ + + case 0: + break; + case cmpp_TT_Word: + if( 0==(flags & cmpp_arg_to_b_F_NO_DEFINES) ){ + cmpp__get_b(dx->pp, arg->z, arg->n, ob, true); + break; + } + goto theDefault; + case cmpp_TT_StringAt:{ + unsigned char const * z = 0; + cmpp_size_t n = 0; + if( 0 ){ + g_warn("ob->z [%.*s] [%s]", (int)ob->n, ob->z, ob->z); + } + if( 0==cmpp__arg_expand_ats(dx, ob, cmpp_atpol_CURRENT, arg, + cmpp_TT_StringAt, &z, &n) + && 0 ){ + g_warn("expanded at [%.*s] [%s]", (int)n, z, z); + g_warn("ob->z [%.*s] [%s]", (int)ob->n, ob->z, ob->z); + } + break; + } + case cmpp_TT_GroupBrace: + if( !(cmpp_arg_to_b_F_NO_BRACE_CALL & flags) + && (cmpp_arg_to_b_F_BRACE_CALL & flags) ){ + cmpp_call_str(dx->pp, arg->z, arg->n, ob, 0); + break; + } + /* fall through */ + default: { + theDefault: ; + cmpp_outputer oss = cmpp_outputer_b; + oss.state = ob;//no: cmpp_b_reuse(ob); Append instead. + cmpp__out2(dx->pp, &oss, arg->z, arg->n); + break; + } + } + return dxppCode; +} + +int cmpp__bind_arg(cmpp_dx * const dx, sqlite3_stmt * const q, + int bindNdx, cmpp_arg const * const arg){ + + if( 0 ){ + g_warn("bind #%d %s <<%.*s>>", bindNdx, + cmpp__tt_cstr(arg->ttype, true), + (int)arg->n, arg->z); + } + switch( arg->ttype ){ + default: + case cmpp_TT_Int: + case cmpp_TT_String: + cmpp__bind_textn(dx->pp, q, bindNdx, arg->z, (int)arg->n); + break; + + case cmpp_TT_Word: + case cmpp_TT_StringAt:{ + cmpp_b os = cmpp_b_empty; + if( 0==cmpp_arg_to_b(dx, arg, &os, 0) ){ + if( 0 ){ + g_warn("bind #%d <<%s>> => <<%.*s>>", + bindNdx, arg->z, (int)os.n, os.z); + } + cmpp__bind_textn(dx->pp, q, bindNdx, os.z, (int)os.n); + } + cmpp_b_clear(&os); + break; + } + + case cmpp_TT_GroupParen:{ + cmpp_args sub = cmpp_args_empty; + int i = 0; + if( 0==cmpp_args_parse(dx, &sub, arg->z, arg->n, 0) + && 0==cmpp__args_evalToInt(dx, &sub, &i) ){ + /* See comment above about cmpp_TT_Int. */ + cmpp__bind_int_text(dx->pp, q, bindNdx, i); + } + cmpp_args_cleanup(&sub); + break; + } + + case cmpp_TT_GroupBrace:{ + cmpp_b b = cmpp_b_empty; + cmpp_call_str(dx->pp, arg->z, arg->n, &b, 0); + cmpp__bind_textn(dx->pp, q, bindNdx, b.z, b.n); + cmpp_b_clear(&b); + break; + } + + } + return dxppCode; +} + +/** + If a is in li->list, return its non-const pointer from li->list + (O(1)), else return NULL. +*/ +static cmpp_arg * CmppArgList_arg_nc(CmppArgList *li, cmpp_arg const * a){ + if( li->nAlloc && a>=li->list && a<(li->list + li->nAlloc) ){ + return li->list + (a - li->list); + } + return NULL; +} + +/** + To be called only by cmpp_dx_args_parse() and only if the current + directive asked for it via cmpp_d::flags cmpp_d_F_NOT_SIMPLIFY. + + Filter chains of "not" operators from pArgs, removing unnecessary + ones. Also collapse "not glob" into a single cmpp_TT_OpNotGlob argument. + Performs some basic validation as well to simplify downstream + operations. Returns p->err.code and is a no-op if that's set + before this is called. +*/ +static int cmpp_args__not_simplify(cmpp * const pp, cmpp_args *pArgs){ + cmpp_arg * pPrev = 0; + cmpp_arg * pNext = 0; + CmppArgList * const ali = &pArgs->pimpl->argli; + pArgs->argc = 0; + for( cmpp_arg * arg = ali->n ? &ali->list[0] : NULL; + arg && !ppCode; + pPrev=arg, arg = pNext ){ + pNext = CmppArgList_arg_nc(ali, arg->next); + assert( pNext || !arg->next ); + if( cmpp_TT_OpNot==arg->ttype ){ + if( !pNext ){ + serr("Missing '%s' RHS", arg->z); + break; + } + cmpp_argOp const * const nop = cmpp_argOp_for_tt(pNext->ttype); + if( nop && nop->arity>1 && cmpp_TT_OpGlob!=nop->ttype ){ + serr("Illegal '%s' RHS: binary '%s' operator", + arg->z, pNext->z); + break; + } + int bNeg = 1; + if( '!'==*arg->z ){ + /* odd number of ! == negate */ + bNeg = arg->n & 1; + } + while( pNext && cmpp_TT_OpNot==pNext->ttype ){ + bNeg = !bNeg; + arg->next = pNext = CmppArgList_arg_nc(ali, pNext->next); + } + if( pNext && cmpp_TT_OpGlob==pNext->ttype ){ + /* Transform it to a cmpp_TT_OpNotGlob or cmpp_TT_OpGlob. */ + assert( pNext->z > arg->z + arg->n ); + arg->n = pNext->z + pNext->n - arg->z; + arg->next = pNext->next; + arg->ttype = bNeg + ? cmpp_TT_OpNotGlob + : pNext->ttype; + ++pArgs->argc; + }else if( pPrev ){ + if( bNeg ){ + ++pArgs->argc; + }else{ + /* Snip this node out. */ + pPrev->next = pNext; + } + }else{ + assert( 0==pArgs->argc ); + ++pArgs->argc; + if( !bNeg ){ + arg->ttype = cmpp_TT_Noop; + } + } + /* Potential bug in waiting/fixme: by eliding all nots we are + ** changing the behavior from forced coercion to bool to + ** coercion to whatever the LHS wants. */ + }else{ + ++pArgs->argc; + } + } + pArgs->arg0 = pArgs->argc ? &ali->list[0] : NULL; + return ppCode; +} + +CMPP__EXPORT(int, cmpp_dx_args_parse)(cmpp_dx *dx, + cmpp_args *args){ + if( !dxppCode + && 0==cmpp_args_parse(dx, args, dx->args.z, dx->args.nz, + cmpp_args_F_NO_PARENS) + && (cmpp_d_F_NOT_SIMPLIFY & dx->d->flags) ){ + cmpp_args__not_simplify(dx->pp, args); + } + return dxppCode; +} + +/* Helper for cmpp_kav_each() and friends. */ +static +int cmpp__each_parse_args(cmpp_dx *dx, + cmpp_args *args, + unsigned char const *zBegin, + cmpp_ssize_t nz, + cmpp_flag32_t flags){ + if( 0==cmpp_args_parse(dx, args, zBegin, nz, cmpp_args_F_NO_PARENS) ){ + if( !args->argc + && (cmpp_kav_each_F_NOT_EMPTY & flags) ){ + cmpp_err_set(dx->pp, CMPP_RC_RANGE, + "Empty list is not permitted here."); + } + } + return dxppCode; +} + +/* Helper for cmpp_kav_each() and friends. */ +static +int cmpp__each_paren_expr(cmpp_dx *dx, cmpp_arg const * arg, + unsigned char * pOut, size_t nOut){ + cmpp_args sub = cmpp_args_empty; + int rc = cmpp_args_parse(dx, &sub, arg->z, arg->n, 0); + if( 0==rc ){ + int d = 0; + rc = cmpp__args_evalToInt(dx, &sub, &d); + if( 0==rc ){ + snprintf((char *)pOut, nOut, "%d", d); + } + } + cmpp_args_cleanup(&sub); + return rc; +} + +CMPP__EXPORT(int, cmpp_kav_each)(cmpp_dx *dx, + unsigned char const *zBegin, + cmpp_ssize_t nIn, + cmpp_kav_each_f callback, + void *callbackState, + cmpp_flag32_t flags){ + if( dxppCode ) return dxppCode; + /* Reminder to self: we cannot reuse internal buffers here because a + callback could recurse into this or otherwise use APIs which use + those same buffers. */ + cmpp_b bKey = cmpp_b_empty; + cmpp_b bVal = cmpp_b_empty; + bool const reqArrow = 0==(cmpp_kav_each_F_NO_ARROW & flags); + cmpp_args args = cmpp_args_empty; + unsigned char exprBuf[32] = {0}; + cmpp_size_t const nz = cmpp__strlenu(zBegin,nIn); + unsigned char const * const zEnd = zBegin + nz; + cmpp_flag32_t a2bK = 0, a2bV = 0 /*cmpp_arg_to_b() flags*/; + assert( zBegin ); + assert( zEnd ); + assert( zEnd>=zBegin ); + + if( cmpp__each_parse_args(dx, &args, zBegin, nz, flags) ){ + goto cleanup; + }else if( reqArrow && 0!=args.argc%3 ){ + cmpp_err_set(dx->pp, CMPP_RC_RANGE, + "Expecting a list of 3 tokens per entry: " + "KEY -> VALUE"); + }else if( !reqArrow && 0!=args.argc%2 ){ + cmpp_err_set(dx->pp, CMPP_RC_RANGE, + "Expecting a list of 2 tokens per entry: " + "KEY VALUE"); + } + if( cmpp_kav_each_F_CALL_KEY & flags ){ + a2bK |= cmpp_arg_to_b_F_BRACE_CALL; + flags |= cmpp_kav_each_F_EXPAND_KEY; + } + if( cmpp_kav_each_F_CALL_VAL & flags ){ + a2bV |= cmpp_arg_to_b_F_BRACE_CALL; + flags |= cmpp_kav_each_F_EXPAND_VAL; + } + cmpp_arg const * aNext = 0; + for( cmpp_arg const * aKey = args.arg0; + !dxppCode && aKey; + aKey = aNext ){ + aNext = aKey->next; + cmpp_arg const * aVal = aKey->next; + if( !aVal ){ + dxserr("Expecting %s after key '%s'.", + (reqArrow ? "->" : "a value"), + aKey->z); + break; + } + if( reqArrow ){ + if( cmpp_TT_ArrowR!=aVal->ttype ){ + dxserr("Expecting -> after key '%s'.", aKey->z); + break; + } + aVal = aVal->next; + if( !aVal ){ + dxserr("Expecting a value after '%s' ->.", aKey->z); + break; + } + } + //g_warn("\nkey=[%s]\nval=[%s]", aKey->z, aVal->z); + /* Expand the key/value parts if needed... */ + unsigned char const *zKey; + unsigned char const *zVal; + cmpp_size_t nKey, nVal; + if( cmpp_kav_each_F_EXPAND_KEY & flags ){ + if( cmpp_arg_to_b(dx, aKey, cmpp_b_reuse(&bKey), + a2bK) ){ + break; + } + zKey = bKey.z; + nKey = bKey.n; + }else{ + zKey = aKey->z; + nKey = aKey->n; + } + if( cmpp_TT_GroupParen==aVal->ttype + && (cmpp_kav_each_F_PARENS_EXPR & flags) ){ + if( cmpp__each_paren_expr(dx, aVal, &exprBuf[0], + sizeof(exprBuf)-1) ){ + break; + } + zVal = &exprBuf[0]; + nVal = cmpp__strlenu(zVal, -1); + }else if( cmpp_kav_each_F_EXPAND_VAL & flags ){ + if( cmpp_arg_to_b(dx, aVal, cmpp_b_reuse(&bVal), + a2bV) ){ + break; + } + zVal = bVal.z; + nVal = bVal.n; + }else{ + zVal = aVal->z; + nVal = aVal->n; + } + aNext = aVal->next; + if( 0!=callback(dx, zKey, nKey, zVal, nVal, callbackState) ){ + break; + } + } +cleanup: + cmpp_b_clear(&bKey); + cmpp_b_clear(&bVal); + cmpp_args_cleanup(&args); + return dxppCode; +} + +CMPP__EXPORT(int, cmpp_str_each)(cmpp_dx *dx, + unsigned char const *zBegin, + cmpp_ssize_t nIn, + cmpp_kav_each_f callback, void *callbackState, + cmpp_flag32_t flags){ + g_warn0("UNTESTED!"); + if( dxppCode ) return dxppCode; + /* Reminder to self: we cannot reuse internal buffers here because a + callback could recurse into this or otherwise use APIs which use + those same buffers. */ + cmpp_b ob = cmpp_b_empty; + cmpp_args args = cmpp_args_empty; + unsigned char exprBuf[32] = {0}; + cmpp_size_t const nz = cmpp__strlenu(zBegin,nIn); + unsigned char const * const zEnd = zBegin + nz; + assert( zBegin ); + assert( zEnd ); + assert( zEnd>=zBegin ); + + if( cmpp__each_parse_args(dx, &args, zBegin, nz, flags) ){ + goto cleanup; + } + cmpp_arg const * aNext = 0; + for( cmpp_arg const * arg = args.arg0; + !dxppCode && arg; + arg = aNext ){ + aNext = arg->next; + //g_warn("\nkey=[%s]\nval=[%s]", arg->z, aVal->z); + /* Expand the key/value parts if needed... */ + unsigned char const *zVal; + cmpp_size_t nVal; + if( cmpp_TT_GroupParen==arg->ttype + && (cmpp_kav_each_F_PARENS_EXPR & flags) ){ + if( cmpp__each_paren_expr(dx, arg, &exprBuf[0], + sizeof(exprBuf)-1) ){ + break; + } + zVal = &exprBuf[0]; + nVal = cmpp__strlenu(zVal, -1); + }else if( cmpp_kav_each_F_EXPAND_VAL & flags ){ + if( cmpp_arg_to_b(dx, arg, cmpp_b_reuse(&ob), 0) ){ + break; + } + zVal = ob.z; + nVal = ob.n; + }else{ + zVal = arg->z; + nVal = arg->n; + } + if( 0!=callback(dx, arg->z, arg->n, zVal, nVal, callbackState) ){ + break; + } + } +cleanup: + cmpp_b_clear(&ob); + cmpp_args_cleanup(&args); + return dxppCode; +} + +/** + Returns true if z _might_ be a cmpp_TT_StringAt, else false. It may have + false positives but won't have false negatives. + + This is only intended to be used on NUL-terminated strings, not a + pointer into a cmpp input source. +*/ +static bool cmpp__might_be_atstring(unsigned char const *z){ + char const * const x = strchr((char const *)z, '@'); + return x && !!strchr(x+1, '@'); +} + +int cmpp__arg_expand_ats(cmpp_dx const * const dx, + cmpp_b * os, + cmpp_atpol_e atPolicy, + cmpp_arg const * const arg, + cmpp_tt thisTtype, + unsigned char const **pExp, + cmpp_size_t * nExp){ + assert( os ); + cmpp_b_reuse(os); + if( 0==dxppCode + && (cmpp_TT_AnyType==thisTtype || thisTtype==arg->ttype) + && cmpp__might_be_atstring(arg->z) + && 0==cmpp__StringAtIsOk(dx->pp, atPolicy) ){ +#if 0 + if( !os->nAlloc ){ + cmpp_b_reserve3(os, 128); + } +#endif + cmpp_outputer oos = cmpp_outputer_b; + oos.state = os; + assert( !os->n ); + if( !cmpp_dx_out_expand(dx, &oos, arg->z, arg->n, + atPolicy ) ){ + *pExp = os->z; + if( nExp ) *nExp = os->n; + if( 0 ){ + g_warn("os->n=%u os->z=[%.*s]\n", os->n, (int)os->n, + os->z); + } + + } + }else if( !dxppCode ){ + *pExp = arg->z; + if( nExp ) *nExp = arg->n; + } + return dxppCode; +} + +bool cmpp__arg_wordIsPathOrFlag( + cmpp_arg const * const arg +){ + return cmpp_TT_Word==arg->ttype + && ('-'==(char)arg->z[0] + || strchr((char*)arg->z, '.') + || strchr((char*)arg->z, '-') + || strchr((char*)arg->z, '/') + || strchr((char*)arg->z, '\\')); +} +/* +** 2022-11-12: +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +** +************************************************************************ +** This file houses the cmpp_popen() pieces. +*/ +#if !defined(_POSIX_C_SOURCE) +# define _POSIX_C_SOURCE 200809L /* for fdopen() in stdio.h */ +#endif +#include <signal.h> + +const cmpp_popen_t cmpp_popen_t_empty = cmpp_popen_t_empty_m; + +#if CMPP_PLATFORM_IS_UNIX +#include <signal.h> +static int cmpp__err_errno(cmpp *pp, int errNo, char const *zContext){ + return cmpp_err_set(pp, cmpp_errno_rc(errNo, CMPP_RC_ERROR), + "errno #%d: %s", errNo, zContext); +} +#endif + +/** + Uses fork()/exec() to run a command in a separate process and open + a two-way stream to it. + + If azCmd is NULL then zCmd must contain the command to run and + any flags. It is passed as the 4th argument to + execl("/bin/sh", "/bin/sh", "-c", zCmd, NULL). + + If azCmd is not NULL then it must be suitable for use as the 2nd + argument to execv(2). execv(X, azCmd) is used in this case, where + X is (zCmd ? zCmd : azCmd[0]). + + Flags: + + - cmpp_popen_F_DIRECT: if azCmd is NULL and flags has this bit set then + zCmd is instead passed to execl(zCmd, zCmd, NULL). That can only + work if zCmd is a single command without arguments. + cmpp_popen_F_DIRECT has no effect if azCmd is not NULL. + + - cmpp_popen_F_PATH: tells it to use execlp() or execvp(), which + performs path lookup of its initial argument. + + On success: + + - po->fdFromChild is the child's stdout. Read from it to read from + the child. + + - If po->fpToChild is not NULL then *po->fpToChild is set to the + child's stdin. Write to it to send the child stuff. Be sure to + flush() and/or close() it to keep it from hanging forever. If + po->fpToChild is NULL then the stdin of the child is closed. + + - po->childPid will be set to the PID of the child process. + + On error: you know the drill. + + After calling this, the caller is obligated to pass po to + cmpp_pclose(). If the caller fcloses() *po->fpToChild then they + must set it to NULL so that passing it to cmpp_pclose() knows not + to close it. + + Bugs: because the command is run via /bin/sh -c ... we cannot tell + if it's actually found. All we can tell is that /bin/sh ran. + + Also: this doesn't capture stderr, so commands should redirect + stderr to stdout. Adding the child's stderr handle to cmpp_popen_t is + a potential TODO without a current use case. +*/ +static +int cmpp__popen_impl(cmpp *pp, unsigned char const *zCmd, + char * const * azCmd, cmpp_flag32_t flags, + cmpp_popen_t *po){ +#if !CMPP_PLATFORM_IS_UNIX + return cmpp__err(pp, CMPP_RC_UNSUPPORTED, + "Piping is not supported in this build."); +#else + if( ppCode ) return ppCode; +#define shut(P,N) close(P[N]) + /** Attribution: this impl is derived from one found in + the Fossil SCM. */ + int pin[2]; + int pout[2]; + + po->fdFromChild = -1; + if( po->fpToChild ) *po->fpToChild = 0; + if( pipe(pin)<0 ){ + return cmpp__err_errno(pp, errno, "pipe(in) failed"); + } + if( pipe(pout)<0 ){ + int const rc = cmpp__err_errno(pp, errno, + "pipe(out) failed"); + shut(pin,0); + shut(pin,1); + return rc; + } + po->childPid = fork(); + if( po->childPid<0 ){ + int const rc = cmpp__err_errno(pp, errno, "fork() failed"); + shut(pin,0); + shut(pin,1); + shut(pout,0); + shut(pout,1); + return rc; + } + signal(SIGPIPE,SIG_IGN); + if( po->childPid==0 ){ + /* The child process. */ + int fd; + close(0); + fd = dup(pout[0]); + if( fd!=0 ) { + cmpp__fatal("Error opening file descriptor 0."); + }; + shut(pout,0); + shut(pout,1); + close(1); + fd = dup(pin[1]); + if(fd!=1) { + cmpp__fatal("Error opening file descriptor 1."); + }; + shut(pin,0); + shut(pin,1); + if( azCmd ){ + if( pp->pimpl->flags.doDebug>1 ){ + for( int i = 0; azCmd[i]; ++i ){ + g_warn("execv arg[%d]=%s", i, azCmd[i]); + } + } + int (*exc)(const char *, char *const []) = + (cmpp_popen_F_PATH & flags) ? execvp : execv; + exc(zCmd ? (char*)zCmd : azCmd[0], azCmd); + cmpp__fatal("execv() failed"); + }else{ + g_debug(pp,2,("zCmd=%s\n", zCmd)); + int (*exc)(const char *, char const *, ...) = + (cmpp_popen_F_PATH & flags) ? execlp : execl; + if( cmpp_popen_F_DIRECT & flags ){ + exc((char*)zCmd, (char*)zCmd, (char*)0); + }else{ + exc("/bin/sh", "/bin/sh", "-c", zCmd, (char*)0); + } + cmpp__fatal("execl() failed"); + } + /* not reached */ + }else{ + /* The parent process. */ + //cmpp_outputer_flush(&pp->pimpl->out.ch); + po->fdFromChild = pin[0]; + shut(pin,1); + shut(pout,0); + if( po->fpToChild ){ + *po->fpToChild = fdopen(pout[1], "w"); + if( !*po->fpToChild ){ + shut(pin,0); + shut(pout,1); + po->fdFromChild = -1; + cmpp__err_errno(pp, errno, + "Error opening child process's stdin " + "FILE handle from its descriptor."); + } + }else{ + shut(pout,1); + } + return ppCode; + } +#undef shut +#endif +} + +int cmpp_popen(cmpp *pp, unsigned char const *zCmd, + cmpp_flag32_t flags, cmpp_popen_t *po){ + return cmpp__popen_impl(pp, zCmd, NULL, flags, po); +} + +int cmpp_popenv(cmpp *pp, char * const * azCmd, + cmpp_flag32_t flags, cmpp_popen_t *po){ + return cmpp__popen_impl(pp, NULL, azCmd, flags, po); +} + +int cmpp_popen_args(cmpp_dx *dx, cmpp_args const * args, + cmpp_popen_t *po){ +#if !CMPP_PLATFORM_IS_UNIX + return cmpp__popen_impl(dx->pp, NULL, 0, po) /* will fail */; +#else + if( dxppCode ) return dxppCode; + enum { MaxArgs = 128 }; + char * argv[MaxArgs] = {0}; + cmpp_size_t offsets[MaxArgs] = {0}; + cmpp_b osAll = cmpp_b_empty; + cmpp_b os1 = cmpp_b_empty; + if( args->argc >= MaxArgs ){ + return cmpp_dx_err_set(dx, CMPP_RC_RANGE, + "Too many arguments (%d). Max is %d.", + args->argc, (int)MaxArgs); + } + int i = 0; + for(cmpp_arg const * a = args->arg0; + a; ++i, a = a->next ){ + offsets[i] = osAll.n; + cmpp_flag32_t a2bFlags = cmpp_arg_to_b_F_BRACE_CALL; + if( cmpp__arg_wordIsPathOrFlag(a) ){ + a2bFlags |= cmpp_arg_to_b_F_FORCE_STRING; + } + if( cmpp_arg_to_b(dx, a, cmpp_b_reuse(&os1), a2bFlags) + || cmpp_b_append4(dx->pp, &osAll, os1.z, os1.n+1/*NUL*/) ){ + goto end; + } + assert( osAll.n > offsets[i] ); + if( 0 ){ + g_warn("execv arg[%d] = %s => %s", i, a->z, + osAll.z+offsets[i]); + } + } + argv[i] = 0; + for( --i; i >= 0; --i ){ + argv[i] = (char*)(osAll.z + offsets[i]); + if( 0 ){ + g_warn("execv arg[%d] = %s", i, argv[i]); + } + } +end: + if( 0==dxppCode ){ + cmpp__popen_impl(dx->pp, NULL, argv, 0, po); + } + cmpp_b_clear(&osAll); + cmpp_b_clear(&os1); + return dxppCode; +#endif +} + +int cmpp_pclose(cmpp_popen_t *po){ +#if CMPP_PLATFORM_IS_UNIX + if( po->fdFromChild>=0 ) close(po->fdFromChild); + if( po->fpToChild && *po->fpToChild ) fclose(*po->fpToChild); + int const childPid = po->childPid; + *po = cmpp_popen_t_empty; +#if 1 + int wp, rc = 0; + if( childPid>0 ){ + //kill(childPid, SIGINT); // really needed? + do{ + wp = waitpid(childPid, &rc, WNOHANG); + if( wp>0 ){ + if( WIFEXITED(rc) ){ + rc = WEXITSTATUS(rc); + }else if( WIFSIGNALED(rc) ){ + rc = WTERMSIG(rc); + }else{ + rc = 0/*???*/; + } + } + } while( wp>0 ); + } + return rc; +#elif 0 + while( waitpid(childPid, NULL, WNOHANG)>0 ){} +#else + if( childPid>0 ){ + kill(childPid, SIGINT); // really needed? + waitpid((pid_t)childPid, NULL, WNOHANG); + }else{ + while( waitpid( (pid_t)0, NULL, WNOHANG)>0 ){} + } +#endif +#endif +} +/* +** 2022-11-12: +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +** +************************************************************************ +** This file houses the module-loading pieces libcmpp. +*/ + +#if CMPP_ENABLE_DLLS +static const CmppSohList CmppSohList_empty = + CmppSohList_empty_m; +#endif + +#if CMPP_ENABLE_DLLS +/** + If compiled without CMPP_ENABLE_DLLS defined to a true value + then this function always returns CMPP_RC_UNSUPPORTED and updates + the error state of its first argument with information about that + code. + + Its first argument is the controlling cmpp. It can actually be + NULL - it's only used for reporting error details. + + Its second argument is the name of a DLL file. + + Its third argument is the name of a symbol in the given DLL which + resolves to a cmpp_module pointer. This name may be NULL, + in which case a default symbol name of "cmpp_module1" is used + (which is only useful when plugins are built one per DLL). + + The fourth argument is the output pointer to store the + resulting module handle in. + + The fifth argument is an optional list to append the DLL's + native handle to. It may be NULL. + + This function tries to open a DLL named fname using the system's + DLL loader. If none is found, CMPP_RC_NOT_FOUND is returned and the + cmpp's error state is populated with info about the error. If + one is found, it looks for a symbol in the DLL: if symName is not + NULL and is not empty then the symbol "cmpp_module_symName" is + sought, else "cmpp_module". (e.g. if symName is "foo" then it + searches for a symbol names "cmpp_module_foo".) If no such symbol is + found then CMPP_RC_NOT_FOUND (again) is returned and the + cmpp's error state is populated, else the symbol is assumed to + be a (cmpp_module*) and *mod is assigned to it. + + All errors update pp's error state but all are recoverable. + + Returns 0 on success. + + On success: + + - `*mod` is set to the module object. Its ownship is kinda murky: it + lives in memory made available via the module loader. It remains + valid memory until the DLL is closed. The module might also + actually be statically linked with the application, in which case + it will live as long as the app. + + - If soli is not NULL then the native DLL handle is appended to it. + Allocation errors when appending the DLL handle to the target list + are ignored - failure to retain a DLL handle for closing later is + not considered critical (and it would be extraordinarily rare (and + closing them outside of late-/post-main() cleanup is ill-advised, + anyway)). + + @see cmpp_module_load() + @see CMPP_MODULE_DECL + @see CMPP_MODULE_IMPL2 + @see CMPP_MODULE_IMPL3 + @see CMPP_MODULE_IMPL_SOLO + @see CMPP_MODULE_REGISTER2 + @see CMPP_MODULE_REGISTER3 +*/ +static +int cmpp__module_extract(cmpp * pp, + char const * dllFileName, + char const * symName, + cmpp_module const ** mod); +#endif + +#if CMPP_ENABLE_DLLS && !defined(CMPP_OMIT_D_MODULE) +# define CMPP_D_MODULE 1 +#else +# define CMPP_D_MODULE 0 +#endif + +#if CMPP_D_MODULE +/** + The #module directive: + + #module dll ?moduleName? + + Uses cmpp_module_load(dx, dll, moduleName||NULL) to try to load a + directive module. +*/ +//static +void cmpp_dx_f_module(cmpp_dx *dx) { + cmpp_arg const * aName = 0; + cmpp_b obDll = cmpp_b_empty; + for( cmpp_arg const *arg = dx->args.arg0; + arg; arg = arg->next ){ + //MARKER(("arg %s=%s\n", cmpp_tt_cstr(arg->ttype), arg->z)); + if( cmpp_dx_err_check(dx) ) goto end; + else if( !obDll.z ){ + cmpp_arg_to_b( + dx, arg, &obDll, + 0//cmpp_arg_to_b_F_NO_DEFINES + ); + continue; + }else if( !aName ){ + aName = arg; + continue; + } + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Unhandled argument: %s", arg->z); + goto end; + } + if( !obDll.z ){ + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, + "Expecting a DLL name argument."); + goto end; + } + cmpp_module_load(dx->pp, (char const *)obDll.z, + aName ? (char const *)aName->z : NULL); +end: + cmpp_b_clear(&obDll); + return; +#if 0 + missing_arg: + cmpp_dx_err_set(dx, CMPP_RC_MISUSE, "Expecting an argument after %s.", + arg->z); + return; +#endif +} +#endif /* #module */ + +/** + Module loader pedantic licensing note: Most of cmpp's + module-loading code was copied verbatim from another project[^1], + but was written by the same author who relicenses it in + cmpp. + + [^1]: https://fossil.wanderinghorse.net/r/cwal +*/ +#if CMPP_ENABLE_DLLS +#if CMPP_HAVE_DLOPEN +typedef void * cmpp_soh; +# include <dlfcn.h> /* this actually has a different name on some platforms! */ +#elif CMPP_HAVE_LTDLOPEN +# include <ltdl.h> +typedef lt_dlhandle cmpp_soh; +#elif CMPP_ENABLE_DLLS +# error "We have no dlopen() impl for this configuration." +#endif + +static cmpp_soh cmpp__dlopen(char const * fname, + char const **errMsg){ + static int once = 0; + cmpp_soh soh = 0; + if(!once && ++once){ +#if CMPP_HAVE_DLOPEN + dlopen( 0, RTLD_NOW | RTLD_GLOBAL ); +#elif CMPP_HAVE_LTDLOPEN + lt_dlinit(); + lt_dlopen( 0 ); +#endif + } +#if CMPP_HAVE_DLOPEN + soh = dlopen(fname, RTLD_NOW | RTLD_GLOBAL); +#elif CMPP_HAVE_LTDLOPEN + soh = lt_dlopen(fname); +#endif + if(!soh && errMsg){ +#if CMPP_HAVE_DLOPEN + *errMsg = dlerror(); +#elif CMPP_HAVE_LTDLOPEN + *errMsg = lt_dlerror(); +#endif + } + return soh; +} + +static +cmpp_module const * cmpp__dlsym(cmpp_soh soh, + char const * mname){ + cmpp_module const ** sym = +#if CMPP_HAVE_DLOPEN + dlsym(soh, mname) +#elif CMPP_HAVE_LTDLOPEN + lt_dlsym(soh, mname) +#else + NULL +#endif + ; + return sym ? *sym : NULL; +} + +static void cmpp__dlclose(cmpp_soh soh){ + if( soh ) { +#if CMPP_CLOSE_DLLS + /* MARKER(("Closing loaded module @%p.\n", (void const *)soh)); */ +#if CMPP_HAVE_DLOPEN + dlclose(soh); +#elif CMPP_HAVE_LTDLOPEN + lt_dlclose(soh); +#endif +#endif + } +} +#endif /* CMPP_ENABLE_DLLS */ + +#define CmppSohList_works (CMPP_ENABLE_DLLS && CMPP_CLOSE_DLLS) + +int CmppSohList_append(cmpp *pp, CmppSohList *soli, void *soh){ +#if CmppSohList_works + int const rc = cmpp_array_reserve(pp, (void**)&soli->list, + soli->n + ? (soli->n==soli->nAlloc + ? soli->nAlloc*2 + : soli->n+1) + : 8, + &soli->nAlloc, sizeof(void*)); + if( 0==rc ){ + soli->list[soli->n++] = soh; + } + return rc; +#else + (void)pp; (void)soli; (void)soh; + return 0; +#endif +} + +void CmppSohList_close(CmppSohList *s){ +#if CmppSohList_works + while( s->nAlloc ){ + if( s->list[--s->nAlloc] ){ + //MARKER(("closing soh %p\n", s->list[s->nAlloc])); + cmpp__dlclose(s->list[s->nAlloc]); + s->list[s->nAlloc] = 0; + } + } + cmpp_mfree(s->list); + *s = CmppSohList_empty; +#else + (void)s; +#endif +} + +#if 0 +/** + Passes soli to CmppSohList_close() then frees soli. Results are + undefined if soli is not NULL but was not returned from + CmppSohList_new(). + + Special case: if built without DLL-closing support, this is a no-op. +*/ +//static void CmppSohList_free(CmppSohList *soli); +void CmppSohList_free(CmppSohList *s){ + if( s ){ +#if CmppSohList_works + CmppSohList_close(s); + cmpp_mfree(s); +#endif + } +} + +/** + Returns a new, cleanly-initialized CmppSohList or NULL + on allocation error. The returned instance must eventually be + passed to CmppSohList_free(). + + Special case: if built without DLL-closing support, this returns a + no-op singleton instance. +*/ +//static CmppSohList * CmppSohList_new(void); +CmppSohList * CmppSohList_new(void){ +#if CmppSohList_works + CmppSohList * s = cmpp_malloc(sizeof(*s)); + if( s ) *s = CmppSohList_empty; + return s; +#else + static CmppSohList soli = CmppSohList_empty; + return &soli; +#endif +} +#endif + +#undef CmppSohList_works + +#if CMPP_ENABLE_DLLS +/** + Default entry point symbol name for loadable modules. This must + match the symbolic name defined by CMPP_MODULE_IMPL_SOLO(). +*/ +static char const * const cmppModDfltSym = "cmpp_module1"; + +/** + Looks for a symbol in the given DLL handle. If symName is NULL or + empty, the symbol "cmpp_module" is used, else the symbols + ("cmpp_module__" + symName) is used. If it finds one, it casts it to + cmpp_module and returns it. On error it may update pp's + error state with the error information if pp is not NULL. + + Errors: + + - symName is too long. + + - cmpp__dlsym() lookup failure. +*/ +static cmpp_module const * +cmpp__module_fish_out_entry_pt(cmpp * pp, + cmpp_soh soh, + char const * symName){ + enum { MaxLen = 128 }; + char buf[MaxLen] = {0}; + cmpp_size_t const slen = symName ? strlen(symName) : 0; + cmpp_module const * mod = 0; + if(slen > (MaxLen-20)){ + cmpp_err_set(pp, CMPP_RC_RANGE, + "DLL symbol name '%.*s' is too long. Max is %d.", + (int)slen, symName, (int)MaxLen-20); + }else{ + if(symName && *symName){ + snprintf(buf, MaxLen,"cmpp_module__%s", symName); + symName = &buf[0]; + }else{ + symName = cmppModDfltSym; + } + mod = cmpp__dlsym(soh, symName); + } + /*MARKER(("%s() [%s] ==> %p\n",__func__, symName, + (void const *)mod));*/ + return mod; +} +#endif/*CMPP_ENABLE_DLLS*/ + +#if CMPP_ENABLE_DLLS +/** + Tries to dlsym() the given cmpp_module symbol from the given + DLL handle. On success, 0 is returned and *mod is assigned to the + memory. On error, non-0 is returned and pp's error state may be + updated. + + Ownership of the returned module ostensibly lies with the first + argument, but that's not entirely true. If CMPP_CLOSE_DLLS is true + then a copy of the module's pointer is stored in the engine for + later closing. The memory itself is owned by the module loader, and + "should" stay valid until the DLL is closed. +*/ +static int cmpp__module_get_sym(cmpp * pp, + cmpp_soh soh, + char const * symName, + cmpp_module const ** mod){ + + cmpp_module const * lm = 0; + int rc = cmpp_err_has(pp); + if( 0==rc ){ + lm = cmpp__module_fish_out_entry_pt(pp, soh, symName); + rc = cmpp_err_has(pp); + } + if(0==rc){ + if(lm){ + *mod = lm; + }else{ + cmpp__dlclose(soh); + rc = cmpp_err_set(pp, CMPP_RC_NOT_FOUND, + "Did not find module entry point symbol '%s'.", + symName ? symName : cmppModDfltSym); + } + } + return rc; +} +#endif/*CMPP_ENABLE_DLLS*/ + +#if !CMPP_ENABLE_DLLS +static int cmpp__err_no_dlls(cmpp * const pp){ + return cmpp_err_set(pp, CMPP_RC_UNSUPPORTED, + "No dlopen() equivalent is installed " + "for this build configuration."); +} +#endif + +#if CMPP_ENABLE_DLLS +//no: CMPP_WASM_EXPORT +int cmpp__module_extract(cmpp * pp, + char const * fname, + char const * symName, + cmpp_module const ** mod){ + int rc = cmpp_err_has(pp); + if( rc ) return rc; + else if( cmpp_is_safemode(pp) ){ + return cmpp_err_set(pp, CMPP_RC_ACCESS, + "Cannot use DLLs in safe mode."); + }else{ + cmpp_soh soh; + char const * errMsg = 0; + soh = cmpp__dlopen(fname, &errMsg); + if(soh){ + if( pp ){ + CmppSohList_append(NULL/*alloc error here can be ignored*/, + &pp->pimpl->mod.sohList, soh); + } + cmpp_module const * x = 0; + rc = cmpp__module_get_sym(pp, soh, symName, &x); + if(!rc && mod) *mod = x; + return rc; + }else{ + return errMsg + ? cmpp_err_set(pp, CMPP_RC_ERROR, "DLL open failed: %s", + errMsg) + : cmpp_err_set(pp, CMPP_RC_ERROR, + "DLL open failed for unknown reason."); + } + } +} +#endif + +//no: CMPP_WASM_EXPORT +int cmpp_module_load(cmpp * pp, char const * fname, + char const * symName){ +#if CMPP_ENABLE_DLLS + if( ppCode ){ + /* fall through */ + }else if( cmpp_ctor_F_SAFEMODE & pp->pimpl->flags.newFlags ){ + cmpp_err_set(pp, CMPP_RC_ACCESS, + "%s() is disallowed in safe-mode."); + }else{ + cmpp__pi(pp); + char * zName = 0; + if( fname ){ + zName = cmpp_path_search(pp, (char const *)pi->mod.path.z, + pi->mod.pathSep, fname, + pi->mod.soExt); + if( !zName ){ + return cmpp_err_set(pp, CMPP_RC_NOT_FOUND, + "Did not find [%s] or [%s%s] " + "in search path [%s].", + fname, fname, pi->mod.soExt, + pi->mod.path.z); + } + } + cmpp_module const * mod = 0; + if( 0==cmpp__module_extract(pp, zName, symName, &mod) ){ + assert(mod); + assert(mod->init); + int const rc = mod->init(pp); + if( rc && !ppCode ){ + cmpp_err_set(pp, CMPP_RC_ERROR, + "Module %s::init() failed with code #%d/%s " + "without providing additional info.", + symName ? symName : "cmpp_module", + rc, cmpp_rc_cstr(rc)); + } + cmpp_mfree(zName); + } + } + return ppCode; +#else + (void)fname; (void)symName; + return cmpp__err_no_dlls(pp); +#endif +} +/* +** 2015-08-18, 2023-04-28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file demonstrates how to create a table-valued-function using +** a virtual table. This demo implements the generate_series() function +** which gives the same results as the eponymous function in PostgreSQL, +** within the limitation that its arguments are signed 64-bit integers. +** +** Considering its equivalents to generate_series(start,stop,step): A +** value V[n] sequence is produced for integer n ascending from 0 where +** ( V[n] == start + n * step && sgn(V[n] - stop) * sgn(step) >= 0 ) +** for each produced value (independent of production time ordering.) +** +** All parameters must be either integer or convertable to integer. +** The start parameter is required. +** The stop parameter defaults to (1<<32)-1 (aka 4294967295 or 0xffffffff) +** The step parameter defaults to 1 and 0 is treated as 1. +** +** Examples: +** +** SELECT * FROM generate_series(0,100,5); +** +** The query above returns integers from 0 through 100 counting by steps +** of 5. In other words, 0, 5, 10, 15, ..., 90, 95, 100. There are a total +** of 21 rows. +** +** SELECT * FROM generate_series(0,100); +** +** Integers from 0 through 100 with a step size of 1. 101 rows. +** +** SELECT * FROM generate_series(20) LIMIT 10; +** +** Integers 20 through 29. 10 rows. +** +** SELECT * FROM generate_series(0,-100,-5); +** +** Integers 0 -5 -10 ... -100. 21 rows. +** +** SELECT * FROM generate_series(0,-1); +** +** Empty sequence. +** +** HOW IT WORKS +** +** The generate_series "function" is really a virtual table with the +** following schema: +** +** CREATE TABLE generate_series( +** value, +** start HIDDEN, +** stop HIDDEN, +** step HIDDEN +** ); +** +** The virtual table also has a rowid which is an alias for the value. +** +** Function arguments in queries against this virtual table are translated +** into equality constraints against successive hidden columns. In other +** words, the following pairs of queries are equivalent to each other: +** +** SELECT * FROM generate_series(0,100,5); +** SELECT * FROM generate_series WHERE start=0 AND stop=100 AND step=5; +** +** SELECT * FROM generate_series(0,100); +** SELECT * FROM generate_series WHERE start=0 AND stop=100; +** +** SELECT * FROM generate_series(20) LIMIT 10; +** SELECT * FROM generate_series WHERE start=20 LIMIT 10; +** +** The generate_series virtual table implementation leaves the xCreate method +** set to NULL. This means that it is not possible to do a CREATE VIRTUAL +** TABLE command with "generate_series" as the USING argument. Instead, there +** is a single generate_series virtual table that is always available without +** having to be created first. +** +** The xBestIndex method looks for equality constraints against the hidden +** start, stop, and step columns, and if present, it uses those constraints +** to bound the sequence of generated values. If the equality constraints +** are missing, it uses 0 for start, 4294967295 for stop, and 1 for step. +** xBestIndex returns a small cost when both start and stop are available, +** and a very large cost if either start or stop are unavailable. This +** encourages the query planner to order joins such that the bounds of the +** series are well-defined. +** +** Update on 2024-08-22: +** xBestIndex now also looks for equality and inequality constraints against +** the value column and uses those constraints as additional bounds against +** the sequence range. Thus, a query like this: +** +** SELECT value FROM generate_series($SA,$EA) +** WHERE value BETWEEN $SB AND $EB; +** +** Is logically the same as: +** +** SELECT value FROM generate_series(max($SA,$SB),min($EA,$EB)); +** +** Constraints on the value column can server as substitutes for constraints +** on the hidden start and stop columns. So, the following two queries +** are equivalent: +** +** SELECT value FROM generate_series($S,$E); +** SELECT value FROM generate_series WHERE value BETWEEN $S and $E; +** +*/ +#if 0 +#include "sqlite3ext.h" +#else +#include "sqlite3.h" +#endif +#include <assert.h> +#include <string.h> +#include <limits.h> +#include <math.h> + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* series_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result. +** +** iOBase, iOTerm, and iOStep are the original values of the +** start=, stop=, and step= constraints on the query. These are +** the values reported by the start, stop, and step columns of the +** virtual table. +** +** iBase, iTerm, iStep, and bDescp are the actual values used to generate +** the sequence. These might be different from the iOxxxx values. +** For example in +** +** SELECT value FROM generate_series(1,11,2) +** WHERE value BETWEEN 4 AND 8; +** +** The iOBase is 1, but the iBase is 5. iOTerm is 11 but iTerm is 7. +** Another example: +** +** SELECT value FROM generate_series(1,15,3) ORDER BY value DESC; +** +** The cursor initialization for the above query is: +** +** iOBase = 1 iBase = 13 +** iOTerm = 15 iTerm = 1 +** iOStep = 3 iStep = 3 bDesc = 1 +** +** The actual step size is unsigned so that can have a value of +** +9223372036854775808 which is needed for querys like this: +** +** SELECT value +** FROM generate_series(9223372036854775807, +** -9223372036854775808, +** -9223372036854775808) +** ORDER BY value ASC; +** +** The setup for the previous query will be: +** +** iOBase = 9223372036854775807 iBase = -1 +** iOTerm = -9223372036854775808 iTerm = 9223372036854775807 +** iOStep = -9223372036854775808 iStep = 9223372036854775808 bDesc = 0 +*/ +typedef unsigned char u8; +typedef struct series_cursor series_cursor; +struct series_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_int64 iOBase; /* Original starting value ("start") */ + sqlite3_int64 iOTerm; /* Original terminal value ("stop") */ + sqlite3_int64 iOStep; /* Original step value */ + sqlite3_int64 iBase; /* Starting value to actually use */ + sqlite3_int64 iTerm; /* Terminal value to actually use */ + sqlite3_uint64 iStep; /* The step size */ + sqlite3_int64 iValue; /* Current value */ + u8 bDesc; /* iStep is really negative */ + u8 bDone; /* True if stepped past last element */ +}; + +/* +** Computed the difference between two 64-bit signed integers using a +** convoluted computation designed to work around the silly restriction +** against signed integer overflow in C. +*/ +static sqlite3_uint64 span64(sqlite3_int64 a, sqlite3_int64 b){ + assert( a>=b ); + return (*(sqlite3_uint64*)&a) - (*(sqlite3_uint64*)&b); +} + +/* +** Add or substract an unsigned 64-bit integer from a signed 64-bit integer +** and return the new signed 64-bit integer. +*/ +static sqlite3_int64 add64(sqlite3_int64 a, sqlite3_uint64 b){ + sqlite3_uint64 x = *(sqlite3_uint64*)&a; + x += b; + return *(sqlite3_int64*)&x; +} +static sqlite3_int64 sub64(sqlite3_int64 a, sqlite3_uint64 b){ + sqlite3_uint64 x = *(sqlite3_uint64*)&a; + x -= b; + return *(sqlite3_int64*)&x; +} + +/* +** The seriesConnect() method is invoked to create a new +** series_vtab that describes the generate_series virtual table. +** +** Think of this routine as the constructor for series_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the series_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against generate_series will look like. +*/ +static int seriesConnect( + sqlite3 *db, + void *pUnused, + int argcUnused, const char *const*argvUnused, + sqlite3_vtab **ppVtab, + char **pzErrUnused +){ + sqlite3_vtab *pNew; + int rc; + +/* Column numbers */ +#define SERIES_COLUMN_ROWID (-1) +#define SERIES_COLUMN_VALUE 0 +#define SERIES_COLUMN_START 1 +#define SERIES_COLUMN_STOP 2 +#define SERIES_COLUMN_STEP 3 + + (void)pUnused; + (void)argcUnused; + (void)argvUnused; + (void)pzErrUnused; + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(value,start hidden,stop hidden,step hidden)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + } + return rc; +} + +/* +** This method is the destructor for series_cursor objects. +*/ +static int seriesDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new series_cursor object. +*/ +static int seriesOpen(sqlite3_vtab *pUnused, sqlite3_vtab_cursor **ppCursor){ + series_cursor *pCur; + (void)pUnused; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a series_cursor. +*/ +static int seriesClose(sqlite3_vtab_cursor *cur){ + sqlite3_free(cur); + return SQLITE_OK; +} + + +/* +** Advance a series_cursor to its next row of output. +*/ +static int seriesNext(sqlite3_vtab_cursor *cur){ + series_cursor *pCur = (series_cursor*)cur; + if( pCur->iValue==pCur->iTerm ){ + pCur->bDone = 1; + }else if( pCur->bDesc ){ + pCur->iValue = sub64(pCur->iValue, pCur->iStep); + assert( pCur->iValue>=pCur->iTerm ); + }else{ + pCur->iValue = add64(pCur->iValue, pCur->iStep); + assert( pCur->iValue<=pCur->iTerm ); + } + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the series_cursor +** is currently pointing. +*/ +static int seriesColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + series_cursor *pCur = (series_cursor*)cur; + sqlite3_int64 x = 0; + switch( i ){ + case SERIES_COLUMN_START: x = pCur->iOBase; break; + case SERIES_COLUMN_STOP: x = pCur->iOTerm; break; + case SERIES_COLUMN_STEP: x = pCur->iOStep; break; + default: x = pCur->iValue; break; + } + sqlite3_result_int64(ctx, x); + return SQLITE_OK; +} + +#ifndef LARGEST_UINT64 +#define LARGEST_INT64 ((sqlite3_int64)0x7fffffffffffffffLL) +#define LARGEST_UINT64 ((sqlite3_uint64)0xffffffffffffffffULL) +#define SMALLEST_INT64 ((sqlite3_int64)0x8000000000000000LL) +#endif + +/* +** The rowid is the same as the value. +*/ +static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + series_cursor *pCur = (series_cursor*)cur; + *pRowid = pCur->iValue; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int seriesEof(sqlite3_vtab_cursor *cur){ + series_cursor *pCur = (series_cursor*)cur; + return pCur->bDone; +} + +/* True to cause run-time checking of the start=, stop=, and/or step= +** parameters. The only reason to do this is for testing the +** constraint checking logic for virtual tables in the SQLite core. +*/ +#ifndef SQLITE_SERIES_CONSTRAINT_VERIFY +# define SQLITE_SERIES_CONSTRAINT_VERIFY 0 +#endif + +/* +** Return the number of steps between pCur->iBase and pCur->iTerm if +** the step width is pCur->iStep. +*/ +static sqlite3_uint64 seriesSteps(series_cursor *pCur){ + if( pCur->bDesc ){ + assert( pCur->iBase >= pCur->iTerm ); + return span64(pCur->iBase, pCur->iTerm)/pCur->iStep; + }else{ + assert( pCur->iBase <= pCur->iTerm ); + return span64(pCur->iTerm, pCur->iBase)/pCur->iStep; + } +} + +#if defined(SQLITE_ENABLE_MATH_FUNCTIONS) || defined(_WIN32) +/* +** Case 1 (the most common case): +** The standard math library is available so use ceil() and floor() from there. +*/ +static double seriesCeil(double r){ return ceil(r); } +static double seriesFloor(double r){ return floor(r); } +#elif defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) +/* +** Case 2 (2nd most common): Use GCC/Clang builtins +*/ +static double seriesCeil(double r){ return __builtin_ceil(r); } +static double seriesFloor(double r){ return __builtin_floor(r); } +#else +/* +** Case 3 (rarely happens): Use home-grown ceil() and floor() routines. +*/ +static double seriesCeil(double r){ + sqlite3_int64 x; + if( r!=r ) return r; + if( r<=(-4503599627370496.0) ) return r; + if( r>=(+4503599627370496.0) ) return r; + x = (sqlite3_int64)r; + if( r==(double)x ) return r; + if( r>(double)x ) x++; + return (double)x; +} +static double seriesFloor(double r){ + sqlite3_int64 x; + if( r!=r ) return r; + if( r<=(-4503599627370496.0) ) return r; + if( r>=(+4503599627370496.0) ) return r; + x = (sqlite3_int64)r; + if( r==(double)x ) return r; + if( r<(double)x ) x--; + return (double)x; +} +#endif + +/* +** This method is called to "rewind" the series_cursor object back +** to the first row of output. This method is always called at least +** once prior to any call to seriesColumn() or seriesRowid() or +** seriesEof(). +** +** The query plan selected by seriesBestIndex is passed in the idxNum +** parameter. (idxStr is not used in this implementation.) idxNum +** is a bitmask showing which constraints are available: +** +** 0x0001: start=VALUE +** 0x0002: stop=VALUE +** 0x0004: step=VALUE +** 0x0008: descending order +** 0x0010: ascending order +** 0x0020: LIMIT VALUE +** 0x0040: OFFSET VALUE +** 0x0080: value=VALUE +** 0x0100: value>=VALUE +** 0x0200: value>VALUE +** 0x1000: value<=VALUE +** 0x2000: value<VALUE +** +** This routine should initialize the cursor and position it so that it +** is pointing at the first row, or pointing off the end of the table +** (so that seriesEof() will return true) if the table is empty. +*/ +static int seriesFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStrUnused, + int argc, sqlite3_value **argv +){ + series_cursor *pCur = (series_cursor *)pVtabCursor; + int iArg = 0; /* Arguments used so far */ + int i; /* Loop counter */ + sqlite3_int64 iMin = SMALLEST_INT64; /* Smallest allowed output value */ + sqlite3_int64 iMax = LARGEST_INT64; /* Largest allowed output value */ + sqlite3_int64 iLimit = 0; /* if >0, the value of the LIMIT */ + sqlite3_int64 iOffset = 0; /* if >0, the value of the OFFSET */ + + (void)idxStrUnused; + + /* If any constraints have a NULL value, then return no rows. + ** See ticket https://sqlite.org/src/info/fac496b61722daf2 + */ + for(i=0; i<argc; i++){ + if( sqlite3_value_type(argv[i])==SQLITE_NULL ){ + goto series_no_rows; + } + } + + /* Capture the three HIDDEN parameters to the virtual table and insert + ** default values for any parameters that are omitted. + */ + if( idxNum & 0x01 ){ + pCur->iOBase = sqlite3_value_int64(argv[iArg++]); + }else{ + pCur->iOBase = 0; + } + if( idxNum & 0x02 ){ + pCur->iOTerm = sqlite3_value_int64(argv[iArg++]); + }else{ + pCur->iOTerm = 0xffffffff; + } + if( idxNum & 0x04 ){ + pCur->iOStep = sqlite3_value_int64(argv[iArg++]); + if( pCur->iOStep==0 ) pCur->iOStep = 1; + }else{ + pCur->iOStep = 1; + } + + /* If there are constraints on the value column but there are + ** no constraints on the start, stop, and step columns, then + ** initialize the default range to be the entire range of 64-bit signed + ** integers. This range will contracted by the value column constraints + ** further below. + */ + if( (idxNum & 0x05)==0 && (idxNum & 0x0380)!=0 ){ + pCur->iOBase = SMALLEST_INT64; + } + if( (idxNum & 0x06)==0 && (idxNum & 0x3080)!=0 ){ + pCur->iOTerm = LARGEST_INT64; + } + pCur->iBase = pCur->iOBase; + pCur->iTerm = pCur->iOTerm; + if( pCur->iOStep>0 ){ + pCur->iStep = pCur->iOStep; + }else if( pCur->iOStep>SMALLEST_INT64 ){ + pCur->iStep = -pCur->iOStep; + }else{ + pCur->iStep = LARGEST_INT64; + pCur->iStep++; + } + pCur->bDesc = pCur->iOStep<0; + if( pCur->bDesc==0 && pCur->iBase>pCur->iTerm ){ + goto series_no_rows; + } + if( pCur->bDesc!=0 && pCur->iBase<pCur->iTerm ){ + goto series_no_rows; + } + + /* Extract the LIMIT and OFFSET values, but do not apply them yet. + ** The range must first be constrained by the limits on value. + */ + if( idxNum & 0x20 ){ + iLimit = sqlite3_value_int64(argv[iArg++]); + if( idxNum & 0x40 ){ + iOffset = sqlite3_value_int64(argv[iArg++]); + } + } + + /* Narrow the range of iMin and iMax (the minimum and maximum outputs) + ** based on equality and inequality constraints on the "value" column. + */ + if( idxNum & 0x3380 ){ + if( idxNum & 0x0080 ){ /* value=X */ + if( sqlite3_value_numeric_type(argv[iArg])==SQLITE_FLOAT ){ + double r = sqlite3_value_double(argv[iArg++]); + if( r==seriesCeil(r) + && r>=(double)SMALLEST_INT64 + && r<=(double)LARGEST_INT64 + ){ + iMin = iMax = (sqlite3_int64)r; + }else{ + goto series_no_rows; + } + }else{ + iMin = iMax = sqlite3_value_int64(argv[iArg++]); + } + }else{ + if( idxNum & 0x0300 ){ /* value>X or value>=X */ + if( sqlite3_value_numeric_type(argv[iArg])==SQLITE_FLOAT ){ + double r = sqlite3_value_double(argv[iArg++]); + if( r<(double)SMALLEST_INT64 ){ + iMin = SMALLEST_INT64; + }else if( (idxNum & 0x0200)!=0 && r==seriesCeil(r) ){ + iMin = (sqlite3_int64)seriesCeil(r+1.0); + }else{ + iMin = (sqlite3_int64)seriesCeil(r); + } + }else{ + iMin = sqlite3_value_int64(argv[iArg++]); + if( (idxNum & 0x0200)!=0 ){ + if( iMin==LARGEST_INT64 ){ + goto series_no_rows; + }else{ + iMin++; + } + } + } + } + if( idxNum & 0x3000 ){ /* value<X or value<=X */ + if( sqlite3_value_numeric_type(argv[iArg])==SQLITE_FLOAT ){ + double r = sqlite3_value_double(argv[iArg++]); + if( r>(double)LARGEST_INT64 ){ + iMax = LARGEST_INT64; + }else if( (idxNum & 0x2000)!=0 && r==seriesFloor(r) ){ + iMax = (sqlite3_int64)(r-1.0); + }else{ + iMax = (sqlite3_int64)seriesFloor(r); + } + }else{ + iMax = sqlite3_value_int64(argv[iArg++]); + if( idxNum & 0x2000 ){ + if( iMax==SMALLEST_INT64 ){ + goto series_no_rows; + }else{ + iMax--; + } + } + } + } + if( iMin>iMax ){ + goto series_no_rows; + } + } + + /* Try to reduce the range of values to be generated based on + ** constraints on the "value" column. + */ + if( pCur->bDesc==0 ){ + if( pCur->iBase<iMin ){ + sqlite3_uint64 span = span64(iMin,pCur->iBase); + pCur->iBase = add64(pCur->iBase, (span/pCur->iStep)*pCur->iStep); + if( pCur->iBase<iMin ){ + if( pCur->iBase > sub64(LARGEST_INT64, pCur->iStep) ){ + goto series_no_rows; + } + pCur->iBase = add64(pCur->iBase, pCur->iStep); + } + } + if( pCur->iTerm>iMax ){ + pCur->iTerm = iMax; + } + }else{ + if( pCur->iBase>iMax ){ + sqlite3_uint64 span = span64(pCur->iBase,iMax); + pCur->iBase = sub64(pCur->iBase, (span/pCur->iStep)*pCur->iStep); + if( pCur->iBase>iMax ){ + if( pCur->iBase < add64(SMALLEST_INT64, pCur->iStep) ){ + goto series_no_rows; + } + pCur->iBase = sub64(pCur->iBase, pCur->iStep); + } + } + if( pCur->iTerm<iMin ){ + pCur->iTerm = iMin; + } + } + } + + /* Adjust iTerm so that it is exactly the last value of the series. + */ + if( pCur->bDesc==0 ){ + if( pCur->iBase>pCur->iTerm ){ + goto series_no_rows; + } + pCur->iTerm = sub64(pCur->iTerm, + span64(pCur->iTerm,pCur->iBase) % pCur->iStep); + }else{ + if( pCur->iBase<pCur->iTerm ){ + goto series_no_rows; + } + pCur->iTerm = add64(pCur->iTerm, + span64(pCur->iBase,pCur->iTerm) % pCur->iStep); + } + + /* Transform the series generator to output values in the requested + ** order. + */ + if( ((idxNum & 0x0008)!=0 && pCur->bDesc==0) + || ((idxNum & 0x0010)!=0 && pCur->bDesc!=0) + ){ + sqlite3_int64 tmp = pCur->iBase; + pCur->iBase = pCur->iTerm; + pCur->iTerm = tmp; + pCur->bDesc = !pCur->bDesc; + } + + /* Apply LIMIT and OFFSET constraints, if any */ + assert( pCur->iStep!=0 ); + if( idxNum & 0x20 ){ + if( iOffset>0 ){ + if( seriesSteps(pCur) < (sqlite3_uint64)iOffset ){ + goto series_no_rows; + }else if( pCur->bDesc ){ + pCur->iBase = sub64(pCur->iBase, pCur->iStep*iOffset); + }else{ + pCur->iBase = add64(pCur->iBase, pCur->iStep*iOffset); + } + } + if( iLimit>=0 && seriesSteps(pCur) > (sqlite3_uint64)iLimit ){ + pCur->iTerm = add64(pCur->iBase, (iLimit - 1)*pCur->iStep); + } + } + pCur->iValue = pCur->iBase; + pCur->bDone = 0; + return SQLITE_OK; + +series_no_rows: + pCur->iBase = 0; + pCur->iTerm = 0; + pCur->iStep = 1; + pCur->bDesc = 0; + pCur->bDone = 1; + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the generate_series virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +** +** In this implementation idxNum is used to represent the +** query plan. idxStr is unused. +** +** The query plan is represented by bits in idxNum: +** +** 0x0001 start = $num +** 0x0002 stop = $num +** 0x0004 step = $num +** 0x0008 output is in descending order +** 0x0010 output is in ascending order +** 0x0020 LIMIT $num +** 0x0040 OFFSET $num +** 0x0080 value = $num +** 0x0100 value >= $num +** 0x0200 value > $num +** 0x1000 value <= $num +** 0x2000 value < $num +** +** Only one of 0x0100 or 0x0200 will be returned. Similarly, only +** one of 0x1000 or 0x2000 will be returned. If the 0x0080 is set, then +** none of the 0xff00 bits will be set. +** +** The order of parameters passed to xFilter is as follows: +** +** * The argument to start= if bit 0x0001 is in the idxNum mask +** * The argument to stop= if bit 0x0002 is in the idxNum mask +** * The argument to step= if bit 0x0004 is in the idxNum mask +** * The argument to LIMIT if bit 0x0020 is in the idxNum mask +** * The argument to OFFSET if bit 0x0040 is in the idxNum mask +** * The argument to value=, or value>= or value> if any of +** bits 0x0380 are in the idxNum mask +** * The argument to value<= or value< if either of bits 0x3000 +** are in the mask +** +*/ +static int seriesBestIndex( + sqlite3_vtab *pVTab, + sqlite3_index_info *pIdxInfo +){ + int i, j; /* Loop over constraints */ + int idxNum = 0; /* The query plan bitmask */ +#ifndef ZERO_ARGUMENT_GENERATE_SERIES + int bStartSeen = 0; /* EQ constraint seen on the START column */ +#endif + int unusableMask = 0; /* Mask of unusable constraints */ + int nArg = 0; /* Number of arguments that seriesFilter() expects */ + int aIdx[7]; /* Constraints on start, stop, step, LIMIT, OFFSET, + ** and value. aIdx[5] covers value=, value>=, and + ** value>, aIdx[6] covers value<= and value< */ + const struct sqlite3_index_constraint *pConstraint; + + /* This implementation assumes that the start, stop, and step columns + ** are the last three columns in the virtual table. */ + assert( SERIES_COLUMN_STOP == SERIES_COLUMN_START+1 ); + assert( SERIES_COLUMN_STEP == SERIES_COLUMN_START+2 ); + + aIdx[0] = aIdx[1] = aIdx[2] = aIdx[3] = aIdx[4] = aIdx[5] = aIdx[6] = -1; + pConstraint = pIdxInfo->aConstraint; + for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){ + int iCol; /* 0 for start, 1 for stop, 2 for step */ + int iMask; /* bitmask for those column */ + int op = pConstraint->op; + if( op>=SQLITE_INDEX_CONSTRAINT_LIMIT + && op<=SQLITE_INDEX_CONSTRAINT_OFFSET + ){ + if( pConstraint->usable==0 ){ + /* do nothing */ + }else if( op==SQLITE_INDEX_CONSTRAINT_LIMIT ){ + aIdx[3] = i; + idxNum |= 0x20; + }else{ + assert( op==SQLITE_INDEX_CONSTRAINT_OFFSET ); + aIdx[4] = i; + idxNum |= 0x40; + } + continue; + } + if( pConstraint->iColumn<SERIES_COLUMN_START ){ + if( (pConstraint->iColumn==SERIES_COLUMN_VALUE || + pConstraint->iColumn==SERIES_COLUMN_ROWID) + && pConstraint->usable + ){ + switch( op ){ + case SQLITE_INDEX_CONSTRAINT_EQ: + case SQLITE_INDEX_CONSTRAINT_IS: { + idxNum |= 0x0080; + idxNum &= ~0x3300; + aIdx[5] = i; + aIdx[6] = -1; +#ifndef ZERO_ARGUMENT_GENERATE_SERIES + bStartSeen = 1; +#endif + break; + } + case SQLITE_INDEX_CONSTRAINT_GE: { + if( idxNum & 0x0080 ) break; + idxNum |= 0x0100; + idxNum &= ~0x0200; + aIdx[5] = i; +#ifndef ZERO_ARGUMENT_GENERATE_SERIES + bStartSeen = 1; +#endif + break; + } + case SQLITE_INDEX_CONSTRAINT_GT: { + if( idxNum & 0x0080 ) break; + idxNum |= 0x0200; + idxNum &= ~0x0100; + aIdx[5] = i; +#ifndef ZERO_ARGUMENT_GENERATE_SERIES + bStartSeen = 1; +#endif + break; + } + case SQLITE_INDEX_CONSTRAINT_LE: { + if( idxNum & 0x0080 ) break; + idxNum |= 0x1000; + idxNum &= ~0x2000; + aIdx[6] = i; + break; + } + case SQLITE_INDEX_CONSTRAINT_LT: { + if( idxNum & 0x0080 ) break; + idxNum |= 0x2000; + idxNum &= ~0x1000; + aIdx[6] = i; + break; + } + } + } + continue; + } + iCol = pConstraint->iColumn - SERIES_COLUMN_START; + assert( iCol>=0 && iCol<=2 ); + iMask = 1 << iCol; +#ifndef ZERO_ARGUMENT_GENERATE_SERIES + if( iCol==0 && op==SQLITE_INDEX_CONSTRAINT_EQ ){ + bStartSeen = 1; + } +#endif + if( pConstraint->usable==0 ){ + unusableMask |= iMask; + continue; + }else if( op==SQLITE_INDEX_CONSTRAINT_EQ ){ + idxNum |= iMask; + aIdx[iCol] = i; + } + } + if( aIdx[3]==0 ){ + /* Ignore OFFSET if LIMIT is omitted */ + idxNum &= ~0x60; + aIdx[4] = 0; + } + for(i=0; i<7; i++){ + if( (j = aIdx[i])>=0 ){ + pIdxInfo->aConstraintUsage[j].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[j].omit = + !SQLITE_SERIES_CONSTRAINT_VERIFY || i>=3; + } + } + /* The current generate_column() implementation requires at least one + ** argument (the START value). Legacy versions assumed START=0 if the + ** first argument was omitted. Compile with -DZERO_ARGUMENT_GENERATE_SERIES + ** to obtain the legacy behavior */ +#ifndef ZERO_ARGUMENT_GENERATE_SERIES + if( !bStartSeen ){ + sqlite3_free(pVTab->zErrMsg); + pVTab->zErrMsg = sqlite3_mprintf( + "first argument to \"generate_series()\" missing or unusable"); + return SQLITE_ERROR; + } +#endif + if( (unusableMask & ~idxNum)!=0 ){ + /* The start, stop, and step columns are inputs. Therefore if there + ** are unusable constraints on any of start, stop, or step then + ** this plan is unusable */ + return SQLITE_CONSTRAINT; + } + if( (idxNum & 0x03)==0x03 ){ + /* Both start= and stop= boundaries are available. This is the + ** the preferred case */ + pIdxInfo->estimatedCost = (double)(2 - ((idxNum&4)!=0)); + pIdxInfo->estimatedRows = 1000; + if( pIdxInfo->nOrderBy>=1 && pIdxInfo->aOrderBy[0].iColumn==0 ){ + if( pIdxInfo->aOrderBy[0].desc ){ + idxNum |= 0x08; + }else{ + idxNum |= 0x10; + } + pIdxInfo->orderByConsumed = 1; + } + }else if( (idxNum & 0x21)==0x21 ){ + /* We have start= and LIMIT */ + pIdxInfo->estimatedRows = 2500; + }else{ + /* If either boundary is missing, we have to generate a huge span + ** of numbers. Make this case very expensive so that the query + ** planner will work hard to avoid it. */ + pIdxInfo->estimatedRows = 2147483647; + } + pIdxInfo->idxNum = idxNum; +#ifdef SQLITE_INDEX_SCAN_HEX + pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_HEX; +#endif + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** generate_series virtual table. +*/ +static sqlite3_module seriesModule = { + 0, /* iVersion */ + 0, /* xCreate */ + seriesConnect, /* xConnect */ + seriesBestIndex, /* xBestIndex */ + seriesDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + seriesOpen, /* xOpen - open a cursor */ + seriesClose, /* xClose - close a cursor */ + seriesFilter, /* xFilter - configure scan constraints */ + seriesNext, /* xNext - advance a cursor */ + seriesEof, /* xEof - check for end of scan */ + seriesColumn, /* xColumn - read data */ + seriesRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_series_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + (void)pApi; +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( sqlite3_libversion_number()<3008012 && pzErrMsg!=0 ){ + *pzErrMsg = sqlite3_mprintf( + "generate_series() requires SQLite 3.8.12 or later"); + return SQLITE_ERROR; + } + rc = sqlite3_create_module(db, "generate_series", &seriesModule, 0); +#endif + return rc; +} +#define CMPP_D_DEMO +/* +** 2025-10-18: +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +** +************************************************************************ +** +** This file contains demonstration client-side directives for the +** c-pp API. +*/ +#if defined(CMPP_D_DEMO) +/* Only when building with the main c-pp app. */ +#elif !defined(CMPP_MODULE_REGISTER1) +/** + Assume a standalone module build. Arrange for the default module + entry point to be installed so that cmpp_module_load() does not + require that the user know the entry point name. +*/ +#define CMPP_MODULE_STANDALONE +#define CMPP_API_THUNK +#include "libcmpp.h" +#endif + +#include <stdlib.h> +#include <assert.h> +#include <string.h> + +/** + cmpp_d_autoload_f() impl for this file's directives and its close + friends. +*/ +int cmpp_d_autoload_f_demos(cmpp *pp, char const *dname, void *state); +/** + Registers demo and utility directives with pp. +*/ +int cmpp_module__demo_register(cmpp *pp); + +/** + Simply says hello and emits info about its arguments. +*/ +static void cmpp_dx_f_demo1(cmpp_dx *dx){ + cmpp_dx_outf(dx, "Hello from %s%s\n", + cmpp_dx_delim(dx), dx->d->name.z); + for( cmpp_arg const * a = dx->args.arg0; + 0==cmpp_dx_err_check(dx) && a; + a = a->next ){ + cmpp_dx_outf(dx, "arg type=%s n=%u z=%.*s\n", + cmpp_tt_cstr(a->ttype), + (unsigned)a->n, (int)a->n, a->z); + } +} + +/** + Internal helper for other directives. Emits an HTML <div> tag. If + passed any arguments, each is assumed to be a CSS class name and is + applied to the DIV. This does _not_ emit the lcosing DIV + tag. Returns 0 on success. +*/ +static int divOpener(cmpp_dx *dx){ + cmpp_dx_out_raw(dx, "<div", 4); + int nClass = 0; + for( cmpp_arg const * a = dx->args.arg0; + 0==cmpp_dx_err_check(dx) && a; + a = a->next ){ + if( 1==++nClass ){ + cmpp_dx_out_raw(dx, " class='", 8); + }else{ + cmpp_dx_out_raw(dx, " ", 1); + } + cmpp_dx_out_raw(dx, a->z, a->n); + } + return cmpp_dx_out_raw(dx, nClass ? "'>" : ">", 1 + !!nClass); +} + +/** + Opens an HTML DIV tag, as per divOpener(). +*/ +static void cmpp_dx_f_divOpen(cmpp_dx *dx){ + if( 0==divOpener(dx) ){ + int * const nDiv = dx->d->impl.state; + assert( nDiv ); + ++*nDiv; + } +} + +/** + Closes an HTML DIV tag which was opened by cmpp_dx_f_divOpen(). +*/ +static void cmpp_dx_f_divClose(cmpp_dx *dx){ + int * const nDiv = dx->d->impl.state; + assert( nDiv ); + if( *nDiv > 0 ){ + --*nDiv; + }else{ + char const * const zDelim = cmpp_dx_delim(dx); + cmpp_dx_err_set( + dx, CMPP_RC_MISUSE, + "%s/%s was used without an opening %s%s directive", + zDelim, dx->d->name.z, zDelim, dx->d->name.z + ); + } +} + +/** + Another HTML DIV-inspired wrapper which consumes a block of input + and wraps it in a DIV. This is functionally the same as the + divOpen/divClose examples but demonstrates how to slurp up the + content between the open/close directives from within the opening + directive's callback. +*/ +static void cmpp_dx_f_divWrapper(cmpp_dx *dx){ + if( divOpener(dx) ) return; + cmpp_b os = {0}; + if( 0==cmpp_dx_consume_b( + dx, &os, &dx->d->closer, 1, + cmpp_dx_consume_F_PROCESS_OTHER_D + ) + /* ^^^ says "read to the matching #/div" accounting for, and + processing, nested directives). The #/div closing tag is + identified by dx->d->closer. */ + ){ + cmpp_b_chomp( &os ); + /* Recall that most cmpp APIs become no-ops if dx->pp has an error + set, so we don't strictly need to error-check these calls: */ + cmpp_dx_out_raw(dx, os.z, os.n); + cmpp_dx_out_raw(dx, "</div>\n", 7); + } + cmpp_b_clear(&os); +} + +/** + A cmpp_d_autoload_f() impl for testing and demonstration + purposes. +*/ +int cmpp_d_autoload_f_demos(cmpp *pp, char const *dname, void *state){ + (void)state; + cmpp_api_init(pp); + +#define M(NAME) (0==strcmp(NAME,dname)) +#define MOC(NAME) (M(NAME) || M("/"NAME)) + +#define DREG0(SYMNAME, NAME, OPENER, OFLAGS, CLOSER, CFLAGS) \ + cmpp_d_reg SYMNAME = { \ + .name = NAME, \ + .opener = { \ + .f = OPENER, \ + .flags = OFLAGS \ + }, \ + .closer = { \ + .f = CLOSER, \ + .flags = CFLAGS \ + }, \ + .dtor = 0, \ + .state = 0 \ + } + +#define CHECK(NAME,CHECKCLOSER,OPENER,OFLAGS,CLOSER,CFLAGS) \ + if( M(NAME) || (CHECKCLOSER && M("/"NAME)) ){ \ + DREG0(reg,NAME,OPENER,OFLAGS,CLOSER,CFLAGS); \ + return cmpp_d_register(pp, &reg, NULL); \ + } (void)0 + + + CHECK("demo1", 0, cmpp_dx_f_demo1, cmpp_d_F_ARGS_LIST, 0, 0); + + CHECK("demo-div-wrapper", 1, cmpp_dx_f_divWrapper, + cmpp_d_F_ARGS_LIST | cmpp_d_F_NO_CALL, + cmpp_dx_f_dangling_closer, 0); + + if( MOC("demo-div") ){ + cmpp_d_reg const r = { + .name = "demo-div", + .opener = { + .f = cmpp_dx_f_divOpen, + .flags = cmpp_d_F_ARGS_LIST | cmpp_d_F_NO_CALL + }, + .closer = { + .f = cmpp_dx_f_divClose + }, + .state = cmpp_malloc(sizeof(int)), + .dtor = cmpp_mfree + }; + /* State for one of the custom directives. */; + int const rc = cmpp_d_register(pp, &r, NULL); + if( 0==rc ){ + *((int*)r.state) = 0 + /* else reg.state was freed by cmpp_d_register() */; + } + return rc; + } + +#undef M +#undef MOC +#undef CHECK +#undef DREG0 + return CMPP_RC_NO_DIRECTIVE; +} + +int cmpp_module__demo_register(cmpp *pp){ + cmpp_api_init(pp); + int rc; +#define X(D) \ + rc = cmpp_d_autoload_f_demos(pp, D, NULL); \ + if( rc && CMPP_RC_NO_DIRECTIVE!=rc ) goto end + X("demo1"); + X("demo-div"); + X("demo-div-wrapper"); +#undef X +end: + return cmpp_err_get(pp, NULL); +} +CMPP_MODULE_REGISTER1(demo); +#endif /* include guard */ diff --git a/ext/wasm/mkdist.sh b/ext/wasm/mkdist.sh index 84780668b..78b93e5e0 100755 --- a/ext/wasm/mkdist.sh +++ b/ext/wasm/mkdist.sh @@ -52,7 +52,7 @@ for arg in $@; do ;; --snapshot) - snapshotSuffix=$(date +%Y%m%d) + snapshotSuffix=-snapshot-$(date +%Y%m%d) ;; -?|--help) diff --git a/ext/wasm/mkwasmbuilds.c b/ext/wasm/mkwasmbuilds.c index 2730d9d76..1f60a89bb 100644 --- a/ext/wasm/mkwasmbuilds.c +++ b/ext/wasm/mkwasmbuilds.c @@ -40,14 +40,22 @@ /* ** Flags for use with BuildDef::flags. ** -** Maintenance reminder: do not combine flags within this enum, -** e.g. F_BUNDLER_FRIENDLY=0x02|F_ESM, as that will lead -** to breakage in some of the flag checks. +** Maintenance reminder: do not combine F_... flags within this enum, +** e.g. F_BUNDLER_FRIENDLY=0x02|F_ESM, as that will lead to breakage +** in some of the flag checks. */ enum BuildDefFlags { /* Indicates an ESM module build. */ F_ESM = 0x01, - /* Indicates a "bundler-friendly" build mode. */ + /* Indicates a "bundler-friendly" build mode. These are untested and + ** unsupported, provided solely for the downstream npm subproject + ** (who is responsible for any testing of these). + ** + ** The only difference beween bundler-friendly and esm builds is + ** that bundlers require static filename strings in a few places due + ** to limitations of bundler tooling, whereas vanilla and JS can + ** both work with dynamic strings. + */ F_BUNDLER_FRIENDLY = 1<<1, /* Indicates that this build is unsupported. Such builds are not ** added to the 'all' target. The unsupported builds exist primarily @@ -57,8 +65,11 @@ enum BuildDefFlags { F_NOT_IN_ALL = 1<<3, /* If it's a 64-bit build. */ F_64BIT = 1<<4, - /* Indicates a node.js-for-node.js build (untested and - ** unsupported). */ + /* Indicates a node.js-for-node.js build. This build is very + ** specificially untested and unsupported, but the downstream npm + ** project makes use of it. None of our JS code is specific to node, + ** but Emscripten's generated sqlite3.js differs between + ** for-the-browser and for-node builds. */ F_NODEJS = 1<<5, /* Indicates a wasmfs build (untested and unsupported). */ F_WASMFS = 1<<6, @@ -71,13 +82,13 @@ enum BuildDefFlags { ** only their JS file and patch their JS to use the WASM file from a ** canonical build which uses that same WASM file. Reusing X.wasm ** that way can only work for builds which are processed identically - ** by Emscripten. For a given set of C flags (as opposed to + ** by Emscripten. For a given set of C flags (as distinct from ** JS-influencing flags), all builds of X.js and Y.js will produce ** identical X.wasm and Y.wasm files. Their JS files may well ** differ, however. */ - CP_JS = 1 << 30, - CP_WASM = 1 << 31, + CP_JS = 1 << 30, /* X.js or X.mjs, depending on F_ESM */ + CP_WASM = 1 << 31, /* X.wasm */ CP_ALL = CP_JS | CP_WASM }; @@ -94,11 +105,10 @@ enum BuildDefFlags { ** final build directory $(dir.dout). ** ** To keep parallel builds from stepping on each other, each distinct -** build goes into its own subdir $(dir.dout.BuildName)[^1], i.e. -** $(dir.dout)/BuildName. Builds which produce deliverables we'd like -** to keep/distribute copy their final results into the build dir -** $(dir.dout). See the notes for the CP_JS enum entry for more -** details on that. +** build goes into its own subdir $(dir.dout.$(BuildDef::zBaseName). +** Builds which produce deliverables we'd like to keep/distribute copy +** their final results into the build dir $(dir.dout). See the notes +** for the CP_JS enum entry for more details on that. ** ** The final result of each build is a pair of JS/WASM files, but ** getting there requires generation of several files, primarily as @@ -116,9 +126,9 @@ enum BuildDefFlags { ** ** --extern-post-js = gets injected immediately after ** sqlite3InitModule(), in the global scope. In this step we replace -** sqlite3InitModule() with a slightly customized, the main purpose of -** which is to (A) give us (not Emscripten) control over the arguments -** it accepts and (B) to run the library bootstrap step. +** sqlite3InitModule() with a slightly customized one, the main +** purpose of which is to (A) give us (not Emscripten) control over +** the arguments it accepts and (B) to run the library bootstrap step. ** ** Then there's sqlite3-api.BuildName.js, which is the entire SQLite3 ** JS API (generated from the list defined in $(sqlite3-api.jses)). It @@ -126,9 +136,7 @@ enum BuildDefFlags { ** ** Each of those inputs has to be generated before passing them on to ** Emscripten so that any build-specific capabilities can get filtered -** in or out (using ./c-pp.c). -** -** [^1]: The legal BuildNames are in this file's BuildDef_map macro. +** in or out (using ./c-pp-lite.c). */ struct BuildDef { /* @@ -143,8 +151,8 @@ struct BuildDef { ** ** The convention for 32- vs 64-bit pairs is to give them similar ** emoji, e.g. a cookie for 32-bit and a donut or cake for 64. - ** Alternately, the same emoji a "64" suffix, excep that that throws - ** off the output alignment in parallel builds ;). + ** Alternately, the same emoji with a "64" suffix, except that that + ** throws off the output alignment in parallel builds ;). */ const char *zEmo; /* @@ -161,13 +169,13 @@ struct BuildDef { const char *zCmppD; /* Extra -D... flags for c-pp */ const char *zEmcc; /* Full flags for emcc. Normally NULL for default. */ const char *zEmccExtra; /* Extra flags for emcc */ - const char *zDeps; /* Extra deps */ + const char *zDeps; /* Extra make target deps */ const char *zEnv; /* emcc -sENVIRONMENT=... value */ /* ** Makefile code "ifeq (...)". If set, this build is enclosed in a ** $zIfCond/endif block. */ - const char *zIfCond; /* makefile "ifeq (...)" or similar */ + const char *zIfCond; int flags; /* Flags from BuildDefFlags */ }; typedef struct BuildDef BuildDef; @@ -183,7 +191,7 @@ typedef struct BuildDef BuildDef; ** logtag.NAME = Used for decorating log output ** ** etc. -***/ +*/ #define BuildDefs_map(E) \ E(vanilla) E(vanilla64) \ E(esm) E(esm64) \ @@ -196,9 +204,8 @@ typedef struct BuildDef BuildDef; ** The set of WASM builds for the library (as opposed to the apps ** (fiddle, speedtest1)). Their order in BuildDefs_map is mostly ** insignificant, but some makefile vars used by some builds are set -** up by prior builds. Because of that, the (sqlite3, vanilla), -** (sqlite3, esm), and (sqlite3, bundler-friendly) builds should be -** defined first (in that order). +** up by prior builds. Because of that, the vanilla, esm, and +** bundler-friendly builds should be defined first (in that order). */ struct BuildDefs { #define E(N) BuildDef N; @@ -239,7 +246,7 @@ const BuildDefs oBuildDefs = { .zEnv = 0, .zDeps = 0, .zIfCond = 0, - .flags = CP_ALL | F_64BIT + .flags = CP_ALL | F_64BIT | F_NOT_IN_ALL }, /* The canonical esm build. */ @@ -267,7 +274,7 @@ const BuildDefs oBuildDefs = { .zEnv = 0, .zDeps = 0, .zIfCond = 0, - .flags = CP_JS | F_ESM | F_64BIT + .flags = CP_JS | F_ESM | F_64BIT | F_NOT_IN_ALL }, /* speedtest1, our primary benchmarking tool */ @@ -284,7 +291,7 @@ const BuildDefs oBuildDefs = { " -DSQLITE_SPEEDTEST1_WASM" " $(SQLITE_OPT)" " -USQLITE_WASM_BARE_BONES" - " -USQLITE_C -DSQLITE_C=$(sqlite3.canonical.c)" + " -USQLITE_C -DSQLITE_C=$(sqlite3.c)" " $(speedtest1.exit-runtime0)" " $(speedtest1.c.in)" " -lm", @@ -312,7 +319,7 @@ const BuildDefs oBuildDefs = { " -DSQLITE_SPEEDTEST1_WASM" " $(SQLITE_OPT)" " -USQLITE_WASM_BARE_BONES" - " -USQLITE_C -DSQLITE_C=$(sqlite3.canonical.c)" + " -USQLITE_C -DSQLITE_C=$(sqlite3.c)" " $(speedtest1.exit-runtime0)" " $(speedtest1.c.in)" " -lm", @@ -345,8 +352,7 @@ const BuildDefs oBuildDefs = { .zEnv = 0, .zDeps = 0, .zIfCond = 0, - .flags = CP_JS | F_BUNDLER_FRIENDLY | F_ESM - //| F_NOT_IN_ALL + .flags = CP_JS | F_BUNDLER_FRIENDLY | F_ESM | F_NOT_IN_ALL }, /* 64-bit bundler-friendly. */ @@ -371,7 +377,7 @@ const BuildDefs oBuildDefs = { .node = { .zEmo = "🍟", .zBaseName = "sqlite3-node", - .zDotWasm = 0, + .zDotWasm = "sqlite3", .zCmppD = "-Dtarget:node $(c-pp.D.bundler)", .zEmcc = 0, .zEmccExtra = 0, @@ -382,21 +388,21 @@ const BuildDefs oBuildDefs = { ** node. */, .zDeps = 0, .zIfCond = 0, - .flags = CP_ALL | F_UNSUPPORTED | F_NODEJS + .flags = CP_JS | F_UNSUPPORTED | F_ESM | F_NODEJS }, /* 64-bit node. */ .node64 = { .zEmo = "🍔", .zBaseName = "sqlite3-node-64bit", - .zDotWasm = 0, + .zDotWasm = "sqlite3-64bit", .zCmppD = "-Dtarget:node $(c-pp.D.bundler)", .zEmcc = 0, .zEmccExtra = 0, .zEnv = "node", .zDeps = 0, .zIfCond = 0, - .flags = CP_ALL | F_UNSUPPORTED | F_NODEJS | F_64BIT + .flags = CP_JS | F_UNSUPPORTED | F_ESM | F_NODEJS | F_64BIT }, /* Entirely unsupported. */ @@ -487,8 +493,8 @@ static void mk_prologue(void){ ** configuration). Comments like "saves nothing" may not be ** technically correct: "nothing" means "some neglible amount." ** - ** Note that performance gains/losses are _not_ taken into - ** account here: only wasm file size. + ** Performance gains or losses are _not_ taken into account + ** here, only wasm file size. */ "--enable-bulk-memory-opt " /* required */ "--all-features " /* required */ @@ -638,7 +644,7 @@ static void mk_pre_post(char const *zBuildName, BuildDef const * pB){ pB->zDotWasm); } ps(""); - pf("\t@$(call b.c-pp.shcmd," + pf("\t@$(call b.mkdir@); $(call b.c-pp.shcmd," "%s," "$(pre-js.in.js)," "$(pre-js.%s.js)," @@ -741,6 +747,12 @@ static void emit_api_js(char const *zBuildName){ zBuildName, zBuildName, zBuildName); pf("$(out.%s.js): $(sqlite3-api.%s.js)\n", zBuildName, zBuildName); + pf("$(sqlite3-api.%s.js):" + /* Extra deps needed by the OPFS pieces... */ + " $(dir.api)/opfs-common-shared.c-pp.js" + " $(dir.api)/opfs-common-inline.c-pp.js" + "\n", + zBuildName); } /* @@ -777,13 +789,14 @@ static void mk_lib_mode(const char *zBuildName, const BuildDef * pB){ zBuildName, zBuildName, zBaseName); pf("dir.dout.%s ?= $(dir.dout)/%s\n", zBuildName, zBuildName); - pf("out.%s.base ?= $(dir.dout.%s)/%s\n", - zBuildName, zBuildName, zBaseName); pf("c-pp.D.%s ?= %s\n", zBuildName, pB->zCmppD ? pB->zCmppD : ""); if( pB->flags & F_64BIT ){ pf("c-pp.D.%s += $(c-pp.D.64bit)\n", zBuildName); } + if( pB->flags & F_UNSUPPORTED ){ + pf("c-pp.D.%s += -Dunsupported-build\n", zBuildName); + } pf("emcc.environment.%s ?= %s\n", zBuildName, pB->zEnv ? pB->zEnv : oBuildDefs.vanilla.zEnv); @@ -914,10 +927,9 @@ static void mk_lib_mode(const char *zBuildName, const BuildDef * pB){ pf("\n%dbit: $(out.%s.js)\n" "$(out.%s.wasm): $(out.%s.js)\n" - "b-%s: $(out.%s.js) $(out.%s.wasm)\n", + "b-%s: $(out.%s.wasm)\n", (F_64BIT & pB->flags) ? 64 : 32, zBuildName, - zBuildName, zBuildName, - zBuildName, zBuildName, zBuildName); + zBuildName, zBuildName, zBuildName, zBuildName); if( CP_JS & pB->flags ){ pf("$(dir.dout)/%s%s: $(out.%s.js)\n", @@ -988,11 +1000,19 @@ static void mk_fiddle(void){ pf("$(out.%s.js): $(MAKEFILE_LIST) " "$(EXPORTED_FUNCTIONS.fiddle) " "$(fiddle.c.in) " - "$(pre-post.%s.deps)\n", + "$(pre-post.%s.deps) " + "$(dir.dout)/sqlite3-opfs-async-proxy.js", zBuildName, zBuildName); + if( isDebug ){ + pf(" $(dir.fiddle)/fiddle-worker.js" + " $(dir.fiddle)/fiddle.js" + " $(dir.fiddle)/index.html"); + } + ps(""); emit_compile_start(zBuildName); - pf("\t$(b.cmd@)$(bin.emcc) -o $@" - " $(emcc.flags.%s)" /* set in fiddle.make */ + pf("\t@$(call b.mkdir@)\n" + "\t$(b.cmd@)$(bin.emcc) -o $@" + " $(emcc.flags.%s)" /* set in GNUmakefile */ " $(pre-post.%s.flags)" " $(fiddle.c.in)" "\n", @@ -1001,10 +1021,6 @@ static void mk_fiddle(void){ pf("\t@$(call b.call.wasm-strip,%s)\n", zBuildName); pf("\t@$(call b.strip-js-emcc-bindings,$(logtag.%s))\n", zBuildName); - pf("\t@$(call b.cp," - "%s," - "$(dir.api)/sqlite3-opfs-async-proxy.js," - "$(dir $@))\n", zBuildName); if( isDebug ){ pf("\t@$(call b.cp,%s," "$(dir.fiddle)/index.html " @@ -1057,7 +1073,7 @@ int main(int argc, char const ** argv){ BuildDefs_map(E) if( 0==strcmp("prologue",zArg) ){ mk_prologue(); }else { - fprintf(stderr,"Unkown build name: %s\n", zArg); + fprintf(stderr,"Unknown build name: %s\n", zArg); rc = 1; break; } diff --git a/ext/wasm/speedtest1-worker.js b/ext/wasm/speedtest1-worker.js index ba11fd163..9355ba93f 100644 --- a/ext/wasm/speedtest1-worker.js +++ b/ext/wasm/speedtest1-worker.js @@ -1,7 +1,7 @@ 'use strict'; (function(){ let speedtestJs = 'speedtest1.js'; - const urlParams = new URL(self.location.href).searchParams; + const urlParams = new URL(globalThis.location.href).searchParams; if(urlParams.has('sqlite3.dir')){ speedtestJs = urlParams.get('sqlite3.dir') + '/' + speedtestJs; } @@ -14,9 +14,9 @@ const wasmfsDir = function f(wasmUtil){ if(undefined !== f._) return f._; const pdir = '/opfs'; - if( !self.FileSystemHandle - || !self.FileSystemDirectoryHandle - || !self.FileSystemFileHandle){ + if( !globalThis.FileSystemHandle + || !globalThis.FileSystemDirectoryHandle + || !globalThis.FileSystemFileHandle){ return f._ = ""; } try{ @@ -92,7 +92,7 @@ } }; - self.onmessage = function(msg){ + globalThis.onmessage = function(msg){ msg = msg.data; switch(msg.type){ case 'run': @@ -111,7 +111,8 @@ setStatus: (text)=>mPost('load-status',text) }; log("Initializing speedtest1 module..."); - self.sqlite3InitModule(EmscriptenModule).then(async (sqlite3)=>{ + globalThis.sqlite3InitModule.__isUnderTest = true; + globalThis.sqlite3InitModule(EmscriptenModule).then(async (sqlite3)=>{ const S = globalThis.S = App.sqlite3 = sqlite3; log("Loaded speedtest1 module. Setting up..."); App.pDir = wasmfsDir(S.wasm); diff --git a/ext/wasm/speedtest1.html b/ext/wasm/speedtest1.html index cce617185..d41f20a4e 100644 --- a/ext/wasm/speedtest1.html +++ b/ext/wasm/speedtest1.html @@ -44,132 +44,137 @@ mounted, else it returns an empty string. */ const wasmfsDir = function f(wasmUtil){ - if(undefined !== f._) return f._; - const pdir = '/persistent'; - if( !self.FileSystemHandle - || !self.FileSystemDirectoryHandle - || !self.FileSystemFileHandle){ - return f._ = ""; - } - try{ - if(0===wasmUtil.xCallWrapped( - 'sqlite3_wasm_init_wasmfs', 'i32', ['string'], pdir - )){ - return f._ = pdir; - }else{ - return f._ = ""; - } - }catch(e){ - // sqlite3_wasm_init_wasmfs() is not available - return f._ = ""; + if(undefined !== f._) return f._; + const pdir = '/persistent'; + if( !self.FileSystemHandle + || !self.FileSystemDirectoryHandle + || !self.FileSystemFileHandle){ + return f._ = ""; + } + try{ + if(0===wasmUtil.xCallWrapped( + 'sqlite3_wasm_init_wasmfs', 'i32', ['string'], pdir + )){ + return f._ = pdir; + }else{ + return f._ = ""; } + }catch(e){ + // sqlite3_wasm_init_wasmfs() is not available + return f._ = ""; + } }; wasmfsDir._ = undefined; const eOut = document.querySelector('#test-output'); const log2 = function(cssClass,...args){ - const ln = document.createElement('div'); - if(cssClass) ln.classList.add(cssClass); - ln.append(document.createTextNode(args.join(' '))); - eOut.append(ln); - //this.e.output.lastElementChild.scrollIntoViewIfNeeded(); + const ln = document.createElement('div'); + if(cssClass) ln.classList.add(cssClass); + ln.append(document.createTextNode(args.join(' '))); + eOut.append(ln); + //this.e.output.lastElementChild.scrollIntoViewIfNeeded(); }; const logList = []; const dumpLogList = function(){ - logList.forEach((v)=>log2('',v)); - logList.length = 0; + logList.forEach((v)=>log2('',v)); + logList.length = 0; }; /* we cannot update DOM while speedtest is running unless we run speedtest in a worker thread. */; const log = (...args)=>{ - console.log(...args); - logList.push(args.join(' ')); + console.log(...args); + logList.push(args.join(' ')); }; const logErr = function(...args){ - console.error(...args); - logList.push('ERROR: '+args.join(' ')); + console.error(...args); + logList.push('ERROR: '+args.join(' ')); }; const runTests = function(sqlite3){ - const capi = sqlite3.capi, wasm = sqlite3.wasm; - //console.debug('sqlite3 =',sqlite3); - const pDir = wasmfsDir(wasm); - if(pDir){ - console.warn("Persistent storage:",pDir); - } - const scope = wasm.scopedAllocPush(); - let dbFile = pDir+"/speedtest1.db"; - const urlParams = new URL(self.location.href).searchParams; - const argv = ["speedtest1"]; - if(urlParams.has('flags')){ - argv.push(...(urlParams.get('flags').split(','))); - } + globalThis.S = sqlite3; + const capi = sqlite3.capi, wasm = sqlite3.wasm; + //console.debug('sqlite3 (global S) =',sqlite3); + const pDir = wasmfsDir(wasm); + if(pDir){ + console.warn("wasmfs storage:",pDir); + } + const scope = wasm.scopedAllocPush(); + let dbFile = pDir+"/speedtest1.db"; + const urlParams = new URL(self.location.href).searchParams; + const argv = ["speedtest1"]; + if(urlParams.has('flags')){ + argv.push(...(urlParams.get('flags').split(','))); + } - let forceSize = 0; - let vfs, pVfs = 0; - if(urlParams.has('vfs')){ - vfs = urlParams.get('vfs'); - pVfs = capi.sqlite3_vfs_find(vfs); - if(!pVfs){ - log2('error',"Unknown VFS:",vfs); - return; - } - argv.push("--vfs", vfs); - log2('',"Using VFS:",vfs); - if('kvvfs' === vfs){ - forceSize = 2 /* >2 is too big as of mid-2025 */; - dbFile = 'session'; - log2('warning',"kvvfs VFS: forcing --size",forceSize, - "and filename '"+dbFile+"'."); - capi.sqlite3_js_kvvfs_clear(dbFile); - } + let forceSize = 0; + let vfs, pVfs = 0; + if(urlParams.has('vfs')){ + vfs = urlParams.get('vfs'); + pVfs = capi.sqlite3_vfs_find(vfs); + if(!pVfs){ + log2('error',"Unknown VFS:",vfs); + return; } - if(forceSize){ - argv.push('--size',forceSize); - }else{ - [ - 'size' - ].forEach(function(k){ - const v = urlParams.get(k); - if(v) argv.push('--'+k, urlParams[k]); - }); + argv.push("--vfs", vfs); + log2('',"Using VFS:",vfs); + if('kvvfs' === vfs){ + forceSize = 5 + /* --size > 2 is too big for local/session storage + as of mid-2025, but kvvfs v2 (late 2025) + lets us use transient storage. */; + dbFile = 'transient'; + log2('warning',"kvvfs VFS: forcing --size",forceSize, + "and filename '"+dbFile+"'."); + capi.sqlite3_js_kvvfs_clear(dbFile); + } + } + if(forceSize){ + argv.push('--size',forceSize); + }else{ + [ + 'size' + ].forEach(function(k){ + const v = urlParams.get(k); + if(v && +v) argv.push('--'+k, urlParams[k]); + }); + } + argv.push( + "--singlethread", + //"--nomutex", + //"--nosync", + //"--memdb", // note that memdb trumps the filename arg + "--nomemstat" + ); + argv.push("--big-transactions"/*important for tests 410 and 510!*/, + dbFile); + console.log("argv =",argv); + // These log messages are not emitted to the UI until after main() returns. Fixing that + // requires moving the main() call and related cleanup into a timeout handler. + if(pDir) wasm.sqlite3_wasm_vfs_unlink(pVfs,dbFile); + log2('',"WASM heap size:",sqlite3.wasm.heap8().byteLength,"bytes"); + log2('',"WASM pointer size:",sqlite3.wasm.ptr.size); + log2('',"Starting native app:\n ",argv.join(' ')); + log2('',"This will take a while and the browser might warn about the runaway JS.", + "Give it time..."); + logList.length = 0; + setTimeout(function(){ + wasm.xCall('wasm_main', argv.length, + wasm.scopedAllocMainArgv(argv)); + wasm.scopedAllocPop(scope); + if('kvvfs'===vfs){ + logList.unshift("KVVFS "+dbFile+" size = "+ + capi.sqlite3_js_kvvfs_size(dbFile)); } - argv.push( - "--singlethread", - //"--nomutex", - //"--nosync", - //"--memdb", // note that memdb trumps the filename arg - "--nomemstat" - ); - argv.push("--big-transactions"/*important for tests 410 and 510!*/, - dbFile); - console.log("argv =",argv); - // These log messages are not emitted to the UI until after main() returns. Fixing that - // requires moving the main() call and related cleanup into a timeout handler. if(pDir) wasm.sqlite3_wasm_vfs_unlink(pVfs,dbFile); + logList.unshift("Done running native main(). Output:"); + dumpLogList(); log2('',"WASM heap size:",sqlite3.wasm.heap8().byteLength,"bytes"); - log2('',"WASM pointer size:",sqlite3.wasm.ptr.size); - log2('',"Starting native app:\n ",argv.join(' ')); - log2('',"This will take a while and the browser might warn about the runaway JS.", - "Give it time..."); - logList.length = 0; - setTimeout(function(){ - wasm.xCall('wasm_main', argv.length, - wasm.scopedAllocMainArgv(argv)); - wasm.scopedAllocPop(scope); - if('kvvfs'===vfs){ - logList.unshift("KVVFS "+dbFile+" size = "+ - capi.sqlite3_js_kvvfs_size(dbFile)); - } - if(pDir) wasm.sqlite3_wasm_vfs_unlink(pVfs,dbFile); - logList.unshift("Done running native main(). Output:"); - dumpLogList(); - log2('',"WASM heap size:",sqlite3.wasm.heap8().byteLength,"bytes"); - }, 50); + }, 50); }/*runTests()*/; self.sqlite3TestModule.print = log; self.sqlite3TestModule.printErr = logErr; + sqlite3InitModule.__isUnderTest = true; sqlite3InitModule(self.sqlite3TestModule).then(runTests); })();</script> </body> diff --git a/ext/wasm/tester1-worker.c-pp.html b/ext/wasm/tester1-worker.c-pp.html index e461b6cbf..37153d84f 100644 --- a/ext/wasm/tester1-worker.c-pp.html +++ b/ext/wasm/tester1-worker.c-pp.html @@ -1,5 +1,5 @@ <!doctype html> -//#@policy error +//#@ policy error <html lang="en-us"> <head> <meta charset="utf-8"> @@ -13,10 +13,18 @@ <body> <h1 id='color-target'>sqlite3 tester #1: Worker thread (@bitness@-bit WASM)</h1> <div>Variants: - <a href='tester1.html' target='tester1.html'>conventional UI thread</a>, - <a href='tester1-worker.html' target='tester1-worker.html'>conventional worker</a>, - <a href='tester1-esm.html' target='tester1-esm.html'>ESM in UI thread</a>, - <a href='tester1-worker.html?esm' target='tester1-worker.html?esm'>ESM worker</a> + conventional UI thread: + (<a href='tester1.html' target='tester1.html'>32-bit</a>, + <a href='tester1-64bit.html' target='tester1-64bit.html'>64-bit</a>), + conventional worker: + (<a href='tester1-worker.html' target='tester1-worker.html'>32-bit</a>, + <a href='tester1-worker-64bit.html' target='tester1-worker-64bit.html'>64-bit</a>), + ESM in UI thread: + (<a href='tester1-esm.html' target='tester1-esm.html'>32-bit</a>, + <a href='tester1-esm-64bit.html' target='tester1-esm-64bit.html'>64-bit</a>), + ESM worker:</a> + (<a href='tester1-worker.html?esm' target='tester1-worker.html?esm'>32-bit</a> + <a href='tester1-worker-64bit.html?esm' target='tester1-worker-64bit.html?esm'>64-bit</a>) </div> <div class='input-wrapper'> <input type='checkbox' id='cb-log-reverse'> diff --git a/ext/wasm/tester1.c-pp.html b/ext/wasm/tester1.c-pp.html index 95fe52219..8424cc85c 100644 --- a/ext/wasm/tester1.c-pp.html +++ b/ext/wasm/tester1.c-pp.html @@ -1,5 +1,5 @@ <!doctype html> -//#@policy error +//#@ policy error <html lang="en-us"> <head> <meta charset="utf-8"> @@ -11,11 +11,19 @@ <style></style> </head> <body><h1 id='color-target'></h1> - <div>Variants: - <a href='tester1.html' target='tester1.html'>conventional UI thread</a>, - <a href='tester1-worker.html' target='tester1-worker.html'>conventional worker</a>, - <a href='tester1-esm.html' target='tester1-esm.html'>ESM in UI thread</a>, - <a href='tester1-worker.html?esm' target='tester1-worker.html?esm'>ESM worker</a> + <div>Variants (32-bit): + conventional UI thread: + (<a href='tester1.html' target='tester1.html'>32-bit</a>, + <a href='tester1-64bit.html' target='tester1-64bit.html'>64-bit</a>), + conventional worker: + (<a href='tester1-worker.html' target='tester1-worker.html'>32-bit</a>, + <a href='tester1-worker-64bit.html' target='tester1-worker-64bit.html'>64-bit</a>), + ESM in UI thread: + (<a href='tester1-esm.html' target='tester1-esm.html'>32-bit</a>, + <a href='tester1-esm-64bit.html' target='tester1-esm-64bit.html'>64-bit</a>), + ESM worker:</a> + (<a href='tester1-worker.html?esm' target='tester1-worker.html?esm'>32-bit</a> + <a href='tester1-worker-64bit.html?esm' target='tester1-worker-64bit.html?esm'>64-bit</a>) </div> <div class='input-wrapper'> <input type='checkbox' id='cb-log-reverse'> @@ -31,6 +39,6 @@ //#else <script src="@sqlite3.js@"></script> <script src="@tester1.js@"></script> -//#endif +//#/if </body> </html> diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index f72e0803f..346d49bb0 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -43,13 +43,13 @@ ./c-pp -f tester1.c-pp.js -o tester1-esm.mjs -Dtarget:es6-module */ -//#@policy error +//#@ policy error //#if target:es6-module import {default as sqlite3InitModule} from "@sqlite3.js@"; globalThis.sqlite3InitModule = sqlite3InitModule; //#else 'use strict'; -//#endif +//#/if (function(self){ /** Set up our output channel differently depending @@ -66,15 +66,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule; const haveWasmCTests = ()=>{ return !!wasm.exports.sqlite3__wasm_test_intptr; }; - const hasOpfs = ()=>{ - return globalThis.FileSystemHandle - && globalThis.FileSystemDirectoryHandle - && globalThis.FileSystemFileHandle - && globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle - && navigator?.storage?.getDirectory; - }; let SQLite3 /* populated after module load */; + const hasOpfs = ()=>!!SQLite3?.oo1?.OpfsDb; { const mapToString = (v)=>{ @@ -129,17 +123,6 @@ globalThis.sqlite3InitModule = sqlite3InitModule; }; } } - const reportFinalTestStatus = function(pass){ - if(isUIThread()){ - let e = document.querySelector('#color-target'); - e.classList.add(pass ? 'tests-pass' : 'tests-fail'); - e = document.querySelector('title'); - e.innerText = (pass ? 'PASS' : 'FAIL') + ': ' + e.innerText; - }else{ - postMessage({type:'test-result', payload:{pass}}); - } - TestUtil.checkHeapSize(true); - }; const log = (...args)=>{ //console.log(...args); logClass('',...args); @@ -153,6 +136,23 @@ globalThis.sqlite3InitModule = sqlite3InitModule; logClass('error',...args); }; + const debug = (...args)=>{ + console.debug('tester1',...args); + }; + + const reportFinalTestStatus = function(pass){ + debug("Final test status:",pass); + if(isUIThread()){ + let e = document.querySelector('#color-target'); + e.classList.add(pass ? 'tests-pass' : 'tests-fail'); + e = document.querySelector('title'); + e.innerText = (pass ? 'PASS' : 'FAIL') + ': ' + e.innerText; + }else{ + postMessage({type:'test-result', payload:{pass}}); + } + TestUtil.checkHeapSize(true); + }; + const toss = (...args)=>{ error(...args); throw new Error(args.join(' ')); @@ -210,8 +210,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule; filter.test(error.message) passes. If it's a function, the test passes if filter(error) returns truthy. If it's a string, the test passes if the filter matches the exception message - precisely. In all other cases the test fails, throwing an - Error. + precisely. If filter is a number then it is compared against + the resultCode property of the exception. In all other cases + the test fails, throwing an Error. If it throws, msg is used as the error report unless it's falsy, in which case a default is used. @@ -225,7 +226,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule; if(filter instanceof RegExp) pass = filter.test(err.message); else if(filter instanceof Function) pass = filter(err); else if('string' === typeof filter) pass = (err.message === filter); + else if('number' === typeof filter) pass = (err.resultCode === filter); if(!pass){ + console.error("Filter",filter,"rejected exception",err); throw new Error(msg || ("Filter rejected this exception: <<"+err.message+">>")); } return this; @@ -369,6 +372,210 @@ globalThis.sqlite3InitModule = sqlite3InitModule; clearOnInit: true, initialCapacity: 6 }; + +//#if enable-see + /** + Code consolidator for SEE sanity checks for various VFSes. ctor + is the VFS's oo1.DB-type constructor. ctorOptFunc(bool) is a + function which must return a constructor args object for ctor. It + is passed true if the db needs to be cleaned up/unlinked before + opening it (OPFS) and false if not (how that is done is + VFS-dependent). dbUnlink is a function which is expected to + unlink() the db file if the ctorOpfFunc does not do so when + passed true (kvvfs). + + This function initializes the db described by ctorOptFunc(...), + writes some secret info into it, and re-opens it twice to + confirmi that it can be read with an SEE key and cannot be read + without one. + */ + T.seeBaseCheck = function(ctor, ctorOptFunc, dbUnlink){ + let initDb = true; + const tryKey = function(keyKey, key, expectCount){ + let db; + //console.debug('tryKey()',arguments); + try { + if (initDb) { + const ctoropt = ctorOptFunc(initDb); + initDb = false; + db = new ctor({ + ...ctoropt, + [keyKey]: key + }); + db.exec([ + "drop table if exists t;", + "create table t(a);" + ]); + db.close(); + db = null; + // Ensure that it's actually encrypted... + let err; + try { + db = new ctor(ctorOptFunc(false)); + T.assert(db, 'db opened') /* opening is fine, but... */; + db.exec("select 1 from sqlite_schema"); + console.warn("(should not be reached) sessionStorage =", sessionStorage); + } catch (e) { + err = e; + } finally { + db.close() + db = null; + } + T.assert(err, "Expecting an exception") + .assert(capi.SQLITE_NOTADB == err.resultCode, + "Expecting NOTADB"); + }/*initDb*/ + db = new ctor({ + ...ctorOptFunc(false), + [keyKey]: key + }); + db.exec("insert into t(a) values (1),(2)"); + T.assert(expectCount === db.selectValue('select sum(a) from t')); + } finally { + if (db) db.close(); + } + }; + const hexFoo = new Uint8Array([0x66,0x6f,0x6f]/*=="foo"*/); + dbUnlink(); + tryKey('textkey', 'foo', 3); + T.assert( !initDb ); + tryKey('textkey', 'foo', 6); + dbUnlink(); + initDb = true; + tryKey('key', 'foo', 3); + T.assert( !initDb ); + tryKey('key', hexFoo, 6); + dbUnlink(); + initDb = true; + tryKey('hexkey', hexFoo, 3); + T.assert( !initDb ); + tryKey('hexkey', hexFoo, 6); + dbUnlink(); + }; +//#/if enable-see + + /* Tests common to "opfs" and "opfs-wl". These tests manipulate + "this" and must run in order. + */ + T.opfsCommon = { + sanityChecks: async function(vfsName, oo1Ctor, sqlite3){ + T.assert(capi.sqlite3_vfs_find(vfsName)); + const opfs = sqlite3.opfs; + const filename = this.opfsDbFile = '/dir/sqlite3-tester1.db'; + const fileUri = 'file://'+filename+'?delete-before-open=1'; + const initSql = [ + 'create table p(a);', + 'insert into p(a) values(1),(2),(3)' + ]; + let db = new oo1Ctor(fileUri); + try { + db.exec(initSql); + T.assert(3 === db.selectValue('select count(*) from p')); + db.close(); + db = new oo1Ctor(filename); + db.exec('insert into p(a) values(4),(5),(6)'); + T.assert(6 === db.selectValue('select count(*) from p')); + this.opfsDbExport = capi.sqlite3_js_db_export(db); + T.assert(this.opfsDbExport instanceof Uint8Array) + .assert(this.opfsDbExport.byteLength>0 + && 0===this.opfsDbExport.byteLength % 512); + }finally{ + db.close(); + db = null; + } + T.assert(await opfs.entryExists(filename)); + try { + db = new oo1Ctor(fileUri); + db.exec(initSql) /* will throw if delete-before-open did not work */; + T.assert(3 === db.selectValue('select count(*) from p')); + }finally{ + if(db) db.close(); + } + }, + + importer: async function(vfsName, oo1Ctor, sqlite3){ + let db; + const filename = this.opfsDbFile; + try { + const exp = this.opfsDbExport; + delete this.opfsDbExport; + this.opfsImportSize = await oo1Ctor.importDb(filename, exp); + db = new oo1Ctor(this.opfsDbFile); + T.assert(6 === db.selectValue('select count(*) from p')). + assert( this.opfsImportSize == exp.byteLength ); + db.close(); + db = null; + this.opfsUnlink = + (fn=filename)=>sqlite3.util.sqlite3__wasm_vfs_unlink(vfsName, fn); + this.opfsUnlink(filename); + T.assert(!(await sqlite3.opfs.entryExists(filename))); + // Try again with a function as an input source: + let cursor = 0; + const blockSize = 512, end = exp.byteLength; + const reader = async function(){ + if(cursor >= exp.byteLength){ + return undefined; + } + const rv = exp.subarray(cursor, cursor+blockSize>end ? end : cursor+blockSize); + cursor += blockSize; + return rv; + }; + this.opfsImportSize = await oo1Ctor.importDb(filename, reader); + db = new oo1Ctor(this.opfsDbFile); + T.assert(6 === db.selectValue('select count(*) from p')). + assert( this.opfsImportSize == exp.byteLength ); + }finally{ + if(db) db.close(); + } + }, + + opfsUtil: async function(vfsName, oo1Ctor, sqlite3){ + const filename = this.opfsDbFile; + const unlink = this.opfsUnlink; + T.assert(filename && !!unlink); + delete this.opfsDbFile; + delete this.opfsUnlink; + /************************************************************** + ATTENTION CLIENT-SIDE USERS: sqlite3.opfs is NOT intended + for client-side use. It is only for this project's own + internal use. Its APIs are subject to change or removal at + any time. The sqlite3.opfs namespace is REMOVED from the + sqlite3 namespace in non-test runs of the library. + ***************************************************************/ + const opfs = sqlite3.opfs; + const fSize = this.opfsImportSize; + delete this.opfsImportSize; + let sh; + try{ + T.assert(await opfs.entryExists(filename)); + const [dirHandle, filenamePart] = await opfs.getDirForFilename(filename, false); + const fh = await dirHandle.getFileHandle(filenamePart); + sh = await fh.createSyncAccessHandle(); + T.assert(fSize === await sh.getSize()); + await sh.close(); + sh = undefined; + unlink(); + T.assert(!(await opfs.entryExists(filename))); + }finally{ + if(sh) await sh.close(); + unlink(); + } + + // Some sanity checks of the opfs utility functions... + const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12); + const aDir = testDir+'/test/dir'; + T.assert(await opfs.mkdir(aDir), "mkdir failed") + .assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists") + .assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)") + .assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed") + .assert(!(await opfs.unlink(testDir+'/test/dir')), + "delete 2b should have failed (dir already deleted)") + .assert((await opfs.unlink(testDir, true)), "delete 3 failed") + .assert(!(await opfs.entryExists(testDir)), + "entryExists(",testDir,") should have failed"); + } + }/*T.opfsCommon*/; + //////////////////////////////////////////////////////////////////////// // End of infrastructure setup. Now define the tests... //////////////////////////////////////////////////////////////////////// @@ -431,7 +638,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; assert(wasmCtypes.structs[1/*sqlite3_io_methods*/ ].members.xFileSize.offset>0); [ /* Spot-check a handful of constants to make sure they got installed... */ - 'SQLITE_SCHEMA','SQLITE_NULL','SQLITE_UTF8', + 'SQLITE_SCHEMA','SQLITE_NULL','SQLITE_UTF8','SQLITE_UTF8_ZT', 'SQLITE_STATIC', 'SQLITE_DIRECTONLY', 'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE' ].forEach((k)=>T.assert('number' === typeof capi[k])); @@ -945,7 +1152,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; let ptr = k1.pointer; k1.dispose(); T.assert(undefined === k1.pointer). - mustThrowMatching(()=>{k1.$pP=1}, /disposed instance/); + mustThrowMatching(()=>{k1.$pP=1}, /disposed/); }finally{ k1.dispose(); k2.dispose(); @@ -1549,6 +1756,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule; T.assert(blob instanceof Uint8Array). assert(0x68===blob[0] && 0x69===blob[1]); blob = null; + blob = db.selectValue("select ?1", new Uint8Array([97,0,98,0,99]), + sqlite3.capi.SQLITE_TEXT); + T.assert("a\0b\0c"===blob, "Something is amiss with embedded NULs"); let counter = 0, colNames = []; list.length = 0; db.exec(new TextEncoder('utf-8').encode("SELECT a a, b b FROM t"),{ @@ -2543,7 +2753,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .assert(wasm.isPtr(tmplMod.$xRowid)) .assert(wasm.isPtr(tmplMod.$xCreate)) .assert(tmplMod.$xCreate === tmplMod.$xConnect, - "setup() must make these equivalent and "+ + "setupModule() must make these equivalent and "+ "installMethods() must avoid re-compiling identical functions"); tmplMod.$xCreate = wasm.ptr.null /* make tmplMod eponymous-only */; let rc = capi.sqlite3_create_module( @@ -2794,25 +3004,69 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// T.g('kvvfs') .t({ - name: 'kvvfs is disabled in worker', - predicate: ()=>(isWorker() || "test is only valid in a Worker"), + name: 'kvvfs v1 API availability', test: function(sqlite3){ - T.assert( - !capi.sqlite3_vfs_find('kvvfs'), - "Expecting kvvfs to be unregistered." - ); + const capi = sqlite3.capi; + if( isUIThread() ){ + T.assert( !!capi.sqlite3_js_kvvfs_size ) + .assert( !!capi.sqlite3_js_kvvfs_clear ); + }else{ + /* Historical behaviour retained not for compatibility but + to help avoid some confusion between the v1 and v2 kvvfs + APIs (namely in how the v1 variants handle empty + strings). */ + T.assert( !capi.sqlite3_js_kvvfs_clear ) + .assert( !capi.sqlite3_js_kvvfs_size ); + } + const k = sqlite3.kvvfs; + T.assert( k && 'object'===typeof k ); + for(const n of ['reserve', 'import', 'export', + 'unlink', 'listen', 'unlisten', + 'exists', + 'estimateSize', 'clear'] ){ + T.assert( k[n] instanceof Function ); + } + + if( 0 ){ + const scope = wasm.scopedAllocPush(); + try{ + const pg = [ + "53514C69746520666F726D61742033b20b0101b402020d02d02l01d0", + "4l01jb02b2E91E00Dd011FD3b1FD3dxl2B010617171701377461626C", + "656B767666736B7676667302435245415445205441424C45206B76766673286129" + ].join(''); + const n = pg.length; + const pI = wasm.scopedAlloc( n+1 ); + const nO = 8192 * 2; + const pO = wasm.scopedAlloc( nO ); + const heap = wasm.heap8u(); + let i; + for( i=0; i<n; ++i ){ + heap[wasm.ptr.add(pI, i)] = pg.codePointAt(i) & 0xff; + } + heap[wasm.ptr.add(pI, i)] = 0; + const rc = wasm.exports.sqlite3__wasm_kvvfs_decode(pI, pO, nO); + const u = heap.slice(pO, wasm.ptr.add(pO,rc)); + debug("decode rc=", rc, u); + }finally{ + wasm.scopedAllocPop(scope); + } + } } - }) + }/*kvvfs API availability*/) .t({ - name: 'kvvfs in main thread', - predicate: ()=>(isUIThread() - || "local/sessionStorage are unavailable in a Worker"), + name: 'kvvfs sessionStorage', + predicate: ()=>(globalThis.sessionStorage || "sessionStorage is unavailable"), test: function(sqlite3){ - const filename = this.kvvfsDbFile = 'session'; + const JDb = sqlite3.oo1.JsStorageDb; const pVfs = capi.sqlite3_vfs_find('kvvfs'); T.assert(looksLikePtr(pVfs)); - const JDb = this.JDb = sqlite3.oo1.JsStorageDb; - const unlink = this.kvvfsUnlink = ()=>JDb.clearStorage(this.kvvfsDbFile); + let x = sqlite3.kvvfs.internal.storageForZClass('session'); + T.assert( 0 === x.files.length ) + .assert( globalThis.sessionStorage===x.storage ) + .assert( 'kvvfs-session-' === x.keyPrefix ); + const filename = this.kvvfsDbFile = 'session'; + const unlink = this.kvvfsUnlink = ()=>sqlite3.kvvfs.clear(filename); unlink(); let db = new JDb(filename); try { @@ -2820,90 +3074,538 @@ globalThis.sqlite3InitModule = sqlite3InitModule; 'create table kvvfs(a);', 'insert into kvvfs(a) values(1),(2),(3)' ]); - T.assert(3 === db.selectValue('select count(*) from kvvfs')); + T.assert(3 === db.selectValue('select count(*) from kvvfs')) + .assert( db.storageSize() > 0, "Db size counting is broken" ); db.close(); + db = undefined; db = new JDb(filename); db.exec('insert into kvvfs(a) values(4),(5),(6)'); T.assert(6 === db.selectValue('select count(*) from kvvfs')); }finally{ - db.close(); + if( db ) db.close(); } + //console.debug("sessionStorage",globalThis.sessionStorage); } }/*kvvfs sanity checks*/) -//#if enable-see .t({ - name: 'kvvfs with SEE encryption', - predicate: ()=>(isUIThread() - || "Only available in main thread."), + name: 'transient kvvfs', + //predicate: ()=>false, + test: function(sqlite3){ + const filename = '.' /* preinstalled instance */; + const JDb = sqlite3.oo1.JsStorageDb; + const DB = sqlite3.oo1.DB; + T.mustThrowMatching(()=>new JDb(""), capi.SQLITE_MISUSE); + T.mustThrowMatching(()=>{ + new JDb("this\ns an illegal - contains control characters"); + /* We don't have a way to get error strings from xOpen() + to this point? xOpen() does not have a handle to the + db and SQLite is not calling xGetLastError() to fetch + the error string. */ + }, capi.SQLITE_RANGE); + T.mustThrowMatching(()=>{new JDb("foo-journal");}, + capi.SQLITE_MISUSE); + T.mustThrowMatching(()=>{new JDb("foo-wal");}, + capi.SQLITE_MISUSE); + T.mustThrowMatching(()=>{new JDb("foo-shm");}, + capi.SQLITE_MISUSE); + T.mustThrowMatching(()=>{ + new JDb("01234567890123456789"+ + "01234567890123456789"+ + "01234567890123456789"+ + "01234567890123456789"+ + "01234567890123456789"+ + "01234567890123456789"+ + "0"/*too long*/); + }, capi.SQLITE_RANGE); + { + const name = "01234567890123456789012" /* max name length */; + (new JDb(name)).close(); + T.assert( sqlite3.kvvfs.unlink(name) ); + } + + sqlite3.kvvfs.clear(filename); + let db = new JDb(filename); + const sqlSetup = [ + 'create table kvvfs(a);', + 'insert into kvvfs(a) values(1),(2),(3)' + ]; + try { + T.assert( 0===db.storageSize(), "expecting 0 storage size" ); + T.mustThrowMatching(()=>db.clearStorage(), /in-use/); + //db.clearStorage(); + T.assert( 0===db.storageSize(), "expecting 0 storage size" ); + db.exec(sqlSetup); + T.assert( 0<db.storageSize(), "expecting non-0 db size" ); + T.mustThrowMatching(()=>db.clearStorage(), /in-use/); + //db.clearStorage(/*wiping everything out from under it*/); + T.assert( 0<db.storageSize(), "expecting non-0 storage size" ); + //db.exec(sqlSetup/*that actually worked*/); + /* Clearing the storage out from under the db does actually + work so long as we re-initialize it before reading. + It is tentatively disallowed for sanity's sake rather + than it not working. + */ + T.assert( 0<db.storageSize(), "expecting non-0 db size" ); + const close = ()=>{ + db.close(); + db = undefined; + }; + T.assert(3 === db.selectValue('select count(*) from kvvfs')); + close(); + + const exportDb = sqlite3.kvvfs.export; + db = new JDb(filename); + db.exec('insert into kvvfs(a) values(4),(5),(6)'); + T.assert(6 === db.selectValue('select count(*) from kvvfs')); + const exp = exportDb({name:filename,includeJournal:true}); + T.assert( filename===exp.name, "Broken export filename" ) + .assert( exp?.size > 0, "Missing db size" ) + .assert( exp?.pages?.length > 0, "Missing db pages" ); + console.debug("kvvfs to Object:",exp); + close(); + + const dbFileRaw = 'file:new-storage?vfs=kvvfs&delete-on-close=1'; + db = new DB({ + filename: dbFileRaw, + //flags: 'ct' + }); + db.exec(sqlSetup); + const dbFilename = db.dbFilename(); + //console.warn("db.dbFilename() =",dbFilename); + T.assert(3 === db.selectValue('select count(*) from kvvfs')); + debug("kvvfs to Object:",exportDb(dbFilename)); + const n = sqlite3.kvvfs.estimateSize( dbFilename ); + T.assert( n>0, "Db size count failed" ); + + if( 1 ){ + // Concurrent open of that same name uses the same storage + const x = new JDb(dbFilename); + T.assert(3 === db.selectValue('select count(*) from kvvfs')); + x.close(); + } + close(); + // When the final instance of a name is closed, the storage + // disappears... + T.mustThrowMatching(function(){ + /* Ensure that 'new-storage' was deleted when its refcount + went to 0, because of its 'transient' flag. By default + the objects are retained, like a filesystem would. */ + let ddb = new JDb(dbFilename); + try{ddb.selectValue('select a from kvvfs')} + finally{ddb.close()} + }, /no such table: kvvfs/); + }finally{ + if( db ) db.close(); + } + } + }/*transient kvvfs*/) + .t({ + name: 'concurrent transient kvvfs', + //predicate: ()=>false, test: function(sqlite3){ - this.kvvfsUnlink(); - let initDb = true; - const tryKey = function(keyKey, key, expectCount){ - let db; - //console.debug('tryKey()',arguments); - const ctoropt = { - filename: this.kvvfsDbFile - //vfs: 'kvvfs' - //,flags: 'ct' + const filename = 'myStorage'; + const kvvfs = sqlite3.kvvfs; + const DB = sqlite3.oo1.DB; + const JDb = sqlite3.oo1.JsStorageDb; + let db; + let duo; + let q1, q2; + const sqlSetup = [ + 'create table kvvfs(a);', + 'insert into kvvfs(a) values(1),(2),(3)' + ]; + const sqlCount = 'select count(*) from kvvfs'; + + try { + const exportDb = sqlite3.kvvfs.export; + const dbFileRaw = 'file:'+filename+'?vfs=kvvfs&delete-on-close=1'; + sqlite3.kvvfs.clear(filename); + db = new DB(dbFileRaw); + db.exec(sqlSetup); + T.assert(3 === db.selectValue(sqlCount)); + + duo = new JDb(filename); + duo.exec('insert into kvvfs(a) values(4),(5),(6)'); + T.assert(6 === db.selectValue(sqlCount)); + const expOpt = { + name: filename, + decodePages: true }; - try { - if (initDb) { - initDb = false; - db = new this.JDb({ - ...ctoropt, - [keyKey]: key - }); - db.exec([ - "drop table if exists t;", - "create table t(a);" - ]); - db.close(); - // Ensure that it's actually encrypted... - let err; - try { - db = new this.JDb(ctoropt); - T.assert(db, 'db opened') /* opening is fine, but... */; - db.exec("select 1 from sqlite_schema"); - console.warn("(should not be reached) sessionStorage =", sessionStorage); - } catch (e) { - err = e; - } finally { - db.close() - } - T.assert(err, "Expecting an exception") - .assert(sqlite3.capi.SQLITE_NOTADB == err.resultCode, - "Expecting NOTADB"); - }/*initDb*/ - //console.debug('tryKey()',arguments); - db = new sqlite3.oo1.DB({ - ...ctoropt, - vfs: 'kvvfs', - [keyKey]: key + let exp = exportDb(expOpt); + let expectRows = 6; + debug("exported db",exp); + db.close(); + T.assert(expectRows === duo.selectValue(sqlCount)); + duo.close(); + T.mustThrowMatching(function(){ + let ddb = new JDb(filename); + try{ddb.selectValue('select a from kvvfs')} + finally{ddb.close()} + }, /.*no such table: kvvfs.*/); + + T.assert( kvvfs.unlink(filename) ) + .assert( !kvvfs.exists(filename) ); + + const importDb = sqlite3.kvvfs.import; + duo = new JDb(dbFileRaw); + T.mustThrowMatching(()=>importDb(exp,true), /.*in use.*/); + duo.close(); + importDb(exp, true); + duo = new JDb(dbFileRaw); + T.assert(expectRows === duo.selectValue(sqlCount)); + let newCount; + try{ + duo.transaction(()=>{ + duo.exec("insert into kvvfs(a) values(7)"); + newCount = duo.selectValue(sqlCount); + T.assert(false, "rolling back"); }); - db.exec("insert into t(a) values (1),(2)"); - T.assert(expectCount === db.selectValue('select sum(a) from t')); - } finally { - if (db) db.close(); + }catch(e){/*ignored*/} + T.assert(7===newCount, "Unexpected row count before rollback") + .assert(expectRows === duo.selectValue(sqlCount), + "Unexpected row count after rollback"); + duo.close(); + + T.assert( kvvfs.unlink(filename) ) + .assert( !kvvfs.exists(filename) ); + + importDb(exp, true); + db = new JDb({ + filename, + flags: 'c' + /* BUG: without the 'c' flag, the db works until we try to + vacuum, at which point it fails with "read only db". */ + }); + duo = new JDb(filename); + T.assert(expectRows === duo.selectValue(sqlCount)); + const sqlIns1 = "insert into kvvfs(a) values(?)"; + q1 = db.prepare(sqlIns1); + q2 = duo.prepare(sqlIns1); + if( 0 ){ + q1.bind('from q1').stepFinalize(); + ++expectRows; + T.assert(expectRows === duo.selectValue(sqlCount), + "Unexpected record count."); + q2.bind('from q1').stepFinalize(); + ++expectRows; + }else{ + q1.bind('from q1'); + T.assert(capi.SQLITE_DONE===capi.sqlite3_step(q1), + "Unexpected step result"); + ++expectRows; + T.assert(expectRows === duo.selectValue(sqlCount), + "Unexpected record count."); + q2.bind('from q1').step(); + ++expectRows; + } + T.assert(expectRows === db.selectValue(sqlCount), + "Unexpected record count."); + q1.finalize(); + q2.finalize(); + + if( 1 ){ + debug("Begin vacuum/page size test..."); + const defaultPageSize = 1024 * 8 /* build-time default */; + const pageSize = 0 + ? defaultPageSize + : 1024 * 16 /* any valid value other than the default */; + if( 0 ){ + debug("Export before vacuum", exportDb(expOpt)); + debug("page size before vacuum", + db.selectArray( + "select page_size from pragma_page_size()" + )); + } + //kvvfs.log.xFileControl = true; + //kvvfs.log.xAccess = true; + db.exec([ + "BEGIN;", + "insert into kvvfs(a) values(randomblob(16000/*>pg size*/));", + "COMMIT;", + "delete from kvvfs where octet_length(a)>100;", + "pragma page_size="+pageSize+";", + "vacuum;", + "select 1;" + ]); + const expectPageSize = kvvfs.internal.disablePageSizeChange + ? defaultPageSize + : pageSize; + const gotPageSize = db.selectValue( + "select page_size from pragma_page_size()" + ); + T.assert(+gotPageSize === expectPageSize, + "Expecting page size",expectPageSize, + "got",gotPageSize); + T.assert(expectRows === duo.selectValue(sqlCount), + "Unexpected record count."); + kvvfs.log.xAccess = kvvfs.log.xFileControl = false; + T.assert(expectRows === duo.selectValue(sqlCount), + "Unexpected record count."); + exp = exportDb(expOpt); + debug("Exported page-expanded db",exp); + if( 0 ){ + debug("vacuumed export",exp); + } + debug("End vacuum/page size test."); + }else{ + expectRows = 6; + } + + db.close(); + duo.close(); + T.assert( kvvfs.unlink(exp.name) ) + .assert( !kvvfs.exists(exp.name) ); + importDb(exp); + T.assert( kvvfs.exists(exp.name) ); + db = new JDb(exp.name); + //debug("column count after export",db.selectValue(sqlCount)); + T.assert(expectRows === db.selectValue(sqlCount), + "Unexpected record count."); + + /* + TODO: more advanced concurrent use tests, e.g. looping + over a query in one connection while writing from + another. Currently that will probably corrupt the db, and + kvvfs's journaling does not support multiple journals per + storage unit. We need to test the locking and fix it as + appropriate. + */ + }finally{ + q1?.finalize?.(); + q2?.finalize?.(); + db?.close?.(); + duo?.close?.(); + } + } + }/*concurrent transient kvvfs*/) + + .t({ + name: 'kvvfs listeners (experiment)', + test: function(sqlite3){ + const kvvfs = sqlite3.kvvfs; + const filename = 'listen'; + let db; + try { + const DB = sqlite3.oo1.DB; + const sqlSetup = [ + 'create table kvvfs(a);', + 'insert into kvvfs(a) values(1),(2),(3)' + ]; + const sqlCount = "select count(*) from kvvfs"; + const sqlSelectSchema = "select * from sqlite_schema"; + const counts = Object.create(null); + const incr = (key)=>counts[key] = 1 + (counts[key] ?? 0); + const pglog = Object.assign(Object.create(null),{ + pages: [], + jrnl: undefined, + size: undefined, + includeJournal: false, + decodePages: true, + exception: new Error("Testing that exceptions from listeners do not interfere") + }); + const toss = ()=>{ + if( pglog.exception ){ + const e = pglog.exception; + delete pglog.exception; + throw e; + } + }; + + const listener = { + storage: filename, + reserve: true, + includeJournal: pglog.includeJournal, + decodePages: pglog.decodePages, + events: { + /** + These may be async but must not be in this case + because we can't test their result without a lot of + hoop-jumping if they are. Kvvfs calls these + asynchronously, though. + */ + 'open': (ev)=>{ + //console.warn('open',ev); + incr(ev.type); + T.assert(filename===ev.storageName) + .assert('number'===typeof ev.data); + }, + 'close': (ev)=>{ + //console.warn('close',ev); + incr(ev.type); + T.assert('number'===typeof ev.data); + toss(); + }, + 'delete': (ev)=>{ + //console.warn('delete',ev); + incr(ev.type); + T.assert('string'===typeof ev.data); + switch(ev.data){ + case 'jrnl': + T.assert(pglog.includeJournal); + pglog.jrnl = null; + break; + default:{ + const n = +ev.data; + T.assert( n>0, "Expecting positive db page number" ); + if( n < pglog.pages.length ){ + pglog.size = undefined; + } + pglog.pages[n] = undefined; + break; + } + } + }, + 'sync': (ev)=>{ + incr(ev.data ? 'xSync' : 'xFileControlSync'); + }, + 'write': (ev)=>{ + //console.warn('write',ev); + incr(ev.type); + T.assert(Array.isArray(ev.data)); + const key = ev.data[0], val = ev.data[1]; + T.assert('string'===typeof key); + switch( key ){ + case 'jrnl': + T.assert(pglog.includeJournal); + pglog.jrnl = val; + break; + case 'sz':{ + const sz = +val; + T.assert( sz>0, "Expecting a db page number" ); + if( sz < pglog.sz ){ + pglog.pages.length = sz / pglog.pages.length; + } + pglog.size = sz; + break; + } + default: + T.assert( +key>0, "Expecting a positive db page number" ); + pglog.pages[+key] = val; + if( pglog.decodePages ){ + T.assert( val instanceof Uint8Array ); + }else{ + T.assert( 'string'===typeof val ); + } + break; + } + } + } + }; + + kvvfs.listen(listener); + const dbFileRaw = 'file:'+filename+'?vfs=kvvfs&delete-on-close=1'; + const expOpt = { + name: filename, + //decodePages: true + }; + db = new DB(dbFileRaw); + db.exec(sqlSetup); + T.assert(db.selectObjects(sqlSelectSchema)?.length>0, + "Unexpected empty schema"); + db.close(); + debug("kvvfs listener counts:",counts); + T.assert( counts.open ); + T.assert( counts.close ); + T.assert( listener.includeJournal ? counts.delete : !counts.delete ); + T.assert( counts.write ); + T.assert( counts.xSync ); + T.assert( counts.xFileControlSync>=counts.xSync ); + T.assert( counts.open===counts.close ); + T.assert( pglog.includeJournal + ? (null===pglog.jrnl) + : (undefined===pglog.jrnl), + "Unexpected pglog.jrnl value: "+pglog.jrnl ); + if( 1 ){ + T.assert(undefined===pglog.pages[0], "Expecting empty slot 0"); + pglog.pages.shift(); + //debug("kvvfs listener pageLog", pglog); } - }.bind(this); - const hexFoo = new Uint8Array([0x66,0x6f,0x6f]/*=="foo"*/); - tryKey('textkey', 'foo', 3); - T.assert( !initDb ); - tryKey('textkey', 'foo', 6); - this.kvvfsUnlink(); - initDb = true; - tryKey('key', 'foo', 3); - T.assert( !initDb ); - tryKey('key', hexFoo, 6); - this.kvvfsUnlink(); - initDb = true; - tryKey('hexkey', hexFoo, 3); - T.assert( !initDb ); - tryKey('hexkey', hexFoo, 6); - this.kvvfsUnlink(); + const before = JSON.stringify(counts); + T.assert( kvvfs.unlisten(listener) ); + T.assert( !kvvfs.unlisten(listener) ); + db = new DB(dbFileRaw); + T.assert( db.selectObjects(sqlSelectSchema)?.length>0 ); + const exp = kvvfs.export(expOpt); + const expectRows = db.selectValue(sqlCount); + db.exec("delete from kvvfs"); + db.close(); + const after = JSON.stringify(counts); + T.assert( before===after, "Expecting no events after unlistening." ); + if( 0 ){ + exp = kvvfs.export(expOpt); + debug("Post-delete export:",exp); + } + if( 1 ){ + // Replace the storage with the pglog state... + const bogoExp = { + name: filename, + size: pglog.size, + timestamp: Date.now(), + pages: pglog.pages + }; + //debug("exp",exp); + //debug("bogoExp",bogoExp); + kvvfs.import(bogoExp, true); + //debug("Re-exported", kvvfs.export(expOpt)); + db = new DB(dbFileRaw); + // Failing on the next line despite exports looking good + T.assert(db.selectObjects(sqlSelectSchema)?.length>0, + "Empty schema on imported db"); + T.assert(expectRows===db.selectValue(sqlCount)); + db.close(); + } + }finally{ + db?.close?.(); + kvvfs.unlink(filename); + } + } + })/*kvvfs listeners */ + + .t({ + name: 'kvvfs vtab', + predicate: (sqlite3)=>!!sqlite3.kvvfs.create_module, + test: function(sqlite3){ + const kvvfs = sqlite3.kvvfs; + const db = new sqlite3.oo1.DB(); + const db2 = new sqlite3.oo1.DB('file:foo?vfs=kvvfs&delete-on-close=1'); + try{ + kvvfs.create_module(db); + let rc = db.selectObjects("select * from sqlite_kvvfs order by name"); + debug("sqlite_kvvfs vtab:", rc); + const nDb = rc.length; + rc = db.selectObject("select * from sqlite_kvvfs where name='foo'"); + T.assert(rc, "Expecting foo storage record") + .assert('foo'===rc.name, "Unexpected name") + .assert(1===rc.nRef, "Unexpected refcount"); + db2.close(); + rc = db.selectObjects("select * from sqlite_kvvfs"); + T.assert( !rc.filter((o)=>o.name==='foo').length, + "Expecting foo storage to be gone"); + debug("sqlite_kvvfs vtab:", rc); + T.assert( rc.length+1 === nDb, + "Unexpected storage count: got",rc.length,"expected",nDb); + }finally{ + db.close(); + db2.close(); + } + } + })/* kvvfs vtab */ + +//#if enable-see + .t({ + name: 'kvvfs SEE encryption in sessionStorage', + predicate: ()=>(!!globalThis.sessionStorage + || "sessionStorage is not available"), + test: function(sqlite3){ + const JDb = sqlite3.oo1.JsStorageDb; + T.seeBaseCheck(JDb, + (isInit)=>{ + return {filename: "session"}; + }, + ()=>JDb.clearStorage('session')); } })/*kvvfs with SEE*/ -//#endif enable-see +//#/if enable-see ;/* end kvvfs tests */ //////////////////////////////////////////////////////////////////////// @@ -2915,7 +3617,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; let countCommit = 0, countRollback = 0;; const db = new sqlite3.oo1.DB(':memory:',1 ? 'c' : 'ct'); let rc = capi.sqlite3_commit_hook(db, (p)=>{ - //console.debug("commit hook",arguments); + //debug("commit hook",arguments); ++countCommit; return (17 == p) ? 0 : capi.SQLITE_ERROR; }, 17); @@ -3188,206 +3890,57 @@ globalThis.sqlite3InitModule = sqlite3InitModule; ;/*end of session API group*/; //////////////////////////////////////////////////////////////////////// - T.g('OPFS: Origin-Private File System', - (sqlite3)=>(sqlite3.capi.sqlite3_vfs_find("opfs") - || 'requires "opfs" VFS')) + // Tests for "opfs" and "opfs-wl" are essentially identical, so... +//#query { + select 'opfs' vfsName, 'OPFS: Origin-Private File System' label, 'OpfsDb' oo1Ctor + UNION ALL + select 'opfs-wl', 'OPFS with Web Locks', 'OpfsWlDb' +} + T.g('@vfsName@: @label@', + (sqlite3)=>(!!capi.sqlite3_vfs_find("@vfsName@") || 'requires "@vfsName@" VFS')) .t({ - name: 'OPFS db sanity checks', - test: async function(sqlite3){ - T.assert(capi.sqlite3_vfs_find('opfs')); - const opfs = sqlite3.opfs; - const filename = this.opfsDbFile = '/dir/sqlite3-tester1.db'; - const fileUri = 'file://'+filename+'?delete-before-open=1'; - const initSql = [ - 'create table p(a);', - 'insert into p(a) values(1),(2),(3)' - ]; - let db = new sqlite3.oo1.OpfsDb(fileUri); - try { - db.exec(initSql); - T.assert(3 === db.selectValue('select count(*) from p')); - db.close(); - db = new sqlite3.oo1.OpfsDb(filename); - db.exec('insert into p(a) values(4),(5),(6)'); - T.assert(6 === db.selectValue('select count(*) from p')); - this.opfsDbExport = capi.sqlite3_js_db_export(db); - T.assert(this.opfsDbExport instanceof Uint8Array) - .assert(this.opfsDbExport.byteLength>0 - && 0===this.opfsDbExport.byteLength % 512); - }finally{ - db.close(); - } - T.assert(await opfs.entryExists(filename)); - try { - db = new sqlite3.oo1.OpfsDb(fileUri); - db.exec(initSql) /* will throw if delete-before-open did not work */; - T.assert(3 === db.selectValue('select count(*) from p')); - }finally{ - if(db) db.close(); - } + name: '@vfsName@ db sanity checks', + test: async (sqlite3)=>{ + await T.opfsCommon.sanityChecks('@vfsName@', sqlite3.oo1.@oo1Ctor@, sqlite3); } - }/*OPFS db sanity checks*/) + }) .t({ - name: 'OPFS import', - test: async function(sqlite3){ - let db; - const filename = this.opfsDbFile; - try { - const exp = this.opfsDbExport; - delete this.opfsDbExport; - this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(filename, exp); - db = new sqlite3.oo1.OpfsDb(this.opfsDbFile); - T.assert(6 === db.selectValue('select count(*) from p')). - assert( this.opfsImportSize == exp.byteLength ); - db.close(); - this.opfsUnlink = - (fn=filename)=>sqlite3.util.sqlite3__wasm_vfs_unlink("opfs",fn); - this.opfsUnlink(filename); - T.assert(!(await sqlite3.opfs.entryExists(filename))); - // Try again with a function as an input source: - let cursor = 0; - const blockSize = 512, end = exp.byteLength; - const reader = async function(){ - if(cursor >= exp.byteLength){ - return undefined; - } - const rv = exp.subarray(cursor, cursor+blockSize>end ? end : cursor+blockSize); - cursor += blockSize; - return rv; - }; - this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(filename, reader); - db = new sqlite3.oo1.OpfsDb(this.opfsDbFile); - T.assert(6 === db.selectValue('select count(*) from p')). - assert( this.opfsImportSize == exp.byteLength ); - }finally{ - if(db) db.close(); - } + name: '@vfsName@ import/export', + test: async (sqlite3)=>{ + await T.opfsCommon.importer('@vfsName@', sqlite3.oo1.@oo1Ctor@, sqlite3); } - }/*OPFS export/import*/) + }) +//#if vfsName = "opfs" +//#// This is independent of the VFS, so only test this once .t({ name: '(Internal-use) OPFS utility APIs', - test: async function(sqlite3){ - const filename = this.opfsDbFile; - const unlink = this.opfsUnlink; - T.assert(filename && !!unlink); - delete this.opfsDbFile; - delete this.opfsUnlink; - /************************************************************** - ATTENTION CLIENT-SIDE USERS: sqlite3.opfs is NOT intended - for client-side use. It is only for this project's own - internal use. Its APIs are subject to change or removal at - any time. - ***************************************************************/ - const opfs = sqlite3.opfs; - const fSize = this.opfsImportSize; - delete this.opfsImportSize; - let sh; - try{ - T.assert(await opfs.entryExists(filename)); - const [dirHandle, filenamePart] = await opfs.getDirForFilename(filename, false); - const fh = await dirHandle.getFileHandle(filenamePart); - sh = await fh.createSyncAccessHandle(); - T.assert(fSize === await sh.getSize()); - await sh.close(); - sh = undefined; - unlink(); - T.assert(!(await opfs.entryExists(filename))); - }finally{ - if(sh) await sh.close(); - unlink(); - } - - // Some sanity checks of the opfs utility functions... - const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12); - const aDir = testDir+'/test/dir'; - T.assert(await opfs.mkdir(aDir), "mkdir failed") - .assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists") - .assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)") - .assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed") - .assert(!(await opfs.unlink(testDir+'/test/dir')), - "delete 2b should have failed (dir already deleted)") - .assert((await opfs.unlink(testDir, true)), "delete 3 failed") - .assert(!(await opfs.entryExists(testDir)), - "entryExists(",testDir,") should have failed"); - } - }/*OPFS util sanity checks*/) + test: async (sqlite3)=>{ + await T.opfsCommon.opfsUtil("@vfsName@", sqlite3.oo1.@oo1Ctor@, sqlite3); + } + }) +//#/if //#if enable-see .t({ - name: 'OPFS with SEE encryption', + name: '@vfsName@ with SEE encryption', + predicate: (sqlite3)=>!!sqlite3.oo1.@oo1Ctor@, test: function(sqlite3){ - const dbFile = 'file:///sqlite3-see.edb'; - const dbCtor = sqlite3.oo1.OpfsDb; - const hexFoo = new Uint8Array([0x66,0x6f,0x6f]/*=="foo"*/); - let initDb = true; - const tryKey = function(keyKey, key, expectCount){ - let db; - //console.debug('tryKey()',arguments); - const ctoropt = { - filename: dbFile, - flags: 'c' - }; - try { - if (initDb) { - initDb = false; - const opt = { - ...ctoropt, - [keyKey]: key - }; - opt.filename += '?delete-before-open=1'; - db = new dbCtor(opt); - db.exec([ - "drop table if exists t;", - "create table t(a);" - ]); - db.close(); - // Ensure that it's actually encrypted... - let err; - try { - db = new dbCtor(ctoropt); - T.assert(db, 'db opened') /* opening is fine, but... */; - const rv = db.exec({ - sql:"select count(*) from sqlite_schema", - returnValue: 'resultRows' - }); - console.warn("(should not be reached) rv =",rv); - } catch (e) { - err = e; - } finally { - db.close() - } - T.assert(err, "Expecting an exception") - .assert(sqlite3.capi.SQLITE_NOTADB == err.resultCode, - "Expecting NOTADB"); - }/*initDb*/ - db = new dbCtor({ - ...ctoropt, - [keyKey]: key - }); - db.exec("insert into t(a) values (1),(2)"); - T.assert(expectCount === db.selectValue('select sum(a) from t')); - } finally { - if (db) db.close(); - } - }; - tryKey('textkey', 'foo', 3); - T.assert( !initDb ); - tryKey('textkey', 'foo', 6); - initDb = true; - tryKey('key', 'foo', 3); - T.assert( !initDb ); - tryKey('key', hexFoo, 6); - initDb = true; - tryKey('hexkey', hexFoo, 3); - T.assert( !initDb ); - tryKey('hexkey', hexFoo, 6); - } - })/*OPFS with SEE*/ -//#endif enable-see + T.seeBaseCheck( + sqlite3.oo1.@oo1Ctor@, + function(isInit){ + const opt = {filename: 'file:///sqlite3-see.edb'}; + if( isInit ) opt.filename += '?delete-before-open=1'; + return opt; + }, + ()=>{} + ); + } + }) +//#/if enable-see ;/* end OPFS tests */ - +//#/query //////////////////////////////////////////////////////////////////////// T.g('OPFS SyncAccessHandle Pool VFS', - (sqlite3)=>(hasOpfs() || "requires OPFS APIs")) + (sqlite3)=>(!!sqlite3.installOpfsSAHPoolVfs || "requires OPFS SAH Pool APIs")) .t({ name: 'SAH sanity checks', test: async function(sqlite3){ @@ -3584,73 +4137,15 @@ globalThis.sqlite3InitModule = sqlite3InitModule; let poolUtil; const P1 = await inst(poolConfig).then(u=>poolUtil = u).catch(catcher); const dbFile = '/sqlite3-see.edb'; - const dbCtor = poolUtil.OpfsSAHPoolDb; - const hexFoo = new Uint8Array([0x66,0x6f,0x6f]/*=="foo"*/); - let initDb = true; - const tryKey = function(keyKey, key, expectCount){ - let db; - //console.debug('tryKey()',arguments); - const ctoropt = { - filename: dbFile, - flags: 'c' - }; - try { - if (initDb) { - initDb = false; - poolUtil.unlink(dbFile); - db = new dbCtor({ - ...ctoropt, - [keyKey]: key - }); - db.exec([ - "drop table if exists t;", - "create table t(a);" - ]); - db.close(); - // Ensure that it's actually encrypted... - let err; - try { - db = new dbCtor(ctoropt); - T.assert(db, 'db opened') /* opening is fine, but... */; - const rv = db.exec({ - sql:"select count(*) from sqlite_schema", - returnValue: 'resultRows' - }); - console.warn("(should not be reached) rv =",rv); - } catch (e) { - err = e; - } finally { - db.close() - } - T.assert(err, "Expecting an exception") - .assert(sqlite3.capi.SQLITE_NOTADB == err.resultCode, - "Expecting NOTADB"); - }/*initDb*/ - db = new dbCtor({ - ...ctoropt, - [keyKey]: key - }); - db.exec("insert into t(a) values (1),(2)"); - T.assert(expectCount === db.selectValue('select sum(a) from t')); - } finally { - if (db) db.close(); - } - }; - tryKey('textkey', 'foo', 3); - T.assert( !initDb ); - tryKey('textkey', 'foo', 6); - initDb = true; - tryKey('key', 'foo', 3); - T.assert( !initDb ); - tryKey('key', hexFoo, 6); - initDb = true; - tryKey('hexkey', hexFoo, 3); - T.assert( !initDb ); - tryKey('hexkey', hexFoo, 6); + T.seeBaseCheck( + poolUtil.OpfsSAHPoolDb, + (isInit)=>{return {filename: dbFile}}, + ()=>poolUtil.unlink(dbFile) + ); poolUtil.removeVfs(); } })/*opfs-sahpool with SEE*/ -//#endif enable-see +//#/if enable-see ; //////////////////////////////////////////////////////////////////////// @@ -3715,44 +4210,64 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .t("Misc. stmt_...", function(sqlite3){ const db = new sqlite3.oo1.DB(); db.exec("create table t(a doggiebiscuits); insert into t(a) values(123)"); - const stmt = db.prepare("select a, a+1 from t"); - T.assert( stmt.isReadOnly() ) - .assert( 0===capi.sqlite3_stmt_isexplain(stmt) ) - .assert( 0===capi.sqlite3_stmt_explain(stmt, 1) ) - .assert( 0!==capi.sqlite3_stmt_isexplain(stmt) ) - .assert( 0===capi.sqlite3_stmt_explain(stmt, 2) ) - .assert( 0!==capi.sqlite3_stmt_isexplain(stmt) ) - .assert( 0===capi.sqlite3_stmt_explain(stmt, 0) ) - .assert( 0===capi.sqlite3_stmt_isexplain(stmt) ); - let n = 0; - while( capi.SQLITE_ROW === capi.sqlite3_step(stmt) ){ - ++n; - T.assert( 0!==capi.sqlite3_stmt_explain(stmt, 1), - "Because stmt is busy" ) - .assert( capi.sqlite3_stmt_busy(stmt) ) - .assert( stmt.isBusy() ) - .assert( 0!==capi.sqlite3_stmt_readonly(stmt) ) - .assert( true===stmt.isReadOnly() ); - const sv = capi.sqlite3_column_value(stmt, 0); - T.assert( 123===capi.sqlite3_value_int(sv) ) - .assert( "doggiebiscuits"===capi.sqlite3_column_decltype(stmt,0) ) - .assert( null===capi.sqlite3_column_decltype(stmt,1) ); - } - T.assert( 1===n ) - .assert( 0===capi.sqlite3_stmt_busy(stmt) ) - .assert( !stmt.isBusy() ); - - if( wasm.exports.sqlite3_column_origin_name ){ - log("Column metadata APIs enabled"); - T.assert( "t" === capi.sqlite3_column_table_name(stmt, 0)) - .assert("a" === capi.sqlite3_column_origin_name(stmt, 0)) - .assert("main" === capi.sqlite3_column_database_name(stmt, 0)) - }else{ - log("Column metadata APIs not enabled"); - } // column metadata APIs + let stmt; + try{ + stmt = db.prepare("select a, a+1 from t"); + T.assert( stmt.isReadOnly() ) + .assert( 0===capi.sqlite3_stmt_isexplain(stmt) ) + .assert( 0===capi.sqlite3_stmt_explain(stmt, 1) ) + .assert( 0!==capi.sqlite3_stmt_isexplain(stmt) ) + .assert( 0===capi.sqlite3_stmt_explain(stmt, 2) ) + .assert( 0!==capi.sqlite3_stmt_isexplain(stmt) ) + .assert( 0===capi.sqlite3_stmt_explain(stmt, 0) ) + .assert( 0===capi.sqlite3_stmt_isexplain(stmt) ); + let n = 0; + while( capi.SQLITE_ROW === capi.sqlite3_step(stmt) ){ + ++n; + T.assert( 0!==capi.sqlite3_stmt_explain(stmt, 1), + "Because stmt is busy" ) + .assert( capi.sqlite3_stmt_busy(stmt) ) + .assert( stmt.isBusy() ) + .assert( 0!==capi.sqlite3_stmt_readonly(stmt) ) + .assert( true===stmt.isReadOnly() ); + const sv = capi.sqlite3_column_value(stmt, 0); + T.assert( 123===capi.sqlite3_value_int(sv) ) + .assert( "doggiebiscuits"===capi.sqlite3_column_decltype(stmt,0) ) + .assert( null===capi.sqlite3_column_decltype(stmt,1) ); + } + T.assert( 1===n ) + .assert( 0===capi.sqlite3_stmt_busy(stmt) ) + .assert( !stmt.isBusy() ); + + if( wasm.exports.sqlite3_column_origin_name ){ + log("Column metadata APIs enabled"); + T.assert( "t" === capi.sqlite3_column_table_name(stmt, 0)) + .assert("a" === capi.sqlite3_column_origin_name(stmt, 0)) + .assert("main" === capi.sqlite3_column_database_name(stmt, 0)) + }else{ + log("Column metadata APIs not enabled"); + } // column metadata APIs + stmt.finalize(); stmt = null; + stmt = db.prepare("select ?1").bind(new Uint8Array([97,0,98,0,99])); + stmt.step(); + const sv = capi.sqlite3_column_value(stmt,0); + T.assert("a\0b\0c"===capi.sqlite3_value_text(sv), + "Expecting NULs to have survived."); + stmt.finalize(); stmt = null; + + /* sqlite3_bind_zeroblob() (added in 3.53) */ + stmt = db.prepare("select ?1"); + T.assert( 0===capi.sqlite3_bind_zeroblob(stmt, 1, 53) ); + T.assert( stmt.step() ); + const b = stmt.get(0); + stmt.finalize(); stmt = null; + T.assert( b instanceof Uint8Array ) + .assert( 53===b.length ); - stmt.finalize(); - db.close(); + }finally{ + if(stmt) stmt.finalize(); + db.close(); + } }) //////////////////////////////////////////////////////////////////// @@ -3951,12 +4466,17 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// log("Loading and initializing sqlite3 WASM module..."); - if(0){ + if(1){ globalThis.sqlite3ApiConfig = { - debug: ()=>{}, - log: ()=>{}, - warn: ()=>{}, - error: ()=>{} + //debug: ()=>{}, log: ()=>{}, warn: ()=>{}, error: ()=>{}, + disable: { + vfs: { + kvvfs: false, + opfs: false, + "opfs-sahpool": false, + "opfs-wl": false + } + } } } //#if not target:es6-module @@ -3987,7 +4507,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; } importScripts(sqlite3Js); } -//#endif +//#/if globalThis.sqlite3InitModule.__isUnderTest = true /* disables certain API-internal cleanup so that we can test internal APIs from here */; diff --git a/ext/wasm/tests/opfs/concurrency/index.html b/ext/wasm/tests/opfs/concurrency/index.html index 54ed04a4f..b7f77824b 100644 --- a/ext/wasm/tests/opfs/concurrency/index.html +++ b/ext/wasm/tests/opfs/concurrency/index.html @@ -25,22 +25,21 @@ <h1></h1> the <code>workers=N</code> URL flag. Set the time between each workload with <code>interval=N</code> (milliseconds). Set the number of worker iterations with <code>iterations=N</code>. - Enable OPFS VFS verbosity with <code>verbose=1-3</code> (output - goes to the dev console). Disable/enable "unlock ASAP" mode - (higher concurrency, lower speed) with <code>unlock-asap=0-1</code>. + Disable/enable "unlock ASAP" mode (potentially higher + concurrency but lower speed) with <code>unlock-asap=0-1</code>. </p> <p>Achtung: if it does not start to do anything within a couple of seconds, check the dev console: Chrome sometimes fails to load - the wasm module due to "cannot allocate WasmMemory." Closing and - re-opening the tab usually resolves it, but sometimes restarting - the browser is required. + the wasm module due to "cannot allocate WasmMemory" when + reloading the page. Closing and re-opening the tab usually + resolves it, but sometimes restarting the browser is required. </p> <p> Links for various testing options: <ul id='testlinks'></ul> </p> <div class='input-wrapper'> - <input type='checkbox' id='cb-log-reverse'> - <label for='cb-log-reverse'>Reverse log order?</label> + <button id='gogogo'><strong>Start test</strong></button> + <label><input type='checkbox' id='cb-log-reverse'>Reverse log order?</label> </div> <div id='test-output'></div> <script>(function(){ diff --git a/ext/wasm/tests/opfs/concurrency/test.js b/ext/wasm/tests/opfs/concurrency/test.js index 1848901af..91a3f4856 100644 --- a/ext/wasm/tests/opfs/concurrency/test.js +++ b/ext/wasm/tests/opfs/concurrency/test.js @@ -1,4 +1,5 @@ (async function(self){ + const btnGo = document.querySelector('#gogogo'); const logCss = (function(){ const mapToString = (v)=>{ @@ -8,6 +9,7 @@ return ''+v; default: break; } + if( v instanceof Date ) return v.toString(); if(null===v) return 'null'; if(v instanceof Error){ v = { @@ -55,11 +57,11 @@ const options = Object.create(null); options.sqlite3Dir = urlArgsJs.get('sqlite3.dir'); options.workerCount = ( - urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 3 - ) || 4; + urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 0 + ) || 3; options.opfsVerbose = ( urlArgsHtml.has('verbose') ? +urlArgsHtml.get('verbose') : 1 - ) || 1; + ) || 0; options.interval = ( urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 1000 ) || 1000; @@ -69,47 +71,61 @@ options.unlockAsap = ( urlArgsHtml.has('unlock-asap') ? +urlArgsHtml.get('unlock-asap') : 0 ) || 0; + options.vfs = urlArgsHtml.get('vfs') || 'opfs'; options.noUnlink = !!urlArgsHtml.has('no-unlink'); const workers = []; workers.post = (type,...args)=>{ for(const w of workers) w.postMessage({type, payload:args}); }; workers.counts = {loaded: 0, passed: 0, failed: 0}; + let timeStart; + const calcTime = (endDate)=>{ + return endDate.getTime() - timeStart?.getTime?.(); + }; const checkFinished = function(){ if(workers.counts.passed + workers.counts.failed !== workers.length){ return; } if(workers.counts.failed>0){ - logCss('tests-fail',"Finished with",workers.counts.failed,"failure(s)."); + logCss('tests-fail',"Finished with",workers.counts.failed,"failure(s) in", + calcTime(new Date()),"ms"); }else{ - logCss('tests-pass',"All",workers.length,"workers finished."); + logCss('tests-pass',"All",workers.length,"workers finished in", + calcTime(new Date()),"ms"); } + logCss("Reload page to run the test again."); }; + workers.onmessage = function(msg){ msg = msg.data; const prefix = 'Worker #'+msg.worker+':'; switch(msg.type){ - case 'loaded': - stdout(prefix,"loaded"); - if(++workers.counts.loaded === workers.length){ - stdout("All",workers.length,"workers loaded. Telling them to run..."); - workers.post('run'); - } - break; - case 'stdout': stdout(prefix,...msg.payload); break; - case 'stderr': stderr(prefix,...msg.payload); break; - case 'error': stderr(prefix,"ERROR:",...msg.payload); break; - case 'finished': - ++workers.counts.passed; - logCss('tests-pass',prefix,...msg.payload); - checkFinished(); - break; - case 'failed': - ++workers.counts.failed; - logCss('tests-fail',prefix,"FAILED:",...msg.payload); - checkFinished(); - break; - default: logCss('error',"Unhandled message type:",msg); break; + case 'loaded': + stdout(prefix,"loaded"); + if(++workers.counts.loaded === workers.length){ + timeStart = new Date(); + stdout(timeStart,"All",workers.length,"workers loaded. Telling them to run..."); + workers.post('run'); + } + break; + case 'stdout': stdout(prefix,...msg.payload); break; + case 'stderr': stderr(prefix,...msg.payload); break; + case 'error': stderr(prefix,"ERROR:",...msg.payload); break; + case 'finished': { + ++workers.counts.passed; + const timeEnd = new Date(); + logCss('tests-pass',prefix,timeEnd,"("+calcTime(timeEnd)+"ms) ", ...msg.payload); + checkFinished(); + break; + } + case 'failed': { + ++workers.counts.failed; + const timeEnd = new Date(); + logCss('tests-fail',prefix,timeEnd,"("+calcTime(timeEnd)+"ms) FAILED:",...msg.payload); + checkFinished(); + break; + } + default: logCss('error',"Unhandled message type:",msg); break; } }; @@ -118,7 +134,10 @@ const eTestLinks = document.querySelector('#testlinks'); const optArgs = function(obj){ const li = []; - for(const k of ['interval','iterations','workers','verbose','unlock-asap']){ + for(const k of [ + 'interval', 'iterations', 'unlock-asap', + 'opfsVerbose', 'vfs', 'workers' + ]){ if( obj.hasOwnProperty(k) ) li.push(k+'='+obj[k]); } return li.join('&'); @@ -127,7 +146,10 @@ {interval: 1000, workers: 5, iterations: 30}, {interval: 500, workers: 5, iterations: 30}, {interval: 250, workers: 3, iterations: 30}, - {interval: 600, workers: 5, iterations: 100} + {interval: 600, workers: 5, iterations: 100}, + {interval: 500, workers: 1, iterations: 5, vfs: 'opfs-wl'}, + {interval: 500, workers: 2, iterations: 10, vfs: 'opfs-wl'}, + {interval: 500, workers: 5, iterations: 20, vfs: 'opfs-wl'}, ]){ const li = document.createElement('li'); eTestLinks.appendChild(li); @@ -138,24 +160,28 @@ a.innerText = args; } - stdout("Launching",options.workerCount,"workers. Options:",options); - workers.uri = ( - 'worker.js?' - + 'sqlite3.dir='+options.sqlite3Dir - + '&interval='+options.interval - + '&iterations='+options.iterations - + '&opfs-verbose='+options.opfsVerbose - + '&opfs-unlock-asap='+options.unlockAsap - ); - for(let i = 0; i < options.workerCount; ++i){ - stdout("Launching worker..."); - workers.push(new Worker( - workers.uri+'&workerId='+(i+1)+( - (i || options.noUnlink) ? '' : '&unlink-db' - ) - )); - } - // Have to delay onmessage assignment until after the loop - // to avoid that early workers get an undue head start. - workers.forEach((w)=>w.onmessage = workers.onmessage); -})(self); + btnGo.addEventListener('click', ()=>{ + btnGo.remove(); + stdout("Launching",options.workerCount,"workers. Options:",options); + workers.uri = ( + 'worker.js?' + + 'sqlite3.dir='+options.sqlite3Dir + + '&vfs='+options.vfs + + '&interval='+options.interval + + '&iterations='+options.iterations + + '&opfs-verbose='+options.opfsVerbose + + '&opfs-unlock-asap='+options.unlockAsap + ); + for(let i = 0; i < options.workerCount; ++i){ + stdout("Launching worker...", i, ); + workers.push(new Worker( + workers.uri+'&workerId='+(i+1)+( + (i || options.noUnlink) ? '' : '&unlink-db' + ) + )); + } + // Have to delay onmessage assignment until after the loop + // to avoid that early workers get an undue head start. + workers.forEach((w)=>w.onmessage = workers.onmessage); + }); +})(globalThis); diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 5d28bedee..398e5b5a7 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -1,19 +1,25 @@ -importScripts( - (new URL(self.location.href).searchParams).get('sqlite3.dir') + '/sqlite3.js' -); -self.sqlite3InitModule().then(async function(sqlite3){ - const urlArgs = new URL(self.location.href).searchParams; - const options = { - workerName: urlArgs.get('workerId') || Math.round(Math.random()*10000), - unlockAsap: urlArgs.get('opfs-unlock-asap') || 0 /*EXPERIMENTAL*/ - }; +'use strict'; +const urlArgs = new URL(globalThis.location.href).searchParams; +const options = { + workerName: urlArgs.get('workerId') || Math.round(Math.random()*10000), + unlockAsap: urlArgs.get('opfs-unlock-asap') || 0, + vfs: urlArgs.get('vfs') +}; +const jsSqlite = urlArgs.get('sqlite3.dir') + +'/sqlite3.js?opfs-async-proxy-id=' + +options.workerName; +importScripts(jsSqlite)/*Sigh - URL args are not propagated this way*/; +//const sqlite3InitModule = (await import(jsSqlite)).default; +globalThis.sqlite3InitModule.__isUnderTest = + true /* causes sqlite3.opfs to be retained */; +globalThis.sqlite3InitModule().then(async function(sqlite3){ const wPost = (type,...payload)=>{ postMessage({type, worker: options.workerName, payload}); }; const stdout = (...args)=>wPost('stdout',...args); const stderr = (...args)=>wPost('stderr',...args); if(!sqlite3.opfs){ - stderr("OPFS support not detected. Aborting."); + stderr("This code requires the (private) sqlite3.opfs object. Aborting."); return; } @@ -34,80 +40,107 @@ self.sqlite3InitModule().then(async function(sqlite3){ count: 0 }); const finish = ()=>{ - if(db){ - if(!db.pointer) return; + if(db?.pointer){ db.close(); } if(interval.error){ - wPost('failed',"Ending work after interval #"+interval.count, + stderr("Ending work at interval #"+interval.count, + "due to error:", interval.error); + wPost('failed', "at interval #"+interval.count, "due to error:",interval.error); }else{ wPost('finished',"Ending work after",interval.count,"intervals."); } }; const run = async function(){ - db = new sqlite3.oo1.OpfsDb({ - filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, - flags: 'c' + const Ctors = Object.assign(Object.create(null),{ + opfs: sqlite3.oo1.OpfsDb, + 'opfs-wl': sqlite3.oo1.OpfsWlDb }); + const ctor = Ctors[options.vfs]; + if( !ctor ){ + stderr("Invalid VFS name:",vfs); + return; + } + stdout("ctor:", options.vfs); + while(true){ + try{ + db = new ctor({ + filename: 'file:'+dbName+'?opfs-unlock-asap='+options.unlockAsap, + flags: 'c' + }); + break; + }catch(e){ + if(e instanceof sqlite3.SQLite3Error + && sqlite3.capi.SQLITE_BUSY===e.resultCode){ + stderr("Retrying open() for BUSY:",e.message); + continue; + } + throw e; + } + } sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000); - db.transaction((db)=>{ - db.exec([ - "create table if not exists t1(w TEXT UNIQUE ON CONFLICT REPLACE,v);", - "create table if not exists t2(w TEXT UNIQUE ON CONFLICT REPLACE,v);" - ]); + sqlite3.capi.sqlite3_js_retry_busy(-1, ()=>{ + db.transaction((db)=>{ + db.exec([ + "create table if not exists t1(w TEXT UNIQUE ON CONFLICT REPLACE,v);", + "create table if not exists t2(w TEXT UNIQUE ON CONFLICT REPLACE,v);" + ]); + }); }); const maxIterations = urlArgs.has('iterations') ? (+urlArgs.get('iterations') || 10) : 10; - stdout("Starting interval-based db updates with delay of",interval.delay,"ms."); + stdout("Starting",maxIterations,"interval-based db updates with delay of",interval.delay,"ms."); const doWork = async ()=>{ const tm = new Date().getTime(); ++interval.count; const prefix = "v(#"+interval.count+")"; stdout("Setting",prefix,"=",tm); try{ - db.exec({ - sql:"INSERT OR REPLACE INTO t1(w,v) VALUES(?,?)", - bind: [options.workerName, new Date().getTime()] + sqlite3.capi.sqlite3_js_retry_busy(-1, ()=>{ + db.exec({ + sql:"INSERT OR REPLACE INTO t1(w,v) VALUES(?,?)", + bind: [options.workerName, new Date().getTime()] + }); + }, (n)=>{ + stderr("Attempt #",n,"for interval #",interval.count,"due to BUSY"); }); - //stdout("Set",prefix); }catch(e){ + stderr("Error: ",e.message); interval.error = e; + throw e; } + //stdout("doWork()",prefix,"error ",interval.error); }; - if(1){/*use setInterval()*/ - setTimeout(async function timer(){ - await doWork(); - if(interval.error || maxIterations === interval.count){ - finish(); - }else{ - setTimeout(timer, interval.delay); - } - }, interval.delay); - }else{ - /*This approach provides no concurrency whatsoever: each worker - is run to completion before any others can work.*/ - let i; - for(i = 0; i < maxIterations; ++i){ - await doWork(); - if(interval.error) break; - await wait(interval.ms); + setTimeout(async function timer(){ + await doWork(); + if(interval.error || maxIterations === interval.count){ + finish(); + }else{ + const jitter = (Math.random()>=0.5) + ? ((Math.random()*100 - Math.random()*100) | 0) + : 0; + const n = interval.delay + jitter; + setTimeout(timer, n<50 ? interval.delay : n); } - finish(); - } + }, interval.delay); }/*run()*/; - self.onmessage = function({data}){ + globalThis.onmessage = function({data}){ switch(data.type){ - case 'run': run().catch((e)=>{ + case 'run': + run().catch((e)=>{ if(!interval.error) interval.error = e; + }).catch(e=>{ + /* Don't do this in finally() - it fires too soon. */ finish(); + throw e; }); - break; - default: - stderr("Unhandled message type '"+data.type+"'."); - break; + break; + default: + stderr("Unhandled message type '"+data.type+"'."); + break; } }; }); diff --git a/main.mk b/main.mk index ac0fdb513..8cc654031 100644 --- a/main.mk +++ b/main.mk @@ -269,6 +269,11 @@ EXTRA_SRC ?= # invocations of $(B.cc)). The configure process does not set either # of $(OPTIONS) or $(OPTS). # +# The difference between $(OPT_FEATURE_FLAGS) and $(OPTS) is that the +# former is historically provided by the configure script, whereas +# $(OPTS) is intended to be provided as an argument to the make +# invocation. +# OPT_FEATURE_FLAGS ?= # # $(SHELL_OPT) = @@ -328,6 +333,7 @@ all: sqlite3.h sqlite3.c CFLAGS.core ?= CFLAGS.env = $(CFLAGS) T.cc += $(CFLAGS.core) $(CFLAGS.env) +T.cc += $(OPT_FEATURE_FLAGS) $(OPTS) # # $(LDFLAGS.configure) represents any LDFLAGS=... the client passes to @@ -349,20 +355,6 @@ T.cc += $(CFLAGS.core) $(CFLAGS.env) # LDFLAGS.configure ?= -# -# The difference between $(OPT_FEATURE_FLAGS) and $(OPTS) is that the -# former is historically provided by the configure script, whereas -# $(OPTS) is intended to be provided as arguments to the make -# invocation. -# -T.cc += $(OPT_FEATURE_FLAGS) - -# -# Add in any optional global compilation flags on the make command -# line i.e. make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1". -# -T.cc += $(OPTS) - # # $(INSTALL) invocation for use with non-executable files. # @@ -386,19 +378,13 @@ T.link.gcov ?= # T.compile = $(T.cc) $(T.compile.gcov) -# -# Optionally set by the configure script to include -DSQLITE_DEBUG=1 -# and other debug-related flags. -# -T.cc.TARGET_DEBUG ?= - # # Extra CFLAGS for both the core sqlite3 components and extensions. # # Define -D_HAVE_SQLITE_CONFIG_H so that the code knows it # can include the generated sqlite_cfg.h. # -T.cc.sqlite.extras = -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite $(T.cc.TARGET_DEBUG) +T.cc.sqlite.extras = -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite # # $(T.cc.sqlite) is $(T.cc) plus any flags which are desired for the @@ -652,7 +638,7 @@ SRC = \ $(TOP)/src/sqliteInt.h \ $(TOP)/src/sqliteLimit.h \ $(TOP)/src/table.c \ - $(TOP)/src/tclsqlite.c \ + tclsqlite-ex.c \ $(TOP)/src/threads.c \ $(TOP)/src/tokenize.c \ $(TOP)/src/treeview.c \ @@ -952,30 +938,12 @@ FUZZDATA = \ # TESTOPTS = --verbose=file --output=test-out.txt -# -# Extra compiler options for various shell tools -# -# Note that some of these will only apply when embedding sqlite3.c -# into the shell, as these flags are not otherwise passed on to the -# library. -SHELL_OPT += -DSQLITE_DQS=0 -SHELL_OPT += -DSQLITE_ENABLE_FTS4 -#SHELL_OPT += -DSQLITE_ENABLE_FTS5 -SHELL_OPT += -DSQLITE_ENABLE_RTREE -SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS -SHELL_OPT += -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION -SHELL_OPT += -DSQLITE_ENABLE_STMTVTAB -SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB -SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB -SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB -SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC -SHELL_OPT += -DSQLITE_ENABLE_PERCENTILE -SHELL_OPT += -DSQLITE_STRICT_SUBTYPE=1 FUZZERSHELL_OPT = FUZZCHECK_OPT += -I$(TOP)/test FUZZCHECK_OPT += -I$(TOP)/ext/recover FUZZCHECK_OPT += \ -DSQLITE_OSS_FUZZ \ + -DSQLITE_DECIMAL_MAX_DIGIT=1000 \ -DSQLITE_ENABLE_BYTECODE_VTAB \ -DSQLITE_ENABLE_CARRAY \ -DSQLITE_ENABLE_DBPAGE_VTAB \ @@ -1007,13 +975,26 @@ FUZZCHECK_OPT += \ -DSQLITE_STRICT_SUBTYPE=1 \ -DSQLITE_STATIC_RANDOMJSON -FUZZCHECK_SRC += $(TOP)/test/fuzzcheck.c -FUZZCHECK_SRC += $(TOP)/test/ossfuzz.c -FUZZCHECK_SRC += $(TOP)/test/fuzzinvariants.c -FUZZCHECK_SRC += $(TOP)/ext/recover/dbdata.c -FUZZCHECK_SRC += $(TOP)/ext/recover/sqlite3recover.c -FUZZCHECK_SRC += $(TOP)/test/vt02.c -FUZZCHECK_SRC += $(TOP)/ext/misc/randomjson.c +FUZZCHECK_SRC = sqlite3.c \ + $(TOP)/test/fuzzcheck.c \ + $(TOP)/test/ossfuzz.c \ + $(TOP)/test/fuzzinvariants.c \ + $(TOP)/ext/recover/dbdata.c \ + $(TOP)/ext/recover/sqlite3recover.c \ + $(TOP)/test/vt02.c \ + $(TOP)/ext/misc/base64.c \ + $(TOP)/ext/misc/base85.c \ + $(TOP)/ext/misc/completion.c \ + $(TOP)/ext/misc/decimal.c \ + $(TOP)/ext/misc/ieee754.c \ + $(TOP)/ext/misc/randomjson.c \ + $(TOP)/ext/misc/regexp.c \ + $(TOP)/ext/misc/series.c \ + $(TOP)/ext/misc/shathree.c \ + $(TOP)/ext/misc/sha1.c \ + $(TOP)/ext/misc/stmtrand.c + +FUZZCHECK_DEP = sqlite3.h DBFUZZ_OPT = ST_OPT = -DSQLITE_OS_KV_OPTIONAL @@ -1428,15 +1409,15 @@ whereexpr.o: $(TOP)/src/whereexpr.c $(DEPS_OBJ_COMMON) window.o: $(TOP)/src/window.c $(DEPS_OBJ_COMMON) $(T.cc.sqlite) -c $(TOP)/src/window.c -tclsqlite.o: $(T.tcl.env.sh) $(TOP)/src/tclsqlite.c $(DEPS_OBJ_COMMON) +tclsqlite.o: $(T.tcl.env.sh) tclsqlite-ex.c $(DEPS_OBJ_COMMON) $(T.compile.tcl) -DUSE_TCL_STUBS=1 $$TCL_INCLUDE_SPEC \ - -c $(TOP)/src/tclsqlite.c + -c tclsqlite-ex.c -o tclsqlite.o -tclsqlite-shell.o: $(T.tcl.env.sh) $(TOP)/src/tclsqlite.c $(DEPS_OBJ_COMMON) - $(T.compile.tcl) -DTCLSH -o $@ -c $(TOP)/src/tclsqlite.c $$TCL_INCLUDE_SPEC +tclsqlite-shell.o: $(T.tcl.env.sh) tclsqlite-ex.c $(DEPS_OBJ_COMMON) + $(T.compile.tcl) -DTCLSH -o $@ -c tclsqlite-ex.c $$TCL_INCLUDE_SPEC -tclsqlite-stubs.o: $(T.tcl.env.sh) $(TOP)/src/tclsqlite.c $(DEPS_OBJ_COMMON) - $(T.compile.tcl) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c $$TCL_INCLUDE_SPEC +tclsqlite-stubs.o: $(T.tcl.env.sh) tclsqlite-ex.c $(DEPS_OBJ_COMMON) + $(T.compile.tcl) -DUSE_TCL_STUBS=1 -o $@ -c tclsqlite-ex.c $$TCL_INCLUDE_SPEC # # STATIC_TCLSQLITE3 = 1 to statically link tclsqlite3, else @@ -1680,11 +1661,19 @@ install-tcl-0 install-tcl-: install-tcl: install-tcl-$(HAVE_TCL) install: install-tcl -tclsqlite3.c: sqlite3.c +TCLSQLITEEX = \ + $(TOP)/ext/qrf/qrf.h \ + $(TOP)/ext/qrf/qrf.c \ + $(TOP)/src/tclsqlite.c + +tclsqlite-ex.c: $(TCLSQLITEEX) $(TOP)/tool/mkcombo.tcl $(B.tclsh) + $(B.tclsh) $(TOP)/tool/mkcombo.tcl $(TCLSQLITEEX) -o $@ + +tclsqlite3.c: sqlite3.c tclsqlite-ex.c echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c cat sqlite3.c >>tclsqlite3.c echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c - cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c + cat tclsqlite-ex.c >>tclsqlite3.c # # $(CFLAGS.tclextension) = CFLAGS for the tclextension* targets. @@ -1801,33 +1790,19 @@ TESTFIXTURE_FLAGS += -DSQLITE_STRICT_SUBTYPE=1 TESTFIXTURE_SRC0 = $(TESTSRC2) $(libsqlite3.LIB) TESTFIXTURE_SRC1 = sqlite3.c -TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c +TESTFIXTURE_SRC = $(TESTSRC) tclsqlite-ex.c TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION)) testfixture$(T.exe): $(T.tcl.env.sh) has_tclsh85 $(TESTFIXTURE_SRC) $(T.link.tcl) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \ -o $@ $(TESTFIXTURE_SRC) \ - $$TCL_LIB_SPEC $$TCL_INCLUDE_SPEC \ + $$TCL_LIB_SPEC $$TCL_INCLUDE_SPEC $$TCL_LIBS \ $(LDFLAGS.libsqlite3) coretestprogs: testfixture$(B.exe) sqlite3$(B.exe) testprogs: $(TESTPROGS) srcck1$(B.exe) fuzzcheck$(T.exe) sessionfuzz$(T.exe) -# A very detailed test running most or all test cases -fulltest: alltest fuzztest - -# Run most or all tcl test cases -alltest: $(TESTPROGS) - ./testfixture$(T.exe) $(TOP)/test/all.test $(TESTOPTS) - -# Really really long testing -soaktest: $(TESTPROGS) - ./testfixture$(T.exe) $(TOP)/test/all.test -soak=1 $(TESTOPTS) - -# Do extra testing but not everything. -fulltestonly: $(TESTPROGS) fuzztest - ./testfixture$(T.exe) $(TOP)/test/full.test # # Fuzz testing @@ -1874,6 +1849,15 @@ mdevtest: srctree-check has_tclsh85 sdevtest: has_tclsh85 $(TCLSH_CMD) $(TOP)/test/testrunner.tcl sdevtest $(TSTRNNR_OPTS) +retest: has_tclsh85 + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl retest + +errors: + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl errors + +status: + $(TCLSH_CMD) $(TOP)/test/testrunner.tcl status -d 2 + # Like releasetest, except it omits srctree-check and verify-source so # that it can be used on a modified source tree. # @@ -1894,19 +1878,15 @@ releasetest: srctree-check has_tclsh85 verify-source $(TCLSH_CMD) $(TOP)/test/testrunner.tcl release $(TSTRNNR_OPTS) # -# Minimal testing that runs in less than 3 minutes -# -quicktest: ./testfixture$(T.exe) - ./testfixture$(T.exe) $(TOP)/test/extraquick.test $(TESTOPTS) - -# -# Try to run tests on whatever options are specified by the -# ./configure. The developers seldom use this target. Instead -# they use "make devtest" which runs tests on a standard set of -# options regardless of how SQLite is configured. This "test" -# target is provided for legacy only. +# Legacy testing targets, no longer used by the developers and +# now aliased to one of the commonly used testing targets. # -test: srctree-check fuzztest sourcetest $(TESTPROGS) tcltest +quicktest: devtest +test: devtest +fulltest: releasetest +alltest: releasetest +soaktest: releasetest +fulltestonly: releasetest # # Run a test using valgrind. This can take a really long time @@ -1926,12 +1906,18 @@ smoketest: $(TESTPROGS) fuzzcheck$(T.exe) shelltest: $(TCLSH_CMD) $(TOP)/test/testrunner.tcl release shell +# Test performance of floating-point conversions. +# +fp-speed-test: fp-speed-1$(T.exe) fp-speed-2$(T.exe) + ./fp-speed-1 1000000 + ./fp-speed-2 1000000 + # # sqlite3_analyzer.c build depends on $(LINK_TOOLS_DYNAMICALLY). # sqlite3_analyzer.c.flags.0 = -DINCLUDE_SQLITE3_C=1 sqlite3_analyzer.c.flags.1 = -sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl \ +sqlite3_analyzer.c: sqlite3.c tclsqlite-ex.c $(TOP)/tool/spaceanal.tcl \ $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in $(B.tclsh) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in \ $(sqlite3_analyzer.c.flags.$(LINK_TOOLS_DYNAMICALLY)) \ @@ -1955,7 +1941,7 @@ sqlite3_analyzer$(T.exe): $(T.tcl.env.sh) sqlite3_analyzer.c \ # can cause the $@ to link to an out-of-tree libsqlite3.so, which may # or may not fail or otherwise cause confusion. -sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl \ +sqltclsh.c: sqlite3.c tclsqlite-ex.c $(TOP)/tool/sqltclsh.tcl \ $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl \ $(TOP)/tool/sqltclsh.c.in $(B.tclsh) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c @@ -1973,24 +1959,6 @@ sqlite3_expert$(T.exe): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqli $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert $(LDFLAGS.libsqlite3) xbin: sqlite3_expert$(T.exe) -CHECKER_DEPS =\ - $(TOP)/tool/mkccode.tcl \ - sqlite3.c \ - $(TOP)/src/tclsqlite.c \ - $(TOP)/ext/repair/sqlite3_checker.tcl \ - $(TOP)/ext/repair/checkindex.c \ - $(TOP)/ext/repair/checkfreelist.c \ - $(TOP)/ext/misc/btreeinfo.c \ - $(TOP)/ext/repair/sqlite3_checker.c.in - -sqlite3_checker.c: $(CHECKER_DEPS) - $(B.tclsh) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@ - -sqlite3_checker$(T.exe): $(T.tcl.env.sh) sqlite3_checker.c - $(T.link.tcl) sqlite3_checker.c -o $@ $$TCL_INCLUDE_SPEC \ - $$TCL_LIB_SPEC $(LDFLAGS.libsqlite3) -xbin: sqlite3_checker$(T.exe) - dbdump$(T.exe): $(TOP)/ext/misc/dbdump.c sqlite3.o $(T.link) -DDBDUMP_STANDALONE -o $@ \ $(TOP)/ext/misc/dbdump.c sqlite3.o $(LDFLAGS.libsqlite3) @@ -2020,6 +1988,10 @@ showshm$(T.exe): $(TOP)/tool/showshm.c $(T.link) -o $@ $(TOP)/tool/showshm.c $(LDFLAGS.configure) xbin: showshm$(T.exe) +showtmlog$(T.exe): $(TOP)/tool/showtmlog.c + $(T.link) -o $@ $(TOP)/tool/showtmlog.c $(LDFLAGS.configure) +xbin: showshm$(T.exe) + index_usage$(T.exe): $(TOP)/tool/index_usage.c sqlite3.o $(T.link) $(SHELL_OPT) -o $@ $(TOP)/tool/index_usage.c sqlite3.o \ $(LDFLAGS.libsqlite3) @@ -2057,6 +2029,14 @@ speedtest1$(T.exe): $(TOP)/test/speedtest1.c sqlite3.c Makefile $(LDFLAGS.libsqlite3) xbin: speedtest1$(T.exe) +fp-speed-1$(T.exe): $(TOP)/test/fp-speed-1.c sqlite3.o Makefile + $(T.link) $(ST_OPT) -o $@ $(TOP)/test/fp-speed-1.c sqlite3.o \ + $(LDFLAGS.libsqlite3) + +fp-speed-2$(T.exe): $(TOP)/test/fp-speed-2.c sqlite3.o Makefile + $(T.link) $(ST_OPT) -o $@ $(TOP)/test/fp-speed-2.c sqlite3.o \ + $(LDFLAGS.libsqlite3) + startup$(T.exe): $(TOP)/test/startup.c sqlite3.c $(T.link) -Os -g -USQLITE_THREADSAFE -DSQLITE_THREADSAFE=0 \ -o $@ $(TOP)/test/startup.c sqlite3.c $(LDFLAGS.libsqlite3) @@ -2118,16 +2098,19 @@ src-archives: sqlite-amalgamation.zip amalgamation-tarball sqlite-src.zip # Build a ZIP archive containing various command-line tools. # -tool-zip: testfixture$(T.exe) sqlite3$(T.exe) sqldiff$(T.exe) \ +tool-zip: sqlite3$(T.exe) sqldiff$(T.exe) \ sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) $(TOP)/tool/mktoolzip.tcl strip sqlite3$(T.exe) sqldiff$(T.exe) sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) - ./testfixture$(T.exe) $(TOP)/tool/mktoolzip.tcl + $(TCLSH_CMD) $(TOP)/tool/mktoolzip.tcl + snapshot-zip: testfixture$(T.exe) sqlite3$(T.exe) sqldiff$(T.exe) \ sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) $(TOP)/tool/mktoolzip.tcl strip sqlite3$(T.exe) sqldiff$(T.exe) sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) - ./testfixture$(T.exe) $(TOP)/tool/mktoolzip.tcl --snapshot + $(TCLSH_CMD) $(TOP)/tool/mktoolzip.tcl --snapshot + clean-tool-zip: rm -f sqlite-tools-*.zip + clean: clean-tool-zip # @@ -2274,23 +2257,23 @@ fuzzershell$(T.exe): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h fuzzy: fuzzershell$(T.exe) xbin: fuzzershell$(T.exe) -fuzzcheck$(T.exe): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) - $(T.link) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(LDFLAGS.libsqlite3) +fuzzcheck$(T.exe): $(FUZZCHECK_SRC) $(FUZZCHECK_DEP) + $(T.link) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) $(LDFLAGS.libsqlite3) fuzzy: fuzzcheck$(T.exe) xbin: fuzzcheck$(T.exe) # -fsanitize=... flags for fuzzcheck-asan. CFLAGS.fuzzcheck-asan.fsanitize ?= -fsanitize=address -fuzzcheck-asan$(T.exe): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) +fuzzcheck-asan$(T.exe): $(FUZZCHECK_SRC) $(FUZZCHECK_DEP) $(T.link) -o $@ $(CFLAGS.fuzzcheck-asan.fsanitize) $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) \ - sqlite3.c $(LDFLAGS.libsqlite3) + $(LDFLAGS.libsqlite3) fuzzy: fuzzcheck-asan$(T.exe) xbin: fuzzcheck-asan$(T.exe) -fuzzcheck-ubsan$(T.exe): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) +fuzzcheck-ubsan$(T.exe): $(FUZZCHECK_SRC) $(FUZZCHECK_DEP) $(T.link) -o $@ -fsanitize=undefined $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) \ - sqlite3.c $(LDFLAGS.libsqlite3) + $(LDFLAGS.libsqlite3) fuzzy: fuzzcheck-ubsan$(T.exe) xbin: fuzzcheck-ubsan$(T.exe) @@ -2352,6 +2335,8 @@ mptest: mptester$(T.exe) # Source and header files that shell.c depends on SHELL_DEP = \ $(TOP)/src/shell.c.in \ + $(TOP)/ext/qrf/qrf.c \ + $(TOP)/ext/qrf/qrf.h \ $(TOP)/ext/expert/sqlite3expert.c \ $(TOP)/ext/expert/sqlite3expert.h \ $(TOP)/ext/intck/sqlite3intck.c \ @@ -2497,6 +2482,7 @@ tidy: rm -f tclsqlite3$(T.exe) $(TESTPROGS) rm -f LogEst$(T.exe) fts3view$(T.exe) rollback-test$(T.exe) showdb$(T.exe) rm -f showjournal$(T.exe) showstat4$(T.exe) showwal$(T.exe) speedtest1$(T.exe) + rm -f fp-speed-1$(T.exe) fp-speed-2$(T.exe) rm -f wordcount$(T.exe) changeset$(T.exe) version-info$(T.exe) rm -f *.exp *.vsix pkgIndex.tcl rm -f sqlite3_analyzer$(T.exe) sqlite3_rsync$(T.exe) sqlite3_expert$(T.exe) diff --git a/make.bat b/make.bat new file mode 100644 index 000000000..2dc9b61c0 --- /dev/null +++ b/make.bat @@ -0,0 +1,2 @@ +@echo off +nmake /f Makefile.msc %* diff --git a/manifest b/manifest index 648b118a6..f875b8a5a 100644 --- a/manifest +++ b/manifest @@ -1,14 +1,14 @@ -C Version\s3.51.3 -D 2026-03-13T10:38:09.694 +C Version\s3.53.0 +D 2026-04-09T11:41:38.498 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md 6bc480fc673fb4acbc4094e77edb326267dd460162d7723c7f30bee2d3d9e97d -F Makefile.in 3ce07126d7e87c7464301482e161fdae6a51d0a2aa06b200b8f0000ef4d6163b +F Makefile.in 5fda086f33b144da08119255da1d2557f983d0764a13707f05acf0159fd89ba5 F Makefile.linux-generic bd3e3cacd369821a6241d4ea1967395c962dfe3057e38cb0a435cee0e8b789d0 -F Makefile.msc d4459fad28b388063698cbb7a73bfce8684da998a844a04b21d4b9b10291196a -F README.md dae499194b75deed76a13a4a83c82493f2530331882d7dfe5754d63287d3f8f7 -F VERSION 04eea51aa150c240c9a4dbd740393c575e73b04428482dc2b14ca6b98967b53c +F Makefile.msc 92391304cf70f4c178b127aa83b88637abd28d1b83ede451616144037ea1d3dd +F README.md f49fbd826941842e348242f3ab62f240c985ceafdf8fbe576abf4eb75317468c +F VERSION 31435e19ded2aae3c1c67dacf06a995a37fd1b253baec5899b78d64cd29db4f7 F art/icon-243x273.gif 9750b734f82fdb3dc43127753d5e6fbf3b62c9f4e136c2fbf573b2f57ea87af5 F art/icon-80x90.gif 65509ce3e5f86a9cd64fe7fca2d23954199f31fe44c1e09e208c80fb83d87031 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -17,8 +17,8 @@ F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F art/sqlite370.svg 40b7e2fe8aac3add5d56dd86ab8d427a4eca5bcb3fe4f8946cb3794e1821d531 F auto.def 44a0d1bf09d78355fc88251ccbf8e64e6341fd89c11de68a01c3645e53a2bade F autoconf/Makefile.fallback 22fe523eb36dfce31e0f6349f782eb084e86a5620b2b0b4f84a2d6133f53f5ac -F autoconf/Makefile.in 118aa2c4d49173672d065fdda19eb8a28642e2c684212d7a626d6db5e6762521 -F autoconf/Makefile.msc 9c1ca648062fd5a4a83ba7590c4422090cccd6399002af0346b7572f086c7483 +F autoconf/Makefile.in efab1f56c7961d301efc030baa4391b9faae38733d94385a3d56b54720a74aba +F autoconf/Makefile.msc e6c596e6e63ea17aa7a97eb8ded18cd984c8dd4203766644fbd57b8fcf720225 F autoconf/README.first f1d3876e9a7852c22f275a6f06814e64934cecbc0b5b9617d64849094c1fd136 F autoconf/README.txt b749816b8452b3af994dc6d607394bef3df1736d7e09359f1087de8439a52807 F autoconf/auto.def 3d994f3a9cc9b712dbce92a5708570ddcf3b988141b6eb738f2ed16127a9f0ac @@ -33,7 +33,7 @@ F autoconf/tea/teaish.tcl 81feb417e718ed75cdd7e2fdf6771f3da80dae97377a90c4d5b62b F autoconf/tea/teaish.test.tcl cfe94e1fb79dd078f650295be59843d470125e0cc3a17a1414c1fb8d77f4aea6 F autosetup/LICENSE 41a26aebdd2cd185d1e2b210f71b7ce234496979f6b35aef2cbf6b80cbed4ce4 F autosetup/README.autosetup a78ff8c4a3d2636a4268736672a74bf14a82f42687fcf0631a70c516075c031e -F autosetup/README.md ce0f95980a687bb861bd830b76bc4b48513567be5cf5ee7004f4f3439ffe3841 +F autosetup/README.md 18478aa27d13664abbdf9d40b90b040f7796570ef0591214731ef202692464e4 F autosetup/autosetup b16e44924c197783df67366762dda985b45d49ebc4af15f4054e3ee0e3b65169 x F autosetup/autosetup-config.guess dfa101c5e8220e864d5e9c72a85e87110df60260d36cb951ad0a85d6d9eaa463 x F autosetup/autosetup-config.sub a38fb074d0dece01cf919e9fb534a26011608aa8fa606490864295328526cd73 x @@ -44,10 +44,10 @@ F autosetup/cc-lib.tcl 493c5935b5dd3bf9bd4eca89b07c8b1b1a9356d61783035144e21795f F autosetup/cc-shared.tcl 163eda58c14cd662fd8a504bd2ad8a716ef4db7015dc1de0095d5de8dd601a4b F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e45f F autosetup/find_tclconfig.tcl e64886ffe3b982d4df42cd28ed91fe0b5940c2c5785e126c1821baf61bc86a7e -F autosetup/jimsh0.c a57c16e65dcffc9c76e496757cb3f7fb47e01ecbd1631a0a5e01751fc856f049 +F autosetup/jimsh0.c 916bbdf8023fbda9937afae57d81a853d8c2ea00f2320aa27becbc33574f963d F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba -F autosetup/proj.tcl 6fc14ef82b19b77a95788ffbcfad7989b4e3cb4ce96a21dcb5cf7312f362fba9 -F autosetup/sqlite-config.tcl 5d779fce20c11fde3fe99d157dcd5b5569d729b301141b8dfb8d5aacf9d48cba +F autosetup/proj.tcl ce301197f364f7ce2acabbbd84b43d19e917ec73653157ca134a06f32d322712 +F autosetup/sqlite-config.tcl 8fecce2838b7e7d990d161d08998034f3e3b0b2ddf4d7a99dbfafba9c902e302 F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9 F autosetup/teaish/README.txt b40071e6f8506500a2f7f71d5fc69e0bf87b9d7678dd9da1e5b4d0acbf40b1ca F autosetup/teaish/core.tcl e014dd95900c7f9a34e8e0f460f47e94841059827bce8b4c49668b0c7ae3f1a0 @@ -56,14 +56,14 @@ F autosetup/teaish/tester.tcl 1799514c2652db49561b3386c5242b94534d1663f2cfac861a F configure 9a00b21dfd13757bbfb8d89b30660a89ec1f8f3a79402b8f9f9b6fc475c3303a x F contrib/sqlitecon.tcl eb4c6578e08dd353263958da0dc620f8400b869a50d06e271ab0be85a51a08d3 F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd -F doc/compile-for-unix.md c9dce1ddd4bf0d25efccc5c63eb047e78c01ce06a6ff29c73e0a8af4a0f4adbc -F doc/compile-for-windows.md f9e74d74da88f384edd5809f825035e071608f00f7f39c0e448df7b3982f979c +F doc/compile-for-unix.md ebd6e461c71d43291e7e8c6487c9c2716bd997053556ec757509011e77e25581 +F doc/compile-for-windows.md 36601c95fa4070eebfe757684271d17a7c4a586912ba706d0b5e7817e1df54ad F doc/json-enhancements.md e356fc834781f1f1aa22ee300027a270b2c960122468499bf347bb123ce1ea4f F doc/jsonb.md acd77fc3a709f51242655ad7803510c886aa8304202fa9cf2abc5f5c4e9d7ae5 -F doc/lemon.html 89ea833a6f71773ab1a9063fbb7fb9b32147bc0b1057b53ecab94a3b30c0aef5 +F doc/lemon.html 2085fda0a90a94fe92159a79dccc5c30d5a2313524887a31659cd66162f17233 F doc/pager-invariants.txt 83aa3a4724b2d7970cc3f3461f0295c46d4fc19a835a5781cbb35cb52feb0577 F doc/tcl-extension-testing.md b88861804fc1eaf83249f8e206334189b61e150c360e1b80d0dcf91af82354f5 -F doc/testrunner.md 5ee928637e03f136a25fef852c5ed975932e31927bd9b05a574424ae18c31019 +F doc/testrunner.md daffa0ebbbe397a73537ae1b19b3124d489ce5f89dfe570781d1f1ef1809597c F doc/trusted-schema.md 33625008620e879c7bcfbbfa079587612c434fa094d338b08242288d358c3e8a F doc/vdbesort-memory.md 4da2639c14cd24a31e0af694b1a8dd37eaf277aff3867e9a8cc14046bc49df56 F doc/vfs-shm.txt 1a55f3f0e7b6745931b117ba5c9df3640d7a0536f532ef0052563100f4416f86 @@ -71,7 +71,7 @@ F doc/wal-lock.md 7db0cd61e2000b545b78ce89b0c2a9a8dd8d64c097839258ac10d7c5c4156e F ext/README.md 6eb1ac267d917767952ed0ef63f55de003b6a5da433ce1fa389e1a9532e73132 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3 F ext/expert/expert.c d548d603a4cc9e61f446cc179c120c6713511c413f82a4a32b1e1e69d3f086a4 -F ext/expert/expert1.test 1d2da6606623b57bb47064e02140823ce1daecd4cacbf402c73ad3473d7f000c +F ext/expert/expert1.test d9dfbf7fb527cfd43049e30a6238ef02c94484041fa4461ed41acbc6435425d6 F ext/expert/sqlite3expert.c 546010043fbec93544f762de5161b3d553165859e6bd853c4b85c05f93484260 F ext/expert/sqlite3expert.h ca81efc2679a92373a13a3e76a6138d0310e32be53d6c3bfaedabd158ea8969b F ext/expert/test_expert.c c395134bd6d4efa594a7d26578a1cb624c4027b79b4b5fcd44736c5ef1f5f725 @@ -79,10 +79,10 @@ F ext/fts3/README.content b9078d0843a094d86af0d48dffbff13c906702b4c3558012e67b9c F ext/fts3/README.syntax b72477722e9b4fe43f8403227d790a1c94221bfad15c27863a4b36d1052e892b F ext/fts3/README.tokenizers b92bdeb8b46503f0dd301d364efc5ef59ef9fa8e2758b8e742f39fa93a2e422d F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 4f02858ab845a97bedf06e6cc1fba49a81fe5e00a26df68d0ad0f00a5814fa70 +F ext/fts3/fts3.c 6cc7bbc307f27e7b6ee2e1d5ff63ffff4df3b42529dfe00eb34ddded417961b3 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h ed9b8bc5ed5be402069651e49d4855cb849af706cf3fe68548f58a2c21eefc7f -F ext/fts3/fts3_aux.c 7eab82a9cf0830f6551ba3abfdbe73ed39e322a4d3940ee82fbf723674ecd9f3 +F ext/fts3/fts3Int.h 277f32f304e82f4397fc2a74793c0a95318b7abb9670b519e4805a00946cbd9b +F ext/fts3/fts3_aux.c 37ba1b10bbd163ddc6b05dbc3a306a0d5d68aee3d30f33b4dc5794955cc50887 F ext/fts3/fts3_expr.c 5c13796638d8192c388777166075cdc8bc4b6712024cd5b72c31acdbefce5984 F ext/fts3/fts3_hash.c d9dba473741445789330c7513d4f65737c92df23c3212784312931641814672a F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf @@ -97,7 +97,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c c1de4ae28356ad98ccb8b2e3388a7fdcce7607b5523738c9afb6275dab765154 F ext/fts3/fts3_unicode.c de426ff05c1c2e7bce161cf6b706638419c3a1d9c2667de9cb9dc0458c18e226 F ext/fts3/fts3_unicode2.c 416eb7e1e81142703520d284b768ca2751d40e31fa912cae24ba74860532bf0f -F ext/fts3/fts3_write.c 2bee1c5828f6401adffd07ca64260aeb79d64138958273a56de8fa5e8759a0c1 +F ext/fts3/fts3_write.c d218b687fb55bce8c9340c6dbb368a10d94647cbe39801d85492d576a4e7da75 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73 F ext/fts3/tool/fts3view.c 413c346399159df81f86c4928b7c4a455caab73bfbc8cd68f950f632e5751674 @@ -108,21 +108,21 @@ F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a0 F ext/fts5/extract_api_docs.tcl 009cf59c77afa86d137b0cca3e3b1a5efbe2264faa2df233f9a7aa8563926d15 F ext/fts5/fts5.h ff5d3cc88b29e41612bfb29eb723e29e38973de62ca75ba3e8f94ccb67f5b5f2 F ext/fts5/fts5Int.h 8d98f8e180fe28d6067e240ed45b9011735d29d5cfb5bac194e1e376baa7c708 -F ext/fts5/fts5_aux.c da4a7a9a11ec15c6df0699d908915a209bcde48f0b04101461316b59f71abffb -F ext/fts5/fts5_buffer.c f1e6d0324d7c55329d340673befc26681a372a4d36086caa8d1ec7d7c53066c7 -F ext/fts5/fts5_config.c e7d8dd062b44a66cd77e5a0f74f23a2354cd1f3f8575afb967b2773c3384f7f8 -F ext/fts5/fts5_expr.c b8c32da1127bafaf10d6b4768b0dcb92285798524bed2d87a8686f99a8e8d259 -F ext/fts5/fts5_hash.c a6266cedd801ab7964fa9e74ebcdda6d30ec6a96107fa24148ec6b7b5b80f6e0 -F ext/fts5/fts5_index.c f0562b4adb9dc2d56addcb8833edab50817725032b1cbd46335c0b32d7f1525d -F ext/fts5/fts5_main.c 42025174a556257287071e90516d3ab8115daf1dd525a301883544469a260014 +F ext/fts5/fts5_aux.c 042da27e97d38071312c111cf18f3cb7983b75ba5e724aa1c3164e61e90f428a +F ext/fts5/fts5_buffer.c dcc3f0352339fe79c9d8abbc1c2009bc3469206467880bf43558447ef4f846fb +F ext/fts5/fts5_config.c bfba970fe1e4eed18ee57c8d51458e226db9a960ddf775c5e50e3d76603a667e +F ext/fts5/fts5_expr.c 71d48e8cf0358deace4949276647d317ff7665db6db09f40b81e2e7fe6664c7c +F ext/fts5/fts5_hash.c d5871df92ce3fa210a650cf419ee916b87c29977e86084d06612edf772bff6f5 +F ext/fts5/fts5_index.c f8cfa37bb7397e5ede20242e4c9cb030bc8b4584ce3f23a5e2495038c0ae64bd +F ext/fts5/fts5_main.c b0fed47b3b4420ba6810373480a75bc28a9c0b7d16478d19a396436fb3ff17d7 F ext/fts5/fts5_storage.c 19bc7c4cbe1e6a2dd9849ef7d84b5ca1fcbf194cefc3e386b901e00e08bf05c2 -F ext/fts5/fts5_tcl.c 7fb5a3d3404099075aaa2457307cb459bbc257c0de3dbd52b1e80a5b503e0329 +F ext/fts5/fts5_tcl.c 2be6cc14f9448f720fd4418339cd202961a0801ea9424cb3d9de946f8f5a051c F ext/fts5/fts5_test_mi.c 4308d5658cb1f5eee5998dcbaac7d5bdf7a2ef43c8192ca6e0c843f856ccee26 -F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b -F ext/fts5/fts5_tokenize.c 49aea8cc400a690a6c4f83c4cedc67f4f8830c6789c4ee343404f62bcaebca7b +F ext/fts5/fts5_test_tok.c 6021033bd4f4feffe8579efb6e1f58156ed462256bf99a2acdbd629246529204 +F ext/fts5/fts5_tokenize.c cfc16dde905552fe238c0403670852e75c0330ba508a9fb4836c1f596618561d F ext/fts5/fts5_unicode2.c 536a6dae41d16edadd6a6b58c56e2ebbb133f0dfe757562a2edbcdc9b8362e50 F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d09885d1dadf80 -F ext/fts5/fts5_vocab.c 23e263ad94ac357cfffd19bd7e001c3f15c4420fb10fa35b5993142127e780e6 +F ext/fts5/fts5_vocab.c bebee4aabcd056a44b3731166433cfdecf17ece750c08cb58733216222bd39e2 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl c5aa7cf7148b6dcffb5b61520ae18212baf169936af734ab265143f59db328fe @@ -168,7 +168,7 @@ F ext/fts5/test/fts5corrupt5.test 73985d4fe6d8f0d5d5c7bcf79ae7c6522c376cd6ad710a F ext/fts5/test/fts5corrupt6.test 2d72db743db7b5d9c9a6d0cfef24d799ed1aa5e8192b66c40e871a37ed9eed06 F ext/fts5/test/fts5corrupt7.test 814aab492d7a09abb5bfdd81cc66fc206d7f3868f9a3bae91876e02efc466fb3 F ext/fts5/test/fts5corrupt8.test 0b10750caf8aa23fa1c379ca4caf6130d41454505e4d5315590f4061eedcbe44 -F ext/fts5/test/fts5corrupt9.test 65a3a64f0e930fcc37e0389ae26224a4faba3a2c938d2f8eb4811a45b5b495fe +F ext/fts5/test/fts5corrupt9.test 4253b9b59f33effac8b67da72ec34309c738aca2d5e8e2656bfbbd6a489a1dfe F ext/fts5/test/fts5corruptbig.test 9f95b40fa36e292feceab02b2ef06e21878bfa1ac7afefa138aae05518b51774 F ext/fts5/test/fts5delete.test 2a5008f8b1174ef41d1974e606928c20e4f9da77d9f8347aed818994d89cced4 F ext/fts5/test/fts5detail.test 54015e9c43ec4ba542cfb93268abdf280e0300f350efd08ee411284b03595cc4 @@ -199,10 +199,10 @@ F ext/fts5/test/fts5first.test bfd685b96905bf541d99d8644e0a7219d1d833455a08ab64e F ext/fts5/test/fts5full.test 97d263c1072f4a560929cca31e70f65d2ae232610e17e6affcf7e979df59547b F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e F ext/fts5/test/fts5hash.test fd3e0367fbf0b0944d6936fdb22696350f57b9871069c6766251578a103e8a14 -F ext/fts5/test/fts5integrity.test c423ce16fd1ccadcac7fc22f794226b2bb00f5a187c0ab1d9f8502521b1bae05 +F ext/fts5/test/fts5integrity.test 613efcebe16b2d7a4096f03bcfb164f79a000b3354420ceda4a6f3e035090789 F ext/fts5/test/fts5integrity2.test 4c3636615c0201232c44a8105d5cb14fd5499fd0ee3014d7ffd7e83aac76ece8 -F ext/fts5/test/fts5interrupt.test 20d04204d3e341b104c0c24a41596b6393a3a81eba1044c168db0e106f9ac92c -F ext/fts5/test/fts5join.test 48b7ed36956948c5b8456c8bcaa5b087808d99000675918a43c4f51a925f1514 +F ext/fts5/test/fts5interrupt.test af7834ac6c2e71c05aea42d92f272eef3655e89b7a14a5620a2cd9de35e2e8ea +F ext/fts5/test/fts5join.test 3791e30d034050281191cd8141d969849f6c060135f1e3938fea3eea955f9da9 F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1 F ext/fts5/test/fts5leftjoin.test 1c14b51f4d1344a89e488160882f05a2246dd7e70c5cf077c8fb473e03c66338 F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c @@ -271,6 +271,7 @@ F ext/fts5/test/fts5update2.test c5baa76799ac605ebb8e5e21035db2014b396cef25c903e F ext/fts5/test/fts5version.test 44ab35566267b7618c090443de2d9ad84f633df5d20bf72e9bad199ae5fced84 F ext/fts5/test/fts5vocab.test 2a2bdb60d0998fa3124d541b6d30b019504918dc43a6584645b63a24be72f992 F ext/fts5/test/fts5vocab2.test 4265137a3747b27deb1e2e2bde5654120c6de72bfed3238e67806d85af60fc4c +F ext/fts5/tool/fts5cost.tcl 188a802e69422619c526698b92f0e5935f7d00b964e155bf4d5b4d4094989f60 F ext/fts5/tool/fts5speed.tcl b0056f91a55b2d1a3684ec05729de92b042e2f85 F ext/fts5/tool/fts5txt2db.tcl c0d43c8590656f8240e622b00957b3a0facc49482411a9fdc2870b45c0c82f9f F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093 @@ -285,14 +286,14 @@ F ext/intck/intck_common.tcl a61fd2697ae55b0a3d89847ca0b590c6e0d8ff64bebb70920d9 F ext/intck/intckbusy.test d5ed4ef85a4b1dc1dee2484bd14a4bb68529659cca743327df0c775f005fa387 F ext/intck/intckcorrupt.test f6c302792326fb3db9dcfc70b554c55369bc4b52882eaaf039cfe0b74c821029 F ext/intck/intckfault.test cff3f75dff74abb3edfcb13f6aa53f6436746ab64b09fe5e2028f051e985efab -F ext/intck/sqlite3intck.c b1c8a86f90fc00741d13314db9c58f7e2f92d1d19c5ad1c6904ec83a6bbd5c96 +F ext/intck/sqlite3intck.c 3c4a166645a1624731f63acd342e24e81e4ffd497116d94a427d72e6cc6caa69 F ext/intck/sqlite3intck.h 2b40c38e7063ab822c974c0bd4aed97dabb579ccfe2e180a4639bb3bbef0f1c9 F ext/intck/test_intck.c 4f9eaadaedccb9df1d26ba41116a0a8e5b0c5556dc3098c8ff68633adcccdea8 F ext/jni/GNUmakefile 8a94e3a1953b88cf117fb2a5380480feada8b4f5316f02572cab425030a720b4 -F ext/jni/README.md e3fbd47c774683539b7fdc95a667eb9cd6e64d8510f3ee327e7fa0c61c8aa787 +F ext/jni/README.md 1479c83dbe26125264a060ee6873531795a7082dbc0d43c4067683371331559f F ext/jni/jar-dist.make 030aaa4ae71dd86e4ec5e7c1e6cd86f9dfa47c4592c070d2e35157e42498e1fa -F ext/jni/src/c/sqlite3-jni.c 3d84a0176af779737ae977ba1c90d2ffe2537b8299c5d9f6552620493f12ac4b -F ext/jni/src/c/sqlite3-jni.h ac180ba9b21978727006c790d3006a95a2402a4c3ec7a0def92707ed89b79945 +F ext/jni/src/c/sqlite3-jni.c 6ccc50b0e98b8ef8fd6a8b837223e1c8ebb92bfe6b976128cc757c26257d994a +F ext/jni/src/c/sqlite3-jni.h df43024cced914c49485633d0f90168689e70577b3b17b0ecbdaf16e4a417bff F ext/jni/src/org/sqlite/jni/annotation/Experimental.java 8603498634e41d0f7c70f661f64e05df64376562ea8f126829fd1e0cdd47e82b F ext/jni/src/org/sqlite/jni/annotation/NotNull.java be6cc3e8e114485822331630097cc0f816377e8503af2fc02f9305ff2b353917 F ext/jni/src/org/sqlite/jni/annotation/Nullable.java 56e3dee1f3f703a545dfdeddc1c3d64d1581172b1ad01ffcae95c18547fafd90 @@ -302,7 +303,7 @@ F ext/jni/src/org/sqlite/jni/capi/AggregateFunction.java 0e28a0df51368c7127e505f F ext/jni/src/org/sqlite/jni/capi/AuthorizerCallback.java c045a5b47e02bb5f1af91973814a905f12048c428a3504fbc5266d1c1be3de5a F ext/jni/src/org/sqlite/jni/capi/AutoExtensionCallback.java 74cc4998a73d6563542ecb90804a3c4f4e828cb4bd69e61226d1a51f4646e759 F ext/jni/src/org/sqlite/jni/capi/BusyHandlerCallback.java 7b8e19810c42b0ad21a04b5d8c804b32ee5905d137148703f16a75b612c380ca -F ext/jni/src/org/sqlite/jni/capi/CApi.java 3d275f5f4fbdbe4fff15f4d42cf5ff559f9a4897e7373fa99f3b1dc9cf7f849c +F ext/jni/src/org/sqlite/jni/capi/CApi.java a9701cbe736b8bef5bc72ae465be0250e137f67bdb5a3ab62497972b6f51572d F ext/jni/src/org/sqlite/jni/capi/CallbackProxy.java 1b3baf5b772f266e8baf8f35f0ddc5bd87fc8c4927ec69115c46fd6fba6701c3 F ext/jni/src/org/sqlite/jni/capi/CollationCallback.java e29bcfc540fdd343e2f5cca4d27235113f2886acb13380686756d5cabdfd065a F ext/jni/src/org/sqlite/jni/capi/CollationNeededCallback.java 5bfa226a8e7a92e804fd52d6e42b4c7b875fa7a94f8e2c330af8cc244a8920ab @@ -320,7 +321,7 @@ F ext/jni/src/org/sqlite/jni/capi/SQLFunction.java 0d1e9afc9ff8a2adb94a155b72385 F ext/jni/src/org/sqlite/jni/capi/SQLTester.java 3c0babc067d8560627a9ed1b07979f9d4393464e2282c2fca4832052e982c7bc F ext/jni/src/org/sqlite/jni/capi/ScalarFunction.java 93b9700fca4c68075ccab12fe0fbbc76c91cafc9f368e835b9bd7cd7732c8615 F ext/jni/src/org/sqlite/jni/capi/TableColumnMetadata.java 9133bb7685901d2edf07801191284975e33b5583ce09dce1c05202ff91e7bb99 -F ext/jni/src/org/sqlite/jni/capi/Tester1.java 4c3d16fdf6e979f839b2ecdb14d0a0c04bd3d0e41500fc9e8110b588883b140b +F ext/jni/src/org/sqlite/jni/capi/Tester1.java 378d142435d220b20b7ce7343c62a03e853bb8c51e80447ee0f8ac5c37362d9a F ext/jni/src/org/sqlite/jni/capi/TraceV2Callback.java 0a25e117a0daae3394a77f24713e36d7b44c67d6e6d30e9e1d56a63442eef723 F ext/jni/src/org/sqlite/jni/capi/UpdateHookCallback.java c8bdf7848e6599115d601bcc9427ff902cb33129b9be32870ac6808e04b6ae56 F ext/jni/src/org/sqlite/jni/capi/ValueHolder.java 2ce069f3e007fdbbe1f4e507a5a407fc9679da31a0aa40985e6317ed4d5ec7b5 @@ -347,7 +348,7 @@ F ext/jni/src/org/sqlite/jni/test-script-interpreter.md d7987b432870d23f7c72a780 F ext/jni/src/org/sqlite/jni/wrapper1/AggregateFunction.java d5c108b02afd3c63c9e5e53f71f85273c1bfdc461ae526e0a0bb2b25e4df6483 F ext/jni/src/org/sqlite/jni/wrapper1/ScalarFunction.java 326ffba29aab836a6ea189703c3d7fb573305fd93da2d14b0f9e9dcf314c8290 F ext/jni/src/org/sqlite/jni/wrapper1/SqlFunction.java e920f7a031e04975579240d4a07ac5e4a9d0f8de31b0aa7a4be753c98ae596c9 -F ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java c82bc00c1988f86246a89f721d3c41f0d952f33f934aa6677ec87f7ca42519a0 +F ext/jni/src/org/sqlite/jni/wrapper1/Sqlite.java ce8aef867e06a21685658eb0173768c355e9e1e2dfdc75f382643fa62c7ee779 F ext/jni/src/org/sqlite/jni/wrapper1/SqliteException.java 982538ddb4c0719ef87dfa664cd137b09890b546029a7477810bd64d4c47ee35 F ext/jni/src/org/sqlite/jni/wrapper1/Tester2.java 08f92d52be2cec28a7b4555479cc54b7ebeeb94985256144eeb727154ec3f85b F ext/jni/src/org/sqlite/jni/wrapper1/ValueHolder.java a84e90c43724a69c2ecebd601bc8e5139f869b7d08cb705c77ef757dacdd0593 @@ -356,28 +357,28 @@ F ext/jni/src/tests/000-000-sanity.test c3427a0e0ac84d7cbe4c95fdc1cd4b61f9ddcf43 F ext/jni/src/tests/000-001-ignored.test e17e874c6ab3c437f1293d88093cf06286083b65bf162317f91bbfd92f961b70 F ext/jni/src/tests/900-001-fts.test bf0ce17a8d082773450e91f2388f5bbb2dfa316d0b676c313c637a91198090f0 F ext/misc/README.md 6243cdc4d7eb791c41ef0716f3980b8b5f6aa8c61ff76a3958cbf0031c6ebfa7 -F ext/misc/amatch.c 0e0124c1e03ee4cb99b25969f6b7b39c53a847b8bf12279efbcb896b0df1059a +F ext/misc/amatch.c 8d237cc014b3736922c26a76a451050d244aa4980c47c531f368f817b1e77b49 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 -F ext/misc/base64.c 8dc0a08cee11722822858a62625f1b63e5d5f1adac1cf4492d5732b571e37aa0 -F ext/misc/base85.c ff54cc676c6ec86231f75ecc86ea45416fcb69751dfb79690d5f5da5f7d39867 +F ext/misc/base64.c 1445761667c16356e827fc6418294c869468be934429aaa8315035e76dd58acf +F ext/misc/base85.c c713c7c81b05558c488cbdd8c68d612d5a97f9f0b8adf1167148dd7c89ce3f96 F ext/misc/basexx.c 89ad6b76558efbceb627afd5e2ef1d84b2e96d9aaf9b7ecb20e3d00b51be6fcf F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a -F ext/misc/btreeinfo.c 8f5e6da2c82ec2f06ee0216e922370a436dafdbb06ffa7a552203515ff9e7ddf +F ext/misc/btreeinfo.c 13bc9e9f1c13cde370d0e4a6a2683e9f1926a4cead7fb72c71871b11a06d78a1 F ext/misc/cksumvfs.c 9d7d0cf1a8893ac5d48922bfe9f3f217b4a61a6265f559263a02bb2001259913 -F ext/misc/closure.c 5559daf1daf742228431db929d1aa86dd535a4224cc634a81d2fd0d1e6ad7839 -F ext/misc/completion.c c27b64fdd0943c1b7f152376599814cee2641f7d67a7bb9bd2b957c2a64a5591 +F ext/misc/closure.c c983987a8d7846c3e52b1885ed3e20af7d4ca52a81a8f94ec6d1cd68f93acc86 +F ext/misc/completion.c 3f5db28e88c3313103b2dd86d910a2944fd500c46754e473493968ce81e994a4 F ext/misc/compress.c 8191118b9b73e7796c961790db62d35d9b0fb724b045e005a5713dc9e0795565 -F ext/misc/csv.c d9dab032420d81cdf0abc4b8523711d20a355704f82d958f963c86be48387dd9 +F ext/misc/csv.c 5e9d4dd749e762c144104c0f01db5bf4458735b19081ebe481a64e589a66687a F ext/misc/dbdump.c 678f1b9ae2317b4473f65d03132a2482c3f4b08920799ed80feedd2941a06680 -F ext/misc/decimal.c d4883de142f6dcd36eda23da40b55e2b51374e7b01eb54a7173940191389fc5e +F ext/misc/decimal.c 23698283d9365ce66d54b5bb97c01e69b4aa7ac804f226f9117a0d42efd15a65 F ext/misc/eval.c 04bc9aada78c888394204b4ed996ab834b99726fb59603b0ee3ed6e049755dc1 -F ext/misc/explain.c 606100185fb90d6a1eade1ed0414d53503c86820d8956a06e3b0a56291894f2b -F ext/misc/fileio.c d80268a5328fe839062a9d3103ea0fc7cacc6d42605959275675cb37867c84f7 -F ext/misc/fossildelta.c 86dfa83f85f7ccd640591d8a5c6865346d0c2ee6a949d78591eceb892f1cbfec -F ext/misc/fuzzer.c 6b231352815304ba60d8e9ec2ee73d4918e74d9b76bda8940ba2b64e8777515e -F ext/misc/ieee754.c 176c061c94857b543313959289cb60cf777c999fd002f82b53d194b95e9f347a -F ext/misc/memstat.c 43705d795090efb78c85c736b89251e743c291e23daaa8382fe7a0df2c6a283d +F ext/misc/explain.c 9670c8ff7b255eea7845abc5123a4958e74016c16990b10497e56380f91704b9 +F ext/misc/fileio.c e72033f987894ed821bf324a426ed462743399c274f1290dcb646349b6d7548f +F ext/misc/fossildelta.c 40add35db7f355d29ae856fe09043e66802fceff6f2551baccb28d794cadbc77 +F ext/misc/fuzzer.c decaca5a3479dfba69576cd41d4e17161eaf154a5438e12d316bbc5853571802 +F ext/misc/ieee754.c 2901d08a586d00a1d3c0fd89e03c57ee9e2b5f013b0daab9e49c7a48a9d5946b +F ext/misc/memstat.c 03ab52d2d841eb3f55118105c1964d5225f152b23bd708844c648b48d14ccbcf F ext/misc/memtrace.c 7c0d115d2ef716ad0ba632c91e05bd119cb16c1aedf3bec9f06196ead2d5537b F ext/misc/mmapwarm.c a81af4aaec00f24f308e2f4c19bf1d88f3ac3ce848c36daa7a4cd38145c4080d F ext/misc/nextchar.c 7877914c2a80c2f181dd04c3dbef550dfb54c93495dc03da2403b5dd58f34edd @@ -385,42 +386,47 @@ F ext/misc/noop.c f1a21cc9b7a4e667e5c8458d80ba680b8bd4315a003f256006046879f679c5 F ext/misc/normalize.c fbb144a861809686ff2b5b6eee8bb2e1207f9bf13ce7376e5273c700a1eafbd5 F ext/misc/pcachetrace.c f4227ce03fb16aa8d6f321b72dd051097419d7a028a9853af048bee7645cb405 F ext/misc/percentile.c 72e05a21db20a2fa85264b99515941f00ae698824c9db82d7edfbb16cea8ec80 -F ext/misc/prefixes.c 82645f79229877afab08c8b08ca1e7fa31921280906b90a61c294e4f540cd2a6 -F ext/misc/qpvtab.c fc189e127f68f791af90a487f4460ec91539a716daf45a0c357e963fd47cc06c +F ext/misc/prefixes.c aa15fd268e7d1336d1a1d4bc79265860d1d2ad68bba4cbcab61f83e4989dc68c +F ext/misc/qpvtab.c 470a5fffba005c8e1994209e59c1848122351e19522de71beb68d666c4fa39a5 F ext/misc/randomjson.c ef835fc64289e76ac4873b85fe12f9463a036168d7683cf2b773e36e6262c4ed -F ext/misc/regexp.c f1f7cfe90fc027b33d2b5ae7d6235eecce69c3aca71c9afce56fec62342c8b44 +F ext/misc/regexp.c 9dada9e9aa91f0cc23e35429e5d1111f110cc201b4c8dcc49aa6d2fc4b2a865d F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c -F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 -F ext/misc/series.c 22c6d8f00cc1b5089b1b37392e9097e9df9a5db53be86daf9a7669d95bb179f4 -F ext/misc/sha1.c cb5002148c2661b5946f34561701e9105e9d339b713ec8ac057fd888b196dcb9 +F ext/misc/scrub.c df54e202887e480bf9cd73168c9ac829e3b0211381b68b9809e0cb5bc1bdc2cf +F ext/misc/series.c 9454c7ef27d3729432496f31caabc892aeee1c21377d403f681126d93c09d6e9 +F ext/misc/sha1.c 8bf60344c11a525384c2efd1ae77f160b06be336db679effaadf292d4b41451c F ext/misc/shathree.c fd22d70620f86a0467acfdd3acd8435d5cb54eb1e2d9ff36ae44e389826993df F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c 693c8fd3293087fa821322967a97e59dfa24051e5d2ca7fa85790a4034db6fa4 -F ext/misc/sqlar.c a6175790482328171da47095f87608b48a476d4fac78d8a9ff18b03a2454f634 -F ext/misc/sqlite3_stdio.c 0fe5a45bd332b30aef2b68c64edbe69e31e9c42365b0fa79ce95a034bca6fbb0 -F ext/misc/sqlite3_stdio.h f05eaf5e0258f0573910324a789a9586fc360a57678c57a6d63cfaa2245b6176 +F ext/misc/sqlar.c 97c100b010159c08a7a9acd8eb1ea510a5522e64741aaafcd7b6c629de682edc +F ext/misc/sqlite3_stdio.c b43a0f530c6f0fb3d41d9af8c0b40f3f71198a1db55ab8ffffbef5c8cc329d22 +F ext/misc/sqlite3_stdio.h 27a4ecea47e61bc9574ccdf2806f468afe23af2f95028c9b689bfa08ab1ce99f F ext/misc/stmt.c b090086cd6bd6281c21271d38d576eeffe662f0e6b67536352ce32bbaa438321 -F ext/misc/stmtrand.c 59cffa5d8e158943ff1ce078956d8e208e8c04e67307e8f249dece2436dcb7fc -F ext/misc/templatevtab.c 10f15b165b95423ddef593bc5dcb915ec4eb5e0f1066d585e5435a368b8bc22b +F ext/misc/stmtrand.c 1c7c6a478e9c808d53a2efa9a744c86c265b6b95b4a96964e1b960c7c642ad49 +F ext/misc/templatevtab.c f2771161158bb78a0bfeaed9b50a018a731c7566df6a67181df0acea1920ab9c +F ext/misc/tmstmpvfs.c 240caad4441328dc52bd2871f48811db46dff858d5598030e389176837a2f4df F ext/misc/totype.c ba11aac3c0b52c685bd25aa4e0f80c41c624fb1cc5ab763250e09ddc762bc3a8 F ext/misc/uint.c 327afc166058acf566f33a15bf47c869d2d3564612644d9ff81a23efc8b36039 F ext/misc/unionvtab.c 716d385256d5fb4beea31b0efede640807e423e85c9784d21d22f0cce010a785 F ext/misc/urifuncs.c f71360d14fa9e7626b563f1f781c6148109462741c5235ac63ae0f8917b9c751 F ext/misc/uuid.c 5bb2264c1b64d163efa46509544fd7500cb8769cb7c16dd52052da8d961505cf F ext/misc/vfslog.c 3932ab932eeb2601dbc4447cb14d445aaa9fbe43b863ef5f014401c3420afd20 -F ext/misc/vfsstat.c 0b23c0a69a2b63dc0ef0af44f9c1fc977300c480a1f7a9814500369d8211f56e -F ext/misc/vfstrace.c 0e4b8b17ac0675ea90f6d168d8214687e06ca3efbc0060aad4814994d82b41fb -F ext/misc/vtablog.c 2d04386c2f5a3bb93bc9ae978f0b7dcd5a264e126abd640dd6d82aa9067cbd48 -F ext/misc/vtshim.c e5bce24ab8c532f4fdc600148718fe1802cb6ed57417f1c1032d8961f72b0e8f -F ext/misc/wholenumber.c 0fa0c082676b7868bf2fa918e911133f2b349bcdceabd1198bba5f65b4fc0668 +F ext/misc/vfsstat.c 2b21efa93062ce814fbe28e6dff2acfafa4073a14b8d02cacfb4da1d604d05a5 +F ext/misc/vfstrace.c fc8c393a8316a8c20867b7e6e92908c4f81060c9e1f24d6ad9aefdc91c01dd13 +F ext/misc/vtablog.c 6c0c11c4822ab6c1a205718ea7c6d1bb561d96b27104b9c1fe84d01aa62d6c9c +F ext/misc/vtshim.c f5ab480d1e33fa46a0b138359bedc9979e32798d72348e04bbe6093f9ae95c7b +F ext/misc/wholenumber.c aa5e6d786fe8d79bc100ea0e852249c026a91ae65a5c1bcb2b869cd1a7cdd6d5 F ext/misc/windirent.h 02211ce51f3034c675f2dbf4d228194d51b3ee05734678bad5106fff6292e60c -F ext/misc/zipfile.c 837591f0505d21f7f7937ea046c9b0fc594f7fa3ca00c2bd54ffa1c94bfccd63 +F ext/misc/zipfile.c 5a583b5e72b4d777dc9f845529e6bd185d58024b633aafc93588679c78719700 F ext/misc/zorder.c bddff2e1b9661a90c95c2a9a9c7ecd8908afab5763256294dd12d609d4664eee +F ext/qrf/README.md 9e644615d7d7b77ef7e9db798765679e50c5ed12eda48bce21c9ef9eb4715e9d +F ext/qrf/dev-notes.md e68a6d91ce4c7eb296ef2daadc2bb79c95c317ad15b9fafe40850c67b29c2430 +F ext/qrf/qrf.c 9bef1f01e0c33a8693a9a7c1b666b8dee1aa60b8bc880d562eb4dfff551a6009 +F ext/qrf/qrf.h fbb223ff5789b324b3e9c22e787e4c1f53e217cff7cc5a243164d4b2e8410f4b F ext/rbu/rbu.c 801450b24eaf14440d8fd20385aacc751d5c9d6123398df41b1b5aa804bf4ce8 F ext/rbu/rbu1.test 25870dd7db7eb5597e2b4d6e29e7a7e095abf332660f67d89959552ce8f8f255 F ext/rbu/rbu10.test 7c22caa32c2ff26983ca8320779a31495a6555737684af7aba3daaf762ef3363 -F ext/rbu/rbu11.test 8584f80ef4be00e6beec4154f638847ffc40b5f2832ffadfbaf558ae40e50cb5 +F ext/rbu/rbu11.test 3b71377c018b05dd39c30ea2feb272a087eb0faeff1b7b4511144f87219c478e F ext/rbu/rbu12.test ec63aa7bfc3c65c1d774bf4357ed731723827d211d9d7cb0efa171bbaeeebaf4 F ext/rbu/rbu13.test 658edbc3325d79252a98b761fde95460e439f80e820ff29e10261e25f870b3b6 F ext/rbu/rbu14.test 05dac607a62f62102f4db92135979a8a4501143638060019aca08c753822cf39 @@ -461,7 +467,7 @@ F ext/rbu/rbuvacuum.test 542561741ff2b262e3694bc6012b44694ee62c545845319a06f3237 F ext/rbu/rbuvacuum2.test 1a9bd41f127be2826de2a65204df9118525a8af8d16e61e6bc63ba3ac0010a23 F ext/rbu/rbuvacuum3.test 3ce42695fdf21aaa3499e857d7d4253bc499ad759bcd6c9362042c13cd37d8de F ext/rbu/rbuvacuum4.test ffccd22f67e2d0b380d2889685742159dfe0d19a3880ca3d2d1d69eefaebb205 -F ext/rbu/sqlite3rbu.c 3fb2390575b261c365d3f6fea61ff15e74d5d89e373f2a2bfa4d80c24321e793 +F ext/rbu/sqlite3rbu.c e99400d29d029936075e27495b269a2dcdceae3cf8c86b1d0869b4af487be3ab F ext/rbu/sqlite3rbu.h e3a5bf21e09ca93ce4e8740e00d6a853e90a697968ec0ea98f40826938bdb68e F ext/rbu/test_rbu.c 8b6e64e486c28c41ef29f6f4ea6be7b3091958987812784904f5e903f6b56418 F ext/recover/dbdata.c 10d3c56968a9af6853722a47280805ad1564714d79ea45ac6f7da14bb57fd137 @@ -483,18 +489,9 @@ F ext/recover/recoversql.test e66d01f95302a223bcd3fd42b5ee58dc2b53d70afa90b0d00e F ext/recover/sqlite3recover.c 56c216332ea91233d6d820d429f3384adbec9ecedda67aa98186b691d427cc57 F ext/recover/sqlite3recover.h 011c799f02deb70ab685916f6f538e6bb32c4e0025e79bfd0e24ff9c74820959 F ext/recover/test_recover.c 3d0fb1df7823f5bc22a0b93955034d16a2dfa2eb1e443e9a0123a77f120599a3 -F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 -F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 -F ext/repair/checkindex.c 7639b4f8928f82c10b950169e60cc45a7f6798df0b299771d17bef025736f657 -F ext/repair/sqlite3_checker.c.in 445118c5f7fea958b36fba1b2c464283e60ed4842039ddee3265f1698115ebf7 -F ext/repair/sqlite3_checker.tcl a9a2caa9660567257c177a91124d8c0dccdfa341e25c51e6da7f1fd9e601eafa -F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e -F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc78249442da72ff3f8297398a69 -F ext/repair/test/checkindex01.test b530f141413b587c9eb78ff734de6bb79bc3515c335096108c12c01bddbadcec -F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c F ext/rtree/README 734aa36238bcd2dee91db5dba107d5fcbdb02396612811377a8ad50f1272b1c1 -F ext/rtree/geopoly.c f0573d5109fdc658a180db0db6eec86ab2a1cf5ce58ec66cbf3356167ea757eb -F ext/rtree/rtree.c 9331997a76b88a9bc04e156bdfd6e2fe35c0aa93bc338ebc6aa0ae470fe4a852 +F ext/rtree/geopoly.c bd1971479184d559499ff3087c37f2823977d7b0ec80916141ae66f70345c88d +F ext/rtree/rtree.c 44abdd5df278ca1901daf29c82cce6785f0ee82ce59e28160ee988c17a9a185b F ext/rtree/rtree.h 4a690463901cb5e6127cf05eb8e642f127012fd5003830dbc974eca5802d9412 F ext/rtree/rtree1.test e0608db762b2aadca0ecb6f97396cf66244490adc3ba88f2a292b27be3e1da3e F ext/rtree/rtree2.test 9d9deddbb16fd0c30c36e6b4fdc3ee3132d765567f0f9432ee71e1303d32603d @@ -536,14 +533,14 @@ F ext/session/changesetfuzz1.test 15b629004e58d5ffcc852e6842a603775bb64b1ce51254 F ext/session/session1.test 5d2502922d38a1579076863827342379a1609ca6bae78c40691a2be1ed1be2aa x F ext/session/session2.test ee83bb973b9ce17ccce4db931cdcdae65eb40bbb22089b2fe6aa4f6be3b9303f F ext/session/session3.test 2cc1629cfb880243aec1a7251145e07b78411d851b39b2aa1390704550db8e6a -F ext/session/session4.test 823f6f018fcbb8dacf61e2960f8b3b848d492b094f8b495eae1d9407d9ab7219 +F ext/session/session4.test ad0ddaaddb9a99dac433d83fc6674aae2af072b8f57e63a6b3f2d76f1a66e98c F ext/session/session5.test 716bc6fafd625ce60dfa62ae128971628c1a1169 F ext/session/session6.test 35279f2ec45448cd2e24a61688219dc6cf7871757716063acf4a8b5455e1e926 F ext/session/session8.test 326f3273abf9d5d2d7d559eee8f5994c4ea74a5d935562454605e6607ee29904 F ext/session/session9.test 0c4a8fbe7a5031f50855f020f3408e1f07fd7859f1daa1629eadcec3422072d6 F ext/session/sessionA.test 1feeab0b8e03527f08f2f1defb442da25480138f F ext/session/sessionB.test c4fb7f8a688787111606e123a555f18ee04f65bb9f2a4bb2aa71d55ce4e6d02c -F ext/session/sessionC.test f8a5508bc059ae646e5ec9bdbca66ad24bc92fe99fda5790ac57e1f59fce2fdf +F ext/session/sessionC.test 2bd42225efdf5f5b1a20f75b672665bcd4f67e2a6d7ddf7420fe7bf523ba41f8 F ext/session/sessionD.test 470ff917dc849e2eb78142ade63aaabd729d773833cff0ff01bca0eda68a21ce F ext/session/sessionE.test b2010949c9d7415306f64e3c2072ddabc4b8250c98478d3c0c4d064bce83111d F ext/session/sessionF.test d37ed800881e742c208df443537bf29aa49fd56eac520d0f0c6df3e6320f3401 @@ -558,11 +555,12 @@ F ext/session/sessionat.test 00c8badb35e43a2f12a716d2734a44d614ff62361979b6b8541 F ext/session/sessionbig.test 47c381e7acfabeef17d98519a3080d69151723354d220afa2053852182ca7adf F ext/session/sessionblob.test 87faf667870b72f08e91969abd9f52a383ab7b514506ee194d64a39d8faff00a F ext/session/sessionchange.test 6618cb1c1338a4b6df173b6ac42d09623fb71269962abf23ebb7617fe9f45a50 +F ext/session/sessionchange2.test 6b5b7e3d1cc4ede43817f7fb68e3771aac4aa6500adb21a458b3e5a9fd841f83 F ext/session/sessionconflict.test 19e4a53795c4c930bfec49e809311e09b2a9e202d9446e56d7a8b139046a0c07 x F ext/session/sessiondiff.test e89f7aedcdd89e5ebac3a455224eb553a171e9586fc3e1e6a7b3388d2648ba8d F ext/session/sessionfault.test c2b43d01213b389a3f518e90775fca2120812ba51e50444c4066962263e45c11 F ext/session/sessionfault2.test b0d6a7c1d7398a7e800d84657404909c7d385965ea8576dc79ed344c46fbf41c -F ext/session/sessionfault3.test ce0b5d182133935c224d72507dbf1c5be1a1febf7e85d0b0fbd6d2f724b32b96 +F ext/session/sessionfault3.test 9397819ec25b0960c5bc03c78613f9cb5cacc970f83e817aec1775c2a839a787 F ext/session/sessioninvert.test 9018f6a7387ac745084b6374c5e1aa14d648b372e6e1181cfab3df632b662d26 x F ext/session/sessionmem.test f2a735db84a3e9e19f571033b725b0b2daf847f3f28b1da55a0c1a4e74f1de09 F ext/session/sessionnoact.test 4c7ae5c7d351cb5323bca62b6b095592ad24bd90a6713c178b62ab0063d23e19 @@ -573,11 +571,10 @@ F ext/session/sessionrowid.test 85187c2f1b38861a5844868126f69f9ec62223a03449a98a F ext/session/sessionsize.test 8fcf4685993c3dbaa46a24183940ab9f5aa9ed0d23e5fb63bfffbdb56134b795 F ext/session/sessionstat1.test 5e718d5888c0c49bbb33a7a4f816366db85f59f6a4f97544a806421b85dc2dec F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009ecef8511f4cf3fc -F ext/session/sqlite3session.c b3de195ce668cace9b324599bf6255a70290cbfb5451e826e946f3aee6e64c54 -F ext/session/sqlite3session.h 7404723606074fcb2afdc6b72c206072cdb2b7d8ba097ca1559174a80bc26f7a -F ext/session/test_session.c 8766b5973a6323934cb51248f621c3dc87ad2a98f023c3cc280d79e7d78d36fb -F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 3dc01e673c456d3b752674c9407276e8fef35dec1d304b3cc1de362f019b2a09 +F ext/session/sqlite3session.c d5c91d5b07d2b8e860f2782ae23f7b44ce929280e00645418ee84a0fd14525b2 +F ext/session/sqlite3session.h 063e7bf7be2fff874456f452a224b5b3013b25682d108933b0351c93a1279b9c +F ext/session/test_session.c 2a02a68b522e2f3d4a64b2a4733af54b0f3e500769aeccd5bcbdd440103db069 +F ext/wasm/GNUmakefile 68c750f173106d9d63f12c1edf1256c6f4bad9894b155da5db64322f4912de4b F ext/wasm/README-dist.txt f01081a850ce38a56706af6b481e3a7878e24e42b314cfcd4b129f0f8427066a F ext/wasm/README.md 2e87804e12c98f1d194b7a06162a88441d33bb443efcfe00dc6565a780d2f259 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff @@ -585,81 +582,82 @@ F ext/wasm/SQLTester/SQLTester.mjs 6b3c52ed36a5573ca4883176f326332a8d4c0cecf5efd F ext/wasm/SQLTester/SQLTester.run.mjs 57f2adb33f43f2784abbf8026c1bfd049d8013af1998e7dcb8b50c89ffc332e0 F ext/wasm/SQLTester/index.html 64f3435084c7d6139b08d1f2a713828a73f68de2ae6a3112cbb5980d991ba06f F ext/wasm/SQLTester/touint8array.c 2d5ece04ec1393a6a60c4bf96385bda5e1a10ad49f3038b96460fc5e5aa7e536 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core ef34398a903d0a2425fbbfbd4ed2cd596daea55b8515e2617c8dc7ad7c0767dd -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-extras 9eae68943ce91ab145892b31370819c2103525240eb72e0fce53c498b8d8275a -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b -F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 -F ext/wasm/api/README.md f4c0d67caaee21a77b8938c30b5f79667bfc9d0c95d01b51df77ea35ee773884 -F ext/wasm/api/extern-post-js.c-pp.js 205f55aacfc62c580985db5c790300779de3876a76a5c7e1bfb13e71c8b4506b +F ext/wasm/api/EXPORTED_FUNCTIONS.c-pp de2ce128aebeff9ef4161cff7a0ff0089be866121bcb8941ad44a38a65f1d436 +F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b47244781a7e81 +F ext/wasm/api/extern-post-js.c-pp.js 80accc53cc6ea1e61c721595f42ba95baa7c7ea636807d9507e69403301f8c54 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 -F ext/wasm/api/post-js-footer.js 5bd7170b5e8ce7b62102702bbcf47ef7b3b49cd56ed40c043fd990aa715b74ee -F ext/wasm/api/post-js-header.js 79d078aec33d93b640a19c574b504d88bb2446432f38e2fbb3bb8e36da436e70 -F ext/wasm/api/pre-js.c-pp.js a876c6399dff29b6fe9e434036beb89889164cc872334e184291723ecc7cb072 -F ext/wasm/api/sqlite3-api-cleanup.js a3d6b9e449aefbb8bba283c2ba9477e2333a0eeb94a7a26b5bf952736f65a6dd -F ext/wasm/api/sqlite3-api-glue.c-pp.js d2b8263b3ce0cefc6c5a68d0a4d448a9770eda4bf9d9ded9d7eb0198e4ce4da1 -F ext/wasm/api/sqlite3-api-oo1.c-pp.js c4260f3fdc553c56ee530c20cc1119029067b503f0d6d7b472705536cb45aa1d -F ext/wasm/api/sqlite3-api-prologue.js 307583ff39a978c897c4ef4ce53fe231dce5c73dc84785969c81c1ab5960a293 -F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 -F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966 +F ext/wasm/api/opfs-common-inline.c-pp.js 496ca858af09b7fef2efaece467960611d35f57254078424bcdeac42ded9e85d +F ext/wasm/api/opfs-common-shared.c-pp.js d0bff21d34fa05a7eb975c4ee608165f96387332101cb05e976953c77434374d +F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b +F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 +F ext/wasm/api/pre-js.c-pp.js d6bf82f83f60caa2904bddb95a29cb738b310f672d2796cdc5fe54463ab0d6cd +F ext/wasm/api/sqlite3-api-glue.c-pp.js 31a721ada7225838a61310a9f3f797fa5275353f8e9b0ae769d85b437be061f5 +F ext/wasm/api/sqlite3-api-oo1.c-pp.js 5f203f5bb5d48a9e43ec51e791dc411c24dca825842bfb6a2d979d871bf8cfaf +F ext/wasm/api/sqlite3-api-prologue.js 4336f02ac24ba58e68e355adb9da2804a1eef15a4aee961fa813c7d812a56b54 +F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1fa34e9b0e3b90a8898e4f700d7125e44c81877f182627bb8564b97989bc6e78 +F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 25e31482b04293a33d7599f1459eb552b3eb36ca10c02c816122d3308bf80cb2 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 26cb41d5a62f46a106b6371eb00fef02de3cdbfaa51338ba087a45f53028e0d0 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 418c33fe284739564daab3c7a7a88882fdd3c99137497900f98eddec1e409af5 -F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 9097074724172e31e56ce20ccd7482259cf72a76124213cbc9469d757676da86 -F ext/wasm/api/sqlite3-wasm.c dd7fc1d535281f0d5d2732bb1b662d1d403a762f07b63c2ea5663053377b2804 -F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bda1c75bd674a92a0e27cc2f3d46dbbf21e422413f8046814515a0bd7409328a -F ext/wasm/api/sqlite3-worker1.c-pp.js 802d69ead8c38dc1be52c83afbfc77e757da8a91a2e159e7ed3ecda8b8dba2e7 -F ext/wasm/c-pp-lite.c 8fa0148e73782a86274db688c4730e2962cd675af329490493adddaf3322f16f -F ext/wasm/common/SqliteTestUtil.js 7adaeffef757d8708418dc9190f72df22367b531831775804b31598b44f6aa51 +F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 27fb135ba3b805b66c90a7333b56080345bf1db79335c3e48b6d01ad7aa09607 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1c742ed92c0515c8ef97d2b132feb01168252bfd847fed5d52607d8a42d23c6a +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 3dbd918ef037cd8fa9c7b4dccb3c8637b228638654c429e7df6acab5981c75e2 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 3da8fe72dc9e76614a9c102b92e777ce03f81d788b607701c828d8fcbac44b06 +F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 +F ext/wasm/api/sqlite3-wasm.c ddf9d435b2e901eaceb805ff694e9609c2f32b5cf89a4f164e734a6fa303fdd2 +F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 46919740f8176c10f691431a81c769c35657e3b537e2ed2bde691f9017b0ee8a +F ext/wasm/api/sqlite3-worker1.c-pp.js 21b20d3b8d43471d1647123616a7e5435b1e4b6fa447714f91ffc7f950ea01e8 +F ext/wasm/common/SqliteTestUtil.js dae753b95e72248c4395d8de8359e0d055cd9928488e8dd84aef89e46d23b32e F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15 F ext/wasm/common/testing.css e97549bab24126c24e0daabfe2de9bb478fb0a69fdb2ddd0a73a992c091aad6f -F ext/wasm/common/whwasmutil.js 0d539324097fc83b953e9844267359ba0fd02286caa784ea2f597ced279ea640 +F ext/wasm/common/whwasmutil.js 831f07a0d9bb61713164871370811432e96d0f813806a4d2c783d3c77c2373a0 F ext/wasm/config.make.in c424ae1cc3c89274520ad312509d36c4daa34a3fce5d0c688e5f8f4365e1049a F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 F ext/wasm/demo-123.js c7b3cca50c55841c381a9ca4f9396e5bbdc6114273d0b10a43e378e32e7be5bf F ext/wasm/demo-jsstorage.html 409c4be4af5f207fb2877160724b91b33ea36a3cd8c204e8da1acb828ffe588e -F ext/wasm/demo-jsstorage.js 42131ddfa18e817d0e39ac63745e9ea31553980a5ebd2222e04d4fac60c19837 -F ext/wasm/demo-worker1-promiser.c-pp.html 635cf90685805e21772a5f7a35d1ace80f98a9ef7c42ff04d7a125ddca7e5db8 -F ext/wasm/demo-worker1-promiser.c-pp.js f40ec65810048e368896be71461028bd10de01e24277208c59266edf23bb9f52 +F ext/wasm/demo-jsstorage.js 467cb4126ff679ebcdb112d100d073af26b9808d0a0b52d66a40e28f59c5099b +F ext/wasm/demo-worker1-promiser.c-pp.html 38e917d4dafa75b4aa0e9e1663abe734c1f701f6da141318bd01ffe3f6e47a77 +F ext/wasm/demo-worker1-promiser.c-pp.js d210aa1e8a74ea6244fe2290de5710bb55b3a9ed681aaba04e38ceb66550f417 F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d -F ext/wasm/demo-worker1.js 08720227e98fa5b44761cf6e219269cee3e9dd0421d8d91459535da776950314 +F ext/wasm/demo-worker1.js fdfa90aa9d6b402bfed802cf1595fe4da6cc834ac38c8ff854bf1ee01f5ff9bb F ext/wasm/example_extra_init.c 2347cd69d19d839ef4e5e77b7855103a7fe3ef2af86f2e8c95839afd8b05862f -F ext/wasm/fiddle/fiddle-worker.js 7798af02e672e088ff192716f80626c8895e19301a65b8af6d5d12b2d13d2451 +F ext/wasm/fiddle/fiddle-worker.js 6c72acac2d381480bc9f5eb538e3f2faf2c1f72dd4fcbd05d3b409818a9a8fd5 F ext/wasm/fiddle/fiddle.js 84fd75967e0af8b69d3dd849818342227d0f81d13db92e0dcbc63649b31a4893 -F ext/wasm/fiddle/index.html a27b8127ef9ecf19612da93b2a6a73bdb3777b5c56b5450bb7200a94bc108ff9 +F ext/wasm/fiddle/index.c-pp.html 02f063ef30b8124f311029855c4439e77bc6505d1bf65a163d88c064a63ee9d9 F ext/wasm/index-dist.html db23748044e286773f2768eec287669501703b5d5f72755e8db73607dc54d290 -F ext/wasm/index.html 54e27db740695ab2cb296e02d42c4c66b3f11b65797340d19fa6590f5b287da1 -F ext/wasm/jaccwabyt/jaccwabyt.js bbac67bc7a79dca34afe6215fd16b27768d84e22273507206f888c117e2ede7d -F ext/wasm/jaccwabyt/jaccwabyt.md 167fc0b624c9bc2c477846e336de9403842d81b1a24fc4d3b24317cb9eba734f -F ext/wasm/mkdist.sh 64d53f469c823ed311f6696f69cec9093f745e467334b34f5ceabdf9de3c5b28 x -F ext/wasm/mkwasmbuilds.c 1b53c4d2a1350c19a96a8cdfbda6a39baea9d2142bfe0cbef0ccb0e898787f47 +F ext/wasm/index.html 5bf6cf1b0a3c8b9f5f54d77f2219d7ae87a15162055ce308109c49b1dcab4239 +F ext/wasm/jaccwabyt/jaccwabyt.js 4e2b797dc170851c9c530c3567679f4aa509eec0fab73b466d945b00b356574b +F ext/wasm/jaccwabyt/jaccwabyt.md 6aa90fa1a973d0ad10d077088bea163b241d8470c75eafdef87620a1de1dea41 +F ext/wasm/libcmpp.c 0e2cec7de7c9cbe7941e6a4dc0c123cec736578a3eeae7828ac18706acd3cea6 +F ext/wasm/mkdist.sh f8883b077a2ca47cf92e6f0ce305fbf72ca648c3501810125056c4b09c2d5554 x +F ext/wasm/mkwasmbuilds.c d4479af2774a43104b819d9286961b7c09793ca39440e20191e693ceff21f911 F ext/wasm/module-symbols.html e54f42112e0aac2a31f850ab33e7f2630a2ea4f63496f484a12469a2501e07e2 F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96 F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d F ext/wasm/speedtest1-wasmfs.mjs 60dd5842f6d2a70a6d0bef12633a11491bde6984aff75a37c2040980d8cbf36a F ext/wasm/speedtest1-worker.html 068d4190f304fa1c34e6501a1b3a4c32fe8d8dac93c2d0f53d667a1cb386eedc -F ext/wasm/speedtest1-worker.js 958a2d3c710bf8e82567277f656193a0248216db99a3c2c86966124b84309efb -F ext/wasm/speedtest1.html c90d63dfa795f0cb1ad188de587be9024b1ff73b4adc5fdf7efc0d781be94d03 +F ext/wasm/speedtest1-worker.js 8acad67bfd6aeeb799bd5ae007ea32af85a082a287d8877c5a10adf4bd7efd89 +F ext/wasm/speedtest1.html f32c66997eb0b036c4546e6302cd0673157912661df0b290ab65816f713feac6 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555e685bce3da8c3f F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c -F ext/wasm/tester1-worker.c-pp.html 883881eeac14eeeecc8ff22acf9fe0f18a97cacb48be08ebb0bae891ceded584 -F ext/wasm/tester1.c-pp.html 949920126dcf477925d8d540093d9cc374d3ab4c4ddee920c1dcadcf37917306 -F ext/wasm/tester1.c-pp.js 2b014884dadf28928fabcb688746ca87145673eef75e154486505a266203fc15 -F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e -F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 -F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 +F ext/wasm/tester1-worker.c-pp.html 7171022e7f4da8f46e5f50ea81dd6ce840b9235c47653a5deeb3764ccc2fe472 +F ext/wasm/tester1.c-pp.html bd927ccf51ddd65e924660a0487add99e1b044afe03950e49d87ccf44efdddb6 +F ext/wasm/tester1.c-pp.js 581be6f3228351bc77512ba743912d1ad52f40dde1fef1a943fd568b0cfc51b1 +F ext/wasm/tests/opfs/concurrency/index.html 706eab6308343c04ac2360aba6001af4ffaf46d8f33a0ccd02c64d93e3216a43 +F ext/wasm/tests/opfs/concurrency/test.js 6919778fceaac1b7cc78caf41d796f545d2c4433b31188aa9689f05b5ad28828 +F ext/wasm/tests/opfs/concurrency/worker.js 704d82c5e287e47f612349e027765943a58ad967dcf178fb5a1c3a8eaafb09af F ext/wasm/tests/opfs/sahpool/digest-worker.js b0ab6218588f1f0a6d15a363b493ceaf29bfb87804d9e0165915a9996377cf79 F ext/wasm/tests/opfs/sahpool/digest.html 206d08a34dc8bd570b2581d3d9ab3ecad3201b516a598dd096dcf3cf8cd81df8 F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca01385e2732294b53f4c842328 F ext/wasm/tests/opfs/sahpool/sahpool-pausing.js f264925cfc82155de38cecb3d204c36e0f6991460fff0cb7c15079454679a4e2 F ext/wasm/tests/opfs/sahpool/sahpool-worker.js bd25a43fc2ab2d1bafd8f2854ad3943ef673f7c3be03e95ecf1612ff6e8e2a61 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk 00dd631c66c1f7922b2d691631163899eff1c3d1da780ff37a62f8d997b368e1 +F main.mk bf77c630aeef4ed4b9270e1fd52c35147b529412c00b74ed6eb8aca3466763f6 +F make.bat a136fd0b1c93e89854a86d5f4edcf0386d211e5d5ec2434480f6eea436c7420c F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -669,38 +667,38 @@ F mptest/multiwrite01.test dab5c5f8f9534971efce679152c5146da265222d F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 1b9c24374a85dfc7eb8fa7c4266ee0db4f9609cceecfc5481cd8307e5af04366 F sqlite3.pc.in e6dee284fba59ef500092fdc1843df3be8433323a3733c91da96690a50a5b398 -F src/alter.c fc7bbbeb9e89c7124bf5772ce474b333b7bdc18d6e080763211a40fde69fb1da +F src/alter.c fc36b19273ffe364aeb4d00ba04bda8798ad7a67fec7a035ee8ee56272e1bdbe F src/analyze.c 03bcfc083fc0cccaa9ded93604e1d4244ea245c17285d463ef6a60425fcb247d -F src/attach.c 9af61b63b10ee702b1594ecd24fb8cea0839cfdb6addee52fba26fa879f5db9d -F src/auth.c 54ab9c6c5803b47c0d45b76ce27eff22a03b4b1f767c5945a3a4eb13aa4c78dc +F src/attach.c 7cf07d4fa42b9fc8662237c60c40b730326c30aa90ae5fffc0b18b2d726ebf61 +F src/auth.c ebec42df26b34a62b6750d30d9c2c03554a1c522020182476f7729a439fef04f F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 F src/bitvec.c e242d4496774dfc88fa278177dd23b607dce369ccafb3f61b41638eea2c9b399 F src/btmutex.c 30dada73a819a1ef5b7583786370dce1842e12e1ad941e4d05ac29695528daea -F src/btree.c 240fa5b4ced4733ac5882b43448f433a9742a76d9d7e28aa9ce48d13a38ceb5d +F src/btree.c fb350c445316c1cc0529703c0b76450770a1de0ab0440641a56b19f05d6fefbe F src/btree.h e823c46d87f63d904d735a24b76146d19f51f04445ea561f71cc3382fd1307f0 F src/btreeInt.h 9c0f9ea5c9b5f4dcaea18111d43efe95f2ac276cd86d770dce10fd99ccc93886 -F src/build.c 611e07299d72ff04bbcb9e7109183467e30925d203c3e121ef9bb3cf6876289b +F src/build.c 8581de0af3b6c448f5d64e2d18a91ac1e7057b3bcb8b8827e1240f80d87486a4 F src/callback.c 3605bbf02bd7ed46c79cd48346db4a32fc51d67624400539c0532f4eead804ad -F src/carray.c ff6081a31878fc34df8fa1052a9cbf17ddc22652544dcb3e2326886ed1053b55 +F src/carray.c 3efe3982d5fb323334c29328a4e189ccaef6b95612a6084ad5fa124fd5db1179 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e -F src/date.c e19e0cfff9a41bfdd884c655755f6f00bca4c1a22272b56e0dd6667b7ea893a2 +F src/date.c 61e92f1f7e2e88e1cd91e91dc69eb2b2854e7877254470f9fabd776bfac922b8 F src/dbpage.c c9ea81c11727f27e02874611e92773e68e2a90a875ef2404b084564c235fd91f F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c -F src/delete.c 03a77ba20e54f0f42ebd8eddf15411ed6bdb06a2c472ac4b6b336521bf7cea42 -F src/expr.c 28b1cc3d2f147cc888703d5482f9581f17656d02abfa331c34370cb3350776be +F src/delete.c 1f2268d6fe3c78fc1bf794ba65d7026498b78e2342ffaf85825dedae546e6fde +F src/expr.c 51e9c77ff5d9a21439e611fe6571a3cd50387e526e13c5614fd407e5b8571930 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 -F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f -F src/func.c 0b802107498048d3dcac0b757720bcb8506507ce02159e213ab8161458eb293b +F src/fkey.c 931f74cec1dc8038a0217ef340c91ce147dd1bbed08dc40c47ee0ec6edfffb08 +F src/func.c 706ac012bf87d8ad7416a56a1d2b1f19e5dea03506a4606a01aa9d3bacf392c7 F src/global.c a19e4b1ca1335f560e9560e590fc13081e21f670643367f99cb9e8f9dc7d615b F src/hash.c 03c8c0f4be9e8bcb6de65aa26d34a61d48a9430747084a69f9469fbb00ea52ca F src/hash.h 46b92795a95bfefb210f52f0c316e9d7cdbcdd7e7fcfb0d8be796d3a5767cddf -F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 +F src/hwtime.h 21c2cf1f736e7b97502c3674d0c386db3f06870d6f10d0cf8174e2a4b8cb726e F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c dfd311b0ac2d4f6359e62013db67799757f4d2cc56cca5c10f4888acfbbfa3fd -F src/json.c fb031340edee159c07ad37dbe668ffe945ed86f525b0eb3822e4a67cbc498a72 +F src/json.c 5027b856cd9b621dc9ba66b211e21a440ccdc63cefdefb44c51e7d3ac550d1a4 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa -F src/loadext.c a3bc9a2522dc3b960e38b7582d1818f6245a49289387c2c7b19f27bfeabf1e81 -F src/main.c 65d11c17890966d271c925c6cc55e3ba50fa08374633cb99c0dee4719a20915a +F src/loadext.c 56a542244fbefc739a2ef57fac007c16b2aefdb4377f584e9547db2ce3e071f9 +F src/main.c 387bb9d0216d6d35b221481ba8e661d94ad043060cd89581b6422c269ce680a0 F src/malloc.c 422f7e0498e1c9ef967f06283b6f2c0b16db6b905d8e06f6dbc8baaa3e4e6c5a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2 @@ -710,45 +708,45 @@ F src/mem5.c b7da5c10a726aacacc9ad7cdcb0667deec643e117591cc69cf9b4b9e7f3e96ff F src/memdb.c a3feb427cdd4036ea2db0ba56d152f14c8212ca760ccb05fb7aa49ff6b897df3 F src/memjournal.c c283c6c95d940eb9dc70f1863eef3ee40382dbd35e5a1108026e7817c206e8a0 F src/msvc.h 80b35f95d93bf996ccb3e498535255f2ef1118c78764719a7cd15ab4106ccac9 -F src/mutex.c 06bcd9c3dbf2d9b21fcd182606c00fafb9bfe0287983c8e17acd13d2c81a2fa9 +F src/mutex.c 00b8cee206a67fd764d001f3a148494331d8d0b3b9c3974ecd69ff29bb444462 F src/mutex.h a7b2293c48db5f27007c3bdb21d438873637d12658f5a0bf8ad025bb96803c4a F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 F src/mutex_unix.c f7ee5a2061a4c11815a2bf4fc0e2bfa6fb8d9dc89390eb613ca0cec32fc9a3d1 -F src/mutex_w32.c 28f8d480387db5b2ef5248705dd4e19db0cfc12c3ba426695a7d2c45c48e6885 +F src/mutex_w32.c e1d317d29cb623667d43de94714264d1e1871cc4bb39fa67dd17048e8138c739 F src/notify.c 57c2d1a2805d6dee32acd5d250d928ab94e02d76369ae057dee7d445fd64e878 F src/os.c 509452169d5ea739723e213b8e2481cf0e587f0e88579a912d200db5269f5f6d F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06 -F src/os_kv.c fb7ba8d6204197357f1eb7e1c7450d09c10043bf7e99aba602f4aa46b8fb11a3 +F src/os_kv.c e7d96727db5b67e39d590a68cc61c86daf4c093c36c011a09ebfb521182ec28d F src/os_setup.h 8efc64eda6a6c2f221387eefc2e7e45fd5a3d5c8337a7a83519ba4fbd2957ae2 -F src/os_unix.c dcf7988ddbdd68619b821c9a722f9377abb46f1d26c9279eb5a50402fd43d749 -F src/os_win.c a89b501fc195085c7d6c9eec7f5bd782625e94bb2a96b000f4d009703df1083f -F src/os_win.h 4c247cdb6d407c75186c94a1e84d5a22cbae4adcec93fcae8d2bc1f956fd1f19 -F src/pager.c cd562b878ea1b44d021ba199abc9d3b54f6b3347500a9fed03f66d6000620945 +F src/os_unix.c fa5e09b4df35ad845440cad67b86908cfe1fd4c28c51915f82e23633d1992bf4 +F src/os_win.c 0d553b6e8b92c8eb85e7f1b4a8036fe8638c8b32c9ad8d9d72a861c10f81b4c5 +F src/os_win.h 5e168adf482484327195d10f9c3bce3520f598e04e07ffe62c9c5a8067c1037b +F src/pager.c fe34fd22ec251436985d7b6ebdd05bf238a17901c2cb23d3d28974dd2361a912 F src/pager.h 6137149346e6c8a3ddc1eeb40aee46381e9bc8b0fcc6dda8a1efde993c2275b8 -F src/parse.y 619c3e92a54686c5e47923688c4b9bf7ec534a4690db5677acc28b299c403250 +F src/parse.y 3b784d6083380a950e3b1b32ce5ddd303e8c7c209d8ab788df2c62aaf9ee8eb3 F src/pcache.c 588cc3c5ccaaadde689ed35ce5c5c891a1f7b1f4d1f56f6cf0143b74d8ee6484 -F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 +F src/pcache.h 092b758d2c5e4dabb30eae46d8dfad77c0f70b16bf3ff1943f7a232b0fe0d4ba F src/pcache1.c 131ca0daf4e66b4608d2945ae76d6ed90de3f60539afbd5ef9ec65667a5f2fcd -F src/pragma.c ecec75795c1821520266e4f93fa8840cce48979af532db06f085e36a7813860f -F src/prepare.c 2af0b5c1ec787c8eebd21baa9d79caf4a4dc3a18e76ce2edbf2027d706bca37a -F src/printf.c 7297c2aeed4d90d80c5ba82920d9e57b7bfad04b3466be1d7e042db382fe296e +F src/pragma.c 789ef67117b74b5be0a2db6681f7f0c55e6913791b9da309aefd280de2c8a74d +F src/prepare.c f6a6e28a281bd1d1da12f47d370a81af46159b40f73bf7fa0b276b664f9c8b7d +F src/printf.c 41fb76fcb5ed7e16aaddc659d3b23891abebea45549fe125fc2e6ec380cc7175 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c -F src/resolve.c 243888da9f31c48d0890f8c9df30ed4be6fd098d295b0fe1c297a7f3c453ca53 +F src/resolve.c 928ff887f2a7c64275182060d94d06fdddbe32226c569781cf7e7edc6f58d7fd F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c 5abbd22cae7469dae0a600d78636be77a5682cc372cf8da744d68bbe23298df2 -F src/shell.c.in 2c7e751795f38bb1855c35b556419cab5b8ba22e0f6758f5a629338065d6b79f -F src/sqlite.h.in c0979f9ac1f5be887397dd2a0bb485636893a81b34d64df85123aae9650c42f2 +F src/select.c ffe199f025a0dd74670d2a77232bdea364a4d7b36f32c64a6572d39ba6a11576 +F src/shell.c.in a3288615aba1b375f7dca9801874c806e8d0d047dfe4e9253a944f4b47c7459e +F src/sqlite.h.in e2915e4a86d5e0783afb5cb72411df38d987c7f3c5aa2d5441b8e74d30b649d8 F src/sqlite3.rc 015537e6ac1eec6c7050e17b616c2ffe6f70fca241835a84a4f0d5937383c479 -F src/sqlite3ext.h 7f236ca1b175ffe03316d974ef57df79b3938466c28d2f95caef5e08c57f3a52 -F src/sqliteInt.h 601512a60e48f88e5148fc0e15f1efd03372df2a18bdee312d1590d14da6b5c1 -F src/sqliteLimit.h 4bc72c1519a27c538b7575c240a4472c829d78d27d69a00ddd5a046a0dbfd73a +F src/sqlite3ext.h 1b7a0ee438bb5c2896d0609c537e917d8057b3340f6ad004d2de44f03e3d3cca +F src/sqliteInt.h bc1cbc0c23dba35b324ae85a7dbb5fb182321bbd30857fb21f3d0cba049001a5 +F src/sqliteLimit.h c70656b67ab5b96741a8f1c812bdd80c81f2b1c1e443d0cc3ea8c33bb1f1a092 F src/status.c 7565d63a79aa2f326339a24a0461a60096d0bd2bce711fefb50b5c89335f3592 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 -F src/tclsqlite.c 3c604c49e6cf4211960a9ddb9505280fd22cde32175f40884c641c0f5a286036 +F src/tclsqlite.c 7401c73c917a4d1b380c896a324c8d8eb533a999559d9e339d479596553bebfd F src/tclsqlite.h 614b3780a62522bc9f8f2b9fb22689e8009958e7aa77e572d0f3149050af348a -F src/test1.c 5d061afe479c7364842e0170be7220dea13389575fa6030d30b3e20bec4e1f75 -F src/test2.c 62f0830958f9075692c29c6de51b495ae8969e1bef85f239ffcd9ba5fb44a5ff +F src/test1.c 3e3b013f59ffcb57dce00c90d55907072d71d4e970cb0a590cb261efe11bae9c +F src/test2.c 2b9ab96bba63a1c369d5769390475259ad210f144a877805f2e32e563f9e93c1 F src/test3.c 432646f581d8af1bb495e58fc98234380250954f5d5535e507fc785eccc3987a F src/test4.c 0ac87fc13cdb334ab3a71823f99b6c32a6bebe5d603cd6a71d84c823d43a25a0 F src/test5.c 38fa635a70a94f2aa8b47ecbab15d821386205d27ad4159c3551ab3ba45efa11 @@ -757,10 +755,10 @@ F src/test8.c 206d8f3cc73950d252906656e2646b5de0d580b07187b635fcb3edd8c2c5fbc0 F src/test9.c df9ddc7db6ef1b8cf745866ee229090779728bcbe660c7f297d3127ab21d92af F src/test_autoext.c 14d4bbd3d0bd1eec0f6d16b29e28cf1e2d0b020d454835f0721a5f68121ac10f F src/test_backup.c a2bfd90d2ff2511b8635507bdb30fa9b605ade19c16b533066cae3077f5bdb72 -F src/test_bestindex.c a9428931bec06de830b2630f57a7b1f2711761269f04df62b7aa1affcbce15bb +F src/test_bestindex.c d75fad21369d80910238032bcf8d9ca1f2bffda13c1ceec63bfbb7f704448b15 F src/test_blob.c 77b994e17f2c87055f44fd96c9a206c5a7155bae2cda2769af60c2f3582f962c F src/test_btree.c 28283787d32b8fa953eb77412ad0de2c9895260e4e5bd5a94b3c7411664f90d5 -F src/test_config.c 18aa596d37de1d5968c439fd58ebf38bc4d9c9d1db63621504e241fde375cecd +F src/test_config.c e02566c2c4ee2916324ce17123a798b47663cead2de546cfbd71d8cddb46bb26 F src/test_delete.c d0e8f6dc55cfc98a7c27c057fb88d512260564bf0b611482656c68b8f7f401ed F src/test_demovfs.c 3efa2adf4f21e10d95521721687d5ca047aea91fa62dd8cc22ac9e5a9c942383 F src/test_devsym.c 649434ed34d0b03fbd5a6b42df80f0f9a7e53f94dd1710aad5dd8831e91c4e86 @@ -780,7 +778,7 @@ F src/test_mutex.c dacae6790956c0d4e705aaed2090227792e291b0496cccd688e9994c1e21f F src/test_onefile.c f31e52e891c5fef6709b9fcef54ce660648a34172423a9cbdf4cbce3ba0049f4 F src/test_osinst.c 269039d9c0820a02ee928014c30860d57ee757ecda54df42e463d0ca1377b835 F src/test_pcache.c 496da3f7e2ca66aefbc36bbf22138b1eff43ba0dff175c228b760fa020a37bd0 -F src/test_quota.c 180e87437250bed7e17e4e61c106730939e39fec9be73d28961f27f579a92078 +F src/test_quota.c 5bb44452b9c6c248bb3c82d2466a20915aa6d12801f6c1784b6499aaa04d9811 F src/test_quota.h 2a8ad1952d1d2ca9af0ce0465e56e6c023b5e15d F src/test_rtree.c d844d746a3cc027247318b970025a927f14772339c991f40e7911583ea5ed0d9 F src/test_schema.c b06d3ddc3edc173c143878f3edb869dd200d57d918ae2f38820534f9a5e3d7d9 @@ -794,35 +792,35 @@ F src/test_vdbecov.c 5c426d9cd2b351f5f9ceb30cabf8c64a63bfcad644c507e0bd9ce2f6ae1 F src/test_vfs.c b4135c1308516adf0dfd494e6d6c33114e03732be899eace0502919b674586b5 F src/test_window.c 6d80e11fba89a1796525e6f0048ff0c7789aa2c6b0b11c80827dc1437bd8ea72 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c cb3294cf23c11106b50d9af6998a6c1bf389b52e15b17698c9fab97bbaa9b37f -F src/treeview.c 3ce7ac9835d2d70cc1c868b01b747ae8a062322e155701e58e3d62ca79aada7a -F src/trigger.c d5cf2541ff048f30b6a0507eb3d1ec4e695c53584e3b2298a5bf248714fe185e +F src/threads.c 85d8b9f05f78211c61e3739ab5db761d7118766d1916ae7f2764735106bc4e13 +F src/tokenize.c e9d52d9f7374d82dadcd11726bea8a597d43709d6672704f3f0375bf1d726912 +F src/treeview.c feaa59f14db4f7b5aacca9c5ad5aeb562c1f98262c1ffd74371f4186ade91fc5 +F src/trigger.c 4bf3bfb3851d165e4404a9f9e69357345f3f7103378c07e07139fdd8aeb7bd20 F src/update.c 3e5e7ff66fa19ebe4d1b113d480639a24cc1175adbefabbd1a948a07f28e37cf F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1 F src/utf.c 7267c3fb9e2467020507601af3354c2446c61f444387e094c779dccd5ca62165 -F src/util.c 36fb1150062957280777655976f3f9a75db236cb8207a0770ceae8d5ec17fcd3 -F src/vacuum.c 1bacdd0a81d2b5dc1c508fbf0d938c89fa78dd8d5b46ec92686d44030d4f4789 -F src/vdbe.c b44c366e83412d3b8c190feb1f029b7d02e1bd69252a57b32f195107f0d03964 -F src/vdbe.h be33bd7b17f2ec92939642416030491508c51071f6c14e27cd195983fec56b63 -F src/vdbeInt.h 2aaeb6df2938b181b4700a9328688a3986f2bba71e8b96f6a80671316618fa49 -F src/vdbeapi.c 790f199ec0d9c423f5efef58d7538ac9c6b34248d84180eb0dca3d635f1c9c9b -F src/vdbeaux.c 908d8a191aed444b2e4c920159249127f3ff67b94c56a16fad1dfdf9c7488f20 +F src/util.c 4f0abc15f63829e12cdfeeb490faf25ac65894b0bcc20d660e3f3757b8e2360b +F src/vacuum.c d3d35d8ae893d419ade5fa196d761a83bddcbb62137a1a157ae751ef38b26e82 +F src/vdbe.c 6c57525d7db0232d52687d30da1093db0c152f14206c2ef1adf0c19a09d863e3 +F src/vdbe.h 70e862ac8a11b590f8c1eaac17a0078429d42bc4ea3f757a9af0f451dd966a71 +F src/vdbeInt.h c31ba4dc8d280c2b1dc89c6fcee68f2555e3813ab34279552c20b964c0e338b1 +F src/vdbeapi.c 6cdcbe5c7afa754c998e73d2d5d2805556268362914b952811bdfb9c78a37cf1 +F src/vdbeaux.c 81687c55682b9f4d942186695f4f7fa4743c564a985e0889def52eded9076d61 F src/vdbeblob.c b3f0640db9642fbdc88bd6ebcc83d6009514cafc98f062f675f2c8d505d82692 -F src/vdbemem.c fdd023e357ad3129e1dcae46df47fccceeb8bd1ffa6c5d43a1e3f04460bb59b7 +F src/vdbemem.c efacb8f229422d2a4db0ed38e49b7f3897862a98d82b261aa3b43d7a2d98c6da F src/vdbesort.c b69220f4ea9ffea5fdef34d968c60305444eea909252a81933b54c296d9cca70 F src/vdbetrace.c 49e689f751505839742f4a243a1a566e57d5c9eaf0d33bbaa26e2de3febf7b41 F src/vdbevtab.c fc46b9cbd759dc013f0b3724549cc0d71379183c667df3a5988f7e2f1bd485f3 F src/vtab.c 5437ce986db2f70e639ce8a3fe68dcdfe64b0f1abb14eaebecdabd5e0766cc68 F src/vxworks.h 9d18819c5235b49c2340a8a4d48195ec5d5afb637b152406de95a9436beeaeab -F src/wal.c 88d94fd15a75f6eda831fa32d1148a267ea37bf0a4b69829a73dfde06244b08f +F src/wal.c 7340d4f9bb827bd349127cac6b2cf0cb7f76b9fda645f7b9b0bf7a6e0b1e2e7c F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 -F src/where.c 287324fe73a0ae8e55b3be89bb2fe4148e3a8394e1e2f10ed2113713a037d8a3 +F src/where.c bffca5e4ef20d0bfbdc24f1dc13fd3f955284225a8ad25a4454635f6be39aad0 F src/whereInt.h 8d94cb116c9e06205c3d5ac87af065fc044f8cf08bfdccd94b6ea1c1308e65da -F src/wherecode.c fe08356c7f20f4e2290204b9147bded3bbfe5453e2c590be0f9b0b5f1c959e76 -F src/whereexpr.c 403a44eeec1a0f0914fccc6a59376b6924bc00ef6728fe6ffce4cf3051b320fc -F src/window.c 538195bbc75bb924e18e368fbd4ed731a3fe3f901351b44f6466ec486f53affe +F src/wherecode.c 676cb6cb02878643e817d9917a2d3522b83a3736b2cedd3dc8a01d7bb92af6c2 +F src/whereexpr.c e9f7185fba366d9365aa7a97329609e4cf00b3dd0400d069fbaa5187350c17c6 +F src/window.c c0a38cd32473e8e8e7bc435039f914a36ca42465506dc491c65870c01ddac9fb F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test 4d7a34d328e58ca2a2d78fd76c27614a41ca7ddf4312ded9c68c04f430b3b47d F test/affinity3.test 9b7d1133e11d5edd7805573c4ab6f3ba73b0b74a1f280d5b130d4bf3506a93ff @@ -837,12 +835,14 @@ F test/alter2.test 7e3d26ab409df52df887b366a63902c3429b935c41cb962fd58ffc25784f2 F test/alter3.test dcdd5f850f30656a45a0f05e41abfb52b74bbf6ccba165d0f7adf6b0116e4fd6 F test/alter4.test 37cafe164067a6590a0ee4cec780bddbbaa33dc50b11542dcfbe0e65626494fd F test/alterauth.test 63442ba61ceb0c1eeb63aac1f4f5cebfa509d352276059d27106ae256bafc959 -F test/alterauth2.test 48967abae0494d9a300d1c92473d99fcb66edfcc23579c89322f033f49410adc -F test/altercol.test b43fb5725332f4cf8cff0280605202c1672e808281accea60a066d2ccc5129e5 +F test/alterauth2.test 4b74fa8f184f4736497317feb587b65759eb87d87acfe3a8ef433d4d18bb002b +F test/altercol.test 3661c432aacb42bc2198dd4611bbb9c3b09fc73251b59edda046109103b8ac00 +F test/altercons.test ea18def4a0f26b9066da56095c9c480df705df4d02e4ae151708fae76f7e3884 +F test/altercons2.test 3c1f58312817df43aeada3b1827fdc3ce3fc50c6f49a95ef62cf4cbbae8583a0 F test/altercorrupt.test 2e1d705342cf9d7de884518ddbb053fd52d7e60d2b8869b7b63b2fda68435c12 F test/alterdropcol.test a653a3945f964d26845ec0cd0a8e74189f46de3119a984c5bc45457da392612e F test/alterdropcol2.test 527fce683b200d620f560f666c44ae33e22728e990a10a48a543280dfd4b4d41 -F test/alterfault.test 289067108947bedca27534edd4ff251bcd298cf84402d7b24eaa3749305418c6 +F test/alterfault.test 2bb3103954ea60f2e2777b1ae12e79ec3e1fd278f2b1398ad316c68835a62898 F test/alterlegacy.test f38c6d06cda39e1f7b955bbce57f2e3ef5b7cb566d3d1234502093e228c15811 F test/altermalloc.test 167a47de41b5c638f5f5c6efb59784002b196fff70f98d9b4ed3cd74a3fb80c9 F test/altermalloc2.test 17fb3724c4b004c469c27dc4ef181608aa644555fbd3f3236767584f73747c81 @@ -850,8 +850,8 @@ F test/altermalloc3.test 8040e486368403f2fdd6fc3998258b499bd4cc2f3ddbb5f8f874cd4 F test/alterqf.test 8ec03d776de9c391daa0078ea8f838903bdcfb11dfae4ba3576b48436834ccba F test/altertab.test 8a2712f9076da5012a002d0b5cc0a421398a5bf61c25bab41b77c427586a7a27 F test/altertab2.test 0889ba0700cc1cdb7bc7d25975aa61fece34f621de963d0886e2395716b38576 -F test/altertab3.test 471b8898d10bbc6488db9c23dc76811f405de6707d2d342b1b8b6fd1f13cd3c8 -F test/altertrig.test aacc980b657354fe2d3d4d3a004f07d04ccc1a93e5ef82d68a79088c274ddc6b +F test/altertab3.test 575e771e2f02b13eb98798dc92eabacd187d6dbcf596e70f11d699b0b6b5d0b2 +F test/altertrig.test b1590647076add5a47aea0f2236c609ca0bc8a7a2462463edd3e5882c7894802 F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 2fb21d7d64748636384e6cb8998dbf83968caf644c07fcb4f76c18f2e7ede94b F test/analyze3.test c5156cef33f04b90a6b9e9d5d0bbc273a0fb44147d4508407bf1080811e2c6c8 @@ -868,6 +868,7 @@ F test/analyzeF.test 40b5cc3ad7b10e81020d7ca86f1417647ecfae7477cfd88acc5aa7ae106 F test/analyzeG.test 623be33038c49648872746c8dd8b23b5792c08fef173c55e82f1b12fca259852 F test/analyzer1.test b6a624ec0af92eec209e1328465b66937c8fdf2fb442a3fa45321ddb3700f4aa F test/atof1.test bd21c4a0e718ab1470de07a2a79f2544d7903be34feebcc80de04beee4807b00 +F test/atof2.test 12912add57230495450e2fc94cb8ad1c9f3277f8843a3bc27079cae45c9782a1 F test/atomic.test 065a453dde33c77ff586d91ccaa6ed419829d492dbb1a5694b8a09f3f9d7d061 F test/atomic2.test b6863b4aa552543874f80b42fb3063f1c8c2e3d8e56b6562f00a3cc347b5c1da F test/atrc.c c388fac43dbba05c804432a7135ae688b32e8f25818e9994ffba4b64cf60c27c @@ -876,12 +877,12 @@ F test/attach2.test 6d1e3a457ce260d6fc8e5945c07fba6c76dc2aa90e1c701f067b50ee88f7 F test/attach3.test c59d92791070c59272e00183b7353eeb94915976 F test/attach4.test 00e754484859998d124d144de6d114d920f2ed6ca2f961e6a7f4183c714f885e F test/attachmalloc.test 67309af95c6b765c13e7d2279d7fccbef78e6eb0565d75d51cefd5dc88784549 -F test/auth.test 5b8558a40571ebc55c1581cb7cec3b2348a699542a0a51b83ef21c6a953d95e3 +F test/auth.test 2a01bf5bf3a0f10adf8ae3a3fd2c05af8a8c1b7a52fae227adb4ccd931915b5c F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1 F test/auth3.test 76d20a7fa136d63bcfcf8bcb65c0b1455ed71078d81f22bcd0550d3eb18594ab F test/autoanalyze1.test b9cc3f32a990fa56669b668d237c6d53e983554ae80c0604992e18869a0b2dec F test/autoinc.test 9df9930966dbe92c55ef37a4d89112cfd537be0d0596d397177c12db9e581be0 -F test/autoindex1.test 65931519206bbec71948b11e125af0656435a0937973fe5fed70d776a712911f +F test/autoindex1.test 2523a76f30734742c3f4d948d0cbf3b6627f775e7833814f425a2e289ba58b22 F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df F test/autoindex3.test ca502c8050166ac6107a7b4fe4e951f4d3270a23a958af02b14f1b962b83c4b6 F test/autoindex4.test 3c2105e9172920e26f950ba3c5823e4972190e022c1e6f260ba476b0af24c593 @@ -889,7 +890,7 @@ F test/autoindex5.test 3fb938cbf4e7f3896563ce04e2a24b0bc653fc6245b4bf3268cd7b20f F test/autovacuum.test 00671369bbf96c6a49989a9425f5b78b94075d6a4b031e5e00000c2c32f365df F test/autovacuum2.test 76f7eb4fe6a6bf6d33a196a7141dba98886d2fb53a268d7feca285d5da4759d7 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 -F test/avfs.test 76f59743dc1f5fa533840d1818b420fe1ee45e21c0fd6bbac7942ba677903128 +F test/avfs.test 95bb8d04f8edad6dc9e600221d103f7e2cc3da398af84df215a3a819e560c45c F test/avtrans.test 7a6eae44763293024b137b53ff824d8500d754dbae060a8d940afbacfc1d4a15 F test/backcompat.test f2431465ed668f09fc3f6998e56e893a1506ccea6e8b6f409f085f759f431b48 F test/backup.test 3b08fd4af69f0fa786931103a31f4542b184aba16e239e5f22b18c3c2476697f @@ -909,13 +910,14 @@ F test/bestindex4.test 3039894f2dad50f3a68443dffad1b44c9b067ac03870102df1ce3d9a4 F test/bestindex5.test a0c90b2dad7836e80a01379e200e5f8ec9476d49b349af02c0dbff2fb75dc98d F test/bestindex6.test 16942535b551273f3ad9df8d7cc4b7f22b1fcd8882714358859eb049a6f99dd4 F test/bestindex7.test f094c669a6400777f4d2ddc3ed28e39169f1adb5be3d59b55f22ccf8c414b71e -F test/bestindex8.test b63a4f171a2c83d481bb14c431a8b72e85d27b2ffdaa0435a95d58ca941678f9 +F test/bestindex8.test 4d8b1e8f30a7f405988ce4dbcc2b95c0775f0bed9ec08e0291a07e2f35f7e653 F test/bestindex9.test 1a4b93db117fd8abe74ae9be982f86aa72f01e60cd4ac541e6ede39673a451a0 F test/bestindexA.test e1b5def6b190797cacf008e6815ffb78fb30261999030d60a728d572eef44c7f -F test/bestindexB.test 328b97b69cd1a20928d5997f9ecb04d2e00f1d18e19ab27f9e9adb44d7bc51ce +F test/bestindexB.test 14db2f66ec9cc5064a74996033b74e5eec0fd2f3a327fbe34ff18de67e9d2671 F test/bestindexC.test 95b4a527b1a5d07951d731604a6d4cf7e5a806b39cea0e7819d4c9667e11c3fc F test/bestindexD.test 6a8f6f84990bcf17dfa59652a1f935beddb7afd96f8302830fbc86b0a13df3c3 F test/bestindexE.test 297f3ea8500a8f3c17d6f78e55bdfee089064c6144ee84a110bd005a03338f49 +F test/bestindexF.test 4e53d606cbde40a2254aa016d500c5b71766a4065b8541202d195a3d9fe11b1c F test/between.test e7587149796101cbe8d5f8abae8d2a7b87f04d8226610aa1091615005dcf4d54 F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc @@ -950,7 +952,7 @@ F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 F test/capi3c.test 31d3a6778f2d06f2d9222bd7660c41a516d1518a059b069e96ebbeadb5a490f7 F test/capi3d.test 8b778794af891b0dca3d900bd345fbc8ebd2aa2aae425a9dccdd10d5233dfbde F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe -F test/carray01.test 49e2aedfdf2c715bc002d2773cdc1217166679639542c79c8aa4115f06421407 +F test/carray01.test 17c1cf8287862b15dda949dba626fd5fee5c58471dcc1cae0341471c2ae7da01 F test/carray02.test 9d070b54f24a34d1f3b3c552ba34db0375a9d1c4219067416fb07d1595987c9d F test/carrayfault.test 108a7d83904fc267c448e27c13b2a857c700bd6ddaa2f1e2518be718b159cb6b F test/cast.test a2a3b32df86e3c0601ffa2e9f028a18796305d251801efea807092dbf374a040 @@ -968,7 +970,7 @@ F test/collate1.test 0890fa372753b59eba53832d37328af815f6b8e4b16761823180eeb62c8 F test/collate2.test 471c6f74573382b89b0f8b88a05256faa52f7964f9e4799e76708a3b1ece6ba4 F test/collate3.test 89defc49983ddfbf0a0555aca8c0521a676f56a5 F test/collate4.test c953715fb498b87163e3e73dd94356bff1f317bd -F test/collate5.test b1dfeff239ea69ee9225832553f423d37a6184eb730cee06f6846ab4e3c6dbef +F test/collate5.test 42daaf7799b04221206a219fb3c0f9efeede03e760f9562b8c0114b8df183fe3 F test/collate6.test 8be65a182abaac8011a622131486dafb8076e907 F test/collate7.test 8ec29d98f3ee4ccebce6e16ce3863fb6b8c7b868 F test/collate8.test cd9b3d3f999b8520ffaa7cc1647061fc5bab1334 @@ -1005,7 +1007,7 @@ F test/corruptK.test ac13504593d89d69690d45479547616ed12644d42b5cb7eeb2e759a76fc F test/corruptL.test f15de2b4729c0851ea89916a26766b094d74bac79f9f9f2b0191935aa3b344c9 F test/corruptM.test 7d574320e08c1b36caa3e47262061f186367d593a7e305d35f15289cc2c3e067 F test/corruptN.test a034bb217bebd8d007625dfb078e76ec3d53515052dbceb68bd47b2c27674d5c -F test/cost.test cc434a026b1e9d0d98137a147e24e5daf1b1ad09e9ff7da63b34c83ddd136d92 +F test/cost.test 3786cd1cc6d1ab416004a1e39387fb0db0c8e259f46b0bce62cbdc328f2c55a0 F test/count.test cd4bd531066e8d77ef8fe1e3fc8253d042072e117ccab214b290cf83f1602249 F test/countofview.test 4088e461a10ee33e69803c177a69aa1d7bba81a9ffc2df66d76465a22ca7fdfc F test/coveridxscan.test f35c7208dedc4f98e471c569df64c0f95a49f6e072d8dc7c8f99bdee2697de1b @@ -1036,7 +1038,7 @@ F test/dbfuzz.c fc566102f72c8af84ae8077b4faf7f056c571e6fa7a32e98b66e42b7505f47b6 F test/dbfuzz001.test 6c9a4622029d69dc38926f115864b055cb2f39badd25ec22cbfb130c8ba8e9c3 F test/dbfuzz2-seed1.db e6225c6f3d7b63f9c5b6867146a5f329d997ab105bee64644dc2b3a2f2aebaee F test/dbfuzz2.c 4b3c12de4d98b1b2d908ab03d217d4619e47c8b23d5e67f8a6f2b1bdee7cae23 -F test/dblwidth-a.sql eb4141518610e52f931a55a984310075e98dc31eee5a28ae806b1e35377be85a +F test/dblwidth-a.sql 59dd59aa78ce8fd8ab631a3816516831f4e947b143039257e6fe132c3cea4171 F test/dbpage.test 63fab1eb026bada121107e53436fa749bbf83281dc9dea17af422f7a5c0f289f F test/dbpagefault.test ea39de2ca86041a9c6df1135645180a76d0a8da93ac159e2fafe38e39636530b F test/dbstatus.test 4a4221a883025ffd39696b3d1b3910b928fb097d77e671351acb35f3aed42759 @@ -1053,8 +1055,9 @@ F test/descidx2.test a0ba347037ff3b811f4c6ceca5fd0f9d5d72e74e59f2d9de346a9d2f6ad F test/descidx3.test 953c831df7ea219c73826dfbf2f6ee02d95040725aa88ccb4fa43d1a1999b926 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 691c9e850b0d0b56b66e7e235453198cb4cf0760e324b7403d3c5abbeab0a014 -F test/distinct2.test 4d6316b6487a0aa5a90bee111575c957e2a5ba5a9be9156febe9533ce78876e8 +F test/distinct2.test 072f33e1348b5cae4156e7ca4c124d21053f77d96d5d960a1ba21806416074ab F test/distinctagg.test 40d7169ae5846caaf62c6e307d2ca3c333daf9b6f7cde888956a339a97afe85f +F test/dotcmd01.sql dd5b0475df723fdb9f91031a7b9684abb279f6a686178f0d8e3b5f1da93c27c2 F test/e_blobbytes.test 4c01dfe4f12087b92b20705a3fdfded45dc4ed16d5a211fed4e1d2786ba68a52 F test/e_blobclose.test 692fc02a058476c2222a63d97e3f3b2b809c1842e5525ded7f854d540ac2e075 F test/e_blobopen.test 29f6055ee453b8e679fe9570c4d3acfedbef821622c5dad16875148c5952ef50 @@ -1064,7 +1067,7 @@ F test/e_createtable.test 31b9bcb6ac8876bc7ec342d86d9c231a84c62b442093a6651dfd0f F test/e_delete.test ab39084f26ae1f033c940b70ebdbbd523dc4962e F test/e_droptrigger.test 235c610f8bf8ec44513e222b9085c7e49fad65ad0c1975ac2577109dd06fd8fa F test/e_dropview.test 74e405df7fa0f762e0c9445b166fe03955856532e2bb234c372f7c51228d75e7 -F test/e_expr.test 0a1e175caddc78b27306647cb4ce2362c55790190f8cdd178b75fd6262eb8f76 +F test/e_expr.test 9bdb347b78b9f4eff9153ea97797facc179a821898588471a70808b4471a69b0 F test/e_fkey.test feeba6238aeff9d809fb6236b351da8df4ae9bda89e088e54526b31a0cbfeec5 F test/e_fts3.test 17ba7c373aba4d4f5696ba147ee23fd1a1ef70782af050e03e262ca187c5ee07 F test/e_insert.test f02f7f17852b2163732c6611d193f84fc67bc641fb4882c77a464076e5eba80e @@ -1073,19 +1076,19 @@ F test/e_resolve.test a61751c368b109db73df0f20fc75fb47e166b1d8 F test/e_select.test 327a15f14068bbd6f647cedc67210f8680fcb2f05e481a0a855fccd2abfa1292 F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f F test/e_totalchanges.test c927f7499dc3aa28b9b556b7d6d115a2f0fe41f012b128d16bf1f3b30e9b41e4 -F test/e_update.test f46c2554d915c9197548681e8d8c33a267e84528 +F test/e_update.test 9f8bb82b8760ac66f2c9c2aadb78a418bd639d22f041d712570c9db56f20afda F test/e_uri.test 86564382132d9c453845eeb5293c7e375487b625900ab56c181a0464908417d8 F test/e_vacuum.test 89fc48e8beee2f9dfd6de1fbb2edea6542dae9121dc0fbe6313764169e742104 F test/e_wal.test db7c33642711cf3c7959714b5f012aca08cacfa78da0382f95e849eb3ba66aa4 F test/e_walauto.test 248af31e73c98df23476a22bdb815524c9dc3ba8 -F test/e_walckpt.test 28c371a6bb5e5fe7f31679c1df1763a19d19e8a0 +F test/e_walckpt.test 16e7d006e8687654ee59e7ad5a6d285ba23f0fe0eeb87f790afd6bc9cf1d1924 F test/e_walhook.test 01b494287ba9e60b70f6ebf3c6c62e0ffe01788e344a4846b08e5de0b344cb66 F test/emptytable.test a38110becbdfa6325cd65cb588dca658cd885f62 F test/enc.test b5503a87b31cea8a5084c6e447383f9ca08933bd2f29d97b6b6201081b2343eb F test/enc2.test 872afe58db772e7dfa1ad8e0759f8cc820e9efc8172d460fae83023101c2e435 F test/enc3.test 55ef64416d72975c66167310a51dc9fc544ba3ae4858b8d5ab22f4cb6500b087 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 -F test/eqp.test 746db9fe11629a0d00328e1721cc2a2e4726d574b677ab14de35fd914f54cc82 +F test/eqp.test 1d653fe8d2612cd6764e5ea2f16dcf5f13a9f50448b9233bb1573804bccd7579 F test/eqp2.test 6e8996148de88f0e7670491e92e712a2920a369b4406f21a27c3c9b6a46b68dd F test/errmsg.test eae9f091eb39ce7e20305de45d8e5d115b68fa856fba4ea6757b6ca3705ff7f9 F test/errofst1.test 6da78363739ba8991f498396ab331b5d64e7ab5c4172c12b5884683ef523ac53 @@ -1101,11 +1104,12 @@ F test/expr.test db981f8a85520e99ae20aab7ad2e9b5b0437ed09159b57ced434c672075d2e6 F test/expr2.test c27327ae9c017a7ff6280123f67aff496f912da74d78c888926d68b46ec75fd8 F test/exprfault.test da33606d799718e2f8e34efd0e5858884a1ad87f608774c552a7f5517cc27181 F test/exprfault2.test c49e84273898969af5dbc4fe6a3f4335f14639799f343590336c9ddf84425965 +F test/expridx1.test b464520126e1d781a7800f8540621c82e8bbc526f089ff8b0b57ffc51feea6b7 F test/extension01.test 5de412c66276105901c370770175003381fdcb0c4da7054fa43cf4a31e0bfa3a F test/external_reader.test 6fdec43eeca23eb32faad1e95a4d1abc402bc8b3db70df12d6fc08a637f4a2b5 F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79 F test/fallocate.test 37a62e396a68eeede8f8d2ecf23573a80faceb630788d314d0a073d862616717 -F test/filectrl.test 4b720117388cf6766d0b798e2dddd785953f8f371633b0c0084d2f34cf72336a +F test/filectrl.test 3344987f143f3cb9914895dc63d9e9518a535ef1b1438d22caf3ba2fa76cc6da F test/filefmt.test f393e80c4b8d493b7a7f8f3809a8425bbf4292af1f5140f01cb1427798a2bbd4 F test/filter1.test 590f8ba9a0cd0823b80d89ac75c5ce72276189cef9225d2436adaf1ee87f3727 F test/filter2.tcl 44e525497ce07382915f01bd29ffd0fa49dab3adb87253b5e5103ba8f93393e8 @@ -1118,13 +1122,15 @@ F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d F test/fkey5.test 6727452e163a427147e84e739da18713da553d79f9783559b04fdcd36d5c7421 F test/fkey6.test 668a7299e75899b0a3342c36df655be57f76a05aca3544bda939a6e676e2f000 F test/fkey7.test 64fb28da03da5dfe3cdef5967aa7e832c2507bf7fb8f0780cacbca1f2338d031 -F test/fkey8.test 51deda7f1a1448bca95875e4a6e1a3a75b4bd7215e924e845bd60de60e4d84bf +F test/fkey8.test 00ecfcaba529d4d9ef608551714e6a4d1e9d7083406f65e2f108c539aff6e8e3 F test/fkey_malloc.test 594a7ea1fbab553c036c70813cd8bd9407d63749 F test/fordelete.test ba98f14446b310f9c9d935b97ec748753d0144a28b356ba30d1f4f6958fdde5c F test/fork-test.c 9ac2e6423a1d38df3d6be0e8ac15608b545de21e2b19d9d876254c5931b63edb F test/format4.test eeae341953db8b6bda7f549044797c3278a6cc345d11ada81471671b654f8ef4 -F test/fp-speed-1.c b37de94eba034e1703668816225f54510ec60fb0685406608cc707afe6b8234d -F test/fpconv1.test d5d8aa0c427533006c112fb1957cdd1ea68c1d0709470dabb9ca02c2e4c06ad8 +F test/fp-speed-1.c 34a2fe8dc30d0b24bb3e7c16efca0a30452287da6caff7dde86eef882db155a4 +F test/fp-speed-2.c 15830b061832bb51f68f0985cf7a06f4f5d49eb25cc399854f1ff60d6a878ee9 +F test/fpconv1.test 63f352682fa65601a326563ad633086df6ab194e6ed5e7366786f38a525a7fd7 +F test/fptest01.sql 3d84f10bb1cc220b59207354c887d720289903adeb9972a29d6bfcb3fec0df95 F test/fts-9fd058691.test 78b887e30ae6816df0e1fed6259de4b5a64ad33c F test/fts3.test 672a040ea57036fb4b6fdc09027c18d7d24ab654 F test/fts3_common.tcl dffad248f9ce090800e272017d2898005c28ee6314fc1dd5550643a02666907a @@ -1150,7 +1156,7 @@ F test/fts3aux1.test 1880eaa75c586cd10f53080479a2b819b3915ae7ce55c4e0ba8f1fe05ac F test/fts3aux2.test 2459e7fa3e22734aed237d1e2ae192f5541c4d8b218956ad2d90754977bf907f F test/fts3b.test c15c4a9d04e210d0be67e54ce6a87b927168fbf9c1e3faec8c1a732c366fd491 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 -F test/fts3comp1.test a0f5b16a2df44dd0b15751787130af2183167c0c +F test/fts3comp1.test 73a53ada3d25bf242c4b2a24cfe9d39e658be56cfa74754279b9e6db776ed7ce F test/fts3conf.test c9cd45433b6787d48a43e84949aa2eb8b3b3d242bac7276731c1476290d31f29 F test/fts3corrupt.test 6732477c5ace050c5758a40a8b5706c8c0cccd416b9c558e0e15224805a40e57 F test/fts3corrupt2.test e318f0676e5e78d5a4b702637e2bb25265954c08a1b1e4aaf93c7880bb0c67d0 @@ -1200,7 +1206,7 @@ F test/fts3tok_err.test 52273cd193b9036282f7bacb43da78c6be87418d F test/fts3varint.test 0b84a3fd4eba8a39f3687523804d18f3b322e6d4539a55bf342079c3614f2ada F test/fts4aa.test 0e6bfd6a81695a39b23e448dda25d864e63dda75bde6949c45ddc95426c6c3f5 F test/fts4check.test f0ea5e5581951d8ef7a341eea14486daf6c5f516a2f3273b0d5e8cb8a6cd3bd2 -F test/fts4content.test 73bbb123420d2c46ef2fb3b24761e9acdb78b0877179d3a5d7d57aada08066f6 +F test/fts4content.test 7f441866207f5b1e76e0f18bde5d9925d1ee8f60388054613dd14a29a79f0bc4 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 F test/fts4growth.test 289833c34ad45a5e6e6133b53b6a71647231fb89d36ddcb8d9c87211b6721d7f F test/fts4growth2.test 13ad4e76451af6e6906c95cdc725d01b00044269 @@ -1212,7 +1218,7 @@ F test/fts4merge.test 57d093660a5093ae6e9fbd2d17592a88b45bbd66db2703c4b640b28828 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test 8d9ccb4a3d41c4c617a149d6c4b13ad02de797d0 F test/fts4merge4.test 66fce89934cd9508cbdc67de486558c34912ffb2e8ffe5c9a1bbb9b8a4408ba7 -F test/fts4merge5.test 69932d85cda8a1c4dcfb742865900ed8fbda51724b8cf9a45bbe226dfd06c596 +F test/fts4merge5.test 987af90c930e8555f74ab994f597431caec7f8defc52de7718655c32da07af9e F test/fts4min.test 1c11e4bde16674a0c795953509cbc3731a7d9cbd1ddc7f35467bf39d632d749f F test/fts4noti.test d5d933705b1b1516b67a5e3f8e514ecb19c6522fb3357bb744776d48427c2292 F test/fts4onepass.test d69ddc4ee3415e40b0c5d1d0408488a87614d4f63ba9c44f3e52db541d6b7cc7 @@ -1239,7 +1245,7 @@ F test/fuzz3.test 70ba57260364b83e964707b9d4b5625284239768ab907dd387c740c0370ce3 F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c 34a025386f84d818cd3343e69e9d9083091af83153e226d71d4e1c126b5f1dd0 +F test/fuzzcheck.c 207d3d1c1d578928f16dec20df781c5d13c2fd4694b4d70e72242f4dcb799830 F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -1251,7 +1257,7 @@ F test/fuzzdata8.db 8f34ae00d8d5d4747dd80983cf46161065e4f78324dcff3c893506ff8db3 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test f64c4aef4c9e9edf1d6dc0d3f1e65dcc81e67c996403c88d14f09b74807a42bc -F test/fuzzinvariants.c 3ddfec7f5b970b018f1a982532de905cf180e0c1e48cd653be9365d3e6177625 +F test/fuzzinvariants.c 6768bcd03290776cd982624729d2abee2e89e6aba62b4a2b839a98332725a167 F test/gcfault.test 4ea410ac161e685f17b19e1f606f58514a2850e806c65b846d05f60d436c5b0d F test/gencol1.test ceb3163b59cb77f4ad57ae4f01a143ce36b06fdd6a8dab1149235db89979ffd8 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 @@ -1262,6 +1268,8 @@ F test/hook.test 2d89bf9480646feb8093be3a58ea502d6521906779ed960de31dd9c4502c054 F test/hook2.test b9ff3b8c6519fb67f33192f1afe86e7782ee4ac8 F test/icu.test 8da7d52cd9722c82f33b0466ed915460cb03c23a38f18a9a2d3ff97da9a4a8c0 F test/ieee754.test 0d3ab84ab2069c9994c833a7cd820ee6037f0cf888e206a4a7fc05f735d5790a +F test/import01.sql 11a5f8325b8b04c66fe489d30558d462411432adf750cef1c0f7b06564691a8c +F test/imposter1.sql fc5ad0945bb19622688c7a1cd7dfd1cefa4b013bac9e2628c22b03c7309f021f F test/imposter1.test 5a20b2cdeb53e65fc57cdb10a33750bd4ef6259909eaf1972253b9e79f7a3fb2 F test/in.test edf979bff3244b9e47849e2b43886631354c8213791f42da92216f08012141af F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 @@ -1301,11 +1309,12 @@ F test/insert.test 97cfb30b83ca1622b9422a1e4c4831b4cb767cf5d654660945036d1e72067 F test/insert2.test 4d14b8f1b810a41995f6286b64a6943215d52208 F test/insert3.test 1b7db95a03ad9c5013fdf7d6722b6cd66ee55e30 F test/insert4.test 2bf81535a990c969665d66db51fcf76c23499b39893b5109f413d1de4ad34cd3 -F test/insert5.test 394f96728d1258f406fe5f5aeb0aaf29487c39a6 +F test/insert5.test 79f6b6efd0d3db5f4e3ff442300b7d9e7185adb345b29aacc3ea5a9c58ab9beb F test/insertfault.test ac63d14ea3b49c573673a572f4014b9117383a03e497c58f308b5c776e4a7f74 F test/instr.test 67ba309e9697c24a304e98a7c8f372456177dd4e32237d2a305e1e05f7bb79c2 F test/instrfault.test 95e28efade652e6d51ae11b377088fe523a581a07ec428009e152a4dd0e0f44c F test/intarray.test bb976b0b3df0ebb6a2eddfb61768280440e672beba5460ed49679ea984ccf440 +F test/intck01.sql f2d88bf41cdd64f2ed8c3d4f357cf520f017aa2986999ab9a62eb6506ef18106 F test/interrupt.test ac1ef50ec9ab8e4f0e17c47629f82539d4b22558904e321ed5abea2e6187da7a F test/interrupt2.test e4408ca770a6feafbadb0801e54a0dcd1a8d108d F test/intpkey.test 7d54711acf553cdd641a40e9c6cfc2bf1a76070074940c1b126442517054320f @@ -1318,7 +1327,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 5984da7bf74b6540aa356f2ab0c6ae68a6d12039a3d798a9ac6a100abc17d520 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/istrue.test e7f285bb70282625c258e866ce6337d4c762922f5a300e1b50f958aef6e7d9c9 -F test/join.test 2fcfd84640cfd9ff48f31b4b0d370c4d5498c355ae4384544668ca54d37ae186 +F test/join.test c706b382ed09ddc89eee7ad0ffd08d862655b0abc292a690d41d995c18c17b3f F test/join2.test f59d63264fb24784ae9c3bc9d867eb569cd6d442da5660f8852effe5c1938c27 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 @@ -1344,16 +1353,17 @@ F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/json/README.md de59d5ba0bd2796d797115688630a6405bbf43a2891bad445ac6b9f38b83f236 F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd28656fb261bddc8a3f F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307 -F test/json/json-speed-check.sh 7d5898808ce7542762318306ae6075a30f5e7ee115c4a409f487e123afe91d88 x +F test/json/json-speed-check.sh 45862b216f1f8bbf16d74e2ad6ea20c773ee77774b299a7e0f76a22eb98e91f1 x F test/json/jsonb-q1.txt 1e180fe6491efab307e318b22879e3a736ac9a96539bbde7911a13ee5b33abc7 F test/json101.test cf53254f0f0c1399a01b21fc58fee0e63a12a556be91b9ee9faccdb8b82c083c -F test/json102.test 9b2e5ada10845ff84853b3feaae2ce51ce7145ae458f74c6a6cecc6ef6ee3ae1 -F test/json103.test 355746a6b66aa438f214b4fae454b13068fad2444b5f693e0d538ad1c059b264 +F test/json102.test ea5c9811e408e115c8fc539548deef431fda4924c23cacd79dd4b783f4449f07 +F test/json103.test e626d109cd0bdb8282ec9bf755af3befa50e3e03a255362fc53433d31e1d66d4 F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 -F test/json105.test 043838b56e68f3252a0dcf5be1689016f6f3f05056f8dcfcdc9d074f4d932988 +F test/json105.test 9900caa21888289873bc6c14f5ee41213d28ac9b7ca3395f8afb73d540e80f66 F test/json106.test 4aed3afd16549045d198a8d9cea00deea96e1f2ecf55864dce96cac558b8abef F test/json107.test 59054e815c8f6b67d634d44ace421cf975828fb5651c4460aa66015c8e19d562 F test/json108.test 0a5f1e2d4b35a1bc33052563d2a5ede03052e2099e58cb424547656c898e0f49 +F test/json109.test 441cea5d73c24a1a34d284101740dfae5a082237c048c8a66b03aeebe5e3643e F test/json501.test b95e2d14988b682a5cadf079dd6162f0f85fb74cd59c6b1f1624110104a974eb F test/json502.test 4ef68e4f272dfb083d4cbceb4e9e51d67ec1186a185e0c13637c50a4dc2f9796 F test/jsonb01.test f4cdfb4cf5a0c940091b17675ed9583f45add0c938f07d65b0de0e19d3a9a101 @@ -1426,7 +1436,7 @@ F test/misc1.test e3e36262aff1bd9b8b9bf1eeb3af04adb3fc1e23f0a92dbff708bba9e939ac F test/misc2.test a1a3573cc02662becd967766021d6f16c54684d56df5f227481c7ef0d9df0bd0 F test/misc3.test 651b88bca19b8ff6a7b6af73dae00c3fd5b3ea5bee0c0d1d91abd4c4b4748718 F test/misc4.test 10cd6addb2fa9093df4751a1b92b50440175dd5468a6ec84d0386e78f087db0e -F test/misc5.test 02fcaf4d42405be02ec975e946270a50b0282dac98c78303ade0d1392839d2b8 +F test/misc5.test 0a5d7604e197f10ee471280bfcaaf8229f9d8e2eebfef2c8853222cbc1ea9cd5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test d595599972ec0b436985f0f02f243b68500ffc977b9b3194ec66c0866cfddcab F test/misc8.test 08d2380bc435486b12161521f225043ac2be26f02471c2c1ea4cac0b1548edbd @@ -1439,11 +1449,12 @@ F test/mmap4.test 2e2b4e32555b58da15176e6fe750f17c9dcf7f93 F test/mmapcorrupt.test 470fb44fe92e99c1d23701d156f8c17865f5b027063c9119dcfdb842791f4465 F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3 F test/mmapwarm.test 2272005969cd17a910077bd5082f70bc1fefad9a875afec7fc9af483898ecaf3 +F test/modeA.sql fc64f646b0a1d0806af122fad6db2c89de63c51106655c09d44a652052a14d05 F test/multiplex.test d74c034e52805f6de8cc5432cef8c9eb774bb64ec29b83a22effc8ca4dac1f08 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test fac575e0b1b852025575a6a8357701d80933e98b5d2fe6d35ddaa68f92f6a1f7 F test/multiplex4.test e8ae4c4bd70606a5727743241f13b5701990abe4 -F test/mutex1.test 42cb5e244c3a77bb0ef2b967e06fa5e7ba7d32d90a9b20bed98f6f5ede185a25 +F test/mutex1.test 2cdc320a3320521d73b8090a04a2245c1e625e5f90672882517bf5fedcec8f13 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test 73ea63ab43668313e2f8cc9ef9e9a966672c7934f3ce76926fbe991235d07d91 F test/nockpt.test 3db354270fc63b6871eebd40285d4c55324fb27be629c958adbff6d7fcaa8e14 @@ -1453,13 +1464,13 @@ F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161 F test/notify3.test 796c7b7157f55c93b4e672b724e9c923a6fc6aa72ac419379a623e2350472e22 F test/notnull.test a37b663d5bb728d66fc182016613fb8e4a0a4bbf3d75b8876a7527f7d4ed3f18 -F test/notnull2.test 5b7dd6e82c409b2d011ad6acf19ae4bf0816a9c69ccf600b529d7405d7c49874 +F test/notnull2.test c2c7b670fb8fa6ffe5f9cc08af88864fbb8237e28b56ad528e8dee921019c5fe F test/notnullfault.test fc4bb7845582a2b3db376001ef49118393b1b11abe0d24adb03db057ee2b73d5 F test/null.test b7ff206a1c60fe01aa2abd33ef9ea83c93727d993ca8a613de86e925c9f2bc6f F test/nulls1.test 7a5e4346ee4285034100b4cd20e6784f16a9d6c927e44ecdf10034086bbee9c9 F test/numcast.test 5d126f7f581432e86a90d1e35cac625164aec4a1 F test/numindex1.test 20a5450d4b056e48cd5db30e659f13347a099823 -F test/offset1.test 72cca52482cbd5bc687cfa67aa2566c859081b5a353fd2f9da9bbd3914dea1ef +F test/offset1.test c21e67d2d5ae8ed310243fbe84fc2f0dca49e9ffed3ea89110c0d5914c0de620 F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 F test/optfuzz-db01.c 9f2fa80b8f84ebbf1f2e8b13421a4e0477fe300f6686fbd76cac1d2db66e0fdc F test/optfuzz-db01.txt 21f6bdeadc701cf11528276e2a55c70bfcb846ba42df327f979bd9e7b6ce7041 @@ -1503,11 +1514,17 @@ F test/pragma5.test 7b33fc43e2e41abf17f35fb73f71b49671a380ea92a6c94b6ce530a25f8d F test/pragma6.test c5ec577ba087954b4dfa619a3cbe97b155b60a0af487527abe89b10fc17e6512 F test/pragmafault.test 275edaf3161771d37de60e5c2b412627ac94cef11739236bec12ed1258b240f8 F test/prefixes.test b524a1c44bffec225b9aec98bd728480352aa8532ac4c15771fb85e8beef65d9 -F test/printf.test 685fec5a0c5af2490ab0632775a301554361d674211d690f5bee0a97b05333de +F test/printf.test bcb093ef5cbd17e2d94d93d62045ee61ed0f465c1ca123f284774e474e73a9ea F test/printf2.test 3f55c1871a5a65507416076f6eb97e738d5210aeda7595a74ee895f2224cce60 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc F test/pushdown.test 46a626ef1c0ca79b85296ff2e078b9da20a50e9b804b38f441590c3987580ddd +F test/qrf01.test 09caa00e6b4deea5fcd8958b4062233fcef5ef1b354312ec956ec4bc3a09572a +F test/qrf02.test 39b4afdc000bedccdafc0aecf17638df67a67aaa2d2942865ae6abcc48ba0e92 +F test/qrf03.test e7efe46d204671726b4707585126cd78d107368de4a7d0c7b8d5157cdd8624ed +F test/qrf04.test 0894692c998d2401dcc33449c02051b503ecce0c94217be54fb007c82d2d1379 +F test/qrf05.test 8ade5bfa7ef0b448e531687203fa8ae9ef41f1d7e4c11d5ba0c4846af75b13d5 +F test/qrf06.test cd7d0f0e2904904ab88141630a8fff5718ef7e3cc23e5a9c519cf29bb0919d89 F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 F test/quickcheck.test a4b7e878cd97e46108291c409b0bf8214f29e18fddd68a42bc5c1375ad1fb80a @@ -1520,8 +1537,9 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 F test/readonly.test 0d307c335b3421898cfe64a783a376138aa003849b6bff61ee2d21e805bc0051 -F test/recover.test c76d05f33f0271fba0f0752170e03b0ab5952dc61dcea7ab3ba40df03c4c42de -F test/regexp1.test 8f2a8bc1569666e29a4cee6c1a666cd224eb6d50e2470d1dc1df995170f3e0f1 +F test/recover.test 643139b911ac880a1e881d7621f02cfb546b608b8f2494d7d26fd5ed103b1ceb +F test/regexp1.sql fd15d317973dd0c9cd9ee7b8bda45ef1c0a5e126223fc53c86d71b5d407e6fb4 +F test/regexp1.test 0023eae4073265641b826a70d81ba34d4dd66ad71871a5b4a1b7cf500d5c0c51 F test/regexp2.test 64f9726b2ddc71aea06725fcad53231833d038d58b936d49083ace658b370a13 F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2118d F test/reservebytes.test 6163640b5a5120c0dee6591481e673a0fa0bf0d12d4da7513bad692c1a49a162 @@ -1539,13 +1557,13 @@ F test/rowid.test d27191b5ce794c05bf61081e8b2c546a1844c1641321dcaf7fb785234256cc F test/rowvalue.test 93474d8e1c496e970bdcc3a7f54ac835adda667d2fd971957b4bce0c0b11707b F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b F test/rowvalue3.test 103e9a224ca0548dd0d67e439f39c5dd16de4200221a333927372408c025324c -F test/rowvalue4.test bac9326d1e886656650f67c0ec484eb5f452244a8209c6af508e9a862ace08ed +F test/rowvalue4.test 6e160977d44ee715e142f63ec0e339586c61f12bbbffacee369b1cdc0b7390f0 F test/rowvalue5.test 00740304ea6a53a8704640c7405690f0045d5d2a6b4b04dde7bccc14c3068ea7 F test/rowvalue6.test d19b54feb604d5601f8614b15e214e0774c01087 F test/rowvalue7.test 06ec0aca725bf683313d03793aa2943bc7f45a901848c7056a9665b769c8fc38 F test/rowvalue8.test 5900eddad9e2c3c2e26f1a95f74aafc1232ee5e0 F test/rowvalue9.test 7499a8fd7ca3a3f0e19d94e135355439aa2b596f86b775ca8de79672da2ca378 -F test/rowvalueA.test be8d6ad8b476eb24c151bb20bfd487e0d50c5e99618b7b0e656035069d2fc2cf +F test/rowvalueA.test 7d28bf6c2f8a2af8adbba2c4eda9136a4dd36250b8966ba05b34c0954f78efd7 F test/rowvaluefault.test 963ae9cdaed30a85a29668dd514e639f3556cae903ee9f172ea972d511c54fff F test/rowvaluevtab.test cd9747bb3f308086944c07968f547ad6b05022e698d80b9ffbdfe09ce0b8da6f F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 @@ -1559,7 +1577,7 @@ F test/savepoint7.test 24c69af86d750c80d51cf6500fde9270717f2b6e5658f055b5e75af75 F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 F test/scanstatus.test b249328caf4d317e71058006872b8012598a5fa045b30bf24a81eeff650ab49e F test/scanstatus2.test d85d17f2b0b4c013dde95232f7beab749f11f0ef847f5ecffb9486d2f5ecf9f9 -F test/schema.test 5dd11c96ba64744de955315d2e4f8992e447533690153b93377dffb2a5ef5431 +F test/schema.test e615575f2d756df4629596523f11d9322384ecf9f980e58c774cff80ff041c33 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 8ed4ae66e082cdd8b1b1f22d8549e1e7a0db4527a8e6ee8b6193053ee1e5c9ce F test/schema4.test 3b26c9fa916abb6dadf894137adcf41b7796f7b9 @@ -1577,7 +1595,7 @@ F test/select5.test 8afc5e5dcdebc2be54472e73ebd9cd1adef1225fd15d37a1c62f969159f3 F test/select6.test da91e61d26b8dea4b61e4a862088dd6ab19998f7be22a16a5b0cfe806e597639 F test/select7.test b825420da8a0b5722fdb77f3369f6396a3d198c46e8787eb26ff9425d4ac9d27 F test/select8.test 8c8f5ae43894c891efc5755ed905467d1d67ad5d -F test/select9.test f7586b207ce2304ab80dc93d3146469a28fd4403621dd3a82d06644563d3c812 +F test/select9.test 108ceff733f31698fef41eb9a0c332f150c54e98be534ee38019a19943f3f5ae F test/selectA.test 1da8ce3884c326e11d2855baffb76436b0d7e044404af8a2a70d1399a4ff7e29 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 F test/selectC.test 38c530b0cc5728b793c3c11f52b52c70290d39822224acd39011c89c1853bd31 @@ -1601,16 +1619,17 @@ F test/sharedA.test 64bdd21216dda2c6a3bd3475348ccdc108160f34682c97f2f51c19fc0e21 F test/sharedB.test 1a84863d7a2204e0d42f2e1606577c5e92e4473fa37ea0f5bdf829e4bf8ee707 F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 -F test/shell1.test ebe953d64c937ad42a0f33170ac0d2d2568faae26813fc7a95203756446d54aa -F test/shell2.test ab23f01ea2347e4b72bb2399af7ee82aa00f9c059141749f7c4064abca5ad728 -F test/shell3.test 603b448e917537cf77be0f265c05c6f63bc677c63a533c8e96aae923b56f4a0e -F test/shell4.test 03593fa7908a55f255916ffeda707cdf55680c777736e3da62b1d78cde0d684d -F test/shell5.test d17e7927ab8b7f720efbdd9b5d05fceb6c3c56c25917901b315400214bf24ef4 +F test/shell1.test eda2e527435f139224dda67db6bbd2466597408d4fe5883d647d67fa32d88f7c +F test/shell2.test dc541d2681503e55466a24d35a4cbf8ca5b90b8fcdef37fc4db07373a67d31d3 +F test/shell3.test 91efdd545097a61a1f72cf79c9ad5b49da080f3f10282eaf4c3c272cd1012db2 +F test/shell4.test e25580a792b7b54560c3a76b6968bd8189261f38979fe28e6bc6312c5db280db +F test/shell5.test a9cd2c8b62e125049ef500937674f47dd6787f0157ac0515aa554044a4dc3ea9 F test/shell6.test e3b883b61d4916b6906678a35f9d19054861123ad91b856461e0a456273bdbb8 F test/shell7.test 43fd8e511c533bab5232e95c7b4be93b243451709e89582600d4b6e67693d5c3 -F test/shell8.test 641cf21a99c59404c24e3062923734951c4099a6b6b6520de00cf7a1249ee871 -F test/shell9.test 8742a5b390cdcef6369f5aa223e415aa4255a4129ef249b177887dc635a87209 -F test/shellA.test 4ecff8b7b2c0122ba8174abfbcc4b0f59e44d80f2a911068f8cd4cfc6661032d +F test/shell8.test 38c9e4d7e85d2a3ecfacaa9f6cda4f7a81bf4fffb5f3f37f9cd76827c6883192 +F test/shell9.test c0e8871061a92151450b3332279a893b516fa73a6c46d4f51a0998407cbf8c89 +F test/shellA.test 05cdaafa1f79913654487ce3aefa038d4106245d58f52e02faf506140a76d480 +F test/shellB.test 31df04230f6062069bb7c5d0e5c5439ca44448fa9da1a55aa461a4b872fe6bd9 F test/shmlock.test 9f1f729a7fe2c46c88b156af819ac9b72c0714ac6f7246638a73c5752b5fd13c F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 @@ -1644,8 +1663,8 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 377a0c48e5a92e0b11c1c5ebb1bc9d83a7312c922bc0cb05970ef5d6a96d1f0c -F test/speedtest.md ee958457ae1b729d9715ae33c0320600000bf1d9ddea1a88dcf79f56729d6fad -F test/speedtest.tcl 6b66974d833d35a63d0e9ec344e0ffa92fbbfac83e173556f700a61cb3be96fc x +F test/speedtest.md ea0c85ebe0ecff8b45ba6cdb26e694871f469009a5a29dcfe634b055f05ab241 +F test/speedtest.tcl b06f6321ef90bb68f18f7b0e430e25203d9da79b80f8926986a0d5f21ac485fb x F test/speedtest1.c 6c01252e66f46de0b6b8d5316e03521e2151782104f3608c10262aa5dce85721 F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 @@ -1680,23 +1699,24 @@ F test/sync.test a619e407ede58a7b6e3e44375328628559fc9695a9c24c47cb5690a866b0031 F test/sync2.test 06152269ed73128782c450c355988fe8dd794d305833af75e1a5e79edd4dae47 F test/syscall.test a067468b43b8cb2305e9f9fe414e5f40c875bb5d2cba5f00b8154396e95fcf37 F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 -F test/tabfunc01.test 56eeae736217204bb1d9f9ef38340d48058f809b64249217cf77ff4ba600cc21 +F test/tabfunc01.test cfa96a9a235c39fb0cae69928b989b28bfec108f62d2533486f76e32dcedfdfb F test/table.test e87294bf1c80bfd7792142b84ab32ea5beb4f3f71e535d7fb263a6b2068377bf F test/tableapi.test e37c33e6be2276e3a96bb54b00eea7f321277115d10e5b30fdb52a112b432750 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 -F test/tclsqlite.test 3f697424cfc1cdc9c076ec0cadb0e700f059400a3e3ce134b7d856fc9f880e1c +F test/tclsqlite.test 5d6c73bfe7006c85e2f7fb7db8638b521eb2043d5451aaacdac4851eab895443 F test/tempdb.test 4cdaa23ddd8acb4d79cbb1b68ccdfd09b0537aaba909ca69a876157c2a2cbd08 F test/tempdb2.test 353864e96fd3ae2f70773d0ffbf8b1fe48589b02c2ec05013b540879410c3440 F test/tempfault.test 0c0d349c9a99bf5f374655742577f8712c647900 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d1631311a16 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 -F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl 463ae33b8bf75ac77451df19bd65e7c415c2e9891227c7c9e657d0a2d8e1074a +F test/temptrigfault.tes fc5918e64f3867156fefe7cfca9d8e1f495134a5229b2b511b0dc11c07f2eab4 +F test/temptrigger.test a00f258ed8d21a0e8fd4f322f15e8cfb5cef2e43655670e07a753e3fb4769d61 +F test/tester.tcl 2d943f60200e0a36bcd3f1f0baf181a751cd3604ef6b6bd4c8dc39b4e8a53116 F test/testloadext.c 862b848783eaed9985fbce46c65cd214664376b549fae252b364d5d1ef350a27 -F test/testrunner.tcl 60d7efa1816c5dfc37df3e3454b94b9042c0c8c50b27ae296d4a797cd309ace6 x -F test/testrunner_data.tcl c507a9afa911c03446ed90442ffd4a98aca02882c3d51bd1177c24795674def8 -F test/testrunner_estwork.tcl 7927a84327259a32854926f68a75292e33a61e7e052fdbfcb01f18696c99c724 +F test/testrunner.tcl 6b232f0d4825dec8b967754503080fc9609fad077f582d02f86bd2d95bec4110 x +F test/testrunner_data.tcl 48c8a230fcada37f4809f95c2ba49e44bc3d520b6165c09173249c6e65b01cc1 +F test/testrunner_estwork.tcl 81e2ae10238f50540f42fbf2d94913052a99bfb494b69e546506323f195dcff9 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1745,7 +1765,7 @@ F test/tkt-8454a207b9.test ead80b7a01438ca1436cee029694a96c821346cf1e24f06de12f8 F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356 F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 -F test/tkt-99378177930f87bd.test 9d6cff39b50d062c813ae1cb0ebbd1b7acf81ecc23ae5d5215e5bb05667dc137 +F test/tkt-99378177930f87bd.test 1ee631d155f0d51a4547e9405ef35a3a9a32977352a37a10bcbbacc5e38356ad F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667 F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6 F test/tkt-9f2eb3abac.test cb6123ac695a08b4454c3792fbe85108f67fabf8 @@ -1793,7 +1813,7 @@ F test/tkt2213.test a9702175601a57b61aba095a233b001d6f362474 F test/tkt2251.test 5aab8c7898cd2df2a68fe19289cc29e8f5cf8c82 F test/tkt2285.test cca17be61cf600b397188e77e7143844d2b977e9 F test/tkt2332.test fc955609b958ca86dfa102832243370a0cc84070 -F test/tkt2339.test 73bd17818924cd2ac442e5fd9916b58565739450 +F test/tkt2339.test bad48bd064594aa7b4de23f6d59a72b0b0c4175fd917f4b66907584732d41652 F test/tkt2391.test ab7a11be7402da8b51a5be603425367aa0684567 F test/tkt2409.test be0d60e7d283f639dccea4b0b5e1cd3a4851fb5b F test/tkt2450.test 77ed94863f2049c1420288ddfea2d41e5e0971d6 @@ -1927,15 +1947,15 @@ F test/vacuum4.test 7ea76b769fffeb41f925303b04cbcf5a5bbeabe55e4c60ae754ff24eeeb7 F test/vacuum5.test 263b144d537e92ad8e9ca8a73cc6e1583f41cfd0dda9432b87f7806174a2f48c F test/vacuum6.test b137b04bf3392d3f5c3b8fda0ce85a6775a70ca112f6559f74ff52dc9ce042fd F test/vacuummem.test 4b30f5b95a9ff86e9d5c20741e50a898b2dc10b0962a3211571eb165357003fb -F test/values.test 0eda08a6ce6545f1ab012dff4cc72a7dd0fee2510f42444136bb2b2b5ed84bc0 +F test/values.test 0e037c50789ac2a308746567d07b53b2f6026c1bb3a435d1b099424600e64caf F test/valuesfault.test 2ef23ed965e3bd08e268cdc38a0d11653390ddbbe1e8e2e98d16f55edd30f6e8 F test/varint.test bbce22cda8fc4d135bcc2b589574be8410614e62 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 F test/view.test 3c23d7a068e9e4a0c4e6907498042772adea725f0630c3d9638ffd4e5a08b92b F test/view2.test db32c8138b5b556f610b35dfddd38c5a58a292f07fda5281eedb0851b2672679 F test/view3.test ad8a8290ee2b55ff6ce66c9ef1ce3f1e47926273a3814e1c425293e128a95456 -F test/vt02.c 5b44ac67b1a283fedecf2d6e2ceda61e7a157f01d44dcb4490dcb1e87d057060 -F test/vt100-a.sql 631eeab18c5adb531bab79aecf64eee3934b42c75a309ee395c814717a6a7651 +F test/vt02.c c2faf56d74470d569cd00741acb3f1719ee95d668f84ef58acc3872635789680 +F test/vt100-a.sql a3e188a118ca78c08b41681a4db6d0f353e554ceb33f1573b1872d16e2d30596 F test/vtab1.test 09a72330d0f31eda2ffaa828b06a6b917fb86250ee72de0301570af725774c07 F test/vtab2.test 14d4ab26cee13ba6cf5c5601b158e4f57552d3b055cdd9406cf7f711e9c84082 F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e @@ -1976,7 +1996,7 @@ F test/wal_common.tcl 204d1721ac13c5e0c7fae6380315b5ab7f4e8423f580d826c5e9df1995 F test/walbak.test 018d4e5a3d45c6298d11b99f09a8ef6876527946 F test/walbig.test f437473a16cfb314867c6b5d1dbcd519e73e3434 F test/walblock.test 6bb472e82730e7e4e81395e907a01d8cfc2bd9e1f01f8a9184ca572e2955a4bf -F test/walckptnoop.test b13a2c3140f2c913cfd422d9a224544757d04b8b14ab4c267ab9910467c0b9be +F test/walckptnoop.test 5f6123750f40cb86633a7e014f9fb805d0eb494b811840086dc72e554e68c7c1 F test/walcksum.test 50e204500eed9c691b6045e467bb2923f49aa93d8adf315e2be135fdb202c1c2 F test/walcrash.test 21038858cc552077b0522f50b0fa87e38139306a F test/walcrash2.test a0edab4e5390f03b99a790de89aad15d6ec70b36 @@ -1991,7 +2011,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8d78a1ec91299163451417b451a2bac3481f8eb9f455b1ca507a6625c927ca6e F test/walprotocol.test 1b3f922125e341703f6e946d77fdc564d38fb3e07a9385cfdc6c99cac1ecf878 F test/walprotocol2.test 7d3b6b4bf0b12f8007121b1e6ef714bc99101fb3b48e46371df1db868eebc131 -F test/walrestart.test 3b0a9198ad2eb9f716d8f3846b133ba9f4619fb56decb1e67bba27743c766289 +F test/walrestart.test 5168c0c2414d1971d8dec949c1070a0144cf15402361ba0d0e6a8054f5598a64 F test/walro.test 78a84bc0fdae1385c06b017215c426b6845734d6a5a3ac75c918dd9b801b1b9d F test/walro2.test 33955a6fd874dd9724005e17f77fef89d334b3171454a1256fe4941a96766cdc F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68 @@ -2006,7 +2026,7 @@ F test/walslow.test 0c51843836c9dcf40a5ac05aa781bfb977b396ee2c872d92bd48b79d5dd9 F test/walthread.test d562f51a61191ccfab64940df7aa1cef87c902fa5ab742590ef7f859dfe6a44b F test/walvfs.test e1a6ad0f3c78e98b55c3d5f0889cf366cc0d0a1cb2bccb44ac9ec67384adc4a1 F test/where.test 5087c72d26fd075a1644c8512be9fe18de9bf2d2b0754f7fd9b74a1c6540c4fc -F test/where2.test 52237a8cb27ebbf6583469429bb5733d1b94ac37d09c3dbd0f487952ed0ab3f8 +F test/where2.test 1dbff4ab847068a52b927a0c1a7cf7faed4c8cae081fbcdac9b052a3a209aefa F test/where3.test 4ccb156ae33de86414a52775a6f590a9d60ba2cbc7a93a24fa331b7bcf5b6030 F test/where4.test 4a371bfcc607f41d233701bdec33ac2972908ba8 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 @@ -2024,7 +2044,7 @@ F test/whereG.test 875d020ac0a47828b31e36c54f1bf0cf81c9ea43b257bc21286eca1fe9a48 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/whereI.test c4bb7e2ca56d49bd8ab5c7bd085b8b83e353922b46904d68aefb3c7468643581 F test/whereJ.test fc05e374cc9f2dc204148d6c06822c380ad388895fe97a6d335b94a26a08aecf -F test/whereK.test 0270ab7f04ba5436fb9156d31d642a1c82727f4c4bfe5ba90d435c78cf44684a +F test/whereK.test 4fb96b078f2ecedc467fa53177787378ff659539e415a4256cae7ae4e2a804b2 F test/whereL.test cb115604cc9bd61acbc99a1f1df0eb1ea7a7875a77fef25ba9282f01d10283e1 F test/whereM.test 0dbc9998783458ddcf3cc078ca7c2951d8b2677d472ecf0028f449ed327c0250 F test/whereN.test 63a3584b71acfb6963416de82f26c6b1644abc5ca6080c76546b9246734c8803 @@ -2036,7 +2056,7 @@ F test/wherelimit3.test 22d73e046870cf8bbe15573eda6b432b07ebe64a88711f9f849c6b36 F test/widetab1.test c296a98e123762de79917350e45fa33fdf88577a2571eb3a64c8bf7e44ef74d1 F test/win32heap.test 1ec2ce646aee491ec23bfcdfd005b33c79f13bf91467966f374a76ffe7c7e85f F test/win32lock.test e56d7a9b6cf9d5f3867c2dd19ff36c5326881e4038c6867610ecb3a9868ea4eb -F test/win32longpath.test 0f9837039b306735c13521c5f25b6ed42937b600dace58e28a3d2f8baf429b6a +F test/win32longpath.test 2641e3a5dbb59f49456f6caf78c0acd6ec7dbba27cb56363bab8fbfe93995caa F test/win32nolock.test 95854dc0206b8a95e4aee15a76acc082767b38f079b2e24676aed6cbb0f32798 F test/window1.test b46d28b9698559e66aa4adafd8074b940faee498bf0c4fbdb62548bfcccc67e7 F test/window2.tcl 492c125fa550cda1dd3555768a2303b3effbeceee215293adf8871efc25f1476 @@ -2061,11 +2081,11 @@ F test/windowerr.tcl f5acd6fbc210d7b5546c0e879d157888455cd4a17a1d3f28f07c1c8a387 F test/windowerr.test a8b752402109c15aa1c5efe1b93ccb0ce1ef84fa964ae1cd6684dd0b3cc1819b F test/windowfault.test 15094c1529424e62f798bc679e3fe9dfab6e8ba2f7dfe8c923b6248c31660a7c F test/windowpushd.test c420e2265f0e09a0e798d0513a660d71b51602088d81b3dbd038918ee1339dcc -F test/with1.test 1ee171d7c306ab8b0771f3511d870f56c735607729836585bbceb1fc2f47e0b1 +F test/with1.test 31db84788e0429885b63995149fab57d32e26196b752a3a926249ae74c0adddd F test/with2.test 181674a6cc86a601ca2ac052741cdfad5b529e07e870435d2f6cdb92d589ff17 F test/with3.test e30369ea27aa27eb1bda4c5e510c8a9f782c8afd2ab99d1a02b8a7f25a5d3e65 F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205 -F test/with5.test 6248213c41fab36290b5b73aa3f937309dfba337004d9d8434c3fabc8c7d4be8 +F test/with5.test 0e5e141fee75aa170289467542a2ffd71933d4fb006bfb135275b1787bdc8fbe F test/with6.test 281e4861b5e517f6c3c2f08517a520c1e2ee7c11966545d3901f258a4fe8ef76 F test/withM.test 693b61765f2b387b5e3e24a4536e2e82de15ff64 F test/without_rowid1.test f6e75e32821eb423ac3812434d12bdd8098f17e3b2206da61575e1db77f82428 @@ -2081,12 +2101,12 @@ F test/zeroblob.test 7b74cefc7b281dfa2b07cd237987fbe94b4a2037a7771e9e83f2d5f608b F test/zeroblobfault.test 861d8191a0d944dfebb3cb4d2c5b4e46a5a119eaec5a63dd996c2389f8063441 F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc F test/zipfile.test a3fcfc43115e4226fdddadd43bdf31c8ca805ad08dad435634f1633d8f5840d9 -F test/zipfile2.test a577e0775e32ef8972e7d5e9a45bc071a5ae061b5b965a08c9c4b709ad036a25 +F test/zipfile2.test 21afaffcf4f7769df38bf16e4a9c4dfa6ba1b0f5b695f844ec61fafb92db0db7 F test/zipfilefault.test 44d4d7a7f7cca7521d569d7f71026b241d65a6b1757aa409c1a168827edbbc2c F tool/GetFile.cs 47852aa0d806fe47ed1ac5138bdce7f000fe87aaa7f28107d0cb1e26682aeb44 F tool/GetTclKit.bat d84033c6a93dfe735d247f48ba00292a1cc284dcf69963e5e672444e04534bbf F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 -F tool/build-all-msvc.bat 1960a7a3e5d8176c4329e31476f6e3dfa9543675355fa9020a569f4452628458 x +F tool/build-all-msvc.bat 1ee9dbadcc07fc23268025854e97b392bcbad72376b47aee7b22f3797a4f2c87 x F tool/build-shell.sh 369c4b171cc877ad974fef691e4da782b4c1e99fe8f4361316c735f64d49280f F tool/buildtclext.tcl d09b753d7858314104eeaf5f4def85d35784470279809e47a633f142226f2b3f F tool/cg_anno.tcl c1f875f5a4c9caca3d59937b16aff716f8b1883935f1b4c9ae23124705bc8099 x @@ -2095,7 +2115,7 @@ F tool/cktclsh.sh 6075eef9c6b9ba4b38fef2ca2a66d25f2311bd3c610498d18a9b01f861629c F tool/cp.tcl 9a0d663ad45828de13763ee7ca0200f31f56c6d742cf104a56ae80e027c242d8 F tool/custom.txt 24ed55e71c5edae0067ba159bbf09240d58b160331f7716e95816cd3aa0ba5c4 F tool/dbhash.c 5da0c61032d23d74f2ab84ffc5740f0e8abec94f2c45c0b4306be7eb3ae96df0 -F tool/dbtotxt.c ca48d34eaca6d6b6e4bd6a7be2b72caf34475869054240244c60fa7e69a518d6 +F tool/dbtotxt.c cfeb957571735af345f253ba8417256031fa0dddf79468eefad184262d17211e F tool/dbtotxt.md c9a57af8739957ef36d2cfad5c4b1443ff3688ed33e4901ee200c8b651f43f3c F tool/emcc.sh.in 41a049468c8155433e37e656ba5bae063a000768b1d627025f277732c4e7c4a4 F tool/enlargedb.c 3e8b2612b985cfa7e3e8800031ee191b43ae80de96abb5abbd5eada62651ee21 @@ -2108,24 +2128,26 @@ F tool/genfkey.README e550911fa984c8255ebed2ef97824125d83806eb5232582700de949edf F tool/genfkey.test b6afd7b825d797a1e1274f519ab5695373552ecad5cd373530c63533638a5a4f F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/index_usage.c f62a0c701b2c7ff2f3e21d206f093c123f222dbf07136a10ffd1ca15a5c706c5 -F tool/lemon.c 8f6c122e5727cb0e5f302b8efc91489b1947a8d98206d7a1b1cfc0ed685b6e7c -F tool/lempar.c bdffd3b233a4e4e78056c9c01fadd2bb3fe902435abde3bce3d769fdf0d5cca2 +F tool/lemon.c 3fdc16b23f1ea0c91c049b518fc3f75c71843dbfe2b447fcb3cd92d9e4f219f8 +F tool/lempar.c b57e1780bf8098dd4a9a5bba537f994276ea825a420f6165153e5894dc2dfb07 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c 63412f9790e5e8538fbde0b4f6db154aaaf80f7a10a01e3c94d14b773a8dd5a6 F tool/logest.c c34e5944318415de513d29a6098df247a9618c96d83c38d4abd88641fe46e669 F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439 F tool/merge-test.tcl de76b62f2de2a92d4c1ca4f976bce0aea6899e0229e250479b229b2a1914b176 F tool/mkamalzip.tcl 8aa5ebe7973c8b8774062d34e15fea9815c4cc2ceea3a9b184695f005910876a -F tool/mkautoconfamal.sh 647dada5e34c466bef62a4408e1c99a7e5e1922805479dd57944f33f9803f2f8 +F tool/mkautoconfamal.sh 06fbe090b81c24e592c1f22b404334f805ba74d482a9260f2ac81e6f3d3386d8 F tool/mkccode.tcl c42a8f8cf78f92e83795d5447460dbce7aaf78a3bbf9082f1507dc71a3665f3c x -F tool/mkctimec.tcl 3fb5cad05922f5da61262cb6bcd5868a34e94a49ca8833ae2d7796e7df075576 x -F tool/mkkeywordhash.c 6b0be901c47f9ad42215fc995eb2f4384ac49213b1fba395102ec3e999acf559 +F tool/mkcombo.tcl 2a5189b219c4a495e1ff7fc980bd568d3cfb82ae9d50c84e77f7a161e96fc132 +F tool/mkctimec.tcl 68f3ee9e2e6a06524b25d79b5b61a883053466080a39da59e0a0697f0c4d9a3b x +F tool/mkfptab.c 24ea40113f96584caca3f6dd06b4ad5adffa0f39023910cb83fa7ef2ffa9b0ba +F tool/mkkeywordhash.c 82d5af1d0e677900739fba59155cddac172d8c712c2d91ab73d6e6bcb30060f0 F tool/mkmsvcmin.tcl d76c45efda1cce2d4005bcea7b8a22bb752e3256009f331120fb4fecb14ebb7a F tool/mkopcodec.tcl 33d20791e191df43209b77d37f0ff0904620b28465cca6990cf8d60da61a07ef F tool/mkopcodeh.tcl 2b4e6967a670ef21bf53a164964c35c6163277d002a4c6f56fa231d68c88d023 F tool/mkopts.tcl 680f785fdb09729fd9ac50632413da4eadbdf9071535e3f26d03795828ab07fa F tool/mkpragmatab.tcl 3801ce32f8c55fe63a3b279f231fb26c2c1a2ea9a09d2dd599239d87a609acec -F tool/mkshellc.tcl bab0a72a68384181a5706712dfdf6815f6526446d4e8aacace2de5e80cda91b2 +F tool/mkshellc.tcl da6918b128e928a8f0d663519e14829153e59465bd5eb596442e99fa10a411b7 F tool/mksourceid.c 36aa8020014aed0836fd13c51d6dc9219b0df1761d6b5f58ff5b616211b079b9 F tool/mksqlite3c-noext.tcl 351c55256213154cabb051a3c870ef9f4487de905015141ae50dc7578a901b84 F tool/mksqlite3c.tcl 7a268139158e5deef27a370bc2f8db6ccf100c1ad7ac5e5b23743c0fd354f609 @@ -2137,26 +2159,27 @@ F tool/mkvsix.tcl 67b40996a50f985a573278eea32fc5a5eb6110bdf14d33f1d8086e48c69e54 F tool/mkwinarm64ec.tcl 171f79234fa53552a129b360356df5599fdab15239caffb3d29c571292728033 F tool/offsets.c 8ed2b344d33f06e71366a9b93ccedaa38c096cc1dbd4c3c26ad08c6115285845 F tool/omittest-msvc.tcl d6b8f501ac1d7798c4126065030f89812379012cad98a1735d6d7221492abc08 -F tool/omittest.tcl bec70ef0e16255c8d9eb06ecd7edf823c07a60a836186cdbce3528fb34b67995 +F tool/omittest.tcl 436b7072e00e25e9b77145a9f67aa8e0eeabd186168827435fd03f8f981aac32 F tool/opcodesum.tcl 740ed206ba8c5040018988129abbf3089a0ccf4a F tool/pagesig.c f98909b4168d9cac11a2de7f031adea0e2f3131faa7515a72807c03ec58eafeb F tool/replace.tcl 511c61acfe563dfb58675efb4628bb158a13d48ff8322123ac447e9d25a82d9a F tool/restore_jrnl.tcl 1079ecba47cc82fa82115b81c1f68097ab1f956f357ee8da5fc4b2589af6bd98 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 -F tool/showdb.c 3956d71e5193162609a60e8c9edfcf09274c00cfea2b1d221261427adb2b5cca +F tool/showdb.c 1faa3661d2d634f206c76794cb21d89d3ea9082d07d5e983be0f025e40f21320 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818 F tool/showlocks.c 9cc5e66d4ebbf2d194f39db2527ece92077e86ae627ddd233ee48e16e8142564 F tool/showshm.c a0ab6ec32dd1f11218ca2a4018f8fb875b59414801ab8ceed8b2e69b7b45a809 F tool/showstat4.c b706fcbc4cd1a6e4a73ac32549afc4b460479d650402d64b23e8d813516e8de4 +F tool/showtmlog.c 2e9da6c4b4767113a0ad5ddabd4337ea100d38ff9c7fee260f9ccdefb2ffdc23 F tool/showwal.c 11eca547980a066b081f512636151233350ac679f29ecf4ebfce7f4530230b3d F tool/soak1.tcl a3892082ed1079671565c044e93b55c3c7f38829aedf53cc597c65d23ffdaddf F tool/spaceanal.tcl 1f83962090a6b60e1d7bf92495d643e622bef9fe82ea3f2d22350dcbce9a12d0 F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 4969fd642dad0ea483e4e104163021d92baf98f6a8eac981fe48525f9b873430 -F tool/sqldiff.c 134be7866be19f8beb32043d5aea5657f01aaeae2df8d33d758ff722c78666b9 +F tool/sqldiff.c 847edc1e0d1e1feb652d3d6128e504456deaf254ab9ad3e7cebd4317d2037182 F tool/sqlite3_analyzer.c.in 14f02cb5ec3c264cd6107d1f1dad77092b1cf440fc196c30b69ae87b56a1a43b -F tool/sqlite3_rsync.c d0e58a1e49fe2192c3ee0b697aed182d502bebfe5b4b406ba6b2baa52a04ecbe -F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 +F tool/sqlite3_rsync.c f510a8b230e1c5b0f62842acd0e94ff15d2f77a00ae782f7d20f9e39919fa19b +F tool/sqltclsh.c.in c103c6fc7d42bce611f9d4596774d60b7ef3d0b291a1f58c9e6184e458b89296 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 F tool/src-verify.c 6c655d9a8d6b30f3648fc78a79bf3838ed68f8543869d380c43ea9f17b3b8501 F tool/srcck1.c 559e703c6cca1d70398bdba1d7f91036c1a71adf718a1aaa6401a562ccaed154 @@ -2171,12 +2194,14 @@ F tool/vdbe-compress.tcl fa2f37ab39b2a0087fafb6a7f3ce19503e25e624ffa8ed9951717ab F tool/vdbe_profile.tcl 3ac5a4a9449f4baf77059358ea050db3e34395ccf59c5464d29b91746d5b961e F tool/version-info.c 33d0390ef484b3b1cb685d59362be891ea162123cea181cb8e6d2cf6ddf2700c F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 -F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 +F tool/warnings.sh a554d13f6e5cf3760f041b87939e3d616ec6961859c3245e8ef701d1eafc2ca2 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6ca269b677c6cdc03628fc4f3634ee1b9b956170f9bc41ba8d37b15cc6571a88 -R 6723709c10ad0e4cbb0f48842d99491e +F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c +P cda5dd9bcbef2135bb9855139a11d0e22a092f9498d82eb18e7d4401264b6eb8 +R 78f8c181262c6bdd7ab10b5492ff03ea +T +sym-major-release * T +sym-release * -T +sym-version-3.51.3 * +T +sym-version-3.53.0 * U drh -Z 8a6b03453203488391a91516890c1c7d +Z 5cc2d306947f7e965f5f798b08602c26 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.tags b/manifest.tags index cb78ee568..368ef71f6 100644 --- a/manifest.tags +++ b/manifest.tags @@ -1,4 +1,5 @@ -branch branch-3.51 +branch trunk +tag trunk tag release -tag branch-3.51 -tag version-3.51.3 +tag major-release +tag version-3.53.0 diff --git a/manifest.uuid b/manifest.uuid index bae34afb9..dd0205708 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -737ae4a34738ffa0c3ff7f9bb18df914dd1cad163f28fd6b6e114a344fe6d618 +4525003a53a7fc63ca75c59b22c79608659ca12f0131f52c18637f829977f20b diff --git a/src/alter.c b/src/alter.c index a7255e75e..fb5a37935 100644 --- a/src/alter.c +++ b/src/alter.c @@ -491,7 +491,7 @@ void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ /* Look up the table being altered. */ assert( pParse->pNewTable==0 ); assert( sqlite3BtreeHoldsAllMutexes(db) ); - if( db->mallocFailed ) goto exit_begin_add_column; + if( NEVER(db->mallocFailed) ) goto exit_begin_add_column; pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); if( !pTab ) goto exit_begin_add_column; @@ -563,7 +563,7 @@ void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ ** Or, if pTab is not a view or virtual table, zero is returned. */ #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) -static int isRealTable(Parse *pParse, Table *pTab, int bDrop){ +static int isRealTable(Parse *pParse, Table *pTab, int iOp){ const char *zType = 0; #ifndef SQLITE_OMIT_VIEW if( IsView(pTab) ){ @@ -576,9 +576,12 @@ static int isRealTable(Parse *pParse, Table *pTab, int bDrop){ } #endif if( zType ){ + const char *azMsg[] = { + "rename columns of", "drop column from", "edit constraints of" + }; + assert( iOp>=0 && iOp<ArraySize(azMsg) ); sqlite3ErrorMsg(pParse, "cannot %s %s \"%s\"", - (bDrop ? "drop column from" : "rename columns of"), - zType, pTab->zName + azMsg[iOp], zType, pTab->zName ); return 1; } @@ -1049,6 +1052,25 @@ static RenameToken *renameColumnTokenNext(RenameCtx *pCtx){ return pBest; } +/* +** Set the error message of the context passed as the first argument to +** the result of formatting zFmt using printf() style formatting. +*/ +static void errorMPrintf(sqlite3_context *pCtx, const char *zFmt, ...){ + sqlite3 *db = sqlite3_context_db_handle(pCtx); + char *zErr = 0; + va_list ap; + va_start(ap, zFmt); + zErr = sqlite3VMPrintf(db, zFmt, ap); + va_end(ap); + if( zErr ){ + sqlite3_result_error(pCtx, zErr, -1); + sqlite3DbFree(db, zErr); + }else{ + sqlite3_result_error_nomem(pCtx); + } +} + /* ** An error occurred while parsing or otherwise processing a database ** object (either pParse->pNewTable, pNewIndex or pNewTrigger) as part of an @@ -1346,8 +1368,8 @@ static int renameResolveTrigger(Parse *pParse){ sqlite3SelectPrep(pParse, pStep->pSelect, &sNC); if( pParse->nErr ) rc = pParse->rc; } - if( rc==SQLITE_OK && pStep->zTarget ){ - SrcList *pSrc = sqlite3TriggerStepSrc(pParse, pStep); + if( rc==SQLITE_OK && pStep->pSrc ){ + SrcList *pSrc = sqlite3SrcListDup(db, pStep->pSrc, 0); if( pSrc ){ Select *pSel = sqlite3SelectNew( pParse, pStep->pExprList, pSrc, 0, 0, 0, 0, 0, 0 @@ -1375,10 +1397,10 @@ static int renameResolveTrigger(Parse *pParse){ pSel->pSrc = 0; sqlite3SelectDelete(db, pSel); } - if( pStep->pFrom ){ + if( ALWAYS(pStep->pSrc) ){ int i; - for(i=0; i<pStep->pFrom->nSrc && rc==SQLITE_OK; i++){ - SrcItem *p = &pStep->pFrom->a[i]; + for(i=0; i<pStep->pSrc->nSrc && rc==SQLITE_OK; i++){ + SrcItem *p = &pStep->pSrc->a[i]; if( p->fg.isSubquery ){ assert( p->u4.pSubq!=0 ); sqlite3SelectPrep(pParse, p->u4.pSubq->pSelect, 0); @@ -1447,13 +1469,13 @@ static void renameWalkTrigger(Walker *pWalker, Trigger *pTrigger){ sqlite3WalkExpr(pWalker, pUpsert->pUpsertWhere); sqlite3WalkExpr(pWalker, pUpsert->pUpsertTargetWhere); } - if( pStep->pFrom ){ + if( pStep->pSrc ){ int i; - SrcList *pFrom = pStep->pFrom; - for(i=0; i<pFrom->nSrc; i++){ - if( pFrom->a[i].fg.isSubquery ){ - assert( pFrom->a[i].u4.pSubq!=0 ); - sqlite3WalkSelect(pWalker, pFrom->a[i].u4.pSubq->pSelect); + SrcList *pSrc = pStep->pSrc; + for(i=0; i<pSrc->nSrc; i++){ + if( pSrc->a[i].fg.isSubquery ){ + assert( pSrc->a[i].u4.pSubq!=0 ); + sqlite3WalkSelect(pWalker, pSrc->a[i].u4.pSubq->pSelect); } } } @@ -1624,8 +1646,8 @@ static void renameColumnFunc( if( rc!=SQLITE_OK ) goto renameColumnFunc_done; for(pStep=sParse.pNewTrigger->step_list; pStep; pStep=pStep->pNext){ - if( pStep->zTarget ){ - Table *pTarget = sqlite3LocateTable(&sParse, 0, pStep->zTarget, zDb); + if( pStep->pSrc ){ + Table *pTarget = sqlite3LocateTableItem(&sParse, 0, &pStep->pSrc->a[0]); if( pTarget==pTab ){ if( pStep->pUpsert ){ ExprList *pUpsertSet = pStep->pUpsert->pUpsertSet; @@ -1637,7 +1659,6 @@ static void renameColumnFunc( } } - /* Find tokens to edit in UPDATE OF clause */ if( sParse.pTriggerTab==pTab ){ renameColumnIdlistNames(&sParse, &sCtx,sParse.pNewTrigger->pColumns,zOld); @@ -1839,13 +1860,10 @@ static void renameTableFunc( if( rc==SQLITE_OK ){ renameWalkTrigger(&sWalker, pTrigger); for(pStep=pTrigger->step_list; pStep; pStep=pStep->pNext){ - if( pStep->zTarget && 0==sqlite3_stricmp(pStep->zTarget, zOld) ){ - renameTokenFind(&sParse, &sCtx, pStep->zTarget); - } - if( pStep->pFrom ){ + if( pStep->pSrc ){ int i; - for(i=0; i<pStep->pFrom->nSrc; i++){ - SrcItem *pItem = &pStep->pFrom->a[i]; + for(i=0; i<pStep->pSrc->nSrc; i++){ + SrcItem *pItem = &pStep->pSrc->a[i]; if( 0==sqlite3_stricmp(pItem->zName, zOld) ){ renameTokenFind(&sParse, &sCtx, pItem->zName); } @@ -2092,6 +2110,57 @@ static void renameTableTest( #endif } + +/* +** Return the number of bytes until the end of the next non-whitespace and +** non-comment token. For the purpose of this function, a "(" token includes +** all of the bytes through and including the matching ")", or until the +** first illegal token, whichever comes first. +** +** Write the token type into *piToken. +** +** The value returned is the number of bytes in the token itself plus +** the number of bytes of leading whitespace and comments skipped plus +** all bytes through the next matching ")" if the token is TK_LP. +** +** Example: (Note: '.' used in place of '*' in the example z[] text) +** +** ,--------- *piToken := TK_RP +** v +** z[] = " /.comment./ --comment\n (two three four) five" +** | | +** |<-------------------------------------->| +** | +** `--- return value +*/ +static int getConstraintToken(const u8 *z, int *piToken){ + int iOff = 0; + int t = 0; + do { + iOff += sqlite3GetToken(&z[iOff], &t); + }while( t==TK_SPACE || t==TK_COMMENT ); + + *piToken = t; + + if( t==TK_LP ){ + int nNest = 1; + while( nNest>0 ){ + iOff += sqlite3GetToken(&z[iOff], &t); + if( t==TK_LP ){ + nNest++; + }else if( t==TK_RP ){ + t = TK_LP; + nNest--; + }else if( t==TK_ILLEGAL ){ + break; + } + } + } + + *piToken = t; + return iOff; +} + /* ** The implementation of internal UDF sqlite_drop_column(). ** @@ -2136,15 +2205,24 @@ static void dropColumnFunc( goto drop_column_done; } - pCol = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol].zCnName); if( iCol<pTab->nCol-1 ){ RenameToken *pEnd; + pCol = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol].zCnName); pEnd = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol+1].zCnName); zEnd = (const char*)pEnd->t.z; }else{ + int eTok; assert( IsOrdinaryTable(pTab) ); + assert( iCol!=0 ); + /* Point pCol->t.z at the "," immediately preceding the definition of + ** the column being dropped. To do this, start at the name of the + ** previous column, and tokenize until the next ",". */ + pCol = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol-1].zCnName); + do { + pCol->t.z += getConstraintToken((const u8*)pCol->t.z, &eTok); + }while( eTok!=TK_COMMA ); + pCol->t.z--; zEnd = (const char*)&zSql[pTab->u.tab.addColOffset]; - while( ALWAYS(pCol->t.z[0]!=0) && pCol->t.z[0]!=',' ) pCol->t.z--; } zNew = sqlite3MPrintf(db, "%.*s%s", pCol->t.z-zSql, zSql, zEnd); @@ -2313,6 +2391,651 @@ void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const Token *pName){ sqlite3SrcListDelete(db, pSrc); } +/* +** Return the number of bytes of leading whitespace/comments in string z[]. +*/ +static int getWhitespace(const u8 *z){ + int nRet = 0; + while( 1 ){ + int t = 0; + int n = sqlite3GetToken(&z[nRet], &t); + if( t!=TK_SPACE && t!=TK_COMMENT ) break; + nRet += n; + } + return nRet; +} + + +/* +** Argument z points into the body of a constraint - specifically the +** second token of the constraint definition. For a named constraint, +** z points to the first token past the CONSTRAINT keyword. For an +** unnamed NOT NULL constraint, z points to the first byte past the NOT +** keyword. +** +** Return the number of bytes until the end of the constraint. +*/ +static int getConstraint(const u8 *z){ + int iOff = 0; + int t = 0; + + /* Now, the current constraint proceeds until the next occurence of one + ** of the following tokens: + ** + ** CONSTRAINT, PRIMARY, NOT, UNIQUE, CHECK, DEFAULT, + ** COLLATE, REFERENCES, FOREIGN, GENERATED, AS, RP, or COMMA + ** + ** Also exit the loop if ILLEGAL turns up. + */ + while( 1 ){ + int n = getConstraintToken(&z[iOff], &t); + if( t==TK_CONSTRAINT || t==TK_PRIMARY || t==TK_NOT || t==TK_UNIQUE + || t==TK_CHECK || t==TK_DEFAULT || t==TK_COLLATE || t==TK_REFERENCES + || t==TK_FOREIGN || t==TK_RP || t==TK_COMMA || t==TK_ILLEGAL + || t==TK_AS || t==TK_GENERATED + ){ + break; + } + iOff += n; + } + + return iOff; +} + +/* +** Compare two constraint names. +** +** Summary: *pRes := zQuote != zCmp +** +** Details: +** Compare the (possibly quoted) constraint name zQuote[0..nQuote-1] +** against zCmp[]. Write zero into *pRes if they are the same and +** non-zero if they differ. Normally return SQLITE_OK, except if there +** is an OOM, set the OOM error condition on ctx and return SQLITE_NOMEM. +*/ +static int quotedCompare( + sqlite3_context *ctx, /* Function context on which to report errors */ + int t, /* Token type */ + const u8 *zQuote, /* Possibly quoted text. Not zero-terminated. */ + int nQuote, /* Length of zQuote in bytes */ + const u8 *zCmp, /* Zero-terminated, unquoted name to compare against */ + int *pRes /* OUT: Set to 0 if equal, non-zero if unequal */ +){ + char *zCopy = 0; /* De-quoted, zero-terminated copy of zQuote[] */ + + if( t==TK_ILLEGAL ){ + *pRes = 1; + return SQLITE_OK; + } + zCopy = sqlite3MallocZero(nQuote+1); + if( zCopy==0 ){ + sqlite3_result_error_nomem(ctx); + return SQLITE_NOMEM_BKPT; + } + memcpy(zCopy, zQuote, nQuote); + sqlite3Dequote(zCopy); + *pRes = sqlite3_stricmp((const char*)zCopy, (const char*)zCmp); + sqlite3_free(zCopy); + return SQLITE_OK; +} + +/* +** zSql[] is a CREATE TABLE statement, supposedly. Find the offset +** into zSql[] of the first character past the first "(" and write +** that offset into *piOff and return SQLITE_OK. Or, if not found, +** set the SQLITE_CORRUPT error code and return SQLITE_ERROR. +*/ +static int skipCreateTable(sqlite3_context *ctx, const u8 *zSql, int *piOff){ + int iOff = 0; + + if( zSql==0 ) return SQLITE_ERROR; + + /* Jump past the "CREATE TABLE" bit. */ + while( 1 ){ + int t = 0; + iOff += sqlite3GetToken(&zSql[iOff], &t); + if( t==TK_LP ) break; + if( t==TK_ILLEGAL ){ + sqlite3_result_error_code(ctx, SQLITE_CORRUPT_BKPT); + return SQLITE_ERROR; + } + } + + *piOff = iOff; + return SQLITE_OK; +} + +/* +** Internal SQL function sqlite3_drop_constraint(): Given an input +** CREATE TABLE statement, return a revised CREATE TABLE statement +** with a constraint removed. Two forms, depending on the datatype +** of argv[2]: +** +** sqlite_drop_constraint(SQL, INT) -- Omit NOT NULL from the INT-th column +** sqlite_drop_constraint(SQL, TEXT) -- OMIT constraint with name TEXT +** +** In the first case, the left-most column is 0. +*/ +static void dropConstraintFunc( + sqlite3_context *ctx, + int NotUsed, + sqlite3_value **argv +){ + const u8 *zSql = sqlite3_value_text(argv[0]); + const u8 *zCons = 0; + int iNotNull = -1; + int ii; + int iOff = 0; + int iStart = 0; + int iEnd = 0; + char *zNew = 0; + int t = 0; + sqlite3 *db; + UNUSED_PARAMETER(NotUsed); + + if( zSql==0 ) return; + + /* Jump past the "CREATE TABLE" bit. */ + if( skipCreateTable(ctx, zSql, &iOff) ) return; + + if( sqlite3_value_type(argv[1])==SQLITE_INTEGER ){ + iNotNull = sqlite3_value_int(argv[1]); + }else{ + zCons = sqlite3_value_text(argv[1]); + } + + /* Search for the named constraint within column definitions. */ + for(ii=0; iEnd==0; ii++){ + + /* Now parse the column or table constraint definition. Search + ** for the token CONSTRAINT if this is a DROP CONSTRAINT command, or + ** NOT in the right column if this is a DROP NOT NULL. */ + while( 1 ){ + iStart = iOff; + iOff += getConstraintToken(&zSql[iOff], &t); + if( t==TK_CONSTRAINT && (zCons || iNotNull==ii) ){ + /* Check if this is the constraint we are searching for. */ + int nTok = 0; + int cmp = 1; + + /* Skip past any whitespace. */ + iOff += getWhitespace(&zSql[iOff]); + + /* Compare the next token - which may be quoted - with the name of + ** the constraint being dropped. */ + nTok = getConstraintToken(&zSql[iOff], &t); + if( zCons ){ + if( quotedCompare(ctx, t, &zSql[iOff], nTok, zCons, &cmp) ) return; + } + iOff += nTok; + + /* The next token is usually the first token of the constraint + ** definition. This is enough to tell the type of the constraint - + ** TK_NOT means it is a NOT NULL, TK_CHECK a CHECK constraint etc. + ** + ** There is also the chance that the next token is TK_CONSTRAINT + ** (or TK_DEFAULT or TK_COLLATE), for example if a table has been + ** created as follows: + ** + ** CREATE TABLE t1(cols, CONSTRAINT one CONSTRAINT two NOT NULL); + ** + ** In this case, allow the "CONSTRAINT one" bit to be dropped by + ** this command if that is what is requested, or to advance to + ** the next iteration of the loop with &zSql[iOff] still pointing + ** to the CONSTRAINT keyword. */ + nTok = getConstraintToken(&zSql[iOff], &t); + if( t==TK_CONSTRAINT || t==TK_DEFAULT || t==TK_COLLATE + || t==TK_COMMA || t==TK_RP || t==TK_GENERATED || t==TK_AS + ){ + t = TK_CHECK; + }else{ + iOff += nTok; + iOff += getConstraint(&zSql[iOff]); + } + + if( cmp==0 || (iNotNull>=0 && t==TK_NOT) ){ + if( t!=TK_NOT && t!=TK_CHECK ){ + errorMPrintf(ctx, "constraint may not be dropped: %s", zCons); + return; + } + iEnd = iOff; + break; + } + + }else if( t==TK_NOT && iNotNull==ii ){ + iEnd = iOff + getConstraint(&zSql[iOff]); + break; + }else if( t==TK_RP || t==TK_ILLEGAL ){ + iEnd = -1; + break; + }else if( t==TK_COMMA ){ + break; + } + } + } + + /* If the constraint has not been found it is an error. */ + if( iEnd<=0 ){ + if( zCons ){ + errorMPrintf(ctx, "no such constraint: %s", zCons); + }else{ + /* SQLite follows postgres in that a DROP NOT NULL on a column that is + ** not NOT NULL is not an error. So just return the original SQL here. */ + sqlite3_result_text(ctx, (const char*)zSql, -1, SQLITE_TRANSIENT); + } + }else{ + + /* Figure out if an extra space should be inserted after the constraint + ** is removed. And if an additional comma preceding the constraint + ** should be removed. */ + const char *zSpace = " "; + iEnd += getWhitespace(&zSql[iEnd]); + sqlite3GetToken(&zSql[iEnd], &t); + if( t==TK_RP || t==TK_COMMA ){ + zSpace = ""; + if( zSql[iStart-1]==',' ) iStart--; + } + + db = sqlite3_context_db_handle(ctx); + zNew = sqlite3MPrintf(db, "%.*s%s%s", iStart, zSql, zSpace, &zSql[iEnd]); + sqlite3_result_text(ctx, zNew, -1, SQLITE_DYNAMIC); + } +} + +/* +** Internal SQL function: +** +** sqlite_add_constraint(SQL, CONSTRAINT-TEXT, ICOL) +** +** SQL is a CREATE TABLE statement. Return a modified version of +** SQL that adds CONSTRAINT-TEXT at the end of the ICOL-th column +** definition. (The left-most column defintion is 0.) +*/ +static void addConstraintFunc( + sqlite3_context *ctx, + int NotUsed, + sqlite3_value **argv +){ + const u8 *zSql = sqlite3_value_text(argv[0]); + const char *zCons = (const char*)sqlite3_value_text(argv[1]); + int iCol = sqlite3_value_int(argv[2]); + int iOff = 0; + int ii; + char *zNew = 0; + int t = 0; + sqlite3 *db; + UNUSED_PARAMETER(NotUsed); + + if( skipCreateTable(ctx, zSql, &iOff) ) return; + + for(ii=0; ii<=iCol || (iCol<0 && t!=TK_RP); ii++){ + iOff += getConstraintToken(&zSql[iOff], &t); + while( 1 ){ + int nTok = getConstraintToken(&zSql[iOff], &t); + if( t==TK_COMMA || t==TK_RP ) break; + if( t==TK_ILLEGAL ){ + sqlite3_result_error_code(ctx, SQLITE_CORRUPT_BKPT); + return; + } + iOff += nTok; + } + } + + iOff += getWhitespace(&zSql[iOff]); + + db = sqlite3_context_db_handle(ctx); + if( iCol<0 ){ + zNew = sqlite3MPrintf(db, "%.*s, %s%s", iOff, zSql, zCons, &zSql[iOff]); + }else{ + zNew = sqlite3MPrintf(db, "%.*s %s%s", iOff, zSql, zCons, &zSql[iOff]); + } + sqlite3_result_text(ctx, zNew, -1, SQLITE_DYNAMIC); +} + +/* +** Find a column named pCol in table pTab. If successful, set output +** parameter *piCol to the index of the column in the table and return +** SQLITE_OK. Otherwise, set *piCol to -1 and return an SQLite error +** code. +*/ +static int alterFindCol(Parse *pParse, Table *pTab, Token *pCol, int *piCol){ + sqlite3 *db = pParse->db; + char *zName = sqlite3NameFromToken(db, pCol); + int rc = SQLITE_NOMEM; + int iCol = -1; + + if( zName ){ + iCol = sqlite3ColumnIndex(pTab, zName); + if( iCol<0 ){ + sqlite3ErrorMsg(pParse, "no such column: %s", zName); + rc = SQLITE_ERROR; + }else{ + rc = SQLITE_OK; + } + } + +#ifndef SQLITE_OMIT_AUTHORIZATION + if( rc==SQLITE_OK ){ + const char *zDb = db->aDb[sqlite3SchemaToIndex(db, pTab->pSchema)].zDbSName; + const char *zCol = pTab->aCol[iCol].zCnName; + if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, zCol) ){ + pTab = 0; + } + } +#endif + + sqlite3DbFree(db, zName); + *piCol = iCol; + return rc; +} + + +/* +** Find the table named by the first entry in source list pSrc. If successful, +** return a pointer to the Table structure and set output variable (*pzDb) +** to point to the name of the database containin the table (i.e. "main", +** "temp" or the name of an attached database). +** +** If the table cannot be located, return NULL. The value of the two output +** parameters is undefined in this case. +*/ +static Table *alterFindTable( + Parse *pParse, /* Parsing context */ + SrcList *pSrc, /* Name of the table to look for */ + int *piDb, /* OUT: write the iDb here */ + const char **pzDb, /* OUT: write name of schema here */ + int bAuth /* Do ALTER TABLE authorization checks if true */ +){ + sqlite3 *db = pParse->db; + Table *pTab = 0; + assert( sqlite3BtreeHoldsAllMutexes(db) ); + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); + if( pTab ){ + int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + *pzDb = db->aDb[iDb].zDbSName; + *piDb = iDb; + + if( SQLITE_OK!=isRealTable(pParse, pTab, 2) + || SQLITE_OK!=isAlterableTable(pParse, pTab) + ){ + pTab = 0; + } + } +#ifndef SQLITE_OMIT_AUTHORIZATION + if( pTab && bAuth ){ + if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, *pzDb, pTab->zName, 0) ){ + pTab = 0; + } + } +#endif + sqlite3SrcListDelete(db, pSrc); + return pTab; +} + +/* +** Generate bytecode for one of: +** +** (1) ALTER TABLE pSrc DROP CONSTRAINT pCons +** (2) ALTER TABLE pSrc ALTER pCol DROP NOT NULL +** +** One of pCons and pCol must be NULL and the other non-null. +*/ +void sqlite3AlterDropConstraint( + Parse *pParse, /* Parsing context */ + SrcList *pSrc, /* The table being altered */ + Token *pCons, /* Name of the constraint to drop */ + Token *pCol /* Name of the column from which to remove the NOT NULL */ +){ + sqlite3 *db = pParse->db; + Table *pTab = 0; + int iDb = 0; + const char *zDb = 0; + char *zArg = 0; + + assert( (pCol==0)!=(pCons==0) ); + assert( pSrc->nSrc==1 ); + pTab = alterFindTable(pParse, pSrc, &iDb, &zDb, pCons!=0); + if( !pTab ) return; + + if( pCons ){ + zArg = sqlite3MPrintf(db, "%.*Q", pCons->n, pCons->z); + }else{ + int iCol; + if( alterFindCol(pParse, pTab, pCol, &iCol) ) return; + zArg = sqlite3MPrintf(db, "%d", iCol); + } + + /* Edit the SQL for the named table. */ + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE " SET " + "sql = sqlite_drop_constraint(sql, %s) " + "WHERE type='table' AND tbl_name=%Q COLLATE nocase" + , zDb, zArg, pTab->zName + ); + sqlite3DbFree(db, zArg); + + /* Finally, reload the database schema. */ + renameReloadSchema(pParse, iDb, INITFLAG_AlterDropCons); +} + +/* +** The implementation of SQL function sqlite_fail(MSG). This takes a single +** argument, and returns it as an error message with the error code set to +** SQLITE_CONSTRAINT. +*/ +static void failConstraintFunc( + sqlite3_context *ctx, + int NotUsed, + sqlite3_value **argv +){ + const char *zText = (const char*)sqlite3_value_text(argv[0]); + int err = sqlite3_value_int(argv[1]); + (void)NotUsed; + sqlite3_result_error(ctx, zText, -1); + sqlite3_result_error_code(ctx, err); +} + +/* +** Buffer pCons, which is nCons bytes in size, contains the text of a +** NOT NULL or CHECK constraint that will be inserted into a CREATE TABLE +** statement. If successful, this function returns the size of the buffer in +** bytes not including any trailing whitespace or "--" style comments. Or, +** if an OOM occurs, it returns 0 and sets db->mallocFailed to true. +** +** C-style comments at the end are preserved. "--" style comments are +** removed because the comment terminator might be \000, and we are about +** to insert the pCons[] text into the middle of a larger string, and that +** will have the effect of removing the comment terminator and messing up +** the syntax. +*/ +static int alterRtrimConstraint( + sqlite3 *db, /* used to record OOM error */ + const char *pCons, /* Buffer containing constraint */ + int nCons /* Size of pCons in bytes */ +){ + u8 *zTmp = (u8*)sqlite3MPrintf(db, "%.*s", nCons, pCons); + int iOff = 0; + int iEnd = 0; + + if( zTmp==0 ) return 0; + + while( 1 ){ + int t = 0; + int nToken = sqlite3GetToken(&zTmp[iOff], &t); + if( t==TK_ILLEGAL ) break; + if( t!=TK_SPACE && (t!=TK_COMMENT || zTmp[iOff]!='-') ){ + iEnd = iOff+nToken; + } + iOff += nToken; + } + + sqlite3DbFree(db, zTmp); + return iEnd; +} + +/* +** Prepare a statement of the form: +** +** ALTER TABLE pSrc ALTER pCol SET NOT NULL +*/ +void sqlite3AlterSetNotNull( + Parse *pParse, /* Parsing context */ + SrcList *pSrc, /* Name of the table being altered */ + Token *pCol, /* Name of the column to add a NOT NULL constraint to */ + Token *pFirst /* The NOT token of the NOT NULL constraint text */ +){ + Table *pTab = 0; + int iCol = 0; + int iDb = 0; + const char *zDb = 0; + const char *pCons = 0; + int nCons = 0; + + /* Look up the table being altered. */ + assert( pSrc->nSrc==1 ); + pTab = alterFindTable(pParse, pSrc, &iDb, &zDb, 0); + if( !pTab ) return; + + /* Find the column being altered. */ + if( alterFindCol(pParse, pTab, pCol, &iCol) ){ + return; + } + + /* Find the length in bytes of the constraint definition */ + pCons = pFirst->z; + nCons = alterRtrimConstraint(pParse->db, pCons, pParse->sLastToken.z - pCons); + + /* Search for a constraint violation. Throw an exception if one is found. */ + sqlite3NestedParse(pParse, + "SELECT sqlite_fail('constraint failed', %d) " + "FROM %Q.%Q AS x WHERE x.%.*s IS NULL", + SQLITE_CONSTRAINT, zDb, pTab->zName, (int)pCol->n, pCol->z + ); + + /* Edit the SQL for the named table. */ + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE " SET " + "sql = sqlite_add_constraint(sqlite_drop_constraint(sql, %d), %.*Q, %d) " + "WHERE type='table' AND tbl_name=%Q COLLATE nocase" + , zDb, iCol, nCons, pCons, iCol, pTab->zName + ); + + /* Finally, reload the database schema. */ + renameReloadSchema(pParse, iDb, INITFLAG_AlterDropCons); +} + +/* +** Implementation of internal SQL function: +** +** sqlite_find_constraint(SQL, CONSTRAINT-NAME) +** +** This function returns true if the SQL passed as the first argument is a +** CREATE TABLE that contains a constraint with the name CONSTRAINT-NAME, +** or false otherwise. +*/ +static void findConstraintFunc( + sqlite3_context *ctx, + int NotUsed, + sqlite3_value **argv +){ + const u8 *zSql = 0; + const u8 *zCons = 0; + int iOff = 0; + int t = 0; + + (void)NotUsed; + zSql = sqlite3_value_text(argv[0]); + zCons = sqlite3_value_text(argv[1]); + + if( zSql==0 || zCons==0 ) return; + while( t!=TK_LP && t!=TK_ILLEGAL ){ + iOff += sqlite3GetToken(&zSql[iOff], &t); + } + + while( 1 ){ + iOff += getConstraintToken(&zSql[iOff], &t); + if( t==TK_CONSTRAINT ){ + int nTok = 0; + int cmp = 0; + iOff += getWhitespace(&zSql[iOff]); + nTok = getConstraintToken(&zSql[iOff], &t); + if( quotedCompare(ctx, t, &zSql[iOff], nTok, zCons, &cmp) ) return; + if( cmp==0 ){ + sqlite3_result_int(ctx, 1); + return; + } + }else if( t==TK_ILLEGAL ){ + break; + } + } + + sqlite3_result_int(ctx, 0); +} + +/* +** Generate bytecode to implement: +** +** ALTER TABLE pSrc ADD [CONSTRAINT pName] CHECK(pExpr) +** +** Any "ON CONFLICT" text that occurs after the "CHECK(...)", up +** until pParse->sLastToken, is included as part of the new constraint. +*/ +void sqlite3AlterAddConstraint( + Parse *pParse, /* Parse context */ + SrcList *pSrc, /* Table to add constraint to */ + Token *pFirst, /* First token of new constraint */ + Token *pName, /* Name of new constraint. NULL if name omitted. */ + const char *pExpr, /* Text of CHECK expression */ + int nExpr /* Size of pExpr in bytes */ +){ + Table *pTab = 0; /* Table identified by pSrc */ + int iDb = 0; /* Which schema does pTab live in */ + const char *zDb = 0; /* Name of the schema in which pTab lives */ + const char *pCons = 0; /* Text of the constraint */ + int nCons; /* Bytes of text to use from pCons[] */ + + /* Look up the table being altered. */ + assert( pSrc->nSrc==1 ); + pTab = alterFindTable(pParse, pSrc, &iDb, &zDb, 1); + if( !pTab ) return; + + /* If this new constraint has a name, check that it is not a duplicate of + ** an existing constraint. It is an error if it is. */ + if( pName ){ + char *zName = sqlite3NameFromToken(pParse->db, pName); + + sqlite3NestedParse(pParse, + "SELECT sqlite_fail('constraint %q already exists', %d) " + "FROM \"%w\"." LEGACY_SCHEMA_TABLE " " + "WHERE type='table' AND tbl_name=%Q COLLATE nocase " + "AND sqlite_find_constraint(sql, %Q)", + zName, SQLITE_ERROR, zDb, pTab->zName, zName + ); + sqlite3DbFree(pParse->db, zName); + } + + /* Search for a constraint violation. Throw an exception if one is found. */ + sqlite3NestedParse(pParse, + "SELECT sqlite_fail('constraint failed', %d) " + "FROM %Q.%Q WHERE (%.*s) IS NOT TRUE", + SQLITE_CONSTRAINT, zDb, pTab->zName, nExpr, pExpr + ); + + /* Edit the SQL for the named table. */ + pCons = pFirst->z; + nCons = alterRtrimConstraint(pParse->db, pCons, pParse->sLastToken.z - pCons); + + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE " SET " + "sql = sqlite_add_constraint(sql, %.*Q, -1) " + "WHERE type='table' AND tbl_name=%Q COLLATE nocase" + , zDb, nCons, pCons, pTab->zName + ); + + /* Finally, reload the database schema. */ + renameReloadSchema(pParse, iDb, INITFLAG_AlterDropCons); +} + /* ** Register built-in functions used to help implement ALTER TABLE */ @@ -2323,6 +3046,10 @@ void sqlite3AlterFunctions(void){ INTERNAL_FUNCTION(sqlite_rename_test, 7, renameTableTest), INTERNAL_FUNCTION(sqlite_drop_column, 3, dropColumnFunc), INTERNAL_FUNCTION(sqlite_rename_quotefix,2, renameQuotefixFunc), + INTERNAL_FUNCTION(sqlite_drop_constraint,2, dropConstraintFunc), + INTERNAL_FUNCTION(sqlite_fail, 2, failConstraintFunc), + INTERNAL_FUNCTION(sqlite_add_constraint, 3, addConstraintFunc), + INTERNAL_FUNCTION(sqlite_find_constraint,2, findConstraintFunc), }; sqlite3InsertBuiltinFuncs(aAlterTableFuncs, ArraySize(aAlterTableFuncs)); } diff --git a/src/attach.c b/src/attach.c index 085e1b0ec..f27c1e6be 100644 --- a/src/attach.c +++ b/src/attach.c @@ -596,7 +596,7 @@ int sqlite3FixTriggerStep( if( sqlite3WalkSelect(&pFix->w, pStep->pSelect) || sqlite3WalkExpr(&pFix->w, pStep->pWhere) || sqlite3WalkExprList(&pFix->w, pStep->pExprList) - || sqlite3FixSrcList(pFix, pStep->pFrom) + || sqlite3FixSrcList(pFix, pStep->pSrc) ){ return 1; } diff --git a/src/auth.c b/src/auth.c index 9ec2e7d04..1088f844a 100644 --- a/src/auth.c +++ b/src/auth.c @@ -78,7 +78,7 @@ int sqlite3_set_authorizer( sqlite3_mutex_enter(db->mutex); db->xAuth = (sqlite3_xauth)xAuth; db->pAuthArg = pArg; - if( db->xAuth ) sqlite3ExpirePreparedStatements(db, 1); + sqlite3ExpirePreparedStatements(db, 1); sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } diff --git a/src/btree.c b/src/btree.c index 8d0f92222..66a423830 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1262,7 +1262,7 @@ static void btreeParseCellPtr( CellInfo *pInfo /* Fill in this structure */ ){ u8 *pIter; /* For scanning through pCell */ - u32 nPayload; /* Number of bytes of cell payload */ + u64 nPayload; /* Number of bytes of cell payload */ u64 iKey; /* Extracted Key value */ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); @@ -1284,6 +1284,7 @@ static void btreeParseCellPtr( do{ nPayload = (nPayload<<7) | (*++pIter & 0x7f); }while( (*pIter)>=0x80 && pIter<pEnd ); + nPayload &= 0xffffffff; } pIter++; @@ -1327,11 +1328,10 @@ static void btreeParseCellPtr( pIter++; pInfo->nKey = *(i64*)&iKey; - pInfo->nPayload = nPayload; + pInfo->nPayload = (u32)nPayload; pInfo->pPayload = pIter; testcase( nPayload==pPage->maxLocal ); testcase( nPayload==(u32)pPage->maxLocal+1 ); - assert( nPayload>=0 ); assert( pPage->maxLocal <= BT_MAX_LOCAL ); if( nPayload<=pPage->maxLocal ){ /* This is the (easy) common case where the entire payload fits @@ -3673,6 +3673,30 @@ static SQLITE_NOINLINE int btreeBeginTrans( } #endif +#ifdef SQLITE_EXPERIMENTAL_PRAGMA_20251114 + /* If both a read and write transaction will be opened by this call, + ** then issue a file-control as if the following pragma command had + ** been evaluated: + ** + ** PRAGMA experimental_pragma_20251114 = 1|2 + ** + ** where the RHS is "1" if wrflag is 1 (RESERVED lock), or "2" if wrflag + ** is 2 (EXCLUSIVE lock). Ignore any result or error returned by the VFS. + ** + ** WARNING: This code will likely remain part of SQLite only temporarily - + ** it exists to allow users to experiment with certain types of blocking + ** locks in custom VFS implementations. It MAY BE REMOVED AT ANY TIME. */ + if( pBt->pPage1==0 && wrflag ){ + sqlite3_file *fd = sqlite3PagerFile(pPager); + char *aFcntl[3] = {0,0,0}; + aFcntl[1] = "experimental_pragma_20251114"; + assert( wrflag==1 || wrflag==2 ); + aFcntl[2] = (wrflag==1 ? "1" : "2"); + sqlite3OsFileControlHint(fd, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); + sqlite3_free(aFcntl[0]); + } +#endif + /* Call lockBtree() until either pBt->pPage1 is populated or ** lockBtree() returns something other than SQLITE_OK. lockBtree() ** may return SQLITE_OK but leave pBt->pPage1 set to 0 if after @@ -5677,7 +5701,7 @@ int sqlite3BtreeIsEmpty(BtCursor *pCur, int *pRes){ assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - if( pCur->eState==CURSOR_VALID ){ + if( NEVER(pCur->eState==CURSOR_VALID) ){ *pRes = 0; return SQLITE_OK; } @@ -9758,7 +9782,7 @@ int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ }while( rc==SQLITE_OK && nOut>0 ); if( rc==SQLITE_OK && nRem>0 && ALWAYS(pPgnoOut) ){ - Pgno pgnoNew; + Pgno pgnoNew = 0; /* Prevent harmless static-analyzer warning */ MemPage *pNew = 0; rc = allocateBtreePage(pBt, &pNew, &pgnoNew, 0, 0); put4byte(pPgnoOut, pgnoNew); diff --git a/src/build.c b/src/build.c index de890c2e9..6e06e6604 100644 --- a/src/build.c +++ b/src/build.c @@ -486,6 +486,7 @@ Table *sqlite3LocateTableItem( const char *zDb; if( p->fg.fixedSchema ){ int iDb = sqlite3SchemaToIndex(pParse->db, p->u4.pSchema); + assert( iDb>=0 && iDb<pParse->db->nDb ); zDb = pParse->db->aDb[iDb].zDbSName; }else{ assert( !p->fg.isSubquery ); @@ -2059,8 +2060,8 @@ void sqlite3ChangeCookie(Parse *pParse, int iDb){ ** The estimate is conservative. It might be larger that what is ** really needed. */ -static int identLength(const char *z){ - int n; +static i64 identLength(const char *z){ + i64 n; for(n=0; *z; n++, z++){ if( *z=='"' ){ n++; } } @@ -2493,9 +2494,10 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ if( !hasColumn(pPk->aiColumn, j, i) && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){ + const char *zColl = sqlite3ColumnColl(&pTab->aCol[i]); assert( j<pPk->nColumn ); pPk->aiColumn[j] = i; - pPk->azColl[j] = sqlite3StrBINARY; + pPk->azColl[j] = zColl ? zColl : sqlite3StrBINARY; j++; } } @@ -2570,13 +2572,14 @@ void sqlite3MarkAllShadowTablesOf(sqlite3 *db, Table *pTab){ ** restored to its original value prior to this routine returning. */ int sqlite3ShadowTableName(sqlite3 *db, const char *zName){ - char *zTail; /* Pointer to the last "_" in zName */ + const char *zTail; /* Pointer to the last "_" in zName */ Table *pTab; /* Table that zName is a shadow of */ + char *zCopy; zTail = strrchr(zName, '_'); if( zTail==0 ) return 0; - *zTail = 0; - pTab = sqlite3FindTable(db, zName, 0); - *zTail = '_'; + zCopy = sqlite3DbStrNDup(db, zName, (int)(zTail-zName)); + pTab = zCopy ? sqlite3FindTable(db, zCopy, 0) : 0; + sqlite3DbFree(db, zCopy); if( pTab==0 ) return 0; if( !IsVirtual(pTab) ) return 0; return sqlite3IsShadowTableOf(db, pTab, zName); @@ -2729,6 +2732,7 @@ void sqlite3EndTable( convertToWithoutRowidTable(pParse, p); } iDb = sqlite3SchemaToIndex(db, p->pSchema); + assert( iDb>=0 && iDb<=db->nDb ); #ifndef SQLITE_OMIT_CHECK /* Resolve names in all CHECK constraint expressions. @@ -3024,6 +3028,7 @@ void sqlite3CreateView( sqlite3TwoPartName(pParse, pName1, pName2, &pName); iDb = sqlite3SchemaToIndex(db, p->pSchema); + assert( iDb>=0 && iDb<db->nDb ); sqlite3FixInit(&sFix, pParse, iDb, "view", pName); if( sqlite3FixSelect(&sFix, pSelect) ) goto create_view_fail; @@ -4620,6 +4625,7 @@ void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists){ goto exit_drop_index; } iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); + assert( iDb>=0 && iDb<db->nDb ); #ifndef SQLITE_OMIT_AUTHORIZATION { int code = SQLITE_DROP_INDEX; @@ -5517,8 +5523,7 @@ void sqlite3RowidConstraint( } /* -** Check to see if pIndex uses the collating sequence pColl. Return -** true if it does and false if it does not. +** Return true if any column of pIndex uses the zColl collation */ #ifndef SQLITE_OMIT_REINDEX static int collationMatch(const char *zColl, Index *pIndex){ @@ -5526,8 +5531,8 @@ static int collationMatch(const char *zColl, Index *pIndex){ assert( zColl!=0 ); for(i=0; i<pIndex->nColumn; i++){ const char *z = pIndex->azColl[i]; - assert( z!=0 || pIndex->aiColumn[i]<0 ); - if( pIndex->aiColumn[i]>=0 && 0==sqlite3StrICmp(z, zColl) ){ + assert( z!=0 ); + if( 0==sqlite3StrICmp(z, zColl) ){ return 1; } } @@ -5535,73 +5540,39 @@ static int collationMatch(const char *zColl, Index *pIndex){ } #endif -/* -** Recompute all indices of pTab that use the collating sequence pColl. -** If pColl==0 then recompute all indices of pTab. -*/ -#ifndef SQLITE_OMIT_REINDEX -static void reindexTable(Parse *pParse, Table *pTab, char const *zColl){ - if( !IsVirtual(pTab) ){ - Index *pIndex; /* An index associated with pTab */ - - for(pIndex=pTab->pIndex; pIndex; pIndex=pIndex->pNext){ - if( zColl==0 || collationMatch(zColl, pIndex) ){ - int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - sqlite3BeginWriteOperation(pParse, 0, iDb); - sqlite3RefillIndex(pParse, pIndex, -1); - } - } - } -} -#endif - -/* -** Recompute all indices of all tables in all databases where the -** indices use the collating sequence pColl. If pColl==0 then recompute -** all indices everywhere. -*/ -#ifndef SQLITE_OMIT_REINDEX -static void reindexDatabases(Parse *pParse, char const *zColl){ - Db *pDb; /* A single database */ - int iDb; /* The database index number */ - sqlite3 *db = pParse->db; /* The database connection */ - HashElem *k; /* For looping over tables in pDb */ - Table *pTab; /* A table in the database */ - - assert( sqlite3BtreeHoldsAllMutexes(db) ); /* Needed for schema access */ - for(iDb=0, pDb=db->aDb; iDb<db->nDb; iDb++, pDb++){ - assert( pDb!=0 ); - for(k=sqliteHashFirst(&pDb->pSchema->tblHash); k; k=sqliteHashNext(k)){ - pTab = (Table*)sqliteHashData(k); - reindexTable(pParse, pTab, zColl); - } - } -} -#endif - /* ** Generate code for the REINDEX command. ** ** REINDEX -- 1 ** REINDEX <collation> -- 2 -** REINDEX ?<database>.?<tablename> -- 3 -** REINDEX ?<database>.?<indexname> -- 4 +** REINDEX ?<database>.?<indexname> -- 3 +** REINDEX ?<database>.?<tablename> -- 4 +** REINDEX EXPRESSIONS -- 5 ** -** Form 1 causes all indices in all attached databases to be rebuilt. -** Form 2 rebuilds all indices in all databases that use the named +** Form 1 causes all indexes in all attached databases to be rebuilt. +** Form 2 rebuilds all indexes in all databases that use the named ** collating function. Forms 3 and 4 rebuild the named index or all -** indices associated with the named table. +** indexes associated with the named table, respectively. Form 5 +** rebuilds all expression indexes in addition to all collations, +** indexes, or tables named "EXPRESSIONS". +** +** If the name is ambiguous such that it matches two or more of +** forms 2 through 5, then rebuild the union of all matching indexes, +** taken care to avoid rebuilding the same index more than once. */ #ifndef SQLITE_OMIT_REINDEX void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ - CollSeq *pColl; /* Collating sequence to be reindexed, or NULL */ - char *z; /* Name of a table or index */ - const char *zDb; /* Name of the database */ - Table *pTab; /* A table in the database */ - Index *pIndex; /* An index associated with pTab */ - int iDb; /* The database index number */ + char *z = 0; /* Name of a table or index or collation */ + const char *zDb = 0; /* Name of the database */ + int iReDb = -1; /* The database index number */ sqlite3 *db = pParse->db; /* The database connection */ Token *pObjName; /* Name of the table or index to be reindexed */ + int bMatch = 0; /* At least one name match */ + const char *zColl = 0; /* Rebuild indexes using this collation */ + Table *pReTab = 0; /* Rebuild all indexes of this table */ + Index *pReIndex = 0; /* Rebuild this index */ + int isExprIdx = 0; /* Rebuild all expression indexes */ + int bAll = 0; /* Rebuild all indexes */ /* Read the database schema. If an error occurs, leave an error message ** and code in pParse and return NULL. */ @@ -5610,41 +5581,66 @@ void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ } if( pName1==0 ){ - reindexDatabases(pParse, 0); - return; + /* rebuild all indexes */ + bMatch = 1; + bAll = 1; }else if( NEVER(pName2==0) || pName2->z==0 ){ - char *zColl; assert( pName1->z ); - zColl = sqlite3NameFromToken(pParse->db, pName1); - if( !zColl ) return; - pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); - if( pColl ){ - reindexDatabases(pParse, zColl); - sqlite3DbFree(db, zColl); - return; + z = sqlite3NameFromToken(pParse->db, pName1); + if( z==0 ) return; + }else{ + iReDb = sqlite3TwoPartName(pParse, pName1, pName2, &pObjName); + if( iReDb<0 ) return; + z = sqlite3NameFromToken(db, pObjName); + if( z==0 ) return; + zDb = db->aDb[iReDb].zDbSName; + } + if( !bAll ){ + if( zDb==0 && sqlite3StrICmp(z, "expressions")==0 ){ + isExprIdx = 1; + bMatch = 1; + } + if( zDb==0 && sqlite3FindCollSeq(db, ENC(db), z, 0)!=0 ){ + zColl = z; + bMatch = 1; + } + if( zColl==0 && (pReTab = sqlite3FindTable(db, z, zDb))!=0 ){ + bMatch = 1; + } + if( zColl==0 && (pReIndex = sqlite3FindIndex(db, z, zDb))!=0 ){ + bMatch = 1; } - sqlite3DbFree(db, zColl); } - iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pObjName); - if( iDb<0 ) return; - z = sqlite3NameFromToken(db, pObjName); - if( z==0 ) return; - zDb = pName2->n ? db->aDb[iDb].zDbSName : 0; - pTab = sqlite3FindTable(db, z, zDb); - if( pTab ){ - reindexTable(pParse, pTab, 0); - sqlite3DbFree(db, z); - return; + if( bMatch ){ + int iDb; + HashElem *k; + Table *pTab; + Index *pIdx; + Db *pDb; + for(iDb=0, pDb=db->aDb; iDb<db->nDb; iDb++, pDb++){ + assert( pDb!=0 ); + if( iReDb>=0 && iReDb!=iDb ) continue; + for(k=sqliteHashFirst(&pDb->pSchema->tblHash); k; k=sqliteHashNext(k)){ + pTab = (Table*)sqliteHashData(k); + if( IsVirtual(pTab) ) continue; + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( bAll + || pTab==pReTab + || pIdx==pReIndex + || (isExprIdx && pIdx->bHasExpr) + || (zColl!=0 && collationMatch(zColl,pIdx)) + ){ + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3RefillIndex(pParse, pIdx, -1); + } + } /* End loop over indexes of pTab */ + } /* End loop over tables of iDb */ + } /* End loop over databases */ + }else{ + sqlite3ErrorMsg(pParse, "unable to identify the object to be reindexed"); } - pIndex = sqlite3FindIndex(db, z, zDb); sqlite3DbFree(db, z); - if( pIndex ){ - iDb = sqlite3SchemaToIndex(db, pIndex->pTable->pSchema); - sqlite3BeginWriteOperation(pParse, 0, iDb); - sqlite3RefillIndex(pParse, pIndex, -1); - return; - } - sqlite3ErrorMsg(pParse, "unable to identify the object to be reindexed"); + return; } #endif diff --git a/src/carray.c b/src/carray.c index 154d107dd..ff0691a85 100644 --- a/src/carray.c +++ b/src/carray.c @@ -79,6 +79,7 @@ struct carray_bind { int nData; /* Number of elements */ int mFlags; /* Control flags */ void (*xDel)(void*); /* Destructor for aData */ + void *pDel; /* Alternative argument to xDel() */ }; @@ -411,7 +412,7 @@ static sqlite3_module carrayModule = { static void carrayBindDel(void *pPtr){ carray_bind *p = (carray_bind*)pPtr; if( p->xDel!=SQLITE_STATIC ){ - p->xDel(p->aData); + p->xDel(p->pDel); } sqlite3_free(p); } @@ -419,14 +420,26 @@ static void carrayBindDel(void *pPtr){ /* ** Invoke this interface in order to bind to the single-argument ** version of CARRAY(). +** +** pStmt The prepared statement to which to bind +** idx The index of the parameter of pStmt to which to bind +** aData The data to be bound +** nData The number of elements in aData +** mFlags One of SQLITE_CARRAY_xxxx indicating datatype of aData +** xDestroy Destructor for pDestroy or aData if pDestroy==NULL. +** pDestroy Invoke xDestroy on this pointer if not NULL +** +** The destructor is called pDestroy if pDestroy!=NULL, or against +** aData if pDestroy==NULL. */ -SQLITE_API int sqlite3_carray_bind( +SQLITE_API int sqlite3_carray_bind_v2( sqlite3_stmt *pStmt, int idx, void *aData, int nData, int mFlags, - void (*xDestroy)(void*) + void (*xDestroy)(void*), + void *pDestroy ){ carray_bind *pNew = 0; int i; @@ -503,20 +516,38 @@ SQLITE_API int sqlite3_carray_bind( memcpy(pNew->aData, aData, sz); } pNew->xDel = sqlite3_free; + pNew->pDel = pNew->aData; }else{ pNew->aData = aData; pNew->xDel = xDestroy; + pNew->pDel = pDestroy; } return sqlite3_bind_pointer(pStmt, idx, pNew, "carray-bind", carrayBindDel); carray_bind_error: if( xDestroy!=SQLITE_STATIC && xDestroy!=SQLITE_TRANSIENT ){ - xDestroy(aData); + xDestroy(pDestroy); } sqlite3_free(pNew); return rc; } +/* +** Invoke this interface in order to bind to the single-argument +** version of CARRAY(). Same as sqlite3_carray_bind_v2() with the +** pDestroy parameter set to NULL. +*/ +SQLITE_API int sqlite3_carray_bind( + sqlite3_stmt *pStmt, + int idx, + void *aData, + int nData, + int mFlags, + void (*xDestroy)(void*) +){ + return sqlite3_carray_bind_v2(pStmt,idx,aData,nData,mFlags,xDestroy,aData); +} + /* ** Invoke this routine to register the carray() function. */ diff --git a/src/date.c b/src/date.c index 5e7ae6f1f..58a7ce544 100644 --- a/src/date.c +++ b/src/date.c @@ -429,7 +429,7 @@ static int parseDateOrTime( return 0; }else if( sqlite3StrICmp(zDate,"now")==0 && sqlite3NotPureFunc(context) ){ return setDateTimeToCurrent(context, p); - }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8)>0 ){ + }else if( sqlite3AtoF(zDate, &r)>0 ){ setRawDateNumber(p, r); return 0; }else if( (sqlite3StrICmp(zDate,"subsec")==0 @@ -875,7 +875,7 @@ static int parseModifier( ** date is already on the appropriate weekday, this is a no-op. */ if( sqlite3_strnicmp(z, "weekday ", 8)==0 - && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8)>0 + && sqlite3AtoF(&z[8], &r)>0 && r>=0.0 && r<7.0 && (n=(int)r)==r ){ sqlite3_int64 Z; computeYMD_HMS(p); @@ -946,9 +946,11 @@ static int parseModifier( case '8': case '9': { double rRounder; - int i; + int i, rx; int Y,M,D,h,m,x; const char *z2 = z; + char *zCopy; + sqlite3 *db = sqlite3_context_db_handle(pCtx); char z0 = z[0]; for(n=1; z[n]; n++){ if( z[n]==':' ) break; @@ -958,7 +960,11 @@ static int parseModifier( if( n==6 && getDigits(&z[1], "50f", &Y)==1 ) break; } } - if( sqlite3AtoF(z, &r, n, SQLITE_UTF8)<=0 ){ + zCopy = sqlite3DbStrNDup(db, z, n); + if( zCopy==0 ) break; + rx = sqlite3AtoF(zCopy, &r)<=0; + sqlite3DbFree(db, zCopy); + if( rx ){ assert( rc==1 ); break; } @@ -1778,7 +1784,7 @@ static void datedebugFunc( char *zJson; zJson = sqlite3_mprintf( "{iJD:%lld,Y:%d,M:%d,D:%d,h:%d,m:%d,tz:%d," - "s:%.3f,validJD:%d,validYMS:%d,validHMS:%d," + "s:%.3f,validJD:%d,validYMD:%d,validHMS:%d," "nFloor:%d,rawS:%d,isError:%d,useSubsec:%d," "isUtc:%d,isLocal:%d}", x.iJD, x.Y, x.M, x.D, x.h, x.m, x.tz, diff --git a/src/delete.c b/src/delete.c index 8fac7c2f3..95020c4df 100644 --- a/src/delete.c +++ b/src/delete.c @@ -86,7 +86,7 @@ static int vtabIsReadOnly(Parse *pParse, Table *pTab){ ** * Only allow DELETE, INSERT, or UPDATE of non-SQLITE_VTAB_INNOCUOUS ** virtual tables if PRAGMA trusted_schema=ON. */ - if( pParse->pToplevel!=0 + if( (pParse->pToplevel!=0 || (pParse->prepFlags & SQLITE_PREPARE_FROM_DDL)) && pTab->u.vtab.p->eVtabRisk > ((pParse->db->flags & SQLITE_TrustedSchema)!=0) ){ @@ -923,8 +923,9 @@ void sqlite3GenerateRowIndexDelete( r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 1, &iPartIdxLabel, pPrior, r1); sqlite3VdbeAddOp3(v, OP_IdxDelete, iIdxCur+i, r1, - pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); - sqlite3VdbeChangeP5(v, 1); /* Cause IdxDelete to error if no entry found */ + pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn + ); + sqlite3VdbeChangeP4(v, -1, (const char*)pIdx, P4_INDEX); sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); pPrior = pIdx; } diff --git a/src/expr.c b/src/expr.c index fdc05366c..9bac0ba7a 100644 --- a/src/expr.c +++ b/src/expr.c @@ -935,34 +935,22 @@ Expr *sqlite3ExprAlloc( int dequote /* True to dequote */ ){ Expr *pNew; - int nExtra = 0; - int iValue = 0; + int nExtra = pToken ? pToken->n+1 : 0; assert( db!=0 ); - if( pToken ){ - if( op!=TK_INTEGER || pToken->z==0 - || sqlite3GetInt32(pToken->z, &iValue)==0 ){ - nExtra = pToken->n+1; /* tag-20240227-a */ - assert( iValue>=0 ); - } - } pNew = sqlite3DbMallocRawNN(db, sizeof(Expr)+nExtra); if( pNew ){ memset(pNew, 0, sizeof(Expr)); pNew->op = (u8)op; pNew->iAgg = -1; - if( pToken ){ - if( nExtra==0 ){ - pNew->flags |= EP_IntValue|EP_Leaf|(iValue?EP_IsTrue:EP_IsFalse); - pNew->u.iValue = iValue; - }else{ - pNew->u.zToken = (char*)&pNew[1]; - assert( pToken->z!=0 || pToken->n==0 ); - if( pToken->n ) memcpy(pNew->u.zToken, pToken->z, pToken->n); - pNew->u.zToken[pToken->n] = 0; - if( dequote && sqlite3Isquote(pNew->u.zToken[0]) ){ - sqlite3DequoteExpr(pNew); - } + if( nExtra ){ + assert( pToken!=0 ); + pNew->u.zToken = (char*)&pNew[1]; + assert( pToken->z!=0 || pToken->n==0 ); + if( pToken->n ) memcpy(pNew->u.zToken, pToken->z, pToken->n); + pNew->u.zToken[pToken->n] = 0; + if( dequote && sqlite3Isquote(pNew->u.zToken[0]) ){ + sqlite3DequoteExpr(pNew); } } #if SQLITE_MAX_EXPR_DEPTH>0 @@ -987,6 +975,24 @@ Expr *sqlite3Expr( return sqlite3ExprAlloc(db, op, &x, 0); } +/* +** Allocate an expression for a 32-bit signed integer literal. +*/ +Expr *sqlite3ExprInt32(sqlite3 *db, int iVal){ + Expr *pNew = sqlite3DbMallocRawNN(db, sizeof(Expr)); + if( pNew ){ + memset(pNew, 0, sizeof(Expr)); + pNew->op = TK_INTEGER; + pNew->iAgg = -1; + pNew->flags = EP_IntValue|EP_Leaf|(iVal?EP_IsTrue:EP_IsFalse); + pNew->u.iValue = iVal; +#if SQLITE_MAX_EXPR_DEPTH>0 + pNew->nHeight = 1; +#endif + } + return pNew; +} + /* ** Attach subtrees pLeft and pRight to the Expr node pRoot. ** @@ -1149,7 +1155,7 @@ Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ ){ sqlite3ExprDeferredDelete(pParse, pLeft); sqlite3ExprDeferredDelete(pParse, pRight); - return sqlite3Expr(db, TK_INTEGER, "0"); + return sqlite3ExprInt32(db, 0); }else{ return sqlite3PExpr(pParse, TK_AND, pLeft, pRight); } @@ -1274,7 +1280,9 @@ void sqlite3ExprFunctionUsable( ){ assert( !IN_RENAME_OBJECT ); assert( (pDef->funcFlags & (SQLITE_FUNC_DIRECT|SQLITE_FUNC_UNSAFE))!=0 ); - if( ExprHasProperty(pExpr, EP_FromDDL) ){ + if( ExprHasProperty(pExpr, EP_FromDDL) + || pParse->prepFlags & SQLITE_PREPARE_FROM_DDL + ){ if( (pDef->funcFlags & SQLITE_FUNC_DIRECT)!=0 || (pParse->db->flags & SQLITE_TrustedSchema)==0 ){ @@ -1970,9 +1978,7 @@ Select *sqlite3SelectDup(sqlite3 *db, const Select *pDup, int flags){ pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); pNew->iLimit = 0; pNew->iOffset = 0; - pNew->selFlags = p->selFlags & ~(u32)SF_UsesEphemeral; - pNew->addrOpenEphm[0] = -1; - pNew->addrOpenEphm[1] = -1; + pNew->selFlags = p->selFlags; pNew->nSelectRow = p->nSelectRow; pNew->pWith = sqlite3WithDup(db, p->pWith); #ifndef SQLITE_OMIT_WINDOWFUNC @@ -2624,7 +2630,7 @@ static int exprIsConst(Parse *pParse, Expr *p, int initFlag){ /* ** Walk an expression tree. Return non-zero if the expression is constant -** and 0 if it involves variables or function calls. +** or return zero if the expression involves variables or function calls. ** ** For the purposes of this function, a double-quoted string (ex: "abc") ** is considered a variable but a single-quoted string (ex: 'abc') is @@ -3414,6 +3420,7 @@ int sqlite3FindInIndex( */ u32 savedNQueryLoop = pParse->nQueryLoop; int rMayHaveNull = 0; + int bloomOk = (inFlags & IN_INDEX_MEMBERSHIP)!=0; eType = IN_INDEX_EPH; if( inFlags & IN_INDEX_LOOP ){ pParse->nQueryLoop = 0; @@ -3421,7 +3428,13 @@ int sqlite3FindInIndex( *prRhsHasNull = rMayHaveNull = ++pParse->nMem; } assert( pX->op==TK_IN ); - sqlite3CodeRhsOfIN(pParse, pX, iTab); + if( !bloomOk + && ExprUseXSelect(pX) + && (pX->x.pSelect->selFlags & SF_ClonedRhsIn)!=0 + ){ + bloomOk = 1; + } + sqlite3CodeRhsOfIN(pParse, pX, iTab, bloomOk); if( rMayHaveNull ){ sqlite3SetHasNullFlag(v, iTab, rMayHaveNull); } @@ -3579,7 +3592,8 @@ static int findCompatibleInRhsSubrtn( void sqlite3CodeRhsOfIN( Parse *pParse, /* Parsing context */ Expr *pExpr, /* The IN operator */ - int iTab /* Use this cursor number */ + int iTab, /* Use this cursor number */ + int allowBloom /* True to allow the use of a Bloom filter */ ){ int addrOnce = 0; /* Address of the OP_Once instruction at top */ int addr; /* Address of OP_OpenEphemeral instruction */ @@ -3701,7 +3715,10 @@ void sqlite3CodeRhsOfIN( sqlite3SelectDestInit(&dest, SRT_Set, iTab); dest.zAffSdst = exprINAffinity(pParse, pExpr); pSelect->iLimit = 0; - if( addrOnce && OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){ + if( addrOnce + && allowBloom + && OptimizationEnabled(pParse->db, SQLITE_BloomFilter) + ){ int regBloom = ++pParse->nMem; addrBloom = sqlite3VdbeAddOp2(v, OP_Blob, 10000, regBloom); VdbeComment((v, "Bloom filter")); @@ -3922,7 +3939,7 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ || (pLeft->u.iValue!=1 && pLeft->u.iValue!=0) ){ sqlite3 *db = pParse->db; - pLimit = sqlite3Expr(db, TK_INTEGER, "0"); + pLimit = sqlite3ExprInt32(db, 0); if( pLimit ){ pLimit->affExpr = SQLITE_AFF_NUMERIC; pLimit = sqlite3PExpr(pParse, TK_NE, @@ -3933,7 +3950,7 @@ int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ } }else{ /* If there is no pre-existing limit add a limit of 1 */ - pLimit = sqlite3Expr(pParse->db, TK_INTEGER, "1"); + pLimit = sqlite3ExprInt32(pParse->db, 1); pSel->pLimit = sqlite3PExpr(pParse, TK_LIMIT, pLimit, 0); } pSel->iLimit = 0; @@ -4194,8 +4211,9 @@ static void sqlite3ExprCodeIN( if( ExprHasProperty(pExpr, EP_Subrtn) ){ const VdbeOp *pOp = sqlite3VdbeGetOp(v, pExpr->y.sub.iAddr); assert( pOp->opcode==OP_Once || pParse->nErr ); - if( pOp->opcode==OP_Once && pOp->p3>0 ){ /* tag-202407032019 */ - assert( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ); + if( pOp->p3>0 ){ /* tag-202407032019 */ + assert( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) + || pParse->nErr ); sqlite3VdbeAddOp4Int(v, OP_Filter, pOp->p3, destIfFalse, rLhs, nVector); VdbeCoverage(v); } @@ -4285,7 +4303,7 @@ static void sqlite3ExprCodeIN( static void codeReal(Vdbe *v, const char *z, int negateFlag, int iMem){ if( ALWAYS(z!=0) ){ double value; - sqlite3AtoF(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); + sqlite3AtoF(z, &value); assert( !sqlite3IsNaN(value) ); /* The new AtoF never returns NaN */ if( negateFlag ) value = -value; sqlite3VdbeAddOp4Dup8(v, OP_Real, 0, iMem, 0, (u8*)&value, P4_REAL); @@ -5144,7 +5162,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ case TK_ISNOT: op = (op==TK_IS) ? TK_EQ : TK_NE; p5 = SQLITE_NULLEQ; - /* fall-through */ + /* no break */ deliberate_fall_through case TK_LT: case TK_LE: case TK_GT: @@ -7386,7 +7404,10 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ if( pIEpr==0 ) break; if( NEVER(!ExprUseYTab(pExpr)) ) break; for(i=0; i<pSrcList->nSrc; i++){ - if( pSrcList->a[0].iCursor==pIEpr->iDataCur ) break; + if( pSrcList->a[i].iCursor==pIEpr->iDataCur ){ + testcase( i>0 ); + break; + } } if( i>=pSrcList->nSrc ) break; if( NEVER(pExpr->pAggInfo!=0) ) break; /* Resolved by outer context */ diff --git a/src/fkey.c b/src/fkey.c index f1117a884..3e5d9d11a 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -688,6 +688,7 @@ FKey *sqlite3FkReferences(Table *pTab){ static void fkTriggerDelete(sqlite3 *dbMem, Trigger *p){ if( p ){ TriggerStep *pStep = p->step_list; + sqlite3SrcListDelete(dbMem, pStep->pSrc); sqlite3ExprDelete(dbMem, pStep->pWhere); sqlite3ExprListDelete(dbMem, pStep->pExprList); sqlite3SelectDelete(dbMem, pStep->pSelect); @@ -907,6 +908,7 @@ void sqlite3FkCheck( if( !IsOrdinaryTable(pTab) ) return; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDb>=00 && iDb<db->nDb ); zDb = db->aDb[iDb].zDbSName; /* Loop through all the foreign key constraints for which pTab is the @@ -1324,7 +1326,6 @@ static Trigger *fkActionTrigger( nFrom = sqlite3Strlen30(zFrom); if( action==OE_Restrict ){ - int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); SrcList *pSrc; Expr *pRaise; @@ -1335,10 +1336,10 @@ static Trigger *fkActionTrigger( } pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); if( pSrc ){ - assert( pSrc->nSrc==1 ); - pSrc->a[0].zName = sqlite3DbStrDup(db, zFrom); - assert( pSrc->a[0].fg.fixedSchema==0 && pSrc->a[0].fg.isSubquery==0 ); - pSrc->a[0].u4.zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); + SrcItem *pItem = &pSrc->a[0]; + pItem->zName = sqlite3DbStrDup(db, zFrom); + pItem->fg.fixedSchema = 1; + pItem->u4.pSchema = pTab->pSchema; } pSelect = sqlite3SelectNew(pParse, sqlite3ExprListAppend(pParse, 0, pRaise), @@ -1354,14 +1355,17 @@ static Trigger *fkActionTrigger( pTrigger = (Trigger *)sqlite3DbMallocZero(db, sizeof(Trigger) + /* struct Trigger */ - sizeof(TriggerStep) + /* Single step in trigger program */ - nFrom + 1 /* Space for pStep->zTarget */ + sizeof(TriggerStep) /* Single step in trigger program */ ); if( pTrigger ){ pStep = pTrigger->step_list = (TriggerStep *)&pTrigger[1]; - pStep->zTarget = (char *)&pStep[1]; - memcpy((char *)pStep->zTarget, zFrom, nFrom); - + pStep->pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); + if( pStep->pSrc ){ + SrcItem *pItem = &pStep->pSrc->a[0]; + pItem->zName = sqlite3DbStrNDup(db, zFrom, nFrom); + pItem->u4.pSchema = pTab->pSchema; + pItem->fg.fixedSchema = 1; + } pStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); pStep->pExprList = sqlite3ExprListDup(db, pList, EXPRDUP_REDUCE); pStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); diff --git a/src/func.c b/src/func.c index 6dac7195a..4260f3e72 100644 --- a/src/func.c +++ b/src/func.c @@ -466,7 +466,7 @@ static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ sqlite3_result_error_nomem(context); return; } - sqlite3AtoF(zBuf, &r, sqlite3Strlen30(zBuf), SQLITE_UTF8); + sqlite3AtoF(zBuf, &r); sqlite3_free(zBuf); } sqlite3_result_double(context, r); @@ -1098,18 +1098,11 @@ void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue, int bEscape){ switch( sqlite3_value_type(pValue) ){ case SQLITE_FLOAT: { - double r1, r2; - const char *zVal; - r1 = sqlite3_value_double(pValue); - sqlite3_str_appendf(pStr, "%!0.15g", r1); - zVal = sqlite3_str_value(pStr); - if( zVal ){ - sqlite3AtoF(zVal, &r2, pStr->nChar, SQLITE_UTF8); - if( r1!=r2 ){ - sqlite3_str_reset(pStr); - sqlite3_str_appendf(pStr, "%!0.20e", r1); - } - } + /* ,--- Show infinity as 9.0e+999 + ** | + ** | ,--- 17 precision guarantees round-trip + ** v v */ + sqlite3_str_appendf(pStr, "%!0.17g", sqlite3_value_double(pValue)); break; } case SQLITE_INTEGER: { @@ -1201,7 +1194,7 @@ static void unistrFunc( } i = j = 0; while( i<nIn ){ - char *z = strchr(&zIn[i],'\\'); + const char *z = strchr(&zIn[i],'\\'); if( z==0 ){ n = nIn - i; memmove(&zOut[j], &zIn[i], n); @@ -1238,7 +1231,7 @@ static void unistrFunc( } } zOut[j] = 0; - sqlite3_result_text64(context, zOut, j, sqlite3_free, SQLITE_UTF8); + sqlite3_result_text64(context, zOut, j, sqlite3_free, SQLITE_UTF8_ZT); return; unistr_error: @@ -1331,7 +1324,7 @@ static void charFunc( } \ } *zOut = 0; - sqlite3_result_text64(context, (char*)z, zOut-z, sqlite3_free, SQLITE_UTF8); + sqlite3_result_text64(context, (char*)z, zOut-z,sqlite3_free,SQLITE_UTF8_ZT); } /* @@ -1360,7 +1353,7 @@ static void hexFunc( } *z = 0; sqlite3_result_text64(context, zHex, (u64)(z-zHex), - sqlite3_free, SQLITE_UTF8); + sqlite3_free, SQLITE_UTF8_ZT); } } @@ -1698,7 +1691,7 @@ static void concatFuncCore( } z[j] = 0; assert( j<=n ); - sqlite3_result_text64(context, z, j, sqlite3_free, SQLITE_UTF8); + sqlite3_result_text64(context, z, j, sqlite3_free, SQLITE_UTF8_ZT); } /* @@ -2364,6 +2357,8 @@ void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive){ sqlite3CreateFunc(db, "like", nArg, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0, 0, 0); pDef = sqlite3FindFunction(db, "like", nArg, SQLITE_UTF8, 0); + assert( pDef!=0 ); /* The sqlite3CreateFunc() call above cannot fail + ** because the "like" SQL-function already exists */ pDef->funcFlags |= flags; pDef->funcFlags &= ~SQLITE_FUNC_UNSAFE; } diff --git a/src/hwtime.h b/src/hwtime.h index f808fa40e..cf637edaf 100644 --- a/src/hwtime.h +++ b/src/hwtime.h @@ -16,17 +16,19 @@ #ifndef SQLITE_HWTIME_H #define SQLITE_HWTIME_H -/* -** The following routine only works on Pentium-class (or newer) processors. -** It uses the RDTSC opcode to read the cycle count value out of the -** processor and returns that value. This can be used for high-res -** profiling. -*/ -#if !defined(__STRICT_ANSI__) && \ - (defined(__GNUC__) || defined(_MSC_VER)) && \ - (defined(i386) || defined(__i386__) || defined(_M_IX86)) +#if defined(_MSC_VER) && defined(_WIN32) + + #include "windows.h" + #include <profileapi.h> - #if defined(__GNUC__) + __inline sqlite3_uint64 sqlite3Hwtime(void){ + LARGE_INTEGER tm; + QueryPerformanceCounter(&tm); + return (sqlite3_uint64)tm.QuadPart; + } + +#elif !defined(__STRICT_ANSI__) && defined(__GNUC__) && \ + (defined(i386) || defined(__i386__) || defined(_M_IX86)) __inline__ sqlite_uint64 sqlite3Hwtime(void){ unsigned int lo, hi; @@ -34,17 +36,6 @@ return (sqlite_uint64)hi << 32 | lo; } - #elif defined(_MSC_VER) - - __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ - __asm { - rdtsc - ret ; return value at EDX:EAX - } - } - - #endif - #elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) __inline__ sqlite_uint64 sqlite3Hwtime(void){ @@ -52,6 +43,14 @@ __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); return (sqlite_uint64)hi << 32 | lo; } + +#elif !defined(__STRICT_ANSI__) && defined(__GNUC__) && defined(__aarch64__) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + sqlite3_uint64 cnt; + __asm__ __volatile__ ("mrs %0, cntvct_el0" : "=r" (cnt)); + return cnt; + } #elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) diff --git a/src/json.c b/src/json.c index d0d3c53a2..f3d5aadcf 100644 --- a/src/json.c +++ b/src/json.c @@ -313,7 +313,8 @@ struct JsonString { /* Allowed values for JsonString.eErr */ #define JSTRING_OOM 0x01 /* Out of memory */ #define JSTRING_MALFORMED 0x02 /* Malformed JSONB */ -#define JSTRING_ERR 0x04 /* Error already sent to sqlite3_result */ +#define JSTRING_TOODEEP 0x04 /* JSON nested too deep */ +#define JSTRING_ERR 0x08 /* Error already sent to sqlite3_result */ /* The "subtype" set for text JSON values passed through using ** sqlite3_result_subtype() and sqlite3_value_subtype(). @@ -328,7 +329,10 @@ struct JsonString { #define JSON_SQL 0x02 /* Result is always SQL */ #define JSON_ABPATH 0x03 /* Allow abbreviated JSON path specs */ #define JSON_ISSET 0x04 /* json_set(), not json_insert() */ -#define JSON_BLOB 0x08 /* Use the BLOB output format */ +#define JSON_AINS 0x08 /* json_array_insert(), not json_insert() */ +#define JSON_BLOB 0x10 /* Use the BLOB output format */ + +#define JSON_INSERT_TYPE(X) (((X)&0xC)>>2) /* A parsed JSON value. Lifecycle: @@ -374,6 +378,7 @@ struct JsonParse { #define JEDIT_REPL 2 /* Overwrite if exists */ #define JEDIT_INS 3 /* Insert if not exists */ #define JEDIT_SET 4 /* Insert or overwrite */ +#define JEDIT_AINS 5 /* array_insert() */ /* ** Maximum nesting depth of JSON for this implementation. @@ -399,7 +404,7 @@ struct JsonParse { **************************************************************************/ static void jsonReturnStringAsBlob(JsonString*); static int jsonArgIsJsonb(sqlite3_value *pJson, JsonParse *p); -static u32 jsonTranslateBlobToText(const JsonParse*,u32,JsonString*); +static u32 jsonTranslateBlobToText(JsonParse*,u32,JsonString*); static void jsonReturnParse(sqlite3_context*,JsonParse*); static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); static void jsonParseFree(JsonParse*); @@ -557,6 +562,15 @@ static void jsonStringOom(JsonString *p){ jsonStringReset(p); } +/* Report JSON nested too deep +*/ +static void jsonStringTooDeep(JsonString *p){ + p->eErr |= JSTRING_TOODEEP; + assert( p->pCtx!=0 ); + sqlite3_result_error(p->pCtx, "JSON nested too deep", -1); + jsonStringReset(p); +} + /* Enlarge pJson->zBuf so that it can hold at least N more bytes. ** Return zero on success. Return non-zero on an OOM error */ @@ -846,6 +860,7 @@ static void jsonReturnString( ){ assert( (pParse!=0)==(ctx!=0) ); assert( ctx==0 || ctx==p->pCtx ); + jsonStringTerminate(p); if( p->eErr==0 ){ int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(p->pCtx)); if( flags & JSON_BLOB ){ @@ -853,7 +868,7 @@ static void jsonReturnString( }else if( p->bStatic ){ sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, SQLITE_TRANSIENT, SQLITE_UTF8); - }else if( jsonStringTerminate(p) ){ + }else{ if( pParse && pParse->bJsonIsRCStr==0 && pParse->nBlobAlloc>0 ){ int rc; pParse->zJson = sqlite3RCStrRef(p->zBuf); @@ -869,11 +884,11 @@ static void jsonReturnString( sqlite3_result_text64(p->pCtx, sqlite3RCStrRef(p->zBuf), p->nUsed, sqlite3RCStrUnref, SQLITE_UTF8); - }else{ - sqlite3_result_error_nomem(p->pCtx); } }else if( p->eErr & JSTRING_OOM ){ sqlite3_result_error_nomem(p->pCtx); + }else if( p->eErr & JSTRING_TOODEEP ){ + /* error already in p->pCtx */ }else if( p->eErr & JSTRING_MALFORMED ){ sqlite3_result_error(p->pCtx, "malformed JSON", -1); } @@ -1204,11 +1219,11 @@ static void jsonBlobAppendOneByte(JsonParse *pParse, u8 c){ /* Slow version of jsonBlobAppendNode() that first resizes the ** pParse->aBlob structure. */ -static void jsonBlobAppendNode(JsonParse*,u8,u32,const void*); +static void jsonBlobAppendNode(JsonParse*,u8,u64,const void*); static SQLITE_NOINLINE void jsonBlobExpandAndAppendNode( JsonParse *pParse, u8 eType, - u32 szPayload, + u64 szPayload, const void *aPayload ){ if( jsonBlobExpand(pParse, pParse->nBlob+szPayload+9) ) return; @@ -1228,7 +1243,7 @@ static SQLITE_NOINLINE void jsonBlobExpandAndAppendNode( static void jsonBlobAppendNode( JsonParse *pParse, /* The JsonParse object under construction */ u8 eType, /* Node type. One of JSONB_* */ - u32 szPayload, /* Number of bytes of payload */ + u64 szPayload, /* Number of bytes of payload */ const void *aPayload /* The payload. Might be NULL */ ){ u8 *a; @@ -2084,12 +2099,8 @@ static int jsonConvertTextToBlob( */ static void jsonReturnStringAsBlob(JsonString *pStr){ JsonParse px; + assert( pStr->eErr==0 ); memset(&px, 0, sizeof(px)); - jsonStringTerminate(pStr); - if( pStr->eErr ){ - sqlite3_result_error_nomem(pStr->pCtx); - return; - } px.zJson = pStr->zBuf; px.nJson = pStr->nUsed; px.db = sqlite3_context_db_handle(pStr->pCtx); @@ -2179,7 +2190,7 @@ static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){ ** The pOut->eErr JSTRING_OOM flag is set on a OOM. */ static u32 jsonTranslateBlobToText( - const JsonParse *pParse, /* the complete parse of the JSON */ + JsonParse *pParse, /* the complete parse of the JSON */ u32 i, /* Start rendering at this index */ JsonString *pOut /* Write JSON here */ ){ @@ -2361,10 +2372,14 @@ static u32 jsonTranslateBlobToText( jsonAppendChar(pOut, '['); j = i+n; iEnd = j+sz; + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + jsonStringTooDeep(pOut); + } while( j<iEnd && pOut->eErr==0 ){ j = jsonTranslateBlobToText(pParse, j, pOut); jsonAppendChar(pOut, ','); } + pParse->iDepth--; if( j>iEnd ) pOut->eErr |= JSTRING_MALFORMED; if( sz>0 ) jsonStringTrimOneChar(pOut); jsonAppendChar(pOut, ']'); @@ -2375,10 +2390,14 @@ static u32 jsonTranslateBlobToText( jsonAppendChar(pOut, '{'); j = i+n; iEnd = j+sz; + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + jsonStringTooDeep(pOut); + } while( j<iEnd && pOut->eErr==0 ){ j = jsonTranslateBlobToText(pParse, j, pOut); jsonAppendChar(pOut, (x++ & 1) ? ',' : ':'); } + pParse->iDepth--; if( (x & 1)!=0 || j>iEnd ) pOut->eErr |= JSTRING_MALFORMED; if( sz>0 ) jsonStringTrimOneChar(pOut); jsonAppendChar(pOut, '}'); @@ -2435,7 +2454,7 @@ static u32 jsonTranslateBlobToPrettyText( u32 i /* Start rendering at this index */ ){ u32 sz, n, j, iEnd; - const JsonParse *pParse = pPretty->pParse; + JsonParse *pParse = pPretty->pParse; JsonString *pOut = pPretty->pOut; n = jsonbPayloadSize(pParse, i, &sz); if( n==0 ){ @@ -2450,6 +2469,9 @@ static u32 jsonTranslateBlobToPrettyText( if( j<iEnd ){ jsonAppendChar(pOut, '\n'); pPretty->nIndent++; + if( pPretty->nIndent >= JSON_MAX_DEPTH ){ + jsonStringTooDeep(pOut); + } while( pOut->eErr==0 ){ jsonPrettyIndent(pPretty); j = jsonTranslateBlobToPrettyText(pPretty, j); @@ -2471,6 +2493,10 @@ static u32 jsonTranslateBlobToPrettyText( if( j<iEnd ){ jsonAppendChar(pOut, '\n'); pPretty->nIndent++; + if( pPretty->nIndent >= JSON_MAX_DEPTH ){ + jsonStringTooDeep(pOut); + } + pParse->iDepth = pPretty->nIndent; while( pOut->eErr==0 ){ jsonPrettyIndent(pPretty); j = jsonTranslateBlobToText(pParse, j, pOut); @@ -2628,6 +2654,7 @@ static void jsonBlobEdit( u32 nIns /* Bytes of content to insert */ ){ i64 d = (i64)nIns - (i64)nDel; + assert( pParse->nBlob >= (u64)iDel + (u64)nDel ); if( d<0 && d>=(-8) && aIns!=0 && jsonBlobOverwrite(&pParse->aBlob[iDel], aIns, nIns, (int)-d) ){ @@ -2870,7 +2897,9 @@ static int jsonLabelCompare( */ #define JSON_LOOKUP_ERROR 0xffffffff #define JSON_LOOKUP_NOTFOUND 0xfffffffe -#define JSON_LOOKUP_PATHERROR 0xfffffffd +#define JSON_LOOKUP_NOTARRAY 0xfffffffd +#define JSON_LOOKUP_TOODEEP 0xfffffffc +#define JSON_LOOKUP_PATHERROR 0xfffffffb #define JSON_LOOKUP_ISERROR(x) ((x)>=JSON_LOOKUP_PATHERROR) /* Forward declaration */ @@ -2899,7 +2928,7 @@ static u32 jsonLookupStep(JsonParse*,u32,const char*,u32); static u32 jsonCreateEditSubstructure( JsonParse *pParse, /* The original JSONB that is being edited */ JsonParse *pIns, /* Populate this with the blob data to insert */ - const char *zTail /* Tail of the path that determins substructure */ + const char *zTail /* Tail of the path that determines substructure */ ){ static const u8 emptyObject[] = { JSONB_ARRAY, JSONB_OBJECT }; int rc; @@ -2917,7 +2946,12 @@ static u32 jsonCreateEditSubstructure( pIns->eEdit = pParse->eEdit; pIns->nIns = pParse->nIns; pIns->aIns = pParse->aIns; + pIns->iDepth = pParse->iDepth+1; + if( pIns->iDepth >= JSON_MAX_DEPTH ){ + return JSON_LOOKUP_TOODEEP; + } rc = jsonLookupStep(pIns, 0, zTail, 0); + pParse->iDepth--; pParse->oom |= pIns->oom; } return rc; /* Error code only */ @@ -2934,9 +2968,9 @@ static u32 jsonCreateEditSubstructure( ** Return one of the JSON_LOOKUP error codes if problems are seen. ** ** This routine will also modify the blob. If pParse->eEdit is one of -** JEDIT_DEL, JEDIT_REPL, JEDIT_INS, or JEDIT_SET, then changes might be -** made to the selected value. If an edit is performed, then the return -** value does not necessarily point to the select element. If an edit +** JEDIT_DEL, JEDIT_REPL, JEDIT_INS, JEDIT_SET, or JEDIT_AINS, then changes +** might be made to the selected value. If an edit is performed, then the +** return value does not necessarily point to the select element. If an edit ** is performed, the return value is only useful for detecting error ** conditions. */ @@ -2962,6 +2996,13 @@ static u32 jsonLookupStep( jsonBlobEdit(pParse, iRoot, sz, 0, 0); }else if( pParse->eEdit==JEDIT_INS ){ /* Already exists, so json_insert() is a no-op */ + }else if( pParse->eEdit==JEDIT_AINS ){ + /* json_array_insert() */ + if( zPath[-1]!=']' ){ + return JSON_LOOKUP_NOTARRAY; + }else{ + jsonBlobEdit(pParse, iRoot, 0, pParse->aIns, pParse->nIns); + } }else{ /* json_set() or json_replace() */ jsonBlobEdit(pParse, iRoot, sz, pParse->aIns, pParse->nIns); @@ -3016,7 +3057,11 @@ static u32 jsonLookupStep( n = jsonbPayloadSize(pParse, v, &sz); if( n==0 || v+n+sz>iEnd ) return JSON_LOOKUP_ERROR; assert( j>0 ); + if( ++pParse->iDepth >= JSON_MAX_DEPTH ){ + return JSON_LOOKUP_TOODEEP; + } rc = jsonLookupStep(pParse, v, &zPath[i], j); + pParse->iDepth--; if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); return rc; } @@ -3033,6 +3078,10 @@ static u32 jsonLookupStep( JsonParse ix; /* Header of the label to be inserted */ testcase( pParse->eEdit==JEDIT_INS ); testcase( pParse->eEdit==JEDIT_SET ); + testcase( pParse->eEdit==JEDIT_AINS ); + if( pParse->eEdit==JEDIT_AINS && sqlite3_strglob("*]",&zPath[i])!=0 ){ + return JSON_LOOKUP_NOTARRAY; + } memset(&ix, 0, sizeof(ix)); ix.db = pParse->db; jsonBlobAppendNode(&ix, rawKey?JSONB_TEXTRAW:JSONB_TEXT5, nKey, 0); @@ -3060,28 +3109,32 @@ static u32 jsonLookupStep( return rc; } }else if( zPath[0]=='[' ){ + u64 kk = 0; x = pParse->aBlob[iRoot] & 0x0f; if( x!=JSONB_ARRAY ) return JSON_LOOKUP_NOTFOUND; n = jsonbPayloadSize(pParse, iRoot, &sz); - k = 0; i = 1; while( sqlite3Isdigit(zPath[i]) ){ - k = k*10 + zPath[i] - '0'; + if( kk<0xffffffff ) kk = kk*10 + zPath[i] - '0'; + /* ^^^^^^^^^^--- Allow kk to be bigger than any JSON array so that + ** we get NOTFOUND instead of PATHERROR, without overflowing kk. */ i++; } if( i<2 || zPath[i]!=']' ){ if( zPath[1]=='#' ){ - k = jsonbArrayCount(pParse, iRoot); + kk = jsonbArrayCount(pParse, iRoot); i = 2; if( zPath[2]=='-' && sqlite3Isdigit(zPath[3]) ){ - unsigned int nn = 0; + u64 nn = 0; i = 3; do{ - nn = nn*10 + zPath[i] - '0'; + if( nn<0xffffffff ) nn = nn*10 + zPath[i] - '0'; + /* ^^^^^^^^^^--- Allow nn to be bigger than any JSON array to + ** get NOTFOUND instead of PATHERROR, without overflowing nn. */ i++; }while( sqlite3Isdigit(zPath[i]) ); - if( nn>k ) return JSON_LOOKUP_NOTFOUND; - k -= nn; + if( nn>kk ) return JSON_LOOKUP_NOTFOUND; + kk -= nn; } if( zPath[i]!=']' ){ return JSON_LOOKUP_PATHERROR; @@ -3093,21 +3146,26 @@ static u32 jsonLookupStep( j = iRoot+n; iEnd = j+sz; while( j<iEnd ){ - if( k==0 ){ + if( kk==0 ){ + if( ++pParse->iDepth >= JSON_MAX_DEPTH ){ + return JSON_LOOKUP_TOODEEP; + } rc = jsonLookupStep(pParse, j, &zPath[i+1], 0); + pParse->iDepth--; if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); return rc; } - k--; + kk--; n = jsonbPayloadSize(pParse, j, &sz); if( n==0 ) return JSON_LOOKUP_ERROR; j += n+sz; } if( j>iEnd ) return JSON_LOOKUP_ERROR; - if( k>0 ) return JSON_LOOKUP_NOTFOUND; + if( kk>0 ) return JSON_LOOKUP_NOTFOUND; if( pParse->eEdit>=JEDIT_INS ){ JsonParse v; testcase( pParse->eEdit==JEDIT_INS ); + testcase( pParse->eEdit==JEDIT_AINS ); testcase( pParse->eEdit==JEDIT_SET ); rc = jsonCreateEditSubstructure(pParse, &v, &zPath[i+1]); if( !JSON_LOOKUP_ISERROR(rc) @@ -3245,7 +3303,7 @@ static void jsonReturnFromBlob( to_double: z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); if( z==0 ) goto returnfromblob_oom; - rc = sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); + rc = sqlite3AtoF(z, &r); sqlite3DbFree(db, z); if( rc<=0 ) goto returnfromblob_malformed; sqlite3_result_double(pCtx, r); @@ -3425,16 +3483,35 @@ static int jsonFunctionArgToBlob( } /* -** Generate a bad path error. +** Generate a path error. +** +** The specifics of the error are determined by the rc argument. +** +** rc error +** ----------------- ---------------------- +** JSON_LOOKUP_ARRAY "not an array" +** JSON_LOOKUP_TOODEEP "JSON nested too deep" +** JSON_LOOKUP_ERROR "malformed JSON" +** otherwise... "bad JSON path" ** ** If ctx is not NULL then push the error message into ctx and return NULL. ** If ctx is NULL, then return the text of the error message. */ static char *jsonBadPathError( sqlite3_context *ctx, /* The function call containing the error */ - const char *zPath /* The path with the problem */ + const char *zPath, /* The path with the problem */ + int rc /* Maybe JSON_LOOKUP_NOTARRAY */ ){ - char *zMsg = sqlite3_mprintf("bad JSON path: %Q", zPath); + char *zMsg; + if( rc==(int)JSON_LOOKUP_NOTARRAY ){ + zMsg = sqlite3_mprintf("not an array element: %Q", zPath); + }else if( rc==(int)JSON_LOOKUP_ERROR ){ + zMsg = sqlite3_mprintf("malformed JSON"); + }else if( rc==(int)JSON_LOOKUP_TOODEEP ){ + zMsg = sqlite3_mprintf("JSON path too deep"); + }else{ + zMsg = sqlite3_mprintf("bad JSON path: %Q", zPath); + } if( ctx==0 ) return zMsg; if( zMsg ){ sqlite3_result_error(ctx, zMsg, -1); @@ -3451,13 +3528,13 @@ static char *jsonBadPathError( ** and return the result. ** ** The specific operation is determined by eEdit, which can be one -** of JEDIT_INS, JEDIT_REPL, or JEDIT_SET. +** of JEDIT_INS, JEDIT_REPL, JEDIT_SET, or JEDIT_AINS. */ static void jsonInsertIntoBlob( sqlite3_context *ctx, int argc, sqlite3_value **argv, - int eEdit /* JEDIT_INS, JEDIT_REPL, or JEDIT_SET */ + int eEdit /* JEDIT_INS, JEDIT_REPL, JEDIT_SET, JEDIT_AINS */ ){ int i; u32 rc = 0; @@ -3494,6 +3571,7 @@ static void jsonInsertIntoBlob( p->nIns = ax.nBlob; p->aIns = ax.aBlob; p->delta = 0; + p->iDepth = 0; rc = jsonLookupStep(p, 0, zPath+1, 0); } jsonParseReset(&ax); @@ -3506,11 +3584,7 @@ static void jsonInsertIntoBlob( jsonInsertIntoBlob_patherror: jsonParseFree(p); - if( rc==JSON_LOOKUP_ERROR ){ - sqlite3_result_error(ctx, "malformed JSON", -1); - }else{ - jsonBadPathError(ctx, zPath); - } + jsonBadPathError(ctx, zPath, rc); return; } @@ -3950,10 +4024,8 @@ static void jsonArrayLengthFunc( if( JSON_LOOKUP_ISERROR(i) ){ if( i==JSON_LOOKUP_NOTFOUND ){ /* no-op */ - }else if( i==JSON_LOOKUP_PATHERROR ){ - jsonBadPathError(ctx, zPath); }else{ - sqlite3_result_error(ctx, "malformed JSON", -1); + jsonBadPathError(ctx, zPath, i); } eErr = 1; i = 0; @@ -4056,7 +4128,7 @@ static void jsonExtractFunc( j = jsonLookupStep(p, 0, jx.zBuf, 0); jsonStringReset(&jx); }else{ - jsonBadPathError(ctx, zPath); + jsonBadPathError(ctx, zPath, 0); goto json_extract_error; } if( j<p->nBlob ){ @@ -4087,11 +4159,8 @@ static void jsonExtractFunc( jsonAppendSeparator(&jx); jsonAppendRawNZ(&jx, "null", 4); } - }else if( j==JSON_LOOKUP_ERROR ){ - sqlite3_result_error(ctx, "malformed JSON", -1); - goto json_extract_error; }else{ - jsonBadPathError(ctx, zPath); + jsonBadPathError(ctx, zPath, j); goto json_extract_error; } } @@ -4115,6 +4184,7 @@ static void jsonExtractFunc( #define JSON_MERGE_BADTARGET 1 /* Malformed TARGET blob */ #define JSON_MERGE_BADPATCH 2 /* Malformed PATCH blob */ #define JSON_MERGE_OOM 3 /* Out-of-memory condition */ +#define JSON_MERGE_TOODEEP 4 /* Nested too deep */ /* ** RFC-7396 MergePatch for two JSONB blobs. @@ -4166,7 +4236,8 @@ static int jsonMergePatch( JsonParse *pTarget, /* The JSON parser that contains the TARGET */ u32 iTarget, /* Index of TARGET in pTarget->aBlob[] */ const JsonParse *pPatch, /* The PATCH */ - u32 iPatch /* Index of PATCH in pPatch->aBlob[] */ + u32 iPatch, /* Index of PATCH in pPatch->aBlob[] */ + u32 iDepth /* Nesting depth */ ){ u8 x; /* Type of a single node */ u32 n, sz=0; /* Return values from jsonbPayloadSize() */ @@ -4275,7 +4346,8 @@ static int jsonMergePatch( /* Algorithm line 12 */ int rc, savedDelta = pTarget->delta; pTarget->delta = 0; - rc = jsonMergePatch(pTarget, iTValue, pPatch, iPValue); + if( iDepth>=JSON_MAX_DEPTH ) return JSON_MERGE_TOODEEP; + rc = jsonMergePatch(pTarget, iTValue, pPatch, iPValue, iDepth+1); if( rc ) return rc; pTarget->delta += savedDelta; } @@ -4296,7 +4368,8 @@ static int jsonMergePatch( pTarget->aBlob[iTEnd+szNew] = 0x00; savedDelta = pTarget->delta; pTarget->delta = 0; - rc = jsonMergePatch(pTarget, iTEnd+szNew,pPatch,iPValue); + if( iDepth>=JSON_MAX_DEPTH ) return JSON_MERGE_TOODEEP; + rc = jsonMergePatch(pTarget, iTEnd+szNew,pPatch,iPValue,iDepth+1); if( rc ) return rc; pTarget->delta += savedDelta; } @@ -4327,11 +4400,13 @@ static void jsonPatchFunc( if( pTarget==0 ) return; pPatch = jsonParseFuncArg(ctx, argv[1], 0); if( pPatch ){ - rc = jsonMergePatch(pTarget, 0, pPatch, 0); + rc = jsonMergePatch(pTarget, 0, pPatch, 0, 0); if( rc==JSON_MERGE_OK ){ jsonReturnParse(ctx, pTarget); }else if( rc==JSON_MERGE_OOM ){ sqlite3_result_error_nomem(ctx); + }else if( rc==JSON_MERGE_TOODEEP ){ + sqlite3_result_error(ctx, "JSON nested too deep", -1); }else{ sqlite3_result_error(ctx, "malformed JSON", -1); } @@ -4419,10 +4494,8 @@ static void jsonRemoveFunc( if( JSON_LOOKUP_ISERROR(rc) ){ if( rc==JSON_LOOKUP_NOTFOUND ){ continue; /* No-op */ - }else if( rc==JSON_LOOKUP_PATHERROR ){ - jsonBadPathError(ctx, zPath); }else{ - sqlite3_result_error(ctx, "malformed JSON", -1); + jsonBadPathError(ctx, zPath, rc); } goto json_remove_done; } @@ -4432,7 +4505,7 @@ static void jsonRemoveFunc( return; json_remove_patherror: - jsonBadPathError(ctx, zPath); + jsonBadPathError(ctx, zPath, 0); json_remove_done: jsonParseFree(p); @@ -4476,16 +4549,18 @@ static void jsonSetFunc( int argc, sqlite3_value **argv ){ - int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); - int bIsSet = (flags&JSON_ISSET)!=0; + int eInsType = JSON_INSERT_TYPE(flags); + static const char *azInsType[] = { "insert", "set", "array_insert" }; + static const u8 aEditType[] = { JEDIT_INS, JEDIT_SET, JEDIT_AINS }; if( argc<1 ) return; + assert( eInsType>=0 && eInsType<=2 ); if( (argc&1)==0 ) { - jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); + jsonWrongNumArgs(ctx, azInsType[eInsType]); return; } - jsonInsertIntoBlob(ctx, argc, argv, bIsSet ? JEDIT_SET : JEDIT_INS); + jsonInsertIntoBlob(ctx, argc, argv, aEditType[eInsType]); } /* @@ -4510,17 +4585,15 @@ static void jsonTypeFunc( zPath = (const char*)sqlite3_value_text(argv[1]); if( zPath==0 ) goto json_type_done; if( zPath[0]!='$' ){ - jsonBadPathError(ctx, zPath); + jsonBadPathError(ctx, zPath, 0); goto json_type_done; } i = jsonLookupStep(p, 0, zPath+1, 0); if( JSON_LOOKUP_ISERROR(i) ){ if( i==JSON_LOOKUP_NOTFOUND ){ /* no-op */ - }else if( i==JSON_LOOKUP_PATHERROR ){ - jsonBadPathError(ctx, zPath); }else{ - sqlite3_result_error(ctx, "malformed JSON", -1); + jsonBadPathError(ctx, zPath, i); } goto json_type_done; } @@ -4774,12 +4847,12 @@ static void jsonArrayStep( } static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ JsonString *pStr; + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); if( pStr ){ - int flags; pStr->pCtx = ctx; - jsonAppendChar(pStr, ']'); - flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + jsonAppendRawNZ(pStr, "]", 2); + jsonStringTrimOneChar(pStr); if( pStr->eErr ){ jsonReturnString(pStr, 0, 0); return; @@ -4800,6 +4873,9 @@ static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); jsonStringTrimOneChar(pStr); } + }else if( flags & JSON_BLOB ){ + static const u8 emptyArray = 0x0b; + sqlite3_result_blob(ctx, &emptyArray, 1, SQLITE_STATIC); }else{ sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC); } @@ -4896,12 +4972,12 @@ static void jsonObjectStep( } static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ JsonString *pStr; + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); if( pStr ){ - int flags; - jsonAppendChar(pStr, '}'); + jsonAppendRawNZ(pStr, "}", 2); + jsonStringTrimOneChar(pStr); pStr->pCtx = ctx; - flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); if( pStr->eErr ){ jsonReturnString(pStr, 0, 0); return; @@ -4922,6 +4998,9 @@ static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); jsonStringTrimOneChar(pStr); } + }else if( flags & JSON_BLOB ){ + static const unsigned char emptyObject = 0x0c; + sqlite3_result_blob(ctx, &emptyObject, 1, SQLITE_STATIC); }else{ sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC); } @@ -5422,7 +5501,7 @@ static int jsonEachFilter( if( zRoot==0 ) return SQLITE_OK; if( zRoot[0]!='$' ){ sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot); + cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot, 0); jsonEachCursorReset(p); return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; } @@ -5440,7 +5519,7 @@ static int jsonEachFilter( return SQLITE_OK; } sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot); + cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot, 0); jsonEachCursorReset(p); return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; } @@ -5530,6 +5609,8 @@ void sqlite3RegisterJsonFunctions(void){ JFUNCTION(jsonb, 1,1,0, 0,1,0, jsonRemoveFunc), JFUNCTION(json_array, -1,0,1, 1,0,0, jsonArrayFunc), JFUNCTION(jsonb_array, -1,0,1, 1,1,0, jsonArrayFunc), + JFUNCTION(json_array_insert, -1,1,1, 1,0,JSON_AINS, jsonSetFunc), + JFUNCTION(jsonb_array_insert,-1,1,0, 1,1,JSON_AINS, jsonSetFunc), JFUNCTION(json_array_length, 1,1,0, 0,0,0, jsonArrayLengthFunc), JFUNCTION(json_array_length, 2,1,0, 0,0,0, jsonArrayLengthFunc), JFUNCTION(json_error_position,1,1,0, 0,0,0, jsonErrorFunc), diff --git a/src/loadext.c b/src/loadext.c index c5177715e..55325edd7 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -522,7 +522,17 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_setlk_timeout, /* Version 3.51.0 and later */ sqlite3_set_errmsg, - sqlite3_db_status64 + sqlite3_db_status64, + /* Version 3.52.0 and later */ + sqlite3_str_truncate, + sqlite3_str_free, +#ifdef SQLITE_ENABLE_CARRAY + sqlite3_carray_bind, + sqlite3_carray_bind_v2 +#else + 0, + 0 +#endif }; /* True if x is the directory separator character @@ -624,33 +634,42 @@ static int sqlite3LoadExtension( ** entry point name "sqlite3_extension_init" was not found, then ** construct an entry point name "sqlite3_X_init" where the X is ** replaced by the lowercase value of every ASCII alphabetic - ** character in the filename after the last "/" upto the first ".", - ** and eliding the first three characters if they are "lib". + ** character in the filename after the last "/" up to the first ".", + ** and skipping the first three characters if they are "lib". ** Examples: ** ** /usr/local/lib/libExample5.4.3.so ==> sqlite3_example_init ** C:/lib/mathfuncs.dll ==> sqlite3_mathfuncs_init + ** + ** If that still finds no entry point, repeat a second time but this + ** time include both alphabetic and numeric characters up to the first + ** ".". Example: + ** + ** /usr/local/lib/libExample5.4.3.so ==> sqlite3_example5_init */ if( xInit==0 && zProc==0 ){ int iFile, iEntry, c; int ncFile = sqlite3Strlen30(zFile); + int cnt = 0; zAltEntry = sqlite3_malloc64(ncFile+30); if( zAltEntry==0 ){ sqlite3OsDlClose(pVfs, handle); return SQLITE_NOMEM_BKPT; } - memcpy(zAltEntry, "sqlite3_", 8); - for(iFile=ncFile-1; iFile>=0 && !DirSep(zFile[iFile]); iFile--){} - iFile++; - if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; - for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ - if( sqlite3Isalpha(c) ){ - zAltEntry[iEntry++] = (char)sqlite3UpperToLower[(unsigned)c]; + do{ + memcpy(zAltEntry, "sqlite3_", 8); + for(iFile=ncFile-1; iFile>=0 && !DirSep(zFile[iFile]); iFile--){} + iFile++; + if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; + for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ + if( sqlite3Isalpha(c) || (cnt && sqlite3Isdigit(c)) ){ + zAltEntry[iEntry++] = (char)sqlite3UpperToLower[(unsigned)c]; + } } - } - memcpy(zAltEntry+iEntry, "_init", 6); - zEntry = zAltEntry; - xInit = (sqlite3_loadext_entry)sqlite3OsDlSym(pVfs, handle, zEntry); + memcpy(zAltEntry+iEntry, "_init", 6); + zEntry = zAltEntry; + xInit = (sqlite3_loadext_entry)sqlite3OsDlSym(pVfs, handle, zEntry); + }while( xInit==0 && (++cnt)<2 ); } if( xInit==0 ){ if( pzErrMsg ){ diff --git a/src/main.c b/src/main.c index 6efe538d4..990d16c70 100644 --- a/src/main.c +++ b/src/main.c @@ -971,6 +971,14 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){ rc = setupLookaside(db, pBuf, sz, cnt); break; } + case SQLITE_DBCONFIG_FP_DIGITS: { + int nIn = va_arg(ap, int); + int *pOut = va_arg(ap, int*); + if( nIn>3 && nIn<24 ) db->nFpDigit = (u8)nIn; + if( pOut ) *pOut = db->nFpDigit; + rc = SQLITE_OK; + break; + } default: { static const struct { int op; /* The opcode */ @@ -2526,6 +2534,9 @@ void *sqlite3_wal_hook( sqlite3_mutex_leave(db->mutex); return pRet; #else + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(xCallback); + UNUSED_PARAMETER(pArg); return 0; #endif } @@ -2541,6 +2552,11 @@ int sqlite3_wal_checkpoint_v2( int *pnCkpt /* OUT: Total number of frames checkpointed */ ){ #ifdef SQLITE_OMIT_WAL + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(zDb); + UNUSED_PARAMETER(eMode); + UNUSED_PARAMETER(pnLog); + UNUSED_PARAMETER(pnCkpt); return SQLITE_OK; #else int rc; /* Return code */ @@ -2554,11 +2570,12 @@ int sqlite3_wal_checkpoint_v2( if( pnLog ) *pnLog = -1; if( pnCkpt ) *pnCkpt = -1; + assert( SQLITE_CHECKPOINT_NOOP==-1 ); assert( SQLITE_CHECKPOINT_PASSIVE==0 ); assert( SQLITE_CHECKPOINT_FULL==1 ); assert( SQLITE_CHECKPOINT_RESTART==2 ); assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); - if( eMode<SQLITE_CHECKPOINT_PASSIVE || eMode>SQLITE_CHECKPOINT_TRUNCATE ){ + if( eMode<SQLITE_CHECKPOINT_NOOP || eMode>SQLITE_CHECKPOINT_TRUNCATE ){ /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint ** mode: */ return SQLITE_MISUSE_BKPT; @@ -2922,6 +2939,7 @@ static const int aHardLimit[] = { SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */ SQLITE_MAX_TRIGGER_DEPTH, SQLITE_MAX_WORKER_THREADS, + SQLITE_MAX_PARSER_DEPTH, }; /* @@ -2936,6 +2954,9 @@ static const int aHardLimit[] = { #if SQLITE_MAX_SQL_LENGTH>SQLITE_MAX_LENGTH # error SQLITE_MAX_SQL_LENGTH must not be greater than SQLITE_MAX_LENGTH #endif +#if SQLITE_MAX_SQL_LENGTH>2147482624 /* 1024 less than 2^31 */ +# error SQLITE_MAX_SQL_LENGTH must not be greater than 2147482624 +#endif #if SQLITE_MAX_COMPOUND_SELECT<2 # error SQLITE_MAX_COMPOUND_SELECT must be at least 2 #endif @@ -2991,6 +3012,7 @@ int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ assert( aHardLimit[SQLITE_LIMIT_SQL_LENGTH]==SQLITE_MAX_SQL_LENGTH ); assert( aHardLimit[SQLITE_LIMIT_COLUMN]==SQLITE_MAX_COLUMN ); assert( aHardLimit[SQLITE_LIMIT_EXPR_DEPTH]==SQLITE_MAX_EXPR_DEPTH ); + assert( aHardLimit[SQLITE_LIMIT_PARSER_DEPTH]==SQLITE_MAX_PARSER_DEPTH ); assert( aHardLimit[SQLITE_LIMIT_COMPOUND_SELECT]==SQLITE_MAX_COMPOUND_SELECT); assert( aHardLimit[SQLITE_LIMIT_VDBE_OP]==SQLITE_MAX_VDBE_OP ); assert( aHardLimit[SQLITE_LIMIT_FUNCTION_ARG]==SQLITE_MAX_FUNCTION_ARG ); @@ -3000,7 +3022,7 @@ int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS ); - assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-1) ); + assert( SQLITE_LIMIT_PARSER_DEPTH==(SQLITE_N_LIMIT-1) ); if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ @@ -3364,7 +3386,7 @@ static int openDatabase( db = sqlite3MallocZero( sizeof(sqlite3) ); if( db==0 ) goto opendb_out; if( isThreadsafe -#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS +#if defined(SQLITE_THREAD_MISUSE_WARNINGS) || sqlite3GlobalConfig.bCoreMutex #endif ){ @@ -3385,6 +3407,7 @@ static int openDatabase( db->aDb = db->aDbStatic; db->lookaside.bDisable = 1; db->lookaside.sz = 0; + db->nFpDigit = 17; assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); @@ -3830,6 +3853,12 @@ int sqlite3_collation_needed16( */ void *sqlite3_get_clientdata(sqlite3 *db, const char *zName){ DbClientData *p; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zName || !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); for(p=db->pDbData; p; p=p->pNext){ if( strcmp(p->zName, zName)==0 ){ @@ -4687,6 +4716,17 @@ int sqlite3_test_control(int op, ...){ break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_ATOF, const char *z, double *p); + ** + ** Test access to the sqlite3AtoF() routine. + */ + case SQLITE_TESTCTRL_ATOF: { + const char *z = va_arg(ap,const char*); + double *pR = va_arg(ap,double*); + rc = sqlite3AtoF(z,pR); + break; + } + #if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_WSD) /* sqlite3_test_control(SQLITE_TESTCTRL_TUNE, id, *piValue) ** @@ -4903,6 +4943,7 @@ const char *sqlite3_filename_journal(const char *zFilename){ } const char *sqlite3_filename_wal(const char *zFilename){ #ifdef SQLITE_OMIT_WAL + UNUSED_PARAMETER(zFilename); return 0; #else zFilename = sqlite3_filename_journal(zFilename); diff --git a/src/mutex.c b/src/mutex.c index 62e09cb4f..21b47e511 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -27,23 +27,28 @@ static SQLITE_WSD int mutexIsInit = 0; #ifndef SQLITE_MUTEX_OMIT -#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS +#ifdef SQLITE_THREAD_MISUSE_WARNINGS /* -** This block (enclosed by SQLITE_ENABLE_MULTITHREADED_CHECKS) contains +** This block (enclosed by SQLITE_THREAD_MISUSE_WARNINGS) contains ** the implementation of a wrapper around the system default mutex ** implementation (sqlite3DefaultMutex()). ** ** Most calls are passed directly through to the underlying default ** mutex implementation. Except, if a mutex is configured by calling ** sqlite3MutexWarnOnContention() on it, then if contention is ever -** encountered within xMutexEnter() a warning is emitted via sqlite3_log(). +** encountered within xMutexEnter() then a warning is emitted via +** sqlite3_log(). Furthermore, if SQLITE_THREAD_MISUSE_ABORT is +** defined then abort() is called after the sqlite3_log() warning. ** -** This type of mutex is used as the database handle mutex when testing -** apps that usually use SQLITE_CONFIG_MULTITHREAD mode. +** This type of mutex is used on the database handle mutex when testing +** apps that usually use SQLITE_CONFIG_MULTITHREAD mode. A failure +** indicates that the app ought to be using SQLITE_OPEN_FULLMUTEX or +** similar because it is trying to use the same database handle from +** two different connections at the same time. */ /* -** Type for all mutexes used when SQLITE_ENABLE_MULTITHREADED_CHECKS +** Type for all mutexes used when SQLITE_THREAD_MISUSE_WARNINGS ** is defined. Variable CheckMutex.mutex is a pointer to the real mutex ** allocated by the system mutex implementation. Variable iType is usually set ** to the type of mutex requested - SQLITE_MUTEX_RECURSIVE, SQLITE_MUTEX_FAST @@ -79,11 +84,12 @@ static int checkMutexNotheld(sqlite3_mutex *p){ */ static int checkMutexInit(void){ pGlobalMutexMethods = sqlite3DefaultMutex(); - return SQLITE_OK; + return pGlobalMutexMethods->xMutexInit(); } static int checkMutexEnd(void){ + int rc = pGlobalMutexMethods->xMutexEnd(); pGlobalMutexMethods = 0; - return SQLITE_OK; + return rc; } /* @@ -160,6 +166,9 @@ static void checkMutexEnter(sqlite3_mutex *p){ sqlite3_log(SQLITE_MISUSE, "illegal multi-threaded access to database connection" ); +#if SQLITE_THREAD_MISUSE_ABORT + abort(); +#endif } pGlobalMutexMethods->xMutexEnter(pCheck->mutex); } @@ -211,7 +220,7 @@ void sqlite3MutexWarnOnContention(sqlite3_mutex *p){ pCheck->iType = SQLITE_MUTEX_WARNONCONTENTION; } } -#endif /* ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS */ +#endif /* ifdef SQLITE_THREAD_MISUSE_WARNINGS */ /* ** Initialize the mutex system. @@ -228,7 +237,7 @@ int sqlite3MutexInit(void){ sqlite3_mutex_methods *pTo = &sqlite3GlobalConfig.mutex; if( sqlite3GlobalConfig.bCoreMutex ){ -#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS +#ifdef SQLITE_THREAD_MISUSE_WARNINGS pFrom = multiThreadedCheckMutex(); #else pFrom = sqlite3DefaultMutex(); diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 7eb5b50be..20e41b161 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -129,11 +129,7 @@ static int winMutexInit(void){ if( InterlockedCompareExchange(&winMutex_lock, 1, 0)==0 ){ int i; for(i=0; i<ArraySize(winMutex_staticMutexes); i++){ -#if SQLITE_OS_WINRT - InitializeCriticalSectionEx(&winMutex_staticMutexes[i].mutex, 0, 0); -#else InitializeCriticalSection(&winMutex_staticMutexes[i].mutex); -#endif } winMutex_isInit = 1; }else{ @@ -223,11 +219,7 @@ static sqlite3_mutex *winMutexAlloc(int iType){ p->trace = 1; #endif #endif -#if SQLITE_OS_WINRT - InitializeCriticalSectionEx(&p->mutex, 0, 0); -#else InitializeCriticalSection(&p->mutex); -#endif } break; } diff --git a/src/os_kv.c b/src/os_kv.c index c2d1f9b7a..1fd1c8e8c 100644 --- a/src/os_kv.c +++ b/src/os_kv.c @@ -21,7 +21,7 @@ ** Debugging logic */ -/* SQLITE_KV_TRACE() is used for tracing calls to kvstorage routines. */ +/* SQLITE_KV_TRACE() is used for tracing calls to kvrecord routines. */ #if 0 #define SQLITE_KV_TRACE(X) printf X #else @@ -35,7 +35,6 @@ #define SQLITE_KV_LOG(X) #endif - /* ** Forward declaration of objects used by this VFS implementation */ @@ -43,6 +42,11 @@ typedef struct KVVfsFile KVVfsFile; /* A single open file. There are only two files represented by this ** VFS - the database and the rollback journal. +** +** Maintenance reminder: if this struct changes in any way, the JSON +** rendering of its structure must be updated in +** sqlite3-wasm.c:sqlite3__wasm_enum_json(). There are no binary +** compatibility concerns, so it does not need an iVersion member. */ struct KVVfsFile { sqlite3_file base; /* IO methods */ @@ -92,7 +96,7 @@ static int kvvfsCurrentTime(sqlite3_vfs*, double*); static int kvvfsCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); static sqlite3_vfs sqlite3OsKvvfsObject = { - 1, /* iVersion */ + 2, /* iVersion */ sizeof(KVVfsFile), /* szOsFile */ 1024, /* mxPathname */ 0, /* pNext */ @@ -168,23 +172,37 @@ static sqlite3_io_methods kvvfs_jrnl_io_methods = { /* Forward declarations for the low-level storage engine */ -static int kvstorageWrite(const char*, const char *zKey, const char *zData); -static int kvstorageDelete(const char*, const char *zKey); -static int kvstorageRead(const char*, const char *zKey, char *zBuf, int nBuf); -#define KVSTORAGE_KEY_SZ 32 +#ifndef SQLITE_WASM +/* In WASM builds these are implemented in JS. */ +static int kvrecordWrite(const char*, const char *zKey, const char *zData); +static int kvrecordDelete(const char*, const char *zKey); +static int kvrecordRead(const char*, const char *zKey, char *zBuf, int nBuf); +#endif +#ifndef KVRECORD_KEY_SZ +#define KVRECORD_KEY_SZ 32 +#endif /* Expand the key name with an appropriate prefix and put the result ** in zKeyOut[]. The zKeyOut[] buffer is assumed to hold at least -** KVSTORAGE_KEY_SZ bytes. +** KVRECORD_KEY_SZ bytes. */ -static void kvstorageMakeKey( +static void kvrecordMakeKey( const char *zClass, const char *zKeyIn, char *zKeyOut ){ - sqlite3_snprintf(KVSTORAGE_KEY_SZ, zKeyOut, "kvvfs-%s-%s", zClass, zKeyIn); + assert( zKeyIn ); + assert( zKeyOut ); + assert( zClass ); + sqlite3_snprintf(KVRECORD_KEY_SZ, zKeyOut, "kvvfs-%s-%s", + zClass, zKeyIn); } +#ifndef SQLITE_WASM +/* In WASM builds do not define APIs which use fopen(), fwrite(), +** and the like because those APIs are a portability issue for +** WASM. +*/ /* Write content into a key. zClass is the particular namespace of the ** underlying key/value store to use - either "local" or "session". ** @@ -192,14 +210,14 @@ static void kvstorageMakeKey( ** ** Return the number of errors. */ -static int kvstorageWrite( +static int kvrecordWrite( const char *zClass, const char *zKey, const char *zData ){ FILE *fd; - char zXKey[KVSTORAGE_KEY_SZ]; - kvstorageMakeKey(zClass, zKey, zXKey); + char zXKey[KVRECORD_KEY_SZ]; + kvrecordMakeKey(zClass, zKey, zXKey); fd = fopen(zXKey, "wb"); if( fd ){ SQLITE_KV_TRACE(("KVVFS-WRITE %-15s (%d) %.50s%s\n", zXKey, @@ -217,9 +235,9 @@ static int kvstorageWrite( ** namespace given by zClass. If the key does not previously exist, ** this routine is a no-op. */ -static int kvstorageDelete(const char *zClass, const char *zKey){ - char zXKey[KVSTORAGE_KEY_SZ]; - kvstorageMakeKey(zClass, zKey, zXKey); +static int kvrecordDelete(const char *zClass, const char *zKey){ + char zXKey[KVRECORD_KEY_SZ]; + kvrecordMakeKey(zClass, zKey, zXKey); unlink(zXKey); SQLITE_KV_TRACE(("KVVFS-DELETE %-15s\n", zXKey)); return 0; @@ -240,7 +258,7 @@ static int kvstorageDelete(const char *zClass, const char *zKey){ ** zero-terminates zBuf at zBuf[0] and returns the size of the data ** without reading it. */ -static int kvstorageRead( +static int kvrecordRead( const char *zClass, const char *zKey, char *zBuf, @@ -248,8 +266,8 @@ static int kvstorageRead( ){ FILE *fd; struct stat buf; - char zXKey[KVSTORAGE_KEY_SZ]; - kvstorageMakeKey(zClass, zKey, zXKey); + char zXKey[KVRECORD_KEY_SZ]; + kvrecordMakeKey(zClass, zKey, zXKey); if( access(zXKey, R_OK)!=0 || stat(zXKey, &buf)!=0 || !S_ISREG(buf.st_mode) @@ -281,6 +299,8 @@ static int kvstorageRead( return (int)n; } } +#endif /* #ifndef SQLITE_WASM */ + /* ** An internal level of indirection which enables us to replace the @@ -288,17 +308,27 @@ static int kvstorageRead( ** Maintenance reminder: if this struct changes in any way, the JSON ** rendering of its structure must be updated in ** sqlite3-wasm.c:sqlite3__wasm_enum_json(). There are no binary -** compatibility concerns, so it does not need an iVersion -** member. +** compatibility concerns, so it does not need an iVersion member. */ typedef struct sqlite3_kvvfs_methods sqlite3_kvvfs_methods; struct sqlite3_kvvfs_methods { - int (*xRead)(const char *zClass, const char *zKey, char *zBuf, int nBuf); - int (*xWrite)(const char *zClass, const char *zKey, const char *zData); - int (*xDelete)(const char *zClass, const char *zKey); + int (*xRcrdRead)(const char*, const char *zKey, char *zBuf, int nBuf); + int (*xRcrdWrite)(const char*, const char *zKey, const char *zData); + int (*xRcrdDelete)(const char*, const char *zKey); const int nKeySize; + const int nBufferSize; +#ifndef SQLITE_WASM +# define MAYBE_CONST const +#else +# define MAYBE_CONST +#endif + MAYBE_CONST sqlite3_vfs * pVfs; + MAYBE_CONST sqlite3_io_methods *pIoDb; + MAYBE_CONST sqlite3_io_methods *pIoJrnl; +#undef MAYBE_CONST }; + /* ** This object holds the kvvfs I/O methods which may be swapped out ** for JavaScript-side implementations in WASM builds. In such builds @@ -313,10 +343,20 @@ struct sqlite3_kvvfs_methods { const #endif sqlite3_kvvfs_methods sqlite3KvvfsMethods = { -kvstorageRead, -kvstorageWrite, -kvstorageDelete, -KVSTORAGE_KEY_SZ +#ifndef SQLITE_WASM + .xRcrdRead = kvrecordRead, + .xRcrdWrite = kvrecordWrite, + .xRcrdDelete = kvrecordDelete, +#else + .xRcrdRead = 0, + .xRcrdWrite = 0, + .xRcrdDelete = 0, +#endif + .nKeySize = KVRECORD_KEY_SZ, + .nBufferSize = SQLITE_KVOS_SZ, + .pVfs = &sqlite3OsKvvfsObject, + .pIoDb = &kvvfs_db_io_methods, + .pIoJrnl = &kvvfs_jrnl_io_methods }; /****** Utility subroutines ************************************************/ @@ -343,7 +383,10 @@ KVSTORAGE_KEY_SZ ** of hexadecimal and base-26 numbers, it is always clear where ** one stops and the next begins. */ -static int kvvfsEncode(const char *aData, int nData, char *aOut){ +#ifndef SQLITE_WASM +static +#endif +int kvvfsEncode(const char *aData, int nData, char *aOut){ int i, j; const unsigned char *a = (const unsigned char*)aData; for(i=j=0; i<nData; i++){ @@ -394,9 +437,13 @@ static const signed char kvvfsHexValue[256] = { ** Decode the text encoding back to binary. The binary content is ** written into pOut, which must be at least nOut bytes in length. ** -** The return value is the number of bytes actually written into aOut[]. +** The return value is the number of bytes actually written into aOut[], or +** -1 for malformed inputs. */ -static int kvvfsDecode(const char *a, char *aOut, int nOut){ +#ifndef SQLITE_WASM +static +#endif +int kvvfsDecode(const char *a, char *aOut, int nOut){ int i, j; int c; const unsigned char *aIn = (const unsigned char*)a; @@ -421,7 +468,7 @@ static int kvvfsDecode(const char *a, char *aOut, int nOut){ }else{ aOut[j] = c<<4; c = kvvfsHexValue[aIn[++i]]; - if( c<0 ) break; + if( c<0 ) return -1 /* hex bytes are always in pairs */; aOut[j++] += c; i++; } @@ -474,13 +521,14 @@ static void kvvfsDecodeJournal( static sqlite3_int64 kvvfsReadFileSize(KVVfsFile *pFile){ char zData[50]; zData[0] = 0; - sqlite3KvvfsMethods.xRead(pFile->zClass, "sz", zData, sizeof(zData)-1); + sqlite3KvvfsMethods.xRcrdRead(pFile->zClass, "sz", zData, + sizeof(zData)-1); return strtoll(zData, 0, 0); } static int kvvfsWriteFileSize(KVVfsFile *pFile, sqlite3_int64 sz){ char zData[50]; sqlite3_snprintf(sizeof(zData), zData, "%lld", sz); - return sqlite3KvvfsMethods.xWrite(pFile->zClass, "sz", zData); + return sqlite3KvvfsMethods.xRcrdWrite(pFile->zClass, "sz", zData); } /****** sqlite3_io_methods methods ******************************************/ @@ -491,10 +539,13 @@ static int kvvfsWriteFileSize(KVVfsFile *pFile, sqlite3_int64 sz){ static int kvvfsClose(sqlite3_file *pProtoFile){ KVVfsFile *pFile = (KVVfsFile *)pProtoFile; - SQLITE_KV_LOG(("xClose %s %s\n", pFile->zClass, + SQLITE_KV_LOG(("xClose %s %s\n", pFile->zClass, pFile->isJournal ? "journal" : "db")); sqlite3_free(pFile->aJrnl); sqlite3_free(pFile->aData); +#ifdef SQLITE_WASM + memset(pFile, 0, sizeof(*pFile)); +#endif return SQLITE_OK; } @@ -503,24 +554,30 @@ static int kvvfsClose(sqlite3_file *pProtoFile){ */ static int kvvfsReadJrnl( sqlite3_file *pProtoFile, - void *zBuf, - int iAmt, + void *zBuf, + int iAmt, sqlite_int64 iOfst ){ KVVfsFile *pFile = (KVVfsFile*)pProtoFile; assert( pFile->isJournal ); SQLITE_KV_LOG(("xRead('%s-journal',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); if( pFile->aJrnl==0 ){ - int szTxt = kvstorageRead(pFile->zClass, "jrnl", 0, 0); + int rc; + int szTxt = sqlite3KvvfsMethods.xRcrdRead(pFile->zClass, "jrnl", + 0, 0); char *aTxt; if( szTxt<=4 ){ return SQLITE_IOERR; } aTxt = sqlite3_malloc64( szTxt+1 ); if( aTxt==0 ) return SQLITE_NOMEM; - kvstorageRead(pFile->zClass, "jrnl", aTxt, szTxt+1); - kvvfsDecodeJournal(pFile, aTxt, szTxt); + rc = sqlite3KvvfsMethods.xRcrdRead(pFile->zClass, "jrnl", + aTxt, szTxt+1); + if( rc>=0 ){ + kvvfsDecodeJournal(pFile, aTxt, szTxt); + } sqlite3_free(aTxt); + if( rc ) return rc; if( pFile->aJrnl==0 ) return SQLITE_IOERR; } if( iOfst+iAmt>pFile->nJrnl ){ @@ -535,8 +592,8 @@ static int kvvfsReadJrnl( */ static int kvvfsReadDb( sqlite3_file *pProtoFile, - void *zBuf, - int iAmt, + void *zBuf, + int iAmt, sqlite_int64 iOfst ){ KVVfsFile *pFile = (KVVfsFile*)pProtoFile; @@ -560,8 +617,8 @@ static int kvvfsReadDb( pgno = 1; } sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); - got = sqlite3KvvfsMethods.xRead(pFile->zClass, zKey, - aData, SQLITE_KVOS_SZ-1); + got = sqlite3KvvfsMethods.xRcrdRead(pFile->zClass, zKey, + aData, SQLITE_KVOS_SZ-1); if( got<0 ){ n = 0; }else{ @@ -593,8 +650,8 @@ static int kvvfsReadDb( */ static int kvvfsWriteJrnl( sqlite3_file *pProtoFile, - const void *zBuf, - int iAmt, + const void *zBuf, + int iAmt, sqlite_int64 iOfst ){ KVVfsFile *pFile = (KVVfsFile*)pProtoFile; @@ -621,14 +678,15 @@ static int kvvfsWriteJrnl( */ static int kvvfsWriteDb( sqlite3_file *pProtoFile, - const void *zBuf, - int iAmt, + const void *zBuf, + int iAmt, sqlite_int64 iOfst ){ KVVfsFile *pFile = (KVVfsFile*)pProtoFile; unsigned int pgno; char zKey[30]; char *aData = pFile->aData; + int rc; SQLITE_KV_LOG(("xWrite('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); assert( iAmt>=512 && iAmt<=65536 ); assert( (iAmt & (iAmt-1))==0 ); @@ -637,13 +695,13 @@ static int kvvfsWriteDb( pgno = 1 + iOfst/iAmt; sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); kvvfsEncode(zBuf, iAmt, aData); - if( sqlite3KvvfsMethods.xWrite(pFile->zClass, zKey, aData) ){ - return SQLITE_IOERR; - } - if( iOfst+iAmt > pFile->szDb ){ - pFile->szDb = iOfst + iAmt; + rc = sqlite3KvvfsMethods.xRcrdWrite(pFile->zClass, zKey, aData); + if( 0==rc ){ + if( iOfst+iAmt > pFile->szDb ){ + pFile->szDb = iOfst + iAmt; + } } - return SQLITE_OK; + return rc; } /* @@ -653,7 +711,7 @@ static int kvvfsTruncateJrnl(sqlite3_file *pProtoFile, sqlite_int64 size){ KVVfsFile *pFile = (KVVfsFile *)pProtoFile; SQLITE_KV_LOG(("xTruncate('%s-journal',%lld)\n", pFile->zClass, size)); assert( size==0 ); - sqlite3KvvfsMethods.xDelete(pFile->zClass, "jrnl"); + sqlite3KvvfsMethods.xRcrdDelete(pFile->zClass, "jrnl"); sqlite3_free(pFile->aJrnl); pFile->aJrnl = 0; pFile->nJrnl = 0; @@ -662,7 +720,7 @@ static int kvvfsTruncateJrnl(sqlite3_file *pProtoFile, sqlite_int64 size){ static int kvvfsTruncateDb(sqlite3_file *pProtoFile, sqlite_int64 size){ KVVfsFile *pFile = (KVVfsFile *)pProtoFile; if( pFile->szDb>size - && pFile->szPage>0 + && pFile->szPage>0 && (size % pFile->szPage)==0 ){ char zKey[50]; @@ -672,7 +730,7 @@ static int kvvfsTruncateDb(sqlite3_file *pProtoFile, sqlite_int64 size){ pgnoMax = 2 + pFile->szDb/pFile->szPage; while( pgno<=pgnoMax ){ sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); - sqlite3KvvfsMethods.xDelete(pFile->zClass, zKey); + sqlite3KvvfsMethods.xRcrdDelete(pFile->zClass, zKey); pgno++; } pFile->szDb = size; @@ -704,7 +762,7 @@ static int kvvfsSyncJrnl(sqlite3_file *pProtoFile, int flags){ }while( n>0 ); zOut[i++] = ' '; kvvfsEncode(pFile->aJrnl, pFile->nJrnl, &zOut[i]); - i = sqlite3KvvfsMethods.xWrite(pFile->zClass, "jrnl", zOut); + i = sqlite3KvvfsMethods.xRcrdWrite(pFile->zClass, "jrnl", zOut); sqlite3_free(zOut); return i ? SQLITE_IOERR : SQLITE_OK; } @@ -818,33 +876,32 @@ static int kvvfsOpen( KVVfsFile *pFile = (KVVfsFile*)pProtoFile; if( zName==0 ) zName = ""; SQLITE_KV_LOG(("xOpen(\"%s\")\n", zName)); - if( strcmp(zName, "local")==0 - || strcmp(zName, "session")==0 - ){ - pFile->isJournal = 0; - pFile->base.pMethods = &kvvfs_db_io_methods; - }else - if( strcmp(zName, "local-journal")==0 - || strcmp(zName, "session-journal")==0 - ){ + assert(!pFile->zClass); + assert(!pFile->aData); + assert(!pFile->aJrnl); + assert(!pFile->nJrnl); + assert(!pFile->base.pMethods); + pFile->szPage = -1; + pFile->szDb = -1; + if( 0==sqlite3_strglob("*-journal", zName) ){ pFile->isJournal = 1; pFile->base.pMethods = &kvvfs_jrnl_io_methods; + if( 0==strcmp("session-journal",zName) ){ + pFile->zClass = "session"; + }else if( 0==strcmp("local-journal",zName) ){ + pFile->zClass = "local"; + } }else{ - return SQLITE_CANTOPEN; + pFile->isJournal = 0; + pFile->base.pMethods = &kvvfs_db_io_methods; } - if( zName[0]=='s' ){ - pFile->zClass = "session"; - }else{ - pFile->zClass = "local"; + if( !pFile->zClass ){ + pFile->zClass = zName; } pFile->aData = sqlite3_malloc64(SQLITE_KVOS_SZ); if( pFile->aData==0 ){ return SQLITE_NOMEM; } - pFile->aJrnl = 0; - pFile->nJrnl = 0; - pFile->szPage = -1; - pFile->szDb = -1; return SQLITE_OK; } @@ -854,13 +911,17 @@ static int kvvfsOpen( ** returning. */ static int kvvfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + int rc /* The JS impl can fail with OOM in argument conversion */; if( strcmp(zPath, "local-journal")==0 ){ - sqlite3KvvfsMethods.xDelete("local", "jrnl"); + rc = sqlite3KvvfsMethods.xRcrdDelete("local", "jrnl"); }else if( strcmp(zPath, "session-journal")==0 ){ - sqlite3KvvfsMethods.xDelete("session", "jrnl"); + rc = sqlite3KvvfsMethods.xRcrdDelete("session", "jrnl"); } - return SQLITE_OK; + else{ + rc = 0; + } + return rc; } /* @@ -868,27 +929,48 @@ static int kvvfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ ** is available, or false otherwise. */ static int kvvfsAccess( - sqlite3_vfs *pProtoVfs, - const char *zPath, - int flags, + sqlite3_vfs *pProtoVfs, + const char *zPath, + int flags, int *pResOut ){ SQLITE_KV_LOG(("xAccess(\"%s\")\n", zPath)); +#if 0 && defined(SQLITE_WASM) + /* + ** This is not having the desired effect in the JS bindings. + ** It's ostensibly the same logic as the #else block, but + ** it's not behaving that way. + ** + ** In JS we map all zPaths to Storage objects, and -journal files + ** are mapped to the storage for the main db (which is is exactly + ** what the mapping of "local-journal" -> "local" is doing). + */ + const char *zKey = (0==sqlite3_strglob("*-journal", zPath)) + ? "jrnl" : "sz"; + *pResOut = + sqlite3KvvfsMethods.xRcrdRead(zPath, zKey, 0, 0)>0; +#else if( strcmp(zPath, "local-journal")==0 ){ - *pResOut = sqlite3KvvfsMethods.xRead("local", "jrnl", 0, 0)>0; + *pResOut = + sqlite3KvvfsMethods.xRcrdRead("local", "jrnl", 0, 0)>0; }else if( strcmp(zPath, "session-journal")==0 ){ - *pResOut = sqlite3KvvfsMethods.xRead("session", "jrnl", 0, 0)>0; + *pResOut = + sqlite3KvvfsMethods.xRcrdRead("session", "jrnl", 0, 0)>0; }else if( strcmp(zPath, "local")==0 ){ - *pResOut = sqlite3KvvfsMethods.xRead("local", "sz", 0, 0)>0; + *pResOut = + sqlite3KvvfsMethods.xRcrdRead("local", "sz", 0, 0)>0; }else if( strcmp(zPath, "session")==0 ){ - *pResOut = sqlite3KvvfsMethods.xRead("session", "sz", 0, 0)>0; + *pResOut = + sqlite3KvvfsMethods.xRcrdRead("session", "sz", 0, 0)>0; }else { *pResOut = 0; } + /*all current JS tests avoid triggering: assert( *pResOut == 0 ); */ +#endif SQLITE_KV_LOG(("xAccess returns %d\n",*pResOut)); return SQLITE_OK; } @@ -899,9 +981,9 @@ static int kvvfsAccess( ** of at least (INST_MAX_PATHNAME+1) bytes. */ static int kvvfsFullPathname( - sqlite3_vfs *pVfs, - const char *zPath, - int nOut, + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, char *zOut ){ size_t nPath; @@ -924,7 +1006,7 @@ static void *kvvfsDlOpen(sqlite3_vfs *pVfs, const char *zPath){ } /* -** Populate the buffer pointed to by zBufOut with nByte bytes of +** Populate the buffer pointed to by zBufOut with nByte bytes of ** random data. */ static int kvvfsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ @@ -933,7 +1015,7 @@ static int kvvfsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ } /* -** Sleep for nMicro microseconds. Return the number of microseconds +** Sleep for nMicro microseconds. Return the number of microseconds ** actually slept. */ static int kvvfsSleep(sqlite3_vfs *pVfs, int nMicro){ @@ -961,7 +1043,7 @@ static int kvvfsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ #endif /* SQLITE_OS_KV || SQLITE_OS_UNIX */ #if SQLITE_OS_KV -/* +/* ** This routine is called initialize the KV-vfs as the default VFS. */ int sqlite3_os_init(void){ diff --git a/src/os_unix.c b/src/os_unix.c index d73d89924..2f75829c8 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5190,7 +5190,7 @@ static int unixShmMap( } /* Map the requested memory region into this processes address space. */ - apNew = (char **)sqlite3_realloc( + apNew = (char **)sqlite3_realloc64( pShmNode->apRegion, nReqRegion*sizeof(char *) ); if( !apNew ){ diff --git a/src/os_win.c b/src/os_win.c index a6b25f2e8..7583ecc1f 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -43,7 +43,7 @@ ** Are most of the Win32 ANSI APIs available (i.e. with certain exceptions ** based on the sub-platform)? */ -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(SQLITE_WIN32_NO_ANSI) +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_ANSI) # define SQLITE_WIN32_HAS_ANSI #endif @@ -51,7 +51,7 @@ ** Are most of the Win32 Unicode APIs available (i.e. with certain exceptions ** based on the sub-platform)? */ -#if (SQLITE_OS_WINCE || SQLITE_OS_WINNT || SQLITE_OS_WINRT) && \ +#if (SQLITE_OS_WINCE || SQLITE_OS_WINNT) && \ !defined(SQLITE_WIN32_NO_WIDE) # define SQLITE_WIN32_HAS_WIDE #endif @@ -190,16 +190,7 @@ */ #if SQLITE_WIN32_FILEMAPPING_API && \ (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) -/* -** Two of the file mapping APIs are different under WinRT. Figure out which -** set we need. -*/ -#if SQLITE_OS_WINRT -WINBASEAPI HANDLE WINAPI CreateFileMappingFromApp(HANDLE, \ - LPSECURITY_ATTRIBUTES, ULONG, ULONG64, LPCWSTR); -WINBASEAPI LPVOID WINAPI MapViewOfFileFromApp(HANDLE, ULONG, ULONG64, SIZE_T); -#else #if defined(SQLITE_WIN32_HAS_ANSI) WINBASEAPI HANDLE WINAPI CreateFileMappingA(HANDLE, LPSECURITY_ATTRIBUTES, \ DWORD, DWORD, DWORD, LPCSTR); @@ -211,7 +202,6 @@ WINBASEAPI HANDLE WINAPI CreateFileMappingW(HANDLE, LPSECURITY_ATTRIBUTES, \ #endif /* defined(SQLITE_WIN32_HAS_WIDE) */ WINBASEAPI LPVOID WINAPI MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); -#endif /* SQLITE_OS_WINRT */ /* ** These file mapping APIs are common to both Win32 and WinRT. @@ -502,7 +492,7 @@ static LONG SQLITE_WIN32_VOLATILE sqlite3_os_type = 0; ** This function is not available on Windows CE or WinRT. */ -#if SQLITE_OS_WINCE || SQLITE_OS_WINRT +#if SQLITE_OS_WINCE # define osAreFileApisANSI() 1 #endif @@ -517,7 +507,7 @@ static struct win_syscall { sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ sqlite3_syscall_ptr pDefault; /* Default value */ } aSyscall[] = { -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE { "AreFileApisANSI", (SYSCALL)AreFileApisANSI, 0 }, #else { "AreFileApisANSI", (SYSCALL)0, 0 }, @@ -556,7 +546,7 @@ static struct win_syscall { #define osCreateFileA ((HANDLE(WINAPI*)(LPCSTR,DWORD,DWORD, \ LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[4].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if defined(SQLITE_WIN32_HAS_WIDE) { "CreateFileW", (SYSCALL)CreateFileW, 0 }, #else { "CreateFileW", (SYSCALL)0, 0 }, @@ -565,7 +555,7 @@ static struct win_syscall { #define osCreateFileW ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD, \ LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[5].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_ANSI) && \ +#if defined(SQLITE_WIN32_HAS_ANSI) && \ (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) && \ SQLITE_WIN32_CREATEFILEMAPPINGA { "CreateFileMappingA", (SYSCALL)CreateFileMappingA, 0 }, @@ -576,8 +566,8 @@ static struct win_syscall { #define osCreateFileMappingA ((HANDLE(WINAPI*)(HANDLE,LPSECURITY_ATTRIBUTES, \ DWORD,DWORD,DWORD,LPCSTR))aSyscall[6].pCurrent) -#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ - (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) +#if (SQLITE_OS_WINCE || defined(SQLITE_WIN32_HAS_WIDE)) && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) { "CreateFileMappingW", (SYSCALL)CreateFileMappingW, 0 }, #else { "CreateFileMappingW", (SYSCALL)0, 0 }, @@ -586,7 +576,7 @@ static struct win_syscall { #define osCreateFileMappingW ((HANDLE(WINAPI*)(HANDLE,LPSECURITY_ATTRIBUTES, \ DWORD,DWORD,DWORD,LPCWSTR))aSyscall[7].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if defined(SQLITE_WIN32_HAS_WIDE) { "CreateMutexW", (SYSCALL)CreateMutexW, 0 }, #else { "CreateMutexW", (SYSCALL)0, 0 }, @@ -672,7 +662,7 @@ static struct win_syscall { #define osGetDiskFreeSpaceA ((BOOL(WINAPI*)(LPCSTR,LPDWORD,LPDWORD,LPDWORD, \ LPDWORD))aSyscall[18].pCurrent) -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_WIDE) { "GetDiskFreeSpaceW", (SYSCALL)GetDiskFreeSpaceW, 0 }, #else { "GetDiskFreeSpaceW", (SYSCALL)0, 0 }, @@ -689,7 +679,7 @@ static struct win_syscall { #define osGetFileAttributesA ((DWORD(WINAPI*)(LPCSTR))aSyscall[20].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if defined(SQLITE_WIN32_HAS_WIDE) { "GetFileAttributesW", (SYSCALL)GetFileAttributesW, 0 }, #else { "GetFileAttributesW", (SYSCALL)0, 0 }, @@ -706,11 +696,7 @@ static struct win_syscall { #define osGetFileAttributesExW ((BOOL(WINAPI*)(LPCWSTR,GET_FILEEX_INFO_LEVELS, \ LPVOID))aSyscall[22].pCurrent) -#if !SQLITE_OS_WINRT { "GetFileSize", (SYSCALL)GetFileSize, 0 }, -#else - { "GetFileSize", (SYSCALL)0, 0 }, -#endif #define osGetFileSize ((DWORD(WINAPI*)(HANDLE,LPDWORD))aSyscall[23].pCurrent) @@ -723,7 +709,7 @@ static struct win_syscall { #define osGetFullPathNameA ((DWORD(WINAPI*)(LPCSTR,DWORD,LPSTR, \ LPSTR*))aSyscall[24].pCurrent) -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_WIDE) { "GetFullPathNameW", (SYSCALL)GetFullPathNameW, 0 }, #else { "GetFullPathNameW", (SYSCALL)0, 0 }, @@ -758,16 +744,10 @@ static struct win_syscall { #define osGetProcAddressA ((FARPROC(WINAPI*)(HMODULE, \ LPCSTR))aSyscall[27].pCurrent) -#if !SQLITE_OS_WINRT { "GetSystemInfo", (SYSCALL)GetSystemInfo, 0 }, -#else - { "GetSystemInfo", (SYSCALL)0, 0 }, -#endif - #define osGetSystemInfo ((VOID(WINAPI*)(LPSYSTEM_INFO))aSyscall[28].pCurrent) { "GetSystemTime", (SYSCALL)GetSystemTime, 0 }, - #define osGetSystemTime ((VOID(WINAPI*)(LPSYSTEMTIME))aSyscall[29].pCurrent) #if !SQLITE_OS_WINCE @@ -787,7 +767,7 @@ static struct win_syscall { #define osGetTempPathA ((DWORD(WINAPI*)(DWORD,LPSTR))aSyscall[31].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if defined(SQLITE_WIN32_HAS_WIDE) { "GetTempPathW", (SYSCALL)GetTempPathW, 0 }, #else { "GetTempPathW", (SYSCALL)0, 0 }, @@ -795,11 +775,7 @@ static struct win_syscall { #define osGetTempPathW ((DWORD(WINAPI*)(DWORD,LPWSTR))aSyscall[32].pCurrent) -#if !SQLITE_OS_WINRT { "GetTickCount", (SYSCALL)GetTickCount, 0 }, -#else - { "GetTickCount", (SYSCALL)0, 0 }, -#endif #define osGetTickCount ((DWORD(WINAPI*)(VOID))aSyscall[33].pCurrent) @@ -812,7 +788,7 @@ static struct win_syscall { #define osGetVersionExA ((BOOL(WINAPI*)( \ LPOSVERSIONINFOA))aSyscall[34].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ +#if defined(SQLITE_WIN32_HAS_WIDE) && \ SQLITE_WIN32_GETVERSIONEX { "GetVersionExW", (SYSCALL)GetVersionExW, 0 }, #else @@ -827,20 +803,12 @@ static struct win_syscall { #define osHeapAlloc ((LPVOID(WINAPI*)(HANDLE,DWORD, \ SIZE_T))aSyscall[36].pCurrent) -#if !SQLITE_OS_WINRT { "HeapCreate", (SYSCALL)HeapCreate, 0 }, -#else - { "HeapCreate", (SYSCALL)0, 0 }, -#endif #define osHeapCreate ((HANDLE(WINAPI*)(DWORD,SIZE_T, \ SIZE_T))aSyscall[37].pCurrent) -#if !SQLITE_OS_WINRT { "HeapDestroy", (SYSCALL)HeapDestroy, 0 }, -#else - { "HeapDestroy", (SYSCALL)0, 0 }, -#endif #define osHeapDestroy ((BOOL(WINAPI*)(HANDLE))aSyscall[38].pCurrent) @@ -858,16 +826,12 @@ static struct win_syscall { #define osHeapSize ((SIZE_T(WINAPI*)(HANDLE,DWORD, \ LPCVOID))aSyscall[41].pCurrent) -#if !SQLITE_OS_WINRT { "HeapValidate", (SYSCALL)HeapValidate, 0 }, -#else - { "HeapValidate", (SYSCALL)0, 0 }, -#endif #define osHeapValidate ((BOOL(WINAPI*)(HANDLE,DWORD, \ LPCVOID))aSyscall[42].pCurrent) -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE { "HeapCompact", (SYSCALL)HeapCompact, 0 }, #else { "HeapCompact", (SYSCALL)0, 0 }, @@ -883,7 +847,7 @@ static struct win_syscall { #define osLoadLibraryA ((HMODULE(WINAPI*)(LPCSTR))aSyscall[44].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ +#if defined(SQLITE_WIN32_HAS_WIDE) && \ !defined(SQLITE_OMIT_LOAD_EXTENSION) { "LoadLibraryW", (SYSCALL)LoadLibraryW, 0 }, #else @@ -892,15 +856,11 @@ static struct win_syscall { #define osLoadLibraryW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[45].pCurrent) -#if !SQLITE_OS_WINRT { "LocalFree", (SYSCALL)LocalFree, 0 }, -#else - { "LocalFree", (SYSCALL)0, 0 }, -#endif #define osLocalFree ((HLOCAL(WINAPI*)(HLOCAL))aSyscall[46].pCurrent) -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE { "LockFile", (SYSCALL)LockFile, 0 }, #else { "LockFile", (SYSCALL)0, 0 }, @@ -922,8 +882,7 @@ static struct win_syscall { LPOVERLAPPED))aSyscall[48].pCurrent) #endif -#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && \ - (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) +#if SQLITE_OS_WINCE || !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 { "MapViewOfFile", (SYSCALL)MapViewOfFile, 0 }, #else { "MapViewOfFile", (SYSCALL)0, 0 }, @@ -951,20 +910,12 @@ static struct win_syscall { #define osSetEndOfFile ((BOOL(WINAPI*)(HANDLE))aSyscall[53].pCurrent) -#if !SQLITE_OS_WINRT { "SetFilePointer", (SYSCALL)SetFilePointer, 0 }, -#else - { "SetFilePointer", (SYSCALL)0, 0 }, -#endif #define osSetFilePointer ((DWORD(WINAPI*)(HANDLE,LONG,PLONG, \ DWORD))aSyscall[54].pCurrent) -#if !SQLITE_OS_WINRT { "Sleep", (SYSCALL)Sleep, 0 }, -#else - { "Sleep", (SYSCALL)0, 0 }, -#endif #define osSleep ((VOID(WINAPI*)(DWORD))aSyscall[55].pCurrent) @@ -973,7 +924,7 @@ static struct win_syscall { #define osSystemTimeToFileTime ((BOOL(WINAPI*)(const SYSTEMTIME*, \ LPFILETIME))aSyscall[56].pCurrent) -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE { "UnlockFile", (SYSCALL)UnlockFile, 0 }, #else { "UnlockFile", (SYSCALL)0, 0 }, @@ -1011,15 +962,6 @@ static struct win_syscall { #define osWriteFile ((BOOL(WINAPI*)(HANDLE,LPCVOID,DWORD,LPDWORD, \ LPOVERLAPPED))aSyscall[61].pCurrent) -#if SQLITE_OS_WINRT - { "CreateEventExW", (SYSCALL)CreateEventExW, 0 }, -#else - { "CreateEventExW", (SYSCALL)0, 0 }, -#endif - -#define osCreateEventExW ((HANDLE(WINAPI*)(LPSECURITY_ATTRIBUTES,LPCWSTR, \ - DWORD,DWORD))aSyscall[62].pCurrent) - /* ** For WaitForSingleObject(), MSDN says: ** @@ -1029,7 +971,7 @@ static struct win_syscall { { "WaitForSingleObject", (SYSCALL)WaitForSingleObject, 0 }, #define osWaitForSingleObject ((DWORD(WINAPI*)(HANDLE, \ - DWORD))aSyscall[63].pCurrent) + DWORD))aSyscall[62].pCurrent) #if !SQLITE_OS_WINCE { "WaitForSingleObjectEx", (SYSCALL)WaitForSingleObjectEx, 0 }, @@ -1038,69 +980,12 @@ static struct win_syscall { #endif #define osWaitForSingleObjectEx ((DWORD(WINAPI*)(HANDLE,DWORD, \ - BOOL))aSyscall[64].pCurrent) - -#if SQLITE_OS_WINRT - { "SetFilePointerEx", (SYSCALL)SetFilePointerEx, 0 }, -#else - { "SetFilePointerEx", (SYSCALL)0, 0 }, -#endif - -#define osSetFilePointerEx ((BOOL(WINAPI*)(HANDLE,LARGE_INTEGER, \ - PLARGE_INTEGER,DWORD))aSyscall[65].pCurrent) + BOOL))aSyscall[63].pCurrent) -#if SQLITE_OS_WINRT - { "GetFileInformationByHandleEx", (SYSCALL)GetFileInformationByHandleEx, 0 }, -#else - { "GetFileInformationByHandleEx", (SYSCALL)0, 0 }, -#endif - -#define osGetFileInformationByHandleEx ((BOOL(WINAPI*)(HANDLE, \ - FILE_INFO_BY_HANDLE_CLASS,LPVOID,DWORD))aSyscall[66].pCurrent) - -#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) - { "MapViewOfFileFromApp", (SYSCALL)MapViewOfFileFromApp, 0 }, -#else - { "MapViewOfFileFromApp", (SYSCALL)0, 0 }, -#endif - -#define osMapViewOfFileFromApp ((LPVOID(WINAPI*)(HANDLE,ULONG,ULONG64, \ - SIZE_T))aSyscall[67].pCurrent) - -#if SQLITE_OS_WINRT - { "CreateFile2", (SYSCALL)CreateFile2, 0 }, -#else - { "CreateFile2", (SYSCALL)0, 0 }, -#endif - -#define osCreateFile2 ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD,DWORD, \ - LPCREATEFILE2_EXTENDED_PARAMETERS))aSyscall[68].pCurrent) - -#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_LOAD_EXTENSION) - { "LoadPackagedLibrary", (SYSCALL)LoadPackagedLibrary, 0 }, -#else - { "LoadPackagedLibrary", (SYSCALL)0, 0 }, -#endif - -#define osLoadPackagedLibrary ((HMODULE(WINAPI*)(LPCWSTR, \ - DWORD))aSyscall[69].pCurrent) - -#if SQLITE_OS_WINRT - { "GetTickCount64", (SYSCALL)GetTickCount64, 0 }, -#else - { "GetTickCount64", (SYSCALL)0, 0 }, -#endif - -#define osGetTickCount64 ((ULONGLONG(WINAPI*)(VOID))aSyscall[70].pCurrent) - -#if SQLITE_OS_WINRT { "GetNativeSystemInfo", (SYSCALL)GetNativeSystemInfo, 0 }, -#else - { "GetNativeSystemInfo", (SYSCALL)0, 0 }, -#endif #define osGetNativeSystemInfo ((VOID(WINAPI*)( \ - LPSYSTEM_INFO))aSyscall[71].pCurrent) + LPSYSTEM_INFO))aSyscall[64].pCurrent) #if defined(SQLITE_WIN32_HAS_ANSI) { "OutputDebugStringA", (SYSCALL)OutputDebugStringA, 0 }, @@ -1108,7 +993,7 @@ static struct win_syscall { { "OutputDebugStringA", (SYSCALL)0, 0 }, #endif -#define osOutputDebugStringA ((VOID(WINAPI*)(LPCSTR))aSyscall[72].pCurrent) +#define osOutputDebugStringA ((VOID(WINAPI*)(LPCSTR))aSyscall[65].pCurrent) #if defined(SQLITE_WIN32_HAS_WIDE) { "OutputDebugStringW", (SYSCALL)OutputDebugStringW, 0 }, @@ -1116,20 +1001,11 @@ static struct win_syscall { { "OutputDebugStringW", (SYSCALL)0, 0 }, #endif -#define osOutputDebugStringW ((VOID(WINAPI*)(LPCWSTR))aSyscall[73].pCurrent) +#define osOutputDebugStringW ((VOID(WINAPI*)(LPCWSTR))aSyscall[66].pCurrent) { "GetProcessHeap", (SYSCALL)GetProcessHeap, 0 }, -#define osGetProcessHeap ((HANDLE(WINAPI*)(VOID))aSyscall[74].pCurrent) - -#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) - { "CreateFileMappingFromApp", (SYSCALL)CreateFileMappingFromApp, 0 }, -#else - { "CreateFileMappingFromApp", (SYSCALL)0, 0 }, -#endif - -#define osCreateFileMappingFromApp ((HANDLE(WINAPI*)(HANDLE, \ - LPSECURITY_ATTRIBUTES,ULONG,ULONG64,LPCWSTR))aSyscall[75].pCurrent) +#define osGetProcessHeap ((HANDLE(WINAPI*)(VOID))aSyscall[67].pCurrent) /* ** NOTE: On some sub-platforms, the InterlockedCompareExchange "function" @@ -1144,25 +1020,25 @@ static struct win_syscall { { "InterlockedCompareExchange", (SYSCALL)InterlockedCompareExchange, 0 }, #define osInterlockedCompareExchange ((LONG(WINAPI*)(LONG \ - SQLITE_WIN32_VOLATILE*, LONG,LONG))aSyscall[76].pCurrent) + SQLITE_WIN32_VOLATILE*, LONG,LONG))aSyscall[68].pCurrent) #endif /* defined(InterlockedCompareExchange) */ -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID +#if !SQLITE_OS_WINCE && SQLITE_WIN32_USE_UUID { "UuidCreate", (SYSCALL)UuidCreate, 0 }, #else { "UuidCreate", (SYSCALL)0, 0 }, #endif -#define osUuidCreate ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[77].pCurrent) +#define osUuidCreate ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[69].pCurrent) -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID +#if !SQLITE_OS_WINCE && SQLITE_WIN32_USE_UUID { "UuidCreateSequential", (SYSCALL)UuidCreateSequential, 0 }, #else { "UuidCreateSequential", (SYSCALL)0, 0 }, #endif #define osUuidCreateSequential \ - ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[78].pCurrent) + ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[70].pCurrent) #if !defined(SQLITE_NO_SYNC) && SQLITE_MAX_MMAP_SIZE>0 { "FlushViewOfFile", (SYSCALL)FlushViewOfFile, 0 }, @@ -1171,7 +1047,7 @@ static struct win_syscall { #endif #define osFlushViewOfFile \ - ((BOOL(WINAPI*)(LPCVOID,SIZE_T))aSyscall[79].pCurrent) + ((BOOL(WINAPI*)(LPCVOID,SIZE_T))aSyscall[71].pCurrent) /* ** If SQLITE_ENABLE_SETLK_TIMEOUT is defined, we require CreateEvent() @@ -1188,7 +1064,7 @@ static struct win_syscall { #define osCreateEvent ( \ (HANDLE(WINAPI*) (LPSECURITY_ATTRIBUTES,BOOL,BOOL,LPCSTR)) \ - aSyscall[80].pCurrent \ + aSyscall[72].pCurrent \ ) /* @@ -1205,7 +1081,7 @@ static struct win_syscall { { "CancelIo", (SYSCALL)0, 0 }, #endif -#define osCancelIo ((BOOL(WINAPI*)(HANDLE))aSyscall[81].pCurrent) +#define osCancelIo ((BOOL(WINAPI*)(HANDLE))aSyscall[73].pCurrent) #if defined(SQLITE_WIN32_HAS_WIDE) && defined(_WIN32) { "GetModuleHandleW", (SYSCALL)GetModuleHandleW, 0 }, @@ -1213,7 +1089,7 @@ static struct win_syscall { { "GetModuleHandleW", (SYSCALL)0, 0 }, #endif -#define osGetModuleHandleW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[82].pCurrent) +#define osGetModuleHandleW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[74].pCurrent) #ifndef _WIN32 { "getenv", (SYSCALL)getenv, 0 }, @@ -1221,7 +1097,7 @@ static struct win_syscall { { "getenv", (SYSCALL)0, 0 }, #endif -#define osGetenv ((const char *(*)(const char *))aSyscall[83].pCurrent) +#define osGetenv ((const char *(*)(const char *))aSyscall[75].pCurrent) #ifndef _WIN32 { "getcwd", (SYSCALL)getcwd, 0 }, @@ -1229,7 +1105,7 @@ static struct win_syscall { { "getcwd", (SYSCALL)0, 0 }, #endif -#define osGetcwd ((char*(*)(char*,size_t))aSyscall[84].pCurrent) +#define osGetcwd ((char*(*)(char*,size_t))aSyscall[76].pCurrent) #ifndef _WIN32 { "readlink", (SYSCALL)readlink, 0 }, @@ -1237,7 +1113,7 @@ static struct win_syscall { { "readlink", (SYSCALL)0, 0 }, #endif -#define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[85].pCurrent) +#define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[77].pCurrent) #ifndef _WIN32 { "lstat", (SYSCALL)lstat, 0 }, @@ -1245,7 +1121,7 @@ static struct win_syscall { { "lstat", (SYSCALL)0, 0 }, #endif -#define osLstat ((int(*)(const char*,struct stat*))aSyscall[86].pCurrent) +#define osLstat ((int(*)(const char*,struct stat*))aSyscall[78].pCurrent) #ifndef _WIN32 { "__errno", (SYSCALL)__errno, 0 }, @@ -1253,7 +1129,7 @@ static struct win_syscall { { "__errno", (SYSCALL)0, 0 }, #endif -#define osErrno (*((int*(*)(void))aSyscall[87].pCurrent)()) +#define osErrno (*((int*(*)(void))aSyscall[79].pCurrent)()) #ifndef _WIN32 { "cygwin_conv_path", (SYSCALL)cygwin_conv_path, 0 }, @@ -1262,7 +1138,7 @@ static struct win_syscall { #endif #define osCygwin_conv_path ((size_t(*)(unsigned int, \ - const void *, void *, size_t))aSyscall[88].pCurrent) + const void *, void *, size_t))aSyscall[80].pCurrent) }; /* End of the overrideable system calls */ @@ -1366,10 +1242,10 @@ int sqlite3_win32_compact_heap(LPUINT pnLargest){ hHeap = winMemGetHeap(); assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); #endif -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE if( (nLargest=osHeapCompact(hHeap, SQLITE_WIN32_HEAP_FLAGS))==0 ){ DWORD lastErrno = osGetLastError(); if( lastErrno==NO_ERROR ){ @@ -1482,28 +1358,11 @@ void sqlite3_win32_write_debug(const char *zBuf, int nBuf){ } #endif /* _WIN32 */ -/* -** The following routine suspends the current thread for at least ms -** milliseconds. This is equivalent to the Win32 Sleep() interface. -*/ -#if SQLITE_OS_WINRT -static HANDLE sleepObj = NULL; -#endif - void sqlite3_win32_sleep(DWORD milliseconds){ -#if SQLITE_OS_WINRT - if ( sleepObj==NULL ){ - sleepObj = osCreateEventExW(NULL, NULL, CREATE_EVENT_MANUAL_RESET, - SYNCHRONIZE); - } - assert( sleepObj!=NULL ); - osWaitForSingleObjectEx(sleepObj, milliseconds, FALSE); -#else osSleep(milliseconds); -#endif } -#if SQLITE_MAX_WORKER_THREADS>0 && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && \ +#if SQLITE_MAX_WORKER_THREADS>0 && !SQLITE_OS_WINCE && \ SQLITE_THREADSAFE>0 DWORD sqlite3Win32Wait(HANDLE hObject){ DWORD rc; @@ -1527,7 +1386,7 @@ DWORD sqlite3Win32Wait(HANDLE hObject){ #if !SQLITE_WIN32_GETVERSIONEX # define osIsNT() (1) -#elif SQLITE_OS_WINCE || SQLITE_OS_WINRT || !defined(SQLITE_WIN32_HAS_ANSI) +#elif SQLITE_OS_WINCE || !defined(SQLITE_WIN32_HAS_ANSI) # define osIsNT() (1) #elif !defined(SQLITE_WIN32_HAS_WIDE) # define osIsNT() (0) @@ -1540,13 +1399,7 @@ DWORD sqlite3Win32Wait(HANDLE hObject){ ** based on the NT kernel. */ int sqlite3_win32_is_nt(void){ -#if SQLITE_OS_WINRT - /* - ** NOTE: The WinRT sub-platform is always assumed to be based on the NT - ** kernel. - */ - return 1; -#elif SQLITE_WIN32_GETVERSIONEX +#if SQLITE_WIN32_GETVERSIONEX if( osInterlockedCompareExchange(&sqlite3_os_type, 0, 0)==0 ){ #if defined(SQLITE_WIN32_HAS_ANSI) OSVERSIONINFOA sInfo; @@ -1588,7 +1441,7 @@ static void *winMemMalloc(int nBytes){ hHeap = winMemGetHeap(); assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); #endif assert( nBytes>=0 ); @@ -1610,7 +1463,7 @@ static void winMemFree(void *pPrior){ hHeap = winMemGetHeap(); assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ); #endif if( !pPrior ) return; /* Passing NULL to HeapFree is undefined. */ @@ -1631,7 +1484,7 @@ static void *winMemRealloc(void *pPrior, int nBytes){ hHeap = winMemGetHeap(); assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ); #endif assert( nBytes>=0 ); @@ -1659,7 +1512,7 @@ static int winMemSize(void *p){ hHeap = winMemGetHeap(); assert( hHeap!=0 ); assert( hHeap!=INVALID_HANDLE_VALUE ); -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, p) ); #endif if( !p ) return 0; @@ -1689,7 +1542,7 @@ static int winMemInit(void *pAppData){ assert( pWinMemData->magic1==WINMEM_MAGIC1 ); assert( pWinMemData->magic2==WINMEM_MAGIC2 ); -#if !SQLITE_OS_WINRT && SQLITE_WIN32_HEAP_CREATE +#if SQLITE_WIN32_HEAP_CREATE if( !pWinMemData->hHeap ){ DWORD dwInitialSize = SQLITE_WIN32_HEAP_INIT_SIZE; DWORD dwMaximumSize = (DWORD)sqlite3GlobalConfig.nHeap; @@ -1722,7 +1575,7 @@ static int winMemInit(void *pAppData){ #endif assert( pWinMemData->hHeap!=0 ); assert( pWinMemData->hHeap!=INVALID_HANDLE_VALUE ); -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(pWinMemData->hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); #endif return SQLITE_OK; @@ -1740,7 +1593,7 @@ static void winMemShutdown(void *pAppData){ if( pWinMemData->hHeap ){ assert( pWinMemData->hHeap!=INVALID_HANDLE_VALUE ); -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) +#if defined(SQLITE_WIN32_MALLOC_VALIDATE) assert( osHeapValidate(pWinMemData->hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); #endif if( pWinMemData->bOwned ){ @@ -2121,17 +1974,6 @@ static int winGetLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ char *zOut = 0; if( osIsNT() ){ -#if SQLITE_OS_WINRT - WCHAR zTempWide[SQLITE_WIN32_MAX_ERRMSG_CHARS+1]; - dwLen = osFormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - lastErrno, - 0, - zTempWide, - SQLITE_WIN32_MAX_ERRMSG_CHARS, - 0); -#else LPWSTR zTempWide = NULL; dwLen = osFormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | @@ -2142,16 +1984,13 @@ static int winGetLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ (LPWSTR) &zTempWide, 0, 0); -#endif if( dwLen > 0 ){ /* allocate a buffer and convert to UTF8 */ sqlite3BeginBenignMalloc(); zOut = winUnicodeToUtf8(zTempWide); sqlite3EndBenignMalloc(); -#if !SQLITE_OS_WINRT /* free the system buffer allocated by FormatMessage */ osLocalFree(zTempWide); -#endif } } #ifdef SQLITE_WIN32_HAS_ANSI @@ -2812,7 +2651,6 @@ static int winHandleUnlock(HANDLE h, int iOff, int nByte){ static int winHandleSeek(HANDLE h, sqlite3_int64 iOffset){ int rc = SQLITE_OK; /* Return value */ -#if !SQLITE_OS_WINRT LONG upperBits; /* Most sig. 32 bits of new offset */ LONG lowerBits; /* Least sig. 32 bits of new offset */ DWORD dwRet; /* Value returned by SetFilePointer() */ @@ -2834,20 +2672,7 @@ static int winHandleSeek(HANDLE h, sqlite3_int64 iOffset){ rc = SQLITE_IOERR_SEEK; } } -#else - /* This implementation works for WinRT. */ - LARGE_INTEGER x; /* The new offset */ - BOOL bRet; /* Value returned by SetFilePointerEx() */ - - x.QuadPart = iOffset; - bRet = osSetFilePointerEx(h, x, 0, FILE_BEGIN); - - if(!bRet){ - rc = SQLITE_IOERR_SEEK; - } -#endif - - OSTRACE(("SEEK file=%p, offset=%lld rc=%s\n", h, iOffset, sqlite3ErrName(rc))); + OSTRACE(("SEEK file=%p, offset=%lld rc=%s\n", h, iOffset,sqlite3ErrName(rc))); return rc; } @@ -3148,17 +2973,6 @@ static int winHandleTruncate(HANDLE h, sqlite3_int64 nByte){ */ static int winHandleSize(HANDLE h, sqlite3_int64 *pnByte){ int rc = SQLITE_OK; - -#if SQLITE_OS_WINRT - FILE_STANDARD_INFO info; - BOOL b; - b = osGetFileInformationByHandleEx(h, FileStandardInfo, &info, sizeof(info)); - if( b ){ - *pnByte = info.EndOfFile.QuadPart; - }else{ - rc = SQLITE_IOERR_FSTAT; - } -#else DWORD upperBits = 0; DWORD lowerBits = 0; @@ -3168,8 +2982,6 @@ static int winHandleSize(HANDLE h, sqlite3_int64 *pnByte){ if( lowerBits==INVALID_FILE_SIZE && osGetLastError()!=NO_ERROR ){ rc = SQLITE_IOERR_FSTAT; } -#endif - return rc; } @@ -3368,20 +3180,6 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ assert( pSize!=0 ); SimulateIOError(return SQLITE_IOERR_FSTAT); OSTRACE(("SIZE file=%p, pSize=%p\n", pFile->h, pSize)); - -#if SQLITE_OS_WINRT - { - FILE_STANDARD_INFO info; - if( osGetFileInformationByHandleEx(pFile->h, FileStandardInfo, - &info, sizeof(info)) ){ - *pSize = info.EndOfFile.QuadPart; - }else{ - pFile->lastErrno = osGetLastError(); - rc = winLogError(SQLITE_IOERR_FSTAT, pFile->lastErrno, - "winFileSize", pFile->zPath); - } - } -#else { DWORD upperBits; DWORD lowerBits; @@ -3396,7 +3194,6 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ "winFileSize", pFile->zPath); } } -#endif OSTRACE(("SIZE file=%p, pSize=%p, *pSize=%lld, rc=%s\n", pFile->h, pSize, *pSize, sqlite3ErrName(rc))); return rc; @@ -4358,20 +4155,6 @@ static int winHandleOpen( ** TODO: retry-on-ioerr. */ if( osIsNT() ){ -#if SQLITE_OS_WINRT - CREATEFILE2_EXTENDED_PARAMETERS extendedParameters; - memset(&extendedParameters, 0, sizeof(extendedParameters)); - extendedParameters.dwSize = sizeof(extendedParameters); - extendedParameters.dwFileAttributes = FILE_ATTRIBUTE_NORMAL; - extendedParameters.dwFileFlags = flag_overlapped; - extendedParameters.dwSecurityQosFlags = SECURITY_ANONYMOUS; - h = osCreateFile2((LPCWSTR)zConverted, - (GENERIC_READ | (bReadonly ? 0 : GENERIC_WRITE)),/* dwDesiredAccess */ - FILE_SHARE_READ | FILE_SHARE_WRITE, /* dwShareMode */ - OPEN_ALWAYS, /* dwCreationDisposition */ - &extendedParameters - ); -#else h = osCreateFileW((LPCWSTR)zConverted, /* lpFileName */ (GENERIC_READ | (bReadonly ? 0 : GENERIC_WRITE)), /* dwDesiredAccess */ FILE_SHARE_READ | FILE_SHARE_WRITE, /* dwShareMode */ @@ -4380,7 +4163,6 @@ static int winHandleOpen( FILE_ATTRIBUTE_NORMAL|flag_overlapped, NULL ); -#endif }else{ /* Due to pre-processor directives earlier in this file, ** SQLITE_WIN32_HAS_ANSI is always defined if osIsNT() is false. */ @@ -4848,9 +4630,7 @@ static int winShmMap( HANDLE hMap = NULL; /* file-mapping handle */ void *pMap = 0; /* Mapped memory region */ -#if SQLITE_OS_WINRT - hMap = osCreateFileMappingFromApp(hShared, NULL, protect, nByte, NULL); -#elif defined(SQLITE_WIN32_HAS_WIDE) +#if defined(SQLITE_WIN32_HAS_WIDE) hMap = osCreateFileMappingW(hShared, NULL, protect, 0, nByte, NULL); #elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA hMap = osCreateFileMappingA(hShared, NULL, protect, 0, nByte, NULL); @@ -4862,15 +4642,9 @@ static int winShmMap( if( hMap ){ int iOffset = pShmNode->nRegion*szRegion; int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; -#if SQLITE_OS_WINRT - pMap = osMapViewOfFileFromApp(hMap, flags, - iOffset - iOffsetShift, szRegion + iOffsetShift - ); -#else pMap = osMapViewOfFile(hMap, flags, 0, iOffset - iOffsetShift, szRegion + iOffsetShift ); -#endif OSTRACE(("SHM-MAP-MAP pid=%lu, region=%d, offset=%d, size=%d, rc=%s\n", osGetCurrentProcessId(), pShmNode->nRegion, iOffset, szRegion, pMap ? "ok" : "failed")); @@ -5003,9 +4777,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ flags |= FILE_MAP_WRITE; } #endif -#if SQLITE_OS_WINRT - pFd->hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); -#elif defined(SQLITE_WIN32_HAS_WIDE) +#if defined(SQLITE_WIN32_HAS_WIDE) pFd->hMap = osCreateFileMappingW(pFd->h, NULL, protect, (DWORD)((nMap>>32) & 0xffffffff), (DWORD)(nMap & 0xffffffff), NULL); @@ -5025,11 +4797,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ } assert( (nMap % winSysInfo.dwPageSize)==0 ); assert( sizeof(SIZE_T)==sizeof(sqlite3_int64) || nMap<=0xffffffff ); -#if SQLITE_OS_WINRT - pNew = osMapViewOfFileFromApp(pFd->hMap, flags, 0, (SIZE_T)nMap); -#else pNew = osMapViewOfFile(pFd->hMap, flags, 0, 0, (SIZE_T)nMap); -#endif if( pNew==NULL ){ osCloseHandle(pFd->hMap); pFd->hMap = NULL; @@ -5364,7 +5132,6 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ } #endif -#if !SQLITE_OS_WINRT && defined(_WIN32) else if( osIsNT() ){ char *zMulti; LPWSTR zWidePath = sqlite3MallocZero( nMax*sizeof(WCHAR) ); @@ -5418,7 +5185,6 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ } } #endif /* SQLITE_WIN32_HAS_ANSI */ -#endif /* !SQLITE_OS_WINRT */ /* ** Check to make sure the temporary directory ends with an appropriate @@ -5593,13 +5359,6 @@ static int winOpen( memset(pFile, 0, sizeof(winFile)); pFile->h = INVALID_HANDLE_VALUE; -#if SQLITE_OS_WINRT - if( !zUtf8Name && !sqlite3_temp_directory ){ - sqlite3_log(SQLITE_ERROR, - "sqlite3_temp_directory variable should be set for WinRT"); - } -#endif - /* If the second argument to this function is NULL, generate a ** temporary file name to use */ @@ -5682,31 +5441,6 @@ static int winOpen( #endif if( osIsNT() ){ -#if SQLITE_OS_WINRT - CREATEFILE2_EXTENDED_PARAMETERS extendedParameters; - extendedParameters.dwSize = sizeof(CREATEFILE2_EXTENDED_PARAMETERS); - extendedParameters.dwFileAttributes = - dwFlagsAndAttributes & FILE_ATTRIBUTE_MASK; - extendedParameters.dwFileFlags = dwFlagsAndAttributes & FILE_FLAG_MASK; - extendedParameters.dwSecurityQosFlags = SECURITY_ANONYMOUS; - extendedParameters.lpSecurityAttributes = NULL; - extendedParameters.hTemplateFile = NULL; - do{ - h = osCreateFile2((LPCWSTR)zConverted, - dwDesiredAccess, - dwShareMode, - dwCreationDisposition, - &extendedParameters); - if( h!=INVALID_HANDLE_VALUE ) break; - if( isReadWrite ){ - int rc2; - sqlite3BeginBenignMalloc(); - rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ|NORETRY, &isRO); - sqlite3EndBenignMalloc(); - if( rc2==SQLITE_OK && isRO ) break; - } - }while( winRetryIoerr(&cnt, &lastErrno) ); -#else do{ h = osCreateFileW((LPCWSTR)zConverted, dwDesiredAccess, @@ -5723,7 +5457,6 @@ static int winOpen( if( rc2==SQLITE_OK && isRO ) break; } }while( winRetryIoerr(&cnt, &lastErrno) ); -#endif } #ifdef SQLITE_WIN32_HAS_ANSI else{ @@ -5860,25 +5593,7 @@ static int winDelete( } if( osIsNT() ){ do { -#if SQLITE_OS_WINRT - WIN32_FILE_ATTRIBUTE_DATA sAttrData; - memset(&sAttrData, 0, sizeof(sAttrData)); - if ( osGetFileAttributesExW(zConverted, GetFileExInfoStandard, - &sAttrData) ){ - attr = sAttrData.dwFileAttributes; - }else{ - lastErrno = osGetLastError(); - if( lastErrno==ERROR_FILE_NOT_FOUND - || lastErrno==ERROR_PATH_NOT_FOUND ){ - rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ - }else{ - rc = SQLITE_ERROR; - } - break; - } -#else attr = osGetFileAttributesW(zConverted); -#endif if ( attr==INVALID_FILE_ATTRIBUTES ){ lastErrno = osGetLastError(); if( lastErrno==ERROR_FILE_NOT_FOUND @@ -6001,6 +5716,7 @@ static int winAccess( attr = sAttrData.dwFileAttributes; } }else{ + if( noRetry ) lastErrno = osGetLastError(); winLogIoerr(cnt, __LINE__); if( lastErrno!=ERROR_FILE_NOT_FOUND && lastErrno!=ERROR_PATH_NOT_FOUND ){ sqlite3_free(zConverted); @@ -6169,7 +5885,7 @@ static int winFullPathnameNoMutex( int nFull, /* Size of output buffer in bytes */ char *zFull /* Output buffer */ ){ -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE int nByte; void *zConverted; char *zOut; @@ -6258,7 +5974,7 @@ static int winFullPathnameNoMutex( } #endif /* __CYGWIN__ */ -#if (SQLITE_OS_WINCE || SQLITE_OS_WINRT) && defined(_WIN32) +#if SQLITE_OS_WINCE && defined(_WIN32) SimulateIOError( return SQLITE_ERROR ); /* WinCE has no concept of a relative pathname, or so I am told. */ /* WinRT has no way to convert a relative path to an absolute one. */ @@ -6277,7 +5993,7 @@ static int winFullPathnameNoMutex( return SQLITE_OK; #endif -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE #if defined(_WIN32) /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this @@ -6409,11 +6125,7 @@ static void *winDlOpen(sqlite3_vfs *pVfs, const char *zFilename){ return 0; } if( osIsNT() ){ -#if SQLITE_OS_WINRT - h = osLoadPackagedLibrary((LPCWSTR)zConverted, 0); -#else h = osLoadLibraryW((LPCWSTR)zConverted); -#endif } #ifdef SQLITE_WIN32_HAS_ANSI else{ @@ -6495,23 +6207,16 @@ static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ DWORD pid = osGetCurrentProcessId(); xorMemory(&e, (unsigned char*)&pid, sizeof(DWORD)); } -#if SQLITE_OS_WINRT - { - ULONGLONG cnt = osGetTickCount64(); - xorMemory(&e, (unsigned char*)&cnt, sizeof(ULONGLONG)); - } -#else { DWORD cnt = osGetTickCount(); xorMemory(&e, (unsigned char*)&cnt, sizeof(DWORD)); } -#endif /* SQLITE_OS_WINRT */ { LARGE_INTEGER i; osQueryPerformanceCounter(&i); xorMemory(&e, (unsigned char*)&i, sizeof(LARGE_INTEGER)); } -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID +#if !SQLITE_OS_WINCE && SQLITE_WIN32_USE_UUID { UUID id; memset(&id, 0, sizeof(UUID)); @@ -6521,7 +6226,7 @@ static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ osUuidCreateSequential(&id); xorMemory(&e, (unsigned char*)&id, sizeof(UUID)); } -#endif /* !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID */ +#endif /* !SQLITE_OS_WINCE && SQLITE_WIN32_USE_UUID */ return e.nXor>nBuf ? nBuf : e.nXor; #endif /* defined(SQLITE_TEST) || defined(SQLITE_OMIT_RANDOMNESS) */ } @@ -6752,15 +6457,16 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==89 ); + assert( ArraySize(aSyscall)==81 ); + assert( strcmp(aSyscall[0].zName,"AreFileApisANSI")==0 ); + assert( strcmp(aSyscall[20].zName,"GetFileAttributesA")==0 ); + assert( strcmp(aSyscall[40].zName,"HeapReAlloc")==0 ); + assert( strcmp(aSyscall[60].zName,"WideCharToMultiByte")==0 ); + assert( strcmp(aSyscall[80].zName,"cygwin_conv_path")==0 ); /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); -#if SQLITE_OS_WINRT - osGetNativeSystemInfo(&winSysInfo); -#else osGetSystemInfo(&winSysInfo); -#endif assert( winSysInfo.dwAllocationGranularity>0 ); assert( winSysInfo.dwPageSize>0 ); @@ -6784,17 +6490,9 @@ int sqlite3_os_init(void){ } int sqlite3_os_end(void){ -#if SQLITE_OS_WINRT - if( sleepObj!=NULL ){ - osCloseHandle(sleepObj); - sleepObj = NULL; - } -#endif - #ifndef SQLITE_OMIT_WAL winBigLock = 0; #endif - return SQLITE_OK; } diff --git a/src/os_win.h b/src/os_win.h index a0845f003..696486c19 100644 --- a/src/os_win.h +++ b/src/os_win.h @@ -58,14 +58,6 @@ # define SQLITE_OS_WINCE 0 #endif -/* -** Determine if we are dealing with WinRT, which provides only a subset of -** the full Win32 API. -*/ -#if !defined(SQLITE_OS_WINRT) -# define SQLITE_OS_WINRT 0 -#endif - /* ** For WinCE, some API function parameters do not appear to be declared as ** volatile. @@ -80,7 +72,7 @@ ** For some Windows sub-platforms, the _beginthreadex() / _endthreadex() ** functions are not available (e.g. those not using MSVC, Cygwin, etc). */ -#if SQLITE_OS_WIN && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && \ +#if SQLITE_OS_WIN && !SQLITE_OS_WINCE && \ SQLITE_THREADSAFE>0 && !defined(__CYGWIN__) # define SQLITE_OS_WIN_THREADS 1 #else diff --git a/src/pager.c b/src/pager.c index cbaef186a..61b391d6b 100644 --- a/src/pager.c +++ b/src/pager.c @@ -813,6 +813,8 @@ int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){ (void)sqlite3WalFindFrame(pPager->pWal, pgno, &iRead); if( iRead ) return 0; /* Case (4) */ } +#else + UNUSED_PARAMETER(pgno); #endif assert( pPager->fd->pMethods->xDeviceCharacteristics!=0 ); if( (pPager->fd->pMethods->xDeviceCharacteristics(pPager->fd) @@ -1233,17 +1235,17 @@ static int jrnlBufferSize(Pager *pPager){ */ #ifdef SQLITE_CHECK_PAGES /* -** Return a 32-bit hash of the page data for pPage. +** Return a 64-bit hash of the page data for pPage. */ -static u32 pager_datahash(int nByte, unsigned char *pData){ - u32 hash = 0; +static u64 pager_datahash(int nByte, unsigned char *pData){ + u64 hash = 0; int i; for(i=0; i<nByte; i++){ hash = (hash*1039) + pData[i]; } return hash; } -static u32 pager_pagehash(PgHdr *pPage){ +static u64 pager_pagehash(PgHdr *pPage){ return pager_datahash(pPage->pPager->pageSize, (unsigned char *)pPage->pData); } static void pager_set_pagehash(PgHdr *pPage){ @@ -4192,6 +4194,8 @@ int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize,a); pPager->pWal = 0; } +#else + UNUSED_PARAMETER(db); #endif pager_reset(pPager); if( MEMDB ){ diff --git a/src/parse.y b/src/parse.y index 617eb7303..f5a6bed14 100644 --- a/src/parse.y +++ b/src/parse.y @@ -21,9 +21,11 @@ */ } -// Function used to enlarge the parser stack, if needed -%realloc parserStackRealloc -%free sqlite3_free +// Setup for the parser stack +%stack_size 50 // Initial stack size +%stack_size_limit parserStackSizeLimit // Function returning max stack size +%realloc parserStackRealloc // realloc() for the stack +%free parserStackFree // free() for the stack // All token codes are small integers with #defines that begin with "TK_" %token_prefix TK_ @@ -49,7 +51,7 @@ } } %stack_overflow { - sqlite3OomFault(pParse->db); + if( pParse->nErr==0 ) sqlite3ErrorMsg(pParse, "Recursion limit"); } // The name of the generated procedure that implements the parser @@ -583,8 +585,23 @@ cmd ::= select(X). { ** sqlite3_realloc() that includes a call to sqlite3FaultSim() to facilitate ** testing. */ - static void *parserStackRealloc(void *pOld, sqlite3_uint64 newSize){ - return sqlite3FaultSim(700) ? 0 : sqlite3_realloc(pOld, newSize); + static void *parserStackRealloc( + void *pOld, /* Prior allocation */ + sqlite3_uint64 newSize, /* Requested new alloation size */ + Parse *pParse /* Parsing context */ + ){ + void *p = sqlite3FaultSim(700) ? 0 : sqlite3_realloc(pOld, newSize); + if( p==0 ) sqlite3OomFault(pParse->db); + return p; + } + static void parserStackFree(void *pOld, Parse *pParse){ + (void)pParse; + sqlite3_free(pOld); + } + + /* Return an integer that is the maximum allowed stack size */ + static int parserStackSizeLimit(Parse *pParse){ + return pParse->db->aLimit[SQLITE_LIMIT_PARSER_DEPTH]; } } @@ -816,19 +833,36 @@ fullname(A) ::= nm(X) DOT nm(Y). { %type xfullname {SrcList*} %destructor xfullname {sqlite3SrcListDelete(pParse->db, $$);} -xfullname(A) ::= nm(X). - {A = sqlite3SrcListAppend(pParse,0,&X,0); /*A-overwrites-X*/} -xfullname(A) ::= nm(X) DOT nm(Y). - {A = sqlite3SrcListAppend(pParse,0,&X,&Y); /*A-overwrites-X*/} -xfullname(A) ::= nm(X) DOT nm(Y) AS nm(Z). { - A = sqlite3SrcListAppend(pParse,0,&X,&Y); /*A-overwrites-X*/ - if( A ) A->a[0].zAlias = sqlite3NameFromToken(pParse->db, &Z); +xfullname(A) ::= nm(X). { + A = sqlite3SrcListAppend(pParse,0,&X,0); + if( IN_RENAME_OBJECT && A ) sqlite3RenameTokenMap(pParse, A->a[0].zName, &X); } -xfullname(A) ::= nm(X) AS nm(Z). { - A = sqlite3SrcListAppend(pParse,0,&X,0); /*A-overwrites-X*/ - if( A ) A->a[0].zAlias = sqlite3NameFromToken(pParse->db, &Z); +xfullname(A) ::= nm(X) DOT nm(Y). { + A = sqlite3SrcListAppend(pParse,0,&X,&Y); + if( IN_RENAME_OBJECT && A ) sqlite3RenameTokenMap(pParse, A->a[0].zName, &Y); +} +xfullname(A) ::= nm(X) AS nm(Z). { + A = sqlite3SrcListAppend(pParse,0,&X,0); + if( A ){ + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, A->a[0].zName, &X); + }else{ + A->a[0].zAlias = sqlite3NameFromToken(pParse->db, &Z); + } + } +} +xfullname(A) ::= nm(X) DOT nm(Y) AS nm(Z). { + A = sqlite3SrcListAppend(pParse,0,&X,&Y); + if( A ){ + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, A->a[0].zName, &Y); + }else{ + A->a[0].zAlias = sqlite3NameFromToken(pParse->db, &Z); + } + } } + %type joinop {int} joinop(X) ::= COMMA|JOIN. { X = JT_INNER; } joinop(X) ::= JOIN_KW(A) JOIN. @@ -1159,7 +1193,12 @@ expr(A) ::= nm(X) DOT nm(Y) DOT nm(Z). { term(A) ::= NULL|FLOAT|BLOB(X). {A=tokenExpr(pParse,@X,X); /*A-overwrites-X*/} term(A) ::= STRING(X). {A=tokenExpr(pParse,@X,X); /*A-overwrites-X*/} term(A) ::= INTEGER(X). { - A = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &X, 1); + int iValue; + if( sqlite3GetInt32(X.z, &iValue)==0 ){ + A = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &X, 0); + }else{ + A = sqlite3ExprInt32(pParse->db, iValue); + } if( A ) A->w.iOfst = (int)(X.z - pParse->zTail); } expr(A) ::= VARIABLE(X). { @@ -1341,43 +1380,67 @@ expr(A) ::= expr(A) likeop(OP) expr(Y) ESCAPE expr(E). [LIKE_KW] { if( A ) A->flags |= EP_InfixFunc; } -expr(A) ::= expr(A) ISNULL|NOTNULL(E). {A = sqlite3PExpr(pParse,@E,A,0);} -expr(A) ::= expr(A) NOT NULL. {A = sqlite3PExpr(pParse,TK_NOTNULL,A,0);} - %include { - /* A routine to convert a binary TK_IS or TK_ISNOT expression into a - ** unary TK_ISNULL or TK_NOTNULL expression. */ - static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){ - sqlite3 *db = pParse->db; - if( pA && pY && pY->op==TK_NULL && !IN_RENAME_OBJECT ){ - pA->op = (u8)op; - sqlite3ExprDelete(db, pA->pRight); - pA->pRight = 0; + /* Create a TK_ISNULL or TK_NOTNULL expression, perhaps optimized to + ** to TK_TRUEFALSE, if possible */ + static Expr *sqlite3PExprIsNull( + Parse *pParse, /* Parsing context */ + int op, /* TK_ISNULL or TK_NOTNULL */ + Expr *pLeft /* Operand */ + ){ + Expr *p = pLeft; + assert( op==TK_ISNULL || op==TK_NOTNULL ); + assert( pLeft!=0 ); + while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ + p = p->pLeft; + assert( p!=0 ); + } + switch( p->op ){ + case TK_INTEGER: + case TK_STRING: + case TK_FLOAT: + case TK_BLOB: + sqlite3ExprDeferredDelete(pParse, pLeft); + return sqlite3ExprInt32(pParse->db, op==TK_NOTNULL); + default: + break; } + return sqlite3PExpr(pParse, op, pLeft, 0); + } + + /* Create a TK_IS or TK_ISNOT operator, perhaps optimized to + ** TK_ISNULL or TK_NOTNULL or TK_TRUEFALSE. */ + static Expr *sqlite3PExprIs( + Parse *pParse, /* Parsing context */ + int op, /* TK_IS or TK_ISNOT */ + Expr *pLeft, /* Left operand */ + Expr *pRight /* Right operand */ + ){ + if( pRight && pRight->op==TK_NULL ){ + sqlite3ExprDeferredDelete(pParse, pRight); + return sqlite3PExprIsNull(pParse, op==TK_IS ? TK_ISNULL : TK_NOTNULL, pLeft); + } + return sqlite3PExpr(pParse, op, pLeft, pRight); } } -// expr1 IS expr2 -// expr1 IS NOT expr2 +expr(A) ::= expr(A) ISNULL|NOTNULL(E). {A = sqlite3PExprIsNull(pParse,@E,A);} +expr(A) ::= expr(A) NOT NULL. {A = sqlite3PExprIsNull(pParse,TK_NOTNULL,A);} + +// expr1 IS expr2 same as expr1 IS NOT DISTINCT FROM expr2 +// expr1 IS NOT expr2 same as expr1 IS DISTINCT FROM expr2 // -// If expr2 is NULL then code as TK_ISNULL or TK_NOTNULL. If expr2 -// is any other expression, code as TK_IS or TK_ISNOT. -// expr(A) ::= expr(A) IS expr(Y). { - A = sqlite3PExpr(pParse,TK_IS,A,Y); - binaryToUnaryIfNull(pParse, Y, A, TK_ISNULL); + A = sqlite3PExprIs(pParse, TK_IS, A, Y); } expr(A) ::= expr(A) IS NOT expr(Y). { - A = sqlite3PExpr(pParse,TK_ISNOT,A,Y); - binaryToUnaryIfNull(pParse, Y, A, TK_NOTNULL); + A = sqlite3PExprIs(pParse, TK_ISNOT, A, Y); } expr(A) ::= expr(A) IS NOT DISTINCT FROM expr(Y). { - A = sqlite3PExpr(pParse,TK_IS,A,Y); - binaryToUnaryIfNull(pParse, Y, A, TK_ISNULL); + A = sqlite3PExprIs(pParse, TK_IS, A, Y); } expr(A) ::= expr(A) IS DISTINCT FROM expr(Y). { - A = sqlite3PExpr(pParse,TK_ISNOT,A,Y); - binaryToUnaryIfNull(pParse, Y, A, TK_NOTNULL); + A = sqlite3PExprIs(pParse, TK_ISNOT, A, Y); } expr(A) ::= NOT(B) expr(X). @@ -1707,28 +1770,13 @@ when_clause(A) ::= WHEN expr(X). { A = X; } %type trigger_cmd_list {TriggerStep*} %destructor trigger_cmd_list {sqlite3DeleteTriggerStep(pParse->db, $$);} trigger_cmd_list(A) ::= trigger_cmd_list(A) trigger_cmd(X) SEMI. { - assert( A!=0 ); A->pLast->pNext = X; A->pLast = X; } trigger_cmd_list(A) ::= trigger_cmd(A) SEMI. { - assert( A!=0 ); A->pLast = A; } -// Disallow qualified table names on INSERT, UPDATE, and DELETE statements -// within a trigger. The table to INSERT, UPDATE, or DELETE is always in -// the same database as the table that the trigger fires on. -// -%type trnm {Token} -trnm(A) ::= nm(A). -trnm(A) ::= nm DOT nm(X). { - A = X; - sqlite3ErrorMsg(pParse, - "qualified table names are not allowed on INSERT, UPDATE, and DELETE " - "statements within triggers"); -} - // Disallow the INDEX BY and NOT INDEXED clauses on UPDATE and DELETE // statements within triggers. We make a specific error message for this // since it is an exception to the default grammar rules. @@ -1751,17 +1799,17 @@ tridxby ::= NOT INDEXED. { %destructor trigger_cmd {sqlite3DeleteTriggerStep(pParse->db, $$);} // UPDATE trigger_cmd(A) ::= - UPDATE(B) orconf(R) trnm(X) tridxby SET setlist(Y) from(F) where_opt(Z) scanpt(E). - {A = sqlite3TriggerUpdateStep(pParse, &X, F, Y, Z, R, B.z, E);} + UPDATE(B) orconf(R) xfullname(X) tridxby SET setlist(Y) from(F) where_opt(Z) scanpt(E). + {A = sqlite3TriggerUpdateStep(pParse, X, F, Y, Z, R, B.z, E);} // INSERT trigger_cmd(A) ::= scanpt(B) insert_cmd(R) INTO - trnm(X) idlist_opt(F) select(S) upsert(U) scanpt(Z). { - A = sqlite3TriggerInsertStep(pParse,&X,F,S,R,U,B,Z);/*A-overwrites-R*/ + xfullname(X) idlist_opt(F) select(S) upsert(U) scanpt(Z). { + A = sqlite3TriggerInsertStep(pParse,X,F,S,R,U,B,Z);/*A-overwrites-R*/ } // DELETE -trigger_cmd(A) ::= DELETE(B) FROM trnm(X) tridxby where_opt(Y) scanpt(E). - {A = sqlite3TriggerDeleteStep(pParse, &X, Y, B.z, E);} +trigger_cmd(A) ::= DELETE(B) FROM xfullname(X) tridxby where_opt(Y) scanpt(E). + {A = sqlite3TriggerDeleteStep(pParse, X, Y, B.z, E);} // SELECT trigger_cmd(A) ::= scanpt(B) select(X) scanpt(E). @@ -1831,22 +1879,42 @@ cmd ::= ANALYZE nm(X) dbnm(Y). {sqlite3Analyze(pParse, &X, &Y);} cmd ::= ALTER TABLE fullname(X) RENAME TO nm(Z). { sqlite3AlterRenameTable(pParse,X,&Z); } -cmd ::= ALTER TABLE add_column_fullname - ADD kwcolumn_opt columnname(Y) carglist. { + +// The ALTER TABLE ADD COLUMN command. This is broken into two sections so +// that sqlite3AlterBeginAddColumn() is called before parsing the various +// constraints and so on (carglist) attached to the new column definition. +cmd ::= alter_add(Y) carglist. { Y.n = (int)(pParse->sLastToken.z-Y.z) + pParse->sLastToken.n; sqlite3AlterFinishAddColumn(pParse, &Y); } -cmd ::= ALTER TABLE fullname(X) DROP kwcolumn_opt nm(Y). { - sqlite3AlterDropColumn(pParse, X, &Y); -} - -add_column_fullname ::= fullname(X). { +alter_add(A) ::= ALTER TABLE fullname(X) ADD kwcolumn_opt nm(Y) typetoken(Z). { disableLookaside(pParse); sqlite3AlterBeginAddColumn(pParse, X); + sqlite3AddColumn(pParse, Y, Z); + A = Y; +} + +cmd ::= ALTER TABLE fullname(X) DROP kwcolumn_opt nm(Y). { + sqlite3AlterDropColumn(pParse, X, &Y); } cmd ::= ALTER TABLE fullname(X) RENAME kwcolumn_opt nm(Y) TO nm(Z). { sqlite3AlterRenameColumn(pParse, X, &Y, &Z); } +cmd ::= ALTER TABLE fullname(X) DROP CONSTRAINT nm(Y). { + sqlite3AlterDropConstraint(pParse, X, &Y, 0); +} +cmd ::= ALTER TABLE fullname(X) ALTER kwcolumn_opt nm(Y) DROP NOT NULL. { + sqlite3AlterDropConstraint(pParse, X, 0, &Y); +} +cmd ::= ALTER TABLE fullname(X) ALTER kwcolumn_opt nm(Y) SET NOT(Z) NULL onconf. { + sqlite3AlterSetNotNull(pParse, X, &Y, &Z); +} +cmd ::= ALTER TABLE fullname(X) ADD CONSTRAINT(Y) nm(Z) CHECK LP(A) expr RP(B) onconf. { + sqlite3AlterAddConstraint(pParse, X, &Y, &Z, A.z+1, (B.z-A.z-1)); +} +cmd ::= ALTER TABLE fullname(X) ADD CHECK(Y) LP(A) expr RP(B) onconf. { + sqlite3AlterAddConstraint(pParse, X, &Y, 0, A.z+1, (B.z-A.z-1)); +} kwcolumn_opt ::= . kwcolumn_opt ::= COLUMNKW. diff --git a/src/pcache.h b/src/pcache.h index f945dab1a..dafb59390 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -29,10 +29,10 @@ struct PgHdr { PCache *pCache; /* PRIVATE: Cache that owns this page */ PgHdr *pDirty; /* Transient list of dirty sorted by pgno */ Pager *pPager; /* The pager this page is part of */ - Pgno pgno; /* Page number for this page */ #ifdef SQLITE_CHECK_PAGES - u32 pageHash; /* Hash of page content */ + u64 pageHash; /* Hash of page content */ #endif + Pgno pgno; /* Page number for this page */ u16 flags; /* PGHDR flags defined below */ /********************************************************************** diff --git a/src/pragma.c b/src/pragma.c index 791508ea4..961ba0b0e 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2067,8 +2067,20 @@ void sqlite3Pragma( pPrior = pIdx; sqlite3VdbeAddOp2(v, OP_AddImm, 8+j, 1);/* increment entry count */ /* Verify that an index entry exists for the current table row */ - jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, + sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, pIdx->nColumn); VdbeCoverage(v); + jmp2 = sqlite3VdbeAddOp3(v, OP_IFindKey, iIdxCur+j, ckUniq, r1); + VdbeCoverage(v); + sqlite3VdbeChangeP4(v, -1, (const char*)pIdx, P4_INDEX); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, + sqlite3MPrintf(db, "index %s stores an imprecise floating-point " + "value for row ", pIdx->zName), + P4_DYNAMIC); + sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); + integrityCheckResultRow(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, ckUniq); + + sqlite3VdbeJumpHere(v, jmp2); sqlite3VdbeLoadString(v, 3, "row "); sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); sqlite3VdbeLoadString(v, 4, " missing from index "); @@ -2076,7 +2088,7 @@ void sqlite3Pragma( jmp5 = sqlite3VdbeLoadString(v, 4, pIdx->zName); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); jmp4 = integrityCheckResultRow(v); - sqlite3VdbeJumpHere(v, jmp2); + sqlite3VdbeResolveLabel(v, ckUniq); /* The OP_IdxRowid opcode is an optimized version of OP_Column ** that extracts the rowid off the end of the index record. diff --git a/src/prepare.c b/src/prepare.c index 539360b74..be9e496f1 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -33,7 +33,8 @@ static void corruptSchema( static const char *azAlterType[] = { "rename", "drop column", - "add column" + "add column", + "drop constraint" }; *pData->pzErrMsg = sqlite3MPrintf(db, "error in %s %s after %s: %s", azObj[0], azObj[1], diff --git a/src/printf.c b/src/printf.c index f75ed3b8a..c9fc1a72c 100644 --- a/src/printf.c +++ b/src/printf.c @@ -496,9 +496,11 @@ void sqlite3_str_vappendf( }while( longvalue>0 ); } length = (int)(&zOut[nOut-1]-bufpt); - while( precision>length ){ - *(--bufpt) = '0'; /* Zero pad */ - length++; + if( precision>length ){ /* zero pad */ + int nn = precision-length; + bufpt -= nn; + memset(bufpt,'0',nn); + length = precision; } if( cThousand ){ int nn = (length - 1)/3; /* Number of "," to insert */ @@ -529,6 +531,7 @@ void sqlite3_str_vappendf( FpDecode s; int iRound; int j; + i64 szBufNeeded; /* Size needed to hold the output */ if( bArgList ){ realvalue = getDoubleArg(pArgList); @@ -549,7 +552,7 @@ void sqlite3_str_vappendf( }else{ iRound = precision+1; } - sqlite3FpDecode(&s, realvalue, iRound, flag_altform2 ? 26 : 16); + sqlite3FpDecode(&s, realvalue, iRound, flag_altform2 ? 20 : 16); if( s.isSpecial ){ if( s.isSpecial==2 ){ bufpt = flag_zeropad ? "null" : "NaN"; @@ -617,17 +620,15 @@ void sqlite3_str_vappendf( }else{ e2 = s.iDP - 1; } - bufpt = buf; - { - i64 szBufNeeded; /* Size of a temporary buffer needed */ - szBufNeeded = MAX(e2,0)+(i64)precision+(i64)width+15; - if( cThousand && e2>0 ) szBufNeeded += (e2+2)/3; - if( szBufNeeded > etBUFSIZE ){ - bufpt = zExtra = printfTempBuf(pAccum, szBufNeeded); - if( bufpt==0 ) return; - } + + szBufNeeded = MAX(e2,0)+(i64)precision+(i64)width+10; + if( cThousand && e2>0 ) szBufNeeded += (e2+2)/3; + if( sqlite3StrAccumEnlargeIfNeeded(pAccum, szBufNeeded) ){ + width = length = 0; + break; } - zOut = bufpt; + bufpt = zOut = pAccum->zText + pAccum->nChar; + flag_dp = (precision>0 ?1:0) | flag_alternateform | flag_altform2; /* The sign in front of the number */ if( prefix ){ @@ -635,12 +636,24 @@ void sqlite3_str_vappendf( } /* Digits prior to the decimal point */ j = 0; + assert( s.n>0 ); if( e2<0 ){ *(bufpt++) = '0'; - }else{ + }else if( cThousand ){ for(; e2>=0; e2--){ *(bufpt++) = j<s.n ? s.z[j++] : '0'; - if( cThousand && (e2%3)==0 && e2>1 ) *(bufpt++) = ','; + if( (e2%3)==0 && e2>1 ) *(bufpt++) = ','; + } + }else{ + j = e2+1; + if( j>s.n ) j = s.n; + memcpy(bufpt, s.z, j); + bufpt += j; + e2 -= j; + if( e2>=0 ){ + memset(bufpt, '0', e2+1); + bufpt += e2+1; + e2 = -1; } } /* The decimal point */ @@ -649,12 +662,26 @@ void sqlite3_str_vappendf( } /* "0" digits after the decimal point but before the first ** significant digit of the number */ - for(e2++; e2<0 && precision>0; precision--, e2++){ - *(bufpt++) = '0'; + if( e2<(-1) && precision>0 ){ + int nn = -1-e2; + if( nn>precision ) nn = precision; + memset(bufpt, '0', nn); + bufpt += nn; + precision -= nn; } /* Significant digits after the decimal point */ - while( (precision--)>0 ){ - *(bufpt++) = j<s.n ? s.z[j++] : '0'; + if( precision>0 ){ + int nn = s.n - j; + if( NEVER(nn>precision) ) nn = precision; + if( nn>0 ){ + memcpy(bufpt, s.z+j, nn); + bufpt += nn; + precision -= nn; + } + if( precision>0 && !flag_rtz ){ + memset(bufpt, '0', precision); + bufpt += precision; + } } /* Remove trailing zeros and the "." if no digits follow the "." */ if( flag_rtz && flag_dp ){ @@ -684,27 +711,31 @@ void sqlite3_str_vappendf( *(bufpt++) = (char)(exp/10+'0'); /* 10's digit */ *(bufpt++) = (char)(exp%10+'0'); /* 1's digit */ } - *bufpt = 0; - /* The converted number is in buf[] and zero terminated. Output it. - ** Note that the number is in the usual order, not reversed as with - ** integer conversions. */ length = (int)(bufpt-zOut); - bufpt = zOut; - - /* Special case: Add leading zeros if the flag_zeropad flag is - ** set and we are not left justified */ - if( flag_zeropad && !flag_leftjustify && length < width){ - int i; - int nPad = width - length; - for(i=width; i>=nPad; i--){ - bufpt[i] = bufpt[i-nPad]; + assert( length <= szBufNeeded ); + if( length<width ){ + i64 nPad = width - length; + if( flag_leftjustify ){ + memset(bufpt, ' ', nPad); + }else if( !flag_zeropad ){ + memmove(zOut+nPad, zOut, length); + memset(zOut, ' ', nPad); + }else{ + int adj = prefix!=0; + memmove(zOut+nPad+adj, zOut+adj, length-adj); + memset(zOut+adj, '0', nPad); } - i = prefix!=0; - while( nPad-- ) bufpt[i++] = '0'; length = width; } - break; + pAccum->nChar += length; + zOut[length] = 0; + + /* Floating point conversions render directly into the output + ** buffer. Hence, don't just break out of the switch(). Bypass the + ** output buffer writing that occurs after the switch() by continuing + ** to the next character in the format string. */ + continue; } case etSIZE: if( !bArgList ){ @@ -748,11 +779,10 @@ void sqlite3_str_vappendf( i64 nCopyBytes; if( nPrior > precision-1 ) nPrior = precision - 1; nCopyBytes = length*nPrior; - if( nCopyBytes + pAccum->nChar >= pAccum->nAlloc ){ - sqlite3StrAccumEnlarge(pAccum, nCopyBytes); + if( sqlite3StrAccumEnlargeIfNeeded(pAccum, nCopyBytes) ){ + break; } - if( pAccum->accError ) break; - sqlite3_str_append(pAccum, + sqlite3_str_append(pAccum, &pAccum->zText[pAccum->nChar-nCopyBytes], nCopyBytes); precision -= nPrior; nPrior *= 2; @@ -1098,6 +1128,13 @@ int sqlite3StrAccumEnlarge(StrAccum *p, i64 N){ return (int)N; } +int sqlite3StrAccumEnlargeIfNeeded(StrAccum *p, i64 N){ + if( N + p->nChar >= p->nAlloc ){ + sqlite3StrAccumEnlarge(p, N); + } + return p->accError; +} + /* ** Append N copies of character c to the given string buffer. */ @@ -1228,6 +1265,14 @@ int sqlite3_str_length(sqlite3_str *p){ return p ? p->nChar : 0; } +/* Truncate the text of the string to be no more than N bytes. */ +void sqlite3_str_truncate(sqlite3_str *p, int N){ + if( p!=0 && N>=0 && (u32)N<p->nChar ){ + p->nChar = N; + p->zText[p->nChar] = 0; + } +} + /* Return the current value for p */ char *sqlite3_str_value(sqlite3_str *p){ if( p==0 || p->nChar==0 ) return 0; @@ -1248,6 +1293,17 @@ void sqlite3_str_reset(StrAccum *p){ p->zText = 0; } +/* +** Destroy a dynamically allocate sqlite3_str object and all +** of its content, all in one call. +*/ +void sqlite3_str_free(sqlite3_str *p){ + if( p ){ + sqlite3_str_reset(p); + sqlite3_free(p); + } +} + /* ** Initialize a string accumulator. ** diff --git a/src/resolve.c b/src/resolve.c index c3ec56136..5b6d6c9c5 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -936,7 +936,7 @@ static int exprProbability(Expr *p){ double r = -1.0; if( p->op!=TK_FLOAT ) return -1; assert( !ExprHasProperty(p, EP_IntValue) ); - sqlite3AtoF(p->u.zToken, &r, sqlite3Strlen30(p->u.zToken), SQLITE_UTF8); + sqlite3AtoF(p->u.zToken, &r); assert( r>=0.0 ); if( r>1.0 ) return -1; return (int)(r*134217728.0); @@ -1656,10 +1656,8 @@ static int resolveCompoundOrderBy( /* Convert the ORDER BY term into an integer column number iCol, ** taking care to preserve the COLLATE clause if it exists. */ if( !IN_RENAME_OBJECT ){ - Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); + Expr *pNew = sqlite3ExprInt32(db, iCol); if( pNew==0 ) return 1; - pNew->flags |= EP_IntValue; - pNew->u.iValue = iCol; if( pItem->pExpr==pE ){ pItem->pExpr = pNew; }else{ @@ -2013,10 +2011,6 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ } #endif - /* The ORDER BY and GROUP BY clauses may not refer to terms in - ** outer queries - */ - sNC.pNext = 0; sNC.ncFlags |= NC_AllowAgg|NC_AllowWin; /* If this is a converted compound query, move the ORDER BY clause from diff --git a/src/select.c b/src/select.c index 3ef7cc0a3..e8e9f36a8 100644 --- a/src/select.c +++ b/src/select.c @@ -21,7 +21,7 @@ */ typedef struct DistinctCtx DistinctCtx; struct DistinctCtx { - u8 isTnct; /* 0: Not distinct. 1: DISTICT 2: DISTINCT and ORDER BY */ + u8 isTnct; /* 0: Not distinct. 1: DISTINCT 2: DISTINCT and ORDER BY */ u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */ int tabTnct; /* Ephemeral table used for DISTINCT processing */ int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */ @@ -151,8 +151,6 @@ Select *sqlite3SelectNew( pNew->iLimit = 0; pNew->iOffset = 0; pNew->selId = ++pParse->nSelect; - pNew->addrOpenEphm[0] = -1; - pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = 0; if( pSrc==0 ) pSrc = sqlite3DbMallocZero(pParse->db, SZ_SRCLIST_1); pNew->pSrc = pSrc; @@ -661,6 +659,10 @@ static int sqlite3ProcessJoin(Parse *pParse, Select *p){ pRight->fg.isOn = 1; p->selFlags |= SF_OnToWhere; } + + if( IsVirtual(pRightTab) && joinType==EP_OuterON && pRight->u1.pFuncArg ){ + p->selFlags |= SF_OnToWhere; + } } return 0; } @@ -1300,29 +1302,6 @@ static void selectInnerLoop( } switch( eDest ){ - /* In this mode, write each query result to the key of the temporary - ** table iParm. - */ -#ifndef SQLITE_OMIT_COMPOUND_SELECT - case SRT_Union: { - int r1; - r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); - sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol); - sqlite3ReleaseTempReg(pParse, r1); - break; - } - - /* Construct a record from the query result, but instead of - ** saving that record, use it as a key to delete elements from - ** the temporary table iParm. - */ - case SRT_Except: { - sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nResultCol); - break; - } -#endif /* SQLITE_OMIT_COMPOUND_SELECT */ - /* Store the result as data using a unique key. */ case SRT_Fifo: @@ -2435,8 +2414,8 @@ void sqlite3SubqueryColumnTypes( } } if( zType ){ - const i64 k = sqlite3Strlen30(zType); - n = sqlite3Strlen30(pCol->zCnName); + const i64 k = strlen(zType); + n = strlen(pCol->zCnName); pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+k+2); pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL); if( pCol->zCnName ){ @@ -2609,9 +2588,9 @@ static CollSeq *multiSelectCollSeq(Parse *pParse, Select *p, int iCol){ ** function is responsible for ensuring that this structure is eventually ** freed. */ -static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){ +static KeyInfo *multiSelectByMergeKeyInfo(Parse *pParse, Select *p, int nExtra){ ExprList *pOrderBy = p->pOrderBy; - int nOrderBy = ALWAYS(pOrderBy!=0) ? pOrderBy->nExpr : 0; + int nOrderBy = (pOrderBy!=0) ? pOrderBy->nExpr : 0; sqlite3 *db = pParse->db; KeyInfo *pRet = sqlite3KeyInfoAlloc(db, nOrderBy+nExtra, 1); if( pRet ){ @@ -2744,7 +2723,7 @@ static void generateWithRecursiveQuery( regCurrent = ++pParse->nMem; sqlite3VdbeAddOp3(v, OP_OpenPseudo, iCurrent, regCurrent, nCol); if( pOrderBy ){ - KeyInfo *pKeyInfo = multiSelectOrderByKeyInfo(pParse, p, 1); + KeyInfo *pKeyInfo = multiSelectByMergeKeyInfo(pParse, p, 1); sqlite3VdbeAddOp4(v, OP_OpenEphemeral, iQueue, pOrderBy->nExpr+2, 0, (char*)pKeyInfo, P4_KEYINFO); destQueue.pOrderBy = pOrderBy; @@ -2753,8 +2732,28 @@ static void generateWithRecursiveQuery( } VdbeComment((v, "Queue table")); if( iDistinct ){ - p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iDistinct, 0); - p->selFlags |= SF_UsesEphemeral; + /* Generate an ephemeral table used to enforce distinctness on the + ** output of the recursive part of the CTE. + */ + KeyInfo *pKeyInfo; /* Collating sequence for the result set */ + CollSeq **apColl; /* For looping through pKeyInfo->aColl[] */ + + assert( p->pNext==0 ); + assert( p->pEList!=0 ); + nCol = p->pEList->nExpr; + pKeyInfo = sqlite3KeyInfoAlloc(pParse->db, nCol, 1); + if( pKeyInfo ){ + for(i=0, apColl=pKeyInfo->aColl; i<nCol; i++, apColl++){ + *apColl = multiSelectCollSeq(pParse, p, i); + if( 0==*apColl ){ + *apColl = pParse->db->pDfltColl; + } + } + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, iDistinct, nCol, 0, + (void*)pKeyInfo, P4_KEYINFO); + }else{ + assert( pParse->nErr>0 ); + } } /* Detach the ORDER BY clause from the compound SELECT */ @@ -2829,7 +2828,7 @@ static void generateWithRecursiveQuery( #endif /* SQLITE_OMIT_CTE */ /* Forward references */ -static int multiSelectOrderBy( +static int multiSelectByMerge( Parse *pParse, /* Parsing context */ Select *p, /* The right-most of SELECTs to be coded */ SelectDest *pDest /* What to do with query results */ @@ -2978,12 +2977,26 @@ static int multiSelect( generateWithRecursiveQuery(pParse, p, &dest); }else #endif - - /* Compound SELECTs that have an ORDER BY clause are handled separately. - */ if( p->pOrderBy ){ - return multiSelectOrderBy(pParse, p, pDest); + /* If the compound has an ORDER BY clause, then always use the merge + ** algorithm. */ + return multiSelectByMerge(pParse, p, pDest); + }else if( p->op!=TK_ALL ){ + /* If the compound is EXCEPT, INTERSECT, or UNION (anything other than + ** UNION ALL) then also always use the merge algorithm. However, the + ** multiSelectByMerge() routine requires that the compound have an + ** ORDER BY clause, and it doesn't right now. So invent one first. */ + Expr *pOne = sqlite3ExprInt32(db, 1); + p->pOrderBy = sqlite3ExprListAppend(pParse, 0, pOne); + if( pParse->nErr ) goto multi_select_end; + assert( p->pOrderBy!=0 ); + p->pOrderBy->a[0].u.x.iOrderByCol = 1; + return multiSelectByMerge(pParse, p, pDest); }else{ + /* For a UNION ALL compound without ORDER BY, simply run the left + ** query, then run the right query */ + int addr = 0; + int nLimit = 0; /* Initialize to suppress harmless compiler warning */ #ifndef SQLITE_OMIT_EXPLAIN if( pPrior->pPrior==0 ){ @@ -2991,300 +3004,49 @@ static int multiSelect( ExplainQueryPlan((pParse, 1, "LEFT-MOST SUBQUERY")); } #endif - - /* Generate code for the left and right SELECT statements. - */ - switch( p->op ){ - case TK_ALL: { - int addr = 0; - int nLimit = 0; /* Initialize to suppress harmless compiler warning */ - assert( !pPrior->pLimit ); - pPrior->iLimit = p->iLimit; - pPrior->iOffset = p->iOffset; - pPrior->pLimit = p->pLimit; - TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL left...\n")); - rc = sqlite3Select(pParse, pPrior, &dest); - pPrior->pLimit = 0; - if( rc ){ - goto multi_select_end; - } - p->pPrior = 0; - p->iLimit = pPrior->iLimit; - p->iOffset = pPrior->iOffset; - if( p->iLimit ){ - addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v); - VdbeComment((v, "Jump ahead if LIMIT reached")); - if( p->iOffset ){ - sqlite3VdbeAddOp3(v, OP_OffsetLimit, - p->iLimit, p->iOffset+1, p->iOffset); - } - } - ExplainQueryPlan((pParse, 1, "UNION ALL")); - TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL right...\n")); - rc = sqlite3Select(pParse, p, &dest); - testcase( rc!=SQLITE_OK ); - pDelete = p->pPrior; - p->pPrior = pPrior; - p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); - if( p->pLimit - && sqlite3ExprIsInteger(p->pLimit->pLeft, &nLimit, pParse) - && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) - ){ - p->nSelectRow = sqlite3LogEst((u64)nLimit); - } - if( addr ){ - sqlite3VdbeJumpHere(v, addr); - } - break; - } - case TK_EXCEPT: - case TK_UNION: { - int unionTab; /* Cursor number of the temp table holding result */ - u8 op = 0; /* One of the SRT_ operations to apply to self */ - int priorOp; /* The SRT_ operation to apply to prior selects */ - Expr *pLimit; /* Saved values of p->nLimit */ - int addr; - int emptyBypass = 0; /* IfEmpty opcode to bypass RHS */ - SelectDest uniondest; - - - testcase( p->op==TK_EXCEPT ); - testcase( p->op==TK_UNION ); - priorOp = SRT_Union; - if( dest.eDest==priorOp ){ - /* We can reuse a temporary table generated by a SELECT to our - ** right. - */ - assert( p->pLimit==0 ); /* Not allowed on leftward elements */ - unionTab = dest.iSDParm; - }else{ - /* We will need to create our own temporary table to hold the - ** intermediate results. - */ - unionTab = pParse->nTab++; - assert( p->pOrderBy==0 ); - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, unionTab, 0); - assert( p->addrOpenEphm[0] == -1 ); - p->addrOpenEphm[0] = addr; - findRightmost(p)->selFlags |= SF_UsesEphemeral; - assert( p->pEList ); - } - - - /* Code the SELECT statements to our left - */ - assert( !pPrior->pOrderBy ); - sqlite3SelectDestInit(&uniondest, priorOp, unionTab); - TREETRACE(0x200, pParse, p, ("multiSelect EXCEPT/UNION left...\n")); - rc = sqlite3Select(pParse, pPrior, &uniondest); - if( rc ){ - goto multi_select_end; - } - - /* Code the current SELECT statement - */ - if( p->op==TK_EXCEPT ){ - op = SRT_Except; - emptyBypass = sqlite3VdbeAddOp1(v, OP_IfEmpty, unionTab); - VdbeCoverage(v); - }else{ - assert( p->op==TK_UNION ); - op = SRT_Union; - } - p->pPrior = 0; - pLimit = p->pLimit; - p->pLimit = 0; - uniondest.eDest = op; - ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", - sqlite3SelectOpName(p->op))); - TREETRACE(0x200, pParse, p, ("multiSelect EXCEPT/UNION right...\n")); - rc = sqlite3Select(pParse, p, &uniondest); - testcase( rc!=SQLITE_OK ); - assert( p->pOrderBy==0 ); - pDelete = p->pPrior; - p->pPrior = pPrior; - p->pOrderBy = 0; - if( p->op==TK_UNION ){ - p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); - } - if( emptyBypass ) sqlite3VdbeJumpHere(v, emptyBypass); - sqlite3ExprDelete(db, p->pLimit); - p->pLimit = pLimit; - p->iLimit = 0; - p->iOffset = 0; - - /* Convert the data in the temporary table into whatever form - ** it is that we currently need. - */ - assert( unionTab==dest.iSDParm || dest.eDest!=priorOp ); - assert( p->pEList || db->mallocFailed ); - if( dest.eDest!=priorOp && db->mallocFailed==0 ){ - int iCont, iBreak, iStart; - iBreak = sqlite3VdbeMakeLabel(pParse); - iCont = sqlite3VdbeMakeLabel(pParse); - computeLimitRegisters(pParse, p, iBreak); - sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v); - iStart = sqlite3VdbeCurrentAddr(v); - selectInnerLoop(pParse, p, unionTab, - 0, 0, &dest, iCont, iBreak); - sqlite3VdbeResolveLabel(v, iCont); - sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); VdbeCoverage(v); - sqlite3VdbeResolveLabel(v, iBreak); - sqlite3VdbeAddOp2(v, OP_Close, unionTab, 0); - } - break; - } - default: assert( p->op==TK_INTERSECT ); { - int tab1, tab2; - int iCont, iBreak, iStart; - Expr *pLimit; - int addr, iLimit, iOffset; - SelectDest intersectdest; - int r1; - int emptyBypass; - - /* INTERSECT is different from the others since it requires - ** two temporary tables. Hence it has its own case. Begin - ** by allocating the tables we will need. - */ - tab1 = pParse->nTab++; - tab2 = pParse->nTab++; - assert( p->pOrderBy==0 ); - - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab1, 0); - assert( p->addrOpenEphm[0] == -1 ); - p->addrOpenEphm[0] = addr; - findRightmost(p)->selFlags |= SF_UsesEphemeral; - assert( p->pEList ); - - /* Code the SELECTs to our left into temporary table "tab1". - */ - sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1); - TREETRACE(0x400, pParse, p, ("multiSelect INTERSECT left...\n")); - rc = sqlite3Select(pParse, pPrior, &intersectdest); - if( rc ){ - goto multi_select_end; - } - - /* Initialize LIMIT counters before checking to see if the LHS - ** is empty, in case the jump is taken */ - iBreak = sqlite3VdbeMakeLabel(pParse); - computeLimitRegisters(pParse, p, iBreak); - emptyBypass = sqlite3VdbeAddOp1(v, OP_IfEmpty, tab1); VdbeCoverage(v); - - /* Code the current SELECT into temporary table "tab2" - */ - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab2, 0); - assert( p->addrOpenEphm[1] == -1 ); - p->addrOpenEphm[1] = addr; - - /* Disable prior SELECTs and the LIMIT counters during the computation - ** of the RHS select */ - pLimit = p->pLimit; - iLimit = p->iLimit; - iOffset = p->iOffset; - p->pPrior = 0; - p->pLimit = 0; - p->iLimit = 0; - p->iOffset = 0; - - intersectdest.iSDParm = tab2; - ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", - sqlite3SelectOpName(p->op))); - TREETRACE(0x400, pParse, p, ("multiSelect INTERSECT right...\n")); - rc = sqlite3Select(pParse, p, &intersectdest); - testcase( rc!=SQLITE_OK ); - pDelete = p->pPrior; - p->pPrior = pPrior; - if( p->nSelectRow>pPrior->nSelectRow ){ - p->nSelectRow = pPrior->nSelectRow; - } - sqlite3ExprDelete(db, p->pLimit); - - /* Reinstate the LIMIT counters prior to running the final intersect */ - p->pLimit = pLimit; - p->iLimit = iLimit; - p->iOffset = iOffset; - - /* Generate code to take the intersection of the two temporary - ** tables. - */ - if( rc ) break; - assert( p->pEList ); - sqlite3VdbeAddOp1(v, OP_Rewind, tab1); - r1 = sqlite3GetTempReg(pParse); - iStart = sqlite3VdbeAddOp2(v, OP_RowData, tab1, r1); - iCont = sqlite3VdbeMakeLabel(pParse); - sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); - VdbeCoverage(v); - sqlite3ReleaseTempReg(pParse, r1); - selectInnerLoop(pParse, p, tab1, - 0, 0, &dest, iCont, iBreak); - sqlite3VdbeResolveLabel(v, iCont); - sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v); - sqlite3VdbeResolveLabel(v, iBreak); - sqlite3VdbeAddOp2(v, OP_Close, tab2, 0); - sqlite3VdbeJumpHere(v, emptyBypass); - sqlite3VdbeAddOp2(v, OP_Close, tab1, 0); - break; - } - } - - #ifndef SQLITE_OMIT_EXPLAIN - if( p->pNext==0 ){ - ExplainQueryPlanPop(pParse); - } - #endif - } - if( pParse->nErr ) goto multi_select_end; - - /* Compute collating sequences used by - ** temporary tables needed to implement the compound select. - ** Attach the KeyInfo structure to all temporary tables. - ** - ** This section is run by the right-most SELECT statement only. - ** SELECT statements to the left always skip this part. The right-most - ** SELECT might also skip this part if it has no ORDER BY clause and - ** no temp tables are required. - */ - if( p->selFlags & SF_UsesEphemeral ){ - int i; /* Loop counter */ - KeyInfo *pKeyInfo; /* Collating sequence for the result set */ - Select *pLoop; /* For looping through SELECT statements */ - CollSeq **apColl; /* For looping through pKeyInfo->aColl[] */ - int nCol; /* Number of columns in result set */ - - assert( p->pNext==0 ); - assert( p->pEList!=0 ); - nCol = p->pEList->nExpr; - pKeyInfo = sqlite3KeyInfoAlloc(db, nCol, 1); - if( !pKeyInfo ){ - rc = SQLITE_NOMEM_BKPT; + assert( !pPrior->pLimit ); + pPrior->iLimit = p->iLimit; + pPrior->iOffset = p->iOffset; + pPrior->pLimit = sqlite3ExprDup(db, p->pLimit, 0); + TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL left...\n")); + rc = sqlite3Select(pParse, pPrior, &dest); + sqlite3ExprDelete(db, pPrior->pLimit); + pPrior->pLimit = 0; + if( rc ){ goto multi_select_end; } - for(i=0, apColl=pKeyInfo->aColl; i<nCol; i++, apColl++){ - *apColl = multiSelectCollSeq(pParse, p, i); - if( 0==*apColl ){ - *apColl = db->pDfltColl; - } + p->pPrior = 0; + p->iLimit = pPrior->iLimit; + p->iOffset = pPrior->iOffset; + if( p->iLimit ){ + addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v); + VdbeComment((v, "Jump ahead if LIMIT reached")); + if( p->iOffset ){ + sqlite3VdbeAddOp3(v, OP_OffsetLimit, + p->iLimit, p->iOffset+1, p->iOffset); + } + } + ExplainQueryPlan((pParse, 1, "UNION ALL")); + TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL right...\n")); + rc = sqlite3Select(pParse, p, &dest); + testcase( rc!=SQLITE_OK ); + pDelete = p->pPrior; + p->pPrior = pPrior; + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); + if( p->pLimit + && sqlite3ExprIsInteger(p->pLimit->pLeft, &nLimit, pParse) + && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) + ){ + p->nSelectRow = sqlite3LogEst((u64)nLimit); } - - for(pLoop=p; pLoop; pLoop=pLoop->pPrior){ - for(i=0; i<2; i++){ - int addr = pLoop->addrOpenEphm[i]; - if( addr<0 ){ - /* If [0] is unused then [1] is also unused. So we can - ** always safely abort as soon as the first unused slot is found */ - assert( pLoop->addrOpenEphm[1]<0 ); - break; - } - sqlite3VdbeChangeP2(v, addr, nCol); - sqlite3VdbeChangeP4(v, addr, (char*)sqlite3KeyInfoRef(pKeyInfo), - P4_KEYINFO); - pLoop->addrOpenEphm[i] = -1; - } + if( addr ){ + sqlite3VdbeJumpHere(v, addr); } - sqlite3KeyInfoUnref(pKeyInfo); +#ifndef SQLITE_OMIT_EXPLAIN + if( p->pNext==0 ){ + ExplainQueryPlanPop(pParse); + } +#endif } multi_select_end: @@ -3316,8 +3078,8 @@ void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p){ ** Code an output subroutine for a coroutine implementation of a ** SELECT statement. ** -** The data to be output is contained in pIn->iSdst. There are -** pIn->nSdst columns to be output. pDest is where the output should +** The data to be output is contained in an array of pIn->nSdst registers +** starting at register pIn->iSdst. pDest is where the output should ** be sent. ** ** regReturn is the number of the register holding the subroutine @@ -3346,6 +3108,8 @@ static int generateOutputSubroutine( int iContinue; int addr; + assert( pIn->eDest==SRT_Coroutine ); + addr = sqlite3VdbeCurrentAddr(v); iContinue = sqlite3VdbeMakeLabel(pParse); @@ -3367,23 +3131,60 @@ static int generateOutputSubroutine( */ codeOffset(v, p->iOffset, iContinue); - assert( pDest->eDest!=SRT_Exists ); - assert( pDest->eDest!=SRT_Table ); switch( pDest->eDest ){ /* Store the result as data using a unique key. */ + case SRT_Fifo: + case SRT_DistFifo: + case SRT_Table: case SRT_EphemTab: { int r1 = sqlite3GetTempReg(pParse); int r2 = sqlite3GetTempReg(pParse); + int iParm = pDest->iSDParm; + testcase( pDest->eDest==SRT_Table ); + testcase( pDest->eDest==SRT_EphemTab ); + testcase( pDest->eDest==SRT_Fifo ); + testcase( pDest->eDest==SRT_DistFifo ); sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r1); - sqlite3VdbeAddOp2(v, OP_NewRowid, pDest->iSDParm, r2); - sqlite3VdbeAddOp3(v, OP_Insert, pDest->iSDParm, r1, r2); +#if !defined(SQLITE_ENABLE_NULL_TRIM) && defined(SQLITE_DEBUG) + /* A destination of SRT_Table and a non-zero iSDParm2 parameter means + ** that this is an "UPDATE ... FROM" on a virtual table or view. In this + ** case set the p5 parameter of the OP_MakeRecord to OPFLAG_NOCHNG_MAGIC. + ** This does not affect operation in any way - it just allows MakeRecord + ** to process OPFLAG_NOCHANGE values without an assert() failing. */ + if( pDest->eDest==SRT_Table && pDest->iSDParm2 ){ + sqlite3VdbeChangeP5(v, OPFLAG_NOCHNG_MAGIC); + } +#endif +#ifndef SQLITE_OMIT_CTE + if( pDest->eDest==SRT_DistFifo ){ + /* If the destination is DistFifo, then cursor (iParm+1) is open + ** on an ephemeral index that is used to enforce uniqueness on the + ** total result. At this point, we are processing the setup portion + ** of the recursive CTE using the merge algorithm, so the results are + ** guaranteed to be unique anyhow. But we still need to populate the + ** (iParm+1) cursor for use by the subsequent recursive phase. + */ + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm+1, r1, + pIn->iSdst, pIn->nSdst); + } +#endif + sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); + sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, r2); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3ReleaseTempReg(pParse, r2); sqlite3ReleaseTempReg(pParse, r1); break; } + /* If any row exist in the result set, record that fact and abort. + */ + case SRT_Exists: { + sqlite3VdbeAddOp2(v, OP_Integer, 1, pDest->iSDParm); + /* The LIMIT clause will terminate the loop for us */ + break; + } + #ifndef SQLITE_OMIT_SUBQUERY /* If we are creating a set for an "expr IN (SELECT ...)". */ @@ -3430,9 +3231,51 @@ static int generateOutputSubroutine( break; } +#ifndef SQLITE_OMIT_CTE + /* Write the results into a priority queue that is order according to + ** pDest->pOrderBy (in pSO). pDest->iSDParm (in iParm) is the cursor for an + ** index with pSO->nExpr+2 columns. Build a key using pSO for the first + ** pSO->nExpr columns, then make sure all keys are unique by adding a + ** final OP_Sequence column. The last column is the record as a blob. + */ + case SRT_DistQueue: + case SRT_Queue: { + int nKey; + int r1, r2, r3, ii; + ExprList *pSO; + int iParm = pDest->iSDParm; + pSO = pDest->pOrderBy; + assert( pSO ); + nKey = pSO->nExpr; + r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3GetTempRange(pParse, nKey+2); + r3 = r2+nKey+1; + + sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r3); + if( pDest->eDest==SRT_DistQueue ){ + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r3); + } + for(ii=0; ii<nKey; ii++){ + sqlite3VdbeAddOp2(v, OP_SCopy, + pIn->iSdst + pSO->a[ii].u.x.iOrderByCol - 1, + r2+ii); + } + sqlite3VdbeAddOp2(v, OP_Sequence, iParm, r2+nKey); + sqlite3VdbeAddOp3(v, OP_MakeRecord, r2, nKey+2, r1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, r2, nKey+2); + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempRange(pParse, r2, nKey+2); + break; + } +#endif /* SQLITE_OMIT_CTE */ + + /* Ignore the output */ + case SRT_Discard: { + break; + } + /* If none of the above, then the result destination must be - ** SRT_Output. This routine is never called with any other - ** destination other than the ones handled above or SRT_Output. + ** SRT_Output. ** ** For SRT_Output, results are stored in a sequence of registers. ** Then the OP_ResultRow opcode is used to cause sqlite3_step() to @@ -3460,8 +3303,9 @@ static int generateOutputSubroutine( } /* -** Alternative compound select code generator for cases when there -** is an ORDER BY clause. +** Generate code for a compound SELECT statement using a merge +** algorithm. The compound must have an ORDER BY clause for this +** to work. ** ** We assume a query of the following form: ** @@ -3478,7 +3322,7 @@ static int generateOutputSubroutine( ** ** outB: Move the output of the selectB coroutine into the output ** of the compound query. (Only generated for UNION and -** UNION ALL. EXCEPT and INSERTSECT never output a row that +** UNION ALL. EXCEPT and INTERSECT never output a row that ** appears only in B.) ** ** AltB: Called when there is data from both coroutines and A<B. @@ -3531,10 +3375,8 @@ static int generateOutputSubroutine( ** AeqB: ... ** AgtB: ... ** Init: initialize coroutine registers -** yield coA -** if eof(A) goto EofA -** yield coB -** if eof(B) goto EofB +** yield coA, on eof goto EofA +** yield coB, on eof goto EofB ** Cmpr: Compare A, B ** Jump AltB, AeqB, AgtB ** End: ... @@ -3542,10 +3384,10 @@ static int generateOutputSubroutine( ** We call AltB, AeqB, AgtB, EofA, and EofB "subroutines" but they are not ** actually called using Gosub and they do not Return. EofA and EofB loop ** until all data is exhausted then jump to the "end" label. AltB, AeqB, -** and AgtB jump to either L2 or to one of EofA or EofB. +** and AgtB jump to either Cmpr or to one of EofA or EofB. */ #ifndef SQLITE_OMIT_COMPOUND_SELECT -static int multiSelectOrderBy( +static int multiSelectByMerge( Parse *pParse, /* Parsing context */ Select *p, /* The right-most of SELECTs to be coded */ SelectDest *pDest /* What to do with query results */ @@ -3617,10 +3459,8 @@ static int multiSelectOrderBy( if( pItem->u.x.iOrderByCol==i ) break; } if( j==nOrderBy ){ - Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); + Expr *pNew = sqlite3ExprInt32(db, i); if( pNew==0 ) return SQLITE_NOMEM_BKPT; - pNew->flags |= EP_IntValue; - pNew->u.iValue = i; p->pOrderBy = pOrderBy = sqlite3ExprListAppend(pParse, pOrderBy, pNew); if( pOrderBy ) pOrderBy->a[nOrderBy++].u.x.iOrderByCol = (u16)i; } @@ -3628,26 +3468,29 @@ static int multiSelectOrderBy( } /* Compute the comparison permutation and keyinfo that is used with - ** the permutation used to determine if the next - ** row of results comes from selectA or selectB. Also add explicit - ** collations to the ORDER BY clause terms so that when the subqueries - ** to the right and the left are evaluated, they use the correct - ** collation. + ** the permutation to determine if the next row of results comes + ** from selectA or selectB. Also add literal collations to the + ** ORDER BY clause terms so that when selectA and selectB are + ** evaluated, they use the correct collation. */ aPermute = sqlite3DbMallocRawNN(db, sizeof(u32)*(nOrderBy + 1)); if( aPermute ){ struct ExprList_item *pItem; + int bKeep = 0; aPermute[0] = nOrderBy; for(i=1, pItem=pOrderBy->a; i<=nOrderBy; i++, pItem++){ assert( pItem!=0 ); assert( pItem->u.x.iOrderByCol>0 ); assert( pItem->u.x.iOrderByCol<=p->pEList->nExpr ); aPermute[i] = pItem->u.x.iOrderByCol - 1; + if( aPermute[i]!=(u32)i-1 ) bKeep = 1; + } + if( bKeep==0 ){ + sqlite3DbFreeNN(db, aPermute); + aPermute = 0; } - pKeyMerge = multiSelectOrderByKeyInfo(pParse, p, 1); - }else{ - pKeyMerge = 0; } + pKeyMerge = multiSelectByMergeKeyInfo(pParse, p, 1); /* Allocate a range of temporary registers and the KeyInfo needed ** for the logic that removes duplicate result rows when the @@ -3726,7 +3569,7 @@ static int multiSelectOrderBy( */ addrSelectA = sqlite3VdbeCurrentAddr(v) + 1; addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); - VdbeComment((v, "left SELECT")); + VdbeComment((v, "SUBR: next-A")); pPrior->iLimit = regLimitA; ExplainQueryPlan((pParse, 1, "LEFT")); sqlite3Select(pParse, pPrior, &destA); @@ -3738,7 +3581,7 @@ static int multiSelectOrderBy( */ addrSelectB = sqlite3VdbeCurrentAddr(v) + 1; addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB); - VdbeComment((v, "right SELECT")); + VdbeComment((v, "SUBR: next-B")); savedLimit = p->iLimit; savedOffset = p->iOffset; p->iLimit = regLimitB; @@ -3752,7 +3595,7 @@ static int multiSelectOrderBy( /* Generate a subroutine that outputs the current row of the A ** select as the next output row of the compound select. */ - VdbeNoopComment((v, "Output routine for A")); + VdbeNoopComment((v, "SUBR: out-A")); addrOutA = generateOutputSubroutine(pParse, p, &destA, pDest, regOutA, regPrev, pKeyDup, labelEnd); @@ -3761,7 +3604,7 @@ static int multiSelectOrderBy( ** select as the next output row of the compound select. */ if( op==TK_ALL || op==TK_UNION ){ - VdbeNoopComment((v, "Output routine for B")); + VdbeNoopComment((v, "SUBR: out-B")); addrOutB = generateOutputSubroutine(pParse, p, &destB, pDest, regOutB, regPrev, pKeyDup, labelEnd); @@ -3774,10 +3617,12 @@ static int multiSelectOrderBy( if( op==TK_EXCEPT || op==TK_INTERSECT ){ addrEofA_noB = addrEofA = labelEnd; }else{ - VdbeNoopComment((v, "eof-A subroutine")); + VdbeNoopComment((v, "SUBR: eof-A")); addrEofA = sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); + VdbeComment((v, "out-B")); addrEofA_noB = sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, labelEnd); VdbeCoverage(v); + VdbeComment((v, "next-B")); sqlite3VdbeGoto(v, addrEofA); p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); } @@ -3789,17 +3634,20 @@ static int multiSelectOrderBy( addrEofB = addrEofA; if( p->nSelectRow > pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; }else{ - VdbeNoopComment((v, "eof-B subroutine")); + VdbeNoopComment((v, "SUBR: eof-B")); addrEofB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); + VdbeComment((v, "out-A")); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, labelEnd); VdbeCoverage(v); + VdbeComment((v, "next-A")); sqlite3VdbeGoto(v, addrEofB); } /* Generate code to handle the case of A<B */ - VdbeNoopComment((v, "A-lt-B subroutine")); addrAltB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); + VdbeComment((v, "out-A")); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA); VdbeCoverage(v); + VdbeComment((v, "next-A")); sqlite3VdbeGoto(v, labelCmpr); /* Generate code to handle the case of A==B @@ -3810,36 +3658,48 @@ static int multiSelectOrderBy( addrAeqB = addrAltB; addrAltB++; }else{ - VdbeNoopComment((v, "A-eq-B subroutine")); - addrAeqB = - sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA); VdbeCoverage(v); - sqlite3VdbeGoto(v, labelCmpr); + addrAeqB = addrAltB + 1; } /* Generate code to handle the case of A>B */ - VdbeNoopComment((v, "A-gt-B subroutine")); addrAgtB = sqlite3VdbeCurrentAddr(v); if( op==TK_ALL || op==TK_UNION ){ sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); + VdbeComment((v, "out-B")); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); + VdbeComment((v, "next-B")); + sqlite3VdbeGoto(v, labelCmpr); + }else{ + addrAgtB++; /* Just do next-B. Might as well use the next-B call + ** in the next code block */ } - sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); - sqlite3VdbeGoto(v, labelCmpr); /* This code runs once to initialize everything. */ sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA_noB); VdbeCoverage(v); + VdbeComment((v, "next-A")); + /* v--- Also the A>B case for EXCEPT and INTERSECT */ sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); + VdbeComment((v, "next-B")); /* Implement the main merge loop */ + if( aPermute!=0 ){ + sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY); + } sqlite3VdbeResolveLabel(v, labelCmpr); - sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY); sqlite3VdbeAddOp4(v, OP_Compare, destA.iSdst, destB.iSdst, nOrderBy, (char*)pKeyMerge, P4_KEYINFO); - sqlite3VdbeChangeP5(v, OPFLAG_PERMUTE); - sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); VdbeCoverage(v); + if( aPermute!=0 ){ + sqlite3VdbeChangeP5(v, OPFLAG_PERMUTE); + } + sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); + VdbeCoverageIf(v, op==TK_ALL); + VdbeCoverageIf(v, op==TK_UNION); + VdbeCoverageIf(v, op==TK_EXCEPT); + VdbeCoverageIf(v, op==TK_INTERSECT); /* Jump to the this point in order to terminate the query. */ @@ -4756,7 +4616,7 @@ static int flattenSubquery( } pSubitem->fg.jointype |= jointype; - /* Now begin substituting subquery result set expressions for + /* Begin substituting subquery result set expressions for ** references to the iParent in the outer query. ** ** Example: @@ -4768,7 +4628,7 @@ static int flattenSubquery( ** We look at every expression in the outer query and every place we see ** "a" we substitute "x*3" and every place we see "b" we substitute "y+10". */ - if( pSub->pOrderBy && (pParent->selFlags & SF_NoopOrderBy)==0 ){ + if( pSub->pOrderBy ){ /* At this point, any non-zero iOrderByCol values indicate that the ** ORDER BY column expression is identical to the iOrderByCol'th ** expression returned by SELECT statement pSub. Since these values @@ -4776,9 +4636,9 @@ static int flattenSubquery( ** zero them before transferring the ORDER BY clause. ** ** Not doing this may cause an error if a subsequent call to this - ** function attempts to flatten a compound sub-query into pParent - ** (the only way this can happen is if the compound sub-query is - ** currently part of pSub->pSrc). See ticket [d11a6e908f]. */ + ** function attempts to flatten a compound sub-query into pParent. + ** See ticket [d11a6e908f]. + */ ExprList *pOrderBy = pSub->pOrderBy; for(i=0; i<pOrderBy->nExpr; i++){ pOrderBy->a[i].u.x.iOrderByCol = 0; @@ -5396,6 +5256,16 @@ static int pushDownWhereTerms( x.pEList = pSubq->pEList; x.pCList = findLeftmostExprlist(pSubq); pNew = substExpr(&x, pNew); + assert( pNew!=0 || pParse->nErr!=0 ); + if( pParse->nErr==0 && pNew->op==TK_IN && ExprUseXSelect(pNew) ){ + assert( pNew->x.pSelect!=0 ); + pNew->x.pSelect->selFlags |= SF_ClonedRhsIn; + assert( pWhere!=0 ); + assert( pWhere->op==TK_IN ); + assert( ExprUseXSelect(pWhere) ); + assert( pWhere->x.pSelect!=0 ); + pWhere->x.pSelect->selFlags |= SF_ClonedRhsIn; + } #ifndef SQLITE_OMIT_WINDOWFUNC if( pSubq->pWin && 0==pushDownWindowCheck(pParse, pSubq, pNew) ){ /* Restriction 6c has prevented push-down in this case */ @@ -5630,14 +5500,14 @@ int sqlite3IndexedByLookup(Parse *pParse, SrcItem *pFrom){ ** SELECT * FROM (SELECT ... FROM t1 EXCEPT SELECT ... FROM t2) ** ORDER BY ... COLLATE ... ** -** This transformation is necessary because the multiSelectOrderBy() routine +** This transformation is necessary because the multiSelectByMerge() routine ** above that generates the code for a compound SELECT with an ORDER BY clause ** uses a merge algorithm that requires the same collating sequence on the ** result columns as on the ORDER BY clause. See ticket ** http://sqlite.org/src/info/6709574d2a ** ** This transformation is only needed for EXCEPT, INTERSECT, and UNION. -** The UNION ALL operator works fine with multiSelectOrderBy() even when +** The UNION ALL operator works fine with multiSelectByMerge() even when ** there are COLLATE terms in the ORDER BY. */ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ @@ -6183,7 +6053,7 @@ static int selectExpander(Walker *pWalker, Select *p){ } #ifndef SQLITE_OMIT_VIRTUALTABLE else if( ALWAYS(IsVirtual(pTab)) - && pFrom->fg.fromDDL + && (pFrom->fg.fromDDL || (pParse->prepFlags & SQLITE_PREPARE_FROM_DDL)) && ALWAYS(pTab->u.vtab.p!=0) && pTab->u.vtab.p->eVtabRisk > ((db->flags & SQLITE_TrustedSchema)!=0) ){ @@ -7136,7 +7006,7 @@ static int havingToWhereExprCb(Walker *pWalker, Expr *pExpr){ && pExpr->pAggInfo==0 ){ sqlite3 *db = pWalker->pParse->db; - Expr *pNew = sqlite3Expr(db, TK_INTEGER, "1"); + Expr *pNew = sqlite3ExprInt32(db, 1); if( pNew ){ Expr *pWhere = pS->pWhere; SWAP(Expr, *pNew, *pExpr); @@ -7487,7 +7357,6 @@ static SQLITE_NOINLINE void existsToJoin( ExprSetProperty(pWhere, EP_IntValue); assert( p->pWhere!=0 ); pSub->pSrc->a[0].fg.fromExists = 1; - pSub->pSrc->a[0].fg.jointype |= JT_CROSS; p->pSrc = sqlite3SrcListAppendList(pParse, p->pSrc, pSub->pSrc); if( pSubWhere ){ p->pWhere = sqlite3PExpr(pParse, TK_AND, p->pWhere, pSubWhere); @@ -7515,6 +7384,7 @@ typedef struct CheckOnCtx CheckOnCtx; struct CheckOnCtx { SrcList *pSrc; /* SrcList for this context */ int iJoin; /* Cursor numbers must be =< than this */ + int bFuncArg; /* True for table-function arg */ CheckOnCtx *pParent; /* Parent context */ }; @@ -7566,7 +7436,9 @@ static int selectCheckOnClausesExpr(Walker *pWalker, Expr *pExpr){ if( iTab>=pSrc->a[0].iCursor && iTab<=pSrc->a[pSrc->nSrc-1].iCursor ){ if( pCtx->iJoin && iTab>pCtx->iJoin ){ sqlite3ErrorMsg(pWalker->pParse, - "ON clause references tables to its right"); + "%s references tables to its right", + (pCtx->bFuncArg ? "table-function argument" : "ON clause") + ); return WRC_Abort; } break; @@ -7604,6 +7476,7 @@ static int selectCheckOnClausesSelect(Walker *pWalker, Select *pSelect){ void sqlite3SelectCheckOnClauses(Parse *pParse, Select *pSelect){ Walker w; CheckOnCtx sCtx; + int ii; assert( pSelect->selFlags & SF_OnToWhere ); assert( pSelect->pSrc!=0 && pSelect->pSrc->nSrc>=2 ); memset(&w, 0, sizeof(w)); @@ -7613,8 +7486,46 @@ void sqlite3SelectCheckOnClauses(Parse *pParse, Select *pSelect){ w.u.pCheckOnCtx = &sCtx; memset(&sCtx, 0, sizeof(sCtx)); sCtx.pSrc = pSelect->pSrc; - sqlite3WalkExprNN(&w, pSelect->pWhere); + sqlite3WalkExpr(&w, pSelect->pWhere); pSelect->selFlags &= ~SF_OnToWhere; + + /* Check for any table-function args that are attached to virtual tables + ** on the RHS of an outer join. They are subject to the same constraints + ** as ON clauses. */ + sCtx.bFuncArg = 1; + for(ii=0; ii<pSelect->pSrc->nSrc; ii++){ + SrcItem *pItem = &pSelect->pSrc->a[ii]; + if( pItem->fg.isTabFunc + && (pItem->fg.jointype & JT_OUTER) + ){ + sCtx.iJoin = pItem->iCursor; + sqlite3WalkExprList(&w, pItem->u1.pFuncArg); + } + } +} + +/* +** If p2 exists and p1 and p2 have the same number of terms, then change +** every term of p1 to have the same sort order as p2 and return true. +** +** If p2 is NULL or p1 and p2 are different lengths, then make no changes +** and return false. +** +** p1 must be non-NULL. +*/ +static int sqlite3CopySortOrder(ExprList *p1, ExprList *p2){ + assert( p1 ); + if( p2 && p1->nExpr==p2->nExpr ){ + int ii; + for(ii=0; ii<p1->nExpr; ii++){ + u8 sortFlags; + sortFlags = p2->a[ii].fg.sortFlags & KEYINFO_ORDER_DESC; + p1->a[ii].fg.sortFlags = sortFlags; + } + return 1; + }else{ + return 0; + } } /* @@ -7712,8 +7623,7 @@ int sqlite3Select( assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_Queue ); if( IgnorableDistinct(pDest) ){ - assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union || - pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard || + assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Discard || pDest->eDest==SRT_DistQueue || pDest->eDest==SRT_DistFifo ); /* All of these destinations are also able to ignore the ORDER BY clause */ if( p->pOrderBy ){ @@ -7729,7 +7639,6 @@ int sqlite3Select( p->pOrderBy = 0; } p->selFlags &= ~(u32)SF_Distinct; - p->selFlags |= SF_NoopOrderBy; } sqlite3SelectPrep(pParse, p, 0); if( pParse->nErr ){ @@ -8257,7 +8166,8 @@ int sqlite3Select( ** BY and DISTINCT, and an index or separate temp-table for the other. */ if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct - && sqlite3ExprListCompare(sSort.pOrderBy, pEList, -1)==0 + && sqlite3CopySortOrder(pEList, sSort.pOrderBy) + && sqlite3ExprListCompare(pEList, sSort.pOrderBy, -1)==0 && OptimizationEnabled(db, SQLITE_GroupByOrder) #ifndef SQLITE_OMIT_WINDOWFUNC && p->pWin==0 @@ -8471,21 +8381,10 @@ int sqlite3Select( ** but not actually sorted. Either way, record the fact that the ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp ** variable. */ - if( sSort.pOrderBy && pGroupBy->nExpr==sSort.pOrderBy->nExpr ){ - int ii; - /* The GROUP BY processing doesn't care whether rows are delivered in - ** ASC or DESC order - only that each group is returned contiguously. - ** So set the ASC/DESC flags in the GROUP BY to match those in the - ** ORDER BY to maximize the chances of rows being delivered in an - ** order that makes the ORDER BY redundant. */ - for(ii=0; ii<pGroupBy->nExpr; ii++){ - u8 sortFlags; - sortFlags = sSort.pOrderBy->a[ii].fg.sortFlags & KEYINFO_ORDER_DESC; - pGroupBy->a[ii].fg.sortFlags = sortFlags; - } - if( sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 ){ - orderByGrp = 1; - } + if( sqlite3CopySortOrder(pGroupBy, sSort.pOrderBy) + && sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 + ){ + orderByGrp = 1; } }else{ assert( 0==sqlite3LogEst(1) ); diff --git a/src/shell.c.in b/src/shell.c.in index bd4483ff7..b121ed836 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file contains code to implement the "sqlite" command line +** This file contains code to implement the "sqlite3" command line ** utility for accessing SQLite databases. */ #if (defined(_WIN32) || defined(WIN32)) && !defined(_CRT_SECURE_NO_WARNINGS) @@ -19,6 +19,22 @@ typedef unsigned int u32; typedef unsigned short int u16; +/* +** Limit input nesting via .read or any other input redirect. +** It's not too expensive, so a generous allowance can be made. +*/ +#define MAX_INPUT_NESTING 25 + +/* +** Used to prevent warnings about unused parameters +*/ +#define UNUSED_PARAMETER(x) (void)(x) + +/* +** Number of elements in an array +*/ +#define ArraySize(X) (int)(sizeof(X)/sizeof(X[0])) + /* ** Optionally #include a user-defined header, whereby compilation options ** may be set prior to where they take effect, but after platform setup. @@ -31,14 +47,6 @@ typedef unsigned short int u16; # include SHELL_STRINGIFY(SQLITE_CUSTOM_INCLUDE) #endif -/* -** Determine if we are dealing with WinRT, which provides only a subset of -** the full Win32 API. -*/ -#if !defined(SQLITE_OS_WINRT) -# define SQLITE_OS_WINRT 0 -#endif - /* ** If SQLITE_SHELL_FIDDLE is defined then the shell is modified ** somewhat for use as a WASM module in a web browser. This flag @@ -47,6 +55,10 @@ typedef unsigned short int u16; ** and this build mode rewires the user input subsystem to account for ** that. */ +#if defined(SQLITE_SHELL_FIDDLE) +# undef SQLITE_OMIT_LOAD_EXTENSION +# define SQLITE_OMIT_LOAD_EXTENSION 1 +#endif /* ** Warning pragmas copied from msvc.h in the core. @@ -100,6 +112,7 @@ typedef unsigned short int u16; #include <stdio.h> #include <assert.h> #include <math.h> +#include <stdint.h> #include "sqlite3.h" typedef sqlite3_int64 i64; typedef sqlite3_uint64 u64; @@ -108,6 +121,7 @@ typedef unsigned char u8; #include <stdarg.h> #ifndef _WIN32 # include <sys/time.h> +# include <sys/ioctl.h> #endif #if !defined(_WIN32) && !defined(WIN32) @@ -177,9 +191,6 @@ typedef unsigned char u8; #endif #if defined(_WIN32) || defined(WIN32) -# if SQLITE_OS_WINRT -# define SQLITE_OMIT_POPEN 1 -# else # include <io.h> # include <fcntl.h> # define isatty(h) _isatty(h) @@ -194,7 +205,6 @@ typedef unsigned char u8; # endif # undef pclose # define pclose _pclose -# endif #else /* Make sure isatty() has a prototype. */ extern int isatty(int); @@ -225,9 +235,6 @@ typedef unsigned char u8; #define IsAlpha(X) isalpha((unsigned char)X) #if defined(_WIN32) || defined(WIN32) -#if SQLITE_OS_WINRT -#include <intrin.h> -#endif #undef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #include <windows.h> @@ -239,6 +246,8 @@ extern LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText); INCLUDE ../ext/misc/sqlite3_stdio.h INCLUDE ../ext/misc/sqlite3_stdio.c +INCLUDE ../ext/qrf/qrf.h +INCLUDE ../ext/qrf/qrf.c /* Use console I/O package as a direct INCLUDE. */ #define SQLITE_INTERNAL_LINKAGE static @@ -252,212 +261,376 @@ INCLUDE ../ext/misc/sqlite3_stdio.c # define SQLITE_CIO_NO_FLUSH #endif -#define eputz(z) sqlite3_fputs(z,stderr) -#define sputz(fp,z) sqlite3_fputs(z,fp) - -/* True if the timer is enabled */ -static int enableTimer = 0; +/* +** The source code for several run-time loadable extensions is inserted +** below by the ../tool/mkshellc.tcl script. Before processing that included +** code, we need to override some macros to make the included program code +** work here in the middle of this regular program. +*/ +#define SQLITE_EXTENSION_INIT1 +#define SQLITE_EXTENSION_INIT2(X) (void)(X) -/* A version of strcmp() that works with NULL values */ -static int cli_strcmp(const char *a, const char *b){ - if( a==0 ) a = ""; - if( b==0 ) b = ""; - return strcmp(a,b); -} -static int cli_strncmp(const char *a, const char *b, size_t n){ - if( a==0 ) a = ""; - if( b==0 ) b = ""; - return strncmp(a,b,n); -} +INCLUDE ../ext/misc/windirent.h +INCLUDE ../ext/misc/memtrace.c +INCLUDE ../ext/misc/pcachetrace.c +INCLUDE ../ext/misc/shathree.c +INCLUDE ../ext/misc/sha1.c +INCLUDE ../ext/misc/uint.c +INCLUDE ../ext/misc/decimal.c +INCLUDE ../ext/misc/base64.c +INCLUDE ../ext/misc/base85.c +INCLUDE ../ext/misc/ieee754.c +INCLUDE ../ext/misc/series.c +INCLUDE ../ext/misc/regexp.c +#ifndef SQLITE_SHELL_FIDDLE +INCLUDE ../ext/misc/fileio.c +INCLUDE ../ext/misc/completion.c +INCLUDE ../ext/misc/appendvfs.c +#endif +#ifdef SQLITE_HAVE_ZLIB +INCLUDE ../ext/misc/zipfile.c +INCLUDE ../ext/misc/sqlar.c +#endif +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) +INCLUDE ../ext/expert/sqlite3expert.h +INCLUDE ../ext/expert/sqlite3expert.c +#endif +INCLUDE ../ext/intck/sqlite3intck.h +INCLUDE ../ext/intck/sqlite3intck.c +INCLUDE ../ext/misc/stmtrand.c +INCLUDE ../ext/misc/vfstrace.c -/* Return the current wall-clock time in microseconds since the -** Unix epoch (1970-01-01T00:00:00Z) -*/ -static sqlite3_int64 timeOfDay(void){ -#if defined(_WIN64) - sqlite3_uint64 t; - FILETIME tm; - GetSystemTimePreciseAsFileTime(&tm); - t = ((u64)tm.dwHighDateTime<<32) | (u64)tm.dwLowDateTime; - t += 116444736000000000LL; - t /= 10; - return t; -#elif defined(_WIN32) - static sqlite3_vfs *clockVfs = 0; - sqlite3_int64 t; - if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); - if( clockVfs==0 ) return 0; /* Never actually happens */ - if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){ - clockVfs->xCurrentTimeInt64(clockVfs, &t); - }else{ - double r; - clockVfs->xCurrentTime(clockVfs, &r); - t = (sqlite3_int64)(r*86400000.0); - } - return t*1000; +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) +#define SQLITE_SHELL_HAVE_RECOVER 1 #else - struct timeval sNow; - (void)gettimeofday(&sNow,0); - return ((i64)sNow.tv_sec)*1000000 + sNow.tv_usec; +#define SQLITE_SHELL_HAVE_RECOVER 0 +#endif +#if SQLITE_SHELL_HAVE_RECOVER +INCLUDE ../ext/recover/sqlite3recover.h +# ifndef SQLITE_HAVE_SQLITE3R +INCLUDE ../ext/recover/dbdata.c +INCLUDE ../ext/recover/sqlite3recover.c +# endif /* SQLITE_HAVE_SQLITE3R */ +#endif +#ifdef SQLITE_SHELL_EXTSRC +# include SHELL_STRINGIFY(SQLITE_SHELL_EXTSRC) #endif -} - -#if !defined(_WIN32) && !defined(WIN32) && !defined(__minux) -#include <sys/time.h> -#include <sys/resource.h> -/* VxWorks does not support getrusage() as far as we can determine */ -#if defined(_WRS_KERNEL) || defined(__RTP__) -struct rusage { - struct timeval ru_utime; /* user CPU time used */ - struct timeval ru_stime; /* system CPU time used */ +#if defined(SQLITE_ENABLE_SESSION) +/* +** State information for a single open session +*/ +typedef struct OpenSession OpenSession; +struct OpenSession { + char *zName; /* Symbolic name for this session */ + int nFilter; /* Number of xFilter rejection GLOB patterns */ + char **azFilter; /* Array of xFilter rejection GLOB patterns */ + sqlite3_session *p; /* The open session */ }; -#define getrusage(A,B) memset(B,0,sizeof(*B)) #endif +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) +typedef struct ExpertInfo ExpertInfo; +struct ExpertInfo { + sqlite3expert *pExpert; + int bVerbose; +}; +#endif -/* Saved resource information for the beginning of an operation */ -static struct rusage sBegin; /* CPU time at start */ -static sqlite3_int64 iBegin; /* Wall-clock time at start */ - -/* -** Begin timing an operation +/* All the parameters that determine how to render query results. */ -static void beginTimer(void){ - if( enableTimer ){ - getrusage(RUSAGE_SELF, &sBegin); - iBegin = timeOfDay(); - } -} +typedef struct Mode { + u8 autoExplain; /* Automatically turn on .explain mode */ + u8 autoEQP; /* Run EXPLAIN QUERY PLAN prior to each SQL stmt */ + u8 autoEQPtrace; /* autoEQP is in trace mode */ + u8 scanstatsOn; /* True to display scan stats before each finalize */ + u8 bAutoScreenWidth; /* Using the TTY to determine screen width */ + u8 mFlags; /* MFLG_ECHO, MFLG_CRLF, etc. */ + u8 eMode; /* One of the MODE_ values */ + sqlite3_qrf_spec spec; /* Spec to be passed into QRF */ +} Mode; + +/* Flags for Mode.mFlags */ +#define MFLG_ECHO 0x01 /* Echo inputs to output */ +#define MFLG_CRLF 0x02 /* Use CR/LF output line endings */ +#define MFLG_HDR 0x04 /* .header used to change headers on/off */ -/* Return the difference of two time_structs in seconds */ -static double timeDiff(struct timeval *pStart, struct timeval *pEnd){ - return (pEnd->tv_usec - pStart->tv_usec)*0.000001 + - (double)(pEnd->tv_sec - pStart->tv_sec); -} /* -** Print the timing results. +** State information about the database connection is contained in an +** instance of the following structure. */ -static void endTimer(FILE *out){ - if( enableTimer ){ - sqlite3_int64 iEnd = timeOfDay(); - struct rusage sEnd; - getrusage(RUSAGE_SELF, &sEnd); - sqlite3_fprintf(out, "Run Time: real %.6f user %.6f sys %.6f\n", - (iEnd - iBegin)*0.000001, - timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), - timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); - } -} +typedef struct ShellState ShellState; +struct ShellState { + sqlite3 *db; /* The database */ + u8 openMode; /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */ + u8 doXdgOpen; /* Invoke start/open/xdg-open in output_reset() */ + u8 nEqpLevel; /* Depth of the EQP output graph */ + u8 eTraceType; /* SHELL_TRACE_* value for type of trace */ + u8 bSafeMode; /* True to prohibit unsafe operations */ + u8 bSafeModePersist; /* The long-term value of bSafeMode */ + u8 eRestoreState; /* See comments above doAutoDetectRestore() */ + unsigned statsOn; /* True to display memory stats before each finalize */ + unsigned mEqpLines; /* Mask of vertical lines in the EQP output graph */ + u8 nPopOutput; /* Revert .output settings when reaching zero */ + u8 nPopMode; /* Revert .mode settings when reaching zero */ + u8 enableTimer; /* Enable the timer. 2: permanently 1: only once */ + int inputNesting; /* Track nesting level of .read and other redirects */ + double prevTimer; /* Last reported timer value */ + double tmProgress; /* --timeout option for .progress */ + i64 lineno; /* Line number of last line read from in */ + const char *zInFile; /* Name of the input file */ + int openFlags; /* Additional flags to open. (SQLITE_OPEN_NOFOLLOW) */ + FILE *in; /* Read commands from this stream */ + FILE *out; /* Write results here */ + FILE *traceOut; /* Output for sqlite3_trace() */ + int nErr; /* Number of errors seen */ + int writableSchema; /* True if PRAGMA writable_schema=ON */ + int nCheck; /* Number of ".check" commands run */ + unsigned nProgress; /* Number of progress callbacks encountered */ + unsigned mxProgress; /* Maximum progress callbacks before failing */ + unsigned flgProgress; /* Flags for the progress callback */ + unsigned shellFlgs; /* Various flags */ + unsigned nTestRun; /* Number of test cases run */ + unsigned nTestErr; /* Number of test cases that failed */ + sqlite3_int64 szMax; /* --maxsize argument to .open */ + char *zDestTable; /* Name of destination table when MODE_Insert */ + char *zTempFile; /* Temporary file that might need deleting */ + char *zErrPrefix; /* Alternative error message prefix */ + char zTestcase[30]; /* Name of current test case */ + char outfile[FILENAME_MAX]; /* Filename for *out */ + sqlite3_stmt *pStmt; /* Current statement if any. */ + FILE *pLog; /* Write log output here */ + Mode mode; /* Current display mode */ + Mode modePrior; /* Backup */ + struct SavedMode { /* Ability to define custom mode configurations */ + char *zTag; /* Name of this saved mode */ + Mode mode; /* The saved mode */ + } *aSavedModes; /* Array of saved .mode settings. system malloc() */ + int nSavedModes; /* Number of saved .mode settings */ + struct AuxDb { /* Storage space for auxiliary database connections */ + sqlite3 *db; /* Connection pointer */ + const char *zDbFilename; /* Filename used to open the connection */ + char *zFreeOnClose; /* Free this memory allocation on close */ +#if defined(SQLITE_ENABLE_SESSION) + int nSession; /* Number of active sessions */ + OpenSession aSession[4]; /* Array of sessions. [0] is in focus. */ +#endif + } aAuxDb[5], /* Array of all database connections */ + *pAuxDb; /* Currently active database connection */ + char *zNonce; /* Nonce for temporary safe-mode escapes */ +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) + ExpertInfo expert; /* Valid if previous command was ".expert OPT..." */ +#endif + struct DotCmdLine { /* Info about arguments to a dot-command */ + const char *zOrig; /* Original text of the dot-command */ + char *zCopy; /* Copy of zOrig, from malloc() */ + int nAlloc; /* Size of allocates for arrays below */ + int nArg; /* Number of argument slots actually used */ + char **azArg; /* Pointer to each argument, dequoted */ + int *aiOfst; /* Offset into zOrig[] for start of each arg */ + char *abQuot; /* True if the argment was originally quoted */ + } dot; +#ifdef SQLITE_SHELL_FIDDLE + struct { + const char * zInput; /* Input string from wasm/JS proxy */ + const char * zPos; /* Cursor pos into zInput */ + const char * zDefaultDbName; /* Default name for db file */ + } wasm; +#endif +}; -#define BEGIN_TIMER beginTimer() -#define END_TIMER(X) endTimer(X) -#define HAS_TIMER 1 +#ifdef SQLITE_SHELL_FIDDLE +static ShellState shellState; +#endif -#elif (defined(_WIN32) || defined(WIN32)) -/* Saved resource information for the beginning of an operation */ -static HANDLE hProcess; -static FILETIME ftKernelBegin; -static FILETIME ftUserBegin; -static sqlite3_int64 ftWallBegin; -typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME, - LPFILETIME, LPFILETIME); -static GETPROCTIMES getProcessTimesAddr = NULL; +/* Allowed values for ShellState.mode.autoEQP +*/ +#define AUTOEQP_off 0 /* Automatic EXPLAIN QUERY PLAN is off */ +#define AUTOEQP_on 1 /* Automatic EQP is on */ +#define AUTOEQP_trigger 2 /* On and also show plans for triggers */ +#define AUTOEQP_full 3 /* Show full EXPLAIN */ -/* -** Check to see if we have timer support. Return 1 if necessary -** support found (or found previously). +/* Allowed values for ShellState.openMode */ -static int hasTimer(void){ - if( getProcessTimesAddr ){ - return 1; - } else { -#if !SQLITE_OS_WINRT - /* GetProcessTimes() isn't supported in WIN95 and some other Windows - ** versions. See if the version we are running on has it, and if it - ** does, save off a pointer to it and the current process handle. - */ - hProcess = GetCurrentProcess(); - if( hProcess ){ - HINSTANCE hinstLib = LoadLibrary(TEXT("Kernel32.dll")); - if( NULL != hinstLib ){ - getProcessTimesAddr = - (GETPROCTIMES) GetProcAddress(hinstLib, "GetProcessTimes"); - if( NULL != getProcessTimesAddr ){ - return 1; - } - FreeLibrary(hinstLib); - } - } -#endif - } - return 0; -} +#define SHELL_OPEN_UNSPEC 0 /* No open-mode specified */ +#define SHELL_OPEN_NORMAL 1 /* Normal database file */ +#define SHELL_OPEN_APPENDVFS 2 /* Use appendvfs */ +#define SHELL_OPEN_ZIPFILE 3 /* Use the zipfile virtual table */ +#define SHELL_OPEN_DESERIALIZE 4 /* Open using sqlite3_deserialize() */ +#define SHELL_OPEN_HEXDB 5 /* Use "dbtotxt" output as data source */ -/* -** Begin timing an operation +/* Allowed values for ShellState.eTraceType */ -static void beginTimer(void){ - if( enableTimer && getProcessTimesAddr ){ - FILETIME ftCreation, ftExit; - getProcessTimesAddr(hProcess,&ftCreation,&ftExit, - &ftKernelBegin,&ftUserBegin); - ftWallBegin = timeOfDay(); - } -} +#define SHELL_TRACE_PLAIN 0 /* Show input SQL text */ +#define SHELL_TRACE_EXPANDED 1 /* Show expanded SQL text */ +#define SHELL_TRACE_NORMALIZED 2 /* Show normalized SQL text */ -/* Return the difference of two FILETIME structs in seconds */ -static double timeDiff(FILETIME *pStart, FILETIME *pEnd){ - sqlite_int64 i64Start = *((sqlite_int64 *) pStart); - sqlite_int64 i64End = *((sqlite_int64 *) pEnd); - return (double) ((i64End - i64Start) / 10000000.0); -} +/* Bits in the ShellState.flgProgress variable */ +#define SHELL_PROGRESS_QUIET 0x01 /* Omit announcing every progress callback */ +#define SHELL_PROGRESS_RESET 0x02 /* Reset the count when the progress + ** callback limit is reached, and for each + ** top-level SQL statement */ +#define SHELL_PROGRESS_ONCE 0x04 /* Cancel the --limit after firing once */ +#define SHELL_PROGRESS_TMOUT 0x08 /* Stop after tmProgress seconds */ + +/* Names of values for Mode.spec.eEsc and Mode.spec.eText +*/ +static const char *qrfEscNames[] = { "auto", "off", "ascii", "symbol" }; +static const char *qrfQuoteNames[] = + { "off","off","sql","hex","csv","tcl","json","relaxed"}; /* -** Print the timing results. +** These are the allowed shellFlgs values */ -static void endTimer(FILE *out){ - if( enableTimer && getProcessTimesAddr){ - FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; - sqlite3_int64 ftWallEnd = timeOfDay(); - getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); -#ifdef _WIN64 - /* microsecond precision on 64-bit windows */ - sqlite3_fprintf(out, "Run Time: real %.6f user %f sys %f\n", - (ftWallEnd - ftWallBegin)*0.000001, - timeDiff(&ftUserBegin, &ftUserEnd), - timeDiff(&ftKernelBegin, &ftKernelEnd)); -#else - /* millisecond precisino on 32-bit windows */ - sqlite3_fprintf(out, "Run Time: real %.3f user %.3f sys %.3f\n", - (ftWallEnd - ftWallBegin)*0.000001, - timeDiff(&ftUserBegin, &ftUserEnd), - timeDiff(&ftKernelBegin, &ftKernelEnd)); -#endif - } -} +#define SHFLG_Pagecache 0x00000001 /* The --pagecache option is used */ +#define SHFLG_Lookaside 0x00000002 /* Lookaside memory is used */ +#define SHFLG_Backslash 0x00000004 /* The --backslash option is used */ +#define SHFLG_PreserveRowid 0x00000008 /* .dump preserves rowid values */ +#define SHFLG_NoErrLineno 0x00000010 /* Omit line numbers from error msgs */ +#define SHFLG_CountChanges 0x00000020 /* .changes setting */ +#define SHFLG_DumpDataOnly 0x00000100 /* .dump show data only */ +#define SHFLG_DumpNoSys 0x00000200 /* .dump omits system tables */ +#define SHFLG_TestingMode 0x00000400 /* allow unsafe testing features */ -#define BEGIN_TIMER beginTimer() -#define END_TIMER(X) endTimer(X) -#define HAS_TIMER hasTimer() +/* +** Macros for testing and setting shellFlgs +*/ +#define ShellHasFlag(P,X) (((P)->shellFlgs & (X))!=0) +#define ShellSetFlag(P,X) ((P)->shellFlgs|=(X)) +#define ShellClearFlag(P,X) ((P)->shellFlgs&=(~(X))) -#else -#define BEGIN_TIMER -#define END_TIMER(X) /*no-op*/ -#define HAS_TIMER 0 -#endif +/* +** These are the allowed values for Mode.eMode. There is a lot of overlap +** between these values and the Mode.spec.eStyle values, but they are not +** one-to-one, and thus need to be tracked separately. +*/ +#define MODE_Ascii 0 /* Use ASCII unit and record separators (0x1F/0x1E) */ +#define MODE_Box 1 /* Unicode box-drawing characters */ +#define MODE_C 2 /* Comma-separated list of C-strings */ +#define MODE_Column 3 /* One record per line in neat columns */ +#define MODE_Count 4 /* Output only a count of the rows of output */ +#define MODE_Csv 5 /* Quote strings, numbers are plain */ +#define MODE_Html 6 /* Generate an XHTML table */ +#define MODE_Insert 7 /* Generate SQL "insert" statements */ +#define MODE_JAtom 8 /* Comma-separated list of JSON atoms */ +#define MODE_JObject 9 /* One JSON object per row */ +#define MODE_Json 10 /* Output JSON */ +#define MODE_Line 11 /* One column per line. Blank line between records */ +#define MODE_List 12 /* One record per line with a separator */ +#define MODE_Markdown 13 /* Markdown formatting */ +#define MODE_Off 14 /* No query output shown */ +#define MODE_Psql 15 /* Similar to psql */ +#define MODE_QBox 16 /* BOX with SQL-quoted content */ +#define MODE_Quote 17 /* Quote values as for SQL */ +#define MODE_Split 18 /* Split-column mode */ +#define MODE_Table 19 /* MySQL-style table formatting */ +#define MODE_Tabs 20 /* Tab-separated values */ +#define MODE_Tcl 21 /* Space-separated list of TCL strings */ +#define MODE_Www 22 /* Full web-page output */ + +#define MODE_BUILTIN 22 /* Maximum built-in mode */ +#define MODE_BATCH 50 /* Default mode for batch processing */ +#define MODE_TTY 51 /* Default mode for interactive processing */ +#define MODE_USER 75 /* First user-defined mode */ +#define MODE_N_USER 25 /* Maximum number of user-defined modes */ + +/* +** Information about built-in display modes +*/ +typedef struct ModeInfo ModeInfo; +struct ModeInfo { + char zName[9]; /* Symbolic name of the mode */ + unsigned char eCSep; /* Column separator */ + unsigned char eRSep; /* Row separator */ + unsigned char eNull; /* Null representation */ + unsigned char eText; /* Default text encoding */ + unsigned char eHdr; /* Default header encoding. */ + unsigned char eBlob; /* Default blob encoding. */ + unsigned char bHdr; /* Show headers by default. 0: n/a, 1: no 2: yes */ + unsigned char eStyle; /* Underlying QRF style */ + unsigned char eCx; /* 0: other, 1: line, 2: columnar */ + unsigned char mFlg; /* Flags. 1=border-off 2=split-column */ +}; +/* String constants used by built-in modes */ +static const char *aModeStr[] = + /* 0 1 2 3 4 5 6 7 8 */ + { 0, "\n", "|", " ", ",", "\r\n", "\036", "\037", "\t", + "", "NULL", "null", "\"\"", ": ", }; + /* 9 10 11 12 13 */ + +static const ModeInfo aModeInfo[] = { +/* zName eCSep eRSep eNull eText eHdr eBlob bHdr eStyle eCx mFlg */ + { "ascii", 7, 6, 9, 1, 1, 0, 1, 12, 0, 0 }, + { "box", 0, 0, 9, 1, 1, 0, 2, 1, 2, 0 }, + { "c", 4, 1, 10, 5, 5, 4, 1, 12, 0, 0 }, + { "column", 0, 0, 9, 1, 1, 0, 2, 2, 2, 0 }, + { "count", 0, 0, 0, 0, 0, 0, 0, 3, 0, 0 }, + { "csv", 4, 5, 9, 3, 3, 0, 1, 12, 0, 0 }, + { "html", 0, 0, 9, 4, 4, 0, 2, 7, 0, 0 }, + { "insert", 0, 0, 10, 2, 2, 0, 1, 8, 0, 0 }, + { "jatom", 4, 1, 11, 6, 6, 0, 1, 12, 0, 0 }, + { "jobject", 0, 1, 11, 6, 6, 0, 0, 10, 0, 0 }, + { "json", 0, 0, 11, 6, 6, 0, 0, 9, 0, 0 }, + { "line", 13, 1, 9, 1, 1, 0, 0, 11, 1, 0 }, + { "list", 2, 1, 9, 1, 1, 0, 1, 12, 0, 0 }, + { "markdown", 0, 0, 9, 1, 1, 0, 2, 13, 2, 0 }, + { "off", 0, 0, 0, 0, 0, 0, 0, 14, 0, 0 }, + { "psql", 0, 0, 9, 1, 1, 0, 2, 19, 2, 1 }, + { "qbox", 0, 0, 10, 2, 1, 0, 2, 1, 2, 0 }, + { "quote", 4, 1, 10, 2, 2, 0, 1, 12, 0, 0 }, + { "split", 0, 0, 9, 1, 1, 0, 1, 2, 2, 2 }, + { "table", 0, 0, 9, 1, 1, 0, 2, 19, 2, 0 }, + { "tabs", 8, 1, 9, 3, 3, 0, 1, 12, 0, 0 }, + { "tcl", 3, 1, 12, 5, 5, 4, 1, 12, 0, 0 }, + { "www", 0, 0, 9, 4, 4, 0, 2, 7, 0, 0 } +}; /* | / / | / / | | \ + ** | / / | / / | | \_ 2: columnar + ** Index into aModeStr[] | / / | | 1: line + ** | / / | | 0: other + ** | / / | \ + ** text encoding |/ | show | \ + ** v-------------------' | hdrs? | The QRF style + ** 0: n/a blob | v-----' + ** 1: plain v_---------' 0: n/a + ** 2: sql 0: auto 1: no + ** 3: csv 1: as-text 2: yes + ** 4: html 2: sql + ** 5: c 3: hex + ** 6: json 4: c + ** 5: json + ** 6: size + ******************************************************************/ /* -** Used to prevent warnings about unused parameters +** These are the column/row/line separators used by the various +** import/export modes. */ -#define UNUSED_PARAMETER(x) (void)(x) +#define SEP_Column "|" +#define SEP_Row "\n" +#define SEP_Tab "\t" +#define SEP_Space " " +#define SEP_Comma "," +#define SEP_CrLf "\r\n" +#define SEP_Unit "\x1F" +#define SEP_Record "\x1E" /* -** Number of elements in an array +** Default values for the various QRF limits */ -#define ArraySize(X) (int)(sizeof(X)/sizeof(X[0])) +#ifndef DFLT_CHAR_LIMIT +# define DFLT_CHAR_LIMIT 300 +#endif +#ifndef DFLT_LINE_LIMIT +# define DFLT_LINE_LIMIT 5 +#endif +#ifndef DFLT_TITLE_LIMIT +# define DFLT_TITLE_LIMIT 20 +#endif +#ifndef DFLT_MULTI_INSERT +# define DFLT_MULTI_INSERT 3000 +#endif /* ** If the following flag is set, then command execution stops @@ -472,12 +645,17 @@ static int bail_on_error = 0; static int stdin_is_interactive = 1; /* -** On Windows systems we need to know if standard output is a console -** in order to show that UTF-16 translation is done in the sign-on -** banner. The following variable is true if it is the console. +** Treat stdout like a TTY if true. */ static int stdout_is_console = 1; +/* +** Use this value as the width of the output device. Or, figure it +** out at runtime if the value is negative. Or use a default width +** if this value is zero. +*/ +static int stdout_tty_width = -1; + /* ** The following is the open SQLite database. We make a pointer ** to this database a static variable so that it can be accessed @@ -506,6 +684,121 @@ static char mainPrompt[PROMPT_LEN_MAX]; /* Continuation prompt. default: " ...> " */ static char continuePrompt[PROMPT_LEN_MAX]; +/* +** Write I/O traces to the following stream. +*/ +#ifdef SQLITE_ENABLE_IOTRACE +static FILE *iotrace = 0; +#endif + +/* +** Output routines that are able to redirect to memory rather than +** doing actually I/O. +** Works like. +** -------------- +** cli_printf(FILE*, const char*, ...); fprintf() +** cli_puts(const char*, FILE*); fputs() +** cli_vprintf(FILE*, const char*, va_list); vfprintf() +** +** These are just thin wrappers with the following added semantics: +** If the file-scope variable cli_output_capture is not NULL, and +** if the FILE* argument is stdout or stderr, then rather than +** writing to stdout/stdout, append the text to the cli_output_capture +** variable. +** +** The cli_exit(int) routine works like exit() except that it +** first dumps any capture output to stdout. +*/ +static sqlite3_str *cli_output_capture = 0; +static int cli_printf(FILE *out, const char *zFormat, ...){ + va_list ap; + int rc; + va_start(ap,zFormat); + if( cli_output_capture && (out==stdout || out==stderr) ){ + sqlite3_str_vappendf(cli_output_capture, zFormat, ap); + rc = 1; + }else{ + rc = sqlite3_vfprintf(out, zFormat, ap); + } + va_end(ap); + return rc; +} +static int cli_puts(const char *zText, FILE *out){ + if( cli_output_capture && (out==stdout || out==stderr) ){ + sqlite3_str_appendall(cli_output_capture, zText); + return 1; + } + return sqlite3_fputs(zText, out); +} +#if 0 /* Not currently used - available if we need it later */ +static int cli_vprintf(FILE *out, const char *zFormat, va_list ap){ + if( cli_output_capture && (out==stdout || out==stderr) ){ + sqlite3_str_vappendf(cli_output_capture, zFormat, ap); + return 1; + }else{ + return sqlite3_vfprintf(out, zFormat, ap); + } +} +#endif +static void cli_exit(int rc){ + if( cli_output_capture ){ + char *z = sqlite3_str_finish(cli_output_capture); + sqlite3_fputs(z, stdout); + fflush(stdout); + } + exit(rc); +} + + +#define eputz(z) cli_puts(z,stderr) +#define sputz(fp,z) cli_puts(z,fp) + +/* A version of strcmp() that works with NULL values */ +static int cli_strcmp(const char *a, const char *b){ + if( a==0 ) a = ""; + if( b==0 ) b = ""; + return strcmp(a,b); +} +static int cli_strncmp(const char *a, const char *b, size_t n){ + if( a==0 ) a = ""; + if( b==0 ) b = ""; + return strncmp(a,b,n); +} + +/* Return the current wall-clock time in microseconds since the +** Unix epoch (1970-01-01T00:00:00Z) +*/ +static sqlite3_int64 timeOfDay(void){ +#if defined(_WIN64) && _WIN32_WINNT >= _WIN32_WINNT_WIN8 + sqlite3_uint64 t; + FILETIME tm; + GetSystemTimePreciseAsFileTime(&tm); + t = ((u64)tm.dwHighDateTime<<32) | (u64)tm.dwLowDateTime; + t += 116444736000000000LL; + t /= 10; + return t; +#elif defined(_WIN32) + static sqlite3_vfs *clockVfs = 0; + sqlite3_int64 t; + if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); + if( clockVfs==0 ) return 0; /* Never actually happens */ + if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){ + clockVfs->xCurrentTimeInt64(clockVfs, &t); + }else{ + double r; + clockVfs->xCurrentTime(clockVfs, &r); + t = (sqlite3_int64)(r*86400000.0); + } + return t*1000; +#else + struct timeval sNow; + (void)gettimeofday(&sNow,0); + return ((i64)sNow.tv_sec)*1000000 + sNow.tv_usec; +#endif +} + + + /* This is variant of the standard-library strncpy() routine with the ** one change that the destination string is always zero-terminated, even ** if there is no zero-terminator in the first n-1 characters of the source @@ -615,7 +908,7 @@ static char *dynamicContinuePrompt(void){ /* Indicate out-of-memory and exit. */ static void shell_out_of_memory(void){ eputz("Error: out of memory\n"); - exit(1); + cli_exit(1); } /* Check a pointer to see if it is NULL. If it is NULL, exit with an @@ -625,13 +918,6 @@ static void shell_check_oom(const void *p){ if( p==0 ) shell_out_of_memory(); } -/* -** Write I/O traces to the following stream. -*/ -#ifdef SQLITE_ENABLE_IOTRACE -static FILE *iotrace = 0; -#endif - /* ** This routine works like printf in that its first argument is a ** format string and subsequent arguments are values to be substituted @@ -646,349 +932,67 @@ static void SQLITE_CDECL iotracePrintf(const char *zFormat, ...){ va_start(ap, zFormat); z = sqlite3_vmprintf(zFormat, ap); va_end(ap); - sqlite3_fprintf(iotrace, "%s", z); + cli_printf(iotrace, "%s", z); sqlite3_free(z); } #endif -/* Lookup table to estimate the number of columns consumed by a Unicode -** character. -*/ -static const struct { - unsigned char w; /* Width of the character in columns */ - int iFirst; /* First character in a span having this width */ -} aUWidth[] = { - /* {1, 0x00000}, */ - {0, 0x00300}, {1, 0x00370}, {0, 0x00483}, {1, 0x00487}, {0, 0x00488}, - {1, 0x0048a}, {0, 0x00591}, {1, 0x005be}, {0, 0x005bf}, {1, 0x005c0}, - {0, 0x005c1}, {1, 0x005c3}, {0, 0x005c4}, {1, 0x005c6}, {0, 0x005c7}, - {1, 0x005c8}, {0, 0x00600}, {1, 0x00604}, {0, 0x00610}, {1, 0x00616}, - {0, 0x0064b}, {1, 0x0065f}, {0, 0x00670}, {1, 0x00671}, {0, 0x006d6}, - {1, 0x006e5}, {0, 0x006e7}, {1, 0x006e9}, {0, 0x006ea}, {1, 0x006ee}, - {0, 0x0070f}, {1, 0x00710}, {0, 0x00711}, {1, 0x00712}, {0, 0x00730}, - {1, 0x0074b}, {0, 0x007a6}, {1, 0x007b1}, {0, 0x007eb}, {1, 0x007f4}, - {0, 0x00901}, {1, 0x00903}, {0, 0x0093c}, {1, 0x0093d}, {0, 0x00941}, - {1, 0x00949}, {0, 0x0094d}, {1, 0x0094e}, {0, 0x00951}, {1, 0x00955}, - {0, 0x00962}, {1, 0x00964}, {0, 0x00981}, {1, 0x00982}, {0, 0x009bc}, - {1, 0x009bd}, {0, 0x009c1}, {1, 0x009c5}, {0, 0x009cd}, {1, 0x009ce}, - {0, 0x009e2}, {1, 0x009e4}, {0, 0x00a01}, {1, 0x00a03}, {0, 0x00a3c}, - {1, 0x00a3d}, {0, 0x00a41}, {1, 0x00a43}, {0, 0x00a47}, {1, 0x00a49}, - {0, 0x00a4b}, {1, 0x00a4e}, {0, 0x00a70}, {1, 0x00a72}, {0, 0x00a81}, - {1, 0x00a83}, {0, 0x00abc}, {1, 0x00abd}, {0, 0x00ac1}, {1, 0x00ac6}, - {0, 0x00ac7}, {1, 0x00ac9}, {0, 0x00acd}, {1, 0x00ace}, {0, 0x00ae2}, - {1, 0x00ae4}, {0, 0x00b01}, {1, 0x00b02}, {0, 0x00b3c}, {1, 0x00b3d}, - {0, 0x00b3f}, {1, 0x00b40}, {0, 0x00b41}, {1, 0x00b44}, {0, 0x00b4d}, - {1, 0x00b4e}, {0, 0x00b56}, {1, 0x00b57}, {0, 0x00b82}, {1, 0x00b83}, - {0, 0x00bc0}, {1, 0x00bc1}, {0, 0x00bcd}, {1, 0x00bce}, {0, 0x00c3e}, - {1, 0x00c41}, {0, 0x00c46}, {1, 0x00c49}, {0, 0x00c4a}, {1, 0x00c4e}, - {0, 0x00c55}, {1, 0x00c57}, {0, 0x00cbc}, {1, 0x00cbd}, {0, 0x00cbf}, - {1, 0x00cc0}, {0, 0x00cc6}, {1, 0x00cc7}, {0, 0x00ccc}, {1, 0x00cce}, - {0, 0x00ce2}, {1, 0x00ce4}, {0, 0x00d41}, {1, 0x00d44}, {0, 0x00d4d}, - {1, 0x00d4e}, {0, 0x00dca}, {1, 0x00dcb}, {0, 0x00dd2}, {1, 0x00dd5}, - {0, 0x00dd6}, {1, 0x00dd7}, {0, 0x00e31}, {1, 0x00e32}, {0, 0x00e34}, - {1, 0x00e3b}, {0, 0x00e47}, {1, 0x00e4f}, {0, 0x00eb1}, {1, 0x00eb2}, - {0, 0x00eb4}, {1, 0x00eba}, {0, 0x00ebb}, {1, 0x00ebd}, {0, 0x00ec8}, - {1, 0x00ece}, {0, 0x00f18}, {1, 0x00f1a}, {0, 0x00f35}, {1, 0x00f36}, - {0, 0x00f37}, {1, 0x00f38}, {0, 0x00f39}, {1, 0x00f3a}, {0, 0x00f71}, - {1, 0x00f7f}, {0, 0x00f80}, {1, 0x00f85}, {0, 0x00f86}, {1, 0x00f88}, - {0, 0x00f90}, {1, 0x00f98}, {0, 0x00f99}, {1, 0x00fbd}, {0, 0x00fc6}, - {1, 0x00fc7}, {0, 0x0102d}, {1, 0x01031}, {0, 0x01032}, {1, 0x01033}, - {0, 0x01036}, {1, 0x01038}, {0, 0x01039}, {1, 0x0103a}, {0, 0x01058}, - {1, 0x0105a}, {2, 0x01100}, {0, 0x01160}, {1, 0x01200}, {0, 0x0135f}, - {1, 0x01360}, {0, 0x01712}, {1, 0x01715}, {0, 0x01732}, {1, 0x01735}, - {0, 0x01752}, {1, 0x01754}, {0, 0x01772}, {1, 0x01774}, {0, 0x017b4}, - {1, 0x017b6}, {0, 0x017b7}, {1, 0x017be}, {0, 0x017c6}, {1, 0x017c7}, - {0, 0x017c9}, {1, 0x017d4}, {0, 0x017dd}, {1, 0x017de}, {0, 0x0180b}, - {1, 0x0180e}, {0, 0x018a9}, {1, 0x018aa}, {0, 0x01920}, {1, 0x01923}, - {0, 0x01927}, {1, 0x01929}, {0, 0x01932}, {1, 0x01933}, {0, 0x01939}, - {1, 0x0193c}, {0, 0x01a17}, {1, 0x01a19}, {0, 0x01b00}, {1, 0x01b04}, - {0, 0x01b34}, {1, 0x01b35}, {0, 0x01b36}, {1, 0x01b3b}, {0, 0x01b3c}, - {1, 0x01b3d}, {0, 0x01b42}, {1, 0x01b43}, {0, 0x01b6b}, {1, 0x01b74}, - {0, 0x01dc0}, {1, 0x01dcb}, {0, 0x01dfe}, {1, 0x01e00}, {0, 0x0200b}, - {1, 0x02010}, {0, 0x0202a}, {1, 0x0202f}, {0, 0x02060}, {1, 0x02064}, - {0, 0x0206a}, {1, 0x02070}, {0, 0x020d0}, {1, 0x020f0}, {2, 0x02329}, - {1, 0x0232b}, {2, 0x02e80}, {0, 0x0302a}, {2, 0x03030}, {1, 0x0303f}, - {2, 0x03040}, {0, 0x03099}, {2, 0x0309b}, {1, 0x0a4d0}, {0, 0x0a806}, - {1, 0x0a807}, {0, 0x0a80b}, {1, 0x0a80c}, {0, 0x0a825}, {1, 0x0a827}, - {2, 0x0ac00}, {1, 0x0d7a4}, {2, 0x0f900}, {1, 0x0fb00}, {0, 0x0fb1e}, - {1, 0x0fb1f}, {0, 0x0fe00}, {2, 0x0fe10}, {1, 0x0fe1a}, {0, 0x0fe20}, - {1, 0x0fe24}, {2, 0x0fe30}, {1, 0x0fe70}, {0, 0x0feff}, {2, 0x0ff00}, - {1, 0x0ff61}, {2, 0x0ffe0}, {1, 0x0ffe7}, {0, 0x0fff9}, {1, 0x0fffc}, - {0, 0x10a01}, {1, 0x10a04}, {0, 0x10a05}, {1, 0x10a07}, {0, 0x10a0c}, - {1, 0x10a10}, {0, 0x10a38}, {1, 0x10a3b}, {0, 0x10a3f}, {1, 0x10a40}, - {0, 0x1d167}, {1, 0x1d16a}, {0, 0x1d173}, {1, 0x1d183}, {0, 0x1d185}, - {1, 0x1d18c}, {0, 0x1d1aa}, {1, 0x1d1ae}, {0, 0x1d242}, {1, 0x1d245}, - {2, 0x20000}, {1, 0x2fffe}, {2, 0x30000}, {1, 0x3fffe}, {0, 0xe0001}, - {1, 0xe0002}, {0, 0xe0020}, {1, 0xe0080}, {0, 0xe0100}, {1, 0xe01f0} -}; - /* -** Return an estimate of the width, in columns, for the single Unicode -** character c. For normal characters, the answer is always 1. But the -** estimate might be 0 or 2 for zero-width and double-width characters. -** -** Different display devices display unicode using different widths. So -** it is impossible to know that true display width with 100% accuracy. -** Inaccuracies in the width estimates might cause columns to be misaligned. -** Unfortunately, there is nothing we can do about that. -*/ -int cli_wcwidth(int c){ - int iFirst, iLast; - - /* Fast path for common characters */ - if( c<=0x300 ) return 1; - - /* The general case */ - iFirst = 0; - iLast = sizeof(aUWidth)/sizeof(aUWidth[0]) - 1; - while( iFirst<iLast-1 ){ - int iMid = (iFirst+iLast)/2; - int cMid = aUWidth[iMid].iFirst; - if( cMid < c ){ - iFirst = iMid; - }else if( cMid > c ){ - iLast = iMid - 1; - }else{ - return aUWidth[iMid].w; - } - } - if( aUWidth[iLast].iFirst > c ) return aUWidth[iFirst].w; - return aUWidth[iLast].w; +** Compute a string length that is limited to what can be stored in +** lower 30 bits of a 32-bit signed integer. +*/ +static int strlen30(const char *z){ + size_t n; + if( z==0 ) return 0; + n = strlen(z); + return n>0x3fffffff ? 0x3fffffff : (int)n; } /* -** Compute the value and length of a multi-byte UTF-8 character that -** begins at z[0]. Return the length. Write the Unicode value into *pU. -** -** This routine only works for *multi-byte* UTF-8 characters. +** Return open FILE * if zFile exists, can be opened for read +** and is an ordinary file or a character stream source. +** Otherwise return 0. */ -static int decodeUtf8(const unsigned char *z, int *pU){ - if( (z[0] & 0xe0)==0xc0 && (z[1] & 0xc0)==0x80 ){ - *pU = ((z[0] & 0x1f)<<6) | (z[1] & 0x3f); - return 2; - } - if( (z[0] & 0xf0)==0xe0 && (z[1] & 0xc0)==0x80 && (z[2] & 0xc0)==0x80 ){ - *pU = ((z[0] & 0x0f)<<12) | ((z[1] & 0x3f)<<6) | (z[2] & 0x3f); - return 3; +static FILE * openChrSource(const char *zFile){ +#if defined(_WIN32) || defined(WIN32) + struct __stat64 x = {0}; +# define STAT_CHR_SRC(mode) ((mode & (_S_IFCHR|_S_IFIFO|_S_IFREG))!=0) + /* On Windows, open first, then check the stream nature. This order + ** is necessary because _stat() and sibs, when checking a named pipe, + ** effectively break the pipe as its supplier sees it. */ + FILE *rv = sqlite3_fopen(zFile, "rb"); + if( rv==0 ) return 0; + if( _fstat64(_fileno(rv), &x) != 0 + || !STAT_CHR_SRC(x.st_mode)){ + fclose(rv); + rv = 0; } - if( (z[0] & 0xf8)==0xf0 && (z[1] & 0xc0)==0x80 && (z[2] & 0xc0)==0x80 - && (z[3] & 0xc0)==0x80 - ){ - *pU = ((z[0] & 0x0f)<<18) | ((z[1] & 0x3f)<<12) | ((z[2] & 0x3f))<<6 - | (z[3] & 0x3f); - return 4; + return rv; +#else + struct stat x = {0}; + int rc = stat(zFile, &x); +# define STAT_CHR_SRC(mode) (S_ISREG(mode)||S_ISFIFO(mode)||S_ISCHR(mode)) + if( rc!=0 ) return 0; + if( STAT_CHR_SRC(x.st_mode) ){ + return sqlite3_fopen(zFile, "rb"); + }else{ + return 0; } - *pU = 0; - return 1; +#endif +#undef STAT_CHR_SRC } - -#if 0 /* NOT USED */ /* -** Return the width, in display columns, of a UTF-8 string. +** This routine reads a line of text from FILE in, stores +** the text in memory obtained from malloc() and returns a pointer +** to the text. NULL is returned at end of file, or if malloc() +** fails, or if the length of the line is longer than about a gigabyte. ** -** Each normal character counts as 1. Zero-width characters count -** as zero, and double-width characters count as 2. +** If zLine is not NULL then it is a malloced buffer returned from +** a previous call to this routine that may be reused. */ -int cli_wcswidth(const char *z){ - const unsigned char *a = (const unsigned char*)z; - int n = 0; - int i = 0; - unsigned char c; - while( (c = a[i])!=0 ){ - if( c>=0xc0 ){ - int u; - int len = decodeUtf8(&a[i], &u); - i += len; - n += cli_wcwidth(u); - }else if( c>=' ' ){ - n++; - i++; - }else{ - i++; - } - } - return n; -} -#endif - -/* -** Check to see if z[] is a valid VT100 escape. If it is, then -** return the number of bytes in the escape sequence. Return 0 if -** z[] is not a VT100 escape. -** -** This routine assumes that z[0] is \033 (ESC). -*/ -static int isVt100(const unsigned char *z){ - int i; - if( z[1]!='[' ) return 0; - i = 2; - while( z[i]>=0x30 && z[i]<=0x3f ){ i++; } - while( z[i]>=0x20 && z[i]<=0x2f ){ i++; } - if( z[i]<0x40 || z[i]>0x7e ) return 0; - return i+1; -} - -/* -** Output string zUtf to stdout as w characters. If w is negative, -** then right-justify the text. W is the width in UTF-8 characters, not -** in bytes. This is different from the %*.*s specification in printf -** since with %*.*s the width is measured in bytes, not characters. -** -** Take into account zero-width and double-width Unicode characters. -** In other words, a zero-width character does not count toward the -** the w limit. A double-width character counts as two. -** -** w should normally be a small number. A couple hundred at most. This -** routine caps w at 100 million to avoid integer overflow issues. -*/ -static void utf8_width_print(FILE *out, int w, const char *zUtf){ - const unsigned char *a = (const unsigned char*)zUtf; - static const int mxW = 10000000; - unsigned char c; - int i = 0; - int n = 0; - int k; - int aw; - if( w<-mxW ){ - w = -mxW; - }else if( w>mxW ){ - w= mxW; - } - aw = w<0 ? -w : w; - if( zUtf==0 ) zUtf = ""; - while( (c = a[i])!=0 ){ - if( (c&0xc0)==0xc0 ){ - int u; - int len = decodeUtf8(a+i, &u); - int x = cli_wcwidth(u); - if( x+n>aw ){ - break; - } - i += len; - n += x; - }else if( c==0x1b && (k = isVt100(&a[i]))>0 ){ - i += k; - }else if( n>=aw ){ - break; - }else{ - n++; - i++; - } - } - if( n>=aw ){ - sqlite3_fprintf(out, "%.*s", i, zUtf); - }else if( w<0 ){ - sqlite3_fprintf(out, "%*s%s", aw-n, "", zUtf); - }else{ - sqlite3_fprintf(out, "%s%*s", zUtf, aw-n, ""); - } -} - - -/* -** Determines if a string is a number of not. -*/ -static int isNumber(const char *z, int *realnum){ - if( *z=='-' || *z=='+' ) z++; - if( !IsDigit(*z) ){ - return 0; - } - z++; - if( realnum ) *realnum = 0; - while( IsDigit(*z) ){ z++; } - if( *z=='.' ){ - z++; - if( !IsDigit(*z) ) return 0; - while( IsDigit(*z) ){ z++; } - if( realnum ) *realnum = 1; - } - if( *z=='e' || *z=='E' ){ - z++; - if( *z=='+' || *z=='-' ) z++; - if( !IsDigit(*z) ) return 0; - while( IsDigit(*z) ){ z++; } - if( realnum ) *realnum = 1; - } - return *z==0; -} - -/* -** Compute a string length that is limited to what can be stored in -** lower 30 bits of a 32-bit signed integer. -*/ -static int strlen30(const char *z){ - const char *z2 = z; - while( *z2 ){ z2++; } - return 0x3fffffff & (int)(z2 - z); -} - -/* -** Return the length of a string in characters. Multibyte UTF8 characters -** count as a single character for single-width characters, or as two -** characters for double-width characters. -*/ -static int strlenChar(const char *z){ - int n = 0; - while( *z ){ - if( (0x80&z[0])==0 ){ - n++; - z++; - }else{ - int u = 0; - int len = decodeUtf8((const u8*)z, &u); - z += len; - n += cli_wcwidth(u); - } - } - return n; -} - -/* -** Return open FILE * if zFile exists, can be opened for read -** and is an ordinary file or a character stream source. -** Otherwise return 0. -*/ -static FILE * openChrSource(const char *zFile){ -#if defined(_WIN32) || defined(WIN32) - struct __stat64 x = {0}; -# define STAT_CHR_SRC(mode) ((mode & (_S_IFCHR|_S_IFIFO|_S_IFREG))!=0) - /* On Windows, open first, then check the stream nature. This order - ** is necessary because _stat() and sibs, when checking a named pipe, - ** effectively break the pipe as its supplier sees it. */ - FILE *rv = sqlite3_fopen(zFile, "rb"); - if( rv==0 ) return 0; - if( _fstat64(_fileno(rv), &x) != 0 - || !STAT_CHR_SRC(x.st_mode)){ - fclose(rv); - rv = 0; - } - return rv; -#else - struct stat x = {0}; - int rc = stat(zFile, &x); -# define STAT_CHR_SRC(mode) (S_ISREG(mode)||S_ISFIFO(mode)||S_ISCHR(mode)) - if( rc!=0 ) return 0; - if( STAT_CHR_SRC(x.st_mode) ){ - return sqlite3_fopen(zFile, "rb"); - }else{ - return 0; - } -#endif -#undef STAT_CHR_SRC -} - -/* -** This routine reads a line of text from FILE in, stores -** the text in memory obtained from malloc() and returns a pointer -** to the text. NULL is returned at end of file, or if malloc() -** fails, or if the length of the line is longer than about a gigabyte. -** -** If zLine is not NULL then it is a malloced buffer returned from -** a previous call to this routine that may be reused. -*/ -static char *local_getline(char *zLine, FILE *in){ - int nLine = zLine==0 ? 0 : 100; +static char *local_getline(char *zLine, FILE *in){ + int nLine = zLine==0 ? 0 : 100; int n = 0; while( 1 ){ @@ -1035,9 +1039,10 @@ static char *local_getline(char *zLine, FILE *in){ ** zPrior argument for reuse. */ #ifndef SQLITE_SHELL_FIDDLE -static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ +static char *one_input_line(ShellState *p, char *zPrior, int isContinuation){ char *zPrompt; char *zResult; + FILE *in = p->in; if( in!=0 ){ zResult = local_getline(zPrior, in); }else{ @@ -1305,7 +1310,7 @@ static void shellDtostr( char z[400]; if( n<1 ) n = 1; if( n>350 ) n = 350; - sqlite3_snprintf(sizeof(z), z, "%#+.*e", n, r); + sprintf(z, "%#+.*e", n, r); sqlite3_result_text(pCtx, z, -1, SQLITE_TRANSIENT); } @@ -1382,333 +1387,363 @@ static void shellAddSchemaName( sqlite3_result_value(pCtx, apVal[0]); } + +/************************* BEGIN PERFORMANCE TIMER *****************************/ +#if !defined(_WIN32) && !defined(WIN32) && !defined(__minux) +#include <sys/time.h> +#include <sys/resource.h> +/* VxWorks does not support getrusage() as far as we can determine */ +#if defined(_WRS_KERNEL) || defined(__RTP__) +struct rusage { + struct timeval ru_utime; /* user CPU time used */ + struct timeval ru_stime; /* system CPU time used */ +}; +#define getrusage(A,B) memset(B,0,sizeof(*B)) +#endif + +/* Saved resource information for the beginning of an operation */ +static struct rusage sBegin; /* CPU time at start */ +static sqlite3_int64 iBegin; /* Wall-clock time at start */ + /* -** The source code for several run-time loadable extensions is inserted -** below by the ../tool/mkshellc.tcl script. Before processing that included -** code, we need to override some macros to make the included program code -** work here in the middle of this regular program. +** Begin timing an operation */ -#define SQLITE_EXTENSION_INIT1 -#define SQLITE_EXTENSION_INIT2(X) (void)(X) +static void beginTimer(ShellState *p){ + if( p->enableTimer || (p->flgProgress & SHELL_PROGRESS_TMOUT)!=0 ){ + getrusage(RUSAGE_SELF, &sBegin); + iBegin = timeOfDay(); + } +} -INCLUDE ../ext/misc/windirent.h -INCLUDE ../ext/misc/memtrace.c -INCLUDE ../ext/misc/pcachetrace.c -INCLUDE ../ext/misc/shathree.c -INCLUDE ../ext/misc/sha1.c -INCLUDE ../ext/misc/uint.c -INCLUDE ../ext/misc/decimal.c -#undef sqlite3_base_init -#define sqlite3_base_init sqlite3_base64_init -INCLUDE ../ext/misc/base64.c -#undef sqlite3_base_init -#define sqlite3_base_init sqlite3_base85_init -#define OMIT_BASE85_CHECKER -INCLUDE ../ext/misc/base85.c -INCLUDE ../ext/misc/ieee754.c -INCLUDE ../ext/misc/series.c -INCLUDE ../ext/misc/regexp.c -#ifndef SQLITE_SHELL_FIDDLE -INCLUDE ../ext/misc/fileio.c -INCLUDE ../ext/misc/completion.c -INCLUDE ../ext/misc/appendvfs.c -#endif -#ifdef SQLITE_HAVE_ZLIB -INCLUDE ../ext/misc/zipfile.c -INCLUDE ../ext/misc/sqlar.c -#endif -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) -INCLUDE ../ext/expert/sqlite3expert.h -INCLUDE ../ext/expert/sqlite3expert.c -#endif -INCLUDE ../ext/intck/sqlite3intck.h -INCLUDE ../ext/intck/sqlite3intck.c -INCLUDE ../ext/misc/stmtrand.c -INCLUDE ../ext/misc/vfstrace.c +/* Return the difference of two time_structs in seconds */ +static double timeDiff(struct timeval *pStart, struct timeval *pEnd){ + return (pEnd->tv_usec - pStart->tv_usec)*0.000001 + + (double)(pEnd->tv_sec - pStart->tv_sec); +} -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) -#define SQLITE_SHELL_HAVE_RECOVER 1 -#else -#define SQLITE_SHELL_HAVE_RECOVER 0 -#endif -#if SQLITE_SHELL_HAVE_RECOVER -INCLUDE ../ext/recover/sqlite3recover.h -# ifndef SQLITE_HAVE_SQLITE3R -INCLUDE ../ext/recover/dbdata.c -INCLUDE ../ext/recover/sqlite3recover.c -# endif /* SQLITE_HAVE_SQLITE3R */ -#endif -#ifdef SQLITE_SHELL_EXTSRC -# include SHELL_STRINGIFY(SQLITE_SHELL_EXTSRC) -#endif +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK +/* Return the time since the start of the timer in +** seconds. */ +static double elapseTime(ShellState *NotUsed){ + (void)NotUsed; + if( iBegin==0 ) return 0.0; + return (timeOfDay() - iBegin)*0.000001; +} +#endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ -#if defined(SQLITE_ENABLE_SESSION) /* -** State information for a single open session +** Print the timing results. */ -typedef struct OpenSession OpenSession; -struct OpenSession { - char *zName; /* Symbolic name for this session */ - int nFilter; /* Number of xFilter rejection GLOB patterns */ - char **azFilter; /* Array of xFilter rejection GLOB patterns */ - sqlite3_session *p; /* The open session */ -}; -#endif +static void endTimer(ShellState *p){ + if( p->enableTimer ){ + sqlite3_int64 iEnd = timeOfDay(); + struct rusage sEnd; + getrusage(RUSAGE_SELF, &sEnd); + p->prevTimer = (iEnd - iBegin)*0.000001; + cli_printf(p->out, "Run Time: real %.6f user %.6f sys %.6f\n", + p->prevTimer, + timeDiff(&sBegin.ru_utime, &sEnd.ru_utime), + timeDiff(&sBegin.ru_stime, &sEnd.ru_stime)); + if( p->enableTimer==1 ) p->enableTimer = 0; + iBegin = 0; + } +} -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) -typedef struct ExpertInfo ExpertInfo; -struct ExpertInfo { - sqlite3expert *pExpert; - int bVerbose; -}; -#endif +#define BEGIN_TIMER(X) beginTimer(X) +#define END_TIMER(X) endTimer(X) +#define ELAPSE_TIME(X) elapseTime(X) +#define HAS_TIMER 1 -/* A single line in the EQP output */ -typedef struct EQPGraphRow EQPGraphRow; -struct EQPGraphRow { - int iEqpId; /* ID for this row */ - int iParentId; /* ID of the parent row */ - EQPGraphRow *pNext; /* Next row in sequence */ - char zText[1]; /* Text to display for this row */ -}; +#elif (defined(_WIN32) || defined(WIN32)) -/* All EQP output is collected into an instance of the following */ -typedef struct EQPGraph EQPGraph; -struct EQPGraph { - EQPGraphRow *pRow; /* Linked list of all rows of the EQP output */ - EQPGraphRow *pLast; /* Last element of the pRow list */ - char zPrefix[100]; /* Graph prefix */ -}; +/* Saved resource information for the beginning of an operation */ +static HANDLE hProcess; +static FILETIME ftKernelBegin; +static FILETIME ftUserBegin; +static sqlite3_int64 ftWallBegin; +typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME, + LPFILETIME, LPFILETIME); +static GETPROCTIMES getProcessTimesAddr = NULL; -/* Parameters affecting columnar mode result display (defaulting together) */ -typedef struct ColModeOpts { - int iWrap; /* In columnar modes, wrap lines reaching this limit */ - u8 bQuote; /* Quote results for .mode box and table */ - u8 bWordWrap; /* In columnar modes, wrap at word boundaries */ -} ColModeOpts; -#define ColModeOpts_default { 60, 0, 0 } -#define ColModeOpts_default_qbox { 60, 1, 0 } +/* +** Check to see if we have timer support. Return 1 if necessary +** support found (or found previously). +*/ +static int hasTimer(void){ + if( getProcessTimesAddr ){ + return 1; + } else { + /* GetProcessTimes() isn't supported in WIN95 and some other Windows + ** versions. See if the version we are running on has it, and if it + ** does, save off a pointer to it and the current process handle. + */ + hProcess = GetCurrentProcess(); + if( hProcess ){ + HINSTANCE hinstLib = LoadLibrary(TEXT("Kernel32.dll")); + if( NULL != hinstLib ){ + getProcessTimesAddr = + (GETPROCTIMES) GetProcAddress(hinstLib, "GetProcessTimes"); + if( NULL != getProcessTimesAddr ){ + return 1; + } + FreeLibrary(hinstLib); + } + } + } + return 0; +} /* -** State information about the database connection is contained in an -** instance of the following structure. +** Begin timing an operation */ -typedef struct ShellState ShellState; -struct ShellState { - sqlite3 *db; /* The database */ - u8 autoExplain; /* Automatically turn on .explain mode */ - u8 autoEQP; /* Run EXPLAIN QUERY PLAN prior to each SQL stmt */ - u8 autoEQPtest; /* autoEQP is in test mode */ - u8 autoEQPtrace; /* autoEQP is in trace mode */ - u8 scanstatsOn; /* True to display scan stats before each finalize */ - u8 openMode; /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */ - u8 doXdgOpen; /* Invoke start/open/xdg-open in output_reset() */ - u8 nEqpLevel; /* Depth of the EQP output graph */ - u8 eTraceType; /* SHELL_TRACE_* value for type of trace */ - u8 bSafeMode; /* True to prohibit unsafe operations */ - u8 bSafeModePersist; /* The long-term value of bSafeMode */ - u8 eRestoreState; /* See comments above doAutoDetectRestore() */ - u8 crlfMode; /* Do NL-to-CRLF translations when enabled (maybe) */ - u8 eEscMode; /* Escape mode for text output */ - ColModeOpts cmOpts; /* Option values affecting columnar mode output */ - unsigned statsOn; /* True to display memory stats before each finalize */ - unsigned mEqpLines; /* Mask of vertical lines in the EQP output graph */ - int inputNesting; /* Track nesting level of .read and other redirects */ - int outCount; /* Revert to stdout when reaching zero */ - int cnt; /* Number of records displayed so far */ - i64 lineno; /* Line number of last line read from in */ - int openFlags; /* Additional flags to open. (SQLITE_OPEN_NOFOLLOW) */ - FILE *in; /* Read commands from this stream */ - FILE *out; /* Write results here */ - FILE *traceOut; /* Output for sqlite3_trace() */ - int nErr; /* Number of errors seen */ - int mode; /* An output mode setting */ - int modePrior; /* Saved mode */ - int cMode; /* temporary output mode for the current query */ - int normalMode; /* Output mode before ".explain on" */ - int writableSchema; /* True if PRAGMA writable_schema=ON */ - int showHeader; /* True to show column names in List or Column mode */ - int nCheck; /* Number of ".check" commands run */ - unsigned nProgress; /* Number of progress callbacks encountered */ - unsigned mxProgress; /* Maximum progress callbacks before failing */ - unsigned flgProgress; /* Flags for the progress callback */ - unsigned shellFlgs; /* Various flags */ - unsigned priorShFlgs; /* Saved copy of flags */ - sqlite3_int64 szMax; /* --maxsize argument to .open */ - char *zDestTable; /* Name of destination table when MODE_Insert */ - char *zTempFile; /* Temporary file that might need deleting */ - char zTestcase[30]; /* Name of current test case */ - char colSeparator[20]; /* Column separator character for several modes */ - char rowSeparator[20]; /* Row separator character for MODE_Ascii */ - char colSepPrior[20]; /* Saved column separator */ - char rowSepPrior[20]; /* Saved row separator */ - int *colWidth; /* Requested width of each column in columnar modes */ - int *actualWidth; /* Actual width of each column */ - int nWidth; /* Number of slots in colWidth[] and actualWidth[] */ - char nullValue[20]; /* The text to print when a NULL comes back from - ** the database */ - char outfile[FILENAME_MAX]; /* Filename for *out */ - sqlite3_stmt *pStmt; /* Current statement if any. */ - FILE *pLog; /* Write log output here */ - struct AuxDb { /* Storage space for auxiliary database connections */ - sqlite3 *db; /* Connection pointer */ - const char *zDbFilename; /* Filename used to open the connection */ - char *zFreeOnClose; /* Free this memory allocation on close */ -#if defined(SQLITE_ENABLE_SESSION) - int nSession; /* Number of active sessions */ - OpenSession aSession[4]; /* Array of sessions. [0] is in focus. */ -#endif - } aAuxDb[5], /* Array of all database connections */ - *pAuxDb; /* Currently active database connection */ - int *aiIndent; /* Array of indents used in MODE_Explain */ - int nIndent; /* Size of array aiIndent[] */ - int iIndent; /* Index of current op in aiIndent[] */ - char *zNonce; /* Nonce for temporary safe-mode escapes */ - EQPGraph sGraph; /* Information for the graphical EXPLAIN QUERY PLAN */ -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) - ExpertInfo expert; /* Valid if previous command was ".expert OPT..." */ -#endif -#ifdef SQLITE_SHELL_FIDDLE - struct { - const char * zInput; /* Input string from wasm/JS proxy */ - const char * zPos; /* Cursor pos into zInput */ - const char * zDefaultDbName; /* Default name for db file */ - } wasm; -#endif -}; - -#ifdef SQLITE_SHELL_FIDDLE -static ShellState shellState; -#endif +static void beginTimer(ShellState *p){ + if( (p->enableTimer || (p->flgProgress & SHELL_PROGRESS_TMOUT)!=0) + && getProcessTimesAddr + ){ + FILETIME ftCreation, ftExit; + getProcessTimesAddr(hProcess,&ftCreation,&ftExit, + &ftKernelBegin,&ftUserBegin); + ftWallBegin = timeOfDay(); + } +} +/* Return the difference of two FILETIME structs in seconds */ +static double timeDiff(FILETIME *pStart, FILETIME *pEnd){ + sqlite_int64 i64Start = *((sqlite_int64 *) pStart); + sqlite_int64 i64End = *((sqlite_int64 *) pEnd); + return (double) ((i64End - i64Start) / 10000000.0); +} -/* Allowed values for ShellState.autoEQP -*/ -#define AUTOEQP_off 0 /* Automatic EXPLAIN QUERY PLAN is off */ -#define AUTOEQP_on 1 /* Automatic EQP is on */ -#define AUTOEQP_trigger 2 /* On and also show plans for triggers */ -#define AUTOEQP_full 3 /* Show full EXPLAIN */ +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK +/* Return the time since the start of the timer in +** seconds. */ +static double elapseTime(ShellState *NotUsed){ + (void)NotUsed; + if( ftWallBegin==0 ) return 0.0; + return (timeOfDay() - ftWallBegin)*0.000001; +} +#endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ -/* Allowed values for ShellState.openMode +/* +** Print the timing results. */ -#define SHELL_OPEN_UNSPEC 0 /* No open-mode specified */ -#define SHELL_OPEN_NORMAL 1 /* Normal database file */ -#define SHELL_OPEN_APPENDVFS 2 /* Use appendvfs */ -#define SHELL_OPEN_ZIPFILE 3 /* Use the zipfile virtual table */ -#define SHELL_OPEN_DESERIALIZE 4 /* Open using sqlite3_deserialize() */ -#define SHELL_OPEN_HEXDB 5 /* Use "dbtotxt" output as data source */ +static void endTimer(ShellState *p){ + if( p->enableTimer && getProcessTimesAddr){ + FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; + sqlite3_int64 ftWallEnd = timeOfDay(); + getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); + p->prevTimer = (ftWallEnd - ftWallBegin)*0.000001; +#ifdef _WIN64 + /* microsecond precision on 64-bit windows */ + cli_printf(p->out, "Run Time: real %.6f user %f sys %f\n", + p->prevTimer, + timeDiff(&ftUserBegin, &ftUserEnd), + timeDiff(&ftKernelBegin, &ftKernelEnd)); +#else + /* millisecond precisino on 32-bit windows */ + cli_printf(p->out, "Run Time: real %.3f user %.3f sys %.3f\n", + p->prevTimer, + timeDiff(&ftUserBegin, &ftUserEnd), + timeDiff(&ftKernelBegin, &ftKernelEnd)); +#endif + if( p->enableTimer==1 ) p->enableTimer = 0; + ftWallBegin = 0; + } +} -/* Allowed values for ShellState.eTraceType -*/ -#define SHELL_TRACE_PLAIN 0 /* Show input SQL text */ -#define SHELL_TRACE_EXPANDED 1 /* Show expanded SQL text */ -#define SHELL_TRACE_NORMALIZED 2 /* Show normalized SQL text */ +#define BEGIN_TIMER(X) beginTimer(X) +#define ELAPSE_TIME(X) elapseTime(X) +#define END_TIMER(X) endTimer(X) +#define HAS_TIMER hasTimer() -/* Bits in the ShellState.flgProgress variable */ -#define SHELL_PROGRESS_QUIET 0x01 /* Omit announcing every progress callback */ -#define SHELL_PROGRESS_RESET 0x02 /* Reset the count when the progress - ** callback limit is reached, and for each - ** top-level SQL statement */ -#define SHELL_PROGRESS_ONCE 0x04 /* Cancel the --limit after firing once */ +#else +#define BEGIN_TIMER(X) /* no-op */ +#define ELAPSE_TIME(X) 0.0 +#define END_TIMER(X) /*no-op*/ +#define HAS_TIMER 0 +#endif +/************************* END PERFORMANCE TIMER ******************************/ -/* Allowed values for ShellState.eEscMode. The default value should -** be 0, so to change the default, reorder the names. +/* +** Clear a display mode, freeing any allocated memory that it +** contains. */ -#define SHELL_ESC_ASCII 0 /* Substitute ^Y for X where Y=X+0x40 */ -#define SHELL_ESC_SYMBOL 1 /* Substitute U+2400 graphics */ -#define SHELL_ESC_OFF 2 /* Send characters verbatim */ - -static const char *shell_EscModeNames[] = { "ascii", "symbol", "off" }; +static void modeFree(Mode *p){ + u8 autoExplain = p->autoExplain; + free(p->spec.aWidth); + free(p->spec.aAlign); + free(p->spec.zColumnSep); + free(p->spec.zRowSep); + free(p->spec.zTableName); + free(p->spec.zNull); + memset(p, 0, sizeof(*p)); + p->spec.iVersion = 1; + p->autoExplain = autoExplain; +} /* -** These are the allowed shellFlgs values +** Duplicate Mode pSrc into pDest. pDest is assumed to be +** uninitialized prior to invoking this routine. */ -#define SHFLG_Pagecache 0x00000001 /* The --pagecache option is used */ -#define SHFLG_Lookaside 0x00000002 /* Lookaside memory is used */ -#define SHFLG_Backslash 0x00000004 /* The --backslash option is used */ -#define SHFLG_PreserveRowid 0x00000008 /* .dump preserves rowid values */ -#define SHFLG_Newlines 0x00000010 /* .dump --newline flag */ -#define SHFLG_CountChanges 0x00000020 /* .changes setting */ -#define SHFLG_Echo 0x00000040 /* .echo on/off, or --echo setting */ -#define SHFLG_HeaderSet 0x00000080 /* showHeader has been specified */ -#define SHFLG_DumpDataOnly 0x00000100 /* .dump show data only */ -#define SHFLG_DumpNoSys 0x00000200 /* .dump omits system tables */ -#define SHFLG_TestingMode 0x00000400 /* allow unsafe testing features */ +static void modeDup(Mode *pDest, Mode *pSrc){ + memcpy(pDest, pSrc, sizeof(*pDest)); + if( pDest->spec.aWidth ){ + size_t sz = sizeof(pSrc->spec.aWidth[0]) * pSrc->spec.nWidth; + pDest->spec.aWidth = malloc( sz ); + if( pDest->spec.aWidth ){ + memcpy(pDest->spec.aWidth, pSrc->spec.aWidth, sz); + }else{ + pDest->spec.nWidth = 0; + } + } + if( pDest->spec.aAlign ){ + size_t sz = sizeof(pSrc->spec.aAlign[0]) * pSrc->spec.nAlign; + pDest->spec.aAlign = malloc( sz ); + if( pDest->spec.aAlign ){ + memcpy(pDest->spec.aAlign, pSrc->spec.aAlign, sz); + }else{ + pDest->spec.nAlign = 0; + } + } + if( pDest->spec.zColumnSep ){ + pDest->spec.zColumnSep = strdup(pSrc->spec.zColumnSep); + } + if( pDest->spec.zRowSep ){ + pDest->spec.zRowSep = strdup(pSrc->spec.zRowSep); + } + if( pDest->spec.zTableName ){ + pDest->spec.zTableName = strdup(pSrc->spec.zTableName); + } + if( pDest->spec.zNull ){ + pDest->spec.zNull = strdup(pSrc->spec.zNull); + } +} /* -** Macros for testing and setting shellFlgs +** Set a string value to a copy of the zNew string in memory +** obtained from system malloc(). */ -#define ShellHasFlag(P,X) (((P)->shellFlgs & (X))!=0) -#define ShellSetFlag(P,X) ((P)->shellFlgs|=(X)) -#define ShellClearFlag(P,X) ((P)->shellFlgs&=(~(X))) +static void modeSetStr(char **az, const char *zNew){ + free(*az); + if( zNew==0 ){ + *az = 0; + }else{ + size_t n = strlen(zNew); + *az = malloc( n+1 ); + if( *az ){ + memcpy(*az, zNew, n+1 ); + } + } +} /* -** These are the allowed modes. -*/ -#define MODE_Line 0 /* One column per line. Blank line between records */ -#define MODE_Column 1 /* One record per line in neat columns */ -#define MODE_List 2 /* One record per line with a separator */ -#define MODE_Semi 3 /* Same as MODE_List but append ";" to each line */ -#define MODE_Html 4 /* Generate an XHTML table */ -#define MODE_Insert 5 /* Generate SQL "insert" statements */ -#define MODE_Quote 6 /* Quote values as for SQL */ -#define MODE_Tcl 7 /* Generate ANSI-C or TCL quoted elements */ -#define MODE_Csv 8 /* Quote strings, numbers are plain */ -#define MODE_Explain 9 /* Like MODE_Column, but do not truncate data */ -#define MODE_Ascii 10 /* Use ASCII unit and record separators (0x1F/0x1E) */ -#define MODE_Pretty 11 /* Pretty-print schemas */ -#define MODE_EQP 12 /* Converts EXPLAIN QUERY PLAN output into a graph */ -#define MODE_Json 13 /* Output JSON */ -#define MODE_Markdown 14 /* Markdown formatting */ -#define MODE_Table 15 /* MySQL-style table formatting */ -#define MODE_Box 16 /* Unicode box-drawing characters */ -#define MODE_Count 17 /* Output only a count of the rows of output */ -#define MODE_Off 18 /* No query output shown */ -#define MODE_ScanExp 19 /* Like MODE_Explain, but for ".scanstats vm" */ -#define MODE_Www 20 /* Full web-page output */ - -static const char *modeDescr[] = { - "line", - "column", - "list", - "semi", - "html", - "insert", - "quote", - "tcl", - "csv", - "explain", - "ascii", - "prettyprint", - "eqp", - "json", - "markdown", - "table", - "box", - "count", - "off", - "scanexp", - "www", -}; +** Change the mode to eMode +*/ +static void modeChange(ShellState *p, unsigned char eMode){ + const ModeInfo *pI; + if( eMode<ArraySize(aModeInfo) ){ + Mode *pM = &p->mode; + pI = &aModeInfo[eMode]; + pM->eMode = eMode; + if( pI->eCSep ) modeSetStr(&pM->spec.zColumnSep, aModeStr[pI->eCSep]); + if( pI->eRSep ) modeSetStr(&pM->spec.zRowSep, aModeStr[pI->eRSep]); + if( pI->eNull ) modeSetStr(&pM->spec.zNull, aModeStr[pI->eNull]); + pM->spec.eText = pI->eText; + pM->spec.eBlob = pI->eBlob; + if( (pM->mFlags & MFLG_HDR)==0 ){ + pM->spec.bTitles = pI->bHdr; + } + pM->spec.eTitle = pI->eHdr; + if( pI->mFlg & 0x01 ){ + pM->spec.bBorder = QRF_No; + }else{ + pM->spec.bBorder = QRF_Auto; + } + if( pI->mFlg & 0x02 ){ + pM->spec.bSplitColumn = QRF_Yes; + pM->bAutoScreenWidth = 1; + }else{ + pM->spec.bSplitColumn = QRF_No; + } + }else if( eMode>=MODE_USER && eMode-MODE_USER<p->nSavedModes ){ + modeFree(&p->mode); + modeDup(&p->mode, &p->aSavedModes[eMode-MODE_USER].mode); + }else if( eMode==MODE_BATCH ){ + u8 mFlags = p->mode.mFlags; + modeFree(&p->mode); + modeChange(p, MODE_List); + p->mode.mFlags = mFlags; + }else if( eMode==MODE_TTY ){ + u8 mFlags = p->mode.mFlags; + modeFree(&p->mode); + modeChange(p, MODE_QBox); + p->mode.bAutoScreenWidth = 1; + p->mode.spec.eText = QRF_TEXT_Relaxed; + p->mode.spec.nCharLimit = DFLT_CHAR_LIMIT; + p->mode.spec.nLineLimit = DFLT_LINE_LIMIT; + p->mode.spec.bTextJsonb = QRF_Yes; + p->mode.spec.nTitleLimit = DFLT_TITLE_LIMIT; + p->mode.spec.nMultiInsert = DFLT_MULTI_INSERT; + p->mode.mFlags = mFlags; + } +} + +/* +** Set the mode to the default. It assumed that the mode has +** already been freed and zeroed prior to calling this routine. +*/ +static void modeDefault(ShellState *p){ + p->mode.spec.iVersion = 1; + p->mode.autoExplain = 1; + if( stdin_is_interactive || stdout_is_console ){ + modeChange(p, MODE_TTY); + }else{ + modeChange(p, MODE_BATCH); + } +} /* -** These are the column/row/line separators used by the various -** import/export modes. +** Find the number of a display mode given its name. Return -1 if +** the name does not match any mode. +** +** Saved modes are also searched if p!=NULL. The number returned +** for a saved mode is the index into the p->aSavedModes[] array +** plus MODE_USER. +** +** Two special mode names are also available: "batch" and "tty". +** evaluate to the default mode for batch operation and interactive +** operation on a TTY, respectively. */ -#define SEP_Column "|" -#define SEP_Row "\n" -#define SEP_Tab "\t" -#define SEP_Space " " -#define SEP_Comma "," -#define SEP_CrLf "\r\n" -#define SEP_Unit "\x1F" -#define SEP_Record "\x1E" +static int modeFind(ShellState *p, const char *zName){ + int i; + for(i=0; i<ArraySize(aModeInfo); i++){ + if( cli_strcmp(aModeInfo[i].zName,zName)==0 ) return i; + } + for(i=0; i<p->nSavedModes; i++){ + if( cli_strcmp(p->aSavedModes[i].zTag,zName)==0 ) return i+MODE_USER; + } + if( strcmp(zName,"batch")==0 ) return MODE_BATCH; + if( strcmp(zName,"tty")==0 ) return MODE_TTY; + return -1; +} /* -** Limit input nesting via .read or any other input redirect. -** It's not too expensive, so a generous allowance can be made. +** Save or restore the current output mode */ -#define MAX_INPUT_NESTING 25 +static void modePush(ShellState *p){ + if( p->nPopMode==0 ){ + modeFree(&p->modePrior); + modeDup(&p->modePrior,&p->mode); + } +} +static void modePop(ShellState *p){ + if( p->modePrior.spec.iVersion>0 ){ + modeFree(&p->mode); + p->mode = p->modePrior; + memset(&p->modePrior, 0, sizeof(p->modePrior)); + } +} + /* ** A callback for the sqlite3_log() interface. @@ -1716,7 +1751,7 @@ static const char *modeDescr[] = { static void shellLog(void *pArg, int iErrCode, const char *zMsg){ ShellState *p = (ShellState*)pArg; if( p->pLog==0 ) return; - sqlite3_fprintf(p->pLog, "(%d) %s\n", iErrCode, zMsg); + cli_printf(p->pLog, "(%d) %s\n", iErrCode, zMsg); fflush(p->pLog); } @@ -1733,10 +1768,26 @@ static void shellPutsFunc( ){ ShellState *p = (ShellState*)sqlite3_user_data(pCtx); (void)nVal; - sqlite3_fprintf(p->out, "%s\n", sqlite3_value_text(apVal[0])); + cli_printf(p->out, "%s\n", sqlite3_value_text(apVal[0])); sqlite3_result_value(pCtx, apVal[0]); } +/* +** Compute the name of the location of an input error in memory +** obtained from sqlite3_malloc(). +*/ +static char *shellErrorLocation(ShellState *p){ + char *zLoc; + if( p->zErrPrefix ){ + zLoc = sqlite3_mprintf("%s", p->zErrPrefix); + }else if( p->zInFile==0 || strcmp(p->zInFile,"<stdin>")==0){ + zLoc = sqlite3_mprintf("line %lld:", p->lineno); + }else{ + zLoc = sqlite3_mprintf("%s:%lld:", p->zInFile, p->lineno); + } + return zLoc; +} + /* ** If in safe mode, print an error message described by the arguments ** and exit immediately. @@ -1749,13 +1800,49 @@ static void failIfSafeMode( if( p->bSafeMode ){ va_list ap; char *zMsg; + char *zLoc = shellErrorLocation(p); va_start(ap, zErrMsg); zMsg = sqlite3_vmprintf(zErrMsg, ap); va_end(ap); - sqlite3_fprintf(stderr, "line %lld: %s\n", p->lineno, zMsg); - exit(1); + cli_printf(stderr, "%s %s\n", zLoc, zMsg); + cli_exit(1); + } +} + +/* +** Issue an error message from a dot-command. +*/ +static void dotCmdError( + ShellState *p, /* Shell state */ + int iArg, /* Index of argument on which error occurred */ + const char *zBrief, /* Brief (<20 character) error description */ + const char *zDetail, /* Error details */ + ... +){ + FILE *out = stderr; + char *zLoc = shellErrorLocation(p); + if( zBrief!=0 && iArg>=0 && iArg<p->dot.nArg ){ + int i = p->dot.aiOfst[iArg]; + int nPrompt = strlen30(zBrief) + 5; + cli_printf(out, "%s %s\n", zLoc, p->dot.zOrig); + if( i > nPrompt ){ + cli_printf(out, "%s %*s%s ---^\n", zLoc, 1+i-nPrompt, "", zBrief); + }else{ + cli_printf(out, "%s %*s^--- %s\n", zLoc, i, "", zBrief); + } + } + if( zDetail ){ + char *zMsg; + va_list ap; + va_start(ap, zDetail); + zMsg = sqlite3_vmprintf(zDetail,ap); + va_end(ap); + cli_printf(out,"%s %s\n", zLoc, zMsg); + sqlite3_free(zMsg); } + sqlite3_free(zLoc); } + /* ** SQL function: edit(VALUE) @@ -1901,28 +1988,12 @@ edit_func_end: } #endif /* SQLITE_NOHAVE_SYSTEM */ -/* -** Save or restore the current output mode -*/ -static void outputModePush(ShellState *p){ - p->modePrior = p->mode; - p->priorShFlgs = p->shellFlgs; - memcpy(p->colSepPrior, p->colSeparator, sizeof(p->colSeparator)); - memcpy(p->rowSepPrior, p->rowSeparator, sizeof(p->rowSeparator)); -} -static void outputModePop(ShellState *p){ - p->mode = p->modePrior; - p->shellFlgs = p->priorShFlgs; - memcpy(p->colSeparator, p->colSepPrior, sizeof(p->colSeparator)); - memcpy(p->rowSeparator, p->rowSepPrior, sizeof(p->rowSeparator)); -} - /* ** Set output mode to text or binary for Windows. */ static void setCrlfMode(ShellState *p){ #ifdef _WIN32 - if( p->crlfMode ){ + if( p->mode.mFlags & MFLG_CRLF ){ sqlite3_fsetmode(p->out, _O_TEXT); }else{ sqlite3_fsetmode(p->out, _O_BINARY); @@ -1932,126 +2003,6 @@ static void setCrlfMode(ShellState *p){ #endif } -/* -** Output the given string as a hex-encoded blob (eg. X'1234' ) -*/ -static void output_hex_blob(FILE *out, const void *pBlob, int nBlob){ - int i; - unsigned char *aBlob = (unsigned char*)pBlob; - - char *zStr = sqlite3_malloc64((i64)nBlob*2 + 1); - shell_check_oom(zStr); - - for(i=0; i<nBlob; i++){ - static const char aHex[] = { - '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' - }; - zStr[i*2] = aHex[ (aBlob[i] >> 4) ]; - zStr[i*2+1] = aHex[ (aBlob[i] & 0x0F) ]; - } - zStr[i*2] = '\0'; - - sqlite3_fprintf(out, "X'%s'", zStr); - sqlite3_free(zStr); -} - -/* -** Output the given string as a quoted string using SQL quoting conventions: -** -** (1) Single quotes (') within the string are doubled -** (2) The while string is enclosed in '...' -** (3) Control characters other than \n, \t, and \r\n are escaped -** using \u00XX notation and if such substitutions occur, -** the whole string is enclosed in unistr('...') instead of '...'. -** -** Step (3) is omitted if the control-character escape mode is OFF. -** -** See also: output_quoted_escaped_string() which does the same except -** that it does not make exceptions for \n, \t, and \r\n in step (3). -*/ -static void output_quoted_string(ShellState *p, const char *zInX){ - int i; - int needUnistr = 0; - int needDblQuote = 0; - const unsigned char *z = (const unsigned char*)zInX; - unsigned char c; - FILE *out = p->out; - sqlite3_fsetmode(out, _O_BINARY); - if( z==0 ) return; - for(i=0; (c = z[i])!=0; i++){ - if( c=='\'' ){ needDblQuote = 1; } - if( c>0x1f ) continue; - if( c=='\t' || c=='\n' ) continue; - if( c=='\r' && z[i+1]=='\n' ) continue; - needUnistr = 1; - break; - } - if( (needDblQuote==0 && needUnistr==0) - || (needDblQuote==0 && p->eEscMode==SHELL_ESC_OFF) - ){ - sqlite3_fprintf(out, "'%s'",z); - }else if( p->eEscMode==SHELL_ESC_OFF ){ - char *zEncoded = sqlite3_mprintf("%Q", z); - sqlite3_fputs(zEncoded, out); - sqlite3_free(zEncoded); - }else{ - if( needUnistr ){ - sqlite3_fputs("unistr('", out); - }else{ - sqlite3_fputs("'", out); - } - while( *z ){ - for(i=0; (c = z[i])!=0; i++){ - if( c=='\'' ) break; - if( c>0x1f ) continue; - if( c=='\t' || c=='\n' ) continue; - if( c=='\r' && z[i+1]=='\n' ) continue; - break; - } - if( i ){ - sqlite3_fprintf(out, "%.*s", i, z); - z += i; - } - if( c==0 ) break; - if( c=='\'' ){ - sqlite3_fputs("''", out); - }else{ - sqlite3_fprintf(out, "\\u%04x", c); - } - z++; - } - if( needUnistr ){ - sqlite3_fputs("')", out); - }else{ - sqlite3_fputs("'", out); - } - } - setCrlfMode(p); -} - -/* -** Output the given string as a quoted string using SQL quoting conventions. -** Additionallly , escape the "\n" and "\r" characters so that they do not -** get corrupted by end-of-line translation facilities in some operating -** systems. -** -** This is like output_quoted_string() but with the addition of the \r\n -** escape mechanism. -*/ -static void output_quoted_escaped_string(ShellState *p, const char *z){ - char *zEscaped; - sqlite3_fsetmode(p->out, _O_BINARY); - if( p->eEscMode==SHELL_ESC_OFF ){ - zEscaped = sqlite3_mprintf("%Q", z); - }else{ - zEscaped = sqlite3_mprintf("%#Q", z); - } - sqlite3_fputs(zEscaped, p->out); - sqlite3_free(zEscaped); - setCrlfMode(p); -} - /* ** Find earliest of chars within s specified in zAny. ** With ns == ~0, is like strpbrk(s,zAny) and s must be 0-terminated. @@ -2115,13 +2066,14 @@ static void output_c_string(FILE *out, const char *z){ static const char *zDQBSRO = "\"\\\x7f"; /* double-quote, backslash, rubout */ char ace[3] = "\\?"; char cbsSay; - sqlite3_fputs(zq, out); + cli_puts(zq, out); + if( z==0 ) z = ""; while( *z!=0 ){ const char *pcDQBSRO = anyOfInStr(z, zDQBSRO, ~(size_t)0); const char *pcPast = zSkipValidUtf8(z, INT_MAX, ctrlMask); const char *pcEnd = (pcDQBSRO && pcDQBSRO < pcPast)? pcDQBSRO : pcPast; if( pcEnd > z ){ - sqlite3_fprintf(out, "%.*s", (int)(pcEnd-z), z); + cli_printf(out, "%.*s", (int)(pcEnd-z), z); } if( (c = *pcEnd)==0 ) break; ++pcEnd; @@ -2137,250 +2089,106 @@ static void output_c_string(FILE *out, const char *z){ } if( cbsSay ){ ace[1] = cbsSay; - sqlite3_fputs(ace, out); + cli_puts(ace, out); }else if( !isprint(c&0xff) ){ - sqlite3_fprintf(out, "\\%03o", c&0xff); + cli_printf(out, "\\%03o", c&0xff); }else{ ace[1] = (char)c; - sqlite3_fputs(ace+1, out); + cli_puts(ace+1, out); } z = pcEnd; } - sqlite3_fputs(zq, out); + cli_puts(zq, out); } -/* -** Output the given string as quoted according to JSON quoting rules. +/* Encode input string z[] as a C-language string literal and +** append it to the sqlite3_str. If z is NULL render and empty string. */ -static void output_json_string(FILE *out, const char *z, i64 n){ - unsigned char c; +static void append_c_string(sqlite3_str *out, const char *z){ + char c; static const char *zq = "\""; static long ctrlMask = ~0L; - static const char *zDQBS = "\"\\"; - const char *pcLimit; + static const char *zDQBSRO = "\"\\\x7f"; /* double-quote, backslash, rubout */ char ace[3] = "\\?"; char cbsSay; - if( z==0 ) z = ""; - pcLimit = z + ((n<0)? strlen(z) : (size_t)n); - sqlite3_fputs(zq, out); - while( z < pcLimit ){ - const char *pcDQBS = anyOfInStr(z, zDQBS, pcLimit-z); - const char *pcPast = zSkipValidUtf8(z, (int)(pcLimit-z), ctrlMask); - const char *pcEnd = (pcDQBS && pcDQBS < pcPast)? pcDQBS : pcPast; + sqlite3_str_appendall(out,zq); + while( *z!=0 ){ + const char *pcDQBSRO = anyOfInStr(z, zDQBSRO, ~(size_t)0); + const char *pcPast = zSkipValidUtf8(z, INT_MAX, ctrlMask); + const char *pcEnd = (pcDQBSRO && pcDQBSRO < pcPast)? pcDQBSRO : pcPast; if( pcEnd > z ){ - sqlite3_fprintf(out, "%.*s", (int)(pcEnd-z), z); - z = pcEnd; + sqlite3_str_appendf(out, "%.*s", (int)(pcEnd-z), z); } - if( z >= pcLimit ) break; - c = (unsigned char)*(z++); + if( (c = *pcEnd)==0 ) break; + ++pcEnd; switch( c ){ - case '"': case '\\': + case '\\': case '"': cbsSay = (char)c; break; - case '\b': cbsSay = 'b'; break; - case '\f': cbsSay = 'f'; break; + case '\t': cbsSay = 't'; break; case '\n': cbsSay = 'n'; break; case '\r': cbsSay = 'r'; break; - case '\t': cbsSay = 't'; break; + case '\f': cbsSay = 'f'; break; default: cbsSay = 0; break; } if( cbsSay ){ ace[1] = cbsSay; - sqlite3_fputs(ace, out); - }else if( c<=0x1f || c>=0x7f ){ - sqlite3_fprintf(out, "\\u%04x", c); + sqlite3_str_appendall(out,ace); + }else if( !isprint(c&0xff) ){ + sqlite3_str_appendf(out, "\\%03o", c&0xff); }else{ ace[1] = (char)c; - sqlite3_fputs(ace+1, out); + sqlite3_str_appendall(out, ace+1); } + z = pcEnd; } - sqlite3_fputs(zq, out); -} - -/* -** Escape the input string if it is needed and in accordance with -** eEscMode. -** -** Escaping is needed if the string contains any control characters -** other than \t, \n, and \r\n -** -** If no escaping is needed (the common case) then set *ppFree to NULL -** and return the original string. If escaping is needed, write the -** escaped string into memory obtained from sqlite3_malloc64() or the -** equivalent, and return the new string and set *ppFree to the new string -** as well. -** -** The caller is responsible for freeing *ppFree if it is non-NULL in order -** to reclaim memory. -*/ -static const char *escapeOutput( - ShellState *p, - const char *zInX, - char **ppFree -){ - i64 i, j; - i64 nCtrl = 0; - unsigned char *zIn; - unsigned char c; - unsigned char *zOut; - - - /* No escaping if disabled */ - if( p->eEscMode==SHELL_ESC_OFF ){ - *ppFree = 0; - return zInX; - } - - /* Count the number of control characters in the string. */ - zIn = (unsigned char*)zInX; - for(i=0; (c = zIn[i])!=0; i++){ - if( c<=0x1f - && c!='\t' - && c!='\n' - && (c!='\r' || zIn[i+1]!='\n') - ){ - nCtrl++; - } - } - if( nCtrl==0 ){ - *ppFree = 0; - return zInX; - } - if( p->eEscMode==SHELL_ESC_SYMBOL ) nCtrl *= 2; - zOut = sqlite3_malloc64( i + nCtrl + 1 ); - shell_check_oom(zOut); - for(i=j=0; (c = zIn[i])!=0; i++){ - if( c>0x1f - || c=='\t' - || c=='\n' - || (c=='\r' && zIn[i+1]=='\n') - ){ - continue; - } - if( i>0 ){ - memcpy(&zOut[j], zIn, i); - j += i; - } - zIn += i+1; - i = -1; - switch( p->eEscMode ){ - case SHELL_ESC_SYMBOL: - zOut[j++] = 0xe2; - zOut[j++] = 0x90; - zOut[j++] = 0x80+c; - break; - case SHELL_ESC_ASCII: - zOut[j++] = '^'; - zOut[j++] = 0x40+c; - break; - } - } - if( i>0 ){ - memcpy(&zOut[j], zIn, i); - j += i; - } - zOut[j] = 0; - *ppFree = (char*)zOut; - return (char*)zOut; + sqlite3_str_appendall(out, zq); } /* -** Output the given string with characters that are special to -** HTML escaped. +** This routine runs when the user presses Ctrl-C */ -static void output_html_string(FILE *out, const char *z){ - int i; - if( z==0 ) z = ""; - while( *z ){ - for(i=0; z[i] - && z[i]!='<' - && z[i]!='&' - && z[i]!='>' - && z[i]!='\"' - && z[i]!='\''; - i++){} - if( i>0 ){ - sqlite3_fprintf(out, "%.*s",i,z); - } - if( z[i]=='<' ){ - sqlite3_fputs("&lt;", out); - }else if( z[i]=='&' ){ - sqlite3_fputs("&amp;", out); - }else if( z[i]=='>' ){ - sqlite3_fputs("&gt;", out); - }else if( z[i]=='\"' ){ - sqlite3_fputs("&quot;", out); - }else if( z[i]=='\'' ){ - sqlite3_fputs("&#39;", out); - }else{ - break; - } - z += i + 1; - } +static void interrupt_handler(int NotUsed){ + UNUSED_PARAMETER(NotUsed); + if( ++seenInterrupt>1 ) cli_exit(1); + if( globalDb ) sqlite3_interrupt(globalDb); } -/* -** If a field contains any character identified by a 1 in the following -** array, then the string must be quoted for CSV. -*/ -static const char needCsvQuote[] = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -}; - -/* -** Output a single term of CSV. Actually, p->colSeparator is used for -** the separator, which may or may not be a comma. p->nullValue is -** the null value. Strings are quoted if necessary. The separator -** is only issued if bSep is true. +/* Try to determine the screen width. Use the default if unable. */ -static void output_csv(ShellState *p, const char *z, int bSep){ - if( z==0 ){ - sqlite3_fprintf(p->out, "%s",p->nullValue); +int shellScreenWidth(void){ + if( stdout_tty_width>0 ){ + return stdout_tty_width; }else{ - unsigned i; - for(i=0; z[i]; i++){ - if( needCsvQuote[((unsigned char*)z)[i]] ){ - i = 0; - break; - } +#if defined(TIOCGSIZE) + struct ttysize ts; + if( ioctl(STDIN_FILENO, TIOCGSIZE, &ts)>=0 + || ioctl(STDOUT_FILENO, TIOCGSIZE, &ts)>=0 + || ioctl(STDERR_FILENO, TIOCGSIZE, &ts)>=0 + ){ + return ts.ts_cols; } - if( i==0 || strstr(z, p->colSeparator)!=0 ){ - char *zQuoted = sqlite3_mprintf("\"%w\"", z); - shell_check_oom(zQuoted); - sqlite3_fputs(zQuoted, p->out); - sqlite3_free(zQuoted); - }else{ - sqlite3_fputs(z, p->out); +#elif defined(TIOCGWINSZ) + struct winsize ws; + if( ioctl(STDIN_FILENO, TIOCGWINSZ, &ws)>=0 + || ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws)>=0 + || ioctl(STDERR_FILENO, TIOCGWINSZ, &ws)>=0 + ){ + return ws.ws_col; } +#elif defined(_WIN32) + CONSOLE_SCREEN_BUFFER_INFO csbi; + if( GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi) + || GetConsoleScreenBufferInfo(GetStdHandle(STD_ERROR_HANDLE), &csbi) + || GetConsoleScreenBufferInfo(GetStdHandle(STD_INPUT_HANDLE), &csbi) + ){ + return csbi.srWindow.Right - csbi.srWindow.Left + 1; + } +#endif +#define DEFAULT_SCREEN_WIDTH 80 + return DEFAULT_SCREEN_WIDTH; } - if( bSep ){ - sqlite3_fputs(p->colSeparator, p->out); - } -} - -/* -** This routine runs when the user presses Ctrl-C -*/ -static void interrupt_handler(int NotUsed){ - UNUSED_PARAMETER(NotUsed); - if( ++seenInterrupt>1 ) exit(1); - if( globalDb ) sqlite3_interrupt(globalDb); } #if (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE) @@ -2416,6 +2224,7 @@ static int safeModeAuth( "fts3_tokenizer", "load_extension", "readfile", + "realpath", "writefile", "zipfile", "zipfile_cds", @@ -2478,23 +2287,23 @@ static int shellAuth( az[1] = zA2; az[2] = zA3; az[3] = zA4; - sqlite3_fprintf(p->out, "authorizer: %s", azAction[op]); + cli_printf(p->out, "authorizer: %s", azAction[op]); for(i=0; i<4; i++){ - sqlite3_fputs(" ", p->out); + cli_puts(" ", p->out); if( az[i] ){ output_c_string(p->out, az[i]); }else{ - sqlite3_fputs("NULL", p->out); + cli_puts("NULL", p->out); } } - sqlite3_fputs("\n", p->out); + cli_puts("\n", p->out); if( p->bSafeMode ) (void)safeModeAuth(pClientData, op, zA1, zA2, zA3, zA4); return SQLITE_OK; } #endif /* -** Print a schema statement. Part of MODE_Semi and MODE_Pretty output. +** Print a schema statement. This is helper routine to dump_callbac(). ** ** This routine converts some CREATE TABLE statements for shadow tables ** in FTS3/4/5 into CREATE TABLE IF NOT EXISTS statements. @@ -2525,18 +2334,12 @@ static void printSchemaLine(FILE *out, const char *z, const char *zTail){ } } if( sqlite3_strglob("CREATE TABLE ['\"]*", z)==0 ){ - sqlite3_fprintf(out, "CREATE TABLE IF NOT EXISTS %s%s", z+13, zTail); + cli_printf(out, "CREATE TABLE IF NOT EXISTS %s%s", z+13, zTail); }else{ - sqlite3_fprintf(out, "%s%s", z, zTail); + cli_printf(out, "%s%s", z, zTail); } sqlite3_free(zToFree); } -static void printSchemaLineN(FILE *out, char *z, int n, const char *zTail){ - char c = z[n]; - z[n] = 0; - printSchemaLine(out, z, zTail); - z[n] = c; -} /* ** Return true if string z[] has nothing but whitespace and comments to the @@ -2554,95 +2357,130 @@ static int wsToEol(const char *z){ } /* -** Add a new entry to the EXPLAIN QUERY PLAN data -*/ -static void eqp_append(ShellState *p, int iEqpId, int p2, const char *zText){ - EQPGraphRow *pNew; - i64 nText; - if( zText==0 ) return; - nText = strlen(zText); - if( p->autoEQPtest ){ - sqlite3_fprintf(p->out, "%d,%d,%s\n", iEqpId, p2, zText); - } - pNew = sqlite3_malloc64( sizeof(*pNew) + nText ); - shell_check_oom(pNew); - pNew->iEqpId = iEqpId; - pNew->iParentId = p2; - memcpy(pNew->zText, zText, nText+1); - pNew->pNext = 0; - if( p->sGraph.pLast ){ - p->sGraph.pLast->pNext = pNew; - }else{ - p->sGraph.pRow = pNew; - } - p->sGraph.pLast = pNew; -} - -/* -** Free and reset the EXPLAIN QUERY PLAN data that has been collected -** in p->sGraph. +** SQL Function: shell_format_schema(SQL,FLAGS) +** +** This function is internally by the CLI to assist with the +** ".schema", ".fullschema", and ".dump" commands. The first +** argument is the value from sqlite_schema.sql. The value returned +** is a modification of the input that can actually be run as SQL +** to recreate the schema object. +** +** When FLAGS is zero, the only changes is to append ";". If the +** 0x01 bit of FLAGS is set, then transformations are made to implement +** ".schema --indent". */ -static void eqp_reset(ShellState *p){ - EQPGraphRow *pRow, *pNext; - for(pRow = p->sGraph.pRow; pRow; pRow = pNext){ - pNext = pRow->pNext; - sqlite3_free(pRow); +static void shellFormatSchema( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + int flags; /* Value of 2nd parameter */ + const char *zSql; /* Value of 1st parameter */ + int nSql; /* Bytes of text in zSql[] */ + sqlite3_str *pOut; /* Output buffer */ + char *z; /* Writable copy of zSql */ + int i, j; /* Loop counters */ + int nParen = 0; + char cEnd = 0; + char c; + int nLine = 0; + int isIndex; + int isWhere = 0; + + assert( nVal==2 ); + pOut = sqlite3_str_new(sqlite3_context_db_handle(pCtx)); + nSql = sqlite3_value_bytes(apVal[0]); + zSql = (const char*)sqlite3_value_text(apVal[0]); + if( zSql==0 || zSql[0]==0 ) goto shellFormatSchema_finish; + flags = sqlite3_value_int(apVal[1]); + if( (flags & 0x01)==0 ){ + sqlite3_str_append(pOut, zSql, nSql); + sqlite3_str_append(pOut, ";", 1); + goto shellFormatSchema_finish; + } + if( sqlite3_strlike("CREATE VIEW%", zSql, 0)==0 + || sqlite3_strlike("CREATE TRIG%", zSql, 0)==0 + ){ + sqlite3_str_append(pOut, zSql, nSql); + sqlite3_str_append(pOut, ";", 1); + goto shellFormatSchema_finish; } - memset(&p->sGraph, 0, sizeof(p->sGraph)); -} - -/* Return the next EXPLAIN QUERY PLAN line with iEqpId that occurs after -** pOld, or return the first such line if pOld is NULL -*/ -static EQPGraphRow *eqp_next_row(ShellState *p, int iEqpId, EQPGraphRow *pOld){ - EQPGraphRow *pRow = pOld ? pOld->pNext : p->sGraph.pRow; - while( pRow && pRow->iParentId!=iEqpId ) pRow = pRow->pNext; - return pRow; -} - -/* Render a single level of the graph that has iEqpId as its parent. Called -** recursively to render sublevels. -*/ -static void eqp_render_level(ShellState *p, int iEqpId){ - EQPGraphRow *pRow, *pNext; - i64 n = strlen(p->sGraph.zPrefix); - char *z; - for(pRow = eqp_next_row(p, iEqpId, 0); pRow; pRow = pNext){ - pNext = eqp_next_row(p, iEqpId, pRow); - z = pRow->zText; - sqlite3_fprintf(p->out, "%s%s%s\n", p->sGraph.zPrefix, - pNext ? "|--" : "`--", z); - if( n<(i64)sizeof(p->sGraph.zPrefix)-7 ){ - memcpy(&p->sGraph.zPrefix[n], pNext ? "| " : " ", 4); - eqp_render_level(p, pRow->iEqpId); - p->sGraph.zPrefix[n] = 0; - } + isIndex = sqlite3_strlike("CREATE INDEX%", zSql, 0)==0 + || sqlite3_strlike("CREATE UNIQUE INDEX%", zSql, 0)==0; + z = sqlite3_mprintf("%s", zSql); + if( z==0 ){ + sqlite3_str_free(pOut); + sqlite3_result_error_nomem(pCtx); + return; } -} - -/* -** Display and reset the EXPLAIN QUERY PLAN data -*/ -static void eqp_render(ShellState *p, i64 nCycle){ - EQPGraphRow *pRow = p->sGraph.pRow; - if( pRow ){ - if( pRow->zText[0]=='-' ){ - if( pRow->pNext==0 ){ - eqp_reset(p); - return; + j = 0; + for(i=0; IsSpace(z[i]); i++){} + for(; (c = z[i])!=0; i++){ + if( IsSpace(c) ){ + if( z[j-1]=='\r' ) z[j-1] = '\n'; + if( IsSpace(z[j-1]) || z[j-1]=='(' ) continue; + }else if( (c=='(' || c==')') && j>0 && IsSpace(z[j-1]) ){ + j--; + } + z[j++] = c; + } + while( j>0 && IsSpace(z[j-1]) ){ j--; } + z[j] = 0; + if( strlen30(z)>=79 ){ + for(i=j=0; (c = z[i])!=0; i++){ /* Copy from z[i] back to z[j] */ + if( c==cEnd ){ + cEnd = 0; + }else if( cEnd!=0){ + /* No-op */ + }else if( c=='"' || c=='\'' || c=='`' ){ + cEnd = c; + }else if( c=='[' ){ + cEnd = ']'; + }else if( c=='-' && z[i+1]=='-' ){ + cEnd = '\n'; + }else if( c=='(' ){ + nParen++; + }else if( c==')' ){ + nParen--; + if( nLine>0 && nParen==0 && j>0 && !isWhere ){ + sqlite3_str_append(pOut, z, j); + sqlite3_str_append(pOut, "\n", 1); + j = 0; + } + }else if( (c=='w' || c=='W') + && nParen==0 && isIndex + && sqlite3_strnicmp("WHERE",&z[i],5)==0 + && !IsAlnum(z[i+5]) && z[i+5]!='_' ){ + isWhere = 1; + }else if( isWhere && (c=='A' || c=='a') + && nParen==0 + && sqlite3_strnicmp("AND",&z[i],3)==0 + && !IsAlnum(z[i+3]) && z[i+3]!='_' ){ + sqlite3_str_append(pOut, z, j); + sqlite3_str_append(pOut, "\n ", 5); + j = 0; + } + z[j++] = c; + if( nParen==1 && cEnd==0 + && (c=='(' || c=='\n' || (c==',' && !wsToEol(z+i+1))) + && !isWhere + ){ + if( c=='\n' ) j--; + sqlite3_str_append(pOut, z, j); + sqlite3_str_append(pOut, "\n ", 3); + j = 0; + nLine++; + while( IsSpace(z[i+1]) ){ i++; } } - sqlite3_fprintf(p->out, "%s\n", pRow->zText+3); - p->sGraph.pRow = pRow->pNext; - sqlite3_free(pRow); - }else if( nCycle>0 ){ - sqlite3_fprintf(p->out, "QUERY PLAN (cycles=%lld [100%%])\n", nCycle); - }else{ - sqlite3_fputs("QUERY PLAN\n", p->out); } - p->sGraph.zPrefix[0] = 0; - eqp_render_level(p, 0); - eqp_reset(p); + z[j] = 0; } + sqlite3_str_appendall(pOut, z); + sqlite3_str_append(pOut, ";", 1); + sqlite3_free(z); + +shellFormatSchema_finish: + sqlite3_result_text(pCtx, sqlite3_str_finish(pOut), -1, sqlite3_free); } #ifndef SQLITE_OMIT_PROGRESS_CALLBACK @@ -2652,493 +2490,26 @@ static void eqp_render(ShellState *p, i64 nCycle){ static int progress_handler(void *pClientData) { ShellState *p = (ShellState*)pClientData; p->nProgress++; + if( (p->flgProgress & SHELL_PROGRESS_TMOUT)!=0 + && ELAPSE_TIME(p)>=p->tmProgress + ){ + cli_printf(p->out, "Progress timeout after %.6f seconds\n", + ELAPSE_TIME(p)); + return 1; + } if( p->nProgress>=p->mxProgress && p->mxProgress>0 ){ - sqlite3_fprintf(p->out, "Progress limit reached (%u)\n", p->nProgress); + cli_printf(p->out, "Progress limit reached (%u)\n", p->nProgress); if( p->flgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0; if( p->flgProgress & SHELL_PROGRESS_ONCE ) p->mxProgress = 0; return 1; } if( (p->flgProgress & SHELL_PROGRESS_QUIET)==0 ){ - sqlite3_fprintf(p->out, "Progress %u\n", p->nProgress); + cli_printf(p->out, "Progress %u\n", p->nProgress); } return 0; } #endif /* SQLITE_OMIT_PROGRESS_CALLBACK */ -/* -** Print N dashes -*/ -static void print_dashes(FILE *out, int N){ - const char zDash[] = "--------------------------------------------------"; - const int nDash = sizeof(zDash) - 1; - while( N>nDash ){ - sqlite3_fputs(zDash, out); - N -= nDash; - } - sqlite3_fprintf(out, "%.*s", N, zDash); -} - -/* -** Print a markdown or table-style row separator using ascii-art -*/ -static void print_row_separator( - ShellState *p, - int nArg, - const char *zSep -){ - int i; - if( nArg>0 ){ - sqlite3_fputs(zSep, p->out); - print_dashes(p->out, p->actualWidth[0]+2); - for(i=1; i<nArg; i++){ - sqlite3_fputs(zSep, p->out); - print_dashes(p->out, p->actualWidth[i]+2); - } - sqlite3_fputs(zSep, p->out); - } - sqlite3_fputs("\n", p->out); -} - -/* -** This is the callback routine that the shell -** invokes for each row of a query result. -*/ -static int shell_callback( - void *pArg, - int nArg, /* Number of result columns */ - char **azArg, /* Text of each result column */ - char **azCol, /* Column names */ - int *aiType /* Column types. Might be NULL */ -){ - int i; - ShellState *p = (ShellState*)pArg; - - if( azArg==0 ) return 0; - switch( p->cMode ){ - case MODE_Count: - case MODE_Off: { - break; - } - case MODE_Line: { - int w = 5; - if( azArg==0 ) break; - for(i=0; i<nArg; i++){ - int len = strlen30(azCol[i] ? azCol[i] : ""); - if( len>w ) w = len; - } - if( p->cnt++>0 ) sqlite3_fputs(p->rowSeparator, p->out); - for(i=0; i<nArg; i++){ - char *pFree = 0; - const char *pDisplay; - pDisplay = escapeOutput(p, azArg[i] ? azArg[i] : p->nullValue, &pFree); - sqlite3_fprintf(p->out, "%*s = %s%s", w, azCol[i], - pDisplay, p->rowSeparator); - if( pFree ) sqlite3_free(pFree); - } - break; - } - case MODE_ScanExp: - case MODE_Explain: { - static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13}; - static const int aExplainMap[] = {0, 1, 2, 3, 4, 5, 6, 7 }; - static const int aScanExpWidth[] = {4, 15, 6, 13, 4, 4, 4, 13, 2, 13}; - static const int aScanExpMap[] = {0, 9, 8, 1, 2, 3, 4, 5, 6, 7 }; - - const int *aWidth = aExplainWidth; - const int *aMap = aExplainMap; - int nWidth = ArraySize(aExplainWidth); - int iIndent = 1; - - if( p->cMode==MODE_ScanExp ){ - aWidth = aScanExpWidth; - aMap = aScanExpMap; - nWidth = ArraySize(aScanExpWidth); - iIndent = 3; - } - if( nArg>nWidth ) nArg = nWidth; - - /* If this is the first row seen, print out the headers */ - if( p->cnt++==0 ){ - for(i=0; i<nArg; i++){ - utf8_width_print(p->out, aWidth[i], azCol[ aMap[i] ]); - sqlite3_fputs(i==nArg-1 ? "\n" : " ", p->out); - } - for(i=0; i<nArg; i++){ - print_dashes(p->out, aWidth[i]); - sqlite3_fputs(i==nArg-1 ? "\n" : " ", p->out); - } - } - - /* If there is no data, exit early. */ - if( azArg==0 ) break; - - for(i=0; i<nArg; i++){ - const char *zSep = " "; - int w = aWidth[i]; - const char *zVal = azArg[ aMap[i] ]; - if( i==nArg-1 ) w = 0; - if( zVal && strlenChar(zVal)>w ){ - w = strlenChar(zVal); - zSep = " "; - } - if( i==iIndent && p->aiIndent && p->pStmt ){ - if( p->iIndent<p->nIndent ){ - sqlite3_fprintf(p->out, "%*.s", p->aiIndent[p->iIndent], ""); - } - p->iIndent++; - } - utf8_width_print(p->out, w, zVal ? zVal : p->nullValue); - sqlite3_fputs(i==nArg-1 ? "\n" : zSep, p->out); - } - break; - } - case MODE_Semi: { /* .schema and .fullschema output */ - printSchemaLine(p->out, azArg[0], ";\n"); - break; - } - case MODE_Pretty: { /* .schema and .fullschema with --indent */ - char *z; - int j; - int nParen = 0; - char cEnd = 0; - char c; - int nLine = 0; - int isIndex; - int isWhere = 0; - assert( nArg==1 ); - if( azArg[0]==0 ) break; - if( sqlite3_strlike("CREATE VIEW%", azArg[0], 0)==0 - || sqlite3_strlike("CREATE TRIG%", azArg[0], 0)==0 - ){ - sqlite3_fprintf(p->out, "%s;\n", azArg[0]); - break; - } - isIndex = sqlite3_strlike("CREATE INDEX%", azArg[0], 0)==0 - || sqlite3_strlike("CREATE UNIQUE INDEX%", azArg[0], 0)==0; - z = sqlite3_mprintf("%s", azArg[0]); - shell_check_oom(z); - j = 0; - for(i=0; IsSpace(z[i]); i++){} - for(; (c = z[i])!=0; i++){ - if( IsSpace(c) ){ - if( z[j-1]=='\r' ) z[j-1] = '\n'; - if( IsSpace(z[j-1]) || z[j-1]=='(' ) continue; - }else if( (c=='(' || c==')') && j>0 && IsSpace(z[j-1]) ){ - j--; - } - z[j++] = c; - } - while( j>0 && IsSpace(z[j-1]) ){ j--; } - z[j] = 0; - if( strlen30(z)>=79 ){ - for(i=j=0; (c = z[i])!=0; i++){ /* Copy from z[i] back to z[j] */ - if( c==cEnd ){ - cEnd = 0; - }else if( c=='"' || c=='\'' || c=='`' ){ - cEnd = c; - }else if( c=='[' ){ - cEnd = ']'; - }else if( c=='-' && z[i+1]=='-' ){ - cEnd = '\n'; - }else if( c=='(' ){ - nParen++; - }else if( c==')' ){ - nParen--; - if( nLine>0 && nParen==0 && j>0 && !isWhere ){ - printSchemaLineN(p->out, z, j, "\n"); - j = 0; - } - }else if( (c=='w' || c=='W') - && nParen==0 && isIndex - && sqlite3_strnicmp("WHERE",&z[i],5)==0 - && !IsAlnum(z[i+5]) && z[i+5]!='_' ){ - isWhere = 1; - }else if( isWhere && (c=='A' || c=='a') - && nParen==0 - && sqlite3_strnicmp("AND",&z[i],3)==0 - && !IsAlnum(z[i+3]) && z[i+3]!='_' ){ - printSchemaLineN(p->out, z, j, "\n "); - j = 0; - } - z[j++] = c; - if( nParen==1 && cEnd==0 - && (c=='(' || c=='\n' || (c==',' && !wsToEol(z+i+1))) - && !isWhere - ){ - if( c=='\n' ) j--; - printSchemaLineN(p->out, z, j, "\n "); - j = 0; - nLine++; - while( IsSpace(z[i+1]) ){ i++; } - } - } - z[j] = 0; - } - printSchemaLine(p->out, z, ";\n"); - sqlite3_free(z); - break; - } - case MODE_List: { - if( p->cnt++==0 && p->showHeader ){ - for(i=0; i<nArg; i++){ - char *z = azCol[i]; - char *pFree; - const char *zOut = escapeOutput(p, z, &pFree); - sqlite3_fprintf(p->out, "%s%s", zOut, - i==nArg-1 ? p->rowSeparator : p->colSeparator); - if( pFree ) sqlite3_free(pFree); - } - } - if( azArg==0 ) break; - for(i=0; i<nArg; i++){ - char *z = azArg[i]; - char *pFree; - const char *zOut; - if( z==0 ) z = p->nullValue; - zOut = escapeOutput(p, z, &pFree); - sqlite3_fputs(zOut, p->out); - if( pFree ) sqlite3_free(pFree); - sqlite3_fputs((i<nArg-1)? p->colSeparator : p->rowSeparator, p->out); - } - break; - } - case MODE_Www: - case MODE_Html: { - if( p->cnt==0 && p->cMode==MODE_Www ){ - sqlite3_fputs( - "</PRE>\n" - "<TABLE border='1' cellspacing='0' cellpadding='2'>\n" - ,p->out - ); - } - if( p->cnt==0 && (p->showHeader || p->cMode==MODE_Www) ){ - sqlite3_fputs("<TR>", p->out); - for(i=0; i<nArg; i++){ - sqlite3_fputs("<TH>", p->out); - output_html_string(p->out, azCol[i]); - sqlite3_fputs("</TH>\n", p->out); - } - sqlite3_fputs("</TR>\n", p->out); - } - p->cnt++; - if( azArg==0 ) break; - sqlite3_fputs("<TR>", p->out); - for(i=0; i<nArg; i++){ - sqlite3_fputs("<TD>", p->out); - output_html_string(p->out, azArg[i] ? azArg[i] : p->nullValue); - sqlite3_fputs("</TD>\n", p->out); - } - sqlite3_fputs("</TR>\n", p->out); - break; - } - case MODE_Tcl: { - if( p->cnt++==0 && p->showHeader ){ - for(i=0; i<nArg; i++){ - output_c_string(p->out, azCol[i] ? azCol[i] : ""); - if(i<nArg-1) sqlite3_fputs(p->colSeparator, p->out); - } - sqlite3_fputs(p->rowSeparator, p->out); - } - if( azArg==0 ) break; - for(i=0; i<nArg; i++){ - output_c_string(p->out, azArg[i] ? azArg[i] : p->nullValue); - if(i<nArg-1) sqlite3_fputs(p->colSeparator, p->out); - } - sqlite3_fputs(p->rowSeparator, p->out); - break; - } - case MODE_Csv: { - sqlite3_fsetmode(p->out, _O_BINARY); - if( p->cnt++==0 && p->showHeader ){ - for(i=0; i<nArg; i++){ - output_csv(p, azCol[i] ? azCol[i] : "", i<nArg-1); - } - sqlite3_fputs(p->rowSeparator, p->out); - } - if( nArg>0 ){ - for(i=0; i<nArg; i++){ - output_csv(p, azArg[i], i<nArg-1); - } - sqlite3_fputs(p->rowSeparator, p->out); - } - setCrlfMode(p); - break; - } - case MODE_Insert: { - if( azArg==0 ) break; - sqlite3_fprintf(p->out, "INSERT INTO %s",p->zDestTable); - if( p->showHeader ){ - sqlite3_fputs("(", p->out); - for(i=0; i<nArg; i++){ - if( i>0 ) sqlite3_fputs(",", p->out); - if( quoteChar(azCol[i]) ){ - char *z = sqlite3_mprintf("\"%w\"", azCol[i]); - shell_check_oom(z); - sqlite3_fputs(z, p->out); - sqlite3_free(z); - }else{ - sqlite3_fprintf(p->out, "%s", azCol[i]); - } - } - sqlite3_fputs(")", p->out); - } - p->cnt++; - for(i=0; i<nArg; i++){ - sqlite3_fputs(i>0 ? "," : " VALUES(", p->out); - if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ - sqlite3_fputs("NULL", p->out); - }else if( aiType && aiType[i]==SQLITE_TEXT ){ - if( ShellHasFlag(p, SHFLG_Newlines) ){ - output_quoted_string(p, azArg[i]); - }else{ - output_quoted_escaped_string(p, azArg[i]); - } - }else if( aiType && aiType[i]==SQLITE_INTEGER ){ - sqlite3_fputs(azArg[i], p->out); - }else if( aiType && aiType[i]==SQLITE_FLOAT ){ - char z[50]; - double r = sqlite3_column_double(p->pStmt, i); - sqlite3_uint64 ur; - memcpy(&ur,&r,sizeof(r)); - if( ur==0x7ff0000000000000LL ){ - sqlite3_fputs("9.0e+999", p->out); - }else if( ur==0xfff0000000000000LL ){ - sqlite3_fputs("-9.0e+999", p->out); - }else{ - sqlite3_int64 ir = (sqlite3_int64)r; - if( r==(double)ir ){ - sqlite3_snprintf(50,z,"%lld.0", ir); - }else{ - sqlite3_snprintf(50,z,"%!.20g", r); - } - sqlite3_fputs(z, p->out); - } - }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ - const void *pBlob = sqlite3_column_blob(p->pStmt, i); - int nBlob = sqlite3_column_bytes(p->pStmt, i); - output_hex_blob(p->out, pBlob, nBlob); - }else if( isNumber(azArg[i], 0) ){ - sqlite3_fputs(azArg[i], p->out); - }else if( ShellHasFlag(p, SHFLG_Newlines) ){ - output_quoted_string(p, azArg[i]); - }else{ - output_quoted_escaped_string(p, azArg[i]); - } - } - sqlite3_fputs(");\n", p->out); - break; - } - case MODE_Json: { - if( azArg==0 ) break; - if( p->cnt==0 ){ - sqlite3_fputs("[{", p->out); - }else{ - sqlite3_fputs(",\n{", p->out); - } - p->cnt++; - for(i=0; i<nArg; i++){ - output_json_string(p->out, azCol[i], -1); - sqlite3_fputs(":", p->out); - if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ - sqlite3_fputs("null", p->out); - }else if( aiType && aiType[i]==SQLITE_FLOAT ){ - char z[50]; - double r = sqlite3_column_double(p->pStmt, i); - sqlite3_uint64 ur; - memcpy(&ur,&r,sizeof(r)); - if( ur==0x7ff0000000000000LL ){ - sqlite3_fputs("9.0e+999", p->out); - }else if( ur==0xfff0000000000000LL ){ - sqlite3_fputs("-9.0e+999", p->out); - }else{ - sqlite3_snprintf(50,z,"%!.20g", r); - sqlite3_fputs(z, p->out); - } - }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ - const void *pBlob = sqlite3_column_blob(p->pStmt, i); - int nBlob = sqlite3_column_bytes(p->pStmt, i); - output_json_string(p->out, pBlob, nBlob); - }else if( aiType && aiType[i]==SQLITE_TEXT ){ - output_json_string(p->out, azArg[i], -1); - }else{ - sqlite3_fputs(azArg[i], p->out); - } - if( i<nArg-1 ){ - sqlite3_fputs(",", p->out); - } - } - sqlite3_fputs("}", p->out); - break; - } - case MODE_Quote: { - if( azArg==0 ) break; - if( p->cnt==0 && p->showHeader ){ - for(i=0; i<nArg; i++){ - if( i>0 ) sqlite3_fputs(p->colSeparator, p->out); - output_quoted_string(p, azCol[i]); - } - sqlite3_fputs(p->rowSeparator, p->out); - } - p->cnt++; - for(i=0; i<nArg; i++){ - if( i>0 ) sqlite3_fputs(p->colSeparator, p->out); - if( (azArg[i]==0) || (aiType && aiType[i]==SQLITE_NULL) ){ - sqlite3_fputs("NULL", p->out); - }else if( aiType && aiType[i]==SQLITE_TEXT ){ - output_quoted_string(p, azArg[i]); - }else if( aiType && aiType[i]==SQLITE_INTEGER ){ - sqlite3_fputs(azArg[i], p->out); - }else if( aiType && aiType[i]==SQLITE_FLOAT ){ - char z[50]; - double r = sqlite3_column_double(p->pStmt, i); - sqlite3_snprintf(50,z,"%!.20g", r); - sqlite3_fputs(z, p->out); - }else if( aiType && aiType[i]==SQLITE_BLOB && p->pStmt ){ - const void *pBlob = sqlite3_column_blob(p->pStmt, i); - int nBlob = sqlite3_column_bytes(p->pStmt, i); - output_hex_blob(p->out, pBlob, nBlob); - }else if( isNumber(azArg[i], 0) ){ - sqlite3_fputs(azArg[i], p->out); - }else{ - output_quoted_string(p, azArg[i]); - } - } - sqlite3_fputs(p->rowSeparator, p->out); - break; - } - case MODE_Ascii: { - if( p->cnt++==0 && p->showHeader ){ - for(i=0; i<nArg; i++){ - if( i>0 ) sqlite3_fputs(p->colSeparator, p->out); - sqlite3_fputs(azCol[i] ? azCol[i] : "", p->out); - } - sqlite3_fputs(p->rowSeparator, p->out); - } - if( azArg==0 ) break; - for(i=0; i<nArg; i++){ - if( i>0 ) sqlite3_fputs(p->colSeparator, p->out); - sqlite3_fputs(azArg[i] ? azArg[i] : p->nullValue, p->out); - } - sqlite3_fputs(p->rowSeparator, p->out); - break; - } - case MODE_EQP: { - eqp_append(p, atoi(azArg[0]), atoi(azArg[1]), azArg[3]); - break; - } - } - return 0; -} - -/* -** This is the callback routine that the SQLite library -** invokes for each row of a query result. -*/ -static int callback(void *pArg, int nArg, char **azArg, char **azCol){ - /* since we don't have type info, call the shell_callback with a NULL value */ - return shell_callback(pArg, nArg, azArg, azCol, NULL); -} - /* ** This is the callback routine from sqlite3_exec() that appends all ** output onto the end of a ShellText object. @@ -3198,7 +2569,7 @@ static void createSelftestTable(ShellState *p){ "DROP TABLE [_shell$self];" ,0,0,&zErrMsg); if( zErrMsg ){ - sqlite3_fprintf(stderr, "SELFTEST initialization failure: %s\n", zErrMsg); + cli_printf(stderr, "SELFTEST initialization failure: %s\n", zErrMsg); sqlite3_free(zErrMsg); } sqlite3_exec(p->db, "RELEASE selftest_init",0,0,0); @@ -3216,11 +2587,7 @@ static void set_table_name(ShellState *p, const char *zName){ p->zDestTable = 0; } if( zName==0 ) return; - if( quoteChar(zName) ){ - p->zDestTable = sqlite3_mprintf("\"%w\"", zName); - }else{ - p->zDestTable = sqlite3_mprintf("%s", zName); - } + p->zDestTable = sqlite3_mprintf("%s", zName); shell_check_oom(p->zDestTable); } @@ -3290,7 +2657,7 @@ static int run_table_dump_query( rc = sqlite3_prepare_v2(p->db, zSelect, -1, &pSelect, 0); if( rc!=SQLITE_OK || !pSelect ){ char *zContext = shell_error_context(zSelect, p->db); - sqlite3_fprintf(p->out, "/**** ERROR: (%d) %s *****/\n%s", + cli_printf(p->out, "/**** ERROR: (%d) %s *****/\n%s", rc, sqlite3_errmsg(p->db), zContext); sqlite3_free(zContext); if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; @@ -3300,22 +2667,22 @@ static int run_table_dump_query( nResult = sqlite3_column_count(pSelect); while( rc==SQLITE_ROW ){ z = (const char*)sqlite3_column_text(pSelect, 0); - sqlite3_fprintf(p->out, "%s", z); + cli_printf(p->out, "%s", z); for(i=1; i<nResult; i++){ - sqlite3_fprintf(p->out, ",%s", sqlite3_column_text(pSelect, i)); + cli_printf(p->out, ",%s", sqlite3_column_text(pSelect, i)); } if( z==0 ) z = ""; while( z[0] && (z[0]!='-' || z[1]!='-') ) z++; if( z[0] ){ - sqlite3_fputs("\n;\n", p->out); + cli_puts("\n;\n", p->out); }else{ - sqlite3_fputs(";\n", p->out); + cli_puts(";\n", p->out); } rc = sqlite3_step(pSelect); } rc = sqlite3_finalize(pSelect); if( rc!=SQLITE_OK ){ - sqlite3_fprintf(p->out, "/**** ERROR: (%d) %s *****/\n", + cli_printf(p->out, "/**** ERROR: (%d) %s *****/\n", rc, sqlite3_errmsg(p->db)); if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; } @@ -3375,7 +2742,7 @@ static void displayLinuxIoStats(FILE *out){ for(i=0; i<ArraySize(aTrans); i++){ int n = strlen30(aTrans[i].zPattern); if( cli_strncmp(aTrans[i].zPattern, z, n)==0 ){ - sqlite3_fprintf(out, "%-36s %s", aTrans[i].zDesc, &z[n]); + cli_printf(out, "%-36s %s", aTrans[i].zDesc, &z[n]); break; } } @@ -3407,7 +2774,7 @@ static void displayStatLine( }else{ sqlite3_snprintf(sizeof(zLine), zLine, zFormat, iHiwtr); } - sqlite3_fprintf(out, "%-36s %s\n", zLabel, zLine); + cli_printf(out, "%-36s %s\n", zLabel, zLine); } /* @@ -3429,22 +2796,22 @@ static int display_stats( sqlite3_stmt *pStmt = pArg->pStmt; char z[100]; nCol = sqlite3_column_count(pStmt); - sqlite3_fprintf(out, "%-36s %d\n", "Number of output columns:", nCol); + cli_printf(out, "%-36s %d\n", "Number of output columns:", nCol); for(i=0; i<nCol; i++){ sqlite3_snprintf(sizeof(z),z,"Column %d %nname:", i, &x); - sqlite3_fprintf(out, "%-36s %s\n", z, sqlite3_column_name(pStmt,i)); + cli_printf(out, "%-36s %s\n", z, sqlite3_column_name(pStmt,i)); #ifndef SQLITE_OMIT_DECLTYPE sqlite3_snprintf(30, z+x, "declared type:"); - sqlite3_fprintf(out, "%-36s %s\n", z, sqlite3_column_decltype(pStmt, i)); + cli_printf(out, "%-36s %s\n", z, sqlite3_column_decltype(pStmt, i)); #endif #ifdef SQLITE_ENABLE_COLUMN_METADATA sqlite3_snprintf(30, z+x, "database name:"); - sqlite3_fprintf(out, "%-36s %s\n", z, + cli_printf(out, "%-36s %s\n", z, sqlite3_column_database_name(pStmt,i)); sqlite3_snprintf(30, z+x, "table name:"); - sqlite3_fprintf(out, "%-36s %s\n", z, sqlite3_column_table_name(pStmt,i)); + cli_printf(out, "%-36s %s\n", z, sqlite3_column_table_name(pStmt,i)); sqlite3_snprintf(30, z+x, "origin name:"); - sqlite3_fprintf(out, "%-36s %s\n", z,sqlite3_column_origin_name(pStmt,i)); + cli_printf(out, "%-36s %s\n", z,sqlite3_column_origin_name(pStmt,i)); #endif } } @@ -3452,7 +2819,7 @@ static int display_stats( if( pArg->statsOn==3 ){ if( pArg->pStmt ){ iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP,bReset); - sqlite3_fprintf(out, "VM-steps: %d\n", iCur); + cli_printf(out, "VM-steps: %d\n", iCur); } return 0; } @@ -3481,55 +2848,55 @@ static int display_stats( iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, &iCur, &iHiwtr, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Lookaside Slots Used: %d (max %d)\n", iCur, iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT, &iCur, &iHiwtr, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Successful lookaside attempts: %d\n", iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, &iCur, &iHiwtr, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Lookaside failures due to size: %d\n", iHiwtr); sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, &iCur, &iHiwtr, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Lookaside failures due to OOM: %d\n", iHiwtr); } iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Pager Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1); - sqlite3_fprintf(out, + cli_printf(out, "Page cache hits: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_MISS, &iCur, &iHiwtr, 1); - sqlite3_fprintf(out, + cli_printf(out, "Page cache misses: %d\n", iCur); iHiwtr64 = iCur64 = -1; sqlite3_db_status64(db, SQLITE_DBSTATUS_TEMPBUF_SPILL, &iCur64, &iHiwtr64, 0); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_WRITE, &iCur, &iHiwtr, 1); - sqlite3_fprintf(out, + cli_printf(out, "Page cache writes: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_SPILL, &iCur, &iHiwtr, 1); - sqlite3_fprintf(out, + cli_printf(out, "Page cache spills: %d\n", iCur); - sqlite3_fprintf(out, + cli_printf(out, "Temporary data spilled to disk: %lld\n", iCur64); sqlite3_db_status64(db, SQLITE_DBSTATUS_TEMPBUF_SPILL, &iCur64, &iHiwtr64, 1); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Schema Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_STMT_USED, &iCur, &iHiwtr, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Statement Heap/Lookaside Usage: %d bytes\n", iCur); } @@ -3537,33 +2904,33 @@ static int display_stats( int iHit, iMiss; iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Fullscan Steps: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_SORT, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Sort Operations: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX,bReset); - sqlite3_fprintf(out, + cli_printf(out, "Autoindex Inserts: %d\n", iCur); iHit = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_HIT, bReset); iMiss = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FILTER_MISS, bReset); if( iHit || iMiss ){ - sqlite3_fprintf(out, + cli_printf(out, "Bloom filter bypass taken: %d/%d\n", iHit, iHit+iMiss); } iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Virtual Machine Steps: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_REPREPARE,bReset); - sqlite3_fprintf(out, + cli_printf(out, "Reprepare operations: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_RUN, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Number of times run: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_MEMUSED, bReset); - sqlite3_fprintf(out, + cli_printf(out, "Memory used by prepared stmt: %d\n", iCur); } @@ -3576,282 +2943,21 @@ static int display_stats( return 0; } - -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS -static int scanStatsHeight(sqlite3_stmt *p, int iEntry){ - int iPid = 0; - int ret = 1; - sqlite3_stmt_scanstatus_v2(p, iEntry, - SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid - ); - while( iPid!=0 ){ - int ii; - for(ii=0; 1; ii++){ - int iId; - int res; - res = sqlite3_stmt_scanstatus_v2(p, ii, - SQLITE_SCANSTAT_SELECTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iId - ); - if( res ) break; - if( iId==iPid ){ - sqlite3_stmt_scanstatus_v2(p, ii, - SQLITE_SCANSTAT_PARENTID, SQLITE_SCANSTAT_COMPLEX, (void*)&iPid - ); - } - } - ret++; - } - return ret; -} -#endif - -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS -static void display_explain_scanstats( - sqlite3 *db, /* Database to query */ - ShellState *pArg /* Pointer to ShellState */ -){ - static const int f = SQLITE_SCANSTAT_COMPLEX; - sqlite3_stmt *p = pArg->pStmt; - int ii = 0; - i64 nTotal = 0; - int nWidth = 0; - eqp_reset(pArg); - - for(ii=0; 1; ii++){ - const char *z = 0; - int n = 0; - if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&z) ){ - break; - } - n = (int)strlen(z) + scanStatsHeight(p, ii)*3; - if( n>nWidth ) nWidth = n; - } - nWidth += 4; - - sqlite3_stmt_scanstatus_v2(p, -1, SQLITE_SCANSTAT_NCYCLE, f, (void*)&nTotal); - for(ii=0; 1; ii++){ - i64 nLoop = 0; - i64 nRow = 0; - i64 nCycle = 0; - int iId = 0; - int iPid = 0; - const char *zo = 0; - const char *zName = 0; - char *zText = 0; - double rEst = 0.0; - - if( sqlite3_stmt_scanstatus_v2(p,ii,SQLITE_SCANSTAT_EXPLAIN,f,(void*)&zo) ){ - break; - } - sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_EST,f,(void*)&rEst); - sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NLOOP,f,(void*)&nLoop); - sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NVISIT,f,(void*)&nRow); - sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NCYCLE,f,(void*)&nCycle); - sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_SELECTID,f,(void*)&iId); - sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_PARENTID,f,(void*)&iPid); - sqlite3_stmt_scanstatus_v2(p, ii, SQLITE_SCANSTAT_NAME,f,(void*)&zName); - - zText = sqlite3_mprintf("%s", zo); - if( nCycle>=0 || nLoop>=0 || nRow>=0 ){ - char *z = 0; - if( nCycle>=0 && nTotal>0 ){ - z = sqlite3_mprintf("%zcycles=%lld [%d%%]", z, - nCycle, ((nCycle*100)+nTotal/2) / nTotal - ); - } - if( nLoop>=0 ){ - z = sqlite3_mprintf("%z%sloops=%lld", z, z ? " " : "", nLoop); - } - if( nRow>=0 ){ - z = sqlite3_mprintf("%z%srows=%lld", z, z ? " " : "", nRow); - } - - if( zName && pArg->scanstatsOn>1 ){ - double rpl = (double)nRow / (double)nLoop; - z = sqlite3_mprintf("%z rpl=%.1f est=%.1f", z, rpl, rEst); - } - - zText = sqlite3_mprintf( - "% *z (%z)", -1*(nWidth-scanStatsHeight(p, ii)*3), zText, z - ); - } - - eqp_append(pArg, iId, iPid, zText); - sqlite3_free(zText); - } - - eqp_render(pArg, nTotal); -} -#endif - - -/* -** Parameter azArray points to a zero-terminated array of strings. zStr -** points to a single nul-terminated string. Return non-zero if zStr -** is equal, according to strcmp(), to any of the strings in the array. -** Otherwise, return zero. -*/ -static int str_in_array(const char *zStr, const char **azArray){ - int i; - for(i=0; azArray[i]; i++){ - if( 0==cli_strcmp(zStr, azArray[i]) ) return 1; - } - return 0; -} - -/* -** If compiled statement pSql appears to be an EXPLAIN statement, allocate -** and populate the ShellState.aiIndent[] array with the number of -** spaces each opcode should be indented before it is output. -** -** The indenting rules are: -** -** * For each "Next", "Prev", "VNext" or "VPrev" instruction, indent -** all opcodes that occur between the p2 jump destination and the opcode -** itself by 2 spaces. -** -** * Do the previous for "Return" instructions for when P2 is positive. -** See tag-20220407a in wherecode.c and vdbe.c. -** -** * For each "Goto", if the jump destination is earlier in the program -** and ends on one of: -** Yield SeekGt SeekLt RowSetRead Rewind -** or if the P1 parameter is one instead of zero, -** then indent all opcodes between the earlier instruction -** and "Goto" by 2 spaces. -*/ -static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ - int *abYield = 0; /* True if op is an OP_Yield */ - int nAlloc = 0; /* Allocated size of p->aiIndent[], abYield */ - int iOp; /* Index of operation in p->aiIndent[] */ - - const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", - "Return", 0 }; - const char *azYield[] = { "Yield", "SeekLT", "SeekGT", "RowSetRead", - "Rewind", 0 }; - const char *azGoto[] = { "Goto", 0 }; - - /* The caller guarantees that the leftmost 4 columns of the statement - ** passed to this function are equivalent to the leftmost 4 columns - ** of EXPLAIN statement output. In practice the statement may be - ** an EXPLAIN, or it may be a query on the bytecode() virtual table. */ - assert( sqlite3_column_count(pSql)>=4 ); - assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 0), "addr" ) ); - assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 1), "opcode" ) ); - assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 2), "p1" ) ); - assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 3), "p2" ) ); - - for(iOp=0; SQLITE_ROW==sqlite3_step(pSql); iOp++){ - int i; - int iAddr = sqlite3_column_int(pSql, 0); - const char *zOp = (const char*)sqlite3_column_text(pSql, 1); - int p1 = sqlite3_column_int(pSql, 2); - int p2 = sqlite3_column_int(pSql, 3); - - /* Assuming that p2 is an instruction address, set variable p2op to the - ** index of that instruction in the aiIndent[] array. p2 and p2op may be - ** different if the current instruction is part of a sub-program generated - ** by an SQL trigger or foreign key. */ - int p2op = (p2 + (iOp-iAddr)); - - /* Grow the p->aiIndent array as required */ - if( iOp>=nAlloc ){ - nAlloc += 100; - p->aiIndent = (int*)sqlite3_realloc64(p->aiIndent, nAlloc*sizeof(int)); - shell_check_oom(p->aiIndent); - abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int)); - shell_check_oom(abYield); - } - - abYield[iOp] = str_in_array(zOp, azYield); - p->aiIndent[iOp] = 0; - p->nIndent = iOp+1; - if( str_in_array(zOp, azNext) && p2op>0 ){ - for(i=p2op; i<iOp; i++) p->aiIndent[i] += 2; - } - if( str_in_array(zOp, azGoto) && p2op<iOp && (abYield[p2op] || p1) ){ - for(i=p2op; i<iOp; i++) p->aiIndent[i] += 2; - } - } - - p->iIndent = 0; - sqlite3_free(abYield); - sqlite3_reset(pSql); -} - -/* -** Free the array allocated by explain_data_prepare(). -*/ -static void explain_data_delete(ShellState *p){ - sqlite3_free(p->aiIndent); - p->aiIndent = 0; - p->nIndent = 0; - p->iIndent = 0; -} - -static void exec_prepared_stmt(ShellState*, sqlite3_stmt*); - -/* -** Display scan stats. -*/ -static void display_scanstats( - sqlite3 *db, /* Database to query */ - ShellState *pArg /* Pointer to ShellState */ -){ -#ifndef SQLITE_ENABLE_STMT_SCANSTATUS - UNUSED_PARAMETER(db); - UNUSED_PARAMETER(pArg); -#else - if( pArg->scanstatsOn==3 ){ - const char *zSql = - " SELECT addr, opcode, p1, p2, p3, p4, p5, comment, nexec," - " format('% 6s (%.2f%%)'," - " CASE WHEN ncycle<100_000 THEN ncycle || ' '" - " WHEN ncycle<100_000_000 THEN (ncycle/1_000) || 'K'" - " WHEN ncycle<100_000_000_000 THEN (ncycle/1_000_000) || 'M'" - " ELSE (ncycle/1000_000_000) || 'G' END," - " ncycle*100.0/(sum(ncycle) OVER ())" - " ) AS cycles" - " FROM bytecode(?)"; - - int rc = SQLITE_OK; - sqlite3_stmt *pStmt = 0; - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_stmt *pSave = pArg->pStmt; - pArg->pStmt = pStmt; - sqlite3_bind_pointer(pStmt, 1, pSave, "stmt-pointer", 0); - - pArg->cnt = 0; - pArg->cMode = MODE_ScanExp; - explain_data_prepare(pArg, pStmt); - exec_prepared_stmt(pArg, pStmt); - explain_data_delete(pArg); - - sqlite3_finalize(pStmt); - pArg->pStmt = pSave; - } - }else{ - display_explain_scanstats(db, pArg); - } -#endif -} - -/* -** Disable and restore .wheretrace and .treetrace/.selecttrace settings. -*/ -static unsigned int savedSelectTrace; -static unsigned int savedWhereTrace; -static void disable_debug_trace_modes(void){ - unsigned int zero = 0; - sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 0, &savedSelectTrace); - sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &zero); - sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 2, &savedWhereTrace); - sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &zero); -} -static void restore_debug_trace_modes(void){ - sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &savedSelectTrace); - sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &savedWhereTrace); +/* +** Disable and restore .wheretrace and .treetrace/.selecttrace settings. +*/ +static unsigned int savedSelectTrace; +static unsigned int savedWhereTrace; +static void disable_debug_trace_modes(void){ + unsigned int zero = 0; + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 0, &savedSelectTrace); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &zero); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 2, &savedWhereTrace); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &zero); +} +static void restore_debug_trace_modes(void){ + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 1, &savedSelectTrace); + sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, 3, &savedWhereTrace); } /* Create the TEMP table used to store parameter bindings */ @@ -3928,6 +3034,8 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ memcpy(zBuf, &zVar[6], szVar-5); sqlite3_bind_text64(pStmt, i, zBuf, szVar-6, sqlite3_free, SQLITE_UTF8); } + }else if( strcmp(zVar, "$TIMER")==0 ){ + sqlite3_bind_double(pStmt, i, pArg->prevTimer); #ifdef SQLITE_ENABLE_CARRAY }else if( strncmp(zVar, "$carray_", 8)==0 ){ static char *azColorNames[] = { @@ -3966,580 +3074,6 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ sqlite3_finalize(pQ); } -/* -** UTF8 box-drawing characters. Imagine box lines like this: -** -** 1 -** | -** 4 --+-- 2 -** | -** 3 -** -** Each box characters has between 2 and 4 of the lines leading from -** the center. The characters are here identified by the numbers of -** their corresponding lines. -*/ -#define BOX_24 "\342\224\200" /* U+2500 --- */ -#define BOX_13 "\342\224\202" /* U+2502 | */ -#define BOX_23 "\342\224\214" /* U+250c ,- */ -#define BOX_34 "\342\224\220" /* U+2510 -, */ -#define BOX_12 "\342\224\224" /* U+2514 '- */ -#define BOX_14 "\342\224\230" /* U+2518 -' */ -#define BOX_123 "\342\224\234" /* U+251c |- */ -#define BOX_134 "\342\224\244" /* U+2524 -| */ -#define BOX_234 "\342\224\254" /* U+252c -,- */ -#define BOX_124 "\342\224\264" /* U+2534 -'- */ -#define BOX_1234 "\342\224\274" /* U+253c -|- */ - -/* Draw horizontal line N characters long using unicode box -** characters -*/ -static void print_box_line(FILE *out, int N){ - const char zDash[] = - BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 - BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24 BOX_24; - const int nDash = sizeof(zDash) - 1; - N *= 3; - while( N>nDash ){ - sqlite3_fputs(zDash, out); - N -= nDash; - } - sqlite3_fprintf(out, "%.*s", N, zDash); -} - -/* -** Draw a horizontal separator for a MODE_Box table. -*/ -static void print_box_row_separator( - ShellState *p, - int nArg, - const char *zSep1, - const char *zSep2, - const char *zSep3 -){ - int i; - if( nArg>0 ){ - sqlite3_fputs(zSep1, p->out); - print_box_line(p->out, p->actualWidth[0]+2); - for(i=1; i<nArg; i++){ - sqlite3_fputs(zSep2, p->out); - print_box_line(p->out, p->actualWidth[i]+2); - } - sqlite3_fputs(zSep3, p->out); - } - sqlite3_fputs("\n", p->out); -} - -/* -** z[] is a line of text that is to be displayed the .mode box or table or -** similar tabular formats. z[] might contain control characters such -** as \n, \t, \f, or \r. -** -** Compute characters to display on the first line of z[]. Stop at the -** first \r, \n, or \f. Expand \t into spaces. Return a copy (obtained -** from malloc()) of that first line, which caller should free sometime. -** Write anything to display on the next line into *pzTail. If this is -** the last line, write a NULL into *pzTail. (*pzTail is not allocated.) -*/ -static char *translateForDisplayAndDup( - ShellState *p, /* To access current settings */ - const unsigned char *z, /* Input text to be transformed */ - const unsigned char **pzTail, /* OUT: Tail of the input for next line */ - int mxWidth, /* Max width. 0 means no limit */ - u8 bWordWrap /* If true, avoid breaking mid-word */ -){ - int i; /* Input bytes consumed */ - int j; /* Output bytes generated */ - int k; /* Input bytes to be displayed */ - int n; /* Output column number */ - unsigned char *zOut; /* Output text */ - - if( z==0 ){ - *pzTail = 0; - return 0; - } - if( mxWidth<0 ) mxWidth = -mxWidth; - if( mxWidth==0 ) mxWidth = 1000000; - i = j = n = 0; - while( n<mxWidth ){ - unsigned char c = z[i]; - if( c>=0xc0 ){ - int u; - int len = decodeUtf8(&z[i], &u); - i += len; - j += len; - n += cli_wcwidth(u); - continue; - } - if( c>=' ' ){ - n++; - i++; - j++; - continue; - } - if( c==0 || c=='\n' || (c=='\r' && z[i+1]=='\n') ) break; - if( c=='\t' ){ - do{ - n++; - j++; - }while( (n&7)!=0 && n<mxWidth ); - i++; - continue; - } - if( c==0x1b && p->eEscMode==SHELL_ESC_OFF && (k = isVt100(&z[i]))>0 ){ - i += k; - j += k; - }else{ - n++; - j += 3; - i++; - } - } - if( n>=mxWidth && bWordWrap ){ - /* Perhaps try to back up to a better place to break the line */ - for(k=i; k>i/2; k--){ - if( IsSpace(z[k-1]) ) break; - } - if( k<=i/2 ){ - for(k=i; k>i/2; k--){ - if( IsAlnum(z[k-1])!=IsAlnum(z[k]) && (z[k]&0xc0)!=0x80 ) break; - } - } - if( k<=i/2 ){ - k = i; - }else{ - i = k; - while( z[i]==' ' ) i++; - } - }else{ - k = i; - } - if( n>=mxWidth && z[i]>=' ' ){ - *pzTail = &z[i]; - }else if( z[i]=='\r' && z[i+1]=='\n' ){ - *pzTail = z[i+2] ? &z[i+2] : 0; - }else if( z[i]==0 || z[i+1]==0 ){ - *pzTail = 0; - }else{ - *pzTail = &z[i+1]; - } - zOut = malloc( j+1 ); - shell_check_oom(zOut); - i = j = n = 0; - while( i<k ){ - unsigned char c = z[i]; - if( c>=0xc0 ){ - int u; - int len = decodeUtf8(&z[i], &u); - do{ zOut[j++] = z[i++]; }while( (--len)>0 ); - n += cli_wcwidth(u); - continue; - } - if( c>=' ' ){ - n++; - zOut[j++] = z[i++]; - continue; - } - if( c==0 ) break; - if( z[i]=='\t' ){ - do{ - n++; - zOut[j++] = ' '; - }while( (n&7)!=0 && n<mxWidth ); - i++; - continue; - } - switch( p->eEscMode ){ - case SHELL_ESC_SYMBOL: - zOut[j++] = 0xe2; - zOut[j++] = 0x90; - zOut[j++] = 0x80 + c; - break; - case SHELL_ESC_ASCII: - zOut[j++] = '^'; - zOut[j++] = 0x40 + c; - break; - case SHELL_ESC_OFF: { - int nn; - if( c==0x1b && (nn = isVt100(&z[i]))>0 ){ - memcpy(&zOut[j], &z[i], nn); - j += nn; - i += nn - 1; - }else{ - zOut[j++] = c; - } - break; - } - } - i++; - } - zOut[j] = 0; - return (char*)zOut; -} - -/* Return true if the text string z[] contains characters that need -** unistr() escaping. -*/ -static int needUnistr(const unsigned char *z){ - unsigned char c; - if( z==0 ) return 0; - while( (c = *z)>0x1f || c=='\t' || c=='\n' || (c=='\r' && z[1]=='\n') ){ z++; } - return c!=0; -} - -/* Extract the value of the i-th current column for pStmt as an SQL literal -** value. Memory is obtained from sqlite3_malloc64() and must be freed by -** the caller. -*/ -static char *quoted_column(sqlite3_stmt *pStmt, int i){ - switch( sqlite3_column_type(pStmt, i) ){ - case SQLITE_NULL: { - return sqlite3_mprintf("NULL"); - } - case SQLITE_INTEGER: - case SQLITE_FLOAT: { - return sqlite3_mprintf("%s",sqlite3_column_text(pStmt,i)); - } - case SQLITE_TEXT: { - const unsigned char *zText = sqlite3_column_text(pStmt,i); - return sqlite3_mprintf(needUnistr(zText)?"%#Q":"%Q",zText); - } - case SQLITE_BLOB: { - int j; - sqlite3_str *pStr = sqlite3_str_new(0); - const unsigned char *a = sqlite3_column_blob(pStmt,i); - int n = sqlite3_column_bytes(pStmt,i); - sqlite3_str_append(pStr, "x'", 2); - for(j=0; j<n; j++){ - sqlite3_str_appendf(pStr, "%02x", a[j]); - } - sqlite3_str_append(pStr, "'", 1); - return sqlite3_str_finish(pStr); - } - } - return 0; /* Not reached */ -} - -/* -** Run a prepared statement and output the result in one of the -** table-oriented formats: MODE_Column, MODE_Markdown, MODE_Table, -** or MODE_Box. -** -** This is different from ordinary exec_prepared_stmt() in that -** it has to run the entire query and gather the results into memory -** first, in order to determine column widths, before providing -** any output. -*/ -static void exec_prepared_stmt_columnar( - ShellState *p, /* Pointer to ShellState */ - sqlite3_stmt *pStmt /* Statement to run */ -){ - sqlite3_int64 nRow = 0; - int nColumn = 0; - char **azData = 0; - sqlite3_int64 nAlloc = 0; - char *abRowDiv = 0; - const unsigned char *uz; - const char *z; - char **azQuoted = 0; - int rc; - sqlite3_int64 i, nData; - int j, nTotal, w, n; - const char *colSep = 0; - const char *rowSep = 0; - const unsigned char **azNextLine = 0; - int bNextLine = 0; - int bMultiLineRowExists = 0; - int bw = p->cmOpts.bWordWrap; - const char *zEmpty = ""; - const char *zShowNull = p->nullValue; - - rc = sqlite3_step(pStmt); - if( rc!=SQLITE_ROW ) return; - nColumn = sqlite3_column_count(pStmt); - if( nColumn==0 ) goto columnar_end; - nAlloc = nColumn*4; - if( nAlloc<=0 ) nAlloc = 1; - azData = sqlite3_malloc64( nAlloc*sizeof(char*) ); - shell_check_oom(azData); - azNextLine = sqlite3_malloc64( nColumn*sizeof(char*) ); - shell_check_oom(azNextLine); - memset((void*)azNextLine, 0, nColumn*sizeof(char*) ); - if( p->cmOpts.bQuote ){ - azQuoted = sqlite3_malloc64( nColumn*sizeof(char*) ); - shell_check_oom(azQuoted); - memset(azQuoted, 0, nColumn*sizeof(char*) ); - } - abRowDiv = sqlite3_malloc64( nAlloc/nColumn ); - shell_check_oom(abRowDiv); - if( nColumn>p->nWidth ){ - p->colWidth = realloc(p->colWidth, (nColumn+1)*2*sizeof(int)); - shell_check_oom(p->colWidth); - for(i=p->nWidth; i<nColumn; i++) p->colWidth[i] = 0; - p->nWidth = nColumn; - p->actualWidth = &p->colWidth[nColumn]; - } - memset(p->actualWidth, 0, nColumn*sizeof(int)); - for(i=0; i<nColumn; i++){ - w = p->colWidth[i]; - if( w<0 ) w = -w; - p->actualWidth[i] = w; - } - for(i=0; i<nColumn; i++){ - const unsigned char *zNotUsed; - int wx = p->colWidth[i]; - if( wx==0 ){ - wx = p->cmOpts.iWrap; - } - if( wx<0 ) wx = -wx; - uz = (const unsigned char*)sqlite3_column_name(pStmt,i); - if( uz==0 ) uz = (u8*)""; - azData[i] = translateForDisplayAndDup(p, uz, &zNotUsed, wx, bw); - } - do{ - int useNextLine = bNextLine; - bNextLine = 0; - if( (nRow+2)*nColumn >= nAlloc ){ - nAlloc *= 2; - azData = sqlite3_realloc64(azData, nAlloc*sizeof(char*)); - shell_check_oom(azData); - abRowDiv = sqlite3_realloc64(abRowDiv, nAlloc/nColumn); - shell_check_oom(abRowDiv); - } - abRowDiv[nRow] = 1; - nRow++; - for(i=0; i<nColumn; i++){ - int wx = p->colWidth[i]; - if( wx==0 ){ - wx = p->cmOpts.iWrap; - } - if( wx<0 ) wx = -wx; - if( useNextLine ){ - uz = azNextLine[i]; - if( uz==0 ) uz = (u8*)zEmpty; - }else if( p->cmOpts.bQuote ){ - assert( azQuoted!=0 ); - sqlite3_free(azQuoted[i]); - azQuoted[i] = quoted_column(pStmt,i); - uz = (const unsigned char*)azQuoted[i]; - }else{ - uz = (const unsigned char*)sqlite3_column_text(pStmt,i); - if( uz==0 ) uz = (u8*)zShowNull; - } - azData[nRow*nColumn + i] - = translateForDisplayAndDup(p, uz, &azNextLine[i], wx, bw); - if( azNextLine[i] ){ - bNextLine = 1; - abRowDiv[nRow-1] = 0; - bMultiLineRowExists = 1; - } - } - }while( bNextLine || sqlite3_step(pStmt)==SQLITE_ROW ); - nTotal = nColumn*(nRow+1); - for(i=0; i<nTotal; i++){ - z = azData[i]; - if( z==0 ) z = (char*)zEmpty; - n = strlenChar(z); - j = i%nColumn; - if( n>p->actualWidth[j] ) p->actualWidth[j] = n; - } - if( seenInterrupt ) goto columnar_end; - switch( p->cMode ){ - case MODE_Column: { - colSep = " "; - rowSep = "\n"; - if( p->showHeader ){ - for(i=0; i<nColumn; i++){ - w = p->actualWidth[i]; - if( p->colWidth[i]<0 ) w = -w; - utf8_width_print(p->out, w, azData[i]); - sqlite3_fputs(i==nColumn-1?"\n":" ", p->out); - } - for(i=0; i<nColumn; i++){ - print_dashes(p->out, p->actualWidth[i]); - sqlite3_fputs(i==nColumn-1?"\n":" ", p->out); - } - } - break; - } - case MODE_Table: { - colSep = " | "; - rowSep = " |\n"; - print_row_separator(p, nColumn, "+"); - sqlite3_fputs("| ", p->out); - for(i=0; i<nColumn; i++){ - w = p->actualWidth[i]; - n = strlenChar(azData[i]); - sqlite3_fprintf(p->out, "%*s%s%*s", (w-n)/2, "", - azData[i], (w-n+1)/2, ""); - sqlite3_fputs(i==nColumn-1?" |\n":" | ", p->out); - } - print_row_separator(p, nColumn, "+"); - break; - } - case MODE_Markdown: { - colSep = " | "; - rowSep = " |\n"; - sqlite3_fputs("| ", p->out); - for(i=0; i<nColumn; i++){ - w = p->actualWidth[i]; - n = strlenChar(azData[i]); - sqlite3_fprintf(p->out, "%*s%s%*s", (w-n)/2, "", - azData[i], (w-n+1)/2, ""); - sqlite3_fputs(i==nColumn-1?" |\n":" | ", p->out); - } - print_row_separator(p, nColumn, "|"); - break; - } - case MODE_Box: { - colSep = " " BOX_13 " "; - rowSep = " " BOX_13 "\n"; - print_box_row_separator(p, nColumn, BOX_23, BOX_234, BOX_34); - sqlite3_fputs(BOX_13 " ", p->out); - for(i=0; i<nColumn; i++){ - w = p->actualWidth[i]; - n = strlenChar(azData[i]); - sqlite3_fprintf(p->out, "%*s%s%*s%s", - (w-n)/2, "", azData[i], (w-n+1)/2, "", - i==nColumn-1?" "BOX_13"\n":" "BOX_13" "); - } - print_box_row_separator(p, nColumn, BOX_123, BOX_1234, BOX_134); - break; - } - } - for(i=nColumn, j=0; i<nTotal; i++, j++){ - if( j==0 && p->cMode!=MODE_Column ){ - sqlite3_fputs(p->cMode==MODE_Box?BOX_13" ":"| ", p->out); - } - z = azData[i]; - if( z==0 ) z = p->nullValue; - w = p->actualWidth[j]; - if( p->colWidth[j]<0 ) w = -w; - utf8_width_print(p->out, w, z); - if( j==nColumn-1 ){ - sqlite3_fputs(rowSep, p->out); - if( bMultiLineRowExists && abRowDiv[i/nColumn-1] && i+1<nTotal ){ - if( p->cMode==MODE_Table ){ - print_row_separator(p, nColumn, "+"); - }else if( p->cMode==MODE_Box ){ - print_box_row_separator(p, nColumn, BOX_123, BOX_1234, BOX_134); - }else if( p->cMode==MODE_Column ){ - sqlite3_fputs("\n", p->out); - } - } - j = -1; - if( seenInterrupt ) goto columnar_end; - }else{ - sqlite3_fputs(colSep, p->out); - } - } - if( p->cMode==MODE_Table ){ - print_row_separator(p, nColumn, "+"); - }else if( p->cMode==MODE_Box ){ - print_box_row_separator(p, nColumn, BOX_12, BOX_124, BOX_14); - } -columnar_end: - if( seenInterrupt ){ - sqlite3_fputs("Interrupt\n", p->out); - } - nData = (nRow+1)*nColumn; - for(i=0; i<nData; i++){ - z = azData[i]; - if( z!=zEmpty && z!=zShowNull ) free(azData[i]); - } - sqlite3_free(azData); - sqlite3_free((void*)azNextLine); - sqlite3_free(abRowDiv); - if( azQuoted ){ - for(i=0; i<nColumn; i++) sqlite3_free(azQuoted[i]); - sqlite3_free(azQuoted); - } -} - -/* -** Run a prepared statement -*/ -static void exec_prepared_stmt( - ShellState *pArg, /* Pointer to ShellState */ - sqlite3_stmt *pStmt /* Statement to run */ -){ - int rc; - sqlite3_uint64 nRow = 0; - - if( pArg->cMode==MODE_Column - || pArg->cMode==MODE_Table - || pArg->cMode==MODE_Box - || pArg->cMode==MODE_Markdown - ){ - exec_prepared_stmt_columnar(pArg, pStmt); - return; - } - - /* perform the first step. this will tell us if we - ** have a result set or not and how wide it is. - */ - rc = sqlite3_step(pStmt); - /* if we have a result set... */ - if( SQLITE_ROW == rc ){ - /* allocate space for col name ptr, value ptr, and type */ - int nCol = sqlite3_column_count(pStmt); - void *pData = sqlite3_malloc64(3*nCol*sizeof(const char*) + 1); - if( !pData ){ - shell_out_of_memory(); - }else{ - char **azCols = (char **)pData; /* Names of result columns */ - char **azVals = &azCols[nCol]; /* Results */ - int *aiTypes = (int *)&azVals[nCol]; /* Result types */ - int i, x; - assert(sizeof(int) <= sizeof(char *)); - /* save off ptrs to column names */ - for(i=0; i<nCol; i++){ - azCols[i] = (char *)sqlite3_column_name(pStmt, i); - } - do{ - nRow++; - /* extract the data and data types */ - for(i=0; i<nCol; i++){ - aiTypes[i] = x = sqlite3_column_type(pStmt, i); - if( x==SQLITE_BLOB - && pArg - && (pArg->cMode==MODE_Insert || pArg->cMode==MODE_Quote) - ){ - azVals[i] = ""; - }else{ - azVals[i] = (char*)sqlite3_column_text(pStmt, i); - } - if( !azVals[i] && (aiTypes[i]!=SQLITE_NULL) ){ - rc = SQLITE_NOMEM; - break; /* from for */ - } - } /* end for */ - - /* if data and types extracted successfully... */ - if( SQLITE_ROW == rc ){ - /* call the supplied callback with the result row data */ - if( shell_callback(pArg, nCol, azVals, azCols, aiTypes) ){ - rc = SQLITE_ABORT; - }else{ - rc = sqlite3_step(pStmt); - } - } - } while( SQLITE_ROW == rc ); - sqlite3_free(pData); - if( pArg->cMode==MODE_Json ){ - sqlite3_fputs("]\n", pArg->out); - }else if( pArg->cMode==MODE_Www ){ - sqlite3_fputs("</TABLE>\n<PRE>\n", pArg->out); - }else if( pArg->cMode==MODE_Count ){ - char zBuf[200]; - sqlite3_snprintf(sizeof(zBuf), zBuf, "%llu row%s\n", - nRow, nRow!=1 ? "s" : ""); - printf("%s", zBuf); - } - } - } -} - #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) /* ** This function is called to process SQL if the previous shell command @@ -4591,8 +3125,8 @@ static int expertFinish( if( bVerbose ){ const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES); - sqlite3_fputs("-- Candidates -----------------------------\n", out); - sqlite3_fprintf(out, "%s\n", zCand); + cli_puts("-- Candidates -----------------------------\n", out); + cli_printf(out, "%s\n", zCand); } for(i=0; i<nQuery; i++){ const char *zSql = sqlite3_expert_report(p, i, EXPERT_REPORT_SQL); @@ -4600,12 +3134,12 @@ static int expertFinish( const char *zEQP = sqlite3_expert_report(p, i, EXPERT_REPORT_PLAN); if( zIdx==0 ) zIdx = "(no new indexes)\n"; if( bVerbose ){ - sqlite3_fprintf(out, + cli_printf(out, "-- Query %d --------------------------------\n" "%s\n\n" ,i+1, zSql); } - sqlite3_fprintf(out, "%s\n%s\n", zIdx, zEQP); + cli_printf(out, "%s\n%s\n", zIdx, zEQP); } } } @@ -4640,18 +3174,18 @@ static int expertDotCommand( } else if( n>=2 && 0==cli_strncmp(z, "-sample", n) ){ if( i==(nArg-1) ){ - sqlite3_fprintf(stderr, "option requires an argument: %s\n", z); + cli_printf(stderr, "option requires an argument: %s\n", z); rc = SQLITE_ERROR; }else{ iSample = (int)integerValue(azArg[++i]); if( iSample<0 || iSample>100 ){ - sqlite3_fprintf(stderr,"value out of range: %s\n", azArg[i]); + cli_printf(stderr,"value out of range: %s\n", azArg[i]); rc = SQLITE_ERROR; } } } else{ - sqlite3_fprintf(stderr,"unknown option: %s\n", z); + cli_printf(stderr,"unknown option: %s\n", z); rc = SQLITE_ERROR; } } @@ -4659,7 +3193,7 @@ static int expertDotCommand( if( rc==SQLITE_OK ){ pState->expert.pExpert = sqlite3_expert_new(pState->db, &zErr); if( pState->expert.pExpert==0 ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "sqlite3_expert_new: %s\n", zErr ? zErr : "out of memory"); rc = SQLITE_ERROR; }else{ @@ -4674,6 +3208,15 @@ static int expertDotCommand( } #endif /* !SQLITE_OMIT_VIRTUALTABLE && !SQLITE_OMIT_AUTHORIZATION */ +/* +** QRF write callback +*/ +static int shellWriteQR(void *pX, const char *z, sqlite3_int64 n){ + ShellState *pArg = (ShellState*)pX; + cli_printf(pArg->out, "%.*s", (int)n, z); + return SQLITE_OK; +} + /* ** Execute a statement or set of statements. Print ** any result rows/columns depending on the current mode @@ -4693,10 +3236,33 @@ static int shell_exec( int rc2; const char *zLeftover; /* Tail of unprocessed SQL */ sqlite3 *db = pArg->db; + unsigned char eStyle; + sqlite3_qrf_spec spec; if( pzErrMsg ){ *pzErrMsg = NULL; } + memcpy(&spec, &pArg->mode.spec, sizeof(spec)); + spec.xWrite = shellWriteQR; + spec.pWriteArg = (void*)pArg; + if( pArg->mode.eMode==MODE_Insert && ShellHasFlag(pArg,SHFLG_PreserveRowid) ){ + spec.bTitles = QRF_SW_On; + } + /* ,- This is true, but it is omitted + ** vvvvvvvvvvvvvvvvvvv ----- to avoid compiler warnings. */ + assert( /*pArg->mode.eMode>=0 &&*/ pArg->mode.eMode<ArraySize(aModeInfo) ); + eStyle = aModeInfo[pArg->mode.eMode].eStyle; + if( pArg->mode.bAutoScreenWidth ){ + spec.nScreenWidth = shellScreenWidth(); + } + if( spec.eBlob==QRF_BLOB_Auto ){ + switch( spec.eText ){ + case QRF_TEXT_Relaxed: /* fall through */ + case QRF_TEXT_Sql: spec.eBlob = QRF_BLOB_Sql; break; + case QRF_TEXT_Json: spec.eBlob = QRF_BLOB_Json; break; + default: spec.eBlob = QRF_BLOB_Text; break; + } + } #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) if( pArg->expert.pExpert ){ @@ -4705,7 +3271,7 @@ static int shell_exec( } #endif - while( zSql[0] && (SQLITE_OK == rc) ){ + while( zSql && zSql[0] && (SQLITE_OK == rc) ){ static const char *zStmtSql; rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover); if( SQLITE_OK != rc ){ @@ -4713,6 +3279,7 @@ static int shell_exec( *pzErrMsg = save_err_msg(db, "in prepare", rc, zSql); } }else{ + int isExplain; if( !pStmt ){ /* this happens for a comment or white-space */ zSql = zLeftover; @@ -4723,80 +3290,58 @@ static int shell_exec( if( zStmtSql==0 ) zStmtSql = ""; while( IsSpace(zStmtSql[0]) ) zStmtSql++; - /* save off the prepared statement handle and reset row count */ + /* save off the prepared statement handle */ if( pArg ){ pArg->pStmt = pStmt; - pArg->cnt = 0; } - + /* Show the EXPLAIN QUERY PLAN if .eqp is on */ - if( pArg && pArg->autoEQP && sqlite3_stmt_isexplain(pStmt)==0 ){ - sqlite3_stmt *pExplain; + isExplain = sqlite3_stmt_isexplain(pStmt); + if( pArg && pArg->mode.autoEQP && isExplain==0 && pArg->dot.nArg==0 ){ int triggerEQP = 0; + u8 savedEnableTimer = pArg->enableTimer; + pArg->enableTimer = 0; disable_debug_trace_modes(); sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, -1, &triggerEQP); - if( pArg->autoEQP>=AUTOEQP_trigger ){ + if( pArg->mode.autoEQP>=AUTOEQP_trigger ){ sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 1, 0); } - pExplain = pStmt; - sqlite3_reset(pExplain); - rc = sqlite3_stmt_explain(pExplain, 2); - if( rc==SQLITE_OK ){ - bind_prepared_stmt(pArg, pExplain); - while( sqlite3_step(pExplain)==SQLITE_ROW ){ - const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3); - int iEqpId = sqlite3_column_int(pExplain, 0); - int iParentId = sqlite3_column_int(pExplain, 1); - if( zEQPLine==0 ) zEQPLine = ""; - if( zEQPLine[0]=='-' ) eqp_render(pArg, 0); - eqp_append(pArg, iEqpId, iParentId, zEQPLine); - } - eqp_render(pArg, 0); - } - if( pArg->autoEQP>=AUTOEQP_full ){ - /* Also do an EXPLAIN for ".eqp full" mode */ - sqlite3_reset(pExplain); - rc = sqlite3_stmt_explain(pExplain, 1); - if( rc==SQLITE_OK ){ - pArg->cMode = MODE_Explain; - assert( sqlite3_stmt_isexplain(pExplain)==1 ); - bind_prepared_stmt(pArg, pExplain); - explain_data_prepare(pArg, pExplain); - exec_prepared_stmt(pArg, pExplain); - explain_data_delete(pArg); - } + sqlite3_reset(pStmt); + spec.eStyle = QRF_STYLE_Auto; + sqlite3_stmt_explain(pStmt, 2); + sqlite3_format_query_result(pStmt, &spec, 0); + if( pArg->mode.autoEQP>=AUTOEQP_full ){ + sqlite3_reset(pStmt); + sqlite3_stmt_explain(pStmt, 1); + sqlite3_format_query_result(pStmt, &spec, 0); } - if( pArg->autoEQP>=AUTOEQP_trigger && triggerEQP==0 ){ + + if( pArg->mode.autoEQP>=AUTOEQP_trigger && triggerEQP==0 ){ sqlite3_db_config(db, SQLITE_DBCONFIG_TRIGGER_EQP, 0, 0); } sqlite3_reset(pStmt); sqlite3_stmt_explain(pStmt, 0); restore_debug_trace_modes(); - } - - if( pArg ){ - int bIsExplain = (sqlite3_stmt_isexplain(pStmt)==1); - pArg->cMode = pArg->mode; - if( pArg->autoExplain ){ - if( bIsExplain ){ - pArg->cMode = MODE_Explain; - } - if( sqlite3_stmt_isexplain(pStmt)==2 ){ - pArg->cMode = MODE_EQP; - } - } - - /* If the shell is currently in ".explain" mode, gather the extra - ** data required to add indents to the output.*/ - if( pArg->cMode==MODE_Explain && bIsExplain ){ - explain_data_prepare(pArg, pStmt); - } + pArg->enableTimer = savedEnableTimer; } bind_prepared_stmt(pArg, pStmt); - exec_prepared_stmt(pArg, pStmt); - explain_data_delete(pArg); - eqp_render(pArg, 0); + if( isExplain && pArg->mode.autoExplain ){ + spec.eStyle = isExplain==1 ? QRF_STYLE_Explain : QRF_STYLE_Eqp; + sqlite3_format_query_result(pStmt, &spec, pzErrMsg); + }else if( pArg->mode.eMode==MODE_Www ){ + cli_printf(pArg->out, + "</PRE>\n" + "<TABLE border='1' cellspacing='0' cellpadding='2'>\n"); + spec.eStyle = QRF_STYLE_Html; + sqlite3_format_query_result(pStmt, &spec, pzErrMsg); + cli_printf(pArg->out, + "</TABLE>\n" + "<PRE>"); + }else{ + spec.eStyle = eStyle; + sqlite3_format_query_result(pStmt, &spec, pzErrMsg); + } /* print usage stats if stats on */ if( pArg && pArg->statsOn ){ @@ -4804,8 +3349,19 @@ static int shell_exec( } /* print loop-counters if required */ - if( pArg && pArg->scanstatsOn ){ - display_scanstats(db, pArg); + if( pArg && pArg->mode.scanstatsOn ){ + char *zErr = 0; + switch( pArg->mode.scanstatsOn ){ + case 1: spec.eStyle = QRF_STYLE_Stats; break; + case 2: spec.eStyle = QRF_STYLE_StatsEst; break; + default: spec.eStyle = QRF_STYLE_StatsVm; break; + } + sqlite3_reset(pStmt); + rc = sqlite3_format_query_result(pStmt, &spec, &zErr); + if( rc ){ + cli_printf(stderr, "Stats query failed: %s\n", zErr); + sqlite3_free(zErr); + } } /* Finalize the statement just executed. If this fails, save a @@ -5001,14 +3557,14 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ */ if( db_int(p->db, "SELECT count(*) FROM sqlite_sequence")>0 ){ if( !p->writableSchema ){ - sqlite3_fputs("PRAGMA writable_schema=ON;\n", p->out); + cli_puts("PRAGMA writable_schema=ON;\n", p->out); p->writableSchema = 1; } - sqlite3_fputs("CREATE TABLE IF NOT EXISTS sqlite_sequence(name,seq);\n" + cli_puts("CREATE TABLE IF NOT EXISTS sqlite_sequence(name,seq);\n" "DELETE FROM sqlite_sequence;\n", p->out); } }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 && !noSys ){ - if( !dataOnly ) sqlite3_fputs("ANALYZE sqlite_schema;\n", p->out); + if( !dataOnly ) cli_puts("ANALYZE sqlite_schema;\n", p->out); }else if( cli_strncmp(zTable, "sqlite_", 7)==0 ){ return 0; }else if( dataOnly ){ @@ -5016,7 +3572,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ }else if( cli_strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){ char *zIns; if( !p->writableSchema ){ - sqlite3_fputs("PRAGMA writable_schema=ON;\n", p->out); + cli_puts("PRAGMA writable_schema=ON;\n", p->out); p->writableSchema = 1; } zIns = sqlite3_mprintf( @@ -5024,7 +3580,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ "VALUES('table','%q','%q',0,'%q');", zTable, zTable, zSql); shell_check_oom(zIns); - sqlite3_fprintf(p->out, "%s\n", zIns); + cli_printf(p->out, "%s\n", zIns); sqlite3_free(zIns); return 0; }else{ @@ -5036,8 +3592,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ ShellText sTable; char **azCol; int i; - char *savedDestTable; - int savedMode; + Mode savedMode; azCol = tableColumnList(p, zTable); if( azCol==0 ){ @@ -5080,18 +3635,21 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azNotUsed){ appendText(&sSelect, " FROM ", 0); appendText(&sSelect, zTable, quoteChar(zTable)); - savedDestTable = p->zDestTable; + savedMode = p->mode; - p->zDestTable = sTable.zTxt; - p->mode = p->cMode = MODE_Insert; + p->mode.spec.zTableName = (char*)zTable; + p->mode.eMode = MODE_Insert; + p->mode.spec.eText = QRF_TEXT_Sql; + p->mode.spec.eBlob = QRF_BLOB_Sql; + p->mode.spec.bTitles = QRF_No; + p->mode.spec.nCharLimit = 0; rc = shell_exec(p, sSelect.zTxt, 0); if( (rc&0xff)==SQLITE_CORRUPT ){ - sqlite3_fputs("/****** CORRUPTION ERROR *******/\n", p->out); + cli_puts("/****** CORRUPTION ERROR *******/\n", p->out); toggleSelectOrder(p->db); shell_exec(p, sSelect.zTxt, 0); toggleSelectOrder(p->db); } - p->zDestTable = savedDestTable; p->mode = savedMode; freeText(&sTable); freeText(&sSelect); @@ -5117,9 +3675,9 @@ static int run_schema_dump_query( if( rc==SQLITE_CORRUPT ){ char *zQ2; int len = strlen30(zQuery); - sqlite3_fputs("/****** CORRUPTION ERROR *******/\n", p->out); + cli_puts("/****** CORRUPTION ERROR *******/\n", p->out); if( zErr ){ - sqlite3_fprintf(p->out, "/****** %s ******/\n", zErr); + cli_printf(p->out, "/****** %s ******/\n", zErr); sqlite3_free(zErr); zErr = 0; } @@ -5128,7 +3686,7 @@ static int run_schema_dump_query( sqlite3_snprintf(len+100, zQ2, "%s ORDER BY rowid DESC", zQuery); rc = sqlite3_exec(p->db, zQ2, dump_callback, p, &zErr); if( rc ){ - sqlite3_fprintf(p->out, "/****** ERROR: %s ******/\n", zErr); + cli_printf(p->out, "/****** ERROR: %s ******/\n", zErr); }else{ rc = SQLITE_CORRUPT; } @@ -5186,8 +3744,8 @@ static const char *(azHelp[]) = { ".cd DIRECTORY Change the working directory to DIRECTORY", #endif ".changes on|off Show number of rows changed by SQL", + ".check OPTIONS ... Verify the results of a .testcase", #ifndef SQLITE_SHELL_FIDDLE - ".check GLOB Fail if output since .testcase does not match", ".clone NEWDB Clone data into NEWDB from the existing database", #endif ".connection [close] [#] Open or close an auxiliary database connection", @@ -5229,30 +3787,18 @@ static const char *(azHelp[]) = { " --schema SCHEMA Use SCHEMA instead of \"main\"", " --help Show CMD details", ".fullschema ?--indent? Show schema and the content of sqlite_stat tables", - ".headers on|off Turn display of headers on or off", + ",headers on|off Turn display of headers on or off", ".help ?-all? ?PATTERN? Show help text for PATTERN", #ifndef SQLITE_SHELL_FIDDLE ".import FILE TABLE Import data from FILE into TABLE", - " Options:", - " --ascii Use \\037 and \\036 as column and row separators", - " --csv Use , and \\n as column and row separators", - " --skip N Skip the first N rows of input", - " --schema S Target table to be S.TABLE", - " -v \"Verbose\" - increase auxiliary output", - " Notes:", - " * If TABLE does not exist, it is created. The first row of input", - " determines the column names.", - " * If neither --csv or --ascii are used, the input mode is derived", - " from the \".mode\" output mode", - " * If FILE begins with \"|\" then it is a command that generates the", - " input text.", #endif #ifndef SQLITE_OMIT_TEST_CONTROL ".imposter INDEX TABLE Create imposter table TABLE on index INDEX", #endif - ".indexes ?TABLE? Show names of indexes", - " If TABLE is specified, only show indexes for", - " tables matching TABLE using the LIKE operator.", + ".indexes ?PATTERN? Show names of indexes matching PATTERN", + " -a|--all Also show system-generated indexes", + " --expr Show only expression indexes", + " --sys Show only system-generated indexes", ".intck ?STEPS_PER_UNLOCK? Run an incremental integrity check on the db", #ifdef SQLITE_ENABLE_IOTRACE ",iotrace FILE Enable I/O diagnostic logging to FILE", @@ -5270,42 +3816,12 @@ static const char *(azHelp[]) = { ".log on|off Turn logging on or off.", #endif ".mode ?MODE? ?OPTIONS? Set output mode", - " MODE is one of:", - " ascii Columns/rows delimited by 0x1F and 0x1E", - " box Tables using unicode box-drawing characters", - " csv Comma-separated values", - " column Output in columns. (See .width)", - " html HTML <table> code", - " insert SQL insert statements for TABLE", - " json Results in a JSON array", - " line One value per line", - " list Values delimited by \"|\"", - " markdown Markdown table format", - " qbox Shorthand for \"box --wrap 60 --quote\"", - " quote Escape answers as for SQL", - " table ASCII-art table", - " tabs Tab-separated values", - " tcl TCL list elements", - " OPTIONS: (for columnar modes or insert mode):", - " --escape T ctrl-char escape; T is one of: symbol, ascii, off", - " --wrap N Wrap output lines to no longer than N characters", - " --wordwrap B Wrap or not at word boundaries per B (on/off)", - " --ww Shorthand for \"--wordwrap 1\"", - " --quote Quote output text as SQL literals", - " --noquote Do not quote output text", - " TABLE The name of SQL table used for \"insert\" mode", #ifndef SQLITE_SHELL_FIDDLE ".nonce STRING Suspend safe mode for one command if nonce matches", #endif ".nullvalue STRING Use STRING in place of NULL values", #ifndef SQLITE_SHELL_FIDDLE ".once ?OPTIONS? ?FILE? Output for the next SQL command only to FILE", - " If FILE begins with '|' then open as a pipe", - " --bom Put a UTF8 byte-order mark at the beginning", - " -e Send output to the system text editor", - " --plain Use text/plain output instead of HTML for -w option", - " -w Send output as HTML to a web browser (same as \".www\")", - " -x Send output as CSV to a spreadsheet (same as \".excel\")", /* Note that .open is (partially) available in WASM builds but is ** currently only intended to be used by the fiddle tool, not ** end users, so is "undocumented." */ @@ -5331,14 +3847,6 @@ static const char *(azHelp[]) = { " --zip FILE is a ZIP archive", #ifndef SQLITE_SHELL_FIDDLE ".output ?FILE? Send output to FILE or stdout if FILE is omitted", - " If FILE begins with '|' then open it as a pipe.", - " If FILE is 'off' then output is disabled.", - " Options:", - " --bom Prefix output with a UTF8 byte-order mark", - " -e Send output to the system text editor", - " --plain Use text/plain for -w option", - " -w Send output to a web browser", - " -x Send output as CSV to a spreadsheet", #endif ".parameter CMD ... Manage SQL parameter bindings", " clear Erase all bindings", @@ -5354,6 +3862,7 @@ static const char *(azHelp[]) = { " --once Do no more than one progress interrupt", " --quiet|-q No output except at interrupts", " --reset Reset the count for each input and interrupt", + " --timeout S Halt after running for S seconds", #endif ".prompt MAIN CONTINUE Replace the standard prompts", #ifndef SQLITE_SHELL_FIDDLE @@ -5381,7 +3890,7 @@ static const char *(azHelp[]) = { " Options:", " --init Create a new SELFTEST table", " -v Verbose output", - ".separator COL ?ROW? Change the column and row separators", + ",separator COL ?ROW? Change the column and row separators", #if defined(SQLITE_ENABLE_SESSION) ".session ?NAME? CMD ... Create or control sessions", " Subcommands:", @@ -5408,7 +3917,7 @@ static const char *(azHelp[]) = { #if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) ".shell CMD ARGS... Run CMD ARGS... in a system shell", #endif - ".show Show the current values for various settings", + ",show Show the current values for various settings", ".stats ?ARG? Show stats or turn stats on or off", " off Turn off automatic stat display", " on Turn on automatic stat display", @@ -5418,13 +3927,11 @@ static const char *(azHelp[]) = { ".system CMD ARGS... Run CMD ARGS... in a system shell", #endif ".tables ?TABLE? List names of tables matching LIKE pattern TABLE", -#ifndef SQLITE_SHELL_FIDDLE - ",testcase NAME Begin redirecting output to 'testcase-out.txt'", -#endif + ".testcase NAME Begin a test case.", ",testctrl CMD ... Run various sqlite3_test_control() operations", " Run \".testctrl\" with no arguments for details", ".timeout MS Try opening locked tables for MS milliseconds", - ".timer on|off Turn SQL timer on or off", + ".timer on|off|once Turn SQL timer on or off.", #ifndef SQLITE_OMIT_TRACE ".trace ?OPTIONS? Output each SQL statement as it is run", " FILE Send output to FILE", @@ -5449,7 +3956,7 @@ static const char *(azHelp[]) = { ".vfsinfo ?AUX? Information about the top-level VFS", ".vfslist List all available VFSes", ".vfsname ?AUX? Print the name of the VFS stack", - ".width NUM1 NUM2 ... Set minimum column widths for columnar output", + ",width NUM1 NUM2 ... Set minimum column widths for columnar output", " Negative values right-justify", #ifndef SQLITE_SHELL_FIDDLE ".www Display output of the next command in web browser", @@ -5457,6 +3964,19 @@ static const char *(azHelp[]) = { #endif }; +INSERT-USAGE-TEXT-HERE + +/* +** Return a pointer to usage text for zCmd, or NULL if none exists. +*/ +static const char *findUsage(const char *zCmd){ + int i; + for(i=0; i<ArraySize(aUsage); i++){ + if( sqlite3_strglob(zCmd, aUsage[i].zCmd)==0 ) return aUsage[i].zUsage; + } + return 0; +} + /* ** Output help text for commands that match zPattern. ** @@ -5486,6 +4006,7 @@ static int showHelp(FILE *out, const char *zPattern){ int j = 0; int n = 0; char *zPat; + const char *zHit = 0; if( zPattern==0 ){ /* Show just the first line for all help topics */ zPattern = "[a-z]"; @@ -5503,37 +4024,46 @@ static int showHelp(FILE *out, const char *zPattern){ show = 0; }else if( azHelp[i][0]==',' ){ show = 1; - sqlite3_fprintf(out, ".%s\n", &azHelp[i][1]); + cli_printf(out, ".%s\n", &azHelp[i][1]); n++; }else if( show ){ - sqlite3_fprintf(out, "%s\n", azHelp[i]); + cli_printf(out, "%s\n", azHelp[i]); } } return n; } /* Seek documented commands for which zPattern is an exact prefix */ - zPat = sqlite3_mprintf(".%s*", zPattern); + zPat = sqlite3_mprintf(".%s*", zPattern[0]=='.' ? &zPattern[1] : zPattern); shell_check_oom(zPat); for(i=0; i<ArraySize(azHelp); i++){ if( sqlite3_strglob(zPat, azHelp[i])==0 ){ - sqlite3_fprintf(out, "%s\n", azHelp[i]); + if( zHit ) cli_printf(out, "%s\n", zHit); + zHit = azHelp[i]; j = i+1; n++; } } - sqlite3_free(zPat); if( n ){ if( n==1 ){ - /* when zPattern is a prefix of exactly one command, then include - ** the details of that command, which should begin at offset j */ - while( j<ArraySize(azHelp)-1 && azHelp[j][0]==' ' ){ - sqlite3_fprintf(out, "%s\n", azHelp[j]); - j++; - } + const char *zUsage = findUsage(zPat); + if( zUsage ){ + cli_puts(zUsage, out); + }else{ + /* when zPattern is a prefix of exactly one command, then include + ** the details of that command, which should begin at offset j */ + cli_printf(out, "%s\n", zHit); + while( j<ArraySize(azHelp)-1 && azHelp[j][0]==' ' ){ + cli_printf(out, "%s\n", azHelp[j]); + j++; + } + } + }else{ + cli_printf(out, "%s\n", zHit); } - return n; } + sqlite3_free(zPat); + if( n ) return n; /* Look for documented commands that contain zPattern anywhere. ** Show complete text of all documented commands that match. */ @@ -5546,10 +4076,10 @@ static int showHelp(FILE *out, const char *zPattern){ } if( azHelp[i][0]=='.' ) j = i; if( sqlite3_strlike(zPat, azHelp[i], 0)==0 ){ - sqlite3_fprintf(out, "%s\n", azHelp[j]); + cli_printf(out, "%s\n", azHelp[j]); while( j<ArraySize(azHelp)-1 && azHelp[j+1][0]==' ' ){ j++; - sqlite3_fprintf(out, "%s\n", azHelp[j]); + cli_printf(out, "%s\n", azHelp[j]); } i = j; n++; @@ -5586,7 +4116,7 @@ static char *readFile(const char *zName, int *pnByte){ if( in==0 ) return 0; rc = fseek(in, 0, SEEK_END); if( rc!=0 ){ - sqlite3_fprintf(stderr,"Error: '%s' not seekable\n", zName); + cli_printf(stderr,"Error: '%s' not seekable\n", zName); fclose(in); return 0; } @@ -5594,7 +4124,7 @@ static char *readFile(const char *zName, int *pnByte){ rewind(in); pBuf = sqlite3_malloc64( nIn+1 ); if( pBuf==0 ){ - sqlite3_fputs("Error: out of memory\n", stderr); + cli_puts("Error: out of memory\n", stderr); fclose(in); return 0; } @@ -5602,7 +4132,7 @@ static char *readFile(const char *zName, int *pnByte){ fclose(in); if( nRead!=1 ){ sqlite3_free(pBuf); - sqlite3_fprintf(stderr,"Error: cannot read '%s'\n", zName); + cli_printf(stderr,"Error: cannot read '%s'\n", zName); return 0; } pBuf[nIn] = 0; @@ -5658,6 +4188,52 @@ static int session_filter(void *pCtx, const char *zTab){ } #endif +/* +** Return the size of the named file in bytes. Or return a negative +** number if the file does not exist. +*/ +static sqlite3_int64 fileSize(const char *zFile){ +#if defined(_WIN32) || defined(WIN32) + struct _stat64 x; + if( _stat64(zFile, &x)!=0 ) return -1; + return (sqlite3_int64)x.st_size; +#else + struct stat x; + if( stat(zFile, &x)!=0 ) return -1; + return (sqlite3_int64)x.st_size; +#endif +} + +/* +** Return true if zFile is an SQLite database. +** +** Algorithm: +** * If the file does not exist -> return false +** * If the size of the file is not a multiple of 512 -> return false +** * If sqlite3_open() fails -> return false +** * if sqlite3_prepare() or sqlite3_step() fails -> return false +** * Otherwise -> return true +*/ +static int isDatabaseFile(const char *zFile, int openFlags){ + sqlite3 *db = 0; + sqlite3_stmt *pStmt = 0; + int rc; + sqlite3_int64 sz = fileSize(zFile); + if( sz<512 || (sz%512)!=0 ) return 0; + if( sqlite3_open_v2(zFile, &db, openFlags, 0)==SQLITE_OK + && sqlite3_prepare_v2(db,"SELECT count(*) FROM sqlite_schema",-1,&pStmt,0) + ==SQLITE_OK + && sqlite3_step(pStmt)==SQLITE_ROW + ){ + rc = 1; + }else{ + rc = 0; + } + sqlite3_finalize(pStmt); + sqlite3_close(db); + return rc; +} + /* ** Try to deduce the type of file for zName based on its content. Return ** one of the SHELL_OPEN_* constants. @@ -5670,20 +4246,12 @@ static int session_filter(void *pCtx, const char *zTab){ int deduceDatabaseType(const char *zName, int dfltZip, int openFlags){ FILE *f; size_t n; - sqlite3 *db = 0; - sqlite3_stmt *pStmt = 0; int rc = SHELL_OPEN_UNSPEC; char zBuf[100]; if( access(zName,0)!=0 ) goto database_type_by_name; - if( sqlite3_open_v2(zName, &db, openFlags, 0)==SQLITE_OK - && sqlite3_prepare_v2(db,"SELECT count(*) FROM sqlite_schema",-1,&pStmt,0) - ==SQLITE_OK - && sqlite3_step(pStmt)==SQLITE_ROW - ){ + if( isDatabaseFile(zName, openFlags) ){ rc = SHELL_OPEN_NORMAL; } - sqlite3_finalize(pStmt); - sqlite3_close(db); if( rc==SHELL_OPEN_NORMAL ) return SHELL_OPEN_NORMAL; f = sqlite3_fopen(zName, "rb"); if( f==0 ) goto database_type_by_name; @@ -5718,6 +4286,35 @@ database_type_by_name: return rc; } +/* +** If the text in z[] is the name of a readable file and that file appears +** to contain SQL text and/or dot-commands, then return true. If z[] is +** not a file, or if the file is unreadable, or if the file is a database +** or anything else that is not SQL text and dot-commands, then return false. +** +** If the bLeaveUninit flag is set, then be sure to leave SQLite in an +** uninitialized state. This means invoking sqlite3_shutdown() after any +** SQLite API is used. +** +** Some amount of guesswork is involved in this decision. +*/ +static int isScriptFile(const char *z, int bLeaveUninit){ + sqlite3_int64 sz = fileSize(z); + if( sz<=0 ) return 0; + if( (sz%512)==0 ){ + int rc; + sqlite3_initialize(); + rc = isDatabaseFile(z, SQLITE_OPEN_READONLY); + if( bLeaveUninit ){ + sqlite3_shutdown(); + } + if( rc ) return 0; /* Is a database */ + } + if( sqlite3_strlike("%.sql",z,0)==0 ) return 1; + if( sqlite3_strlike("%.txt",z,0)==0 ) return 1; + return 0; +} + #ifndef SQLITE_OMIT_DESERIALIZE /* ** Reconstruct an in-memory database using the output from the "dbtotxt" @@ -5739,7 +4336,7 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){ if( zDbFilename ){ in = sqlite3_fopen(zDbFilename, "r"); if( in==0 ){ - sqlite3_fprintf(stderr,"cannot open \"%s\" for reading\n", zDbFilename); + cli_printf(stderr,"cannot open \"%s\" for reading\n", zDbFilename); return 0; } nLine = 0; @@ -5755,7 +4352,7 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){ if( rc!=2 ) goto readHexDb_error; if( n<0 ) goto readHexDb_error; if( pgsz<512 || pgsz>65536 || (pgsz & (pgsz-1))!=0 ){ - sqlite3_fputs("invalid pagesize\n", stderr); + cli_puts("invalid pagesize\n", stderr); goto readHexDb_error; } sz = ((i64)n+pgsz-1)&~(pgsz-1); /* Round up to nearest multiple of pgsz */ @@ -5766,7 +4363,7 @@ static unsigned char *readHexDb(ShellState *p, int *pnData){ int j = 0; /* Page number from "| page" line */ int k = 0; /* Offset from "| page" line */ if( nLine>=2000000000 ){ - sqlite3_fprintf(stderr, "input too big\n"); + cli_printf(stderr, "input too big\n"); goto readHexDb_error; } rc = sscanf(zLine, "| page %d offset %d", &j, &k); @@ -5807,7 +4404,7 @@ readHexDb_error: p->lineno = nLine; } sqlite3_free(a); - sqlite3_fprintf(stderr,"Error on line %lld of --hexdb input\n", nLine); + cli_printf(stderr,"Error on line %lld of --hexdb input\n", nLine); return 0; } #endif /* SQLITE_OMIT_DESERIALIZE */ @@ -5912,19 +4509,19 @@ static void open_db(ShellState *p, int openFlags){ } } if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ - sqlite3_fprintf(stderr,"Error: unable to open database \"%s\": %s\n", + cli_printf(stderr,"Error: unable to open database \"%s\": %s\n", zDbFilename, sqlite3_errmsg(p->db)); if( (openFlags & OPEN_DB_KEEPALIVE)==0 ){ - exit(1); + cli_exit(1); } sqlite3_close(p->db); sqlite3_open(":memory:", &p->db); if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ - sqlite3_fputs("Also: unable to open substitute in-memory database.\n", + cli_puts("Also: unable to open substitute in-memory database.\n", stderr); - exit(1); + cli_exit(1); }else{ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Notice: using substitute in-memory database instead of \"%s\"\n", zDbFilename); } @@ -6002,6 +4599,8 @@ static void open_db(ShellState *p, int openFlags){ shellModuleSchema, 0, 0); sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p, shellPutsFunc, 0, 0); + sqlite3_create_function(p->db, "shell_format_schema", 2, SQLITE_UTF8, p, + shellFormatSchema, 0, 0); sqlite3_create_function(p->db, "usleep",1,SQLITE_UTF8,0, shellUSleepFunc, 0, 0); #ifndef SQLITE_NOHAVE_SYSTEM @@ -6036,7 +4635,7 @@ static void open_db(ShellState *p, int openFlags){ SQLITE_DESERIALIZE_RESIZEABLE | SQLITE_DESERIALIZE_FREEONCLOSE); if( rc ){ - sqlite3_fprintf(stderr,"Error: sqlite3_deserialize() returns %d\n", rc); + cli_printf(stderr,"Error: sqlite3_deserialize() returns %d\n", rc); } if( p->szMax>0 ){ sqlite3_file_control(p->db, "main", SQLITE_FCNTL_SIZE_LIMIT, &p->szMax); @@ -6051,7 +4650,7 @@ static void open_db(ShellState *p, int openFlags){ } #endif sqlite3_db_config( - p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->scanstatsOn, (int*)0 + p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->mode.scanstatsOn, (int*)0 ); } } @@ -6062,7 +4661,7 @@ static void open_db(ShellState *p, int openFlags){ void close_db(sqlite3 *db){ int rc = sqlite3_close(db); if( rc ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Error: sqlite3_close() returns %d: %s\n", rc, sqlite3_errmsg(db)); } } @@ -6235,7 +4834,7 @@ static int booleanValue(const char *zArg){ if( sqlite3_stricmp(zArg, "off")==0 || sqlite3_stricmp(zArg,"no")==0 ){ return 0; } - sqlite3_fprintf(stderr, + cli_printf(stderr, "ERROR: Not a boolean value: \"%s\". Assuming \"no\".\n", zArg); return 0; } @@ -6263,18 +4862,18 @@ static void output_file_close(FILE *f){ ** recognized and do the right thing. NULL is returned if the output ** filename is "off". */ -static FILE *output_file_open(const char *zFile){ +static FILE *output_file_open(ShellState *p, const char *zFile){ FILE *f; if( cli_strcmp(zFile,"stdout")==0 ){ f = stdout; }else if( cli_strcmp(zFile, "stderr")==0 ){ f = stderr; - }else if( cli_strcmp(zFile, "off")==0 ){ + }else if( cli_strcmp(zFile, "off")==0 || p->bSafeMode ){ f = 0; }else{ f = sqlite3_fopen(zFile, "w"); if( f==0 ){ - sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zFile); + cli_printf(stderr,"Error: cannot open \"%s\"\n", zFile); } } return f; @@ -6327,12 +4926,12 @@ static int sql_trace_callback( switch( mType ){ case SQLITE_TRACE_ROW: case SQLITE_TRACE_STMT: { - sqlite3_fprintf(p->traceOut, "%.*s;\n", (int)nSql, zSql); + cli_printf(p->traceOut, "%.*s;\n", (int)nSql, zSql); break; } case SQLITE_TRACE_PROFILE: { sqlite3_int64 nNanosec = pX ? *(sqlite3_int64*)pX : 0; - sqlite3_fprintf(p->traceOut, + cli_printf(p->traceOut, "%.*s; -- %lld ns\n", (int)nSql, zSql, nNanosec); break; } @@ -6361,7 +4960,9 @@ struct ImportCtx { const char *zFile; /* Name of the input file */ FILE *in; /* Read the CSV text from this input stream */ int (SQLITE_CDECL *xCloser)(FILE*); /* Func to close in */ + char *zIn; /* Input text */ char *z; /* Accumulated text for a field */ + i64 nUsed; /* Bytes of zIn[] used so far */ i64 n; /* Number of bytes in z */ i64 nAlloc; /* Space allocated for z[] */ int nLine; /* Current line number */ @@ -6371,6 +4972,8 @@ struct ImportCtx { int cTerm; /* Character that terminated the most recent field */ int cColSep; /* The column separator character. (Usually ",") */ int cRowSep; /* The row separator character. (Usually "\n") */ + int cQEscape; /* Escape character with "...". 0 for none */ + int cUQEscape; /* Escape character not with "...". 0 for none */ }; /* Clean up resourced used by an ImportCtx */ @@ -6381,9 +4984,28 @@ static void import_cleanup(ImportCtx *p){ } sqlite3_free(p->z); p->z = 0; + if( p->zIn ){ + sqlite3_free(p->zIn); + p->zIn = 0; + } } -/* Append a single byte to z[] */ +/* Read a single character of the .import input text. Return EOF +** at end-of-file. +*/ +static int import_getc(ImportCtx *p){ + if( p->in ){ + return fgetc(p->in); + }else if( p->zIn && p->zIn[p->nUsed]!=0 ){ + return p->zIn[p->nUsed++]; + }else{ + return EOF; + } +} + +/* Append a single byte to the field value begin constructed +** in the p->z[] buffer +*/ static void import_append_char(ImportCtx *p, int c){ if( p->n+1>=p->nAlloc ){ p->nAlloc += p->nAlloc + 100; @@ -6399,8 +5021,8 @@ static void import_append_char(ImportCtx *p, int c){ ** + Input comes from p->in. ** + Store results in p->z of length p->n. Space to hold p->z comes ** from sqlite3_malloc64(). -** + Use p->cSep as the column separator. The default is ",". -** + Use p->rSep as the row separator. The default is "\n". +** + Use p->cColSep as the column separator. The default is ",". +** + Use p->cRowSep as the row separator. The default is "\n". ** + Keep track of the line number in p->nLine. ** + Store the character that terminates the field in p->cTerm. Store ** EOF on end-of-file. @@ -6411,7 +5033,7 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ int cSep = (u8)p->cColSep; int rSep = (u8)p->cRowSep; p->n = 0; - c = fgetc(p->in); + c = import_getc(p); if( c==EOF || seenInterrupt ){ p->cTerm = EOF; return 0; @@ -6420,10 +5042,17 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ int pc, ppc; int startLine = p->nLine; int cQuote = c; + int cEsc = (u8)p->cQEscape; pc = ppc = 0; while( 1 ){ - c = fgetc(p->in); + c = import_getc(p); if( c==rSep ) p->nLine++; + if( c==cEsc && cEsc!=0 ){ + c = import_getc(p); + import_append_char(p, c); + ppc = pc = 0; + continue; + } if( c==cQuote ){ if( pc==cQuote ){ pc = 0; @@ -6440,11 +5069,11 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ break; } if( pc==cQuote && c!='\r' ){ - sqlite3_fprintf(stderr,"%s:%d: unescaped %c character\n", - p->zFile, p->nLine, cQuote); + cli_printf(stderr,"%s:%d: unescaped %c character\n", + p->zFile, p->nLine, cQuote); } if( c==EOF ){ - sqlite3_fprintf(stderr,"%s:%d: unterminated %c-quoted field\n", + cli_printf(stderr,"%s:%d: unterminated %c-quoted field\n", p->zFile, startLine, cQuote); p->cTerm = c; break; @@ -6456,12 +5085,13 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ }else{ /* If this is the first field being parsed and it begins with the ** UTF-8 BOM (0xEF BB BF) then skip the BOM */ + int cEsc = p->cUQEscape; if( (c&0xff)==0xef && p->bNotFirst==0 ){ import_append_char(p, c); - c = fgetc(p->in); + c = import_getc(p); if( (c&0xff)==0xbb ){ import_append_char(p, c); - c = fgetc(p->in); + c = import_getc(p); if( (c&0xff)==0xbf ){ p->bNotFirst = 1; p->n = 0; @@ -6470,8 +5100,9 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ } } while( c!=EOF && c!=cSep && c!=rSep ){ + if( c==cEsc && cEsc!=0 ) c = import_getc(p); import_append_char(p, c); - c = fgetc(p->in); + c = import_getc(p); } if( c==rSep ){ p->nLine++; @@ -6489,8 +5120,8 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ ** + Input comes from p->in. ** + Store results in p->z of length p->n. Space to hold p->z comes ** from sqlite3_malloc64(). -** + Use p->cSep as the column separator. The default is "\x1F". -** + Use p->rSep as the row separator. The default is "\x1E". +** + Use p->cColSep as the column separator. The default is "\x1F". +** + Use p->cRowSep as the row separator. The default is "\x1E". ** + Keep track of the row number in p->nLine. ** + Store the character that terminates the field in p->cTerm. Store ** EOF on end-of-file. @@ -6501,14 +5132,14 @@ static char *SQLITE_CDECL ascii_read_one_field(ImportCtx *p){ int cSep = (u8)p->cColSep; int rSep = (u8)p->cRowSep; p->n = 0; - c = fgetc(p->in); + c = import_getc(p); if( c==EOF || seenInterrupt ){ p->cTerm = EOF; return 0; } while( c!=EOF && c!=cSep && c!=rSep ){ import_append_char(p, c); - c = fgetc(p->in); + c = import_getc(p); } if( c==rSep ){ p->nLine++; @@ -6543,7 +5174,7 @@ static void tryToCloneData( shell_check_oom(zQuery); rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); if( rc ){ - sqlite3_fprintf(stderr,"Error %d: %s on [%s]\n", + cli_printf(stderr,"Error %d: %s on [%s]\n", sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery); goto end_data_xfer; } @@ -6560,7 +5191,7 @@ static void tryToCloneData( memcpy(zInsert+i, ");", 3); rc = sqlite3_prepare_v2(newDb, zInsert, -1, &pInsert, 0); if( rc ){ - sqlite3_fprintf(stderr,"Error %d: %s on [%s]\n", + cli_printf(stderr,"Error %d: %s on [%s]\n", sqlite3_extended_errcode(newDb), sqlite3_errmsg(newDb), zInsert); goto end_data_xfer; } @@ -6596,7 +5227,7 @@ static void tryToCloneData( } /* End for */ rc = sqlite3_step(pInsert); if( rc!=SQLITE_OK && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){ - sqlite3_fprintf(stderr,"Error %d: %s\n", + cli_printf(stderr,"Error %d: %s\n", sqlite3_extended_errcode(newDb), sqlite3_errmsg(newDb)); } sqlite3_reset(pInsert); @@ -6614,7 +5245,7 @@ static void tryToCloneData( shell_check_oom(zQuery); rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); if( rc ){ - sqlite3_fprintf(stderr,"Warning: cannot step \"%s\" backwards", zTable); + cli_printf(stderr,"Warning: cannot step \"%s\" backwards", zTable); break; } } /* End for(k=0...) */ @@ -6651,7 +5282,7 @@ static void tryToCloneSchema( shell_check_oom(zQuery); rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); if( rc ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Error: (%d) %s on [%s]\n", sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery); goto end_schema_xfer; @@ -6661,10 +5292,10 @@ static void tryToCloneSchema( zSql = sqlite3_column_text(pQuery, 1); if( zName==0 || zSql==0 ) continue; if( sqlite3_stricmp((char*)zName, "sqlite_sequence")!=0 ){ - sqlite3_fprintf(stdout, "%s... ", zName); fflush(stdout); + cli_printf(stdout, "%s... ", zName); fflush(stdout); sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg); if( zErrMsg ){ - sqlite3_fprintf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql); + cli_printf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql); sqlite3_free(zErrMsg); zErrMsg = 0; } @@ -6682,7 +5313,7 @@ static void tryToCloneSchema( shell_check_oom(zQuery); rc = sqlite3_prepare_v2(p->db, zQuery, -1, &pQuery, 0); if( rc ){ - sqlite3_fprintf(stderr,"Error: (%d) %s on [%s]\n", + cli_printf(stderr,"Error: (%d) %s on [%s]\n", sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zQuery); goto end_schema_xfer; } @@ -6691,10 +5322,10 @@ static void tryToCloneSchema( zSql = sqlite3_column_text(pQuery, 1); if( zName==0 || zSql==0 ) continue; if( sqlite3_stricmp((char*)zName, "sqlite_sequence")==0 ) continue; - sqlite3_fprintf(stdout, "%s... ", zName); fflush(stdout); + cli_printf(stdout, "%s... ", zName); fflush(stdout); sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg); if( zErrMsg ){ - sqlite3_fprintf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql); + cli_printf(stderr,"Error: %s\nSQL: [%s]\n", zErrMsg, zSql); sqlite3_free(zErrMsg); zErrMsg = 0; } @@ -6718,12 +5349,12 @@ static void tryToClone(ShellState *p, const char *zNewDb){ int rc; sqlite3 *newDb = 0; if( access(zNewDb,0)==0 ){ - sqlite3_fprintf(stderr,"File \"%s\" already exists.\n", zNewDb); + cli_printf(stderr,"File \"%s\" already exists.\n", zNewDb); return; } rc = sqlite3_open(zNewDb, &newDb); if( rc ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Cannot create output database: %s\n", sqlite3_errmsg(newDb)); }else{ sqlite3_exec(p->db, "PRAGMA writable_schema=ON;", 0, 0, 0); @@ -6742,12 +5373,12 @@ static void tryToClone(ShellState *p, const char *zNewDb){ */ static void output_redir(ShellState *p, FILE *pfNew){ if( p->out != stdout ){ - sqlite3_fputs("Output already redirected.\n", stderr); + cli_puts("Output already redirected.\n", stderr); }else{ p->out = pfNew; setCrlfMode(p); - if( p->mode==MODE_Www ){ - sqlite3_fputs( + if( p->mode.eMode==MODE_Www ){ + cli_puts( "<!DOCTYPE html>\n" "<HTML><BODY><PRE>\n", p->out @@ -6769,8 +5400,8 @@ static void output_reset(ShellState *p){ pclose(p->out); #endif }else{ - if( p->mode==MODE_Www ){ - sqlite3_fputs("</PRE></BODY></HTML>\n", p->out); + if( p->mode.eMode==MODE_Www ){ + cli_puts("</PRE></BODY></HTML>\n", p->out); } output_file_close(p->out); #ifndef SQLITE_NOHAVE_SYSTEM @@ -6786,7 +5417,7 @@ static void output_reset(ShellState *p){ char *zCmd; zCmd = sqlite3_mprintf("%s %s", zXdgOpenCmd, p->zTempFile); if( system(zCmd) ){ - sqlite3_fprintf(stderr,"Failed: [%s]\n", zCmd); + cli_printf(stderr,"Failed: [%s]\n", zCmd); }else{ /* Give the start/open/xdg-open command some time to get ** going before we continue, and potential delete the @@ -6794,7 +5425,7 @@ static void output_reset(ShellState *p){ sqlite3_sleep(2000); } sqlite3_free(zCmd); - outputModePop(p); + modePop(p); p->doXdgOpen = 0; } #endif /* !defined(SQLITE_NOHAVE_SYSTEM) */ @@ -6802,6 +5433,10 @@ static void output_reset(ShellState *p){ p->outfile[0] = 0; p->out = stdout; setCrlfMode(p); + if( cli_output_capture ){ + sqlite3_str_free(cli_output_capture); + cli_output_capture = 0; + } } #else # define output_redir(SS,pfO) @@ -6886,7 +5521,7 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ "SELECT data FROM sqlite_dbpage(?1) WHERE pgno=1", -1, &pStmt, 0); if( rc ){ - sqlite3_fprintf(stderr,"error: %s\n", sqlite3_errmsg(p->db)); + cli_printf(stderr,"error: %s\n", sqlite3_errmsg(p->db)); sqlite3_finalize(pStmt); return 1; } @@ -6899,28 +5534,28 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ memcpy(aHdr, pb, 100); sqlite3_finalize(pStmt); }else{ - sqlite3_fputs("unable to read database header\n", stderr); + cli_puts("unable to read database header\n", stderr); sqlite3_finalize(pStmt); return 1; } i = get2byteInt(aHdr+16); if( i==1 ) i = 65536; - sqlite3_fprintf(p->out, "%-20s %d\n", "database page size:", i); - sqlite3_fprintf(p->out, "%-20s %d\n", "write format:", aHdr[18]); - sqlite3_fprintf(p->out, "%-20s %d\n", "read format:", aHdr[19]); - sqlite3_fprintf(p->out, "%-20s %d\n", "reserved bytes:", aHdr[20]); + cli_printf(p->out, "%-20s %d\n", "database page size:", i); + cli_printf(p->out, "%-20s %d\n", "write format:", aHdr[18]); + cli_printf(p->out, "%-20s %d\n", "read format:", aHdr[19]); + cli_printf(p->out, "%-20s %d\n", "reserved bytes:", aHdr[20]); for(i=0; i<ArraySize(aField); i++){ int ofst = aField[i].ofst; unsigned int val = get4byteInt(aHdr + ofst); - sqlite3_fprintf(p->out, "%-20s %u", aField[i].zName, val); + cli_printf(p->out, "%-20s %u", aField[i].zName, val); switch( ofst ){ case 56: { - if( val==1 ) sqlite3_fputs(" (utf8)", p->out); - if( val==2 ) sqlite3_fputs(" (utf16le)", p->out); - if( val==3 ) sqlite3_fputs(" (utf16be)", p->out); + if( val==1 ) cli_puts(" (utf8)", p->out); + if( val==2 ) cli_puts(" (utf16le)", p->out); + if( val==3 ) cli_puts(" (utf16be)", p->out); } } - sqlite3_fputs("\n", p->out); + cli_puts("\n", p->out); } if( zDb==0 ){ zSchemaTab = sqlite3_mprintf("main.sqlite_schema"); @@ -6931,11 +5566,11 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ } for(i=0; i<ArraySize(aQuery); i++){ int val = db_int(p->db, aQuery[i].zSql, zSchemaTab); - sqlite3_fprintf(p->out, "%-20s %d\n", aQuery[i].zName, val); + cli_printf(p->out, "%-20s %d\n", aQuery[i].zName, val); } sqlite3_free(zSchemaTab); sqlite3_file_control(p->db, zDb, SQLITE_FCNTL_DATA_VERSION, &iDataVersion); - sqlite3_fprintf(p->out, "%-20s %u\n", "data version", iDataVersion); + cli_printf(p->out, "%-20s %u\n", "data version", iDataVersion); return 0; } #endif /* SQLITE_SHELL_HAVE_RECOVER */ @@ -6995,7 +5630,7 @@ static int shell_dbtotxt_command(ShellState *p, int nArg, char **azArg){ } zName = strdup(zTail); shell_check_oom(zName); - sqlite3_fprintf(p->out, "| size %lld pagesize %d filename %s\n", + cli_printf(p->out, "| size %lld pagesize %d filename %s\n", nPage*pgSz, pgSz, zName); sqlite3_finalize(pStmt); pStmt = 0; @@ -7011,27 +5646,27 @@ static int shell_dbtotxt_command(ShellState *p, int nArg, char **azArg){ for(j=0; j<16 && aLine[j]==0; j++){} if( j==16 ) continue; if( !seenPageLabel ){ - sqlite3_fprintf(p->out, "| page %lld offset %lld\n",pgno,(pgno-1)*pgSz); + cli_printf(p->out, "| page %lld offset %lld\n",pgno,(pgno-1)*pgSz); seenPageLabel = 1; } - sqlite3_fprintf(p->out, "| %5d:", i); - for(j=0; j<16; j++) sqlite3_fprintf(p->out, " %02x", aLine[j]); - sqlite3_fprintf(p->out, " "); + cli_printf(p->out, "| %5d:", i); + for(j=0; j<16; j++) cli_printf(p->out, " %02x", aLine[j]); + cli_printf(p->out, " "); for(j=0; j<16; j++){ unsigned char c = (unsigned char)aLine[j]; - sqlite3_fprintf(p->out, "%c", bShow[c]); + cli_printf(p->out, "%c", bShow[c]); } - sqlite3_fprintf(p->out, "\n"); + cli_printf(p->out, "\n"); } } sqlite3_finalize(pStmt); - sqlite3_fprintf(p->out, "| end %s\n", zName); + cli_printf(p->out, "| end %s\n", zName); free(zName); return 0; dbtotxt_error: if( rc ){ - sqlite3_fprintf(stderr, "ERROR: %s\n", sqlite3_errmsg(p->db)); + cli_printf(stderr, "ERROR: %s\n", sqlite3_errmsg(p->db)); } sqlite3_finalize(pStmt); free(zName); @@ -7042,7 +5677,7 @@ dbtotxt_error: ** Print the given string as an error message. */ static void shellEmitError(const char *zErr){ - sqlite3_fprintf(stderr,"Error: %s\n", zErr); + cli_printf(stderr,"Error: %s\n", zErr); } /* ** Print the current sqlite3_errmsg() value to stderr and return 1. @@ -7225,39 +5860,42 @@ static void clearTempFile(ShellState *p){ p->zTempFile = 0; } +/* Forward reference */ +static char *find_home_dir(int clearFlag); + /* ** Create a new temp file name with the given suffix. +** +** Because the classic temp folders like /tmp are no longer +** accessible to web browsers, for security reasons, create the +** temp file in the user's home directory. */ static void newTempFile(ShellState *p, const char *zSuffix){ - clearTempFile(p); - sqlite3_free(p->zTempFile); - p->zTempFile = 0; - if( p->db ){ - sqlite3_file_control(p->db, 0, SQLITE_FCNTL_TEMPFILENAME, &p->zTempFile); - } - if( p->zTempFile==0 ){ - /* If p->db is an in-memory database then the TEMPFILENAME file-control - ** will not work and we will need to fallback to guessing */ - char *zTemp; - sqlite3_uint64 r; - sqlite3_randomness(sizeof(r), &r); - zTemp = getenv("TEMP"); - if( zTemp==0 ) zTemp = getenv("TMP"); - if( zTemp==0 ){ + char *zHome; /* Home directory */ + int i; /* Loop counter */ + sqlite3_uint64 r = 0; /* Integer with 64 bits of randomness */ + char zRand[32]; /* Text string with 160 bits of randomness */ #ifdef _WIN32 - zTemp = "\\tmp"; + const char cDirSep = '\\'; #else - zTemp = "/tmp"; + const char cDirSep = '/'; #endif - } - p->zTempFile = sqlite3_mprintf("%s/temp%llx.%s", zTemp, r, zSuffix); - }else{ - p->zTempFile = sqlite3_mprintf("%z.%s", p->zTempFile, zSuffix); + + for(i=0; i<31; i++){ + if( (i%12)==0 ) sqlite3_randomness(sizeof(r),&r); + zRand[i] = "0123456789abcdefghijklmnopqrstuvwxyz"[r%36]; + r /= 36; } + zRand[i] = 0; + clearTempFile(p); + sqlite3_free(p->zTempFile); + p->zTempFile = 0; + zHome = find_home_dir(0); + p->zTempFile = sqlite3_mprintf("%s%ctemp-%s.%s", + zHome,cDirSep,zRand,zSuffix); shell_check_oom(p->zTempFile); } - /* ** The implementation of SQL scalar function fkey_collate_clause(), used ** by the ".lint fkey-indexes" command. This scalar function is always @@ -7402,7 +6040,7 @@ static int lintFkeyIndexes( zIndent = " "; } else{ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Usage: %s %s ?-verbose? ?-groupbyparent?\n", azArg[0], azArg[1]); return SQLITE_ERROR; } @@ -7447,22 +6085,22 @@ static int lintFkeyIndexes( if( rc!=SQLITE_OK ) break; if( res<0 ){ - sqlite3_fputs("Error: internal error", stderr); + cli_puts("Error: internal error", stderr); break; }else{ if( bGroupByParent && (bVerbose || res==0) && (zPrev==0 || sqlite3_stricmp(zParent, zPrev)) ){ - sqlite3_fprintf(out, "-- Parent table %s\n", zParent); + cli_printf(out, "-- Parent table %s\n", zParent); sqlite3_free(zPrev); zPrev = sqlite3_mprintf("%s", zParent); } if( res==0 ){ - sqlite3_fprintf(out, "%s%s --> %s\n", zIndent, zCI, zTarget); + cli_printf(out, "%s%s --> %s\n", zIndent, zCI, zTarget); }else if( bVerbose ){ - sqlite3_fprintf(out, + cli_printf(out, "%s/* no extra indexes required for %s -> %s */\n", zIndent, zFrom, zTarget ); @@ -7472,16 +6110,16 @@ static int lintFkeyIndexes( sqlite3_free(zPrev); if( rc!=SQLITE_OK ){ - sqlite3_fprintf(stderr,"%s\n", sqlite3_errmsg(db)); + cli_printf(stderr,"%s\n", sqlite3_errmsg(db)); } rc2 = sqlite3_finalize(pSql); if( rc==SQLITE_OK && rc2!=SQLITE_OK ){ rc = rc2; - sqlite3_fprintf(stderr,"%s\n", sqlite3_errmsg(db)); + cli_printf(stderr,"%s\n", sqlite3_errmsg(db)); } }else{ - sqlite3_fprintf(stderr,"%s\n", sqlite3_errmsg(db)); + cli_printf(stderr,"%s\n", sqlite3_errmsg(db)); } return rc; @@ -7501,9 +6139,9 @@ static int lintDotCommand( return lintFkeyIndexes(pState, azArg, nArg); usage: - sqlite3_fprintf(stderr,"Usage %s sub-command ?switches...?\n", azArg[0]); - sqlite3_fprintf(stderr, "Where sub-commands are:\n"); - sqlite3_fprintf(stderr, " fkey-indexes\n"); + cli_printf(stderr,"Usage %s sub-command ?switches...?\n", azArg[0]); + cli_printf(stderr, "Where sub-commands are:\n"); + cli_printf(stderr, " fkey-indexes\n"); return SQLITE_ERROR; } @@ -7517,7 +6155,7 @@ static void shellPrepare( if( *pRc==SQLITE_OK ){ int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); if( rc!=SQLITE_OK ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "sql error: %s (%d)\n", sqlite3_errmsg(db), sqlite3_errcode(db)); *pRc = rc; } @@ -7562,7 +6200,7 @@ static void shellFinalize( int rc = sqlite3_finalize(pStmt); if( *pRc==SQLITE_OK ){ if( rc!=SQLITE_OK ){ - sqlite3_fprintf(stderr,"SQL error: %s\n", sqlite3_errmsg(db)); + cli_printf(stderr,"SQL error: %s\n", sqlite3_errmsg(db)); } *pRc = rc; } @@ -7584,7 +6222,7 @@ void shellReset( if( *pRc==SQLITE_OK ){ if( rc!=SQLITE_OK ){ sqlite3 *db = sqlite3_db_handle(pStmt); - sqlite3_fprintf(stderr,"SQL error: %s\n", sqlite3_errmsg(db)); + cli_printf(stderr,"SQL error: %s\n", sqlite3_errmsg(db)); } *pRc = rc; } @@ -7637,9 +6275,9 @@ static int arErrorMsg(ArCommand *pAr, const char *zFmt, ...){ va_end(ap); shellEmitError(z); if( pAr->fromCmdLine ){ - sqlite3_fputs("Use \"-A\" for more help\n", stderr); + cli_puts("Use \"-A\" for more help\n", stderr); }else{ - sqlite3_fputs("Use \".archive --help\" for more help\n", stderr); + cli_puts("Use \".archive --help\" for more help\n", stderr); } sqlite3_free(z); return SQLITE_ERROR; @@ -7739,7 +6377,7 @@ static int arParseCommand( struct ArSwitch *pEnd = &aSwitch[nSwitch]; if( nArg<=1 ){ - sqlite3_fprintf(stderr, "Wrong number of arguments. Usage:\n"); + cli_printf(stderr, "Wrong number of arguments. Usage:\n"); return arUsage(stderr); }else{ char *z = azArg[1]; @@ -7845,7 +6483,7 @@ static int arParseCommand( } } if( pAr->eCmd==0 ){ - sqlite3_fprintf(stderr, "Required argument missing. Usage:\n"); + cli_printf(stderr, "Required argument missing. Usage:\n"); return arUsage(stderr); } return SQLITE_OK; @@ -7888,7 +6526,7 @@ static int arCheckEntries(ArCommand *pAr){ } shellReset(&rc, pTest); if( rc==SQLITE_OK && bOk==0 ){ - sqlite3_fprintf(stderr,"not found in archive: %s\n", z); + cli_printf(stderr,"not found in archive: %s\n", z); rc = SQLITE_ERROR; } } @@ -7971,15 +6609,15 @@ static int arListCommand(ArCommand *pAr){ shellPreparePrintf(pAr->db, &rc, &pSql, zSql, azCols[pAr->bVerbose], pAr->zSrcTable, zWhere); if( pAr->bDryRun ){ - sqlite3_fprintf(pAr->out, "%s\n", sqlite3_sql(pSql)); + cli_printf(pAr->out, "%s\n", sqlite3_sql(pSql)); }else{ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ if( pAr->bVerbose ){ - sqlite3_fprintf(pAr->out, "%s % 10d %s %s\n", + cli_printf(pAr->out, "%s % 10d %s %s\n", sqlite3_column_text(pSql, 0), sqlite3_column_int(pSql, 1), sqlite3_column_text(pSql, 2),sqlite3_column_text(pSql, 3)); }else{ - sqlite3_fprintf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0)); + cli_printf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0)); } } } @@ -8006,7 +6644,7 @@ static int arRemoveCommand(ArCommand *pAr){ zSql = sqlite3_mprintf("DELETE FROM %s WHERE %s;", pAr->zSrcTable, zWhere); if( pAr->bDryRun ){ - sqlite3_fprintf(pAr->out, "%s\n", zSql); + cli_printf(pAr->out, "%s\n", zSql); }else{ char *zErr = 0; rc = sqlite3_exec(pAr->db, "SAVEPOINT ar;", 0, 0, 0); @@ -8019,7 +6657,7 @@ static int arRemoveCommand(ArCommand *pAr){ } } if( zErr ){ - sqlite3_fprintf(stdout, "ERROR: %s\n", zErr); /* stdout? */ + cli_printf(stdout, "ERROR: %s\n", zErr); /* stdout? */ sqlite3_free(zErr); } } @@ -8034,11 +6672,15 @@ static int arRemoveCommand(ArCommand *pAr){ */ static int arExtractCommand(ArCommand *pAr){ const char *zSql1 = - "SELECT " - " ($dir || name)," - " writefile(($dir || name), %s, mode, mtime) " - "FROM %s WHERE (%s) AND (data IS NULL OR $dirOnly = 0)" - " AND name NOT GLOB '*..[/\\]*'"; + "WITH dest(dpath,dlen) AS (SELECT realpath($dir),length(realpath($dir)))\n" + "SELECT ($dir || name),\n" + " CASE WHEN $dryrun THEN 0\n" + " ELSE writefile($dir||name, %s, mode, mtime) END\n" + " FROM dest CROSS JOIN %s\n" + " WHERE (%s)\n" + " AND (data IS NULL OR $pass==0)\n" /* Dirs both passes */ + " AND dpath=substr(realpath($dir||name),1,dlen)\n" /* No escapes */ + " AND name NOT GLOB '*..[/\\]*'\n"; /* No /../ in paths */ const char *azExtraArg[] = { "sqlar_uncompress(data, sz)", @@ -8073,24 +6715,28 @@ static int arExtractCommand(ArCommand *pAr){ if( rc==SQLITE_OK ){ j = sqlite3_bind_parameter_index(pSql, "$dir"); sqlite3_bind_text(pSql, j, zDir, -1, SQLITE_STATIC); - - /* Run the SELECT statement twice. The first time, writefile() is called - ** for all archive members that should be extracted. The second time, - ** only for the directories. This is because the timestamps for - ** extracted directories must be reset after they are populated (as - ** populating them changes the timestamp). */ + j = sqlite3_bind_parameter_index(pSql, "$dryrun"); + sqlite3_bind_int(pSql, j, pAr->bDryRun); + + /* Run the SELECT statement twice + ** (0) writefile() all files and directories + ** (1) writefile() for directory again + ** The second pass is so that the timestamps for extracted directories + ** will be reset to the value in the archive, since populating them + ** in the first pass will have changed the timestamp. */ for(i=0; i<2; i++){ - j = sqlite3_bind_parameter_index(pSql, "$dirOnly"); + j = sqlite3_bind_parameter_index(pSql, "$pass"); sqlite3_bind_int(pSql, j, i); if( pAr->bDryRun ){ - sqlite3_fprintf(pAr->out, "%s\n", sqlite3_sql(pSql)); - }else{ - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ - if( i==0 && pAr->bVerbose ){ - sqlite3_fprintf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0)); - } + cli_printf(pAr->out, "%s\n", sqlite3_sql(pSql)); + if( pAr->bVerbose==0 ) break; + } + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ + if( i==0 && pAr->bVerbose ){ + cli_printf(pAr->out, "%s\n", sqlite3_column_text(pSql, 0)); } } + if( pAr->bDryRun ) break; shellReset(&rc, pSql); } shellFinalize(&rc, pSql); @@ -8107,13 +6753,13 @@ static int arExtractCommand(ArCommand *pAr){ static int arExecSql(ArCommand *pAr, const char *zSql){ int rc; if( pAr->bDryRun ){ - sqlite3_fprintf(pAr->out, "%s\n", zSql); + cli_printf(pAr->out, "%s\n", zSql); rc = SQLITE_OK; }else{ char *zErr = 0; rc = sqlite3_exec(pAr->db, zSql, 0, 0, &zErr); if( zErr ){ - sqlite3_fprintf(stdout, "ERROR: %s\n", zErr); + cli_printf(stdout, "ERROR: %s\n", zErr); sqlite3_free(zErr); } } @@ -8289,13 +6935,13 @@ static int arDotCommand( } cmd.db = 0; if( cmd.bDryRun ){ - sqlite3_fprintf(cmd.out, "-- open database '%s'%s\n", cmd.zFile, + cli_printf(cmd.out, "-- open database '%s'%s\n", cmd.zFile, eDbType==SHELL_OPEN_APPENDVFS ? " using 'apndvfs'" : ""); } rc = sqlite3_open_v2(cmd.zFile, &cmd.db, flags, eDbType==SHELL_OPEN_APPENDVFS ? "apndvfs" : 0); if( rc!=SQLITE_OK ){ - sqlite3_fprintf(stderr, "cannot open file: %s (%s)\n", + cli_printf(stderr, "cannot open file: %s (%s)\n", cmd.zFile, sqlite3_errmsg(cmd.db)); goto end_ar_command; } @@ -8309,7 +6955,7 @@ static int arDotCommand( if( cmd.eCmd!=AR_CMD_CREATE && sqlite3_table_column_metadata(cmd.db,0,"sqlar","name",0,0,0,0,0) ){ - sqlite3_fprintf(stderr, "database does not contain an 'sqlar' table\n"); + cli_printf(stderr, "database does not contain an 'sqlar' table\n"); rc = SQLITE_ERROR; goto end_ar_command; } @@ -8367,7 +7013,7 @@ end_ar_command: */ static int recoverSqlCb(void *pCtx, const char *zSql){ ShellState *pState = (ShellState*)pCtx; - sqlite3_fprintf(pState->out, "%s;\n", zSql); + cli_printf(pState->out, "%s;\n", zSql); return SQLITE_OK; } @@ -8410,7 +7056,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ bRowids = 0; } else{ - sqlite3_fprintf(stderr,"unexpected option: %s\n", azArg[i]); + cli_printf(stderr,"unexpected option: %s\n", azArg[i]); showHelp(pState->out, azArg[0]); return 1; } @@ -8420,17 +7066,19 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ pState->db, "main", recoverSqlCb, (void*)pState ); - sqlite3_recover_config(p, 789, (void*)zRecoveryDb); /* Debug use only */ + if( !pState->bSafeMode ){ + sqlite3_recover_config(p, 789, (void*)zRecoveryDb); /* Debug use only */ + } sqlite3_recover_config(p, SQLITE_RECOVER_LOST_AND_FOUND, (void*)zLAF); sqlite3_recover_config(p, SQLITE_RECOVER_ROWIDS, (void*)&bRowids); sqlite3_recover_config(p, SQLITE_RECOVER_FREELIST_CORRUPT,(void*)&bFreelist); - sqlite3_fprintf(pState->out, ".dbconfig defensive off\n"); + cli_printf(pState->out, ".dbconfig defensive off\n"); sqlite3_recover_run(p); if( sqlite3_recover_errcode(p)!=SQLITE_OK ){ const char *zErr = sqlite3_recover_errmsg(p); int errCode = sqlite3_recover_errcode(p); - sqlite3_fprintf(stderr,"sql error: %s (%d)\n", zErr, errCode); + cli_printf(stderr,"sql error: %s (%d)\n", zErr, errCode); } rc = sqlite3_recover_finish(p); return rc; @@ -8452,7 +7100,7 @@ static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){ while( SQLITE_OK==sqlite3_intck_step(p) ){ const char *zMsg = sqlite3_intck_message(p); if( zMsg ){ - sqlite3_fprintf(pState->out, "%s\n", zMsg); + cli_printf(pState->out, "%s\n", zMsg); nError++; } nStep++; @@ -8462,11 +7110,11 @@ static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){ } rc = sqlite3_intck_error(p, &zErr); if( zErr ){ - sqlite3_fprintf(stderr,"%s\n", zErr); + cli_printf(stderr,"%s\n", zErr); } sqlite3_intck_close(p); - sqlite3_fprintf(pState->out, "%lld steps, %lld errors\n", nStep, nError); + cli_printf(pState->out, "%lld steps, %lld errors\n", nStep, nError); } return rc; @@ -8489,7 +7137,7 @@ static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){ #define rc_err_oom_die(rc) \ if( rc==SQLITE_NOMEM ) shell_check_oom(0); \ else if(!(rc==SQLITE_OK||rc==SQLITE_DONE)) \ - sqlite3_fprintf(stderr,"E:%d\n",rc), assert(0) + cli_printf(stderr,"E:%d\n",rc), assert(0) #else static void rc_err_oom_die(int rc){ if( rc==SQLITE_NOMEM ) shell_check_oom(0); @@ -8604,7 +7252,7 @@ SELECT CASE WHEN (nc < 10) THEN 1 WHEN (nc < 100) THEN 2 \ SELECT\ '('||x'0a'\ || group_concat(\ - cname||' TEXT',\ + cname||' ANY',\ ','||iif((cpos-1)%4>0, ' ', x'0a'||' '))\ ||')' AS ColsSpec \ FROM (\ @@ -8683,77 +7331,1645 @@ FROM (\ *pzRenamed = 0; } } - sqlite3_finalize(pStmt); - sqlite3_close(*pDb); - *pDb = 0; - return zColsSpec; + sqlite3_finalize(pStmt); + sqlite3_close(*pDb); + *pDb = 0; + return zColsSpec; + } +} + +/* +** Check if the sqlite_schema table contains one or more virtual tables. If +** parameter zLike is not NULL, then it is an SQL expression that the +** sqlite_schema row must also match. If one or more such rows are found, +** print the following warning to the output: +** +** WARNING: Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled +*/ +static int outputDumpWarning(ShellState *p, const char *zLike){ + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + shellPreparePrintf(p->db, &rc, &pStmt, + "SELECT 1 FROM sqlite_schema o WHERE " + "sql LIKE 'CREATE VIRTUAL TABLE%%' AND (%s)", zLike ? zLike : "true" + ); + if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + cli_puts("/* WARNING: " + "Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled */\n", + p->out + ); + } + shellFinalize(&rc, pStmt); + return rc; +} + +/* +** Fault-Simulator state and logic. +*/ +static struct { + int iId; /* ID that triggers a simulated fault. -1 means "any" */ + int iErr; /* The error code to return on a fault */ + int iCnt; /* Trigger the fault only if iCnt is already zero */ + int iInterval; /* Reset iCnt to this value after each fault */ + int eVerbose; /* When to print output */ + int nHit; /* Number of hits seen so far */ + int nRepeat; /* Turn off after this many hits. 0 for never */ + int nSkip; /* Skip this many before first fault */ +} faultsim_state = {-1, 0, 0, 0, 0, 0, 0, 0}; + +/* +** This is the fault-sim callback +*/ +static int faultsim_callback(int iArg){ + if( faultsim_state.iId>0 && faultsim_state.iId!=iArg ){ + return SQLITE_OK; + } + if( faultsim_state.iCnt ){ + if( faultsim_state.iCnt>0 ) faultsim_state.iCnt--; + if( faultsim_state.eVerbose>=2 ){ + cli_printf(stdout, + "FAULT-SIM id=%d no-fault (cnt=%d)\n", iArg, faultsim_state.iCnt); + } + return SQLITE_OK; + } + if( faultsim_state.eVerbose>=1 ){ + cli_printf(stdout, + "FAULT-SIM id=%d returns %d\n", iArg, faultsim_state.iErr); + } + faultsim_state.iCnt = faultsim_state.iInterval; + faultsim_state.nHit++; + if( faultsim_state.nRepeat>0 && faultsim_state.nRepeat<=faultsim_state.nHit ){ + faultsim_state.iCnt = -1; + } + return faultsim_state.iErr; +} + +/* +** pickStr(zArg, &zErr, zS1, zS2, ..., ""); +** +** Try to match zArg against zS1, zS2, and so forth until the first +** emptry string. Return the index of the match or -1 if none is found. +** If no match is found, and &zErr is not NULL, then write into +** zErr a message describing the valid choices. +*/ +static int pickStr(const char *zArg, char **pzErr, ...){ + int i, n; + const char *z; + sqlite3_str *pMsg; + va_list ap; + va_start(ap, pzErr); + i = 0; + while( (z = va_arg(ap,const char*))!=0 && z[0]!=0 ){ + if( cli_strcmp(zArg, z)==0 ) return i; + i++; + } + va_end(ap); + if( pzErr==0 ) return -1; + n = i; + pMsg = sqlite3_str_new(0); + va_start(ap, pzErr); + sqlite3_str_appendall(pMsg, "should be"); + i = 0; + while( (z = va_arg(ap, const char*))!=0 && z[0]!=0 ){ + if( i==n-1 ){ + sqlite3_str_append(pMsg,", or",4); + }else if( i>0 ){ + sqlite3_str_append(pMsg, ",", 1); + } + sqlite3_str_appendf(pMsg, " %s", z); + i++; + } + va_end(ap); + *pzErr = sqlite3_str_finish(pMsg); + return -1; +} + +/* +** DOT-COMMAND: .import +** +** USAGE: .import [OPTIONS] FILE TABLE +** +** Import CSV or similar text from FILE into TABLE. If TABLE does +** not exist, it is created using the first row of FILE as the column +** names. If FILE begins with "|" then it is a command that is run +** and the output from the command is used as the input data. If +** FILE begins with "<<" followed by a label, then content is read from +** the script until the first line that matches the label. +** +** The content of FILE is interpreted using RFC-4180 ("CSV") quoting +** rules unless the current mode is "ascii" or "tabs" or unless one +** the --ascii option is used. +** +** The column and row separators must be single ASCII characters. If +** multiple characters or a Unicode character are specified for the +** separators, then only the first byte of the separator is used. Except, +** if the row separator is \n and the mode is not --ascii, then \r\n is +** understood as a row separator too. +** +** Options: +** --ascii Do not use RFC-4180 quoting. Use \037 and \036 +** as column and row separators on input, unless other +** delimiters are specified using --colsep and/or --rowsep +** --colsep CHAR Use CHAR as the column separator. +** --csv Input is standard RFC-4180 CSV. +** --esc CHAR Use CHAR as an escape character in unquoted CSV inputs. +** --qesc CHAR Use CHAR as an escape character in quoted CSV inputs. +** --rowsep CHAR Use CHAR as the row separator. +** --schema S When creating TABLE, put it in schema S +** --skip N Ignore the first N rows of input +** -v Verbose mode +*/ +static int dotCmdImport(ShellState *p){ + int nArg = p->dot.nArg; /* Number of arguments */ + char **azArg = p->dot.azArg;/* Argument list */ + char *zTable = 0; /* Insert data into this table */ + char *zSchema = 0; /* Schema of zTable */ + char *zFile = 0; /* Name of file to extra content from */ + sqlite3_stmt *pStmt = NULL; /* A statement */ + int nCol; /* Number of columns in the table */ + i64 nByte; /* Number of bytes in an SQL string */ + int i, j; /* Loop counters */ + int needCommit; /* True to COMMIT or ROLLBACK at end */ + char *zSql = 0; /* An SQL statement */ + ImportCtx sCtx; /* Reader context */ + char *(SQLITE_CDECL *xRead)(ImportCtx*); /* Func to read one value */ + int eVerbose = 0; /* Larger for more console output */ + i64 nSkip = 0; /* Initial lines to skip */ + i64 iLineOffset = 0; /* Offset to the first line of input */ + char *zCreate = 0; /* CREATE TABLE statement text */ + int rc; /* Result code */ + + failIfSafeMode(p, "cannot run .import in safe mode"); + memset(&sCtx, 0, sizeof(sCtx)); + if( p->mode.eMode==MODE_Ascii ){ + xRead = ascii_read_one_field; + }else{ + xRead = csv_read_one_field; + } + for(i=1; i<nArg; i++){ + char *z = azArg[i]; + if( z[0]=='-' && z[1]=='-' ) z++; + if( z[0]!='-' ){ + if( zFile==0 ){ + zFile = z; + }else if( zTable==0 ){ + zTable = z; + }else{ + dotCmdError(p, i, "unknown argument", 0); + return 1; + } + }else if( cli_strcmp(z,"-v")==0 ){ + eVerbose++; + }else if( cli_strcmp(z,"-schema")==0 && i<nArg-1 ){ + zSchema = azArg[++i]; + }else if( cli_strcmp(z,"-skip")==0 && i<nArg-1 ){ + nSkip = integerValue(azArg[++i]); + }else if( cli_strcmp(z,"-ascii")==0 ){ + if( sCtx.cColSep==0 ) sCtx.cColSep = SEP_Unit[0]; + if( sCtx.cRowSep==0 ) sCtx.cRowSep = SEP_Record[0]; + xRead = ascii_read_one_field; + }else if( cli_strcmp(z,"-csv")==0 ){ + if( sCtx.cColSep==0 ) sCtx.cColSep = ','; + if( sCtx.cRowSep==0 ) sCtx.cRowSep = '\n'; + xRead = csv_read_one_field; + }else if( cli_strcmp(z,"-esc")==0 ){ + sCtx.cUQEscape = azArg[++i][0]; + }else if( cli_strcmp(z,"-qesc")==0 ){ + sCtx.cQEscape = azArg[++i][0]; + }else if( cli_strcmp(z,"-colsep")==0 ){ + if( i==nArg-1 ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + i++; + sCtx.cColSep = azArg[i][0]; + }else if( cli_strcmp(z,"-rowsep")==0 ){ + if( i==nArg-1 ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + i++; + sCtx.cRowSep = azArg[i][0]; + }else{ + dotCmdError(p, i, "unknown option", 0); + return 1; + } + } + if( zTable==0 ){ + dotCmdError(p, nArg, 0, "Missing %s argument\n", + zFile==0 ? "FILE" : "TABLE"); + return 1; + } + seenInterrupt = 0; + open_db(p, 0); + if( sCtx.cColSep==0 ){ + if( p->mode.spec.zColumnSep && p->mode.spec.zColumnSep[0]!=0 ){ + sCtx.cColSep = p->mode.spec.zColumnSep[0]; + }else{ + sCtx.cColSep = ','; + } + } + if( (sCtx.cColSep & 0x80)!=0 ){ + eputz("Error: .import column separator must be ASCII\n"); + return 1; + } + if( sCtx.cRowSep==0 ){ + if( p->mode.spec.zRowSep && p->mode.spec.zRowSep[0]!=0 ){ + sCtx.cRowSep = p->mode.spec.zRowSep[0]; + }else{ + sCtx.cRowSep = '\n'; + } + } + if( sCtx.cRowSep=='\r' && xRead!=ascii_read_one_field ){ + sCtx.cRowSep = '\n'; + } + if( (sCtx.cRowSep & 0x80)!=0 ){ + eputz("Error: .import row separator must be ASCII\n"); + return 1; + } + sCtx.zFile = zFile; + sCtx.nLine = 1; + if( sCtx.zFile[0]=='|' ){ +#ifdef SQLITE_OMIT_POPEN + eputz("Error: pipes are not supported in this OS\n"); + return 1; +#else + sCtx.in = sqlite3_popen(sCtx.zFile+1, "r"); + sCtx.zFile = "<pipe>"; + sCtx.xCloser = pclose; +#endif + }else if( sCtx.zFile[0]=='<' && sCtx.zFile[1]=='<' && sCtx.zFile[2]!=0 ){ + /* Input text comes from subsequent lines of script until the zFile + ** delimiter */ + int nEndMark = strlen30(zFile)-2; + char *zEndMark = &zFile[2]; + sqlite3_str *pContent = sqlite3_str_new(p->db); + int ckEnd = 1; + i64 iStart = p->lineno; + char zLine[2000]; + sCtx.zFile = p->zInFile; + sCtx.nLine = p->lineno+1; + iLineOffset = p->lineno; + while( sqlite3_fgets(zLine,sizeof(zLine),p->in) ){ + if( ckEnd && cli_strncmp(zLine,zEndMark,nEndMark)==0 ){ + ckEnd = 2; + if( strchr(zLine,'\n') ) p->lineno++; + break; + } + if( strchr(zLine,'\n') ){ + p->lineno++; + ckEnd = 1; + }else{ + ckEnd = 0; + } + sqlite3_str_appendall(pContent, zLine); + } + sCtx.zIn = sqlite3_str_finish(pContent); + if( sCtx.zIn==0 ){ + sCtx.zIn = sqlite3_mprintf(""); + } + if( ckEnd<2 ){ + i64 savedLn = p->lineno; + p->lineno = iStart; + dotCmdError(p, 0, 0,"Content terminator \"%s\" not found.",zEndMark); + p->lineno = savedLn; + import_cleanup(&sCtx); + return 1; + } + }else{ + sCtx.in = sqlite3_fopen(sCtx.zFile, "rb"); + sCtx.xCloser = fclose; + } + if( sCtx.in==0 && sCtx.zIn==0 ){ + dotCmdError(p, 0, 0, "cannot open \"%s\"", zFile); + import_cleanup(&sCtx); + return 1; + } + if( eVerbose>=1 ){ + char zSep[2]; + zSep[1] = 0; + zSep[0] = sCtx.cColSep; + cli_puts("Column separator ", p->out); + output_c_string(p->out, zSep); + cli_puts(", row separator ", p->out); + zSep[0] = sCtx.cRowSep; + output_c_string(p->out, zSep); + cli_puts("\n", p->out); + } + sCtx.z = sqlite3_malloc64(120); + if( sCtx.z==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + /* Below, resources must be freed before exit. */ + while( nSkip>0 ){ + nSkip--; + while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){} + } + import_append_char(&sCtx, 0); /* To ensure sCtx.z is allocated */ + if( sqlite3_table_column_metadata(p->db, zSchema, zTable,0,0,0,0,0,0) + && 0==db_int(p->db, "SELECT count(*) FROM \"%w\".sqlite_schema" + " WHERE name=%Q AND type='view'", + zSchema ? zSchema : "main", zTable) + ){ + /* Table does not exist. Create it. */ + sqlite3 *dbCols = 0; + char *zRenames = 0; + char *zColDefs; + zCreate = sqlite3_mprintf("CREATE TABLE \"%w\".\"%w\"", + zSchema ? zSchema : "main", zTable); + while( xRead(&sCtx) ){ + zAutoColumn(sCtx.z, &dbCols, 0); + if( sCtx.cTerm!=sCtx.cColSep ) break; + } + zColDefs = zAutoColumn(0, &dbCols, &zRenames); + if( zRenames!=0 ){ + cli_printf((stdin_is_interactive && p->in==stdin)? p->out : stderr, + "Columns renamed during .import %s due to duplicates:\n" + "%s\n", sCtx.zFile, zRenames); + sqlite3_free(zRenames); + } + assert(dbCols==0); + if( zColDefs==0 ){ + cli_printf(stderr,"%s: empty file\n", sCtx.zFile); + import_cleanup(&sCtx); + sqlite3_free(zCreate); + return 1; + } + zCreate = sqlite3_mprintf("%z%z\n", zCreate, zColDefs); + if( zCreate==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + if( eVerbose>=1 ){ + cli_printf(p->out, "%s\n", zCreate); + } + rc = sqlite3_exec(p->db, zCreate, 0, 0, 0); + if( rc ){ + cli_printf(stderr, + "%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db)); + } + sqlite3_free(zCreate); + zCreate = 0; + if( rc ){ + import_cleanup(&sCtx); + return 1; + } + } + zSql = sqlite3_mprintf("SELECT count(*) FROM pragma_table_info(%Q,%Q);", + zTable, zSchema); + if( zSql==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + zSql = 0; + if( rc ){ + if (pStmt) sqlite3_finalize(pStmt); + shellDatabaseError(p->db); + import_cleanup(&sCtx); + return 1; + } + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + nCol = sqlite3_column_int(pStmt, 0); + }else{ + nCol = 0; + } + sqlite3_finalize(pStmt); + pStmt = 0; + if( nCol==0 ) return 0; /* no columns, no error */ + + nByte = 64 /* space for "INSERT INTO", "VALUES(", ")\0" */ + + (zSchema ? strlen(zSchema)*2 + 2: 0) /* Quoted schema name */ + + strlen(zTable)*2 + 2 /* Quoted table name */ + + nCol*2; /* Space for ",?" for each column */ + zSql = sqlite3_malloc64( nByte ); + if( zSql==0 ){ + import_cleanup(&sCtx); + shell_out_of_memory(); + } + if( zSchema ){ + sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\".\"%w\" VALUES(?", + zSchema, zTable); + }else{ + sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); + } + j = strlen30(zSql); + for(i=1; i<nCol; i++){ + zSql[j++] = ','; + zSql[j++] = '?'; + } + zSql[j++] = ')'; + zSql[j] = 0; + assert( j<nByte ); + if( eVerbose>=2 ){ + cli_printf(p->out, "Insert using: %s\n", zSql); + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + zSql = 0; + if( rc ){ + shellDatabaseError(p->db); + if (pStmt) sqlite3_finalize(pStmt); + import_cleanup(&sCtx); + return 1; + } + needCommit = sqlite3_get_autocommit(p->db); + if( needCommit ) sqlite3_exec(p->db, "BEGIN", 0, 0, 0); + do{ + int startLine = sCtx.nLine; + for(i=0; i<nCol; i++){ + char *z = xRead(&sCtx); + /* + ** Did we reach end-of-file before finding any columns? + ** If so, stop instead of NULL filling the remaining columns. + */ + if( z==0 && i==0 ) break; + /* + ** Did we reach end-of-file OR end-of-line before finding any + ** columns in ASCII mode? If so, stop instead of NULL filling + ** the remaining columns. + */ + if( p->mode.eMode==MODE_Ascii && (z==0 || z[0]==0) && i==0 ) break; + /* + ** For CSV mode, per RFC 4180, accept EOF in lieu of final + ** record terminator but only for last field of multi-field row. + ** (If there are too few fields, it's not valid CSV anyway.) + */ + if( z==0 && (xRead==csv_read_one_field) && i==nCol-1 && i>0 ){ + z = ""; + } + sqlite3_bind_text(pStmt, i+1, z, -1, SQLITE_TRANSIENT); + if( i<nCol-1 && sCtx.cTerm!=sCtx.cColSep ){ + if( i==0 && (strcmp(z,"\n")==0 || strcmp(z,"\r\n")==0) ){ + /* Ignore trailing \n or \r\n when some other row separator */ + break; + } + cli_printf(stderr,"%s:%d: expected %d columns but found %d" + " - filling the rest with NULL\n", + sCtx.zFile, startLine, nCol, i+1); + i += 2; + while( i<=nCol ){ sqlite3_bind_null(pStmt, i); i++; } + } + } + if( sCtx.cTerm==sCtx.cColSep ){ + do{ + xRead(&sCtx); + i++; + }while( sCtx.cTerm==sCtx.cColSep ); + cli_printf(stderr, + "%s:%d: expected %d columns but found %d - extras ignored\n", + sCtx.zFile, startLine, nCol, i); + } + if( i>=nCol ){ + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + cli_printf(stderr,"%s:%d: INSERT failed: %s\n", + sCtx.zFile, startLine, sqlite3_errmsg(p->db)); + sCtx.nErr++; + if( bail_on_error ) break; + }else{ + sCtx.nRow++; + } + } + }while( sCtx.cTerm!=EOF ); + + import_cleanup(&sCtx); + sqlite3_finalize(pStmt); + if( needCommit ) sqlite3_exec(p->db, "COMMIT", 0, 0, 0); + if( eVerbose>0 ){ + cli_printf(p->out, + "Added %d rows with %d errors using %d lines of input\n", + sCtx.nRow, sCtx.nErr, sCtx.nLine-1-iLineOffset); + } + return sCtx.nErr ? 1 : 0; +} + + +/* +** This function computes what to show the user about the configured +** titles (or column-names). Output is an integer between 0 and 3: +** +** 0: The titles do not matter. Never show anything. +** 1: Show "--titles off" +** 2: Show "--titles on" +** 3: Show "--title VALUE" where VALUE is an encoding method +** to use, one of: plain sql csv html tcl json +** +** Inputs are: +** +** spec.bTitles (bT) Whether or not to show the titles +** spec.eTitle (eT) The actual encoding to be used for titles +** ModeInfo.bHdr (bH) Default value for spec.bTitles +** ModeInfo.eHdr (eH) Default value for spec.eTitle +** bAll Whether the -v option is used +*/ +static int modeTitleDsply(ShellState *p, int bAll){ + int eMode = p->mode.eMode; + const ModeInfo *pI = &aModeInfo[eMode]; + int bT = p->mode.spec.bTitles; + int eT = p->mode.spec.eTitle; + int bH = pI->bHdr; + int eH = pI->eHdr; + + /* Variable "v" is the truth table that will determine the answer + ** + ** Actual encoding is different from default + ** vvvvvvvv */ + sqlite3_uint64 v = UINT64_C(0x0133013311220102); + /* ^^^^ ^^^^ + ** Upper 2-byte groups for when ON/OFF disagrees with + ** the default. */ + + if( bH==0 ) return 0; /* Header not appliable. Ex: off, count */ + + if( eT==0 ) eT = eH; /* Fill in missing spec.eTitle */ + if( bT==0 ) bT = bH; /* Fill in missing spec.bTitles */ + + if( eT!=eH ) v >>= 32; /* Encoding disagree in upper 4-bytes */ + if( bT!=bH ) v >>= 16; /* ON/OFF disagree in upper 2-byte pairs */ + if( bT<2 ) v >>= 8; /* ON in even bytes, OFF in odd bytes (1st byte 0) */ + if( !bAll ) v >>= 4; /* bAll values are in the lower half-byte */ + + return v & 3; /* Return the selected truth-table entry */ +} + +/* +** DOT-COMMAND: .mode +** +** USAGE: .mode [MODE] [OPTIONS] +** +** Change the output mode to MODE and/or apply OPTIONS to the output mode. +** Arguments are processed from left to right. If no arguments, show the +** current output mode and relevant options. +** +** Options: +** --align STRING Set the alignment of text in columnar modes +** String consists of characters 'L', 'C', 'R' +** meaning "left", "centered", and "right", with +** one letter per column starting from the left. +** Unspecified alignment defaults to 'L'. +** --blob-quote ARG ARG can be "auto", "text", "sql", "hex", "tcl", +** "json", or "size". Default is "auto". +** --border on|off Show outer border on "box" and "table" modes. +** --charlimit N Set the maximum number of output characters to +** show for any single SQL value to N. Longer values +** truncated. Zero means "no limit". +** --colsep STRING Use STRING as the column separator +** --escape ESC Enable/disable escaping of control characters +** found in the output. ESC can be "off", "ascii", +** or "symbol". +** --linelimit N Set the maximum number of output lines to show for +** any single SQL value to N. Longer values are +** truncated. Zero means "no limit". Only works +** in "line" mode and in columnar modes. +** --limits L,C,T Shorthand for "--linelimit L --charlimit C +** --titlelimit T". The ",T" can be omitted in which +** case the --titlelimit is unchanged. The argument +** can also be "off" to mean "0,0,0" or "on" to +** mean "5,300,20". +** --list List available modes +** --multiinsert N In "insert" mode, put multiple rows on a single +** INSERT statement until the size exceeds N bytes. +** --null STRING Render SQL NULL values as the given string +** --once Setting changes to the right are reverted after +** the next SQL command. +** --quote ARG Enable/disable quoting of text. ARG can be +** "off", "on", "sql", "relaxed", "csv", "html", +** "tcl", or "json". "off" means show the text as-is. +** "on" is an alias for "sql". +** --reset Changes all mode settings back to their default. +** --rowsep STRING Use STRING as the row separator +** --sw|--screenwidth N Declare the screen width of the output device +** to be N characters. An attempt may be made to +** wrap output text to fit within this limit. Zero +** means "no limit". Or N can be "auto" to set the +** width automatically. +** --tablename NAME Set the name of the table for "insert" mode. +** --tag NAME Save mode to the left as NAME. +** --textjsonb BOOLEAN If enabled, JSONB text is displayed as text JSON. +** --title ARG Whether or not to show column headers, and if so +** how to encode them. ARG can be "off", "on", +** "sql", "csv", "html", "tcl", or "json". +** --titlelimit N Limit the length of column titles to N characters. +** -v|--verbose Verbose output +** --widths LIST Set the columns widths for columnar modes. The +** argument is a list of integers, one for each +** column. A "0" width means use a dynamic width +** based on the actual width of data. If there are +** fewer entries in LIST than columns, "0" is used +** for the unspecified widths. +** --wordwrap BOOLEAN Enable/disable word wrapping +** --wrap N Wrap columns wider than N characters +** --ww Shorthand for "--wordwrap on" +*/ +static int dotCmdMode(ShellState *p){ + int nArg = p->dot.nArg; /* Number of arguments */ + char **azArg = p->dot.azArg;/* Argument list */ + int eMode = -1; /* New mode value, or -1 for none */ + int iMode = -1; /* Index of the argument that is the mode name */ + int i; /* Loop counter */ + int k; /* Misc index variable */ + int chng = 0; /* True if anything has changed */ + int bAll = 0; /* Show all details of the mode */ + + for(i=1; i<nArg; i++){ + const char *z = azArg[i]; + if( z[0]=='-' && z[1]=='-' ) z++; + if( z[0]!='-' + && iMode<0 + && (eMode = modeFind(p, azArg[i]))>=0 + && eMode!=MODE_Www + ){ + iMode = i; + modeChange(p, eMode); + /* (Legacy) If the mode is MODE_Insert and the next argument + ** is not an option, then the next argument must be the table + ** name. + */ + if( i+1<nArg && azArg[i+1][0]!='-' ){ + i++; + modeSetStr(&p->mode.spec.zTableName, azArg[i]); + } + chng = 1; + }else if( optionMatch(z,"align") ){ + char *zAlign; + int nAlign; + int nErr = 0; + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + i++; + zAlign = azArg[i]; + nAlign = 0x3fff & strlen(zAlign); + free(p->mode.spec.aAlign); + p->mode.spec.aAlign = malloc(nAlign); + shell_check_oom(p->mode.spec.aAlign); + for(k=0; k<nAlign; k++){ + unsigned char c = 0; + switch( zAlign[k] ){ + case 'l': case 'L': c = QRF_ALIGN_Left; break; + case 'c': case 'C': c = QRF_ALIGN_Center; break; + case 'r': case 'R': c = QRF_ALIGN_Right; break; + default: nErr++; break; + } + p->mode.spec.aAlign[k] = c; + } + p->mode.spec.nAlign = nAlign; + chng = 1; + if( nErr ){ + dotCmdError(p, i, "bad alignment string", + "Should contain only characters L, C, and R."); + return 1; + } + }else if( pickStr(z,0,"-blob","-blob-quote","")>=0 ){ + if( (++i)>=nArg ){ + dotCmdError(p, i-1, "missing argument", 0); + return 1; + } + k = pickStr(azArg[i], 0, + "auto", "text", "sql", "hex", "tcl", "json", "size", ""); + /* 0 1 2 3 4 5 6 + ** Must match QRF_BLOB_xxxx values. See also tag-20251124a */ + if( k>=0 ){ + p->mode.spec.eBlob = k & 0xff; + } + chng = 1; + }else if( optionMatch(z,"border") ){ + if( (++i)>=nArg ){ + dotCmdError(p, i-1, "missing argument", 0); + return 1; + } + k = pickStr(azArg[i], 0, "auto", "off", "on", ""); + if( k>=0 ){ + p->mode.spec.bBorder = k & 0x3; + } + chng = 1; + }else if( 0<=(k=pickStr(z,0, + "-charlimit","-linelimit","-titlelimit","-multiinsert","")) ){ + int w; /* 0 1 2 3 */ + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + w = integerValue(azArg[++i]); + switch( k ){ + case 0: p->mode.spec.nCharLimit = w; break; + case 1: p->mode.spec.nLineLimit = w; break; + case 2: p->mode.spec.nTitleLimit = w; break; + default: p->mode.spec.nMultiInsert = w; break; + } + chng = 1; + }else if( 0<=(k=pickStr(z,0,"-tablename","-rowsep","-colsep","-null","")) ){ + /* 0 1 2 3 */ + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + i++; + switch( k ){ + case 0: modeSetStr(&p->mode.spec.zTableName, azArg[i]); break; + case 1: modeSetStr(&p->mode.spec.zRowSep, azArg[i]); break; + case 2: modeSetStr(&p->mode.spec.zColumnSep, azArg[i]); break; + case 3: modeSetStr(&p->mode.spec.zNull, azArg[i]); break; + } + chng = 1; + }else if( optionMatch(z,"escape") ){ + /* See similar code at tag-20250224-1 */ + char *zErr = 0; + if( (++i)>=nArg ){ + dotCmdError(p, i-1, "missing argument", 0); + return 1; + } /* 0 1 2 <-- One less than QRF_ESC_ */ + k = pickStr(azArg[i],&zErr,"off","ascii","symbol",""); + if( k<0 ){ + dotCmdError(p, i, "unknown escape type", "%s", zErr); + sqlite3_free(zErr); + return 1; + } + p->mode.spec.eEsc = k+1; + chng = 1; + }else if( optionMatch(z,"limits") ){ + if( (++i)>=nArg ){ + dotCmdError(p, i-1, "missing argument", 0); + return 1; + } + k = pickStr(azArg[i],0,"on","off",""); + if( k==0 ){ + p->mode.spec.nLineLimit = DFLT_LINE_LIMIT; + p->mode.spec.nCharLimit = DFLT_CHAR_LIMIT; + p->mode.spec.nTitleLimit = DFLT_TITLE_LIMIT; + }else if( k==1 ){ + p->mode.spec.nLineLimit = 0; + p->mode.spec.nCharLimit = 0; + p->mode.spec.nTitleLimit = 0; + }else{ + int L, C, T = 0; + int nNum = sscanf(azArg[i], "%d,%d,%d", &L, &C, &T); + if( nNum<2 || L<0 || C<0 || T<0){ + dotCmdError(p, i, "bad argument", "Should be \"L,C,T\" where L, C" + " and T are unsigned integers"); + return 1; + } + p->mode.spec.nLineLimit = L; + p->mode.spec.nCharLimit = C; + if( nNum==3 ) p->mode.spec.nTitleLimit = T; + } + chng = 1; + }else if( optionMatch(z,"list") ){ + int ii; + cli_puts("available modes:", p->out); + for(ii=0; ii<ArraySize(aModeInfo); ii++){ + if( ii==MODE_Www ) continue; + cli_printf(p->out, " %s", aModeInfo[ii].zName); + } + for(ii=0; ii<p->nSavedModes; ii++){ + cli_printf(p->out, " %s", p->aSavedModes[ii].zTag); + } + cli_puts(" batch tty\n", p->out); + chng = 1; /* Not really a change, but we still want to suppress the + ** "current mode" output */ + }else if( optionMatch(z,"once") ){ + p->nPopMode = 0; + modePush(p); + p->nPopMode = 1; + }else if( optionMatch(z,"noquote") ){ + /* (undocumented legacy) --noquote always turns quoting off */ + p->mode.spec.eText = QRF_TEXT_Plain; + p->mode.spec.eBlob = QRF_BLOB_Auto; + chng = 1; + }else if( optionMatch(z,"quote") ){ + if( i+1<nArg + && azArg[i+1][0]!='-' + && (iMode>0 || strcmp(azArg[i+1],"off")==0 || modeFind(p, azArg[i+1])<0) + ){ + /* --quote is followed by an argument other that is not an option + ** or a mode name. See it must be a boolean or a keyword to describe + ** how to set quoting. */ + i++; + if( (k = pickStr(azArg[i],0,"no","yes","0","1",""))>=0 ){ + k &= 1; /* 0 for "off". 1 for "on". */ + }else{ + char *zErr = 0; + k = pickStr(azArg[i],&zErr, + "off","on","sql","csv","html","tcl","json","relaxed",""); + /* 0 1 2 3 4 5 6 7 */ + if( k<0 ){ + dotCmdError(p, i, "unknown", "%z", zErr); + return 1; + } + } + }else{ + /* (Legacy) no following boolean argument. Turn quoting on */ + k = 1; + } + switch( k ){ + case 1: /* on */ + modeSetStr(&p->mode.spec.zNull, "NULL"); + /* Fall through */ + case 2: /* sql */ + p->mode.spec.eText = QRF_TEXT_Sql; + break; + case 3: /* csv */ + p->mode.spec.eText = QRF_TEXT_Csv; + break; + case 4: /* html */ + p->mode.spec.eText = QRF_TEXT_Html; + break; + case 5: /* tcl */ + p->mode.spec.eText = QRF_TEXT_Tcl; + break; + case 6: /* json */ + p->mode.spec.eText = QRF_TEXT_Json; + break; + case 7: /* relaxed */ + p->mode.spec.eText = QRF_TEXT_Relaxed; + break; + default: /* off */ + p->mode.spec.eText = QRF_TEXT_Plain; + break; + } + chng = 1; + }else if( optionMatch(z,"reset") ){ + int saved_eMode = p->mode.eMode; + modeFree(&p->mode); + modeChange(p, saved_eMode); + }else if( optionMatch(z,"screenwidth") || optionMatch(z,"sw") ){ + if( (++i)>=nArg ){ + dotCmdError(p, i-1, "missing argument", 0); + return 1; + } + k = pickStr(azArg[i],0,"off","auto",""); + if( k==0 ){ + p->mode.bAutoScreenWidth = 0; + p->mode.spec.nScreenWidth = 0; + }else if( k==1 ){ + p->mode.bAutoScreenWidth = 1; + }else{ + i64 w = integerValue(azArg[i]); + p->mode.bAutoScreenWidth = 0; + if( w<0 ) w = 0; + if( w>QRF_MAX_WIDTH ) w = QRF_MAX_WIDTH; + p->mode.spec.nScreenWidth = w; + } + chng = 1; + }else if( optionMatch(z,"tag") ){ + size_t nByte; + int n; + const char *zTag; + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + zTag = azArg[++i]; + if( modeFind(p, zTag)>=0 ){ + dotCmdError(p, i, "mode already exists", 0); + return 1; + } + if( p->nSavedModes > MODE_N_USER ){ + dotCmdError(p, i-1, "cannot add more modes", 0); + return 1; + } + n = p->nSavedModes++; + nByte = sizeof(p->aSavedModes[0]); + nByte *= n+1; + p->aSavedModes = realloc( p->aSavedModes, nByte ); + shell_check_oom(p->aSavedModes); + p->aSavedModes[n].zTag = strdup(zTag); + shell_check_oom(p->aSavedModes[n].zTag); + modeDup(&p->aSavedModes[n].mode, &p->mode); + chng = 1; + }else if( optionMatch(z,"textjsonb") ){ + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + p->mode.spec.bTextJsonb = booleanValue(azArg[++i]) ? QRF_Yes : QRF_No; + chng = 1; + }else if( optionMatch(z,"titles") || optionMatch(z,"title") ){ + char *zErr = 0; + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + k = pickStr(azArg[++i],&zErr, + "off","on","plain","sql","csv","html","tcl","json",""); + /* 0 1 2 3 4 5 6 7 */ + if( k<0 ){ + dotCmdError(p, i, "bad --titles value","%z", zErr); + return 1; + } + p->mode.spec.bTitles = k>=1 ? QRF_Yes : QRF_No; + p->mode.mFlags &= ~MFLG_HDR; + p->mode.spec.eTitle = k>1 ? k-1 : aModeInfo[p->mode.eMode].eHdr; + chng = 1; + }else if( optionMatch(z,"widths") || optionMatch(z,"width") ){ + int nWidth = 0; + short int *aWidth; + const char *zW; + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + zW = azArg[++i]; + /* Every width value takes at least 2 bytes in the input string to + ** specify, so strlen(zW) bytes should be plenty of space to hold the + ** result. */ + aWidth = malloc( strlen(zW) ); + while( IsSpace(zW[0]) ) zW++; + while( zW[0] ){ + int w = 0; + int nDigit = 0; + k = zW[0]=='-' && IsDigit(zW[1]); + while( IsDigit(zW[k]) ){ + w = w*10 + zW[k] - '0'; + if( w>QRF_MAX_WIDTH ){ + dotCmdError(p,i+1,"width too big", + "Maximum column width is %d", QRF_MAX_WIDTH); + free(aWidth); + return 1; + } + nDigit++; + k++; + } + if( nDigit==0 ){ + dotCmdError(p,i+1,"syntax error", + "should be a comma-separated list if integers"); + free(aWidth); + return 1; + } + if( zW[0]=='-' ) w = -w; + aWidth[nWidth++] = w; + zW += k; + if( zW[0]==',' ) zW++; + while( IsSpace(zW[0]) ) zW++; + } + free(p->mode.spec.aWidth); + p->mode.spec.aWidth = aWidth; + p->mode.spec.nWidth = nWidth; + chng = 1; + }else if( optionMatch(z,"wrap") ){ + int w; + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + w = integerValue(azArg[++i]); + if( w<(-QRF_MAX_WIDTH) ) w = -QRF_MAX_WIDTH; + if( w>QRF_MAX_WIDTH ) w = QRF_MAX_WIDTH; + p->mode.spec.nWrap = w; + chng = 1; + }else if( optionMatch(z,"ww") ){ + p->mode.spec.bWordWrap = QRF_Yes; + chng = 1; + }else if( optionMatch(z,"wordwrap") ){ + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + p->mode.spec.bWordWrap = (u8)booleanValue(azArg[++i]) ? QRF_Yes : QRF_No; + chng = 1; + }else if( optionMatch(z,"v") || optionMatch(z,"verbose") ){ + bAll = 1; + }else if( z[0]=='-' ){ + dotCmdError(p, i, "bad option", "Use \".help .mode\" for more info"); + return 1; + }else{ + dotCmdError(p, i, iMode>0?"bad argument":"unknown mode", + "Use \".help .mode\" for more info"); + return 1; + } + } + if( !chng || bAll ){ + const ModeInfo *pI = aModeInfo + p->mode.eMode; + sqlite3_str *pDesc = sqlite3_str_new(p->db); + char *zDesc; + const char *zSetting; + + if( p->nPopMode ) sqlite3_str_appendall(pDesc, "--once "); + sqlite3_str_appendall(pDesc,pI->zName); + if( bAll || (p->mode.spec.nAlign && pI->eCx==2) ){ + int ii; + sqlite3_str_appendall(pDesc, " --align \""); + for(ii=0; ii<p->mode.spec.nAlign; ii++){ + unsigned char a = p->mode.spec.aAlign[ii]; + sqlite3_str_appendchar(pDesc, 1, "LLCR"[a&3]); + } + sqlite3_str_append(pDesc, "\"", 1); + } + if( bAll + || (p->mode.spec.bBorder==QRF_No) != ((pI->mFlg&1)!=0) + ){ + sqlite3_str_appendf(pDesc," --border %s", + p->mode.spec.bBorder==QRF_No ? "off" : "on"); + } + if( bAll || p->mode.spec.eBlob!=QRF_BLOB_Auto ){ + const char *azBQuote[] = + { "auto", "text", "sql", "hex", "tcl", "json", "size" }; + /* 0 1 2 3 4 5 6 + ** Must match QRF_BLOB_xxxx values. See all instances of tag-20251124a */ + u8 e = p->mode.spec.eBlob; + sqlite3_str_appendf(pDesc, " --blob-quote %s", azBQuote[e]); + } + zSetting = aModeStr[pI->eCSep]; + if( bAll || (zSetting && cli_strcmp(zSetting,p->mode.spec.zColumnSep)!=0) ){ + sqlite3_str_appendf(pDesc, " --colsep "); + append_c_string(pDesc, p->mode.spec.zColumnSep); + } + if( bAll || p->mode.spec.eEsc!=QRF_Auto ){ + sqlite3_str_appendf(pDesc, " --escape %s",qrfEscNames[p->mode.spec.eEsc]); + } + if( bAll + || (p->mode.spec.nLineLimit>0 && pI->eCx>0) + || p->mode.spec.nCharLimit>0 + || (p->mode.spec.nTitleLimit>0 && pI->eCx>0) + ){ + if( p->mode.spec.nLineLimit==0 + && p->mode.spec.nCharLimit==0 + && p->mode.spec.nTitleLimit==0 + ){ + sqlite3_str_appendf(pDesc, " --limits off"); + }else if( p->mode.spec.nLineLimit==DFLT_LINE_LIMIT + && p->mode.spec.nCharLimit==DFLT_CHAR_LIMIT + && p->mode.spec.nTitleLimit==DFLT_TITLE_LIMIT + ){ + sqlite3_str_appendf(pDesc, " --limits on"); + }else{ + sqlite3_str_appendf(pDesc, " --limits %d,%d,%d", + p->mode.spec.nLineLimit, p->mode.spec.nCharLimit, + p->mode.spec.nTitleLimit); + } + } + if( bAll + || (p->mode.spec.nMultiInsert && p->mode.spec.eStyle==QRF_STYLE_Insert) + ){ + sqlite3_str_appendf(pDesc, " --multiinsert %u", + p->mode.spec.nMultiInsert); + } + zSetting = aModeStr[pI->eNull]; + if( bAll || (zSetting && cli_strcmp(zSetting,p->mode.spec.zNull)!=0) ){ + sqlite3_str_appendf(pDesc, " --null "); + append_c_string(pDesc, p->mode.spec.zNull); + } + if( bAll + || (pI->eText!=p->mode.spec.eText && (pI->eText>1 || p->mode.spec.eText>1)) + ){ + sqlite3_str_appendf(pDesc," --quote %s",qrfQuoteNames[p->mode.spec.eText]); + } + zSetting = aModeStr[pI->eRSep]; + if( bAll || (zSetting && cli_strcmp(zSetting,p->mode.spec.zRowSep)!=0) ){ + sqlite3_str_appendf(pDesc, " --rowsep "); + append_c_string(pDesc, p->mode.spec.zRowSep); + } + if( bAll + || (pI->eCx && (p->mode.spec.nScreenWidth>0 || p->mode.bAutoScreenWidth)) + ){ + if( p->mode.bAutoScreenWidth ){ + sqlite3_str_appendall(pDesc, " --sw auto"); + }else{ + sqlite3_str_appendf(pDesc," --sw %d", + p->mode.spec.nScreenWidth); + } + } + if( bAll || p->mode.eMode==MODE_Insert ){ + sqlite3_str_appendf(pDesc," --tablename "); + append_c_string(pDesc, p->mode.spec.zTableName); + } + if( bAll || p->mode.spec.bTextJsonb ){ + sqlite3_str_appendf(pDesc," --textjsonb %s", + p->mode.spec.bTextJsonb==QRF_Yes ? "on" : "off"); + } + k = modeTitleDsply(p, bAll); + if( k==1 ){ + sqlite3_str_appendall(pDesc, " --titles off"); + }else if( k==2 ){ + sqlite3_str_appendall(pDesc, " --titles on"); + }else if( k==3 ){ + static const char *azTitle[] = + { "plain", "sql", "csv", "html", "tcl", "json"}; + sqlite3_str_appendf(pDesc, " --titles %s", + azTitle[p->mode.spec.eTitle-1]); + } + if( p->mode.spec.nWidth>0 && (bAll || pI->eCx==2) ){ + int ii; + const char *zSep = " --widths "; + for(ii=0; ii<p->mode.spec.nWidth; ii++){ + sqlite3_str_appendf(pDesc, "%s%d", zSep, (int)p->mode.spec.aWidth[ii]); + zSep = ","; + } + }else if( bAll ){ + sqlite3_str_appendall(pDesc, " --widths \"\""); + } + if( bAll || (pI->eCx>0 && p->mode.spec.bWordWrap) ){ + if( bAll ){ + sqlite3_str_appendf(pDesc, " --wordwrap %s", + p->mode.spec.bWordWrap==QRF_Yes ? "on" : "off"); + } + if( p->mode.spec.nWrap ){ + sqlite3_str_appendf(pDesc, " --wrap %d", p->mode.spec.nWrap); + } + if( !bAll ) sqlite3_str_append(pDesc, " --ww", 5); + } + zDesc = sqlite3_str_finish(pDesc); + cli_printf(p->out, ".mode %s\n", zDesc); + fflush(p->out); + sqlite3_free(zDesc); + } + return 0; +} + +/* +** DOT-COMMAND: .output +** USAGE: .output [OPTIONS] [FILE] +** +** Begin redirecting output to FILE. Or if FILE is omitted, revert +** to sending output to the console. If FILE begins with "|" then +** the remainder of file is taken as a pipe and output is directed +** into that pipe. If FILE is "memory" then output is captured in an +** internal memory buffer. If FILE is "off" then output is redirected +** into /dev/null or the equivalent. +** +** Options: +** --bom Prepend a byte-order mark to the output +** -e Accumulate output in a temporary text file then +** launch a text editor when the redirection ends. +** --error-prefix X Use X as the left-margin prefix for error messages. +** Set to an empty string to restore the default. +** --keep Keep redirecting output to its current destination. +** Use this option in combination with --show or +** with --error-prefix when you do not want to stop +** a current redirection. +** --plain Use plain text rather than HTML tables with -w +** --show Show output text captured by .testcase or by +** redirecting to "memory". +** -w Show the output in a web browser. Output is +** written into a temporary HTML file until the +** redirect ends, then the web browser is launched. +** Query results are shown as HTML tables, unless +** the --plain is used too. +** -x Show the output in a spreadsheet. Output is +** written to a temp file as CSV then the spreadsheet +** is launched when +** +** DOT-COMMAND: .once +** USAGE: .once [OPTIONS] FILE ... +** +** Write the output for the next line of SQL or the next dot-command into +** FILE. If FILE begins with "|" then it is a program into which output +** is written. The FILE argument should be omitted if one of the -e, -w, +** or -x options is used. +** +** Options: +** -e Capture output into a temporary file then bring up +** a text editor on that temporary file. +** --plain Use plain text rather than HTML tables with -w +** -w Capture output into an HTML file then bring up that +** file in a web browser +** -x Show the output in a spreadsheet. Output is +** written to a temp file as CSV then the spreadsheet +** is launched when +** +** DOT-COMMAND: .excel +** Shorthand for ".once -x" +** +** DOT-COMMAND: .www [--plain] +** Shorthand for ".once -w" or ".once --plain -w" +*/ +static int dotCmdOutput(ShellState *p){ + int nArg = p->dot.nArg; /* Number of arguments */ + char **azArg = p->dot.azArg; /* Text of the arguments */ + char *zFile = 0; /* The FILE argument */ + int i; /* Loop counter */ + int eMode = 0; /* 0: .outout/.once, 'x'=.excel, 'w'=.www */ + int bOnce = 0; /* 0: .output, 1: .once, 2: .excel/.www */ + int bPlain = 0; /* --plain option */ + int bKeep = 0; /* Keep redirecting */ + static const char *zBomUtf8 = "\357\273\277"; + const char *zBom = 0; + char c = azArg[0][0]; + int n = strlen30(azArg[0]); + + failIfSafeMode(p, "cannot run .%s in safe mode", azArg[0]); + if( c=='e' ){ + eMode = 'x'; + bOnce = 2; + }else if( c=='w' ){ + eMode = 'w'; + bOnce = 2; + }else if( n>=2 && cli_strncmp(azArg[0],"once",n)==0 ){ + bOnce = 1; + } + for(i=1; i<nArg; i++){ + char *z = azArg[i]; + if( z[0]=='-' ){ + if( z[1]=='-' ) z++; + if( cli_strcmp(z,"-bom")==0 ){ + zBom = zBomUtf8; + }else if( cli_strcmp(z,"-plain")==0 ){ + bPlain = 1; + }else if( c=='o' && z[0]=='1' && z[1]!=0 && z[2]==0 + && (z[1]=='x' || z[1]=='e' || z[1]=='w') ){ + if( bKeep || eMode ){ + dotCmdError(p, i, "incompatible with prior options",0); + goto dotCmdOutput_error; + } + eMode = z[1]; + }else if( cli_strcmp(z,"-show")==0 ){ + if( cli_output_capture ){ + sqlite3_fprintf(stdout, "%s", sqlite3_str_value(cli_output_capture)); + } + }else if( cli_strcmp(z,"-keep")==0 ){ + bKeep = 1; + }else if( optionMatch(z,"error-prefix") ){ + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + free(p->zErrPrefix); + i++; + p->zErrPrefix = azArg[i][0]==0 ? 0 : strdup(azArg[i]); + }else{ + dotCmdError(p, i, "unknown option", 0); + sqlite3_free(zFile); + return 1; + } + }else if( zFile==0 && eMode==0 ){ + if( bKeep ){ + dotCmdError(p, i, "incompatible with prior options",0); + goto dotCmdOutput_error; + } + if( cli_strcmp(z, "memory")==0 && bOnce ){ + dotCmdError(p, 0, "cannot redirect to \"memory\"", 0); + goto dotCmdOutput_error; + } + if( cli_strcmp(z, "off")==0 ){ +#ifdef _WIN32 + zFile = sqlite3_mprintf("nul"); +#else + zFile = sqlite3_mprintf("/dev/null"); +#endif + }else{ + zFile = sqlite3_mprintf("%s", z); + } + if( zFile && zFile[0]=='|' ){ + while( i+1<nArg ) zFile = sqlite3_mprintf("%z %s", zFile, azArg[++i]); + break; + } + }else{ + dotCmdError(p, i, "surplus argument", 0); + sqlite3_free(zFile); + return 1; + } + } + if( zFile==0 && !bKeep ){ + zFile = sqlite3_mprintf("stdout"); + shell_check_oom(zFile); + } + if( bOnce ){ + p->nPopOutput = 2; + }else{ + p->nPopOutput = 0; + } + if( !bKeep ) output_reset(p); +#ifndef SQLITE_NOHAVE_SYSTEM + if( eMode=='e' || eMode=='x' || eMode=='w' ){ + p->doXdgOpen = 1; + modePush(p); + if( eMode=='x' ){ + /* spreadsheet mode. Output as CSV. */ + newTempFile(p, "csv"); + p->mode.mFlags &= ~MFLG_ECHO; + p->mode.eMode = MODE_Csv; + modeSetStr(&p->mode.spec.zColumnSep, SEP_Comma); + modeSetStr(&p->mode.spec.zRowSep, SEP_CrLf); +#ifdef _WIN32 + zBom = zBomUtf8; /* Always include the BOM on Windows, as Excel does + ** not work without it. */ +#endif + }else if( eMode=='w' ){ + /* web-browser mode. */ + newTempFile(p, "html"); + if( !bPlain ) p->mode.eMode = MODE_Www; + }else{ + /* text editor mode */ + newTempFile(p, "txt"); + } + sqlite3_free(zFile); + zFile = sqlite3_mprintf("%s", p->zTempFile); + } +#endif /* SQLITE_NOHAVE_SYSTEM */ + if( !bKeep ) shell_check_oom(zFile); + if( bKeep ){ + /* no-op */ + }else if( cli_strcmp(zFile,"memory")==0 ){ + if( cli_output_capture ){ + sqlite3_str_free(cli_output_capture); + } + cli_output_capture = sqlite3_str_new(0); + }else if( zFile[0]=='|' ){ +#ifdef SQLITE_OMIT_POPEN + eputz("Error: pipes are not supported in this OS\n"); + output_redir(p, stdout); + goto dotCmdOutput_error; +#else + FILE *pfPipe = sqlite3_popen(zFile + 1, "w"); + if( pfPipe==0 ){ + assert( stderr!=NULL ); + cli_printf(stderr,"Error: cannot open pipe \"%s\"\n", zFile + 1); + goto dotCmdOutput_error; + }else{ + output_redir(p, pfPipe); + if( zBom ) cli_puts(zBom, pfPipe); + sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); + } +#endif + }else{ + FILE *pfFile = output_file_open(p, zFile); + if( pfFile==0 ){ + if( cli_strcmp(zFile,"off")!=0 ){ + assert( stderr!=NULL ); + cli_printf(stderr,"Error: cannot write to \"%s\"\n", zFile); + } + goto dotCmdOutput_error; + } else { + output_redir(p, pfFile); + if( zBom ) cli_puts(zBom, pfFile); + if( bPlain && eMode=='w' ){ + cli_puts( + "<!DOCTYPE html>\n<BODY>\n<PLAINTEXT>\n", + pfFile + ); + } + sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); + } + } + sqlite3_free(zFile); + return 0; + +dotCmdOutput_error: + sqlite3_free(zFile); + return 1; +} + +/* +** DOT-COMMAND: .check +** USAGE: .check [OPTIONS] PATTERN +** +** Verify results of commands since the most recent .testcase command. +** Restore output to the console, unless --keep is used. +** +** If PATTERN starts with "<<ENDMARK" then the actual pattern is taken from +** subsequent lines of text up to the first line that begins with ENDMARK. +** All pattern lines and the ENDMARK are discarded. +** +** Options: +** --exact Do an exact comparison including leading and +** trailing whitespace. +** --glob Treat PATTERN as a GLOB +** --keep Do not reset the testcase. More .check commands +** will follow. +** --notglob Output should not match PATTERN +** --show Write testcase output to the screen, for debugging. +*/ +static int dotCmdCheck(ShellState *p){ + int nArg = p->dot.nArg; /* Number of arguments */ + char **azArg = p->dot.azArg; /* Text of the arguments */ + int i; /* Loop counter */ + int k; /* Result of pickStr() */ + char *zTest; /* Textcase result */ + int bKeep = 0; /* --keep option */ + char *zCheck = 0; /* PATTERN argument */ + char *zPattern = 0; /* Actual test pattern */ + int eCheck = 0; /* 1: --glob, 2: --notglob, 3: --exact */ + int isOk; /* True if results are OK */ + sqlite3_int64 iStart = p->lineno; /* Line number of .check statement */ + + if( p->zTestcase[0]==0 ){ + dotCmdError(p, 0, "no .testcase is active", 0); + return 1; + } + for(i=1; i<nArg; i++){ + char *z = azArg[i]; + if( z[0]=='-' && z[1]=='-' && z[2]!=0 ) z++; + if( cli_strcmp(z,"-keep")==0 ){ + bKeep = 1; + }else if( cli_strcmp(z,"-show")==0 ){ + if( cli_output_capture ){ + sqlite3_fprintf(stdout, "%s", sqlite3_str_value(cli_output_capture)); + } + bKeep = 1; + }else if( z[0]=='-' + && (k = pickStr(&z[1],0,"glob","notglob","exact",""))>=0 + ){ + if( eCheck && eCheck!=k+1 ){ + dotCmdError(p, i, "incompatible with prior options",0); + return 1; + } + eCheck = k+1; + }else if( zCheck ){ + dotCmdError(p, i, "unknown option", 0); + return 1; + }else{ + zCheck = azArg[i]; + } + } + if( zCheck==0 ){ + dotCmdError(p, 0, "no PATTERN specified", 0); + return 1; + } + if( cli_output_capture && sqlite3_str_length(cli_output_capture) ){ + zTest = sqlite3_str_value(cli_output_capture); + shell_check_oom(zTest); + }else{ + zTest = ""; + } + p->nTestRun++; + if( zCheck[0]=='<' && zCheck[1]=='<' && zCheck[2]!=0 ){ + int nCheck = strlen30(zCheck); + sqlite3_str *pPattern = sqlite3_str_new(p->db); + char zLine[2000]; + while( sqlite3_fgets(zLine,sizeof(zLine),p->in) ){ + if( strchr(zLine,'\n') ) p->lineno++; + if( cli_strncmp(&zCheck[2],zLine,nCheck-2)==0 ) break; + sqlite3_str_appendall(pPattern, zLine); + } + zPattern = sqlite3_str_finish(pPattern); + if( zPattern==0 ){ + zPattern = sqlite3_mprintf(""); + } + }else{ + zPattern = zCheck; + } + shell_check_oom(zPattern); + switch( eCheck ){ + case 1: { + char *zGlob = sqlite3_mprintf("*%s*", zPattern); + isOk = testcase_glob(zGlob, zTest)!=0; + sqlite3_free(zGlob); + break; + } + case 2: { + char *zGlob = sqlite3_mprintf("*%s*", zPattern); + isOk = testcase_glob(zGlob, zTest)==0; + sqlite3_free(zGlob); + break; + } + case 3: { + isOk = cli_strcmp(zTest,zPattern)==0; + break; + } + default: { + /* Skip leading and trailing \n and \r on both pattern and test output */ + const char *z1 = zPattern; + const char *z2 = zTest; + size_t n1, n2; + while( z1[0]=='\n' || z1[0]=='\r' ) z1++; + n1 = strlen(z1); + while( n1>0 && (z1[n1-1]=='\n' || z1[n1-1]=='\r') ) n1--; + while( z2[0]=='\n' || z2[0]=='\r' ) z2++; + n2 = strlen(z2); + while( n2>0 && (z2[n2-1]=='\n' || z2[n2-1]=='\r') ) n2--; + isOk = n1==n2 && memcmp(z1,z2,n1)==0; + break; + } + } + if( !isOk ){ + sqlite3_fprintf(stderr, + "%s:%lld: .check failed for testcase %s\n", + p->zInFile, iStart, p->zTestcase); + p->nTestErr++; + sqlite3_fprintf(stderr, "Expected: [%s]\n", zPattern); + sqlite3_fprintf(stderr, "Got: [%s]\n", zTest); + } + if( zPattern!=zCheck ){ + sqlite3_free(zPattern); + } + if( !bKeep ){ + output_reset(p); + p->zTestcase[0] = 0; } + return 0; } /* -** Check if the sqlite_schema table contains one or more virtual tables. If -** parameter zLike is not NULL, then it is an SQL expression that the -** sqlite_schema row must also match. If one or more such rows are found, -** print the following warning to the output: +** DOT-COMMAND: .testcase +** USAGE: .testcase [OPTIONS] NAME ** -** WARNING: Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled +** Start a new test case identified by NAME. All output +** through the next ".check" command is captured for comparison. See the +** ".check" commandn for additional informatioon. +** +** Options: +** --error-prefix TEXT Change error message prefix text to TEXT */ -static int outputDumpWarning(ShellState *p, const char *zLike){ - int rc = SQLITE_OK; - sqlite3_stmt *pStmt = 0; - shellPreparePrintf(p->db, &rc, &pStmt, - "SELECT 1 FROM sqlite_schema o WHERE " - "sql LIKE 'CREATE VIRTUAL TABLE%%' AND %s", zLike ? zLike : "true" - ); - if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ - sqlite3_fputs("/* WARNING: " - "Script requires that SQLITE_DBCONFIG_DEFENSIVE be disabled */\n", - p->out - ); +static int dotCmdTestcase(ShellState *p){ + int nArg = p->dot.nArg; /* Number of arguments */ + char **azArg = p->dot.azArg; /* Text of the arguments */ + int i; /* Loop counter */ + const char *zName = 0; /* Testcase name */ + + for(i=1; i<nArg; i++){ + char *z = azArg[i]; + if( z[0]=='-' && z[1]=='-' && z[2]!=0 ) z++; + if( optionMatch(z,"error-prefix") ){ + if( i+1>=nArg ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + free(p->zErrPrefix); + i++; + p->zErrPrefix = azArg[i][0]==0 ? 0 : strdup(azArg[i]); + }else if( zName ){ + dotCmdError(p, i, "unknown option", 0); + return 1; + }else{ + zName = azArg[i]; + } } - shellFinalize(&rc, pStmt); - return rc; + output_reset(p); + if( cli_output_capture ){ + sqlite3_str_free(cli_output_capture); + } + cli_output_capture = sqlite3_str_new(0); + if( zName ){ + sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "%s", zName); + }else{ + sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "%s:%lld", + p->zInFile, p->lineno); + } + return 0; } /* -** Fault-Simulator state and logic. +** Enlarge the space allocated in p->dot so that it can hold more +** than nArg parsed command-line arguments. */ -static struct { - int iId; /* ID that triggers a simulated fault. -1 means "any" */ - int iErr; /* The error code to return on a fault */ - int iCnt; /* Trigger the fault only if iCnt is already zero */ - int iInterval; /* Reset iCnt to this value after each fault */ - int eVerbose; /* When to print output */ - int nHit; /* Number of hits seen so far */ - int nRepeat; /* Turn off after this many hits. 0 for never */ - int nSkip; /* Skip this many before first fault */ -} faultsim_state = {-1, 0, 0, 0, 0, 0, 0, 0}; +static void parseDotRealloc(ShellState *p, int nArg){ + p->dot.nAlloc = nArg+22; + p->dot.azArg = realloc(p->dot.azArg,p->dot.nAlloc*sizeof(char*)); + shell_check_oom(p->dot.azArg); + p->dot.aiOfst = realloc(p->dot.aiOfst,p->dot.nAlloc*sizeof(int)); + shell_check_oom(p->dot.aiOfst); + p->dot.abQuot = realloc(p->dot.abQuot,p->dot.nAlloc); + shell_check_oom(p->dot.abQuot); +} + /* -** This is the fault-sim callback +** Parse input line zLine up into individual arguments. Retain the +** parse in the p->dot substructure. */ -static int faultsim_callback(int iArg){ - if( faultsim_state.iId>0 && faultsim_state.iId!=iArg ){ - return SQLITE_OK; - } - if( faultsim_state.iCnt ){ - if( faultsim_state.iCnt>0 ) faultsim_state.iCnt--; - if( faultsim_state.eVerbose>=2 ){ - sqlite3_fprintf(stdout, - "FAULT-SIM id=%d no-fault (cnt=%d)\n", iArg, faultsim_state.iCnt); +static void parseDotCmdArgs(const char *zLine, ShellState *p){ + char *z; + int h = 1; + int nArg = 0; + size_t szLine; + + p->dot.zOrig = zLine; + free(p->dot.zCopy); + z = p->dot.zCopy = strdup(zLine); + shell_check_oom(z); + szLine = strlen(z); + while( szLine>0 && IsSpace(z[szLine-1]) ) szLine--; + if( szLine>0 && z[szLine-1]==';' ){ + szLine--; + while( szLine>0 && IsSpace(z[szLine-1]) ) szLine--; + } + z[szLine] = 0; + parseDotRealloc(p, 2); + while( z[h] ){ + while( IsSpace(z[h]) ){ h++; } + if( z[h]==0 ) break; + if( nArg+2>p->dot.nAlloc ){ + parseDotRealloc(p, nArg); + } + if( z[h]=='\'' || z[h]=='"' ){ + int delim = z[h++]; + p->dot.abQuot[nArg] = 1; + p->dot.azArg[nArg] = &z[h]; + p->dot.aiOfst[nArg] = h; + while( z[h] && z[h]!=delim ){ + if( z[h]=='\\' && delim=='"' && z[h+1]!=0 ) h++; + h++; + } + if( z[h]==delim ){ + z[h++] = 0; + } + if( delim=='"' ) resolve_backslashes(p->dot.azArg[nArg]); + }else{ + p->dot.abQuot[nArg] = 0; + p->dot.azArg[nArg] = &z[h]; + p->dot.aiOfst[nArg] = h; + while( z[h] && !IsSpace(z[h]) ){ h++; } + if( z[h] ) z[h++] = 0; } - return SQLITE_OK; - } - if( faultsim_state.eVerbose>=1 ){ - sqlite3_fprintf(stdout, - "FAULT-SIM id=%d returns %d\n", iArg, faultsim_state.iErr); - } - faultsim_state.iCnt = faultsim_state.iInterval; - faultsim_state.nHit++; - if( faultsim_state.nRepeat>0 && faultsim_state.nRepeat<=faultsim_state.nHit ){ - faultsim_state.iCnt = -1; + nArg++; } - return faultsim_state.iErr; + p->dot.nArg = nArg; + p->dot.azArg[nArg] = 0; } /* @@ -8762,12 +8978,11 @@ static int faultsim_callback(int iArg){ ** ** Return 1 on error, 2 to exit, and 0 otherwise. */ -static int do_meta_command(char *zLine, ShellState *p){ - int h = 1; - int nArg = 0; +static int do_meta_command(const char *zLine, ShellState *p){ + int nArg; int n, c; int rc = 0; - char *azArg[52]; + char **azArg; #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) if( p->expert.pExpert ){ @@ -8775,29 +8990,11 @@ static int do_meta_command(char *zLine, ShellState *p){ } #endif - /* Parse the input line into tokens. + /* Parse the input line into tokens stored in p->dot. */ - while( zLine[h] && nArg<ArraySize(azArg)-1 ){ - while( IsSpace(zLine[h]) ){ h++; } - if( zLine[h]==0 ) break; - if( zLine[h]=='\'' || zLine[h]=='"' ){ - int delim = zLine[h++]; - azArg[nArg++] = &zLine[h]; - while( zLine[h] && zLine[h]!=delim ){ - if( zLine[h]=='\\' && delim=='"' && zLine[h+1]!=0 ) h++; - h++; - } - if( zLine[h]==delim ){ - zLine[h++] = 0; - } - if( delim=='"' ) resolve_backslashes(azArg[nArg-1]); - }else{ - azArg[nArg++] = &zLine[h]; - while( zLine[h] && !IsSpace(zLine[h]) ){ h++; } - if( zLine[h] ) zLine[h++] = 0; - } - } - azArg[nArg] = 0; + parseDotCmdArgs(zLine, p); + nArg = p->dot.nArg; + azArg = p->dot.azArg; /* Process the input line. */ @@ -8809,7 +9006,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #ifndef SQLITE_OMIT_AUTHORIZATION if( c=='a' && cli_strncmp(azArg[0], "auth", n)==0 ){ if( nArg!=2 ){ - sqlite3_fprintf(stderr, "Usage: .auth ON|OFF\n"); + cli_printf(stderr, "Usage: .auth ON|OFF\n"); rc = 1; goto meta_command_exit; } @@ -8856,7 +9053,7 @@ static int do_meta_command(char *zLine, ShellState *p){ bAsync = 1; }else { - sqlite3_fprintf(stderr,"unknown option: %s\n", azArg[j]); + dotCmdError(p, j, "unknown option", "should be -append or -async"); return 1; } }else if( zDestFile==0 ){ @@ -8865,19 +9062,19 @@ static int do_meta_command(char *zLine, ShellState *p){ zDb = zDestFile; zDestFile = azArg[j]; }else{ - sqlite3_fprintf(stderr, "Usage: .backup ?DB? ?OPTIONS? FILENAME\n"); + cli_printf(stderr, "Usage: .backup ?DB? ?OPTIONS? FILENAME\n"); return 1; } } if( zDestFile==0 ){ - sqlite3_fprintf(stderr, "missing FILENAME argument on .backup\n"); + cli_printf(stderr, "missing FILENAME argument on .backup\n"); return 1; } if( zDb==0 ) zDb = "main"; rc = sqlite3_open_v2(zDestFile, &pDest, SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, zVfs); if( rc!=SQLITE_OK ){ - sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zDestFile); + cli_printf(stderr,"Error: cannot open \"%s\"\n", zDestFile); close_db(pDest); return 1; } @@ -8938,7 +9135,7 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = chdir(azArg[1]); #endif if( rc ){ - sqlite3_fprintf(stderr,"Cannot change to directory \"%s\"\n", azArg[1]); + cli_printf(stderr,"Cannot change to directory \"%s\"\n", azArg[1]); rc = 1; } }else{ @@ -8957,31 +9154,13 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else -#ifndef SQLITE_SHELL_FIDDLE /* Cancel output redirection, if it is currently set (by .testcase) ** Then read the content of the testcase-out.txt file and compare against ** azArg[1]. If there are differences, report an error and exit. */ if( c=='c' && n>=3 && cli_strncmp(azArg[0], "check", n)==0 ){ - char *zRes = 0; - output_reset(p); - if( nArg!=2 ){ - eputz("Usage: .check GLOB-PATTERN\n"); - rc = 2; - }else if( (zRes = readFile("testcase-out.txt", 0))==0 ){ - rc = 2; - }else if( testcase_glob(azArg[1],zRes)==0 ){ - sqlite3_fprintf(stderr, - "testcase-%s FAILED\n Expected: [%s]\n Got: [%s]\n", - p->zTestcase, azArg[1], zRes); - rc = 1; - }else{ - sqlite3_fprintf(p->out, "testcase-%s ok\n", p->zTestcase); - p->nCheck++; - } - sqlite3_free(zRes); + rc = dotCmdCheck(p); }else -#endif /* !defined(SQLITE_SHELL_FIDDLE) */ #ifndef SQLITE_SHELL_FIDDLE if( c=='c' && cli_strncmp(azArg[0], "clone", n)==0 ){ @@ -9009,9 +9188,9 @@ static int do_meta_command(char *zLine, ShellState *p){ zFile = "(temporary-file)"; } if( p->pAuxDb == &p->aAuxDb[i] ){ - sqlite3_fprintf(stdout, "ACTIVE %d: %s\n", i, zFile); + cli_printf(stdout, "ACTIVE %d: %s\n", i, zFile); }else if( p->aAuxDb[i].db!=0 ){ - sqlite3_fprintf(stdout, " %d: %s\n", i, zFile); + cli_printf(stdout, " %d: %s\n", i, zFile); } } }else if( nArg==2 && IsDigit(azArg[1][0]) && azArg[1][1]==0 ){ @@ -9047,12 +9226,17 @@ static int do_meta_command(char *zLine, ShellState *p){ ){ if( nArg==2 ){ #ifdef _WIN32 - p->crlfMode = booleanValue(azArg[1]); + if( booleanValue(azArg[1]) ){ + p->mode.mFlags |= MFLG_CRLF; + }else{ + p->mode.mFlags &= ~MFLG_CRLF; + } #else - p->crlfMode = 0; + p->mode.mFlags &= ~MFLG_CRLF; #endif } - sqlite3_fprintf(stderr, "crlf is %s\n", p->crlfMode ? "ON" : "OFF"); + cli_printf(stderr, "crlf is %s\n", + (p->mode.mFlags & MFLG_CRLF)!=0 ? "ON" : "OFF"); }else if( c=='d' && n>1 && cli_strncmp(azArg[0], "databases", n)==0 ){ @@ -9082,7 +9266,7 @@ static int do_meta_command(char *zLine, ShellState *p){ int eTxn = sqlite3_txn_state(p->db, azName[i*2]); int bRdonly = sqlite3_db_readonly(p->db, azName[i*2]); const char *z = azName[i*2+1]; - sqlite3_fprintf(p->out, "%s: %s %s%s\n", + cli_printf(p->out, "%s: %s %s%s\n", azName[i*2], z && z[0] ? z : "\"\"", bRdonly ? "r/o" : "r/w", eTxn==SQLITE_TXN_NONE ? "" : eTxn==SQLITE_TXN_READ ? " read-txn" : " write-txn"); @@ -9108,6 +9292,7 @@ static int do_meta_command(char *zLine, ShellState *p){ { "enable_trigger", SQLITE_DBCONFIG_ENABLE_TRIGGER }, { "enable_view", SQLITE_DBCONFIG_ENABLE_VIEW }, { "fts3_tokenizer", SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER }, + { "fp_digits", SQLITE_DBCONFIG_FP_DIGITS }, { "legacy_alter_table", SQLITE_DBCONFIG_LEGACY_ALTER_TABLE }, { "legacy_file_format", SQLITE_DBCONFIG_LEGACY_FILE_FORMAT }, { "load_extension", SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION }, @@ -9124,16 +9309,24 @@ static int do_meta_command(char *zLine, ShellState *p){ for(ii=0; ii<ArraySize(aDbConfig); ii++){ if( nArg>1 && cli_strcmp(azArg[1], aDbConfig[ii].zName)!=0 ) continue; if( nArg>=3 ){ - sqlite3_db_config(p->db, aDbConfig[ii].op, booleanValue(azArg[2]), 0); + if( aDbConfig[ii].op==SQLITE_DBCONFIG_FP_DIGITS ){ + sqlite3_db_config(p->db, aDbConfig[ii].op, atoi(azArg[2]), 0); + }else{ + sqlite3_db_config(p->db, aDbConfig[ii].op, booleanValue(azArg[2]), 0); + } } sqlite3_db_config(p->db, aDbConfig[ii].op, -1, &v); - sqlite3_fprintf(p->out, "%19s %s\n", - aDbConfig[ii].zName, v ? "on" : "off"); + if( aDbConfig[ii].op==SQLITE_DBCONFIG_FP_DIGITS ){ + cli_printf(p->out, "%19s %d\n", aDbConfig[ii].zName, v); + }else{ + cli_printf(p->out, "%19s %s\n", + aDbConfig[ii].zName, v ? "on" : "off"); + } if( nArg>1 ) break; } if( nArg>1 && ii==ArraySize(aDbConfig) ){ - sqlite3_fprintf(stderr,"Error: unknown dbconfig \"%s\"\n", azArg[1]); - eputz("Enter \".dbconfig\" with no arguments for a list\n"); + dotCmdError(p, 1, "unknown dbconfig", + "Enter \".dbconfig\" with no arguments for a list"); } }else @@ -9152,19 +9345,19 @@ static int do_meta_command(char *zLine, ShellState *p){ char *zLike = 0; char *zSql; int i; - int savedShowHeader = p->showHeader; int savedShellFlags = p->shellFlgs; + Mode saved_mode; ShellClearFlag(p, - SHFLG_PreserveRowid|SHFLG_Newlines|SHFLG_Echo - |SHFLG_DumpDataOnly|SHFLG_DumpNoSys); + SHFLG_PreserveRowid|SHFLG_DumpDataOnly|SHFLG_DumpNoSys); for(i=1; i<nArg; i++){ if( azArg[i][0]=='-' ){ const char *z = azArg[i]+1; if( z[0]=='-' ) z++; if( cli_strcmp(z,"preserve-rowids")==0 ){ #ifdef SQLITE_OMIT_VIRTUALTABLE - eputz("The --preserve-rowids option is not compatible" - " with SQLITE_OMIT_VIRTUALTABLE\n"); + dotCmdError(p, i, "unable", + "The --preserve-rowids option is not compatible" + " with SQLITE_OMIT_VIRTUALTABLE"); rc = 1; sqlite3_free(zLike); goto meta_command_exit; @@ -9173,7 +9366,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif }else if( cli_strcmp(z,"newlines")==0 ){ - ShellSetFlag(p, SHFLG_Newlines); + /*ShellSetFlag(p, SHFLG_Newlines);*/ }else if( cli_strcmp(z,"data-only")==0 ){ ShellSetFlag(p, SHFLG_DumpDataOnly); @@ -9182,8 +9375,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ShellSetFlag(p, SHFLG_DumpNoSys); }else { - sqlite3_fprintf(stderr, - "Unknown option \"%s\" on \".dump\"\n", azArg[i]); + dotCmdError(p, i, "unknown option", 0); rc = 1; sqlite3_free(zLike); goto meta_command_exit; @@ -9213,16 +9405,17 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); + modeDup(&saved_mode, &p->mode); outputDumpWarning(p, zLike); if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){ /* When playing back a "dump", the content might appear in an order ** which causes immediate foreign key constraints to be violated. ** So disable foreign-key constraint enforcement to prevent problems. */ - sqlite3_fputs("PRAGMA foreign_keys=OFF;\n", p->out); - sqlite3_fputs("BEGIN TRANSACTION;\n", p->out); + cli_puts("PRAGMA foreign_keys=OFF;\n", p->out); + cli_puts("BEGIN TRANSACTION;\n", p->out); } p->writableSchema = 0; - p->showHeader = 0; + p->mode.spec.bTitles = QRF_No; /* Set writable_schema=ON since doing so forces SQLite to initialize ** as much of the schema as it can even if the sqlite_schema table is ** corrupt. */ @@ -9251,22 +9444,27 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_free(zLike); if( p->writableSchema ){ - sqlite3_fputs("PRAGMA writable_schema=OFF;\n", p->out); + cli_puts("PRAGMA writable_schema=OFF;\n", p->out); p->writableSchema = 0; } sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0); sqlite3_exec(p->db, "RELEASE dump;", 0, 0, 0); if( (p->shellFlgs & SHFLG_DumpDataOnly)==0 ){ - sqlite3_fputs(p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n", p->out); + cli_puts(p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n", p->out); } - p->showHeader = savedShowHeader; p->shellFlgs = savedShellFlags; + modeFree(&p->mode); + p->mode = saved_mode; rc = p->nErr>0; }else if( c=='e' && cli_strncmp(azArg[0], "echo", n)==0 ){ if( nArg==2 ){ - setOrClearFlag(p, SHFLG_Echo, azArg[1]); + if( booleanValue(azArg[1]) ){ + p->mode.mFlags |= MFLG_ECHO; + }else{ + p->mode.mFlags &= ~MFLG_ECHO; + } }else{ eputz("Usage: .echo on|off\n"); rc = 1; @@ -9280,28 +9478,24 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='e' && cli_strncmp(azArg[0], "eqp", n)==0 ){ if( nArg==2 ){ - p->autoEQPtest = 0; - if( p->autoEQPtrace ){ + if( p->mode.autoEQPtrace ){ if( p->db ) sqlite3_exec(p->db, "PRAGMA vdbe_trace=OFF;", 0, 0, 0); - p->autoEQPtrace = 0; + p->mode.autoEQPtrace = 0; } if( cli_strcmp(azArg[1],"full")==0 ){ - p->autoEQP = AUTOEQP_full; + p->mode.autoEQP = AUTOEQP_full; }else if( cli_strcmp(azArg[1],"trigger")==0 ){ - p->autoEQP = AUTOEQP_trigger; + p->mode.autoEQP = AUTOEQP_trigger; #ifdef SQLITE_DEBUG - }else if( cli_strcmp(azArg[1],"test")==0 ){ - p->autoEQP = AUTOEQP_on; - p->autoEQPtest = 1; }else if( cli_strcmp(azArg[1],"trace")==0 ){ - p->autoEQP = AUTOEQP_full; - p->autoEQPtrace = 1; + p->mode.autoEQP = AUTOEQP_full; + p->mode.autoEQPtrace = 1; open_db(p, 0); sqlite3_exec(p->db, "SELECT name FROM sqlite_schema LIMIT 1", 0, 0, 0); sqlite3_exec(p->db, "PRAGMA vdbe_trace=ON;", 0, 0, 0); #endif }else{ - p->autoEQP = (u8)booleanValue(azArg[1]); + p->mode.autoEQP = (u8)booleanValue(azArg[1]); } }else{ eputz("Usage: .eqp off|on|trace|trigger|full\n"); @@ -9311,7 +9505,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #ifndef SQLITE_SHELL_FIDDLE if( c=='e' && cli_strncmp(azArg[0], "exit", n)==0 ){ - if( nArg>1 && (rc = (int)integerValue(azArg[1]))!=0 ) exit(rc); + if( nArg>1 && (rc = (int)integerValue(azArg[1]))!=0 ) cli_exit(rc); rc = 2; }else #endif @@ -9319,31 +9513,19 @@ static int do_meta_command(char *zLine, ShellState *p){ /* The ".explain" command is automatic now. It is largely pointless. It ** retained purely for backwards compatibility */ if( c=='e' && cli_strncmp(azArg[0], "explain", n)==0 ){ - int val = 1; if( nArg>=2 ){ if( cli_strcmp(azArg[1],"auto")==0 ){ - val = 99; + p->mode.autoExplain = 1; }else{ - val = booleanValue(azArg[1]); + p->mode.autoExplain = booleanValue(azArg[1]); } } - if( val==1 && p->mode!=MODE_Explain ){ - p->normalMode = p->mode; - p->mode = MODE_Explain; - p->autoExplain = 0; - }else if( val==0 ){ - if( p->mode==MODE_Explain ) p->mode = p->normalMode; - p->autoExplain = 0; - }else if( val==99 ){ - if( p->mode==MODE_Explain ) p->mode = p->normalMode; - p->autoExplain = 1; - } }else #if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_AUTHORIZATION) if( c=='e' && cli_strncmp(azArg[0], "expert", n)==0 ){ if( p->bSafeMode ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Cannot run experimental commands such as \"%s\" in safe mode\n", azArg[0]); rc = 1; @@ -9401,9 +9583,9 @@ static int do_meta_command(char *zLine, ShellState *p){ /* --help lists all file-controls */ if( cli_strcmp(zCmd,"help")==0 ){ - sqlite3_fputs("Available file-controls:\n", p->out); + cli_puts("Available file-controls:\n", p->out); for(i=0; i<ArraySize(aCtrl); i++){ - sqlite3_fprintf(p->out, + cli_printf(p->out, " .filectrl %s %s\n", aCtrl[i].zCtrlName, aCtrl[i].zUsage); } rc = 1; @@ -9419,7 +9601,7 @@ static int do_meta_command(char *zLine, ShellState *p){ filectrl = aCtrl[i].ctrlCode; iCtrl = i; }else{ - sqlite3_fprintf(stderr,"Error: ambiguous file-control: \"%s\"\n" + cli_printf(stderr,"Error: ambiguous file-control: \"%s\"\n" "Use \".filectrl --help\" for help\n", zCmd); rc = 1; goto meta_command_exit; @@ -9427,7 +9609,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } } if( filectrl<0 ){ - sqlite3_fprintf(stderr,"Error: unknown file-control: %s\n" + cli_printf(stderr,"Error: unknown file-control: %s\n" "Use \".filectrl --help\" for help\n", zCmd); }else{ switch(filectrl){ @@ -9471,7 +9653,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nArg!=2 ) break; sqlite3_file_control(p->db, zSchema, filectrl, &z); if( z ){ - sqlite3_fprintf(p->out, "%s\n", z); + cli_printf(p->out, "%s\n", z); sqlite3_free(z); } isOk = 2; @@ -9485,31 +9667,30 @@ static int do_meta_command(char *zLine, ShellState *p){ } x = -1; sqlite3_file_control(p->db, zSchema, filectrl, &x); - sqlite3_fprintf(p->out, "%d\n", x); + cli_printf(p->out, "%d\n", x); isOk = 2; break; } } } if( isOk==0 && iCtrl>=0 ){ - sqlite3_fprintf(p->out, "Usage: .filectrl %s %s\n", + cli_printf(p->out, "Usage: .filectrl %s %s\n", zCmd, aCtrl[iCtrl].zUsage); rc = 1; }else if( isOk==1 ){ char zBuf[100]; sqlite3_snprintf(sizeof(zBuf), zBuf, "%lld", iRes); - sqlite3_fprintf(p->out, "%s\n", zBuf); + cli_printf(p->out, "%s\n", zBuf); } }else if( c=='f' && cli_strncmp(azArg[0], "fullschema", n)==0 ){ ShellState data; int doStats = 0; - memcpy(&data, p, sizeof(data)); - data.showHeader = 0; - data.cMode = data.mode = MODE_Semi; + int hasStat[5]; + int flgs = 0; + char *zSql; if( nArg==2 && optionMatch(azArg[1], "indent") ){ - data.cMode = data.mode = MODE_Pretty; nArg = 1; } if( nArg!=1 ){ @@ -9518,44 +9699,61 @@ static int do_meta_command(char *zLine, ShellState *p){ goto meta_command_exit; } open_db(p, 0); - rc = sqlite3_exec(p->db, - "SELECT sql FROM" + zSql = sqlite3_mprintf( + "SELECT shell_format_schema(sql,%d) FROM" " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" " FROM sqlite_schema UNION ALL" " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_schema) " "WHERE type!='meta' AND sql NOTNULL" - " AND name NOT LIKE 'sqlite__%' ESCAPE '_' " - "ORDER BY x", - callback, &data, 0 - ); + " AND name NOT LIKE 'sqlite__%%' ESCAPE '_' " + "ORDER BY x", flgs); + memcpy(&data, p, sizeof(data)); + data.mode.spec.bTitles = QRF_No; + data.mode.eMode = MODE_List; + data.mode.spec.eText = QRF_TEXT_Plain; + data.mode.spec.nCharLimit = 0; + data.mode.spec.zRowSep = "\n"; + rc = shell_exec(&data,zSql,0); + sqlite3_free(zSql); if( rc==SQLITE_OK ){ sqlite3_stmt *pStmt; + memset(hasStat, 0, sizeof(hasStat)); rc = sqlite3_prepare_v2(p->db, - "SELECT rowid FROM sqlite_schema" + "SELECT substr(name,12,1) FROM sqlite_schema" " WHERE name GLOB 'sqlite_stat[134]'", -1, &pStmt, 0); if( rc==SQLITE_OK ){ - doStats = sqlite3_step(pStmt)==SQLITE_ROW; - sqlite3_finalize(pStmt); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + int k = sqlite3_column_int(pStmt,0); + assert( k==1 || k==3 || k==4 ); + hasStat[k] = 1; + doStats = 1; + } } + sqlite3_finalize(pStmt); } if( doStats==0 ){ - sqlite3_fputs("/* No STAT tables available */\n", p->out); + cli_puts("/* No STAT tables available */\n", p->out); }else{ - sqlite3_fputs("ANALYZE sqlite_schema;\n", p->out); - data.cMode = data.mode = MODE_Insert; - data.zDestTable = "sqlite_stat1"; - shell_exec(&data, "SELECT * FROM sqlite_stat1", 0); - data.zDestTable = "sqlite_stat4"; - shell_exec(&data, "SELECT * FROM sqlite_stat4", 0); - sqlite3_fputs("ANALYZE sqlite_schema;\n", p->out); + cli_puts("ANALYZE sqlite_schema;\n", p->out); + data.mode.eMode = MODE_Insert; + if( hasStat[1] ){ + data.mode.spec.zTableName = "sqlite_stat1"; + shell_exec(&data, "SELECT * FROM sqlite_stat1", 0); + } + if( hasStat[4] ){ + data.mode.spec.zTableName = "sqlite_stat4"; + shell_exec(&data, "SELECT * FROM sqlite_stat4", 0); + } + cli_puts("ANALYZE sqlite_schema;\n", p->out); } }else if( c=='h' && cli_strncmp(azArg[0], "headers", n)==0 ){ if( nArg==2 ){ - p->showHeader = booleanValue(azArg[1]); - p->shellFlgs |= SHFLG_HeaderSet; + p->mode.spec.bTitles = booleanValue(azArg[1]) ? QRF_Yes : QRF_No; + p->mode.mFlags |= MFLG_HDR; + p->mode.spec.eTitle = aModeInfo[p->mode.eMode].eHdr; }else{ eputz("Usage: .headers on|off\n"); rc = 1; @@ -9566,7 +9764,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nArg>=2 ){ n = showHelp(p->out, azArg[1]); if( n==0 ){ - sqlite3_fprintf(p->out, "Nothing matches '%s'\n", azArg[1]); + cli_printf(p->out, "Nothing matches '%s'\n", azArg[1]); } }else{ showHelp(p->out, 0); @@ -9575,327 +9773,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #ifndef SQLITE_SHELL_FIDDLE if( c=='i' && cli_strncmp(azArg[0], "import", n)==0 ){ - char *zTable = 0; /* Insert data into this table */ - char *zSchema = 0; /* Schema of zTable */ - char *zFile = 0; /* Name of file to extra content from */ - sqlite3_stmt *pStmt = NULL; /* A statement */ - int nCol; /* Number of columns in the table */ - i64 nByte; /* Number of bytes in an SQL string */ - int i, j; /* Loop counters */ - int needCommit; /* True to COMMIT or ROLLBACK at end */ - int nSep; /* Number of bytes in p->colSeparator[] */ - char *zSql = 0; /* An SQL statement */ - ImportCtx sCtx; /* Reader context */ - char *(SQLITE_CDECL *xRead)(ImportCtx*); /* Func to read one value */ - int eVerbose = 0; /* Larger for more console output */ - i64 nSkip = 0; /* Initial lines to skip */ - int useOutputMode = 1; /* Use output mode to determine separators */ - char *zCreate = 0; /* CREATE TABLE statement text */ - - failIfSafeMode(p, "cannot run .import in safe mode"); - memset(&sCtx, 0, sizeof(sCtx)); - if( p->mode==MODE_Ascii ){ - xRead = ascii_read_one_field; - }else{ - xRead = csv_read_one_field; - } - rc = 1; - for(i=1; i<nArg; i++){ - char *z = azArg[i]; - if( z[0]=='-' && z[1]=='-' ) z++; - if( z[0]!='-' ){ - if( zFile==0 ){ - zFile = z; - }else if( zTable==0 ){ - zTable = z; - }else{ - sqlite3_fprintf(p->out, "ERROR: extra argument: \"%s\". Usage:\n",z); - showHelp(p->out, "import"); - goto meta_command_exit; - } - }else if( cli_strcmp(z,"-v")==0 ){ - eVerbose++; - }else if( cli_strcmp(z,"-schema")==0 && i<nArg-1 ){ - zSchema = azArg[++i]; - }else if( cli_strcmp(z,"-skip")==0 && i<nArg-1 ){ - nSkip = integerValue(azArg[++i]); - }else if( cli_strcmp(z,"-ascii")==0 ){ - sCtx.cColSep = SEP_Unit[0]; - sCtx.cRowSep = SEP_Record[0]; - xRead = ascii_read_one_field; - useOutputMode = 0; - }else if( cli_strcmp(z,"-csv")==0 ){ - sCtx.cColSep = ','; - sCtx.cRowSep = '\n'; - xRead = csv_read_one_field; - useOutputMode = 0; - }else{ - sqlite3_fprintf(p->out, "ERROR: unknown option: \"%s\". Usage:\n", z); - showHelp(p->out, "import"); - goto meta_command_exit; - } - } - if( zTable==0 ){ - sqlite3_fprintf(p->out, "ERROR: missing %s argument. Usage:\n", - zFile==0 ? "FILE" : "TABLE"); - showHelp(p->out, "import"); - goto meta_command_exit; - } - seenInterrupt = 0; - open_db(p, 0); - if( useOutputMode ){ - /* If neither the --csv or --ascii options are specified, then set - ** the column and row separator characters from the output mode. */ - nSep = strlen30(p->colSeparator); - if( nSep==0 ){ - eputz("Error: non-null column separator required for import\n"); - goto meta_command_exit; - } - if( nSep>1 ){ - eputz("Error: multi-character column separators not allowed" - " for import\n"); - goto meta_command_exit; - } - nSep = strlen30(p->rowSeparator); - if( nSep==0 ){ - eputz("Error: non-null row separator required for import\n"); - goto meta_command_exit; - } - if( nSep==2 && p->mode==MODE_Csv - && cli_strcmp(p->rowSeparator,SEP_CrLf)==0 - ){ - /* When importing CSV (only), if the row separator is set to the - ** default output row separator, change it to the default input - ** row separator. This avoids having to maintain different input - ** and output row separators. */ - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - nSep = strlen30(p->rowSeparator); - } - if( nSep>1 ){ - eputz("Error: multi-character row separators not allowed" - " for import\n"); - goto meta_command_exit; - } - sCtx.cColSep = (u8)p->colSeparator[0]; - sCtx.cRowSep = (u8)p->rowSeparator[0]; - } - sCtx.zFile = zFile; - sCtx.nLine = 1; - if( sCtx.zFile[0]=='|' ){ -#ifdef SQLITE_OMIT_POPEN - eputz("Error: pipes are not supported in this OS\n"); - goto meta_command_exit; -#else - sCtx.in = sqlite3_popen(sCtx.zFile+1, "r"); - sCtx.zFile = "<pipe>"; - sCtx.xCloser = pclose; -#endif - }else{ - sCtx.in = sqlite3_fopen(sCtx.zFile, "rb"); - sCtx.xCloser = fclose; - } - if( sCtx.in==0 ){ - sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zFile); - goto meta_command_exit; - } - if( eVerbose>=2 || (eVerbose>=1 && useOutputMode) ){ - char zSep[2]; - zSep[1] = 0; - zSep[0] = sCtx.cColSep; - sqlite3_fputs("Column separator ", p->out); - output_c_string(p->out, zSep); - sqlite3_fputs(", row separator ", p->out); - zSep[0] = sCtx.cRowSep; - output_c_string(p->out, zSep); - sqlite3_fputs("\n", p->out); - } - sCtx.z = sqlite3_malloc64(120); - if( sCtx.z==0 ){ - import_cleanup(&sCtx); - shell_out_of_memory(); - } - /* Below, resources must be freed before exit. */ - while( nSkip>0 ){ - nSkip--; - while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){} - } - import_append_char(&sCtx, 0); /* To ensure sCtx.z is allocated */ - if( sqlite3_table_column_metadata(p->db, zSchema, zTable,0,0,0,0,0,0) - && 0==db_int(p->db, "SELECT count(*) FROM \"%w\".sqlite_schema" - " WHERE name=%Q AND type='view'", - zSchema ? zSchema : "main", zTable) - ){ - /* Table does not exist. Create it. */ - sqlite3 *dbCols = 0; - char *zRenames = 0; - char *zColDefs; - zCreate = sqlite3_mprintf("CREATE TABLE \"%w\".\"%w\"", - zSchema ? zSchema : "main", zTable); - while( xRead(&sCtx) ){ - zAutoColumn(sCtx.z, &dbCols, 0); - if( sCtx.cTerm!=sCtx.cColSep ) break; - } - zColDefs = zAutoColumn(0, &dbCols, &zRenames); - if( zRenames!=0 ){ - sqlite3_fprintf((stdin_is_interactive && p->in==stdin)? p->out : stderr, - "Columns renamed during .import %s due to duplicates:\n" - "%s\n", sCtx.zFile, zRenames); - sqlite3_free(zRenames); - } - assert(dbCols==0); - if( zColDefs==0 ){ - sqlite3_fprintf(stderr,"%s: empty file\n", sCtx.zFile); - import_cleanup(&sCtx); - rc = 1; - sqlite3_free(zCreate); - goto meta_command_exit; - } - zCreate = sqlite3_mprintf("%z%z\n", zCreate, zColDefs); - if( zCreate==0 ){ - import_cleanup(&sCtx); - shell_out_of_memory(); - } - if( eVerbose>=1 ){ - sqlite3_fprintf(p->out, "%s\n", zCreate); - } - rc = sqlite3_exec(p->db, zCreate, 0, 0, 0); - if( rc ){ - sqlite3_fprintf(stderr, - "%s failed:\n%s\n", zCreate, sqlite3_errmsg(p->db)); - } - sqlite3_free(zCreate); - zCreate = 0; - if( rc ){ - import_cleanup(&sCtx); - rc = 1; - goto meta_command_exit; - } - } - zSql = sqlite3_mprintf("SELECT count(*) FROM pragma_table_info(%Q,%Q);", - zTable, zSchema); - if( zSql==0 ){ - import_cleanup(&sCtx); - shell_out_of_memory(); - } - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - zSql = 0; - if( rc ){ - if (pStmt) sqlite3_finalize(pStmt); - shellDatabaseError(p->db); - import_cleanup(&sCtx); - rc = 1; - goto meta_command_exit; - } - if( sqlite3_step(pStmt)==SQLITE_ROW ){ - nCol = sqlite3_column_int(pStmt, 0); - }else{ - nCol = 0; - } - sqlite3_finalize(pStmt); - pStmt = 0; - if( nCol==0 ) return 0; /* no columns, no error */ - - nByte = 64 /* space for "INSERT INTO", "VALUES(", ")\0" */ - + (zSchema ? strlen(zSchema)*2 + 2: 0) /* Quoted schema name */ - + strlen(zTable)*2 + 2 /* Quoted table name */ - + nCol*2; /* Space for ",?" for each column */ - zSql = sqlite3_malloc64( nByte ); - if( zSql==0 ){ - import_cleanup(&sCtx); - shell_out_of_memory(); - } - if( zSchema ){ - sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\".\"%w\" VALUES(?", - zSchema, zTable); - }else{ - sqlite3_snprintf(nByte, zSql, "INSERT INTO \"%w\" VALUES(?", zTable); - } - j = strlen30(zSql); - for(i=1; i<nCol; i++){ - zSql[j++] = ','; - zSql[j++] = '?'; - } - zSql[j++] = ')'; - zSql[j] = 0; - assert( j<nByte ); - if( eVerbose>=2 ){ - sqlite3_fprintf(p->out, "Insert using: %s\n", zSql); - } - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - zSql = 0; - if( rc ){ - shellDatabaseError(p->db); - if (pStmt) sqlite3_finalize(pStmt); - import_cleanup(&sCtx); - rc = 1; - goto meta_command_exit; - } - needCommit = sqlite3_get_autocommit(p->db); - if( needCommit ) sqlite3_exec(p->db, "BEGIN", 0, 0, 0); - do{ - int startLine = sCtx.nLine; - for(i=0; i<nCol; i++){ - char *z = xRead(&sCtx); - /* - ** Did we reach end-of-file before finding any columns? - ** If so, stop instead of NULL filling the remaining columns. - */ - if( z==0 && i==0 ) break; - /* - ** Did we reach end-of-file OR end-of-line before finding any - ** columns in ASCII mode? If so, stop instead of NULL filling - ** the remaining columns. - */ - if( p->mode==MODE_Ascii && (z==0 || z[0]==0) && i==0 ) break; - /* - ** For CSV mode, per RFC 4180, accept EOF in lieu of final - ** record terminator but only for last field of multi-field row. - ** (If there are too few fields, it's not valid CSV anyway.) - */ - if( z==0 && (xRead==csv_read_one_field) && i==nCol-1 && i>0 ){ - z = ""; - } - sqlite3_bind_text(pStmt, i+1, z, -1, SQLITE_TRANSIENT); - if( i<nCol-1 && sCtx.cTerm!=sCtx.cColSep ){ - sqlite3_fprintf(stderr,"%s:%d: expected %d columns but found %d" - " - filling the rest with NULL\n", - sCtx.zFile, startLine, nCol, i+1); - i += 2; - while( i<=nCol ){ sqlite3_bind_null(pStmt, i); i++; } - } - } - if( sCtx.cTerm==sCtx.cColSep ){ - do{ - xRead(&sCtx); - i++; - }while( sCtx.cTerm==sCtx.cColSep ); - sqlite3_fprintf(stderr, - "%s:%d: expected %d columns but found %d - extras ignored\n", - sCtx.zFile, startLine, nCol, i); - } - if( i>=nCol ){ - sqlite3_step(pStmt); - rc = sqlite3_reset(pStmt); - if( rc!=SQLITE_OK ){ - sqlite3_fprintf(stderr,"%s:%d: INSERT failed: %s\n", - sCtx.zFile, startLine, sqlite3_errmsg(p->db)); - sCtx.nErr++; - }else{ - sCtx.nRow++; - } - } - }while( sCtx.cTerm!=EOF ); - - import_cleanup(&sCtx); - sqlite3_finalize(pStmt); - if( needCommit ) sqlite3_exec(p->db, "COMMIT", 0, 0, 0); - if( eVerbose>0 ){ - sqlite3_fprintf(p->out, - "Added %d rows with %d errors using %d lines of input\n", - sCtx.nRow, sCtx.nErr, sCtx.nLine-1); - } + rc = dotCmdImport(p); }else #endif /* !defined(SQLITE_SHELL_FIDDLE) */ @@ -9928,10 +9806,10 @@ static int do_meta_command(char *zLine, ShellState *p){ } zSql = sqlite3_mprintf( "SELECT rootpage, 0 FROM sqlite_schema" - " WHERE name='%q' AND type='index'" + " WHERE type='index' AND lower(name)=lower('%q')" "UNION ALL " "SELECT rootpage, 1 FROM sqlite_schema" - " WHERE name='%q' AND type='table'" + " WHERE type='table' AND lower(name)=lower('%q')" " AND sql LIKE '%%without%%rowid%%'", azArg[1], azArg[1] ); @@ -9969,7 +9847,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_finalize(pStmt); if( i==0 || tnum==0 ){ - sqlite3_fprintf(stderr,"no such index: \"%s\"\n", azArg[1]); + cli_printf(stderr,"no such index: \"%s\"\n", azArg[1]); rc = 1; sqlite3_free(zCollist); goto meta_command_exit; @@ -9984,18 +9862,104 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = sqlite3_exec(p->db, zSql, 0, 0, 0); sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->db, "main", 0, 0); if( rc ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Error in [%s]: %s\n", zSql, sqlite3_errmsg(p->db)); }else{ - sqlite3_fprintf(stdout, "%s;\n", zSql); + cli_printf(stdout, "%s;\n", zSql); + } + }else{ + cli_printf(stderr,"SQLITE_TESTCTRL_IMPOSTER returns %d\n", rc); + rc = 1; + } + sqlite3_free(zSql); + }else +#endif /* !defined(SQLITE_OMIT_TEST_CONTROL) */ + + if( c=='i' && (cli_strncmp(azArg[0], "indices", n)==0 + || cli_strncmp(azArg[0], "indexes", n)==0) + ){ + sqlite3_str *pSql; + int i; + int allFlag = 0; + int sysFlag = 0; + int exprFlag = 0; + int debugFlag = 0; /* Undocument --debug flag */ + const char *zPattern = 0; + const char *zSep = "WHERE"; + + for(i=1; i<nArg; i++){ + if( azArg[i][0]=='-' ){ + const char *z = azArg[i]+1; + if( z[0]=='-' ) z++; + if( cli_strcmp(z,"all")==0 || cli_strcmp(z,"a")==0 ){ + allFlag = 1; + }else + if( cli_strcmp(z,"sys")==0 ){ + sysFlag = 1; + allFlag = 0; + }else + if( cli_strcmp(z,"expr")==0 ){ + exprFlag = 1; + allFlag = 0; + }else + if( cli_strcmp(z,"debug")==0 ){ + debugFlag = 1; + }else + { + dotCmdError(p, i, "unknown option", 0); + rc = 1; + goto meta_command_exit; + } + }else if( zPattern==0 ){ + zPattern = azArg[i]; + }else{ + dotCmdError(p, i, "unknown argument", 0); + rc = 1; + goto meta_command_exit; } + } + + open_db(p, 0); + pSql = sqlite3_str_new(p->db); + sqlite3_str_appendf(pSql, + "SELECT if(t.schema='main',i.name,t.schema||'.'||i.name)\n" + "FROM pragma_table_list t, pragma_index_list(t.name,t.schema) i\n" + ); + if( exprFlag ){ + allFlag = 0; + sqlite3_str_appendf(pSql, + "%s (EXISTS(SELECT 1 FROM pragma_index_xinfo(i.name) WHERE cid=-2)\n" + " OR\n" + " EXISTS(SELECT cid FROM pragma_table_xinfo(t.name) WHERE hidden=2" + " INTERSECT " + " SELECT cid FROM pragma_index_info(i.name)))\n", zSep); + zSep = "AND"; + } + if( sysFlag ){ + sqlite3_str_appendf(pSql, + "%s i.name LIKE 'sqlite__autoindex__%%' ESCAPE '_'\n", zSep); + zSep = "AND"; + }else if( !allFlag ){ + sqlite3_str_appendf(pSql, + "%s i.name NOT LIKE 'sqlite__%%' ESCAPE '_'\n", zSep); + zSep = "AND"; + } + if( zPattern ){ + sqlite3_str_appendf(pSql, "%s i.name LIKE '%%%q%%'\n", zSep, zPattern); + } + sqlite3_str_appendf(pSql, "ORDER BY 1"); + + /* Run the SQL statement in "split" mode. */ + if( debugFlag ){ + cli_printf(stdout,"%s;\n", sqlite3_str_value(pSql)); }else{ - sqlite3_fprintf(stderr,"SQLITE_TESTCTRL_IMPOSTER returns %d\n", rc); - rc = 1; + modePush(p); + modeChange(p, MODE_Split); + shell_exec(p, sqlite3_str_value(pSql), 0); + modePop(p); } - sqlite3_free(zSql); + sqlite3_str_free(pSql); }else -#endif /* !defined(SQLITE_OMIT_TEST_CONTROL) */ if( c=='i' && cli_strncmp(azArg[0], "intck", n)==0 ){ i64 iArg = 0; @@ -10004,7 +9968,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( iArg==0 ) iArg = -1; } if( (nArg!=1 && nArg!=2) || iArg<0 ){ - sqlite3_fprintf(stderr,"%s","Usage: .intck STEPS_PER_UNLOCK\n"); + cli_printf(stderr,"%s","Usage: .intck STEPS_PER_UNLOCK\n"); rc = 1; goto meta_command_exit; } @@ -10025,7 +9989,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else{ iotrace = sqlite3_fopen(azArg[1], "w"); if( iotrace==0 ){ - sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); + cli_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); sqlite3IoTrace = 0; rc = 1; }else{ @@ -10044,6 +10008,7 @@ static int do_meta_command(char *zLine, ShellState *p){ { "sql_length", SQLITE_LIMIT_SQL_LENGTH }, { "column", SQLITE_LIMIT_COLUMN }, { "expr_depth", SQLITE_LIMIT_EXPR_DEPTH }, + { "parser_depth", SQLITE_LIMIT_PARSER_DEPTH }, { "compound_select", SQLITE_LIMIT_COMPOUND_SELECT }, { "vdbe_op", SQLITE_LIMIT_VDBE_OP }, { "function_arg", SQLITE_LIMIT_FUNCTION_ARG }, @@ -10057,7 +10022,7 @@ static int do_meta_command(char *zLine, ShellState *p){ open_db(p, 0); if( nArg==1 ){ for(i=0; i<ArraySize(aLimit); i++){ - sqlite3_fprintf(stdout, "%20s %d\n", aLimit[i].zLimitName, + cli_printf(stdout, "%20s %d\n", aLimit[i].zLimitName, sqlite3_limit(p->db, aLimit[i].limitCode, -1)); } }else if( nArg>3 ){ @@ -10072,14 +10037,14 @@ static int do_meta_command(char *zLine, ShellState *p){ if( iLimit<0 ){ iLimit = i; }else{ - sqlite3_fprintf(stderr,"ambiguous limit: \"%s\"\n", azArg[1]); + cli_printf(stderr,"ambiguous limit: \"%s\"\n", azArg[1]); rc = 1; goto meta_command_exit; } } } if( iLimit<0 ){ - sqlite3_fprintf(stderr,"unknown limit: \"%s\"\n" + cli_printf(stderr,"unknown limit: \"%s\"\n" "enter \".limits\" with no arguments for a list.\n", azArg[1]); rc = 1; @@ -10088,9 +10053,10 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nArg==3 ){ sqlite3_limit(p->db, aLimit[iLimit].limitCode, (int)integerValue(azArg[2])); + }else{ + cli_printf(stdout, "%20s %d\n", aLimit[iLimit].zLimitName, + sqlite3_limit(p->db, aLimit[iLimit].limitCode, -1)); } - sqlite3_fprintf(stdout, "%20s %d\n", aLimit[iLimit].zLimitName, - sqlite3_limit(p->db, aLimit[iLimit].limitCode, -1)); } }else @@ -10138,174 +10104,12 @@ static int do_meta_command(char *zLine, ShellState *p){ } output_file_close(p->pLog); if( cli_strcmp(zFile,"on")==0 ) zFile = "stdout"; - p->pLog = output_file_open(zFile); + p->pLog = output_file_open(p, zFile); } }else if( c=='m' && cli_strncmp(azArg[0], "mode", n)==0 ){ - const char *zMode = 0; - const char *zTabname = 0; - int i, n2; - int chng = 0; /* 0x01: change to cmopts. 0x02: Any other change */ - ColModeOpts cmOpts = ColModeOpts_default; - for(i=1; i<nArg; i++){ - const char *z = azArg[i]; - if( optionMatch(z,"wrap") && i+1<nArg ){ - cmOpts.iWrap = integerValue(azArg[++i]); - chng |= 1; - }else if( optionMatch(z,"ww") ){ - cmOpts.bWordWrap = 1; - chng |= 1; - }else if( optionMatch(z,"wordwrap") && i+1<nArg ){ - cmOpts.bWordWrap = (u8)booleanValue(azArg[++i]); - chng |= 1; - }else if( optionMatch(z,"quote") ){ - cmOpts.bQuote = 1; - chng |= 1; - }else if( optionMatch(z,"noquote") ){ - cmOpts.bQuote = 0; - chng |= 1; - }else if( optionMatch(z,"escape") && i+1<nArg ){ - /* See similar code at tag-20250224-1 */ - const char *zEsc = azArg[++i]; - int k; - for(k=0; k<ArraySize(shell_EscModeNames); k++){ - if( sqlite3_stricmp(zEsc,shell_EscModeNames[k])==0 ){ - p->eEscMode = k; - chng |= 2; - break; - } - } - if( k>=ArraySize(shell_EscModeNames) ){ - sqlite3_fprintf(stderr, "unknown control character escape mode \"%s\"" - " - choices:", zEsc); - for(k=0; k<ArraySize(shell_EscModeNames); k++){ - sqlite3_fprintf(stderr, " %s", shell_EscModeNames[k]); - } - sqlite3_fprintf(stderr, "\n"); - rc = 1; - goto meta_command_exit; - } - }else if( zMode==0 ){ - zMode = z; - /* Apply defaults for qbox pseudo-mode. If that - * overwrites already-set values, user was informed of this. - */ - chng |= 1; - if( cli_strcmp(z, "qbox")==0 ){ - ColModeOpts cmo = ColModeOpts_default_qbox; - zMode = "box"; - cmOpts = cmo; - } - }else if( zTabname==0 ){ - zTabname = z; - }else if( z[0]=='-' ){ - sqlite3_fprintf(stderr,"unknown option: %s\n", z); - eputz("options:\n" - " --escape MODE\n" - " --noquote\n" - " --quote\n" - " --wordwrap on/off\n" - " --wrap N\n" - " --ww\n"); - rc = 1; - goto meta_command_exit; - }else{ - sqlite3_fprintf(stderr,"extra argument: \"%s\"\n", z); - rc = 1; - goto meta_command_exit; - } - } - if( !chng ){ - if( p->mode==MODE_Column - || (p->mode>=MODE_Markdown && p->mode<=MODE_Box) - ){ - sqlite3_fprintf(p->out, - "current output mode: %s --wrap %d --wordwrap %s " - "--%squote --escape %s\n", - modeDescr[p->mode], p->cmOpts.iWrap, - p->cmOpts.bWordWrap ? "on" : "off", - p->cmOpts.bQuote ? "" : "no", - shell_EscModeNames[p->eEscMode] - ); - }else{ - sqlite3_fprintf(p->out, - "current output mode: %s --escape %s\n", - modeDescr[p->mode], - shell_EscModeNames[p->eEscMode] - ); - } - } - if( zMode==0 ){ - zMode = modeDescr[p->mode]; - if( (chng&1)==0 ) cmOpts = p->cmOpts; - } - n2 = strlen30(zMode); - if( cli_strncmp(zMode,"lines",n2)==0 ){ - p->mode = MODE_Line; - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( cli_strncmp(zMode,"columns",n2)==0 ){ - p->mode = MODE_Column; - if( (p->shellFlgs & SHFLG_HeaderSet)==0 ){ - p->showHeader = 1; - } - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - p->cmOpts = cmOpts; - }else if( cli_strncmp(zMode,"list",n2)==0 ){ - p->mode = MODE_List; - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Column); - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( cli_strncmp(zMode,"html",n2)==0 ){ - p->mode = MODE_Html; - }else if( cli_strncmp(zMode,"tcl",n2)==0 ){ - p->mode = MODE_Tcl; - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Space); - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( cli_strncmp(zMode,"csv",n2)==0 ){ - p->mode = MODE_Csv; - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf); - }else if( cli_strncmp(zMode,"tabs",n2)==0 ){ - p->mode = MODE_List; - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Tab); - }else if( cli_strncmp(zMode,"insert",n2)==0 ){ - p->mode = MODE_Insert; - set_table_name(p, zTabname ? zTabname : "table"); - if( p->eEscMode==SHELL_ESC_OFF ){ - ShellSetFlag(p, SHFLG_Newlines); - }else{ - ShellClearFlag(p, SHFLG_Newlines); - } - }else if( cli_strncmp(zMode,"quote",n2)==0 ){ - p->mode = MODE_Quote; - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); - }else if( cli_strncmp(zMode,"ascii",n2)==0 ){ - p->mode = MODE_Ascii; - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Unit); - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Record); - }else if( cli_strncmp(zMode,"markdown",n2)==0 ){ - p->mode = MODE_Markdown; - p->cmOpts = cmOpts; - }else if( cli_strncmp(zMode,"table",n2)==0 ){ - p->mode = MODE_Table; - p->cmOpts = cmOpts; - }else if( cli_strncmp(zMode,"box",n2)==0 ){ - p->mode = MODE_Box; - p->cmOpts = cmOpts; - }else if( cli_strncmp(zMode,"count",n2)==0 ){ - p->mode = MODE_Count; - }else if( cli_strncmp(zMode,"off",n2)==0 ){ - p->mode = MODE_Off; - }else if( cli_strncmp(zMode,"json",n2)==0 ){ - p->mode = MODE_Json; - }else{ - eputz("Error: mode should be one of: " - "ascii box column csv html insert json line list markdown " - "qbox quote table tabs tcl\n"); - rc = 1; - } - p->cMode = p->mode; + rc = dotCmdMode(p); }else #ifndef SQLITE_SHELL_FIDDLE @@ -10314,9 +10118,9 @@ static int do_meta_command(char *zLine, ShellState *p){ eputz("Usage: .nonce NONCE\n"); rc = 1; }else if( p->zNonce==0 || cli_strcmp(azArg[1],p->zNonce)!=0 ){ - sqlite3_fprintf(stderr,"line %lld: incorrect nonce: \"%s\"\n", + cli_printf(stderr,"line %lld: incorrect nonce: \"%s\"\n", p->lineno, azArg[1]); - exit(1); + cli_exit(1); }else{ p->bSafeMode = 0; return 0; /* Return immediately to bypass the safe mode reset @@ -10327,8 +10131,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='n' && cli_strncmp(azArg[0], "nullvalue", n)==0 ){ if( nArg==2 ){ - sqlite3_snprintf(sizeof(p->nullValue), p->nullValue, - "%.*s", (int)ArraySize(p->nullValue)-1, azArg[1]); + modeSetStr(&p->mode.spec.zNull, azArg[1]); }else{ eputz("Usage: .nullvalue STRING\n"); rc = 1; @@ -10343,6 +10146,8 @@ static int do_meta_command(char *zLine, ShellState *p){ int openMode = SHELL_OPEN_UNSPEC; int openFlags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; + if( p->bSafeMode ) openFlags = SQLITE_OPEN_READONLY; + /* Check for command-line arguments */ for(iName=1; iName<nArg; iName++){ const char *z = azArg[iName]; @@ -10350,10 +10155,10 @@ static int do_meta_command(char *zLine, ShellState *p){ if( optionMatch(z,"new") ){ newFlag = 1; #ifdef SQLITE_HAVE_ZLIB - }else if( optionMatch(z, "zip") ){ + }else if( optionMatch(z, "zip") && !p->bSafeMode ){ openMode = SHELL_OPEN_ZIPFILE; #endif - }else if( optionMatch(z, "append") ){ + }else if( optionMatch(z, "append") && !p->bSafeMode ){ openMode = SHELL_OPEN_APPENDVFS; }else if( optionMatch(z, "readonly") ){ openFlags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); @@ -10377,11 +10182,11 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif /* !SQLITE_SHELL_FIDDLE */ if( z[0]=='-' ){ - sqlite3_fprintf(stderr,"unknown option: %s\n", z); + cli_printf(stderr,"unknown option: %s\n", z); rc = 1; goto meta_command_exit; }else if( zFN ){ - sqlite3_fprintf(stderr,"extra argument: \"%s\"\n", z); + cli_printf(stderr,"extra argument: \"%s\"\n", z); rc = 1; goto meta_command_exit; }else{ @@ -10432,7 +10237,7 @@ static int do_meta_command(char *zLine, ShellState *p){ p->pAuxDb->zDbFilename = zNewFilename; open_db(p, OPEN_DB_KEEPALIVE); if( p->db==0 ){ - sqlite3_fprintf(stderr,"Error: cannot open '%s'\n", zNewFilename); + cli_printf(stderr,"Error: cannot open '%s'\n", zNewFilename); sqlite3_free(zNewFilename); }else{ p->pAuxDb->zFreeOnClose = zNewFilename; @@ -10452,145 +10257,7 @@ static int do_meta_command(char *zLine, ShellState *p){ || (c=='e' && n==5 && cli_strcmp(azArg[0],"excel")==0) || (c=='w' && n==3 && cli_strcmp(azArg[0],"www")==0) ){ - char *zFile = 0; - int i; - int eMode = 0; /* 0: .outout/.once, 'x'=.excel, 'w'=.www */ - int bOnce = 0; /* 0: .output, 1: .once, 2: .excel/.www */ - int bPlain = 0; /* --plain option */ - static const char *zBomUtf8 = "\357\273\277"; - const char *zBom = 0; - - failIfSafeMode(p, "cannot run .%s in safe mode", azArg[0]); - if( c=='e' ){ - eMode = 'x'; - bOnce = 2; - }else if( c=='w' ){ - eMode = 'w'; - bOnce = 2; - }else if( cli_strncmp(azArg[0],"once",n)==0 ){ - bOnce = 1; - } - for(i=1; i<nArg; i++){ - char *z = azArg[i]; - if( z[0]=='-' ){ - if( z[1]=='-' ) z++; - if( cli_strcmp(z,"-bom")==0 ){ - zBom = zBomUtf8; - }else if( cli_strcmp(z,"-plain")==0 ){ - bPlain = 1; - }else if( c=='o' && cli_strcmp(z,"-x")==0 ){ - eMode = 'x'; /* spreadsheet */ - }else if( c=='o' && cli_strcmp(z,"-e")==0 ){ - eMode = 'e'; /* text editor */ - }else if( c=='o' && cli_strcmp(z,"-w")==0 ){ - eMode = 'w'; /* Web browser */ - }else{ - sqlite3_fprintf(p->out, - "ERROR: unknown option: \"%s\". Usage:\n", azArg[i]); - showHelp(p->out, azArg[0]); - rc = 1; - sqlite3_free(zFile); - goto meta_command_exit; - } - }else if( zFile==0 && eMode==0 ){ - if( cli_strcmp(z, "off")==0 ){ -#ifdef _WIN32 - zFile = sqlite3_mprintf("nul"); -#else - zFile = sqlite3_mprintf("/dev/null"); -#endif - }else{ - zFile = sqlite3_mprintf("%s", z); - } - if( zFile && zFile[0]=='|' ){ - while( i+1<nArg ) zFile = sqlite3_mprintf("%z %s", zFile, azArg[++i]); - break; - } - }else{ - sqlite3_fprintf(p->out, - "ERROR: extra parameter: \"%s\". Usage:\n", azArg[i]); - showHelp(p->out, azArg[0]); - rc = 1; - sqlite3_free(zFile); - goto meta_command_exit; - } - } - if( zFile==0 ){ - zFile = sqlite3_mprintf("stdout"); - } - shell_check_oom(zFile); - if( bOnce ){ - p->outCount = 2; - }else{ - p->outCount = 0; - } - output_reset(p); -#ifndef SQLITE_NOHAVE_SYSTEM - if( eMode=='e' || eMode=='x' || eMode=='w' ){ - p->doXdgOpen = 1; - outputModePush(p); - if( eMode=='x' ){ - /* spreadsheet mode. Output as CSV. */ - newTempFile(p, "csv"); - ShellClearFlag(p, SHFLG_Echo); - p->mode = MODE_Csv; - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, SEP_Comma); - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_CrLf); -#ifdef _WIN32 - zBom = zBomUtf8; /* Always include the BOM on Windows, as Excel does - ** not work without it. */ -#endif - }else if( eMode=='w' ){ - /* web-browser mode. */ - newTempFile(p, "html"); - if( !bPlain ) p->mode = MODE_Www; - }else{ - /* text editor mode */ - newTempFile(p, "txt"); - } - sqlite3_free(zFile); - zFile = sqlite3_mprintf("%s", p->zTempFile); - } -#endif /* SQLITE_NOHAVE_SYSTEM */ - shell_check_oom(zFile); - if( zFile[0]=='|' ){ -#ifdef SQLITE_OMIT_POPEN - eputz("Error: pipes are not supported in this OS\n"); - rc = 1; - output_redir(p, stdout); -#else - FILE *pfPipe = sqlite3_popen(zFile + 1, "w"); - if( pfPipe==0 ){ - assert( stderr!=NULL ); - sqlite3_fprintf(stderr,"Error: cannot open pipe \"%s\"\n", zFile + 1); - rc = 1; - }else{ - output_redir(p, pfPipe); - if( zBom ) sqlite3_fputs(zBom, pfPipe); - sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); - } -#endif - }else{ - FILE *pfFile = output_file_open(zFile); - if( pfFile==0 ){ - if( cli_strcmp(zFile,"off")!=0 ){ - assert( stderr!=NULL ); - sqlite3_fprintf(stderr,"Error: cannot write to \"%s\"\n", zFile); - } - rc = 1; - } else { - output_redir(p, pfFile); - if( zBom ) sqlite3_fputs(zBom, pfFile); - if( bPlain && eMode=='w' ){ - sqlite3_fputs( - "<!DOCTYPE html>\n<BODY>\n<PLAINTEXT>\n", - pfFile - ); - } - sqlite3_snprintf(sizeof(p->outfile), p->outfile, "%s", zFile); - } - } - sqlite3_free(zFile); + rc = dotCmdOutput(p); }else #endif /* !defined(SQLITE_SHELL_FIDDLE) */ @@ -10627,7 +10294,7 @@ static int do_meta_command(char *zLine, ShellState *p){ "SELECT key, quote(value) " "FROM temp.sqlite_parameters;", -1, &pStmt, 0); while( rx==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ - sqlite3_fprintf(p->out, + cli_printf(p->out, "%-*s %s\n", len, sqlite3_column_text(pStmt,0), sqlite3_column_text(pStmt,1)); } @@ -10673,7 +10340,7 @@ static int do_meta_command(char *zLine, ShellState *p){ rx = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); if( rx!=SQLITE_OK ){ - sqlite3_fprintf(p->out, "Error: %s\n", sqlite3_errmsg(p->db)); + cli_printf(p->out, "Error: %s\n", sqlite3_errmsg(p->db)); sqlite3_finalize(pStmt); pStmt = 0; rc = 1; @@ -10703,10 +10370,10 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='p' && n>=3 && cli_strncmp(azArg[0], "print", n)==0 ){ int i; for(i=1; i<nArg; i++){ - if( i>1 ) sqlite3_fputs(" ", p->out); - sqlite3_fputs(azArg[i], p->out); + if( i>1 ) cli_puts(" ", p->out); + cli_puts(azArg[i], p->out); } - sqlite3_fputs("\n", p->out); + cli_puts("\n", p->out); }else #ifndef SQLITE_OMIT_PROGRESS_CALLBACK @@ -10733,6 +10400,19 @@ static int do_meta_command(char *zLine, ShellState *p){ p->flgProgress |= SHELL_PROGRESS_ONCE; continue; } + if( cli_strcmp(z,"timeout")==0 ){ + if( i==nArg-1 ){ + dotCmdError(p, i, "missing argument", 0); + return 1; + } + i++; + p->tmProgress = atof(azArg[i]); + if( p->tmProgress>0.0 ){ + p->flgProgress = SHELL_PROGRESS_QUIET|SHELL_PROGRESS_TMOUT; + if( nn==0 ) nn = 100; + } + continue; + } if( cli_strcmp(z,"limit")==0 ){ if( i+1>=nArg ){ eputz("Error: missing argument on --limit\n"); @@ -10743,7 +10423,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } continue; } - sqlite3_fprintf(stderr,"Error: unknown option: \"%s\"\n", azArg[i]); + cli_printf(stderr,"Error: unknown option: \"%s\"\n", azArg[i]); rc = 1; goto meta_command_exit; }else{ @@ -10787,7 +10467,7 @@ static int do_meta_command(char *zLine, ShellState *p){ #else p->in = sqlite3_popen(azArg[1]+1, "r"); if( p->in==0 ){ - sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); + cli_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); rc = 1; }else{ rc = process_input(p, "<pipe>"); @@ -10795,10 +10475,12 @@ static int do_meta_command(char *zLine, ShellState *p){ } #endif }else if( (p->in = openChrSource(azArg[1]))==0 ){ - sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); + cli_printf(stderr,"Error: cannot open \"%s\"\n", azArg[1]); rc = 1; }else{ - rc = process_input(p, azArg[1]); + char *zFilename = strdup(azArg[1]); + rc = process_input(p, zFilename); + free(zFilename); fclose(p->in); } p->in = inSaved; @@ -10828,7 +10510,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } rc = sqlite3_open(zSrcFile, &pSrc); if( rc!=SQLITE_OK ){ - sqlite3_fprintf(stderr,"Error: cannot open \"%s\"\n", zSrcFile); + cli_printf(stderr,"Error: cannot open \"%s\"\n", zSrcFile); close_db(pSrc); return 1; } @@ -10866,21 +10548,21 @@ static int do_meta_command(char *zLine, ShellState *p){ ){ if( nArg==2 ){ if( cli_strcmp(azArg[1], "vm")==0 ){ - p->scanstatsOn = 3; + p->mode.scanstatsOn = 3; }else if( cli_strcmp(azArg[1], "est")==0 ){ - p->scanstatsOn = 2; + p->mode.scanstatsOn = 2; }else{ - p->scanstatsOn = (u8)booleanValue(azArg[1]); + p->mode.scanstatsOn = (u8)booleanValue(azArg[1]); } open_db(p, 0); sqlite3_db_config( - p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->scanstatsOn, (int*)0 + p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->mode.scanstatsOn, (int*)0 ); #if !defined(SQLITE_ENABLE_STMT_SCANSTATUS) eputz("Warning: .scanstats not available in this build.\n"); #elif !defined(SQLITE_ENABLE_BYTECODE_VTAB) - if( p->scanstatsOn==3 ){ + if( p->mode.scanstatsOn==3 ){ eputz("Warning: \".scanstats vm\" not available in this build.\n"); } #endif @@ -10891,7 +10573,6 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( c=='s' && cli_strncmp(azArg[0], "schema", n)==0 ){ - ShellText sSelect; ShellState data; char *zErrMsg = 0; const char *zDiv = "("; @@ -10899,22 +10580,27 @@ static int do_meta_command(char *zLine, ShellState *p){ int iSchema = 0; int bDebug = 0; int bNoSystemTabs = 0; + int bIndent = 0; int ii; - + sqlite3_str *pSql; + sqlite3_stmt *pStmt = 0; + open_db(p, 0); memcpy(&data, p, sizeof(data)); - data.showHeader = 0; - data.cMode = data.mode = MODE_Semi; - initText(&sSelect); + data.mode.spec.bTitles = QRF_No; + data.mode.eMode = MODE_List; + data.mode.spec.eText = QRF_TEXT_Plain; + data.mode.spec.nCharLimit = 0; + data.mode.spec.zRowSep = "\n"; for(ii=1; ii<nArg; ii++){ if( optionMatch(azArg[ii],"indent") ){ - data.cMode = data.mode = MODE_Pretty; + bIndent = 1; }else if( optionMatch(azArg[ii],"debug") ){ bDebug = 1; }else if( optionMatch(azArg[ii],"nosys") ){ bNoSystemTabs = 1; }else if( azArg[ii][0]=='-' ){ - sqlite3_fprintf(stderr,"Unknown option: \"%s\"\n", azArg[ii]); + cli_printf(stderr,"Unknown option: \"%s\"\n", azArg[ii]); rc = 1; goto meta_command_exit; }else if( zName==0 ){ @@ -10931,96 +10617,85 @@ static int do_meta_command(char *zLine, ShellState *p){ || sqlite3_strlike(zName,"sqlite_temp_master", '\\')==0 || sqlite3_strlike(zName,"sqlite_temp_schema", '\\')==0; if( isSchema ){ - char *new_argv[2], *new_colv[2]; - new_argv[0] = sqlite3_mprintf( - "CREATE TABLE %s (\n" + cli_printf(p->out, + "CREATE TABLE %ssqlite_schema (\n" " type text,\n" " name text,\n" " tbl_name text,\n" " rootpage integer,\n" " sql text\n" - ")", zName); - shell_check_oom(new_argv[0]); - new_argv[1] = 0; - new_colv[0] = "sql"; - new_colv[1] = 0; - callback(&data, 1, new_argv, new_colv); - sqlite3_free(new_argv[0]); + ");\n", + sqlite3_strlike("sqlite_t%",zName,0)==0 ? "temp." : "" + ); } } - if( zDiv ){ - sqlite3_stmt *pStmt = 0; - rc = sqlite3_prepare_v2(p->db, "SELECT name FROM pragma_database_list", - -1, &pStmt, 0); - if( rc ){ - shellDatabaseError(p->db); - sqlite3_finalize(pStmt); - rc = 1; - goto meta_command_exit; - } - appendText(&sSelect, "SELECT sql FROM", 0); - iSchema = 0; - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - const char *zDb = (const char*)sqlite3_column_text(pStmt, 0); - char zScNum[30]; - sqlite3_snprintf(sizeof(zScNum), zScNum, "%d", ++iSchema); - appendText(&sSelect, zDiv, 0); - zDiv = " UNION ALL "; - appendText(&sSelect, "SELECT shell_add_schema(sql,", 0); - if( sqlite3_stricmp(zDb, "main")!=0 ){ - appendText(&sSelect, zDb, '\''); - }else{ - appendText(&sSelect, "NULL", 0); - } - appendText(&sSelect, ",name) AS sql, type, tbl_name, name, rowid,", 0); - appendText(&sSelect, zScNum, 0); - appendText(&sSelect, " AS snum, ", 0); - appendText(&sSelect, zDb, '\''); - appendText(&sSelect, " AS sname FROM ", 0); - appendText(&sSelect, zDb, quoteChar(zDb)); - appendText(&sSelect, ".sqlite_schema", 0); - } + rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); + if( rc ){ + shellDatabaseError(p->db); sqlite3_finalize(pStmt); -#ifndef SQLITE_OMIT_INTROSPECTION_PRAGMAS - if( zName ){ - appendText(&sSelect, - " UNION ALL SELECT shell_module_schema(name)," - " 'table', name, name, name, 9e+99, 'main' FROM pragma_module_list", - 0); - } -#endif - appendText(&sSelect, ") WHERE ", 0); - if( zName ){ - char *zQarg = sqlite3_mprintf("%Q", zName); - int bGlob; - shell_check_oom(zQarg); - bGlob = strchr(zName, '*') != 0 || strchr(zName, '?') != 0 || - strchr(zName, '[') != 0; - if( strchr(zName, '.') ){ - appendText(&sSelect, "lower(printf('%s.%s',sname,tbl_name))", 0); - }else{ - appendText(&sSelect, "lower(tbl_name)", 0); - } - appendText(&sSelect, bGlob ? " GLOB " : " LIKE ", 0); - appendText(&sSelect, zQarg, 0); - if( !bGlob ){ - appendText(&sSelect, " ESCAPE '\\' ", 0); - } - appendText(&sSelect, " AND ", 0); - sqlite3_free(zQarg); + + rc = 1; + goto meta_command_exit; + } + pSql = sqlite3_str_new(p->db); + sqlite3_str_appendf(pSql, "SELECT sql FROM", 0); + iSchema = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zDb = (const char*)sqlite3_column_text(pStmt, 1); + char zScNum[30]; + sqlite3_snprintf(sizeof(zScNum), zScNum, "%d", ++iSchema); + sqlite3_str_appendall(pSql, zDiv); + zDiv = " UNION ALL "; + if( sqlite3_stricmp(zDb, "main")==0 ){ + sqlite3_str_appendf(pSql, + "SELECT shell_format_schema(shell_add_schema(sql,NULL,name),%d)", + bIndent); + }else{ + sqlite3_str_appendf(pSql, + "SELECT shell_format_schema(shell_add_schema(sql,%Q,name),%d)", + zDb, bIndent); } - if( bNoSystemTabs ){ - appendText(&sSelect, "name NOT LIKE 'sqlite__%%' ESCAPE '_' AND ", 0); + sqlite3_str_appendf(pSql, + " AS sql, type, tbl_name, name, rowid, %d AS snum, %Q as sname", + ++iSchema, zDb); + sqlite3_str_appendf(pSql," FROM \"%w\".sqlite_schema", zDb); + } + sqlite3_finalize(pStmt); +#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) \ + && !defined(SQLITE_OMIT_VIRTUALTABLE) + if( zName ){ + sqlite3_str_appendall(pSql, + " UNION ALL SELECT shell_module_schema(name)," + " 'table', name, name, name, 9e+99, 'main' FROM pragma_module_list"); + } +#endif + sqlite3_str_appendf(pSql, ") WHERE ", 0); + if( zName ){ + int bGlob; + bGlob = strchr(zName, '*') != 0 || strchr(zName, '?') != 0 || + strchr(zName, '[') != 0; + if( strchr(zName, '.') ){ + sqlite3_str_appendall(pSql, "lower(format('%%s.%%s',sname,tbl_name))"); + }else{ + sqlite3_str_appendall(pSql, "lower(tbl_name)"); } - appendText(&sSelect, "sql IS NOT NULL" - " ORDER BY snum, rowid", 0); - if( bDebug ){ - sqlite3_fprintf(p->out, "SQL: %s;\n", sSelect.zTxt); + if( bGlob ){ + sqlite3_str_appendf(pSql, " GLOB %Q AND ", zName); }else{ - rc = sqlite3_exec(p->db, sSelect.zTxt, callback, &data, &zErrMsg); + sqlite3_str_appendf(pSql, " LIKE %Q ESCAPE '\\' AND ", zName); } - freeText(&sSelect); } + if( bNoSystemTabs ){ + sqlite3_str_appendf(pSql, " name NOT LIKE 'sqlite__%%' ESCAPE '_' AND "); + } + sqlite3_str_appendf(pSql, "sql IS NOT NULL ORDER BY snum, rowid"); + if( bDebug ){ + cli_printf(p->out, "SQL: %s;\n", sqlite3_str_value(pSql)); + }else{ + rc = shell_exec(&data, sqlite3_str_value(pSql), &zErrMsg); + } + sqlite3_str_free(pSql); + if( zErrMsg ){ shellEmitError(zErrMsg); sqlite3_free(zErrMsg); @@ -11076,7 +10751,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else{ rc = sqlite3session_attach(pSession->p, azCmd[1]); if( rc ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "ERROR: sqlite3session_attach() returns %d\n",rc); rc = 0; } @@ -11096,7 +10771,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( pSession->p==0 ) goto session_not_open; out = sqlite3_fopen(azCmd[1], "wb"); if( out==0 ){ - sqlite3_fprintf(stderr,"ERROR: cannot open \"%s\" for writing\n", + cli_printf(stderr,"ERROR: cannot open \"%s\" for writing\n", azCmd[1]); }else{ int szChng; @@ -11107,12 +10782,12 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = sqlite3session_patchset(pSession->p, &szChng, &pChng); } if( rc ){ - sqlite3_fprintf(stdout, "Error: error code %d\n", rc); + cli_printf(stdout, "Error: error code %d\n", rc); rc = 0; } if( pChng && fwrite(pChng, szChng, 1, out)!=1 ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "ERROR: Failed to write entire %d-byte output\n", szChng); } sqlite3_free(pChng); @@ -11140,7 +10815,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); if( pAuxDb->nSession ){ ii = sqlite3session_enable(pSession->p, ii); - sqlite3_fprintf(p->out, + cli_printf(p->out, "session %s enable flag = %d\n", pSession->zName, ii); } }else @@ -11177,7 +10852,7 @@ static int do_meta_command(char *zLine, ShellState *p){ ii = nCmd==1 ? -1 : booleanValue(azCmd[1]); if( pAuxDb->nSession ){ ii = sqlite3session_indirect(pSession->p, ii); - sqlite3_fprintf(p->out, + cli_printf(p->out, "session %s indirect flag = %d\n", pSession->zName, ii); } }else @@ -11190,7 +10865,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( nCmd!=1 ) goto session_syntax_error; if( pAuxDb->nSession ){ ii = sqlite3session_isempty(pSession->p); - sqlite3_fprintf(p->out, + cli_printf(p->out, "session %s isempty flag = %d\n", pSession->zName, ii); } }else @@ -11200,7 +10875,7 @@ static int do_meta_command(char *zLine, ShellState *p){ */ if( cli_strcmp(azCmd[0],"list")==0 ){ for(i=0; i<pAuxDb->nSession; i++){ - sqlite3_fprintf(p->out, "%d %s\n", i, pAuxDb->aSession[i].zName); + cli_printf(p->out, "%d %s\n", i, pAuxDb->aSession[i].zName); } }else @@ -11215,19 +10890,19 @@ static int do_meta_command(char *zLine, ShellState *p){ if( zName[0]==0 ) goto session_syntax_error; for(i=0; i<pAuxDb->nSession; i++){ if( cli_strcmp(pAuxDb->aSession[i].zName,zName)==0 ){ - sqlite3_fprintf(stderr,"Session \"%s\" already exists\n", zName); + cli_printf(stderr,"Session \"%s\" already exists\n", zName); goto meta_command_exit; } } if( pAuxDb->nSession>=ArraySize(pAuxDb->aSession) ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Maximum of %d sessions\n", ArraySize(pAuxDb->aSession)); goto meta_command_exit; } pSession = &pAuxDb->aSession[pAuxDb->nSession]; rc = sqlite3session_create(p->db, azCmd[1], &pSession->p); if( rc ){ - sqlite3_fprintf(stderr,"Cannot open session: error code=%d\n", rc); + cli_printf(stderr,"Cannot open session: error code=%d\n", rc); rc = 0; goto meta_command_exit; } @@ -11251,7 +10926,7 @@ static int do_meta_command(char *zLine, ShellState *p){ int i, v; for(i=1; i<nArg; i++){ v = booleanValue(azArg[i]); - sqlite3_fprintf(p->out, "%s: %d 0x%x\n", azArg[i], v, v); + cli_printf(p->out, "%s: %d 0x%x\n", azArg[i], v, v); } } if( cli_strncmp(azArg[0]+9, "integer", n-9)==0 ){ @@ -11260,7 +10935,7 @@ static int do_meta_command(char *zLine, ShellState *p){ char zBuf[200]; v = integerValue(azArg[i]); sqlite3_snprintf(sizeof(zBuf),zBuf,"%s: %lld 0x%llx\n", azArg[i],v,v); - sqlite3_fputs(zBuf, p->out); + cli_puts(zBuf, p->out); } } }else @@ -11287,9 +10962,9 @@ static int do_meta_command(char *zLine, ShellState *p){ bVerbose++; }else { - sqlite3_fprintf(stderr, + cli_printf(stderr, "Unknown option \"%s\" on \"%s\"\n", azArg[i], azArg[0]); - sqlite3_fputs("Should be one of: --init -v\n", stderr); + cli_puts("Should be one of: --init -v\n", stderr); rc = 1; goto meta_command_exit; } @@ -11334,10 +11009,10 @@ static int do_meta_command(char *zLine, ShellState *p){ if( zAns==0 ) continue; k = 0; if( bVerbose>0 ){ - sqlite3_fprintf(stdout, "%d: %s %s\n", tno, zOp, zSql); + cli_printf(stdout, "%d: %s %s\n", tno, zOp, zSql); } if( cli_strcmp(zOp,"memo")==0 ){ - sqlite3_fprintf(p->out, "%s\n", zSql); + cli_printf(p->out, "%s\n", zSql); }else if( cli_strcmp(zOp,"run")==0 ){ char *zErrMsg = 0; @@ -11346,22 +11021,22 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = sqlite3_exec(p->db, zSql, captureOutputCallback, &str, &zErrMsg); nTest++; if( bVerbose ){ - sqlite3_fprintf(p->out, "Result: %s\n", str.zTxt); + cli_printf(p->out, "Result: %s\n", str.zTxt); } if( rc || zErrMsg ){ nErr++; rc = 1; - sqlite3_fprintf(p->out, "%d: error-code-%d: %s\n", tno, rc,zErrMsg); + cli_printf(p->out, "%d: error-code-%d: %s\n", tno, rc,zErrMsg); sqlite3_free(zErrMsg); }else if( cli_strcmp(zAns,str.zTxt)!=0 ){ nErr++; rc = 1; - sqlite3_fprintf(p->out, "%d: Expected: [%s]\n", tno, zAns); - sqlite3_fprintf(p->out, "%d: Got: [%s]\n", tno, str.zTxt); + cli_printf(p->out, "%d: Expected: [%s]\n", tno, zAns); + cli_printf(p->out, "%d: Got: [%s]\n", tno, str.zTxt); } } else{ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Unknown operation \"%s\" on selftest line %d\n", zOp, tno); rc = 1; break; @@ -11370,7 +11045,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_finalize(pStmt); } /* End loop over k */ freeText(&str); - sqlite3_fprintf(p->out, "%d errors out of %d tests\n", nErr, nTest); + cli_printf(p->out, "%d errors out of %d tests\n", nErr, nTest); }else if( c=='s' && cli_strncmp(azArg[0], "separator", n)==0 ){ @@ -11379,12 +11054,10 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = 1; } if( nArg>=2 ){ - sqlite3_snprintf(sizeof(p->colSeparator), p->colSeparator, - "%.*s", (int)ArraySize(p->colSeparator)-1, azArg[1]); + modeSetStr(&p->mode.spec.zColumnSep, azArg[1]); } if( nArg>=3 ){ - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, - "%.*s", (int)ArraySize(p->rowSeparator)-1, azArg[2]); + modeSetStr(&p->mode.spec.zRowSep,azArg[2]); } }else @@ -11418,7 +11091,7 @@ static int do_meta_command(char *zLine, ShellState *p){ bDebug = 1; }else { - sqlite3_fprintf(stderr, + cli_printf(stderr, "Unknown option \"%s\" on \"%s\"\n", azArg[i], azArg[0]); showHelp(p->out, azArg[0]); rc = 1; @@ -11497,7 +11170,7 @@ static int do_meta_command(char *zLine, ShellState *p){ freeText(&sQuery); freeText(&sSql); if( bDebug ){ - sqlite3_fprintf(p->out, "%s\n", zSql); + cli_printf(p->out, "%s\n", zSql); }else{ shell_exec(p, zSql, 0); } @@ -11527,7 +11200,7 @@ static int do_meta_command(char *zLine, ShellState *p){ "' OR ') as query, tname from tabcols group by tname)" , zRevText); shell_check_oom(zRevText); - if( bDebug ) sqlite3_fprintf(p->out, "%s\n", zRevText); + if( bDebug ) cli_printf(p->out, "%s\n", zRevText); lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0); if( lrc!=SQLITE_OK ){ /* assert(lrc==SQLITE_NOMEM); // might also be SQLITE_ERROR if the @@ -11540,7 +11213,7 @@ static int do_meta_command(char *zLine, ShellState *p){ const char *zGenQuery = (char*)sqlite3_column_text(pStmt,0); sqlite3_stmt *pCheckStmt; lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0); - if( bDebug ) sqlite3_fprintf(p->out, "%s\n", zGenQuery); + if( bDebug ) cli_printf(p->out, "%s\n", zGenQuery); if( lrc!=SQLITE_OK ){ rc = 1; }else{ @@ -11548,7 +11221,7 @@ static int do_meta_command(char *zLine, ShellState *p){ double countIrreversible = sqlite3_column_double(pCheckStmt, 0); if( countIrreversible>0 ){ int sz = (int)(countIrreversible + 0.5); - sqlite3_fprintf(stderr, + cli_printf(stderr, "Digest includes %d invalidly encoded text field%s.\n", sz, (sz>1)? "s": ""); } @@ -11587,7 +11260,7 @@ static int do_meta_command(char *zLine, ShellState *p){ x = zCmd!=0 ? system(zCmd) : 1; /*consoleRenewSetup();*/ sqlite3_free(zCmd); - if( x ) sqlite3_fprintf(stderr,"System command returns %d\n", x); + if( x ) cli_printf(stderr,"System command returns %d\n", x); }else #endif /* !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) */ @@ -11600,48 +11273,51 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = 1; goto meta_command_exit; } - sqlite3_fprintf(p->out, "%12.12s: %s\n","echo", - azBool[ShellHasFlag(p, SHFLG_Echo)]); - sqlite3_fprintf(p->out, "%12.12s: %s\n","eqp", azBool[p->autoEQP&3]); - sqlite3_fprintf(p->out, "%12.12s: %s\n","explain", - p->mode==MODE_Explain ? "on" : p->autoExplain ? "auto" : "off"); - sqlite3_fprintf(p->out, "%12.12s: %s\n","headers", - azBool[p->showHeader!=0]); - if( p->mode==MODE_Column - || (p->mode>=MODE_Markdown && p->mode<=MODE_Box) + cli_printf(p->out, "%12.12s: %s\n","echo", + azBool[(p->mode.mFlags & MFLG_ECHO)!=0]); + cli_printf(p->out, "%12.12s: %s\n","eqp", azBool[p->mode.autoEQP&3]); + cli_printf(p->out, "%12.12s: %s\n","explain", + p->mode.autoExplain ? "auto" : "off"); + cli_printf(p->out, "%12.12s: %s\n","headers", + azBool[p->mode.spec.bTitles==QRF_Yes]); + if( p->mode.spec.eStyle==QRF_STYLE_Column + || p->mode.spec.eStyle==QRF_STYLE_Box + || p->mode.spec.eStyle==QRF_STYLE_Table + || p->mode.spec.eStyle==QRF_STYLE_Markdown ){ - sqlite3_fprintf(p->out, + cli_printf(p->out, "%12.12s: %s --wrap %d --wordwrap %s --%squote\n", "mode", - modeDescr[p->mode], p->cmOpts.iWrap, - p->cmOpts.bWordWrap ? "on" : "off", - p->cmOpts.bQuote ? "" : "no"); + aModeInfo[p->mode.eMode].zName, p->mode.spec.nWrap, + p->mode.spec.bWordWrap==QRF_Yes ? "on" : "off", + p->mode.spec.eText==QRF_TEXT_Sql ? "" : "no"); }else{ - sqlite3_fprintf(p->out, "%12.12s: %s\n","mode", modeDescr[p->mode]); + cli_printf(p->out, "%12.12s: %s\n","mode", + aModeInfo[p->mode.eMode].zName); } - sqlite3_fprintf(p->out, "%12.12s: ", "nullvalue"); - output_c_string(p->out, p->nullValue); - sqlite3_fputs("\n", p->out); - sqlite3_fprintf(p->out, "%12.12s: %s\n","output", + cli_printf(p->out, "%12.12s: ", "nullvalue"); + output_c_string(p->out, p->mode.spec.zNull); + cli_puts("\n", p->out); + cli_printf(p->out, "%12.12s: %s\n","output", strlen30(p->outfile) ? p->outfile : "stdout"); - sqlite3_fprintf(p->out, "%12.12s: ", "colseparator"); - output_c_string(p->out, p->colSeparator); - sqlite3_fputs("\n", p->out); - sqlite3_fprintf(p->out, "%12.12s: ", "rowseparator"); - output_c_string(p->out, p->rowSeparator); - sqlite3_fputs("\n", p->out); + cli_printf(p->out, "%12.12s: ", "colseparator"); + output_c_string(p->out, p->mode.spec.zColumnSep); + cli_puts("\n", p->out); + cli_printf(p->out, "%12.12s: ", "rowseparator"); + output_c_string(p->out, p->mode.spec.zRowSep); + cli_puts("\n", p->out); switch( p->statsOn ){ case 0: zOut = "off"; break; default: zOut = "on"; break; case 2: zOut = "stmt"; break; case 3: zOut = "vmstep"; break; } - sqlite3_fprintf(p->out, "%12.12s: %s\n","stats", zOut); - sqlite3_fprintf(p->out, "%12.12s: ", "width"); - for (i=0;i<p->nWidth;i++) { - sqlite3_fprintf(p->out, "%d ", p->colWidth[i]); + cli_printf(p->out, "%12.12s: %s\n","stats", zOut); + cli_printf(p->out, "%12.12s: ", "width"); + for(i=0; i<p->mode.spec.nWidth; i++){ + cli_printf(p->out, "%d ", (int)p->mode.spec.aWidth[i]); } - sqlite3_fputs("\n", p->out); - sqlite3_fprintf(p->out, "%12.12s: %s\n", "filename", + cli_puts("\n", p->out); + cli_printf(p->out, "%12.12s: %s\n", "filename", p->pAuxDb->zDbFilename ? p->pAuxDb->zDbFilename : ""); }else @@ -11662,16 +11338,11 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else - if( (c=='t' && n>1 && cli_strncmp(azArg[0], "tables", n)==0) - || (c=='i' && (cli_strncmp(azArg[0], "indices", n)==0 - || cli_strncmp(azArg[0], "indexes", n)==0) ) - ){ + if( (c=='t' && n>1 && cli_strncmp(azArg[0], "tables", n)==0) ){ sqlite3_stmt *pStmt; - char **azResult; - int nRow, nAlloc; - int ii; - ShellText s; - initText(&s); + sqlite3_str *pSql; + const char *zPattern = nArg>1 ? azArg[1] : 0; + open_db(p, 0); rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); if( rc ){ @@ -11679,112 +11350,46 @@ static int do_meta_command(char *zLine, ShellState *p){ return shellDatabaseError(p->db); } - if( nArg>2 && c=='i' ){ - /* It is an historical accident that the .indexes command shows an error - ** when called with the wrong number of arguments whereas the .tables - ** command does not. */ - eputz("Usage: .indexes ?LIKE-PATTERN?\n"); - rc = 1; - sqlite3_finalize(pStmt); - goto meta_command_exit; - } - for(ii=0; sqlite3_step(pStmt)==SQLITE_ROW; ii++){ + pSql = sqlite3_str_new(p->db); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ const char *zDbName = (const char*)sqlite3_column_text(pStmt, 1); if( zDbName==0 ) continue; - if( s.zTxt && s.zTxt[0] ) appendText(&s, " UNION ALL ", 0); + if( sqlite3_str_length(pSql) ){ + sqlite3_str_appendall(pSql, " UNION ALL "); + } if( sqlite3_stricmp(zDbName, "main")==0 ){ - appendText(&s, "SELECT name FROM ", 0); + sqlite3_str_appendall(pSql, "SELECT name FROM "); }else{ - appendText(&s, "SELECT ", 0); - appendText(&s, zDbName, '\''); - appendText(&s, "||'.'||name FROM ", 0); - } - appendText(&s, zDbName, '"'); - appendText(&s, ".sqlite_schema ", 0); - if( c=='t' ){ - appendText(&s," WHERE type IN ('table','view')" - " AND name NOT LIKE 'sqlite__%' ESCAPE '_'" - " AND name LIKE ?1", 0); - }else{ - appendText(&s," WHERE type='index'" - " AND tbl_name LIKE ?1", 0); + sqlite3_str_appendf(pSql, "SELECT %Q||'.'||name FROM ", zDbName); + } + sqlite3_str_appendf(pSql, "\"%w\".sqlite_schema", zDbName); + sqlite3_str_appendf(pSql, + " WHERE type IN ('table','view')" + " AND name NOT LIKE 'sqlite__%%' ESCAPE '_'" + ); + if( zPattern ){ + sqlite3_str_appendf(pSql," AND name LIKE %Q", zPattern); } } rc = sqlite3_finalize(pStmt); if( rc==SQLITE_OK ){ - appendText(&s, " ORDER BY 1", 0); - rc = sqlite3_prepare_v2(p->db, s.zTxt, -1, &pStmt, 0); - } - freeText(&s); - if( rc ) return shellDatabaseError(p->db); - - /* Run the SQL statement prepared by the above block. Store the results - ** as an array of nul-terminated strings in azResult[]. */ - nRow = nAlloc = 0; - azResult = 0; - if( nArg>1 ){ - sqlite3_bind_text(pStmt, 1, azArg[1], -1, SQLITE_TRANSIENT); - }else{ - sqlite3_bind_text(pStmt, 1, "%", -1, SQLITE_STATIC); - } - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - if( nRow>=nAlloc ){ - char **azNew; - sqlite3_int64 n2 = 2*(sqlite3_int64)nAlloc + 10; - azNew = sqlite3_realloc64(azResult, sizeof(azResult[0])*n2); - shell_check_oom(azNew); - nAlloc = (int)n2; - azResult = azNew; - } - azResult[nRow] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); - shell_check_oom(azResult[nRow]); - nRow++; - } - if( sqlite3_finalize(pStmt)!=SQLITE_OK ){ - rc = shellDatabaseError(p->db); - } - - /* Pretty-print the contents of array azResult[] to the output */ - if( rc==0 && nRow>0 ){ - int len, maxlen = 0; - int i, j; - int nPrintCol, nPrintRow; - for(i=0; i<nRow; i++){ - len = strlen30(azResult[i]); - if( len>maxlen ) maxlen = len; - } - nPrintCol = 80/(maxlen+2); - if( nPrintCol<1 ) nPrintCol = 1; - nPrintRow = (nRow + nPrintCol - 1)/nPrintCol; - for(i=0; i<nPrintRow; i++){ - for(j=i; j<nRow; j+=nPrintRow){ - char *zSp = j<nPrintRow ? "" : " "; - sqlite3_fprintf(p->out, - "%s%-*s", zSp, maxlen, azResult[j] ? azResult[j]:""); - } - sqlite3_fputs("\n", p->out); - } + sqlite3_str_appendall(pSql, " ORDER BY 1"); } - for(ii=0; ii<nRow; ii++) sqlite3_free(azResult[ii]); - sqlite3_free(azResult); + /* Run the SQL statement in "split" mode. */ + modePush(p); + modeChange(p, MODE_Split); + shell_exec(p, sqlite3_str_value(pSql), 0); + sqlite3_str_free(pSql); + modePop(p); + if( rc ) return shellDatabaseError(p->db); }else -#ifndef SQLITE_SHELL_FIDDLE - /* Begin redirecting output to the file "testcase-out.txt" */ + /* Set the p->zTestcase name and begin redirecting output into + ** the cli_output_capture sqlite3_str */ if( c=='t' && cli_strcmp(azArg[0],"testcase")==0 ){ - output_reset(p); - p->out = output_file_open("testcase-out.txt"); - if( p->out==0 ){ - eputz("Error: cannot open 'testcase-out.txt'\n"); - } - if( nArg>=2 ){ - sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "%s", azArg[1]); - }else{ - sqlite3_snprintf(sizeof(p->zTestcase), p->zTestcase, "?"); - } + rc = dotCmdTestcase(p); }else -#endif /* !defined(SQLITE_SHELL_FIDDLE) */ #ifndef SQLITE_UNTESTABLE if( c=='t' && n>=8 && cli_strncmp(azArg[0], "testctrl", n)==0 ){ @@ -11837,10 +11442,10 @@ static int do_meta_command(char *zLine, ShellState *p){ /* --help lists all test-controls */ if( cli_strcmp(zCmd,"help")==0 ){ - sqlite3_fputs("Available test-controls:\n", p->out); + cli_puts("Available test-controls:\n", p->out); for(i=0; i<ArraySize(aCtrl); i++){ if( aCtrl[i].unSafe && !ShellHasFlag(p,SHFLG_TestingMode) ) continue; - sqlite3_fprintf(p->out, " .testctrl %s %s\n", + cli_printf(p->out, " .testctrl %s %s\n", aCtrl[i].zCtrlName, aCtrl[i].zUsage); } rc = 1; @@ -11857,7 +11462,7 @@ static int do_meta_command(char *zLine, ShellState *p){ testctrl = aCtrl[i].ctrlCode; iCtrl = i; }else{ - sqlite3_fprintf(stderr,"Error: ambiguous test-control: \"%s\"\n" + cli_printf(stderr,"Error: ambiguous test-control: \"%s\"\n" "Use \".testctrl --help\" for help\n", zCmd); rc = 1; goto meta_command_exit; @@ -11865,7 +11470,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } } if( testctrl<0 ){ - sqlite3_fprintf(stderr,"Error: unknown test-control: %s\n" + cli_printf(stderr,"Error: unknown test-control: %s\n" "Use \".testctrl --help\" for help\n", zCmd); }else{ switch(testctrl){ @@ -11945,13 +11550,13 @@ static int do_meta_command(char *zLine, ShellState *p){ if( sqlite3_stricmp(zLabel, aLabel[jj].zLabel)==0 ) break; } if( jj>=ArraySize(aLabel) ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Error: no such optimization: \"%s\"\n", zLabel); - sqlite3_fputs("Should be one of:", stderr); + cli_puts("Should be one of:", stderr); for(jj=0; jj<ArraySize(aLabel); jj++){ - sqlite3_fprintf(stderr," %s", aLabel[jj].zLabel); + cli_printf(stderr," %s", aLabel[jj].zLabel); } - sqlite3_fputs("\n", stderr); + cli_puts("\n", stderr); rc = 1; goto meta_command_exit; } @@ -11969,23 +11574,23 @@ static int do_meta_command(char *zLine, ShellState *p){ if( m & newOpt ) nOff++; } if( nOff<12 ){ - sqlite3_fputs("+All", p->out); + cli_puts("+All", p->out); for(ii=0; ii<ArraySize(aLabel); ii++){ if( !aLabel[ii].bDsply ) continue; if( (newOpt & aLabel[ii].mask)!=0 ){ - sqlite3_fprintf(p->out, " -%s", aLabel[ii].zLabel); + cli_printf(p->out, " -%s", aLabel[ii].zLabel); } } }else{ - sqlite3_fputs("-All", p->out); + cli_puts("-All", p->out); for(ii=0; ii<ArraySize(aLabel); ii++){ if( !aLabel[ii].bDsply ) continue; if( (newOpt & aLabel[ii].mask)==0 ){ - sqlite3_fprintf(p->out, " +%s", aLabel[ii].zLabel); + cli_printf(p->out, " +%s", aLabel[ii].zLabel); } } } - sqlite3_fputs("\n", p->out); + cli_puts("\n", p->out); rc2 = isOk = 3; break; } @@ -12025,7 +11630,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3 *db; if( ii==0 && cli_strcmp(azArg[2],"random")==0 ){ sqlite3_randomness(sizeof(ii),&ii); - sqlite3_fprintf(stdout, "-- random seed: %d\n", ii); + cli_printf(stdout, "-- random seed: %d\n", ii); } if( nArg==3 ){ db = 0; @@ -12078,7 +11683,7 @@ static int do_meta_command(char *zLine, ShellState *p){ case SQLITE_TESTCTRL_SEEK_COUNT: { u64 x = 0; rc2 = sqlite3_test_control(testctrl, p->db, &x); - sqlite3_fprintf(p->out, "%llu\n", x); + cli_printf(p->out, "%llu\n", x); isOk = 3; break; } @@ -12109,11 +11714,11 @@ static int do_meta_command(char *zLine, ShellState *p){ int val = 0; rc2 = sqlite3_test_control(testctrl, -id, &val); if( rc2!=SQLITE_OK ) break; - if( id>1 ) sqlite3_fputs(" ", p->out); - sqlite3_fprintf(p->out, "%d: %d", id, val); + if( id>1 ) cli_puts(" ", p->out); + cli_printf(p->out, "%d: %d", id, val); id++; } - if( id>1 ) sqlite3_fputs("\n", p->out); + if( id>1 ) cli_puts("\n", p->out); isOk = 3; } break; @@ -12152,7 +11757,7 @@ static int do_meta_command(char *zLine, ShellState *p){ int ii, jj, x; int *aOp; if( nArg!=4 ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "ERROR - should be: \".testctrl bitvec_test SIZE INT-ARRAY\"\n" ); rc = 1; @@ -12175,7 +11780,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } aOp[jj] = x; x = sqlite3_test_control(testctrl, iSize, aOp); - sqlite3_fprintf(p->out, "result: %d\n", x); + cli_printf(p->out, "result: %d\n", x); free(aOp); break; } @@ -12198,21 +11803,21 @@ static int do_meta_command(char *zLine, ShellState *p){ faultsim_state.nHit = 0; sqlite3_test_control(testctrl, faultsim_callback); }else if( cli_strcmp(z,"status")==0 ){ - sqlite3_fprintf(p->out, "faultsim.iId: %d\n", + cli_printf(p->out, "faultsim.iId: %d\n", faultsim_state.iId); - sqlite3_fprintf(p->out, "faultsim.iErr: %d\n", + cli_printf(p->out, "faultsim.iErr: %d\n", faultsim_state.iErr); - sqlite3_fprintf(p->out, "faultsim.iCnt: %d\n", + cli_printf(p->out, "faultsim.iCnt: %d\n", faultsim_state.iCnt); - sqlite3_fprintf(p->out, "faultsim.nHit: %d\n", + cli_printf(p->out, "faultsim.nHit: %d\n", faultsim_state.nHit); - sqlite3_fprintf(p->out, "faultsim.iInterval: %d\n", + cli_printf(p->out, "faultsim.iInterval: %d\n", faultsim_state.iInterval); - sqlite3_fprintf(p->out, "faultsim.eVerbose: %d\n", + cli_printf(p->out, "faultsim.eVerbose: %d\n", faultsim_state.eVerbose); - sqlite3_fprintf(p->out, "faultsim.nRepeat: %d\n", + cli_printf(p->out, "faultsim.nRepeat: %d\n", faultsim_state.nRepeat); - sqlite3_fprintf(p->out, "faultsim.nSkip: %d\n", + cli_printf(p->out, "faultsim.nSkip: %d\n", faultsim_state.nSkip); }else if( cli_strcmp(z,"-v")==0 ){ if( faultsim_state.eVerbose<2 ) faultsim_state.eVerbose++; @@ -12231,7 +11836,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( cli_strcmp(z,"-?")==0 || sqlite3_strglob("*help*",z)==0){ bShowHelp = 1; }else{ - sqlite3_fprintf(stderr, + cli_printf(stderr, "Unrecognized fault_install argument: \"%s\"\n", azArg[kk]); rc = 1; @@ -12240,7 +11845,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } } if( bShowHelp ){ - sqlite3_fputs( + cli_puts( "Usage: .testctrl fault_install ARGS\n" "Possible arguments:\n" " off Disable faultsim\n" @@ -12262,13 +11867,13 @@ static int do_meta_command(char *zLine, ShellState *p){ } } if( isOk==0 && iCtrl>=0 ){ - sqlite3_fprintf(p->out, + cli_printf(p->out, "Usage: .testctrl %s %s\n", zCmd,aCtrl[iCtrl].zUsage); rc = 1; }else if( isOk==1 ){ - sqlite3_fprintf(p->out, "%d\n", rc2); + cli_printf(p->out, "%d\n", rc2); }else if( isOk==2 ){ - sqlite3_fprintf(p->out, "0x%08x\n", rc2); + cli_printf(p->out, "0x%08x\n", rc2); } }else #endif /* !defined(SQLITE_UNTESTABLE) */ @@ -12280,13 +11885,17 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='t' && n>=5 && cli_strncmp(azArg[0], "timer", n)==0 ){ if( nArg==2 ){ - enableTimer = booleanValue(azArg[1]); - if( enableTimer && !HAS_TIMER ){ + if( cli_strcmp(azArg[1],"once")==0 ){ + p->enableTimer = 1; + }else{ + p->enableTimer = 2*booleanValue(azArg[1]); + } + if( p->enableTimer && !HAS_TIMER ){ eputz("Error: timer not available on this system.\n"); - enableTimer = 0; + p->enableTimer = 0; } }else{ - eputz("Usage: .timer on|off\n"); + eputz("Usage: .timer on|off|once\n"); rc = 1; } }else @@ -12323,13 +11932,13 @@ static int do_meta_command(char *zLine, ShellState *p){ mType |= SQLITE_TRACE_CLOSE; } else { - sqlite3_fprintf(stderr,"Unknown option \"%s\" on \".trace\"\n", z); + cli_printf(stderr,"Unknown option \"%s\" on \".trace\"\n", z); rc = 1; goto meta_command_exit; } }else{ output_file_close(p->traceOut); - p->traceOut = output_file_open(z); + p->traceOut = output_file_open(p, z); } } if( p->traceOut==0 ){ @@ -12368,21 +11977,21 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='v' && cli_strncmp(azArg[0], "version", n)==0 ){ char *zPtrSz = sizeof(void*)==8 ? "64-bit" : "32-bit"; - sqlite3_fprintf(p->out, "SQLite %s %s\n" /*extra-version-info*/, + cli_printf(p->out, "SQLite %s %s\n" /*extra-version-info*/, sqlite3_libversion(), sqlite3_sourceid()); #if SQLITE_HAVE_ZLIB - sqlite3_fprintf(p->out, "zlib version %s\n", zlibVersion()); + cli_printf(p->out, "zlib version %s\n", zlibVersion()); #endif #define CTIMEOPT_VAL_(opt) #opt #define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt) #if defined(__clang__) && defined(__clang_major__) - sqlite3_fprintf(p->out, "clang-" CTIMEOPT_VAL(__clang_major__) "." + cli_printf(p->out, "clang-" CTIMEOPT_VAL(__clang_major__) "." CTIMEOPT_VAL(__clang_minor__) "." CTIMEOPT_VAL(__clang_patchlevel__) " (%s)\n", zPtrSz); #elif defined(_MSC_VER) - sqlite3_fprintf(p->out, "msvc-" CTIMEOPT_VAL(_MSC_VER) " (%s)\n", zPtrSz); + cli_printf(p->out, "msvc-" CTIMEOPT_VAL(_MSC_VER) " (%s)\n", zPtrSz); #elif defined(__GNUC__) && defined(__VERSION__) - sqlite3_fprintf(p->out, "gcc-" __VERSION__ " (%s)\n", zPtrSz); + cli_printf(p->out, "gcc-" __VERSION__ " (%s)\n", zPtrSz); #endif }else @@ -12392,10 +12001,10 @@ static int do_meta_command(char *zLine, ShellState *p){ if( p->db ){ sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFS_POINTER, &pVfs); if( pVfs ){ - sqlite3_fprintf(p->out, "vfs.zName = \"%s\"\n", pVfs->zName); - sqlite3_fprintf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); - sqlite3_fprintf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); - sqlite3_fprintf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); + cli_printf(p->out, "vfs.zName = \"%s\"\n", pVfs->zName); + cli_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); + cli_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); + cli_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); } } }else @@ -12407,13 +12016,13 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_file_control(p->db, "main", SQLITE_FCNTL_VFS_POINTER, &pCurrent); } for(pVfs=sqlite3_vfs_find(0); pVfs; pVfs=pVfs->pNext){ - sqlite3_fprintf(p->out, "vfs.zName = \"%s\"%s\n", pVfs->zName, + cli_printf(p->out, "vfs.zName = \"%s\"%s\n", pVfs->zName, pVfs==pCurrent ? " <--- CURRENT" : ""); - sqlite3_fprintf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); - sqlite3_fprintf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); - sqlite3_fprintf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); + cli_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); + cli_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); + cli_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); if( pVfs->pNext ){ - sqlite3_fputs("-----------------------------------\n", p->out); + cli_puts("-----------------------------------\n", p->out); } } }else @@ -12424,7 +12033,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( p->db ){ sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFSNAME, &zVfsName); if( zVfsName ){ - sqlite3_fprintf(p->out, "%s\n", zVfsName); + cli_printf(p->out, "%s\n", zVfsName); sqlite3_free(zVfsName); } } @@ -12437,31 +12046,31 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='w' && cli_strncmp(azArg[0], "width", n)==0 ){ int j; - assert( nArg<=ArraySize(azArg) ); - p->nWidth = nArg-1; - p->colWidth = realloc(p->colWidth, (p->nWidth+1)*sizeof(int)*2); - if( p->colWidth==0 && p->nWidth>0 ) shell_out_of_memory(); - if( p->nWidth ) p->actualWidth = &p->colWidth[p->nWidth]; + p->mode.spec.nWidth = nArg-1; + p->mode.spec.aWidth = realloc(p->mode.spec.aWidth, + (p->mode.spec.nWidth+1)*sizeof(short int)); + shell_check_oom(p->mode.spec.aWidth); for(j=1; j<nArg; j++){ i64 w = integerValue(azArg[j]); - if( w < -30000 ) w = -30000; - if( w > +30000 ) w = +30000; - p->colWidth[j-1] = (int)w; + if( w < -QRF_MAX_WIDTH ) w = -QRF_MAX_WIDTH; + if( w > QRF_MAX_WIDTH ) w = QRF_MAX_WIDTH; + p->mode.spec.aWidth[j-1] = (short int)w; } }else { - sqlite3_fprintf(stderr,"Error: unknown command or invalid arguments: " + cli_printf(stderr,"Error: unknown command or invalid arguments: " " \"%s\". Enter \".help\" for help\n", azArg[0]); rc = 1; } meta_command_exit: - if( p->outCount ){ - p->outCount--; - if( p->outCount==0 ) output_reset(p); + if( p->nPopOutput ){ + p->nPopOutput--; + if( p->nPopOutput==0 ) output_reset(p); } p->bSafeMode = p->bSafeModePersist; + p->dot.nArg = 0; return rc; } @@ -12691,16 +12300,21 @@ static int doAutoDetectRestore(ShellState *p, const char *zSql){ /* ** Run a single line of SQL. Return the number of errors. */ -static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ +static int runOneSqlLine( + ShellState *p, /* Execution context */ + char *zSql, /* SQL to be run */ + const char *zFilename, /* Source file of the sql */ + int startline /* linenumber */ +){ int rc; char *zErrMsg = 0; open_db(p, 0); if( ShellHasFlag(p,SHFLG_Backslash) ) resolve_backslashes(zSql); if( p->flgProgress & SHELL_PROGRESS_RESET ) p->nProgress = 0; - BEGIN_TIMER; + BEGIN_TIMER(p); rc = shell_exec(p, zSql, &zErrMsg); - END_TIMER(p->out); + END_TIMER(p); if( rc || zErrMsg ){ char zPrefix[100]; const char *zErrorTail; @@ -12718,13 +12332,21 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ zErrorType = "Error"; zErrorTail = zErrMsg; } - if( in!=0 || !stdin_is_interactive ){ - sqlite3_snprintf(sizeof(zPrefix), zPrefix, - "%s near line %d:", zErrorType, startline); + if( zFilename || !stdin_is_interactive ){ + if( cli_strcmp(zFilename,"cmdline")==0 ){ + sqlite3_snprintf(sizeof(zPrefix), zPrefix, + "%s in %r command line argument:", zErrorType, startline); + }else if( cli_strcmp(zFilename,"<stdin>")==0 ){ + sqlite3_snprintf(sizeof(zPrefix), zPrefix, + "%s near line %d:", zErrorType, startline); + }else{ + sqlite3_snprintf(sizeof(zPrefix), zPrefix, + "%s near line %d of %s:", zErrorType, startline, zFilename); + } }else{ sqlite3_snprintf(sizeof(zPrefix), zPrefix, "%s:", zErrorType); } - sqlite3_fprintf(stderr,"%s %s\n", zPrefix, zErrorTail); + cli_printf(stderr,"%s %s\n", zPrefix, zErrorTail); sqlite3_free(zErrMsg); zErrMsg = 0; return 1; @@ -12733,7 +12355,7 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ sqlite3_snprintf(sizeof(zLineBuf), zLineBuf, "changes: %lld total_changes: %lld", sqlite3_changes64(p->db), sqlite3_total_changes64(p->db)); - sqlite3_fprintf(p->out, "%s\n", zLineBuf); + cli_printf(p->out, "%s\n", zLineBuf); } if( doAutoDetectRestore(p, zSql) ) return 1; @@ -12741,8 +12363,8 @@ static int runOneSqlLine(ShellState *p, char *zSql, FILE *in, int startline){ } static void echo_group_input(ShellState *p, const char *zDo){ - if( ShellHasFlag(p, SHFLG_Echo) ){ - sqlite3_fprintf(p->out, "%s\n", zDo); + if( p->mode.mFlags & MFLG_ECHO ){ + cli_printf(p->out, "%s\n", zDo); fflush(p->out); } } @@ -12753,12 +12375,13 @@ static void echo_group_input(ShellState *p, const char *zDo){ ** impl because we need the global shellState and cannot access it from that ** function without moving lots of code around (creating a larger/messier diff). */ -static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ +static char *one_input_line(ShellState *p, char *zPrior, int isContinuation){ /* Parse the next line from shellState.wasm.zInput. */ const char *zBegin = shellState.wasm.zPos; const char *z = zBegin; char *zLine = 0; i64 nZ = 0; + FILE *in = p->in; UNUSED_PARAMETER(in); UNUSED_PARAMETER(isContinuation); @@ -12799,22 +12422,27 @@ static int process_input(ShellState *p, const char *zSrc){ int errCnt = 0; /* Number of errors seen */ i64 startline = 0; /* Line number for start of current input */ QuickScanState qss = QSS_Start; /* Accumulated line status (so far) */ + const char *saved_zInFile; /* Prior value of p->zInFile */ + i64 saved_lineno; /* Prior value of p->lineno */ if( p->inputNesting==MAX_INPUT_NESTING ){ /* This will be more informative in a later version. */ - sqlite3_fprintf(stderr,"%s: Input nesting limit (%d) reached at line %lld." + cli_printf(stderr,"%s: Input nesting limit (%d) reached at line %lld." " Check recursion.\n", zSrc, MAX_INPUT_NESTING, p->lineno); return 1; } ++p->inputNesting; + saved_zInFile = p->zInFile; + p->zInFile = zSrc; + saved_lineno = p->lineno; p->lineno = 0; CONTINUE_PROMPT_RESET; while( errCnt==0 || !bail_on_error || (p->in==0 && stdin_is_interactive) ){ fflush(p->out); - zLine = one_input_line(p->in, zLine, nSql>0); + zLine = one_input_line(p, zLine, nSql>0); if( zLine==0 ){ /* End of input */ - if( p->in==0 && stdin_is_interactive ) sqlite3_fputs("\n", p->out); + if( p->in==0 && stdin_is_interactive ) cli_puts("\n", p->out); break; } if( seenInterrupt ){ @@ -12871,22 +12499,26 @@ static int process_input(ShellState *p, const char *zSrc){ if( nSql>0x7fff0000 ){ char zSize[100]; sqlite3_snprintf(sizeof(zSize),zSize,"%,lld",nSql); - sqlite3_fprintf(stderr, "%s:%lld: Input SQL is too big: %s bytes\n", + cli_printf(stderr, "%s:%lld: Input SQL is too big: %s bytes\n", zSrc, startline, zSize); nSql = 0; errCnt++; break; }else if( nSql && QSS_SEMITERM(qss) && sqlite3_complete(zSql) ){ echo_group_input(p, zSql); - errCnt += runOneSqlLine(p, zSql, p->in, startline); + errCnt += runOneSqlLine(p, zSql, p->zInFile, startline); CONTINUE_PROMPT_RESET; nSql = 0; - if( p->outCount ){ + if( p->nPopOutput ){ output_reset(p); - p->outCount = 0; + p->nPopOutput = 0; }else{ clearTempFile(p); } + if( p->nPopMode ){ + modePop(p); + p->nPopMode = 0; + } p->bSafeMode = p->bSafeModePersist; qss = QSS_Start; }else if( nSql && QSS_PLAINWHITE(qss) ){ @@ -12898,12 +12530,14 @@ static int process_input(ShellState *p, const char *zSrc){ if( nSql ){ /* This may be incomplete. Let the SQL parser deal with that. */ echo_group_input(p, zSql); - errCnt += runOneSqlLine(p, zSql, p->in, startline); + errCnt += runOneSqlLine(p, zSql, p->zInFile, startline); CONTINUE_PROMPT_RESET; } free(zSql); free(zLine); --p->inputNesting; + p->zInFile = saved_zInFile; + p->lineno = saved_lineno; return errCnt>0; } @@ -13064,13 +12698,13 @@ static void process_sqliterc( p->in = sqliterc ? sqlite3_fopen(sqliterc,"rb") : 0; if( p->in ){ if( stdin_is_interactive ){ - sqlite3_fprintf(stderr,"-- Loading resources from %s\n", sqliterc); + cli_printf(stderr,"-- Loading resources from %s\n", sqliterc); } - if( process_input(p, sqliterc) && bail_on_error ) exit(1); + if( process_input(p, sqliterc) && bail_on_error ) cli_exit(1); fclose(p->in); }else if( sqliterc_override!=0 ){ - sqlite3_fprintf(stderr,"cannot open: \"%s\"\n", sqliterc); - if( bail_on_error ) exit(1); + cli_printf(stderr,"cannot open: \"%s\"\n", sqliterc); + if( bail_on_error ) cli_exit(1); } p->in = inSaved; p->lineno = savedLineno; @@ -13092,8 +12726,8 @@ static const char zOptions[] = " -bail stop after hitting an error\n" " -batch force batch I/O\n" " -box set output mode to 'box'\n" - " -column set output mode to 'column'\n" " -cmd COMMAND run \"COMMAND\" before reading stdin\n" + " -column set output mode to 'column'\n" " -csv set output mode to 'csv'\n" #if !defined(SQLITE_OMIT_DESERIALIZE) " -deserialize open the database using sqlite3_deserialize()\n" @@ -13124,6 +12758,7 @@ static const char zOptions[] = #endif " -newline SEP set output row separator. Default: '\\n'\n" " -nofollow refuse to open symbolic links to database files\n" + " -noinit Do not read the ~/.sqliterc file at startup\n" " -nonce STRING set the safe-mode escape nonce\n" " -no-rowid-in-view Disable rowid-in-view using sqlite3_config()\n" " -nullvalue TEXT set text string for NULL values. Default ''\n" @@ -13132,6 +12767,7 @@ static const char zOptions[] = " -quote set output mode to 'quote'\n" " -readonly open the database read-only\n" " -safe enable safe-mode\n" + " -screenwidth N use N as the default screenwidth \n" " -separator SEP set output column separator. Default: '|'\n" #ifdef SQLITE_ENABLE_SORTER_REFERENCES " -sorterref SIZE sorter references threshold size\n" @@ -13148,11 +12784,11 @@ static const char zOptions[] = #endif ; static void usage(int showDetail){ - sqlite3_fprintf(stderr,"Usage: %s [OPTIONS] [FILENAME [SQL...]]\n" + cli_printf(stderr,"Usage: %s [OPTIONS] [FILENAME [SQL...]]\n" "FILENAME is the name of an SQLite database. A new database is created\n" "if the file does not previously exist. Defaults to :memory:.\n", Argv0); if( showDetail ){ - sqlite3_fprintf(stderr,"OPTIONS include:\n%s", zOptions); + cli_printf(stderr,"OPTIONS include:\n%s", zOptions); }else{ eputz("Use the -help option for additional information\n"); } @@ -13173,19 +12809,11 @@ static void verify_uninitialized(void){ /* ** Initialize the state information in data */ -static void main_init(ShellState *data) { - memset(data, 0, sizeof(*data)); - data->normalMode = data->cMode = data->mode = MODE_List; - data->autoExplain = 1; -#ifdef _WIN32 - data->crlfMode = 1; -#endif - data->pAuxDb = &data->aAuxDb[0]; - memcpy(data->colSeparator,SEP_Column, 2); - memcpy(data->rowSeparator,SEP_Row, 2); - data->showHeader = 0; - data->shellFlgs = SHFLG_Lookaside; - sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); +static void main_init(ShellState *p) { + memset(p, 0, sizeof(*p)); + p->pAuxDb = &p->aAuxDb[0]; + p->shellFlgs = SHFLG_Lookaside; + sqlite3_config(SQLITE_CONFIG_LOG, shellLog, p); #if !defined(SQLITE_SHELL_FIDDLE) verify_uninitialized(); #endif @@ -13200,22 +12828,18 @@ static void main_init(ShellState *data) { */ #if defined(_WIN32) || defined(WIN32) static void printBold(const char *zText){ -#if !SQLITE_OS_WINRT HANDLE out = GetStdHandle(STD_OUTPUT_HANDLE); CONSOLE_SCREEN_BUFFER_INFO defaultScreenInfo; GetConsoleScreenBufferInfo(out, &defaultScreenInfo); SetConsoleTextAttribute(out, FOREGROUND_RED|FOREGROUND_INTENSITY ); -#endif sputz(stdout, zText); -#if !SQLITE_OS_WINRT SetConsoleTextAttribute(out, defaultScreenInfo.wAttributes); -#endif } #else static void printBold(const char *zText){ - sqlite3_fprintf(stdout, "\033[1m%s\033[0m", zText); + cli_printf(stdout, "\033[1m%s\033[0m", zText); } #endif @@ -13225,9 +12849,9 @@ static void printBold(const char *zText){ */ static char *cmdline_option_value(int argc, char **argv, int i){ if( i==argc ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "%s: Error: missing argument to %s\n", argv[0], argv[argc-1]); - exit(1); + cli_exit(1); } return argv[i]; } @@ -13240,34 +12864,85 @@ static void sayAbnormalExit(void){ */ static int vfstraceOut(const char *z, void *pArg){ ShellState *p = (ShellState*)pArg; - sqlite3_fputs(z, p->out); + cli_puts(z, p->out); fflush(p->out); return 1; } -#ifndef SQLITE_SHELL_IS_UTF8 -# if (defined(_WIN32) || defined(WIN32)) \ - && (defined(_MSC_VER) || (defined(UNICODE) && defined(__GNUC__))) -# define SQLITE_SHELL_IS_UTF8 (0) -# else -# define SQLITE_SHELL_IS_UTF8 (1) -# endif +#if defined(SQLITE_DEBUG) && !defined(SQLITE_SHELL_FIDDLE) +/* Ensure that sqlite3_reset_auto_extension() clears auto-extension +** memory. https://sqlite.org/forum/forumpost/310cb231b07c80d1. +** Testing this: if we (inadvertently) remove the +** sqlite3_reset_auto_extension() call from main(), most shell tests +** will fail because of a leak message in their output. */ +static int auto_ext_leak_tester( + sqlite3 *db, + char **pzErrMsg, + const struct sqlite3_api_routines *pThunk +){ + (void)db; (void)pzErrMsg; (void)pThunk; + return SQLITE_OK; +} #endif +/* Alternative name to the entry point for Fiddle */ #ifdef SQLITE_SHELL_FIDDLE # define main fiddle_main #endif -#if SQLITE_SHELL_IS_UTF8 -int SQLITE_CDECL main(int argc, char **argv){ -#else +/* Use the wmain() entry point on Windows. Translate arguments to +** UTF8, then invoke the traditional main() entry point which is +** renamed using a #define to utf8_main() . +*/ +#if defined(_WIN32) && !defined(__MINGW32__) && !defined(main) +# define main utf8_main /* Rename entry point to utf_main() */ +int SQLITE_CDECL utf8_main(int,char**); /* Forward declaration */ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ - char **argv; -#endif + int rc, i; + char **argv = malloc( sizeof(char*) * (argc+1) ); + char **orig = argv; + if( argv==0 ){ + fprintf(stderr, "malloc failed\n"); + exit(1); + } + for(i=0; i<argc; i++){ + int nByte = WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, 0, 0, 0, 0); + if( nByte==0 ){ + argv[i] = 0; + }else{ + argv[i] = malloc( nByte ); + if( argv[i]==0 ){ + fprintf(stderr, "malloc failed\n"); + exit(1); + } + nByte = WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, argv[i],nByte,0,0); + if( nByte==0 ){ + free(argv[i]); + argv[i] = 0; + } + } + } + argv[argc] = 0; + rc = utf8_main(argc, argv); + for(i=0; i<argc; i++) free(orig[i]); + free(argv); + return rc; +} +#endif /* WIN32 */ + +/* +** This is the main entry point for the process. Everything starts here. +** +** The "main" identifier may have been #defined to something else: +** +** utf8_main On Windows +** fiddle_main In Fiddle +** sqlite3_shell Other projects that use shell.c as a subroutine +*/ +int SQLITE_CDECL main(int argc, char **argv){ #ifdef SQLITE_DEBUG sqlite3_int64 mem_main_enter = 0; #endif - char *zErrMsg = 0; #ifdef SQLITE_SHELL_FIDDLE # define data shellState #else @@ -13278,15 +12953,13 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ int rc = 0; int warnInmemoryDb = 0; int readStdin = 1; + int noInit = 0; /* Do not read ~/.sqliterc if true */ int nCmd = 0; int nOptsEnd = argc; int bEnableVfstrace = 0; char **azCmd = 0; + int *aiCmd = 0; const char *zVfs = 0; /* Value of -vfs command-line option */ -#if !SQLITE_SHELL_IS_UTF8 - char **argvToFree = 0; - int argcToFree = 0; -#endif setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */ #ifdef SQLITE_SHELL_FIDDLE @@ -13305,7 +12978,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( getenv("SQLITE_DEBUG_BREAK") ){ if( isatty(0) && isatty(2) ){ char zLine[100]; - sqlite3_fprintf(stderr, + cli_printf(stderr, "attach debugger to process %d and press ENTER to continue...", GETPID()); if( sqlite3_fgets(zLine, sizeof(zLine), stdin)!=0 @@ -13315,11 +12988,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } }else{ #if defined(_WIN32) || defined(WIN32) -#if SQLITE_OS_WINRT - __debugbreak(); -#else DebugBreak(); -#endif #elif defined(SIGTRAP) raise(SIGTRAP); #endif @@ -13337,7 +13006,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #if USE_SYSTEM_SQLITE+0!=1 if( cli_strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){ - sqlite3_fprintf(stderr, + cli_printf(stderr, "SQLite header and source version mismatch\n%s\n%s\n", sqlite3_sourceid(), SQLITE_SOURCE_ID); exit(1); @@ -13345,32 +13014,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #endif main_init(&data); - /* On Windows, we must translate command-line arguments into UTF-8. - ** The SQLite memory allocator subsystem has to be enabled in order to - ** do this. But we want to run an sqlite3_shutdown() afterwards so that - ** subsequent sqlite3_config() calls will work. So copy all results into - ** memory that does not come from the SQLite memory allocator. - */ -#if !SQLITE_SHELL_IS_UTF8 - sqlite3_initialize(); - argvToFree = malloc(sizeof(argv[0])*argc*2); - shell_check_oom(argvToFree); - argcToFree = argc; - argv = argvToFree + argc; - for(i=0; i<argc; i++){ - char *z = sqlite3_win32_unicode_to_utf8(wargv[i]); - i64 n; - shell_check_oom(z); - n = strlen(z); - argv[i] = malloc( n+1 ); - shell_check_oom(argv[i]); - memcpy(argv[i], z, n+1); - argvToFree[i] = argv[i]; - sqlite3_free(z); - } - sqlite3_shutdown(); -#endif - assert( argc>=1 && argv && argv[0] ); Argv0 = argv[0]; @@ -13386,7 +13029,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } #endif - /* Do an initial pass through the command-line argument to locate + /* Do an initial pass through the command-line arguments to locate ** the name of the database file, the name of the initialization file, ** the size of the alternative malloc heap, options affecting commands ** or SQL run from the command line, and the first command to execute. @@ -13398,16 +13041,20 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ char *z; z = argv[i]; if( z[0]!='-' || i>nOptsEnd ){ - if( data.aAuxDb->zDbFilename==0 ){ + if( data.aAuxDb->zDbFilename==0 && !isScriptFile(z,1) ){ data.aAuxDb->zDbFilename = z; }else{ /* Excess arguments are interpreted as SQL (or dot-commands) and ** mean that nothing is read from stdin */ readStdin = 0; + stdin_is_interactive = 0; nCmd++; azCmd = realloc(azCmd, sizeof(azCmd[0])*nCmd); shell_check_oom(azCmd); + aiCmd = realloc(aiCmd, sizeof(aiCmd[0])*nCmd); + shell_check_oom(azCmd); azCmd[nCmd-1] = z; + aiCmd[nCmd-1] = i; } continue; } @@ -13430,6 +13077,15 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ ** we do the actual processing of arguments later in a second pass. */ stdin_is_interactive = 0; + stdout_is_console = 0; + modeChange(&data, MODE_BATCH); + }else if( cli_strcmp(z,"-screenwidth")==0 ){ + int n = atoi(cmdline_option_value(argc, argv, ++i)); + if( n<2 ){ + sqlite3_fprintf(stderr,"minimum --screenwidth is 2\n"); + exit(1); + } + stdout_tty_width = n; }else if( cli_strcmp(z,"-utf8")==0 ){ }else if( cli_strcmp(z,"-no-utf8")==0 ){ }else if( cli_strcmp(z,"-no-rowid-in-view")==0 ){ @@ -13463,7 +13119,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ int szHdr = 0; sqlite3_config(SQLITE_CONFIG_PCACHE_HDRSZ, &szHdr); sz += szHdr; - sqlite3_fprintf(stdout, "Page cache size increased to %d to accommodate" + cli_printf(stdout, "Page cache size increased to %d to accommodate" " the %d-byte headers\n", (int)sz, szHdr); } verify_uninitialized(); @@ -13524,6 +13180,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ data.openFlags |= SQLITE_OPEN_READONLY; }else if( cli_strcmp(z,"-nofollow")==0 ){ data.openFlags |= SQLITE_OPEN_NOFOLLOW; + }else if( cli_strcmp(z,"-noinit")==0 ){ + noInit = 1; }else if( cli_strcmp(z,"-exclusive")==0 ){ /* UNDOCUMENTED */ data.openFlags |= SQLITE_OPEN_EXCLUSIVE; }else if( cli_strcmp(z,"-ifexists")==0 ){ @@ -13551,6 +13209,16 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( cli_strcmp(z,"-escape")==0 && i+1<argc ){ /* skip over the argument */ i++; + }else if( cli_strcmp(z,"-test-argv")==0 ){ + /* Undocumented test option. Print the values in argv[] and exit. + ** Use this to verify that any translation of the argv[], for example + ** on Windows that receives wargv[] from the OS and must convert + ** to UTF8 prior to calling this routine. */ + int kk; + for(kk=0; kk<argc; kk++){ + sqlite3_fprintf(stdout,"argv[%d] = \"%s\"\n", kk, argv[kk]); + } + return 0; } } #ifndef SQLITE_SHELL_FIDDLE @@ -13577,8 +13245,27 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ sqlite3_vfs *pVfs = sqlite3_vfs_find(zVfs); if( pVfs ){ sqlite3_vfs_register(pVfs, 1); - }else{ - sqlite3_fprintf(stderr,"no such VFS: \"%s\"\n", zVfs); + } +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) + else if( access(zVfs,0)==0 ){ + /* If the VFS name is not the name of an existing VFS, but it is + ** the name of a file, then try to load that file as an extension. + ** Presumably the extension implements the desired VFS. */ + sqlite3 *db = 0; + char *zErr = 0; + sqlite3_open(":memory:", &db); + sqlite3_enable_load_extension(db, 1); + rc = sqlite3_load_extension(db, zVfs, 0, &zErr); + sqlite3_close(db); + if( (rc&0xff)!=SQLITE_OK ){ + cli_printf(stderr, "could not load extension VFS \"%s\": %s\n", + zVfs, zErr); + exit(1); + } + } +#endif + else{ + cli_printf(stderr,"no such VFS: \"%s\"\n", zVfs); exit(1); } } @@ -13588,9 +13275,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ data.pAuxDb->zDbFilename = ":memory:"; warnInmemoryDb = argc==1; #else - sqlite3_fprintf(stderr, + cli_printf(stderr, "%s: Error: no database filename specified\n", Argv0); - return 1; + rc = 1; + goto shell_main_exit; #endif } data.out = stdout; @@ -13599,7 +13287,11 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } #ifndef SQLITE_SHELL_FIDDLE sqlite3_appendvfs_init(0,0,0); +#ifdef SQLITE_DEBUG + sqlite3_auto_extension( (void (*)())auto_ext_leak_tester ); +#endif #endif + modeDefault(&data); /* Go ahead and open the database file if it already exists. If the ** file does not exist, delay opening it. This prevents empty database @@ -13614,9 +13306,9 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ ** is given on the command line, look for a file named ~/.sqliterc and ** try to process it. */ - process_sqliterc(&data,zInitFile); + if( !noInit ) process_sqliterc(&data,zInitFile); - /* Make a second pass through the command-line argument and set + /* Make a second pass through the command-line arguments and set ** options. This second pass is delayed until after the initialization ** file is processed so that the command-line arguments will override ** settings in the initialization file. @@ -13628,45 +13320,44 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( cli_strcmp(z,"-init")==0 ){ i++; }else if( cli_strcmp(z,"-html")==0 ){ - data.mode = MODE_Html; + modeChange(&data, MODE_Html); }else if( cli_strcmp(z,"-list")==0 ){ - data.mode = MODE_List; + modeChange(&data, MODE_List); }else if( cli_strcmp(z,"-quote")==0 ){ - data.mode = MODE_Quote; - sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator, SEP_Comma); - sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator, SEP_Row); + modeChange(&data, MODE_Quote); }else if( cli_strcmp(z,"-line")==0 ){ - data.mode = MODE_Line; + modeChange(&data, MODE_Line); }else if( cli_strcmp(z,"-column")==0 ){ - data.mode = MODE_Column; + modeChange(&data, MODE_Column); }else if( cli_strcmp(z,"-json")==0 ){ - data.mode = MODE_Json; + modeChange(&data, MODE_Json); }else if( cli_strcmp(z,"-markdown")==0 ){ - data.mode = MODE_Markdown; + modeChange(&data, MODE_Markdown); }else if( cli_strcmp(z,"-table")==0 ){ - data.mode = MODE_Table; + modeChange(&data, MODE_Table); + }else if( cli_strcmp(z,"-psql")==0 ){ + modeChange(&data, MODE_Psql); }else if( cli_strcmp(z,"-box")==0 ){ - data.mode = MODE_Box; + modeChange(&data, MODE_Box); }else if( cli_strcmp(z,"-csv")==0 ){ - data.mode = MODE_Csv; - memcpy(data.colSeparator,",",2); + modeChange(&data, MODE_Csv); }else if( cli_strcmp(z,"-escape")==0 && i+1<argc ){ /* See similar code at tag-20250224-1 */ const char *zEsc = argv[++i]; int k; - for(k=0; k<ArraySize(shell_EscModeNames); k++){ - if( sqlite3_stricmp(zEsc,shell_EscModeNames[k])==0 ){ - data.eEscMode = k; + for(k=0; k<ArraySize(qrfEscNames); k++){ + if( sqlite3_stricmp(zEsc,qrfEscNames[k])==0 ){ + data.mode.spec.eEsc = k; break; } } - if( k>=ArraySize(shell_EscModeNames) ){ - sqlite3_fprintf(stderr, "unknown control character escape mode \"%s\"" + if( k>=ArraySize(qrfEscNames) ){ + cli_printf(stderr, "unknown control character escape mode \"%s\"" " - choices:", zEsc); - for(k=0; k<ArraySize(shell_EscModeNames); k++){ - sqlite3_fprintf(stderr, " %s", shell_EscModeNames[k]); + for(k=0; k<ArraySize(qrfEscNames); k++){ + cli_printf(stderr, " %s", qrfEscNames[k]); } - sqlite3_fprintf(stderr, "\n"); + cli_printf(stderr, "\n"); exit(1); } #ifdef SQLITE_HAVE_ZLIB @@ -13686,44 +13377,40 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ data.openFlags |= SQLITE_OPEN_READONLY; }else if( cli_strcmp(z,"-nofollow")==0 ){ data.openFlags |= SQLITE_OPEN_NOFOLLOW; + }else if( cli_strcmp(z,"-noinit")==0 ){ + /* No-op */ }else if( cli_strcmp(z,"-exclusive")==0 ){ /* UNDOCUMENTED */ data.openFlags |= SQLITE_OPEN_EXCLUSIVE; }else if( cli_strcmp(z,"-ifexists")==0 ){ data.openFlags &= ~(SQLITE_OPEN_CREATE); if( data.openFlags==0 ) data.openFlags = SQLITE_OPEN_READWRITE; }else if( cli_strcmp(z,"-ascii")==0 ){ - data.mode = MODE_Ascii; - sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,SEP_Unit); - sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator,SEP_Record); + modeChange(&data, MODE_Ascii); }else if( cli_strcmp(z,"-tabs")==0 ){ - data.mode = MODE_List; - sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator,SEP_Tab); - sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator,SEP_Row); + modeChange(&data, MODE_Tabs); }else if( cli_strcmp(z,"-separator")==0 ){ - sqlite3_snprintf(sizeof(data.colSeparator), data.colSeparator, - "%s",cmdline_option_value(argc,argv,++i)); + modeSetStr(&data.mode.spec.zColumnSep, + cmdline_option_value(argc,argv,++i)); }else if( cli_strcmp(z,"-newline")==0 ){ - sqlite3_snprintf(sizeof(data.rowSeparator), data.rowSeparator, - "%s",cmdline_option_value(argc,argv,++i)); + modeSetStr(&data.mode.spec.zRowSep, + cmdline_option_value(argc,argv,++i)); }else if( cli_strcmp(z,"-nullvalue")==0 ){ - sqlite3_snprintf(sizeof(data.nullValue), data.nullValue, - "%s",cmdline_option_value(argc,argv,++i)); + modeSetStr(&data.mode.spec.zNull, + cmdline_option_value(argc,argv,++i)); }else if( cli_strcmp(z,"-header")==0 ){ - data.showHeader = 1; - ShellSetFlag(&data, SHFLG_HeaderSet); + data.mode.spec.bTitles = QRF_Yes; }else if( cli_strcmp(z,"-noheader")==0 ){ - data.showHeader = 0; - ShellSetFlag(&data, SHFLG_HeaderSet); + data.mode.spec.bTitles = QRF_No; }else if( cli_strcmp(z,"-echo")==0 ){ - ShellSetFlag(&data, SHFLG_Echo); + data.mode.mFlags |= MFLG_ECHO; }else if( cli_strcmp(z,"-eqp")==0 ){ - data.autoEQP = AUTOEQP_on; + data.mode.autoEQP = AUTOEQP_on; }else if( cli_strcmp(z,"-eqpfull")==0 ){ - data.autoEQP = AUTOEQP_full; + data.mode.autoEQP = AUTOEQP_full; }else if( cli_strcmp(z,"-stats")==0 ){ data.statsOn = 1; }else if( cli_strcmp(z,"-scanstats")==0 ){ - data.scanstatsOn = 1; + data.mode.scanstatsOn = 1; }else if( cli_strcmp(z,"-backslash")==0 ){ /* Undocumented command-line option: -backslash ** Causes C-style backslash escapes to be evaluated in SQL statements @@ -13734,9 +13421,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( cli_strcmp(z,"-bail")==0 ){ /* No-op. The bail_on_error flag should already be set. */ }else if( cli_strcmp(z,"-version")==0 ){ - sqlite3_fprintf(stdout, "%s %s (%d-bit)\n", + cli_printf(stdout, "%s %s (%d-bit)\n", sqlite3_libversion(), sqlite3_sourceid(), 8*(int)sizeof(char*)); - return 0; + rc = 0; + goto shell_main_exit; }else if( cli_strcmp(z,"-interactive")==0 ){ /* Need to check for interactive override here to so that it can ** affect console setup (for Windows only) and testing thereof. @@ -13744,6 +13432,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ stdin_is_interactive = 1; }else if( cli_strcmp(z,"-batch")==0 ){ /* already handled */ + }else if( cli_strcmp(z,"-screenwidth")==0 ){ + i++; }else if( cli_strcmp(z,"-utf8")==0 ){ /* already handled */ }else if( cli_strcmp(z,"-no-utf8")==0 ){ @@ -13789,23 +13479,18 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ z = cmdline_option_value(argc,argv,++i); if( z[0]=='.' ){ rc = do_meta_command(z, &data); - if( rc && bail_on_error ) return rc==2 ? 0 : rc; - }else{ - open_db(&data, 0); - rc = shell_exec(&data, z, &zErrMsg); - if( zErrMsg!=0 ){ - shellEmitError(zErrMsg); - sqlite3_free(zErrMsg); - if( bail_on_error ) return rc!=0 ? rc : 1; - }else if( rc!=0 ){ - sqlite3_fprintf(stderr,"Error: unable to process SQL \"%s\"\n", z); - if( bail_on_error ) return rc; + if( rc && (bail_on_error || rc==2) ){ + if( rc==2 ) rc = 0; + goto shell_main_exit; } + }else{ + rc = runOneSqlLine(&data, z, "cmdline", i); + if( bail_on_error ) goto shell_main_exit; } #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) }else if( cli_strncmp(z, "-A", 2)==0 ){ if( nCmd>0 ){ - sqlite3_fprintf(stderr,"Error: cannot mix regular SQL or dot-commands" + cli_printf(stderr,"Error: cannot mix regular SQL or dot-commands" " with \"%s\"\n", z); rc = 1; goto shell_main_exit; @@ -13818,6 +13503,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ arDotCommand(&data, 1, argv+i, argc-i); } readStdin = 0; + stdin_is_interactive = 0; break; #endif }else if( cli_strcmp(z,"-safe")==0 ){ @@ -13825,11 +13511,11 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( cli_strcmp(z,"-unsafe-testing")==0 ){ /* Acted upon in first pass. */ }else{ - sqlite3_fprintf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); + cli_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); eputz("Use -help for a list of options.\n"); - return 1; + rc = 1; + goto shell_main_exit; } - data.cMode = data.mode; } if( !readStdin ){ @@ -13839,27 +13525,43 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ */ for(i=0; i<nCmd; i++){ echo_group_input(&data, azCmd[i]); - if( azCmd[i][0]=='.' ){ + if( isScriptFile(azCmd[i],0) ){ + FILE *inSaved = data.in; + i64 savedLineno = data.lineno; + int res = 1; + if( (data.in = openChrSource(azCmd[i]))!=0 ){ + res = process_input(&data, azCmd[i]); + fclose(data.in); + } + data.in = inSaved; + data.lineno = savedLineno; + if( res ) i = nCmd; + }else if( azCmd[i][0]=='.' ){ + char *zErrCtx = malloc( 64 ); + shell_check_oom(zErrCtx); + sqlite3_snprintf(64,zErrCtx,"argv[%i]:",aiCmd[i]); + data.zInFile = "<cmdline>"; + data.zErrPrefix = zErrCtx; rc = do_meta_command(azCmd[i], &data); + free(data.zErrPrefix); + data.zErrPrefix = 0; if( rc ){ if( rc==2 ) rc = 0; goto shell_main_exit; } }else{ - open_db(&data, 0); - rc = shell_exec(&data, azCmd[i], &zErrMsg); - if( zErrMsg || rc ){ - if( zErrMsg!=0 ){ - shellEmitError(zErrMsg); - }else{ - sqlite3_fprintf(stderr, - "Error: unable to process SQL: %s\n", azCmd[i]); - } - sqlite3_free(zErrMsg); - if( rc==0 ) rc = 1; - goto shell_main_exit; + rc = runOneSqlLine(&data, azCmd[i], "cmdline", aiCmd[i]); + if( data.nPopMode ){ + modePop(&data); + data.nPopMode = 0; } } + if( data.nPopOutput && azCmd[i][0]!='.' ){ + output_reset(&data); + data.nPopOutput = 0; + }else{ + clearTempFile(&data); + } } }else{ /* Run commands received from standard input @@ -13867,7 +13569,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( stdin_is_interactive ){ char *zHome; char *zHistory; - sqlite3_fprintf(stdout, + cli_printf(stdout, "SQLite version %s %.19s\n" /*extra-version-info*/ "Enter \".help\" for usage hints.\n", sqlite3_libversion(), sqlite3_sourceid()); @@ -13920,6 +13622,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #endif shell_main_exit: free(azCmd); + free(aiCmd); set_table_name(&data, 0); if( data.db ){ session_close_all(&data, -1); @@ -13936,21 +13639,38 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ output_reset(&data); data.doXdgOpen = 0; clearTempFile(&data); -#if !SQLITE_SHELL_IS_UTF8 - for(i=0; i<argcToFree; i++) free(argvToFree[i]); - free(argvToFree); -#endif - free(data.colWidth); + modeFree(&data.mode); + if( data.nSavedModes ){ + int ii; + for(ii=0; ii<data.nSavedModes; ii++){ + modeFree(&data.aSavedModes[ii].mode); + free(data.aSavedModes[ii].zTag); + } + free(data.aSavedModes); + } + free(data.zErrPrefix); free(data.zNonce); + free(data.dot.zCopy); + free(data.dot.azArg); + free(data.dot.aiOfst); + free(data.dot.abQuot); + if( data.nTestRun ){ + sqlite3_fprintf(stdout, "%d test%s run with %d error%s\n", + data.nTestRun, data.nTestRun==1 ? "" : "s", + data.nTestErr, data.nTestErr==1 ? "" : "s"); + fflush(stdout); + rc = data.nTestErr>0; + } /* Clear the global data structure so that valgrind will detect memory ** leaks */ memset(&data, 0, sizeof(data)); if( bEnableVfstrace ){ vfstrace_unregister("trace"); } + sqlite3_reset_auto_extension(); #ifdef SQLITE_DEBUG if( sqlite3_memory_used()>mem_main_enter ){ - sqlite3_fprintf(stderr,"Memory leaked: %u bytes\n", + cli_printf(stderr,"Memory leaked: %u bytes\n", (unsigned int)(sqlite3_memory_used()-mem_main_enter)); } #endif @@ -13990,7 +13710,7 @@ sqlite3_vfs * fiddle_db_vfs(const char *zDbName){ /* Only for emcc experimentation purposes. */ sqlite3 * fiddle_db_arg(sqlite3 *arg){ - sqlite3_fprintf(stdout, "fiddle_db_arg(%p)\n", (const void*)arg); + cli_printf(stdout, "fiddle_db_arg(%p)\n", (const void*)arg); return arg; } @@ -14027,7 +13747,7 @@ void fiddle_reset_db(void){ ** Resolve problem reported in ** https://sqlite.org/forum/forumpost/0b41a25d65 */ - sqlite3_fputs("Rolling back in-progress transaction.\n", stdout); + cli_puts("Rolling back in-progress transaction.\n", stdout); sqlite3_exec(globalDb,"ROLLBACK", 0, 0, 0); } rc = sqlite3_db_config(globalDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index f6ed48c20..9323ce5ee 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -578,7 +578,7 @@ int sqlite3_exec( #define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) #define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) #define SQLITE_OK_LOAD_PERMANENTLY (SQLITE_OK | (1<<8)) -#define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) /* internal use only */ +#define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) /* internal only */ /* ** CAPI3REF: Flags For File Open Operations @@ -1290,6 +1290,12 @@ struct sqlite3_io_methods { #define SQLITE_SET_LOCKPROXYFILE SQLITE_FCNTL_SET_LOCKPROXYFILE #define SQLITE_LAST_ERRNO SQLITE_FCNTL_LAST_ERRNO +/* reserved file-control numbers: +** 101 +** 102 +** 103 +*/ + /* ** CAPI3REF: Mutex Handle @@ -1490,7 +1496,7 @@ typedef const char *sqlite3_filename; ** greater and the function pointer is not NULL) and will fall back ** to xCurrentTime() if xCurrentTimeInt64() is unavailable. ** -** ^The xSetSystemCall(), xGetSystemCall(), and xNestSystemCall() interfaces +** ^The xSetSystemCall(), xGetSystemCall(), and xNextSystemCall() interfaces ** are not used by the SQLite core. These optional interfaces are provided ** by some VFSes to facilitate testing of the VFS code. By overriding ** system calls with functions under its control, a test program can @@ -1711,7 +1717,8 @@ int sqlite3_os_end(void); ** are called "anytime configuration options". ** ^If sqlite3_config() is called after [sqlite3_initialize()] and before ** [sqlite3_shutdown()] with a first argument that is not an anytime -** configuration option, then the sqlite3_config() call will return SQLITE_MISUSE. +** configuration option, then the sqlite3_config() call will +** return SQLITE_MISUSE. ** Note, however, that ^sqlite3_config() can be called as part of the ** implementation of an application-defined [sqlite3_os_init()]. ** @@ -2277,9 +2284,10 @@ struct sqlite3_mem_methods { ** is less than 8. The "sz" argument should be a multiple of 8 less than ** 65536. If "sz" does not meet this constraint, it is reduced in size until ** it does. -** <li><p>The third argument ("cnt") is the number of slots. Lookaside is disabled -** if "cnt"is less than 1. The "cnt" value will be reduced, if necessary, so -** that the product of "sz" and "cnt" does not exceed 2,147,418,112. The "cnt" +** <li><p>The third argument ("cnt") is the number of slots. +** Lookaside is disabled if "cnt"is less than 1. +* The "cnt" value will be reduced, if necessary, so +** that the product of "sz" and "cnt" does not exceed 2,147,418,112. The "cnt" ** parameter is usually chosen so that the product of "sz" and "cnt" is less ** than 1,000,000. ** </ol> @@ -2567,12 +2575,15 @@ struct sqlite3_mem_methods { ** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]] ** <dt>SQLITE_DBCONFIG_STMT_SCANSTATUS</dt> ** <dd>The SQLITE_DBCONFIG_STMT_SCANSTATUS option is only useful in -** SQLITE_ENABLE_STMT_SCANSTATUS builds. In this case, it sets or clears -** a flag that enables collection of the sqlite3_stmt_scanstatus_v2() -** statistics. For statistics to be collected, the flag must be set on -** the database handle both when the SQL statement is prepared and when it -** is stepped. The flag is set (collection of statistics is enabled) -** by default. <p>This option takes two arguments: an integer and a pointer to +** [SQLITE_ENABLE_STMT_SCANSTATUS] builds. In this case, it sets or clears +** a flag that enables collection of run-time performance statistics +** used by [sqlite3_stmt_scanstatus_v2()] and the [nexec and ncycle] +** columns of the [bytecode virtual table]. +** For statistics to be collected, the flag must be set on +** the database handle both when the SQL statement is +** [sqlite3_prepare|prepared] and when it is [sqlite3_step|stepped]. +** The flag is set (collection of statistics is enabled) by default. +** <p>This option takes two arguments: an integer and a pointer to ** an integer. The first argument is 1, 0, or -1 to enable, disable, or ** leave unchanged the statement scanstatus option. If the second argument ** is not NULL, then the value of the statement scanstatus setting after @@ -2641,20 +2652,38 @@ struct sqlite3_mem_methods { ** to an integer. The first argument is 1, 0, or -1 to enable, disable, or ** leave unchanged the ability to use comments in SQL text, ** respectively. If the second argument is not NULL, then 0 or 1 is written -** into the integer that the second argument points to depending on if +** into the integer that the second argument points to depending on if ** comments are allowed in SQL text after processing the first argument. ** </dd> ** +** [[SQLITE_DBCONFIG_FP_DIGITS]] +** <dt>SQLITE_DBCONFIG_FP_DIGITS</dt> +** <dd>The SQLITE_DBCONFIG_FP_DIGITS setting is a small integer that determines +** the number of significant digits that SQLite will attempt to preserve when +** converting floating point numbers (IEEE 754 "doubles") into text. The +** default value 17, as of SQLite version 3.52.0. The value was 15 in all +** prior versions.<p> +** This option takes two arguments which are an integer and a pointer +** to an integer. The first argument is a small integer, between 3 and 23, or +** zero. The FP_DIGITS setting is changed to that small integer, or left +** unaltered if the first argument is zero or out of range. The second argument +** is a pointer to an integer. If the pointer is not NULL, then the value of +** the FP_DIGITS setting, after possibly being modified by the first +** arguments, is written into the integer to which the second argument points. +** </dd> +** ** </dl> ** ** [[DBCONFIG arguments]] <h3>Arguments To SQLITE_DBCONFIG Options</h3> ** ** <p>Most of the SQLITE_DBCONFIG options take two arguments, so that the ** overall call to [sqlite3_db_config()] has a total of four parameters. -** The first argument (the third parameter to sqlite3_db_config()) is an integer. -** The second argument is a pointer to an integer. If the first argument is 1, -** then the option becomes enabled. If the first integer argument is 0, then the -** option is disabled. If the first argument is -1, then the option setting +** The first argument (the third parameter to sqlite3_db_config()) is +** an integer. +** The second argument is a pointer to an integer. If the first argument is 1, +** then the option becomes enabled. If the first integer argument is 0, +** then the option is disabled. +** If the first argument is -1, then the option setting ** is unchanged. The second argument, the pointer to an integer, may be NULL. ** If the second argument is not NULL, then a value of 0 or 1 is written into ** the integer to which the second argument points, depending on whether the @@ -2662,9 +2691,10 @@ struct sqlite3_mem_methods { ** the first argument. ** ** <p>While most SQLITE_DBCONFIG options use the argument format -** described in the previous paragraph, the [SQLITE_DBCONFIG_MAINDBNAME] -** and [SQLITE_DBCONFIG_LOOKASIDE] options are different. See the -** documentation of those exceptional options for details. +** described in the previous paragraph, the [SQLITE_DBCONFIG_MAINDBNAME], +** [SQLITE_DBCONFIG_LOOKASIDE], and [SQLITE_DBCONFIG_FP_DIGITS] options +** are different. See the documentation of those exceptional options for +** details. */ #define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ #define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ @@ -2689,7 +2719,8 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE 1020 /* int int* */ #define SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE 1021 /* int int* */ #define SQLITE_DBCONFIG_ENABLE_COMMENTS 1022 /* int int* */ -#define SQLITE_DBCONFIG_MAX 1022 /* Largest DBCONFIG */ +#define SQLITE_DBCONFIG_FP_DIGITS 1023 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1023 /* Largest DBCONFIG */ /* ** CAPI3REF: Enable Or Disable Extended Result Codes @@ -4171,6 +4202,7 @@ void sqlite3_free_filename(sqlite3_filename); ** <li> sqlite3_errmsg() ** <li> sqlite3_errmsg16() ** <li> sqlite3_error_offset() +** <li> sqlite3_db_handle() ** </ul> ** ** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language @@ -4217,7 +4249,7 @@ const char *sqlite3_errstr(int); int sqlite3_error_offset(sqlite3 *db); /* -** CAPI3REF: Set Error Codes And Message +** CAPI3REF: Set Error Code And Message ** METHOD: sqlite3 ** ** Set the error code of the database handle passed as the first argument @@ -4336,6 +4368,10 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** [[SQLITE_LIMIT_EXPR_DEPTH]] ^(<dt>SQLITE_LIMIT_EXPR_DEPTH</dt> ** <dd>The maximum depth of the parse tree on any expression.</dd>)^ ** +** [[SQLITE_LIMIT_PARSER_DEPTH]] ^(<dt>SQLITE_LIMIT_PARSER_DEPTH</dt> +** <dd>The maximum depth of the LALR(1) parser stack used to analyze +** input SQL statements.</dd>)^ +** ** [[SQLITE_LIMIT_COMPOUND_SELECT]] ^(<dt>SQLITE_LIMIT_COMPOUND_SELECT</dt> ** <dd>The maximum number of terms in a compound SELECT statement.</dd>)^ ** @@ -4380,6 +4416,7 @@ int sqlite3_limit(sqlite3*, int id, int newVal); #define SQLITE_LIMIT_VARIABLE_NUMBER 9 #define SQLITE_LIMIT_TRIGGER_DEPTH 10 #define SQLITE_LIMIT_WORKER_THREADS 11 +#define SQLITE_LIMIT_PARSER_DEPTH 12 /* ** CAPI3REF: Prepare Flags @@ -4424,12 +4461,29 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** fails, the sqlite3_prepare_v3() call returns the same error indications ** with or without this flag; it just omits the call to [sqlite3_log()] that ** logs the error. +** +** [[SQLITE_PREPARE_FROM_DDL]] <dt>SQLITE_PREPARE_FROM_DDL</dt> +** <dd>The SQLITE_PREPARE_FROM_DDL flag causes the SQL compiler to enforce +** security constraints that would otherwise only be enforced when parsing +** the database schema. In other words, the SQLITE_PREPARE_FROM_DDL flag +** causes the SQL compiler to treat the SQL statement being prepared as if +** it had come from an attacker. When SQLITE_PREPARE_FROM_DDL is used and +** [SQLITE_DBCONFIG_TRUSTED_SCHEMA] is off, SQL functions may only be called +** if they are tagged with [SQLITE_INNOCUOUS] and virtual tables may only +** be used if they are tagged with [SQLITE_VTAB_INNOCUOUS]. Best practice +** is to use the SQLITE_PREPARE_FROM_DDL option when preparing any SQL that +** is derived from parts of the database schema. In particular, virtual +** table implementations that run SQL statements that are derived from +** arguments to their CREATE VIRTUAL TABLE statement should always use +** [sqlite3_prepare_v3()] and set the SQLITE_PREPARE_FROM_DDL flag to +** prevent bypass of the [SQLITE_DBCONFIG_TRUSTED_SCHEMA] security checks. ** </dl> */ #define SQLITE_PREPARE_PERSISTENT 0x01 #define SQLITE_PREPARE_NORMALIZE 0x02 #define SQLITE_PREPARE_NO_VTAB 0x04 #define SQLITE_PREPARE_DONT_LOG 0x10 +#define SQLITE_PREPARE_FROM_DDL 0x20 /* ** CAPI3REF: Compiling An SQL Statement @@ -4443,8 +4497,9 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** ** The preferred routine to use is [sqlite3_prepare_v2()]. The ** [sqlite3_prepare()] interface is legacy and should be avoided. -** [sqlite3_prepare_v3()] has an extra "prepFlags" option that is used -** for special purposes. +** [sqlite3_prepare_v3()] has an extra +** [SQLITE_PREPARE_FROM_DDL|"prepFlags" option] that is sometimes +** needed for special purpose or to pass along security restrictions. ** ** The use of the UTF-8 interfaces is preferred, as SQLite currently ** does all parsing using UTF-8. The UTF-16 interfaces are provided @@ -4849,8 +4904,8 @@ typedef struct sqlite3_context sqlite3_context; ** it should be a pointer to well-formed UTF16 text. ** ^If the third parameter to sqlite3_bind_text64() is not NULL, then ** it should be a pointer to a well-formed unicode string that is -** either UTF8 if the sixth parameter is SQLITE_UTF8, or UTF16 -** otherwise. +** either UTF8 if the sixth parameter is SQLITE_UTF8 or SQLITE_UTF8_ZT, +** or UTF16 otherwise. ** ** [[byte-order determination rules]] ^The byte-order of ** UTF16 input text is determined by the byte-order mark (BOM, U+FEFF) @@ -4896,10 +4951,15 @@ typedef struct sqlite3_context sqlite3_context; ** object and pointer to it must remain valid until then. ^SQLite will then ** manage the lifetime of its private copy. ** -** ^The sixth argument to sqlite3_bind_text64() must be one of -** [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE] -** to specify the encoding of the text in the third parameter. If -** the sixth argument to sqlite3_bind_text64() is not one of the +** ^The sixth argument (the E argument) +** to sqlite3_bind_text64(S,K,Z,N,D,E) must be one of +** [SQLITE_UTF8], [SQLITE_UTF8_ZT], [SQLITE_UTF16], [SQLITE_UTF16BE], +** or [SQLITE_UTF16LE] to specify the encoding of the text in the +** third parameter, Z. The special value [SQLITE_UTF8_ZT] means that the +** string argument is both UTF-8 encoded and is zero-terminated. In other +** words, SQLITE_UTF8_ZT means that the Z array is allocated to hold at +** least N+1 bytes and that the Z&#91;N&#93; byte is zero. If +** the E argument to sqlite3_bind_text64(S,K,Z,N,D,E) is not one of the ** allowed values shown above, or if the text encoding is different ** from the encoding specified by the sixth parameter, then the behavior ** is undefined. @@ -5766,6 +5826,52 @@ int sqlite3_create_window_function( ** ** These constants define integer codes that represent the various ** text encodings supported by SQLite. +** +** <dl> +** [[SQLITE_UTF8]] <dt>SQLITE_UTF8</dt><dd>Text is encoding as UTF-8</dd> +** +** [[SQLITE_UTF16LE]] <dt>SQLITE_UTF16LE</dt><dd>Text is encoding as UTF-16 +** with each code point being expressed "little endian" - the least significant +** byte first. This is the usual encoding, for example on Windows.</dd> +** +** [[SQLITE_UTF16BE]] <dt>SQLITE_UTF16BE</dt><dd>Text is encoding as UTF-16 +** with each code point being expressed "big endian" - the most significant +** byte first. This encoding is less common, but is still sometimes seen, +** specially on older systems. +** +** [[SQLITE_UTF16]] <dt>SQLITE_UTF16</dt><dd>Text is encoding as UTF-16 +** with each code point being expressed either little endian or as big +** endian, according to the native endianness of the host computer. +** +** [[SQLITE_ANY]] <dt>SQLITE_ANY</dt><dd>This encoding value may only be used +** to declare the preferred text for [application-defined SQL functions] +** created using [sqlite3_create_function()] and similar. If the preferred +** encoding (the 4th parameter to sqlite3_create_function() - the eTextRep +** parameter) is SQLITE_ANY, that indicates that the function does not have +** a preference regarding the text encoding of its parameters and can take +** any text encoding that the SQLite core find convenient to supply. This +** option is deprecated. Please do not use it in new applications. +** +** [[SQLITE_UTF16_ALIGNED]] <dt>SQLITE_UTF16_ALIGNED</dt><dd>This encoding +** value may be used as the 3rd parameter (the eTextRep parameter) to +** [sqlite3_create_collation()] and similar. This encoding value means +** that the application-defined collating sequence created expects its +** input strings to be in UTF16 in native byte order, and that the start +** of the strings must be aligned to a 2-byte boundary. +** +** [[SQLITE_UTF8_ZT]] <dt>SQLITE_UTF8_ZT</dt><dd>This option can only be +** used to specify the text encoding to strings input to +** [sqlite3_result_text64()] and [sqlite3_bind_text64()]. +** The SQLITE_UTF8_ZT encoding means that the input string (call it "z") +** is UTF-8 encoded and that it is zero-terminated. If the length parameter +** (call it "n") is non-negative, this encoding option means that the caller +** guarantees that z array contains at least n+1 bytes and that the z&#91;n&#93; +** byte has a value of zero. +** This option gives the same output as SQLITE_UTF8, but can be more efficient +** by avoiding the need to make a copy of the input string, in some cases. +** However, if z is allocated to hold fewer than n+1 bytes or if the +** z&#91;n&#93; byte is not zero, undefined behavior may result. +** </dl> */ #define SQLITE_UTF8 1 /* IMP: R-37514-35566 */ #define SQLITE_UTF16LE 2 /* IMP: R-03371-37637 */ @@ -5773,6 +5879,7 @@ int sqlite3_create_window_function( #define SQLITE_UTF16 4 /* Use native byte order */ #define SQLITE_ANY 5 /* Deprecated */ #define SQLITE_UTF16_ALIGNED 8 /* sqlite3_create_collation only */ +#define SQLITE_UTF8_ZT 16 /* Zero-terminated UTF8 */ /* ** CAPI3REF: Function Flags @@ -6007,26 +6114,22 @@ SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), ** the SQL function that supplied the [sqlite3_value*] parameters. ** ** As long as the input parameter is correct, these routines can only -** fail if an out-of-memory error occurs during a format conversion. -** Only the following subset of interfaces are subject to out-of-memory -** errors: -** -** <ul> -** <li> sqlite3_value_blob() -** <li> sqlite3_value_text() -** <li> sqlite3_value_text16() -** <li> sqlite3_value_text16le() -** <li> sqlite3_value_text16be() -** <li> sqlite3_value_bytes() -** <li> sqlite3_value_bytes16() -** </ul> -** +** fail if an out-of-memory error occurs while trying to do a +** UTF8&rarr;UTF16 or UTF16&rarr;UTF8 conversion. ** If an out-of-memory error occurs, then the return value from these ** routines is the same as if the column had contained an SQL NULL value. -** Valid SQL NULL returns can be distinguished from out-of-memory errors -** by invoking the [sqlite3_errcode()] immediately after the suspect +** If the input sqlite3_value was not obtained from [sqlite3_value_dup()], +** then valid SQL NULL returns can also be distinguished from +** out-of-memory errors after extracting the value +** by invoking the [sqlite3_errcode()] immediately after the suspicious ** return value is obtained and before any ** other SQLite interface is called on the same [database connection]. +** If the input sqlite3_value was obtained from sqlite3_value_dup() then +** it is disconnected from the database connection and so sqlite3_errcode() +** will not work. +** In that case, the only way to distinguish an out-of-memory +** condition from a true SQL NULL is to invoke sqlite3_value_type() on the +** input to see if it is NULL prior to trying to extract the value. */ const void *sqlite3_value_blob(sqlite3_value*); double sqlite3_value_double(sqlite3_value*); @@ -6053,7 +6156,8 @@ int sqlite3_value_frombind(sqlite3_value*); ** of the value X, assuming that X has type TEXT.)^ If sqlite3_value_type(X) ** returns something other than SQLITE_TEXT, then the return value from ** sqlite3_value_encoding(X) is meaningless. ^Calls to -** [sqlite3_value_text(X)], [sqlite3_value_text16(X)], [sqlite3_value_text16be(X)], +** [sqlite3_value_text(X)], [sqlite3_value_text16(X)], +** [sqlite3_value_text16be(X)], ** [sqlite3_value_text16le(X)], [sqlite3_value_bytes(X)], or ** [sqlite3_value_bytes16(X)] might change the encoding of the value X and ** thus change the return from subsequent calls to sqlite3_value_encoding(X). @@ -6184,17 +6288,17 @@ sqlite3 *sqlite3_context_db_handle(sqlite3_context*); ** query execution, under some circumstances the associated auxiliary data ** might be preserved. An example of where this might be useful is in a ** regular-expression matching function. The compiled version of the regular -** expression can be stored as auxiliary data associated with the pattern string. -** Then as long as the pattern string remains the same, +** expression can be stored as auxiliary data associated with the pattern +** string. Then as long as the pattern string remains the same, ** the compiled regular expression can be reused on multiple ** invocations of the same function. ** -** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the auxiliary data -** associated by the sqlite3_set_auxdata(C,N,P,X) function with the Nth argument -** value to the application-defined function. ^N is zero for the left-most -** function argument. ^If there is no auxiliary data -** associated with the function argument, the sqlite3_get_auxdata(C,N) interface -** returns a NULL pointer. +** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the auxiliary +** data associated by the sqlite3_set_auxdata(C,N,P,X) function with the +** Nth argument value to the application-defined function. ^N is zero +** for the left-most function argument. ^If there is no auxiliary data +** associated with the function argument, the sqlite3_get_auxdata(C,N) +** interface returns a NULL pointer. ** ** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as auxiliary data for the ** N-th argument of the application-defined function. ^Subsequent @@ -6278,10 +6382,14 @@ void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); ** ** There is no limit (other than available memory) on the number of different ** client data pointers (with different names) that can be attached to a -** single database connection. However, the implementation is optimized -** for the case of having only one or two different client data names. -** Applications and wrapper libraries are discouraged from using more than -** one client data name each. +** single database connection. However, the current implementation stores +** the content on a linked list. Insert and retrieval performance will +** be proportional to the number of entries. The design use case, and +** the use case for which the implementation is optimized, is +** that an application will store only small number of client data names, +** typically just one or two. This interface is not intended to be a +** generalized key/value store for thousands or millions of keys. It +** will work for that, but performance might be disappointing. ** ** There is no way to enumerate the client data pointers ** associated with a database connection. The N parameter can be thought @@ -6389,10 +6497,14 @@ typedef void (*sqlite3_destructor_type)(void*); ** set the return value of the application-defined function to be ** a text string which is represented as UTF-8, UTF-16 native byte order, ** UTF-16 little endian, or UTF-16 big endian, respectively. -** ^The sqlite3_result_text64() interface sets the return value of an +** ^The sqlite3_result_text64(C,Z,N,D,E) interface sets the return value of an ** application-defined function to be a text string in an encoding -** specified by the fifth (and last) parameter, which must be one -** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]. +** specified the E parameter, which must be one +** of [SQLITE_UTF8], [SQLITE_UTF8_ZT], [SQLITE_UTF16], [SQLITE_UTF16BE], +** or [SQLITE_UTF16LE]. ^The special value [SQLITE_UTF8_ZT] means that +** the result text is both UTF-8 and zero-terminated. In other words, +** SQLITE_UTF8_ZT means that the Z array holds at least N+1 bytes and that +** the Z&#91;N&#93; is zero. ** ^SQLite takes the text result from the application from ** the 2nd parameter of the sqlite3_result_text* interfaces. ** ^If the 3rd parameter to any of the sqlite3_result_text* interfaces @@ -6479,7 +6591,7 @@ void sqlite3_result_int(sqlite3_context*, int); void sqlite3_result_int64(sqlite3_context*, sqlite3_int64); void sqlite3_result_null(sqlite3_context*); void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); -void sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64, +void sqlite3_result_text64(sqlite3_context*, const char *z, sqlite3_uint64 n, void(*)(void*), unsigned char encoding); void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); @@ -7418,7 +7530,7 @@ int sqlite3_table_column_metadata( ** ^The sqlite3_load_extension() interface attempts to load an ** [SQLite extension] library contained in the file zFile. If ** the file cannot be loaded directly, attempts are made to load -** with various operating-system specific extensions added. +** with various operating-system specific filename extensions added. ** So for example, if "samplelib" cannot be loaded, then names like ** "samplelib.so" or "samplelib.dylib" or "samplelib.dll" might ** be tried also. @@ -7426,10 +7538,10 @@ int sqlite3_table_column_metadata( ** ^The entry point is zProc. ** ^(zProc may be 0, in which case SQLite will try to come up with an ** entry point name on its own. It first tries "sqlite3_extension_init". -** If that does not work, it constructs a name "sqlite3_X_init" where -** X consists of the lower-case equivalent of all ASCII alphabetic -** characters in the filename from the last "/" to the first following -** "." and omitting any initial "lib".)^ +** If that does not work, it tries names of the form "sqlite3_X_init" +** where X consists of the lower-case equivalent of all ASCII alphabetic +** characters or all ASCII alphanumeric characters in the filename from +** the last "/" to the first following "." and omitting any initial "lib".)^ ** ^The sqlite3_load_extension() interface returns ** [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. ** ^If an error occurs and pzErrMsg is not 0, then the @@ -7503,7 +7615,7 @@ int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** <blockquote><pre> ** &nbsp; int xEntryPoint( ** &nbsp; sqlite3 *db, -** &nbsp; const char **pzErrMsg, +** &nbsp; char **pzErrMsg, ** &nbsp; const struct sqlite3_api_routines *pThunk ** &nbsp; ); ** </pre></blockquote>)^ @@ -8253,13 +8365,6 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); ** SQLITE_MUTEX_W32 implementations are appropriate for use on Unix ** and Windows. ** -** If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor -** macro defined (with "-DSQLITE_MUTEX_APPDEF=1"), then no mutex -** implementation is included with the library. In this case the -** application must supply a custom mutex implementation using the -** [SQLITE_CONFIG_MUTEX] option of the sqlite3_config() function -** before calling sqlite3_initialize() or any other public sqlite3_ -** function that calls sqlite3_initialize(). ** ** ^The sqlite3_mutex_alloc() routine allocates a new ** mutex and returns a pointer to it. ^The sqlite3_mutex_alloc() @@ -8614,6 +8719,7 @@ int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_TUNE 32 #define SQLITE_TESTCTRL_LOGEST 33 #define SQLITE_TESTCTRL_USELONGDOUBLE 34 /* NOT USED */ +#define SQLITE_TESTCTRL_ATOF 34 #define SQLITE_TESTCTRL_LAST 34 /* Largest TESTCTRL */ /* @@ -8722,17 +8828,22 @@ sqlite3_str *sqlite3_str_new(sqlite3*); ** pass the returned value to [sqlite3_free()] to avoid a memory leak. ** ^The [sqlite3_str_finish(X)] interface may return a NULL pointer if any ** errors were encountered during construction of the string. ^The -** [sqlite3_str_finish(X)] interface will also return a NULL pointer if the +** [sqlite3_str_finish(X)] interface might also return a NULL pointer if the ** string in [sqlite3_str] object X is zero bytes long. +** +** ^The [sqlite3_str_free(X)] interface destroys both the sqlite3_str object +** X and the string content it contains. Calling sqlite3_str_free(X) is +** the equivalent of calling [sqlite3_free](sqlite3_str_finish(X)). */ char *sqlite3_str_finish(sqlite3_str*); +void sqlite3_str_free(sqlite3_str*); /* ** CAPI3REF: Add Content To A Dynamic String ** METHOD: sqlite3_str ** -** These interfaces add content to an sqlite3_str object previously obtained -** from [sqlite3_str_new()]. +** These interfaces add or remove content to an sqlite3_str object +** previously obtained from [sqlite3_str_new()]. ** ** ^The [sqlite3_str_appendf(X,F,...)] and ** [sqlite3_str_vappendf(X,F,V)] interfaces uses the [built-in printf] @@ -8753,7 +8864,11 @@ char *sqlite3_str_finish(sqlite3_str*); ** ^This method can be used, for example, to add whitespace indentation. ** ** ^The [sqlite3_str_reset(X)] method resets the string under construction -** inside [sqlite3_str] object X back to zero bytes in length. +** inside [sqlite3_str] object X back to zero bytes in length. +** +** ^The [sqlite3_str_truncate(X,N)] method changes the length of the string +** under construction to be N bytes or less. This routine is a no-op if +** N is negative or if the string is already N bytes or smaller in size. ** ** These methods do not return a result code. ^If an error occurs, that fact ** is recorded in the [sqlite3_str] object and can be recovered by a @@ -8765,6 +8880,7 @@ void sqlite3_str_append(sqlite3_str*, const char *zIn, int N); void sqlite3_str_appendall(sqlite3_str*, const char *zIn); void sqlite3_str_appendchar(sqlite3_str*, int N, char C); void sqlite3_str_reset(sqlite3_str*); +void sqlite3_str_truncate(sqlite3_str*,int N); /* ** CAPI3REF: Status Of A Dynamic String @@ -10295,7 +10411,8 @@ const char *sqlite3_vtab_collation(sqlite3_index_info*,int); ** <tr> ** <td valign="top">sqlite3_vtab_distinct() return value ** <td valign="top">Rows are returned in aOrderBy order -** <td valign="top">Rows with the same value in all aOrderBy columns are adjacent +** <td valign="top">Rows with the same value in all aOrderBy columns are +** adjacent ** <td valign="top">Duplicates over all colUsed columns may be omitted ** <tr><td>0<td>yes<td>yes<td>no ** <tr><td>1<td>no<td>yes<td>no @@ -10304,8 +10421,8 @@ const char *sqlite3_vtab_collation(sqlite3_index_info*,int); ** </table> ** ** ^For the purposes of comparing virtual table output values to see if the -** values are the same value for sorting purposes, two NULL values are considered -** to be the same. In other words, the comparison operator is "IS" +** values are the same value for sorting purposes, two NULL values are +** considered to be the same. In other words, the comparison operator is "IS" ** (or "IS NOT DISTINCT FROM") and not "==". ** ** If a virtual table implementation is unable to meet the requirements @@ -10598,9 +10715,9 @@ int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); ** a variable pointed to by the "pOut" parameter. ** ** The "flags" parameter must be passed a mask of flags. At present only -** one flag is defined - SQLITE_SCANSTAT_COMPLEX. If SQLITE_SCANSTAT_COMPLEX +** one flag is defined - [SQLITE_SCANSTAT_COMPLEX]. If SQLITE_SCANSTAT_COMPLEX ** is specified, then status information is available for all elements -** of a query plan that are reported by "EXPLAIN QUERY PLAN" output. If +** of a query plan that are reported by "[EXPLAIN QUERY PLAN]" output. If ** SQLITE_SCANSTAT_COMPLEX is not specified, then only query plan elements ** that correspond to query loops (the "SCAN..." and "SEARCH..." elements of ** the EXPLAIN QUERY PLAN output) are available. Invoking API @@ -10614,7 +10731,8 @@ int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); ** elements used to implement the statement - a non-zero value is returned and ** the variable that pOut points to is unchanged. ** -** See also: [sqlite3_stmt_scanstatus_reset()] +** See also: [sqlite3_stmt_scanstatus_reset()] and the +** [nexec and ncycle] columns of the [bytecode virtual table]. */ int sqlite3_stmt_scanstatus( sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ @@ -11156,19 +11274,42 @@ int sqlite3_deserialize( /* ** CAPI3REF: Bind array values to the CARRAY table-valued function ** -** The sqlite3_carray_bind(S,I,P,N,F,X) interface binds an array value to -** one of the first argument of the [carray() table-valued function]. The -** S parameter is a pointer to the [prepared statement] that uses the carray() -** functions. I is the parameter index to be bound. P is a pointer to the -** array to be bound, and N is the number of eements in the array. The -** F argument is one of constants [SQLITE_CARRAY_INT32], [SQLITE_CARRAY_INT64], -** [SQLITE_CARRAY_DOUBLE], [SQLITE_CARRAY_TEXT], or [SQLITE_CARRAY_BLOB] to -** indicate the datatype of the array being bound. The X argument is not a -** NULL pointer, then SQLite will invoke the function X on the P parameter -** after it has finished using P, even if the call to -** sqlite3_carray_bind() fails. The special-case finalizer -** SQLITE_TRANSIENT has no effect here. -*/ +** The sqlite3_carray_bind_v2(S,I,P,N,F,X,D) interface binds an array value to +** parameter that is the first argument of the [carray() table-valued function]. +** The S parameter is a pointer to the [prepared statement] that uses the +** carray() functions. I is the parameter index to be bound. I must be the +** index of the parameter that is the first argument to the carray() +** table-valued function. P is a pointer to the array to be bound, and N +** is the number of elements in the array. The F argument is one of +** constants [SQLITE_CARRAY_INT32], [SQLITE_CARRAY_INT64], +** [SQLITE_CARRAY_DOUBLE], [SQLITE_CARRAY_TEXT], +** or [SQLITE_CARRAY_BLOB] to indicate the datatype of the array P. +** +** If the X argument is not a NULL pointer or one of the special +** values [SQLITE_STATIC] or [SQLITE_TRANSIENT], then SQLite will invoke +** the function X with argument D when it is finished using the data in P. +** The call to X(D) is a destructor for the array P. The destructor X(D) +** is invoked even if the call to sqlite3_carray_bind_v2() fails. If the X +** parameter is the special-case value [SQLITE_STATIC], then SQLite assumes +** that the data static and the destructor is never invoked. If the X +** parameter is the special-case value [SQLITE_TRANSIENT], then +** sqlite3_carray_bind_v2() makes its own private copy of the data prior +** to returning and never invokes the destructor X. +** +** The sqlite3_carray_bind() function works the same as sqlite3_carray_bind_v2() +** with a D parameter set to P. In other words, +** sqlite3_carray_bind(S,I,P,N,F,X) is same as +** sqlite3_carray_bind_v2(S,I,P,N,F,X,P). +*/ +int sqlite3_carray_bind_v2( + sqlite3_stmt *pStmt, /* Statement to be bound */ + int i, /* Parameter index */ + void *aData, /* Pointer to array data */ + int nData, /* Number of data elements */ + int mFlags, /* CARRAY flags */ + void (*xDel)(void*), /* Destructor for aData */ + void *pDel /* Optional argument to xDel() */ +); int sqlite3_carray_bind( sqlite3_stmt *pStmt, /* Statement to be bound */ int i, /* Parameter index */ diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index 5258faaed..cad1a2a00 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -371,7 +371,11 @@ struct sqlite3_api_routines { /* Version 3.51.0 and later */ int (*set_errmsg)(sqlite3*,int,const char*); int (*db_status64)(sqlite3*,int,sqlite3_int64*,sqlite3_int64*,int); - + /* Version 3.52.0 and later */ + void (*str_truncate)(sqlite3_str*,int); + void (*str_free)(sqlite3_str*); + int (*carray_bind)(sqlite3_stmt*,int,void*,int,int,void(*)(void*)); + int (*carray_bind_v2)(sqlite3_stmt*,int,void*,int,int,void(*)(void*),void*); }; /* @@ -710,6 +714,11 @@ typedef int (*sqlite3_loadext_entry)( /* Version 3.51.0 and later */ #define sqlite3_set_errmsg sqlite3_api->set_errmsg #define sqlite3_db_status64 sqlite3_api->db_status64 +/* Version 3.52.0 and later */ +#define sqlite3_str_truncate sqlite3_api->str_truncate +#define sqlite3_str_free sqlite3_api->str_free +#define sqlite3_carray_bind sqlite3_api->carray_bind +#define sqlite3_carray_bind_v2 sqlite3_api->carray_bind_v2 #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) diff --git a/src/sqliteInt.h b/src/sqliteInt.h index fa53fe694..94cc9c671 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -246,9 +246,7 @@ /* ** Include standard header files as necessary */ -#ifdef HAVE_STDINT_H #include <stdint.h> -#endif #ifdef HAVE_INTTYPES_H #include <inttypes.h> #endif @@ -661,6 +659,7 @@ # define float sqlite_int64 # define fabs(X) ((X)<0?-(X):(X)) # define sqlite3IsOverflow(X) 0 +# define INFINITY (9223372036854775807LL) # ifndef SQLITE_BIG_DBL # define SQLITE_BIG_DBL (((sqlite3_int64)1)<<50) # endif @@ -1070,6 +1069,7 @@ typedef INT16_TYPE LogEst; #else # define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&7)==0) #endif +#define TWO_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&1)==0) /* ** Disable MMAP on platforms where it is known to not work @@ -1537,7 +1537,7 @@ struct Schema { ** The number of different kinds of things that can be limited ** using the sqlite3_limit() interface. */ -#define SQLITE_N_LIMIT (SQLITE_LIMIT_WORKER_THREADS+1) +#define SQLITE_N_LIMIT (SQLITE_LIMIT_PARSER_DEPTH+1) /* ** Lookaside malloc is a set of fixed-size buffers that can be used @@ -1691,6 +1691,7 @@ struct sqlite3 { u8 noSharedCache; /* True if no shared-cache backends */ u8 nSqlExec; /* Number of pending OP_SqlExec opcodes */ u8 eOpenState; /* Current condition of the connection */ + u8 nFpDigit; /* Significant digits to keep on double->text */ int nextPagesize; /* Pagesize after VACUUM if >0 */ i64 nChange; /* Value returned by sqlite3_changes() */ i64 nTotalChange; /* Value returned by sqlite3_total_changes() */ @@ -3585,19 +3586,6 @@ struct Upsert { /* ** An instance of the following structure contains all information ** needed to generate code for a single SELECT statement. -** -** See the header comment on the computeLimitRegisters() routine for a -** detailed description of the meaning of the iLimit and iOffset fields. -** -** addrOpenEphm[] entries contain the address of OP_OpenEphemeral opcodes. -** These addresses must be stored so that we can go back and fill in -** the P4_KEYINFO and P2 parameters later. Neither the KeyInfo nor -** the number of columns in P2 can be computed at the same time -** as the OP_OpenEphm instruction is coded because not -** enough information about the compound query is known at that point. -** The KeyInfo for addrOpenTran[0] and [1] contains collating sequences -** for the result set. The KeyInfo for addrOpenEphm[2] contains collating -** sequences for the ORDER BY clause. */ struct Select { u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ @@ -3605,7 +3593,6 @@ struct Select { u32 selFlags; /* Various SF_* values */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ u32 selId; /* Unique identifier number for this SELECT */ - int addrOpenEphm[2]; /* OP_OpenEphem opcodes related to this select */ ExprList *pEList; /* The fields of the result */ SrcList *pSrc; /* The FROM clause */ Expr *pWhere; /* The WHERE clause */ @@ -3637,7 +3624,7 @@ struct Select { #define SF_Resolved 0x0000004 /* Identifiers have been resolved */ #define SF_Aggregate 0x0000008 /* Contains agg functions or a GROUP BY */ #define SF_HasAgg 0x0000010 /* Contains aggregate functions */ -#define SF_UsesEphemeral 0x0000020 /* Uses the OpenEphemeral opcode */ +#define SF_ClonedRhsIn 0x0000020 /* Cloned RHS of an IN operator */ #define SF_Expanded 0x0000040 /* sqlite3SelectExpand() called on this */ #define SF_HasTypeInfo 0x0000080 /* FROM subqueries have Table metadata */ #define SF_Compound 0x0000100 /* Part of a compound query */ @@ -3647,14 +3634,14 @@ struct Select { #define SF_MinMaxAgg 0x0001000 /* Aggregate containing min() or max() */ #define SF_Recursive 0x0002000 /* The recursive part of a recursive CTE */ #define SF_FixedLimit 0x0004000 /* nSelectRow set by a constant LIMIT */ -#define SF_MaybeConvert 0x0008000 /* Need convertCompoundSelectToSubquery() */ +/* 0x0008000 // available for reuse */ #define SF_Converted 0x0010000 /* By convertCompoundSelectToSubquery() */ #define SF_IncludeHidden 0x0020000 /* Include hidden columns in output */ #define SF_ComplexResult 0x0040000 /* Result contains subquery or function */ #define SF_WhereBegin 0x0080000 /* Really a WhereBegin() call. Debug Only */ #define SF_WinRewrite 0x0100000 /* Window function rewrite accomplished */ #define SF_View 0x0200000 /* SELECT statement is a view */ -#define SF_NoopOrderBy 0x0400000 /* ORDER BY is ignored for this query */ +/* 0x0400000 // available for reuse */ #define SF_UFSrcCheck 0x0800000 /* Check pSrc as required by UPDATE...FROM */ #define SF_PushDown 0x1000000 /* Modified by WHERE-clause push-down opt */ #define SF_MultiPart 0x2000000 /* Has multiple incompatible PARTITIONs */ @@ -3674,11 +3661,6 @@ struct Select { ** by one of the following macros. The "SRT" prefix means "SELECT Result ** Type". ** -** SRT_Union Store results as a key in a temporary index -** identified by pDest->iSDParm. -** -** SRT_Except Remove results from the temporary index pDest->iSDParm. -** ** SRT_Exists Store a 1 in memory cell pDest->iSDParm if the result ** set is not empty. ** @@ -3742,30 +3724,28 @@ struct Select { ** table. (pDest->iSDParm) is the number of key columns in ** each index record in this case. */ -#define SRT_Union 1 /* Store result as keys in an index */ -#define SRT_Except 2 /* Remove result from a UNION index */ -#define SRT_Exists 3 /* Store 1 if the result is not empty */ -#define SRT_Discard 4 /* Do not save the results anywhere */ -#define SRT_DistFifo 5 /* Like SRT_Fifo, but unique results only */ -#define SRT_DistQueue 6 /* Like SRT_Queue, but unique results only */ +#define SRT_Exists 1 /* Store 1 if the result is not empty */ +#define SRT_Discard 2 /* Do not save the results anywhere */ +#define SRT_DistFifo 3 /* Like SRT_Fifo, but unique results only */ +#define SRT_DistQueue 4 /* Like SRT_Queue, but unique results only */ /* The DISTINCT clause is ignored for all of the above. Not that ** IgnorableDistinct() implies IgnorableOrderby() */ #define IgnorableDistinct(X) ((X->eDest)<=SRT_DistQueue) -#define SRT_Queue 7 /* Store result in an queue */ -#define SRT_Fifo 8 /* Store result as data with an automatic rowid */ +#define SRT_Queue 5 /* Store result in an queue */ +#define SRT_Fifo 6 /* Store result as data with an automatic rowid */ /* The ORDER BY clause is ignored for all of the above */ #define IgnorableOrderby(X) ((X->eDest)<=SRT_Fifo) -#define SRT_Output 9 /* Output each row of result */ -#define SRT_Mem 10 /* Store result in a memory cell */ -#define SRT_Set 11 /* Store results as keys in an index */ -#define SRT_EphemTab 12 /* Create transient tab and store like SRT_Table */ -#define SRT_Coroutine 13 /* Generate a single row of result */ -#define SRT_Table 14 /* Store result as data with an automatic rowid */ -#define SRT_Upfrom 15 /* Store result as data with rowid */ +#define SRT_Output 7 /* Output each row of result */ +#define SRT_Mem 8 /* Store result in a memory cell */ +#define SRT_Set 9 /* Store results as keys in an index */ +#define SRT_EphemTab 10 /* Create transient tab and store like SRT_Table */ +#define SRT_Coroutine 11 /* Generate a single row of result */ +#define SRT_Table 12 /* Store result as data with an automatic rowid */ +#define SRT_Upfrom 13 /* Store result as data with rowid */ /* ** An instance of this object describes where to put of the results of @@ -3901,17 +3881,12 @@ struct Parse { u8 nested; /* Number of nested calls to the parser/code generator */ u8 nTempReg; /* Number of temporary registers in aTempReg[] */ u8 isMultiWrite; /* True if statement may modify/insert multiple rows */ - u8 mayAbort; /* True if statement may throw an ABORT exception */ - u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ u8 disableLookaside; /* Number of times lookaside has been disabled */ u8 prepFlags; /* SQLITE_PREPARE_* flags */ u8 withinRJSubrtn; /* Nesting level for RIGHT JOIN body subroutines */ - u8 bHasExists; /* Has a correlated "EXISTS (SELECT ....)" expression */ u8 mSubrtnSig; /* mini Bloom filter on available SubrtnSig.selId */ u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ - u8 bReturning; /* Coding a RETURNING trigger */ u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ - u8 disableTriggers; /* True to disable triggers */ #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) u8 earlyCleanup; /* OOM inside sqlite3ParserAddCleanup() */ #endif @@ -3920,10 +3895,15 @@ struct Parse { u8 isCreate; /* CREATE TABLE, INDEX, or VIEW (but not TRIGGER) ** and ALTER TABLE ADD COLUMN. */ #endif - bft colNamesSet :1; /* TRUE after OP_ColumnName has been issued to pVdbe */ - bft bHasWith :1; /* True if statement contains WITH */ - bft okConstFactor :1; /* OK to factor out constants */ - bft checkSchema :1; /* Causes schema cookie check after an error */ + bft disableTriggers:1; /* True to disable triggers */ + bft mayAbort :1; /* True if statement may throw an ABORT exception */ + bft hasCompound :1; /* Need to invoke convertCompoundSelectToSubquery() */ + bft bReturning :1; /* Coding a RETURNING trigger */ + bft bHasExists :1; /* Has a correlated "EXISTS (SELECT ....)" expression */ + bft colNamesSet :1; /* TRUE after OP_ColumnName has been issued to pVdbe */ + bft bHasWith :1; /* True if statement contains WITH */ + bft okConstFactor:1; /* OK to factor out constants */ + bft checkSchema :1; /* Causes schema cookie check after an error */ int nRangeReg; /* Size of the temporary register block */ int iRangeReg; /* First register in temporary register block */ int nErr; /* Number of errors seen */ @@ -4152,19 +4132,19 @@ struct Trigger { ** orconf -> stores the ON CONFLICT algorithm ** pSelect -> The content to be inserted - either a SELECT statement or ** a VALUES clause. -** zTarget -> Dequoted name of the table to insert into. +** pSrc -> Table to insert into. ** pIdList -> If this is an INSERT INTO ... (<column-names>) VALUES ... ** statement, then this stores the column-names to be ** inserted into. ** pUpsert -> The ON CONFLICT clauses for an Upsert ** ** (op == TK_DELETE) -** zTarget -> Dequoted name of the table to delete from. +** pSrc -> Table to delete from ** pWhere -> The WHERE clause of the DELETE statement if one is specified. ** Otherwise NULL. ** ** (op == TK_UPDATE) -** zTarget -> Dequoted name of the table to update. +** pSrc -> Table to update, followed by any FROM clause tables. ** pWhere -> The WHERE clause of the UPDATE statement if one is specified. ** Otherwise NULL. ** pExprList -> A list of the columns to update and the expressions to update @@ -4184,8 +4164,7 @@ struct TriggerStep { u8 orconf; /* OE_Rollback etc. */ Trigger *pTrig; /* The trigger that this step is a part of */ Select *pSelect; /* SELECT statement or RHS of INSERT INTO SELECT ... */ - char *zTarget; /* Target table for DELETE, UPDATE, INSERT */ - SrcList *pFrom; /* FROM clause for UPDATE statement (if any) */ + SrcList *pSrc; /* Table to insert/update/delete */ Expr *pWhere; /* The WHERE clause for DELETE or UPDATE steps */ ExprList *pExprList; /* SET clause for UPDATE, or RETURNING clause */ IdList *pIdList; /* Column names for INSERT */ @@ -4268,10 +4247,11 @@ typedef struct { /* ** Allowed values for mInitFlags */ -#define INITFLAG_AlterMask 0x0003 /* Types of ALTER */ +#define INITFLAG_AlterMask 0x0007 /* Types of ALTER */ #define INITFLAG_AlterRename 0x0001 /* Reparse after a RENAME */ #define INITFLAG_AlterDrop 0x0002 /* Reparse after a DROP COLUMN */ #define INITFLAG_AlterAdd 0x0003 /* Reparse after an ADD COLUMN */ +#define INITFLAG_AlterDropCons 0x0004 /* Reparse after an ADD COLUMN */ /* Tuning parameters are set using SQLITE_TESTCTRL_TUNE and are controlled ** on debug-builds of the CLI using ".testctrl tune ID VALUE". Tuning @@ -4401,6 +4381,7 @@ struct Walker { NameContext *pNC; /* Naming context */ int n; /* A counter */ int iCur; /* A cursor number */ + int sz; /* String literal length */ SrcList *pSrcList; /* FROM clause */ struct CCurHint *pCCurHint; /* Used by codeCursorHint() */ struct RefSrcList *pRefSrcList; /* sqlite3ReferencesSrcList() */ @@ -4805,7 +4786,20 @@ int sqlite3LookasideUsed(sqlite3*,int*); sqlite3_mutex *sqlite3Pcache1Mutex(void); sqlite3_mutex *sqlite3MallocMutex(void); -#if defined(SQLITE_ENABLE_MULTITHREADED_CHECKS) && !defined(SQLITE_MUTEX_OMIT) + +/* The SQLITE_THREAD_MISUSE_WARNINGS compile-time option used to be called +** SQLITE_ENABLE_MULTITHREADED_CHECKS. Keep that older macro for backwards +** compatibility, at least for a while... */ +#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS +# define SQLITE_THREAD_MISUSE_WARNINGS 1 +#endif + +/* SQLITE_THREAD_MISUSE_ABORT implies SQLITE_THREAD_MISUSE_WARNINGS */ +#ifdef SQLITE_THREAD_MISUSE_ABORT +# define SQLITE_THREAD_MISUSE_WARNINGS 1 +#endif + +#if defined(SQLITE_THREAD_MISUSE_WARNINGS) && !defined(SQLITE_MUTEX_OMIT) void sqlite3MutexWarnOnContention(sqlite3_mutex*); #else # define sqlite3MutexWarnOnContention(x) @@ -4834,17 +4828,22 @@ struct PrintfArguments { sqlite3_value **apArg; /* The argument values */ }; +/* +** Maxium number of base-10 digits in an unsigned 64-bit integer +*/ +#define SQLITE_U64_DIGITS 20 + /* ** An instance of this object receives the decoding of a floating point ** value into an approximate decimal representation. */ struct FpDecode { - char sign; /* '+' or '-' */ - char isSpecial; /* 1: Infinity 2: NaN */ - int n; /* Significant digits in the decode */ - int iDP; /* Location of the decimal point */ - char *z; /* Start of significant digits */ - char zBuf[24]; /* Storage for significant digits */ + int n; /* Significant digits in the decode */ + int iDP; /* Location of the decimal point */ + char *z; /* Start of significant digits */ + char zBuf[SQLITE_U64_DIGITS+1]; /* Storage for significant digits */ + char sign; /* '+' or '-' */ + char isSpecial; /* 1: Infinity 2: NaN */ }; void sqlite3FpDecode(FpDecode*,double,int,int); @@ -4933,6 +4932,7 @@ int sqlite3NoTempsInRange(Parse*,int,int); #endif Expr *sqlite3ExprAlloc(sqlite3*,int,const Token*,int); Expr *sqlite3Expr(sqlite3*,int,const char*); +Expr *sqlite3ExprInt32(sqlite3*,int); void sqlite3ExprAttachSubtrees(sqlite3*,Expr*,Expr*,Expr*); Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*); void sqlite3PExprAddSelect(Parse*, Expr*, Select*); @@ -5184,6 +5184,7 @@ int sqlite3ExprContainsSubquery(Expr*); int sqlite3ExprIsInteger(const Expr*, int*, Parse*); int sqlite3ExprCanBeNull(const Expr*); int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); +int sqlite3ExprIsLikeOperator(const Expr*); int sqlite3IsRowid(const char*); const char *sqlite3RowidAlias(Table *pTab); void sqlite3GenerateRowDelete( @@ -5252,17 +5253,16 @@ void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,int); void sqlite3DeleteTriggerStep(sqlite3*, TriggerStep*); TriggerStep *sqlite3TriggerSelectStep(sqlite3*,Select*, const char*,const char*); - TriggerStep *sqlite3TriggerInsertStep(Parse*,Token*, IdList*, + TriggerStep *sqlite3TriggerInsertStep(Parse*,SrcList*, IdList*, Select*,u8,Upsert*, const char*,const char*); - TriggerStep *sqlite3TriggerUpdateStep(Parse*,Token*,SrcList*,ExprList*, + TriggerStep *sqlite3TriggerUpdateStep(Parse*,SrcList*,SrcList*,ExprList*, Expr*, u8, const char*,const char*); - TriggerStep *sqlite3TriggerDeleteStep(Parse*,Token*, Expr*, + TriggerStep *sqlite3TriggerDeleteStep(Parse*,SrcList*, Expr*, const char*,const char*); void sqlite3DeleteTrigger(sqlite3*, Trigger*); void sqlite3UnlinkAndDeleteTrigger(sqlite3*,int,const char*); u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Table*,int); - SrcList *sqlite3TriggerStepSrc(Parse*, TriggerStep*); # define sqlite3ParseToplevel(p) ((p)->pToplevel ? (p)->pToplevel : (p)) # define sqlite3IsToplevel(p) ((p)->pToplevel==0) #else @@ -5276,7 +5276,6 @@ void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,int); # define sqlite3ParseToplevel(p) p # define sqlite3IsToplevel(p) 1 # define sqlite3TriggerColmask(A,B,C,D,E,F,G) 0 -# define sqlite3TriggerStepSrc(A,B) 0 #endif int sqlite3JoinType(Parse*, Token*, Token*, Token*); @@ -5309,7 +5308,7 @@ int sqlite3FixTriggerStep(DbFixer*, TriggerStep*); int sqlite3RealSameAsInt(double,sqlite3_int64); i64 sqlite3RealToI64(double); int sqlite3Int64ToText(i64,char*); -int sqlite3AtoF(const char *z, double*, int, u8); +int sqlite3AtoF(const char *z, double*); int sqlite3GetInt32(const char *, int*); int sqlite3GetUInt32(const char*, u32*); int sqlite3Atoi(const char*); @@ -5453,10 +5452,13 @@ void sqlite3Reindex(Parse*, Token*, Token*); void sqlite3AlterFunctions(void); void sqlite3AlterRenameTable(Parse*, SrcList*, Token*); void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*); +void sqlite3AlterDropConstraint(Parse*,SrcList*,Token*,Token*); +void sqlite3AlterAddConstraint(Parse*,SrcList*,Token*,Token*,const char*,int); +void sqlite3AlterSetNotNull(Parse*, SrcList*, Token*, Token*); i64 sqlite3GetToken(const unsigned char *, int *); void sqlite3NestedParse(Parse*, const char*, ...); void sqlite3ExpirePreparedStatements(sqlite3*, int); -void sqlite3CodeRhsOfIN(Parse*, Expr*, int); +void sqlite3CodeRhsOfIN(Parse*, Expr*, int, int); int sqlite3CodeSubselect(Parse*, Expr*); void sqlite3SelectPrep(Parse*, Select*, NameContext*); int sqlite3ExpandSubquery(Parse*, SrcItem*); @@ -5529,6 +5531,7 @@ char *sqlite3RCStrResize(char*,u64); void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int); int sqlite3StrAccumEnlarge(StrAccum*, i64); +int sqlite3StrAccumEnlargeIfNeeded(StrAccum*, i64); char *sqlite3StrAccumFinish(StrAccum*); void sqlite3StrAccumSetError(StrAccum*, u8); void sqlite3ResultStrAccum(sqlite3_context*,StrAccum*); diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index ee467836a..778112471 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -81,21 +81,42 @@ ** It used to be the case that setting this value to zero would ** turn the limit off. That is no longer true. It is not possible ** to turn this limit off. +** +** The hard limit is the largest possible 32-bit signed integer less +** 1024, or 2147482624. */ #ifndef SQLITE_MAX_SQL_LENGTH # define SQLITE_MAX_SQL_LENGTH 1000000000 #endif /* -** The maximum depth of an expression tree. This is limited to -** some extent by SQLITE_MAX_SQL_LENGTH. But sometime you might -** want to place more severe limits on the complexity of an -** expression. A value of 0 means that there is no limit. +** The maximum depth of an expression tree. The expression tree depth +** is also limited indirectly by SQLITE_MAX_SQL_LENGTH and by +** SQLITE_MAX_PARSER_DEPTH. Reducing the maximum complexity of +** expressions can help prevent excess memory usage by hostile SQL. +** +** A value of 0 for this compile-time option causes all expression +** depth limiting code to be omitted. */ #ifndef SQLITE_MAX_EXPR_DEPTH # define SQLITE_MAX_EXPR_DEPTH 1000 #endif +/* +** The maximum depth of the LALR(1) stack used in the parser that +** interprets SQL inputs. The parser stack depth can also be limited +** indirectly by SQLITE_MAX_SQL_LENGTH. Limiting the parser stack +** depth can help prevent excess memory usage and excess CPU stack +** usage when processing hostile SQL. +** +** Prior to version 3.45.0 (2024-01-15), the parser stack was +** hard-coded to 100 entries, and that worked fine for almost all +** applications. So the upper bound on this limit need not be large. +*/ +#ifndef SQLITE_MAX_PARSER_DEPTH +# define SQLITE_MAX_PARSER_DEPTH 2500 +#endif + /* ** The maximum number of terms in a compound SELECT statement. ** The code generator for compound SELECT statements does one @@ -211,6 +232,10 @@ # undef SQLITE_MAX_DEFAULT_PAGE_SIZE # define SQLITE_MAX_DEFAULT_PAGE_SIZE SQLITE_MAX_PAGE_SIZE #endif +#if SQLITE_MAX_DEFAULT_PAGE_SIZE<SQLITE_DEFAULT_PAGE_SIZE +# undef SQLITE_MAX_DEFAULT_PAGE_SIZE +# define SQLITE_MAX_DEFAULT_PAGE_SIZE SQLITE_DEFAULT_PAGE_SIZE +#endif /* diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 02a4d84e4..8f46f7e81 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -124,6 +124,15 @@ /* Forward declaration */ typedef struct SqliteDb SqliteDb; +/* Add -DSQLITE_ENABLE_QRF_IN_TCL to add the Query Result Formatter (QRF) +** into the build of the TCL extension, when building using separate +** source files. The QRF is included automatically when building from +** the tclsqlite3.c amalgamation. +*/ +#if defined(SQLITE_ENABLE_QRF_IN_TCL) +#include "qrf.h" +#endif + /* ** New SQL functions can be created as TCL scripts. Each such function ** is described by an instance of the following structure. @@ -2035,6 +2044,376 @@ static void DbHookCmd( sqlite3_wal_hook(db, (pDb->pWalHook?DbWalHandler:0), pDb); } +/* +** Implementation of the "db format" command. +** +** Based on provided options, format the results of the SQL statement(s) +** provided into human-readable form using the Query Result Formatter (QRF) +** and return the resuling text. +** +** Syntax: db format OPTIONS SQL +** +** OPTIONS may be: +** +** -style ("auto"|"box"|"column"|...) Output style +** -esc ("auto"|"off"|"ascii"|"symbol") How to deal with ctrl chars +** -text ("auto"|"off"|"sql"|"csv"|...) How to escape TEXT values +** -title ("auto"|"off"|"sql"|...|"off") How to escape column names +** -blob ("auto"|"text"|"sql"|...) How to escape BLOB values +** -wordwrap ("auto"|"off"|"on") Try to wrap at word boundry? +** -textjsonb ("auto"|"off"|"on") Auto-convert JSONB to text? +** -splitcolumn ("auto"|"off"|"on") Enable split-column mode +** -defaultalign ("auto"|"left"|...) Default alignment +** -titalalign ("auto"|"left"|"right"|...) Default column name alignment +** -border ("auto"|"off"|"on") Border for box and table styles +** -wrap NUMBER Max width of any single column +** -screenwidth NUMBER Width of the display TTY +** -linelimit NUMBER Max lines for any cell +** -charlimit NUMBER Content truncated to this size +** -titlelimit NUMBER Max width of column titles +** -multiinsert NUMBER Multi-row INSERT byte size +** -align LIST-OF-ALIGNMENT Alignment of columns +** -widths LIST-OF-NUMBERS Widths for individual columns +** -columnsep TEXT Column separator text +** -rowsep TEXT Row separator text +** -tablename TEXT Table name for style "insert" +** -null TEXT Text for NULL values +** +** A mapping from TCL "format" command options to sqlite3_qrf_spec fields +** is below. Use this to reference the QRF documentation: +** +** TCL Option spec field +** ---------- ---------- +** -style eStyle +** -esc eEsc +** -text eText +** -title eTitle, bTitle +** -blob eBlob +** -wordwrap bWordWrap +** -textjsonb bTextJsonb +** -splitcolumn bSplitColumn +** -defaultalign eDfltAlign +** -titlealign eTitleAlign +** -border bBorder +** -wrap nWrap +** -screenwidth nScreenWidth +** -linelimit nLineLimit +** -charlimit nCharLimit +** -titlelimit nTitleLimit +** -multiinsert nMultiInsert +** -align nAlign, aAlign +** -widths nWidth, aWidth +** -columnsep zColumnSep +** -rowsep zRowSep +** -tablename zTableName +** -null zNull +*/ +static int dbQrf(SqliteDb *pDb, int objc, Tcl_Obj *const*objv){ +#ifndef SQLITE_QRF_H + Tcl_SetResult(pDb->interp, "QRF not available in this build", TCL_VOLATILE); + return TCL_ERROR; +#else + char *zResult = 0; /* Result to be returned */ + const char *zSql = 0; /* SQL to run */ + int i; /* Loop counter */ + int rc; /* Result code */ + sqlite3_qrf_spec qrf; /* Formatting spec */ + static const char *azAlign[] = { + "auto", "bottom", "c", + "center", "e", "left", + "middle", "n", "ne", + "nw", "right", "s", + "se", "sw", "top", + "w", 0 + }; + static const unsigned char aAlignMap[] = { + QRF_ALIGN_Auto, QRF_ALIGN_Bottom, QRF_ALIGN_C, + QRF_ALIGN_Center, QRF_ALIGN_E, QRF_ALIGN_Left, + QRF_ALIGN_Middle, QRF_ALIGN_N, QRF_ALIGN_NE, + QRF_ALIGN_NW, QRF_ALIGN_Right, QRF_ALIGN_S, + QRF_ALIGN_SE, QRF_ALIGN_SW, QRF_ALIGN_Top, + QRF_ALIGN_W + }; + + memset(&qrf, 0, sizeof(qrf)); + qrf.iVersion = 1; + qrf.pzOutput = &zResult; + for(i=2; i<objc; i++){ + const char *zArg = Tcl_GetString(objv[i]); + const char *azBool[] = { "auto", "yes", "no", "on", "off", 0 }; + const unsigned char aBoolMap[] = { 0, 2, 1, 2, 1 }; + if( zArg[0]!='-' ){ + if( zSql ){ + Tcl_AppendResult(pDb->interp, "unknown argument: ", zArg, (char*)0); + rc = TCL_ERROR; + goto format_failed; + } + zSql = zArg; + }else if( i==objc-1 ){ + Tcl_AppendResult(pDb->interp, "option has no argument: ", zArg, (char*)0); + rc = TCL_ERROR; + goto format_failed; + }else if( strcmp(zArg,"-style")==0 ){ + static const char *azStyles[] = { + "auto", "box", "column", + "count", "csv", "eqp", + "explain", "html", "insert", + "jobject", "json", "line", + "list", "markdown", "quote", + "stats", "stats-est", "stats-vm", + "table", 0 + }; + static unsigned char aStyleMap[] = { + QRF_STYLE_Auto, QRF_STYLE_Box, QRF_STYLE_Column, + QRF_STYLE_Count, QRF_STYLE_Csv, QRF_STYLE_Eqp, + QRF_STYLE_Explain, QRF_STYLE_Html, QRF_STYLE_Insert, + QRF_STYLE_JObject, QRF_STYLE_Json, QRF_STYLE_Line, + QRF_STYLE_List, QRF_STYLE_Markdown, QRF_STYLE_Quote, + QRF_STYLE_Stats, QRF_STYLE_StatsEst, QRF_STYLE_StatsVm, + QRF_STYLE_Table + }; + int style; + rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azStyles, + "format style (-style)", 0, &style); + if( rc ) goto format_failed; + qrf.eStyle = aStyleMap[style]; + i++; + }else if( strcmp(zArg,"-esc")==0 ){ + static const char *azEsc[] = { + "ascii", "auto", "off", "symbol", 0 + }; + static unsigned char aEscMap[] = { + QRF_ESC_Ascii, QRF_ESC_Auto, QRF_ESC_Off, QRF_ESC_Symbol + }; + int esc; + rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azEsc, + "control character escape (-esc)", 0, &esc); + if( rc ) goto format_failed; + qrf.eEsc = aEscMap[esc]; + i++; + }else if( strcmp(zArg,"-text")==0 || strcmp(zArg, "-title")==0 ){ + /* NB: --title can be "off" or "on but --text may not be. Thus we put + ** the "off" and "on" choices first and start the search on the + ** thrid element of the array when processing --text */ + static const char *azText[] = { "off", "on", + "auto", "csv", "html", + "json", "plain", "relaxed", + "sql", "tcl", 0 + }; + static unsigned char aTextMap[] = { + QRF_TEXT_Auto, QRF_TEXT_Csv, QRF_TEXT_Html, + QRF_TEXT_Json, QRF_TEXT_Plain, QRF_TEXT_Relaxed, + QRF_TEXT_Sql, QRF_TEXT_Tcl + }; + int txt; + int k = zArg[2]=='e'; + rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], &azText[k*2], zArg, + 0, &txt); + if( rc ) goto format_failed; + if( k ){ + qrf.eText = aTextMap[txt]; + }else if( txt<=1 ){ + qrf.bTitles = txt ? QRF_Yes : QRF_No; + qrf.eTitle = QRF_TEXT_Auto; + }else{ + qrf.bTitles = QRF_Yes; + qrf.eTitle = aTextMap[txt-2]; + } + i++; + }else if( strcmp(zArg,"-blob")==0 ){ + static const char *azBlob[] = { + "auto", "hex", "json", + "tcl", "text", "sql", + "size", 0 + }; + static unsigned char aBlobMap[] = { + QRF_BLOB_Auto, QRF_BLOB_Hex, QRF_BLOB_Json, + QRF_BLOB_Tcl, QRF_BLOB_Text, QRF_BLOB_Sql, + QRF_BLOB_Size + }; + int blob; + rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azBlob, + "BLOB encoding (-blob)", 0, &blob); + if( rc ) goto format_failed; + qrf.eBlob = aBlobMap[blob]; + i++; + }else if( strcmp(zArg,"-wordwrap")==0 ){ + int v = 0; + rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azBool, + "-wordwrap", 0, &v); + if( rc ) goto format_failed; + qrf.bWordWrap = aBoolMap[v]; + i++; + }else if( strcmp(zArg,"-textjsonb")==0 + || strcmp(zArg,"-splitcolumn")==0 + || strcmp(zArg,"-border")==0 + ){ + int v = 0; + rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azBool, + zArg, 0, &v); + if( rc ) goto format_failed; + if( zArg[1]=='t' ){ + qrf.bTextJsonb = aBoolMap[v]; + }else if( zArg[1]=='b' ){ + qrf.bBorder = aBoolMap[v]; + }else{ + qrf.bSplitColumn = aBoolMap[v]; + } + i++; + }else if( strcmp(zArg,"-defaultalign")==0 || strcmp(zArg,"-titlealign")==0){ + int ax = 0; + rc = Tcl_GetIndexFromObj(pDb->interp, objv[i+1], azAlign, + zArg[1]=='d' ? "default alignment (-defaultalign)" : + "title alignment (-titlealign)", + 0, &ax); + if( rc ) goto format_failed; + if( zArg[1]=='d' ){ + qrf.eDfltAlign = aAlignMap[ax]; + }else{ + qrf.eTitleAlign = aAlignMap[ax]; + } + i++; + }else if( strcmp(zArg,"-wrap")==0 + || strcmp(zArg,"-screenwidth")==0 + || strcmp(zArg,"-linelimit")==0 + || strcmp(zArg,"-titlelimit")==0 + ){ + int v = 0; + rc = Tcl_GetIntFromObj(pDb->interp, objv[i+1], &v); + if( rc ) goto format_failed; + if( v<QRF_MIN_WIDTH ){ + v = QRF_MIN_WIDTH; + }else if( v>QRF_MAX_WIDTH ){ + v = QRF_MAX_WIDTH; + } + if( zArg[1]=='w' ){ + qrf.nWrap = v; + }else if( zArg[1]=='s' ){ + qrf.nScreenWidth = v; + }else if( zArg[1]=='t' ){ + qrf.nTitleLimit = v; + }else{ + qrf.nLineLimit = v; + } + i++; + }else if( strcmp(zArg,"-charlimit")==0 ){ + int v = 0; + rc = Tcl_GetIntFromObj(pDb->interp, objv[i+1], &v); + if( rc ) goto format_failed; + if( v<0 ) v = 0; + qrf.nCharLimit = v; + i++; + }else if( strcmp(zArg,"-multiinsert")==0 ){ + int v = 0; + rc = Tcl_GetIntFromObj(pDb->interp, objv[i+1], &v); + if( rc ) goto format_failed; + if( v<0 ) v = 0; + qrf.nMultiInsert = v; + i++; + }else if( strcmp(zArg,"-align")==0 ){ + Tcl_Size n = 0; + int jj; + rc = Tcl_ListObjLength(pDb->interp, objv[i+1], &n); + if( rc ) goto format_failed; + sqlite3_free(qrf.aAlign); + qrf.aAlign = sqlite3_malloc64( (n+1)*sizeof(qrf.aAlign[0]) ); + if( qrf.aAlign==0 ){ + Tcl_AppendResult(pDb->interp, "out of memory", (char*)0); + rc = TCL_ERROR; + goto format_failed; + } + memset(qrf.aAlign, 0, (n+1)*sizeof(qrf.aAlign[0])); + qrf.nAlign = n; + for(jj=0; jj<n; jj++){ + int x; + Tcl_Obj *pTerm; + rc = Tcl_ListObjIndex(pDb->interp, objv[i+1], jj, &pTerm); + if( rc ) goto format_failed; + rc = Tcl_GetIndexFromObj(pDb->interp, pTerm, azAlign, + "column alignment (-align)", 0, &x); + if( rc ) goto format_failed; + qrf.aAlign[jj] = aAlignMap[x]; + } + i++; + }else if( strcmp(zArg,"-widths")==0 ){ + Tcl_Size n = 0; + int jj; + rc = Tcl_ListObjLength(pDb->interp, objv[i+1], &n); + if( rc ) goto format_failed; + sqlite3_free(qrf.aWidth); + qrf.aWidth = sqlite3_malloc64( (n+1)*sizeof(qrf.aWidth[0]) ); + if( qrf.aWidth==0 ){ + Tcl_AppendResult(pDb->interp, "out of memory", (char*)0); + rc = TCL_ERROR; + goto format_failed; + } + memset(qrf.aWidth, 0, (n+1)*sizeof(qrf.aWidth[0])); + qrf.nWidth = n; + for(jj=0; jj<n; jj++){ + Tcl_Obj *pTerm; + int v; + rc = Tcl_ListObjIndex(pDb->interp, objv[i+1], jj, &pTerm); + if( rc ) goto format_failed; + rc = Tcl_GetIntFromObj(pDb->interp, pTerm, &v); + if( v<(-QRF_MAX_WIDTH) ){ + v = -QRF_MAX_WIDTH; + }else if( v>QRF_MAX_WIDTH ){ + v = QRF_MAX_WIDTH; + } + qrf.aWidth[jj] = (short int)v; + } + i++; + }else if( strcmp(zArg,"-columnsep")==0 ){ + qrf.zColumnSep = Tcl_GetString(objv[i+1]); + i++; + }else if( strcmp(zArg,"-rowsep")==0 ){ + qrf.zRowSep = Tcl_GetString(objv[i+1]); + i++; + }else if( strcmp(zArg,"-tablename")==0 ){ + qrf.zTableName = Tcl_GetString(objv[i+1]); + i++; + }else if( strcmp(zArg,"-null")==0 ){ + qrf.zNull = Tcl_GetString(objv[i+1]); + i++; + }else if( strcmp(zArg,"-version")==0 ){ + /* Undocumented. Testing use only */ + qrf.iVersion = atoi(Tcl_GetString(objv[i+1])); + i++; + }else{ + Tcl_AppendResult(pDb->interp, "unknown option: ", zArg, (char*)0); + rc = TCL_ERROR; + goto format_failed; + } + } + while( zSql && zSql[0] ){ + SqlPreparedStmt *pStmt = 0; /* Next statement to run */ + char *zErr = 0; /* Error message from QRF */ + + rc = dbPrepareAndBind(pDb, zSql, &zSql, &pStmt); + if( rc ) goto format_failed; + if( pStmt==0 ) continue; + rc = sqlite3_format_query_result(pStmt->pStmt, &qrf, &zErr); + dbReleaseStmt(pDb, pStmt, 0); + if( rc ){ + Tcl_SetResult(pDb->interp, zErr, TCL_VOLATILE); + sqlite3_free(zErr); + rc = TCL_ERROR; + goto format_failed; + } + } + Tcl_SetResult(pDb->interp, zResult, TCL_VOLATILE); + rc = TCL_OK; + /* Fall through...*/ + +format_failed: + sqlite3_free(qrf.aWidth); + sqlite3_free(qrf.aAlign); + sqlite3_free(zResult); + return rc; + +#endif +} + /* ** The "sqlite" command below creates a new Tcl command for each ** connection it opens to an SQLite database. This routine is invoked @@ -2064,15 +2443,15 @@ static int SQLITE_TCLAPI DbObjCmd( "commit_hook", "complete", "config", "copy", "deserialize", "enable_load_extension", "errorcode", "erroroffset", "eval", - "exists", "function", "incrblob", - "interrupt", "last_insert_rowid", "nullvalue", - "onecolumn", "preupdate", "profile", - "progress", "rekey", "restore", - "rollback_hook", "serialize", "status", - "timeout", "total_changes", "trace", - "trace_v2", "transaction", "unlock_notify", - "update_hook", "version", "wal_hook", - 0 + "exists", "format", "function", + "incrblob", "interrupt", "last_insert_rowid", + "nullvalue", "onecolumn", "preupdate", + "profile", "progress", "rekey", + "restore", "rollback_hook", "serialize", + "status", "timeout", "total_changes", + "trace", "trace_v2", "transaction", + "unlock_notify", "update_hook", "version", + "wal_hook", 0 }; enum DB_enum { DB_AUTHORIZER, DB_BACKUP, DB_BIND_FALLBACK, @@ -2081,14 +2460,15 @@ static int SQLITE_TCLAPI DbObjCmd( DB_COMMIT_HOOK, DB_COMPLETE, DB_CONFIG, DB_COPY, DB_DESERIALIZE, DB_ENABLE_LOAD_EXTENSION, DB_ERRORCODE, DB_ERROROFFSET, DB_EVAL, - DB_EXISTS, DB_FUNCTION, DB_INCRBLOB, - DB_INTERRUPT, DB_LAST_INSERT_ROWID, DB_NULLVALUE, - DB_ONECOLUMN, DB_PREUPDATE, DB_PROFILE, - DB_PROGRESS, DB_REKEY, DB_RESTORE, - DB_ROLLBACK_HOOK, DB_SERIALIZE, DB_STATUS, - DB_TIMEOUT, DB_TOTAL_CHANGES, DB_TRACE, - DB_TRACE_V2, DB_TRANSACTION, DB_UNLOCK_NOTIFY, - DB_UPDATE_HOOK, DB_VERSION, DB_WAL_HOOK, + DB_EXISTS, DB_FORMAT, DB_FUNCTION, + DB_INCRBLOB, DB_INTERRUPT, DB_LAST_INSERT_ROWID, + DB_NULLVALUE, DB_ONECOLUMN, DB_PREUPDATE, + DB_PROFILE, DB_PROGRESS, DB_REKEY, + DB_RESTORE, DB_ROLLBACK_HOOK, DB_SERIALIZE, + DB_STATUS, DB_TIMEOUT, DB_TOTAL_CHANGES, + DB_TRACE, DB_TRACE_V2, DB_TRANSACTION, + DB_UNLOCK_NOTIFY, DB_UPDATE_HOOK, DB_VERSION, + DB_WAL_HOOK }; /* don't leave trailing commas on DB_enum, it confuses the AIX xlc compiler */ @@ -2978,6 +3358,18 @@ static int SQLITE_TCLAPI DbObjCmd( break; } + /* + ** $db format [OPTIONS] SQL + ** + ** Run the SQL statement(s) given as the final argument. Use the + ** Query Result Formatter extension of SQLite to format the output as + ** text and return that text. + */ + case DB_FORMAT: { + rc = dbQrf(pDb, objc, objv); + break; + } + /* ** $db function NAME [OPTIONS] SCRIPT ** diff --git a/src/test1.c b/src/test1.c index f8e83dc42..3ca5c837a 100644 --- a/src/test1.c +++ b/src/test1.c @@ -4411,7 +4411,7 @@ static void delIntptr(void *p){ } /* -** bind_carray_intptr STMT IPARAM INT0 INT1 INT2... +** bind_carray_intptr STMT IPARAM INT-0 INT-1 INT-2... */ static int SQLITE_TCLAPI bind_carray_intptr( void * clientData, @@ -4455,6 +4455,7 @@ static int SQLITE_TCLAPI bind_carray_intptr( ** -malloc ** -transient ** -static +** -v2 ** -int32 ** -int64 ** -double @@ -4477,6 +4478,7 @@ static int SQLITE_TCLAPI test_carray_bind( void *aData = 0; int isTransient = 0; int isStatic = 0; + int isV2 = 0; int isMalloc = 0; /* True to use custom xDel function */ int idx; int i, j; @@ -4509,16 +4511,22 @@ static int SQLITE_TCLAPI test_carray_bind( const char *z = Tcl_GetString(objv[i]); if( strcmp(z, "-transient")==0 ){ isTransient = 1; + isStatic = isMalloc = 0; xDel = SQLITE_TRANSIENT; }else if( strcmp(z, "-static")==0 ){ isStatic = 1; + isMalloc = isTransient = 0; xDel = SQLITE_STATIC; }else if( strcmp(z, "-malloc")==0 ){ isMalloc = 1; + isStatic = isTransient = 0; xDel = testCarrayFree; }else + if( strcmp(z, "-v2")==0 ){ + isV2 = 1; + }else if( strcmp(z, "-int32")==0 ){ eType = 0; /* CARRAY_INT32 */ }else @@ -4687,7 +4695,20 @@ static int SQLITE_TCLAPI test_carray_bind( if( rc==SQLITE_OK ){ if( mFlagsOverride==0 ) mFlagsOverride = eType; - rc = sqlite3_carray_bind(pStmt, idx, aData, nData, mFlagsOverride, xDel); + if( isV2 ){ + void *pDel; + if( xDel==testCarrayFree ){ + u8 *p2 = (u8*)aData; + pDel = (void*)&p2[-16]; + xDel = sqlite3_free; + }else{ + pDel = aData; + } + rc = sqlite3_carray_bind_v2(pStmt, idx, aData, nData, mFlagsOverride, + xDel, pDel); + }else{ + rc = sqlite3_carray_bind(pStmt, idx, aData, nData, mFlagsOverride, xDel); + } } if( isTransient ){ if( eType==3 && aData ){ @@ -7370,6 +7391,7 @@ static int SQLITE_TCLAPI test_limit( { "SQLITE_LIMIT_SQL_LENGTH", SQLITE_LIMIT_SQL_LENGTH }, { "SQLITE_LIMIT_COLUMN", SQLITE_LIMIT_COLUMN }, { "SQLITE_LIMIT_EXPR_DEPTH", SQLITE_LIMIT_EXPR_DEPTH }, + { "SQLITE_LIMIT_PARSER_DEPTH", SQLITE_LIMIT_PARSER_DEPTH }, { "SQLITE_LIMIT_COMPOUND_SELECT", SQLITE_LIMIT_COMPOUND_SELECT }, { "SQLITE_LIMIT_VDBE_OP", SQLITE_LIMIT_VDBE_OP }, { "SQLITE_LIMIT_FUNCTION_ARG", SQLITE_LIMIT_FUNCTION_ARG }, @@ -7381,7 +7403,7 @@ static int SQLITE_TCLAPI test_limit( /* Out of range test cases */ { "SQLITE_LIMIT_TOOSMALL", -1, }, - { "SQLITE_LIMIT_TOOBIG", SQLITE_LIMIT_WORKER_THREADS+1 }, + { "SQLITE_LIMIT_TOOBIG", SQLITE_LIMIT_PARSER_DEPTH+1 }, }; int i, id = 0; int val; @@ -7675,7 +7697,8 @@ static int SQLITE_TCLAPI test_wal_checkpoint_v2( int nCkpt = -555; Tcl_Obj *pRet; - const char * aMode[] = { "passive", "full", "restart", "truncate", 0 }; + const char * aMode[] = {"noop", "passive", "full", "restart", "truncate", 0}; + assert( SQLITE_CHECKPOINT_NOOP==-1 ); assert( SQLITE_CHECKPOINT_PASSIVE==0 ); assert( SQLITE_CHECKPOINT_FULL==1 ); assert( SQLITE_CHECKPOINT_RESTART==2 ); @@ -7689,12 +7712,15 @@ static int SQLITE_TCLAPI test_wal_checkpoint_v2( if( objc==4 ){ zDb = Tcl_GetString(objv[3]); } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) || ( - TCL_OK!=Tcl_GetIntFromObj(0, objv[2], &eMode) - && TCL_OK!=Tcl_GetIndexFromObj(interp, objv[2], aMode, "mode", 0, &eMode) - )){ + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ return TCL_ERROR; } + if( TCL_OK!=Tcl_GetIntFromObj(0, objv[2], &eMode) ){ + if( TCL_OK!=Tcl_GetIndexFromObj(interp, objv[2], aMode, "mode", 0,&eMode) ){ + return TCL_ERROR; + } + eMode = eMode - 1; + } rc = sqlite3_wal_checkpoint_v2(db, zDb, eMode, &nLog, &nCkpt); if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ @@ -8618,6 +8644,7 @@ static int SQLITE_TCLAPI test_sqlite3_db_config( { "ATTACH_CREATE", SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE }, { "ATTACH_WRITE", SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE }, { "COMMENTS", SQLITE_DBCONFIG_ENABLE_COMMENTS }, + { "FP_DIGITS", SQLITE_DBCONFIG_FP_DIGITS }, }; int i; int v = 0; diff --git a/src/test2.c b/src/test2.c index 899728ead..9afce7b7a 100644 --- a/src/test2.c +++ b/src/test2.c @@ -636,6 +636,7 @@ static int SQLITE_TCLAPI faultInstallCmd( if( argc!=1 && argc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], " SCRIPT\"", (void*)0); + return SQLITE_ERROR; } zScript = argc==2 ? argv[1] : ""; nScript = (int)strlen(zScript); diff --git a/src/test_bestindex.c b/src/test_bestindex.c index f6b5db0fb..963abfec0 100644 --- a/src/test_bestindex.c +++ b/src/test_bestindex.c @@ -828,7 +828,6 @@ static int tclUpdate( tcl_vtab *pTab = (tcl_vtab*)tab; Tcl_Interp *interp = pTab->interp; Tcl_Obj *pEval = Tcl_DuplicateObj(pTab->pCmd); - Tcl_Obj *pRes = 0; int rc = TCL_OK; Tcl_IncrRefCount(pEval); diff --git a/src/test_config.c b/src/test_config.c index 3dbef3c9a..bebf8625a 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -82,7 +82,7 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options","configslower","1.0",TCL_GLOBAL_ONLY); #endif -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT +#if !SQLITE_OS_WINCE Tcl_SetVar2(interp, "sqlite_options", "curdir", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "curdir", "0", TCL_GLOBAL_ONLY); @@ -94,12 +94,6 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "win32malloc", "0", TCL_GLOBAL_ONLY); #endif -#if defined(SQLITE_OS_WINRT) && SQLITE_OS_WINRT - Tcl_SetVar2(interp, "sqlite_options", "winrt", "1", TCL_GLOBAL_ONLY); -#else - Tcl_SetVar2(interp, "sqlite_options", "winrt", "0", TCL_GLOBAL_ONLY); -#endif - #ifdef SQLITE_DEBUG Tcl_SetVar2(interp, "sqlite_options", "debug", "1", TCL_GLOBAL_ONLY); #else @@ -685,6 +679,14 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); Tcl_SetVar2(interp, "sqlite_options", "trace", "1", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_THREAD_MISUSE_WARNINGS + Tcl_SetVar2(interp, "sqlite_options", "thread_misuse_warnings", + "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "thread_misuse_warnings", + "0", TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_OMIT_TRIGGER Tcl_SetVar2(interp, "sqlite_options", "trigger", "0", TCL_GLOBAL_ONLY); #else diff --git a/src/test_quota.c b/src/test_quota.c index d2f9cddd1..3eeacc7e7 100644 --- a/src/test_quota.c +++ b/src/test_quota.c @@ -387,11 +387,7 @@ static char *quota_utf8_to_mbcs(const char *zUtf8){ zTmpWide = (LPWSTR)sqlite3_malloc( (nWide+1)*sizeof(zTmpWide[0]) ); if( zTmpWide==0 ) return 0; MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, zTmpWide, nWide); -#ifdef SQLITE_OS_WINRT - codepage = CP_ACP; -#else codepage = AreFileApisANSI() ? CP_ACP : CP_OEMCP; -#endif nMbcs = WideCharToMultiByte(codepage, 0, zTmpWide, nWide, 0, 0, 0, 0); zMbcs = nMbcs ? (char*)sqlite3_malloc( nMbcs+1 ) : 0; if( zMbcs ){ diff --git a/src/threads.c b/src/threads.c index f128d69fc..c7b2e893f 100644 --- a/src/threads.c +++ b/src/threads.c @@ -196,6 +196,7 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ rc = sqlite3Win32Wait((HANDLE)p->tid); assert( rc!=WAIT_IO_COMPLETION ); bRc = CloseHandle((HANDLE)p->tid); + (void)bRc; /* Prevent warning when assert() is a no-op */ assert( bRc ); } if( rc==WAIT_OBJECT_0 ) *ppOut = p->pResult; diff --git a/src/tokenize.c b/src/tokenize.c index 152ada64f..fa9a5627d 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -508,7 +508,7 @@ i64 sqlite3GetToken(const unsigned char *z, int *tokenType){ } case CC_DOLLAR: case CC_VARALPHA: { - int n = 0; + i64 n = 0; testcase( z[0]=='$' ); testcase( z[0]=='@' ); testcase( z[0]==':' ); testcase( z[0]=='#' ); *tokenType = TK_VARIABLE; @@ -604,7 +604,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql){ int tokenType; /* type of the next token */ int lastTokenParsed = -1; /* type of the previous token */ sqlite3 *db = pParse->db; /* The database connection */ - int mxSqlLen; /* Max length of an SQL string */ + i64 mxSqlLen; /* Max length of an SQL string */ Parse *pParentParse = 0; /* Outer parse context, if any */ #ifdef sqlite3Parser_ENGINEALWAYSONSTACK yyParser sEngine; /* Space to hold the Lemon-generated Parser object */ @@ -734,7 +734,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql){ } if( pParse->zErrMsg || (pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE) ){ if( pParse->zErrMsg==0 ){ - pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc)); + pParse->zErrMsg = sqlite3DbStrDup(db, sqlite3ErrStr(pParse->rc)); } if( (pParse->prepFlags & SQLITE_PREPARE_DONT_LOG)==0 ){ sqlite3_log(pParse->rc, "%s in \"%s\"", pParse->zErrMsg, pParse->zTail); @@ -815,7 +815,7 @@ char *sqlite3Normalize( sqlite3_str_append(pStr, " NULL", 5); break; } - /* Fall through */ + /* no break */ deliberate_fall_through } case TK_STRING: case TK_INTEGER: @@ -879,7 +879,7 @@ char *sqlite3Normalize( } case TK_SELECT: { iStartIN = 0; - /* fall through */ + /* no break */ deliberate_fall_through } default: { if( sqlite3IsIdChar(zSql[i]) ) addSpaceSeparator(pStr); diff --git a/src/treeview.c b/src/treeview.c index 153fec88d..b482e1581 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -1300,7 +1300,13 @@ void sqlite3TreeViewTrigger( void sqlite3ShowExpr(const Expr *p){ sqlite3TreeViewExpr(0,p,0); } void sqlite3ShowExprList(const ExprList *p){ sqlite3TreeViewExprList(0,p,0,0);} void sqlite3ShowIdList(const IdList *p){ sqlite3TreeViewIdList(0,p,0,0); } -void sqlite3ShowSrcList(const SrcList *p){ sqlite3TreeViewSrcList(0,p); } +void sqlite3ShowSrcList(const SrcList *p){ + TreeView *pView = 0; + sqlite3TreeViewPush(&pView, 0); + sqlite3TreeViewLine(pView, "SRCLIST"); + sqlite3TreeViewSrcList(pView,p); + sqlite3TreeViewPop(&pView); +} void sqlite3ShowSelect(const Select *p){ sqlite3TreeViewSelect(0,p,0); } void sqlite3ShowWith(const With *p){ sqlite3TreeViewWith(0,p,0); } void sqlite3ShowUpsert(const Upsert *p){ sqlite3TreeViewUpsert(0,p,0); } diff --git a/src/trigger.c b/src/trigger.c index 799fbe57f..4f9068ad8 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -26,7 +26,7 @@ void sqlite3DeleteTriggerStep(sqlite3 *db, TriggerStep *pTriggerStep){ sqlite3SelectDelete(db, pTmp->pSelect); sqlite3IdListDelete(db, pTmp->pIdList); sqlite3UpsertDelete(db, pTmp->pUpsert); - sqlite3SrcListDelete(db, pTmp->pFrom); + sqlite3SrcListDelete(db, pTmp->pSrc); sqlite3DbFree(db, pTmp->zSpan); sqlite3DbFree(db, pTmp); @@ -215,11 +215,16 @@ void sqlite3BeginTrigger( } } + /* NB: The SQLITE_ALLOW_TRIGGERS_ON_SYSTEM_TABLES compile-time option is + ** experimental and unsupported. Do not use it unless understand the + ** implications and you cannot get by without this capability. */ +#if !defined(SQLITE_ALLOW_TRIGGERS_ON_SYSTEM_TABLES) /* Experimental */ /* Do not create a trigger on a system table */ if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ sqlite3ErrorMsg(pParse, "cannot create trigger on system table"); goto trigger_cleanup; } +#endif /* INSTEAD of triggers are only for views and views only support INSTEAD ** of triggers. @@ -331,6 +336,7 @@ void sqlite3FinishTrigger( if( NEVER(pParse->nErr) || !pTrig ) goto triggerfinish_cleanup; zName = pTrig->zName; iDb = sqlite3SchemaToIndex(pParse->db, pTrig->pSchema); + assert( iDb>=00 && iDb<db->nDb ); pTrig->step_list = pStepList; while( pStepList ){ pStepList->pTrig = pTrig; @@ -365,12 +371,12 @@ void sqlite3FinishTrigger( if( sqlite3ReadOnlyShadowTables(db) ){ TriggerStep *pStep; for(pStep=pTrig->step_list; pStep; pStep=pStep->pNext){ - if( pStep->zTarget!=0 - && sqlite3ShadowTableName(db, pStep->zTarget) + if( pStep->pSrc!=0 + && sqlite3ShadowTableName(db, pStep->pSrc->a[0].zName) ){ sqlite3ErrorMsg(pParse, "trigger \"%s\" may not write to shadow table \"%s\"", - pTrig->zName, pStep->zTarget); + pTrig->zName, pStep->pSrc->a[0].zName); goto triggerfinish_cleanup; } } @@ -461,26 +467,39 @@ TriggerStep *sqlite3TriggerSelectStep( static TriggerStep *triggerStepAllocate( Parse *pParse, /* Parser context */ u8 op, /* Trigger opcode */ - Token *pName, /* The target name */ + SrcList *pTabList, /* Target table */ const char *zStart, /* Start of SQL text */ const char *zEnd /* End of SQL text */ ){ + Trigger *pNew = pParse->pNewTrigger; sqlite3 *db = pParse->db; - TriggerStep *pTriggerStep; + TriggerStep *pTriggerStep = 0; - if( pParse->nErr ) return 0; - pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep) + pName->n + 1); - if( pTriggerStep ){ - char *z = (char*)&pTriggerStep[1]; - memcpy(z, pName->z, pName->n); - sqlite3Dequote(z); - pTriggerStep->zTarget = z; - pTriggerStep->op = op; - pTriggerStep->zSpan = triggerSpanDup(db, zStart, zEnd); - if( IN_RENAME_OBJECT ){ - sqlite3RenameTokenMap(pParse, pTriggerStep->zTarget, pName); + if( pParse->nErr==0 ){ + if( pNew + && pNew->pSchema!=db->aDb[1].pSchema + && pTabList->a[0].u4.zDatabase + ){ + sqlite3ErrorMsg(pParse, + "qualified table names are not allowed on INSERT, UPDATE, and DELETE " + "statements within triggers"); + }else{ + pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep)); + if( pTriggerStep ){ + pTriggerStep->pSrc = sqlite3SrcListDup(db, pTabList, EXPRDUP_REDUCE); + pTriggerStep->op = op; + pTriggerStep->zSpan = triggerSpanDup(db, zStart, zEnd); + if( pTriggerStep->pSrc && IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, + pTriggerStep->pSrc->a[0].zName, + pTabList->a[0].zName + ); + } + } } } + + sqlite3SrcListDelete(db, pTabList); return pTriggerStep; } @@ -493,7 +512,7 @@ static TriggerStep *triggerStepAllocate( */ TriggerStep *sqlite3TriggerInsertStep( Parse *pParse, /* Parser */ - Token *pTableName, /* Name of the table into which we insert */ + SrcList *pTabList, /* Table to INSERT into */ IdList *pColumn, /* List of columns in pTableName to insert into */ Select *pSelect, /* A SELECT statement that supplies values */ u8 orconf, /* The conflict algorithm (OE_Abort, OE_Replace, etc.) */ @@ -506,7 +525,7 @@ TriggerStep *sqlite3TriggerInsertStep( assert(pSelect != 0 || db->mallocFailed); - pTriggerStep = triggerStepAllocate(pParse, TK_INSERT, pTableName,zStart,zEnd); + pTriggerStep = triggerStepAllocate(pParse, TK_INSERT, pTabList, zStart, zEnd); if( pTriggerStep ){ if( IN_RENAME_OBJECT ){ pTriggerStep->pSelect = pSelect; @@ -538,7 +557,7 @@ TriggerStep *sqlite3TriggerInsertStep( */ TriggerStep *sqlite3TriggerUpdateStep( Parse *pParse, /* Parser */ - Token *pTableName, /* Name of the table to be updated */ + SrcList *pTabList, /* Name of the table to be updated */ SrcList *pFrom, /* FROM clause for an UPDATE-FROM, or NULL */ ExprList *pEList, /* The SET clause: list of column and new values */ Expr *pWhere, /* The WHERE clause */ @@ -549,21 +568,36 @@ TriggerStep *sqlite3TriggerUpdateStep( sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; - pTriggerStep = triggerStepAllocate(pParse, TK_UPDATE, pTableName,zStart,zEnd); + pTriggerStep = triggerStepAllocate(pParse, TK_UPDATE, pTabList, zStart, zEnd); if( pTriggerStep ){ + SrcList *pFromDup = 0; if( IN_RENAME_OBJECT ){ pTriggerStep->pExprList = pEList; pTriggerStep->pWhere = pWhere; - pTriggerStep->pFrom = pFrom; + pFromDup = pFrom; pEList = 0; pWhere = 0; pFrom = 0; }else{ pTriggerStep->pExprList = sqlite3ExprListDup(db, pEList, EXPRDUP_REDUCE); pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); - pTriggerStep->pFrom = sqlite3SrcListDup(db, pFrom, EXPRDUP_REDUCE); + pFromDup = sqlite3SrcListDup(db, pFrom, EXPRDUP_REDUCE); } pTriggerStep->orconf = orconf; + + if( pFromDup && !IN_RENAME_OBJECT){ + Select *pSub; + Token as = {0, 0}; + pSub = sqlite3SelectNew(pParse, 0, pFromDup, 0,0,0,0, SF_NestedFrom, 0); + pFromDup = sqlite3SrcListAppendFromTerm(pParse, 0, 0, 0, &as, pSub ,0); + } + if( pFromDup && pTriggerStep->pSrc ){ + pTriggerStep->pSrc = sqlite3SrcListAppendList( + pParse, pTriggerStep->pSrc, pFromDup + ); + }else{ + sqlite3SrcListDelete(db, pFromDup); + } } sqlite3ExprListDelete(db, pEList); sqlite3ExprDelete(db, pWhere); @@ -578,7 +612,7 @@ TriggerStep *sqlite3TriggerUpdateStep( */ TriggerStep *sqlite3TriggerDeleteStep( Parse *pParse, /* Parser */ - Token *pTableName, /* The table from which rows are deleted */ + SrcList *pTabList, /* The table from which rows are deleted */ Expr *pWhere, /* The WHERE clause */ const char *zStart, /* Start of SQL text */ const char *zEnd /* End of SQL text */ @@ -586,7 +620,7 @@ TriggerStep *sqlite3TriggerDeleteStep( sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; - pTriggerStep = triggerStepAllocate(pParse, TK_DELETE, pTableName,zStart,zEnd); + pTriggerStep = triggerStepAllocate(pParse, TK_DELETE, pTabList, zStart, zEnd); if( pTriggerStep ){ if( IN_RENAME_OBJECT ){ pTriggerStep->pWhere = pWhere; @@ -786,6 +820,7 @@ static SQLITE_NOINLINE Trigger *triggersReallyExist( p = pList; if( (pParse->db->flags & SQLITE_EnableTrigger)==0 && pTab->pTrigger!=0 + && sqlite3SchemaToIndex(pParse->db, pTab->pTrigger->pSchema)!=1 ){ /* The SQLITE_DBCONFIG_ENABLE_TRIGGER setting is off. That means that ** only TEMP triggers are allowed. Truncate the pList so that it @@ -848,52 +883,6 @@ Trigger *sqlite3TriggersExist( return triggersReallyExist(pParse,pTab,op,pChanges,pMask); } -/* -** Convert the pStep->zTarget string into a SrcList and return a pointer -** to that SrcList. -** -** This routine adds a specific database name, if needed, to the target when -** forming the SrcList. This prevents a trigger in one database from -** referring to a target in another database. An exception is when the -** trigger is in TEMP in which case it can refer to any other database it -** wants. -*/ -SrcList *sqlite3TriggerStepSrc( - Parse *pParse, /* The parsing context */ - TriggerStep *pStep /* The trigger containing the target token */ -){ - sqlite3 *db = pParse->db; - SrcList *pSrc; /* SrcList to be returned */ - char *zName = sqlite3DbStrDup(db, pStep->zTarget); - pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); - assert( pSrc==0 || pSrc->nSrc==1 ); - assert( zName || pSrc==0 ); - if( pSrc ){ - Schema *pSchema = pStep->pTrig->pSchema; - pSrc->a[0].zName = zName; - if( pSchema!=db->aDb[1].pSchema ){ - assert( pSrc->a[0].fg.fixedSchema || pSrc->a[0].u4.zDatabase==0 ); - pSrc->a[0].u4.pSchema = pSchema; - pSrc->a[0].fg.fixedSchema = 1; - } - if( pStep->pFrom ){ - SrcList *pDup = sqlite3SrcListDup(db, pStep->pFrom, 0); - if( pDup && pDup->nSrc>1 && !IN_RENAME_OBJECT ){ - Select *pSubquery; - Token as; - pSubquery = sqlite3SelectNew(pParse,0,pDup,0,0,0,0,SF_NestedFrom,0); - as.n = 0; - as.z = 0; - pDup = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&as,pSubquery,0); - } - pSrc = sqlite3SrcListAppendList(pParse, pSrc, pDup); - } - }else{ - sqlite3DbFree(db, zName); - } - return pSrc; -} - /* ** Return true if the pExpr term from the RETURNING clause argument ** list is of the form "*". Raise an error if the terms if of the @@ -1159,7 +1148,7 @@ static int codeTriggerProgram( switch( pStep->op ){ case TK_UPDATE: { sqlite3Update(pParse, - sqlite3TriggerStepSrc(pParse, pStep), + sqlite3SrcListDup(db, pStep->pSrc, 0), sqlite3ExprListDup(db, pStep->pExprList, 0), sqlite3ExprDup(db, pStep->pWhere, 0), pParse->eOrconf, 0, 0, 0 @@ -1169,7 +1158,7 @@ static int codeTriggerProgram( } case TK_INSERT: { sqlite3Insert(pParse, - sqlite3TriggerStepSrc(pParse, pStep), + sqlite3SrcListDup(db, pStep->pSrc, 0), sqlite3SelectDup(db, pStep->pSelect, 0), sqlite3IdListDup(db, pStep->pIdList), pParse->eOrconf, @@ -1180,7 +1169,7 @@ static int codeTriggerProgram( } case TK_DELETE: { sqlite3DeleteFrom(pParse, - sqlite3TriggerStepSrc(pParse, pStep), + sqlite3SrcListDup(db, pStep->pSrc, 0), sqlite3ExprDup(db, pStep->pWhere, 0), 0, 0 ); sqlite3VdbeAddOp0(v, OP_ResetCount); diff --git a/src/util.c b/src/util.c index 8e4fd516e..67e6e2ee0 100644 --- a/src/util.c +++ b/src/util.c @@ -458,266 +458,613 @@ u8 sqlite3StrIHash(const char *z){ return h; } -/* Double-Double multiplication. (x[0],x[1]) *= (y,yy) +/* +** Two inputs are multiplied to get a 128-bit result. Write the +** lower 64-bits of the result into *pLo, and return the high-order +** 64 bits. +*/ +static u64 sqlite3Multiply128(u64 a, u64 b, u64 *pLo){ +#if (defined(__GNUC__) || defined(__clang__)) \ + && (defined(__x86_64__) || defined(__aarch64__) || defined(__riscv)) \ + && !defined(SQLITE_DISABLE_INTRINSIC) + __uint128_t r = (__uint128_t)a * b; + *pLo = (u64)r; + return (u64)(r>>64); +#elif defined(_WIN64) && !defined(SQLITE_DISABLE_INTRINSIC) + *pLo = a*b; + return __umulh(a, b); +#else + u64 a0 = (u32)a; + u64 a1 = a >> 32; + u64 b0 = (u32)b; + u64 b1 = b >> 32; + u64 a0b0 = a0 * b0; + u64 a1b1 = a1 * b1; + u64 a0b1 = a0 * b1; + u64 a1b0 = a1 * b0; + u64 t = (a0b0 >> 32) + (u32)a0b1 + (u32)a1b0; + *pLo = (a0b0 & UINT64_C(0xffffffff)) | (t << 32); + return a1b1 + (a0b1>>32) + (a1b0>>32) + (t>>32); +#endif +} + +/* +** A is an unsigned 96-bit integer formed by (a<<32)+aLo. +** B is an unsigned 64-bit integer. +** +** Compute the upper 96 bits of 160-bit result of A*B. +** +** Write ((A*B)>>64 & 0xffffffff) (the middle 32 bits of A*B) +** into *pLo. Return the upper 64 bits of A*B. ** -** Reference: -** T. J. Dekker, "A Floating-Point Technique for Extending the -** Available Precision". 1971-07-26. +** The lower 64 bits of A*B are discarded. */ -static void dekkerMul2(volatile double *x, double y, double yy){ - /* - ** The "volatile" keywords on parameter x[] and on local variables - ** below are needed force intermediate results to be truncated to - ** binary64 rather than be carried around in an extended-precision - ** format. The truncation is necessary for the Dekker algorithm to - ** work. Intel x86 floating point might omit the truncation without - ** the use of volatile. - */ - volatile double tx, ty, p, q, c, cc; - double hx, hy; - u64 m; - memcpy(&m, (void*)&x[0], 8); - m &= 0xfffffffffc000000LL; - memcpy(&hx, &m, 8); - tx = x[0] - hx; - memcpy(&m, &y, 8); - m &= 0xfffffffffc000000LL; - memcpy(&hy, &m, 8); - ty = y - hy; - p = hx*hy; - q = hx*ty + tx*hy; - c = p+q; - cc = p - c + q + tx*ty; - cc = x[0]*yy + x[1]*y + cc; - x[0] = c + cc; - x[1] = c - x[0]; - x[1] += cc; +static u64 sqlite3Multiply160(u64 a, u32 aLo, u64 b, u32 *pLo){ +#if (defined(__GNUC__) || defined(__clang__)) \ + && (defined(__x86_64__) || defined(__aarch64__) || defined(__riscv)) \ + && !defined(SQLITE_DISABLE_INTRINSIC) + __uint128_t r = (__uint128_t)a * b; + r += ((__uint128_t)aLo * b) >> 32; + *pLo = (r>>32)&0xffffffff; + return r>>64; +#elif defined(_WIN64) && !defined(SQLITE_DISABLE_INTRINSIC) + u64 r1_hi = __umulh(a,b); + u64 r1_lo = a*b; + u64 r2 = (__umulh((u64)aLo,b)<<32) + ((aLo*b)>>32); + u64 t = r1_lo + r2; + if( t<r1_lo ) r1_hi++; + *pLo = t>>32; + return r1_hi; +#else + u64 x2 = a>>32; + u64 x1 = a&0xffffffff; + u64 x0 = aLo; + u64 y1 = b>>32; + u64 y0 = b&0xffffffff; + u64 x2y1 = x2*y1; + u64 r4 = x2y1>>32; + u64 x2y0 = x2*y0; + u64 x1y1 = x1*y1; + u64 r3 = (x2y1 & 0xffffffff) + (x2y0 >>32) + (x1y1 >>32); + u64 x1y0 = x1*y0; + u64 x0y1 = x0*y1; + u64 r2 = (x2y0 & 0xffffffff) + (x1y1 & 0xffffffff) + + (x1y0 >>32) + (x0y1>>32); + u64 x0y0 = x0*y0; + u64 r1 = (x1y0 & 0xffffffff) + (x0y1 & 0xffffffff) + + (x0y0 >>32); + r2 += r1>>32; + r3 += r2>>32; + *pLo = r2&0xffffffff; + return (r4<<32) + r3; +#endif +} + +/* +** Return a u64 with the N-th bit set. +*/ +#define U64_BIT(N) (((u64)1)<<(N)) + +/* +** Range of powers of 10 that we need to deal with when converting +** IEEE754 doubles to and from decimal. +*/ +#define POWERSOF10_FIRST (-348) +#define POWERSOF10_LAST (+347) + +/* +** For any p between -348 and +347, return the integer part of +** +** pow(10,p) * pow(2,63-pow10to2(p)) +** +** Or, in other words, for any p in range, return the most significant +** 64 bits of pow(10,p). The pow(10,p) value is shifted left or right, +** as appropriate so the most significant 64 bits fit exactly into a +** 64-bit unsigned integer. +** +** Write into *pLo the next 32 significant bits of the answer after +** the first 64. +** +** Algorithm: +** +** (1) For p between 0 and 26, return the value directly from the aBase[] +** lookup table. +** +** (2) For p outside the range 0 to 26, use aScale[] for the initial value +** then refine that result (if necessary) by a single multiplication +** against aBase[]. +** +** The constant tables aBase[], aScale[], and aScaleLo[] are generated +** by the C program at ../tool/mkfptab.c run with the --round option. +*/ +static u64 powerOfTen(int p, u32 *pLo){ + static const u64 aBase[] = { + UINT64_C(0x8000000000000000), /* 0: 1.0e+0 << 63 */ + UINT64_C(0xa000000000000000), /* 1: 1.0e+1 << 60 */ + UINT64_C(0xc800000000000000), /* 2: 1.0e+2 << 57 */ + UINT64_C(0xfa00000000000000), /* 3: 1.0e+3 << 54 */ + UINT64_C(0x9c40000000000000), /* 4: 1.0e+4 << 50 */ + UINT64_C(0xc350000000000000), /* 5: 1.0e+5 << 47 */ + UINT64_C(0xf424000000000000), /* 6: 1.0e+6 << 44 */ + UINT64_C(0x9896800000000000), /* 7: 1.0e+7 << 40 */ + UINT64_C(0xbebc200000000000), /* 8: 1.0e+8 << 37 */ + UINT64_C(0xee6b280000000000), /* 9: 1.0e+9 << 34 */ + UINT64_C(0x9502f90000000000), /* 10: 1.0e+10 << 30 */ + UINT64_C(0xba43b74000000000), /* 11: 1.0e+11 << 27 */ + UINT64_C(0xe8d4a51000000000), /* 12: 1.0e+12 << 24 */ + UINT64_C(0x9184e72a00000000), /* 13: 1.0e+13 << 20 */ + UINT64_C(0xb5e620f480000000), /* 14: 1.0e+14 << 17 */ + UINT64_C(0xe35fa931a0000000), /* 15: 1.0e+15 << 14 */ + UINT64_C(0x8e1bc9bf04000000), /* 16: 1.0e+16 << 10 */ + UINT64_C(0xb1a2bc2ec5000000), /* 17: 1.0e+17 << 7 */ + UINT64_C(0xde0b6b3a76400000), /* 18: 1.0e+18 << 4 */ + UINT64_C(0x8ac7230489e80000), /* 19: 1.0e+19 >> 0 */ + UINT64_C(0xad78ebc5ac620000), /* 20: 1.0e+20 >> 3 */ + UINT64_C(0xd8d726b7177a8000), /* 21: 1.0e+21 >> 6 */ + UINT64_C(0x878678326eac9000), /* 22: 1.0e+22 >> 10 */ + UINT64_C(0xa968163f0a57b400), /* 23: 1.0e+23 >> 13 */ + UINT64_C(0xd3c21bcecceda100), /* 24: 1.0e+24 >> 16 */ + UINT64_C(0x84595161401484a0), /* 25: 1.0e+25 >> 20 */ + UINT64_C(0xa56fa5b99019a5c8), /* 26: 1.0e+26 >> 23 */ + }; + static const u64 aScale[] = { + UINT64_C(0x8049a4ac0c5811ae), /* 0: 1.0e-351 << 1229 */ + UINT64_C(0xcf42894a5dce35ea), /* 1: 1.0e-324 << 1140 */ + UINT64_C(0xa76c582338ed2621), /* 2: 1.0e-297 << 1050 */ + UINT64_C(0x873e4f75e2224e68), /* 3: 1.0e-270 << 960 */ + UINT64_C(0xda7f5bf590966848), /* 4: 1.0e-243 << 871 */ + UINT64_C(0xb080392cc4349dec), /* 5: 1.0e-216 << 781 */ + UINT64_C(0x8e938662882af53e), /* 6: 1.0e-189 << 691 */ + UINT64_C(0xe65829b3046b0afa), /* 7: 1.0e-162 << 602 */ + UINT64_C(0xba121a4650e4ddeb), /* 8: 1.0e-135 << 512 */ + UINT64_C(0x964e858c91ba2655), /* 9: 1.0e-108 << 422 */ + UINT64_C(0xf2d56790ab41c2a2), /* 10: 1.0e-81 << 333 */ + UINT64_C(0xc428d05aa4751e4c), /* 11: 1.0e-54 << 243 */ + UINT64_C(0x9e74d1b791e07e48), /* 12: 1.0e-27 << 153 */ + UINT64_C(0xcccccccccccccccc), /* 13: 1.0e-1 << 67 (special case) */ + UINT64_C(0xcecb8f27f4200f3a), /* 14: 1.0e+27 >> 26 */ + UINT64_C(0xa70c3c40a64e6c51), /* 15: 1.0e+54 >> 116 */ + UINT64_C(0x86f0ac99b4e8dafd), /* 16: 1.0e+81 >> 206 */ + UINT64_C(0xda01ee641a708de9), /* 17: 1.0e+108 >> 295 */ + UINT64_C(0xb01ae745b101e9e4), /* 18: 1.0e+135 >> 385 */ + UINT64_C(0x8e41ade9fbebc27d), /* 19: 1.0e+162 >> 475 */ + UINT64_C(0xe5d3ef282a242e81), /* 20: 1.0e+189 >> 564 */ + UINT64_C(0xb9a74a0637ce2ee1), /* 21: 1.0e+216 >> 654 */ + UINT64_C(0x95f83d0a1fb69cd9), /* 22: 1.0e+243 >> 744 */ + UINT64_C(0xf24a01a73cf2dccf), /* 23: 1.0e+270 >> 833 */ + UINT64_C(0xc3b8358109e84f07), /* 24: 1.0e+297 >> 923 */ + UINT64_C(0x9e19db92b4e31ba9), /* 25: 1.0e+324 >> 1013 */ + }; + static const unsigned int aScaleLo[] = { + 0x205b896d, /* 0: 1.0e-351 << 1229 */ + 0x52064cad, /* 1: 1.0e-324 << 1140 */ + 0xaf2af2b8, /* 2: 1.0e-297 << 1050 */ + 0x5a7744a7, /* 3: 1.0e-270 << 960 */ + 0xaf39a475, /* 4: 1.0e-243 << 871 */ + 0xbd8d794e, /* 5: 1.0e-216 << 781 */ + 0x547eb47b, /* 6: 1.0e-189 << 691 */ + 0x0cb4a5a3, /* 7: 1.0e-162 << 602 */ + 0x92f34d62, /* 8: 1.0e-135 << 512 */ + 0x3a6a07f9, /* 9: 1.0e-108 << 422 */ + 0xfae27299, /* 10: 1.0e-81 << 333 */ + 0xaa97e14c, /* 11: 1.0e-54 << 243 */ + 0x775ea265, /* 12: 1.0e-27 << 153 */ + 0xcccccccc, /* 13: 1.0e-1 << 67 (special case) */ + 0x00000000, /* 14: 1.0e+27 >> 26 */ + 0x999090b6, /* 15: 1.0e+54 >> 116 */ + 0x69a028bb, /* 16: 1.0e+81 >> 206 */ + 0xe80e6f48, /* 17: 1.0e+108 >> 295 */ + 0x5ec05dd0, /* 18: 1.0e+135 >> 385 */ + 0x14588f14, /* 19: 1.0e+162 >> 475 */ + 0x8f1668c9, /* 20: 1.0e+189 >> 564 */ + 0x6d953e2c, /* 21: 1.0e+216 >> 654 */ + 0x4abdaf10, /* 22: 1.0e+243 >> 744 */ + 0xbc633b39, /* 23: 1.0e+270 >> 833 */ + 0x0a862f81, /* 24: 1.0e+297 >> 923 */ + 0x6c07a2c2, /* 25: 1.0e+324 >> 1013 */ + }; + int g, n; + u64 s, x; + u32 lo; + + assert( p>=POWERSOF10_FIRST && p<=POWERSOF10_LAST ); + if( p<0 ){ + if( p==(-1) ){ + *pLo = aScaleLo[13]; + return aScale[13]; + } + g = p/27; + n = p%27; + if( n ){ + g--; + n += 27; + } + }else if( p<27 ){ + *pLo = 0; + return aBase[p]; + }else{ + g = p/27; + n = p%27; + } + s = aScale[g+13]; + if( n==0 ){ + *pLo = aScaleLo[g+13]; + return s; + } + x = sqlite3Multiply160(s,aScaleLo[g+13],aBase[n],&lo); + if( (U64_BIT(63) & x)==0 ){ + x = x<<1 | ((lo>>31)&1); + lo = (lo<<1) | 1; + } + *pLo = lo; + return x; +} + +/* +** pow10to2(x) computes floor(log2(pow(10,x))). +** pow2to10(y) computes floor(log10(pow(2,y))). +** +** Conceptually, pow10to2(p) converts a base-10 exponent p into +** a corresponding base-2 exponent, and pow2to10(e) converts a base-2 +** exponent into a base-10 exponent. +** +** The conversions are based on the observation that: +** +** ln(10.0)/ln(2.0) == 108853/32768 (approximately) +** ln(2.0)/ln(10.0) == 78913/262144 (approximately) +** +** These ratios are approximate, but they are accurate to 5 digits, +** which is close enough for the usage here. Right-shift is used +** for division so that rounding of negative numbers happens in the +** right direction. +*/ +static int pwr10to2(int p){ return (p*108853) >> 15; } +static int pwr2to10(int p){ return (p*78913) >> 18; } + +/* +** Count leading zeros for a 64-bit unsigned integer. +*/ +static int countLeadingZeros(u64 m){ +#if (defined(__GNUC__) || defined(__clang__)) \ + && !defined(SQLITE_DISABLE_INTRINSIC) + return __builtin_clzll(m); +#else + int n = 0; + if( m <= 0x00000000ffffffffULL) { n += 32; m <<= 32; } + if( m <= 0x0000ffffffffffffULL) { n += 16; m <<= 16; } + if( m <= 0x00ffffffffffffffULL) { n += 8; m <<= 8; } + if( m <= 0x0fffffffffffffffULL) { n += 4; m <<= 4; } + if( m <= 0x3fffffffffffffffULL) { n += 2; m <<= 2; } + if( m <= 0x7fffffffffffffffULL) { n += 1; } + return n; +#endif +} + +/* +** Given m and e, which represent a quantity r == m*pow(2,e), +** return values *pD and *pP such that r == (*pD)*pow(10,*pP), +** approximately. *pD should contain at least n significant digits. +** +** The input m is required to have its highest bit set. In other words, +** m should be left-shifted, and e decremented, to maximize the value of m. +*/ +static void sqlite3Fp2Convert10(u64 m, int e, int n, u64 *pD, int *pP){ + int p; + u64 h, d1; + u32 d2; + assert( n>=1 && n<=18 ); + p = n - 1 - pwr2to10(e+63); + h = sqlite3Multiply128(m, powerOfTen(p,&d2), &d1); + assert( -(e + pwr10to2(p) + 2) >= 0 ); + assert( -(e + pwr10to2(p) + 1) <= 63 ); + if( n==18 ){ + h >>= -(e + pwr10to2(p) + 2); + *pD = (h + ((h<<1)&2))>>1; + }else{ + *pD = h >> -(e + pwr10to2(p) + 1); + } + *pP = -p; +} + +/* +** Return an IEEE754 floating point value that approximates d*pow(10,p). +** +** The (current) algorithm is adapted from the work of Ross Cox at +** https://github.com/rsc/fpfmt +*/ +static double sqlite3Fp10Convert2(u64 d, int p){ + int b, lp, e, adj, s; + u32 pwr10l, mid1; + u64 pwr10h, x, hi, lo, sticky, u, m; + double r; + if( p<POWERSOF10_FIRST ) return 0.0; + if( p>POWERSOF10_LAST ) return INFINITY; + b = 64 - countLeadingZeros(d); + lp = pwr10to2(p); + e = 53 - b - lp; + if( e > 1074 ){ + if( e>=1130 ) return 0.0; + e = 1074; + } + s = -(e-(64-b) + lp + 3); + pwr10h = powerOfTen(p, &pwr10l); + if( pwr10l!=0 ){ + pwr10h++; + pwr10l = ~pwr10l; + } + x = d<<(64-b); + hi = sqlite3Multiply128(x,pwr10h,&lo); + mid1 = lo>>32; + sticky = 1; + if( (hi & (U64_BIT(s)-1))==0 ) { + u32 mid2 = sqlite3Multiply128(x,((u64)pwr10l)<<32,&lo)>>32; + sticky = (mid1-mid2 > 1); + hi -= mid1 < mid2; + } + u = (hi>>s) | sticky; + adj = (u >= U64_BIT(55)-2); + if( adj ){ + u = (u>>adj) | (u&1); + e -= adj; + } + m = (u + 1 + ((u>>2)&1)) >> 2; + if( e<=(-972) ) return INFINITY; + if((m & U64_BIT(52)) != 0){ + m = (m & ~U64_BIT(52)) | ((u64)(1075-e)<<52); + } + memcpy(&r,&m,8); + return r; } /* ** The string z[] is an text representation of a real number. ** Convert this string to a double and write it into *pResult. ** -** The string z[] is length bytes in length (bytes, not characters) and -** uses the encoding enc. The string is not necessarily zero-terminated. +** z[] must be UTF-8 and zero-terminated. +** +** Return positive if the result is a valid real number (or integer) and +** zero or negative if the string is empty or contains extraneous text. +** Lower bits of the return value contain addition information about the +** parse: +** +** bit 0 => Set if any prefix of the input is valid. Clear if +** there is no prefix of the input that can be seen as +** a valid floating point number. +** bit 1 => Set if the input contains a decimal point or eNNN +** clause. Zero if the input is an integer. +** bit 2 => The input is exactly 0.0, not an underflow from +** some value near zero. +** bit 3 => Set if there are more than about 19 significant +** digits in the input. ** -** Return TRUE if the result is a valid real number (or integer) and FALSE -** if the string is empty or contains extraneous text. More specifically -** return -** 1 => The input string is a pure integer -** 2 or more => The input has a decimal point or eNNN clause -** 0 or less => The input string is not a valid number -** -1 => Not a valid number, but has a valid prefix which -** includes a decimal point and/or an eNNN clause +** If the input contains a syntax error but begins with text that might +** be a valid number of some kind, then the result is negative. The +** result is only zero if no prefix of the input could be interpreted as +** a number. ** -** Valid numbers are in one of these formats: +** Leading and trailing whitespace is ignored. Valid numbers are in +** one of the formats below: ** ** [+-]digits[E[+-]digits] ** [+-]digits.[digits][E[+-]digits] ** [+-].digits[E[+-]digits] ** -** Leading and trailing whitespace is ignored for the purpose of determining -** validity. -** -** If some prefix of the input string is a valid number, this routine -** returns FALSE but it still converts the prefix and writes the result -** into *pResult. +** Algorithm sketch: Compute an unsigned 64-bit integer s and a base-10 +** exponent d such that the value encoding by the input is s*pow(10,d). +** Then invoke sqlite3Fp10Convert2() to calculated the closest possible +** IEEE754 double. The sign is added back afterwards, if the input string +** starts with a "-". The use of an unsigned 64-bit s mantissa means that +** only about the first 19 significant digits of the input can contribute +** to the result. This can result in suboptimal rounding decisions when +** correct rounding requires more than 19 input digits. For example, +** this routine renders "3500000000000000.2500001" as +** 3500000000000000.0 instead of 3500000000000000.5 because the decision +** to round up instead of using banker's rounding to round down is determined +** by the 23rd significant digit, which this routine ignores. It is not +** possible to do better without some kind of BigNum. */ -#if defined(_MSC_VER) -#pragma warning(disable : 4756) -#endif -int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ +int sqlite3AtoF(const char *zIn, double *pResult){ #ifndef SQLITE_OMIT_FLOATING_POINT - int incr; - const char *zEnd; - /* sign * significand * (10 ^ (esign * exponent)) */ - int sign = 1; /* sign of significand */ - u64 s = 0; /* significand */ - int d = 0; /* adjust exponent for shifting decimal point */ - int esign = 1; /* sign of exponent */ - int e = 0; /* exponent */ - int eValid = 1; /* True exponent is either not used or is well-formed */ - int nDigit = 0; /* Number of digits processed */ - int eType = 1; /* 1: pure integer, 2+: fractional -1 or less: bad UTF16 */ - u64 s2; /* round-tripped significand */ - double rr[2]; - - assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); - *pResult = 0.0; /* Default return value, in case of an error */ - if( length==0 ) return 0; - - if( enc==SQLITE_UTF8 ){ - incr = 1; - zEnd = z + length; - }else{ - int i; - incr = 2; - length &= ~1; - assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); - testcase( enc==SQLITE_UTF16LE ); - testcase( enc==SQLITE_UTF16BE ); - for(i=3-enc; i<length && z[i]==0; i+=2){} - if( i<length ) eType = -100; - zEnd = &z[i^1]; - z += (enc&1); - } - - /* skip leading spaces */ - while( z<zEnd && sqlite3Isspace(*z) ) z+=incr; - if( z>=zEnd ) return 0; - - /* get sign of significand */ - if( *z=='-' ){ - sign = -1; - z+=incr; - }else if( *z=='+' ){ - z+=incr; - } - - /* copy max significant digits to significand */ - while( z<zEnd && sqlite3Isdigit(*z) ){ - s = s*10 + (*z - '0'); - z+=incr; nDigit++; - if( s>=((LARGEST_UINT64-9)/10) ){ - /* skip non-significant significand digits - ** (increase exponent by d to shift decimal left) */ - while( z<zEnd && sqlite3Isdigit(*z) ){ z+=incr; d++; } + const unsigned char *z = (const unsigned char*)zIn; + int neg = 0; /* True for a negative value */ + u64 s = 0; /* mantissa */ + int d = 0; /* Value is s * pow(10,d) */ + int mState = 0; /* 1: digit seen 2: fp 4: hard-zero */ + unsigned v; /* Value of a single digit */ + + start_of_text: + if( (v = (unsigned)z[0] - '0')<10 ){ + parse_integer_part: + mState = 1; + s = v; + z++; + while( (v = (unsigned)z[0] - '0')<10 ){ + s = s*10 + v; + z++; + if( s>=(LARGEST_UINT64-9)/10 ){ + mState = 9; + while( sqlite3Isdigit(z[0]) ){ z++; d++; } + break; + } } + }else if( z[0]=='-' ){ + neg = 1; + z++; + if( (v = (unsigned)z[0] - '0')<10 ) goto parse_integer_part; + }else if( z[0]=='+' ){ + z++; + if( (v = (unsigned)z[0] - '0')<10 ) goto parse_integer_part; + }else if( sqlite3Isspace(z[0]) ){ + do{ z++; }while( sqlite3Isspace(z[0]) ); + goto start_of_text; + }else{ + s = 0; } - if( z>=zEnd ) goto do_atof_calc; /* if decimal point is present */ if( *z=='.' ){ - z+=incr; - eType++; - /* copy digits from after decimal to significand - ** (decrease exponent by d to shift decimal right) */ - while( z<zEnd && sqlite3Isdigit(*z) ){ - if( s<((LARGEST_UINT64-9)/10) ){ - s = s*10 + (*z - '0'); - d--; - nDigit++; - } - z+=incr; + z++; + if( sqlite3Isdigit(z[0]) ){ + mState |= 1; + do{ + if( s<(LARGEST_UINT64-9)/10 ){ + s = s*10 + z[0] - '0'; + d--; + }else{ + mState = 11; + } + }while( sqlite3Isdigit(*++z) ); + }else if( mState==0 ){ + *pResult = 0.0; + return 0; } + mState |= 2; + }else if( mState==0 ){ + *pResult = 0.0; + return 0; } - if( z>=zEnd ) goto do_atof_calc; /* if exponent is present */ if( *z=='e' || *z=='E' ){ - z+=incr; - eValid = 0; - eType++; - - /* This branch is needed to avoid a (harmless) buffer overread. The - ** special comment alerts the mutation tester that the correct answer - ** is obtained even if the branch is omitted */ - if( z>=zEnd ) goto do_atof_calc; /*PREVENTS-HARMLESS-OVERREAD*/ - + int esign; + z++; + /* get sign of exponent */ if( *z=='-' ){ esign = -1; - z+=incr; - }else if( *z=='+' ){ - z+=incr; + z++; + }else{ + esign = +1; + if( *z=='+' ){ + z++; + } } /* copy digits to exponent */ - while( z<zEnd && sqlite3Isdigit(*z) ){ - e = e<10000 ? (e*10 + (*z - '0')) : 10000; - z+=incr; - eValid = 1; + if( (v = (unsigned)z[0] - '0')<10 ){ + int exp = v; + z++; + mState |= 2; + while( (v = (unsigned)z[0] - '0')<10 ){ + exp = exp<10000 ? (exp*10 + v) : 10000; + z++; + } + d += esign*exp; + }else{ + z--; /* Leave z[0] at 'e' or '+' or '-', + ** so that the return is 0 or -1 */ } } - /* skip trailing spaces */ - while( z<zEnd && sqlite3Isspace(*z) ) z+=incr; - -do_atof_calc: - /* Zero is a special case */ + /* Convert s*pow(10,d) into real */ if( s==0 ){ - *pResult = sign<0 ? -0.0 : +0.0; - goto atof_return; - } - - /* adjust exponent by d, and update sign */ - e = (e*esign) + d; - - /* Try to adjust the exponent to make it smaller */ - while( e>0 && s<((LARGEST_UINT64-0x7ff)/10) ){ - s *= 10; - e--; - } - while( e<0 && (s%10)==0 ){ - s /= 10; - e++; - } - - rr[0] = (double)s; - assert( sizeof(s2)==sizeof(rr[0]) ); -#ifdef SQLITE_DEBUG - rr[1] = 18446744073709549568.0; - memcpy(&s2, &rr[1], sizeof(s2)); - assert( s2==0x43efffffffffffffLL ); -#endif - /* Largest double that can be safely converted to u64 - ** vvvvvvvvvvvvvvvvvvvvvv */ - if( rr[0]<=18446744073709549568.0 ){ - s2 = (u64)rr[0]; - rr[1] = s>=s2 ? (double)(s - s2) : -(double)(s2 - s); - }else{ - rr[1] = 0.0; - } - assert( rr[1]<=1.0e-10*rr[0] ); /* Equal only when rr[0]==0.0 */ - - if( e>0 ){ - while( e>=100 ){ - e -= 100; - dekkerMul2(rr, 1.0e+100, -1.5902891109759918046e+83); - } - while( e>=10 ){ - e -= 10; - dekkerMul2(rr, 1.0e+10, 0.0); - } - while( e>=1 ){ - e -= 1; - dekkerMul2(rr, 1.0e+01, 0.0); - } + *pResult = 0.0; + mState |= 4; }else{ - while( e<=-100 ){ - e += 100; - dekkerMul2(rr, 1.0e-100, -1.99918998026028836196e-117); - } - while( e<=-10 ){ - e += 10; - dekkerMul2(rr, 1.0e-10, -3.6432197315497741579e-27); - } - while( e<=-1 ){ - e += 1; - dekkerMul2(rr, 1.0e-01, -5.5511151231257827021e-18); - } + *pResult = sqlite3Fp10Convert2(s,d); } - *pResult = rr[0]+rr[1]; - if( sqlite3IsNaN(*pResult) ) *pResult = 1e300*1e300; - if( sign<0 ) *pResult = -*pResult; + if( neg ) *pResult = -*pResult; assert( !sqlite3IsNaN(*pResult) ); -atof_return: /* return true if number and no extra non-whitespace characters after */ - if( z==zEnd && nDigit>0 && eValid && eType>0 ){ - return eType; - }else if( eType>=2 && (eType==3 || eValid) && nDigit>0 ){ - return -1; - }else{ - return 0; + if( z[0]==0 ){ + return mState; + } + if( sqlite3Isspace(z[0]) ){ + do{ z++; }while( sqlite3Isspace(*z) ); + if( z[0]==0 ){ + return mState; + } } + return 0xfffffff0 | mState; #else - return !sqlite3Atoi64(z, pResult, length, enc); + return sqlite3Atoi64(z, pResult, strlen(z), SQLITE_UTF8)==0; #endif /* SQLITE_OMIT_FLOATING_POINT */ } -#if defined(_MSC_VER) -#pragma warning(default : 4756) + +/* +** Digit pairs used to convert a U64 or I64 into text, two digits +** at a time. +*/ +static const union { + char a[201]; + short int forceAlignment; +} sqlite3DigitPairs = { + "00010203040506070809" + "10111213141516171819" + "20212223242526272829" + "30313233343536373839" + "40414243444546474849" + "50515253545556575859" + "60616263646566676869" + "70717273747576777879" + "80818283848586878889" + "90919293949596979899" +}; + +/* +** ARMv6, ARMv7, PPC32 are known to not support hardware u64 division. +*/ +#if (defined(__arm__) && !defined(__aarch64__)) || \ + (defined(__ppc__) && !defined(__ppc64__)) +# define SQLITE_AVOID_U64_DIVIDE 1 #endif +#ifdef SQLITE_AVOID_U64_DIVIDE +/* +** Render an unsigned 64-bit integer as text onto the end of a 2-byte +** aligned buffer that is SQLITE_U64_DIGIT+1 bytes long. The last byte +** of the buffer will be filled with a \000 byte. +** +** Return the index into the buffer of the first byte. +** +** This routine is used on platforms where u64-division is slow because +** it is not available in hardware and has to be emulated in software. +** It seeks to minimize the number of u64 divisions and use u32 divisions +** instead. It is slower on platforms that have hardware u64 division, +** but much faster on platforms that do not. +*/ +static int sqlite3UInt64ToText(u64 v, char *zOut){ + u32 x32, kk; + int i; + zOut[SQLITE_U64_DIGITS] = 0; + i = SQLITE_U64_DIGITS; + assert( TWO_BYTE_ALIGNMENT(&sqlite3DigitPairs.a[0]) ); + assert( TWO_BYTE_ALIGNMENT(zOut) ); + while( (v>>32)!=0 ){ + u32 y, x0, x1, y0, y1; + x32 = v % 100000000; + v = v / 100000000; + y = x32 % 10000; + x32 /= 10000; + x1 = x32 / 100; + x0 = x32 % 100; + y1 = y / 100; + y0 = y % 100; + assert( i>=8 ); + i -= 8; + *(u16*)(&zOut[i]) = *(u16*)&sqlite3DigitPairs.a[x1*2]; + *(u16*)(&zOut[i+2]) = *(u16*)&sqlite3DigitPairs.a[x0*2]; + *(u16*)(&zOut[i+4]) = *(u16*)&sqlite3DigitPairs.a[y1*2]; + *(u16*)(&zOut[i+6]) = *(u16*)&sqlite3DigitPairs.a[y0*2]; + } + x32 = v; + while( x32>=10 ){ + kk = x32 % 100; + x32 = x32 / 100; + assert( TWO_BYTE_ALIGNMENT(&sqlite3DigitPairs.a[kk*2]) ); + assert( i>=2 ); + i -= 2; + assert( TWO_BYTE_ALIGNMENT(&zOut[i]) ); + *(u16*)(&zOut[i]) = *(u16*)&sqlite3DigitPairs.a[kk*2]; + } + if( x32 ){ + assert( i>0 ); + zOut[--i] = x32 + '0'; + } + return i; +} +#endif /* defined(SQLITE_AVOID_U64_DIVIDE) */ + /* ** Render an signed 64-bit integer as text. Store the result in zOut[] and ** return the length of the string that was stored, in bytes. The value @@ -729,23 +1076,39 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ int sqlite3Int64ToText(i64 v, char *zOut){ int i; u64 x; - char zTemp[22]; - if( v<0 ){ - x = (v==SMALLEST_INT64) ? ((u64)1)<<63 : (u64)-v; - }else{ + union { + char a[SQLITE_U64_DIGITS+1]; + u16 forceAlignment; + } u; + if( v>0 ){ x = v; + }else if( v==0 ){ + zOut[0] = '0'; + zOut[1] = 0; + return 1; + }else{ + x = (v==SMALLEST_INT64) ? ((u64)1)<<63 : (u64)-v; } - i = sizeof(zTemp)-2; - zTemp[sizeof(zTemp)-1] = 0; - while( 1 /*exit-by-break*/ ){ - zTemp[i] = (x%10) + '0'; - x = x/10; - if( x==0 ) break; - i--; - }; - if( v<0 ) zTemp[--i] = '-'; - memcpy(zOut, &zTemp[i], sizeof(zTemp)-i); - return sizeof(zTemp)-1-i; +#ifdef SQLITE_AVOID_U64_DIVIDE + i = sqlite3UInt64ToText(x, u.a); +#else + i = sizeof(u.a)-1; + u.a[i] = 0; + while( x>=10 ){ + int kk = (x%100)*2; + assert( TWO_BYTE_ALIGNMENT(&sqlite3DigitPairs.a[kk]) ); + assert( TWO_BYTE_ALIGNMENT(&u.a[i-2]) ); + *(u16*)(&u.a[i-2]) = *(u16*)&sqlite3DigitPairs.a[kk]; + i -= 2; + x /= 100; + } + if( x ){ + u.a[--i] = x + '0'; + } +#endif /* SQLITE_AVOID_U64_DIVIDE */ + if( v<0 ) u.a[--i] = '-'; + memcpy(zOut, &u.a[i], sizeof(u.a)-i); + return sizeof(u.a)-1-i; } /* @@ -799,8 +1162,8 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ int incr; u64 u = 0; int neg = 0; /* assume positive */ - int i; - int c = 0; + int i, j; + unsigned int c = 0; int nonNum = 0; /* True if input contains UTF16 with high byte non-zero */ int rc; /* Baseline return code */ const char *zStart; @@ -828,8 +1191,8 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ } zStart = zNum; while( zNum<zEnd && zNum[0]=='0' ){ zNum+=incr; } /* Skip leading zeros. */ - for(i=0; &zNum[i]<zEnd && (c=zNum[i])>='0' && c<='9'; i+=incr){ - u = u*10 + c - '0'; + for(i=0; &zNum[i]<zEnd && (c=(unsigned)zNum[i]-'0')<=9; i+=incr){ + u = u*10 + c; } testcase( i==18*incr ); testcase( i==19*incr ); @@ -866,14 +1229,14 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ return rc; }else{ /* zNum is a 19-digit numbers. Compare it against 9223372036854775808. */ - c = i>19*incr ? 1 : compare2pow63(zNum, incr); - if( c<0 ){ + j = i>19*incr ? 1 : compare2pow63(zNum, incr); + if( j<0 ){ /* zNum is less than 9223372036854775808 so it fits */ assert( u<=LARGEST_INT64 ); return rc; }else{ *pNum = neg ? SMALLEST_INT64 : LARGEST_INT64; - if( c>0 ){ + if( j>0 ){ /* zNum is greater than 9223372036854775808 so it overflows */ return 2; }else{ @@ -1002,7 +1365,7 @@ int sqlite3Atoi(const char *z){ ** representation. ** ** If iRound<=0 then round to -iRound significant digits to the -** the left of the decimal point, or to a maximum of mxRound total +** the right of the decimal point, or to a maximum of mxRound total ** significant digits. ** ** If iRound>0 round to min(iRound,mxRound) significant digits total. @@ -1015,13 +1378,14 @@ int sqlite3Atoi(const char *z){ ** The p->z[] array is *not* zero-terminated. */ void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRound){ - int i; - u64 v; - int e, exp = 0; - double rr[2]; + int i; /* Index into zBuf[] where to put next character */ + int n; /* Number of digits */ + u64 v; /* mantissa */ + int e, exp = 0; /* Base-2 and base-10 exponent */ + char *zBuf; /* Local alias for p->zBuf */ + char *z; /* Local alias for p->z */ p->isSpecial = 0; - p->z = p->zBuf; assert( mxRound>0 ); /* Convert negative numbers to positive. Deal with Infinity, 0.0, and @@ -1039,78 +1403,100 @@ void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRound){ p->sign = '+'; } memcpy(&v,&r,8); - e = v>>52; - if( (e&0x7ff)==0x7ff ){ + e = (v>>52)&0x7ff; + if( e==0x7ff ){ p->isSpecial = 1 + (v!=0x7ff0000000000000LL); p->n = 0; p->iDP = 0; + p->z = p->zBuf; return; } - - /* Multiply r by powers of ten until it lands somewhere in between - ** 1.0e+19 and 1.0e+17. - ** - ** Use Dekker-style double-double computation to increase the - ** precision. - ** - ** The error terms on constants like 1.0e+100 computed using the - ** decimal extension, for example as follows: - ** - ** SELECT decimal_exp(decimal_sub('1.0e+100',decimal(1.0e+100))); - */ - rr[0] = r; - rr[1] = 0.0; - if( rr[0]>9.223372036854774784e+18 ){ - while( rr[0]>9.223372036854774784e+118 ){ - exp += 100; - dekkerMul2(rr, 1.0e-100, -1.99918998026028836196e-117); - } - while( rr[0]>9.223372036854774784e+28 ){ - exp += 10; - dekkerMul2(rr, 1.0e-10, -3.6432197315497741579e-27); - } - while( rr[0]>9.223372036854774784e+18 ){ - exp += 1; - dekkerMul2(rr, 1.0e-01, -5.5511151231257827021e-18); - } + v &= 0x000fffffffffffffULL; + if( e==0 ){ + int nn = countLeadingZeros(v); + v <<= nn; + e = -1074 - nn; }else{ - while( rr[0]<9.223372036854774784e-83 ){ - exp -= 100; - dekkerMul2(rr, 1.0e+100, -1.5902891109759918046e+83); - } - while( rr[0]<9.223372036854774784e+07 ){ - exp -= 10; - dekkerMul2(rr, 1.0e+10, 0.0); - } - while( rr[0]<9.22337203685477478e+17 ){ - exp -= 1; - dekkerMul2(rr, 1.0e+01, 0.0); - } + v = (v<<11) | U64_BIT(63); + e -= 1086; } - v = rr[1]<0.0 ? (u64)rr[0]-(u64)(-rr[1]) : (u64)rr[0]+(u64)rr[1]; + sqlite3Fp2Convert10(v, e, (iRound<=0||iRound>=18)?18:iRound+1, &v, &exp); - /* Extract significant digits. */ - i = sizeof(p->zBuf)-1; + /* Extract significant digits, start at the right-most slot in p->zBuf + ** and working back to the right. "i" keeps track of the next slot in + ** which to store a digit. */ + assert( sizeof(p->zBuf)==SQLITE_U64_DIGITS+1 ); assert( v>0 ); - while( v ){ p->zBuf[i--] = (v%10) + '0'; v /= 10; } - assert( i>=0 && i<sizeof(p->zBuf)-1 ); - p->n = sizeof(p->zBuf) - 1 - i; - assert( p->n>0 ); - assert( p->n<sizeof(p->zBuf) ); - p->iDP = p->n + exp; + zBuf = p->zBuf; +#ifdef SQLITE_AVOID_U64_DIVIDE + i = sqlite3UInt64ToText(v, zBuf); +#else + i = SQLITE_U64_DIGITS; + while( v>=10 ){ + int kk = (v%100)*2; + assert( TWO_BYTE_ALIGNMENT(&sqlite3DigitPairs.a[kk]) ); + assert( TWO_BYTE_ALIGNMENT(&zBuf[i]) ); + assert( i-2>=0 ); + *(u16*)(&zBuf[i-2]) = *(u16*)&sqlite3DigitPairs.a[kk]; + i -= 2; + v /= 100; + } + if( v ){ + assert( v<10 ); + assert( i>0 ); + zBuf[--i] = v + '0'; + } +#endif /* SQLITE_AVOID_U64_DIVIDE */ + assert( i>=0 && i<SQLITE_U64_DIGITS ); + n = SQLITE_U64_DIGITS - i; /* Total number of digits extracted */ + assert( n>0 ); + assert( n<=SQLITE_U64_DIGITS ); + p->iDP = n + exp; if( iRound<=0 ){ iRound = p->iDP - iRound; - if( iRound==0 && p->zBuf[i+1]>='5' ){ + if( iRound==0 && zBuf[i]>='5' ){ iRound = 1; - p->zBuf[i--] = '0'; - p->n++; + zBuf[--i] = '0'; + n++; p->iDP++; } } - if( iRound>0 && (iRound<p->n || p->n>mxRound) ){ - char *z = &p->zBuf[i+1]; + z = &zBuf[i]; /* z points to the first digit */ + if( iRound>0 && (iRound<n || n>mxRound) ){ if( iRound>mxRound ) iRound = mxRound; - p->n = iRound; + if( iRound==17 ){ + /* If the precision is exactly 17, which only happens with the "!" + ** flag (ex: "%!.17g") then try to reduce the precision if that + ** yields text that will round-trip to the original floating-point. + ** value. Thus, for exaple, 49.47 will render as 49.47, rather than + ** as 49.469999999999999. */ + if( z[15]=='9' && z[14]=='9' ){ + int jj, kk; + u64 v2; + for(jj=14; jj>0 && z[jj-1]=='9'; jj--){} + if( jj==0 ){ + v2 = 1; + }else{ + v2 = z[0] - '0'; + for(kk=1; kk<jj; kk++) v2 = (v2*10) + z[kk] - '0'; + v2++; + } + if( r==sqlite3Fp10Convert2(v2, exp + n - jj) ){ + iRound = jj+1; + } + }else if( p->iDP>=n || (z[15]=='0' && z[14]=='0' && z[13]=='0') ){ + int jj, kk; + u64 v2; + assert( z[0]!='0' ); + for(jj=13; z[jj-1]=='0'; jj--){} + v2 = z[0] - '0'; + for(kk=1; kk<jj; kk++) v2 = (v2*10) + z[kk] - '0'; + if( r==sqlite3Fp10Convert2(v2, exp + n - jj) ){ + iRound = jj+1; + } + } + } + n = iRound; if( z[iRound]>='5' ){ int j = iRound-1; while( 1 /*exit-by-break*/ ){ @@ -1118,8 +1504,9 @@ void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRound){ if( z[j]<='9' ) break; z[j] = '0'; if( j==0 ){ - p->z[i--] = '1'; - p->n++; + z--; + z[0] = '1'; + n++; p->iDP++; break; }else{ @@ -1128,13 +1515,13 @@ void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRound){ } } } - p->z = &p->zBuf[i+1]; - assert( i+p->n < sizeof(p->zBuf) ); - assert( p->n>0 ); - while( p->z[p->n-1]=='0' ){ - p->n--; - assert( p->n>0 ); + assert( n>0 ); + while( z[n-1]=='0' ){ + n--; + assert( n>0 ); } + p->n = n; + p->z = z; } /* diff --git a/src/vacuum.c b/src/vacuum.c index 1b4838040..70e62e1ef 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -230,9 +230,11 @@ SQLITE_NOINLINE int sqlite3RunVacuum( pDb = &db->aDb[nDb]; assert( strcmp(pDb->zDbSName,zDbVacuum)==0 ); pTemp = pDb->pBt; + nRes = sqlite3BtreeGetRequestedReserve(pMain); if( pOut ){ sqlite3_file *id = sqlite3PagerFile(sqlite3BtreePager(pTemp)); i64 sz = 0; + const char *zFilename; if( id->pMethods!=0 && (sqlite3OsFileSize(id, &sz)!=SQLITE_OK || sz>0) ){ rc = SQLITE_ERROR; sqlite3SetString(pzErrMsg, db, "output file already exists"); @@ -244,8 +246,16 @@ SQLITE_NOINLINE int sqlite3RunVacuum( ** they are for the database being vacuumed, except that PAGER_CACHESPILL ** is always set. */ pgflags = db->aDb[iDb].safety_level | (db->flags & PAGER_FLAGS_MASK); + + /* If the VACUUM INTO target file is a URI filename and if the + ** "reserve=N" query parameter is present, reset the reserve to the + ** amount specified, if the amount is within range */ + zFilename = sqlite3BtreeGetFilename(pTemp); + if( ALWAYS(zFilename) ){ + int nNew = (int)sqlite3_uri_int64(zFilename, "reserve", nRes); + if( nNew>=0 && nNew<=255 ) nRes = nNew; + } } - nRes = sqlite3BtreeGetRequestedReserve(pMain); sqlite3BtreeSetCacheSize(pTemp, db->aDb[iDb].pSchema->cache_size); sqlite3BtreeSetSpillSize(pTemp, sqlite3BtreeSetSpillSize(pMain,0)); diff --git a/src/vdbe.c b/src/vdbe.c index b5a262e63..20b96d472 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -353,12 +353,11 @@ static int alsoAnInt(Mem *pRec, double rValue, i64 *piValue){ */ static void applyNumericAffinity(Mem *pRec, int bTryForInt){ double rValue; - u8 enc = pRec->enc; int rc; assert( (pRec->flags & (MEM_Str|MEM_Int|MEM_Real|MEM_IntReal))==MEM_Str ); - rc = sqlite3AtoF(pRec->z, &rValue, pRec->n, enc); + rc = sqlite3MemRealValueRC(pRec, &rValue); if( rc<=0 ) return; - if( rc==1 && alsoAnInt(pRec, rValue, &pRec->u.i) ){ + if( (rc&2)==0 && alsoAnInt(pRec, rValue, &pRec->u.i) ){ pRec->flags |= MEM_Int; }else{ pRec->u.r = rValue; @@ -438,7 +437,10 @@ int sqlite3_value_numeric_type(sqlite3_value *pVal){ int eType = sqlite3_value_type(pVal); if( eType==SQLITE_TEXT ){ Mem *pMem = (Mem*)pVal; + assert( pMem->db!=0 ); + sqlite3_mutex_enter(pMem->db->mutex); applyNumericAffinity(pMem, 0); + sqlite3_mutex_leave(pMem->db->mutex); eType = sqlite3_value_type(pVal); } return eType; @@ -471,15 +473,15 @@ static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ pMem->u.i = 0; return MEM_Int; } - rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); + rc = sqlite3MemRealValueRC(pMem, &pMem->u.r); if( rc<=0 ){ - if( rc==0 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1 ){ + if( (rc&2)==0 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1 ){ pMem->u.i = ix; return MEM_Int; }else{ return MEM_Real; } - }else if( rc==1 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)==0 ){ + }else if( (rc&2)==0 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)==0 ){ pMem->u.i = ix; return MEM_Int; } @@ -6620,20 +6622,17 @@ case OP_SorterInsert: { /* in2 */ break; } -/* Opcode: IdxDelete P1 P2 P3 * P5 +/* Opcode: IdxDelete P1 P2 P3 P4 * ** Synopsis: key=r[P2@P3] ** ** The content of P3 registers starting at register P2 form ** an unpacked index key. This opcode removes that entry from the ** index opened by cursor P1. ** -** If P5 is not zero, then raise an SQLITE_CORRUPT_INDEX error -** if no matching index entry is found. This happens when running -** an UPDATE or DELETE statement and the index entry to be updated -** or deleted is not found. For some uses of IdxDelete -** (example: the EXCEPT operator) it does not matter that no matching -** entry is found. For those cases, P5 is zero. Also, do not raise -** this (self-correcting and non-critical) error if in writable_schema mode. +** P4 is a pointer to an Index structure. +** +** Raise an SQLITE_CORRUPT_INDEX error if no matching index entry is found +** and not in writable_schema mode. */ case OP_IdxDelete: { VdbeCursor *pC; @@ -6656,13 +6655,22 @@ case OP_IdxDelete: { r.aMem = &aMem[pOp->p2]; rc = sqlite3BtreeIndexMoveto(pCrsr, &r, &res); if( rc ) goto abort_due_to_error; - if( res==0 ){ - rc = sqlite3BtreeDelete(pCrsr, BTREE_AUXDELETE); - if( rc ) goto abort_due_to_error; - }else if( pOp->p5 && !sqlite3WritableSchema(db) ){ - rc = sqlite3ReportError(SQLITE_CORRUPT_INDEX, __LINE__, "index corruption"); - goto abort_due_to_error; + if( res!=0 ){ + rc = sqlite3VdbeFindIndexKey(pCrsr, pOp->p4.pIdx, &r, &res, 0); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + if( res!=0 ){ + if( !sqlite3WritableSchema(db) ){ + rc = sqlite3ReportError( + SQLITE_CORRUPT_INDEX, __LINE__, "index corruption"); + goto abort_due_to_error; + } + pC->cacheStatus = CACHE_STALE; + pC->seekResult = 0; + break; + } } + rc = sqlite3BtreeDelete(pCrsr, BTREE_AUXDELETE); + if( rc ) goto abort_due_to_error; assert( pC->deferredMoveto==0 ); pC->cacheStatus = CACHE_STALE; pC->seekResult = 0; @@ -7289,6 +7297,58 @@ case OP_IntegrityCk: { sqlite3VdbeChangeEncoding(pIn1, encoding); goto check_for_interrupt; } + +/* Opcode: IFindKey P1 P2 P3 P4 * +** +** This instruction always follows an OP_Found with the same P1, P2 and P3 +** values as this instruction and a non-zero P4 value. The P4 value to +** this opcode is of type P4_INDEX and contains a pointer to the Index +** object of for the index being searched. +** +** This opcode uses sqlite3VdbeFindIndexKey() to search around the current +** cursor location for an index key that exactly matches all fields that +** are not indexed expressions or references to VIRTUAL generated columns, +** and either exactly match or are real numbers that are within 2 ULPs of +** each other if the don't match. +** +** To put it another way, this opcode looks for nearby index entries that +** are very close to the search key, but which might have small differences +** in floating-point values that come via an expression. +** +** If no nearby alternative entry is found in cursor P1, then jump to P2. +** But if a close match is found, fall through. +** +** This opcode is used by PRAGMA integrity_check to help distinguish +** between truely corrupt indexes and expression indexes that are holding +** floating-point values that are off by one or two ULPs. +*/ +case OP_IFindKey: { /* jump, in3 */ + VdbeCursor *pC; + int res; + UnpackedRecord r; + + assert( pOp[-1].opcode==OP_Found ); + assert( pOp[-1].p1==pOp->p1 ); + assert( pOp[-1].p3==pOp->p3 ); + pC = p->apCsr[pOp->p1]; + assert( pOp->p4type==P4_INDEX ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0 ); + assert( pC->isTable==0 ); + + memset(&r, 0, sizeof(r)); + r.aMem = &aMem[pOp->p3]; + r.nField = pOp->p4.pIdx->nColumn; + r.pKeyInfo = pC->pKeyInfo; + + rc = sqlite3VdbeFindIndexKey(pC->uc.pCursor, pOp->p4.pIdx, &r, &res, 1); + if( rc || res!=0 ){ + rc = SQLITE_OK; + goto jump_to_p2; + } + pC->nullRow = 0; + break; +}; #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ /* Opcode: RowSetAdd P1 P2 * * * diff --git a/src/vdbe.h b/src/vdbe.h index 28df764bc..7120e38d8 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -75,6 +75,7 @@ struct VdbeOp { SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ Table *pTab; /* Used when p4type is P4_TABLE */ SubrtnSig *pSubrtnSig; /* Used when p4type is P4_SUBRTNSIG */ + Index *pIdx; /* Used when p4type is P4_INDEX */ #ifdef SQLITE_ENABLE_CURSOR_HINTS Expr *pExpr; /* Used when p4type is P4_EXPR */ #endif @@ -129,20 +130,21 @@ typedef struct VdbeOpList VdbeOpList; #define P4_INT32 (-3) /* P4 is a 32-bit signed integer */ #define P4_SUBPROGRAM (-4) /* P4 is a pointer to a SubProgram structure */ #define P4_TABLE (-5) /* P4 is a pointer to a Table structure */ +#define P4_INDEX (-6) /* P4 is a pointer to an Index structure */ /* Above do not own any resources. Must free those below */ -#define P4_FREE_IF_LE (-6) -#define P4_DYNAMIC (-6) /* Pointer to memory from sqliteMalloc() */ -#define P4_FUNCDEF (-7) /* P4 is a pointer to a FuncDef structure */ -#define P4_KEYINFO (-8) /* P4 is a pointer to a KeyInfo structure */ -#define P4_EXPR (-9) /* P4 is a pointer to an Expr tree */ -#define P4_MEM (-10) /* P4 is a pointer to a Mem* structure */ -#define P4_VTAB (-11) /* P4 is a pointer to an sqlite3_vtab structure */ -#define P4_REAL (-12) /* P4 is a 64-bit floating point value */ -#define P4_INT64 (-13) /* P4 is a 64-bit signed integer */ -#define P4_INTARRAY (-14) /* P4 is a vector of 32-bit integers */ -#define P4_FUNCCTX (-15) /* P4 is a pointer to an sqlite3_context object */ -#define P4_TABLEREF (-16) /* Like P4_TABLE, but reference counted */ -#define P4_SUBRTNSIG (-17) /* P4 is a SubrtnSig pointer */ +#define P4_FREE_IF_LE (-7) +#define P4_DYNAMIC (-7) /* Pointer to memory from sqliteMalloc() */ +#define P4_FUNCDEF (-8) /* P4 is a pointer to a FuncDef structure */ +#define P4_KEYINFO (-9) /* P4 is a pointer to a KeyInfo structure */ +#define P4_EXPR (-10) /* P4 is a pointer to an Expr tree */ +#define P4_MEM (-11) /* P4 is a pointer to a Mem* structure */ +#define P4_VTAB (-12) /* P4 is a pointer to an sqlite3_vtab structure */ +#define P4_REAL (-13) /* P4 is a 64-bit floating point value */ +#define P4_INT64 (-14) /* P4 is a 64-bit signed integer */ +#define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */ +#define P4_FUNCCTX (-16) /* P4 is a pointer to an sqlite3_context object */ +#define P4_TABLEREF (-17) /* Like P4_TABLE, but reference counted */ +#define P4_SUBRTNSIG (-18) /* P4 is a SubrtnSig pointer */ /* Error message codes for OP_Halt */ #define P5_ConstraintNotNull 1 @@ -186,7 +188,7 @@ typedef struct VdbeOpList VdbeOpList; ** Additional non-public SQLITE_PREPARE_* flags */ #define SQLITE_PREPARE_SAVESQL 0x80 /* Preserve SQL text */ -#define SQLITE_PREPARE_MASK 0x1f /* Mask of public flags */ +#define SQLITE_PREPARE_MASK 0x3f /* Mask of public flags */ /* ** Prototypes for the VDBE interface. See comments on the implementation diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 8b68c339a..24c422f79 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -630,6 +630,7 @@ void sqlite3VdbeMemShallowCopy(Mem*, const Mem*, int); void sqlite3VdbeMemMove(Mem*, Mem*); int sqlite3VdbeMemNulTerminate(Mem*); int sqlite3VdbeMemSetStr(Mem*, const char*, i64, u8, void(*)(void*)); +int sqlite3VdbeMemSetText(Mem*, const char*, i64, void(*)(void*)); void sqlite3VdbeMemSetInt64(Mem*, i64); #ifdef SQLITE_OMIT_FLOATING_POINT # define sqlite3VdbeMemSetDouble sqlite3VdbeMemSetInt64 @@ -648,13 +649,14 @@ int sqlite3VdbeMemSetZeroBlob(Mem*,int); int sqlite3VdbeMemIsRowSet(const Mem*); #endif int sqlite3VdbeMemSetRowSet(Mem*); -void sqlite3VdbeMemZeroTerminateIfAble(Mem*); +int sqlite3VdbeMemZeroTerminateIfAble(Mem*); int sqlite3VdbeMemMakeWriteable(Mem*); int sqlite3VdbeMemStringify(Mem*, u8, u8); int sqlite3IntFloatCompare(i64,double); i64 sqlite3VdbeIntValue(const Mem*); int sqlite3VdbeMemIntegerify(Mem*); double sqlite3VdbeRealValue(Mem*); +int sqlite3MemRealValueRC(Mem*, double*); int sqlite3VdbeBooleanValue(Mem*, int ifNull); void sqlite3VdbeIntegerAffinity(Mem*); int sqlite3VdbeMemRealify(Mem*); @@ -685,6 +687,7 @@ void sqlite3VdbePreUpdateHook( Vdbe*,VdbeCursor*,int,const char*,Table*,i64,int,int); #endif int sqlite3VdbeTransferError(Vdbe *p); +int sqlite3VdbeFindIndexKey(BtCursor*, Index*, UnpackedRecord*, int*, int); int sqlite3VdbeSorterInit(sqlite3 *, int, VdbeCursor *); void sqlite3VdbeSorterReset(sqlite3 *, VdbeSorter *); diff --git a/src/vdbeapi.c b/src/vdbeapi.c index ec849cc4f..9fd4715ce 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -392,7 +392,23 @@ static void setResultStrOrError( void (*xDel)(void*) /* Destructor function */ ){ Mem *pOut = pCtx->pOut; - int rc = sqlite3VdbeMemSetStr(pOut, z, n, enc, xDel); + int rc; + if( enc==SQLITE_UTF8 ){ + rc = sqlite3VdbeMemSetText(pOut, z, n, xDel); + }else if( enc==SQLITE_UTF8_ZT ){ + /* It is usually considered improper to assert() on an input. However, + ** the following assert() is checking for inputs that are documented + ** to result in undefined behavior. */ + assert( z==0 + || n<0 + || n>pOut->db->aLimit[SQLITE_LIMIT_LENGTH] + || z[n]==0 + ); + rc = sqlite3VdbeMemSetText(pOut, z, n, xDel); + pOut->flags |= MEM_Term; + }else{ + rc = sqlite3VdbeMemSetStr(pOut, z, n, enc, xDel); + } if( rc ){ if( rc==SQLITE_TOOBIG ){ sqlite3_result_error_toobig(pCtx); @@ -585,7 +601,7 @@ void sqlite3_result_text64( #endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); assert( xDel!=SQLITE_DYNAMIC ); - if( enc!=SQLITE_UTF8 ){ + if( enc!=SQLITE_UTF8 && enc!=SQLITE_UTF8_ZT ){ if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; n &= ~(u64)1; } @@ -736,6 +752,8 @@ static int doWalCallbacks(sqlite3 *db){ } } } +#else + UNUSED_PARAMETER(db); #endif return rc; } @@ -1692,13 +1710,25 @@ static int bindText( assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */ if( zData!=0 ){ pVar = &p->aVar[i-1]; - rc = sqlite3VdbeMemSetStr(pVar, zData, nData, encoding, xDel); - if( rc==SQLITE_OK ){ - if( encoding==0 ){ - pVar->enc = ENC(p->db); - }else{ - rc = sqlite3VdbeChangeEncoding(pVar, ENC(p->db)); - } + if( encoding==SQLITE_UTF8 ){ + rc = sqlite3VdbeMemSetText(pVar, zData, nData, xDel); + }else if( encoding==SQLITE_UTF8_ZT ){ + /* It is usually consider improper to assert() on an input. + ** However, the following assert() is checking for inputs + ** that are documented to result in undefined behavior. */ + assert( zData==0 + || nData<0 + || nData>pVar->db->aLimit[SQLITE_LIMIT_LENGTH] + || ((u8*)zData)[nData]==0 + ); + rc = sqlite3VdbeMemSetText(pVar, zData, nData, xDel); + pVar->flags |= MEM_Term; + }else{ + rc = sqlite3VdbeMemSetStr(pVar, zData, nData, encoding, xDel); + if( encoding==0 ) pVar->enc = ENC(p->db); + } + if( rc==SQLITE_OK && encoding!=0 ){ + rc = sqlite3VdbeChangeEncoding(pVar, ENC(p->db)); } if( rc ){ sqlite3Error(p->db, rc); @@ -1810,7 +1840,7 @@ int sqlite3_bind_text64( unsigned char enc ){ assert( xDel!=SQLITE_DYNAMIC ); - if( enc!=SQLITE_UTF8 ){ + if( enc!=SQLITE_UTF8 && enc!=SQLITE_UTF8_ZT ){ if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; nData &= ~(u64)1; } diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 5368c0c42..9448b68f1 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2003,6 +2003,10 @@ char *sqlite3VdbeDisplayP4(sqlite3 *db, Op *pOp){ zP4 = pOp->p4.pTab->zName; break; } + case P4_INDEX: { + zP4 = pOp->p4.pIdx->zName; + break; + } case P4_SUBRTNSIG: { SubrtnSig *pSig = pOp->p4.pSubrtnSig; sqlite3_str_appendf(&x, "subrtnsig:%d,%s", pSig->selId, pSig->zAff); @@ -2901,7 +2905,7 @@ int sqlite3VdbeSetColName( } assert( p->aColName!=0 ); pColName = &(p->aColName[idx+var*p->nResAlloc]); - rc = sqlite3VdbeMemSetStr(pColName, zName, -1, SQLITE_UTF8, xDel); + rc = sqlite3VdbeMemSetText(pColName, zName, -1, xDel); assert( rc!=0 || !zName || (pColName->flags&MEM_Term)!=0 ); return rc; } @@ -5393,6 +5397,223 @@ void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ } } +/* +** Helper function for vdbeIsMatchingIndexKey(). Return true if column +** iCol should be ignored when comparing a record with a record from +** an index on disk. The field should be ignored if: +** +** * the corresponding bit in mask is set, and +** * either: +** - bIntegrity is false, or +** - the two Mem values are both real values that differ by +** BTREE_ULPDISTORTION or fewer ULPs. +*/ +static int vdbeSkipField( + Bitmask mask, /* Mask of indexed expression fields */ + int iCol, /* Column of index being considered */ + Mem *pMem1, /* Expected index value */ + Mem *pMem2, /* Actual indexed value */ + int bIntegrity /* True if running PRAGMA integrity_check */ +){ +#define BTREE_ULPDISTORTION 2 + if( iCol>=BMS || (mask & MASKBIT(iCol))==0 ) return 0; + if( bIntegrity==0 ) return 1; + if( (pMem1->flags & MEM_Real) && (pMem2->flags & MEM_Real) ){ + u64 m1, m2; + memcpy(&m1,&pMem1->u.r,8); + memcpy(&m2,&pMem2->u.r,8); + if( (m1<m2 ? m2-m1 : m1-m2) <= BTREE_ULPDISTORTION ){ + return 1; + } + } + return 0; +} + +/* +** This function compares the unpacked record with the current key that +** cursor pCur points to. If bInt is false, all fields for which the +** corresponding bit in parameter "mask" is set are ignored. Or, if +** bInt is true, then a difference of BTREE_ULPDISTORTION or fewer ULPs +** in real values is overlooked for fields with the corresponding bit +** set in mask. +** +** Return the usual less than zero, zero, or greater than zero if the +** remaining fields of the cursor cursor key are less than, equal to or +** greater than those in (*p). +*/ +static int vdbeIsMatchingIndexKey( + BtCursor *pCur, /* Cursor open on index */ + int bInt, /* True for integrity_check-style search */ + Bitmask mask, /* Mask of columns to skip */ + UnpackedRecord *p, /* Index key being deleted */ + int *piRes /* 0 for a match, non-zero for not a match */ +){ + u8 *aRec = 0; + u32 nRec = 0; + Mem mem; + int rc = SQLITE_OK; + + memset(&mem, 0, sizeof(mem)); + mem.enc = p->pKeyInfo->enc; + mem.db = p->pKeyInfo->db; + nRec = sqlite3BtreePayloadSize(pCur); + if( nRec>0x7fffffff ){ + return SQLITE_CORRUPT_BKPT; + } + + /* Allocate 5 extra bytes at the end of the buffer. This allows the + ** getVarint32() call below to read slightly past the end of the buffer + ** if the record is corrupt. */ + aRec = sqlite3MallocZero(nRec+5); + if( aRec==0 ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + rc = sqlite3BtreePayload(pCur, 0, nRec, aRec); + } + + if( rc==SQLITE_OK ){ + u32 szHdr = 0; /* Size of record header in bytes */ + u32 idxHdr = 0; /* Current index in header */ + + idxHdr = getVarint32(aRec, szHdr); + if( szHdr>98307 ){ + rc = SQLITE_CORRUPT; + }else{ + int res = 0; /* Result of this function call */ + u32 idxRec = szHdr; /* Index of next field in record body */ + int ii = 0; /* Iterator variable */ + + int nCol = p->pKeyInfo->nAllField; + for(ii=0; ii<nCol && rc==SQLITE_OK; ii++){ + u32 iSerial = 0; + int nSerial = 0; + + if( idxHdr>=szHdr ){ + rc = SQLITE_CORRUPT_BKPT; + break; + } + idxHdr += getVarint32(&aRec[idxHdr], iSerial); + nSerial = sqlite3VdbeSerialTypeLen(iSerial); + if( (idxRec+nSerial)>nRec ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + sqlite3VdbeSerialGet(&aRec[idxRec], iSerial, &mem); + if( vdbeSkipField(mask, ii, &p->aMem[ii], &mem, bInt)==0 ){ + res = sqlite3MemCompare(&mem, &p->aMem[ii], p->pKeyInfo->aColl[ii]); + if( res!=0 ) break; + } + } + idxRec += sqlite3VdbeSerialTypeLen(iSerial); + } + + *piRes = res; + } + } + + sqlite3_free(aRec); + return rc; +} + +/* +** This is called when the record in (*p) should be found in the index +** opened by cursor pCur, but was not. This may happen as part of a DELETE +** operation or an integrity check. +** +** One reason that an exact match was not found may be the EIIB bug - that +** a text-to-float conversion may have caused a real value in record (*p) +** to be slightly different from its counterpart on disk. This function +** attempts to find the right index record. If it does find the right +** record, it leaves *pCur pointing to it and sets (*pRes) to 0 before +** returning. Otherwise, (*pRes) is set to non-zero and an SQLite error +** code returned. +** +** The algorithm used to find the correct record is: +** +** * Scan up to BTREE_FDK_RANGE entries either side of the current entry. +** If parameter bIntegrity is false, then all fields that are indexed +** expressions or virtual table columns are omitted from the comparison. +** If bIntegrity is true, then small differences in real values in +** such fields are overlooked, but they are not omitted from the comparison +** altogether. +** +** * If the above fails to find an entry and bIntegrity is false, search +** the entire index. +*/ +int sqlite3VdbeFindIndexKey( + BtCursor *pCur, + Index *pIdx, + UnpackedRecord *p, + int *pRes, + int bIntegrity +){ +#define BTREE_FDK_RANGE 10 + int nStep = 0; + int res = 1; + int rc = SQLITE_OK; + int ii = 0; + + /* Calculate a mask based on the first 64 columns of the index. The mask + ** bit is set if the corresponding index field is either an expression + ** or a virtual column of the table. */ + Bitmask mask = 0; + for(ii=0; ii<MIN(pIdx->nColumn, BMS); ii++){ + int iCol = pIdx->aiColumn[ii]; + if( (iCol==XN_EXPR) + || (iCol>=0 && (pIdx->pTable->aCol[iCol].colFlags & COLFLAG_VIRTUAL)) + ){ + mask |= MASKBIT(ii); + } + } + + /* If the mask is 0 at this point, then the index contains no expressions + ** or virtual columns. So do not search for a match - return so that the + ** caller may declare the db corrupt immediately. Or, if mask is non-zero, + ** proceed. */ + if( mask!=0 ){ + + /* Move the cursor back BTREE_FDK_RANGE entries. If this hits an EOF, + ** position the cursor at the first entry in the index and set nStep + ** to -1 so that the first loop below scans the entire index. Otherwise, + ** set nStep to BTREE_FDK_RANGE*2 so that the first loop below scans + ** just that many entries. */ + for(ii=0; sqlite3BtreeEof(pCur)==0 && ii<BTREE_FDK_RANGE; ii++){ + rc = sqlite3BtreePrevious(pCur, 0); + } + if( rc==SQLITE_DONE ){ + rc = sqlite3BtreeFirst(pCur, &res); + nStep = -1; + }else{ + nStep = BTREE_FDK_RANGE*2; + } + + /* This loop runs at most twice to search for a key with matching PK + ** fields in the index. The second iteration always searches the entire + ** index. The first iteration searches nStep entries starting with the + ** current cursor entry if (nStep>=0), or the entire index if (nStep<0). */ + while( sqlite3BtreeCursorIsValidNN(pCur) ){ + for(ii=0; rc==SQLITE_OK && (ii<nStep || nStep<0); ii++){ + rc = vdbeIsMatchingIndexKey(pCur, bIntegrity, mask, p, &res); + if( res==0 || rc!=SQLITE_OK ) break; + rc = sqlite3BtreeNext(pCur, 0); + } + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + assert( res!=0 ); + } + if( nStep<0 || rc!=SQLITE_OK || res==0 || bIntegrity ) break; + + /* The first, non-exhaustive, search failed to find an entry with + ** matching PK fields. So restart for an exhaustive search of the + ** entire index. */ + nStep = -1; + rc = sqlite3BtreeFirst(pCur, &res); + } + } + + *pRes = res; + return rc; +} + #ifndef SQLITE_OMIT_DATETIME_FUNCS /* ** Cause a function to throw an error if it was call from OP_PureFunc diff --git a/src/vdbemem.c b/src/vdbemem.c index 144f88936..77aedbe9f 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -107,21 +107,27 @@ static void vdbeMemRenderNum(int sz, char *zBuf, Mem *p){ StrAccum acc; assert( p->flags & (MEM_Int|MEM_Real|MEM_IntReal) ); assert( sz>22 ); - if( p->flags & MEM_Int ){ -#if GCC_VERSION>=7000000 - /* Work-around for GCC bug - ** https://gcc.gnu.org/bugzilla/show_bug.cgi?id=96270 */ + if( p->flags & (MEM_Int|MEM_IntReal) ){ +#if GCC_VERSION>=7000000 && GCC_VERSION<15000000 && defined(__i386__) + /* Work-around for GCC bug or bugs: + ** https://gcc.gnu.org/bugzilla/show_bug.cgi?id=96270 + ** https://gcc.gnu.org/bugzilla/show_bug.cgi?id=114659 + ** The problem appears to be fixed in GCC 15 */ i64 x; - assert( (p->flags&MEM_Int)*2==sizeof(x) ); - memcpy(&x, (char*)&p->u, (p->flags&MEM_Int)*2); + assert( (MEM_Str&~p->flags)*4==sizeof(x) ); + memcpy(&x, (char*)&p->u, (MEM_Str&~p->flags)*4); p->n = sqlite3Int64ToText(x, zBuf); #else p->n = sqlite3Int64ToText(p->u.i, zBuf); #endif + if( p->flags & MEM_IntReal ){ + memcpy(zBuf+p->n,".0", 3); + p->n += 2; + } }else{ sqlite3StrAccumInit(&acc, 0, zBuf, sz, 0); - sqlite3_str_appendf(&acc, "%!.15g", - (p->flags & MEM_IntReal)!=0 ? (double)p->u.i : p->u.r); + sqlite3_str_appendf(&acc, "%!.*g", + (p->db ? p->db->nFpDigit : 17), p->u.r); assert( acc.zText==zBuf && acc.mxAlloc<=0 ); zBuf[acc.nChar] = 0; /* Fast version of sqlite3StrAccumFinish(&acc) */ p->n = acc.nChar; @@ -170,6 +176,9 @@ int sqlite3VdbeMemValidStrRep(Mem *p){ assert( p->enc==SQLITE_UTF8 || p->z[((p->n+1)&~1)+1]==0 ); } if( (p->flags & (MEM_Int|MEM_Real|MEM_IntReal))==0 ) return 1; + if( p->db==0 ){ + return 1; /* db->nFpDigit required to validate p->z[] */ + } memcpy(&tmp, p, sizeof(tmp)); vdbeMemRenderNum(sizeof(zBuf), zBuf, &tmp); z = p->z; @@ -320,13 +329,16 @@ int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ ** ** This is an optimization. Correct operation continues even if ** this routine is a no-op. +** +** Return true if the strig is zero-terminated after this routine is +** called and false if it is not. */ -void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ +int sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ if( (pMem->flags & (MEM_Str|MEM_Term|MEM_Ephem|MEM_Static))!=MEM_Str ){ /* pMem must be a string, and it cannot be an ephemeral or static string */ - return; + return 0; } - if( pMem->enc!=SQLITE_UTF8 ) return; + if( pMem->enc!=SQLITE_UTF8 ) return 0; assert( pMem->z!=0 ); if( pMem->flags & MEM_Dyn ){ if( pMem->xDel==sqlite3_free @@ -334,18 +346,19 @@ void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ ){ pMem->z[pMem->n] = 0; pMem->flags |= MEM_Term; - return; + return 1; } if( pMem->xDel==sqlite3RCStrUnref ){ /* Blindly assume that all RCStr objects are zero-terminated */ pMem->flags |= MEM_Term; - return; + return 1; } }else if( pMem->szMalloc >= pMem->n+1 ){ pMem->z[pMem->n] = 0; pMem->flags |= MEM_Term; - return; + return 1; } + return 0; } /* @@ -643,18 +656,117 @@ i64 sqlite3VdbeIntValue(const Mem *pMem){ } } +/* +** This routine implements the uncommon and slower path for +** sqlite3MemRealValueRC() that has to deal with input strings +** that are not UTF8 or that are not zero-terminated. It is +** broken out into a separate no-inline routine so that the +** main sqlite3MemRealValueRC() routine can avoid unnecessary +** stack pushes. +** +** A text->float translation of pMem->z is written into *pValue. +** +** Result code invariants: +** +** rc==0 => ERROR: Input string not well-formed, or OOM +** rc<0 => Some prefix of the input is well-formed +** rc>0 => All of the input is well-formed +** (rc&2)==0 => The number is expressed as an integer, with no +** decimal point or eNNN suffix. +*/ +static SQLITE_NOINLINE int sqlite3MemRealValueRCSlowPath( + Mem *pMem, + double *pValue +){ + int rc = SQLITE_OK; + *pValue = 0.0; + if( pMem->enc==SQLITE_UTF8 ){ + char *zCopy = sqlite3DbStrNDup(pMem->db, pMem->z, pMem->n); + if( zCopy ){ + rc = sqlite3AtoF(zCopy, pValue); + sqlite3DbFree(pMem->db, zCopy); + } + return rc; + }else{ + int n, i, j; + char *zCopy; + const char *z; + + n = pMem->n & ~1; + zCopy = sqlite3DbMallocRaw(pMem->db, n/2 + 2); + if( zCopy ){ + z = pMem->z; + if( pMem->enc==SQLITE_UTF16LE ){ + for(i=j=0; i<n-1; i+=2, j++){ + zCopy[j] = z[i]; + if( z[i+1]!=0 ) break; + } + }else{ + for(i=j=0; i<n-1; i+=2, j++){ + if( z[i]!=0 ) break; + zCopy[j] = z[i+1]; + } + } + assert( j<=n/2 ); + zCopy[j] = 0; + rc = sqlite3AtoF(zCopy, pValue); + if( i<n ) rc = -100; + sqlite3DbFree(pMem->db, zCopy); + } + return rc; + } +} + +/* +** Invoke sqlite3AtoF() on the text value of pMem. Write the +** translation of the text input into *pValue. +** +** The caller must ensure that pMem->db!=0 and that pMem is in +** mode MEM_Str or MEM_Blob. +** +** Result code invariants: +** +** rc==0 => ERROR: Input string not well-formed, or OOM +** rc<0 => Some prefix of the input is well-formed +** rc>0 => All of the input is well-formed +** (rc&2)==0 => The number is expressed as an integer, with no +** decimal point or eNNN suffix. +*/ +int sqlite3MemRealValueRC(Mem *pMem, double *pValue){ + testcase( pMem->db==0 ); + assert( pMem->flags & (MEM_Str|MEM_Blob) ); + if( pMem->z==0 ){ + *pValue = 0.0; + return 0; + }else if( pMem->enc==SQLITE_UTF8 + && ((pMem->flags & MEM_Term)!=0 || sqlite3VdbeMemZeroTerminateIfAble(pMem)) + ){ + return sqlite3AtoF(pMem->z, pValue); + }else if( pMem->n==0 ){ + *pValue = 0.0; + return 0; + }else{ + return sqlite3MemRealValueRCSlowPath(pMem, pValue); + } +} + +/* +** This routine acts as a bridge from sqlite3VdbeRealValue() to +** sqlite3VdbeRealValueRC, allowing sqlite3VdbeRealValue() to avoid +** stuffing values onto the stack. +*/ +static SQLITE_NOINLINE double sqlite3MemRealValueNoRC(Mem *pMem){ + double r; + (void)sqlite3MemRealValueRC(pMem, &r); + return r; +} + /* ** Return the best representation of pMem that we can get into a ** double. If pMem is already a double or an integer, return its ** value. If it is a string or blob, try to convert it to a double. ** If it is a NULL, return 0.0. */ -static SQLITE_NOINLINE double memRealValue(Mem *pMem){ - /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ - double val = (double)0; - sqlite3AtoF(pMem->z, &val, pMem->n, pMem->enc); - return val; -} double sqlite3VdbeRealValue(Mem *pMem){ assert( pMem!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); @@ -665,7 +777,7 @@ double sqlite3VdbeRealValue(Mem *pMem){ testcase( pMem->flags & MEM_IntReal ); return (double)pMem->u.i; }else if( pMem->flags & (MEM_Str|MEM_Blob) ){ - return memRealValue(pMem); + return sqlite3MemRealValueNoRC(pMem); }else{ /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ return (double)0; @@ -789,8 +901,8 @@ int sqlite3VdbeMemNumerify(Mem *pMem){ sqlite3_int64 ix; assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); - if( ((rc==0 || rc==1) && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1) + rc = sqlite3MemRealValueRC(pMem, &pMem->u.r); + if( ((rc&2)==0 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<2) || sqlite3RealSameAsInt(pMem->u.r, (ix = sqlite3RealToI64(pMem->u.r))) ){ pMem->u.i = ix; @@ -1254,6 +1366,84 @@ int sqlite3VdbeMemSetStr( return SQLITE_OK; } +/* Like sqlite3VdbeMemSetStr() except: +** +** enc is always SQLITE_UTF8 +** pMem->db is always non-NULL +*/ +int sqlite3VdbeMemSetText( + Mem *pMem, /* Memory cell to set to string value */ + const char *z, /* String pointer */ + i64 n, /* Bytes in string, or negative */ + void (*xDel)(void*) /* Destructor function */ +){ + i64 nByte = n; /* New value for pMem->n */ + u16 flags; + + assert( pMem!=0 ); + assert( pMem->db!=0 ); + assert( sqlite3_mutex_held(pMem->db->mutex) ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); + + /* If z is a NULL pointer, set pMem to contain an SQL NULL. */ + if( !z ){ + sqlite3VdbeMemSetNull(pMem); + return SQLITE_OK; + } + + if( nByte<0 ){ + nByte = strlen(z); + flags = MEM_Str|MEM_Term; + }else{ + flags = MEM_Str; + } + if( nByte>(i64)pMem->db->aLimit[SQLITE_LIMIT_LENGTH] ){ + if( xDel && xDel!=SQLITE_TRANSIENT ){ + if( xDel==SQLITE_DYNAMIC ){ + sqlite3DbFree(pMem->db, (void*)z); + }else{ + xDel((void*)z); + } + } + sqlite3VdbeMemSetNull(pMem); + return sqlite3ErrorToParser(pMem->db, SQLITE_TOOBIG); + } + + /* The following block sets the new values of Mem.z and Mem.xDel. It + ** also sets a flag in local variable "flags" to indicate the memory + ** management (one of MEM_Dyn or MEM_Static). + */ + if( xDel==SQLITE_TRANSIENT ){ + i64 nAlloc = nByte + 1; + testcase( nAlloc==31 ); + testcase( nAlloc==32 ); + if( sqlite3VdbeMemClearAndResize(pMem, (int)MAX(nAlloc,32)) ){ + return SQLITE_NOMEM_BKPT; + } + assert( pMem->z!=0 ); + memcpy(pMem->z, z, nByte); + pMem->z[nByte] = 0; + }else{ + sqlite3VdbeMemRelease(pMem); + pMem->z = (char *)z; + if( xDel==SQLITE_DYNAMIC ){ + pMem->zMalloc = pMem->z; + pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); + pMem->xDel = 0; + }else if( xDel==SQLITE_STATIC ){ + pMem->xDel = xDel; + flags |= MEM_Static; + }else{ + pMem->xDel = xDel; + flags |= MEM_Dyn; + } + } + pMem->flags = flags; + pMem->n = (int)(nByte & 0x7fffffff); + pMem->enc = SQLITE_UTF8; + return SQLITE_OK; +} + /* ** Move data out of a btree key or data field and into a Mem structure. ** The data is payload from the entry that pCur is currently pointing @@ -1682,7 +1872,7 @@ static int valueFromExpr( if( affinity==SQLITE_AFF_BLOB ){ if( op==TK_FLOAT ){ assert( pVal && pVal->z && pVal->flags==(MEM_Str|MEM_Term) ); - sqlite3AtoF(pVal->z, &pVal->u.r, pVal->n, SQLITE_UTF8); + sqlite3AtoF(pVal->z, &pVal->u.r); pVal->flags = MEM_Real; }else if( op==TK_INTEGER ){ /* This case is required by -9223372036854775808 and other strings @@ -1950,6 +2140,11 @@ int sqlite3Stat4ValueFromExpr( ** ** If *ppVal is initially NULL then the caller is responsible for ** ensuring that the value written into *ppVal is eventually freed. +** +** If the buffer does not contain a well-formed record, this routine may +** read several bytes past the end of the buffer. Callers must therefore +** ensure that any buffer which may contain a corrupt record is padded +** with at least 8 bytes of addressable memory. */ int sqlite3Stat4Column( sqlite3 *db, /* Database handle */ diff --git a/src/wal.c b/src/wal.c index 7f7bee626..cc8ed326c 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1126,7 +1126,7 @@ static void walUnlockExclusive(Wal *pWal, int lockIdx, int n){ /* ** Compute a hash on a page number. The resulting hash value must land -** between 0 and (HASHTABLE_NSLOT-1). The walHashNext() function advances +** between 0 and (HASHTABLE_NSLOT-1). The walNextHash() function advances ** the hash to the next value in the event of a collision. */ static int walHash(u32 iPage){ @@ -1334,7 +1334,7 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ for(iKey=walHash(iPage); sLoc.aHash[iKey]; iKey=walNextHash(iKey)){ if( (nCollide--)==0 ) return SQLITE_CORRUPT_BKPT; } - sLoc.aPgno[idx-1] = iPage; + sLoc.aPgno[(idx-1)&(HASHTABLE_NPAGE-1)] = iPage; AtomicStore(&sLoc.aHash[iKey], (ht_slot)idx); #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT @@ -3582,7 +3582,10 @@ static int walFindFrame( SEH_INJECT_FAULT; while( (iH = AtomicLoad(&sLoc.aHash[iKey]))!=0 ){ u32 iFrame = iH + sLoc.iZero; - if( iFrame<=iLast && iFrame>=pWal->minFrame && sLoc.aPgno[iH-1]==pgno ){ + if( iFrame<=iLast + && iFrame>=pWal->minFrame + && sLoc.aPgno[(iH-1)&(HASHTABLE_NPAGE-1)]==pgno + ){ assert( iFrame>iRead || CORRUPT_DB ); iRead = iFrame; } diff --git a/src/where.c b/src/where.c index c4f2c5543..e3c107778 100644 --- a/src/where.c +++ b/src/where.c @@ -1514,11 +1514,14 @@ static sqlite3_index_info *allocateIndexInfo( break; } if( i==n ){ + int bSortByGroup = (pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0; nOrderBy = n; if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) && !pSrc->fg.rowidUsed ){ - eDistinct = 2 + ((pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0); + eDistinct = 2 + bSortByGroup; }else if( pWInfo->wctrlFlags & WHERE_GROUPBY ){ - eDistinct = 1; + eDistinct = 1 - bSortByGroup; + }else if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ + eDistinct = 3; } } } @@ -2446,11 +2449,16 @@ void sqlite3WhereClausePrint(WhereClause *pWC){ void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC){ WhereInfo *pWInfo; if( pWC ){ + int nb; + SrcItem *pItem; + Table *pTab; + Bitmask mAll; + pWInfo = pWC->pWInfo; - int nb = 1+(pWInfo->pTabList->nSrc+3)/4; - SrcItem *pItem = pWInfo->pTabList->a + p->iTab; - Table *pTab = pItem->pSTab; - Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1; + nb = 1+(pWInfo->pTabList->nSrc+3)/4; + pItem = pWInfo->pTabList->a + p->iTab; + pTab = pItem->pSTab; + mAll = (((Bitmask)1)<<(nb*4)) - 1; sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, p->iTab, nb, p->maskSelf, nb, p->prereq & mAll); sqlite3DebugPrintf(" %12s", @@ -2929,6 +2937,67 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ return rc; } +/* +** Callback for estLikePatternLength(). +** +** If this node is a string literal that is longer pWalker->sz, then set +** pWalker->sz to the byte length of that string literal. +** +** pWalker->eCode indicates how to count characters: +** +** eCode==0 Count as a GLOB pattern +** eCode==1 Count as a LIKE pattern +*/ +static int exprNodePatternLengthEst(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_STRING ){ + int sz = 0; /* Pattern size in bytes */ + u8 *z = (u8*)pExpr->u.zToken; /* The pattern */ + u8 c; /* Next character of the pattern */ + u8 c1, c2, c3; /* Wildcards */ + if( pWalker->eCode ){ + c1 = '%'; + c2 = '_'; + c3 = 0; + }else{ + c1 = '*'; + c2 = '?'; + c3 = '['; + } + while( (c = *(z++))!=0 ){ + if( c==c3 ){ + if( *z ) z++; + while( *z && *z!=']' ) z++; + }else if( c!=c1 && c!=c2 ){ + sz++; + } + } + if( sz>pWalker->u.sz ) pWalker->u.sz = sz; + } + return WRC_Continue; +} + +/* +** Return the length of the longest string literal in the given +** expression. +** +** eCode indicates how to count characters: +** +** eCode==0 Count as a GLOB pattern +** eCode==1 Count as a LIKE pattern +*/ +static int estLikePatternLength(Expr *p, u16 eCode){ + Walker w; + w.u.sz = 0; + w.eCode = eCode; + w.xExprCallback = exprNodePatternLengthEst; + w.xSelectCallback = sqlite3SelectWalkFail; +#ifdef SQLITE_DEBUG + w.xSelectCallback2 = sqlite3SelectWalkAssert2; +#endif + sqlite3WalkExpr(&w, p); + return w.u.sz; +} + /* ** Adjust the WhereLoop.nOut value downward to account for terms of the ** WHERE clause that reference the loop but which are not used by an @@ -2957,6 +3026,13 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** "x" column is boolean or else -1 or 0 or 1 is a common default value ** on the "x" column and so in that case only cap the output row estimate ** at 1/2 instead of 1/4. +** +** Heuristic 3: If there is a LIKE or GLOB (or REGEXP or MATCH) operator +** with a large constant pattern, then reduce the size of the search +** space according to the length of the pattern, under the theory that +** longer patterns are less likely to match. This heuristic was added +** to give better output-row count estimates when preparing queries for +** the Join-Order Benchmarks. See forum thread 2026-01-30T09:57:54z */ static void whereLoopOutputAdjust( WhereClause *pWC, /* The WHERE clause */ @@ -3006,13 +3082,14 @@ static void whereLoopOutputAdjust( }else{ /* In the absence of explicit truth probabilities, use heuristics to ** guess a reasonable truth probability. */ + Expr *pOpExpr = pTerm->pExpr; pLoop->nOut--; if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && (pTerm->wtFlags & TERM_HIGHTRUTH)==0 /* tag-20200224-1 */ ){ - Expr *pRight = pTerm->pExpr->pRight; + Expr *pRight = pOpExpr->pRight; int k = 0; - testcase( pTerm->pExpr->op==TK_IS ); + testcase( pOpExpr->op==TK_IS ); if( sqlite3ExprIsInteger(pRight, &k, 0) && k>=(-1) && k<=1 ){ k = 10; }else{ @@ -3022,6 +3099,23 @@ static void whereLoopOutputAdjust( pTerm->wtFlags |= TERM_HEURTRUTH; iReduce = k; } + }else + if( ExprHasProperty(pOpExpr, EP_InfixFunc) + && pOpExpr->op==TK_FUNCTION + ){ + int eOp; + assert( ExprUseXList(pOpExpr) ); + assert( pOpExpr->x.pList->nExpr>=2 ); + eOp = sqlite3ExprIsLikeOperator(pOpExpr); + if( ALWAYS(eOp>0) ){ + int szPattern; + Expr *pRHS = pOpExpr->x.pList->a[0].pExpr; + eOp = eOp==SQLITE_INDEX_CONSTRAINT_LIKE; + szPattern = estLikePatternLength(pRHS, eOp); + if( szPattern>0 ){ + pLoop->nOut -= szPattern*2; + } + } } } } @@ -3093,7 +3187,7 @@ static int whereRangeVectorLen( idxaff = sqlite3TableColumnAffinity(pIdx->pTable, pLhs->iColumn); if( aff!=idxaff ) break; - pColl = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); + pColl = sqlite3ExprCompareCollSeq(pParse, pTerm->pExpr); if( pColl==0 ) break; if( sqlite3StrICmp(pColl->zName, pIdx->azColl[i+nEq]) ) break; } @@ -3482,6 +3576,7 @@ static int whereLoopAddBtreeIndex( pNew->rRun += nInMul + nIn; pNew->nOut += nInMul + nIn; whereLoopOutputAdjust(pBuilder->pWC, pNew, rSize); + if( pSrc->fg.fromExists ) pNew->nOut = 0; rc = whereLoopInsert(pBuilder, pNew); if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ @@ -4077,7 +4172,14 @@ static int whereLoopAddBtree( whereLoopOutputAdjust(pWC, pNew, rSize); if( pSrc->fg.isSubquery ){ if( pSrc->fg.viaCoroutine ) pNew->wsFlags |= WHERE_COROUTINE; - pNew->u.btree.pOrderBy = pSrc->u4.pSubq->pSelect->pOrderBy; + /* Do not set btree.pOrderBy for a recursive CTE. In this case + ** the ORDER BY clause does not determine the overall order that + ** rows are emitted from the CTE in. */ + if( (pSrc->u4.pSubq->pSelect->selFlags & SF_Recursive)==0 ){ + pNew->u.btree.pOrderBy = pSrc->u4.pSubq->pSelect->pOrderBy; + } + }else if( pSrc->fg.fromExists ){ + pNew->nOut = 0; } rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; @@ -4180,6 +4282,7 @@ static int whereLoopAddBtree( ** positioned to the correct row during the right-join no-match ** loop. */ }else{ + if( pSrc->fg.fromExists ) pNew->nOut = 0; rc = whereLoopInsert(pBuilder, pNew); } pNew->nOut = rSize; @@ -4842,7 +4945,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ sqlite3 *db = pWInfo->pParse->db; int rc = SQLITE_OK; int bFirstPastRJ = 0; - int hasRightJoin = 0; + int hasRightCrossJoin = 0; WhereLoop *pNew; @@ -4869,15 +4972,34 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ ** prevents the right operand of a RIGHT JOIN from being swapped with ** other elements even further to the right. ** - ** The JT_LTORJ case and the hasRightJoin flag work together to - ** prevent FROM-clause terms from moving from the right side of - ** a LEFT JOIN over to the left side of that join if the LEFT JOIN - ** is itself on the left side of a RIGHT JOIN. + ** The hasRightCrossJoin flag prevent FROM-clause terms from moving + ** from the right side of a LEFT JOIN or CROSS JOIN over to the + ** left side of that same join. This is a required restriction in + ** the case of LEFT JOIN - an incorrect answer may results if it is + ** not enforced. This restriction is not required for CROSS JOIN. + ** It is provided merely as a means of controlling join order, under + ** the theory that no real-world queries that care about performance + ** actually use the CROSS JOIN syntax. */ - if( pItem->fg.jointype & JT_LTORJ ) hasRightJoin = 1; + if( pItem->fg.jointype & (JT_LTORJ|JT_CROSS) ){ + testcase( pItem->fg.jointype & JT_LTORJ ); + testcase( pItem->fg.jointype & JT_CROSS ); + hasRightCrossJoin = 1; + } mPrereq |= mPrior; bFirstPastRJ = (pItem->fg.jointype & JT_RIGHT)!=0; - }else if( !hasRightJoin ){ + }else if( pItem->fg.fromExists ){ + /* joins that result from the EXISTS-to-JOIN optimization should not + ** be moved to the left of any of their dependencies */ + WhereClause *pWC = &pWInfo->sWC; + WhereTerm *pTerm; + int i; + for(i=pWC->nBase, pTerm=pWC->a; i>0; i--, pTerm++){ + if( (pNew->maskSelf & pTerm->prereqAll)!=0 ){ + mPrereq |= (pTerm->prereqAll & (pNew->maskSelf-1)); + } + } + }else if( !hasRightCrossJoin ){ mPrereq = 0; } #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -5100,9 +5222,7 @@ static i8 wherePathSatisfiesOrderBy( pLoop = pLast; } if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ - if( pLoop->u.vtab.isOrdered - && ((wctrlFlags&(WHERE_DISTINCTBY|WHERE_SORTBYGROUP))!=WHERE_DISTINCTBY) - ){ + if( pLoop->u.vtab.isOrdered && pWInfo->pOrderBy==pOrderBy ){ obSat = obDone; }else{ /* No further ORDER BY terms may be matched. So this call should @@ -5478,12 +5598,21 @@ static LogEst whereSortingCost( ** 12 otherwise ** ** For the purposes of this heuristic, a star-query is defined as a query -** with a large central table that is joined using an INNER JOIN, -** not CROSS or OUTER JOINs, against four or more smaller tables. -** The central table is called the "fact" table. The smaller tables -** that get joined are "dimension tables". Also, any table that is -** self-joined cannot be a dimension table; we assume that dimension -** tables may only be joined against fact tables. +** with a central "fact" table that is joined against multiple +** "dimension" tables, subject to the following constraints: +** +** (aa) Only a five-way or larger join is considered for this +** optimization. If there are fewer than four terms in the FROM +** clause, this heuristic does not apply. +** +** (bb) The join between the fact table and the dimension tables must +** be an INNER join. CROSS and OUTER JOINs do not qualify. +** +** (cc) A table must have 3 or more dimension tables in order to be +** considered a fact table. (Was 4 prior to 2026-02-10.) +** +** (dd) A table that is a self-join cannot be a dimension table. +** Dimension tables are joined against fact tables. ** ** SIDE EFFECT: (and really the whole point of this subroutine) ** @@ -5536,7 +5665,7 @@ static int computeMxChoice(WhereInfo *pWInfo){ } #endif /* SQLITE_DEBUG */ - if( nLoop>=5 + if( nLoop>=4 /* Constraint (aa) */ && !pWInfo->bStarDone && OptimizationEnabled(pWInfo->pParse->db, SQLITE_StarQuery) ){ @@ -5548,7 +5677,7 @@ static int computeMxChoice(WhereInfo *pWInfo){ pWInfo->bStarDone = 1; /* Only do this computation once */ - /* Look for fact tables with four or more dimensions where the + /* Look for fact tables with three or more dimensions where the ** dimension tables are not separately from the fact tables by an outer ** or cross join. Adjust cost weights if found. */ @@ -5565,18 +5694,17 @@ static int computeMxChoice(WhereInfo *pWInfo){ if( (pFactTab->fg.jointype & (JT_OUTER|JT_CROSS))!=0 ){ /* If the candidate fact-table is the right table of an outer join ** restrict the search for dimension-tables to be tables to the right - ** of the fact-table. */ - if( iFromIdx+4 > nLoop ) break; /* Impossible to reach nDep>=4 */ + ** of the fact-table. Constraint (bb) */ + if( iFromIdx+3 > nLoop ){ + break; /* ^-- Impossible to reach nDep>=2 - Constraint (cc) */ + } while( pStart && pStart->iTab<=iFromIdx ){ pStart = pStart->pNextLoop; } } for(pWLoop=pStart; pWLoop; pWLoop=pWLoop->pNextLoop){ if( (aFromTabs[pWLoop->iTab].fg.jointype & (JT_OUTER|JT_CROSS))!=0 ){ - /* Fact-tables and dimension-tables cannot be separated by an - ** outer join (at least for the definition of fact- and dimension- - ** used by this heuristic). */ - break; + break; /* Constraint (bb) */ } if( (pWLoop->prereq & m)!=0 /* pWInfo depends on iFromIdx */ && (pWLoop->maskSelf & mSeen)==0 /* pWInfo not already a dependency */ @@ -5590,7 +5718,9 @@ static int computeMxChoice(WhereInfo *pWInfo){ } } } - if( nDep<=3 ) continue; + if( nDep<=2 ){ + continue; /* Constraint (cc) */ + } /* If we reach this point, it means that pFactTab is a fact table ** with four or more dimensions connected by inner joins. Proceed @@ -5603,6 +5733,23 @@ static int computeMxChoice(WhereInfo *pWInfo){ pWLoop->rStarDelta = 0; } } +#endif +#ifdef WHERETRACE_ENABLED /* 0x80000 */ + if( sqlite3WhereTrace & 0x80000 ){ + Bitmask mShow = mSeen; + sqlite3DebugPrintf("Fact table %s(%d), dimensions:", + pFactTab->zAlias ? pFactTab->zAlias : pFactTab->pSTab->zName, + iFromIdx); + for(pWLoop=pStart; pWLoop; pWLoop=pWLoop->pNextLoop){ + if( mShow & pWLoop->maskSelf ){ + SrcItem *pDim = aFromTabs + pWLoop->iTab; + mShow &= ~pWLoop->maskSelf; + sqlite3DebugPrintf(" %s(%d)", + pDim->zAlias ? pDim->zAlias: pDim->pSTab->zName, pWLoop->iTab); + } + } + sqlite3DebugPrintf("\n"); + } #endif pWInfo->bStarUsed = 1; @@ -5626,10 +5773,8 @@ static int computeMxChoice(WhereInfo *pWInfo){ if( sqlite3WhereTrace & 0x80000 ){ SrcItem *pDim = aFromTabs + pWLoop->iTab; sqlite3DebugPrintf( - "Increase SCAN cost of dimension %s(%d) of fact %s(%d) to %d\n", - pDim->zAlias ? pDim->zAlias: pDim->pSTab->zName, pWLoop->iTab, - pFactTab->zAlias ? pFactTab->zAlias : pFactTab->pSTab->zName, - iFromIdx, mxRun + "Increase SCAN cost of %s to %d\n", + pDim->zAlias ? pDim->zAlias: pDim->pSTab->zName, mxRun ); } pWLoop->rStarDelta = mxRun - pWLoop->rRun; @@ -6443,6 +6588,7 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin( for(pTerm=pWInfo->sWC.a; pTerm<pEnd; pTerm++){ if( (pTerm->prereqAll & pLoop->maskSelf)!=0 ){ pTerm->wtFlags |= TERM_CODED; + pTerm->prereqAll = 0; } } if( i!=pWInfo->nLevel-1 ){ @@ -7418,6 +7564,10 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ ){ int r1 = pParse->nMem+1; int j, op; + int addrIfNull = 0; /* Init to avoid false-positive compiler warning */ + if( pLevel->iLeftJoin ){ + addrIfNull = sqlite3VdbeAddOp2(v, OP_IfNullRow, pLevel->iIdxCur, r1); + } for(j=0; j<n; j++){ sqlite3VdbeAddOp3(v, OP_Column, pLevel->iIdxCur, j, r1+j); } @@ -7427,17 +7577,21 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ VdbeCoverageIf(v, op==OP_SeekLT); VdbeCoverageIf(v, op==OP_SeekGT); sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2); + if( pLevel->iLeftJoin ){ + sqlite3VdbeJumpHere(v, addrIfNull); + } } #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */ } - if( pTabList->a[pLevel->iFrom].fg.fromExists && i==pWInfo->nLevel-1 ){ - /* If the EXISTS-to-JOIN optimization was applied, then the EXISTS - ** loop(s) will be the inner-most loops of the join. There might be - ** multiple EXISTS loops, but they will all be nested, and the join - ** order will not have been changed by the query planner. If the - ** inner-most EXISTS loop sees a single successful row, it should - ** break out of *all* EXISTS loops. But only the inner-most of the - ** nested EXISTS loops should do this breakout. */ + if( pTabList->a[pLevel->iFrom].fg.fromExists + && (i==pWInfo->nLevel-1 + || pTabList->a[pWInfo->a[i+1].iFrom].fg.fromExists==0) + ){ + /* This is an EXISTS-to-JOIN optimization which is either the + ** inner-most loop, or the inner-most of a group of nested + ** EXISTS-to-JOIN optimization loops. If this loop sees a successful + ** row, it should break out of itself as well as other EXISTS-to-JOIN + ** loops in which is is directly nested. */ int nOuter = 0; /* Nr of outer EXISTS that this one is nested within */ while( nOuter<i ){ if( !pTabList->a[pLevel[-nOuter-1].iFrom].fg.fromExists ) break; @@ -7445,7 +7599,11 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ } testcase( nOuter>0 ); sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel[-nOuter].addrBrk); - VdbeComment((v, "EXISTS break")); + if( nOuter ){ + VdbeComment((v, "EXISTS break %d..%d", i-nOuter, i)); + }else{ + VdbeComment((v, "EXISTS break %d", i)); + } } sqlite3VdbeResolveLabel(v, pLevel->addrCont); if( pLevel->op!=OP_Noop ){ diff --git a/src/wherecode.c b/src/wherecode.c index 65ed980b8..4375ac5f5 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -1573,7 +1573,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( if( SMASKBIT32(j) & pLoop->u.vtab.mHandleIn ){ int iTab = pParse->nTab++; int iCache = ++pParse->nMem; - sqlite3CodeRhsOfIN(pParse, pTerm->pExpr, iTab); + sqlite3CodeRhsOfIN(pParse, pTerm->pExpr, iTab, 0); sqlite3VdbeAddOp3(v, OP_VInitIn, iTab, iTarget, iCache); }else{ codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); diff --git a/src/whereexpr.c b/src/whereexpr.c index 0d99ca85e..74bf624c8 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -293,13 +293,14 @@ static int isLikeOrGlob( ){ int isNum; double rDummy; - isNum = sqlite3AtoF(zNew, &rDummy, iTo, SQLITE_UTF8); + assert( zNew[iTo]==0 ); + isNum = sqlite3AtoF(zNew, &rDummy); if( isNum<=0 ){ if( iTo==1 && zNew[0]=='-' ){ isNum = +1; }else{ zNew[iTo-1]++; - isNum = sqlite3AtoF(zNew, &rDummy, iTo, SQLITE_UTF8); + isNum = sqlite3AtoF(zNew, &rDummy); zNew[iTo-1]--; } } @@ -342,6 +343,34 @@ static int isLikeOrGlob( } #endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ +/* +** If pExpr is one of "like", "glob", "match", or "regexp", then +** return the corresponding SQLITE_INDEX_CONSTRAINT_xxxx value. +** If not, return 0. +** +** pExpr is guaranteed to be a TK_FUNCTION. +*/ +int sqlite3ExprIsLikeOperator(const Expr *pExpr){ + static const struct { + const char *zOp; + unsigned char eOp; + } aOp[] = { + { "match", SQLITE_INDEX_CONSTRAINT_MATCH }, + { "glob", SQLITE_INDEX_CONSTRAINT_GLOB }, + { "like", SQLITE_INDEX_CONSTRAINT_LIKE }, + { "regexp", SQLITE_INDEX_CONSTRAINT_REGEXP } + }; + int i; + assert( pExpr->op==TK_FUNCTION ); + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + for(i=0; i<ArraySize(aOp); i++){ + if( sqlite3StrICmp(pExpr->u.zToken, aOp[i].zOp)==0 ){ + return aOp[i].eOp; + } + } + return 0; +} + #ifndef SQLITE_OMIT_VIRTUALTABLE /* @@ -378,15 +407,6 @@ static int isAuxiliaryVtabOperator( Expr **ppRight /* Expression to left of MATCH/op2 */ ){ if( pExpr->op==TK_FUNCTION ){ - static const struct Op2 { - const char *zOp; - unsigned char eOp2; - } aOp[] = { - { "match", SQLITE_INDEX_CONSTRAINT_MATCH }, - { "glob", SQLITE_INDEX_CONSTRAINT_GLOB }, - { "like", SQLITE_INDEX_CONSTRAINT_LIKE }, - { "regexp", SQLITE_INDEX_CONSTRAINT_REGEXP } - }; ExprList *pList; Expr *pCol; /* Column reference */ int i; @@ -406,16 +426,11 @@ static int isAuxiliaryVtabOperator( */ pCol = pList->a[1].pExpr; assert( pCol->op!=TK_COLUMN || (ExprUseYTab(pCol) && pCol->y.pTab!=0) ); - if( ExprIsVtab(pCol) ){ - for(i=0; i<ArraySize(aOp); i++){ - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - if( sqlite3StrICmp(pExpr->u.zToken, aOp[i].zOp)==0 ){ - *peOp2 = aOp[i].eOp2; - *ppRight = pList->a[0].pExpr; - *ppLeft = pCol; - return 1; - } - } + if( ExprIsVtab(pCol) && (i = sqlite3ExprIsLikeOperator(pExpr))!=0 ){ + *peOp2 = i; + *ppRight = pList->a[0].pExpr; + *ppLeft = pCol; + return 1; } /* We can also match against the first column of overloaded @@ -549,16 +564,22 @@ static void whereCombineDisjuncts( Expr *pNew; /* New virtual expression */ int op; /* Operator for the combined expression */ int idxNew; /* Index in pWC of the next virtual term */ + Expr *pA, *pB; /* Expressions associated with pOne and pTwo */ if( (pOne->wtFlags | pTwo->wtFlags) & TERM_VNULL ) return; if( (pOne->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; if( (pTwo->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; if( (eOp & (WO_EQ|WO_LT|WO_LE))!=eOp && (eOp & (WO_EQ|WO_GT|WO_GE))!=eOp ) return; - assert( pOne->pExpr->pLeft!=0 && pOne->pExpr->pRight!=0 ); - assert( pTwo->pExpr->pLeft!=0 && pTwo->pExpr->pRight!=0 ); - if( sqlite3ExprCompare(0,pOne->pExpr->pLeft, pTwo->pExpr->pLeft, -1) ) return; - if( sqlite3ExprCompare(0,pOne->pExpr->pRight, pTwo->pExpr->pRight,-1) )return; + pA = pOne->pExpr; + pB = pTwo->pExpr; + assert( pA->pLeft!=0 && pA->pRight!=0 ); + assert( pB->pLeft!=0 && pB->pRight!=0 ); + if( sqlite3ExprCompare(0,pA->pLeft, pB->pLeft, -1) ) return; + if( sqlite3ExprCompare(0,pA->pRight, pB->pRight,-1) ) return; + if( ExprHasProperty(pA,EP_Commuted)!=ExprHasProperty(pB,EP_Commuted) ){ + return; + } /* If we reach this point, it means the two subterms can be combined */ if( (eOp & (eOp-1))!=0 ){ if( eOp & (WO_LT|WO_LE) ){ @@ -569,7 +590,7 @@ static void whereCombineDisjuncts( } } db = pWC->pWInfo->pParse->db; - pNew = sqlite3ExprDup(db, pOne->pExpr, 0); + pNew = sqlite3ExprDup(db, pA, 0); if( pNew==0 ) return; for(op=TK_EQ; eOp!=(WO_EQ<<(op-TK_EQ)); op++){ assert( op<TK_GE ); } pNew->op = op; @@ -1609,13 +1630,11 @@ static void whereAddLimitExpr( int iVal = 0; if( sqlite3ExprIsInteger(pExpr, &iVal, pParse) && iVal>=0 ){ - Expr *pVal = sqlite3Expr(db, TK_INTEGER, 0); + Expr *pVal = sqlite3ExprInt32(db, iVal); if( pVal==0 ) return; - ExprSetProperty(pVal, EP_IntValue); - pVal->u.iValue = iVal; pNew = sqlite3PExpr(pParse, TK_MATCH, 0, pVal); }else{ - Expr *pVal = sqlite3Expr(db, TK_REGISTER, 0); + Expr *pVal = sqlite3ExprAlloc(db, TK_REGISTER, 0, 0); if( pVal==0 ) return; pVal->iTable = iReg; pNew = sqlite3PExpr(pParse, TK_MATCH, 0, pVal); diff --git a/src/window.c b/src/window.c index 1f22ab194..ea2781864 100644 --- a/src/window.c +++ b/src/window.c @@ -717,7 +717,7 @@ void sqlite3WindowUpdate( pWin->eEnd = aUp[i].eEnd; pWin->eExclude = 0; if( pWin->eStart==TK_FOLLOWING ){ - pWin->pStart = sqlite3Expr(db, TK_INTEGER, "1"); + pWin->pStart = sqlite3ExprInt32(db, 1); } break; } @@ -1062,9 +1062,7 @@ int sqlite3WindowRewrite(Parse *pParse, Select *p){ ** keep everything legal in this case. */ if( pSublist==0 ){ - pSublist = sqlite3ExprListAppend(pParse, 0, - sqlite3Expr(db, TK_INTEGER, "0") - ); + pSublist = sqlite3ExprListAppend(pParse, 0, sqlite3ExprInt32(db, 0)); } pSub = sqlite3SelectNew( diff --git a/test/alterauth2.test b/test/alterauth2.test index 6f9242d36..260b635c3 100644 --- a/test/alterauth2.test +++ b/test/alterauth2.test @@ -116,4 +116,55 @@ do_auth_test 1.3 { {SQLITE_UPDATE sqlite_temp_master sql temp {}} } +do_auth_test 1.4 { + ALTER TABLE t2 ALTER COLUMN b SET NOT NULL; +} { + {SQLITE_ALTER_TABLE main t2 b {}} + {SQLITE_FUNCTION {} sqlite_add_constraint {} {}} + {SQLITE_FUNCTION {} sqlite_drop_constraint {} {}} + {SQLITE_FUNCTION {} sqlite_fail {} {}} + {SQLITE_READ sqlite_master sql main {}} + {SQLITE_READ sqlite_master tbl_name main {}} + {SQLITE_READ sqlite_master type main {}} + {SQLITE_READ t2 b main {}} + {SQLITE_SELECT {} {} {} {}} + {SQLITE_UPDATE sqlite_master sql main {}} +} +do_auth_test 1.5 { + ALTER TABLE t2 ALTER COLUMN 'b' DROP NOT NULL; +} { + {SQLITE_ALTER_TABLE main t2 b {}} + {SQLITE_FUNCTION {} sqlite_drop_constraint {} {}} + {SQLITE_READ sqlite_master sql main {}} + {SQLITE_READ sqlite_master tbl_name main {}} + {SQLITE_READ sqlite_master type main {}} + {SQLITE_UPDATE sqlite_master sql main {}} +} + +do_auth_test 1.6 { + ALTER TABLE t2 ADD CONSTRAINT abc CHECK (aaa>b) +} { + {SQLITE_ALTER_TABLE main t2 {} {}} + {SQLITE_FUNCTION {} sqlite_add_constraint {} {}} + {SQLITE_FUNCTION {} sqlite_fail {} {}} + {SQLITE_FUNCTION {} sqlite_find_constraint {} {}} + {SQLITE_READ sqlite_master sql main {}} + {SQLITE_READ sqlite_master tbl_name main {}} + {SQLITE_READ sqlite_master type main {}} + {SQLITE_READ t2 aaa main {}} + {SQLITE_READ t2 b main {}} + {SQLITE_SELECT {} {} {} {}} + {SQLITE_UPDATE sqlite_master sql main {}} +} +do_auth_test 1.7 { + ALTER TABLE t2 DROP CONSTRAINT abc; +} { + {SQLITE_ALTER_TABLE main t2 {} {}} + {SQLITE_FUNCTION {} sqlite_drop_constraint {} {}} + {SQLITE_READ sqlite_master sql main {}} + {SQLITE_READ sqlite_master tbl_name main {}} + {SQLITE_READ sqlite_master type main {}} + {SQLITE_UPDATE sqlite_master sql main {}} +} + finish_test diff --git a/test/altercol.test b/test/altercol.test index 5f7de57a4..d94e0529c 100644 --- a/test/altercol.test +++ b/test/altercol.test @@ -946,4 +946,17 @@ do_execsql_test 23.20 { ROLLBACK; } {t4new} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 24.0 { + CREATE TABLE t1(a PRIMARY KEY, b); + CREATE TABLE t2(a, b, c); + INSERT INTO t2 VALUES(1, 1, 1); +} + +do_catchsql_test 24.1 { + PRAGMA foreign_keys = 1; + ALTER TABLE t2 ADD COLUMN d REFERENCES t1 DEFAULT 123; +} {1 {Cannot add a REFERENCES column with non-NULL default value}} + finish_test diff --git a/test/altercons.test b/test/altercons.test new file mode 100644 index 000000000..fecdf858b --- /dev/null +++ b/test/altercons.test @@ -0,0 +1,442 @@ +# 2025 September 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix altercons + +# If SQLITE_OMIT_ALTERTABLE is defined, omit this file. +ifcapable !altertable { + finish_test + return +} + +foreach {tn before after} { + 1 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b)) } + { CREATE TABLE t1(a, b) } + + 2 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b) NOT NULL) } + { CREATE TABLE t1(a, b NOT NULL) } + + 3 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b)NOT NULL) } + { CREATE TABLE t1(a, b NOT NULL) } + + 3 { CREATE TABLE t1(a, b NOT NULL CONSTRAINT abc CHECK(t1.a != t1.b)); } + { CREATE TABLE t1(a, b NOT NULL) } + + 4 { CREATE TABLE t1(a, b, CONSTRAINT abc CHECK(t1.a != t1.b)) } + { CREATE TABLE t1(a, b) } + + 5 { CREATE TABLE t1(a, b, CONSTRAINT abc CHECK(t1.a != t1.b), PRIMARY KEY(a))} + { CREATE TABLE t1(a, b, PRIMARY KEY(a)) } + + 6 { CREATE TABLE t1(a, b,CONSTRAINT abc CHECK(t1.a != t1.b),PRIMARY KEY(a))} + { CREATE TABLE t1(a, b,PRIMARY KEY(a)) } + + 7 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b) CONSTRAINT def UNIQUE) } + { CREATE TABLE t1(a, b CONSTRAINT def UNIQUE) } + + 8 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b) CHECK (123)) } + { CREATE TABLE t1(a, b CHECK (123)) } + + 9 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b) DEFAULT NULL) } + { CREATE TABLE t1(a, b DEFAULT NULL) } + + 10 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b) COLLATE nocase) } + { CREATE TABLE t1(a, b COLLATE nocase) } + + 11 { CREATE TABLE t1(a, b CONSTRAINT abc CHECK(t1.a != t1.b) REFERENCES t2) } + { CREATE TABLE t1(a, b REFERENCES t2) } + + 12 { CREATE TABLE t1(a, b, c, CONSTRAINT one CONSTRAINT abc CHECK(a!=b) CONSTRAINT three) } + { CREATE TABLE t1(a, b, c, CONSTRAINT one CONSTRAINT three) } + + 13 { CREATE TABLE t1(a, b, c, CONSTRAINT abc CONSTRAINT one CHECK(a!=b) CONSTRAINT three) } + { CREATE TABLE t1(a, b, c, CONSTRAINT one CHECK(a!=b) CONSTRAINT three) } + + 14 { CREATE TABLE t1(a, b, c, CONSTRAINT abc) } + { CREATE TABLE t1(a, b, c) } + + 15 { CREATE TABLE t1(a, b, c, + CONSTRAINT abc, CHECK( a!=b )) } + { CREATE TABLE t1(a, b, c, CHECK( a!=b )) } + + 16 { CREATE TABLE t1(a, b, c, CONSTRAINT abc /* hello */ CHECK( a!=b )) } + { CREATE TABLE t1(a, b, c) } + + 17 { CREATE TABLE t1(a, b, c, /* world */ CONSTRAINT abc CHECK( a!=b )) } + { CREATE TABLE t1(a, b, c) } + + 18 { CREATE TABLE t1(a, b, c -- comment + CONSTRAINT abc NOT NULL + ) } + { CREATE TABLE t1(a, b, c) } + + 19 { CREATE TABLE t1(a, b, c, -- comment + CONSTRAINT abc CHECK (a>b) CONSTRAINT two + ) } + { CREATE TABLE t1(a, b, c, CONSTRAINT two + ) } + + 20 { CREATE TABLE t1(a, b, c, CONSTRAINT one CONSTRAINT abc CHECK (a>b)CONSTRAINT two) } + { CREATE TABLE t1(a, b, c, CONSTRAINT one CONSTRAINT two) } + + 21 { CREATE TABLE t1(a, b, c CONSTRAINT abc AS (b+1)) } + { CREATE TABLE t1(a, b, c AS (b+1)) } + + 22 { CREATE TABLE t1(a, b, c CONSTRAINT abc GENERATED ALWAYS AS (b+1) STORED) } + { CREATE TABLE t1(a, b, c GENERATED ALWAYS AS (b+1) STORED) } +} { + reset_db + + do_execsql_test 1.$tn.0 $before + + do_execsql_test 1.$tn.1 { + ALTER TABLE t1 DROP CONSTRAINT abc; + } {} + + do_execsql_test 1.$tn.2 { + SELECT sql FROM sqlite_schema WHERE name='t1' + } [list [string trim $after]] +} + +#------------------------------------------------------------------------- + +do_execsql_test 2.0 { + CREATE TABLE t2(x, y CONSTRAINT ccc UNIQUE); +} +do_catchsql_test 2.1 { + ALTER TABLE t2 DROP CONSTRAINT ccc +} {1 {constraint may not be dropped: ccc}} +do_catchsql_test 2.2 { + ALTER TABLE t2 DROP CONSTRAINT ddd +} {1 {no such constraint: ddd}} + +#------------------------------------------------------------------------- +reset_db +foreach {tn col before after} { + 1 a { CREATE TABLE t1(a NOT NULL, b) } + { CREATE TABLE t1(a, b) } + + 2 a { CREATE TABLE t1(a NOT NULL ON CONFLICT FAIL, b) } + { CREATE TABLE t1(a, b) } + + 3 a { CREATE TABLE t1(a NOT NULL ON CONFLICT FAIL UNIQUE, b) } + { CREATE TABLE t1(a UNIQUE, b) } + + 4 b { CREATE TABLE t1(a NOT NULL ON CONFLICT FAIL UNIQUE, b) } + { CREATE TABLE t1(a NOT NULL ON CONFLICT FAIL UNIQUE, b) } + + 5 a { CREATE TABLE t1(a CHECK(a<b) NOT NULL, b) } + { CREATE TABLE t1(a CHECK(a<b), b) } + + 6 a { CREATE TABLE t1(a CHECK(a<b) CONSTRAINT nn NOT NULL, b) } + { CREATE TABLE t1(a CHECK(a<b), b) } + + 7 b { CREATE TABLE t1(a, b NOT NULL PRIMARY KEY) } + { CREATE TABLE t1(a, b PRIMARY KEY) } + + 8 b { CREATE TABLE t1(a, b CHECK ((b+a) IS NOT NULL) NOT NULL PRIMARY KEY) } + { CREATE TABLE t1(a, b CHECK ((b+a) IS NOT NULL) PRIMARY KEY) } + + 9 b { CREATE TABLE t1(a, b CONSTRAINT nn CHECK (b IS NOT NULL) NOT NULL) } + { CREATE TABLE t1(a, b CONSTRAINT nn CHECK (b IS NOT NULL)) } + + 10 b { CREATE TABLE t1(a, b NOT NULL AS (a+1)) } + { CREATE TABLE t1(a, b AS (a+1)) } + + 11 b { CREATE TABLE t1(a, b NOT NULL GENERATED ALWAYS AS (a+1)) } + { CREATE TABLE t1(a, b GENERATED ALWAYS AS (a+1)) } +} { + reset_db + + do_execsql_test 3.$tn.0 $before + + do_execsql_test 3.$tn.1 " + ALTER TABLE t1 ALTER COLUMN $col DROP NOT NULL + " + + do_execsql_test 3.$tn.2 { + SELECT sql FROM sqlite_schema WHERE name='t1' + } [list [string trim $after]] +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE TABLE t2(x, y CONSTRAINT ccc UNIQUE); +} +do_execsql_test 4.1 { + ALTER TABLE t2 ALTER x DROP NOT NULL; + ALTER TABLE t2 ALTER x DROP NOT NULL; + ALTER TABLE t2 ALTER x DROP NOT NULL; +} {} + +#------------------------------------------------------------------------- +# +reset_db + +do_execsql_test 5.1 { + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + INSERT INTO t3 VALUES(1000, NULL); +} + +do_catchsql_test 5.2.1 { + ALTER TABLE t3 ALTER b SET NOT NULL +} {1 {constraint failed}} + +do_test 5.2.2 { + sqlite3_errcode db +} {SQLITE_CONSTRAINT} + +foreach {tn before alter after} { + 1 { CREATE TABLE t1(a, b) } + { ALTER TABLE t1 ALTER a SET NOT NULL } + { CREATE TABLE t1(a NOT NULL, b) } + + 2 { CREATE TABLE t1(a, b) } + { ALTER TABLE t1 ALTER a SET NOT NULL ON CONFLICT FAIL } + { CREATE TABLE t1(a NOT NULL ON CONFLICT FAIL, b) } + + 3 { CREATE TABLE t1(a, b) } + { ALTER TABLE t1 ALTER a SET NOT NULL ON CONFLICT fail; } + { CREATE TABLE t1(a NOT NULL ON CONFLICT fail, b) } + + 4 { CREATE TABLE t1(a, b) } + { ALTER TABLE t1 ALTER b SET NOT NULL ON CONFLICT IGNORE ; } + { CREATE TABLE t1(a, b NOT NULL ON CONFLICT IGNORE) } + + 5 { CREATE TABLE t1(a, 'a b c' VARCHAR(10), UNIQUE(a)) } + { ALTER TABLE t1 ALTER 'a b c' SET NOT NULL } + { CREATE TABLE t1(a, 'a b c' VARCHAR(10) NOT NULL, UNIQUE(a)) } +} { + reset_db + do_execsql_test 5.3.$tn.1 $before + do_execsql_test 5.3.$tn.2 $alter + do_execsql_test 5.3.$tn.3 { + SELECT sql FROM sqlite_schema WHERE name='t1'; + } [list [string trim $after]] +} + +do_execsql_test 5.4.1 { + CREATE TABLE x1(a, b, c); +} +do_catchsql_test 5.4.2 { + ALTER TABLE x1 ALTER d SET NOT NULL; +} {1 {no such column: d}} +do_catchsql_test 5.4.3 { + ALTER TABLE x2 ALTER c SET NOT NULL; +} {1 {no such table: x2}} +do_catchsql_test 5.4.4 { + ALTER TABLE temp.x1 ALTER c SET NOT NULL; +} {1 {no such table: temp.x1}} + +#------------------------------------------------------------------------- +# +reset_db + +do_execsql_test 6.1 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); +} + +do_catchsql_test 6.2.1 { + ALTER TABLE t1 ADD CONSTRAINT nn CHECK (c!=6); +} {1 {constraint failed}} +do_execsql_test 6.2.2 { + DELETE FROM t1 WHERE c=6; + ALTER TABLE t1 ADD CONSTRAINT nn CHECK (c!=6); +} {} +do_catchsql_test 6.2.3 { + INSERT INTO t1 VALUES(4, 5, 6); +} {1 {CHECK constraint failed: nn}} + +foreach {tn before alter after} { + 1 { CREATE TABLE t1(a, b) } + { ALTER TABLE t1 ADD CONSTRAINT nn CHECK (a>=0) } + { CREATE TABLE t1(a, b, CONSTRAINT nn CHECK (a>=0)) } + + 2 { CREATE TABLE t1(a, b ) } + { ALTER TABLE t1 ADD CONSTRAINT nn CHECK (a>=0) } + { CREATE TABLE t1(a, b , CONSTRAINT nn CHECK (a>=0)) } + + 3 { CREATE TABLE t1(a, b ) } + { ALTER TABLE t1 ADD CHECK (a>=0) } + { CREATE TABLE t1(a, b , CHECK (a>=0)) } +} { + reset_db + do_execsql_test 6.3.$tn.1 $before + do_execsql_test 6.3.$tn.2 $alter + do_execsql_test 6.3.$tn.3 { + SELECT sql FROM sqlite_schema WHERE type='table'; + } [list [string trim $after]] +} + +do_execsql_test 6.4.1 { + CREATE TABLE b1(a, b, CONSTRAINT abc CHECK (a!=2)); +} +do_catchsql_test 6.4.2 { + ALTER TABLE b1 ADD CONSTRAINT abc CHECK (a!=3); +} {1 {constraint abc already exists}} +do_execsql_test 6.4.1 { + SELECT sql FROM sqlite_schema WHERE tbl_name='b1' +} {{CREATE TABLE b1(a, b, CONSTRAINT abc CHECK (a!=2))}} + +do_execsql_test 6.5 { + CREATE TABLE abc(x,y); +} + +do_catchsql_test 6.6 { + ALTER TABLE abc ADD CHECK (z>=0); +} {1 {no such column: z}} + +#------------------------------------------------------------------------- +# Try attaching a NOT NULL to a generated column. +# +reset_db +do_execsql_test 7.0 { + CREATE TABLE x1(a, b AS (a+1)); + INSERT INTO x1 VALUES(1), (2), (3), (NULL); +} + +do_catchsql_test 7.1 { + ALTER TABLE x1 ALTER b SET NOT NULL; +} {1 {constraint failed}} + +do_catchsql_test 7.2 { + DELETE FROM x1 WHERE b IS NULL; + ALTER TABLE x1 ALTER b SET NOT NULL; +} {0 {}} + +do_execsql_test 7.3 { + SELECT b FROM x1 +} {2 3 4} + +do_catchsql_test 7.4 { + ALTER TABLE x1 ALTER rowid SET NOT NULL; +} {1 {no such column: rowid}} + +do_execsql_test 7.5 { + CREATE VIEW v1 AS SELECT a, b FROM x1; +} +do_catchsql_test 7.6 { + ALTER TABLE v1 RENAME a TO c; +} {1 {cannot rename columns of view "v1"}} +do_catchsql_test 7.7 { + ALTER TABLE v1 ALTER a SET NOT NULL; +} {1 {cannot edit constraints of view "v1"}} +do_catchsql_test 7.8 { + ALTER TABLE sqlite_schema ALTER sql SET NOT NULL; +} {1 {table sqlite_master may not be altered}} +do_catchsql_test 7.9 { + ALTER TABLE v1 ALTER a DROP NOT NULL +} {1 {cannot edit constraints of view "v1"}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 8.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b NOT NULL, c CHECK (c!=555), d); + INSERT INTO t1 VALUES(1, 1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3, 3); +} + +do_execsql_test 8.1.1 { + ALTER TABLE t1 ALTER a SET NOT NULL; + ALTER TABLE t1 ALTER b SET NOT NULL; + ALTER TABLE t1 ALTER c SET NOT NULL; + ALTER TABLE t1 ALTER d SET NOT NULL; +} + +do_execsql_test 8.1.2 { + SELECT sql FROM sqlite_schema WHERE tbl_name = 't1' +} {{CREATE TABLE t1(a INTEGER PRIMARY KEY NOT NULL, b NOT NULL, c CHECK (c!=555) NOT NULL, d NOT NULL)}} + +do_execsql_test 8.1.3 { + SELECT * FROM t1 WHERE a=2; +} {2 2 2 2} + +do_execsql_test 8.2.1 { + ALTER TABLE t1 ALTER a DROP NOT NULL; + ALTER TABLE t1 ALTER b DROP NOT NULL; + ALTER TABLE t1 ALTER c DROP NOT NULL; + ALTER TABLE t1 ALTER d DROP NOT NULL; +} + +do_execsql_test 8.2.2 { + SELECT sql FROM sqlite_schema WHERE tbl_name = 't1' +} {{CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c CHECK (c!=555), d)}} + +do_execsql_test 8.2.3 { + SELECT * FROM t1 WHERE a=3; +} {3 3 3 3} + +#------------------------------------------------------------------------- +reset_db +forcedelete test.db2 +do_execsql_test 9.0 { + CREATE TABLE t1(x, y, z); + ATTACH 'test.db2' AS aux; + CREATE TABLE aux.t1(x, y, z); + INSERT INTO aux.t1 VALUES(1, 1, 1); + INSERT INTO aux.t1 VALUES(2, 2, 2); + INSERT INTO aux.t1 VALUES(3, 3, NULL); + + CREATE TABLE aux.t2(x, y, z); +} + +do_catchsql_test 9.1.1 { + ALTER TABLE aux.t1 ALTER COLUMN z SET NOT NULL +} {1 {constraint failed}} +do_execsql_test 9.1.2 { + UPDATE aux.t1 SET z=x; + ALTER TABLE aux.t1 ALTER COLUMN z SET NOT NULL; + SELECT sql FROM aux.sqlite_schema WHERE name='t1'; +} {{CREATE TABLE t1(x, y, z NOT NULL)}} +do_execsql_test 9.1.3 { + ALTER TABLE aux.t1 ALTER z DROP NOT NULL; + SELECT sql FROM aux.sqlite_schema WHERE name='t1'; +} {{CREATE TABLE t1(x, y, z)}} +do_execsql_test 9.1.4 { + ALTER TABLE t2 ALTER x SET NOT NULL; + SELECT sql FROM aux.sqlite_schema WHERE name='t2'; +} {{CREATE TABLE t2(x NOT NULL, y, z)}} +do_execsql_test 9.1.5 { + ALTER TABLE t2 ALTER x DROP NOT NULL; + SELECT sql FROM aux.sqlite_schema WHERE name='t2'; +} {{CREATE TABLE t2(x, y, z)}} + +do_catchsql_test 9.2.1 { + ALTER TABLE aux.t1 ADD CONSTRAINT bill CHECK (y!=2); +} {1 {constraint failed}} +do_execsql_test 9.2.2 { + UPDATE aux.t1 SET y=4 WHERE y=2; + ALTER TABLE aux.t1 ADD CONSTRAINT bill CHECK (y!=2); + SELECT sql FROM aux.sqlite_schema WHERE name='t1'; +} {{CREATE TABLE t1(x, y, z, CONSTRAINT bill CHECK (y!=2))}} +do_execsql_test 9.2.3 { + ALTER TABLE aux.t1 DROP CONSTRAINT bill; + SELECT sql FROM aux.sqlite_schema WHERE name='t1'; +} {{CREATE TABLE t1(x, y, z)}} +do_execsql_test 9.2.4 { + ALTER TABLE t2 ADD CONSTRAINT william CHECK (z!=''); + SELECT sql FROM aux.sqlite_schema WHERE name='t2'; +} {{CREATE TABLE t2(x, y, z, CONSTRAINT william CHECK (z!=''))}} +do_execsql_test 9.2.5 { + ALTER TABLE t2 DROP CONSTRAINT william; + SELECT sql FROM aux.sqlite_schema WHERE name='t2'; +} {{CREATE TABLE t2(x, y, z)}} + +finish_test + diff --git a/test/altercons2.test b/test/altercons2.test new file mode 100644 index 000000000..a5bbaf6fc --- /dev/null +++ b/test/altercons2.test @@ -0,0 +1,247 @@ +# 2025 September 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix altercons2 + +# If SQLITE_OMIT_ALTERTABLE is defined, omit this file. +ifcapable !altertable { + finish_test + return +} + +foreach {tn newsql alter res final} { + 1 "CREATE TABLE t1(a, b" + "ALTER TABLE t1 ALTER c SET NOT NULL" + {1 {database disk image is malformed}} + "CREATE TABLE t1(a, b" + + 2 "CREATE TABLE t1(a, b, " + "ALTER TABLE t1 ALTER c DROP NOT NULL" + {0 {}} + "CREATE TABLE t1(a, b, " + + 3 "CREATE TABLE t1(a, b, CHECK( ..." + "ALTER TABLE t1 ALTER c DROP NOT NULL" + {0 {}} + "CREATE TABLE t1(a, b, CHECK( ..." + + 4 "CREATE TABLE t1(a, b, c NOT NULL" + "ALTER TABLE t1 ALTER c DROP NOT NULL" + {0 {}} + "CREATE TABLE t1(a, b, c " + + 5 "CREATE TABLE" + "ALTER TABLE t1 ALTER c DROP NOT NULL" + {1 {database disk image is malformed}} + "CREATE TABLE" + + 6 "CREATE TABLE" + "ALTER TABLE t1 ADD CONSTRAINT nn CHECK (a!=0)" + {1 {database disk image is malformed}} + "CREATE TABLE" + +} { + reset_db + do_execsql_test 1.$tn.0 { + CREATE TABLE t1(a, b, c NOT NULL, CONSTRAINT xyz CHECK( a!=0 )); + } + do_execsql_test 1.$tn.1 { + PRAGMA writable_schema = 1; + UPDATE sqlite_schema SET sql = $::newsql + } + do_catchsql_test 1.$tn.2 $alter $res + + do_execsql_test 1.$tn.3 { + SELECT sql FROM sqlite_schema WHERE name='t1' + } [list $final] +} + +#------------------------------------------------------------------------- + +reset_db +proc xAuth {t args} { + if {$t=="SQLITE_ALTER_TABLE"} { + return "SQLITE_DENY" + } + return "SQLITE_OK" +} +sqlite3 db test.db +db auth xAuth + +do_execsql_test 2.0 { + CREATE TABLE x1(a PRIMARY KEY, b CHECK(a!=b) NOT NULL, c); +} + +do_catchsql_test 2.1.1 { + ALTER TABLE x1 ADD CONSTRAINT ccc CHECK (a!='a') +} {1 {not authorized}} +do_execsql_test 2.1.2 { + SELECT sql FROM sqlite_schema WHERE name='x1' +} {{CREATE TABLE x1(a PRIMARY KEY, b CHECK(a!=b) NOT NULL, c)}} + +do_catchsql_test 2.2.1 { + ALTER TABLE x1 ALTER c SET NOT NULL +} {1 {not authorized}} +do_execsql_test 2.2.2 { + SELECT sql FROM sqlite_schema WHERE name='x1' +} {{CREATE TABLE x1(a PRIMARY KEY, b CHECK(a!=b) NOT NULL, c)}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(x); +} +do_execsql_test 3.1 { + ALTER TABLE t1 ALTER x SET NOT NULL; +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE TABLE abc(a, b, c, CONSTRAINT one CONSTRAINT two CHECK (b!=c)); +} +do_execsql_test 4.1 { + ALTER TABLE abc DROP CONSTRAINT one +} +do_execsql_test 4.2 { + SELECT sql FROM sqlite_schema +} { + {CREATE TABLE abc(a, b, c, CONSTRAINT two CHECK (b!=c))} +} + +#------------------------------------------------------------------------- +reset_db + +# The columns must come before the table constraints in a CREATE TABLE +# statement. This is useful, as it means the DROP CONSTRAINT code does +# not have to handle the constraint immediately following the '(' at +# the start of the column-list. +do_catchsql_test 5.0 { + CREATE TABLE abc(a, b, c, CONSTRAINT two CHECK (b!=c), d) +} {1 {near "d": syntax error}} +do_catchsql_test 5.1 { + CREATE TABLE def(CONSTRAINT abc CHECK( b!=c ), a, b, c); +} {1 {near "CONSTRAINT": syntax error}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 6.0 { + CREATE TABLE abc(a, b CONSTRAINT two COLLATE nocase CHECK (a!=b), c CONSTRAINT one DEFAULT 'abc'); +} + +do_execsql_test 6.1 { + ALTER TABLE abc DROP CONSTRAINT one; + ALTER TABLE abc DROP CONSTRAINT two; +} + +do_execsql_test 6.2 { + SELECT sql FROM sqlite_schema +} { + {CREATE TABLE abc(a, b COLLATE nocase CHECK (a!=b), c DEFAULT 'abc')} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 7.0 { + CREATE TABLE abc(a, b, c, CONSTRAINT one CHECK (a>b) FOREIGN KEY(a) REFERENCES abc); +} +do_execsql_test 7.1 { + ALTER TABLE abc DROP CONSTRAINT one +} +do_execsql_test 7.2 { + SELECT sql FROM sqlite_schema +} { + {CREATE TABLE abc(a, b, c, FOREIGN KEY(a) REFERENCES abc)} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 8.0 { + CREATE TABLE abc(a, b, c, CONSTRAINT one FOREIGN KEY(a) REFERENCES abc); +} +do_catchsql_test 8.1 { + ALTER TABLE abc DROP CONSTRAINT one +} {1 {constraint may not be dropped: one}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 9.0 { + CREATE TABLE abc(a, b NOT NULL AS (a+1)) +} +do_execsql_test 9.1 { + ALTER TABLE abc ALTER b DROP NOT NULL; + SELECT sql FROM sqlite_schema; +} {{CREATE TABLE abc(a, b AS (a+1))}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 10.0 { + CREATE TABLE abc(a, b GENERATED ALWAYS AS (a+1)); + INSERT INTO abc VALUES(1), (2); + SELECT * FROM abc; +} {1 2 2 3} + +do_execsql_test 10.1 { + ALTER TABLE abc ALTER b SET NOT NULL; +} +do_catchsql_test 10.2 { + INSERT INTO abc VALUES(NULL); +} {1 {NOT NULL constraint failed: abc.b}} +do_execsql_test 10.3 { + INSERT INTO abc VALUES(3); + ALTER TABLE abc ALTER COLUMN b DROP NOT NULL; +} +do_execsql_test 10.4 { + INSERT INTO abc VALUES(NULL); +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 11.0 { + CREATE TABLE t1(a, b, c); +} + +do_execsql_test 11.1.1 { + ALTER TABLE t1 ADD CONSTRAINT c1 CHECK(a=b) --comment + ; +} + +do_execsql_test 11.1.2 {ALTER TABLE t1 ADD CONSTRAINT c2 CHECK(a=b) --comment} + +do_execsql_test 11.1.3 { + SELECT sql FROM sqlite_schema; +} { + {CREATE TABLE t1(a, b, c, CONSTRAINT c1 CHECK(a=b), CONSTRAINT c2 CHECK(a=b))} +} + +do_execsql_test 11.2.1 { + CREATE TABLE t2(a, b); +} +do_execsql_test 11.2.2 {ALTER TABLE t2 ALTER b SET NOT NULL --new cons} +do_execsql_test 11.2.3 { + SELECT sql FROM sqlite_schema WHERE name='t2'; +} { + {CREATE TABLE t2(a, b NOT NULL)} +} +do_execsql_test 11.2.3 { + ALTER TABLE t2 ALTER b DROP NOT NULL; + SELECT sql FROM sqlite_schema WHERE name='t2'; +} { + {CREATE TABLE t2(a, b)} +} +do_execsql_test 11.2.2 {ALTER TABLE t2 ALTER b SET NOT NULL --new cons +; +} +finish_test + diff --git a/test/alterfault.test b/test/alterfault.test index b6b42973e..c9c0d77ba 100644 --- a/test/alterfault.test +++ b/test/alterfault.test @@ -23,6 +23,9 @@ ifcapable !altertable { do_execsql_test 1.0 { CREATE TABLE t1(a); + CREATE TEMP TRIGGER tr1 AFTER INSERT ON t1 BEGIN + SELECT 123; + END; } faultsim_save_and_close @@ -36,6 +39,40 @@ do_faultsim_test 1.1 -faults oom* -prep { faultsim_test_result {0 {}} } +reset_db +do_execsql_test 2.0 { + CREATE TABLE x1(d, e CONSTRAINT abc NOT NULL, f); +} +faultsim_save_and_close + +foreach {tn sql} { + 1 { ALTER TABLE x1 ADD CHECK (d!=1) } + 2 { ALTER TABLE x1 ADD CONSTRAINT xyz CHECK (f>d+e); } + 3 { ALTER TABLE x1 DROP CONSTRAINT abc } + 4 { ALTER TABLE x1 ALTER f SET NOT NULL } + 5 { ALTER TABLE x1 ALTER e DROP NOT NULL } +} { + do_faultsim_test 2.$tn -faults oom* -prep { + faultsim_restore_and_reopen + } -body { + execsql $::sql + } -test { + faultsim_test_result {0 {}} + } +} +# Test an OOM when returning an error. +# +do_faultsim_test 2.e -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { + ALTER TABLE x1 DROP CONSTRAINT nosuchconstraint + } +} -test { + faultsim_test_result \ + {1 {no such constraint: nosuchconstraint}} \ + {1 {SQL logic error}} +} finish_test diff --git a/test/altertab3.test b/test/altertab3.test index 92060fb41..36e08c769 100644 --- a/test/altertab3.test +++ b/test/altertab3.test @@ -778,4 +778,52 @@ do_execsql_test 31.2 { SELECT rr FROM t1 LIMIT 1 } {5.0} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 32.1.0 { + CREATE TABLE t1( + a INT, + b INT, + -- comment with comma + c INT + ); +} +do_execsql_test 32.1.1 { + ALTER TABLE t1 DROP COLUMN c; +} +do_execsql_test 32.1.2 { + SELECT sql FROM sqlite_schema +} {{CREATE TABLE t1( + a INT, + b INT)}} + +reset_db +do_execsql_test 32.2.0 { + CREATE TABLE t1( + a INT, + b INT, + -- comment with, comma + c INT + ); +} +do_execsql_test 32.2.1 { + ALTER TABLE t1 DROP COLUMN c; +} +do_execsql_test 32.2.2 { + SELECT sql FROM sqlite_schema +} {{CREATE TABLE t1( + a INT, + b INT)}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 33.1 { + CREATE TABLE x1(a TEXT, b INTEGER, c CHECK(c!=0)); +} + +do_execsql_test 33.2 { + ALTER TABLE x1 DROP COLUMN b; + SELECT sql FROM sqlite_schema; +} {{CREATE TABLE x1(a TEXT, c CHECK(c!=0))}} + finish_test diff --git a/test/altertrig.test b/test/altertrig.test index 556dc3fea..223feaf8f 100644 --- a/test/altertrig.test +++ b/test/altertrig.test @@ -41,7 +41,7 @@ do_execsql_test 1.0 { CREATE TABLE t4(a); CREATE TRIGGER r1 INSERT ON t1 BEGIN - UPDATE t1 SET d='xyz' FROM t2, t3; + UPDATE t1 SET d='xyz' FROM t2, t3; END; } diff --git a/test/atof2.test b/test/atof2.test new file mode 100644 index 000000000..5a68d1352 --- /dev/null +++ b/test/atof2.test @@ -0,0 +1,35 @@ +# 2026-02-20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests of the sqlite3AtoF() function. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# Rounding cases: +# +do_execsql_test atof2-1.0 { + SELECT format('%g',192.496475); +} 192.496 +do_execsql_test atof2-1.1 { + SELECT format('%g',192.496501); +} 192.497 + +load_static_extension db ieee754 +do_execsql_test atof2-2.1 { + SELECT format('%!.30f',ieee754_inc(100.0,-1)); +} 99.9999999999999858 +do_execsql_test atof2-2.2 { + SELECT format('%!.30f',ieee754_inc(100.0,-2)); +} 99.9999999999999716 + +finish_test diff --git a/test/auth.test b/test/auth.test index 1d56f7034..f378b900a 100644 --- a/test/auth.test +++ b/test/auth.test @@ -1922,8 +1922,8 @@ do_test auth-1.283 { execsql { REINDEX BINARY; } - set ::authargs -} {t3_idx1 {} main {} sqlite_autoindex_t3_1 {} main {}} + lsort -unique $::authargs +} {{} main sqlite_autoindex_t3_1 t3_idx1 t3_idx2} do_test auth-1.284 { set ::authargs {} execsql { @@ -1963,8 +1963,8 @@ ifcapable tempdb { execsql { REINDEX BINARY; } - set ::authargs - } {t3_idx1 {} temp {} sqlite_autoindex_t3_1 {} temp {}} + lsort -unique $::authargs + } {{} sqlite_autoindex_t3_1 t3_idx1 t3_idx2 temp} do_test auth-1.290 { set ::authargs {} execsql { diff --git a/test/autoindex1.test b/test/autoindex1.test index 1c8ce007f..be41d702c 100644 --- a/test/autoindex1.test +++ b/test/autoindex1.test @@ -185,8 +185,7 @@ do_eqp_test autoindex1-500.1 { QUERY PLAN |--SEARCH t501 USING INTEGER PRIMARY KEY (rowid=?) `--LIST SUBQUERY xxxxxx - |--SCAN t502 - `--CREATE BLOOM FILTER + `--SCAN t502 } do_eqp_test autoindex1-501 { SELECT b FROM t501 diff --git a/test/avfs.test b/test/avfs.test index ffd6b309f..1503e8613 100644 --- a/test/avfs.test +++ b/test/avfs.test @@ -332,9 +332,11 @@ do_test 4.3 { set ofd [open $shdo w] if {$::cliDoesAr} { puts $ofd ".ar -u $shdo" + puts $ofd ".mode list" puts $ofd "select count(*) from sqlar where name = '$shdo';" } else { puts $ofd "insert into sqlar values (1);" + puts $ofd ".mode list" puts $ofd "select count(*) from sqlar;" } puts $ofd ".q" diff --git a/test/bestindex8.test b/test/bestindex8.test index 3ed7f6703..43d9cf6af 100644 --- a/test/bestindex8.test +++ b/test/bestindex8.test @@ -80,17 +80,17 @@ do_execsql_test 1.0 { foreach {tn sql bDistinct idxinsert bConsumed res} { 1 "SELECT a, b FROM vt1" 0 0 0 {a b c d a b c d} - 2 "SELECT DISTINCT a, b FROM vt1" 2 1 1 {a b c d} - 3 "SELECT DISTINCT a FROM vt1" 2 1 1 {a c} + 2 "SELECT DISTINCT a, b FROM vt1" 2 0 1 {a b c d} + 3 "SELECT DISTINCT a FROM vt1" 2 0 1 {a c} 4 "SELECT DISTINCT b FROM vt1" 2 1 0 {b d} - 5 "SELECT DISTINCT b FROM vt1 ORDER BY a" 0 1 1 {b d} - 6 "SELECT DISTINCT t0.c0 FROM vt1, t0 ORDER BY vt1.a" 0 1 1 {1 0} + 5 "SELECT DISTINCT b FROM vt1 ORDER BY a" 3 1 1 {b d} + 6 "SELECT DISTINCT t0.c0 FROM vt1, t0 ORDER BY vt1.a" 3 1 1 {1 0} 7 "SELECT DISTINCT a, b FROM vt1 ORDER BY a, b" 3 0 1 {a b c d} - 8 "SELECT DISTINCT a, b FROM vt1 ORDER BY a" 0 1 1 {a b c d} - 9 "SELECT DISTINCT a FROM vt1 ORDER BY a, b" 0 1 1 {a c} + 8 "SELECT DISTINCT a, b FROM vt1 ORDER BY a" 3 1 1 {a b c d} + 9 "SELECT DISTINCT a FROM vt1 ORDER BY a, b" 3 1 1 {a c} - 10 "SELECT DISTINCT a, b FROM vt1 WHERE b='b'" 2 1 1 {a b} - 11 "SELECT DISTINCT a, b FROM vt1 WHERE +b='b'" 2 1 1 {a b} + 10 "SELECT DISTINCT a, b FROM vt1 WHERE b='b'" 2 0 1 {a b} + 11 "SELECT DISTINCT a, b FROM vt1 WHERE +b='b'" 2 0 1 {a b} } { set ::lBestIndexDistinct "" set ::lOrderByConsumed 0 diff --git a/test/bestindexB.test b/test/bestindexB.test index b50e74fee..5850e35bd 100644 --- a/test/bestindexB.test +++ b/test/bestindexB.test @@ -34,7 +34,7 @@ proc vtab_command {method args} { set orderby [$hdl orderby] if {[info exists ::xbestindex_sql]} { - explain_i $::xbestindex_sql + # explain_i $::xbestindex_sql set ::xbestindex_res [ execsql $::xbestindex_sql ] } diff --git a/test/bestindexF.test b/test/bestindexF.test new file mode 100644 index 000000000..4f49f610d --- /dev/null +++ b/test/bestindexF.test @@ -0,0 +1,294 @@ +# 2025-12-15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix bestindexF + +ifcapable !vtab { + finish_test + return +} + + +proc vtab_command {method args} { + switch -- $method { + xConnect { + return "CREATE TABLE t1(a, b, c)" + } + + xBestIndex { + set hdl [lindex $args 0] + set ::vtab_orderby [$hdl orderby] + set ::vtab_distinct [$hdl distinct] + + if {$::vtab_orderby == "{column 0 desc 0} {column 1 desc 0}" + || $::vtab_orderby == "{column 0 desc 0}" + } { + return [list orderby 1] + } + + return "" + } + + xFilter { + set sql { + SELECT 1, 1, 'a', 555 + UNION ALL + SELECT 2, 1, 'a', NULL + UNION ALL + SELECT 3, 1, 'b', 'text' + UNION ALL + SELECT 4, 2, 'a', 3.14 + UNION ALL + SELECT 5, 2, 'b', 0 + } + return [list sql $sql] + } + } + + return {} +} + +register_tcl_module db + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING tcl(vtab_command) +} + +proc uses_idxinsert {sql} { + return [expr [lsearch [db eval "explain $sql"] IdxInsert]>=0] +} +proc do_idxinsert_test {tn sql res} { + set uses [uses_idxinsert $sql] + uplevel [list do_execsql_test $tn "SELECT $uses ; $sql" $res] +} + +do_idxinsert_test 1.1.1 { + SELECT DISTINCT a, b FROM t1 +} {0 1 a 1 b 2 a 2 b} + +do_test 1.1.2 { + list $::vtab_distinct $::vtab_orderby +} {2 {{column 0 desc 0} {column 1 desc 0}}} + +do_execsql_test 1.3 { + CREATE TABLE t0(c0); + INSERT INTO t0 VALUES(0); + INSERT INTO t0 VALUES(1); +} + +do_idxinsert_test 1.4.1 { + SELECT DISTINCT t0.c0 FROM t1, t0 ORDER BY t1.a; +} {1 0 1} + +do_test 1.4.2 { + list $::vtab_distinct $::vtab_orderby +} {3 {{column 0 desc 0}}} + +#------------------------------------------------------------------------- +# +reset_db +proc vtab_command {method args} { + switch -- $method { + xConnect { + return "CREATE TABLE t1(a, b, c)" + } + + xBestIndex { + set hdl [lindex $args 0] + set ::vtab_orderby [$hdl orderby] + set ::vtab_distinct [$hdl distinct] + + # Set idxNum to 1 if DISTINCT is to be used in xFilter. + # + set idxStr [list ""] + if {$::vtab_distinct==2 || $::vtab_distinct==3} { + set idxStr [list DISTINCT] + } + + set orderby 0 + if {$::vtab_orderby == "{column 0 desc 1}" + || $::vtab_orderby == "{column 0 desc 0}" + } { + set orderby 1 + if {$::vtab_distinct==1 || $::vtab_distinct==2} { + lappend idxStr "ORDER BY ((a+2)%5)" + } else { + set sort "ORDER BY a" + if {$::vtab_orderby == "{column 0 desc 1}"} { + append sort " DESC" + } + lappend idxStr $sort + } + } else { + lappend idxStr "" + } + + return [list orderby $orderby idxstr $idxStr] + return "" + } + + xFilter { + set idxstr [lindex $args 1] + + set distinct [lindex $idxstr 0] + set orderby [lindex $idxstr 1] + set sql " + SELECT $distinct 0, a, b FROM real_t1 $orderby + " + return [list sql $sql] + } + } + + return {} +} + +do_execsql_test 2.0 { + CREATE TABLE real_t1(a, b); + + INSERT INTO real_t1 VALUES (1, 'a'); + INSERT INTO real_t1 VALUES (2, 'a'); + INSERT INTO real_t1 VALUES (1, 'a'); + + INSERT INTO real_t1 VALUES (2, 'b'); + INSERT INTO real_t1 VALUES (1, 'b'); + INSERT INTO real_t1 VALUES (2, 'b'); + + INSERT INTO real_t1 VALUES (3, 'a'); + INSERT INTO real_t1 VALUES (4, 'b'); + INSERT INTO real_t1 VALUES (3, 'a'); + + INSERT INTO real_t1 VALUES (4, 'b'); + INSERT INTO real_t1 VALUES (3, 'a'); + INSERT INTO real_t1 VALUES (4, 'b'); +} + +register_tcl_module db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING tcl(vtab_command) +} + +do_execsql_test 2.1 { + SELECT a, b FROM t1 +} { + 1 a 2 a 1 a + 2 b 1 b 2 b + 3 a 4 b 3 a + 4 b 3 a 4 b +} + +# This is like do_execsql_test, except one value is prepended to the +# expected result - the P4 (idxStr) of the VFilter opcode. It is an error +# if $sql generates two or more VFilter instructions. +# +proc do_vtabsorter_test {tn sql expect} { + set vm [db eval "EXPLAIN $sql"] + + set ii [lsearch $vm VFilter] + set ::res [lindex $vm [expr $ii+4]] + + set ::idx [expr [lsearch $vm IdxInsert]>=0] + + set iSort [lsearch $vm SorterSort] + if {$iSort>=0} { + error "query is using sorter" + } + + uplevel [list do_test $tn.0 { set ::idx } [lindex $expect 0]] + uplevel [list do_test $tn.1 { set ::res } [lindex $expect 1]] + uplevel [list do_execsql_test $tn.2 $sql [lrange $expect 2 end]] +} + +do_vtabsorter_test 2.2 { + SELECT a, b FROM t1 +} { 0 "{} {}" + 1 a 2 a 1 a + 2 b 1 b 2 b + 3 a 4 b 3 a + 4 b 3 a 4 b +} + +do_vtabsorter_test 2.3 { + SELECT DISTINCT a FROM t1 +} { 0 "DISTINCT {ORDER BY ((a+2)%5)}" + 3 4 1 2 +} + +do_vtabsorter_test 2.4 { + SELECT DISTINCT a FROM t1 ORDER BY a +} { 0 "DISTINCT {ORDER BY a}" + 1 2 3 4 +} + +do_vtabsorter_test 2.5 { + SELECT DISTINCT a FROM t1 ORDER BY a DESC +} { 0 "DISTINCT {ORDER BY a DESC}" + 4 3 2 1 +} + +do_vtabsorter_test 2.6 { + SELECT a FROM t1 ORDER BY a +} { 0 "{} {ORDER BY a}" + 1 1 1 + 2 2 2 + 3 3 3 + 4 4 4 +} + +do_vtabsorter_test 2.7 { + SELECT a FROM t1 ORDER BY a DESC +} { 0 "{} {ORDER BY a DESC}" + 4 4 4 + 3 3 3 + 2 2 2 + 1 1 1 +} + +do_vtabsorter_test 2.8 { + SELECT a, count(*) FROM t1 GROUP BY a ORDER BY a +} { 0 "{} {ORDER BY a}" + 1 3 + 2 3 + 3 3 + 4 3 +} + +do_vtabsorter_test 2.9 { + SELECT a, count(*) FROM t1 GROUP BY a ORDER BY a DESC +} { 0 "{} {ORDER BY a DESC}" + 4 3 + 3 3 + 2 3 + 1 3 +} + +do_vtabsorter_test 2.10 { + SELECT a, count(*) FROM t1 GROUP BY a +} { 0 "{} {ORDER BY ((a+2)%5)}" + 3 3 + 4 3 + 1 3 + 2 3 +} + +do_vtabsorter_test 2.11 { + SELECT DISTINCT a, count(*) FROM t1 GROUP BY a +} { 1 "{} {ORDER BY ((a+2)%5)}" + 3 3 + 4 3 + 1 3 + 2 3 +} + +finish_test + diff --git a/test/carray01.test b/test/carray01.test index 86ea06996..b17a481e1 100644 --- a/test/carray01.test +++ b/test/carray01.test @@ -51,6 +51,10 @@ do_test 101 { run_stmt $STMT } {1} do_test 102 { + sqlite3_carray_bind -v2 -malloc $STMT 3 1 2 3 4 5 6 7 + run_stmt $STMT +} {1} +do_test 103 { set STMT2 [sqlite3_prepare_v2 db { SELECT DISTINCT typeof(value) FROM carray(?3)} -1] sqlite3_carray_bind $STMT2 3 1 2 3 4 5 6 7 @@ -124,6 +128,10 @@ do_test 160 { sqlite3_carray_bind -double $STMT 3 1 2 3 4 5 6 7 run_stmt $STMT } {1} +do_test 161 { + sqlite3_carray_bind -double -v2 $STMT 3 1 2 3 4 5 6 7 + run_stmt $STMT +} {1} do_test 170 { sqlite3_carray_bind -text -static $STMT 3 1 2 3 4 6 7 run_stmt $STMT diff --git a/test/collate5.test b/test/collate5.test index 71d4efe25..b474c39d5 100644 --- a/test/collate5.test +++ b/test/collate5.test @@ -122,9 +122,9 @@ do_test collate5-2.0 { } {} do_test collate5-2.1.1 { - execsql { + string toupper [execsql { SELECT a FROM collate5t1 UNION select a FROM collate5t2; - } + }] } {A B N} do_test collate5-2.1.2 { execsql { @@ -132,10 +132,10 @@ do_test collate5-2.1.2 { } } {A B N a b n} do_test collate5-2.1.3 { - execsql { + string tolower [execsql { SELECT a, b FROM collate5t1 UNION select a, b FROM collate5t2; - } -} {A Apple A apple B Banana b banana N {}} + }] +} {a apple a apple b banana b banana n {}} do_test collate5-2.1.4 { execsql { SELECT a, b FROM collate5t2 UNION select a, b FROM collate5t1; @@ -143,9 +143,9 @@ do_test collate5-2.1.4 { } {A Apple B banana N {} a apple b banana n {}} do_test collate5-2.2.1 { - execsql { + string toupper [execsql { SELECT a FROM collate5t1 EXCEPT select a FROM collate5t2; - } + }] } {N} do_test collate5-2.2.2 { execsql { @@ -153,10 +153,10 @@ do_test collate5-2.2.2 { } } {A a} do_test collate5-2.2.3 { - execsql { + string tolower [execsql { SELECT a, b FROM collate5t1 EXCEPT select a, b FROM collate5t2; - } -} {A Apple N {}} + }] +} {a apple n {}} do_test collate5-2.2.4 { execsql { SELECT a, b FROM collate5t2 EXCEPT select a, b FROM collate5t1 @@ -165,9 +165,9 @@ do_test collate5-2.2.4 { } {A apple a apple} do_test collate5-2.3.1 { - execsql { + string toupper [execsql { SELECT a FROM collate5t1 INTERSECT select a FROM collate5t2; - } + }] } {A B} do_test collate5-2.3.2 { execsql { @@ -175,10 +175,10 @@ do_test collate5-2.3.2 { } } {B b} do_test collate5-2.3.3 { - execsql { + string tolower [execsql { SELECT a, b FROM collate5t1 INTERSECT select a, b FROM collate5t2; - } -} {a apple B banana} + }] +} {a apple b banana} do_test collate5-2.3.4 { execsql { SELECT a, b FROM collate5t2 INTERSECT select a, b FROM collate5t1; diff --git a/test/cost.test b/test/cost.test index 6106caba8..5ef84c2b2 100644 --- a/test/cost.test +++ b/test/cost.test @@ -203,8 +203,8 @@ do_eqp_test 8.2 { } { QUERY PLAN |--SCAN track - |--SEARCH album USING INTEGER PRIMARY KEY (rowid=?) |--SEARCH composer USING INTEGER PRIMARY KEY (rowid=?) + |--SEARCH album USING INTEGER PRIMARY KEY (rowid=?) `--USE TEMP B-TREE FOR DISTINCT } diff --git a/test/dblwidth-a.sql b/test/dblwidth-a.sql index 38c219698..bcd60359f 100644 --- a/test/dblwidth-a.sql +++ b/test/dblwidth-a.sql @@ -1,20 +1,50 @@ +#!sqlite3 /* ** Run this script using "sqlite3" to confirm that the command-line ** shell properly handles the output of double-width characters. ** ** https://sqlite.org/forum/forumpost/008ac80276 */ +.testcase 100 .mode box CREATE TABLE data(word TEXT, description TEXT); INSERT INTO data VALUES('〈οὐκέτι〉','Greek without dblwidth <...>'); -.print .mode box SELECT * FROM data; +.check <<END +╭────────────┬──────────────────────────────╮ +│ word │ description │ +╞════════════╪══════════════════════════════╡ +│ 〈οὐκέτι〉 │ Greek without dblwidth <...> │ +╰────────────┴──────────────────────────────╯ +END + +.testcase 200 .mode table -.print .mode table SELECT * FROM data; +.check <<END ++------------+------------------------------+ +| word | description | ++------------+------------------------------+ +| 〈οὐκέτι〉 | Greek without dblwidth <...> | ++------------+------------------------------+ +END + +.testcase 300 .mode qbox -.print .mode qbox SELECT * FROM data; +.check <<END +╭──────────────┬────────────────────────────────╮ +│ word │ description │ +╞══════════════╪════════════════════════════════╡ +│ '〈οὐκέτι〉' │ 'Greek without dblwidth <...>' │ +╰──────────────┴────────────────────────────────╯ +END + +.testcase 400 .mode column -.print .mode column SELECT * FROM data; +.check <<END + word description +---------- ---------------------------- +〈οὐκέτι〉 Greek without dblwidth <...> +END diff --git a/test/distinct2.test b/test/distinct2.test index 980b0b1e3..a410d9abb 100644 --- a/test/distinct2.test +++ b/test/distinct2.test @@ -311,7 +311,7 @@ do_execsql_test 4010 { } do_execsql_test 4020 { SELECT b FROM t1 UNION SELECT 1; -} {1 { }} +} {1 {}} #------------------------------------------------------------------------- # @@ -380,4 +380,36 @@ do_execsql_test 5080 { ); } 0 +#------------------------------------------------------------------------- +# 2026-03-05 - do not "skip-ahead" of a null-row on the RHS of a +# LEFT JOIN. +# +reset_db + +do_execsql_test 6000 { + CREATE TABLE t1(c1 UNIQUE NOT NULL); + INSERT INTO t1 VALUES(1); + CREATE TABLE t0(c0 UNIQUE); + INSERT INTO t0 VALUES(0); + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<1000) + INSERT INTO t0(c0) SELECT NULL FROM c; +} {} + +do_execsql_test 6010 { + SELECT DISTINCT * FROM t1 LEFT OUTER JOIN t0 ON c0>c1; +} {1 {}} + +do_execsql_test 6020 { + SELECT DISTINCT * FROM t1 FULL OUTER JOIN t0 ON c0>c1; +} {1 {} {} 0 {} {}} + +do_execsql_test 6040 { + ANALYZE; + SELECT DISTINCT * FROM t1 LEFT OUTER JOIN t0 ON c0>c1; +} {1 {}} + +do_execsql_test 6050 { + SELECT DISTINCT * FROM t1 FULL OUTER JOIN t0 ON c0>c1; +} {1 {} {} 0 {} {}} + finish_test diff --git a/test/dotcmd01.sql b/test/dotcmd01.sql new file mode 100644 index 000000000..75595d647 --- /dev/null +++ b/test/dotcmd01.sql @@ -0,0 +1,63 @@ +#!sqlite3 +# +# 2026-01-30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Miscellaneous tests for dot-commands +# +# ./sqlite3 test/dotcmd01.sql +# + +.testcase setup +.open :memory: +.mode tty +.check '' + +# The ".eqp on" setting does not affect the output from .fullschema +# and similar. +# +.testctrl opt -Stat4 +.testcase 100 +CREATE TABLE t1(a,b,c); +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<300) + INSERT INTO t1(a,b,c) + SELECT n%10, n%30, n%100 FROM c; +CREATE INDEX t1a ON t1(a); +CREATE INDEX t1b ON t1(b); +ANALYZE; +.eqp on +.fullschema +.check <<END +CREATE TABLE t1(a,b,c); +CREATE INDEX t1a ON t1(a); +CREATE INDEX t1b ON t1(b); +ANALYZE sqlite_schema; +INSERT INTO sqlite_stat1 VALUES('t1','t1b','300 10'), + ('t1','t1a','300 30'); +ANALYZE sqlite_schema; +END + +.testcase 110 +.schema +.check <<END +CREATE TABLE t1(a,b,c); +CREATE INDEX t1a ON t1(a); +CREATE INDEX t1b ON t1(b); +CREATE TABLE sqlite_stat1(tbl,idx,stat); +END + +.testcase 120 +.tables +.check --glob "t1" + +.testcase 130 +.indexes +.check --glob t1a*t1b diff --git a/test/e_expr.test b/test/e_expr.test index 81d2fd172..5c0bfb0c1 100644 --- a/test/e_expr.test +++ b/test/e_expr.test @@ -1744,10 +1744,10 @@ do_execsql_test e_expr-32.2.8 { integer 9223372036854775807 \ integer 9223372036854775807 \ integer 9223372036854775807 \ - real 9.22337203685478e+18 \ - real 9.22337203685478e+18 \ - real 9.22337203685478e+18 \ - real 9.22337203685478e+18 \ + real 9.2233720368547758e+18 \ + real 9.2233720368547758e+18 \ + real 9.2233720368547758e+18 \ + real 9.2233720368547758e+18 \ integer -5 \ integer -5 \ ] diff --git a/test/e_update.test b/test/e_update.test index a13b059b3..2b3341dc7 100644 --- a/test/e_update.test +++ b/test/e_update.test @@ -325,6 +325,8 @@ foreach {tn sql error ac data } { # EVIDENCE-OF: R-43190-62442 In other words, the schema-name. prefix on # the table name of the UPDATE is not allowed within triggers. # +# Update: Unless the trigger is in the temp schema. +# do_update_tests e_update-2.1 -error { qualified table names are not allowed on INSERT, UPDATE, and DELETE statements within triggers } { @@ -339,12 +341,14 @@ do_update_tests e_update-2.1 -error { UPDATE aux.t1 SET a=1, b=2; END; } {} +} - 3 { - CREATE TRIGGER tr1 AFTER DELETE ON t4 BEGIN - UPDATE main.t1 SET a=1, b=2; - END; - } {} +# Qualified table name is allowed as t4 is a temp table. +do_execsql_test e_update-2.1.3 { + CREATE TRIGGER tr1 AFTER DELETE ON t4 BEGIN + UPDATE main.t1 SET a=1, b=2; + END; + DROP TRIGGER tr1; } # EVIDENCE-OF: R-06085-13761 Unless the table to which the trigger is diff --git a/test/e_walckpt.test b/test/e_walckpt.test index 3b1f3b015..4aae52a78 100644 --- a/test/e_walckpt.test +++ b/test/e_walckpt.test @@ -605,14 +605,15 @@ foreach {tn script} { sqlite3 db test.db foreach {tn mode res} { 0 -1001 {1 {SQLITE_MISUSE - not an error}} - 1 -1 {1 {SQLITE_MISUSE - not an error}} - 2 0 {0 {0 -1 -1}} - 3 1 {0 {0 -1 -1}} - 4 2 {0 {0 -1 -1}} - 5 3 {0 {0 -1 -1}} - 6 4 {1 {SQLITE_MISUSE - not an error}} - 7 114 {1 {SQLITE_MISUSE - not an error}} - 8 1000000 {1 {SQLITE_MISUSE - not an error}} + 1 -2 {1 {SQLITE_MISUSE - not an error}} + 2 -1 {0 {0 -1 -1}} + 3 0 {0 {0 -1 -1}} + 4 1 {0 {0 -1 -1}} + 5 2 {0 {0 -1 -1}} + 6 3 {0 {0 -1 -1}} + 7 4 {1 {SQLITE_MISUSE - not an error}} + 8 114 {1 {SQLITE_MISUSE - not an error}} + 9 1000000 {1 {SQLITE_MISUSE - not an error}} } { do_test 4.$tn { list [catch "wal_checkpoint_v2 db $mode" msg] $msg diff --git a/test/eqp.test b/test/eqp.test index 147b5ceaf..d2bdc49e3 100644 --- a/test/eqp.test +++ b/test/eqp.test @@ -124,10 +124,10 @@ do_eqp_test 1.8 { } { QUERY PLAN |--CO-ROUTINE (subquery-xxxxxx) - | `--COMPOUND QUERY - | |--LEFT-MOST SUBQUERY + | `--MERGE (UNION) + | |--LEFT | | `--SCAN CONSTANT ROW - | `--UNION USING TEMP B-TREE + | `--RIGHT | `--SCAN CONSTANT ROW |--SCAN (subquery-xxxxxx) `--SCAN t3 @@ -137,24 +137,26 @@ do_eqp_test 1.9 { } { QUERY PLAN |--CO-ROUTINE abc - | `--COMPOUND QUERY - | |--LEFT-MOST SUBQUERY + | `--MERGE (EXCEPT) + | |--LEFT | | `--SCAN CONSTANT ROW - | `--EXCEPT USING TEMP B-TREE - | `--SCAN t3 - |--SCAN abc - `--SCAN t3 + | `--RIGHT + | |--SCAN t3 + | `--USE TEMP B-TREE FOR ORDER BY + |--SCAN t3 + `--SCAN abc } do_eqp_test 1.10 { SELECT * FROM t3 JOIN (SELECT 1 INTERSECT SELECT a FROM t3 LIMIT 17) AS abc } { QUERY PLAN |--CO-ROUTINE abc - | `--COMPOUND QUERY - | |--LEFT-MOST SUBQUERY + | `--MERGE (INTERSECT) + | |--LEFT | | `--SCAN CONSTANT ROW - | `--INTERSECT USING TEMP B-TREE - | `--SCAN t3 + | `--RIGHT + | |--SCAN t3 + | `--USE TEMP B-TREE FOR ORDER BY |--SCAN abc `--SCAN t3 } @@ -455,10 +457,11 @@ do_eqp_test 4.3.1 { SELECT x FROM t1 UNION SELECT x FROM t2 } { QUERY PLAN - `--COMPOUND QUERY - |--LEFT-MOST SUBQUERY - | `--SCAN t1 - `--UNION USING TEMP B-TREE + `--MERGE (UNION) + |--LEFT + | |--SCAN t1 + | `--USE TEMP B-TREE FOR ORDER BY + `--RIGHT `--SCAN t2 USING COVERING INDEX t2i1 } @@ -466,13 +469,17 @@ do_eqp_test 4.3.2 { SELECT x FROM t1 UNION SELECT x FROM t2 UNION SELECT x FROM t1 } { QUERY PLAN - `--COMPOUND QUERY - |--LEFT-MOST SUBQUERY - | `--SCAN t1 - |--UNION USING TEMP B-TREE - | `--SCAN t2 USING COVERING INDEX t2i1 - `--UNION USING TEMP B-TREE - `--SCAN t1 + `--MERGE (UNION) + |--LEFT + | `--MERGE (UNION) + | |--LEFT + | | |--SCAN t1 + | | `--USE TEMP B-TREE FOR ORDER BY + | `--RIGHT + | `--SCAN t2 USING COVERING INDEX t2i1 + `--RIGHT + |--SCAN t1 + `--USE TEMP B-TREE FOR ORDER BY } do_eqp_test 4.3.3 { SELECT x FROM t1 UNION SELECT x FROM t2 UNION SELECT x FROM t1 ORDER BY 1 diff --git a/test/expridx1.test b/test/expridx1.test new file mode 100644 index 000000000..e0efa5658 --- /dev/null +++ b/test/expridx1.test @@ -0,0 +1,273 @@ +# 2026-03-16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains test cases for handling stale expression indexes - +# expression indexes for which the value of the expression is different +# from (though usually very close to) the value stored on disk. +# +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix expridx1 + +# Check that OP_IdxDelete works when: +# +# 1.1.* The real value in the index is greater than that in the table, +# 1.2.* The real value in the index is smaller than that in the table, +# 1.3.* Duplicate distorted real values. +# + +do_execsql_test 1.0.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b REAL); + INSERT INTO t1 VALUES(10, 10.0); + INSERT INTO t1 VALUES(15, 15.0); + INSERT INTO t1 VALUES(20, 20.0); + INSERT INTO t1 VALUES(25, 25.0); + INSERT INTO t1 VALUES(30, 30.0); + CREATE INDEX i1 ON t1((b+0.0)); +} + +# Return a list of rowids from table t1 for which there is no exact +# match in the index. +set idxcheck { + SELECT rowid FROM t1 AS o NOT INDEXED + WHERE NOT EXISTS (SELECT 1 FROM t1 WHERE +a=o.a AND b+0.0=o.b+0.0) +} + +set root [db one {SELECT rootpage FROM sqlite_schema WHERE name='i1'}] + +do_test 1.0.2 { + # Create an imposter table for index i1 + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $root + db eval { + CREATE TABLE x1(b, rowid, PRIMARY KEY(b, rowid)) WITHOUT ROWID; + } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 + + db one { PRAGMA integrity_check } +} ok + +do_execsql_test 1.1.1 { + UPDATE x1 SET b=21.0 WHERE rowid=20; +} +do_execsql_test 1.1.1b { + PRAGMA integrity_check; +} {{row 3 missing from index i1}} +do_execsql_test 1.1.1c $idxcheck {20} + +do_execsql_test 1.1.2 { + DELETE FROM t1 WHERE a=20; +} {} +do_execsql_test 1.1.3 { + PRAGMA integrity_check; +} {ok} +do_execsql_test 1.1.4 $idxcheck {} + +do_execsql_test 1.2.1 { + UPDATE x1 SET b=26.0 WHERE rowid=25; + PRAGMA integrity_check; +} { + {row 3 missing from index i1} +} +do_execsql_test 1.2.2 $idxcheck {25} + +do_execsql_test 1.2.3 { + DELETE FROM t1 WHERE a=25; +} {} +do_execsql_test 1.2.4 { + PRAGMA integrity_check; +} {ok} +do_execsql_test 1.2.5 $idxcheck {} + +do_execsql_test 1.3.1 { + DELETE FROM t1; + INSERT INTO t1 VALUES(5, 15.0); + INSERT INTO t1 VALUES(10, 20.0); + INSERT INTO t1 VALUES(15, 20.0); + INSERT INTO t1 VALUES(20, 20.0); + INSERT INTO t1 VALUES(25, 20.0); + INSERT INTO t1 VALUES(30, 20.0); + INSERT INTO t1 VALUES(35, 25.0); + + UPDATE x1 SET b=19.0 WHERE b=20.0; + PRAGMA integrity_check; +} { + {row 2 missing from index i1} + {row 3 missing from index i1} + {row 4 missing from index i1} + {row 5 missing from index i1} + {row 6 missing from index i1} +} + +do_execsql_test 1.3.2 $idxcheck {10 15 20 25 30} + +foreach {tn a} { + 1 15 2 30 3 20 4 10 5 25 +} { + do_execsql_test 1.3.3.$tn { + DELETE FROM t1 WHERE a=$a + } +} +do_execsql_test 1.3.4 { + PRAGMA integrity_check +} {ok} +do_execsql_test 1.3.5 $idxcheck {} + +#------------------------------------------------------------------------- +reset_db +set nRow [expr 1000] +do_execsql_test 2.0 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a, b)) WITHOUT ROWID; + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$nRow + ) + INSERT INTO t1 SELECT i, random(), hex(randomblob(50)) FROM s; + CREATE INDEX t1c ON t1(+c); +} + +set idxcheck { + SELECT a, b FROM t1 AS o NOT INDEXED + WHERE NOT EXISTS (SELECT 1 FROM t1 WHERE +a=o.a AND +b=o.b AND +c=o.c) +} + +set root [db one {SELECT rootpage FROM sqlite_schema WHERE name='t1c'}] + +do_test 2.1 { + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $root + db eval { + CREATE TABLE x1(a, b, c, PRIMARY KEY(c, a, b)) WITHOUT ROWID; + } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 +} {} + +do_execsql_test 2.2 { + UPDATE x1 SET c=hex(randomblob(50)) WHERE (a%2)!=0 +} + +do_execsql_test 2.3 " + SELECT count(*) FROM ( $idxcheck ) +" [expr $nRow/2] + +do_test 2.4 { + for {set ii 1} {$ii<$nRow} {incr ii 2} { + execsql { DELETE FROM t1 WHERE a=$ii } + } + execsql {PRAGMA integrity_check} +} {ok} + +do_execsql_test 2.5 " + SELECT count(*) FROM ( $idxcheck ) +" 0 + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE TABLE y1(a, b, c GENERATED ALWAYS AS (a*b) VIRTUAL); + CREATE INDEX i1 ON y1(c); + INSERT INTO y1 VALUES(2, 3); + INSERT INTO y1 VALUES(4, 5); +} + +set idxcheck { + SELECT rowid FROM y1 AS o NOT INDEXED + WHERE NOT EXISTS (SELECT 1 FROM y1 WHERE +rowid=o.rowid AND c=o.c) +} + +set root [db one {SELECT rootpage FROM sqlite_schema WHERE name='i1'}] + +do_test 3.1 { + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $root + db eval { CREATE TABLE x1(c, rowid, PRIMARY KEY(c, rowid)) WITHOUT ROWID; } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 +} {} + +do_execsql_test 3.2 { + UPDATE x1 SET c=19 WHERE rowid=2; +} + +do_execsql_test 3.3 $idxcheck 2 + +do_execsql_test 3.4 { + DELETE FROM y1 WHERE a=4; +} +do_execsql_test 3.5 $idxcheck {} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE TABLE z1(a INTEGER PRIMARY KEY, b); + CREATE INDEX z1b ON z1(b+0.0); + INSERT INTO z1 VALUES(1, 1.0); + INSERT INTO z1 VALUES(2, 4.0); + INSERT INTO z1 VALUES(3, 4.0); + INSERT INTO z1 VALUES(4, 4.0); + INSERT INTO z1 VALUES(5, 4.0); + INSERT INTO z1 VALUES(6, 4.0); + INSERT INTO z1 VALUES(7, 4.0); + INSERT INTO z1 VALUES(8, 1.0); +} + +set root [db one {SELECT rootpage FROM sqlite_schema WHERE name='z1b'}] +do_test 4.1 { + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $root + db eval { CREATE TABLE x1(b, a, PRIMARY KEY(b, a)) WITHOUT ROWID; } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 +} {} + +do_execsql_test 4.2 { + UPDATE x1 SET b=4.000000000000001 WHERE a=2; -- 1 ULP + UPDATE x1 SET b=4.000000000000002 WHERE a=3; -- 2 ULP + UPDATE x1 SET b=4.000000000000003 WHERE a=4; -- 3 ULP + UPDATE x1 SET b=3.9999999999999996 WHERE a=5; -- -1 ULP + UPDATE x1 SET b=3.9999999999999992 WHERE a=6; -- -2 ULP + UPDATE x1 SET b=3.9999999999999988 WHERE a=7; -- -3 ULP +} + +do_execsql_test 4.3 { + PRAGMA integrity_check +} { + {index z1b stores an imprecise floating-point value for row 2} + {index z1b stores an imprecise floating-point value for row 3} + {row 4 missing from index z1b} + {index z1b stores an imprecise floating-point value for row 5} + {index z1b stores an imprecise floating-point value for row 6} + {row 7 missing from index z1b} +} + +do_execsql_test 4.4 { + UPDATE z1 SET b=-4.0 WHERE b=4.0; + PRAGMA integrity_check; +} {ok} + +do_execsql_test 4.5 { + UPDATE x1 SET b=-4.000000000000001 WHERE a=2; -- -1 ULP + UPDATE x1 SET b=-4.000000000000002 WHERE a=3; -- -2 ULP + UPDATE x1 SET b=-4.000000000000003 WHERE a=4; -- -3 ULP + UPDATE x1 SET b=-3.9999999999999996 WHERE a=5; -- 1 ULP + UPDATE x1 SET b=-3.9999999999999992 WHERE a=6; -- 2 ULP + UPDATE x1 SET b=-3.9999999999999988 WHERE a=7; -- 3 ULP +} + +do_execsql_test 4.6 { + PRAGMA integrity_check +} { + {index z1b stores an imprecise floating-point value for row 2} + {index z1b stores an imprecise floating-point value for row 3} + {row 4 missing from index z1b} + {index z1b stores an imprecise floating-point value for row 5} + {index z1b stores an imprecise floating-point value for row 6} + {row 7 missing from index z1b} +} + + + + + +finish_test diff --git a/test/filectrl.test b/test/filectrl.test index 9b1a1c758..eea9a4773 100644 --- a/test/filectrl.test +++ b/test/filectrl.test @@ -36,13 +36,11 @@ do_test filectrl-1.5 { sqlite3 db test_control_lockproxy.db file_control_lockproxy_test db [get_pwd] } {} -ifcapable !winrt { - do_test filectrl-1.6 { - sqlite3 db test.db - set fn [file_control_tempfilename db] - set fn - } {/etilqs_/} -} +do_test filectrl-1.6 { + sqlite3 db test.db + set fn [file_control_tempfilename db] + set fn +} {/etilqs_/} db close forcedelete .test_control_lockproxy.db-conch test.proxy forcedelete test.db test2.db diff --git a/test/fkey8.test b/test/fkey8.test index 2d72f6fcf..5eb97c39d 100644 --- a/test/fkey8.test +++ b/test/fkey8.test @@ -249,5 +249,45 @@ do_execsql_test 6.3 { DELETE FROM aux.p1 WHERE a=123; } +#------------------------------------------------------------------------- +# Forum: https://sqlite.org/forum/forumpost/636bd0180a +# +reset_db +do_execsql_test 7.0 { + PRAGMA foreign_keys = ON; + CREATE TABLE p1 (pid PRIMARY KEY); + CREATE TABLE c1 (cid PRIMARY KEY, + pid REFERENCES p1(pid) ON UPDATE CASCADE + ); +} + +do_execsql_test 7.1 { + ATTACH ':memory:' AS aux; + CREATE TABLE aux.p1 (pid PRIMARY KEY); + CREATE TABLE aux.c1 (cid PRIMARY KEY, + pid REFERENCES p1(pid) ON UPDATE CASCADE); + + INSERT INTO aux.p1 VALUES (10); + INSERT INTO aux.p1 VALUES (20); + + INSERT INTO aux.c1 VALUES(11, 10); + INSERT INTO aux.c1 VALUES(12, 10); + INSERT INTO aux.c1 VALUES(21, 20); + INSERT INTO aux.c1 VALUES(22, 20); +} + +do_catchsql_test 7.2 { + UPDATE aux.p1 SET pid = pid * 10; +} {0 {}} + +do_execsql_test 7.3 { + SELECT * FROM aux.p1; +} {100 200} + +do_execsql_test 7.4 { + SELECT * FROM aux.c1; +} {11 100 12 100 21 200 22 200} + + finish_test diff --git a/test/fp-speed-1.c b/test/fp-speed-1.c index cb4e0409c..c55e723cc 100644 --- a/test/fp-speed-1.c +++ b/test/fp-speed-1.c @@ -2,6 +2,9 @@ ** Performance testing of floating-point binary-to-decimal conversion for ** SQLite versus the standard library. ** +** This module compares library sprintf() against SQLite's sqlite3_snprintf(). +** To go the other direction (decimal-to-binary) see fp-speed-2.c. +** ** To compile: ** ** make sqlite3.c @@ -10,9 +13,17 @@ ** ** To run the test: ** -** time ./a.out 0 10000000 <-- standard library -** time ./a.out 1 10000000 <-- SQLite +** ./a.out 10000000 */ +#include "sqlite3.h" +#include <stdio.h> +#include <stdlib.h> +#ifdef _WIN32 +# include <windows.h> +#else +# include <sys/time.h> +#endif + static double aVal[] = { -1.0163830486285643089e+063, +0.0049243807391586981e-019, @@ -117,38 +128,80 @@ static double aVal[] = { }; #define NN (sizeof(aVal)/sizeof(aVal[0])) -#include "sqlite3.h" -#include <stdio.h> -#include <stdlib.h> +/* Return the current wall-clock time in microseconds since the +** Unix epoch (1970-01-01T00:00:00Z) +*/ +static sqlite3_int64 timeOfDay(void){ +#if defined(_WIN64) && _WIN32_WINNT >= _WIN32_WINNT_WIN8 + sqlite3_uint64 t; + FILETIME tm; + GetSystemTimePreciseAsFileTime(&tm); + t = ((sqlite3_uint64)tm.dwHighDateTime<<32) | + (sqlite3_uint64)tm.dwLowDateTime; + t += 116444736000000000LL; + t /= 10; + return t; +#elif defined(_WIN32) + static sqlite3_vfs *clockVfs = 0; + sqlite3_int64 t; + if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); + if( clockVfs==0 ) return 0; /* Never actually happens */ + if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){ + clockVfs->xCurrentTimeInt64(clockVfs, &t); + }else{ + double r; + clockVfs->xCurrentTime(clockVfs, &r); + t = (sqlite3_int64)(r*86400000.0); + } + return t*1000; +#else + struct timeval sNow; + (void)gettimeofday(&sNow,0); + return ((sqlite3_int64)sNow.tv_sec)*1000000 + sNow.tv_usec; +#endif +} int main(int argc, char **argv){ int i; int cnt; - int fg; + static const char *zFmt = "%.17g"; + sqlite3_int64 tm1, tm2; char zBuf[1000]; - if( argc!=3 ){ - fprintf(stderr, "Usage: %s FLAG COUNT\n", argv[0]); + if( argc!=2 ){ + printf("Usage: %s COUNT\n", argv[0]); + printf("Suggested value for COUNT is 10 million\n"); + return 1; + } + cnt = atoi(argv[1]); + if( cnt<100 ){ + printf("Minimum COUNT value is 100"); return 1; } - cnt = atoi(argv[2]); - fg = atoi(argv[1]); - switch( fg % 3 ){ - case 0: { - printf("Doing %d calls to C-lib sprintf()\n", cnt); - for(i=0; i<cnt; i++){ - sprintf(zBuf, "%.26g", aVal[i%NN]); - } - break; - } - case 1: { - printf("Doing %d calls to sqlite3_snprintf()\n", cnt); - for(i=0; i<cnt; i++){ - sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.26g", aVal[i%NN]); - } - break; - } + printf("C-library sprintf(\"%s\"): ", zFmt); + fflush(stdout); + tm1 = timeOfDay(); + for(i=0; i<cnt; i++){ + sprintf(zBuf, zFmt, aVal[i%NN]); + } + tm1 = timeOfDay() - tm1; + printf("%6.1f ns/call, %9.6f sec total\n", tm1*1.0e+3/(double)cnt,tm1*1.0e-6); + + printf("sqlite3_snprintf(\"%s\"): ", zFmt); + tm2 = timeOfDay(); + for(i=0; i<cnt; i++){ + sqlite3_snprintf(sizeof(zBuf), zBuf, zFmt, aVal[i%NN]); + } + tm2 = timeOfDay() - tm2; + printf("%6.1f ns/call, %9.6f sec total\n", tm2*1.0e+3/(double)cnt,tm2*1.0e-6); + + if( tm1 < tm2 ){ + printf("sprintf() is about %g times faster than sqlite3_snprintf()\n", + (double)tm2/(double)tm1); + }else{ + printf("sqlite3_snprintf() is about %g times faster than sprintf()\n", + (double)tm1/(double)tm2); } return 0; } diff --git a/test/fp-speed-2.c b/test/fp-speed-2.c new file mode 100644 index 000000000..466eee4f0 --- /dev/null +++ b/test/fp-speed-2.c @@ -0,0 +1,263 @@ +/* +** Performance testing of floating-point decimal-to-binary conversion for +** SQLite versus the standard library. +** +** This module compares library atof() against SQLite's sqlite3AtoF(). +** To go the other direction (binary-to-decimal) see fp-speed-1.c. +** +** To compile: +** +** make sqlite3.c +** gcc -Os -c sqlite3.c +** gcc -I. -Os test/fp-speed-2.c sqlite3.o -ldl -lm -lpthread +** +** To run the test: +** +** ./a.out 10000000 +** +** Notes: +** +** * A first loop is run to measure the testing overhead. This +** overhead is then subtracted from the times of subsequent loops +** so that the estimates are for the calls to atof() or +** sqlite3AtoF() only. +*/ +#include "sqlite3.h" +#include <string.h> +#include <stdlib.h> +#include <stdio.h> +#ifdef _WIN32 +# include <windows.h> +#else +# include <sys/time.h> +#endif + +static const char *aVal[] = { + "-1.01638304862856430", + "+0.00492438073915869", + "+7.38187324073439948", + "+7.06785952192257171", + "+9.28072663198500256", + "+5.88710508619333946", + "-2.29980236212596628", + "-1.59035819249108474", + "+2.43134418168449782", + "-3.82909873289453990", + "+1.87879140627440013", + "+0.72706535604871466", + "+0.06395776979131836", + "+5.22923388793158619", + "+6.37476826728722313", + "+8.69723395383291069", + "-9.50744860515976919", + "-8.64802578453687530", + "-3.56575957757974703", + "-7.83231061797317613", + "+7.78138752741208003", + "-1.87397189283601965", + "+8.68981459155933572", + "+6.05667663598778378", + "+4.17033792171481606", + "+2.12838712887466515", + "-6.83952360838918106", + "-6.21142997639395291", + "-2.07535257426146373", + "+5.87274598039442902", + "+8.58889910620021018", + "+6.86244610313559176", + "-3.30539867566709051", + "-4.35968431527634449", + "+0.08341395201040996", + "-8.85819865489042224", + "-3.66220954287276982", + "-6.69658522970250632", + "+1.82041693474064884", + "+6.52345080386490003", + "+1.59230060182190114", + "+1.73625552916563894", + "+7.28754319768547858", + "+1.28358801059589267", + "+8.05162533203208194", + "+6.63246333993811454", + "-1.71265000702800620", + "+1.69957383415837123", + "+7.60487669237386637", + "+0.61591172354494550", + "+5.75448943554159432", + "+8.29702285926905818", + "-6.55319253601630674", + "+5.83213346061870300", + "+5.65571665095718917", + "+0.33227897084384087", + "-7.12106487766986866", + "-9.67212625267063433", + "-3.45839165713773950", + "+4.78960943232147507", + "-9.69260280400041378", + "+7.06838482753813854", + "-5.29701141821629619", + "-4.42870212009053932", + "+0.07288911557328087", + "-9.18554620258794474", + "+3.72941262341310077", + "+2.68574218827927192", + "-4.70706073336246853", + "+7.21758207682793342", + "-8.36784125342611634", + "+2.21748443042418221", + "+0.19498245886068610", + "-9.73340529556720719", + "-9.77938877669369998", + "-5.15611645874169315", + "-7.50489935777651747", + "+7.35560765686877845", + "-5.06816285755335998", + "+1.52097056420277478", + "-7.59897825350482960", + "+1.36541372033897758", + "-1.64417205546513720", + "-4.90424331961411259", + "-7.70636119616491307", + "+0.16994274609307662", + "+8.33743178495722168", + "-5.23553304804695800", + "-3.85100459421941479", + "-6.35136225443263398", + "+2.38693034844544289", + "+3.83527158716203602", + "-3.12631204931368879", + "-5.57947970025564908", + "-8.81098744795956043", + "-4.37273601202032169", + "-3.11099511896685399", + "-9.48418780317042682", + "-3.73984516683044072", + "+4.89840420089159599", +}; +#define NN (sizeof(aVal)/sizeof(aVal[0])) + +/* Return the current wall-clock time in microseconds since the +** Unix epoch (1970-01-01T00:00:00Z) +*/ +static sqlite3_int64 timeOfDay(void){ +#if defined(_WIN64) && _WIN32_WINNT >= _WIN32_WINNT_WIN8 + sqlite3_uint64 t; + FILETIME tm; + GetSystemTimePreciseAsFileTime(&tm); + t = ((sqlite3_uint64)tm.dwHighDateTime<<32) | + (sqlite3_uint64)tm.dwLowDateTime; + t += 116444736000000000LL; + t /= 10; + return t; +#elif defined(_WIN32) + static sqlite3_vfs *clockVfs = 0; + sqlite3_int64 t; + if( clockVfs==0 ) clockVfs = sqlite3_vfs_find(0); + if( clockVfs==0 ) return 0; /* Never actually happens */ + if( clockVfs->iVersion>=2 && clockVfs->xCurrentTimeInt64!=0 ){ + clockVfs->xCurrentTimeInt64(clockVfs, &t); + }else{ + double r; + clockVfs->xCurrentTime(clockVfs, &r); + t = (sqlite3_int64)(r*86400000.0); + } + return t*1000; +#else + struct timeval sNow; + (void)gettimeofday(&sNow,0); + return ((sqlite3_int64)sNow.tv_sec)*1000000 + sNow.tv_usec; +#endif +} + +/* +** Generate text of the i-th test floating-point literal. +*/ +static int fpLiteral(int i, char *z){ + int e, ex, len, ix; + ex = i%401; + e = ex - 200; + len = (i/401)%16 + 4; + ix = (i/(401*16))%NN; + memcpy(z, aVal[ix], len); + z[len++] = 'e'; + if( e<0 ){ + z[len++] = '-'; + e = -e; + }else{ + z[len++] = '+'; + } + z[len++] = e/100 + '0'; + z[len++] = (e/10)%10 + '0'; + z[len++] = e%10 + '0'; + z[len] = 0; + return ex; +} + +int main(int argc, char **argv){ + int i; + int cnt; + sqlite3_int64 tm[3]; + double arSum[401]; + char z[1000]; + + if( argc!=2 ){ + printf("Usage: %s COUNT\n", argv[0]); + printf("Suggested value for COUNT is 10 million\n"); + return 1; + } + cnt = atoi(argv[1]); + if( cnt<100 ){ + printf("Minimum COUNT value is 100"); + return 1; + } + + printf("test-overhead: "); + fflush(stdout); + memset(arSum, 0, sizeof(arSum)); + tm[2] = timeOfDay(); + for(i=0; i<cnt; i++){ + double r = (double)i; + int ex = fpLiteral(i,z); + arSum[ex] += r; + } + tm[2] = timeOfDay() - tm[2]; + for(i=1; i<=400; i++) arSum[0] += arSum[i]; + printf("%6.1f ns/test, %9.6f sec total\n", tm[2]*1e3/cnt, tm[2]*1e-6); + + printf("C-lib atof(): "); + fflush(stdout); + memset(arSum, 0, sizeof(arSum)); + tm[0] = timeOfDay(); + for(i=0; i<cnt; i++){ + int ex = fpLiteral(i,z); + arSum[ex] += atof(z); + } + tm[0] = timeOfDay() - tm[0] - tm[2]; + for(i=1; i<=400; i++) arSum[0] += arSum[i]; + printf("%6.1f ns/test, %9.6f sec net", tm[0]*1e3/cnt, tm[0]*1e-6); + printf(", cksum: %g\n", arSum[0]); + + printf("sqlite3AtoF(): "); + fflush(stdout); + memset(arSum, 0, sizeof(arSum)); + tm[1] = timeOfDay(); + for(i=0; i<cnt; i++){ + double r = 0.0; + int ex = fpLiteral(i,z); + sqlite3_test_control(SQLITE_TESTCTRL_ATOF, z, &r); + arSum[ex] += r; + } + tm[1] = timeOfDay() - tm[1] - tm[2]; + for(i=1; i<=400; i++) arSum[0] += arSum[i]; + printf("%6.1f ns/test, %9.6f sec net", tm[1]*1e3/cnt, tm[1]*1e-6); + printf(", cksum: %g\n", arSum[0]); + + if( tm[0] < tm[1] ){ + printf("atof() is about %g times faster than sqlite3AtoF()\n", + (double)tm[1]/(double)tm[0]); + }else{ + printf("sqlite3AtoF() is about %g times faster than atof()\n", + (double)tm[0]/(double)tm[1]); + } + return 0; +} diff --git a/test/fpconv1.test b/test/fpconv1.test index 195fdf990..a93489907 100644 --- a/test/fpconv1.test +++ b/test/fpconv1.test @@ -17,28 +17,82 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl + +# Unusual rendering cases: +# +do_execsql_test fpconv1-1.0 { + SELECT 1.23 - 2.34; +} {-1.1099999999999999} +# ^--- Not -1.11 as you would expect. -1.11 has a different bit pattern + +do_execsql_test fpconv1-1.1 { + SELECT 1.23 * 2.34; +} {2.8781999999999996} +# ^--- Not 2.8782 as you would expect. 2.8782 has a different bit pattern + +# Change significant digits to 15 and get a different result. +sqlite3_db_config db FP_DIGITS 15 +do_execsql_test fpconv1-1.2 { + SELECT 1.23 - 2.34; +} {-1.11} +do_execsql_test fpconv1-1.3 { + SELECT 1.23 * 2.34; +} {2.8782} +sqlite3_db_config db FP_DIGITS 17 + + if {[catch {load_static_extension db decimal} error]} { puts "Skipping decimal tests, hit load error: $error" finish_test; return } sqlite3_create_function db -do_execsql_test fpconv1-1.0 { +do_execsql_test fpconv1-2.0 { WITH RECURSIVE /* Number of random floating-point values to try. - ** On a circa 2016 x64 linux box, this test runs at - ** about 80000 cases per second -------------------vvvvvv */ - c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100000), + ** On a circa 2021 Ryzen 5950X running Mint Linux, and + ** compiled with -O0 -DSQLITE_DEBUG, this test runs at + ** about 150000 cases per second ------------------vvvvvvv */ + c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<500_000), fp(y) AS MATERIALIZED ( SELECT CAST( format('%+d.%019d0e%+03d', random()%10,abs(random()),random()%200) AS real) FROM c ) SELECT y FROM fp - WHERE -log10(abs(decimal_sub(dtostr(y,24),format('%!.24e',y))/y))<15.0; + WHERE -log10(abs(decimal_sub(dtostr(y,24),format('%!.24e',y))/y))<17.0; /* Number of digits of accuracy required -------^^^^ */ } {} # ^---- Expect a empty set as the result. The output is all tested numbers -# that fail to preserve at least 15 significant digits of accuracy. +# that fail to preserve at least 16 significant digits of accuracy. + +######################################################################## +# Random test to ensure that double -> text -> double conversions +# round-trip exactly. +# + +load_static_extension db ieee754 + +do_execsql_test fpconv1-3.0 { + WITH RECURSIVE + c(x,s) AS MATERIALIZED (VALUES(1,random()&0xffefffffffffffff) + UNION ALL + SELECT x+1,random()&0xffefffffffffffff + FROM c WHERE x<1_000_000), + fp(y,s) AS ( + SELECT ieee754_from_int(s),s FROM c + ), + fp2(yt,y,s) AS ( + SELECT y||'', y, s FROM fp + ) + SELECT format('%#016x',s) aS 'orig-hex', + format('%#016x',ieee754_to_int(CAST(yt AS real))) AS 'full-roundtrip', + yt AS 'rendered-as', + decimal_exp(yt,30) AS 'text-decimal', + decimal_exp(ieee754_from_int(s),30) AS 'float-decimal' + FROM fp2 + WHERE ieee754_to_int(CAST(yt AS real))<>s; +} {} +# ^---- Values that fail to round-trip will be reported finish_test diff --git a/test/fptest01.sql b/test/fptest01.sql new file mode 100644 index 000000000..9c82f4d59 --- /dev/null +++ b/test/fptest01.sql @@ -0,0 +1,186 @@ +#!sqlite3 +# +# 2026-03-01 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Floating-point <-> text conversions +# +# FAILURES IN THIS SCRIPT ARE NOT NECESSARILY THE FAULT OF SQLITE. +# +# Some of the tests below use the system strtod() function as +# an oracle of truth. These tests assume that the system strtod() +# is always correct. That is the case for Win11, Macs, most Linux +# boxes and so forth. But it possible to find a machine for which +# is not true. (One example, is Macs from around 2005.) On such +# machines, some of these tests might fail. +# +# So, in other words, a failure in any of the tests below does not +# necessarily mean that SQLite is wrong. It might mean that the +# strtod() function in the standard library of the machine on which +# the test is running is wrong. +# + +# Verify that binary64 -> text -> binary64 conversions round-trip +# successfully for 98,256 different edge-case binary64 values. The +# query result is all cases that do not round-trip without change, +# and so the query result should be an empty set. +# +.testcase 100 +.mode list +WITH + i1(i) AS (VALUES(0) UNION ALL SELECT i+1 FROM i1 WHERE i<15), + i2(j) AS (VALUES(0) UNION ALL SELECT j+1 FROM i2 WHERE j<0x7fe), + i3(k) AS (VALUES(0x0000000000000000), + (0x000ffffffffffff0), + (0x0008080808080800)), + fpint(n) AS (SELECT (j<<52)+i+k FROM i2, i1, i3), + fp(n,r) AS (SELECT n, ieee754_from_int(n) FROM fpint) +SELECT n, r FROM fp WHERE r<>(0.0 + (r||'')); +.check '' + +# Another batch of 106,444 edge cases: All postiive floating point +# values that have only a single bit set in the mantissa part of the +# number. +# +.testcase 110 +WITH + i1(i) AS (VALUES(0) UNION ALL SELECT i+1 FROM i1 WHERE i<51), + i2(j) AS (VALUES(0) UNION ALL SELECT j+1 FROM i2 WHERE j<0x7fe), + fpint(n) AS (SELECT (j<<52)+(1<<i) FROM i2, i1), + fp(n,r) AS (SELECT n, ieee754_from_int(n) FROM fpint) +SELECT n, r FROM fp WHERE r<>(0.0 + (r||'')); +.check '' + +# Verify that text -> binary64 conversions agree with system strtod(). +# for 98,256 different edge-cases. +# +.testcase 200 +.mode list +WITH + i1(i) AS (VALUES(0) UNION ALL SELECT i+1 FROM i1 WHERE i<15), + i2(j) AS (VALUES(0) UNION ALL SELECT j+1 FROM i2 WHERE j<0x7fe), + i3(k) AS (VALUES(0x0000000000000000), + (0x000ffffffffffff0), + (0x0008080808080800)), + fpint(n) AS (SELECT (j<<52)+i+k FROM i2, i1, i3), + fp(r) AS (SELECT ieee754_from_int(n) FROM fpint) +SELECT r FROM fp WHERE r<>strtod(r||''); +.check '' + + +# Another batch of 106,444 edge cases: All postiive floating point +# values that have only a single bit set in the mantissa part of the +# number. +# +.testcase 210 +WITH + i1(i) AS (VALUES(0) UNION ALL SELECT i+1 FROM i1 WHERE i<51), + i2(j) AS (VALUES(0) UNION ALL SELECT j+1 FROM i2 WHERE j<0x7fe), + fpint(n) AS (SELECT (j<<52)+(1<<i) FROM i2, i1), + fp(r) AS (SELECT ieee754_from_int(n) FROM fpint) +SELECT r FROM fp WHERE r<>strtod(r||''); +.check '' + +# Comparing SQLite's text-to-double conversion against strtod() +# for 200,000 random floating-point literals. +# +.param set $N 50_000 +.testcase 300 +WITH RECURSIVE fp(n,x) AS MATERIALIZED ( + VALUES(1,'1234.5789') + UNION ALL + SELECT n+1, format('%.*s.%.*se%+d', + (n%3)+1, + random()%1000, + abs(random()%16)+1, + abs(random()), + random()%308) + FROM fp WHERE n<$N +) SELECT x FROM fp WHERE (x+0.0)<>strtod(x); +.check '' +.testcase 301 +WITH RECURSIVE fp(n,x) AS MATERIALIZED ( + VALUES(1,'1234.5789') + UNION ALL + SELECT n+1, format('%.*s.%.*s', + (n%3)+1, + random()%1000, + abs(random()%16)+1, + abs(random())) + FROM fp WHERE n<$N +) SELECT x FROM fp WHERE (x+0.0)<>strtod(x); +.check '' +.testcase 302 +WITH RECURSIVE fp(n,x) AS MATERIALIZED ( + VALUES(1,'1234.5789') + UNION ALL + SELECT n+1, format('%.*s.%.*s', + abs(random()%7)+1, + random(), + abs(random()%10)+1, + abs(random())) + FROM fp WHERE n<$N +) SELECT x FROM fp WHERE (x+0.0)<>strtod(x); +.check '' +.testcase 303 +WITH RECURSIVE fp(n,x) AS MATERIALIZED ( + VALUES(1,'1234.5789') + UNION ALL + SELECT n+1, format('%de%+03d', + random(), + random()%308) + FROM fp WHERE n<$N +) SELECT x FROM fp WHERE (x+0.0)<>strtod(x); +.check '' + +# Another 500,000 comparisions between SQLite and strtod() from completely +# random floating point literals. +# +.testcase 310 +WITH RECURSIVE fp(n,x) AS MATERIALIZED ( + SELECT 1, format('%+.*g',abs(random()%18), ieee754_from_int(random())) + UNION ALL + SELECT n+1, format('%+.*g',abs(random()%18), ieee754_from_int(random())) + FROM fp WHERE n<$N*2 +) SELECT x FROM fp WHERE (x+0.0)<>strtod(x); +.check '' +.testcase 311 +WITH RECURSIVE fp(n,x) AS MATERIALIZED ( + SELECT 1, format('%+.*g',abs(random()%18), ieee754_from_int(random())) + UNION ALL + SELECT n+1, format('%+.*g',abs(random()%18), ieee754_from_int(random())) + FROM fp WHERE n<$N*2 +) SELECT x FROM fp WHERE (x+0.0)<>strtod(x); +.check '' +.testcase 312 +WITH RECURSIVE fp(n,x) AS MATERIALIZED ( + SELECT 1, format('%+.*g',abs(random()%18), ieee754_from_int(random())) + UNION ALL + SELECT n+1, format('%+.*g',abs(random()%18), ieee754_from_int(random())) + FROM fp WHERE n<$N*2 +) SELECT x FROM fp WHERE (x+0.0)<>strtod(x); +.check '' +.testcase 313 +WITH RECURSIVE fp(n,x) AS MATERIALIZED ( + SELECT 1, format('%+.*g',abs(random()%18), ieee754_from_int(random())) + UNION ALL + SELECT n+1, format('%+.*g',abs(random()%18), ieee754_from_int(random())) + FROM fp WHERE n<$N*2 +) SELECT x FROM fp WHERE (x+0.0)<>strtod(x); +.check '' +.testcase 314 +WITH RECURSIVE fp(n,x) AS MATERIALIZED ( + SELECT 1, format('%+.*g',abs(random()%18), ieee754_from_int(random())) + UNION ALL + SELECT n+1, format('%+.*g',abs(random()%18), ieee754_from_int(random())) + FROM fp WHERE n<$N*2 +) SELECT x FROM fp WHERE (x+0.0)<>strtod(x); +.check '' diff --git a/test/fts3comp1.test b/test/fts3comp1.test index 9f13aaa64..b5077a355 100644 --- a/test/fts3comp1.test +++ b/test/fts3comp1.test @@ -112,4 +112,81 @@ do_catchsql_test 2.2 { CREATE VIRTUAL TABLE t2 USING fts4(x, uncompress=unzip) } {1 {missing compress parameter in fts4 constructor}} +#-------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + PRAGMA trusted_schema = OFF; +} + +set ::myfunc_invoked 0 +proc myfunc {data} { + incr ::myfunc_invoked + return $data +} +db func myfunc myfunc + +do_execsql_test 3.1 { + CREATE VIEW v1 AS SELECT myfunc('xyz'); +} + +do_catchsql_test 3.2 { + SELECT * FROM v1 +} {1 {unsafe use of myfunc()}} + +do_execsql_test 3.3 { + CREATE VIRTUAL TABLE f1 USING fts4(x, compress=myfunc, uncompress=myfunc); +} + +do_catchsql_test 3.4 { + INSERT INTO f1(rowid, x) VALUES(123, 'one two three'); +} {1 {SQL logic error}} + +do_test 3.5 { + set ::myfunc_invoked +} {0} + +do_execsql_test 3.6.1 { + CREATE TABLE t1(x); + CREATE TABLE t2(y); + + CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN + INSERT INTO t2 VALUES( myfunc(new.x) ); + END; +} + +do_catchsql_test 3.6.2 { + INSERT INTO t1 VALUES('hello world'); +} {1 {unsafe use of myfunc()}} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE v1 USING fts4(x, compress=comp, uncompress=uncomp); +} + +proc comp {data} { return $data } +proc uncomp {data} { return $data } + +db func comp comp +db func uncomp uncomp + +do_catchsql_test 4.1 { + INSERT INTO v1 VALUES('one two three'); +} {0 {}} + +db close +sqlite3 db test.db +db func comp -directonly comp + +do_catchsql_test 4.2 { + INSERT INTO v1 VALUES('one two three'); +} {1 {SQL logic error}} + +db func uncomp -directonly uncomp + +do_catchsql_test 4.3 { + SELECT * FROM v1 +} {1 {SQL logic error}} + + finish_test diff --git a/test/fts4content.test b/test/fts4content.test index 980586ea3..8268e734a 100644 --- a/test/fts4content.test +++ b/test/fts4content.test @@ -638,7 +638,6 @@ do_catchsql_test 11.1 { # Check that an fts4 table cannot be its own content table. # reset_db -breakpoint do_execsql_test 12.1.1 { CREATE VIRTUAL TABLE t1 USING fts4(a, content=t1 ); INSERT INTO t1(rowid, a) VALUES(1, 'abc'); @@ -669,6 +668,64 @@ do_catchsql_test 12.2.4 { SELECT count(*) FROM t1; } {1 {SQL logic error}} +#--------------------------------------------------------------------------- +# Check that an fts4 table cannot read from an unsafe vtab in a non-trusted +# schema. +reset_db +do_execsql_test 13.0 { + PRAGMA trusted_schema = off; + CREATE VIRTUAL TABLE t1 USING fts4(data, content=sqlite_dbpage); +} + +do_catchsql_test 13.1 { + INSERT INTO t1(t1) VALUES('rebuild'); +} {1 {SQL logic error}} + +proc vtab_command {method args} { + switch -- $method { + xConnect { + return "CREATE TABLE t1(a)" + } + + xBestIndex { + return "" + } + + xFilter { + return [list sql {SELECT 1, 123}] + } + + xUpdate { + return 123 + } + } + + return {} +} + +register_tcl_module db xyz + +do_execsql_test 13.2.0 { + CREATE VIRTUAL TABLE aa USING tcl(vtab_command); +} + +do_execsql_test 13.2.1 { + INSERT INTO aa VALUES('one two three'); +} + +do_test 13.2.2 { + set ::stmt [sqlite3_prepare_v3 db \ + "INSERT INTO aa VALUES('one two three');" -1 0x00 + ] + sqlite3_finalize $::stmt +} {SQLITE_OK} +do_test 13.2.2 { + list [catch { + set ::stmt [sqlite3_prepare_v3 db \ + "INSERT INTO aa VALUES('one two three');" -1 0x20 + ] + } msg] $msg +} {1 {(1) unsafe use of virtual table "aa"}} finish_test diff --git a/test/fts4merge5.test b/test/fts4merge5.test index 1fad778b9..90870ca34 100644 --- a/test/fts4merge5.test +++ b/test/fts4merge5.test @@ -53,6 +53,9 @@ for {set tn 1} {1} {incr tn} { } } +do_catchsql_test 1.5 { + INSERT INTO x1(x1) VALUES('maxpendinAB64'); +} {1 {SQL logic error}} finish_test diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index a3377770a..46328fb14 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -169,8 +169,18 @@ static struct GlobalVars { /* ** Include various extensions. */ -extern int sqlite3_vt02_init(sqlite3*,char**,const sqlite3_api_routines*); -extern int sqlite3_randomjson_init(sqlite3*,char**,const sqlite3_api_routines*); +extern int sqlite3_vt02_init(sqlite3*,char**,void*); +extern int sqlite3_randomjson_init(sqlite3*,char**,void*); +extern int sqlite3_series_init(sqlite3*,char**,void*); +extern int sqlite3_base64_init(sqlite3*,char**,void*); +extern int sqlite3_base85_init(sqlite3*,char**,void*); +extern int sqlite3_completion_init(sqlite3*,char**,void*); +extern int sqlite3_decimal_init(sqlite3*,char**,void*); +extern int sqlite3_ieee_init(sqlite3*,char**,void*); +extern int sqlite3_regexp_init(sqlite3*,char**,void*); +extern int sqlite3_shathree_init(sqlite3*,char**,void*); +extern int sqlite3_sha_init(sqlite3*,char**,void*); +extern int sqlite3_stmtrand_init(sqlite3*,char**,void*); /* ** Print an error message and quit. @@ -1382,6 +1392,16 @@ int runCombinedDbSqlInput( /* Activate extensions */ sqlite3_randomjson_init(cx.db, 0, 0); + sqlite3_series_init(cx.db, 0, 0); + sqlite3_base64_init(cx.db, 0, 0); + sqlite3_base85_init(cx.db, 0, 0); + sqlite3_completion_init(cx.db, 0, 0); + sqlite3_decimal_init(cx.db, 0, 0); + sqlite3_ieee_init(cx.db, 0, 0); + sqlite3_regexp_init(cx.db, 0, 0); + sqlite3_shathree_init(cx.db, 0, 0); + sqlite3_sha_init(cx.db, 0, 0); + sqlite3_stmtrand_init(cx.db, 0, 0); /* Add support for sqlite_dbdata and sqlite_dbptr virtual tables used ** by the recovery API */ @@ -1954,6 +1974,7 @@ int main(int argc, char **argv){ char **azSrcDb = 0; /* Array of source database names */ int iSrcDb; /* Loop over all source databases */ int nTest = 0; /* Total number of tests performed */ + int nSliceSkip = 0; /* Skipped due to --slice */ char *zDbName = ""; /* Appreviated name of a source database */ const char *zFailCode = 0; /* Value of the TEST_FAILURE env variable */ int cellSzCkFlag = 0; /* --cell-size-check */ @@ -2520,6 +2541,7 @@ int main(int argc, char **argv){ if( isDbSql(pSql->a, pSql->sz) ){ if( iSliceSz>0 && (nTest%iSliceSz)!=iSliceIdx ){ nTest++; + nSliceSkip++; continue; } sqlite3_snprintf(sizeof(g.zTestName), g.zTestName, "sqlid=%d",pSql->id); @@ -2582,6 +2604,7 @@ int main(int argc, char **argv){ const char *zVfs = "inmem"; if( iSliceSz>0 && (nTest%iSliceSz)!=iSliceIdx ){ nTest++; + nSliceSkip++; continue; } sqlite3_snprintf(sizeof(g.zTestName), g.zTestName, "sqlid=%d,dbid=%d", @@ -2693,12 +2716,13 @@ int main(int argc, char **argv){ if( briefFlag ){ sqlite3_int64 iElapse = timeOfDay() - iBegin; if( iSliceSz>0 ){ - printf("%s %s: slice %d/%d of %d tests, %d.%03d seconds\n", - pathTail(argv[0]), pathTail(g.zDbFile), - iSliceIdx, iSliceSz, nTest, - (int)(iElapse/1000), (int)(iElapse%1000)); + printf( + "%s %s: 0 errors out of %d tests, slice %d/%d, %d.%03d seconds\n", + pathTail(argv[0]), pathTail(g.zDbFile), + nTest - nSliceSkip, iSliceIdx, iSliceSz, + (int)(iElapse/1000), (int)(iElapse%1000)); }else{ - printf("%s %s: 0 errors, %d tests, %d.%03d seconds\n", + printf("%s %s: 0 errors out of %d tests, %d.%03d seconds\n", pathTail(argv[0]), pathTail(g.zDbFile), nTest, (int)(iElapse/1000), (int)(iElapse%1000)); } diff --git a/test/fuzzinvariants.c b/test/fuzzinvariants.c index 6a5cfda68..9341b2056 100644 --- a/test/fuzzinvariants.c +++ b/test/fuzzinvariants.c @@ -261,9 +261,12 @@ int fuzz_invariant( sqlite3_finalize(pCk); /* Invariants do not necessarily work if there are virtual tables - ** involved in the query */ - rc = sqlite3_prepare_v2(db, - "SELECT 1 FROM bytecode(?1) WHERE opcode='VOpen'", -1, &pCk, 0); + ** or scalar subqueries involved in the query */ + rc = sqlite3_prepare_v2(db, + "SELECT 1 FROM bytecode(?1)" + " WHERE opcode='VOpen' OR" + " (opcode='Explain' AND p4 GLOB 'SCALAR SUBQUERY*')", + -1, &pCk, 0); if( rc==SQLITE_OK ){ if( eVerbosity>=2 ){ char *zSql = sqlite3_expanded_sql(pCk); diff --git a/test/import01.sql b/test/import01.sql new file mode 100644 index 000000000..6e029b8b2 --- /dev/null +++ b/test/import01.sql @@ -0,0 +1,217 @@ +#!sqlite3 +# +# 2025-12-28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the ".import" command of the CLI. +# To run these tests: +# +# ./sqlite3 test/import01.sql +# + +.testcase setup +.open :memory: +.mode tty +.check '' + +.testcase 100 +CREATE TABLE t1(a,b,c); +.import -csv <<END t1 +111,222,333 +abc,def,ghi +END +SELECT * FROM t1; +.check <<END +╭───────┬───────┬───────╮ +│ a │ b │ c │ +╞═══════╪═══════╪═══════╡ +│ '111' │ '222' │ '333' │ +│ abc │ def │ ghi │ +╰───────┴───────┴───────╯ +END + +.testcase 110 +DELETE FROM t1; +.import -colsep ";" <<END t1 +this;is a;test +one;two;three +END +SELECT * FROM t1; +.check <<END +╭──────┬──────┬───────╮ +│ a │ b │ c │ +╞══════╪══════╪═══════╡ +│ this │ is a │ test │ +│ one │ two │ three │ +╰──────┴──────┴───────╯ +END + +.testcase 120 +DELETE FROM t1; +.import -colsep "," -rowsep ';' <<END t1 +this,is a,test;one,two,three; +END +SELECT * FROM t1; +.check <<END +╭──────┬──────┬───────╮ +│ a │ b │ c │ +╞══════╪══════╪═══════╡ +│ this │ is a │ test │ +│ one │ two │ three │ +╰──────┴──────┴───────╯ +END + +.testcase 130 +DELETE FROM t1; +.import -csv -colsep "," -rowsep "\n" <<END t1 +this,"is a","test ""with quotes""" +"second",,"line" +END +SELECT * FROM t1; +.check <<END +╭────────┬──────┬────────────────────╮ +│ a │ b │ c │ +╞════════╪══════╪════════════════════╡ +│ this │ is a │ test "with quotes" │ +│ second │ │ line │ +╰────────┴──────┴────────────────────╯ +END +.testcase 131 +DELETE FROM t1; +.import -ascii -colsep "," -rowsep "\n" <<END t1 +this,"is a","test ""with quotes""" +"second",,"line" +END +SELECT * FROM t1; +.check <<END +╭──────────┬────────┬────────────────────────╮ +│ a │ b │ c │ +╞══════════╪════════╪════════════════════════╡ +│ this │ "is a" │ "test ""with quotes""" │ +│ "second" │ │ "line" │ +╰──────────┴────────┴────────────────────────╯ +END + +.testcase 140 +DROP TABLE t1; +.import -csv <<END t1 +"abc","def","xy z" +"This","is","a" +"test","...", +END +SELECT * FROM t1; +.check <<END +╭──────┬─────┬──────╮ +│ abc │ def │ xy z │ +╞══════╪═════╪══════╡ +│ This │ is │ a │ +│ test │ ... │ │ +╰──────┴─────┴──────╯ +END +.testcase 141 +SELECT sql FROM sqlite_schema WHERE name='t1'; +.check <<END +╭───────────────────────────────────╮ +│ sql │ +╞═══════════════════════════════════╡ +│ CREATE TABLE "t1"( │ +│ "abc" ANY, "def" ANY, "xy z" ANY) │ +╰───────────────────────────────────╯ +END + +.testcase 150 +DROP TABLE t1; +.import -csv -v <<END t1 +"abc","def","xy z" +"This","is","a" +"test","...", +END +SELECT * FROM t1; +.check <<END +Column separator ",", row separator "\n" +CREATE TABLE "main"."t1"( +"abc" ANY, "def" ANY, "xy z" ANY) + +Added 2 rows with 0 errors using 3 lines of input +╭──────┬─────┬──────╮ +│ abc │ def │ xy z │ +╞══════╪═════╪══════╡ +│ This │ is │ a │ +│ test │ ... │ │ +╰──────┴─────┴──────╯ +END + +.testcase 160 +DROP TABLE t1; +.import -csv -schema TEMP <<END t2 +"x" +"abcdef" +END +SELECT * FROM t2; +.tables +.check <<END +╭────────╮ +│ x │ +╞════════╡ +│ abcdef │ +╰────────╯ +temp.t2 +END + +.testcase 170 +.import -csv -skip 2 <<END t3 +a,b,c +d,e,f +g,h,i +j,k,l +m,n,o +END +SELECT * FROM t3; +.check <<END +╭───┬───┬───╮ +│ g │ h │ i │ +╞═══╪═══╪═══╡ +│ j │ k │ l │ +│ m │ n │ o │ +╰───┴───┴───╯ +END + +.testcase 180 +DELETE FROM t3; +.import -csv -skip 7 <<END t3 +a,b,c +d,e,f +g,h,i +j,k,l +m,n,o +END +SELECT * FROM t3; +.check <<END +END + +.testcase 200 --error-prefix ERROR: +.import -csv +.check 'ERROR: Missing FILE argument' +.testcase 201 +.import -csv file1.csv +.check 'ERROR: Missing TABLE argument' +.testcase --error-prefix test-error: 202 +.import -csvxyzzy file1.csv +.check <<END +test-error: .import -csvxyzzy file1.csv +test-error: ^--- unknown option +END +.testcase 203 +.import -csv file1.csv t4 -colsep +.check <<END +test-error: .import -csv file1.csv t4 -colsep +test-error: missing argument ---^ +END diff --git a/test/imposter1.sql b/test/imposter1.sql new file mode 100644 index 000000000..9604b43f4 --- /dev/null +++ b/test/imposter1.sql @@ -0,0 +1,32 @@ +#!sqlite3 +# +# 2025-12-16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the .imposter command. +# +.mode box -reset +.testcase 100 +CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c INT); +INSERT INTO t1 VALUES(1,'two',3),(4,'five',6); +CREATE INDEX t1bc ON t1(b,c); +.imposter T1BC x1 +----------^^^^--- Different case that the original +SELECT * FROM x1; +.check <<END +CREATE TABLE "x1"("b","c","_ROWID_",PRIMARY KEY("b","c","_ROWID_"))WITHOUT ROWID; +╭──────┬───┬─────────╮ +│ b │ c │ _ROWID_ │ +╞══════╪═══╪═════════╡ +│ five │ 6 │ 4 │ +│ two │ 3 │ 1 │ +╰──────┴───┴─────────╯ +END diff --git a/test/insert5.test b/test/insert5.test index 1e58902e0..3ae99a5d2 100644 --- a/test/insert5.test +++ b/test/insert5.test @@ -96,22 +96,19 @@ do_test insert5-2.8 { } } {1} -# UPDATE: Using a column from the outer query (main.id) in the GROUP BY -# or ORDER BY of a sub-query is no longer supported. -# -# do_test insert5-2.9 { -# uses_temp_table { -# INSERT INTO b -# SELECT * FROM main -# WHERE id > 10 AND (SELECT count(*) FROM v2 GROUP BY main.id) -# } -# } {} do_test insert5-2.9 { catchsql { INSERT INTO b SELECT * FROM main WHERE id > 10 AND (SELECT count(*) FROM v2 GROUP BY main.id) } -} {1 {no such column: main.id}} +} {0 {}} +do_execsql_test insert5-2.10 { + CREATE TABLE t1(a INT); + INSERT INTO t1 VALUES(2); + CREATE TABLE t2(c INT, d INT); + INSERT INTO t2 VALUES(3,4),(10,NULL); + SELECT (SELECT c FROM t2 ORDER BY coalesce(d,a) LIMIT 1) FROM t1; +} {10} finish_test diff --git a/test/intck01.sql b/test/intck01.sql new file mode 100644 index 000000000..b1996aeeb --- /dev/null +++ b/test/intck01.sql @@ -0,0 +1,23 @@ +#!sqlite3 +# +# 2026-03-01 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Bug report sqlite.org/forum/forumpost/efc9bc9cb3 +# +.testcase 100 +.mode quote +.intck 1 +SELECT parse_create_index('CREATE IDEX i ON t("x',0); +.check <<END +1 steps, 0 errors +NULL +END diff --git a/test/join.test b/test/join.test index b33a7560a..a1ce7da0c 100644 --- a/test/join.test +++ b/test/join.test @@ -1369,4 +1369,45 @@ do_execsql_test join-32.3 { INNER JOIN t2 ON +y IS z; } {NULL NULL 123 NULL} +# 2025-12-24 https://sqlite.org/forum/forumpost/11a53f2bad +# +# Chained omit-noop-join optimization +# +reset_db +db null NULL +do_execsql_test join-33.1 { + CREATE TABLE t1(a1 INTEGER PRIMARY KEY, b1); + CREATE TABLE t2(a2 INTEGER PRIMARY KEY, b2); + CREATE TABLE t3(a3 INTEGER PRIMARY KEY, b3); + CREATE TABLE t4(a4 INTEGER PRIMARY KEY, b4); + INSERT INTO t1 VALUES(1,11),(2,12),(3,13), (5,15); + INSERT INTO t2 VALUES(1,21), (3,23),(4,24),(5,25); + INSERT INTO t3 VALUES (2,32),(3,33), (5,35); + INSERT INTO t4 VALUES(1,41),(2,42), (4,44),(5,45); + CREATE VIEW vchain AS + SELECT a1, b1, b2, b3, b4 + FROM t1 LEFT JOIN t2 ON a1=a2 + LEFT JOIN t3 ON a2=a3 + LEFT JOIN t4 ON a3=a4; +} +do_execsql_test join-33.2 { + SELECT a1 FROM vchain ORDER BY a1; +} {1 2 3 5} +do_eqp_test join-33.2-eqp { + SELECT a1 FROM vchain ORDER BY a1; +} { + QUERY PLAN + `--SCAN t1 +} +do_execsql_test join-33.3 { + SELECT a1, b2 FROM vchain ORDER BY a1; +} {1 21 2 NULL 3 23 5 25} +do_eqp_test join-33.3-eqp { + SELECT a1, b2 FROM vchain ORDER BY a1; +} { + QUERY PLAN + |--SCAN t1 + `--SEARCH t2 USING INTEGER PRIMARY KEY (rowid=?) LEFT-JOIN +} + finish_test diff --git a/test/json/json-speed-check.sh b/test/json/json-speed-check.sh index 1eabbb3db..9b0796183 100755 --- a/test/json/json-speed-check.sh +++ b/test/json/json-speed-check.sh @@ -72,7 +72,7 @@ done echo "NAME = $NAME" | tee summary-$NAME.txt echo "CC_OPTS = $CC_OPTS" | tee -a summary-$NAME.txt rm -f cachegrind.out.* jsonshell -$CC -g -Os -Wall -I. $CC_OPTS ./shell.c ./sqlite3.c -o jsonshell -ldl -lpthread +$CC -g -Os -Wall -I. $CC_OPTS ./shell.c ./sqlite3.c -o jsonshell -lm -ldl -lpthread ls -l jsonshell | tee -a summary-$NAME.txt home=`echo $0 | sed -e 's,/[^/]*$,,'` DB=$TYPE''100mb.db diff --git a/test/json102.test b/test/json102.test index 54a0e1e0e..54f66a83f 100644 --- a/test/json102.test +++ b/test/json102.test @@ -375,6 +375,27 @@ do_execsql_test json102-440-3 { do_execsql_test json102-440-4 { SELECT json(jsonb_remove(jsonb('[0,1,2,3,4]'),'$[2]')); } {{[0,1,3,4]}} +do_execsql_test json102-445-1 { + SELECT json_remove('[0,1,2,3,4]','$[5]'); +} {{[0,1,2,3,4]}} +do_execsql_test json102-445-2 { + SELECT json_remove('[0,1,2,3,4]','$[6]'); +} {{[0,1,2,3,4]}} +do_execsql_test json102-445-3 { + SELECT json_remove('[0,1,2,3,4]','$[4294967295]'); +} {{[0,1,2,3,4]}} +do_execsql_test json102-445-4 { + SELECT json_remove('[0,1,2,3,4]','$[4294967296]'); +} {{[0,1,2,3,4]}} +do_execsql_test json102-445-5 { + SELECT json_remove('[0,1,2,3,4]','$[4294967297]'); +} {{[0,1,2,3,4]}} +do_execsql_test json102-445-6 { + SELECT json_remove('[0,1,2,3,4]','$[42949672950]'); +} {{[0,1,2,3,4]}} +do_execsql_test json102-445-7 { + SELECT json_remove('[0,1,2,3,4]','$[42949672960]'); +} {{[0,1,2,3,4]}} do_execsql_test json102-450 { SELECT json_remove('[0,1,2,3,4]','$[2]','$[0]'); } {{[1,3,4]}} diff --git a/test/json103.test b/test/json103.test index f94217ac1..9eadf29f8 100644 --- a/test/json103.test +++ b/test/json103.test @@ -27,6 +27,9 @@ do_execsql_test json103-100 { do_catchsql_test json103-101 { SELECT json_group_array(a) FROM t1; } {1 {JSON cannot hold BLOB values}} +do_execsql_test json103-102 { + SELECT quote(jsonb_group_array(a)) FROM t1 WHERE a<0 AND typeof(a)!='blob'; +} {X'0B'} do_execsql_test json103-110 { SELECT json_group_array(a) FROM t1 WHERE rowid BETWEEN 31 AND 39; @@ -45,6 +48,10 @@ do_execsql_test json103-200 { do_catchsql_test json103-201 { SELECT json_group_object(c,a) FROM t1; } {1 {JSON cannot hold BLOB values}} +do_execsql_test json103-202 { + SELECT quote(jsonb_group_object(c,a)) FROM t1 WHERE a<0 AND typeof(a)!='blob'; +} {X'0C'} + do_execsql_test json103-210 { SELECT json_group_object(c,a) FROM t1 diff --git a/test/json105.test b/test/json105.test index 509db94e1..4a5572cf0 100644 --- a/test/json105.test +++ b/test/json105.test @@ -29,6 +29,11 @@ json_extract_test 30 {'$.b[#-2]'} {'[2,3]'} json_extract_test 31 {'$.b[#-02]'} {'[2,3]'} json_extract_test 40 {'$.b[#-3]'} 1 json_extract_test 50 {'$.b[#-4]'} NULL +json_extract_test 51 {'$.b[#-4296967295]'} NULL +json_extract_test 52 {'$.b[#-4296967296]'} NULL +json_extract_test 53 {'$.b[#-4296967297]'} NULL +json_extract_test 54 {'$.b[#-42969672950]'} NULL +json_extract_test 55 {'$.b[#-42969672960]'} NULL json_extract_test 60 {'$.b[#-2][#-1]'} 3 json_extract_test 70 {'$.b[0]','$.b[#-1]'} {'[1,4]'} diff --git a/test/json109.test b/test/json109.test new file mode 100644 index 000000000..1631a1f0f --- /dev/null +++ b/test/json109.test @@ -0,0 +1,72 @@ +# 2026-01-17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for json_array_insert(). +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix json109 + +do_execsql_test 1.1 { + SELECT json_array_insert('[1,2,3]','$[0]',999,'$[0]',888); +} {{[888,999,1,2,3]}} +do_execsql_test 1.2 { + SELECT json_array_insert('[1,2,3]','$[0]',999,'$[#]',888); +} {{[999,1,2,3,888]}} +do_execsql_test 1.3 { + SELECT json_array_insert('[1,2,3]','$[1]',888); +} {{[1,888,2,3]}} +do_execsql_test 1.4 { + SELECT json_array_insert('[1,2,3]','$[2]',888); +} {{[1,2,888,3]}} +do_execsql_test 1.5 { + SELECT json_array_insert('[1,2,3]','$[3]',888); +} {{[1,2,3,888]}} +do_execsql_test 1.6 { + SELECT json_array_insert('[1,2,3]','$[#-1]',888); +} {{[1,2,888,3]}} +do_execsql_test 1.7 { + SELECT json_array_insert('[1,2,3]','$[#-2]',888); +} {{[1,888,2,3]}} +do_execsql_test 1.8 { + SELECT json_array_insert('[1,2,3]','$[#-3]',888); +} {{[888,1,2,3]}} +do_execsql_test 1.9 { + SELECT json_array_insert('[1,2,3]','$[#-4]',888); +} {{[1,2,3]}} + +do_catchsql_test 2.1 { + SELECT json_array_insert('{a:[1,2,3]}','$.a',888); +} {1 {not an array element: '$.a'}} +do_catchsql_test 2.2 { + SELECT json_array_insert('{a:[1,2,3]}','$.b',888); +} {1 {not an array element: '$.b'}} +do_catchsql_test 2.3 { + SELECT json_array_insert('{a:[1,2,3]}','$.b[0]',888); +} {0 {{{"a":[1,2,3],"b":[888]}}}} +do_catchsql_test 2.4 { + SELECT json_array_insert('{a:[1,2,3]}','$.b.c.d[0]',888); +} {0 {{{"a":[1,2,3],"b":{"c":{"d":[888]}}}}}} +do_catchsql_test 2.5 { + SELECT json_array_insert('{a:[1,2,3]}','$.b.c.d[0',888); +} {1 {not an array element: '$.b.c.d[0'}} +do_catchsql_test 2.6 { + SELECT json_array_insert('{a:[1,2,3]}','$.b.c.d',888); +} {1 {not an array element: '$.b.c.d'}} +do_catchsql_test 2.7 { + SELECT json_array_insert('{a:[1,2,3]}','$[0]',888); +} {0 {{{"a":[1,2,3]}}}} +do_catchsql_test 2.8 { + SELECT json_array_insert('{a:[1,2,3]}','$.b[0]',888,'$.a[1]','999','$.c',0); +} {1 {not an array element: '$.c'}} + +finish_test diff --git a/test/misc5.test b/test/misc5.test index 43ee2781a..80b8d3c67 100644 --- a/test/misc5.test +++ b/test/misc5.test @@ -595,7 +595,7 @@ do_test misc5-7.1.2 { } append sql "0$tail); SELECT * FROM t1;" catchsql $sql -} {0 900} +} {1 {Recursion limit}} # Parser stack overflow is silently ignored when it occurs while parsing the diff --git a/test/modeA.sql b/test/modeA.sql new file mode 100644 index 000000000..48f71c0c9 --- /dev/null +++ b/test/modeA.sql @@ -0,0 +1,323 @@ +#!sqlite3 +# +# 2025-11-12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the ".mode" command of the CLI. +# To run these tests: +# +# ./sqlite3 test/modeA.sql +# +# +.open :memory: +CREATE TABLE t1(a,b,c,d,e); +INSERT INTO t1 VALUES(1,2.5,'three',x'4444',NULL); +INSERT INTO t1 SELECT b,c,d,e,a FROM t1; +INSERT INTO t1 SELECT d,e,a,b,c FROM t1; +.mode box + +.testcase 100 +SELECT * FROM t1; +.check <<END +╭─────┬───────┬───────┬───────┬───────╮ +│ a │ b │ c │ d │ e │ +╞═════╪═══════╪═══════╪═══════╪═══════╡ +│ 1 │ 2.5 │ three │ DD │ │ +│ 2.5 │ three │ DD │ │ 1 │ +│ DD │ │ 1 │ 2.5 │ three │ +│ │ 1 │ 2.5 │ three │ DD │ +╰─────┴───────┴───────┴───────┴───────╯ +END + +.testcase 110 +.mode --null xyz +SELECT * FROM t1; +.check <<END +╭─────┬───────┬───────┬───────┬───────╮ +│ a │ b │ c │ d │ e │ +╞═════╪═══════╪═══════╪═══════╪═══════╡ +│ 1 │ 2.5 │ three │ DD │ xyz │ +│ 2.5 │ three │ DD │ xyz │ 1 │ +│ DD │ xyz │ 1 │ 2.5 │ three │ +│ xyz │ 1 │ 2.5 │ three │ DD │ +╰─────┴───────┴───────┴───────┴───────╯ +END + +# Default output mode is qbox --quote relaxed +# +.mode tty --wrap 10 +CREATE TABLE t2(a,b,c,d); +INSERT INTO t2 VALUES(1,2.5,'three',x'4444'); +INSERT INTO t2 VALUES('The quick fox jumps over the lazy brown dog',2,3,4); +INSERT INTO t2 VALUES('10','', -1.25,NULL); +INSERT INTO t2 VALUES('a,b,c','"Double-Quoted"','-1.25','NULL'); +.testcase 120 +SELECT * FROM t2; +.check <<END +╭────────────┬────────────┬─────────┬─────────╮ +│ a │ b │ c │ d │ +╞════════════╪════════════╪═════════╪═════════╡ +│ 1 │ 2.5 │ three │ x'4444' │ +├────────────┼────────────┼─────────┼─────────┤ +│ The quick │ 2 │ 3 │ 4 │ +│ fox jumps │ │ │ │ +│ over the │ │ │ │ +│ lazy brown │ │ │ │ +│ dog │ │ │ │ +├────────────┼────────────┼─────────┼─────────┤ +│ '10' │ │ -1.25 │ NULL │ +├────────────┼────────────┼─────────┼─────────┤ +│ a,b,c │ "Double- │ '-1.25' │ 'NULL' │ +│ │ Quoted" │ │ │ +╰────────────┴────────────┴─────────┴─────────╯ +END +.testcase 130 +.mode +.check <<END +.mode qbox --limits on --quote relaxed --sw auto --textjsonb on +END +.testcase 140 +.mode -v +.check <<END +.mode qbox --align "" --border on --blob-quote auto --colsep "" --escape auto --limits on --multiinsert 3000 --null "NULL" --quote relaxed --rowsep "" --sw auto --tablename "" --textjsonb on --titles on --widths "" --wordwrap off --wrap 10 +END +.testcase 150 --error-prefix "Error:" +.mode foo +.check <<END +Error: .mode foo +Error: ^--- unknown mode +Error: Use ".help .mode" for more info +END + +.testcase 160 +.mode --null xyzzy -v +.output -glob ' --null "xyzzy"' +.testcase 170 +.mode -null abcde -v +.output -glob ' --null "abcde"' + +# Test cases for the ".explain off" command +.mode box -reset +.testcase 180 +EXPLAIN SELECT * FROM t1; +.output --notglob *────* --keep +.output --notglob "* id │ parent │ notused │ detail *" --keep +.output --glob "* Init *" +.testcase 190 +EXPLAIN QUERY PLAN SELECT * FROM t1; +.output --glob "*`--SCAN *" +.explain off +.testcase 200 +EXPLAIN SELECT * FROM t1; +.output --glob *────* +.testcase 210 +EXPLAIN QUERY PLAN SELECT * FROM t1; +.output --glob "* id │ parent │ notused │ detail *" +.explain auto + +# Test cases for limit settings in the .mode command. +.testcase 300 +.mode box --reset +.mode +.check <<END +.mode box +END +.testcase 310 +.mode --limits 5,300,20 +.mode +.check <<END +.mode box --limits on +END +.testcase 320 +.mode --limits 5,300,19 +.mode +.check <<END +.mode box --limits 5,300,19 +END +.testcase 330 +.mode --limits 0,0,0 +.mode -v +.check <<END +.mode box --align "" --border on --blob-quote auto --colsep "" --escape auto --limits off --multiinsert 0 --null "" --quote off --rowsep "" --sw 0 --tablename "" --textjsonb off --titles on --widths "" --wordwrap off +END + +.testcase 400 +.mode --linelimit 123 +.mode +.check <<END +.mode box --limits 123,0,0 +END + +.testcase 410 +.mode --linelimit 0 -charlimit 123 +.mode +.check <<END +.mode box --limits 0,123,0 +END + +.testcase 420 +.mode --charlimit 0 -titlelimit 123 +.mode +.check <<END +.mode box --limits 0,0,123 +END + +.testcase 430 +.mode list +.mode +.check <<END +.mode list +END + +.testcase 440 +.mode -limits 0,123,0 +.mode +.check <<END +.mode list --limits 0,123,0 +END + +.testcase 450 +.mode -limits 123,0,0 +.mode +.check <<END +.mode list +END + +# --titlelimit functionality +# +.testcase 500 +.mode line --limits off --titlelimit 20 +SELECT a AS 'abcdefghijklmnopqrstuvwxyz', b FROM t2 WHERE c=3; +.check <<END +abcdefghijklmnopq...: The quick fox jumps over the lazy brown dog + b: 2 +END +.testcase 510 +.mode line --titlelimit 10 +SELECT a AS 'abcdefghijklmnopqrstuvwxyz', b FROM t2 WHERE c=3; +.check <<END +abcdefg...: The quick fox jumps over the lazy brown dog + b: 2 +END +.testcase 520 +.mode line --titlelimit 2 +SELECT a AS 'abcdefghijklmnopqrstuvwxyz', b FROM t2 WHERE c=3; +.check <<END +ab: The quick fox jumps over the lazy brown dog + b: 2 +END +.testcase 530 +.mode line --titlelimit 4 +SELECT a AS 'abcd', b FROM t2 WHERE c=3; +.check <<END +abcd: The quick fox jumps over the lazy brown dog + b: 2 +END +.testcase 540 +.mode line --titlelimit 3 +SELECT a AS 'abcd', b FROM t2 WHERE c=3; +.check <<END +...: The quick fox jumps over the lazy brown dog + b: 2 +END + +# line --screenwidth and --colsep +# +.testcase 550 +.mode line --sw 40 --colsep ":-hi-:" +SELECT a AS 'abc', b FROM t2 WHERE c=3; +.check <<END +abc:-hi-:The quick fox jumps over the + lazy brown dog + b:-hi-:2 +END +.testcase 551 +.mode line --sw 40 --colsep ":-hi-:" --wordwrap off +SELECT a AS 'abc', b FROM t2 WHERE c=3; +.check <<END +abc:-hi-:The quick fox jumps over the la + zy brown dog + b:-hi-:2 +END +# 23456789 123456789 123456789 123456789 + +# https://sqlite.org/forum/forumpost/2025-12-31T19:14:24z +# +# For legacy compatibility, ".header" settings are not changed +# by ".mode" unless the --title or --reset option is used on .mode. +# +.testcase 600 +DROP TABLE IF EXISTS t1; +CREATE TABLE t1(a,b,c); +INSERT INTO t1 VALUES(1,2,3); +.header on +.mode csv +SELECT * FROM t1; +.check --glob a,b,c* + +.testcase 610 +.mode csv -reset +SELECT * FROM t1; +.check 1,2,3 + +.testcase 620 +.mode tty +.mode csv +.header on +SELECT * FROM t1; +.check --glob a,b,c* + +.testcase 630 +.mode tty +.mode csv --title on +SELECT * FROM t1; +.check --glob a,b,c* +.testcase 631 +.mode tty +.mode csv --title off +SELECT * FROM t1; +.check 1,2,3 + +# Verification of claims about .insert mode in the climode.html +# documentation. +.testcase 700 +CREATE TABLE tbl1(one,two); +INSERT INTO tbl1 VALUES('hello!',10),('goodbye',20); +.mode insert new_table --multiinsert 0 +SELECT * FROM tbl1; +.check <<END +INSERT INTO new_table VALUES('hello!',10); +INSERT INTO new_table VALUES('goodbye',20); +END +.testcase 710 +.mode insert new_table --titles on +SELECT * FROM tbl1; +.check <<END +INSERT INTO new_table(one,two) VALUES('hello!',10); +INSERT INTO new_table(one,two) VALUES('goodbye',20); +END +.testcase 720 +.mode insert new_table --titles off +SELECT * FROM tbl1; +.check <<END +INSERT INTO new_table VALUES('hello!',10); +INSERT INTO new_table VALUES('goodbye',20); +END + +# QRF reports an error if the string is too big. +# +.testcase 800 +.mode box +.limit length 1000 +WITH c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<100) +SELECT hex(randomblob(100)) c; +.check -glob "*: string or blob too big" +.limit length 10000000 diff --git a/test/mutex1.test b/test/mutex1.test index cb189a7a8..de291f4c9 100644 --- a/test/mutex1.test +++ b/test/mutex1.test @@ -115,6 +115,10 @@ ifcapable threadsafe1&&shared_cache { } } { + ifcapable thread_misuse_warnings { + if {$mode ne "serialized"} continue + } + # For journal_mode=memory, the static_prng mutex is not required. This # is because the header of an in-memory journal does not contain # any random bytes, and so no call to sqlite3_randomness() is made. @@ -177,16 +181,18 @@ ifcapable threadsafe1&&shared_cache { # Open and use a connection in "nomutex" mode. Test that no recursive # mutexes are obtained. - do_test mutex1.3.1 { - catch {db close} - clear_mutex_counters - sqlite3 db test.db -nomutex 1 - execsql { SELECT * FROM abc } - } {1 2 3 1 2 3 1 2 3} - do_test mutex1.3.2 { - mutex_counters counters - set counters(recursive) - } {0} + ifcapable !thread_misuse_warnings { + do_test mutex1.3.1 { + catch {db close} + clear_mutex_counters + sqlite3 db test.db -nomutex 1 + execsql { SELECT * FROM abc } + } {1 2 3 1 2 3 1 2 3} + do_test mutex1.3.2 { + mutex_counters counters + set counters(recursive) + } {0} + } } # Test the sqlite3_db_mutex() function. diff --git a/test/notnull2.test b/test/notnull2.test index 67d7c26a8..f49a13b56 100644 --- a/test/notnull2.test +++ b/test/notnull2.test @@ -66,7 +66,7 @@ do_vmstep_test 1.5.2 { SELECT count(*) FROM t2 WHERE EXISTS( SELECT 1 FROM t1 WHERE t1.a=450 AND t2.c IS NULL ) -} 4000 {0} +} 5000 {0} #------------------------------------------------------------------------- reset_db diff --git a/test/offset1.test b/test/offset1.test index 5b04bd836..fb68f0d02 100644 --- a/test/offset1.test +++ b/test/offset1.test @@ -190,6 +190,20 @@ do_execsql_test offset1-2.0 { ORDER BY salary asc); } {} do_execsql_test offset1-2.1 { + SELECT * FROM v ORDER BY +id; +} { + 11 Diane London hr 70 + 12 Bob London hr 78 + 21 Emma London it 84 + 22 Grace Berlin it 90 + 23 Henry London it 104 + 24 Irene Berlin it 104 + 25 Frank Berlin it 120 + 31 Cindy Berlin sales 96 + 32 Dave London sales 96 + 33 Alice Berlin sales 100 +} +do_execsql_test offset1-2.2 { SELECT * FROM v LIMIT 5 OFFSET 2; } { 22 Grace Berlin it 90 @@ -198,5 +212,19 @@ do_execsql_test offset1-2.1 { 11 Diane London hr 70 33 Alice Berlin sales 100 } +do_execsql_test offset1-2.3 { + SELECT * FROM v LIMIT 3 OFFSET 6; +} { + 33 Alice Berlin sales 100 + 23 Henry London it 104 + 24 Irene Berlin it 104 +} +do_execsql_test offset1-2.4 { + SELECT * FROM v LIMIT 3 OFFSET 1; +} { + 32 Dave London sales 96 + 22 Grace Berlin it 90 + 21 Emma London it 84 +} finish_test diff --git a/test/printf.test b/test/printf.test index cc439e617..1f8ab25a5 100644 --- a/test/printf.test +++ b/test/printf.test @@ -3823,6 +3823,11 @@ do_execsql_test printf-17.11 { SELECT format('%.30f',1.0000000000000000076e-50); } 0.000000000000000000000000000000 +# Reported by OpenAI Codex Security on 2026-04-08 +do_execsql_test printf-17.12 { + SELECT format('%!.0e',-1e100); +} -1.0e+100 + #------------------------------------------------------------------------- # dbsqlfuzz ad651aad4bb2100f3a724129a555d8d773366d46 # diff --git a/test/qrf01.test b/test/qrf01.test new file mode 100644 index 000000000..2eb42699c --- /dev/null +++ b/test/qrf01.test @@ -0,0 +1,1186 @@ +# 2025-11-05 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the Query Result Formatter (QRF) +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix qrf01 + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1,2.5,'three'),(x'424c4f42',NULL,'Ἀμήν'); +} + +do_test 1.10 { + set result "\n[db format {SELECT * FROM t1}]" +} { +╭──────┬─────┬───────╮ +│ a │ b │ c │ +╞══════╪═════╪═══════╡ +│ 1 │ 2.5 │ three │ +│ BLOB │ │ Ἀμήν │ +╰──────┴─────┴───────╯ +} +do_test 1.11a { + set result "\n[db format -title off {SELECT * FROM t1}]" +} { +╭──────┬─────┬───────╮ +│ 1 │ 2.5 │ three │ +│ BLOB │ │ Ἀμήν │ +╰──────┴─────┴───────╯ +} +do_test 1.11b { + set result "\n[db format -text sql {SELECT * FROM t1}]" +} { +╭─────────────┬─────┬─────────╮ +│ a │ b │ c │ +╞═════════════╪═════╪═════════╡ +│ 1 │ 2.5 │ 'three' │ +│ x'424c4f42' │ │ 'Ἀμήν' │ +╰─────────────┴─────┴─────────╯ +} +do_test 1.11c { + set result "\n[db format -text sql -border off {SELECT * FROM t1}]" +} { + a │ b │ c +═════════════╪═════╪═════════ + 1 │ 2.5 │ 'three' + x'424c4f42' │ │ 'Ἀμήν' +} +do_test 1.11d { + set result "\n[db format -text relaxed -blob sql -border off \ + {SELECT * FROM t1}]" +} { + a │ b │ c +═════════════╪═════╪═══════ + 1 │ 2.5 │ three + x'424c4f42' │ │ Ἀμήν +} +do_test 1.12 { + set result "\n[db format -text csv {SELECT * FROM t1}]" +} { +╭────────────────────┬─────┬────────╮ +│ a │ b │ c │ +╞════════════════════╪═════╪════════╡ +│ 1 │ 2.5 │ three │ +│ "\102\114\117\102" │ │ "Ἀμήν" │ +╰────────────────────┴─────┴────────╯ +} +do_test 1.13 { + set result "\n[db format -text csv -blob hex {SELECT * FROM t1}]" +} { +╭──────────┬─────┬────────╮ +│ a │ b │ c │ +╞══════════╪═════╪════════╡ +│ 1 │ 2.5 │ three │ +│ 424c4f42 │ │ "Ἀμήν" │ +╰──────────┴─────┴────────╯ +} +do_test 1.14 { + catch {db format -text unk -blob hex {SELECT * FROM t1}} res + set res +} {bad -text "unk": must be auto, csv, html, json, plain, relaxed, sql, or tcl} +do_test 1.15 { + catch {db format -text sql -blob unk {SELECT * FROM t1}} res + set res +} {bad BLOB encoding (-blob) "unk": must be auto, hex, json, tcl, text, sql, or size} +do_test 1.16 { + catch {db format -text sql -style unk {SELECT * FROM t1}} res + set res +} {bad format style (-style) "unk": must be auto, box, column, count, csv, eqp, explain, html, insert, jobject, json, line, list, markdown, quote, stats, stats-est, stats-vm, or table} + + +do_test 1.20 { + set result "\n[db format -style box {SELECT * FROM t1}]" +} { +╭──────┬─────┬───────╮ +│ a │ b │ c │ +╞══════╪═════╪═══════╡ +│ 1 │ 2.5 │ three │ +│ BLOB │ │ Ἀμήν │ +╰──────┴─────┴───────╯ +} + +do_test 1.30 { + set result "\n[db format -style table {SELECT * FROM t1}]" +} { ++------+-----+-------+ +| a | b | c | ++------+-----+-------+ +| 1 | 2.5 | three | +| BLOB | | Ἀμήν | ++------+-----+-------+ +} +do_test 1.31 { + set result "\n[db format -style table -title off {SELECT * FROM t1}]" +} { ++------+-----+-------+ +| 1 | 2.5 | three | +| BLOB | | Ἀμήν | ++------+-----+-------+ +} +do_test 1.32 { + set result "\n[db format -style table -border off {SELECT * FROM t1}]" +} { + a | b | c +------+-----+------- + 1 | 2.5 | three + BLOB | | Ἀμήν +} +do_test 1.33 { + set result "\n[db format -style table -border off \ + -screenwidth 15 \ + {SELECT * FROM t1}]" +} { + a | b | c +----+---+----- + 1|2.5|three +BLOB| |Ἀμήν +} +do_test 1.34 { + set result "\n[db format -style box -border off \ + -screenwidth 30 \ + {SELECT * FROM t1}]" +} { + a │ b │ c +══════╪═════╪═══════ + 1 │ 2.5 │ three + BLOB │ │ Ἀμήν +} +do_test 1.35 { + set result "\n[db format -style box -border off \ + -screenwidth 15 \ + {SELECT * FROM t1}]" +} { + a │ b │ c +════╪═══╪═════ + 1│2.5│three +BLOB│ │Ἀμήν +} + +do_test 1.40 { + set result "\n[db format -style column {SELECT * FROM t1}]" +} { +a b c +---- --- ----- +1 2.5 three +BLOB Ἀμήν +} +do_test 1.41 { + set result "\n[db format -style column -title off {SELECT * FROM t1}]" +} { +1 2.5 three +BLOB Ἀμήν +} + +do_test 1.50 { + db format -style count {SELECT * FROM t1} +} 2 + +do_test 1.60a { + db format -style list -columnsep , -rowsep \r\n -text csv -blob tcl {SELECT * FROM t1} +} "1,2.5,three\r\n\"\\102\\114\\117\\102\",,\"Ἀμήν\"\r\n" +do_test 1.60b { + db format -style csv -columnsep xyz -rowsep pqr -text sql -blob sql {SELECT * FROM t1} +} "1,2.5,three\r\nx'424c4f42',,\"Ἀμήν\"\r\n" +do_test 1.61a { + db format -style list -columnsep , -rowsep \r\n -text csv -title auto -blob tcl {SELECT * FROM t1} +} "a,b,c\r\n1,2.5,three\r\n\"\\102\\114\\117\\102\",,\"Ἀμήν\"\r\n" +do_test 1.61b { + db format -style csv -title auto -blob tcl {SELECT * FROM t1} +} "a,b,c\r\n1,2.5,three\r\n\"\\102\\114\\117\\102\",,\"Ἀμήν\"\r\n" +do_test 1.62a { + db format -style list -columnsep , -rowsep \r\n -text csv -title csv -blob tcl {SELECT a AS 'a x y', b, c FROM t1} +} "\"a x y\",b,c\r\n1,2.5,three\r\n\"\\102\\114\\117\\102\",,\"Ἀμήν\"\r\n" +do_test 1.62b { + db format -style csv -title csv -blob tcl {SELECT a AS 'a x y', b, c FROM t1} +} "\"a x y\",b,c\r\n1,2.5,three\r\n\"\\102\\114\\117\\102\",,\"Ἀμήν\"\r\n" + +do_test 1.70 { + set result "\n[db format -style html {SELECT * FROM t1}]" +} { +<TR> +<TD>1 +<TD>2.5 +<TD>three +</TR> +<TR> +<TD>BLOB +<TD>null +<TD>Ἀμήν +</TR> +} +do_test 1.71 { + set result "\n[db format -style html -title auto {SELECT * FROM t1}]" +} { +<TR> +<TH>a +<TH>b +<TH>c +</TR> +<TR> +<TD>1 +<TD>2.5 +<TD>three +</TR> +<TR> +<TD>BLOB +<TD>null +<TD>Ἀμήν +</TR> +} + +do_test 1.80 { + set result "\n[db format -style insert {SELECT * FROM t1}]" +} { +INSERT INTO tab VALUES(1,2.5,'three'); +INSERT INTO tab VALUES(x'424c4f42',NULL,'Ἀμήν'); +} +do_test 1.81 { + set result "\n[db format -style insert -tablename t1 {SELECT * FROM t1}]" +} { +INSERT INTO t1 VALUES(1,2.5,'three'); +INSERT INTO t1 VALUES(x'424c4f42',NULL,'Ἀμήν'); +} +do_test 1.82 { + set result "\n[db format -style insert -tablename t1 -title auto \ + {SELECT * FROM t1}]" +} { +INSERT INTO t1(a,b,c) VALUES(1,2.5,'three'); +INSERT INTO t1(a,b,c) VALUES(x'424c4f42',NULL,'Ἀμήν'); +} +do_test 1.83 { + set result "\n[db format -style insert -tablename drop -title on \ + {SELECT a AS "a-b", b, c AS "123" FROM t1}]" +} { +INSERT INTO "drop"("a-b",b,"123") VALUES(1,2.5,'three'); +INSERT INTO "drop"("a-b",b,"123") VALUES(x'424c4f42',NULL,'Ἀμήν'); +} +do_test 1.84 { + set result "\n[db format -style insert {SELECT * FROM t1} -multiinsert 2000]" +} { +INSERT INTO tab VALUES(1,2.5,'three'), + (x'424c4f42',NULL,'Ἀμήν'); +} +do_test 1.85 { + set result "\n[db format -style insert { + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<7) + SELECT n, n*10 AS m FROM c;} -multiinsert 70]" +} { +INSERT INTO tab VALUES(1,10), + (2,20), + (3,30), + (4,40), + (5,50); +INSERT INTO tab VALUES(6,60), + (7,70); +} + +do_test 1.90 { + set result "\n[db format -style json {SELECT * FROM t1}]" +} { +[{"a":1,"b":2.5,"c":"three"}, +{"a":"\u0042\u004c\u004f\u0042","b":null,"c":"Ἀμήν"}] +} +do_test 1.91 { + set result "\n[db format -style jobject {SELECT * FROM t1}]" +} { +{"a":1,"b":2.5,"c":"three"} +{"a":"\u0042\u004c\u004f\u0042","b":null,"c":"Ἀμήν"} +} +do_test 1.92 { + set result "\n[db format -style jobject {SELECT *, unistr('abc\u000a123\u000d\u000axyz') AS xyz FROM t1}]" +} { +{"a":1,"b":2.5,"c":"three","xyz":"abc\n123\r\nxyz"} +{"a":"\u0042\u004c\u004f\u0042","b":null,"c":"Ἀμήν","xyz":"abc\n123\r\nxyz"} +} + +do_test 1.100 { + set result "\n[db format -style line {SELECT * FROM t1}]" +} { +a: 1 +b: 2.5 +c: three + +a: BLOB +b: +c: Ἀμήν +} +do_test 1.101 { + set result "\n[db format -style line -null (NULL) {SELECT * FROM t1}]" +} { +a: 1 +b: 2.5 +c: three + +a: BLOB +b: (NULL) +c: Ἀμήν +} +do_test 1.102 { + set result "\n[db format -style line -null (NULL) -columnsep { = } \ + -text sql {SELECT * FROM t1}]" +} { +a = 1 +b = 2.5 +c = 'three' + +a = x'424c4f42' +b = (NULL) +c = 'Ἀμήν' +} + +do_test 1.110 { + set result "\n[db format -style list {SELECT * FROM t1}]" +} { +1|2.5|three +BLOB||Ἀμήν +} +do_test 1.111 { + set result "\n[db format -style list -title on {SELECT * FROM t1}]" +} { +a|b|c +1|2.5|three +BLOB||Ἀμήν +} +do_test 1.112 { + set result "\n[db format -style list -title on -text sql -null NULL \ + -title plain {SELECT * FROM t1}]" +} { +a|b|c +1|2.5|'three' +x'424c4f42'|NULL|'Ἀμήν' +} +do_test 1.118 { + set rc [catch {db format -style list -title unk {SELECT * FROM t1}} res] + lappend rc $res +} {1 {bad -title "unk": must be off, on, auto, csv, html, json, plain, relaxed, sql, or tcl}} + + +do_test 1.120 { + set result "\n[db format -style markdown {SELECT * FROM t1}]" +} { +| a | b | c | +|------|-----|-------| +| 1 | 2.5 | three | +| BLOB | | Ἀμήν | +} +do_test 1.121 { + set result "\n[db format -style markdown -title off {SELECT * FROM t1}]" +} { +| 1 | 2.5 | three | +| BLOB | | Ἀμήν | +} + +do_test 1.130 { + set result "\n[db format -style quote {SELECT * FROM t1}]" +} { +1,2.5,'three' +x'424c4f42',NULL,'Ἀμήν' +} +do_test 1.131 { + set result "\n[db format -style quote -title on {SELECT * FROM t1}]" +} { +'a','b','c' +1,2.5,'three' +x'424c4f42',NULL,'Ἀμήν' +} + + +do_execsql_test 2.0 { + DELETE FROM t1; + INSERT INTO t1 VALUES(1,2,'The quick fox jumps over the lazy brown dog.'); +} +do_test 2.1 { + set result "\n[db format -widths {5 -5 19} -wordwrap on \ + {SELECT * FROM t1}]" +} { +╭───────┬───────┬─────────────────────╮ +│ a │ b │ c │ +╞═══════╪═══════╪═════════════════════╡ +│ 1 │ 2 │ The quick fox jumps │ +│ │ │ over the lazy brown │ +│ │ │ dog. │ +╰───────┴───────┴─────────────────────╯ +} +do_test 2.2 { + set result "\n[db format -widths {5 -5 19} -wordwrap off \ + {SELECT * FROM t1}]" +} { +╭───────┬───────┬─────────────────────╮ +│ a │ b │ c │ +╞═══════╪═══════╪═════════════════════╡ +│ 1 │ 2 │ The quick fox jumps │ +│ │ │ over the lazy brown │ +│ │ │ dog. │ +╰───────┴───────┴─────────────────────╯ +} +do_test 2.3 { + set result "\n[db format -widths {5 -5 18} -wordwrap on \ + {SELECT * FROM t1}]" +} { +╭───────┬───────┬────────────────────╮ +│ a │ b │ c │ +╞═══════╪═══════╪════════════════════╡ +│ 1 │ 2 │ The quick fox │ +│ │ │ jumps over the │ +│ │ │ lazy brown dog. │ +╰───────┴───────┴────────────────────╯ +} +do_test 2.4 { + set result "\n[db format -widths {5 -5 -18} -wordwrap on \ + {SELECT * FROM t1}]" +} { +╭───────┬───────┬────────────────────╮ +│ a │ b │ c │ +╞═══════╪═══════╪════════════════════╡ +│ 1 │ 2 │ The quick fox │ +│ │ │ jumps over the │ +│ │ │ lazy brown dog. │ +╰───────┴───────┴────────────────────╯ +} +do_test 2.5 { + set result "\n[db format -widths {5 -5 19} -wordwrap off \ + {SELECT * FROM t1}]" +} { +╭───────┬───────┬─────────────────────╮ +│ a │ b │ c │ +╞═══════╪═══════╪═════════════════════╡ +│ 1 │ 2 │ The quick fox jumps │ +│ │ │ over the lazy brown │ +│ │ │ dog. │ +╰───────┴───────┴─────────────────────╯ +} +do_test 2.6 { + set result "\n[db format -widths {5 -5 18} -wordwrap off \ + {SELECT * FROM t1}]" +} { +╭───────┬───────┬────────────────────╮ +│ a │ b │ c │ +╞═══════╪═══════╪════════════════════╡ +│ 1 │ 2 │ The quick fox jump │ +│ │ │ s over the lazy br │ +│ │ │ own dog. │ +╰───────┴───────┴────────────────────╯ +} +do_test 2.7 { + set result "\n[db format -widths {5 5 18} -wordwrap yes \ + -align {left center right} -titlealign right \ + {SELECT * FROM t1}]" +} { +╭───────┬───────┬────────────────────╮ +│ a │ b │ c │ +╞═══════╪═══════╪════════════════════╡ +│ 1 │ 2 │ The quick fox │ +│ │ │ jumps over the │ +│ │ │ lazy brown dog. │ +╰───────┴───────┴────────────────────╯ +} +do_test 2.8 { + set result "\n[db format -widths {5 8 11} -wordwrap yes \ + -align {auto auto center} -titlealign left \ + -defaultalign right \ + {SELECT * FROM t1}]" +} { +╭───────┬──────────┬─────────────╮ +│ a │ b │ c │ +╞═══════╪══════════╪═════════════╡ +│ 1 │ 2 │ The quick │ +│ │ │ fox jumps │ +│ │ │ over the │ +│ │ │ lazy brown │ +│ │ │ dog. │ +╰───────┴──────────┴─────────────╯ +} +do_test 2.9 { + catch {db format -align {auto xyz 123} {SELECT * FROM t1}} res + set res +} {bad column alignment (-align) "xyz": must be auto, bottom, c, center, e, left, middle, n, ne, nw, right, s, se, sw, top, or w} +do_test 2.10 { + catch {db format -defaultalign xyz {SELECT * FROM t1}} res + set res +} {bad default alignment (-defaultalign) "xyz": must be auto, bottom, c, center, e, left, middle, n, ne, nw, right, s, se, sw, top, or w} +do_test 2.11 { + catch {db format -titlealign xyz {SELECT * FROM t1}} res + set res +} {bad title alignment (-titlealign) "xyz": must be auto, bottom, c, center, e, left, middle, n, ne, nw, right, s, se, sw, top, or w} + + +do_execsql_test 2.30 { + UPDATE t1 SET c='Η γρήγορη αλεπού πηδάει πάνω από το τεμπέλικο καφέ σκυλί'; + SELECT hex(c) FROM t1; +} {CE9720CEB3CF81CEAECEB3CEBFCF81CEB720CEB1CEBBCEB5CF80CEBFCF8D20CF80CEB7CEB4CEACCEB5CEB920CF80CEACCEBDCF8920CEB1CF80CF8C20CF84CEBF20CF84CEB5CEBCCF80CEADCEBBCEB9CEBACEBF20CEBACEB1CF86CEAD20CF83CEBACF85CEBBCEAF} +do_test 2.31 { + set result "\n[db format -widths {5 -5 18} -wordwrap on \ + {SELECT * FROM t1}]" +} { +╭───────┬───────┬────────────────────╮ +│ a │ b │ c │ +╞═══════╪═══════╪════════════════════╡ +│ 1 │ 2 │ Η γρήγορη αλεπού │ +│ │ │ πηδάει πάνω από το │ +│ │ │ τεμπέλικο καφέ │ +│ │ │ σκυλί │ +╰───────┴───────┴────────────────────╯ +} +do_test 2.32 { + set result "\n[db format -widths {5 5 18} -align {left center center} -wordwrap on \ + {SELECT * FROM t1}]" +} { +╭───────┬───────┬────────────────────╮ +│ a │ b │ c │ +╞═══════╪═══════╪════════════════════╡ +│ 1 │ 2 │ Η γρήγορη αλεπού │ +│ │ │ πηδάει πάνω από το │ +│ │ │ τεμπέλικο καφέ │ +│ │ │ σκυλί │ +╰───────┴───────┴────────────────────╯ +} + + +do_execsql_test 3.0 { + DELETE FROM t1; + INSERT INTO t1 VALUES(1,2,unistr('abc\u001b[1;31m123\u001b[0mxyz')); +} +do_test 3.1 { + set result "\n[db format {SELECT * FROM t1}]" +} { +╭───┬───┬────────────────────────╮ +│ a │ b │ c │ +╞═══╪═══╪════════════════════════╡ +│ 1 │ 2 │ abc^[[1;31m123^[[0mxyz │ +╰───┴───┴────────────────────────╯ +} +do_test 3.2 { + set result "\n[db format -esc off {SELECT * FROM t1}]" + string map [list \033 X] $result +} { +╭───┬───┬───────────╮ +│ a │ b │ c │ +╞═══╪═══╪═══════════╡ +│ 1 │ 2 │ abcX[1;31m123X[0mxyz │ +╰───┴───┴───────────╯ +} +do_test 3.3 { + set result "\n[db format -esc symbol {SELECT * FROM t1}]" +} { +╭───┬───┬──────────────────────╮ +│ a │ b │ c │ +╞═══╪═══╪══════════════════════╡ +│ 1 │ 2 │ abc␛[1;31m123␛[0mxyz │ +╰───┴───┴──────────────────────╯ +} +do_test 3.4 { + set result "\n[db format -esc ascii {SELECT * FROM t1}]" +} { +╭───┬───┬────────────────────────╮ +│ a │ b │ c │ +╞═══╪═══╪════════════════════════╡ +│ 1 │ 2 │ abc^[[1;31m123^[[0mxyz │ +╰───┴───┴────────────────────────╯ +} +do_test 3.5 { + catch {db format -esc unk {SELECT * FROM t1}} res + set res +} {bad control character escape (-esc) "unk": must be ascii, auto, off, or symbol} + +do_execsql_test 4.0 { + DELETE FROM t1; + INSERT INTO t1 VALUES(json('{a:5,b:6}'), jsonb('{c:1,d:2}'), 99); +} +do_test 4.1 { + set result "\n[db format -text sql {SELECT * FROM t1}]" +} { +╭─────────────────┬───────────────────────┬────╮ +│ a │ b │ c │ +╞═════════════════╪═══════════════════════╪════╡ +│ '{"a":5,"b":6}' │ x'8c1763133117641332' │ 99 │ +╰─────────────────┴───────────────────────┴────╯ +} +do_test 4.2 { + set result "\n[db format -text sql -textjsonb on {SELECT * FROM t1}]" +} { +╭─────────────────┬────────────────────────┬────╮ +│ a │ b │ c │ +╞═════════════════╪════════════════════════╪════╡ +│ '{"a":5,"b":6}' │ jsonb('{"c":1,"d":2}') │ 99 │ +╰─────────────────┴────────────────────────┴────╯ +} +do_test 4.3 { + set result "\n[db format -text plain -textjsonb on -wrap 11 \ + {SELECT a AS json, b AS jsonb, c AS num FROM t1}]" +} { +╭─────────────┬─────────────┬─────╮ +│ json │ jsonb │ num │ +╞═════════════╪═════════════╪═════╡ +│ {"a":5,"b": │ {"c":1,"d": │ 99 │ +│ 6} │ 2} │ │ +╰─────────────┴─────────────┴─────╯ +} + +do_execsql_test 5.0 { + DROP TABLE t1; + CREATE TABLE t1(name, mtime, value); + INSERT INTO t1 VALUES + ('entry-one',1708791504,zeroblob(300)), + (unistr('one\u000atwo\u000athree'),1333206973,NULL), + ('sample-jsonb',1333101221,jsonb('{ + "alpha":53.11688723, + "beta":"qrfWidthPrint(p, p->pOut, -p->u.sLine.mxColWth);", + "zeta":[15,null,1333206973,"fd8ffe000104a46494600010101"]}')); +} +do_test 5.1 { + set sql {SELECT name, mtime, datetime(mtime,'unixepoch') AS time, + value FROM t1 ORDER BY mtime} + set result "\n[db format -style line -screenwidth 60 -blob sql \ + -text sql -wordwrap off -linelimit 77 \ + -columnsep { = } $sql]" +} { + name = 'sample-jsonb' +mtime = 1333101221 + time = '2012-03-30 09:53:41' +value = x'cc7c57616c706861b535332e31313638383732334762657461 + c73071726657696474685072696e7428702c20702d3e704f7574 + 2c202d702d3e752e734c696e652e6d78436f6c577468293b477a + 657461cb2c23313500a331333333323036393733c71b66643866 + 6665303030313034613436343934363030303130313031' + + name = unistr('one\u000atwo\u000athree') +mtime = 1333206973 + time = '2012-03-31 15:16:13' +value = + + name = 'entry-one' +mtime = 1708791504 + time = '2024-02-24 16:18:24' +value = x'00000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000' +} +do_test 5.2a { + set sql {SELECT name, mtime, datetime(mtime,'unixepoch') AS time, + value FROM t1 ORDER BY mtime} + set result "\n[db format -style line -screenwidth 60 -blob sql \ + -text plain -esc off -textjsonb yes -columnsep { = }\ + -wordwrap yes -linelimit 3 $sql]" +} { + name = sample-jsonb +mtime = 1333101221 + time = 2012-03-30 09:53:41 +value = {"alpha":53.11688723,"beta":"qrfWidthPrint(p, + p->pOut, -p->u.sLine.mxColWth);","zeta":[15,null, + 1333206973,"fd8ffe000104a46494600010101"]} + + name = one + two + three +mtime = 1333206973 + time = 2012-03-31 15:16:13 +value = + + name = entry-one +mtime = 1708791504 + time = 2024-02-24 16:18:24 +value = x'00000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000 + ... +} +set sqlnolabel "SELECT name, mtime, datetime(mtime,'unixepoch'),\ + value FROM t1 ORDER BY mtime" +do_test 5.2b { + set result "\n[db format -style line -screenwidth 60 -blob sql \ + -text plain -esc off -textjsonb no -titlelimit 12 \ + -wordwrap yes -linelimit 3 $sqlnolabel ]" +} { + name: sample-jsonb + mtime: 1333101221 +datetime(...: 2012-03-30 09:53:41 + value: x'cc7c57616c706861b535332e31313638383732334762 + 657461c73071726657696474685072696e7428702c2070 + 2d3e704f75742c202d702d3e752e734c696e652e6d7843 + ... + + name: one + two + three + mtime: 1333206973 +datetime(...: 2012-03-31 15:16:13 + value: + + name: entry-one + mtime: 1708791504 +datetime(...: 2024-02-24 16:18:24 + value: x'00000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000 + ... +} +set sql "SELECT name, mtime, datetime(mtime,'unixepoch') AS time,\ + value FROM t1 ORDER BY mtime" +do_test 5.3a { + set result "\n[db format -style box -widths {0 10 10 14}\ + -align {left right right center} \ + -blob sql \ + -text plain -esc off -textjsonb no \ + -wordwrap yes -linelimit 2 $sql]" +} { +╭──────────────┬────────────┬────────────┬────────────────╮ +│ name │ mtime │ time │ value │ +╞══════════════╪════════════╪════════════╪════════════════╡ +│ sample-jsonb │ 1333101221 │ 2012-03-30 │ x'cc7c57616c70 │ +│ │ │ 09:53:41 │ 6861b535332e31 │ +│ │ │ │ ... │ +├──────────────┼────────────┼────────────┼────────────────┤ +│ one │ 1333206973 │ 2012-03-31 │ │ +│ two │ │ 15:16:13 │ │ +│ ... │ │ │ │ +├──────────────┼────────────┼────────────┼────────────────┤ +│ entry-one │ 1708791504 │ 2024-02-24 │ x'000000000000 │ +│ │ │ 16:18:24 │ 00000000000000 │ +│ │ │ │ ... │ +╰──────────────┴────────────┴────────────┴────────────────╯ +} +do_test 5.3b { + set result "\n[db format -style box -widths {0 10 0 14} \ + -align {left right right center} \ + -blob sql -titlelimit 12 \ + -text plain -esc off -textjsonb no \ + -wordwrap yes -linelimit 2 $sqlnolabel ]" +} { +╭──────────────┬────────────┬─────────────────────┬────────────────╮ +│ name │ mtime │ datetime(... │ value │ +╞══════════════╪════════════╪═════════════════════╪════════════════╡ +│ sample-jsonb │ 1333101221 │ 2012-03-30 09:53:41 │ x'cc7c57616c70 │ +│ │ │ │ 6861b535332e31 │ +│ │ │ │ ... │ +├──────────────┼────────────┼─────────────────────┼────────────────┤ +│ one │ 1333206973 │ 2012-03-31 15:16:13 │ │ +│ two │ │ │ │ +│ ... │ │ │ │ +├──────────────┼────────────┼─────────────────────┼────────────────┤ +│ entry-one │ 1708791504 │ 2024-02-24 16:18:24 │ x'000000000000 │ +│ │ │ │ 00000000000000 │ +│ │ │ │ ... │ +╰──────────────┴────────────┴─────────────────────┴────────────────╯ +} +do_test 5.3c { + set result "\n[db format -style table -widths {0 10 10 14}\ + -align {center right right right} \ + -blob sql \ + -text plain -esc off -textjsonb no \ + -wordwrap yes -linelimit 2 $sql]" +} { ++--------------+------------+------------+----------------+ +| name | mtime | time | value | ++--------------+------------+------------+----------------+ +| sample-jsonb | 1333101221 | 2012-03-30 | x'cc7c57616c70 | +| | | 09:53:41 | 6861b535332e31 | +| | | | ... | ++--------------+------------+------------+----------------+ +| one | 1333206973 | 2012-03-31 | | +| two | | 15:16:13 | | +| ... | | | | ++--------------+------------+------------+----------------+ +| entry-one | 1708791504 | 2024-02-24 | x'000000000000 | +| | | 16:18:24 | 00000000000000 | +| | | | ... | ++--------------+------------+------------+----------------+ +} +do_test 5.3c { + set result "\n[db format -style column -widths {0 10 10 14}\ + -align {center right right right} \ + -blob sql \ + -text plain -esc off -textjsonb no \ + -wordwrap yes -linelimit 2 $sql]" +} { + name mtime time value +------------ ---------- ---------- -------------- +sample-jsonb 1333101221 2012-03-30 x'cc7c57616c70 + 09:53:41 6861b535332e31 + ... + + one 1333206973 2012-03-31 + two 15:16:13 + ... + + entry-one 1708791504 2024-02-24 x'000000000000 + 16:18:24 00000000000000 + ... +} +do_test 5.4 { + db eval { + CREATE TABLE t2(a,b,c,d,e); + WITH v(x) AS (SELECT 'abcdefghijklmnopqrstuvwxyz') + INSERT INTO t2 SELECT x,x,x,x,x FROM v; + } + set sql {SELECT char(0x61,0xa,0x62,0xa,0x63,0xa,0x64) a, + mtime b, mtime c, mtime d, mtime e FROM t1} + set result "\n[db format -style box -widths {1 2 3 4 5}\ + -linelimit 3 -wordwrap off {SELECT *, 'x' AS x FROM t2}]" +} { +╭────┬────┬─────┬──────┬───────┬───╮ +│ a │ b │ c │ d │ e │ x │ +╞════╪════╪═════╪══════╪═══════╪═══╡ +│ ab │ ab │ abc │ abcd │ abcde │ x │ +│ cd │ cd │ def │ efgh │ fghij │ │ +│ ef │ ef │ ghi │ ijkl │ klmno │ │ +│ .. │ .. │ ... │ ... │ ... │ │ +╰────┴────┴─────┴──────┴───────┴───╯ +} + +do_execsql_test 6.0 { + DELETE FROM t2; + INSERT INTO t2 VALUES + (1, 2.5, 'three', x'342028666f757229', null); +} +do_test 6.1a { + set result "\n[db format -style list -null NULL \ + -text tcl -columnsep , \ + {SELECT * FROM t2}]" +} { +1,2.5,"three","\064\040\050\146\157\165\162\051",NULL +} + +do_execsql_test 7.0 { + CREATE TABLE t7(a,b); + INSERT INTO t7 VALUES('abcdefghijklmnop', + 'abcぁdefかghiのjklはmnop'); +} +do_test 7.1 { + set result "\n[db format -style list -charlimit 13 \ + {SELECT * FROM t7}]" +} { +abcdefghijklm...|abcぁdefかghi... +} +do_test 7.2 { + set result "\n[db format -style list -charlimit 14 \ + {SELECT * FROM t7}]" +} { +abcdefghijklmn...|abcぁdefかghi... +} +do_test 7.3 { + set result "\n[db format -style list -charlimit 15 \ + {SELECT * FROM t7}]" +} { +abcdefghijklmno...|abcぁdefかghiの... +} +do_test 7.4 { + set result "\n[db format -style list -charlimit 16 \ + {SELECT * FROM t7}]" +} { +abcdefghijklmnop|abcぁdefかghiのj... +} + +do_test 8.0 { + set result "\n[db format -style table { + WITH c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<10) + SELECT 'aaa' AS 'a', format('%.*c',n,'b')||char(9)||'xx' AS x FROM c}]" +} { ++-----+--------------------+ +| a | x | ++-----+--------------------+ +| aaa | b xx | +| aaa | bb xx | +| aaa | bbb xx | +| aaa | bbbb xx | +| aaa | bbbbb xx | +| aaa | bbbbbb xx | +| aaa | bbbbbbb xx | +| aaa | bbbbbbbb xx | +| aaa | bbbbbbbbb xx | +| aaa | bbbbbbbbbb xx | ++-----+--------------------+ +} +do_test 8.1 { + set result "\n[db format -style table { + WITH c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<10) + SELECT 'aaaa' AS 'a', format('%.*c',n,'b')||char(9)||'xx' AS x FROM c}]" +} { ++------+--------------------+ +| a | x | ++------+--------------------+ +| aaaa | b xx | +| aaaa | bb xx | +| aaaa | bbb xx | +| aaaa | bbbb xx | +| aaaa | bbbbb xx | +| aaaa | bbbbbb xx | +| aaaa | bbbbbbb xx | +| aaaa | bbbbbbbb xx | +| aaaa | bbbbbbbbb xx | +| aaaa | bbbbbbbbbb xx | ++------+--------------------+ +} +do_test 8.3 { + set result "\n[db format -style table -esc off { + WITH c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<15) + SELECT 'aaa' AS 'a',n, 'xx'||char(n)||'yy' AS xy FROM c + WHERE n NOT IN (8,10,13,14)}]" +} { ++-----+----+------------+ +| a | n | xy | ++-----+----+------------+ +| aaa | 1 | xx␁yy | +| aaa | 2 | xx␂yy | +| aaa | 3 | xx␃yy | +| aaa | 4 | xx␄yy | +| aaa | 5 | xx␅yy | +| aaa | 6 | xx␆yy | +| aaa | 7 | xx␇yy | +| aaa | 9 | xx yy | +| aaa | 11 | xx␋yy | +| aaa | 12 | xx␌yy | +| aaa | 15 | xx␏yy | ++-----+----+------------+ +} +do_test 8.4 { + set result "\n[db format -style table -esc off { + WITH c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<15) + SELECT 'aaa' AS 'a',n, 'xx'||char(n)||'yy'||char(9)||'zz' AS xyz FROM c + WHERE n NOT IN (8,10,13,14)}]" +} { ++-----+----+--------------------+ +| a | n | xyz | ++-----+----+--------------------+ +| aaa | 1 | xx␁yy zz | +| aaa | 2 | xx␂yy zz | +| aaa | 3 | xx␃yy zz | +| aaa | 4 | xx␄yy zz | +| aaa | 5 | xx␅yy zz | +| aaa | 6 | xx␆yy zz | +| aaa | 7 | xx␇yy zz | +| aaa | 9 | xx yy zz | +| aaa | 11 | xx␋yy zz | +| aaa | 12 | xx␌yy zz | +| aaa | 15 | xx␏yy zz | ++-----+----+--------------------+ +} + +do_test 9.1 { + db eval { + CREATE TABLE t9(x); + INSERT INTO t9 VALUES + (x'4331323334'), + (x'c30431323334'), + (x'd3000431323334'), + (x'e30000000431323334'), + (x'f3000000000000000431323334'); + } + db format -style list -text plain -rowsep , -textjsonb on \ + {SELECT * FROM t9} +} {1234,1234,1234,1234,1234,} +do_test 9.2 { + db format -style list -text sql -rowsep , -textjsonb on \ + {SELECT * FROM t9} +} {jsonb('1234'),jsonb('1234'),jsonb('1234'),jsonb('1234'),jsonb('1234'),} +do_test 9.3 { + db format -style json {SELECT * FROM t9 WHERE rowid<0} +} {} +do_test 9.4 { + db format -style jobject {SELECT * FROM t9 WHERE rowid<0} +} {} + +do_test 10.1 { + db eval { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(x); + INSERT INTO t1(x) VALUES + ('alice'), + ('bob'), + ('cinderella-cinderella'), + ('daniel'), + ('emma'), + ('fred'), + ('gertrude'), + ('harold'), + ('ingrid'), + ('jake'), + ('lisa'), + ('mike'), + ('nina'), + ('octavian'), + ('paula'), + ('quintus'), + ('rita'), + ('sam'), + ('tammy'), + ('ulysses'), + ('violet'), + ('william'), + ('xanthippe'), + ('yates'), + ('zoe'); + } + set result "\n[db format -style column -title off -screenwidth 41 -splitcolumn on \ + {SELECT x FROM t1}]" +} { +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +} +do_test 10.2 { + set result "\n[db format -style column -title off -screenwidth 42 -splitcolumn on \ + {SELECT x FROM t1}]" +} { +alice jake tammy +bob lisa ulysses +cinderella-cinderella mike violet +daniel nina william +emma octavian xanthippe +fred paula yates +gertrude quintus zoe +harold rita +ingrid sam +} +do_test 10.3 { + set result "\n[db format -style column -title off -screenwidth 51 -splitcolumn on \ + {SELECT x FROM t1}]" +} { +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +} +do_test 10.4 { + set result "\n[db format -style column -title off -screenwidth 61 -splitcolumn on \ + {SELECT x FROM t1}]" +} { +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +} +do_test 10.5 { + set result "\n[db format -style column -title off -screenwidth 74 -splitcolumn on \ + {SELECT x FROM t1}]" +} { +alice emma ingrid nina rita violet zoe +bob fred jake octavian sam william +cinderella-cinderella gertrude lisa paula tammy xanthippe +daniel harold mike quintus ulysses yates +} + +do_test 11.1 { + set result "\n[db format -style table -blob size {SELECT randomblob(1234)}]" +} { ++------------------+ +| randomblob(1234) | ++------------------+ +| (1234-byte blob) | ++------------------+ +} + +do_test 12.1 { + set result "\n[db format -style box -text html \ + {SELECT 'abc','','xyz'}]" +} { +╭───────┬────┬───────╮ +│ 'abc' │ '' │ 'xyz' │ +╞═══════╪════╪═══════╡ +│ abc │ │ xyz │ +╰───────┴────┴───────╯ +} + +# Tests for "relaxed" quoting +# +do_test 13.2 { + db eval { + CREATE TABLE t13(a,b); + INSERT INTO t13(a,b) VALUES + (1,'NULL'), + (0,'-NULL-'), + (0,''), + (1,'''abcde'), + (1,'abcde'''), + (0,'abcde'), + (1,' abcde'), + (1,'abcde '), + (1,'+0'), + (1,'-0'), + (1,'012345'), + (0,'012xyz345'), + (1,'0123.45'), + (0,'12.34.56'), + (0,'12.3e'), + (1,'12.3e+123'), + (1,'12.3e-34'), + (1,'12.3E56'), + (1,'12E56'), + (0,'12.5E5.6'), + (0,'12.5e+'), + (0,'12.5e-'), + (1,'+Inf'),(1,'-Inf'),(1,'Inf'); + } + set result \n[db format -style box -text relaxed -null NULL \ + -align {center left} \ + {SELECT if(a,'yes','') AS 'quoted?', b AS string + FROM t13 ORDER BY rowid}] +} { +╭─────────┬─────────────╮ +│ quoted? │ string │ +╞═════════╪═════════════╡ +│ yes │ 'NULL' │ +│ │ -NULL- │ +│ │ │ +│ yes │ '''abcde' │ +│ yes │ 'abcde''' │ +│ │ abcde │ +│ yes │ ' abcde' │ +│ yes │ 'abcde ' │ +│ yes │ '+0' │ +│ yes │ '-0' │ +│ yes │ '012345' │ +│ │ 012xyz345 │ +│ yes │ '0123.45' │ +│ │ 12.34.56 │ +│ │ 12.3e │ +│ yes │ '12.3e+123' │ +│ yes │ '12.3e-34' │ +│ yes │ '12.3E56' │ +│ yes │ '12E56' │ +│ │ 12.5E5.6 │ +│ │ 12.5e+ │ +│ │ 12.5e- │ +│ yes │ '+Inf' │ +│ yes │ '-Inf' │ +│ yes │ 'Inf' │ +╰─────────┴─────────────╯ +} + +db close + +finish_test diff --git a/test/qrf02.test b/test/qrf02.test new file mode 100644 index 000000000..07e1568f7 --- /dev/null +++ b/test/qrf02.test @@ -0,0 +1,47 @@ +# 2025-11-05 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the Query Result Formatter (QRF) +# +# These tests are for EXPLAIN and EXPLAIN QUERY PLAN formatting, the +# output of which can change when enhancments are made to the query +# planner. So expect to have to modify the expected results of these +# test cases in the future. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix qrf02 + +do_execsql_test 1.0 { + CREATE TABLE t1(a); + INSERT INTO t1 VALUES(1); +} + +set result [db format {EXPLAIN SELECT * FROM t1}] +do_test 1.10 { + set result +} {/*addr opcode p1 p2 p3 p4 p5 comment +---- ------------- ---- ---- ---- ------------- -- ------------- +0 Init */} +regsub -all {\d+} $result {N} result2 +do_test 1.11 { + set result2 +} "/.*\nN Rewind .*\nN Column .*/" + +do_test 1.20 { + set result "\n[db format {EXPLAIN QUERY PLAN SELECT * FROM t1}]" +} { +QUERY PLAN +`--SCAN t1 +} + +finish_test diff --git a/test/qrf03.test b/test/qrf03.test new file mode 100644 index 000000000..c0457df7f --- /dev/null +++ b/test/qrf03.test @@ -0,0 +1,176 @@ +# 2025-11-15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the Query Result Formatter (QRF) +# +# Format narrowing due to nScreenWidth +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix qrf03 + +do_execsql_test 1.0 { + CREATE TABLE mlink( + mid INTEGER, + fid INTEGER, + pmid INTEGER, + pid INTEGER, + fnid INTEGER REFERENCES filename, + pfnid INTEGER, + mperm INTEGER, + isaux BOOLEAN DEFAULT 0 + ); + INSERT INTO mlink VALUES(28775,28774,28773,28706,1,0,0,0); + INSERT INTO mlink VALUES(28773,28706,28770,28685,1,0,0,0); + INSERT INTO mlink VALUES(28770,28736,28769,28695,2,0,0,0); + INSERT INTO mlink VALUES(28770,28697,28769,28698,3,0,0,0); + INSERT INTO mlink VALUES(28767,28768,28759,28746,4,0,0,0); + CREATE TABLE event( + type TEXT, + mtime DATETIME, + objid INTEGER PRIMARY KEY, + tagid INTEGER, + uid INTEGER REFERENCES user, + bgcolor TEXT, + euser TEXT, + user TEXT, + ecomment TEXT, + comment TEXT, + brief TEXT, + omtime DATETIME + ); + INSERT INTO event VALUES('ci',2460994.978048461023,126223,NULL,NULL,NULL,NULL,'drh',NULL,unistr('Data structure improvements on columnar layout. Prep work for getting\u000acolumnar layouts to respond to nScreenWidth.'),NULL,2460994.978048461023); + INSERT INTO event VALUES('ci',2460994.836955601816,126218,NULL,NULL,NULL,NULL,'stephan',NULL,'API doc typo fix.',NULL,2460994.836955601816); + INSERT INTO event VALUES('ci',2460994.88823369192,126212,NULL,NULL,NULL,NULL,'stephan',NULL,'Move sqlite3-api-cleanup.js into post-js-footer.js to remove the final direct Emscripten dependency from the intermediary build product sqlite3-api.js (the whole library, waiting to be bootstrapped). This is partly in response to [forum:4b7d45433731d2e0|forum post 4b7d45433731d2e0], which demonstrates a potential use case for a standalone sqlite3-api.js. This is a build/doc change, not a functional one.',NULL,2460994.88823369192); + INSERT INTO event VALUES('ci',2460994.516081551089,126211,NULL,NULL,NULL,NULL,'drh',NULL,unistr('Improve columnar layout in QRF so that it correctly deals with control\u000acharacters, and especially tabs.'),NULL,2460994.516081551089); + INSERT INTO event VALUES('ci',2460994.409343171865,126208,NULL,NULL,NULL,NULL,'drh',NULL,'Make use of the new sqlite3_str_free() interface in the CLI.',NULL,2460994.409343171865); +} + +do_test 1.10 { + set x "\n[db format -style box -screenwidth 68 \ + {SELECT * FROM mlink ORDER BY rowid}]" + +} { +╭───────┬───────┬───────┬───────┬──────┬───────┬───────┬───────╮ +│ mid │ fid │ pmid │ pid │ fnid │ pfnid │ mperm │ isaux │ +╞═══════╪═══════╪═══════╪═══════╪══════╪═══════╪═══════╪═══════╡ +│ 28775 │ 28774 │ 28773 │ 28706 │ 1 │ 0 │ 0 │ 0 │ +│ 28773 │ 28706 │ 28770 │ 28685 │ 1 │ 0 │ 0 │ 0 │ +│ 28770 │ 28736 │ 28769 │ 28695 │ 2 │ 0 │ 0 │ 0 │ +│ 28770 │ 28697 │ 28769 │ 28698 │ 3 │ 0 │ 0 │ 0 │ +│ 28767 │ 28768 │ 28759 │ 28746 │ 4 │ 0 │ 0 │ 0 │ +╰───────┴───────┴───────┴───────┴──────┴───────┴───────┴───────╯ +} +do_test 1.11 { + set x "\n[db format -style box -screenwidth 52 \ + {SELECT * FROM mlink ORDER BY rowid}]" + +} { +╭─────┬─────┬─────┬─────┬────┬─────┬─────┬─────╮ +│ mid │ fid │pmid │ pid │fnid│pfnid│mperm│isaux│ +╞═════╪═════╪═════╪═════╪════╪═════╪═════╪═════╡ +│28775│28774│28773│28706│ 1│ 0│ 0│ 0│ +│28773│28706│28770│28685│ 1│ 0│ 0│ 0│ +│28770│28736│28769│28695│ 2│ 0│ 0│ 0│ +│28770│28697│28769│28698│ 3│ 0│ 0│ 0│ +│28767│28768│28759│28746│ 4│ 0│ 0│ 0│ +╰─────┴─────┴─────┴─────┴────┴─────┴─────┴─────╯ +} + +do_test 1.20 { + set x "\n[db format -style table -screenwidth 68 \ + {SELECT * FROM mlink ORDER BY rowid}]" + +} { ++-------+-------+-------+-------+------+-------+-------+-------+ +| mid | fid | pmid | pid | fnid | pfnid | mperm | isaux | ++-------+-------+-------+-------+------+-------+-------+-------+ +| 28775 | 28774 | 28773 | 28706 | 1 | 0 | 0 | 0 | +| 28773 | 28706 | 28770 | 28685 | 1 | 0 | 0 | 0 | +| 28770 | 28736 | 28769 | 28695 | 2 | 0 | 0 | 0 | +| 28770 | 28697 | 28769 | 28698 | 3 | 0 | 0 | 0 | +| 28767 | 28768 | 28759 | 28746 | 4 | 0 | 0 | 0 | ++-------+-------+-------+-------+------+-------+-------+-------+ +} +do_test 1.21 { + set x "\n[db format -style table -screenwidth 52 \ + {SELECT * FROM mlink ORDER BY rowid}]" + +} { ++-----+-----+-----+-----+----+-----+-----+-----+ +| mid | fid |pmid | pid |fnid|pfnid|mperm|isaux| ++-----+-----+-----+-----+----+-----+-----+-----+ +|28775|28774|28773|28706| 1| 0| 0| 0| +|28773|28706|28770|28685| 1| 0| 0| 0| +|28770|28736|28769|28695| 2| 0| 0| 0| +|28770|28697|28769|28698| 3| 0| 0| 0| +|28767|28768|28759|28746| 4| 0| 0| 0| ++-----+-----+-----+-----+----+-----+-----+-----+ +} + +do_test 1.30 { + set x "\n[db format -style markdown -screenwidth 68 \ + {SELECT * FROM mlink ORDER BY rowid}]" + +} { +| mid | fid | pmid | pid | fnid | pfnid | mperm | isaux | +|-------|-------|-------|-------|------|-------|-------|-------| +| 28775 | 28774 | 28773 | 28706 | 1 | 0 | 0 | 0 | +| 28773 | 28706 | 28770 | 28685 | 1 | 0 | 0 | 0 | +| 28770 | 28736 | 28769 | 28695 | 2 | 0 | 0 | 0 | +| 28770 | 28697 | 28769 | 28698 | 3 | 0 | 0 | 0 | +| 28767 | 28768 | 28759 | 28746 | 4 | 0 | 0 | 0 | +} +do_test 1.31 { + set x "\n[db format -style markdown -screenwidth 52 \ + {SELECT * FROM mlink ORDER BY rowid}]" + +} { +| mid | fid |pmid | pid |fnid|pfnid|mperm|isaux| +|-----|-----|-----|-----|----|-----|-----|-----| +|28775|28774|28773|28706| 1| 0| 0| 0| +|28773|28706|28770|28685| 1| 0| 0| 0| +|28770|28736|28769|28695| 2| 0| 0| 0| +|28770|28697|28769|28698| 3| 0| 0| 0| +|28767|28768|28759|28746| 4| 0| 0| 0| +} + +do_test 1.40 { + set x "\n[db format -style column -screenwidth 68 \ + {SELECT * FROM mlink ORDER BY rowid}]" + +} { + mid fid pmid pid fnid pfnid mperm isaux +----- ----- ----- ----- ---- ----- ----- ----- +28775 28774 28773 28706 1 0 0 0 +28773 28706 28770 28685 1 0 0 0 +28770 28736 28769 28695 2 0 0 0 +28770 28697 28769 28698 3 0 0 0 +28767 28768 28759 28746 4 0 0 0 +} +do_test 1.41 { + set x "\n[db format -style column -screenwidth 52 \ + {SELECT * FROM mlink ORDER BY rowid}]" + +} { + mid fid pmid pid fnid pfnid mperm isaux +----- ----- ----- ----- ---- ----- ----- ----- +28775 28774 28773 28706 1 0 0 0 +28773 28706 28770 28685 1 0 0 0 +28770 28736 28769 28695 2 0 0 0 +28770 28697 28769 28698 3 0 0 0 +28767 28768 28759 28746 4 0 0 0 +} + + + +finish_test diff --git a/test/qrf04.test b/test/qrf04.test new file mode 100644 index 000000000..0b231d921 --- /dev/null +++ b/test/qrf04.test @@ -0,0 +1,750 @@ +# 2025-11-23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the Query Result Formatter (QRF), and especially +# the bSplitColumn feature. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix qrf01 + +# The expected output from test 1.1. The "do_test" procedure normally +# ignores differences in whitespace, but whitespace is important for +# this test, so we have to do the comparison ourselves. +# +set expected { +<---- 22 ----> +alice +bob +cinderella-cinderella +daniel +emma +fred +gertrude +harold +ingrid +jake +lisa +mike +nina +octavian +paula +quintus +rita +sam +tammy +ulysses +violet +william +xanthippe +yates +zoe +<---- 23 ----> +alice +bob +cinderella-cinderella +daniel +emma +fred +gertrude +harold +ingrid +jake +lisa +mike +nina +octavian +paula +quintus +rita +sam +tammy +ulysses +violet +william +xanthippe +yates +zoe +<---- 24 ----> +alice +bob +cinderella-cinderella +daniel +emma +fred +gertrude +harold +ingrid +jake +lisa +mike +nina +octavian +paula +quintus +rita +sam +tammy +ulysses +violet +william +xanthippe +yates +zoe +<---- 25 ----> +alice +bob +cinderella-cinderella +daniel +emma +fred +gertrude +harold +ingrid +jake +lisa +mike +nina +octavian +paula +quintus +rita +sam +tammy +ulysses +violet +william +xanthippe +yates +zoe +<---- 26 ----> +alice +bob +cinderella-cinderella +daniel +emma +fred +gertrude +harold +ingrid +jake +lisa +mike +nina +octavian +paula +quintus +rita +sam +tammy +ulysses +violet +william +xanthippe +yates +zoe +<---- 27 ----> +alice +bob +cinderella-cinderella +daniel +emma +fred +gertrude +harold +ingrid +jake +lisa +mike +nina +octavian +paula +quintus +rita +sam +tammy +ulysses +violet +william +xanthippe +yates +zoe +<---- 28 ----> +alice +bob +cinderella-cinderella +daniel +emma +fred +gertrude +harold +ingrid +jake +lisa +mike +nina +octavian +paula +quintus +rita +sam +tammy +ulysses +violet +william +xanthippe +yates +zoe +<---- 29 ----> +alice +bob +cinderella-cinderella +daniel +emma +fred +gertrude +harold +ingrid +jake +lisa +mike +nina +octavian +paula +quintus +rita +sam +tammy +ulysses +violet +william +xanthippe +yates +zoe +<---- 30 ----> +alice +bob +cinderella-cinderella +daniel +emma +fred +gertrude +harold +ingrid +jake +lisa +mike +nina +octavian +paula +quintus +rita +sam +tammy +ulysses +violet +william +xanthippe +yates +zoe +<---- 31 ----> +alice +bob +cinderella-cinderella +daniel +emma +fred +gertrude +harold +ingrid +jake +lisa +mike +nina +octavian +paula +quintus +rita +sam +tammy +ulysses +violet +william +xanthippe +yates +zoe +<---- 32 ----> +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +<---- 33 ----> +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +<---- 34 ----> +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +<---- 35 ----> +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +<---- 36 ----> +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +<---- 37 ----> +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +<---- 38 ----> +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +<---- 39 ----> +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +<---- 40 ----> +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +<---- 41 ----> +alice octavian +bob paula +cinderella-cinderella quintus +daniel rita +emma sam +fred tammy +gertrude ulysses +harold violet +ingrid william +jake xanthippe +lisa yates +mike zoe +nina +<---- 42 ----> +alice jake tammy +bob lisa ulysses +cinderella-cinderella mike violet +daniel nina william +emma octavian xanthippe +fred paula yates +gertrude quintus zoe +harold rita +ingrid sam +<---- 43 ----> +alice jake tammy +bob lisa ulysses +cinderella-cinderella mike violet +daniel nina william +emma octavian xanthippe +fred paula yates +gertrude quintus zoe +harold rita +ingrid sam +<---- 44 ----> +alice jake tammy +bob lisa ulysses +cinderella-cinderella mike violet +daniel nina william +emma octavian xanthippe +fred paula yates +gertrude quintus zoe +harold rita +ingrid sam +<---- 45 ----> +alice jake tammy +bob lisa ulysses +cinderella-cinderella mike violet +daniel nina william +emma octavian xanthippe +fred paula yates +gertrude quintus zoe +harold rita +ingrid sam +<---- 46 ----> +alice jake tammy +bob lisa ulysses +cinderella-cinderella mike violet +daniel nina william +emma octavian xanthippe +fred paula yates +gertrude quintus zoe +harold rita +ingrid sam +<---- 47 ----> +alice jake tammy +bob lisa ulysses +cinderella-cinderella mike violet +daniel nina william +emma octavian xanthippe +fred paula yates +gertrude quintus zoe +harold rita +ingrid sam +<---- 48 ----> +alice jake tammy +bob lisa ulysses +cinderella-cinderella mike violet +daniel nina william +emma octavian xanthippe +fred paula yates +gertrude quintus zoe +harold rita +ingrid sam +<---- 49 ----> +alice jake tammy +bob lisa ulysses +cinderella-cinderella mike violet +daniel nina william +emma octavian xanthippe +fred paula yates +gertrude quintus zoe +harold rita +ingrid sam +<---- 50 ----> +alice jake tammy +bob lisa ulysses +cinderella-cinderella mike violet +daniel nina william +emma octavian xanthippe +fred paula yates +gertrude quintus zoe +harold rita +ingrid sam +<---- 51 ----> +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +<---- 52 ----> +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +<---- 53 ----> +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +<---- 54 ----> +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +<---- 55 ----> +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +<---- 56 ----> +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +<---- 57 ----> +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +<---- 58 ----> +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +<---- 59 ----> +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +<---- 60 ----> +alice harold paula william +bob ingrid quintus xanthippe +cinderella-cinderella jake rita yates +daniel lisa sam zoe +emma mike tammy +fred nina ulysses +gertrude octavian violet +<---- 61 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 62 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 63 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 64 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 65 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 66 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 67 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 68 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 69 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 70 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 71 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 72 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 73 ----> +alice fred lisa quintus violet +bob gertrude mike rita william +cinderella-cinderella harold nina sam xanthippe +daniel ingrid octavian tammy yates +emma jake paula ulysses zoe +<---- 74 ----> +alice emma ingrid nina rita violet zoe +bob fred jake octavian sam william +cinderella-cinderella gertrude lisa paula tammy xanthippe +daniel harold mike quintus ulysses yates +<---- 75 ----> +alice emma ingrid nina rita violet zoe +bob fred jake octavian sam william +cinderella-cinderella gertrude lisa paula tammy xanthippe +daniel harold mike quintus ulysses yates +<---- 76 ----> +alice emma ingrid nina rita violet zoe +bob fred jake octavian sam william +cinderella-cinderella gertrude lisa paula tammy xanthippe +daniel harold mike quintus ulysses yates +<---- 77 ----> +alice emma ingrid nina rita violet zoe +bob fred jake octavian sam william +cinderella-cinderella gertrude lisa paula tammy xanthippe +daniel harold mike quintus ulysses yates +<---- 78 ----> +alice emma ingrid nina rita violet zoe +bob fred jake octavian sam william +cinderella-cinderella gertrude lisa paula tammy xanthippe +daniel harold mike quintus ulysses yates +<---- 79 ----> +alice emma ingrid nina rita violet zoe +bob fred jake octavian sam william +cinderella-cinderella gertrude lisa paula tammy xanthippe +daniel harold mike quintus ulysses yates +<---- 80 ----> +alice emma ingrid nina rita violet zoe +bob fred jake octavian sam william +cinderella-cinderella gertrude lisa paula tammy xanthippe +daniel harold mike quintus ulysses yates +} + +do_test 1.0 { + db eval { + CREATE TABLE t1(x); + INSERT INTO t1(x) VALUES + ('alice'), + ('bob'), + ('cinderella-cinderella'), + ('daniel'), + ('emma'), + ('fred'), + ('gertrude'), + ('harold'), + ('ingrid'), + ('jake'), + ('lisa'), + ('mike'), + ('nina'), + ('octavian'), + ('paula'), + ('quintus'), + ('rita'), + ('sam'), + ('tammy'), + ('ulysses'), + ('violet'), + ('william'), + ('xanthippe'), + ('yates'), + ('zoe'); + } + set res \n + for {set i 22} {$i<=80} {incr i} { + set sp [expr {$i-13}] + append res [format "<----%*s%3d%*s---->\n" \ + [expr {$sp/2}] {} $i [expr {$sp-$sp/2}] {}] + append res [db format -style column -title off \ + -screenwidth $i -splitcolumn on \ + {SELECT x FROM t1 ORDER BY x ASC}] + } + expr {$res eq $::expected} +} {1} diff --git a/test/qrf05.test b/test/qrf05.test new file mode 100644 index 000000000..0d5a4d7f9 --- /dev/null +++ b/test/qrf05.test @@ -0,0 +1,37 @@ +# 2025-12-02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the Query Result Formatter (QRF) +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix qrf05 + +do_execsql_test 1.0 { + CREATE TABLE t1(a INT NOT NULL); +} +do_test 1.1 { + set rc [catch {db format -style list \ + {INSERT INTO t1 VALUES(123) RETURNING *}} msg] + list $rc [string trim $msg] +} {0 123} +do_test 1.2 { + set rc [catch {db format -style list \ + {INSERT INTO t1 VALUES(NULL) RETURNING *}} msg] + list $rc [string trim $msg] +} {1 {NOT NULL constraint failed: t1.a}} +do_test 1.3 { + set rc [catch {db format -version 99 {SELECT * FROM t1}} msg] + list $rc [string trim $msg] +} {1 {unusable sqlite3_qrf_spec.iVersion (99)}} + +finish_test diff --git a/test/qrf06.test b/test/qrf06.test new file mode 100644 index 000000000..5fa62c26f --- /dev/null +++ b/test/qrf06.test @@ -0,0 +1,576 @@ +# 2025-12-02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the Query Result Formatter (QRF), and especially +# the sqlite3_qrf_wcwidth() function and its utilization. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix qrf06 + +# Data +db eval { + BEGIN TRANSACTION; + CREATE TABLE language(name TEXT); + INSERT INTO language(name) VALUES + ('العربية'), + ('Deutsch'), + ('English'), + ('Español'), + ('فارسی'), + ('Français'), + ('Italiano'), + ('مصرى'), + ('Nederlands'), + ('日本語'), + ('Polski'), + ('Português'), + ('Sinugboanong Binisaya'), + ('Svenska'), + ('Українська'), + ('Tiếng Việt'), + ('Winaray'), + ('中文'), + ('Русский'), + ('Afrikaans'), + ('Shqip'), + ('Asturianu'), + ('Azərbaycanca'), + ('Български'), + ('閩南語 / Bân-lâm-gú'), + ('বাংলা'), + ('Беларуская'), + ('Català'), + ('Čeština'), + ('Cymraeg'), + ('Dansk'), + ('Eesti'), + ('Ελληνικά'), + ('Esperanto'), + ('Euskara'), + ('Galego'), + ('한국어'), + ('Հայերեն'), + ('हिन्दी'), + ('Hrvatski'), + ('Bahasa Indonesia'), + ('עברית'), + ('ქართული'), + ('Ladin'), + ('Latina'), + ('Latviešu'), + ('Lietuvių'), + ('Magyar'), + ('Македонски'), + ('Malagasy'), + ('मराठी'), + ('Bahasa Melayu'), + ('Bahaso Minangkabau'), + ('မြန်မာဘာသာ'), + ('Norskbokmålnynorsk'), + ('Нохчийн'), + ('Oʻzbekcha / Ўзбекча'), + ('Қазақша / Qazaqşa / قازاقشا'), + ('Română'), + ('Simple English'), + ('Slovenčina'), + ('Slovenščina'), + ('Српски / Srpski'), + ('Srpskohrvatski / Српскохрватски'), + ('Suomi'), + ('Kiswahili'), + ('தமிழ்'), + ('Татарча / Tatarça'), + ('తెలుగు'), + ('ภาษาไทย'), + ('Тоҷикӣ'), + ('تۆرکجه'), + ('Türkçe'), + ('اردو'), + ('粵語'), + ('Bahsa Acèh'), + ('Alemannisch'), + ('አማርኛ'), + ('Aragonés'), + ('Արեւմտահայերէն'), + ('Bahasa Hulontalo'), + ('Basa Bali'), + ('Bahasa Banjar'), + ('Basa Banyumasan'), + ('Башҡортса'), + ('Беларуская (тарашкевіца)'), + ('Bikol Central'), + ('বিষ্ণুপ্রিয়া মণিপুরী'), + ('Boarisch'), + ('Bosanski'), + ('Brezhoneg'), + ('Чӑвашла'), + ('Dagbanli'), + ('الدارجة'), + ('Diné Bizaad'), + ('Emigliàn–Rumagnòl'), + ('Fiji Hindi'), + ('Føroyskt'), + ('Frysk'), + ('Fulfulde'), + ('Gaeilge'), + ('Gàidhlig'), + ('گیلکی'), + ('ગુજરાતી'), + ('Hak-kâ-ngî / 客家語'), + ('Hausa'), + ('Hornjoserbsce'), + ('Ido'), + ('Igbo'), + ('Ilokano'), + ('Interlingua'), + ('Interlingue'), + ('Ирон'), + ('Íslenska'), + ('Jawa'), + ('ಕನ್ನಡ'), + ('Kapampangan'), + ('ភាសាខ្មែរ'), + ('Kotava'), + ('Kreyòl Ayisyen'), + ('Kurdî / كوردی'), + ('کوردیی ناوەندی'), + ('Кыргызча'), + ('Кырык мары'), + ('Lëtzebuergesch'), + ('Lìgure'), + ('Limburgs'), + ('Lombard'), + ('मैथिली'), + ('മലയാളം'), + ('მარგალური'), + ('مازِرونی'), + ('Mìng-dĕ̤ng-ngṳ̄ / 閩東語'), + ('Монгол'), + ('Napulitano'), + ('नेपाल भाषा'), + ('Nordfriisk'), + ('Occitan'), + ('Олык марий'), + ('ଓଡି଼ଆ'), + ('অসমীযা়'), + ('ਪੰਜਾਬੀ'), + ('پنجابی (شاہ مکھی)'), + ('پښتو'), + ('Piemontèis'), + ('Plattdüütsch'), + ('Qaraqalpaqsha'), + ('Qırımtatarca'), + ('Runa Simi'), + ('Русиньскый'), + ('संस्कृतम्'), + ('ᱥᱟᱱᱛᱟᱲᱤ'), + ('سرائیکی'), + ('Саха Тыла'), + ('Scots'), + ('ChiShona'), + ('Sicilianu'), + ('සිංහල'), + ('سنڌي'), + ('Ślůnski'), + ('Basa Sunda'), + ('Taclḥit'), + ('Tagalog'), + ('ၽႃႇသႃႇတႆး'), + ('ⵜⴰⵎⴰⵣⵉⵖⵜ ⵜⴰⵏⴰⵡⴰⵢⵜ'), + ('tolışi'), + ('chiTumbuka'), + ('Basa Ugi'), + ('Vèneto'), + ('Volapük'), + ('Walon'), + ('文言'), + ('吴语'), + ('ייִדיש'), + ('Yorùbá'), + ('Zazaki'), + ('žemaitėška'), + ('isiZulu'), + ('नेपाली'), + ('ꯃꯤꯇꯩ ꯂꯣꯟ'), + ('Dzhudezmo / לאדינו'), + ('Адыгэбзэ'), + ('Ænglisc'), + ('Anarâškielâ'), + ('अंगिका'), + ('Аԥсшәа'), + ('armãneashti'), + ('Arpitan'), + ('atikamekw'), + ('ܐܬܘܪܝܐ'), + ('Avañe’ẽ'), + ('Авар'), + ('Aymar'), + ('Batak Toba'), + ('Betawi'), + ('भोजपुरी'), + ('Bislama'), + ('བོད་ཡིག'), + ('Буряад'), + ('Chavacano de Zamboanga'), + ('Chichewa'), + ('Corsu'), + ('Vahcuengh / 話僮'), + ('Dagaare'), + ('Davvisámegiella'), + ('Deitsch'), + ('ދިވެހިބަސް'), + ('Dolnoserbski'), + ('Dusun Bundu-liwan'), + ('Эрзянь'), + ('Estremeñu'), + ('Eʋegbe'), + ('Farefare'), + ('Fɔ̀ngbè'), + ('Furlan'), + ('Gaelg'), + ('Gagauz'), + ('ГӀалгӀай'), + ('Ghanaian Pidgin'), + ('Gĩkũyũ'), + ('赣语 / 贛語'), + ('Gungbe'), + ('Хальмг'), + ('ʻŌlelo Hawaiʻi'), + ('Ikinyarwanda'), + ('Jaku Iban'), + ('Kabɩyɛ'), + ('Yerwa Kanuri'), + ('Kaszëbsczi'), + ('Kernewek'), + ('Коми'), + ('Перем коми'), + ('Kongo'), + ('कोंकणी / Konknni'), + ('كٲشُر'), + ('Kriyòl Gwiyannen'), + ('Kumoring'), + ('Kʋsaal'), + ('ພາສາລາວ'), + ('Лакку'), + ('Latgaļu'), + ('Лезги'), + ('Li Niha'), + ('Lingála'), + ('Lingua Franca Nova'), + ('livvinkarjala'), + ('lojban'), + ('Luganda'), + ('Madhurâ'), + ('Malti'), + ('Mandailing'), + ('Māori'), + ('Mfantse'), + ('Mirandés'), + ('Мокшень'), + ('ဘာသာ မန်'), + ('Moore'), + ('ߒߞߏ'), + ('Na Vosa Vaka-Viti'), + ('Nāhuatlahtōlli'), + ('Naijá'), + ('Nedersaksisch'), + ('Nouormand / Normaund'), + ('Novial'), + ('Afaan Oromoo'), + ('ပအိုဝ်ႏဘာႏသာႏ'), + ('Pangasinán'), + ('Pangcah'), + ('Papiamentu'), + ('Patois'), + ('Pfälzisch'), + ('Picard'), + ('Къарачай–малкъар'), + ('Ripoarisch'), + ('Rumantsch'), + ('Sakizaya'), + ('Gagana Sāmoa'), + ('Sardu'), + ('Seediq'), + ('Seeltersk'), + ('Sesotho'), + ('Sesotho sa Leboa'), + ('Setswana'), + ('ꠍꠤꠟꠐꠤ'), + ('Словѣ́ньскъ / ⰔⰎⰑⰂⰡⰐⰠⰔⰍⰟ'), + ('Soomaaliga'), + ('Sranantongo'), + ('SiSwati'), + ('Reo tahiti'), + ('Taqbaylit'), + ('Tarandíne'), + ('Tayal'), + ('Tetun'), + ('Tok Pisin'), + ('faka Tonga'), + ('Türkmençe'), + ('Twi'), + ('Tyap'), + ('Тыва дыл'), + ('Удмурт'), + ('ئۇيغۇرچه'), + ('Vepsän'), + ('võro'), + ('West-Vlams'), + ('Wolof'), + ('isiXhosa'), + ('Zeêuws'), + ('алтай тил'), + ('अवधी'), + ('डोटेली'), + ('ತುಳು'), + ('ရခိုင်'), + ('Bajau Sama'), + ('Bamanankan'), + ('Chamoru'), + ('རྫོང་ཁ'), + ('𐌲𐌿𐍄𐌹𐍃𐌺x'), + ('Igala'), + ('ᐃᓄᒃᑎᑐᑦ / Inuktitut'), + ('Iñupiak'), + ('isiNdebele seSewula'), + ('Kalaallisut'), + ('Nupe'), + ('Obolo'), + ('पालि'), + ('pinayuanan'), + ('Ποντιακά'), + ('romani čhib'), + ('Ikirundi'), + ('руски'), + ('Sängö'), + ('ᥖᥭᥰᥖᥬᥳᥑᥨᥒᥰ'), + ('ትግርኛ'), + ('Thuɔŋjäŋ'), + ('ᏣᎳᎩ'), + ('Tsėhesenėstsestotse'), + ('Xitsonga'), + ('Tshivenḓa'), + ('Wayuunaiki'), + ('адыгабзэ'); + COMMIT; +} + +do_test 1.2 { + set res \n[db format -style box { + SELECT name, rowid AS id FROM language + WHERE length(name)=2 + ORDER BY name + }] + set exp { +╭──────┬─────╮ +│ name │ id │ +╞══════╪═════╡ +│ 中文 │ 18 │ +│ 吴语 │ 173 │ +│ 文言 │ 172 │ +│ 粵語 │ 75 │ +╰──────┴─────╯ +} + if {$res ne $exp} { + puts [list $res] + puts [list $exp] + } + string compare $res $exp +} {0} + +do_test 1.3 { + set res \n[db format -style box { + SELECT name, rowid AS id FROM language + WHERE length(name)=3 + ORDER BY name + }] + set exp { +╭────────┬─────╮ +│ name │ id │ +╞════════╪═════╡ +│ Ido │ 108 │ +│ Twi │ 297 │ +│ ߒߞߏ │ 258 │ +│ ᏣᎳᎩ │ 335 │ +│ 日本語 │ 10 │ +│ 한국어 │ 37 │ +╰────────┴─────╯ +} + if {$res ne $exp} { + puts [list $res] + puts [list $exp] + } + string compare $res $exp +} {0} + +do_test 1.4 { + set res \n[db format -style box { + SELECT name, rowid AS id FROM language + WHERE length(name)=4 + ORDER BY name + }] + set exp { +╭──────┬─────╮ +│ name │ id │ +╞══════╪═════╡ +│ Igbo │ 109 │ +│ Jawa │ 115 │ +│ Nupe │ 323 │ +│ Tyap │ 298 │ +│ võro │ 303 │ +│ Авар │ 192 │ +│ Ирон │ 113 │ +│ Коми │ 231 │ +│ اردو │ 74 │ +│ سنڌي │ 159 │ +│ مصرى │ 8 │ +│ پښتو │ 144 │ +│ अवधी │ 309 │ +│ पालि │ 325 │ +│ ತುಳು │ 311 │ +│ ትግርኛ │ 333 │ +│ አማርኛ │ 78 │ +╰──────┴─────╯ +} + if {$res ne $exp} { + puts [list $res] + puts [list $exp] + } + string compare $res $exp +} {0} + +do_test 1.5 { + set res \n[db format -style box { + SELECT name, rowid AS id FROM language + WHERE length(name)=5 + ORDER BY name + }] + set exp { +╭───────┬─────╮ +│ name │ id │ +╞═══════╪═════╡ +│ Aymar │ 193 │ +│ Corsu │ 202 │ +│ Dansk │ 31 │ +│ Eesti │ 32 │ +│ Frysk │ 99 │ +│ Gaelg │ 216 │ +│ Hausa │ 106 │ +│ Igala │ 318 │ +│ Kongo │ 233 │ +│ Ladin │ 44 │ +│ Malti │ 250 │ +│ Moore │ 257 │ +│ Māori │ 252 │ +│ Naijá │ 261 │ +│ Obolo │ 324 │ +│ Sardu │ 278 │ +│ Scots │ 155 │ +│ Shqip │ 21 │ +│ Suomi │ 65 │ +│ Sängö │ 331 │ +│ Tayal │ 292 │ +│ Tetun │ 293 │ +│ Walon │ 171 │ +│ Wolof │ 305 │ +│ Лакку │ 240 │ +│ Лезги │ 242 │ +│ руски │ 330 │ +│ עברית │ 42 │ +│ فارسی │ 5 │ +│ كٲشُر │ 235 │ +│ گیلکی │ 103 │ +│ मराठी │ 51 │ +│ বাংলা │ 26 │ +│ ଓଡି଼ଆ │ 140 │ +│ தமிழ் │ 67 │ +│ ಕನ್ನಡ │ 116 │ +│ සිංහල │ 158 │ +│ ꠍꠤꠟꠐꠤ │ 284 │ +╰───────┴─────╯ +} + if {$res ne $exp} { + puts [list $res] + puts [list $exp] + } + string compare $res $exp +} {0} + +do_test 1.6 { + set res \n[db format -style box { + SELECT name, rowid AS id FROM language + WHERE length(name)=6 + ORDER BY name + }] + set exp { +╭────────┬─────╮ +│ name │ id │ +╞════════╪═════╡ +│ Betawi │ 195 │ +│ Català │ 28 │ +│ Eʋegbe │ 212 │ +│ Furlan │ 215 │ +│ Gagauz │ 217 │ +│ Galego │ 36 │ +│ Gungbe │ 222 │ +│ Gĩkũyũ │ 220 │ +│ Kabɩyɛ │ 227 │ +│ Kotava │ 119 │ +│ Kʋsaal │ 238 │ +│ Latina │ 45 │ +│ Lìgure │ 126 │ +│ Magyar │ 48 │ +│ Novial │ 264 │ +│ Patois │ 270 │ +│ Picard │ 272 │ +│ Polski │ 11 │ +│ Română │ 59 │ +│ Seediq │ 279 │ +│ Türkçe │ 73 │ +│ Vepsän │ 302 │ +│ Vèneto │ 169 │ +│ Yorùbá │ 175 │ +│ Zazaki │ 176 │ +│ Zeêuws │ 307 │ +│ lojban │ 247 │ +│ tolışi │ 166 │ +│ Аԥсшәа │ 186 │ +│ Буряад │ 199 │ +│ Монгол │ 134 │ +│ Тоҷикӣ │ 71 │ +│ Удмурт │ 300 │ +│ Хальмг │ 223 │ +│ Эрзянь │ 210 │ +│ ייִדיש │ 174 │ +│ تۆرکجه │ 72 │ +│ ܐܬܘܪܝܐ │ 190 │ +│ अंगिका │ 185 │ +│ डोटेली │ 310 │ +│ नेपाली │ 179 │ +│ मैथिली │ 129 │ +│ हिन्दी │ 39 │ +│ ਪੰਜਾਬੀ │ 142 │ +│ తెలుగు │ 69 │ +│ മലയാളം │ 130 │ +│ རྫོང་ཁ │ 316 │ +│ ရခိုင် │ 312 │ +╰────────┴─────╯ +} + if {$res ne $exp} { + puts [list $res] + puts [list $exp] + } + string compare $res $exp +} {0} + +finish_test diff --git a/test/recover.test b/test/recover.test index ad6b7298d..7035e407c 100644 --- a/test/recover.test +++ b/test/recover.test @@ -42,7 +42,7 @@ proc compare_dbs {db1 db2} { proc recover_with_opts {opts} { set cmd ".recover $opts" - set fd [open [list |$::CLI test.db $cmd]] + set fd [open [list |$::CLI -noinit test.db $cmd]] fconfigure $fd -translation binary set sql [read $fd] close $fd diff --git a/test/regexp1.sql b/test/regexp1.sql new file mode 100644 index 000000000..00bf2b41a --- /dev/null +++ b/test/regexp1.sql @@ -0,0 +1,32 @@ +#!sqlite3 +# +# 2025-12-16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the oversized patterns in the REGEXP extension found +# at ext/misc/regexp.c. +# +.mode list +.testcase 100 +-- 0 1 2 3 4 +-- 123456789 123456789 123456789 123456789 123 +SELECT 'abcdefg' REGEXP '((((((((((((((((((abcdefg))))))))))))))))))'; +.check 1 + +.testcase 110 +.limit like_pattern_length 42 +SELECT 'abcdefg' REGEXP '((((((((((((((((((abcdefg))))))))))))))))))'; +.check -glob "Error near line *: REGEXP pattern too big*" + +.testcase 120 +.limit like_pattern_length 43 +SELECT 'abcdefg' REGEXP '((((((((((((((((((abcdefg))))))))))))))))))'; +.check 1 diff --git a/test/regexp1.test b/test/regexp1.test index 0401b13d7..fb123284b 100644 --- a/test/regexp1.test +++ b/test/regexp1.test @@ -331,5 +331,29 @@ do_execsql_test regexp1-7.12 { SELECT char(0x61,0x10ffff,0x62) REGEXP char(0x10ffff); } 1 +do_execsql_test regexp1-8.0 { + CREATE TABLE t2(a); + INSERT INTO t2 VALUES('abc-def'); + SELECT length(a) FROM t2; +} {7} + +do_execsql_test regexp1-8.1 { + SELECT rowid FROM t2 WHERE a REGEXP '[1-5]'; +} {} +do_execsql_test regexp1-8.2 { + SELECT rowid FROM t2 WHERE a REGEXP '[1\-5]'; +} {1} +do_execsql_test regexp1-8.3 { + SELECT rowid FROM t2 WHERE a REGEXP '[x\-]'; +} {1} +do_catchsql_test regexp1-8.4 { + SELECT rowid FROM t2 WHERE a REGEXP '[x-]'; +} {1 {unclosed '['}} +do_execsql_test regexp1-8.5 { + SELECT rowid FROM t2 WHERE a REGEXP '-'; +} {1} +do_execsql_test regexp1-8.6 { + SELECT rowid FROM t2 WHERE a REGEXP '\-'; +} {1} finish_test diff --git a/test/rowvalue4.test b/test/rowvalue4.test index 1ef5fc292..5e02f0fc2 100644 --- a/test/rowvalue4.test +++ b/test/rowvalue4.test @@ -236,8 +236,7 @@ do_eqp_test 5.1 { QUERY PLAN |--SEARCH d2 USING INDEX d2ab (a=? AND b=?) |--LIST SUBQUERY xxxxxx - | |--SCAN d1 - | `--CREATE BLOOM FILTER + | `--SCAN d1 `--LIST SUBQUERY xxxxxx |--SCAN d1 `--CREATE BLOOM FILTER diff --git a/test/rowvalueA.test b/test/rowvalueA.test index 8760c2c39..247b7bf88 100644 --- a/test/rowvalueA.test +++ b/test/rowvalueA.test @@ -73,4 +73,53 @@ do_catchsql_test 2.3 { SELECT 2 IN ( (1, 2), (3, 4), (5, 6) ) } {1 {row value misused}} +#------------------------------------------------------------------------- +# Test the fix for forum post https://sqlite.org/forum/forumpost/6ceca07fc3 +# +do_execsql_test 3.0 { + CREATE TABLE x2 (x, y); + INSERT INTO x2 VALUES (1234, 'abc'); + + CREATE TABLE x1 (a, b PRIMARY KEY COLLATE NOCASE) WITHOUT ROWID; + INSERT INTO x1 VALUES (1234, 'ABCD'); +} + +do_execsql_test 3.1 { + SELECT * FROM x2 CROSS JOIN x1 WHERE (1234, x2.y) > (x1.a, x1.b); +} {1234 abc 1234 ABCD} + +do_execsql_test 3.2 { + CREATE INDEX x1a ON x1(a); +} + +do_execsql_test 3.3 { + SELECT * FROM x2 CROSS JOIN x1 WHERE (1234, x2.y) > (x1.a, x1.b); +} {1234 abc 1234 ABCD} + +#------------------------------------------------------------------------- +# Test the fix for forum post https://sqlite.org/forum/forumpost/7a308e933d +# +do_execsql_test 4.0 { + CREATE TABLE t1 (a PRIMARY KEY, b COLLATE NOCASE) WITHOUT ROWID; + INSERT INTO t1 VALUES ('BBB', 'a'); +} +do_execsql_test 4.1 { + SELECT * FROM t1 WHERE (t1.a, t1.b) <= ('BBB', 'CCC'); +} {BBB a} + +do_execsql_test 4.2 { + DROP TABLE t1; + CREATE TABLE t0 (c0); + INSERT INTO t0 VALUES ('True'); + + CREATE TABLE t1 (c0 COLLATE NOCASE, c1 PRIMARY KEY) WITHOUT ROWID; + INSERT INTO t1 VALUES ('a', 1); + INSERT INTO t1 VALUES ('a', 'True'); +} + +do_execsql_test 4.3 { + SELECT * FROM t0, t1 WHERE (t1.c1, t1.c0) <= (t0.c0, t0.c0); +} {True a 1 True a True} + + finish_test diff --git a/test/schema.test b/test/schema.test index c7daef20b..a6564293b 100644 --- a/test/schema.test +++ b/test/schema.test @@ -227,10 +227,10 @@ ifcapable auth { set ::STMT [sqlite3_prepare $::DB {SELECT * FROM sqlite_master} -1 TAIL] db auth {} sqlite3_step $::STMT - } {SQLITE_ROW} + } {SQLITE_ERROR} do_test schema-8.12 { sqlite3_finalize $::STMT - } {SQLITE_OK} + } {SQLITE_SCHEMA} } diff --git a/test/select9.test b/test/select9.test index bbed8e18f..bef56d83f 100644 --- a/test/select9.test +++ b/test/select9.test @@ -406,7 +406,7 @@ do_test select9-4.4 { do_test select9-4.5 { execsql { CREATE VIEW v1 AS SELECT a FROM t1 UNION SELECT d FROM t2 } cksort { SELECT a FROM v1 ORDER BY 1 LIMIT 5 } -} {1 2 3 4 5 sort} +} {1 2 3 4 5 nosort} do_test select9-4.X { execsql { DROP INDEX i1; diff --git a/test/shell1.test b/test/shell1.test index abf214a90..4f12bc096 100644 --- a/test/shell1.test +++ b/test/shell1.test @@ -54,7 +54,7 @@ do_test shell1-1.1.2 { # error on extra options do_test shell1-1.1.3 { catchcmd "test.db FOO test.db BAD" ".quit" -} {/1 .Error: in prepare, near "FOO": syntax error*/} +} {/1 .Parse error in 2nd command line argument: near "FOO": syntax error*/} # -help do_test shell1-1.2.1 { @@ -79,7 +79,7 @@ do_test shell1-1.3.2 { } {0 {}} do_test shell1-1.3.3 { catchcmd "-init FOO test.db BAD .quit" "" -} {/1 .Error: in prepare, near "BAD": syntax error*/} +} {/1 .Parse error in 4th command line argument: near "BAD": syntax error*/} # -echo print commands before execution do_test shell1-1.4.1 { @@ -216,10 +216,14 @@ do_test shell1-2.2.4 { } {0 {}} do_test shell1-2.2.5 { catchcmd "test.db" ".mode \"insert FOO" -} {1 {Error: mode should be one of: ascii box column csv html insert json line list markdown qbox quote table tabs tcl}} +} {1 {line 1: .mode "insert FOO +line 1: ^--- unknown mode +line 1: Use ".help .mode" for more info}} do_test shell1-2.2.6 { catchcmd "test.db" ".mode \'insert FOO" -} {1 {Error: mode should be one of: ascii box column csv html insert json line list markdown qbox quote table tabs tcl}} +} {1 {line 1: .mode 'insert FOO +line 1: ^--- unknown mode +line 1: Use ".help .mode" for more info}} # check multiple tokens, and quoted tokens do_test shell1-2.3.1 { @@ -247,7 +251,9 @@ do_test shell1-2.3.7 { # check quoted args are unquoted do_test shell1-2.4.1 { catchcmd "test.db" ".mode FOO" -} {1 {Error: mode should be one of: ascii box column csv html insert json line list markdown qbox quote table tabs tcl}} +} {1 {line 1: .mode FOO +line 1: ^--- unknown mode +line 1: Use ".help .mode" for more info}} do_test shell1-2.4.2 { catchcmd "test.db" ".mode csv" } {0 {}} @@ -296,9 +302,7 @@ do_test shell1-3.2.4 { catchcmd "test.db" ".bail OFF BAD" } {1 {Usage: .bail on|off}} -# This test will not work on winrt, as winrt has no concept of the absolute -# paths that the test expects in the result. It uses relative paths only. -ifcapable vtab&&!winrt { +ifcapable vtab { # .databases List names and files of attached databases do_test shell1-3.3.1 { catchcmd "-csv test.db" ".databases" @@ -364,7 +368,6 @@ do_test shell1-3.7.4 { catchcmd "test.db" ".explain OFF BAD" } {0 {}} - # .header(s) ON|OFF Turn display of headers on or off do_test shell1-3.9.1 { catchcmd "test.db" ".header" @@ -400,7 +403,7 @@ do_test shell1-3.10.1 { # look for a few of the possible help commands list [regexp {.help} $res] \ [regexp {.quit} $res] \ - [regexp {.show} $res] + [regexp {.mode} $res] } {1 1 1} do_test shell1-3.10.2 { # we allow .help to take extra args (it is help after all) @@ -408,20 +411,24 @@ do_test shell1-3.10.2 { # look for a few of the possible help commands list [regexp {.help} $res] \ [regexp {.quit} $res] \ - [regexp {.show} $res] + [regexp {.mode} $res] } {1 1 1} # .import FILE TABLE Import data from FILE into TABLE do_test shell1-3.11.1 { catchcmd "test.db" ".import" -} {/1 .ERROR: missing FILE argument.*/} +} {/1 .line 1: Missing FILE argument.*/} do_test shell1-3.11.2 { catchcmd "test.db" ".import FOO" -} {/1 .ERROR: missing TABLE argument.*/} +} {/1 .line 1: Missing TABLE argument.*/} do_test shell1-3.11.3 { # too many arguments catchcmd "test.db" ".import FOO BAR BAD" -} {/1 .ERROR: extra argument: "BAD".*./} +} {1 {line 1: .import FOO BAR BAD +line 1: ^--- unknown argument}} +do_test shell1-3.11.4 { + catchcmd "test.db" ".import <<END t1\na,b,c\n1,2,3" +} {1 {line 1: Content terminator "END" not found.}} # .indexes ?TABLE? Show names of all indexes # If TABLE specified, only show indexes for tables @@ -438,7 +445,8 @@ do_test shell1-3.12.2-legacy { do_test shell1-3.12.3 { # too many arguments catchcmd "test.db" ".indexes FOO BAD" -} {1 {Usage: .indexes ?LIKE-PATTERN?}} +} {1 {line 1: .indexes FOO BAD +line 1: ^--- unknown argument}} # .mode MODE ?TABLE? Set output mode where MODE is one of: # ascii Columns/rows delimited by 0x1F and 0x1E @@ -451,11 +459,13 @@ do_test shell1-3.12.3 { # tabs Tab-separated values # tcl TCL list elements do_test shell1-3.13.1 { - catchcmd "test.db" ".mode" -} {0 {current output mode: list --escape ascii}} + catchcmd "test.db" ".mode batch\n.mode" +} {0 {.mode list}} do_test shell1-3.13.2 { catchcmd "test.db" ".mode FOO" -} {1 {Error: mode should be one of: ascii box column csv html insert json line list markdown qbox quote table tabs tcl}} +} {1 {line 1: .mode FOO +line 1: ^--- unknown mode +line 1: Use ".help .mode" for more info}} do_test shell1-3.13.3 { catchcmd "test.db" ".mode csv" } {0 {}} @@ -503,7 +513,7 @@ do_test shell1-3.15.1 { .print x" } {0 x} do_test shell1-3.15.2 { - catchcmd "test.db" ".output FOO + catchcmd "test.db" ".mode batch\n.output FOO .print x .output SELECT readfile('FOO');" @@ -512,17 +522,8 @@ SELECT readfile('FOO');" do_test shell1-3.15.3 { # too many arguments catchcmd "test.db" ".output FOO BAD" -} {1 {ERROR: extra parameter: "BAD". Usage: -.output ?FILE? Send output to FILE or stdout if FILE is omitted - If FILE begins with '|' then open it as a pipe. - If FILE is 'off' then output is disabled. - Options: - --bom Prefix output with a UTF8 byte-order mark - -e Send output to the system text editor - --plain Use text/plain for -w option - -w Send output to a web browser - -x Send output as CSV to a spreadsheet -child process exited abnormally}} +} {1 {line 1: .output FOO BAD +line 1: ^--- surplus argument}} # .output stdout Send output to the screen do_test shell1-3.16.1 { @@ -531,17 +532,8 @@ do_test shell1-3.16.1 { do_test shell1-3.16.2 { # too many arguments catchcmd "test.db" ".output stdout BAD" -} {1 {ERROR: extra parameter: "BAD". Usage: -.output ?FILE? Send output to FILE or stdout if FILE is omitted - If FILE begins with '|' then open it as a pipe. - If FILE is 'off' then output is disabled. - Options: - --bom Prefix output with a UTF8 byte-order mark - -e Send output to the system text editor - --plain Use text/plain for -w option - -w Send output to a web browser - -x Send output as CSV to a spreadsheet -child process exited abnormally}} +} {1 {line 1: .output stdout BAD +line 1: ^--- surplus argument}} # .prompt MAIN CONTINUE Replace the standard prompts do_test shell1-3.17.1 { @@ -625,6 +617,20 @@ CREATE VIEW v1 AS SELECT y+1 FROM v2 catch {db eval {DROP VIEW v1; DROP VIEW v2; DROP TABLE t1;}} } +do_test shell1-3.21.5 { + exec {*}$CLI -noinit test.db \ + {CREATE TABLE t2(a INTEGER PRIMARY KEY, b BLOB DEFAULT(jsonb('[]')),c TEXT NOT NULL)STRICT;} \ + {.schema -indent t2} +} {CREATE TABLE t2( + a INTEGER PRIMARY KEY, + b BLOB DEFAULT(jsonb('[]')), + c TEXT NOT NULL +)STRICT;} +do_test shell1-3.21.6 { + exec {*}$CLI -noinit test.db \ + {DROP TABLE t2;} \ + {.schema -indent t2} +} {} # .separator STRING Change column separator used by output and .import do_test shell1-3.22.1 { @@ -643,7 +649,7 @@ do_test shell1-3.22.4 { # .show Show the current values for various settings do_test shell1-3.23.1 { - set res [catchcmd "test.db" ".show"] + set res [catchcmd "test.db" ".mode batch\n.show"] list [regexp {echo:} $res] \ [regexp {explain:} $res] \ [regexp {headers:} $res] \ @@ -679,7 +685,7 @@ do_test shell1-3.23b.4 { # Adverse interaction between .stats and .eqp # do_test shell1-3.23b.5 { - catchcmd "test.db" [string map {"\n " "\n"} { + catchcmd "test.db" [string map {"\n " "\n"} {.mode batch CREATE TEMP TABLE t1(x); INSERT INTO t1 VALUES(1),(2); .stats on @@ -741,30 +747,27 @@ do_test shell1-3.26.5 { do_test shell1-3.26.6 { catchcmd "test.db" ".mode column\n.header off\n.width -10 10\nSELECT 'abcdefg', 123456;" # this should be treated the same as a '1' width for col 1 and 2 -} {0 { abcdefg 123456 }} +} {0 { abcdefg 123456}} # .timer ON|OFF Turn the CPU timer measurement on or off do_test shell1-3.27.1 { catchcmd "test.db" ".timer" -} {1 {Usage: .timer on|off}} -ifcapable !winrt { - # No timer support on winrt. - do_test shell1-3.27.2 { - catchcmd "test.db" ".timer ON" - } {0 {}} -} +} {1 {Usage: .timer on|off|once}} +do_test shell1-3.27.2 { + catchcmd "test.db" ".timer ON" +} {0 {}} do_test shell1-3.27.3 { catchcmd "test.db" ".timer OFF" } {0 {}} do_test shell1-3.27.4 { # too many arguments catchcmd "test.db" ".timer OFF BAD" -} {1 {Usage: .timer on|off}} +} {1 {Usage: .timer on|off|once}} -do_test shell1-3-28.1 { +do_test shell1-3.28.1 { catchcmd test.db \ - ".log stdout\nSELECT coalesce(sqlite_log(123,'hello'),'456');" + ".mode batch\n.log stdout\nSELECT coalesce(sqlite_log(123,'hello'),'456');" } "0 {(123) hello\n456}" do_test shell1-3-29.1 { @@ -803,14 +806,14 @@ INSERT INTO t1 VALUES(''); INSERT INTO t1 VALUES(1); INSERT INTO t1 VALUES(2.25); INSERT INTO t1 VALUES('hello'); -INSERT INTO t1 VALUES(X'807f'); +INSERT INTO t1 VALUES(x'807f'); CREATE TABLE t3(x,y); INSERT INTO t3 VALUES(1,NULL); INSERT INTO t3 VALUES(2,''); INSERT INTO t3 VALUES(3,1); INSERT INTO t3 VALUES(4,2.25); INSERT INTO t3 VALUES(5,'hello'); -INSERT INTO t3 VALUES(6,X'807f'); +INSERT INTO t3 VALUES(6,x'807f'); COMMIT;}} @@ -828,14 +831,14 @@ INSERT INTO t1(rowid,x) VALUES(2,''); INSERT INTO t1(rowid,x) VALUES(3,1); INSERT INTO t1(rowid,x) VALUES(4,2.25); INSERT INTO t1(rowid,x) VALUES(5,'hello'); -INSERT INTO t1(rowid,x) VALUES(6,X'807f'); +INSERT INTO t1(rowid,x) VALUES(6,x'807f'); CREATE TABLE t3(x,y); INSERT INTO t3(rowid,x,y) VALUES(1,1,NULL); INSERT INTO t3(rowid,x,y) VALUES(2,2,''); INSERT INTO t3(rowid,x,y) VALUES(3,3,1); INSERT INTO t3(rowid,x,y) VALUES(4,4,2.25); INSERT INTO t3(rowid,x,y) VALUES(5,5,'hello'); -INSERT INTO t3(rowid,x,y) VALUES(6,6,X'807f'); +INSERT INTO t3(rowid,x,y) VALUES(6,6,x'807f'); COMMIT;}} # If the table contains an INTEGER PRIMARY KEY, do not record a separate @@ -854,12 +857,12 @@ do_test shell1-4.1.2 { } {0 {PRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE t1(x INTEGER PRIMARY KEY, y); -INSERT INTO t1 VALUES(1,NULL); -INSERT INTO t1 VALUES(2,''); -INSERT INTO t1 VALUES(3,1); -INSERT INTO t1 VALUES(4,2.25); -INSERT INTO t1 VALUES(5,'hello'); -INSERT INTO t1 VALUES(6,X'807f'); +INSERT INTO t1(x,y) VALUES(1,NULL); +INSERT INTO t1(x,y) VALUES(2,''); +INSERT INTO t1(x,y) VALUES(3,1); +INSERT INTO t1(x,y) VALUES(4,2.25); +INSERT INTO t1(x,y) VALUES(5,'hello'); +INSERT INTO t1(x,y) VALUES(6,x'807f'); COMMIT;}} # Verify that the table named [table] is correctly quoted and that @@ -883,7 +886,7 @@ INSERT INTO "table"(rowid,x,y) VALUES(2,12,''); INSERT INTO "table"(rowid,x,y) VALUES(3,23,1); INSERT INTO "table"(rowid,x,y) VALUES(4,34,2.25); INSERT INTO "table"(rowid,x,y) VALUES(5,45,'hello'); -INSERT INTO "table"(rowid,x,y) VALUES(6,56,X'807f'); +INSERT INTO "table"(rowid,x,y) VALUES(6,56,x'807f'); COMMIT;}} # Do not record rowids for a WITHOUT ROWID table. Also check correct quoting @@ -902,12 +905,12 @@ do_test shell1-4.1.4 { } {0 {PRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE [ta<>ble](x INTEGER PRIMARY KEY, y) WITHOUT ROWID; -INSERT INTO "ta<>ble" VALUES(1,NULL); -INSERT INTO "ta<>ble" VALUES(12,''); -INSERT INTO "ta<>ble" VALUES(23,1); -INSERT INTO "ta<>ble" VALUES(34,2.25); -INSERT INTO "ta<>ble" VALUES(45,'hello'); -INSERT INTO "ta<>ble" VALUES(56,X'807f'); +INSERT INTO "ta<>ble"(x,y) VALUES(1,NULL); +INSERT INTO "ta<>ble"(x,y) VALUES(12,''); +INSERT INTO "ta<>ble"(x,y) VALUES(23,1); +INSERT INTO "ta<>ble"(x,y) VALUES(34,2.25); +INSERT INTO "ta<>ble"(x,y) VALUES(45,'hello'); +INSERT INTO "ta<>ble"(x,y) VALUES(56,x'807f'); COMMIT;}} # Do not record rowids if the rowid is inaccessible @@ -924,9 +927,9 @@ do_test shell1-4.1.5 { } {0 {PRAGMA foreign_keys=OFF; BEGIN TRANSACTION; CREATE TABLE t1(_ROWID_,rowid,oid); -INSERT INTO t1 VALUES(1,NULL,'alpha'); -INSERT INTO t1 VALUES(12,'',99); -INSERT INTO t1 VALUES(23,1,X'b0b1b2'); +INSERT INTO t1(_ROWID_,rowid,oid) VALUES(1,NULL,'alpha'); +INSERT INTO t1(_ROWID_,rowid,oid) VALUES(12,'',99); +INSERT INTO t1(_ROWID_,rowid,oid) VALUES(23,1,x'b0b1b2'); COMMIT;}} } else { @@ -941,7 +944,7 @@ do_test shell1-4.1.6 { (4,2.25), (5,'hello'), (6,x'807f'); } catchcmd test2.db {.dump --preserve-rowids} -} {1 {The --preserve-rowids option is not compatible with SQLITE_OMIT_VIRTUALTABLE}} +} {/.* --preserve-rowids option is not compatible with SQLITE_OMIT_VIRTUALTABLE/} } @@ -1020,7 +1023,7 @@ INSERT INTO t1 VALUES(''); INSERT INTO t1 VALUES(1); INSERT INTO t1 VALUES(2.25); INSERT INTO t1 VALUES('hello'); -INSERT INTO t1 VALUES(X'807f');}} +INSERT INTO t1 VALUES(x'807f');}} # Test the output of ".mode insert" with headers # @@ -1031,7 +1034,7 @@ INSERT INTO t1(x) VALUES(''); INSERT INTO t1(x) VALUES(1); INSERT INTO t1(x) VALUES(2.25); INSERT INTO t1(x) VALUES('hello'); -INSERT INTO t1(x) VALUES(X'807f');}} +INSERT INTO t1(x) VALUES(x'807f');}} # Test the output of ".mode insert" # @@ -1042,7 +1045,7 @@ INSERT INTO t3 VALUES(2,''); INSERT INTO t3 VALUES(3,1); INSERT INTO t3 VALUES(4,2.25); INSERT INTO t3 VALUES(5,'hello'); -INSERT INTO t3 VALUES(6,X'807f');}} +INSERT INTO t3 VALUES(6,x'807f');}} # Test the output of ".mode insert" with headers # @@ -1053,7 +1056,7 @@ INSERT INTO t3(x,y) VALUES(2,''); INSERT INTO t3(x,y) VALUES(3,1); INSERT INTO t3(x,y) VALUES(4,2.25); INSERT INTO t3(x,y) VALUES(5,'hello'); -INSERT INTO t3(x,y) VALUES(6,X'807f');}} +INSERT INTO t3(x,y) VALUES(6,x'807f');}} # Test the output of ".mode tcl" # @@ -1069,8 +1072,8 @@ do_test shell1-4.3 { catchcmd test.db ".mode tcl\nselect * from t1;" } {0 {"" "" -"1" -"2.25" +1 +2.25 "hello" "\200\177"}} @@ -1083,15 +1086,15 @@ do_test shell1-4.4 { } catchcmd test.db ".mode tcl\nselect * from t2;" } {0 {"" "" -"1" "2.25" +1 2.25 "hello" "\200\177"}} # Test the output of ".mode tcl" with ".nullvalue" # do_test shell1-4.5 { catchcmd test.db ".mode tcl\n.nullvalue NULL\nselect * from t2;" -} {0 {"NULL" "" -"1" "2.25" +} {0 {NULL "" +1 2.25 "hello" "\200\177"}} # Test the output of ".mode tcl" with Tcl reserved characters @@ -1115,10 +1118,11 @@ do_test shell1-4.6 { # do_test shell1-4.7 { catchcmd test.db ".mode quote\nselect x'0123456789ABCDEF';" -} {0 X'0123456789abcdef'} +} {0 x'0123456789abcdef'} # Test using arbitrary byte data with the shell via standard input/output. # +if 0 { # Causes a valgrind error in TCL. Seems to be a TCL problem. do_test shell1-5.0 { # # NOTE: Skip NUL byte because it appears to be incompatible with command @@ -1185,6 +1189,7 @@ do_test shell1-5.0 { } } } {} +} # These test cases do not work on MinGW if 0 { @@ -1274,7 +1279,7 @@ do_test shell1-7.1.7 { # information. # do_test shell1-8.1 { - catchcmd ":memory:" { + catchcmd ":memory:" {.mode batch -- The pow2 table will hold all the necessary powers of two. CREATE TABLE pow2(x INTEGER PRIMARY KEY, v TEXT); WITH RECURSIVE c(x,v) AS ( @@ -1300,20 +1305,20 @@ do_test_with_ansi_output shell1-8.2 { .mode box SELECT ieee754(47.49) AS x; } -} {0 {┌───────────────────────────────┐ +} {0 {╭───────────────────────────────╮ │ x │ -├───────────────────────────────┤ +╞═══════════════════════════════╡ │ ieee754(6683623321994527,-47) │ -└───────────────────────────────┘}} +╰───────────────────────────────╯}} do_test_with_ansi_output shell1-8.3 { catchcmd ":memory: --box" { select ieee754(6683623321994527,-47) as x; } -} {0 {┌───────┐ +} {0 {╭───────╮ │ x │ -├───────┤ +╞═══════╡ │ 47.49 │ -└───────┘}} +╰───────╯}} do_test shell1-8.4 { catchcmd ":memory: --table" {SELECT ieee754_mantissa(47.49) AS M, ieee754_exponent(47.49) AS E;} } {0 {+------------------+-----+ @@ -1321,35 +1326,42 @@ do_test shell1-8.4 { +------------------+-----+ | 6683623321994527 | -47 | +------------------+-----+}} +do_test shell1-8.4b { + catchcmd ":memory: --psql" \ + {SELECT ieee754_mantissa(47.49) AS M, ieee754_exponent(47.49) AS E;} +} {0 { M | E +------------------+----- + 6683623321994527 | -47}} do_test_with_ansi_output shell1-8.5 { catchcmd ":memory: --box" { create table t(a text, b int); insert into t values ('too long for one line', 1), ('shorter', NULL); .header on +.mode box --wordwrap off .width 10 10 .nullvalue NADA select * from t;} -} {0 {┌────────────┬────────────┐ +} {0 {╭────────────┬────────────╮ │ a │ b │ -├────────────┼────────────┤ -│ too long f │ 1 │ +╞════════════╪════════════╡ +│ too long f │ 1 │ │ or one lin │ │ │ e │ │ ├────────────┼────────────┤ │ shorter │ NADA │ -└────────────┴────────────┘}} +╰────────────┴────────────╯}} #---------------------------------------------------------------------------- # Test cases shell1-9.*: Basic test that "dot" commands and SQL intermix ok. # do_test shell1-9.1 { catchcmd :memory: { -.mode csv +.mode csv --rowsep "\n" /* x */ select 1,2; --x -- .nada ; -.mode csv +.mode csv --rowsep "\n" --x select 2,1; select 3,4; } @@ -1387,4 +1399,30 @@ select base85(zeroblob(2000000000)); } } {/1.*too big.*/} +#---------------------------------------------------------------------------- +# As of 2025-11-17, the default mode is: +# +# qbox --screenwidth auto --linelimit 5 --charlimit 300 --textjsonb on +# +do_test shell1-12.1 { + catchcmd :memory: {.mode tty -quote sql +.print +SELECT jsonb(1234) AS x;} +} {0 { +╭───────────────╮ +│ x │ +╞═══════════════╡ +│ jsonb('1234') │ +╰───────────────╯}} +do_test shell1-12.2 { + catchcmd :memory: {.mode box --textjsonb on +.print +SELECT jsonb(1234) AS x;} +} {0 { +╭──────╮ +│ x │ +╞══════╡ +│ 1234 │ +╰──────╯}} + finish_test diff --git a/test/shell2.test b/test/shell2.test index 5f700a9a1..7141c4d49 100644 --- a/test/shell2.test +++ b/test/shell2.test @@ -44,7 +44,7 @@ do_test shell2-1.1.1 { # Shell silently ignores extra parameters. # Ticket [f5cb008a65]. do_test shell2-1.2.1 { - catchcmdex {:memory: "select+3" "select+4"} + catchcmdex {:memory: -list "select+3" "select+4"} } {0 {3 4 }} @@ -64,7 +64,7 @@ do_test shell2-1.3 { UPDATE OR REPLACE t5 SET a = 4 WHERE a = 1; } -} {1 {Runtime error near line 9: too many levels of trigger recursion}} +} {1 {Error near line 9: too many levels of trigger recursion}} @@ -75,7 +75,8 @@ do_test shell2-1.3 { # NB. whitespace is important do_test shell2-1.4.1 { forcedelete foo.db - catchcmd "foo.db" {CREATE TABLE foo(a); + catchcmd "foo.db" {.mode batch +CREATE TABLE foo(a); INSERT INTO foo(a) VALUES(1); SELECT * FROM foo;} } {0 1} @@ -96,7 +97,9 @@ SELECT * FROM foo; # NB. whitespace is important do_test shell2-1.4.3 { forcedelete foo.db - catchcmd "foo.db" {.echo ON + catchcmd "foo.db" { +.mode batch +.echo ON CREATE TABLE foo(a); INSERT INTO foo(a) VALUES(1); SELECT * FROM foo;} @@ -110,7 +113,9 @@ SELECT * FROM foo; # NB. whitespace is important do_test shell2-1.4.4 { forcedelete foo.db - catchcmd "foo.db" {.echo ON + catchcmd "foo.db" { +.mode batch +.echo ON CREATE TABLE foo(a); .echo OFF INSERT INTO foo(a) VALUES(1); @@ -124,7 +129,9 @@ SELECT * FROM foo;} # NB. whitespace is important do_test shell2-1.4.5 { forcedelete foo.db - catchcmdex "foo.db" {.echo ON + catchcmdex "foo.db" { +.mode batch +.echo ON CREATE TABLE foo1(a); INSERT INTO foo1(a) VALUES(1); CREATE TABLE foo2(b); @@ -153,7 +160,9 @@ SELECT * FROM foo1; SELECT * FROM foo2; # NB. whitespace is important do_test shell2-1.4.6 { forcedelete foo.db - catchcmdex "foo.db" {.echo ON + catchcmdex "foo.db" { +.mode batch +.echo ON .headers ON CREATE TABLE foo1(a); INSERT INTO foo1(a) VALUES(1); @@ -208,6 +217,7 @@ do_test shell2-1.4.9 { do_test shell2-1.4.9 { forcedelete clone.db set res [catchcmd :memory: [string trim { +.mode batch CREATE TABLE t(id INTEGER PRIMARY KEY AUTOINCREMENT); INSERT INTO t VALUES (1),(2); .clone clone.db @@ -222,6 +232,7 @@ ifcapable vtab { # See overflow report at https://sqlite.org/forum/forumpost/5d34ce5280 do_test shell2-1.4.10 { set res [catchcmd :memory: [string trim { + .mode batch SELECT * FROM generate_series(9223372036854775807,9223372036854775807,1); SELECT * FROM generate_series(9223372036854775807,9223372036854775807,-1); SELECT avg(value),min(value),max(value) FROM generate_series( @@ -248,6 +259,65 @@ do_test shell2-1.4.10 { 0 1 2}} +do_test shell2-1.4.10b { + set res [catchcmd :memory: [string trim { + .mode tty +.print + SELECT * FROM generate_series(9223372036854775807,9223372036854775807,1); + SELECT * FROM generate_series(9223372036854775807,9223372036854775807,-1); + SELECT avg(value),min(value),max(value) FROM generate_series( + -9223372036854775808,9223372036854775807,1085102592571150095); + SELECT * FROM generate_series(-9223372036854775808,9223372036854775807, + 9223372036854775807); + SELECT value FROM generate_series(-4611686018427387904, + 4611686018427387904, 4611686018427387904) ORDER BY value DESC; + SELECT * FROM generate_series(0,-2,-1); + SELECT * FROM generate_series(0,-2); + SELECT * FROM generate_series(0,2) LIMIT 3;}]] +} {0 { +╭─────────────────────╮ +│ value │ +╞═════════════════════╡ +│ 9223372036854775807 │ +╰─────────────────────╯ +╭─────────────────────╮ +│ value │ +╞═════════════════════╡ +│ 9223372036854775807 │ +╰─────────────────────╯ +╭────────────┬──────────────────────┬─────────────────────╮ +│ avg(value) │ min(value) │ max(value) │ +╞════════════╪══════════════════════╪═════════════════════╡ +│ -0.5 │ -9223372036854775808 │ 9223372036854775807 │ +╰────────────┴──────────────────────┴─────────────────────╯ +╭──────────────────────╮ +│ value │ +╞══════════════════════╡ +│ -9223372036854775808 │ +│ -1 │ +│ 9223372036854775806 │ +╰──────────────────────╯ +╭──────────────────────╮ +│ value │ +╞══════════════════════╡ +│ 4611686018427387904 │ +│ 0 │ +│ -4611686018427387904 │ +╰──────────────────────╯ +╭───────╮ +│ value │ +╞═══════╡ +│ 0 │ +│ -1 │ +│ -2 │ +╰───────╯ +╭───────╮ +│ value │ +╞═══════╡ +│ 0 │ +│ 1 │ +│ 2 │ +╰───────╯}} } ;# ifcapable vtab ifcapable vtab { @@ -260,17 +330,17 @@ do_test shell2-1.4.11 { close $df set res [catchcmd :memory: [string trim { CREATE TABLE t(line text); -.mode ascii -.separator "\377" "\n" +.mode ascii -colsep "\377" -rowsep "\n" .import dummy.csv t SELECT count(*) FROM t;}]] -} {0 1} +} {1 {0 +Error: .import column separator must be ASCII}} } ;# ifcapable vtab # Bug from forum post 7cbe081746dd3803 # Keywords as column names were producing an error message. do_test shell2-1.4.12 { - set res [catchcmd :memory: [string trim { + set res [catchcmd :memory: [string trim {.mode batch CREATE TABLE "group"("order" text); INSERT INTO "group" VALUES ('ABC'); .sha3sum}]] diff --git a/test/shell3.test b/test/shell3.test index 6bb49f5c3..3b1246dfe 100644 --- a/test/shell3.test +++ b/test/shell3.test @@ -69,7 +69,7 @@ do_test shell3-1.6 { } {0 {}} do_test shell3-1.7 { catchcmd "foo.db \"CREATE TABLE\"" -} {1 {Error: in prepare, incomplete input}} +} {1 {Parse error in 2nd command line argument: incomplete input}} #---------------------------------------------------------------------------- # shell3-2.*: Basic tests for running SQL file from command line. diff --git a/test/shell4.test b/test/shell4.test index 3ced0702e..3614909c7 100644 --- a/test/shell4.test +++ b/test/shell4.test @@ -136,15 +136,15 @@ SELECT * FROM t1;} do_test shell4-3.1 { set fd [open t1.txt wb] - puts $fd "SELECT 'squirrel';" + puts $fd ".mode list\nSELECT 'squirrel';" close $fd - exec $::CLI_ONLY :memory: --interactive ".read t1.txt" + exec $::CLI_ONLY --noinit :memory: --interactive ".read t1.txt" } {squirrel} do_test_with_ansi_output shell4-3.2 { set fd [open t1.txt wb] - puts $fd "SELECT 'pound: \302\243';" + puts $fd ".mode list\nSELECT 'pound: \302\243';" close $fd - exec $::CLI_ONLY :memory: --interactive ".read t1.txt" + exec $::CLI_ONLY --noinit :memory: --interactive ".read t1.txt" } {pound: £} do_test shell4-4.1 { diff --git a/test/shell5.test b/test/shell5.test index 70a2298bc..559dc3ce7 100644 --- a/test/shell5.test +++ b/test/shell5.test @@ -33,14 +33,15 @@ forcedelete test.db test.db-journal test.db-wal # .import FILE TABLE Import data from FILE into TABLE do_test shell5-1.1.1 { catchcmd "test.db" ".import" -} {/1 .ERROR: missing FILE argument.*/} +} {/1 .line 1: Missing FILE argument.*/} do_test shell5-1.1.2 { catchcmd "test.db" ".import FOO" -} {/1 .ERROR: missing TABLE argument.*/} +} {/1 .line 1: Missing TABLE argument.*/} do_test shell5-1.1.3 { # too many arguments catchcmd "test.db" ".import FOO BAR BAD" -} {/1 .ERROR: extra argument.*/} +} {1 {line 1: .import FOO BAR BAD +line 1: ^--- unknown argument}} # .separator STRING Change separator used by output mode and .import do_test shell5-1.2.1 { @@ -59,13 +60,13 @@ do_test shell5-1.2.4 { # column separator should default to "|" do_test shell5-1.3.1.1 { - set res [catchcmd "test.db" ".show"] + set res [catchcmd "test.db" ".mode list\n.show"] list [regexp {colseparator: \"\|\"} $res] } {1} # row separator should default to "\n" do_test shell5-1.3.1.2 { - set res [catchcmd "test.db" ".show"] + set res [catchcmd "test.db" ".mode list\n.show"] list [regexp {rowseparator: \"\\n\"} $res] } {1} @@ -82,7 +83,7 @@ do_test shell5-1.4.1 { forcedelete FOO set res [catchcmd "test.db" {CREATE TABLE t1(a, b); .import FOO t1}] -} {1 {Error: cannot open "FOO"}} +} {1 {line 2: cannot open "FOO"}} # the remainder of these test cases require virtual tables. # @@ -97,7 +98,9 @@ do_test shell5-1.4.2 { forcedelete shell5.csv set in [open shell5.csv w] close $in - set res [catchcmd ":memory:" {ATTACH 'test.db' AS test; + set res [catchcmd ":memory:" { +.mode list +ATTACH 'test.db' AS test; .import -schema test shell5.csv t1 SELECT COUNT(*) FROM test.t1;}] } {0 0} @@ -116,7 +119,7 @@ do_test shell5-1.4.4 { set in [open shell5.csv w] puts $in "1|2|3" close $in - set res [catchcmd ":memory:" {ATTACH 'test.db' AS test; + set res [catchcmd ":memory: --list" {ATTACH 'test.db' AS test; .import --schema test shell5.csv t1}] } {1 {shell5.csv:1: expected 2 columns but found 3 - extras ignored}} @@ -125,7 +128,7 @@ do_test shell5-1.4.5 { set in [open shell5.csv w] puts $in "1|2" close $in - set res [catchcmd "test.db" {DELETE FROM t1; + set res [catchcmd "test.db -list" {DELETE FROM t1; .import shell5.csv t1 SELECT COUNT(*) FROM t1;}] } {0 1} @@ -138,7 +141,9 @@ do_test shell5-1.4.6 { puts $in "2|3" puts $in "3|4" close $in - set res [catchcmd ":memory:" {ATTACH 'test.db' AS test; + set res [catchcmd ":memory:" { +.mode list +ATTACH 'test.db' AS test; .import -schema test shell5.csv t1 SELECT COUNT(*) FROM test.t1;}] } {0 3} @@ -148,7 +153,9 @@ do_test shell5-1.4.7 { set in [open shell5.csv w] puts $in "4,5" close $in - set res [catchcmd ":memory:" {ATTACH 'test.db' AS test; + set res [catchcmd ":memory:" { +.mode list +ATTACH 'test.db' AS test; .separator , .import --schema test shell5.csv t1 SELECT COUNT(*) FROM test.t1;}] @@ -159,12 +166,13 @@ do_test shell5-1.4.8.1 { set in [open shell5.csv w] puts $in "5|Now is the time for all good men to come to the aid of their country." close $in - set res [catchcmd "test.db" {.import shell5.csv t1 + set res [catchcmd "test.db" {.mode list +.import shell5.csv t1 SELECT COUNT(*) FROM t1;}] } {0 5} do_test shell5-1.4.8.2 { - catchcmd "test.db" {SELECT b FROM t1 WHERE a='5';} + catchcmd "test.db -list" {SELECT b FROM t1 WHERE a='5';} } {0 {Now is the time for all good men to come to the aid of their country.}} # import file with 1 row, 2 columns, quoted text data @@ -174,12 +182,12 @@ do_test shell5-1.4.9.1 { set in [open shell5.csv w] puts $in "6|'Now is the time for all good men to come to the aid of their country.'" close $in - set res [catchcmd "test.db" {.import shell5.csv t1 + set res [catchcmd "test.db -list" {.import shell5.csv t1 SELECT COUNT(*) FROM t1;}] } {0 6} do_test shell5-1.4.9.2 { - catchcmd "test.db" {SELECT b FROM t1 WHERE a='6';} + catchcmd "test.db -list" {SELECT b FROM t1 WHERE a='6';} } {0 {'Now is the time for all good men to come to the aid of their country.'}} # import file with 1 row, 2 columns, quoted text data @@ -187,12 +195,12 @@ do_test shell5-1.4.10.1 { set in [open shell5.csv w] puts $in "7|\"Now is the time for all good men to come to the aid of their country.\"" close $in - set res [catchcmd "test.db" {.import shell5.csv t1 + set res [catchcmd "test.db -list" {.import shell5.csv t1 SELECT COUNT(*) FROM t1;}] } {0 7} do_test shell5-1.4.10.2 { - catchcmd "test.db" {SELECT b FROM t1 WHERE a='7';} + catchcmd "test.db -list" {SELECT b FROM t1 WHERE a='7';} } {0 {Now is the time for all good men to come to the aid of their country.}} # import file with 2 rows, 2 columns and an initial BOM @@ -203,7 +211,7 @@ do_test shell5-1.4.11 { puts $in "2|3" puts $in "4|5" close $in - set res [catchcmd "test.db" {CREATE TABLE t2(x INT, y INT); + set res [catchcmd "test.db -list" {CREATE TABLE t2(x INT, y INT); .import shell5.csv t2 .mode quote .header on @@ -218,7 +226,7 @@ do_test shell5-1.4.12 { puts $in "\xef\xbb\xbf\"two\"|3" puts $in "4|5" close $in - set res [catchcmd "test.db" {DELETE FROM t2; + set res [catchcmd "test.db -list" {DELETE FROM t2; .import shell5.csv t2 .mode quote .header on @@ -232,7 +240,7 @@ do_test shell5-1.5.1 { set in [open shell5.csv w] puts $in "8|$str" close $in - set res [catchcmd "test.db" {.import shell5.csv t1 + set res [catchcmd "test.db -list" {.import shell5.csv t1 SELECT length(b) FROM t1 WHERE a='8';}] } {0 999} @@ -252,7 +260,7 @@ do_test shell5-1.6.1 { set in [open shell5.csv w] puts $in $data close $in - set res [catchcmd "test.db" {DROP TABLE IF EXISTS t2; + set res [catchcmd "test.db -list" {DROP TABLE IF EXISTS t2; .import shell5.csv t2 SELECT COUNT(*) FROM t2;}] } {0 1} @@ -268,6 +276,7 @@ do_test shell5-1.7.1 { close $in set res [catchcmd "test.db" {.mode csv .import shell5.csv t3 +.mode quote SELECT COUNT(*) FROM t3;}] } [list 0 $rows] @@ -329,6 +338,24 @@ do_test shell5-1.10 { db eval {SELECT hex(c) FROM t1 ORDER BY rowid} } {636F6C756D6E33 783320220D0A64617461222033 783320220A64617461222033} +# The --escape option +# +do_test shell5-1.10.1 { + set out [open shell5.csv w] + fconfigure $out -translation lf + puts $out {column1,column2,column3,column4} + puts $out "x1,x2%\"x3,\"x3\\\"data\\\"3\",x4" + close $out + db close + forcedelete test.db + catchcmd test.db { + CREATE TABLE t1(a,b,c,d); +.import --csv --qesc \\ --esc % shell5.csv t1 + } + sqlite3 db test.db + db eval {SELECT b, c FROM t1 ORDER BY rowid} +} {column2 column3 x2\"x3 x3\"data\"3} + # Blank last column with \r\n line endings. do_test shell5-1.11 { set out [open shell5.csv w] @@ -495,9 +522,10 @@ do_test shell5-4.4 { CREATE TEMP TABLE t8(a, b, c); .import shell5.csv t8 .nullvalue # +.mode quote SELECT * FROM temp.t8 }] -} {0 1,2,3} +} {0 '1','2','3'} #---------------------------------------------------------------------------- # Tests for the shell automatic column rename. @@ -513,7 +541,7 @@ do_test shell5-5.1 { close $out forcedelete test.db catchcmd test.db {.import -csv shell5.csv t1 -.mode line +.mode line --colsep ' = ' SELECT * FROM t1;} } {1 { ? = 0 x_02 = x2 @@ -529,7 +557,7 @@ Columns renamed during .import shell5.csv due to duplicates: "z" to "z_05", "z" to "z_08"}} -do_test shell5-5.1 { +do_test shell5-5.1b { set out [open shell5.csv w] fconfigure $out -translation lf puts $out {"COW","cow","CoW","cOw"} @@ -539,10 +567,10 @@ do_test shell5-5.1 { catchcmd test.db {.import -csv shell5.csv t1 .mode line SELECT * FROM t1;} -} {1 {COW_1 = uuu -cow_2 = lll -CoW_3 = ulu -cOw_4 = lul +} {1 {COW_1: uuu +cow_2: lll +CoW_3: ulu +cOw_4: lul Columns renamed during .import shell5.csv due to duplicates: "COW" to "COW_1", "cow" to "cow_2", @@ -561,7 +589,7 @@ do_test_with_ansi_output shell5-6.1 { close $out forcedelete test.db catchcmd test.db {.import -csv shell5.csv t1 -.mode line +.mode line --colsep " = " SELECT * FROM t1;} } {0 { あい = 1 うえお = 2}} @@ -576,8 +604,8 @@ do_test_with_ansi_output shell5-6.2 { catchcmd test.db {.import -csv shell5.csv t1 .mode line SELECT * FROM t1;} -} {0 { 1 = あい - 2 = うえお}} +} {0 {1: あい +2: うえお}} # 2024-03-11 https://sqlite.org/forum/forumpost/ca014d7358 # Import into a table that contains computed columns. @@ -588,7 +616,7 @@ do_test shell5-7.1 { puts $out {aaa|bbb} close $out forcedelete test.db - catchcmd :memory: {CREATE TABLE t1(a TEXT, b TEXT, c AS (a||b)); + catchcmd ":memory: -list" {CREATE TABLE t1(a TEXT, b TEXT, c AS (a||b)); .import shell5.csv t1 SELECT * FROM t1;} } {0 aaa|bbb|aaabbb} @@ -605,4 +633,36 @@ do_test shell5-8.1 { catchcmd :memory: {.import --csv shell5.csv '""""""""""""""""""""""""""""""""""""""""""""""'} } {0 {}} + +# 2025-12-29 https://sqlite.org/forum/forumpost/6c1c0e213d +# .import honor .bail +# +do_test shell5-9.1 { + catchcmd ":memory:" { + CREATE TABLE t1(a,b,c INT CHECK(c<>5)); +.bail on +.import -csv <<END t1 +1,2,3 +"a","b","c" +3,4,5 +"q","r","s" +END +SELECT * FROM t1;} +} {1 {<stdin>:7: INSERT failed: CHECK constraint failed: c<>5}} +do_test shell5-9.2 { + catchcmd ":memory:" { + CREATE TABLE t1(a,b,c INT CHECK(c<>5)); +.bail off +.import -csv <<END t1 +1,2,3 +"a","b","c" +3,4,5 +"q","r","s" +END +SELECT * FROM t1;} +} {1 {1|2|3 +a|b|c +q|r|s +<stdin>:7: INSERT failed: CHECK constraint failed: c<>5}} + finish_test diff --git a/test/shell8.test b/test/shell8.test index e55539636..40579a599 100644 --- a/test/shell8.test +++ b/test/shell8.test @@ -217,6 +217,46 @@ if {$tcl_platform(platform)=="unix"} { do_test 3.3 { catchcmd shell8.db {.ar -x} } {0 {}} + + # Test defenses against using symlinks to write outside + # of the destination directory. See forum thread at + # sqlite.org/forum/forumpost/2026-02-21T11:04:36z + # + forcedelete shell8.db + forcedelete ar1 + forcedelete ar2 + forcedelete ar3 + file mkdir ar2 + file mkdir ar3 + set pwd [pwd] + sqlite3 db shell8.db + db eval { + CREATE TABLE sqlar( + name TEXT PRIMARY KEY, -- name of the file + mode INT, -- access permissions + mtime INT, -- last modification time + sz INT, -- original file size + data BLOB -- compressed content + ); + INSERT INTO sqlar VALUES + ('abc',33188,0,-1,'content for abc'), + ('escape',40960,0,-1,$pwd||'/ar3'), + ('escape/def',33188,0,-1,'content for escape/def'), + ('ghi',33188,0,-1,'content for ghi'); + } + do_test 3.4.1 { + catchcmd shell8.db {.ar -x --directory ar2} + lsort [glob -tails -directory ar2 -nocomplain *] + } {abc escape ghi} + do_test 3.4.2 { + lsort [glob -tails -directory ar3 -nocomplain *] + } {} + # ^^--- An extraction into ar2 should not leak any files into ar3 + + forcedelete shell8.db + forcedelete ar2 + forcedelete ar3 + } finish_test diff --git a/test/shell9.test b/test/shell9.test index 869cb075d..173de3b06 100644 --- a/test/shell9.test +++ b/test/shell9.test @@ -74,7 +74,7 @@ do_execsql_test 1.2.1 { db close do_test 1.2.2 { catchcmd test.db ".read testdump.txt" -} {1 {Parse error near line 5: table sqlite_master may not be modified}} +} {1 {Parse error near line 5 of testdump.txt: table sqlite_master may not be modified}} # Check testdump.txt cannot be processed if the db is in safe mode # diff --git a/test/shellA.test b/test/shellA.test index f3959d428..3b28c921c 100644 --- a/test/shellA.test +++ b/test/shellA.test @@ -36,11 +36,11 @@ do_execsql_test shellA-1.0 { # and that our calls to the CLI are working. # do_test_with_ansi_output shellA-1.2 { - exec {*}$CLI test.db {.mode box --escape symbol} {SELECT * FROM t1;} + exec {*}$CLI -noinit test.db {.mode box -quote off --escape symbol} {SELECT * FROM t1;} } { -┌───┬──────────────────────────┐ +╭───┬──────────────────────────╮ │ a │ x │ -├───┼──────────────────────────┤ +╞═══╪══════════════════════════╡ │ 1 │ line with ' single quote │ ├───┼──────────────────────────┤ │ 2 │ ␛[31mVT-100 codes␛[0m │ @@ -57,39 +57,39 @@ do_test_with_ansi_output shellA-1.2 { │ 7 │ carriage␍return │ ├───┼──────────────────────────┤ │ 8 │ last line │ -└───┴──────────────────────────┘ +╰───┴──────────────────────────╯ } # ".mode list" # do_test shellA-1.3 { - exec {*}$CLI test.db {SELECT x FROM t1 WHERE a=2;} + exec {*}$CLI -noinit -list test.db {SELECT x FROM t1 WHERE a=2;} } { ^[[31mVT-100 codes^[[0m } do_test_with_ansi_output shellA-1.4 { - exec {*}$CLI test.db --escape symbol {SELECT x FROM t1 WHERE a=2;} + exec {*}$CLI -noinit -list test.db --escape symbol {SELECT x FROM t1 WHERE a=2;} } { ␛[31mVT-100 codes␛[0m } do_test shellA-1.5 { - exec {*}$CLI test.db --escape ascii {SELECT x FROM t1 WHERE a=2;} + exec {*}$CLI -noinit -list test.db --escape ascii {SELECT x FROM t1 WHERE a=2;} } { ^[[31mVT-100 codes^[[0m } do_test_with_ansi_output shellA-1.6 { - exec {*}$CLI test.db {.mode list --escape symbol} {SELECT x FROM t1 WHERE a=2;} + exec {*}$CLI -noinit test.db {.mode list --escape symbol} {SELECT x FROM t1 WHERE a=2;} } { ␛[31mVT-100 codes␛[0m } do_test shellA-1.7 { - exec {*}$CLI test.db {.mode list --escape ascii} {SELECT x FROM t1 WHERE a=2;} + exec {*}$CLI -noinit test.db {.mode list --escape ascii} {SELECT x FROM t1 WHERE a=2;} } { ^[[31mVT-100 codes^[[0m } do_test shellA-1.8 { file delete -force out.txt - exec {*}$CLI test.db {.mode list --escape off} {SELECT x FROM t1 WHERE a=7;} \ + exec {*}$CLI -noinit test.db {.mode list --escape off} {SELECT x FROM t1 WHERE a=7;} \ >out.txt set fd [open out.txt rb] set res [read $fd] @@ -98,92 +98,93 @@ do_test shellA-1.8 { } "carriage\rreturn" do_test shellA-1.9 { set rc [catch { - exec {*}$CLI test.db {.mode test --escape xyz} + exec {*}$CLI -noinit test.db {.mode test --escape xyz} } msg] lappend rc $msg -} {1 {unknown control character escape mode "xyz" - choices: ascii symbol off}} +} {1 {argv[3]: .mode test --escape xyz +argv[3]: ^--- unknown mode +argv[3]: Use ".help .mode" for more info}} do_test shellA-1.10 { set rc [catch { - exec {*}$CLI --escape abc test.db .q + exec {*}$CLI --noinit --escape abc test.db .q } msg] lappend rc $msg -} {1 {unknown control character escape mode "abc" - choices: ascii symbol off}} +} {1 {unknown control character escape mode "abc" - choices: auto off ascii symbol}} # ".mode quote" # do_test shellA-2.1 { - exec {*}$CLI test.db --quote {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} + exec {*}$CLI -noinit test.db --quote {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} } { 1,'line with '' single quote' 2,unistr('\u001b[31mVT-100 codes\u001b[0m') -6,'new -line' +6,unistr('new\u000aline') 7,unistr('carriage\u000dreturn') 8,'last line' } do_test shellA-2.2 { - exec {*}$CLI test.db --quote {.mode} -} {current output mode: quote --escape ascii} + exec {*}$CLI -noinit test.db --quote {.mode -v} +} {/*.mode quote* --escape auto*/} do_test shellA-2.3 { - exec {*}$CLI test.db --quote --escape SYMBOL {.mode} -} {current output mode: quote --escape symbol} + exec {*}$CLI -noinit test.db --quote --escape SYMBOL {.mode} +} {.mode quote --escape symbol} do_test shellA-2.4 { - exec {*}$CLI test.db --quote --escape OFF {.mode} -} {current output mode: quote --escape off} + exec {*}$CLI -noinit test.db --quote --escape OFF {.mode} +} {.mode quote --escape off} # ".mode line" # do_test_with_ansi_output shellA-3.1 { - exec {*}$CLI test.db --line --escape symbol \ + exec {*}$CLI -noinit test.db --line --escape symbol \ {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} } { - a = 1 - x = line with ' single quote + a: 1 + x: line with ' single quote - a = 2 - x = ␛[31mVT-100 codes␛[0m + a: 2 + x: ␛[31mVT-100 codes␛[0m - a = 6 - x = new -line + a: 6 + x: new + line - a = 7 - x = carriage␍return + a: 7 + x: carriage␍return - a = 8 - x = last line + a: 8 + x: last line } do_test shellA-3.2 { - exec {*}$CLI test.db --line --escape ascii \ + exec {*}$CLI -noinit test.db --line --escape ascii \ {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} } { - a = 1 - x = line with ' single quote + a: 1 + x: line with ' single quote - a = 2 - x = ^[[31mVT-100 codes^[[0m + a: 2 + x: ^[[31mVT-100 codes^[[0m - a = 6 - x = new -line + a: 6 + x: new + line - a = 7 - x = carriage^Mreturn + a: 7 + x: carriage^Mreturn - a = 8 - x = last line + a: 8 + x: last line } # ".mode box" # do_test_with_ansi_output shellA-4.1 { - exec {*}$CLI test.db --box --escape ascii \ + exec {*}$CLI -noinit test.db --box --escape ascii \ {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} } { -┌───┬──────────────────────────┐ +╭───┬──────────────────────────╮ │ a │ x │ -├───┼──────────────────────────┤ +╞═══╪══════════════════════════╡ │ 1 │ line with ' single quote │ ├───┼──────────────────────────┤ │ 2 │ ^[[31mVT-100 codes^[[0m │ @@ -194,31 +195,56 @@ do_test_with_ansi_output shellA-4.1 { │ 7 │ carriage^Mreturn │ ├───┼──────────────────────────┤ │ 8 │ last line │ -└───┴──────────────────────────┘ +╰───┴──────────────────────────╯ +} +do_test_with_ansi_output shellA-4.1b { + exec {*}$CLI -noinit test.db --box --escape ascii \ + {.mode -border off} \ + {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} +} { + a │ x +═══╪══════════════════════════ + 1 │ line with ' single quote +───┼────────────────────────── + 2 │ ^[[31mVT-100 codes^[[0m +───┼────────────────────────── + 6 │ new + │ line +───┼────────────────────────── + 7 │ carriage^Mreturn +───┼────────────────────────── + 8 │ last line } do_test_with_ansi_output shellA-4.2 { - exec {*}$CLI test.db {.mode qbox} {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} + exec {*}$CLI -noinit test.db {.mode qbox} {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} } { -┌───┬───────────────────────────────────────────┐ +╭───┬───────────────────────────────────────────╮ │ a │ x │ -├───┼───────────────────────────────────────────┤ +╞═══╪═══════════════════════════════════════════╡ │ 1 │ 'line with '' single quote' │ -├───┼───────────────────────────────────────────┤ │ 2 │ unistr('\u001b[31mVT-100 codes\u001b[0m') │ -├───┼───────────────────────────────────────────┤ -│ 6 │ 'new │ -│ │ line' │ -├───┼───────────────────────────────────────────┤ +│ 6 │ unistr('new\u000aline') │ │ 7 │ unistr('carriage\u000dreturn') │ -├───┼───────────────────────────────────────────┤ │ 8 │ 'last line' │ -└───┴───────────────────────────────────────────┘ +╰───┴───────────────────────────────────────────╯ +} +do_test_with_ansi_output shellA-4.2b { + exec {*}$CLI -noinit test.db {.mode qbox -border off} \ + {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} +} { + a │ x +═══╪═══════════════════════════════════════════ + 1 │ 'line with '' single quote' + 2 │ unistr('\u001b[31mVT-100 codes\u001b[0m') + 6 │ unistr('new\u000aline') + 7 │ unistr('carriage\u000dreturn') + 8 │ 'last line' } # ".mode insert" # do_test shellA-5.1 { - exec {*}$CLI test.db {.mode insert t1 --escape ascii} \ + exec {*}$CLI -noinit test.db {.mode insert t1 --escape ascii} \ {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} } { INSERT INTO t1 VALUES(1,'line with '' single quote'); @@ -228,7 +254,7 @@ INSERT INTO t1 VALUES(7,unistr('carriage\u000dreturn')); INSERT INTO t1 VALUES(8,'last line'); } do_test shellA-5.2 { - exec {*}$CLI test.db {.mode insert t1 --escape symbol} \ + exec {*}$CLI -noinit test.db {.mode insert t1 --escape symbol} \ {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} } { INSERT INTO t1 VALUES(1,'line with '' single quote'); @@ -239,7 +265,7 @@ INSERT INTO t1 VALUES(8,'last line'); } do_test shellA-5.3 { file delete -force out.txt - exec {*}$CLI test.db {.mode insert t1 --escape off} \ + exec {*}$CLI -noinit test.db {.mode insert t1 --escape off} \ {SELECT a, x FROM t1 WHERE a IN (1,2,6,7,8)} >out.txt set fd [open out.txt rb] set res [read $fd] @@ -254,4 +280,83 @@ INSERT INTO t1 VALUES(7,'carriage\rreturn'); INSERT INTO t1 VALUES(8,'last line'); " +# ".mode split" +# +do_test shellA-6.1 { + db eval { + CREATE TABLE t2(x); + INSERT INTO t2(x) VALUES + ('one'), ('two'), ('three'), ('four'), ('five'), + ('six'), ('seven'), ('eight'), ('nine'), ('ten'), + ('eleven'), ('twelve'), ('thirteen'), ('fourteen'); + } + exec {*}$CLI -noinit test.db \ + {.print} \ + {.mode split -screenwidth 30} \ + {SELECT x FROM t2} +} { +one five nine thirteen +two six ten fourteen +three seven eleven +four eight twelve} +# 3456789 123456789 123456789 + +do_test shellA-6.2 { + exec {*}$CLI -noinit test.db \ + {.print} \ + {.mode split -screenwidth 30} \ + {SELECT x FROM t2} \ + {.mode column -titles off} \ + {SELECT x FROM t2} +} { +one five nine thirteen +two six ten fourteen +three seven eleven +four eight twelve +one +two +three +four +five +six +seven +eight +nine +ten +eleven +twelve +thirteen +fourteen} + +do_test shellA-6.3 { + exec {*}$CLI -noinit test.db \ + {.print} \ + {.mode table} \ + {.mode --once split -screenwidth 30} \ + {SELECT x FROM t2} \ + {SELECT x FROM t2} +} { +one five nine thirteen +two six ten fourteen +three seven eleven +four eight twelve ++----------+ +| x | ++----------+ +| one | +| two | +| three | +| four | +| five | +| six | +| seven | +| eight | +| nine | +| ten | +| eleven | +| twelve | +| thirteen | +| fourteen | ++----------+} + finish_test diff --git a/test/shellB.test b/test/shellB.test new file mode 100644 index 000000000..5be2f7b6d --- /dev/null +++ b/test/shellB.test @@ -0,0 +1,53 @@ +# 2025-11-12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# TESTRUNNER: shell +# +# Test cases for the command-line shell using the newly renovated +# ".testcase" and ".check" commands. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set CLI [test_cli_invocation] + +# Run an instance of the CLI on the file $name. +# Capture the number of test cases and the number of +# errors and increment the counts. +# +proc do_clitest {name} { + set mapping [list <NAME> $::testdir/$name <CLI> $::CLI] + set script [string map $mapping { + catch {exec <CLI> :memory: ".read <NAME>" 2>@stdout} res + set ntest 0 + set nerr 999 + regexp {(\d+) tests? run with (\d+) errors?} $res all ntest nerr + set_test_counter count [expr {[set_test_counter count]+$ntest-1}] + set_test_counter errors [expr {[set_test_counter errors]+$nerr}] + if {$nerr==0} {set res "error count: 0"} + set res + }] + # puts $script + do_test shellB-$name $script {error count: 0} +} + +do_clitest modeA.sql +do_clitest dblwidth-a.sql +do_clitest vt100-a.sql +do_clitest regexp1.sql +do_clitest imposter1.sql +ifcapable vtab { + do_clitest dotcmd01.sql + do_clitest import01.sql + do_clitest intck01.sql +} +do_clitest fptest01.sql + +finish_test diff --git a/test/speedtest.md b/test/speedtest.md index 135e562ae..98cba93ff 100644 --- a/test/speedtest.md +++ b/test/speedtest.md @@ -8,8 +8,9 @@ You will need: * valgrind * tclsh * A script or program named "open" that brings up *.txt files in an - editor for viewing. (Macs provide this by default. You'll need to - come up with your own on Linux and Windows.) + editor for viewing. (Macs provide this by default. On Linux it's + called xdg-open and some distributions symlink it to "open". You'll + need to come up with your own on Windows.) * An SQLite source tree The procedure described in this document is not the only way to make diff --git a/test/speedtest.tcl b/test/speedtest.tcl index 9cb81c0fc..26bc27530 100755 --- a/test/speedtest.tcl +++ b/test/speedtest.tcl @@ -27,7 +27,7 @@ Other options include: --lookaside N SZ Lookahead uses N slots of SZ bytes each. --osmalloc Use the OS native malloc() instead of MEMSYS5 --pagesize N Use N as the page size. - --quiet | -q "Quite". Put results in file but don't pop up editor + --quiet | -q "Quiet". Put results in file but don't pop up editor --size N Change the test size. 100 means 100%. Default: 5. --testset TEST Specify the specific testset to use. The default is "mix1". Other options include: "main", "json", diff --git a/test/tabfunc01.test b/test/tabfunc01.test index 9a2017c46..60f546ce4 100644 --- a/test/tabfunc01.test +++ b/test/tabfunc01.test @@ -668,6 +668,20 @@ do_execsql_test 1370 { SELECT * FROM generate_series(0,0,0); } {} +reset_db +load_static_extension db series +do_execsql_test 1400 { + CREATE TABLE t1(x); + CREATE TABLE t2(y); +} +do_catchsql_test 1410 { + SELECT x, y, value + FROM (t1 RIGHT JOIN generate_series(t2.y,5) AS value) JOIN t2; +} {1 {table-function argument references tables to its right}} +do_catchsql_test 1420 { + SELECT x, y, value + FROM t2 JOIN (t1 RIGHT JOIN generate_series(t2.y,5) AS value) +} {1 {no such column: t2.y}} # Free up memory allocations diff --git a/test/tclsqlite.test b/test/tclsqlite.test index 5f373ea18..6cababad3 100644 --- a/test/tclsqlite.test +++ b/test/tclsqlite.test @@ -42,7 +42,7 @@ do_test tcl-1.1.1 { do_test tcl-1.2 { set v [catch {db bogus} msg] lappend v $msg -} {1 {bad option "bogus": must be authorizer, backup, bind_fallback, busy, cache, changes, close, collate, collation_needed, commit_hook, complete, config, copy, deserialize, enable_load_extension, errorcode, erroroffset, eval, exists, function, incrblob, interrupt, last_insert_rowid, nullvalue, onecolumn, preupdate, profile, progress, rekey, restore, rollback_hook, serialize, status, timeout, total_changes, trace, trace_v2, transaction, unlock_notify, update_hook, version, or wal_hook}} +} {1 {bad option "bogus": must be authorizer, backup, bind_fallback, busy, cache, changes, close, collate, collation_needed, commit_hook, complete, config, copy, deserialize, enable_load_extension, errorcode, erroroffset, eval, exists, format, function, incrblob, interrupt, last_insert_rowid, nullvalue, onecolumn, preupdate, profile, progress, rekey, restore, rollback_hook, serialize, status, timeout, total_changes, trace, trace_v2, transaction, unlock_notify, update_hook, version, or wal_hook}} do_test tcl-1.2.1 { set v [catch {db cache bogus} msg] lappend v $msg diff --git a/test/temptrigfault.tes b/test/temptrigfault.tes new file mode 100644 index 000000000..4b124e1f7 --- /dev/null +++ b/test/temptrigfault.tes @@ -0,0 +1,120 @@ +# 2025 November 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix temptrigfault + +forcedelete test.db2 +do_execsql_test 1.0 { + CREATE TABLE t1(x, y); + ATTACH 'test.db2' AS aux; + CREATE TABLE aux.t1(x, y); +} + +do_faultsim_test 1.1 -faults oom* -prep { +} -body { + execsql { + CREATE TEMP TRIGGER tmptrig AFTER INSERT ON t1 BEGIN + INSERT INTO aux.t1 VALUES(new.x, new.y); + END; + } +} -test { + faultsim_test_result {0 {}} + catchsql { DROP TRIGGER tmptrig } +} + +do_execsql_test 2.0 { + CREATE TEMP TRIGGER tmptrig AFTER INSERT ON t1 BEGIN + INSERT INTO aux.t1 VALUES(new.x, new.y); + END; +} + +do_faultsim_test 2 -faults oom* -prep { +} -body { + execsql { + INSERT INTO t1 VALUES('x', 'y'); + } +} -test { + faultsim_test_result {0 {}} +} + +do_execsql_test 3.0.1 { + SELECT * FROM t1; + DELETE FROM t1; + DELETE FROM aux.t1; +} [db eval {SELECT * FROM aux.t1}] + +do_execsql_test 3.0.2 { + CREATE TEMP TRIGGER tmptrig2 AFTER INSERT ON aux.t1 BEGIN + INSERT INTO t1 VALUES(new.x||'2', new.y||'2'); + END; +} + +do_faultsim_test 3 -faults oom* -prep { +} -body { + execsql { + INSERT INTO aux.t1 VALUES('aaa', 'bbb'); + } +} -test { + faultsim_test_result {0 {}} +} + +proc repeatlist {list n} { + set ret [list] + for {set i 0} {$i < $n} {incr i} { + set ret [concat $ret $list] + } + set ret +} + +do_execsql_test 3.x.1 { + SELECT * FROM main.t1; +} [repeatlist {aaa2 bbb2} 5] + +do_execsql_test 3.x.2 { + SELECT * FROM aux.t1; +} [repeatlist {aaa bbb aaa2 bbb2} 5] + +faultsim_save_and_close +do_faultsim_test 4 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { ATTACH 'test.db2' AS aux; } +} -body { + execsql { + CREATE TEMP TRIGGER xyz AFTER DELETE ON main.t1 BEGIN + DELETE FROM aux.t1 WHERE rowid=old.rowid; + END; + + DELETE FROM t1 WHERE rowid=2; + } +} -test { + faultsim_test_result {0 {}} {1 {unable to open a temporary database file for storing temporary tables}} +} + +faultsim_save_and_close +do_faultsim_test 5 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { ATTACH 'test.db2' AS aux; } +} -body { + execsql { + CREATE TEMP TRIGGER xyz AFTER UPDATE ON aux.t1 BEGIN + UPDATE main.t1 SET x=new.x, y=new.y WHERE rowid=new.rowid; + END; + UPDATE aux.t1 SET x=x||x WHERE rowid=1+abs(random() % 5); + } +} -test { + faultsim_test_result {0 {}} {1 {unable to open a temporary database file for storing temporary tables}} +} + + +finish_test diff --git a/test/temptrigger.test b/test/temptrigger.test index e4277adf6..74390f3b8 100644 --- a/test/temptrigger.test +++ b/test/temptrigger.test @@ -276,4 +276,191 @@ do_catchsql_test 6.3 { } {1 error} db2 close +#------------------------------------------------------------------------- +reset_db +forcedelete test.db2 + +do_execsql_test 7.0 { + CREATE TABLE m1(a, b); + ATTACH 'test.db2' AS aux; + CREATE TABLE aux.a1(c, d); +} + +do_execsql_test 7.1 { + CREATE TEMP TRIGGER tr1 AFTER INSERT ON m1 BEGIN + INSERT INTO a1 VALUES(new.a, new.b); + END; + + INSERT INTO m1 VALUES(5, 6); + SELECT * FROM aux.a1; +} {5 6} + +do_execsql_test 7.2 { + CREATE TABLE a1(e, f); + INSERT INTO m1 VALUES(7, 8); +} + +do_execsql_test 7.3.1 { SELECT * FROM main.a1 } {7 8} +do_execsql_test 7.3.2 { SELECT * FROM aux.a1 } {5 6} + +do_execsql_test 7.4 { + DROP TRIGGER tr1; + CREATE TEMP TRIGGER tr1 AFTER INSERT ON m1 BEGIN + INSERT INTO a1 SELECT d, c FROM aux.a1; + END; + + DELETE FROM aux.a1; + DELETE FROM main.a1; + INSERT INTO aux.a1 VALUES('hello', 'world'); +} + +do_execsql_test 7.5 { + INSERT INTO m1 VALUES(9, 10); + SELECT * FROM main.a1; +} {world hello} + +do_catchsql_test 7.6 { + DROP TRIGGER tr1; + CREATE TRIGGER tr1 AFTER INSERT ON m1 BEGIN + INSERT INTO a1 SELECT d, c FROM aux.a1; + END; +} {1 {trigger tr1 cannot reference objects in database aux}} + +#------------------------------------------------------------------------- +# Check that temp triggers may INSERT/UPDATE/DELETE to fully qualified +# table names. +reset_db +forcedelete {*}[glob -nocomplain *mj*] +forcedelete test.db2 +do_execsql_test 8.0 { + ATTACH 'test.db2' AS aux; + CREATE TABLE t1(a, b); + CREATE TABLE t2(c, d); + CREATE TABLE aux.t1(e, f); + CREATE TABLE aux.t2(g, h); +} + +do_catchsql_test 8.1.1 { + CREATE TRIGGER tr1 AFTER INSERT ON t2 BEGIN + INSERT INTO aux.t1 VALUES(new.c, new.d); + END; +} {1 {qualified table names are not allowed on INSERT, UPDATE, and DELETE statements within triggers}} + +do_execsql_test 8.1.2 { + CREATE TEMP TRIGGER tr1 AFTER INSERT ON t2 BEGIN + INSERT INTO aux.t1 VALUES(new.c, new.d); + END; + + INSERT INTO main.t2 VALUES('x', 'y'); + SELECT * FROM aux.t1; +} {x y} + +do_execsql_test 8.1.3 { SELECT * FROM t1 } {} + +do_catchsql_test 8.2.1 { + CREATE TRIGGER aux.tr2 AFTER UPDATE ON aux.t1 BEGIN + UPDATE main.t2 SET c=new.e, d=new.f; + END; +} {1 {qualified table names are not allowed on INSERT, UPDATE, and DELETE statements within triggers}} + +do_execsql_test 8.2.2 { + CREATE TEMP TRIGGER tr2 AFTER UPDATE ON aux.t1 BEGIN + UPDATE main.t2 SET c=new.e, d=new.f; + END; + + UPDATE aux.t1 SET e=1, f=2; + SELECT * FROM t2; +} {1 2} + +do_execsql_test 8.2.3 { SELECT * FROM aux.t2 } {} + +do_catchsql_test 8.3.1 { + CREATE TRIGGER tr3 AFTER DELETE ON t2 BEGIN + DELETE FROM aux.t1; + END; +} {1 {qualified table names are not allowed on INSERT, UPDATE, and DELETE statements within triggers}} + +do_execsql_test 8.3.2 { + INSERT INTO main.t1 VALUES('a', 'b'); + CREATE TEMP TRIGGER tr3 AFTER DELETE ON t2 BEGIN + DELETE FROM aux.t1; + END; + + DELETE FROM main.t2; + SELECT * FROM aux.t1; +} {} + +do_execsql_test 8.3.3 { SELECT * FROM t1 } {a b} + +#------------------------------------------------------------------------- +reset_db +set nDb 8 +do_test 9.0 { + for {set ii 0} {$ii < $nDb} {incr ii} { + db eval "ATTACH ':memory:' AS db$ii" + db eval "CREATE TABLE db$ii.tbl(a, b, c)" + } + + for {set ii 0} {$ii < ($nDb-1)} {incr ii} { + set jj [expr $ii+1] + db eval " + CREATE TEMP TRIGGER tr$ii AFTER INSERT ON db$ii.tbl BEGIN + INSERT INTO db$jj.tbl VALUES(new.b, new.c, new.a); + END; + " + } +} {} + +do_execsql_test 9.1 { INSERT INTO db0.tbl VALUES('a', 'b', 'c'); } +do_execsql_test 9.1.1 { SELECT * FROM db0.tbl } {a b c} +do_execsql_test 9.1.2 { SELECT * FROM db1.tbl } {b c a} +do_execsql_test 9.1.3 { SELECT * FROM db2.tbl } {c a b} +do_execsql_test 9.1.1 { SELECT * FROM db3.tbl } {a b c} +do_execsql_test 9.1.2 { SELECT * FROM db4.tbl } {b c a} +do_execsql_test 9.1.3 { SELECT * FROM db5.tbl } {c a b} +do_execsql_test 9.1.1 { SELECT * FROM db6.tbl } {a b c} +do_execsql_test 9.1.2 { SELECT * FROM db7.tbl } {b c a} + +do_test 9.2 { + for {set ii 0} {$ii < ($nDb-1)} {incr ii} { + set jj [expr $ii+1] + db eval " + CREATE TEMP TRIGGER tru$ii AFTER UPDATE ON db$ii.tbl BEGIN + UPDATE db$jj.tbl SET a=new.b, b=new.c, c=new.a; + END; + " + } +} {} + +do_execsql_test 9.3 { UPDATE db0.tbl SET a=1, b=2, c=3 } +do_execsql_test 9.3.1 { SELECT * FROM db0.tbl } {1 2 3} +do_execsql_test 9.3.2 { SELECT * FROM db1.tbl } {2 3 1} +do_execsql_test 9.3.3 { SELECT * FROM db2.tbl } {3 1 2} +do_execsql_test 9.3.1 { SELECT * FROM db3.tbl } {1 2 3} +do_execsql_test 9.3.2 { SELECT * FROM db4.tbl } {2 3 1} +do_execsql_test 9.3.3 { SELECT * FROM db5.tbl } {3 1 2} +do_execsql_test 9.3.1 { SELECT * FROM db6.tbl } {1 2 3} +do_execsql_test 9.3.2 { SELECT * FROM db7.tbl } {2 3 1} + +do_test 9.4 { + for {set ii 0} {$ii < ($nDb-1)} {incr ii} { + set jj [expr $ii+1] + db eval " + CREATE TEMP TRIGGER trd$ii BEFORE DELETE ON db$ii.tbl BEGIN + DELETE FROM db$jj.tbl; + END; + " + } +} {} + +do_execsql_test 9.5 { DELETE FROM db0.tbl } +do_execsql_test 9.5.1 { SELECT * FROM db0.tbl } {} +do_execsql_test 9.5.2 { SELECT * FROM db1.tbl } {} +do_execsql_test 9.5.3 { SELECT * FROM db2.tbl } {} +do_execsql_test 9.5.1 { SELECT * FROM db3.tbl } {} +do_execsql_test 9.5.2 { SELECT * FROM db4.tbl } {} +do_execsql_test 9.5.3 { SELECT * FROM db5.tbl } {} +do_execsql_test 9.5.1 { SELECT * FROM db6.tbl } {} +do_execsql_test 9.5.2 { SELECT * FROM db7.tbl } {} + finish_test diff --git a/test/tester.tcl b/test/tester.tcl index 3fad39668..856df5421 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -1785,11 +1785,6 @@ proc crashsql {args} { # cfSync(), which can be different then what TCL uses by # default, so here we force it to the "nativename" format. set cfile [string map {\\ \\\\} [file nativename [file join [get_pwd] $crashfile]]] - ifcapable winrt { - # Except on winrt. Winrt has no way to transform a relative path into - # an absolute one, so it just uses the relative paths. - set cfile $crashfile - } set f [open crash.tcl w] puts $f "sqlite3_initialize ; sqlite3_shutdown" diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 515036368..62dce8e62 100755 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -101,6 +101,7 @@ Usage: $a0 help $a0 joblist ?PATTERN? $a0 njob ?NJOB? + $a0 retest $a0 script ?-msvc? CONFIG $a0 status ?-d SECS? ?--cls? $a0 halt @@ -120,26 +121,30 @@ Usage: --stop-on-error Stop running after any reported error --zipvfs ZIPVFSDIR ZIPVFS source directory -Special values for PERMUTATION that work with plain tclsh: +Special values for PERMUTATION include: - list - show all allowed PERMUTATION arguments. + list - show allowed PERMUTATION arguments. mdevtest - tests recommended prior to normal development check-ins. + devtest - alias for "mdevtest" release - full release test with various builds. sdevtest - like mdevtest but using ASAN and UBSAN. - -Other PERMUTATION arguments must be run using testfixture, not tclsh: - all - all tcl test scripts, plus a subset of test scripts rerun with various permutations. full - all tcl test scripts. veryquick - a fast subset of the tcl test scripts. This is the default. +The interpreter that runs this script can be an ordinary "tclsh" as long +as "package require sqlite3" works, or it can be "testfixture". + If no PATTERN arguments are present, all tests specified by the PERMUTATION are run. Otherwise, each pattern is interpreted as a glob pattern. Only those tcl tests for which the final component of the filename matches at least one specified pattern are run. The glob wildcard '*' is prepended to the pattern if it does not start with '^' and appended to every -pattern that does not end with '$'. +pattern that does not end with '$'. If PATTERN begins with "~", then it +is an anti-pattern that only matches tests that do not match PATTERN. +Tests or only run if they match one or more patterns and match no +anti-patterns. If no PATTERN arguments are present, then various fuzztest, threadtest and other tests are run as part of the "release" permutation. These are @@ -167,6 +172,9 @@ only the parts that contain the error messages. The --summary option just shows the jobs that failed. If PATTERN are provided, the error information is only provided for jobs that match PATTERN. +The "retest" command reruns tests that failed or were never completed +by a prior invocation of testrunner.tcl. + Full documentation here: https://sqlite.org/src/doc/trunk/doc/testrunner.md }]] @@ -212,7 +220,8 @@ proc default_njob {} { if {$nCore<=2} { set nHelper 1 } else { - set nHelper [expr int($nCore*0.5)] + set nHelper [expr int($nCore*0.8)] + if {$nHelper>20} {set nHelper 20} } return $nHelper } @@ -297,6 +306,8 @@ switch -nocase -glob -- $tcl_platform(os) { error "cannot determine platform!" } } +set TRG(testfixture-fullpath) [file join $dir $TRG(testfixture)] +set TRG(interp) [info nameofexec] #------------------------------------------------------------------------- #------------------------------------------------------------------------- @@ -706,9 +717,18 @@ if {[llength $argv]>=1 } } - if {![file readable $TRG(dbname)]} { - puts "Database missing: $TRG(dbname)" - exit + set once 1 + while {![file readable $TRG(dbname)]} { + if {$delay==0} { + puts "Database missing: $TRG(dbname)" + exit + } + if {$once} { + set once 0 + puts "Waiting for testing to start...." + flush stdout + } + after [expr {$delay*1000}] } sqlite3 mydb $TRG(dbname) mydb timeout 2000 @@ -1107,10 +1127,21 @@ proc add_job {args} { # # An empty patternlist matches everything # +# Entries of patternlist that begin with "~" mean "match anything that +# does not match the following pattern". For example, a patternlist +# of {fuzzcheck ~san} will match "fuzzcheck" but not "fuzzcheck-asan". +# proc job_matches_any_pattern {patternlist jobcmd} { set bMatch 0 + set bMiss 0 if {[llength $patternlist]==0} {return 1} foreach p $patternlist { + if {[string index $p 0] eq "~"} { + set p [string range $p 1 end] + set not 1 + } else { + set not 0 + } set p [string trim $p *] if {[string index $p 0]=="^"} { set p [string range $p 1 end] @@ -1122,11 +1153,19 @@ proc job_matches_any_pattern {patternlist jobcmd} { } else { set p "$p*" } - if {[string match $p $jobcmd]} { - set bMatch 1 - break + if {$not} { + if {[string match $p $jobcmd]} {return 0} + } else { + if {[string match $p $jobcmd]} { + set bMatch 1 + } else { + set bMiss 1 + } } } + if {!$bMiss} { + set bMatch 1 + } return $bMatch } @@ -1148,7 +1187,7 @@ proc add_tcl_jobs {build config patternlist {shelldepid ""}} { set testrunner_tcl [file normalize [info script]] if {$build==""} { - set testfixture [info nameofexec] + set testfixture $TRG(interp) } else { set testfixture [file join [lindex $build 1] $TRG(testfixture)] } @@ -1269,6 +1308,26 @@ proc add_fuzztest_jobs {buildname patternlist} { set subcmd [lrange $interpreter 1 end] set interpreter [lindex $interpreter 0] + # For fuzzcheck-asan and fuzzcheck-ubsan, break up some + # fuzzdata files into multiple slices, for improved + # concurrency. + # + if {[string match *fuzzcheck-*san $interpreter]} { + set newscripts {} + foreach s $scripts { + if {[string match {*fuzzdata[12].db} $s] + && ![string match slice $s]} { + set N 6 + for {set i 0} {$i<$N} {incr i} { + lappend newscripts [list --slice $i $N $s] + } + } else { + lappend newscripts $s + } + } + set scripts $newscripts + } + if {[string match fuzzcheck* $interpreter] && [info exists env(FUZZDB)] && [file readable $env(FUZZDB)] @@ -1359,14 +1418,30 @@ proc add_devtest_jobs {lBld patternlist} { } } -# Check to ensure that the interpreter is a full-blown "testfixture" -# build and not just a "tclsh". If this is not the case, issue an -# error message and exit. +# Check to ensure that TRG(interp) is a full-blown "testfixture" and +# not just a "tclsh". +# +# The value of TRG(interp) defaults to whatever interpreter is running +# this script, which might be either tclsh or testfixture. If tclsh is +# running this script, change $TRG(interp) to be an instance of testfixture. +# If no testfixture exists in the directory from which this script is run, +# attempt to build one. +# +# Do not return unless $TRG(interp) is a valid testfixture. If unable +# to find and/or construct one, abort with an error message. # proc must_be_testfixture {} { + global TRG if {[lsearch [info commands] sqlite3_soft_heap_limit]<0} { - puts "Use testfixture, not tclsh, for these arguments." - exit 1 + if {![file exec $TRG(testfixture-fullpath)]} { + puts "make testfixture" + catch {exec make testfixture >@stdout 2>@stderr} + } + if {![file exec $TRG(testfixture-fullpath)]} { + puts "Requires testfixture, and I was unable to build it." + exit 1 + } + set TRG(interp) $TRG(testfixture-fullpath) } } @@ -1441,11 +1516,15 @@ proc add_jobs_from_cmdline {patternlist} { list { set allperm [array names ::testspec] - lappend allperm all mdevtest sdevtest release list + lappend allperm all devtest mdevtest sdevtest release list puts "Allowed values for the PERMUTATION argument: [lsort $allperm]" exit 0 } + retest { + # no-op + } + default { must_be_testfixture if {[info exists ::testspec($first)]} { @@ -1482,6 +1561,8 @@ proc add_jobs_from_cmdline {patternlist} { } } +# Initializer, or reinitialize, the testrunner.db database file. +# proc make_new_testset {} { global TRG @@ -1525,7 +1606,8 @@ proc mark_job_as_finished {jobid output state endtm} { SET output=$output, state=$state, endtime=$endtm, span=$endtm-starttime, ntest=$ntest, nerr=$nerr, svers=$svers, pltfm=$pltfm WHERE jobid=$jobid; - UPDATE jobs SET state=$childstate WHERE depid=$jobid AND state!='halt'; + UPDATE jobs SET state=$childstate + WHERE depid=$jobid AND state!='halt' AND state!='done'; UPDATE config SET value=value+$nerr WHERE name='nfail'; UPDATE config SET value=value+$ntest WHERE name='ntest'; } @@ -1597,7 +1679,7 @@ proc launch_another_job {iJob} { global O global T - set testfixture [info nameofexec] + set testfixture $TRG(interp) set script $TRG(info_script) set O($iJob) "" @@ -1798,6 +1880,27 @@ proc run_testset {} { } +# If the argument is "retest", simply rerun all tests from the previous +# run that are marked as one of "ready", "running", "failed", or "omit" +# plus redo any build of dependencies those tests. +# +proc handle_retest {} { + set cnt 0 + if {[catch {trdb exists {SELECT jobid FROM jobs}} cnt] || $cnt==0} { + puts "No test available to rerun" + exit 1 + } + trdb eval {UPDATE jobs SET state='ready' + WHERE state IN ('running','failed','omit')} + for {set kk 0} {$kk<2} {incr kk} { + trdb eval { + UPDATE jobs SET state='ready' + WHERE jobid IN (SELECT depid FROM jobs WHERE state='ready'); + UPDATE jobs SET state='' WHERE state='ready' AND depid<>''; + } + } +} + # Handle the --buildonly option, if it was specified. # proc handle_buildonly {} { @@ -1836,14 +1939,21 @@ proc explain_tests {} { sqlite3 trdb $TRG(dbname) trdb timeout $TRG(timeout) -set tm [lindex [time { make_new_testset }] 0] +if {[llength $TRG(patternlist)]==1 && $TRG(patternlist) eq "retest"} { + set tm 0 + handle_retest +} else { + set tm [lindex [time { make_new_testset }] 0] +} if {$TRG(explain)} { explain_tests } else { if {$TRG(nJob)>1} { puts "splitting work across $TRG(nJob) cores" } - puts "built testset in [expr $tm/1000]ms.." + if {$tm>0} { + puts "built testset in [expr $tm/1000]ms.." + } handle_buildonly run_testset } diff --git a/test/testrunner_data.tcl b/test/testrunner_data.tcl index e74caee1d..57ab05443 100644 --- a/test/testrunner_data.tcl +++ b/test/testrunner_data.tcl @@ -37,7 +37,6 @@ namespace eval trd { set tcltest(win.Windows-Memdebug) veryquick set tcltest(win.Windows-Win32Heap) veryquick set tcltest(win.Windows-Sanitize) veryquick - set tcltest(win.Windows-WinRT) veryquick set tcltest(win.Default) {full win_unc_locking} # Extra [make xyz] tests that should be run for various builds. @@ -193,6 +192,7 @@ namespace eval trd { -DSQLITE_ENABLE_HIDDEN_COLUMNS -DSQLITE_MAX_ATTACHED=125 -DSQLITE_MUTATION_TEST + -DSQLITE_THREAD_MISUSE_ABORT --enable-fts5 } set build(Debug-Two) { @@ -364,12 +364,6 @@ namespace eval trd { set build(Windows-Sanitize) { ASAN=1 } - - set build(Windows-WinRT) { - FOR_WINRT=1 - ENABLE_SETLK=1 - -DSQLITE_TEMP_STORE=3 - } } @@ -487,7 +481,7 @@ proc make_sh_script {srcdir opts cflags makeOpts configOpts} { set myopts "" if {[info exists ::env(OPTS)]} { append myopts "# From environment variable:\n" - append myopts "OPTS=$::env(OPTS)\n\n" + append myopts "OPTS=\"$::env(OPTS)\"\n\n" } foreach o [lsort $opts] { append myopts "OPTS=\"\$OPTS $o\"\n" diff --git a/test/testrunner_estwork.tcl b/test/testrunner_estwork.tcl index c139394a5..e02eb22dc 100644 --- a/test/testrunner_estwork.tcl +++ b/test/testrunner_estwork.tcl @@ -364,6 +364,7 @@ set estwork(shell6.test) 3 set estwork(shell8.test) 104 set estwork(shell9.test) 3 set estwork(shellA.test) 2 +set estwork(shellB.test) 2 set estwork(shmlock.test) 27 set estwork(sidedelete.test) 10 set estwork(skipscan1.test) 7 diff --git a/test/tkt-99378177930f87bd.test b/test/tkt-99378177930f87bd.test index ba9fdc702..495867280 100644 --- a/test/tkt-99378177930f87bd.test +++ b/test/tkt-99378177930f87bd.test @@ -33,6 +33,8 @@ do_execsql_test tkt-99378-100 { (2, '{"x":2}', 4, 5), (3, '{"x":1}', 6, 7); CREATE INDEX t1x ON t1(d, a, b->>'x', c); + CREATE TABLE t2(y); + INSERT INTO t2(y) VALUES(9); } {} do_execsql_test tkt-99378-110 { SELECT a, @@ -48,6 +50,20 @@ do_execsql_test tkt-99378-110 { 2 2 1 26 22 3 1 1 6 6 } +do_execsql_test tkt-99378-111 { + SELECT if(a,a,y), + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t2 CROSS JOIN t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} { + 1 2 1 16 12 + 2 2 1 26 22 + 3 1 1 6 6 +} # The proof that the index on the expression is being used is in the # fact that the byte code contains no "Function" opcodes. In other words, @@ -65,6 +81,17 @@ do_execsql_test tkt-99378-120 { WHERE d BETWEEN 0 and 10 GROUP BY a; } {~/Function/} +do_execsql_test tkt-99378-121 { + EXPLAIN + SELECT if(a,a,y), + SUM(1) AS t1, + SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2, + SUM(c) AS t3, + SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4 + FROM t2 CROSS JOIN t1 + WHERE d BETWEEN 0 and 10 + GROUP BY a; +} {~/Function/} do_execsql_test tkt-99378-130 { @@ -182,6 +209,7 @@ do_execsql_test tkt-99378-310 { # do_execsql_test tkt-99378-400 { DROP TABLE t1; + DROP TABLE t2; CREATE TABLE t0(w); INSERT INTO t0(w) VALUES(1); CREATE TABLE t1(x); diff --git a/test/tkt2339.test b/test/tkt2339.test index 41acd377c..4bbedb828 100644 --- a/test/tkt2339.test +++ b/test/tkt2339.test @@ -47,12 +47,12 @@ do_test tkt2339.2 { } } {4 3 14 13} do_test tkt2339.3 { - execsql { + lsort -integer [execsql { SELECT * FROM (SELECT * FROM t1 ORDER BY num DESC) UNION ALL SELECT * FROM (SELECT * FROM t2 ORDER BY num DESC LIMIT 2) - } -} {4 3 2 1 14 13} + }] +} {1 2 3 4 13 14} do_test tkt2339.4 { execsql { SELECT * FROM (SELECT * FROM t1 ORDER BY num DESC LIMIT 2) diff --git a/test/values.test b/test/values.test index c3c52ceb1..58e764ce6 100644 --- a/test/values.test +++ b/test/values.test @@ -21,9 +21,9 @@ do_execsql_test 1.0 { } -explain_i { - INSERT INTO x1(a, b, c) VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4); -} +#explain_i { +# INSERT INTO x1(a, b, c) VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4); +#} do_execsql_test 1.1.1 { INSERT INTO x1 VALUES(1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4); } diff --git a/test/vt02.c b/test/vt02.c index 06fade096..e08d649c1 100644 --- a/test/vt02.c +++ b/test/vt02.c @@ -45,8 +45,29 @@ ** a=... AND b=... AND c=... ** a=... AND b=... AND c=... AND d=... ** -** Various ORDER BY constraints are also recognized and consumed. The -** OFFSET constraint is recognized and consumed. +** The table will also recognize IN constraints on column D using the +** sqlite3_vtab_in_first() and sqlite3_vtab_in_next() interfaces: +** +** a=... AND d IN (...) +** a=... AND b=... AND d IN (...) +** a=... AND b=... AND c=... AND d IN (...) +** +** Various ORDER BY constraints are also recognized and consumed. +** +** ORDER BY x +** ORDER BY a +** ORDER BY a, b +** ORDER BY a, b, c +** ORDER BY a, b, c, d +** +** The above also work if every term is DESC rather than ASC. However, +** the orderByConsumed is not set if there are a mixture of ASC and DESC +** terms in the ORDER BY clause. +** +** The sqlite3_vtab_distinct() interface is used to recognize DISTINCT +** and GROUP BY constraints and consume the corresponding sorting requirements. +** +** The OFFSET constraint is recognized and consumed. ** ** ## TABLE-VALUED FUNCTION ** @@ -90,6 +111,13 @@ ** vector of results sent to xFilter. Only the first ** few are used, as required by idxNum. ** +** 0x80 If sqlite3_vtab_distinct() says that duplicate rows +** may be omitted (values 2 or 3), then maybe omit +** some but not all of the duplicate rows. +** +** 0x100 Do not omit any duplicate rows even if +** sqlite3_vtab_distinct() says that is ok to do. +** ** Because these flags take effect during xBestIndex, the RHS of the ** flag= constraint must be accessible. In other words, the RHS of flag= ** needs to be an integer literal, not another column of a join or a @@ -136,14 +164,12 @@ ** ## COMPILING AND RUNNING ** ** This file can also be compiled separately as a loadable extension -** for SQLite (as long as the -DTH3_VERSION is not defined). To compile as a -** loadable extension do his: -** -** gcc -Wall -g -shared -fPIC -I. -DSQLITE_DEBUG vt02.c -o vt02.so +** for SQLite (as long as the -DTH3_VERSION is not defined). To compile as a +** loadable extension do something like this: ** -** Or on Windows: -** -** cl vt02.c -link -dll -out:vt02.dll +** (linux) gcc -shared -fPIC -I. vt02.c -o vt02.so +** (mac) clang -dynamiclib -fPIC -I. vt02.c -o vt02.dylib +** (windows) cl vt02.c -link -dll -out:vt02.dll ** ** Then load into the CLI using: ** @@ -167,6 +193,7 @@ ** 2x increment by 100 ** 3x increment by 1000 ** 1xx Use offset provided by argv[N] +** 1xxx Reverse output order (to implement ORDER BY ... DESC) */ #ifndef TH3_VERSION /* These bits for separate compilation as a loadable extension, only */ @@ -195,6 +222,8 @@ struct vt02_vtab { #define VT02_NO_OFFSET 0x0004 /* Omit the offset optimization */ #define VT02_ALLOC_IDXSTR 0x0008 /* Alloate an idxStr */ #define VT02_BAD_IDXNUM 0x0010 /* Generate an invalid idxNum */ +#define VT02_PARTIAL_DEDUP 0x0080 /* Omit some but not all duplicate rows */ +#define VT02_NO_DEDUP 0x0100 /* Include all duplicate rows */ /* ** A cursor @@ -203,6 +232,7 @@ struct vt02_cur { sqlite3_vtab_cursor parent; /* Base class. Must be first */ sqlite3_int64 i; /* Current entry */ sqlite3_int64 iEof; /* Indicate EOF when reaching this value */ + sqlite3_int64 iMin; /* EOF if dropping below this value */ int iIncr; /* Amount by which to increment */ unsigned int mD; /* Mask of allowed D-column values */ }; @@ -292,7 +322,7 @@ static int vt02Close(sqlite3_vtab_cursor *pCursor){ */ static int vt02Eof(sqlite3_vtab_cursor *pCursor){ vt02_cur *pCur = (vt02_cur*)pCursor; - return pCur->i<0 || pCur->i>=pCur->iEof; + return pCur->i<pCur->iMin || pCur->i>=pCur->iEof; } /* Advance the cursor to the next row in the table @@ -301,8 +331,8 @@ static int vt02Next(sqlite3_vtab_cursor *pCursor){ vt02_cur *pCur = (vt02_cur*)pCursor; do{ pCur->i += pCur->iIncr; - if( pCur->i<0 ) pCur->i = pCur->iEof; - }while( (pCur->mD & (1<<(pCur->i%10)))==0 && pCur->i<pCur->iEof ); + if( pCur->i<pCur->iMin || pCur->i>=pCur->iEof ) break; + }while( (pCur->mD & (1<<(pCur->i%10)))==0 ); return SQLITE_OK; } @@ -324,6 +354,7 @@ static int vt02Next(sqlite3_vtab_cursor *pCursor){ ** 2x increment by 100 ** 3x increment by 1000 ** 1xx Use offset provided by argv[N] +** 1xxx Output rows in reverse order */ static int vt02Filter( sqlite3_vtab_cursor *pCursor, /* The cursor to rewind */ @@ -334,11 +365,17 @@ static int vt02Filter( ){ vt02_cur *pCur = (vt02_cur*)pCursor; /* The vt02 cursor */ int bUseOffset = 0; /* True to use OFFSET value */ + int bReverse = 0; /* Output rows in reverse order */ int iArg = 0; /* argv[] values used so far */ int iOrigIdxNum = idxNum; /* Original value for idxNum */ pCur->iIncr = 1; + pCur->iMin = 0; pCur->mD = 0x3ff; + if( idxNum>=1000 ){ + bReverse = 1; + idxNum -= 1000; + } if( idxNum>=100 ){ bUseOffset = 1; idxNum -= 100; @@ -416,9 +453,17 @@ static int vt02Filter( }else{ goto vt02_bad_idxnum; } + if( bReverse ){ + sqlite3_int64 x; + x = pCur->i + ((pCur->iEof - pCur->i)/pCur->iIncr)*pCur->iIncr; + if( x>=pCur->iEof ) x -= pCur->iIncr; + pCur->iIncr = -pCur->iIncr; + pCur->iMin = pCur->i; + pCur->i = x; + } if( bUseOffset ){ int nSkip = sqlite3_value_int(argv[iArg]); - while( nSkip-- > 0 && pCur->i<pCur->iEof ) vt02Next(pCursor); + while( nSkip-- > 0 && !vt02Eof(pCursor) ) vt02Next(pCursor); } return SQLITE_OK; @@ -838,35 +883,52 @@ static int vt02BestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ ** the same answer. */ if( pInfo->nOrderBy>0 && (flags & VT02_NO_SORT_OPT)==0 ){ + int eDistinct = sqlite3_vtab_distinct(pInfo); if( pInfo->idxNum==1 ){ /* There will only be one row of output. So it is always sorted. */ pInfo->orderByConsumed = 1; }else - if( pInfo->aOrderBy[0].iColumn<=0 - && pInfo->aOrderBy[0].desc==0 - ){ - /* First column of order by is X ascending */ + if( pInfo->aOrderBy[0].iColumn<=0 ){ + /* First column of order by is X */ + if( pInfo->aOrderBy[0].desc ){ + pInfo->idxNum += 1000; /* Reverse output order */ + } pInfo->orderByConsumed = 1; }else - if( sqlite3_vtab_distinct(pInfo)>=1 ){ + if( eDistinct>=1 ){ unsigned int x = 0; + int nDesc = 0; + int nAsc = 0; for(i=0; i<pInfo->nOrderBy; i++){ int iCol = pInfo->aOrderBy[i].iColumn; if( iCol<0 ) iCol = 0; + if( pInfo->aOrderBy[i].desc ){ + nDesc++; + }else{ + nAsc++; + } x |= 1<<iCol; } - if( sqlite3_vtab_distinct(pInfo)==2 ){ + if( nDesc>0 && nAsc>0 ){ + if( eDistinct!=1 ) eDistinct = -999; /* Never set orderByConsumed */ + }else if( nAsc==0 ){ + pInfo->idxNum += 1000; /* Reverse output order */ + } + if( eDistinct>=2 && (flags & VT02_NO_DEDUP)!=0 ){ + eDistinct = 1; + } + if( eDistinct>=2 ){ /* DISTINCT or (DISTINCT and ORDER BY) */ if( x==0x02 ){ /* DISTINCT A */ - pInfo->idxNum += 30; + pInfo->idxNum += (flags & VT02_PARTIAL_DEDUP)!=0 ? 20 : 30; pInfo->orderByConsumed = 1; }else if( x==0x06 ){ /* DISTINCT A,B */ - pInfo->idxNum += 20; + pInfo->idxNum += (flags & VT02_PARTIAL_DEDUP)!=0 ? 10 : 20; pInfo->orderByConsumed = 1; }else if( x==0x0e ){ /* DISTINCT A,B,C */ - pInfo->idxNum += 10; + pInfo->idxNum += (flags & VT02_PARTIAL_DEDUP)!=0 ? 0 : 10; pInfo->orderByConsumed = 1; }else if( x & 0x01 ){ /* DISTINCT X */ @@ -875,7 +937,7 @@ static int vt02BestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ /* DISTINCT A,B,C,D */ pInfo->orderByConsumed = 1; } - }else{ + }else if( eDistinct==1 ){ /* GROUP BY */ if( x==0x02 ){ /* GROUP BY A */ pInfo->orderByConsumed = 1; @@ -893,6 +955,21 @@ static int vt02BestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ pInfo->orderByConsumed = 1; } } + }else{ + int nDesc = 0; + int nAsc = 0; + for(i=0; i<pInfo->nOrderBy; i++){ + if( pInfo->aOrderBy[i].iColumn!=i+1 ) break; + if( pInfo->aOrderBy[i].desc ){ + nDesc++; + }else{ + nAsc++; + } + } + if( i==pInfo->nOrderBy && (nDesc==0 || nAsc==0) ){ + pInfo->orderByConsumed = 1; + if( nDesc ) pInfo->idxNum += 1000; + } } } @@ -901,7 +978,7 @@ static int vt02BestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ pInfo->needToFreeIdxStr = 1; } if( flags & VT02_BAD_IDXNUM ){ - pInfo->idxNum += 1000; + pInfo->idxNum += 10000; } if( iOffset>=0 ){ diff --git a/test/vt100-a.sql b/test/vt100-a.sql index a0d3f46be..f141f637f 100644 --- a/test/vt100-a.sql +++ b/test/vt100-a.sql @@ -9,11 +9,40 @@ INSERT INTO t1 VALUES ('one','twotwotwo','thirty-three'), (unistr('\u001b[91mRED\u001b[0m'),'fourfour','fifty-five'), ('six','seven','eighty-eight'); -.print With -escape off +.testcase 100 SELECT * FROM t1; +.check <<END +╭─────┬───────────┬──────────────╮ +│ a │ b │ c │ +╞═════╪═══════════╪══════════════╡ +│ one │ twotwotwo │ thirty-three │ +│ RED │ fourfour │ fifty-five │ +│ six │ seven │ eighty-eight │ +╰─────┴───────────┴──────────────╯ +END + .mode box -escape ascii -.print With -escape ascii +.testcase 200 SELECT * FROM t1; +.check <<END +╭────────────────┬───────────┬──────────────╮ +│ a │ b │ c │ +╞════════════════╪═══════════╪══════════════╡ +│ one │ twotwotwo │ thirty-three │ +│ ^[[91mRED^[[0m │ fourfour │ fifty-five │ +│ six │ seven │ eighty-eight │ +╰────────────────┴───────────┴──────────────╯ +END + +.testcase 300 .mode box -escape symbol -.print With -escape symbol SELECT * FROM t1; +.check <<END +╭──────────────┬───────────┬──────────────╮ +│ a │ b │ c │ +╞══════════════╪═══════════╪══════════════╡ +│ one │ twotwotwo │ thirty-three │ +│ ␛[91mRED␛[0m │ fourfour │ fifty-five │ +│ six │ seven │ eighty-eight │ +╰──────────────┴───────────┴──────────────╯ +END diff --git a/test/walckptnoop.test b/test/walckptnoop.test index 7ff8e90b8..89055316f 100644 --- a/test/walckptnoop.test +++ b/test/walckptnoop.test @@ -102,7 +102,11 @@ do_catchsql_test 1.8 { PRAGMA wal_checkpoint = noop; } {0 {0 5 0}} -do_execsql_test 1.9 { +do_test 1.9 { + sqlite3_wal_checkpoint_v2 db noop +} {0 5 0} + +do_execsql_test 1.10 { PRAGMA journal_mode = delete; PRAGMA wal_checkpoint = noop; } {delete 0 -1 -1} diff --git a/test/walrestart.test b/test/walrestart.test index 4274b2e33..cf27a4098 100644 --- a/test/walrestart.test +++ b/test/walrestart.test @@ -78,10 +78,12 @@ do_execsql_test -db db2 1.3 { proc faultsim {n} { return 0 } do_execsql_test 1.4 { PRAGMA wal_checkpoint; -} {0 58 58} +} {/0 5. 5./} do_catchsql_test 1.5 { PRAGMA integrity_check } {0 ok} +sqlite3_test_control_fault_install + finish_test diff --git a/test/where2.test b/test/where2.test index 83740bffd..6045cb117 100644 --- a/test/where2.test +++ b/test/where2.test @@ -794,4 +794,41 @@ do_execsql_test where2-15.1 { ORDER BY a,EXISTS(SELECT 1 FROM t1 LEFT JOIN (SELECT x AS y FROM t2) AS s2 ON t1.b=s2.y),x; } {12 34 NULL | 56 78 78 | 90 12 12 |} +# Demonstrate that CROSS JOIN is a join reordering barrier. +# +reset_db +do_execsql_test where2-16.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b INT); + CREATE TABLE t2(c INTEGER PRIMARY KEY, d INT); + CREATE TABLE t3(e INTEGER PRIMARY KEY, f INT); + CREATE TABLE t4(g INTEGER PRIMARY KEY, h INT); +} + +# Here the query planner wants to move t4 forward so that it is in front of +# t1. Ensure that does not happen. +do_execsql_test where2-16.2 { + EXPLAIN QUERY PLAN + SELECT * + FROM t1, t2 CROSS JOIN t3, t4 + WHERE t4.g=1 + AND t1.a=t4.h + AND t2.c=t1.b + AND t3.e=t2.d; +} {~/.* t4 .* t[12] .*/} + +# In this case the planner wants to move t1 to come after t4. Ensure that +# does not happen. +do_execsql_test where2-16.2 { + EXPLAIN QUERY PLAN + SELECT * + FROM t1, t2 CROSS JOIN t3, t4 + WHERE t2.c=1 + AND t3.e=t2.d + AND t4.g=t3.f + AND t1.a=t4.h; +} {~/.* t[34] .* t1 .*/} + + + + finish_test diff --git a/test/whereK.test b/test/whereK.test index 060d470ff..995c08371 100644 --- a/test/whereK.test +++ b/test/whereK.test @@ -69,4 +69,17 @@ do_execsql_test 1.5eqp { ORDER BY +a; } {/SEARCH t1 USING INDEX t1bc/} +# https://sqlite.org/forum/forumpost/2026-01-16T11:35:28Z +do_execsql_test 2.1 { + DROP TABLE t1; + CREATE TABLE t0(x COLLATE NOCASE); + CREATE INDEX t0x ON t0(x); + CREATE TABLE t1(y); + INSERT INTO t0 VALUES('a'); + INSERT INTO t1 VALUES('AB'); + SELECT count(*) FROM t0, t1 WHERE (y BETWEEN 1 AND x) OR (x>=y AND x); + SELECT count(*) FROM t0, t1 WHERE (x>=y AND x) OR (y BETWEEN 1 AND x); +} {1 1} + + finish_test diff --git a/test/win32longpath.test b/test/win32longpath.test index 2545c55a1..43905f26e 100644 --- a/test/win32longpath.test +++ b/test/win32longpath.test @@ -115,11 +115,7 @@ do_test 1.6 { db3 close -# winrt platforms do not handle paths with unix-style '/' directory separators. -# set lUri [list 1a 1b 1c 1d 1e 1f] -ifcapable winrt { set lUri [list 1a 1c 1e] } - foreach tn $lUri { sqlite3 db3 $uri($tn) -vfs win32-longpath -uri 1 -translatefilename 0 diff --git a/test/with1.test b/test/with1.test index 5ddf9dce0..c87082583 100644 --- a/test/with1.test +++ b/test/with1.test @@ -1237,4 +1237,23 @@ do_execsql_test 27.1 { SELECT k, cte_map, main_map, '|' FROM log ORDER BY k; } {1 cte1 main1 | 2 cte2 main2 |} +# forum post https://sqlite.org/forum/forumpost/2026-03-04T05:06:26Z +# +db null NULL +do_execsql_test 28.1 { + DROP TABLE t1; + CREATE TABLE t1(x INTEGER PRIMARY KEY); + INSERT INTO t1 VALUES(1),(4),(999); + SELECT ( + WITH RECURSIVE t2(y) AS ( + SELECT 4 + UNION + SELECT NULL + UNION + SELECT y+1 FROM t2 WHERE y=4 ORDER BY 1 + ) + SELECT 1 FROM t2 WHERE y=x + ) FROM t1; +} {NULL 1 NULL} + finish_test diff --git a/test/with5.test b/test/with5.test index 430c5f2de..d5c89efcf 100644 --- a/test/with5.test +++ b/test/with5.test @@ -190,5 +190,26 @@ do_execsql_test 220 { SELECT x FROM closure ORDER BY +x; } {1 2 3 4 5 6 7 8 9 11 13} +#------------------------------------------------------------------------- +# Forum: https://sqlite.org/forum/forumpost/deb1eadf4d677bd5 +# +# For a recursive CTE, do not assume rows are delivered in the order +# specified by the ORDER BY clause. In this case ORDER BY governs the +# order in which the queue is processed, not the overall order in which +# rows are emitted by the CTE. +# +reset_db +do_execsql_test 300 { + CREATE TABLE tree(id INTEGER PRIMARY KEY, parent INTEGER); + INSERT INTO tree VALUES(3, 1); + INSERT INTO tree VALUES(2, 3); +} + +do_execsql_test 310 { + WITH RECURSIVE tt(ii) AS ( + VALUES(1) UNION ALL SELECT id FROM tree, tt WHERE parent=ii ORDER BY id + ) + SELECT * FROM tt ORDER BY ii; +} {1 2 3} finish_test diff --git a/test/zipfile2.test b/test/zipfile2.test index 8ee90d310..c1498872a 100644 --- a/test/zipfile2.test +++ b/test/zipfile2.test @@ -302,4 +302,19 @@ do_execsql_test 7.1 { SELECT length(name) FROM t1; } {60000} + +# https://sqlite.org/forum/forumpost/721a05d2c5 +# +if {[catch { load_static_extension db fileio }]==0} { + forcedelete test.zip + set fd [open test.zip wb] + fconfigure $fd -translation binary + puts -nonewline $fd [db one {SELECT X'504b0506000000000100010030000000160000000000504b01021400140000000000000000000000000000000000000000000100010000000000000000000000000000006100'}] + close $fd + + do_catchsql_test 8.0 { + SELECT name,sz FROM zipfile(readfile('test.zip')); + } {1 {failed to read LFH at offset 0}} +} + finish_test diff --git a/tool/build-all-msvc.bat b/tool/build-all-msvc.bat index 83d660deb..ae0d0e5b6 100755 --- a/tool/build-all-msvc.bat +++ b/tool/build-all-msvc.bat @@ -105,7 +105,6 @@ REM When set, these values are expanded and passed to the NMAKE command line, REM after its other arguments. These may be used to specify additional NMAKE REM options, for example: REM -REM SET NMAKE_ARGS=FOR_WINRT=1 REM SET NMAKE_ARGS_DEBUG=MEMDEBUG=1 REM SET NMAKE_ARGS_RETAIL=WIN32HEAP=1 REM @@ -861,4 +860,4 @@ GOTO no_errors GOTO end_of_file :end_of_file -%__ECHO% EXIT /B %ERRORLEVEL% +%__ECHO% EXIT /B %ERRORLEVEL% diff --git a/tool/dbtotxt.c b/tool/dbtotxt.c index fbd6e3d51..ed347840a 100644 --- a/tool/dbtotxt.c +++ b/tool/dbtotxt.c @@ -1,6 +1,12 @@ /* -** Copyright 2008 D. Richard Hipp and Hipp, Wyrick & Company, Inc. -** All Rights Reserved +** 2018-12-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** ****************************************************************************** ** diff --git a/tool/lemon.c b/tool/lemon.c index 324dda0c5..9071e7719 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -494,6 +494,7 @@ struct lemon { char *filename; /* Name of the input file */ char *outname; /* Name of the current output file */ char *tokenprefix; /* A prefix added to token names in the .h file */ + char *stackSizeLimit; /* Function to return the stack size limit */ char *reallocFunc; /* Function to use to allocate stack space */ char *freeFunc; /* Function to use to free stack space */ int nconflict; /* Number of parsing conflicts */ @@ -2638,6 +2639,9 @@ static void parseonetoken(struct pstate *psp) }else if( strcmp(x,"default_type")==0 ){ psp->declargslot = &(psp->gp->vartype); psp->insertLineMacro = 0; + }else if( strcmp(x,"stack_size_limit")==0 ){ + psp->declargslot = &(psp->gp->stackSizeLimit); + psp->insertLineMacro = 0; }else if( strcmp(x,"realloc")==0 ){ psp->declargslot = &(psp->gp->reallocFunc); psp->insertLineMacro = 0; @@ -3715,7 +3719,7 @@ PRIVATE int compute_action(struct lemon *lemp, struct action *ap) return act; } -#define LINESIZE 1000 +#define LINESIZE 10000 /* The next cluster of routines are for reading the template file ** and writing the results to the generated parser */ /* The first function transfers data from "in" to "out" until @@ -3750,12 +3754,9 @@ PRIVATE void tplt_xfer(char *name, FILE *in, FILE *out, int *lineno) /* Skip forward past the header of the template file to the first "%%" */ -PRIVATE void tplt_skip_header(FILE *in, int *lineno) -{ +PRIVATE void tplt_skip_header(FILE *in){ char line[LINESIZE]; - while( fgets(line,LINESIZE,in) && (line[0]!='%' || line[1]!='%') ){ - (*lineno)++; - } + while( fgets(line,LINESIZE,in) && (line[0]!='%' || line[1]!='%') ){} } /* The next function finds the template file and opens it, returning @@ -3825,12 +3826,14 @@ PRIVATE void tplt_linedir(FILE *out, int lineno, char *filename) filename++; } fprintf(out,"\"\n"); + fflush(out); } /* Print a string to the file and keep the linenumber up to date */ PRIVATE void tplt_print(FILE *out, struct lemon *lemp, char *str, int *lineno) { if( str==0 ) return; + fflush(out); while( *str ){ putc(*str,out); if( *str=='\n' ) (*lineno)++; @@ -3843,6 +3846,7 @@ PRIVATE void tplt_print(FILE *out, struct lemon *lemp, char *str, int *lineno) if (!lemp->nolinenosflag) { (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); } + fflush(out); return; } @@ -4407,6 +4411,13 @@ static void writeRuleText(FILE *out, struct rule *rp){ } } +/* +** Return true if the string is not NULL and not empty. +*/ +static int notnull(const char *z){ + return z && z[0]; +} + /* Generate C source code for the parser */ void ReportTable( @@ -4414,7 +4425,7 @@ void ReportTable( int mhflag, /* Output in makeheaders format if true */ int sqlFlag /* Generate the *.sql file too */ ){ - FILE *out, *in, *sql; + FILE *out, *in; int lineno; struct state *stp; struct action *ap; @@ -4439,18 +4450,10 @@ void ReportTable( in = tplt_open(lemp); if( in==0 ) return; - out = file_open(lemp,".c","wb"); - if( out==0 ){ - fclose(in); - return; - } - if( sqlFlag==0 ){ - sql = 0; - }else{ - sql = file_open(lemp, ".sql", "wb"); + if( sqlFlag ){ + FILE *sql = file_open(lemp, ".sql", "wb"); if( sql==0 ){ fclose(in); - fclose(out); return; } fprintf(sql, @@ -4515,6 +4518,12 @@ void ReportTable( } } fprintf(sql, "COMMIT;\n"); + fclose(sql); + } + out = file_open(lemp,".c","wb"); + if( out==0 ){ + fclose(in); + return; } lineno = 1; @@ -4543,7 +4552,7 @@ void ReportTable( } } if( lemp->include[0]=='/' ){ - tplt_skip_header(in,&lineno); + tplt_skip_header(in); }else{ tplt_xfer(lemp->name,in,out,&lineno); } @@ -4563,7 +4572,7 @@ void ReportTable( if( mhflag ){ fprintf(out,"#if INTERFACE\n"); lineno++; }else{ - fprintf(out,"#ifndef %s%s\n", prefix, lemp->symbols[1]->name); + fprintf(out,"#ifndef %s%s\n", prefix, lemp->symbols[1]->name); lineno++; } for(i=1; i<lemp->nterminal; i++){ fprintf(out,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i); @@ -4612,25 +4621,33 @@ void ReportTable( fprintf(out,"#define %sARG_FETCH\n",name); lineno++; fprintf(out,"#define %sARG_STORE\n",name); lineno++; } + fprintf(out, "#undef YYREALLOC\n"); lineno++; if( lemp->reallocFunc ){ fprintf(out,"#define YYREALLOC %s\n", lemp->reallocFunc); lineno++; }else{ fprintf(out,"#define YYREALLOC realloc\n"); lineno++; } + fprintf(out, "#undef YYFREE\n"); lineno++; if( lemp->freeFunc ){ fprintf(out,"#define YYFREE %s\n", lemp->freeFunc); lineno++; }else{ fprintf(out,"#define YYFREE free\n"); lineno++; } + fprintf(out, "#undef YYDYNSTACK\n"); lineno++; if( lemp->reallocFunc && lemp->freeFunc ){ fprintf(out,"#define YYDYNSTACK 1\n"); lineno++; }else{ fprintf(out,"#define YYDYNSTACK 0\n"); lineno++; } - if( lemp->ctx && lemp->ctx[0] ){ + fprintf(out, "#undef YYSIZELIMIT\n"); lineno++; + if( notnull(lemp->ctx) ){ i = lemonStrlen(lemp->ctx); while( i>=1 && ISSPACE(lemp->ctx[i-1]) ) i--; while( i>=1 && (ISALNUM(lemp->ctx[i-1]) || lemp->ctx[i-1]=='_') ) i--; + if( notnull(lemp->stackSizeLimit) ){ + fprintf(out,"#define YYSIZELIMIT %s\n", lemp->stackSizeLimit); lineno++; + } + fprintf(out,"#define %sCTX(P) ((P)->%s)\n",name,&lemp->ctx[i]); lineno++; fprintf(out,"#define %sCTX_SDECL %s;\n",name,lemp->ctx); lineno++; fprintf(out,"#define %sCTX_PDECL ,%s\n",name,lemp->ctx); lineno++; fprintf(out,"#define %sCTX_PARAM ,%s\n",name,&lemp->ctx[i]); lineno++; @@ -4639,6 +4656,7 @@ void ReportTable( fprintf(out,"#define %sCTX_STORE yypParser->%s=%s;\n", name,&lemp->ctx[i],&lemp->ctx[i]); lineno++; }else{ + fprintf(out,"#define %sCTX(P) 0\n",name); lineno++; fprintf(out,"#define %sCTX_SDECL\n",name); lineno++; fprintf(out,"#define %sCTX_PDECL\n",name); lineno++; fprintf(out,"#define %sCTX_PARAM\n",name); lineno++; @@ -4648,10 +4666,13 @@ void ReportTable( if( mhflag ){ fprintf(out,"#endif\n"); lineno++; } + fprintf(out, "#undef YYERRORSYMBOL\n"); lineno++; + fprintf(out, "#undef YYERRSYMDT\n"); lineno++; if( lemp->errsym && lemp->errsym->useCnt ){ fprintf(out,"#define YYERRORSYMBOL %d\n",lemp->errsym->index); lineno++; fprintf(out,"#define YYERRSYMDT yy%d\n",lemp->errsym->dtnum); lineno++; } + fprintf(out,"#undef YYFALLBACK\n"); lineno++; if( lemp->has_fallback ){ fprintf(out,"#define YYFALLBACK 1\n"); lineno++; } @@ -5003,7 +5024,6 @@ void ReportTable( sp2->destLineno = -1; /* Avoid emitting this destructor again */ } } - emit_destructor_code(out,lemp->symbols[i],lemp,&lineno); fprintf(out," break;\n"); lineno++; } @@ -5103,7 +5123,6 @@ void ReportTable( acttab_free(pActtab); fclose(in); fclose(out); - if( sql ) fclose(sql); return; } diff --git a/tool/lempar.c b/tool/lempar.c index 74314efea..7b654e650 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -299,15 +299,24 @@ static int yyGrowStack(yyParser *p){ int newSize; int idx; yyStackEntry *pNew; +#ifdef YYSIZELIMIT + int nLimit = YYSIZELIMIT(ParseCTX(p)); +#endif newSize = oldSize*2 + 100; +#ifdef YYSIZELIMIT + if( newSize>nLimit ){ + newSize = nLimit; + if( newSize<=oldSize ) return 1; + } +#endif idx = (int)(p->yytos - p->yystack); if( p->yystack==p->yystk0 ){ - pNew = YYREALLOC(0, newSize*sizeof(pNew[0])); + pNew = YYREALLOC(0, newSize*sizeof(pNew[0]), ParseCTX(p)); if( pNew==0 ) return 1; memcpy(pNew, p->yystack, oldSize*sizeof(pNew[0])); }else{ - pNew = YYREALLOC(p->yystack, newSize*sizeof(pNew[0])); + pNew = YYREALLOC(p->yystack, newSize*sizeof(pNew[0]), ParseCTX(p)); if( pNew==0 ) return 1; } p->yystack = pNew; @@ -459,7 +468,9 @@ void ParseFinalize(void *p){ } #if YYGROWABLESTACK - if( pParser->yystack!=pParser->yystk0 ) YYFREE(pParser->yystack); + if( pParser->yystack!=pParser->yystk0 ){ + YYFREE(pParser->yystack, ParseCTX(pParser)); + } #endif } diff --git a/tool/mkautoconfamal.sh b/tool/mkautoconfamal.sh index 3835799a6..002a3b8ee 100644 --- a/tool/mkautoconfamal.sh +++ b/tool/mkautoconfamal.sh @@ -59,7 +59,8 @@ cp $TOP/src/sqlite3.rc $TMPSPACE cp $TOP/tool/Replace.cs $TMPSPACE cp $TOP/VERSION $TMPSPACE cp $TOP/main.mk $TMPSPACE - +cp $TOP/make.bat $TMPSPACE +tree $TMPSPACE cd $TMPSPACE #if true; then diff --git a/tool/mkcombo.tcl b/tool/mkcombo.tcl new file mode 100644 index 000000000..71368ec41 --- /dev/null +++ b/tool/mkcombo.tcl @@ -0,0 +1,106 @@ +#!/usr/bin/tclsh +# +# Use this script to combine multiple source code files into a single +# file. Example: +# +# tclsh mkcombo.tcl file1.c file2.c file3.c -o file123.c +# + +set help {Usage: tclsh mkcombo.tcl [OPTIONS] [FILELIST] + where OPTIONS is zero or more of the following with these effects: + --linemacros=? => Emit #line directives into output or not. (? = 1 or 0) + --o FILE => write to alternative output file named FILE + --help => See this. +} + +set linemacros 0 +set fname {} +set src [list] + + +for {set i 0} {$i<[llength $argv]} {incr i} { + set x [lindex $argv $i] + if {[regexp {^-?-linemacros(?:=([01]))?$} $x ma ulm]} { + if {$ulm == ""} {set ulm 1} + set linemacros $ulm + } elseif {[regexp {^-o$} $x]} { + incr i + if {$i==[llength $argv]} { + error "No argument following $x" + } + set fname [lindex $argv $i] + } elseif {[regexp {^-?-((help)|\?)$} $x]} { + puts $help + exit 0 + } elseif {[regexp {^-?-} $x]} { + error "unknown command-line option: $x" + } else { + lappend src $x + } +} + +# Open the output file and write a header comment at the beginning +# of the file. +# +if {![info exists fname]} { + set fname sqlite3.c + if {$enable_recover} { set fname sqlite3r.c } +} +set out [open $fname wb] + +# Return a string consisting of N "*" characters. +# +proc star N { + set r {} + for {set i 0} {$i<$N} {incr i} {append r *} + return $r +} + +# Force the output to use unix line endings, even on Windows. +fconfigure $out -translation binary +puts $out "/[star 78]" +puts $out {** The following is an amalgamation of these source code files:} +puts $out {**} +foreach s $src { + regsub {^.*/(src|ext)/} $s {\1/} s2 + puts $out "** $s2" +} +puts $out {**} +puts $out "[star 78]/" + +# Insert a comment into the code +# +proc section_comment {text} { + global out s78 + set n [string length $text] + set nstar [expr {60 - $n}] + puts $out "/************** $text [star $nstar]/" +} + +# Read the source file named $filename and write it into the +# sqlite3.c output file. The only transformation is the trimming +# of EOL whitespace. +# +proc copy_file_verbatim {filename} { + global out + set in [open $filename rb] + set tail [file tail $filename] + section_comment "Begin file $tail" + while {![eof $in]} { + set line [string trimright [gets $in]] + puts $out $line + } + section_comment "End of $tail" +} +set taillist "" +foreach file $src { + copy_file_verbatim $file + append taillist ", [file tail $file]" +} + +set taillist "End of the amalgamation of [string range $taillist 2 end]" +set n [string length $taillist] +set ns [expr {(75-$n)/2}] +if {$ns<3} {set ns 3} +puts $out "/[star $ns] $taillist [star $ns]/" +close $out diff --git a/tool/mkctimec.tcl b/tool/mkctimec.tcl index 64d4a121a..66caacb60 100755 --- a/tool/mkctimec.tcl +++ b/tool/mkctimec.tcl @@ -276,6 +276,7 @@ set boolean_defnil_options { SQLITE_SECURE_DELETE SQLITE_SMALL_STACK SQLITE_SOUNDEX + SQLITE_STRICT_SUBTYPE SQLITE_SUBSTR_COMPATIBILITY SQLITE_TCL SQLITE_TEST diff --git a/tool/mkfptab.c b/tool/mkfptab.c new file mode 100644 index 000000000..e40e04b00 --- /dev/null +++ b/tool/mkfptab.c @@ -0,0 +1,238 @@ +/* +** This program generates C code for the tables used to generate powers +** of 10 in the powerOfTen() subroutine in util.c. +** +** The objective of the powerOfTen() subroutine is to provide the most +** significant 96 bits of any power of 10 between -348 and +347. Rather +** than generate a massive 8K table, three much smaller tables are constructed, +** which can then generate the requested power of 10 using a single +** 160-bit multiple. +** +** This program works by internally generating a table of powers of +** 10 accurate to 256 bits each. It then that full-sized, high-accuracy +** table to construct the three smaller tables needed by powerOfTen(). +** +** LIMITATION: +** +** This program uses the __uint128_t datatype, available in gcc/clang. +** It won't build using other compilers. +*/ +#include <stdint.h> +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include <assert.h> + +typedef unsigned __int128 u128; /* 128-bit unsigned integer */ +typedef unsigned long long int u64; /* 64-bit unsigned integer */ + +/* There is no native 256-bit unsigned integer type, so synthesize one +** using four 64-bit unsigned integers. Must significant first. */ +struct u256 { + u64 a[4]; /* big-endian */ +}; +typedef struct u256 u256; + +/* +** Return a u64 with the N-th bit set. +*/ +#define U64_BIT(N) (((u64)1)<<(N)) + +/* Multiple *pX by 10, in-place */ +static void u256_times_10(u256 *pX){ + u64 carry = 0; + int i; + for(i=3; i>=0; i--){ + u128 y = (10 * (u128)pX->a[i]) + carry; + pX->a[i] = (u64)y; + carry = (u64)(y>>64); + } +} + +/* Multiple *pX by 2, in-place. AKA, left-shift */ +static void u256_times_2(u256 *pX){ + u64 carry = 0; + int i; + for(i=3; i>=0; i--){ + u64 y = pX->a[i]; + pX->a[i] = (y<<1) | carry; + carry = y>>63; + } +} + +/* Divide *pX by 10, in-place */ +static void u256_div_10(u256 *pX){ + u64 rem = 0; + int i; + for(i=0; i<4; i++){ + u128 acc = (((u128)rem)<<64) | pX->a[i]; + pX->a[i] = acc/10; + rem = (u64)(acc%10); + } +} + +/* Divide *pX by 2, in-place, AKA, right-shift */ +static void u256_div_2(u256 *pX){ + u64 rem = 0; + int i; + for(i=0; i<4; i++){ + u64 y = pX->a[i]; + pX->a[i] = (y>>1) | (rem<<63); + rem = y&1; + } +} + +/* Note: The main table is a little larger on the low end than the required +** range of -348..+347, since we need the -351 value for the reduced tables. +*/ +#define SCALE_FIRST (-351) /* Least power-of-10 */ +#define SCALE_LAST (+347) /* Largest power-of-10 */ +#define SCALE_COUNT (SCALE_LAST - SCALE_FIRST + 1) /* Size of main table */ +#define SCALE_ZERO (351) + +int main(int argc, char **argv){ + int i, j, iNext; + int e; + int bRound = 0; + int bTruth = 0; + const u64 top = ((u64)1)<<63; + u256 v; + u64 aHi[SCALE_COUNT]; + u64 aLo[SCALE_COUNT]; + int aE[SCALE_COUNT]; + + for(i=1; i<argc; i++){ + const char *z = argv[i]; + if( z[0]=='-' && z[1]=='-' && z[2]!=0 ) z++; + if( strcmp(z,"-round")==0 ){ + bRound = 1; + }else + if( strcmp(z,"-truth")==0 ){ + bTruth = 1; + }else + { + fprintf(stderr, "unknown option: \"%s\"\n", argv[i]); + exit(1); + } + } + + /* Generate the master 256-bit power-of-10 table */ + v.a[0] = top; + v.a[1] = 0; + v.a[2] = 0; + v.a[3] = 0; + for(i=SCALE_ZERO, e=63; i>=0; i--){ + aHi[i] = v.a[0]; + aLo[i] = v.a[1]; + aE[i] = e; + u256_div_10(&v); + while( v.a[0] < top ){ + e++; + u256_times_2(&v); + } + } + v.a[0] = 0; + v.a[1] = top; + v.a[2] = 0; + v.a[3] = 0; + for(i=SCALE_ZERO+1, e=63; i<SCALE_COUNT; i++){ + u256_times_10(&v); + while( v.a[0]>0 ){ + e--; + u256_div_2(&v); + } + aHi[i] = v.a[1]; + aLo[i] = v.a[2]; + aE[i] = e; + } + + if( bTruth ){ + /* With the --truth flag, also output the aTruth[] table that + ** contains 128 bits of every power-of-two in the range */ + printf(" /* Powers of ten, accurate to 128 bits each */\n"); + printf(" static const struct {u64 hi; u64 lo;} aTruth[] = {\n"); + for(i=0; i<SCALE_COUNT; i++){ + u64 x = aHi[i]; + u64 y = aLo[i]; + int e = aE[i]; + char *zOp; + if( e>0 ){ + zOp = "<<"; + }else{ + e = -e; + zOp = ">>"; + } + printf(" {0x%016llx, 0x%016llx}, /* %2d: 1.0e%+d %s %d */\n", + x, y, i, i+SCALE_FIRST, zOp, e); + } + printf(" };\n"); + } + + /* The aBase[] table contains powers of 10 between 0 and 26. These + ** all fit in a single 64-bit integer. + */ + printf(" static const u64 aBase[] = {\n"); + for(i=SCALE_ZERO, j=0; i<SCALE_ZERO+27; i++, j++){ + u64 x = aHi[i]; + int e = aE[i]; + char *zOp; + if( e>0 ){ + zOp = "<<"; + }else{ + e = -e; + zOp = ">>"; + } + printf(" UINT64_C(0x%016llx), /* %2d: 1.0e%+d %s %d */\n", + x, j, i+SCALE_FIRST, zOp, e); + } + printf(" };\n"); + + /* For powers of 10 outside the range [0..26], we have to multiple + ** on of the aBase[] entries by a scaling factor to get the true + ** power of ten. The scaling factors are all approximates accurate + ** to 96 bytes, represented by a 64-bit integer in aScale[] for the + ** most significant bits and a 32-bit integer in aScaleLo[] for the + ** next 32 bites. + ** + ** The scale factors are at increments of 27. Except, the entry for 0 + ** is replaced by the -1 value as a special case. + */ + printf(" static const u64 aScale[] = {\n"); + for(i=j=0; i<SCALE_COUNT; i=iNext, j++){ + const char *zExtra = ""; + iNext = i+27; + if( i==SCALE_ZERO ){ i--; zExtra = " (special case)"; } + u64 x = aHi[i]; + int e = aE[i]; + char *zOp; + if( e>0 ){ + zOp = "<<"; + }else{ + e = -e; + zOp = ">>"; + } + printf(" UINT64_C(0x%016llx), /* %2d: 1.0e%+d %s %d%s */\n", + x, j, i+SCALE_FIRST, zOp, e, zExtra); + } + printf(" };\n"); + printf(" static const unsigned int aScaleLo[] = {\n"); + for(i=j=0; i<SCALE_COUNT; i=iNext, j++){ + const char *zExtra = ""; + iNext = i+27; + if( i==SCALE_ZERO ){ i--; zExtra = " (special case)"; } + u64 x = aLo[i]; + int e = aE[i]; + char *zOp; + if( bRound && (x & U64_BIT(31))!=0 && i!=SCALE_ZERO-1 ) x += U64_BIT(32); + if( e>0 ){ + zOp = "<<"; + }else{ + e = -e; + zOp = ">>"; + } + printf(" 0x%08llx, /* %2d: 1.0e%+d %s %d%s */\n", + x>>32, j, i+SCALE_FIRST, zOp, e, zExtra); + } + printf(" };\n"); + return 0; +} diff --git a/tool/mkkeywordhash.c b/tool/mkkeywordhash.c index 188c0a29a..d6d54a5a4 100644 --- a/tool/mkkeywordhash.c +++ b/tool/mkkeywordhash.c @@ -668,8 +668,8 @@ int main(int argc, char **argv){ printf("/* Check to see if z[0..n-1] is a keyword. If it is, write the\n"); printf("** parser symbol code for that keyword into *pType. Always\n"); printf("** return the integer n (the length of the token). */\n"); - printf("static int keywordCode(const char *z, int n, int *pType){\n"); - printf(" int i, j;\n"); + printf("static i64 keywordCode(const char *z, i64 n, int *pType){\n"); + printf(" i64 i, j;\n"); printf(" const char *zKW;\n"); printf(" assert( n>=2 );\n"); printf(" i = ((charMap(z[0])*%d) %c", HASH_C0, HASH_CC); diff --git a/tool/mkshellc.tcl b/tool/mkshellc.tcl index 2f7a6ea25..45452621b 100644 --- a/tool/mkshellc.tcl +++ b/tool/mkshellc.tcl @@ -13,28 +13,90 @@ set topdir [file dir [file dir [file normal $argv0]]] set out stdout fconfigure stdout -translation binary if {[lindex $argv 0]!=""} { - set out [open [lindex $argv 0] wb] + set output_file [lindex $argv 0] + file delete -force $output_file + set out [open $output_file wb] +} else { + set output_file {} } -puts $out {/* DO NOT EDIT! -** This file is automatically generated by the script in the canonical -** SQLite source tree at tool/mkshellc.tcl. That script combines source -** code from various constituent source files of SQLite into this single -** "shell.c" file used to implement the SQLite command-line shell. -** -** Most of the code found below comes from the "src/shell.c.in" file in -** the canonical SQLite source tree. That main file contains "INCLUDE" -** lines that specify other files in the canonical source tree that are -** inserted to getnerate this complete program source file. -** -** The code from multiple files is combined into this single "shell.c" -** source file to help make the command-line program easier to compile. -** -** To modify this program, get a copy of the canonical SQLite source tree, -** edit the src/shell.c.in" and/or some of the other files that are included -** by "src/shell.c.in", then rerun the tool/mkshellc.tcl script. -*/} + +############################## FIRST PASS ################################ +# Read through the shell.c.in source file to gather information. Do not +# yet generate any code +# set in [open $topdir/src/shell.c.in] fconfigure $in -translation binary +set allSource(src/shell.c.in) 1 +set inUsage 0 +set dotcmd {} +while {1} { + set lx [gets $in] + if {[eof $in]} break; + if {[regexp {^INCLUDE } $lx]} { + set cfile [lindex $lx 1] + if {[string match ../* $cfile]} { + set xfile [string range $cfile 3 end] + } else { + set xfile "src/$cfile" + } + set allSource($xfile) 1 + } elseif {[regexp {^\*\* USAGE:\s+([^\s]+)} $lx all dotcmd]} { + set inUsage 1 + set details [string trim [string range $lx 2 end]] + } elseif {$inUsage} { + if {![regexp {^\*\*} $lx] || [regexp { DOT-COMMAND: } $lx]} { + set inUsage 0 + set Usage($dotcmd) [string trim $details] + } else { + append details \n + append details [string range [string trimright $lx] 3 end] + } + } +} + +# Generate dot-command usage text based on the data accumulated in +# the Usage() array. +# +proc generate_usage {out} { + global Usage + puts $out "/**************************************************************" + puts $out "** \"Usage\" help text automatically generated from comments */" + puts $out "static const struct \173" + puts $out " const char *zCmd; /* Name of the dot-command */" + puts $out " const char *zUsage; /* Documentation */" + puts $out "\175 aUsage\[\] = \173" + foreach dotcmd [array names Usage] { + puts $out " \173 \"$dotcmd\"," + foreach line [split $Usage($dotcmd) \n] { + set x [string map [list \\ \\\\ \" \\\"] $line] + puts $out "\"$x\\n\"" + } + puts $out " \175," + } + puts $out "\175;" +} +# generate_usage stderr + +###### SECOND PASS ####### +# Make a second pass through shell.c.in to generate the the final +# output, based on data gathered during the first pass. +# + +puts $out {/* +** This is the amalgamated source code to the "sqlite3" or "sqlite3.exe" +** command-line shell (CLI) for SQLite. This file is automatically +** generated by the tool/mkshellc.tcl script from the following sources: +**} +foreach fn [lsort [array names allSource]] { + puts $out "** $fn" +} +puts $out {** +** To modify this program, get a copy of the canonical SQLite source tree, +** edit the src/shell.c.in file and/or some of the other files that are +** listed above, then rerun the command "make shell.c". +*/} +seek $in 0 start +puts $out "/************************* Begin src/shell.c.in ******************/" proc omit_redundant_typedefs {line} { global typedef_seen if {[regexp {^typedef .* ([a-zA-Z0-9_]+);} $line all typename]} { @@ -53,9 +115,14 @@ while {1} { incr iLine if {[regexp {^INCLUDE } $lx]} { set cfile [lindex $lx 1] - puts $out "/************************* Begin $cfile ******************/" -# puts $out "#line 1 \"$cfile\"" - set in2 [open $topdir/src/$cfile] + if {[string match ../* $cfile]} { + set xfile [string range $cfile 3 end] + } else { + set xfile "src/$cfile" + } + puts $out "/************************* Begin $xfile ******************/" +# puts $out "#line 1 \"$xfile\"" + set in2 [open $topdir/$xfile] fconfigure $in2 -translation binary while {![eof $in2]} { set lx [omit_redundant_typedefs [gets $in2]] @@ -69,11 +136,14 @@ while {1} { puts $out $lx } close $in2 - puts $out "/************************* End $cfile ********************/" + puts $out "/************************* End $xfile ********************/" # puts $out "#line [expr $iLine+1] \"shell.c.in\"" - continue + } elseif {[regexp {^INSERT-USAGE-TEXT-HERE} $lx]} { + generate_usage $out + } else { + puts $out $lx } - puts $out $lx } +puts $out "/************************* End src/shell.c.in ******************/" close $in close $out diff --git a/tool/omittest.tcl b/tool/omittest.tcl index 0452a4c6f..03c9220cd 100644 --- a/tool/omittest.tcl +++ b/tool/omittest.tcl @@ -123,7 +123,6 @@ set CompileOptionsToTest { SQLITE_ENABLE_MEMSYS SQLITE_ENABLE_MODULE_COMMENTS SQLITE_ENABLE_MULTIPLEX - SQLITE_ENABLE_MULTITHREADED_CHECKS SQLITE_ENABLE_NORMALIZE SQLITE_ENABLE_NULL_TRIM SQLITE_ENABLE_OFFSET_SQL_FUNC @@ -152,6 +151,7 @@ set CompileOptionsToTest { SQLITE_ENABLE_VFSTRACE SQLITE_ENABLE_WHERETRACE SQLITE_ENABLE_ZIPVFS + SQLITE_THREAD_MISUSE_WARNINGS } # Parse command-line options. diff --git a/tool/showdb.c b/tool/showdb.c index f0bd9737c..84813b839 100644 --- a/tool/showdb.c +++ b/tool/showdb.c @@ -8,6 +8,7 @@ #include <sys/types.h> #include <sys/stat.h> #include <fcntl.h> +#include <stdint.h> #if !defined(_MSC_VER) #include <unistd.h> @@ -28,13 +29,21 @@ typedef sqlite3_uint64 u64; /* unsigned 64-bit */ static struct GlobalData { i64 pagesize; /* Size of a database page */ + i64 usablesize; /* pagesize-nRes */ int dbfd; /* File descriptor for reading the DB */ u32 mxPage; /* Last page number */ + u32 nRes; /* Amount of reserve space */ int perLine; /* HEX elements to print per line */ int bRaw; /* True to access db file via OS APIs */ + int bCSV; /* CSV output for "pgidx" */ + int bTmstmp; /* Interpret tmstmpvfs tags on "pgidx" */ sqlite3_file *pFd; /* File descriptor for non-raw mode */ sqlite3 *pDb; /* Database handle that owns pFd */ -} g = {1024, -1, 0, 16, 0, 0, 0}; + char **zPageUse; /* Use for each page */ + struct TmstmpTag { + unsigned char a[16]; /* tmstmpvfs tag for each page */ + } *aPageTag; +} g = {4096, 4096, -1, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0}; /* ** Convert the var-int format into i64. Return the number of bytes @@ -142,11 +151,12 @@ static void fileClose(){ static unsigned char *fileRead(sqlite3_int64 ofst, int nByte){ unsigned char *aData; int got; + int rc; aData = sqlite3_malloc64(32+(i64)nByte); if( aData==0 ) out_of_memory(); memset(aData, 0, nByte+32); if( g.bRaw==0 ){ - int rc = g.pFd->pMethods->xRead(g.pFd, (void*)aData, nByte, ofst); + rc = g.pFd->pMethods->xRead(g.pFd, (void*)aData, nByte, ofst); if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ fprintf(stderr, "error in xRead() - %d\n", rc); exit(1); @@ -154,7 +164,21 @@ static unsigned char *fileRead(sqlite3_int64 ofst, int nByte){ }else{ lseek(g.dbfd, (long)ofst, SEEK_SET); got = read(g.dbfd, aData, nByte); - if( got>0 && got<nByte ) memset(aData+got, 0, nByte-got); + if( got==nByte ){ + rc = SQLITE_OK; + }else if( got>0 && got<nByte ){ + memset(aData+got, 0, nByte-got); + rc = SQLITE_IOERR_SHORT_READ; + }else{ + memset(aData,0,nByte); + rc = SQLITE_IOERR; + } + } + if( g.aPageTag && nByte==(int)g.pagesize ){ + unsigned int pgno = (unsigned int)(ofst/g.pagesize) + 1; + if( pgno>0 && pgno<=g.mxPage ){ + memcpy(g.aPageTag[pgno].a, &aData[nByte-16], 16); + } } return aData; } @@ -382,14 +406,14 @@ static i64 localPayload(i64 nPayload, char cType){ i64 nLocal; if( cType==13 ){ /* Table leaf */ - maxLocal = g.pagesize-35; - minLocal = (g.pagesize-12)*32/255-23; + maxLocal = g.usablesize-35; + minLocal = (g.usablesize-12)*32/255-23; }else{ - maxLocal = (g.pagesize-12)*64/255-23; - minLocal = (g.pagesize-12)*32/255-23; + maxLocal = (g.usablesize-12)*64/255-23; + minLocal = (g.usablesize-12)*32/255-23; } if( nPayload>maxLocal ){ - surplus = minLocal + (nPayload-minLocal)%(g.pagesize-4); + surplus = minLocal + (nPayload-minLocal)%(g.usablesize-4); if( surplus<=maxLocal ){ nLocal = surplus; }else{ @@ -751,7 +775,7 @@ static void decode_trunk_page( print_decode_line(a, 4, 4, "Number of entries on this page"); if( detail ){ n = decodeInt32(&a[4]); - for(i=0; i<n && i<g.pagesize/4; i++){ + for(i=0; i<n && i<g.usablesize/4; i++){ u32 x = decodeInt32(&a[8+4*i]); char zIdx[13]; sprintf(zIdx, "[%d]", i); @@ -769,11 +793,6 @@ static void decode_trunk_page( } } -/* -** A short text comment on the use of each page. -*/ -static char **zPageUse; - /* ** Add a comment on the use of a page. */ @@ -790,13 +809,13 @@ static void page_usage_msg(u32 pgno, const char *zFormat, ...){ sqlite3_free(zMsg); return; } - if( zPageUse[pgno]!=0 ){ + if( g.zPageUse[pgno]!=0 ){ printf("ERROR: page %d used multiple times:\n", pgno); - printf("ERROR: previous: %s\n", zPageUse[pgno]); + printf("ERROR: previous: %s\n", g.zPageUse[pgno]); printf("ERROR: current: %s\n", zMsg); - sqlite3_free(zPageUse[pgno]); + sqlite3_free(g.zPageUse[pgno]); } - zPageUse[pgno] = zMsg; + g.zPageUse[pgno] = zMsg; } /* @@ -837,7 +856,7 @@ static void page_usage_cell( while( ovfl && (cnt++)<g.mxPage ){ page_usage_msg(ovfl, "overflow %d from cell %d of page %u", cnt, cellno, pgno); - a = fileRead((ovfl-1)*(sqlite3_int64)g.pagesize, 4); + a = fileRead((ovfl-1)*(sqlite3_int64)g.pagesize, g.pagesize); ovfl = decodeInt32(a); sqlite3_free(a); } @@ -916,12 +935,12 @@ static void page_usage_btree( u32 ofst; cellidx = cellstart + i*2; - if( cellidx+1 >= g.pagesize ){ + if( cellidx+1 >= g.usablesize ){ printf("ERROR: page %d too many cells (%d)\n", pgno, nCell); break; } ofst = a[cellidx]*256 + a[cellidx+1]; - if( ofst<cellidx+2 || ofst+4>=g.pagesize ){ + if( ofst<cellidx+2 || ofst+4>=g.usablesize ){ printf("ERROR: page %d cell %d out of bounds\n", pgno, i); continue; } @@ -959,9 +978,9 @@ static void page_usage_freelist(u32 pgno){ a = fileRead((pgno-1)*g.pagesize, g.pagesize); iNext = decodeInt32(a); n = decodeInt32(a+4); - if( n>(g.pagesize - 8)/4 ){ + if( n>(g.usablesize - 8)/4 ){ printf("ERROR: page %d too many freelist entries (%d)\n", pgno, n); - n = (g.pagesize - 8)/4; + n = (g.usablesize - 8)/4; } for(i=0; i<n; i++){ int child = decodeInt32(a + (i*4+8)); @@ -990,6 +1009,63 @@ static void page_usage_ptrmap(u8 *a){ } } +/* +** The six bytes at a[] are a big-endian unsigned integer which is the +** number of milliseconds since 1970. Decode that value into an ISO 8601 +** date/time string stored in static space and return a pointer to that +** string. +*/ +static const char *decodeTimestamp(const unsigned char *a){ + uint64_t ms; /* Milliseconds since 1970 */ + uint64_t days; /* Days since 1970-01-01 */ + uint64_t sod; /* Start of date specified by ms */ + uint64_t z; /* Days since 0000-03-01 */ + uint64_t era; /* 400-year era */ + int i; /* Loop counter */ + int h; /* hour */ + int m; /* minute */ + int s; /* second */ + int f; /* millisecond */ + int Y; /* year */ + int M; /* month */ + int D; /* day */ + int y; /* year assuming March is first month */ + unsigned int doe; /* day of 400-year era */ + unsigned int yoe; /* year of 400-year era */ + unsigned int doy; /* day of year */ + unsigned int mp; /* month with March==0 */ + static char zOut[50]; /* Return results here */ + + for(ms=0, i=0; i<=5; i++) ms = (ms<<8) + a[i]; + if( ms==0 ){ + return " "; + }else if( ms>4102444800000LL ){ /* 2100-01-01 */ + /* YYYY-MM-DD HH:MM:SS.SSS */ + return " (bad date) "; + } + days = ms/86400000; + sod = (ms%86400000)/1000; + f = (int)(ms%1000); + + h = sod/3600; + m = (sod%3600)/60; + s = sod%60; + z = days + 719468; + era = z/146097; + doe = (unsigned)(z - era*146097); + yoe = (doe - doe/1460 + doe/36524 - doe/146096)/365; + y = (int)yoe + era*400; + doy = doe - (365*yoe + yoe/4 - yoe/100); + mp = (5*doy + 2)/153; + D = doy - (153*mp + 2)/5 + 1; + M = mp + (mp<10 ? 3 : -9); + Y = y + (M <=2); + snprintf(zOut, sizeof(zOut), + "%04d-%02d-%02d %02d:%02d:%02d.%03d", + Y, M, D, h, m, s, f); + return zOut; +} + /* ** Try to figure out how every page in the database file is being used. */ @@ -1010,14 +1086,22 @@ static void page_usage_report(const char *zPrg, const char *zDbName){ /* Open the database file */ db = openDatabase(zPrg, zDbName); - /* Set up global variables zPageUse[] and g.mxPage to record page + /* Set up global variables g.zPageUse[] and g.mxPage to record page ** usages */ - zPageUse = sqlite3_malloc64( sizeof(zPageUse[0])*(g.mxPage+1) ); - if( zPageUse==0 ) out_of_memory(); - memset(zPageUse, 0, sizeof(zPageUse[0])*(g.mxPage+1)); + g.zPageUse = sqlite3_malloc64( sizeof(g.zPageUse[0])*(g.mxPage+1) ); + if( g.zPageUse==0 ) out_of_memory(); + memset(g.zPageUse, 0, sizeof(g.zPageUse[0])*(g.mxPage+1)); /* Discover the usage of each page */ a = fileRead(0, 100); + if( g.bTmstmp && a[20]==16 ){ + g.aPageTag = sqlite3_malloc64( sizeof(struct TmstmpTag)*(g.mxPage+1) ); + if( g.aPageTag==0 ) out_of_memory(); + memset(g.aPageTag, 0, sizeof(struct TmstmpTag)*(g.mxPage+1) ); + }else{ + g.bTmstmp = 0; + g.aPageTag = 0; + } page_usage_freelist(decodeInt32(a+32)); page_usage_ptrmap(a); sqlite3_free(a); @@ -1042,15 +1126,68 @@ static void page_usage_report(const char *zPrg, const char *zDbName){ sqlite3_close(db); /* Print the report and free memory used */ + if( g.bCSV ){ + if( g.bTmstmp ){ + printf("pgno,tm,frame,flg,salt,parent,child,ovfl,txt\r\n"); + }else{ + printf("pgno,parent,child,ovfl,txt\r\n"); + } + } for(i=1; i<=g.mxPage; i++){ - if( zPageUse[i]==0 ) page_usage_btree(i, -1, 0, 0); - printf("%5u: %s\n", i, zPageUse[i] ? zPageUse[i] : "???"); + if( g.zPageUse[i]==0 ){ + g.zPageUse[i] = sqlite3_mprintf("???"); + if( g.zPageUse[i]==0 ) continue; + } + if( g.bCSV ){ + const char *z = g.zPageUse[i]; + const char *s; + printf("%u,", i); + if( g.bTmstmp ){ + const unsigned char *a = g.aPageTag[i].a; + sqlite3_uint64 tm = 0; + unsigned int x; + int k; + for(k=2; k<=7; k++) tm = (tm<<8)+a[k]; + printf("%llu.%03u,", tm/1000, (unsigned int)(tm%1000)); + for(x=0, k=8; k<=11; k++) x = (x<<8)+a[k]; + printf("%u,", x); + printf("%u,", a[12]); + for(x=0, k=13; k<=15; k++) x = (x<<8)+a[k]; + printf("%u,", x); + } + if( (s = strstr(z, " of page "))!=0 ){ + printf("%d,", atoi(s+9)); + }else if( (s = strstr(z, " of trunk page "))!=0 ){ + printf("%d,", atoi(s+15)); + }else{ + printf("0,"); + } + if( (s = strstr(z, "], child "))!=0 ){ + printf("%d,", atoi(s+9)); + }else if( (s = strstr(z, " from cell "))!=0 ){ + printf("%d,", atoi(s+12)); + }else{ + printf("-1,"); + } + if( strncmp(z,"overflow ", 9)==0 ){ + printf("%d,", atoi(z+9)); + }else{ + printf("-1,"); + } + printf("\"%s\"\r\n", z); + }else if( g.bTmstmp ){ + printf("%5u: %s %s\n", i, + decodeTimestamp(&g.aPageTag[i].a[2]), + g.zPageUse[i]); + }else{ + printf("%5u: %s\n", i, g.zPageUse[i]); + } } for(i=1; i<=g.mxPage; i++){ - sqlite3_free(zPageUse[i]); + sqlite3_free(g.zPageUse[i]); } - sqlite3_free(zPageUse); - zPageUse = 0; + sqlite3_free(g.zPageUse); + g.zPageUse = 0; } /* @@ -1083,7 +1220,7 @@ static void ptrmap_coverage_report(const char *zDbName){ for(pgno=2; pgno<=g.mxPage; pgno += perPage+1){ printf("%5llu: PTRMAP page covering %llu..%llu\n", pgno, pgno+1, pgno+perPage); - a = fileRead((pgno-1)*g.pagesize, usable); + a = fileRead((pgno-1)*g.pagesize, g.pagesize); for(i=0; i+5<=usable; i+=5){ const char *zType; u32 iFrom = decodeInt32(&a[i+1]); @@ -1115,7 +1252,7 @@ static void ptrmap_coverage_report(const char *zDbName){ ** Check the range validity for a page number. Print an error and ** exit if the page is out of range. */ -static void checkPageValidity(int iPage){ +static void checkPageValidity(unsigned int iPage){ if( iPage<1 || iPage>g.mxPage ){ fprintf(stderr, "Invalid page number %d: valid range is 1..%d\n", iPage, g.mxPage); @@ -1130,7 +1267,9 @@ static void usage(const char *argv0){ fprintf(stderr, "Usage %s ?--uri? FILENAME ?args...?\n\n", argv0); fprintf(stderr, "switches:\n" + " --csv CSV output for \"pgidx\"\n" " --raw Read db file directly, bypassing SQLite VFS\n" + " --tmstmp Interpret tmstmpvfs tags\n" "args:\n" " dbheader Show database header\n" " pgidx Index of how each page is used\n" @@ -1154,14 +1293,28 @@ int main(int argc, char **argv){ char **azArg = argv; int nArg = argc; - /* Check for the "--uri" or "-uri" switch. */ - if( nArg>1 ){ - if( sqlite3_stricmp("-raw", azArg[1])==0 - || sqlite3_stricmp("--raw", azArg[1])==0 - ){ + /* Check for the switches. */ + while( nArg>1 && azArg[1][0]=='-' ){ + const char *z = azArg[1]; + if( z[1]=='-' && z[2]!=0 ) z++; + if( sqlite3_stricmp("-raw", z)==0 ){ g.bRaw = 1; azArg++; nArg--; + }else + if( strcmp("-csv", z)==0 ){ + g.bCSV = 1; + azArg++; + nArg--; + }else + if( strcmp("-tmstmp", z)==0 ){ + g.bTmstmp = 1; + azArg++; + nArg--; + }else + { + usage(zPrg); + exit(1); } } @@ -1173,15 +1326,19 @@ int main(int argc, char **argv){ fileOpen(zPrg, azArg[1]); szFile = fileGetsize(); - zPgSz = fileRead(16, 2); - g.pagesize = zPgSz[0]*256 + zPgSz[1]*65536; - if( g.pagesize==0 ) g.pagesize = 1024; + zPgSz = fileRead(0, 24); + g.pagesize = zPgSz[16]*256 + zPgSz[17]*65536; + if( g.pagesize==0 ) g.pagesize = 4096; + g.nRes = zPgSz[20]; + g.usablesize = g.pagesize - g.nRes; sqlite3_free(zPgSz); - - printf("Pagesize: %d\n", (int)g.pagesize); g.mxPage = (u32)((szFile+g.pagesize-1)/g.pagesize); - printf("Available pages: 1..%u\n", g.mxPage); + if( !g.bCSV ){ + printf("Pagesize: %d\n", (int)g.pagesize); + if( g.nRes ) printf("Useable-size: %d\n", (int)g.usablesize); + printf("Available pages: 1..%u\n", g.mxPage); + } if( nArg==2 ){ u32 i; for(i=1; i<=g.mxPage; i++) print_page(i); diff --git a/tool/showtmlog.c b/tool/showtmlog.c new file mode 100644 index 000000000..4c35b777b --- /dev/null +++ b/tool/showtmlog.c @@ -0,0 +1,254 @@ +/* +** A utility program to decode tmstmpvfs log files. +*/ +#include <stdio.h> +#include <assert.h> +#include <string.h> +#include <stdint.h> +#include <stdlib.h> + +/* +** The six bytes at a[] are a big-endian unsigned integer which is the +** number of milliseconds since 1970. Decode that value into an ISO 8601 +** date/time string stored in static space and return a pointer to that +** string. +*/ +static const char *decodeTimestamp(const unsigned char *a){ + uint64_t ms; /* Milliseconds since 1970 */ + uint64_t days; /* Days since 1970-01-01 */ + uint64_t sod; /* Start of date specified by ms */ + uint64_t z; /* Days since 0000-03-01 */ + uint64_t era; /* 400-year era */ + int i; /* Loop counter */ + int h; /* hour */ + int m; /* minute */ + int s; /* second */ + int f; /* millisecond */ + int Y; /* year */ + int M; /* month */ + int D; /* day */ + int y; /* year assuming March is first month */ + unsigned int doe; /* day of 400-year era */ + unsigned int yoe; /* year of 400-year era */ + unsigned int doy; /* day of year */ + unsigned int mp; /* month with March==0 */ + static char zOut[50]; /* Return results here */ + + for(ms=0, i=0; i<=5; i++) ms = (ms<<8) + a[i]; + if( ms==0 ){ + return " "; + }else if( ms>4102444800000LL ){ /* 2100-01-01 */ + /* YYYY-MM-DD HH:MM:SS.SSS */ + return " (bad date) "; + } + days = ms/86400000; + sod = (ms%86400000)/1000; + f = (int)(ms%1000); + + h = sod/3600; + m = (sod%3600)/60; + s = sod%60; + z = days + 719468; + era = z/146097; + doe = (unsigned)(z - era*146097); + yoe = (doe - doe/1460 + doe/36524 - doe/146096)/365; + y = (int)yoe + era*400; + doy = doe - (365*yoe + yoe/4 - yoe/100); + mp = (5*doy + 2)/153; + D = doy - (153*mp + 2)/5 + 1; + M = mp + (mp<10 ? 3 : -9); + Y = y + (M <=2); + snprintf(zOut, sizeof(zOut), + "%04d-%02d-%02d %02d:%02d:%02d.%03d", + Y, M, D, h, m, s, f); + return zOut; +} + +/* +** Render a single 16-byte tmstmpvfs log record as a line to a CSV file. +** +** Columns: tmstmp,fileno,op,pid,pgno,frame,salt,txn +*/ +static void renderCSV(int iFile, unsigned char *a){ + unsigned int a2, a3; + int j; + uint64_t ms; + + for(ms=0, j=2; j<=7; j++) ms = (ms<<8) + a[j]; + printf("%u.%03u,%d,", (unsigned int)(ms/1000), (unsigned)(ms%1000), iFile); + for(a2=0, j=8; j<=11; j++) a2 = (a2<<8)+a[j]; + for(a3=0, j=12; j<=15; j++) a3 = (a3<<8)+a[j]; + switch( a[0] ){ + case 0x01: { + printf("\"open-db\",%u,,,,\r\n",a2); + break; + } + case 0x02: { + printf("\"open-wal\",%u,,,,\r\n", a2); + break; + } + case 0x03: { + printf("\"wal-page\",,%u,%u,,%d\r\n", a2, a3, a[1]); + break; + } + case 0x04: { + printf("\"db-page\",,%u,,,\r\n", a2); + break; + } + case 0x05: { + printf("\"ckpt-start\",,,,,\r\n"); + break; + } + case 0x06: { + printf("\"ckpt-page\",,%u,%u,,\r\n", a2, a3); + break; + } + case 0x07: { + printf("\"ckpt-end\",,,,,\r\n"); + break; + } + case 0x08: { + printf("\"wal-reset\",,,,%u,\r\n", a3); + break; + } + case 0x0e: { + printf("\"close-wal\",,,,,\r\n"); + break; + } + case 0x0f: { + printf("\"close-db\",,,,,\r\n"); + break; + } + default: { + printf("\"invalid-record\",,,,,\r\n"); + break; + } + } +} + +/* +** Render a single 16-byte tmstmpvfs log record as human-readable text +** on stdout. +*/ +static void renderText(unsigned char *a){ + unsigned int a2, a3; + int j; + + printf("%s ", decodeTimestamp(a+2)); + for(a2=0, j=8; j<=11; j++) a2 = (a2<<8)+a[j]; + for(a3=0, j=12; j<=15; j++) a3 = (a3<<8)+a[j]; + switch( a[0] ){ + case 0x01: { + printf("open-db pid %u\n", a2); + break; + } + case 0x02: { + printf("open-wal pid %u\n", a2); + break; + } + case 0x03: { + printf("wal-page pgno %-8u frame %-8u%s\n", a2, a3, + a[1]==1 ? " txn" : ""); + break; + } + case 0x04: { + printf("db-page pgno %-8u\n", a2); + break; + } + case 0x05: { + printf("ckpt-start\n"); + break; + } + case 0x06: { + printf("ckpt-page pgno %-8u frame %-8u\n", a2, a3); + break; + } + case 0x07: { + printf("ckpt-end\n"); + break; + } + case 0x08: { + printf("wal-reset salt1 0x%08x\n", a3); + break; + } + case 0x0e: { + printf("close-wal\n"); + break; + } + case 0x0f: { + printf("close-db\n"); + break; + } + default: { + printf("invalid-record\n"); + break; + } + } +} + +static void usage(const char *argv0){ + printf("Usage: %s [--csv] LOGFILE ...\n", argv0); + printf("Decode one or more tmstmpvfs log files and display the results\n" + "on stdout. Render as CSV if the --csv option is used.\n"); +} + +int main(int argc, char **argv){ + int i; + FILE *in; + unsigned char a[16]; + int bCSV = 0; + const char *z; + int nFile = 0; + int iFile; + for(i=1; i<argc; i++){ + z = argv[i]; + if( z[0]=='-' ){ + if( z[1]=='-' ) z++; + if( strcmp(z,"-csv")==0 ){ + bCSV = 1; + }else + if( strcmp(z,"-help")==0 || strcmp(z,"-?")==0 ){ + usage(argv[0]); + return 0; + }else + { + printf("unknown command-line option: \"%s\"\n", + argv[i]); + usage(argv[0]); + return 1; + } + }else{ + nFile++; + } + } + if( nFile==0 ){ + usage(argv[0]); + return 1; + } + iFile = 0; + if( bCSV ){ + printf("tmstmp,fileno,op,pid,pgno,frame,salt,txn\r\n"); + } + for(i=1; i<argc; i++){ + z = argv[i]; + if( z[0]=='-' ) continue; + in = fopen(z, "rb"); + if( in==0 ){ + printf("%s: can't open\n", z); + continue; + } + iFile++; + if( nFile>1 && !bCSV ){ + printf("*** %s ***\n", z); + } + while( 16==fread(a, 1, 16, in) ){ + if( bCSV ){ + renderCSV(iFile, a); + }else{ + renderText(a); + } + } + fclose(in); + } + return 0; +} diff --git a/tool/sqldiff.c b/tool/sqldiff.c index 44bf488f8..d27a62e14 100644 --- a/tool/sqldiff.c +++ b/tool/sqldiff.c @@ -28,6 +28,9 @@ #include "sqlite3.h" #include "sqlite3_stdio.h" +typedef sqlite3_int64 i64; +typedef sqlite3_uint64 u64; + /* ** All global variables are gathered into the "g" singleton. */ @@ -202,12 +205,12 @@ static char **columnNames( int *pbRowid /* OUT: True if PK is an implicit rowid */ ){ char **az = 0; /* List of column names to be returned */ - int naz = 0; /* Number of entries in az[] */ + i64 naz = 0; /* Number of entries in az[] */ sqlite3_stmt *pStmt; /* SQL statement being run */ char *zPkIdxName = 0; /* Name of the PRIMARY KEY index */ int truePk = 0; /* PRAGMA table_info identifies the PK to use */ - int nPK = 0; /* Number of PRIMARY KEY columns */ - int i, j; /* Loop counters */ + i64 nPK = 0; /* Number of PRIMARY KEY columns */ + i64 i, j; /* Loop counters */ if( g.bSchemaPK==0 ){ /* Normal case: Figure out what the true primary key is for the table. @@ -271,7 +274,7 @@ static char **columnNames( } *pnPKey = nPK; naz = nPK; - az = sqlite3_malloc( sizeof(char*)*(nPK+1) ); + az = sqlite3_malloc64( sizeof(char*)*(nPK+1) ); if( az==0 ) runtimeError("out of memory"); memset(az, 0, sizeof(char*)*(nPK+1)); if( g.bSchemaCompare ){ @@ -288,7 +291,7 @@ static char **columnNames( || !(strcmp(sid,"rootpage")==0 ||strcmp(sid,"name")==0 ||strcmp(sid,"type")==0)){ - az = sqlite3_realloc(az, sizeof(char*)*(naz+2) ); + az = sqlite3_realloc64(az, sizeof(char*)*(naz+2) ); if( az==0 ) runtimeError("out of memory"); az[naz++] = sid; } @@ -954,7 +957,7 @@ static int rbuDeltaCreate( unsigned int i, base; char *zOrigDelta = zDelta; hash h; - int nHash; /* Number of hash table entries */ + i64 nHash; /* Number of hash table entries */ int *landmark; /* Primary hash table */ int *collide; /* Collision chain */ int lastRead = -1; /* Last byte of zSrc read by a COPY command */ @@ -982,7 +985,7 @@ static int rbuDeltaCreate( ** source file. */ nHash = lenSrc/NHASH; - collide = sqlite3_malloc( nHash*2*sizeof(int) ); + collide = sqlite3_malloc64( nHash*2*sizeof(int) ); landmark = &collide[nHash]; memset(landmark, -1, nHash*sizeof(int)); memset(collide, -1, nHash*sizeof(int)); @@ -1286,9 +1289,9 @@ static void rbudiff_one_table(const char *zTab, FILE *out){ } }else{ char *zOtaControl; - int nOtaControl = sqlite3_column_bytes(pStmt, nCol); + i64 nOtaControl = sqlite3_column_bytes(pStmt, nCol); - zOtaControl = (char*)sqlite3_malloc(nOtaControl+1); + zOtaControl = (char*)sqlite3_malloc64(nOtaControl+1); memcpy(zOtaControl, sqlite3_column_text(pStmt, nCol), nOtaControl+1); for(i=0; i<nCol; i++){ @@ -1300,11 +1303,11 @@ static void rbudiff_one_table(const char *zTab, FILE *out){ const char *aSrc = sqlite3_column_blob(pStmt, nCol+1+i); int nSrc = sqlite3_column_bytes(pStmt, nCol+1+i); const char *aFinal = sqlite3_column_blob(pStmt, i); - int nFinal = sqlite3_column_bytes(pStmt, i); + i64 nFinal = sqlite3_column_bytes(pStmt, i); char *aDelta; int nDelta; - aDelta = sqlite3_malloc(nFinal + 60); + aDelta = sqlite3_malloc64(nFinal + 60); nDelta = rbuDeltaCreate(aSrc, nSrc, aFinal, nFinal, aDelta); if( nDelta<nFinal ){ int j; @@ -1549,10 +1552,10 @@ static void changeset_one_table(const char *zTab, FILE *out){ sqlite3_stmt *pStmt; /* SQL statment */ char *zId = safeId(zTab); /* Escaped name of the table */ char **azCol = 0; /* List of escaped column names */ - int nCol = 0; /* Number of columns */ + i64 nCol = 0; /* Number of columns */ int *aiFlg = 0; /* 0 if column is not part of PK */ int *aiPk = 0; /* Column numbers for each PK column */ - int nPk = 0; /* Number of PRIMARY KEY columns */ + i64 nPk = 0; /* Number of PRIMARY KEY columns */ sqlite3_str *pSql; /* SQL for the diff query */ int i, k; /* Loop counters */ const char *zSep; /* List separator */ @@ -1564,16 +1567,16 @@ static void changeset_one_table(const char *zTab, FILE *out){ pStmt = db_prepare("PRAGMA main.table_info=%Q", zTab); while( SQLITE_ROW==sqlite3_step(pStmt) ){ nCol++; - azCol = sqlite3_realloc(azCol, sizeof(char*)*nCol); + azCol = sqlite3_realloc64(azCol, sizeof(char*)*nCol); if( azCol==0 ) runtimeError("out of memory"); - aiFlg = sqlite3_realloc(aiFlg, sizeof(int)*nCol); + aiFlg = sqlite3_realloc64(aiFlg, sizeof(int)*nCol); if( aiFlg==0 ) runtimeError("out of memory"); azCol[nCol-1] = safeId((const char*)sqlite3_column_text(pStmt,1)); aiFlg[nCol-1] = i = sqlite3_column_int(pStmt,5); if( i>0 ){ if( i>nPk ){ nPk = i; - aiPk = sqlite3_realloc(aiPk, sizeof(int)*nPk); + aiPk = sqlite3_realloc64(aiPk, sizeof(int)*nPk); if( aiPk==0 ) runtimeError("out of memory"); } aiPk[i-1] = nCol-1; @@ -1896,6 +1899,11 @@ static void showHelp(void){ ); } +/* work-around the Microsoft "WorstFit" bug */ +#ifdef _WIN32 +#define main utf8_main +#endif + int main(int argc, char **argv){ const char *zDb1 = 0; const char *zDb2 = 0; @@ -1908,7 +1916,7 @@ int main(int argc, char **argv){ FILE *out = stdout; void (*xDiff)(const char*,FILE*) = diff_one_table; #ifndef SQLITE_OMIT_LOAD_EXTENSION - int nExt = 0; + i64 nExt = 0; char **azExt = 0; #endif int useTransaction = 0; diff --git a/tool/sqlite3_rsync.c b/tool/sqlite3_rsync.c index ad9f132bb..b10224b2f 100644 --- a/tool/sqlite3_rsync.c +++ b/tool/sqlite3_rsync.c @@ -32,6 +32,7 @@ static const char zUsage[] = "\n" " --exe PATH Name of the sqlite3_rsync program on the remote side\n" " --help Show this help screen\n" + " -p|--port PORT Run SSH over TCP port PORT instead of the default 22\n" " --protocol N Use sync protocol version N.\n" " --ssh PATH Name of the SSH program used to reach the remote side\n" " -v Verbose. Multiple v's for increasing output\n" @@ -324,15 +325,29 @@ static int popen2( ** Close the connection to a child process previously created using ** popen2(). */ -static void pclose2(FILE *pIn, FILE *pOut, int childPid){ +static int pclose2(FILE *pIn, FILE *pOut, int childPid){ #ifdef _WIN32 /* Not implemented, yet */ fclose(pIn); fclose(pOut); + return 0; #else + int wp, rc = 0; fclose(pIn); fclose(pOut); - while( waitpid(0, 0, WNOHANG)>0 ) {} + do{ + wp = waitpid(0, &rc, WNOHANG); + if( wp>0 ){ + if( WIFEXITED(rc) ){ + rc = WEXITSTATUS(rc); + }else if( WIFSIGNALED(rc) ){ + rc = WTERMSIG(rc); + }else{ + rc = 0/*???*/; + } + } + } while( wp>0 ); + return rc; #endif } /***************************************************************************** @@ -2054,6 +2069,7 @@ int main(int argc, char const * const *argv){ FILE *pOut = 0; int childPid = 0; const char *zSsh = "ssh"; + int iPort = 0; const char *zExe = "sqlite3_rsync"; char *zCmd = 0; sqlite3_int64 tmStart; @@ -2094,6 +2110,15 @@ int main(int argc, char const * const *argv){ zSsh = cli_opt_val; continue; } + if( strcmp(z, "-port")==0 || strcmp(z, "-p")==0 ){ + const char *zPort = cli_opt_val; + iPort = atoi(zPort); + if( iPort<1 || iPort>65535 ){ + fprintf(stderr, "invalid TCP port number: \"%s\"\n", zPort); + return 1; + } + continue; + } if( strcmp(z, "-exe")==0 ){ zExe = cli_opt_val; continue; @@ -2235,6 +2260,7 @@ int main(int argc, char const * const *argv){ for(iRetry=0; 1 /*exit-via-break*/; iRetry++){ sqlite3_str *pStr = sqlite3_str_new(0); append_escaped_arg(pStr, zSsh, 1); + if( iPort>0 ) sqlite3_str_appendf(pStr, " -p %d", iPort); sqlite3_str_appendf(pStr, " -e none"); append_escaped_arg(pStr, ctx.zOrigin, 0); if( iRetry ) add_path_argument(pStr); @@ -2283,6 +2309,7 @@ int main(int argc, char const * const *argv){ for(iRetry=0; 1 /*exit-by-break*/; iRetry++){ sqlite3_str *pStr = sqlite3_str_new(0); append_escaped_arg(pStr, zSsh, 1); + if( iPort>0 ) sqlite3_str_appendf(pStr, " -p %d", iPort); sqlite3_str_appendf(pStr, " -e none"); append_escaped_arg(pStr, ctx.zReplica, 0); if( iRetry==1 ) add_path_argument(pStr); @@ -2349,7 +2376,7 @@ int main(int argc, char const * const *argv){ } originSide(&ctx); } - pclose2(ctx.pIn, ctx.pOut, childPid); + ctx.nErr += !!pclose2(ctx.pIn, ctx.pOut, childPid); if( ctx.pLog ) fclose(ctx.pLog); tmEnd = currentTime(); tmElapse = tmEnd - tmStart; /* Elapse time in milliseconds */ diff --git a/tool/sqltclsh.c.in b/tool/sqltclsh.c.in index da354ee93..3e4d38b2d 100644 --- a/tool/sqltclsh.c.in +++ b/tool/sqltclsh.c.in @@ -33,7 +33,7 @@ INCLUDE $ROOT/ext/misc/appendvfs.c INCLUDE $ROOT/ext/misc/zipfile.c INCLUDE $ROOT/ext/misc/sqlar.c #endif -INCLUDE $ROOT/src/tclsqlite.c +INCLUDE tclsqlite-ex.c const char *sqlite3_tclapp_init_proc(Tcl_Interp *interp){ (void)interp; diff --git a/tool/warnings.sh b/tool/warnings.sh index 3619ce70a..4bf0ac9a7 100644 --- a/tool/warnings.sh +++ b/tool/warnings.sh @@ -68,6 +68,7 @@ gcc -c $WARNING_OPTS -std=c99 \ echo '**** Optimized -O3. Includes FTS4/5, GEOPOLY, JSON1 ******' echo '****' $WARNING_OPTS gcc -O3 -c $WARNING_OPTS -std=c99 \ + -Wdeclaration-after-statement \ -DHAVE_STDINT_H \ -DSQLITE_ENABLE_FTS4 \ -DSQLITE_ENABLE_FTS5 \ diff --git a/tool/winmain.c b/tool/winmain.c new file mode 100644 index 000000000..72ae00d5c --- /dev/null +++ b/tool/winmain.c @@ -0,0 +1,79 @@ +/* +** 2025-12-26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements a substitute process entry point for windows +** that correctly interprets command-line arguments as UTF-8. This is +** a work-around for the (unfixed) Windows bug known as "WorstFit" and +** described at: +** +** * https://news.ycombinator.com/item?id=42647101 +** +** USAGE: +** +** If you have a command-line program coded to C-language standard in which +** the entry point is: +** +** int main(int argc, char **argv){...} +** +** That does not work on Windows if there are non-ASCII characters on the +** command line. Programs are expected to use an alternative entry point: +** +** int wmain(int wargc, wchar_t **wargv){...} +** +** This file implements _wmain() but then converts all of the wargv elements +** to UTF-8 and passes them off to utf8_main(). +** +** So, a command-line tool that implements the standard entry point can be +** modified by adding the following lines just prior to main(): +** +** #ifdef _WIN32 +** #define main utf8_main +** #endif +** +** Then, include this file in the list of files that implement the program, +** and the program will work correctly, even on Windows. +*/ +#include <windows.h> +#include <stdio.h> + +extern int utf8_main(int,char**); +int wmain(int argc, wchar_t **wargv){ + int rc, i; + char **argv = malloc( sizeof(char*) * (argc+1) ); + char **orig = argv; + if( argv==0 ){ + fprintf(stderr, "malloc failed\n"); + exit(1); + } + for(i=0; i<argc; i++){ + int nByte = WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, 0, 0, 0, 0); + if( nByte==0 ){ + argv[i] = 0; + }else{ + argv[i] = malloc( nByte ); + if( argv[i]==0 ){ + fprintf(stderr, "malloc failed\n"); + exit(1); + } + nByte = WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, argv[i],nByte,0,0); + if( nByte==0 ){ + free(argv[i]); + argv[i] = 0; + } + } + } + argv[argc] = 0; + rc = utf8_main(argc, argv); + for(i=0; i<argc; i++) free(orig[i]); + free(argv); + return rc; +} From 30842cda35c88cdfc7a8adba1c9b20821d2c08d1 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo <sjlombardo@zetetic.net> Date: Thu, 23 Apr 2026 17:02:49 -0400 Subject: [PATCH 73/75] Sanitize source database name passed to sqlcipher_export --- src/sqlcipher.c | 11 +++++++++-- test/sqlcipher-compatibility.test | 24 ++++++++++++++++++++---- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/sqlcipher.c b/src/sqlcipher.c index 56c5223d3..0a2eceb95 100644 --- a/src/sqlcipher.c +++ b/src/sqlcipher.c @@ -3756,7 +3756,7 @@ static int sqlcipher_execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql) static void sqlcipher_exportFunc(sqlite3_context *context, int argc, sqlite3_value **argv) { sqlite3 *db = sqlite3_context_db_handle(context); const char* targetDb, *sourceDb; - int targetDb_idx = 0; + int targetDb_idx = 0, sourceDb_idx = 0; u64 saved_flags = db->flags; /* Saved value of the db->flags */ u32 saved_mDbFlags = db->mDbFlags; /* Saved value of the db->mDbFlags */ int saved_nChange = db->nChange; /* Saved value of db->nChange */ @@ -3790,12 +3790,19 @@ static void sqlcipher_exportFunc(sqlite3_context *context, int argc, sqlite3_val sourceDb = (char *) sqlite3_value_text(argv[1]); } + /* if the source database is not valid, do not proceed. */ + sourceDb_idx = sqlcipher_find_db_index(db, sourceDb); + if(sourceDb_idx < 0) { + rc = SQLITE_ERROR; + pzErrMsg = sqlite3_mprintf("invalid source database %s", sourceDb); + goto end_of_export; + } /* if the target database is not valid, do not proceed. */ targetDb_idx = sqlcipher_find_db_index(db, targetDb); if(targetDb_idx < 0) { rc = SQLITE_ERROR; - pzErrMsg = sqlite3_mprintf("unknown database %s", targetDb); + pzErrMsg = sqlite3_mprintf("invalid target database %s", targetDb); goto end_of_export; } db->init.iDb = targetDb_idx; diff --git a/test/sqlcipher-compatibility.test b/test/sqlcipher-compatibility.test index 4053d78d9..fc97314ec 100644 --- a/test/sqlcipher-compatibility.test +++ b/test/sqlcipher-compatibility.test @@ -433,9 +433,9 @@ file delete -force test.db file delete -force test2.db # use the sqlcipher_export function -# on a non-existent database. Verify -# the error gets through. -do_test export-error { +# on a non-existent target database +# and verify an error is reported +do_test export-target-error { sqlite_orig db test.db catchsql { @@ -443,7 +443,23 @@ do_test export-error { CREATE TABLE t1(a,b); SELECT sqlcipher_export('nodb'); } -} {1 {unknown database nodb}} +} {1 {invalid target database nodb}} +db close +file delete -force test.db + +# use the sqlcipher_export function +# on a non-existent source database +# and verify an error is reported +do_test export-source-error { + sqlite_orig db test.db + + catchsql { + PRAGMA key = 'testkey'; + CREATE TABLE t1(a,b); + SELECT sqlcipher_export('main', 'nodb'); + } + +} {1 {invalid source database nodb}} db close file delete -force test.db From cd3eef0f9abc1c11434aed6292274ca0b98a8af5 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo <sjlombardo@zetetic.net> Date: Fri, 24 Apr 2026 11:43:51 -0400 Subject: [PATCH 74/75] Update CHANGELOG.md with relevant changes for 4.15.0 --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f4bab97c..3a9a7781f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,12 @@ # SQLCipher Change Log Notable changes to this project are documented in this file. -## [4.15.0] - (? 2026 - [4.15.0 changes]) +## [4.15.0] - (April 2026 - [4.15.0 changes]) +- Update baseline to SQLite 3.53.0 +- Sanitize source database name passed to `sqlcipher_export` +- Improve error handling in `sqlcipher_extra_init` +- Remove const from pzErrMesg in `sqlcipher_export_init` (issue #590) +- Minor code cleanups ## [4.14.0] - (March 2026 - [4.14.0 changes]) - Updates baseline to SQLite 3.51.3 From 0b4d16090a71579c4c88220b7fb67bb7396f1434 Mon Sep 17 00:00:00 2001 From: Stephen Lombardo <sjlombardo@zetetic.net> Date: Mon, 27 Apr 2026 10:15:53 -0400 Subject: [PATCH 75/75] Update CHANGELOG.md with reporter information for 4.15.0 release --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a9a7781f..d4f001b61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,8 @@ Notable changes to this project are documented in this file. ## [4.15.0] - (April 2026 - [4.15.0 changes]) - Update baseline to SQLite 3.53.0 -- Sanitize source database name passed to `sqlcipher_export` +- Sanitize source database name passed to `sqlcipher_export` (reported by + Dima Petschke from Deutsche Telekom Security GmbH) - Improve error handling in `sqlcipher_extra_init` - Remove const from pzErrMesg in `sqlcipher_export_init` (issue #590) - Minor code cleanups